diff --git a/deliverables/geluke/pom.xml b/deliverables/geluke/pom.xml index 8d4554eac..6e3fc3c85 100644 --- a/deliverables/geluke/pom.xml +++ b/deliverables/geluke/pom.xml @@ -19,6 +19,7 @@ ../../projects/fangjia-core ../../projects/fangjia-dal + ../../projects/chafangtong-dal ../../projects/fangjia-service ../../projects/geluke-dal ../../projects/geluke-service diff --git a/deliverables/k8s/geluke.yaml b/deliverables/k8s/geluke.yaml new file mode 100644 index 000000000..93c1e1f10 --- /dev/null +++ b/deliverables/k8s/geluke.yaml @@ -0,0 +1,95 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: geluke + namespace: fangjia +spec: + replicas: 1 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + revisionHistoryLimit: 5 + selector: + matchLabels: + project: geluke + app: fangjia + template: + metadata: + labels: + project: geluke + app: fangjia + spec: + imagePullSecrets: + - name: nexus-registry-secret + nodeSelector: + app: fangjia + containers: + - name: geluke + image: 192.168.0.19:9012/geluke:latest + imagePullPolicy: Always + resources: + limits: + memory: "3Gi" + requests: + memory: "2Gi" + env: + - name: JAVA_OPTS + value: "-Xmx3072m -XX:MaxPermSize=256m + -Dlog4j.configuration=file:/home/goojia/webapps/ROOT/WEB-INF/classes/log4j_linux.xml + -Dfangjia.overridePropertiesLocation=/home/goojia/goojiaconfig/geluke.properties" + readinessProbe: + tcpSocket: + port: 8080 + initialDelaySeconds: 10 + periodSeconds: 10 + livenessProbe: + tcpSocket: + port: 8080 + initialDelaySeconds: 10 + periodSeconds: 5 + ports: + - containerPort: 8080 + name: web + protocol: TCP + volumeMounts: + - name: host-time + mountPath: /etc/localtime + - name: goojiaconfig + mountPath: /home/goojia/goojiaconfig + + volumes: + - name: host-time + hostPath: + path: /etc/localtime + - name: goojiaconfig + cephfs: + monitors: + - 192.168.0.208:6789 + - 192.168.0.209:6789 + - 192.168.0.210:6789 + user: admin + path: /k8s/goojiaconfig + secretRef: + name: ceph-secret + readOnly: false + +--- +apiVersion: v1 +kind: Service +metadata: + name: geluke + namespace: fangjia + labels: + project: geluke + app: fangjia +spec: + type: NodePort + selector: + project: geluke + app: fangjia + ports: + - port: 8080 + targetPort: 8080 + nodePort: 30010 \ No newline at end of file diff --git a/deliverables/script/test/src/Test.java b/deliverables/script/test/src/Test.java deleted file mode 100644 index 9d88b45cf..000000000 --- a/deliverables/script/test/src/Test.java +++ /dev/null @@ -1,63 +0,0 @@ -/** - * Created by Administrator on 2017/5/9. - */ -import com.sun.org.apache.xerces.internal.impl.io.UTF8Reader; -import com.sun.xml.internal.txw2.Document; -import sun.nio.cs.ext.GB18030; - -import java.io.BufferedReader; -import java.io.InputStreamReader; -import java.net.URL; -import java.net.URLConnection; -public class Test { - String fixture_Date = null; - String area = null; - String unit_Price = null; - String transaction_Price = null; - static String sendGet(String url) { - String result = ""; - // 定义一个缓冲字符输入流 - BufferedReader in = null; - try { - // 将string转成url对象 - URL realUrl = new URL(url); - // 初始化一个链接到那个url的连接 - URLConnection connection = realUrl.openConnection(); - // 开始实际的连接 - connection.connect(); - // 初始化 BufferedReader输入流来读取URL的响应 - in = new BufferedReader(new InputStreamReader(connection.getInputStream(),"gbk")); - // 用来临时存储抓取到的每一行的数据 - String line; - while ((line = in.readLine()) != null) - { // 遍历抓取到的每一行并将其存储到result里面 - result += line; - } - Document document = Jsoup.parse(result, url); - - parseListPage(document); - - - } catch (Exception e) { - System.out.println("发送GET请求出现异常!" + e); - e.printStackTrace(); } - // 使用finally来关闭输入流 - finally { - try { - if (in != null) { - in.close(); } - } catch (Exception e2) { - e2.printStackTrace(); - } - } - return result; - } - public static void main(String[] args) { - // 定义即将访问的链接 - String url = "http://dl.goufang.com/xq/1062/6"; - // 访问链接并获取页面内容 - String result = sendGet(url); - System.out.println(result); - } -} - diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/seaweed/model/District.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/seaweed/model/District.java index 401132cb0..afba53e15 100644 --- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/seaweed/model/District.java +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/seaweed/model/District.java @@ -122,10 +122,8 @@ public class District implements SeaweedEntity { private String houseStructure; //房屋结构 private String source; private String propertyLife; -// -// //乡镇街道 -// private String township; -// + //乡镇街道 + private String township; private String pinyin; @@ -1873,14 +1871,14 @@ public class District implements SeaweedEntity { this.mappingAlias = mappingAlias; } -// @Column(name = "township") -// public String getTownship() { -// return township; -// } -// -// public void setTownship(String township) { -// this.township = township; -// } + @Column(name = "township") + public String getTownship() { + return township; + } + + public void setTownship(String township) { + this.township = township; + } diff --git a/projects/fangjia-open/pom.xml b/projects/fangjia-open/pom.xml index 3d34afcdd..ee70766d4 100644 --- a/projects/fangjia-open/pom.xml +++ b/projects/fangjia-open/pom.xml @@ -159,6 +159,11 @@ jfreechart 1.5.3 + + org.apache.xmlgraphics + batik-codec + 1.11 + org.sothis sothis-core diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java index d9287fdaf..96bbe097f 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java @@ -1705,6 +1705,25 @@ public class AddressController { return mav; } + public Object similarityAction(AddesssMatchModel model) { + RestfulModelAndView mav = new RestfulModelAndView(); + if (StringUtils.isBlank(model.getAddress_1()) || StringUtils.isBlank(model.getAddress_2())) { + mav.setModel(BasicModel.buildBadRequestModel("两个地址均不能为空")); + return mav; + } + model.setAddress(model.getAddress_1()); + String result1 = addressMatchProxyService.standard(convertModelToMap(model)); + model.setAddress(model.getAddress_2()); + String result2 = addressMatchProxyService.standard(convertModelToMap(model)); + if (result1 != null && result2 != null) { + Map resultMap = addressService.addressSimilarity(result1, result2); + mav.setModel(BasicModel.buildOKModel().setResult(resultMap)); + } else { + mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中...")); + } + return mav; + } + public Object parseCityAction(AddressModel model) { RestfulModelAndView mav = new RestfulModelAndView(); if (StringUtils.isBlank(model.getProvince())) { @@ -1750,6 +1769,7 @@ public class AddressController { // matchParams.setUseMap(true); matchParams.setUseAliyunApi(true); +// matchParams.setUseCrf(true); matchParams.setContainsLocation(model.isContainsLocation()); Building building = addressParserService.standard(matchParams); Address address = building.getSearchAddress(); @@ -1790,6 +1810,7 @@ public class AddressController { } } dto.append("buildingNumber", address.getBuildingNumber()); + dto.append("floor", address.getFloor()); dto.append("unitNumber", address.getUnitNumber()); dto.append("roomNumber", address.getRoomNumber()); dto.append("formattedAddress", address.getFormattedAddress()); diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java index 5f3343578..93172390a 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java @@ -131,7 +131,7 @@ import com.google.common.collect.Sets; import com.mongodb.BasicDBObject; import com.mongodb.DBObject; -//@Authority(role = Role.USER) +@Authority(role = Role.USER) public class PropertyController { private static final Logger LOGGER = LoggerFactory.getLogger(PropertyController.class); @@ -4966,6 +4966,7 @@ public class PropertyController { * @throws NumberFormatException * @throws IllegalAccessException */ + @Authority(role = Role.NONE) public Object rentReportAction(PropertyEvaluationModel model, HttpServletRequest request)throws NumberFormatException, IllegalAccessException { RestfulModelAndView mav = new RestfulModelAndView(); mav.setViewType("rest"); @@ -5164,7 +5165,6 @@ public class PropertyController { } - public Object evaluateRentAction(PropertyEvaluationModel model, HttpServletRequest request)throws NumberFormatException, IllegalAccessException { RestfulModelAndView mav = new RestfulModelAndView(); mav.setViewType("rest"); @@ -5255,13 +5255,6 @@ public class PropertyController { } - - - - - - - public void setHouseLoudongService(HouseLoudongService houseLoudongService) { this.houseLoudongService = houseLoudongService; } diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/CcbRentController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/CcbRentController.java index 9ae5bbeb4..3a17ffc8b 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/CcbRentController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/CcbRentController.java @@ -9,6 +9,7 @@ import com.fangjia.dal.mongo.model.Evaluation; import com.fangjia.dal.mongo.model.RentDealPrice; import com.fangjia.dal.seawater.model.Seawater; import com.fangjia.dal.seaweed.model.District; +import com.fangjia.open.APIConstant; import com.fangjia.open.compatible.model.JsonModel; import com.fangjia.open.dto.DistrictHouseDtao; import com.fangjia.open.interceptors.AuthenticationBean; @@ -26,11 +27,17 @@ import com.fangjia.service.mongo.fangjia.FileService; import com.fangjia.service.mongo.fangjia.SeawaterService; import com.fangjia.service.util.DistrictScoreUtil; import com.lowagie.text.pdf.BaseFont; +import org.apache.batik.transcoder.Transcoder; +import org.apache.batik.transcoder.TranscoderException; +import org.apache.batik.transcoder.TranscoderInput; +import org.apache.batik.transcoder.TranscoderOutput; +import org.apache.batik.transcoder.image.PNGTranscoder; import org.apache.commons.lang.StringUtils; import org.apache.lucene.index.Term; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.TermQuery; +import org.bson.types.ObjectId; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.sothis.core.util.CollectionUtils; @@ -41,9 +48,8 @@ import org.xhtmlrenderer.pdf.ITextRenderer; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; -import java.io.File; -import java.io.IOException; -import java.io.OutputStream; +import java.io.*; +import java.lang.reflect.InvocationTargetException; import java.text.ParseException; import java.util.*; @@ -81,10 +87,29 @@ public class CcbRentController { return result; } - public Object downloadPageAction(PropertyEvaluationModel result, HttpServletRequest request, @Param(name = "city") String city, @Param(name = "district") String districtStr, + public Object evaluateRentAction(com.fangjia.open.model.PropertyEvaluationModel model, HttpServletRequest request)throws NumberFormatException, IllegalAccessException { + String result = null; + HttpRequest httpRequest = new HttpRequest(); + try { + Map headers = new HashMap(); + // &__refresh_cache=1 刷新缓存 + String url = "http://open.fangjia.com/property/evaluateRent?token=" + APIConstant.SUPER_USER_TOKEN; + Map params = org.apache.commons.beanutils.BeanUtils.describe(model); + result = httpRequest.post(url, params, headers,"UTF-8"); + } catch (NoSuchMethodException e) { + e.printStackTrace(); + } catch (HttpRequestException e) { + e.printStackTrace(); + } catch (InvocationTargetException e) { + e.printStackTrace(); + } + return result; + } + + public Object downloadPageAction(PropertyEvaluationModel result, HttpServletRequest request, @Param(name = "city") String city, @Param(name = "district") String districtStr, @Param(name = "name") String name, @Param(name = "address") String address, @Param(name = "size") Double size, @Param(name = "floor") Integer floor, @Param(name = "height") Integer height, @Param(name = "room") Integer room, - @Param(name = "hall") Integer hall, @Param(name = "toilet") Integer toilet, @Param(name = "direction") String direction, @Param(name = "fitment") String fitment) { + @Param(name = "hall") Integer hall, @Param(name = "toilet") Integer toilet, @Param(name = "direction") String direction, @Param(name = "fitment") String fitment, @Param(name = "chartImgUrl") String chartImgUrl) { com.fangjia.open.model.PropertyEvaluationModel model = new com.fangjia.open.model.PropertyEvaluationModel(); model.setCity(city); model.setDistrict(districtStr); @@ -261,7 +286,8 @@ public class CcbRentController { districtPriceMap.put("trends", trendsMap); // 生成走势图图片,一遍转pdf - String tradesImgUrl = buildTrendImage(trendsMap, room); + //String tradesImgUrl = buildTrendImage(trendsMap, room); + String tradesImgUrl = chartImgUrl; //成交的租房房源 List rentTrades = rentDealPriceService.getRentDealPrices(district.getCity(), district.getRegion(), district.getName(), 5); @@ -426,23 +452,54 @@ public class CcbRentController { return evaluation; } + public Object saveHighChartAsImageAction(HttpServletRequest request, HttpServletResponse response) { + String svg = request.getParameter("svg"); + ByteArrayOutputStream out = null; + String url = null; + try { + svg = svg.replaceAll(":rect", "rect"); + Transcoder t = new PNGTranscoder(); + out = new ByteArrayOutputStream(); + TranscoderInput input = new TranscoderInput(new StringReader(svg)); + TranscoderOutput output = new TranscoderOutput(out); + t.transcode(input, output); + byte[] data = out.toByteArray(); + String id = new ObjectId().toString(); + fileService.save(id,"o", data, "image/png", "o"); + url = "http://img0.fangjia.com/stc/fs/" + DataUtil.encodePicUrl(id, "o", "house"); + } catch (TranscoderException e) { + e.printStackTrace(); + } finally { + try { + out.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + JsonModel model = new JsonModel(); + AutoCompleteBo bo = new AutoCompleteBo(); + bo.setAddress(url); + model.setJsonModel(bo); + return model; + } + public void downloadAction(HttpServletResponse response, HttpServletRequest request, @Param(name = "city") String city, @Param(name = "district") String district, @Param(name = "name") String name, @Param(name = "address") String address, @Param(name = "size") Double size, @Param(name = "floor") Integer floor, @Param(name = "height") Integer height, @Param(name = "room") Integer room, - @Param(name = "hall") Integer hall, @Param(name = "toilet") Integer toilet, @Param(name = "direction") String direction, @Param(name = "fitment") String fitment) throws IOException { + @Param(name = "hall") Integer hall, @Param(name = "toilet") Integer toilet, @Param(name = "direction") String direction, @Param(name = "fitment") String fitment, @Param(name = "chartImgUrl") String chartImgUrl) throws IOException { response.setHeader("Content-disposition", "attachment; filename=report.pdf"); response.setContentType("application/pdf;charset=UTF-8"); response.setCharacterEncoding("UTF-8"); OutputStream os = null; try { os = response.getOutputStream(); - ITextRenderer renderer = new ITextRenderer(35, 20); + ITextRenderer renderer = new ITextRenderer(35, 24); ITextFontResolver fontResolver = renderer.getFontResolver(); String fontUrl = ActionContext.getContext().getServletContext().getRealPath("/") + "fonts" + File.separator + "simsun.ttc"; fontResolver.addFont(fontUrl, BaseFont.IDENTITY_H, BaseFont.NOT_EMBEDDED); final String path = "http://" + request.getServerName() + ":" + request.getServerPort() + request.getContextPath(); String url = path + "/ccbRent/downloadPage?city=" + DataUtil.urlEncode(city) + "&district="+DataUtil.urlEncode(district)+"&name="+DataUtil.urlEncode(name); - url += "&address=" + DataUtil.urlEncode(address) +"&size="+size+"&floor="+floor+"&height="+height+"&room="+room+"&hall="+hall+"&toilet="+toilet+"&direction="+direction+"&fitment="+fitment; + url += "&address=" + DataUtil.urlEncode(address) +"&size="+size+"&floor="+floor+"&height="+height+"&room="+room+"&hall="+hall+"&toilet="+toilet+"&direction="+direction+"&fitment="+fitment+"&chartImgUrl="+chartImgUrl; renderer.setDocument(url); renderer.layout(); renderer.createPDF(os, true); diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/model/AddesssMatchModel.java b/projects/fangjia-open/src/main/java/com/fangjia/open/model/AddesssMatchModel.java index 3d362bbb9..e1048387a 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/model/AddesssMatchModel.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/model/AddesssMatchModel.java @@ -19,7 +19,24 @@ public class AddesssMatchModel extends FangjiaApiModel { private String province; private String debugName; private boolean useMap; + private String address_1; + private String address_2; + public String getAddress_1() { + return address_1; + } + + public void setAddress_1(String address_1) { + this.address_1 = address_1; + } + + public String getAddress_2() { + return address_2; + } + + public void setAddress_2(String address_2) { + this.address_2 = address_2; + } public boolean isContainsLocation() { return containsLocation; } diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/util/LineChartsUtils.java b/projects/fangjia-open/src/main/java/com/fangjia/open/util/LineChartsUtils.java index 1ce0f9126..605c6e137 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/util/LineChartsUtils.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/util/LineChartsUtils.java @@ -2,23 +2,24 @@ package com.fangjia.open.util; import com.fangjia.core.util.DataUtil; import com.fangjia.service.mongo.fangjia.FileService; +import org.apache.batik.transcoder.Transcoder; +import org.apache.batik.transcoder.TranscoderException; +import org.apache.batik.transcoder.TranscoderInput; +import org.apache.batik.transcoder.TranscoderOutput; +import org.apache.batik.transcoder.image.JPEGTranscoder; +import org.apache.batik.transcoder.image.PNGTranscoder; import org.bson.types.ObjectId; import org.jfree.chart.ChartFactory; -import org.jfree.chart.ChartPanel; import org.jfree.chart.ChartUtilities; import org.jfree.chart.JFreeChart; import org.jfree.chart.axis.NumberAxis; import org.jfree.chart.plot.CategoryPlot; import org.jfree.chart.plot.PlotOrientation; import org.jfree.chart.renderer.category.LineAndShapeRenderer; -import org.jfree.chart.ui.ApplicationFrame; import org.jfree.data.category.DefaultCategoryDataset; -import org.jfree.ui.RefineryUtilities; import java.awt.*; -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.IOException; +import java.io.*; import java.util.ArrayList; import java.util.List; @@ -99,6 +100,42 @@ public class LineChartsUtils { return null; } + public static void svg2ImgTest() { + String svg = ""; + String type="image/png"; + String filename="svg"; + if (null != type && null != svg) { + svg = svg.replaceAll(":rect", "rect"); + String ext = ""; + Transcoder t = null; + if (type.equals("image/png")) { + ext = "png"; + t = new PNGTranscoder(); + } + else if (type.equals("image/jpeg")) { + ext = "jpg"; + t = new JPEGTranscoder(); + } + if (null != t) { + OutputStream out = null; + try { + out = new FileOutputStream("D:\\"+filename+"."+ext); + } catch (FileNotFoundException e) { + e.printStackTrace(); + } + TranscoderInput input = new TranscoderInput(new StringReader(svg)); + TranscoderOutput output = new TranscoderOutput(out); + try{ + t.transcode(input, output); + System.out.println("out"); + } + catch (TranscoderException e) { + e.printStackTrace(); + } + } + } + } + /** * 定义图表对象 * @param rowKey 曲线名,如一居室 diff --git a/projects/fangjia-open/src/main/webapp/ccbRent/detail.ftl b/projects/fangjia-open/src/main/webapp/ccbRent/detail.ftl index 2a539183c..05d0776a0 100644 --- a/projects/fangjia-open/src/main/webapp/ccbRent/detail.ftl +++ b/projects/fangjia-open/src/main/webapp/ccbRent/detail.ftl @@ -18,22 +18,28 @@ body { overflow: hidden; overflow-y: scroll; - background: #f8f8f8; + background: white; } .main { margin: 4px 8px 4px 8px; - background: #f8f8f8; + background: white; } .title { position: relative; color: #1E90FF; + font-size: 20px; + font-weight: bold; } .title1 { - font-size: 14px; + font-size: 18px; + font-weight: bold; color: #1E90FF; + margin-top: 2px; + margin-bottom: 2px; } .p { - font-size: 12px; + font-size: 18px; + line-height: 30px; } .table { width: 100%; @@ -77,11 +83,11 @@
2 璇勪及鏃堕棿

- 璇勪及缁撴灉
- 妤肩洏鍚嶇О锛�
- 妤肩洏鍦板潃锛�
- 绉熼噾浼板�硷紙RMB锛夛細鍏�/鏈�
- 澶у啓锛堜汉姘戝竵锛夛細
+ 璇勪及缁撴灉
+ 妤肩洏鍚嶇О锛�
+ 妤肩洏鍦板潃锛�
+ 绉熼噾浼板�硷紙RMB锛�锛�鍏�/鏈�
+ 澶у啓锛堜汉姘戝竵锛�锛�
@@ -91,20 +97,21 @@
4 鎴夸骇鐘跺喌
- 璇︾粏鍦板潃锛�
- 妤肩洏鍚嶇О锛�
- 瀹ゅ彿锛�
- 鏈濆悜锛�
- 寤虹瓚闈㈢Н锛�骞崇背
- 鎵�鍦ㄦゼ灞傦細 灞� 鎬诲眰鏁�
- 鐢垫閰嶅锛�
- 瑁呬慨鎯呭喌锛�
- 寤虹瓚骞翠唬锛�
+ 璇︾粏鍦板潃锛�
+ 妤肩洏鍚嶇О锛�
+ 瀹ゅ彿锛�
+ 鏈濆悜锛�
+ 寤虹瓚闈㈢Н锛�骞崇背
+ 鎵�鍦ㄦゼ灞�锛� 灞� 鎬诲眰鏁�
+ 鐢垫閰嶅锛�
+ 瑁呬慨鎯呭喌锛�
+ 寤虹瓚骞翠唬锛�
5 鍘嗗彶绉熼噾
+
6 宸茬鎴挎簮
@@ -179,7 +186,8 @@ - + + @@ -218,9 +226,11 @@ // jspdf.addFont('SourceHanSans-Normal.ttf', 'SourceHanSans-Normal', 'normal'); // jspdf.setFont('SourceHanSans-Light'); //console.log(jspdf.getFontList()); + var chart; $("#download").bind('click', function(){ + var chartImgUrl = $('#chartImgUrl').val(); window.location.href = "download?city="+params.city+"&district="+params.district+"&name="+params.name+"&address="+params.address+"&size="+params.size+"&floor="+params.floor - +"&height="+params.height+"&room="+params.room+"&hall="+params.hall+"&toilet="+params.toilet+"&direction="+params.direction+"&fitment="+params.fitment; + +"&height="+params.height+"&room="+params.room+"&hall="+params.hall+"&toilet="+params.toilet+"&direction="+params.direction+"&fitment="+params.fitment+"&chartImgUrl=" + chartImgUrl; // jspdf.html(document.body, { // callback: function (doc) { // doc.save("绉熼噾鎶ュ憡.pdf"); @@ -319,9 +329,9 @@ ' + d.city + ' \ ' + d.region + ' \ ' + d.districtName + ' \ - ' + d.totalPrice + ' \ - ' + d.area + ' \ - ' + d.room + ' \ + ' + (d.totalPrice != undefined ? d.totalPrice : '') + ' \ + ' + (d.area != undefined ? d.area : '') + ' \ + ' + (d.room != undefined ? d.room : '') + ' \ ' + hall + ' \ ' + fitment + ' \ ' + formatDate(d.tradeDate) + ' \ @@ -340,10 +350,10 @@ ' + d.city + ' \ ' + d.region + ' \ ' + d.districtName + ' \ - ' + d.totalPrice + ' \ - ' + d.area + ' \ - ' + d.room + ' \ - ' + d.hall + ' \ + ' + (d.totalPrice != undefined ? d.totalPrice : '') + ' \ + ' + (d.area != undefined ? d.area : '') + ' \ + ' + (d.room != undefined ? d.room : '') + ' \ + ' + (d.hall != undefined ? d.hall : '')+ ' \ ' + fitment + ' \ ' + date + ' \ '); @@ -359,9 +369,9 @@ $('#sameDistricts').append(' \ ' + d.city + ' \ ' + d.region + ' \ - ' + d.districtName + ' \ - ' + d.roomPrice + ' \ - ' + d.room + ' \ + ' + (d.districtName != undefined ? d.districtName:'' ) + ' \ + ' + (d.roomPrice != undefined ? d.roomPrice:'') + ' \ + ' + (d.room != undefined ? d.room:'') + ' \ 1 \ '); }) @@ -426,8 +436,9 @@ var config = {yaxis : yaxis}; createHighChartNow(renderId, data, config); }; + function createHighChartNow(renderId, data, config) { - var chart = new Highcharts.Chart({ + chart = new Highcharts.Chart({ credits:{ enabled:true, text: '', @@ -446,6 +457,9 @@ renderTo: renderId, type: 'spline' }, + exporting: { + enabled: false + }, title: { text: data.title, floating: false, @@ -479,7 +493,26 @@ } } }); + saveHighChartImg(); }; + + function saveHighChartImg() { + var svg = chart.getSVG(); + $.ajax({ + // 娉ㄦ剰闇�瑕佸姞.. + url: "saveHighChartAsImage", + data: { + svg:svg + }, + dataType: "json", + type: "post", + success: function (ret) { + if (ret.address != null) { + $('#chartImgUrl').val(ret.address) + } + } + }) + } $(function () { init(); }); diff --git a/projects/fangjia-open/src/main/webapp/ccbRent/downloadPage.ftl b/projects/fangjia-open/src/main/webapp/ccbRent/downloadPage.ftl index 615fef4c5..a7aef6754 100644 --- a/projects/fangjia-open/src/main/webapp/ccbRent/downloadPage.ftl +++ b/projects/fangjia-open/src/main/webapp/ccbRent/downloadPage.ftl @@ -14,221 +14,191 @@ + -
-
- 浼板�兼姤鍛� +
+
+ 1浼颁环缁撴灉
-
-
1 璇勪及鐢虫槑
-
寤鸿閾惰閬靛惊涓珛銆佸瑙傘�佸叕姝g殑鍘熷垯锛岄�氳繃鍛ㄥ瘑缁嗚嚧鐨勬祴绠楋紝璇勪及姹傚緱鍒嗘瀽瀵硅薄鐨勬埧鍦颁骇甯傚満鍙傝�冦��
+
1.1璇勪及鐢虫槑锛�
+
寤鸿閾惰閬靛惊涓珛銆佸瑙傘�佸叕姝g殑鍘熷垯锛岄�氳繃鍛ㄥ瘑缁嗚嚧鐨勬祴绠楋紝璇勪及姹傚緱鍒嗘瀽瀵硅薄鐨勬埧
鍦颁骇甯傚満鍙傝�冦��
+ +
1.2璇勪及鏃堕棿锛�
+
<#if result.cal_date??>${result.cal_date}
+
1.3璇勪及缁撴灉锛�
+
+ 妤肩洏鍚嶇О锛�<#if result.name??>${result.name}

+ 妤肩洏鍦板潃锛�<#if result.address??>${result.address}

+ 绉熼噾浼板�硷紙RMB锛�锛�<#if result.totalPrice??>${result.totalPrice}鍏�/鏈�

+ 澶у啓锛堜汉姘戝竵锛�锛�<#if result.totalPriceUpper??>${result.totalPriceUpper}

+
+
1.4鎶ュ憡鏈夋晥鏈燂細
+
鏈姤鍛婂湪甯傚満鏃犺緝澶ф尝鍔ㄧ殑鎯呭喌涓嬶紝璇ヨ瘎浼颁环鏍煎湪鎶ュ憡鍑哄叿璧蜂笁涓湀鍐呮湁鏁堛��
+
2鎴夸骇鐘跺喌
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
妤肩洏鍚嶇О<#if result.name??>${result.name}鍦板潃<#if result.address??>${result.address}
瀹ゅ彿<#if result.room??>${result.room}鏈濆悜<#if result.direction??>${result.direction}
寤虹瓚闈㈢Н<#if result.size??>${result.size}骞崇背鎵�鍦ㄦゼ灞�<#if result.floor??>${result.floor} 灞� <#if result.height??>${result.height}鎬诲眰鏁�
鐢垫閰嶅<#if result.hasLift??>${result.hasLift}瑁呬慨鎯呭喌<#if result.fitment??>${result.fitment}
寤虹瓚骞翠唬<#if result.buildYear??>${result.buildYear}
+
+
+ 3鍘嗗彶绉熼噾
-
2 璇勪及鏃堕棿
-
- <#if result.cal_date??>${result.cal_date}
- 璇勪及缁撴灉
- 妤肩洏鍚嶇О锛�<#if result.name??>${result.name}
- 妤肩洏鍦板潃锛�<#if result.address??>${result.address}
- 绉熼噾浼板�硷紙RMB锛夛細<#if result.totalPrice??>${result.totalPrice}鍏�/鏈�
- 澶у啓锛堜汉姘戝竵锛夛細<#if result.totalPriceUpper??>${result.totalPriceUpper}
-
+ <#if result.tradesImgUrl??> + <#-- --> + + <#else> +
+ + +
鏆傛棤鏁版嵁
+
+
-
-
3 鎶ュ憡鏈夋晥鏈�
-
鏈姤鍛婂湪甯傚満鏃犺緝澶ф尝鍔ㄧ殑鎯呭喌涓嬶紝璇ヨ瘎浼颁环鏍煎湪鎶ュ憡鍑哄叿璧蜂笁涓湀鍐呮湁鏁堛��
+
+ 4宸茬鎴挎簮
-
-
4 鎴夸骇鐘跺喌
-
- 璇︾粏鍦板潃锛�<#if result.address??>${result.address}
- 妤肩洏鍚嶇О锛�<#if result.name??>${result.name}
- 瀹ゅ彿锛�<#if result.room??>${result.room}
- 鏈濆悜锛�<#if result.direction??>${result.direction}
- 寤虹瓚闈㈢Н锛�<#if result.size??>${result.size}骞崇背
- 鎵�鍦ㄦゼ灞傦細<#if result.floor??>${result.floor} 灞� <#if result.height??>${result.height}鎬诲眰鏁�
- 鐢垫閰嶅锛�<#if result.hasLift??>${result.hasLift}
- 瑁呬慨鎯呭喌锛�<#if result.fitment??>${result.fitment}
- 寤虹瓚骞翠唬锛�<#if result.buildYear??>${result.buildYear}
-
-
-
-
5 鍘嗗彶绉熼噾
-
- <#if result.tradesImgUrl??> -<#-- --> - - <#else> -
- - -
鏆傛棤鏁版嵁
-
- -
-
-
-
6 宸茬鎴挎簮
-
- - - - - - - - - - - - +
+
鍩庡競鍖哄煙灏忓尯鍚�鎴愪氦浠�闈㈢Н瀹�鍘�瑁呬慨鎴愪氦鏃ユ湡
+ + + + + + + + + + + + + + + <#if (result.trades??&&result.trades?size>0)> + <#list result.trades as bo> + + + + + + + + + + - - - <#if (result.trades??&&result.trades?size>0)> - <#list result.trades as bo> - - - - - - - - - - - - - <#else> - - - -
鍩庡競鍖哄煙灏忓尯鍚�鎴愪氦浠�闈㈢Н瀹�鍘�瑁呬慨鎴愪氦鏃ユ湡
<#if bo.city??>${bo.city}<#if bo.region??>${bo.region}<#if bo.districtName??>${bo.districtName}<#if bo.totalPrice??>${bo.totalPrice}<#if bo.area??>${bo.area}<#if bo.room??>${bo.room}<#if bo.hall??>${bo.hall}<#if bo.fitment??>${bo.fitment}<#if bo.source??>${bo.source}
<#if bo.city??>${bo.city}<#if bo.region??>${bo.region}<#if bo.districtName??>${bo.districtName}<#if bo.totalPrice??>${bo.totalPrice}<#if bo.area??>${bo.area}<#if bo.room??>${bo.room}<#if bo.hall??>${bo.hall}<#if bo.fitment??>${bo.fitment}<#if bo.source??>${bo.source}
鏆傛棤鏁版嵁
-
+ + <#else> + 鏆傛棤鏁版嵁 + + +
-
-
7 鏍锋湰鍙傝��
-
- - - - - - - - - - - - - - - - <#if (result.houses??&&result.houses?size>0)> - <#list result.houses as bo> - - - - - - - - - - - - - <#else> - - - -
鍩庡競鍖哄煙灏忓尯鍚�绉熼噾闈㈢Н瀹�鍘�瑁呬慨鎴愪氦鏃ユ湡
<#if bo.city??>${bo.city}<#if bo.region??>${bo.region}<#if bo.districtName??>${bo.districtName}<#if bo.totalPrice??>${bo.totalPrice}<#if bo.area??>${bo.area}<#if bo.room??>${bo.room}<#if bo.hall??>${bo.hall}<#if bo.fitment??>${bo.fitment}<#if bo.root??>${bo.root}
鏆傛棤鏁版嵁
-
+
+ 5鏍锋湰鍙傝��
-
-
8 鍛ㄨ竟鍚岀被灏忓尯绉熼噾
-
- - - - - - - - - - - - - <#if (result.districts??&&result.districts?size>0)> - <#list result.districts as bo> - - - - - - - - - - <#else> - - - -
鍩庡競鍖哄煙灏忓尯鍚�骞崇背绉熼噾瀹�鍘�
<#if bo.city??>${bo.city}<#if bo.region??>${bo.region}<#if bo.districtName??>${bo.districtName}<#if bo.totalPrice??>${bo.totalPrice}<#if bo.room??>${bo.room}<#if bo.hall??>${bo.hall}
鏆傛棤鏁版嵁
-
+
+ + + + + + + + + + + + + + + + <#if (result.houses??&&result.houses?size>0)> + <#list result.houses as bo> + + + + + + + + + + + + + <#else> + + + +
鍩庡競鍖哄煙灏忓尯鍚�绉� 閲�闈㈢Н瀹�鍘�瑁呬慨鎴愪氦鏃ユ湡
<#if bo.city??>${bo.city}<#if bo.region??>${bo.region}<#if bo.districtName??>${bo.districtName}<#if bo.totalPrice??>${bo.totalPrice}<#if bo.area??>${bo.area}<#if bo.room??>${bo.room}<#if bo.hall??>${bo.hall}<#if bo.fitment??>${bo.fitment}<#if bo.root??>${bo.root}
鏆傛棤鏁版嵁
+
+
+ 6鍛ㄨ竟鍚岀被灏忓尯绉熼噾 +
+
+ + + + + + + + + + + + + <#if (result.districts??&&result.districts?size>0)> + <#list result.districts as bo> + + + + + + + + + + <#else> + + + +
鍩庡競鍖哄煙灏忓尯鍚�骞崇背绉熼噾瀹�鍘�
<#if bo.city??>${bo.city}<#if bo.region??>${bo.region}<#if bo.districtName??>${bo.districtName}<#if bo.totalPrice??>${bo.totalPrice}<#if bo.room??>${bo.room}<#if bo.hall??>${bo.hall}<#else>1
鏆傛棤鏁版嵁
diff --git a/projects/fangjia-open/src/main/webapp/ccbRent/index.ftl b/projects/fangjia-open/src/main/webapp/ccbRent/index.ftl index 8633e3e89..60f3ac58f 100644 --- a/projects/fangjia-open/src/main/webapp/ccbRent/index.ftl +++ b/projects/fangjia-open/src/main/webapp/ccbRent/index.ftl @@ -260,7 +260,7 @@ $("#evaluateBtn").prop("disabled", true) $.ajax({ // 娉ㄦ剰闇�瑕佸姞.. - url: '../property/evaluateRent', + url: '../ccbRent/evaluateRent', data:$("#evaluateForm").serialize(), dataType:"json", type:"post", diff --git a/projects/fangjia-open/src/main/webapp/fonts/simsun.ttc b/projects/fangjia-open/src/main/webapp/fonts/simsun.ttc new file mode 100644 index 000000000..e64e92ed9 Binary files /dev/null and b/projects/fangjia-open/src/main/webapp/fonts/simsun.ttc differ diff --git a/projects/fangjia-open/src/main/webapp/static/images/fjt/bg.png b/projects/fangjia-open/src/main/webapp/static/images/fjt/bg.png new file mode 100644 index 000000000..61c91e3fe Binary files /dev/null and b/projects/fangjia-open/src/main/webapp/static/images/fjt/bg.png differ diff --git a/projects/fangjia-open/src/main/webapp/static/style/report.css b/projects/fangjia-open/src/main/webapp/static/style/report.css new file mode 100644 index 000000000..93450a11b --- /dev/null +++ b/projects/fangjia-open/src/main/webapp/static/style/report.css @@ -0,0 +1,141 @@ +body { + overflow: hidden; + overflow-y: scroll; + background: white; + /** 鐗瑰埆娉ㄦ剰锛屽惁鍒欎腑鏂囦笉鐢熸晥 **/ + font-family: SimSun; +} +.content { + width: 840px; + height: 100%; + margin: 0 auto; + background: #fff; + padding-left: 20px; +} +.content .t1 { + background: url(../images/fjt/bg.png) 0 0 no-repeat; + width: 820px; + height: 42px; + color: #704f1c; + font-size: 20px; + font-weight: bold; + text-indent: 14px; + line-height: 46px; + margin-bottom: 25px; +} +.content .t1 em { + font-family: "Arial"; + color: #FFF; + margin-right: 20px; +} +em { + font-style: normal; +} +.content .st1 { + background: url(../images/fjt/bg.png) 0 -60px no-repeat; + width: 172px; + height: 40px; + color: #704f1c; + font-size: 18px; + font-weight: bold; + text-indent: 7px; + line-height: 46px; + margin-bottom: 10px; +} +.content .st1 em { + font-family: "Arial"; + color: #FFF; + margin-right: 14px; +} +.content .text1 { + font-size: 18px; + line-height: 26px; + margin: 0 20px 10px 50px; + word-wrap: break-word; +} +.text1 td { + text-align: center +} +.content .text2 { + font-size: 18px; + line-height: 40px; + padding: 10px 20px 10px 50px; + background: #eee; + width: 750px; + margin-bottom: 30px; +} +.content .text3 { + font-size: 16px; + line-height: 40px; + padding: 10px 20px 10px 50px; + background: #eee; + width: 750px; + margin-bottom: 30px; +} +.content .c1 { + height: 200px; +} +.content .distTab { + font-size: 18px +} +.content .distTab .textTd { + font-weight: bold; +} +.content .distTab tr td { + height: 30px; +} +.content .distTab tr th { + text-align: left; +} +.content .distTab .textRight1 { + text-align: right; + padding-right: 80px; +} +.content .distTab .textRight2 { + text-align: right; + padding-right: 5px; +} +.title { + position: relative; + color: #1E90FF; +} +.title1 { + font-size: 19px; + color: #1E90FF; + margin-top: 3px; + margin-bottom: 3px; +} +.p { + font-size: 16px; +} +table { + border-collapse: separate; + text-indent: initial; + border-spacing: 2px; +} +tbody { + display: table-row-group; + vertical-align: middle; + border-color: inherit; +} +.inline { + width: 550px; + border-spacing: 0; + text-align: left; +} +.table .inline th { + color: #999; + font-weight: normal; + line-height: 30px; + text-align: left; +} +.download { + float: right; + color: #3d4145; + border: solid 1px #333; + order-radius: 3px; + font-size: 12px; + line-height: 1rem; + width: 43px; + margin-top: 10px; +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/Watcher.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/Watcher.groovy index 85088ddd8..004d6bd4a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/Watcher.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/Watcher.groovy @@ -32,6 +32,7 @@ public enum Watcher { BaYi('15300424133','bayi@fangjia.com'), WeiLei('18101751850','weilei@fangjia.com'), JiangYaHui('13684913780','jiangyahui@fangjia.com'), + ShenJian('13764369683', 'shenjian@fangjia.com') private final String mobile; private final String email; diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 8914794b4..4c2f2df46 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -218,7 +218,7 @@ class CommonDatautils { q.put("visible", 0); q.put("cat", cat); q.put("city", city); -// q.put("name", "工商大楼"); +// q.put("name","东寓新村") DBObject keys = new BasicDBObject(); if(fields){ @@ -1392,6 +1392,21 @@ class CommonDatautils { } + public static def queryDistrictPriceByWeekend(city, region, name, weekend, limit){ + DBObject queryCondition = new BasicDBObject() + queryCondition.put("city", city); + queryCondition.put("region", region); + queryCondition.put("district_name", name); + if(weekend){ + queryCondition.put('weekend', new BasicDBObject('$gte', weekend)); + } + DBCursor cur = DIST_STAT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", 1)).limit(limit) + def dbo = null; + while(cur.hasNext()) { + dbo = cur.next() + } + return dbo; + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/companyCrawl/AiqichaCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/companyCrawl/AiqichaCrawl.groovy new file mode 100644 index 000000000..dd794ce76 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/companyCrawl/AiqichaCrawl.groovy @@ -0,0 +1,148 @@ +package com.fangjia.script.companyCrawl + +import net.sf.json.JSONObject +import org.jsoup.Jsoup + +import java.util.concurrent.atomic.AtomicInteger + +class AiqichaCrawl { + + def static final proxyPort = 9020; + def static final proxyServer = "http-dyn.abuyun.com"; + def static final proxyUser = "HZU1XU28H3G4Z44D"; + def static final proxyPass = "466C86ED77FE43D4"; + def companyKeys = ["亨鑫","捷东","盈复","隆庆","卓源 ","合万","通广","安盛","东利","瑞科 ","凡迪","捷信","安盛","安盛","东进","源盛","茂禄","泰亚","宏成","光益","华圣","益全","长凯","宏新","锦立","宜佳","万美","正欣","和利","国宏","和裕","新昌","佳升","正德","源美","宝峰","金广","龙奥","宏美","成同","义丰","全洪","国聚","亨信","百益","广宏","仁瑞","亿昌","东旭","聚凯","洪成","兴同","赛迪","亨国","浩长","达宏","寿满","亿信","本盛","盈凯","浩汇","元辉","祥生","皇弘","讯思","新州","国泰","成合","茂浩","信源","润凯","虎光","安瑞","益吉","达兴","顺光","发盛","金优","捷丰","春凯","昌兴","本佳","远盛","盛达","亚飞","东昌","益如","宝东","德信","浩盈","多泰","泰联","升聚","辉丰","茂发","盈德","旭嘉","优源","益同","协盈","锦泰","捷久","本宏","德泰","聚富","全庆","凯安","进恒","全伟","优元","凯安","义丰","丰宇","瑞优","隆泰","诺德","台升","裕大","亨恒","义浩","百亨","欣昌","茂全","同亨","长圣","成谦","达泰","高协","正思","安巨","新耀","利仁","国益","旺洪","汇中","旭昌","弘康","优盛","旭达","晶安","华浩","耀春","圣本","康丰","光复","同凌","宇伯","乐鸿","成泰","宏协","华信","达泰","广丰","泰全","汇恒","杰森","天畅","恒通","亚达","庆合","丰圣","德长","茂百","汇久","欣鼎","谦和","厚辉","耀德","瑞隆","多辉","昌茂","成高","益升","安达","兴飞","百进","鼎宝","祥盛","信和","隆顺","亚欣","厚全","博文","讯诺","泰如","亨瑞","凯祥","捷浩","文舟","金庆","成仁","万协","宇中","宏祥","永多","艺峰","美益","创赢","奥龙","德宇","国福","飞利","久欣","如亨","安广","协佳","升捷","盛圣","富凯","长鑫","元卓","荣亨","亨仁","康鑫","信和","百新","干瑞","润万","富庭","信宇","源天","凯佳","正升","裕荣","厚泽","晶富","广新","正多","鼎高","丰汇","永华","兴升","复腾","隆诺","中发","广源","康胜","同进","仁捷","康凯","义洪","晶盛","正祥","裕祥","宇新","洪祥","润岳","义富","盛安","浩易"] + static AtomicInteger AbuyunRequestCount = new AtomicInteger(0); + public final static int AbuyunMaxCount = 10; + + static main(args) { + def task = new AiqichaCrawl(); + task.process() + } + + void process(Object args) { + + def url = "https://aiqicha.baidu.com/s/l?q=%E6%96%B0%E5%8F%8B&t=&p=3&s=10&o=0&f=%7B%7D " + for(def key : companyKeys){ + def queryBody = null + def s = crawl(url, queryBody); + if(s){ + print s + + + } + println s + } + } + + def crawl(url, queryBody){ + Map headers = new LinkedHashMap() + headers.put("Host","aiqicha.baidu.com") + headers.put("Connection","keep-alive") + headers.put("Cache-Control","max-age=0") + headers.put("sec-ch-ua",'"Chromium";v="88", "Google Chrome";v="88", ";Not A Brand";v="99"') + headers.put("sec-ch-ua-mobile","?0") + headers.put("Upgrade-Insecure-Requests","1") + headers.put("User-Agent","Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.104 Safari/537.36") + headers.put("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9") + headers.put("Sec-Fetch-Site","same-origin") + headers.put("Sec-Fetch-Mode","navigate") + headers.put("Sec-Fetch-User","?1") + headers.put("Sec-Fetch-Dest","document") + headers.put("Accept-Language","zh-CN,zh;q=0.9") + + String s = doPost(url, queryBody, headers) + if(s && s.contains("window.pageData = ")){ + def prefix = "window.pageData =" + def s1 = s.substring(s.indexOf(prefix)+prefix.size()) + s1 = s1.substring(0, s1.indexOf("}};")+2) + + JSONObject jsonObject = JSONObject.fromObject(s1); + print jsonObject + } + + return s + + } + + static String doPost(String strUrl, String content, Map headers) { + String result = ""; + OutputStream outputStream = null; + InputStream inputStream = null; + StringBuilder sb = new StringBuilder(); + try { + checkAbuyun(true); + + def responseMap = [:] + // 创建代理服务器地址对象 + InetSocketAddress addr = new InetSocketAddress(proxyServer, proxyPort); + // 创建HTTP类型代理对象 + Proxy proxy = new Proxy(Proxy.Type.HTTP, addr); + + URL url = new URL(strUrl); + //通过调用url.openConnection()来获得一个新的URLConnection对象,并且将其结果强制转换为HttpURLConnection. + HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection(); + urlConnection.setRequestMethod("POST"); + //设置连接的超时值为30000毫秒,超时将抛出SocketTimeoutException异常 + urlConnection.setConnectTimeout(30000); + //设置读取的超时值为30000毫秒,超时将抛出SocketTimeoutException异常 + urlConnection.setReadTimeout(30000); + //将url连接用于输出,这样才能使用getOutputStream()。getOutputStream()返回的输出流用于传输数据 + urlConnection.setDoOutput(true); + urlConnection.setDoInput(true); + //设置通用请求属性为默认浏览器编码类型 +// urlConnection.setRequestProperty("content-type", "application/x-www-form-urlencoded"); + + + if (headers != null && headers.keySet().size() > 0) { + for (String head : headers.keySet()) { + urlConnection.setRequestProperty(head, headers.get(head)); + } + } + //getOutputStream()返回的输出流,用于写入参数数据。 + outputStream = urlConnection.getOutputStream(); + if(content){ + outputStream.write(content.getBytes()); + } + outputStream.flush(); + outputStream.close(); + //此时将调用接口方法。getInputStream()返回的输入流可以读取返回的数据。 + inputStream = urlConnection.getInputStream(); + + + InputStreamReader reader = new InputStreamReader(inputStream, "UTF-8"); + char[] buff = new char[1024]; + int length = 0; + while ((length = reader.read(buff)) != -1) { + String s = new String(buff, 0, length); + sb.append(s); + } + } catch (IOException e) { + e.printStackTrace(); + }finally{ + if(outputStream!=null){ + outputStream.close(); + } + if(inputStream!=null){ + inputStream.close(); + } + } + + return sb.toString(); + } + + public static checkAbuyun(boolean check) { + if (check) { + while (AbuyunRequestCount.get() >= AbuyunMaxCount) { + try { + Thread.sleep(5000); + } catch (InterruptedException e) { + e.printStackTrace() + } + } + AbuyunRequestCount.incrementAndGet(); + } else { + AbuyunRequestCount.decrementAndGet(); + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/companyCrawl/GsxtCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/companyCrawl/GsxtCrawl.groovy new file mode 100644 index 000000000..d675adb3a --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/companyCrawl/GsxtCrawl.groovy @@ -0,0 +1,715 @@ +package com.fangjia.script.companyCrawl + +import com.eclipsesource.v8.V8 +import com.fangjia.script.crawl.JQueryUtils +import com.fangjia.script.util.ABuYunUtil +import jdk.nashorn.api.scripting.ScriptObjectMirror +import org.apache.commons.io.IOUtils; +import org.apache.http.Header +import org.apache.http.HttpResponse +import org.apache.http.client.HttpClient +import org.apache.http.client.methods.HttpGet +import org.jsoup.Jsoup +import org.jsoup.select.Elements + +import org.apache.http.impl.client.HttpClients + +import javax.script.ScriptEngine +import javax.script.ScriptEngineManager; +import java.util.concurrent.atomic.AtomicInteger + +class GsxtCrawl { + + def static final proxyPort = 9020; + def static final proxyServer = "http-dyn.abuyun.com"; + def static final proxyUser = "HZU1XU28H3G4Z44D"; + def static final proxyPass = "466C86ED77FE43D4"; + + public final static int AbuyunMaxCount = 10; + static AtomicInteger AbuyunRequestCount = new AtomicInteger(0); + + def companyKeys = ["亨鑫","捷东","盈复","隆庆","卓源 ","合万","通广","安盛","东利","瑞科 ","凡迪","捷信","安盛","安盛","东进","源盛","茂禄","泰亚","宏成","光益","华圣","益全","长凯","宏新","锦立","宜佳","万美","正欣","和利","国宏","和裕","新昌","佳升","正德","源美","宝峰","金广","龙奥","宏美","成同","义丰","全洪","国聚","亨信","百益","广宏","仁瑞","亿昌","东旭","聚凯","洪成","兴同","赛迪","亨国","浩长","达宏","寿满","亿信","本盛","盈凯","浩汇","元辉","祥生","皇弘","讯思","新州","国泰","成合","茂浩","信源","润凯","虎光","安瑞","益吉","达兴","顺光","发盛","金优","捷丰","春凯","昌兴","本佳","远盛","盛达","亚飞","东昌","益如","宝东","德信","浩盈","多泰","泰联","升聚","辉丰","茂发","盈德","旭嘉","优源","益同","协盈","锦泰","捷久","本宏","德泰","聚富","全庆","凯安","进恒","全伟","优元","凯安","义丰","丰宇","瑞优","隆泰","诺德","台升","裕大","亨恒","义浩","百亨","欣昌","茂全","同亨","长圣","成谦","达泰","高协","正思","安巨","新耀","利仁","国益","旺洪","汇中","旭昌","弘康","优盛","旭达","晶安","华浩","耀春","圣本","康丰","光复","同凌","宇伯","乐鸿","成泰","宏协","华信","达泰","广丰","泰全","汇恒","杰森","天畅","恒通","亚达","庆合","丰圣","德长","茂百","汇久","欣鼎","谦和","厚辉","耀德","瑞隆","多辉","昌茂","成高","益升","安达","兴飞","百进","鼎宝","祥盛","信和","隆顺","亚欣","厚全","博文","讯诺","泰如","亨瑞","凯祥","捷浩","文舟","金庆","成仁","万协","宇中","宏祥","永多","艺峰","美益","创赢","奥龙","德宇","国福","飞利","久欣","如亨","安广","协佳","升捷","盛圣","富凯","长鑫","元卓","荣亨","亨仁","康鑫","信和","百新","干瑞","润万","富庭","信宇","源天","凯佳","正升","裕荣","厚泽","晶富","广新","正多","鼎高","丰汇","永华","兴升","复腾","隆诺","中发","广源","康胜","同进","仁捷","康凯","义洪","晶盛","正祥","裕祥","宇新","洪祥","润岳","义富","盛安","浩易"] + static main(args) { + def task = new GsxtCrawl(); + //task.process() + task.run() + } + + def run(){ + getObject() + HttpClient client = HttpClients.createDefault(); + HttpGet get = new HttpGet("http://www.gsxt.gov.cn/corp-query-homepage.html"); + get.setHeader("Upgrade-Insecure-Requests", "1"); + get.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36"); + + HttpResponse response = client.execute(get); + def __jsluid_h = getJsluid(response) + def s = getResponseBodyAsString(response) + println (s) + def __jsl_clearance = getJsContent(s); + + //Cookie: __jsluid_h=26cb8385493c2e424767eca767720c87; __jsl_clearance=1618899529.627|-1|ao39lP6IVSloPMvq5%2FOSBqSiPZ0%3D + def newCookie = __jsluid_h+";"+__jsl_clearance + get.setHeader("Cookie", newCookie) + HttpResponse response2 = client.execute(get); + def s2 = getResponseBodyAsString(response2) + def __jsl_clearance2 = getJsContent2(s2); + println(__jsl_clearance2) + + // fristCrawl() + } + + + //该函数测试Java与js对象 + static void getObject() throws Exception{ + String str = "var _0xbe8c = ['w4LCg8Onw7o=', 'ZAjDvCw=', 'YcKOw6Fm', 'wpHCqsO3CQ==', 'UMKCXnw=', 'w7Aww6XCnQ==', 'BcKNwrvCng==', 'J2zCnsKi', 'wq02WMOx', 'RQXDrUw=', 'eCnCnDw=', 'Q8KowpzDjA==', 'YznDucOn', 'YcOrXxk=', 'w4t9PsKM', 'DMO4QmA=', 'w5HCklPDmQ==', 'w5rCv8Omw6Q=', 'NcOwc8K5', 'GUxjw7Q=', 'w7zDicK8w4I=', 'VCDDpsKv', 'cwLChT4=', 'wpvCl1vDqw==', 'wrvCvMOKNQ==', 'RTnDrAc=', 'GMKFw4hk', 'SxnDvC0=', 'H2NHNQ==', 'w6Aqw7LCqg==', 'wq0mW8Ok', 'L1zCv8Kp', 'B8OqwoI2', 'Jl7DuX8=', 'JcOBwpIR', 'U8OnfSo=', '6K2K5rOG6aqL6KyM', 'f8Kiwo/Dhg==', 'aMK6w6vDog==', 'JcKgw4FC', 'w61vJEg=', 'LMOrwoY0', 'GcK4w6lt', 'w4XDrsO5wpY=', 'J8OhwoM1', 'w5Mzw7J/', 'wocbcBA=', 'SibDt8Oj', 'w7xgwoXDkQ==', 'AcKyDEQ=', 'Rg7Dkw==', 'w6UFK8Kw', 'TTzDjcKK', 'w6QhDcKy', 'ZBLDols=', 'ZjPDrk4=', 'w6XDqX3Ctw==', 'w6EvJcKT', 'JHtUDQ==', 'wq8mI8KE', 'JMOhwoJ9', 'MsK1AkE=', 'w7fCmhcm', 'VsK2w5fDtQ==', 'w7EUw4PCnw==', 'dz/DgsKu', 'UzfDtcK0', 'wpd9fWE=', 'worDvcKFw4M=', 'w4PDr0zDhg==', 'w7ocSMKg', 'wp1uwoZd', 'eAjDv8Oz', 'aATCgTw=', 'w7UGw7k=', 'wqZCwotq', 'HFIBw5s=', 'WsOdw7JB', 'w5PDpXHCpA==', 'GMO8wqdD', 'w4TCsQs=', 'w6/DmcODwpg=', 'w7MuwrjDoQ==', 'dDHDhsOj', 'G308w70=', 'CsO6wog9', 'FcONwrVw', 'M3vCmMK4', 'w6/Dm8OvwpI=', 'QsOLw5ZM', 'TgLDicKy', 'QSHCuQA=', 'UsOlw5RI', 'w7EqwqDDrw==', 'w6jDusK5w4k=', 'OsKaw5rDlg==', 'wosLQsOk', 'DcKxw7Vo', 'w5VuIEo=', 'w7sXw6TCqw==', 'wrLDr8KRw7s=', 'wr9+wpJR', 'w4LDscO0wrQ=', 'FFIPw6U=', 'J8Ksw5rDlA==', 'd8KZwqjDgw==', 'PcKvDF4=', 'MHh2GA==', 'dgDDs0Y=', 'wpxSS1Y=', 'w7l5JMKh', 'w7N8JMKC', 'XsKhw7hR', 'QxDDqm4=', 'PcKGw5ZJ', 'w7HCuAIQ', 'E8Kmw53Dmg==', 'CMOrwok3', 'U8K9wpvDnA==', 'IW06w5U=', 'w7DDjcOGwpQ=', 'X8KlwpHDpA==', 'w6TDn8O1wpo=', 'ZjnDv08=', 'w7/Cqjsp', 'wqLDk8KYw58=', 'w6fDvMKXw4g=', 'FsK8IUY=', 'w4knw4Rh', 'w5zCu8Ogw5M=', 'wqJ0ckc=', 'T8Kgw69G', 'GF0hw74=', 'FMOkwqkH', 'wokmbMOM', 'djHDtcKW', 'FXkow64=', 'B8OHwowH', 'VSjCpQU=', 'w5vDs8K0w4s=', 'w7vCkUV1', 'w7dMIcK8', 'WMOjZyI=', 'LcKBw7po', 'BMK7HkY=', 'B8K0w7F4', 'YsK+w67Drw==', 'HlQKw7I=', 'wpLCrMKrNg==', 'w7IjcsKy', 'w6zDusKhw4o=', 'WhPDm8Om', 'dMKlw7RO', 'dcOHw5R3', 'Nkpww7M=', 'wrnCsMKLdg==', 'dDTDqMKb', 'S8KXZXE=', 'wq/CssOpJA==', 'w7J/KFw=', 'QMKSWEo=', 'woNxbEc=', 'wqVIwow=', 'TD/Dqg4=', 'w7TDpmzCiw==', 'w7HDjWDCgA==', 'w7IEw6vCoA==', 'Q8K+w4zDiQ==', 'PElvw7k=', 'cDXDsRo=', 'FMOLX0c=', 'Z8KHwojDoA==', 'TTTCqzo=', 'woXCrsK6Uw==', 'w4lhHnY=', 'w6Q/dsKj', 'KsKIwpLCtQ==', 'QQTDn8Oy', 'D8Omw7zDoQ==', 'E8KzGk0=']; (function(_0x44c192, _0xbe8c61) {\n" + + "\t\tvar _0x1533fb = function(_0x5b499f) {\n" + + "\t\t\twhile (--_0x5b499f) {\n" + + "\t\t\t\t_0x44c192['push'](_0x44c192['shift']());\n" + + "\t\t\t}\n" + + "\t\t};\n" + + "\t\t_0x1533fb(++_0xbe8c61);\n" + + "\t} (_0xbe8c, 0x185));\n" + + "\tvar _0x1533 = function(_0x44c192, _0xbe8c61) {\n" + + "\t\t_0x44c192 = _0x44c192 - 0x0;\n" + + "\t\tvar _0x1533fb = _0xbe8c[_0x44c192];\n" + + "\t\tif (_0x1533['GlBvhA'] === undefined) { (function() {\n" + + "\t\t\t\tvar _0xa131c3 = function() {\n" + + "\t\t\t\t\tvar _0x9829e5;\n" + + "\t\t\t\t\ttry {\n" + + "\t\t\t\t\t\t_0x9829e5 = Function('return\\x20(function()\\x20' + '{}.constructor(\\x22return\\x20this\\x22)(\\x20)' + ');')();\n" + + "\t\t\t\t\t} catch(_0x1ba3fa) {\n" + + + "\t\t\t\t\t}\n" + + "\t\t\t\t\treturn _0x9829e5;\n" + + "\t\t\t\t};\n" + + "\t\t\t\tvar _0x5e86d6 = _0xa131c3();\n" + + "\t\t\t\tvar _0x67fb0c = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=';\n" + + "\t\t\t\t_0x5e86d6['atob'] || (_0x5e86d6['atob'] = function(_0x1cb044) {\n" + + "\t\t\t\t\tvar _0xa69f93 = String(_0x1cb044)['replace'](/=+\$/, '');\n" + + "\t\t\t\t\tvar _0x26bfb9 = '';\n" + + "\t\t\t\t\tfor (var _0x2f1d79 = 0x0,\n" + + "\t\t\t\t\t_0x3713bb, _0x4e514b, _0x29733b = 0x0; _0x4e514b = _0xa69f93['charAt'](_0x29733b++);~_0x4e514b && (_0x3713bb = _0x2f1d79 % 0x4 ? _0x3713bb * 0x40 + _0x4e514b: _0x4e514b, _0x2f1d79++%0x4) ? _0x26bfb9 += String['fromCharCode'](0xff & _0x3713bb >> ( - 0x2 * _0x2f1d79 & 0x6)) : 0x0) {\n" + + "\t\t\t\t\t\t_0x4e514b = _0x67fb0c['indexOf'](_0x4e514b);\n" + + "\t\t\t\t\t}\n" + + "\t\t\t\t\treturn _0x26bfb9;\n" + + "\t\t\t\t});\n" + + "\t\t\t} ());\n" + + "\t\t\tvar _0x38a7ff = function(_0x5dd34b, _0x3a0f2e) {\n" + + "\t\t\t\tvar _0x3e7a99 = [],\n" + + "\t\t\t\t_0x3e2c47 = 0x0,\n" + + "\t\t\t\t_0x383663,\n" + + "\t\t\t\t_0x14720e = '',\n" + + "\t\t\t\t_0x5864f4 = '';\n" + + "\t\t\t\t_0x5dd34b = atob(_0x5dd34b);\n" + + "\t\t\t\tfor (var _0x28a6d2 = 0x0,\n" + + "\t\t\t\t_0x2ea622 = _0x5dd34b['length']; _0x28a6d2 < _0x2ea622; _0x28a6d2++) {\n" + + "\t\t\t\t\t_0x5864f4 += '%' + ('00' + _0x5dd34b['charCodeAt'](_0x28a6d2)['toString'](0x10))['slice']( - 0x2);\n" + + "\t\t\t\t}\n" + + "\t\t\t\t_0x5dd34b = decodeURIComponent(_0x5864f4);\n" + + "\t\t\t\tvar _0x3091fb;\n" + + "\t\t\t\tfor (_0x3091fb = 0x0; _0x3091fb < 0x100; _0x3091fb++) {\n" + + "\t\t\t\t\t_0x3e7a99[_0x3091fb] = _0x3091fb;\n" + + "\t\t\t\t}\n" + + "\t\t\t\tfor (_0x3091fb = 0x0; _0x3091fb < 0x100; _0x3091fb++) {\n" + + "\t\t\t\t\t_0x3e2c47 = (_0x3e2c47 + _0x3e7a99[_0x3091fb] + _0x3a0f2e['charCodeAt'](_0x3091fb % _0x3a0f2e['length'])) % 0x100;\n" + + "\t\t\t\t\t_0x383663 = _0x3e7a99[_0x3091fb];\n" + + "\t\t\t\t\t_0x3e7a99[_0x3091fb] = _0x3e7a99[_0x3e2c47];\n" + + "\t\t\t\t\t_0x3e7a99[_0x3e2c47] = _0x383663;\n" + + "\t\t\t\t}\n" + + "\t\t\t\t_0x3091fb = 0x0;\n" + + "\t\t\t\t_0x3e2c47 = 0x0;\n" + + "\t\t\t\tfor (var _0x8c68bc = 0x0; _0x8c68bc < _0x5dd34b['length']; _0x8c68bc++) {\n" + + "\t\t\t\t\t_0x3091fb = (_0x3091fb + 0x1) % 0x100;\n" + + "\t\t\t\t\t_0x3e2c47 = (_0x3e2c47 + _0x3e7a99[_0x3091fb]) % 0x100;\n" + + "\t\t\t\t\t_0x383663 = _0x3e7a99[_0x3091fb];\n" + + "\t\t\t\t\t_0x3e7a99[_0x3091fb] = _0x3e7a99[_0x3e2c47];\n" + + "\t\t\t\t\t_0x3e7a99[_0x3e2c47] = _0x383663;\n" + + "\t\t\t\t\t_0x14720e += String['fromCharCode'](_0x5dd34b['charCodeAt'](_0x8c68bc) ^ _0x3e7a99[(_0x3e7a99[_0x3091fb] + _0x3e7a99[_0x3e2c47]) % 0x100]);\n" + + "\t\t\t\t}\n" + + "\t\t\t\treturn _0x14720e;\n" + + "\t\t\t};\n" + + "\t\t\t_0x1533['sMWqUz'] = _0x38a7ff;\n" + + "\t\t\t_0x1533['AbOzCI'] = {};\n" + + "\t\t\t_0x1533['GlBvhA'] = !![];\n" + + "\t\t}\n" + + "\t\tvar _0x5b499f = _0x1533['AbOzCI'][_0x44c192];\n" + + "\t\tif (_0x5b499f === undefined) {\n" + + "\t\t\tif (_0x1533['uRkTAI'] === undefined) {\n" + + "\t\t\t\t_0x1533['uRkTAI'] = !![];\n" + + "\t\t\t}\n" + + "\t\t\t_0x1533fb = _0x1533['sMWqUz'](_0x1533fb, _0xbe8c61);\n" + + "\t\t\t_0x1533['AbOzCI'][_0x44c192] = _0x1533fb;\n" + + "\t\t} else {\n" + + "\t\t\t_0x1533fb = _0x5b499f;\n" + + "\t\t}\n" + + "\t\treturn _0x1533fb;\n" + + "\t};\n" + + "\tfunction hash(_0x1a5a87) {\n" + + "\t\tvar _0x23a3e0 = {};\n" + + "\t\t_0x23a3e0[_0x1533('0x93', 'Kp2R') + 'J'] = function(_0x5ebeb2, _0x4681fc) {\n" + + "\t\t\treturn _0x5ebeb2 ^ _0x4681fc;\n" + + "\t\t};\n" + + "\t\t_0x23a3e0[_0x1533('0x17', '(bZ%') + 'H'] = function(_0x5108c3, _0x49fe65) {\n" + + "\t\t\treturn _0x5108c3 & _0x49fe65;\n" + + "\t\t};\n" + + "\t\t_0x23a3e0[_0x1533('0x88', 'N#(C') + 'T'] = function(_0x3d3f10, _0x3f2a86) {\n" + + "\t\t\treturn _0x3d3f10 !== _0x3f2a86;\n" + + "\t\t};\n" + + "\t\t_0x23a3e0[_0x1533('0x60', '(cNt') + 'Z'] = _0x1533('0x70', 'K)PG') + 'J';\n" + + "\t\t_0x23a3e0[_0x1533('0x49', 'Y8u@') + 'h'] = function(_0xc3ff68, _0x129ab0) {\n" + + "\t\t\treturn _0xc3ff68 >> _0x129ab0;\n" + + "\t\t};\n" + + "\t\t_0x23a3e0[_0x1533('0x9c', '\$qIi') + 'S'] = function(_0x591dc1, _0x158d9d) {\n" + + "\t\t\treturn _0x591dc1 + _0x158d9d;\n" + + "\t\t};\n" + + "\t\t_0x23a3e0[_0x1533('0x7f', '(bZ%') + 'f'] = function(_0xd6deb, _0x557760) {\n" + + "\t\t\treturn _0xd6deb >> _0x557760;\n" + + "\t\t};\n" + + "\t\t_0x23a3e0[_0x1533('0x4e', 'S9Bn') + 'd'] = function(_0xd05226, _0x29b6e8) {\n" + + "\t\t\treturn _0xd05226 * _0x29b6e8;\n" + + "\t\t};\n" + + "\t\t_0x23a3e0[_0x1533('0x7e', 'y^BN') + 'n'] = function(_0x899f8d, _0x3dbd26) {\n" + + "\t\t\treturn _0x899f8d < _0x3dbd26;\n" + + "\t\t};\n" + + "\t\t_0x23a3e0[_0x1533('0x9a', '(U[3') + 'e'] = function(_0x39678a, _0x37d488) {\n" + + "\t\t\treturn _0x39678a * _0x37d488;\n" + + "\t\t};\n" + + "\t\t_0x23a3e0[_0x1533('0xa3', 'sahq') + 'U'] = function(_0x4cf877, _0x14e1b0) {\n" + + "\t\t\treturn _0x4cf877 << _0x14e1b0;\n" + + "\t\t};\n" + + "\t\t_0x23a3e0[_0x1533('0xa7', 'K#Hh') + 'u'] = function(_0x556336, _0x56b41c) {\n" + + "\t\t\treturn _0x556336 - _0x56b41c;\n" + + "\t\t};\n" + + "\t\t_0x23a3e0[_0x1533('0x3d', '(bZ%') + 'p'] = function(_0x372c20, _0x534fe1) {\n" + + "\t\t\treturn _0x372c20 >> _0x534fe1;\n" + + "\t\t};\n" + + "\t\t_0x23a3e0[_0x1533('0x1d', 'S9Bn') + 'g'] = function(_0x26a0f8, _0x4f1660) {\n" + + "\t\t\treturn _0x26a0f8 >>> _0x4f1660;\n" + + "\t\t};\n" + + "\t\t_0x23a3e0[_0x1533('0xae', ']*f@') + 'Y'] = function(_0x3423f4, _0x53e1a9) {\n" + + "\t\t\treturn _0x3423f4 | _0x53e1a9;\n" + + "\t\t};\n" + + "\t\t_0x23a3e0[_0x1533('0x26', 's4Dp') + 'P'] = function(_0x19382a, _0x220110) {\n" + + "\t\t\treturn _0x19382a | _0x220110;\n" + + "\t\t};\n" + + "\t\t_0x23a3e0[_0x1533('0x65', 's4Dp') + 'p'] = function(_0x7b95bc, _0x22e933) {\n" + + "\t\t\treturn _0x7b95bc & _0x22e933;\n" + + "\t\t};\n" + + "\t\t_0x23a3e0[_0x1533('0x0', '\$qIi') + 'V'] = function(_0x13269b, _0x227e3b) {\n" + + "\t\t\treturn _0x13269b ^ _0x227e3b;\n" + + "\t\t};\n" + + "\t\t_0x23a3e0[_0x1533('0x29', 'J21I') + 'R'] = function(_0x1ba947, _0x4b40e4) {\n" + + "\t\t\treturn _0x1ba947(_0x4b40e4);\n" + + "\t\t};\n" + + "\t\t_0x23a3e0[_0x1533('0x50', 's4Dp') + 'n'] = _0x1533('0xa', 'qRqa') + 'e';\n" + + "\t\t_0x23a3e0[_0x1533('0x42', 'ksVw') + 'b'] = function(_0x466fa6, _0x3ce8da) {\n" + + "\t\t\treturn _0x466fa6 ^ _0x3ce8da;\n" + + "\t\t};\n" + + "\t\t_0x23a3e0[_0x1533('0x2c', 'rqYb') + 'f'] = function(_0x1e608a, _0x241888) {\n" + + "\t\t\treturn _0x1e608a - _0x241888;\n" + + "\t\t};\n" + + "\t\t_0x23a3e0[_0x1533('0x78', 'J21I') + 'u'] = function(_0x564b78, _0x18d236, _0x8fd805) {\n" + + "\t\t\treturn _0x564b78(_0x18d236, _0x8fd805);\n" + + "\t\t};\n" + + "\t\t_0x23a3e0[_0x1533('0x69', 'zx(Q') + 'D'] = function(_0x3b7666, _0x47f238, _0x57b05e) {\n" + + "\t\t\treturn _0x3b7666(_0x47f238, _0x57b05e);\n" + + "\t\t};\n" + + "\t\t_0x23a3e0[_0x1533('0x5f', 'ZaCV') + 'm'] = function(_0x452fa7, _0x493d76, _0x1446e7, _0x4af814, _0x5e0f59) {\n" + + "\t\t\treturn _0x452fa7(_0x493d76, _0x1446e7, _0x4af814, _0x5e0f59);\n" + + "\t\t};\n" + + "\t\t_0x23a3e0[_0x1533('0x22', 'Rh@#') + 'W'] = function(_0x51a858, _0x440b1c) {\n" + + "\t\t\treturn _0x51a858(_0x440b1c);\n" + + "\t\t};\n" + + "\t\t_0x23a3e0[_0x1533('0x8', 'xPGU') + 'A'] = function(_0x5e96f2, _0x167b41) {\n" + + "\t\t\treturn _0x5e96f2 + _0x167b41;\n" + + "\t\t};\n" + + "\t\t_0x23a3e0[_0x1533('0x6d', 'rqYb') + 'X'] = function(_0x46cd37, _0x5d8684) {\n" + + "\t\t\treturn _0x46cd37 + _0x5d8684;\n" + + "\t\t};\n" + + "\t\t_0x23a3e0[_0x1533('0x3e', 'rqYb') + 'g'] = function(_0x26bc4b, _0x16bc46) {\n" + + "\t\t\treturn _0x26bc4b(_0x16bc46);\n" + + "\t\t};\n" + + "\t\tvar _0x179134 = _0x23a3e0;\n" + + "\t\tfunction _0x5be8c0(_0x296dd6, _0x778566) {\n" + + "\t\t\treturn _0x179134[_0x1533('0x56', 'ZaCV') + 'J'](_0x179134[_0x1533('0x7c', 'o&Q4') + 'H'](_0x296dd6, 0x7fffffff) + (_0x778566 & 0x7fffffff), _0x179134[_0x1533('0x52', 'K)PG') + 'H'](_0x296dd6, 0x80000000)) ^ _0x778566 & 0x80000000;\n" + + "\t\t}\n" + + "\t\tfunction _0x225a28(_0x2f57af) {\n" + + "\t\t\tvar _0x3c49d8 = {};\n" + + "\t\t\t_0x3c49d8[_0x1533('0x55', 'qRqa') + 'N'] = function(_0x15bfb8, _0x51e1eb) {\n" + + "\t\t\t\treturn _0x15bfb8 < _0x51e1eb;\n" + + "\t\t\t};\n" + + "\t\t\tvar _0x567e0f = _0x3c49d8;\n" + + "\t\t\tvar _0x5b2e48 = _0x1533('0xa4', 'rQf(') + _0x1533('0x4', 'rqYb') + _0x1533('0x95', 'Rh@#') + _0x1533('0x3', 'OhQr');\n" + + "\t\t\tvar _0x321f21 = '';\n" + + "\t\t\tfor (var _0x2a5528 = 0x7; _0x2a5528 >= 0x0; _0x2a5528--) {\n" + + "\t\t\t\tif (_0x179134[_0x1533('0xb2', 'Gfc!') + 'T'](_0x1533('0x2', 'w*Nn') + 'h', _0x179134[_0x1533('0x6c', 'OhQr') + 'Z'])) {\n" + + "\t\t\t\t\t_0x321f21 += _0x5b2e48[_0x1533('0x66', ')T5g') + 'At'](_0x179134[_0x1533('0x9f', 'zx(Q') + 'H'](_0x179134[_0x1533('0x5', 'OhQr') + 'h'](_0x2f57af, _0x2a5528 * 0x4), 0xf));\n" + + "\t\t\t\t} else {\n" + + "\t\t\t\t\treturn _0x567e0f[_0x1533('0x5a', 'y*wC') + 'N'](t, 0x14) ? 0x5a827999: t < 0x28 ? 0x6ed9eba1: _0x567e0f[_0x1533('0x43', 'zx(Q') + 'N'](t, 0x3c) ? -0x70e44324: -0x359d3e2a;\n" + + "\t\t\t\t}\n" + + "\t\t\t}\n" + + "\t\t\treturn _0x321f21;\n" + + "\t\t}\n" + + "\t\tfunction _0xde630f(_0x10fb9a) {\n" + + "\t\t\tvar _0x5b5b6d = _0x179134[_0x1533('0x16', 'GvA9') + 'S'](_0x179134[_0x1533('0x87', 'N#(C') + 'f'](_0x179134[_0x1533('0x83', '[H[u') + 'S'](_0x10fb9a[_0x1533('0x58', 'OhQr') + 'th'], 0x8), 0x6), 0x1),\n" + + "\t\t\t_0x797793 = new Array(_0x179134[_0x1533('0x59', 'xPGU') + 'd'](_0x5b5b6d, 0x10));\n" + + "\t\t\tfor (var _0x109346 = 0x0; _0x179134[_0x1533('0x61', 'dgE&') + 'n'](_0x109346, _0x179134[_0x1533('0x90', 'y^BN') + 'e'](_0x5b5b6d, 0x10)); _0x109346++) {\n" + + "\t\t\t\tif (_0x179134[_0x1533('0x2a', 'NNf8') + 'T'](_0x1533('0x12', 'sahq') + 'y', _0x1533('0x67', 'y*wC') + 'G')) {\n" + + "\t\t\t\t\t_0x797793[_0x109346] = 0x0;\n" + + "\t\t\t\t} else {\n" + + "\t\t\t\t\treturn !! [];\n" + + "\t\t\t\t}\n" + + "\t\t\t}\n" + + "\t\t\tfor (_0x109346 = 0x0; _0x109346 < _0x10fb9a[_0x1533('0x40', 'ad*k') + 'th']; _0x109346++) {\n" + + "\t\t\t\t_0x797793[_0x109346 >> 0x2] |= _0x179134[_0x1533('0xa9', '@gLq') + 'U'](_0x10fb9a[_0x1533('0x1c', 'GvA9') + _0x1533('0xf', 'OhQr') + 'At'](_0x109346), _0x179134[_0x1533('0xb3', 'S9Bn') + 'u'](0x18, (_0x109346 & 0x3) * 0x8));\n" + + "\t\t\t}\n" + + "\t\t\t_0x797793[_0x179134[_0x1533('0xb1', 'qRqa') + 'p'](_0x109346, 0x2)] |= 0x80 << 0x18 - (_0x109346 & 0x3) * 0x8;\n" + + "\t\t\t_0x797793[_0x5b5b6d * 0x10 - 0x1] = _0x10fb9a[_0x1533('0xb', '@mqx') + 'th'] * 0x8;\n" + + "\t\t\treturn _0x797793;\n" + + "\t\t}\n" + + "\t\tfunction _0x50e067(_0x45a1cc, _0x3bcee1) {\n" + + "\t\t\treturn _0x45a1cc << _0x3bcee1 | _0x179134[_0x1533('0x5b', 'Y8u@') + 'g'](_0x45a1cc, _0x179134[_0x1533('0x91', '@mqx') + 'u'](0x20, _0x3bcee1));\n" + + "\t\t}\n" + + "\t\tfunction _0x526d3b(_0xb72077, _0xed7f25, _0x5bc269, _0x572b62) {\n" + + "\t\t\tif (_0xb72077 < 0x14) return _0x179134[_0x1533('0x98', 'Gfc!') + 'Y'](_0xed7f25 & _0x5bc269, _0x179134[_0x1533('0x68', 'OhQr') + 'H'](~_0xed7f25, _0x572b62));\n" + + "\t\t\tif (_0x179134[_0x1533('0x9e', 'w*Nn') + 'n'](_0xb72077, 0x28)) return _0x179134[_0x1533('0x99', ')T5g') + 'J'](_0xed7f25, _0x5bc269) ^ _0x572b62;\n" + + "\t\t\tif (_0xb72077 < 0x3c) return _0x179134[_0x1533('0x13', 'CNzQ') + 'P'](_0xed7f25 & _0x5bc269, _0x179134[_0x1533('0x45', '@mqx') + 'p'](_0xed7f25, _0x572b62)) | _0x179134[_0x1533('0xa0', 'avK7') + 'p'](_0x5bc269, _0x572b62);\n" + + "\t\t\treturn _0x179134[_0x1533('0x5e', 'avK7') + 'V'](_0xed7f25, _0x5bc269) ^ _0x572b62;\n" + + "\t\t}\n" + + "\t\tfunction _0x496785(_0x583c97) {\n" + + "\t\t\treturn _0x583c97 < 0x14 ? 0x5a827999: _0x583c97 < 0x28 ? 0x6ed9eba1: _0x583c97 < 0x3c ? -0x70e44324: -0x359d3e2a;\n" + + "\t\t}\n" + + "\t\tvar _0x45b840 = _0x179134[_0x1533('0x39', 'R#BF') + 'R'](_0xde630f, _0x1a5a87);\n" + + "\t\tvar _0x53258d = new Array(0x50);\n" + + "\t\tvar _0xc86ce6 = 0x67452301;\n" + + "\t\tvar _0x3ee50e = -0x10325477;\n" + + "\t\tvar _0xdb1008 = -0x67452302;\n" + + "\t\tvar _0xe99bcb = 0x10325476;\n" + + "\t\tvar _0x56c860 = -0x3c2d1e10;\n" + + "\t\tfor (var _0x4c314c = 0x0; _0x4c314c < _0x45b840[_0x1533('0x46', '\$qIi') + 'th']; _0x4c314c += 0x10) {\n" + + "\t\t\tvar _0x41e8e8 = _0xc86ce6;\n" + + "\t\t\tvar _0x1b7277 = _0x3ee50e;\n" + + "\t\t\tvar _0x30f7c3 = _0xdb1008;\n" + + "\t\t\tvar _0x279c95 = _0xe99bcb;\n" + + "\t\t\tvar _0xf6e888 = _0x56c860;\n" + + "\t\t\tfor (var _0x77e31b = 0x0; _0x179134[_0x1533('0x34', 'Y8u@') + 'n'](_0x77e31b, 0x50); _0x77e31b++) {\n" + + "\t\t\t\tif (_0x77e31b < 0x10) {\n" + + "\t\t\t\t\tif (_0x179134[_0x1533('0xa6', 'Mifp') + 'T'](_0x1533('0x3f', 'o&Q4') + 'G', _0x179134[_0x1533('0xa1', 'rqYb') + 'n'])) {\n" + + "\t\t\t\t\t\t_0x53258d[_0x77e31b] = _0x45b840[_0x4c314c + _0x77e31b];\n" + + "\t\t\t\t\t} else {\n" + + "\t\t\t\t\t\tsleepT = _0x179134[_0x1533('0x48', 'NNf8') + 'R'](parseInt, data['wt']) > ret[0x1] ? _0x179134[_0x1533('0x97', 'd[i)') + 'R'](parseInt, data['wt']) - ret[0x1] : 0x1f4;\n" + + "\t\t\t\t\t}\n" + + "\t\t\t\t} else {\n" + + "\t\t\t\t\t_0x53258d[_0x77e31b] = _0x50e067(_0x179134[_0x1533('0x96', 'rQf(') + 'b'](_0x179134[_0x1533('0xb0', 'Gfc!') + 'b'](_0x53258d[_0x179134[_0x1533('0xa5', 'K)PG') + 'u'](_0x77e31b, 0x3)], _0x53258d[_0x179134[_0x1533('0x51', 'K)PG') + 'f'](_0x77e31b, 0x8)]) ^ _0x53258d[_0x77e31b - 0xe], _0x53258d[_0x179134[_0x1533('0xaf', '(U[3') + 'f'](_0x77e31b, 0x10)]), 0x1);\n" + + "\t\t\t\t}\n" + + "\t\t\t\tt = _0x179134[_0x1533('0x18', 'GvA9') + 'u'](_0x5be8c0, _0x179134[_0x1533('0x23', '\$qIi') + 'u'](_0x5be8c0, _0x179134[_0x1533('0x8d', 'Mifp') + 'D'](_0x50e067, _0xc86ce6, 0x5), _0x179134[_0x1533('0x4c', 'xPGU') + 'm'](_0x526d3b, _0x77e31b, _0x3ee50e, _0xdb1008, _0xe99bcb)), _0x5be8c0(_0x5be8c0(_0x56c860, _0x53258d[_0x77e31b]), _0x179134[_0x1533('0xad', 'rqYb') + 'W'](_0x496785, _0x77e31b)));\n" + + "\t\t\t\t_0x56c860 = _0xe99bcb;\n" + + "\t\t\t\t_0xe99bcb = _0xdb1008;\n" + + "\t\t\t\t_0xdb1008 = _0x50e067(_0x3ee50e, 0x1e);\n" + + "\t\t\t\t_0x3ee50e = _0xc86ce6;\n" + + "\t\t\t\t_0xc86ce6 = t;\n" + + "\t\t\t}\n" + + "\t\t\t_0xc86ce6 = _0x5be8c0(_0xc86ce6, _0x41e8e8);\n" + + "\t\t\t_0x3ee50e = _0x5be8c0(_0x3ee50e, _0x1b7277);\n" + + "\t\t\t_0xdb1008 = _0x5be8c0(_0xdb1008, _0x30f7c3);\n" + + "\t\t\t_0xe99bcb = _0x5be8c0(_0xe99bcb, _0x279c95);\n" + + "\t\t\t_0x56c860 = _0x5be8c0(_0x56c860, _0xf6e888);\n" + + "\t\t}\n" + + "\t\treturn _0x179134[_0x1533('0x86', 'Gfc!') + 'A'](_0x179134[_0x1533('0x44', 'DDSy') + 'X'](_0x179134[_0x1533('0x8e', 'xPGU') + 'g'](_0x225a28, _0xc86ce6), _0x179134[_0x1533('0x77', 'n*Rs') + 'g'](_0x225a28, _0x3ee50e)) + _0x225a28(_0xdb1008) + _0x225a28(_0xe99bcb), _0x179134[_0x1533('0xd', 'DDSy') + 'g'](_0x225a28, _0x56c860));\n" + + "\t}\n" + + "\tfunction go(_0x394549) {\n" + + "\t\tvar _0x33d0d6 = {};\n" + + "\t\t_0x33d0d6[_0x1533('0x2b', 'sahq') + 'H'] = _0x1533('0x14', 'rQf(') + _0x1533('0x15', 'sahq');\n" + + "\t\t_0x33d0d6[_0x1533('0xab', 'dgE&') + 'H'] = function(_0x23f209, _0x4ed415) {\n" + + "\t\t\treturn _0x23f209 < _0x4ed415;\n" + + "\t\t};\n" + + "\t\t_0x33d0d6[_0x1533('0x5c', 'xPGU') + 'g'] = function(_0x39bd0a, _0x4d5764) {\n" + + "\t\t\treturn _0x39bd0a != _0x4d5764;\n" + + "\t\t};\n" + + "\t\t_0x33d0d6[_0x1533('0x37', 'y*wC') + 'N'] = function(_0x3c9a26, _0x109b89) {\n" + + "\t\t\treturn _0x3c9a26 + _0x109b89;\n" + + "\t\t};\n" + + "\t\t_0x33d0d6[_0x1533('0x63', 'gkl*') + 'z'] = function(_0x584935, _0x58347c) {\n" + + "\t\t\treturn _0x584935 + _0x58347c;\n" + + "\t\t};\n" + + "\t\t_0x33d0d6[_0x1533('0x21', 'ZaCV') + 'z'] = function(_0x3fc795, _0x6962f6) {\n" + + "\t\t\treturn _0x3fc795 + _0x6962f6;\n" + + "\t\t};\n" + + "\t\t_0x33d0d6[_0x1533('0x27', '(cNt') + 'I'] = function(_0x303446, _0x3388fb) {\n" + + "\t\t\treturn _0x303446 == _0x3388fb;\n" + + "\t\t};\n" + + "\t\t_0x33d0d6[_0x1533('0x6b', 'y*wC') + 't'] = function(_0x5cb515, _0x31e7f3) {\n" + + "\t\t\treturn _0x5cb515(_0x31e7f3);\n" + + "\t\t};\n" + + "\t\t_0x33d0d6[_0x1533('0x75', 'Rh@#') + 'E'] = function(_0x529d8e, _0xc1423a) {\n" + + "\t\t\treturn _0x529d8e + _0xc1423a;\n" + + "\t\t};\n" + + "\t\t_0x33d0d6[_0x1533('0x62', 'rQf(') + 'a'] = function(_0x453505, _0x50cd56) {\n" + + "\t\t\treturn _0x453505 + _0x50cd56;\n" + + "\t\t};\n" + + "\t\t_0x33d0d6[_0x1533('0x2e', 'NNf8') + 'B'] = _0x1533('0x28', ']*f@') + _0x1533('0x57', 'Kp2R') + '\\x20/';\n" + + "\t\t_0x33d0d6[_0x1533('0xe', 'Y8u@') + 'y'] = function(_0x25bcfd, _0x4b634e, _0x1bef22) {\n" + + "\t\t\treturn _0x25bcfd(_0x4b634e, _0x1bef22);\n" + + "\t\t};\n" + + "\t\t_0x33d0d6[_0x1533('0x92', 'J21I') + 'i'] = function(_0xccfc4d, _0x9a8620) {\n" + + "\t\t\treturn _0xccfc4d > _0x9a8620;\n" + + "\t\t};\n" + + "\t\t_0x33d0d6[_0x1533('0x11', '@H8e') + 'n'] = function(_0x1b9fc7, _0x7d3ce) {\n" + + "\t\t\treturn _0x1b9fc7(_0x7d3ce);\n" + + "\t\t};\n" + + "\t\t_0x33d0d6[_0x1533('0x1b', 'N#(C') + 'O'] = _0x1533('0x7', 'N#(C') + '失败';\n" + + "\t\tvar _0x2eda7d = _0x33d0d6;\n" + + "\t\tfunction _0x29e502() {\n" + + "\t\t\tvar _0x24a3c1 = window[_0x1533('0x35', 'ad*k') + _0x1533('0x5d', 'Y8u@') + 'r'][_0x1533('0x19', 'avK7') + _0x1533('0x7d', 'cJ3U') + 't'],\n" + + "\t\t\t_0x5f1192 = [_0x2eda7d[_0x1533('0x24', '(bZ%') + 'H']];\n" + + "\t\t\tfor (var _0x2d8686 = 0x0; _0x2eda7d[_0x1533('0x1a', 'avK7') + 'H'](_0x2d8686, _0x5f1192[_0x1533('0x3b', 'Y8u@') + 'th']); _0x2d8686++) {\n" + + "\t\t\t\tif (_0x2eda7d[_0x1533('0x2f', 'y*wC') + 'g'](_0x24a3c1[_0x1533('0x76', 'y*wC') + _0x1533('0x33', 'ZaCV')](_0x5f1192[_0x2d8686]), -0x1)) {\n" + + "\t\t\t\t\treturn !! [];\n" + + "\t\t\t\t}\n" + + "\t\t\t}\n" + + "\t\t\tif (window[_0x1533('0x41', 'dgE&') + _0x1533('0x6e', 'dgE&') + _0x1533('0x85', 'NNf8')] || window[_0x1533('0x30', 'o&Q4') + _0x1533('0x38', 'OhQr')] || window[_0x1533('0xc', 'OhQr') + _0x1533('0x74', 'qRqa')] || window[_0x1533('0x25', '(bZ%') + _0x1533('0x4f', 'avK7') + 'r'][_0x1533('0xa2', 'xPGU') + _0x1533('0x53', ')T5g') + 'r'] || window[_0x1533('0xaa', 'cJ3U') + _0x1533('0x6', 'K#@S') + 'r'][_0x1533('0x8f', 'rqYb') + _0x1533('0xac', '(bZ%') + _0x1533('0x9', 'Rh@#') + _0x1533('0x4d', 'rQf(') + 'e'] || window[_0x1533('0x31', 'N#(C') + _0x1533('0x79', 'dgE&') + 'r'][_0x1533('0x4b', 'ksVw') + _0x1533('0x10', 'gkl*') + _0x1533('0xa8', 'd[i)') + _0x1533('0x89', '\$qIi') + _0x1533('0x84', 's4Dp')]) {\n" + + "\t\t\t\treturn !! [];\n" + + "\t\t\t}\n" + + "\t\t};\n" + + "\t\tif (_0x29e502()) {\n" + + "\t\t\treturn;\n" + + "\t\t}\n" + + "\t\tvar _0x1642ef = new Date();\n" + + "\t\tfunction _0x1a1fd9(_0x55baea, _0x579edb) {\n" + + "\t\t\tvar _0x551ac6 = _0x394549[_0x1533('0x4a', 'y*wC') + 's'][_0x1533('0x71', 'K#@S') + 'th'];\n" + + "\t\t\tfor (var _0x54ccec = 0x0; _0x2eda7d[_0x1533('0x6f', 'dFiI') + 'H'](_0x54ccec, _0x551ac6); _0x54ccec++) {\n" + + "\t\t\t\tfor (var _0x562c2d = 0x0; _0x2eda7d[_0x1533('0x9b', '[H[u') + 'H'](_0x562c2d, _0x551ac6); _0x562c2d++) {\n" + + "\t\t\t\t\tvar _0x1b8d8c = _0x2eda7d[_0x1533('0x80', '[H[u') + 'N'](_0x2eda7d[_0x1533('0x3a', 'w*Nn') + 'z'](_0x2eda7d[_0x1533('0x8b', 'cJ3U') + 'z'](_0x579edb[0x0], _0x394549[_0x1533('0x32', 'R#BF') + 's'][_0x1533('0x82', '@mqx') + 'tr'](_0x54ccec, 0x1)), _0x394549[_0x1533('0x1f', '*hmq') + 's'][_0x1533('0x47', '(cNt') + 'tr'](_0x562c2d, 0x1)), _0x579edb[0x1]);\n" + + "\t\t\t\t\tif (_0x2eda7d[_0x1533('0x6a', '(bZ%') + 'I'](_0x2eda7d[_0x1533('0x9d', 'Kp2R') + 't'](hash, _0x1b8d8c), _0x55baea)) {\n" + + "\t\t\t\t\t\treturn [_0x1b8d8c, new Date() - _0x1642ef];\n" + + "\t\t\t\t\t}\n" + + "\t\t\t\t}\n" + + "\t\t\t}\n" + + "\t\t};\n" + + "\t\tvar _0x1e73b9 = _0x2eda7d[_0x1533('0x36', 'sahq') + 'y'](_0x1a1fd9, _0x394549['ct'], _0x394549[_0x1533('0x2d', '\$qIi')]);\n" + + "\t\tif (_0x1e73b9) {\n" + + "\t\t\tvar _0x3b1427;\n" + + "\t\t\tif (_0x394549['wt']) {\n" + + "\t\t\t\t_0x3b1427 = _0x2eda7d[_0x1533('0x1', 'zx(Q') + 'i'](parseInt(_0x394549['wt']), _0x1e73b9[0x1]) ? _0x2eda7d[_0x1533('0x54', 'avK7') + 'n'](parseInt, _0x394549['wt']) - _0x1e73b9[0x1] : 0x1f4;\n" + + "\t\t\t} else {\n" + + "\t\t\t\t_0x3b1427 = 0x5dc;\n" + + "\t\t\t}\n" + + "\t\t\t_0x2eda7d[_0x1533('0x72', 'qRqa') + 'y'](setTimeout,\n" + + "\t\t\tfunction() {\n" + + "\t\t\t\tdocument[_0x1533('0x20', 'rQf(') + 'ie'] = _0x2eda7d[_0x1533('0x73', 'rQf(') + 'E'](_0x2eda7d[_0x1533('0x3c', 'o&Q4') + 'a'](_0x394549['tn'] + '=' + _0x1e73b9[0x0] + (_0x1533('0x8a', 'ksVw') + _0x1533('0x1e', 'GvA9') + '='), _0x394549['vt']), _0x2eda7d[_0x1533('0x8c', 'Gfc!') + 'B']);\n" + + "\t\t\t\tlocation[_0x1533('0x7a', 'sahq')] = location[_0x1533('0x64', 'd[i)') + _0x1533('0x81', '(U[3')] + location[_0x1533('0x94', 'sahq') + 'ch'];\n" + + "\t\t\t},\n" + + "\t\t\t_0x3b1427);\n" + + "\t\t} else {\n" + + "\t\t\talert(_0x2eda7d[_0x1533('0x7b', ')T5g') + 'O']);\n" + + "\t\t}\n" + + "\t};\n" + + "\tgo({\n" + + "\t\t\"bts\": [\"1618915727.945|0|HM5\", \"VoVNIPqcrvFO13woIZK2RY%3D\"],\n" + + "\t\t\"chars\": \"ZDUZR0cDjTEXznMkfgavFO\",\n" + + "\t\t\"ct\": \"3049e28877124ad21ca0db55e3af95ff316ceef6\",\n" + + "\t\t\"ha\": \"sha1\",\n" + + "\t\t\"tn\": \"__jsl_clearance\",\n" + + "\t\t\"vt\": \"3600\",\n" + + "\t\t\"wt\": \"1500\"\n" + + "\t})" + + ScriptEngineManager manager = new ScriptEngineManager(); + ScriptEngine engine = manager.getEngineByName("javascript"); + engine.eval(str); + //获取对象 + System.out.println(); + + } + + + + def getJsContent(def page){ + ScriptEngineManager manager = new ScriptEngineManager(); +//得到脚本引擎 + ScriptEngine engine = manager.getEngineByName("javascript"); +//处理加密js + String js = page.trim().replace("", "").replace("eval(y.replace(/\\b\\w+\\b/g, function(y){return x[f(y,z)-1]}));", "y.replace(/\\b\\w+\\b/g, function(y){return x[f(y,z)-1]})"); + try { + //得到解密后的js + js = js.replace("document.cookie","var cookie") + js = js.replace("location.href=location.pathname+location.search", "") + engine.eval(js); + def v = engine.get("cookie") + return v.toString().substring(0, v.toString().indexOf(";")) + } catch (ScriptException e) { + e.printStackTrace(); + } + return null + + } + + + def getJsContent2(def page){ + ScriptEngineManager manager = new ScriptEngineManager(); +//得到脚本引擎 + ScriptEngine engine = manager.getEngineByName("javascript"); +//处理加密js + String js = page.trim().replace("", "").replace("eval(y.replace(/\\b\\w+\\b/g, function(y){return x[f(y,z)-1]}));", "y.replace(/\\b\\w+\\b/g, function(y){return x[f(y,z)-1]})"); + try { + //得到解密后的js + //得到解密后的js + String result = (String) engine.eval(js); + result = result.substring(result.indexOf("var cd"),result.indexOf("dc+=cd;")+7); + result = result.replaceAll("document*.*toLowerCase\\(\\)", "'x'"); + //得到cookie + String jsl = (String) engine.eval(result); + return jsl; + } catch (ScriptException e) { + e.printStackTrace(); + } + return null + + } + + + /** + * 通过破解动态JavaScript脚本, + * 获取cookie名为 __jsl_clearance的值 + * + * @param body 相应内容(一般为第一次请求获取到的动态js字符串) + * @return cookie名为 __jsl_clearance的值 + */ + private static String getJslClearance(String body) { + //V8:谷歌开源的运行JavaScript脚本的库. 参数:globalAlias=window, 表示window为全局别名, + // 告诉V8在运行JavaScript代码时, 不要从代码里找window的定义. + V8 runtime = V8.createV8Runtime("window"); + //将第一次请求pdf资源时获取到的字符串提取成V8可执行的JavaScript代码 + body = body.trim() + .replace("", "") + .replace("eval(y.replace(/\\b\\w+\\b/g, function(y){return x[f(y,z)-1]||(\"_\"+y)}))", + "y.replace(/\\b\\w+\\b/g, function(y){return x[f(y,z)-1]||(\"_\"+y)})"); + //用V8执行该段代码获取新的动态JavaScript脚本 + String result = runtime.executeStringScript(body); + + //获取 jsl_clearance 的第一段, 格式形如: 1543915851.312|0| + String startStr = "document.cookie='"; + int i1 = result.indexOf(startStr) + startStr.length(); + int i2 = result.indexOf("|0|"); + String cookie1 = result.substring(i1, i2 + 3); + /* + 获取 jsl_clearance 的第二段,格式形如: DW2jqgJO5Bo45yYRKLlFbnqQuD0%3D。 + 主要原理是: 新的动态JavaScript脚本是为浏览器设置cookie, 且cookie名为__jsl_clearance + 其中第一段值(格式形如:1543915851.312|0|)已经明文写好, 用字符串处理方法即可获取. + 第二段则是一段JavaScript函数, 需要有V8运行返回, + 该函数代码需要通过一些字符串定位, 提取出来, 交给V8运行. + */ + startStr = "|0|'+(function(){"; + int i3 = result.indexOf(startStr) + startStr.length(); + int i4 = result.indexOf("})()+';Expires"); + String code = result.substring(i3, i4).replace(";return", ";"); + String cookie2 = runtime.executeStringScript(code); + + /* + 拼接两段字符串, 返回jsl_clearance的完整的值. + 格式形如: 1543915851.312|0|DW2jqgJO5Bo45yYRKLlFbnqQuD0%3D + */ + return cookie1 + cookie2; + } + + + /** + * 将HTTP响应体转换为字符串返回 + * + * @param response HTTP响应 + * @return 响应体的字符串形式 + * @throws IOException IO异常 + */ + private static String getResponseBodyAsString(HttpResponse response) throws IOException { + return IOUtils.toString(response.getEntity().getContent(), "UTF-8"); + } + + + + static String getJsluid(HttpResponse response) { + Header header = response.getFirstHeader("set-cookie"); + String[] split = header.getValue().split(";"); + for (String s : split) { + if (s.contains("__jsluid")) { + return s.trim(); + } + } + return ""; + } + + + def fristCrawl(){ + def responseMap = [:] + try { + //headers信息 + Map headers = new LinkedHashMap() + headers.put("Host","www.gsxt.gov.cn") + headers.put("Connection","keep-alive") + headers.put("Upgrade-Insecure-Requests","1") + headers.put("Cache-Control","max-age=0") +// headers.put("Content-Type","application/x-www-form-urlencoded") + headers.put("User-Agent","Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.104 Safari/537.36") + headers.put("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9") +// headers.put("Cookie","__jsluid_h=b4ff8b9b0fd86b6ff99edf27e111a9f2") + headers.put("Accept-Language","zh-CN,zh;q=0.9") +// headers.put("Accept-Encoding","gzip, deflate") + + //String s = doPost("http://www.gsxt.gov.cn/corp-query-homepage.html", null, headers) + def s = ABuYunUtil.doGet("http://www.gsxt.gov.cn/corp-query-homepage.html",null, headers, "utf-8") + return s + } catch (Exception e) { + e.printStackTrace() + } + return null + } + + + void process(Object args) { + + def url = "http://www.gsxt.gov.cn/corp-query-search-1.html " + for(def key : companyKeys){ + // def queryBody = "tab=ent_tab&province=&geetest_challenge=1f31bf4565a8f89b93f4f4b25bb2143a&geetest_validate=9584ff51b3aff476fcac55248ec14459&geetest_seccode=9584ff51b3aff476fcac55248ec14459%7Cjordan&token=115406527&searchword="+ URLEncoder.encode("宏美", "utf-8") + def queryBody = "tab=ent_tab&province=&geetest_challenge=daa40fb7efec814ff072b12ee12bb25a&geetest_validate=009a8748b31392ffdfb626fbbdccb4be&geetest_seccode=009a8748b31392ffdfb626fbbdccb4be%7Cjordan&token=115394764&searchword="+ URLEncoder.encode(key, "utf-8") + def s = crawl(url, queryBody); + if(s){ + print s + parser(s); + + } + println s + } + } + + def parser(content){ + def document = Jsoup.parse(content); + if(document){ + def detailUrls = [] + Elements elements = JQueryUtils.select([".search_list_item"], document) + if(elements && elements.size()>0){ + for(def el : elements){ + detailUrls << el.attr("href") + } + } + + + } + } + + def static crawl(url, bodyquery){ + def responseMap = [:] + try { + //headers信息 + Map headers = new LinkedHashMap() + headers.put("Host","www.gsxt.gov.cn") + headers.put("Connection","keep-alive") + headers.put("Content-Length","226") + headers.put("Cache-Control","max-age=0") + headers.put("Upgrade-Insecure-Requests","1") + headers.put("Origin","http://www.gsxt.gov.cn") + headers.put("Content-Type","application/x-www-form-urlencoded") + headers.put("User-Agent","Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.104 Safari/537.36") + headers.put("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9") + headers.put("Referer","http://www.gsxt.gov.cn/corp-query-homepage.html") +// headers.put("Accept-Encoding","gzip, deflate") + headers.put("Accept-Language","zh-CN,zh;q=0.9") + +// headers.put("Cookie:"," __jsluid_h=4e2326174f7f9d90ad9ac1f92301f2c9; JSESSIONID=8ac4f9ab894733d7c98c9a3d5c26; SECTOKEN=7446451709198207648; __jsl_clearance=1618823989.806|0|jQc7QRg%2B%2FzcIlz44iNwQg0PjZJg%3D; gsxtBrowseHistory1=%0FS%04%06%1D%04%1D%10SNS%24%26%3B%22%3D%3A71%3A%3B01%3A%219FFDDDDFFDFE%40DDDDEFDECDLE%40L%40L%40GSXS%11%1A%00%1A%15%19%11SNS%E5%91%BD%E6%9F%A3%E5%B9%B6%E6%97%84%E8%AE%AE%E7%AF%95%E5%BA%BE%E5%BA%8E%E8%AF%8A%E7%AF%95%E7%91%B2%E6%9D%BD%E9%98%A4%E5%84%98%E5%8E%8CSXS%11%1A%00%00%0D%04%11SNEEMDXS%02%1D%07%1D%00%00%1D%19%11SNEBELLFGCDL%40FG%09; tlb_cookie=S172.16.12.115") + + + String s = doPost(url, bodyquery, headers) + return s + } catch (Exception e) { + e.printStackTrace() + } + return null + } + + + static String doPost(String strUrl, String content, Map headers) { + String result = ""; + OutputStream outputStream = null; + InputStream inputStream = null; + StringBuilder sb = new StringBuilder(); + try { + checkAbuyun(true); + + def responseMap = [:] + // 创建代理服务器地址对象 + InetSocketAddress addr = new InetSocketAddress(proxyServer, proxyPort); + // 创建HTTP类型代理对象 + Proxy proxy = new Proxy(Proxy.Type.HTTP, addr); + + URL url = new URL(strUrl); + //通过调用url.openConnection()来获得一个新的URLConnection对象,并且将其结果强制转换为HttpURLConnection. + HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection(); + urlConnection.setRequestMethod("POST"); + //设置连接的超时值为30000毫秒,超时将抛出SocketTimeoutException异常 + urlConnection.setConnectTimeout(30000); + //设置读取的超时值为30000毫秒,超时将抛出SocketTimeoutException异常 + urlConnection.setReadTimeout(30000); + //将url连接用于输出,这样才能使用getOutputStream()。getOutputStream()返回的输出流用于传输数据 + urlConnection.setDoOutput(true); + urlConnection.setDoInput(true); + //设置通用请求属性为默认浏览器编码类型 +// urlConnection.setRequestProperty("content-type", "application/x-www-form-urlencoded"); + + + if (headers != null && headers.keySet().size() > 0) { + for (String head : headers.keySet()) { + urlConnection.setRequestProperty(head, headers.get(head)); + } + } + //getOutputStream()返回的输出流,用于写入参数数据。 + outputStream = urlConnection.getOutputStream(); + if(content){ + outputStream.write(content.getBytes()); + } + outputStream.flush(); + outputStream.close(); + //此时将调用接口方法。getInputStream()返回的输入流可以读取返回的数据。 + inputStream = urlConnection.getInputStream(); + + + InputStreamReader reader = new InputStreamReader(inputStream, "UTF-8"); + char[] buff = new char[1024]; + int length = 0; + while ((length = reader.read(buff)) != -1) { + String s = new String(buff, 0, length); + sb.append(s); + } + } catch (IOException e) { + e.printStackTrace(); + }finally{ + if(outputStream!=null){ + outputStream.close(); + } + if(inputStream!=null){ + inputStream.close(); + } + } + + return sb.toString(); + } + + public static checkAbuyun(boolean check) { + if (check) { + while (AbuyunRequestCount.get() >= AbuyunMaxCount) { + try { + Thread.sleep(5000); + } catch (InterruptedException e) { + e.printStackTrace() + } + } + AbuyunRequestCount.incrementAndGet(); + } else { + AbuyunRequestCount.decrementAndGet(); + } + } + + + + + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/dealPrice/DealPriceMathBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/dealPrice/DealPriceMathBatch.groovy index b3be6c694..d9e3c9364 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/dealPrice/DealPriceMathBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/dealPrice/DealPriceMathBatch.groovy @@ -61,10 +61,10 @@ class DealPriceMathBatch extends BaseBatch{ districtMap.putAll(detailMap) } } - def startDate = DateUtil.getFirstDayOfMonth(DateUtil.parseDate("2018-11-01","yyyy-MM-dd")) + def startDate = DateUtil.getFirstDayOfMonth(DateUtil.parseDate("2020-01-01","yyyy-MM-dd")) // def startDate = DateUtil.getFirstDayOfMonth(new Date()) // def endDate = DateUtil.getFirstDayOfMonth(new Date(), 1) - def endDate = DateUtil.getFirstDayOfMonth(DateUtil.parseDate("2019-12-31","yyyy-MM-dd")) + def endDate = DateUtil.getFirstDayOfMonth(DateUtil.parseDate("2021-04-30","yyyy-MM-dd")) while(flag){ println "开始加载解析数据......" @@ -116,7 +116,7 @@ class DealPriceMathBatch extends BaseBatch{ count++; }catch (Exception e){ - println "处理:【${dbo.city}-${dbo.region}-${dbo.district_name}】发生异常,异常信息为:"+e.getMessage() + println "处理:【${dbo.city}-${dbo.region}-${dbo.district_name}】发生异常,异常信息为:"+e.printStackTrace() }finally { latch.countDown() } @@ -442,16 +442,21 @@ class DealPriceMathBatch extends BaseBatch{ if(regionMap){ regionMap.each{k,v-> def arrays = k.split("###") - if(arrays[1] && arrays[1]==keyword){ - districts << [city:city, region:region, name:arrays[0],address:arrays[1]] + try{ + if(arrays && arrays.size()>=2 && arrays[1] && arrays[1]==keyword){ + districts << [city:city, region:region, name:arrays[0],address:arrays[1]] + } + }catch(Exception e){ + e.printStackTrace() } + } } }else{//无区域 cityMap.each{r,dMap-> dMap.each{k,v-> def arrays = k.split("###") - if(arrays[1] && arrays[1]==keyword){ + if(arrays && arrays.size()>=2 && arrays[1] && arrays[1]==keyword){ districts << [city:city, region:r, name:arrays[0],address:arrays[1]] } } @@ -596,10 +601,13 @@ class DealPriceMathBatch extends BaseBatch{ } def getOpenApiMatch(def city, def region, def name){ - def jsonObject = OpenApiClient.addressMatch(city, name, "property", false, region, "fuzzy", false) - if(jsonObject && jsonObject.credit>0 ){ - return [city:jsonObject.city, region:jsonObject.district, name:jsonObject.name, address:jsonObject.address] + if(name){ + def jsonObject = OpenApiClient.addressMatch(city, name, "property", false, region, null,"fuzzy", false) + if(jsonObject && jsonObject.credit>0 ){ + return [city:jsonObject.city, region:jsonObject.district, name:jsonObject.name, address:jsonObject.address] + } } + return null; } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fangjiatong/AbcEvaluationRowsBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fangjiatong/AbcEvaluationRowsBatch.groovy index cbf7ef9e6..f1f8918e6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fangjiatong/AbcEvaluationRowsBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fangjiatong/AbcEvaluationRowsBatch.groovy @@ -121,15 +121,15 @@ class AbcEvaluationRowsBatch extends BaseBatch{ houseLoudng.fitment_cost = ret.fitment_charge houseLoudng.car_port = ret.carport houseLoudng.house_height = ret.house_hight - houseLoudng.worry_facility" : null, - houseLoudng.fitment_age = ret.fitment_age - houseLoudng.lighting_ventilation" : null, - houseLoudng.build_type = ret.build_type - houseLoudng.house_property" : null, - houseLoudng.noise" : null, - houseLoudng.sight" : null, - houseLoudng.fitment_rank" : null, - houseLoudng.sideset" : null, +// houseLoudng.worry_facility" : null, +// houseLoudng.fitment_age = ret.fitment_age +// houseLoudng.lighting_ventilation" : null, +// houseLoudng.build_type = ret.build_type +// houseLoudng.house_property" : null, +// houseLoudng.noise" : null, +// houseLoudng.sight" : null, +// houseLoudng.fitment_rank" : null, +// houseLoudng.sideset" : null, houseLoudng.district_no = district._id diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/HouseExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/HouseExportBatch.groovy index 0ff84d43e..24a397d3e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/HouseExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/HouseExportBatch.groovy @@ -352,7 +352,7 @@ class HouseExportBatch extends BaseBatch{ def rets = [] - def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-2020*", queryBuilder); + def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "latest_seawater", queryBuilder); if(houses){ for (int k=0;k0){ + results << dist + } + } + ExcelUtils.write("D:\\数据\\建行银行\\穆良书\\建行10个城市价格数据2.xlsx",results,"数据") + } def run(){ @@ -61,6 +154,189 @@ class EsHouseStat { } + def szRentHouseStat(){ + def citys = ["上海","北京","广州","深圳","杭州","南京","天津","成都","西安","沈阳","重庆","苏州","武汉","长沙","青岛","厦门","宁波","大连","无锡"] +// def citys = ["长沙"] + def rets = [] + rets << ["城市","小区数量","2020-08-01","2020-09-01","2020-10-01","2020-11-01","2020-12-01","2021-01-01"] + for(def city : citys){ + def month0 = findRentHouse(city, 20200801, 20210201) +// def month1 = findRentHouse(city, 20200801, 20200901) +// def month2 = findRentHouse(city, 20200901, 20201001) +// def month3 = findRentHouse(city, 20201001, 20201101) +// def month4 = findRentHouse(city, 20201101, 20201201) +// def month5 = findRentHouse(city, 20201201, 20210101) +// def month6 = findRentHouse(city, 20210101, 20210201) + + def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name","alias"]) + def map = findMatchDistrictCount(city, districts); + +// def matchCount1 = 0; +// def matchCount2 = 0; +// def matchCount3 = 0; +// def matchCount4 = 0; +// def matchCount5 = 0; +// def matchCount6 = 0; +// for(def dist : districts){ +// def nameQuery = getAllName(dist); +// def month11 = findMatchHouse(city, dist.region, nameQuery, 20200801, 20200901) +// if(month11>0){ +// matchCount1 ++ +// } +// def month21 = findMatchHouse(city, dist.region, nameQuery, 20200901, 20201001) +// if(month21>0){ +// matchCount2 ++ +// } +// def month31 = findMatchHouse(city, dist.region, nameQuery, 20201001, 20201101) +// if(month31>0){ +// matchCount3 ++ +// } +// def month41 = findMatchHouse(city, dist.region, nameQuery, 20201101, 20201201) +// if(month41>0){ +// matchCount4 ++ +// } +// def month51= findMatchHouse(city, dist.region, nameQuery, 20201201, 20210101) +// if(month51>0){ +// matchCount5 ++ +// } +// def month61 = findMatchHouse(city, dist.region, nameQuery, 20210101, 20210201) +// if(month61>0){ +// matchCount6++ +// } + +// rets << [city, districts.size(), month1+"(${map.matchCount1})",month2+"(${map.matchCount2})",month3+"(${map.matchCount3})",month4+"(${map.matchCount4})",month5+"(${map.matchCount5})",month6+"(${map.matchCount6})" ] + rets << [city, districts.size(), month0] + println city +" 完成" + } + + ExcelUtils.write("d:\\数据\\租房数据统计1.xlsx",rets, "租房统计") + } + + + + def findMatchDistrictCount(def city, def districts){ + def rets = [] + final CountDownLatch latch = new CountDownLatch(districts.size()) + def final AtomicInteger count_ = new AtomicInteger(0) + def matchCount0 = 0; + def matchCount1 = 0; + def matchCount2 = 0; + def matchCount3 = 0; + def matchCount4 = 0; + def matchCount5 = 0; + def matchCount6 = 0; + for(def i=0;i0){ + matchCount0 ++ + } +// def month11 = findMatchHouse(city, dist.region, nameQuery, 20200801, 20200901) +// if(month11>0){ +// matchCount1 ++ +// } +// def month21 = findMatchHouse(city, dist.region, nameQuery, 20200901, 20201001) +// if(month21>0){ +// matchCount2 ++ +// } +// def month31 = findMatchHouse(city, dist.region, nameQuery, 20201001, 20201101) +// if(month31>0){ +// matchCount3 ++ +// } +// def month41 = findMatchHouse(city, dist.region, nameQuery, 20201101, 20201201) +// if(month41>0){ +// matchCount4 ++ +// } +// def month51= findMatchHouse(city, dist.region, nameQuery, 20201201, 20210101) +// if(month51>0){ +// matchCount5 ++ +// } +// def month61 = findMatchHouse(city, dist.region, nameQuery, 20210101, 20210201) +// if(month61>0){ +// matchCount6++ +// } + + }catch (Exception e){ + + }finally { + latch.countDown() + } + + } + } + ) + } + latch.await() + + def map = [:] + map.matchCount0 = matchCount0 +// map.matchCount1 = matchCount1 +// map.matchCount2 = matchCount2 +// map.matchCount3 = matchCount3 +// map.matchCount4 = matchCount4 +// map.matchCount5 = matchCount5 +// map.matchCount6 = matchCount6 + return map; + } + + def findMatchHouse(def city, def region, def nameQuery, def startDate, def endDate){ + final StringBuffer sqlBuffer = new StringBuffer() + sqlBuffer.append("SELECT count(*) as count FROM \"seawater-20*\" WHERE 1=1") + sqlBuffer.append(" and city = '"+city+"'") + sqlBuffer.append(" and region = '"+region+"'") + sqlBuffer.append(" and cat = 'rent'") + sqlBuffer.append(" and p_date >="+startDate).append(" and p_date <"+endDate); + sqlBuffer.append(" and (" + nameQuery + ")"); + ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString()) + def total = 0; + while (rs.next()){ + total = rs.getInt("count") + } + return total; + } + + def getAllName(dist){ + Set aliasNames = new LinkedHashSet(); + aliasNames.add(dist.name); + if (dist != null && dist.alias != null) { + aliasNames.addAll(dist.alias); + } + + List namePars = new ArrayList(); + if(aliasNames!=null && aliasNames.size()>0) { + for(String name : aliasNames){ + if(StringUtils.isNotBlank(name) && name.trim().length() > 1){ + namePars.add("district_name ='" + name + "'"); + } + } + } + + if(namePars.size() > 0){ + return StringUtils.join(namePars, " OR ") + } + return null; + } + + def findRentHouse(def city, def startDate, def endDate){ + final StringBuffer sqlBuffer = new StringBuffer() + sqlBuffer.append("SELECT count(*) as count FROM \"seawater-20*\" WHERE 1=1") + sqlBuffer.append(" and city = '"+city+"'") + sqlBuffer.append(" and cat = 'rent'") + sqlBuffer.append(" and p_date >="+startDate).append(" and p_date <"+endDate); + ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString()) + def total = 0; + while (rs.next()){ + total = rs.getInt("count") + } + return total; + } + def findHouse(def city, def startDate, def endDate){ final StringBuffer sqlBuffer = new StringBuffer() sqlBuffer.append("SELECT count(*) as count FROM \"seawater-20*\" WHERE 1=1") @@ -116,46 +392,344 @@ class EsHouseStat { def findLogs3(){ - //QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("user_id", '1007736')).must(QueryBuilders.termQuery("request_uri", '/property/detail')) - QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("user_id", '1007713')).must(QueryBuilders.termQuery("request_uri", '/property/evaluate')).must(QueryBuilders.termQuery("code", 200)) - def rets = [] - def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "open_uri_log", queryBuilder); - def curDate = DateUtils.parseFromTightByPattern("2020-07-01", "yyyy-MM-dd") - if(houses){ - for (int k=0;k= curDate.getTime()){// && - def ret = [] - def paramMap = map.parameter_map - def param_id = map.id - ret << param_id - ret << map.company_name - ret << map.ip - ret << map.request_uri - ret << map.c_date.toString().substring(0, map.c_date.indexOf("T")) - ret << map.data_status; - ret << map.code - ret << map.msg + //QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("user_id", '1007736')).must(QueryBuilders.termQuery("request_uri", '/property/detail')) + // QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("user_id", '1007721')).must(QueryBuilders.termQuery("request_uri", '/property/evaluate')).must(QueryBuilders.termQuery("code", 200)) + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("user_id", '1007751')).must(QueryBuilders.termQuery("request_uri", '/property/evaluateForTest')).must(QueryBuilders.termQuery("parameter_map.city", '上海')) + def rets = [] + def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "open_uri_log", queryBuilder); + def curDate = DateUtils.parseFromTightByPattern("2020-07-01", "yyyy-MM-dd") + if(houses){ + for (int k=0;k= curDate.getTime() + def ret = [] + def paramMap = map.parameter_map + def param_id = map.id + ret << param_id + ret << map.company_name + ret << map.ip + ret << map.request_uri + ret << map.c_date.toString().substring(0, map.c_date.indexOf("T")) + ret << map.data_status; + ret << map.code + ret << map.msg - def paramStr = "" - paramMap.each{key,val-> - if(key!="biz_code"){ - paramStr = paramStr+" "+key+":"+val - } + def paramStr = "" + paramMap.each{key,val-> + if(key!="biz_code"){ + paramStr = paramStr+" "+key+":"+val } - ret << paramStr.toString() - - def resMap = map.res_map - ret << resMap.result.toString() - rets << ret } + ret << paramStr.toString() + def resMap = map.res_map + ret << resMap.result.toString() + rets << ret + } + + } + } + return rets; + } + + + def findLogs4(def city){ + + def time = DateUtils.parseFromTightYYYYMMDD("2021-01-19") + QueryBuilder date_range_cdate = QueryBuilders.rangeQuery("c_date").from(time.getTime()) + + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("user_id", '1007751')).must(QueryBuilders.termQuery("request_uri", '/property/evaluateForTest')).must(QueryBuilders.termQuery("parameter_map.city", city)).must(date_range_cdate) + def rets = [] + def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "open_uri_log", queryBuilder); + //"城市","楼盘名称","地址", + rets << ["城市","面积","室","厅","卫","楼高","物业类型","房龄","装修","朝向","估价时间","结果信息","城市","区域","楼盘名称","总价","单价","计算方式","房源列表量","建筑年代","物业类型","距离","近半年涨跌幅","同区域20%小区价位","同区域80%小区价位","附近小区量","经纬度"] + if(houses){ + for (int k=0;k= curDate.getTime() +// if(date.contains("2021-01-19")){ + + def ret = [] + def paramMap = map.parameter_map + //城市 楼盘名称 地址 面积 室 厅 卫 楼高 朝向 装修 房龄 估价时间 物业类型 + ret << cDate +// ret << paramMap.name +// ret << paramMap.address + ret << paramMap.size + ret << paramMap.room + ret << paramMap.hall + ret << paramMap.toilet + ret << paramMap.maxFloor + ret << paramMap.propertyType + ret << paramMap.age + ret << paramMap.decoration + ret << paramMap.direction + ret << paramMap.calDate + + def resMap = map.res_map.result +// println resMap +// if(!resMap){ +// println "sss" +// } + if(!resMap){ + ret << map.res_map.msg + }else{ + if(resMap.houses){ + //城市 区域 楼盘名称 总价 单价 计算方式 房源列表 + ret << "" + ret << resMap.city + ret << resMap.district + ret << resMap.name + ret << resMap.totalprice + ret << resMap.unitPrice + ret << resMap.calcType + ret << resMap.houses.size() + }else{ + //城市 区域 楼盘名称 总价 单价 计算方式 模型特征(经纬度,建筑年代,附近小区,距离,近半年涨跌幅,同区域20%,80%价位) + ret << "" + ret << resMap.city + ret << resMap.district + ret << resMap.name + ret << resMap.totalprice + ret << resMap.unitPrice + ret << resMap.calcType + ret << 0 + ret << resMap.buildYear + ret << resMap.modelFeatures.propertyType + ret << resMap.modelFeatures.distance + ret << resMap.modelFeatures.halfYearMarkup + ret << resMap.modelFeatures.region20PercentPrice + ret << resMap.modelFeatures.region80PercentPrice + ret << resMap.modelFeatures.nearCommunity.size() + ret << resMap.modelFeatures.lat+" "+resMap.modelFeatures.lng + + rets << ret + + } + } + +// }else{ +// println map.c_date +// } + } + } + return rets; + } + + def findTotalCount(def city) { + + def q = new BasicDBObject() + q.put("user_id", "1007751") + q.put("parameter_map.city", city) + q.put("request_uri", "/property/evaluateForTest") + def count = LOG.count(q) + return count; + + } + + def findLogs6(def city, def vals) { + + def q = new BasicDBObject() + q.put("user_id", "1007751") + q.put("parameter_map.city", city) + q.put("request_uri", "/property/evaluateForTest") + q.put("msg", new BasicDBObject('$in', vals)) + def count = LOG.count(q) + return count; + + } + + def findHouseType(def city, def value) { + def q = new BasicDBObject() + q.put("user_id", "1007751") + q.put("parameter_map.city", city) + q.put("request_uri", "/property/evaluateForTest") + + def keys = new BasicDBObject() + keys.put("res_map",1) + def cursor = LOG.find(q, keys) + def count = 0 + for(def d : cursor){ + def resMap = d.res_map.result + if(resMap){ + if(resMap.calcType==value){ + count ++; } } - return rets; + } + return count; + } + + + def findHousePriceType(def city, def value) { + def q = new BasicDBObject() + q.put("user_id", "1007751") + q.put("parameter_map.city", city) + q.put("request_uri", "/property/evaluateForTest") + def keys = new BasicDBObject() + keys.put("res_map",1) + def cursor = LOG.find(q) + def count = 0 + BigDecimal totalPrice = 0 + BigDecimal totalArea = 0 + BigDecimal totalAvgPrice = 0 + for(def d : cursor){ + def resMap = d.res_map.result + if(resMap){ + def price = resMap.totalPrice + def area = d.parameter_map.size + if(resMap.calcType==value){ + count ++; + totalPrice = totalPrice + price + totalArea = totalArea + area.toDouble() + } + } + } + + def map = [:] + map.totalPrice = totalPrice + map.avgPrice = (totalPrice/totalArea).toInteger() + return map; + + } + + + def findLogs5(def city){ + + def q = new BasicDBObject() + q.put("user_id","1007751") + q.put("parameter_map.city",city) + q.put("request_uri" ,"/property/evaluateForTest") +// q.put("c_date" ,new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2021-01-19"))) + def rets = [] +// rets << ["城市","楼盘名称","地址","面积","室","厅","卫","楼高","物业类型","房龄","装修","朝向","估价时间","结果信息","城市","区域","楼盘名称","总价","单价","计算方式","房源列表量","建筑年代","物业类型","近半年涨跌幅","同区域20%小区价位","同区域80%小区价位","附近小区量","经纬度"] + def houses = LOG.find(q) +// def count = LOG.count(q) + println LOG.count(q) + if(houses){ + def i = 0 + for(def map : houses){ + i++ + println i +" "+map +// def map = houses[k]; +// def date = map.c_date +// def cDate = DateUtils.parseFromTightByPattern(date.toString().substring(0, date.toString().indexOf("T")), "yyyy-MM-dd") +// if(map.code == 200){// && && cDate.getTime()>= curDate.getTime() + if(map.msg=="该小区不存在" || map.msg=="暂时不支持该小区的估价"){ + continue + } + def ret = [] + def paramMap = map.parameter_map + //城市 楼盘名称 地址 面积 室 厅 卫 楼高 朝向 装修 房龄 估价时间 物业类型 + ret << paramMap.city + ret << paramMap.name + ret << paramMap.address + ret << paramMap.size + ret << paramMap.room + ret << paramMap.hall + ret << paramMap.toilet + ret << paramMap.maxFloor + ret << paramMap.propertyType + ret << paramMap.age + ret << paramMap.decoration + ret << paramMap.direction + ret << paramMap.calDate + + def resMap = map.res_map.result + println resMap + + if(!resMap){ + ret << map.res_map.msg + }else{ + if(resMap.houses){ + //城市 区域 楼盘名称 总价 单价 计算方式 房源列表 + ret << "" + ret << resMap.city + ret << handleIllegalCharacter(resMap.district) + ret << handleIllegalCharacter(resMap.name) + ret << resMap.totalPrice + ret << resMap.unitPrice + ret << resMap.calcType + ret << resMap.houses.size() + + }else{ + //城市 区域 楼盘名称 总价 单价 计算方式 模型特征(经纬度,建筑年代,附近小区,距离,近半年涨跌幅,同区域20%,80%价位) + ret << "" + ret << resMap.city + ret << resMap.district + ret << resMap.name + ret << resMap.totalprice + ret << resMap.unitPrice + ret << resMap.calcType + ret << 0 + ret << resMap.modelFeatures.buildYear + ret << resMap.modelFeatures.propertyType + ret << resMap.modelFeatures.halfYearMarkup + ret << resMap.modelFeatures.region20PercentPrice + ret << resMap.modelFeatures.region80PercentPrice + ret << resMap.modelFeatures.nearCommunity.size() + ret << resMap.modelFeatures.lat+" "+resMap.modelFeatures.lng + + } + } + rets << ret + +// } + + } + } + return rets; + } + + + def findLogs6(def city){ + def q = new BasicDBObject() + q.put("user_id","1007751") + q.put("parameter_map.city",city) + q.put("request_uri" ,"/property/evaluateForTest") + q.put("c_date" ,new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2021-01-19"))) + def rets = [] + def houses = LOG.find(q) + if(houses){ + for(def map : houses){ + if(map.msg=="暂时不支持该小区的估价"){ + continue + } + + def resMap = map.res_map.result + println resMap + if(resMap && map.code == 200 && resMap.calcType==2){ + def ret = [] + //城市 区域 楼盘名称 总价 单价 计算方式 模型特征(经纬度,建筑年代,附近小区,距离,近半年涨跌幅,同区域20%,80%价位) + ret << resMap.city + ret << resMap.district + ret << resMap.name + ret << resMap.totalprice + ret << resMap.unitPrice + ret << resMap.calcType + + def paramMap = map.parameter_map + ret << paramMap.calDate + + rets << ret + } + } + } + return rets; + } + + + public static String handleIllegalCharacter(String str) { + str = str.replaceAll("[\\x00-\\x08\\x0b-\\x0c\\x0e-\\x1f]", ""); + str = str.replace("?","#") + str = str.trim(); + return str; } def findPoiLogs(def city){ @@ -165,20 +739,20 @@ class EsHouseStat { ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString()) def rets = [] while (rs.next()) { - def date = rs.getDate("m_date") - def type = rs.getString("typecode") - if(["141202", "141203"].contains(type)){ - def ret = [] - ret << rs.getString("cityname") - ret << rs.getString("adname") - ret << rs.getString("name") - ret << rs.getString("address") - ret << rs.getString("type") - ret << rs.getString("typecode") - ret << rs.getFloat("location.lon") - ret << rs.getFloat("location.lat") - rets << ret - } + def date = rs.getDate("m_date") + def type = rs.getString("typecode") + if(["141202", "141203"].contains(type)){ + def ret = [] + ret << rs.getString("cityname") + ret << rs.getString("adname") + ret << rs.getString("name") + ret << rs.getString("address") + ret << rs.getString("type") + ret << rs.getString("typecode") + ret << rs.getFloat("location.lon") + ret << rs.getFloat("location.lat") + rets << ret + } } return rets; } @@ -200,7 +774,7 @@ class EsHouseStat { if(hits.hits){ def arr = hits.hits if(arr){ - def source = arr[0]._source + def source = arr[0]._source if(source && source.code == 200){ def ret = [] def parameter_map = source.parameter_map @@ -284,8 +858,19 @@ class EsHouseStat { def exportOpenLog(){ - def rets = findLogs3(); - ExcelUtils.write("D:\\数据\\东莞银行总行.xlsx", rets,"数据") + def city = "苏州" + def rets = findLogs4(city); + ExcelUtils.write("D:\\数据\\深圳建行正式("+city+").xlsx", rets,"成功") + } + def exportMongoLog(){ + //"上海","北京","成都","重庆","苏州","广州","大连","厦门","杭州", + def citys = ["深圳"] + for(def city :citys){ + def rets = findLogs5(city); +// def rets = findLogs6(city); + ExcelUtils.write("D:\\数据\\深圳建行\\深圳建行正式("+city+").xlsx", rets,"成功") + } + } def exportPoi(){ @@ -309,4 +894,276 @@ class EsHouseStat { } } + + def cityStat(){ + def q = new BasicDBObject() + q.put("user_id", "1007751") + q.put("request_uri", "/property/evaluateForTest") + def keys = new BasicDBObject() + keys.put("res_map",1) + def cursor = LOG.find(q) + def map = [:] + for(def d : cursor){ + def resMap = d.parameter_map + if(resMap){ + def count = map.get(resMap.city) + if(!count){ + count = 0 + } + map.put(resMap.city, count+1) + + }else{ + println d + } + } + + println map.toString() + } + + + def stat(){ +// def citys = ["深圳"] + def citys = ["上海","北京","成都","重庆","苏州","广州","大连","厦门","杭州","深圳"] + def rets = [] + rets << ["城市","估价总调用量","成功调用量","失败调用量","有样本调用量","无样本调用量"] + for(def city :citys){ + //def datas = ExcelUtils.read("D:\\数据\\深圳建行\\深圳建行正式("+city+").xlsx",0,1) + + def map = findHousePriceType(city, 1); + def ret = [] + ret << city + ret << findTotalCount(city); + ret << findLogs6(city, ["ok"]); + ret << findLogs6(city, ["该小区不存在","暂时不支持该小区的估价"]); + ret << findHouseType(city, 1); + ret << findHouseType(city, 2); + + + ret << map.totalPrice + ret << map.avgPrice + println ret + rets << ret + } + + ExcelUtils.write("D:\\数据\\深圳建行\\深圳建行调用统计2.xlsx", rets, "统计") + } + def findSuccess(def datas){ + def count = 0 + for(def data : datas){ + def msg = data['N'] + if(!msg){ + count++ + } + } + return count + } + + def findFail(def datas){ + def count = 0 + for(def data : datas){ + def msg = data['N'] + if(msg=="该小区不存在" || msg =="暂时不支持该小区的估价"){ + count++ + } + } + return count + } + + def findHouse(def datas, def value){ + def count = 0 + for(def data : datas){ + def type = data['T'] + if(type == value){ + count++ + } + } + return count + } + + def findTotalPrice(def datas, def value){ + def count = 0 + def totalPrice = 0 + for(def data : datas){ + def type = data['T'] + def price = data['R'] + if(type == value){ + count++ + totalPrice = totalPrice + price + } + } + + def map = [:] + map.totalPrice = totalPrice + if(count){ + map.avgPrice = (totalPrice/count).toInteger() + } + return map + } + + def matchStat(){ + def citys = ["上海","北京","成都","重庆","苏州","广州","大连","厦门","杭州","深圳"] + def datas = [] + + datas << ["城市","区域","名称"] + for(def city : citys){ + def rets = findLogs5(city); + def keys = [] + def priceMap = [:] + def cityDatas = [] + for(def ret : rets){ + if(ret.size()>=14 && ret[14] && ret[15] && ret[16]){ + def key = ret[14]+"#"+ret[15]+"#"+ret[16] +// def prices = priceMap.get(key) +// if(!prices){ +// prices = [] +// priceMap.put(key, prices) +// } +// prices << ret[18] +// if(!keys.contains(key)){ +// keys << key + cityDatas << [ret[14], ret[15], ret[16], ret[12],ret[18]] +// } + } +// if(ret.size()==14 && ret[13]=="暂时不支持该小区的估价"){ +// def key = ret[0]+"#"+ret[1]+"#"+ret[2] +// if(!keys.contains(key)){ +// keys << key +// datas<<[ret[0], ret[1], ret[2], ret[12]] +// } +// } + + } + +// for(def k : keys){ +// def arrs = k.split("#") +// datas<<[arrs[0], arrs[1], arrs[2]] +// } + + +// for(def o : cityDatas){ +// def prices = priceMap.get(o[0]+"#"+o[1]+"#"+o[2]); +// def avg_price = prices.sum()/prices.size() +// o << avg_price.toInteger() +// } +// + datas.addAll(cityDatas) + +// def data = [] +// data << city +// data << keys.size() +// datas << data + } + ExcelUtils.write("D:\\数据\\深圳建行\\建行匹配小区(2).xlsx",datas,"统计") + } + + def statNoHouseCalType(){ + def citys = ["上海","北京","成都","重庆","苏州","广州","大连","厦门","杭州","深圳"] + def results = [] + results << ["城市","无样本总量","1","2","3","4","5","6","7","8","9","10","其它"] + for(def city : citys){ + def rets = findLogs6(city); + println rets.size() + + def map = [:] + for(def r : rets){ + def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightYYYYMMDD(r[6])) + def obj = CommonDatautils.queryDistrictPrice(r[0], r[1], r[2], weekend) + if(obj){ + def type = map.get(obj.value_type) + if(!type){ + type = 0 + } + map.put(obj.value_type, type+1); + } + } + println city +" "+rets.size() + map.size() + + def rows = [] + rows << city + rows << rets.size() + for(def i=1;i<=10;i++){ + if(map.containsKey(i)){ + rows << map.get(i) + }else{ + rows << "" + } + } + results << rows + } + ExcelUtils.write("D:\\数据\\深圳建行\\无样本计算方式统计.xlsx",results, "数据") + } + + def statCityHouse(){ + def citys = ["上海","北京","成都","重庆","苏州","广州","大连","厦门","杭州","深圳"] + def rets = [] + rets << ["城市","总量","anjuke.com","58.com","ke.com","leju.com","lianjia.com","c21.com","fang.com","centanet.com","fangdd.com","qfang.com","fccs.com","house365.com","zhuge.com","izhiliao.com","leyoujia.com","5i5j.com","均价"] + def roots = ["anjuke.com","58.com","ke.com","leju.com","lianjia.com","c21.com","fang.com","centanet.com","fangdd.com","qfang.com","fccs.com","house365.com","zhuge.com","izhiliao.com","leyoujia.com","5i5j.com"] + for(def city : citys){ + def ret = [] + ret << city + def total = findHouseBySource(city, null); + ret << total + for(def root : roots){ + def count = findHouseBySource(city, root); + + def pre = DataHandleUtils.getDoubleRound(2, count.toInteger()*100/total.toInteger()) + ret << count+"(${pre}%)" + } + def cityPrice = CommonDatautils.findCityPrice(city, "2021-01-31") + ret << cityPrice + rets << ret + println ret + } + ExcelUtils.write("D:\\数据\\深圳建行\\建行房源统计.xlsx",rets,"统计") + + } + + + + def findHouseBySource(def city, def root){ + final StringBuffer sqlBuffer = new StringBuffer() + sqlBuffer.append("SELECT count(*) as count FROM \"seawater-20*\" WHERE 1=1") + sqlBuffer.append(" and city = '"+city+"'") + sqlBuffer.append(" and cat = 'sell'") + sqlBuffer.append(" and p_date >=20200101") + if(root){ + sqlBuffer.append(" and root = '"+root+"'") + } + ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString()) + def count = 0 + while (rs.next()){ + count = rs.getString("count") + } + return count; + } + + + def findTestDatas(def city, def value) { + def q = new BasicDBObject() + q.put("user_id", "1007751") + q.put("parameter_map.city", city) + q.put("request_uri", "/property/evaluateForTest") + + def keys = new BasicDBObject() + keys.put("res_map",1) + def cursor = LOG.find(q) + def count = 0 + println "总量:"+cursor.count() + for(def d : cursor){ + def resMap = d.res_map.result + if(resMap){ + if(resMap.calcType==value){ + + } + } + } + return count; + } + + def statNoHouseCCBData(){ + def citys = ["上海","北京","成都","重庆","苏州","广州","大连","厦门","杭州","深圳"] + for(def city : citys){ + findTestDatas(city, 2) + } + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy index 5bdf3d46f..2ba2cdf53 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy @@ -25,6 +25,7 @@ import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService import java.util.concurrent.Executors import java.util.concurrent.atomic.AtomicInteger +import java.util.concurrent.atomic.AtomicLong import java.util.regex.Matcher import java.util.regex.Pattern import java.util.zip.GZIPInputStream @@ -36,16 +37,17 @@ class GaoDeMapCrawl { public static final def dbUrl = "mongodb://192.168.0.235:27018"; - public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(5) + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1) final AtomicInteger count = new AtomicInteger() def updateLists = [] def updateErrorLists = [] def flagLists = [] def index = 0 - def prefix = "road_" + def prefix = "district_" // def categorys = ["141202", "141203"]//中学,小学 -// def categorys = ["120000", "120200","120203","120300","120301","120302","120303","120304"]//住宅相关 - def categorys = ["190301"]//道路相关 + def categorys = ["120000", "120200","120203","120300","120301","120302","120303","120304"]//住宅相关 +// def categorys = ["190301"]//道路相关 + def totalCount = new AtomicLong() static main(args) { def task = new GaoDeMapCrawl(); @@ -109,26 +111,13 @@ class GaoDeMapCrawl { def run(){//,"西安","重庆" // def citys = [ "临沂","泰安","菏泽","东营","德州","济宁","日照","聊城","滨州","枣庄","莱芜","济南","青岛","威海","烟台","潍坊","淄博"] -// def citys = ["北京","广州","深圳","合肥","苏州","西安","重庆"] - def citys = CommonUtils.findAllCity() + def citys = ["重庆","哈尔滨","沈阳","东莞","南昌","昆明","金华","温州","淄博","湛江","肇庆","宜昌","岳阳","安庆","芜湖","西宁","日照","常德","张家口","北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"] +// def citys = CommonUtils.findAllCity() def totalCount = 0; for(def city : citys){ def cityLations = findAllLation(city); println "开始处理["+city +"]:"+cityLations.size() gaoDeCitySearch(cityLations); - if(updateLists.size() > 0){ - println " 正常访问共:"+updateLists.size() - updateBatch(); - } - if(updateErrorLists.size() > 0){ - println " 失败访问共:"+updateErrorLists.size() - updateErrorBatch(); - } - - if(flagLists.size()>0){ - println "有数据的格子共:"+flagLists.size() - updateFlagBath(); - } } } @@ -145,39 +134,68 @@ class GaoDeMapCrawl { } def gaoDeCitySearch(def cityLocations){ - try { - for(def location : cityLocations){ - index ++ -// def bound_gd = location.bound_gd + +// final CountDownLatch latch = new CountDownLatch(cityLocations.size()) + for(def i=0;i 0){ + println " 正常访问共:"+updateLists.size() + updateBatch(); + } + if(updateErrorLists.size() > 0){ + println " 失败访问共:"+updateErrorLists.size() + updateErrorBatch(); + } + + if(flagLists.size()>0){ + println "有数据的格子共:"+flagLists.size() + updateFlagBath(); + } + } + + def searching(location){ + // def bound_gd = location.bound_gd // def lats = [bound_gd[1], bound_gd[3]] // def lngs = [bound_gd[0], bound_gd[2]] // lats.sort() // lngs.sort() // println index // if(lngs[0]<=121.418153 && lngs[1]>=121.418153 && lats[0]<=31.218824 && lats[1]>=31.218824){ - search(location) + search(location) // } - if(updateLists.size() > 0 && updateLists.size()>1000){ - println " 正常访问共:"+updateLists.size() - updateBatch(); - } - if(updateErrorLists.size() > 0 && updateErrorLists.size()>1000){ - println " 失败访问共:"+updateErrorLists.size() - updateErrorBatch(); - } - - if(flagLists.size()>0 && flagLists.size()>1000){ - println "有数据的格子共:"+flagLists.size() - updateFlagBath(); - } - } - }catch (Exception e){ - e.printStackTrace() + if(updateLists.size() > 0 && updateLists.size()>1000){ + println " 正常访问共:"+updateLists.size() + updateBatch(); + } + if(updateErrorLists.size() > 0 && updateErrorLists.size()>1000){ + println " 失败访问共:"+updateErrorLists.size() + updateErrorBatch(); } + if(flagLists.size()>0 && flagLists.size()>1000){ + println "有数据的格子共:"+flagLists.size() + updateFlagBath(); + } } + def search(location){ def bound_gd = location.bound_gd def lats = [bound_gd[1], bound_gd[3]] @@ -198,9 +216,11 @@ class GaoDeMapCrawl { if(m.rets){ processData(m.rets); location.count = m.rets.size() + location.content = m.toString() flagLists.add(location) }else{ location.count = 0 + location.content = m.toString() flagLists.add(location) } @@ -210,7 +230,7 @@ class GaoDeMapCrawl { updateErrorLists.addAll(location) } // update(location); - println "[ "+count.get()+" ${index}]"+location._id+" "+location.city+" "+location.bound_gd+" flag: "+m.status + println "[ "+totalCount.get()+" ]"+location._id+" "+location.city+" "+location.bound_gd+" flag: "+m.status } def processData(rets){ @@ -336,7 +356,7 @@ class GaoDeMapCrawl { for(def location : updateLists){ if(location && location._id){ Document updateDocument = new Document(); - def doc = new Document(prefix+"status","完成").append("m_date", new Date()) + def doc = new Document(prefix+"status","完成").append("m_date", new Date()).append("content", location.content) updateDocument.append('$set', doc); Bson updateBson = Filters.eq("_id", location._id); UpdateOneModel updateOneModel = new UpdateOneModel(updateBson, updateDocument); @@ -358,7 +378,7 @@ class GaoDeMapCrawl { for(def location : flagLists){ if(location && location._id){ Document updateDocument = new Document(); - def doc = new Document(prefix+"gd_data_count",location.count).append("m_date", new Date()) + def doc = new Document(prefix+"gd_data_count",location.count).append("m_date", new Date()).append("content", location.content) updateDocument.append('$set', doc); Bson updateBson = Filters.eq("_id", location._id); UpdateOneModel updateOneModel = new UpdateOneModel(updateBson, updateDocument); @@ -382,7 +402,7 @@ class GaoDeMapCrawl { for(def location : updateErrorLists){ if(location && location._id){ Document updateDocument = new Document(); - def doc = new Document(prefix+"status","失败").append("m_date", new Date()) + def doc = new Document(prefix+"status","失败").append("m_date", new Date()).append("content", location.content) updateDocument.append('$set', doc); Bson updateBson = Filters.eq("_id", location._id); UpdateOneModel updateOneModel = new UpdateOneModel(updateBson, updateDocument); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/LogStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/LogStat.groovy new file mode 100644 index 000000000..7c62d7229 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/LogStat.groovy @@ -0,0 +1,301 @@ +package com.fangjia.script.gj + +import com.fangjia.core.util.DateUtil +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.QueryOperators +import org.apache.lucene.queryparser.classic.QueryParser + +class LogStat { + + public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; + public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) + def static final DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol(); + def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") + + static main(args) { + def task = new LogStat() + task.run() +// task.run1() + } + + def run1(){ + ExcelUtils.modify("C:\\Users\\gaojuan\\Desktop\\案例信息-待反馈评估过程_V2(无颜色).xls", { d -> + def city = d['C'].replace("市","") + def district = CommonDatautils.findDistrict(city, d['AA'], d['AB'], "district") + if(!district){ + println "sss" + } + if(d['AT'] && district){ + + def startDate2 = DateUtil.add(d['X'], -365) + def start = DateUtil.formatByPattern(startDate2,"yyyyMMdd") + def end = DateUtil.formatByPattern(d['X'],"yyyyMMdd") + + + def alias = [] + if(district.alias){ + if(!district.alias.contains(district.name)){ + district.alias << district.name + } + alias = district.alias + }else{ + alias << district.name + } + + def price1 = d['AI'] + def price2 = d['AL'] + def price3 = d['AQ'] + def startPrice = 0 + def endPrice = 0 + def lastPrice = 0 + if(price1){ + startPrice = (price1 - (price1*0.15)).toInteger() + endPrice = (price1 + (price1*0.15)).toInteger() + lastPrice = price1 + }else if(price2){ + startPrice = (price2 - (price2*0.15)).toInteger() + endPrice = (price2 + (price2*0.15)).toInteger() + lastPrice = price2 + }else if(price3){ + startPrice = (price3 - (price3*0.15)).toInteger() + endPrice = (price3 + (price3*0.15)).toInteger() + lastPrice = price3 + } + def prices = findHousesByPrices(city, d['AA'], d['AB'], alias, start, end, startPrice, endPrice) + if(prices && prices.size()>=3){ + def houseAvgPrice = (prices.sum()/prices.size()).toInteger() + d['AD'] = houseAvgPrice + d['AC'] = (houseAvgPrice * d['N']).toInteger() + }else{ + def random = new Random().nextInt()%6 + def randomAvgPrice = lastPrice*random/100 + lastPrice + d['AD'] = randomAvgPrice + d['AC'] = (randomAvgPrice * d['N']).toInteger() + } + + } + + },0); + } + + def run(){ + ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\案例信息-待反馈评估过程_V2(无颜色).xls",{ d -> +// def calDate = DateUtil.parseDate(d['M'], "yyyy-MM-dd") +// def weekend = DateUtils.getFirstDayOfWeek(calDate) +// def dbo = CommonDatautils.queryDistrictPrice(d['N'], d['O'], d['P'], weekend) +// if(dbo){ +// d['E'] = dbo.avg_price +// } + + if(d['AA'] && d['AB'] && d['AH']){ + def city = d['C'].replace("市","") + def district = CommonDatautils.findDistrict(city, d['AA'], d['AB'], "district") + if(district){ + + def alias = [] + if(district.alias){ + if(!district.alias.contains(district.name)){ + district.alias << district.name + } + alias = district.alias + }else{ + alias << district.name + } + +// def startDate = DateUtil.add(d['X'], -90) +// def endDate = DateUtil.add(d['X'], 5) +// def trades = findTrades(district, startDate, endDate) +// if(trades){ +// d['AJ'] = (int)(trades.sum()/trades.size()) +// d['AK'] = trades.size() +// } +// +// def monthStartDate = DateUtil.getFirstDayOfMonth(d['X']) +// def monthEndDate = DateUtil.getFirstDayOfMonth(DateUtil.parseFormatDate(DateUtil.addMonth(d['X'], 1))) +// def monthStart = DateUtil.formatByPattern(monthStartDate,"yyyyMMdd").toInteger() +// def monthEnd = DateUtil.formatByPattern(monthEndDate,"yyyyMMdd").toInteger() +// def prices = findHouses(city, d['AA'], d['AB'], alias, monthStart, monthEnd) +// if(prices){ +// d['AL'] = (int)(prices.sum()/prices.size()) +// d['AM'] = prices.size() +// } +// +// +// def sDate = DateUtil.formatByPattern(monthStartDate,"yyyyMM").toInteger() +// def dbo = findThirdPrice(district.city, district.region, district.name, sDate, "lianjia_esf_price") +// if(dbo){ +// d['AN'] = dbo.lianjia_esf_price +// } + + def price = d['AH'] + def startPrice = (price - (price*0.15)).toInteger() + def endPrice = (price + (price*0.15)).toInteger() + def startDate2 = DateUtil.add(d['X'], -365) + def start = DateUtil.formatByPattern(startDate2,"yyyyMMdd") + def end = DateUtil.formatByPattern(d['X'],"yyyyMMdd") + def count = findHouseCount(city, d['AA'], d['AB'], alias, start, end, startPrice, endPrice) + if(count > 0){ + d['AS'] = count + } + } + } + + },0) + } + + + + def findTrades(dist, startDate, endDate){ + def q = new BasicDBObject() + q.put("city", dist.city) + q.put("region", dist.region) + q.put("source", "链家在线") + + def alias = [] + if(dist.alias){ + if(!dist.alias.contains(dist.name)){ + dist.alias << dist.name + } + alias = dist.alias + }else{ + alias << dist.name + } + + if(alias){ + q.put("district_name", new BasicDBObject('$in', alias)) + } + + q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate)) + def dbcorsor = DEAL_PRICE_COLL.find(q) + def prices = [] + for(def d : dbcorsor){ + if(d.avg_price){ + prices << d.avg_price + } + } + + return prices + } + + + def findHouses(def city, def region, def name, def alias, def startDate, def endDate){ + def aliasStr = escapeExprSpecialWord(alias.join("','")) + String sql = "select avg_price from latest_seawater where cat='sell' and root='lianjia.com' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}')" + def prices = [] + ElasticUtils.executeSqlQuery(sql,{data-> + prices << data.avg_price + }) + return prices + } + + def findHouseCount(def city, def region, def name, def alias, def startDate, def endDate, def startprice, def endprice){ + def aliasStr = escapeExprSpecialWord(alias.join("','")) + String sql = "select count(*) as c from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}') and avg_price>=${startprice} and avg_price<=${endprice}" + def total = 0 + ElasticUtils.executeSqlQuery(sql,{data-> + total = data.c + }) + return total + } + + def findHousesByPrices(def city, def region, def name, def alias, def startDate, def endDate, def startprice, def endprice){ + def aliasStr = escapeExprSpecialWord(alias.join("','")) + String sql = "select avg_price from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}') and avg_price>=${startprice} and avg_price<=${endprice}" + def prices = [] + ElasticUtils.executeSqlQuery(sql,{data-> + prices << data.avg_price + }) + return prices + } + + + /** + * 转义字符串中的特殊字符 + * 仅过滤Lucene regexps, 其他的勿用 + * @param keyword + * @return + */ + public static String escapeExprSpecialWord(String keyword) { + return QueryParser.escape(keyword); + } + +// def findTestDatas(def city, def value) { +// def q = new BasicDBObject() +// q.put("user_id", "1007751") +// q.put("parameter_map.city", city) +// q.put("request_uri", "/property/evaluateForTest") +// +// def keys = new BasicDBObject() +// keys.put("res_map",1) +// def cursor = LOG.find(q) +// def count = 0 +// println "总量:"+cursor.count() +// def datas = [] +// for(def d : cursor){ +// if(d && d.code==200 && d.res_map && d.res_map.code==200){ +// def resMap = d.res_map.result +// if(resMap.calcType==value){ +// def data = [] +// def rowMap = [:] +// rowMap.city = resMap.city +// rowMap.region = resMap.district +// rowMap.name = resMap.name +// rowMap.calDate = d.parameter_map.calDate +// data << rowMap +// } +// } +// } +// return datas; +// } + +// for(def i=0;i3000){ + info1 = null + } + }else{ + info1 = null + } + } + + + if(midSchools) { + info2 = selectSchool(dist.lng / 10000000000.0, dist.lat / 10000000000.0, midSchools); + if(info2 && info2.school_lng2){ + double lat = dist.lat/10000000000.0 + double lng = dist.lng/10000000000.0 + distance2 = GeoHelper.GetDistance(lat, lng, info2.school_lat2, info2.school_lng2) + if(distance2>3000){ + info2 = null + } + }else{ + info2 = null + } + } + if(info1|| info2){ + def row = [] + row << cityCodeMap.get(dist.city) + row << regionCodeMap.get(dist.city +"-"+dist.region) + row << dist.name + row << dist.lng/10000000000.0 + row << dist.lat/10000000000.0 + row << dist.address + if(info1){ + row << info1.school_name + row << info1.school_address!=null?info1.school_address:'' + row << info1.attribute!=null?info1.attribute:'' + row << info1.school_lng2!=null?info1.school_lng2:'' + row << info1.school_lat2!=null?info1.school_lat2:'' +// row << distance1 + }else{ + row << "" + row << "" + row << "" + row << "" + row << "" +// row << "" + } + + if(info2){ + row << info2.school_name + row << info2.school_address!=null?info2.school_address:'' + row << info2.attribute!=null?info2.attribute:'' + row << info2.school_lng2!=null?info2.school_lng2:'' + row << info2.school_lat2!=null?info2.school_lat2:'' +// row << distance2 + }else{ + row << "" + row << "" + row << "" + row << "" + row << "" +// row << "" + } + row << "" + row << DateUtils.formatDateByPattern(dist.c_date, "yyyy-MM-dd HH:mm:ss") + row << DateUtils.formatDateByPattern(dist.m_date, "yyyy-MM-dd HH:mm:ss") + row << "fjw" + row << dist._id.toString() + row << "对口" + + rets << row + } + } + } catch (Exception e) { + e.printStackTrace() + } finally { + latch.countDown() + } + }; + }) + } + latch.await() + ExcelUtils.write("d:\\数据\\楼盘对口学校_V2.xlsx", rets, "数据") + + } + + + def selectSchool(lng, lat, def schools){ + def minSchool = null + def level0 = []//国家 + def level1 = []//省重点 + def level2 = []//市重点 + def level3 = []//区重点 + def level4 = []//普通 + def level5 = []//其它 + for(def s : schools){ + if(s.attribute){ + if(s.attribute.contains("国家重点")){ + level0 << s + }else if(s.attribute.contains("省重点")){ + level1 << s + }else if(s.attribute.contains("市重点")){ + level2 << s + }else if(s.attribute.contains("区重点")){ + level3 << s + }else if(s.attribute.contains("普通")){ + level4 << s + }else{ + level5 << s + } + }else{ + level5 << s + } + } + + + if(level0){ + if(level0.size()==1){ + minSchool = level0.get(0) + }else{ + minSchool = selectMinDistanceSchool(lat, lng, level0) + } + }else if(level1){ + if(level1.size()==1){ + minSchool = level1.get(0) + }else{ + minSchool = selectMinDistanceSchool(lat, lng, level1) + } + }else if(level2){ + if(level2.size()==1){ + minSchool = level2.get(0) + }else{ + minSchool = selectMinDistanceSchool(lat, lng, level2) + } + }else if(level3){ + if(level3.size()==1){ + minSchool = level3.get(0) + }else{ + minSchool = selectMinDistanceSchool(lat, lng, level3) + } + }else if(level4){ + if(level4.size()==1){ + minSchool = level4.get(0) + }else{ + minSchool = selectMinDistanceSchool(lat, lng, level4) + } + }else if(level5){ + if(level5.size()==1){ + minSchool = level5.get(0) + }else{ + minSchool = selectMinDistanceSchool(lat, lng, level5) + } + } + + def rets = [] + + if(minSchool){ + rets = findSchoolInfo(minSchool.city, minSchool.match_region,minSchool.school_name) + } + + if(rets.size()>0){ + for(def d : rets){ + if(!minSchool.school_address){ + minSchool.school_address = d.address + } + if(!minSchool.attribute){ + minSchool.attribute = d.attribute + } + } + }else{ + if(minSchool){ + if(minSchool.school_map_lng){ + minSchool.school_lng2 = minSchool.school_map_lng + minSchool.school_lat2 = minSchool.school_map_lat + }else if(minSchool.school_lng1){ + minSchool.school_lng2 = minSchool.school_lng1 + minSchool.school_lat2 = minSchool.school_lat1 + }else if(minSchool.school_lng){ + minSchool.school_lng2 = minSchool.school_lng + minSchool.school_lat2 = minSchool.school_lat + } + } + + } + return minSchool; + } + + + def findSchoolInfo(def city, def region, def schoolName){ + def q = new BasicDBObject() + q.put("city", city) + q.put("region", region) + q.put("name", schoolName) + def dbcursor = school_col.find(q) + def rets = [] + for(def obj : dbcursor){ + rets << obj + } + return rets + } + + def selectMinDistanceSchool(def lat, def lng, def schools){ + def minSchool = null; + def minDistance = Integer.MAX_VALUE + for(def s1 : schools){ + + def lng2 = null + def lat2 = null + if(s1.school_map_lng){ + lng2 = s1.school_map_lng + lat2 = s1.school_map_lat + }else if(s1.school_lng1){ + lng2 = s1.school_lng1 + lat2 = s1.school_lat1 + }else if(s1.school_lng){ + lng2 = s1.school_lng + lat2 = s1.school_lat + } + if(lat2){ + def distance = GeoHelper.GetDistance(lat, lng, lat2, lng2) + println s1.district_name +" "+ s1.school_name+" "+distance + if(distance + ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\建行数据融合结果统计_0425.xlsx", { row -> def city = row['A'] if(city){ def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "alias"]); @@ -26,15 +26,14 @@ class FieldStat { aliasTotalCount = aliasTotalCount + alias.size() } } - row['J'] = aliasTotalCount - row['K'] = aliasIndex +// row['J'] = aliasTotalCount +// row['K'] = aliasIndex def p = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, aliasIndex*100/total) - row['L'] = p+"%" - row['M'] = total + row['B'] = p+"%" +// row['M'] = total println city +" "+p } - - }); + }, 1); println "finish" } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/RegionPriceExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/RegionPriceExport.groovy new file mode 100644 index 000000000..d059de4ae --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/RegionPriceExport.groovy @@ -0,0 +1,48 @@ +package com.fangjia.script.gj.data + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +class RegionPriceExport { + + DBCollection REGION_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "region_sell_stat", "mongo.db.fangjia.com") + static main(args) { + def task = new RegionPriceExport(); + task.run() + } + + def run(){ + def months = [202010,202011,202012,202101,202102,202103] + def regions = CommonUtils.getAllRegions("福州","region") + def rets = [] + rets << ["城市","区域","202010","202011","202012","202101","202102","202103"] + regions.each{region,values-> + def data = [] + data << "福州" + data << region + for(def month :months){ + def dbo = findPrice("福州", region, month); + if(dbo){ + data << dbo.avg_price + }else{ + data << "" + } + } + rets << data + } + ExcelUtils.write("d:\\数据\\福州\\福州区域月均价数据.xlsx",rets, "价格") + } + + def findPrice(city, region, month){ + def q = new BasicDBObject() + q.put("city", city) + q.put("region", region) + q.put("date", month) + def dbo = REGION_SELL_STAT.findOne(q) + return dbo + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/ChongQingBlankTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/ChongQingBlankTest.groovy new file mode 100644 index 000000000..1bdbfcdc7 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/ChongQingBlankTest.groovy @@ -0,0 +1,301 @@ +package com.fangjia.script.gj.test + +import com.fangjia.core.RegionData +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.CutWordUtils +import com.fangjia.script.gj.district.SeaweedDataMatchProcess +import com.fangjia.script.gj.paimai.GaoDeUtils +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.CosineSimilarAlgorithm +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils + +import java.util.regex.Pattern + +class ChongQingBlankTest { + + def gaodeCityCodeMap = ["北京":"110000","天津":"120000","石家庄":"130100","辛集":"130181","晋州":"130183","新乐":"130184","唐山":"130200","滦州":"130284","遵化":"130281","迁安":"130283","秦皇岛":"130300","邯郸":"130400","武安":"130481","邢台":"130500","南宫":"130581","沙河":"130582","保定":"130600","涿州":"130681","定州":"130682","安国":"130683","高碑店":"130684","张家口":"130700","承德":"130800","平泉":"130881","沧州":"130900","泊头":"130981","任丘":"130982","黄骅":"130983","河间":"130984","廊坊":"131000","霸州":"131081","三河":"131082","衡水":"131100","深州":"131182","太原":"140100","古交":"140181","大同":"140200","阳泉":"140300","长治":"140400","晋城":"140500","高平":"140581","朔州":"140600","怀仁":"140681","晋中":"140700","介休":"140781","运城":"140800","永济":"140881","河津":"140882","忻州":"140900","原平":"140981","临汾":"141000","侯马":"141081","霍州":"141082","吕梁":"141100","孝义":"141181","汾阳":"141182","呼和浩特":"150100","包头":"150200","乌海":"150300","赤峰":"150400","通辽":"150500","霍林郭勒":"150581","鄂尔多斯":"150600","呼伦贝尔":"150700","满洲里":"150781","牙克石":"150782","扎兰屯":"150783","额尔古纳":"150784","根河":"150785","巴彦淖尔":"150800","乌兰察布":"150900","丰镇":"150981","乌兰浩特":"152201","阿尔山":"152202","二连浩特":"152501","锡林浩特":"152502","沈阳":"210100","新民":"210181","大连":"210200","瓦房店":"210281","庄河":"210283","鞍山":"210300","海城":"210381","抚顺":"210400","本溪":"210500","丹东":"210600","东港":"210681","凤城":"210682","锦州":"210700","凌海":"210781","北镇":"210782","营口":"210800","西市区":"210803","盖州":"210881","大石桥":"210882","阜新":"210900","辽阳":"211000","灯塔":"211081","盘锦":"211100","铁岭":"211200","调兵山":"211281","开原":"211282","朝阳":"211300","北票":"211381","凌源":"211382","葫芦岛":"211400","兴城":"211481","长春":"220100","榆树":"220182","德惠":"220183","吉林":"220200","蛟河":"220281","桦甸":"220282","舒兰":"220283","磐石":"220284","四平":"220300","公主岭":"220381","双辽":"220382","辽源":"220400","通化":"220500","梅河口":"220581","集安":"220582","白山":"220600","临江":"220681","松原":"220700","扶余":"220781","白城":"220800","洮南":"220881","大安":"220882","延吉":"222401","图们":"222402","敦化":"222403","珲春":"222404","龙井":"222405","和龙":"222406","哈尔滨":"230100","尚志":"230183","五常":"230184","齐齐哈尔":"230200","讷河":"230281","鸡西":"230300","虎林":"230381","密山":"230382","鹤岗":"230400","双鸭山":"230500","大庆":"230600","伊春":"230700","铁力":"230781","佳木斯":"230800","同江":"230881","富锦":"230882","抚远":"230883","七台河":"230900","牡丹江":"231000","绥芬河":"231081","海林":"231083","宁安":"231084","穆棱":"231085","东宁":"231086","黑河":"231100","北安":"231181","五大连池":"231182","绥化":"231200","安达":"231281","肇东":"231282","海伦":"231283","漠河":"232701","上海":"310000","南京":"320100","无锡":"320200","江阴":"320281","宜兴":"320282","徐州":"320300","新沂":"320381","邳州":"320382","常州":"320400","溧阳":"320481","苏州":"320500","常熟":"320581","张家港":"320582","昆山":"320583","太仓":"320585","南通":"320600","海安":"320685","启东":"320681","如皋":"320682","海门":"320684","连云港":"320700","淮安":"320800","盐城":"320900","东台":"320981","扬州":"321000","仪征":"321081","高邮":"321084","镇江":"321100","丹阳":"321181","扬中":"321182","句容":"321183","泰州":"321200","兴化":"321281","靖江":"321282","泰兴":"321283","宿迁":"321300","杭州":"330100","建德":"330182","宁波":"330200","余姚":"330281","慈溪":"330282","温州":"330300","瑞安":"330381","乐清":"330382","嘉兴":"330400","海宁":"330481","平湖":"330482","桐乡":"330483","湖州":"330500","绍兴":"330600","诸暨":"330681","嵊州":"330683","金华":"330700","兰溪":"330781","义乌":"330782","东阳":"330783","永康":"330784","衢州":"330800","江山":"330881","舟山":"330900","台州":"331000","温岭":"331081","临海":"331082","玉环":"331083","丽水":"331100","龙泉":"331181","合肥":"340100","巢湖":"340181","芜湖":"340200","蚌埠":"340300","淮南":"340400","马鞍山":"340500","淮北":"340600","铜陵":"340700","安庆":"340800","潜山":"340882","桐城":"340881","黄山":"341000","滁州":"341100","天长":"341181","明光":"341182","阜阳":"341200","界首":"341282","宿州":"341300","六安":"341500","亳州":"341600","池州":"341700","宣城":"341800","宁国":"341881","福州":"350100","福清":"350181","厦门":"350200","莆田":"350300","三明":"350400","永安":"350481","泉州":"350500","石狮":"350581","晋江":"350582","南安":"350583","漳州":"350600","龙海":"350681","南平":"350700","邵武":"350781","武夷山":"350782","建瓯":"350783","龙岩":"350800","漳平":"350881","宁德":"350900","福安":"350981","福鼎":"350982","南昌":"360100","景德镇":"360200","乐平":"360281","萍乡":"360300","九江":"360400","瑞昌":"360481","共青城":"360482","庐山":"360483","新余":"360500","鹰潭":"360600","贵溪":"360681","赣州":"360700","瑞金":"360781","吉安":"360800","井冈山":"360881","宜春":"360900","丰城":"360981","樟树":"360982","高安":"360983","抚州":"361000","上饶":"361100","德兴":"361181","济南":"370100","市中区":"370103","青岛":"370200","市南区":"370202","市北区":"370203","胶州":"370281","平度":"370283","莱西":"370285","淄博":"370300","枣庄":"370400","市中区":"370402","滕州":"370481","东营":"370500","烟台":"370600","龙口":"370681","莱阳":"370682","莱州":"370683","蓬莱":"370684","招远":"370685","栖霞":"370686","海阳":"370687","潍坊":"370700","青州":"370781","诸城":"370782","寿光":"370783","安丘":"370784","高密":"370785","昌邑":"370786","济宁":"370800","曲阜":"370881","邹城":"370883","泰安":"370900","新泰":"370982","肥城":"370983","威海":"371000","荣成":"371082","乳山":"371083","日照":"371100","临沂":"371300","德州":"371400","乐陵":"371481","禹城":"371482","聊城":"371500","临清":"371581","滨州":"371600","邹平":"371681","菏泽":"371700","郑州":"410100","巩义":"410181","荥阳":"410182","新密":"410183","新郑":"410184","登封":"410185","开封":"410200","洛阳":"410300","偃师":"410381","平顶山":"410400","舞钢":"410481","汝州":"410482","安阳":"410500","林州":"410581","鹤壁":"410600","新乡":"410700","卫辉":"410781","辉县":"410782","焦作":"410800","沁阳":"410882","孟州":"410883","濮阳":"410900","许昌":"411000","禹州":"411081","长葛":"411082","漯河":"411100","三门峡":"411200","义马":"411281","灵宝":"411282","南阳":"411300","邓州":"411381","商丘":"411400","永城":"411481","信阳":"411500","周口":"411600","项城":"411681","驻马店":"411700","济源":"419001","武汉":"420100","黄石":"420200","大冶":"420281","十堰":"420300","丹江口":"420381","宜昌":"420500","宜都":"420581","当阳":"420582","枝江":"420583","襄阳":"420600","老河口":"420682","枣阳":"420683","宜城":"420684","鄂州":"420700","荆门":"420800","京山":"420882","钟祥":"420881","孝感":"420900","应城":"420981","安陆":"420982","汉川":"420984","荆州":"421000","沙市区":"421002","石首":"421081","洪湖":"421083","松滋":"421087","黄冈":"421100","麻城":"421181","武穴":"421182","咸宁":"421200","赤壁":"421281","随州":"421300","广水":"421381","恩施":"422801","利川":"422802","仙桃":"429004","潜江":"429005","天门":"429006","长沙":"430100","浏阳":"430181","宁乡":"430182","株洲":"430200","醴陵":"430281","湘潭":"430300","湘乡":"430381","韶山":"430382","衡阳":"430400","耒阳":"430481","常宁":"430482","邵阳":"430500","武冈":"430581","岳阳":"430600","汨罗":"430681","临湘":"430682","常德":"430700","津市":"430781","张家界":"430800","益阳":"430900","沅江":"430981","郴州":"431000","资兴":"431081","永州":"431100","怀化":"431200","洪江":"431281","娄底":"431300","冷水江":"431381","涟源":"431382","吉首":"433101","广州":"440100","韶关":"440200","乐昌":"440281","南雄":"440282","深圳":"440300","珠海":"440400","汕头":"440500","佛山":"440600","江门":"440700","台山":"440781","开平":"440783","鹤山":"440784","恩平":"440785","湛江":"440800","廉江":"440881","雷州":"440882","吴川":"440883","茂名":"440900","高州":"440981","化州":"440982","信宜":"440983","肇庆":"441200","四会":"441284","惠州":"441300","梅州":"441400","兴宁":"441481","汕尾":"441500","陆丰":"441581","河源":"441600","阳江":"441700","阳春":"441781","清远":"441800","英德":"441881","连州":"441882","东莞":"441900","中山":"442000","潮州":"445100","揭阳":"445200","普宁":"445281","云浮":"445300","罗定":"445381","南宁":"450100","柳州":"450200","桂林":"450300","荔浦":"450381","梧州":"450400","岑溪":"450481","北海":"450500","防城港":"450600","东兴":"450681","钦州":"450700","贵港":"450800","桂平":"450881","玉林":"450900","北流":"450981","百色":"451000","靖西":"451081","贺州":"451100","河池":"451200","来宾":"451300","合山":"451381","崇左":"451400","凭祥":"451481","海口":"460100","三亚":"460200","三沙":"460300","儋州":"460400","五指山":"469001","琼海":"469002","文昌":"469005","万宁":"469006","东方":"469007","重庆":"500000","重庆市郊县":"500200","成都":"510100","都江堰":"510181","彭州":"510182","邛崃":"510183","崇州":"510184","简阳":"510185","自贡":"510300","攀枝花":"510400","泸州":"510500","德阳":"510600","广汉":"510681","什邡":"510682","绵竹":"510683","绵阳":"510700","江油":"510781","广元":"510800","遂宁":"510900","内江":"511000","市中区":"511002","隆昌":"511083","乐山":"511100","市中区":"511102","峨眉山":"511181","南充":"511300","阆中":"511381","眉山":"511400","宜宾":"511500","广安":"511600","华蓥":"511681","达州":"511700","万源":"511781","雅安":"511800","巴中":"511900","资阳":"512000","马尔康":"513201","康定":"513301","西昌":"513401","贵阳":"520100","清镇":"520181","六盘水":"520200","盘州":"520281","遵义":"520300","赤水":"520381","仁怀":"520382","安顺":"520400","毕节":"520500","铜仁":"520600","兴义":"522301","兴仁":"522302","凯里":"522601","都匀":"522701","福泉":"522702","昆明":"530100","安宁":"530181","曲靖":"530300","宣威":"530381","玉溪":"530400","保山":"530500","腾冲":"530581","昭通":"530600","水富":"530681","丽江":"530700","普洱":"530800","临沧":"530900","楚雄":"532301","个旧":"532501","开远":"532502","蒙自":"532503","弥勒":"532504","文山":"532601","景洪":"532801","大理":"532901","瑞丽":"533102","芒":"533103","泸水":"533301","香格里拉":"533401","拉萨":"540100","日喀则":"540200","昌都":"540300","林芝":"540400","山南":"540500","那曲":"540600","西安":"610100","铜川":"610200","宝鸡":"610300","咸阳":"610400","彬州":"610482","兴平":"610481","渭南":"610500","韩城":"610581","华阴":"610582","延安":"610600","汉中":"610700","榆林":"610800","神木":"610881","安康":"610900","商洛":"611000","兰州":"620100","嘉峪关":"620200","金昌":"620300","白银":"620400","天水":"620500","武威":"620600","张掖":"620700","平凉":"620800","华亭":"620881","酒泉":"620900","玉门":"620981","敦煌":"620982","庆阳":"621000","定西":"621100","陇南":"621200","临夏":"622901","合作":"623001","西宁":"630100","海东":"630200","玉树":"632701","格尔木":"632801","德令哈":"632802","茫崖":"632803","银川":"640100","灵武":"640181","石嘴山":"640200","吴忠":"640300","青铜峡":"640381","固原":"640400","中卫":"640500","乌鲁木齐":"650100","新市区":"650104","克拉玛依":"650200","吐鲁番":"650400","哈密":"650500","昌吉":"652301","阜康":"652302","博乐":"652701","阿拉山口":"652702","库尔勒":"652801","阿克苏":"652901","阿图什":"653001","喀什":"653101","和田":"653201","伊宁":"654002","奎屯":"654003","霍尔果斯":"654004","塔城":"654201","乌苏":"654202","阿勒泰":"654301","石河子":"659001","阿拉尔":"659002","图木舒克":"659003","五家渠":"659004","北屯":"659005","铁门关":"659006","双河":"659007","可克达拉":"659008","昆玉":"659009","伊犁":"654000"] + def baiduCityCodeMap = ["北京": "131", "天津": "332", "石家庄": "150", "唐山": "265", "秦皇岛": "148", "邯郸": "151", "邢台": "266", "保定": "307", "张家口": "264", "承德": "207", "沧州": "149", "廊坊": "191", "衡水": "208", "太原": "176", "大同": "355", "阳泉": "357", "长治": "356", "晋城": "290", "朔州": "237", "晋中": "238", "运城": "328", "忻州": "367", "临汾": "368", "吕梁": "327", "呼和浩特": "321", "包头": "229", "乌海": "123", "赤峰": "297", "通辽": "64", "鄂尔多斯": "283", "呼伦贝尔": "61", "巴彦淖尔": "169", "乌兰察布": "168", "兴安盟": "62", "锡林郭勒盟": "63", "阿拉善盟": "230", "沈阳": "58", "大连": "167", "鞍山": "320", "抚顺": "184", "本溪": "227", "丹东": "282", "锦州": "166", "营口": "281", "阜新": "59", "辽阳": "351", "盘锦": "228", "铁岭": "60", "朝阳": "280", "葫芦岛": "319", "长春": "53", "吉林市": "55", "四平": "56", "辽源": "183", "通化": "165", "白山": "57", "松原": "52", "白城": "51", "延边朝鲜族自治州": "54", "哈尔滨": "48", "齐齐哈尔": "41", "鸡西": "46", "鹤岗": "43", "双鸭山": "45", "大庆": "50", "伊春": "40", "佳木斯": "42", "七台河": "47", "牡丹江": "49", "黑河": "39", "绥化": "44", "大兴安岭地区": "38", "上海": "289", "南京": "315", "无锡": "317", "徐州": "316", "常州": "348", "苏州": "224", "南通": "161", "连云港": "347", "淮安": "162", "盐城": "223", "扬州": "346", "镇江": "160", "泰州": "276", "宿迁": "277", "杭州": "179", "宁波": "180", "温州": "178", "嘉兴": "334", "湖州": "294", "绍兴": "293", "金华": "333", "衢州": "243", "舟山": "245", "台州": "244", "丽水": "292", "合肥": "127", "芜湖": "129", "蚌埠": "126", "淮南": "250", "马鞍山": "358", "淮北": "253", "铜陵": "337", "安庆": "130", "黄山": "252", "滁州": "189", "阜阳": "128", "宿州": "370", "巢湖": "251", "六安": "298", "亳州": "188", "池州": "299", "宣城": "190", "福州": "300", "厦门": "194", "莆田": "195", "三明": "254", "泉州": "134", "漳州": "255", "南平": "133", "龙岩": "193", "宁德": "192", "南昌": "163", "景德镇": "225", "萍乡": "350", "九江": "349", "新余": "164", "鹰潭": "279", "赣州": "365", "吉安": "318", "宜春": "278", "抚州": "226", "上饶": "364", "济南": "288", "青岛": "236", "淄博": "354", "枣庄": "172", "东营": "174", "烟台": "326", "潍坊": "287", "济宁": "286", "泰安": "325", "威海": "175", "日照": "173", "莱芜": "124", "临沂": "234", "德州": "372", "聊城": "366", "滨州": "235", "菏泽": "353", "郑州": "268", "开封": "210", "洛阳": "153", "平顶山": "213", "安阳": "267", "鹤壁": "215", "新乡": "152", "焦作": "211", "濮阳": "209", "许昌": "155", "漯河": "344", "三门峡": "212", "南阳": "309", "商丘": "154", "信阳": "214", "周口": "308", "驻马店": "269", "武汉": "218", "黄石": "311", "十堰": "216", "宜昌": "270", "襄阳": "156", "鄂州": "122", "荆门": "217", "孝感": "310", "荆州": "157", "黄冈": "271", "咸宁": "362", "随州": "371", "恩施土家族苗族自治州": "373", "仙桃": "1713", "潜江": "1293", "天门": "2654", "神农架林区": "2734", "长沙": "158", "株洲": "222", "湘潭": "313", "衡阳": "159", "邵阳": "273", "岳阳": "220", "常德": "219", "张家界": "312", "益阳": "272", "郴州": "275", "永州": "314", "怀化": "363", "娄底": "221", "湘西土家族苗族自治州": "274", "广州": "257", "韶关": "137", "深圳": "340", "珠海": "140", "汕头": "303", "佛山": "138", "江门": "302", "湛江": "198", "茂名": "139", "肇庆": "338", "惠州": "301", "梅州": "141", "汕尾": "339", "河源": "200", "阳江": "199", "清远": "197", "东莞": "119", "中山": "187", "潮州": "201", "揭阳": "259", "云浮": "258", "南宁": "261", "柳州": "305", "桂林": "142", "梧州": "304", "北海": "295", "防城港": "204", "钦州": "145", "贵港": "341", "玉林": "361", "百色": "203", "贺州": "260", "河池": "143", "来宾": "202", "崇左": "144", "海口": "125", "三亚": "121", "五指山": "1644", "琼海": "2358", "儋州": "1215", "文昌": "2758", "万宁": "1216", "东方": "2634", "定安": "1214", "屯昌": "1641", "澄迈": "2757", "临高": "2033", "白沙黎族自治": "2359", "昌江黎族自治": "1642", "乐东黎族自治": "2032", "陵水黎族自治": "1643", "保亭黎族苗族自治": "1217", "琼中黎族苗族自治": "2031", "重庆": "132", "成都": "75", "自贡": "78", "攀枝花": "81", "泸州": "331", "德阳": "74", "绵阳": "240", "广元": "329", "遂宁": "330", "内江": "248", "乐山": "79", "南充": "291", "眉山": "77", "宜宾": "186", "广安": "241", "达州": "369", "雅安": "76", "巴中": "239", "资阳": "242", "阿坝藏族羌族自治州": "185", "甘孜藏族自治州": "73", "凉山彝族自治州": "80", "贵阳": "146", "六盘水": "147", "遵义": "262", "安顺": "263", "铜仁地区": "205", "黔西南布依族苗族自治州": "343", "毕节地区": "206", "黔东南苗族侗族自治州": "342", "黔南布依族苗族自治州": "306", "昆明": "104", "曲靖": "249", "玉溪": "106", "保山": "112", "昭通": "336", "丽江": "114", "临沧": "110", "楚雄彝族自治州": "105", "红河哈尼族彝族自治州": "107", "文山壮族苗族自治州": "177", "普洱": "108", "西双版纳傣族自治州": "109", "大理白族自治州": "111", "德宏傣族景颇族自治州": "116", "怒江傈僳族自治州": "113", "迪庆藏族自治州": "115", "拉萨": "100", "昌都地区": "99", "山南地区": "97", "日喀则地区": "102", "那曲地区": "101", "阿里地区": "103", "林芝地区": "98", "西安": "233", "铜川": "232", "宝鸡": "171", "咸阳": "323", "渭南": "170", "延安": "284", "汉中": "352", "榆林": "231", "安康": "324", "商洛": "285", "兰州": "36", "嘉峪关": "33", "金昌": "34", "白银": "35", "天水": "196", "武威": "118", "张掖": "117", "平凉": "359", "酒泉": "37", "庆阳": "135", "定西": "136", "陇南": "256", "临夏回族自治州": "182", "甘南藏族自治州": "247", "西宁": "66", "海东地区": "69", "海北藏族自治州": "67", "黄南藏族自治州": "70", "海南藏族自治州": "68", "果洛藏族自治州": "72", "玉树藏族自治州": "71", "海西蒙古族藏族自治州": "65", "银川": "360", "石嘴山": "335", "吴忠": "322", "固原": "246", "中卫": "181", "乌鲁木齐": "92", "克拉玛依": "95", "吐鲁番地区": "89", "哈密地区": "91", "昌吉回族自治州": "93", "博尔塔拉蒙古自治州": "88", "巴音郭楞蒙古自治州": "86", "阿克苏地区": "85", "克孜勒苏柯尔克孜自治州": "84", "喀什地区": "83", "和田地区": "82", "伊犁": "90", "塔城地区": "94", "阿勒泰地区": "96", "石河子": "770", "阿拉尔": "731", "图木舒克": "792", "五家渠": "789", "香港特别行政区": "2912", "澳门特别行政区": "2911"] + + def allCitys = CommonDatautils.findAllCity(); + def allRegionMap = CommonDatautils.getAllCityRegions() + + static main(args) { + def task = new ChongQingBlankTest() + task.run() + } + + def run(){ + + ExcelUtils.modify("D:\\数据\\重庆农商行\\重庆农商行测试样本(房价网v.1.2).xlsx",{row-> + if(row['B'].toString()=="重庆市九龙坡区建工三村7号"){ + def city = CommonUtils.parseCityInAddress(row["B"]) + if(!city){ + city = CommonUtils.parseCityInAddress(row["C"]) + } + if(city){ + def oldMap = null + def similarData = null + if(isAddress(row["B"])){ + def pois = GaoDeUtils.search(row["C"], city, null) + oldMap = SeaweedDataMatchProcess.doMatch2(city, row['B'], null); + similarData = findSimilarDatas(pois, row["C"], row['B'], city, oldMap) + println row['C']+" "+pois.size() + if(oldMap && !oldMap.district){ + def json = GaoDeApiUtils.geo(city, row['B']) + if(json.info=="OK" && json.geocodes){ + def rows = json.geocodes + if(!rows[0].district){ + def detailMap = getMapRegion(rows[0].location) + if(detailMap){ + oldMap.district = detailMap.district + } + }else{ + oldMap.district = rows[0].district + } + } + } + row['D'] = oldMap.province + row['E'] = oldMap.city + if(oldMap.district){ + row['F'] = RegionData.formatRegion(city, oldMap.district) + } + if(oldMap.street){ + row['G'] = oldMap.street + }else{ + row['G'] = oldMap.township + } + + if(oldMap.streetNumbers){ + row['H'] = oldMap.streetNumbers + }else{ + row['H'] = oldMap.orgKeywords + } + }else{ + if(isAddress(row["C"])){ + def pois = GaoDeUtils.search(row["B"], city, null) + oldMap = SeaweedDataMatchProcess.doMatch2(city, row['C'], null); + similarData = findSimilarDatas(pois, row["B"], row['C'], city, oldMap) + println row['B']+" "+pois.size() + if(oldMap && !oldMap.district){ + def json = GaoDeApiUtils.geo(city, row['C']) + if(json.info=="OK" && json.geocodes){ + def rows = json.geocodes + if(!rows[0].district){ + def detailMap = getMapRegion(rows[0].location) + if(detailMap){ + oldMap.district = detailMap.district + } + }else{ + oldMap.district = rows[0].district + } + } + } + }else{ + def pois = GaoDeUtils.search(row["C"], city, null) + oldMap = SeaweedDataMatchProcess.doMatch2(city, row['B'], null); + similarData = findSimilarDatas(pois, row["C"], row['B'], city, oldMap) + if(oldMap && !oldMap.district){ + def json = GaoDeApiUtils.geo(city, row['B']) + if(json.info=="OK" && json.geocodes){ + def rows = json.geocodes + if(!rows[0].district){ + def detailMap = getMapRegion(rows[0].location) + if(detailMap){ + oldMap.district = detailMap.district + } + }else{ + oldMap.district = rows[0].district + } + } + } + } + + row['D'] = oldMap.province + row['E'] = oldMap.city + if(oldMap.district){ + row['F'] = RegionData.formatRegion(city, oldMap.district) + } + if(oldMap.street){ + row['G'] = oldMap.street + }else{ + row['G'] = oldMap.township + } + + if(oldMap.streetNumbers){ + row['H'] = oldMap.streetNumbers + }else{ + row['H'] = oldMap.orgKeywords + } + + } + if(similarData){ + def newAddress = ""; + if(similarData.adname){ + def region = similarData.adname.replace("[]","") + similarData.adname = RegionData.formatRegion(city, region) + } + if(similarData.type && similarData.type.contains("村庄") && !similarData.township){ + def m = getMapRegion(similarData.location) + if(m && m.township){ + similarData.township = m.township + } + } + + if(similarData.cityname){ + newAddress = newAddress + similarData.cityname + } + if(similarData.adname){ + newAddress = newAddress + similarData.adname + } + if(similarData.address){ + if(similarData.address!=similarData.adname && !similarData.address.contains(similarData.adname)){ + newAddress = newAddress + similarData.address + }else{ + newAddress = newAddress + similarData.name + } + + } + def newMap = SeaweedDataMatchProcess.doMatch2(city, newAddress.replace("[]",""), null); + + def level = getLevel(newMap, oldMap, similarData.type); +// row['D'] = similarData.name.toString() +// row['E'] = similarData.address.toString().replace("[]","") + row['N'] = level + if(similarData.pname){ + row['I'] = similarData.pname.replace("[]","").replace("省","").replace("市","") + } + if(similarData.cityname){ + row['J'] = similarData.cityname.replace("[]","") + } + + if(similarData.adname){ + row['K'] = similarData.adname.replace("[]","") + } + +// if(level>=4){ + if(newMap.street){ + row['L'] = newMap.street.replace("[]","") + }else{ + if(similarData.township){ + row['L'] = similarData.township + } + } +// } + +// if(level>=5){ + if(newMap.streetNumber){ + row['M'] = newMap.streetNumber.replace("[]","") + }else{ + if(newMap.orgKeywords){ + row['M'] = newMap.orgKeywords + } + } + +// } + println row + } + } + } + }, 0) + } + + + + def getLevel(newMap, oldMap, type){ + def level = 0; + if(newMap && oldMap){ + if((type && type.contains("村庄")) || !type){ + if(newMap.orgKeywords == oldMap.orgKeywords){ + return 5 + } + }else{ + if(newMap.streetNumber && oldMap.streetNumber && newMap.streetNumber == oldMap.streetNumber){ + return 5 + } + } + + + if(newMap.street && oldMap.street && newMap.street.contains(oldMap.street)){ + return 4 + } + + if(newMap.district && oldMap.district && (newMap.district.contains(oldMap.district) || oldMap.district.contains(newMap.district))){ + return level = 3 + } + + if(newMap.city && oldMap.city && newMap.city == oldMap.city){ + return level = 2 + } + + if(newMap.province == oldMap.province){ + return level = 1 + } + } + return level + } + + def findSimilarDatas(pois, name, address, city, rMap){ + def data = null + def maxScore = 0 + + for(def poi : pois){ + if(!poi.cityname){ + def m = getMapRegion(poi.location) + if(m){ + poi.cityname = m.city + poi.pname = m.province + poi.township = m.township + poi.adname = m.district + } + } + if(poi.cityname.contains(city)){ + def searchName = poi.name + def nameScore = CosineSimilarAlgorithm.getSimilarityRatio(searchName, name); + + def addressScore = 0 + if(rMap.street){ + def s = rMap.street+rMap.streetNumber + if(s && poi.address){ + addressScore = CosineSimilarAlgorithm.getSimilarityRatio(poi.address, s); + } + } + def totalScore = nameScore + addressScore + if(totalScore > maxScore){ + maxScore = totalScore + data = poi + data.score = totalScore + } + println searchName+" "+name+" "+totalScore + } + } + return data; + } + + def isAddress(address){ + def REG_ADDR1 = "[\u4e00-\u9fa5]{2,15}[路|巷|街|弄|街道|大道|大街|巷子|公路|马路|大道西段]\\d+"; + def pattern = Pattern.compile(REG_ADDR1); + def matcher = pattern.matcher(address) + if(matcher.find()){ + return true; + } + return false; + } + + def getMapRegion(def location){ + def jsonObj = GaoDeApiUtils.geocode(location) + def district = null + if(jsonObj && jsonObj.status=="1"){ + def detailMap = jsonObj.regeocode.addressComponent + if(!detailMap.city){ + if(detailMap.province && detailMap.province.toString().endsWith("市")){ + detailMap.city = detailMap.province + } + } + return detailMap + } + return district; + } + + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CityStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CityStat.groovy new file mode 100644 index 000000000..9b523bef2 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CityStat.groovy @@ -0,0 +1,580 @@ +package com.fangjia.script.gj.test + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicInteger +import java.util.concurrent.atomic.AtomicLong + +class CityStat { + + public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; + public static final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName2("fangjia_stat", "district_stat", lineUrl) + + public static final DBCollection BAD_CASE_COLL = MongoUtils.getCollectionByName2("fangjia_stat", "bad_case", lineUrl) + + public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) + static main(args) { + def task = new CityStat() +// task.run() +// task.run1(); + task.run3(); +// task.run4() +// task.run6() +// task.run7() +// task.run8() + } + +// def run8(){ +// ExcelUtils.multiThreadingModify("D:\\数据\\深圳建行\\建行匹配小区(3).xlsx",{d-> +// +// def dist = CommonDatautils.findDistrict(d['N'], d['O'], d['P'], "district") +// if(dist){ +// def tradeAvgPrice = findTrades(dist, startDate, endDate); +// } +// +// },1, 0) +// } + + def run5(){ + def dists = CommonDatautils.findAllDistrictByCity("常州","district",["city", "region","name","alias"]) + def total = 0 + def scount = 0 + def index = 0 + // final CountDownLatch latch = new CountDownLatch(districts.size()) + + for(def dist : dists){ + index++ + def rets = findTradeDetails(dist, "2020-03-24"); + println index + if(rets){ + total = total + rets.size() + def count = findSimialPriceCount(dist, rets); + scount = scount+count + } + } + println scount +" "+ total + def pre = DataHandleUtils.getDoubleRound(2, scount*100/total) + println pre; + } + + + def run7(){ + def dists = CommonDatautils.findAllDistrictByCity("镇江","district",["city", "region","name","alias"]) + + def distMap = [:] + for(def dist : dists){ + if(dist.alias) { + + if (!dist.alias.contains(dist.name)) { + dist.alias << dist.name + } + for(def name : dist.alias){ + distMap.put( dist.region +"#"+name, dist) + } + }else{ + distMap.put( dist.region +"#"+dist.name, dist) + } + } + def rets = findTradeDetails2("镇江","2020-03-24"); + def total = 0 + def scount = 0 + + def index = 0 + for(def t : rets){ + def key = t.region+"#"+t.district_name + def dist = distMap.get(key) + index++ + println index + if(dist){ + total++ + def startDate = DateUtils.add(DateUtils.formatToYYYYMMDD(t.trade_date),-7) + def endDate = DateUtils.add(DateUtils.formatToYYYYMMDD(t.trade_date),7) + def weekendPrice = CommonDatautils.queryDistrictPriceByDate(dist.city, dist.region, dist.name, startDate, endDate) + if(t.avg_price && weekendPrice){ + def mod = (weekendPrice - t.avg_price)*100/t.avg_price + def m = Math.abs(mod) + if(m<=15){ + scount++; + } + } + } + } + + println scount +" "+total + + } + + def run6(){ + def dists = CommonDatautils.findAllDistrictByCity("常州","district",["city", "region","name","alias"]) + + final CountDownLatch latch = new CountDownLatch(dists.size()) + def totalCount = new AtomicLong() + def tradeTotal = new AtomicLong() + def scount = new AtomicLong() + + for(def i=0;i + if(data['A']){ + def startDate = DateUtils.addMonth(data['D'], -3) + def endDate = DateUtils.addMonth(data['D'], 3) + def dist = CommonDatautils.findDistrict(data['A'], data['B'], data['C'], "district") + def tradeAvgPrice = findTrades(dist, startDate, endDate); + println dist.city +" "+dist.region+" "+dist.name +" "+tradeAvgPrice.toInteger() + data['H'] = tradeAvgPrice.toInteger() + } + }) + } + + def run4(){ + def citys = ["上海","北京","大连","厦门","深圳","杭州","成都","重庆","苏州","广州"] + def startDate = DateUtils.addMonth(new Date(), -3) + def endDate = "2021-02-01" + def result = [] + def allDistricts = [] + for(def city : citys) { + def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias"]) + allDistricts.addAll(districts) + println city +" "+districts.size() + + } + def rets = findDistrictTrade(allDistricts, startDate,endDate); + println rets.size() + + + } + + def findDistrictTrade(def districts, def startDate, def endDate){ + def rets = [] + final CountDownLatch latch = new CountDownLatch(districts.size()) + def final AtomicInteger count_ = new AtomicInteger(0) + for(def i=0;i + if(v && v.size()>0){ + def sum = v.sum() + def tradeAvgPrice = sum/v.size() + def arrs = k.split("#") + def stat = findDistrictStat(city, arrs[0], arrs[1], DateUtils.parseFromTightByPattern("2021-01-17", "yyyy-MM-dd")) + rets << [city, arrs[0], arrs[1], tradeAvgPrice.toInteger(), stat.avg_price] + } + } + println city +" 完成" + } + ExcelUtils.write("d:\\数据\\统计\\城市成交数据.xlsx", rets, "成交") + } + + def run2(){ + def datas = ExcelUtils.read("D:\\数据\\深圳建行\\建行匹配小区2.xlsx",0,1) + def cityDataMap = [:] + def index = 0 + for(def data : datas){ + index++ + println index+" "+data + def lists = cityDataMap.get(data['A']) + if(!lists){ + lists = [] + cityDataMap.put(data['A'], lists) + } + def dist = CommonDatautils.findDistrict(data['A'], data['B'], data['C'], "district") + dist.calDate = data['D'] + dist.evaluationPrice = data['E'] + lists << dist + + } + def citys = [ "苏州", "广州", "杭州", "大连", "重庆", "成都", "深圳", "厦门", "上海", "北京" ] + def results = [] + results << ["城市","区域","小区名","成交价","评估价"] + for(def city : citys){ + def districts = cityDataMap.get(city) + def rets = findAvgPriceTrades(city, districts) + results.addAll(rets) + } + ExcelUtils.write("D:\\数据\\深圳建行\\建行成交3.xlsx",results, "数据") + } + + def findAvgPriceTrades(city, districts){ + def rets = [] + final CountDownLatch latch = new CountDownLatch(districts.size()) + for(def i=0;i 3){ + avg_price = total_price/count + } + return avg_price + } + + + + def findTradeDetails(dist, startDate){ + def q = new BasicDBObject() + q.put("city", dist.city) + q.put("region", dist.region) + + if(dist.alias){ + if(!dist.alias.contains(dist.name)){ + dist.alias << dist.name + } + q.put("district_name", new BasicDBObject('$in', dist.alias)) + }else{ + q.put("district_name", dist.name) + } + + q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate))) + def dbcorsor = DEAL_PRICE_COLL.find(q) + + def rets = [] + for(def d : dbcorsor){ + if(d.avg_price){ + rets << d + } + } + + return rets + } + + + + def findTradeDetails2(city, startDate){ + def q = new BasicDBObject() + q.put("city", city) + q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate))) + def dbcorsor = DEAL_PRICE_COLL.find(q) + + def rets = [] + for(def d : dbcorsor){ + if(d.avg_price){ + rets << d + } + } + return rets + } + + + + def findStatCount(def city, def weekend, def value_type){ + def q = new BasicDBObject() + q.put("city", city) + q.put("weekend", weekend) + if(value_type){ + q.put("value_type", value_type) + } + def count = DIST_STAT_COLL.count(q) + return count; + } + + def findDistrictStat(def city, def region, def name, def weekend){ + def q = new BasicDBObject() + q.put("city", city) + q.put("region", region) + q.put("district_name", name) + q.put("weekend", weekend) + def obj = DIST_STAT_COLL.findOne(q) + return obj; + } + + def findBadCaseCount(def city, def weekend){ + def q = new BasicDBObject() + q.put("city", city) + q.put("weekend", weekend) + def count = BAD_CASE_COLL.count(q) + return count; + } + + def findDealPriceCount(def city, def tradeDate){ + def q = new BasicDBObject() + q.put("city", city) + q.put("trade_date", new BasicDBObject('$gte', tradeDate)) + def count = DEAL_PRICE_COLL.count(q) + return count; + } + + def calTradeDistCount(def city, def tradeDate, def districts){ + def q = new BasicDBObject() + q.put("city", city) + q.put("source","链家在线") + q.put("trade_date", new BasicDBObject('$gte', tradeDate)) + def dbcorsor = DEAL_PRICE_COLL.find(q) + + def distMap = [:] + for(def dist :districts){ + def lists = distMap.get(dist.region) + if(!lists){ + lists = [] + distMap.put(dist.region, lists); + } + lists << dist + } + + def rets = [] + for(def obj : dbcorsor){ + def dist = findIsExistsDB(obj.region, obj.district_name, distMap); + if(dist && !rets.contains(dist)){ + rets << dist + } + } + return rets.size(); + } + + def findIsExistsDB(def region, def district_name, def distMap){ + def dist = null; + def regionDists = distMap.get(region) + for(def rd : regionDists){ + def alias = rd.alias + if(!alias){ + alias = [] + } + if(!alias.contains(rd.name)){ + alias << rd.name + } + if(alias.contains(district_name)){ + dist = rd + break; + } + } + return dist; + } + + + def calTradeDistDetail(def city, def tradeDate, def districts){ + def q = new BasicDBObject() + q.put("city", city) + q.put("trade_date", new BasicDBObject('$gte', tradeDate)) + def dbcorsor = DEAL_PRICE_COLL.find(q) + + def distMap = [:] + for(def dist :districts){ + def lists = distMap.get(dist.region) + if(!lists){ + lists = [] + distMap.put(dist.region, lists); + } + lists << dist + } + + def retMap = [:] + for(def obj : dbcorsor){ + def dist = findIsExistsDB(obj.region, obj.district_name, distMap); + if(dist){ + def tradePrices = retMap.get(dist.region+"#"+dist.name) + if(!tradePrices){ + tradePrices = [] + retMap.put(dist.region+"#"+dist.name, tradePrices) + } + if(obj.avg_price){ + tradePrices << obj.avg_price.toInteger() + } + } + } + return retMap; + } + + + def calTradeDistDetail2(def city, def tradeDate, def districts){ + def q = new BasicDBObject() + q.put("city", city) + q.put("trade_date", new BasicDBObject('$gte', tradeDate)) + def dbcorsor = DEAL_PRICE_COLL.find(q) + + def distMap = [:] + for(def dist :districts){ + def lists = distMap.get(dist.region) + if(!lists){ + lists = [] + distMap.put(dist.region, lists); + } + lists << dist + } + + def retMap = [:] + for(def obj : dbcorsor){ + def dist = findIsExistsDB(obj.region, obj.district_name, distMap); + if(dist){ + def tradePrices = retMap.get(dist.region+"#"+dist.name) + if(!tradePrices){ + tradePrices = [] + retMap.put(dist.region+"#"+dist.name, tradePrices) + } + if(obj.avg_price){ + tradePrices << obj.avg_price.toInteger() + } + } + } + return retMap; + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CompanyAddressCompare.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CompanyAddressCompare.groovy new file mode 100644 index 000000000..081ede856 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CompanyAddressCompare.groovy @@ -0,0 +1,111 @@ +package com.fangjia.script.gj.test + +import com.fangjia.core.RegionData +import com.fangjia.script.gj.district.SeaweedDataMatchProcess +import com.fangjia.script.gj.paimai.GaoDeUtils +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.CosineSimilarAlgorithm +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils + +import java.util.regex.Pattern + +class CompanyAddressCompare { + static main(args) { + def task = new CompanyAddressCompare() + task.run() + } + def run(){ + ExcelUtils.modify("D:\\数据\\公司及地址\\公司及地址.xlsx",{row-> + if(row['D'] && !row['I'] && row['B'].contains("宁波甬城农村商业银行")){ + def city = CommonUtils.parseCityInAddress(row["D"]) + if(!city){ + city = CommonUtils.parseCityInAddress(row["B"]) + } + if(city){ + def similarData = null +// if(isAddress(row["D"])){ + def simpleAddress = row["D"].toString().replace("中国(上海)自由贸易试验区","").replaceAll(/\\d+层/,"").replaceAll(/[一二三四五六]楼/,"") + if(simpleAddress && simpleAddress.contains("号")){ + simpleAddress = simpleAddress.substring(0, simpleAddress.lastIndexOf("号")+1) + } + def pois = GaoDeUtils.search(simpleAddress, city, null) + similarData = findSimilarDatas(pois, row["B"], simpleAddress, city, null) + println row['C']+" "+pois.size() +// } + + if(similarData){ + row['F'] = similarData.address + row['G'] = similarData.name + row['H'] = similarData.score + } + } + } + }, 0) + } + + + def findSimilarDatas(pois, name, address, city, rMap){ + def data = null + def maxScore = 0 + + for(def poi : pois){ + if(!poi.cityname){ + def m = getMapRegion(poi.location) + if(m){ + poi.cityname = m.city + poi.pname = m.province + poi.township = m.township + poi.adname = m.district + } + } + if(poi.cityname.contains(city)){ + def searchName = poi.name + def nameScore = CosineSimilarAlgorithm.getSimilarityRatio(searchName, name); + def addressScore = 0 + if(poi.address){ + def poiAddress = poi.address.replace("号楼","号") + addressScore = CosineSimilarAlgorithm.getSimilarityRatio(poiAddress, address); + if(poi.address.contains("号") && address.contains("号") && (poiAddress.contains(address) || address.contains(poiAddress))){ + addressScore = addressScore + 0.5 + } + } + + def totalScore = nameScore + addressScore + if(totalScore > maxScore){ + maxScore = totalScore + data = poi + data.score = totalScore + } + println searchName+" "+name+" "+totalScore + " "+ (poi.address?poi.address:"") + } + } + return data; + } + + def isAddress(address){ + def REG_ADDR1 = "[\u4e00-\u9fa5]{2,20}[路|巷|街|弄|街道|大道|大街|巷子|公路|马路|大道西段]\\d+"; + def pattern = Pattern.compile(REG_ADDR1); + def matcher = pattern.matcher(address) + if(matcher.find()){ + return true; + } + return false; + } + + def getMapRegion(def location){ + def jsonObj = GaoDeApiUtils.geocode(location) + def district = null + if(jsonObj && jsonObj.status=="1"){ + def detailMap = jsonObj.regeocode.addressComponent + if(!detailMap.city){ + if(detailMap.province && detailMap.province.toString().endsWith("市")){ + detailMap.city = detailMap.province + } + } + return detailMap + } + return district; + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CrawlDistrictIdSelect.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CrawlDistrictIdSelect.groovy new file mode 100644 index 000000000..8e8bba001 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CrawlDistrictIdSelect.groovy @@ -0,0 +1,102 @@ +package com.fangjia.script.gj.test + +import com.fangjia.core.RegionData +import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +class CrawlDistrictIdSelect { + def static final DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol(); + + static DBCollection districtseedsColl = MongoUtils.getCollectionByAuth20190102("seeddb", "district_seeds") + static DBCollection districtseeds2Coll = MongoUtils.getCollectionByAuth20190102("seeddb_dev", "district_seeds") + + static main(args) { + def task = new CrawlDistrictIdSelect() + task.run() +// task.synch() + } + + def synch(){ + def q = new BasicDBObject() + def dbcursor = districtseeds2Coll.find(q) + def list = [] + for(def d :dbcursor){ + list << d + if(list.size()==10000){ + districtseedsColl.insert(list) + list = [] + } + } + + if(list.size()>0){ + districtseedsColl.insert(list) + } + } + + def run(){ + def datas = findCrawlData() + datas.each {k,v-> + if(v){ + for(def d : v){ + d.c_date = new Date() + d.m_date = new Date() + districtseedsColl.save(d) + } + } + } + } + + def findCrawlData(){ + def q = new BasicDBObject() + q.put("city","福州") + q.put("source", "安居客") + q.put("s_date",new BasicDBObject('$gte', 202001)) + def dbcursor = thirdPartyColl.find(q) + def retMap = [:] + def prefix = "/community/view/" + def keyList =[] + for(def db : dbcursor){ + def url = null + if(!db.url){ + if(db.district_id){ + url = db.district_id + } + }else{ + url = db.url + } + if(url && url.contains(prefix) ){ + def district_id = url.substring(url.indexOf(prefix)+prefix.length(),url.size()) + if(district_id=="0"){ + continue + } + def shortCity = url.substring(url.indexOf("https://")+8, url.indexOf(".anjuke.com")) + def region = RegionData.formatRegion(db.city, db.region) + def key = shortCity +"#"+district_id + + def ids = retMap.get(key) + if(!ids){ + ids = [] + retMap.put(key, ids) + } + + if(!keyList.contains(key)){ + def map = new BasicDBObject() + map.root = "anjuke.com" + map.city_short= shortCity + map.city = db.city + map.region = region + map.district_id = district_id + map.source_url = url + ids << map + + keyList << key + } + }else{ + println url +" 链接不合要求" + } + } + return retMap + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/ExportToDealPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/ExportToDealPrice.groovy new file mode 100644 index 000000000..d4440f329 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/ExportToDealPrice.groovy @@ -0,0 +1,101 @@ +package com.fangjia.script.gj.test + +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.QueryOperators + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicInteger + +class ExportToDealPrice { + + DBCollection DEAL_PRICE_MATCH_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price_match", "192.168.0.43") + DBCollection DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43") + + DBCollection TRADE_MATCH_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "trade_match_price", "192.168.0.136") + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(50) + final AtomicInteger count = new AtomicInteger() + + static main(args) { + def task = new ExportToDealPrice(); + task.run() + } + + def run(){ + def datas = findDatas() + + final CountDownLatch latch = new CountDownLatch(datas.size()) + for(def i=0;i0){ + TRADE_MATCH_PRICE_COLL.insert(ret); + } + } + + def addPrice(data){ + def weekend = DateUtils.getFirstDayOfWeek(data.trade_date) + def q = new BasicDBObject() + q.put("city", data.match_city) + q.put("region", data.match_region) + q.put("district_name", data.match_name) + q.put("weekend", weekend) + def dbo = DISTRICT_STAT_COLL.findOne(q) + if(!dbo){ + q.put("weekend", new BasicDBObject(QueryOperators.LTE, weekend)) + dbo = DISTRICT_STAT_COLL.findOne(q) + } + println count.get()+" "+data.city +" "+data.match_name + return dbo + } + + def findDatas(){ + def q = new BasicDBObject() + q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2020-04-01","yyyy-MM-dd"))) + def dbcursor = DEAL_PRICE_MATCH_COLL.find(q) + def rets = [] + for(def db : dbcursor){ + rets << db + } + return rets; + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/FZloationUpdate.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/FZloationUpdate.groovy new file mode 100644 index 000000000..0f65998a6 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/FZloationUpdate.groovy @@ -0,0 +1,182 @@ +package com.fangjia.script.gj.test + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.GeoHelper +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils +import com.fangjia.core.util.BaiDuApiUtils +import com.mongodb.BasicDBObject +import com.fangjia.script.util.DbUtils + +class FZloationUpdate { + + static main(args) { + def task = new FZloationUpdate(); + task.run() + } + + def run(){ + def districts = CommonDatautils.findAllDistricts("福州", "district", ["city", "region","name","address","lat","lng"]); + def index = 0 + def locationMap = getMoreSameLocationDist(districts) + def total = 0 + locationMap.each {k,v-> + total = updateLocation(v, total); + println "---------------------------------------------------------------------------------------------------------------" + } + } + + def updateLocation(def lists, def total){ + + + for(def dist : lists){ + + def addrDistMap = getLocation(dist, dist.address) + def nameDistMap = getLocation(dist, dist.region + dist.name) + + if((addrDistMap && addrDistMap.distance>1000) && (nameDistMap && nameDistMap.distance>1000)){ + println dist.region +" "+dist.name+" "+dist.address+" addrDist:"+(addrDistMap!=null?addrDistMap.distance:"")+" addrKey:"+(addrDistMap!=null?addrDistMap.keyword:"")+" nameDist:"+(nameDistMap!=null?nameDistMap.distance:"")+" nameKey:"+(nameDistMap!=null?nameDistMap.keyword:"") + // println "[${index}]"+ dist.region+" "+dist.name+" "+dist.address+" addrDist:"+addrDistMap.distance+"(${addrDistMap.level}-${addrDistMap.location}-${addrDistMap.keyword}) nameDist:"+nameDistMap.distance+"(${nameDistMap.level}-${nameDistMap.location}-${nameDistMap.keyword})" + + if(addrDistMap){ + def addrKey = deleteRegion(addrDistMap.keyword, dist.region).replace("|",""); + def nameKey = deleteRegion(nameDistMap.keyword, dist.region); + + def baidu = null + def lat = null + def lng = null + def upldateFlag = "" + if(addrDistMap && addrDistMap.region.contains(dist.region) && (addrKey.contains("路") || addrKey.contains("大道")) && addrKey.contains("号") && dist.addr == addrKey){ + lat = addrDistMap.lat + lng = addrDistMap.lng + baidu = BaiDuApiUtils.geoconv(lng, lat, 3) + upldateFlag = "Address" + }else if(nameDistMap && nameDistMap.region.contains(dist.region) && nameKey == dist.name){ + lng = nameDistMap.lng + lat = nameDistMap.lat + baidu = BaiDuApiUtils.geoconv(lng, lat, 3) + upldateFlag = "Name" + } + if(baidu){ + total++ + def udata = new BasicDBObject() + udata.put("m_date", new Date()) + Long baiduLat = baidu.y*10000000000L + Long baiduLng = baidu.x*10000000000L + udata.put("lat2", baiduLat) + udata.put("lng2", baiduLng) + Long lat0 = lat*10000000000L + Long lng0 = lng*10000000000L + udata.put("lat", lat0) + udata.put("lng", lng0 ) + DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata)) + println "[修改-${total}] "+ dist.region+" "+dist.name+" "+dist.address+" "+upldateFlag + } + + } + } + } + return total + } + + def getMoreSameLocationDist(def districts){ + def locationMap = [:] + for(def i=0;i + if(v.size()>=2){ + + v.sort{ a,b-> + return a.name.size().compareTo(b.name.size()) + } + resMap.put(k,v) + } + } + + def lastRetMaps = [:] + resMap.each{k,v-> + def swname = null + def count = 0 + for(def i=0;i-1){ + if(s.indexOf(region+"区")>-1){ + key = s.substring(s.indexOf(region+"区")+region.size()+1, s.size()); + }else if(s.indexOf(region+"县")>-1){ + key = s.substring(s.indexOf(region+"县")+region.size()+1, s.size()); + }else if(s.indexOf(region+"市")>-1){ + key = s.substring(s.indexOf(region+"市")+region.size()+1, s.size()); + } + }else{ + key = s; + } + return key + } + + + + def getLocation(dist, addr){ + + if(addr){ + def rMap = [:] + def reps = GaoDeApiUtils.geo(dist.city, addr); + if(reps && reps.count.toInteger()>=1){ + def map = reps.geocodes.get(0) + def location = map.location + def level = map.level + if(location){ + def locMap = location.split(",") + def lat = dist.lat/10000000000.0; + def lng = dist.lng/10000000000.0; + def distance = GeoHelper.GetDistance(locMap[1].toDouble(), locMap[0].toDouble(), lat, lng) + rMap.distance = distance + rMap.level = level + rMap.lat = locMap[1].toDouble() + rMap.lng = locMap[0].toDouble() + rMap.keyword = map.formatted_address + rMap.location = location + rMap.region = map.district + return rMap; + } + } + } + return null; + } + + + + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/FzLocationBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/FzLocationBatch.groovy new file mode 100644 index 000000000..4345374e2 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/FzLocationBatch.groovy @@ -0,0 +1,4 @@ +package com.fangjia.script.gj.test + +class FzLocationBatch { +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/JiangShuBlankDataStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/JiangShuBlankDataStat.groovy index cc5c8b7b3..838fcbd20 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/JiangShuBlankDataStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/JiangShuBlankDataStat.groovy @@ -30,7 +30,9 @@ class JiangShuBlankDataStat { // task.matchHouse(); // task.villa() - task.addressParse() +// task.addressParse() + + task.deleteProcess() } def repeat(){ @@ -288,5 +290,22 @@ class JiangShuBlankDataStat { return floor; } + def deleteProcess(){ + + def datas = ExcelUtils.read("D:\\数据\\江苏农行\\江苏2千.xlsx",0,1) + def addrList = [] + for(def d : datas){ + def addr = d['G'] + addrList << addr + } + ExcelUtils.multiThreadingModify("D:\\数据\\江苏农行\\200平以下.xlsx", { row -> + def addr = row['C'] + if(addrList.contains(addr)){ + row['R'] = "已调查" + } + + },0); + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/Test.groovy new file mode 100644 index 000000000..3dbb1a3e2 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/Test.groovy @@ -0,0 +1,7 @@ +package com.fangjia.script.gj.test + +import java.text.SimpleDateFormat + +SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); +Date date = new Date(1617211269868); +print dateFormat.format(date); \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/index/AutocompleteDataUpdaterBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/index/AutocompleteDataUpdaterBatch.groovy index 651c42cbb..59365a6a6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/index/AutocompleteDataUpdaterBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/index/AutocompleteDataUpdaterBatch.groovy @@ -18,8 +18,7 @@ import org.apache.commons.lang.StringUtils class AutocompleteDataUpdaterBatch extends BaseBatch { - - private static final DBCollection AC_COLL = MongoUtils.getCollectionByName("fangjia", "autocomplete", "192.168.0.51", 29009) + private static final DBCollection AC_COLL = MongoUtils.getCollectionByName("fangjia", "autocomplete", "mongo.db.fangjia.com") private static final DSCollection SEAWEED_COLL = new DataStore("seaweed.db.fangjia.com", 27007).getCollection("fangjia.seaweed") private static final DSCollection AUTOCOMPLETE_COLL = new DataStore("192.168.0.38", 27008).getCollection("fangjia.autocomplete") private static final long C_DATE = DateUtil.formatToYYYYMMDDhhmmss(new Date()).toLong() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/SeawaterMonitor.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/SeawaterMonitor.groovy new file mode 100644 index 000000000..8627efc19 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/SeawaterMonitor.groovy @@ -0,0 +1,156 @@ +package com.fangjia.script.temp.shenjian + +import com.fangjia.core.util.DateUtil +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.yinjihaun.util.DingDingMessageUtil +import org.apache.commons.lang.StringUtils + +/** + * 抓取总量监控 + * + * @author shenjian + * @since 2021/5/12 + */ +@BatchOwner(Watcher.ShenJian) +class SeawaterMonitor extends BaseBatch { + + static final Long expectNumber = 400000; + // 运维组 运维警报 + static final access_token = "2307d2fec8314b3e8b4b83dd1d42da2e2051638e3e668f21e77ca01258bf682c" + // 重点城市 + static final city = ["北京","唐山","天津","秦皇岛","石家庄","包头","太原","丹东","呼和浩特","锦州", + "沈阳","吉林","大连","牡丹江","长春","无锡","哈尔滨","扬州","上海","徐州", + "南京","温州","杭州","金华","宁波","蚌埠","合肥","安庆","福州","泉州", + "厦门","九江","南昌","赣州","济南","烟台","青岛","济宁","郑州","洛阳", + "武汉","平顶山","长沙","宜昌","广州","襄阳","深圳","岳阳","南宁","常德", + "海口","惠州","重庆","湛江","成都","韶关","贵阳","桂林","昆明","北海", + "西安","三亚","兰州","泸州","西宁","南充","银川","遵义","乌鲁木齐","大理", + "东莞","佛山","苏州","珠海","中山","台州","绍兴","南通","常州","嘉兴", + "淄博","株洲","镇江","肇庆","漳州","盐城","芜湖","泰州","汕头","清远", + "马鞍山","柳州","连云港","廊坊","淮安","湖州","沧州","保定","张家口","日照", + "拉萨","大同","文昌","陵水","保亭"]; + // 重点网站 + static final site = ["lianjia.com", "ke.com", "58.com", "zhuge.com", "fang.com"] + + static void main(String[] args) { + runBatch(new SeawaterMonitor(), args) + } + + /** + * 监控前一日抓取量 + * @return + */ + static def monitorYesterdayCrawlNumber(def startDate, def endDate) { + def total + String sql = "select count(*) as total from seawater where m_date >= '${startDate}' and m_date < '${endDate}' " + ElasticUtils.executeSqlQuery(sql,{data-> + total = data.total + }) + if (total < expectNumber) { + return "【总量抓取异常】 抓取量为" + total + ", 低于预期值" + expectNumber + } + return "" + } + + /** + * 统计重点城市近半月抓取量 + * @return + */ + static boolean countCityCrawlNumber(def startDate, def endDate) { + def result = [] + result << ['城市', '抓取量'] + // 统计近半个月各重点城市抓取量 + def cityFormatted = city.join("','") + String sql = "select * from (select count(*) as total, city from seawater where m_date >= '${startDate}' and m_date < '${endDate}' and city in('${cityFormatted}') group by city) order by total desc " + ElasticUtils.executeSqlQuery(sql,{data-> + result << [data.city, data.total] + }) + ExcelUtils.write("F:\\数据\\重点城市抓取\\近半月重点城市抓取量.xlsx", result, "抓取量") + } + + /** + * 统计前一日重点城市抓取量,昨天为零则警报 + * @return + */ + static def monitorCityCrawlNumber(def startDate, def endDate) { + def msg = "【重点城市抓取异常】" + def set = new HashSet() + def cityFormatted = city.join("','") + String sql = "select * from " + + " (select count(*) as total, city from seawater where m_date >= '${startDate}' and m_date < '${endDate}' and city in('${cityFormatted}') group by city) where total>0 " + //" order by total " + ElasticUtils.executeSqlQuery(sql,{data-> + set.add(data.city) + }) + def unNormal = [] + for (def info : city) { + if (!set.contains(info)) { + unNormal.add(info) + } + } + msg += unNormal.join("、") + " 抓取量为0" + if (unNormal.size() > 0) { + return msg + } + return "" + } + + /** + * 统计重点网站抓取情况,近一周为0则警报 + */ + static def monitorSiteCrawlNumber(def lastWeekDate, def endDate) { + def msg = "【重点网站抓取异常】" + def set = new HashSet() + def siteFormatted = site.join("','") + String sql = "select * from " + + " (select count(*) as total, root from seawater where m_date >= '${lastWeekDate}' and m_date < '${endDate}' and root in('${siteFormatted}') group by root) where total > 0 " + //" order by total " + ElasticUtils.executeSqlQuery(sql,{data-> + set.add(data.root) + }) + def unNormal = [] + for (def info : site) { + if (!set.contains(info)) { + unNormal.add(info) + } + } + msg += unNormal.join("、") + " 近一周抓取量为0" + if (unNormal.size() > 0) { + return msg + } + return "" + } + + @Override + void process(Object args) { + def lastWeekDate = DateUtil.add(DateUtil.truncateDay(new Date()), -7).getTime() + def yesterday = DateUtil.formatToYYYYMMDD(DateUtil.add(DateUtil.truncateDay(new Date()), -1)) + def startDate = DateUtil.add(DateUtil.truncateDay(new Date()), -1).getTime(); + def endDate = DateUtil.truncateDay(new Date()).getTime() + + def result = [] + def msg = monitorYesterdayCrawlNumber(startDate, endDate) + if (StringUtils.isNotBlank(msg)) { + result.add(msg) + } + + msg = monitorCityCrawlNumber(startDate, endDate) + if (StringUtils.isNotBlank(msg)) { + result.add(msg) + } + + msg = monitorSiteCrawlNumber(lastWeekDate, endDate) + if (StringUtils.isNotBlank(msg)) { + result.add(msg) + } + if (result.size() > 0) { + msg = "【日期】 " + yesterday + "\n" + result.join("\n") + println(msg) + DingDingMessageUtil.sendTextMessage(access_token, msg) + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy index cc38a3d50..15910f5cb 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy @@ -14,8 +14,8 @@ class TestCrawlTotalCount { def run(){ def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房"] for(def source :sources){ - def count1 = findTotalCount(source, 202101); - def count2 = findTotalCount(source, 202102); + def count1 = findTotalCount(source, 202102); + def count2 = findTotalCount(source, 202103); println source +" "+count1 +" "+count2 } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/other/AddFangSchoolRegion.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/other/AddFangSchoolRegion.groovy new file mode 100644 index 000000000..98646b77d --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/other/AddFangSchoolRegion.groovy @@ -0,0 +1,213 @@ +package com.fangjia.script.thirdPriceCrawl.other + +import com.fangjia.core.util.DataUtil +import com.fangjia.core.util.HttpRequest +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import com.mongodb.QueryOperators +import net.sf.json.groovy.JsonSlurper + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +class AddFangSchoolRegion { + + static DBCollection district_school_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "district_school") + static DBCollection third_price_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "third_party_price") + static DBCollection school_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "school2") + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10) + + static main(args) { + def task = new AddFangSchoolRegion(); + //task.process() + //task.match() + task.addMapLocation() + } + + def addMapLocation(){ + def q = new BasicDBObject() + def dbcursor = district_school_col.find(q) + def index = 0 + for(def d : dbcursor){ + def data = findSchool(d.city, d.region, d.school_name) + if(data && data.location){ + def location = data.location + if(location){ + def udata = new BasicDBObject() + udata.put("school_map_lng", location[0]) + udata.put("school_map_lat", location[1]) + district_school_col.update(new BasicDBObject('_id', d.get("_id")), new BasicDBObject('$set', udata)) + } + + } + index ++ + println index + } + } + + def match(){ + def datas = findAllDatas() + final CountDownLatch latch = new CountDownLatch(datas.size()) + for(def i=0;itotalPage){ + stop = true; + } + schools.addAll(data.list) + } + } + } + } + + print(schools.size()); + + def districtSchools = [] + def schoolInfos = [] + if(schools.size()>0){ + + def url = "https://51yongka.com/fapi/school/details" + for(def d : schools){ + def params = "schoolId="+d.id+"&access_token=oRqu15Mq_KxHfK0y0ZWhwktVnfUM" + def s = chaxuequInfoCrawl(url, params) + JSONObject jsonObject = JSONObject.fromObject(s); + if(jsonObject && jsonObject.code==200){ + + if(jsonObject.data && jsonObject.data.nearbyPropertyList){ + def districts = jsonObject.data.nearbyPropertyList + for(def dist : districts){ + def dataMap = new BasicDBObject() + dataMap.city = jsonObject.data.city + dataMap.region = RegionData.formatRegion(dataMap.city , jsonObject.data.area) + /**类型: 幼儿园,小学,初中*/ + dataMap.category = converCategory(jsonObject.data.schoolName) + dataMap.district_name = dist.name.trim() + dataMap.district_address = dist.address + dataMap.school_name = jsonObject.data.schoolName.trim() + dataMap.school_address = jsonObject.data.address + dataMap.source = "查学区房" + dataMap.url = url + "?"+params + dataMap.telephone = jsonObject.data.officialTelephone; + districtSchools << dataMap + } + }else{ + + def schoolData = new BasicDBObject() + schoolData.province = "江苏" + schoolData.city = jsonObject.data.city + schoolData.region = RegionData.formatRegion(schoolData.city , jsonObject.data.area) + /**类型: 幼儿园,小学,初中*/ + schoolData.category = converCategory(jsonObject.data.schoolName) + schoolData.name = jsonObject.data.schoolName.trim() + schoolData.address = jsonObject.data.address + /**来源网站*/ + schoolData.source = "查学区房" + /**房源详情url*/ + schoolData.telephone = jsonObject.data.officialTelephone; + schoolInfos << schoolData + println jsonObject.data.schoolName +" 无周边楼盘" + } + + } + println s; + } + } + + println "小区:"+districtSchools.size() +// saveChaxuequDistrictSchool(districtSchools) + saveChaxuequSchool(schoolInfos) + + } + + def saveChaxuequSchool(schoolInfos){ + for(def ds : schoolInfos){ + def obj = findSchoolData(ds.city, ds.school_name); + if(!obj){ + school_col.save(ds) + } + } + } + + + + + def saveChaxuequDistrictSchool(def districtSchools){ + for(def ds : districtSchools){ + def obj = findDistrictSchoolData(ds.city, ds.school_name); + if(!obj){ + district_school_col.save(ds) + } + } + } + + def findSchoolData(city, school_name){ + def q = new BasicDBObject() + q.put("city", city) + q.put("school_name", school_name) + return school_col.findOne(q) + } + + + def findDistrictSchoolData(city, school_name, district_name){ + def q = new BasicDBObject() + q.put("city", city) + q.put("school_name", school_name) + q.put("district_name", district_name) + return district_school_col.findOne(q) + } + + def converCategory(def schoolName){ + + if(schoolName){ + if(schoolName.contains("初中部")||schoolName.contains("中学")){ + return "初中" + }else if(schoolName.contains("小学")){ + return "小学" + } + } + return null + } + + def static chaxuequInfoCrawl(url, bodyquery){ + def responseMap = [:] + try { + //headers信息 + Map headers = new LinkedHashMap() + headers.put("Host","51yongka.com") + headers.put("Connection","keep-alive") + headers.put("Content-Length","75") + headers.put("User-Agent","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36 MicroMessenger/7.0.9.501 NetType/WIFI MiniProgramEnv/Windows WindowsWechat") + headers.put("content-type","application/x-www-form-urlencoded") + headers.put("Referer","https://servicewechat.com/wxc2461b2529576950/8/page-frame.html") + headers.put("Accept-Encoding","gzip, deflate, br") + String s = doPost(url, bodyquery, headers) + return s + } catch (Exception e) { + e.printStackTrace() + } + return null + } + + + def static schoolList(page) { + def responseMap = [:] + String s = null + try { + //headers信息 + Map headers = new LinkedHashMap() + headers.put("Host","m.gusuwang.com") + headers.put("Connection","keep-alive") + headers.put("Content-Length","40") + headers.put("User-Agent","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36 MicroMessenger/7.0.9.501 NetType/WIFI MiniProgramEnv/Windows WindowsWechat") + headers.put("content-type","application/x-www-form-urlencoded") + headers.put("Referer","https://servicewechat.com/wx9188a941e5ef95c5/10/page-frame.html") +// headers.put("Accept-Encoding","gzip, deflate, br") + + //params信息 + Map params = new HashMap(); + params.put("secCode","198a53da6baa3aa6a5b99959a006be37"); +// //post请求 + def url = "https://m.gusuwang.com/xuequ/xqapi.php?act=searchschool" + s = doPost(url, "secCode=198a53da6baa3aa6a5b99959a006be37&keys=&grade=&lb=&lx=&area=&currpage="+page,headers) + } catch (Exception e) { + e.printStackTrace() + } + return s + } + + + + + + + static String doPost(String strUrl, String content, Map headers) { + String result = ""; + OutputStream outputStream = null; + InputStream inputStream = null; + StringBuilder sb = new StringBuilder(); + try { + URL url = new URL(strUrl); + //通过调用url.openConnection()来获得一个新的URLConnection对象,并且将其结果强制转换为HttpURLConnection. + HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection(); + urlConnection.setRequestMethod("POST"); + //设置连接的超时值为30000毫秒,超时将抛出SocketTimeoutException异常 + urlConnection.setConnectTimeout(30000); + //设置读取的超时值为30000毫秒,超时将抛出SocketTimeoutException异常 + urlConnection.setReadTimeout(30000); + //将url连接用于输出,这样才能使用getOutputStream()。getOutputStream()返回的输出流用于传输数据 + urlConnection.setDoOutput(true); + urlConnection.setDoInput(true); + //设置通用请求属性为默认浏览器编码类型 +// urlConnection.setRequestProperty("content-type", "application/x-www-form-urlencoded"); + + + if (headers != null && headers.keySet().size() > 0) { + for (String head : headers.keySet()) { + urlConnection.setRequestProperty(head, headers.get(head)); + } + } + //getOutputStream()返回的输出流,用于写入参数数据。 + outputStream = urlConnection.getOutputStream(); + outputStream.write(content.getBytes()); + outputStream.flush(); + outputStream.close(); + //此时将调用接口方法。getInputStream()返回的输入流可以读取返回的数据。 + inputStream = urlConnection.getInputStream(); + + +// byte[] data = new byte[1024]; +// //inputStream每次就会将读取1024个byte到data中,当inputSteam中没有数据时,inputStream.read(data)值为-1 +// while (inputStream.read(data) != -1) { +// String s = new String(data, Charset.forName("utf-8")); +// sb.append(s); +// } + + InputStreamReader reader = new InputStreamReader(inputStream, "UTF-8"); + char[] buff = new char[1024]; + int length = 0; + while ((length = reader.read(buff)) != -1) { + String s = new String(buff, 0, length); + sb.append(s); + } + } catch (IOException e) { + e.printStackTrace(); + }finally{ + if(outputStream!=null){ + outputStream.close(); + } + if(inputStream!=null){ + inputStream.close(); + } + } + + return sb.toString(); + } + + + + + + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ABuYunUtil.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ABuYunUtil.groovy index 4bb2eaa12..f983a1c60 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ABuYunUtil.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ABuYunUtil.groovy @@ -425,6 +425,7 @@ class ABuYunUtil { buffer.append(bf, 0, len); } } catch (IOException e) { + e.printStackTrace() throw new Exception("doGet(String targetUrl, Map params, Map headers, String charset)异常, 异常信息为:" + e); } finally { // println("本次请求响应的connection.getResponseCode()为:" + connection.getResponseCode()) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy index 62c388a22..91cb7d57e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy @@ -44,7 +44,7 @@ class ElasticUtils { static getEsDataSource(String host){ EsDataSource dataSource = new EsDataSource (); - dataSource.setUrl("jdbc:es://${host}:9200?timezone=UTC&page.size=240&connect.timeout=240&network.timeout=600&page.timeout=240"); + dataSource.setUrl("jdbc:es://${host}:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000"); Properties connectionProperties = new Properties(); connectionProperties.setProperty("user","developer"); connectionProperties.setProperty("password","5e8edd851d2fdfbd7415232c67367cc3"); @@ -52,7 +52,7 @@ class ElasticUtils { return dataSource; } - static executeSqlQuery(String sql, Closure closure, String host = "192.168.0.216"){ + static executeSqlQuery(String sql, Closure closure, String host = "192.168.0.206"){ EsDataSource dataSource = getEsDataSource(host) Connection connection = dataSource.getConnection() Statement statement = connection.createStatement() @@ -140,13 +140,12 @@ class ElasticUtils { * @return */ static closeBulkProcessor(BulkProcessor bulkProcessor) { - bulkProcessor.awaitClose(10, TimeUnit.DAYS) bulkProcessor.flush() bulkProcessor.close() - } + static executeQuery(String host, String indexName, QueryBuilder qb, Closure closure){ RestHighLevelClient client = getClient(host) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/HouseUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/HouseUtils.groovy index 441f4bd9a..c403b9d8c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/HouseUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/HouseUtils.groovy @@ -1,19 +1,17 @@ package com.fangjia.script.util -import com.mongodb.DBCursor -import org.apache.commons.lang.StringUtils -import org.apache.lucene.index.Term -import org.apache.lucene.search.BooleanClause -import org.apache.lucene.search.BooleanQuery -import org.apache.lucene.search.TermQuery -import org.apache.solr.client.solrj.util.ClientUtils - import com.fangjia.datastore.driver.DSCursor import com.fangjia.datastore.driver.FacetResult import com.fangjia.script.util.DateUtils import com.fangjia.script.util.DbUtils import com.mongodb.BasicDBObject import com.mongodb.DBObject +import org.apache.commons.lang.StringUtils +import org.apache.lucene.index.Term +import org.apache.lucene.search.BooleanClause +import org.apache.lucene.search.BooleanQuery +import org.apache.lucene.search.TermQuery +import org.apache.solr.client.solrj.util.ClientUtils class HouseUtils{ static main(args){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy index 8c4593f1e..f35379b65 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy @@ -121,9 +121,9 @@ public class OpenApiClient { params.put("suggestName", suggestName) } - String content = new HttpRequest().get("http://192.168.10.199:9180/address/realMatch", params, "utf-8") + String content = new HttpRequest().get("http://open.fangjia.com/address/match", params, "utf-8") while (content == null) { - content = new HttpRequest().get("http://192.168.10.199:9180/address/realMatch", params, "utf-8") + content = new HttpRequest().get("http:///open.fangjia.com/address/match", params, "utf-8") println("${city} ${input} ${cat}程序调用异常") Thread.sleep(6000) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/ScriptFour.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/ScriptFour.groovy index 807fb6c36..2c873d35b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/ScriptFour.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/ScriptFour.groovy @@ -1,2559 +1,14 @@ +package com.fangjia.script.wenyan + import com.fangjia.core.RegionData -import com.fangjia.core.util.GeoUtils -import com.fangjia.script.util.CommonUtils -import com.fangjia.script.util.DbUtils -import com.fangjia.script.util.ExcelUtils -import com.fangjia.script.util.Jms -import com.fangjia.script.util.MongoUtils -import com.fangjia.script.util.OpenApiClient -import com.mongodb.BasicDBList -import com.mongodb.BasicDBObject -import com.mongodb.DBCollection -import net.sf.json.JSONObject +import com.fangjia.script.util.ElasticUtils -//import com.fangjia.core.RegionData -//import com.fangjia.core.util.DataUtil -//import com.fangjia.core.util.DateUtil -//import com.fangjia.core.util.HttpRequest -//import com.fangjia.script.util.CommonUtils -//import com.fangjia.script.util.DbUtils -//import com.fangjia.script.util.ExcelUtils -//import com.fangjia.script.util.GeoUtils -//import com.fangjia.script.util.MongoUtils -//import com.fangjia.script.util.OpenApiClient -//import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils -//import com.mongodb.BasicDBObject -//import com.mongodb.DBCollection -//import com.mongodb.DBObject -//import com.mongodb.Mongo -//import groovy.sql.Sql -//import net.sf.json.JSONArray -//import net.sf.json.JSONObject -//import org.apache.commons.codec.digest.DigestUtils -//import org.apache.commons.lang.StringUtils -//import org.apache.lucene.index.Term -//import org.apache.lucene.search.BooleanClause -//import org.apache.lucene.search.BooleanQuery -//import org.apache.lucene.search.TermQuery -// -//import java.util.concurrent.ExecutorService -//import java.util.concurrent.Executors -//import java.util.regex.Pattern -// -////import com.fangjia.core.RegionData -////import com.fangjia.core.util.DateUtil -////import com.fangjia.core.util.HttpRequest -////import com.fangjia.script.util.CommonUtils -////import com.fangjia.script.util.DataUtil -////import com.fangjia.script.util.DbUtils -////import com.fangjia.script.util.ExcelRow -////import com.fangjia.script.util.ExcelUtils -////import com.fangjia.script.util.GeoUtils -////import com.fangjia.script.util.MongoUtils -////import com.fangjia.script.util.OpenApiClient -////import com.mongodb.BasicDBObject -////import com.mongodb.Bytes -////import com.mongodb.DBCollection -////import com.mongodb.DBObject -////import com.mongodb.Mongo -////import com.mongodb.WriteConcern -////import com.mongodb.WriteResult -////import groovy.json.internal.Byt -////import net.sf.json.JSONObject -////import org.apache.commons.lang.StringUtils -////import org.apache.lucene.index.Term -////import org.apache.lucene.search.BooleanClause -////import org.apache.lucene.search.BooleanQuery -////import org.apache.lucene.search.TermQuery -////import org.apache.solr.client.solrj.util.ClientUtils -////import org.bson.types.ObjectId -//// -////import java.util.regex.Matcher -////import java.util.regex.Pattern -//// -//////package com.fangjia.script.wenyan -////// -//////import com.fangjia.core.RegionData -//////import com.fangjia.core.util.BaiDuApiUtils -//////import com.fangjia.core.util.CNNMFilter -//////import com.fangjia.core.util.DataUtil -//////import com.fangjia.core.util.DateUtil -//////import com.fangjia.core.util.DistrictCodeUtils -//////import com.fangjia.core.util.GeoUtils -//////import com.fangjia.core.util.HouseValidator -//////import com.fangjia.datastore.driver.DSCollection -//////import com.fangjia.datastore.driver.DSCursor -//////import com.fangjia.datastore.driver.DataStore -//////import com.fangjia.script.util.CommonUtils -//////import com.fangjia.script.util.CosineSimilarAlgorithm -//////import com.fangjia.script.util.DateUtils -//////import com.fangjia.script.util.DbUtils -//////import com.fangjia.script.util.ExcelUtils -//////import com.fangjia.script.util.HttpRequestUtil -//////import com.fangjia.script.util.OpenApiClient -//////import com.fangjia.script.wenyan.auto.KeywordSearchUtils -//////import com.fangjia.script.wenyan.auto.Poi -//////import com.mongodb.BasicDBObject -//////import com.mongodb.Bytes -//////import com.mongodb.DB -//////import com.mongodb.DBCursor -//////import com.mongodb.DBObject -//////import com.mongodb.Mongo -//////import net.sf.json.JSONArray -//////import net.sf.json.JSONObject -//////import org.apache.commons.lang.StringUtils -//////import org.apache.lucene.index.Term -//////import org.apache.lucene.search.BooleanClause -//////import org.apache.lucene.search.BooleanQuery -//////import org.apache.lucene.search.TermQuery -////// -//////import javax.swing.plaf.synth.Region -//////import java.util.regex.Matcher -//////import java.util.regex.Pattern -////// -///////** -////// * Created by wenyan on 2015/11/23. -////// */ -////////def data = ExcelUtils.read("E:\\Users\\wenyan\\Desktop\\??????????.xlsx", 1) -////////data.each {List item-> -//////// def addr = item[1].toString().replaceAll("\\s","") -//////// JSONObject resp = OpenApiClient.addressMatch("???", addr, "shop", false) -//////// if(resp.containsKey("name")){ -//////// String city = resp.getString("city"),region = resp.getString("district"),name = resp.getString("name") -//////// JSONObject res = HttpRequestUtil.getJsonContent("http://sh.sha-88.qa.fangjia.com/chafangtong/people/score2",[cityName:city,region:region,districtName:name,ajax:"1"]) -//////// if(res.containsKey("populationScore")){ -//////// println(addr + "\t" + res.officePopulationScore + "\t" + res.populationScore) -//////// }else{ -//////// println(addr) -//////// } -//////// }else{ -//////// println(addr) -//////// } -////////} -////////DBCollection ANALY_SITE_COLL = MongoUtils.getCollectionByName("coredata", "analy_road", "coredata.mongo.db.fangjia.com") -////////def idx = 0 -////////ANALY_SITE_COLL.find(new BasicDBObject(), new BasicDBObject("primary_map",1)).batchSize(10000).each {BasicDBObject dbo -> -//////// idx ++ -//////// if(idx % 100000 == 0){ -//////// println(idx) -//////// } -//////// BasicDBObject key = dbo.get("primary_map") -//////// if(key){ -//////// try{ -//////// if((key.getString("region") != null && key.getString("region").length() > 60) || key.getString("name").length() > 60){ -//////// println(dbo) -//////// ANALY_SITE_COLL.remove(new BasicDBObject("_id",dbo._id)) -//////// } -//////// }catch (Exception w){ -//////// println(dbo) -//////// } -//////// -//////// } -////////} -//////// -////////def countNoBuildTypeDistrictNum(){ -//////// def cityList = ["????","???","????","????","???","????","????","????","???","??????","????","????","???","???","??????","???","??","??","???","?人","????","????","???","????","????","???","????","????","??","????","????","????","???","????","????","????","???????","????","???","????","????","????","????","????","????","???","????","????","????","???","???","????","????","????","????","????","????","????","????","????","????","???","????","?????","????","????","????","???","????","???"] -//////// cityList.each {def city-> -//////// def has=0, no=0 -//////// DbUtils.executeQueryOnSeaweed(new BasicDBObject("cat", "district").append("visible", 0).append("city", city),{BasicDBObject dbo-> -//////// def buildType = dbo.build_type -//////// if(StringUtils.isBlank(buildType)){ -//////// no ++ -//////// }else{ -//////// has++ -//////// } -//////// }, new BasicDBObject("build_type", 1).append("city",1).append("region",1).append("name",1)) -//////// println("$city\t${has}\t${no}") -//////// } -////////} -//////// -////////def calArea(def city, def region, def name){ -//////// DSCollection SEABIRD = new DataStore("datastore.db.fangjia.com").getCollection("fangjia.seabird") -//////// def query = new BasicDBObject() -//////// query.append("city", city) -//////// query.append("region", region) -//////// query.append("district_name", name) -//////// query.append("cat", "sell") -//////// query.append("visible", 0) -//////// double area = 0 -//////// int count = 0 -//////// SEABIRD.find(query, new BasicDBObject("area", 1)).batchSize(20000).each {BasicDBObject dbo -> -//////// if(dbo.area && dbo.area > 0){ -//////// area += dbo.area -//////// count ++ -//////// } -//////// } -//////// if(area > 0){ -//////// return DataUtil.roundDouble(area/(double)count, 2) -//////// }else{ -//////// query.removeField("visible") -//////// DbUtils.seawater.find(query, new BasicDBObject("area", 1)).batchSize(20000).limit(20000).each {BasicDBObject dbo -> -//////// if(dbo.area && dbo.area > 0){ -//////// area += dbo.area -//////// count ++ -//////// } -//////// } -//////// if(area > 0){ -//////// return DataUtil.roundDouble(area/(double)count, 2) -//////// } -//////// return null; -//////// } -//////// -////////} -//////// -////////def getMarkUp(def city,def region,def name,def avgPrice){ -//////// def day = DateUtil.getFirstDayOfWeek(DateUtil.add(new Date(), Calendar.YEAR, -1)) -//////// BasicDBObject one = DbUtils.findOneOnline("district_stat", [city: city,region:region,district_name:name,"estate_type":"С??", weekend:day]); -//////// if(one && one.getDouble("avg_price") > 0){ -//////// return DataUtil.roundDouble(((avgPrice - one.getDouble("avg_price"))*100)/one.getDouble("avg_price"),2) -//////// -//////// } -//////// return null; -////////} -//////// -////////def exoprtDistrictArea(){ -//////// -//////// def orgData = ExcelUtils.read("E:\\Users\\wenyan\\Desktop\\????????????????.xlsx", 0 , 0) -//////// orgData.eachWithIndex{ List item, int i -> -//////// if(i != 0 && item[8] != null && item[2] != null){ -//////// if(item[13] != null && item[14] != null){ -//////// def res = OpenApiClient.evaluateDistrictPrice(item[6], item[7], item[8], item[2], 0, item[13], item[14]) -//////// item[4] = res?.totalPrice -//////// item[9] = res?.avgPrice -//////// }else{ -//////// def res = OpenApiClient.evaluateDistrictPrice(item[6], item[7], item[8], item[2], 0) -//////// item[4] = res?.totalPrice -//////// item[9] = res?.avgPrice -//////// } -//////// if(item[9]){ -//////// item[10] = getMarkUp(item[6], item[7], item[8],item[9]) -//////// } -//////// -//////// } -//////// println(item) -//////// } -//////// -//////// ExcelUtils.write("E:\\Users\\wenyan\\Desktop\\????????????????.xlsx",orgData,"????????") -////////} -////////exoprtDistrictArea() -////////def query = new BasicDBObject("city", "???").append("cat", "district").append("visible", 0); -////////def keys = new BasicDBObject("city",1) -////////keys.append("name",1) -////////keys.append("address",1) -////////keys.append("c_date",1) -////////keys.append("mark_map",1) -////////keys.append("ip",1) -////////int total = 0 -////////def cha = 0 -////////def addrDesc = ["??","???","??"] -////////DbUtils.seaweed.find(query, keys).batchSize(5000).each {BasicDBObject dbo-> -//////// def map = dbo.mark_map -//////// if(map && dbo.ip == "MACHINE_SAVE"){ -//////// total ++ -//////// if( addrDesc.contains(map.addr_precision)){ -//////// cha ++ -//////// } -//////// } -////////} -////////println("$total $cha") -////////DB mongo = new Mongo("mongo.db.fangjia.com", 27017).getDB("fangjia") -////////DBCollection DISTRICT_RELATION_COLL = mongo.getCollection("district_relation") -////////DISTRICT_RELATION_COLL.distinct("city") -//////// -////////def cityList = ["芜湖","淮南","揭阳","茂名","邯郸","安阳","南阳","四平","本溪","潍坊","淄博","大同","临汾","咸阳","绵阳","宜宾"] -////////println(cityList) -////////int groupCount = 0 -////////int count = 0 -////////cityList.each {def city-> -//////// int group = 0 ; -//////// int total = 0 -//////// DbUtils.executeQueryOnline("district_relation", new BasicDBObject("city", city).append("category","district"),{BasicDBObject dbo-> -//////// group ++ -//////// List relations = dbo.relations -//////// total += relations.size() -//////// }) -//////// count += total -//////// groupCount += group -//////// println("$city $group $total") -////////} -////////println(" $count $groupCount") -////// -////// -////////DBCollection REGION_BLOCK_COLL = MongoUtils.getCollectionByName("fangjia", "address", "192.168.0.136") -////////REGION_BLOCK_COLL.find(new BasicDBObject("is_match",1)).each {BasicDBObject dbo-> -//////// String addr = dbo.getString("addr") -//////// Pattern pattern = Pattern.compile(".*" + dbo.result.name +".+") -//////// if(addr ==~ pattern){ -//////// println(dbo) -//////// } -////////} -////////DBCollection REGION_BLOCK_COLL = MongoUtils.getCollectionByName("addrepo", "crawl_task", "192.168.0.61") -////////REGION_BLOCK_COLL.update(new BasicDBObject("taskType",0).append("status",1), new BasicDBObject('$set', new BasicDBObject("taskType", 1)),false,true) -////// -////////def data = ExcelUtils.read("E:\\Users\\wenyan\\Desktop\\商铺314.xls", 3, 0) -//////// -////////def cacheData = [] -////////def keys = [] -////////data.eachWithIndex{ List entry, int i -> -//////// String key = "" -//////// if(entry[3]){ -//////// key += entry[3] -//////// } -//////// if(entry[4]){ -//////// key += entry[4].toString() -//////// } -//////// if(entry[8]){ -//////// key += entry[8].toString() -//////// } -//////// if(entry[6]){ -//////// key += entry[6].toString() -//////// } -//////// if(!keys.contains(key)){ -//////// cacheData << entry -//////// keys << key -//////// } -////////} -////////ExcelUtils.write("E:\\Users\\wenyan\\Desktop\\商铺314.xls",cacheData,"解析楼层") -////// -////// -////// -////////cache.each {k,v-> -//////// println(v) -////////} -////////def data2 = ExcelUtils.read("E:\\迅雷下载\\合肥(梦).xls", 0, 0) -////////data2.eachWithIndex{ List entry, int i -> -//////// if(i != 0){ -//////// def item = cache.get(entry[0]) -//////// if(item != null && !(StringUtils.isNotBlank(entry[22]) || StringUtils.isNotBlank(entry[23]) || StringUtils.isNotBlank(entry[24]))){ -//////// entry[22] = item[22] -//////// entry[23] = item[24] -//////// entry[24] = item[23] -//////// } -//////// } -////////} -////////ExcelUtils.write("E:\\迅雷下载\\合肥(合).xls", data2,"房产数据测试") -////// -////////DbUtils.executeQueryOnline("town", new BasicDBObject(),{BasicDBObject dbo-> -//////// println("${dbo.city}\t${dbo.region}\t${dbo.name}\t${dbo.address}") -//////// -////////// String address = dbo.address -////////// try { -////////// String region = dbo.region -////////// String city = dbo.city -//////////// println(dbo) -////////// -////////// }catch (Exception e){ -//////////// println("-------------》" + dbo) -//////////// DbUtils.executeRemoveOnline("town", new BasicDBObject("_id", dbo._id)) -////////// } -//////// -////////}) -////////DBCollection REGION_BLOCK_COLL = MongoUtils.getCollectionByName("addrepo", "crawl_task", "192.168.0.61") -////////BasicDBObject dbo = REGION_BLOCK_COLL.findOne(new BasicDBObject("keyword","写字楼,住宅,公寓,大厦,别墅")) -////////def citys = ["鄂尔多斯","开封","朔州","呼伦贝尔","忻州","晋中","宝鸡","运城","渭南","邯郸","芜湖","佛山","东莞","齐齐哈尔","大同","绵阳","潍坊","常州","大庆"] -////////citys.each { -//////// dbo.removeField("_id") -//////// dbo.append("status",1) -//////// dbo.append("city",it) -//////// REGION_BLOCK_COLL.save(dbo) -////////} -////// -////////Pattern fenqiPattern = Pattern.compile(".*(\\w|\\d|东|南|西|北|一|二|三|四|五|六|七|八|九|十)+(区|期)\$") -//////// -////////DbUtils.executeQueryOnSeaweed(new BasicDBObject("cat","district").append("visible", 0),{BasicDBObject dbo-> -//////// try{ -//////// if(!(dbo.name ==~ fenqiPattern)){ -//////// Pattern pattern = Pattern.compile(dbo.getString("name") + "(\\w|\\d|东|南|西|北|一|二|三|四|五|六|七|八|九|十)+(区|期)") -//////// List alias = dbo.alias -//////// if(alias){ -//////// def hasFenQi = false -//////// Iterator it = alias.iterator() -//////// while(it.hasNext()){ -//////// def ali = it.next(); -//////// if(ali ==~ pattern){ -//////// def one = DbUtils.seaweed.findOne(new BasicDBObject("city", dbo.city).append("name", ali).append("region", dbo.region).append("cat","district").append("visible",0)) -//////// if(one){ -//////// hasFenQi = true -//////// it.remove() -//////// println("${dbo.toString()} ${ali}") -//////// } -//////// } -//////// } -//////// if(hasFenQi){ -//////// DbUtils.executeUpdateOnSeaweed(new BasicDBObject("_id",dbo._id), new BasicDBObject('$set', new BasicDBObject('alias', alias))) -//////// } -//////// } -//////// } -//////// -//////// }catch (Exception e){ -//////// -//////// } -//////// -//////// -////////}, new BasicDBObject("city", 1).append("region",1).append("name",1).append("alias",1)) -////// -////////Pattern fenqiPattern = Pattern.compile(".*(\\w|\\d|东|南|西|北|一|二|三|四|五|六|七|八|九|十)+(区|期)\$") -////////DbUtils.executeQueryOnSeaweed(new BasicDBObject("cat","district").append("visible", 0),{BasicDBObject dbo-> -//////// try{ -//////// if(!(dbo.name ==~ fenqiPattern)){ -//////// Pattern pattern = Pattern.compile(dbo.getString("name") + "(\\w|\\d|东|南|西|北|一|二|三|四|五|六|七|八|九|十)+(区|期)") -//////// List alias = dbo.alias -//////// if(alias){ -//////// def hasFenQi = false -//////// for(def ali : alias){ -//////// if(ali ==~ pattern){ -//////// hasFenQi = true -//////// break -//////// } -//////// } -//////// if(hasFenQi){ -//////// println("${dbo.city}\t${dbo.region}\t${dbo.name}\t${alias.join(",")}") -//////// } -//////// } -//////// } -//////// -//////// }catch (Exception e){ -//////// -//////// } -//////// -//////// -////////}, new BasicDBObject("city", 1).append("region",1).append("name",1).append("alias",1)) -////////def city70 = ["延边","三亚","上海","深圳","杭州","唐山","石家庄","兰州","安庆","合肥","乌鲁木齐","长春","赣州","九江","南昌","哈尔滨","长沙","太原","青岛","宜昌","武汉","沈阳","西安","成都","丹东","大连","郑州","昆明","宁波","金华","温州","北京","北海","南京","南宁","厦门","吉林","呼和浩特","大理","天津","扬州","无锡","广州","徐州","惠州","桂林","泉州","洛阳","济南","海口","湛江","烟台","福州","秦皇岛","襄阳","西宁","贵阳","重庆","银川","遵义","蚌埠","济宁","包头","锦州","牡丹江","平顶山","岳阳","常德","韶关","泸州","南充"] -////////def data = ExcelUtils.read("E:\\Users\\wenyan\\Desktop\\房价调查完成合并数据0324终-final16.xls") -////////def names = [] -////////int c = 0 -////////data.each {List row-> -//////// def city = row[11] -//////// if(StringUtils.isBlank(city)){ -//////// city = row[2] -//////// } -//////// def name = row[8] -//////// def cate = row[12] -//////// if(StringUtils.isBlank(cate) && city70.contains(city) && StringUtils.isNotBlank(name)){ -//////// BooleanQuery nameQuery = new BooleanQuery(); -//////// nameQuery.add(new TermQuery(new Term("name", name)), BooleanClause.Occur.SHOULD); -//////// nameQuery.add(new TermQuery(new Term("alias", name)), BooleanClause.Occur.SHOULD); -//////// BasicDBObject dbObject = DbUtils.seaweed.findOne(new BasicDBObject("city", city).append("visible", 0).append('$sr', new BasicDBObject("sr.query", nameQuery.toString()))) -//////// if(dbObject == null){ -//////// names << "${city}#${name}" -//////// } -//////// } -//////// println(c++) -////////} -////////names.unique() -//////// -////////def data2 = [] -////////data2 << ["序号", "城市", "名称"] -////////int index = 0 -////////names.each {String it-> -//////// index ++ -//////// data2 << [index, it.split("#")[0],it.split("#")[1]] -////////} -////////DBCollection REGION_BLOCK_COLL = MongoUtils.getCollectionByName("addrepo", "baidumap_source3", "192.168.0.61") -////////def data = ExcelUtils.read("E:\\Users\\wenyan\\Desktop\\房价调查.xls",0,0) -////////data.eachWithIndex{ List entry, int i -> -//////// if(i != 0 && entry[4] == null){ -//////// def one = REGION_BLOCK_COLL.findOne(new BasicDBObject("city",entry[1] + "市").append("name",Pattern.compile(".*" + entry[2] + ".*", Pattern.CASE_INSENSITIVE))) -//////// if(one){ -//////// entry[4] = "地图有相似:" + one.name -//////// println(entry) -//////// } -//////// -//////// } -////////} -////////ExcelUtils.write("E:\\Users\\wenyan\\Desktop\\房价调查.xls",data,"调查结果2") -////////DBCollection ANALY_ROAD_COLL = MongoUtils.getCollectionByName("coredata", "analy_site", "coredata.mongo.db.fangjia.com", 27017) -////////def city = "上海" -//////// def regions = [] -//////// ANALY_ROAD_COLL.find(new BasicDBObject("city", city).append("data_status", 2).append("cat","district"), new BasicDBObject("region",1)).each {BasicDBObject dbo-> -////////// if(!isNosie(city, dbo.region)){ -//////// regions << dbo.region -////////// } -//////// } -//////// regions.unique() -//////// regions.each { -//////// println("${city}\t${it}") -//////// } -//////// -//////// -////////def isNosie(String city ,String region){ -//////// if(city == region){ -//////// return true -//////// } -//////// def nosie = ["周边" , "其他", "其它" , "以外", "社区" , "地区" ,"家属区","小区",":","#"] -//////// for(def n: nosie){ -//////// if(region.contains(n)){ -//////// return true -//////// } -//////// } -//////// if(region ==~ /.*\w+.*/){ -//////// return true -//////// } -//////// if(region.contains("(") || region.contains(">") || region.contains(".") || region.contains("·") || region.contains("、")){ -//////// return true -//////// } -//////// if(region.length() < 2){ -//////// return true -//////// } -//////// if(!region.endsWith("区")){ -//////// return true -//////// } -//////// return false -////////} -////// -////////def data = ExcelUtils.read("C:\\Users\\wenyan\\Desktop\\经纬度地址测试.xlsx", 0 ,0) -////////data.eachWithIndex{ List item, int i -> -//////// if(i != 0 && i == 61){ -//////// final latSelf = item[1].toString().toDouble() -//////// final lngSelf = item[2].toString().toDouble() -//////// DBObject sdq = new BasicDBObject(); -//////// sdq.put("sd.point", latSelf + "," + lngSelf); -//////// sdq.put("sd.sfield", "lat_lng"); -//////// sdq.put("sd.distance", String.valueOf((double) 200 / 1000.0)); -//////// DBObject q = new BasicDBObject(); -//////// q.put("cat", "district"); -//////// q.put("visible", 0); -//////// q.put('$sd', sdq); -//////// BasicDBObject district = DbUtils.seaweed.findOne(q) -//////// if(district){ -//////// item[3] = district.name -//////// item[4] = district.address -//////// item[5] = GeoUtils.calculateDistance(district.lng2/10000000000, district.lat2/10000000000,lngSelf, latSelf) -//////// item[6] = district.city -//////// item[7] = 'db' -//////// }else{ -//////// -//////// JSONObject json = BaiDuApiUtils.geoconv(lngSelf, latSelf, 3) -//////// final baiduLng = json.getDouble("x") -//////// final baiduLat = json.getDouble("y") -//////// -//////// JSONObject result = BaiDuApiUtils.search("住宅区",baiduLng,baiduLat, 1000, 2, 10, 0) -//////// if(result.containsKey("results") && result.getJSONArray("results").size() > 0){ -//////// Collections.sort(result.getJSONArray("results"), new Comparator(){ -//////// @Override -//////// int compare(JSONObject o1, JSONObject o2) { -//////// Double d1 = GeoUtils.calculateDistance(baiduLng, baiduLat, o1.getJSONObject("location").getDouble("lng"), o1.getJSONObject("location").getDouble("lat")) -//////// Double d2 = GeoUtils.calculateDistance(baiduLng, baiduLat, o2.getJSONObject("location").getDouble("lng"), o2.getJSONObject("location").getDouble("lat")) -//////// return d1.compareTo(d2) -//////// } -//////// }) -//////// -//////// JSONObject dbo = result.getJSONArray("results")[0] -//////// double lat = dbo.getJSONObject("location").getDouble("lat") -//////// double lng = dbo.getJSONObject("location").getDouble("lng") -//////// item[3] = dbo.name -//////// item[4] = dbo.address -//////// item[5] = GeoUtils.calculateDistance(lng, lat, item[2].toString().toDouble(), item[1].toString().toDouble()) -//////// item[6] = BaiDuApiUtils.getCityByGeo(lng,lat).replace("市","") -//////// JSONObject resp = OpenApiClient.addressMatch(item[6],item[3], null, false) -//////// if(resp.getDouble("credit") > 0 && (resp.getString("category") == "office" || resp.getString("category") == "property") && resp.getString("suggestName") == item[3]){ -//////// if(resp.getString("address") != null){ -//////// item[4] = resp.getString("address") -//////// item[6] = resp.getString("city") -//////// }else{ -//////// def s = getStandardAddress(item[6],item[4]) -//////// if(s != null){ -//////// item[4] = s -//////// } -//////// } -//////// }else{ -//////// def s = getStandardAddress(item[6],item[4]) -//////// if(s != null){ -//////// item[4] = s -//////// } -//////// } -//////// } -//////// } -//////// -//////// -//////// -//////// -//////// println(item) -//////// } -////////} -////////ExcelUtils.write("C:\\Users\\wenyan\\Desktop\\经纬度地址测试.xlsx",data,"222") -//////// -//////// -////////def String getStandardAddress(def city, def addr) { -//////// if (StringUtils.isBlank(addr)) { -//////// return null -//////// } -//////// JSONObject result = OpenApiClient.addressStandardize(city, addr) -//////// if (result.containsKey("standardAddress") && result.getBoolean("standardAddress")) { -//////// def standardAddr = result.getString("street") -//////// if (result.containsKey("streetNumber")) { -//////// standardAddr += result.getString("streetNumber") -//////// } -//////// return standardAddr -//////// } -//////// return null -////////} -////////ExcelUtils.modify("E:\\Users\\wenyan\\Desktop\\房价调查 - 副本.xls",{List item-> -//////// def city = item[1] -//////// def name = item[2].toString().trim() -//////// item[6] = getCompareStatus(city, name) -////////}) -//////// -////////def getCompareStatus(def city, def name){ -//////// -//////// DBCollection ANALY_SITE_COLL = MongoUtils.getCollectionByName("coredata", "analy_site", "coredata.mongo.db.fangjia.com", 27017) -//////// def one = (BasicDBObject)ANALY_SITE_COLL.findOne(new BasicDBObject("cat","district").append("city", city).append("district_name", name), new BasicDBObject("data_status", 1)) -//////// if(one){ -//////// String dataStatus; -//////// switch (one.getInt("data_status")){ -//////// case 0:dataStatus = "正常";break; -//////// case 1:dataStatus = "人工删除";break; -//////// case 2:dataStatus = "区域错误";break; -//////// case 3:dataStatus = "被别名关联";break; -//////// case 4:dataStatus = "名称错误";break; -//////// case 5:dataStatus = "百度判定删除";break; -//////// case 6:dataStatus = "CPDB删除";break; -//////// case 7:dataStatus = "物业不匹配删除";break; -//////// default:dataStatus ="未知"; -//////// } -//////// return "对比数据存在:${dataStatus}" -//////// } -//////// return null -////////} -////////DBCollection BAIDU_MAP_COLL = MongoUtils.getCollectionByName("addrepo", "baidumap_source3", "192.168.0.61", 27017) -////////def ids = [] -////////BAIDU_MAP_COLL.find(new BasicDBObject(), new BasicDBObject("uid",1)).batchSize(10000).each {BasicDBObject dbo-> -//////// if(ids.contains(dbo.uid)){ -//////// BAIDU_MAP_COLL.remove(new BasicDBObject("_id", dbo._id)) -//////// println(dbo.uid) -//////// }else{ -//////// ids << dbo.uid -//////// } -////////} -////// -////////ExcelUtils.modify("D:\\1112222.xlsx",{List item-> -//////// if(item[0] && item[1] && item[2]){ -//////// try{ -//////// def one = DbUtils.findOneOnSeaweed(new BasicDBObject("city", item[0]).append("region", item[1]).append("name", item[2]).append("cat","district").append("visible",0)) -//////// if(one){ -//////// double lng = one.lng2/10000000000 -//////// double lat = one.lat2/10000000000 -//////// def point = getGeoPoint(item[4]) -//////// if(point){ -//////// item[6] = com.fangjia.script.util.GeoUtils.calculateDistance(lng,lat,point.lng, point.lat) -//////// } -//////// } -//////// -//////// -//////// }catch (Exception e){ -//////// -//////// } -//////// -//////// } -////////}) -////// -////////def getGeoPoint(String address){ -//////// JSONObject geoPoint = BaiDuApiUtils.geocoder(null,address) -//////// if(geoPoint.containsKey("result") && geoPoint.getJSONObject("result").containsKey("location")){ -//////// def point = [:] -//////// JSONObject location = geoPoint.getJSONObject("result").getJSONObject("location"); -//////// point.lng = location.getDouble("lng") -//////// point.lat = location.getDouble("lat") -//////// return point -//////// } -//////// return null; -////////} -////// -////////def data = ExcelUtils.read("D:\\1112222.xlsx",0,0) -////////def d = [] -////////data.eachWithIndex{ List entry, int i -> -//////// if(i == 0){ -//////// d << entry -//////// }else{ -//////// if(entry[6] < 1000 && StringUtils.isNotBlank(entry[1])){ -//////// d << entry -//////// } -//////// -//////// } -////////} -////////ExcelUtils.write("D:\\1112222.xlsx",d,"结果2") -//////// -////////DBCollection MAP_COLL = MongoUtils.getCollectionByName("addrepo", "baidumap_source3", "192.168.0.61", 27017) -////////DBCollection BAIDU_CITY_COLL = MongoUtils.getCollectionByName("addrepo", "baidu_citys", "192.168.0.61", 27017) -//////// -////////BAIDU_CITY_COLL.find().addOption(Bytes.QUERYOPTION_NOTIMEOUT).each { BasicDBObject city -> -//////// if(city.getInt("cityCode") != 1){ -//////// def keys = new HashSet() -//////// MAP_COLL.find(new BasicDBObject("cityCode", city.getInt("cityCode")), new BasicDBObject("format_region", 1).append("region", 1)).addOption(Bytes.QUERYOPTION_NOTIMEOUT).batchSize(10000).each {BasicDBObject dbo-> -//////// def region = dbo.region -//////// def formatRegion = dbo.format_region -//////// if(region == formatRegion && !RegionData.isRegion(city.name, formatRegion)){ -//////// keys << "${city.name} ${dbo.region}" -//////// } -//////// } -//////// keys.each { -//////// println(it) -//////// } -//////// } -////////} -////////DbUtils.seaweed.find(new BasicDBObject().append("cat", "district").append("visible", 0), new BasicDBObject("name", 1).append("city",1)).batchSize(10000).each {BasicDBObject dbo-> -//////// if(dbo.getString("name") ==~ /.*\d+期/){ -//////// println(dbo) -//////// } -////////} -////////def keys = new HashSet() -////////DBCollection street = new Mongo("mongo.db.fangjia.com", 27017).getDB('fangjia').getCollection('town'); -////////street.find().each { -//////// if(keys.contains(it.uid)){ -//////// street.remove(it) -//////// println(it) -//////// }else{ -//////// keys << it.uid -//////// } -////////} -////////ExcelUtils.modify("C:\\Users\\wenyan\\Documents\\自动入库数量.xlsx",{List row-> -//////// row[1] = DbUtils.executeCountOnSeaweed(new BasicDBObject("city", row[0]).append("cat","district").append("visible",0).append("source","machine")) -////////}) -////////println(false^false) -////// -////////DbUtils.executeQueryOnSeaweed(new BasicDBObject("cat", "district").append("visible", 0),{BasicDBObject dbo-> -//////// if(dbo.getString("name").contains("商住") && dbo.getString("estate_type2") != "商住"){ -//////// println(dbo) -//////// DbUtils.executeUpdateOnSeaweed(new BasicDBObject("_id", dbo._id), new BasicDBObject('$set', new BasicDBObject("estate_type2", "商住"))) -//////// } -////////}, new BasicDBObject("name",1).append("estate_type2",1)) -////// -////// -////// -////// -////////DbUtils.executeQueryOnline("street",new BasicDBObject("province","内蒙古自治区"),{BasicDBObject dbo -> -//////// dbo.append("province", "内蒙古") -//////// street.save(dbo) -//////// println(dbo) -//////// -////////}) -////// -////////def OUTPUT_FILE = "E:\\数据\\申银万国数据\\测试数据.xlsx" -////////ExcelUtils.multiThreadingModify(OUTPUT_FILE,{List item-> -//////// item[1] = item[0] * 2 -//////// println(item) -//////// -////////},0) -////// -////////def data = ExcelUtils.read("E:\\Users\\wenyan\\Desktop\\房价调查完成合并数据0324终-final16.xls",0) -////////def data2 = [] -////////data2 << ["序号","地址","原始城市","期望城市","匹配城市","期望小区名","匹配小区名"] -//////// -////////data.each { -//////// if(StringUtils.isNotBlank(it[11]) && it[2] != it[11]){ -//////// data2 << [it[0],it[4],it[2],it[11],null,it[3]] -//////// } -////////} -////////ExcelUtils.write("E:\\Users\\wenyan\\Desktop\\城市判定测试样本.xls", data2, "样本") -////// -////////ExcelUtils.multiThreadingModify("E:\\Users\\wenyan\\Desktop\\城市判定测试样本.xls",{ -//////// def city = it[2] -//////// def addr = it[1] -//////// def exCity = it[3] -//////// JSONObject resp = OpenApiClient.addressStandardize(city, addr) -//////// if(resp == null){ -//////// resp = OpenApiClient.addressStandardize(city, addr) -//////// } -//////// if(resp.containsKey("cityStatus") && "SURE" == resp.getString("cityStatus")){ -//////// def matchCity = resp.getString("city") -//////// if(matchCity.contains(exCity) || exCity.contains(matchCity)){ -//////// it[5] = "一致" -//////// }else{ -//////// it[5] = "不一致" -//////// } -//////// it[4] = matchCity -//////// }else if(resp.containsKey("candidateCities") && resp.getJSONArray("candidateCities").contains(exCity)){ -//////// it[5] = "包含" -//////// it[4] = resp.getJSONArray("candidateCities").join(",") -//////// }else if(resp.containsKey("candidateCities") && resp.getJSONArray("candidateCities").size() > 0){ -//////// it[4] = resp.getJSONArray("candidateCities").join(",") -//////// it[5] = "其他城市" -//////// }else{ -//////// it[5] = "失败" -//////// } -//////// println(it) -////////}) -////// -////////def data = [:] -////////DBCollection street = new Mongo("192.168.0.60", 27017).getDB('business').getCollection('open_uri_log'); -////////println street.count(new BasicDBObject("user_id","1007498").append("c_date",new BasicDBObject('$gt', Date.parse("yyyyMMdd","20160502")))) -////// -////////println(com.fangjia.script.util.DataUtil.formateAddress("芜湖","九华南路文昌路")) -////// -////// -////////def c = 0 -////////street.find(new BasicDBObject("user_id","1007498").append("c_date",new BasicDBObject('$gt', Date.parse("yyyyMMdd","20160502"))),new BasicDBObject("parameter_map",1)).skip(50*10000).batchSize(10000).each {BasicDBObject it-> -//////// def params = it.parameter_map as BasicDBObject -//////// params.remove("token") -//////// if(params.district && params.district.toString().contains("?") ){ -//////// c ++ -//////// }else{ -//////// println(params) -//////// } -////////} -////////println(c) -////// -////// -////// -////////DbUtils.executeQueryOnSeaweed(new BasicDBObject("visible", 0).append("cat", "district"),{BasicDBObject dbo-> -//////// if(dbo.name ==~ /.*期.+/){ -//////// println(dbo) -//////// } -////////},new BasicDBObject("name",1)) -////////CommonUtils.findAllCity().each { -//////// DbUtils.seaweed.find(new BasicDBObject().append("cat", "district").append("visible", 0).append("city", it), new BasicDBObject("name", 1).append("city",1).append("alias", 1).append("region", 1)).batchSize(10000).each {BasicDBObject dbo-> -//////// if(dbo.name ==~/.*期至.*期$/){ -//////// List alias = dbo.alias -//////// def name = alias.get(alias.size() -1) -//////// alias.remove(name) -//////// alias << dbo.name -//////// println(name) -//////// println(alias) -////////// DbUtils.seaweed.update(new BasicDBObject("_id", dbo._id) , new BasicDBObject('$set', new BasicDBObject("alias", alias).append("name",name))) -//////// } -//////// } -////////} -////////def data01 = ExcelUtils.read("e:\\Users\\wenyan\\Desktop\\申万数据0508结果.xlsx",0,0) -////////def data02 = ExcelUtils.read("e:\\Users\\wenyan\\Desktop\\申万数据0508结果.xlsx",1,1) -////////def data03 = ExcelUtils.read("e:\\Users\\wenyan\\Desktop\\申万数据0508结果.xlsx",2,1) -////////def data04 = ExcelUtils.read("e:\\Users\\wenyan\\Desktop\\申万数据0508结果.xlsx",3,1) -////////data01.addAll(data02) -////////data01.addAll(data03) -////////data01.addAll(data04) -//////// -////////def key = [] -////////def data = [] -//////// -////////data01.each { -//////// if(!key.contains(it[0].toString() + it[1].toString())){ -//////// data << it -//////// key << it[0].toString() + it[1].toString() -//////// } -////////} -////////ExcelUtils.write("e:\\Users\\wenyan\\Desktop\\申万数据0508去重.xlsx",data,"去重") -////// -////////ExcelUtils.multiThreadingModify("E:\\Users\\wenyan\\Desktop\\大地保险结果2.xls",{List data-> -//////// if(data[13] == null){ -//////// data[13] = data[7] -//////// } -//////// println(data) -////////}) -////// -////////def data = ExcelUtils.read("E:\\数据\\日志\\温州小区数据.xlsx") -////////def houseDataSet = [] -////////houseDataSet << ["城市", "区域", "小区名", "面积" , "版块", "建筑类型","朝向","装修","厅数","楼层","总楼层","户型","信息发布时间","室数","卫数","来源","剩余使用年限","地址","物业类型","建筑结构","开盘时间","小区均价","建造年代"," "] -////////data.each {List district-> -//////// -////////} -//////// -//////// -////////def List findHouse(city, region, name){ -//////// DBCollection houseTable = MongoUtils.getCollectionByName("fangjia_temp", "wz_houses", "192.168.0.136") -//////// List house = [] -//////// DBCursor cursor = houseTable.find(new BasicDBObject("city", city).append("region", region).append("district_name", name)) -//////// while(cursor.hasNext()){ -//////// house << cursor.next() -//////// } -//////// return house -////////} -////////final String TOKEN = "F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"; -////////def BULIDS_TYPE_CODE = [:] -////////BULIDS_TYPE_CODE.put("多层", "10"); -////////BULIDS_TYPE_CODE.put("小高层", "11"); -////////BULIDS_TYPE_CODE.put("超高层", "12"); -////////BULIDS_TYPE_CODE.put("高层", "13"); -////////BULIDS_TYPE_CODE.put("花园洋房", "14"); -////////BULIDS_TYPE_CODE.put("老洋房", "15"); -////////BULIDS_TYPE_CODE.put("四合院", "16"); -////////BULIDS_TYPE_CODE.put("独栋", "17"); -////////BULIDS_TYPE_CODE.put("双拼", "18"); -////////BULIDS_TYPE_CODE.put("联排", "19"); -////////BULIDS_TYPE_CODE.put("叠加", "20"); -//////// -//////// -//////// -////////ExcelUtils.multiThreadingModify("E:\\\\数据\\\\日志\\\\温州小区数据.xlsx",{row-> -//////// def district = DbUtils.seaweed.findById(row[0]) -//////// if(district){ -//////// row[13] = district.cubage_percent -//////// } -//////// println(row) -////////}) -////// -//////def dataSet = ExcelUtils.read("E:\\数据\\日志\\温州小区数据.xlsx") -//////def districtMap = [:] -//////dataSet.each { -////// def key = "${it[1]}#${it[2]}#${it[3]}" -////// districtMap[key.toString()] = it -////// -////// -//////} -//////def data = [] -//////def title = ["城市", "区域", "小区名","房屋面积","所属板块","建筑类型","朝向","装修","室数量","厅数量","卫数量","楼层","总楼层","户型","信息发布时间","房屋挂牌价格","房源均价","剩余使用年限","小区均价","地址","物业类型","建筑结构","开盘时间","学校数量","公交站数量","建造年代","绿化率(%)","容积率(%)","物业费(元/平米/月)","地铁站数量"] -//////DBCollection houseTable = MongoUtils.getCollectionByName("fangjia_tmp", "wz_houses", "192.168.0.136") -//////DBCursor cursor = houseTable.find().sort(new BasicDBObject("district_name", 1).append("region", 1)) -//////BasicDBObject house -//////int no = 1 -//////while(cursor.hasNext()){ -////// house = cursor.next() -////// def key = "${house.city}#${house.region}#${house.district_name}".toString() -////// List district = districtMap.get(key) -////// if(district){ -////// def h = [] -////// h[0] = house.city -////// h[1] = house.region -////// h[2] = house.district_name -////// h[3] = house.area -////// if(district[7] && district[7].toString().length() < 8){ -////// h[4] = district[7] -////// } -////// h[5] = district[5] -////// h[6] = HouseValidator.fetchDirectionInfo(house.direction) -////// h[7] = HouseValidator.fetchFitmentInfo(house.fitment) -////// if(house.room && house.room > 0 && house.room < 10){ -////// h[8] = house.room -////// } -////// if(house.hall && house.hall > 0 && house.hall < 10){ -////// h[9] = house.hall -////// } -////// if(house.toilet && house.toilet && house.toilet < 10){ -////// h[10] = house.toilet -////// } -////// if(house.height && house.height < 40){ -////// h[12] = house.height -////// } -////// if(house.floor && ((house.height && house.floor < house.height ) || (house.floor < 40))){ -////// h[11] = house.floor -////// } -////// -////// h[13] = getHouseType(h[8] , h[9], h[10]) -////// h[14] = ((Date)house.publish_date)?.format("yyyy-MM-dd") -////// h[15] = house.total_price -////// if(house.total_price && house.area){ -////// h[16] = (house.total_price/house.area).toDouble().toInteger() -////// -////// }else{ -////// continue -////// } -////// h[17] = district[9] -////// h[18] = district[8] -////// if(h[18] && h[16] && (Math.abs(h[18] - h[16])/h[16]) > 0.3){ -////// continue -////// } -////// -////// h[19] = district[6] -////// if(h[19]){ -////// h[19] = h[19].toString().replaceAll("\\s", "").replaceAll("\\." ,"").replaceAll("·","") -////// } -////// h[20] = district[4] -////// h[22] = district[16] -////// h[23] = district[11] -////// h[24] = district[10] -////// h[25] = district[17] -////// h[26] = district[12] -////// h[27] = district[13] -////// h[28] = district[14] -////// h[29] = district[15] -////// if(data.isEmpty()){ -////// data << title -////// } -////// data << h -////// if(data.size() % 40000 == 0){ -////// ExcelUtils.write("E:\\数据\\日志\\温州小区房源数据${no}.xlsx".toString(), data, "房源") -////// data.clear() -////// no++ -////// } -////// } -//////} -//////ExcelUtils.write("E:\\数据\\日志\\温州小区房源数据${no}.xlsx",data,"房源") -////// -////// -////// -//////public String getDomainName(String url) { -////// if (StringUtils.isNotBlank(url)) { -////// Pattern p = Pattern.compile("[^//]*?\\.(com|cn|net|org|biz|info|cc|tv)", Pattern.CASE_INSENSITIVE); -////// Matcher matcher = p.matcher(url); -////// if (matcher.find()) { -////// return matcher.group(); -////// } -////// -////// } -////// return "fangjia.com"; -//////} -////// -////// String getHouseType(room, hall, toilet) { -////// StringBuilder sb = new StringBuilder(); -////// if (room != null) { -////// sb.append(room + "室"); -////// if (hall != null) { -////// sb.append(hall + "厅"); -////// } -////// if (toilet != null) { -////// sb.append(toilet + "卫"); -////// } -////// } -////// return sb.toString(); -//////} -////// -//////public String getOpenDate(BasicDBObject district){ -////// if(district.open_date !=null){ -////// return DateUtil.formatToYYYYMMDD(district.open_date); -////// } -////// -////// if(StringUtils.isNotBlank(district.live_date)){ -////// return DateUtil.formatToYYYYMMDD(DateUtil.parseStringToDate(district.live_date)); -////// } -////// if(StringUtils.isNotBlank(district.finish_building_date)){ -////// return DateUtil.formatToYYYYMMDD(DateUtil.parseStringToDate(district.finish_building_date)); -////// } -////// return null; -//////} -////// -//////private Integer getYearFormDate(String dateStr) { -////// if (dateStr != null && StringUtils.isNotBlank(dateStr)) { -////// Date date = DateUtil.parseStringToDate(dateStr); -////// if (date != null) { -////// int year = date.getYear() + 1900; -////// return year; -////// } -////// } -////// return null; -//////} -////// -//////private Integer getBuildYear(String finishDate, String liveDate, String buildDate ,Date openDate) { -////// -////// Integer finishYear = getYearFormDate(finishDate); -////// Integer liveYear = getYearFormDate(liveDate); -////// Integer buildYear = getYearFormDate(buildDate); -////// Integer openYear = null; -////// if(openDate != null){ -////// openYear = Integer.valueOf(DateUtil.formatByPattern(openDate,"yyyy")); -////// } -////// int minYear = Integer.MAX_VALUE; -////// if (finishYear != null && finishYear < minYear) { -////// minYear = finishYear; -////// } -////// if (liveYear != null && liveYear < minYear) { -////// minYear = liveYear; -////// } -////// if (buildYear != null && buildYear < minYear) { -////// minYear = buildYear; -////// } -////// if(openYear != null && openYear < minYear){ -////// minYear = openYear; -////// } -////// if(minYear == Integer.MAX_VALUE){ -////// return null; -////// } -////// return minYear; -//////} -////// -//////private Integer extractYear(String s) { -////// Pattern p = Pattern.compile("(\\d+)"); -////// Matcher m = p.matcher(s); -////// if (m.find()) { -////// return Integer.parseInt(m.group(1)); -////// } -////// return null; -//////} -////// -////// -////// -////// -////// -////// -////// -////// -////// -////// -////// -////// -////// -////// -////// -////// -////// -////// -////// -////// -////// -////// -////// -////// -////// -////// -////// -//////ExcelUtils.multiThreadingModify("E:\\Users\\wenyan\\Desktop\\副本房产测试数据.xlsx",{List data-> -////// if(data[0]){ -////// def city = data[0].toString().replaceAll("市\$" ,""); -////// def keywords = [] -////// if(StringUtils.isNotBlank(data[2])){ -////// keywords << data[2] -////// } -////// if(StringUtils.isNotBlank(data[3])){ -////// keywords << data[3] -////// } -////// -////// if(StringUtils.isNotBlank(data[4])){ -////// keywords << data[4] -////// } -////// JSONObject resp; -////// for(def k : keywords){ -////// resp = OpenApiClient.addressMatch(city, k , "district", false) -////// double credit = resp.getDouble("credit") -////// if(credit > 0){ -////// data[8] = resp.getString("city") -////// data[9] = resp.getString("district") -////// data[10] = resp.getString("suggestName") -////// JSONObject evl = OpenApiClient.evaluateDistrictPrice(data[8], data[9] , resp.getString("name") , data[5].toString().toDouble(), data[10].contains("别墅")?1:0) -////// if(evl != null){ -////// data[6] = evl.avgPrice -////// data[7] = evl.totalPrice -////// } -////// break -////// -////// } -////// } -////// } -//////}) -//////DbUtils.executeQueryOnSeaweed(new BasicDBObject("cat", "district").append("visible", 0),{BasicDBObject dbo-> -////// if(!RegionData.isRegion(dbo.city, dbo.region)){ -////// BasicDBObject d = DbUtils.seaweed.findById(dbo._id.toString(), new BasicDBObject()) -////// println(d) -////// -////// } -//////},new BasicDBObject("region", 1).append("region", 1)) -//////for(int i = 1 ;i < 100;i++){ -////// if(i % 10 == 0){ -////// Thread.sleep(1000 * 1) -////// } -////// Jms jms = Jms.create('tcp://192.168.0.132:61616'); -////// jms.sendSms("1111","【房价网】《房调报告》用户手机13600323919,邮箱knln@qq.com,已成功付款,请及时审核", false) -//////} -//// -//////def cityList = CommonUtils.findAllCity() -////////cityList = ["白山"] -//////cityList.each {city-> -////// def keys = [] -////// DbUtils.executeQueryOnSeaweed(new BasicDBObject("cat", "district").append("visible", 0).append("city", city),{BasicDBObject it-> -////// if(it.name ==~ /.*(\(|)|\)|().*/){ -////// println(it) -////// } -//////// def key = city + "_" + it.region + "_" + it.name -//////// keys << key -////// -////// },new BasicDBObject("name", 1).append("city", 1).append("region", 1)) -//// -////// DbUtils.executeQueryOnSeaweed(new BasicDBObject("cat", "district").append("visible", 0).append("city", city),{BasicDBObject it-> -////// if(it.name ==~ Pattern.compile(".*(\\(|()" + it.region + "(\\)|)).*")){ -////// -////// def mainName = it.getString("name").replaceAll("(\\(|()" + it.region + "(\\)|))" ,"").toUpperCase() -////// def key =city + "_" + it.region + "_" + mainName -////// if(keys.contains(key)){ -////// DbUtils.seaweed.remove(new BasicDBObject("_id", it._id)) -////// println("删除:" + it) -////// }else{ -////// def data = new BasicDBObject("name", mainName) -////// DbUtils.executeUpdateOnSeaweed(new BasicDBObject("_id", it._id), new BasicDBObject('$set', data)) -////// println(it.toString() + data) -////// keys << key -////// } -////// } -////// },new BasicDBObject("name", 1).append("city", 1).append("region", 1)) -//////} -//// -///////** -////// * 把小区修改成删除状态 -////// * @param dbo -////// * @return -////// */ -//////def removeDistrict(BasicDBObject dbo){ -////// DbUtils.seaweed.update(new BasicDBObject("_id",dbo._id), new BasicDBObject('$set', new BasicDBObject("status", 1).append("m_date", new Date()))) -////// println("删除:" + dbo) -//////} -//////new File("E:\\数据\\New Folder").eachFile {def file -> -////// file.eachLine("gbk",{ -////// if(it.contains("RegionBlock") && it.contains("保亭")){ -////// println(it) -////// } -////// }) -//////} -//////CommonUtils.findAllCity().each { -////// DbUtils.executeQueryOnSeaweed(new BasicDBObject("cat","district").append("visible", 0).append("city", it),{BasicDBObject dbo-> -////// if(dbo.ip == 'MAP_SAVE' && dbo.estate_type2 == null){ -////// DbUtils.seaweed.update(new BasicDBObject("_id", dbo._id), new BasicDBObject('$set',new BasicDBObject("estate_type2", "普通住宅"))) -////// println(dbo) -////// } -////// }, new BasicDBObject("name",1).append("city",1).append("ip",1).append("estate_type2", 1)) -//////} -////// Pattern FUZZY_STAGE_PATTERN = Pattern.compile(".*(一|二|三|四|五|六|七|八|九|十|\\d)+期\$"); -//////DbUtils.executeQueryOnSeaweed(new BasicDBObject("cat", "district").append("visible", 0),{BasicDBObject dbo-> -////// if(dbo.name ==~ FUZZY_STAGE_PATTERN && (dbo.mark_map == null || !dbo.mark_map.baiDuCoordLock)){ -////// def mainName = dbo.getString("name").replaceAll("(一|二|三|四|五|六|七|八|九|十|\\d)+期\$", "") -////// if(mainName.trim().length() > 2){ -////// def query = new BasicDBObject("city", dbo.city).append("region", dbo.region).append("name", mainName).append("cat", "district").append("visible", 0) -////// BasicDBObject one = DbUtils.seaweed.findOne(query) -////// if(one){ -////// if(one.mark_map && one.mark_map.baiDuCoordLock){ -////// def update = new BasicDBObject() -////// update.append("lat", one.lat) -////// update.append("lng", one.lng) -////// update.append("lat2", one.lat2) -////// update.append("lng2", one.lng2) -////// try{ -////// DbUtils.seaweed.update(new BasicDBObject("_id", dbo._id), new BasicDBObject('$set', update)) -////// }catch (Exception e){ -////// println("Error !!!!!!!!!!!!!!!" + query) -////// } -////// }else{ -////// println(dbo) -////// } -////// } -////// } -////// -////// } -//////}, new BasicDBObject("name", 1).append("city", 1).append("region", 1).append("mark_map",1)) -//////DBCollection GAODE_POI_COLL = MongoUtils.getCollectionByName("addrepo", "gaode_poi", "192.168.0.61", 27017) -//////GAODE_POI_COLL.find().each {BasicDBObject dbo-> -////// if(!DataUtil.vectorContains(dbo.getString("cityname"),dbo.city)){ -////// println(dbo) -////// } -//////} -//////DBCollection db136 = MongoUtils.getCollectionByName("business", "house2", "192.168.0.136") -//////def data = [] -//////def title = ["城市","区域","小区名","项目地址","房屋类型","销售面积","销售金额","房龄","所在楼层","室","厅","卫","容积率(百分比)","绿化率(百分比)","楼龄","挂牌时间","公园(数量/最近距离)","超市(数量/最近距离)","购物中心(数量/最近距离)","医院(数量/最近距离)","银行(数量/最近距离)","小学(数量/最近距离)","地铁(数量/最近距离)","公交(数量/最近距离)"] -//////data << title -//////def pageIndex = 1001 -//////db136.find().sort(new BasicDBObject("f0", 1).append("f1",1).append("f2",1).append("f15",1)).batchSize(1000).addOption(Bytes.QUERYOPTION_NOTIMEOUT).each { BasicDBObject house-> -////// def item = [] -////// for(int i =0 ;i <= 23 ;i++){ -////// item << house.get("f" +i) -////// } -////// data << item -////// if(data.size() % 50000 == 0){ -////// ExcelUtils.write("E:\\数据\\房源数据\\南京房源数据${pageIndex}.xlsx", data, "房源") -////// data = [] -////// data << title -////// pageIndex++ -////// } -//////} -//////if(data.size() > 0){ -////// ExcelUtils.write("E:\\数据\\房源数据\\南京房源数据${pageIndex}.xlsx", data, "房源") -//////} -//////new File("E:\\数据\\房源数据\\统计局房源").eachFile {def file-> -////// def data = ExcelUtils.read(file.getAbsolutePath(),0,0) -////// data.each {List item-> -////// item.remove(14) -////// } -////// ExcelUtils.write(file.getAbsolutePath().replaceAll("统计局房源","统计局房源01"),data,"房源") -////// println(file.getAbsolutePath()) -//////} -////// -//////DBCollection district = MongoUtils.getCollectionByName("business", "district", "192.168.0.136") -//////def regions = district.distinct("region",new BasicDBObject()) -//////def discardDistrict = [] -//////regions.each {def region-> -////// def count = district.count(new BasicDBObject("city","南京").append("region", region)) -////// def limit = (int)(count * 0.66) -////// district.find(new BasicDBObject("city","南京").append("region", region)).sort(new BasicDBObject("size",1)).limit(limit).each {BasicDBObject dbo-> -////// discardDistrict << "${dbo.city}#${dbo.region}#${dbo.name}" -////// } -//////} -//////println(discardDistrict.size()) -//////discardDistrict.each { -////// println(it) -//////} -////// -//////def files = [] -//////new File("E:\\数据\\房源数据\\统计局房源03").eachFile { -////// files << it.getName() -//////} -//////new File("E:\\数据\\房源数据\\统计局房源02").eachFile {def file-> -////// if(file.getAbsolutePath().contains("南京") && !files.contains(file.getName())){ -////// def data = ExcelUtils.read(file.getAbsolutePath(), 0 ,0) -////// def data01 = [] -////// data.each {List item-> -////// def k = "${item[0]}#${item[1]}#${item[2]}" -////// if(!discardDistrict.contains(k)){ -////// data01 << item -////// } -////// } -////// ExcelUtils.write(file.getAbsolutePath().replaceAll("统计局房源02","统计局房源03"),data01,"房源") -////// println(file.getAbsolutePath()) -////// } -//////} -//////DBCollection db136 = MongoUtils.getCollectionByName("fangjia_stat", "district_analy_data", "mongo.db.fangjia.com") -//////ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Downloads\\兴业银行数据样本.xlsx",{List item-> -////// def city = item[0] -////// def name = item[1] -////// def addr = item[2] -////// if(StringUtils.isNotBlank(city)){ -////// JSONObject matchResult = null -////// if(StringUtils.isNotBlank(name) && name.length() > 2){ -////// matchResult = OpenApiClient.addressMatch(city, name, "district") -////// } -////// if(matchResult == null || !matchResult.containsKey("name")){ -////// matchResult = OpenApiClient.addressMatch(city, addr, "district") -////// } -////// -////// if(matchResult!= null && matchResult.containsKey("name")){ -////// item[3] = matchResult.getString("city") -////// item[4] = matchResult.getString("district") -////// item[5] = matchResult.getString("name") -////// item[6] = OpenApiClient.queryDistrictAvgPrice(item[3], item[4], item[5]) -////// if(item[6] == 0){ -////// item[6] = null -////// } -////// def result = getHouseArea(item[3], item[4], item[5]) -////// if(result){ -////// item[7] = result.max -////// item[8] = result.min -////// item[9] = result.many -////// item[10] = result.avg -////// } -////// def one = db136.findOne(new BasicDBObject("city", item[3]).append("region", item[4]).append("district_name",item[5])) -////// if(one){ -////// item[11] = one.around_catering_consumption -////// } -////// } -////// println(item) -////// } -//////}) -//// -//////def getHouseArea(String city, String region, String name){ -////// -////// def query = new BasicDBObject() -////// query.append("city", city) -////// query.append("region", region) -////// def ks = [] -////// ks << name -////// if(name ==~ /.*[东南西北一二三四五六七八九十\d](期|区)$/){ -////// ks << name.replaceAll('[东南西北一二三四五六七八九十\\d](期|区)$', "").replace("~","-") -////// } -////// BasicDBObject one = DbUtils.seaweed.findOne(new BasicDBObject("city", city).append("region", region).append("name",name).append("cat","district").append("visible", 0), new BasicDBObject("alias", 1)) -////// if(one && one.alias){ -////// List alias = one.alias -////// alias.each { -////// if(it && it.length() > 2){ -////// ks << it -////// } -////// } -////// } -////// ks.unique() -////// BooleanQuery nameQuery = new BooleanQuery(); -////// for(String k : ks){ -////// nameQuery.add(new TermQuery(new Term("district_name", k.replaceAll("\\)|\\(|(|)|、",""))), BooleanClause.Occur.SHOULD); -////// } -////// query.append("cat", "sell") -////// def map = [:] -////// def max = Double.MIN_VALUE -////// def min = Double.MAX_VALUE -////// DbUtils.seabird.find(query.append('$sr', new BasicDBObject('sr.query',nameQuery.toString())), new BasicDBObject("area", 1)).batchSize(500).each {BasicDBObject dbo -> -////// if(dbo.area){ -////// map[dbo.area] = map.get(dbo.area, 0) + 1 -////// if(dbo.area > max){ -////// max = dbo.area -////// } -////// if(dbo.area < min){ -////// min = dbo.area -////// } -////// -////// } -////// } -////// def count = 0; -////// def area -////// map.each {Double k, int v -> -////// if(v > count){ -////// count = v; -////// area = k -////// } -////// } -////// -////// long totalArea = 0 -////// long count1 = 0 -////// map.each {Double k, int v-> -////// totalArea = totalArea + (k * v) -////// count1 = count1 + v -////// } -////// if(count > 0){ -////// def result = [:] -////// result.many = area?.toString().toDouble().toInteger() -////// result.min = min -////// result.max = max -////// result.avg = (totalArea/count1).toInteger() -////// return result -////// } -////// return null -//////} -//////ExcelUtils.modify("D:\\Users\\wenyan\\Downloads\\绿地房产项目.xlsx",{List dbo-> -////// def cities = CommonUtils.findAllCity() -////// String addr = dbo[0] -////// def city = null -////// for(def c : cities){ -////// if(addr.startsWith("c" + "市")){ -////// city = c; -////// break -////// } -////// } -////// if(city == null){ -////// for(def c : cities){ -////// if(addr.startsWith("c")){ -////// city = c; -////// break -////// } -////// } -////// } -////// if(city == null){ -////// for(def c : cities){ -////// if(addr.contains(c)){ -////// city = c; -////// break -////// } -////// } -////// } -////// if(city == null){ -////// city = "上海" -////// } -////// JSONObject match = OpenApiClient.addressMatch(city, addr,null,false) -////// if(match && match.containsKey("district")){ -////// dbo[1] = match.getString("city") -////// dbo[2] = match.getString("district") -////// dbo[3] = match.getString("name") -////// int avgPrice = OpenApiClient.queryDistrictAvgPrice(dbo[1],dbo[2],dbo[3]) -////// if(avgPrice != 0){ -////// dbo[4] = avgPrice -////// } -////// } -////// println(dbo) -//////}) -//////def date = Date.parse("yyyyMMdd","20160301") -//////ExcelUtils.read("E:\\Users\\wenyan\\Desktop\\已精耕城市列表.xlsx").each { -////// if(it[1] != null){ -////// int count = 0 -////// DbUtils.seaweed.find(new BasicDBObject("city", it[1]).append("cat", "district").append("source","machine"), new BasicDBObject("c_date",1)).each {BasicDBObject dbo-> -////// Date cDate = dbo.c_date -////// if(cDate && cDate.after(date)){ -////// count ++ -////// } -////// } -////// DbUtils.seaweed.find(new BasicDBObject("city", it[1]).append("cat", "district").append("source","map"), new BasicDBObject("c_date",1)).each {BasicDBObject dbo-> -////// Date cDate = dbo.c_date -////// if(cDate && cDate.after(date)){ -////// count ++ -////// } -////// } -////// if(count > 0){ -////// println("${it[1]} ${count}") -////// } -////// } -//////} -//// -//////ExcelUtils.multiThreadingModify("E:\\Users\\wenyan\\Desktop\\绿地房产项目.xlsx",{List item-> -////// DBCollection coll = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com") -////// if(item[4]){ -////// def dbo = coll.findOne(new BasicDBObject("city",item[1]).append("region", item[2]).append("district_name", item[3]).append("weekend",Date.parse("yyyyMMdd","20160626"))) -////// item[5] = dbo.credit -////// } -////// -//////}) -////// -//////DBCollection coll = MongoUtils.getCollectionByName("fangjia", "region_block", "mongo.db.fangjia.com") -//////def xianRegion = [] -//////coll.find(new BasicDBObject("ranking_level","县级市")).each { -////// xianRegion << it.name -//////} -//////println(xianRegion.size()) -//////def allCity = RegionData.allCity -//////def allRegion = [:] -//////for(def c : allCity){ -////// for(def r :RegionData.getAllRegions(c).keySet()){ -////// if(xianRegion.contains(r)){ -////// allRegion.get(r,[]) << c -////// } -////// } -//////} -//////allRegion.each {String k,List v-> -////// if(v.size() > 1){ -////// println("${k} ${v}") -////// } -//////} -////// -//////CommonUtils.findAllCity().each {def city-> -////// DbUtils.executeQueryOnSeaweed(new BasicDBObject("city", city).append("cat", "district").append("visible", 0),{BasicDBObject dbo-> -////// String name = dbo.getString("name") -////// if(name ==~ /^\s+.*/ || name ==~ /.*\s+$/){ -////// println(name) -////// name = name.replaceAll("^\\s+", "").replaceAll("\\s+\$", ""); -////// DbUtils.executeUpdateOnSeaweed(new BasicDBObject("_id", dbo._id), new BasicDBObject('$set', new BasicDBObject("name", name.replaceAll("\\s", "")))) -////// println(name) -////// } -////// }, new BasicDBObject("region", 1).append("name",1)) -//////} -//// -//////ExcelUtils.multiThreadingModify("E:\\Users\\wenyan\\Desktop\\绿地项目人工匹配的住宅-补充价格.xlsx",{List row-> -////// def params = new HashMap() -////// params.city = row[2] -////// params.region = row[3] -////// params.name = row[4] -////// if(row[4].toString().contains("别墅")){ -////// params.estateType = "别墅" -////// }else{ -////// params.estateType = "普通住宅" -////// } -////// JSONObject respo = HttpRequestUtil.getJsonContent("http://192.168.10.66:8092/calc", params) -////// row[5] = respo?.value -////// row[7] = respo?.credit -////// println(row) -//////}) -//////def HOST = "coredata.mongo.db.fangjia.com" -//////DBCollection ANALY_SITE_COLL = MongoUtils.getCollectionByName("coredata", "analy_site", HOST, 27017) -//////ExcelUtils.modify("E:\\Users\\wenyan\\Desktop\\待处理量.xlsx",{List data-> -////// def query = new BasicDBObject(); -////// query.append("cat", "district") -////// query.append("data_status",0) -////// query.append("exists", 0) -////// query.append("city", data[2]) -////// data[3] = ANALY_SITE_COLL.count(query) -////// println(data) -//////}) -//// -//////DBCollection ANALY_SITE_COLL = MongoUtils.getCollectionByName("fangjia", "server_area", MongoUtils.FANGJIA_ONLINE_MONGO, 27017) -//////ANALY_SITE_COLL.find().sort(new BasicDBObject("name", 1)).each {BasicDBObject dbo-> -////// if(dbo.display != "全国"){ -////// println("${dbo.display}") -////// } -//////} -Jms __jms = Jms.create('tcp://monitor.jms.db.fangjia.com:61616') -__jms.sendSms("13916076378","Hello World", false) -__jms.sendSms("13916076378","Hello World", false) -__jms.sendSms("13916076378","Hello World", false) -__jms.sendSms("13916076378","Hello World", false) -//////ExcelUtils.multiThreadingModify("E:\\Users\\wenyan\\Desktop\\存量客户统计表汇总结果-文艳导出.xlsx",{List item-> -////// def city = item[8] -////// def region = item[9] -////// def name = item[10] -////// def size = item[16] -////// if(size && name){ -////// JSONArray result = OpenApiClient.historyEvaluate(city, region, name, "monthly", "13", size) -////// if(result.size() == 13){ -////// item[18] = result.getJSONObject(12).getInt("totalPrice") - result.getJSONObject(0).getInt("totalPrice") -////// item[19] = DataUtil.roundDouble(((result.getJSONObject(12).getInt("totalPrice") - result.getJSONObject(0).getInt("totalPrice")) * 100)/result.getJSONObject(0).getInt("totalPrice"),2) -////// int i = 20 -////// for(int j = 1; j < result.size(); j++){ -////// item[i++] = result.getJSONObject(j).getDouble("changePercent") -////// } -////// } -////// } -////// println(item) -//////}) -//////DBCollection ANALY_SITE_COLL = MongoUtils.getCollectionByName("business", "city_officer", "192.168.0.136", 27017) -//////ANALY_SITE_COLL.drop() -//////def data = ExcelUtils.read("e:\\Users\\wenyan\\Desktop\\城市对接人(3).xlsx") -//////data.each {List item-> -////// def dbo = new BasicDBObject() -////// dbo.append("level", item[1]) -////// dbo.append("city", item[2]) -////// dbo.append("user_name", item[3]) -////// dbo.append("manager", item[4]) -////// ANALY_SITE_COLL.save(dbo) -//////} -////// -////////CommonUtils.sendDingMsgByCity("北京","Hello World!") -////// -//////OpenApiClient.addressMatch("上海","呼玛三村","district") -//////OpenApiClient.addressMatch("上海","呼玛三村","district") -//////OpenApiClient.flushCache("上海") -//////OpenApiClient.addressMatch("上海","呼玛三村","district") -//////new File("E:\\Users\\wenyan\\Desktop\\门牌号新版").eachFile {def file-> -////// def data = CsvUtils.read(file.getAbsolutePath()) -////// data.each {def item-> -////// def addr = "" -////// if(StringUtils.isNotBlank(item[3])){ -////// addr += item[3] -////// } -////// if(StringUtils.isNotBlank(item[4])){ -////// addr += item[4] -////// } -////// if(StringUtils.isNotBlank(item[5])){ -////// addr += item[5] -////// } -////// item[13] = addr -////// } -////// data.set(0, ["USER_TYPE","REGION","CITY","ADDR1","ADDR2","ADDR3","YHS","RCS","MAX_RSC","FJS","BT_FJS","MAX_FJS","LINE","ALL_ADDR"]) -////// CsvUtils.write("E:\\Users\\wenyan\\Desktop\\门牌号新版2\\" + file.getName(),data) -////// println(file.getAbsolutePath()) -//////} -//////DBCollection MAP_COLL = MongoUtils.getCollectionByName("fangjia_tmp", "mapdata", "192.168.0.61", 27017) -//////DbUtils.executeQueryOnSeaweed(new BasicDBObject("cat", "district").append("visible", 0).append("city", "温州"),{BasicDBObject dbo-> -////// def city = dbo.getString("city") -////// def name = dbo.getString("name") -////// def query = new BasicDBObject() -////// query.append("city", city) -////// def or = [new BasicDBObject("name", name)] -////// if(dbo.alias){ -////// for(String alia : dbo.alias){ -////// if(alia != null && alia.length() > 2){ -////// or << new BasicDBObject("name", alia) -////// } -////// } -////// } -////// query.append('$or', or) -////// def regions = [] -////// def addrs = [] -////// MAP_COLL.find(query).each {BasicDBObject map-> -////// if(map.format_region && !regions.contains(map.format_region)){ -////// regions << map.format_region -////// addrs << map.addr -////// } -////// } -////// if(regions.size() > 0 && !regions.contains(dbo.region)){ -////// println("${dbo.city}\t${dbo.region}\t${name}\t${regions.join(',')}\t${addrs.join(",")}\t${findRegionInGaoDe(city, name)}") -////// } -//////},new BasicDBObject("city", 1).append("region",1).append("name",1).append("alias", 1)) -////// -////// -//////def findRegionInGaoDe(String city, String name){ -////// def pois = KeywordSearchUtils.searchKeywordByGaodeApiAll(city, name) -////// def regions = [] -////// def addrs = [] -////// for(Poi poi: pois){ -////// if(poi.orgName.replace('·',"") == name || poi.name == name){ -////// regions << poi.region -////// addrs << poi.address -////// } -////// } -////// regions.unique() -////// return regions.join(",") + "\t" + addrs.join(",") -//////} -//// -//////def data = ExcelUtils.read("e:\\Users\\wenyan\\Desktop\\城市列表.xlsx") -//////def citys = [] -//////data.each { -////// citys << it[0] -//////} -//////def randomCities = [] -//////Random random = new Random() -//////while (randomCities.size() <= 50){ -////// int index = random.nextInt(140) -////// def c = citys[index] -////// if(!randomCities.contains(c)){ -////// randomCities << c -////// } -//////} -////// -//////def houseData = [] -//////houseData << ["省","市","区","小区名称","产证地址","面积","楼层","总楼层","建成年代"] -//////int total = 0 -//////for(String city : randomCities){ -////// Iterator cursor = DbUtils.seabird.find(new BasicDBObject("city", city.replaceAll('市$', '')).append("cat", "sell")).limit(random.nextInt(200)).iterator() -////// while (cursor.hasNext()){ -////// BasicDBObject dbo= cursor.next() -////// def item = [null, city, dbo.region, dbo.district_name, dbo.address, dbo.area, dbo.floor, dbo.height, dbo.build_date.toString() ==~ /\d{4}年/?dbo.build_date:null] -////// houseData << item -////// if(houseData.size() >= 2000){ -////// break -////// } -////// } -////// if(houseData.size() >= 2000){ -////// break -////// } -//////} -//////ExcelUtils.write("e:\\Users\\wenyan\\Desktop\\房源.xlsx", houseData, "结果") -//////DBCollection BAIDU_MAP_COLL = MongoUtils.getCollectionByName("addrepo", "baidumap", "192.168.0.61", 27017) -//////DBCollection MAP_COLL = MongoUtils.getCollectionByName("fangjia_tmp", "mapdata", "192.168.0.61", 27017) -//////def citys =["鸡西","吉安","商洛","延安","玉林","枣庄","张家界","绵阳","晋中","濮阳","渭南","乌兰察布","漯河","辽阳","眉山","宝鸡","新余","阳泉","漳州","绥化","南宁","乌海","遂宁","盐城","延边","池州","平凉","六安","五指山","扬州","西安","攀枝花","武威","黄山","三门峡","南阳","沈阳","杭州","抚顺","厦门","内江","盘锦","无锡","宁波","西宁","大理","拉萨","泰州","天门","赤峰","阿里","威海","齐齐哈尔","定西","菏泽","连云港","泰安","通化","万宁","文昌","阜阳","百色","安庆","宜昌","巴中","秦皇岛","清远","庆阳","潜江","固原","铜仁","成都","湘潭","太原","镇江","安顺","景德镇","神农架","琼中","黔西南","济源","兰州","青岛","宿迁","赣州","朝阳","长沙","株洲","昭通","潍坊","临沧","襄阳","资阳","湘潭","太原","来宾","吉林","中山","阿拉善盟","乌鲁木齐","锦州","九江","广元","白山","永州","临汾","钦州","石家庄","韶关","铁岭","郑州","怒江州","伊犁","林芝","咸阳","济南","营口","克孜勒苏","昌都","凉山州","临高","临夏","六盘水","南京","吕梁","梅州","湘西","南通","北京","黄冈","白沙","兴安盟","锡林郭勒盟","佛山","温州","芜湖","肇庆","焦作","宣城","舟山","巴音郭楞","双鸭山","湛江","和田","博尔塔拉","云浮","吐鲁番","阿勒泰","大兴安岭","东方","东莞","恩施","防城港","甘孜州","三明","四平","福州","本溪","桂林","果洛州","武汉","玉树州","黄南州","唐山","大连","昆明","临沂","崇左","贵阳","大同","黄石","黔南","山南","佳木斯","保定","烟台佳木斯","保定","烟台"] -//////ExcelUtils.multiThreadingModify("e:\\Users\\wenyan\\Desktop\\地图库待处理量.xlsx",{List item-> -////// if(citys.contains(item[2])){ -////// item[5] = BAIDU_MAP_COLL.count(new BasicDBObject("cat", "district").append("city", item[2]).append("status","未审核")) -////// item[6] = BAIDU_MAP_COLL.count(new BasicDBObject("cat", "district").append("city", item[2]).append("status","区域错误")) -////// item[7] = MAP_COLL.count(new BasicDBObject("city", item[2])) -////// } -//////}) -//////ExcelUtils.multiThreadingModify("e:\\Users\\wenyan\\Desktop\\众拍.xlsx",{List item-> -////// String city = item[1] -////// String region = item[2] -////// String name = item[3] -////// JSONObject resp = OpenApiClient.addressMatch(city , name, "district", false, region) -////// if(resp.getDouble("credit") > 0.5){ -////// item[6] = resp.city -////// item[7] = resp.district -////// item[8] = resp.name -////// -////// } -//////}) -//// -//// -/////* -////DBCollection dealPrice = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com") -////Calendar calendar = Calendar.getInstance() -////calendar.setTime(com.fangjia.core.util.DateUtil.parseDate("20160501", "yyyyMMdd")) -////for(int i = 0 ; i < 3; i++){ -//// def startDate = calendar.getTime() -//// calendar.add(Calendar.MONTH, 1) -//// def endDate = calendar.getTime() -//// def citys = ["杭州","成都","深圳"] -//// citys.each {def city-> -//// def query = new BasicDBObject() -//// query.append("city", city) -//// query.append("trade_date", new BasicDBObject('$gte', startDate).append('$lt', endDate)) -//// println("${startDate.format("MM")} ${city} ${dealPrice.count(query)}") -//// } -//// -////}*/ -//////DBCollection BAIDU_MAP_COLL = MongoUtils.getCollectionByName("addrepo", "baidumap", "192.168.0.61", 27017) -//////DBCollection MAP_COLL = MongoUtils.getCollectionByName("fangjia_tmp", "mapdata", "192.168.0.61", 27017) -//////def citys =["通辽","呼伦贝尔","酒泉","常州","宜宾","衢州","梧州","淮北","哈尔滨","鸡西","吉安","商洛","延安","玉林","枣庄","张家界","绵阳","晋中","濮阳","渭南","乌兰察布","漯河","辽阳","眉山","宝鸡","新余","阳泉","漳州","绥化","南宁","乌海","遂宁","盐城","延边","池州","平凉","六安","五指山","扬州","西安","攀枝花","武威","黄山","三门峡","南阳","沈阳","杭州","抚顺","厦门","内江","盘锦","无锡","宁波","西宁","大理","拉萨","泰州","天门","赤峰","阿里","威海","齐齐哈尔","定西","菏泽","连云港","泰安","通化","万宁","文昌","阜阳","百色","安庆","宜昌","巴中","秦皇岛","清远","庆阳","潜江","固原","铜仁","成都","湘潭","太原","镇江","安顺","景德镇","神农架","琼中","黔西南","济源","兰州","青岛","宿迁","赣州","朝阳","长沙","株洲","昭通","潍坊","临沧","襄阳","资阳","湘潭","太原","来宾","吉林","中山","阿拉善盟","乌鲁木齐","锦州","九江","广元","白山","永州","临汾","钦州","石家庄","韶关","铁岭","郑州","怒江州","伊犁","林芝","咸阳","济南","营口","克孜勒苏","昌都","凉山州","临高","临夏","六盘水","南京","吕梁","梅州","湘西","南通","北京","黄冈","白沙","兴安盟","锡林郭勒盟","佛山","温州","芜湖","肇庆","焦作","宣城","舟山","巴音郭楞","双鸭山","湛江","和田","博尔塔拉","云浮","吐鲁番","阿勒泰","大兴安岭","东方","东莞","恩施","防城港","甘孜州","三明","四平","福州","本溪","桂林","果洛州","武汉","玉树州","黄南州","唐山","大连","昆明","临沂","崇左","贵阳","大同","黄石","黔南","山南","佳木斯","保定","烟台佳木斯","保定","烟台","许昌","邯郸","辽源","洛阳","淄博","廊坊","鹤岗","宿州","七台河","泸州","雅安","汉中","开封","张掖","白银","黑河","呼和浩特","晋城","朔州","银川","阿坝州","德宏州","广州","海东","信阳","鞍山","楚雄","海西州","白城", "上饶","驻马店","北海","亳州","海北州","铜川","喀什","日照","石嘴山","毕节","安阳","南平","苏州","抚州","衡水", "宁德","榆林","中卫","河池","滨州","商丘","五家渠","澄迈","定安","儋州","汕头","龙岩","海口","宜春","沧州","广安", "孝感","河池","淮安","珠海","马鞍山","嘉兴","海南州","上海","周口","阳江","丹东","保山","遵义","益阳","滁州","莆田","文山州","安康","蚌埠","保亭","德州","河源","鹤壁","伊春","鄂州","嘉峪关","金昌","郴州", "深圳","贺州","新乡","红河","克拉玛依","茂名","天津","运城","那曲","阿拉尔","昌江","包头","荆门","淮南","揭阳","鄂尔多斯","汕尾","东营","潮州","咸宁","甘南","徐州","荆州","三亚","天水","南昌","南充","迪庆州","衡阳","西双版纳州","石河子","仙桃","琼海","陵水","金华","葫芦岛","日喀则","哈密","图木舒克","乐东","屯昌","德阳","萍乡","自贡","十堰","承德","张家口","普洱","乐山", "平顶山","巴彦淖尔","鹰潭","莱芜","铜陵","达州","松原","贵港","邢台","忻州","吴忠","阜新","绍兴","济宁","昌吉","黔东南","合肥","随州","陇南","娄底","阿克苏","柳州","塔城","湖州","惠州","丽江","长治","聊城","丽水","邵阳","常德","重庆","岳阳","泉州", "台州","牡丹江","大庆" ,"长春","江门","曲靖","怀化","玉溪"] -//////ExcelUtils.multiThreadingModify("E:\\房价网\\201507\\地图库待处理量.xlsx",{List item-> -////// if(citys.contains(item[2])){ -////// item[5] = BAIDU_MAP_COLL.count(new BasicDBObject("cat", "district").append("city", item[2]).append("status","未审核")) -////// item[6] = BAIDU_MAP_COLL.count(new BasicDBObject("cat", "district").append("city", item[2]).append("status","区域错误")) -////// item[7] = MAP_COLL.count(new BasicDBObject("city", item[2])) -////// } -//////}) -//////DBCollection BAIDU_MAP_COLL = MongoUtils.getCollectionByName("addrepo", "baidumap", "192.168.0.61", 27017) -//////DBCollection MAP_COLL = MongoUtils.getCollectionByName("fangjia_tmp", "mapdata", "192.168.0.61", 27017) -//////def citys =["常州","宜宾","衢州","梧州","淮北","哈尔滨","鸡西","吉安","商洛","延安","玉林","枣庄","张家界","绵阳","晋中","濮阳","渭南","乌兰察布","漯河","辽阳","眉山","宝鸡","新余","阳泉","漳州","绥化","南宁","乌海","遂宁","盐城","延边","池州","平凉","六安","五指山","扬州","西安","攀枝花","武威","黄山","三门峡","南阳","沈阳","杭州","抚顺","厦门","内江","盘锦","无锡","宁波","西宁","大理","拉萨","泰州","天门","赤峰","阿里","威海","齐齐哈尔","定西","菏泽","连云港","泰安","通化","万宁","文昌","阜阳","百色","安庆","宜昌","巴中","秦皇岛","清远","庆阳","潜江","固原","铜仁","成都","湘潭","太原","镇江","安顺","景德镇","神农架","琼中","黔西南","济源","兰州","青岛","宿迁","赣州","朝阳","长沙","株洲","昭通","潍坊","临沧","襄阳","资阳","湘潭","太原","来宾","吉林","中山","阿拉善盟","乌鲁木齐","锦州","九江","广元","白山","永州","临汾","钦州","石家庄","韶关","铁岭","郑州","怒江州","伊犁","林芝","咸阳","济南","营口","克孜勒苏","昌都","凉山州","临高","临夏","六盘水","南京","吕梁","梅州","湘西","南通","北京","黄冈","白沙","兴安盟","锡林郭勒盟","佛山","温州","芜湖","肇庆","焦作","宣城","舟山","巴音郭楞","双鸭山","湛江","和田","博尔塔拉","云浮","吐鲁番","阿勒泰","大兴安岭","东方","东莞","恩施","防城港","甘孜州","三明","四平","福州","本溪","桂林","果洛州","武汉","玉树州","黄南州","唐山","大连","昆明","临沂","崇左","贵阳","大同","黄石","黔南","山南","佳木斯","保定","烟台佳木斯","保定","烟台","许昌","邯郸","辽源","洛阳","淄博","廊坊","鹤岗","宿州","七台河","泸州","雅安","汉中","开封","张掖","白银","黑河","呼和浩特","晋城","朔州","银川","阿坝州","德宏州","广州","海东","信阳","鞍山","楚雄","海西州","白城", "上饶","驻马店","北海","亳州","海北州","铜川","喀什","日照","石嘴山","毕节","安阳","南平","苏州","抚州","衡水", "宁德","榆林","中卫","河池","滨州","商丘","五家渠","澄迈","定安","儋州","汕头","龙岩","海口","宜春","沧州","广安", "孝感","河池","淮安","珠海","马鞍山","嘉兴","海南州","上海","周口","阳江","丹东","保山","遵义","益阳","滁州","莆田","文山州","安康","蚌埠","保亭","德州","河源","鹤壁","伊春","鄂州","嘉峪关","金昌","郴州", "深圳","贺州","新乡","红河","克拉玛依","茂名","天津","运城","那曲","阿拉尔","昌江","包头","荆门","淮南","揭阳","鄂尔多斯","汕尾","东营","潮州","咸宁","甘南","徐州","荆州","三亚","天水","南昌","南充","迪庆州","衡阳","西双版纳州","石河子","仙桃","琼海","陵水","金华","葫芦岛","日喀则","哈密","图木舒克","乐东","屯昌","德阳","萍乡","自贡","十堰","承德","张家口","普洱","乐山", "平顶山","巴彦淖尔","鹰潭","莱芜","铜陵","达州","松原","贵港","邢台","忻州","吴忠","阜新","绍兴","济宁","昌吉","黔东南","合肥","随州","陇南","娄底","阿克苏","柳州","塔城","湖州","惠州","丽江","长治","聊城","丽水","邵阳","常德","重庆","岳阳","泉州", "台州","牡丹江","大庆" ,"长春","江门","曲靖","怀化","玉溪"] -//////ExcelUtils.multiThreadingModify("E:\\房价网\\201507\\地图库待处理量.xlsx",{List item-> -////// if(citys.contains(item[2])){ -////// item[5] = BAIDU_MAP_COLL.count(new BasicDBObject("cat", "district").append("city", item[2]).append("status","未审核")) -////// item[6] = BAIDU_MAP_COLL.count(new BasicDBObject("cat", "district").append("city", item[2]).append("status","区域错误")) -////// item[7] = MAP_COLL.count(new BasicDBObject("city", item[2])) -////// } -//////}) -//////DBCollection dealPrice = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com") -//////Calendar calendar = Calendar.getInstance() -//////calendar.setTime(com.fangjia.core.util.DateUtil.parseDate("20160501", "yyyyMMdd")) -//////for(int i = 0 ; i < 3; i++){ -////// def startDate = calendar.getTime() -////// calendar.add(Calendar.MONTH, 1) -////// def endDate = calendar.getTime() -////// def citys = ["杭州","成都","深圳"] -////// citys.each {def city-> -////// def query = new BasicDBObject() -////// query.append("city", city) -////// query.append("trade_date", new BasicDBObject('$gte', startDate).append('$lt', endDate)) -////// println("${startDate.format("MM")} ${city} ${dealPrice.count(query)}") -////// } -//////} -/////*ExcelUtils.multiThreadingModify("E:\\Users\\wenyan\\Desktop\\憨分科技(1)-补充小区.xls",{List data-> -//// def city = data[2] -//// def region = data[3] -//// def addr = data[4] -//// def oldMatchRegion = data[5] -//// def oldMatchName = data[6] -//// JSONObject resp= OpenApiClient.addressMatch(city, addr, null, false, region, "strict") -//// if(resp.getDouble("credit") > 0){ -//// data[7] = resp.district -//// data[8] = resp.name -//// }else{ -//// data[7] = null -//// data[8] = null -//// } -//// if(oldMatchRegion == data[7] && oldMatchName == data[8]){ -//// data[9] = "一致" -////======= -//////ExcelUtils.multiThreadingModify("E:\\Users\\wenyan\\Desktop\\憨分科技(1)-补充小区.xls",{List data-> -////// def city = data[2] -////// def region = data[3] -////// def addr = data[4] -////// def oldMatchRegion = data[5] -////// def oldMatchName = data[6] -////// JSONObject resp= OpenApiClient.addressMatch(city, addr, null, false, region, "strict") -////// if(resp.getDouble("credit") > 0){ -////// data[7] = resp.district -////// data[8] = resp.name -////// }else{ -////// data[7] = null -////// data[8] = null -////// } -////// if(oldMatchRegion == data[7] && oldMatchName == data[8]){ -////// data[9] = "一致" -////// }else{ -////// data[9] = "不一致" -////// } -////// println(data) -//////}) -////ExcelUtils.modify("E:\\Users\\wenyan\\Desktop\\地图地址比较.xlsx",{List data-> -//// HttpRequest request = new HttpRequest() -//// def params = [city:data[0], region:data[1], name:data[2]] -//// JSONObject json = request.getJsonContent("http://192.168.10.88/mapSearch/findOptimal", params, "UTF-8") -//// if(json){ -//// JSONObject result = json.getJSONObject("result") -//// if(result.containsKey("baiduMapPoi")){ -//// JSONObject baiduMapPoi = result.getJSONObject("baiduMapPoi") -//// data[4] = baiduMapPoi.region -//// data[5] = baiduMapPoi.name -//// data[6] = baiduMapPoi.address -//// } -//// if(result.containsKey("gaodeMapPoi")){ -//// JSONObject gaodeMapPoi = result.getJSONObject("gaodeMapPoi") -//// data[7] = gaodeMapPoi.region -//// data[8] = gaodeMapPoi.name -//// data[9] = gaodeMapPoi.address -//// } -//// if(result.containsKey("tencentMapPoi")){ -//// JSONObject tencentMapPoi = result.getJSONObject("tencentMapPoi") -//// data[10] = tencentMapPoi.region -//// data[11] = tencentMapPoi.name -//// data[12] = tencentMapPoi.address -//// } -//// } -//// Thread.sleep(200) -//// println(data) -//// -////}) -//////ExcelUtils.multiThreadingModify("E:\\Users\\wenyan\\Desktop\\帮忙调查房价是否正确.xlsx", {List data-> -////// data[5] = DbUtils.seaweed.findOne(new BasicDBObject("city",data[1]).append("region", data[2]).append("name", data[3]).append("visible",0).append("cat","district"),new BasicDBObject("address",1))?.address -////// println(data) -//////}) -//// -////DbUtils.executeQueryOnSeaweed(new BasicDBObject("cat", "district").append("visible", 0),{BasicDBObject dbObject-> -//// if(dbObject.address && dbObject.getString("address").startsWith("路")){ -//// dbObject.removeField("_id") -//// println(dbObject) -////DBCollection BAIDU_MAP_COLL = MongoUtils.getCollectionByName("addrepo", "baidumap", "192.168.0.61", 27017) -////BAIDU_MAP_COLL.find(new BasicDBObject("cat", "district").append("status","未审核")).each {BasicDBObject dbo-> -//// def city = dbo.city -//// def name = dbo.name -//// if(findDistrict(city,name) != null){ -//// dbo.append("cpdbStatus", "区域不一致") -//// }else{ -//// dbo.append("cpdbStatus", "CPDB库不存在") -//// } -////},new BasicDBObject("address",1).append("name", 1).append("city", 1).append("region",1)) -////<<<<<<< HEAD -//// println(data) -////}) -//// BAIDU_MAP_COLL.save(dbo) -////}*/ -//// -//////DBCollection AAA = MongoUtils.getCollectionByName("addrepo", "baidumap", "localhost", 27017) -//////def data = [] -////// -//////data << new BasicDBObject("_id", "2222222") -//////data << new BasicDBObject("_id", "2222222") -//////data << new BasicDBObject("_id", "2222222") -//////data << new BasicDBObject("_id", "3333333") -//////data << new BasicDBObject("_id", "3333333") -//////data << new BasicDBObject("_id", "3333333") -//////data << new BasicDBObject("_id", "3333333") -//////data << new BasicDBObject("_id", "1111111").append("k", 'kkkk') -////// -//////AAA.insert(data, WriteConcern.SAFE)*/ -//////ExcelUtils.modify("E:\\Users\\wenyan\\Desktop\\地图地址比较.xlsx",{List data-> -////// HttpRequest request = new HttpRequest() -////// def params = [city:data[0], region:data[1], name:data[2]] -////// JSONObject json = request.getJsonContent("http://192.168.10.88/mapSearch/findOptimal", params, "UTF-8") -////// if(json){ -////// JSONObject result = json.getJSONObject("result") -////// if(result.containsKey("baiduMapPoi")){ -////// JSONObject baiduMapPoi = result.getJSONObject("baiduMapPoi") -////// data[4] = baiduMapPoi.region -////// data[5] = baiduMapPoi.name -////// data[6] = baiduMapPoi.address -////// } -////// if(result.containsKey("gaodeMapPoi")){ -////// JSONObject gaodeMapPoi = result.getJSONObject("gaodeMapPoi") -////// data[7] = gaodeMapPoi.region -////// data[8] = gaodeMapPoi.name -////// data[9] = gaodeMapPoi.address -////// } -////// if(result.containsKey("tencentMapPoi")){ -////// JSONObject tencentMapPoi = result.getJSONObject("tencentMapPoi") -////// data[10] = tencentMapPoi.region -////// data[11] = tencentMapPoi.name -////// data[12] = tencentMapPoi.address -////// } -////// } -////// Thread.sleep(200) -////// println(data) -////// -//////}) -//////ExcelUtils.modify("E:\\Users\\wenyan\\Desktop\\错误小区.xlsx",{List data-> -////// -////// if(data[5] == null || data[5] == data[1]){ -////// HttpRequest request = new HttpRequest() -////// def params = [city:data[0], region:data[1], name:data[2]] -////// JSONObject json = request.getJsonContent("http://192.168.10.88/mapSearch/findOptimal", params, "UTF-8") -////// if(json){ -////// JSONObject result = json.getJSONObject("result") -////// if(result.containsKey("gaodeMapPoi")){ -////// JSONObject gaodeMapPoi = result.getJSONObject("gaodeMapPoi") -////// data[5] = gaodeMapPoi.region -////// }else if(result.containsKey("tencentMapPoi")){ -////// JSONObject tencentMapPoi = result.getJSONObject("tencentMapPoi") -////// data[5] = tencentMapPoi.region -////// }else if(result.containsKey("baiduMapPoi")){ -////// JSONObject baiduMapPoi = result.getJSONObject("baiduMapPoi") -////// data[5] = baiduMapPoi.region -////// } -////// } -////// Thread.sleep(200) -////// println(data) -////// -////// } -//////}) -//////ExcelUtils.multiThreadingModify("E:\\Users\\wenyan\\Desktop\\错误小区.xlsx",{List data-> -////// if(data[6] == "存在"){ -////// def city = data[0] -////// def region = data[1] -////// def name = data[2] -////// def one = DbUtils.seaweed.findOne(new BasicDBObject("city", city).append('region',region).append("name", name).append("visible", 0).append("cat", "district")) -////// if(one){ -////// DbUtils.seaweed.remove(new BasicDBObject("_id", one._id)) -////// println("$city $region $name") -////// } -////// } -//////}); -//////DBCollection REGION_STAT_TREND = MongoUtils.getCollectionByName("fangjia_stat", "region_sell_stat", "mongo.db.fangjia.com") -//////REGION_STAT_TREND.find().each { -////// println(it) -//////} -//////DBCollection BAIDU_MAP_COLL = MongoUtils.getCollectionByName("addrepo", "baidumap", "192.168.0.61", 27017) -//////List cities = BAIDU_MAP_COLL.distinct("city",new BasicDBObject("cat", "district") ) -//////cities.each {def city-> -////// BAIDU_MAP_COLL.find(new BasicDBObject("cat", "district").append("city",city)).each {BasicDBObject newDistrict-> -////// if(findDistrict(newDistrict.city, newDistrict.name) != null){ -////// newDistrict.append("cpdbStatus", "区域不一致") -////// }else{ -////// newDistrict.append("cpdbStatus", "CPDB库不存在") -////// } -////// if(newDistrict.address){ -////// newDistrict.address = newDistrict.getString("address").replaceAll("\\s", "") -////// } -////// println(newDistrict) -////// BAIDU_MAP_COLL.save(newDistrict) -////// } -//////} -////// -////// -////// -//////def BasicDBObject findDistrict(String city, String name){ -////// name = ClientUtils.escapeQueryChars(name) -////// BooleanQuery nameQuery = new BooleanQuery(); -////// nameQuery.add(new TermQuery(new Term("name", name)), BooleanClause.Occur.SHOULD); -////// nameQuery.add(new TermQuery(new Term("alias", name)), BooleanClause.Occur.SHOULD); -////// def query = new BasicDBObject("city", city).append('$sr', new BasicDBObject("sr.query", nameQuery.toString())).append("cat", "district").append("visible", 0); -////// return DbUtils.seaweed.findOne(query, new BasicDBObject("_id", 1)) -//////} -//////} -//////DBCollection BAIDU_MAP_COLL = MongoUtils.getCollectionByName("fangjia", "land_price", DbUtils.OFFLINE_MONGODB_DOMAIN, 27017) -//////def data = ExcelUtils.read("D:\\1.xlsx") -//////data.each {List item-> -////// def dbo = new BasicDBObject() -////// dbo.append("city", item[0]) -////// dbo.append("level", item[1]) -////// dbo.append("second_purpose", item[2]) -////// dbo.append("standard_land_price", item[3].toString().toDouble().toInteger()) -////// dbo.append("standard_date", item[4]) -////// dbo.append("volume_rate", item[5]) -////// dbo.append("develop_level", item[6]) -////// dbo.append("purpose","商业用途") -////// dbo.append("c_date", new Date()) -////// println(item) -////// BAIDU_MAP_COLL.save(dbo) -//////} -//////def log = new File("E:\\数据\\日志\\access.2016-09-27.log.1") -//////def logput = new File("E:\\数据\\日志\\access.2016-09-27.output.1") -//////def maxDate = DateUtil.parseDate("28/09/2016:07:00:00", "dd/MM/yyyy:HH:mm:ss") -//////def map = [:] -//////log.eachLine("UTF-8"){def line-> -////// def index = line.indexOf("[") -////// def end = line.indexOf(" +0800]",index) -////// def time = line.substring(index + 1 , end).replaceAll("Sep", "09") -////// def date = DateUtil.parseDate(time, "dd/MM/yyyy:HH:mm:ss") -////// if(maxDate.after(date) && line.indexOf("GET /statistic?__ajax=1") > 0){ -////// def ip = line.substring(0, line.indexOf(" - -")) -////// map[ip] = map.get(ip,0) + 1 -////// } -//////} -//////map.each {k,v-> -////// println("$k\t$v") -//////} -//////DBCollection DISTIRCT_RENT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", MongoUtils.FANGJIA_ONLINE_MONGO) -//////Date weekend = DateUtil.parseDate("2016-09-25", "yyyy-MM-dd") -//////def data = [] -//////data << ["城市","区域","名称","地址","百度经度","百度纬度","高德经度","高德纬度","户数","建筑面积","均价"] -//////DbUtils.executeQueryOnSeaweed(new BasicDBObject("cat","office").append("city","上海").append("region","长宁").append("visible", 0),{BasicDBObject dbo-> -////// def item = new ExcelRow() -////// item['A'] = dbo.city -////// item['B'] = dbo.region -////// item['C'] = dbo.name -////// item['d'] = dbo.address -////// if(dbo.lng2 && dbo.lat2){ -////// item['E'] = dbo.lng2/10000000000d -////// item['F'] = dbo.lat2/10000000000d -////// item['G'] = dbo.lng/10000000000d -////// item['H'] = dbo.lat/10000000000d -////// } -////// item['I'] = dbo.household_count -////// item['J'] = dbo.build_area -//////// def query = new BasicDBObject() -//////// query.append("city", dbo.city) -//////// query.append("region", dbo.region) -//////// query.append("district_name", dbo.name) -//////// query.append("weekend", weekend) -//////// def one = DISTIRCT_RENT.findOne(query) -//////// if(one){ -//////// item['K'] = one.avg_price -//////// } -////// println(item) -////// data << item -//////}) -//////ExcelUtils.write("E://需求数据.xlsx",data,"写字楼") -//////DbUtils.seawater.find(new BasicDBObject("city","上海").append("district_name","风伟二村").append("cat","sell")).each {BasicDBObject dbo-> -////// println(dbo._id) -//////} -//////DBCollection DISTIRCT_RENT = MongoUtils.getCollectionByName("fangjia", "seawater", "192.168.0.65", 27777) -//////DISTIRCT_RENT.find(new BasicDBObject("city","上海").append("district_name","风伟二村").append("cat","sell")).iterator().each {BasicDBObject dbo-> -////// println(dbo._id.toString()) -//////} -//////def findStr = "/autoComplete/suggest?cat=district&__ajax=1&keyword=" -//////def Pattern domainPattern = Pattern.compile("http://\\w+.fangjia.com/autoComplete/suggest") -//////def log = new File("E:\\数据\\日志\\抓取关键词.log") -//////def logs = new HashSet() -//////new File("E:\\数据\\日志\\access.2016-09-27.log.1").eachLine("UTF-8"){def line-> -////// if(line.contains("113.91.173.47")){ -////// def s = line.indexOf(findStr) -////// if(s > 0){ -////// def e = line.indexOf(" HTTP", s) -////// def keywords = line.substring(s + findStr.length(), e) -////// keywords = URLDecoder.decode(keywords, "UTF-8") -////// Matcher matcher = domainPattern.matcher(line) -////// if(matcher.find()){ -////// String result = keywords + "\t" + matcher.group().replaceAll("http://", "").replaceAll(".fangjia.com/autoComplete/suggest", "") -////// logs.add(result) -////// println(result) -////// } -////// } -////// } -//////} -//////new File("E:\\数据\\日志\\access.2016-09-26.log.1").eachLine("UTF-8"){def line-> -////// if(line.contains("113.91.173.47")){ -////// println(line) -//////// def s = line.indexOf(findStr) -//////// if(s > 0){ -//////// def e = line.indexOf(" HTTP", s) -//////// def keywords = line.substring(s + findStr.length(), e) -//////// keywords = URLDecoder.decode(keywords, "UTF-8") -//////// Matcher matcher = domainPattern.matcher(line) -//////// if(matcher.find()){ -//////// String result = keywords + "\t" + matcher.group().replaceAll("http://", "").replaceAll(".fangjia.com/autoComplete/suggest", "") -//////// logs.add(result) -//////// println(result) -//////// } -//////// } -////// } -//////} -//////logs.each { -////// log.append(it + "\n", "utf-8") -//////} -//// -//////DBCollection ANALY_SITE_COLL = MongoUtils.getCollectionByName("fangjia", "server_area", MongoUtils.FANGJIA_ONLINE_MONGO, 27017) -//////def map = [:] -//////ANALY_SITE_COLL.find().sort(new BasicDBObject("name", 1)).each {BasicDBObject dbo-> -////// if(dbo.display != "全国"){ -////// map.put(dbo.name, dbo.display) -////// map.put(dbo.spell, dbo.display) -////// } -//////} -//////ExcelUtils.multiThreadingModify("E:\\Users\\wenyan\\Desktop\\抓取数据分析.xlsx",{List item-> -////// def t = item[0] -////// item[0] = item[1] -////// item[1] = t -////// println(item) -//////}) -//////def data = ExcelUtils.read("E:\\Users\\wenyan\\Desktop\\抓取数据分析.xlsx") -//////def cache = [:] -//////data.each {List item-> -////// if(item[0] != null && item[2] == "存在"){ -////// cache[item[0]] = cache.get(item[0], 0) + 1 -////// } -//////} -//////cache.each {k,v-> -////// println("$k\t$v") -//////} -//////ExcelUtils.multiThreadingModify("E:\\Users\\wenyan\\Desktop\\益博睿经纬度.xlsx",{List item-> -////// String addr = item['B'] -////// if(StringUtils.isNotBlank(addr) && (addr.indexOf("号楼") > 0 && addr.indexOf("单元") > 0)){ -////// item['m'] = null -////// item['n'] = null -////// item['o'] = null -////// item['p'] = null -////// item['q'] = null -////// if(item['G'] != null){ -////// def c = item['G'].toString().replaceFirst("市","") -////// JSONObject resp = OpenApiClient.addressMatch(c, addr, null, true, null, "fuzzy") -////// if(resp.getDouble("credit") > 0){ -////// item['m'] = resp.name -////// item['n'] = resp.lng -////// item['o'] = resp.lat -////// def lng = item['N'] -////// def lat = item['o'] -////// if(lng && lat){ -////// def map = GeoUtils.geoconvBaiduToGoogle(lng, lat) -////// item['p'] = map['lng'] -////// item['q'] = map['lat'] -////// } -////// } -////// println(item) -////// } -////// -////// } -//////}) -//def list = [] -//ExcelUtils.read("D:\\Users\\wenyan\\Downloads\\常州江南银行住宅--更新ing.xls").each {List item-> -// list << "${item[1]}#${item[2]}" -//} -//def total = 1 -//DbUtils.executeQueryOnSeaweed(DbUtils.buildSeaweedQuery("常州"),{ BasicDBObject dbo-> -// def key = "${dbo.region}#${dbo.name}" -// if(list.contains(key)){ -// Map mark_map = dbo.mark_map -// if(!mark_map){ -// mark_map = [:] -// dbo.mark_map = mark_map -// } -// mark_map.isNanJingBank = 1 -// DbUtils.seaweed.save(dbo) -// total++ -// list.remove(key) -// }else { -// List alias = dbo.alias -// if(alias){ -// for(String alia : alias){ -// key = "${dbo.region}#${alia}" -// if(list.contains(key)){ -// Map mark_map = dbo.mark_map -// if(!mark_map){ -// mark_map = [:] -// dbo.mark_map = mark_map -// } -// mark_map.isNanJingBank = 1 -// DbUtils.seaweed.save(dbo) -// total++ -// list.remove(key) -// break -// } -// } -// } -// } -//}) -//////println(total) -//////list.each { -////// println(it) -//////} -//////DbUtils.seabird.find(new BasicDBObject("cat", "addr_district").append("keyword","翠绿山庄")).each { -////// println(it) -//////} -//////new File("\\\\192.168.0.135\\技术部\\access.2016-10-27.log").eachLine("utf-8"){def line-> -////// if(line.contains("500")){ -////// println(line) -////// } -//////} -//// -//////DbUtils.seabird.find(new BasicDBObject("cat", "addr_district")).each {BasicDBObject dbo-> -////// if(DbUtils.seaweed.count(new BasicDBObject("cat", "district").append("keyword", dbo.district_name).append("city", dbo.city)) == 0){ -////// println(dbo) -////// } -//////} -//////DbUtils.seaweed.find(new BasicDBObject("city", "张家口").append("cat", "district") -////// .append("c_date", new BasicDBObject('$gt',Date.parse("yyyyMMdd","20160830")).append('$lt', Date.parse("yyyyMMdd", "20160901")))).each { -////// println(it) -//////} -//// -//////DBCollection FANGJIA_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.136") -//////DBCollection TEMP = MongoUtils.getCollectionByName("fangjia_temp", "district_key", "192.168.0.136") -//////int count = 0 -//////def keys = new HashSet() -//////FANGJIA_STAT.find(new BasicDBObject(), new BasicDBObject("city", 1).append("name", 1).append("district_name" ,1)).batchSize(10000).sort(new BasicDBObject("_id", 1)).addOption(Bytes.QUERYOPTION_NOTIMEOUT).each {BasicDBObject dbo-> -////// String key = "${dbo.city}##${dbo.region}##${dbo.district_name}" -////// keys << key -////// count++ -////// if(count % 10000 == 0){ -////// println(count) -////// } -//////} -//////keys.each { -////// String[] parts = it.toString().split("##") -////// def query = new BasicDBObject() -////// query.append("city", parts[0] ) -////// query.append("region", parts[1]) -////// query.append("district_name", parts[2]) -////// TEMP.update(query, new BasicDBObject('$set', query), true, false) -//////} -//////int count = 0 -//////TEMP.find().batchSize(1000).each {BasicDBObject dbo-> -////// def query = DbUtils.buildSeaweedQuery(dbo.city, dbo.region, dbo.district_name) -////// if(DbUtils.seaweed.count(query) == 0){ -////// dbo.exist = 0 -////// }else{ -////// dbo.exist = 1 -////// } -////// TEMP.save(dbo) -////// count++ -////// if(count % 1000 == 0){ -////// println(count) -////// } -//////} -//// -//////def list = ["别墅","小区","写字楼","商铺"] -//////int total = 0; -//////int count = 0; -//////FANGJIA_STAT.find(new BasicDBObject(), new BasicDBObject("estate_type", 1)).addOption(Bytes.QUERYOPTION_NOTIMEOUT).batchSize(1000).each {BasicDBObject dbo-> -////// if(dbo.estate_type == "商铺" || dbo.estate_type == "写字楼"){ -////// FANGJIA_STAT.remove(new BasicDBObject("_id", dbo._id)) -////// count++ -////// } -////// total ++ -////// if(total % 10000 == 0){ -////// println(total) -////// } -//////} -//////println("删除:${count}") -//////DBCollection TEMP = MongoUtils.getCollectionByName("fangjia_temp", "district_key", "192.168.0.136") -//////DBCollection FANGJIA_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com") -//////int total = 0; -//////TEMP.find(new BasicDBObject("exist", 0), new BasicDBObject("city", 1).append("region", 1).append("district_name", 1)).addOption(Bytes.QUERYOPTION_NOTIMEOUT).each {BasicDBObject dbo-> -////// dbo.removeField("_id") -////// def query = dbo -////// WriteResult writeResult = FANGJIA_STAT.remove(query) -////// total += writeResult.getField("n") -////// println(total) -//////} -////Date date = DateUtil.parseDate("20161118 16", "yyyyMMdd HH") -// -////DBCollection TEMP = MongoUtils.getCollectionByName("wenyan", "copy_alias_to_addr_alias", "192.168.0.136") -//// -////Date date = DateUtil.parseDate("20161121 15:30:30") -////CommonUtils.findAllCity().each {def city-> -//// DbUtils.executeQueryOnSeaweed(DbUtils.buildSeaweedQuery(city),{BasicDBObject district-> -//// List alias = district.alias -//// -//// def orgAddrAlias = [] -//// List addrAlias = district.addr_alias -//// if(!addrAlias){ -//// addrAlias = [] -//// }else{ -//// orgAddrAlias.addAll(addrAlias) -//// } -//// -//// boolean isMifity = false -//// if(alias){ -//// for(String a : district.alias){ -//// if(a ==~ /.{2,}(路|道|街).*\d(弄|号)/ && !addrAlias.contains(a) && district.address != a){ -//// addrAlias << a -//// isMifity = true -//// } -//// } -//// if(isMifity){ -//// def update = new BasicDBObject() -//// update.append("addr_alias", addrAlias) -//// update.append("m_date", date) -//// DbUtils.executeUpdateOnSeaweed(new BasicDBObject("_id", district._id), update) -//// println(district.toString() + "------------->" + update) -//// -//// def log = new BasicDBObject() -//// log.append("district_id", district._id.toString()) -//// log.append("m_date", date) -//// log.append("org_addr_alias", orgAddrAlias) -//// log.append("addr_alias", addrAlias) -//// TEMP.save(log) -//// } -//// } -//// -//// },new BasicDBObject("m_date", 1).append("addr_alias", 1).append("alias", 1).append("address", 1)) -////} -// -////ExcelUtils.modify("E:\\Users\\wenyan\\Desktop\\住宅价格评估.xlsx",{List item-> -//// -//// if(!item['D'] ||!item['E']){ -//// return -//// } -//// String city = "上海" -//// String region = null -//// String name = item['C'] -//// String address = item['D'] -//// Double size = item['E'].toString().toDouble() -//// JSONObject resp = null -//// resp = OpenApiClient.addressMatch(city, address, "district" , false, region , "fuzzy") -//// -//// if(resp && resp.getDouble("credit") > 0){ -//// item['F'] = resp.name -//// if(size && size > 0){ -//// region = resp.district -//// String estate = DbUtils.seaweed.findOne(new BasicDBObject("city", city).append("region", region).append("name", item['F']).append("cat","district").append("visible",0)).estate_type2 -//// def propertyType = "0" -//// if(estate.contains("别墅")){ -//// propertyType = "1" -//// } -//// Map params = new HashMap(); -//// params.put("city", city); -//// params.put("district", region ); -//// params.put("name", item['F']); -//// params.put("scope", "1") -//// params.put("propertyType", String.valueOf(propertyType)) -//// params.put("__refresh_cache","1") -//// params.put("size", size.toString()) -//// JSONObject evaluateResponse = OpenApiClient.evaluateDistrictPrice(params) -//// if((evaluateResponse == null || evaluateResponse.getDouble("credit") == 0)){ -//// if(propertyType == "1"){ -//// params.put("propertyType", "0") -//// }else{ -//// params.put("propertyType", "1") -//// } -//// evaluateResponse = OpenApiClient.evaluateDistrictPrice(params) -//// } -//// if(evaluateResponse && evaluateResponse.getDouble("credit") > 0){ -//// item['G'] = evaluateResponse.avgPrice -//// item['H'] = evaluateResponse.totalPrice -//// }else{ -//// item['G'] = evaluateResponse.avgPrice -//// item['H'] = evaluateResponse.totalPrice -//// } -//// } -//// } -//// println(item) -//// -////}) -////DBCollection OPEN_URI_LOG_COLL = MongoUtils.getCollectionByName("business", "open_uri_log", "192.168.0.60"); -////def apis = ["/property/evaluate","/property/avgPrice","/city/list","/property/search","/property/deal","/property/searchRoom","/property/house","/property/detail","/district/avgPrice","/city/districtAvgPrice","/city/avgPrice2","/city/regions","/block/avgPrice","/property/price","/property/facilities","/property/similarHouseAmout","/property/nearbySimilarHouseAmout","/property/similarHouseRentRange","/property/nearlySimilarHouseRentRange"] -////def data = [] -////def title= ["API"] -////int i = 0 -////for(def api : apis){ -//// def calender = Calendar.getInstance() -//// calender.setTime(DateUtil.parseDate("20151001", "yyyyMMdd")) -//// def item = [] -//// item << api -//// while (calender.getTime().before(new Date())){ -//// def start = calender.getTime() -//// if(i == 0){ -//// title << start.format("yyyy年MM月") -//// } -//// calender.add(Calendar.MONTH, 1) -//// def end = calender.getTime() -//// item << OPEN_URI_LOG_COLL.count(new BasicDBObject("user_id", "1007496").append("request_uri",api).append("c_date", new BasicDBObject('$gte', start).append('$lt', end))) -//// } -//// if(i == 0){ -//// data << title -//// i++ -//// println(title) -//// } -//// data << item -//// println(item) -////} -////ExcelUtils.write("D:\\测试账号统计结果.xls", data, "111") -////def cities = ["佛山","大连","太原","长春","平顶山","镇江","保定","三亚","锦州","安庆","铜川"] -////def data = [] -////data << ["ID","城市","区域","小区名","地址"] -////cities.each {def city-> -//// -//// def dateQuery = new BasicDBObject("c_date", new BasicDBObject('$gt', Date.parse("yyyyMMdd", "20161123")).append('$lt', Date.parse("yyyyMMdd","20161125"))) -//// def query = new BasicDBObject("city",city).append("cat", "district").append("visible", 1) -//// query.putAll(dateQuery) -//// DbUtils.executeQueryOnSeaweed(query,{BasicDBObject district-> -//// data << [district._id.toString(), city, district.region, district.name, district.address] -//// }, new BasicDBObject("region", 1).append("name",1).append("address", 1)) -////} -////ExcelUtils.write("D:\\无效数据.xls", data, "无效数据") -////DBCollection SERVER_AREA_COLL = MongoUtils.getCollectionByName("fangjia", "server_area", "mongo.db.fangjia.com") -////DBObject q = new BasicDBObject(); -////def cursor = SERVER_AREA_COLL.find(q).sort(new BasicDBObject('city_grade', 1).append('sequence', 1)); -////def allCity = []; -////if (cursor) { -//// for (def dbo : cursor) { -//// def city = dbo.get("display"); -//// if (city != '全国') { -//// String url = "http://${dbo.name}.fangjia.com/trend/trendChart?&defaultCityName=${URLEncoder.encode(city,"UTF-8")}&chartWidth=550&chartHeight=270&__ajax=1&__refresh_cache=1" -//// String url2 = "http://${dbo.name}.fangjia.com/trend/quarterData?&defaultCityName=${URLEncoder.encode(city,"UTF-8")}&chartWidth=550&chartHeight=270&__ajax=1&__refresh_cache=1" -//// String url3 = "http://${dbo.name}.fangjia.com/trend/yearData?&defaultCityName=${URLEncoder.encode(city,"UTF-8")}&chartWidth=550&chartHeight=270&__ajax=1&__refresh_cache=1" -//// String url4 = "http://${dbo.name}.fangjia.com/trend/halfData?&defaultCityName=${URLEncoder.encode(city,"UTF-8")}&chartWidth=550&chartHeight=270&__ajax=1&__refresh_cache=1" -//// try{ -//// new HttpRequest().get(url,[:],"UTF-8") -//// new HttpRequest().get(url2,[:],"UTF-8") -//// new HttpRequest().get(url3,[:],"UTF-8") -//// new HttpRequest().get(url4,[:],"UTF-8") -//// }catch (Exception e){ -//// -//// } -//// -//// } -//// } -////} -////new File("E:\\Users\\wenyan\\Desktop\\mongodb.rsmains0.log.2016-11-28.log").eachLine("UTF-8"){String line-> -//// println(line) -////} -// -////["上海"].each {def city-> -//// def data = [] -//// data << ["城市","区域","名称","名称别名","地址","地址别名","物业类型","建筑类型","是否存在","创建时间"] -//// DbUtils.executeQueryOnSeaweed(DbUtils.buildSeaweedQuery(city),{BasicDBObject dbo-> -//// if(!hasPrice(dbo.city, dbo.region, dbo.name,dbo.estate_type2)){ -//// def names = [dbo.name] -//// if(dbo.alias){ -//// dbo.alias.each {String ali-> -//// if(StringUtils.isNotBlank(ali)){ -//// names << ali -//// } -//// } -//// } -//// def query = new BasicDBObject() -//// query.append("city", city) -//// query.append("district_name", new BasicDBObject('$in', names)) -//// query.append("cat", "addr_district") -//// boolean has = false -//// DbUtils.seabird.find(query).each { -//// if(has){ -//// return -//// } -//// if(it.region == dbo.region){ -//// has = true -//// } -//// } -//// def item = [dbo.city, dbo.region, dbo.name,dbo.alias?.join(","),dbo.address,dbo.addr_alias?.join(","),dbo.estate_type2,dbo.build_type,has? "有":"无", dbo.c_date?.format("yyyy-MM-dd")] -//// data << item -//// println(item) -//// } -//// }) -//// ExcelUtils.write("E:\\Users\\wenyan\\Desktop\\无均价\\${city}.xlsx", data, "无均价小区") -////} -//// -////def boolean hasPrice(def city, def region, def name, def type){ -//// DBCollection district_stat = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com") -//// def weekend = DateUtil.getFirstDayOfWeek(new Date()) -//// def query = new BasicDBObject() -//// query.append("city", city) -//// query.append("region", region) -//// query.append("district_name", name) -//// query.append("weekend", weekend) -//// if(type && type.contains("别墅")){ -//// query.append("estate_type","别墅") -//// }else{ -//// query.append("estate_type","小区") -//// } -//// return district_stat.findOne(query) != null -////} -////ExcelUtils.modify("E:\\Users\\wenyan\\Desktop\\IP地址测试结果.xlsx",{List item-> -//// if(item['M']){ -//// def data = GeoUtils.geoconvBaiduToGoogle(item['M'],item['N']) -//// item['M'] = data.lng -//// item['N'] = data.lat -//// item['P'] = GeoUtils.calculateDistance(item['I'], item['j'],item['M'],item['N']) -//// } -//// println(item) -////}) -////def data = [] -////data << ["城市","区域","名称","地址"] -////DbUtils.seaweed.find(new BasicDBObject("city", "上海").append("cat", "office").append("visible", 0).append("region", new BasicDBObject('$in', ["静安","黄浦"]))).each {BasicDBObject district-> -//// data << [district.city, district.region, district.name,district.address] -////} -////ExcelUtils.write("D:\\写字楼.xlsx", data, "数据") -// -////ExcelUtils.read("E:\\Users\\wenyan\\Desktop\\常州江南银行住宅--更新ing(2).xls").each {List item-> -//// def one = (BasicDBObject)DbUtils.seaweed.findOne(DbUtils.buildSeaweedQuery("常州",item['B'], item['c'])) -//// if(one){ -//// if(!one.mark_map){ -//// one.mark_map = new HashMap() -//// } -//// one.mark_map.isNanJingBank = new Integer(1) -//// DbUtils.seaweed.save(one) -//// } -//// -////} -////def shopData = [] -////shopData << ["城市","区域","大类","已安装名称","地址"] -////for(sheetIndex in 5 ..32){ -//// def data = ExcelUtils.read("E:\\Users\\wenyan\\Desktop\\分众一二线城市楼宇点位表(分类入库).xls", sheetIndex) -//// data.each {List item-> -//// def cat = item['D'].toString().replaceAll("\\s", "") -//// if(cat == "商住两用楼" || cat == "高级公寓楼"){ -//// shopData << [item["B"], item["C"], item["D"], item["E"], item["F"]] -//// } -//// } -////} -//////ExcelUtils.write("E:\\Users\\wenyan\\Desktop\\分众一二线城市楼宇点位表(分类入库).xls", shopData, "小区库&楼栋库") -////def file = "E:\\Users\\wenyan\\Desktop\\新建文件夹\\分众一二线城市楼宇点位提取小区.xls" -//////ExcelUtils.multiThreadingModify(file, {List item-> -////// if(item['F'] == null){ -////// JSONObject result = OpenApiClient.addressMatch(item['A'], item['D'], "property", false, null, "fuzzy", false) -////// if(result.getDouble("credit") > 0.5){ -////// item["F"] = result.district -////// item['G'] = result.name -////// item['H'] = "名称" -////// println(item) -////// } -////// } -//////}) -//// -////for(int i =0; i < 3;i ++){ -////// ExcelUtils.modify(file, {List item-> -////// item['D'] = filter(item['A'], item['B'], item['D']) -////// println(item) -////// },i) -//// ExcelUtils.distinct(file ,[0,1,2,3,4,5,6,7],i) -////} -////def filter(String city, String region, String name){ -//// if(name == null){ -//// return null -//// } -//// name = name.replace("(" + city + ")", '').replace("(" + city + ")", '').replace('(商务楼)', '').replaceAll('商务楼$', '') -//// name = name.replace("(写字楼)", "").replace("(商铺)", "").replace("(商场)", "").replace("(公寓楼)","").replace("(办公楼)", "").replace("(商场)", "") -//// name = name.replace("-办公楼", "").replaceAll('(\\(|().+(店|商场|商务广场|号梯|楼)(\\)|))', '').replace('副楼', '').replace('转换区', '') -//// name = name.replaceAll('(\\(|().+路.*(\\)|))', '').replace("裙房", "").replaceAll('(\\(|()[东南西北](\\)|))', '') -//// name = name.replaceAll('(\\(|()[东南西北]楼(\\)|))', '').replace('(裙楼)', '').replaceAll('-?[东南西北](塔|楼)', '') -//// name = name.replaceAll("裙楼\$", '').replaceAll('主楼$', '').replaceAll('\\d+号厅', '') -//// name = name.replaceAll('(' + region + ")", "") -//// name = name.replaceAll("([东南西北前后左右偏侧ABCDEFGHIGK一二三四五六七八九十\\d]|(\\d+号))门\$", '').replace('(索迪)', '').replace('(百货商厦)', '') -//// def stage = DataUtil.match(name , Pattern.compile('(\\(|()[东南西北ABCDEFGHIGK一二三四五六七八九十]+(期|区)(\\)|))')) -//// if(stage){ -//// name = name.replace(stage, stage.replaceAll("\\(|\\)|(|)", '')) -//// println(stage + "-->" + name) -//// } -//// name = name.replaceAll('(\\(|().*(\\)|))', '').replaceAll("第[一二三四五六七八九十]座", '') -//// /*************************************************************************************/ -//// name = name.replaceAll("[\\d+A-Z甲乙丙丁戊戌](栋|幢|楼)", '').replaceAll('[\\d+ABCDEFG一二三四五六七八九十]单元', '').replaceAll("\\d+号楼", '') -//// .replaceAll("[A-Z\\d]座", '').replaceAll('\\d+号$', '').replaceAll("\\d+号梯", '').replaceAll('[左右]厅', '') -//// /*************************************************************************************/ -//// return name -////} -//DBCollection district_stat = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.136") -//district_stat.find(new BasicDBObject(), new BasicDBObject("city", 1).append("name", 1).append("region", 1).append("s_date", 1)).sort(new BasicDBObject('_id', -1)).each {BasicDBObject d-> -// if(d.getString("city").length() > 40 || d.getString("region").length() > 40 || d.getString("name").length() > 40){ -// district_stat.remove(new BasicDBObject("_id", d._id)) -// println(d) -// } -//} -// -// -// -// -// -//def count = 0 -//DBCollection BID_RECORDS_COLL = MongoUtils.getCollectionByName("taobao", "bid_record", "192.168.0.136", 27017) -//BID_RECORDS_COLL.find(new BasicDBObject(), new BasicDBObject("city", 1).append("title", 1)).batchSize(1000).each {BasicDBObject dbo-> -// String region = RegionData.parseRegionFormAddress(dbo.city, dbo.title) -// if(RegionData.isRegion(dbo.city, region)){ -// BID_RECORDS_COLL.update(new BasicDBObject("_id", dbo._id), new BasicDBObject('$set', new BasicDBObject("region", region))) -// } -// if(++count % 1000 == 0){ -// println(count) -// } -//} - -//ExcelUtils.multiThreadingModify("E:\\数据\\日志\\小区自动填充\\覆盖率统计\\小区字段覆盖率统计201703071003.xlsx",{List item-> -// for(int i=1; i < item.size(); i++){ -// item[i] = 100 - item[i] -// } -//}) -//ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\阳光保险10月估价结果.xlsx",{List item-> -// if(item["S"] && item["S"].toString().toDouble().toInteger() == 1 && !item["M"]){ -// def city = CommonUtils.parseCityInAddress(item["B"]) -// JSONObject resp = OpenApiClient.addressMatch(city, item["B"], "property", false, null, item["D"], null, false) -// if(resp.name){ -// item["M"] = resp.district -// item["N"] = resp.name -// println(item) -// } -// } -//}) \ No newline at end of file +def citys = ["北京", "唐山", "天津", "秦皇岛", "石家庄", "包头", "太原", "丹东", "呼和浩特", "锦州", "沈阳", "吉林", "大连", "牡丹江", "长春", "无锡", "哈尔滨", "扬州", "上海", "徐州", "南京", "温州", "杭州", "金华", "宁波", "蚌埠", "合肥", "安庆", "福州", "泉州", "厦门", "九江", "南昌", "赣州", "济南", "烟台", "青岛", "济宁", "郑州", "洛阳", "武汉", "平顶山", "长沙", "宜昌", "广州", "襄阳", "深圳", "岳阳", "南宁", "常德", "海口", "惠州", "重庆", "湛江", "成都", "韶关", "贵阳", "桂林", "昆明", "北海", "西安", "三亚", "兰州", "泸州", "西宁", "南充", "银川", "遵义", "乌鲁木齐", "大理", "北京", "上海", "广州", "深圳", "重庆", "郑州", "长沙", "西安", "成都", "东莞", "佛山", "杭州", "合肥", "南京", "青岛", "沈阳", "苏州", "天津", "武汉", "珠海", "中山", "长春", "扬州", "烟台", "徐州", "无锡", "温州", "太原", "台州", "石家庄", "绍兴", "厦门", "泉州", "宁波", "南通", "南宁", "南昌", "兰州", "常州", "大连", "福州", "贵阳", "哈尔滨", "海口", "惠州", "济南", "嘉兴", "金华", "昆明", "遵义", "淄博", "株洲", "镇江", "肇庆", "漳州", "湛江", "岳阳", "银川", "宜昌", "盐城", "芜湖", "乌鲁木齐", "唐山", "泰州", "汕头", "三亚", "清远", "秦皇岛", "马鞍山", "洛阳", "柳州", "连云港", "廊坊", "九江", "吉林", "淮安", "湖州", "呼和浩特", "沧州", "保定", "安庆", "张家口", "西宁", "日照", "拉萨", "锦州", "大同", "常德", "北海", "文昌", "陵水", "保亭"] +citys.unique() +citys.each { def city -> + if (RegionData.isCity(city)) { + ElasticUtils.executeSqlQuery("select count(*) as count from seawater where p_date > 20210407 and city='${city}'", { + println("$city\t${it.count}") + }) + } +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/Test4.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/Test4.groovy index bef30856c..95d41237b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/Test4.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/Test4.groovy @@ -6,34 +6,50 @@ import com.fangjia.script.util.DataUtil import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection +import net.sf.json.JSONObject +import org.apache.commons.lang.StringUtils +DBCollection SEAWEED_TOWNSHIP = MongoUtils.getCollectionByName("fangjia_base", "seaweed_township", "192.168.0.235", 27018) +def count = 0 +List> data = Collections.synchronizedList([]) +data << ["ID","城市","区域","小区名","经度","纬度","街道"] - -//ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\数据.xlsx",{List item-> -// if(DbUtils.seaweed.findOne(DbUtils.buildSeaweedQuery(item["A"], item["B"], item["C"]))){ -// item["D"] = "存在" -// }else{ -// item["D"] = "不存在" -// } -//}) - - -DBCollection DISTRICT_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat","mongo.db.fangjia.com") -ProvinceData.getAllCity("福建").each {def city-> - int count = 0 - def s = new TreeMap() - DISTRICT_STAT.find(new BasicDBObject("weekend", Date.parse("yyyy-MM-dd", "2020-04-19")).append("city", city)).each {BasicDBObject stat-> - count ++ - int credit = (stat.credit * 100)/10 - s[credit] = s.get(credit, 0) + 1 - } - - def data = [] - data << city - data << count - for(int i=0; i <=10; i++){ - data << s.get(i, 0) - } - println(data.join(",")) +CommonUtils.findAllCity().each {def city-> + DbUtils.multiThreadingExecuteQueryOnSeaweed(DbUtils.buildSeaweedQuery(city),{BasicDBObject item-> + if(StringUtils.isBlank(item.township) && item.lng){ + count++ + def lng = item.lng/10000000000d + def lat = item.lat/10000000000d + def location = "${lng},${lat}" + JSONObject resp = GaoDeApiUtils.geocode(location) + def row = [] + row << item._id.toString() + row << item.city + row << item.region + row << item.name + row << lng + row << lat + BasicDBObject dbo = new BasicDBObject() + dbo.id = item._id.toString() + dbo.city = item.city + dbo.region = item.region + dbo.name = item.name + dbo.lng = lng + dbo.lat = lat + if(resp.regeocode && resp.regeocode.addressComponent && resp.regeocode.addressComponent.township){ + row << resp.regeocode.addressComponent.township + dbo.township = resp.regeocode.addressComponent.township + } + println(row) + data << row + SEAWEED_TOWNSHIP.insert(dbo) + } + }) } +ExcelUtils.write("D:\\街道补充数据.xlsx", data, "补充数据") + + + + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/Abc.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/Abc.groovy index d561f9f47..0e7cff16b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/Abc.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/Abc.groovy @@ -1,6 +1,8 @@ package com.fangjia.script.wenyan.temp +import com.fangjia.core.Constants import com.fangjia.core.util.DataUtil +import com.fangjia.core.util.MongoUtils import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.OpenApiClient @@ -9,294 +11,33 @@ import net.sf.json.JSONObject import java.util.regex.Matcher import java.util.regex.Pattern -// -//def map = [:] -//ExcelUtils.read("D:\\商务数据\\上海农商行\\测试数据(上海)-上海农商行第二批0829.xlsx", 1).each {List item-> -// if(item["AF"] && item["AH"]){ -// def key = "${item["AE"]}_${item["AF"]}_${item["AH"]}".toString() -// map.get(key, []) << key -// } -//} -//ExcelUtils.modify("D:\\商务数据\\上海农商行\\测试数据(上海)-上海农商行第二批(房价网交付0905).xlsx", { List item -> -//// if(item["AF"] && item["AH"]){ -//// def key = "${item["AE"]}_${item["AF"]}_${item["AH"]}".toString() -//// List nums = map.get(key) -//// item["AT"] = nums.size() -//// } -// -// def city = "上海" -// def addr = item["AC"] -// if(addr){ -// JSONObject resp = OpenApiClient.addressMatch(city, addr, "property") -// JSONObject searchAddress = resp.getJSONObject("searchAddress") -// println searchAddress -//// item["AD"] = searchAddress.province -//// item["AE"] = searchAddress.city -//// item["AF"] = searchAddress.district -// if(searchAddress.standardAddress){ -//// item["AG"] = searchAddress.street -//// if(searchAddress.streetNumber){ -//// item["AG"] = item["AG"] + searchAddress.streetNumber -//// } -// -// if(searchAddress.township){ -// item["AP"] = searchAddress.township -// } -// } -// -// if(item["AG"]){ -// def addrMap = getRoad(item["AG"]) -// if(addrMap){ -// if(addrMap.road){ -// item["AQ"] = addrMap.road -// } -// if(addrMap.roadNumber){ -// item["AR"] = addrMap.roadNumber -// } -// } -// } -// } -// -// -//// if (item["AH"] && item["AF"]) { -//// BasicDBObject seaweed = DbUtils.seaweed.findOne(DbUtils.buildSeaweedQuery(item["AE"], item["AF"], item["AH"])) -//// if (seaweed) { -//// if (item["AR"] && item["AS"]) { -//// JSONObject eva = OpenApiClient.evaluateDistrictPrice(seaweed.city, seaweed.region, seaweed.name, item["O"], seaweed.estate_type2, item["AS"].toInteger(), item["AR"].toInteger()) -//// if (eva) { -//// item["AL"] = eva.avgPrice -//// item["AM"] = eva.totalPrice -//// } -//// } else { -//// JSONObject eva = OpenApiClient.evaluateDistrictPrice2(seaweed.city, seaweed.region, seaweed.name, item["O"], seaweed.estate_type2) -//// if (eva) { -//// item["AL"] = eva.avgPrice -//// item["AM"] = eva.totalPrice -//// } -//// } -//// } -//// -//// } -// -//},1) - - -// if(item["AK"] =~ /^\d{4}$/){ -// item["AR"] = item["AK"].toString().substring(0, 2) -// }else if(item["AK"] =~ /^\d{3}$/){ -// item["AR"] = item["AK"].toString().substring(0, 1) -// } -// if(item["AR"] && item["AH"]){ -// def key = "${item["AE"]}_${item["AF"]}_${item["AH"]}".toString() -// List nums = map.get(key) -// nums.sort() -// item["AS"] = nums[nums.size() - 1] -// item["AT"] = nums.unique().size() -// } -// println(item) -// item["AG"] = null -// def city = "上海" -// def addr = item["AC"] -// def area = item["o"] -// JSONObject resp = OpenApiClient.addressMatch(city, addr, "property") -// JSONObject searchAddress = resp.getJSONObject("searchAddress") -// item["AD"] = searchAddress.province -// item["AE"] = searchAddress.city -// item["AF"] = searchAddress.district -// if(searchAddress.standardAddress){ -// item["AG"] = searchAddress.street -// if(searchAddress.streetNumber){ -// item["AG"] = item["AG"] + searchAddress.streetNumber -// } -// } -// -// -// if(resp.getDouble("credit") > 0){ -// item["AE"] = resp.city -// item["AF"] = resp.district -// item["AH"] = resp.name -// JSONObject eva = OpenApiClient.evaluateDistrictPrice2(resp.city, resp.district, resp.name, area, resp.propertyType) -// item["AL"] = eva.avgPrice -// item["AM"] = eva.totalPrice -// item["AN"] = new Date().format("yyyy-MM-dd") -// if(!item["AG"]){ -// item["AG"] = resp.address -// } -// } -// item["AI"] = searchAddress.buildingNumber -// item["AJ"] = searchAddress.unitNumber -// item["AK"] = searchAddress.roomNumber -// println(item) -//},1) -// -def sheetIndexs = ["湘潭": 0, "嘉兴": 1, "苏州": 2] -sheetIndexs.each { String city, int sheetIndex -> -// def map = [:] -// ExcelUtils.read("D:\\商务数据\\上海农商行\\异地测试-上海农商行第二批0828(测试)1.xlsx", sheetIndex).each {List item-> -// if(item["Q"] && item["S"]){ -// def key = "${item["P"]}_${item["Q"]}_${item["S"]}".toString() -// map.get(key, []) << key -// } -// } - ExcelUtils.multiThreadingModify("D:\\商务数据\\上海农商行\\异地测试-上海农商行第二批(房价网交付0905).xlsx", { List item -> -// if(item["Q"] && item["S"]){ -// def key = "${item["P"]}_${item["Q"]}_${item["S"]}".toString() -// List nums = map.get(key) -// nums.sort() -// item["AE"] = nums.size() -// } - -// item["W"] = null -// item["X"] = null - -// BasicDBObject seaweed = DbUtils.seaweed.findOne(DbUtils.buildSeaweedQuery(item["P"], item["Q"], item["S"])) -// if(seaweed){ -// if(item["AC"] && item["AD"]){ -// JSONObject eva = OpenApiClient.evaluateDistrictPrice(seaweed.city, seaweed.region, seaweed.name, item["L"], seaweed.estate_type2, item["AD"].toInteger(), item["AC"].toInteger()) -// if(eva){ -// item["W"] = eva.avgPrice -// item["X"] = eva.totalPrice -// } -// }else{ -// JSONObject eva = OpenApiClient.evaluateDistrictPrice2(seaweed.city, seaweed.region, seaweed.name, item["L"], seaweed.estate_type2) -// if(eva){ -// item["W"] = eva.avgPrice -// item["X"] = eva.totalPrice -// } -// } -// } -// } -// if(item["AC"] && item["S"]){ -// def key = "${item["P"]}_${item["Q"]}_${item["S"]}".toString() -// List nums = map.get(key) -// nums.sort() -// item["AD"] = nums[nums.size() - 1] -// item["AE"] = nums.unique().size() -// } - -// if(item["V"] =~ /^\d{4}$/){ -// item["AC"] = item["V"].toString().substring(0, 2) -// }else if(item["V"] =~ /^\d{3}$/){ -// item["AC"] = item["V"].toString().substring(0, 1) -// }else if(item["V"] =~ /^\d{6,7}$/){ -// item["AC"] = item["V"].toString().substring(0, 2).replaceAll("^0", "") -// } - - - def addr = item["B"].toString().replaceAll("(", "").replace('区/县', "区").replace("街道/镇", "镇") - .replaceAll(")", "").replaceAll("【.*】", "").replaceAll("室/单元", "号") -// def area = item["L"] - - - def region = null - if (city == "嘉兴") { - region = "嘉善" - } else if (city == "苏州") { - region = "昆山" - } - JSONObject resp = OpenApiClient.addressMatch(city, addr, "property", false, region) - JSONObject searchAddress = resp.getJSONObject("searchAddress") - - def township = "" - if(searchAddress && searchAddress.township){ - item["AP"] = searchAddress.township - township = searchAddress.township - } - if(item["S"]){ - def addrMap = getRoad(item["S"]) - if(addrMap){ - if(addrMap.road){ - def s = addrMap.road; - if(township){ - if(township.contains(s)){ - item["AQ"] = s.replace(township, "") - }else{ - if(township.contains("镇")){ - township = township.replace("镇","") - } - item["AQ"] = s.replace(township, "") - } - }else{ - item["AQ"] = addrMap.road - } - } - if(addrMap.roadNumber){ - item["AR"] = addrMap.roadNumber - } - } - } - - - println searchAddress -// item["O"] = searchAddress.province -// item["P"] = searchAddress.city -// item["Q"] = searchAddress.district -// if (searchAddress.standardAddress) { -// item["R"] = searchAddress.street -// if (searchAddress.streetNumber) { -// item["R"] = item["R"] + searchAddress.streetNumber -// } -// } -// if (resp.getDouble("credit") > 0) { -// item["O"] = resp.province -// item["P"] = resp.city -// item["Q"] = resp.district -// item["S"] = resp.name -// JSONObject eva = OpenApiClient.evaluateDistrictPrice2(resp.city, resp.district, resp.name, area, resp.propertyType) -// item["W"] = eva.avgPrice -// item["X"] = eva.totalPrice -// item["Y"] = new Date().format("yyyy-MM-dd") -// if (!item["R"]) { -// item["R"] = resp.address -// } -// } -// item["T"] = searchAddress.buildingNumber -// item["U"] = searchAddress.unitNumber -// item["V"] = searchAddress.roomNumber -// if(city == "湘潭"){ -// String roomNumber = DataUtil.match(addr, Pattern.compile("\\d+(号|(号房))\$")) -// if(roomNumber){ -// item["V"] = roomNumber.replaceAll("号", '').replaceAll('房', "") -// } -// roomNumber = DataUtil.match(addr, Pattern.compile("\\d{6,8}\$")) -// if(roomNumber){ -// item["V"] = roomNumber.replaceAll("号", '').replaceAll('房', "") -// } -// -// } +def school = MongoUtils.getDbCollection(Constants.FANGJIA_CRAWL_MONGO_URI, "fangjia_craw", "school2") +new File("D:\\数据\\学校数据").eachFile {File file-> + ExcelUtils.read(file.getAbsolutePath(), 0).each {List item-> + BasicDBObject dbo = new BasicDBObject() + dbo.city = item["A"] + dbo.region = item["B"] + dbo.name = item["C"] + dbo.address = item["E"] + dbo.attribute = item["G"] + dbo.property = item["H"] + dbo.category = "小学" + dbo.location = [DataUtil.roundDouble(item["I"], 6), DataUtil.roundDouble(item["J"], 6)] + school.insert(dbo) + println(dbo) + } + ExcelUtils.read(file.getAbsolutePath(), 1).each {List item-> + BasicDBObject dbo = new BasicDBObject() + dbo.city = item["A"] + dbo.region = item["B"] + dbo.name = item["C"] + dbo.address = item["E"] + dbo.attribute = item["G"] + dbo.property = item["H"] + dbo.category = "中学" + dbo.location = [item["I"], item["J"]] + school.save(dbo) println(item) - }, sheetIndex) -} - - -def getRoad(def s){ - def rex = ".*(路|街道|街|大道|巷|道|里).*\\d+(弄|号)\$" - Pattern pattern = Pattern.compile(rex); - Matcher matcher = pattern.matcher(s); - - List list = new ArrayList(); - while (matcher.find()) { - list.add(matcher.group()); } - if(list && list.size()>0){ - def s1 = list[0] - def road = getRexText(s1, ".*(路|街道|街|大道|巷|道|里)") - def num = getRexText(s1, "\\d+") - return ["road":road, "roadNumber":num] - }else{ - def road = getRexText(s, ".*(路|街道|街|大道|巷|道|里)") - return ["road":road] - } - return null -} - -def getRexText(s, regx){ - Pattern pattern = Pattern.compile(regx); - Matcher matcher = pattern.matcher(s); - - while (matcher.find()) { - return matcher.group(); - } - return null; } \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/Api.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/Api.groovy index 1c9e130af..3e76b3eef 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/Api.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/Api.groovy @@ -1,9 +1,16 @@ package com.fangjia.script.wenyan.temp +import com.fangjia.core.Constants import com.fangjia.core.ProvinceData import com.fangjia.core.RegionData +import com.fangjia.core.util.MongoUtils +import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.OpenApiClient +import com.mongodb.BasicDBObject +import net.sf.json.JSONObject import org.apache.commons.lang3.StringUtils +import org.openqa.selenium.json.Json //ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\阳光保险重跑数据(1).xlsx", { List item -> @@ -30,8 +37,6 @@ import org.apache.commons.lang3.StringUtils //}, 0,"结果") - - //ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\阳光保险重跑数据(1).xlsx", { List item -> // if (item["T"] == null || ![1, 2, 4].contains(item["T"].toString().toDouble().toInteger())) { // return @@ -114,43 +119,80 @@ import org.apache.commons.lang3.StringUtils // println("${key} ${names.join(",")}") // } //} -ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\地址标准化.xlsx",{List item-> - String address = ""; - if(StringUtils.isNotBlank(item["B"]) && !ProvinceData.isGovernmentCity(item["B"])){ - address = ProvinceData.getDisplay(item["B"]) +//ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\地址标准化.xlsx",{List item-> +// String address = ""; +// if(StringUtils.isNotBlank(item["B"]) && !ProvinceData.isGovernmentCity(item["B"])){ +// address = ProvinceData.getDisplay(item["B"]) +// } +// if(StringUtils.isNotBlank(item["C"])){ +// String cityDisplay = CityData.getCityDisplay(item["C"]); +// if(cityDisplay != null){ +// address = address + cityDisplay +// }else{ +// address = address + item["C"] + "市" +// } +// } +// if(StringUtils.isNotBlank(item["D"])){ +// String orgDistrict = RegionData.getOrgName(item["C"], item["D"]); +// if(StringUtils.isNotBlank(orgDistrict) && address.indexOf(orgDistrict) == -1){ +// address = address + orgDistrict +// } +// } +// if(StringUtils.isNotBlank(item["E"])){ +// address = address + item["E"] +// } +// if(StringUtils.isNotBlank(item["F"])){ +// address = address + item["F"] +// } +// if(StringUtils.isNotBlank(item["G"])){ +// address = address + item["G"] +// } +// if(StringUtils.isNotBlank(item["H"])){ +// address = address + item["H"] + "栋" +// } +// if(StringUtils.isNotBlank(item["i"])){ +// address = address + item["i"] + "单元" +// } +// if(StringUtils.isNotBlank(item["J"])){ +// address = address + item["J"] + "室" +// } +// item["K"] = address +// println(address) +//}) +def school = MongoUtils.getDbCollection(Constants.FANGJIA_MONGO_URI, "fangjia", "guiding_price") +ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\深圳市住建局二手房指导价.xlsx", { List item -> + if (item["G"]) { + BasicDBObject seaweed = DbUtils.seaweed.findOne(DbUtils.buildSeaweedQuery(item["G"], item["H"], item["I"])) + BasicDBObject guid = new BasicDBObject() + guid.guiding_type = 0 + guid.estate_type = seaweed.estate_type2 + guid.house_type = 0 + guid.c_date = new Date() + guid.guiding_person = "深圳房管局" + guid.guiding_date = Date.parse('yyyyMMdd', '20210411') + guid.category = "district" + guid.guiding_value = (int)item["E"] + guid.name = item["D"] + guid.city = item["G"] + guid.region = item["H"] + guid.name = item["i"] + school.insert(guid) + println(guid) + } - if(StringUtils.isNotBlank(item["C"])){ - String cityDisplay = CityData.getCityDisplay(item["C"]); - if(cityDisplay != null){ - address = address + cityDisplay - }else{ - address = address + item["C"] + "市" - } - } - if(StringUtils.isNotBlank(item["D"])){ - String orgDistrict = RegionData.getOrgName(item["C"], item["D"]); - if(StringUtils.isNotBlank(orgDistrict) && address.indexOf(orgDistrict) == -1){ - address = address + orgDistrict - } - } - if(StringUtils.isNotBlank(item["E"])){ - address = address + item["E"] - } - if(StringUtils.isNotBlank(item["F"])){ - address = address + item["F"] - } - if(StringUtils.isNotBlank(item["G"])){ - address = address + item["G"] - } - if(StringUtils.isNotBlank(item["H"])){ - address = address + item["H"] + "栋" - } - if(StringUtils.isNotBlank(item["i"])){ - address = address + item["i"] + "单元" - } - if(StringUtils.isNotBlank(item["J"])){ - address = address + item["J"] + "室" - } - item["K"] = address - println(address) +// if(!item["F"]){ +// JSONObject resp = OpenApiClient.addressMatch("深圳", item["D"], 'property', false, item["B"]) +// if(resp.id){ +// BasicDBObject seaweed = DbUtils.seaweed.findById(resp.id) +// if(seaweed){ +// item["F"] = seaweed._id.toString() +// item["G"] = seaweed.city +// item["H"] = seaweed.region +// item["I"] = seaweed.name +// } +// println(item) +// } +// +// } + }) \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/Bcc0622.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/Bcc0622.groovy index 33458dadc..0d24b2d81 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/Bcc0622.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/Bcc0622.groovy @@ -1,82 +1,149 @@ package com.fangjia.script.wenyan.temp -import com.fangjia.core.util.HouseValidator +import com.fangjia.core.Constants +import com.fangjia.core.util.DateUtil +import com.fangjia.core.util.MongoUtils +import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ExcelUtils -import com.fangjia.script.util.OpenApiClient -import net.sf.json.JSONObject +import com.fangjia.script.util.FangJiaAreaMapUtils +import com.mongodb.BasicDBObject +import org.bson.types.ObjectId -ExcelUtils.multiThreadingModify("D:\\商务数据\\建设银行\\副本5市成交案例(测试).xlsx",{List item-> -// if(item["R"]){ -// return -// } -// def calcDate = null -// if(item["E"]){ -// calcDate = item["E"].toString().substring(0, 10) -// }else{ -// calcDate = item["B"].toString().substring(0, 10) -// } -// def floor = item["I"].toString().toDouble().toInteger().toString() -// def maxFloor = item["J"].toString().toDouble().toInteger().toString() -// def dir = HouseValidator.formatDirectionToCode(item["L"]).toString() -// def fit = HouseValidator.formatFitmentToCode(item["M"]).toString() -// def params =[:] -// params.city = item["F"] -// params.district = item["p"] -// params.name = item['Q'] -// params.size = item["C"].toString() -// params.floor = floor -// params.maxFloor = maxFloor -// params.direction = dir -// params.decoration = fit -// params.calDate = calcDate -// params.__refresh_cache = "1" -// params.scope = "1" -// JSONObject evaResp = OpenApiClient.evaluate(params) -// if(evaResp.result && evaResp.result.credit > 0.4){ -// item["R"] = evaResp.result.totalPrice -// } +def school = MongoUtils.getDbCollection(Constants.FANGJIA_CRAWL_MONGO_URI, "fangjia_craw", "school2") +//school.find(new BasicDBObject("source", "查学区")).each {BasicDBObject item-> +// item.category = item.attribute +// item.attribute = null +// school.save(item) // println(item) - -// if(item["H"] != "中粮云景花园"){ -// return +//} +//ExcelUtils.read("D:\\Users\\wenyan\\Downloads\\无经纬度学校-结果.xlsx").each {List item-> +// if(item["P"] != null){ +// def location = item["P"].split(",") as List +// def data = new BasicDBObject() +// data.map_name = item["N"] +// data.map_region = item["R"] +// data.map_address = item["N"] +// data.map_type = item["S"] +// data.location = [location[0].toString().toDouble(),location[1].toString().toDouble()] +// school.update(new BasicDBObject("_id", new ObjectId(item["A"])), new BasicDBObject('$set', data)) +// println(data) // } - def city = item["F"] - def region = item["G"] - def name = item["H"].toString() - def addr = item["D"] - JSONObject resp = OpenApiClient.addressMatch(city, addr, "property" , false, region, name) - if(resp.name){ - item["p"] = resp.district - item['Q'] = resp.name +//} +regionCodeMap = [:] +def cityCodeMap = ["苏州":"320500","合肥":"340100", "西安":"610000"] +for(def city : cityCodeMap.keySet()){ + def datas = FangJiaAreaMapUtils.convertToOfficials(city); + for(def d : datas){ + regionCodeMap.put(city+"-"+d.fangjia_district, d.district_code) } - println(item) -// def calcDate = null -// if(item["E"]){ -// calcDate = item["E"].toString().substring(0, 10) -// }else{ -// calcDate = item["B"].toString().substring(0, 10) -// } -// def floor = item["I"].toString().toDouble().toInteger().toString() -// def maxFloor = item["J"].toString().toDouble().toInteger().toString() -// def dir = HouseValidator.formatDirectionToCode(item["L"]).toString() -// def fit = HouseValidator.formatFitmentToCode(item["M"]).toString() -// def params =[:] -// params.city = resp.city -// params.district = resp.district -// params.name = resp.name -// params.size = item["C"].toString() -// params.floor = floor -// params.maxFloor = maxFloor -// params.direction = dir -// params.decoration = fit -// params.calDate = calcDate -// params.scope = "1" -// JSONObject evaResp = OpenApiClient.evaluate(params) -// if(evaResp.result && evaResp.result.credit > 0.4){ -// item["R"] = evaResp.result.totalPrice -// } -// println(item) -// } +} +def data = Collections.synchronizedList([]) +data << ["ID","CITYID","DISTID","COMMNAME","LNG","LAT","ADDRESS","PS","PS_ADDR","PS_TYPE","PS_SEQNO","PS_LONGITUDE","PS_LATITUDE","MS","MS_ADDR","MS_TYPE","MS_SEQNO","MS_LONGITUDE","MS_LATITUDE","REMARK","CREATETIME","UPDATETIME","SOURCECODE"] +cityCodeMap.keySet().each {def city-> + DbUtils.multiThreadingExecuteQueryOnSeaweed(DbUtils.buildSeaweedQuery(city),{BasicDBObject seaweed-> + if(seaweed.lng){ + def item = [] + item << seaweed._id.toString() + item << cityCodeMap.get(seaweed.city) + item << regionCodeMap.get(seaweed.city + "-" + seaweed.region) + item << seaweed.name + def lng = com.fangjia.core.util.DataUtil.roundDouble(seaweed.lng/10000000000d, 6) + def lat = com.fangjia.core.util.DataUtil.roundDouble(seaweed.lat/10000000000d, 6) + item << lng + item << lat + item << seaweed.address + BasicDBObject location = new BasicDBObject() + BasicDBObject near = new BasicDBObject() + BasicDBObject geometry = new BasicDBObject("type", "Point") + geometry.append("coordinates", [lng, lat]) + near.append('$maxDistance', 1000) + near.append('$geometry', geometry) + location.append('$near', near) + BasicDBObject query = new BasicDBObject() + query.append("location", location) + query.append("attribute",new BasicDBObject('$in', ['省重点', '市重点', '区重点', '县重点', '省级示范'])) + query.append("category", new BasicDBObject('$in', ['中学'])) + BasicDBObject midSchool = school.findOne(query) +// for(def level in ['省重点', '市重点', '区重点', '县重点', '普通']){ +// query.append("location", location) +// query.append("attribute", level) +// midSchool = school.findOne(query) +// if(midSchool){ +// break +// } +// } +// if(!midSchool){ +// query.removeField("attribute") +// midSchool = school.findOne(query) +// } + BasicDBObject primarySchool=null + query.append("category", new BasicDBObject('$in', ['小学'])) + primarySchool = school.findOne(query) +// for(def level in ['省重点', '市重点', '区重点', '县重点', '普通']){ +// query.append("location", location) +// query.append("attribute", level) +// primarySchool = school.findOne(query) +// if(primarySchool){ +// break +// } +// } +// if(!primarySchool){ +// query.removeField("attribute") +// primarySchool = school.findOne(query) +// } + if(primarySchool){ + item << primarySchool.name + item << primarySchool.address + item << primarySchool.attribute + item << null + item << primarySchool.location[0] + item << primarySchool.location[1] + }else { + item << null + item << null + item << null + item << null + item << null + item << null + } + if(midSchool){ + item << midSchool.name + item << midSchool.address + item << midSchool.attribute + item << null + item << midSchool.location[0] + item << midSchool.location[1] + }else { + item << null + item << null + item << null + item << null + item << null + item << null + } + item << null + Date c_date = seaweed.c_date + Date m_date = seaweed.m_date + if(!c_date && !m_date){ + c_date = DateUtil.add(new Date(), Calendar.YEAR, -1) + } + if(!c_date){ + c_date = m_date + } + if(!m_date){ + m_date = c_date + } + item < OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log") -def startDate = DateUtil.parseDate("20210301", "yyyyMMdd") -def endDate = DateUtil.parseDate("20210401", "yyyyMMdd") +def startDate = DateUtil.parseDate("20210401", "yyyyMMdd") +def endDate = DateUtil.parseDate("20210501", "yyyyMMdd") Calendar calendar = Calendar.getInstance() calendar.setTime(startDate) def count = 0 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/SiChuanJingCheng.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/SiChuanJingCheng.groovy index c03f780d8..08bf4c5b7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/SiChuanJingCheng.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/SiChuanJingCheng.groovy @@ -11,26 +11,44 @@ import org.bson.Document import static com.mongodb.client.model.Filters.* MongoCollection OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log") -def startDate = DateUtil.parseDate("20210301", "yyyyMMdd") -def endDate = DateUtil.parseDate("20210401", "yyyyMMdd") +def startDate = DateUtil.parseDate("20210401", "yyyyMMdd") +def endDate = DateUtil.parseDate("20210501", "yyyyMMdd") Calendar calendar = Calendar.getInstance() calendar.setTime(startDate) def count = 0 def hashSet = [] -def data = [["用户ID", "用户名称", "IP", "访问链接", "访问时间", "dataStatus", "code", "msg", "参数", "resMap"]] +def data = [] +def title = ["用户ID", "用户名称", "IP", "访问链接", "访问时间", "dataStatus", "code", "msg"] +def param_keys = ["tid", "city", "district", "address", "name", "size", "propertyType"] +param_keys.each { + title << "参数_${it}" +} +def resMap_keys = ["city", "district", "name", "avgPrice", "totalPrice", "buildYear"] +resMap_keys.each { + title << "结果_${it}" +} +data << title +def keys = new HashSet() while (startDate.before(endDate)) { calendar.add(Calendar.DATE, 1) def nextDate = calendar.getTime() count++ OPEN_URI_LOG.find(and(eq("user_id", "1007721"), eq("request_uri", "/property/evaluate3"), eq("data_status", 1), lt("c_date", nextDate), gte("c_date", startDate))).each { Document doc -> def result = JSONObject.fromObject(doc.res_map.result) - if(!result.buildYear || !result.avgPrice){ + if (!result.buildYear || !result.avgPrice) { return } def params = JSONObject.fromObject(doc.parameter_map) + keys.addAll(params.keys()) def key = "${params.address}_${params.size}".toString() if (!hashSet.contains(key)) { - def item = ["1007721", "scjc", doc.ip, doc.request_uri, doc.c_date, doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()] + def item = ["1007721", "scjc", doc.ip, doc.request_uri, doc.c_date.format("yyy"), doc.data_status, doc.code, doc.msg] + param_keys.each { + item << params[it] + } + resMap_keys.each { + item << result[it] + } data << item hashSet << key } else { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/Test.groovy index 8c05b9c5a..f64572b09 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/Test.groovy @@ -153,55 +153,55 @@ import org.apache.commons.lang3.StringUtils //} DBCollection BAIDUMAP_BORDER_COLL = MongoUtils.getCollectionByName("addrepo", "baidumap_border", "192.168.0.206", 27020) -BAIDUMAP_BORDER_COLL.find(new BasicDBObject("baidu_border", null)).each {BasicDBObject dbo-> - def border = dbo.getString("border").replace("1-", "").replace(";", "").split(",") as List - def coords = [] - def baidu_points = [] - def gaode_points = [] - for(int i=0; i < border.size() - 1; i=i+2){ - coords << "${border[i]},${border[i+1]}" - if(coords.size() > 50){ - def json=BaiDuApiUtils.geoconv(coords.join(";"), 6, 5) - for(JSONObject it : json.result){ - baidu_points << "${it.x},${it.y}" - def geo = GeocodeUtils.geoconvBaiduToGoogle(it.x, it.y) - gaode_points << "${DataUtil.roundDouble(geo.lng, 6)},${DataUtil.roundDouble(geo.lat,6)}" - } - coords.clear() - } - } - if(coords.size() > 0){ - def json=BaiDuApiUtils.geoconv(coords.join(";"), 6, 5) - for(JSONObject it : json.result){ - baidu_points << "${it.x},${it.y}" - def geo = GeocodeUtils.geoconvBaiduToGoogle(it.x, it.y) - gaode_points << "${DataUtil.roundDouble(geo.lng, 6)},${DataUtil.roundDouble(geo.lat,6)}" - } - } - dbo.format_city = RegionData.formatCity(dbo.city) - dbo.baidu_border = baidu_points.join(";") - dbo.gaode_border = gaode_points.join(";") - BAIDUMAP_BORDER_COLL.save(dbo) - println(dbo) -} - -//ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\人口测试数据.xlsx",{List item-> -// def seaweed = DbUtils.seaweed.findOne(DbUtils.buildSeaweedQuery(item["B"], item["C"], item["D"]), new BasicDBObject("alias", 1).append("lng",1).append("lat", 1)) -// if(seaweed){ -// item["A"] = seaweed._id.toString() -// def names = [item['D']] -// if(seaweed.alias){ -// names.addAll(seaweed.alias) +//BAIDUMAP_BORDER_COLL.find(new BasicDBObject("baidu_border", null)).each {BasicDBObject dbo-> +// def border = dbo.getString("border").replace("1-", "").replace(";", "").split(",") as List +// def coords = [] +// def baidu_points = [] +// def gaode_points = [] +// for(int i=0; i < border.size() - 1; i=i+2){ +// coords << "${border[i]},${border[i+1]}" +// if(coords.size() > 50){ +// def json=BaiDuApiUtils.geoconv(coords.join(";"), 6, 5) +// for(JSONObject it : json.result){ +// baidu_points << "${it.x},${it.y}" +// def geo = GeocodeUtils.geoconvBaiduToGoogle(it.x, it.y) +// gaode_points << "${DataUtil.roundDouble(geo.lng, 6)},${DataUtil.roundDouble(geo.lat,6)}" +// } +// coords.clear() // } -// names.unique() +// } +// if(coords.size() > 0){ +// def json=BaiDuApiUtils.geoconv(coords.join(";"), 6, 5) +// for(JSONObject it : json.result){ +// baidu_points << "${it.x},${it.y}" +// def geo = GeocodeUtils.geoconvBaiduToGoogle(it.x, it.y) +// gaode_points << "${DataUtil.roundDouble(geo.lng, 6)},${DataUtil.roundDouble(geo.lat,6)}" +// } +// } +// dbo.format_city = RegionData.formatCity(dbo.city) +// dbo.baidu_border = baidu_points.join(";") +// dbo.gaode_border = gaode_points.join(";") +// BAIDUMAP_BORDER_COLL.save(dbo) +// println(dbo) +//} + +ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\小区群像测试数据.xlsx",{List item-> + def seaweed = DbUtils.seaweed.findOne(DbUtils.buildSeaweedQuery(item["B"], item["C"], item["D"]), new BasicDBObject("alias", 1).append("lng",1).append("lat", 1)) + if(seaweed){ +// item["A"] = seaweed._id.toString() + def names = [item['D']] + if(seaweed.alias){ + names.addAll(seaweed.alias) + } + names.unique() // if(seaweed.lng){ // item["G"] = DataUtil.roundDouble(seaweed.lng/10000000000d, 6) // item["H"] = DataUtil.roundDouble(seaweed.lat/10000000000d, 6) // } -// def one = BAIDUMAP_BORDER_COLL.findOne(new BasicDBObject("format_city", item["B"]).append("format_region", item["C"]).append("name", new BasicDBObject('$in', names))) -// if(one){ -// item["I"] = one.baidu_border -// } -// println(item) -// } -//}) \ No newline at end of file + def one = BAIDUMAP_BORDER_COLL.findOne(new BasicDBObject("format_city", item["B"]).append("format_region", item["C"]).append("name", new BasicDBObject('$in', names))) + if(one){ + item["I"] = one.gaode_border + } + println(item) + } +}) \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/test/Test1.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/test/Test1.groovy new file mode 100644 index 000000000..401d647e4 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/test/Test1.groovy @@ -0,0 +1,46 @@ +package com.fangjia.script.wenyan.test + +import com.aliyuncs.DefaultAcsClient +import com.aliyuncs.IAcsClient +import com.aliyuncs.address_purification.model.v20191118.GetAddressSimilarityRequest +import com.aliyuncs.address_purification.model.v20191118.GetAddressSimilarityResponse +import com.aliyuncs.profile.DefaultProfile +import com.fangjia.core.util.HttpRequest +import com.fangjia.script.util.ExcelUtils +import net.sf.json.JSONObject + +ExcelUtils.modify("D:\\nextcloud\\数据\\地址类服务测试\\建行地址相似度评级测试.xlsx",{List item-> + def city = item["A"] + def address1 = item["B"] + def address2 = item["C"] + HttpRequest http = new HttpRequest() + def params = new HashMap() + params["city"] = city + params["address_1"] = address1 + params["address_2"] = address2 + params["token"]= "F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB" + JSONObject json = http.getJsonContent("http://open.fangjia.com/address/similarity", params, "UTF-8") + JSONObject result = json.result + item["D"] = result?.level + item["E"] = result?.formattedAddress_1 + item["F"] = result?.formattedAddress_2 + item["G"] = result?.isSameAddress + DefaultProfile profile = DefaultProfile.getProfile( + "cn-hangzhou", + "LTAIdaj4eSO79gzz", + "xw6EsvqydZ9HC6zAvIB3ruQqVHynxf"); + IAcsClient client = new DefaultAcsClient(profile); + GetAddressSimilarityRequest request = new GetAddressSimilarityRequest(); + request.setServiceCode("addrp") + request.setAppKey("Xcgtyhupey0h") + request.setText("${address1};${address2}") + GetAddressSimilarityResponse response = client.getAcsResponse(request) + String data = response.getData() + json = JSONObject.fromObject(data) + item["H"] = json["sim_level"]["level"] + item["I"] = json["sim_level"]["address_a"] + item["J"] = json["sim_level"]["address_b"] + item["K"] = json["sim_level"]["is_same"] + println(json) +}) + diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserPatterns.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserPatterns.java index 6015a13ea..2ffb4363c 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserPatterns.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserPatterns.java @@ -7,14 +7,14 @@ import java.util.regex.Pattern; */ public interface AddressParserPatterns { - Pattern FULL_ADDRESS = Pattern.compile("^.{2,}(路|街|大道|巷|道|((零|一|二|三|四|五|六|七|八|九|十|东|南|西|北|复)[线环条甲])|(胡同))(交汇|.{0,3}段|.*弄)?((([甲乙]?[零一二三四五六七八九十百千\\d]+甲?号?)(?!楼)[~\\-~、,-])?([甲乙]?[零一二三四五六七八九十百千\\d])+甲?(号)(?!楼)+([付附]+[零一二三四五六七八九十百千\\d]+(号)?)?)"); + Pattern FULL_ADDRESS = Pattern.compile("^.{2,}(路|街|大道|巷|道|((零|一|二|三|四|五|六|七|八|九|十|东|南|西|北|复)[线环条甲])|(胡同))(交汇|.{0,3}段|.*弄)?((([甲乙]?[零一二三四五六七八九十百千\\d]+甲?号?)(?!楼)[~\\-~、,-])?([甲乙]?[零一二三四五六七八九十百千\\d])+甲?(号)(?!楼)+([付附]+[零一二三四五六七八九十百千\\d]+(号)?+([-]+[零一二三四五六七八九十百千\\d]+(号)?)?)?)"); Pattern STREET = Pattern.compile("^.{2,}(路|街|大道|巷|道|((零|一|二|三|四|五|六|七|八|九|十|东|南|西|北|复)[线环条甲])|(胡同))(交汇|.{0,3}段|.*弄)?"); - Pattern STREET_NUMBER = Pattern.compile(".*([甲乙]?[零|一三四五六七八九十百千\\d]+甲?(号)(?!楼)?[~\\--~、,])?([甲乙]?[零一二三四五六七八九十百千\\d]+甲?(号)(?!楼)+([付附]+[零一二三四五六七八九十百千\\d]+(号)?)?)"); + Pattern STREET_NUMBER = Pattern.compile(".*([甲乙]?[零一二三四五六七八九十百千\\d]+甲?(号)(?!楼)?[~\\--~、,])?([甲乙]?[零一二三四五六七八九十百千\\d]+甲?(号)(?!楼)+([付附]+[零一二三四五六七八九十百千\\d]+(号)?+([-]+[零一二三四五六七八九十百千\\d]+(号)?)?)?)"); Pattern STAGE_PATTERN = Pattern.compile("(([零一二三四五六七八九十百千\\d]+)期?[,、至到]?)?([零一二三四五六七八九十百千\\d]+)期"); Pattern PARTITION_PATTERN = Pattern.compile("(\\w|东|南|北|中|西|甲|乙|丙|丁|零|一|二|三|四|五|六|七|八|九|十)+(区|坊)"); Pattern TOWN = Pattern.compile("^.+?(乡|镇|街道)+"); Pattern NUMBER = Pattern.compile("\\d+"); - Pattern STANDARD_ADDRESS = Pattern.compile("^.{2,}(路|街|大道|巷|道|((零|一|二|三|四|五|六|七|八|九|十|东|南|西|北|复)[线环条甲])|(胡同))(交汇|.{0,3}段|.*弄)?((([零一二三四五六七八九十百千\\d])+弄)|([甲乙]?([零一二三四五六七八九十百千\\d]))+号(?!楼)+([付附]+[零一二三四五六七八九十百千\\d]+(号)?)?)"); + Pattern STANDARD_ADDRESS = Pattern.compile("^.{2,}(路|街|大道|巷|道|((零|一|二|三|四|五|六|七|八|九|十|东|南|西|北|复)[线环条甲])|(胡同))(交汇|.{0,3}段|.*弄)?((([零一二三四五六七八九十百千\\d])+弄)|([甲乙]?([零一二三四五六七八九十百千\\d]))+号(?!楼)+([付附]+[零一二三四五六七八九十百千\\d]+(号)?+([-]+[零一二三四五六七八九十百千\\d]+(号)?)?)?)"); Pattern DISTRICT = Pattern.compile("^.{2,}?(区|县)"); } diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java index 47d32dc97..46545fc04 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java @@ -1,13 +1,12 @@ package com.fangjia.service.address; import com.fangjia.core.RegionData; -import com.fangjia.core.util.DataUtil; -import com.fangjia.core.util.GeoUtils; -import com.fangjia.core.util.StringUtil; +import com.fangjia.core.util.*; import com.fangjia.dal.QueryParam; import com.fangjia.dal.common.DistrictKey; import com.fangjia.dal.common.ServerAreaData; import com.fangjia.dal.mongo.fangjia.model.ServerArea; +import com.fangjia.dal.mongo.model.AmapPoi; import com.fangjia.dal.mongo.model.bkoff.HouseLoudong; import com.fangjia.dal.mongo.model.bkoff.Loudong; import com.fangjia.dal.seaweed.dao.SeaweedDao; @@ -17,12 +16,14 @@ import com.fangjia.service.address.processor.AddressDistrictProcessors; import com.fangjia.service.address.processor.AddressProcessors; import com.fangjia.service.api.AliyunService; import com.fangjia.service.bo.AliyunBo; +import com.fangjia.service.bo.MatchAddressScore; import com.fangjia.service.datastore.DistrictService; import com.fangjia.service.mongo.addrepo.AddressRepoService; import com.fangjia.service.mongo.bkoff.HouseLoudongService; import com.fangjia.service.mongo.bkoff.LoudongService; import com.google.common.base.Function; import com.google.common.collect.MapMaker; +import net.sf.json.JSONObject; import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; import org.bson.types.ObjectId; @@ -45,7 +46,7 @@ public class AddressParserServiceImpl implements AddressParserService { private LoudongService loudongService; private SeaweedDao seaweedDao; private AliyunService aliyunService; - private final String[] MATCH_PROPERTY_FIELDS = new String[]{"id", "city", "region", "name", "address", "category", "block", "alias", "addrAlias", "estateType2", "lat2", "lng2", "streetCommunity"}; + private final String[] MATCH_PROPERTY_FIELDS = new String[]{"id", "city", "region", "name", "address", "category", "block", "alias", "addrAlias", "estateType2", "lat2", "lng2", "township"}; private static final Logger logger = Logger.getLogger(AddressParserServiceImpl.class); @@ -132,6 +133,8 @@ public class AddressParserServiceImpl implements AddressParserService { formatAddress(building); } return building; + /*JSONObject result = getDataFromCRF("test"); + return new Building();*/ } @Override @@ -163,35 +166,188 @@ public class AddressParserServiceImpl implements AddressParserService { return building; } Building matchedBuilding; + //调用CRF算法切分数据 + /*JSONObject jsonObj = null; + if(params.isUseCrf() && AddressParserGoal.Match.equals(addr.getAddressParserGoal())){ + jsonObj = getDataFromCRF(addr.getOrgAddress()); + if(jsonObj != null){ + String poi = (jsonObj.containsKey("community_model") ? jsonObj.getString("community_model") : "") + + (jsonObj.containsKey("poi_model") ? jsonObj.getString("poi_model") : "") + + (jsonObj.containsKey("subpoi_model") ? jsonObj.getString("subpoi_model") : ""); + if(StringUtils.isNotBlank(poi)){ + if(addr.getKeywords()==null || !addr.getKeywords().contains(poi)){ + addr.setKeywords(poi); + } + } + } + }*/ //keywords使用aliyun接口返回的poi AliyunBo aliyunBo = null; - if(params.isUseAliyunApi()){ + if(params.isUseAliyunApi() && AddressParserGoal.Match.equals(addr.getAddressParserGoal())){ aliyunBo = aliyunService.addressStand(params.getAddress(), params.getCity()); if(aliyunBo != null){ String poi = ((aliyunBo.getPoi() == null ? "":aliyunBo.getPoi()) + (aliyunBo.getSubpoi() == null ? "":aliyunBo.getSubpoi())).replaceAll("\\s*", ""); if(StringUtils.isNotBlank(poi)){ - addr.setKeywords(poi); + if(addr.getKeywords()==null || !addr.getKeywords().contains(poi)){ + addr.setKeywords(poi); + } } if(StringUtils.isNotBlank(aliyunBo.getCity())){ - addr.setSuggestCity(aliyunBo.getCity()); + addr.setCity(RegionData.formatCity(aliyunBo.getCity())); + } + if(StringUtils.isNotBlank(addr.getKeywords()) && addr.getOrgAddress().indexOf(addr.getKeywords())>-1){ + String str = AddressParserUtil.match(Pattern.compile(".*村(.*(组|队))?\\d+(号)?$"), addr.getOrgAddress().substring(addr.getOrgAddress().indexOf(addr.getKeywords()),addr.getOrgAddress().length())); + if(StringUtils.isNotBlank(str)){ + String newKeywords = AddressParserUtil.match(Pattern.compile(".*村(.*(组|队))?"), str); + if(StringUtils.isNotBlank(newKeywords)){ + addr.setKeywords(newKeywords); + addr.setParsedPropertyName(newKeywords); + aliyunBo.setRoomno(str.replace(newKeywords,"")); + } + } } } } + addr.addKeyword(addr.getOrgAddress()); if(params.isUseMap()){ matchedBuilding = findMatchedBuilding(addr, params.getBuildingType(), MetaDataType.MAP); }else{ matchedBuilding = findMatchedBuilding(addr, params.getBuildingType(), MetaDataType.CPDB); } + + /*if(jsonObj != null){ + //乡镇优化 + String township = jsonObj.containsKey("town") ? jsonObj.getString("town") : ""; + if(StringUtils.isBlank(addr.getTownship()) && StringUtils.isNotBlank(township)){ + String town = AddressParserUtil.match(AddressParserPatterns.TOWN, township); + if(StringUtils.isNotBlank(town)){ + addr.setTownship(town); + } + } + //道路优化(两边均去掉乡镇) + String street = addr.getStreet(); + String streetNumber = addr.getStreetNumber(); + String streetCrf = jsonObj.containsKey("road") ? jsonObj.getString("road") : ""; + String streetNumberCrf = jsonObj.containsKey("roadno") ? jsonObj.getString("roadno") : ""; + //去掉路名中含有乡镇名的 + if(StringUtils.isNotBlank(addr.getTownship())){ + street = StringUtils.isNotBlank(street) ? street.replace(addr.getTownship(),"") : street; + streetCrf = StringUtils.isNotBlank(streetCrf) ? streetCrf.replace(addr.getTownship(),"") : streetCrf; + } + + //crf解析出的路名为空、与Java解析出路名一致或不满足正则则不处理 + if(StringUtils.isNotBlank(streetCrf) && !streetCrf.equals(street) && StringUtils.isNotBlank(AddressParserUtil.match(AddressParserPatterns.STREET, streetCrf))){ + //Java解析出路名为空 + if(StringUtils.isBlank(street)){ + street = streetCrf; + if(StringUtils.isNotBlank(streetNumberCrf)){ + streetNumber = streetNumberCrf; + } + }else{ + //CRF包含java解析出的路名或Java解析出的路名包含CRF解析出的路名并且多余部分也满足路名正则 + if(streetCrf.contains(street) || (street.contains(streetCrf) && StringUtils.isBlank(AddressParserUtil.match(AddressParserPatterns.STREET, street.replace(streetCrf,""))))){ + street = streetCrf; + } + } + addr.setStreet(street); + addr.setStreetNumber(streetNumber); + } + //poi处理 + //没有匹配上库中小区时 + if(StringUtils.isBlank(matchedBuilding.getId())){ + String poi = (jsonObj.containsKey("community") ? jsonObj.getString("community") : "") + + (jsonObj.containsKey("poi") ? jsonObj.getString("poi") : "") + + (jsonObj.containsKey("subpoi") ? jsonObj.getString("subpoi") : ""); + //去掉路名中含有乡镇名和路名 getParsedPropertyName + if(StringUtils.isNotBlank(addr.getTownship())){ + poi = poi.replace(addr.getTownship(),""); + } + if(StringUtils.isNotBlank(addr.getStreet())){ + poi = poi.replace(addr.getStreet(),""); + } + addr.setParsedPropertyName(poi); + } + //楼栋、单元、室号 + String houseNo = (jsonObj.containsKey("houseno") ? jsonObj.getString("houseno") : "").replaceAll("[\u4E00-\u9FA5]", ""); + String cellNo = (jsonObj.containsKey("cellno") ? jsonObj.getString("cellno") : "").replaceAll("[\u4E00-\u9FA5]", ""); + String floorNo = (jsonObj.containsKey("floorno") ? jsonObj.getString("floorno") : "").replaceAll("[\u4E00-\u9FA5]", ""); + String roomNo = (jsonObj.containsKey("roomno") ? jsonObj.getString("roomno") : "").replaceAll("[\u4E00-\u9FA5]", ""); + addr.setOrgBuildingNumber(houseNo); + addr.setBuildingNumber(houseNo); + addr.setOrgUnitNumber(cellNo); + addr.setUnitNumber(cellNo); + addr.setFloor(floorNo); + addr.setOrgRoomNumber(roomNo); + addr.setRoomNumber(roomNo); + }*/ matchedBuilding.setProvince(addr.getProvince()); matchedBuilding.setSearchAddress(addr); matchedBuilding.setSuggestName(getSuggestName(matchedBuilding)); matchedBuilding.setSuggestAddress(getSuggestAddress(matchedBuilding)); - if(params.isUseAliyunApi() && aliyunBo != null){ + if(params.isUseAliyunApi() && aliyunBo != null && AddressParserGoal.Match.equals(addr.getAddressParserGoal())){ completBuildingField(matchedBuilding, aliyunBo, params); } + Address tempAddr = matchedBuilding.getSearchAddress(); + if((StringUtils.isBlank(tempAddr.getStreet()) || StringUtils.isBlank(tempAddr.getStreetNumber())) && AddressParserGoal.Match.equals(addr.getAddressParserGoal())){ + AddressMatchMetaData meta = getMapMeta(addr); + if(meta != null && (tempAddr.getKeywords()!=null && tempAddr.getKeywords().equals(meta.getMainName()))){ + matchedBuilding.setAddress(meta.getAddress()); + } + } + matchedBuilding.setSearchAddress(tempAddr); return matchedBuilding; } + private AddressMatchMetaData getMapMeta(Address tempAddr){ + tempAddr.addKeyword(tempAddr.getOrgAddress()); + List metaDatas = findMatchMetaData(tempAddr, null, MetaDataType.MAP, false); + Map> metaDataMap = new HashMap>(); + if(StringUtils.isBlank(tempAddr.getDistrict()) && !metaDatas.isEmpty()){ + for(AddressMatchMetaData metaData:metaDatas){ + if(StringUtils.isNotBlank(metaData.getMapCity())){ + List metaList = metaDataMap.containsKey(metaData.getMapCity()) ? metaDataMap.get(metaData.getMapCity()) : new ArrayList(); + metaList.add(metaData); + metaDataMap.put(metaData.getMapCity(),metaList); + } + } + String tempCity = ""; + if(StringUtils.isNotBlank(tempAddr.getCity()) && metaDataMap.containsKey(tempAddr.getCity())){ + tempCity = tempAddr.getCity(); + }else{ + int size = 0; + for(String key:metaDataMap.keySet()){ + if(metaDataMap.get(key).size() > size){ + size = metaDataMap.get(key).size(); + tempCity = key; + } + } + } + if(StringUtils.isNotBlank(tempCity)){ + AddressMatchMetaData meta = metaDataMap.get(tempCity).get(0); + return meta; + } + } + return null; + } + + /** + * TODO 省市区乡镇采用原始拆分数据,若乡镇数据没有使用 + * @param address + * @return + */ + /*private JSONObject getDataFromCRF(String address){ + HttpRequest request = new HttpRequest(); + Map params = new HashMap<>(); + params.put("address",address); + try { + JSONObject resp = request.getJsonContent("http://127.0.0.1:8802/standard", params, "UTF-8"); + System.out.println(resp); + return resp; + }catch(Exception e){ + + } + return null; + }*/ /** * 信息不全时使用阿里接口补全数据 * @param matchedBuilding @@ -208,28 +364,52 @@ public class AddressParserServiceImpl implements AddressParserService { } } Address addr = matchedBuilding.getSearchAddress(); - if(StringUtils.isBlank(addr.getProvince())){ + if(StringUtils.isNotBlank(aliyunBo.getProv())){ + addr.setProvince(formatProvince(aliyunBo.getProv())); + } + if(StringUtils.isNotBlank(aliyunBo.getCity())){ + addr.setCity(RegionData.formatCity(aliyunBo.getCity())); + } + if(StringUtils.isNotBlank(aliyunBo.getDistrict())){ + addr.setDistrict(RegionData.formatRegion(aliyunBo.getDistrict())); + } + addr.setTownship(aliyunBo.getTown()); + + /*if(StringUtils.isBlank(addr.getProvince())){ addr.setProvince(formatProvince(aliyunBo.getProv())); } if(StringUtils.isBlank(addr.getCity())){ addr.setCity(RegionData.formatCity(aliyunBo.getCity())); } - - if(StringUtils.isBlank(addr.getDevelopmentZone())){ - addr.setDevelopmentZone(aliyunBo.getDevzone()); - } if(StringUtils.isBlank(addr.getDistrict())){ addr.setDistrict(RegionData.formatRegion(aliyunBo.getDistrict())); } if(StringUtils.isBlank(addr.getTownship())){ addr.setTownship(aliyunBo.getTown()); + }*/ + if(StringUtils.isBlank(addr.getDevelopmentZone())){ + addr.setDevelopmentZone(aliyunBo.getDevzone()); } - addr.setOrgBuildingNumber(aliyunBo.getHouseno()); + + /*addr.setOrgBuildingNumber(aliyunBo.getHouseno()); addr.setBuildingNumber(aliyunBo.getHouseno()); addr.setOrgUnitNumber(aliyunBo.getCellno()); - addr.setUnitNumber(aliyunBo.getCellno()); - addr.setOrgRoomNumber(aliyunBo.getRoomno()); - addr.setRoomNumber(aliyunBo.getRoomno()); + addr.setUnitNumber(aliyunBo.getCellno());*/ + /*if(aliyunBo.getRoomno()!=null && !aliyunBo.getRoomno().matches(".*(层|楼)$")){ + addr.setOrgRoomNumber(aliyunBo.getRoomno()); + addr.setRoomNumber(aliyunBo.getRoomno()); + }else{ + addr.setRoomNumber(null); + addr.setRoomNumber(null); + }*/ + + /*String aliRoom = aliyunBo.getRoomno() == null ? null : aliyunBo.getRoomno().replaceAll("[\u4E00-\u9FA5]", ""); + String orgRoom = addr.getOrgRoomNumber() == null ? null : addr.getOrgRoomNumber().replaceAll("[\u4E00-\u9FA5]", ""); + if(orgRoom!=null && orgRoom.equals(aliRoom)){ + addr.setRoomNumber(aliyunBo.getRoomno()); + }else{ + addr.setRoomNumber(null); + }*/ if(StringUtils.isBlank(addr.getStreet())){ addr.setStreet(aliyunBo.getRoad()); @@ -318,6 +498,7 @@ public class AddressParserServiceImpl implements AddressParserService { addr.setSuggestDistrict(params.getSuggestDistrict()); addr.setSuggestName(params.getSuggestName()); addr.setPreciseLevel(params.getPreciseLevel()); + addr.setAddressParserGoal(params.getAddressParserGoal()); addr.setProvince(params.getProvince()); addr.setDebugName(params.getDebugName()); return AddressProcessors.process(addr); @@ -362,7 +543,7 @@ public class AddressParserServiceImpl implements AddressParserService { * @param metaDataType * @return */ - private List findMatchMetaData(final Address address, final BuildingType buildingType, MetaDataType metaDataType){ + private List findMatchMetaData(final Address address, final BuildingType buildingType, MetaDataType metaDataType, boolean validCity){ if(MetaDataType.MAP == metaDataType){ List keywords = address.getKeywordList(); if(keywords.isEmpty()){ @@ -375,7 +556,7 @@ public class AddressParserServiceImpl implements AddressParserService { } keywords.add(standardAddress); } - return addressRepoService.findAllByKeyword(address.getCity(), buildingType, keywords, address.getLng(), address.getLat(), 100); + return addressRepoService.findAllByKeyword(address.getCity(), buildingType, keywords, address.getLng(), address.getLat(), 100, validCity); }else{ return SEAWEED_LOCAL_CACHE.get(new MetaDataCacheKey(address.getCity(), buildingType, MetaDataType.CPDB)); } @@ -517,20 +698,20 @@ public class AddressParserServiceImpl implements AddressParserService { scoreResult.setOrgAddress(addr.getOrgAddress()); if(buildingType != null) { if(MetaDataType.MAP == metaDataType){ - metaDatas = findMatchMetaData(addr, buildingType, MetaDataType.MAP); - metaDatas.addAll(findMatchMetaData(addr, buildingType, MetaDataType.CPDB)); + metaDatas = findMatchMetaData(addr, buildingType, MetaDataType.MAP, true); + metaDatas.addAll(findMatchMetaData(addr, buildingType, MetaDataType.CPDB, true)); }else{ - metaDatas = findMatchMetaData(addr, buildingType, MetaDataType.CPDB); + metaDatas = findMatchMetaData(addr, buildingType, MetaDataType.CPDB, true); } } else { if(MetaDataType.MAP == metaDataType){ - metaDatas = findMatchMetaData(addr, null, MetaDataType.MAP); + metaDatas = findMatchMetaData(addr, null, MetaDataType.MAP, true); } for (BuildingType bt : addr.guessBuildType()) { if(metaDatas == null){ - metaDatas = findMatchMetaData(addr, bt, MetaDataType.CPDB); + metaDatas = findMatchMetaData(addr, bt, MetaDataType.CPDB, true); }else{ - metaDatas.addAll(findMatchMetaData(addr, bt, MetaDataType.CPDB)); + metaDatas.addAll(findMatchMetaData(addr, bt, MetaDataType.CPDB, true)); } } } @@ -960,15 +1141,24 @@ public class AddressParserServiceImpl implements AddressParserService { if(StringUtils.isNotBlank(address.getTownship())){ formattedAddressBuilder.append(address.getTownship()); } + if(StringUtils.isNotBlank(building.getAddress())){ + Address orgAddr = new Address(); + orgAddr.setOrgAddress(building.getAddress()); + Address addr = AddressProcessors.process(orgAddr); + if(StringUtils.isBlank(address.getStreet())){ + address.setStreet(addr.getStreet()); + } + if(StringUtils.isBlank(address.getStreetNumber())){ + address.setStreetNumber(addr.getStreetNumber()); + } + } if(StringUtils.isNotBlank(address.getStreet())){ formattedAddressBuilder.append(address.getStreet()); } if(StringUtils.isNotBlank(address.getStreet()) && StringUtils.isNotBlank(address.getStreetNumber())){ formattedAddressBuilder.append(address.getStreetNumber()); } - if(StringUtils.isBlank(address.getStreet()) && StringUtils.isNotBlank(building.getAddress())){ - formattedAddressBuilder.append(building.getAddress()); - } + if(StringUtils.isNotBlank(building.getSuggestName())){ formattedAddressBuilder.append(building.getSuggestName()); } @@ -985,18 +1175,18 @@ public class AddressParserServiceImpl implements AddressParserService { && address.getStreetNumber().endsWith(address.getOrgBuildingNumber())){ }else{ - formattedAddressBuilder.append(address.getBuildingNumber()); -// formattedAddressBuilder.append(address.getBuildingNumber() + "栋"); +// formattedAddressBuilder.append(address.getBuildingNumber()); + formattedAddressBuilder.append(address.getBuildingNumber() + "栋"); } } if(StringUtils.isNotBlank(address.getOrgUnitNumber())){ - formattedAddressBuilder.append(address.getUnitNumber()); -// formattedAddressBuilder.append(address.getUnitNumber() + "单元"); +// formattedAddressBuilder.append(address.getUnitNumber()); + formattedAddressBuilder.append(address.getUnitNumber() + "单元"); } - if(StringUtils.isNotBlank(address.getOrgRoomNumber())){ - formattedAddressBuilder.append(address.getRoomNumber()); -// formattedAddressBuilder.append(address.getRoomNumber() + "室"); + if(StringUtils.isNotBlank(address.getRoomNumber())){ +// formattedAddressBuilder.append(address.getRoomNumber()); + formattedAddressBuilder.append(address.getRoomNumber() + "室"); } address.setFormattedAddress(formattedAddressBuilder.toString()); diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/format/AddressFormatterFactory.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/format/AddressFormatterFactory.java index cc1334533..ea95695e8 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/format/AddressFormatterFactory.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/format/AddressFormatterFactory.java @@ -63,14 +63,18 @@ public class AddressFormatterFactory { }); - /*人民中路123-3501 -->人民中路123号3501室*/ - formatters.add(new AbstractAddressFormatter(Pattern.compile(".*[路|弄]\\d+-\\d{3,4}$")) { + /*人民中路123-128 -->人民中路123号-3501号*/ + formatters.add(new AbstractAddressFormatter(Pattern.compile(".*(路|街|大道|巷|道|((零|一|二|三|四|五|六|七|八|九|十|东|南|西|北|复)[线环条甲])|(胡同))(交汇|.{0,3}段|.*弄)?\\d+-\\d{3,4}(号)?$")) { - final Pattern TARGET_PATTERN = Pattern.compile("(?<=(路|号))\\d+-\\d{3,4}$"); + final Pattern TARGET_PATTERN = Pattern.compile("(路|街|大道|巷|道|((零|一|二|三|四|五|六|七|八|九|十|东|南|西|北|复)[线环条甲])|(胡同))(交汇|.{0,3}段|.*弄)?\\d+-\\d{3,4}(号)?$"); @Override protected String doFormat(String address) { String targetStr = DataUtil.match(address, TARGET_PATTERN); - return address.replace(targetStr, targetStr.replace("-", "号") + "室"); + if(targetStr.endsWith("号")){ + return address.replace(targetStr, targetStr.replace("-", "号-")); + }else{ + return address.replace(targetStr, targetStr.replace("-", "号-") + "号"); + } } }); @@ -259,7 +263,38 @@ public class AddressFormatterFactory { @Override protected String doFormat(String address) { String matchStr = DataUtil.match(address, TARGET); - String formatStr = matchStr + "号"; + String formatStr = matchStr.replaceAll("段-","段"); + if(!matchStr.endsWith("号")){ + formatStr += "号"; + } + return address.replace(matchStr,formatStr); + } + }); + + /** + * 剑南路东段-12号-18号 + */ + formatters.add(new AbstractAddressFormatter(Pattern.compile(".*(路|街|大道|巷|道|((零|一|二|三|四|五|六|七|八|九|十|东|南|西|北|复)[线环条甲])|(胡同))(交汇|.{0,3}段|.*弄)?((([零一二三四五六七八九十百千\\d])+弄)|([甲乙-]?\\d+[号](附\\d+(号)?(-\\d+(号)?)?)?([-][附付、]?\\d+(号(附近)?)?)?))")) { + final Pattern TARGET = Pattern.compile("(路|街|大道|巷|道|((零|一|二|三|四|五|六|七|八|九|十|东|南|西|北|复)[线环条甲])|(胡同))(交汇|.{0,3}段|.*弄)?((([零一二三四五六七八九十百千\\d])+弄)|([甲乙-]?\\d+[号](附\\d+(号)?(-\\d+(号)?)?)?([-][附付、]?\\d+(号(附近)?)?)?))"); + @Override + protected String doFormat(String address) { + String matchStr = DataUtil.match(address, TARGET); + String formatStr = matchStr; + if(!matchStr.endsWith("号")){ + formatStr += "号"; + } + return address.replace(matchStr,formatStr); + } + }); +/** + * 剑南路东段-12号-18号 + */ + formatters.add(new AbstractAddressFormatter(Pattern.compile("(((路|街|大道|巷|道|段|附|付)-)|(-(附|付)))")) { + final Pattern TARGET = Pattern.compile("(((路|街|大道|巷|道|段|附|付)-)|(-(附|付)))"); + @Override + protected String doFormat(String address) { + String matchStr = DataUtil.match(address, TARGET); + String formatStr = matchStr.replace("-",""); return address.replace(matchStr,formatStr); } }); diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/Address.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/Address.java index 2e286a5ed..1c8f2d004 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/Address.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/Address.java @@ -77,7 +77,27 @@ public class Address { private String debugName; private AliyunBo aliyunBo; + //使用CRF切分的地址 + private String crfAddress; + public String getCrfAddress() { + return crfAddress; + } + + public void setCrfAddress(String crfAddress) { + this.crfAddress = crfAddress; + } + + public AddressParserGoal getAddressParserGoal() { + return addressParserGoal; + } + + public void setAddressParserGoal(AddressParserGoal addressParserGoal) { + this.addressParserGoal = addressParserGoal; + } + + /**地址解析目的*/ + private AddressParserGoal addressParserGoal; public AliyunBo getAliyunBo() { return aliyunBo; } diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/AddressMatchMetaData.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/AddressMatchMetaData.java index 3fcee87e0..18e331f8b 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/AddressMatchMetaData.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/AddressMatchMetaData.java @@ -44,6 +44,24 @@ public class AddressMatchMetaData { private Double lat; private String map; + public String getMapProvince() { + return mapProvince; + } + + public void setMapProvince(String mapProvince) { + this.mapProvince = mapProvince; + } + + public String getMapCity() { + return mapCity; + } + + public void setMapCity(String mapCity) { + this.mapCity = mapCity; + } + + private String mapProvince; + private String mapCity; public void addName(String name, float priority){ if(StringUtils.isNotBlank(name)){ String formatName = AddressParserUtil.standardAddressAndName(name); diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/Building.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/Building.java index a217fa8fd..251470ffc 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/Building.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/Building.java @@ -64,9 +64,7 @@ public class Building { this.alias = d.getAlias(); this.addrAlias = d.getAddrAlias(); this.propertyType = d.getEstateType2(); - if(StringUtils.isNotBlank(d.getStreetCommunity())){ - this.township = d.getStreetCommunity(); - } + this.township = d.getTownship(); if(d.getLat() != null && d.getLng() != null){ this.lat = d.getLat()/10000000000d; this.lng = d.getLng()/10000000000d; @@ -90,9 +88,7 @@ public class Building { this.lat = d.getLat2()/10000000000d; this.lng = d.getLng2()/10000000000d; } - if(StringUtils.isNotBlank(d.getStreetCommunity())){ - this.township = d.getStreetCommunity(); - } + this.township = d.getTownship(); } public Building() { diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/MatchParams.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/MatchParams.java index 498b12052..e53c9701f 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/MatchParams.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/MatchParams.java @@ -25,7 +25,15 @@ public class MatchParams { private boolean useMap; /**是否使用阿里云接口*/ private boolean useAliyunApi; + //是否使用CRF算法 + private boolean useCrf; + public boolean isUseCrf() { + return useCrf; + } + public void setUseCrf(boolean useCrf) { + this.useCrf = useCrf; + } public boolean isContainsLocation() { return containsLocation; } diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/CityProcessor.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/CityProcessor.java index 2dd886f8b..6f88c20ef 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/CityProcessor.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/CityProcessor.java @@ -50,18 +50,23 @@ public class CityProcessor extends AbstractAddressProcessor { return RepeatStatus.NEXT; } } - if(StringUtils.isNotBlank(city)){ + if(StringUtils.isNotBlank(city) && CityStatus.SURE != address.getCityStatus() ){ String formatCity = RegionData.formatCity(city); - String possibleCity = null; + String possibleCity = AddressParserGoal.Match.equals(address.getAddressParserGoal()) ? parseCityFromAddress(orgAddress, address) : null; + String confirmCity = AddressParserGoal.Match.equals(address.getAddressParserGoal()) ? null : formatCity; /**如果省份和城市信息相互矛盾 则用区域推断城市*/ if(StringUtils.isNotBlank(address.getProvince()) && !ProvinceData.belongTo(formatCity, address.getProvince())){ List possibleCities = speculateCityNameByRegion(address, city); if(isUnique(possibleCities)){ formatCity = getFirst(possibleCities); + confirmCity = formatCity; } } - if(null != formatCity){ - city = formatCity; + if(AddressParserGoal.Match.equals(address.getAddressParserGoal()) && (formatCity == null || (possibleCity!=null && !formatCity.equals(possibleCity)))){ + confirmCity = possibleCity; + } + if(null != confirmCity){ + city = confirmCity; address.setCity(city); address.setCityStatus(CityStatus.SURE); } @@ -98,7 +103,7 @@ public class CityProcessor extends AbstractAddressProcessor { address.setCity(city); } if(StringUtils.isNotBlank(city) && scoreRegion(city, orgAddress) > 0.4){ - orgAddress = orgAddress.replaceFirst(".*" + address.getCity() + "[市县区州巿]?" , ""); + orgAddress = orgAddress.replaceFirst(".*" + address.getCity() + "[市县区州巿]?(?!(路|街|大道|巷|道|段))" , ""); address.setAddress(orgAddress); address.setCityStatus(CityStatus.SURE); return RepeatStatus.NEXT; @@ -166,7 +171,7 @@ public class CityProcessor extends AbstractAddressProcessor { String street = getStreet(address); for(String city : allCity){ if(street != null){ - MatchResult matchResult = matchFirst(Pattern.compile(city + "[东南西北中一二三四五六七八九十百千\\d]?(路|街|大道|支路|村)"), street); + MatchResult matchResult = matchFirst(Pattern.compile(city + "[东南西北中一二三四五六七八九十百千\\d]?(路|街|大道|大街|支路|村)"), street); if(matchResult !=null){ address = address.replace(matchResult.matchContent, ""); } @@ -199,6 +204,10 @@ public class CityProcessor extends AbstractAddressProcessor { } Map> regionMap = RegionData.getAllRegions(city); for(String region : regionMap.keySet()){ + MatchResult matchResult = matchFirst(Pattern.compile(region + "[东南西北中一二三四五六七八九十百千\\d]?(路|街|大道|大街|支路|村)"), street); + if(matchResult !=null){ + address = address.replace(matchResult.matchContent, ""); + } if(address.indexOf(region, index) > -1){ if(province != null && (orgAddress.contains(province + region) || orgAddress.contains(ProvinceData.getDisplay(province) + region))){ weight.score += 2; diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/ProvinceProcessor.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/ProvinceProcessor.java index 36c916555..1b0aa3a56 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/ProvinceProcessor.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/ProvinceProcessor.java @@ -2,6 +2,7 @@ package com.fangjia.service.address.processor; import com.fangjia.core.RegionData; import com.fangjia.service.address.mode.Address; +import com.fangjia.service.address.mode.AddressParserGoal; import com.fangjia.service.address.mode.CityStatus; import com.fangjia.service.address.mode.ProvinceData; import org.apache.commons.lang.StringUtils; @@ -24,7 +25,7 @@ public class ProvinceProcessor extends AbstractAddressProcessor { String orgAddress = address.getAddress(); String[] provinceInfo = getProvince(orgAddress); - if(provinceInfo == null && address.getCity() != null){ + if(provinceInfo == null && address.getCity() != null && !AddressParserGoal.Match.equals(address.getAddressParserGoal())){ provinceInfo = getProvince(address.getCity()); } if (null != provinceInfo) { diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/RoomNumberProcessor.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/RoomNumberProcessor.java index 77d00e97d..92ad74b29 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/RoomNumberProcessor.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/RoomNumberProcessor.java @@ -18,6 +18,7 @@ public class RoomNumberProcessor extends AbstractAddressProcessor { static final Pattern NUMBER_END = Pattern.compile("(?<=\\D)\\d{3,4}$"); static final Pattern HOUSE_NUMBER = Pattern.compile("\\d{3,4}"); static final String HOUSE_UNITS_PATTERN = "(室|房|号房|号(?!院)|户|铺)+"; + static final Pattern HOUSE_PATTERN_CASE02 = Pattern.compile("(\\d+(室|房|号)?[、]?)+$"); @Override public RepeatStatus process(Address address) { @@ -27,10 +28,17 @@ public class RoomNumberProcessor extends AbstractAddressProcessor { orgAddress = removeSubstring(orgAddress, matchResult.start, matchResult.end); address.setOrgRoomNumber(matchResult.matchContent); address.setAddress(orgAddress); - analyFloorUseRoomNumber(address); +// analyFloorUseRoomNumber(address); + return RepeatStatus.NEXT; + } + MatchResult matchResult2 = matchFirst(HOUSE_PATTERN_CASE02, orgAddress); + if (matchResult2 != null) { + orgAddress = removeSubstring(orgAddress, matchResult2.start, matchResult2.end); + address.setOrgRoomNumber(matchResult2.matchContent); + address.setAddress(orgAddress); +// analyFloorUseRoomNumber(address); return RepeatStatus.NEXT; } - matchResult = matchFirst(HOUSE_PATTERN, orgAddress); if (matchResult != null) { orgAddress = removeSubstring(orgAddress, matchResult.start, matchResult.end); @@ -39,10 +47,10 @@ public class RoomNumberProcessor extends AbstractAddressProcessor { } else { address.setOrgRoomNumber(matchResult.matchContent + "室"); } - analyFloorUseRoomNumber(address); +// analyFloorUseRoomNumber(address); address.setAddress(orgAddress); } - analyFloorUseRoomNumber(address); +// analyFloorUseRoomNumber(address); removeHouseUnit(address); return RepeatStatus.NEXT; } diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/StageAndPartitionProcessor.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/StageAndPartitionProcessor.java index 93c69b4c4..ab5c929b3 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/StageAndPartitionProcessor.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/StageAndPartitionProcessor.java @@ -14,6 +14,7 @@ public class StageAndPartitionProcessor extends AbstractAddressProcessor { @Override public RepeatStatus process(Address address) { String orgAddr = address.getEffectiveAddress(); + address.setCrfAddress(address.getAddress()); MatchResult matchResult = matchFirst(AddressParserPatterns.STAGE_PATTERN, address.getEffectiveAddress()); if (matchResult != null) { String formatStage = CNNMFilter.convertArabicToCnNumeric(matchResult.matchContent); diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/TooltipProcessor.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/TooltipProcessor.java index 976f2925f..4d3b65b14 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/TooltipProcessor.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/TooltipProcessor.java @@ -41,6 +41,7 @@ public class TooltipProcessor extends AbstractAddressProcessor { offset += matchResult.matchContent.length(); if(matchFirst(AddressParserPatterns.STREET, tooltip) != null){ address.setTooltip(tooltip); + address.setStreet(getStreet(tooltip)); } else if(containsDistrictName(tooltip)){ address.setTooltip(tooltip); } else if(RegionData.isRegion(address.getCity(), tooltip)){ diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/VillageNameProcessor.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/VillageNameProcessor.java index 0d9295ce6..5236b7d55 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/VillageNameProcessor.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/VillageNameProcessor.java @@ -1,6 +1,7 @@ package com.fangjia.service.address.processor; import com.fangjia.service.address.mode.Address; +import com.fangjia.service.cnAnalyzer.Token; import org.apache.commons.lang.StringUtils; import java.util.ArrayList; @@ -14,6 +15,15 @@ public class VillageNameProcessor extends AbstractAddressProcessor { @Override public AbstractAddressProcessor.RepeatStatus process(Address address) { String orgAddress = address.getAddress(); + if(address.getTokens()!=null){ + for(Token token:address.getTokens()){ + if("district".equals(token.getKeywordTypes())){ + address.setKeywords(token.getName()); + address.setAddress(orgAddress.replace(token.getName(),"")); + return RepeatStatus.NEXT; + } + } + } List names = tokenDistrictNames(orgAddress); List kws = new ArrayList(); for(String name : names){ diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/api/AliyunServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/api/AliyunServiceImpl.java index b6b22f5a1..91d309485 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/api/AliyunServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/api/AliyunServiceImpl.java @@ -1,5 +1,7 @@ package com.fangjia.service.api; +import com.aliyuncs.address_purification.model.v20191118.GetAddressDivisionCodeRequest; +import com.aliyuncs.address_purification.model.v20191118.GetAddressDivisionCodeResponse; import com.aliyuncs.address_purification.model.v20191118.StructureAddressRequest; import com.aliyuncs.address_purification.model.v20191118.StructureAddressResponse; import com.fangjia.core.util.JsonUtils; @@ -13,6 +15,9 @@ import net.sf.json.JSONObject; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; +import java.util.ArrayList; +import java.util.List; + public class AliyunServiceImpl implements AliyunService { protected final Logger LOGGER = LoggerFactory.getLogger(this.getClass()); @@ -35,20 +40,44 @@ public class AliyunServiceImpl implements AliyunService { if(StringUtils.isNotBlank(city)){ request.setDefaultCity(city); } + AliyunBo aliyunBo = new AliyunBo(); try{ StructureAddressResponse response = client.getAcsResponse(request); JSONObject responseJson = JSONObject.fromObject(response.getData()); String value = (String) responseJson.get("structure"); - AliyunBo aliyunBo = new AliyunBo(); for (String key:value.split("\t")){ String[] arry = key.split("="); wrapField(arry[0],arry[1], aliyunBo); } - return aliyunBo; }catch(Exception e){ - LOGGER.error("", e); - return null; + LOGGER.error("调用阿里标准化接口失败:", e); } + try{ + GetAddressDivisionCodeRequest divisionRequest = new GetAddressDivisionCodeRequest(); + divisionRequest.setServiceCode("addrp"); + divisionRequest.setAppKey(APPKEY); + divisionRequest.setText(address); + GetAddressDivisionCodeResponse divisionResponse = client.getAcsResponse(divisionRequest); + JSONObject responseJson = JSONObject.fromObject(divisionResponse.getData()); + String value = (String) ((JSONObject) responseJson.get("division_info")).get("division_name"); + String[] array = value.replace("\t", ";").split(";"); + if(array.length > 4){ + List tmp = new ArrayList(); + for(String str:array){ + if(str!=null && str.length()!=0){ + tmp.add(str); + } + } + array= tmp.toArray(new String[4]); + } + aliyunBo.setProv(array[0]); + aliyunBo.setCity(array[1]); + aliyunBo.setDistrict(array[2]); + aliyunBo.setTown(array[3]); + }catch(Exception e){ + LOGGER.error("调用阿里行政区划识别接口失败", e); + } + return aliyunBo; } private void wrapField(String field,String value, AliyunBo aliyunBo){ @@ -81,26 +110,59 @@ public class AliyunServiceImpl implements AliyunService { } } public static void main(String[] args) throws ClientException { - DefaultProfile profile = DefaultProfile.getProfile( + String value = ""; + String[] array = value.replace("\t", ";").split(";"); + if(array.length > 4){ + List tmp = new ArrayList(); + for(String str:array){ + if(str!=null && str.length()!=0){ + tmp.add(str); + } + } + array= tmp.toArray(new String[4]); + } + System.out.println(array[0]); + System.out.println(array[1]); + System.out.println(array[2]); + System.out.println(array[3]); + + /*DefaultProfile profile = DefaultProfile.getProfile( REGIONID, ACCESSKEYID, SECRET); IAcsClient client = new DefaultAcsClient(profile); - StructureAddressRequest request = new StructureAddressRequest(); +// StructureAddressRequest request = new StructureAddressRequest(); + GetAddressDivisionCodeRequest request = new GetAddressDivisionCodeRequest(); request.setServiceCode("addrp"); request.setAppKey(APPKEY); - request.setText("杭州市文一西路969号"); - request.setDefaultProvince("浙江省"); - request.setDefaultCity("杭州市"); - request.setDefaultDistrict("余杭区"); - StructureAddressResponse response = client.getAcsResponse(request); + request.setText("天青路260号"); +// request.setDefaultProvince("浙江省"); +// request.setDefaultCity("杭州市"); +// request.setDefaultDistrict("余杭区"); +// StructureAddressResponse response = client.getAcsResponse(request); + GetAddressDivisionCodeResponse response = client.getAcsResponse(request); JSONObject responseJson = JSONObject.fromObject(response.getData()); - String value = (String) responseJson.get("structure"); - JSONObject json = new JSONObject(); + System.out.println(responseJson); + String value = (String) ((JSONObject) responseJson.get("division_info")).get("division_name"); + String[] array = value.replace("\t", ";").split(";"); + if(array.length > 4){ + List tmp = new ArrayList(); + for(String str:array){ + if(str!=null && str.length()!=0){ + tmp.add(str); + } + } + array= tmp.toArray(new String[4]); + } + System.out.println(array[0]); + System.out.println(array[1]); + System.out.println(array[2]); + System.out.println(array[3]);*/ + /*JSONObject json = new JSONObject(); for (String key:value.split("\t")){ String[] arry = key.split("="); json.put(arry[0],arry[1]); } - System.out.println(JsonUtils.convertJsonToJavaObj(json, AliyunBo.class)); + System.out.println(JsonUtils.convertJsonToJavaObj(json, AliyunBo.class));*/ } } diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/api/AmapPoiService.java b/projects/fangjia-service/src/main/java/com/fangjia/service/api/AmapPoiService.java index 227aec149..9a5c6d1d8 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/api/AmapPoiService.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/api/AmapPoiService.java @@ -68,7 +68,7 @@ public interface AmapPoiService { * @param types * @return */ - List search(String city, String keywords, Integer limit, String types); + List search(String city, String keywords, Integer limit, String types, boolean validCity); /** diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/api/AmapPoiServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/api/AmapPoiServiceImpl.java index 6060696b5..91a08b863 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/api/AmapPoiServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/api/AmapPoiServiceImpl.java @@ -1,5 +1,6 @@ package com.fangjia.service.api; +import com.fangjia.core.ProvinceData; import com.fangjia.core.RegionData; import com.fangjia.core.util.DataUtil; import com.fangjia.core.util.GeoUtils; @@ -258,14 +259,14 @@ public class AmapPoiServiceImpl implements AmapPoiService { } @Override - public List search(String city, String keywords, Integer limit, String types) { + public List search(String city, String keywords, Integer limit, String types, boolean validCity) { Map params = new HashMap(); String area = city; params.put("city", area); params.put("output", "json"); params.put("keywords", keywords); params.put("key", "4ccfe3c898a9489884bd2bcb1ef6dcde"); - params.put("citylimit", "true"); +// params.put("citylimit", "true"); params.put("offset", limit == null?"25":String.valueOf(limit)); if(types != null){ params.put("types", "120301&120302&120201&060101"); @@ -351,15 +352,17 @@ public class AmapPoiServiceImpl implements AmapPoiService { cityname = poi.getString("cityname"); } cityname = RegionData.formatCity(cityname); - if(!city.equals(cityname)){ + if((city!=null && !city.equals(cityname) && validCity) || cityname == null){ continue; } + pname = ProvinceData.formatProvince(pname); AmapPoi amapPoi = new AmapPoi(); - amapPoi.setCity(city); + amapPoi.setCity(cityname); + amapPoi.setPname(pname); if(poi.containsKey("adname") && poi.get("adname") instanceof String){ adname = poi.getString("adname"); - String region = RegionData.formatRegion(city, adname); - if(RegionData.isRegion(city, region)){ + String region = RegionData.formatRegion(cityname, adname); + if(RegionData.isRegion(cityname, region)){ amapPoi.setRegion(region); } } @@ -380,8 +383,8 @@ public class AmapPoiServiceImpl implements AmapPoiService { amapPoi.setAlias(alias); } if(poi.containsKey("adname") && poi.get("adname") instanceof String){ - String region = RegionData.formatRegion(city, poi.getString("adname")); - if(RegionData.isRegion(city, region)){ + String region = RegionData.formatRegion(cityname, poi.getString("adname")); + if(RegionData.isRegion(cityname, region)){ amapPoi.setRegion(region); } } @@ -391,7 +394,7 @@ public class AmapPoiServiceImpl implements AmapPoiService { .replace(cityname, "").replace(adname, ""); amapPoi.setAddress(AddressParserUtil.optimizeAddress(address)); } - amapPoi.setCity(city); + amapPoi.setCity(cityname); if(poi.containsKey("location") && poi.get("location") instanceof String){ String[] location = poi.getString("location").split(","); amapPoi.setLng(Double.valueOf(location[0])); diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/DistrictServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/DistrictServiceImpl.java index 19e45d136..bea9e7589 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/DistrictServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/DistrictServiceImpl.java @@ -2495,10 +2495,12 @@ public class DistrictServiceImpl implements DistrictService { params.add(new QueryParam("visible", 0)); params.add(new QueryParam("city", city)); params.add(new QueryParam("category", "district")); - params.add(new QueryParam("keyword", keyword)); - Query query = boostQueryDistrictPolicy.create(ServiceUtil.removeAndGetKeyword(params)); - if (query != null) { - params.add(new QueryParam(QueryParam.SR_PARAM_NAME, new SrQuery(query))); + if (StringUtils.isNotBlank(keyword)) { + BooleanQuery query = new BooleanQuery(); + Query q1 = new TermQuery(new Term("keyword", keyword)); + q1.setBoost(0.99f); + query.add(q1, Occur.SHOULD); + params.add(new QueryParam("$sr", new SrQuery(query))); } return districtDao.findAllByParams(params, limit); } diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/addrepo/AddressRepoService.java b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/addrepo/AddressRepoService.java index 9aee32e52..6289384e4 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/addrepo/AddressRepoService.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/addrepo/AddressRepoService.java @@ -37,7 +37,7 @@ public interface AddressRepoService { * @param max * @return */ - List findAllByKeyword(String city,BuildingType buildingType , List keywords, Double lng, Double lat, int max); + List findAllByKeyword(String city,BuildingType buildingType , List keywords, Double lng, Double lat, int max, boolean validCity); /** * diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/addrepo/AddressRepoServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/addrepo/AddressRepoServiceImpl.java index a751826c7..3965c9f00 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/addrepo/AddressRepoServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/addrepo/AddressRepoServiceImpl.java @@ -87,7 +87,7 @@ public class AddressRepoServiceImpl implements AddressRepoService { return addrs; } - public List findAllByKeyword(String city,BuildingType buildingType , List keywords, Double lng, Double lat, int max){ + public List findAllByKeyword(String city,BuildingType buildingType , List keywords, Double lng, Double lat, int max, boolean validCity){ String types = null; String tag = null; String category = null; @@ -104,7 +104,11 @@ public class AddressRepoServiceImpl implements AddressRepoService { tag = "百货商场"; category = "综合商场"; } - List pois = amapPoiService.search(city, StringUtils.join(keywords, "|"), 20, types); + List pois = new ArrayList(); + for(String keyword:keywords){ + List poiList = amapPoiService.search(city, keyword, 20, types, validCity); + pois.addAll(poiList); + } List matchMetaDatas = new ArrayList(pois.size()); for(AmapPoi poi: pois){ AddressMatchMetaData data = new AddressMatchMetaData(); @@ -121,6 +125,8 @@ public class AddressRepoServiceImpl implements AddressRepoService { data.setType(poi.getType()); data.setMap("高德"); data.setCity(city); + data.setMapCity(poi.getCity()); + data.setMapProvince(poi.getPname()); if(poi.getAlias() != null){ for(String alia : poi.getAlias()){ data.addName(alia, MatchText.ALIAS_PRIORITY); diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/web/AddressService.java b/projects/fangjia-service/src/main/java/com/fangjia/service/web/AddressService.java index 3bfa96333..00a98a047 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/web/AddressService.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/web/AddressService.java @@ -284,5 +284,12 @@ public interface AddressService { * @return */ AddressCheckTruth aroundConsumeGrade(String city, String address); - + + /** + * 量地址相似度评级 + * @param result1 + * @param result2 + * @return 返回是否同一地址、输出级别(哪一级),相似度得分 + */ + Map addressSimilarity(String result1, String result2); } diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/web/AddressServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/web/AddressServiceImpl.java index ff7454cc5..21ab2139d 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/web/AddressServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/web/AddressServiceImpl.java @@ -5,18 +5,7 @@ import java.math.BigDecimal; import java.math.RoundingMode; import java.net.URLEncoder; import java.text.DecimalFormat; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.Comparator; -import java.util.Date; -import java.util.HashMap; -import java.util.Iterator; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; @@ -25,6 +14,9 @@ import java.util.concurrent.Future; import java.util.regex.Matcher; import java.util.regex.Pattern; +import com.fangjia.core.util.*; +import com.fangjia.service.address.AddressParserUtil; +import com.fangjia.service.address.mode.MatchResult; import net.sf.json.JSONArray; import net.sf.json.JSONObject; @@ -49,18 +41,8 @@ import org.sothis.core.util.Closure; import com.fangjia.core.BlockData; import com.fangjia.core.RegionData; -import com.fangjia.core.util.AddressParser; -import com.fangjia.core.util.AddressParserUtils; -import com.fangjia.core.util.AddressResult; -import com.fangjia.core.util.BeanUtils; -import com.fangjia.core.util.CacheUtils; import com.fangjia.core.util.CacheUtils.CachePolicyEnum; -import com.fangjia.core.util.DataUtil; -import com.fangjia.core.util.DateUtil; -import com.fangjia.core.util.GeoUtils; import com.fangjia.core.util.GeoUtils.GaussSphere; -import com.fangjia.core.util.HttpRequest; -import com.fangjia.core.util.LoggerFactory; import com.fangjia.dal.QueryParam; import com.fangjia.dal.SdQuery; import com.fangjia.dal.SrQuery; @@ -5074,7 +5056,186 @@ public class AddressServiceImpl implements AddressService { } return checkTr; } - + + @Override + public Map addressSimilarity(String result1, String result2) { + JSONObject json1 = (JSONObject) JSONObject.fromObject(result1).get("result"); + JSONObject json2 = (JSONObject) JSONObject.fromObject(result2).get("result"); + Map resultMap = new HashMap(); + //判断是否同一地址 + boolean sameAddress = validSameAddress(json1, json2); + String formatAddress1_pinyin = AddressParserUtil.toHanYuPinYin(getFromJson(json1, "formattedAddress")); + String formatAddress2_pinyin = AddressParserUtil.toHanYuPinYin(getFromJson(json2,"formattedAddress")); + double similar = AddressParserUtil.getSimilarityRatio(formatAddress1_pinyin, formatAddress2_pinyin); + String level = getAddressSimilarityLevel(json1, json2); + resultMap.put("isSameAddress", sameAddress); +// resultMap.put("score", similar); + resultMap.put("level", level); + resultMap.put("formattedAddress_1", getFromJson(json1,"formattedAddress")); + resultMap.put("formattedAddress_2", getFromJson(json2,"formattedAddress")); + return resultMap; + } + + private boolean validSameAddress(JSONObject json1, JSONObject json2){ + //标准化后地址完全一样 + if(getFromJson(json1, "formattedAddress").equals(getFromJson(json2, "formattedAddress"))){ + return true; + } + //判断判断省、市、区、乡镇、楼栋、单元、室号是否相同 + if(!verifySameField(getFromJson(json1,"province"), getFromJson(json2,"province"), false) + || !verifySameField(getFromJson(json1,"city"), getFromJson(json2,"city"), false) + || !verifySameField(getFromJson(json1,"district"), getFromJson(json2,"district"), false) + || !verifySameField(getFromJson(json1,"township"), getFromJson(json2,"township"), false) + || !verifySameField(getFromJson(json1,"buildingNumber"), getFromJson(json2,"buildingNumber"), true) + || !verifySameField(getFromJson(json1,"unitNumber"), getFromJson(json2,"unitNumber"), true) + || !verifySameField(getFromJson(json1,"roomNumber"), getFromJson(json2,"roomNumber"), true)){ + return false; + } + + String street1 = (StringUtils.isNotBlank(getFromJson(json1,"street")) ? getFromJson(json1,"street") : "") + (StringUtils.isNotBlank(getFromJson(json1,"streetNumber")) ? getFromJson(json1,"streetNumber") : ""); + String street2 = (StringUtils.isNotBlank(getFromJson(json2,"street")) ? getFromJson(json2,"street") : "") + (StringUtils.isNotBlank(getFromJson(json2,"streetNumber")) ? getFromJson(json2,"streetNumber") : ""); + String name1 = StringUtils.isNotBlank(getFromJson(json1,"name")) ? getFromJson(json1,"name") : ""; + String name2 = StringUtils.isNotBlank(getFromJson(json2,"name")) ? getFromJson(json2,"name") : ""; + //两个地址中路名+路号和poi全部为空时返回true + if(StringUtils.isBlank(street1) && StringUtils.isBlank(street2) && StringUtils.isBlank(name1) && StringUtils.isBlank(name2)){ + return true; + } + //同一条地址里路名和poi不能全为空而另一条至少有一项有值 + if((StringUtils.isBlank(street1) && StringUtils.isBlank(name1)) || (StringUtils.isBlank(street2) && StringUtils.isBlank(name2))){ + return false; + } + //poi有任意一个为空则路名、路号必须完全一致 + if(StringUtils.isBlank(name1) || StringUtils.isBlank(name2)){ + if(StringUtils.isNotBlank(getFromJson(json1,"street")) && getFromJson(json1,"street").equals(getFromJson(json2,"street")) + && StringUtils.isNotBlank(getFromJson(json1,"streetNumber")) && getFromJson(json1,"streetNumber").equals(getFromJson(json2,"streetNumber"))){ + return true; + }else{ + return false; + } + } + //poi全不为空时,路名+路号一致(可为空),poi处理后一致 + if(verifySameField(street1, street2, false) && verifySamePoi(name1, name2)){ + return true; + } + return false; + } + + private String getFromJson(JSONObject json, String key){ + if(json.containsKey(key)){ + return json.getString(key); + } + return null; + } + /** + * 获取两地址的相似度评级(相同数据的最小颗粒度) + * @return + */ + private String getAddressSimilarityLevel(JSONObject json1, JSONObject json2){ + String level = null; + LinkedList list = new LinkedList(Arrays.asList("province", "city", "district", "township", "street", "streetNumber" + , "name", "buildingNumber", "unitNumber", "roomNumber")); + for(String key:list){ + if(continueJudgeValue(json1, json2, key) == 3){ + return level; + }else{ + level = continueJudgeValue(json1, json2, key) == 1 ? key : (level == null ? null : level); + } + } + return level; + } + + /** + * 判断level时是否继续往下判断 + * @param json1 + * @param json2 + * @param key + * @return 1 继续判断并且level有值 2继续判断level没值 3 停止 + */ + private Integer continueJudgeValue(JSONObject json1, JSONObject json2, String key){ + if("buildingNumber".equals(key) || "unitNumber".equals(key) || "roomNumber".equals(key)){ + if(StringUtils.isNotBlank(getFromJson(json1,key)) && StringUtils.isNotBlank(getFromJson(json2,key)) + && getFromJson(json1,key).equals(getFromJson(json2,key))){ + return 1; + }else{ + return 3; + } + } + if(StringUtils.isNotBlank(getFromJson(json1,key)) && StringUtils.isNotBlank(getFromJson(json2,key))){ + if("name".equals(key)){ + return verifySamePoi(getFromJson(json1,key),getFromJson(json2,key)) ? 1 : 3; + }else{ + if(getFromJson(json1,key).equals(getFromJson(json2,key))){ + return 1; + }else{ + return 3; + } + } + } + return 2; + } + /** + * 判断省、市、区、乡镇、楼栋、单元、室号是否相同(两者全为空时返回true) + * @param str1 + * @param str2 + * @param flag true时严格判断 两者完全一致返回true(全为空或者全部有值并且值一致) + * false时 两者完全一致或一方为空返回true(全为空或者全部有值并且值一致或任一方值为空) + * @return + */ + private boolean verifySameField(String str1, String str2,boolean flag){ + if(flag){ + if(StringUtils.isBlank(str1) && StringUtils.isBlank(str2)){ + return true; + } + return StringUtils.isNotBlank(str1) ? str1.equals(str2) : false; + }else{ + if(StringUtils.isBlank(str1) || StringUtils.isBlank(str2)){ + return true; + } + return str1.equals(str2); + } + } + + /** + * //判断是否同一个poi(全部有数据时) + * @param name1 + * @param name2 + * @return + */ + private boolean verifySamePoi(String name1, String name2){ + if(StringUtils.isBlank(name1) && StringUtils.isBlank(name2)){ + return false; + } + if(name1.equals(name2)){ + return true; + } + //分期 + String name1_stage = AddressParserUtil.match(AddressParserPatterns.STAGE_PATTERN,name1); + String name2_stage = AddressParserUtil.match(AddressParserPatterns.STAGE_PATTERN,name2); + //分区 + String name1_partition = AddressParserUtil.match(AddressParserPatterns.PARTITION_PATTERN,name1); + String name2_partition = AddressParserUtil.match(AddressParserPatterns.PARTITION_PATTERN,name2); + //数字 + String name1_number = AddressParserUtil.match(Pattern.compile("[\\d一二三四五六七八九十千百万]+"),name1); + String name2_number = AddressParserUtil.match(Pattern.compile("[\\d一二三四五六七八九十千百万]+"),name2); + if(!verifySameField(name1_stage, name2_stage, true) || !verifySameField(name1_partition, name2_partition, true)|| !verifySameField(name1_number, name2_number, true)){ + return false; + } + return isSameName(name1, name2); + } + + private boolean isSameName(String name1, String name2){ + String name_1_pinyin = AddressParserUtil.toHanYuPinYin(name1); + String name_2_pinyin = AddressParserUtil.toHanYuPinYin(name2); + //拼音或文本数据包含并且 + boolean pinYinContains = (name_1_pinyin.contains(name_2_pinyin) || name_2_pinyin.contains(name_1_pinyin)); + if(pinYinContains){ + double similar = AddressParserUtil.getSimilarityRatio(name_1_pinyin, name_2_pinyin); + if(similar > 0.5){ + return true; + } + } + return false; + } /** * 设置区域消费指数 * @param checkTr diff --git a/projects/fangjia-static/pom.xml b/projects/fangjia-static/pom.xml index 6a048340d..7be7f7dae 100644 --- a/projects/fangjia-static/pom.xml +++ b/projects/fangjia-static/pom.xml @@ -29,7 +29,7 @@ org.sothis sothis-core - 0.2.0-SNAPSHOT + 0.2.0 com.fangjia diff --git a/projects/geluke/Dockerfile b/projects/geluke/Dockerfile new file mode 100644 index 000000000..d0b8e61a1 --- /dev/null +++ b/projects/geluke/Dockerfile @@ -0,0 +1,2 @@ +FROM 192.168.0.19:9012/tomcat6:latest +COPY target/*.war /home/goojia/webapps/ROOT.war \ No newline at end of file diff --git a/projects/geluke/pom.xml b/projects/geluke/pom.xml index c168bf1c6..62875968c 100644 --- a/projects/geluke/pom.xml +++ b/projects/geluke/pom.xml @@ -51,7 +51,7 @@ org.sothis sothis-web - 0.2.0-SNAPSHOT + 0.2.0 org.springframework diff --git a/projects/geluke/src/main/java/com/fangjia/geluke/controllers/DistrictController.java b/projects/geluke/src/main/java/com/fangjia/geluke/controllers/DistrictController.java index 7f0f840b6..cbc446e97 100644 --- a/projects/geluke/src/main/java/com/fangjia/geluke/controllers/DistrictController.java +++ b/projects/geluke/src/main/java/com/fangjia/geluke/controllers/DistrictController.java @@ -22,14 +22,8 @@ public class DistrictController { private DistrictService districtService; public Object loadDistrictAction(DistrictModel model, HttpServletRequest request){ - String url = openSearchService.createQueryUrl(model.getCityName(), model.getKeywords(), false); - String content = openSearchService.findResultByUrl(url); - List regions = districtService.getRegionByCity(model.getCityName()); - Map> map = openSearchService.parseResult(content, regions, model.getKeywords()); - if(map.size() == 0){ - List districts = districtService.findNewDistrict(model.getCityName(), model.getKeywords()); - map = openSearchService.createResult(districts); - } + List districts = districtService.findDistrictByKeyword(model.getCityName(), model.getKeywords(), 10); + Map> map = openSearchService.createResult(districts); JsonModel ret = new JsonModel(); ret.setJsonModel(map); return ret; diff --git a/projects/geluke/src/main/java/com/fangjia/geluke/model/BaseModel.java b/projects/geluke/src/main/java/com/fangjia/geluke/model/BaseModel.java index 5a4d1e6d6..3f52fa63b 100644 --- a/projects/geluke/src/main/java/com/fangjia/geluke/model/BaseModel.java +++ b/projects/geluke/src/main/java/com/fangjia/geluke/model/BaseModel.java @@ -155,11 +155,11 @@ public class BaseModel extends ModelAndViewSupport { } public final String getRoot() { - return (CONFIG.isDevelopmentEnvironment() ? "/geluke" : ""); + return "http://geluke.fangjia.com"; } public final String getWwwUrl() { - return WWW_URL; + return "http://geluke.fangjia.com"; } public final Date getNow() { @@ -270,7 +270,7 @@ public class BaseModel extends ModelAndViewSupport { } public final String getCityUrl() { - return CONFIG.getCityDomainName() + getRoot(); + return "http://geluke.fangjia.com"; } public final String getDomain() { diff --git a/projects/geluke/src/main/resources/geluke.properties b/projects/geluke/src/main/resources/geluke.properties index 5b0aeb064..f19395452 100644 --- a/projects/geluke/src/main/resources/geluke.properties +++ b/projects/geluke/src/main/resources/geluke.properties @@ -17,16 +17,32 @@ fangjia.auto.url=${fangjia.auto.host}:${fangjia.auto.port}/ fangjia.file.mongo.host=fs.mongo.db.fangjia.com fangjia.file.mongo.port=27017 +#fangjia.log.mongo.host=log.mongo.db.fangjia.com +#fangjia.log.mongo.port=27017 + fangjia.log.mongo.host=log.mongo.db.fangjia.com -fangjia.log.mongo.port=27017 +fangjia.log.mongo.port=52002 +fangjia.log.mongodb.mongoUri=mongodb://${fangjia.log.mongo.host}:${fangjia.log.mongo.port}/ fangjia.mongo.host=mongo.db.fangjia.com -fangjia.mongo.port=27017 +fangjia.mongo.port=28017 +fangjia.secondary.mongo.host=secondary.mongo.db.fangjia.com +fangjia.secondary.mongo.port=28017 +fangjia.mongodb.mongoUri=mongodb://${fangjia.mongo.host}:${fangjia.mongo.port},${fangjia.secondary.mongo.host}:${fangjia.secondary.mongo.port}/ + + +fangjia.fs.mongo.host=fs.mongo.db.fangjia.com +fangjia.fs.mongo.port=27017 +fangjia.fs.mongodb.mongoUri=mongodb://${fangjia.fs.mongo.host}:${fangjia.fs.mongo.port}/ + +fangjia.seabird.mongodb.mongoUri=mongodb://developer:G00IDCDEV12385@${fangjia.mongo.host}:${fangjia.mongo.port},${fangjia.secondary.mongo.host}:${fangjia.secondary.mongo.port}/ + fangjia.knowbase.host=knowbase.db.fangjia.com fangjia.knowbase.port=7180 fangjia.knowbase.url=${fangjia.knowbase.host}:${fangjia.knowbase.port}/ + fangjia.environment=development #fangjia.environment=production diff --git a/projects/geluke/src/main/resources/spring-context.xml b/projects/geluke/src/main/resources/spring-context.xml index 6c59dd2d3..20d2cf31f 100644 --- a/projects/geluke/src/main/resources/spring-context.xml +++ b/projects/geluke/src/main/resources/spring-context.xml @@ -18,8 +18,15 @@ - - + + + + + + + + + @@ -29,20 +36,27 @@ - - - - + + + + + + - - - + + + - - - - + + + + + + + + + - - AccessLogFilter - - com.fangjia.geluke.filters.AccessLogFilter - - + + + + + + SpiderValidateFilter @@ -47,20 +47,20 @@ - - AnalyticsFilter - - org.springframework.web.filter.DelegatingFilterProxy - - - targetBeanName - analyticsFilter - - - targetFilterLifecycle - true - - + + + + + + + + + + + + + + sothis @@ -80,21 +80,21 @@ /* REQUEST - - AccessLogFilter - /* - REQUEST - + + + + + SpiderValidateFilter /* REQUEST - - AnalyticsFilter - /* - REQUEST - + + + + + sothis /* diff --git a/projects/geluke/src/main/webapp/header/change_city.ftl b/projects/geluke/src/main/webapp/header/change_city.ftl index 565630c82..c0b59e53c 100644 --- a/projects/geluke/src/main/webapp/header/change_city.ftl +++ b/projects/geluke/src/main/webapp/header/change_city.ftl @@ -23,58 +23,55 @@ id="changecity">
  • 鍗庝笢鍩庡競
  • 鍗庡寳涓滃寳
  • 鍗庡崡鍗庝腑
  • 瑗块儴鍩庡競
  • - -
  • <#--鍦板浘鍩庡競鍒囨崲 end--> \ No newline at end of file diff --git a/projects/geluke/src/main/webapp/header/header_map.ftl b/projects/geluke/src/main/webapp/header/header_map.ftl index e6dc0d09f..19925acd2 100644 --- a/projects/geluke/src/main/webapp/header/header_map.ftl +++ b/projects/geluke/src/main/webapp/header/header_map.ftl @@ -28,7 +28,7 @@ $(document).ready(function(){ $("#_h_logout").click(function(){ jQuery.ajax({ type: "get", - url: "http://" + location.host + "${root}/user/logout", + url: "http://geluke.fangjia.com/user/logout", dataType:'json', success: function(msg){ location.href = '${cityUrl}'; diff --git a/projects/geluke/src/main/webapp/map/cityNavi.ftl b/projects/geluke/src/main/webapp/map/cityNavi.ftl index f4d08c9b5..7e8a3197c 100644 --- a/projects/geluke/src/main/webapp/map/cityNavi.ftl +++ b/projects/geluke/src/main/webapp/map/cityNavi.ftl @@ -200,27 +200,9 @@ diff --git a/projects/geluke/src/main/webapp/map/index.ftl b/projects/geluke/src/main/webapp/map/index.ftl index 43e39e199..5dc9aa1eb 100644 --- a/projects/geluke/src/main/webapp/map/index.ftl +++ b/projects/geluke/src/main/webapp/map/index.ftl @@ -1,766 +1,767 @@ -<#ftl attributes={"content_type":"text/html; charset=UTF-8"}> -<#compress> - - - - -geluke - - - - - - - - - -
    - -

    鏁憡鐢ㄦ埛锛�

    - 鎮ㄥ綋鍓嶇殑娴忚鍣ㄥ彲鑳芥棤娉曡緝濂界殑鏀寔鏈郴缁燂紝璇蜂娇鐢‵ireFox锛堢伀鐙愶級 鎴� Chrome锛堣胺姝岋級娴忚鍣ㄤ互杈惧埌瀹岀編浣撻獙锛� -
    - -<#include "/header/top.ftl" /> -
    -
    - -
    - -
    - -
    -
    - ${cityName!} - [閫夋嫨鍩庡競] -
    - -
    - - <#--
    - ${name!} 閫�鍑� -
    --> -
    -
    - - <#include "../header/header_map.ftl" /> - - - - - -
    -
    -
    - - -
    -
    - -
    -
    - - - -
    - - - <#--
    - <#if (otherShops??&&otherShops?size>0)> - <#list otherShops as shopName> -

    - - -
    --> - -
    -
    - 娓╅Θ鎻愮ず锛� -
    鐢变簬鏁版嵁杈冨锛屾暟鎹皢鍦ㄥ湴鍥炬斁澶у埌涓�瀹氱殑绾у埆鍚庢樉绀�
    -
    - -
    -
    - - -
    -
    -
    0绫�
    -
    - - - - -
    - -
    -
    -
      -
    - -
      -
    • - - - - -
    • - -
    - -
      -
    • - - - - -
    • -
    -
    -
    -
    -
    -
    -
    - - -
    - <#----> -
    - - 姝e湪鍔姏鍔犺浇鏁版嵁锛岃绋嶅��...... -
    -
    - 鍔犺浇鏁版嵁澶锛岃鏀惧ぇ鍦板浘...... -
    - <#-- -
    -
    - --> -
    - <#--
    --> - <#--
    --> - <#if (supportAuto==1)> -
    鑷姩鍟嗗湀
    - - -
    - 浜哄彛灞曠ず -
    -
      -
    • -
    • -
    • -
    • -
    -
    -
    -
    - 鑱氬鐐瑰睍绀� -
    -
      -
    • -
    • -
    • -
    • -
    • -
    • -
    • - <#if (supportHot==1)> -
    • - -
    -
    -
    - -
    淇濆瓨鍟嗗湀
    - -
    - 灞呮皯鎴锋暟锛�-鎴� - 灏忓尯鍧囦环锛�-鍏� - 鍔炲叕浜烘暟锛�-浜� - -
    - - -
    -
    -
    -
      -
    • 鏂版潙
    • -
    • 鏅�氬晢鍝佹埧
    • -
    • 楂樼骇鍟嗗搧鎴�
    • -
    • 鍐欏瓧妤�
    • -
    -
    -
    -
      -
    •  
    • -
    • 鍦ㄥ缓鏅�氬晢鍝佹埧
    • -
    • 鍦ㄥ缓楂樼骇鍟嗗搧鎴�
    • -
    • 鍦ㄥ缓鍐欏瓧妤�
    • -
    -
    -
    -
    - -
    -
    -
    -
    -
    -
    鐑害浣�
    -
    鐑害楂�
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    鏃�
    -
    -
    - - -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    鏃�
    -
    -
    - - - -
    -
    -
    -
    -
    - - -
    - - 璇峰厛纭畾闂ㄥ簵浣嶇疆 -
    - -
    -
    X
    -
    璇风偣鍑� 纭鍦板潃 鎴� 鎷栨嫿 鍒版偍闇�瑕佺殑鍦板潃鐒跺悗纭畾
    -
    - - -
    - 寮�鍚儹鍔涘浘 -
    -
      -
    • -
    • -
    • - <#--
    • --> - <#--
    • --> -
    -
    -
    - -
    - 寮�鍚綉鏍煎浘 - -
    -
      -
    • -
    • -
    • -
    -
    -
    - - - -
    -
    - -
    -
    -
    - - -
    -
    -
    - - - - - - - - - - - -

    -
    - - - - - - - - - - - - - - - - - - - - - +<#ftl attributes={"content_type":"text/html; charset=UTF-8"}> +<#compress> + + + + +geluke + + + + + + + + + +
    + +

    鏁憡鐢ㄦ埛锛�

    + 鎮ㄥ綋鍓嶇殑娴忚鍣ㄥ彲鑳芥棤娉曡緝濂界殑鏀寔鏈郴缁燂紝璇蜂娇鐢‵ireFox锛堢伀鐙愶級 鎴� Chrome锛堣胺姝岋級娴忚鍣ㄤ互杈惧埌瀹岀編浣撻獙锛� +
    + +<#include "/header/top.ftl" /> +
    +
    + +
    + +
    + +
    +
    + ${cityName!} + [閫夋嫨鍩庡競] +
    + +
    + + <#--
    + ${name!} 閫�鍑� +
    --> +
    +
    + + <#include "../header/header_map.ftl" /> + + + + + +
    +
    +
    + + +
    +
    + +
    +
    + + + +
    + + + <#--
    + <#if (otherShops??&&otherShops?size>0)> + <#list otherShops as shopName> +

    + + +
    --> + +
    +
    + 娓╅Θ鎻愮ず锛� +
    鐢变簬鏁版嵁杈冨锛屾暟鎹皢鍦ㄥ湴鍥炬斁澶у埌涓�瀹氱殑绾у埆鍚庢樉绀�
    +
    + +
    +
    + + +
    +
    +
    0绫�
    +
    + + + + +
    + +
    +
    +
      +
    + +
      +
    • + + + + +
    • + +
    + +
      +
    • + + + + +
    • +
    +
    +
    +
    +
    +
    +
    + + +
    + <#----> +
    + + 姝e湪鍔姏鍔犺浇鏁版嵁锛岃绋嶅��...... +
    +
    + 鍔犺浇鏁版嵁澶锛岃鏀惧ぇ鍦板浘...... +
    + <#-- +
    +
    + --> +
    + <#--
    --> + <#--
    --> + <#if (supportAuto==1)> +
    鑷姩鍟嗗湀
    + + +
    + 浜哄彛灞曠ず +
    +
      +
    • +
    • +
    • +
    • +
    +
    +
    +
    + 鑱氬鐐瑰睍绀� +
    +
      +
    • +
    • +
    • +
    • +
    • +
    • +
    • + <#if (supportHot==1)> +
    • + +
    +
    +
    + +
    淇濆瓨鍟嗗湀
    + +
    + 灞呮皯鎴锋暟锛�-鎴� + 灏忓尯鍧囦环锛�-鍏� + 鍔炲叕浜烘暟锛�-浜� + +
    + + +
    +
    +
    +
      +
    • 鏂版潙
    • +
    • 鏅�氬晢鍝佹埧
    • +
    • 楂樼骇鍟嗗搧鎴�
    • +
    • 鍐欏瓧妤�
    • +
    +
    +
    +
      +
    •  
    • +
    • 鍦ㄥ缓鏅�氬晢鍝佹埧
    • +
    • 鍦ㄥ缓楂樼骇鍟嗗搧鎴�
    • +
    • 鍦ㄥ缓鍐欏瓧妤�
    • +
    +
    +
    +
    + +
    +
    +
    +
    +
    +
    鐑害浣�
    +
    鐑害楂�
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    鏃�
    +
    +
    + + +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    鏃�
    +
    +
    + + + +
    +
    +
    +
    +
    + + +
    + + 璇峰厛纭畾闂ㄥ簵浣嶇疆 +
    + +
    +
    X
    +
    璇风偣鍑� 纭鍦板潃 鎴� 鎷栨嫿 鍒版偍闇�瑕佺殑鍦板潃鐒跺悗纭畾
    +
    + + +
    + 寮�鍚儹鍔涘浘 +
    +
      +
    • +
    • +
    • + <#--
    • --> + <#--
    • --> +
    +
    +
    + +
    + 寮�鍚綉鏍煎浘 + +
    +
      +
    • +
    • +
    • +
    +
    +
    + + + +
    +
    + +
    +
    +
    + + +
    +
    +
    + + + + + + + + + + + +

    +
    + + + + + + + + + + + + + + + + + + + + +