From da38bead89c598b7f7760707d1770ee17f862cd8 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 26 Oct 2022 17:35:13 +0800 Subject: [PATCH] =?UTF-8?q?=E5=BB=BA=E8=A1=8C=E7=9B=B8=E5=85=B3=E8=84=9A?= =?UTF-8?q?=E6=9C=AC=E6=9B=B4=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../BigdataCommunityBatch.groovy | 43 +++++++------------ .../ccbevaluation/BigdataForsaleBatch.groovy | 1 - .../script/ccbevaluation/CcbFtpUtils.groovy | 8 ++++ .../ccbevaluation/ExportOracleData.groovy | 38 ++++++++-------- .../gj/district/FangjiaClientUtils.groovy | 17 ++++++++ .../fangjia/script/jiangnanblank/SFTP.groovy | 16 +++++++ .../ThirdDataToGatherBatch.groovy | 2 +- .../fangjia/script/util/OpenApiClient.groovy | 1 + 8 files changed, 77 insertions(+), 49 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy index c06cef98e..a96bb9b09 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -8,6 +8,7 @@ import com.fangjia.script.Watcher import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.ccb.CcbCodeUtils import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.gj.district.FangjiaClientUtils import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.OpenApiClient @@ -175,38 +176,25 @@ class BigdataCommunityBatch extends BaseBatch{ ret << "pa101" ret << "" if(dist.address){ - if(dist.address.toString().contains("弄")){ - String number = DataUtil.match(dist.address, Pattern.compile("\\d+弄")) - if(number){ - ret << dist.address.toString().substring(0, dist.address.toString().indexOf(number)) - ret << dist.address.toString().substring(dist.address.toString().indexOf(number), dist.address.toString().size()) + def jsonObj = null + if(dist.address){ + jsonObj = FangjiaClientUtils.addressStructure(dist.city, dist.address) + } + if(jsonObj && jsonObj.street){ + if(jsonObj.street){ + ret << jsonObj.street }else{ ret << "" + } + + if(jsonObj.streetNumber){ + ret << jsonObj.streetNumber + }else{ ret << "" } }else{ - def jsonObj = null - if(dist.address){ -// jsonObj = OpenApiClient.addressMatch(dist.city, dist.address, "property", false, dist.region, dist.name) - jsonObj = OpenApiClient.standard1Action(dist.city, dist.address) - } - if(jsonObj && jsonObj.street){ - if(jsonObj.street){ - ret << jsonObj.street - }else{ - ret << "" - } - - if(jsonObj.streetNumber){ - ret << jsonObj.streetNumber - }else{ - ret << "" - } - - }else{ - ret << "" - ret << "" - } + ret << "" + ret << "" } }else{ ret << "" @@ -225,7 +213,6 @@ class BigdataCommunityBatch extends BaseBatch{ println dist.city + " "+dist.name+" error" e.printStackTrace() } - return ret; } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index ca517f31a..5b0edbae1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -448,7 +448,6 @@ class BigdataForsaleBatch extends BaseBatch{ if(!alias.contains(newName)){ alias << newName } - } if(alias && alias.size()>0){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpUtils.groovy index 6b972553e..124911f34 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpUtils.groovy @@ -3,6 +3,7 @@ package com.fangjia.script.ccbevaluation import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.jiangnanblank.SFTP import com.jcraft.jsch.ChannelSftp +import com.jcraft.jsch.SftpATTRS import org.apache.commons.codec.digest.DigestUtils class CcbFtpUtils { @@ -64,6 +65,13 @@ class CcbFtpUtils { return files } + static def getFileLength(String path, String month){ + sftpChannel = reConnect(); + def size = sftp.getFileSize(ftpPath+ month+"/"+path, sftpChannel) + return size + } + + static main(args) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy index eabf59e2a..22861ba1d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -30,24 +30,24 @@ class ExportOracleData { def lastMonthStr = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM") def new_dists = [] //不会变 10.25 -// exportEvaluationRateData() -// exportBigdataPoiData()// -// exportBigdataDistrictData()// -// -// exportGuidedPrice()//随人工改变// -// exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策// -// -//// //处理小区情况 -// exportBigdataCommunity()// -// exportCommunityStaging()//没cityid + exportEvaluationRateData() + exportBigdataPoiData()// + exportBigdataDistrictData()// + + exportGuidedPrice()//随人工改变// + exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策// + +// //处理小区情况 + exportBigdataCommunity()// + exportCommunityStaging()//没cityid exportCommunityBuileType()// -// -// exportThirdPartyPrice2(lastMonth, new_dists)// -// exportBigdataForsale(lastMonthStr +"-19", new_dists) //2022-07-19 -// exportBigdataDeal(lastMonthStr + "-01", new_dists) //2022-07-01 -// -// //第三方网站权重表 -// exportWebsiteRate(); + + exportThirdPartyPrice2(lastMonth, new_dists)// + exportBigdataForsale(lastMonthStr +"-19", new_dists) //2022-07-19 + exportBigdataDeal(lastMonthStr + "-01", new_dists) //2022-07-01 + + //第三方网站权重表 + exportWebsiteRate(); // // // //计算结果相关 @@ -229,8 +229,8 @@ class ExportOracleData { FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); println "BIGDATA_COMMBUILDTYPE_GJ 数据文件输出完成......" -// CcbFtpUtils.ftpUpload(null, fileName, curDate) -// println "BIGDATA_COMMBUILDTYPE_GJ 上传Ftp完成......" + CcbFtpUtils.ftpUpload(null, fileName, curDate) + println "BIGDATA_COMMBUILDTYPE_GJ 上传Ftp完成......" } def exportBigdataDistrictData() { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy index 9f3670a44..ae1dee2e4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy @@ -67,6 +67,7 @@ class FangjiaClientUtils { + //分词 省,市,区 public static def addressStand(def city, def address){ def jsonObj = null def content = null @@ -84,6 +85,22 @@ class FangjiaClientUtils { return jsonObj; } + //分词基础上的标准化,complete=false 不不全 short=true 省市区返回简拼 + public static def addressStructure(def city, def address){ + def jsonObj = null + def content = null + try { + address = address.replace("{","") + def url = "http://192.168.0.19:31080/FangJiaAddressService/structure/v2?address="+URLEncoder.encode(address,"UTF-8")+"&city="+URLEncoder.encode(city,"UTF-8") + content = new HttpRequest().newGet(url, null, "utf-8") + jsonObj = JSONObject.fromObject(content) + } catch (Exception e) { + println address +" "+content + e.printStackTrace(); + } + return jsonObj; + } + public static def arroundPrice(city,address){ def jsonObj = null def content = null diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/SFTP.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/SFTP.groovy index 70082d882..a50827e27 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/SFTP.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/SFTP.groovy @@ -209,4 +209,20 @@ class SFTP { return isDirExistFlag; } + public long getFileSize(String srcSftpFilePath, ChannelSftp sftp) { + long fileSize;//文件大于等于0则存在 + try { + SftpATTRS sftpATTRS = sftp.lstat(srcSftpFilePath); + fileSize = sftpATTRS.getSize(); + } catch (Exception e) { + e.printStackTrace() + fileSize = -1;//获取文件大小异常 + if (e.getMessage().toLowerCase().equals("no such file")) { + fileSize = -2;//文件不存在 + } + } + return fileSize; + } + + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy index 850ef9d42..320e65b2f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy @@ -210,7 +210,7 @@ class ThirdDataToGatherBatch extends BaseBatch{ DBObject udata = new BasicDBObject() // udata.put("level",d.level); // udata.put("last_price", d.last_price); - udata.put("district_id", d._id.toString()) + udata.put("seaweed_id", d._id.toString()) udata.put("fangjia_price", d.fangjia_price); fieldMap.each{k,v-> diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy index cc7b1ad50..7c37cae97 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy @@ -364,6 +364,7 @@ public class OpenApiClient { } + //此接口地址标准化,带补全 public static JSONObject standard1Action(String city, String address) { try { Map params = new HashMap();