From 2cea6c4190c3db222915d7c3a14cfdc3d0c6b43a Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Mon, 21 Feb 2022 11:56:06 +0800 Subject: [PATCH 1/5] =?UTF-8?q?=E4=BC=98=E5=8C=96=E5=89=8D=E6=B5=B7?= =?UTF-8?q?=E6=B5=8B=E8=AF=95=E8=84=9A=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/temp/shenjian/qianhai/QianhaiTest.groovy | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/qianhai/QianhaiTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/qianhai/QianhaiTest.groovy index 7c92e5a3e..6994cf1dc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/qianhai/QianhaiTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/qianhai/QianhaiTest.groovy @@ -8,7 +8,7 @@ import com.fangjia.script.xingyebank.CityUtil import net.sf.json.JSONObject import org.apache.commons.lang.StringUtils -ExcelUtils.multiThreadingModify("D:\\数据\\前海\\前海征信测试数据test.xlsx",{List item-> +ExcelUtils.multiThreadingModify("D:\\数据\\前海\\前海征信测试数据.xlsx",{List item-> if(StringUtils.isBlank(item['B'])){ return; @@ -20,6 +20,11 @@ ExcelUtils.multiThreadingModify("D:\\ params["token"] = "F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB" JSONObject json = request.getJsonContent("http://open.fangjia.com/qhzx/propertyDetail", params, "UTF-8") + if((!json || !json.result) && StringUtils.isNotBlank(item['G'])){ + params["address"] = item['G'] + json = request.getJsonContent("http://open.fangjia.com/qhzx/propertyDetail", params, "UTF-8") + } + if(json.code == 200 && json.result){ JSONObject result = json.result item['C'] = CityUtil.getProvinceByCity(result.city) @@ -28,8 +33,6 @@ ExcelUtils.multiThreadingModify("D:\\ item['F'] = result.name if (result.name) { item['O'] = '住宅地址' - } else { - item['O'] = '非住宅地址' } item['G'] = result.address item['H'] = result.lng @@ -51,6 +54,9 @@ ExcelUtils.multiThreadingModify("D:\\ item['J'] = json.getJSONObject("result").averagePrice } } + if (!item['O']) { + item['O'] = '非住宅地址' + } params.clear() params["city"] = getCityFormAddress(item['B']) params["address"] = item['B'] From 85b918acbe83be12f75693ef789e16a6e8fe6a1f Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Fri, 4 Mar 2022 11:15:49 +0800 Subject: [PATCH 2/5] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E9=87=8D=E5=BA=86?= =?UTF-8?q?=E5=86=9C=E5=95=86=E8=A1=8C=E6=B5=8B=E8=AF=95=E8=84=9A=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../shenjian/poi/GetCommitteeFromExcel.groovy | 25 +++++++++++++++ .../shenjian/poi/GetLngLatFromExcel.groovy | 17 ++++++++++ .../shenjian/poi/GetTownshipFromExcel.groovy | 31 +++++++++++++++++++ 3 files changed, 73 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetCommitteeFromExcel.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetLngLatFromExcel.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetTownshipFromExcel.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetCommitteeFromExcel.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetCommitteeFromExcel.groovy new file mode 100644 index 000000000..8125bf1db --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetCommitteeFromExcel.groovy @@ -0,0 +1,25 @@ +package com.fangjia.script.temp.shenjian.poi + +import com.fangjia.script.statistics.poi.AmapPoi +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import org.apache.commons.collections.CollectionUtils +import org.apache.commons.lang.StringUtils + +com.fangjia.script.util.ExcelUtils.multiThreadingModify("D:\\Users\\Domi\\Documents\\知识库\\重庆农商行\\重庆农商行测试地址V8.xlsx",{List item-> + + // 乡镇以下级政府及事业单位(居委会) + def types = "130106" + def city = item['D'] + def lng = item['M'] + def lat = item['N'] + if (StringUtils.isBlank(item['G'])) { + List pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "") + if (CollectionUtils.isNotEmpty(pois)) { + AmapPoi amapPoi = pois.get(0) + if (amapPoi.getCityname().startsWith(city)) { + item['G'] = amapPoi.getName() + item['S'] = '修改社区' + } + } + } +}) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetLngLatFromExcel.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetLngLatFromExcel.groovy new file mode 100644 index 000000000..6b37becf8 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetLngLatFromExcel.groovy @@ -0,0 +1,17 @@ +package com.fangjia.script.temp.shenjian.poi + +import com.fangjia.core.util.GeoUtils +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import org.apache.commons.lang.StringUtils + +com.fangjia.script.util.ExcelUtils.multiThreadingModify("D:\\Users\\Domi\\Documents\\知识库\\重庆农商行\\重庆农商行测试地址V8.xlsx",{List item-> + + if (item['L'] != null) { + def location = GeoUtils.gcjToGps84(item['L'], item['K']) + if (location) { + item['N'] = location.lat + item['M'] = location.lng + println(item) + } + } +}) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetTownshipFromExcel.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetTownshipFromExcel.groovy new file mode 100644 index 000000000..c8481a44a --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetTownshipFromExcel.groovy @@ -0,0 +1,31 @@ +package com.fangjia.script.temp.shenjian.poi + +import com.fangjia.script.statistics.poi.AmapPoi +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.apache.commons.collections.CollectionUtils +import org.apache.commons.lang.StringUtils + +final DBCollection villagesConn = MongoUtils.getCollectionByName("fangjia", "villages", "192.168.0.43", 28017) + +/** + * 获取国家标准居委街道 + */ +com.fangjia.script.util.ExcelUtils.multiThreadingModify("D:\\Users\\Domi\\Documents\\知识库\\重庆农商行\\重庆农商行测试地址V9(补全省市区代码).xlsx",{List item-> + + def flag = item['S'] + def area = item['E'] + def name = item['G'] + if (StringUtils.isNotBlank(item['G']) && StringUtils.isNotBlank(area) && String && flag == '修改社区' && item['T'] == null) { + villagesConn.find(new BasicDBObject("province_name", "重庆市") + .append("area_name", area + "区").append("name", name), new BasicDBObject()).limit(1).each { + obj -> + item['F'] = obj.street_name + item['L'] = obj.street_code + item['T'] = "补充街道正确" + } + } +}) From 38c28bc2e9484f5ab40206f05dd3b550bee12b40 Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Fri, 4 Mar 2022 17:27:25 +0800 Subject: [PATCH 3/5] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E9=87=8D=E5=BA=86?= =?UTF-8?q?=E5=86=9C=E5=95=86=E8=A1=8C=E6=B5=8B=E8=AF=95=E8=84=9A=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/fangjia/core/util/DataUtil.java | 19 +++++ .../FillTownshipAndCommunityFromExcel.groovy | 39 +++++++++ .../shenjian/address/GetAddressDetail.groovy | 79 +++++++++++++++++++ .../temp/shenjian/poi/GetCodeFromExcel.groovy | 48 +++++++++++ 4 files changed, 185 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/FillTownshipAndCommunityFromExcel.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/GetAddressDetail.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetCodeFromExcel.groovy diff --git a/projects/fangjia-core/src/main/java/com/fangjia/core/util/DataUtil.java b/projects/fangjia-core/src/main/java/com/fangjia/core/util/DataUtil.java index 93218b265..c681c113d 100644 --- a/projects/fangjia-core/src/main/java/com/fangjia/core/util/DataUtil.java +++ b/projects/fangjia-core/src/main/java/com/fangjia/core/util/DataUtil.java @@ -1435,6 +1435,25 @@ public class DataUtil { return null; } + /** + * 匹配符合条件的字符串 + * + * @param str + * @param pattern + * @return + */ + public static List matchAll(String str, Pattern pattern) { + if (str == null) { + return null; + } + List list = new ArrayList(); + Matcher matcher = pattern.matcher(str); + while (matcher.find()) { + list.add(matcher.group()); + } + return list; + } + public static boolean isNumeric(String str) { try { Double.parseDouble(str); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/FillTownshipAndCommunityFromExcel.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/FillTownshipAndCommunityFromExcel.groovy new file mode 100644 index 000000000..79cf910db --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/FillTownshipAndCommunityFromExcel.groovy @@ -0,0 +1,39 @@ +package com.fangjia.script.temp.shenjian.address + +import com.fangjia.script.util.ExcelUtils +import org.apache.commons.lang.StringUtils + +/** + * 补充详细地址中缺失的乡镇街道与居委社区 + */ +ExcelUtils.multiThreadingModify("D:\\Users\\Domi\\Documents\\知识库\\重庆农商行\\重庆农商行测试地址V11(优化详细地址) .xlsx", { + List item -> + def area = item['F'] + def township = item['G'] + def community = item['H'] + String detailAddr = item['I'] + + // 不包含街道与居委 + if (StringUtils.isNotBlank(township) && StringUtils.isNotBlank(community) && + !detailAddr.contains(township) && !detailAddr.contains(community) ) { + if (StringUtils.isNotBlank(area)) { + int index = detailAddr.indexOf(area + "区") + area.length() + 2 + item['K'] = detailAddr.substring(0, index) + township + community + + detailAddr.substring(index) + } + // 不包含社区 + } else if (StringUtils.isNotBlank(township) && StringUtils.isNotBlank(community) && + detailAddr.contains(township) && !detailAddr.contains(community)) { + int index = detailAddr.indexOf(township) + township.length() + 1 + item['K'] = detailAddr.substring(0, index) + community + + detailAddr.substring(index) + } // 只不包含街道 + else if (StringUtils.isNotBlank(township) && StringUtils.isNotBlank(community) && + !detailAddr.contains(township) && detailAddr.contains(community)) { + if (StringUtils.isNotBlank(area)) { + int index = detailAddr.indexOf(area + "区") + area.length() + 2 + item['K'] = detailAddr.substring(0, index) + township + + detailAddr.substring(index) + } + } +}) \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/GetAddressDetail.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/GetAddressDetail.groovy new file mode 100644 index 000000000..b77b6d36c --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/GetAddressDetail.groovy @@ -0,0 +1,79 @@ +package com.fangjia.script.temp.shenjian.address + +import com.fangjia.core.util.DataUtil +import com.fangjia.script.util.ExcelUtils +import net.sf.json.JSONObject +import org.apache.commons.lang.StringUtils + +import java.util.regex.Pattern + +//ExcelUtils.removeEmptyRow("D:\\Users\\Domi\\Documents\\知识库\\重庆农商行\\重庆农商行测试地址V10(优化补充社区冗余信息).xlsx") +ners = ["houseno", "cellno", "floorno", "roomno"] +Pattern GROUP_PATTERN = Pattern.compile("[一二三四五六七八九十甲乙丙丁\\d]+组") +ExcelUtils.multiThreadingModify("D:\\Users\\Domi\\Documents\\知识库\\重庆农商行\\重庆农商行测试地址V11(优化详细地址) .xlsx",{List item-> + + if (StringUtils.isNotBlank(item["J"])) { + //item["I"] = item["J"] + //item["W"] = "详细地址替换" + } +// def addr = item["B"].toString() +// def detailAddr = item["I"]?.toString() +// def pd = item["C"] +// item['J'] = "" +// if (StringUtils.isNotBlank(pd)) { +// int len = addr.lastIndexOf(pd) +// if (len > 3) { +// String aim = addr.substring(len - 3, len) +// int aimIndex = detailAddr.lastIndexOf(aim) +// if (aimIndex > 0) { +// item['J'] = detailAddr.substring(0, aimIndex + 3) + pd +// } +// } +// } + +// if(detailAddr{ +// String group = DataUtil.match(pd, GROUP_PATTERN) +// if(group == null ){ +// return +// } +// def arabic_group = null +// if(!detailAddr.contains(group)){ +// arabic_group = CNNMFilter.qCNNumericToArabic(group) +// } +// if(group){ +// def flg = addr.substring(addr.indexOf(group) - 3, addr.indexOf(group)) +// flg = flg.replace("-", "") +// if(arabic_group != null){ +// group = arabic_group +// } +// if(!detailAddr.contains(flg + group)){ +// println(item) +// detailAddr = detailAddr.replace(group, '') +// def idx = detailAddr.lastIndexOf(flg) +// try{ +// detailAddr = detailAddr.substring(0, idx) + flg + group + detailAddr.substring(idx + flg.length()) +// item["j"] = detailAddr +// }catch(Exception e){ +// item["W"] ="组替换失败" +// } +// +// +// } +// } +// } + +// HttpRequest request = new HttpRequest() +// JSONObject result = request.getJsonContent("http://192.168.0.19:31080/FangJiaNlpService/ner",["address":addr], "utf-8") +// def index = Integer.MAX_VALUE +// for(JSONObject entity : result.entities){ +// if(entity.type in ners){ +// if(entity.start < index){ +// index = entity.start +// } +// } +// } +// if(index != Integer.MAX_VALUE){ +// item["C"] = addr.substring(index) +// println(item) +// } +}) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetCodeFromExcel.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetCodeFromExcel.groovy new file mode 100644 index 000000000..f466ed3be --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetCodeFromExcel.groovy @@ -0,0 +1,48 @@ +package com.fangjia.script.temp.shenjian.poi + +import com.fangjia.script.statistics.poi.AmapPoi +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.apache.commons.collections.CollectionUtils +import org.apache.commons.lang.StringUtils + +final DBCollection villagesConn = MongoUtils.getCollectionByName("fangjia", "villages", "192.168.0.43", 28017) + +/** + * 获取省市区编码 + */ +com.fangjia.script.util.ExcelUtils.multiThreadingModify("D:\\Users\\Domi\\Documents\\知识库\\重庆农商行\\重庆农商行测试地址V9(补全省市区代码).xlsx",{List item-> + + def province = item['C'] + def city = item['D'] + def area = item['E'] + if (StringUtils.isBlank(province) && city == '重庆' && item['K'] == null) { + item['J'] = '5000000' + if (StringUtils.isNotBlank(city) && StringUtils.isNotBlank(area)) { + villagesConn.find(new BasicDBObject("province_name", "重庆市") + .append("area_name", new BasicDBObject('$regex', '^' + area)), new BasicDBObject() ) + .limit(1).each {obj -> + item['J'] = obj.province_code + item['K'] = obj.area_code + } + } + } else if (StringUtils.isNotBlank(province) && StringUtils.isNotBlank(city) && StringUtils.isNotBlank(area) && item['K'] == null) { + villagesConn.find(new BasicDBObject("province_name", new BasicDBObject('$regex', '^' + province)) + .append("city_name", new BasicDBObject('$regex', '^' + city)) + .append("area_name", new BasicDBObject('$regex', '^' + area)), new BasicDBObject() ) + .limit(1).each {obj -> + item['I'] = obj.province_code + item['J'] = obj.city_code + item['K'] = obj.area_code + } + } + if (item['I'] != null) { + item['I'] = (String.valueOf(item['I']).replace(".0", "") + "000000").substring(0, 6); + } + if (item['J'] != null) { + item['J'] = (String.valueOf(item['J']).replace(".0", "") + "000000").substring(0, 6); + } +}) From 0df8ff3d21f2aad1c3033753907fcbdef17a0053 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 9 Mar 2022 13:31:19 +0800 Subject: [PATCH 4/5] update --- .../ccbevaluation/CcbFtpTestUtils.groovy | 12 + .../fangjia/script/ccbevaluation/Test.groovy | 68 +++--- .../ronghe/BigdataCommunityGJRHBatch.groovy | 26 +-- .../ronghe/ExportDistrictRongHeTest.groovy | 5 + .../groovy/com/fangjia/script/gj/Test.groovy | 28 ++- .../script/gj/level/DsElasticUtils.groovy | 215 ++++++++++++++++++ .../gj/paimai/BeiKeCrawlByChrome.groovy | 4 +- .../gj/paimai/LianjiaCrawlByChrome.groovy | 12 +- .../script/gj/price/FzMatchTest.groovy | 85 +++++++ .../script/thirdPriceCrawl/DataStat.groovy | 8 +- .../script/thirdPriceCrawl/Esf58Crawl.groovy | 142 +++++++++++- .../thirdPriceCrawl/SynchDistrictPrice.groovy | 4 +- .../TestCrawlTotalCount.groovy | 4 +- 13 files changed, 548 insertions(+), 65 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DsElasticUtils.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/price/FzMatchTest.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpTestUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpTestUtils.groovy index 4190d947b..7083cb615 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpTestUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpTestUtils.groovy @@ -27,9 +27,21 @@ class CcbFtpTestUtils { }else{ sftp.upload(ftpPath+ month+"/", outputPath_dist, sftpChannel); } + } + + + static def delete(path){ + if(path.startsWith("/")){ + sftp.deleteSFTP(path, sftpChannel) + }else{ + sftp.deleteFile(path) + } } + + + static def reConnect(){ if(sftpChannel == null){ println "sftpChannel 为空,需重连....." diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index 0b63403ce..6a9d4446e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -270,24 +270,24 @@ def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def // } //},0) -//def index = 0 -//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\人工核实需删除数据\\320100_南京_是否独有及第三方是否存在(删除1049).xlsx",{d-> -// def flag = d['M'] -// if(flag==0.0){ -// if(d['D']){ -// def udata = new BasicDBObject() -// udata.put("name_cat", 11) -// udata.put("m_date",new Date()) -// -// def q = new BasicDBObject() -// q.put("_id", new ObjectId(d['D'])) -// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) -// index++ -// println index -// } -// } -// -//},0) +def index = 0 +ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\人工核实需删除数据\\是否独有及第三方是否存在20220217\\9370300_淄博_是否独有及第三方是否存在(删除383).xlsx",{d-> + def flag = d['M'] + if(flag==0.0){ + if(d['D']){ + def udata = new BasicDBObject() + udata.put("name_cat", 11) + udata.put("m_date",new Date()) + + def q = new BasicDBObject() + q.put("_id", new ObjectId(d['D'])) + DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) + index++ + println index + } + } + +},0) //ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\补充删除数据1223_1228.xlsx",{d-> // if(d['A']){ @@ -782,22 +782,22 @@ def getStreetInfo(def city, def glocation){ //} //ExcelUtils.write("C:\\Users\\gaojuan\\Desktop\\上海小区.xlsx", rets, "数据") -def datas1 = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\12月.xlsx", 0, 1) -def datas2 = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\1月.xlsx", 0, 1) - -def keys = [] -for(def d : datas2){ - keys << d['A']+"##"+d['C']+"##"+d['B'] -} - -def index = 0 -for(def d : datas1){ - def k = d['A']+"##"+d['C']+"##"+d['B'] - if(!keys.contains(k)){ - index ++ - println index +" "+k - } -} +//def datas1 = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\12月.xlsx", 0, 1) +//def datas2 = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\1月.xlsx", 0, 1) +// +//def keys = [] +//for(def d : datas2){ +// keys << d['A']+"##"+d['C']+"##"+d['B'] +//} +// +//def index = 0 +//for(def d : datas1){ +// def k = d['A']+"##"+d['C']+"##"+d['B'] +// if(!keys.contains(k)){ +// index ++ +// println index +" "+k +// } +//} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy index 6409c9795..dc70af53b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy @@ -90,19 +90,19 @@ class BigdataCommunityGJRHBatch extends BaseBatch { dist.estatecharge = CcbCodeUtils.processEstateCharge(dist.city, obj.estate_charge, obj.estate_type2) } } -// println index +" "+city +" 小区加载完成 "+districts.size() -// addBigdataCommunityGJRH(districts, cityCodeDataMap.get(city)) -// println index +" "+city +" 1.小区表完成" -// addBigdataCommbuildstruGJ(districts, cityCodeDataMap.get(city)) -// println index +" "+city +" 2.小区建筑结构表" -// addBigdataCommhousetypeGJ(districts, cityCodeDataMap.get(city)) -// println index +" "+city +" 3.小区房屋类型表" -// addBigdataCommbuildyearGj(districts, cityCodeDataMap.get(city)) -// println index +" "+city +" 4.小区建筑年代表" -// addBigdataCommfeeGj(districts, cityCodeDataMap.get(city)) -// println index +" "+city +" 5.小区费用表" -// addBigdataCommhouseuseGJ(districts, cityCodeDataMap.get(city)) -// println index +" "+city +" 6.小区房屋用途表" + println index +" "+city +" 小区加载完成 "+districts.size() + addBigdataCommunityGJRH(districts, cityCodeDataMap.get(city)) + println index +" "+city +" 1.小区表完成" + addBigdataCommbuildstruGJ(districts, cityCodeDataMap.get(city)) + println index +" "+city +" 2.小区建筑结构表" + addBigdataCommhousetypeGJ(districts, cityCodeDataMap.get(city)) + println index +" "+city +" 3.小区房屋类型表" + addBigdataCommbuildyearGj(districts, cityCodeDataMap.get(city)) + println index +" "+city +" 4.小区建筑年代表" + addBigdataCommfeeGj(districts, cityCodeDataMap.get(city)) + println index +" "+city +" 5.小区费用表" + addBigdataCommhouseuseGJ(districts, cityCodeDataMap.get(city)) + println index +" "+city +" 6.小区房屋用途表" addBigdataPositionGJ(districts, cityCodeDataMap.get(city), citySeqnoMap) println index +" "+ city + " 7.地理位置表" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy index 0c1c0c558..11270dab3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy @@ -26,6 +26,11 @@ class ExportDistrictRongHeTest extends BaseBatch{ @Override void process(Object args) { def curDate = 20220310//DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd") + def fileName = "/ccb_rh/${curDate}" +// if(CommonDatautils.isWindow()){ +// fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}" +// } + CcbFtpTestUtils.delete(fileName) exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd")) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/Test.groovy index f2dbd6ff8..2996353c6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/Test.groovy @@ -6,6 +6,7 @@ import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.HttpClient import com.fangjia.script.util.MongoUtils +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils import com.fangjia.script.yinjihaun.qqmap.TengXunApiUtils import com.mongodb.BasicDBObject import com.mongodb.Bytes @@ -42,4 +43,29 @@ println new HttpClient().get("https://longyuanguojiguangchangbl.fang.com/","gbk" // } // //} -//ExcelUtils.write("D:\\数据\\结果\\区域信息.xlsx", ret, "数据") \ No newline at end of file +//ExcelUtils.write("D:\\数据\\结果\\区域信息.xlsx", ret, "数据") + +ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\重庆农商行测试地址.xlsx",{d-> + if(d['B']){ + def json = GaoDeApiUtils.geo("重庆", d['B']) + if(json && json.info=="OK"){ + def datas = json.geocodes + if(datas){ + def row = datas[0] + println row + d['C'] = row.country + d['D'] = row.province + d['E'] = row.city.toString().replace("[]","") + d['F'] = row.citycode.toString().replace("[]","") + d['G'] = row.district.toString().replace("[]","") + d['H'] = row.street.toString().replace("[]","") + d['I'] = row.number.toString().replace("[]","") + d['J'] = row.adcode.toString().replace("[]","") + d['K'] = row.location + d['L'] = row.level + } + } + } + + +}) \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DsElasticUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DsElasticUtils.groovy new file mode 100644 index 000000000..9ef7e6926 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DsElasticUtils.groovy @@ -0,0 +1,215 @@ +package com.fangjia.script.util + +import org.apache.http.HttpHost +import org.apache.http.auth.AuthScope +import org.apache.http.auth.UsernamePasswordCredentials +import org.apache.http.client.CredentialsProvider +import org.apache.http.client.config.RequestConfig +import org.apache.http.impl.client.BasicCredentialsProvider +import org.apache.http.impl.nio.client.HttpAsyncClientBuilder +import org.elasticsearch.action.ActionListener +import org.elasticsearch.action.bulk.BackoffPolicy +import org.elasticsearch.action.bulk.BulkProcessor +import org.elasticsearch.action.bulk.BulkRequest +import org.elasticsearch.action.bulk.BulkResponse +import org.elasticsearch.action.index.IndexRequest +import org.elasticsearch.action.search.ClearScrollRequest +import org.elasticsearch.action.search.SearchRequest +import org.elasticsearch.action.search.SearchResponse +import org.elasticsearch.action.search.SearchScrollRequest +import org.elasticsearch.client.RequestOptions +import org.elasticsearch.client.RestClient +import org.elasticsearch.client.RestClientBuilder +import org.elasticsearch.client.RestHighLevelClient +import org.elasticsearch.common.unit.TimeValue +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.search.Scroll +import org.elasticsearch.search.SearchHit +import org.elasticsearch.search.builder.SearchSourceBuilder +import org.elasticsearch.xpack.sql.jdbc.EsDataSource + +import java.sql.Connection +import java.sql.ResultSet +import java.sql.Statement +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit +import java.util.function.BiConsumer + +/** + * 功能说明:Elasticsearch工具类 + */ +class DsElasticUtils { + + + static getEsDataSource(String host){ + EsDataSource dataSource = new EsDataSource (); + dataSource.setUrl("jdbc:es://${host}:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000"); + Properties connectionProperties = new Properties(); + connectionProperties.setProperty("user","developer"); + connectionProperties.setProperty("password","5e8edd851d2fdfbd7415232c67367cc3"); + dataSource.setProperties(connectionProperties); + return dataSource; + } + + + static RestHighLevelClient getClient(String host) { + RestClientBuilder builder = RestClient.builder(new HttpHost(host, 9200, "http")); + builder.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() { + @Override + RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder requestConfigBuilder) { + requestConfigBuilder.setConnectionRequestTimeout(1000 * 60); + requestConfigBuilder.setSocketTimeout(1000 * 180); + return requestConfigBuilder; + } + }) + builder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() { + @Override + HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpAsyncClientBuilder) { + httpAsyncClientBuilder.disableAuthCaching(); + CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "goojia123456")); + httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider); + return httpAsyncClientBuilder; + } + }) + return new RestHighLevelClient(builder) + } + + + /** + * 创建一个ES批处理器 + * @param host + * @return + */ + static createBulkProcessor(String host, RestHighLevelClient client = null) { + + BulkProcessor.Listener listener = new BulkProcessor.Listener() { + @Override + void beforeBulk(long executionId, BulkRequest request) { + } + + @Override + void afterBulk(long executionId, BulkRequest request, + BulkResponse response) { + } + + @Override + void afterBulk(long executionId, BulkRequest request, + Throwable failure) { + } + }; + if(!client){ + client = getClient(host); + } + return BulkProcessor.builder(new BiConsumer>() { + @Override + void accept(BulkRequest bulkRequest, ActionListener bulkResponseActionListener) { + client.bulkAsync(bulkRequest, RequestOptions.DEFAULT, bulkResponseActionListener) + } + }, listener).setBulkActions(10000) + .setConcurrentRequests(20) + .setBackoffPolicy(BackoffPolicy.constantBackoff(TimeValue.timeValueSeconds(1L), 3)) + .setFlushInterval(TimeValue.timeValueSeconds(10L)).build(); + } + + /** + * 关闭一个ES批处理器 + * @param + * @return + */ + static closeBulkProcessor(BulkProcessor bulkProcessor) { + bulkProcessor.awaitClose(10, TimeUnit.DAYS) + bulkProcessor.flush() + bulkProcessor.close() + } + + + static executeQuery(String host, String indexName, QueryBuilder qb, Closure closure){ + + RestHighLevelClient client = getClient(host) + BulkProcessor bulkProcessor = createBulkProcessor(host, client) + final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(60L)) + SearchRequest searchRequest = new SearchRequest(indexName); + searchRequest.scroll(scroll); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(qb); + searchRequest.source(searchSourceBuilder); + SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + String scrollId = searchResponse.getScrollId(); + SearchHit[] searchHits = searchResponse.getHits().getHits(); + def nThread = Runtime.getRuntime().availableProcessors() * 5 + ExecutorService executorService = Executors.newFixedThreadPool(nThread) + while (searchHits != null && searchHits.length > 0) { + for (int k=0; k < searchHits.size(); k++) { + final SearchHit hit = searchHits[k]; + executorService.submit(new Runnable() { + @Override + void run() { + try{ + if(closure.getMaximumNumberOfParameters() == 1){ + def update = closure.call(hit.getSourceAsMap()) + if(update){ + bulkProcessor.add(new IndexRequest(indexName).id(hit.getId()).source(hit.getSourceAsMap())) + } + }else if(closure.getMaximumNumberOfParameters() == 2){ + def update = closure.call(hit.getId(), hit.getSourceAsMap()) + if(update){ + bulkProcessor.add(new IndexRequest(indexName).id(hit.getId()).source(hit.getSourceAsMap())) + } + } + }catch (Exception e){ + e.printStackTrace() + } + } + }) + } + SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId); + scrollRequest.scroll(scroll); + searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT); + scrollId = searchResponse.getScrollId(); + searchHits = searchResponse.getHits().getHits(); + } + ClearScrollRequest clearScrollRequest = new ClearScrollRequest(); + clearScrollRequest.addScrollId(scrollId); + client.clearScroll(clearScrollRequest, RequestOptions.DEFAULT); + executorService.shutdown() + executorService.awaitTermination(1, TimeUnit.DAYS) + bulkProcessor.close() + bulkProcessor.awaitClose(1, TimeUnit.DAYS) + client.close() + } + + + static def executeSimpleQuery(String host, String indexName, QueryBuilder qb){ + + final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(60L)) + RestHighLevelClient client = getClient(host) + SearchRequest searchRequest = new SearchRequest(indexName); + searchRequest.scroll(scroll); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(qb); + searchRequest.source(searchSourceBuilder); + SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + String scrollId = searchResponse.getScrollId(); + SearchHit[] searchHits = searchResponse.getHits().getHits(); + def results = [] + while (searchHits != null && searchHits.length > 0) { + for (int k=0; k < searchHits.size(); k++) { + SearchHit searchHit = searchHits[k] + searchHit.getSourceAsMap().put("id", searchHit.getId()); + results << searchHit.getSourceAsMap() + } + SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId); + scrollRequest.scroll(scroll); + searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT); + scrollId = searchResponse.getScrollId(); + searchHits = searchResponse.getHits().getHits(); + } + ClearScrollRequest clearScrollRequest = new ClearScrollRequest(); + clearScrollRequest.addScrollId(scrollId); + client.clearScroll(clearScrollRequest, RequestOptions.DEFAULT); + client.close() + return results; + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy index 320b85cdb..5f2bbe1b3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy @@ -110,7 +110,7 @@ public class BeiKeCrawlByChrome { def durl = e.select(".title a").attr("href") def dealDate = e.select(".dealDate").text() dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd") - def endDate = DateUtils.parseFromTightByPattern("2021-11-30", "yyyy-MM-dd") + def endDate = DateUtils.parseFromTightByPattern("2022-01-20", "yyyy-MM-dd") if(dealDate.getTime()>=endDate.getTime()){ detailUrls << durl }else{ @@ -141,7 +141,7 @@ public class BeiKeCrawlByChrome { def durl = e.select(".title a").attr("href") def dealDate = e.select(".dealDate").text() dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd") - def endDate = DateUtils.parseFromTightByPattern("2021-12-31", "yyyy-MM-dd") + def endDate = DateUtils.parseFromTightByPattern("2022-01-20", "yyyy-MM-dd") if(dealDate.getTime()>=endDate.getTime()){ detailUrls << durl parse(city, e) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy index c6eec9d7e..76719fb79 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -34,10 +34,10 @@ public class LianjiaCrawlByChrome { static main(args) { def task = new LianjiaCrawlByChrome(); -// task.run() + task.run() // task.run1() // task.addDealRegion(); - task.standardRegion() +// task.standardRegion() } def run1(){ @@ -187,7 +187,7 @@ public class LianjiaCrawlByChrome { def durl = e.select(".title a").attr("href") def dealDate = e.select(".dealDate").text() dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd") - def endDate = DateUtils.parseFromTightByPattern("2021-11-30", "yyyy-MM-dd") + def endDate = DateUtils.parseFromTightByPattern("2022-01-30", "yyyy-MM-dd") if(dealDate.getTime()>=endDate.getTime()){ detailUrls << durl }else{ @@ -218,7 +218,7 @@ public class LianjiaCrawlByChrome { def durl = e.select(".title a").attr("href") def dealDate = e.select(".dealDate").text() dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd") - def endDate = DateUtils.parseFromTightByPattern("2021-12-31", "yyyy-MM-dd") + def endDate = DateUtils.parseFromTightByPattern("2022-01-31", "yyyy-MM-dd") if(dealDate.getTime()>=endDate.getTime()){ detailUrls << durl parse(city, e) @@ -419,9 +419,9 @@ public class LianjiaCrawlByChrome { if(cityEles){ for(def e : cityEles){ def href = e.attr("href") - // if(!href.contains("fang.lianjia") && e.text().contains("福州")){ +// if(!href.contains("fang.lianjia") && e.text().contains("福州")){ cityMap.put(e.text(), href) - // } +// } } } return cityMap diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/price/FzMatchTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/price/FzMatchTest.groovy new file mode 100644 index 000000000..06218bbcf --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/price/FzMatchTest.groovy @@ -0,0 +1,85 @@ +package com.fangjia.script.gj.price + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.district.SeaweedDataMatchProcess +import com.fangjia.script.util.ExcelUtils +import groovy.sql.Sql + +class FzMatchTest { + + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + + static main(args) { + def task = new FzMatchTest(); +// task.run() + task.caluatePPE() + } + + def caluatePPE(){ + def filePath = "D:\\数据\\福州\\商品住房楼盘评估指导参考价情况表(估家科技_福州五区_结果1).xlsx" + ExcelUtils.multiThreadingModify(filePath, { data -> + if(data['F'] && data['G']){ + def dist = CommonDatautils.findDistrict("福州", data['F'], data['G'], "district") + if(dist){ + def avg_price = findAvgPrice(dist._id.toString()) + if(data['i'] && avg_price){ + def weekendPrice = data['i'].toString().toDouble() + def ppe15 = Math.abs((weekendPrice - avg_price) * 100 / avg_price) + data['J'] = ppe15 + } + } + } + },1) + } + + def findAvgPrice(def commid){ + def sql = "select year_month,target_avg_price from (select year_month,target_avg_price from avm_evaluation_item_gj where commid='${commid}' order by year_month desc) where rownum<2"; + def avg_price = null + sqlInstance.eachRow(sql, { r -> + avg_price = r.target_avg_price + }); + return avg_price + } + + def run(){ + def filePath = "D:\\数据\\福州\\商品住房楼盘评估指导参考价情况表(估家科技_福州五区).xlsx" + def city = "福州" + ExcelUtils.multiThreadingModify(filePath, { data -> +// if(data['G']){ +// if(!data['H']){ +// def dist = CommonDatautils.findDistrict(city, data['F'], data['G'], "district") +// +// if(dist && dist.finish_building_date){ +// data['H'] = dist.finish_building_date.substring(0,4) +// } +// } +// if(!data['I']){ +// def price = CommonDatautils.queryDistrictPrice(city, data['F'], data['G']) +// if(price){ +// data['I'] = price +// } +// } +// +// } + + if(!data['G'] && data['B']){ + def name = data['B'].replace("(", "").replace(")","").replace("\\","").replace("#","") + def rMap = SeaweedDataMatchProcess.doMatch(city, null, name, null); + if(rMap) { + data['F'] = rMap.region + data['G'] = rMap.name + def dist = CommonDatautils.findDistrictById(rMap.id) + if(dist && dist.finish_building_date){ + data['H'] = dist.finish_building_date.substring(0,4) + } + + def price = CommonDatautils.queryDistrictPrice(city, rMap.region, rMap.name) + if(price){ + data['I'] = price + } + } + } + }, 1); + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy index 7f04edab1..5443a7a6f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy @@ -40,11 +40,11 @@ class DataStat { def run(){ - def source = "城市房产"// - def citys = findCrawlCity(source, 202112); + def source = "58同城"// + def citys = findCrawlCity(source, 202201); for(def city : citys){ - def count1 = findCount(source, city, 202112); - def count2 = findCount(source, city, 202201); + def count1 = findCount(source, city, 202201); + def count2 = findCount(source, city, 202202); println city +" "+count1+" "+count2 // println count1 // println "--- ----" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy index 5fb2fef63..69aaa9179 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy @@ -181,7 +181,147 @@ class Esf58Crawl extends BaseBatch{ // "平湖":"pinghushi|501","海盐":"haiyan|504","武义县":"wuyix|14528","嵊州":"shengzhou|359","新昌":"xinchang|361", // "江山":"jiangshanshi|6796","平阳":"pingyangxian|7575" def cityList = [ - "福州":"fz|304" + "福州":"fz|304","蚌埠":"bengbu|3470","淮南":"hn|2319","青岛":"qd|122","广州":"gz|3","天津":"tj|18","南京":"nj|172", "嘉兴":"jx|497","大连":"dl|147","武汉":"wh|158", + "兰州":"lz|952","合肥":"hf|837", "乌鲁木齐":"xj|984","南昌":"nc|669","沈阳":"sy|188","西安":"xa|483","苏州":"su|5","无锡":"wx|93","潍坊":"wf|362", + "芜湖":"wuhu|2045","东莞":"dg|413","常州":"cz|463","佛山":"fs|222", + "南通":"nt|394","东莞":"dg|413","吉林":"jl|700","中山":"zs|771","阜阳":"fy|2325", + "呼和浩特":"hu|811", + "济南":"jn|265","温州":"wz|330","台州":"tz|403","东营":"dy|623","咸阳":"xianyang|7453","安庆":"anqing|3251","宿州":"suzhou|3359","六安":"la|2328","唐山":"ts|276","商丘":"sq|1029","大同":"dt|6964","威海":"weihai|518","安阳":"ay|1096", + "临汾":"linfen|5669","临沂":"linyi|505","九江":"jj|2247","淄博":"zb|385","北京":"bj|1","上海":"sh|2", + "三亚":"sanya|2422","烟台":"yt|228","宁波":"nb|135","金华":"jh|531", "绍兴":"sx|355","湖州":"huzhou|831","张家口":"zjk|3328","珠海":"zh|910","海口":"haikou|2053","保定":"bd|424","保亭":"baoting|10367","南宁":"nn|845", "扬州":"yz|637","汕头":"st|783","泰州":"taizhou|693","淮安":"ha|968","湛江":"zhanjiang|791","锦州":"jinzhou|2354","昆明":"km|541","柳州":"liuzhou|7133","肇庆":"zq|901","连云港":"lyg|2049","日照":"rizhao|3177","常德":"changde|872", + "沧州":"cangzhou|652", "泉州":"qz|291","清远":"qingyuan|7303", "西宁":"xn|2052","遵义":"zunyi|7620","株洲":"zhuzhou|1086","贵阳":"gy|2015","岳阳":"yy|821","徐州":"xz|471", "哈尔滨":"hrb|202","宜昌":"yc|858", "廊坊":"lf|772","秦皇岛":"qhd|1078","惠州":"huizhou|722","洛阳":"luoyang|556", + "深圳":"sz|4","石家庄":"sjz|241","太原":"ty|740","成都":"cd|102","杭州":"hz|79","重庆":"cq|37","盐城":"yancheng|613","信阳":"xy|8694","南充":"nanchong|2378","南阳":"ny|592","厦门":"xm|606", + "绵阳":"mianyang|1057","济宁":"jining|450","邯郸":"hd|572", + "淮北":"huaibei|9357","滁州":"chuzhou|10266", + "马鞍山":"mas|2039","铜陵":"tongling|10285","宣城":"xuancheng|5633","亳州":"bozhou|2329","黄山":"huangshan|2323", + "池州":"chizhou|10260","巢湖":"ch|10229","和县":"hexian|10892","霍邱":"hq|11226","桐城":"tongcheng|11296", + "宁国":"ningguo|5645","天长":"tianchang|10273","东至":"dongzhi|10262","无为":"wuweixian|10232", + "莆田":"pt|2429","漳州":"zhangzhou|710","宁德":"nd|7951","三明":"sm|2048","南平":"np|10291","龙岩":"ly|6752","武夷山":"wuyishan|10761", + "石狮":"shishi|296","晋江":"jinjiangshi|297","南安":"nananshi|293","龙海":"longhai|713","上杭":"shanghangxian|6757", + "福安":"fuanshi|7969","福鼎":"fudingshi|7970","安溪":"anxixian|7100","永春":"yongchunxian|7101","永安":"yongan|2133", + "漳浦":"zhangpu|717", "珠海":"zh|910","江门":"jm|629", + "茂名":"mm|679","揭阳":"jy|927","梅州":"mz|9389", + "阳江":"yj|2284","韶关":"sg|2192","河源":"heyuan|10467","云浮":"yf|10485","汕尾":"sw|9449", + "潮州":"chaozhou|10461","台山":"taishan|11263","阳春":"yangchun|8566","顺德":"sd|8716","惠东":"huidong|725", + "博罗":"boluo|726","海丰":"haifengxian|9444","开平":"kaipingshi|634","陆丰":"lufengshi|9456", + "桂林":"gl|1039","玉林":"yulin|2337","梧州":"wuzhou|2046", + "北海":"bh|10536","贵港":"gg|6770","钦州":"qinzhou|2335","百色":"baise|10513","河池":"hc|2340", + "来宾":"lb|10552","贺州":"hezhou|10549","防城港":"fcg|10539","崇左":"chongzuo|10524","桂平":"guipingqu|6774", + "北流":"beiliushi|9168","博白":"bobaixian|9173","岑溪":"cenxi|2119", + "黔东南":"qdn|9363","黔南":"qn|10492","六盘水":"lps|10506", + "毕节":"bijie|10564","铜仁":"tr|10417","安顺":"anshun|7468","黔西南":"qxn|10434","仁怀":"renhuaishi|7628", + "清镇":"qingzhen|12703","天水":"tianshui|8601","白银":"by|10304","庆阳":"qingyang|10475","平凉":"pl|7154", + "酒泉":"jq|10387","张掖":"zhangye|10454","武威":"wuwei|10448","定西":"dx|10322","金昌":"jinchang|7428", + "陇南":"ln|10415","临夏":"linxia|7112","嘉峪关":"jyg|10362","甘南":"gn|10343","敦煌":"dunhuang|10390", + "五指山":"wzs|9952","三沙":"sansha|13722","琼海":"qh|10136", + "文昌":"wenchang|9984","万宁":"wanning|10022","屯昌":"tunchang|10044","琼中":"qiongzhong|10064","陵水":"lingshui|10184", + "东方":"df|10250","定安":"da|10303","澄迈":"cm|10331","白沙":"baish|10380", + "儋州":"danzhou|10394", "郑州":"zz|342","新乡":"xx|1016","许昌":"xc|977", + "平顶山":"pds|1005","焦作":"jiaozuo|3266","开封":"kaifeng|2342", + "濮阳":"puyang|2346","周口":"zk|933","驻马店":"zmd|1067","漯河":"luohe|2347", + "三门峡":"smx|9317","鹤壁":"hb|2344","济源":"jiyuan|9918","明港":"mg|8541","鄢陵":"yanling|9123", + "禹州":"yuzhou|979","长葛":"changge|9344","灵宝":"lingbaoshi|9307","杞县":"qixianqu|7389","汝州":"ruzhou|1010", + "项城":"xiangchengshi|935","偃师":"yanshiqu|7121","长垣":"changyuan|5936","滑县":"huaxian|5405","林州":"linzhou|1101", + "沁阳":"qinyang|3268","孟州":"mengzhou|3267","温县":"wenxian|7312","尉氏":"weishixian|7391","兰考":"lankaoxian|7393", + "通许":"tongxuxian|7390","新安":"lyxinan|11217","伊川":"yichuan|11220","孟津":"mengjinqu|7122","宜阳":"lyyiyang|11219", + "舞钢":"wugang|1011","永城":"yongcheng|1032","睢县":"suixian|1038","鹿邑":"luyi|939","渑池":"yingchixian|9322", + "沈丘":"shenqiu|942","太康":"taikang|938","商水":"shangshui|936","淇县":"qixianq|9186","浚县":"junxian|9185", + "范县":"fanxian|7285","固始":"gushixian|8698","淮滨":"huaibinxian|8702","邓州":"dengzhou|595","新野":"xinye|603", + "大庆":"dq|375","齐齐哈尔":"qqhr|5853","牡丹江":"mdj|3489","绥化":"suihua|6718", + "佳木斯":"jms|6776","鸡西":"jixi|7289","双鸭山":"sys|9837","鹤岗":"hegang|9061","黑河":"heihe|9862", + "伊春":"yich|9773","七台河":"qth|9848","大兴安岭":"dxal|9878","安达":"shanda|6720","肇东":"shzhaodong|6721", + "肇州":"zhaozhou|382", "襄阳":"xf|891","荆州":"jingzhou|3479","十堰":"shiyan|2032", + "黄石":"hshi|1734","孝感":"xiaogan|3434","黄冈":"hg|2299","恩施":"es|2302","荆门":"jingmen|2296", + "咸宁":"xianning|9617","鄂州":"ez|9709","随州":"suizhou|9656","潜江":"qianjiang|9669","天门":"tm|9517", + "仙桃":"xiantao|9736","神农架":"snj|9605","宜都":"yidou|864","汉川":"hanchuan|3439","枣阳":"zaoyang|896", + "武穴":"wuxueshi|7362","钟祥":"zhongxiangshi|9119","京山":"jingshanxian|9117","沙洋":"shayangxian|9118","松滋":"songzi|3484", + "广水":"guangshuishi|9657","赤壁":"chibishi|9623","老河口":"laohekou|895","谷城":"gucheng|899","宜城":"yichengshi|897", + "南漳":"nanzhang|898","云梦":"yunmeng|3438","安陆":"anlu|3442","大悟":"dawu|3437","孝昌":"xiaochang|3436", + "当阳":"dangyang|865","枝江":"zhijiang|866","嘉鱼":"jiayuxian|9624","随县":"suixia|9660", + "长沙":"cs|414","益阳":"yiyang|10198","衡阳":"hy|914", + "湘潭":"xiangtan|2047","郴州":"chenzhou|5695","邵阳":"shaoyang|2303","怀化":"hh|5756", + "永州":"yongzhou|2307","娄底":"ld|9481","湘西":"xiangxi|10219","张家界":"zjj|6788","醴陵":"liling|1091", + "澧县":"lixian|876","桂阳":"czguiyang|5699","资兴":"zixing|5698","永兴":"yongxing|5701","常宁":"changningshi|921", + "祁东":"qidongxian|5690","衡东":"hengdong|5693","冷水江":"lengshuijiangshi|9470","涟源":"lianyuanshi|9471","双峰":"shuangfengxian|9473", + "邵阳县":"shaoyangxian|6955","邵东":"shaodongxian|6954","沅江":"yuanjiangs|10201","南县":"nanxian|10202","祁阳":"qiyang|8532", + "湘阴":"xiangyin|828","华容":"huarong|830","慈利":"cilixian|6791","攸县":"zzyouxian|1095", + "邢台":"xt|751","衡水":"hs|993", + "承德":"chengde|6760","定州":"dingzhou|8398","馆陶":"gt|8706","张北":"zhangbei|11201","赵县":"zx|9048", + "正定":"zd|3198","迁安市":"qianan|284","任丘":"renqiu|656","三河":"sanhe|776","武安":"wuan|577", + "雄安新区":"xionganxinqu|111234","燕郊":"lfyanjiao|12804","涿州":"zhuozhou|428","河间":"hejian|658","黄骅":"huanghua|657", + "沧县":"cangxian|659","磁县":"cixian|591","涉县":"shexian|14059","霸州":"bazhou|775","香河":"xianghe|5395", + "固安":"lfguan|12803","遵化市":"zunhua|283","迁西":"qianxixian|7061","玉田":"yutianxian|7060","滦南":"luannanxian|7066", + "沙河":"shaheshi|755", + "宿迁":"suqian|2350","镇江":"zj|645","沭阳":"shuyang|5772","大丰":"dafeng|11279", + "如皋":"rugao|397","启东":"qidong|400","溧阳":"liyang|469","海门":"haimen|399","东海":"donghai|2147", + "扬中":"yangzhong|649","兴化":"xinghuashi|699","新沂":"xinyishi|478","泰兴":"taixing|696","如东":"rudong|402", + "邳州":"pizhou|477","沛县":"xzpeixian|11349","靖江":"jingjiang|698","建湖":"jianhu|618","海安":"haian|401", + "东台":"dongtai|615","丹阳":"danyang|648","宝应县":"baoyingx|14451","灌南":"guannan|2150","灌云":"guanyun|2148", + "姜堰":"jiangyan|697","金坛":"jintan|468","昆山":"szkunshan|16","泗洪":"sihong|5958","泗阳":"siyang|5959", + "句容":"jurong|650","射阳":"sheyang|621","阜宁":"funingxian|620","响水":"xiangshui|619","盱眙":"xuyi|976", + "金湖":"jinhu|975","江阴":"jiangyins|34984", "赣州":"ganzhou|2363","宜春":"yichun|5709","吉安":"ja|2364", + "上饶":"sr|10120","萍乡":"px|2248","抚州":"fuzhou|10134","景德镇":"jdz|2360","新余":"xinyu|10115", + "鹰潭":"yingtan|3209","永新":"yxx|11077","乐平":"lepingshi|9048","进贤":"jinxian|677","分宜":"fenyi|10118", + "丰城":"fengchengshi|5711","樟树":"zhangshu|5713","高安":"gaoan|5712","余江":"yujiang|3210","南城":"nanchengx|10137", + "浮梁":"fuliangxian|9071", + "长春":"cc|319","四平":"sp|10171","延边":"yanbian|3184","松原":"songyuan|2315", + "白城":"bc|5918","通化":"th|10159","白山":"baishan|10179","辽源":"liaoyuan|2501","公主岭":"gongzhuling|10171", + "梅河口":"meihekou|10162","扶余":"fuyuxian|9085","长岭":"changlingxian|9084","桦甸":"huadian|706","磐石":"panshi|708", + "梨树县":"lishu|10176", + "鞍山":"as|523","抚顺":"fushun|5722", + "营口":"yk|5898","盘锦":"pj|2041","朝阳":"cy|10106","丹东":"dandong|3445","辽阳":"liaoyang|2038", + "本溪":"benxi|5845","葫芦岛":"hld|10088","铁岭":"tl|6729","阜新":"fx|10097","庄河":"pld|3306", + "瓦房店":"wfd|3279","灯塔":"dengta|2071","凤城":"fengcheng|3450","北票":"beipiao|10109","开原":"kaiyuan|6733", + "银川":"yinchuan|2054","吴忠":"wuzhong|9962","石嘴山":"szs|9971","中卫":"zw|9951","固原":"guyuan|2421", + "包头":"bt|801","赤峰":"chifeng|6700","鄂尔多斯":"erds|2037","通辽":"tongliao|10015", + "呼伦贝尔":"hlbe|10039","巴彦淖尔市":"bycem|10070","乌兰察布":"wlcb|9993","锡林郭勒":"xl|2408","兴安盟":"xam|9976", + "乌海":"wuhai|2404","阿拉善盟":"alsm|10083","海拉尔":"hlr|2043", + "海西":"hx|9902","海北":"haibei|9917","果洛":"guoluo|9936","海东":"haidong|9909", + "黄南":"huangnan|9896","玉树":"ys|9888","海南":"hainan|10574","格尔木":"geermushi|9904", + "泰安":"ta|686","聊城":"lc|882", + "枣庄":"zaozhuang|961","德州":"dz|728","菏泽":"heze|5632", + "滨州":"bz|944","莱芜":"lw|2292","章丘":"zhangqiu|8680","垦利":"kl|11313","诸城":"zc|9146", + "寿光":"shouguang|369","龙口":"longkou|233","曹县":"caoxian|5638","单县":"shanxian|5636","肥城":"feicheng|690", + "高密":"gaomi|371","广饶":"guangrao|627","桓台":"huantaixian|7335","莒县":"juxian|3180","莱州":"laizhou|235", + "蓬莱":"penglai|237","青州":"qingzhou|367","荣成":"rongcheng|522","乳山":"rushan|520","滕州":"tengzhou|967", + "新泰":"xintai|689","招远":"zhaoyuan|3325","邹城":"zoucheng|455","邹平":"zouping|946","临清":"linqing|884", + "茌平":"chiping|887","郓城":"hzyc|5637","博兴":"boxing|949","东明":"dongming|5641","巨野":"juye|5640", + "无棣":"wudi|951","齐河":"qihe|734","微山":"weishan|459","禹城":"yuchengshi|731","临邑":"linyixianq|739", + "乐陵":"leling|730","莱阳":"laiyang|234","宁津":"ningjin|733","高唐":"gaotang|885","莘县":"shenxian|888", + "阳谷":"yanggu|886","冠县":"guanxian|890","平邑":"pingyi|514","郯城":"tancheng|510","沂源":"yiyuanxian|7334", + "汶上":"wenshang|460","梁山":"liangshanx|462","利津":"lijin|628","沂南":"yinanxian|7301","栖霞":"qixia|238", + "宁阳":"ningyang|691","东平":"dongping|692","昌邑":"changyishi|372","安丘":"anqiu|370","昌乐":"changle|373", + "临朐":"linqu|374","鄄城":"juancheng|5635", + "运城":"yuncheng|5653","晋中":"jz|8854", + "长治":"changzhi|6921","晋城":"jincheng|3350","阳泉":"yq|8760","吕梁":"lvliang|3222","忻州":"xinzhou|3453", + "朔州":"shuozhou|9871","临猗":"linyixian|9193","清徐":"qingxu|10908","柳林":"liulin|3225","高平":"gaoping|3354", + "泽州":"zezhou|3353","襄垣":"xiangyuanxian|6928","孝义":"xiaoyi|3227", + "宝鸡":"baoji|2044","渭南":"wn|5733","汉中":"hanzhong|3163", + "榆林":"yl|5942","延安":"yanan|8973","安康":"ankang|3157","商洛":"sl|9854","铜川":"tc|9832", + "神木":"shenmu|5944","韩城":"hancheng|5735","府谷":"fugu|5945","靖边":"jingbian|5947","定边":"dingbian|5948", + "德阳":"deyang|2373","宜宾":"yb|2380", + "自贡":"zg|6745","乐山":"ls|3237","泸州":"luzhou|2372","达州":"dazhou|9799","内江":"scnj|5928", + "遂宁":"suining|9688","攀枝花":"panzhihua|2371","眉山":"ms|9704","广安":"ga|2381","资阳":"zy|6803", + "凉山":"liangshan|9717","广元":"guangyuan|9749","雅安":"ya|9687","巴中":"bazhong|9811","阿坝":"ab|9817", + "甘孜":"ganzi|9764","安岳":"anyuexian|6806","广汉":"guanghanshi|8719","简阳":"jianyangshi|6805","仁寿":"renshouxian|9706", + "射洪":"shehongxian|9694","大竹":"dazu|9806","宣汉":"xuanhan|9804","渠县":"qux|9807","长宁":"changningx|7148", + "昌吉":"changji|8582","巴音郭楞":"bygl|9530","伊犁":"yili|9472","阿克苏":"aks|9499", + "喀什":"ks|9326","哈密":"hami|7452","克拉玛依":"klmy|2042","博尔塔拉":"betl|9529","吐鲁番":"tlf|9475", + "和田":"ht|9489","石河子":"shz|9551","克孜勒苏":"kzls|9519","阿拉尔":"ale|9539","五家渠":"wjq|9562", + "图木舒克":"tmsk|9559","库尔勒":"kel|7168","阿勒泰":"alt|18837","塔城":"tac|18845", + "拉萨":"lasa|2055","日喀则":"rkz|9615","山南":"sn|9576","林芝":"linzhi|9646","昌都":"changdu|9648", + "那曲":"nq|9618","阿里":"al|9678","日土":"rituxian|9682","改则":"gaizexian|9684", + "曲靖":"qj|2389","大理":"dali|2398","红河":"honghe|2394","玉溪":"yx|2040", + "丽江":"lj|2392","文山":"ws|2395","楚雄":"cx|2393","西双版纳":"bn|2397","昭通":"zt|9409", + "德宏":"dh|9437","普洱":"pe|9444","保山":"bs|2390","临沧":"lincang|9422","迪庆":"diqing|9432", + "怒江":"nujiang|9462","弥勒":"milexian|8892","安宁":"anningshi|547","宣威":"xuanwushi|7533", + "丽水":"lishui|7921","衢州":"quzhou|6793", + "舟山":"zhoushan|8481","乐清":"yueqingcity|13950","瑞安":"ruiancity|13951","义乌":"yiwu|12291","余姚":"yuyao|5333", + "诸暨":"zhuji|357","象山":"xiangshanxian|6738","温岭":"wenling|408","桐乡":"tongxiang|502","慈溪":"cixi|5334", + "长兴":"changxing|834","嘉善":"jiashanx|14357","海宁":"haining|500","德清":"deqing|835","东阳":"dongyang|536", + "安吉":"anji|836","苍南":"cangnanxian|7579","临海":"linhai|407","永康":"yongkang|537","玉环":"yuhuan|409", + "平湖":"pinghushi|501","海盐":"haiyan|504","武义县":"wuyix|14528","嵊州":"shengzhou|359","新昌":"xinchang|361", + "江山":"jiangshanshi|6796","平阳":"pingyangxian|7575" ] static main(args) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy index 0b75f8545..2d055dcbb 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy @@ -15,10 +15,10 @@ class SynchDistrictPrice { def static final SHORT_DATE = Utils.getLastMonth() def thirdPartyColl = CrawalDBUtil.getThirdPartyCol(); def districtDetailColl = CrawalDBUtil.getDistrictDetailColl(); -// def sourceMap = ["jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多","house365.com":"365淘房", +// def sourceMap = ["fang.com":"搜房","jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fangdd.com":"房多多","house365.com":"365淘房", // "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家"] //"fang.com":"搜房", - def sourceMap = ["ke.com":"贝壳找房","cityhouse.cn":"城市房产"] + def sourceMap = ["cityhouse.cn":"城市房产"] static main(args) { def task = new SynchDistrictPrice() task.run() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy index ed17686b9..db48455df 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy @@ -14,8 +14,8 @@ class TestCrawlTotalCount { def run(){ def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"] for(def source :sources){ - def count1 = findTotalCount(source, 202112); - def count2 = findTotalCount(source, 202201); + def count1 = findTotalCount(source, 202201); + def count2 = findTotalCount(source, 202202); println source +" "+count1 +" "+count2 } } From fbef302f79dfe7735a7ff88abad978a9e3ec58ec Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 9 Mar 2022 13:31:39 +0800 Subject: [PATCH 5/5] update --- .../fangjia/script/jiangnanblank/SFTP.groovy | 51 ++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/SFTP.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/SFTP.groovy index 811de10a1..70082d882 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/SFTP.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/SFTP.groovy @@ -42,6 +42,55 @@ class SFTP { return sftp; } + + + /** + * 删除本地文件 + * + * @param filePath + * @return + */ + public boolean deleteFile(String filePath) { + File file = new File(filePath); + if (!file.exists()) { + return false; + } + if (!file.isFile()) { + file.deleteDir(); + return false; + } + boolean rs = file.delete(); + return rs; + } + + public static void deleteSFTP(String directory, ChannelSftp sftp) { + try { + if (isDirExist(directory, sftp)) { + Vector vector = sftp.ls(directory); + if (vector.size() == 1) { // 文件,直接删除 + sftp.rm(directory); + } else if (vector.size() == 2) { // 空文件夹,直接删除 + sftp.rmdir(directory); + } else { + String fileName = ""; + // 删除文件夹下所有文件 + for (ChannelSftp.LsEntry en : vector) { + fileName = en.getFilename(); + if (".".equals(fileName) || "..".equals(fileName)) { + continue; + } else { + deleteSFTP(directory + "/" + fileName); + } + } + // 删除文件夹 + sftp.rmdir(directory); + } + } + } catch (Exception e) { + e.printStackTrace(); + } + } + /** * 上传文件 * @param directory 上传的目录 @@ -132,7 +181,7 @@ class SFTP { sftp.cd(filePath.toString()); } else { // 建立目录 - sftp.mkdir(filePath.toString()); + sftp.mkdir(filePath.toString()) // 进入并设置为当前目录 sftp.cd(filePath.toString()); }