From a437fe2240cbd9c36da257c3f527f679b1893bd7 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 15 Jan 2024 11:00:29 +0800 Subject: [PATCH 01/22] =?UTF-8?q?=E5=86=B2=E7=AA=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../main/groovy/com/fangjia/script/util/ExcelUtils.groovy | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy index 5c155319f..d6ef502f2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy @@ -454,12 +454,7 @@ public class ExcelUtils{ def data = ExcelUtils.read(fileName, srcSheetIndex, 0) def body = data.subList(1, data.size()) -<<<<<<< HEAD - def nThread = 1 -======= - def nThread = 10 ->>>>>>> f0061ade19d549f26467ab6be7e9eaaa66cd3011 - println nThread + ExecutorService executorService = Executors.newFixedThreadPool(3) final AtomicInteger counter = new AtomicInteger(0) final double total = body.size() From 6c88842f3611a52297ae83af756f9b8826b697e9 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 28 Feb 2024 16:20:23 +0800 Subject: [PATCH 02/22] =?UTF-8?q?=E5=8F=91=E9=82=AE=E4=BB=B6=E6=94=B9?= =?UTF-8?q?=E4=B8=BAwangwenyan@fangjia.com?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../groovy/com/fangjia/script/Watcher.groovy | 51 +++++++-------- .../script/dailyCount/ApiDailyCount.groovy | 4 +- .../fangjia/script/gj/YouDaDataParser.groovy | 4 +- .../script/gj/village/GuangdaTest.groovy | 65 +++++++++++++++---- .../gj/xiamen/GaoDePoiDataProcess.groovy | 13 ++-- .../thirdPriceCrawl/SynchDistrictPrice.groovy | 4 +- .../ApiDailyCountOfficialAccount.groovy | 4 +- 7 files changed, 92 insertions(+), 53 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/Watcher.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/Watcher.groovy index 004d6bd4a..40fe526e8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/Watcher.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/Watcher.groovy @@ -2,36 +2,35 @@ package com.fangjia.script public enum Watcher { //离职同事 - LiuPeiPei('13621626685', 'duxiaoyu@fangjia.com'), - YanZhou('13621626685','duxiaoyu@fangjia.com'), - HuangXiaoHu('13621626685', 'duxiaoyu@fangjia.com'), - SongYingXia('13621626685', 'duxiaoyu@fangjia.com'), - YuKai('13621626685', 'duxiaoyu@fangjia.com'), - RuanYaDong('13621626685', 'duxiaoyu@fangjia.com'), - Reda('13621626685', 'duxiaoyu@fangjia.com'), - Tomzhao('13621626685','duxiaoyu@fangjia.com'), - YuHai('13621626685','duxiaoyu@fangjia.com'), - Shibb('13621626685','duxiaoyu@fangjia.com'), - HeLvYuan('13621626685','duxiaoyu@fangjia.com'), - WangYuHui('13621626685','duxiaoyu@fangjia.com'), - LinXiaoShuai('15801810393','linxiaoshuai@fangjia.com'), - FangRui('17717680589','fangrui@fangjia.com'), - //DuXiaoYu('13621626685', 'duxiaoyu@fangjia.com'), + LiuPeiPei('13621626685', 'wangwenyan@fangjia.com'), + YanZhou('13621626685','wangwenyan@fangjia.com'), + HuangXiaoHu('13621626685', 'wangwenyan@fangjia.com'), + SongYingXia('13621626685', 'wangwenyan@fangjia.com'), + YuKai('13621626685', 'wangwenyan@fangjia.com'), + RuanYaDong('13621626685', 'wangwenyan@fangjia.com'), + Reda('13621626685', 'wangwenyan@fangjia.com'), + Tomzhao('13621626685','wangwenyan@fangjia.com'), + YuHai('13621626685','wangwenyan@fangjia.com'), + Shibb('13621626685','wangwenyan@fangjia.com'), + HeLvYuan('13621626685','wangwenyan@fangjia.com'), + WangYuHui('13621626685','wangwenyan@fangjia.com'), + LinXiaoShuai('15801810393','wangwenyan@fangjia.com'), + FangRui('17717680589','wangwenyan@fangjia.com'), + DuXiaoYu('13621626685', 'wangwenyan@fangjia.com'), //在职同事 - DuXiaoYu('13916076378', 'wangwenyan@fangjia.com'), - LiXue('13311649646','lixue@fangjia.com'), + LiXue('13311649646','wangwenyan@fangjia.com'), WangWenYan('13916076378','wangwenyan@fangjia.com'), GaoJuan('15021384995','gaojuan@fangjia.com'), - Hubingliang('15179856208','hubingliang@126.com'), - ZhouTongYun('13585834921','zhoutongyun@fangjia.com'), - YinJiHuan('18217642009','yinjihuan@fangjia.com'), - Xuhe('13862416422','xuhe@fangjia.com'), - ZongZhiMin('18260046486','zongzhimin@fangjia.com'), - RaoYanHui('18516539752','raoyanhui@fangjia.com'), - BaYi('15300424133','bayi@fangjia.com'), - WeiLei('18101751850','weilei@fangjia.com'), - JiangYaHui('13684913780','jiangyahui@fangjia.com'), + Hubingliang('15179856208','wangwenyan@fangjia.com'), + ZhouTongYun('13585834921','wangwenyan@fangjia.com'), + YinJiHuan('18217642009','wangwenyan@fangjia.com'), + Xuhe('13862416422','wangwenyan@fangjia.com'), + ZongZhiMin('18260046486','wangwenyan@fangjia.com'), + RaoYanHui('18516539752','wangwenyan@fangjia.com'), + BaYi('15300424133','wangwenyan@fangjia.com'), + WeiLei('18101751850','wangwenyan@fangjia.com'), + JiangYaHui('13684913780','wangwenyan@fangjia.com'), ShenJian('13764369683', 'shenjian@fangjia.com') private final String mobile; private final String email; diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/dailyCount/ApiDailyCount.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/dailyCount/ApiDailyCount.groovy index 6212900b5..a0ed5c281 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/dailyCount/ApiDailyCount.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/dailyCount/ApiDailyCount.groovy @@ -27,8 +27,8 @@ class ApiDailyCount extends BaseBatch{ private static final Jms jms = Jms.create('tcp://monitor.jms.db.fangjia.com:61616') def static emails = [ - 'xuqi@fangjia.com', - 'zongzhimin@fangjia.com' + 'wangwenyan@fangjia.com', + 'gaojuan@fangjia.com' ] static void main(String[] args){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy index 621b548e7..c40aca736 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy @@ -52,9 +52,9 @@ class YouDaDataParser { static main(args) { def task = new YouDaDataParser(); // task.newHouseparse();//友达一手房存入mongo - task.esfParser("D:\\数据\\友达\\2024年1月\\查询1.DBF");//友达二手房存入mongo +// task.esfParser("D:\\数据\\友达\\2024年2月\\查询1.DBF");//友达二手房存入mongo // task.newhouseToSaveEs();//新房保存到es -// task.esfhouseToSaveEs();//二手房保存到es + task.esfhouseToSaveEs();//二手房保存到es // task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间 } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/GuangdaTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/GuangdaTest.groovy index 7d683c976..1196b6bd2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/GuangdaTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/GuangdaTest.groovy @@ -18,7 +18,44 @@ DBCollection development_zone_CONN = MongoUtils.getCollectionByName2("fangjia", DBCollection community_CONN = MongoUtils.getCollectionByName2("fangjia", "community", MONGO100_URL) -def filePath = "D:\\数据\\光大\\全量.xlsx" +def filePath = "D:\\数据\\南京\\南京银行测试-新跑.xlsx" +ExcelUtils.multiThreadingModify(filePath, { row -> + def communityObj = null + if(row['E']){ + communityObj = findCommunityTown(community_CONN, row['C'], row['D'], row['E']) + } + if(communityObj){ + row['M'] = communityObj.province_code + row['N'] = communityObj.city_code + row['O'] = communityObj.area_code + row['P'] = communityObj.street_code + } + if(row['D']){ + if(row['D'].toString().endsWith("区")){ + row['F'] = "非农" + }else{ + if(communityObj){ + if(communityObj.has_gov && communityObj.has_gov=="是"){ + row['I'] = "是" + row['F'] = "非农" + }else{ + row['I'] = "否" + if(row['E']){ + def townObj = isTown(row['C'], row['D'], row['E']) + if(townObj){ + row['J'] = "是" + row['F'] = "非农" + }else{ + row['F'] = "农" + } + } + } + } + } + } + +}); +//def filePath = "D:\\数据\\光大\\全量.xlsx" //ExcelUtils.modify(filePath, { row -> // if(!row['O']){ // def jsonObj = FangjiaClientUtils.villageIdentify(null, row['A']) @@ -62,19 +99,19 @@ def filePath = "D:\\ // } // } - def retMap = SignatureUtils.getAliAddressStrc(row['A']) - println row['A'] +" " + retMap.toString() - if(retMap){ - row['Y'] = retMap.prov - row['Z'] = retMap.city - row['AA'] = retMap.district - row['AB'] = retMap.town - row['AC'] = retMap.devzone - row['AD'] = retMap.road - row['AE'] = retMap.roadno - row['AF'] = retMap.poi - row['AG'] = retMap.subpoi - } +// def retMap = SignatureUtils.getAliAddressStrc(row['A']) +// println row['A'] +" " + retMap.toString() +// if(retMap){ +// row['Y'] = retMap.prov +// row['Z'] = retMap.city +// row['AA'] = retMap.district +// row['AB'] = retMap.town +// row['AC'] = retMap.devzone +// row['AD'] = retMap.road +// row['AE'] = retMap.roadno +// row['AF'] = retMap.poi +// row['AG'] = retMap.subpoi +// } // }) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy index d5d8303d5..cfda2228d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy @@ -66,8 +66,10 @@ class GaoDePoiDataProcess extends BaseBatch{ @Override void process(Object args) { // println ProvinceData.getProvince("成都") - synchMapPoi(); -// synchCompany() +// synchMapPoi(); + + + synchCompany() // poiCatProcess() // addressAndCatProcess() // synchNetwork() @@ -99,10 +101,10 @@ class GaoDePoiDataProcess extends BaseBatch{ def synchCompany(){ fullCityMap = CommonDatautils.queryFullNameCity() //"三明市","宁德市","泉州市","厦门市","福州市","漳州市","南平市","龙岩市","莆田市", - def citys = ["湖州市"] + def citys = ["杭州","宁波","温州","嘉兴","台州","绍兴","金华","舟山","衢州","丽水"] for(def sampleCity : citys){ - def cityObj = fullCityMap.get("湖州"); + def cityObj = fullCityMap.get(sampleCity); def city = null if(cityObj){ city = cityObj.city @@ -112,6 +114,7 @@ class GaoDePoiDataProcess extends BaseBatch{ println city+ " 加载公司数据:"+companyDatas.size() def rows = running(city, companyDatas, "company", townMap) save(city, rows) +// saveToMG(city, rows) println "完成" } } @@ -182,7 +185,7 @@ class GaoDePoiDataProcess extends BaseBatch{ def createCompanyData(map, townMap){ def retMap = [:] - retMap._id = map._id + retMap.data_id = map._id.toString() retMap.fjw_province = map.province_name retMap.fjw_city = map.city_name diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy index 7c9ed802c..33e407462 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy @@ -26,8 +26,8 @@ class SynchDistrictPrice extends BaseBatch{ // "吉屋":"吉屋","365淘房":"365淘房","我爱我家":"我爱我家","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","麦田":"麦田", // "中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产","兔博士PC端":"兔博士" -// def esSources = ["我爱我家":"我爱我家","兔博士PC端":"兔博士","吉屋":"吉屋","贝壳找房":"贝壳找房","乐有家":"乐有家","365淘房":"365淘房","诸葛找房APP":"诸葛找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","C21":"C21","城市房产":"城市房产"] - def esSources = ["我爱我家":"我爱我家","兔博士PC端":"兔博士","吉屋":"吉屋","贝壳找房":"贝壳找房","乐有家":"乐有家","365淘房":"365淘房","诸葛找房APP":"诸葛找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","C21":"C21","城市房产":"城市房产","幸福里APP":"幸福里"] +// def esSources = ["搜房":"搜房"] + def esSources = ["我爱我家":"我爱我家","兔博士PC端":"兔博士","吉屋":"吉屋","贝壳找房":"贝壳找房","乐有家":"乐有家","365淘房":"365淘房","诸葛找房APP":"诸葛找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","C21":"C21","城市房产小程序":"城市房产","幸福里APP":"幸福里"] static main(args) { runBatch(new SynchDistrictPrice(), args); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/zongzhimin/ApiDailyCountOfficialAccount.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/zongzhimin/ApiDailyCountOfficialAccount.groovy index 4d695b799..e6425178c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/zongzhimin/ApiDailyCountOfficialAccount.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/zongzhimin/ApiDailyCountOfficialAccount.groovy @@ -27,8 +27,8 @@ class ApiDailyCountOfficialAccount extends BaseBatch{ private static final Jms jms = Jms.create('tcp://monitor.jms.db.fangjia.com:61616') def static emails = [ - 'xuqi@fangjia.com', - 'zongzhimin@fangjia.com' + 'wangwenyan@fangjia.com', + 'gaojuan@fangjia.com' ] static void main(String[] args){ From 3791c3dd8ff9def9a2bdfa9c8b32d2cf686ee482 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 28 Feb 2024 16:25:59 +0800 Subject: [PATCH 03/22] updadte --- .../script/zongzhimin/ApiDailyCountOfficialAccount.groovy | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/zongzhimin/ApiDailyCountOfficialAccount.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/zongzhimin/ApiDailyCountOfficialAccount.groovy index e6425178c..366adad0f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/zongzhimin/ApiDailyCountOfficialAccount.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/zongzhimin/ApiDailyCountOfficialAccount.groovy @@ -27,8 +27,7 @@ class ApiDailyCountOfficialAccount extends BaseBatch{ private static final Jms jms = Jms.create('tcp://monitor.jms.db.fangjia.com:61616') def static emails = [ - 'wangwenyan@fangjia.com', - 'gaojuan@fangjia.com' + 'wangwenyan@fangjia.com' ] static void main(String[] args){ From 38c3b4d762b5b713aac7e6c9a391bccf1174292a Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 28 Feb 2024 16:52:02 +0800 Subject: [PATCH 04/22] =?UTF-8?q?=E5=8E=BB=E9=98=BF=E9=87=8C=E4=BA=91?= =?UTF-8?q?=E7=9A=84=E5=BC=95=E7=94=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- projects/fangjia-script/pom.xml | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/projects/fangjia-script/pom.xml b/projects/fangjia-script/pom.xml index e04dc02ff..b98b25ab1 100644 --- a/projects/fangjia-script/pom.xml +++ b/projects/fangjia-script/pom.xml @@ -537,21 +537,21 @@ - - com.aliyun - aliyun-java-sdk-address-purification - 1.0.1 - - - com.aliyun - aliyun-java-sdk-core - 4.5.9 - - - com.alibaba - fastjson - 1.2.83 - + + + + + + + + + + + + + + + From bc9df89425ca28cf7388bb0a68a92380ee9b86c1 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 28 Feb 2024 17:50:44 +0800 Subject: [PATCH 05/22] =?UTF-8?q?=E5=88=A0=E9=99=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/ana/ManualOperationDataStat.groovy | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ana/ManualOperationDataStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ana/ManualOperationDataStat.groovy index 9e0bfffd0..796f9b27b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ana/ManualOperationDataStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ana/ManualOperationDataStat.groovy @@ -124,13 +124,13 @@ public class ManualOperationDataStat extends BaseBatch{ println(content) sendEmail(Watcher.WangWenYan.email, "后台统计量", content) sendEmail("wangchencen@fangjia.com", "后台统计量", content) - sendEmail("caoyuyan@fangjia.com", "后台统计量", content) - sendEmail("shenyuan@fangjia.com", "后台统计量", content) - sendEmail("xuminxia@fangjia.com", "后台统计量", content) - sendEmail("huangcunliang@fangjia.com", "后台统计量", content) - sendEmail("yaojingmeng@fangjia.com", "后台统计量", content) - sendEmail("muliangshu@fangjia.com", "后台统计量", content) - sendEmail("sunjian@fangjia.com", "后台统计量", content) +// sendEmail("caoyuyan@fangjia.com", "后台统计量", content) +// sendEmail("shenyuan@fangjia.com", "后台统计量", content) +// sendEmail("xuminxia@fangjia.com", "后台统计量", content) +// sendEmail("huangcunliang@fangjia.com", "后台统计量", content) +// sendEmail("yaojingmeng@fangjia.com", "后台统计量", content) +// sendEmail("muliangshu@fangjia.com", "后台统计量", content) +// sendEmail("sunjian@fangjia.com", "后台统计量", content) } } \ No newline at end of file From 203798e5e98a54ca94bd33561c2424df5a9a7c29 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 28 Feb 2024 18:00:03 +0800 Subject: [PATCH 06/22] =?UTF-8?q?=E5=88=A0=E9=99=A4=E9=82=AE=E7=AE=B1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/cover/LoudongDataCoverBatch.groovy | 14 +++++++------- .../fangjia/script/fangrui/LngLatEachWeek.groovy | 2 +- .../script/hubingliang/PdsUseStatisticBatch.groovy | 8 ++++---- .../log/AnalyticsApiLogPerformanceBatch.groovy | 4 ++-- .../fangjia/script/log/CommonLogAnaBatch.groovy | 10 +++++----- .../fangjia/script/log/DaikuanLogAnaBatch.groovy | 12 ++++++------ .../script/log/ReadBkoffSafeLogBatch.groovy | 14 +++++++------- .../groovy/com/fangjia/script/util/DBHelper.groovy | 2 +- .../script/wenyan/ComparisonDataStatistics.groovy | 2 +- .../script/wenyan/EvaluationSourceAnalyze.groovy | 2 +- 10 files changed, 35 insertions(+), 35 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/cover/LoudongDataCoverBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/cover/LoudongDataCoverBatch.groovy index 74ec3b25f..63fbf843d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/cover/LoudongDataCoverBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/cover/LoudongDataCoverBatch.groovy @@ -24,16 +24,16 @@ class LoudongDataCoverBatch extends BaseBatch { static Jms jms = Jms.create('tcp://jms.db.fangjia.com:61616'); public DBCollection mongoColl = MongoUtils.getCollectionByName("fangjia", "house_loudong", "mongo.db.fangjia.com") public static final def EMAILS = [ - "duxiaoyu@fangjia.com", +// "duxiaoyu@fangjia.com", // "ruanyadong@fangjia.com", // "xiamin@fangjia.com", // "zhutianli@fangjia.com", - "joe@fangjia.com", - "yukai@fangjia.com", - "jiangxiaochun@fangjia.com", - "gaojuan@fangjia.com", - "shijing@fangjia.com", - "xiaohai@fangjia.com", +// "joe@fangjia.com", +// "yukai@fangjia.com", +// "jiangxiaochun@fangjia.com", + "gaojuan@fangjia.com" +// "shijing@fangjia.com", +// "xiaohai@fangjia.com", ] static def FIELDS_MAP = [ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fangrui/LngLatEachWeek.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fangrui/LngLatEachWeek.groovy index ff8e2fb8a..29478337f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fangrui/LngLatEachWeek.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fangrui/LngLatEachWeek.groovy @@ -82,7 +82,7 @@ class LngLatEachWeek extends BaseBatch { href = "点击下载附件" jms.sendMail("caoyuyan@fangjia.com","未匹配上的小区经纬度",href) jms.sendMail("wangchencen@fangjia.com","未匹配上的小区经纬度",href) - jms.sendMail("sunjian@fangjia.com","未匹配上的小区经纬度",href) +// jms.sendMail("sunjian@fangjia.com","未匹配上的小区经纬度",href) } public static Map getLngLatByAddress(String city, String address){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/hubingliang/PdsUseStatisticBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/hubingliang/PdsUseStatisticBatch.groovy index 3579922b1..c7a13eea0 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/hubingliang/PdsUseStatisticBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/hubingliang/PdsUseStatisticBatch.groovy @@ -45,10 +45,10 @@ class PdsUseStatisticBatch extends BaseBatch{ final static DBCollection fsDB = MongoUtils.getCollectionByName("fangjia_fs", "files", MongoUtils.MongoDB_Domain_fs_mongo, 27000) def static emails = [ - "duweiliang@fangjia.com", - "yaojingmeng@fangjia.com", - "xuqi@fangjia.com", - "wangpeili@fangjia.com" +// "duweiliang@fangjia.com", +// "yaojingmeng@fangjia.com", +// "xuqi@fangjia.com", + "gaojuan@fangjia.com" ] static main(args) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/log/AnalyticsApiLogPerformanceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/log/AnalyticsApiLogPerformanceBatch.groovy index 7ec752606..26b8c5aa5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/log/AnalyticsApiLogPerformanceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/log/AnalyticsApiLogPerformanceBatch.groovy @@ -163,9 +163,9 @@ class AnalyticsApiLogPerformanceBatch extends BaseBatch { } static void sendEmail(String sub, String msg) { - jms.sendMail("duxiaoyu@fangjia.com", sub, msg) +// jms.sendMail("duxiaoyu@fangjia.com", sub, msg) jms.sendMail("wangwenyan@fangjia.com", sub, msg) - jms.sendMail("helvyuan@fangjia.com", sub, msg) +// jms.sendMail("helvyuan@fangjia.com", sub, msg) jms.sendMail("gaojuan@fangjia.com", sub, msg) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/log/CommonLogAnaBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/log/CommonLogAnaBatch.groovy index 30da55b3e..94ea4a405 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/log/CommonLogAnaBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/log/CommonLogAnaBatch.groovy @@ -32,11 +32,11 @@ class CommonLogAnaBatch extends BaseBatch { def static bjEvaReport = []; public static final def EMAILS = [ - "jiangxiaochun@fangjia.com", - "limingmin@fangjia.com", - "xiaohai@fangjia.com", - "luxuping@fangjia.com", - "yukai@fangjia.com" + "jiangxiaochun@fangjia.com" +// "limingmin@fangjia.com", +// "xiaohai@fangjia.com", +// "luxuping@fangjia.com", +// "yukai@fangjia.com" ]; static main(args) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/log/DaikuanLogAnaBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/log/DaikuanLogAnaBatch.groovy index bd412eda3..41c9da3f7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/log/DaikuanLogAnaBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/log/DaikuanLogAnaBatch.groovy @@ -26,12 +26,12 @@ class DaikuanLogAnaBatch extends BaseBatch { static Jms jms = Jms.create('tcp://jms.db.fangjia.com:61616'); public static final def EMAILS = [ - "jiangxiaochun@fangjia.com", - "limingmin@fangjia.com", - "zhongjingyi@fangjia.com", - "xiaohai@fangjia.com", - "luxuping@fangjia.com", - "yukai@fangjia.com" + "jiangxiaochun@fangjia.com" +// "limingmin@fangjia.com", +// "zhongjingyi@fangjia.com", +// "xiaohai@fangjia.com", +// "luxuping@fangjia.com", +// "yukai@fangjia.com" ]; static main(args) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/log/ReadBkoffSafeLogBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/log/ReadBkoffSafeLogBatch.groovy index b2cda8f17..b2d649195 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/log/ReadBkoffSafeLogBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/log/ReadBkoffSafeLogBatch.groovy @@ -148,14 +148,14 @@ class ReadBkoffSafeLogBatch extends BaseBatch{ } static void sendEmail(String sub, String msg) { - jms.sendMail("duxiaoyu@fangjia.com", sub, msg) - jms.sendMail("joe@fangjia.com", sub, msg) - jms.sendMail("xiaohai@fangjia.com", sub, msg) +// jms.sendMail("duxiaoyu@fangjia.com", sub, msg) +// jms.sendMail("joe@fangjia.com", sub, msg) +// jms.sendMail("xiaohai@fangjia.com", sub, msg) jms.sendMail("caoyuyan@fangjia.com", sub, msg) - jms.sendMail("wangchencen@fangjia.com", sub, msg) - jms.sendMail("yaojingmeng@fangjia.com", sub, msg) - jms.sendMail("chichilook@qq.com", sub, msg) - jms.sendMail("sunjian@fangjia.com", sub, msg) +// jms.sendMail("wangchencen@fangjia.com", sub, msg) +// jms.sendMail("yaojingmeng@fangjia.com", sub, msg) +// jms.sendMail("chichilook@qq.com", sub, msg) +// jms.sendMail("sunjian@fangjia.com", sub, msg) } static public String getCatById(String id) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DBHelper.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DBHelper.groovy index cd31567e6..438522549 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DBHelper.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DBHelper.groovy @@ -36,7 +36,7 @@ class DBHelper { static final DB offline61PoseidonDb = new Mongo("192.168.0.61", 27037).getDB("poseidon"); static final online42GoojiaFangjiaDbConn = Sql.newInstance("jdbc:mysql://222.73.242.42:4306/goojia_fangjia?useUnicode=true&characterEncoding=utf-8", "sa", "G00jia", "com.mysql.jdbc.Driver"); static final offline89GoojiaFangjiaStatDbConn = Sql.newInstance("jdbc:mysql://192.168.0.89:3306/goojia_fangjia_stat?useUnicode=true&characterEncoding=utf-8", "root", "123456", "com.mysql.jdbc.Driver"); - static final emailsGroup0 = ['gaojuan@fangjia.com','yukai@fangjia.com','joe@fangjia.com','duxiaoyu@fangjia.com','tangye@fangjia.com','ruanyadong@fangjia.com','xiamin@fangjia.com','zhutianli@fangjia.com']; + static final emailsGroup0 = ['gaojuan@fangjia.com']; static final emailsFangjiaWang = ['all.list@fangjia.com']; static final mobilesYanzhou = ['13621626685']; static final emailsYanzhou = ['duxiaoyu@fangjia.com']; diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/ComparisonDataStatistics.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/ComparisonDataStatistics.groovy index 0f3c5e23a..9236574d7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/ComparisonDataStatistics.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/ComparisonDataStatistics.groovy @@ -23,7 +23,7 @@ public class ComparisonDataStatistics extends com.fangjia.script.BaseBatch{ static final DATE_NUMBER = new Date().format("yyyyMMdd").toInteger() static final DBCollection MESSENGER_COLL = MongoUtils.getCollectionByName("fangjia", "messenger", MongoUtils.FANGJIA_ONLINE_MONGO ,27017) static final def categories = ["district":"住宅", "office":"写字楼" , "shop":"商铺"] - def emails = ["wangwenyan@fangjia.com","duxiaoyu@fangjia.com","joe@fangjia.com","caoyuyan@fangjia.com"] + def emails = ["wangwenyan@fangjia.com","caoyuyan@fangjia.com"] static Jms jms = Jms.create('tcp://jms.db.fangjia.com:61616'); def static main(args){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/EvaluationSourceAnalyze.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/EvaluationSourceAnalyze.groovy index e7a3c1c7e..7241889ae 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/EvaluationSourceAnalyze.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/EvaluationSourceAnalyze.groovy @@ -14,7 +14,7 @@ class EvaluationSourceAnalyze extends BaseBatch { static Jms jms = Jms.create('tcp://jms.db.fangjia.com:61616'); public static final DBCollection STAT_COLL = MongoUtils.getCollectionByName("fangjia", "evaluation", "mongo.db.fangjia.com") - def emails = ["wangwenyan@fangjia.com","duxiaoyu@fangjia.com","joe@fangjia.com","xiaohai@fangjia.com"] + def emails = ["wangwenyan@fangjia.com","xiaohai@fangjia.com"] static main(args) { new EvaluationSourceAnalyze().process(null); From e993a2d0bce4f5d3f47838c95ecfd42c2ff4bfd8 Mon Sep 17 00:00:00 2001 From: wangwenyan Date: Fri, 1 Mar 2024 10:25:53 +0800 Subject: [PATCH 07/22] =?UTF-8?q?=E8=AE=BE=E7=BD=AE=E6=8C=87=E5=AF=BC?= =?UTF-8?q?=E4=BB=B7=E4=BF=AE=E6=94=B9=E6=97=B6=E9=97=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/script/wenyan/Test3.groovy | 109 ++++++++---------- 1 file changed, 47 insertions(+), 62 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/Test3.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/Test3.groovy index d2ba9d116..f4430e6a9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/Test3.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/Test3.groovy @@ -1,7 +1,16 @@ +import com.fangjia.core.RegionData +import com.fangjia.core.util.GeoUtils +import com.fangjia.core.util.HttpRequest import com.fangjia.core.util.Mongo3Utils +import com.fangjia.core.util.MongoUtils +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject import com.mongodb.client.MongoCollection +import net.sf.json.JSONObject import org.bson.Document +import org.bson.types.ObjectId import org.elasticsearch.xpack.sql.jdbc.EsDataSource import java.sql.Connection @@ -11,67 +20,43 @@ import java.util.function.Consumer import static com.mongodb.client.model.Filters.and import static com.mongodb.client.model.Filters.eq +//REGION_CONN = MongoUtils.getCollectionByName("fangjia", "region_block", MongoUtils.MongoDB_Domain_mongo, MongoUtils.MongoDB_Port_28017); +//TOWNSHIP_CONN = MongoUtils.getCollectionByName("fangjia", "township", MongoUtils.MongoDB_Domain_mongo, MongoUtils.MongoDB_Port_28017); +//TOWNSHIP_CONN.find().each {BasicDBObject township-> +// township.area_station = township.central +// township.removeField("central") +// TOWNSHIP_CONN.save(township) +// +//} +//REGION_CONN.find(new BasicDBObject("category", "region")).each {BasicDBObject regionBlock-> +// def lng = regionBlock.lng/10000000000l +// def lat = regionBlock.lat/10000000000l +// HttpRequest request = new HttpRequest() +// JSONObject resp = request.getJsonContent("http://localhost:8080/FangJiaAddressService/geocode/township", ['location':"$lng,$lat".toString()], "UTF-8") +// if(resp.result){ +// TOWNSHIP_CONN.update(new BasicDBObject("_id", new ObjectId(resp.result.id)), new BasicDBObject('$set', new BasicDBObject('central', 1))) +// println(resp.result) +// } +//} -def cityList = ["上海", "广州", "北京", "深圳", "成都", "杭州", "重庆", "武汉", "西安", "天津"] -def roots = ["lianjia.com", "zhuge.com", "fang.com", "anjuke.com", "58.com", "soufy.cn"] -MongoCollection DISTRICT_STAT = Mongo3Utils.getDbCollection("fangjia", "district_stat_rent") -MongoCollection DEAL_PRICE = Mongo3Utils.getDbCollection("fangjia", "deal_price") -def data = [["城市", "区域", "小区名", "版块", "装修", "朝向", "室", "厅", "卫", '标题', '总价', '面积', '单价', '发布时间', '来源']] -cityList.each { def city -> - def statMap = [:] - DISTRICT_STAT.find(and(eq("city", city), eq("end_month_date", Date.parse("yyyyMMdd", "20190228")))).forEach(new Consumer() { - @Override - void accept(Document t) { - def key = "${t.city}_${t.region}_${t.district_name}_${t.house_type}".toString() - statMap[key] = t.avg_price - } - }) - roots.each { def root -> - EsDataSource dataSource = new EsDataSource(); - String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" - dataSource.setUrl(address) - Properties connectionProperties = new Properties() - connectionProperties.put("user", "developer") - connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3") - dataSource.setProperties(connectionProperties) - Connection connection = dataSource.getConnection() - Statement statement = connection.createStatement() - ResultSet rs = statement.executeQuery("""select * from "seawater-2019" where city='${city}' and cat = 'rent' order by p_date""") - int count = 0 - while (rs.next()) { - String region = rs.getString("region") - String district_name = rs.getString("district_name") - int room = rs.getInt("room") - def key = "${city}_${region}_${district_name}_${room}".toString() - if (!statMap.containsKey(key)) { - continue - } - int total_price = rs.getInt("total_price") - double area = rs.getDouble("area") - int avg_price = (int) (total_price / area) - if (total_price * 1.3 || total_price < statMap[key] * 0.7) { - continue - } - String block = rs.getString("block") - String name = rs.getString("name") - String fitment = rs.getString("fitment") - String direction = rs.getString("direction") - int hall = rs.getInt("hall") - int toilet = rs.getInt("toilet") - int p_date = rs.getInt("p_date") - if (!block || !name || !fitment || !direction || room == 0 || hall == 0 || toilet == 0) { - continue - } - println(rs.getString('url')) - def item = [city, region, district_name, block, fitment, direction, room, hall, toilet, name, total_price, area, avg_price, p_date, root] - println(item) - data << item - count++ - if(count > 10){ - break - } - } - } -} -ExcelUtils.write("D:\\建设银行\\租金样本.xlsx", data, "结果") \ No newline at end of file +//CommonUtils.findAllCity().each {def city-> +// DbUtils.multiThreadingExecuteQueryOnSeaweed(DbUtils.buildSeaweedQuery(city, null, null, "district"),{BasicDBObject office-> +// if(!office.township){ +// def lng = office.lng/10000000000l +// def lat = office.lat/10000000000l +// HttpRequest request = new HttpRequest() +// JSONObject resp = request.getJsonContent("http://localhost:8080/FangJiaAddressService/geocode/township", ['location':"$lng,$lat".toString()], "UTF-8") +// if(resp.result){ +// println(office._id.toString() + "\t" + office.township + "\t-->\t" + resp.result.streetName) +// DbUtils.seaweed.update(new BasicDBObject("_id", office._id), new BasicDBObject('$set', new BasicDBObject('township', resp.result.streetName))) +// } +// } +// }) +//} +ExcelUtils.multiThreadingModify("C:\\Users\\goojia\\Desktop\\地图查询数据导出-添加经纬度-兴趣点.xlsx",{List item-> + def lng = item["AB"].toString().split(",")[0].toDouble() + def lat = item["AB"].toString().split(",")[1].toDouble() + item["AD"] = GeoUtils.calculateDistance(121.526672,31.142637,lng, lat) + println(item) +}) \ No newline at end of file From 51189231edb137921d5170abe2c4339cff9570dc Mon Sep 17 00:00:00 2001 From: wangwenyan Date: Fri, 1 Mar 2024 11:34:27 +0800 Subject: [PATCH 08/22] =?UTF-8?q?=E4=BC=98=E5=8C=96Trigger?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/service/index/SellTriggerServiceImpl.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/index/SellTriggerServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/index/SellTriggerServiceImpl.java index af7d6be61..1f7ab6869 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/index/SellTriggerServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/index/SellTriggerServiceImpl.java @@ -165,8 +165,11 @@ public class SellTriggerServiceImpl implements TriggerService { newObj.put("districtname_key", district.getName() + "_" + newObj.get("region") + "_" + object.get("city")); newObj.put("district_name", district.getName()); } + }else{ + if(object.containsField("city") && object.containsField("district_name") && object.containsField("region")){ + newObj.put("districtname_key", object.get("district_name") + "_" + object.get("region") + "_" + object.get("city")); + } } - // newObj.put("if_school", 0); // if (districtSchoolMap != null && districtSchoolMap.containsKey(districtKey)) { // List skList = districtSchoolMap.get(districtKey); From f0061ade19d549f26467ab6be7e9eaaa66cd3011 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Sun, 3 Mar 2024 19:45:57 +0800 Subject: [PATCH 09/22] =?UTF-8?q?=E6=88=BF=E6=BA=90=E7=BB=9F=E8=AE=A1?= =?UTF-8?q?=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../averager/HouseChangingMonitorBatch.groovy | 2 +- .../script/avgprice/WebAverageLabor.groovy | 2 +- .../cover/DistrictDataCoverBatch.groovy | 18 ++++---- .../cover/DistrictFieldCoverBatch.groovy | 5 +-- .../script/district/DistrictHouseBatch.groovy | 44 +++++++++++++++++-- .../evaluation/EvaluationFieldProcent.groovy | 4 +- .../fangjiatong/FjtAnalyCrawlerIp.groovy | 2 +- .../com/fangjia/script/util/DBHelper.groovy | 2 +- .../com/fangjia/script/wenyan/Watcher.groovy | 4 +- 9 files changed, 57 insertions(+), 26 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/averager/HouseChangingMonitorBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/averager/HouseChangingMonitorBatch.groovy index 41a7088f2..2f785e37c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/averager/HouseChangingMonitorBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/averager/HouseChangingMonitorBatch.groovy @@ -29,7 +29,7 @@ class HouseChangingMonitorBatch extends BaseBatch { DBCollection COLLECTION = MongoUtils.getCollectionByName("fangjia_log", "house_changing_monitor", "log.mongo.db.fangjia.com") Jms jms = Jms.create('tcp://jms.db.fangjia.com:61616'); def EMAILS = [ - 'duxiaoyu@fangjia.com' + 'gaojuan@fangjia.com' ] public static void main(String[] args){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/avgprice/WebAverageLabor.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/avgprice/WebAverageLabor.groovy index a1cf124eb..fdf9f1e5c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/avgprice/WebAverageLabor.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/avgprice/WebAverageLabor.groovy @@ -39,7 +39,7 @@ class WebAverageLabor extends BaseBatch{ public static final def CITIES = ["上海", "成都", "北京", "广州", "深圳"] - public static final def EMAILS = ["duxiaoyu@fangjia.com"] + public static final def EMAILS = ["wangwenyan@fangjia.com"] static Jms jms = Jms.create('tcp://jms.db.fangjia.com:61616'); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/cover/DistrictDataCoverBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/cover/DistrictDataCoverBatch.groovy index 1f060779c..083b4973d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/cover/DistrictDataCoverBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/cover/DistrictDataCoverBatch.groovy @@ -24,16 +24,16 @@ class DistrictDataCoverBatch extends BaseBatch { public DSCollection dsCollection = new DataStore("seaweed.db.fangjia.com", 27007).getCollection("fangjia.seaweed") public DBCollection mongoColl = MongoUtils.getCollectionByName("fangjia", "district_facilitys", "mongo.db.fangjia.com") public static final def EMAILS =[ - "duxiaoyu@fangjia.com", - "ruanyadong@fangjia.com", - "xiamin@fangjia.com", - "zhutianli@fangjia.com", - "joe@fangjia.com", - "yukai@fangjia.com", - "jiangxiaochun@fangjia.com", +// "duxiaoyu@fangjia.com", +// "ruanyadong@fangjia.com", +// "xiamin@fangjia.com", +// "zhutianli@fangjia.com", +// "joe@fangjia.com", +// "yukai@fangjia.com", +// "jiangxiaochun@fangjia.com", "gaojuan@fangjia.com", - "liuxingwang@fangjia.com", - "xiaohai@fangjia.com", +// "liuxingwang@fangjia.com", +// "xiaohai@fangjia.com", ] static def FIELDS_MAP = [ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/cover/DistrictFieldCoverBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/cover/DistrictFieldCoverBatch.groovy index 84bb4a078..9b187a3f5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/cover/DistrictFieldCoverBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/cover/DistrictFieldCoverBatch.groovy @@ -33,10 +33,7 @@ class DistrictFieldCoverBatch extends BaseBatch { final static String coverDomain = "mongo.db.fangjia.com" final static String datastoreDomain = "seaweed.db.fangjia.com" final static def emails = [ - "duxiaoyu@fangjia.com", - "joe@fangjia.com", "yukai@fangjia.com", "yanzhou@fangjia.com", - "gaojuan@fangjia.com", "yuanyadong@fangjia.com", "liuxingwang@fangjia.com", - "xiamin@fangjia.com", "zhutianli@fangjia.com" + "gaojuan@fangjia.com" ] // final static def emails = [ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/district/DistrictHouseBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/district/DistrictHouseBatch.groovy index fa3d57a71..dbc92b4dd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/district/DistrictHouseBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/district/DistrictHouseBatch.groovy @@ -1,8 +1,10 @@ package com.fangjia.script.district +import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.util.MongoUtils import com.mongodb.DBCursor +import org.bson.types.ObjectId import java.util.List @@ -40,6 +42,19 @@ class DistrictHouseBatch extends BaseBatch { @Override public void process(Object args) { + +// DBObject q = new BasicDBObject(); +// q.put("_id", new ObjectId("6501174f6f5638447b264189")); +// def obj = CommonDatautils.getHouseDataStoreConnection().findOne(q) +// obj.put("c_date", new Date()) +// CommonDatautils.getHouseDataStoreConnection().save(obj) + +// CommonDatautils.getHouseDataStoreConnection().find(new BasicDBObject("districtname_key","新城帝景南区_武进_常州")).each{ +// println(it) +// } + + + def cityList = cityList(); int c = 0 @@ -52,9 +67,28 @@ class DistrictHouseBatch extends BaseBatch { Map rentCountMap = facetDistrictInHouse("rent", city); println c + "\t" + city + "\t" + "\tsn:" + sn + "\t" + districts.size() + "\t" + DateUtil.formatToTightYYYYMMDDhhmmss(new Date()) for(DBObject district : districts) { - String key = district.get("name").toString().trim() + "_" + district.get("region") + "_" + district.get("city") - long sellCount = sellCountMap.get(key) ? sellCountMap.get(key) : 0 - long rentCount = rentCountMap.get(key) ? rentCountMap.get(key) : 0 + def names = [] + names << district.name + if(district.alias && districts.size()>0){ + for(def name : district.alias){ + if(!names.contains(name)){ + names << name + } + } + } + + long sellCount = 0 + long rentCount = 0 + for(def name : names){ + String key = name.toString().trim() + "_" + district.get("region") + "_" + district.get("city") + if(!sellCount){ + sellCount = sellCountMap.get(key) ? sellCountMap.get(key) : 0 + } + if(!rentCount){ + rentCount = rentCountMap.get(key) ? rentCountMap.get(key) : 0 + } + } + saveToDistrictHouse(district.get('city'), district.get("region"), district.get("name"), sellCount, rentCount, sn) } @@ -75,6 +109,7 @@ class DistrictHouseBatch extends BaseBatch { sellQuery.put('$facet', facet) sellQuery.put('city', city) sellQuery.put('cat',cat) +// sellQuery.put("district_name","新城帝景南区") // sellQuery.put('visible',0) DSCursor sellDSCursor = SEABIRD.find(sellQuery).skip(0).limit(0) List sellFacetResult = sellDSCursor.getFacetResults(); @@ -109,13 +144,14 @@ class DistrictHouseBatch extends BaseBatch { fields.put("city", 1) fields.put("region", 1) fields.put("name", 1) - + fields.put("alias", 1) def ret = [] DBObject q = new BasicDBObject() q.put('city', city) q.put('cat', 'district') q.put('visible', 0) +// q.put("name","新城帝景") DSCursor cur = SEAWEED.find(q, fields).batchSize(2000) Iterator iter = cur.iterator() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/evaluation/EvaluationFieldProcent.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/evaluation/EvaluationFieldProcent.groovy index 46195928b..9c1578e2d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/evaluation/EvaluationFieldProcent.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/evaluation/EvaluationFieldProcent.groovy @@ -29,9 +29,7 @@ class EvaluationFieldProcent extends BaseBatch{ public static final DBCollection EVALUA_COLL = MongoUtils.getCollectionByName("fangjia", "evaluation", "mongo.db.fangjia.com") public static final DBCollection EVALUA_COVER_COLL = MongoUtils.getCollectionByName("fangjia", "evaluation_field_coverage", "mongo.db.fangjia.com") - final static def emails = ["xiaohai@fangjia.com","jiangxiaochun@fangjia.com","limingmin@fangjia.com","duxiaoyu@fangjia.com", - "joe@fangjia.com", "yukai@fangjia.com", "gaojuan@fangjia.com", "yuanyadong@fangjia.com", - "xiamin@fangjia.com", "zhutianli@fangjia.com"] + final static def emails = [ "gaojuan@fangjia.com"] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fangjiatong/FjtAnalyCrawlerIp.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fangjiatong/FjtAnalyCrawlerIp.groovy index a1043fb29..5091edaa5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fangjiatong/FjtAnalyCrawlerIp.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fangjiatong/FjtAnalyCrawlerIp.groovy @@ -30,7 +30,7 @@ class FjtAnalyCrawlerIp extends BaseBatch{ public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017"; public static final DBCollection addressMatchLogColl = MongoUtils.getCollectionByName2("fangjia", "address_match_log", lineUrl) - final static def emails = ["gaojuan@fangjia.com","jiangxiaochun@fangjia.com","duxiaoyu@fangjia.com","wangwenyan@fangjia.com","luxuping@fangjia.com","yaojiayi@fangjia.com"] + final static def emails = ["gaojuan@fangjia.com"] static Jms jms = Jms.create('tcp://jms.db.fangjia.com:61616'); static main(args) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DBHelper.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DBHelper.groovy index 438522549..448d8aaea 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DBHelper.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DBHelper.groovy @@ -39,7 +39,7 @@ class DBHelper { static final emailsGroup0 = ['gaojuan@fangjia.com']; static final emailsFangjiaWang = ['all.list@fangjia.com']; static final mobilesYanzhou = ['13621626685']; - static final emailsYanzhou = ['duxiaoyu@fangjia.com']; + static final emailsYanzhou = ['wangwenyan@fangjia.com']; static List getCityNames(){ DBObject dbObject = new BasicDBObject("status",0); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/Watcher.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/Watcher.groovy index d0134fa35..d20623baa 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/Watcher.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/Watcher.groovy @@ -3,9 +3,9 @@ package com.fangjia.script.wenyan public enum Watcher { //在职同事 - DuXiaoYu('13621626685', 'duxiaoyu@fangjia.com'), + DuXiaoYu('13621626685', 'wangwenyan@fangjia.com'), WangWenYan('13916076378','wangwenyan@fangjia.com'), - Tomzhao('15021021464','zhaoshumao@fangjia.com'), + Tomzhao('15021021464','wangwenyan@fangjia.com'), GaoJuan('15021384995','gaojuan@fangjia.com'); private final String mobile; From b356c2cea392e32fb303242236770b9b9467197c Mon Sep 17 00:00:00 2001 From: mjt <13607600062@163.com> Date: Mon, 4 Mar 2024 08:59:29 +0800 Subject: [PATCH 10/22] =?UTF-8?q?=E6=B1=9F=E8=8B=8F=E9=93=B6=E8=A1=8C?= =?UTF-8?q?=E6=B6=89=E5=86=9C=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../JSBankTest/JSBankAliAddress.groovy | 50 ++++++ .../JSBankTest/JSBankDevelopmentZOne.groovy | 85 ++++++++++ .../JSBankTest/JSBankSheNongTest.groovy | 85 ++++++++++ .../sdklog/sdkclient_CommunicateLog_info | 0 .../sdklog/sdkclient_EMAYSDK_info | 0 .../majintao/test/AliAddressTest.groovy | 51 ++++++ .../test/AliAddressTestChecked.groovy | 76 +++++++++ .../script/majintao/test/FiveLevelTest.groovy | 64 ++++++++ .../script/majintao/test/SheNongTest.groovy | 123 +++++++++++++++ .../script/majintao/test/SheNongTest2.groovy | 147 ++++++++++++++++++ .../majintao/test/SheNongTestChecked.groovy | 44 ++++++ .../majintao/test/nanjingBankTest.groovy | 121 ++++++++++++++ .../test/sdklog/sdkclient_CommunicateLog_info | 0 .../test/sdklog/sdkclient_EMAYSDK_info | 0 .../fangjia/script/majintao/test/t1.groovy | 15 ++ .../fangjia/script/majintao/test/test.groovy | 72 +++++++++ .../com/fangjia/script/util/ExcelUtils.groovy | 2 +- 17 files changed, 934 insertions(+), 1 deletion(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/JSBankTest/JSBankAliAddress.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/JSBankTest/JSBankDevelopmentZOne.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/JSBankTest/JSBankSheNongTest.groovy rename projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/{ => JSBankTest}/sdklog/sdkclient_CommunicateLog_info (100%) rename projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/{ => JSBankTest}/sdklog/sdkclient_EMAYSDK_info (100%) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/test/AliAddressTest.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/test/AliAddressTestChecked.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/test/FiveLevelTest.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/test/SheNongTest.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/test/SheNongTest2.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/test/SheNongTestChecked.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/test/nanjingBankTest.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/test/sdklog/sdkclient_CommunicateLog_info create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/test/sdklog/sdkclient_EMAYSDK_info create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/test/t1.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/test/test.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/JSBankTest/JSBankAliAddress.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/JSBankTest/JSBankAliAddress.groovy new file mode 100644 index 000000000..cb9ef1193 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/JSBankTest/JSBankAliAddress.groovy @@ -0,0 +1,50 @@ +package com.fangjia.script.majintao.test + +import com.fangjia.core.util.HttpRequest +import com.fangjia.script.gj.village.ailiyun.SignatureUtils +import com.fangjia.script.util.ExcelUtils + +//def address = "江苏省徐州市贾汪区经济开发区大黄山镇孤山小区B3.801" +String excelFilePath = "C:\\Users\\Goojia\\Desktop\\1.xlsx"; +ExcelUtils.removeEmptyRow(excelFilePath) +def count = 0 +ExcelUtils.singleThreadingModify(excelFilePath, { List items -> + + count = count + 1 + if (count % 10 == 0) { + println("已处理:" + count) + } + sleep(80) + + def address = items['A'] + def map = SignatureUtils.getAliAddressStrc(address) + //prov=河南省 city=信阳市 district=息县 town=孙庙乡 community=前赵楼村 road=前赵楼村 poi=爱婆店 roomno=012 + if ("prov" in map) { + def province = map.get("prov") + items['E'] = province + } + if ("city" in map) { + def city = map.get("city") + items['G'] = city + } + if ("district" in map) { + def district = map.get("district") + items['I'] = district + } + if ("town" in map) { + def town = map.get("town") + items['K'] = town + } +// if ("community" in map) { +// def community = map.get("community") +// items['S'] = community +// } + if ("devzone" in map) { + def devzone = map.get("devzone") + items['Q'] = devzone + + } + +}) + + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/JSBankTest/JSBankDevelopmentZOne.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/JSBankTest/JSBankDevelopmentZOne.groovy new file mode 100644 index 000000000..7a2098677 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/JSBankTest/JSBankDevelopmentZOne.groovy @@ -0,0 +1,85 @@ +package com.fangjia.script.majintao.JSBankTest; + +import com.fangjia.core.util.HttpRequest +import com.fangjia.script.util.ExcelUtils; +import net.sf.json.JSONObject; +import java.util.Base64; +import net.sf.json.JSONArray; + + +String excelFilePath = "C:\\Users\\Goojia\\Desktop\\测试.xlsx"; +ExcelUtils.removeEmptyRow(excelFilePath) +def count = 0 +ExcelUtils.singleThreadingModify(excelFilePath, { List items -> + count += 1 + if (count % 10 == 0) { + println("已处理:" + count) + } + def zx = ["北京", "天津", "上海", "重庆"] +// String province = "天津"; + String province = items['E']; + if (zx.contains(province)) { + province = province + "市"; + } +// String name = "天津经济技术开发区"; + String name = items['Q']; + String elasticUsername = "elastic"; + String elasticPassword = "elasticsearch"; + + String seawaterUrl = "http://192.168.0.89:9200/development_zone/_search"; + String basicAuth = "Basic " + Base64.getEncoder().encodeToString((elasticUsername + ":" + elasticPassword).getBytes("UTF-8")); + + String body = """ + { + "query": { + "bool": { + "must": [ + { + "match": { + "name": { + "query": "${name}", + "operator": "and" + } + } + }, + { + "term": {"province": {"value": "${province}"}} + } + ] + } + } + } + """; + + HttpRequest request = new HttpRequest(); + JSONObject jsonBody = JSONObject.fromObject(body); + JSONObject header = new JSONObject(); + header.put("Authorization", basicAuth); + String res = request.postWithBody(seawaterUrl, jsonBody, header, "UTF-8"); + + JSONObject response = JSONObject.fromObject(res); + JSONObject obj = null; + if (response != null && response.containsKey("hits")) { + JSONObject hits = response.getJSONObject("hits"); + if (hits != null && hits.size() > 0) { + JSONArray hitsArray = hits.getJSONArray("hits"); + // 省略了之前的部分... + + if (hitsArray != null && hitsArray.size() > 0) { + for (int i = 0; i < hitsArray.size(); i++) { + JSONObject hit = hitsArray.getJSONObject(i); + JSONObject source = hit.getJSONObject("_source"); + String devName = source.optString("name"); // Assuming "name" is a String + String level = source.optString("level"); // Assuming "level" is a String + if (devName.equals(name)) { + items['O'] = "是"; + items['P'] = level; + println("省份:" + province + " 名称:" + name + " 开发区名:" + devName + " 开发区级别:" + level); + break + } + } + } + + } + } +}); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/JSBankTest/JSBankSheNongTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/JSBankTest/JSBankSheNongTest.groovy new file mode 100644 index 000000000..438897f64 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/JSBankTest/JSBankSheNongTest.groovy @@ -0,0 +1,85 @@ +package com.fangjia.script.majintao.JSBankTest + +import com.fangjia.core.util.HttpRequest +import com.fangjia.script.util.ExcelUtils +import org.apache.tools.ant.types.resources.selectors.None + + +String excelFilePath = "C:\\Users\\Goojia\\Desktop\\测试.xlsx"; +ExcelUtils.removeEmptyRow(excelFilePath) +def count = 0 +ExcelUtils.singleThreadingModify(excelFilePath, { List items -> + count += 1 + if (count % 10 == 0) { + println("已处理:" + count) + } + + + String address = items['B'] + + String url = "http://192.168.0.100:32080/FangJiaAddressService/villageIdentify"; + + HttpRequest httpRequest = new HttpRequest(); + map = new HashMap() + + + def map = [:] + map.put("address", address) + map.put("showFullValue", "2") + map.put("showTestValue", "2") + map.put("type", "2") + + def content = httpRequest.getJsonContent(url, map, "UTF-8") + def result = content.get("result") + if (result != null) { + if ("addrVerify") { + items['C'] = result.get("addrVerify") + } + + if ("province" in result) { + items['D'] = result.get("province") + } + if ("city" in result) { + items['F'] = result.get("city") + } + if ("district" in result) { + + items['H'] = result.get("district") + } + if ("town" in result) { + items['J'] = result.get("town") + } + if ("committee" in result) { + + items['L'] = result.get("committee") + } + if ("typeCode" in result) { + items['M'] = result.get("typeCode") + } + if ("countyGov") { + items['N'] = result.get("countyGov") + } + def devDistrict = "" + if ("devDistrict" in result) { + devDistrict = result.get("devDistrict") + } + def dev = "" + if ("dev" in result) { + dev = result.get("dev") + } + // 开发区级别暂时没有 + + // 暂时使用 + def defGrade = "" + if (!dev.equals("") && devDistrict.equals("否")) { + defGrade = "非国际级或省级" + devDistrict = "是" + } + if (!devDistrict.equals("")) { + items['O'] = devDistrict + } + items["P"] = defGrade + } + +}) + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/sdklog/sdkclient_CommunicateLog_info b/projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/JSBankTest/sdklog/sdkclient_CommunicateLog_info similarity index 100% rename from projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/sdklog/sdkclient_CommunicateLog_info rename to projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/JSBankTest/sdklog/sdkclient_CommunicateLog_info diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/sdklog/sdkclient_EMAYSDK_info b/projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/JSBankTest/sdklog/sdkclient_EMAYSDK_info similarity index 100% rename from projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/sdklog/sdkclient_EMAYSDK_info rename to projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/JSBankTest/sdklog/sdkclient_EMAYSDK_info diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/test/AliAddressTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/test/AliAddressTest.groovy new file mode 100644 index 000000000..16d11d368 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/test/AliAddressTest.groovy @@ -0,0 +1,51 @@ +package com.fangjia.script.majintao.test + +import com.fangjia.core.util.HttpRequest +import com.fangjia.script.gj.village.ailiyun.SignatureUtils +import com.fangjia.script.util.ExcelUtils + +//def address = "江苏省徐州市贾汪区经济开发区大黄山镇孤山小区B3.801" +String excelFilePath = "C:\\Users\\Goojia\\Desktop\\注册地址-太仓农商行POC测试.xlsx"; +ExcelUtils.removeEmptyRow(excelFilePath) +def count = 0 +ExcelUtils.singleThreadingModify(excelFilePath, { List items -> + + count = count + 1 + if (count % 10 == 0) { + println("已处理:" + count) + } + sleep(80) + + def address = items['A'] + def map = SignatureUtils.getAliAddressStrc(address) + //prov=河南省 city=信阳市 district=息县 town=孙庙乡 community=前赵楼村 road=前赵楼村 poi=爱婆店 roomno=012 + if ("prov" in map) { + def province = map.get("prov") + items['C'] = province + } + if ("city" in map) { + def city = map.get("city") + items['G'] = city + } + if ("district" in map) { + def district = map.get("district") + items['K'] = district + } + if ("town" in map) { + def town = map.get("town") + items['O'] = town + } + if ("community" in map) { + def community = map.get("community") + items['S'] = community + + } + if ("devzone" in map) { + def devzone = map.get("devzone") + items['AE'] = devzone + + } + +}) + + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/test/AliAddressTestChecked.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/test/AliAddressTestChecked.groovy new file mode 100644 index 000000000..e744b76d4 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/test/AliAddressTestChecked.groovy @@ -0,0 +1,76 @@ +package com.fangjia.script.majintao.test + +import com.fangjia.core.util.HttpRequest +import com.fangjia.script.gj.village.ailiyun.SignatureUtils +import com.fangjia.script.util.ExcelUtils + +//def address = "江苏省徐州市贾汪区经济开发区大黄山镇孤山小区B3.801" +String excelFilePath = "C:\\Users\\Goojia\\Desktop\\涉农对私测试数据.xlsx"; +ExcelUtils.removeEmptyRow(excelFilePath) +def count = 0 +ExcelUtils.singleThreadingModify(excelFilePath, { List items -> + + count = count + 1 + if (count % 10 == 0) { + println("已处理:" + count) + } + sleep(50) + + def address = items['A'] + def pro = items['D'] + if (pro == null) { + + def map = SignatureUtils.getAliAddressStrc(address) + //prov=河南省 city=信阳市 district=息县 town=孙庙乡 community=前赵楼村 road=前赵楼村 poi=爱婆店 roomno=012 + if ("prov" in map) { + def province = map.get("prov") + items['D'] = province + } + if ("city" in map) { + def city = map.get("city") + items['H'] = city + } + if ("district" in map) { + def district = map.get("district") + items['L'] = district + } + if ("town" in map) { + def town = map.get("town") + items['P'] = town + } + if ("community" in map) { + def community = map.get("community") + items['T'] = community + + } +// if ("road" in map) { +// def road = map.get("road") +// items['G'] = road +// +// } +// if ("poi" in map) { +// +// def poi = map.get("poi") +// items['H'] = poi +// +// } +// if ("roomno" in map) { +// def roomno = map.get("roomno") +// items['I'] = roomno +// +// } +// if ("devzone" in map) { +// def devzone = map.get("devzone") +// items['J'] = devzone +// +// } +// if ("roadno" in map) { +// def roadno = map.get("roadno") +// items['K'] = roadno +// } + } + + +}) + + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/test/FiveLevelTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/test/FiveLevelTest.groovy new file mode 100644 index 000000000..a430f9480 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/test/FiveLevelTest.groovy @@ -0,0 +1,64 @@ +package com.fangjia.script.majintao.test + +import com.fangjia.core.util.HttpRequest +import com.fangjia.script.util.ExcelUtils +import net.sf.json.JSONObject + + +String excelFilePath = "C:\\Users\\Goojia\\Desktop\\临时数据\\2.xlsx"; +ExcelUtils.removeEmptyRow(excelFilePath) +ExcelUtils.singleThreadingModify(excelFilePath, { List items -> + + String url = "http://192.168.0.19:31080/FangJiaAddressService/v3/structure"; + + HttpRequest httpRequest = new HttpRequest(); + map = new HashMap() + + String address = items['A'] + + def map = [:] + map.put("address", address) + map.put("complete", "true") + map.put("addressAssessment", "true") + + def content = httpRequest.getJsonContent(url, map, "UTF-8") + def result = content.get("result") + if ("province" in result) { + items['B'] = result.get("province") + } + if ("provinceCode" in result) { + items['C'] = result.get("provinceCode") + } + if ("city" in result){ + items['D'] = result.get("city") + } + if("cityCode" in result){ + items['E'] = result.get("cityCode") + } + if ("district" in result){ + + items['F'] = result.get("district") + } + if ("districtCode" in result){ + + items['G'] = result.get("districtCode") + } + if ("township" in result){ + + items['H'] = result.get("township") + } + if ("townshipCode" in result){ + + items['I'] = result.get("townshipCode") + print(result.get("townshipCode")) + } + if ("address" in result){ + + items['J'] = result.get("address") + } + if ("formatAddress" in result){ + items['K'] = result.get("formatAddress") + } + +}) + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/test/SheNongTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/test/SheNongTest.groovy new file mode 100644 index 000000000..a10f0b701 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/test/SheNongTest.groovy @@ -0,0 +1,123 @@ +package com.fangjia.script.majintao.test + +import com.fangjia.core.util.HttpRequest +import com.fangjia.script.util.ExcelUtils + + +String excelFilePath = "C:\\Users\\Goojia\\Desktop\\注册地址-太仓农商行POC测试.xlsx"; +ExcelUtils.removeEmptyRow(excelFilePath) +def count = 0 +ExcelUtils.singleThreadingModify(excelFilePath, { List items -> + count += 1 + if (count % 10 == 0) { + println("已处理:" + count) + } + + + String address = items['A'] +// if (province == null) { + String url = "http://192.168.0.100:32080/FangJiaAddressService/villageIdentify"; + + HttpRequest httpRequest = new HttpRequest(); + map = new HashMap() + + + def map = [:] + map.put("address", address) + map.put("showFullValue", "2") + map.put("showTestValue", "2") + map.put("type", "2") + + def content = httpRequest.getJsonContent(url, map, "UTF-8") + def result = content.get("result") + if (result != null) { + if ("province" in result) { + items['B'] = result.get("province") + } + if ("provinceCode" in result) { + items['E'] = result.get("provinceCode") + } + if ("city" in result) { + items['F'] = result.get("city") + } + if ("cityCode" in result) { + items['I'] = result.get("cityCode") + } + if ("district" in result) { + + items['J'] = result.get("district") + } + if ("districtCode" in result) { + + items['M'] = result.get("districtCode") + } + if ("town" in result) { + + items['N'] = result.get("town") + } + if ("townCode" in result) { + + items['Q'] = result.get("townCode") + } + + if ("committee" in result) { + + items['R'] = result.get("committee") + } + if ("committeeCode" in result) { + + items['U'] = result.get("committeeCode") + } + if ("typeCode" in result) { + + items['V'] = result.get("typeCode") + } + if ("typeName") { + items['W'] = result.get("typeName") + } + if ("verifyFlag") { + items['X'] = result.get("verifyFlag") + } + if ("addrVerify") { + items['Y'] = result.get("addrVerify") + } + if ("countyLevelCity") { + items['Z'] = result.get("countyLevelCity") + } + if ("countyGov") { + items['AA'] = result.get("countyGov") + } + if ("communitySource") { + items['AB'] = result.get("communitySource") + } + + if ("administrativeTowns" in result) { + items['AD'] = result.get("administrativeTowns") + } + if ("devDistrict" in result) { + items['AF'] = result.get("devDistrict") + } + if ("devDistrictName" in result) { + items['AG'] = result.get("devDistrictName") + } + if ("splitCommittee" in result) { + items['AH'] = result.get("splitCommittee") + } + if ("splitStreet" in result) { + items['AI'] = result.get("splitStreet") + } + if ("poiName" in result) { + items['AJ'] = result.get("poiName") + } + if ("splitStreetNum" in result) { + items['AK'] = result.get("splitStreetNum") + } + if ("dev" in result) { + items['AL'] = result.get("dev") + } + } + +// } + +}) + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/test/SheNongTest2.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/test/SheNongTest2.groovy new file mode 100644 index 000000000..7b84a95d5 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/test/SheNongTest2.groovy @@ -0,0 +1,147 @@ +package com.fangjia.script.majintao.test + +import com.fangjia.core.util.HttpRequest +import com.fangjia.script.util.ExcelUtils + + +String excelFilePath = "C:\\Users\\Goojia\\Desktop\\对公.xlsx"; +ExcelUtils.removeEmptyRow(excelFilePath) +def count = 0 +ExcelUtils.singleThreadingModify(excelFilePath, { List items -> + count+=1 + if (count%10==0){ + println("已处理:"+count) + } + + String url = "http://192.168.0.100:32080/FangJiaAddressService/villageIdentify"; + + HttpRequest httpRequest = new HttpRequest(); + map = new HashMap() + + String address = items['A'] +// print(address) + +// province; +// city; +// district; +// town; +// committee; +// type; +// provinceCode; +// cityCode; +// districtCode; +// townCode; +// committeeCode; +// typeCode; +// typeName; +// verifyFlag;//涉农标识 +// addrVerify;//地址是否有效信息 +// countyLevelCity;//是否为县级市 +// administrativeTowns;//是否市辖建制镇 +// countyGov;//是否为县政府 +// devDistrict;//是否为开发区 +// devDistrictName;//开发区 +// splitCommittee; +// splitStreet; +// poiName; +// splitStreetNum; +// dev; +// communitySource;//居委来源 + + def map = [:] + map.put("address", address) + map.put("complete", "true") + map.put("addressAssessment", "true") + map.put("type", "2") + + def content = httpRequest.getJsonContent(url, map, "UTF-8") + def result = content.get("result") + if ("province" in result) { + items['B'] = result.get("province") + } + if ("provinceCode" in result) { + items['C'] = result.get("provinceCode") + } + if ("city" in result){ + items['D'] = result.get("city") + } + if("cityCode" in result){ + items['E'] = result.get("cityCode") + } + if ("district" in result){ + + items['F'] = result.get("district") + } + if ("districtCode" in result){ + + items['G'] = result.get("districtCode") + } + if ("town" in result){ + + items['H'] = result.get("town") + } + if ("townCode" in result){ + + items['I'] = result.get("townCode") + } + + if ("committee" in result){ + + items['J'] = result.get("committee") + } + if ("committeeCode" in result){ + + items['K'] = result.get("committeeCode") + } + if ("typeCode" in result){ + + items['L'] = result.get("typeCode") + } + if ("typeName"){ + items['M'] = result.get("typeName") + } + if ("verifyFlag"){ + items['N'] = result.get("verifyFlag") + } + if ("addrVerify"){ + items['O'] = result.get("addrVerify") + } + if ("countyLevelCity"){ + items['P'] = result.get("countyLevelCity") + } + if ("countyGov"){ + items['Q'] = result.get("countyGov") + } + if ("communitySource"){ + items['R'] = result.get("communitySource") + } + + if ("administrativeTowns" in result){ + items['T'] = result.get("administrativeTowns") + } + if ("devDistrict" in result){ + items['V'] = result.get("devDistrict") + } + if ("devDistrictName" in result){ + items['W'] = result.get("devDistrictName") + } + if ("splitCommittee" in result){ + items['X'] = result.get("splitCommittee") + } + if ("splitStreet" in result){ + items['Y'] = result.get("splitStreet") + } + if ("poiName" in result){ + items['Z'] = result.get("poiName") + } + if ("splitStreetNum" in result){ + items['AA'] = result.get("splitStreetNum") + } + if ("dev" in result){ + items['AB'] = result.get("dev") + } + + + +}) + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/test/SheNongTestChecked.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/test/SheNongTestChecked.groovy new file mode 100644 index 000000000..8388fae47 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/test/SheNongTestChecked.groovy @@ -0,0 +1,44 @@ +package com.fangjia.script.majintao.test + +import com.fangjia.core.util.HttpRequest +import com.fangjia.script.util.ExcelUtils + + +String excelFilePath = "C:\\Users\\Goojia\\Desktop\\涉农对私测试数据 - 副本.xlsx"; +ExcelUtils.removeEmptyRow(excelFilePath) +def count = 0 +ExcelUtils.singleThreadingModify(excelFilePath, { List items -> + count += 1 + if (count % 10 == 0) { + println("已处理:" + count) + } + + + String flag = items['X'] + String address = items['A'] + String tt = items['Y'] + if (flag == null&&"地址有效".equals(tt)) { + String url = "http://192.168.0.100:32080/FangJiaAddressService/villageIdentify"; + + HttpRequest httpRequest = new HttpRequest(); + map = new HashMap() + + + def map = [:] + map.put("address", address) + map.put("complete", "true") + map.put("addressAssessment", "true") + map.put("type", "1") + + def content = httpRequest.getJsonContent(url, map, "UTF-8") + def result = content.get("result") + if (result != null) { + if ("verifyFlag" in result) { + items['X'] = result.get("verifyFlag") + } + } + + } + +}) + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/test/nanjingBankTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/test/nanjingBankTest.groovy new file mode 100644 index 000000000..b88be8d68 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/test/nanjingBankTest.groovy @@ -0,0 +1,121 @@ +package com.fangjia.script.majintao.test + + + +import com.fangjia.core.util.HttpRequest +import com.fangjia.script.util.ExcelUtils + + +String excelFilePath = "C:\\Users\\Goojia\\Desktop\\南京银行测试.xlsx"; +ExcelUtils.removeEmptyRow(excelFilePath) +def count = 0 +ExcelUtils.singleThreadingModify(excelFilePath, { List items -> + count+=1 + if (count%10==0){ + println("已处理:"+count) + } + + String url = "http://192.168.0.100:32080/FangJiaAddressService/villageIdentify"; + + HttpRequest httpRequest = new HttpRequest(); + map = new HashMap() + + String address = items['A'] + + def map = [:] + map.put("address", address) + map.put("complete", "2") + map.put("addressAssessment", "2") + map.put("type", "2") + + def content = httpRequest.getJsonContent(url, map, "UTF-8") + def result = content.get("result") + if ("province" in result) { + items['B'] = result.get("province") + } + if ("provinceCode" in result) { + items['C'] = result.get("provinceCode") + } + if ("city" in result){ + items['D'] = result.get("city") + } + if("cityCode" in result){ + items['E'] = result.get("cityCode") + } + if ("district" in result){ + + items['F'] = result.get("district") + } + if ("districtCode" in result){ + + items['G'] = result.get("districtCode") + } + if ("town" in result){ + + items['H'] = result.get("town") + } + if ("townCode" in result){ + + items['I'] = result.get("townCode") + } + + if ("committee" in result){ + + items['J'] = result.get("committee") + } + if ("committeeCode" in result){ + + items['K'] = result.get("committeeCode") + } + if ("typeCode" in result){ + + items['L'] = result.get("typeCode") + } + if ("typeName"){ + items['M'] = result.get("typeName") + } + if ("verifyFlag"){ + items['N'] = result.get("verifyFlag") + } + if ("addrVerify"){ + items['O'] = result.get("addrVerify") + } + if ("countyLevelCity"){ + items['P'] = result.get("countyLevelCity") + } + if ("countyGov"){ + items['Q'] = result.get("countyGov") + } +// if ("communitySource"){ +// items['R'] = result.get("communitySource") +// } +// +// if ("administrativeTowns" in result){ +// items['T'] = result.get("administrativeTowns") +// } +// if ("devDistrict" in result){ +// items['V'] = result.get("devDistrict") +// } +// if ("devDistrictName" in result){ +// items['W'] = result.get("devDistrictName") +// } +// if ("splitCommittee" in result){ +// items['X'] = result.get("splitCommittee") +// } +// if ("splitStreet" in result){ +// items['Y'] = result.get("splitStreet") +// } +// if ("poiName" in result){ +// items['Z'] = result.get("poiName") +// } +// if ("splitStreetNum" in result){ +// items['AA'] = result.get("splitStreetNum") +// } +// if ("dev" in result){ +// items['AB'] = result.get("dev") +// } + + + +}) + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/test/sdklog/sdkclient_CommunicateLog_info b/projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/test/sdklog/sdkclient_CommunicateLog_info new file mode 100644 index 000000000..e69de29bb diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/test/sdklog/sdkclient_EMAYSDK_info b/projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/test/sdklog/sdkclient_EMAYSDK_info new file mode 100644 index 000000000..e69de29bb diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/test/t1.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/test/t1.groovy new file mode 100644 index 000000000..9f339f5f0 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/test/t1.groovy @@ -0,0 +1,15 @@ +package com.fangjia.script.majintao.test + +import com.fangjia.core.util.HttpRequest + +String url = "http://192.168.0.19:31080/FangJiaAddressService/villageIdentify"; + +HttpRequest httpRequest = new HttpRequest(); +map = new HashMap() + + +def map = [:] +map.put("address", address) +map.put("showFullValue", "2") +map.put("showTestValue", "2") +map.put("type", "2") \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/test/test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/test/test.groovy new file mode 100644 index 000000000..22a5ae18e --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/majintao/test/test.groovy @@ -0,0 +1,72 @@ +package com.fangjia.script.majintao.test + +import com.fangjia.core.util.HttpRequest +import com.fangjia.script.util.ExcelUtils +import net.sf.json.JSONObject + + +String excelFilePath = "F:\\估家\\文件\\北交所\\testcopy1.xlsx"; +ExcelUtils.removeEmptyRow(excelFilePath) +ExcelUtils.singleThreadingModify(excelFilePath, { List items -> + + String comprehensiveUrl = "http://localhost:9180/comprehensiveQuery/houseValueInformation"; + String communityUrl = "http://open.fangjia.com/addrSvc/avgPrice"; + String valueUrl = "http://open.fangjia.com/property/addressEvaluate"; + String caseUrl = "http://open.fangjia.com/house/case"; + String poiUrl = "http://open.fangjia.com/district/addrFacilitys"; + + HttpRequest httpRequest = new HttpRequest(); + map = new HashMap() + + String city = items['A'] + String address = items['B'] + double size = items['C'] + + System.out.println(city + "---" + address + "----" + size); + + map.put("city", city) + map.put("address", address) + map.put("size", size.toString()) + map.put("strctureArea", size.toString()) + map.put("token", "F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB") + + if (!address.contains("%")) { + // 发送HTTP请求 + long start + long end + +// start = System.currentTimeMillis(); +// httpRequest.getJsonContent(communityUrl, map, "UTF-8") +// +// +// end = System.currentTimeMillis(); +// items['D'] = String.valueOf(end - start); +// +// start = System.currentTimeMillis(); +// httpRequest.getJsonContent(valueUrl, map, "UTF-8") +// end = System.currentTimeMillis(); +// items['E'] = String.valueOf(end - start); +// +// start = System.currentTimeMillis(); +// httpRequest.getJsonContent(caseUrl, map, "UTF-8") +// end = System.currentTimeMillis(); +// items['F'] = String.valueOf(end - start); +// +// start = System.currentTimeMillis(); +// httpRequest.getJsonContent(poiUrl, map, "UTF-8") +// end = System.currentTimeMillis(); +// items['G'] = String.valueOf(end - start); + + start = System.currentTimeMillis(); + def content = httpRequest.getJsonContent(comprehensiveUrl, map, "UTF-8") + end = System.currentTimeMillis(); + items['H'] = String.valueOf(end - start); + println content.get("msg") + if(content.get("msg").contains("该地址不支持估价!")) { + items['I'] = "该地址不支持估价!" + } + } + + +}) + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy index 87c14b741..4858d096f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy @@ -454,7 +454,7 @@ public class ExcelUtils{ def data = ExcelUtils.read(fileName, srcSheetIndex, 0) def body = data.subList(1, data.size()) - def nThread = Runtime.getRuntime().availableProcessors() * 3 + def nThread = 1 println nThread ExecutorService executorService = Executors.newFixedThreadPool(3) final AtomicInteger counter = new AtomicInteger(0) From 3c487b12495eae103d050cd09820927390eeae5c Mon Sep 17 00:00:00 2001 From: wangwenyan Date: Tue, 5 Mar 2024 15:10:50 +0800 Subject: [PATCH 11/22] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E6=A0=B7=E6=9C=AC?= =?UTF-8?q?=E7=BB=9F=E8=AE=A1=E8=84=9A=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/script/district/DistrictHouseBatch.groovy | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/district/DistrictHouseBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/district/DistrictHouseBatch.groovy index fa3d57a71..a66decde6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/district/DistrictHouseBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/district/DistrictHouseBatch.groovy @@ -3,6 +3,7 @@ package com.fangjia.script.district import com.fangjia.script.util.MongoUtils import com.mongodb.DBCursor +import org.apache.commons.lang3.StringUtils import java.util.List @@ -55,6 +56,15 @@ class DistrictHouseBatch extends BaseBatch { String key = district.get("name").toString().trim() + "_" + district.get("region") + "_" + district.get("city") long sellCount = sellCountMap.get(key) ? sellCountMap.get(key) : 0 long rentCount = rentCountMap.get(key) ? rentCountMap.get(key) : 0 + if(district.alias != null){ + for(String alia: district.alias){ + if(StringUtils.isNotBlank(alia)){ + key = alia.trim() + "_" + district.get("region") + "_" + district.get("city") + sellCount = sellCount + (sellCountMap.get(key)? sellCountMap.get(key) : 0) + rentCount = rentCount + (rentCountMap.get(key)? rentCountMap.get(key) : 0) + } + } + } saveToDistrictHouse(district.get('city'), district.get("region"), district.get("name"), sellCount, rentCount, sn) } @@ -109,6 +119,7 @@ class DistrictHouseBatch extends BaseBatch { fields.put("city", 1) fields.put("region", 1) fields.put("name", 1) + fields.put("alias", 1) def ret = [] From def60cbc6dff23982420816d558f633c11f4c48b Mon Sep 17 00:00:00 2001 From: wangwenyan Date: Tue, 5 Mar 2024 15:26:59 +0800 Subject: [PATCH 12/22] =?UTF-8?q?=E8=AE=BE=E7=BD=AE=E6=8C=87=E5=AF=BC?= =?UTF-8?q?=E4=BB=B7=E4=BF=AE=E6=94=B9=E6=97=B6=E9=97=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/script/district/DistrictHouseBatch.groovy | 11 +++-------- .../com/fangjia/script/wenyan/test/Test1.groovy | 10 +++++++--- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/district/DistrictHouseBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/district/DistrictHouseBatch.groovy index dbc92b4dd..fea372b37 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/district/DistrictHouseBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/district/DistrictHouseBatch.groovy @@ -69,7 +69,7 @@ class DistrictHouseBatch extends BaseBatch { for(DBObject district : districts) { def names = [] names << district.name - if(district.alias && districts.size()>0){ + if(district.alias && district.alias.size()>0){ for(def name : district.alias){ if(!names.contains(name)){ names << name @@ -81,14 +81,9 @@ class DistrictHouseBatch extends BaseBatch { long rentCount = 0 for(def name : names){ String key = name.toString().trim() + "_" + district.get("region") + "_" + district.get("city") - if(!sellCount){ - sellCount = sellCountMap.get(key) ? sellCountMap.get(key) : 0 - } - if(!rentCount){ - rentCount = rentCountMap.get(key) ? rentCountMap.get(key) : 0 - } + sellCount = sellCount + (sellCountMap.get(key) ? sellCountMap.get(key) : 0) + rentCount = rentCount + (rentCountMap.get(key) ? rentCountMap.get(key) : 0) } - saveToDistrictHouse(district.get('city'), district.get("region"), district.get("name"), sellCount, rentCount, sn) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/test/Test1.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/test/Test1.groovy index e6c27c934..ce4d04b65 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/test/Test1.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/test/Test1.groovy @@ -5,7 +5,11 @@ import com.fangjia.datastore.driver.DataStore import com.fangjia.script.util.DbUtils import com.mongodb.BasicDBObject -DSCollection seabird = new DataStore("192.168.0.38", 27007).getCollection("fangjia.seaweed") -seabird.find(DbUtils.buildSeaweedQuery("常州", "溧阳", "吾悦广场").append("if_sell", 1)).each { - println(it) +//、DSCollection seabird = new DataStore("192.168.0.38", 27007).getCollection("fangjia.seaweed") +//seabird.find(DbUtils.buildSeaweedQuery("常州", "溧阳", "吾悦广场").append("if_sell", 1)).each { +// println(it) +//} +long l = 0 +if(!l){ + println(l) } \ No newline at end of file From c2e26c5c17efa1313a26cb1600beddcdea5886a3 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 8 Mar 2024 10:36:40 +0800 Subject: [PATCH 13/22] =?UTF-8?q?=E5=8A=A0=E7=BA=BF=E7=A8=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/district/DistrictHouseBatch.groovy | 23 ++++++--- .../fangjia/script/gj/CitySellStatTest.groovy | 50 ++++++++++++++++--- .../gj/village/ailiyun/SignatureUtils.groovy | 5 ++ .../gj/xiamen/SuggestionDataSynch.groovy | 2 +- 4 files changed, 65 insertions(+), 15 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/district/DistrictHouseBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/district/DistrictHouseBatch.groovy index dbc92b4dd..394463a8b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/district/DistrictHouseBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/district/DistrictHouseBatch.groovy @@ -44,12 +44,17 @@ class DistrictHouseBatch extends BaseBatch { public void process(Object args) { // DBObject q = new BasicDBObject(); -// q.put("_id", new ObjectId("6501174f6f5638447b264189")); -// def obj = CommonDatautils.getHouseDataStoreConnection().findOne(q) -// obj.put("c_date", new Date()) -// CommonDatautils.getHouseDataStoreConnection().save(obj) +// q.put("cat", "sell"); +// q.put("city","常州") +// q.put("district_name","清河苑一区") +// def dbcursor = CommonDatautils.getHouseDataStoreConnection().find(q) +// for(def obj : dbcursor){ +// obj.put("c_date", new Date()) +// CommonDatautils.getHouseDataStoreConnection().save(obj) +// } -// CommonDatautils.getHouseDataStoreConnection().find(new BasicDBObject("districtname_key","新城帝景南区_武进_常州")).each{ + +// CommonDatautils.getHouseDataStoreConnection().find(new BasicDBObject("districtname_key","清河苑一区_溧阳_常州")).each{ // println(it) // } @@ -62,14 +67,15 @@ class DistrictHouseBatch extends BaseBatch { c++ try { long sn = Long.parseLong("${new Date().format('yyyyMMddHHmmssSSS')}") - List districts = findDistrictByCity(city) +// List districts = findDistrictByCity(city) + def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name","alias"]) Map sellCountMap = facetDistrictInHouse("sell", city); Map rentCountMap = facetDistrictInHouse("rent", city); println c + "\t" + city + "\t" + "\tsn:" + sn + "\t" + districts.size() + "\t" + DateUtil.formatToTightYYYYMMDDhhmmss(new Date()) for(DBObject district : districts) { def names = [] names << district.name - if(district.alias && districts.size()>0){ + if(district.alias && districts.alias.size()>0){ for(def name : district.alias){ if(!names.contains(name)){ names << name @@ -93,6 +99,7 @@ class DistrictHouseBatch extends BaseBatch { } HOUSE_COLL.remove(new BasicDBObject("city", city).append('sn', new BasicDBObject('$ne', sn))) + println c + " " + city }catch (Exception e) { e.printStackTrace(); } @@ -151,7 +158,7 @@ class DistrictHouseBatch extends BaseBatch { q.put('city', city) q.put('cat', 'district') q.put('visible', 0) -// q.put("name","新城帝景") + q.put("name","清河苑") DSCursor cur = SEAWEED.find(q, fields).batchSize(2000) Iterator iter = cur.iterator() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/CitySellStatTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/CitySellStatTest.groovy index fc989d72f..49b75a911 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/CitySellStatTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/CitySellStatTest.groovy @@ -1,5 +1,7 @@ package com.fangjia.script.gj +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.temp.shenjian.poi.entity.Community import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject @@ -12,7 +14,41 @@ class CitySellStatTest { static main(args) { def task = new CitySellStatTest(); - task.run() +// task.run() + task.cityPriceRun() + } + + def cityPriceRun(){ + def cityObjs = CommonDatautils.findCitys(); + def rets = [] + rets << ["序号","城市","城市级别","城市均价起始月","起始均价","城市均价最新月","最新均价"] + def index = 1 + for(def o : cityObjs){ + def ret = [] + ret << index + ret << o.display + ret << o.new_city_grade + def row1 = findCityMonthPrice(o.display, 1) + if(row1){ + ret << row1.date + ret << row1.avg_price + }else{ + ret << "" + ret << "" + } + def row2 = findCityMonthPrice(o.display, -1) + if(row2){ + ret << row2.date + ret << row2.avg_price + }else{ + ret << "" + ret << "" + } + rets << ret + index++ + } + + ExcelUtils.write("D:\\数据\\其它\\全国城市均价.xlsx", rets, "城市均价") } def run(){ @@ -25,18 +61,20 @@ class CitySellStatTest { },0) } - def findFirstMonth(def city){ + def findCityMonthPrice(def city, def asc){ def q = new BasicDBObject(); q.put("city", city) - def dbcursor = CITY_STAT_COLL.find(q).sort(new BasicDBObject("_id",1)).limit(1) - def month; + def dbcursor = CITY_STAT_COLL.find(q).sort(new BasicDBObject("date", asc)).limit(1) + def obj = null for(def o : dbcursor){ - month = o.date + obj = o } - return month + return obj } + + def findTotalMonth(def city, def month){ def q = new BasicDBObject(); q.put("city", city) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/ailiyun/SignatureUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/ailiyun/SignatureUtils.groovy index 6dd3714bb..b2dbb9204 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/ailiyun/SignatureUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/ailiyun/SignatureUtils.groovy @@ -200,4 +200,9 @@ public class SignatureUtils { } return null; } + + public static void main(String[] args){ + def a = getAliAddressStrc("上海市长宁区新泾三村") + println a + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SuggestionDataSynch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SuggestionDataSynch.groovy index 8aa6511ac..2896b5c0b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SuggestionDataSynch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SuggestionDataSynch.groovy @@ -21,7 +21,7 @@ import java.util.concurrent.atomic.AtomicLong class SuggestionDataSynch extends BaseBatch{ static def targetUrl = "http://192.168.0.12:9200/seaweed/" - public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(50) + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(100) def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; DBCollection SEAWEED_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed", MONGO100_URL) From 46b6bf7268f758ea8cf182ec397f436850446b73 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 8 Mar 2024 10:41:54 +0800 Subject: [PATCH 14/22] =?UTF-8?q?=E6=88=BF=E6=BA=90=E7=BB=9F=E8=AE=A1?= =?UTF-8?q?=E6=9B=B4=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/script/district/DistrictHouseBatch.groovy | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/district/DistrictHouseBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/district/DistrictHouseBatch.groovy index dbc8722a3..5542b2c16 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/district/DistrictHouseBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/district/DistrictHouseBatch.groovy @@ -87,8 +87,12 @@ class DistrictHouseBatch extends BaseBatch { long rentCount = 0 for(def name : names){ String key = name.toString().trim() + "_" + district.get("region") + "_" + district.get("city") - sellCount = sellCount + (sellCountMap.get(key) ? sellCountMap.get(key) : 0) - rentCount = rentCount + (rentCountMap.get(key) ? rentCountMap.get(key) : 0) + if(!sellCount){ + sellCount = sellCountMap.get(key) ? sellCountMap.get(key) : 0 + } + if(!rentCount){ + rentCount = rentCountMap.get(key) ? rentCountMap.get(key) : 0 + } } saveToDistrictHouse(district.get('city'), district.get("region"), district.get("name"), sellCount, rentCount, sn) } From 58e277a6a3c3984d7443e1d9f35ea05e1f918cbd Mon Sep 17 00:00:00 2001 From: goojiayangqi <1052755581@163.com> Date: Fri, 8 Mar 2024 15:15:20 +0800 Subject: [PATCH 15/22] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E5=AF=BC=E5=87=BA?= =?UTF-8?q?=E4=B8=A4=E5=BC=A0=E8=A1=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ronghe/ExportDistrictRongHe.groovy | 80 +++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy index 3a55f12b6..75d052af1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy @@ -69,6 +69,13 @@ class ExportDistrictRongHe extends BaseBatch{ exportBigdataCityPrice(queryDate) exportBigdataDistPrice(queryDate) exportBigdataCommPrice(queryDate) + exportBigdataCommPrice(queryDate) + // 新增两张数据表 + // avm_evaluate_rate_city_gj_h + // avm_evaluate_rate_comm_gj_h + exportEvaluateRateCity(queryDate) + exportEvaluateRateComm(queryDate) + //1月10日 新增5张表,导92城,增量数据,2月10日导344城,其中252城所有历史数据,92城增量,3月后可能不用提供数据 // exportCommunityBasePrice(lastMonth) // exportCommunityPdescribeGj(lastMonth) @@ -228,6 +235,79 @@ class ExportDistrictRongHe extends BaseBatch{ CcbFtpUtils.ftpUpload("rh",fileName, curDate) println "GUIDED_PRICE_GJ 上传Ftp完成......" } + // avm_evaluate_rate_city_gj_h gujia_clc_yrmo_val =202402 and gujia_clc_yrmo_val<=202403 gujia_clc_yrmo_val>=to_date('${startDateStr}','yyyy-MM-dd') and gujia_clc_yrmo_val<=to_date('${endDateStr}','yyyy-MM-dd') + def exportEvaluateRateComm(){ + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select urbn_id,gujia_bld_tp_val,gujia_bld_tp_val_coef,gujia_clc_yrmo_val,data_crt_tm,data_udt_tm,gujia_src_co from t_app_p9pha_plda.avm_evaluate_rate_city_gj_h where urbn_id='${cityCode}' and gujia_clc_yrmo_val = ${queryDate}"; + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.urbn_id + data << r.gujia_bld_tp_val + data << r.gujia_bld_tp_val_coef + data << r.gujia_clc_yrmo_val + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + } + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" + // FJW_000000_000000_EVALUATE_RATE_CITY_FLW_ADD_ + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_CITY_FLW_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_EVALUATE_RATE_CITY_FLW_ADD_${curDate}_0001.dat" + } + + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + + println "EVALUATE_RATE_CITY_FLW_ADD 数据文件输出完成......" + CcbFtpUtils.ftpUpload("rh",fileName, curDate) + println "EVALUATE_RATE_CITY_FLW_ADD 上传Ftp完成......" + } + + def exportEvaluateRateCity(){ + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select urbn_id,gujia_admndiv_id,gujia_bds_id,comm_nm,gujia_clc_yrmo_val,basic_price,area_avg,unitprc_avg,unitprc_avg,comm_type,area_avg_coef,unitprc_avg_coef,basic_price_coef,buildarea_coef,type_coef,data_crt_tm,data_udt_tm,gujia_src_co from t_app_p9pha_plda.avm_evaluate_rate_city_gj_h where urbn_id='${cityCode}' and gujia_clc_yrmo_val = ${queryDate} "; + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.urbn_id + data << r.gujia_admndiv_id + data << r.gujia_bds_id + data << r.comm_nm + data << r.gujia_clc_yrmo_val + data << r.basic_price + data << r.area_avg + data << r.unitprc_avg + data << r.comm_type + data << r.area_avg_coef + data << r.basic_price_coef + data << r.buildarea_coef + data << r.type_coef + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + } + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" + // FJW_000000_000000_EVALUATE_RATE_COMM_FLW_ADD_ avm_evaluate_rate_comm_gj_h + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_COMM_FLW_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMMUNITY_PRICE_GJ_ALL_${curDate}_0001.dat" + } + + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + + println "FJW_000000_000000_EVALUATE_RATE_COMM_FLW_ADD_ 数据文件输出完成......" + CcbFtpUtils.ftpUpload("rh",fileName, curDate) + println "FJW_000000_000000_EVALUATE_RATE_COMM_FLW_ADD_ 上传Ftp完成......" + } def exportBigdataCommbuildstruGJ(){ From bb42620c92ded15281a1f9fa5a6dab7c7533ead5 Mon Sep 17 00:00:00 2001 From: goojiayangqi <1052755581@163.com> Date: Fri, 8 Mar 2024 17:46:11 +0800 Subject: [PATCH 16/22] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E5=AF=BC=E5=87=BA?= =?UTF-8?q?=E4=B8=A4=E5=BC=A0=E8=A1=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ronghe/ExportDistrictRongHe.groovy | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy index 75d052af1..4d3833534 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy @@ -236,11 +236,11 @@ class ExportDistrictRongHe extends BaseBatch{ println "GUIDED_PRICE_GJ 上传Ftp完成......" } // avm_evaluate_rate_city_gj_h gujia_clc_yrmo_val =202402 and gujia_clc_yrmo_val<=202403 gujia_clc_yrmo_val>=to_date('${startDateStr}','yyyy-MM-dd') and gujia_clc_yrmo_val<=to_date('${endDateStr}','yyyy-MM-dd') - def exportEvaluateRateComm(){ + def exportEvaluateRateCity(queryDate){ def allRows = [] for(def city : citys) { def cityCode = cityCodeMap.get(city) - def sql = "select urbn_id,gujia_bld_tp_val,gujia_bld_tp_val_coef,gujia_clc_yrmo_val,data_crt_tm,data_udt_tm,gujia_src_co from t_app_p9pha_plda.avm_evaluate_rate_city_gj_h where urbn_id='${cityCode}' and gujia_clc_yrmo_val = ${queryDate}"; + def sql = "select urbn_id,gujia_bld_tp_val,gujia_bld_tp_val_coef,gujia_clc_yrmo_val,data_crt_tm,data_udt_tm,gujia_src_co from t_app_p9pha_plda.avm_evaluate_rate_city_gj_h where urbn_id='${cityCode}' and gujia_clc_yrmo_val = '${queryDate}'"; pgsqlInstance.eachRow(sql, { r -> def data = [] data << r.urbn_id @@ -256,23 +256,23 @@ class ExportDistrictRongHe extends BaseBatch{ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" // FJW_000000_000000_EVALUATE_RATE_CITY_FLW_ADD_ - def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_CITY_FLW_ADD_${curDate}_0001.dat" + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_CITY_FJW_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_EVALUATE_RATE_CITY_FLW_ADD_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_EVALUATE_RATE_CITY_FJW_ADD_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); - println "EVALUATE_RATE_CITY_FLW_ADD 数据文件输出完成......" + println "EVALUATE_RATE_CITY_FJW_ADD 数据文件输出完成......" CcbFtpUtils.ftpUpload("rh",fileName, curDate) - println "EVALUATE_RATE_CITY_FLW_ADD 上传Ftp完成......" + println "EVALUATE_RATE_CITY_FJW_ADD 上传Ftp完成......" } - def exportEvaluateRateCity(){ + def exportEvaluateRateComm(queryDate){ def allRows = [] for(def city : citys) { def cityCode = cityCodeMap.get(city) - def sql = "select urbn_id,gujia_admndiv_id,gujia_bds_id,comm_nm,gujia_clc_yrmo_val,basic_price,area_avg,unitprc_avg,unitprc_avg,comm_type,area_avg_coef,unitprc_avg_coef,basic_price_coef,buildarea_coef,type_coef,data_crt_tm,data_udt_tm,gujia_src_co from t_app_p9pha_plda.avm_evaluate_rate_city_gj_h where urbn_id='${cityCode}' and gujia_clc_yrmo_val = ${queryDate} "; + def sql = "select urbn_id,gujia_admndiv_id,gujia_bds_id,comm_nm,gujia_clc_yrmo_val,basic_price,area_avg,unitprc_avg,unitprc_avg,comm_type,area_avg_coef,unitprc_avg_coef,basic_price_coef,buildarea_coef,type_coef,data_crt_tm,data_udt_tm,gujia_src_co from t_app_p9pha_plda.avm_evaluate_rate_comm_gj_h where urbn_id='${cityCode}' and gujia_clc_yrmo_val = '${queryDate}' "; pgsqlInstance.eachRow(sql, { r -> def data = [] data << r.urbn_id @@ -297,16 +297,16 @@ class ExportDistrictRongHe extends BaseBatch{ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" // FJW_000000_000000_EVALUATE_RATE_COMM_FLW_ADD_ avm_evaluate_rate_comm_gj_h - def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_COMM_FLW_ADD_${curDate}_0001.dat" + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_COMM_FJW_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMMUNITY_PRICE_GJ_ALL_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_EVALUATE_RATE_COMM_FJW_ADD_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); - println "FJW_000000_000000_EVALUATE_RATE_COMM_FLW_ADD_ 数据文件输出完成......" + println "FJW_000000_000000_EVALUATE_RATE_COMM_FJW_ADD_ 数据文件输出完成......" CcbFtpUtils.ftpUpload("rh",fileName, curDate) - println "FJW_000000_000000_EVALUATE_RATE_COMM_FLW_ADD_ 上传Ftp完成......" + println "FJW_000000_000000_EVALUATE_RATE_COMM_FJW_ADD_ 上传Ftp完成......" } From 6e48ce05ee1f6343488c1499c0ea1774726d30bc Mon Sep 17 00:00:00 2001 From: goojiayangqi <1052755581@163.com> Date: Fri, 8 Mar 2024 17:56:16 +0800 Subject: [PATCH 17/22] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E5=AF=BC=E5=87=BA?= =?UTF-8?q?=E4=B8=A4=E5=BC=A0=E8=A1=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy | 1 - 1 file changed, 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy index 4d3833534..50cc6343f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy @@ -69,7 +69,6 @@ class ExportDistrictRongHe extends BaseBatch{ exportBigdataCityPrice(queryDate) exportBigdataDistPrice(queryDate) exportBigdataCommPrice(queryDate) - exportBigdataCommPrice(queryDate) // 新增两张数据表 // avm_evaluate_rate_city_gj_h // avm_evaluate_rate_comm_gj_h From 1635d3892d0d88e5e5f0fef729b79fd99675afcb Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 13 Mar 2024 09:26:06 +0800 Subject: [PATCH 18/22] =?UTF-8?q?=E5=9B=A0=E5=8A=9E=E5=85=AC=E6=A5=BC?= =?UTF-8?q?=E9=9D=A2=E7=A7=AF=E8=BF=87=E5=A4=A7=EF=BC=8C=E4=BB=A5=E5=89=8D?= =?UTF-8?q?=E9=92=88=E5=AF=B9=E4=BD=8F=E5=AE=85=EF=BC=8C=E7=8E=B0=E5=B0=86?= =?UTF-8?q?1000-=E3=80=8B3000?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia-open/src/main/webapp/weiXinEvaluation/bhform.ftl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-open/src/main/webapp/weiXinEvaluation/bhform.ftl b/projects/fangjia-open/src/main/webapp/weiXinEvaluation/bhform.ftl index 5160cbd1a..d9b670394 100644 --- a/projects/fangjia-open/src/main/webapp/weiXinEvaluation/bhform.ftl +++ b/projects/fangjia-open/src/main/webapp/weiXinEvaluation/bhform.ftl @@ -133,7 +133,7 @@
浜ц瘉寤虹瓚闈㈢Н锛�
-
From 6ac16b027fe11c2ed171970e1a91b7234875ea62 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 15 Mar 2024 15:25:12 +0800 Subject: [PATCH 19/22] update --- .../BatchAddPersonGuiPrice.groovy | 19 +-- .../ccbevaluation/PgsqlCommUtils.groovy | 9 ++ .../script/commData/CommonDatautils.groovy | 118 +++++++++++++++ .../fuzhou/DistrictSellStatBatch.groovy | 20 ++- .../script/fuzhou/FzDataResultStat.groovy | 32 +++- .../script/fuzhou/FzDistrictMatch.groovy | 36 ++++- .../fangjia/script/gj/YouDaDataParser.groovy | 2 +- .../script/gj/city/CityStatExport.groovy | 29 +++- .../gj/data_handle/TownshipPrcoess.groovy | 51 +++++++ .../gj/district/GXDdistrictMatch.groovy | 138 ++++++++++++++++++ .../fangjia/script/gj/es/EsSelectTest.groovy | 9 +- .../script/gj/village/QiliangTest.groovy | 107 ++++++++++++++ .../script/thirdPriceCrawl/DataStat.groovy | 6 +- .../TestCrawlTotalCount.groovy | 4 +- 14 files changed, 551 insertions(+), 29 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownshipPrcoess.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/GXDdistrictMatch.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/QiliangTest.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BatchAddPersonGuiPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BatchAddPersonGuiPrice.groovy index a69810dab..ea0274d5f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BatchAddPersonGuiPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BatchAddPersonGuiPrice.groovy @@ -24,15 +24,16 @@ class BatchAddPersonGuiPrice { def run(){ - def path = "D:\\数据\\价格\\常南苏徐无人工指导价录入.xlsx" + def path = "D:\\数据\\福州\\价格比对_20240315 -价格入库.xlsx" def datas = ExcelUtils.read(path, 0, 1) for(def d : datas){ - def row = findGuidingPrice(d['B'], d['C'], d['D']) + def row = findGuidingPrice(d['B'], d['C'], d['C']) if(!row){ - def obj = CommonDatautils.findDistrictById(d['A']) - save(d['B'], d['C'], d['D'], d['M'], obj.estate_type2) +// def obj = CommonDatautils.findDistrictById(d['A']) + def obj = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district") + save(d['A'], d['B'], d['C'], d['Q'], obj.estate_type2, d['P']) //发重算均价命令 - addPrice(d['B'], d['C'], d['D'], obj.estate_type2, DateUtils.parseFromTightByPattern('2023-01-01','yyyy-MM-dd'),new Date()) + addPrice(d['A'], d['B'], d['C'], obj.estate_type2, DateUtils.parseFromTightByPattern('2023-03-01','yyyy-MM-dd'),new Date()) } } } @@ -43,13 +44,13 @@ class BatchAddPersonGuiPrice { q.put("city", city) q.put("region", region) q.put("name", name) - q.put("guiding_person" , "付贵生") - q.put("guiding_date", new BasicDBObject('$gte', DateUtils.parseFromTightByPattern("2022-11-01", "yyyy-MM-dd"))) +// q.put("guiding_person" , person) + q.put("guiding_date", new BasicDBObject('$gte', DateUtils.parseFromTightByPattern("2024-03-20", "yyyy-MM-dd"))) def dbo = GUIDING_PRICE_COLL.findOne(q) return dbo } - def save(city, region, name, price, estate_type){ + def save(city, region, name, price, estate_type, person){ DBObject obj = new BasicDBObject(); obj.put("city", city); obj.put("region", region); @@ -62,7 +63,7 @@ class BatchAddPersonGuiPrice { obj.put("m_date", new Date()); obj.put("estate_type" , estate_type); obj.put("house_type" , 0) - obj.put("guiding_person" , "陈杰") + obj.put("guiding_person" , person) GUIDING_PRICE_COLL.save(obj); } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/PgsqlCommUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/PgsqlCommUtils.groovy index 2fd7e0c2f..cfbd78a2d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/PgsqlCommUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/PgsqlCommUtils.groovy @@ -27,4 +27,13 @@ class PgsqlCommUtils { }); return count } + + public static def findDistrictMap(){ + def sql = "select urbn_id,gujia_admndiv_id,gujia_admndiv_nm from base.t0831_bigdata_district_gj_h" + def map = [:] + sqlInstance.eachRow(sql,{r-> + map.put(r.urbn_id+"#"+r.gujia_admndiv_id, r.gujia_admndiv_nm) + }); + return map + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index ad59e008c..1af08d6b3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -1175,6 +1175,28 @@ class CommonDatautils { return null; } + + + public static def findAllStatusDistrict(String city,String region, String name, String cat){ + + DBObject q = new BasicDBObject(); + q.put("city", city); + q.put("region", region); + q.put("name", name); + q.put("cat", cat); + + def rets = [] + def cursor = SEAWEED_MONGO_COLL.find(q); + if(cursor){ + for(def obj : cursor){ + rets << obj; + } + + } + return rets; + } + + public static def findDistrictsByName(String city, String name, String cat){ DBObject q = new BasicDBObject(); @@ -1537,6 +1559,32 @@ class CommonDatautils { } + static def findAllStatusDistrictByAlias(def city, def region, def name, def cat){ + DBObject q = new BasicDBObject(); + if(region){ + q.put('region', region) + } + + q.put('city',city) + q.put('cat', cat) + + def names = []; + names << name; + BasicDBObject query = new BasicDBObject(); + query.putAt(QueryOperators.IN, names); + q.put("alias", query); + + + def dsCursor = getDataStoreConnection().find(q); + def ret = []; + for(def obj : dsCursor){ + ret << obj; + } + + return ret; + } + + static def findDistrictByAddress(def city, def region, def address, def cat){ DBObject q = new BasicDBObject(); if(region){ @@ -3919,6 +3967,29 @@ class CommonDatautils { return avg_price } + static def findMatchTrades(city, region, name, startDate, endDate){ + def q = new BasicDBObject() + q.put("city", city) + q.put("match_region", region) + q.put("match_name", name) + q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)).append('$lt',DateUtils.parseFromTightYYYYMMDD(endDate))) + def dbcorsor = DEAL_PRICE_COLL.find(q) + def avg_price = 0 + def total_price = 0 + def count = 0 + + for(def d : dbcorsor){ + if(d.avg_price){ + total_price = total_price + d.avg_price + count++ + } + } + if(count > 3){ + avg_price = total_price/count + } + return avg_price + } + /** @@ -3993,6 +4064,32 @@ class CommonDatautils { return total; } + + static def findLogs(user_id, requestUri, startDate, endDate){ + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("user_id", user_id)).must(QueryBuilders.termQuery("request_uri", requestUri)) + if(startDate){ + def startTime = DateUtils.parseFromTightYYYYMMDD(startDate) + def endTime = DateUtils.parseFromTightYYYYMMDD(endDate) + def date_range_cdate = QueryBuilders.rangeQuery("c_date").from(startTime.getTime()).to(endTime.getTime()).includeLower(true).includeUpper(false); + queryBuilder.must(date_range_cdate); + } + + def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "open_uri_log", queryBuilder); + if(houses){ + for (int k=0;k3000){ dataMap.put("beike_esf_price", db.beike_esf_price) } + if(!dataMap.containsKey("maitian_esf_price") && db.maitian_esf_price && db.maitian_esf_price>3000){ + dataMap.put("maitian_esf_price", db.maitian_esf_price) + } + } return map } @@ -4337,6 +4438,23 @@ class CommonDatautils { } + static def findHouseAvgPrice(city, region , start, end){ + def prices = [] + def sql = "select avg_price from latest_seawater where cat='sell' and city='${city}' and p_date > ${start} and p_date<${end} and (root='lianjia.com' or root='ke.com')" + if(region){ + sql = sql + " and region='${region}'" + } + ElasticUtils.executeSqlQuery(sql, { + prices << it.avg_price + }) + if(prices && prices.size()>3){ + def avg_price = (int)(prices.sum()/prices.size()) + println region + " "+ prices.size() +" "+avg_price +" "+start+" "+end + return avg_price + } + return null + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy index 0b6a136ec..beeafcb09 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy @@ -7,6 +7,7 @@ import com.fangjia.script.Watcher import com.fangjia.script.util.CommonUtils import com.fangjia.script.util.DateUtils import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection @@ -74,7 +75,7 @@ class DistrictSellStatBatch extends BaseBatch { def day = new Date().getDate() - def startMDate = DateUtils.truncateDay(new Date()-1) + def startMDate = DateUtils.truncateDay(new Date()-7) def endMDate = DateUtils.truncateDay(new Date()) def index = 0 @@ -144,7 +145,8 @@ class DistrictSellStatBatch extends BaseBatch { void run() { try { totalCount.addAndGet(1) - def key = dist.region +"##" + dist.name + def type = dist.estate_type2=="别墅"?"别墅":"小区" + def key = dist.region +"##" + dist.name+"##"+ type def weekendPriceObj = weekendPriceMap.get(key) if(weekendPriceObj){ @@ -153,8 +155,13 @@ class DistrictSellStatBatch extends BaseBatch { def lastYearPriceObj = lastYearPriceMap.get(key) double mom = 0d - if(lastMonthPriceObj) { - mom = DataUtil.roundDouble(((weekendPriceObj.avg_price - lastMonthPriceObj.avg_price)/(double)lastMonthPriceObj.avg_price),4) + if(lastMonthPriceObj && lastMonthPriceObj.avg_price>0) { + try{ + mom = DataUtil.roundDouble(((weekendPriceObj.avg_price - lastMonthPriceObj.avg_price)/(double)lastMonthPriceObj.avg_price),4) + }catch(Exception e){ + println key+" 出错................." + } + } def curPrice = weekendPriceObj.avg_price @@ -192,7 +199,7 @@ class DistrictSellStatBatch extends BaseBatch { curPrice = 0 } if(curPriceObj){ - if(curPrice!=weekendPriceObj.avg_price){ + if(curPrice!=curPriceObj.avg_price){ def data = new BasicDBObject() data.put("m_date", new Date()) data.put("commid", dist._id.toString()) @@ -263,7 +270,7 @@ class DistrictSellStatBatch extends BaseBatch { } } if(!priceMap.containsKey(d.region +"##"+d.district_name)){ - priceMap.put(d.region +"##"+d.district_name, d) + priceMap.put(d.region +"##"+d.district_name+"##"+d.estate_type, d) } } return priceMap @@ -355,4 +362,5 @@ class DistrictSellStatBatch extends BaseBatch { // } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy index 02d267a37..893ad9876 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy @@ -9,6 +9,7 @@ import com.mongodb.Bytes import com.mongodb.DBCollection import com.mongodb.DBCursor import com.mongodb.DBObject +import com.rabbitmq.client.AMQP import com.rabbitmq.client.impl.AMQImpl import org.apache.commons.io.FileUtils import org.bson.types.ObjectId @@ -28,10 +29,11 @@ class FzDataResultStat { static main(args) { def task = new FzDataResultStat() - task.priceMarkupRun(); - task.regionPriceTrendRun() +// task.priceMarkupRun(); +// task.regionPriceTrendRun() //// task.margeExcel() // task.saveCsvData() + task.testNewPrice() } def margeExcel(){ @@ -163,7 +165,7 @@ class FzDataResultStat { def exportRegion(city, months){ - def regions = ["鼓楼","仓山","台江","晋安","马尾","长乐"]; + def regions = ["鼓楼","仓山","台江","晋安","马尾","长乐","福清","闽侯","连江","罗源","闽清","永泰","平潭"]; def datas =[] def header = ["城市", "区域"] for(def m : months){ @@ -206,8 +208,12 @@ class FzDataResultStat { DBObject dbo = cur.next() prices << dbo.get("avg_price"); } - def avg_price = (int)(prices.sum()/prices.size()) - return avg_price; + if(prices){ + def avg_price = (int)(prices.sum()/prices.size()) + return avg_price; + } + + return null; } @@ -293,4 +299,20 @@ class FzDataResultStat { } return rets } + + def testNewPrice(){ + ExcelUtils.modify("D:\\数据\\福州\\价格比对_20240315 -价格入库.xlsx",{d-> + if(d['A']){ + def q = new BasicDBObject() + q.put("city", d['A']) + q.put("region",d['B']) + q.put("name",d['C']) + q.put("month", 202402) + def obj = FZ_DISTRICT_STAT_COLL.findOne(q) + if(obj){ + d['S'] = obj.avg_price + } + } + }) + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDistrictMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDistrictMatch.groovy index cae471308..baccfa579 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDistrictMatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDistrictMatch.groovy @@ -19,7 +19,41 @@ class FzDistrictMatch { def task = new FzDistrictMatch() // task.run(); // task.addPrice() - task.addThirdPrice() +// task.addThirdPrice() + + task.addThirdPrice2("D:\\数据\\福州\\价格比对_20240315.xlsx") + } + + def addThirdPrice2(path){ + + ExcelUtils.multiThreadingModify(path, { row -> + def cityMap = CommonDatautils.findThirdPartyPrice(row['A'], row['B'], row['C'], 202312, 202402) + def priceMap = cityMap.get(row['A']+"##"+row['B']+"##"+row['C']) + if(priceMap){ + if(priceMap.containsKey("lianjia_esf_price")){ + row['J'] = priceMap.get("lianjia_esf_price") + } + + if(!row['J'] && priceMap.containsKey("beike_esf_price")){ + row['J'] = priceMap.get("beike_esf_price") + } + + if(priceMap.containsKey("maitian_esf_price")){ + row['K'] = priceMap.get("maitian_esf_price") + } + } + + def tradeAvgPrice = CommonDatautils.findMatchTrades(row['A'], row['B'], row['C'], "2023-08-01", "2024-01-31"); + if(tradeAvgPrice){ + row['L'] = (int)tradeAvgPrice + } + + def priceObj = CommonDatautils.findDistrictPrice(row['A'], row['B'], row['C']) + if(priceObj){ + row['M'] = priceObj.avg_price_2b + } + + }) } def addThirdPrice(){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy index c40aca736..0db9c5bb7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy @@ -52,7 +52,7 @@ class YouDaDataParser { static main(args) { def task = new YouDaDataParser(); // task.newHouseparse();//友达一手房存入mongo -// task.esfParser("D:\\数据\\友达\\2024年2月\\查询1.DBF");//友达二手房存入mongo + task.esfParser("D:\\数据\\友达\\2024年2月\\查询1.DBF");//友达二手房存入mongo // task.newhouseToSaveEs();//新房保存到es task.esfhouseToSaveEs();//二手房保存到es // task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/city/CityStatExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/city/CityStatExport.groovy index 3289104fe..7ecf720df 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/city/CityStatExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/city/CityStatExport.groovy @@ -31,7 +31,34 @@ class CityStatExport { // task.regionStat() // task.stat() // task.houseStat() - task.addFieldStat() +// task.addFieldStat() + + task.exportRegionAvgPrice() + } + + def exportRegionAvgPrice(){ + def regions = ["鼓楼","仓山","台江","晋安","马尾","长乐","福清","闽侯","连江","罗源","闽清","永泰","平潭"]; + def add = -9; + def months = [] + while(add<=0){ + months << DateUtils.addMonth(new Date(), add) + add++ + } + def rets = [] + rets << months + for(def region : regions){ + def row = [] + row << region + for(def i=0; i + if(!d['G']){ +// def name1 = d['E'].replace("街道","").replace("镇","").replace("乡","") +// def name2 = d['F'].replace("街道","").replace("镇","").replace("乡","") +// if(name1 == name2){ +// d['G'] = "不处理" +// } + + def lists1 = findCommunityTown(community_CONN, d['C'], d['D'], d['E']) + def lists2 = findCommunityTown(community_CONN, d['C'], d['D'], d['F']) + if(lists1 && lists2){ + d['G'] = "需删除" + } + } + }) + } + + + def findCommunityTown(community_CONN, city, region, town){ + def q = new BasicDBObject() + q.put("city_name", city) + q.put("area_name", region) + q.put("street_name", town) + def rets = community_CONN.find(q).toList() + return rets; + + } +} + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/GXDdistrictMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/GXDdistrictMatch.groovy new file mode 100644 index 000000000..85b3a9a65 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/GXDdistrictMatch.groovy @@ -0,0 +1,138 @@ +package com.fangjia.script.gj.district + +import com.fangjia.core.RegionData +import com.fangjia.script.ccbevaluation.CcbCommonUtils +import com.fangjia.script.ccbevaluation.PgsqlCommUtils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.ExcelUtils + +class GXDdistrictMatch { + + static def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + static def regionCodeDataMap = PgsqlCommUtils.findDistrictMap() + + static main(args) { + def task = new GXDdistrictMatch(); + task.run() + } + + def run(){ + def fileName = "D:\\数据\\国信达\\国信达11城2.xlsx" + ExcelUtils.multiThreadingModify(fileName, { d -> + if(!d['V']){ + def city = getCityFromCityCode(d['A']) + def region = regionCodeDataMap.get(d['A'] + "#" + d['E']) + if (city && region) { + region = RegionData.formatRegion(city, region) + if (region) { + d = match(city, region, d) + } else { + println d['A'] + "#" + d['E'] + " 区域无法格式化" + } + } else { + println d['A'] + "#" + d['E'] + " 无法格式化" + } + } + }) + } + + + def match(city, region, d){ + d = matching(city, region, d, d['C'], d['D'], "") + if(!d['U']){ + if(d['C'].contains("小区")){ + d = matching(city, region, d, d['C'].replace("小区",""), d['D'], "(Update)") + }else{ + d = matching(city, region, d, d['C']+"小区", d['D'], "(Update)") + } + if(!d['U']){ + d = matching2(city, region, d, d['C'], d['D'], "") + } + + if(!d['U']){ + if(d['C'].contains("小区")){ + d = matching2(city, region, d, d['C'].replace("小区",""), d['D'], "(Update)") + }else{ + d = matching2(city, region, d, d['C']+"小区", d['D'], "(Update)") + } + } + + } + } + + + def matching(city, region, d, name, alias, str){ + //正常状态下有没有 + def rets = CommonDatautils.findDistrictList(city, region, name, "district") + if(rets){ + if(rets.size()==1){ + d = setRowValue(rets[0], d, "名称匹配"+str) + }else{ + d['U'] = "名称在多个小区中"+str + } + }else{ + rets = CommonDatautils.findDistrictByAlias(city, region, name, "district") + if(rets && rets.size()==1){ + d = setRowValue(rets[0], d, "名称在别名中"+str) + }else{ + if (d['D']) { + rets = CommonDatautils.findDistrictByAlias(city, region, alias, "district") + if (rets) { + if (rets.size() == 1) { + d = setRowValue(rets[0], d, "别名匹配"+str) + } else { + d['U'] = "别名在多个小区中"+str + } + } + } + } + } + + return d; + } + + def matching2(city, region, d, name, alias, str){ + def rets = CommonDatautils.findAllStatusDistrict(city, region, name, "district")//包含删除的 + if(rets){ + if (rets.size() == 1) { + d = setRowValue(rets[0], d, "名称匹配(非正常)"+str) + } + }else{ + + rets = CommonDatautils.findAllStatusDistrictByAlias(city, region, name, "district") + if (rets && rets.size() == 1) { + d = setRowValue(rets[0], d, "别名匹配(非正常)"+str) + }else{ + rets = CommonDatautils.findAllStatusDistrictByAlias(city, region, alias, "district") + if (rets.size() == 1) { + d = setRowValue(rets[0], d, "别名匹配(非正常)"+str) + } + } + } + return d; + } + + def setRowValue(dbo, d, remark){ + if (dbo) { + d['V'] = dbo.city + d['W'] = dbo.region + d['X'] = dbo.name + d['Y'] = dbo.estate_type2 + d['Z'] = dbo.status + d['AA'] = dbo.name_cat + d['U'] = remark + } + return d; + } + + def getCityFromCityCode(cityCode){ + def city = null + for(def m : cityCodeDataMap){ + if(m.value == cityCode){ + city = m.key + break; + } + } + return city; + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsSelectTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsSelectTest.groovy index 6f2307343..d259d858b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsSelectTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsSelectTest.groovy @@ -1,6 +1,7 @@ package com.fangjia.script.gj.es import com.fangjia.core.util.DateUtil +import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.util.ElasticUtils import org.elasticsearch.index.query.QueryBuilder import org.elasticsearch.index.query.QueryBuilders @@ -16,7 +17,13 @@ class EsSelectTest { static main(args) { def task = new EsSelectTest() - task.run() +// task.run() + task.test() + } + + + def test(){ + def ret = CommonDatautils.findLogs("1007703", "/property/evaluate", "2024-03-13", "2024-03-16") } def run(){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/QiliangTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/QiliangTest.groovy new file mode 100644 index 000000000..c153747fc --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/QiliangTest.groovy @@ -0,0 +1,107 @@ +package com.fangjia.script.gj.village + +import com.fangjia.core.util.HttpRequest +import com.fangjia.script.temp.shenjian.util.HttpUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.yinjihaun.util.HttpClient +import net.sf.json.JSONObject + +class QiliangTest { + + static main(args) { + def task = new QiliangTest() + task.run() + } + + def run(){//地址测试-2024.3.4-结果.xlsx + ExcelUtils.modify("D:\\数据\\省联社\\对比.xlsx", { row -> + if(!row['N']){ + def address = null + if(row['B'] && row['C'] && !row['C'].toString().contains(row['B'])){ + address = row['B']+row['C'] + }else{ + address = row['C'] + } + if(address){ + def result = getHttpResult(address) + if(result && result.code==20000){ + def data = result.data + if(data){ + if(data.province && data.province!='null'){ + row['N'] = data.province + } + if(data.city && data.city!='null'){ + row['O'] = data.city + } + if(data.district && data.district!='null'){ + row['P'] = data.district + } + if(data.town && data.town!='null'){ + row['Q'] = data.town + } + if(data.village && data.village!='null'){ + row['R'] = data.village + } + + if(data.areaCode && data.areaCode!='null'){ + row['S'] = data.areaCode + } + + if(data.farmer == "0"){ + row['T'] = "非农" + }else{ + row['T'] = "农" + } + if(data.isGovt == "0"){ + row['U'] = "否" + }else{ + row['U'] = "是" + } + + def list = data.indusParkList + if(list && list!="null"){ + def s = "" + for(def r : list){ + s = s + r.indusParkName +"-"+r.indusParkLevel+"\n" + } + row['V'] = s + } + if(data.govtAddr && data.govtAddr!='null'){ + row['W'] = data.govtAddr?data.govtAddr:'' + } + + } + } + } + } + }) + } + + def getHttpResult(address){ + def data = new JSONObject() + data.put("address", address) + data.put("type","1") + + Map headers = new HashMap(); + headers.put("Accept", "application/json, text/plain, */*"); + headers.put("Accept-Encoding", "gzip, deflate"); + headers.put("Accept-Language","zh-CN,zh;q=0.9") + headers.put("Content-Type","application/json;charset=UTF-8") + headers.put("Host","146.56.217.44:9010") + headers.put("Origin","http://146.56.217.44:8088") + headers.put("Proxy-Connection","keep-alive") + headers.put("Referer","http://146.56.217.44:8088/") + headers.put("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36") + + HttpRequest request1 = new HttpRequest() + try{ + String res = request1.postWithBody("http://146.56.217.44:9010/uams/farmer/all/v3/sign", JSONObject.fromObject(data.toString()), headers, "UTF-8") + def response = JSONObject.fromObject(res) + println response + return response + }catch(Exception e){ + e.printStackTrace() + } + return null; + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy index 49136a392..42972bc05 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy @@ -23,8 +23,8 @@ class DataStat { // for(def city :citys){ // println city // } -// task.run(); - task.stat() + task.run(); +// task.stat() // def citys = ["福州", "厦门", "泉州", "莆田", "宁德", "三明", "龙岩", "南平", "漳州"] // def rets = [] @@ -76,7 +76,7 @@ class DataStat { } def stat(){ - def dates = [202301,202302,202303,202304,202305,202306,202307,202308,202309,202310,202311,202401] + def dates = [202303,202304,202305,202306,202307,202308,202309,202310,202311,202401,202402] def sources = ["时间","我爱我家","兔博士","吉屋","贝壳找房","乐有家","365淘房","诸葛找房","链家","Q房","麦田","中原地产","安居客","搜房","C21","城市房产","幸福里"] // crawlCount(dates,sources); matchCount(dates,sources); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy index e396b2e96..a53d2d7c7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy @@ -14,8 +14,8 @@ class TestCrawlTotalCount { def run(){ def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋","幸福里"] for(def source :sources){ - def count1 = findTotalCount(source, 202307); - def count2 = findTotalCount(source, 202401); + def count1 = findTotalCount(source, 202401); + def count2 = findTotalCount(source, 202402); println source +" "+count1 +" "+count2 } } From 98e8db125bddbdf2343ffbb1fbb6218407574bdd Mon Sep 17 00:00:00 2001 From: mjt <13607600062@163.com> Date: Mon, 1 Apr 2024 12:28:55 +0800 Subject: [PATCH 20/22] =?UTF-8?q?=E5=85=89=E5=A4=A7=E9=93=B6=E8=A1=8C?= =?UTF-8?q?=E6=B6=89=E5=86=9C=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../open/controllers/GdBankController.java | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 projects/fangjia-open/src/main/java/com/fangjia/open/controllers/GdBankController.java diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/GdBankController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/GdBankController.java new file mode 100644 index 000000000..845358f5b --- /dev/null +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/GdBankController.java @@ -0,0 +1,71 @@ +package com.fangjia.open.controllers; + +import com.fangjia.core.util.HttpRequest; +import com.fangjia.open.interceptors.Authority; +import com.fangjia.open.interceptors.Role; +import com.fangjia.open.model.AddressModel; +import com.fangjia.open.model.BasicModel; +import com.fangjia.open.model.RestfulModelAndView; +import net.sf.json.JSONObject; +import org.apache.commons.lang.StringUtils; + +import java.util.HashMap; +import java.util.Map; + +/** + * 光大接口 + */ +@Authority(role = Role.USER) +public class GdBankController { + + /** + * 涉农识别-新规 + * + * @param model + * @return + */ + public Object villageIdentifyAction(AddressModel model) { + + RestfulModelAndView mav = new RestfulModelAndView(); + if (StringUtils.isBlank(model.getAddress())) { + mav.setModel(BasicModel.buildBadRequestModel("地址不能为空")); + return mav; + } + + HttpRequest request = new HttpRequest(); + JSONObject json = null; + Map params = new HashMap(); + params.put("city", StringUtils.isNotBlank(model.getCity()) ? model.getCity() : null); + params.put("type", StringUtils.isNotBlank(model.getType()) ? model.getType() : null); + params.put("redis", (model.getRadius() != null && model.getRadius() != 0) ? model.getRadius().toString() : null); + params.put("address", model.getAddress()); + params.put("showFullValue", "2"); + try { + json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/villageIdentify", params, "UTF-8"); + } catch (Exception e) { + try { + json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/villageIdentify", params, "UTF-8"); + } catch (Exception ex) { + ex.printStackTrace(); + } + } + if (json != null) { + int code = json.getInt("msgCode"); + String msg = json.getString("msg"); + BasicModel basicModel = new BasicModel(code, msg); + if (json.containsKey("result")) { + JSONObject result = json.getJSONObject("result"); + if (("是").equals(result.get("countyLevelCity"))){ + result.put("prefectureLevelCity", "否"); + } else if(("否").equals(result.get("countyLevelCity"))){ + result.put("prefectureLevelCity", "是"); + } + basicModel.setResult(result); + } + mav.setModel(basicModel); + } else { + mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中...")); + } + return mav; + } +} From d83d586b2fe059394dc2049692f36cd15c96e1f8 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 2 Apr 2024 14:20:12 +0800 Subject: [PATCH 21/22] =?UTF-8?q?=E5=AF=BC=E5=87=BA=E8=84=9A=E6=9C=AC?= =?UTF-8?q?=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...rtMonitorSquareRentCalcCommandBatch.groovy | 1 + .../DealAndThirdPriceDataProcess.groovy | 41 +- .../ronghe/BigdataCommunityGJRHBatch.groovy | 14 +- .../script/commData/CommonDatautils.groovy | 146 +++++- .../script/fuzhou/FzDistrictMatch.groovy | 87 +++- .../com/fangjia/script/fuzhou/Test1.groovy | 2 +- .../fangjia/script/gj/AddAvgpriceBatch.groovy | 10 +- .../fangjia/script/gj/EsSearchUtils.groovy | 26 +- .../script/gj/commStat/CityStat.groovy | 11 +- .../script/gj/data/ZhengzhouBlankStat.groovy | 112 +++++ .../script/gj/data_handle/DevProcess.groovy | 86 ++++ .../gj/data_handle/GXDdistrictMatch.groovy | 467 ++++++++++++++++++ .../gj/data_handle/TradeDatasDistBatch.groovy | 13 + .../gj/district/FangjiaClientUtils.groovy | 10 +- .../gj/district/GXDdistrictMatch.groovy | 138 ------ .../fangjia/script/gj/es/EsSelectTest.groovy | 2 +- .../script/gj/village/QiliangTest.groovy | 94 +++- .../com/fangjia/script/util/ExcelUtils.groovy | 2 +- 18 files changed, 1073 insertions(+), 189 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/ZhengzhouBlankStat.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/GXDdistrictMatch.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TradeDatasDistBatch.groovy delete mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/GXDdistrictMatch.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/averager/InsertMonitorSquareRentCalcCommandBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/averager/InsertMonitorSquareRentCalcCommandBatch.groovy index dc3ee5d27..9be5b8c21 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/averager/InsertMonitorSquareRentCalcCommandBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/averager/InsertMonitorSquareRentCalcCommandBatch.groovy @@ -21,6 +21,7 @@ import org.bson.types.ObjectId /** * 通知计算二手房均价 + * 发送平米租金命令 * @author duxiaoyu * */ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy index 8bc6aff7c..fca28ce3e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy @@ -387,26 +387,43 @@ class DealPriceDataProcess { } def deleteThirdPartyData(){ - def citys = CcbCommonUtils.citys_all +// ,"陵水","塔城","保亭","长沙","苏州","南昌","沧州","文昌","衡水" + def citys = ["469005"]//CcbCommonUtils.citys_all def month = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));//上个月日期 def rows = [] def cityCodeDatas = CcbCommonUtils.findCityCodeValue() def set = new HashSet() - for(def city : citys) { - def rets = findPeGteThirdParty2(city, DateUtils.formatDateByPattern(new Date(),"yyyyMM"))//当前月日期 - def total = 0 - println city +" "+rets.size() + for(def cityid : citys) { + def rets = findPeGteThirdParty2(cityid, DateUtils.formatDateByPattern(new Date(),"yyyyMM"))//当前月日期 + println cityid +" "+rets.size() if(rets){ - set.add(city) + set.add(cityid) } for(def m : rets){ - def cityId = cityCodeDatas.get(city) +// def cityId = cityCodeDatas.get(city) // def sql = "delete from BIGDATA_THIRD_PARTY_PRICE_GJ where cityid='${cityId}' and commid='${m.commid}' and price_date=${month} and community_price=${m.price}" // sqlInstance236.executeUpdate(sql) + def querySql = "select count(1) as count from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} and gujia_bds_comm_avprc=${m.price}" + def total = 0 + pgsqlInstance.eachRow(querySql, { r -> + total = r.count + }) + if(total>0){ + println cityid +" "+m.commid+" (按价格)删除:"+total + def sql = "delete from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} and gujia_bds_comm_avprc=${m.price}" + pgsqlInstance.executeUpdate(sql) + }else{ - def sql = "delete from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityId}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} and gujia_bds_comm_avprc=${m.price}" - pgsqlInstance.executeUpdate(sql) - + querySql = "select count(1) as count from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} " + pgsqlInstance.eachRow(querySql, { r -> + total = r.count + }) + if(total>0){ + println cityId +" "+m.commid+" (按小区)删除:"+total + def sql = "delete from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} " + pgsqlInstance.executeUpdate(sql) + } + } } } println set.toString() @@ -432,8 +449,8 @@ class DealPriceDataProcess { return rets } - def findPeGteThirdParty2(cityName, yearmonth){ - def sql = "SELECT gujia_comm_id,gujiabds_to_mkprc_mny from t_app_p9pha_plda.avm_evaluation_item_gj_h where gujiabds_clc_yrmo_val=${yearmonth} and gujia_data_tp='第三方' and urbn_nm='${cityName}' and gujia_mns_rto>20" + def findPeGteThirdParty2(cityid, yearmonth){ + def sql = "SELECT gujia_comm_id,gujiabds_to_mkprc_mny from t_app_p9pha_plda.avm_evaluation_item_gj_h where gujiabds_clc_yrmo_val=${yearmonth} and gujia_data_tp='第三方' and urbn_id='${cityid}' and gujia_mns_rto>20" def rets = [] pgsqlInstance.eachRow(sql,{r-> def map = [:] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy index d2ad41623..00a1ddd5e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy @@ -164,7 +164,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch { fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMUNITY_FJW.dat" } def headers = [] - FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", true); + FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", false); println "BIGDATA_COMMUNITY_FJW 数据文件输出完成......" @@ -251,7 +251,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch { fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMBUILDSTRU_FJW.dat" } def headers = [] - FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", true); + FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", false); println "BIGDATA_COMMBUILDSTRU_FJW 数据文件输出完成......" // sqlInstance.executeUpdate("delete from BIGDATA_COMMBUILDSTRU_FJW where CITYID='${cityid}'") @@ -309,7 +309,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch { fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMHOUSETYPE_FJW.dat" } def headers = [] - FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", true); + FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", false); println "BIGDATA_COMMHOUSETYPE_FJW 数据文件输出完成......" @@ -367,7 +367,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch { fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMBUILDYEAR_FJW.dat" } def headers = [] - FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", true); + FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", false); println "BIGDATA_COMMBUILDYEAR_FJW 数据文件输出完成......" // sqlInstance.executeUpdate("delete from BIGDATA_COMMBUILDYEAR_FJW where CITYID='${cityid}'") @@ -430,7 +430,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch { fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMFEE_FJW.dat" } def headers = [] - FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", true); + FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", false); println "BIGDATA_COMMFEE_FJW 数据文件输出完成......" @@ -509,7 +509,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch { fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMHOUSEUSE_FJW.dat" } def headers = [] - FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", true); + FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", false); println "BIGDATA_COMMHOUSEUSE_FJW 数据文件输出完成......" // sqlInstance.executeUpdate("delete from BIGDATA_COMMHOUSEUSE_FJW where CITYID='${cityid}'") @@ -595,7 +595,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch { fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_POSITION_FJW.dat" } def headers = [] - FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", true); + FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", false); println "BIGDATA_POSITION_FJW 数据文件输出完成......" } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 1af08d6b3..e0e58814f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -40,6 +40,8 @@ import java.util.zip.GZIPInputStream class CommonDatautils { + public static final String[] TAGS = ["独栋", "联排", "叠加", "双拼", "花园面积", "泳池", "别墅", "带花园", "洋房", "叠拼", "独院", "联体", "微墅", "大平墅"]; + public static final def mongoDbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017"; public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017"; @@ -3812,6 +3814,135 @@ class CommonDatautils { return rets; } + + + static def findHouses2(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2){ + + QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(true).includeUpper(true); + QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false); + + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)) + .must(QueryBuilders.termQuery("cat", cat)).must(date_range).must(area_range) + + if(region){ + queryBuilder.must(QueryBuilders.termQuery("region", region)) + } + + if(name || alias){ + if(!alias){ + alias = [] + } + if(!alias.contains(name)){ + alias.add(name) + } + + if(name.contains("弄小区")){ + def newName = name.replace("弄小区","弄") + if(!alias.contains(newName)){ + alias << newName + } + } + + if(alias && alias.size()>0){ + BoolQueryBuilder inBuilder = QueryBuilders.boolQuery(); + for(def s : alias){ + inBuilder.should(QueryBuilders.matchQuery("district_name", s)); + } + queryBuilder.must(inBuilder) + } + } + + def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder); + + def avgPrices = [] + def areas = [] + def totalPrices = [] + + if(houses){ + for (int k=0;k1000){ + continue + } + + if(seawater.avg_price && seawater.avg_price<10){ + continue + } + } + avgPrices << seawater.avg_price + areas << seawater.area + totalPrices << seawater.total_price + } + } + + def retMap = [:] + if(areas){ + retMap.avg_price = avgPrices + retMap.area = areas + retMap.total_price = totalPrices + } + return retMap; + } + + /** + * 是否是别墅 + * @return + */ + public static boolean isVilla(Double area, String name, Integer height, Integer room) { + if (area == null) { + return false; + } + if (area >= 150 && StringUtils.indexOfAny(name, TAGS) > -1) { + return true; + } + + boolean ret = false; + if (height != null && room != null) { + if (height <= 3) { + if (area > 200 && room >= 5) { + ret = true; + } + } + }else if(height != null){ + if (height <= 3) { + if (area > 200){ + ret = true; + } + } + } else if (height == null && room != null) { + if (area > 360 && room >= 6) { + ret = true; + } + } else if (height == null && room == null) { + if (area > 500) { + ret = true; + } + } + + return ret; + } + + + static def parseFloor(def floorDesc){ if(floorDesc){ floorDesc = floorDesc.replaceAll("[\\s ]+", "") @@ -4081,7 +4212,7 @@ class CommonDatautils { def map = houses[k]; if(map.parameter_map){ def pMap = map.parameter_map; - if((pMap.keyword && pMap.keyword.contains("碧桂园")) || (pMap.name && pMap.name.contains("碧桂园"))){ + if((pMap.keyword && pMap.keyword.contains("紫金山蘭园")) || (pMap.name && pMap.name.contains("紫金山蘭园"))){ println map } } @@ -4319,7 +4450,20 @@ class CommonDatautils { if(!dataMap.containsKey("maitian_esf_price") && db.maitian_esf_price && db.maitian_esf_price>3000){ dataMap.put("maitian_esf_price", db.maitian_esf_price) } + if(!dataMap.containsKey("xingfuli_esf_price") && db.xingfuli_esf_price && db.xingfuli_esf_price>3000){ + dataMap.put("xingfuli_esf_price", db.xingfuli_esf_price) + } + if(!dataMap.containsKey("5i5j_esf_price") && db.get("5i5j_esf_price") && db.get("5i5j_esf_price")>3000){ + dataMap.put("5i5j_esf_price", db.get("5i5j_esf_price")) + } + + if(!dataMap.containsKey("tuboshi_esf_price") && db.tuboshi_esf_price && db.tuboshi_esf_price>3000){ + dataMap.put("tuboshi_esf_price", db.tuboshi_esf_price) + } + if(!dataMap.containsKey("zhongyuan_esf_price") && db.zhongyuan_esf_price && db.zhongyuan_esf_price>3000){ + dataMap.put("zhongyuan_esf_price", db.zhongyuan_esf_price) + } } return map } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDistrictMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDistrictMatch.groovy index baccfa579..2b96edc91 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDistrictMatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDistrictMatch.groovy @@ -1,6 +1,7 @@ package com.fangjia.script.fuzhou import com.fangjia.core.RegionData +import com.fangjia.core.util.DateUtil import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils @@ -21,7 +22,38 @@ class FzDistrictMatch { // task.addPrice() // task.addThirdPrice() - task.addThirdPrice2("D:\\数据\\福州\\价格比对_20240315.xlsx") +// task.addThirdPrice2("D:\\数据\\福州\\价格比对_20240315.xlsx") + task.longyanPrice() + } + + def longyanPrice(){ +// def dists = CommonDatautils.findAllDistricts("龙岩","district", ["city","region","name","estate_type2"]) +// def rets = [] +// rets << ["Id", "城市","区域", "名称", "物业类型", "上个月价格toB","上个月价格","最新价格toB","最新价格"] +// for(def d : dists){ +// def ret = [] +// ret << d._id.toString() +// ret << d.city +// ret << d.region +// ret << d.name +// ret << d.estate_type2 +// def obj2 = CommonDatautils.queryDistrictPriceByWeekend(d.city, d.region, d.name, DateUtil.parseFormatDate("2024-03-10"), 1) +// if(obj2){ +// ret << obj2.avg_price_2b +// ret << obj2.avg_price +// } +// +// def obj1 = CommonDatautils.queryDistrictPriceByWeekend(d.city, d.region, d.name, DateUtil.parseFormatDate("2024-04-07"), 1) +// if(obj1){ +// ret << obj1.avg_price_2b +// ret << obj1.avg_price +// } +// rets << ret +// } +// ExcelUtils.write("D:\\数据\\福州\\龙岩价格对比.xlsx", rets, "龙岩") + + + addThirdPrice("D:\\数据\\福州\\龙岩价格对比.xlsx") } def addThirdPrice2(path){ @@ -207,4 +239,57 @@ class FzDistrictMatch { } },0) } + + + def addThirdPrice(path){ + ExcelUtils.multiThreadingModify(path, { d -> + if(d['C'] && d['C']=="新罗"){ + def cityMap = CommonDatautils.findThirdPartyPrice(d['B'], d['C'], d['D'], 202312, 202403) + def priceMap = cityMap.get(d['B']+"##"+d['C']+"##"+d['D']) + + if(priceMap && priceMap.zhugezhaofang_esf_price){ + d['J'] = priceMap.zhugezhaofang_esf_price + } + + if(priceMap && priceMap.anjuke_esf_price){ + d['K'] = priceMap.anjuke_esf_price + } + + if(priceMap && priceMap.soufang_esf_price) { + d['L'] = priceMap.soufang_esf_price + } + + if(priceMap && priceMap.lianjia_esf_price){ + d['M'] = priceMap.lianjia_esf_price + } + + if(priceMap && priceMap.cityhouse_esf_price){ + d['N'] = priceMap.cityhouse_esf_price + } + + if(priceMap && priceMap.zhongyuan_esf_price){ + d['O'] = priceMap.zhongyuan_esf_price + } + + if(priceMap && priceMap.xingfuli_esf_price){ + d['P'] = priceMap.xingfuli_esf_price + } + if(priceMap && priceMap.tuboshi_esf_price){ + d['Q'] = priceMap.tuboshi_esf_price + } + + if(priceMap && priceMap.get("5i5j_esf_price")){ + d['R'] = priceMap.get("5i5j_esf_price") + } + + def sellMap = CommonDatautils.findHouses2(d['B'], d['C'], "sell", 20231001, 20240407, d['D'], null, d['E']) + if(sellMap && sellMap.avg_price){ + if(sellMap.avg_price.size()>=3){ + d['S'] = Math.round(sellMap.avg_price.sum() / sellMap.avg_price.size()); + } + } + } + },0) + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy index 9e365230a..ad2cf8430 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy @@ -14,7 +14,7 @@ import com.mongodb.DBCollection //福州第三方均价统计,发送钉钉消息 //def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1) //def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv"); -def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202309.csv") +def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202402.csv") def count0 = 0 def count1 = 0 def count2 = 0 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddAvgpriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddAvgpriceBatch.groovy index 28fb38a92..cb2fcce9c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddAvgpriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddAvgpriceBatch.groovy @@ -20,16 +20,16 @@ class AddAvgpriceBatch { public static final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com") static { - citys = ["徐州"]//CommonDatautils.findAllCity() + citys = ["宜昌"]//CommonDatautils.findAllCity() println "加载城市完成:"+citys.size() } static main(args) { def task = new AddAvgpriceBatch(); -// task.run1(); + task.run1(); // task.test() // task.sendPriceCommand() - task.sendNewDistPrice() +// task.sendNewDistPrice() } def sendNewDistPrice(){ @@ -107,7 +107,7 @@ class AddAvgpriceBatch { println "------------------------------------------------------------" println "${city}小区量:"+districtKeysMap.size() if(districtKeysMap){ - def lastWeekendStats = queryAllDistrictPriceByWeekend(city, DateUtils.parseFromTightYYYYMMDD("2023-03-26")) + def lastWeekendStats = queryAllDistrictPriceByWeekend(city, DateUtils.parseFromTightYYYYMMDD("2024-04-07")) println "【2023-03-26】共计均价小区:"+lastWeekendStats.size() //库中小区没有计算过均价的小区 def districtKeys = districtKeysMap.keySet(); @@ -118,7 +118,7 @@ class AddAvgpriceBatch { def array = disKey.split("##"); def estateType2 = districtKeysMap.get(disKey) //发重算均价命令 - addPrice(array[0], array[1], array[2], estateType2, DateUtils.parseFromTightByPattern('2023-03-01','yyyy-MM-dd'),DateUtils.parseFromTightByPattern('2023-04-03','yyyy-MM-dd')) + addPrice(array[0], array[1], array[2], estateType2, DateUtils.parseFromTightByPattern('2024-03-01','yyyy-MM-dd'),DateUtils.parseFromTightByPattern('2024-04-07','yyyy-MM-dd')) } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsSearchUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsSearchUtils.groovy index e6812a2e4..82893abe6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsSearchUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsSearchUtils.groovy @@ -23,18 +23,22 @@ class EsSearchUtils { ResultSet rs = dataSource.connection.createStatement().executeQuery(sql.toString()) def rets = [] ElasticUtils.executeSqlQuery(sql,{ data-> - def m = [:] - m.city = data.cityname - m.region = data.adname - m.name = data.name - m.address = data.address - m.type = data.type - m.code = data.typecode + def mapName = data.name.replaceAll('[·\\(\\)()]|·', '').replace("建设中", "") + def newName = name.replaceAll('[·\\(\\)()]|·', '') + if(newName == mapName){ + def m = [:] + m.city = data.cityname + m.region = data.adname + m.name = data.name + m.address = data.address + m.type = data.type + m.code = data.typecode - def location = data.location - m.lng = location.lon - m.lat = location.lat - rets << m + def location = data.location + m.lng = location.lon + m.lat = location.lat + rets << m + } }) return rets; diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/commStat/CityStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/commStat/CityStat.groovy index 960b75f38..fad0c6bd1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/commStat/CityStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/commStat/CityStat.groovy @@ -33,7 +33,16 @@ class CityStat { // task.run6() // task.run7() // task.run8() - task.dealPriceStat() +// task.dealPriceStat() + task.countDist() + } + def countDist(){ + ExcelUtils.modify("D:\\数据\\其它\\小区统计.xlsx", { d -> + if(d['B']){ + def districts = CommonDatautils.findAllDistrictByCity(d['B'], "district", ["city", "region", "name"]) + d['D'] = districts.size() + } + },0); } def dealPriceStat(){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/ZhengzhouBlankStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/ZhengzhouBlankStat.groovy new file mode 100644 index 000000000..0979849cf --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/ZhengzhouBlankStat.groovy @@ -0,0 +1,112 @@ +package com.fangjia.script.gj.data + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.DBCollection + +//郑州银行数据导出 +class ZhengzhouBlankStat { + public static DBCollection cityInfoColl = MongoUtils.getCollectionByName("fangjia_data_ana", "city_info", "mongo.db.fangjia.com") + + public static final def DISTRICT_FIELDS_MAP = [ + name :'name',region :'region',block :'block',address :'address',estate_type2 :'estateType2',household_count :'householdCount',open_date :'openDate',live_date :'liveDate',build_date:'buildDate', + all_area :'allArea',build_area :'buildArea',build_type :'buildType',developer :'developer',estate_company :'estateCompany',cubage_percent :'cubagePercent',virescence_percent :'virescencePercent', + car_site :'carSite',estate_charge :'estateCharge',finish_building_date :'finishBuildingDate',business_quarter :'businessQuarter', + start_building_date :'startBuildingDate',loop_line_info :'loopLineInfo',property_life :'propertyLife',house_property :'houseProperty',build_structure :'buildStructure', + lat : 'lat',lng : 'lng',lat2 : 'lat2',lng2 : 'lng2', street_community:'streetCommunity',committee:'committee',schools:'schools',house_structure:'houseStructure', + alias:'alias',addr_alias:'addrAlias','main_type_area':'mainTypeArea',loudong_count:'loudongCount',car_site_count:'carSiteCount',mapping_alias:'mappingAlias',is_elevator:'elevator', + township:'township',kindergartens:'kindergartens',main_type:'mainType',land_property:'landProperty',usage_card:'usageCard',occupancy_rate:'occupancyRate',land_usage_start:'landUsageStart' + ] + + static main(args) { + def task = new ZhengzhouBlankStat(); + task.run() + } + + def run(){ + def citys = ["郑州","洛阳","平顶山","安阳","许昌","南阳","新乡","焦作","开封","信阳","驻马店","漯河","鹤壁","周口","濮阳","三门峡","商丘","济源"] + //statRegionDistrictCount(citys) +// statCityProcent(citys) + statCityHouseloudongProcent(citys) + + } + + def statRegionDistrictCount(citys){ + def rets = [] + rets << ["城市", "区县", "楼盘数量"] + for(def c : citys){ + def regions = CommonDatautils.getCityRegions(c) + + for(def r : regions){ + def count = CommonDatautils.findRegionDistrictCount("district", c, r) + def ret = [] + ret << c + ret << r + ret << count + rets << ret + } + + } + ExcelUtils.write("D:\\数据\\郑州银行\\郑州银行统计.xlsx", rets, "河南省区县小区量统计") + } + + def statCityProcent(citys){ + def rets = [] + rets << ["城市","数据量","地址","地址别名","别名","楼栋总数","户数","产权年限","产权性质","建筑结构","竣工时间","小区当前均价"] + for(def c : citys){ + def dbo = CommonDatautils.findCityInfo(c) + if(dbo){ + def detail = dbo.district_detail + def ret = [] + ret << c + if(detail){ + ret << dbo.existing_district_count//数据量 + ret << detail.address*100// 地址 + ret << detail.addrAlias*100// 地址别名 + ret << detail.alias*100// 别名 + ret << detail.loudongCount*100// 楼栋总数 + ret << detail.householdCount*100// 户数 + ret << detail.propertyLife*100// 产权年限 + ret << detail.houseProperty*100// 产权性质 + ret << detail.buildStructure*100// 建筑结构(按材料划分) + ret << detail.finishBuildingDate*100// 竣工时间 + ret << dbo.price_coverage*100// 小区当前均价 + } + rets << ret + } + } + + ExcelUtils.write("D:\\数据\\郑州银行\\郑州银行统计.xlsx", rets, "河南省城市字段覆盖率统计") + } + + def statCityHouseloudongProcent(citys){ + def rets = [] + + rets << ["城市","楼栋","单元","所在层数","房号","面积","套型","楼层"] + for(def c : citys){ + def dbo = CommonDatautils.findCityInfo(c) + if(dbo){ + def detail = dbo.house_loudong_detail + def ret = [] + ret << c + if(detail){ + ret << detail.house_num_rate*100//楼栋 + ret << detail.unit_num_rate*100// 单元 + ret << detail.room_num_rate*100// 所在层数 + ret << detail.room_num_rate*100// 房号 + ret << detail.area_rate*100// 面积 + ret << detail.room_info_rate*100// 套型 + } + def loudong_detail = dbo.loudong_detail + if(loudong_detail){ + ret << loudong_detail.loudong_height_rate*100// 楼层 + } + rets << ret + } + } + + ExcelUtils.write("D:\\数据\\郑州银行\\郑州银行统计.xlsx", rets, "河南省房产信息字段覆盖率统计") + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy new file mode 100644 index 000000000..a2ec7c80b --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy @@ -0,0 +1,86 @@ +package com.fangjia.script.gj.data_handle + +import com.fangjia.core.RegionData +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import net.sf.json.JSONObject +import org.apache.poi.ss.util.RegionUtil + +class DevProcess { + + def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; + DBCollection development_zone_CONN = MongoUtils.getCollectionByName2("fangjia", "development_zone", MONGO100_URL) + + static main(args) { + def task = new DevProcess(); + task.run() + } + def run(){ + def rets = ExcelUtils.read("D:\\数据\\开发区\\附件全国农村创业园区(基地)目录2021.xls") + def index = 0 + + def mcaDistMap = CommonDatautils.queryDistNameCity() + for(def r : rets){ + index ++ + if(index >2){ + def udata = new BasicDBObject() + JSONObject jsonObject = FangjiaClientUtils.formatFivelevel(null, r['E']); + if (jsonObject.get("district") && jsonObject.get("district") == "null") { + jsonObject = FangjiaClientUtils.formatFivelevel(null, r['C']+r['E']); + } + if (jsonObject.get("district") && jsonObject.get("district") == "null") { + jsonObject = FangjiaClientUtils.formatFivelevel(null, r['B']+r['C']+r['E']); + } + + udata.address = r['E'] + udata.create_time = new Date() + udata.industry = r['F'] + udata.level = "国家级" + udata.name = r['D'] + udata.province = r['B'] + udata.type = "农村创业园区" + + if(jsonObject) { + if (jsonObject.get("city") && jsonObject.get("city") != "null") { + udata.city = jsonObject.get("city") + } + if (jsonObject.get("district") && jsonObject.get("district") != "null") { + udata.district = jsonObject.get("district") + } + if(!udata.district && udata.city){ + def city = RegionData.formatCity(udata.city) + def region = RegionData.formatRegion(udata.city, r['C']) + if(region){ + def regionObj = mcaDistMap.get(city+"#"+region) + if(regionObj){ + udata.district = regionObj.district + } + } + } + if(jsonObject.get("township") && jsonObject.get("township")!="null"){ + udata.township = jsonObject.get("township") + } + + if(!udata.district){ + println index+" "+ udata + } + + } + + def obj = development_zone_CONN.findOne(new BasicDBObject("name", udata.name)) + if(!obj){ + development_zone_CONN.save(udata) + }else{ + println udata.name +" 已存在" + } + + + } + + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/GXDdistrictMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/GXDdistrictMatch.groovy new file mode 100644 index 000000000..2a638980a --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/GXDdistrictMatch.groovy @@ -0,0 +1,467 @@ +package com.fangjia.script.gj.data_handle + +import com.fangjia.core.RegionData +import com.fangjia.script.ccbevaluation.CcbCommonUtils +import com.fangjia.script.ccbevaluation.PgsqlCommUtils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.EsSearchUtils +import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.util.OpenApiClient +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +class GXDdistrictMatch { + + def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + + def regionCodeDataMap = PgsqlCommUtils.findDistrictMap(); + + def fullCityMap = CommonDatautils.queryFullNameCity(); + + DBCollection OPERATION_LOG_COLL = MongoUtils.getCollectionByName("fangjia", "operation_log", "mongo.db.fangjia.com") + + static main(args) { + def task = new GXDdistrictMatch(); +// task.run() +// task.addOldMatch() +// task.runDb() + +// task.updateDistStatus()//恢复name_cat不正常数据 +// task.saveToDB() //保存新增小区 + task.restoreDist()//恢复小区status=0 + } + + def addOldMatch(){ + //老匹配 + def fileName = "D:\\数据\\国信达\\非正常数据.xlsx"//11城重新匹配.xlsx + ExcelUtils.multiThreadingModify(fileName, { d -> + println d['C'] + if(!d['AJ']){ + def jsonObject = OpenApiClient.addressMatch(d['AD'], d['H'], null, false, d['AE'], d['C']) + if (jsonObject != null && !jsonObject.containsKey("result")) { + d['AJ'] = jsonObject.get("id") + d['AK'] = jsonObject.get("city") + d['AL'] = jsonObject.get("district") + d['AM'] = jsonObject.get("name") + d['AN'] = jsonObject.get("address") + + def dbo = CommonDatautils.findDistrictById(jsonObject.get("id")) + if(dbo){ + d['AO'] = dbo.status + d['AP'] = dbo.name_cat + d['AQ'] = dbo.estate_type2 + } + } + } + }); + + } + + def runDb(){ + def fileName = "D:\\数据\\国信达\\国信达11城未匹配数据.xlsx" + ExcelUtils.modify(fileName, { d -> + if(!d['U']){ + def fullCityObj = fullCityMap.get(d['AD']) + if(fullCityObj){ + def fullRegion = regionCodeDataMap.get(d['A']+"#"+d['E']) + def obj = findPoiFromDB(fullCityObj.city, fullRegion, d['C']) + if(obj){ + d['U'] = "地图库" + d['V'] = obj.city + d['W'] = obj.region + d['X'] = obj.name + d['Y'] = obj.type + d['Z'] = obj.address + d['AA'] = obj.lng + d['AB'] = obj.lat + }else{ + obj = findDatas(d['AD'], d['AE'], d['C'], d['C'].replaceAll('[·\\(\\)()]|·', '')) + if(obj){ + d['U'] = "楼盘抓取库" + d['V'] = obj.city + d['W'] = obj.region + d['X'] = obj.name + d['Y'] = obj.property_type + d['Z'] = obj.address + def loc = obj.location + if(loc){ + d['AA'] = loc.x + d['AB'] = loc.y + } + } + } + } + } + }) + } + + def findPoiFromDB(city, region, name){ + def rets = EsSearchUtils.findPois(city, region, name) + def obj = null + if(rets && rets.size()==1){ + obj = rets[0] + } + return obj + } + + + def run(){ + def fileName = "D:\\数据\\国信达\\接口匹配.xlsx"//11城重新匹配.xlsx + ExcelUtils.multiThreadingModify(fileName, { d -> + println d['C'] + if(!d['V'] && d['A']){ + def city = getCityFromCityCode(d['A']) + def region = regionCodeDataMap.get(d['A'] + "#" + d['E']) + if (city && region) { + region = RegionData.formatRegion(city, region) + d['AD'] = city + d['AE'] = region + if (region) { + d = match(city, region, d) + } else { + println d['A'] + "#" + d['E'] + " 区域无法格式化" + } + + } else { + println d['A'] + "#" + d['E'] + " 无法格式化" + } + } + }) + } + + def updateDistStatus(){ + def fileName = "D:\\数据\\国信达\\国信达11城(全).xlsx" + def index = 0 + ExcelUtils.modify(fileName, { d -> + if(d['U'] && ["名称匹配","名称匹配(Update)","名称在别名中","名称在别名中(Update)","别名匹配"].contains(d['U'])){ + index++ + + def status = d['Z'] + def name_cat = d['AA'] + + println d["V"]+" "+d["X"]+name_cat + + if(status!=null && status.toString().toDouble().toInteger()==0){ + if(name_cat==null){//&& name_cat.toString().toDouble().toInteger()!=0 + + def dist = CommonDatautils.findDistrict(d["V"],d["W"],d["X"], "district") + if(dist){ + def delUdata = new BasicDBObject() + delUdata.put("name_cat", 0) + delUdata.put("m_date", new Date()) + DbUtils.seaweed.update(new BasicDBObject('_id',dist._id), new BasicDBObject('$set', delUdata)) + saveOptLog(dist.city, dist.region, dist.name, "update") + } + } + } + } + }) + } + + def restoreDist(){ + def fileName = "D:\\数据\\国信达\\11城小区需恢复数据.xlsx"//国信达11城(全).xlsx + def index = 0 + ExcelUtils.modify(fileName, { d -> + if(d['U']){ + index++ + def status = d['Z'] + println d["V"]+" "+d["X"] + if(status && status.toString().toDouble().toInteger()==1){ + def dists = CommonDatautils.findAllStatusDistrict(d["V"],d["W"],d["X"], "district") + if(dists && dists.size()==1){ + def dist = dists[0] + def delUdata = new BasicDBObject() + delUdata.put("name_cat", 0) + delUdata.put("status", 0) + delUdata.put("m_date", new Date()) + DbUtils.seaweed.update(new BasicDBObject('_id',dist._id), new BasicDBObject('$set', delUdata)) + saveOptLog(dist.city, dist.region, dist.name, "update-status") + } + } + } + }) + } + + def saveToDB(){ + def fileName = "D:\\数据\\国信达\\国信达11城未匹配数据.xlsx" + ExcelUtils.modify(fileName, { d -> + if(d['U'] && d['AG']!='过滤'){ + def udata = new BasicDBObject() + udata.put("city", d['AD']) + udata.put("region", d['AE']) + def name = d['X'].replaceAll('[·\\(\\)()]|·', '').replace("建设中", "") + udata.put("name", name) + + def alias = [] + if(name!=d['C']){ + alias << d['C'] + } + if(d['AI'] && !alias.contains(d['AI']) && d['AI']!=name){ + alias << d['AI'] + } + if(alias){ + udata.put("alias", alias) + } + + udata.put("address", d['AF']) + udata.put("status", "0") + udata.put("name_cat", "0") + udata.put("cat", "district") + udata.put("username", "高娟") + + def estateType2 = "普通住宅" + def type = "" + if(d['AH']){ + type = d['AH'] + }else{ + type = d['Y'] + } + if(type){ + if(type.contains("别墅")){ + estateType2 = "别墅" + }else if(type.contains("商住")){ + estateType2 = "商住" + } + } + udata.put("estateType2", estateType2) + if(d['AA']){ + Long lng = d['AA']*10000000000 + udata.put("lng", lng) + } + if(d['AB']){ + Long lat = d['AB'] * 10000000000 + udata.put("lat", lat) + } + + def s = FangjiaClientUtils.saveOrUpdateDistrict(udata) + def opt = "add" + if(s && s=='成功'){ + saveOptLog(udata.city, udata.region, udata.name, opt) + } + } + }) + + } + + + def match(city, region, d){ + d = matching(city, region, d, d['C'].replaceAll('[·\\(\\)()]|·', ''), d['D'], "") + if(!d['U']){ + d = matching(city, region, d, d['C'].replace("小区","").replace("号楼","号院").replace("号","号院"), d['D'], "(Update)") + if(!d['U']){ + d = matching(city, region, d, d['C']+"小区", d['D'], "(Update)") + } + + if(!d['U']){ + if(d['F'] && d['G']){ + d = matchAddr(city, region, d, "") + } + } + + if(!d['U']){ + d = matchApi(city, region, d, "property", false) + if(!d['U']){ + d = matchApi(city, region, d, null, true) + } + } + + if(!d['U']){ + d = matching2(city, region, d, d['C'], d['D'], "") + } + + if(!d['U']){ + if(d['C'].contains("小区")){ + d = matching2(city, region, d, d['C'].replace("小区",""), d['D'], "(Update)") + }else{ + d = matching2(city, region, d, d['C']+"小区", d['D'], "(Update)") + } + } + } + return d + } + + def matchApi(city, region, d, cat, useMap){ + def jsonObject = FangjiaClientUtils.newMatch(city, d['H'], cat, useMap) + if(!jsonObject){ + jsonObject = FangjiaClientUtils.newMatch(city, region+d['C'], cat, useMap) + if(!jsonObject && d['D']){ + jsonObject = FangjiaClientUtils.newMatch(city, region+d['D'], cat, useMap) + } + } + if(jsonObject != null && jsonObject.credit>0) { + d['U'] = "接口匹配" + d['V'] = jsonObject.get("city") + d['W'] = jsonObject.get("district") + d['X'] = jsonObject.get("name") + def type = jsonObject.get("category")//propertyType + if(type && type!='null'){ + d['Y'] = type + } + def dbo = CommonDatautils.findDistrictById(jsonObject.get("id")) + if(dbo){ + d['Z'] = dbo.status + d['AA'] = dbo.name_cat + } + d['AC'] = jsonObject.get("credit") + if(jsonObject.get("address") && jsonObject.get("address")!='null'){ + d['AH'] = jsonObject.get("address") + } + + } + return d + } + + + def matching(city, region, d, name, alias, str){ + //正常状态下有没有 + def rets = CommonDatautils.findDistrictList(city, region, name, "district") + if(rets){ + if(rets.size()>=1){ + d = setRowValue(rets[0], d, "名称匹配"+str) + } + }else{ + rets = CommonDatautils.findDistrictByAlias(city, region, name, "district") + if(rets && rets.size()>=1){ + d = setRowValue(rets[0], d, "名称在别名中"+str) + }else{ + rets = CommonDatautils.findDistrictByAddress(city, region, name, "district") + if(rets && rets.size()>=1){ + d = setRowValue(rets[0], d, "名称在地址中"+str) + }else{ + if (d['D']) { + rets = CommonDatautils.findDistrictByAlias(city, region, alias.replaceAll('[·\\(\\)()]|·', ''), "district") + if (rets) { + if (rets.size() >= 1) { + d = setRowValue(rets[0], d, "别名匹配"+str) + } + } + } + } + } + } + + if(rets){ + d['AI'] = rets.size() + } + return d; + } + + def matching2(city, region, d, name, alias, str){ + def rets = CommonDatautils.findAllStatusDistrict(city, region, name, "district")//包含删除的 + if(rets){ + if (rets.size() >= 1) { + d = setRowValue(rets[0], d, "名称匹配(非正常)"+str) + } + }else{ + + rets = CommonDatautils.findAllStatusDistrictByAlias(city, region, name, "district") + if (rets && rets.size() >= 1) { + d = setRowValue(rets[0], d, "别名匹配(非正常)"+str) + }else{ + rets = CommonDatautils.findAllStatusDistrictByAlias(city, region, alias, "district") + if (rets.size() >= 1) { + d = setRowValue(rets[0], d, "别名匹配(非正常)"+str) + } + } + } + + if(rets){ + d['AI'] = rets.size() + } + return d; + } + + def matchAddr(city, region, d, str){ + def rets = CommonDatautils.findDistrictByAddress(city, region, d['F']+d['G'],"district") + if(rets){ + if (rets.size() >= 1) { + d = setRowValue(rets[0], d, "地址匹配") + } + } + if(rets){ + d['AI'] = rets.size() + } + return d + } + + def setRowValue(dbo, d, remark){ + if (dbo) { + d['V'] = dbo.city + d['W'] = dbo.region + d['X'] = dbo.name + d['Y'] = dbo.estate_type2 + d['Z'] = dbo.status + d['AA'] = dbo.name_cat + d['U'] = remark + d['AH'] = dbo.address + } + return d; + } + + def getCityFromCityCode(cityCode){ + def city = null + for(def m : cityCodeDataMap){ + if(m.value == cityCode){ + city = m.key + break; + } + } + return city; + } + + def filterSpacialChar(def s){ + def pattern = ~/[()!;\·]\[【】()]/ + def matcher = pattern.matcher(s) + + def count = matcher.getCount() + if(count && count>0){ + return matcher.replaceAll('_') + } + return null; + } + + + def findDatas(def city, def region, def name1, def name2){ + def sql = "select * from community where city='${city}' and region='${region}' and (name='${name1}' or name='${name2}')" + def rets = [] + ElasticUtils.executeSqlQuery(sql, { d-> + rets << d + println d.property_type + }) + + if(rets){ + def obj = null + for(def r : rets){ + if(r.property_type && (r.property_type.contains("住宅") || r.property_type.contains("别墅") || r.property_type.contains("公寓"))){ + obj = r + break + } + } + if(obj){ + return obj; + }else{ + return rets[0]; + } + + } + return null; + } + + def saveOptLog(city, region, name, type){ + def obj = new BasicDBObject(); + obj.put("city",city); + obj.put("region",region); + obj.put("district_name",name); + obj.put("category","district"); + obj.put("c_date", new Date()) + obj.put("m_date", new Date()) + obj.put("type", type) + obj.put("user_name", "高娟") + obj.put("origin", "script") + OPERATION_LOG_COLL.save(obj) + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TradeDatasDistBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TradeDatasDistBatch.groovy new file mode 100644 index 000000000..7c9472e42 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TradeDatasDistBatch.groovy @@ -0,0 +1,13 @@ +package com.fangjia.script.gj.data_handle + +import com.fangjia.script.util.MongoUtils +import com.mongodb.DBCollection + +class TradeDatasDistBatch { + + public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com") + + static main(args) { +// def + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy index 3abfbaa22..260ef5cb7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy @@ -228,9 +228,9 @@ class FangjiaClientUtils { def url = null; try { address = address.replace("{","").replace("\\","") - url = "http://192.168.0.100:32080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}" -// url = "http://192.168.0.19:31080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}" - + url = "http://192.168.0.93:8080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}" +// url = "http://192.168.20.242:5001/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}" +// url = "http://192.168.0.100:32080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}" if(city){ url = url + "&city="+URLEncoder.encode(city,"UTF-8") } @@ -314,13 +314,13 @@ class FangjiaClientUtils { def content = null try { address = address.replace("{","") - def url = "http://192.168.0.19:31080/FangJiaNlpService/formatFivelevel?address="+URLEncoder.encode(address,"UTF-8") +// def url = "http://192.168.0.19:31080/FangJiaNlpService/formatFivelevel?address="+URLEncoder.encode(address,"UTF-8") + def url = "http://192.168.0.100:32080/FangJiaNlpService/formatFivelevel?address="+URLEncoder.encode(address,"UTF-8") if(city){ url = url +"&city="+URLEncoder.encode(city,"UTF-8") } content = new HttpRequest().newGet(url, null, "utf-8") jsonObj = JSONObject.fromObject(content) - println jsonObj } catch (Exception e) { println address +" "+content e.printStackTrace(); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/GXDdistrictMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/GXDdistrictMatch.groovy deleted file mode 100644 index 85b3a9a65..000000000 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/GXDdistrictMatch.groovy +++ /dev/null @@ -1,138 +0,0 @@ -package com.fangjia.script.gj.district - -import com.fangjia.core.RegionData -import com.fangjia.script.ccbevaluation.CcbCommonUtils -import com.fangjia.script.ccbevaluation.PgsqlCommUtils -import com.fangjia.script.commData.CommonDatautils -import com.fangjia.script.util.ExcelUtils - -class GXDdistrictMatch { - - static def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() - static def regionCodeDataMap = PgsqlCommUtils.findDistrictMap() - - static main(args) { - def task = new GXDdistrictMatch(); - task.run() - } - - def run(){ - def fileName = "D:\\数据\\国信达\\国信达11城2.xlsx" - ExcelUtils.multiThreadingModify(fileName, { d -> - if(!d['V']){ - def city = getCityFromCityCode(d['A']) - def region = regionCodeDataMap.get(d['A'] + "#" + d['E']) - if (city && region) { - region = RegionData.formatRegion(city, region) - if (region) { - d = match(city, region, d) - } else { - println d['A'] + "#" + d['E'] + " 区域无法格式化" - } - } else { - println d['A'] + "#" + d['E'] + " 无法格式化" - } - } - }) - } - - - def match(city, region, d){ - d = matching(city, region, d, d['C'], d['D'], "") - if(!d['U']){ - if(d['C'].contains("小区")){ - d = matching(city, region, d, d['C'].replace("小区",""), d['D'], "(Update)") - }else{ - d = matching(city, region, d, d['C']+"小区", d['D'], "(Update)") - } - if(!d['U']){ - d = matching2(city, region, d, d['C'], d['D'], "") - } - - if(!d['U']){ - if(d['C'].contains("小区")){ - d = matching2(city, region, d, d['C'].replace("小区",""), d['D'], "(Update)") - }else{ - d = matching2(city, region, d, d['C']+"小区", d['D'], "(Update)") - } - } - - } - } - - - def matching(city, region, d, name, alias, str){ - //正常状态下有没有 - def rets = CommonDatautils.findDistrictList(city, region, name, "district") - if(rets){ - if(rets.size()==1){ - d = setRowValue(rets[0], d, "名称匹配"+str) - }else{ - d['U'] = "名称在多个小区中"+str - } - }else{ - rets = CommonDatautils.findDistrictByAlias(city, region, name, "district") - if(rets && rets.size()==1){ - d = setRowValue(rets[0], d, "名称在别名中"+str) - }else{ - if (d['D']) { - rets = CommonDatautils.findDistrictByAlias(city, region, alias, "district") - if (rets) { - if (rets.size() == 1) { - d = setRowValue(rets[0], d, "别名匹配"+str) - } else { - d['U'] = "别名在多个小区中"+str - } - } - } - } - } - - return d; - } - - def matching2(city, region, d, name, alias, str){ - def rets = CommonDatautils.findAllStatusDistrict(city, region, name, "district")//包含删除的 - if(rets){ - if (rets.size() == 1) { - d = setRowValue(rets[0], d, "名称匹配(非正常)"+str) - } - }else{ - - rets = CommonDatautils.findAllStatusDistrictByAlias(city, region, name, "district") - if (rets && rets.size() == 1) { - d = setRowValue(rets[0], d, "别名匹配(非正常)"+str) - }else{ - rets = CommonDatautils.findAllStatusDistrictByAlias(city, region, alias, "district") - if (rets.size() == 1) { - d = setRowValue(rets[0], d, "别名匹配(非正常)"+str) - } - } - } - return d; - } - - def setRowValue(dbo, d, remark){ - if (dbo) { - d['V'] = dbo.city - d['W'] = dbo.region - d['X'] = dbo.name - d['Y'] = dbo.estate_type2 - d['Z'] = dbo.status - d['AA'] = dbo.name_cat - d['U'] = remark - } - return d; - } - - def getCityFromCityCode(cityCode){ - def city = null - for(def m : cityCodeDataMap){ - if(m.value == cityCode){ - city = m.key - break; - } - } - return city; - } -} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsSelectTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsSelectTest.groovy index d259d858b..674198bd0 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsSelectTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsSelectTest.groovy @@ -23,7 +23,7 @@ class EsSelectTest { def test(){ - def ret = CommonDatautils.findLogs("1007703", "/property/evaluate", "2024-03-13", "2024-03-16") + def ret = CommonDatautils.findLogs("1007741", "/property/evaluate", "2024-03-01", "2024-03-28") } def run(){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/QiliangTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/QiliangTest.groovy index c153747fc..dade0fa1f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/QiliangTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/QiliangTest.groovy @@ -5,13 +5,22 @@ import com.fangjia.script.temp.shenjian.util.HttpUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.yinjihaun.util.HttpClient import net.sf.json.JSONObject +import okhttp3.*; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.Proxy; +import java.util.concurrent.TimeUnit; + + +import okhttp3.Request.Builder; class QiliangTest { - static main(args) { - def task = new QiliangTest() - task.run() - } +// static main(args) { +// def task = new QiliangTest() +//// task.run() +// task.getHttpResult("扬州扬子江北路399号11041") +// } def run(){//地址测试-2024.3.4-结果.xlsx ExcelUtils.modify("D:\\数据\\省联社\\对比.xlsx", { row -> @@ -80,7 +89,7 @@ class QiliangTest { def getHttpResult(address){ def data = new JSONObject() data.put("address", address) - data.put("type","1") + data.put("type","2") Map headers = new HashMap(); headers.put("Accept", "application/json, text/plain, */*"); @@ -104,4 +113,79 @@ class QiliangTest { } return null; } + + + final static String proxyIp = "tunnel2.qg.net"; + final static Integer proxyPort = 14186; + final static String authKey = "113BBC06"; + final static String password = "3B2FBD05B0AF"; + public static final MediaType JSON = MediaType.parse("application/json;charset=utf-8"); + +// public String request(params) throws IOException { +// Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyIp, proxyPort)); +// OkHttpClient client = new OkHttpClient.Builder() +// .proxy(proxy) +// .proxyAuthenticator((route, response){ +// String credential = Credentials.basic(authKey, password); +// return response.request().newBuilder().header("Proxy-Authorization", credential).build(); +// }). +// build(); +//// Request request = new Request.Builder().url("http://146.56.217.44:9010/uams/farmer/all/v3/sign").get().build(); +// +// RequestBody body = RequestBody.create(JSON, JSON.toJSONString(params)); +// Request request = new Builder().url("http://146.56.217.44:9010/uams/farmer/all/v3/sign").post(body).build(); +// +// Response response = client.newCall(request).execute(); +// return response.body().string(); +// +//// return client.newCall(request).execute(); +// } + public static void main(String[] args) { + def proxyHost = "tunnel2.qg.net" + def proxyPort = 14186 + def proxyUsername = '113BBC06' + def proxyPassword = '3B2FBD05B0AF' + + def targetUrl = 'http://146.56.217.44:9010/uams/farmer/all/v3/sign' + def queryParams = ['address': '扬州扬子江北路399号11041', 'type': '2'] + + +// 创建代理服务器的连接 + def proxy = new URL("http://${proxyUsername}:${proxyPassword}@${proxyHost}:${proxyPort}") + +// 打开代理连接 + def proxyConnection = proxy.openConnection() + +// 创建目标URL的连接 + def url = new URL(targetUrl) + def connection = url.openConnection(proxy) + +// 设置请求方法 + connection.setRequestMethod("GET") + +// 添加请求头(可选) + connection.setRequestProperty("Content-Type", "application/json") + +// 构建查询参数 + def queryString = queryParams.collect { k, v -> "${k}=${v}" }.join('&') + +// 将查询参数附加到目标URL + def finalUrl = "${targetUrl}?${queryString}" + +// 发送请求 + connection.connect() + +// 获取响应 + def responseCode = connection.getResponseCode() + def responseMessage = connection.getResponseMessage() + +// 读取响应内容 + def reader = new BufferedReader(new InputStreamReader(connection.getInputStream())) + def responseContent = reader.readLine() + reader.close() + + println "Response Code: ${responseCode}" + println "Response Message: ${responseMessage}" + println "Response Content: ${responseContent}" + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy index d6ef502f2..50882747b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy @@ -414,7 +414,7 @@ public class ExcelUtils{ def data = ExcelUtils.read(fileName, srcSheetIndex, 0) def body = data.subList(1, data.size()) - def nThread = Runtime.getRuntime().availableProcessors() * 3 + def nThread = Runtime.getRuntime().availableProcessors() * 4 println nThread ExecutorService executorService = Executors.newFixedThreadPool(nThread) final AtomicInteger counter = new AtomicInteger(0) From 9d985e920d7dd836a21e6c778049ece2f93fff3a Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 2 Apr 2024 16:02:59 +0800 Subject: [PATCH 22/22] update --- .../ronghe/BigdataCommunityGJRHBatch.groovy | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy index 00a1ddd5e..d2ad41623 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy @@ -164,7 +164,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch { fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMUNITY_FJW.dat" } def headers = [] - FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", false); + FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", true); println "BIGDATA_COMMUNITY_FJW 数据文件输出完成......" @@ -251,7 +251,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch { fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMBUILDSTRU_FJW.dat" } def headers = [] - FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", false); + FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", true); println "BIGDATA_COMMBUILDSTRU_FJW 数据文件输出完成......" // sqlInstance.executeUpdate("delete from BIGDATA_COMMBUILDSTRU_FJW where CITYID='${cityid}'") @@ -309,7 +309,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch { fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMHOUSETYPE_FJW.dat" } def headers = [] - FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", false); + FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", true); println "BIGDATA_COMMHOUSETYPE_FJW 数据文件输出完成......" @@ -367,7 +367,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch { fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMBUILDYEAR_FJW.dat" } def headers = [] - FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", false); + FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", true); println "BIGDATA_COMMBUILDYEAR_FJW 数据文件输出完成......" // sqlInstance.executeUpdate("delete from BIGDATA_COMMBUILDYEAR_FJW where CITYID='${cityid}'") @@ -430,7 +430,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch { fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMFEE_FJW.dat" } def headers = [] - FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", false); + FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", true); println "BIGDATA_COMMFEE_FJW 数据文件输出完成......" @@ -509,7 +509,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch { fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMHOUSEUSE_FJW.dat" } def headers = [] - FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", false); + FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", true); println "BIGDATA_COMMHOUSEUSE_FJW 数据文件输出完成......" // sqlInstance.executeUpdate("delete from BIGDATA_COMMHOUSEUSE_FJW where CITYID='${cityid}'") @@ -595,7 +595,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch { fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_POSITION_FJW.dat" } def headers = [] - FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", false); + FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", true); println "BIGDATA_POSITION_FJW 数据文件输出完成......" }