From 95ade7a506c016c007adf13a21ea977daa8269e8 Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Mon, 28 Nov 2022 17:34:43 +0800 Subject: [PATCH 01/99] =?UTF-8?q?=E6=96=B0=E5=A2=9Ejenkins=E6=96=87?= =?UTF-8?q?=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 1b1725b85..d849b8ef3 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -5,7 +5,7 @@ pipeline { choice(name: 'PROJECT_NAME', choices: ['fangjiabkoff', 'chafangtong', 'fangjia', 'fangjiapds', 'fangjiastatic'], description: '请选择å�‘布的项目') } tools { - jdk 'JDK8' + jdk 'JDK6' } stages { From c87bb7cf27edb3c013fb1a5aa7bac8e7e50e8361 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 30 Nov 2022 14:27:51 +0800 Subject: [PATCH 02/99] =?UTF-8?q?=E6=B8=A4=E6=B5=B7=E5=9C=B0=E5=9D=80?= =?UTF-8?q?=E6=94=B9=E4=B8=BA=E8=87=AA=E5=B7=B1=E8=BE=93=E5=85=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../weixin/WeiXinEvaluationController.java | 21 +++++++++++-------- .../src/main/webapp/static/js/select.js | 5 ++--- .../main/webapp/weiXinEvaluation/bhform.ftl | 14 ++++++++----- .../main/webapp/weiXinEvaluation/bhreport.ftl | 2 +- 4 files changed, 24 insertions(+), 18 deletions(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java index 3a03aa8d1..800a2360d 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java @@ -454,9 +454,7 @@ public class WeiXinEvaluationController { evaluation.setCity(district.getCity()); evaluation.setRegion(district.getRegion()); evaluation.setDistrictName(district.getName()); - if(StringUtils.isBlank(evaluation.getAddress())){ - evaluation.setAddress(district.getAddress()); - } + evaluation.setAddress(district.getAddress()); evaluation.setUserId(userId); evaluation.setDateCreated(new Date()); evaluation.setLastUpdated(new Date()); @@ -491,8 +489,8 @@ public class WeiXinEvaluationController { if(fjtUser!=null && fjtUser.getRole()==17) { - System.out.println("²³º£µØÖ·ËÑË÷ "+evaluation.getAddress()); - List bhDists = BHBankUtils.getDistrictByBhbank(evaluation.getAddress()); + System.out.println("²³º£µØÖ·ËÑË÷ "+evaluation.getCertificateAddress()); + List bhDists = BHBankUtils.getDistrictByBhbank(evaluation.getCertificateAddress()); JSONObject o = null; if(bhDists.size()==0){ bhDists = BHBankUtils.getDistrictByBhbank(district.getName()); @@ -545,7 +543,7 @@ public class WeiXinEvaluationController { if(StringUtils.isBlank(usersDetail.getMobile())){ usersDetail.setMobile(fjtUser.getEmail()); } - System.out.println("²³º£Ñ¯¼ÛËÑË÷ "+evaluation.getAddress()); + System.out.println("²³º£Ñ¯¼ÛËÑË÷ "+evaluation.getCertificateAddress()); JSONObject bhApiResult = BHBankUtils.getInquirePrice(model, bhMatchMap, usersDetail, evaluation.getTotalPrice()); if(bhApiResult!=null && (Integer)bhApiResult.get("errCode")==0){ System.out.println("²³º£Ñ¯¼ÛËÑË÷½á¹û "+bhApiResult.toString()); @@ -1411,11 +1409,16 @@ public class WeiXinEvaluationController { } if(uploadfiles.size()>0){ JSONObject jsonObj = BHBankUtils.inquireUploadFiles2(usersDetail.getName(),model.getQueryId(), model.getFileClass(), uploadfiles); - if(jsonObj!=null && (Integer)jsonObj.get("errCode")!=0) { - mav.setModel(BasicModel.buildServerErrorModel().setMsg((String) jsonObj.get("errMsg"))); + if(jsonObj!=null){ + if(jsonObj!=null && (Integer)jsonObj.get("errCode")!=0) { + mav.setModel(BasicModel.buildServerErrorModel().setMsg((String) jsonObj.get("errMsg"))); + }else{ + mav.setModel(BasicModel.buildServerErrorModel().setMsg(("ÎļþÉÏ´«³É¹¦£¡"))); + } }else{ - mav.setModel(BasicModel.buildServerErrorModel().setMsg(("ÎļþÉÏ´«³É¹¦£¡"))); + mav.setModel(BasicModel.buildServerErrorModel().setMsg("ÉÏ´«Ê§°Ü£¡")); } + } } catch (Exception e) { e.printStackTrace(); diff --git a/projects/fangjia-open/src/main/webapp/static/js/select.js b/projects/fangjia-open/src/main/webapp/static/js/select.js index b7c01d2f6..1202c4ad6 100644 --- a/projects/fangjia-open/src/main/webapp/static/js/select.js +++ b/projects/fangjia-open/src/main/webapp/static/js/select.js @@ -1,9 +1,7 @@ // JavaScript Document - /* 选择æ“�作 */ - // 选择æ“�作-地å�€ -$(document).on('click', '.create-actions00', function () { +/*$(document).on('click', '.create-actions00', function () { var buttons1 = [ { text: '请选择地å�€', @@ -31,6 +29,7 @@ $(document).on('click', '.create-actions00', function () { var groups = [buttons1, buttons2]; $.actions(groups); }); + */ // 选择æ“�作-房型-室 $(document).on('click', '.create-actions01', function () { diff --git a/projects/fangjia-open/src/main/webapp/weiXinEvaluation/bhform.ftl b/projects/fangjia-open/src/main/webapp/weiXinEvaluation/bhform.ftl index 68519a79b..5160cbd1a 100644 --- a/projects/fangjia-open/src/main/webapp/weiXinEvaluation/bhform.ftl +++ b/projects/fangjia-open/src/main/webapp/weiXinEvaluation/bhform.ftl @@ -47,13 +47,17 @@
  • -
    -
    -
    地�:
    - -

    请选择地�

    +
    产�地�:
    +
    +
    +<#--
    --> +<#--
    地�:
    --> +<#-- --> +<#--

    请选择地�

    --> +<#--
    -->
    (必填项)
    diff --git a/projects/fangjia-open/src/main/webapp/weiXinEvaluation/bhreport.ftl b/projects/fangjia-open/src/main/webapp/weiXinEvaluation/bhreport.ftl index 6c6e97d28..a64bb2f90 100644 --- a/projects/fangjia-open/src/main/webapp/weiXinEvaluation/bhreport.ftl +++ b/projects/fangjia-open/src/main/webapp/weiXinEvaluation/bhreport.ftl @@ -34,7 +34,7 @@

    ${fjtEvaluation.districtName!}

    -

    ${fjtEvaluation.address!} <#if fjtEvaluation.houseNum?? && fjtEvaluation.houseNum?contains("å�·")>${fjtEvaluation.houseNum!}<#else>${fjtEvaluation.houseNum!}å�·${fjtEvaluation.roomNum!}室 é‡�新评估 >

    +

    <#if fjtEvaluation.certificateAddress??>${fjtEvaluation.certificateAddress!}<#else>${fjtEvaluation.address!} <#if fjtEvaluation.houseNum?? && fjtEvaluation.houseNum?contains("å�·")>${fjtEvaluation.houseNum!}<#else>${fjtEvaluation.houseNum!}å�·${fjtEvaluation.roomNum!}室 é‡�新评估 >

    *请确认与产�地�一致

    <#if bohaiEvaluationAudit?? && bohaiEvaluationAudit.auditStatus!=0 && bohaiEvaluationAudit.inquirePrice gt 0> From acaf893c4fcf1569a1e47c8b262d2677c3b5dac9 Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Wed, 30 Nov 2022 16:53:16 +0800 Subject: [PATCH 03/99] =?UTF-8?q?=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Jenkinsfile | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index d849b8ef3..90340301f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -16,11 +16,13 @@ pipeline { } stage('Maven Build') { steps { - def projects = ['fangjiabkoff', 'chafangtong', 'fangjia', 'fangjiapds', 'fangjiastatic'] - for (def project : projects) { - dir('deliverables/${project}') { - withMaven(globalMavenSettingsConfig: '426b9a25-aa0d-47d3-a24c-8abdfc4421ff') { - sh 'mvn clean package -Dmaven.test.skip=true' + script { + def projects = ['fangjiabkoff', 'chafangtong', 'fangjia', 'fangjiapds', 'fangjiastatic'] + for (def project : projects) { + dir('deliverables/${project}') { + withMaven(globalMavenSettingsConfig: '426b9a25-aa0d-47d3-a24c-8abdfc4421ff') { + sh 'mvn clean package -Dmaven.test.skip=true' + } } } } From e68c3a77ead3cbcdb53c4e9abb85632e74c6a4f1 Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Wed, 30 Nov 2022 17:02:06 +0800 Subject: [PATCH 04/99] =?UTF-8?q?=E4=BF=AE=E6=94=B9jdk?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 90340301f..9af2ad01b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -5,7 +5,7 @@ pipeline { choice(name: 'PROJECT_NAME', choices: ['fangjiabkoff', 'chafangtong', 'fangjia', 'fangjiapds', 'fangjiastatic'], description: '请选择å�‘布的项目') } tools { - jdk 'JDK6' + jdk 'JDK8' } stages { From 17ab402b32ea4d3ef84e855c7d0adee878e499c3 Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Wed, 30 Nov 2022 17:19:45 +0800 Subject: [PATCH 05/99] =?UTF-8?q?=E4=BF=AE=E6=94=B9jdk?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 9af2ad01b..ad7973bc1 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -19,7 +19,7 @@ pipeline { script { def projects = ['fangjiabkoff', 'chafangtong', 'fangjia', 'fangjiapds', 'fangjiastatic'] for (def project : projects) { - dir('deliverables/${project}') { + dir("deliverables/${project}") { withMaven(globalMavenSettingsConfig: '426b9a25-aa0d-47d3-a24c-8abdfc4421ff') { sh 'mvn clean package -Dmaven.test.skip=true' } @@ -35,7 +35,7 @@ pipeline { def imagesName = ['fangjiabkoff': 'fangjia-bkoff', 'chafangtong': 'chafangtong', 'fangjia': 'fangjia-coresite', 'fangjiapds': 'fangjia-pds', 'fangjiastatic': 'fangjia-static'] for (def project : projects) { if (params.PROJECT_NAME == project) { - dir('projects/${imagesName["project"]}') { + dir("projects/${imagesName["project"]}") { withDockerRegistry(credentialsId: 'DockerNexus', url: 'http://192.168.0.19:9012') { sh "docker build -t 192.168.0.19:9011/${imagesName[project]}:${env.BUILD_TAG.toLowerCase()} ." withDockerRegistry(credentialsId: 'DockerNexus', url: 'http://192.168.0.19:9011') { From 098cfee97088988b7fd24fd2e210994fbd22ac13 Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Wed, 30 Nov 2022 17:47:58 +0800 Subject: [PATCH 06/99] =?UTF-8?q?=E4=BF=AE=E6=94=B9jdk?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Jenkinsfile | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index ad7973bc1..ec9643d7a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -19,9 +19,11 @@ pipeline { script { def projects = ['fangjiabkoff', 'chafangtong', 'fangjia', 'fangjiapds', 'fangjiastatic'] for (def project : projects) { - dir("deliverables/${project}") { - withMaven(globalMavenSettingsConfig: '426b9a25-aa0d-47d3-a24c-8abdfc4421ff') { - sh 'mvn clean package -Dmaven.test.skip=true' + if (params.PROJECT_NAME == project) { + dir("deliverables/${project}") { + withMaven(globalMavenSettingsConfig: '426b9a25-aa0d-47d3-a24c-8abdfc4421ff') { + sh 'mvn clean package -Dmaven.test.skip=true' + } } } } From 2aa9bdab9f0fb9ffe778cd609ffe15885d710e4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=96=87=E8=89=B3?= Date: Wed, 30 Nov 2022 22:05:05 +0800 Subject: [PATCH 07/99] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E6=A0=B7=E6=9C=AC?= =?UTF-8?q?=E6=9F=A5=E8=AF=A2bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/service/sql/UserServiceImpl.java | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/sql/UserServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/sql/UserServiceImpl.java index e0e763666..d849f26c6 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/sql/UserServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/sql/UserServiceImpl.java @@ -7,12 +7,14 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import com.fangjia.core.util.CacheUtils; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; import com.fangjia.dal.QueryParam; import com.fangjia.dal.sql.dao.UserDao; import com.fangjia.dal.sql.model.User; +import org.sothis.core.util.Closure; public class UserServiceImpl implements UserService { private UserDao userDao; @@ -28,10 +30,15 @@ public class UserServiceImpl implements UserService { return Collections.emptyList(); } - public User findByUserName(String userName) { - List params = new ArrayList(1); - params.add(new QueryParam("userName", userName)); - return userDao.findFirstByParams(params); + public User findByUserName(final String userName) { + return CacheUtils.getValue(UserServiceImpl.class, "findByUserName", userName, CacheUtils.CachePolicyEnum.TODAY, new Closure(){ + @Override + public User execute(String input) { + List params = new ArrayList(1); + params.add(new QueryParam("userName", userName)); + return userDao.findFirstByParams(params); + } + }); } public void updateUser(User user) { From f4e3cdfb99123f61ea23f11879182f16fe0d7698 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=96=87=E8=89=B3?= Date: Wed, 30 Nov 2022 22:43:40 +0800 Subject: [PATCH 08/99] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E6=A0=B7=E6=9C=AC?= =?UTF-8?q?=E6=9F=A5=E8=AF=A2bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/open/interceptors/AuthenticationInterceptor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/interceptors/AuthenticationInterceptor.java b/projects/fangjia-open/src/main/java/com/fangjia/open/interceptors/AuthenticationInterceptor.java index 5733be72d..eff33a38f 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/interceptors/AuthenticationInterceptor.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/interceptors/AuthenticationInterceptor.java @@ -213,7 +213,7 @@ public class AuthenticationInterceptor implements Interceptor { if (null == apiUser) { return ret; } - if(apiUser.getDeadTime() != null && new Date().after(apiUser.getDeadTime())){ + if(apiUser.getDeadTime() != null && DateUtil.truncateDay(new Date()).after(apiUser.getDeadTime())){ return new HttpCode(OpenHttpCodeEnum.USER_EXPIRED); } if(!isOldUser(apiUser) && isExpired(token)){ From daec6d15e11beb92c904ad83fea144232a52b99d Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Wed, 30 Nov 2022 23:04:00 +0800 Subject: [PATCH 09/99] =?UTF-8?q?bug=E4=BF=AE=E5=A4=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index ec9643d7a..37a7f1e92 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -37,7 +37,7 @@ pipeline { def imagesName = ['fangjiabkoff': 'fangjia-bkoff', 'chafangtong': 'chafangtong', 'fangjia': 'fangjia-coresite', 'fangjiapds': 'fangjia-pds', 'fangjiastatic': 'fangjia-static'] for (def project : projects) { if (params.PROJECT_NAME == project) { - dir("projects/${imagesName["project"]}") { + dir("projects/${imagesName[project]}") { withDockerRegistry(credentialsId: 'DockerNexus', url: 'http://192.168.0.19:9012') { sh "docker build -t 192.168.0.19:9011/${imagesName[project]}:${env.BUILD_TAG.toLowerCase()} ." withDockerRegistry(credentialsId: 'DockerNexus', url: 'http://192.168.0.19:9011') { From e0acf09b56dbeaaefb44e9abae01e30b01b63d34 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 6 Dec 2022 14:56:42 +0800 Subject: [PATCH 10/99] =?UTF-8?q?=E8=9E=8D=E5=90=88=E5=9D=87=E4=BB=B7?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E8=BF=9E=E6=8E=A5=E6=95=B0=E6=8D=AE=E5=BA=93?= =?UTF-8?q?=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ccbevaluation/AddThirdPartyPrice.groovy | 4 +- .../ccbevaluation/BigdataDealBatch.groovy | 22 +- .../ccbevaluation/CcbCommonUtils.groovy | 2 +- .../ccbevaluation/CcbFtpTestUtils.groovy | 20 +- .../DealAndThirdPriceDataProcess.groovy | 4 +- .../ExportOracleAllDataTest.groovy | 8 +- .../ccbevaluation/GuiDingPriceBatch.groovy | 7 +- .../ccbevaluation/ThirdPartyPriceBatch.groovy | 4 +- .../script/ccbevaluation/TradeStatLog.groovy | 19 +- .../ronghe/ExportDistrictRongHeTest.groovy | 121 +++---- .../script/commData/CommonDatautils.groovy | 32 +- .../fangjia/script/gj/YinLianMeiShi.groovy | 305 ++++++++++++++++++ .../gj/crawl/AnjukeDistrictMatch.groovy | 205 +++++++++--- .../gj/paimai/BeiKeCrawlByChrome.groovy | 2 +- .../gj/paimai/LianjiaCrawlByChrome.groovy | 2 +- .../com/fangjia/script/gj/test/Test.groovy | 124 +++++-- .../fangjia/script/wenyan/temp/QinLin.groovy | 10 +- 17 files changed, 730 insertions(+), 161 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YinLianMeiShi.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddThirdPartyPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddThirdPartyPrice.groovy index 7a641d018..ca14ea514 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddThirdPartyPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddThirdPartyPrice.groovy @@ -17,8 +17,8 @@ class AddThirdPartyPrice { } def run(){ -//"º£Î÷ÖÝ","ÌìÃÅ","´óÐ˰²Áë","ÙÙÖÝ",,"º×¸Ú","ÈÕ¿¦Ôò","ͳ·¬", "°¢ÀÕÌ©","µÏÇìÖÝ","DZ½­","Ëþ³Ç","¿Ë×ÎÀÕËÕ","ºÍÌï","¼ÎÓø¹Ø" - def citys = ["ÁÖÖ¥"] +// + def citys = ["ÁÖÖ¥","º£Î÷ÖÝ","ÌìÃÅ","´óÐ˰²Áë","ÙÙÖÝ","º×¸Ú","ÈÕ¿¦Ôò","ͳ·¬", "°¢ÀÕÌ©","µÏÇìÖÝ","DZ½­","Ëþ³Ç","¿Ë×ÎÀÕËÕ","ºÍÌï","¼ÎÓø¹Ø"] def rows = [] def cityCodeDatas = CcbCommonUtils.findCityCodeValue() for(def c : citys){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 28c543e60..d73c64d44 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -58,7 +58,7 @@ class BigdataDealBatch extends BaseBatch{ public void process(Object args) { //def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date()) // def citys = CcbCommonUtils.citys; - def citys = CcbCommonUtils.getRunningCitys(args) + def citys = ["ÉϺ£"]//CcbCommonUtils.getRunningCitys(args) if(args && args.size()>=2){ curDate = args[1].toString().toInteger() } @@ -87,16 +87,16 @@ class BigdataDealBatch extends BaseBatch{ } - def trades = getCityData(city, statMap, ppeDists) - if(trades){ - def rets = createData(trades, cityCodeDataMap, distCodeMap, "other") - rows.addAll(rets) - println "¡¾${index}¡¿"+city +" ³É½»¹²£º"+rets.size() - - def cityDatas = [] - cityDatas << headers - cityDatas.addAll(rows) - } +// def trades = getCityData(city, statMap, ppeDists) +// if(trades){ +// def rets = createData(trades, cityCodeDataMap, distCodeMap, "other") +// rows.addAll(rets) +// println "¡¾${index}¡¿"+city +" ³É½»¹²£º"+rets.size() +// +// def cityDatas = [] +// cityDatas << headers +// cityDatas.addAll(rows) +// } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index 517f8c44c..a551c50b3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -52,7 +52,7 @@ class CcbCommonUtils { def countStr = args[0] if(countStr=="344"){ citys = CcbCommonUtils.citys_all - sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl", + sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") }else if(countStr=="92"){ citys = CcbCommonUtils.citys_92 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpTestUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpTestUtils.groovy index 2a9ad549f..4bac05c3f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpTestUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpTestUtils.groovy @@ -13,7 +13,7 @@ class CcbFtpTestUtils { static def ftpPassword = "ccb!@#" static def ftpPath = "/home/ccb_evaluation/output/" - static def ftpPath_rh = "/home/ccb_rh/output/" + static def ftpPath_rh = "/home/ccb_ronghe/output/" static SFTP sftp = new SFTP(); static ChannelSftp sftpChannel = sftp.connect(ftpServer, ftpPort, ftpUser, ftpPassword); @@ -70,16 +70,26 @@ class CcbFtpTestUtils { } - static def listFileNames(month){ + static def listFileNames(type, month){ //ÉÏ´«µ½ftp sftpChannel = reConnect(); - Vector files = sftp.listFiles(ftpPath+ month+"/", sftpChannel); + Vector files = null; + if(type && type=="rh"){ + files = sftp.listFiles(ftpPath_rh+ month+"/", sftpChannel); + }else{ + files = sftp.listFiles(ftpPath+ month+"/", sftpChannel); + } return files } - static def getFileLength(String path, String month){ + static def getFileLength(String type, String path, String month){ sftpChannel = reConnect(); - def size = sftp.getFileSize(ftpPath+ month+"/"+path, sftpChannel) + def size = 0; + if(type && type=="rh"){ + size = sftp.getFileSize(ftpPath_rh+ month+"/"+path, sftpChannel) + }else{ + size = sftp.getFileSize(ftpPath+ month+"/"+path, sftpChannel) + } return size } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy index a96478424..e67f2a8cc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy @@ -371,11 +371,11 @@ class DealPriceDataProcess { def deleteThirdPartyData(){ def citys = CcbCommonUtils.citys_92 - def month = 202209 + def month = 202210 def rows = [] def cityCodeDatas = CcbCommonUtils.findCityCodeValue() for(def city : citys) { - def rets = findPeGteThirdParty(city, 202210) + def rets = findPeGteThirdParty(city, 202211) def total = 0 println city +" "+rets.size() for(def m : rets){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy index 71c378532..d35310b33 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy @@ -54,7 +54,7 @@ class ExportOracleAllDataTest { // exportCommunityStaging()//ûcityid // exportCommunityBuileType()// - exportThirdPartyPrice2(new_dists)// +// exportThirdPartyPrice2(new_dists)// // exportBigdataForsale(new_dists) //2022-07-19 // exportBigdataDeal( new_dists) //2022-07-01 // @@ -325,7 +325,7 @@ class ExportOracleAllDataTest { def allRows = [] for(def city : citys) { def cityCode = cityCodeMap.get(city) - def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, AVGPRICE, SOURCETYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME , SOURCECODE from community_guide_price_gj where CITYID='${cityCode}' and createtime>=to_date('${date}','yyyyMMdd')"; + def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, AVGPRICE, SOURCETYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME , SOURCECODE from community_guide_price_gj where CITYID='${cityCode}' ";//and createtime>=to_date('${date}','yyyyMMdd') sqlInstance.eachRow(sql, { r -> def data = [] data << r.ID @@ -1172,14 +1172,14 @@ class ExportOracleAllDataTest { def exportInOk(){ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" - Vector files = CcbFtpTestUtils.listFileNames(curDate) + Vector files = CcbFtpTestUtils.listFileNames(null, curDate) Iterator fileVec = files.iterator() def fileRows = [] while(fileVec.hasNext()){ ChannelSftp.LsEntry file = fileVec.next() if(org.apache.commons.lang.StringUtils.isNotBlank(file.getFilename()) && file.getFilename().endsWith(".dat")){ - fileRows << file.getFilename() +" "+CcbFtpTestUtils.getFileLength(file.getFilename(), curDate) + fileRows << file.getFilename() +" "+CcbFtpTestUtils.getFileLength(null, file.getFilename(), curDate) } } def fileName = "/ccb_evaluation/${curDate}.ok" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index bd64cc3f8..0d15402de 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -38,16 +38,17 @@ class GuiDingPriceBatch extends BaseBatch{ @Override public void process(Object args) { - def citys = CcbCommonUtils.getRunningCitys(args) + def citys = ["ÉϺ£"]//CcbCommonUtils.getRunningCitys(args) def rows = [] - CcbCommonUtils.deleteGuidedPrice(curDate) +// CcbCommonUtils.deleteGuidedPrice(curDate) for (def city : citys) { def cityCodeDatas = CcbCommonUtils.findCityCodeValue() def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) //Ìí¼Ó×Ô¼º¿âÖеÄÈ˹¤Ö¸µ¼¼Û - def prices = CcbCommonUtils.findPersonPrice(city, "2022-10-27", "2022-11-18") + def prices = CcbCommonUtils.findPersonPrice(city, "2022-11-24", "2022-11-25") +// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22") if(prices){ def distMap = getDistrictMap(city) def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy index 03d4cfcb7..027e71834 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy @@ -45,7 +45,7 @@ class ThirdPartyPriceBatch extends BaseBatch { def citys = CcbCommonUtils.getRunningCitys(args) def rows = [] def index = 0 - def weekend = DateUtils.getFirstDayOfWeek(new Date()) + def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightByPattern(sDate+"15", "yyyyMMdd")) def cityCodeDatas = CcbCommonUtils.findCityCodeValue() for(def city : citys){ index++ @@ -67,7 +67,7 @@ class ThirdPartyPriceBatch extends BaseBatch { } def header = []//["µÚÈý·½¾ù¼ÛID","Â¥ÅÌID","Ð¡ÇøÃû³Æ","³ÇÊÐID","³ÇÊÐÃû³Æ","ÐÐÕþÇøID","ÐÐÕþÇøÃû³Æ","¾ù¼Û¶ÔÓ¦µÄÔ·Ý","Öî¸ðÕÒ·¿¾ù¼Û","°²¾Ó¿Í¾ù¼Û","·¿ÌìϾù¼Û","Á´¼Ò¾ù¼Û","³ÇÊз¿²ú¾ù¼Û","ÂóÌï¾ù¼Û","±´¿Ç¾ù¼Û","ÌÔ·¿365","ÖÐÔ­¾ù¼Û","Íò©Ê¿¾ù¼Û","Q·¿¾ù¼Û","·¿Ì½¾ù¼Û","·¿¶à¶à¾ù¼Û","ÎÒ°®ÎÒ¼Ò¾ù¼Û","´´½¨Ê±¼ä","¸üÐÂʱ¼ä","À´Ô´¹«Ë¾"] - FileUtils.batchCreateFile(rows, fileName, header, "utf-8", false); + FileUtils.batchCreateFile(rows, fileName, header, "utf-8", true); } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy index fd96c4d80..ae3aa4800 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy @@ -40,7 +40,7 @@ class TradeStatLog { static main(args) { def task = new TradeStatLog() - task.run()//ͳ¼Æ×î½ü3¸öÔÂץȡµÄ³É½»Êý¾ÝÁ¿ +// task.run()//ͳ¼Æ×î½ü3¸öÔÂץȡµÄ³É½»Êý¾ÝÁ¿ // task.exportTrade()//µ¼³öδÈë¿â³É½» // task.thirdPartyStat() // println Calendar.getInstance().get(Calendar.YEAR) @@ -50,8 +50,25 @@ class TradeStatLog { // task.deleteDealData2() // task.statTableCount() // task.filterTrade() + task.updateDealPriceType() + } + def updateDealPriceType(){ + def citys = CommonDatautils.findAllCity() + def index = 0 + for(def c : citys){ + DEAL_PRICE_COLL.find(new BasicDBObject("city", c)).sort(new BasicDBObject("trade_date",-1)).batchSize(5000).each {it-> + if(it.total_price instanceof Double){ + index ++ + println index+" "+it.city+" "+it.total_price+" "+it._id.toString() + + def udata = new BasicDBObject() + udata.put("total_price", (int)it.total_price) + DEAL_PRICE_COLL.update(new BasicDBObject("_id", it._id), new BasicDBObject('$set', udata)) + } + } + } } def statTableCount(){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy index 161f82f00..312032ca4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy @@ -8,6 +8,7 @@ import com.fangjia.script.ccbevaluation.CcbFtpTestUtils import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.ccb.FileUtils import com.fangjia.script.util.DateUtils +import com.jcraft.jsch.ChannelSftp import groovy.sql.Sql @BatchOwner(Watcher.GaoJuan) @@ -25,7 +26,7 @@ class ExportDistrictRongHeTest extends BaseBatch{ @Override void process(Object args) { citys = CcbCommonUtils.getRunningCitys(args) - def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd") + def curDate = "20221110"//DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd") def new_dists = []//["350604","350605","410307","410308","450181"] //°´Ììµ¼£¨Ã¿ÌìÒ»¸öÎļþ¼Ð£© // exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-1), "yyyy-MM-dd"), new_dists) @@ -48,17 +49,17 @@ class ExportDistrictRongHeTest extends BaseBatch{ // exportBigdataDistrictFJW()// // exportBigdataAliasFJW()// - def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger() - exportBigdataCityPrice(queryDate) - exportBigdataDistPrice(queryDate) - exportBigdataCommPrice(queryDate) +// def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger() +// exportBigdataCityPrice(queryDate) +// exportBigdataDistPrice(queryDate) +// exportBigdataCommPrice(queryDate) + + exportInOk() // } + SSHHelperTest jschCommand = new SSHHelperTest(); - System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}")); - - exportOk(curDate) - + System.out.println(jschCommand.executeCommand("sh /home/ccb_ronghe/output/rh_package.sh ${curDate}")); } @@ -66,9 +67,9 @@ class ExportDistrictRongHeTest extends BaseBatch{ def exportBigdataCommunityGJRH(){ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat" + def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat" } def index = 0 for(def city : citys){ @@ -133,9 +134,9 @@ class ExportDistrictRongHeTest extends BaseBatch{ } def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat" + def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) @@ -162,9 +163,9 @@ class ExportDistrictRongHeTest extends BaseBatch{ } def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat" + def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) @@ -190,9 +191,9 @@ class ExportDistrictRongHeTest extends BaseBatch{ } def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat" + def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); CcbFtpTestUtils.ftpUpload("rh", fileName, curDate) @@ -228,9 +229,9 @@ class ExportDistrictRongHeTest extends BaseBatch{ }); } def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat" + def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) @@ -258,9 +259,9 @@ class ExportDistrictRongHeTest extends BaseBatch{ } def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat" + def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) @@ -291,9 +292,9 @@ class ExportDistrictRongHeTest extends BaseBatch{ } def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat" + def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) @@ -323,9 +324,9 @@ class ExportDistrictRongHeTest extends BaseBatch{ } def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat" + def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); CcbFtpTestUtils.ftpUpload("rh", fileName, curDate) @@ -358,9 +359,9 @@ class ExportDistrictRongHeTest extends BaseBatch{ } def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat" + def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) @@ -382,9 +383,9 @@ class ExportDistrictRongHeTest extends BaseBatch{ index++ def curStartStr = curStartDateStr - def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat" + def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat" + fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat" } def allRows = [] @@ -482,9 +483,9 @@ class ExportDistrictRongHeTest extends BaseBatch{ def endDateStr = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd") // def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd") - def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" + def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" + fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" } def allRows = [] @@ -675,9 +676,9 @@ class ExportDistrictRongHeTest extends BaseBatch{ allRows << data }); def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" + def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" + fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); println "BIGDATA_DEAL_FJW µ¼³öÍê³É......" @@ -686,9 +687,9 @@ class ExportDistrictRongHeTest extends BaseBatch{ def exportBigdataBuildingFJW(){ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat" + def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat" } def index = 0 for(def city : citys) { @@ -743,9 +744,9 @@ class ExportDistrictRongHeTest extends BaseBatch{ def exportBigdataCityPrice(queryDate){ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_CITYPRICE_ADD_${curDate}_0001.dat" + def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_CITYPRICE_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_CITYPRICE_ADD_${curDate}_0001.dat" + fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_CITYPRICE_ADD_${curDate}_0001.dat" } def allRows = [] for(def city : citys) { @@ -789,9 +790,9 @@ class ExportDistrictRongHeTest extends BaseBatch{ def exportBigdataDistPrice(queryDate){ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DISTPRICE_ADD_${curDate}_0001.dat" + def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_DISTPRICE_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DISTPRICE_ADD_${curDate}_0001.dat" + fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_DISTPRICE_ADD_${curDate}_0001.dat" } def allRows = [] for(def city : citys) { @@ -836,9 +837,9 @@ class ExportDistrictRongHeTest extends BaseBatch{ def exportBigdataCommPrice(queryDate){ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMPRICE_ADD_${curDate}_0001.dat" + def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_COMMPRICE_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMPRICE_ADD_${curDate}_0001.dat" + fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_COMMPRICE_ADD_${curDate}_0001.dat" } def allRows = [] for(def city : citys) { @@ -899,9 +900,9 @@ class ExportDistrictRongHeTest extends BaseBatch{ allRows << data }); def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat" + def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) @@ -910,9 +911,9 @@ class ExportDistrictRongHeTest extends BaseBatch{ def exportBigdataBuildprojFjw(){ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat" + def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile([], fileName, [], "utf-8", false); CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) @@ -921,23 +922,35 @@ class ExportDistrictRongHeTest extends BaseBatch{ def exportBigdataCommlayoutpicFjw(){ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat" + def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile([], fileName, [], "utf-8", false); CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) println "BIGDATA_COMMLAYOUTPIC_FJW ÉÏ´«FtpÍê³É......" } - def exportOk(curDate){ -// def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd") - def dataCodeFileName = "/ccb_rh/${curDate}/${curDate}.ok" - if(CommonDatautils.isWindow()){ - dataCodeFileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\ccb_rh\\${curDate}\\${curDate}.ok" + + def exportInOk(){ + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" + Vector files = CcbFtpTestUtils.listFileNames("rh", curDate) + Iterator fileVec = files.iterator() + def fileRows = [] + while(fileVec.hasNext()){ + ChannelSftp.LsEntry file = fileVec.next() + + if(org.apache.commons.lang.StringUtils.isNotBlank(file.getFilename()) && file.getFilename().endsWith(".dat")){ + fileRows << file.getFilename() +" "+CcbFtpTestUtils.getFileLength("rh", file.getFilename(), curDate) + } } - FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false); - CcbFtpTestUtils.ftpUpload("rh",dataCodeFileName, curDate) + def fileName = "/ccb_ronghe/${curDate}/${curDate}.ok" + if(CommonDatautils.isWindow()){ + fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\ccb_ronghe\\${curDate}\\${curDate}.ok" + } + FileUtils.createTxtFile(fileRows, fileName, "utf-8", false); + CcbFtpTestUtils.ftpUpload("rh", fileName, curDate) + println "ÈÚºÏͳ¼ÆÎļþ³¤¶ÈokÎļþ ÉÏ´«FtpÍê³É......" } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 49bacf4ee..c6d2373a5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -755,7 +755,7 @@ class CommonDatautils { q.put("visible",0) q.put("cat", cat); q.put("city", city); -// q.put("name","ºþÅϼÑÔ·") +// q.put("name","Íò´ï»ª¸®Ò»ÆÚ") DBObject keys = new BasicDBObject(); if(fields){ @@ -903,6 +903,36 @@ class CommonDatautils { } return null; } + + + public static def findSeaweedById(String id, def filelds){ + if(!id){ + return null; + } + try{ + DBObject q = new BasicDBObject(); + q.put("_id", new ObjectId(id.toString().replace("[","").replace("]",""))); + + DBObject keys = new BasicDBObject(); + filelds.each{ + keys.put(it, 1); + } + try{ + DSCursor cursor = getDataStoreConnection().find(q, keys); + if(cursor){ + for(def obj : cursor){ + return obj; + } + } + }catch(Exception e){ + println "error_id:"+id + e.printStackTrace() + } + }catch(Exception e){ + + } + return null; + } public static def findDistrictBySid(String sid){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YinLianMeiShi.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YinLianMeiShi.groovy new file mode 100644 index 000000000..d1aca25f7 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YinLianMeiShi.groovy @@ -0,0 +1,305 @@ +package com.fangjia.script.gj + +import com.fangjia.core.RegionData +import com.fangjia.core.util.BaiDuApiUtils +import com.fangjia.core.util.GeocodeUtils +import com.fangjia.core.util.HttpRequest +import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.util.CosineSimilarAlgorithm +import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject +import net.sf.json.JSONObject +import org.apache.poi.ss.util.RegionUtil + +class YinLianMeiShi { + + def baiduCityCodeMap = ["±±¾©": "131", "Ìì½ò": "332", "ʯ¼Òׯ": "150", "ÌÆÉ½": "265", "ÇØ»Êµº": "148", "ºªµ¦": "151", "ÐĮ̈": "266", "±£¶¨": "307", "ÕżҿÚ": "264", "³ÐµÂ": "207", "²×ÖÝ": "149", "ÀÈ·»": "191", "ºâË®": "208", "̫ԭ": "176", "´óͬ": "355", "ÑôȪ": "357", "³¤ÖÎ": "356", "½ú³Ç": "290", "Ë·ÖÝ": "237", "½úÖÐ": "238", "Ô˳Ç": "328", "ÐÃÖÝ": "367", "ÁÙ·Ú": "368", "ÂÀÁº": "327", "ºôºÍºÆÌØ": "321", "°üÍ·": "229", "ÎÚº£": "123", "³à·å": "297", "ͨÁÉ": "64", "¶õ¶û¶à˹": "283", "ºôÂ×±´¶û": "61", "°ÍÑåÄ×¶û": "169", "ÎÚÀ¼²ì²¼": "168", "Ð˰²ÃË": "62", "ÎýÁÖ¹ùÀÕÃË": "63", "°¢À­ÉÆÃË": "230", "ÉòÑô": "58", "´óÁ¬": "167", "°°É½": "320", "¸§Ë³": "184", "±¾Ïª": "227", "µ¤¶«": "282", "½õÖÝ": "166", "Óª¿Ú": "281", "¸·ÐÂ": "59", "ÁÉÑô": "351", "Å̽õ": "228", "ÌúÁë": "60", "³¯Ñô": "280", "ºù«µº": "319", "³¤´º": "53", "¼ªÁÖÊÐ": "55", "ËÄÆ½": "56", "ÁÉÔ´": "183", "ͨ»¯": "165", "°×ɽ": "57", "ËÉÔ­": "52", "°×³Ç": "51", "Ñӱ߳¯ÏÊ×å×ÔÖÎÖÝ": "54", "¹þ¶û±õ": "48", "ÆëÆë¹þ¶û": "41", "¼¦Î÷": "46", "º×¸Ú": "43", "˫Ѽɽ": "45", "´óÇì": "50", "ÒÁ´º": "40", "¼Ñľ˹": "42", "Æß̨ºÓ": "47", "ĵµ¤½­": "49", "ºÚºÓ": "39", "Ë绯": "44", "´óÐ˰²ÁëµØÇø": "38", "ÉϺ£": "289", "ÄϾ©": "315", "ÎÞÎý": "317", "ÐìÖÝ": "316", "³£ÖÝ": "348", "ËÕÖÝ": "224", "ÄÏͨ": "161", "Á¬ÔƸÛ": "347", "»´°²": "162", "ÑγÇ": "223", "ÑïÖÝ": "346", "Õò½­": "160", "Ì©ÖÝ": "276", "ËÞǨ": "277", "º¼ÖÝ": "179", "Äþ²¨": "180", "ÎÂÖÝ": "178", "¼ÎÐË": "334", "ºþÖÝ": "294", "ÉÜÐË": "293", "½ð»ª": "333", "áéÖÝ": "243", "ÖÛɽ": "245", "̨ÖÝ": "244", "ÀöË®": "292", "ºÏ·Ê": "127", "Îߺþ": "129", "°ö²º": "126", "»´ÄÏ": "250", "Âí°°É½": "358", "»´±±": "253", "Í­Áê": "337", "°²Çì": "130", "»ÆÉ½": "252", "³üÖÝ": "189", "¸·Ñô": "128", "ËÞÖÝ": "370", "³²ºþ": "251", "Áù°²": "298", "ÙñÖÝ": "188", "³ØÖÝ": "299", "Ðû³Ç": "190", "¸£ÖÝ": "300", "ÏÃÃÅ": "194", "ÆÎÌï": "195", "ÈýÃ÷": "254", "ȪÖÝ": "134", "ÕÄÖÝ": "255", "ÄÏÆ½": "133", "ÁúÑÒ": "193", "ÄþµÂ": "192", "Äϲý": "163", "¾°µÂÕò": "225", "ƼÏç": "350", "¾Å½­": "349", "ÐÂÓà": "164", "ӥ̶": "279", "¸ÓÖÝ": "365", "¼ª°²": "318", "ÒË´º": "278", "¸§ÖÝ": "226", "ÉÏÈÄ": "364", "¼ÃÄÏ": "288", "Çൺ": "236", "×Ͳ©": "354", "Ôæ×¯": "172", "¶«Óª": "174", "ÑĮ̀": "326", "Ϋ·»": "287", "¼ÃÄþ": "286", "Ì©°²": "325", "Íþº£": "175", "ÈÕÕÕ": "173", "À³Îß": "124", "ÁÙÒÊ": "234", "µÂÖÝ": "372", "ÁijÇ": "366", "±õÖÝ": "235", "ºÊÔó": "353", "Ö£ÖÝ": "268", "¿ª·â": "210", "ÂåÑô": "153", "ƽ¶¥É½": "213", "°²Ñô": "267", "º×±Ú": "215", "ÐÂÏç": "152", "½¹×÷": "211", "å§Ñô": "209", "Ðí²ý": "155", "äðºÓ": "344", "ÈýÃÅÏ¿": "212", "ÄÏÑô": "309", "ÉÌÇð": "154", "ÐÅÑô": "214", "ÖÜ¿Ú": "308", "פÂíµê": "269", "Î人": "218", "»ÆÊ¯": "311", "Ê®Ñß": "216", "Ò˲ý": "270", "ÏåÑô": "156", "¶õÖÝ": "122", "¾£ÃÅ": "217", "Т¸Ð": "310", "¾£ÖÝ": "157", "»Æ¸Ô": "271", "ÏÌÄþ": "362", "ËæÖÝ": "371", "¶÷Ê©ÍÁ¼Ò×åÃç×å×ÔÖÎÖÝ": "373", "ÏÉÌÒ": "1713", "DZ½­": "1293", "ÌìÃÅ": "2654", "ÉñÅ©¼ÜÁÖÇø": "2734", "³¤É³": "158", "ÖêÖÞ": "222", "ÏæÌ¶": "313", "ºâÑô": "159", "ÉÛÑô": "273", "ÔÀÑô": "220", "³£µÂ": "219", "Õżҽç": "312", "ÒæÑô": "272", "³»ÖÝ": "275", "ÓÀÖÝ": "314", "»³»¯": "363", "¦µ×": "221", "ÏæÎ÷ÍÁ¼Ò×åÃç×å×ÔÖÎÖÝ": "274", "¹ãÖÝ": "257", "ÉØ¹Ø": "137", "ÉîÛÚ": "340", "Ö麣": "140", "ÉÇÍ·": "303", "·ðɽ": "138", "½­ÃÅ": "302", "Õ¿½­": "198", "ïÃû": "139", "ÕØÇì": "338", "»ÝÖÝ": "301", "÷ÖÝ": "141", "ÉÇβ": "339", "ºÓÔ´": "200", "Ñô½­": "199", "ÇåÔ¶": "197", "¶«Ý¸": "119", "ÖÐɽ": "187", "³±ÖÝ": "201", "½ÒÑô": "259", "ÔÆ¸¡": "258", "ÄÏÄþ": "261", "ÁøÖÝ": "305", "¹ðÁÖ": "142", "ÎàÖÝ": "304", "±±º£": "295", "·À³Ç¸Û": "204", "ÇÕÖÝ": "145", "¹ó¸Û": "341", "ÓñÁÖ": "361", "°ÙÉ«": "203", "ºØÖÝ": "260", "ºÓ³Ø": "143", "À´±ö": "202", "³ç×ó": "144", "º£¿Ú": "125", "ÈýÑÇ": "121", "Îåָɽ": "1644", "Çíº£": "2358", "ÙÙÖÝ": "1215", "ÎIJý": "2758", "ÍòÄþ": "1216", "¶«·½": "2634", "¶¨°²": "1214", "ÍͲý": "1641", "³ÎÂõ": "2757", "ÁÙ¸ß": "2033", "°×ɳÀè×å×ÔÖÎ": "2359", "²ý½­Àè×å×ÔÖÎ": "1642", "ÀÖ¶«Àè×å×ÔÖÎ": "2032", "ÁêË®Àè×å×ÔÖÎ": "1643", "±£Í¤Àè×åÃç×å×ÔÖÎ": "1217", "ÇíÖÐÀè×åÃç×å×ÔÖÎ": "2031", "ÖØÇì": "132", "³É¶¼": "75", "×Ô¹±": "78", "ÅÊÖ¦»¨": "81", "ãòÖÝ": "331", "µÂÑô": "74", "ÃàÑô": "240", "¹ãÔª": "329", "ËìÄþ": "330", "ÄÚ½­": "248", "ÀÖɽ": "79", "Äϳä": "291", "üɽ": "77", "Ò˱ö": "186", "¹ã°²": "241", "´ïÖÝ": "369", "ÑŰ²": "76", "°ÍÖÐ": "239", "×ÊÑô": "242", "°¢°Ó²Ø×åǼ×å×ÔÖÎÖÝ": "185", "¸Ê×βØ×å×ÔÖÎÖÝ": "73", "Á¹É½ÒÍ×å×ÔÖÎÖÝ": "80", "¹óÑô": "146", "ÁùÅÌË®": "147", "×ñÒå": "262", "°²Ë³": "263", "Í­ÈʵØÇø": "205", "Ç­Î÷Äϲ¼ÒÀ×åÃç×å×ÔÖÎÖÝ": "343", "±Ï½ÚµØÇø": "206", "Ç­¶«ÄÏÃç×å¶±×å×ÔÖÎÖÝ": "342", "Ç­Äϲ¼ÒÀ×åÃç×å×ÔÖÎÖÝ": "306", "À¥Ã÷": "104", "Çú¾¸": "249", "ÓñϪ": "106", "±£É½": "112", "ÕÑͨ": "336", "Àö½­": "114", "ÁÙ²×": "110", "³þÐÛÒÍ×å×ÔÖÎÖÝ": "105", "ºìºÓ¹þÄá×åÒÍ×å×ÔÖÎÖÝ": "107", "ÎÄɽ׳×åÃç×å×ÔÖÎÖÝ": "177", "ÆÕ¶ý": "108", "Î÷Ë«°æÄÉ´ö×å×ÔÖÎÖÝ": "109", "´óÀí°××å×ÔÖÎÖÝ": "111", "µÂºê´ö×å¾°ÆÄ×å×ÔÖÎÖÝ": "116", "Å­½­ÀüËÛ×å×ÔÖÎÖÝ": "113", "µÏÇì²Ø×å×ÔÖÎÖÝ": "115", "À­Èø": "100", "²ý¶¼µØÇø": "99", "ɽÄϵØÇø": "97", "ÈÕ¿¦ÔòµØÇø": "102", "ÄÇÇúµØÇø": "101", "°¢ÀïµØÇø": "103", "ÁÖÖ¥µØÇø": "98", "Î÷°²": "233", "Í­´¨": "232", "±¦¼¦": "171", "ÏÌÑô": "323", "μÄÏ": "170", "ÑÓ°²": "284", "ººÖÐ": "352", "ÓÜÁÖ": "231", "°²¿µ": "324", "ÉÌÂå": "285", "À¼ÖÝ": "36", "¼ÎÓø¹Ø": "33", "½ð²ý": "34", "°×Òø": "35", "ÌìË®": "196", "ÎäÍþ": "118", "ÕÅÒ´": "117", "ƽÁ¹": "359", "¾ÆÈª": "37", "ÇìÑô": "135", "¶¨Î÷": "136", "¤ÄÏ": "256", "ÁÙÏÄ»Ø×å×ÔÖÎÖÝ": "182", "¸ÊÄϲØ×å×ÔÖÎÖÝ": "247", "Î÷Äþ": "66", "º£¶«µØÇø": "69", "º£±±²Ø×å×ÔÖÎÖÝ": "67", "»ÆÄϲØ×å×ÔÖÎÖÝ": "70", "º£ÄϲØ×å×ÔÖÎÖÝ": "68", "¹ûÂ岨×å×ÔÖÎÖÝ": "72", "ÓñÊ÷²Ø×å×ÔÖÎÖÝ": "71", "º£Î÷ÃɹÅ×岨×å×ÔÖÎÖÝ": "65", "Òø´¨": "360", "ʯ×ìɽ": "335", "ÎâÖÒ": "322", "¹ÌÔ­": "246", "ÖÐÎÀ": "181", "ÎÚ³ľÆë": "92", "¿ËÀ­ÂêÒÀ": "95", "ͳ·¬µØÇø": "89", "¹þÃܵØÇø": "91", "²ý¼ª»Ø×å×ÔÖÎÖÝ": "93", "²©¶ûËþÀ­ÃɹÅ×ÔÖÎÖÝ": "88", "°ÍÒô¹ùÀãÃɹÅ×ÔÖÎÖÝ": "86", "°¢¿ËËÕµØÇø": "85", "¿Ë×ÎÀÕËտ¶û¿Ë×Î×ÔÖÎÖÝ": "84", "¿¦Ê²µØÇø": "83", "ºÍÌïµØÇø": "82", "ÒÁÀç": "90", "Ëþ³ÇµØÇø": "94", "°¢ÀÕÌ©µØÇø": "96", "ʯºÓ×Ó": "770", "°¢À­¶û": "731", "Í¼Ä¾Êæ¿Ë": "792", "Îå¼ÒÇþ": "789", "Ïã¸ÛÌØ±ðÐÐÕþÇø": "2912", "°ÄÃÅÌØ±ðÐÐÕþÇø": "2911"] + + def filters = [] + + static main(args) { + def task = new YinLianMeiShi(); + task.run() + } + + def run(){ + ExcelUtils.modify("D:\\Êý¾Ý\\ÒøÁª\\Êý¾Ý500Ìõ-¹Ø¼ü×Ö.xlsx",{row-> + def score = row['Q'] + if(row['E'] && (!score || (score && score.toString().toDouble()<0.8))){ + def city = RegionData.formatCity(row['E']); + + def baiduCode = getBaiduMapCode(city); + def newKeys = filterName(row['D'], row['E'], row['F'], row['B']) + row['I'] = newKeys + + if(baiduCode){ + def rets = searchBaiDuMap(row['B'], baiduCode.toInteger()); + def simpleAddr = getSimpleAddress(city, row['C']) + def bdRes = searchMapBySimilar(city, row['F'], row['B'], simpleAddr, rets, newKeys); + if (bdRes && (!score || (score && bdRes.score>score.toString().toDouble()))) { + row['J'] = bdRes.uid + row['K'] = bdRes.region + row['L'] = bdRes.name + row['M'] = bdRes.old_addr + row['N'] = bdRes.type + row['O'] = bdRes.baidu_lng2 + row['P'] = bdRes.baidu_lat2 + row['Q'] = bdRes.score + if(bdRes.matchType){ + row['R'] = bdRes.matchType + } + } + println row['B'] +" "+ bdRes + } + + } + }) + } + + + def getSimpleAddress(city, address){ + def jsonObj = FangjiaClientUtils.addressStructure(city, address) + def old_addr = null + if(jsonObj && jsonObj.result){ + if(jsonObj.result.street){ + def street = jsonObj.result.street + if(street){ + old_addr = street + } + } +// if(jsonObj.result.streetNumber){ +// def street_number = jsonObj.result.streetNumber +// if(street_number){ +// old_addr = old_addr + street_number +// } +// } + } + return old_addr; + } + + def filterName(pro,city, region, name){ + def newKey = name.replace("ÓÐÏÞÔðÈι«Ë¾", "").replace("ÓÐÏÞ¹«Ë¾","") + if(pro && name.contains(pro)){ + newKey = newKey.replace(pro, "") + } + + def newCity = RegionData.formatCity(city) + if(city && (name.contains(city) || name.contains(newCity))){ + newKey = newKey.replace(city, "").replace(newCity, "") + } + + + if(region && newKey.contains(region)){ + newKey = newKey.replace(region, "") + } + if(newKey.contains("(")){ + def subName = newKey.substring(newKey.indexOf("(")+1, newKey.indexOf(")")); + if(subName.endsWith("½Ö") || subName.endsWith("·") || subName.contains("*")){ + newKey = newKey.replace("("+subName+")", ""); + } + newKey = newKey.replace("(","").replace(")","") + } + return newKey; + } + + def getBaiduMapCode(city){ + if(city){ + return baiduCityCodeMap[city] + } + return city + } + + static def searchBaiDuMap(String address, int cityCode) { + def rets = [] + try { + Thread.sleep(500) + String url = "https://map.baidu.com/?newmap=1&reqflag=pcmap&biz=1&from=webmap&da_par=baidu&pcevaname=pc4.1&qt=s&da_src=searchBox.button&wd="+URLEncoder.encode(address,"UTF-8")+"&c=${cityCode}&src=0&wd2=&pn=0&sug=0&l=11&from=webmap&sug_forward=&device_ratio=1&tn=B_NORMAL_MAP&nn=0&u_loc=13509900,3640104&ie=utf-8&t=${System.currentTimeMillis()}&auth=AUeHbGFG3ZMg2Bv8Y2b8GfVB3OGc35VOuxLNVzzzLHHtSAFSAS7I7vICu2I%400GdFvkGcuVtvvhguVtvyheuVtvCMGuVtvCQMuVtvIPcuxtw8wkvyuw904v7uv%40vcuVtvc3CuVtvcPPuVtveGvuNzHtAimNzCywiKDv7uvhgMuzVVtcvY1SGpuxRtKHZz3%40SdJMjjejCEBf0wd0vyMO7Ay7MOI&seckey=Im7p%2FanwNmQx078ijew%2Fsx5QUzMQU8M7SbrqWr3JGEc%3D%2C0-Ulo82T3hd3wuKMf_tTM6DbOSuS4Zlg9ZEtPAqKkNKXysGCBLK7zryshf7oP6p8_Hynb5vDXmd6C0TtcF9impZZK2kzkn1_SWeBVEJH4Zdh0n81wXv-1j6enHaAwiYj7T7eCaCw3g5vDcHvZxUxtBsRphpZI-Rco19dgpXwT9xb-XVcCb8GyvbMt5s2wIpb"; + def content = new HttpRequest().get(url, null, "utf-8") + JSONObject json = JSONObject.fromObject(content) + if (json && json.content && json.content.size() > 0) { + for (def loc : json.content) { +// String name = formatName(address) + def x = loc.x + def y = loc.y + if(x){ + x = x/100 + } + if(y){ + y = y/100 + } + if(x && y){ + def locJson = BaiDuApiUtils.geoconv(x, y, 6) + if(locJson){ + def baiduCity = RegionData.parseCityInAddress(loc.area_name) + def bean = [:] + def lng2 = locJson.get("x") + def lat2 = locJson.get("y") + def gmap = GeocodeUtils.geoconvBaiduToGoogle(lng2, lat2) + def lng = gmap.get("lng") + def lat = gmap.get("lat") + bean.uid = loc.uid + bean.baidu_lng2 = lng2 + bean.baidu_lat2 = lat2 + bean.longitude = lng + bean.latitude = lat + bean.name = loc.name + bean.address = getAddressInfo(loc.address_norm, loc.addr); + bean.old_addr = loc.addr + +// def baiduCity = RegionData.formatCity(loc.city_name); + bean.cityname = baiduCity + def regionStr = loc.area_name.replace(loc.city_name, "") + def baiduRegion = RegionData.formatRegion(baiduCity, regionStr); + bean.region = baiduRegion + if(loc.std_tag){ + bean.type = loc.std_tag + } + bean.flag = loc.acc_flag + rets << bean; + } + } + } + } + } catch(Exception e) { + } + return rets; + } + + def static formatName(String name) { + if(name.indexOf("¡¤") != -1) { + name = name.replaceAll("¡¤", "") + } + + if(name.indexOf(".") != -1) { + name = name.replace(".", "") + } + if(name.indexOf("(") != -1) { + name = name.replace("(", "") + } + if(name.indexOf(")") != -1) { + name = name.replace(")", "") + } + if(name.indexOf("-")!=-1){ + name = name.replace("-","") + } + return name.replaceAll("\\s+", "").toLowerCase() + } + + + static def getAddressInfo(def s, def addr){ + def arrs = s.split("]") + def map = [:] + if(arrs && arrs.size()>=3){ + if(arrs[0] && arrs[0].contains("PROV")){ + map.prov = arrs[0].substring(1, arrs[0].indexOf("(")) + } + if(arrs[1] && arrs[1].contains("CITY")){ + map.city = arrs[1].substring(1, arrs[1].indexOf("(")) + } + if(arrs[2] && arrs[2].contains("AREA")){ + map.region = arrs[2].substring(1, arrs[2].indexOf("(")) + } + } + if(addr){ + addr = addr.replace("[]","") + } + if(addr && map){ + if(map.region && !addr.contains(map.region)){ + addr = map.region + addr + } + if(map.city && !addr.contains(map.city)){ + addr = map.city + addr + } + + if(map.prov && !addr.contains(map.prov)){ + addr = map.prov + addr + } + } + return addr; + } + + + + def searchMapBySimilar(def city, def region, def name, def inpAddr, def rets, newKey){ + def bo = null; + if(rets){ + def maxScore = 0; + if(!bo || !maxScore){ + for(def o :rets){ + + if(!region.contains(o.region)){ + continue; + } + def s = o.name +// if(o.name && o.name.contains("(") && !o.name.toString().startsWith("(")){ +// s = o.name.substring(0, o.name.indexOf("(")) +// } +// s = formatName(s) + def addr = o.address + def mapKey = filterName(null, o.cityname, o.region, o.name) + + if(mapKey){ + def score = CosineSimilarAlgorithm.getSimilarityRatio(mapKey, newKey); + if(city==o.cityname){ + if(score > maxScore){ + maxScore = score; + o.matchType = "³ÇÊÐÏàͬ£¬Ãû³ÆÏàËÆ" + o.city = o.cityname + bo = o + bo.score = maxScore + } + if(bo && score!=1){ + if(inpAddr){ + def baidu_addr = getSimpleAddress(o.cityname, o.old_addr) + if(baidu_addr){ + def addr_score = CosineSimilarAlgorithm.getSimilarityRatio(inpAddr, baidu_addr); + if(addr_score==1){ + o.matchType = "³ÇÊÐÏàͬ£¬Ãû³ÆÏàËÆ£¬µØÖ·ÓйØ" + bo.score = bo.score + 0.1 + } + } + } + +// for(def word : filters){ +// if(mapKeyName.toString().endsWith(word)){ +// mapKeyName = mapKeyName.replace(word, "") +// } +// if(newName.toString().endsWith(word)){ +// newName = newName.replace(word, ""); +// } +// } +// +// +// if(mapKeyName && newName && mapKeyName.length()>=2){ +// score = CosineSimilarAlgorithm.getSimilarityRatio(mapKeyName, newName); +// if(score > maxScore){ +// maxScore = score +// o.matchType = "Ãû³ÆÏàËÆ" +// o.city = obj.city +// bo = o +// bo.score = maxScore +// }else{ +// if(mapKeyName.length() == newName.length()){ +// def mapKeyNameBY = com.fangjia.core.util.DataUtil.hanyuQuanpin(mapKeyName, " ") +// def newNameBY = com.fangjia.core.util.DataUtil.hanyuQuanpin(newName, " ") +// if(mapKeyNameBY == newNameBY){ +// maxScore = 0.977; +// o.matchType = "Æ´ÒôÏàËÆ" +// o.city = obj.city +// bo = o +// bo.score = maxScore +// } +// } +// } +// } + } + } + } + } + } + } + return bo; + } + + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDistrictMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDistrictMatch.groovy index f990711be..7a1d976ee 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDistrictMatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDistrictMatch.groovy @@ -2,51 +2,172 @@ package com.fangjia.script.gj.crawl import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.temp.shenjian.util.SeaweedUtils +import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.OpenApiClient +import com.mongodb.BasicDBObject +import org.bson.types.ObjectId class AnjukeDistrictMatch { static main(args) { def task = new AnjukeDistrictMatch(); - task.run(); +// task.run(); // task.run1() +// task.updateField() + task.fenQiRun() } - def run1(){ - def regionPointsMap = findRegionRange("ÖØÇì") - def flag = isInArea(dist.lng/10000000000.0, dist.lat/10000000000.0, regionPointsMap.get(dist.region)) + def fenQiRun(){ + def citys = ['ËÕÖÝ','º¼ÖÝ','Äþ²¨','ÎÂÖÝ','¼ÎÐË','ºþÖÝ','ÉÜÐË','½ð»ª','áéÖÝ','ÖÛɽ','̨ÖÝ','ÀöË®'] + for(def c : citys){ + def dists = CommonDatautils.findAllDistrictByCity(c, "district", ["city","region","name","virescence_percent","build_area","all_area","cubage_percent","master_district_name"]) + def distMap = findFiqiDatas(dists) + println c +" "+dists.size() +" "+ distMap.size() + addFieldValue(distMap); + } } + + def addFieldValue(distMap){ + def index = 0 + distMap.each {k,v-> + if(v.master_district_name!="master"){ + def masterKey = v.city +"##"+v.region+"##"+v.master_district_name; +// def masterObj = distMap.get(masterKey) + def masterObj = CommonDatautils.findDistrict(v.city, v.region, v.master_district_name, "district"); + if(masterObj){ + def udata = new BasicDBObject() + if(!v.virescence_percent && masterObj.virescence_percent){ + udata.put("virescence_percent", masterObj.virescence_percent) + } + + if(!v.build_area && masterObj.build_area){ + udata.put("build_area", masterObj.build_area) + } + + if(!v.all_area && masterObj.all_area){ + udata.put("all_area", masterObj.all_area) + } + + if(!v.cubage_percent && masterObj.cubage_percent){ + udata.put("cubage_percent", masterObj.cubage_percent) + } + + if(udata){ + index ++ + udata.put("m_date",new Date()) + def q = new BasicDBObject() + q.put("_id", new ObjectId(v._id.toString())) + DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) + } + + } + } + } + println index; + } + + def findFiqiDatas(dists){ + def distMap = [:] + for(def d : dists){ + def master_district_name = d.master_district_name + if(master_district_name && master_district_name!="null"){ + distMap.put(d.city+"##"+d.region+"##"+d.name, d); + } + } + return distMap; + } + + + def updateField(){ + def filePath = "D:\\Êý¾Ý\\½¨ÐÐ\\¶þÆÚ\\µÚÈý·½Ð¡Çø×ֶβ¹³ä\\ÂÌ»¯ÂÊ.xlsx" + ExcelUtils.multiThreadingModify(filePath, { row -> + if(row['Q'] && !row['Q'].contains("½Ó¿Ú")){ + def dist = CommonDatautils.findSeaweedById(row['R'], ["virescence_percent"]) + if(dist && !dist.virescence_percent){ + def val = row['K'] + if(!val){ + val = row['J'] + } + if(!val){ + val = row['M'] + } + if(!val){ + val = row['O'] + } + if(!val){ + val = row['G'] + } + if(!val){ + val = row['E'] + } + if(!val){ + val = row['F'] + } + if(!val){ + val = row['H'] + } + if(!val){ + val = row['I'] + } + if(!val){ + val = row['L'] + } + if(!val){ + val = row['N'] + } + if(!val){ + val = row['O'] + } + + if(val){ + if(val.toString().toDouble().toInteger()==val.toString().toDouble()){ + val = val.toString().toDouble().toInteger() + } + def udata = new BasicDBObject() + udata.put("virescence_percent", val.toString()) + udata.put("m_date",new Date()) + + def q = new BasicDBObject() + q.put("_id", new ObjectId(dist._id.toString())) + DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) + row['U'] = 1 + } + } + } + }) + } + +// def run1(){ +// def regionPointsMap = findRegionRange("ÖØÇì") +// def flag = isInArea(dist.lng/10000000000.0, dist.lat/10000000000.0, regionPointsMap.get(dist.region)) +// } def run(){ - - - def filePath = "D:\\Êý¾Ý\\ÖØÇìÅ©ÉÌÐÐ\\°²¾Ó¿Í\\ÖØÇìÐ¡Çø.xlsx" + def filePath = "D:\\Êý¾Ý\\½¨ÐÐ\\¶þÆÚ\\µÚÈý·½Ð¡Çø×ֶβ¹³ä\\ÂÌ»¯ÂÊ.xlsx" ExcelUtils.multiThreadingModify(filePath,{row-> - if(row['A'] && !row['J']){ + if(row['A'] && !row['Q']){ def city = row['B'] def region = row['C'] def name = row['D'] - def address = row['E'] -// def flag = "±ðÃûÆ¥Åä" - -// def dist = matchDist(city, null, name) -// if(!row['J'] && name.contains("(")){ -// name = name.replace("(${region})","").replace("(","").replace(")","") -// flag = "È¥À¨ºÅÆ¥Åä" -// dist = matchDist(city, region, name) -// } -// if(!dist){ -// dist = CommonDatautils.findSeaweedByMappingAlias(city, region, name,"district") -// flag = "Ó³Éä±ðÃûÆ¥Åä" -// } -// if(!dist) { -// def rets = CommonDatautils.findDistrictByAlias(city, region, name, "district") -// if(rets && rets.size()==1){ -// dist = rets[0] -// } -// } + def flag = "Ãû³ÆÆ¥Åä" + def dist = matchDist(city, region, name) + if(!row['D'] && name.contains("(")){ + name = name.replace("(${region})","").replace("(","").replace(")","") + flag = "È¥À¨ºÅÆ¥Åä" + dist = matchDist(city, region, name) + } + if(!dist){ + dist = CommonDatautils.findSeaweedByMappingAlias(city, region, name,"district") + flag = "Ó³Éä±ðÃûÆ¥Åä" + } + if(!dist) { + def rets = CommonDatautils.findDistrictByAlias(city, region, name, "district") + if(rets && rets.size()==1){ + dist = rets[0] + } + } // // if(!dist){ @@ -61,23 +182,21 @@ class AnjukeDistrictMatch { // } // } -// if(dist){ -// row['J'] = flag -// row['K'] = dist._id.toString() -// row['L'] = dist.region -// row['M'] = dist.name -// row['N'] = dist.address -// } - - def jsonObject = OpenApiClient.addressMatch(city, address, "property", false,region,name) - if(jsonObject != null && jsonObject.precise!=-1) { -// row['J'] = "Æ¥Åä½Ó¿Ú" -// row['K'] = jsonObject.get("id") -// row['L'] = jsonObject.get("district") -// row['M'] = jsonObject.get("name") -// row['N'] = jsonObject.get("address") - row['M'] = jsonObject.get("name") + if(dist){ + row['Q'] = flag + row['R'] = dist._id.toString() + row['S'] = dist.region + row['T'] = dist.name + }else{ + def jsonObject = OpenApiClient.addressMatch(city, name, "property", false,region, null) + if(jsonObject != null && jsonObject.precise!=-1) { + row['Q'] = "Æ¥Åä½Ó¿Ú" + row['R'] = jsonObject.get("id") + row['S'] = jsonObject.get("district") + row['T'] = jsonObject.get("name") + } } + } }) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy index ce6242a31..a01d0b3eb 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy @@ -240,7 +240,7 @@ public class BeiKeCrawlByChrome { def price = e.select(".totalPrice .number").text() if (price && price.isNumber()) { - detail.total_price = price.toDouble() * 10000 + detail.total_price = (int)(price.toDouble() * 10000) } if (detail.total_price && detail.area) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy index 8e74425cb..bb9afd6f8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -316,7 +316,7 @@ public class LianjiaCrawlByChrome { def price = e.select(".totalPrice .number").text() if (price && price.isNumber()) { - detail.total_price = price.toDouble() * 10000 + detail.total_price = (int)(price.toDouble() * 10000) } if (detail.total_price && detail.area) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/Test.groovy index e1976f1cb..ca7c9e388 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/Test.groovy @@ -3,12 +3,15 @@ package com.fangjia.script.gj.test import ch.qos.logback.core.db.dialect.DBUtil import com.fangjia.core.RegionData import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.gj.district.SeaweedDataMatchProcess import com.fangjia.script.lixue.util.DBUtils import com.fangjia.script.util.CommonUtils import com.fangjia.script.util.DateUtils import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils +import com.fangjia.script.util.OpenApiClient import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.mongodb.Mongo @@ -24,6 +27,75 @@ DBCollection coll = MongoUtils.getCollectionByName("seeds_db", "city_seeds", "1 def ZS_MONGODB_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; DBCollection TOWNSHIP_CONN = MongoUtils.getCollectionByName2("fangjia", "township", ZS_MONGODB_URL) + +DBCollection FANGJIA_AREA_CODE = MongoUtils.getCollectionByName("business", "fangjia_area_code", "mongo.db.fangjia.com") +DBCollection GUIDING_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "guiding_price", "mongo.db.fangjia.com") +//ExcelUtils.modify("D:\\Êý¾Ý\\֤ȯ\\¹ú¾ý¼û֤ʧ°ÜÔ­Òò.xls", { d -> +//}) + + +ExcelUtils.modify("D:\\Êý¾Ý\\֤ȯ\\¹ú¾ý¼û֤ʧ°ÜÔ­Òò(±ê×¼»¯Ê¡ÊÐÇø).xls", { d -> + def info = d['A'] + if(info && !d['E']){ + def arrs = info.split("!~") + if(arrs && arrs[2] && arrs[3]){ +// def rMap = SeaweedDataMatchProcess.doMatch(d['B'], d['C'], arrs[4], arrs[4]); +// if(rMap) { +// d['E'] = rMap.name +// d['F'] = rMap.id +// } + + def address = null + if(d['G']){ + address = d['G'] + }else{ + address = arrs[4] + } + + def jsonObject = OpenApiClient.addressMatch(d['C'], address, "property", false, d['D'], null) + if(jsonObject != null && jsonObject.precise!=-1) { + d['H'] = jsonObject.get("city") + d['I'] = jsonObject.get("district") + d['J'] = jsonObject.get("name") + d['K'] = jsonObject.get("id") + } + +// def jsonObj = FangjiaClientUtils.addressStructure(d['B'], arrs[4]) +// if(jsonObj && jsonObj.result) { +// def street = null +// def street_number = null +// if (jsonObj.result.street) { +// d['E'] = jsonObj.result.street +// } +// if (jsonObj.result.streetNumber) { +// d['F'] = jsonObj.result.streetNumber +// } +// d['G'] = jsonObj.result.name +// } + +// def rets = findProinvceAndCityAndRegion(FANGJIA_AREA_CODE, arrs[1], arrs[3]) +// if(rets && rets.size()==1){ +// def obj = rets[0] +// d['B'] = obj.province +// d['C'] = obj.fjw_city +// d['D'] = obj.fjw_district +// } + } + } +}); +// +//def findProinvceAndCityAndRegion(FANGJIA_AREA_CODE, cityCode, regionCode){ +// def q = new BasicDBObject() +// q.put("city_code", cityCode.toString().toInteger()) +// q.put("district_code", regionCode.toString().toInteger()) +// def dbcursor = FANGJIA_AREA_CODE.find(q) +// def rets = [] +// for(def d : dbcursor){ +// rets << d +// } +// return rets; +//} + //ExcelUtils.modify("D:\\Êý¾Ý\\½¨ÐÐ\\¶þÆÚ\\¹ú¼ÒÐÐÕþÇø»®ÌØÊâ³ÇÊе¼³ö.xlsx",{d-> // def city = RegionData.formatCity(d['E']) // if(city && !d['J']){ @@ -103,33 +175,33 @@ DBCollection TOWNSHIP_CONN = MongoUtils.getCollectionByName2("fangjia", "townshi // } //}) -ExcelUtils.modify("D:\\Êý¾Ý\\Ç×ÁÚ\\·¿¼ÛÍøÊ¡ÊÐÇø.xlsx",{d-> - if(!d['D']){ - def infos = findTownshipInfo(d['B'], d['C'], TOWNSHIP_CONN) - if(infos && infos.size()==1){ - def info = infos[0] - d['D'] = info.province_code - d['E'] = info.area_code - d['F'] = info.street_code - } - } -}) - - -def findTownshipInfo(city, region, coll){ - def q = new BasicDBObject() - q.put("city_name", Pattern.compile(city, Pattern.CASE_INSENSITIVE)) - q.put("street_name", Pattern.compile('^.*'+region+'½ÖµÀ.*$', Pattern.CASE_INSENSITIVE)) - - def dbcursor = coll.find(q) - def rows = [] - for(def d : dbcursor){ -// if(d.street_name.contains(region)){ - rows << d +//ExcelUtils.modify("D:\\Êý¾Ý\\Ç×ÁÚ\\·¿¼ÛÍøÊ¡ÊÐÇø.xlsx",{d-> +// if(!d['D']){ +// def infos = findTownshipInfo(d['B'], d['C'], TOWNSHIP_CONN) +// if(infos && infos.size()==1){ +// def info = infos[0] +// d['D'] = info.province_code +// d['E'] = info.area_code +// d['F'] = info.street_code // } - } - return rows -} +// } +//}) +// +// +//def findTownshipInfo(city, region, coll){ +// def q = new BasicDBObject() +// q.put("city_name", Pattern.compile(city, Pattern.CASE_INSENSITIVE)) +// q.put("street_name", Pattern.compile('^.*'+region+'½ÖµÀ.*$', Pattern.CASE_INSENSITIVE)) +// +// def dbcursor = coll.find(q) +// def rows = [] +// for(def d : dbcursor){ +//// if(d.street_name.contains(region)){ +// rows << d +//// } +// } +// return rows +//} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy index 8bca701d7..04e45fb88 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy @@ -11,8 +11,8 @@ import org.bson.Document import static com.mongodb.client.model.Filters.* MongoCollection OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log") -def startDate = DateUtil.parseDate("20220601", "yyyyMMdd") -def endDate = DateUtil.parseDate("20220701", "yyyyMMdd") +def startDate = DateUtil.parseDate("20220901", "yyyyMMdd") +def endDate = DateUtil.parseDate("20221001", "yyyyMMdd") Calendar calendar = Calendar.getInstance() calendar.setTime(startDate) def count = 0 @@ -25,7 +25,7 @@ while (startDate.before(endDate)) { OPEN_URI_LOG.find(and(eq("user_id", "1007705"), eq("request_uri", "/property/detail"), eq("data_status", 1), lt("c_date", nextDate), gte("c_date", startDate))).each { Document doc -> def result = JSONObject.fromObject(doc.res_map.result) if (!hashSet.contains(result)) { - def item = ["60a3202c0b42a00001aad385", "qlkjgx", doc.ip, doc.request_uri, doc.c_date.format("yyyy-MM-dd HH:mm:ss"), doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()] + def item = ["60a3202c0b42a00001aad385", "qlkj", doc.ip, doc.request_uri, doc.c_date.format("yyyy-MM-dd HH:mm:ss"), doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()] data << item hashSet << result } else { @@ -38,4 +38,6 @@ while (startDate.before(endDate)) { startDate = nextDate } -ExcelUtils.write("D:\\Ç×ÁÚÊý¾Ý${startDate.format('yyyyMM')}.xlsx", data, "²éѯÈÕÖ¾") \ No newline at end of file +ExcelUtils.write("D:\\Ç×ÁÚÊý¾Ý${startDate.format('yyyyMM')}.xlsx", data, "²éѯÈÕÖ¾") + + From 084130a68b403ae8da303b964221870e35cb75a5 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 6 Dec 2022 16:50:15 +0800 Subject: [PATCH 11/99] =?UTF-8?q?=E6=8D=A2=E5=BA=93?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ronghe/BigdataCommunityGJRHBatch.groovy | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy index 770ea6914..d2ad41623 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy @@ -31,8 +31,8 @@ class BigdataCommunityGJRHBatch extends BaseBatch { public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80) - - def sqlInstance = null + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") public static def batchsize = 1000 static main(args) { @@ -42,13 +42,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch { @Override public void process(Object args) { def citys = CcbCommonUtils.getRunningCitys(args) - if(citys.size()==92){ - sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", - "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") - }else{ - sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl", - "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") - } + def rows = [] def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() // for (def city : citys) { From 42676bf294faeca93c7bfa0fe1f44bd0256a9fb5 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 7 Dec 2022 11:42:17 +0800 Subject: [PATCH 12/99] =?UTF-8?q?=E8=9E=8D=E5=90=88=E6=97=B6=E9=97=B4?= =?UTF-8?q?=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/script/ccbevaluation/Test.groovy | 84 +++++++++++++++---- .../ronghe/ExportDistrictRongHeTest.groovy | 54 ++++++------ 2 files changed, 96 insertions(+), 42 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index d36f08cc1..947d57cb8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -45,6 +45,56 @@ DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_craw def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017"; def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business", "res_esfdealprice", offlineUrl) + + +def citys = ['ËÕÖÝ','º¼ÖÝ','Äþ²¨','ÎÂÖÝ','¼ÎÐË','ºþÖÝ','ÉÜÐË','½ð»ª','áéÖÝ','ÖÛɽ','̨ÖÝ','ÀöË®'] + +def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() +def rets = [] +for(def c : citys){ + def citycode = cityCodeDataMap.get(c) + def datas = findAllData(sqlInstance236, citycode) + def totalCount = datas.size() + def landareaCount = 0; + def buildareaCount = 0 + def greeningrateCount = 0 + def plotratioCount = 0 + for(def m : datas){ + if(m.landarea){ + landareaCount ++ + } + if(m.buildarea){ + buildareaCount++ + } + if(m.greeningrate){ + greeningrateCount++ + } + if(m.plotratio){ + plotratioCount++ + } + } + + def landAreaPre = landareaCount*100/totalCount + def buildAreaPre = buildareaCount*100/totalCount + def greeningratePre = greeningrateCount*100/totalCount + def plotratioPre = plotratioCount*100/totalCount + rets << [c, totalCount, landAreaPre, buildAreaPre, greeningratePre, plotratioPre] +} + +ExcelUtils.write("D:\\Êý¾Ý\\½¨ÐÐ\\²âÊÔ\\12³Ç¸²¸ÇÂÊͳ¼Æ.xlsx",rets, "¸²¸ÇÂÊ") + +def findAllData(sqlInstance236, citycode){ + def sql = "select landarea,buildarea,greeningrate,plotratio from bigdata_community_fjw where cityid='${citycode}'" + def rets = [] + sqlInstance236.eachRow(sql,{r-> + rets << ["landarea":r.landarea, "buildarea":r.buildarea, "greeningrate":r.greeningrate, "plotratio":r.plotratio] + }); + return rets + +} + + + ////def url = "https://sh.anjuke.com" ////def s = DataHandleUtils.findMatchPattern(name, "https://\\w+.anjuke.com") ////println s @@ -447,23 +497,23 @@ def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business ////},0) // -ExcelUtils.multiThreadingModify("D:\\Êý¾Ý\\½¨ÐÐ\\¶þÆÚ\\252¸ö³ÇÊÐɾ³ýÐ¡Çø.xlsx",{d-> - if(d['A']){ - def dbo = CommonDatautils.findDistrictById(d['D']) - if(dbo){ - def udata = new BasicDBObject() -// udata.put("name_cat", 11) - udata.put("status", 1) - udata.put("m_date",new Date()) - - def q = new BasicDBObject() - q.put("_id", dbo._id) - DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) - }else{ - println dbo._id.toString() +" ûÕÒµ½" - } - } -},0) +//ExcelUtils.multiThreadingModify("D:\\Êý¾Ý\\½¨ÐÐ\\¶þÆÚ\\252¸ö³ÇÊÐɾ³ýÐ¡Çø.xlsx",{d-> +// if(d['A']){ +// def dbo = CommonDatautils.findDistrictById(d['D']) +// if(dbo){ +// def udata = new BasicDBObject() +//// udata.put("name_cat", 11) +// udata.put("status", 1) +// udata.put("m_date",new Date()) +// +// def q = new BasicDBObject() +// q.put("_id", dbo._id) +// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) +// }else{ +// println dbo._id.toString() +" ûÕÒµ½" +// } +// } +//},0) //def citys = ["º£¿Ú","ÈýÑÇ","Èýɳ","Îåָɽ","Çíº£","ÙÙÖÝ","ÎIJý","ÍòÄþ","¶«·½","¶¨°²","ÍͲý","³ÎÂõ","ÁÙ¸ß","°×ɳ","²ý½­","ÀÖ¶«","ÁêË®","±£Í¤","ÇíÖÐ"] //for(def c : citys){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy index 312032ca4..e91198746 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy @@ -26,35 +26,37 @@ class ExportDistrictRongHeTest extends BaseBatch{ @Override void process(Object args) { citys = CcbCommonUtils.getRunningCitys(args) - def curDate = "20221110"//DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd") + def curDate = "20221210"//DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd") def new_dists = []//["350604","350605","410307","410308","450181"] //°´Ììµ¼£¨Ã¿ÌìÒ»¸öÎļþ¼Ð£© // exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-1), "yyyy-MM-dd"), new_dists) // if(curDate.toString().endsWith("10")){ -// exportBigdataBuildprojFjw() -// exportBigdataCommlayoutpicFjw() -// exportEvaluateRate() -// -// exportBigdataCommunityGJRH()// -// exportBigdataBuildingFJW()// -// exportBigdataCommfeeGj()// -// -// exportBigdataCommbuildstruGJ()// -// exportBigdataCommhousetypeGJ()// -// exportBigdataCommbuildyearGj()// -// exportBigdataCommhouseuseGJ()// -// exportBigdataPositionGJ()// -// exportBigdataDistrictFJW()// -// exportBigdataAliasFJW()// + exportBigdataBuildprojFjw() + exportBigdataCommlayoutpicFjw() + exportEvaluateRate() -// def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger() -// exportBigdataCityPrice(queryDate) -// exportBigdataDistPrice(queryDate) -// exportBigdataCommPrice(queryDate) + exportBigdataCommunityGJRH()// + exportBigdataBuildingFJW()// + exportBigdataCommfeeGj()// - exportInOk() + exportBigdataCommbuildstruGJ()// + exportBigdataCommhousetypeGJ()// + exportBigdataCommbuildyearGj()// + exportBigdataCommhouseuseGJ()// + exportBigdataPositionGJ()// + exportBigdataDistrictFJW()// + exportBigdataAliasFJW()// + + + //±¾12.10ÉÏ´«10£¬11ÔÂËùÓÐÊý¾Ý£¬23Äê1ÔÂÐè¸ÄΪËùÓгÇÊа´ÈÕÆÚͬ²½Ò»ÆÚÊý¾Ý + def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger() + exportBigdataCityPrice(queryDate) + exportBigdataDistPrice(queryDate) + exportBigdataCommPrice(queryDate) + + exportInOk() // } @@ -469,7 +471,7 @@ class ExportDistrictRongHeTest extends BaseBatch{ stop = true } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); - CcbFtpTestUtils.ftpUpload("rh", fileName, curDate) + CcbFtpTestUtils.ftpUpload("ccb_ronghe", fileName, curDate) println "BIGDATA_DEAL_FJW ÉÏ´«FtpÍê³É......${curStartStr}" def curEndStr = DateUtils.add(curStartStr, 1) curStartDateStr = curEndStr @@ -751,7 +753,9 @@ class ExportDistrictRongHeTest extends BaseBatch{ def allRows = [] for(def city : citys) { def cityCode = cityCodeMap.get(city) - def sql = "select CITYID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_CITYPRICE where CITYID='${cityCode}' and YMID=${queryDate}";// + // def sql = "select CITYID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_CITYPRICE where CITYID='${cityCode}' and YMID=${queryDate}";// + def sql = "select CITYID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_CITYPRICE where CITYID='${cityCode}'";// + sqlInstance.eachRow(sql,{r-> def data = [] data << r.CITYID @@ -797,7 +801,7 @@ class ExportDistrictRongHeTest extends BaseBatch{ def allRows = [] for(def city : citys) { def cityCode = cityCodeMap.get(city) - def sql = "select CITYID,DISTID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_DISTPRICE where CITYID='${cityCode}' and YMID=${queryDate} ";// + def sql = "select CITYID,DISTID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_DISTPRICE where CITYID='${cityCode}'";// and YMID=${queryDate} sqlInstance.eachRow(sql,{r-> def data = [] data << r.CITYID @@ -844,7 +848,7 @@ class ExportDistrictRongHeTest extends BaseBatch{ def allRows = [] for(def city : citys) { def cityCode = cityCodeMap.get(city) - def sql = "select CITYID, YMID,COMMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_COMMPRICE where CITYID='${cityCode}' and YMID=${queryDate}";// + def sql = "select CITYID, YMID,COMMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_COMMPRICE where CITYID='${cityCode}' ";// and YMID=${queryDate} sqlInstance.eachRow(sql,{r-> def data = [] data << r.CITYID From a7b4bb1938840a63be6778adb70a7f2350242275 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 7 Dec 2022 13:20:36 +0800 Subject: [PATCH 13/99] update --- .../com/fangjia/script/ccbevaluation/CcbFtpTestUtils.groovy | 2 +- .../ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpTestUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpTestUtils.groovy index 4bac05c3f..da0ac8e8e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpTestUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpTestUtils.groovy @@ -7,7 +7,7 @@ import org.apache.commons.codec.digest.DigestUtils class CcbFtpTestUtils { - static def ftpServer = "192.168.0.86" + static def ftpServer = "192.168.0.45" static def ftpPort = 17495 static def ftpUser = "ccb" static def ftpPassword = "ccb!@#" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy index e91198746..983495eb0 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy @@ -33,9 +33,10 @@ class ExportDistrictRongHeTest extends BaseBatch{ // if(curDate.toString().endsWith("10")){ + exportEvaluateRate() exportBigdataBuildprojFjw() exportBigdataCommlayoutpicFjw() - exportEvaluateRate() + exportBigdataCommunityGJRH()// exportBigdataBuildingFJW()// From 91454464410cd0dd143ee7cb31e3d1d9a208ae3f Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 7 Dec 2022 14:31:42 +0800 Subject: [PATCH 14/99] =?UTF-8?q?=E6=B5=8B=E8=AF=95=E6=9C=8D=E5=8A=A1?= =?UTF-8?q?=E5=99=A8=E6=8D=A2=E7=BA=BF=E4=B8=8A45?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ronghe/ExportDistrictRongHeTest.groovy | 46 +++++++++---------- .../ccbevaluation/ronghe/SSHHelperTest.groovy | 2 +- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy index 983495eb0..ed6d771e7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy @@ -33,29 +33,29 @@ class ExportDistrictRongHeTest extends BaseBatch{ // if(curDate.toString().endsWith("10")){ - exportEvaluateRate() - exportBigdataBuildprojFjw() - exportBigdataCommlayoutpicFjw() - - - exportBigdataCommunityGJRH()// - exportBigdataBuildingFJW()// - exportBigdataCommfeeGj()// - - exportBigdataCommbuildstruGJ()// - exportBigdataCommhousetypeGJ()// - exportBigdataCommbuildyearGj()// - exportBigdataCommhouseuseGJ()// - exportBigdataPositionGJ()// - exportBigdataDistrictFJW()// - exportBigdataAliasFJW()// - - - //±¾12.10ÉÏ´«10£¬11ÔÂËùÓÐÊý¾Ý£¬23Äê1ÔÂÐè¸ÄΪËùÓгÇÊа´ÈÕÆÚͬ²½Ò»ÆÚÊý¾Ý - def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger() - exportBigdataCityPrice(queryDate) - exportBigdataDistPrice(queryDate) - exportBigdataCommPrice(queryDate) +// exportEvaluateRate() +// exportBigdataBuildprojFjw() +// exportBigdataCommlayoutpicFjw() +// +// +// exportBigdataCommunityGJRH()// +// exportBigdataBuildingFJW()// +// exportBigdataCommfeeGj()// +// +// exportBigdataCommbuildstruGJ()// +// exportBigdataCommhousetypeGJ()// +// exportBigdataCommbuildyearGj()// +// exportBigdataCommhouseuseGJ()// +// exportBigdataPositionGJ()// +// exportBigdataDistrictFJW()// +// exportBigdataAliasFJW()// +// +// +// //±¾12.10ÉÏ´«10£¬11ÔÂËùÓÐÊý¾Ý£¬23Äê1ÔÂÐè¸ÄΪËùÓгÇÊа´ÈÕÆÚͬ²½Ò»ÆÚÊý¾Ý +// def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger() +// exportBigdataCityPrice(queryDate) +// exportBigdataDistPrice(queryDate) +// exportBigdataCommPrice(queryDate) exportInOk() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/SSHHelperTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/SSHHelperTest.groovy index 0e94c9551..07d453ef7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/SSHHelperTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/SSHHelperTest.groovy @@ -16,7 +16,7 @@ class SSHHelperTest { private Session session = null; private Channel channel = null; - private String sftpHost = "192.168.0.86"; + private String sftpHost = "192.168.0.45"; private int sftpPort = 17495; private String sftpUserName = "ccb"; private String sftpPassword = "ccb!@#"; From f2e3f1fc09b2521a23bf4ae6493e871befc6b514 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 7 Dec 2022 14:39:57 +0800 Subject: [PATCH 15/99] update --- .../ronghe/ExportDistrictRongHeTest.groovy | 58 +++++++++---------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy index ed6d771e7..f6c44ace0 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy @@ -33,29 +33,29 @@ class ExportDistrictRongHeTest extends BaseBatch{ // if(curDate.toString().endsWith("10")){ -// exportEvaluateRate() -// exportBigdataBuildprojFjw() -// exportBigdataCommlayoutpicFjw() -// -// -// exportBigdataCommunityGJRH()// -// exportBigdataBuildingFJW()// -// exportBigdataCommfeeGj()// -// -// exportBigdataCommbuildstruGJ()// -// exportBigdataCommhousetypeGJ()// -// exportBigdataCommbuildyearGj()// -// exportBigdataCommhouseuseGJ()// -// exportBigdataPositionGJ()// -// exportBigdataDistrictFJW()// -// exportBigdataAliasFJW()// -// -// -// //±¾12.10ÉÏ´«10£¬11ÔÂËùÓÐÊý¾Ý£¬23Äê1ÔÂÐè¸ÄΪËùÓгÇÊа´ÈÕÆÚͬ²½Ò»ÆÚÊý¾Ý -// def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger() -// exportBigdataCityPrice(queryDate) -// exportBigdataDistPrice(queryDate) -// exportBigdataCommPrice(queryDate) + exportEvaluateRate() + exportBigdataBuildprojFjw() + exportBigdataCommlayoutpicFjw() + + + exportBigdataCommunityGJRH()// + exportBigdataBuildingFJW()// + exportBigdataCommfeeGj()// + + exportBigdataCommbuildstruGJ()// + exportBigdataCommhousetypeGJ()// + exportBigdataCommbuildyearGj()// + exportBigdataCommhouseuseGJ()// + exportBigdataPositionGJ()// + exportBigdataDistrictFJW()// + exportBigdataAliasFJW()// + + + //±¾12.10ÉÏ´«10£¬11ÔÂËùÓÐÊý¾Ý£¬23Äê1ÔÂÐè¸ÄΪËùÓгÇÊа´ÈÕÆÚͬ²½Ò»ÆÚÊý¾Ý + def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger() + exportBigdataCityPrice(queryDate) + exportBigdataDistPrice(queryDate) + exportBigdataCommPrice(queryDate) exportInOk() @@ -747,9 +747,9 @@ class ExportDistrictRongHeTest extends BaseBatch{ def exportBigdataCityPrice(queryDate){ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_CITYPRICE_ADD_${curDate}_0001.dat" + def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_CITYPRICE_FJW_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_CITYPRICE_ADD_${curDate}_0001.dat" + fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_CITYPRICE_FJW_ADD_${curDate}_0001.dat" } def allRows = [] for(def city : citys) { @@ -795,9 +795,9 @@ class ExportDistrictRongHeTest extends BaseBatch{ def exportBigdataDistPrice(queryDate){ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_DISTPRICE_ADD_${curDate}_0001.dat" + def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_DISTPRICE_FJW_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_DISTPRICE_ADD_${curDate}_0001.dat" + fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_DISTPRICE_FJW_ADD_${curDate}_0001.dat" } def allRows = [] for(def city : citys) { @@ -842,9 +842,9 @@ class ExportDistrictRongHeTest extends BaseBatch{ def exportBigdataCommPrice(queryDate){ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_COMMPRICE_ADD_${curDate}_0001.dat" + def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_COMMPRICE_FJW_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_COMMPRICE_ADD_${curDate}_0001.dat" + fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_COMMPRICE_FJW_ADD_${curDate}_0001.dat" } def allRows = [] for(def city : citys) { From 914a152bd752c43f2013a88cfa960fdc7ebd8a9c Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 7 Dec 2022 14:55:44 +0800 Subject: [PATCH 16/99] update --- .../ronghe/ExportDistrictRongHeTest.groovy | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy index f6c44ace0..c274629b5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy @@ -29,7 +29,7 @@ class ExportDistrictRongHeTest extends BaseBatch{ def curDate = "20221210"//DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd") def new_dists = []//["350604","350605","410307","410308","450181"] //°´Ììµ¼£¨Ã¿ÌìÒ»¸öÎļþ¼Ð£© -// exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-1), "yyyy-MM-dd"), new_dists) + exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-1), "yyyy-MM-dd"), new_dists) // if(curDate.toString().endsWith("10")){ @@ -515,8 +515,13 @@ class ExportDistrictRongHeTest extends BaseBatch{ // } // } - def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ ";// - allRows.addAll(createDeal(sql)) + for(def city : citys) { + if(CcbCommonUtils.citys_252.contains(city)){ + def cityCode = cityCodeMap.get(city) + def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where CITYID='${cityCode}'"; + allRows.addAll(createDeal(sql)) + } + } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) From 3eee09c3e382766f12de1c733286b4fd46b0bb3a Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 8 Dec 2022 11:08:47 +0800 Subject: [PATCH 17/99] =?UTF-8?q?=E6=9B=B4=E6=96=B0=E5=88=AB=E5=90=8D?= =?UTF-8?q?=E8=A1=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ronghe/ExportDistrictRongHeTest.groovy | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy index c274629b5..cb23d1b7b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy @@ -29,33 +29,33 @@ class ExportDistrictRongHeTest extends BaseBatch{ def curDate = "20221210"//DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd") def new_dists = []//["350604","350605","410307","410308","450181"] //°´Ììµ¼£¨Ã¿ÌìÒ»¸öÎļþ¼Ð£© - exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-1), "yyyy-MM-dd"), new_dists) +// exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-1), "yyyy-MM-dd"), new_dists) // if(curDate.toString().endsWith("10")){ - exportEvaluateRate() - exportBigdataBuildprojFjw() - exportBigdataCommlayoutpicFjw() - - - exportBigdataCommunityGJRH()// - exportBigdataBuildingFJW()// - exportBigdataCommfeeGj()// - - exportBigdataCommbuildstruGJ()// - exportBigdataCommhousetypeGJ()// - exportBigdataCommbuildyearGj()// - exportBigdataCommhouseuseGJ()// - exportBigdataPositionGJ()// - exportBigdataDistrictFJW()// +// exportEvaluateRate() +// exportBigdataBuildprojFjw() +// exportBigdataCommlayoutpicFjw() +// +// +// exportBigdataCommunityGJRH()// +// exportBigdataBuildingFJW()// +// exportBigdataCommfeeGj()// +// +// exportBigdataCommbuildstruGJ()// +// exportBigdataCommhousetypeGJ()// +// exportBigdataCommbuildyearGj()// +// exportBigdataCommhouseuseGJ()// +// exportBigdataPositionGJ()// +// exportBigdataDistrictFJW()// exportBigdataAliasFJW()// //±¾12.10ÉÏ´«10£¬11ÔÂËùÓÐÊý¾Ý£¬23Äê1ÔÂÐè¸ÄΪËùÓгÇÊа´ÈÕÆÚͬ²½Ò»ÆÚÊý¾Ý - def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger() - exportBigdataCityPrice(queryDate) - exportBigdataDistPrice(queryDate) - exportBigdataCommPrice(queryDate) +// def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger() +// exportBigdataCityPrice(queryDate) +// exportBigdataDistPrice(queryDate) +// exportBigdataCommPrice(queryDate) exportInOk() From 974dfc1ce3243f9f78bcb65643ccc00ba0d98726 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 12 Dec 2022 10:43:28 +0800 Subject: [PATCH 18/99] =?UTF-8?q?=E5=90=8C=E6=AD=A5=E7=AC=AC=E4=B8=89?= =?UTF-8?q?=E6=96=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy index f49a13675..7cfd81e44 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy @@ -26,7 +26,7 @@ class SynchDistrictPrice extends BaseBatch{ // "¼ªÎÝ":"¼ªÎÝ","365ÌÔ·¿":"365ÌÔ·¿","ÎÒ°®ÎÒ¼Ò":"ÎÒ°®ÎÒ¼Ò","Öî¸ðÕÒ·¿APP":"Öî¸ðÕÒ·¿","±´¿Ç":"±´¿ÇÕÒ·¿","Á´¼Ò":"Á´¼Ò","Q·¿":"Q·¿","ÂóÌï":"ÂóÌï", // "ÖÐÔ­µØ²ú":"ÖÐÔ­µØ²ú","°²¾Ó¿ÍС³ÌÐò":"°²¾Ó¿Í","ËÑ·¿":"ËÑ·¿","ÀÖÓмÒ":"ÀÖÓмÒ","C21":"C21","³ÇÊз¿²ú":"³ÇÊз¿²ú","Íò©Ê¿PC¶Ë":"Íò©Ê¿" - def esSources = ["Á´¼Ò":"Á´¼Ò"] + def esSources = ["¼ªÎÝ":"¼ªÎÝ","365ÌÔ·¿":"365ÌÔ·¿","ÎÒ°®ÎÒ¼Ò":"ÎÒ°®ÎÒ¼Ò","Öî¸ðÕÒ·¿APP":"Öî¸ðÕÒ·¿","±´¿Ç":"±´¿ÇÕÒ·¿","Á´¼Ò":"Á´¼Ò","Q·¿":"Q·¿","ÂóÌï":"ÂóÌï","ÖÐÔ­µØ²ú":"ÖÐÔ­µØ²ú","°²¾Ó¿ÍС³ÌÐò":"°²¾Ó¿Í","ËÑ·¿":"ËÑ·¿","ÀÖÓмÒ":"ÀÖÓмÒ","C21":"C21","³ÇÊз¿²ú":"³ÇÊз¿²ú","Íò©Ê¿PC¶Ë":"Íò©Ê¿"] static main(args) { From ceb92cf60382505c839c0c0901637792dc75a1fb Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 12 Dec 2022 13:33:47 +0800 Subject: [PATCH 19/99] =?UTF-8?q?=E5=BB=BA=E8=A1=8C=E5=AF=BC=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E8=84=9A=E6=9C=AC=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/ccbevaluation/ExportOracleAllDataTest.groovy | 6 +++--- .../com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy | 3 ++- .../com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy | 2 +- .../fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy | 2 +- .../script/thirdPriceCrawl/TestCrawlTotalCount.groovy | 4 ++-- 5 files changed, 9 insertions(+), 8 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy index d35310b33..d198493d1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy @@ -46,7 +46,7 @@ class ExportOracleAllDataTest { // exportBigdataPoiData()// // exportBigdataDistrictData()// -// exportGuidedPrice()//ËæÈ˹¤¸Ä±ä// + exportGuidedPrice()//ËæÈ˹¤¸Ä±ä// // exportCommunityGuidedPrice(lastMonth.toString()+"28")//ËæÕþ²ß// // // //´¦ÀíÐ¡ÇøÇé¿ö @@ -54,8 +54,8 @@ class ExportOracleAllDataTest { // exportCommunityStaging()//ûcityid // exportCommunityBuileType()// -// exportThirdPartyPrice2(new_dists)// -// exportBigdataForsale(new_dists) //2022-07-19 + exportThirdPartyPrice2(new_dists)// + exportBigdataForsale(new_dists) //2022-07-19 // exportBigdataDeal( new_dists) //2022-07-01 // // //µÚÈý·½ÍøÕ¾È¨Öرí diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy index a11d0ce17..088626285 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy @@ -19,6 +19,7 @@ import java.util.concurrent.ExecutorService import java.util.concurrent.Executors import java.util.concurrent.atomic.AtomicLong +//Ð¡ÇøÔ¶Ⱦù¼Û±í£¨´øÕǵø·ù¿ØÖÆ£© @BatchOwner(Watcher.GaoJuan) class DistrictSellStatBatch extends BaseBatch { @@ -58,7 +59,7 @@ class DistrictSellStatBatch extends BaseBatch { } def calcMonthlyPrice(){ - def citys =["°°É½", '´óÇì', "ºâÑô", "ÄÏÑô", "ÆÎÌï"] // CommonUtils.findAllCity() + def citys = CommonUtils.findAllCity() def keys = new BasicDBObject("city", 1).append("name", 1).append("region", 1).append("estate_type2", 1) def lastMonthWeekend = DateUtils.truncateDay(DateUtils.getLastSundayInWeekOfLastMonth(new Date())) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy index a01d0b3eb..24f03c7cc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy @@ -183,7 +183,7 @@ public class BeiKeCrawlByChrome { def durl = e.select(".title a").attr("href") def dealDate = e.select(".dealDate").text() dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd") - def endDate = DateUtils.parseFromTightByPattern(startMonth + "-30", "yyyy-MM-dd") + def endDate = DateUtils.parseFromTightByPattern(startMonth + "-20", "yyyy-MM-dd") if(dealDate && dealDate.getTime()>=endDate.getTime()){ detailUrls << durl parse(city, e, region) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy index bb9afd6f8..0ecdddeb1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -251,7 +251,7 @@ public class LianjiaCrawlByChrome { def durl = e.select(".title a").attr("href") def dealDate = e.select(".dealDate").text() dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd") - def endDate = DateUtils.parseFromTightByPattern(startMonth + "-30", "yyyy-MM-dd") + def endDate = DateUtils.parseFromTightByPattern(startMonth + "-20", "yyyy-MM-dd") if(dealDate.getTime()>=endDate.getTime()){ detailUrls << durl parse(city, e, regionText) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy index 22eedff7f..56fbf3de1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy @@ -14,8 +14,8 @@ class TestCrawlTotalCount { def run(){ def sources = ["·¿¹À¹À","Öî¸ðÕÒ·¿","°²¾Ó¿Í","ËÑ·¿","ÖÐÔ­µØ²ú","¸Ï¼¯","Á´¼Ò","Íò©Ê¿","³ÇÊз¿²ú","±´¿ÇÕÒ·¿","Q·¿","Q·¿(ÐÂ)","ÖÐÔ­µØ²ú(ÐÂ)","ÀÖÓмÒ","ÀÖÓмÒ(ÐÂ)","·¿Ì½","·¿¶à¶à","С×ʼÒ","ÎÒ°®ÎÒ¼Ò","ÂóÌï","·¿Í¾","C21","ÀÖ¾Ó","·¿²ú³¬ÊÐ","58ͬ³Ç","365ÌÔ·¿","¼ªÎÝ"] for(def source :sources){ - def count1 = findTotalCount(source, 202209); - def count2 = findTotalCount(source, 202210); + def count1 = findTotalCount(source, 202210); + def count2 = findTotalCount(source, 202211); println source +" "+count1 +" "+count2 } } From 07af0707f811c8a19a2ddbb2f342a1633cca60de Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 12 Dec 2022 17:07:14 +0800 Subject: [PATCH 20/99] =?UTF-8?q?=E5=AF=BC=E4=BC=B0=E5=80=BC=E6=B5=8B?= =?UTF-8?q?=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ExportOracleAllDataTest.groovy | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy index d198493d1..7b1c88343 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy @@ -42,24 +42,24 @@ class ExportOracleAllDataTest { def lastMonthStr = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM") def new_dists = [] //²»»á±ä 10.25 -// exportEvaluationRateData() -// exportBigdataPoiData()// -// exportBigdataDistrictData()// + exportEvaluationRateData() + exportBigdataPoiData()// + exportBigdataDistrictData()// exportGuidedPrice()//ËæÈ˹¤¸Ä±ä// -// exportCommunityGuidedPrice(lastMonth.toString()+"28")//ËæÕþ²ß// -// -// //´¦ÀíÐ¡ÇøÇé¿ö -// exportBigdataCommunity()// -// exportCommunityStaging()//ûcityid -// exportCommunityBuileType()// + exportCommunityGuidedPrice(lastMonth.toString()+"28")//ËæÕþ²ß// + + //´¦ÀíÐ¡ÇøÇé¿ö + exportBigdataCommunity()// + exportCommunityStaging()//ûcityid + exportCommunityBuileType()// exportThirdPartyPrice2(new_dists)// exportBigdataForsale(new_dists) //2022-07-19 -// exportBigdataDeal( new_dists) //2022-07-01 + exportBigdataDeal( new_dists) //2022-07-01 // // //µÚÈý·½ÍøÕ¾È¨Öرí -// exportWebsiteRate(); + exportWebsiteRate(); // // //¼ÆËã½á¹ûÏà¹Ø // exportAvmEvaluationGJNB()// From 6e5a9c739ba1db6b4c5ada371f3b264010c8609e Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 12 Dec 2022 17:09:23 +0800 Subject: [PATCH 21/99] =?UTF-8?q?94-=E3=80=8B236?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy index 7b1c88343..02aa217b8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy @@ -15,7 +15,7 @@ import java.sql.ResultSet class ExportOracleAllDataTest { - def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl", + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") def cityCodeMap = CcbCommonUtils.findCityCodeValue() From 09a80b9a831705a9cdb6abf077650c63aa6b218f Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 14 Dec 2022 15:57:08 +0800 Subject: [PATCH 22/99] =?UTF-8?q?=E5=BB=BA=E8=A1=8C15=E5=8F=B7=E5=90=8C?= =?UTF-8?q?=E6=AD=A5=E5=87=86=E5=A4=87?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ExportOracleAllDataTest.groovy | 18 ++--- .../ronghe/ExportDistrictRongHe.groovy | 68 +++++++++++++++++++ .../gj/district/FangjiaClientUtils.groovy | 13 ++-- 3 files changed, 86 insertions(+), 13 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy index 02aa217b8..a8b061813 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy @@ -62,17 +62,17 @@ class ExportOracleAllDataTest { exportWebsiteRate(); // // //¼ÆËã½á¹ûÏà¹Ø -// exportAvmEvaluationGJNB()// -// exportAVMEvaluationItemGJNB()// + exportAvmEvaluationGJNB()// + exportAVMEvaluationItemGJNB()// + + exportCityPrice()// + exportRegionPrice()// // -// exportCityPrice()// -// exportRegionPrice()// + exportCommunityBasePriceGj()// + exportCommunityBasePrice()// // -// exportCommunityBasePriceGj()// -// exportCommunityBasePrice()// -// -// exportCommunityPdescribeGj()// -// exportCommCaseDetailGj()// + exportCommunityPdescribeGj()// + exportCommCaseDetailGj()// def curDate = curMonth.toString()+"28" // okÎļþ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy index a4a366773..a16f783fb 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy @@ -49,6 +49,10 @@ class ExportDistrictRongHe extends BaseBatch{ exportBigdataBuildprojFjw() exportBigdataCommlayoutpicFjw() exportEvaluateRate() + + //ÒòΪÈÚºÏÔݲ»Çп⣬ËùÒÔÐèÒª¼Ó´«Ò»ÕŵÚÈý·½¾ù¼Û±íµ½ÈÚºÏÖУ¬ ´Ó1ÔÂ10ÈÕ¿ªÊ¼´«Êä + def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); + exportThirdPartyPrice2(lastMonth, new_dists) } SSHHelper jschCommand = new SSHHelper(); System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}")); @@ -801,6 +805,70 @@ class ExportDistrictRongHe extends BaseBatch{ println "EVALUATE_RATE_FJW ÉÏ´«FtpÍê³É......" } + def exportThirdPartyPrice2(price_date, new_dists) { + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def fileName = "/ccb_rh/FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" + } + def headers = [] + def index = 0 + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def allRows = [] + if(new_dists) { + def newstr = "('"+new_dists.join("','")+"')" + def sql1 = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, " + + "COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from BIGDATA_THIRD_PARTY_PRICE_GJ where " + + "CITYID='${cityCode}' and price_date=${price_date} and distid not in ${newstr} "; + def rets1 = getThirdPartyDatas(sql1) + allRows.addAll(rets1) + + def sql2 = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, " + + "COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from BIGDATA_THIRD_PARTY_PRICE_GJ where " + + "CITYID='${cityCode}' and distid in ${newstr} "; + def rets2 = getThirdPartyDatas(sql2) + allRows.addAll(rets2) + }else{ + def sql = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from BIGDATA_THIRD_PARTY_PRICE_GJ where CITYID='${cityCode}' and price_date=${price_date} "; + allRows = getThirdPartyDatas(sql) + } + if(allRows && allRows.size()>0){ + index = index + 1 + if(index==1){ + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + }else{ + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); + } + } + } + + println "BIGDATA_THIRD_PARTY_PRICE_GJ Êý¾ÝÎļþÊä³öÍê³É......" + CcbFtpUtils.ftpUpload('rh', fileName, curDate) + } + + def getThirdPartyDatas(def sql){ + def allRows = [] + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.ID + data << r.CITYID + data << r.DISTID + data << r.COMMID + data << r.COMMNAME + data << r.DATA_SOURCE_ID + data << r.DATA_SOURCE + data << r.PRICE_DATE + data << r.COMMUNITY_PRICE + data << r.COMMUNITY_MAX_PRICE + data << r.COMMUNITY_MIN_PRICE + data << r.UPDATETIME + allRows << data + }) + return allRows; + } + def exportBigdataBuildprojFjw(){ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy index ae1dee2e4..4b363b071 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy @@ -90,10 +90,15 @@ class FangjiaClientUtils { def jsonObj = null def content = null try { - address = address.replace("{","") - def url = "http://192.168.0.19:31080/FangJiaAddressService/structure/v2?address="+URLEncoder.encode(address,"UTF-8")+"&city="+URLEncoder.encode(city,"UTF-8") - content = new HttpRequest().newGet(url, null, "utf-8") - jsonObj = JSONObject.fromObject(content) + if(address){ + address = address.replace("{","") + def url = "http://192.168.0.19:31080/FangJiaAddressService/structure/v2?address="+URLEncoder.encode(address,"UTF-8") + if(city){ + url = url +"&city="+URLEncoder.encode(city,"UTF-8") + } + content = new HttpRequest().newGet(url, null, "utf-8") + jsonObj = JSONObject.fromObject(content) + } } catch (Exception e) { println address +" "+content e.printStackTrace(); From a21a64cad950a9b4b533f94ae62dee91fe57fc18 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 14 Dec 2022 18:47:22 +0800 Subject: [PATCH 23/99] update --- .../ExportOracleAllDataTest.groovy | 46 +++++++++++-------- 1 file changed, 28 insertions(+), 18 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy index a8b061813..b9b854e62 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy @@ -41,6 +41,9 @@ class ExportOracleAllDataTest { def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); def lastMonthStr = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM") def new_dists = [] + + exportBigdataForsale(new_dists) //2022-07-19 + //²»»á±ä 10.25 exportEvaluationRateData() exportBigdataPoiData()// @@ -55,7 +58,6 @@ class ExportOracleAllDataTest { exportCommunityBuileType()// exportThirdPartyPrice2(new_dists)// - exportBigdataForsale(new_dists) //2022-07-19 exportBigdataDeal( new_dists) //2022-07-01 // // //µÚÈý·½ÍøÕ¾È¨Öرí @@ -669,22 +671,24 @@ class ExportOracleAllDataTest { " BUILDTYPE, HOUSESTRU,BUILDSTRU,BUILDAREA,LIVINGAREA,HOUSESTYLE,ROOMCNT,LIVINGROOMCNT,KITCHENCNT, BATHROOMCNT,HOUSEASPECT,FITMENT,IFELEVATOR,ELEVATORHOUSE,to_char(PUBLISHTIME,'yyyy-mm-dd hh24:mi:ss') as PUBLISHTIME,to_char(EXPIRETIME,'yyyy-mm-dd hh24:mi:ss') as EXPIRETIME,\n" + " IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH from bigdata_forsale_gj " + "where cityId='${cityCode}' and distid not in ${newstr}"; - def rets1 = getHouseDatas(sql1) - allRows.addAll(rets1) + def rets1 = getHouseDatas(sql1, fileName) + if(rets1){ + allRows.addAll(rets1) + } + }else{ def sql = "select CITYID,FORSALEID,PROPCODE,CITYNAME,DISTID,DISTNAME,COMMID,COMMNAME, BUILDID,BUILDNO, BUILDNAME,UNITID,UNITNO,UNITNAME,HOUSEID,ADDRESS,BUILDYEAR,PROPTYPE,PROPYEARS,UNITPRICE,TOTALPRICE,PAYTYPE,FLOORNUM,FLOORCNT,\n" + " BUILDTYPE, HOUSESTRU,BUILDSTRU,BUILDAREA,LIVINGAREA,HOUSESTYLE,ROOMCNT,LIVINGROOMCNT,KITCHENCNT, BATHROOMCNT,HOUSEASPECT,FITMENT,IFELEVATOR,ELEVATORHOUSE,to_char(PUBLISHTIME,'yyyy-mm-dd hh24:mi:ss') as PUBLISHTIME,to_char(EXPIRETIME,'yyyy-mm-dd hh24:mi:ss') as EXPIRETIME,\n" + " IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH from bigdata_forsale_gj where cityId='${cityCode}' "; - allRows = getHouseDatas(sql) + allRows = getHouseDatas(sql, fileName) } - if(allRows.size()>0){ - index = index + 1 - if(index == 1){ - FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); - }else{ + if(allRows && allRows.size()>0){ +// index = index + 1 +// if(index == 1){ +// FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); +// }else{ FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); - } - +// } } } @@ -694,7 +698,7 @@ class ExportOracleAllDataTest { " BUILDTYPE, HOUSESTRU,BUILDSTRU,BUILDAREA,LIVINGAREA,HOUSESTYLE,ROOMCNT,LIVINGROOMCNT,KITCHENCNT, BATHROOMCNT,HOUSEASPECT,FITMENT,IFELEVATOR,ELEVATORHOUSE,to_char(PUBLISHTIME,'yyyy-mm-dd hh24:mi:ss') as PUBLISHTIME,to_char(EXPIRETIME,'yyyy-mm-dd hh24:mi:ss') as EXPIRETIME,\n" + " IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH " + "from bigdata_forsale_gj where distid in ${newstr}" - def rets2 = getHouseDatas(sql2) + def rets2 = getHouseDatas(sql2, fileName) if(rets2.size()>0){ FileUtils.batchCreateFile(rets2, fileName, headers, "utf-8", true); } @@ -704,7 +708,7 @@ class ExportOracleAllDataTest { println "BIGDATA_FORSALE_GJ ÉÏ´«FtpÍê³É......" } - def getHouseDatas(sql){ + def getHouseDatas(sql, fileName){ def allRows = [] def index = 0 sqlInstance.eachRow(sql,{r-> @@ -764,12 +768,18 @@ class ExportOracleAllDataTest { data << r.SOURCECODE data << r.ATTACH allRows << data -// if(allRows.size()%200000==0){ -// FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); -// allRows.clear() -// } + if(allRows.size()%200000==0){ + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", true); + allRows.clear() + } }); - return allRows + + if(allRows.size>0){ + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", true); + allRows.clear() + } +// return allRows + return null } From 73a036beb93621cce6a35dbdae23e900ecf4449e Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 15 Dec 2022 08:10:18 +0800 Subject: [PATCH 24/99] update --- .../ExportOracleAllDataTest.groovy | 50 +++++++++---------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy index b9b854e62..3f7954e53 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy @@ -42,33 +42,33 @@ class ExportOracleAllDataTest { def lastMonthStr = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM") def new_dists = [] - exportBigdataForsale(new_dists) //2022-07-19 - - //²»»á±ä 10.25 - exportEvaluationRateData() - exportBigdataPoiData()// - exportBigdataDistrictData()// - - exportGuidedPrice()//ËæÈ˹¤¸Ä±ä// - exportCommunityGuidedPrice(lastMonth.toString()+"28")//ËæÕþ²ß// - - //´¦ÀíÐ¡ÇøÇé¿ö - exportBigdataCommunity()// - exportCommunityStaging()//ûcityid - exportCommunityBuileType()// - - exportThirdPartyPrice2(new_dists)// - exportBigdataDeal( new_dists) //2022-07-01 +// exportBigdataForsale(new_dists) //2022-07-19 // -// //µÚÈý·½ÍøÕ¾È¨Öرí - exportWebsiteRate(); +// //²»»á±ä 10.25 +// exportEvaluationRateData() +// exportBigdataPoiData()// +// exportBigdataDistrictData()// // -// //¼ÆËã½á¹ûÏà¹Ø - exportAvmEvaluationGJNB()// - exportAVMEvaluationItemGJNB()// - - exportCityPrice()// - exportRegionPrice()// +// exportGuidedPrice()//ËæÈ˹¤¸Ä±ä// +// exportCommunityGuidedPrice(lastMonth.toString()+"28")//ËæÕþ²ß// +// +// //´¦ÀíÐ¡ÇøÇé¿ö +// exportBigdataCommunity()// +// exportCommunityStaging()//ûcityid +// exportCommunityBuileType()// +// +// exportThirdPartyPrice2(new_dists)// +// exportBigdataDeal( new_dists) //2022-07-01 +//// +//// //µÚÈý·½ÍøÕ¾È¨Öرí +// exportWebsiteRate(); +//// +//// //¼ÆËã½á¹ûÏà¹Ø +// exportAvmEvaluationGJNB()// +// exportAVMEvaluationItemGJNB()// +// +// exportCityPrice()// +// exportRegionPrice()// // exportCommunityBasePriceGj()// exportCommunityBasePrice()// From d58d54fd9cc61f98728e66a399b22eb321cac77f Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 15 Dec 2022 09:43:51 +0800 Subject: [PATCH 25/99] update --- .../ExportOracleAllDataTest.groovy | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy index 3f7954e53..a9b26861a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy @@ -949,6 +949,12 @@ class ExportOracleAllDataTest { def exportCommunityBasePriceGj(){ def allRows = [] + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; + def fileName = "/ccb_evaluation/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" + } + def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"] for(def city : citys) { def cityCode = cityCodeMap.get(city) def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, FAKE_RATE, DIFF_RATE, CONFI_RATE, CORR_RATE, CONFIDENCE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_base_price_gj where CITYID='${cityCode}' "; @@ -974,15 +980,17 @@ class ExportOracleAllDataTest { data << r.UPDATETIME data << r.SOURCECODE allRows << data + if(allRows.size()%200000==0){ + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", true); + allRows.clear() + } }); } - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; - def fileName = "/ccb_evaluation/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" - if(CommonDatautils.isWindow()){ - fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" + + if(allRows.size()>0){ + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", true); } - def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"] - FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "COMMUNITY_BASE_PRICE_GJ Êý¾ÝÎļþÊä³öÍê³É......" CcbFtpTestUtils.ftpUpload(null, fileName, curDate) println "COMMUNITY_BASE_PRICE_GJ ÉÏ´«FtpÍê³É......" From 813b58967fc6a3006983369755dd044e2f29fdf1 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 15 Dec 2022 10:39:42 +0800 Subject: [PATCH 26/99] update --- .../ExportOracleAllDataTest.groovy | 72 ++++++++++++------- 1 file changed, 45 insertions(+), 27 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy index a9b26861a..bbfa5d1de 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy @@ -70,17 +70,17 @@ class ExportOracleAllDataTest { // exportCityPrice()// // exportRegionPrice()// // - exportCommunityBasePriceGj()// +// exportCommunityBasePriceGj()// exportCommunityBasePrice()// -// +//// exportCommunityPdescribeGj()// exportCommCaseDetailGj()// - - def curDate = curMonth.toString()+"28" - // okÎļþ - exportInOk() - SSHHelperTest jschCommand = new SSHHelperTest(); - System.out.println(jschCommand.executeCommand("sh /home/ccb_evaluation/output/eval_package.sh ${curDate}")); +// +// def curDate = curMonth.toString()+"28" +// // okÎļþ +// exportInOk() +// SSHHelperTest jschCommand = new SSHHelperTest(); +// System.out.println(jschCommand.executeCommand("sh /home/ccb_evaluation/output/eval_package.sh ${curDate}")); } @@ -904,6 +904,15 @@ class ExportOracleAllDataTest { def exportCommunityBasePrice(){ def allRows = [] + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; + def fileName = "/ccb_evaluation/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat" + def fileName1 = "/ccb_evaluation/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat" + fileName1 = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" + } + def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"] + for(def city : citys) { def cityCode = cityCodeMap.get(city) def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, FAKE_RATE, DIFF_RATE, CONFI_RATE, CORR_RATE, CONFIDENCE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_base_price_gj where CITYID='${cityCode}' "; @@ -929,18 +938,19 @@ class ExportOracleAllDataTest { data << r.UPDATETIME data << r.SOURCECODE allRows << data + if(allRows.size()%200000==0){ + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); + FileUtils.batchCreateFile(allRows, fileName1, headers, "utf-8", true); + allRows.clear() + } }); } - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; - def fileName = "/ccb_evaluation/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat" - def fileName1 = "/ccb_evaluation/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" - if(CommonDatautils.isWindow()){ - fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat" - fileName1 = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" + + if(allRows.size()>0){ + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); + FileUtils.batchCreateFile(allRows, fileName1, headers, "utf-8", true); } - def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"] - FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); - FileUtils.batchCreateFile(allRows, fileName1, headers, "utf-8", false); + println "COMMUNITY_BASE_PRICE_GJNB Êý¾ÝÎļþÊä³öÍê³É......" CcbFtpTestUtils.ftpUpload(null, fileName, curDate) CcbFtpTestUtils.ftpUpload(null, fileName1, curDate) @@ -1120,11 +1130,19 @@ class ExportOracleAllDataTest { data << r.SATUR_RATIO data << r.NEIGHBOR_PRICE allRows << data + + if(allRows.size()%200000==0){ + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); + allRows.clear() + } }); } println allRows.size() - FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + if(allRows.size()%200000==0){ + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); + allRows.clear() + } println "community_p_describe_gj Êý¾ÝÎļþÊä³öÍê³É......" CcbFtpTestUtils.ftpUpload(null, fileName, curDate) println "community_p_describe_gj ÉÏ´«FtpÍê³É......" @@ -1138,12 +1156,11 @@ class ExportOracleAllDataTest { fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\${curDate}\\MODEL_000000_000000_COMM_CASE_DETAIL_GJ_ADD_${curDate}_0001.dat" } def headers = [] - + def allRows = [] def index = 0 for(def city : citys) { def cityCode = cityCodeMap.get(city) def sql = "select ID, CITYID, COMMUNITY_ID, NETLISTING_ID, PRICE_DATE, to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE from comm_case_detail_gj where cityid='${cityCode}'"; - def allRows = [] sqlInstance.eachRow(sql, { r -> def data = [] data << r.ID @@ -1153,15 +1170,16 @@ class ExportOracleAllDataTest { data << r.PRICE_DATE data << r.CREATE_DATE allRows << data - }); - if (allRows && allRows.size() > 0) { - index = index + 1 - if (index == 1) { - FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); - } else { + if(allRows.size()%200000==0){ FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); + allRows.clear() } - } + }); + } + + if(allRows.size()>0){ + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); + allRows.clear() } println "comm_case_detail_gj Êý¾ÝÎļþÊä³öÍê³É......" CcbFtpTestUtils.ftpUpload(null, fileName, curDate) From 70f0ed82f83b0fcd9f11c7f64a4e67e6f08cb7b1 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 15 Dec 2022 12:34:29 +0800 Subject: [PATCH 27/99] update --- .../script/ccbevaluation/ExportOracleAllDataTest.groovy | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy index bbfa5d1de..6c2dbcf26 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy @@ -71,10 +71,10 @@ class ExportOracleAllDataTest { // exportRegionPrice()// // // exportCommunityBasePriceGj()// - exportCommunityBasePrice()// -//// +// exportCommunityBasePrice()// +// exportCommunityPdescribeGj()// - exportCommCaseDetailGj()// +// exportCommCaseDetailGj()// // // def curDate = curMonth.toString()+"28" // // okÎļþ @@ -1139,7 +1139,7 @@ class ExportOracleAllDataTest { } println allRows.size() - if(allRows.size()%200000==0){ + if(allRows.size()>0){ FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); allRows.clear() } From 5753e55512a5b52b153c94b0643dcda04486c5e1 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 19 Dec 2022 14:28:19 +0800 Subject: [PATCH 28/99] =?UTF-8?q?=E5=BB=BA=E8=A1=8C=E6=88=BF=E6=BA=90?= =?UTF-8?q?=E5=8F=91=E5=B8=83=20=E6=97=B6=E9=97=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index d89e8501e..639400868 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -74,8 +74,8 @@ class BigdataForsaleBatch extends BaseBatch{ // def distIds = getDistrictIds(cityCodeDataMap.get(city)) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]); -// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20221018, 20221119); - exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210901, 20221119); + exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20221118, 20221219); +// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210901, 20221119); // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20211001, 20221011); // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831); //²âÊÔͨ¹ýÆ¥Åä½Ó¿ÚÅ䷿ԴЧ¹û From c35e244ab150aa38beb471fc30aca02d05ccd12a Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 20 Dec 2022 15:15:07 +0800 Subject: [PATCH 29/99] update --- .../ronghe/ExportDistrictRongHeTest.groovy | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy index cb23d1b7b..c274629b5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy @@ -29,33 +29,33 @@ class ExportDistrictRongHeTest extends BaseBatch{ def curDate = "20221210"//DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd") def new_dists = []//["350604","350605","410307","410308","450181"] //°´Ììµ¼£¨Ã¿ÌìÒ»¸öÎļþ¼Ð£© -// exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-1), "yyyy-MM-dd"), new_dists) + exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-1), "yyyy-MM-dd"), new_dists) // if(curDate.toString().endsWith("10")){ -// exportEvaluateRate() -// exportBigdataBuildprojFjw() -// exportBigdataCommlayoutpicFjw() -// -// -// exportBigdataCommunityGJRH()// -// exportBigdataBuildingFJW()// -// exportBigdataCommfeeGj()// -// -// exportBigdataCommbuildstruGJ()// -// exportBigdataCommhousetypeGJ()// -// exportBigdataCommbuildyearGj()// -// exportBigdataCommhouseuseGJ()// -// exportBigdataPositionGJ()// -// exportBigdataDistrictFJW()// + exportEvaluateRate() + exportBigdataBuildprojFjw() + exportBigdataCommlayoutpicFjw() + + + exportBigdataCommunityGJRH()// + exportBigdataBuildingFJW()// + exportBigdataCommfeeGj()// + + exportBigdataCommbuildstruGJ()// + exportBigdataCommhousetypeGJ()// + exportBigdataCommbuildyearGj()// + exportBigdataCommhouseuseGJ()// + exportBigdataPositionGJ()// + exportBigdataDistrictFJW()// exportBigdataAliasFJW()// //±¾12.10ÉÏ´«10£¬11ÔÂËùÓÐÊý¾Ý£¬23Äê1ÔÂÐè¸ÄΪËùÓгÇÊа´ÈÕÆÚͬ²½Ò»ÆÚÊý¾Ý -// def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger() -// exportBigdataCityPrice(queryDate) -// exportBigdataDistPrice(queryDate) -// exportBigdataCommPrice(queryDate) + def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger() + exportBigdataCityPrice(queryDate) + exportBigdataDistPrice(queryDate) + exportBigdataCommPrice(queryDate) exportInOk() From 884d836c5446f233d31e3d2c199d520aa6cb45dd Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 20 Dec 2022 15:30:09 +0800 Subject: [PATCH 30/99] =?UTF-8?q?=E6=B5=8B=E8=AF=95=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E4=BD=8D=E7=BD=AE=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/script/ccbevaluation/CcbFtpTestUtils.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpTestUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpTestUtils.groovy index da0ac8e8e..81958589e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpTestUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpTestUtils.groovy @@ -7,7 +7,7 @@ import org.apache.commons.codec.digest.DigestUtils class CcbFtpTestUtils { - static def ftpServer = "192.168.0.45" + static def ftpServer = "192.168.0.86"//45 static def ftpPort = 17495 static def ftpUser = "ccb" static def ftpPassword = "ccb!@#" From 9743a72aeb493e86a8ce8f4776ac49e2bcb44c84 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 20 Dec 2022 17:33:23 +0800 Subject: [PATCH 31/99] =?UTF-8?q?=E7=AC=AC=E4=B8=89=E6=96=B9=E6=97=B6?= =?UTF-8?q?=E9=97=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ExportOracleAllDataTest.groovy | 6 ++--- .../ccbevaluation/FjwPricePkCcbPrice.groovy | 22 ++++++++++++++++++- .../ccbevaluation/ThirdPartyPriceBatch.groovy | 6 ++--- 3 files changed, 27 insertions(+), 7 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy index 6c2dbcf26..42f71fc0a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy @@ -73,12 +73,12 @@ class ExportOracleAllDataTest { // exportCommunityBasePriceGj()// // exportCommunityBasePrice()// // - exportCommunityPdescribeGj()// +// exportCommunityPdescribeGj()// // exportCommCaseDetailGj()// // // def curDate = curMonth.toString()+"28" // // okÎļþ -// exportInOk() + exportInOk() // SSHHelperTest jschCommand = new SSHHelperTest(); // System.out.println(jschCommand.executeCommand("sh /home/ccb_evaluation/output/eval_package.sh ${curDate}")); @@ -1207,7 +1207,7 @@ class ExportOracleAllDataTest { } def exportInOk(){ - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"15" Vector files = CcbFtpTestUtils.listFileNames(null, curDate) Iterator fileVec = files.iterator() def fileRows = [] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/FjwPricePkCcbPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/FjwPricePkCcbPrice.groovy index b28516075..9c6b96508 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/FjwPricePkCcbPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/FjwPricePkCcbPrice.groovy @@ -29,7 +29,9 @@ class FjwPricePkCcbPrice { // task.updatePrice() // task.addFjwPrice() // task.addCcbPrice() - task.addMarketPrice() + + task.addCcbPriceFromGuided() +// task.addMarketPrice() } def addMarketPrice(){ @@ -66,6 +68,24 @@ class FjwPricePkCcbPrice { }) } + def addCcbPriceFromGuided(){ + def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + ExcelUtils.multiThreadingModify("D:\\Êý¾Ý\\½¨ÐÐ\\¶þÆÚ\\È˹¤Ö¸µ¼¼Û2.xls",{d-> + if(d['B'] && !d['O']){ + def cityId = d['D'] + def dist = CommonDatautils.findDistrictById(d['B']) + if(dist){ + def map = getCommunityBasePriceGjByCommid(cityId, dist._id.toString(), 202211) + if(map){ + d['O'] = map.avg_price + d['P'] = map.calc_type + d['Q'] = map.credit + } + } + } + }) + } + def updatePrice(){ def index = 0 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy index 027e71834..295ab8915 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy @@ -22,9 +22,9 @@ import java.util.concurrent.ConcurrentHashMap @BatchOwner(Watcher.GaoJuan) class ThirdPartyPriceBatch extends BaseBatch { - def sDate= 202210//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); - def startDate = 202208 - def endDate = 202210 + def sDate= 202211//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); + def startDate = 202209 + def endDate = 202211 // def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM")); static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") From 73f4ebbd5c19b95d257ea869215e0091d9b33b85 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 21 Dec 2022 18:04:47 +0800 Subject: [PATCH 32/99] =?UTF-8?q?=E6=88=90=E4=BA=A4=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ccbevaluation/BigdataDealBatch.groovy | 35 +- .../ccbevaluation/FjwPricePkCcbPrice.groovy | 2 +- .../ccbevaluation/GuiDingPriceBatch.groovy | 6 +- .../fangjia/script/ccbevaluation/Test.groovy | 94 +++-- .../script/ccbevaluation/TradeStatLog.groovy | 14 +- .../com/fangjia/script/fuzhou/Test.groovy | 73 +++- .../com/fangjia/script/fuzhou/Test1.groovy | 2 +- .../fangjia/script/gj/YinLianMeiShi.groovy | 394 +++++++++++++++--- .../fangjia/script/gj/YouDaDataParser.groovy | 2 +- .../gj/paimai/LianjiaCrawlByChrome.groovy | 2 +- .../temp/shenjian/util/GaoDeUtils.groovy | 29 ++ .../script/thirdPriceCrawl/DataStat.groovy | 8 +- .../com/fangjia/script/util/MongoUtils.groovy | 2 + 13 files changed, 518 insertions(+), 145 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index d73c64d44..af28b3fd4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -58,17 +58,17 @@ class BigdataDealBatch extends BaseBatch{ public void process(Object args) { //def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date()) // def citys = CcbCommonUtils.citys; - def citys = ["ÉϺ£"]//CcbCommonUtils.getRunningCitys(args) + def citys = CcbCommonUtils.getRunningCitys(args) if(args && args.size()>=2){ curDate = args[1].toString().toInteger() } -// println "curDate:"+curDate -// for(def city : citys){ -//// if(!ccbCitys.contains(city)){ -// addDealPriceMatch(city); -//// } -// } + println "curDate:"+curDate + for(def city : citys){ +// if(!ccbCitys.contains(city)){ + addDealPriceMatch(city); +// } + } def rows = [] def headers = []//["³ÇÊÐID","³É½»ID","³ÇÊÐÃû³Æ","ÐÐÕþÇøID","ÐÐÕþÇøÃû³Æ","½ÖµÀ","ÃÅÅÆºÅ","Ð¡ÇøID","Ð¡ÇøÃû³Æ","Â¥¶°ID","Â¥¶°±àºÅ","Â¥¶°Ãû³Æ","µ¥ÔªID","µ¥Ôª±àºÅ","µ¥ÔªÃû³Æ","»§ID","ÏêϸµØÖ·","½¨ÖþÄê´ú","ȨÊô","ȨÊôÄêÏÞ","ÆÚÊý","ËùÔÚÂ¥²ã","²ãÀàÐÍ","×ÜÂ¥²ãÊý","½¨ÖþÀàÐÍ","½¨Öþ½á¹¹","½¨ÖþÃæ»ý","Ì×ÄÚÃæ»ý","»§ÐÍ","ÊÒ","Ìü","³ø","ÎÀ","·¿ÎÝÓÃ;","ÊÒÄڽṹ","³¯Ïò","×°ÐÞÇé¿ö","ÆÀ¹À±àºÅ","³É½»µ¥¼Û","³É½»×ܼÛ","É걨×ܼÛ","ÆÀ¹À×ܼÛ","ÍøÇ©ºÏͬ±àºÅ","½»Ò×ÈÕÆÚ","½»Ò×ÀàÐÍ","³öÊÛ¸¶¿î·½Ê½","³É½»ÖÜÆÚ","ÊÇ·ñÓеçÌÝ","ÔùËÍÃæ»ý£¨µØÃ滨԰£©","ÔùËÍÃæ»ý£¨Â¶Ì¨£©","ÔùËÍÃæ»ý£¨Ñǫ̂£©","ÔùËÍÃæ»ý£¨Æ®´°£©","ÔùËÍÃæ»ý£¨¸óÂ¥£©","ÔùËÍÃæ»ý£¨³µ¿â£©","ÔùËÍÃæ»ý£¨µØÏÂÊÒ£©","ÔùËÍÃæ»ý£¨ÆäËû£©","·¿Ô´Êý¾ÝÀ´Ô´","Öн鹫˾ID","Öн鹫˾Ãû³Æ","±¸×¢","´´½¨Ê±¼ä","¸üÐÂʱ¼ä","À´Ô´¹«Ë¾","³É½»½ð¶î","Êý¾Ýʱ¼ä"] def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() @@ -86,17 +86,16 @@ class BigdataDealBatch extends BaseBatch{ println city +" "+youdaDatas.size() } + def trades = getCityData(city, statMap, ppeDists) + if(trades){ + def rets = createData(trades, cityCodeDataMap, distCodeMap, "other") + rows.addAll(rets) + println "¡¾${index}¡¿"+city +" ³É½»¹²£º"+rets.size() -// def trades = getCityData(city, statMap, ppeDists) -// if(trades){ -// def rets = createData(trades, cityCodeDataMap, distCodeMap, "other") -// rows.addAll(rets) -// println "¡¾${index}¡¿"+city +" ³É½»¹²£º"+rets.size() -// -// def cityDatas = [] -// cityDatas << headers -// cityDatas.addAll(rows) -// } + def cityDatas = [] + cityDatas << headers + cityDatas.addAll(rows) + } } @@ -105,7 +104,7 @@ class BigdataDealBatch extends BaseBatch{ fileName = "d:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\BIGDATA_DEAL_GJ.dat" } - FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", true); + FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); println "txt Éú³É " + rows.size() } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/FjwPricePkCcbPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/FjwPricePkCcbPrice.groovy index 9c6b96508..4d8b7fa02 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/FjwPricePkCcbPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/FjwPricePkCcbPrice.groovy @@ -75,7 +75,7 @@ class FjwPricePkCcbPrice { def cityId = d['D'] def dist = CommonDatautils.findDistrictById(d['B']) if(dist){ - def map = getCommunityBasePriceGjByCommid(cityId, dist._id.toString(), 202211) + def map = getCommunityBasePriceGjByCommid(cityId, dist._id.toString(), 202209) if(map){ d['O'] = map.avg_price d['P'] = map.calc_type diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index 0d15402de..0882eba51 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -38,16 +38,16 @@ class GuiDingPriceBatch extends BaseBatch{ @Override public void process(Object args) { - def citys = ["ÉϺ£"]//CcbCommonUtils.getRunningCitys(args) + def citys = CcbCommonUtils.getRunningCitys(args) def rows = [] -// CcbCommonUtils.deleteGuidedPrice(curDate) + CcbCommonUtils.deleteGuidedPrice(curDate) for (def city : citys) { def cityCodeDatas = CcbCommonUtils.findCityCodeValue() def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) //Ìí¼Ó×Ô¼º¿âÖеÄÈ˹¤Ö¸µ¼¼Û - def prices = CcbCommonUtils.findPersonPrice(city, "2022-11-24", "2022-11-25") + def prices = CcbCommonUtils.findPersonPrice(city, "2022-11-25", "2022-12-19") // def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22") if(prices){ def distMap = getDistrictMap(city) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index 947d57cb8..b7afc9818 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -47,41 +47,41 @@ def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business -def citys = ['ËÕÖÝ','º¼ÖÝ','Äþ²¨','ÎÂÖÝ','¼ÎÐË','ºþÖÝ','ÉÜÐË','½ð»ª','áéÖÝ','ÖÛɽ','̨ÖÝ','ÀöË®'] - -def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() -def rets = [] -for(def c : citys){ - def citycode = cityCodeDataMap.get(c) - def datas = findAllData(sqlInstance236, citycode) - def totalCount = datas.size() - def landareaCount = 0; - def buildareaCount = 0 - def greeningrateCount = 0 - def plotratioCount = 0 - for(def m : datas){ - if(m.landarea){ - landareaCount ++ - } - if(m.buildarea){ - buildareaCount++ - } - if(m.greeningrate){ - greeningrateCount++ - } - if(m.plotratio){ - plotratioCount++ - } - } - - def landAreaPre = landareaCount*100/totalCount - def buildAreaPre = buildareaCount*100/totalCount - def greeningratePre = greeningrateCount*100/totalCount - def plotratioPre = plotratioCount*100/totalCount - rets << [c, totalCount, landAreaPre, buildAreaPre, greeningratePre, plotratioPre] -} - -ExcelUtils.write("D:\\Êý¾Ý\\½¨ÐÐ\\²âÊÔ\\12³Ç¸²¸ÇÂÊͳ¼Æ.xlsx",rets, "¸²¸ÇÂÊ") +//def citys = ['ËÕÖÝ','º¼ÖÝ','Äþ²¨','ÎÂÖÝ','¼ÎÐË','ºþÖÝ','ÉÜÐË','½ð»ª','áéÖÝ','ÖÛɽ','̨ÖÝ','ÀöË®'] +// +//def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() +//def rets = [] +//for(def c : citys){ +// def citycode = cityCodeDataMap.get(c) +// def datas = findAllData(sqlInstance236, citycode) +// def totalCount = datas.size() +// def landareaCount = 0; +// def buildareaCount = 0 +// def greeningrateCount = 0 +// def plotratioCount = 0 +// for(def m : datas){ +// if(m.landarea){ +// landareaCount ++ +// } +// if(m.buildarea){ +// buildareaCount++ +// } +// if(m.greeningrate){ +// greeningrateCount++ +// } +// if(m.plotratio){ +// plotratioCount++ +// } +// } +// +// def landAreaPre = landareaCount*100/totalCount +// def buildAreaPre = buildareaCount*100/totalCount +// def greeningratePre = greeningrateCount*100/totalCount +// def plotratioPre = plotratioCount*100/totalCount +// rets << [c, totalCount, landAreaPre, buildAreaPre, greeningratePre, plotratioPre] +//} +// +//ExcelUtils.write("D:\\Êý¾Ý\\½¨ÐÐ\\²âÊÔ\\12³Ç¸²¸ÇÂÊͳ¼Æ.xlsx",rets, "¸²¸ÇÂÊ") def findAllData(sqlInstance236, citycode){ def sql = "select landarea,buildarea,greeningrate,plotratio from bigdata_community_fjw where cityid='${citycode}'" @@ -1149,6 +1149,30 @@ def findThirdCommCount(cityid, sqlInstance){ +ExcelUtils.modify("D:\\Êý¾Ý\\½¨ÐÐ\\¶þÆÚ\\¸¶.xlsx",{d-> + def obj = CommonDatautils.findDistrict(d['D'],d['L'], d['K'], "district") + if(obj){ + def count = findGuidedPrice(sqlInstance236, obj._id.toString()) + if(count){ + d['M'] = "´æÔÚ" + } + d['N'] = obj.name_cat + d['O'] = obj.estate_type2 + } + +}) + +def findGuidedPrice(sqlInstance, id){ + def count = 0 + def sql = "select count(*) as count from guided_price_gj where commid='${id}'" + sqlInstance.eachRow(sql,{r-> + count = r.count + }); + return count +} + + + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy index ae3aa4800..e87103a90 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy @@ -50,7 +50,7 @@ class TradeStatLog { // task.deleteDealData2() // task.statTableCount() // task.filterTrade() - task.updateDealPriceType() +// task.updateDealPriceType() } @@ -90,15 +90,15 @@ class TradeStatLog { } def run(){ - def citys = CcbCommonUtils.citys_92; + def citys = CcbCommonUtils.citys_all; def rets = [] - rets << ["³ÇÊÐ","8ÔÂ×ÜÁ¿","9ÔÂ×ÜÁ¿","10ÔÂ×ÜÁ¿","Õ¼±È","±´¿Ç","Á´¼Ò", "ÎÒ°®ÎÒ¼Ò","·¿ÌìÏÂ","ÖÐÔ­"] + rets << ["³ÇÊÐ","9ÔÂ×ÜÁ¿","10ÔÂ×ÜÁ¿","11ÔÂ×ÜÁ¿","Õ¼±È","±´¿Ç","Á´¼Ò", "ÎÒ°®ÎÒ¼Ò","·¿ÌìÏÂ","ÖÐÔ­"] def sources = ["±´¿ÇÕÒ·¿","Á´¼ÒÔÚÏß","ÎÒ°®ÎÒ¼Ò","·¿ÌìÏÂ","ÖÐÔ­µØ²ú"] //def startDate = "2021-05-01" - def startDate = "2022-08-01" - def date = "2022-09-01" - def endDate = "2022-10-01" - def newDate = "2022-11-01" + def startDate = "2022-09-01" + def date = "2022-10-01" + def endDate = "2022-11-01" + def newDate = "2022-12-01" for(def city : citys){ def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd")) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test.groovy index 0231b21c0..47a3b331f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test.groovy @@ -5,9 +5,36 @@ import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.EsUtils import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection import org.apache.commons.io.FileUtils import org.bson.types.ObjectId + +DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com") + +def datas = FileUtils.readLines("D:\\Êý¾Ý\\ÆäËü\\FJW_COMM_20221015.csv") +def rets = [] +for(def d : datas){ + def arrs = d.split("@") + rets << ["commId":arrs[0], "avg_price":arrs[4]] +} + +for(def r : rets){ + if(r.avg_price){ + def udata = new BasicDBObject() + udata.put("date", 202210) + udata.put("avg", r.avg_price.toInteger()) + DISTRICT_SELL_STAT.update(new BasicDBObject("commid", r.commId) , new BasicDBObject('$set', data),false,false) + }else{ + println r.commId +" ²»´æÔÚ" + } +} + + + + //List list = FileUtils.readLines(new File("C:\\Users\\gaojuan\\Desktop\\FJW_HOUSE_201911.csv")) //println "¹ýÂËǰ£º"+list.size()+ " "+new Date().toLocaleString() //def ret = [] @@ -48,29 +75,29 @@ import org.bson.types.ObjectId // return location; //} -List list1= FileUtils.readLines(new File("D:\\Êý¾Ý\\¸£ÖÝ\\FJW_DISTRICT_STAT_202112.csv")) -def map1 = [:] -for(def s : list1){ - def row = s.split("@\\|@") - def key = row[2]+"##"+row[3]+"##"+row[4] - map1.put(key, row[5]) -} - -List list2 = FileUtils.readLines(new File("D:\\Êý¾Ý\\¸£ÖÝ\\FJW_DISTRICT_STAT_202111.csv")) -def map2 = [:] -for(def s : list2) { - def row = s.split("@\\|@") - def key = row[2] + "##" + row[3] + "##" + row[4] - map2.put(key, row[5]) -} -def rows = [] -map1.each {k,v-> - def row = [] - def keys = k.split("##") - def lastPrice = map2.get(k) - rows << [keys[0], keys[1], keys[2], v.toString(), lastPrice.toString()] -} -ExcelUtils.write("D:\\Êý¾Ý\\¸£ÖÝ\\¼Û¸ñ±È¶Ô12-2.xlsx", rows, "Êý¾Ý") +//List list1= FileUtils.readLines(new File("D:\\Êý¾Ý\\¸£ÖÝ\\FJW_DISTRICT_STAT_202112.csv")) +//def map1 = [:] +//for(def s : list1){ +// def row = s.split("@\\|@") +// def key = row[2]+"##"+row[3]+"##"+row[4] +// map1.put(key, row[5]) +//} +// +//List list2 = FileUtils.readLines(new File("D:\\Êý¾Ý\\¸£ÖÝ\\FJW_DISTRICT_STAT_202111.csv")) +//def map2 = [:] +//for(def s : list2) { +// def row = s.split("@\\|@") +// def key = row[2] + "##" + row[3] + "##" + row[4] +// map2.put(key, row[5]) +//} +//def rows = [] +//map1.each {k,v-> +// def row = [] +// def keys = k.split("##") +// def lastPrice = map2.get(k) +// rows << [keys[0], keys[1], keys[2], v.toString(), lastPrice.toString()] +//} +//ExcelUtils.write("D:\\Êý¾Ý\\¸£ÖÝ\\¼Û¸ñ±È¶Ô12-2.xlsx", rows, "Êý¾Ý") //ExcelUtils.multiThreadingModify("D:\\Êý¾Ý\\¸£ÖÝ\\¸£ÖÝÕÇ·ùÔÚ10%ÒÔÉÏÐ¡Çø.xlsx",{d-> // def obj = CommonDatautils.queryDistrictPrice(d['A'],d['B'], d['C'], DateUtils.parseFromTightByPattern("2021-10-10","yyyy-MM-dd")) // if(obj){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy index bcbe886de..7e4a0bd2c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy @@ -14,7 +14,7 @@ import com.mongodb.DBCollection //¸£ÖݵÚÈý·½¾ù¼Ûͳ¼Æ,·¢ËͶ¤¶¤ÏûÏ¢ //def datas = ExcelUtils.read("d:\\Êý¾Ý\\¸£ÖÝ\\¸£ÖÝ4ÔÂ2.xlsx",0,1) //def datas = CsvUtils.read("d:\\Êý¾Ý\\¸£ÖÝ\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv"); -def datas = readTxt("d:\\Êý¾Ý\\¸£ÖÝ\\FJW_THIRD_PARTY_AVG_PRICE_202210.csv") +def datas = readTxt("d:\\Êý¾Ý\\¸£ÖÝ\\FJW_THIRD_PARTY_AVG_PRICE_202211.csv") def count0 = 0 def count1 = 0 def count2 = 0 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YinLianMeiShi.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YinLianMeiShi.groovy index d1aca25f7..3075ff1db 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YinLianMeiShi.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YinLianMeiShi.groovy @@ -5,38 +5,112 @@ import com.fangjia.core.util.BaiDuApiUtils import com.fangjia.core.util.GeocodeUtils import com.fangjia.core.util.HttpRequest import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.linxsh.util.AddressUtil +import com.fangjia.script.statistics.poi.AmapPoi +import com.fangjia.script.temp.shenjian.util.GaoDeUtils import com.fangjia.script.util.CosineSimilarAlgorithm import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject +import com.mongodb.DBCollection import net.sf.json.JSONObject import org.apache.poi.ss.util.RegionUtil +import java.util.regex.Pattern + class YinLianMeiShi { def baiduCityCodeMap = ["±±¾©": "131", "Ìì½ò": "332", "ʯ¼Òׯ": "150", "ÌÆÉ½": "265", "ÇØ»Êµº": "148", "ºªµ¦": "151", "ÐĮ̈": "266", "±£¶¨": "307", "ÕżҿÚ": "264", "³ÐµÂ": "207", "²×ÖÝ": "149", "ÀÈ·»": "191", "ºâË®": "208", "̫ԭ": "176", "´óͬ": "355", "ÑôȪ": "357", "³¤ÖÎ": "356", "½ú³Ç": "290", "Ë·ÖÝ": "237", "½úÖÐ": "238", "Ô˳Ç": "328", "ÐÃÖÝ": "367", "ÁÙ·Ú": "368", "ÂÀÁº": "327", "ºôºÍºÆÌØ": "321", "°üÍ·": "229", "ÎÚº£": "123", "³à·å": "297", "ͨÁÉ": "64", "¶õ¶û¶à˹": "283", "ºôÂ×±´¶û": "61", "°ÍÑåÄ×¶û": "169", "ÎÚÀ¼²ì²¼": "168", "Ð˰²ÃË": "62", "ÎýÁÖ¹ùÀÕÃË": "63", "°¢À­ÉÆÃË": "230", "ÉòÑô": "58", "´óÁ¬": "167", "°°É½": "320", "¸§Ë³": "184", "±¾Ïª": "227", "µ¤¶«": "282", "½õÖÝ": "166", "Óª¿Ú": "281", "¸·ÐÂ": "59", "ÁÉÑô": "351", "Å̽õ": "228", "ÌúÁë": "60", "³¯Ñô": "280", "ºù«µº": "319", "³¤´º": "53", "¼ªÁÖÊÐ": "55", "ËÄÆ½": "56", "ÁÉÔ´": "183", "ͨ»¯": "165", "°×ɽ": "57", "ËÉÔ­": "52", "°×³Ç": "51", "Ñӱ߳¯ÏÊ×å×ÔÖÎÖÝ": "54", "¹þ¶û±õ": "48", "ÆëÆë¹þ¶û": "41", "¼¦Î÷": "46", "º×¸Ú": "43", "˫Ѽɽ": "45", "´óÇì": "50", "ÒÁ´º": "40", "¼Ñľ˹": "42", "Æß̨ºÓ": "47", "ĵµ¤½­": "49", "ºÚºÓ": "39", "Ë绯": "44", "´óÐ˰²ÁëµØÇø": "38", "ÉϺ£": "289", "ÄϾ©": "315", "ÎÞÎý": "317", "ÐìÖÝ": "316", "³£ÖÝ": "348", "ËÕÖÝ": "224", "ÄÏͨ": "161", "Á¬ÔƸÛ": "347", "»´°²": "162", "ÑγÇ": "223", "ÑïÖÝ": "346", "Õò½­": "160", "Ì©ÖÝ": "276", "ËÞǨ": "277", "º¼ÖÝ": "179", "Äþ²¨": "180", "ÎÂÖÝ": "178", "¼ÎÐË": "334", "ºþÖÝ": "294", "ÉÜÐË": "293", "½ð»ª": "333", "áéÖÝ": "243", "ÖÛɽ": "245", "̨ÖÝ": "244", "ÀöË®": "292", "ºÏ·Ê": "127", "Îߺþ": "129", "°ö²º": "126", "»´ÄÏ": "250", "Âí°°É½": "358", "»´±±": "253", "Í­Áê": "337", "°²Çì": "130", "»ÆÉ½": "252", "³üÖÝ": "189", "¸·Ñô": "128", "ËÞÖÝ": "370", "³²ºþ": "251", "Áù°²": "298", "ÙñÖÝ": "188", "³ØÖÝ": "299", "Ðû³Ç": "190", "¸£ÖÝ": "300", "ÏÃÃÅ": "194", "ÆÎÌï": "195", "ÈýÃ÷": "254", "ȪÖÝ": "134", "ÕÄÖÝ": "255", "ÄÏÆ½": "133", "ÁúÑÒ": "193", "ÄþµÂ": "192", "Äϲý": "163", "¾°µÂÕò": "225", "ƼÏç": "350", "¾Å½­": "349", "ÐÂÓà": "164", "ӥ̶": "279", "¸ÓÖÝ": "365", "¼ª°²": "318", "ÒË´º": "278", "¸§ÖÝ": "226", "ÉÏÈÄ": "364", "¼ÃÄÏ": "288", "Çൺ": "236", "×Ͳ©": "354", "Ôæ×¯": "172", "¶«Óª": "174", "ÑĮ̀": "326", "Ϋ·»": "287", "¼ÃÄþ": "286", "Ì©°²": "325", "Íþº£": "175", "ÈÕÕÕ": "173", "À³Îß": "124", "ÁÙÒÊ": "234", "µÂÖÝ": "372", "ÁijÇ": "366", "±õÖÝ": "235", "ºÊÔó": "353", "Ö£ÖÝ": "268", "¿ª·â": "210", "ÂåÑô": "153", "ƽ¶¥É½": "213", "°²Ñô": "267", "º×±Ú": "215", "ÐÂÏç": "152", "½¹×÷": "211", "å§Ñô": "209", "Ðí²ý": "155", "äðºÓ": "344", "ÈýÃÅÏ¿": "212", "ÄÏÑô": "309", "ÉÌÇð": "154", "ÐÅÑô": "214", "ÖÜ¿Ú": "308", "פÂíµê": "269", "Î人": "218", "»ÆÊ¯": "311", "Ê®Ñß": "216", "Ò˲ý": "270", "ÏåÑô": "156", "¶õÖÝ": "122", "¾£ÃÅ": "217", "Т¸Ð": "310", "¾£ÖÝ": "157", "»Æ¸Ô": "271", "ÏÌÄþ": "362", "ËæÖÝ": "371", "¶÷Ê©ÍÁ¼Ò×åÃç×å×ÔÖÎÖÝ": "373", "ÏÉÌÒ": "1713", "DZ½­": "1293", "ÌìÃÅ": "2654", "ÉñÅ©¼ÜÁÖÇø": "2734", "³¤É³": "158", "ÖêÖÞ": "222", "ÏæÌ¶": "313", "ºâÑô": "159", "ÉÛÑô": "273", "ÔÀÑô": "220", "³£µÂ": "219", "Õżҽç": "312", "ÒæÑô": "272", "³»ÖÝ": "275", "ÓÀÖÝ": "314", "»³»¯": "363", "¦µ×": "221", "ÏæÎ÷ÍÁ¼Ò×åÃç×å×ÔÖÎÖÝ": "274", "¹ãÖÝ": "257", "ÉØ¹Ø": "137", "ÉîÛÚ": "340", "Ö麣": "140", "ÉÇÍ·": "303", "·ðɽ": "138", "½­ÃÅ": "302", "Õ¿½­": "198", "ïÃû": "139", "ÕØÇì": "338", "»ÝÖÝ": "301", "÷ÖÝ": "141", "ÉÇβ": "339", "ºÓÔ´": "200", "Ñô½­": "199", "ÇåÔ¶": "197", "¶«Ý¸": "119", "ÖÐɽ": "187", "³±ÖÝ": "201", "½ÒÑô": "259", "ÔÆ¸¡": "258", "ÄÏÄþ": "261", "ÁøÖÝ": "305", "¹ðÁÖ": "142", "ÎàÖÝ": "304", "±±º£": "295", "·À³Ç¸Û": "204", "ÇÕÖÝ": "145", "¹ó¸Û": "341", "ÓñÁÖ": "361", "°ÙÉ«": "203", "ºØÖÝ": "260", "ºÓ³Ø": "143", "À´±ö": "202", "³ç×ó": "144", "º£¿Ú": "125", "ÈýÑÇ": "121", "Îåָɽ": "1644", "Çíº£": "2358", "ÙÙÖÝ": "1215", "ÎIJý": "2758", "ÍòÄþ": "1216", "¶«·½": "2634", "¶¨°²": "1214", "ÍͲý": "1641", "³ÎÂõ": "2757", "ÁÙ¸ß": "2033", "°×ɳÀè×å×ÔÖÎ": "2359", "²ý½­Àè×å×ÔÖÎ": "1642", "ÀÖ¶«Àè×å×ÔÖÎ": "2032", "ÁêË®Àè×å×ÔÖÎ": "1643", "±£Í¤Àè×åÃç×å×ÔÖÎ": "1217", "ÇíÖÐÀè×åÃç×å×ÔÖÎ": "2031", "ÖØÇì": "132", "³É¶¼": "75", "×Ô¹±": "78", "ÅÊÖ¦»¨": "81", "ãòÖÝ": "331", "µÂÑô": "74", "ÃàÑô": "240", "¹ãÔª": "329", "ËìÄþ": "330", "ÄÚ½­": "248", "ÀÖɽ": "79", "Äϳä": "291", "üɽ": "77", "Ò˱ö": "186", "¹ã°²": "241", "´ïÖÝ": "369", "ÑŰ²": "76", "°ÍÖÐ": "239", "×ÊÑô": "242", "°¢°Ó²Ø×åǼ×å×ÔÖÎÖÝ": "185", "¸Ê×βØ×å×ÔÖÎÖÝ": "73", "Á¹É½ÒÍ×å×ÔÖÎÖÝ": "80", "¹óÑô": "146", "ÁùÅÌË®": "147", "×ñÒå": "262", "°²Ë³": "263", "Í­ÈʵØÇø": "205", "Ç­Î÷Äϲ¼ÒÀ×åÃç×å×ÔÖÎÖÝ": "343", "±Ï½ÚµØÇø": "206", "Ç­¶«ÄÏÃç×å¶±×å×ÔÖÎÖÝ": "342", "Ç­Äϲ¼ÒÀ×åÃç×å×ÔÖÎÖÝ": "306", "À¥Ã÷": "104", "Çú¾¸": "249", "ÓñϪ": "106", "±£É½": "112", "ÕÑͨ": "336", "Àö½­": "114", "ÁÙ²×": "110", "³þÐÛÒÍ×å×ÔÖÎÖÝ": "105", "ºìºÓ¹þÄá×åÒÍ×å×ÔÖÎÖÝ": "107", "ÎÄɽ׳×åÃç×å×ÔÖÎÖÝ": "177", "ÆÕ¶ý": "108", "Î÷Ë«°æÄÉ´ö×å×ÔÖÎÖÝ": "109", "´óÀí°××å×ÔÖÎÖÝ": "111", "µÂºê´ö×å¾°ÆÄ×å×ÔÖÎÖÝ": "116", "Å­½­ÀüËÛ×å×ÔÖÎÖÝ": "113", "µÏÇì²Ø×å×ÔÖÎÖÝ": "115", "À­Èø": "100", "²ý¶¼µØÇø": "99", "ɽÄϵØÇø": "97", "ÈÕ¿¦ÔòµØÇø": "102", "ÄÇÇúµØÇø": "101", "°¢ÀïµØÇø": "103", "ÁÖÖ¥µØÇø": "98", "Î÷°²": "233", "Í­´¨": "232", "±¦¼¦": "171", "ÏÌÑô": "323", "μÄÏ": "170", "ÑÓ°²": "284", "ººÖÐ": "352", "ÓÜÁÖ": "231", "°²¿µ": "324", "ÉÌÂå": "285", "À¼ÖÝ": "36", "¼ÎÓø¹Ø": "33", "½ð²ý": "34", "°×Òø": "35", "ÌìË®": "196", "ÎäÍþ": "118", "ÕÅÒ´": "117", "ƽÁ¹": "359", "¾ÆÈª": "37", "ÇìÑô": "135", "¶¨Î÷": "136", "¤ÄÏ": "256", "ÁÙÏÄ»Ø×å×ÔÖÎÖÝ": "182", "¸ÊÄϲØ×å×ÔÖÎÖÝ": "247", "Î÷Äþ": "66", "º£¶«µØÇø": "69", "º£±±²Ø×å×ÔÖÎÖÝ": "67", "»ÆÄϲØ×å×ÔÖÎÖÝ": "70", "º£ÄϲØ×å×ÔÖÎÖÝ": "68", "¹ûÂ岨×å×ÔÖÎÖÝ": "72", "ÓñÊ÷²Ø×å×ÔÖÎÖÝ": "71", "º£Î÷ÃɹÅ×岨×å×ÔÖÎÖÝ": "65", "Òø´¨": "360", "ʯ×ìɽ": "335", "ÎâÖÒ": "322", "¹ÌÔ­": "246", "ÖÐÎÀ": "181", "ÎÚ³ľÆë": "92", "¿ËÀ­ÂêÒÀ": "95", "ͳ·¬µØÇø": "89", "¹þÃܵØÇø": "91", "²ý¼ª»Ø×å×ÔÖÎÖÝ": "93", "²©¶ûËþÀ­ÃɹÅ×ÔÖÎÖÝ": "88", "°ÍÒô¹ùÀãÃɹÅ×ÔÖÎÖÝ": "86", "°¢¿ËËÕµØÇø": "85", "¿Ë×ÎÀÕËտ¶û¿Ë×Î×ÔÖÎÖÝ": "84", "¿¦Ê²µØÇø": "83", "ºÍÌïµØÇø": "82", "ÒÁÀç": "90", "Ëþ³ÇµØÇø": "94", "°¢ÀÕÌ©µØÇø": "96", "ʯºÓ×Ó": "770", "°¢À­¶û": "731", "Í¼Ä¾Êæ¿Ë": "792", "Îå¼ÒÇþ": "789", "Ïã¸ÛÌØ±ðÐÐÕþÇø": "2912", "°ÄÃÅÌØ±ðÐÐÕþÇø": "2911"] + DBCollection QCC_COLL = MongoUtils.getCollectionByName("crawl", "tyc_web_crawl", "192.168.0.206",27020) + def filters = [] static main(args) { def task = new YinLianMeiShi(); - task.run() +// task.run() +// task.matchFlag() +// task.marge() + task.addValueByBaiduId() + } + + def addValueByBaiduId(){ + ExcelUtils.modify("D:\\Êý¾Ý\\ÒøÁª\\Êý¾Ý500Ìõ(12-21).xlsx",{d-> + if(d['A'] && d['T']=="°Ù¶ÈÈ˹¤"){ + def jsonObj = BaiDuApiUtils.getDetailByUid(d['U']) + if(jsonObj && jsonObj.result){ + def result = jsonObj.result + if(result){ + d['J'] = result.uid + d['K'] = RegionData.formatRegion(result.city, result.area) + d['L'] = result.name + d['M'] = result.address + d['N'] = result.detail_info.tag + d['O'] = result.location.lng + d['P'] = result.location.lat + println result + } + } + } + }) + } + def marge(){ + def file1 = "D:\\Êý¾Ý\\ÒøÁª\\ÒøÁªÈ˹¤1221(ÁéÖ¥).xlsx" + def datas1 = ExcelUtils.read(file1, 0, 1) + ExcelUtils.modify("D:\\Êý¾Ý\\ÒøÁª\\Êý¾Ý500Ìõ(12-21).xlsx",{d-> + if(d['A']){ + for(def r : datas1){ + if(r['A'] == d['A']){ + List pois = GaoDeUtils.getAmapById2(r['F']); + if(pois && pois.size()==1){ + def amapPoi = pois[0] + d['J'] = amapPoi.id + d['K'] = amapPoi.adname + d['L'] = amapPoi.name + d['M'] = amapPoi.address + d['N'] = amapPoi.type + def location = amapPoi.location + if(location){ + def arrs = location.split(",") + // ¸ßµÂת°Ù¶È + def baidu = BaiDuApiUtils.geoconv(arrs[0].toString().toDouble(), arrs[1].toString().toDouble(), 3) + def lng2 = baidu.x + def lat2 = baidu.y + d['O'] = lng2 + d['P'] = lat2 + } + } + println pois + } + } + } + }) } def run(){ - ExcelUtils.modify("D:\\Êý¾Ý\\ÒøÁª\\Êý¾Ý500Ìõ-¹Ø¼ü×Ö.xlsx",{row-> + ExcelUtils.modify("D:\\Êý¾Ý\\ÒøÁª\\Êý¾Ý500Ìõ£¨ºÏ²¢È˹¤).xlsx",{row-> def score = row['Q'] - if(row['E'] && (!score || (score && score.toString().toDouble()<0.8))){ + if(row['B'] && !row['L']){ + def baiduCode = getBaiduMapCode(row['E']); def city = RegionData.formatCity(row['E']); + if(!baiduCode){ + baiduCode = getBaiduMapCode(city); + } - def baiduCode = getBaiduMapCode(city); def newKeys = filterName(row['D'], row['E'], row['F'], row['B']) row['I'] = newKeys if(baiduCode){ - def rets = searchBaiDuMap(row['B'], baiduCode.toInteger()); - def simpleAddr = getSimpleAddress(city, row['C']) + def map = getSimpleAddress(city, row['C']) + def simpleAddr = map.addr + if(map.city!="ÊÐÏ½Çø" && map.city!=city){ + city = RegionData.formatCity(map.city); + } + def rets = getBiduSearchDatas(baiduCode , simpleAddr, row['B'], null) + rets = getBiduSearchDatas(baiduCode , null, row['B'], rets) + rets = getBiduSearchDatas(baiduCode , null, simpleAddr, rets) def bdRes = searchMapBySimilar(city, row['F'], row['B'], simpleAddr, rets, newKeys); - if (bdRes && (!score || (score && bdRes.score>score.toString().toDouble()))) { + if (bdRes) { row['J'] = bdRes.uid row['K'] = bdRes.region row['L'] = bdRes.name @@ -56,29 +130,155 @@ class YinLianMeiShi { }) } + def matchFlag(){ + ExcelUtils.multiThreadingModify("D:\\Êý¾Ý\\ÒøÁª\\Êý¾Ý500Ìõ£¨ºÏ²¢È˹¤).xlsx",{row-> + if(row['B'] && row['B'].contains("Âó¼ÓÃÀ¶­´å²ÍÌü")){ + def datas = findQccData(row['B']) + if(datas){ + def max_score = 0 + def last_obj = null + def newKeys = filterName(row['D'], row['E'], row['F'], row['B']) + def city = RegionData.formatCity(row['E']); + for(def d : datas){ + + def map = getSimpleAddress(null, d.address) + def inputMap = getSimpleAddress(null, row['C']) + if(inputMap && inputMap.city){ + city = inputMap.city + } + if(map && city==map.city){ + def qccNameKey = filterName(map.pro, map.city, map.region, d.name) + def name_score = CosineSimilarAlgorithm.getSimilarityRatio(newKeys, qccNameKey); + def addr_score = 0 + if (map.addr && inputMap.addr) { + if(map.addr.contains("Óë")){ + def arrs = map.addr.split("Óë") + for(def s1 : arrs){ + def temp_addr_score = CosineSimilarAlgorithm.getSimilarityRatio(inputMap.addr, s1); + if(temp_addr_score > addr_score){ + addr_score = temp_addr_score + } + } + }else{ + addr_score = CosineSimilarAlgorithm.getSimilarityRatio(inputMap.addr, map.addr); + } + } + + def total_score = name_score + addr_score + if(total_score > max_score){ + max_score = total_score; + last_obj = d + last_obj.simple_addr = map.addr + last_obj.tyc_city = map.city + } + println row['B'] +" "+ d.name+" "+total_score + }else{ + def qccNameKey = filterName(map.pro, map.city, map.region, d.name) + def name_score = CosineSimilarAlgorithm.getSimilarityRatio(newKeys, qccNameKey); + if(map.addr && inputMap.addr && (inputMap.addr.contains(map.addr) || map.addr.contains(inputMap.addr))){ + max_score = name_score; + last_obj = d + last_obj.simple_addr = map.addr + last_obj.tyc_city = map.city + println row['B'] +" "+ d.name+" "+name_score + } + } + } + if(max_score && max_score > 0.4){ + row['AB'] = last_obj.name + row['AC'] = last_obj.address + row['AD'] = last_obj.flag + row['AE'] = max_score + } + } + } + }); + } + + def findQccData(keywords){ + def q = new BasicDBObject() +// q.put("search_terms", keywords) + if(keywords.contains("(*")){ + keywords = keywords.toString().substring(0, keywords.toString().indexOf("(*")) + } + q.put("search_terms", Pattern.compile("^.*${keywords}.*\$", Pattern.CASE_INSENSITIVE)) + def dbcursor = QCC_COLL.find(q) + def rets = [] + for(def d : dbcursor){ + rets << ["name":d.name, "address":d.taxAddress, "flag":d.regStatus] + } + return rets; + } + def getBiduSearchDatas(baiduCode , simpleAddr, nameKey, rets){ + def searchKey = "" + if(simpleAddr){ + searchKey = simpleAddr +" "+nameKey + }else{ + searchKey = nameKey + } + def datas = searchBaiDuMap(searchKey, baiduCode.toInteger()); + if(!rets){ + return datas; + }else{ + def ids = [] + for(def r : rets){ + if(!ids.contains(r.uid)){ + ids << r.uid + } + } + for(def d : datas){ + if(!ids.contains(d.uid)){ + rets << d + } + } + return rets; + } + return null; + } + def getSimpleAddress(city, address){ def jsonObj = FangjiaClientUtils.addressStructure(city, address) - def old_addr = null + def old_addr = "" + def map = [:] if(jsonObj && jsonObj.result){ + + if(jsonObj.result.province){ + map.pro = jsonObj.result.province + } + + if(jsonObj.result.city){ + if(jsonObj.result.city=="ÊÐÏ½Çø"){ + map.city = jsonObj.result.province + }else{ + map.city = jsonObj.result.city + } + map.city = RegionData.formatCity(map.city) + } + + if(jsonObj.result.district){ + map.region = jsonObj.result.district + } if(jsonObj.result.street){ def street = jsonObj.result.street if(street){ old_addr = street + map.street = street.replace("½»²æ¿Ú","").replace("½»²æ","") + } + } + if(jsonObj.result.streetNumber){ + def street_number = jsonObj.result.streetNumber + if(street_number){ + old_addr = old_addr + street_number } } -// if(jsonObj.result.streetNumber){ -// def street_number = jsonObj.result.streetNumber -// if(street_number){ -// old_addr = old_addr + street_number -// } -// } } - return old_addr; + map.addr = old_addr.replace("½»²æ¿Ú","").replace("½»²æ","") + return map; } def filterName(pro,city, region, name){ - def newKey = name.replace("ÓÐÏÞÔðÈι«Ë¾", "").replace("ÓÐÏÞ¹«Ë¾","") + def newKey = name.replace("²ÍÒû·þÎñÓÐÏÞ¹«Ë¾","").replace("¿Ø¹ÉÓÐÏÞ¹«Ë¾","").replace("ÓÐÏÞÔðÈι«Ë¾", "").replace("ÓÐÏÞ¹«Ë¾","").replace("²ÍÒû","") if(pro && name.contains(pro)){ newKey = newKey.replace(pro, "") } @@ -88,17 +288,57 @@ class YinLianMeiShi { newKey = newKey.replace(city, "").replace(newCity, "") } - - if(region && newKey.contains(region)){ + if(region && newKey.contains(region) ){ newKey = newKey.replace(region, "") } - if(newKey.contains("(")){ - def subName = newKey.substring(newKey.indexOf("(")+1, newKey.indexOf(")")); - if(subName.endsWith("½Ö") || subName.endsWith("·") || subName.contains("*")){ - newKey = newKey.replace("("+subName+")", ""); - } - newKey = newKey.replace("(","").replace(")","") + + def newRegion = null + if(region){ + newRegion = RegionData.formatRegion(city, region) } + if(newRegion && newKey.contains(newRegion)){ + newKey = newKey.replace(newRegion, "") + } + + if(newKey.contains("(") || newKey.contains("£¨")){ + if(newKey.contains("(")){ + def subName = newKey.substring(newKey.indexOf("(")+1, newKey.indexOf(")")); + if(subName.endsWith("½Ö") || subName.endsWith("·") || subName.contains("*") || subName.endsWith("µê")){ + newKey = newKey.replace("("+subName+")", ""); + } + } + newKey = newKey.replace("(","").replace(")","").replace("£¨","").replace("£©","") + if(newKey.toString().endsWith("µê")){ + newKey = newKey.substring(0, newKey.size()-1) + } + } + return newKey; + } + + def filterName2(pro,city, region, name){ + def newKey = name.replace("²ÍÒû·þÎñÓÐÏÞ¹«Ë¾","").replace("¿Ø¹ÉÓÐÏÞ¹«Ë¾","").replace("ÓÐÏÞÔðÈι«Ë¾", "").replace("ÓÐÏÞ¹«Ë¾","").replace("²ÍÒû","") + if(pro && name.contains(pro)){ + newKey = newKey.replace(pro, "") + } + + def newCity = RegionData.formatCity(city) + if(city && (name.contains(city) || name.contains(newCity))){ + newKey = newKey.replace(city, "").replace(newCity, "") + } + + if(region && newKey.contains(region) ){ + newKey = newKey.replace(region, "") + } + + def newRegion = null + if(region){ + newRegion = RegionData.formatRegion(city, region) + } + if(newRegion && newKey.contains(newRegion)){ + newKey = newKey.replace(newRegion, "") + } + + newKey = newKey.replace("(","").replace(")","").replace("£¨","").replace("£©","") return newKey; } @@ -113,7 +353,8 @@ class YinLianMeiShi { def rets = [] try { Thread.sleep(500) - String url = "https://map.baidu.com/?newmap=1&reqflag=pcmap&biz=1&from=webmap&da_par=baidu&pcevaname=pc4.1&qt=s&da_src=searchBox.button&wd="+URLEncoder.encode(address,"UTF-8")+"&c=${cityCode}&src=0&wd2=&pn=0&sug=0&l=11&from=webmap&sug_forward=&device_ratio=1&tn=B_NORMAL_MAP&nn=0&u_loc=13509900,3640104&ie=utf-8&t=${System.currentTimeMillis()}&auth=AUeHbGFG3ZMg2Bv8Y2b8GfVB3OGc35VOuxLNVzzzLHHtSAFSAS7I7vICu2I%400GdFvkGcuVtvvhguVtvyheuVtvCMGuVtvCQMuVtvIPcuxtw8wkvyuw904v7uv%40vcuVtvc3CuVtvcPPuVtveGvuNzHtAimNzCywiKDv7uvhgMuzVVtcvY1SGpuxRtKHZz3%40SdJMjjejCEBf0wd0vyMO7Ay7MOI&seckey=Im7p%2FanwNmQx078ijew%2Fsx5QUzMQU8M7SbrqWr3JGEc%3D%2C0-Ulo82T3hd3wuKMf_tTM6DbOSuS4Zlg9ZEtPAqKkNKXysGCBLK7zryshf7oP6p8_Hynb5vDXmd6C0TtcF9impZZK2kzkn1_SWeBVEJH4Zdh0n81wXv-1j6enHaAwiYj7T7eCaCw3g5vDcHvZxUxtBsRphpZI-Rco19dgpXwT9xb-XVcCb8GyvbMt5s2wIpb"; + // String url = "https://map.baidu.com/?newmap=1&reqflag=pcmap&biz=1&from=webmap&da_par=baidu&pcevaname=pc4.1&qt=s&da_src=searchBox.button&wd="+URLEncoder.encode(address,"UTF-8")+"&c=${cityCode}&src=0&wd2=&pn=0&sug=0&l=11&from=webmap&sug_forward=&device_ratio=1&tn=B_NORMAL_MAP&nn=0&u_loc=13509900,3640104&ie=utf-8&t=${System.currentTimeMillis()}&&auth=Dae1RaNR1LK8xdJTxN8SQOTRa2AH%3DL7vuxLNVENTxRBt6BG6B6WJWwGvzBU%40yYxvkGcuVtvvhguVtvyheuVtcvY1SGpuLtB2QKAZzvCQMuVtvIPcuxtw8wkv7uvZgMuVtv%40vcuVtvc3CuVtvcPPuztxgHxwBVDzvFosSE21wiKDv7uvhgMuzVVtvrMhuENLtSrrlrIKHf0wd0vyMO7FOUySy&seckey=Im7p%2FanwNmQx078ijew%2FsyQrDC7YJnynqrZ43JVFlM0%3D%2C0-Ulo82T3hd3wuKMf_tTM-ZXYa3S1Y21RX3DV_l5MHgI_1U55ZQrzxTIKi3BRb2x59mVOurDjnW_8CwMEmjK3y8JfHyV0kv70GMQXxnIEcHpM3Dqhs5EdrN6ciDtoxyrcRfowp_xTBkEgU49ZrCz2KeLyujCOypqjKS3rAmPOfEp1kIZPD4W6RDQzrsWTe7a"; + String url = "https://map.baidu.com/?newmap=1&reqflag=pcmap&biz=1&from=webmap&da_par=after_baidu&pcevaname=pc4.1&qt=s&da_src=searchBox.button&wd="+URLEncoder.encode(address,"UTF-8")+"&c=${cityCode}&src=0&wd2=&pn=0&sug=0&l=19&from=webmap&sug_forward=&auth=E8LCawOwRbEOx6JVTvCIBI6W7zSz5vSXuxLNVENTRVLtUDHUDUYKYyHxzBU%40yYxvkGcuVtvvhguVtvyheuVtvCMGuVtvCQMuVtvIPcuxtw8wkv7uvZgMuVtv%40vcuVtvc3CuVtvcPPuztxgHxwDWBWvveh3uxtwiKDv7uvhgMuzVVtcvY1SGpuxRtKHZz3VP8JSrrlrIKHf0wd0vyMO7FOUS7F&seckey=Im7p%2FanwNmQx078ijew%2FsyQrDC7YJnynqrZ43JVFlM0%3D%2C0-Ulo82T3hd3wuKMf_tTM-ZXYa3S1Y21RX3DV_l5MHgI_1U55ZQrzxTIKi3BRb2x59mVOurDjnW_8CwMEmjK3y8JfHyV0kv70GMQXxnIEcHpM3Dqhs5EdrN6ciDtoxyrcRfowp_xTBkEgU49ZrCz2KeLyujCOypqjKS3rAmPOfEp1kIZPD4W6RDQzrsWTe7a&device_ratio=2&tn=B_NORMAL_MAP&nn=0&u_loc=13513133,3631861&ie=utf-8&t=${System.currentTimeMillis()}&newfrom=zhuzhan_webmap" def content = new HttpRequest().get(url, null, "utf-8") JSONObject json = JSONObject.fromObject(content) if (json && json.content && json.content.size() > 0) { @@ -143,14 +384,21 @@ class YinLianMeiShi { bean.longitude = lng bean.latitude = lat bean.name = loc.name - bean.address = getAddressInfo(loc.address_norm, loc.addr); + if(loc.address_norm){ + bean.address = getAddressInfo(loc.address_norm, loc.addr); + } bean.old_addr = loc.addr -// def baiduCity = RegionData.formatCity(loc.city_name); + if(!baiduCity){ + baiduCity = RegionData.formatCity(loc.city_name) + } bean.cityname = baiduCity - def regionStr = loc.area_name.replace(loc.city_name, "") - def baiduRegion = RegionData.formatRegion(baiduCity, regionStr); - bean.region = baiduRegion + if(loc.city_name){ + def regionStr = loc.area_name.replace(loc.city_name, "") + def baiduRegion = RegionData.formatRegion(baiduCity, regionStr); + bean.region = baiduRegion + } + if(loc.std_tag){ bean.type = loc.std_tag } @@ -161,6 +409,7 @@ class YinLianMeiShi { } } } catch(Exception e) { + searchBaiDuMap(address, cityCode) } return rets; } @@ -226,10 +475,6 @@ class YinLianMeiShi { def maxScore = 0; if(!bo || !maxScore){ for(def o :rets){ - - if(!region.contains(o.region)){ - continue; - } def s = o.name // if(o.name && o.name.contains("(") && !o.name.toString().startsWith("(")){ // s = o.name.substring(0, o.name.indexOf("(")) @@ -237,29 +482,76 @@ class YinLianMeiShi { // s = formatName(s) def addr = o.address def mapKey = filterName(null, o.cityname, o.region, o.name) + def mapKey1 = filterName2(null, o.cityname, o.region, o.name) if(mapKey){ def score = CosineSimilarAlgorithm.getSimilarityRatio(mapKey, newKey); - if(city==o.cityname){ - if(score > maxScore){ - maxScore = score; - o.matchType = "³ÇÊÐÏàͬ£¬Ãû³ÆÏàËÆ" - o.city = o.cityname - bo = o - bo.score = maxScore + def score1 = CosineSimilarAlgorithm.getSimilarityRatio(mapKey1, newKey); + if(score1>score){ + score = score1 + } + def addr_score = 0 + if(inpAddr) { + def baidu_addr_map = getSimpleAddress(o.cityname, o.old_addr) + if(!baidu_addr_map.addr){ + baidu_addr_map.addr = AddressUtil.formatAddress(o.cityname, o.old_addr) } - if(bo && score!=1){ - if(inpAddr){ - def baidu_addr = getSimpleAddress(o.cityname, o.old_addr) - if(baidu_addr){ - def addr_score = CosineSimilarAlgorithm.getSimilarityRatio(inpAddr, baidu_addr); - if(addr_score==1){ - o.matchType = "³ÇÊÐÏàͬ£¬Ãû³ÆÏàËÆ£¬µØÖ·ÓйØ" - bo.score = bo.score + 0.1 + if(!baidu_addr_map.addr){ + if(o.region && !region.contains(o.region)){ + continue; + } + if(city!=o.cityname){ + continue; + } + } + if (baidu_addr_map.addr) { + if(baidu_addr_map.addr.contains("Óë")){ + def arrs = baidu_addr_map.addr.split("Óë") + for(def s1 : arrs){ + def temp_addr_score = CosineSimilarAlgorithm.getSimilarityRatio(inpAddr, s1); + if(temp_addr_score > addr_score && (inpAddr.contains(baidu_addr_map.addr) || baidu_addr_map.addr.contains(inpAddr))){ + addr_score = temp_addr_score } } + }else{ + addr_score = CosineSimilarAlgorithm.getSimilarityRatio(inpAddr, baidu_addr_map.addr); + if(addr_score>0 && inpAddr && baidu_addr_map.street && !inpAddr.contains(baidu_addr_map.street) && !baidu_addr_map.street.contains(inpAddr)){ + addr_score = 0 + } } + } + } + + if(addr_score>0 && score==0){ + addr_score = addr_score/2 + } + def total_score = score + addr_score +// if(city==o.cityname){ + if(total_score > maxScore){ + maxScore = total_score; + o.matchType = "³ÇÊÐÏàͬ£¬Ãû³ÆÏàËÆ" + o.city = o.cityname + bo = o + bo.score = maxScore + + if(addr_score>0.6){ + o.matchType = "³ÇÊÐÏàͬ£¬Ãû³ÆÏàËÆ£¬µØÖ·ÓйØ" + } + } +// if(bo && score!=1){ +// if(inpAddr){ +// def baidu_addr = getSimpleAddress(o.cityname, o.old_addr) +// if(baidu_addr){ +// def addr_score = CosineSimilarAlgorithm.getSimilarityRatio(inpAddr, baidu_addr); +// //if(addr_score==1){ +// // o.matchType = "³ÇÊÐÏàͬ£¬Ãû³ÆÏàËÆ£¬µØÖ·ÓйØ" +// bo.score = bo.score + addr_score +// maxScore = bo.score; +// //} +// } +// } + // for(def word : filters){ // if(mapKeyName.toString().endsWith(word)){ // mapKeyName = mapKeyName.replace(word, "") @@ -292,8 +584,8 @@ class YinLianMeiShi { // } // } // } - } - } +// } +// } } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy index 6d692218b..4f5efcea8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy @@ -52,7 +52,7 @@ class YouDaDataParser { static main(args) { def task = new YouDaDataParser(); // task.newHouseparse();//ÓÑ´ïÒ»ÊÖ·¿´æÈëmongo -// task.esfParser("D:\\Êý¾Ý\\ÓÑ´ï\\2022Äê11ÔÂ\\²éѯ1.DBF");//ÓÑ´ï¶þÊÖ·¿´æÈëmongo +// task.esfParser("D:\\Êý¾Ý\\ÓÑ´ï\\2022Äê12ÔÂ\\²éѯ1.DBF");//ÓÑ´ï¶þÊÖ·¿´æÈëmongo // task.newhouseToSaveEs();//з¿±£´æµ½es task.esfhouseToSaveEs();//¶þÊÖ·¿±£´æµ½es // task.updateFinishDateByEsfTrade();//¸ù¾Ý¶þÊֳɽ»µÄ¿¢¹¤Ê±¼äÐÞ¸Äseaweed¿âÖеĿ¢¹¤Ê±¼ä diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy index 0ecdddeb1..7f99588f5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -474,7 +474,7 @@ public class LianjiaCrawlByChrome { if(cityEles){ for(def e : cityEles) { def href = e.attr("href") - if (!href.contains("fang.lianjia")){ + if (!href.contains("fang.lianjia") ){ cityMap.put(e.text(), href) } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy index 17b02bb6b..2a34dd4b3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -247,6 +247,35 @@ class GaoDeUtils { return Collections.emptyList() } + + static List getAmapById2(id){ + def params = [:] + params.id = id + params.output = 'json' + params.key = "66e6771767ab293ace441553c36621c1" + JSONObject json = HttpRequestUtil.getJsonContent("https://restapi.amap.com/v5/place/detail", params); + if (json == null) { + json = HttpRequestUtil.getJsonContent("https://restapi.amap.com/v5/place/detail", params); + } + if (json && json.info && json.info == "OK") { + try { + List ret = JSONObject.parseArray(json.pois.toString(), AmapPoi.class) + return ret + } catch(Exception ex) { + ex.printStackTrace() + } + } else if (json && json.infocode && json.infocode == "10044") { + System.exit(1) + } else if (json && json.infocode && json.infocode == "10004") { + // ¿ª·¢Õߵĵ¥Î»Ê±¼äÄÚ£¨1·ÖÖÓ£©·ÃÎÊÁ¿³¬ÏÞ£¬±»ÏµÍ³×Ô¶¯·âÍ££¬ÏÂÒ»·ÖÖÓ×Ô¶¯½â·â¡£ + Thread.sleep(60000) + } else if (json && json.infocode && json.infocode == "10017") { + // ËùÇëÇóµÄ×ÊÔ´²»¿ÉÓà + System.exit(1) + } + return Collections.emptyList() + } + /** * »ñÈ¡ÏêÇé */ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy index 78d3fceac..0ceed1225 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy @@ -40,11 +40,11 @@ class DataStat { def run(){ - def source = "Á´¼Ò"// - def citys = findCrawlCity(source, 202209); + def source = "365ÌÔ·¿"// + def citys = findCrawlCity(source, 202210); for(def city : citys){ - def count1 = findCount(source, city, 202209); - def count2 = findCount(source, city, 202210); + def count1 = findCount(source, city, 202210); + def count2 = findCount(source, city, 202211); println city +" "+count1+" "+count2 // println count1 // println "--- ----" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/MongoUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/MongoUtils.groovy index 34186b830..ec6fa0a0b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/MongoUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/MongoUtils.groovy @@ -65,6 +65,8 @@ class MongoUtils { if (domainUrl == MongoDB_Domain_mongo) { port == 28017; } + + if ((domainUrl == "192.168.0.136") && (port == 27017 || port == 28017)) { return com.fangjia.core.util.MongoUtils.getDbCollection(Constants.TEST_FANGJIA_MONGO_URI, db, name) } From 7a81af678622dcedf19892b6a18c6e3e82b8073c Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 27 Dec 2022 09:14:39 +0800 Subject: [PATCH 33/99] =?UTF-8?q?94-=E3=80=8B236?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ccbevaluation/CcbFtpTestUtils.groovy | 2 +- .../DealAndThirdPriceDataProcess.groovy | 43 +++++++++++++++++-- .../ccbevaluation/ExportOracleDataTest.groovy | 2 +- .../ccbevaluation/GuiDingPriceBatch.groovy | 2 +- .../fangjia/script/ccbevaluation/Test.groovy | 33 ++++++++------ .../ronghe/ExportDistrictRongHe.groovy | 2 +- 6 files changed, 64 insertions(+), 20 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpTestUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpTestUtils.groovy index 81958589e..00ed6bf6e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpTestUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpTestUtils.groovy @@ -7,7 +7,7 @@ import org.apache.commons.codec.digest.DigestUtils class CcbFtpTestUtils { - static def ftpServer = "192.168.0.86"//45 + static def ftpServer = "192.168.0.45"//45 static def ftpPort = 17495 static def ftpUser = "ccb" static def ftpPassword = "ccb!@#" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy index e67f2a8cc..6b28570f1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy @@ -25,8 +25,9 @@ class DealPriceDataProcess { def task = new DealPriceDataProcess() // task.process()//Ô˵±³É½»peÖµ²»ºÃʱÐУ¬À­Éý³É½»¼Û¸ñ¼°É¾³ýpeÌ«´óÊý¾Ý // task.addLianJiaPrice(); - task.deleteThirdPartyData() +// task.deleteThirdPartyData() // task.delDealByPe() + task.delDealData() } def delDealByPe(){ @@ -370,20 +371,33 @@ class DealPriceDataProcess { def deleteThirdPartyData(){ - def citys = CcbCommonUtils.citys_92 - def month = 202210 + def citys = CcbCommonUtils.citys_all + def month = 202211 def rows = [] def cityCodeDatas = CcbCommonUtils.findCityCodeValue() + def set = new HashSet() for(def city : citys) { - def rets = findPeGteThirdParty(city, 202211) + def rets = findPeGteThirdParty(city, 202212) def total = 0 println city +" "+rets.size() + if(rets){ + set.add(city) + } for(def m : rets){ def cityId = cityCodeDatas.get(city) def sql = "delete from BIGDATA_THIRD_PARTY_PRICE_GJ where cityid='${cityId}' and commid='${m.commid}' and price_date=${month} and community_price=${m.price}" sqlInstance236.executeUpdate(sql) } } + println set.toString() + } + + def findThirdCount(sqlInstance236, sql){ + def count = 0 + sqlInstance236.eachRow(sql,{r-> + count = r.count() + }); + return count } def findPeGteThirdParty(cityName, yearmonth){ @@ -398,4 +412,25 @@ class DealPriceDataProcess { return rets } + def delDealData(){ + ExcelUtils.modify("D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\³É½»\\º¼Öݳɽ»É¾³ý12-23.xlsx", { d -> + if(d['A']){ + def count = delDealDataByid(d['A'],d['K'].toString().toDouble(), d['H'].toString().toDouble().toInteger()) + println d['B']+" "+count + } + }) + } + + def delDealDataByid(commid, area, price){ + def sql = "select count(*) as count from bigdata_deal_gj where commid='${commid}' and buildarea=${area} and DEALUNITPRICE=${price} " + + "and dealdate>=TO_DATE('2022-11-01','yyyy-MM-dd')" +// sqlInstance236.executeUpdate(sql) + def count = 0 + sqlInstance236.eachRow(sql,{r-> + count = r.count + }); + return count + } + + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleDataTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleDataTest.groovy index a368aeb87..c4d5579f4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleDataTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleDataTest.groovy @@ -14,7 +14,7 @@ import java.sql.ResultSet class ExportOracleDataTest { - def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl", + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") def cityCodeMap = CcbCommonUtils.findCityCodeValue() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index 0882eba51..537e09d1e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -47,7 +47,7 @@ class GuiDingPriceBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) //Ìí¼Ó×Ô¼º¿âÖеÄÈ˹¤Ö¸µ¼¼Û - def prices = CcbCommonUtils.findPersonPrice(city, "2022-11-25", "2022-12-19") + def prices = CcbCommonUtils.findPersonPrice(city, "2022-11-25", "2022-12-24") // def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22") if(prices){ def distMap = getDistrictMap(city) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index b7afc9818..14a238118 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -1087,6 +1087,15 @@ def findThirdCommCount(cityid, sqlInstance){ //} +def fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Ö¸µ¼¼Û\\¹úÐÅ´ï¼Û¸ñ-º¼ÖÝ.xlsx" +ExcelUtils.multiThreadingModify(fileName, { d -> + def dist = CommonDatautils.findDistrictById(d['A']) + if(dist){ + d['C'] = dist.city + d['D'] = dist.region + d['E'] = dist.name + } +}) //def fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Ö¸µ¼¼Û\\0922Èë¿âÊý¾Ý.xlsx" //def datas = ExcelUtils.read(fileName, 0, 1) @@ -1149,18 +1158,18 @@ def findThirdCommCount(cityid, sqlInstance){ -ExcelUtils.modify("D:\\Êý¾Ý\\½¨ÐÐ\\¶þÆÚ\\¸¶.xlsx",{d-> - def obj = CommonDatautils.findDistrict(d['D'],d['L'], d['K'], "district") - if(obj){ - def count = findGuidedPrice(sqlInstance236, obj._id.toString()) - if(count){ - d['M'] = "´æÔÚ" - } - d['N'] = obj.name_cat - d['O'] = obj.estate_type2 - } - -}) +//ExcelUtils.modify("D:\\Êý¾Ý\\½¨ÐÐ\\¶þÆÚ\\¸¶.xlsx",{d-> +// def obj = CommonDatautils.findDistrict(d['D'],d['L'], d['K'], "district") +// if(obj){ +// def count = findGuidedPrice(sqlInstance236, obj._id.toString()) +// if(count){ +// d['M'] = "´æÔÚ" +// } +// d['N'] = obj.name_cat +// d['O'] = obj.estate_type2 +// } +// +//}) def findGuidedPrice(sqlInstance, id){ def count = 0 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy index a16f783fb..2a16ad76e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy @@ -530,7 +530,7 @@ class ExportDistrictRongHe extends BaseBatch{ for(def city : citys) { def cityCode = cityCodeMap.get(city) - if(curDate.toString().endsWith("28")){ + if(curDate.toString().endsWith("10")){ if(new_dists){ def newstr = "('"+new_dists.join("','")+"')" def sql1 = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ " + From 36464674a1a47c9fbe2259475331090ab71b0208 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 27 Dec 2022 10:04:33 +0800 Subject: [PATCH 34/99] update --- .../ccbevaluation/ExportOracleDataTest.groovy | 64 +-- .../ExportOracleEvaluationData.groovy | 439 +++++++++--------- 2 files changed, 263 insertions(+), 240 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleDataTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleDataTest.groovy index c4d5579f4..412ec7c3a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleDataTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleDataTest.groovy @@ -32,37 +32,37 @@ class ExportOracleDataTest { def lastMonthStr = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM") def new_dists = [] //²»»á±ä 10.25 - exportEvaluationRateData() - exportBigdataPoiData()// - exportBigdataDistrictData()// - - exportGuidedPrice()//ËæÈ˹¤¸Ä±ä// - exportCommunityGuidedPrice(lastMonth.toString()+"28")//ËæÕþ²ß// - - //´¦ÀíÐ¡ÇøÇé¿ö - exportBigdataCommunity()// - exportCommunityStaging()//ûcityid - exportCommunityBuileType()// - - exportThirdPartyPrice2(lastMonth, new_dists)// - exportBigdataForsale(lastMonthStr +"-19", new_dists) //2022-07-19 - exportBigdataDeal(lastMonthStr + "-01", new_dists) //2022-07-01 - - //µÚÈý·½ÍøÕ¾È¨Öرí - exportWebsiteRate(); - - //¼ÆËã½á¹ûÏà¹Ø - exportAvmEvaluationGJNB(curMonth)//ÎÞ³ÇÊÐÇø·Ö - exportAVMEvaluationItemGJNB(curMonth)//ÎÞ³ÇÊÐÇø·Ö - - exportCityPrice(curMonth)//ÎÞ³ÇÊÐÇø·Ö - exportRegionPrice(curMonth)//ÎÞ³ÇÊÐÇø·Ö - - exportCommunityBasePriceGj(curMonth)//ÒѺÏÏÂÃæµÄºÏ²¢ //ÎÞ³ÇÊÐÇø·Ö - exportCommunityBasePrice(curMonth)//ÎÞ³ÇÊÐÇø·Ö - - exportCommunityPdescribeGj(curMonth)//ÎÞ³ÇÊÐÇø·Ö - exportCommCaseDetailGj(curMonth)//ÎÞ³ÇÊÐÇø·Ö +// exportEvaluationRateData() +// exportBigdataPoiData()// +// exportBigdataDistrictData()// +// +// exportGuidedPrice()//ËæÈ˹¤¸Ä±ä// +// exportCommunityGuidedPrice(lastMonth.toString()+"28")//ËæÕþ²ß// +// +// //´¦ÀíÐ¡ÇøÇé¿ö +// exportBigdataCommunity()// +// exportCommunityStaging()//ûcityid +// exportCommunityBuileType()// +// +// exportThirdPartyPrice2(lastMonth, new_dists)// +// exportBigdataForsale(lastMonthStr +"-19", new_dists) //2022-07-19 +// exportBigdataDeal(lastMonthStr + "-01", new_dists) //2022-07-01 +// +// //µÚÈý·½ÍøÕ¾È¨Öرí +// exportWebsiteRate(); +// +// //¼ÆËã½á¹ûÏà¹Ø +// exportAvmEvaluationGJNB(curMonth)//ÎÞ³ÇÊÐÇø·Ö +// exportAVMEvaluationItemGJNB(curMonth)//ÎÞ³ÇÊÐÇø·Ö +// +// exportCityPrice(curMonth)//ÎÞ³ÇÊÐÇø·Ö +// exportRegionPrice(curMonth)//ÎÞ³ÇÊÐÇø·Ö +// +// exportCommunityBasePriceGj(curMonth)//ÒѺÏÏÂÃæµÄºÏ²¢ //ÎÞ³ÇÊÐÇø·Ö +// exportCommunityBasePrice(curMonth)//ÎÞ³ÇÊÐÇø·Ö +// +// exportCommunityPdescribeGj(curMonth)//ÎÞ³ÇÊÐÇø·Ö +// exportCommCaseDetailGj(curMonth)//ÎÞ³ÇÊÐÇø·Ö def curDate = curMonth.toString()+"28" // okÎļþ @@ -1149,7 +1149,7 @@ class ExportOracleDataTest { def exportInOk(){ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" - Vector files = CcbFtpTestUtils.listFileNames(curDate) + Vector files = CcbFtpTestUtils.listFileNames(null, curDate) Iterator fileVec = files.iterator() def fileRows = [] while(fileVec.hasNext()){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleEvaluationData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleEvaluationData.groovy index b37ee5b94..419f170b0 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleEvaluationData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleEvaluationData.groovy @@ -13,6 +13,8 @@ class ExportOracleEvaluationData { "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") def citys = null + def cityCodeMap = CcbCommonUtils.findCityCodeValue() + static main(args) { def task = new ExportOracleEvaluationData() task.run(args) @@ -45,49 +47,53 @@ class ExportOracleEvaluationData { } def exportAvmEvaluationGJNB(yearMonth){ - def sql = "select ID,CITYID,CITYNAME, YEAR_MONTH, COMMUNITY_NUM, HAS_PRICE_COMMUNITY_NUM, PRICE_COVERAGE, ARTIFICIAL_CALC_NUM, ARTIFICIAL_CALC_COVERAGE, SAMPLE_CALC_NUM, SAMPLE_CALC_COVERAGE, \n" + - "ANN_CALC_NUM, ANN_CALC_COVERAGE, TEST_NUM, TEST_COMM_NUM, TEST_COMM_COVERAGE, PPE15_COVERAGE, RATE_MEDIAN, COEFFICIENT,DEAL_TEST_CASE_NUM, DEAL_TEST_COMM_NUM, DEAL_TEST_COMM_COVERAGE, DEAL_PPE15_COVERAGE, \n" + - "DEAL_COEFFICIENT, DEAL_RATE_MEDIAN, THIRD_TEST_CASE_NUM, THIRD_TEST_COMM_NUM, THIRD_TEST_COMM_COVERAGE, THIRD_PPE15_COVERAGE, THIRD_COEFFICIENT, THIRD_RATE_MEDIAN, \n" + - "to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from avm_evaluation_gj where YEAR_MONTH = ${yearMonth}"; + def allRows = [] - sqlInstance.eachRow(sql,{r-> - def data = [] - data << r.ID - data << r.CITYID - data << r.CITYNAME - data << r.YEAR_MONTH - data << r.COMMUNITY_NUM - data << r.HAS_PRICE_COMMUNITY_NUM - data << r.PRICE_COVERAGE - data << r.ARTIFICIAL_CALC_NUM - data << r.ARTIFICIAL_CALC_COVERAGE - data << r.SAMPLE_CALC_NUM - data << r.SAMPLE_CALC_COVERAGE - data << r.ANN_CALC_NUM - data << r.ANN_CALC_COVERAGE - data << r.TEST_NUM - data << r.TEST_COMM_NUM - data << r.TEST_COMM_COVERAGE - data << r.PPE15_COVERAGE - data << r.RATE_MEDIAN - data << r.COEFFICIENT - data << r.DEAL_TEST_CASE_NUM - data << r.DEAL_TEST_COMM_NUM - data << r.DEAL_TEST_COMM_COVERAGE - data << r.DEAL_PPE15_COVERAGE - data << r.DEAL_COEFFICIENT - data << r.DEAL_RATE_MEDIAN - data << r.THIRD_TEST_CASE_NUM - data << r.THIRD_TEST_COMM_NUM - data << r.THIRD_TEST_COMM_COVERAGE - data << r.THIRD_PPE15_COVERAGE - data << r.THIRD_COEFFICIENT - data << r.THIRD_RATE_MEDIAN - data << r.CREATETIME - data << r.UPDATETIME - data << r.SOURCECODE - allRows << data - }); + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select ID,CITYID,CITYNAME, YEAR_MONTH, COMMUNITY_NUM, HAS_PRICE_COMMUNITY_NUM, PRICE_COVERAGE, ARTIFICIAL_CALC_NUM, ARTIFICIAL_CALC_COVERAGE, SAMPLE_CALC_NUM, SAMPLE_CALC_COVERAGE, \n" + + "ANN_CALC_NUM, ANN_CALC_COVERAGE, TEST_NUM, TEST_COMM_NUM, TEST_COMM_COVERAGE, PPE15_COVERAGE, RATE_MEDIAN, COEFFICIENT,DEAL_TEST_CASE_NUM, DEAL_TEST_COMM_NUM, DEAL_TEST_COMM_COVERAGE, DEAL_PPE15_COVERAGE, \n" + + "DEAL_COEFFICIENT, DEAL_RATE_MEDIAN, THIRD_TEST_CASE_NUM, THIRD_TEST_COMM_NUM, THIRD_TEST_COMM_COVERAGE, THIRD_PPE15_COVERAGE, THIRD_COEFFICIENT, THIRD_RATE_MEDIAN, \n" + + "to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from avm_evaluation_gj where YEAR_MONTH = ${yearMonth} and CITYID='${cityCode}'"; + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.ID + data << r.CITYID + data << r.CITYNAME + data << r.YEAR_MONTH + data << r.COMMUNITY_NUM + data << r.HAS_PRICE_COMMUNITY_NUM + data << r.PRICE_COVERAGE + data << r.ARTIFICIAL_CALC_NUM + data << r.ARTIFICIAL_CALC_COVERAGE + data << r.SAMPLE_CALC_NUM + data << r.SAMPLE_CALC_COVERAGE + data << r.ANN_CALC_NUM + data << r.ANN_CALC_COVERAGE + data << r.TEST_NUM + data << r.TEST_COMM_NUM + data << r.TEST_COMM_COVERAGE + data << r.PPE15_COVERAGE + data << r.RATE_MEDIAN + data << r.COEFFICIENT + data << r.DEAL_TEST_CASE_NUM + data << r.DEAL_TEST_COMM_NUM + data << r.DEAL_TEST_COMM_COVERAGE + data << r.DEAL_PPE15_COVERAGE + data << r.DEAL_COEFFICIENT + data << r.DEAL_RATE_MEDIAN + data << r.THIRD_TEST_CASE_NUM + data << r.THIRD_TEST_COMM_NUM + data << r.THIRD_TEST_COMM_COVERAGE + data << r.THIRD_PPE15_COVERAGE + data << r.THIRD_COEFFICIENT + data << r.THIRD_RATE_MEDIAN + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + } def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; def fileName = "/ccb/MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_${curDate}_0001.dat" @@ -108,38 +114,43 @@ class ExportOracleEvaluationData { } def exportAVMEvaluationItemGJNB(yearMonth){ - def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, CONFIDENCE,TARGET_AVG_PRICE, \n" + - "to_char(DEALDATE,'yyyy-mm-dd hh24:mi:ss') as DEALDATE, UNIT_PRICE, BUILDAREA, FITMENT, HOUSEASPECT, FLOORNUM, FLOORCNT, DATA_TYPE, RATE, PE, \n" + - "to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME from avm_evaluation_item_gj where YEAR_MONTH = ${yearMonth}"; + def allRows = [] - sqlInstance.eachRow(sql,{r-> - def data = [] - data << r.ID - data << r.CITYID - data << r.CITYNAME - data << r.DISTID - data << r.DISTNAME - data << r.COMMID - data << r.COMMNAME - data << r.YEAR_MONTH - data << r.CALC_AVG_PRICE - data << r.AVG_PRICE - data << r.CALC_TYPE - data << r.CONFIDENCE - data << r.TARGET_AVG_PRICE - data << r.DEALDATE - data << r.UNIT_PRICE - data << r.BUILDAREA - data << r.FITMENT - data << r.HOUSEASPECT - data << r.FLOORNUM - data << r.FLOORCNT - data << r.DATA_TYPE - data << r.RATE - data << r.PE - data << r.CREATETIME - allRows << data - }); + + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, CONFIDENCE,TARGET_AVG_PRICE, \n" + + "to_char(DEALDATE,'yyyy-mm-dd hh24:mi:ss') as DEALDATE, UNIT_PRICE, BUILDAREA, FITMENT, HOUSEASPECT, FLOORNUM, FLOORCNT, DATA_TYPE, RATE, PE, \n" + + "to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME from avm_evaluation_item_gj where YEAR_MONTH = ${yearMonth} AND CITYID='${cityCode}'"; + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.ID + data << r.CITYID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.COMMID + data << r.COMMNAME + data << r.YEAR_MONTH + data << r.CALC_AVG_PRICE + data << r.AVG_PRICE + data << r.CALC_TYPE + data << r.CONFIDENCE + data << r.TARGET_AVG_PRICE + data << r.DEALDATE + data << r.UNIT_PRICE + data << r.BUILDAREA + data << r.FITMENT + data << r.HOUSEASPECT + data << r.FLOORNUM + data << r.FLOORCNT + data << r.DATA_TYPE + data << r.RATE + data << r.PE + data << r.CREATETIME + allRows << data + }); + } def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; def fileName = "/ccb/MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_${curDate}_0001.dat" @@ -158,31 +169,35 @@ class ExportOracleEvaluationData { } def exportCommunityBasePrice(yearMonth){ - def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, FAKE_RATE, DIFF_RATE, CONFI_RATE, CORR_RATE, CONFIDENCE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_base_price_gj where year_month=${yearMonth}"; + def allRows = [] - sqlInstance.eachRow(sql,{r-> - def data = [] - data << r.ID - data << r.CITYID - data << r.CITYNAME - data << r.DISTID - data << r.DISTNAME - data << r.COMMID - data << r.COMMNAME - data << r.YEAR_MONTH - data << r.CALC_AVG_PRICE - data << r.AVG_PRICE - data << r.CALC_TYPE - data << r.FAKE_RATE - data << r.DIFF_RATE - data << r.CONFI_RATE - data << r.CORR_RATE - data << r.CONFIDENCE - data << r.CREATETIME - data << r.UPDATETIME - data << r.SOURCECODE - allRows << data - }); + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, FAKE_RATE, DIFF_RATE, CONFI_RATE, CORR_RATE, CONFIDENCE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_base_price_gj where year_month=${yearMonth} AND CITYID='${cityCode}'"; + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.ID + data << r.CITYID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.COMMID + data << r.COMMNAME + data << r.YEAR_MONTH + data << r.CALC_AVG_PRICE + data << r.AVG_PRICE + data << r.CALC_TYPE + data << r.FAKE_RATE + data << r.DIFF_RATE + data << r.CONFI_RATE + data << r.CORR_RATE + data << r.CONFIDENCE + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + } def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat" @@ -201,31 +216,35 @@ class ExportOracleEvaluationData { } def exportCommunityBasePriceGj(yearMonth){ - def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, FAKE_RATE, DIFF_RATE, CONFI_RATE, CORR_RATE, CONFIDENCE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_base_price_gj where year_month=${yearMonth}"; + def allRows = [] - sqlInstance.eachRow(sql,{r-> - def data = [] - data << r.ID - data << r.CITYID - data << r.CITYNAME - data << r.DISTID - data << r.DISTNAME - data << r.COMMID - data << r.COMMNAME - data << r.YEAR_MONTH - data << r.CALC_AVG_PRICE - data << r.AVG_PRICE - data << r.CALC_TYPE - data << r.FAKE_RATE - data << r.DIFF_RATE - data << r.CONFI_RATE - data << r.CORR_RATE - data << r.CONFIDENCE - data << r.CREATETIME - data << r.UPDATETIME - data << r.SOURCECODE - allRows << data - }); + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, FAKE_RATE, DIFF_RATE, CONFI_RATE, CORR_RATE, CONFIDENCE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_base_price_gj where year_month=${yearMonth} AND CITYID='${cityCode}'"; + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.ID + data << r.CITYID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.COMMID + data << r.COMMNAME + data << r.YEAR_MONTH + data << r.CALC_AVG_PRICE + data << r.AVG_PRICE + data << r.CALC_TYPE + data << r.FAKE_RATE + data << r.DIFF_RATE + data << r.CONFI_RATE + data << r.CORR_RATE + data << r.CONFIDENCE + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + } def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" @@ -240,21 +259,24 @@ class ExportOracleEvaluationData { } def exportCityPrice(month){ - def sql = "select ID, CITYID, CITYNAME, YEAR_MONTH, CALC_AVG_PRICE, PRICE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from city_price_gj where year_month=${month}"; def allRows = [] - sqlInstance.eachRow(sql,{r-> - def data = [] - data << r.ID - data << r.CITYID - data << r.CITYNAME - data << r.YEAR_MONTH - data << r.CALC_AVG_PRICE - data << r.PRICE - data << r.CREATETIME - data << r.UPDATETIME - data << r.SOURCECODE - allRows << data - }); + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select ID, CITYID, CITYNAME, YEAR_MONTH, CALC_AVG_PRICE, PRICE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from city_price_gj where year_month=${month} AND CITYID='${cityCode}'"; + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.ID + data << r.CITYID + data << r.CITYNAME + data << r.YEAR_MONTH + data << r.CALC_AVG_PRICE + data << r.PRICE + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + } def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" def fileName = "/ccb/MODEL_000000_000000_CITY_PRICE_GJ_ALL_${curDate}_0001.dat" @@ -269,23 +291,27 @@ class ExportOracleEvaluationData { } def exportRegionPrice(month){ - def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, YEAR_MONTH, CALC_AVG_PRICE, PRICE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from DISTRICT_PRICE_GJ where year_month=${month}"; def allRows = [] - sqlInstance.eachRow(sql,{r-> - def data = [] - data << r.ID - data << r.CITYID - data << r.CITYNAME - data << r.DISTID - data << r.DISTNAME - data << r.YEAR_MONTH - data << r.CALC_AVG_PRICE - data << r.PRICE - data << r.CREATETIME - data << r.UPDATETIME - data << r.SOURCECODE - allRows << data - }); + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, YEAR_MONTH, CALC_AVG_PRICE, PRICE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from DISTRICT_PRICE_GJ where year_month=${month} AND CITYID='${cityCode}'"; + + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.ID + data << r.CITYID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.YEAR_MONTH + data << r.CALC_AVG_PRICE + data << r.PRICE + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + } def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" def fileName = "/ccb/MODEL_000000_000000_DISTRICT_PRICE_GJ_ALL_${curDate}_0001.dat" @@ -301,68 +327,67 @@ class ExportOracleEvaluationData { def exportCommunityPdescribeGj(month){ -// def citys = CcbCommonUtils.citys; -// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat" } def headers = [] -// -// for(def city : citys) { -// def cityCode = cityCodeDataMap.get(city) - def sql = "select ID,COMMUNITY_ID,CITYID ,CITY_NAME,DISTRICT_CODE ,DISTRICT_NAME,COMMUNITY_NAME,PQ_NAME,BASE_PRICE,CONFIDENCE_LEVEL,CNT,MIN_PRICE,MEDIANPRICE,MEANPRICE,MAX_PRICE,TIME,FIRSTDATAT,THIRDDATA,RELATIVEERROR,SKEWNESS,PQ_PRICE_RANGE,DISTRICT_PRICE_RANGE ,DESCRIPTION,PRIV_BASE_PRICE,IS_ARTTFICIAL,PRIV_FINAL_PRICE,to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE,PRICE,PRICE_SOURCE,CASE_MD_PRICE,CASE_AVG_PRICE,INDEX_LABEL,MAIN_PROPERTY,COMMTYPE,NEW_DEAL_PRICE,to_char(NEW_DEAL_DATE,'yyyy-mm-dd') as NEW_DEAL_DATE,PRICE_MOM_RATIO,MAX_MOM_RATIO,PRICE_YOY_RATIO ,SATUR_RATIO,NEIGHBOR_PRICE from community_p_describe_gj where time=${month}"; -// def sql = "select ID,COMMUNITY_ID,CITYID ,CITY_NAME,DISTRICT_CODE ,DISTRICT_NAME,COMMUNITY_NAME,PQ_NAME,BASE_PRICE,CONFIDENCE_LEVEL,CNT,MIN_PRICE,MEDIANPRICE,MEANPRICE,MAX_PRICE,TIME,FIRSTDATAT,THIRDDATA,RELATIVEERROR,SKEWNESS,PQ_PRICE_RANGE,DISTRICT_PRICE_RANGE ,DESCRIPTION,PRIV_BASE_PRICE,IS_ARTTFICIAL,PRIV_FINAL_PRICE,to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE,PRICE,PRICE_SOURCE,CASE_MD_PRICE,CASE_AVG_PRICE,INDEX_LABEL,MAIN_PROPERTY,COMMTYPE,NEW_DEAL_PRICE,to_char(NEW_DEAL_DATE,'yyyy-mm-dd') as NEW_DEAL_DATE,PRICE_MOM_RATIO,MAX_MOM_RATIO,PRICE_YOY_RATIO ,SATUR_RATIO,NEIGHBOR_PRICE from community_p_describe_gj where time in (202110,202111,202112,202201)"; def allRows = [] - sqlInstance.eachRow(sql, { r -> - def data = [] - data << r.ID - data << r.COMMUNITY_ID - data << r.CITYID - data << r.CITY_NAME - data << r.DISTRICT_CODE - data << r.DISTRICT_NAME - data << r.COMMUNITY_NAME - data << r.PQ_NAME - data << r.BASE_PRICE - data << r.CONFIDENCE_LEVEL - data << r.CNT - data << r.MIN_PRICE - data << r.MEDIANPRICE - data << r.MEANPRICE - data << r.MAX_PRICE - data << r.TIME - data << r.FIRSTDATAT - data << r.THIRDDATA - data << r.RELATIVEERROR - data << r.SKEWNESS - data << r.PQ_PRICE_RANGE - data << r.DISTRICT_PRICE_RANGE - data << r.DESCRIPTION - data << r.PRIV_BASE_PRICE - data << r.IS_ARTTFICIAL - data << r.PRIV_FINAL_PRICE - data << r.CREATE_DATE - data << r.PRICE - data << r.PRICE_SOURCE - data << r.CASE_MD_PRICE - data << r.CASE_AVG_PRICE - data << r.INDEX_LABEL - data << r.MAIN_PROPERTY - data << r.COMMTYPE - data << r.NEW_DEAL_PRICE - data << r.NEW_DEAL_DATE - data << r.PRICE_MOM_RATIO - data << r.MAX_MOM_RATIO - data << r.PRICE_YOY_RATIO - data << r.SATUR_RATIO - data << r.NEIGHBOR_PRICE - allRows << data - }); + + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select ID,COMMUNITY_ID,CITYID ,CITY_NAME,DISTRICT_CODE ,DISTRICT_NAME,COMMUNITY_NAME,PQ_NAME,BASE_PRICE,CONFIDENCE_LEVEL,CNT,MIN_PRICE,MEDIANPRICE,MEANPRICE,MAX_PRICE,TIME,FIRSTDATAT,THIRDDATA,RELATIVEERROR,SKEWNESS,PQ_PRICE_RANGE,DISTRICT_PRICE_RANGE ,DESCRIPTION,PRIV_BASE_PRICE,IS_ARTTFICIAL,PRIV_FINAL_PRICE,to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE,PRICE,PRICE_SOURCE,CASE_MD_PRICE,CASE_AVG_PRICE,INDEX_LABEL,MAIN_PROPERTY,COMMTYPE,NEW_DEAL_PRICE,to_char(NEW_DEAL_DATE,'yyyy-mm-dd') as NEW_DEAL_DATE,PRICE_MOM_RATIO,MAX_MOM_RATIO,PRICE_YOY_RATIO ,SATUR_RATIO,NEIGHBOR_PRICE from community_p_describe_gj where time=${month} AND CITYID='${cityCode}'"; + + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.ID + data << r.COMMUNITY_ID + data << r.CITYID + data << r.CITY_NAME + data << r.DISTRICT_CODE + data << r.DISTRICT_NAME + data << r.COMMUNITY_NAME + data << r.PQ_NAME + data << r.BASE_PRICE + data << r.CONFIDENCE_LEVEL + data << r.CNT + data << r.MIN_PRICE + data << r.MEDIANPRICE + data << r.MEANPRICE + data << r.MAX_PRICE + data << r.TIME + data << r.FIRSTDATAT + data << r.THIRDDATA + data << r.RELATIVEERROR + data << r.SKEWNESS + data << r.PQ_PRICE_RANGE + data << r.DISTRICT_PRICE_RANGE + data << r.DESCRIPTION + data << r.PRIV_BASE_PRICE + data << r.IS_ARTTFICIAL + data << r.PRIV_FINAL_PRICE + data << r.CREATE_DATE + data << r.PRICE + data << r.PRICE_SOURCE + data << r.CASE_MD_PRICE + data << r.CASE_AVG_PRICE + data << r.INDEX_LABEL + data << r.MAIN_PROPERTY + data << r.COMMTYPE + data << r.NEW_DEAL_PRICE + data << r.NEW_DEAL_DATE + data << r.PRICE_MOM_RATIO + data << r.MAX_MOM_RATIO + data << r.PRICE_YOY_RATIO + data << r.SATUR_RATIO + data << r.NEIGHBOR_PRICE + allRows << data + }); + } println allRows.size() FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); -// } + println "community_p_describe_gj Êý¾ÝÎļþÊä³öÍê³É......" CcbFtpUtils.ftpUpload(null, fileName, curDate) println "community_p_describe_gj ÉÏ´«FtpÍê³É......" @@ -370,8 +395,6 @@ class ExportOracleEvaluationData { def exportCommCaseDetailGj(month){ -// def citys = CcbCommonUtils.citys; - def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" def fileName = "/ccb/MODEL_000000_000000_COMM_CASE_DETAIL_GJ_ADD_${curDate}_0001.dat" @@ -381,8 +404,8 @@ class ExportOracleEvaluationData { def headers = [] for(def city : citys){ - def cityCode = cityCodeDataMap.get(city) - def sql = "select ID, CITYID, COMMUNITY_ID, NETLISTING_ID, PRICE_DATE, to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE from comm_case_detail_gj where cityid='${cityCode}' and price_date=${month}"; + def cityCode = cityCodeMap.get(city) + def sql = "select ID, CITYID, COMMUNITY_ID, NETLISTING_ID, PRICE_DATE, to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE from comm_case_detail_gj where cityid='${cityCode}' and price_date=${month} AND CITYID='${cityCode}'"; def allRows = [] sqlInstance.eachRow(sql,{r-> def data = [] From e9ac78eff35a890566aee5ba23bd47a340ccac89 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 27 Dec 2022 10:51:59 +0800 Subject: [PATCH 35/99] update --- .../ExportOracleAllDataTest.groovy | 70 +++++++++---------- .../ccbevaluation/ExportOracleDataTest.groovy | 64 ++++++++--------- 2 files changed, 67 insertions(+), 67 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy index 42f71fc0a..20fd78c85 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy @@ -42,45 +42,45 @@ class ExportOracleAllDataTest { def lastMonthStr = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM") def new_dists = [] -// exportBigdataForsale(new_dists) //2022-07-19 + exportBigdataForsale(new_dists) //2022-07-19 + + //²»»á±ä 10.25 + exportEvaluationRateData() + exportBigdataPoiData()// + exportBigdataDistrictData()// + + exportGuidedPrice()//ËæÈ˹¤¸Ä±ä// + exportCommunityGuidedPrice(lastMonth.toString()+"28")//ËæÕþ²ß// + + //´¦ÀíÐ¡ÇøÇé¿ö + exportBigdataCommunity()// + exportCommunityStaging()//ûcityid + exportCommunityBuileType()// + + exportThirdPartyPrice2(new_dists)// + exportBigdataDeal( new_dists) //2022-07-01 // -// //²»»á±ä 10.25 -// exportEvaluationRateData() -// exportBigdataPoiData()// -// exportBigdataDistrictData()// +// //µÚÈý·½ÍøÕ¾È¨Öرí + exportWebsiteRate(); // -// exportGuidedPrice()//ËæÈ˹¤¸Ä±ä// -// exportCommunityGuidedPrice(lastMonth.toString()+"28")//ËæÕþ²ß// +// //¼ÆËã½á¹ûÏà¹Ø + exportAvmEvaluationGJNB()// + exportAVMEvaluationItemGJNB()// + + exportCityPrice()// + exportRegionPrice()// + + exportCommunityBasePriceGj()// + exportCommunityBasePrice()// + + exportCommunityPdescribeGj()// + exportCommCaseDetailGj()// // -// //´¦ÀíÐ¡ÇøÇé¿ö -// exportBigdataCommunity()// -// exportCommunityStaging()//ûcityid -// exportCommunityBuileType()// -// -// exportThirdPartyPrice2(new_dists)// -// exportBigdataDeal( new_dists) //2022-07-01 -//// -//// //µÚÈý·½ÍøÕ¾È¨Öرí -// exportWebsiteRate(); -//// -//// //¼ÆËã½á¹ûÏà¹Ø -// exportAvmEvaluationGJNB()// -// exportAVMEvaluationItemGJNB()// -// -// exportCityPrice()// -// exportRegionPrice()// -// -// exportCommunityBasePriceGj()// -// exportCommunityBasePrice()// -// -// exportCommunityPdescribeGj()// -// exportCommCaseDetailGj()// -// -// def curDate = curMonth.toString()+"28" + def curDate = curMonth.toString()+"28" // // okÎļþ exportInOk() -// SSHHelperTest jschCommand = new SSHHelperTest(); -// System.out.println(jschCommand.executeCommand("sh /home/ccb_evaluation/output/eval_package.sh ${curDate}")); + SSHHelperTest jschCommand = new SSHHelperTest(); + System.out.println(jschCommand.executeCommand("sh /home/ccb_evaluation/output/eval_package.sh ${curDate}")); } @@ -1207,7 +1207,7 @@ class ExportOracleAllDataTest { } def exportInOk(){ - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"15" + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" Vector files = CcbFtpTestUtils.listFileNames(null, curDate) Iterator fileVec = files.iterator() def fileRows = [] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleDataTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleDataTest.groovy index 412ec7c3a..f1f2e17bc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleDataTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleDataTest.groovy @@ -32,37 +32,37 @@ class ExportOracleDataTest { def lastMonthStr = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM") def new_dists = [] //²»»á±ä 10.25 -// exportEvaluationRateData() -// exportBigdataPoiData()// -// exportBigdataDistrictData()// -// -// exportGuidedPrice()//ËæÈ˹¤¸Ä±ä// -// exportCommunityGuidedPrice(lastMonth.toString()+"28")//ËæÕþ²ß// -// -// //´¦ÀíÐ¡ÇøÇé¿ö -// exportBigdataCommunity()// -// exportCommunityStaging()//ûcityid -// exportCommunityBuileType()// -// -// exportThirdPartyPrice2(lastMonth, new_dists)// -// exportBigdataForsale(lastMonthStr +"-19", new_dists) //2022-07-19 -// exportBigdataDeal(lastMonthStr + "-01", new_dists) //2022-07-01 -// -// //µÚÈý·½ÍøÕ¾È¨Öرí -// exportWebsiteRate(); -// -// //¼ÆËã½á¹ûÏà¹Ø -// exportAvmEvaluationGJNB(curMonth)//ÎÞ³ÇÊÐÇø·Ö -// exportAVMEvaluationItemGJNB(curMonth)//ÎÞ³ÇÊÐÇø·Ö -// -// exportCityPrice(curMonth)//ÎÞ³ÇÊÐÇø·Ö -// exportRegionPrice(curMonth)//ÎÞ³ÇÊÐÇø·Ö -// -// exportCommunityBasePriceGj(curMonth)//ÒѺÏÏÂÃæµÄºÏ²¢ //ÎÞ³ÇÊÐÇø·Ö -// exportCommunityBasePrice(curMonth)//ÎÞ³ÇÊÐÇø·Ö -// -// exportCommunityPdescribeGj(curMonth)//ÎÞ³ÇÊÐÇø·Ö -// exportCommCaseDetailGj(curMonth)//ÎÞ³ÇÊÐÇø·Ö + exportEvaluationRateData() + exportBigdataPoiData()// + exportBigdataDistrictData()// + + exportGuidedPrice()//ËæÈ˹¤¸Ä±ä// + exportCommunityGuidedPrice(lastMonth.toString()+"28")//ËæÕþ²ß// + + //´¦ÀíÐ¡ÇøÇé¿ö + exportBigdataCommunity()// + exportCommunityStaging()//ûcityid + exportCommunityBuileType()// + + exportThirdPartyPrice2(lastMonth, new_dists)// + exportBigdataForsale(lastMonthStr +"-19", new_dists) //2022-07-19 + exportBigdataDeal(lastMonthStr + "-01", new_dists) //2022-07-01 + + //µÚÈý·½ÍøÕ¾È¨Öرí + exportWebsiteRate(); + + //¼ÆËã½á¹ûÏà¹Ø + exportAvmEvaluationGJNB(curMonth)//ÎÞ³ÇÊÐÇø·Ö + exportAVMEvaluationItemGJNB(curMonth)//ÎÞ³ÇÊÐÇø·Ö + + exportCityPrice(curMonth)//ÎÞ³ÇÊÐÇø·Ö + exportRegionPrice(curMonth)//ÎÞ³ÇÊÐÇø·Ö + + exportCommunityBasePriceGj(curMonth)//ÒѺÏÏÂÃæµÄºÏ²¢ //ÎÞ³ÇÊÐÇø·Ö + exportCommunityBasePrice(curMonth)//ÎÞ³ÇÊÐÇø·Ö + + exportCommunityPdescribeGj(curMonth)//ÎÞ³ÇÊÐÇø·Ö + exportCommCaseDetailGj(curMonth)//ÎÞ³ÇÊÐÇø·Ö def curDate = curMonth.toString()+"28" // okÎļþ @@ -1156,7 +1156,7 @@ class ExportOracleDataTest { ChannelSftp.LsEntry file = fileVec.next() if(org.apache.commons.lang.StringUtils.isNotBlank(file.getFilename()) && file.getFilename().endsWith(".dat")){ - fileRows << file.getFilename() +" "+CcbFtpTestUtils.getFileLength(file.getFilename(), curDate) + fileRows << file.getFilename() +" "+CcbFtpTestUtils.getFileLength(null, file.getFilename(), curDate) } } def fileName = "/ccb_evaluation/${curDate}.ok" From 03feeb1f0a7bb6c3ac97e27183981068f40c09f2 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 5 Jan 2023 00:14:06 +0800 Subject: [PATCH 36/99] add --- projects/fangjia-script/pom.xml | 5 + .../BigdataCommunityBatch.groovy | 2 +- .../pgsql/ExportPgsqlBaseEvaluation.groovy | 623 ++++++++++++++++++ .../pgsql/ExportPgsqlEvaluationResult.groovy | 382 +++++++++++ .../pgsql/ExportPgsqlRonghe.groovy | 14 + .../script/gj/test/DongGuangApiTest.groovy | 155 +++++ .../fangjia/script/util/ElasticUtils.groovy | 3 + .../fangjia/script/util/OpenApiClient.groovy | 24 + 8 files changed, 1207 insertions(+), 1 deletion(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlBaseEvaluation.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlEvaluationResult.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlRonghe.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/DongGuangApiTest.groovy diff --git a/projects/fangjia-script/pom.xml b/projects/fangjia-script/pom.xml index 9469b6d02..811ec290c 100644 --- a/projects/fangjia-script/pom.xml +++ b/projects/fangjia-script/pom.xml @@ -171,6 +171,11 @@ ojdbc14 10.2.0.4.0 + + org.postgresql + postgresql + 42.2.1 + com.yahoo.platform.yui yuicompressor diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy index bfce0b81b..27ed0ad49 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -28,7 +28,7 @@ import java.util.regex.Pattern @BatchOwner(Watcher.GaoJuan) class BigdataCommunityBatch extends BaseBatch{ - public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1) + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10) static main(args) { runBatch(new BigdataCommunityBatch(), args); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlBaseEvaluation.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlBaseEvaluation.groovy new file mode 100644 index 000000000..428aaa792 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlBaseEvaluation.groovy @@ -0,0 +1,623 @@ +package com.fangjia.script.ccbevaluation.pgsql + +import com.fangjia.script.ccbevaluation.CcbCommonUtils +import com.fangjia.script.ccbevaluation.CcbFtpUtils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import groovy.sql.Sql + +class ExportPgsqlBaseEvaluation { + + def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/postgres", + "postgres", "postgres", "org.postgresql.Driver") + def citys = null + def cityCodeMap = CcbCommonUtils.findCityCodeValue() + + def database = "base" + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def linux_prefix = "/home/goojia/ccb_data/evaluation/" + def win_prefix = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\evaluation\\${curDate}\\" + + static main(args) { + def task = new ExportPgsqlBaseEvaluation() + task.run(args) + } + + def run(args) { + citys = CcbCommonUtils.getRunningCitys(args) + def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")); + def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); + def lastMonthStr = "2022-09"//DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM") + def new_dists = [] + +// exportEvaluationRateData() +// exportBigdataPoiData() +// exportBigdataDistrictData() + + //µÚÈý·½ÍøÕ¾È¨Öرí +// exportWebsiteRate(); + +// exportGuidedPrice()//ËæÈ˹¤¸Ä±ä// +// exportCommunityGuidedPrice(lastMonth.toString()+"28")//ËæÕþ²ß// +// +//// //´¦ÀíÐ¡ÇøÇé¿ö +// exportBigdataCommunity()// +// exportCommunityStaging()//ûcityid +// exportCommunityBuildType()// +// +// exportThirdPartyPrice2(lastMonth, new_dists)// + exportBigdataForsale(lastMonthStr +"-19", new_dists) //2022-07-19 +// exportBigdataDeal(lastMonthStr + "-01", new_dists) //2022-07-01 + + + } + + def exportEvaluationRateData() { + def sql = "select gujia_ceft_id,gujia_ceft_tp,gujia_ceft,gujia_seq, gujia_urbn,rmrk_dsc, to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm, gujia_src_co from "+database+".t0831_evaluate_rate_gj_w"; + def allRows = [] + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_ceft_id + data << r.gujia_ceft_tp + data << r.gujia_ceft + data << r.gujia_seq + data << r.gujia_urbn + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + + + def fileName = linux_prefix + "MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = win_prefix + "MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_${curDate}_0001.dat" + } + createFileToFtp("EVALUATE_RATE", allRows, fileName, curDate, false) + } + + def exportBigdataPoiData() { + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select urbn_id,gujia_admndiv_id,gujia_fsid,poi_nm,gujia_tp,gujia_tp_ecd,lgt,ltt,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+".t0831_bigdata_poi_gj_w where urbn_id='${cityCode}'"; + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.urbn_id + data << r.gujia_admndiv_id + data << r.gujia_fsid + data << r.poi_nm + data << r.gujia_tp + data << r.gujia_tp_ecd + data << r.lgt + data << r.ltt + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + } + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + + def fileName = linux_prefix + "MODEL_000000_000000_BIGDATA_POI_GJ_ALL_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = win_prefix + "MODEL_000000_000000_BIGDATA_POI_GJ_ALL_${curDate}_0001.dat" + } + createFileToFtp("BIGDATA_POI_GJ", allRows, fileName, curDate, false) + } + + def exportBigdataDistrictData() { + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select gujia_admndiv_id,gujia_admndiv_nm,gujia_admndiv_nm_cpa,urbn_id,gujia_natstd_cd,gujia_dspl_seq,gujia_admndiv_cgy,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_lgc_admndiv_dsc,gujia_lgc_admndiv_cd,gujia_adiv_cd_val,gujia_data_tm from "+database+".t0831_bigdata_district_gj_w where urbn_id='${cityCode}'"; + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_admndiv_id + data << r.gujia_admndiv_nm + data << r.gujia_admndiv_nm_cpa + data << r.urbn_id + data << r.gujia_natstd_cd + data << r.gujia_dspl_seq + data << r.gujia_admndiv_cgy + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + data << r.gujia_lgc_admndiv_dsc + data << r.gujia_lgc_admndiv_cd + data << r.gujia_adiv_cd_val + data << r.gujia_data_tm + allRows << data + }); + } + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def fileName = linux_prefix + "MODEL_000000_000000_BIGDATA_DISTRICT_GJ_ALL_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = win_prefix + "MODEL_000000_000000_BIGDATA_DISTRICT_GJ_ALL_${curDate}_0001.dat" + } + createFileToFtp("BIGDATA_DISTRICT_GJ", allRows, fileName, curDate, false) + } + + def exportWebsiteRate(){ + def sql = "select gujiadatawebst_src_id,gujia_data_webst_src,gujiadatawebstmpng_id,gujia_wgt,gujia_tp,gujia_src_co,urbn_id,gujia_urbn,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm, to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm from "+database+".t0831_website_config_gj_w"; + def allRows = [] + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujiadatawebst_src_id + data << r.gujia_data_webst_src + data << r.gujiadatawebstmpng_id + data << r.gujia_wgt + data << r.gujia_tp + data << r.gujia_src_co + data << r.urbn_id + data << r.gujia_urbn + data << r.data_crt_tm + data << r.data_udt_tm + allRows << data + }); + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def fileName = linux_prefix + "MODEL_000000_000000_WEBSITE_CONFIG_GJ_ALL_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = win_prefix + "MODEL_000000_000000_WEBSITE_CONFIG_GJ_ALL_${curDate}_0001.dat" + } + createFileToFtp("WEBSITE_CONFIG_GJ", allRows, fileName, curDate, false) + } + + + def exportGuidedPrice() { + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select gujia_fsid,gujia_bds_id,comm_nm,urbn_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,to_char(gujia_drc_dt,'yyyy-mm-dd') as gujia_drc_dt,gujia_drc_prc,gujia_add_psn,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+".t0831_guided_price_gj_w where urbn_id='${cityCode}'"; + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_fsid + data << r.gujia_bds_id + data << r.comm_nm + data << r.urbn_id + data << r.urbn_nm + data << r.gujia_admndiv_id + data << r.gujia_admndiv_nm + data << r.gujia_drc_dt + data << r.gujia_drc_prc + data << r.gujia_add_psn + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + } + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def fileName = linux_prefix + "MODEL_000000_000000_GUIDED_PRICE_GJ_ALL_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = win_prefix + "MODEL_000000_000000_GUIDED_PRICE_GJ_ALL_${curDate}_0001.dat" + } + createFileToFtp("GUIDED_PRICE_GJ", allRows, fileName, curDate, false) + } + + def exportCommunityGuidedPrice(def date) { + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select gujia_fsid,gujia_bds_id, comm_nm, urbn_id, urbn_nm, gujia_admndiv_id, gujia_admndiv_nm, gujia_drc_prc, gujia_drc_prc_src_val, rmrk_dsc, to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm, to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm, gujia_src_co from "+database+".t0831_community_guide_price_gj_h where urbn_id='${cityCode}' and data_crt_tm>=to_date('${date}','yyyyMMdd')"; + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_fsid + data << r.gujia_bds_id + data << r.comm_nm + data << r.urbn_id + data << r.urbn_nm + data << r.gujia_admndiv_id + data << r.gujia_admndiv_nm + data << r.gujia_drc_prc + data << r.gujia_drc_prc_src_val + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + } + + def fileName = linux_prefix + "MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = win_prefix + "MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.dat" + } + createFileToFtp("COMMUNITY_GUIDE_PRICE_GJ", allRows, fileName, curDate, false) + } + + def exportBigdataCommunity(){ + + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select urbn_id,gujia_admndiv_id,gujia_comm_id,gujia_admndiv_nm,comm_nm,gujia_dtl_adr,lgt,ltt,gujiaocpoflandtotarea,gujia_bld_tot_area,gujia_grnrate,gujia_vol_rate,gujia_park_num,gujia_bld_yr_rc,gujia_hs_use,gujia_totunits,gujia_data_tm,gujia_udt_mark_val,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_comm_tp_val,gujia_comm_fmnm,gujia_str,gujia_hsno,gujiawthr_new_pan_ind,gujia_comm_intd,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,gujia_src_co,gujia_vlgtwn_str_mng,gujia_lgc_admndiv_cd,gujia_lgc_adiv,gujia_bds_st_val from "+database+".t0831_bigdata_community_gj_w where urbn_id='${cityCode}' ";//where cityId in (350200,150100) + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.urbn_id + data << r.gujia_admndiv_id + data << r.gujia_comm_id + data << r.gujia_admndiv_nm + data << r.comm_nm + data << r.gujia_dtl_adr + data << r.lgt + data << r.ltt + data << r.gujiaocpoflandtotarea + data << r.gujia_bld_tot_area + data << r.gujia_grnrate + data << r.gujia_vol_rate + data << r.gujia_park_num + data << r.gujia_bld_yr_rc + data << r.gujia_hs_use + data << r.gujia_totunits + data << r.gujia_data_tm + data << r.gujia_udt_mark_val + data << r.data_udt_tm + data << r.gujia_comm_tp_val + data << r.gujia_comm_fmnm + data << r.gujia_str + data << r.gujia_hsno + data << r.gujiawthr_new_pan_ind + data << r.gujia_comm_intd + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.gujia_src_co + data << r.gujia_vlgtwn_str_mng + data << r.gujia_lgc_admndiv_cd + data << r.gujia_lgc_adiv + data << r.gujia_bds_st_val + allRows << data + }); + } + def fileName = linux_prefix + "MODEL_000000_000000_BIGDATA_COMMUNITY_GJ_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "MODEL_000000_000000_BIGDATA_COMMUNITY_GJ_ALL_${curDate}_0001.dat" + } + createFileToFtp("BIGDATA_COMMUNITY_GJ", allRows, fileName, curDate, false) + } + + def exportCommunityStaging() { + + def allRows = [] + def sql = "select gujia_instm_id,gujia_comm_id,gujiacomm_prim_prd_id,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+".t0831_community_staging_gj_w"; + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_instm_id + data << r.gujia_comm_id + data << r.gujiacomm_prim_prd_id + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + def fileName = linux_prefix + "MODEL_000000_000000_COMMUNITY_STAGING_GJ_ALL_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = win_prefix + "MODEL_000000_000000_COMMUNITY_STAGING_GJ_ALL_${curDate}_0001.dat" + } + createFileToFtp("COMMUNITY_STAGING_GJ", allRows, fileName, curDate, false) + } + + def exportCommunityBuildType() { + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select urbn_id,gujia_comm_id,gujia_bld_tp_val,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_data_tm from "+database+".t0831_bigdata_commbuildtype_gj_w where urbn_id='${cityCode}'"; + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.urbn_id + data << r.gujia_comm_id + data << r.gujia_bld_tp_val + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + data << r.gujia_data_tm + allRows << data + }); + } + def fileName = linux_prefix + "MODEL_000000_000000_BIGDATA_COMMBUILDTYPE_GJ_ALL_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = win_prefix + "MODEL_000000_000000_BIGDATA_COMMBUILDTYPE_GJ_ALL_${curDate}_0001.dat" + } + createFileToFtp("BIGDATA_COMMBUILDTYPE_GJ", allRows, fileName, curDate, false) + } + + def exportThirdPartyPrice2(price_date, new_dists) { + + def fileName = linux_prefix + "MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = win_prefix + "MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" + } + def index = 0 + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def allRows = [] + if(new_dists) { + def newstr = "('"+new_dists.join("','")+"')" + def sql1 = "select gujia_trdpt_avprc_id,urbn_id,gujia_admndiv_id,gujia_bds_id,comm_nm,gujia_dtsr_id,gujia_dtsr_dsc,gujia_avprc_dt_val,gujia_comm_avprc,gujia_comm_hgst_prc,gujia_comm_lwst_prc,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm from "+database+".t0831_bigdata_third_party_price_gj_h where " + + "urbn_id='${cityCode}' and gujia_avprc_dt_val=${price_date} and gujia_admndiv_id not in ${newstr} "; + def rets1 = getThirdPartyDatas(sql1) + allRows.addAll(rets1) + + def sql2 = "select gujia_trdpt_avprc_id,urbn_id,gujia_admndiv_id,gujia_bds_id,comm_nm,gujia_dtsr_id,gujia_dtsr_dsc,gujia_avprc_dt_val,gujia_comm_avprc,gujia_comm_hgst_prc,gujia_comm_lwst_prc,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm from "+database+".t0831_bigdata_third_party_price_gj_h where " + + "urbn_id='${cityCode}' and gujia_admndiv_id in ${newstr} "; + def rets2 = getThirdPartyDatas(sql2) + allRows.addAll(rets2) + }else{ + def sql = "select gujia_trdpt_avprc_id,urbn_id,gujia_admndiv_id,gujia_bds_id,comm_nm,gujia_dtsr_id,gujia_dtsr_dsc,gujia_avprc_dt_val,gujia_comm_avprc,gujia_comm_hgst_prc,gujia_comm_lwst_prc,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm from "+database+".t0831_bigdata_third_party_price_gj_h where urbn_id='${cityCode}' and gujia_avprc_dt_val='${price_date}' "; + allRows = getThirdPartyDatas(sql) + } + if(allRows && allRows.size()>0){ + index = index + 1 + if(index==1){ + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + }else{ + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", true); + } + } + } + + println "BIGDATA_THIRD_PARTY_PRICE_GJ Êý¾ÝÎļþÊä³öÍê³É......" + CcbFtpUtils.ftpUpload(null, fileName, curDate) + } + + def getThirdPartyDatas(def sql){ + def allRows = [] + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_trdpt_avprc_id + data << r.urbn_id + data << r.gujia_admndiv_id + data << r.gujia_bds_id + data << r.comm_nm + data << r.gujia_dtsr_id + data << r.gujia_dtsr_dsc + data << r.gujia_avprc_dt_val + data << r.gujia_comm_avprc + data << r.gujia_comm_hgst_prc + data << r.gujia_comm_lwst_prc + data << r.data_udt_tm + allRows << data + }) + return allRows; + } + + + def exportBigdataDeal(def dealDate, def new_dists) { + + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + if(new_dists){ + def newstr = "('"+new_dists.join("','")+"')" + def sql1 = "select urbn_id,mdl_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_str,gujia_hsno,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_prd_val,gujia_wbt_floor,gujia_lyr_tp_val,gujia_tot_floor_num,gujia_bld_tp_val,gujia_bld_stc,gujiaconstructionarea,gujia_set_innr_area,gujia_acc_tp_val,gujia_room,gujia_dept,gujia_ktch_num,gujia_toliet,gujia_hs_use,gujia_indr_stc_val,gujia_twrd_val,gujia_dcrt_sttn,gujia_ases_bt_ln_num,gujia_trdtckt_prc,gujia_mdl_tot_prc,gujia_dcl_tot_prc,gujia_ases_tot_prc,gujiasigntonlinectrid,to_char(txn_dt,'yyyy-mm-dd') as txn_dt,gujia_txn_tp_val,gujia_sell_pymtmod,gujia_mdl_cyc,gujiawthrext_lift_ind,gujiagvlndfargrdnarea,gujia_gv_lu_tbl_area,gujia_gv_sun_tbl_area,gujiagvbaywindow_area,gujiagvarea_tower_flr,gujia_gv_grg_area,gujiagvundgdroom_area,gujia_gv_othr_area,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_mkdam,gujia_data_tm " + + "from "+database+".t0831_bigdata_deal_gj_h where urbn_id='${cityCode}' and txn_dt>=to_date('${dealDate}','yyyy-MM-dd') and gujia_admndiv_id not in ${newstr}" + def rets1 = dealData(sql1) + + def sql2 = "select urbn_id,mdl_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_str,gujia_hsno,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_prd_val,gujia_wbt_floor,gujia_lyr_tp_val,gujia_tot_floor_num,gujia_bld_tp_val,gujia_bld_stc,gujiaconstructionarea,gujia_set_innr_area,gujia_acc_tp_val,gujia_room,gujia_dept,gujia_ktch_num,gujia_toliet,gujia_hs_use,gujia_indr_stc_val,gujia_twrd_val,gujia_dcrt_sttn,gujia_ases_bt_ln_num,gujia_trdtckt_prc,gujia_mdl_tot_prc,gujia_dcl_tot_prc,gujia_ases_tot_prc,gujiasigntonlinectrid,to_char(txn_dt,'yyyy-mm-dd') as txn_dt,gujia_txn_tp_val,gujia_sell_pymtmod,gujia_mdl_cyc,gujiawthrext_lift_ind,gujiagvlndfargrdnarea,gujia_gv_lu_tbl_area,gujia_gv_sun_tbl_area,gujiagvbaywindow_area,gujiagvarea_tower_flr,gujia_gv_grg_area,gujiagvundgdroom_area,gujia_gv_othr_area,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_mkdam,gujia_data_tm " + + "from "+database+".t0831_bigdata_deal_gj_h where urbn_id='${cityCode}' and gujia_admndiv_id in ${newstr}" + def rets2 = dealData(sql2) + allRows.addAll(rets1) + allRows.addAll(rets2) + }else{ + def sql = "select urbn_id,mdl_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_str,gujia_hsno,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_prd_val,gujia_wbt_floor,gujia_lyr_tp_val,gujia_tot_floor_num,gujia_bld_tp_val,gujia_bld_stc,gujiaconstructionarea,gujia_set_innr_area,gujia_acc_tp_val,gujia_room,gujia_dept,gujia_ktch_num,gujia_toliet,gujia_hs_use,gujia_indr_stc_val,gujia_twrd_val,gujia_dcrt_sttn,gujia_ases_bt_ln_num,gujia_trdtckt_prc,gujia_mdl_tot_prc,gujia_dcl_tot_prc,gujia_ases_tot_prc,gujiasigntonlinectrid,to_char(txn_dt,'yyyy-mm-dd') as txn_dt,gujia_txn_tp_val,gujia_sell_pymtmod,gujia_mdl_cyc,gujiawthrext_lift_ind,gujiagvlndfargrdnarea,gujia_gv_lu_tbl_area,gujia_gv_sun_tbl_area,gujiagvbaywindow_area,gujiagvarea_tower_flr,gujia_gv_grg_area,gujiagvundgdroom_area,gujia_gv_othr_area,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_mkdam,gujia_data_tm " + + "from "+database+".t0831_bigdata_deal_gj_h where urbn_id='${cityCode}' and txn_dt>=to_date('${dealDate}','yyyy-MM-dd')"; + allRows.addAll(dealData(sql)) + } + } + + def fileName = linux_prefix + "MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = win_prefix + "MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_${curDate}_0001.dat" + } + createFileToFtp("BIGDATA_DEAL_GJ", allRows, fileName, curDate, false) + } + + + def dealData(def sql){ + def allRows = [] + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.urbn_id + data << r.urbn_id + data << r.mdl_id + data << r.urbn_nm + data << r.gujia_admndiv_id + data << r.gujia_admndiv_nm + data << r.gujia_str + data << r.gujia_hsno + data << r.gujia_comm_id + data << r.comm_nm + data << r.gujia_bld_id + data << r.bld_id + data << r.gujia_bld_nm + data << r.gujia_unit_id + data << r.unit_id + data << r.gujia_unit_nm + data << r.gujia_acc_id + data << r.gujia_dtl_adr + data << r.gujia_bld_yr_rc + data << r.gujia_own_val + data << r.gujia_own_yrlmt_val + data << r.gujia_prd_val + data << r.gujia_wbt_floor + data << r.gujia_lyr_tp_val + data << r.gujia_tot_floor_num + data << r.gujia_bld_tp_val + data << r.gujia_bld_stc + data << r.gujiaconstructionarea + data << r.gujia_set_innr_area + data << r.gujia_acc_tp_val + data << r.gujia_room + data << r.gujia_dept + data << r.gujia_ktch_num + data << r.gujia_toliet + data << r.gujia_hs_use + data << r.gujia_indr_stc_val + data << r.gujia_twrd_val + data << r.gujia_dcrt_sttn + data << r.gujia_ases_bt_ln_num + data << r.gujia_trdtckt_prc + data << r.gujia_mdl_tot_prc + data << r.gujia_dcl_tot_prc + data << r.gujia_ases_tot_prc + data << r.gujiasigntonlinectrid + data << r.txn_dt + data << r.gujia_txn_tp_val + data << r.gujia_sell_pymtmod + data << r.gujia_mdl_cyc + data << r.gujiawthrext_lift_ind + data << r.gujiagvlndfargrdnarea + data << r.gujia_gv_lu_tbl_area + data << r.gujia_gv_sun_tbl_area + data << r.gujiagvbaywindow_area + data << r.gujiagvarea_tower_flr + data << r.gujia_gv_grg_area + data << r.gujiagvundgdroom_area + data << r.gujia_gv_othr_area + data << r.gujia_hs_src_dtsrc + data << r.intr_co_id + data << r.intr_co_nm + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + data << r.gujia_mkdam + data << r.gujia_data_tm + allRows << data + }); + return allRows + } + + + def exportBigdataForsale(publishDate, new_dists){ + def fileName = linux_prefix + "MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat" + } + def index = 0 + for(def city : citys){ + def cityCode = cityCodeMap.get(city) + + def allRows = [] + if(new_dists){ + def newstr = "('"+new_dists.join("','")+"')" + def sql1 = "select urbn_id,gujia_sell_id,gujia_listg_hs_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_unitprc,gujia_tot_prc,gujia_sell_pymtmod,gujia_wbt_floor,gujia_tot_floor_num,gujia_bld_tp_val,gujia_indr_stc_val,gujia_bld_stc,gujiaconstructionarea,gujia_set_innr_area,gujia_acc_tp_val,gujia_room,gujia_dept,gujia_ktch_num,gujia_toliet,gujia_twrd_val,gujia_dcrt_sttn,gujiawthrext_lift_ind,gujiajiladderjiactval,to_char(anc_tm,'yyyy-mm-dd') as anc_tm,to_char(exdat,'yyyy-mm-dd') as exdat,gujia_wthr_strt_ind,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,gujia_anc_psn,gujia_anc_psn_tel,gujia_dsc_inf_cntnt,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_apd_inf " + + "from "+database+".t0831_bigdata_forsale_gj_h where urbn_id='${cityCode}' and anc_tm>=to_date('${publishDate}','yyyy-MM-dd') and gujia_admndiv_id not in ${newstr}"; + def rets1 = getHouseDatas(sql1) + allRows.addAll(rets1) + }else{ + def sql = "select urbn_id,gujia_sell_id,gujia_listg_hs_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_unitprc,gujia_tot_prc,gujia_sell_pymtmod,gujia_wbt_floor,gujia_tot_floor_num,gujia_bld_tp_val,gujia_indr_stc_val,gujia_bld_stc,gujiaconstructionarea,gujia_set_innr_area,gujia_acc_tp_val,gujia_room,gujia_dept,gujia_ktch_num,gujia_toliet,gujia_twrd_val,gujia_dcrt_sttn,gujiawthrext_lift_ind,gujiajiladderjiactval,to_char(anc_tm,'yyyy-mm-dd') as anc_tm,to_char(exdat,'yyyy-mm-dd') as exdat,gujia_wthr_strt_ind,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,gujia_anc_psn,gujia_anc_psn_tel,gujia_dsc_inf_cntnt,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_apd_inf " + + "from "+database+".t0831_bigdata_forsale_gj_h where urbn_id='${cityCode}' and anc_tm>=to_date('${publishDate}','yyyy-MM-dd')"; + allRows = getHouseDatas(sql) + } + if(allRows.size()>0){ + index = index + 1 + if(index == 1){ + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + }else{ + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", true); + } + + } + } + + if(new_dists){//²¹³ä±»ÐÞÇøÓòµÄÊý¾Ý + def newstr = "('"+new_dists.join("','")+"')" + def sql2 = "select urbn_id,gujia_sell_id,gujia_listg_hs_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_unitprc,gujia_tot_prc,gujia_sell_pymtmod,gujia_wbt_floor,gujia_tot_floor_num,gujia_bld_tp_val,gujia_indr_stc_val,gujia_bld_stc,gujiaconstructionarea,gujia_set_innr_area,gujia_acc_tp_val,gujia_room,gujia_dept,gujia_ktch_num,gujia_toliet,gujia_twrd_val,gujia_dcrt_sttn,gujiawthrext_lift_ind,gujiajiladderjiactval,to_char(anc_tm,'yyyy-mm-dd') as anc_tm,to_char(exdat,'yyyy-mm-dd') as exdat,gujia_wthr_strt_ind,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,gujia_anc_psn,gujia_anc_psn_tel,gujia_dsc_inf_cntnt,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_apd_inf " + + "from "+database+".t0831_bigdata_forsale_gj_h where gujia_admndiv_id in ${newstr}" + def rets2 = getHouseDatas(sql2) + if(rets2.size()>0){ + FileUtils.batchCreateFile(rets2, fileName, [], "utf-8", true); + } + } + println "BIGDATA_FORSALE_GJ Êý¾ÝÎļþÊä³öÍê³É......" + CcbFtpUtils.ftpUpload(null, fileName, curDate) + println "BIGDATA_FORSALE_GJ ÉÏ´«FtpÍê³É......" + } + + def getHouseDatas(sql){ + def allRows = [] + def index = 0 + sqlInstance.eachRow(sql,{r-> + index = index+1 + println r.urbn_nm +" "+index + def data = [] + data << r.urbn_id + data << r.gujia_sell_id + data << r.gujia_listg_hs_id + data << r.urbn_nm + data << r.gujia_admndiv_id + data << r.gujia_admndiv_nm + data << r.gujia_comm_id + data << r.comm_nm + data << r.gujia_bld_id + data << r.bld_id + data << r.gujia_bld_nm + data << r.gujia_unit_id + data << r.unit_id + data << r.gujia_unit_nm + data << r.gujia_acc_id + data << r.gujia_dtl_adr + data << r.gujia_bld_yr_rc + data << r.gujia_own_val + data << r.gujia_own_yrlmt_val + data << r.gujia_unitprc + data << r.gujia_tot_prc + data << r.gujia_sell_pymtmod + data << r.gujia_wbt_floor + data << r.gujia_tot_floor_num + data << r.gujia_bld_tp_val + data << r.gujia_indr_stc_val + data << r.gujia_bld_stc + data << r.gujiaconstructionarea + data << r.gujia_set_innr_area + data << r.gujia_acc_tp_val + data << r.gujia_room + data << r.gujia_dept + data << r.gujia_ktch_num + data << r.gujia_toliet + data << r.gujia_twrd_val + data << r.gujia_dcrt_sttn + data << r.gujiawthrext_lift_ind + data << r.gujiajiladderjiactval + data << r.anc_tm + data << r.exdat + data << r.gujia_wthr_strt_ind + data << r.gujia_hs_src_dtsrc + data << r.intr_co_id + data << r.intr_co_nm + data << r.gujia_anc_psn + data << r.gujia_anc_psn_tel + data << r.gujia_dsc_inf_cntnt + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + data << r.gujia_apd_inf + allRows << data +// if(allRows.size()%200000==0){ +// FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); +// allRows.clear() +// } + }); + return allRows + } + + + def createFileToFtp(tableName,allRows, fileName, curDate, isAppend){ + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", isAppend); + println tableName+" Êý¾ÝÎļþÊä³öÍê³É......" +// CcbFtpUtils.ftpUpload(null, fileName, curDate) +// println tableName+" ÉÏ´«FtpÍê³É......" + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlEvaluationResult.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlEvaluationResult.groovy new file mode 100644 index 000000000..1508d0b9f --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlEvaluationResult.groovy @@ -0,0 +1,382 @@ +package com.fangjia.script.ccbevaluation.pgsql + +import com.fangjia.script.ccbevaluation.CcbCommonUtils +import com.fangjia.script.ccbevaluation.CcbFtpUtils +import com.fangjia.script.ccbevaluation.ronghe.SSHHelper +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import com.jcraft.jsch.ChannelSftp +import groovy.sql.Sql +import org.apache.commons.codec.digest.DigestUtils + +class ExportPgsqlEvaluationResult { + + def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/postgres", + "postgres", "postgres", "org.postgresql.Driver") + def citys = null + def cityCodeMap = CcbCommonUtils.findCityCodeValue() + + def database = "t_app_p9pha_plda." + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def linux_prefix = "/home/goojia/ccb_data/evaluation/" + def win_prefix = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\evaluation\\${curDate}\\" + + static main(args) { + def task = new ExportPgsqlEvaluationResult() + task.run(args) + } + + def run(args) { + citys = CcbCommonUtils.getRunningCitys(args) + def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")); + + //¼ÆËã½á¹ûÏà¹Ø +// exportAvmEvaluationGJNB(curMonth) +// exportAVMEvaluationItemGJNB(curMonth) +// +// exportCityPrice(curMonth) +// exportRegionPrice(curMonth) +// +// exportCommunityBasePrice(curMonth) +// +// exportCommunityPdescribeGj(curMonth) +// exportCommCaseDetailGj(curMonth) +// + def curDate = curMonth.toString()+"28" +// Éú³ÉMD5Îļþ + exportMd5() + SSHHelper jschCommand = new SSHHelper(); + System.out.println(jschCommand.executeCommand("sh /home/ccb/output/gz_package.sh ${curDate}")); +// okÎļþ + exportOkFile() + } + + def exportAvmEvaluationGJNB(yearMonth){ + + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select gujia_fsid,urbn_id,urbn_nm,gujia_yrmo,gujia_comm_num,gujia_val_comm_num,gujia_prc_cvr_rate,gujiamnldrcprccommnum,gujia_mnl_drc_prc_pct,gujia_samp_comm_num,gujiaext_samp_clc_pct,gujianervethruncomnum,gujianervethrunclcpct,gujia_tst_case_num,gujia_tst_comm_num,gujiatst_comm_num_pct,gujia_ppe15_cvr_rate,gujia_rto_mdn,gujia_rto_lv_san_ceft,gujiamdl_tst_case_num,gujiamdl_tst_comm_num,gujiamdltstcommnumpct,gujiamdltst_ppe15_pct,gujiamdlrtolvsan_ceft,gujia_mdl_md_bit_rto,gujiatrdpttstcase_num,gujiatrdpttstcomm_num,gujiatrdpttcommnumpct,gujiatrdpttstppe15pct,gujiatrdptrolvsanceft,gujiatrdpt_md_bit_rto,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"avm_evaluation_gj_h where gujia_yrmo = ${yearMonth} and urbn_id='${cityCode}'"; + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_fsid + data << r.urbn_id + data << r.urbn_nm + data << r.gujia_yrmo + data << r.gujia_comm_num + data << r.gujia_val_comm_num + data << r.gujia_prc_cvr_rate + data << r.gujiamnldrcprccommnum + data << r.gujia_mnl_drc_prc_pct + data << r.gujia_samp_comm_num + data << r.gujiaext_samp_clc_pct + data << r.gujianervethruncomnum + data << r.gujianervethrunclcpct + data << r.gujia_tst_case_num + data << r.gujia_tst_comm_num + data << r.gujiatst_comm_num_pct + data << r.gujia_ppe15_cvr_rate + data << r.gujia_rto_mdn + data << r.gujia_rto_lv_san_ceft + data << r.gujiamdl_tst_case_num + data << r.gujiamdl_tst_comm_num + data << r.gujiamdltstcommnumpct + data << r.gujiamdltst_ppe15_pct + data << r.gujiamdlrtolvsan_ceft + data << r.gujia_mdl_md_bit_rto + data << r.gujiatrdpttstcase_num + data << r.gujiatrdpttstcomm_num + data << r.gujiatrdpttcommnumpct + data << r.gujiatrdpttstppe15pct + data << r.gujiatrdptrolvsanceft + data << r.gujiatrdpt_md_bit_rto + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + } + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; + def fileName = linux_prefix + "MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_${curDate}_0001.dat" + def fileName1 = linux_prefix + "MODEL_000000_000000_AVM_EVALUATION_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_${curDate}_0001.dat" + fileName1 = win_prefix + "MODEL_000000_000000_AVM_EVALUATION_GJ_ADD_${curDate}_0001.dat" + } + createFileToFtp("AVM_EVALUATION_GJNB", allRows, fileName, curDate, false) + createFileToFtp("AVM_EVALUATION_GJ", allRows, fileName1, curDate, false) + } + + def exportAVMEvaluationItemGJNB(yearMonth){ + + def allRows = [] + + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select gujia_fsid,urbn_id,urbn_nm,gujia_rgon_id,gujia_rgon_nm,gujia_comm_id,gujia_comm_nm,gujia_clc_yrmo,gujia_clc_avprc,gujia_avprc,gujia_clc_tp_val,gujia_rlb,gujia_to_mkprc_mny, to_char(mdl_dt,'yyyy-mm-dd') as mdl_dt,gujia_valt_unitprc,gujia_area,gujia_dcrt_val,gujia_twrd_val,gujia_wbt_floor,gujia_tot_floor,gujia_data_tp,gujia_rto,gujia_mns_rto, to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm from "+database+"avm_evaluation_item_gj_h where gujia_clc_yrmo = ${yearMonth} AND urbn_id='${cityCode}'"; + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_fsid + data << r.urbn_id + data << r.urbn_nm + data << r.gujia_rgon_id + data << r.gujia_rgon_nm + data << r.gujia_comm_id + data << r.gujia_comm_nm + data << r.gujia_clc_yrmo + data << r.gujia_clc_avprc + data << r.gujia_avprc + data << r.gujia_clc_tp_val + data << r.gujia_rlb + data << r.gujia_to_mkprc_mny + data << r.mdl_dt + data << r.gujia_valt_unitprc + data << r.gujia_area + data << r.gujia_dcrt_val + data << r.gujia_twrd_val + data << r.gujia_wbt_floor + data << r.gujia_tot_floor + data << r.gujia_data_tp + data << r.gujia_rto + data << r.gujia_mns_rto + data << r.data_crt_tm + allRows << data + }); + } + + def fileName = linux_prefix + "MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_${curDate}_0001.dat" + def fileName1 = linux_prefix + "MODEL_000000_000000_AVM_EVALUATION_ITEM_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_${curDate}_0001.dat" + fileName1 = win_prefix + "MODEL_000000_000000_AVM_EVALUATION_ITEM_GJ_ADD_${curDate}_0001.dat" + } + + createFileToFtp("AVM_EVALUATION_ITEM_GJNB",allRows, fileName, curDate, false) + createFileToFtp("AVM_EVALUATION_ITEM_GJ",allRows, fileName1, curDate, false) + } + + def exportCityPrice(month){ + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select gujia_trdpt_avprc_id,urbn_id,urbn_nm,gujia_clc_yrmo,gujia_clc_avprc,gujia_urbn_avprc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"avm_city_price_gj_h where gujia_clc_yrmo=${month} AND urbn_id='${cityCode}'"; + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_trdpt_avprc_id + data << r.urbn_id + data << r.urbn_nm + data << r.gujia_clc_yrmo + data << r.gujia_clc_avprc + data << r.gujia_urbn_avprc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + } + + def fileName = linux_prefix + "MODEL_000000_000000_CITY_PRICE_GJ_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "MODEL_000000_000000_CITY_PRICE_GJ_ALL_${curDate}_0001.dat" + } + createFileToFtp("CITY_PRICE_GJ",allRows, fileName, curDate, false) + } + + def exportRegionPrice(month){ + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select gujia_trdpt_avprc_id,urbn_id,urbn_nm,gujia_admndiv_id,gxda_admndiv_nm,gujia_clc_yrmo,gujia_clc_avprc,gujia_urbn_avprc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"avm_district_price_gj_h where gujia_clc_yrmo=${month} AND urbn_id='${cityCode}'"; + + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_trdpt_avprc_id + data << r.urbn_id + data << r.urbn_nm + data << r.gujia_admndiv_id + data << r.gxda_admndiv_nm + data << r.gujia_clc_yrmo + data << r.gujia_clc_avprc + data << r.gujia_urbn_avprc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + } + def fileName = linux_prefix + "MODEL_000000_000000_DISTRICT_PRICE_GJ_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "MODEL_000000_000000_DISTRICT_PRICE_GJ_ALL_${curDate}_0001.dat" + } + + createFileToFtp("DISTRICT_PRICE_GJ",allRows, fileName, curDate, false) + } + + def exportCommunityBasePrice(yearMonth){ + + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select gujia_bss_prc_id,urbn_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_bds_id,comm_nm,gujia_clc_yrmo,gujia_clc_avprc,gujia_comm_basic_prc,gujia_clmtd_val,gujia_fwd_samp_rto,gujia_mns_grd,gujia_cfdrg,gujiaarndcommprc_crlt,gujia_cfdc_dgr_val,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"avm_community_base_price_gj_h where gujia_clc_yrmo=${yearMonth} AND urbn_id='${cityCode}'"; + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_bss_prc_id + data << r.urbn_id + data << r.urbn_nm + data << r.gujia_admndiv_id + data << r.gujia_admndiv_nm + data << r.gujia_bds_id + data << r.comm_nm + data << r.gujia_clc_yrmo + data << r.gujia_clc_avprc + data << r.gujia_comm_basic_prc + data << r.gujia_clmtd_val + data << r.gujia_fwd_samp_rto + data << r.gujia_mns_grd + data << r.gujia_cfdrg + data << r.gujiaarndcommprc_crlt + data << r.gujia_cfdc_dgr_val + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + } + + def fileName = linux_prefix + "MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat" + def fileName1 = linux_prefix + "MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat" + fileName1 = win_prefix + "MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" + } + createFileToFtp("COMMUNITY_BASE_PRICE_GJNB",allRows, fileName, curDate, false) + createFileToFtp("COMMUNITY_BASE_PRICE_GJ",allRows, fileName1, curDate, false) + } + + def exportCommunityPdescribeGj(month){ + + def fileName = linux_prefix + "MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat" + } + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select gujia_fsid,gujia_comm_id,urbn_id,urbn_nm,admndiv_ecd,gujia_admndiv_nm,comm_nm,gujia_als_dsc,gujia_bss_prc,gujia_cfdc_dgr_grd,gujiart3mlistgcasenum,gujialistgcaselwstprc,gujialistgcase_md_val,gujia_listg_case_avg,gujialistgcasemax_prc,gujiaavprc_crpnd_s_mo,gujiaavgwithminvalavg,gujiaavgwithmaxvalavg,gujia_rel_err,gujia_dvtdgr_dsc,gujia_als_prc_rng,gujiaprcminandmax_rng,gujia_dsc,gujia_lmth_bss_prc,gujia_wthr_mnl_ind,gujia_lmth_fnl_prc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,gujianervethruntw_prc,gujia_prc_src,gujiaabtjanlcmdbitval,gujiaabtjanligcaseavg,gujia_indx_lbl,gujia_prim_pty_tp,gujia_comm_tp_val,gujia_ltst_mdl_prc,to_char(gujia_ltst_mdl_dt,'yyyy-mm-dd') as gujia_ltst_mdl_dt,gujia_idxn_mo_moonmo,gujiaabt1yiminmmoonmo,gujia_idxn_yr_moonmo,gujiaabt1yr_samp_scvl,gujiaarnd1kscommavprc from "+database+"avm_community_p_describe_gj_h where gujiaavprc_crpnd_s_mo=${month} AND urbn_id='${cityCode}'"; + + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_fsid + data << r.gujia_comm_id + data << r.urbn_id + data << r.urbn_nm + data << r.admndiv_ecd + data << r.gujia_admndiv_nm + data << r.comm_nm + data << r.gujia_als_dsc + data << r.gujia_bss_prc + data << r.gujia_cfdc_dgr_grd + data << r.gujiart3mlistgcasenum + data << r.gujialistgcaselwstprc + data << r.gujialistgcase_md_val + data << r.gujia_listg_case_avg + data << r.gujialistgcasemax_prc + data << r.gujiaavprc_crpnd_s_mo + data << r.gujiaavgwithminvalavg + data << r.gujiaavgwithmaxvalavg + data << r.gujia_rel_err + data << r.gujia_dvtdgr_dsc + data << r.gujia_als_prc_rng + data << r.gujiaprcminandmax_rng + data << r.gujia_dsc + data << r.gujia_lmth_bss_prc + data << r.gujia_wthr_mnl_ind + data << r.gujia_lmth_fnl_prc + data << r.data_crt_tm + data << r.gujianervethruntw_prc + data << r.gujia_prc_src + data << r.gujiaabtjanlcmdbitval + data << r.gujiaabtjanligcaseavg + data << r.gujia_indx_lbl + data << r.gujia_prim_pty_tp + data << r.gujia_comm_tp_val + data << r.gujia_ltst_mdl_prc + data << r.gujia_ltst_mdl_dt + data << r.gujia_idxn_mo_moonmo + data << r.gujiaabt1yiminmmoonmo + data << r.gujia_idxn_yr_moonmo + data << r.gujiaabt1yr_samp_scvl + data << r.gujiaarnd1kscommavprc + allRows << data + }); + } + createFileToFtp("community_p_describe_gj",allRows, fileName, curDate, false) + } + + def exportCommCaseDetailGj(month){ + def fileName = linux_prefix + "MODEL_000000_000000_COMM_CASE_DETAIL_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "MODEL_000000_000000_COMM_CASE_DETAIL_GJ_ADD_${curDate}_0001.dat" + } + for(def city : citys){ + def cityCode = cityCodeMap.get(city) + def sql = "select gujia_fsid, urbn_id, gujia_comm_id, gujia_case_id,crterm,to_char(data_crt_tm,'yyyy-mm-dd') as data_crt_tm from "+database+"avm_comm_case_detail_gj_h where urbn_id='${cityCode}' and crterm=${month}"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.gujia_fsid + data << r.urbn_id + data << r.gujia_comm_id + data << r.gujia_case_id + data << r.crterm + data << r.data_crt_tm + allRows << data + if(allRows.size()%200000==0){ + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", true); + allRows.clear() + } + }); + if(allRows.size()>0){ + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", true); + } + } + CcbFtpUtils.ftpUpload(null, fileName, curDate) + } + + def exportMd5(){ + Vector files = CcbFtpUtils.listFileNames(curDate) + Iterator fileVec = files.iterator() + def fileRows = [] + while(fileVec.hasNext()){ + ChannelSftp.LsEntry file = fileVec.next() + if(org.apache.commons.lang.StringUtils.isNotBlank(file.getFilename()) && file.getFilename().endsWith(".dat")){ + fileRows << file.getFilename() +":"+ DigestUtils.md5Hex(file.getFilename()) + } + } + def fileName = linux_prefix + "md5.txt" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "md5.txt" + } + createFileToFtp("md5.txt",fileRows, fileName, curDate, false) + } + + + def exportOkFile(){ + def dataCodeFileName = linux_prefix + "${curDate}.ok" + if(CommonDatautils.isWindow()){ + dataCodeFileName = win_prefix + "${curDate}.ok" + } + createFileToFtp("ok txt", [], dataCodeFileName, curDate, false) + } + + + def createFileToFtp(tableName,allRows, fileName, curDate, isAppend){ + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", isAppend); + println tableName+" Êý¾ÝÎļþÊä³öÍê³É......" + CcbFtpUtils.ftpUpload(null, fileName, curDate) +// println tableName+" ÉÏ´«FtpÍê³É......" + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlRonghe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlRonghe.groovy new file mode 100644 index 000000000..8d3d01341 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlRonghe.groovy @@ -0,0 +1,14 @@ +package com.fangjia.script.ccbevaluation.pgsql + +class ExportPgsqlRonghe { + + static main(args) { + def task = new ExportPgsqlRonghe() + task.run(args) + } + + def run(args) { + + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/DongGuangApiTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/DongGuangApiTest.groovy new file mode 100644 index 000000000..2e4cfff4a --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/DongGuangApiTest.groovy @@ -0,0 +1,155 @@ +package com.fangjia.script.gj.test + +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.OpenApiClient +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders + +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +class DongGuangApiTest { + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) + + static main(args) { + def task = new DongGuangApiTest() + task.run() + } + + def run(){ + def rets = findEsData(); + println "ES ¼ÓÔØÍê³É" + def datas = evaluation(rets) + println "ÖØ¹ÀÍê³É" +// ExcelUtils.write("D:\\Êý¾Ý\\¶«Ý¸ÒøÐÐ\\¶«Ý¸ÖعÀ.xlsx", datas, "Êý¾Ý") + FileUtils.batchCreateFile(datas, "/home/goojia/¶«Ý¸ÖعÀ.txt", [], "utf-8", true); + println "txt Éú³É" + exportCsv("/home/goojia/¶«Ý¸ÖعÀ.csv", datas) + println "csv Éú³É" + } + + def evaluation(rets){ + def datas = new CopyOnWriteArrayList() + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(rets.size()) + for(def i=0;i + def paramMap = map.parameter_map + rets << paramMap + if(rets.size()%10000==0){ + println "."+ new Date() + } + }) + return rets; + } + + + def exportCsv(path, list){ + File file = new File(path); + if (file.exists()){ + file.delete(); + } + file.append("Óû§ÊäÈë³ÇÊÐ,") + file.append("Óû§ÊäÈë³ÇÊÐÇøÓò,") + file.append("Óû§ÊäÈë³ÇÊеØÖ·,") + file.append("Óû§ÊäÈë³ÇÊÐÃæ»ý,") + file.append(System.getProperty("line.separator")); + list.each{ + def d = it; + file.append(d[0] + ",","gbk") + file.append(d[1] + ",","gbk") + file.append(d[2] + ",","gbk") + file.append(d[3] + ",","gbk") + if(d[4]){ + file.append(d[4]+",","gbk") + }else{ + file.append(",") + + } + + if(d[5]){ + file.append(d[5] +",","gbk") + }else{ + file.append(",") + + } + + + if(d[6]){ + file.append(d[6] +",","gbk") + }else{ + file.append(",") + + } + if(d[7]){ + file.append(d[7]+",","gbk") + }else{ + file.append(",") + } + file.append(System.getProperty("line.separator")); + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy index c30a1353a..242e1b8a1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy @@ -221,6 +221,9 @@ class ElasticUtils { searchHit.getSourceAsMap().put("id", searchHit.getId()); results << searchHit.getSourceAsMap() } + if(results.size()%100000==0){ + println "."+ new Date() + } SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId); scrollRequest.scroll(scroll); searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy index 7c37cae97..0d0647082 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy @@ -1105,6 +1105,30 @@ public class OpenApiClient { return null } + public static JSONObject addressEvaluation(city, region, address, size){ + try { + HttpRequest request = new HttpRequest(); + Map params = new HashMap(); + params.put("city", city); + params.put("district", region); + params.put("address", address); + params.put("size", String.valueOf(size)); + params.put("token", TOKEN); + String response = request.get("http://open.fangjia.com/property/addressEvaluate", params, "UTF-8"); + JSONObject result = JSONObject.fromObject(response); + if (result.getInt("code") == 200) { + if (result.containsKey("result")) { + return result.getJSONObject("result") + } + } else if (result.getInt("code") == 500) { + println("µØÖ·¹À¼Ûʧ°Ü:${city} ${address} ${size} ${result}") + } + } catch (Exception e) { + e.printStackTrace(); + } + return null + } + public static JSONObject evaluateDistrictPriceByAddress(String city, String address, double size) { try { HttpRequest request = new HttpRequest(); From 20b3ac91e08f7dc1791adabed5566017a2755fc8 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 9 Jan 2023 10:21:57 +0800 Subject: [PATCH 37/99] =?UTF-8?q?=E4=BC=B0=E5=80=BC252=E5=9F=8E86=E5=AF=BC?= =?UTF-8?q?=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/script/ccbevaluation/CcbFtpTestUtils.groovy | 2 +- .../fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpTestUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpTestUtils.groovy index 00ed6bf6e..81958589e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpTestUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpTestUtils.groovy @@ -7,7 +7,7 @@ import org.apache.commons.codec.digest.DigestUtils class CcbFtpTestUtils { - static def ftpServer = "192.168.0.45"//45 + static def ftpServer = "192.168.0.86"//45 static def ftpPort = 17495 static def ftpUser = "ccb" static def ftpPassword = "ccb!@#" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy index 20fd78c85..db9d7956a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy @@ -37,7 +37,7 @@ class ExportOracleAllDataTest { def run(args) { citys = CcbCommonUtils.getRunningCitys(args)//["Ϋ·»","¼ÃÄþ","Íþº£","ÁÙÒÊ","ÃàÑô","¤ÄÏ","Ëþ³Ç","°¢ÀÕÌ©","µÏÇìÖÝ","ÈÕ¿¦Ôò"] //CcbCommonUtils.getRunningCitys(args) - def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")); + def curMonth = "202212"//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")); def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); def lastMonthStr = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM") def new_dists = [] From 800e9467485975008d92d47dd2606771c8000dfb Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 9 Jan 2023 11:11:22 +0800 Subject: [PATCH 38/99] =?UTF-8?q?=E5=86=9C=E8=A1=8C=E6=8A=A5=E5=91=8A?= =?UTF-8?q?=E8=B4=B9=E7=94=A8=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../bkoff/controllers/EvaluationAuditController.java | 6 +++--- .../com/fangjia/bkoff/controllers/YugudanController.java | 6 +++--- .../fangjia-bkoff/src/main/webapp/evaluationAudit/list.ftl | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/EvaluationAuditController.java b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/EvaluationAuditController.java index 52b1ecb2d..a101d08b2 100644 --- a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/EvaluationAuditController.java +++ b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/EvaluationAuditController.java @@ -305,11 +305,11 @@ public class EvaluationAuditController { if(trades!=null && model.getAuditPrice()!=null){ int charge = 0; if(model.getAuditPrice() > 0 && model.getAuditPrice() <= 100){ - charge = 40; + charge = 20; }else if(model.getAuditPrice() > 100 && model.getAuditPrice() <= 300){ - charge = 160; + charge = 70; }else if(model.getAuditPrice() > 300){ - charge = 360; + charge = 80; } trades.setCharge((int) charge); userTradesService.update(trades); diff --git a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/YugudanController.java b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/YugudanController.java index 3a51dcc61..5d15317e7 100644 --- a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/YugudanController.java +++ b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/YugudanController.java @@ -401,11 +401,11 @@ public class YugudanController { double price = model.getPeopleEditPrice(); double charge = 0; if(price > 0 && price <= 100){ - charge = 40; + charge = 20; }else if(price > 100 && price <= 300){ - charge = 160; + charge = 70; }else if(price > 300){ - charge = 360; + charge = 80; } EvaluationAudit audit = evaluationAuditService.findByEvalutionId(String.valueOf(model.getId())); diff --git a/projects/fangjia-bkoff/src/main/webapp/evaluationAudit/list.ftl b/projects/fangjia-bkoff/src/main/webapp/evaluationAudit/list.ftl index efec6b2ea..826290bd0 100644 --- a/projects/fangjia-bkoff/src/main/webapp/evaluationAudit/list.ftl +++ b/projects/fangjia-bkoff/src/main/webapp/evaluationAudit/list.ftl @@ -316,11 +316,11 @@ var price = new Number(val)*10000 var charge = 0; if(price > 0 && price <= 1000000){ - charge = 40; + charge = 20; }else if(price > 1000000 && price <= 3000000){ - charge = 160; + charge = 70; }else if(price > 3000000){ - charge = 360 + charge = 80; } if(price != 0){ $(id).html(charge+'å…ƒ'); From 59eba6b00e0d93e46f8c3958e8399f2658a79de3 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 9 Jan 2023 11:26:18 +0800 Subject: [PATCH 39/99] =?UTF-8?q?=E5=AF=BC=E5=87=BA=E5=BD=93=E5=89=8D?= =?UTF-8?q?=E6=97=B6=E9=97=B4=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ExportOracleAllDataTest.groovy | 34 ++----------------- 1 file changed, 2 insertions(+), 32 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy index db9d7956a..79eda643f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy @@ -20,6 +20,7 @@ class ExportOracleAllDataTest { def cityCodeMap = CcbCommonUtils.findCityCodeValue() def citys = null + def curDate = "20221228"//DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; static main(args) { def task = new ExportOracleAllDataTest() @@ -75,8 +76,7 @@ class ExportOracleAllDataTest { exportCommunityPdescribeGj()// exportCommCaseDetailGj()// -// - def curDate = curMonth.toString()+"28" + // // okÎļþ exportInOk() SSHHelperTest jschCommand = new SSHHelperTest(); @@ -102,7 +102,6 @@ class ExportOracleAllDataTest { allRows << data }); - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; def fileName = "/ccb_evaluation/MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_${curDate}_0001.dat" if (CommonDatautils.isWindow()) { fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\${curDate}\\MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_${curDate}_0001.dat" @@ -133,7 +132,6 @@ class ExportOracleAllDataTest { allRows << data }); - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; def fileName = "/ccb_evaluation/MODEL_000000_000000_WEBSITE_CONFIG_GJ_ALL_${curDate}_0001.dat" if (CommonDatautils.isWindow()) { fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\${curDate}\\MODEL_000000_000000_WEBSITE_CONFIG_GJ_ALL_${curDate}_0001.dat" @@ -168,7 +166,6 @@ class ExportOracleAllDataTest { allRows << data }); } - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; def fileName = "/ccb_evaluation/MODEL_000000_000000_BIGDATA_POI_GJ_ALL_${curDate}_0001.dat" if (CommonDatautils.isWindow()) { fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_POI_GJ_ALL_${curDate}_0001.dat" @@ -196,7 +193,6 @@ class ExportOracleAllDataTest { data << r.SOURCECODE allRows << data }); - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; def fileName = "/ccb_evaluation/MODEL_000000_000000_COMMUNITY_STAGING_GJ_ALL_${curDate}_0001.dat" if (CommonDatautils.isWindow()) { fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_STAGING_GJ_ALL_${curDate}_0001.dat" @@ -230,8 +226,6 @@ class ExportOracleAllDataTest { }); } - - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; def fileName = "/ccb_evaluation/MODEL_000000_000000_BIGDATA_COMMBUILDTYPE_GJ_ALL_${curDate}_0001.dat" if (CommonDatautils.isWindow()) { fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMBUILDTYPE_GJ_ALL_${curDate}_0001.dat" @@ -269,8 +263,6 @@ class ExportOracleAllDataTest { allRows << data }); } - - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; def fileName = "/ccb_evaluation/MODEL_000000_000000_BIGDATA_DISTRICT_GJ_ALL_${curDate}_0001.dat" if (CommonDatautils.isWindow()) { fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_DISTRICT_GJ_ALL_${curDate}_0001.dat" @@ -309,8 +301,6 @@ class ExportOracleAllDataTest { allRows << data }); } - - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; def fileName = "/ccb_evaluation/MODEL_000000_000000_GUIDED_PRICE_GJ_ALL_${curDate}_0001.dat" if (CommonDatautils.isWindow()) { fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\${curDate}\\MODEL_000000_000000_GUIDED_PRICE_GJ_ALL_${curDate}_0001.dat" @@ -346,8 +336,6 @@ class ExportOracleAllDataTest { allRows << data }); } - - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; def fileName = "/ccb_evaluation/MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.dat" if (CommonDatautils.isWindow()) { fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.dat" @@ -392,7 +380,6 @@ class ExportOracleAllDataTest { } } - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; def fileName = "/ccb_evaluation/MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_${curDate}_0001.dat" if (CommonDatautils.isWindow()) { fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_${curDate}_0001.dat" @@ -513,7 +500,6 @@ class ExportOracleAllDataTest { data << r.SOURCECODE.replaceAll(~'\\r', '') allRows << data }) - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; def fileName = "/ccb_evaluation/MODEL_000000_000000_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" if (CommonDatautils.isWindow()) { fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\${curDate}\\MODEL_000000_000000_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" @@ -530,7 +516,6 @@ class ExportOracleAllDataTest { def exportThirdPartyPrice2(new_dists) { - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; def fileName = "/ccb_evaluation/MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" if (CommonDatautils.isWindow()) { fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" @@ -638,7 +623,6 @@ class ExportOracleAllDataTest { }); } - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; def fileName = "/ccb_evaluation/MODEL_000000_000000_BIGDATA_COMMUNITY_GJ_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMUNITY_GJ_ALL_${curDate}_0001.dat" @@ -654,7 +638,6 @@ class ExportOracleAllDataTest { def exportBigdataForsale(new_dists){ //["±±¾©","ÉϺ£"] - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; def fileName = "/ccb_evaluation/MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat" @@ -785,7 +768,6 @@ class ExportOracleAllDataTest { def exportAvmEvaluationGJNB(){ - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; def fileName = "/ccb_evaluation/MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_${curDate}_0001.dat" def fileName1 = "/ccb_evaluation/MODEL_000000_000000_AVM_EVALUATION_GJ_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ @@ -850,7 +832,6 @@ class ExportOracleAllDataTest { } def exportAVMEvaluationItemGJNB(){ - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; def fileName = "/ccb_evaluation/MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_${curDate}_0001.dat" def fileName1 = "/ccb_evaluation/MODEL_000000_000000_AVM_EVALUATION_ITEM_GJ_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ @@ -904,7 +885,6 @@ class ExportOracleAllDataTest { def exportCommunityBasePrice(){ def allRows = [] - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; def fileName = "/ccb_evaluation/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat" def fileName1 = "/ccb_evaluation/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ @@ -959,7 +939,6 @@ class ExportOracleAllDataTest { def exportCommunityBasePriceGj(){ def allRows = [] - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; def fileName = "/ccb_evaluation/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" @@ -1026,7 +1005,6 @@ class ExportOracleAllDataTest { allRows << data }); } - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" def fileName = "/ccb_evaluation/MODEL_000000_000000_CITY_PRICE_GJ_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\${curDate}\\MODEL_000000_000000_CITY_PRICE_GJ_ADD_${curDate}_0001.dat" @@ -1059,9 +1037,6 @@ class ExportOracleAllDataTest { allRows << data }); } - - - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" def fileName = "/ccb_evaluation/MODEL_000000_000000_DISTRICT_PRICE_GJ_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\${curDate}\\MODEL_000000_000000_DISTRICT_PRICE_GJ_ADD_${curDate}_0001.dat" @@ -1075,7 +1050,6 @@ class ExportOracleAllDataTest { def exportCommunityPdescribeGj(){ - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" def fileName = "/ccb_evaluation/MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat" @@ -1150,7 +1124,6 @@ class ExportOracleAllDataTest { def exportCommCaseDetailGj(){ - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" def fileName = "/ccb_evaluation/MODEL_000000_000000_COMM_CASE_DETAIL_GJ_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\${curDate}\\MODEL_000000_000000_COMM_CASE_DETAIL_GJ_ADD_${curDate}_0001.dat" @@ -1187,7 +1160,6 @@ class ExportOracleAllDataTest { } def exportLastFile(){ - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" def dataCodeFileName = "/ccb_evaluation/${curDate}.ok" if(CommonDatautils.isWindow()){ dataCodeFileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\${curDate}\\${curDate}.ok" @@ -1197,7 +1169,6 @@ class ExportOracleAllDataTest { } def exportOkFile(){ - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" def dataCodeFileName = "/ccb_evaluation/${curDate}.ok" if(CommonDatautils.isWindow()){ dataCodeFileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\${curDate}\\${curDate}.ok" @@ -1207,7 +1178,6 @@ class ExportOracleAllDataTest { } def exportInOk(){ - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" Vector files = CcbFtpTestUtils.listFileNames(null, curDate) Iterator fileVec = files.iterator() def fileRows = [] From eb07e7823ce78c189d3b75346e1f40bcbd62e128 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 9 Jan 2023 17:20:57 +0800 Subject: [PATCH 40/99] =?UTF-8?q?=E8=9E=8D=E5=90=88=E4=BF=AE=E6=94=B9?= =?UTF-8?q?=EF=BC=8C=E6=96=B0=E5=A2=9E6=E5=BC=A0=E8=A1=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ronghe/ExportDistrictRongHe.groovy | 267 +++++++++++++++++- .../thirdPriceCrawl/SynchDistrictPrice.groovy | 2 +- .../TestCrawlTotalCount.groovy | 4 +- 3 files changed, 267 insertions(+), 6 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy index 2a16ad76e..3c889c1b6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy @@ -4,10 +4,12 @@ import com.fangjia.script.BaseBatch import com.fangjia.script.BatchOwner import com.fangjia.script.Watcher import com.fangjia.script.ccbevaluation.CcbCommonUtils +import com.fangjia.script.ccbevaluation.CcbFtpTestUtils import com.fangjia.script.ccbevaluation.CcbFtpUtils import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.ccb.FileUtils import com.fangjia.script.util.DateUtils +import com.jcraft.jsch.ChannelSftp import groovy.sql.Sql import java.sql.ResultSet @@ -28,10 +30,11 @@ class ExportDistrictRongHe extends BaseBatch{ @Override void process(Object args) { citys = CcbCommonUtils.getRunningCitys(args) + def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd") def new_dists = []//["350604","350605","410307","410308","450181"] //°´Ììµ¼£¨Ã¿ÌìÒ»¸öÎļþ¼Ð£© - exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-1), "yyyy-MM-dd"), new_dists) + exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists) if(curDate.toString().endsWith("10")){ exportBigdataCommunityGJRH()// @@ -51,8 +54,15 @@ class ExportDistrictRongHe extends BaseBatch{ exportEvaluateRate() //ÒòΪÈÚºÏÔݲ»Çп⣬ËùÒÔÐèÒª¼Ó´«Ò»ÕŵÚÈý·½¾ù¼Û±íµ½ÈÚºÏÖУ¬ ´Ó1ÔÂ10ÈÕ¿ªÊ¼´«Êä - def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); exportThirdPartyPrice2(lastMonth, new_dists) + + + //1ÔÂ10ÈÕ ÐÂÔö5ÕÅ±í£¬µ¼92³Ç£¬ÔöÁ¿Êý¾Ý£¬2ÔÂ10ÈÕµ¼344³Ç£¬ÆäÖÐ252³ÇËùÓÐÀúÊ·Êý¾Ý£¬92³ÇÔöÁ¿£¬3Ôºó¿ÉÄܲ»ÓÃÌṩÊý¾Ý + exportCommunityBasePrice(lastMonth) + exportCommunityPdescribeGj(lastMonth) + exportGuideStatistics(lastMonth) + exportBigdataSupplierStatistics(lastMonth) + } SSHHelper jschCommand = new SSHHelper(); System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}")); @@ -831,7 +841,12 @@ class ExportDistrictRongHe extends BaseBatch{ def rets2 = getThirdPartyDatas(sql2) allRows.addAll(rets2) }else{ - def sql = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from BIGDATA_THIRD_PARTY_PRICE_GJ where CITYID='${cityCode}' and price_date=${price_date} "; + def sql = null + if(CcbCommonUtils.citys_92.contains(city)){ + sql = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from BIGDATA_THIRD_PARTY_PRICE_GJ where CITYID='${cityCode}' and price_date=${price_date} "; + }else{ + sql = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from BIGDATA_THIRD_PARTY_PRICE_GJ where CITYID='${cityCode}'"; + } allRows = getThirdPartyDatas(sql) } if(allRows && allRows.size()>0){ @@ -880,6 +895,252 @@ class ExportDistrictRongHe extends BaseBatch{ println "BIGDATA_BUILDPROJ_FJW ÉÏ´«FtpÍê³É......" } + def exportCommunityBasePrice(yearMonth){ + + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = null; + if(CcbCommonUtils.citys_92.contains(city)){ + sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, FAKE_RATE, DIFF_RATE, CONFI_RATE, CORR_RATE, CONFIDENCE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_base_price_gj where year_month=${yearMonth} AND CITYID='${cityCode}'"; + }else{ + sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, FAKE_RATE, DIFF_RATE, CONFI_RATE, CORR_RATE, CONFIDENCE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_base_price_gj where CITYID='${cityCode}'"; + } + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.ID + data << r.CITYID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.COMMID + data << r.COMMNAME + data << r.YEAR_MONTH + data << r.CALC_AVG_PRICE + data << r.AVG_PRICE + data << r.CALC_TYPE + data << r.FAKE_RATE + data << r.DIFF_RATE + data << r.CONFI_RATE + data << r.CORR_RATE + data << r.CONFIDENCE + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + } + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" + def fileName = "/ccb_rh/FJW_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat" + } + def headers = [] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "COMMUNITY_BASE_PRICE_GJNB Êý¾ÝÎļþÊä³öÍê³É......" + CcbFtpUtils.ftpUpload(null, fileName, curDate) + println "COMMUNITY_BASE_PRICE_GJNB ÉÏ´«FtpÍê³É......" + } + + + def exportCommunityPdescribeGj(month){ + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" + def fileName = "/ccb_rh/FJW_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat" + } + def headers = [] + def allRows = [] + + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = null; + if(CcbCommonUtils.citys_92.contains(city)){ + sql = "select ID,COMMUNITY_ID,CITYID ,CITY_NAME,DISTRICT_CODE ,DISTRICT_NAME,COMMUNITY_NAME,PQ_NAME,BASE_PRICE,CONFIDENCE_LEVEL,CNT,MIN_PRICE,MEDIANPRICE,MEANPRICE,MAX_PRICE,TIME,FIRSTDATAT,THIRDDATA,RELATIVEERROR,SKEWNESS,PQ_PRICE_RANGE,DISTRICT_PRICE_RANGE ,DESCRIPTION,PRIV_BASE_PRICE,IS_ARTTFICIAL,PRIV_FINAL_PRICE,to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE,PRICE,PRICE_SOURCE,CASE_MD_PRICE,CASE_AVG_PRICE,INDEX_LABEL,MAIN_PROPERTY,COMMTYPE,NEW_DEAL_PRICE,to_char(NEW_DEAL_DATE,'yyyy-mm-dd') as NEW_DEAL_DATE,PRICE_MOM_RATIO,MAX_MOM_RATIO,PRICE_YOY_RATIO ,SATUR_RATIO,NEIGHBOR_PRICE from community_p_describe_gj where time=${month} AND CITYID='${cityCode}'"; + }else{ + sql = "select ID,COMMUNITY_ID,CITYID ,CITY_NAME,DISTRICT_CODE ,DISTRICT_NAME,COMMUNITY_NAME,PQ_NAME,BASE_PRICE,CONFIDENCE_LEVEL,CNT,MIN_PRICE,MEDIANPRICE,MEANPRICE,MAX_PRICE,TIME,FIRSTDATAT,THIRDDATA,RELATIVEERROR,SKEWNESS,PQ_PRICE_RANGE,DISTRICT_PRICE_RANGE ,DESCRIPTION,PRIV_BASE_PRICE,IS_ARTTFICIAL,PRIV_FINAL_PRICE,to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE,PRICE,PRICE_SOURCE,CASE_MD_PRICE,CASE_AVG_PRICE,INDEX_LABEL,MAIN_PROPERTY,COMMTYPE,NEW_DEAL_PRICE,to_char(NEW_DEAL_DATE,'yyyy-mm-dd') as NEW_DEAL_DATE,PRICE_MOM_RATIO,MAX_MOM_RATIO,PRICE_YOY_RATIO ,SATUR_RATIO,NEIGHBOR_PRICE from community_p_describe_gj where CITYID='${cityCode}'"; + } + + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.ID + data << r.COMMUNITY_ID + data << r.CITYID + data << r.CITY_NAME + data << r.DISTRICT_CODE + data << r.DISTRICT_NAME + data << r.COMMUNITY_NAME + data << r.PQ_NAME + data << r.BASE_PRICE + data << r.CONFIDENCE_LEVEL + data << r.CNT + data << r.MIN_PRICE + data << r.MEDIANPRICE + data << r.MEANPRICE + data << r.MAX_PRICE + data << r.TIME + data << r.FIRSTDATAT + data << r.THIRDDATA + data << r.RELATIVEERROR + data << r.SKEWNESS + data << r.PQ_PRICE_RANGE + data << r.DISTRICT_PRICE_RANGE + data << r.DESCRIPTION + data << r.PRIV_BASE_PRICE + data << r.IS_ARTTFICIAL + data << r.PRIV_FINAL_PRICE + data << r.CREATE_DATE + data << r.PRICE + data << r.PRICE_SOURCE + data << r.CASE_MD_PRICE + data << r.CASE_AVG_PRICE + data << r.INDEX_LABEL + data << r.MAIN_PROPERTY + data << r.COMMTYPE + data << r.NEW_DEAL_PRICE + data << r.NEW_DEAL_DATE + data << r.PRICE_MOM_RATIO + data << r.MAX_MOM_RATIO + data << r.PRICE_YOY_RATIO + data << r.SATUR_RATIO + data << r.NEIGHBOR_PRICE + allRows << data + }); + } + println allRows.size() + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + + println "community_p_describe_gj Êý¾ÝÎļþÊä³öÍê³É......" + CcbFtpUtils.ftpUpload(null, fileName, curDate) + println "community_p_describe_gj ÉÏ´«FtpÍê³É......" + } + + def exportGuideStatistics(time){ + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" + def fileName = "/ccb_rh/FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat" + } + def headers = [] + def allRows = [] + + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = null; + if(CcbCommonUtils.citys_92.contains(city)){ + sql = "select MODEL_TYPE,CITYID,CITYNAME,YEAR_MONTH,COMM_SUM_COUNT,COMM_COUNT,RATIO,TYPE from GUIDE_STATISTICS where cityid='${cityCode}' and year_month=${time}"; + }else{ + sql = "select MODEL_TYPE,CITYID,CITYNAME,YEAR_MONTH,COMM_SUM_COUNT,COMM_COUNT,RATIO,TYPE from GUIDE_STATISTICS where cityid='${cityCode}'"; + } + + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.MODEL_TYPE + data << r.CITYID + data << r.CITYNAME + data << r.YEAR_MONTH + data << r.COMM_SUM_COUNT + data << r.COMM_COUNT + data << r.RATIO + data << r.TYPE + allRows << data + }); + } + println allRows.size() + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + + println "guide_statistics Êý¾ÝÎļþÊä³öÍê³É......" + CcbFtpUtils.ftpUpload(null, fileName, curDate) + println "guide_statistics ÉÏ´«FtpÍê³É......" + } + + + def exportBigdataSupplierStatistics(time){ + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" + def fileName = "/ccb_rh/FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_${curDate}_0001.dat" + } + def headers = [] + def allRows = [] + + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = null; + if(CcbCommonUtils.citys_92.contains(city)){ + sql = "select CITYID,CITYNAME,YEAR_MONTH,DATA_SOURCE,THIRD_COMM_COUNT,MODEL_TYPE from BIGDATA_SUPPLIER_STATISTICS where cityid='${cityCode}' and year_month=${time}"; + }else{ + sql = "select CITYID,CITYNAME,YEAR_MONTH,DATA_SOURCE,THIRD_COMM_COUNT,MODEL_TYPE from BIGDATA_SUPPLIER_STATISTICS where cityid='${cityCode}'"; + } + + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.CITYID + data << r.CITYNAME + data << r.YEAR_MONTH + data << r.DATA_SOURCE + data << r.THIRD_COMM_COUNT + data << r.MODEL_TYPE + allRows << data + }); + } + println allRows.size() + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + + println "bigdata_supplier_statistics Êý¾ÝÎļþÊä³öÍê³É......" + CcbFtpUtils.ftpUpload(null, fileName, curDate) + println "bigdata_supplier_statistics ÉÏ´«FtpÍê³É......" + } + + def exportBigdataModelingBase(time){ + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" + def fileName = "/ccb_rh/FJW_000000_000000_BIGDATA_MODELING_BASE_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_MODELING_BASE_ADD_${curDate}_0001.dat" + } + def headers = [] + def allRows = [] + + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = null; + if(CcbCommonUtils.citys_92.contains(city)){ + sql = "select CITY_CODE,CITYNAME,YEAR_MONTH,ESTIMATE_DEAL_COUNT,ESTIMATE_COMM_COUNT,DEAL_COUNT,DEAL_COMM_COUNT,SALE_COUNT,SALE_COMM_COUNT,MODEL_TYPE,TYPE from BIGDATA_MODELING_BASE where city_code='${cityCode}' and year_month=${time}"; + }else{ + sql = "select CITY_CODE,CITYNAME,YEAR_MONTH,ESTIMATE_DEAL_COUNT,ESTIMATE_COMM_COUNT,DEAL_COUNT,DEAL_COMM_COUNT,SALE_COUNT,SALE_COMM_COUNT,MODEL_TYPE,TYPE from BIGDATA_MODELING_BASE where city_code='${cityCode}'"; + } + + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.CITY_CODE + data << r.CITYNAME + data << r.YEAR_MONTH + data << r.ESTIMATE_DEAL_COUNT + data << r.ESTIMATE_COMM_COUNT + data << r.DEAL_COUNT + data << r.DEAL_COMM_COUNT + data << r.SALE_COUNT + data << r.SALE_COMM_COUNT + data << r.MODEL_TYPE + data << r.TYPE + allRows << data + }); + } + println allRows.size() + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + + println "bigdata_modeling_base Êý¾ÝÎļþÊä³öÍê³É......" + CcbFtpUtils.ftpUpload(null, fileName, curDate) + println "bigdata_modeling_base ÉÏ´«FtpÍê³É......" + } + + + def exportBigdataCommlayoutpicFjw(){ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy index 7cfd81e44..3e43ef6a3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy @@ -26,7 +26,7 @@ class SynchDistrictPrice extends BaseBatch{ // "¼ªÎÝ":"¼ªÎÝ","365ÌÔ·¿":"365ÌÔ·¿","ÎÒ°®ÎÒ¼Ò":"ÎÒ°®ÎÒ¼Ò","Öî¸ðÕÒ·¿APP":"Öî¸ðÕÒ·¿","±´¿Ç":"±´¿ÇÕÒ·¿","Á´¼Ò":"Á´¼Ò","Q·¿":"Q·¿","ÂóÌï":"ÂóÌï", // "ÖÐÔ­µØ²ú":"ÖÐÔ­µØ²ú","°²¾Ó¿ÍС³ÌÐò":"°²¾Ó¿Í","ËÑ·¿":"ËÑ·¿","ÀÖÓмÒ":"ÀÖÓмÒ","C21":"C21","³ÇÊз¿²ú":"³ÇÊз¿²ú","Íò©Ê¿PC¶Ë":"Íò©Ê¿" - def esSources = ["¼ªÎÝ":"¼ªÎÝ","365ÌÔ·¿":"365ÌÔ·¿","ÎÒ°®ÎÒ¼Ò":"ÎÒ°®ÎÒ¼Ò","Öî¸ðÕÒ·¿APP":"Öî¸ðÕÒ·¿","±´¿Ç":"±´¿ÇÕÒ·¿","Á´¼Ò":"Á´¼Ò","Q·¿":"Q·¿","ÂóÌï":"ÂóÌï","ÖÐÔ­µØ²ú":"ÖÐÔ­µØ²ú","°²¾Ó¿ÍС³ÌÐò":"°²¾Ó¿Í","ËÑ·¿":"ËÑ·¿","ÀÖÓмÒ":"ÀÖÓмÒ","C21":"C21","³ÇÊз¿²ú":"³ÇÊз¿²ú","Íò©Ê¿PC¶Ë":"Íò©Ê¿"] + def esSources = ["¼ªÎÝ":"¼ªÎÝ","±´¿ÇÕÒ·¿":"±´¿ÇÕÒ·¿","ÀÖÓмÒ":"ÀÖÓмÒ","365ÌÔ·¿":"365ÌÔ·¿","ÎÒ°®ÎÒ¼Ò":"ÎÒ°®ÎÒ¼Ò","Öî¸ðÕÒ·¿APP":"Öî¸ðÕÒ·¿","Á´¼Ò":"Á´¼Ò","Q·¿":"Q·¿","ÂóÌï":"ÂóÌï","ÖÐÔ­µØ²ú":"ÖÐÔ­µØ²ú","°²¾Ó¿ÍС³ÌÐò":"°²¾Ó¿Í","ËÑ·¿":"ËÑ·¿","C21":"C21","³ÇÊз¿²ú":"³ÇÊз¿²ú","Íò©Ê¿PC¶Ë":"Íò©Ê¿"] static main(args) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy index 56fbf3de1..df803e640 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy @@ -14,8 +14,8 @@ class TestCrawlTotalCount { def run(){ def sources = ["·¿¹À¹À","Öî¸ðÕÒ·¿","°²¾Ó¿Í","ËÑ·¿","ÖÐÔ­µØ²ú","¸Ï¼¯","Á´¼Ò","Íò©Ê¿","³ÇÊз¿²ú","±´¿ÇÕÒ·¿","Q·¿","Q·¿(ÐÂ)","ÖÐÔ­µØ²ú(ÐÂ)","ÀÖÓмÒ","ÀÖÓмÒ(ÐÂ)","·¿Ì½","·¿¶à¶à","С×ʼÒ","ÎÒ°®ÎÒ¼Ò","ÂóÌï","·¿Í¾","C21","ÀÖ¾Ó","·¿²ú³¬ÊÐ","58ͬ³Ç","365ÌÔ·¿","¼ªÎÝ"] for(def source :sources){ - def count1 = findTotalCount(source, 202210); - def count2 = findTotalCount(source, 202211); + def count1 = findTotalCount(source, 202211); + def count2 = findTotalCount(source, 202212); println source +" "+count1 +" "+count2 } } From 68a9c8379b562b74684e266c8cb1d025bdb1d7ba Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 9 Jan 2023 18:53:07 +0800 Subject: [PATCH 41/99] update --- .../ronghe/ExportDistrictRongHe.groovy | 51 ++++++++++--------- 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy index 3c889c1b6..48e927791 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy @@ -34,34 +34,35 @@ class ExportDistrictRongHe extends BaseBatch{ def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd") def new_dists = []//["350604","350605","410307","410308","450181"] //°´Ììµ¼£¨Ã¿ÌìÒ»¸öÎļþ¼Ð£© - exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists) +// exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists) if(curDate.toString().endsWith("10")){ - exportBigdataCommunityGJRH()// - exportBigdataBuildingFJW()// - exportBigdataCommfeeGj()// +// exportBigdataCommunityGJRH()// +// exportBigdataBuildingFJW()// +// exportBigdataCommfeeGj()// +// +// exportBigdataCommbuildstruGJ()// +// exportBigdataCommhousetypeGJ()// +// exportBigdataCommbuildyearGj()// +// exportBigdataCommhouseuseGJ()// +// exportBigdataPositionGJ()// +// exportBigdataDistrictFJW()// +// exportBigdataAliasFJW()// +// +// exportBigdataBuildprojFjw() +// exportBigdataCommlayoutpicFjw() +// exportEvaluateRate() - exportBigdataCommbuildstruGJ()// - exportBigdataCommhousetypeGJ()// - exportBigdataCommbuildyearGj()// - exportBigdataCommhouseuseGJ()// - exportBigdataPositionGJ()// - exportBigdataDistrictFJW()// - exportBigdataAliasFJW()// - - exportBigdataBuildprojFjw() - exportBigdataCommlayoutpicFjw() - exportEvaluateRate() - - //ÒòΪÈÚºÏÔݲ»Çп⣬ËùÒÔÐèÒª¼Ó´«Ò»ÕŵÚÈý·½¾ù¼Û±íµ½ÈÚºÏÖУ¬ ´Ó1ÔÂ10ÈÕ¿ªÊ¼´«Êä +// //ÒòΪÈÚºÏÔݲ»Çп⣬ËùÒÔÐèÒª¼Ó´«Ò»ÕŵÚÈý·½¾ù¼Û±íµ½ÈÚºÏÖУ¬ ´Ó1ÔÂ10ÈÕ¿ªÊ¼´«Êä exportThirdPartyPrice2(lastMonth, new_dists) - - - //1ÔÂ10ÈÕ ÐÂÔö5ÕÅ±í£¬µ¼92³Ç£¬ÔöÁ¿Êý¾Ý£¬2ÔÂ10ÈÕµ¼344³Ç£¬ÆäÖÐ252³ÇËùÓÐÀúÊ·Êý¾Ý£¬92³ÇÔöÁ¿£¬3Ôºó¿ÉÄܲ»ÓÃÌṩÊý¾Ý +// +// +// //1ÔÂ10ÈÕ ÐÂÔö5ÕÅ±í£¬µ¼92³Ç£¬ÔöÁ¿Êý¾Ý£¬2ÔÂ10ÈÕµ¼344³Ç£¬ÆäÖÐ252³ÇËùÓÐÀúÊ·Êý¾Ý£¬92³ÇÔöÁ¿£¬3Ôºó¿ÉÄܲ»ÓÃÌṩÊý¾Ý exportCommunityBasePrice(lastMonth) exportCommunityPdescribeGj(lastMonth) exportGuideStatistics(lastMonth) exportBigdataSupplierStatistics(lastMonth) + exportBigdataModelingBase(lastMonth) } SSHHelper jschCommand = new SSHHelper(); @@ -939,7 +940,7 @@ class ExportDistrictRongHe extends BaseBatch{ def headers = [] FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); println "COMMUNITY_BASE_PRICE_GJNB Êý¾ÝÎļþÊä³öÍê³É......" - CcbFtpUtils.ftpUpload(null, fileName, curDate) + CcbFtpUtils.ftpUpload("rh", fileName, curDate) println "COMMUNITY_BASE_PRICE_GJNB ÉÏ´«FtpÍê³É......" } @@ -1013,7 +1014,7 @@ class ExportDistrictRongHe extends BaseBatch{ FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); println "community_p_describe_gj Êý¾ÝÎļþÊä³öÍê³É......" - CcbFtpUtils.ftpUpload(null, fileName, curDate) + CcbFtpUtils.ftpUpload("rh", fileName, curDate) println "community_p_describe_gj ÉÏ´«FtpÍê³É......" } @@ -1053,7 +1054,7 @@ class ExportDistrictRongHe extends BaseBatch{ FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); println "guide_statistics Êý¾ÝÎļþÊä³öÍê³É......" - CcbFtpUtils.ftpUpload(null, fileName, curDate) + CcbFtpUtils.ftpUpload("rh", fileName, curDate) println "guide_statistics ÉÏ´«FtpÍê³É......" } @@ -1092,7 +1093,7 @@ class ExportDistrictRongHe extends BaseBatch{ FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); println "bigdata_supplier_statistics Êý¾ÝÎļþÊä³öÍê³É......" - CcbFtpUtils.ftpUpload(null, fileName, curDate) + CcbFtpUtils.ftpUpload("rh", fileName, curDate) println "bigdata_supplier_statistics ÉÏ´«FtpÍê³É......" } @@ -1135,7 +1136,7 @@ class ExportDistrictRongHe extends BaseBatch{ FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); println "bigdata_modeling_base Êý¾ÝÎļþÊä³öÍê³É......" - CcbFtpUtils.ftpUpload(null, fileName, curDate) + CcbFtpUtils.ftpUpload("rh", fileName, curDate) println "bigdata_modeling_base ÉÏ´«FtpÍê³É......" } From 4b869b2797b040328a0278f917195a1478028c0e Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 10 Jan 2023 10:46:44 +0800 Subject: [PATCH 42/99] =?UTF-8?q?=E5=9F=8E=E5=B8=82=E8=A1=8C=E6=83=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy index 90a46050c..e7c767739 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy @@ -34,8 +34,8 @@ class BigdataCitypriceBatch extends BaseBatch{ @Override void process(Object args) { def citys = CcbCommonUtils.getRunningCitys(args) - def startDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-1), "yyyyMMdd").toInteger(); - def endDate = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(new Date(),-1), "yyyyMMdd").toInteger(); + def startDate = 20221201//DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-1), "yyyyMMdd").toInteger(); + def endDate = 20221231//DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(new Date(),-1), "yyyyMMdd").toInteger(); def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() cityRun(citys, startDate, endDate, cityCodeDataMap) From 3bb6bee02bc11a8c6ab3c943eb55ca871c95dfee Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 10 Jan 2023 11:20:28 +0800 Subject: [PATCH 43/99] =?UTF-8?q?=E6=97=B6=E9=97=B4=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ccbevaluation/ronghe/BigdataCitypriceBatch.groovy | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy index e7c767739..2d248a235 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy @@ -27,6 +27,8 @@ class BigdataCitypriceBatch extends BaseBatch{ public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80) + def curRunDate = 202212//DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-1), "yyyyMM").toInteger(); + static main(args) { runBatch(new BigdataCitypriceBatch(), args) } @@ -238,7 +240,7 @@ class BigdataCitypriceBatch extends BaseBatch{ if (sell_avg_price || rent_avg_price) { def row = [] row << cityCodeDataMap.get(dist.city) - row << DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-1), "yyyyMM").toInteger(); + row << curRunDate//DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-1), "yyyyMM").toInteger(); row << dist._id.toString() if (sell_avg_price) { @@ -362,7 +364,7 @@ class BigdataCitypriceBatch extends BaseBatch{ if (sell_avg_price || rent_avg_price) { def row = [] row << cityCodeDataMap.get(city) - row << DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-1), "yyyyMM").toInteger(); + row << curRunDate//DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-1), "yyyyMM").toInteger(); if(region){ row << regionCodeDataMap.district_code From 5b96e0c12bcb1369ccb4e023f4f69902bee63631 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 10 Jan 2023 15:56:54 +0800 Subject: [PATCH 44/99] =?UTF-8?q?=E4=B8=B4=E6=97=B6=E5=AF=BC1=E6=9C=88?= =?UTF-8?q?=E6=95=B0=E6=8D=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ronghe/ExportDistrictRongHe.groovy | 179 ++++++++++++++++-- 1 file changed, 165 insertions(+), 14 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy index 892bd5e6c..919bff95a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy @@ -29,6 +29,10 @@ class ExportDistrictRongHe extends BaseBatch{ @Override void process(Object args) { + + def queryDate = 202212//DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger() + + citys = CcbCommonUtils.getRunningCitys(args) def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd") @@ -36,7 +40,8 @@ class ExportDistrictRongHe extends BaseBatch{ //°´Ììµ¼£¨Ã¿ÌìÒ»¸öÎļþ¼Ð£© exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists) - if(curDate.toString().endsWith("10")){ + +// if(curDate.toString().endsWith("10")){ exportBigdataCommunityGJRH()// exportBigdataBuildingFJW()// exportBigdataCommfeeGj()// @@ -53,21 +58,26 @@ class ExportDistrictRongHe extends BaseBatch{ exportBigdataCommlayoutpicFjw() exportEvaluateRate() -// //ÒòΪÈÚºÏÔݲ»Çп⣬ËùÒÔÐèÒª¼Ó´«Ò»ÕŵÚÈý·½¾ù¼Û±íµ½ÈÚºÏÖУ¬ ´Ó1ÔÂ10ÈÕ¿ªÊ¼´«Êä - def thirdPartyMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")); - exportThirdPartyPrice2(thirdPartyMonth, new_dists) +// def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger() + exportBigdataCityPrice(queryDate) + exportBigdataDistPrice(queryDate) + exportBigdataCommPrice(queryDate) +//// //ÒòΪÈÚºÏÔݲ»Çп⣬ËùÒÔÐèÒª¼Ó´«Ò»ÕŵÚÈý·½¾ù¼Û±íµ½ÈÚºÏÖУ¬ ´Ó1ÔÂ10ÈÕ¿ªÊ¼´«Ê䣬³¤ÆÚ¹© +// def thirdPartyMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")); +// exportThirdPartyPrice2(thirdPartyMonth, new_dists) +// +// +// //1ÔÂ10ÈÕ ÐÂÔö5ÕÅ±í£¬µ¼92³Ç£¬ÔöÁ¿Êý¾Ý£¬2ÔÂ10ÈÕµ¼344³Ç£¬ÆäÖÐ252³ÇËùÓÐÀúÊ·Êý¾Ý£¬92³ÇÔöÁ¿£¬3Ôºó¿ÉÄܲ»ÓÃÌṩÊý¾Ý +// exportCommunityBasePrice(lastMonth) +// exportCommunityPdescribeGj(lastMonth) +// exportGuideStatistics(lastMonth) +// exportBigdataSupplierStatistics(lastMonth) +// exportBigdataModelingBase(lastMonth) - //1ÔÂ10ÈÕ ÐÂÔö5ÕÅ±í£¬µ¼92³Ç£¬ÔöÁ¿Êý¾Ý£¬2ÔÂ10ÈÕµ¼344³Ç£¬ÆäÖÐ252³ÇËùÓÐÀúÊ·Êý¾Ý£¬92³ÇÔöÁ¿£¬3Ôºó¿ÉÄܲ»ÓÃÌṩÊý¾Ý - exportCommunityBasePrice(lastMonth) - exportCommunityPdescribeGj(lastMonth) - exportGuideStatistics(lastMonth) - exportBigdataSupplierStatistics(lastMonth) - exportBigdataModelingBase(lastMonth) - - } - SSHHelper jschCommand = new SSHHelper(); - System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}")); +// } +// SSHHelper jschCommand = new SSHHelper(); +// System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}")); exportOk(curDate) @@ -1141,6 +1151,147 @@ class ExportDistrictRongHe extends BaseBatch{ println "bigdata_modeling_base ÉÏ´«FtpÍê³É......" } + def exportBigdataCityPrice(queryDate){ + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" + def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_CITYPRICE_FJW_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_CITYPRICE_FJW_ADD_${curDate}_0001.dat" + } + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + // def sql = "select CITYID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_CITYPRICE where CITYID='${cityCode}' and YMID=${queryDate}";// + def sql = "select CITYID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_CITYPRICE where CITYID='${cityCode}'";// + + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.YMID + data << r.SALEPRICE?r.SALEPRICE:'' + data << r.RENTPRICE?r.RENTPRICE:'' + if(r.SALECOUNT>0) { + data << r.SALECOUNT + }else{ + data << '' + } + if(r.RENTCOUNT>0){ + data << r.RENTCOUNT + }else{ + data << '' + } + data << r.UPDATETIME + data << r.HOUSEUSE + data << r.SALEBLDGAREA?r.SALEBLDGAREA:'' + data << r.RENTBLDGAREA?r.RENTBLDGAREA:'' + data << r.SALETOTALPRICE?r.SALETOTALPRICE:'' + data << r.RENTTOTALPRICE?r.RENTTOTALPRICE:'' + data << r.DATATYPE + data << r.SDATERANGE + data << r.RDATERANGE + data << r.SOURCECODE + allRows << data + }); + } + + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) + println "BIGDATA_CITYPRICE ÉÏ´«FtpÍê³É......" + } + + + def exportBigdataDistPrice(queryDate){ + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" + def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_DISTPRICE_FJW_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_DISTPRICE_FJW_ADD_${curDate}_0001.dat" + } + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select CITYID,DISTID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_DISTPRICE where CITYID='${cityCode}'";// and YMID=${queryDate} + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.YMID + data << r.DISTID + data << r.SALEPRICE?r.SALEPRICE:'' + data << r.RENTPRICE?r.RENTPRICE:'' + if(r.SALECOUNT>0) { + data << r.SALECOUNT + }else{ + data << '' + } + if(r.RENTCOUNT>0){ + data << r.RENTCOUNT + }else{ + data << '' + } + data << r.UPDATETIME + data << r.HOUSEUSE + data << r.SALEBLDGAREA?r.SALEBLDGAREA:'' + data << r.RENTBLDGAREA?r.RENTBLDGAREA:'' + data << r.SALETOTALPRICE?r.SALETOTALPRICE:'' + data << r.RENTTOTALPRICE?r.RENTTOTALPRICE:'' + data << r.DATATYPE + data << r.SDATERANGE + data << r.RDATERANGE + data << r.SOURCECODE + allRows << data + }); + } + + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) + println "BIGDATA_DISTPRICE ÉÏ´«FtpÍê³É......" + } + + + def exportBigdataCommPrice(queryDate){ + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" + def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_COMMPRICE_FJW_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_COMMPRICE_FJW_ADD_${curDate}_0001.dat" + } + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select CITYID, YMID,COMMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_COMMPRICE where CITYID='${cityCode}' ";// and YMID=${queryDate} + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.YMID + data << r.COMMID + data << r.SALEPRICE?r.SALEPRICE:'' + if(r.SALECOUNT>0) { + data << r.SALECOUNT + }else{ + data << '' + } + data << r.RENTPRICE?r.RENTPRICE:'' + if(r.RENTCOUNT>0){ + data << r.RENTCOUNT + }else{ + data << '' + } + data << r.UPDATETIME + data << r.SALEBLDGAREA?r.SALEBLDGAREA:'' + data << r.RENTBLDGAREA?r.RENTBLDGAREA:'' + data << r.SALETOTALPRICE?r.SALETOTALPRICE:'' + data << r.RENTTOTALPRICE?r.RENTTOTALPRICE:'' + data << r.HOUSEUSE + data << r.SDATERANGE + data << r.RDATERANGE + data << r.DATATYPE + data << r.SOURCECODE + allRows << data + }); + } + + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) + println "BIGDATA_COMMPRICE ÉÏ´«FtpÍê³É......" + } + def exportBigdataCommlayoutpicFjw(){ From 0d98e1ae838d41c6ecd74d8f24c306c95326e3f9 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 10 Jan 2023 16:20:22 +0800 Subject: [PATCH 45/99] =?UTF-8?q?=E8=A1=8C=E6=83=85=E7=9B=AE=E5=BD=95?= =?UTF-8?q?=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy index 919bff95a..59e1387c8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy @@ -1153,7 +1153,7 @@ class ExportDistrictRongHe extends BaseBatch{ def exportBigdataCityPrice(queryDate){ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_CITYPRICE_FJW_ADD_${curDate}_0001.dat" + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_CITYPRICE_FJW_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_CITYPRICE_FJW_ADD_${curDate}_0001.dat" } @@ -1201,7 +1201,7 @@ class ExportDistrictRongHe extends BaseBatch{ def exportBigdataDistPrice(queryDate){ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_DISTPRICE_FJW_ADD_${curDate}_0001.dat" + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DISTPRICE_FJW_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_DISTPRICE_FJW_ADD_${curDate}_0001.dat" } @@ -1248,7 +1248,7 @@ class ExportDistrictRongHe extends BaseBatch{ def exportBigdataCommPrice(queryDate){ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_COMMPRICE_FJW_ADD_${curDate}_0001.dat" + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMPRICE_FJW_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_COMMPRICE_FJW_ADD_${curDate}_0001.dat" } From 66ef95705d823abe7ce3d3956674fc758482561f Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 11 Jan 2023 09:49:50 +0800 Subject: [PATCH 46/99] update --- .../ronghe/ExportDistrictRongHe.groovy | 55 ++++++++++--------- 1 file changed, 28 insertions(+), 27 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy index 48e927791..892bd5e6c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy @@ -34,30 +34,31 @@ class ExportDistrictRongHe extends BaseBatch{ def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd") def new_dists = []//["350604","350605","410307","410308","450181"] //°´Ììµ¼£¨Ã¿ÌìÒ»¸öÎļþ¼Ð£© -// exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists) + exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists) if(curDate.toString().endsWith("10")){ -// exportBigdataCommunityGJRH()// -// exportBigdataBuildingFJW()// -// exportBigdataCommfeeGj()// -// -// exportBigdataCommbuildstruGJ()// -// exportBigdataCommhousetypeGJ()// -// exportBigdataCommbuildyearGj()// -// exportBigdataCommhouseuseGJ()// -// exportBigdataPositionGJ()// -// exportBigdataDistrictFJW()// -// exportBigdataAliasFJW()// -// -// exportBigdataBuildprojFjw() -// exportBigdataCommlayoutpicFjw() -// exportEvaluateRate() + exportBigdataCommunityGJRH()// + exportBigdataBuildingFJW()// + exportBigdataCommfeeGj()// + + exportBigdataCommbuildstruGJ()// + exportBigdataCommhousetypeGJ()// + exportBigdataCommbuildyearGj()// + exportBigdataCommhouseuseGJ()// + exportBigdataPositionGJ()// + exportBigdataDistrictFJW()// + exportBigdataAliasFJW()// + + exportBigdataBuildprojFjw() + exportBigdataCommlayoutpicFjw() + exportEvaluateRate() // //ÒòΪÈÚºÏÔݲ»Çп⣬ËùÒÔÐèÒª¼Ó´«Ò»ÕŵÚÈý·½¾ù¼Û±íµ½ÈÚºÏÖУ¬ ´Ó1ÔÂ10ÈÕ¿ªÊ¼´«Êä - exportThirdPartyPrice2(lastMonth, new_dists) -// -// -// //1ÔÂ10ÈÕ ÐÂÔö5ÕÅ±í£¬µ¼92³Ç£¬ÔöÁ¿Êý¾Ý£¬2ÔÂ10ÈÕµ¼344³Ç£¬ÆäÖÐ252³ÇËùÓÐÀúÊ·Êý¾Ý£¬92³ÇÔöÁ¿£¬3Ôºó¿ÉÄܲ»ÓÃÌṩÊý¾Ý + def thirdPartyMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")); + exportThirdPartyPrice2(thirdPartyMonth, new_dists) + + + //1ÔÂ10ÈÕ ÐÂÔö5ÕÅ±í£¬µ¼92³Ç£¬ÔöÁ¿Êý¾Ý£¬2ÔÂ10ÈÕµ¼344³Ç£¬ÆäÖÐ252³ÇËùÓÐÀúÊ·Êý¾Ý£¬92³ÇÔöÁ¿£¬3Ôºó¿ÉÄܲ»ÓÃÌṩÊý¾Ý exportCommunityBasePrice(lastMonth) exportCommunityPdescribeGj(lastMonth) exportGuideStatistics(lastMonth) @@ -818,8 +819,8 @@ class ExportDistrictRongHe extends BaseBatch{ def exportThirdPartyPrice2(price_date, new_dists) { - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; - def fileName = "/ccb_rh/FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "10"; + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" if (CommonDatautils.isWindow()) { fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" } @@ -933,7 +934,7 @@ class ExportDistrictRongHe extends BaseBatch{ } def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/ccb_rh/FJW_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat" + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat" } @@ -948,7 +949,7 @@ class ExportDistrictRongHe extends BaseBatch{ def exportCommunityPdescribeGj(month){ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/ccb_rh/FJW_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat" + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat" } @@ -1021,7 +1022,7 @@ class ExportDistrictRongHe extends BaseBatch{ def exportGuideStatistics(time){ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/ccb_rh/FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat" + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat" } @@ -1062,7 +1063,7 @@ class ExportDistrictRongHe extends BaseBatch{ def exportBigdataSupplierStatistics(time){ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/ccb_rh/FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_${curDate}_0001.dat" + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_${curDate}_0001.dat" } @@ -1100,7 +1101,7 @@ class ExportDistrictRongHe extends BaseBatch{ def exportBigdataModelingBase(time){ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/ccb_rh/FJW_000000_000000_BIGDATA_MODELING_BASE_ADD_${curDate}_0001.dat" + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_MODELING_BASE_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_MODELING_BASE_ADD_${curDate}_0001.dat" } From 32d64dbd670463f679c073952c547a822b13a374 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 11 Jan 2023 10:11:59 +0800 Subject: [PATCH 47/99] update --- .../fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy index 3e43ef6a3..dce00a5fa 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy @@ -26,7 +26,7 @@ class SynchDistrictPrice extends BaseBatch{ // "¼ªÎÝ":"¼ªÎÝ","365ÌÔ·¿":"365ÌÔ·¿","ÎÒ°®ÎÒ¼Ò":"ÎÒ°®ÎÒ¼Ò","Öî¸ðÕÒ·¿APP":"Öî¸ðÕÒ·¿","±´¿Ç":"±´¿ÇÕÒ·¿","Á´¼Ò":"Á´¼Ò","Q·¿":"Q·¿","ÂóÌï":"ÂóÌï", // "ÖÐÔ­µØ²ú":"ÖÐÔ­µØ²ú","°²¾Ó¿ÍС³ÌÐò":"°²¾Ó¿Í","ËÑ·¿":"ËÑ·¿","ÀÖÓмÒ":"ÀÖÓмÒ","C21":"C21","³ÇÊз¿²ú":"³ÇÊз¿²ú","Íò©Ê¿PC¶Ë":"Íò©Ê¿" - def esSources = ["¼ªÎÝ":"¼ªÎÝ","±´¿ÇÕÒ·¿":"±´¿ÇÕÒ·¿","ÀÖÓмÒ":"ÀÖÓмÒ","365ÌÔ·¿":"365ÌÔ·¿","ÎÒ°®ÎÒ¼Ò":"ÎÒ°®ÎÒ¼Ò","Öî¸ðÕÒ·¿APP":"Öî¸ðÕÒ·¿","Á´¼Ò":"Á´¼Ò","Q·¿":"Q·¿","ÂóÌï":"ÂóÌï","ÖÐÔ­µØ²ú":"ÖÐÔ­µØ²ú","°²¾Ó¿ÍС³ÌÐò":"°²¾Ó¿Í","ËÑ·¿":"ËÑ·¿","C21":"C21","³ÇÊз¿²ú":"³ÇÊз¿²ú","Íò©Ê¿PC¶Ë":"Íò©Ê¿"] + def esSources = ["ÎÒ°®ÎÒ¼Ò":"ÎÒ°®ÎÒ¼Ò","Íò©Ê¿PC¶Ë":"Íò©Ê¿","¼ªÎÝ":"¼ªÎÝ","±´¿ÇÕÒ·¿":"±´¿ÇÕÒ·¿","ÀÖÓмÒ":"ÀÖÓмÒ","365ÌÔ·¿":"365ÌÔ·¿","Öî¸ðÕÒ·¿APP":"Öî¸ðÕÒ·¿","Á´¼Ò":"Á´¼Ò","Q·¿":"Q·¿","ÂóÌï":"ÂóÌï","ÖÐÔ­µØ²ú":"ÖÐÔ­µØ²ú","°²¾Ó¿ÍС³ÌÐò":"°²¾Ó¿Í","ËÑ·¿":"ËÑ·¿","C21":"C21","³ÇÊз¿²ú":"³ÇÊз¿²ú"] static main(args) { From 3a8dbefddc783a4497f35fe3a90737c2f06faa38 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Sun, 15 Jan 2023 14:29:59 +0800 Subject: [PATCH 48/99] =?UTF-8?q?=E5=BB=BA=E8=A1=8C=E6=88=BF=E6=BA=90?= =?UTF-8?q?=E5=90=8C=E6=AD=A5=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ccbevaluation/BigdataForsaleBatch.groovy | 205 +++++++++++++++++- .../ccbevaluation/CcbCommonUtils.groovy | 4 +- .../script/fuzhou/FzDataResultStat.groovy | 4 +- .../gj/paimai/LianjiaCrawlByChrome.groovy | 2 +- .../script/gj/test/DongGuangApiTest.groovy | 27 ++- .../script/thirdPriceCrawl/DataStat.groovy | 6 +- 6 files changed, 235 insertions(+), 13 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index 639400868..e2ae36e0d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -74,7 +74,7 @@ class BigdataForsaleBatch extends BaseBatch{ // def distIds = getDistrictIds(cityCodeDataMap.get(city)) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]); - exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20221118, 20221219); + exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20221218, 20230119); // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210901, 20221119); // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20211001, 20221011); // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831); @@ -429,6 +429,13 @@ class BigdataForsaleBatch extends BaseBatch{ def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2){ + def rets1 = findHousesByPdate(city, region, cat, startDate, endDate, name, alias, estate_type2) + def rets2 = findHousesByMdate(city, region, cat, startDate, endDate, name, alias, estate_type2, rets1) + rets1.addAll(rets2) + return rets1; + } + + def findHousesByPdate(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2){ QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(false).includeUpper(false); QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false); @@ -586,7 +593,201 @@ class BigdataForsaleBatch extends BaseBatch{ def fitment = parseFitment(seawater.fitment) rowMap.fitment = fitment=="null"?null:fitment + if(seawater.p_date instanceof String){ + seawater.p_date = seawater.p_date.toInteger() + } + if(seawater.p_date<=20200101){ + //seawater.p_date.toString().replace("2019","2020")// + rowMap.p_date = DateUtils.getRandomDate("2020-01-01", "2020-07-31") + rowMap.c_date = rowMap.p_date + rowMap.m_date = rowMap.p_date + }else{ + rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss") + rowMap.c_date = DateUtils.getConverDate(seawater.m_date) + rowMap.m_date = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-18 10:00:00"//DateUtils.getConverDate(seawater.m_date) + } + rets << rowMap + } + } + + println name +" "+houses.size()+" last:"+rets.size() + return rets; + } + + + def findHousesByMdate(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2, rets1){ + def date = DateUtils.addMonth(endDate.toString(), -3, "yyyyMMdd") + def startMdate = DateUtils.parseFromTightByPattern(date,"yyyyMMdd") + def endMdate = DateUtils.parseFromTightByPattern(endDate.toString(), "yyyyMMdd") + + QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(date.toInteger()).to(endDate).includeLower(false).includeUpper(false); + QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false); + + + QueryBuilder date_range_mdate = QueryBuilders.rangeQuery("m_date").from(startMdate.getTime()).to(endMdate.getTime()).includeLower(true).includeUpper(false); + + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat)) + .must(date_range).must(area_range).must(date_range_mdate) + + if(!alias){ + alias = [] + } + if(!alias.contains(name)){ + alias.add(name) + } + + + if(name.contains("ÅªÐ¡Çø")){ + def newName = name.replace("ÅªÐ¡Çø","Ū") + if(!alias.contains(newName)){ + alias << newName + } + } + + if(alias && alias.size()>0){ + BoolQueryBuilder inBuilder = QueryBuilders.boolQuery(); + for(def s : alias){ + inBuilder.should(QueryBuilders.matchQuery("district_name", s)); + } + queryBuilder.must(inBuilder) + } + + + def rets = [] + def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder); + + if(houses){ + + def ids = new HashSet() + for(def h : rets1){ + ids << h.houseId + } + for (int k=0;k60){ + continue; + } + +// if(!seawater.room){ +// continue; +// } + if(!seawater.room){ + seawater.room = null + } + + if(seawater.room){ + if(seawater.room>=11 || seawater.room<1){ + continue; + } + } + + if(!seawater.hall){ + seawater.hall = null + } + if(seawater.hall && (seawater.hall>=5 || seawater.hall<1)){ + continue; + } + if(!seawater.toilet){ + seawater.toilet = null + } + if(seawater.toilet && (seawater.toilet>=5 || seawater.toilet<1)){ + continue; + } + + + def floor = parseFloor(seawater.floor_info) + if(seawater.height && floor && floor.isNumber()){ + if(floor.toInteger()>seawater.height){ + floor = null + seawater.height = null + } + + } + + if(!floor){ + seawater.floor = null + } + if(floor && floor.toInteger()<1){ + floor = null + } + + def rowMap = [:]; + if(cat == "rent"){ + rowMap.rent_type = seawater.rent_type + if(seawater.avg_price && seawater.avg_price>1000){ + continue + } + + if(seawater.avg_price && seawater.avg_price<10){ + continue + } + + } + rowMap.houseId = seawater.id + rowMap.city = CcbCommonUtils.getCcbCity(city) + //rowMap.region =seawater.region + //rowMap.address = seawater.address + + + rowMap.area = seawater.area + rowMap.total_price = seawater.total_price + rowMap.avg_price = seawater.avg_price + rowMap.height = seawater.height=="null"?null:seawater.height + if(rowMap.height && rowMap.height<1){ + rowMap.height = null + } + rowMap.floor = floor=="null"?null:floor + rowMap.room = seawater.room=="null"?null:seawater.room + rowMap.hall = seawater.hall=="null"?null:seawater.hall + rowMap.toilet = seawater.toilet=="null"?null:seawater.toilet + if(CcbCommonUtils.rootMap.get(seawater.root)){ + rowMap.root = CcbCommonUtils.rootMap.get(seawater.root) + }else{ + rowMap.root = "99" + } + + def direction = parseDirection(seawater.direction) + rowMap.direction = direction=="null"?null:direction + + def fitment = parseFitment(seawater.fitment) + rowMap.fitment = fitment=="null"?null:fitment if(seawater.p_date instanceof String){ seawater.p_date = seawater.p_date.toInteger() @@ -599,7 +800,7 @@ class BigdataForsaleBatch extends BaseBatch{ }else{ rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss") rowMap.c_date = DateUtils.getConverDate(seawater.m_date) - rowMap.m_date = DateUtils.getConverDate(seawater.m_date) + rowMap.m_date = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-18 10:00:00"//DateUtils.getConverDate(seawater.m_date) } rets << rowMap diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index a551c50b3..b17d97a59 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -65,6 +65,8 @@ class CcbCommonUtils { } }else{ citys = CcbCommonUtils.citys_92 + sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") } println "ÔËÐгÇÊÐÊýÁ¿£º"+citys.size() return citys; @@ -422,7 +424,7 @@ class CcbCommonUtils { q.put("visible",0) q.put("cat", cat); q.put("city", city); -// q.put("name","ÔÆÉ½ÉгÇ") +// q.put("name","Ãù´äÀ¶Íå") // q.put("name_cat", 0); // q.put("estate_type2", new BasicDBObject('$in', ["ÆÕͨסլ","ÉÌס"])) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy index 0722c4464..917ddfc5e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy @@ -111,8 +111,8 @@ class FzDataResultStat { def regions = ["¹ÄÂ¥","²Öɽ","̨½­","½ú°²","Âíβ","³¤ÀÖ"]; def datas1 = [] def datas2 = [] - datas1 << ["³ÇÊÐ","ÇøÓò","Ãû³Æ","${curMonth}¼Û¸ñ","${lastMonth}¼Û¸ñ", "Õǵø·ù"] - datas2 << ["³ÇÊÐ","ÇøÓò","Ãû³Æ","${curMonth}¼Û¸ñ","${lastMonth}¼Û¸ñ", "Õǵø·ù"] + datas1 << ["³ÇÊÐ","ÇøÓò","Ãû³Æ","${curMonth}¼Û¸ñ","${lastMonth}¼Û¸ñ", "Õǵø·ù", "ÊÇ·ñÓÐЧ"] + datas2 << ["³ÇÊÐ","ÇøÓò","Ãû³Æ","${curMonth}¼Û¸ñ","${lastMonth}¼Û¸ñ", "Õǵø·ù", "ÊÇ·ñÓÐЧ"] def row_num = 0 for(def row : rows){ if(row_num>0){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy index 7f99588f5..0ecdddeb1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -474,7 +474,7 @@ public class LianjiaCrawlByChrome { if(cityEles){ for(def e : cityEles) { def href = e.attr("href") - if (!href.contains("fang.lianjia") ){ + if (!href.contains("fang.lianjia")){ cityMap.put(e.text(), href) } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/DongGuangApiTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/DongGuangApiTest.groovy index 2e4cfff4a..3b8227240 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/DongGuangApiTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/DongGuangApiTest.groovy @@ -1,10 +1,12 @@ package com.fangjia.script.gj.test +import com.fangjia.core.util.DataUtil import com.fangjia.script.gj.ccb.FileUtils import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.OpenApiClient +import net.sf.json.JSONObject import org.elasticsearch.index.query.QueryBuilder import org.elasticsearch.index.query.QueryBuilders @@ -24,17 +26,34 @@ class DongGuangApiTest { } def run(){ - def rets = findEsData(); + // def rets = findEsData(); + def rets = readTxt("D:\\Êý¾Ý\\¶«Ý¸ÒøÐÐ\\¶«Ý¸ÒøÐÐ (1).csv") println "ES ¼ÓÔØÍê³É" def datas = evaluation(rets) println "ÖØ¹ÀÍê³É" // ExcelUtils.write("D:\\Êý¾Ý\\¶«Ý¸ÒøÐÐ\\¶«Ý¸ÖعÀ.xlsx", datas, "Êý¾Ý") - FileUtils.batchCreateFile(datas, "/home/goojia/¶«Ý¸ÖعÀ.txt", [], "utf-8", true); + FileUtils.batchCreateFile(datas, "D:\\Êý¾Ý\\¶«Ý¸ÒøÐÐ\\¶«Ý¸ÖعÀ.txt", [], "utf-8", true); println "txt Éú³É" - exportCsv("/home/goojia/¶«Ý¸ÖعÀ.csv", datas) + exportCsv("D:\\Êý¾Ý\\¶«Ý¸ÒøÐÐ\\¶«Ý¸ÖعÀ.csv", datas) println "csv Éú³É" } + def readTxt(def fileName){ + File file = new File(fileName); + def rows = []; + + file.eachLine{ + def row = it.split("#",-1) + if(row && row.size()>=2){ + def s = row[1] + JSONObject result = JSONObject.fromObject(s); + rows << result + } + + } + return rows; + } + def evaluation(rets){ def datas = new CopyOnWriteArrayList() def totalCount = new AtomicLong() @@ -45,7 +64,7 @@ class DongGuangApiTest { void run() { try{ totalCount.addAndGet(1) - def ret = OpenApiClient.addressEvaluation(dist.city, dist.district, dist.name, dist.size) + def ret = OpenApiClient.addressEvaluation(dist.city, dist.district, dist.name, 100) if(ret && ret.avgPrice){ def row = [] row << dist.city diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy index 0ceed1225..62e2524e9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy @@ -41,10 +41,10 @@ class DataStat { def run(){ def source = "365ÌÔ·¿"// - def citys = findCrawlCity(source, 202210); + def citys = findCrawlCity(source, 202211); for(def city : citys){ - def count1 = findCount(source, city, 202210); - def count2 = findCount(source, city, 202211); + def count1 = findCount(source, city, 202211); + def count2 = findCount(source, city, 202212); println city +" "+count1+" "+count2 // println count1 // println "--- ----" From 61bdc844feb8cccc6653bda36eefdecd435da89e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=96=87=E8=89=B3?= Date: Sun, 15 Jan 2023 20:31:41 +0800 Subject: [PATCH 49/99] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E4=BA=BA=E5=B7=A5?= =?UTF-8?q?=E6=8C=87=E5=AF=BC=E4=BB=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../AverageStatisticsController.java | 17 +++--- .../bkoff/util/BaseAvgImportServlet.java | 58 ++++++++++++++++--- .../main/webapp/averageStatistics/base.ftl | 14 ++++- .../webapp/avgPriceUpdate/districtDetail.ftl | 14 ++++- .../dal/mongo/model/AverageStatistics.java | 11 ++++ .../com/fangjia/dal/mongo/model/AvgBase.java | 22 ++++++- .../com/fangjia/script/wenyan/Test4.groovy | 18 ++++++ .../fangjia/AverageStatisticsService.java | 2 + .../fangjia/AverageStatisticsServiceImpl.java | 10 +++- 9 files changed, 144 insertions(+), 22 deletions(-) diff --git a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/AverageStatisticsController.java b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/AverageStatisticsController.java index 3b91e6583..71ff8fb02 100644 --- a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/AverageStatisticsController.java +++ b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/AverageStatisticsController.java @@ -968,9 +968,6 @@ public class AverageStatisticsController { */ public Object baseAction(AverageStatisticsModel model){ Pager pager = model.getPager(25); - if (StringUtils.isBlank(model.getCityDisplay())) { - return model; - } pager.setTotalRows(avgBaseDao.count(model.getCityDisplay(), model.getRegion(), model.getName(),null, model.getStartDate(), model.getEndDate())); model.setBaseList(avgBaseDao.findAll(model.getCityDisplay(), model.getRegion(), model.getName(),null, model.getStartDate(), model.getEndDate(), pager)); return model; @@ -987,6 +984,8 @@ public class AverageStatisticsController { list.add("ÊÇ·ñ¼ÓÈëÖ¸µ¼¼Û"); list.add("µ÷²éʱ¼ä"); list.add("µ÷²éÈË"); + list.add("À´Ô´"); + list.add("À´Ô´Ð¡ÇøÃû"); data.add(list); HSSFWorkbook workbook = new HSSFWorkbook(); HSSFSheet sheet = workbook.createSheet("µ¼ÈëÄ£°å"); @@ -994,10 +993,12 @@ public class AverageStatisticsController { HSSFCell cell0 = row.createCell(0); HSSFCell cell1 = row.createCell(1); HSSFCell cell2 = row.createCell(2); - HSSFCell cell3= row.createCell(3); - HSSFCell cell4= row.createCell(4); - HSSFCell cell5= row.createCell(5); - HSSFCell cell6= row.createCell(6); + HSSFCell cell3 = row.createCell(3); + HSSFCell cell4 = row.createCell(4); + HSSFCell cell5 = row.createCell(5); + HSSFCell cell6 = row.createCell(6); + HSSFCell cell7 = row.createCell(7); + HSSFCell cell8 = row.createCell(8); cell0.setCellValue("³ÇÊÐ"); cell1.setCellValue("ÇøÓò"); cell2.setCellValue("Ð¡ÇøÃû"); @@ -1005,6 +1006,8 @@ public class AverageStatisticsController { cell4.setCellValue("ÊÇ·ñ¼ÓÈëÖ¸µ¼¼Û"); cell5.setCellValue("µ÷²éʱ¼ä"); cell6.setCellValue("µ÷²éÈË"); + cell7.setCellValue("À´Ô´"); + cell8.setCellValue("À´Ô´Ð¡ÇøÃû"); HttpServletResponse response = ActionContext.getContext().getResponse(); response.setCharacterEncoding("UTF-8"); response.setContentType("application/x-download;charset=UTF-8"); diff --git a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/util/BaseAvgImportServlet.java b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/util/BaseAvgImportServlet.java index f4594ad70..080be8617 100644 --- a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/util/BaseAvgImportServlet.java +++ b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/util/BaseAvgImportServlet.java @@ -8,11 +8,13 @@ import com.fangjia.core.util.HttpRequestException; import com.fangjia.dal.QueryParam; import com.fangjia.dal.common.DistrictKey; import com.fangjia.dal.mongo.dao.AvgBaseDao; +import com.fangjia.dal.mongo.model.AverageStatistics; import com.fangjia.dal.mongo.model.AvgBase; import com.fangjia.dal.mongo.model.bkoff.GuidingPrice; import com.fangjia.dal.seaweed.model.District; import com.fangjia.service.datastore.DistrictService; import com.fangjia.service.mongo.bkoff.GuidingPriceService; +import com.fangjia.service.mongo.fangjia.AverageStatisticsService; import net.sf.json.JSONObject; import org.apache.commons.fileupload.FileItem; import org.apache.commons.fileupload.FileItemFactory; @@ -47,12 +49,12 @@ import java.util.*; public class BaseAvgImportServlet extends HttpServlet { - private static final String[] columns = { "³ÇÊÐ", "ÇøÓò", "Ð¡ÇøÃû", "¼Û¸ñ", "ÊÇ·ñ¼ÓÈëÖ¸µ¼¼Û", "µ÷²éʱ¼ä", "µ÷²éÈË" }; + private static final String[] columns = { "³ÇÊÐ", "ÇøÓò", "Ð¡ÇøÃû", "¼Û¸ñ", "ÊÇ·ñ¼ÓÈëÖ¸µ¼¼Û", "µ÷²éʱ¼ä", "µ÷²éÈË" , "À´Ô´", "À´Ô´Ð¡ÇøÃû"}; private AvgBaseDao avgBaseDao; private GuidingPriceService guidingPriceService; private DistrictService districtService; - + private AverageStatisticsService averageStatisticsService; @Override public void init(ServletConfig config) throws ServletException { @@ -62,6 +64,7 @@ public class BaseAvgImportServlet extends HttpServlet { avgBaseDao = (AvgBaseDao) context.getBean("avgBaseDao"); guidingPriceService = (GuidingPriceService) context.getBean("guidingPriceService"); districtService = (DistrictService) context.getBean("districtService"); + averageStatisticsService = (AverageStatisticsService) context.getBean("averageStatisticsService"); } catch (Exception e) { e.printStackTrace(); } @@ -385,17 +388,56 @@ public class BaseAvgImportServlet extends HttpServlet { Date applyDate = (DateUtil.parseDate(map.get(key), "yyyy-MM-dd")); avgBase.setCreationDate(applyDate == null ? curDate : applyDate); } else if (key.equals("µ÷²éÈË") && map.get(key) != null) { - avgBase.setUserName(map.get(key)); + } else if (key.equals("À´Ô´") && map.get(key) !=null){ + avgBase.setSource(map.get(key).replaceAll("\\s", "")); + } else if (key.equals("À´Ô´Ð¡ÇøÃû") && map.get(key) != null){ + avgBase.setSourceCommName(map.get(key).replaceAll("\\s", "")); } avgBase.setCategory("district"); - } - if (avgBase.getIsGuidancePrice() == 1) { - + District dis = null; + if(StringUtils.isNotBlank(avgBase.getSourceCommName()) && !avgBase.getSourceCommName().equals(avgBase.getDistrictName())){ DistrictKey districtKey = new DistrictKey(avgBase.getCity(), avgBase.getRegion(), avgBase.getDistrictName()); - District dis = districtService.findOneByKey(districtKey, "district"); + dis = districtService.findOneByKey(districtKey, "district"); + if(dis.getAlias() == null){ + dis.setAlias(Collections.singletonList(avgBase.getSourceCommName())); + }else if(!dis.getAlias().contains(avgBase.getSourceCommName())){ + dis.getAlias().add(avgBase.getSourceCommName()); + } + if(dis.getMappingAlias() == null){ + dis.setMappingAlias(Collections.singletonList(avgBase.getSourceCommName())); + }else if(!dis.getMappingAlias().contains(avgBase.getSourceCommName())){ + dis.getMappingAlias().add(avgBase.getSourceCommName()); + } + districtService.save(dis); + } + if(StringUtils.isNotBlank(avgBase.getSource()) && avgBase.getSource().equalsIgnoreCase("GXD")){ + AverageStatistics averageStatistics = averageStatisticsService.findSecendAverageStatictics(avgBase.getCity(), avgBase.getRegion(), avgBase.getDistrictName(), "district"); + if(averageStatistics != null){ + averageStatistics.setGxdEsfPrice(avgBase.getPrice()); + }else{ + averageStatistics = new AverageStatistics(); + averageStatistics.setGxdEsfPrice(avgBase.getPrice()); + averageStatistics.setCategory("district"); + averageStatistics.setCity(avgBase.getCity()); + averageStatistics.setRegion(avgBase.getRegion()); + averageStatistics.setName(avgBase.getDistrictName()); + if(dis != null){ + averageStatistics.setMasterDistrictName(dis.getDistrictName()); + averageStatistics.setModificationDate(new Date()); + } + } + averageStatisticsService.save(averageStatistics); + } + + if (avgBase.getIsGuidancePrice() == 1) { + if(dis == null){ + DistrictKey districtKey = new DistrictKey(avgBase.getCity(), avgBase.getRegion(), + avgBase.getDistrictName()); + dis = districtService.findOneByKey(districtKey, "district"); + } GuidingPrice price = guidingPriceService.findByBizKey(avgBase.getCity(), avgBase.getRegion(), avgBase.getDistrictName(),avgBase.getCreationDate()); if (price != null) { @@ -414,7 +456,7 @@ public class BaseAvgImportServlet extends HttpServlet { price.setGuidingPerson(avgBase.getUserName()); price.setGuidingDate(avgBase.getCreationDate()); price.setGuidingValue(avgBase.getPrice()); - price.setEstateType(dis.getEstateType2().indexOf("±ðÊû") > -1 ? "±ðÊû" : "ÆÕͨסլ"); + price.setEstateType(dis.getEstateType2().contains("±ðÊû") ? "±ðÊû" : "ÆÕͨסլ"); price.setCat("district"); guidingPriceService.save(price); HttpRequest request = new HttpRequest(); diff --git a/projects/fangjia-bkoff/src/main/webapp/averageStatistics/base.ftl b/projects/fangjia-bkoff/src/main/webapp/averageStatistics/base.ftl index 417d20240..dd6935446 100644 --- a/projects/fangjia-bkoff/src/main/webapp/averageStatistics/base.ftl +++ b/projects/fangjia-bkoff/src/main/webapp/averageStatistics/base.ftl @@ -66,6 +66,8 @@ 是å�¦åŠ å…¥æŒ‡å¯¼ä»· 调查时间 调查人 + æ�¥æº� + æ�¥æº�å°�区å�� @@ -94,7 +96,17 @@ <#if it.userName??> ${it.userName} - + + + <#if it.source??> + ${it.source} + + + + <#if it.sourceCommName??> + ${it.sourceCommName} + + diff --git a/projects/fangjia-bkoff/src/main/webapp/avgPriceUpdate/districtDetail.ftl b/projects/fangjia-bkoff/src/main/webapp/avgPriceUpdate/districtDetail.ftl index a3cb6949e..b173ee86a 100644 --- a/projects/fangjia-bkoff/src/main/webapp/avgPriceUpdate/districtDetail.ftl +++ b/projects/fangjia-bkoff/src/main/webapp/avgPriceUpdate/districtDetail.ftl @@ -254,6 +254,9 @@ <#if (it.tsfhouseNewPrice)?? || (it.tsfhouseEsfPrice)??> 365房产 + <#if (it.gxdNewPrice)?? || (it.gxdEsfPrice)??> + GXD + @@ -345,7 +348,10 @@ 新房 二手房 - + <#if (it.gxdNewPrice)?? || (it.gxdEsfPrice)??> + 新房 + 二手房 + @@ -430,7 +436,7 @@ <#if (it.ganjiNewPrice)?? || (it.ganjiEsfPrice)??> ${it.ganjiNewPrice!} ${it.ganjiEsfPrice!} - + <#if (it.yungujiaNewPrice)?? || (it.yungujiaEsfPrice)??> ${it.yungujiaNewPrice!} ${it.yungujiaEsfPrice!} @@ -443,6 +449,10 @@ ${it.tsfhouseNewPrice!} ${it.tsfhouseEsfPrice!} + <#if (it.gxdNewPrice)?? || (it.gxdEsfPrice)??> + ${it.gxdNewPrice!} + ${it.gxdEsfPrice!} + diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/AverageStatistics.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/AverageStatistics.java index 7b2405df1..da498918c 100644 --- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/AverageStatistics.java +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/AverageStatistics.java @@ -166,6 +166,8 @@ public class AverageStatistics implements MongoEntity { private Double tsfhouseEsfDiff; private Integer tsfhouseNewPrice; private Double tsfhouseNewDiff; + /**¹úÐÅ´ï*/ + private Integer gxdEsfPrice; private String level;//¼Û¸ñÆÀ¼¶ private Integer lastAvgPrice;//×îÖÕ¼Û @@ -1155,4 +1157,13 @@ public class AverageStatistics implements MongoEntity { public void setTsfhouseNewDiff(Double tsfhouseNewDiff) { this.tsfhouseNewDiff = tsfhouseNewDiff; } + + @Column(name = "gxd_esf_price") + public Integer getGxdEsfPrice() { + return gxdEsfPrice; + } + + public void setGxdEsfPrice(Integer gxdEsfPrice) { + this.gxdEsfPrice = gxdEsfPrice; + } } diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/AvgBase.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/AvgBase.java index 394785d9d..4d1c1dfbf 100644 --- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/AvgBase.java +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/AvgBase.java @@ -30,9 +30,9 @@ public class AvgBase implements MongoEntity{ private String category; private Integer status; private String dateStr; - - private Integer isGuidancePrice; + private String source; + private String sourceCommName; @Id @GeneratedValue @@ -135,4 +135,22 @@ public class AvgBase implements MongoEntity{ public void setDateStr(String dateStr) { this.dateStr = dateStr; } + + @Column(name = "source") + public String getSource() { + return source; + } + + public void setSource(String source) { + this.source = source; + } + + @Column(name = "source_comm_name") + public String getSourceCommName() { + return sourceCommName; + } + + public void setSourceCommName(String sourceCommName) { + this.sourceCommName = sourceCommName; + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/Test4.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/Test4.groovy index 09877e434..f13913aca 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/Test4.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/Test4.groovy @@ -1 +1,19 @@ package com.fangjia.script.wenyan + +import com.fangjia.core.ProvinceData +import com.fangjia.core.util.DateUtil +import com.fangjia.core.util.Mongo3Utils +import com.fangjia.datastore.driver.DSCollection +import com.fangjia.datastore.driver.DataStore +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ElasticUtils +import com.mongodb.client.MongoCollection +import org.bson.Document + +MongoCollection documents = Mongo3Utils.getDbCollection("fangjia", "deal_price") + +ProvinceData.getAllCity("º£ÄÏ").each{def it-> + long c = documents.countDocuments(new Document("city", it).append("trade_date", new Document('$gt', DateUtil.add(new Date(), Calendar.YEAR, -3)))) + println("${it}\t${c}") +} \ No newline at end of file diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/AverageStatisticsService.java b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/AverageStatisticsService.java index 25eb7040d..4fff15570 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/AverageStatisticsService.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/AverageStatisticsService.java @@ -17,4 +17,6 @@ public interface AverageStatisticsService { AverageStatistics findSecendAverageStatictics(String city,String region,String name,String category); Map findFirstAverageStatictics(String city,String region,String name,String category, Date sDate); + + AverageStatistics save(AverageStatistics averageStatistics); } diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/AverageStatisticsServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/AverageStatisticsServiceImpl.java index 772d5594c..fef0b66b5 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/AverageStatisticsServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/AverageStatisticsServiceImpl.java @@ -198,8 +198,14 @@ public class AverageStatisticsServiceImpl implements AverageStatisticsService { } return dataMap; } - - + + @Override + public AverageStatistics save(AverageStatistics averageStatistics) { + averageStatisticsDao.save(averageStatistics); + return averageStatistics; + } + + private boolean isValid(Integer value) { if (value != null && value > 1000) { return true; From c62304a1a0d35fec7fe12e993ffcfd4934606f18 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 16 Jan 2023 12:10:47 +0800 Subject: [PATCH 50/99] =?UTF-8?q?=E5=8E=BB=E9=99=A4=E5=A4=9A=E4=BD=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ExportOracleAllDataTest.groovy | 54 +++++++++---------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy index 79eda643f..39ef500ac 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy @@ -43,39 +43,39 @@ class ExportOracleAllDataTest { def lastMonthStr = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM") def new_dists = [] - exportBigdataForsale(new_dists) //2022-07-19 - - //²»»á±ä 10.25 - exportEvaluationRateData() - exportBigdataPoiData()// - exportBigdataDistrictData()// - - exportGuidedPrice()//ËæÈ˹¤¸Ä±ä// - exportCommunityGuidedPrice(lastMonth.toString()+"28")//ËæÕþ²ß// - - //´¦ÀíÐ¡ÇøÇé¿ö - exportBigdataCommunity()// - exportCommunityStaging()//ûcityid - exportCommunityBuileType()// - - exportThirdPartyPrice2(new_dists)// - exportBigdataDeal( new_dists) //2022-07-01 +// exportBigdataForsale(new_dists) //2022-07-19 // -// //µÚÈý·½ÍøÕ¾È¨Öرí - exportWebsiteRate(); +// //²»»á±ä 10.25 +// exportEvaluationRateData() +// exportBigdataPoiData()// +// exportBigdataDistrictData()// // -// //¼ÆËã½á¹ûÏà¹Ø - exportAvmEvaluationGJNB()// - exportAVMEvaluationItemGJNB()// +// exportGuidedPrice()//ËæÈ˹¤¸Ä±ä// +// exportCommunityGuidedPrice(lastMonth.toString()+"28")//ËæÕþ²ß// +// +// //´¦ÀíÐ¡ÇøÇé¿ö +// exportBigdataCommunity()// +// exportCommunityStaging()//ûcityid +// exportCommunityBuileType()// +// +// exportThirdPartyPrice2(new_dists)// +// exportBigdataDeal( new_dists) //2022-07-01 +//// +//// //µÚÈý·½ÍøÕ¾È¨Öرí +// exportWebsiteRate(); +//// +//// //¼ÆËã½á¹ûÏà¹Ø +// exportAvmEvaluationGJNB()// +// exportAVMEvaluationItemGJNB()// +// +// exportCityPrice()// +// exportRegionPrice()// - exportCityPrice()// - exportRegionPrice()// - exportCommunityBasePriceGj()// exportCommunityBasePrice()// - exportCommunityPdescribeGj()// - exportCommCaseDetailGj()// +// exportCommunityPdescribeGj()// +// exportCommCaseDetailGj()// // // okÎļþ exportInOk() From 61f999e0c1d1a7dfa739ecba66d51afca9fc9854 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 17 Jan 2023 14:03:37 +0800 Subject: [PATCH 51/99] =?UTF-8?q?=E6=97=B6=E9=97=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/ccbevaluation/ThirdPartyPriceBatch.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy index 295ab8915..8a2b1c55e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy @@ -22,9 +22,9 @@ import java.util.concurrent.ConcurrentHashMap @BatchOwner(Watcher.GaoJuan) class ThirdPartyPriceBatch extends BaseBatch { - def sDate= 202211//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); - def startDate = 202209 - def endDate = 202211 + def sDate= 202212//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); + def startDate = 202210 + def endDate = 202212 // def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM")); static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") From 0702327714a3167ca7c2034cfd0be531c5abd505 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 17 Jan 2023 17:27:10 +0800 Subject: [PATCH 52/99] update --- .../fangjia/script/ccbevaluation/BigdataDealBatch.groovy | 3 ++- .../fangjia/script/ccbevaluation/ExportOracleData.groovy | 7 ++++--- .../script/ccbevaluation/ExportOracleDataTest.groovy | 7 ++++--- .../groovy/com/fangjia/script/gj/YouDaDataParser.groovy | 4 ++-- .../fangjia/script/gj/district/FangjiaClientUtils.groovy | 5 ++++- 5 files changed, 16 insertions(+), 10 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index af28b3fd4..42766952e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -633,7 +633,8 @@ class BigdataDealBatch extends BaseBatch{ def addDealPriceMatch(city){ - def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd")) +// def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd")) + def startDate = DateUtils.parseFromTightByPattern(startDate.toString()+"01", "yyyyMMdd") def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd")) // def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210101", "yyyyMMdd")) // def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20221001", "yyyyMMdd")) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy index 22861ba1d..b25c4efbd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -43,7 +43,8 @@ class ExportOracleData { exportCommunityBuileType()// exportThirdPartyPrice2(lastMonth, new_dists)// - exportBigdataForsale(lastMonthStr +"-19", new_dists) //2022-07-19 +// exportBigdataForsale(lastMonthStr +"-19", new_dists) //2022-07-19 ֮ǰ°´publish_dateµ¼³ö + exportBigdataForsale(lastMonthStr +"-18", new_dists) //ÏÖÔÚ°´updatetimeµ¼³ö exportBigdataDeal(lastMonthStr + "-01", new_dists) //2022-07-01 //µÚÈý·½ÍøÕ¾È¨Öرí @@ -659,13 +660,13 @@ class ExportOracleData { def sql1 = "select CITYID,FORSALEID,PROPCODE,CITYNAME,DISTID,DISTNAME,COMMID,COMMNAME, BUILDID,BUILDNO, BUILDNAME,UNITID,UNITNO,UNITNAME,HOUSEID,ADDRESS,BUILDYEAR,PROPTYPE,PROPYEARS,UNITPRICE,TOTALPRICE,PAYTYPE,FLOORNUM,FLOORCNT,\n" + " BUILDTYPE, HOUSESTRU,BUILDSTRU,BUILDAREA,LIVINGAREA,HOUSESTYLE,ROOMCNT,LIVINGROOMCNT,KITCHENCNT, BATHROOMCNT,HOUSEASPECT,FITMENT,IFELEVATOR,ELEVATORHOUSE,to_char(PUBLISHTIME,'yyyy-mm-dd hh24:mi:ss') as PUBLISHTIME,to_char(EXPIRETIME,'yyyy-mm-dd hh24:mi:ss') as EXPIRETIME,\n" + " IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH from bigdata_forsale_gj " + - "where cityId='${cityCode}' and publishtime>=to_date('${publishDate}','yyyy-MM-dd') and distid not in ${newstr}"; + "where cityId='${cityCode}' and updatetime>=to_date('${publishDate}','yyyy-MM-dd') and distid not in ${newstr}"; def rets1 = getHouseDatas(sql1) allRows.addAll(rets1) }else{ def sql = "select CITYID,FORSALEID,PROPCODE,CITYNAME,DISTID,DISTNAME,COMMID,COMMNAME, BUILDID,BUILDNO, BUILDNAME,UNITID,UNITNO,UNITNAME,HOUSEID,ADDRESS,BUILDYEAR,PROPTYPE,PROPYEARS,UNITPRICE,TOTALPRICE,PAYTYPE,FLOORNUM,FLOORCNT,\n" + " BUILDTYPE, HOUSESTRU,BUILDSTRU,BUILDAREA,LIVINGAREA,HOUSESTYLE,ROOMCNT,LIVINGROOMCNT,KITCHENCNT, BATHROOMCNT,HOUSEASPECT,FITMENT,IFELEVATOR,ELEVATORHOUSE,to_char(PUBLISHTIME,'yyyy-mm-dd hh24:mi:ss') as PUBLISHTIME,to_char(EXPIRETIME,'yyyy-mm-dd hh24:mi:ss') as EXPIRETIME,\n" + - " IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH from bigdata_forsale_gj where cityId='${cityCode}' and publishtime>=to_date('${publishDate}','yyyy-MM-dd')"; + " IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH from bigdata_forsale_gj where cityId='${cityCode}' and updatetime>=to_date('${publishDate}','yyyy-MM-dd')"; allRows = getHouseDatas(sql) } if(allRows.size()>0){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleDataTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleDataTest.groovy index f1f2e17bc..88902dc7b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleDataTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleDataTest.groovy @@ -12,6 +12,7 @@ import org.apache.commons.codec.digest.DigestUtils import java.sql.ResultSet +//µ¼³ö¹ÀÖµ ÔöÁ¿Êý¾Ý аæ class ExportOracleDataTest { def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", @@ -45,7 +46,7 @@ class ExportOracleDataTest { exportCommunityBuileType()// exportThirdPartyPrice2(lastMonth, new_dists)// - exportBigdataForsale(lastMonthStr +"-19", new_dists) //2022-07-19 + exportBigdataForsale(lastMonthStr +"-18", new_dists) //2022-07-19 °´¸üÐÂʱ¼äµ¼³öÊý¾Ý exportBigdataDeal(lastMonthStr + "-01", new_dists) //2022-07-01 //µÚÈý·½ÍøÕ¾È¨Öرí @@ -658,13 +659,13 @@ class ExportOracleDataTest { def sql1 = "select CITYID,FORSALEID,PROPCODE,CITYNAME,DISTID,DISTNAME,COMMID,COMMNAME, BUILDID,BUILDNO, BUILDNAME,UNITID,UNITNO,UNITNAME,HOUSEID,ADDRESS,BUILDYEAR,PROPTYPE,PROPYEARS,UNITPRICE,TOTALPRICE,PAYTYPE,FLOORNUM,FLOORCNT,\n" + " BUILDTYPE, HOUSESTRU,BUILDSTRU,BUILDAREA,LIVINGAREA,HOUSESTYLE,ROOMCNT,LIVINGROOMCNT,KITCHENCNT, BATHROOMCNT,HOUSEASPECT,FITMENT,IFELEVATOR,ELEVATORHOUSE,to_char(PUBLISHTIME,'yyyy-mm-dd hh24:mi:ss') as PUBLISHTIME,to_char(EXPIRETIME,'yyyy-mm-dd hh24:mi:ss') as EXPIRETIME,\n" + " IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH from bigdata_forsale_gj " + - "where cityId='${cityCode}' and publishtime>=to_date('${publishDate}','yyyy-MM-dd') and distid not in ${newstr}"; + "where cityId='${cityCode}' and updatetime>=to_date('${publishDate}','yyyy-MM-dd') and distid not in ${newstr}"; def rets1 = getHouseDatas(sql1) allRows.addAll(rets1) }else{ def sql = "select CITYID,FORSALEID,PROPCODE,CITYNAME,DISTID,DISTNAME,COMMID,COMMNAME, BUILDID,BUILDNO, BUILDNAME,UNITID,UNITNO,UNITNAME,HOUSEID,ADDRESS,BUILDYEAR,PROPTYPE,PROPYEARS,UNITPRICE,TOTALPRICE,PAYTYPE,FLOORNUM,FLOORCNT,\n" + " BUILDTYPE, HOUSESTRU,BUILDSTRU,BUILDAREA,LIVINGAREA,HOUSESTYLE,ROOMCNT,LIVINGROOMCNT,KITCHENCNT, BATHROOMCNT,HOUSEASPECT,FITMENT,IFELEVATOR,ELEVATORHOUSE,to_char(PUBLISHTIME,'yyyy-mm-dd hh24:mi:ss') as PUBLISHTIME,to_char(EXPIRETIME,'yyyy-mm-dd hh24:mi:ss') as EXPIRETIME,\n" + - " IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH from bigdata_forsale_gj where cityId='${cityCode}' and publishtime>=to_date('${publishDate}','yyyy-MM-dd')"; + " IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH from bigdata_forsale_gj where cityId='${cityCode}' and updatetime>=to_date('${publishDate}','yyyy-MM-dd')"; allRows = getHouseDatas(sql) } if(allRows.size()>0){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy index 4f5efcea8..bbd0ec13b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy @@ -48,11 +48,11 @@ class YouDaDataParser { def final AtomicInteger count_ = new AtomicInteger(0) - static def statDate = "2022-10-01" + static def statDate = "2022-12-01" static main(args) { def task = new YouDaDataParser(); // task.newHouseparse();//ÓÑ´ïÒ»ÊÖ·¿´æÈëmongo -// task.esfParser("D:\\Êý¾Ý\\ÓÑ´ï\\2022Äê12ÔÂ\\²éѯ1.DBF");//ÓÑ´ï¶þÊÖ·¿´æÈëmongo +// task.esfParser("D:\\Êý¾Ý\\ÓÑ´ï\\2023Äê1ÔÂ\\²éѯ1.DBF");//ÓÑ´ï¶þÊÖ·¿´æÈëmongo // task.newhouseToSaveEs();//з¿±£´æµ½es task.esfhouseToSaveEs();//¶þÊÖ·¿±£´æµ½es // task.updateFinishDateByEsfTrade();//¸ù¾Ý¶þÊֳɽ»µÄ¿¢¹¤Ê±¼äÐÞ¸Äseaweed¿âÖеĿ¢¹¤Ê±¼ä diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy index 4b363b071..70249a1f9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy @@ -73,7 +73,10 @@ class FangjiaClientUtils { def content = null try { address = address.replace("{","") - def url = "http://192.168.0.19:31080/FangJiaNlpService/ner?address="+URLEncoder.encode(address,"UTF-8")+"&city="+URLEncoder.encode(city,"UTF-8") + def url = "http://192.168.0.19:31080/FangJiaNlpService/ner?address="+URLEncoder.encode(address,"UTF-8") + if(city){ + url = url +"&city="+URLEncoder.encode(city,"UTF-8") + } // def url = "http://192.168.0.19:39080/FangJiaAddressService/structure?address="+URLEncoder.encode(address,"UTF-8") content = new HttpRequest().newGet(url, null, "utf-8") From b8a3d7c43357e62a128f75ab1b27ab848be4e174 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=96=87=E8=89=B3?= Date: Tue, 17 Jan 2023 17:35:06 +0800 Subject: [PATCH 53/99] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E4=BA=BA=E5=B7=A5?= =?UTF-8?q?=E6=8C=87=E5=AF=BC=E4=BB=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../main/java/com/fangjia/bkoff/util/BaseAvgImportServlet.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/util/BaseAvgImportServlet.java b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/util/BaseAvgImportServlet.java index 080be8617..7c960c4e0 100644 --- a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/util/BaseAvgImportServlet.java +++ b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/util/BaseAvgImportServlet.java @@ -1,6 +1,7 @@ package com.fangjia.bkoff.util; import com.fangjia.core.Constants; +import com.fangjia.core.RegionData; import com.fangjia.core.util.DataUtil; import com.fangjia.core.util.DateUtil; import com.fangjia.core.util.HttpRequest; @@ -398,7 +399,7 @@ public class BaseAvgImportServlet extends HttpServlet { } District dis = null; if(StringUtils.isNotBlank(avgBase.getSourceCommName()) && !avgBase.getSourceCommName().equals(avgBase.getDistrictName())){ - DistrictKey districtKey = new DistrictKey(avgBase.getCity(), avgBase.getRegion(), + DistrictKey districtKey = new DistrictKey(avgBase.getCity(), RegionData.formatRegion(avgBase.getCity(), avgBase.getRegion()), avgBase.getDistrictName()); dis = districtService.findOneByKey(districtKey, "district"); if(dis.getAlias() == null){ From 0475c8e678528897fbe24e7326cd205b8ed24961 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=96=87=E8=89=B3?= Date: Tue, 17 Jan 2023 17:37:16 +0800 Subject: [PATCH 54/99] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E4=BA=BA=E5=B7=A5?= =?UTF-8?q?=E6=8C=87=E5=AF=BC=E4=BB=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/fangjia/bkoff/util/BaseAvgImportServlet.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/util/BaseAvgImportServlet.java b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/util/BaseAvgImportServlet.java index 7c960c4e0..411d679e6 100644 --- a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/util/BaseAvgImportServlet.java +++ b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/util/BaseAvgImportServlet.java @@ -178,7 +178,7 @@ public class BaseAvgImportServlet extends HttpServlet { } private Map checkUploadData(Sheet sheet, String name, long version, Map infoMap, - String realName) { + String realName) { int totalNum = sheet.getLastRowNum(); System.out.println("ÉÏ´«Îļþ¹²:" + totalNum + "¸ö"); if (totalNum > 5000) { @@ -475,7 +475,7 @@ public class BaseAvgImportServlet extends HttpServlet { } } } - + if(dis !=null) { } } From 7de504cfa6d0f46e1347d442080f78aed7d90a40 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 17 Jan 2023 17:44:44 +0800 Subject: [PATCH 55/99] update --- .../main/groovy/com/fangjia/script/util/DbUtils.groovy | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DbUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DbUtils.groovy index 7e6e73263..96709e230 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DbUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DbUtils.groovy @@ -335,8 +335,15 @@ public class DbUtils{ keys.append("lng2", 1) keys.append("estate_type2", 1) DbUtils.SEAWEED_SLAVE.find(query, keys).each {BasicDBObject dbo-> + if(!district._id || dbo._id != district._id){ - districts << dbo + if(district.estateType){ + if(dbo.estate_type2 == district.estateType){ + districts << dbo + } + }else{ + districts << dbo + } } } } From b8443d0e03626c2e2e78a926184fd4b038fb9221 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 27 Jan 2023 11:39:42 +0800 Subject: [PATCH 56/99] =?UTF-8?q?=E4=BC=B0=E5=80=BC=E8=9E=8D=E5=90=88?= =?UTF-8?q?=E9=BB=98=E8=AE=A4=E5=AF=BC=E5=87=BA344=E5=9F=8E=E6=95=B0?= =?UTF-8?q?=E6=8D=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy | 2 +- .../com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index b17d97a59..24d06b991 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -64,7 +64,7 @@ class CcbCommonUtils { "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") } }else{ - citys = CcbCommonUtils.citys_92 + citys = CcbCommonUtils.citys_all sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index 537e09d1e..af1c15d53 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -47,7 +47,7 @@ class GuiDingPriceBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) //Ìí¼Ó×Ô¼º¿âÖеÄÈ˹¤Ö¸µ¼¼Û - def prices = CcbCommonUtils.findPersonPrice(city, "2022-11-25", "2022-12-24") + def prices = CcbCommonUtils.findPersonPrice(city, "2022-12-24", "2023-01-25") // def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22") if(prices){ def distMap = getDistrictMap(city) From 3e3568bd4e09c7ec70e6ae536b6e32695695df38 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 27 Jan 2023 11:56:34 +0800 Subject: [PATCH 57/99] =?UTF-8?q?=E5=88=A0=E9=99=A4=E5=A4=9A=E4=BD=99?= =?UTF-8?q?=E7=9A=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/script/ccbevaluation/ExportOracleDataTest.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleDataTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleDataTest.groovy index 88902dc7b..2c727edfa 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleDataTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleDataTest.groovy @@ -59,7 +59,7 @@ class ExportOracleDataTest { exportCityPrice(curMonth)//ÎÞ³ÇÊÐÇø·Ö exportRegionPrice(curMonth)//ÎÞ³ÇÊÐÇø·Ö - exportCommunityBasePriceGj(curMonth)//ÒѺÏÏÂÃæµÄºÏ²¢ //ÎÞ³ÇÊÐÇø·Ö +// exportCommunityBasePriceGj(curMonth)//ÒѺÏÏÂÃæµÄºÏ²¢ //ÎÞ³ÇÊÐÇø·Ö exportCommunityBasePrice(curMonth)//ÎÞ³ÇÊÐÇø·Ö exportCommunityPdescribeGj(curMonth)//ÎÞ³ÇÊÐÇø·Ö From 225b288122300a99c30ba97b899b82cbc2b02a1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=96=87=E8=89=B3?= Date: Wed, 1 Feb 2023 17:29:11 +0800 Subject: [PATCH 58/99] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E9=82=AE=E7=AE=B1?= =?UTF-8?q?=E5=AF=86=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/main/groovy/com/fangjia/script/jms/EmailConsumer.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jms/EmailConsumer.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jms/EmailConsumer.groovy index f43bda14e..e0388af3c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jms/EmailConsumer.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jms/EmailConsumer.groovy @@ -133,7 +133,7 @@ public class EmailConsumer extends BaseBatch { email.setHostName(EMAIL_SERVER); email.setSslSmtpPort("465"); email.setCharset("GBK") - email.setAuthentication("service@fangjia.com", "bRvvVvic8HVgHkCi"); + email.setAuthentication("service@fangjia.com", "iXWzdL69CnzxksaE"); email.addTo(mail.receiverEmail); email.setFrom("service@fangjia.com");//·¢ËÍ·½ email.setSubject(mail.subject); //±êÌâ From 3420c987f2231fd1ccb4a7c648391ed0ad344d7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=96=87=E8=89=B3?= Date: Thu, 2 Feb 2023 12:42:30 +0800 Subject: [PATCH 59/99] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E9=82=AE=E7=AE=B1?= =?UTF-8?q?=E5=AF=86=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../groovy/com/fangjia/script/jms/EmailConsumer.groovy | 4 ++-- .../main/groovy/com/fangjia/script/wenyan/Test4.groovy | 9 ++++----- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jms/EmailConsumer.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jms/EmailConsumer.groovy index e0388af3c..3908e4ca6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jms/EmailConsumer.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jms/EmailConsumer.groovy @@ -98,7 +98,7 @@ public class EmailConsumer extends BaseBatch { } } else { - println "send error ${errorCount}, sleep ${sleepTime / 1000}s ..." + println "${mail.receiver_email} send error ${errorCount}, sleep ${sleepTime / 1000}s ..." sleep(sleepTime); } } @@ -168,7 +168,7 @@ public class EmailConsumer extends BaseBatch { email.setHostName(EMAIL_SERVER); email.setSslSmtpPort("465"); email.setCharset("GBK") - email.setAuthentication("service@fangjia.com", "bRvvVvic8HVgHkCi"); + email.setAuthentication("service@fangjia.com", "iXWzdL69CnzxksaE"); email.addTo(mail.receiverEmail); email.setFrom("service@fangjia.com"); email.setSubject(mail.subject); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/Test4.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/Test4.groovy index f13913aca..c6dcfa9bd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/Test4.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/Test4.groovy @@ -1,6 +1,7 @@ package com.fangjia.script.wenyan import com.fangjia.core.ProvinceData +import com.fangjia.script.util.Jms import com.fangjia.core.util.DateUtil import com.fangjia.core.util.Mongo3Utils import com.fangjia.datastore.driver.DSCollection @@ -11,9 +12,7 @@ import com.fangjia.script.util.ElasticUtils import com.mongodb.client.MongoCollection import org.bson.Document -MongoCollection documents = Mongo3Utils.getDbCollection("fangjia", "deal_price") +def jms = Jms.create("tcp://jms.db.fangjia.com:61616") -ProvinceData.getAllCity("º£ÄÏ").each{def it-> - long c = documents.countDocuments(new Document("city", it).append("trade_date", new Document('$gt', DateUtil.add(new Date(), Calendar.YEAR, -3)))) - println("${it}\t${c}") -} \ No newline at end of file +jms.sendMail("1142375881@qq.com", "²âÊÔ", "²âÊÔÄÚÈÝ") +println('1111') \ No newline at end of file From 75503b74f314ace92a6ac2a5cf6bae578aeaba18 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 2 Feb 2023 18:24:16 +0800 Subject: [PATCH 60/99] =?UTF-8?q?=E5=B0=8F=E5=8C=BA=E8=A1=8C=E6=83=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ronghe/ExportDistrictRongHe.groovy | 46 +++++++++---------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy index 59e1387c8..fbf70498c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy @@ -38,29 +38,29 @@ class ExportDistrictRongHe extends BaseBatch{ def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd") def new_dists = []//["350604","350605","410307","410308","450181"] //°´Ììµ¼£¨Ã¿ÌìÒ»¸öÎļþ¼Ð£© - exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists) +// exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists) // if(curDate.toString().endsWith("10")){ - exportBigdataCommunityGJRH()// - exportBigdataBuildingFJW()// - exportBigdataCommfeeGj()// - - exportBigdataCommbuildstruGJ()// - exportBigdataCommhousetypeGJ()// - exportBigdataCommbuildyearGj()// - exportBigdataCommhouseuseGJ()// - exportBigdataPositionGJ()// - exportBigdataDistrictFJW()// - exportBigdataAliasFJW()// - - exportBigdataBuildprojFjw() - exportBigdataCommlayoutpicFjw() - exportEvaluateRate() +// exportBigdataCommunityGJRH()// +// exportBigdataBuildingFJW()// +// exportBigdataCommfeeGj()// +// +// exportBigdataCommbuildstruGJ()// +// exportBigdataCommhousetypeGJ()// +// exportBigdataCommbuildyearGj()// +// exportBigdataCommhouseuseGJ()// +// exportBigdataPositionGJ()// +// exportBigdataDistrictFJW()// +// exportBigdataAliasFJW()// +// +// exportBigdataBuildprojFjw() +// exportBigdataCommlayoutpicFjw() +// exportEvaluateRate() // def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger() - exportBigdataCityPrice(queryDate) - exportBigdataDistPrice(queryDate) +// exportBigdataCityPrice(queryDate) +// exportBigdataDistPrice(queryDate) exportBigdataCommPrice(queryDate) //// //ÒòΪÈÚºÏÔݲ»Çп⣬ËùÒÔÐèÒª¼Ó´«Ò»ÕŵÚÈý·½¾ù¼Û±íµ½ÈÚºÏÖУ¬ ´Ó1ÔÂ10ÈÕ¿ªÊ¼´«Ê䣬³¤ÆÚ¹© @@ -79,7 +79,7 @@ class ExportDistrictRongHe extends BaseBatch{ // SSHHelper jschCommand = new SSHHelper(); // System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}")); - exportOk(curDate) +// exportOk(curDate) } @@ -1161,7 +1161,7 @@ class ExportDistrictRongHe extends BaseBatch{ for(def city : citys) { def cityCode = cityCodeMap.get(city) // def sql = "select CITYID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_CITYPRICE where CITYID='${cityCode}' and YMID=${queryDate}";// - def sql = "select CITYID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_CITYPRICE where CITYID='${cityCode}'";// + def sql = "select CITYID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_CITYPRICE where CITYID='${cityCode}' and YMID=${queryDate}";// sqlInstance.eachRow(sql,{r-> def data = [] @@ -1208,7 +1208,7 @@ class ExportDistrictRongHe extends BaseBatch{ def allRows = [] for(def city : citys) { def cityCode = cityCodeMap.get(city) - def sql = "select CITYID,DISTID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_DISTPRICE where CITYID='${cityCode}'";// and YMID=${queryDate} + def sql = "select CITYID,DISTID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_DISTPRICE where CITYID='${cityCode}' and YMID=${queryDate}";// and YMID=${queryDate} sqlInstance.eachRow(sql,{r-> def data = [] data << r.CITYID @@ -1255,7 +1255,7 @@ class ExportDistrictRongHe extends BaseBatch{ def allRows = [] for(def city : citys) { def cityCode = cityCodeMap.get(city) - def sql = "select CITYID, YMID,COMMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_COMMPRICE where CITYID='${cityCode}' ";// and YMID=${queryDate} + def sql = "select CITYID, YMID,COMMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_COMMPRICE where CITYID='${cityCode}' and YMID=${queryDate} ";// sqlInstance.eachRow(sql,{r-> def data = [] data << r.CITYID @@ -1305,7 +1305,7 @@ class ExportDistrictRongHe extends BaseBatch{ println "BIGDATA_COMMLAYOUTPIC_FJW ÉÏ´«FtpÍê³É......" } - def exportOk(curDate){ + def exportOk(curDate){ // def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd") def dataCodeFileName = "/ccb_rh/${curDate}/${curDate}.ok" if(CommonDatautils.isWindow()){ From edef42a8fa095ac3ff21c5df34d1b43018d0ed5f Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 2 Feb 2023 21:54:33 +0800 Subject: [PATCH 61/99] update --- .../ronghe/ExportDistrictRongHe.groovy | 53 +++++++++---------- 1 file changed, 26 insertions(+), 27 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy index fbf70498c..8fe64bdef 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy @@ -38,34 +38,33 @@ class ExportDistrictRongHe extends BaseBatch{ def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd") def new_dists = []//["350604","350605","410307","410308","450181"] //°´Ììµ¼£¨Ã¿ÌìÒ»¸öÎļþ¼Ð£© -// exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists) + exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists) -// if(curDate.toString().endsWith("10")){ -// exportBigdataCommunityGJRH()// -// exportBigdataBuildingFJW()// -// exportBigdataCommfeeGj()// -// -// exportBigdataCommbuildstruGJ()// -// exportBigdataCommhousetypeGJ()// -// exportBigdataCommbuildyearGj()// -// exportBigdataCommhouseuseGJ()// -// exportBigdataPositionGJ()// -// exportBigdataDistrictFJW()// -// exportBigdataAliasFJW()// -// -// exportBigdataBuildprojFjw() -// exportBigdataCommlayoutpicFjw() -// exportEvaluateRate() + if(curDate.toString().endsWith("10")){ + exportBigdataCommunityGJRH()// + exportBigdataBuildingFJW()// + exportBigdataCommfeeGj()// -// def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger() -// exportBigdataCityPrice(queryDate) -// exportBigdataDistPrice(queryDate) + exportBigdataCommbuildstruGJ()// + exportBigdataCommhousetypeGJ()// + exportBigdataCommbuildyearGj()// + exportBigdataCommhouseuseGJ()// + exportBigdataPositionGJ()// + exportBigdataDistrictFJW()// + exportBigdataAliasFJW()// + + exportBigdataBuildprojFjw() + exportBigdataCommlayoutpicFjw() + exportEvaluateRate() + + exportBigdataCityPrice(queryDate) + exportBigdataDistPrice(queryDate) exportBigdataCommPrice(queryDate) -//// //ÒòΪÈÚºÏÔݲ»Çп⣬ËùÒÔÐèÒª¼Ó´«Ò»ÕŵÚÈý·½¾ù¼Û±íµ½ÈÚºÏÖУ¬ ´Ó1ÔÂ10ÈÕ¿ªÊ¼´«Ê䣬³¤ÆÚ¹© -// def thirdPartyMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")); -// exportThirdPartyPrice2(thirdPartyMonth, new_dists) + //ÒòΪÈÚºÏÔݲ»Çп⣬ËùÒÔÐèÒª¼Ó´«Ò»ÕŵÚÈý·½¾ù¼Û±íµ½ÈÚºÏÖУ¬ ´Ó1ÔÂ10ÈÕ¿ªÊ¼´«Ê䣬³¤ÆÚ¹© + def thirdPartyMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")); + exportThirdPartyPrice2(thirdPartyMonth, new_dists) // // // //1ÔÂ10ÈÕ ÐÂÔö5ÕÅ±í£¬µ¼92³Ç£¬ÔöÁ¿Êý¾Ý£¬2ÔÂ10ÈÕµ¼344³Ç£¬ÆäÖÐ252³ÇËùÓÐÀúÊ·Êý¾Ý£¬92³ÇÔöÁ¿£¬3Ôºó¿ÉÄܲ»ÓÃÌṩÊý¾Ý @@ -75,11 +74,11 @@ class ExportDistrictRongHe extends BaseBatch{ // exportBigdataSupplierStatistics(lastMonth) // exportBigdataModelingBase(lastMonth) -// } -// SSHHelper jschCommand = new SSHHelper(); -// System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}")); + } + SSHHelper jschCommand = new SSHHelper(); + System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}")); -// exportOk(curDate) + exportOk(curDate) } From 039924b31c8bc537a0cbad8a1d691f573b8807ae Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 8 Feb 2023 10:57:54 +0800 Subject: [PATCH 62/99] =?UTF-8?q?=E4=B8=B4=E6=97=B6=E5=AF=BC=E5=87=BA?= =?UTF-8?q?=E8=8B=8F=E5=B7=9E=E6=88=BF=E6=BA=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/ccbevaluation/data/TestData.groovy | 115 ++++++++++++++++ .../ronghe/ExportDistrictRongHe.groovy | 130 +++++++++++++++++- .../com/fangjia/script/chongqing/Des3.groovy | 82 +++++++++++ .../fangjia/script/chongqing/Des3Test.groovy | 100 ++++++++++++++ 4 files changed, 426 insertions(+), 1 deletion(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/TestData.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/Des3.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/Des3Test.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/TestData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/TestData.groovy new file mode 100644 index 000000000..857b62715 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/TestData.groovy @@ -0,0 +1,115 @@ +package com.fangjia.script.ccbevaluation.data + +import com.fangjia.core.util.BaiDuApiUtils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject + +class TestData { + + static main(args) { + def task = new TestData() +// task.run() + task.update() + } + + def update(){ + ExcelUtils.modify("C:\\Users\\gaojuan\\Desktop\\ÁúÑÒ1.xlsx", { d -> + def dist = CommonDatautils.findDistrictById(d['A']) + if(dist){ + if(d['E'] && d['G']=="È˹¤"){ + def q = new BasicDBObject() + def baidu = BaiDuApiUtils.geoconv(d['E'].toString().toDouble(), d['F'].toString().toDouble(), 3) + def lng2 = baidu.x + def lat2 = baidu.y + + Long dLat2 = lat2 * 10000000000L + Long dLng2 = lng2 * 10000000000L + + q.put("lat2", dLat2) + q.put("lng2", dLng2) + + Long lng = d['E'].toString().toDouble() * 10000000000L + Long lat = d['F'].toString().toDouble() * 10000000000L + q.put("lng", lng) + q.put("lat", lat) + DbUtils.seaweed.update(new BasicDBObject('_id',d['A']),new BasicDBObject('$set',q)) + } + } + }); + } + + def run(){ + ExcelUtils.modify("C:\\Users\\gaojuan\\Desktop\\ÁúÑÒ.xlsx", { d -> + def dist = CommonDatautils.findDistrictById(d['A']) + if(dist){ + def loc = getLocation(dist.city, dist.region, dist.name) + if(loc){ + d['E'] = loc.x + d['F'] = loc.y + println d['C']+" "+loc.x+" "+loc.y + } + } + +// if(!d['E']){ +// def loc = getAmapLocation("ÁúÑÒÊÐ", d['B'], d['C']) +// if(loc){ +// d['E'] = loc.x +// d['F'] = loc.y +// } +// } + +// if(!d['E']){ +// def dist = CommonDatautils.findDistrictById(d['A']) +// if(dist){ +// def loc = getAmapOldLocation(dist.city, dist.region, d['C']) +// if(loc){ +// d['E'] = loc.x +// d['F'] = loc.y +// } +// } +// +// } + + }); + } + + def getLocation(city, region, name){ + String sql = "select * from community where city='${city}' and region='${region}' and name='${name}' " + def loc = null + ElasticUtils.executeSqlQuery(sql,{ data-> + if(!loc && data.location){ + loc = data.location + } + }) + return loc; + } + + def getAmapLocation(city, region, name){ + String sql = "select * from amap_poi_detail where cityname='${city}' and adname='${region}' and MATCH(name, '${name}') " + def loc = null + ElasticUtils.executeSqlQuery(sql,{ data-> + if(!loc && data.location){ + if(name == data.name){ + loc = data.location + } + } + }) + return loc; + } + + def getAmapOldLocation(city, region, name){ + String sql = "select * from amap_poi where cityname='${city}' and adname='${region}' and MATCH(name, '${name}') " + def loc = null + ElasticUtils.executeSqlQuery(sql,{ data-> + if(!loc && data.location){ + if(name == data.name){ + loc = data.location + } + } + }) + return loc; + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy index 8fe64bdef..d65a4e181 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy @@ -38,7 +38,7 @@ class ExportDistrictRongHe extends BaseBatch{ def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd") def new_dists = []//["350604","350605","410307","410308","450181"] //°´Ììµ¼£¨Ã¿ÌìÒ»¸öÎļþ¼Ð£© - exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists) + // exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists) if(curDate.toString().endsWith("10")){ @@ -75,6 +75,10 @@ class ExportDistrictRongHe extends BaseBatch{ // exportBigdataModelingBase(lastMonth) } + + //2ÔÂ8ºÅ ÁÙʱµ¼³öËÕÖÝÒ»ÄêµÄÊý¾Ý + exportBigdataForsale("2022-01-01", new_dists) //ÏÖÔÚ°´updatetimeµ¼³ö + SSHHelper jschCommand = new SSHHelper(); System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}")); @@ -1304,6 +1308,130 @@ class ExportDistrictRongHe extends BaseBatch{ println "BIGDATA_COMMLAYOUTPIC_FJW ÉÏ´«FtpÍê³É......" } + def exportBigdataForsale(publishDate, new_dists){ + //["±±¾©","ÉϺ£"] + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"); + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat" + } + def headers = []//["³ÇÊÐID","³öÊÛID","¹ÒÅÆ·¿ÎÝID","³ÇÊÐÃû³Æ","ÐÐÕþÇøID","ÐÐÕþÇøÃû³Æ","Ð¡ÇøID","Ð¡ÇøÃû³Æ","Â¥¶°ID","Â¥¶°±àºÅ","Â¥¶°Ãû³Æ","µ¥ÔªID","µ¥Ôª±àºÅ","µ¥ÔªÃû³Æ","»§ID","ÏêϸµØÖ·","½¨ÖþÄê´ú","ȨÊô","ȨÊôÄêÏÞ","µ¥¼Û","×ܼÛ","³öÊÛ¸¶¿î·½Ê½","ËùÔÚÂ¥²ã","×ÜÂ¥²ã","½¨ÖþÀàÐÍ","ÊÒÄڽṹ","½¨Öþ½á¹¹","½¨ÖþÃæ»ý","Ì×ÄÚÃæ»ý","»§ÐÍ","ÊÒ","Ìü","³ø","ÎÀ","³¯Ïò","×°ÐÞÇé¿ö","ÊÇ·ñÓеçÌÝ","¼¸Ìݼ¸»§","·¢²¼Ê±¼ä","µ½ÆÚʱ¼ä","ÊÇ·ñÊ×·¢","·¿Ô´Êý¾ÝÀ´Ô´","Öн鹫˾ID","Öн鹫˾Ãû³Æ","·¢²¼ÈË","·¢²¼È˵绰","ÃèÊöÐÅÏ¢","±¸×¢","´´½¨Ê±¼ä","¸üÐÂʱ¼ä","À´Ô´¹«Ë¾","¸½¼ÓÐÅÏ¢"] + def index = 0 + citys = ["ËÕÖÝ"] + for(def city : citys){ + def cityCode = cityCodeMap.get(city) + + def allRows = [] + if(new_dists){ + def newstr = "('"+new_dists.join("','")+"')" + def sql1 = "select CITYID,FORSALEID,PROPCODE,CITYNAME,DISTID,DISTNAME,COMMID,COMMNAME, BUILDID,BUILDNO, BUILDNAME,UNITID,UNITNO,UNITNAME,HOUSEID,ADDRESS,BUILDYEAR,PROPTYPE,PROPYEARS,UNITPRICE,TOTALPRICE,PAYTYPE,FLOORNUM,FLOORCNT,\n" + + " BUILDTYPE, HOUSESTRU,BUILDSTRU,BUILDAREA,LIVINGAREA,HOUSESTYLE,ROOMCNT,LIVINGROOMCNT,KITCHENCNT, BATHROOMCNT,HOUSEASPECT,FITMENT,IFELEVATOR,ELEVATORHOUSE,to_char(PUBLISHTIME,'yyyy-mm-dd hh24:mi:ss') as PUBLISHTIME,to_char(EXPIRETIME,'yyyy-mm-dd hh24:mi:ss') as EXPIRETIME,\n" + + " IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH from bigdata_forsale_gj " + + "where cityId='${cityCode}' and updatetime>=to_date('${publishDate}','yyyy-MM-dd') and distid not in ${newstr}"; + def rets1 = getHouseDatas(sql1) + allRows.addAll(rets1) + }else{ + def sql = "select CITYID,FORSALEID,PROPCODE,CITYNAME,DISTID,DISTNAME,COMMID,COMMNAME, BUILDID,BUILDNO, BUILDNAME,UNITID,UNITNO,UNITNAME,HOUSEID,ADDRESS,BUILDYEAR,PROPTYPE,PROPYEARS,UNITPRICE,TOTALPRICE,PAYTYPE,FLOORNUM,FLOORCNT,\n" + + " BUILDTYPE, HOUSESTRU,BUILDSTRU,BUILDAREA,LIVINGAREA,HOUSESTYLE,ROOMCNT,LIVINGROOMCNT,KITCHENCNT, BATHROOMCNT,HOUSEASPECT,FITMENT,IFELEVATOR,ELEVATORHOUSE,to_char(PUBLISHTIME,'yyyy-mm-dd hh24:mi:ss') as PUBLISHTIME,to_char(EXPIRETIME,'yyyy-mm-dd hh24:mi:ss') as EXPIRETIME,\n" + + " IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH from bigdata_forsale_gj where cityId='${cityCode}' and updatetime>=to_date('${publishDate}','yyyy-MM-dd')"; + allRows = getHouseDatas(sql) + } + if(allRows.size()>0){ + index = index + 1 + if(index == 1){ + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + }else{ + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); + } + + } + } + + if(new_dists){//²¹³ä±»ÐÞÇøÓòµÄÊý¾Ý + def newstr = "('"+new_dists.join("','")+"')" + def sql2 = "select CITYID,FORSALEID,PROPCODE,CITYNAME,DISTID,DISTNAME,COMMID,COMMNAME, BUILDID,BUILDNO, BUILDNAME,UNITID,UNITNO,UNITNAME,HOUSEID,ADDRESS,BUILDYEAR,PROPTYPE,PROPYEARS,UNITPRICE,TOTALPRICE,PAYTYPE,FLOORNUM,FLOORCNT,\n" + + " BUILDTYPE, HOUSESTRU,BUILDSTRU,BUILDAREA,LIVINGAREA,HOUSESTYLE,ROOMCNT,LIVINGROOMCNT,KITCHENCNT, BATHROOMCNT,HOUSEASPECT,FITMENT,IFELEVATOR,ELEVATORHOUSE,to_char(PUBLISHTIME,'yyyy-mm-dd hh24:mi:ss') as PUBLISHTIME,to_char(EXPIRETIME,'yyyy-mm-dd hh24:mi:ss') as EXPIRETIME,\n" + + " IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH " + + "from bigdata_forsale_gj where distid in ${newstr}" + def rets2 = getHouseDatas(sql2) + if(rets2.size()>0){ + FileUtils.batchCreateFile(rets2, fileName, headers, "utf-8", true); + } + } + println "ÈÚºÏ BIGDATA_FORSALE_GJ Êý¾ÝÎļþÊä³öÍê³É......" +// CcbFtpUtils.ftpUpload(null, fileName, curDate) +// println "ÈÚºÏ BIGDATA_FORSALE_GJ ÉÏ´«FtpÍê³É......" + } + + def getHouseDatas(sql){ + def allRows = [] + def index = 0 + sqlInstance.eachRow(sql,{r-> + index = index+1 + println r.CITYNAME +" "+index + def data = [] + data << r.CITYID + data << r.FORSALEID + data << r.PROPCODE + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.COMMID + data << r.COMMNAME + data << r.BUILDID + data << r.BUILDNO + data << r.BUILDNAME + data << r.UNITID + data << r.UNITNO + data << r.UNITNAME + data << r.HOUSEID + data << r.ADDRESS + data << r.BUILDYEAR + data << r.PROPTYPE + data << r.PROPYEARS + data << r.UNITPRICE + data << r.TOTALPRICE + data << r.PAYTYPE + data << r.FLOORNUM + data << r.FLOORCNT + data << r.BUILDTYPE + data << r.HOUSESTRU + data << r.BUILDSTRU + data << r.BUILDAREA + data << r.LIVINGAREA + data << r.HOUSESTYLE + data << r.ROOMCNT + data << r.LIVINGROOMCNT + data << r.KITCHENCNT + data << r.BATHROOMCNT + data << r.HOUSEASPECT + data << r.FITMENT + data << r.IFELEVATOR + data << r.ELEVATORHOUSE + data << r.PUBLISHTIME + data << r.EXPIRETIME + data << r.IFFIRST + data << r.DATASOURCE + data << r.AGENCYID + data << r.AGENCYNAME + data << r.PUBLISHER + data << r.PHONE + data << "" + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + data << r.ATTACH + allRows << data +// if(allRows.size()%200000==0){ +// FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); +// allRows.clear() +// } + }); + return allRows + } + + def exportOk(curDate){ // def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd") def dataCodeFileName = "/ccb_rh/${curDate}/${curDate}.ok" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/Des3.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/Des3.groovy new file mode 100644 index 000000000..012ddd921 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/Des3.groovy @@ -0,0 +1,82 @@ +package com.fangjia.script.chongqing +import com.sun.org.apache.xml.internal.security.utils.Base64; +import org.apache.commons.lang3.StringUtils; + +import javax.crypto.Cipher; +import javax.crypto.SecretKeyFactory; +import javax.crypto.spec.DESedeKeySpec; +import javax.crypto.spec.IvParameterSpec; +import java.security.Key; + +class Des3 { + private final static String encoding = "UTF-8"; + + static main(args) { + def task = new Des3() + task.run() + } + def run(){ + try { + String key = "111036369260679051122113"; + String iv = "23235645";//IV length must be 8 bytes long + //¼ÓÃÜ + String encryptStr = "Lk+AtbEWoNF0eQNrSsGoNJjRFuvnCaRKaypcAnaRvGpHtc2KmMOMbufut+JDWP5Rq/4qNAWsEch0iZgE5z2CTnAwCjoRTUl0"//encode("test", key, iv); + System.out.println("ÃÜÎÄ: " + encryptStr); + //½âÃÜ + String decryptStr = decode(encryptStr, key, iv); + System.out.println("Ã÷ÎÄ: " + decryptStr); + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * ·½·¨ÃèÊö£º3DES¼ÓÃÜ + * @author hyang + * + * @param plainText Ã÷ÎÄ + * @param secretKey ÃÜÔ¿ + * @param iv ¼ÓÃÜÏòÁ¿ + * @return String ÃÜÎÄ + * @throws Exception + */ + public static String encode(String plainText, String secretKey, String iv) + throws Exception { + Key deskey = null; + DESedeKeySpec spec = new DESedeKeySpec(secretKey.getBytes()); + SecretKeyFactory keyfactory = SecretKeyFactory.getInstance("desede"); + deskey = keyfactory.generateSecret(spec); + Cipher cipher = Cipher.getInstance("desede/CBC/PKCS5Padding"); + IvParameterSpec ips = new IvParameterSpec(iv.getBytes()); + cipher.init(Cipher.ENCRYPT_MODE, deskey, ips); + byte[] encryptData = cipher.doFinal(plainText.getBytes(encoding)); + return Base64.encode(encryptData); + } + + /** + * ·½·¨ÃèÊö£º 3DES½âÃÜ + * @author hyang + * + * @param encryptText ÃÜÎÄ + * @param secretKey ÃÜÔ¿ + * @param iv ¼ÓÃÜÏòÁ¿ + * @return String Ã÷ÎÄ + * @throws Exception + */ + public static String decode(String encryptText, String secretKey, String iv) + throws Exception { + if (StringUtils.isBlank(encryptText) || StringUtils.isBlank(secretKey)) { + return ""; + } + Key deskey = null; + DESedeKeySpec spec = new DESedeKeySpec(secretKey.getBytes()); + SecretKeyFactory keyfactory = SecretKeyFactory.getInstance("desede"); + deskey = keyfactory.generateSecret(spec); + Cipher cipher = Cipher.getInstance("desede/CBC/NoPadding"); //desede/CBC/NoPadding ÎÞÌî³ä desede/CBC/PKCS5Padding ¼ÓÃÜÐèÒªÌî³ä£¬½âÃÜÒ»°ã¼Ó²»¼Ó¶¼½á¹ûÒ»Ö + IvParameterSpec ips = new IvParameterSpec(iv.getBytes()); + cipher.init(Cipher.DECRYPT_MODE, deskey, ips); + byte[] decryptData = cipher.doFinal(Base64.decode(encryptText)); + return new String(decryptData, encoding).trim(); + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/Des3Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/Des3Test.groovy new file mode 100644 index 000000000..a9a798124 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/Des3Test.groovy @@ -0,0 +1,100 @@ +package com.fangjia.script.chongqing + +import com.fangjia.SecretUtils + +import javax.crypto.Cipher +import javax.crypto.SecretKey +import javax.crypto.spec.SecretKeySpec + +class Des3Test { + //¶¨Òå¼ÓÃÜËã·¨£¬ÓÐDES¡¢DESede(¼´3DES)¡¢Blowfish + private static final String Algorithm = "DESede"; + private static final String PASSWORD_CRYPT_KEY = "111036369260679051122113"; + + + /** + * ¼ÓÃÜ·½·¨ + * @param src Ô´Êý¾ÝµÄ×Ö½ÚÊý×é + * @return + */ + public static byte[] encryptMode(byte[] src) { + try { + SecretKey deskey = new SecretKeySpec(build3DesKey(PASSWORD_CRYPT_KEY), Algorithm); //Éú³ÉÃÜÔ¿ + Cipher c1 = Cipher.getInstance(Algorithm); //ʵÀý»¯¸ºÔð¼ÓÃÜ/½âÃܵÄCipher¹¤¾ßÀà + c1.init(Cipher.ENCRYPT_MODE, deskey); //³õʼ»¯Îª¼ÓÃÜģʽ + return c1.doFinal(src); + } catch (java.security.NoSuchAlgorithmException e1) { + e1.printStackTrace(); + } catch (javax.crypto.NoSuchPaddingException e2) { + e2.printStackTrace(); + } catch (java.lang.Exception e3) { + e3.printStackTrace(); + } + return null; + } + + + /** + * ½âÃܺ¯Êý + * @param src ÃÜÎĵÄ×Ö½ÚÊý×é + * @return + */ + public static byte[] decryptMode(byte[] src) { + try { + SecretKey deskey = new SecretKeySpec(build3DesKey(PASSWORD_CRYPT_KEY), Algorithm); + Cipher c1 = Cipher.getInstance(Algorithm); + c1.init(Cipher.DECRYPT_MODE, deskey); //³õʼ»¯Îª½âÃÜģʽ + return c1.doFinal(src); + } catch (java.security.NoSuchAlgorithmException e1) { + e1.printStackTrace(); + } catch (javax.crypto.NoSuchPaddingException e2) { + e2.printStackTrace(); + } catch (java.lang.Exception e3) { + e3.printStackTrace(); + } + return null; + } + + + /* + * ¸ù¾Ý×Ö·û´®Éú³ÉÃÜÔ¿×Ö½ÚÊý×é + * @param keyStr ÃÜÔ¿×Ö·û´® + * @return + * @throws UnsupportedEncodingException + */ + public static byte[] build3DesKey(String keyStr) throws UnsupportedEncodingException{ + byte[] key = new byte[24]; //ÉùÃ÷Ò»¸ö24λµÄ×Ö½ÚÊý×飬ĬÈÏÀïÃæ¶¼ÊÇ0 + byte[] temp = keyStr.getBytes("UTF-8"); //½«×Ö·û´®×ª³É×Ö½ÚÊý×é + + /* + * Ö´ÐÐÊý×鿽±´ + * System.arraycopy(Ô´Êý×飬´ÓÔ´Êý×éÄÄÀ↑ʼ¿½±´£¬Ä¿±êÊý×飬¿½±´¶àÉÙλ) + */ + if(key.length > temp.length){ + //Èç¹ûtemp²»¹»24룬Ôò¿½±´tempÊý×éÕû¸ö³¤¶ÈµÄÄÚÈݵ½keyÊý×éÖÐ + System.arraycopy(temp, 0, key, 0, temp.length); + }else{ + //Èç¹ûtemp´óÓÚ24룬Ôò¿½±´tempÊý×é24¸ö³¤¶ÈµÄÄÚÈݵ½keyÊý×éÖÐ + System.arraycopy(temp, 0, key, 0, key.length); + } + return key; + } + + /** + * @param args + */ + public static void main(String[] args) { + String msg = "3DES¼ÓÃܽâÃܰ¸Àý"; + System.out.println("¡¾¼ÓÃÜǰ¡¿£º" + msg); + + //¼ÓÃÜ +// byte[] secretArr = Des3Test.encryptMode(msg.getBytes()); +// System.out.println("¡¾¼ÓÃܺ󡿣º" + new String(secretArr)); + + String s = "Lk+AtbEWoNF0eQNrSsGoNJjRFuvnCaRKaypcAnaRvGpHtc2KmMOMbufut+JDWP5Rq/4qNAWsEch0iZgE5z2CTnAwCjoRTUl0" + + //½âÃÜ + byte[] myMsgArr = Des3Test.decryptMode(s.getBytes("UTF-8")); + System.out.println("¡¾½âÃܺ󡿣º" + new String(myMsgArr)); + } +} From 3be00b6aaac94303b258a585a73b42eb5e41c2a9 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 8 Feb 2023 11:19:02 +0800 Subject: [PATCH 63/99] update --- .../ronghe/BigdataCitypriceBatch.groovy | 6 ++--- .../ronghe/ExportDistrictRongHe.groovy | 7 +++-- .../fuzhou/DistrictPriceExportBatch.groovy | 2 +- .../TestCrawlTotalCount.groovy | 4 +-- .../fangjia/script/util/BaiduMapUtils.groovy | 27 +++++++++---------- .../yinjihaun/amap/GaoDeApiUtils.groovy | 2 +- 6 files changed, 22 insertions(+), 26 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy index 2d248a235..e3b08e9e1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy @@ -27,7 +27,7 @@ class BigdataCitypriceBatch extends BaseBatch{ public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80) - def curRunDate = 202212//DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-1), "yyyyMM").toInteger(); + def curRunDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-1), "yyyyMM").toInteger(); static main(args) { runBatch(new BigdataCitypriceBatch(), args) @@ -36,8 +36,8 @@ class BigdataCitypriceBatch extends BaseBatch{ @Override void process(Object args) { def citys = CcbCommonUtils.getRunningCitys(args) - def startDate = 20221201//DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-1), "yyyyMMdd").toInteger(); - def endDate = 20221231//DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(new Date(),-1), "yyyyMMdd").toInteger(); + def startDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-1), "yyyyMMdd").toInteger(); + def endDate = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(new Date(),-1), "yyyyMMdd").toInteger(); def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() cityRun(citys, startDate, endDate, cityCodeDataMap) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy index d65a4e181..33a6b05c2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy @@ -30,7 +30,7 @@ class ExportDistrictRongHe extends BaseBatch{ @Override void process(Object args) { - def queryDate = 202212//DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger() + def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger() citys = CcbCommonUtils.getRunningCitys(args) @@ -38,7 +38,7 @@ class ExportDistrictRongHe extends BaseBatch{ def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd") def new_dists = []//["350604","350605","410307","410308","450181"] //°´Ììµ¼£¨Ã¿ÌìÒ»¸öÎļþ¼Ð£© - // exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists) + exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists) if(curDate.toString().endsWith("10")){ @@ -77,7 +77,7 @@ class ExportDistrictRongHe extends BaseBatch{ } //2ÔÂ8ºÅ ÁÙʱµ¼³öËÕÖÝÒ»ÄêµÄÊý¾Ý - exportBigdataForsale("2022-01-01", new_dists) //ÏÖÔÚ°´updatetimeµ¼³ö +// exportBigdataForsale("2022-01-01", new_dists) //ÏÖÔÚ°´updatetimeµ¼³ö SSHHelper jschCommand = new SSHHelper(); System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}")); @@ -1317,7 +1317,6 @@ class ExportDistrictRongHe extends BaseBatch{ } def headers = []//["³ÇÊÐID","³öÊÛID","¹ÒÅÆ·¿ÎÝID","³ÇÊÐÃû³Æ","ÐÐÕþÇøID","ÐÐÕþÇøÃû³Æ","Ð¡ÇøID","Ð¡ÇøÃû³Æ","Â¥¶°ID","Â¥¶°±àºÅ","Â¥¶°Ãû³Æ","µ¥ÔªID","µ¥Ôª±àºÅ","µ¥ÔªÃû³Æ","»§ID","ÏêϸµØÖ·","½¨ÖþÄê´ú","ȨÊô","ȨÊôÄêÏÞ","µ¥¼Û","×ܼÛ","³öÊÛ¸¶¿î·½Ê½","ËùÔÚÂ¥²ã","×ÜÂ¥²ã","½¨ÖþÀàÐÍ","ÊÒÄڽṹ","½¨Öþ½á¹¹","½¨ÖþÃæ»ý","Ì×ÄÚÃæ»ý","»§ÐÍ","ÊÒ","Ìü","³ø","ÎÀ","³¯Ïò","×°ÐÞÇé¿ö","ÊÇ·ñÓеçÌÝ","¼¸Ìݼ¸»§","·¢²¼Ê±¼ä","µ½ÆÚʱ¼ä","ÊÇ·ñÊ×·¢","·¿Ô´Êý¾ÝÀ´Ô´","Öн鹫˾ID","Öн鹫˾Ãû³Æ","·¢²¼ÈË","·¢²¼È˵绰","ÃèÊöÐÅÏ¢","±¸×¢","´´½¨Ê±¼ä","¸üÐÂʱ¼ä","À´Ô´¹«Ë¾","¸½¼ÓÐÅÏ¢"] def index = 0 - citys = ["ËÕÖÝ"] for(def city : citys){ def cityCode = cityCodeMap.get(city) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy index 8d77112a6..dd6f1e2dc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy @@ -33,7 +33,7 @@ class DistrictPriceExportBatch extends BaseBatch{ static DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com") - def specialDistsPriceMap = ["½ò̩дå7ºÅ¡¢9-17ºÅ":53620,"½ò̩дå14ºÅ":76230,"½ò̩дå1-6ºÅ¡¢8ºÅ":32580] + def specialDistsPriceMap = ["½ò̩дå7ºÅ¡¢9-13ºÅ¡¢15-17ºÅ":53620,"½ò̩дå14ºÅ":76230,"½ò̩дå1-6ºÅ¡¢8ºÅ":32580] static main(args) { runBatch(new DistrictPriceExportBatch(), args) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy index df803e640..2d9309dae 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy @@ -14,8 +14,8 @@ class TestCrawlTotalCount { def run(){ def sources = ["·¿¹À¹À","Öî¸ðÕÒ·¿","°²¾Ó¿Í","ËÑ·¿","ÖÐÔ­µØ²ú","¸Ï¼¯","Á´¼Ò","Íò©Ê¿","³ÇÊз¿²ú","±´¿ÇÕÒ·¿","Q·¿","Q·¿(ÐÂ)","ÖÐÔ­µØ²ú(ÐÂ)","ÀÖÓмÒ","ÀÖÓмÒ(ÐÂ)","·¿Ì½","·¿¶à¶à","С×ʼÒ","ÎÒ°®ÎÒ¼Ò","ÂóÌï","·¿Í¾","C21","ÀÖ¾Ó","·¿²ú³¬ÊÐ","58ͬ³Ç","365ÌÔ·¿","¼ªÎÝ"] for(def source :sources){ - def count1 = findTotalCount(source, 202211); - def count2 = findTotalCount(source, 202212); + def count1 = findTotalCount(source, 202212); + def count2 = findTotalCount(source, 202301); println source +" "+count1 +" "+count2 } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/BaiduMapUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/BaiduMapUtils.groovy index 3bfe575e9..2eb3fcfed 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/BaiduMapUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/BaiduMapUtils.groovy @@ -1,5 +1,6 @@ package com.fangjia.script.util +import com.fangjia.core.util.HttpRequest import com.fangjia.core.util.RandomUtil import com.fangjia.script.addrepo.InvokeHttpRequest; import groovy.json.JsonSlurper @@ -103,24 +104,20 @@ class BaiduMapUtils { int recount = 0; while(!stop && recount<3) { try{ - def ret = ABuYunUtil.doGet(url, null, null, "utf-8"); - if (ret) { - JSONObject jsonObj = JSONObject.fromObject(ret); - - if(jsonObj.getInt("status") == 121){ - println "key over" - changeAK() - recount++; + HttpRequest request = new HttpRequest(); + try{ + String result = request.get(url,null,"UTF-8"); + JSONObject json = JSONObject.fromObject(result); + if(json.getInt("status") >= 300){ + changeAK(); + json = searchByAddress(address); }else{ - def result = jsonObj.result; - if(result){ - stop = true - return result; - }else{ - stop = true - } + return json.result } + }catch (Exception e){ + e.printStackTrace(); } + return null; }catch (Exception e){ recount++; println e.getMessage() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy index fdcf818c3..4d301536b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy @@ -12,7 +12,7 @@ import org.jsoup.Jsoup import java.util.zip.GZIPInputStream class GaoDeApiUtils { - def static GAO_DE_AK = "90010d53a16ce9df544acab297645106" + def static GAO_DE_AK = "c6fcd4bfbc40d73ee762b8115dca3390" /** * ¾­Î³¶ÈתµØÖ·
    From f6bf27b743d8bacdf680f311ea424f56fb168d6d Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 8 Feb 2023 14:33:34 +0800 Subject: [PATCH 64/99] =?UTF-8?q?=E5=AF=BC=E8=8B=8F=E5=B7=9E=E6=88=BF?= =?UTF-8?q?=E6=BA=90=20=E5=87=8F=E5=B0=91=E4=B8=80=E4=B8=AA=E5=AD=97?= =?UTF-8?q?=E6=AE=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ccbevaluation/ronghe/ExportDistrictRongHe.groovy | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy index 33a6b05c2..d43bfa2cb 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy @@ -77,7 +77,7 @@ class ExportDistrictRongHe extends BaseBatch{ } //2ÔÂ8ºÅ ÁÙʱµ¼³öËÕÖÝÒ»ÄêµÄÊý¾Ý -// exportBigdataForsale("2022-01-01", new_dists) //ÏÖÔÚ°´updatetimeµ¼³ö + exportBigdataForsale("2022-01-01", new_dists) //ÏÖÔÚ°´updatetimeµ¼³ö SSHHelper jschCommand = new SSHHelper(); System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}")); @@ -1310,10 +1310,11 @@ class ExportDistrictRongHe extends BaseBatch{ def exportBigdataForsale(publishDate, new_dists){ //["±±¾©","ÉϺ£"] + citys = ["ËÕÖÝ"] def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"); - def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat" + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_FORSALE_FJW_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat" + fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_FORSALE_FJW_ADD_${curDate}_0001.dat" } def headers = []//["³ÇÊÐID","³öÊÛID","¹ÒÅÆ·¿ÎÝID","³ÇÊÐÃû³Æ","ÐÐÕþÇøID","ÐÐÕþÇøÃû³Æ","Ð¡ÇøID","Ð¡ÇøÃû³Æ","Â¥¶°ID","Â¥¶°±àºÅ","Â¥¶°Ãû³Æ","µ¥ÔªID","µ¥Ôª±àºÅ","µ¥ÔªÃû³Æ","»§ID","ÏêϸµØÖ·","½¨ÖþÄê´ú","ȨÊô","ȨÊôÄêÏÞ","µ¥¼Û","×ܼÛ","³öÊÛ¸¶¿î·½Ê½","ËùÔÚÂ¥²ã","×ÜÂ¥²ã","½¨ÖþÀàÐÍ","ÊÒÄڽṹ","½¨Öþ½á¹¹","½¨ÖþÃæ»ý","Ì×ÄÚÃæ»ý","»§ÐÍ","ÊÒ","Ìü","³ø","ÎÀ","³¯Ïò","×°ÐÞÇé¿ö","ÊÇ·ñÓеçÌÝ","¼¸Ìݼ¸»§","·¢²¼Ê±¼ä","µ½ÆÚʱ¼ä","ÊÇ·ñÊ×·¢","·¿Ô´Êý¾ÝÀ´Ô´","Öн鹫˾ID","Öн鹫˾Ãû³Æ","·¢²¼ÈË","·¢²¼È˵绰","ÃèÊöÐÅÏ¢","±¸×¢","´´½¨Ê±¼ä","¸üÐÂʱ¼ä","À´Ô´¹«Ë¾","¸½¼ÓÐÅÏ¢"] def index = 0 @@ -1420,7 +1421,7 @@ class ExportDistrictRongHe extends BaseBatch{ data << r.CREATETIME data << r.UPDATETIME data << r.SOURCECODE - data << r.ATTACH +// data << r.ATTACH //ÈںϷ¿Ô´Ã»ÓÐÕâ¸ö×Ö¶Î allRows << data // if(allRows.size()%200000==0){ // FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); From b550e3b1f244403b06edf0da55ae79cf73127230 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 9 Feb 2023 17:01:58 +0800 Subject: [PATCH 65/99] =?UTF-8?q?=E8=9E=8D=E5=90=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ccbevaluation/CcbCommonUtils.groovy | 3 +- .../ronghe/ExportDistrictRongHe.groovy | 13 +++--- .../script/wenyan/temp/DongGuanBank.groovy | 43 +++++++++++++++++++ 3 files changed, 52 insertions(+), 7 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/DongGuanBank.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index 24d06b991..908f4adf5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -16,7 +16,8 @@ import groovy.sql.Sql class CcbCommonUtils { // - public static def citys_all = [ "ÉϺ£","³¤´º","ÑïÖÝ","±±¾©","ÖØÇì","Ìì½ò","³É¶¼","ÉòÑô","´óÁ¬","°²Çì","Î÷Äþ","ÈÕÕÕ","Î人","Î÷°²","ËÕÖÝ","¹ãÖÝ","ÄϾ©","Çൺ","¼ÃÄÏ","Äþ²¨","³¤É³","ʯ¼Òׯ","·ðɽ","¼ÎÐË","Òø´¨","ÄÏͨ","ÀÈ·»","ÖÐɽ","Ö£ÖÝ","¹þ¶û±õ","ÑĮ̀","ÐìÖÝ","ÉÜÐË","ÏÃÃÅ","À¥Ã÷","Äϲý","¹óÑô","Ö麣","¶«Ý¸","Õò½­","ÂåÑô","ÎÂÖÝ","À¼ÖÝ","¸£ÖÝ","»ÝÖÝ","ºôºÍºÆÌØ","³£ÖÝ","ÌÆÉ½","»´°²","ºþÖÝ","¾Å½­","̫ԭ","±±º£","ÖêÖÞ","×Ͳ©","ÑγÇ","ÎÚ³ľÆë","º£¿Ú","ÄÏÄþ","³£µÂ","̨ÖÝ","½ð»ª","¼ªÁÖ","ÔÀÑô","ÁøÖÝ","ÇåÔ¶","Õ¿½­","ÕżҿÚ","±£¶¨","ÉîÛÚ","º¼ÖÝ","ºÏ·Ê","ÎÞÎý","ȪÖÝ","×ñÒå","ÕØÇì","ÕÄÖÝ","Ò˲ý","Îߺþ","Ì©ÖÝ","ÉÇÍ·","ÈýÑÇ","ÇØ»Êµº","Âí°°É½","Á¬ÔƸÛ","²×ÖÝ","À­Èø","½õÖÝ","´óͬ","ÎIJý","±£Í¤","ÁêË®","Ϋ·»","ÁÙÒÊ","ºªµ¦","¼ÃÄþ","Íþº£","ÃàÑô","÷ÖÝ","ƽ¶¥É½","½­ÃÅ","¸ÓÖÝ","Ì©°²","ÏåÑô","ÄÏÑô","¹ðÁÖ","ÁijÇ","ÁÙ·Ú","µÂÖÝ","ÉÌÇð","ºâÑô","ËÞǨ","°üÍ·","μÄÏ","ÏÌÑô","Ðí²ý","±õÖÝ","ÐĮ̈","Äϳä","°²Ñô","ÐÂÏç","Ôæ×¯","³üÖÝ","ÁúÑÒ","±¦¼¦","ÐÅÑô","Ô˳Ç","ºâË®","¶«Óª","½úÖÐ","Т¸Ð","³¤ÖÎ","¸·Ñô","ÉØ¹Ø","ºÊÔó","µÂÑô","ÄþµÂ","¾£ÖÝ","ÀÖɽ","פÂíµê","ÄÏÆ½","ººÖÐ","ÓÜÁÖ","ÒË´º","°°É½","¿ª·â","ÖÜ¿Ú","Ë绯","ÏæÌ¶","»Æ¸Ô","³ÐµÂ","½¹×÷","ĵµ¤½­","ÆÎÌï","ÉÏÈÄ","üɽ","ÆëÆë¹þ¶û","³»ÖÝ","Óª¿Ú","°ö²º","´óÇì","³à·å","ºÓÔ´","ÂÀÁº","ËìÄþ","äðºÓ","ÈýÃ÷","ïÃû","å§Ñô","Çú¾¸","ƽÁ¹","¶õ¶û¶à˹","Ê®Ñß","¼ª°²","»ÆÊ¯","¼Ñľ˹","ÈýÃÅÏ¿","½ÒÑô","ÐÃÖÝ","ͨÁÉ","ãòÖÝ","Áù°²","³±ÖÝ","áéÖÝ","»´ÄÏ","¾£ÃÅ","ÓÀÖÝ","ºôÂ×±´¶û","ºù«µº","µ¤¶«","ÑÓ±ß","Ðû³Ç","Ò˱ö","ÇìÑô","½ú³Ç","Ñô½­","ËÞÖÝ","ÑÓ°²","ÏÌÄþ","»³»¯","»ÆÉ½","ÖÛɽ","ÉÇβ","Å̽õ","³¯Ñô","ÀöË®","¸§ÖÝ","ÌúÁë","ºìºÓ","¤ÄÏ","¹ã°²","¸§Ë³","×Ô¹±","´ïÖÝ","ÓñÁÖ","ÓñϪ","°×Òø","ÙñÖÝ","ÉÛÑô","¹ãÔª","²ý¼ª","ÒæÑô","ÕÅÒ´","¦µ×","ÑôȪ","ÌìË®","ËæÖÝ","Ë·ÖÝ","´óÀí","ͨ»¯","³þÐÛ","°ÍÑåÄ×¶û","ÎÚÀ¼²ì²¼","ÐÂÓà","¾°µÂÕò","ËÉÔ­","ÁÉÑô","¸·ÐÂ","¾ÆÈª","Ç­ÄÏ","»´±±","ËÄÆ½","ÎàÖÝ","Ç­¶«ÄÏ","°ÍÖÐ","º×±Ú","¼¦Î÷","ÄÚ½­","ÇÕÖÝ","³ØÖÝ","ÏÉÌÒ","¶÷Ê©","±¾Ïª","×ÊÑô","ÁùÅÌË®","°ÙÉ«","°×³Ç","ÁÙÏÄ","ÒÁÀç","¹ó¸Û","°×ɽ","˫Ѽɽ","ÎâÖÒ","¶õÖÝ","Àö½­","°²¿µ","±Ï½Ú","ÔÆ¸¡","°ÍÒô¹ùÀã","°²Ë³","°¢¿ËËÕ","ºÚºÓ","Í­Áê","ÑŰ²","ʯ×ìɽ","ƼÏç","º×¸Ú","ÎýÁÖ¹ùÀÕÃË","Í­ÈÊ","·À³Ç¸Û","¿¦Ê²","Ð˰²ÃË","ºÓ³Ø","Í­´¨","ÆÕ¶ý","±£É½","Á¹É½ÖÝ","ӥ̶","¶¨Î÷","ÒÁ´º","ÖÐÎÀ","Ç­Î÷ÄÏ","ÉÌÂå","ÎÚº£","ÎäÍþ","º£Î÷ÖÝ","À´±ö","ÕÑͨ","ÏæÎ÷","¹þÃÜ","Õżҽç","Î÷Ë«°æÄÉÖÝ","Æß̨ºÓ","¹ÌÔ­","ÅÊÖ¦»¨","µÂºêÖÝ","³ç×ó","ÁÉÔ´","¼ÎÓø¹Ø","¿ËÀ­ÂêÒÀ","ÁÙ²×","DZ½­","ºØÖÝ","ÌìÃÅ","º£¶«","Ëþ³Ç","ÙÙÖÝ","½ð²ý","²©¶ûËþÀ­","ͳ·¬","´óÐ˰²Áë","°¢ÀÕÌ©","ºÍÌï","µÏÇìÖÝ","ÈÕ¿¦Ôò","ÁÖÖ¥","°¢°ÓÖÝ","°¢À­ÉÆÃË","°¢Àï","²ý¶¼","¸ÊÄÏ","¸Ê×ÎÖÝ","¹ûÂåÖÝ","º£±±ÖÝ","º£ÄÏÖÝ","»ÆÄÏÖÝ","¿Ë×ÎÀÕËÕ","ÄÇÇú","Å­½­ÖÝ","Èýɳ","ɽÄÏ","ÉñÅ©¼Ü","ÎÄɽÖÝ","ÓñÊ÷ÖÝ"] +// public static def citys_all = [ "ÉϺ£","³¤´º","ÑïÖÝ","±±¾©","ÖØÇì","Ìì½ò","³É¶¼","ÉòÑô","´óÁ¬","°²Çì","Î÷Äþ","ÈÕÕÕ","Î人","Î÷°²","ËÕÖÝ","¹ãÖÝ","ÄϾ©","Çൺ","¼ÃÄÏ","Äþ²¨","³¤É³","ʯ¼Òׯ","·ðɽ","¼ÎÐË","Òø´¨","ÄÏͨ","ÀÈ·»","ÖÐɽ","Ö£ÖÝ","¹þ¶û±õ","ÑĮ̀","ÐìÖÝ","ÉÜÐË","ÏÃÃÅ","À¥Ã÷","Äϲý","¹óÑô","Ö麣","¶«Ý¸","Õò½­","ÂåÑô","ÎÂÖÝ","À¼ÖÝ","¸£ÖÝ","»ÝÖÝ","ºôºÍºÆÌØ","³£ÖÝ","ÌÆÉ½","»´°²","ºþÖÝ","¾Å½­","̫ԭ","±±º£","ÖêÖÞ","×Ͳ©","ÑγÇ","ÎÚ³ľÆë","º£¿Ú","ÄÏÄþ","³£µÂ","̨ÖÝ","½ð»ª","¼ªÁÖ","ÔÀÑô","ÁøÖÝ","ÇåÔ¶","Õ¿½­","ÕżҿÚ","±£¶¨","ÉîÛÚ","º¼ÖÝ","ºÏ·Ê","ÎÞÎý","ȪÖÝ","×ñÒå","ÕØÇì","ÕÄÖÝ","Ò˲ý","Îߺþ","Ì©ÖÝ","ÉÇÍ·","ÈýÑÇ","ÇØ»Êµº","Âí°°É½","Á¬ÔƸÛ","²×ÖÝ","À­Èø","½õÖÝ","´óͬ","ÎIJý","±£Í¤","ÁêË®","Ϋ·»","ÁÙÒÊ","ºªµ¦","¼ÃÄþ","Íþº£","ÃàÑô","÷ÖÝ","ƽ¶¥É½","½­ÃÅ","¸ÓÖÝ","Ì©°²","ÏåÑô","ÄÏÑô","¹ðÁÖ","ÁijÇ","ÁÙ·Ú","µÂÖÝ","ÉÌÇð","ºâÑô","ËÞǨ","°üÍ·","μÄÏ","ÏÌÑô","Ðí²ý","±õÖÝ","ÐĮ̈","Äϳä","°²Ñô","ÐÂÏç","Ôæ×¯","³üÖÝ","ÁúÑÒ","±¦¼¦","ÐÅÑô","Ô˳Ç","ºâË®","¶«Óª","½úÖÐ","Т¸Ð","³¤ÖÎ","¸·Ñô","ÉØ¹Ø","ºÊÔó","µÂÑô","ÄþµÂ","¾£ÖÝ","ÀÖɽ","פÂíµê","ÄÏÆ½","ººÖÐ","ÓÜÁÖ","ÒË´º","°°É½","¿ª·â","ÖÜ¿Ú","Ë绯","ÏæÌ¶","»Æ¸Ô","³ÐµÂ","½¹×÷","ĵµ¤½­","ÆÎÌï","ÉÏÈÄ","üɽ","ÆëÆë¹þ¶û","³»ÖÝ","Óª¿Ú","°ö²º","´óÇì","³à·å","ºÓÔ´","ÂÀÁº","ËìÄþ","äðºÓ","ÈýÃ÷","ïÃû","å§Ñô","Çú¾¸","ƽÁ¹","¶õ¶û¶à˹","Ê®Ñß","¼ª°²","»ÆÊ¯","¼Ñľ˹","ÈýÃÅÏ¿","½ÒÑô","ÐÃÖÝ","ͨÁÉ","ãòÖÝ","Áù°²","³±ÖÝ","áéÖÝ","»´ÄÏ","¾£ÃÅ","ÓÀÖÝ","ºôÂ×±´¶û","ºù«µº","µ¤¶«","ÑÓ±ß","Ðû³Ç","Ò˱ö","ÇìÑô","½ú³Ç","Ñô½­","ËÞÖÝ","ÑÓ°²","ÏÌÄþ","»³»¯","»ÆÉ½","ÖÛɽ","ÉÇβ","Å̽õ","³¯Ñô","ÀöË®","¸§ÖÝ","ÌúÁë","ºìºÓ","¤ÄÏ","¹ã°²","¸§Ë³","×Ô¹±","´ïÖÝ","ÓñÁÖ","ÓñϪ","°×Òø","ÙñÖÝ","ÉÛÑô","¹ãÔª","²ý¼ª","ÒæÑô","ÕÅÒ´","¦µ×","ÑôȪ","ÌìË®","ËæÖÝ","Ë·ÖÝ","´óÀí","ͨ»¯","³þÐÛ","°ÍÑåÄ×¶û","ÎÚÀ¼²ì²¼","ÐÂÓà","¾°µÂÕò","ËÉÔ­","ÁÉÑô","¸·ÐÂ","¾ÆÈª","Ç­ÄÏ","»´±±","ËÄÆ½","ÎàÖÝ","Ç­¶«ÄÏ","°ÍÖÐ","º×±Ú","¼¦Î÷","ÄÚ½­","ÇÕÖÝ","³ØÖÝ","ÏÉÌÒ","¶÷Ê©","±¾Ïª","×ÊÑô","ÁùÅÌË®","°ÙÉ«","°×³Ç","ÁÙÏÄ","ÒÁÀç","¹ó¸Û","°×ɽ","˫Ѽɽ","ÎâÖÒ","¶õÖÝ","Àö½­","°²¿µ","±Ï½Ú","ÔÆ¸¡","°ÍÒô¹ùÀã","°²Ë³","°¢¿ËËÕ","ºÚºÓ","Í­Áê","ÑŰ²","ʯ×ìɽ","ƼÏç","º×¸Ú","ÎýÁÖ¹ùÀÕÃË","Í­ÈÊ","·À³Ç¸Û","¿¦Ê²","Ð˰²ÃË","ºÓ³Ø","Í­´¨","ÆÕ¶ý","±£É½","Á¹É½ÖÝ","ӥ̶","¶¨Î÷","ÒÁ´º","ÖÐÎÀ","Ç­Î÷ÄÏ","ÉÌÂå","ÎÚº£","ÎäÍþ","º£Î÷ÖÝ","À´±ö","ÕÑͨ","ÏæÎ÷","¹þÃÜ","Õżҽç","Î÷Ë«°æÄÉÖÝ","Æß̨ºÓ","¹ÌÔ­","ÅÊÖ¦»¨","µÂºêÖÝ","³ç×ó","ÁÉÔ´","¼ÎÓø¹Ø","¿ËÀ­ÂêÒÀ","ÁÙ²×","DZ½­","ºØÖÝ","ÌìÃÅ","º£¶«","Ëþ³Ç","ÙÙÖÝ","½ð²ý","²©¶ûËþÀ­","ͳ·¬","´óÐ˰²Áë","°¢ÀÕÌ©","ºÍÌï","µÏÇìÖÝ","ÈÕ¿¦Ôò","ÁÖÖ¥","°¢°ÓÖÝ","°¢À­ÉÆÃË","°¢Àï","²ý¶¼","¸ÊÄÏ","¸Ê×ÎÖÝ","¹ûÂåÖÝ","º£±±ÖÝ","º£ÄÏÖÝ","»ÆÄÏÖÝ","¿Ë×ÎÀÕËÕ","ÄÇÇú","Å­½­ÖÝ","Èýɳ","ɽÄÏ","ÉñÅ©¼Ü","ÎÄɽÖÝ","ÓñÊ÷ÖÝ"] + public static def citys_all = [ "ÉϺ£","³¤´º","ÑïÖÝ","±±¾©","ÖØÇì","Ìì½ò","³É¶¼","ÉòÑô","´óÁ¬","°²Çì","Î÷Äþ","ÈÕÕÕ","Î人","Î÷°²","ËÕÖÝ","¹ãÖÝ","ÄϾ©","Çൺ","¼ÃÄÏ","Äþ²¨","³¤É³","ʯ¼Òׯ","·ðɽ","¼ÎÐË","Òø´¨","ÄÏͨ","ÀÈ·»","ÖÐɽ","Ö£ÖÝ","¹þ¶û±õ","ÑĮ̀","ÐìÖÝ","ÉÜÐË","ÏÃÃÅ","À¥Ã÷","Äϲý","¹óÑô","Ö麣","¶«Ý¸","Õò½­","ÂåÑô","ÎÂÖÝ","À¼ÖÝ","¸£ÖÝ","»ÝÖÝ","ºôºÍºÆÌØ","³£ÖÝ","ÌÆÉ½","»´°²","ºþÖÝ","¾Å½­","̫ԭ","±±º£","ÖêÖÞ","×Ͳ©","ÑγÇ","ÎÚ³ľÆë","º£¿Ú","ÄÏÄþ","³£µÂ","̨ÖÝ","½ð»ª","¼ªÁÖ","ÔÀÑô","ÁøÖÝ","ÇåÔ¶","Õ¿½­","ÕżҿÚ","±£¶¨","ÉîÛÚ","º¼ÖÝ","ºÏ·Ê","ÎÞÎý","ȪÖÝ","×ñÒå","ÕØÇì","ÕÄÖÝ","Ò˲ý","Îߺþ","Ì©ÖÝ","ÉÇÍ·","ÈýÑÇ","ÇØ»Êµº","Âí°°É½","Á¬ÔƸÛ","²×ÖÝ","À­Èø","½õÖÝ","´óͬ","ÎIJý","±£Í¤","ÁêË®","Ϋ·»","ÁÙÒÊ","ºªµ¦","¼ÃÄþ","Íþº£","ÃàÑô","÷ÖÝ","ƽ¶¥É½","½­ÃÅ","¸ÓÖÝ","Ì©°²","ÏåÑô","ÄÏÑô","¹ðÁÖ","ÁijÇ","ÁÙ·Ú","µÂÖÝ","ÉÌÇð","ºâÑô","ËÞǨ","°üÍ·","μÄÏ","ÏÌÑô","Ðí²ý","±õÖÝ","ÐĮ̈","Äϳä","°²Ñô","ÐÂÏç","Ôæ×¯","³üÖÝ","ÁúÑÒ","±¦¼¦","ÐÅÑô","Ô˳Ç","ºâË®","¶«Óª","½úÖÐ","Т¸Ð","³¤ÖÎ","¸·Ñô","ÉØ¹Ø","ºÊÔó","µÂÑô","ÄþµÂ","¾£ÖÝ","ÀÖɽ","פÂíµê","ÄÏÆ½","ººÖÐ","ÓÜÁÖ","ÒË´º","°°É½","¿ª·â","ÖÜ¿Ú","Ë绯","ÏæÌ¶","»Æ¸Ô","³ÐµÂ","½¹×÷","ĵµ¤½­","ÆÎÌï","ÉÏÈÄ","üɽ","ÆëÆë¹þ¶û","³»ÖÝ","Óª¿Ú","°ö²º","´óÇì","³à·å","ºÓÔ´","ÂÀÁº","ËìÄþ","äðºÓ","ÈýÃ÷","ïÃû","å§Ñô","Çú¾¸","ƽÁ¹","¶õ¶û¶à˹","Ê®Ñß","¼ª°²","»ÆÊ¯","¼Ñľ˹","ÈýÃÅÏ¿","½ÒÑô","ÐÃÖÝ","ͨÁÉ","ãòÖÝ","Áù°²","³±ÖÝ","áéÖÝ","»´ÄÏ","¾£ÃÅ","ÓÀÖÝ","ºôÂ×±´¶û","ºù«µº","µ¤¶«","ÑÓ±ß","Ðû³Ç","Ò˱ö","ÇìÑô","½ú³Ç","Ñô½­","ËÞÖÝ","ÑÓ°²","ÏÌÄþ","»³»¯","»ÆÉ½","ÖÛɽ","ÉÇβ","Å̽õ","³¯Ñô","ÀöË®","¸§ÖÝ","ÌúÁë","ºìºÓ","¤ÄÏ","¹ã°²","¸§Ë³","×Ô¹±","´ïÖÝ","ÓñÁÖ","ÓñϪ","°×Òø","ÙñÖÝ","ÉÛÑô","¹ãÔª","²ý¼ª","ÒæÑô","ÕÅÒ´","¦µ×","ÑôȪ","ÌìË®","ËæÖÝ","Ë·ÖÝ","´óÀí","ͨ»¯","³þÐÛ","°ÍÑåÄ×¶û","ÎÚÀ¼²ì²¼","ÐÂÓà","¾°µÂÕò","ËÉÔ­","ÁÉÑô","¸·ÐÂ","¾ÆÈª","Ç­ÄÏ","»´±±","ËÄÆ½","ÎàÖÝ","Ç­¶«ÄÏ","°ÍÖÐ","º×±Ú","¼¦Î÷","ÄÚ½­","ÇÕÖÝ","³ØÖÝ","ÏÉÌÒ","¶÷Ê©","±¾Ïª","×ÊÑô","ÁùÅÌË®","°ÙÉ«","°×³Ç","ÁÙÏÄ","ÒÁÀç","¹ó¸Û","°×ɽ","˫Ѽɽ","ÎâÖÒ","¶õÖÝ","Àö½­","°²¿µ","±Ï½Ú","ÔÆ¸¡","°ÍÒô¹ùÀã","°²Ë³","°¢¿ËËÕ","ºÚºÓ","Í­Áê","ÑŰ²","ʯ×ìɽ","ƼÏç","º×¸Ú","ÎýÁÖ¹ùÀÕÃË","Í­ÈÊ","·À³Ç¸Û","¿¦Ê²","Ð˰²ÃË","ºÓ³Ø","Í­´¨","ÆÕ¶ý","±£É½","Á¹É½ÖÝ","ӥ̶","¶¨Î÷","ÒÁ´º","ÖÐÎÀ","Ç­Î÷ÄÏ","ÉÌÂå","ÎÚº£","ÎäÍþ","º£Î÷ÖÝ","À´±ö","ÕÑͨ","ÏæÎ÷","¹þÃÜ","Õżҽç","Î÷Ë«°æÄÉÖÝ","Æß̨ºÓ","¹ÌÔ­","ÅÊÖ¦»¨","µÂºêÖÝ","³ç×ó","ÁÉÔ´","¼ÎÓø¹Ø","¿ËÀ­ÂêÒÀ","ÁÙ²×","DZ½­","ºØÖÝ","ÌìÃÅ","º£¶«","Ëþ³Ç","ÙÙÖÝ","½ð²ý","²©¶ûËþÀ­","ͳ·¬","´óÐ˰²Áë","°¢ÀÕÌ©","ºÍÌï","µÏÇìÖÝ","ÈÕ¿¦Ôò","ÁÖÖ¥"] public static def citys_92 = [ "ÉϺ£","ÑïÖÝ","±±¾©","ÖØÇì","Ìì½ò","³É¶¼","ÉòÑô","´óÁ¬","°²Çì","Î÷Äþ","ÈÕÕÕ","³¤´º","Î人","Î÷°²","ËÕÖÝ","¹ãÖÝ","ÄϾ©","Çൺ","¼ÃÄÏ","Äþ²¨","³¤É³","ʯ¼Òׯ","·ðɽ","¼ÎÐË","Òø´¨","ÄÏͨ","ÀÈ·»","ÖÐɽ","Ö£ÖÝ","¹þ¶û±õ","ÑĮ̀","ÐìÖÝ","ÉÜÐË","ÏÃÃÅ","À¥Ã÷","Äϲý","¹óÑô","Ö麣","¶«Ý¸","Õò½­","ÂåÑô","ÎÂÖÝ","À¼ÖÝ","¸£ÖÝ","»ÝÖÝ","ºôºÍºÆÌØ","³£ÖÝ","ÌÆÉ½","»´°²","ºþÖÝ","¾Å½­","̫ԭ","±±º£","ÖêÖÞ","×Ͳ©","ÑγÇ","ÎÚ³ľÆë","º£¿Ú","ÄÏÄþ","³£µÂ","̨ÖÝ","½ð»ª","¼ªÁÖ","ÔÀÑô","ÁøÖÝ","ÇåÔ¶","Õ¿½­","ÕżҿÚ","±£¶¨","ÉîÛÚ","º¼ÖÝ","ºÏ·Ê","ÎÞÎý","ȪÖÝ","×ñÒå","ÕØÇì","ÕÄÖÝ","Ò˲ý","Îߺþ","Ì©ÖÝ","ÉÇÍ·","ÈýÑÇ","ÇØ»Êµº","Âí°°É½","Á¬ÔƸÛ","²×ÖÝ","À­Èø","½õÖÝ","´óͬ","ÎIJý","±£Í¤","ÁêË®"] public static def citys_252 = ["Ϋ·»","ÁÙÒÊ","ºªµ¦","¼ÃÄþ","Íþº£","ÃàÑô","÷ÖÝ","ƽ¶¥É½","½­ÃÅ","¸ÓÖÝ","Ì©°²","ÏåÑô","ÄÏÑô","¹ðÁÖ","ÁijÇ","ÁÙ·Ú","µÂÖÝ","ÉÌÇð","ºâÑô","ËÞǨ","°üÍ·","μÄÏ","ÏÌÑô","Ðí²ý","±õÖÝ","ÐĮ̈","Äϳä","°²Ñô","ÐÂÏç","Ôæ×¯","³üÖÝ","ÁúÑÒ","±¦¼¦","ÐÅÑô","Ô˳Ç","ºâË®","¶«Óª","½úÖÐ","Т¸Ð","³¤ÖÎ","¸·Ñô","ÉØ¹Ø","ºÊÔó","µÂÑô","ÄþµÂ","¾£ÖÝ","ÀÖɽ","פÂíµê","ÄÏÆ½","ººÖÐ","ÓÜÁÖ","ÒË´º","°°É½","¿ª·â","ÖÜ¿Ú","Ë绯","ÏæÌ¶","»Æ¸Ô","³ÐµÂ","½¹×÷","ĵµ¤½­","ÆÎÌï","ÉÏÈÄ","üɽ","ÆëÆë¹þ¶û","³»ÖÝ","Óª¿Ú","°ö²º","´óÇì","³à·å","ºÓÔ´","ÂÀÁº","ËìÄþ","äðºÓ","ÈýÃ÷","ïÃû","å§Ñô","Çú¾¸","ƽÁ¹","¶õ¶û¶à˹","Ê®Ñß","¼ª°²","»ÆÊ¯","¼Ñľ˹","ÈýÃÅÏ¿","½ÒÑô","ÐÃÖÝ","ͨÁÉ","ãòÖÝ","Áù°²","³±ÖÝ","áéÖÝ","»´ÄÏ","¾£ÃÅ","ÓÀÖÝ","ºôÂ×±´¶û","ºù«µº","µ¤¶«","ÑÓ±ß","Ðû³Ç","Ò˱ö","ÇìÑô","½ú³Ç","Ñô½­","ËÞÖÝ","ÑÓ°²","ÏÌÄþ","»³»¯","»ÆÉ½","ÖÛɽ","ÉÇβ","Å̽õ","³¯Ñô","ÀöË®","¸§ÖÝ","ÌúÁë","ºìºÓ","¤ÄÏ","¹ã°²","¸§Ë³","×Ô¹±","´ïÖÝ","ÓñÁÖ","ÓñϪ","°×Òø","ÙñÖÝ","ÉÛÑô","¹ãÔª","²ý¼ª","ÒæÑô","ÕÅÒ´","¦µ×","ÑôȪ","ÌìË®","ËæÖÝ","Ë·ÖÝ","´óÀí","ͨ»¯","³þÐÛ","°ÍÑåÄ×¶û","ÎÚÀ¼²ì²¼","ÐÂÓà","¾°µÂÕò","ËÉÔ­","ÁÉÑô","¸·ÐÂ","¾ÆÈª","Ç­ÄÏ","»´±±","ËÄÆ½","ÎàÖÝ","Ç­¶«ÄÏ","°ÍÖÐ","º×±Ú","¼¦Î÷","ÄÚ½­","ÇÕÖÝ","³ØÖÝ","ÏÉÌÒ","¶÷Ê©","±¾Ïª","×ÊÑô","ÁùÅÌË®","°ÙÉ«","°×³Ç","ÁÙÏÄ","ÒÁÀç","¹ó¸Û","°×ɽ","˫Ѽɽ","ÎâÖÒ","¶õÖÝ","Àö½­","°²¿µ","±Ï½Ú","ÔÆ¸¡","°ÍÒô¹ùÀã","°²Ë³","°¢¿ËËÕ","ºÚºÓ","Í­Áê","ÑŰ²","ʯ×ìɽ","ƼÏç","º×¸Ú","ÎýÁÖ¹ùÀÕÃË","Í­ÈÊ","·À³Ç¸Û","¿¦Ê²","Ð˰²ÃË","ºÓ³Ø","Í­´¨","ÆÕ¶ý","±£É½","Á¹É½ÖÝ","ӥ̶","¶¨Î÷","ÒÁ´º","ÖÐÎÀ","Ç­Î÷ÄÏ","ÉÌÂå","ÎÚº£","ÎäÍþ","º£Î÷ÖÝ","À´±ö","ÕÑͨ","ÏæÎ÷","¹þÃÜ","Õżҽç","Î÷Ë«°æÄÉÖÝ","Æß̨ºÓ","¹ÌÔ­","ÅÊÖ¦»¨","µÂºêÖÝ","³ç×ó","ÁÉÔ´","¼ÎÓø¹Ø","¿ËÀ­ÂêÒÀ","ÁÙ²×","DZ½­","ºØÖÝ","ÌìÃÅ","º£¶«","Ëþ³Ç","ÙÙÖÝ","½ð²ý","²©¶ûËþÀ­","ͳ·¬","´óÐ˰²Áë","°¢ÀÕÌ©","ºÍÌï","µÏÇìÖÝ","ÈÕ¿¦Ôò","ÁÖÖ¥","°¢°ÓÖÝ","°¢À­ÉÆÃË","°¢Àï","²ý¶¼","¸ÊÄÏ","¸Ê×ÎÖÝ","¹ûÂåÖÝ","º£±±ÖÝ","º£ÄÏÖÝ","»ÆÄÏÖÝ","¿Ë×ÎÀÕËÕ","ÄÇÇú","Å­½­ÖÝ","Èýɳ","ɽÄÏ","ÉñÅ©¼Ü","ÎÄɽÖÝ","ÓñÊ÷ÖÝ"] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy index d43bfa2cb..efcbda2f1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy @@ -38,7 +38,7 @@ class ExportDistrictRongHe extends BaseBatch{ def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd") def new_dists = []//["350604","350605","410307","410308","450181"] //°´Ììµ¼£¨Ã¿ÌìÒ»¸öÎļþ¼Ð£© - exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists) +// exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists) if(curDate.toString().endsWith("10")){ @@ -77,7 +77,7 @@ class ExportDistrictRongHe extends BaseBatch{ } //2ÔÂ8ºÅ ÁÙʱµ¼³öËÕÖÝÒ»ÄêµÄÊý¾Ý - exportBigdataForsale("2022-01-01", new_dists) //ÏÖÔÚ°´updatetimeµ¼³ö + exportBigdataForsale("2022-07-01", new_dists) //ÏÖÔÚ°´updatetimeµ¼³ö SSHHelper jschCommand = new SSHHelper(); System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}")); @@ -1309,8 +1309,6 @@ class ExportDistrictRongHe extends BaseBatch{ } def exportBigdataForsale(publishDate, new_dists){ - //["±±¾©","ÉϺ£"] - citys = ["ËÕÖÝ"] def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"); def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_FORSALE_FJW_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ @@ -1320,6 +1318,9 @@ class ExportDistrictRongHe extends BaseBatch{ def index = 0 for(def city : citys){ def cityCode = cityCodeMap.get(city) + if(city=="ËÕÖÝ"){ + continue + } def allRows = [] if(new_dists){ @@ -1359,8 +1360,8 @@ class ExportDistrictRongHe extends BaseBatch{ } } println "ÈÚºÏ BIGDATA_FORSALE_GJ Êý¾ÝÎļþÊä³öÍê³É......" -// CcbFtpUtils.ftpUpload(null, fileName, curDate) -// println "ÈÚºÏ BIGDATA_FORSALE_GJ ÉÏ´«FtpÍê³É......" + CcbFtpUtils.ftpUpload("rh",fileName, curDate) + println "ÈÚºÏ BIGDATA_FORSALE_GJ ÉÏ´«FtpÍê³É......" } def getHouseDatas(sql){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/DongGuanBank.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/DongGuanBank.groovy new file mode 100644 index 000000000..6e992809d --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/DongGuanBank.groovy @@ -0,0 +1,43 @@ +package com.fangjia.script.wenyan.temp + +import com.fangjia.core.Constants +import com.fangjia.core.util.DateUtil +import com.fangjia.core.util.Mongo3Utils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.client.MongoCollection +import net.sf.json.JSONObject +import org.bson.Document + +import static com.mongodb.client.model.Filters.* + +MongoCollection OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log") +def startDate = DateUtil.parseDate("20221101", "yyyyMMdd") +def endDate = DateUtil.parseDate("20221201", "yyyyMMdd") +Calendar calendar = Calendar.getInstance() +calendar.setTime(startDate) +def count = 0 +def hashSet = [] +def data = [["±àºÅ", "¹«Ë¾Ãû³Æ", "IP", "·ÃÎÊÁ´½Ó", "·ÃÎÊʱ¼ä", "dataStatus", "code", "msg", "²ÎÊý", "·µ»Ø²ÎÊý"]] +while (startDate.before(endDate)) { + calendar.add(Calendar.DATE, 1) + def nextDate = calendar.getTime() + count++ + OPEN_URI_LOG.find(and(eq("user_id", "1007713"), eq("request_uri", "/property/evaluate"), eq("data_status", 1), lt("c_date", nextDate), gte("c_date", startDate))).each { Document doc -> + def result = JSONObject.fromObject(doc.res_map.result) +// if (!hashSet.contains(result)) { + def item = [doc._id.toString(), "¶«Ý¸ÒøÐÐ×ÜÐÐ", doc.ip, doc.request_uri, doc.c_date.format("yyyy-MM-dd HH:mm:ss"), doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()] + data << item +// hashSet << result +// } else { +// println("ÒÑ´æÔÚ") +// } + } + if (count % 7 == 0) { + hashSet.clear() + } + + startDate = nextDate +} +ExcelUtils.write("D:\\Êý¾Ý\\¶«Ý¸\\¶«Ý¸ÒøÐÐ×ÜÐÐ.xlsx", data, "202211") + + From 5e779c54b3d9b1e4e2358328a373dd05d2930f7c Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 9 Feb 2023 22:28:49 +0800 Subject: [PATCH 66/99] up --- .../ccbevaluation/ronghe/ExportDistrictRongHe.groovy | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy index efcbda2f1..2f34573dd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy @@ -38,7 +38,7 @@ class ExportDistrictRongHe extends BaseBatch{ def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd") def new_dists = []//["350604","350605","410307","410308","450181"] //°´Ììµ¼£¨Ã¿ÌìÒ»¸öÎļþ¼Ð£© -// exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists) + exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists) if(curDate.toString().endsWith("10")){ @@ -77,7 +77,7 @@ class ExportDistrictRongHe extends BaseBatch{ } //2ÔÂ8ºÅ ÁÙʱµ¼³öËÕÖÝÒ»ÄêµÄÊý¾Ý - exportBigdataForsale("2022-07-01", new_dists) //ÏÖÔÚ°´updatetimeµ¼³ö +// exportBigdataForsale("2022-07-01", new_dists) //ÏÖÔÚ°´updatetimeµ¼³ö SSHHelper jschCommand = new SSHHelper(); System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}")); @@ -1197,7 +1197,7 @@ class ExportDistrictRongHe extends BaseBatch{ } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); - CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) + CcbFtpUtils.ftpUpload("rh",fileName, curDate) println "BIGDATA_CITYPRICE ÉÏ´«FtpÍê³É......" } @@ -1244,7 +1244,7 @@ class ExportDistrictRongHe extends BaseBatch{ } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); - CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) + CcbFtpUtils.ftpUpload("rh",fileName, curDate) println "BIGDATA_DISTPRICE ÉÏ´«FtpÍê³É......" } @@ -1291,7 +1291,7 @@ class ExportDistrictRongHe extends BaseBatch{ } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); - CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) + CcbFtpUtils.ftpUpload("rh",fileName, curDate) println "BIGDATA_COMMPRICE ÉÏ´«FtpÍê³É......" } @@ -1309,7 +1309,7 @@ class ExportDistrictRongHe extends BaseBatch{ } def exportBigdataForsale(publishDate, new_dists){ - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"); + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_FORSALE_FJW_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_FORSALE_FJW_ADD_${curDate}_0001.dat" From 54e650f6369571d4153c7a5d44343df6a4f9dc1f Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 10 Feb 2023 09:54:35 +0800 Subject: [PATCH 67/99] =?UTF-8?q?=E8=9E=8D=E5=90=88-=E5=AF=BC=E4=BC=B0?= =?UTF-8?q?=E5=80=BC=E7=9B=B8=E5=85=B3=E7=9A=84=E8=A1=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ronghe/ExportDistrictRongHe.groovy | 66 +++++++++---------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy index 2f34573dd..7c919169c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy @@ -38,51 +38,51 @@ class ExportDistrictRongHe extends BaseBatch{ def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd") def new_dists = []//["350604","350605","410307","410308","450181"] //°´Ììµ¼£¨Ã¿ÌìÒ»¸öÎļþ¼Ð£© - exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists) +// exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists) - if(curDate.toString().endsWith("10")){ - exportBigdataCommunityGJRH()// - exportBigdataBuildingFJW()// - exportBigdataCommfeeGj()// - - exportBigdataCommbuildstruGJ()// - exportBigdataCommhousetypeGJ()// - exportBigdataCommbuildyearGj()// - exportBigdataCommhouseuseGJ()// - exportBigdataPositionGJ()// - exportBigdataDistrictFJW()// + if(curDate.toString().endsWith("11")){ +// exportBigdataCommunityGJRH()// +// exportBigdataBuildingFJW()// +// exportBigdataCommfeeGj()// +// +// exportBigdataCommbuildstruGJ()// +// exportBigdataCommhousetypeGJ()// +// exportBigdataCommbuildyearGj()// +// exportBigdataCommhouseuseGJ()// +// exportBigdataPositionGJ()// +// exportBigdataDistrictFJW()// exportBigdataAliasFJW()// - exportBigdataBuildprojFjw() - exportBigdataCommlayoutpicFjw() - exportEvaluateRate() - - exportBigdataCityPrice(queryDate) - exportBigdataDistPrice(queryDate) - exportBigdataCommPrice(queryDate) - - //ÒòΪÈÚºÏÔݲ»Çп⣬ËùÒÔÐèÒª¼Ó´«Ò»ÕŵÚÈý·½¾ù¼Û±íµ½ÈÚºÏÖУ¬ ´Ó1ÔÂ10ÈÕ¿ªÊ¼´«Ê䣬³¤ÆÚ¹© - def thirdPartyMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")); - exportThirdPartyPrice2(thirdPartyMonth, new_dists) +// exportBigdataBuildprojFjw() +// exportBigdataCommlayoutpicFjw() +// exportEvaluateRate() +// +// exportBigdataCityPrice(queryDate) +// exportBigdataDistPrice(queryDate) +// exportBigdataCommPrice(queryDate) +// +// //ÒòΪÈÚºÏÔݲ»Çп⣬ËùÒÔÐèÒª¼Ó´«Ò»ÕŵÚÈý·½¾ù¼Û±íµ½ÈÚºÏÖУ¬ ´Ó1ÔÂ10ÈÕ¿ªÊ¼´«Ê䣬³¤ÆÚ¹© +// def thirdPartyMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")); +// exportThirdPartyPrice2(thirdPartyMonth, new_dists) // // -// //1ÔÂ10ÈÕ ÐÂÔö5ÕÅ±í£¬µ¼92³Ç£¬ÔöÁ¿Êý¾Ý£¬2ÔÂ10ÈÕµ¼344³Ç£¬ÆäÖÐ252³ÇËùÓÐÀúÊ·Êý¾Ý£¬92³ÇÔöÁ¿£¬3Ôºó¿ÉÄܲ»ÓÃÌṩÊý¾Ý -// exportCommunityBasePrice(lastMonth) -// exportCommunityPdescribeGj(lastMonth) -// exportGuideStatistics(lastMonth) -// exportBigdataSupplierStatistics(lastMonth) -// exportBigdataModelingBase(lastMonth) + //1ÔÂ10ÈÕ ÐÂÔö5ÕÅ±í£¬µ¼92³Ç£¬ÔöÁ¿Êý¾Ý£¬2ÔÂ10ÈÕµ¼344³Ç£¬ÆäÖÐ252³ÇËùÓÐÀúÊ·Êý¾Ý£¬92³ÇÔöÁ¿£¬3Ôºó¿ÉÄܲ»ÓÃÌṩÊý¾Ý + exportCommunityBasePrice(lastMonth) + exportCommunityPdescribeGj(lastMonth) + exportGuideStatistics(lastMonth) + exportBigdataSupplierStatistics(lastMonth) + exportBigdataModelingBase(lastMonth) } //2ÔÂ8ºÅ ÁÙʱµ¼³öËÕÖÝÒ»ÄêµÄÊý¾Ý // exportBigdataForsale("2022-07-01", new_dists) //ÏÖÔÚ°´updatetimeµ¼³ö - SSHHelper jschCommand = new SSHHelper(); - System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}")); - - exportOk(curDate) +// SSHHelper jschCommand = new SSHHelper(); +// System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}")); +// +// exportOk(curDate) } From ba9322e206ba373e023c39e54c29913a358acb81 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 10 Feb 2023 16:41:04 +0800 Subject: [PATCH 68/99] =?UTF-8?q?=E8=BF=98=E5=8E=9F=E8=9E=8D=E5=90=88?= =?UTF-8?q?=E5=AF=BC=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ronghe/ExportDistrictRongHe.groovy | 67 +++++++++---------- .../gj/paimai/BeiKeCrawlByChrome.groovy | 41 ++++++------ .../gj/paimai/LianjiaCrawlByChrome.groovy | 38 ++++++----- 3 files changed, 76 insertions(+), 70 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy index 7c919169c..8215007e5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy @@ -38,35 +38,34 @@ class ExportDistrictRongHe extends BaseBatch{ def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd") def new_dists = []//["350604","350605","410307","410308","450181"] //°´Ììµ¼£¨Ã¿ÌìÒ»¸öÎļþ¼Ð£© -// exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists) + exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists) + if(curDate.toString().endsWith("10")){ + exportBigdataCommunityGJRH()// + exportBigdataBuildingFJW()// + exportBigdataCommfeeGj()// - if(curDate.toString().endsWith("11")){ -// exportBigdataCommunityGJRH()// -// exportBigdataBuildingFJW()// -// exportBigdataCommfeeGj()// -// -// exportBigdataCommbuildstruGJ()// -// exportBigdataCommhousetypeGJ()// -// exportBigdataCommbuildyearGj()// -// exportBigdataCommhouseuseGJ()// -// exportBigdataPositionGJ()// -// exportBigdataDistrictFJW()// + exportBigdataCommbuildstruGJ()// + exportBigdataCommhousetypeGJ()// + exportBigdataCommbuildyearGj()// + exportBigdataCommhouseuseGJ()// + exportBigdataPositionGJ()// + exportBigdataDistrictFJW()// exportBigdataAliasFJW()// -// exportBigdataBuildprojFjw() -// exportBigdataCommlayoutpicFjw() -// exportEvaluateRate() -// -// exportBigdataCityPrice(queryDate) -// exportBigdataDistPrice(queryDate) -// exportBigdataCommPrice(queryDate) -// -// //ÒòΪÈÚºÏÔݲ»Çп⣬ËùÒÔÐèÒª¼Ó´«Ò»ÕŵÚÈý·½¾ù¼Û±íµ½ÈÚºÏÖУ¬ ´Ó1ÔÂ10ÈÕ¿ªÊ¼´«Ê䣬³¤ÆÚ¹© -// def thirdPartyMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")); -// exportThirdPartyPrice2(thirdPartyMonth, new_dists) -// -// + exportBigdataBuildprojFjw() + exportBigdataCommlayoutpicFjw() + exportEvaluateRate() + + exportBigdataCityPrice(queryDate) + exportBigdataDistPrice(queryDate) + exportBigdataCommPrice(queryDate) + + //ÒòΪÈÚºÏÔݲ»Çп⣬ËùÒÔÐèÒª¼Ó´«Ò»ÕŵÚÈý·½¾ù¼Û±íµ½ÈÚºÏÖУ¬ ´Ó1ÔÂ10ÈÕ¿ªÊ¼´«Ê䣬³¤ÆÚ¹© + def thirdPartyMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")); + exportThirdPartyPrice2(thirdPartyMonth, new_dists) + + //1ÔÂ10ÈÕ ÐÂÔö5ÕÅ±í£¬µ¼92³Ç£¬ÔöÁ¿Êý¾Ý£¬2ÔÂ10ÈÕµ¼344³Ç£¬ÆäÖÐ252³ÇËùÓÐÀúÊ·Êý¾Ý£¬92³ÇÔöÁ¿£¬3Ôºó¿ÉÄܲ»ÓÃÌṩÊý¾Ý exportCommunityBasePrice(lastMonth) exportCommunityPdescribeGj(lastMonth) @@ -74,15 +73,15 @@ class ExportDistrictRongHe extends BaseBatch{ exportBigdataSupplierStatistics(lastMonth) exportBigdataModelingBase(lastMonth) + //2ÔÂ8ºÅ ÁÙʱµ¼³öËÕÖÝÒ»ÄêµÄÊý¾Ý +// exportBigdataForsale("2022-07-01", new_dists) //ÏÖÔÚ°´updatetimeµ¼³ö + } - //2ÔÂ8ºÅ ÁÙʱµ¼³öËÕÖÝÒ»ÄêµÄÊý¾Ý -// exportBigdataForsale("2022-07-01", new_dists) //ÏÖÔÚ°´updatetimeµ¼³ö + SSHHelper jschCommand = new SSHHelper(); + System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}")); -// SSHHelper jschCommand = new SSHHelper(); -// System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}")); -// -// exportOk(curDate) + exportOk(curDate) } @@ -1164,7 +1163,7 @@ class ExportDistrictRongHe extends BaseBatch{ for(def city : citys) { def cityCode = cityCodeMap.get(city) // def sql = "select CITYID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_CITYPRICE where CITYID='${cityCode}' and YMID=${queryDate}";// - def sql = "select CITYID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_CITYPRICE where CITYID='${cityCode}' and YMID=${queryDate}";// + def sql = "select CITYID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_CITYPRICE_FJW where CITYID='${cityCode}' and YMID=${queryDate}";// sqlInstance.eachRow(sql,{r-> def data = [] @@ -1211,7 +1210,7 @@ class ExportDistrictRongHe extends BaseBatch{ def allRows = [] for(def city : citys) { def cityCode = cityCodeMap.get(city) - def sql = "select CITYID,DISTID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_DISTPRICE where CITYID='${cityCode}' and YMID=${queryDate}";// and YMID=${queryDate} + def sql = "select CITYID,DISTID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_DISTPRICE_FJW where CITYID='${cityCode}' and YMID=${queryDate}";// and YMID=${queryDate} sqlInstance.eachRow(sql,{r-> def data = [] data << r.CITYID @@ -1258,7 +1257,7 @@ class ExportDistrictRongHe extends BaseBatch{ def allRows = [] for(def city : citys) { def cityCode = cityCodeMap.get(city) - def sql = "select CITYID, YMID,COMMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_COMMPRICE where CITYID='${cityCode}' and YMID=${queryDate} ";// + def sql = "select CITYID, YMID,COMMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_COMMPRICE_FJW where CITYID='${cityCode}' and YMID=${queryDate} ";// sqlInstance.eachRow(sql,{r-> def data = [] data << r.CITYID diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy index 24f03c7cc..f641c214c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy @@ -251,26 +251,29 @@ public class BeiKeCrawlByChrome { detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd") } detail.source = "±´¿ÇÕÒ·¿" - def detailUrl = e.select(".title a").attr("href") - detail.url = detailUrl - def dbo = findTrade(detail) - if (!dbo) { - def houseInfo = e.select(".houseInfo").text() - def houseInfos = houseInfo.split("\\|") - detail.direction = houseInfos[0] - detail.fitment = houseInfos[1] - def floorStr = e.select(".positionInfo").text() - if (floorStr && floorStr.contains("¹²") && floorStr.contains("²ã)")) { - floorStr = floorStr.substring(floorStr.indexOf("¹²") + 1, floorStr.indexOf("²ã)")) - if (floorStr && floorStr.isNumber()) { - detail.height = floorStr.toInteger() - } - } - detail.create_date = new Date() - detail.m_date = new Date() - println detail - DEAL_PRICE_COLL.save(detail) + if(detail.total_price){ + def detailUrl = e.select(".title a").attr("href") + detail.url = detailUrl + def dbo = findTrade(detail) + if (!dbo) { + def houseInfo = e.select(".houseInfo").text() + def houseInfos = houseInfo.split("\\|") + detail.direction = houseInfos[0] + detail.fitment = houseInfos[1] + def floorStr = e.select(".positionInfo").text() + if (floorStr && floorStr.contains("¹²") && floorStr.contains("²ã)")) { + floorStr = floorStr.substring(floorStr.indexOf("¹²") + 1, floorStr.indexOf("²ã)")) + if (floorStr && floorStr.isNumber()) { + detail.height = floorStr.toInteger() + } + } + + detail.create_date = new Date() + detail.m_date = new Date() + println detail + DEAL_PRICE_COLL.save(detail) + } } } }catch(Exception ex){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy index 0ecdddeb1..e82b8bc45 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -328,25 +328,29 @@ public class LianjiaCrawlByChrome { } detail.source = "Á´¼ÒÔÚÏß" detail.url = e.select(".title a").attr("href") - def dbo = findTrade(detail) - if (!dbo) { - def houseInfo = e.select(".houseInfo").text() - def houseInfos = houseInfo.split("\\|") - detail.direction = houseInfos[0] - detail.fitment = houseInfos[1] - def floorStr = e.select(".positionInfo").text() - if (floorStr && floorStr.contains("¹²") && floorStr.contains("²ã)")) { - floorStr = floorStr.substring(floorStr.indexOf("¹²") + 1, floorStr.indexOf("²ã)")) - if (floorStr && floorStr.isNumber()) { - detail.height = floorStr.toInteger() - } - } - detail.create_date = new Date() - detail.m_date = new Date() - println detail - DEAL_PRICE_COLL.save(detail) + if(detail.total_price){ + def dbo = findTrade(detail) + if (!dbo) { + def houseInfo = e.select(".houseInfo").text() + def houseInfos = houseInfo.split("\\|") + detail.direction = houseInfos[0] + detail.fitment = houseInfos[1] + def floorStr = e.select(".positionInfo").text() + if (floorStr && floorStr.contains("¹²") && floorStr.contains("²ã)")) { + floorStr = floorStr.substring(floorStr.indexOf("¹²") + 1, floorStr.indexOf("²ã)")) + if (floorStr && floorStr.isNumber()) { + detail.height = floorStr.toInteger() + } + } + + detail.create_date = new Date() + detail.m_date = new Date() + println detail + DEAL_PRICE_COLL.save(detail) + } } + } } }catch(Exception ex){ From 0d9396305d0de89a6f4cd0c16bbf3e148f46fc73 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 17 Feb 2023 09:30:58 +0800 Subject: [PATCH 69/99] update --- .../BigdataCommunityBatch.groovy | 62 ++++++++++++++++++- 1 file changed, 59 insertions(+), 3 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy index 27ed0ad49..efdf794af 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -12,6 +12,7 @@ import com.fangjia.script.gj.district.FangjiaClientUtils import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.OpenApiClient +import groovy.sql.Sql import java.util.concurrent.CopyOnWriteArrayList import java.util.concurrent.CountDownLatch @@ -30,6 +31,9 @@ class BigdataCommunityBatch extends BaseBatch{ public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10) + def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/postgres?stringtype=unspecified", + "postgres", "postgres", "org.postgresql.Driver") + static main(args) { runBatch(new BigdataCommunityBatch(), args); } @@ -37,7 +41,7 @@ class BigdataCommunityBatch extends BaseBatch{ @Override public void process(Object args) { def citys = CcbCommonUtils.getRunningCitys(args) - +// citys = ["ÎIJý"] def rows = [] def headers = []//["³ÇÊÐID","ÐÐÕþÇøID","Ð¡ÇøID","ÐÐÕþÇøÃû³Æ","Ð¡ÇøÃû³Æ","ÏêϸµØÖ·","γ¶È","¾­¶È","Õ¼µØ×ÜÃæ»ý","½¨Öþ×ÜÃæ»ý","ÂÌ»¯ÂÊ","ÈÝ»ýÂÊ","³µÎ»ÊýÁ¿","½¨ÖþÄê´ú","·¿ÎÝÓÃ;", @@ -71,7 +75,9 @@ class BigdataCommunityBatch extends BaseBatch{ allDatas << headers allDatas.addAll(rows) FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); - println "txt Éú³É" + println "txt Éú³É->µ¼Oracle" + + saveToPgSql(rows) // ExcelUtils.write("d:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\excel\\bigdata_community.xlsx", allDatas, "Êý¾Ý") } @@ -169,7 +175,7 @@ class BigdataCommunityBatch extends BaseBatch{ ret << CcbCodeUtils.parseBuildYear(dist.finish_building_date) ret << CcbCodeUtils.parseHouseUse(dist.estate_type2) ret << dist.household_count - ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd") + ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd").toInteger() ret << dist.status ret << DateUtils.formatDateByPattern(dist.m_date, "yyyy-MM-dd HH:mm:ss") ret << "pa101" @@ -222,4 +228,54 @@ class BigdataCommunityBatch extends BaseBatch{ } return ret; } + + def saveToPgSql(rows){ + for(def r : rows) { + for (def i = 0; i < r.size(); i++) { + if("" == r[i]){ + r[i] = null + } + } + r << null + r << '2023-02-28' + r << null + r << '2999-12-31' + r << null + r << null + r << null + } + + println "start: "+new Date() + def sql = "insert into warehouse.t0831_bigdata_community_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" +// def sql = "insert into base.t0831_bigdata_community_gj_h values " + def index = 0 + sqlInstance.withBatch(20000, sql){ ps-> + for(def r : rows) { + ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20], + r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], null, '2023-02-28', null, '2999-12-31', null, null, null) + ps.toString() + } + } + for(def r : rows){ + index ++ + if(index == 1){ + sql = sql + "("+r[0].toString()+","+r[1].toString()+","+r[2].toString()+")" + }else{ + sql = sql + ",("+")" + } + } +// sqlInstance.withBatch(50000, sql){ ps-> +// def index = 0 +// for(def r : rows){ +// index++ +// for(def i =0 ;i Date: Fri, 17 Feb 2023 10:21:49 +0800 Subject: [PATCH 70/99] update --- .../fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy index efdf794af..1580dda9b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -31,7 +31,7 @@ class BigdataCommunityBatch extends BaseBatch{ public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10) - def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/postgres?stringtype=unspecified", + def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified", "postgres", "postgres", "org.postgresql.Driver") static main(args) { @@ -246,7 +246,7 @@ class BigdataCommunityBatch extends BaseBatch{ } println "start: "+new Date() - def sql = "insert into warehouse.t0831_bigdata_community_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" + def sql = "insert into base.t0831_bigdata_community_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" // def sql = "insert into base.t0831_bigdata_community_gj_h values " def index = 0 sqlInstance.withBatch(20000, sql){ ps-> From aa5ada6ba684802cf804f20ed76ae7e9c07ac711 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 17 Feb 2023 13:09:27 +0800 Subject: [PATCH 71/99] update --- .../groovy/com/fangjia/script/fuzhou/HouseExportBatch.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/HouseExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/HouseExportBatch.groovy index 32c4d1f34..1dd114d8a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/HouseExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/HouseExportBatch.groovy @@ -420,8 +420,8 @@ class HouseExportBatch extends BaseBatch{ row << seawater.name row << seawater.url -// row << seawater.p_date - row << DateUtils.getRandomDate("2022-06-01", "2022-06-30") + row << seawater.p_date +// row << DateUtils.getRandomDate("2022-06-01", "2022-06-30") rets << row } } From b592fa85273914a71d2e8e92c3710854176e9461 Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Fri, 17 Feb 2023 13:13:07 +0800 Subject: [PATCH 72/99] =?UTF-8?q?=E5=8C=B9=E9=85=8D=E6=8E=A5=E5=8F=A3?= =?UTF-8?q?=E6=96=B0=E5=A2=9E=E8=87=AA=E7=84=B6=E6=9D=91=E5=8C=B9=E9=85=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../open/controllers/AddressController.java | 106 +++++++++++++++++- 1 file changed, 104 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java index 2ae672241..0167259de 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java @@ -24,6 +24,7 @@ import com.fangjia.open.model.*; import net.sf.json.JSONArray; import net.sf.json.JSONObject; +import org.apache.commons.codec.binary.Base64; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; import org.apache.lucene.index.Term; @@ -85,7 +86,6 @@ import com.fangjia.service.mongo.fangjia.ServerAreaService; import com.fangjia.service.open.WaitCrawlKeywordsService; import com.fangjia.service.sql.FjtEvaluationService; import com.fangjia.service.web.AddressService; -import org.sothis.web.mvc.annotation.Param; @Authority(role = Role.USER) public class AddressController { @@ -116,6 +116,8 @@ public class AddressController { private static final String GRADE_CONSUME = "Ïû·ÑÆÀ·Ö"; public final static OpenConfig CONFIG = OpenConfig.getConfig(); + private static final String ES_USER = "developer"; + private static final String ES_PASSWORD = "5e8edd851d2fdfbd7415232c67367cc3"; /** * @deprecated see parse2Action @@ -522,7 +524,53 @@ public class AddressController { MatchDistrictResult matchResult = addressMatchProxyService.checkMatchDistrictCorrect(param); jsonResult.put("matchResult", matchResult); } else if (jsonResult != null) { - jsonResult.put("matchResult", new MatchDistrictResult()); + // ÅжÏÊÇ·ñΪ×ÔÈ»´å + boolean matchedVillage = false; + if (jsonResult.get("searchAddress") != null) { + JSONObject searchAddress = (JSONObject) jsonResult.get("searchAddress"); + String city = searchAddress.getString("city"); + String region = searchAddress.getString("district"); + String name = searchAddress.getString("keywords"); + String township = searchAddress.getString("township"); + if (StringUtils.isNotBlank(city) && StringUtils.isNotBlank(region) && StringUtils.isNotBlank(name)) { + List villages = findVillage(city, region, name); + if (CollectionUtils.isNotEmpty(villages)) { + if (villages.size() == 1 || StringUtils.isBlank(township)) { + JSONObject village = villages.get(0).getJSONObject("_source"); + jsonResult.put("id", villages.get(0).getString("_id")); + jsonResult.put("province", village.getString("province")); + jsonResult.put("city", village.getString("city")); + jsonResult.put("district", village.getString("region")); + jsonResult.put("township", village.getString("township")); + jsonResult.put("name", village.getString("name")); + jsonResult.put("lng", village.getJSONObject("location").getDouble("lon")); + jsonResult.put("lat", village.getJSONObject("location").getDouble("lat")); + jsonResult.put("propertyType", "×ÔÈ»´å"); + matchedVillage = true; + } else { + for (JSONObject source : villages) { + JSONObject village = source.getJSONObject("_source"); + if (village.getString("township").contains(township) || township.contains(village.getString("township"))) { + jsonResult.put("id", source.getString("_id")); + jsonResult.put("province", village.getString("province")); + jsonResult.put("city", village.getString("city")); + jsonResult.put("district", village.getString("region")); + jsonResult.put("township", village.getString("township")); + jsonResult.put("name", village.getString("name")); + jsonResult.put("lng", village.getJSONObject("location").getDouble("lon")); + jsonResult.put("lat", village.getJSONObject("location").getDouble("lat")); + jsonResult.put("propertyType", "×ÔÈ»´å"); + matchedVillage = true; + break; + } + } + } + } + } + } + if (matchedVillage == false) { + jsonResult.put("matchResult", new MatchDistrictResult()); + } } resultModel.setResult(json.get("result")); mav.setModel(resultModel); @@ -918,6 +966,60 @@ public class AddressController { return ret; } + private JSONObject addTerm(String name, String value){ + JSONObject term = new JSONObject(); + JSONObject match = new JSONObject(); + match.put(name, value); + term.put("term", match); + return term; + } + + /*** + * »ñÈ¡×ÔÈ»´å + * @param city + * @param region + * @param name + * @return + */ + private List findVillage(String city, String region, String name) { + JSONObject reqBody = new JSONObject(); + reqBody.put("from", 0); + reqBody.put("size", 5); + JSONObject query = new JSONObject(); + JSONObject bool = new JSONObject(); + JSONArray must = new JSONArray(); + must.add(addTerm("city", city)); + must.add(addTerm("region", region)); + must.add(addTerm("name", name)); + bool.put("must", must); + query.put("bool", bool); + reqBody.put("query", query); + Base64 token = new Base64(); + String authenticationEncoding = token.encodeAsString((""+ES_USER+":"+ES_PASSWORD+"").getBytes()); + HttpRequest request = new HttpRequest(); + Map headers = new HashMap(); + headers.put("Content-Type", "application/json"); + headers.put("Authorization", "Basic "+authenticationEncoding+""); + try { + String response = request.postWithBody("http://192.168.0.45:9200/village/_search", reqBody, headers, "UTF-8"); + JSONObject json = JSONObject.fromObject(response); + if(!json.containsKey("hits") || !json.getJSONObject("hits").containsKey("hits")){ + return Collections.emptyList(); + } + JSONArray hits = json.getJSONObject("hits").getJSONArray("hits"); + List results = new ArrayList(); + for(int i=0 ;i < hits.size(); i++){ + JSONObject jsonObject = hits.getJSONObject(i); + results.add(jsonObject); + } + return results; + + } catch (HttpRequestException e) { + e.printStackTrace(); + } + return Collections.emptyList(); + } + /** * ¼ÓÔØ²âÊÔÊý¾Ý * From abe48b48f3618faac8f9607c05ec49239927c500 Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Fri, 17 Feb 2023 13:24:33 +0800 Subject: [PATCH 73/99] =?UTF-8?q?=E5=8C=B9=E9=85=8D=E6=8E=A5=E5=8F=A3?= =?UTF-8?q?=E6=96=B0=E5=A2=9E=E8=87=AA=E7=84=B6=E6=9D=91=E5=8C=B9=E9=85=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../open/controllers/AddressController.java | 80 ++++++++++--------- 1 file changed, 42 insertions(+), 38 deletions(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java index 0167259de..3902aaa98 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java @@ -87,7 +87,7 @@ import com.fangjia.service.open.WaitCrawlKeywordsService; import com.fangjia.service.sql.FjtEvaluationService; import com.fangjia.service.web.AddressService; -@Authority(role = Role.USER) +//@Authority(role = Role.USER) public class AddressController { private AddressService addressService; @@ -492,7 +492,7 @@ public class AddressController { * @param model * @return */ - @Authority(role = Role.USER) + //@Authority(role = Role.USER) public Object matchV2Action(AddesssMatchModel model) { RestfulModelAndView mav = new RestfulModelAndView(); String result = addressMatchProxyService.parser(convertModelToMap(model)); @@ -526,47 +526,51 @@ public class AddressController { } else if (jsonResult != null) { // ÅжÏÊÇ·ñΪ×ÔÈ»´å boolean matchedVillage = false; - if (jsonResult.get("searchAddress") != null) { - JSONObject searchAddress = (JSONObject) jsonResult.get("searchAddress"); - String city = searchAddress.getString("city"); - String region = searchAddress.getString("district"); - String name = searchAddress.getString("keywords"); - String township = searchAddress.getString("township"); - if (StringUtils.isNotBlank(city) && StringUtils.isNotBlank(region) && StringUtils.isNotBlank(name)) { - List villages = findVillage(city, region, name); - if (CollectionUtils.isNotEmpty(villages)) { - if (villages.size() == 1 || StringUtils.isBlank(township)) { - JSONObject village = villages.get(0).getJSONObject("_source"); - jsonResult.put("id", villages.get(0).getString("_id")); - jsonResult.put("province", village.getString("province")); - jsonResult.put("city", village.getString("city")); - jsonResult.put("district", village.getString("region")); - jsonResult.put("township", village.getString("township")); - jsonResult.put("name", village.getString("name")); - jsonResult.put("lng", village.getJSONObject("location").getDouble("lon")); - jsonResult.put("lat", village.getJSONObject("location").getDouble("lat")); - jsonResult.put("propertyType", "×ÔÈ»´å"); - matchedVillage = true; - } else { - for (JSONObject source : villages) { - JSONObject village = source.getJSONObject("_source"); - if (village.getString("township").contains(township) || township.contains(village.getString("township"))) { - jsonResult.put("id", source.getString("_id")); - jsonResult.put("province", village.getString("province")); - jsonResult.put("city", village.getString("city")); - jsonResult.put("district", village.getString("region")); - jsonResult.put("township", village.getString("township")); - jsonResult.put("name", village.getString("name")); - jsonResult.put("lng", village.getJSONObject("location").getDouble("lon")); - jsonResult.put("lat", village.getJSONObject("location").getDouble("lat")); - jsonResult.put("propertyType", "×ÔÈ»´å"); - matchedVillage = true; - break; + try { + if (jsonResult.get("searchAddress") != null) { + JSONObject searchAddress = (JSONObject) jsonResult.get("searchAddress"); + String city = searchAddress.containsKey("city") ? searchAddress.getString("city") : null; + String region = searchAddress.containsKey("district") ? searchAddress.getString("district") : null; + String name = searchAddress.containsKey("keywords") ? searchAddress.getString("keywords") : null; + String township = searchAddress.containsKey("township") ? searchAddress.getString("township") : null; + if (StringUtils.isNotBlank(city) && StringUtils.isNotBlank(region) && StringUtils.isNotBlank(name)) { + List villages = findVillage(city, region, name); + if (CollectionUtils.isNotEmpty(villages)) { + if (villages.size() == 1 || StringUtils.isBlank(township)) { + JSONObject village = villages.get(0).getJSONObject("_source"); + jsonResult.put("id", villages.get(0).getString("_id")); + jsonResult.put("province", village.getString("province")); + jsonResult.put("city", village.getString("city")); + jsonResult.put("district", village.getString("region")); + jsonResult.put("township", village.getString("township")); + jsonResult.put("name", village.getString("name")); + jsonResult.put("lng", village.getJSONObject("location").getDouble("lon")); + jsonResult.put("lat", village.getJSONObject("location").getDouble("lat")); + jsonResult.put("propertyType", "×ÔÈ»´å"); + matchedVillage = true; + } else { + for (JSONObject source : villages) { + JSONObject village = source.getJSONObject("_source"); + if (village.getString("township").contains(township) || township.contains(village.getString("township"))) { + jsonResult.put("id", source.getString("_id")); + jsonResult.put("province", village.getString("province")); + jsonResult.put("city", village.getString("city")); + jsonResult.put("district", village.getString("region")); + jsonResult.put("township", village.getString("township")); + jsonResult.put("name", village.getString("name")); + jsonResult.put("lng", village.getJSONObject("location").getDouble("lon")); + jsonResult.put("lat", village.getJSONObject("location").getDouble("lat")); + jsonResult.put("propertyType", "×ÔÈ»´å"); + matchedVillage = true; + break; + } } } } } } + } catch(Exception ex) { + ex.printStackTrace(); } if (matchedVillage == false) { jsonResult.put("matchResult", new MatchDistrictResult()); From b7816ccf726ea64aa7794d8b8b41f01991fb6872 Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Fri, 17 Feb 2023 13:24:33 +0800 Subject: [PATCH 74/99] =?UTF-8?q?=E5=8C=B9=E9=85=8D=E6=8E=A5=E5=8F=A3?= =?UTF-8?q?=E6=96=B0=E5=A2=9E=E8=87=AA=E7=84=B6=E6=9D=91=E5=8C=B9=E9=85=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../open/controllers/AddressController.java | 76 ++++++++++--------- 1 file changed, 40 insertions(+), 36 deletions(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java index 0167259de..27942a8b1 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java @@ -526,47 +526,51 @@ public class AddressController { } else if (jsonResult != null) { // ÅжÏÊÇ·ñΪ×ÔÈ»´å boolean matchedVillage = false; - if (jsonResult.get("searchAddress") != null) { - JSONObject searchAddress = (JSONObject) jsonResult.get("searchAddress"); - String city = searchAddress.getString("city"); - String region = searchAddress.getString("district"); - String name = searchAddress.getString("keywords"); - String township = searchAddress.getString("township"); - if (StringUtils.isNotBlank(city) && StringUtils.isNotBlank(region) && StringUtils.isNotBlank(name)) { - List villages = findVillage(city, region, name); - if (CollectionUtils.isNotEmpty(villages)) { - if (villages.size() == 1 || StringUtils.isBlank(township)) { - JSONObject village = villages.get(0).getJSONObject("_source"); - jsonResult.put("id", villages.get(0).getString("_id")); - jsonResult.put("province", village.getString("province")); - jsonResult.put("city", village.getString("city")); - jsonResult.put("district", village.getString("region")); - jsonResult.put("township", village.getString("township")); - jsonResult.put("name", village.getString("name")); - jsonResult.put("lng", village.getJSONObject("location").getDouble("lon")); - jsonResult.put("lat", village.getJSONObject("location").getDouble("lat")); - jsonResult.put("propertyType", "×ÔÈ»´å"); - matchedVillage = true; - } else { - for (JSONObject source : villages) { - JSONObject village = source.getJSONObject("_source"); - if (village.getString("township").contains(township) || township.contains(village.getString("township"))) { - jsonResult.put("id", source.getString("_id")); - jsonResult.put("province", village.getString("province")); - jsonResult.put("city", village.getString("city")); - jsonResult.put("district", village.getString("region")); - jsonResult.put("township", village.getString("township")); - jsonResult.put("name", village.getString("name")); - jsonResult.put("lng", village.getJSONObject("location").getDouble("lon")); - jsonResult.put("lat", village.getJSONObject("location").getDouble("lat")); - jsonResult.put("propertyType", "×ÔÈ»´å"); - matchedVillage = true; - break; + try { + if (jsonResult.get("searchAddress") != null) { + JSONObject searchAddress = (JSONObject) jsonResult.get("searchAddress"); + String city = searchAddress.containsKey("city") ? searchAddress.getString("city") : null; + String region = searchAddress.containsKey("district") ? searchAddress.getString("district") : null; + String name = searchAddress.containsKey("keywords") ? searchAddress.getString("keywords") : null; + String township = searchAddress.containsKey("township") ? searchAddress.getString("township") : null; + if (StringUtils.isNotBlank(city) && StringUtils.isNotBlank(region) && StringUtils.isNotBlank(name)) { + List villages = findVillage(city, region, name); + if (CollectionUtils.isNotEmpty(villages)) { + if (villages.size() == 1 || StringUtils.isBlank(township)) { + JSONObject village = villages.get(0).getJSONObject("_source"); + jsonResult.put("id", villages.get(0).getString("_id")); + jsonResult.put("province", village.getString("province")); + jsonResult.put("city", village.getString("city")); + jsonResult.put("district", village.getString("region")); + jsonResult.put("township", village.getString("township")); + jsonResult.put("name", village.getString("name")); + jsonResult.put("lng", village.getJSONObject("location").getDouble("lon")); + jsonResult.put("lat", village.getJSONObject("location").getDouble("lat")); + jsonResult.put("propertyType", "×ÔÈ»´å"); + matchedVillage = true; + } else { + for (JSONObject source : villages) { + JSONObject village = source.getJSONObject("_source"); + if (village.getString("township").contains(township) || township.contains(village.getString("township"))) { + jsonResult.put("id", source.getString("_id")); + jsonResult.put("province", village.getString("province")); + jsonResult.put("city", village.getString("city")); + jsonResult.put("district", village.getString("region")); + jsonResult.put("township", village.getString("township")); + jsonResult.put("name", village.getString("name")); + jsonResult.put("lng", village.getJSONObject("location").getDouble("lon")); + jsonResult.put("lat", village.getJSONObject("location").getDouble("lat")); + jsonResult.put("propertyType", "×ÔÈ»´å"); + matchedVillage = true; + break; + } } } } } } + } catch(Exception ex) { + ex.printStackTrace(); } if (matchedVillage == false) { jsonResult.put("matchResult", new MatchDistrictResult()); From 795352405a1792f30b26b98b093c44428ae53e08 Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Fri, 17 Feb 2023 13:29:43 +0800 Subject: [PATCH 75/99] =?UTF-8?q?=E5=8C=B9=E9=85=8D=E6=8E=A5=E5=8F=A3?= =?UTF-8?q?=E6=96=B0=E5=A2=9E=E8=87=AA=E7=84=B6=E6=9D=91=E5=8C=B9=E9=85=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/fangjia/open/controllers/AddressController.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java index 3902aaa98..27942a8b1 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java @@ -87,7 +87,7 @@ import com.fangjia.service.open.WaitCrawlKeywordsService; import com.fangjia.service.sql.FjtEvaluationService; import com.fangjia.service.web.AddressService; -//@Authority(role = Role.USER) +@Authority(role = Role.USER) public class AddressController { private AddressService addressService; @@ -492,7 +492,7 @@ public class AddressController { * @param model * @return */ - //@Authority(role = Role.USER) + @Authority(role = Role.USER) public Object matchV2Action(AddesssMatchModel model) { RestfulModelAndView mav = new RestfulModelAndView(); String result = addressMatchProxyService.parser(convertModelToMap(model)); From 1ccce2f38ae0ba8188a63686d21d5842ce8691c2 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 17 Feb 2023 14:06:44 +0800 Subject: [PATCH 76/99] update --- .../BigdataCommbuildtypeBatch.groovy | 43 +- .../BigdataCommunityBatch.groovy | 26 +- .../ccbevaluation/BigdataDealBatch.groovy | 74 +- .../ccbevaluation/BigdataForsaleBatch.groovy | 28 +- .../ccbevaluation/CcbFtpTestUtils.groovy | 2 +- .../DealAndThirdPriceDataProcess.groovy | 8 +- .../ExportOracleAllDataTest.groovy | 5 +- .../ExportOracleEvaluationData.groovy | 3 +- .../ccbevaluation/PgsqlCommUtils.groovy | 20 + .../fangjia/script/ccbevaluation/Test.groovy | 101 ++- .../script/ccbevaluation/TradeStatLog.groovy | 12 +- .../pgsql/ExportPgsqlEvaluationResult.groovy | 39 +- .../pgsql/ExportPgsqlRonghe.groovy | 604 +++++++++++++++ .../script/chongqing/HailanTest.groovy | 703 ++++++++++++++++++ .../script/chongqing/MingShenPocStat.groovy | 328 ++++++++ .../script/chongqing/MingShenPocTest.groovy | 459 ++++++++++++ .../script/chongqing/ThreeDESUtils.groovy | 105 +++ .../fuzhou/DistrictSellStatBatch.groovy | 2 + .../com/fangjia/script/fuzhou/Test1.groovy | 2 +- .../fangjia/script/gj/AddAvgpriceBatch.groovy | 1 + .../script/gj/ShLongShangHangLocation.groovy | 52 ++ .../gj/es/FangjiaCorestieLogAnaly.groovy | 42 ++ .../gj/paimai/BeiKeCrawlByChrome.groovy | 4 +- .../gj/paimai/LianjiaCrawlByChrome.groovy | 2 +- .../script/wenyan/temp/DongGuanBank.groovy | 6 +- 25 files changed, 2561 insertions(+), 110 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/PgsqlCommUtils.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/HailanTest.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MingShenPocStat.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MingShenPocTest.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/ThreeDESUtils.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ShLongShangHangLocation.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/FangjiaCorestieLogAnaly.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy index 066318e29..90803d4ca 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy @@ -7,6 +7,7 @@ import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.ccb.CcbCodeUtils import com.fangjia.script.gj.ccb.FileUtils import com.fangjia.script.util.DateUtils +import groovy.sql.Sql import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService @@ -22,6 +23,9 @@ class BigdataCommbuildtypeBatch extends BaseBatch{ public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40) + def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified", + "postgres", "postgres", "org.postgresql.Driver") + static main(args) { runBatch(new BigdataCommbuildtypeBatch(), args); } @@ -29,14 +33,14 @@ class BigdataCommbuildtypeBatch extends BaseBatch{ @Override public void process(Object args) { def citys = CcbCommonUtils.getRunningCitys(args) - citys = citys.unique() def rets = [] def cityCodeDatas = CcbCommonUtils.findCityCodeValue() for(def city : citys){ // def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) // def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2","name_cat","build_type", "lng","c_date", "m_date"]) - def distIds = CcbCommonUtils.findAllDistrictIdsByOracle(cityCodeDatas.get(city)); +// def distIds = CcbCommonUtils.findAllDistrictIdsByOracle(cityCodeDatas.get(city)); + def distIds = PgsqlCommUtils.findAllDistrictIdsByOracle(cityCodeDatas.get(city)) def rows = createDatas(cityCodeDatas.get(city), distIds) rets.addAll(rows) println city +" "+rows.size() @@ -50,7 +54,9 @@ class BigdataCommbuildtypeBatch extends BaseBatch{ def headers = []//["³ÇÊÐID","Ð¡ÇøID"," ½¨ÖþÀàÐÍ","±¸×¢","´´½¨Ê±¼ä","¸üÐÂʱ¼ä","À´Ô´¹«Ë¾","Êý¾Ýʱ¼ä"] FileUtils.batchCreateFile(rets, fileName,headers, "utf-8", false); - println "txt Éú³É" + println "txt Éú³É->µ¼Oracle" + + saveToPgSql(rets) } @@ -103,4 +109,35 @@ class BigdataCommbuildtypeBatch extends BaseBatch{ } return rets; } + + + def saveToPgSql(rows){ + def sqls = [] + for(def r : rows) { + for (def i = 0; i < r.size(); i++) { + if("" == r[i]){ + r[i] = null + } + } + r << null + r << '2023-02-28' + r << null + r << '2999-12-31' + r << null + r << null + r << null + } + + def sql = "insert into base.t0831_bigdata_commbuildtype_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" + println "start: "+new Date() + def index =0 + sqlInstance.withBatch(20000, sql){ ps-> + for(def r : rows) { + index++ + println index + ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14]) + } + } + println "end: "+new Date() + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy index 1580dda9b..123ada0cf 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -41,7 +41,7 @@ class BigdataCommunityBatch extends BaseBatch{ @Override public void process(Object args) { def citys = CcbCommonUtils.getRunningCitys(args) -// citys = ["ÎIJý"] + citys = ["ÉϺ£"] def rows = [] def headers = []//["³ÇÊÐID","ÐÐÕþÇøID","Ð¡ÇøID","ÐÐÕþÇøÃû³Æ","Ð¡ÇøÃû³Æ","ÏêϸµØÖ·","γ¶È","¾­¶È","Õ¼µØ×ÜÃæ»ý","½¨Öþ×ÜÃæ»ý","ÂÌ»¯ÂÊ","ÈÝ»ýÂÊ","³µÎ»ÊýÁ¿","½¨ÖþÄê´ú","·¿ÎÝÓÃ;", @@ -247,8 +247,7 @@ class BigdataCommunityBatch extends BaseBatch{ println "start: "+new Date() def sql = "insert into base.t0831_bigdata_community_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" -// def sql = "insert into base.t0831_bigdata_community_gj_h values " - def index = 0 + sqlInstance.withBatch(20000, sql){ ps-> for(def r : rows) { ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20], @@ -256,26 +255,7 @@ class BigdataCommunityBatch extends BaseBatch{ ps.toString() } } - for(def r : rows){ - index ++ - if(index == 1){ - sql = sql + "("+r[0].toString()+","+r[1].toString()+","+r[2].toString()+")" - }else{ - sql = sql + ",("+")" - } - } -// sqlInstance.withBatch(50000, sql){ ps-> -// def index = 0 -// for(def r : rows){ -// index++ -// for(def i =0 ;i + count = it.count + }); + return count; + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpTestUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpTestUtils.groovy index 81958589e..00ed6bf6e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpTestUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpTestUtils.groovy @@ -7,7 +7,7 @@ import org.apache.commons.codec.digest.DigestUtils class CcbFtpTestUtils { - static def ftpServer = "192.168.0.86"//45 + static def ftpServer = "192.168.0.45"//45 static def ftpPort = 17495 static def ftpUser = "ccb" static def ftpPassword = "ccb!@#" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy index 6b28570f1..3123dc829 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy @@ -25,9 +25,9 @@ class DealPriceDataProcess { def task = new DealPriceDataProcess() // task.process()//Ô˵±³É½»peÖµ²»ºÃʱÐУ¬À­Éý³É½»¼Û¸ñ¼°É¾³ýpeÌ«´óÊý¾Ý // task.addLianJiaPrice(); -// task.deleteThirdPartyData() + task.deleteThirdPartyData() // task.delDealByPe() - task.delDealData() +// task.delDealData() } def delDealByPe(){ @@ -372,12 +372,12 @@ class DealPriceDataProcess { def deleteThirdPartyData(){ def citys = CcbCommonUtils.citys_all - def month = 202211 + def month = 202212 def rows = [] def cityCodeDatas = CcbCommonUtils.findCityCodeValue() def set = new HashSet() for(def city : citys) { - def rets = findPeGteThirdParty(city, 202212) + def rets = findPeGteThirdParty(city, 202301) def total = 0 println city +" "+rets.size() if(rets){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy index 39ef500ac..4df84ea86 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy @@ -13,6 +13,7 @@ import org.apache.commons.codec.digest.DigestUtils import java.sql.ResultSet +//¹ÀÖµ ÐÂ°æ µ¼³öËùÓÐÊý¾Ý class ExportOracleAllDataTest { def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", @@ -20,7 +21,7 @@ class ExportOracleAllDataTest { def cityCodeMap = CcbCommonUtils.findCityCodeValue() def citys = null - def curDate = "20221228"//DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def curDate = "20220128"//DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; static main(args) { def task = new ExportOracleAllDataTest() @@ -38,7 +39,7 @@ class ExportOracleAllDataTest { def run(args) { citys = CcbCommonUtils.getRunningCitys(args)//["Ϋ·»","¼ÃÄþ","Íþº£","ÁÙÒÊ","ÃàÑô","¤ÄÏ","Ëþ³Ç","°¢ÀÕÌ©","µÏÇìÖÝ","ÈÕ¿¦Ôò"] //CcbCommonUtils.getRunningCitys(args) - def curMonth = "202212"//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")); + def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")); def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); def lastMonthStr = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM") def new_dists = [] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleEvaluationData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleEvaluationData.groovy index 419f170b0..513e7025c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleEvaluationData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleEvaluationData.groovy @@ -31,7 +31,7 @@ class ExportOracleEvaluationData { exportCityPrice(curMonth) exportRegionPrice(curMonth) - exportCommunityBasePriceGj(curMonth)//ÒѺÏÏÂÃæµÄºÏ²¢ + // exportCommunityBasePriceGj(curMonth)//ÒѺÏÏÂÃæµÄºÏ²¢ exportCommunityBasePrice(curMonth) exportCommunityPdescribeGj(curMonth) @@ -248,6 +248,7 @@ class ExportOracleEvaluationData { def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/PgsqlCommUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/PgsqlCommUtils.groovy new file mode 100644 index 000000000..f2a564cb7 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/PgsqlCommUtils.groovy @@ -0,0 +1,20 @@ +package com.fangjia.script.ccbevaluation + +import groovy.sql.Sql + +class PgsqlCommUtils { + + public static def database = "warehouse" + + public static def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/"+database+"?stringtype=unspecified", + "postgres", "postgres", "org.postgresql.Driver") + + public static def findAllDistrictIdsByOracle(def cityid){ + def sql = "select gujia_comm_id from base.t0831_bigdata_community_gj_h where urbn_id='${cityid}' and gujia_bds_bds_st=0"; + def set = new HashSet() + sqlInstance.eachRow(sql,{r-> + set << r.gujia_comm_id + }); + return set + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index 14a238118..8aca6bac6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -45,8 +45,91 @@ DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_craw def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017"; def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business", "res_esfdealprice", offlineUrl) +def citys = ["ɽÄÏ","»ÆÄÏÖÝ","¸ÊÄÏ","¿Ë×ÎÀÕËÕ","¼ÎÓø¹Ø","º£±±ÖÝ","°¢À­ÉÆÃË","Å­½­ÖÝ","ºÍÌï","ÈÕ¿¦Ôò","²ý¶¼","º£ÄÏÖÝ","ÓñÊ÷ÖÝ","ͳ·¬","°¢ÀÕÌ©","Èýɳ","Ëþ³Ç","¸Ê×ÎÖÝ","¹ûÂåÖÝ","ÁÖÖ¥","ÄÇÇú","°¢°ÓÖÝ","ÉñÅ©¼Ü"] +def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() +ExcelUtils.multiThreadingModify("D:\\Êý¾Ý\\Ð¡Çø\\Ð¡ÇøµØÖ·Ãûµ¥_¹À¼Ò¿Æ¼¼V2 (3).xlsx",{d-> + if(d['J']){ + def cityId = cityCodeDataMap.get(d['H']) + def map = findCcbPriceCredit(sqlInstance236, cityId, d['J']) + println d['H'] +" "+d['J']+ map.toString() + if(map){ + d['Q'] = map.credit + d['R'] = map.month + d['U'] = map.price + } +// def thirdMap = findThirdPriceBySource(sqlInstance236, cityId, d['J']) +// if(thirdMap){ +// d['S'] = thirdMap.price +// d['T'] = thirdMap.month +// } + } + +}) + +def findCcbPriceCredit(sqlInstance, cityid, commid){ + def sql = "select * from (select CONFIDENCE,year_month,avg_price from community_base_price_gj where cityid='${cityid}' and commid='${commid}' order by year_month desc) WHERE ROWNUM <= 1" + def map = [:] + sqlInstance.eachRow(sql,{r-> + map.month = r.year_month + map.credit = r.CONFIDENCE + map.price = r.avg_price + }); + return map +} + +def findThirdPriceBySource(sqlInstance, cityid, commid){ + def sql = "select * from (select data_source, price_date, community_price,commid from bigdata_third_party_price_gj where cityid='${cityid}' and commid='${commid}' and data_source in ('±´¿Ç','Á´¼Ò') \n" + + "order by price_date desc) WHERE ROWNUM <= 1" + def map = [:] + sqlInstance.eachRow(sql,{r-> + map.month = r.price_date + map.price = r.community_price + }); + return map +} + + + + + +for(def c : citys){ + def cityCode = cityCodeDataMap.get(c) + println c +" "+cityCode + def priceDists = findAllPriceDist(sqlInstance236,cityCode, 202301) +// + checkNoPriceDist(sqlInstance236, priceDists, cityCode) +} + +def checkNoPriceDist(sqlInstance236,priceDists, cityid){ + def commids = [] + def sql = "select commid from bigdata_community_gj where cityid='${cityid}' and status=0"; + sqlInstance236.eachRow(sql, { r -> + commids << r.commid + }) + + for(def commid : commids){ + if(!priceDists.contains(commid)){ + updateDistStatus(sqlInstance236, commid) + } + } + return commids +} + + +def updateDistStatus(sqlInstance236, commid){ + def sql = "update bigdata_community_gj set status=1 where commid='${commid}'"; + sqlInstance236.executeUpdate(sql) +} +def findAllPriceDist(sqlInstance236, cityid, yearMonth){ + def commids = [] + def sql = "select commid from community_base_price_gj where cityid='${cityid}' and year_month=${yearMonth} "; + sqlInstance236.eachRow(sql, { r -> + commids << r.commid + }) + return commids +} //def citys = ['ËÕÖÝ','º¼ÖÝ','Äþ²¨','ÎÂÖÝ','¼ÎÐË','ºþÖÝ','ÉÜÐË','½ð»ª','áéÖÝ','ÖÛɽ','̨ÖÝ','ÀöË®'] // //def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() @@ -1087,15 +1170,15 @@ def findThirdCommCount(cityid, sqlInstance){ //} -def fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Ö¸µ¼¼Û\\¹úÐÅ´ï¼Û¸ñ-º¼ÖÝ.xlsx" -ExcelUtils.multiThreadingModify(fileName, { d -> - def dist = CommonDatautils.findDistrictById(d['A']) - if(dist){ - d['C'] = dist.city - d['D'] = dist.region - d['E'] = dist.name - } -}) +//def fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Ö¸µ¼¼Û\\¹úÐÅ´ï¼Û¸ñ-º¼ÖÝ.xlsx" +//ExcelUtils.multiThreadingModify(fileName, { d -> +// def dist = CommonDatautils.findDistrictById(d['A']) +// if(dist){ +// d['C'] = dist.city +// d['D'] = dist.region +// d['E'] = dist.name +// } +//}) //def fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Ö¸µ¼¼Û\\0922Èë¿âÊý¾Ý.xlsx" //def datas = ExcelUtils.read(fileName, 0, 1) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy index e87103a90..92fb500fc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy @@ -40,7 +40,7 @@ class TradeStatLog { static main(args) { def task = new TradeStatLog() -// task.run()//ͳ¼Æ×î½ü3¸öÔÂץȡµÄ³É½»Êý¾ÝÁ¿ + task.run()//ͳ¼Æ×î½ü3¸öÔÂץȡµÄ³É½»Êý¾ÝÁ¿ // task.exportTrade()//µ¼³öδÈë¿â³É½» // task.thirdPartyStat() // println Calendar.getInstance().get(Calendar.YEAR) @@ -92,13 +92,13 @@ class TradeStatLog { def run(){ def citys = CcbCommonUtils.citys_all; def rets = [] - rets << ["³ÇÊÐ","9ÔÂ×ÜÁ¿","10ÔÂ×ÜÁ¿","11ÔÂ×ÜÁ¿","Õ¼±È","±´¿Ç","Á´¼Ò", "ÎÒ°®ÎÒ¼Ò","·¿ÌìÏÂ","ÖÐÔ­"] + rets << ["³ÇÊÐ","10ÔÂ×ÜÁ¿","11ÔÂ×ÜÁ¿","12ÔÂ×ÜÁ¿","Õ¼±È","±´¿Ç","Á´¼Ò", "ÎÒ°®ÎÒ¼Ò","·¿ÌìÏÂ","ÖÐÔ­"] def sources = ["±´¿ÇÕÒ·¿","Á´¼ÒÔÚÏß","ÎÒ°®ÎÒ¼Ò","·¿ÌìÏÂ","ÖÐÔ­µØ²ú"] //def startDate = "2021-05-01" - def startDate = "2022-09-01" - def date = "2022-10-01" - def endDate = "2022-11-01" - def newDate = "2022-12-01" + def startDate = "2022-10-01" + def date = "2022-11-01" + def endDate = "2022-12-01" + def newDate = "2023-01-01" for(def city : citys){ def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd")) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlEvaluationResult.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlEvaluationResult.groovy index 1508d0b9f..24254df78 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlEvaluationResult.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlEvaluationResult.groovy @@ -1,8 +1,10 @@ package com.fangjia.script.ccbevaluation.pgsql import com.fangjia.script.ccbevaluation.CcbCommonUtils +import com.fangjia.script.ccbevaluation.CcbFtpTestUtils import com.fangjia.script.ccbevaluation.CcbFtpUtils import com.fangjia.script.ccbevaluation.ronghe.SSHHelper +import com.fangjia.script.ccbevaluation.ronghe.SSHHelperTest import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.ccb.FileUtils import com.fangjia.script.util.DateUtils @@ -43,13 +45,11 @@ class ExportPgsqlEvaluationResult { // exportCommunityPdescribeGj(curMonth) // exportCommCaseDetailGj(curMonth) // - def curDate = curMonth.toString()+"28" -// Éú³ÉMD5Îļþ - exportMd5() - SSHHelper jschCommand = new SSHHelper(); - System.out.println(jschCommand.executeCommand("sh /home/ccb/output/gz_package.sh ${curDate}")); -// okÎļþ - exportOkFile() + exportInOk() + + SSHHelperTest jschCommand = new SSHHelperTest(); + System.out.println(jschCommand.executeCommand("sh /home/ccb_evaluation/output/eval_package.sh ${curDate}")); + } def exportAvmEvaluationGJNB(yearMonth){ @@ -363,20 +363,31 @@ class ExportPgsqlEvaluationResult { } - def exportOkFile(){ - def dataCodeFileName = linux_prefix + "${curDate}.ok" - if(CommonDatautils.isWindow()){ - dataCodeFileName = win_prefix + "${curDate}.ok" + def exportInOk(){ + Vector files = CcbFtpTestUtils.listFileNames(null, curDate) + Iterator fileVec = files.iterator() + def fileRows = [] + while(fileVec.hasNext()){ + ChannelSftp.LsEntry file = fileVec.next() + + if(org.apache.commons.lang.StringUtils.isNotBlank(file.getFilename()) && file.getFilename().endsWith(".dat")){ + fileRows << file.getFilename() +" "+CcbFtpTestUtils.getFileLength(null, file.getFilename(), curDate) + } } - createFileToFtp("ok txt", [], dataCodeFileName, curDate, false) + def fileName = linux_prefix + "/${curDate}.ok" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "${curDate}.ok" + } + + createFileToFtp("ok txt",fileRows, fileName, curDate, false) } def createFileToFtp(tableName,allRows, fileName, curDate, isAppend){ FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", isAppend); println tableName+" Êý¾ÝÎļþÊä³öÍê³É......" - CcbFtpUtils.ftpUpload(null, fileName, curDate) -// println tableName+" ÉÏ´«FtpÍê³É......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println tableName+" ÉÏ´«FtpÍê³É......" } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlRonghe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlRonghe.groovy index 8d3d01341..5a4068b7c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlRonghe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlRonghe.groovy @@ -1,14 +1,618 @@ package com.fangjia.script.ccbevaluation.pgsql +import com.fangjia.script.ccbevaluation.CcbCommonUtils +import com.fangjia.script.ccbevaluation.CcbFtpTestUtils +import com.fangjia.script.ccbevaluation.CcbFtpUtils +import com.fangjia.script.ccbevaluation.ronghe.SSHHelperTest +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import com.jcraft.jsch.ChannelSftp +import groovy.sql.Sql + class ExportPgsqlRonghe { + def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/postgres", + "postgres", "postgres", "org.postgresql.Driver") + + def citys = [] + + def database = "base." + def cityCodeMap = CcbCommonUtils.findCityCodeValue() + + def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd") + def linux_prefix = "/ccb_ronghe/${curDate}/" + def win_prefix = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\ccb_ronghe\\${curDate}\\" + static main(args) { def task = new ExportPgsqlRonghe() task.run(args) } def run(args) { + def new_dists = [] + citys = CcbCommonUtils.getRunningCitys(args) + //°´Ììµ¼£¨Ã¿ÌìÒ»¸öÎļþ¼Ð£© + exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date() + 1, -1), "yyyy-MM-dd"), new_dists) + if (curDate.toString().endsWith("10")) { + + exportEvaluateRate() + exportBigdataBuildprojFjw() + exportBigdataCommlayoutpicFjw() + + + exportBigdataCommunityGJRH()// + exportBigdataBuildingFJW()// + exportBigdataCommfeeGj()// + + exportBigdataCommbuildstruGJ()// + exportBigdataCommhousetypeGJ()// + exportBigdataCommbuildyearGj()// + exportBigdataCommhouseuseGJ()// + exportBigdataPositionGJ()// + exportBigdataDistrictFJW()// + exportBigdataAliasFJW()// + + + //±¾12.10ÉÏ´«10£¬11ÔÂËùÓÐÊý¾Ý£¬23Äê1ÔÂÐè¸ÄΪËùÓгÇÊа´ÈÕÆÚͬ²½Ò»ÆÚÊý¾Ý + def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger() +// exportBigdataCityPrice(queryDate) +// exportBigdataDistPrice(queryDate) +// exportBigdataCommPrice(queryDate) + + exportInOk() + +// } + +// SSHHelperTest jschCommand = new SSHHelperTest(); +// System.out.println(jschCommand.executeCommand("sh /home/ccb_ronghe/output/rh_package.sh ${curDate}")); + } + } + + + def exportBigdataDealFJW2(curDate, dealdate, new_dists){ + + def startDateStr = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd") + def endDateStr = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd") + + def fileName = "FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" + } + + def allRows = [] + + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + if(curDate.toString().endsWith("10")){ + if(new_dists){ + def newstr = "('"+new_dists.join("','")+"')" + def sql1 = "select urbn_id,mdl_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_str,gujia_hsno,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_prd_val,gujia_wbt_floor,gujia_lyr_tp_val,gujia_tot_floor_num,gujia_bld_tp_val,gujia_bld_stc,gujiaconstructionarea,gujia_set_innr_area,gujia_acc_tp_val,gujia_room,gujia_dept,gujia_ktch_num,gujia_toliet,gujia_hs_use,gujia_indr_stc_val,gujia_twrd_val,gujia_dcrt_sttn,gujia_ases_bt_ln_num,gujia_trdtckt_prc,gujia_mdl_tot_prc,gujia_dcl_tot_prc,gujia_ases_tot_prc,gujiasigntonlinectrid,to_char(txn_dt,'yyyy-mm-dd') as txn_dt,gujia_txn_tp_val,gujia_sell_pymtmod,gujia_mdl_cyc,gujiawthrext_lift_ind,gujiagvlndfargrdnarea,gujia_gv_lu_tbl_area,gujia_gv_sun_tbl_area,gujiagvbaywindow_area,gujiagvarea_tower_flr,gujia_gv_grg_area,gujiagvundgdroom_area,gujia_gv_othr_area,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_mkdam,gujia_data_tm " + + "from "+database+"t0831_bigdata_deal_gj_h where urbn_id='${cityCode}' and txn_dt>=to_date('${dealDate}','yyyy-MM-dd') and gujia_admndiv_id not in ${newstr}" + def rets1 = dealData(sql1) + + def sql2 = "select urbn_id,mdl_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_str,gujia_hsno,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_prd_val,gujia_wbt_floor,gujia_lyr_tp_val,gujia_tot_floor_num,gujia_bld_tp_val,gujia_bld_stc,gujiaconstructionarea,gujia_set_innr_area,gujia_acc_tp_val,gujia_room,gujia_dept,gujia_ktch_num,gujia_toliet,gujia_hs_use,gujia_indr_stc_val,gujia_twrd_val,gujia_dcrt_sttn,gujia_ases_bt_ln_num,gujia_trdtckt_prc,gujia_mdl_tot_prc,gujia_dcl_tot_prc,gujia_ases_tot_prc,gujiasigntonlinectrid,to_char(txn_dt,'yyyy-mm-dd') as txn_dt,gujia_txn_tp_val,gujia_sell_pymtmod,gujia_mdl_cyc,gujiawthrext_lift_ind,gujiagvlndfargrdnarea,gujia_gv_lu_tbl_area,gujia_gv_sun_tbl_area,gujiagvbaywindow_area,gujiagvarea_tower_flr,gujia_gv_grg_area,gujiagvundgdroom_area,gujia_gv_othr_area,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_mkdam,gujia_data_tm " + + "from "+database+"t0831_bigdata_deal_gj_h where urbn_id='${cityCode}' and gujia_admndiv_id in ${newstr}" + def rets2 = dealData(sql2) + allRows.addAll(rets1) + allRows.addAll(rets2) + }else{ + def sql = "select urbn_id,mdl_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_str,gujia_hsno,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_prd_val,gujia_wbt_floor,gujia_lyr_tp_val,gujia_tot_floor_num,gujia_bld_tp_val,gujia_bld_stc,gujiaconstructionarea,gujia_set_innr_area,gujia_acc_tp_val,gujia_room,gujia_dept,gujia_ktch_num,gujia_toliet,gujia_hs_use,gujia_indr_stc_val,gujia_twrd_val,gujia_dcrt_sttn,gujia_ases_bt_ln_num,gujia_trdtckt_prc,gujia_mdl_tot_prc,gujia_dcl_tot_prc,gujia_ases_tot_prc,gujiasigntonlinectrid,to_char(txn_dt,'yyyy-mm-dd') as txn_dt,gujia_txn_tp_val,gujia_sell_pymtmod,gujia_mdl_cyc,gujiawthrext_lift_ind,gujiagvlndfargrdnarea,gujia_gv_lu_tbl_area,gujia_gv_sun_tbl_area,gujiagvbaywindow_area,gujiagvarea_tower_flr,gujia_gv_grg_area,gujiagvundgdroom_area,gujia_gv_othr_area,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_mkdam,gujia_data_tm " + + "from "+database+".t0831_bigdata_deal_gj_h where urbn_id='${cityCode}' and txn_dt>=to_date('${dealDate}','yyyy-MM-dd')"; + allRows.addAll(dealData(sql)) + } + } + } + createFileToFtp("BIGDATA_DEAL_FJW", allRows, fileName, curDate, false) + } + + + + def dealData(def sql){ + def allRows = [] + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.urbn_id + data << r.urbn_id + data << r.mdl_id + data << r.urbn_nm + data << r.gujia_admndiv_id + data << r.gujia_admndiv_nm + data << r.gujia_str + data << r.gujia_hsno + data << r.gujia_comm_id + data << r.comm_nm + data << r.gujia_bld_id + data << r.bld_id + data << r.gujia_bld_nm + data << r.gujia_unit_id + data << r.unit_id + data << r.gujia_unit_nm + data << r.gujia_acc_id + data << r.gujia_dtl_adr + data << r.gujia_bld_yr_rc + data << r.gujia_own_val + data << r.gujia_own_yrlmt_val + data << r.gujia_prd_val + data << r.gujia_wbt_floor + data << r.gujia_lyr_tp_val + data << r.gujia_tot_floor_num + if(r.gujia_bld_tp_val){ + def arrs = r.gujia_bld_tp_val.replace("[","").replace("]","").split(",") + data << arrs[0].trim() + }else{ + data << '' + } + data << r.gujia_bld_stc + data << r.gujiaconstructionarea + data << r.gujia_set_innr_area + data << r.gujia_acc_tp_val + data << r.gujia_room + data << r.gujia_dept + data << r.gujia_ktch_num + data << r.gujia_toliet + data << r.gujia_hs_use + data << r.gujia_indr_stc_val + data << r.gujia_twrd_val + data << r.gujia_dcrt_sttn + data << r.gujia_ases_bt_ln_num + data << r.gujia_trdtckt_prc + data << r.gujia_mdl_tot_prc + data << r.gujia_dcl_tot_prc + data << r.gujia_ases_tot_prc + data << r.gujiasigntonlinectrid + data << r.txn_dt + data << r.gujia_txn_tp_val + data << r.gujia_sell_pymtmod + data << r.gujia_mdl_cyc + data << r.gujiawthrext_lift_ind + data << r.gujiagvlndfargrdnarea + data << r.gujia_gv_lu_tbl_area + data << r.gujia_gv_sun_tbl_area + data << r.gujiagvbaywindow_area + data << r.gujiagvarea_tower_flr + data << r.gujia_gv_grg_area + data << r.gujiagvundgdroom_area + data << r.gujia_gv_othr_area + data << r.gujia_hs_src_dtsrc + data << r.intr_co_id + data << r.intr_co_nm + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + data << r.gujia_mkdam + data << r.gujia_data_tm + allRows << data + }); + return allRows + } + + def exportEvaluateRate(){ + def sql = "select gujia_ceft_id,gujia_ceft_tp,gujia_ceft,gujia_seq, gujia_urbn,rmrk_dsc, to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm, gujia_src_co from t0831_evaluate_rate_fjw_w"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.gujia_ceft_id + data << r.gujia_ceft_tp + data << r.gujia_ceft + data << r.gujia_seq + data << r.gujia_urbn + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + def fileName = linux_prefix + "FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat" + } + + createFileToFtp("EVALUATE_RATE_FJW", allRows, fileName, curDate, false) + } + + def exportBigdataBuildprojFjw(){ + def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat" + } + createFileToFtp("BIGDATA_BUILDPROJ_FJW", [], fileName, curDate, false) + } + + def exportBigdataCommlayoutpicFjw(){ + def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat" + } + createFileToFtp("BIGDATA_COMMLAYOUTPIC_FJW", [], fileName, curDate, false) + } + + def exportBigdataCommunityGJRH(){ + + def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat" + } + def allRows = [] + for(def city : citys){ + def cityCode = cityCodeMap.get(city) + def sql = "select urbn_id,gujia_comm_id,comm_nm,gujia_comm_fmnm,gujia_admndiv_id,gujia_str,gujia_hsno,gujia_dtl_adr,gujiaocpoflandtotarea,gujia_bld_tot_area,gujia_comm_tp_val,gujiawthr_new_pan_ind,gujia_grnrate,gujia_vol_rate,gujia_park_num,gujia_comm_intd,rmrk_dsc, to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm, to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujiaprhsld_hspty_prc,gujiacomm_tot_rsd_num,gujiacommrntfee_avprc,gujia_comm_avprc from "+database+"t0831_bigdata_community_fjw_w where urbn_id='${cityCode}'"; + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.urbn_id + data << r.gujia_comm_id + data << r.comm_nm + data << r.gujia_comm_fmnm + data << r.gujia_admndiv_id + data << r.gujia_str + data << r.gujia_hsno + data << r.gujia_dtl_adr + data << r.gujiaocpoflandtotarea + data << r.gujia_bld_tot_area + data << r.gujia_comm_tp_val + data << r.gujiawthr_new_pan_ind + data << r.gujia_grnrate + data << r.gujia_vol_rate + data << r.gujia_park_num + data << r.gujia_comm_intd + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + } + + createFileToFtp("BIGDATA_COMMUNITY_FJW", allRows, fileName, curDate, false) + } + + def exportBigdataCommbuildstruGJ(){ + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select urbn_id, gujia_comm_id, gujia_bld_stc, rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm, gujia_src_co from "+database+"t0831_bigdata_commbuildstru_fjw_w where urbn_id='${cityCode}'"; + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.urbn_id + data << r.gujia_comm_id + data << r.gujia_bld_stc + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + } + + def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat" + } + createFileToFtp("BIGDATA_COMMBUILDSTRU_FJW", allRows, fileName, curDate, false) } + def exportBigdataCommfeeGj(){ + + def allRows = [] + for(def city : citys){ + def cityCode = cityCodeMap.get(city) + def sql = "select urbn_id,gujia_comm_id,co_id,gujia_eps_val,gujia_hs_use,gujia_bld_tp_val,gujia_comm_eps_tp,gujia_eps_cmnt,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"t0831_bigdata_commfee_fjw_w where urbn_id='${cityCode}'"; + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.urbn_id + data << r.gujia_comm_id + data << r.co_id + data << r.gujia_eps_val + data << r.gujia_hs_use + if(r.gujia_bld_tp_val){ + def arrs = r.gujia_bld_tp_val.replace("[","").replace("]","").split(",") + data << arrs[0].trim() + }else{ + data << '' + } + data << r.gujia_comm_eps_tp + data << r.gujia_eps_cmnt + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + } + def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat" + } + createFileToFtp("BIGDATA_COMMFEE_FJW", allRows, fileName, curDate, false) + } + + def exportBigdataCommhousetypeGJ(){ + + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select urbn_id,gujia_comm_id,gujia_comm_hs_tp,rmrk_dsc,to_char(data_crt_tm'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"t0831_bigdata_commhousetype_fjw_w where urbn_id='${cityCode}'"; + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.urbn_id + data << r.gujia_comm_id + data << r.gujia_comm_hs_tp + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + } + + def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat" + } + createFileToFtp("BIGDATA_COMMHOUSETYPE_FJW", allRows, fileName, curDate, false) + } + + def exportBigdataCommbuildyearGj(){ + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select urbn_id,gujia_comm_id,gujia_bld_yr_rc,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"t0831_bigdata_commbuildyear_fjw_w where urbn_id='${cityCode}'";// + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.urbn_id + data << r.gujia_comm_id + data << r.gujia_bld_yr_rc + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + } + def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat" + } + createFileToFtp("BIGDATA_COMMBUILDYEAR_FJW", allRows, fileName, curDate, false) + } + + def exportBigdataCommhouseuseGJ(){ + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select urbn_id,gujia_comm_id,gujia_hs_use,gujia_ladder_accnum,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"t0831_bigdata_commhouseuse_fjw_w where urbn_id='${cityCode}'";// + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.urbn_id + data << r.gujia_comm_id + data << r.gujia_hs_use + data << r.gujia_ladder_accnum + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + } + def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat" + } + createFileToFtp("BIGDATA_COMMHOUSEUSE_FJW", allRows, fileName, curDate, false) + } + + def exportBigdataPositionGJ(){ + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select urbn_id,gujia_obj_tp,gujia_obj_id,gujia_lo_tp,gujia_sn,lgt,ltt,gujia_hignas_val,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"t0831_bigdata_position_fjw_w where urbn_id='${cityCode}'";// + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.urbn_id + data << r.gujia_obj_tp + data << r.gujia_obj_id + data << r.gujia_lo_tp + data << r.gujia_sn + data << r.lgt + data << r.ltt + data << r.gujia_hignas_val + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + } + def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat" + } + createFileToFtp("BIGDATA_POSITION_FJW", allRows, fileName, curDate, false) + } + + def exportBigdataDistrictFJW(){ + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select gujia_admndiv_id,gujia_admndiv_nm,gujia_admndiv_nm_cpa,urbn_id,gujia_natstd_cd,gujia_dspl_seq,gujia_admndiv_cgy,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"t0831_bigdata_district_fjw_w where urbn_id='${cityCode}'";// + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.gujia_admndiv_id + data << r.gujia_admndiv_nm + data << r.gujia_admndiv_nm_cpa + data << r.urbn_id + data << r.gujia_natstd_cd + data << r.gujia_dspl_seq + data << r.gujia_admndiv_cgy + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + } + + def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat" + } + createFileToFtp("BIGDATA_DISTRICT_FJW", allRows, fileName, curDate, false) + } + + def exportBigdataAliasFJW(){ + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select urbn_id,rel_id,gujia_comm_id,comm_nm,gujia_pcsst,gujia_vld_dt,gujia_vrsn_dsc,gujia_wthr_prim_nm,gujia_dtsrc,gujia_dsc,gujia_als_lvl,gujia_adr_tp_ecd,gujia_wthr_adt_ind,gujia_wthr_dspl_ind from "+database+"t0831_address_alias_fjw_w where urbn_id='${cityCode}'";// + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.urbn_id + data << r.rel_id + data << r.gujia_comm_id + data << r.comm_nm + data << r.gujia_pcsst + data << r.gujia_vld_dt + data << r.gujia_vrsn_dsc + data << r.gujia_wthr_prim_nm + data << r.gujia_dtsrc + data << r.gujia_dsc + data << r.gujia_als_lvl + data << r.gujia_adr_tp_ecd + data << r.gujia_wthr_adt_ind + data << r.gujia_wthr_dspl_ind + allRows << data + }); + } + def fileName = linux_prefix + "FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat" + } + createFileToFtp("ADDRESS_ALIAS_FJW", allRows, fileName, curDate, false) + } + + def exportBigdataBuildingFJW(){ + def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat" + } + def index = 0 + for(def city : citys) { + def allRows = [] + def cityCode = cityCodeMap.get(city) + def sql = "select urbn_id,gujia_bld_id,bld_id,gujia_bld_nm,gujia_bld_als_dsc,gujia_admndiv_id,gujia_comm_id,gujia_bld_yr_rc,gujia_bld_tp_val,gujia_bld_stc,floor_num,gujia_unit_num,gujia_accnum_val,gujiawthrext_lift_ind,gujia_lift_num,gujiajiladderjiactval,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_vld_ind from "+database+"t0831_bigdata_building_fjw_w where urbn_id='${cityCode}'";// + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.urbn_id + data << r.gujia_bld_id + data << r.bld_id + data << r.gujia_bld_nm + data << r.gujia_bld_als_dsc + data << r.gujia_admndiv_id + data << r.gujia_comm_id + data << r.gujia_bld_yr_rc + if(r.gujia_bld_tp_val){ + def arrs = r.gujia_bld_tp_val.replace("[","").replace("]","").split(",") + data << arrs[0].trim() + }else{ + data << '' + } + data << r.gujia_bld_stc + data << r.floor_num + data << r.gujia_unit_num + data << r.gujia_accnum_val + data << r.gujiawthrext_lift_ind + data << r.gujia_lift_num + data << r.gujiajiladderjiactval + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + data << r.gujia_vld_ind + allRows << data + }); + + if(allRows && allRows.size()>0){ + index = index + 1 + if(index == 1){ + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + }else{ + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", true); + } + } + } + + CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) + println "BIGDATA_BUILDING_FJW ÉÏ´«FtpÍê³É......" + } + + def exportBigdataCityPrice(queryDate){ + def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_CITYPRICE_FJW_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "FJW_000000_000000_BIGDATA_CITYPRICE_FJW_ADD_${curDate}_0001.dat" + } + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + // def sql = "select CITYID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_CITYPRICE where CITYID='${cityCode}' and YMID=${queryDate}";// + def sql = "select CITYID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from t0831_city_price_gj_h where CITYID='${cityCode}'";// + + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.YMID + data << r.SALEPRICE?r.SALEPRICE:'' + data << r.RENTPRICE?r.RENTPRICE:'' + if(r.SALECOUNT>0) { + data << r.SALECOUNT + }else{ + data << '' + } + if(r.RENTCOUNT>0){ + data << r.RENTCOUNT + }else{ + data << '' + } + data << r.UPDATETIME + data << r.HOUSEUSE + data << r.SALEBLDGAREA?r.SALEBLDGAREA:'' + data << r.RENTBLDGAREA?r.RENTBLDGAREA:'' + data << r.SALETOTALPRICE?r.SALETOTALPRICE:'' + data << r.RENTTOTALPRICE?r.RENTTOTALPRICE:'' + data << r.DATATYPE + data << r.SDATERANGE + data << r.RDATERANGE + data << r.SOURCECODE + allRows << data + }); + } + + createFileToFtp("BIGDATA_CITYPRICE", allRows, fileName, curDate, false) + } + + def exportInOk(){ + Vector files = CcbFtpTestUtils.listFileNames("rh", curDate) + Iterator fileVec = files.iterator() + def fileRows = [] + while(fileVec.hasNext()){ + ChannelSftp.LsEntry file = fileVec.next() + + if(org.apache.commons.lang.StringUtils.isNotBlank(file.getFilename()) && file.getFilename().endsWith(".dat")){ + fileRows << file.getFilename() +" "+CcbFtpTestUtils.getFileLength("rh", file.getFilename(), curDate) + } + } + def fileName = linux_prefix+"${curDate}.ok" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "${curDate}.ok" + } + createFileToFtp("ÈÚºÏͳ¼ÆÎļþ³¤¶ÈokÎļþ", fileRows, fileName, curDate, false) + } + + def createFileToFtp(tableName,allRows, fileName, curDate, isAppend){ + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", isAppend); + println tableName+" Êý¾ÝÎļþÊä³öÍê³É......" + CcbFtpTestUtils.ftpUpload("rh", fileName, curDate) + println tableName+" ÉÏ´«FtpÍê³É......" + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/HailanTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/HailanTest.groovy new file mode 100644 index 000000000..2b163d322 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/HailanTest.groovy @@ -0,0 +1,703 @@ +package com.fangjia.script.chongqing + +import com.fangjia.core.RegionData +import com.fangjia.core.util.HttpRequest +import com.fangjia.script.ccbevaluation.CcbCommonUtils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.util.OpenApiClient +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import net.sf.json.JSONObject +import org.elasticsearch.index.query.BoolQueryBuilder +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders + +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong +import java.util.regex.Matcher +import java.util.regex.Pattern + +class HailanTest { + + static def DIRECTIONS = ["¶«±±", "Î÷±±", "¶«ÄÏ", "Î÷ÄÏ", "Äϱ±", "¶«Î÷", "¶«", "ÄÏ", "Î÷", "±±"] + private static final HashMap> FITMENTS_TYPES_MAPS = new HashMap>() + static{ + FITMENTS_TYPES_MAPS.put("ëÅ÷", ["ëÅ÷", "ÇåË®", "ÎÞ×°ÐÞ", "¶À¶°Ã«Å÷", "ëÅß", "°×Å÷", "¾É×°", "Å÷", "Ô­·¿", "뻵"]) + FITMENTS_TYPES_MAPS.put("¼ò×°", ["¼ò×°", "ÆÕͨװÐÞ", "ÆÕͨ", "¼òµ¥×°ÐÞ", "¼òÒ××°ÐÞ", "ÆÕ×°", "Ò»°ã×°ÐÞ", "Ò»°ã", "¼ò", "´Ö×°ÐÞ"]) + FITMENTS_TYPES_MAPS.put("¾«×°", ["¾«×°", "ȫװÐÞ", "ȫװ", "ÖеÈ×°ÐÞ", "ÖÐ×°", "Öеµ×°ÐÞ", "¾«ÖÂ×°ÐÞ", "Öеµ", "ÖеÈ", "¾«", "ÖÐÐÞ", "ÓÅÖÊ×°ÐÞ", "ʱÉÐ×°ÐÞ", "Á¼×°", "Öиߵ²"]) + FITMENTS_TYPES_MAPS.put("ºÀ×°", ["ºÀ»ª×°ÐÞ", "¸ßµµ", "ºÀ»ª", "ºÀ×°", "¸ß¼¶×°ÐÞ", "¸ß×°", "ÉݳÞ"]) + } + + static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80) + def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") + + static main(args) { + def task = new HailanTest(); + task.run(); + } + + def run(){ + def inputfile = "D:\\Êý¾Ý\\º£ÄÏ\\ͳ¼Æ.xlsx" +// match(inputfile) +// addPrice(inputfile) +// addCommId("D:\\Êý¾Ý\\º£ÄÏ\\Æ¥ÅäÉϵĺϲ¢Êý¾Ý.xlsx") +// addHouse(inputfile) +// stat(inputfile) + hailanStat(inputfile) + } + + def addHouse(inputfile){ + + def datas = [] + datas << ["ÀàÐÍ(סլ/±ðÊû£©","ÐÐÕþÇø»®","ÏêϸµØÖ·","·¿ºÅ","Ð¡ÇøÃû³Æ","½¨ÖþÃæ»ý","ËùÔÚÂ¥²ã","ÏêϸµØÖ·","Ð¡ÇøÃû","½»Ò×ÈÕ¾ÓסÓ÷¿¼Û¸ñÖ¸Êý","ÆÀ¹ÀÈÕ¾ÓסÓ÷¿¼Û¸ñÖ¸Êý","ÊÇ·ñÓеçÌÝ","ËùÔÚÂ¥²ã","×ÜÂ¥²ã","ÄÚ²¿×°ÐÞ","Æ½Ãæ²¼ÖÃ","³¯Ïò", + "¾°¹Û","½¨Öþ½á¹¹","½¨ÖþÃæ»ý£¨Æ½·½Ã×£©","·¿ÎÝ¿ÉʹÓÃÄêÏÞ","·¿ÎÝÒÑʹÓÃÄêÏÞ","ÆÀ¹À¼Û£¨µ¥¼Û£©","¹ÒÅÆ¼Û£¨µ¥¼Û£©","³É½»¼Û£¨µ¥¼Û£©","Ð¡Çø/Â¥Å̾ù¼Û£¨µ¥¼Û£©"] + def index = 0 + ExcelUtils.multiThreadingModify(inputfile, { row -> + if(row['L']){ + index ++ + def dist = CommonDatautils.findDistrict(row['H'], row['I'], row['J'], "district") + def rets = findHouses(row['H'], row['I'], "sell", 20200101,20230113, row['J'], null, row['k']) + println index +" "+ row['C']+" "+ rets.size() + if(rets){ + def lastRets = [] + for(def r : rets){ + def fjwPrice = row['M'] + if(fjwPrice){ + def precent = java.lang.Math.abs((fjwPrice-r.avg_price)*100/r.avg_price) + if(precent>=10){ + continue; + } + } + if(row['A'] && row['A']=="±ðÊû"){ + def inputAare = row['F'].toString().toDouble() + def minArea = inputAare + 50 + def maxArea = inputAare + 100 + if(r.areamaxArea){ + continue; + } + } + + lastRets << r + if(lastRets.size()>=3){ + break; + } + } + + if(lastRets){ + for(def lr : lastRets){ + def data = [] + data << row['A'] + data << row['B'] + data << row['C'] + data << row['D'] + data << row['E'] + data << row['F'] + data << row['G'] + if(dist){ + data << dist.address + }else{ + data << "" + } + if(dist){ + data << dist.name + }else{ + data << "" + } + + data << "" + data << "" + if(lr.height){ + if(lr.height>6){ + data << "ÊÇ" + }else{ + data << "·ñ" + } + }else{ + data << "" + } + data << lr.floor + data << lr.height + data << lr.fitment + data << "" + data << lr.direction + data << "" + data << lr.build_type + data << lr.area + data << dist.property_life +// def year = dist.create_year +// if(year){ +// data << getHouseYear(year.toInteger())//ÒÑʹÓÃÄêÏÞ +// }else{ +// data << ""//ÒÑʹÓÃÄêÏÞ +// } + data << getHouseYear(dist.create_year , lr.build_date, dist.build_date) + + data << getEvalutaionPrice(lr, dist)//ÆÀ¹À¼Û£¨µ¥¼Û£© + data << lr.avg_price + data << getDealPrice(dist)//³É½»µ¥¼Û + data << row['M'] + datas << data + } + } + } + } + }) + ExcelUtils.write("D:\\Êý¾Ý\\º£ÄÏ\\º£ÄÏÑù±¾Êý¾Ý.xlsx", datas, "Ñù±¾Êý¾Ý") + } + + def getDealPrice(dist){ + def json = OpenApiClient.searchDealPrice(dist._id.toString(),"20220101", 100) + def prices = [] + if(json && json.code==200){ + def result = json.result + if(result){ + for(def r : result){ + prices << r.avgPrice + } + } + } + def dealAvgPrice = null + if(prices){ + dealAvgPrice = (prices.sum()/prices.size()).toInteger() + } + return dealAvgPrice; + } + + def getHouseYear(create_year , house_build_date, dist_build_date){ + def year = null + if(house_build_date){ + year = getYear(house_build_date) + + }else if(dist_build_date){ + year = getYear(dist_build_date) + }else if(create_year){ + year = create_year + } + if(year){ + year = year.toInteger() + def curYear = new Date().getYear()+1900 + def modYear = curYear - year + return modYear.toInteger().toString() + } + return null + + } + + + def getYear(finishBuildingDate){ + if(finishBuildingDate){ + Pattern pat3 = Pattern.compile("\\d{4}"); + def matcher3 = pat3.matcher(finishBuildingDate); + if(matcher3.find()){ + return matcher3.group() + } + } + return null + } + + + def getEvalutaionPrice(house, dist){ + def params = [:] + params.put("city", dist.city); + params.put("district", dist.region); + params.put("name", dist.name); + params.put("size", String.valueOf(house.area)); + if(house.height) { + params.put("maxFloor", dist.house_height.toString()); + params.put("direction", getDirectionCode(house.direction).toString());//³¯Ïò + if (house.house_floor) { + params.put("floor", house.house_floor.toString()); + } + } + def fitmentCode = null + if(house.fitment){ + fitmentCode = getFitmentCode(house.fitment) + params.put("decoration", fitmentCode.toString());//×°ÐÞ + } + params.put("scope", "1") + def result = OpenApiClient.evaluateDistrictPrice(params); + + if(result){ + return result.avgPrice + } + return null + } + + + def getDirectionCode(def s){ + def directMap = ["Î÷ÄÏ":1, "¶«ÄÏ":2, "ÄÏ":3, "Î÷±±":4, "¶«±±":5, "¶«":6, "Î÷":7, "±±":8, "¶«Î÷":9, "Äϱ±":10] + return directMap.get(s); + } + def getFitmentCode(def s){ + if(s){ + if(["ëÅ÷"].contains(s)){ + return 1; + }else if(["¼òµ¥×°ÐÞ","¼ò×°"].contains(s)){ + return 2; + }else if(["¾«×°ÐÞ","¾«×°"].contains(s)){ + return 3; + }else if(["ºÀ»ª×°ÐÞ","ºÀ×°"].contains(s)){ + return 4; + } + } + return null + } + + + + + def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2){ + + QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(false).includeUpper(false); + QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false); + + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat)) + .must(date_range).must(area_range) + + if(!alias){ + alias = [] + } + if(!alias.contains(name)){ + alias.add(name) + } + + + if(alias && alias.size()>0){ + BoolQueryBuilder inBuilder = QueryBuilders.boolQuery(); + for(def s : alias){ + inBuilder.should(QueryBuilders.matchQuery("district_name", s)); + } + queryBuilder.must(inBuilder) + } + + + def rets = [] + def hosts = ["192.168.0.206","192.168.0.211","192.168.0.212","192.168.0.213","192.168.0.214","192.168.0.215"] + def index = new Random().nextInt(5); + + def houses = ElasticUtils.executeSimpleQuery(hosts[index], "seawater-20*", queryBuilder); + + if(houses){ + for (int k=0;k60){ + continue; + } + +// if(!seawater.room){ +// continue; +// } +// if(!seawater.room){ +// seawater.room = null +// } + + if(seawater.room){ + if(seawater.room>=11 || seawater.room<1){ + continue; + } + } + + if(!seawater.hall){ + seawater.hall = null + } + if(seawater.hall && (seawater.hall>=5 || seawater.hall<1)){ + continue; + } + if(!seawater.toilet){ + seawater.toilet = null + } + if(seawater.toilet && (seawater.toilet>=5 || seawater.toilet<1)){ + continue; + } + + + def floor = parseFloor(seawater.floor_info) + if(seawater.height && floor && floor.isNumber()){ + if(floor.toInteger()>seawater.height){ + floor = null + seawater.height = null + } + + } + + if(!floor){ + seawater.floor = null + } + if(floor && floor.toInteger()<1){ + floor = null + } + + def rowMap = [:]; + + rowMap.houseId = seawater.id +// rowMap.city = CcbCommonUtils.getCcbCity(city) + //rowMap.region =seawater.region + //rowMap.address = seawater.address + rowMap.area = seawater.area + rowMap.address = seawater.address + rowMap.total_price = seawater.total_price + rowMap.avg_price = seawater.avg_price + rowMap.build_type = seawater.build_type + rowMap.build_date = seawater.build_date + rowMap.district_name = seawater.district_name + rowMap.height = seawater.height=="null"?null:seawater.height + if(rowMap.height && rowMap.height<1){ + rowMap.height = null + } + rowMap.floor = floor=="null"?null:floor + rowMap.room = seawater.room=="null"?null:seawater.room + rowMap.hall = seawater.hall=="null"?null:seawater.hall + rowMap.toilet = seawater.toilet=="null"?null:seawater.toilet + + + def direction = parseDirection(seawater.direction) + rowMap.direction = direction=="null"?null:direction + + def fitment = parseFitment(seawater.fitment) + rowMap.fitment = fitment=="null"?null:fitment + + + + if(seawater.p_date instanceof String){ + seawater.p_date = seawater.p_date.toInteger() + } + + rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss") + rowMap.c_date = DateUtils.getConverDate(seawater.m_date) + rowMap.m_date = DateUtils.getConverDate(seawater.m_date) + + rets << rowMap + } + } + return rets; + } + + + static def parseFloor(def floorDesc){ + if(floorDesc){ + floorDesc = floorDesc.replaceAll("[\\s ]+", "") + if(floorDesc){ + Pattern pat1 = Pattern.compile("µÚ(\\d{1,2})²ã"); + Matcher matcher1 = pat1.matcher(floorDesc); + if (matcher1.find()) { + return matcher1.group(1) + } + pat1 = Pattern.compile("(\\d{1,2})²ã?(/|\\()¹²?\\d{1,2}(²ã|F)?"); + matcher1 = pat1.matcher(floorDesc); + if (matcher1.find()) { + return matcher1.group(1) + } +// if(floorDesc.contains("ÖÐÂ¥²ã") || floorDesc.contains("Öвã")){ +// return "Öвã" +// } +// +// if(floorDesc.contains("¸ßÂ¥²ã") || floorDesc.contains("¸ß²ã")){ +// return "¸ß²ã" +// } +// +// +// if(floorDesc.contains("µÍÂ¥²ã") || floorDesc.contains("µÍ²ã")){ +// return "µÍ²ã" +// } + if(floorDesc.isNumber()){ + return floorDesc; + } + } + } + return null; + } + + static String parseDirection(String directionDesc) { + if (directionDesc == null) { + return null + } + directionDesc = directionDesc.replaceAll("(\\(|£¨|¡¶|\"|¡°|¡¾|¡²).+?(\\)|¡·|£©|\"|¡±|¡¿|¡³)", "#_#") + StringBuffer sb = new StringBuffer() + for (int i = 0; i < directionDesc.length(); i++) { + String c = directionDesc.substring(i, i + 1) + if (DIRECTIONS.contains(c)) { + if (directionDesc.substring(i).startsWith(c + "#_#")) { + sb.append(c + " ") + } else { + sb.append(c) + } + } + } + for (String d : DIRECTIONS) { + if (sb.toString().startsWith(d)) { + return d + } + } + for (String d : DIRECTIONS) { + if (sb.toString().contains(d)) { + return d + } + } + return null + + } + + static String parseFitment(String fitmentDesc) { + if (fitmentDesc == null) { + return null + } + fitmentDesc = fitmentDesc.replaceAll("[\\s ]+", "") + if (fitmentDesc.equals("·ÇëÅ÷")) { + return null + } + Iterator>> it = FITMENTS_TYPES_MAPS + .entrySet().iterator() + Map.Entry> entry + while (it.hasNext()) { + entry = it.next() + if (fitmentDesc.contains(entry.getKey())) { + return entry.getKey() + } + for (String fitmentKey : entry.getValue()) { + if (fitmentDesc.contains(fitmentKey)) { + return entry.getKey() + } + } + } + return null + + } + + + + def addCommId(inputfile){ + ExcelUtils.modify(inputfile, { row -> + if(row['H']){ + def dist = CommonDatautils.findDistrict(row['H'], row['I'], row['J'], "district") + if(dist){ + row['L'] = dist._id.toString() + } + } + }) + } + + def addPrice(path){ + ExcelUtils.multiThreadingModify(path, { row -> + def commid = row['L'] + if(commid){ + def dist = CommonDatautils.findDistrictById(commid.trim()) + if(dist){ + def priceObj = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name) + if(priceObj){ + row['M'] = priceObj.avg_price + } + } + } + }) + } + + + def match(path){ + ExcelUtils.multiThreadingModify(path, { row -> + if(row['C']){ + String city = row['H']//RegionData.formatCity(row['B']) + if(city){ + def address = row['C'] +// def region = row['C'] + println "[match] city:"+city+" address:"+address + def jsonObject = OpenApiClient.addressMatch(city, address, "property", false, null, row['E']) + if(jsonObject != null && jsonObject.precise!=-1) { + row['I'] = jsonObject.get("district") + row['J'] = jsonObject.get("name") + row['K'] = jsonObject.get("propertyType") + row['L'] = jsonObject.get("id") + } + }else{ + println row['C'] + } + + } + }) + } + + def stat(path){ + ExcelUtils.modify(path, { row -> + if(row['L'] && !row['H']){ + def city = row['L'] + def region = row['M'] + def dists = null + if(city && region){ + dists = CommonDatautils.findDistrictByCityAndRegions(city, "district", [region], ["city","region","name","alias","estate_type2"]) + }else{ + dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name","alias","estate_type2"]) + } + def distLists = [] + def villaLists = [] + for(def d : dists){ + if(d.estate_type2=="±ðÊû"){ + villaLists << d + }else{ + distLists << d + } + } + row['J'] = dists.size() + + def haveHouseList = findHaveHouseDist(distLists) + def haveVillaList = findHaveHouseDist(villaLists) + row['H'] = haveHouseList + row['I'] = haveVillaList +// row['N'] = distLists.size() +// row['O'] = villaLists.size() + } + }) + } + + def hailanStat(path){ + def total = 0 + def distTotal = 0 + def villaTotal = 0 + def haveHouseDistTotal = 0 + def haveHouseVillaTotal = 0 + def buildStructureTotal = 0 +// ExcelUtils.modify(path, { row -> +// if (row['L'] && !row['K']) { + //"º£¿Ú","ÈýÑÇ","Èýɳ","ÙÙÖÝ", + def citys = ["Îåָɽ","Çíº£","ÎIJý","ÍòÄþ","¶«·½","¶¨°²","ÍͲý","³ÎÂõ","ÁÙ¸ß","°×ɳ","²ý½­","ÀÖ¶«","ÁêË®","±£Í¤","ÇíÖÐ"] + for(def city : citys) { + + def dists = null +// if (city && region) { +// dists = CommonDatautils.findDistrictByCityAndRegions(city, "district", [region], ["city", "region", "name", "alias", "estate_type2", "build_structure"]) +// } else { + dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias", "estate_type2", "build_structure"]) +// } + def distLists = [] + def villaLists = [] + for (def d : dists) { + if (d.estate_type2 == "±ðÊû") { + villaLists << d + } else { + distLists << d + } + } + total = total + dists.size() + +// def structureCount = 0 +// for(def d : dists){ +// if(d.build_structure){ +// structureCount ++ +// } +// } +// println structureCount +// row['K'] = structureCount +// buildStructureTotal = buildStructureTotal + structureCount +// + def haveHouseList = findHaveHouseDist(distLists) + def haveVillaList = findHaveHouseDist(villaLists) + haveHouseDistTotal = haveHouseDistTotal + haveHouseList + haveHouseVillaTotal = haveHouseVillaTotal + haveVillaList + distTotal = distTotal + distLists.size() + villaTotal = villaTotal + villaLists.size() +// println c +" "+total+" "+buildStructureTotal +// } + } +// }) + + println total +" "+haveHouseDistTotal+" "+haveHouseVillaTotal+" "+distTotal+" "+villaTotal+" "+buildStructureTotal + + } + + def findHaveHouseDist(districts){ + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(districts.size()) + println "Ð¡ÇøÁ¿£º"+districts.size() + + def rets = new CopyOnWriteArrayList() + + for(def i=0;i0){ + rets.add(dist) + } + println totalCount.get() +" "+ ret.size() + } catch (Exception e) { + e.printStackTrace() + } finally { + latch.countDown() + } + }; + }) + } + latch.await() + return rets.size() + } + + + def findThirdPrice(dist){ + def q = new BasicDBObject() + q.put("city", dist.city) + q.put("region", dist.region) + q.put("name", dist.name) + q.put("s_date", new BasicDBObject('$gte', 202201)) + def dbcursor = AVERAGE_STATISTICS.find(q) + def prices = [] + if(dbcursor){ + for(def d : dbcursor){ + def set = d.keySet(); + Iterator it = set.iterator() + while(it.hasNext()){ + def key = it.next() + if(key && key.contains("_price") && key!="fangjia_price" && key!="last_price"){ + def price = d[key] + if(price){ + prices << price + } + } + } + } + } + return prices; + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MingShenPocStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MingShenPocStat.groovy new file mode 100644 index 000000000..d1e0db444 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MingShenPocStat.groovy @@ -0,0 +1,328 @@ +package com.fangjia.script.chongqing + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.rabbitmq.client.impl.AMQImpl +import org.apache.lucene.queryparser.classic.QueryParser +import org.elasticsearch.index.query.BoolQueryBuilder +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + + +class MingShenPocStat { + + static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40) + + private static final DBCollection district_sell_stat_coll = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com") + final static DBCollection CITY_INFO_CONN = MongoUtils.getCollectionByName("fangjia_data_ana", "city_info", MongoUtils.MongoDB_Domain_mongo); + + static main(args) { + def task = new MingShenPocStat() +// task.run() + task.officeRun() + } + + def officeRun(){ + def path = "D:\\Êý¾Ý\\ÃñÉú\\³éÑù-д×ÖÂ¥Àà.xlsx" + def datas = [] + ExcelUtils.modify(path, { d -> + if(d['B']){ + + def office = CommonDatautils.findDistrictById(d['C']) + if(office){ + def alias = [] + alias << office.name + if(office.alias){ + def nameAlias = office.alias + for(def o : nameAlias){ + if(o){ + alias << o + } + } + } + def aliasStr = escapeExprSpecialWord(alias.join("','")) + def houses = findOfficeHouseCount(office.city, office.region, aliasStr) + if(houses){ + if(houses.size()>=2){ + houses = houses[0..1] + } + for(def h : houses){ + def obj = findOfficeAvgPrice(office) + def rets = [] + rets << d['A'] + rets << d['B'] + rets << '' + rets << h.region + rets << office.address + rets << h.area + rets << h.floor + rets << h.height + def level = office.bed_kind + rets << level + rets << h.fitment + def date = office.finish_building_date + if(date){ + rets << date.substring(0, 4) + }else{ + rets << "" + } + + rets << "" + rets << "" + def price = "" + if(obj && obj.avg_price){ + price = obj.avg_price + rets << obj.avg_price + }else{ + rets << "" + } + + if(price){ + rets << (price*h.area).toInteger() + } + datas << rets + } + } + }else{ + def rets = [] + rets << d['A'] + rets << d['B'] + datas << rets + } + + // def districts = CommonDatautils.findAllDistrictByCity(d['C'],"office",["city", "region", "name", "alias","address","bed_kind","finish_building_date"]) + + } + }) + + ExcelUtils.write("D:\\Êý¾Ý\\ÃñÉú\\³éÑù-д×ÖÂ¥Àà-Ñù±¾.xlsx", datas, "Ñù±¾") + } + + def run(){ + def path = "D:\\Êý¾Ý\\ÃñÉú\\Ñù±¾×ֶθ²¸ÇÂÊ.xlsx" +// def datas = ExcelUtils.read("C:\\Users\\gaojuan\\Downloads\\¶þÆÚ¹ÀÖµÊý¾ÝÿÖÜͳ¼Æ\\¶þÆÚ¹ÀÖµÊý¾ÝÿÖÜͳ¼Æ\\¹ÀÖµÊý¾Ýͳ¼Æ½á¹û20230112.xlsx", 0 , 1) +// +// ExcelUtils.modify(path, { row -> +// if(row['C']) { +// def total = 0 +// def pre = 0 +// for(def d : datas){ +// if(row['C']==d['A']){ +// total = d['B'] +// pre = d['H'] +// break +// } +// } +// if(total){ +// def c = (total*pre/100).toInteger() +// row['E'] = c +// row['F'] = pre+"%" +// } +// } +// }) + + def totalcount = findHouseTotal("´óÁ¬") + def count = findHouseCount("´óÁ¬", "height") + + println DataHandleUtils.getDoubleRound(2, count*100/totalcount) + +// ExcelUtils.multiThreadingModify(path, { row -> +// if(row['C']) { + //def districts = CommonDatautils.findAllDistrictByCity(row['C'],"district",["city", "region", "name", "alias"]) +// def districts = CommonDatautils.findDistrictByCityAndRegions("´óÁ¬","district",["ÂÃ˳¿Ú"], ["city", "region", "name", "alias"]) +// def total = districts.size() +// if(total){ +// def house_dist_count = findHouseStat(districts) +// def pre = house_dist_count*100/total +// pre = DataHandleUtils.getDoubleRound(2, pre) +// println total+" "+house_dist_count+" "+pre +// } +// def count = findHouseCount(row['C']) +// row['D'] = count +// println row['C']+" "+count +// row['H'] = findDistrictSellStat(row['C']) +// row['G'] = getNewDist(row['C']) + +// def totalcount = findHouseTotal(row['C']) + +// def total = row['Z'] +// def count = findHouseCount(row['C'], "build_date") +// row['Q'] = DataHandleUtils.getDoubleRound(2, count*100/total) +// println count +" "+row['Q'] +// def obj = getFieldPrecent(row['C']) +// if(obj){ +// def detail = obj.district_detail +// row['E'] = detail.alias.toDouble()*100 +// row['F'] = detail.address.toDouble()*100 +// row['G'] = detail.lng.toDouble()*100 +// row['H'] = detail.buildType.toDouble()*100 +// row['I'] = detail.outPics.toDouble()*100 +// row['J'] = detail.developer.toDouble()*100 +// row['Q'] = detail.buildDate.toDouble()*100 +// row['R'] = detail.buildDate.toDouble()*100 +// } +// } +// }); + } + + def getFieldPrecent(city){ + def q = new BasicDBObject() + q.put("city", city) + def rets = CITY_INFO_CONN.find(q).sort(new BasicDBObject('weekend', -1)) + if(rets){ + return rets[0] + } + return null + } + + + def getNewDist(city){ + String sql = "select count(*) as count from community where city='${city}' and source='±´¿ÇÕÒ·¿-з¿' " + def count = null + ElasticUtils.executeSqlQuery(sql,{ data-> + count = data.count + }) + return count; + } + + def findDistrictSellStat(def city){ + def q = new BasicDBObject() + q.put("city", city) + def count = district_sell_stat_coll.count(q) + return count + } + + + def findHouseStat(districts){ + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(districts.size()) + println "Ð¡ÇøÁ¿£º"+districts.size() + + def rets = new CopyOnWriteArrayList() + + for(def i=0;i0){ + rets.add(rMap) + } + println totalCount.get() +" "+ ret.size() + } catch (Exception e) { + e.printStackTrace() + } finally { + latch.countDown() + } + }; + }) + } + latch.await() + return rets.size() + } + + def findHouseTotal(city){ + def count = 0 + ElasticUtils.executeSqlQuery("select count(*) as count from seawater where p_date >= 20220101 and city='${city}'", { + count = it.count + println("$city\t${it.count}") + }) + return count + } + + def findHouseCount(city, field){ + def count = 0 + def sql = "select count(*) as count from seawater where city='${city}' and cat='sell' and "+field+" is not null" + ElasticUtils.executeSqlQuery(sql, { + count = it.count + println("$city\t${it.count}") + }) + return count + } + + def findHouses(def city, def region, def cat, def startDate, def name, def alias){ + + QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).includeLower(false).includeUpper(false); + QueryBuilder area_range = QueryBuilders.rangeQuery("area").includeLower(true).includeUpper(false); + + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat)) + .must(date_range).must(area_range) + + if(!alias){ + alias = [] + } + if(!alias.contains(name)){ + alias.add(name) + } + + + if(name.contains("ÅªÐ¡Çø")){ + def newName = name.replace("ÅªÐ¡Çø","Ū") + if(!alias.contains(newName)){ + alias << newName + } + + } + + if(alias && alias.size()>0){ + BoolQueryBuilder inBuilder = QueryBuilders.boolQuery(); + for(def s : alias){ + inBuilder.should(QueryBuilders.matchQuery("district_name", s)); + } + queryBuilder.must(inBuilder) + } + + + def rets = [] + def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder); + return houses; + } + + def findOfficeAvgPrice(office){ + def sql = "select * from office where city='${office.city}' and region='${office.region}' and name='${office.name}'" + def rets = [] + ElasticUtils.executeSqlQuery(sql, {d-> + rets << d + }) + if(rets){ + def obj = rets[0] + return obj + } + return null + + } + + def findOfficeHouseCount(city, region, names){ + def sql = "select * from seawater where city='${city}' and region='${region}' and cat='office_sell' and district_name in ('${names}') and root='58.com'" + def rets = [] + ElasticUtils.executeSqlQuery(sql, {d-> + rets << d + }) + return rets + } + + + /** + * תÒå×Ö·û´®ÖеÄÌØÊâ×Ö·û + * ½ö¹ýÂËLucene regexps, ÆäËûµÄÎðÓà + * @param keyword + * @return + */ + public static String escapeExprSpecialWord(String keyword) { + return QueryParser.escape(keyword); + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MingShenPocTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MingShenPocTest.groovy new file mode 100644 index 000000000..112de372f --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MingShenPocTest.groovy @@ -0,0 +1,459 @@ +package com.fangjia.script.chongqing + +import com.fangjia.core.RegionData +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.util.BaiduMapUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.GeoHelper +import com.fangjia.script.util.OpenApiClient +import com.mongodb.BasicDBObject +import com.mongodb.DBObject +import org.elasticsearch.index.query.BoolQueryBuilder +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders + +class MingShenPocTest { + + def total = 5000 + static main(args) { + def task = new MingShenPocTest() + task.run() + } + def run(){ + def path = "D:\\Êý¾Ý\\ÃñÉú\\poc²âÊÔ.xlsx" + //9¼¶±ê×¼»¯ + //structureV2(path)//Õë¶Ôº£ÄÏpocµØÖ·Êý¾Ý´Ë½Ó¿Ú²»ºÏÊÊʹÓà + //Ê¡ÊÐÇø²ð·Ö +// ner(path)//³ÇÊУ¬ÇøÓò´øÊдøÇø +// match(path) +// addPrice(path) +// addLatLng(path) +// addAroundPrice(path) +// addAuthor(path) + addHouse(path) + } + + def structureV2(path){ + ExcelUtils.multiThreadingModify(path, { row -> + if(row['A']){ + def jsonObj = FangjiaClientUtils.addressStructure(null, row['A']) + if(jsonObj && jsonObj.result){ + + if(jsonObj.result.province){ + row['B'] = jsonObj.result.province + } + + if(jsonObj.result.city){ + def cityStr = null + if(jsonObj.result.city=="ÊÐÏ½Çø"){ + cityStr = jsonObj.result.province + }else{ + cityStr = jsonObj.result.city + } + row['C'] = RegionData.formatCity(cityStr) + } + + if(jsonObj.result.district){ + row['D'] = jsonObj.result.district + } + } + } + }) + } + + def ner(path){ + ExcelUtils.multiThreadingModify(path, { d -> + if(d['A'] && !d['B']) { + def json = FangjiaClientUtils.addressStand(null, d['A']) + if(json && json.entities){ + def entities = json.entities + for(def m : entities){ + if(m.type=="prov"){ + d['B'] = m.word + } + if(m.type=="city"){ + def cityStr = m.word + if(cityStr){ + d['C'] = cityStr//RegionData.formatCity(cityStr) + } + } + if(m.type=="dist") { + if(d['C']){ + d['D'] = m.word //RegionData.formatRegion(d['C'], m.word) + } + } + } + } + } + }); + } + + //¿ÉÒÔ¸ù¾Ý²âÊÔÊý¾ÝÌí¼Ó ÍÆ¼öÂ¥ÅÌÃû + def match(path){ + ExcelUtils.multiThreadingModify(path, { row -> + if(row['C'] && !row['E']){ + def city = RegionData.formatCity(row['C']) + def region = null + if(city){ + region = RegionData.formatRegion(city, row['D']) + } + def address = row['A'] + println "[match] city:"+city+" region:"+region+" address:"+address + def jsonObject = OpenApiClient.addressMatch(city, address, null, false, region, null)//"property" + if(jsonObject != null && jsonObject.precise!=-1) { + row['E'] = jsonObject.get("city") + row['F'] = jsonObject.get("district") + row['G'] = jsonObject.get("name") + row['H'] = jsonObject.get("propertyType") + row['I'] = jsonObject.get("id") + } + } + }) + } + + def addPrice(path){ + ExcelUtils.multiThreadingModify(path, { row -> + def commid = row['I'] + if(commid && !row['J']){ + def dist = CommonDatautils.findDistrictById(commid.trim()) + if(dist){ + def house = [:] + house.area = 100 + def price = getEvalutaionPrice(house, dist) + if(price){ + row['J'] = price + } +// def priceObj = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name) +// if(priceObj){ +// row['J'] = priceObj.avg_price +// } + } + } + }) + } + + def getEvalutaionPrice(house, dist){ + def params = [:] + params.put("city", dist.city); + params.put("district", dist.region); + params.put("name", dist.name); + params.put("size", String.valueOf(house.area)); + if(house.height) { + params.put("maxFloor", dist.house_height.toString()); + params.put("direction", getDirectionCode(house.direction).toString());//³¯Ïò + if (house.house_floor) { + params.put("floor", house.house_floor.toString()); + } + } + def fitmentCode = null + if(house.fitment){ + fitmentCode = getFitmentCode(house.fitment) + params.put("decoration", fitmentCode.toString());//×°ÐÞ + } + params.put("scope", "1") + def result = OpenApiClient.evaluateDistrictPrice(params); + + if(result){ + return result.avgPrice + } + return null + } + + + def getDirectionCode(def s){ + def directMap = ["Î÷ÄÏ":1, "¶«ÄÏ":2, "ÄÏ":3, "Î÷±±":4, "¶«±±":5, "¶«":6, "Î÷":7, "±±":8, "¶«Î÷":9, "Äϱ±":10] + return directMap.get(s); + } + def getFitmentCode(def s){ + if(s){ + if(["ëÅ÷"].contains(s)){ + return 1; + }else if(["¼òµ¥×°ÐÞ","¼ò×°"].contains(s)){ + return 2; + }else if(["¾«×°ÐÞ","¾«×°"].contains(s)){ + return 3; + }else if(["ºÀ»ª×°ÐÞ","ºÀ×°"].contains(s)){ + return 4; + } + } + return null + } + + def addLatLng(path){ + ExcelUtils.multiThreadingModify(path, { d -> + if(!d['J'] && d['H'] && (d['H']=="ÆÕͨסլ" || d['H']=="ÉÌס" || d['H']=="±ðÊû")) { + def json = BaiduMapUtils.searchByAddress(d['A']) + if(json && json.location){ + d['K'] = json.location.lng + d['L'] = json.location.lat + } + } + }) + } + + + def addAroundPrice(path){ + ExcelUtils.multiThreadingModify(path, { d -> + if(!d['J'] && d['C'] && d['K']){ + def city = RegionData.formatCity(d['C']) + def dist = new BasicDBObject() + dist.city = city + dist.lng = d['K'].toString().toDouble()*10000000000L + dist.lat = d['L'].toString().toDouble()*10000000000L + dist.estateType = "ÆÕͨסլ"//d['H'] + if(dist.lat) { + List list = DbUtils.findAroundDistrictByRadius(dist, 1000) + if (list) { + list = sortByDistance(dist, list); + if (list && list.size() > 10) { + list = list[0..9] + } + if (list) { + def prices = [] + for (def s : list) { + def priceObj = CommonDatautils.findDistrictPrice(s.city, s.region, s.name) + if (priceObj) { + prices << priceObj.avg_price + } + } + if (prices) { + def arround_price = (int) (prices.sum() / prices.size()) + d['J'] = arround_price + } + } + } + } + } + }); + } + + def sortByDistance(d, list){ + for(def nb : list){ + def distance = GeoHelper.GetDistance(d.lat/10000000000.0, d.lng/10000000000.0, nb.lat/10000000000.0, nb.lng/10000000000.0) + nb.distance = distance + } + //ÅÅÐò + Collections.sort(list, new Comparator() { + @Override + int compare(Object o1, Object o2) { + if(o1.distance > o2.distance) + return 1;//µÚ¶þ¸öÔªËØ£¨o1£©±ÈµÚÒ»¸öÔªËØ£¨o2£©´ó£¬·µ»Ø-1 + if(o1.distance == o2.distance) + return 0; + return -1; + } + }) + return list; + } + + + def addAuthor(def path){ + def authors = ["¶­Ñå¾ù","²ÜÓñÑå","¸¶¹óÉú","ÉòÑà·¼","Catherine", "ÍõÀÙ","¼ªçë", "Õű¦½£"] + def personTotal = authors.size() + def avg = Math.ceil(total/personTotal) + + def index = 0 + def count = 0; + ExcelUtils.modify(path, { row -> + if(row['A']){ + if(index < avg){ + row['M'] = authors[count] + } + index++ + if(index >= avg){ + count++ + index = 0 + } + } + }); + } + + def addHouse(path){ + def datas = [] + ExcelUtils.multiThreadingModify(path, { row -> + def commId= row['E'] + if(commId){ + def dist = CommonDatautils.findDistrictById(commId) + if(dist){ + def rets = findHouses(row['H'], row['I'], "sell", 20200101,20230113, row['J'], null, row['k']) + if(rets){ + def data = [] + for(def r : rets){ + data << r.city + data << r.region + data << r.district_name + data << r.floor + data << r.height + data << r.fitment + data << r.direction + data << r.build_type + data << r.area + data << r.avg_price + datas << data + } + } + } + } + }); + + ExcelUtils.write("D:\\Êý¾Ý\\ÃñÉú\\ÃñÉúÑù±¾Êý¾Ý.xlsx", datas, "Ñù±¾Êý¾Ý") + } + + def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2){ + + QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(false).includeUpper(false); + QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false); + + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat)) + .must(date_range).must(area_range) + + if(!alias){ + alias = [] + } + if(!alias.contains(name)){ + alias.add(name) + } + + + if(alias && alias.size()>0){ + BoolQueryBuilder inBuilder = QueryBuilders.boolQuery(); + for(def s : alias){ + inBuilder.should(QueryBuilders.matchQuery("district_name", s)); + } + queryBuilder.must(inBuilder) + } + + + def rets = [] + def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder); + + if(houses){ + for (int k=0;k60){ + continue; + } + +// if(!seawater.room){ +// continue; +// } +// if(!seawater.room){ +// seawater.room = null +// } + + if(seawater.room){ + if(seawater.room>=11 || seawater.room<1){ + continue; + } + } + + if(!seawater.hall){ + seawater.hall = null + } + if(seawater.hall && (seawater.hall>=5 || seawater.hall<1)){ + continue; + } + if(!seawater.toilet){ + seawater.toilet = null + } + if(seawater.toilet && (seawater.toilet>=5 || seawater.toilet<1)){ + continue; + } + + + def floor = parseFloor(seawater.floor_info) + if(seawater.height && floor && floor.isNumber()){ + if(floor.toInteger()>seawater.height){ + floor = null + seawater.height = null + } + + } + + if(!floor){ + seawater.floor = null + } + if(floor && floor.toInteger()<1){ + floor = null + } + + def rowMap = [:]; + + rowMap.houseId = seawater.id +// rowMap.city = CcbCommonUtils.getCcbCity(city) + //rowMap.region =seawater.region + //rowMap.address = seawater.address + rowMap.area = seawater.area + rowMap.address = seawater.address + rowMap.total_price = seawater.total_price + rowMap.avg_price = seawater.avg_price + rowMap.build_type = seawater.build_type + rowMap.build_date = seawater.build_date + rowMap.district_name = seawater.district_name + rowMap.height = seawater.height=="null"?null:seawater.height + if(rowMap.height && rowMap.height<1){ + rowMap.height = null + } + rowMap.floor = floor=="null"?null:floor + rowMap.room = seawater.room=="null"?null:seawater.room + rowMap.hall = seawater.hall=="null"?null:seawater.hall + rowMap.toilet = seawater.toilet=="null"?null:seawater.toilet + + + def direction = parseDirection(seawater.direction) + rowMap.direction = direction=="null"?null:direction + + def fitment = parseFitment(seawater.fitment) + rowMap.fitment = fitment=="null"?null:fitment + + + + if(seawater.p_date instanceof String){ + seawater.p_date = seawater.p_date.toInteger() + } + + rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss") + rowMap.c_date = DateUtils.getConverDate(seawater.m_date) + rowMap.m_date = DateUtils.getConverDate(seawater.m_date) + + rets << rowMap + } + } + return rets; + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/ThreeDESUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/ThreeDESUtils.groovy new file mode 100644 index 000000000..3d685a1a5 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/ThreeDESUtils.groovy @@ -0,0 +1,105 @@ +package com.fangjia.script.chongqing + +import com.fangjia.script.util.ExcelUtils +import org.apache.commons.io.FileUtils + +import javax.crypto.Cipher; +import javax.crypto.SecretKey; +import javax.crypto.spec.SecretKeySpec; +import java.util.Base64; + +/** + * 3DES¼ÓÃܹ¤¾ßÀà + * @author qzz + */ +public class ThreeDESUtils { + + /** + * ¼Ó½âÃÜͳһ±àÂ뷽ʽ + */ + private final static String ENCODING = "utf-8"; + + /** + * ¼Ó½âÃÜ·½Ê½ + */ + private final static String ALGORITHM = "DESede"; + + /** + *¼ÓÃÜģʽ¼°Ìî³ä·½Ê½ + */ + private final static String PATTERN = "DESede/ECB/pkcs5padding"; + + /** + * 3DES¼ÓÃÜ + * + * @param plainText ÆÕͨÎı¾ + * @param sK ÃØÔ¿£¨24λÃÜÂ룩 + * @return + * @throws Exception + */ + public static String encode(String plainText,String sK) throws Exception { + SecretKey secretKey = new SecretKeySpec(sK.getBytes(ENCODING), ALGORITHM); + // 3DES¼ÓÃܲÉÓÃpkcs5paddingÌî³ä + Cipher cipher = Cipher.getInstance(PATTERN); + // ÓÃÃܳ׳õʼ»¯Cipher¶ÔÏó + cipher.init(Cipher.ENCRYPT_MODE, secretKey); + // Ö´ÐмÓÃܲÙ×÷ + byte[] encryptData = cipher.doFinal(plainText.getBytes(ENCODING)); + return Base64.getEncoder().encodeToString(encryptData); + } + + /** + * 3DES½âÃÜ + * + * @param encryptText ¼ÓÃÜÎı¾ + * @return + * @throws Exception + */ + public static String decode(String encryptText, String sK) throws Exception { + SecretKey secretKey = new SecretKeySpec(sK.getBytes(ENCODING), ALGORITHM); + // 3DES¼ÓÃܲÉÓÃpkcs5paddingÌî³ä + Cipher cipher = Cipher.getInstance(PATTERN); + + // ÓÃÃܳ׳õʼ»¯Cipher¶ÔÏó + cipher.init(Cipher.DECRYPT_MODE, secretKey); + // ÕýʽִÐнâÃܲÙ×÷ + byte[] decryptData = cipher.doFinal(Base64.getDecoder().decode(encryptText)); + return new String(decryptData, ENCODING); + } + + public static void main(String[] args) throws Exception { + + def rets = [] + def datas = readTxt("D:\\Êý¾Ý\\ÃñÉú\\³éÑù-סլÀࣨÆÕͨסլ¡¢¹«Ô¢£©.txt") + def index = 0 + for(def d : datas){ + index++ + def row = [] + def str = decode(d,"111036369260679051122113") + println str + def id = str.substring(str.length()-4) + row << id + row << str.substring(0, str.indexOf(id)) + rets << row + } + ExcelUtils.write("D:\\Êý¾Ý\\ÃñÉú\\ÃñÉúÒøÐвâÊÔÊý¾Ý.xlsx", rets, "³éÑù-סլÀࣨÆÕͨסլ¡¢¹«Ô¢£©") + +// ExcelUtils.multiThreadingModify("D:\\Êý¾Ý\\ÃñÉú\\¼ÓÃÜÊý¾Ý.xlsx", { row -> +// //¼ÓÃÜ +// String str = row['A'] +// //½âÃÜ +// row['B'] = (decode(str,"111036369260679051122113")); +// }); + + } + + public static def readTxt(def file){ + def lines = FileUtils.readLines(new File(file)) + def ret = []; + for(line in lines) { + ret << line; + } + return ret; + } +} + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy index 088626285..1b604d376 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy @@ -18,6 +18,7 @@ import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService import java.util.concurrent.Executors import java.util.concurrent.atomic.AtomicLong +import java.util.regex.Pattern //Ð¡ÇøÔ¶Ⱦù¼Û±í£¨´øÕǵø·ù¿ØÖÆ£© @BatchOwner(Watcher.GaoJuan) @@ -78,6 +79,7 @@ class DistrictSellStatBatch extends BaseBatch { }else{ districts = DbUtils.seaweed.find(new BasicDBObject("city", city).append("cat", "district").append("visible", 0), keys).batchSize(2000) } + print city +" "+districts.size() def weekendPriceMap = findPriceMap(city, lastMonthWeekend)//ÉϸöÔÂÐ¡Çø¾ù¼Û def curMonthlyPriceMap = findMonthlyPrice(city, curMonth)//µ±ÔÂÐ¡ÇøÔ¾ù¼Û diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy index 7e4a0bd2c..44e4dd54c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy @@ -14,7 +14,7 @@ import com.mongodb.DBCollection //¸£ÖݵÚÈý·½¾ù¼Ûͳ¼Æ,·¢ËͶ¤¶¤ÏûÏ¢ //def datas = ExcelUtils.read("d:\\Êý¾Ý\\¸£ÖÝ\\¸£ÖÝ4ÔÂ2.xlsx",0,1) //def datas = CsvUtils.read("d:\\Êý¾Ý\\¸£ÖÝ\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv"); -def datas = readTxt("d:\\Êý¾Ý\\¸£ÖÝ\\FJW_THIRD_PARTY_AVG_PRICE_202211.csv") +def datas = readTxt("d:\\Êý¾Ý\\¸£ÖÝ\\FJW_THIRD_PARTY_AVG_PRICE_202301.csv") def count0 = 0 def count1 = 0 def count2 = 0 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddAvgpriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddAvgpriceBatch.groovy index 577aac71d..7a531b0e7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddAvgpriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddAvgpriceBatch.groovy @@ -51,6 +51,7 @@ class AddAvgpriceBatch { for(def disKey : res2){ def array = disKey.split("##"); def estateType2 = districtKeysMap.get(disKey) + //·¢ÖØËã¾ù¼ÛÃüÁî CommonUtils.recalculationAverage(array[0], array[1], array[2], estateType2, 0, new Date()) } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ShLongShangHangLocation.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ShLongShangHangLocation.groovy new file mode 100644 index 000000000..4cc9588c1 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ShLongShangHangLocation.groovy @@ -0,0 +1,52 @@ +package com.fangjia.script.gj + +import com.fangjia.core.util.GeoUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils + +class ShLongShangHangLocation { + static main(args) { + def task = new ShLongShangHangLocation() + task.run() + } + + def run(){ + def rets = ["ÉϺ£ÊлÆÐË·1599ºÅ","ÉϺ£ÊÐÑîÆÖÇø³¤Ñô·1687ºÅA¶°","ÉϺ£ÊÐÑîÆÖÇø¼Ñľ˼·432ºÅ","ÉϺ£ÊÐÑîÆÖÇø¿Ø½­Â·1063ºÅ","ÉϺ£ÊÐÑîÆÖÇø°°É½Â·100ºÅ","ÉϺ£ÊÐÑîÆÖÇøÑÓ¼ª¶«Â·287ºÅ","ÉϺ£ÊÐÑîÆÖÇøÒóÐз408ºÅ","ÉϺ£ÊÐÑîÆÖÇøÕ¢Òó·1555ºÅ102ÊÒ","ÉϺ£ÊÐÑîÆÖÇøÆ½Á¹Â·1500ºÅ"] + def map = [:] + for(def r : rets){ + def location = getLocation(r) + if(location){ + map.put(r, location) + } + } + def titleMap = [0:'C',1:'D',2:'E',3:'F',4:'G',5:'H',6:'I',7:'J',8:'K'] + ExcelUtils.modify("D:\\Êý¾Ý\\ÉϺ£Å©ÉÌÐÐ\\ÍøµãÓë¾Óί·Ö²¼20230118.xlsx",{row-> + if(row['B'] && !row['B'].contains("µØÖ·") && row['B'].contains('ÆëÆë¹þ¶û·')){ + for(def i=0;i OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log") -def startDate = DateUtil.parseDate("20221101", "yyyyMMdd") -def endDate = DateUtil.parseDate("20221201", "yyyyMMdd") +def startDate = DateUtil.parseDate("20230101", "yyyyMMdd") +def endDate = DateUtil.parseDate("20230201", "yyyyMMdd") Calendar calendar = Calendar.getInstance() calendar.setTime(startDate) def count = 0 @@ -38,6 +38,6 @@ while (startDate.before(endDate)) { startDate = nextDate } -ExcelUtils.write("D:\\Êý¾Ý\\¶«Ý¸\\¶«Ý¸ÒøÐÐ×ÜÐÐ.xlsx", data, "202211") +ExcelUtils.write("D:\\Êý¾Ý\\¶«Ý¸\\¶«Ý¸ÒøÐÐ×ÜÐÐ(²»È¥ÖØ).xlsx", data, "202301") From 72d565c7606fa940da4f11508b85c7c4091316c4 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 17 Feb 2023 15:12:07 +0800 Subject: [PATCH 77/99] update --- .../BigdataCommunityBatch.groovy | 3 +- .../ccbevaluation/BigdataForsaleBatch.groovy | 48 +++++++++++++++++-- .../ccbevaluation/StagingDistrictBatch.groovy | 37 ++++++++++++++ 3 files changed, 81 insertions(+), 7 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy index 123ada0cf..0177411a5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -41,7 +41,6 @@ class BigdataCommunityBatch extends BaseBatch{ @Override public void process(Object args) { def citys = CcbCommonUtils.getRunningCitys(args) - citys = ["ÉϺ£"] def rows = [] def headers = []//["³ÇÊÐID","ÐÐÕþÇøID","Ð¡ÇøID","ÐÐÕþÇøÃû³Æ","Ð¡ÇøÃû³Æ","ÏêϸµØÖ·","γ¶È","¾­¶È","Õ¼µØ×ÜÃæ»ý","½¨Öþ×ÜÃæ»ý","ÂÌ»¯ÂÊ","ÈÝ»ýÂÊ","³µÎ»ÊýÁ¿","½¨ÖþÄê´ú","·¿ÎÝÓÃ;", @@ -251,7 +250,7 @@ class BigdataCommunityBatch extends BaseBatch{ sqlInstance.withBatch(20000, sql){ ps-> for(def r : rows) { ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20], - r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], null, '2023-02-28', null, '2999-12-31', null, null, null) + r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38]) ps.toString() } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index 7eac9b9f7..f90b1cfdc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -57,9 +57,13 @@ class BigdataForsaleBatch extends BaseBatch{ public static final String[] TAGS = ["¶À¶°", "ÁªÅÅ", "µþ¼Ó", "˫ƴ", "»¨Ô°Ãæ»ý", "Ó¾³Ø", "±ðÊû", "´ø»¨Ô°", "Ñó·¿", "µþÆ´", "¶ÀÔº", "ÁªÌå", "΢Êû", "´óƽÊû"]; public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(60) + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/postgres?stringtype=unspecified", + "postgres", "postgres", "org.postgresql.Driver") + static main(args) { runBatch(new BigdataForsaleBatch(), args); } @@ -69,6 +73,7 @@ class BigdataForsaleBatch extends BaseBatch{ //Ŀǰ²éѯ¿âÖÐÊÇ·ñÒÑ´æÔÚ£¬ÓõÄÊÇOracle def index = 0 def citys = CcbCommonUtils.getRunningCitys(args) + citys = ["¸£ÖÝ"] def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() for(def city : citys){ index++ @@ -181,7 +186,9 @@ class BigdataForsaleBatch extends BaseBatch{ // headers = ["³ÇÊÐID","³öÊÛID","¹ÒÅÆ·¿ÎÝID","³ÇÊÐÃû³Æ","ÐÐÕþÇøID","ÐÐÕþÇøÃû³Æ","Ð¡ÇøID","Ð¡ÇøÃû³Æ","Â¥¶°ID","Â¥¶°±àºÅ","Â¥¶°Ãû³Æ","µ¥ÔªID","µ¥Ôª±àºÅ","µ¥ÔªÃû³Æ","»§ID","ÏêϸµØÖ·","½¨ÖþÄê´ú","ȨÊô","ȨÊôÄêÏÞ","µ¥¼Û","×ܼÛ","³öÊÛ¸¶¿î·½Ê½","ËùÔÚÂ¥²ã","×ÜÂ¥²ã","½¨ÖþÀàÐÍ","ÊÒÄڽṹ","½¨Öþ½á¹¹","½¨ÖþÃæ»ý","Ì×ÄÚÃæ»ý","»§ÐÍ","ÊÒ","Ìü","³ø","ÎÀ","³¯Ïò","×°ÐÞÇé¿ö","ÊÇ·ñÓеçÌÝ","¼¸Ìݼ¸»§","·¢²¼Ê±¼ä","µ½ÆÚʱ¼ä","ÊÇ·ñÊ×·¢","·¿Ô´Êý¾ÝÀ´Ô´","Öн鹫˾ID","Öн鹫˾Ãû³Æ","·¢²¼ÈË","·¢²¼È˵绰","ÃèÊöÐÅÏ¢","±¸×¢","´´½¨Ê±¼ä","¸üÐÂʱ¼ä","À´Ô´¹«Ë¾","¸½¼ÓÐÅÏ¢"] // } FileUtils.batchCreateFile(allDatas, fileName, headers, "utf-8", true); - println "txtÎļþÉú³É " + println "txtÎļþÉú³É->ÊÖ¶¯µ¼Èëµ½Oracle " + + saveToPgSql(allDatas) } @@ -610,9 +617,9 @@ class BigdataForsaleBatch extends BaseBatch{ def count = isExistFromHouse(seawater.id) if(count){ - rets << rowMap - }else{ println seawater.id +" ÒÑ´æÔÚ" + }else{ + rets << rowMap } } @@ -812,9 +819,9 @@ class BigdataForsaleBatch extends BaseBatch{ def count = isExistFromHouse(seawater.id) if(count){ - rets << rowMap - }else{ println seawater.id +" ÒÑ´æÔÚ" + }else{ + rets << rowMap } } } @@ -1019,4 +1026,35 @@ class BigdataForsaleBatch extends BaseBatch{ return count; } + def saveToPgSql(rows){ + for(def r : rows) { + for (def i = 0; i < r.size(); i++) { + if("" == r[i]){ + r[i] = null + } + } + r << null + r << '2023-02-28' + r << null + r << '2999-12-31' + r << null + r << null + r << null + } + + println "start: "+new Date() + def sql = "insert into base.t0831_bigdata_forsale_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" + + pgsqlInstance.withBatch(20000, sql){ ps-> + for(def r : rows) { + ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20], + r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38], r[39], r[40], + r[41], r[42], r[43], r[44], r[45], r[46], r[47], r[48], r[49], r[50], r[51], r[52], r[53], r[54], r[55], r[56], r[57], r[58]) + ps.toString() + } + } + + println "end: "+new Date() + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy index ed95e4cc6..d11ca41ad 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy @@ -9,6 +9,7 @@ import com.fangjia.script.util.DateUtils import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ExcelUtils import com.mongodb.BasicDBObject +import groovy.sql.Sql import org.bson.types.ObjectId /** @@ -18,6 +19,9 @@ import org.bson.types.ObjectId @BatchOwner(Watcher.GaoJuan) class StagingDistrictBatch extends BaseBatch{ + def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified", + "postgres", "postgres", "org.postgresql.Driver") + static main(args) { runBatch(new StagingDistrictBatch(), args); } @@ -47,6 +51,8 @@ class StagingDistrictBatch extends BaseBatch{ //ExcelUtils.write("d:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\excel\\COMMUNITY_STAGING_GJ.xlsx", results, "Êý¾Ý") FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); + saveToPgSql(rows) + } def createCityData(def dists, def fenQiMap){ @@ -86,4 +92,35 @@ class StagingDistrictBatch extends BaseBatch{ } return fenQiMap; } + + + def saveToPgSql(rows){ + for(def r : rows) { + for (def i = 0; i < r.size(); i++) { + if("" == r[i]){ + r[i] = null + } + } + r << null + r << '2023-02-28' + r << null + r << '2999-12-31' + r << null + r << null + r << null + } + + println "start: "+new Date() + def sql = "insert into base.t0831_community_staging_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" + + pgsqlInstance.withBatch(20000, sql){ ps-> + for(def r : rows) { + ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20], + r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38]) + ps.toString() + } + } + + println "end: "+new Date() + } } From 5e3d4fe6cad9f668978498d5680c12563ed43096 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 17 Feb 2023 15:20:32 +0800 Subject: [PATCH 78/99] =?UTF-8?q?=E5=AD=97=E6=AE=B5=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/script/ccbevaluation/StagingDistrictBatch.groovy | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy index d11ca41ad..899243d11 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy @@ -111,12 +111,11 @@ class StagingDistrictBatch extends BaseBatch{ } println "start: "+new Date() - def sql = "insert into base.t0831_community_staging_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" + def sql = "insert into base.t0831_community_staging_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?)" pgsqlInstance.withBatch(20000, sql){ ps-> for(def r : rows) { - ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20], - r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38]) + ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13]) ps.toString() } } From 470f2257d048ab4b925e19fb3639cb2c118c4e19 Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Fri, 17 Feb 2023 15:29:26 +0800 Subject: [PATCH 79/99] =?UTF-8?q?=E8=A7=84=E8=8C=83=E7=89=A9=E4=B8=9A?= =?UTF-8?q?=E7=B1=BB=E5=9E=8B=E8=BE=93=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/open/controllers/AddressController.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java index 27942a8b1..f998411a0 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java @@ -523,6 +523,13 @@ public class AddressController { } MatchDistrictResult matchResult = addressMatchProxyService.checkMatchDistrictCorrect(param); jsonResult.put("matchResult", matchResult); + if (jsonResult.containsKey("category")) { + if ("office".equals(jsonResult.getString("category"))) { + jsonResult.put("propertyType", "д×ÖÂ¥"); + } else if ("shop".equals(jsonResult.getString("category"))) { + jsonResult.put("propertyType", "ÉÌÆÌ"); + } + } } else if (jsonResult != null) { // ÅжÏÊÇ·ñΪ×ÔÈ»´å boolean matchedVillage = false; From 1f0bacd013cf4421e19a3153619f6c810f3f9f34 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 17 Feb 2023 17:11:19 +0800 Subject: [PATCH 80/99] update --- .../ccbevaluation/BigdataDealBatch.groovy | 37 ++++++++ .../ccbevaluation/BigdataForsaleBatch.groovy | 85 +++++++++++++------ 2 files changed, 94 insertions(+), 28 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index fe8a32a3c..d2cbed0d1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -50,6 +50,9 @@ class BigdataDealBatch extends BaseBatch{ def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified", + "postgres", "postgres", "org.postgresql.Driver") + static main(args) { runBatch(new BigdataDealBatch(), args); } @@ -106,6 +109,8 @@ class BigdataDealBatch extends BaseBatch{ // // FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); // println "txt Éú³É " + rows.size() + + saveToPgSql(rows) } def createData(def rets, cityCodeDatas, distCodeMap, source){ @@ -680,4 +685,36 @@ class BigdataDealBatch extends BaseBatch{ latch.await() println city +" "+trades.size()+" "+new Date().toString() } + + def saveToPgSql(rows){ + for(def r : rows) { + for (def i = 0; i < r.size(); i++) { + if("" == r[i]){ + r[i] = null + } + } + r << null + r << '2023-02-28' + r << null + r << '2999-12-31' + r << null + r << null + r << null + } + + println "start: "+new Date() + def sql = "insert into base.t0831_bigdata_deal_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" + + sqlInstance.withBatch(20000, sql){ ps-> + for(def r : rows) { + ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20], + r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38], r[39], + r[40], r[41], r[42], r[43], r[44], r[45], r[46], r[47], r[48], r[49],r[50], r[51], r[52], r[53], r[54], r[55], r[56], r[57], r[58], r[59], + r[60], r[61], r[62], r[63], r[64], r[65], r[66], r[67], r[68], r[69],r[70], r[71]) + ps.toString() + } + } + + println "end: "+new Date() + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index f90b1cfdc..0f85fe67b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -122,7 +122,8 @@ class BigdataForsaleBatch extends BaseBatch{ totalCount.addAndGet(1) def commId = rMap.get("_id").toString() - def ret = findHouses(city, rMap.region, "sell", startDate,endDate, rMap.name, rMap.alias, rMap.estate_type2) + + def ret = findHouses(city, rMap.region, "sell", startDate,endDate, rMap.name, rMap.alias, rMap.estate_type2, commId) //def ret = findHousesByDistId(city, commId, "sell", startDate,endDate) // if(ret && ret.size()<100){//µ¼ÀúÊ··¿Ô´ÐèÒª£¬ÒÔºó²»ÐèÒª // ret = findHouses(city, rMap.region, "sell", 20190701, endDate, rMap.name, rMap.alias) @@ -435,12 +436,24 @@ class BigdataForsaleBatch extends BaseBatch{ } - def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2){ + def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2, def commId){ def rets1 = findHousesByPdate(city, region, cat, startDate, endDate, name, alias, estate_type2) def rets2 = findHousesByMdate(city, region, cat, startDate, endDate, name, alias, estate_type2, rets1) rets1.addAll(rets2) - return rets1; + if(rets1){ + def newRets = [] + def houseIds = getAllHouseIds(commId) + for(def r : rets1){ + if(!houseIds.contains(r.houseId)){ + newRets << r + }else{ + println r.houseId +" ÒÑ´æÔÚ" + } + } + return newRets + } + return []; } def findHousesByPdate(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2){ @@ -474,8 +487,11 @@ class BigdataForsaleBatch extends BaseBatch{ } + def hosts = ["192.168.0.206","192.168.0.211","192.168.0.212","192.168.0.213","192.168.0.214","192.168.0.215"] + def index = new Random().nextInt(6); + def rets = [] - def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder); + def houses = ElasticUtils.executeSimpleQuery(hosts[index], "seawater-20*", queryBuilder); if(houses){ for (int k=0;k - distIds << it.commid + houseIds << it.FORSALEID }); - return distIds; + return houseIds; } + def isExistFromHouse(def houseId){ def sql = "SELECT count(FORSALEID) as count FROM bigdata_forsale_gj WHERE FORSALEID = '"+houseId+"'"; From 17d0dfafe5289deeafd2fb7a76ffe01f87c21edd Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 17 Feb 2023 17:38:35 +0800 Subject: [PATCH 81/99] =?UTF-8?q?=E6=9F=A5=E6=88=BF=E6=BA=90=E5=8A=A0?= =?UTF-8?q?=E5=9F=8E=E5=B8=82id?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/ccbevaluation/BigdataForsaleBatch.groovy | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index 0f85fe67b..cf248fb61 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -123,7 +123,7 @@ class BigdataForsaleBatch extends BaseBatch{ def commId = rMap.get("_id").toString() - def ret = findHouses(city, rMap.region, "sell", startDate,endDate, rMap.name, rMap.alias, rMap.estate_type2, commId) + def ret = findHouses(city, rMap.region, "sell", startDate,endDate, rMap.name, rMap.alias, rMap.estate_type2, commId, cityCodeDataMap) //def ret = findHousesByDistId(city, commId, "sell", startDate,endDate) // if(ret && ret.size()<100){//µ¼ÀúÊ··¿Ô´ÐèÒª£¬ÒÔºó²»ÐèÒª // ret = findHouses(city, rMap.region, "sell", 20190701, endDate, rMap.name, rMap.alias) @@ -436,14 +436,15 @@ class BigdataForsaleBatch extends BaseBatch{ } - def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2, def commId){ + def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2, def commId, def cityCodeDataMap){ def rets1 = findHousesByPdate(city, region, cat, startDate, endDate, name, alias, estate_type2) def rets2 = findHousesByMdate(city, region, cat, startDate, endDate, name, alias, estate_type2, rets1) rets1.addAll(rets2) if(rets1){ def newRets = [] - def houseIds = getAllHouseIds(commId) + def cityCode = cityCodeDataMap.get(city) + def houseIds = getAllHouseIds(commId, cityCode) for(def r : rets1){ if(!houseIds.contains(r.houseId)){ newRets << r @@ -1034,8 +1035,8 @@ class BigdataForsaleBatch extends BaseBatch{ println ids.size() } - def getAllHouseIds(def commid){ - def sql = "SELECT FORSALEID FROM bigdata_forsale_gj WHERE commid = '"+commid+"'"; + def getAllHouseIds(def commid, def cityCode){ + def sql = "SELECT FORSALEID FROM bigdata_forsale_gj WHERE cityid='"+cityCode+"' and commid = '"+commid+"'"; def houseIds = new HashSet() sqlInstance.eachRow(sql,{it-> From 05673e6b2c34dd4176ab61be30519e23d30ee28a Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 17 Feb 2023 18:18:15 +0800 Subject: [PATCH 82/99] update --- .../com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy | 1 - 1 file changed, 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index cf248fb61..007fc2b86 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -73,7 +73,6 @@ class BigdataForsaleBatch extends BaseBatch{ //Ŀǰ²éѯ¿âÖÐÊÇ·ñÒÑ´æÔÚ£¬ÓõÄÊÇOracle def index = 0 def citys = CcbCommonUtils.getRunningCitys(args) - citys = ["¸£ÖÝ"] def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() for(def city : citys){ index++ From c83a6d13696ad2f8e2b88753369663789ce2bcf5 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Sun, 19 Feb 2023 10:54:55 +0800 Subject: [PATCH 83/99] =?UTF-8?q?=E6=88=BF=E6=BA=90=E5=8E=BB=E9=87=8D?= =?UTF-8?q?=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ccbevaluation/BigdataForsaleBatch.groovy | 102 +++++++++++++----- .../gj/paimai/BeiKeCrawlByChrome.groovy | 4 +- .../gj/paimai/LianjiaCrawlByChrome.groovy | 2 +- 3 files changed, 76 insertions(+), 32 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index 007fc2b86..74740ef87 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -21,6 +21,7 @@ import org.elasticsearch.index.query.QueryBuilders import org.elasticsearch.xpack.sql.jdbc.EsDataSource import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CopyOnWriteArrayList import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService import java.util.concurrent.Executors @@ -107,6 +108,33 @@ class BigdataForsaleBatch extends BaseBatch{ def exportSellCSV(index,city, cityCodeDataMap, distCodeMap, districts, startDate, endDate){ + def retMap = findAllCommHouse(index,city, cityCodeDataMap, distCodeMap, districts, startDate, endDate) + println city + "(Ô­)·¿Ô´×ÜÁ¿£º"+retMap.size() + + def allDatas = retMap.values() + allDatas = repeatHouseId(allDatas) + println city + "(ÐÂ)·¿Ô´×ÜÁ¿£º"+allDatas.size() + + + def fileName = "/home/goojia/ccb/BIGDATA_FORSALE_GJ.dat" + if(CommonDatautils.isWindow()){ + fileName = "d:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\BIGDATA_FORSALE_GJ.dat" + } + + + // def allDatas = createSellDatas(city, retMap.values()); + + def headers = [] +// if(index == 1){ +// headers = ["³ÇÊÐID","³öÊÛID","¹ÒÅÆ·¿ÎÝID","³ÇÊÐÃû³Æ","ÐÐÕþÇøID","ÐÐÕþÇøÃû³Æ","Ð¡ÇøID","Ð¡ÇøÃû³Æ","Â¥¶°ID","Â¥¶°±àºÅ","Â¥¶°Ãû³Æ","µ¥ÔªID","µ¥Ôª±àºÅ","µ¥ÔªÃû³Æ","»§ID","ÏêϸµØÖ·","½¨ÖþÄê´ú","ȨÊô","ȨÊôÄêÏÞ","µ¥¼Û","×ܼÛ","³öÊÛ¸¶¿î·½Ê½","ËùÔÚÂ¥²ã","×ÜÂ¥²ã","½¨ÖþÀàÐÍ","ÊÒÄڽṹ","½¨Öþ½á¹¹","½¨ÖþÃæ»ý","Ì×ÄÚÃæ»ý","»§ÐÍ","ÊÒ","Ìü","³ø","ÎÀ","³¯Ïò","×°ÐÞÇé¿ö","ÊÇ·ñÓеçÌÝ","¼¸Ìݼ¸»§","·¢²¼Ê±¼ä","µ½ÆÚʱ¼ä","ÊÇ·ñÊ×·¢","·¿Ô´Êý¾ÝÀ´Ô´","Öн鹫˾ID","Öн鹫˾Ãû³Æ","·¢²¼ÈË","·¢²¼È˵绰","ÃèÊöÐÅÏ¢","±¸×¢","´´½¨Ê±¼ä","¸üÐÂʱ¼ä","À´Ô´¹«Ë¾","¸½¼ÓÐÅÏ¢"] +// } + FileUtils.batchCreateFile(allDatas, fileName, headers, "utf-8", true); + println "txtÎļþÉú³É->ÊÖ¶¯µ¼Èëµ½Oracle " + + saveToPgSql(allDatas) + } + + def findAllCommHouse(index,city, cityCodeDataMap, distCodeMap, districts, startDate, endDate){ def totalCount = new AtomicLong() final CountDownLatch latch = new CountDownLatch(districts.size()) println city +" Ð¡ÇøÁ¿£º"+districts.size() @@ -172,23 +200,38 @@ class BigdataForsaleBatch extends BaseBatch{ }) } latch.await() + return retMap + } - def fileName = "/home/goojia/ccb/BIGDATA_FORSALE_GJ.dat" - if(CommonDatautils.isWindow()){ - fileName = "d:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\BIGDATA_FORSALE_GJ.dat" + + def repeatHouseId(allDatas){ + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(allDatas.size()) + def rets = new CopyOnWriteArrayList() + for(def i=0;iÊÖ¶¯µ¼Èëµ½Oracle " - - saveToPgSql(allDatas) + latch.await() + return rets } @@ -440,20 +483,21 @@ class BigdataForsaleBatch extends BaseBatch{ def rets1 = findHousesByPdate(city, region, cat, startDate, endDate, name, alias, estate_type2) def rets2 = findHousesByMdate(city, region, cat, startDate, endDate, name, alias, estate_type2, rets1) rets1.addAll(rets2) - if(rets1){ - def newRets = [] - def cityCode = cityCodeDataMap.get(city) - def houseIds = getAllHouseIds(commId, cityCode) - for(def r : rets1){ - if(!houseIds.contains(r.houseId)){ - newRets << r - }else{ - println r.houseId +" ÒÑ´æÔÚ" - } - } - return newRets - } - return []; +// if(rets1){ +// def newRets = [] +// def cityCode = cityCodeDataMap.get(city) +// def houseIds = getAllHouseIds(commId, cityCode) +// for(def r : rets1){ +// if(!houseIds.contains(r.houseId)){ +// newRets << r +// }else{ +// println r.houseId +" ÒÑ´æÔÚ" +// } +// } +// return newRets +// } +// return []; + return rets1; } def findHousesByPdate(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy index 0bab915c0..f641c214c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy @@ -401,13 +401,13 @@ public class BeiKeCrawlByChrome { if(cityEles){ for(def e : cityEles){ def href = e.attr("href") - if(e.text().contains("¸£ÖÝ")){ +// if(e.text().contains("¸£ÖÝ")){ def cityMap = [:] cityMap.city = e.text() cityMap.url = "https:"+href list << cityMap // cityMap.put(e.text(), "https:"+href) - } +// } } } return list diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy index 94a13d8e8..e82b8bc45 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -478,7 +478,7 @@ public class LianjiaCrawlByChrome { if(cityEles){ for(def e : cityEles) { def href = e.attr("href") - if (!href.contains("fang.lianjia") && citys.contains(e.text())){ + if (!href.contains("fang.lianjia")){ cityMap.put(e.text(), href) } } From 2cbd678451f8e6ee38627f768f1806dffb225306 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Sun, 19 Feb 2023 11:41:28 +0800 Subject: [PATCH 84/99] upate --- .../com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index 74740ef87..5b075b74f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -217,7 +217,7 @@ class BigdataForsaleBatch extends BaseBatch{ totalCount.addAndGet(1) def count = isExistFromHouse(houseInfo[1]) if(count){ - println houseInfo[1]+" ÒÑ´æÔÚ" + println houseInfo[1]+" ÒÑ´æÔÚ "+totalCount.get() }else{ rets << houseInfo } From 7d7c9daac21084fe18d17d582be0f25167e7da06 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 20 Feb 2023 10:52:33 +0800 Subject: [PATCH 85/99] =?UTF-8?q?pgsql=20=E6=8D=A2=E5=BA=93?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ccbevaluation/BigdataDealBatch.groovy | 3 +- .../ccbevaluation/BigdataForsaleBatch.groovy | 6 ++- .../ccbevaluation/GuiDingPriceBatch.groovy | 39 ++++++++++++++++-- .../ccbevaluation/ThirdPartyPriceBatch.groovy | 40 +++++++++++++++++-- 4 files changed, 77 insertions(+), 11 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index d2cbed0d1..3c9e030bb 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -705,13 +705,12 @@ class BigdataDealBatch extends BaseBatch{ println "start: "+new Date() def sql = "insert into base.t0831_bigdata_deal_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" - sqlInstance.withBatch(20000, sql){ ps-> + pgsqlInstance.withBatch(20000, sql){ ps-> for(def r : rows) { ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20], r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38], r[39], r[40], r[41], r[42], r[43], r[44], r[45], r[46], r[47], r[48], r[49],r[50], r[51], r[52], r[53], r[54], r[55], r[56], r[57], r[58], r[59], r[60], r[61], r[62], r[63], r[64], r[65], r[66], r[67], r[68], r[69],r[70], r[71]) - ps.toString() } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index 5b075b74f..398fd08bd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -58,11 +58,13 @@ class BigdataForsaleBatch extends BaseBatch{ public static final String[] TAGS = ["¶À¶°", "ÁªÅÅ", "µþ¼Ó", "˫ƴ", "»¨Ô°Ãæ»ý", "Ó¾³Ø", "±ðÊû", "´ø»¨Ô°", "Ñó·¿", "µþÆ´", "¶ÀÔº", "ÁªÌå", "΢Êû", "´óƽÊû"]; public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(60) + public static ExecutorService fixedThreadPool2 = Executors.newFixedThreadPool(100) + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") - def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/postgres?stringtype=unspecified", + def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified", "postgres", "postgres", "org.postgresql.Driver") static main(args) { @@ -211,7 +213,7 @@ class BigdataForsaleBatch extends BaseBatch{ for(def i=0;i + for(def r : rows) { + ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20]) + } + } + + println "end: "+new Date() + } + + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy index 8a2b1c55e..3aedc3aa5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy @@ -22,9 +22,9 @@ import java.util.concurrent.ConcurrentHashMap @BatchOwner(Watcher.GaoJuan) class ThirdPartyPriceBatch extends BaseBatch { - def sDate= 202212//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); - def startDate = 202210 - def endDate = 202212 + def sDate= 202301//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); + def startDate = 202211 + def endDate = 202301 // def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM")); static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") @@ -36,6 +36,10 @@ class ThirdPartyPriceBatch extends BaseBatch { def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + + def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified", + "postgres", "postgres", "org.postgresql.Driver") + static main(args) { runBatch(new ThirdPartyPriceBatch(), args); } @@ -69,6 +73,8 @@ class ThirdPartyPriceBatch extends BaseBatch { def header = []//["µÚÈý·½¾ù¼ÛID","Â¥ÅÌID","Ð¡ÇøÃû³Æ","³ÇÊÐID","³ÇÊÐÃû³Æ","ÐÐÕþÇøID","ÐÐÕþÇøÃû³Æ","¾ù¼Û¶ÔÓ¦µÄÔ·Ý","Öî¸ðÕÒ·¿¾ù¼Û","°²¾Ó¿Í¾ù¼Û","·¿ÌìϾù¼Û","Á´¼Ò¾ù¼Û","³ÇÊз¿²ú¾ù¼Û","ÂóÌï¾ù¼Û","±´¿Ç¾ù¼Û","ÌÔ·¿365","ÖÐÔ­¾ù¼Û","Íò©Ê¿¾ù¼Û","Q·¿¾ù¼Û","·¿Ì½¾ù¼Û","·¿¶à¶à¾ù¼Û","ÎÒ°®ÎÒ¼Ò¾ù¼Û","´´½¨Ê±¼ä","¸üÐÂʱ¼ä","À´Ô´¹«Ë¾"] FileUtils.batchCreateFile(rows, fileName, header, "utf-8", true); + saveToPgSql(rows) + } @@ -618,4 +624,32 @@ class ThirdPartyPriceBatch extends BaseBatch { } + def saveToPgSql(rows){ + for(def r : rows) { + for (def i = 0; i < r.size(); i++) { + if("" == r[i]){ + r[i] = null + } + } + r << null + r << '2023-02-28' + r << null + r << '2999-12-31' + r << null + r << null + r << null + } + + println "start: "+new Date() + def sql = "insert into base.t0831_bigdata_deal_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" + + pgsqlInstance.withBatch(20000, sql){ ps-> + for(def r : rows) { + ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18]) + } + } + + println "end: "+new Date() + } + } From 53d9032bba96b31074bd6c31f1116704a4c4ccdc Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 20 Feb 2023 10:57:36 +0800 Subject: [PATCH 86/99] 94->236 --- .../com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index 908f4adf5..1d9ae5347 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -61,7 +61,7 @@ class CcbCommonUtils { "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") }else if(countStr=="252"){ citys = CcbCommonUtils.citys_252 - sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl", + sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") } }else{ From 5d8b5abfc4b4f191122ef47c2b1bf8ea21cb6e2a Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 20 Feb 2023 10:58:15 +0800 Subject: [PATCH 87/99] =?UTF-8?q?100-=E3=80=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index 398fd08bd..82baa96ee 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -58,7 +58,7 @@ class BigdataForsaleBatch extends BaseBatch{ public static final String[] TAGS = ["¶À¶°", "ÁªÅÅ", "µþ¼Ó", "˫ƴ", "»¨Ô°Ãæ»ý", "Ó¾³Ø", "±ðÊû", "´ø»¨Ô°", "Ñó·¿", "µþÆ´", "¶ÀÔº", "ÁªÌå", "΢Êû", "´óƽÊû"]; public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(60) - public static ExecutorService fixedThreadPool2 = Executors.newFixedThreadPool(100) + public static ExecutorService fixedThreadPool2 = Executors.newFixedThreadPool(200) def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", From 06dde273d46a2baf1e6b5687b730bc5b3782ad46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=96=87=E8=89=B3?= Date: Mon, 20 Feb 2023 19:01:04 +0800 Subject: [PATCH 88/99] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E9=82=AE=E7=AE=B1?= =?UTF-8?q?=E5=AF=86=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/bkoff/util/BaseAvgImportServlet.java | 11 ++++++----- .../src/main/java/com/fangjia/core/Constants.java | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/util/BaseAvgImportServlet.java b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/util/BaseAvgImportServlet.java index 411d679e6..f782a46f9 100644 --- a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/util/BaseAvgImportServlet.java +++ b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/util/BaseAvgImportServlet.java @@ -375,7 +375,8 @@ public class BaseAvgImportServlet extends HttpServlet { } else if (key.equals("¼Û¸ñ") && map.get(key) != null) { String val = formatNumer(map.get(key)); if (StringUtils.isNotBlank(val)) { - int x = Integer.parseInt(val); + double dd = Double.parseDouble(val); + int x = (int) dd; avgBase.setPrice(x); } @@ -395,6 +396,7 @@ public class BaseAvgImportServlet extends HttpServlet { } else if (key.equals("À´Ô´Ð¡ÇøÃû") && map.get(key) != null){ avgBase.setSourceCommName(map.get(key).replaceAll("\\s", "")); } + avgBase.setRegion(RegionData.formatRegion(avgBase.getCity(), avgBase.getRegion())); avgBase.setCategory("district"); } District dis = null; @@ -439,7 +441,7 @@ public class BaseAvgImportServlet extends HttpServlet { avgBase.getDistrictName()); dis = districtService.findOneByKey(districtKey, "district"); } - GuidingPrice price = guidingPriceService.findByBizKey(avgBase.getCity(), avgBase.getRegion(), + GuidingPrice price = guidingPriceService.findByBizKey(avgBase.getCity(), RegionData.formatRegion(avgBase.getCity(), avgBase.getRegion()), avgBase.getDistrictName(),avgBase.getCreationDate()); if (price != null) { price.setGuidingPerson(avgBase.getUserName()); @@ -473,11 +475,10 @@ public class BaseAvgImportServlet extends HttpServlet { } catch (HttpRequestException e) { e.printStackTrace(); } + }else{ + System.out.println("Ð¡Çø²»´æÔÚ"+ avgBase.getDistrictName()); } } - - if(dis !=null) { - } } avgBases.add(avgBase); diff --git a/projects/fangjia-core/src/main/java/com/fangjia/core/Constants.java b/projects/fangjia-core/src/main/java/com/fangjia/core/Constants.java index 5b64d35c0..7a9f45104 100644 --- a/projects/fangjia-core/src/main/java/com/fangjia/core/Constants.java +++ b/projects/fangjia-core/src/main/java/com/fangjia/core/Constants.java @@ -8,6 +8,6 @@ public interface Constants { String LOG_MONGO_URI = "mongodb://192.168.0.42:52002"; String SEABIRD_MONGO_URI = "mongodb://192.168.0.74:57007"; String SEAWATER_MONGO_URI = "mongodb://192.168.0.206:27020"; - String MSVC_FANG_JIA_DATA_SERVICE_BASE_PATH ="http://192.168.0.19:38080/FangJiaDataService"; + String MSVC_FANG_JIA_DATA_SERVICE_BASE_PATH ="http://192.168.0.100:38080/FangJiaDataService"; } From c3157ed42ff4244579b133bedcec16f7bc51d7e6 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 20 Feb 2023 21:51:27 +0800 Subject: [PATCH 89/99] updadte --- .../ccbevaluation/BigdataDealBatch.groovy | 80 +++++++++---------- .../ccbevaluation/ThirdPartyPriceBatch.groovy | 4 +- .../script/ccbevaluation/TradeStatLog.groovy | 10 +-- .../fangjia/script/gj/CityStatExport.groovy | 8 +- .../fangjia/script/gj/YouDaDataParser.groovy | 4 +- .../gj/paimai/BeiKeCrawlByChrome.groovy | 4 +- .../gj/paimai/LianjiaCrawlByChrome.groovy | 4 +- 7 files changed, 57 insertions(+), 57 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 3c9e030bb..b5d601cd4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -67,48 +67,48 @@ class BigdataDealBatch extends BaseBatch{ } println "curDate:"+curDate - for(def city : citys){ -// if(!ccbCitys.contains(city)){ - addDealPriceMatch(city); -// } - } -// def rows = [] -// def headers = []//["³ÇÊÐID","³É½»ID","³ÇÊÐÃû³Æ","ÐÐÕþÇøID","ÐÐÕþÇøÃû³Æ","½ÖµÀ","ÃÅÅÆºÅ","Ð¡ÇøID","Ð¡ÇøÃû³Æ","Â¥¶°ID","Â¥¶°±àºÅ","Â¥¶°Ãû³Æ","µ¥ÔªID","µ¥Ôª±àºÅ","µ¥ÔªÃû³Æ","»§ID","ÏêϸµØÖ·","½¨ÖþÄê´ú","ȨÊô","ȨÊôÄêÏÞ","ÆÚÊý","ËùÔÚÂ¥²ã","²ãÀàÐÍ","×ÜÂ¥²ãÊý","½¨ÖþÀàÐÍ","½¨Öþ½á¹¹","½¨ÖþÃæ»ý","Ì×ÄÚÃæ»ý","»§ÐÍ","ÊÒ","Ìü","³ø","ÎÀ","·¿ÎÝÓÃ;","ÊÒÄڽṹ","³¯Ïò","×°ÐÞÇé¿ö","ÆÀ¹À±àºÅ","³É½»µ¥¼Û","³É½»×ܼÛ","É걨×ܼÛ","ÆÀ¹À×ܼÛ","ÍøÇ©ºÏͬ±àºÅ","½»Ò×ÈÕÆÚ","½»Ò×ÀàÐÍ","³öÊÛ¸¶¿î·½Ê½","³É½»ÖÜÆÚ","ÊÇ·ñÓеçÌÝ","ÔùËÍÃæ»ý£¨µØÃ滨԰£©","ÔùËÍÃæ»ý£¨Â¶Ì¨£©","ÔùËÍÃæ»ý£¨Ñǫ̂£©","ÔùËÍÃæ»ý£¨Æ®´°£©","ÔùËÍÃæ»ý£¨¸óÂ¥£©","ÔùËÍÃæ»ý£¨³µ¿â£©","ÔùËÍÃæ»ý£¨µØÏÂÊÒ£©","ÔùËÍÃæ»ý£¨ÆäËû£©","·¿Ô´Êý¾ÝÀ´Ô´","Öн鹫˾ID","Öн鹫˾Ãû³Æ","±¸×¢","´´½¨Ê±¼ä","¸üÐÂʱ¼ä","À´Ô´¹«Ë¾","³É½»½ð¶î","Êý¾Ýʱ¼ä"] -// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() -// def index = 0 // for(def city : citys){ -// index ++ -// def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) -// def ppeDists = [:] -// -// def statMap = [:]//getDistrictStatByCity(city, weekend) -// if(city=="ÉϺ£"){ -// def youDataTrade = getYouDaData(statMap) -// def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda") -// rows.addAll(youdaDatas) -// println city +" "+youdaDatas.size() -// } -// -// def trades = getCityData(city, statMap, ppeDists) -// if(trades){ -// def rets = createData(trades, cityCodeDataMap, distCodeMap, "other") -// rows.addAll(rets) -// println "¡¾${index}¡¿"+city +" ³É½»¹²£º"+rets.size() -// -// def cityDatas = [] -// cityDatas << headers -// cityDatas.addAll(rows) -// } -// +//// if(!ccbCitys.contains(city)){ +// addDealPriceMatch(city); +//// } // } -// -// def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat" -// if(CommonDatautils.isWindow()){ -// fileName = "d:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\BIGDATA_DEAL_GJ.dat" -// } -// -// FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); -// println "txt Éú³É " + rows.size() + def rows = [] + def headers = []//["³ÇÊÐID","³É½»ID","³ÇÊÐÃû³Æ","ÐÐÕþÇøID","ÐÐÕþÇøÃû³Æ","½ÖµÀ","ÃÅÅÆºÅ","Ð¡ÇøID","Ð¡ÇøÃû³Æ","Â¥¶°ID","Â¥¶°±àºÅ","Â¥¶°Ãû³Æ","µ¥ÔªID","µ¥Ôª±àºÅ","µ¥ÔªÃû³Æ","»§ID","ÏêϸµØÖ·","½¨ÖþÄê´ú","ȨÊô","ȨÊôÄêÏÞ","ÆÚÊý","ËùÔÚÂ¥²ã","²ãÀàÐÍ","×ÜÂ¥²ãÊý","½¨ÖþÀàÐÍ","½¨Öþ½á¹¹","½¨ÖþÃæ»ý","Ì×ÄÚÃæ»ý","»§ÐÍ","ÊÒ","Ìü","³ø","ÎÀ","·¿ÎÝÓÃ;","ÊÒÄڽṹ","³¯Ïò","×°ÐÞÇé¿ö","ÆÀ¹À±àºÅ","³É½»µ¥¼Û","³É½»×ܼÛ","É걨×ܼÛ","ÆÀ¹À×ܼÛ","ÍøÇ©ºÏͬ±àºÅ","½»Ò×ÈÕÆÚ","½»Ò×ÀàÐÍ","³öÊÛ¸¶¿î·½Ê½","³É½»ÖÜÆÚ","ÊÇ·ñÓеçÌÝ","ÔùËÍÃæ»ý£¨µØÃ滨԰£©","ÔùËÍÃæ»ý£¨Â¶Ì¨£©","ÔùËÍÃæ»ý£¨Ñǫ̂£©","ÔùËÍÃæ»ý£¨Æ®´°£©","ÔùËÍÃæ»ý£¨¸óÂ¥£©","ÔùËÍÃæ»ý£¨³µ¿â£©","ÔùËÍÃæ»ý£¨µØÏÂÊÒ£©","ÔùËÍÃæ»ý£¨ÆäËû£©","·¿Ô´Êý¾ÝÀ´Ô´","Öн鹫˾ID","Öн鹫˾Ãû³Æ","±¸×¢","´´½¨Ê±¼ä","¸üÐÂʱ¼ä","À´Ô´¹«Ë¾","³É½»½ð¶î","Êý¾Ýʱ¼ä"] + def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + def index = 0 + for(def city : citys){ + index ++ + def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) + def ppeDists = [:] + + def statMap = [:]//getDistrictStatByCity(city, weekend) + if(city=="ÉϺ£"){ + def youDataTrade = getYouDaData(statMap) + def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda") + rows.addAll(youdaDatas) + println city +" "+youdaDatas.size() + } + + def trades = getCityData(city, statMap, ppeDists) + if(trades){ + def rets = createData(trades, cityCodeDataMap, distCodeMap, "other") + rows.addAll(rets) + println "¡¾${index}¡¿"+city +" ³É½»¹²£º"+rets.size() + + def cityDatas = [] + cityDatas << headers + cityDatas.addAll(rows) + } + + } + + def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat" + if(CommonDatautils.isWindow()){ + fileName = "d:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\BIGDATA_DEAL_GJ.dat" + } + + FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); + println "txt Éú³É " + rows.size() saveToPgSql(rows) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy index 3aedc3aa5..1d774446a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy @@ -641,9 +641,9 @@ class ThirdPartyPriceBatch extends BaseBatch { } println "start: "+new Date() - def sql = "insert into base.t0831_bigdata_deal_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" + def sql = "insert into base.t0831_bigdata_third_party_price_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" - pgsqlInstance.withBatch(20000, sql){ ps-> + pgsqlInstance.withBatch(50000, sql){ ps-> for(def r : rows) { ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18]) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy index 92fb500fc..7f072ef8c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy @@ -92,13 +92,13 @@ class TradeStatLog { def run(){ def citys = CcbCommonUtils.citys_all; def rets = [] - rets << ["³ÇÊÐ","10ÔÂ×ÜÁ¿","11ÔÂ×ÜÁ¿","12ÔÂ×ÜÁ¿","Õ¼±È","±´¿Ç","Á´¼Ò", "ÎÒ°®ÎÒ¼Ò","·¿ÌìÏÂ","ÖÐÔ­"] + rets << ["³ÇÊÐ","11ÔÂ×ÜÁ¿","12ÔÂ×ÜÁ¿","23Äê1ÔÂ×ÜÁ¿","Õ¼±È","±´¿Ç","Á´¼Ò", "ÎÒ°®ÎÒ¼Ò","·¿ÌìÏÂ","ÖÐÔ­"] def sources = ["±´¿ÇÕÒ·¿","Á´¼ÒÔÚÏß","ÎÒ°®ÎÒ¼Ò","·¿ÌìÏÂ","ÖÐÔ­µØ²ú"] //def startDate = "2021-05-01" - def startDate = "2022-10-01" - def date = "2022-11-01" - def endDate = "2022-12-01" - def newDate = "2023-01-01" + def startDate = "2022-11-01" + def date = "2022-12-01" + def endDate = "2023-01-01" + def newDate = "2023-02-01" for(def city : citys){ def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd")) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/CityStatExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/CityStatExport.groovy index 64a84167e..ccca5758b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/CityStatExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/CityStatExport.groovy @@ -17,15 +17,15 @@ class CityStatExport { static main(args) { def task = new CityStatExport(); - //task.run() - task.run1() + task.run() +// task.run1() } def run(){ // def citys = CommonDatautils.findAllCity(); - def citys = ["¸£ÖÝ", "ÏÃÃÅ", "ȪÖÝ", "ÆÎÌï", "ÄþµÂ", "ÈýÃ÷", "ÁúÑÒ", "ÄÏÆ½", "ÕÄÖÝ"] + def citys = ["ËÕÖÝ","ÎÞÎý","ÄÏͨ","ÐìÖÝ","Á¬ÔƸÛ","ËÞǨ","ÑγÇ","Õò½­","Çൺ","ÄϾ©","³£ÖÝ","»´°²","Ì©ÖÝ","ÑïÖÝ","ÉϺ£"] def ret = [] // ret << ["³ÇÊÐ","Ð¡Çø×ÜÁ¿","²úȨÄêÏÞ¸²¸ÇÂÊ","¿¢¹¤Ê±¼ä¸²¸ÇÂÊ"] ret << ["³ÇÊÐ","Ð¡Çø×ÜÁ¿"] @@ -45,7 +45,7 @@ class CityStatExport { ret << m println m } - ExcelUtils.write("D:\\Êý¾Ý\\¸£½¨Ê¡Ð¡ÇøÊý¾Ýͳ¼Æ.xlsx",ret,"Êý¾Ý") + ExcelUtils.write("D:\\Êý¾Ý\\Ð¡ÇøÊý¾Ýͳ¼Æ.xlsx",ret,"Êý¾Ý") } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy index bbd0ec13b..52f2ea980 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy @@ -48,11 +48,11 @@ class YouDaDataParser { def final AtomicInteger count_ = new AtomicInteger(0) - static def statDate = "2022-12-01" + static def statDate = "2023-01-01" static main(args) { def task = new YouDaDataParser(); // task.newHouseparse();//ÓÑ´ïÒ»ÊÖ·¿´æÈëmongo -// task.esfParser("D:\\Êý¾Ý\\ÓÑ´ï\\2023Äê1ÔÂ\\²éѯ1.DBF");//ÓÑ´ï¶þÊÖ·¿´æÈëmongo +// task.esfParser("D:\\Êý¾Ý\\ÓÑ´ï\\2023Äê2ÔÂ\\²éѯ1.DBF");//ÓÑ´ï¶þÊÖ·¿´æÈëmongo // task.newhouseToSaveEs();//з¿±£´æµ½es task.esfhouseToSaveEs();//¶þÊÖ·¿±£´æµ½es // task.updateFinishDateByEsfTrade();//¸ù¾Ý¶þÊֳɽ»µÄ¿¢¹¤Ê±¼äÐÞ¸Äseaweed¿âÖеĿ¢¹¤Ê±¼ä diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy index f641c214c..6c4e5732c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy @@ -401,13 +401,13 @@ public class BeiKeCrawlByChrome { if(cityEles){ for(def e : cityEles){ def href = e.attr("href") -// if(e.text().contains("¸£ÖÝ")){ + if(e.text().contains("ÖØÇì")){ def cityMap = [:] cityMap.city = e.text() cityMap.url = "https:"+href list << cityMap // cityMap.put(e.text(), "https:"+href) -// } + } } } return list diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy index e82b8bc45..2bea3bcce 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -474,11 +474,11 @@ public class LianjiaCrawlByChrome { def cityDoc = Jsoup.parse(driver.getPageSource()); def cityEles = JQueryUtils.select(".city_province a", cityDoc) def cityMap = [:] - def citys= ["¸£ÖÝ"] + def citys= ["ÖØÇì"] if(cityEles){ for(def e : cityEles) { def href = e.attr("href") - if (!href.contains("fang.lianjia")){ + if (!href.contains("fang.lianjia") && citys.contains(e.text())){ cityMap.put(e.text(), href) } } From 37008075739415fe3faf131ebc0769d655d3725c Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 20 Feb 2023 22:38:30 +0800 Subject: [PATCH 90/99] =?UTF-8?q?=E5=8E=BB=EF=BC=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index b5d601cd4..35c9912a8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -703,7 +703,7 @@ class BigdataDealBatch extends BaseBatch{ } println "start: "+new Date() - def sql = "insert into base.t0831_bigdata_deal_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" + def sql = "insert into base.t0831_bigdata_deal_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" pgsqlInstance.withBatch(20000, sql){ ps-> for(def r : rows) { From e20aa1272f707ff3c069891c37cc888fa0c7aed2 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 22 Feb 2023 15:13:21 +0800 Subject: [PATCH 91/99] =?UTF-8?q?pgsql=20=E5=AF=BC=E5=87=BA=E4=BC=B0?= =?UTF-8?q?=E5=80=BC=E5=9F=BA=E6=9C=AC=E6=95=B0=E6=8D=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ccbevaluation/CcbFtpTestUtils.groovy | 2 +- .../ccbevaluation/GuiDingPriceBatch.groovy | 16 +- .../pgsql/ExportPgsqlBaseEvaluation.groovy | 214 +++++++++--------- 3 files changed, 121 insertions(+), 111 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpTestUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpTestUtils.groovy index 00ed6bf6e..81958589e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpTestUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpTestUtils.groovy @@ -7,7 +7,7 @@ import org.apache.commons.codec.digest.DigestUtils class CcbFtpTestUtils { - static def ftpServer = "192.168.0.45"//45 + static def ftpServer = "192.168.0.86"//45 static def ftpPort = 17495 static def ftpUser = "ccb" static def ftpPassword = "ccb!@#" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index ab3444915..32a919245 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) //Ìí¼Ó×Ô¼º¿âÖеÄÈ˹¤Ö¸µ¼¼Û - def prices = CcbCommonUtils.findPersonPrice(city, "2022-12-24", "2023-01-25") + def prices = CcbCommonUtils.findPersonPrice(city, "2023-01-25", "2023-02-21") // def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22") if(prices){ def distMap = getDistrictMap(city) @@ -84,6 +84,9 @@ class GuiDingPriceBatch extends BaseBatch{ saveToPgSql(rows) + //¸üо²Ì¬Êý¾Ý±í + updateVersion(DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28") + } def findFangDatas(def districts){ @@ -230,7 +233,7 @@ class GuiDingPriceBatch extends BaseBatch{ } println "start: "+new Date() - def sql = "insert into base.t0831_bigdata_deal_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" + def sql = "insert into base.t0831_guided_price_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" pgsqlInstance.withBatch(300, sql){ ps-> for(def r : rows) { @@ -241,6 +244,15 @@ class GuiDingPriceBatch extends BaseBatch{ println "end: "+new Date() } + def updateVersion(def time){ + def tables = ["t0831_website_config_gj_h","t0831_bigdata_district_gj_h","t0831_evaluate_rate_gj_h","t0831_bigdata_poi_gj_h","t0831_guided_price_gj_h","t0831_community_staging_gj_h"] + for(def tableName : tables){ + def sql = "update base."+tableName+" set p9_start_date='"+time+"'" + pgsqlInstance.executeUpdate(sql) + } + + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlBaseEvaluation.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlBaseEvaluation.groovy index 428aaa792..2c3853f3f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlBaseEvaluation.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlBaseEvaluation.groovy @@ -1,6 +1,7 @@ package com.fangjia.script.ccbevaluation.pgsql import com.fangjia.script.ccbevaluation.CcbCommonUtils +import com.fangjia.script.ccbevaluation.CcbFtpTestUtils import com.fangjia.script.ccbevaluation.CcbFtpUtils import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.ccb.FileUtils @@ -9,7 +10,7 @@ import groovy.sql.Sql class ExportPgsqlBaseEvaluation { - def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/postgres", + def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse", "postgres", "postgres", "org.postgresql.Driver") def citys = null def cityCodeMap = CcbCommonUtils.findCityCodeValue() @@ -28,40 +29,38 @@ class ExportPgsqlBaseEvaluation { citys = CcbCommonUtils.getRunningCitys(args) def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")); def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); - def lastMonthStr = "2022-09"//DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM") + def lastMonthStr = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM") def new_dists = [] -// exportEvaluationRateData() -// exportBigdataPoiData() -// exportBigdataDistrictData() + exportEvaluationRateData() + exportBigdataPoiData() + exportBigdataDistrictData() //µÚÈý·½ÍøÕ¾È¨Öرí -// exportWebsiteRate(); + exportWebsiteRate(); -// exportGuidedPrice()//ËæÈ˹¤¸Ä±ä// -// exportCommunityGuidedPrice(lastMonth.toString()+"28")//ËæÕþ²ß// -// -//// //´¦ÀíÐ¡ÇøÇé¿ö -// exportBigdataCommunity()// -// exportCommunityStaging()//ûcityid -// exportCommunityBuildType()// -// -// exportThirdPartyPrice2(lastMonth, new_dists)// - exportBigdataForsale(lastMonthStr +"-19", new_dists) //2022-07-19 -// exportBigdataDeal(lastMonthStr + "-01", new_dists) //2022-07-01 + exportGuidedPrice()//ËæÈ˹¤¸Ä±ä// + exportCommunityGuidedPrice(lastMonth.toString()+"28")//ËæÕþ²ß// + //´¦ÀíÐ¡ÇøÇé¿ö + exportCommunityStaging()//ûcityid + exportBigdataCommunity()// + exportCommunityBuildType()// + exportThirdPartyPrice2(lastMonth, new_dists)// + exportBigdataForsale(lastMonthStr +"-18", new_dists) //ÏÖÔÚ°´updatetimeµ¼³ö + exportBigdataDeal(lastMonthStr + "-01", new_dists) //2022-07-01 } def exportEvaluationRateData() { - def sql = "select gujia_ceft_id,gujia_ceft_tp,gujia_ceft,gujia_seq, gujia_urbn,rmrk_dsc, to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm, gujia_src_co from "+database+".t0831_evaluate_rate_gj_w"; + def sql = "select gujia_ceft_id,gujia_ceft_tp,gujia_bds_ceft,gujia_bds_seq, gujia_urbn,rmrk_dsc, to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm, gujia_src_co from "+database+".t0831_evaluate_rate_gj_h"; def allRows = [] sqlInstance.eachRow(sql, { r -> def data = [] data << r.gujia_ceft_id data << r.gujia_ceft_tp - data << r.gujia_ceft - data << r.gujia_seq + data << r.gujia_bds_ceft + data << r.gujia_bds_seq data << r.gujia_urbn data << r.rmrk_dsc data << r.data_crt_tm @@ -82,7 +81,7 @@ class ExportPgsqlBaseEvaluation { def allRows = [] for(def city : citys) { def cityCode = cityCodeMap.get(city) - def sql = "select urbn_id,gujia_admndiv_id,gujia_fsid,poi_nm,gujia_tp,gujia_tp_ecd,lgt,ltt,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+".t0831_bigdata_poi_gj_w where urbn_id='${cityCode}'"; + def sql = "select urbn_id,gujia_admndiv_id,gujia_fsid,poi_nm,gujia_tp,gujia_tp_ecd,gujia_bds_ltt,gujia_bds_lgt,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+".t0831_bigdata_poi_gj_h where urbn_id='${cityCode}'"; sqlInstance.eachRow(sql, { r -> def data = [] data << r.urbn_id @@ -91,8 +90,8 @@ class ExportPgsqlBaseEvaluation { data << r.poi_nm data << r.gujia_tp data << r.gujia_tp_ecd - data << r.lgt - data << r.ltt + data << r.gujia_bds_ltt + data << r.gujia_bds_lgt data << r.rmrk_dsc data << r.data_crt_tm data << r.data_udt_tm @@ -113,7 +112,7 @@ class ExportPgsqlBaseEvaluation { def allRows = [] for(def city : citys) { def cityCode = cityCodeMap.get(city) - def sql = "select gujia_admndiv_id,gujia_admndiv_nm,gujia_admndiv_nm_cpa,urbn_id,gujia_natstd_cd,gujia_dspl_seq,gujia_admndiv_cgy,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_lgc_admndiv_dsc,gujia_lgc_admndiv_cd,gujia_adiv_cd_val,gujia_data_tm from "+database+".t0831_bigdata_district_gj_w where urbn_id='${cityCode}'"; + def sql = "select gujia_admndiv_id,gujia_admndiv_nm,gujia_admndiv_nm_cpa,urbn_id,gujia_natstd_cd,gujia_dspl_seq,gujia_admndiv_cgy,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_lgc_admndiv_dsc,gujia_lgc_admndiv_cd,gujia_adiv_cd_val,gujia_bds_data_tm_val from "+database+".t0831_bigdata_district_gj_h where urbn_id='${cityCode}'"; sqlInstance.eachRow(sql, { r -> def data = [] data << r.gujia_admndiv_id @@ -130,7 +129,7 @@ class ExportPgsqlBaseEvaluation { data << r.gujia_lgc_admndiv_dsc data << r.gujia_lgc_admndiv_cd data << r.gujia_adiv_cd_val - data << r.gujia_data_tm + data << r.gujia_bds_data_tm_val allRows << data }); } @@ -144,15 +143,15 @@ class ExportPgsqlBaseEvaluation { } def exportWebsiteRate(){ - def sql = "select gujiadatawebst_src_id,gujia_data_webst_src,gujiadatawebstmpng_id,gujia_wgt,gujia_tp,gujia_src_co,urbn_id,gujia_urbn,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm, to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm from "+database+".t0831_website_config_gj_w"; + def sql = "select gujiadatawebst_src_id,gujia_data_webst_src,gujiadatawebstmpng_id,gujia_bds_wgt,gujia_bds_tp,gujia_src_co,urbn_id,gujia_urbn,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm, to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm from "+database+".t0831_website_config_gj_h"; def allRows = [] sqlInstance.eachRow(sql, { r -> def data = [] data << r.gujiadatawebst_src_id data << r.gujia_data_webst_src data << r.gujiadatawebstmpng_id - data << r.gujia_wgt - data << r.gujia_tp + data << r.gujia_bds_wgt + data << r.gujia_bds_tp data << r.gujia_src_co data << r.urbn_id data << r.gujia_urbn @@ -174,7 +173,7 @@ class ExportPgsqlBaseEvaluation { def allRows = [] for(def city : citys) { def cityCode = cityCodeMap.get(city) - def sql = "select gujia_fsid,gujia_bds_id,comm_nm,urbn_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,to_char(gujia_drc_dt,'yyyy-mm-dd') as gujia_drc_dt,gujia_drc_prc,gujia_add_psn,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+".t0831_guided_price_gj_w where urbn_id='${cityCode}'"; + def sql = "select gujia_fsid,gujia_bds_id,comm_nm,urbn_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,to_char(gujia_drc_dt,'yyyy-mm-dd') as gujia_drc_dt,gujia_bds_drc_prc,gujia_add_psn,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+".t0831_guided_price_gj_h where urbn_id='${cityCode}'"; sqlInstance.eachRow(sql, { r -> def data = [] data << r.gujia_fsid @@ -185,7 +184,7 @@ class ExportPgsqlBaseEvaluation { data << r.gujia_admndiv_id data << r.gujia_admndiv_nm data << r.gujia_drc_dt - data << r.gujia_drc_prc + data << r.gujia_bds_drc_prc data << r.gujia_add_psn data << r.rmrk_dsc data << r.data_crt_tm @@ -207,7 +206,7 @@ class ExportPgsqlBaseEvaluation { def allRows = [] for(def city : citys) { def cityCode = cityCodeMap.get(city) - def sql = "select gujia_fsid,gujia_bds_id, comm_nm, urbn_id, urbn_nm, gujia_admndiv_id, gujia_admndiv_nm, gujia_drc_prc, gujia_drc_prc_src_val, rmrk_dsc, to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm, to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm, gujia_src_co from "+database+".t0831_community_guide_price_gj_h where urbn_id='${cityCode}' and data_crt_tm>=to_date('${date}','yyyyMMdd')"; + def sql = "select gujia_fsid,gujia_bds_id, comm_nm, urbn_id, urbn_nm, gujia_admndiv_id, gujia_admndiv_nm, gujia_bds_drc_prc, gujia_bds_drc_prc_src, rmrk_dsc, to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm, to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm, gujia_src_co from "+database+".t0831_community_guide_price_gj_h where urbn_id='${cityCode}' and data_crt_tm>=to_date('${date}','yyyyMMdd')"; sqlInstance.eachRow(sql, { r -> def data = [] data << r.gujia_fsid @@ -217,8 +216,8 @@ class ExportPgsqlBaseEvaluation { data << r.urbn_nm data << r.gujia_admndiv_id data << r.gujia_admndiv_nm - data << r.gujia_drc_prc - data << r.gujia_drc_prc_src_val + data << r.gujia_bds_drc_prc + data << r.gujia_bds_drc_prc_src data << r.rmrk_dsc data << r.data_crt_tm data << r.data_udt_tm @@ -239,7 +238,7 @@ class ExportPgsqlBaseEvaluation { def allRows = [] for(def city : citys) { def cityCode = cityCodeMap.get(city) - def sql = "select urbn_id,gujia_admndiv_id,gujia_comm_id,gujia_admndiv_nm,comm_nm,gujia_dtl_adr,lgt,ltt,gujiaocpoflandtotarea,gujia_bld_tot_area,gujia_grnrate,gujia_vol_rate,gujia_park_num,gujia_bld_yr_rc,gujia_hs_use,gujia_totunits,gujia_data_tm,gujia_udt_mark_val,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_comm_tp_val,gujia_comm_fmnm,gujia_str,gujia_hsno,gujiawthr_new_pan_ind,gujia_comm_intd,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,gujia_src_co,gujia_vlgtwn_str_mng,gujia_lgc_admndiv_cd,gujia_lgc_adiv,gujia_bds_st_val from "+database+".t0831_bigdata_community_gj_w where urbn_id='${cityCode}' ";//where cityId in (350200,150100) + def sql = "select urbn_id,gujia_admndiv_id,gujia_comm_id,gujia_admndiv_nm,comm_nm,gujia_dtl_adr,gujia_bds_ltt,gujia_bds_lgt,gujiabdsocpoldtotarea,gujiabds_bld_tot_area,gujia_bds_grnrate,gujia_bds_vol_rate,gujia_bds_park_num,gujia_bds_bld_yr_rc,gujia_bds_hs_use_val,totunits,gujia_bds_data_tm_val,gujiabds_udt_flag_val,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_comm_tp_val,gujia_comm_fmnm,gujia_str,gujia_hsno,gujiawthr_new_pan_ind,gujia_comm_intd,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,gujia_src_co,gujia_vlgtwn_str_mng,gujia_lgc_admndiv_cd,gujia_lgc_adiv,gujia_bds_bds_st from "+database+".t0831_bigdata_community_gj_h where urbn_id='${cityCode}' ";//where cityId in (350200,150100) sqlInstance.eachRow(sql,{r-> def data = [] data << r.urbn_id @@ -248,18 +247,18 @@ class ExportPgsqlBaseEvaluation { data << r.gujia_admndiv_nm data << r.comm_nm data << r.gujia_dtl_adr - data << r.lgt - data << r.ltt - data << r.gujiaocpoflandtotarea - data << r.gujia_bld_tot_area - data << r.gujia_grnrate - data << r.gujia_vol_rate - data << r.gujia_park_num - data << r.gujia_bld_yr_rc - data << r.gujia_hs_use - data << r.gujia_totunits - data << r.gujia_data_tm - data << r.gujia_udt_mark_val + data << r.gujia_bds_ltt + data << r.gujia_bds_lgt + data << r.gujiabdsocpoldtotarea + data << r.gujiabds_bld_tot_area + data << r.gujia_bds_grnrate + data << r.gujia_bds_vol_rate + data << r.gujia_bds_park_num + data << r.gujia_bds_bld_yr_rc + data << r.gujia_bds_hs_use_val + data << r.totunits + data << r.gujia_bds_data_tm_val + data << r.gujiabds_udt_flag_val data << r.data_udt_tm data << r.gujia_comm_tp_val data << r.gujia_comm_fmnm @@ -273,7 +272,7 @@ class ExportPgsqlBaseEvaluation { data << r.gujia_vlgtwn_str_mng data << r.gujia_lgc_admndiv_cd data << r.gujia_lgc_adiv - data << r.gujia_bds_st_val + data << r.gujia_bds_bds_st allRows << data }); } @@ -287,7 +286,7 @@ class ExportPgsqlBaseEvaluation { def exportCommunityStaging() { def allRows = [] - def sql = "select gujia_instm_id,gujia_comm_id,gujiacomm_prim_prd_id,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+".t0831_community_staging_gj_w"; + def sql = "select gujia_instm_id,gujia_comm_id,gujiacomm_prim_prd_id,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+".t0831_community_staging_gj_h"; sqlInstance.eachRow(sql, { r -> def data = [] data << r.gujia_instm_id @@ -310,7 +309,7 @@ class ExportPgsqlBaseEvaluation { def allRows = [] for(def city : citys) { def cityCode = cityCodeMap.get(city) - def sql = "select urbn_id,gujia_comm_id,gujia_bld_tp_val,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_data_tm from "+database+".t0831_bigdata_commbuildtype_gj_w where urbn_id='${cityCode}'"; + def sql = "select urbn_id,gujia_comm_id,gujia_bld_tp_val,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_bds_data_tm_val from "+database+".t0831_bigdata_commbuildtype_gj_h where urbn_id='${cityCode}'"; sqlInstance.eachRow(sql, { r -> def data = [] data << r.urbn_id @@ -320,7 +319,7 @@ class ExportPgsqlBaseEvaluation { data << r.data_crt_tm data << r.data_udt_tm data << r.gujia_src_co - data << r.gujia_data_tm + data << r.gujia_bds_data_tm_val allRows << data }); } @@ -343,17 +342,17 @@ class ExportPgsqlBaseEvaluation { def allRows = [] if(new_dists) { def newstr = "('"+new_dists.join("','")+"')" - def sql1 = "select gujia_trdpt_avprc_id,urbn_id,gujia_admndiv_id,gujia_bds_id,comm_nm,gujia_dtsr_id,gujia_dtsr_dsc,gujia_avprc_dt_val,gujia_comm_avprc,gujia_comm_hgst_prc,gujia_comm_lwst_prc,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm from "+database+".t0831_bigdata_third_party_price_gj_h where " + - "urbn_id='${cityCode}' and gujia_avprc_dt_val=${price_date} and gujia_admndiv_id not in ${newstr} "; + def sql1 = "select gujia_trdpt_avprc_id,urbn_id,gujia_admndiv_id,gujia_bds_id,comm_nm,gujia_dtsr_id,gujia_dtsr_dsc,gujiabds_avprc_dt_val,gujia_bds_comm_avprc,gujiabdscomm_hgst_prc,gujiabdscomm_lwst_prc,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm from "+database+".t0831_bigdata_third_party_price_gj_h where " + + "urbn_id='${cityCode}' and gujiabds_avprc_dt_val=${price_date} and gujia_admndiv_id not in ${newstr} "; def rets1 = getThirdPartyDatas(sql1) allRows.addAll(rets1) - def sql2 = "select gujia_trdpt_avprc_id,urbn_id,gujia_admndiv_id,gujia_bds_id,comm_nm,gujia_dtsr_id,gujia_dtsr_dsc,gujia_avprc_dt_val,gujia_comm_avprc,gujia_comm_hgst_prc,gujia_comm_lwst_prc,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm from "+database+".t0831_bigdata_third_party_price_gj_h where " + + def sql2 = "select gujia_trdpt_avprc_id,urbn_id,gujia_admndiv_id,gujia_bds_id,comm_nm,gujia_dtsr_id,gujia_dtsr_dsc,gujiabds_avprc_dt_val,gujia_bds_comm_avprc,gujiabdscomm_hgst_prc,gujiabdscomm_lwst_prc,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm from "+database+".t0831_bigdata_third_party_price_gj_h where " + "urbn_id='${cityCode}' and gujia_admndiv_id in ${newstr} "; def rets2 = getThirdPartyDatas(sql2) allRows.addAll(rets2) }else{ - def sql = "select gujia_trdpt_avprc_id,urbn_id,gujia_admndiv_id,gujia_bds_id,comm_nm,gujia_dtsr_id,gujia_dtsr_dsc,gujia_avprc_dt_val,gujia_comm_avprc,gujia_comm_hgst_prc,gujia_comm_lwst_prc,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm from "+database+".t0831_bigdata_third_party_price_gj_h where urbn_id='${cityCode}' and gujia_avprc_dt_val='${price_date}' "; + def sql = "select gujia_trdpt_avprc_id,urbn_id,gujia_admndiv_id,gujia_bds_id,comm_nm,gujia_dtsr_id,gujia_dtsr_dsc,gujiabds_avprc_dt_val,gujia_bds_comm_avprc,gujiabdscomm_hgst_prc,gujiabdscomm_lwst_prc,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm from "+database+".t0831_bigdata_third_party_price_gj_h where urbn_id='${cityCode}' and gujiabds_avprc_dt_val='${price_date}' "; allRows = getThirdPartyDatas(sql) } if(allRows && allRows.size()>0){ @@ -367,7 +366,7 @@ class ExportPgsqlBaseEvaluation { } println "BIGDATA_THIRD_PARTY_PRICE_GJ Êý¾ÝÎļþÊä³öÍê³É......" - CcbFtpUtils.ftpUpload(null, fileName, curDate) + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) } def getThirdPartyDatas(def sql){ @@ -381,10 +380,10 @@ class ExportPgsqlBaseEvaluation { data << r.comm_nm data << r.gujia_dtsr_id data << r.gujia_dtsr_dsc - data << r.gujia_avprc_dt_val - data << r.gujia_comm_avprc - data << r.gujia_comm_hgst_prc - data << r.gujia_comm_lwst_prc + data << r.gujiabds_avprc_dt_val + data << r.gujia_bds_comm_avprc + data << r.gujiabdscomm_hgst_prc + data << r.gujiabdscomm_lwst_prc data << r.data_udt_tm allRows << data }) @@ -399,17 +398,17 @@ class ExportPgsqlBaseEvaluation { def cityCode = cityCodeMap.get(city) if(new_dists){ def newstr = "('"+new_dists.join("','")+"')" - def sql1 = "select urbn_id,mdl_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_str,gujia_hsno,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_prd_val,gujia_wbt_floor,gujia_lyr_tp_val,gujia_tot_floor_num,gujia_bld_tp_val,gujia_bld_stc,gujiaconstructionarea,gujia_set_innr_area,gujia_acc_tp_val,gujia_room,gujia_dept,gujia_ktch_num,gujia_toliet,gujia_hs_use,gujia_indr_stc_val,gujia_twrd_val,gujia_dcrt_sttn,gujia_ases_bt_ln_num,gujia_trdtckt_prc,gujia_mdl_tot_prc,gujia_dcl_tot_prc,gujia_ases_tot_prc,gujiasigntonlinectrid,to_char(txn_dt,'yyyy-mm-dd') as txn_dt,gujia_txn_tp_val,gujia_sell_pymtmod,gujia_mdl_cyc,gujiawthrext_lift_ind,gujiagvlndfargrdnarea,gujia_gv_lu_tbl_area,gujia_gv_sun_tbl_area,gujiagvbaywindow_area,gujiagvarea_tower_flr,gujia_gv_grg_area,gujiagvundgdroom_area,gujia_gv_othr_area,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_mkdam,gujia_data_tm " + + def sql1 = "select urbn_id,gujia_bds_mdl_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_str,gujia_hsno,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bds_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_prd_val,gujia_bds_wbt_floor,gujia_lyr_tp_val,gujiabdstot_floor_num,gujia_bld_tp_val,gujia_bld_stc,guiabconstructionarea,gujiabdsset_innr_area,gujia_acc_tp_val,gujia_bds_room_num,gujia_bds_dept_num,gujia_bds_ktch_num,gujia_bds_hlt_btw_num,gujia_hs_use,gujia_indr_stc_val,gujia_twrd_val,gujia_dcrt_sttn,gujia_ases_bt_ln_num,gujia_bds_trdtckt_prc,gujia_bds_mdl_tot_prc,gujia_bds_dcl_tot_prc,gujiabds_ases_tot_prc,gujiasigntonlinectrid,to_char(txn_dt,'yyyy-mm-dd') as txn_dt,gujia_txn_tp_val,gujia_sell_pymtmod,gujia_bds_mdl_cyc,gujiawthrext_lift_ind,gujiabdslndfgrngvarea,gujiabdslutbl_gv_area,gujiabdssuntblgv_area,gujiabdsbayindowgarea,gujiabdstowerfrgvarea,gujia_bds_grg_gv_area,gujiabdsundgdrmgvarea,gujiabds_othr_gv_area,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_bds_mkdam,gujia_bds_data_tm_val " + "from "+database+".t0831_bigdata_deal_gj_h where urbn_id='${cityCode}' and txn_dt>=to_date('${dealDate}','yyyy-MM-dd') and gujia_admndiv_id not in ${newstr}" def rets1 = dealData(sql1) - def sql2 = "select urbn_id,mdl_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_str,gujia_hsno,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_prd_val,gujia_wbt_floor,gujia_lyr_tp_val,gujia_tot_floor_num,gujia_bld_tp_val,gujia_bld_stc,gujiaconstructionarea,gujia_set_innr_area,gujia_acc_tp_val,gujia_room,gujia_dept,gujia_ktch_num,gujia_toliet,gujia_hs_use,gujia_indr_stc_val,gujia_twrd_val,gujia_dcrt_sttn,gujia_ases_bt_ln_num,gujia_trdtckt_prc,gujia_mdl_tot_prc,gujia_dcl_tot_prc,gujia_ases_tot_prc,gujiasigntonlinectrid,to_char(txn_dt,'yyyy-mm-dd') as txn_dt,gujia_txn_tp_val,gujia_sell_pymtmod,gujia_mdl_cyc,gujiawthrext_lift_ind,gujiagvlndfargrdnarea,gujia_gv_lu_tbl_area,gujia_gv_sun_tbl_area,gujiagvbaywindow_area,gujiagvarea_tower_flr,gujia_gv_grg_area,gujiagvundgdroom_area,gujia_gv_othr_area,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_mkdam,gujia_data_tm " + + def sql2 = "select urbn_id,gujia_bds_mdl_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_str,gujia_hsno,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bds_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_prd_val,gujia_bds_wbt_floor,gujia_lyr_tp_val,gujiabdstot_floor_num,gujia_bld_tp_val,gujia_bld_stc,guiabconstructionarea,gujiabdsset_innr_area,gujia_acc_tp_val,gujia_bds_room_num,gujia_bds_dept_num,gujia_bds_ktch_num,gujia_bds_hlt_btw_num,gujia_hs_use,gujia_indr_stc_val,gujia_twrd_val,gujia_dcrt_sttn,gujia_ases_bt_ln_num,gujia_bds_trdtckt_prc,gujia_bds_mdl_tot_prc,gujia_bds_dcl_tot_prc,gujiabds_ases_tot_prc,gujiasigntonlinectrid,to_char(txn_dt,'yyyy-mm-dd') as txn_dt,gujia_txn_tp_val,gujia_sell_pymtmod,gujia_bds_mdl_cyc,gujiawthrext_lift_ind,gujiabdslndfgrngvarea,gujiabdslutbl_gv_area,gujiabdssuntblgv_area,gujiabdsbayindowgarea,gujiabdstowerfrgvarea,gujia_bds_grg_gv_area,gujiabdsundgdrmgvarea,gujiabds_othr_gv_area,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_bds_mkdam,gujia_bds_data_tm_val " + "from "+database+".t0831_bigdata_deal_gj_h where urbn_id='${cityCode}' and gujia_admndiv_id in ${newstr}" def rets2 = dealData(sql2) allRows.addAll(rets1) allRows.addAll(rets2) }else{ - def sql = "select urbn_id,mdl_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_str,gujia_hsno,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_prd_val,gujia_wbt_floor,gujia_lyr_tp_val,gujia_tot_floor_num,gujia_bld_tp_val,gujia_bld_stc,gujiaconstructionarea,gujia_set_innr_area,gujia_acc_tp_val,gujia_room,gujia_dept,gujia_ktch_num,gujia_toliet,gujia_hs_use,gujia_indr_stc_val,gujia_twrd_val,gujia_dcrt_sttn,gujia_ases_bt_ln_num,gujia_trdtckt_prc,gujia_mdl_tot_prc,gujia_dcl_tot_prc,gujia_ases_tot_prc,gujiasigntonlinectrid,to_char(txn_dt,'yyyy-mm-dd') as txn_dt,gujia_txn_tp_val,gujia_sell_pymtmod,gujia_mdl_cyc,gujiawthrext_lift_ind,gujiagvlndfargrdnarea,gujia_gv_lu_tbl_area,gujia_gv_sun_tbl_area,gujiagvbaywindow_area,gujiagvarea_tower_flr,gujia_gv_grg_area,gujiagvundgdroom_area,gujia_gv_othr_area,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_mkdam,gujia_data_tm " + + def sql = "select urbn_id,gujia_bds_mdl_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_str,gujia_hsno,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bds_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_prd_val,gujia_bds_wbt_floor,gujia_lyr_tp_val,gujiabdstot_floor_num,gujia_bld_tp_val,gujia_bld_stc,guiabconstructionarea,gujiabdsset_innr_area,gujia_acc_tp_val,gujia_bds_room_num,gujia_bds_dept_num,gujia_bds_ktch_num,gujia_bds_hlt_btw_num,gujia_hs_use,gujia_indr_stc_val,gujia_twrd_val,gujia_dcrt_sttn,gujia_ases_bt_ln_num,gujia_bds_trdtckt_prc,gujia_bds_mdl_tot_prc,gujia_bds_dcl_tot_prc,gujiabds_ases_tot_prc,gujiasigntonlinectrid,to_char(txn_dt,'yyyy-mm-dd') as txn_dt,gujia_txn_tp_val,gujia_sell_pymtmod,gujia_bds_mdl_cyc,gujiawthrext_lift_ind,gujiabdslndfgrngvarea,gujiabdslutbl_gv_area,gujiabdssuntblgv_area,gujiabdsbayindowgarea,gujiabdstowerfrgvarea,gujia_bds_grg_gv_area,gujiabdsundgdrmgvarea,gujiabds_othr_gv_area,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_bds_mkdam,gujia_bds_data_tm_val " + "from "+database+".t0831_bigdata_deal_gj_h where urbn_id='${cityCode}' and txn_dt>=to_date('${dealDate}','yyyy-MM-dd')"; allRows.addAll(dealData(sql)) } @@ -428,8 +427,7 @@ class ExportPgsqlBaseEvaluation { sqlInstance.eachRow(sql, { r -> def data = [] data << r.urbn_id - data << r.urbn_id - data << r.mdl_id + data << r.gujia_bds_mdl_id data << r.urbn_nm data << r.gujia_admndiv_id data << r.gujia_admndiv_nm @@ -445,45 +443,45 @@ class ExportPgsqlBaseEvaluation { data << r.gujia_unit_nm data << r.gujia_acc_id data << r.gujia_dtl_adr - data << r.gujia_bld_yr_rc + data << r.gujia_bds_bld_yr_rc data << r.gujia_own_val data << r.gujia_own_yrlmt_val data << r.gujia_prd_val - data << r.gujia_wbt_floor + data << r.gujia_bds_wbt_floor data << r.gujia_lyr_tp_val - data << r.gujia_tot_floor_num + data << r.gujiabdstot_floor_num data << r.gujia_bld_tp_val data << r.gujia_bld_stc - data << r.gujiaconstructionarea - data << r.gujia_set_innr_area + data << r.guiabconstructionarea + data << r.gujiabdsset_innr_area data << r.gujia_acc_tp_val - data << r.gujia_room - data << r.gujia_dept - data << r.gujia_ktch_num - data << r.gujia_toliet + data << r.gujia_bds_room_num + data << r.gujia_bds_dept_num + data << r.gujia_bds_ktch_num + data << r.gujia_bds_hlt_btw_num data << r.gujia_hs_use data << r.gujia_indr_stc_val data << r.gujia_twrd_val data << r.gujia_dcrt_sttn data << r.gujia_ases_bt_ln_num - data << r.gujia_trdtckt_prc - data << r.gujia_mdl_tot_prc - data << r.gujia_dcl_tot_prc - data << r.gujia_ases_tot_prc + data << r.gujia_bds_trdtckt_prc + data << r.gujia_bds_mdl_tot_prc + data << r.gujia_bds_dcl_tot_prc + data << r.gujiabds_ases_tot_prc data << r.gujiasigntonlinectrid data << r.txn_dt data << r.gujia_txn_tp_val data << r.gujia_sell_pymtmod - data << r.gujia_mdl_cyc + data << r.gujia_bds_mdl_cyc data << r.gujiawthrext_lift_ind - data << r.gujiagvlndfargrdnarea - data << r.gujia_gv_lu_tbl_area - data << r.gujia_gv_sun_tbl_area - data << r.gujiagvbaywindow_area - data << r.gujiagvarea_tower_flr - data << r.gujia_gv_grg_area - data << r.gujiagvundgdroom_area - data << r.gujia_gv_othr_area + data << r.gujiabdslndfgrngvarea + data << r.gujiabdslutbl_gv_area + data << r.gujiabdssuntblgv_area + data << r.gujiabdsbayindowgarea + data << r.gujiabdstowerfrgvarea + data << r.gujia_bds_grg_gv_area + data << r.gujiabdsundgdrmgvarea + data << r.gujiabds_othr_gv_area data << r.gujia_hs_src_dtsrc data << r.intr_co_id data << r.intr_co_nm @@ -491,8 +489,8 @@ class ExportPgsqlBaseEvaluation { data << r.data_crt_tm data << r.data_udt_tm data << r.gujia_src_co - data << r.gujia_mkdam - data << r.gujia_data_tm + data << r.gujia_bds_mkdam + data << r.gujia_bds_data_tm_val allRows << data }); return allRows @@ -511,13 +509,13 @@ class ExportPgsqlBaseEvaluation { def allRows = [] if(new_dists){ def newstr = "('"+new_dists.join("','")+"')" - def sql1 = "select urbn_id,gujia_sell_id,gujia_listg_hs_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_unitprc,gujia_tot_prc,gujia_sell_pymtmod,gujia_wbt_floor,gujia_tot_floor_num,gujia_bld_tp_val,gujia_indr_stc_val,gujia_bld_stc,gujiaconstructionarea,gujia_set_innr_area,gujia_acc_tp_val,gujia_room,gujia_dept,gujia_ktch_num,gujia_toliet,gujia_twrd_val,gujia_dcrt_sttn,gujiawthrext_lift_ind,gujiajiladderjiactval,to_char(anc_tm,'yyyy-mm-dd') as anc_tm,to_char(exdat,'yyyy-mm-dd') as exdat,gujia_wthr_strt_ind,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,gujia_anc_psn,gujia_anc_psn_tel,gujia_dsc_inf_cntnt,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_apd_inf " + - "from "+database+".t0831_bigdata_forsale_gj_h where urbn_id='${cityCode}' and anc_tm>=to_date('${publishDate}','yyyy-MM-dd') and gujia_admndiv_id not in ${newstr}"; + def sql1 = "select urbn_id,gujia_sell_id,gujia_listg_hs_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bds_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_bds_unitprc,gujia_bds_tot_prc,gujia_sell_pymtmod,gujia_bds_wbt_floor,gujiabdstot_floor_num,gujia_bld_tp_val,gujia_indr_stc_val,gujia_bld_stc,guiabconstructionarea,gujiabdsset_innr_area,gujia_acc_tp_val,gujia_bds_room_num,gujia_bds_dept_num,gujia_bds_ktch_num,gujia_bds_hlt_btw_num,gujia_twrd_val,gujia_dcrt_sttn,gujiawthrext_lift_ind,gujiajiladderjiactval,to_char(anc_tm,'yyyy-mm-dd') as anc_tm,to_char(exdat,'yyyy-mm-dd') as exdat,gujia_wthr_strt,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,gujia_anc_psn,gujia_anc_psn_tel,gujia_dsc_inf_cntnt,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_apd_inf " + + "from "+database+".t0831_bigdata_forsale_gj_h where urbn_id='${cityCode}' and data_udt_tm>=to_date('${publishDate}','yyyy-MM-dd') and gujia_admndiv_id not in ${newstr}"; def rets1 = getHouseDatas(sql1) allRows.addAll(rets1) }else{ - def sql = "select urbn_id,gujia_sell_id,gujia_listg_hs_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_unitprc,gujia_tot_prc,gujia_sell_pymtmod,gujia_wbt_floor,gujia_tot_floor_num,gujia_bld_tp_val,gujia_indr_stc_val,gujia_bld_stc,gujiaconstructionarea,gujia_set_innr_area,gujia_acc_tp_val,gujia_room,gujia_dept,gujia_ktch_num,gujia_toliet,gujia_twrd_val,gujia_dcrt_sttn,gujiawthrext_lift_ind,gujiajiladderjiactval,to_char(anc_tm,'yyyy-mm-dd') as anc_tm,to_char(exdat,'yyyy-mm-dd') as exdat,gujia_wthr_strt_ind,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,gujia_anc_psn,gujia_anc_psn_tel,gujia_dsc_inf_cntnt,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_apd_inf " + - "from "+database+".t0831_bigdata_forsale_gj_h where urbn_id='${cityCode}' and anc_tm>=to_date('${publishDate}','yyyy-MM-dd')"; + def sql = "select urbn_id,gujia_sell_id,gujia_listg_hs_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bds_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_bds_unitprc,gujia_bds_tot_prc,gujia_sell_pymtmod,gujia_bds_wbt_floor,gujiabdstot_floor_num,gujia_bld_tp_val,gujia_indr_stc_val,gujia_bld_stc,guiabconstructionarea,gujiabdsset_innr_area,gujia_acc_tp_val,gujia_bds_room_num,gujia_bds_dept_num,gujia_bds_ktch_num,gujia_bds_hlt_btw_num,gujia_twrd_val,gujia_dcrt_sttn,gujiawthrext_lift_ind,gujiajiladderjiactval,to_char(anc_tm,'yyyy-mm-dd') as anc_tm,to_char(exdat,'yyyy-mm-dd') as exdat,gujia_wthr_strt,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,gujia_anc_psn,gujia_anc_psn_tel,gujia_dsc_inf_cntnt,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_apd_inf " + + "from "+database+".t0831_bigdata_forsale_gj_h where urbn_id='${cityCode}' and data_udt_tm>=to_date('${publishDate}','yyyy-MM-dd')"; allRows = getHouseDatas(sql) } if(allRows.size()>0){ @@ -533,7 +531,7 @@ class ExportPgsqlBaseEvaluation { if(new_dists){//²¹³ä±»ÐÞÇøÓòµÄÊý¾Ý def newstr = "('"+new_dists.join("','")+"')" - def sql2 = "select urbn_id,gujia_sell_id,gujia_listg_hs_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_unitprc,gujia_tot_prc,gujia_sell_pymtmod,gujia_wbt_floor,gujia_tot_floor_num,gujia_bld_tp_val,gujia_indr_stc_val,gujia_bld_stc,gujiaconstructionarea,gujia_set_innr_area,gujia_acc_tp_val,gujia_room,gujia_dept,gujia_ktch_num,gujia_toliet,gujia_twrd_val,gujia_dcrt_sttn,gujiawthrext_lift_ind,gujiajiladderjiactval,to_char(anc_tm,'yyyy-mm-dd') as anc_tm,to_char(exdat,'yyyy-mm-dd') as exdat,gujia_wthr_strt_ind,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,gujia_anc_psn,gujia_anc_psn_tel,gujia_dsc_inf_cntnt,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_apd_inf " + + def sql2 = "select urbn_id,gujia_sell_id,gujia_listg_hs_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bds_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_bds_unitprc,gujia_bds_tot_prc,gujia_sell_pymtmod,gujia_bds_wbt_floor,gujiabdstot_floor_num,gujia_bld_tp_val,gujia_indr_stc_val,gujia_bld_stc,guiabconstructionarea,gujiabdsset_innr_area,gujia_acc_tp_val,gujia_bds_room_num,gujia_bds_dept_num,gujia_bds_ktch_num,gujia_bds_hlt_btw_num,gujia_twrd_val,gujia_dcrt_sttn,gujiawthrext_lift_ind,gujiajiladderjiactval,to_char(anc_tm,'yyyy-mm-dd') as anc_tm,to_char(exdat,'yyyy-mm-dd') as exdat,gujia_wthr_strt,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,gujia_anc_psn,gujia_anc_psn_tel,gujia_dsc_inf_cntnt,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_apd_inf " + "from "+database+".t0831_bigdata_forsale_gj_h where gujia_admndiv_id in ${newstr}" def rets2 = getHouseDatas(sql2) if(rets2.size()>0){ @@ -541,7 +539,7 @@ class ExportPgsqlBaseEvaluation { } } println "BIGDATA_FORSALE_GJ Êý¾ÝÎļþÊä³öÍê³É......" - CcbFtpUtils.ftpUpload(null, fileName, curDate) + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) println "BIGDATA_FORSALE_GJ ÉÏ´«FtpÍê³É......" } @@ -568,31 +566,31 @@ class ExportPgsqlBaseEvaluation { data << r.gujia_unit_nm data << r.gujia_acc_id data << r.gujia_dtl_adr - data << r.gujia_bld_yr_rc + data << r.gujia_bds_bld_yr_rc data << r.gujia_own_val data << r.gujia_own_yrlmt_val - data << r.gujia_unitprc - data << r.gujia_tot_prc + data << r.gujia_bds_unitprc + data << r.gujia_bds_tot_prc data << r.gujia_sell_pymtmod - data << r.gujia_wbt_floor - data << r.gujia_tot_floor_num + data << r.gujia_bds_wbt_floor + data << r.gujiabdstot_floor_num data << r.gujia_bld_tp_val data << r.gujia_indr_stc_val data << r.gujia_bld_stc - data << r.gujiaconstructionarea - data << r.gujia_set_innr_area + data << r.guiabconstructionarea + data << r.gujiabdsset_innr_area data << r.gujia_acc_tp_val - data << r.gujia_room - data << r.gujia_dept - data << r.gujia_ktch_num - data << r.gujia_toliet + data << r.gujia_bds_room_num + data << r.gujia_bds_dept_num + data << r.gujia_bds_ktch_num + data << r.gujia_bds_hlt_btw_num data << r.gujia_twrd_val data << r.gujia_dcrt_sttn data << r.gujiawthrext_lift_ind data << r.gujiajiladderjiactval data << r.anc_tm data << r.exdat - data << r.gujia_wthr_strt_ind + data << r.gujia_wthr_strt data << r.gujia_hs_src_dtsrc data << r.intr_co_id data << r.intr_co_nm @@ -617,7 +615,7 @@ class ExportPgsqlBaseEvaluation { def createFileToFtp(tableName,allRows, fileName, curDate, isAppend){ FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", isAppend); println tableName+" Êý¾ÝÎļþÊä³öÍê³É......" -// CcbFtpUtils.ftpUpload(null, fileName, curDate) -// println tableName+" ÉÏ´«FtpÍê³É......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println tableName+" ÉÏ´«FtpÍê³É......" } } From a3f6e0c37456262b43ed53b5a0ced1029b1b1e7a Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 22 Feb 2023 15:17:06 +0800 Subject: [PATCH 92/99] update --- .../pgsql/ExportPgsqlBaseEvaluation.groovy | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlBaseEvaluation.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlBaseEvaluation.groovy index 2c3853f3f..e1bb0d430 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlBaseEvaluation.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlBaseEvaluation.groovy @@ -10,7 +10,7 @@ import groovy.sql.Sql class ExportPgsqlBaseEvaluation { - def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse", + def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse", "postgres", "postgres", "org.postgresql.Driver") def citys = null def cityCodeMap = CcbCommonUtils.findCityCodeValue() @@ -55,7 +55,7 @@ class ExportPgsqlBaseEvaluation { def exportEvaluationRateData() { def sql = "select gujia_ceft_id,gujia_ceft_tp,gujia_bds_ceft,gujia_bds_seq, gujia_urbn,rmrk_dsc, to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm, gujia_src_co from "+database+".t0831_evaluate_rate_gj_h"; def allRows = [] - sqlInstance.eachRow(sql, { r -> + pgsqlInstance.eachRow(sql, { r -> def data = [] data << r.gujia_ceft_id data << r.gujia_ceft_tp @@ -82,7 +82,7 @@ class ExportPgsqlBaseEvaluation { for(def city : citys) { def cityCode = cityCodeMap.get(city) def sql = "select urbn_id,gujia_admndiv_id,gujia_fsid,poi_nm,gujia_tp,gujia_tp_ecd,gujia_bds_ltt,gujia_bds_lgt,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+".t0831_bigdata_poi_gj_h where urbn_id='${cityCode}'"; - sqlInstance.eachRow(sql, { r -> + pgsqlInstance.eachRow(sql, { r -> def data = [] data << r.urbn_id data << r.gujia_admndiv_id @@ -113,7 +113,7 @@ class ExportPgsqlBaseEvaluation { for(def city : citys) { def cityCode = cityCodeMap.get(city) def sql = "select gujia_admndiv_id,gujia_admndiv_nm,gujia_admndiv_nm_cpa,urbn_id,gujia_natstd_cd,gujia_dspl_seq,gujia_admndiv_cgy,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_lgc_admndiv_dsc,gujia_lgc_admndiv_cd,gujia_adiv_cd_val,gujia_bds_data_tm_val from "+database+".t0831_bigdata_district_gj_h where urbn_id='${cityCode}'"; - sqlInstance.eachRow(sql, { r -> + pgsqlInstance.eachRow(sql, { r -> def data = [] data << r.gujia_admndiv_id data << r.gujia_admndiv_nm @@ -145,7 +145,7 @@ class ExportPgsqlBaseEvaluation { def exportWebsiteRate(){ def sql = "select gujiadatawebst_src_id,gujia_data_webst_src,gujiadatawebstmpng_id,gujia_bds_wgt,gujia_bds_tp,gujia_src_co,urbn_id,gujia_urbn,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm, to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm from "+database+".t0831_website_config_gj_h"; def allRows = [] - sqlInstance.eachRow(sql, { r -> + pgsqlInstance.eachRow(sql, { r -> def data = [] data << r.gujiadatawebst_src_id data << r.gujia_data_webst_src @@ -174,7 +174,7 @@ class ExportPgsqlBaseEvaluation { for(def city : citys) { def cityCode = cityCodeMap.get(city) def sql = "select gujia_fsid,gujia_bds_id,comm_nm,urbn_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,to_char(gujia_drc_dt,'yyyy-mm-dd') as gujia_drc_dt,gujia_bds_drc_prc,gujia_add_psn,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+".t0831_guided_price_gj_h where urbn_id='${cityCode}'"; - sqlInstance.eachRow(sql, { r -> + pgsqlInstance.eachRow(sql, { r -> def data = [] data << r.gujia_fsid data << r.gujia_bds_id @@ -207,7 +207,7 @@ class ExportPgsqlBaseEvaluation { for(def city : citys) { def cityCode = cityCodeMap.get(city) def sql = "select gujia_fsid,gujia_bds_id, comm_nm, urbn_id, urbn_nm, gujia_admndiv_id, gujia_admndiv_nm, gujia_bds_drc_prc, gujia_bds_drc_prc_src, rmrk_dsc, to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm, to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm, gujia_src_co from "+database+".t0831_community_guide_price_gj_h where urbn_id='${cityCode}' and data_crt_tm>=to_date('${date}','yyyyMMdd')"; - sqlInstance.eachRow(sql, { r -> + pgsqlInstance.eachRow(sql, { r -> def data = [] data << r.gujia_fsid data << r.gujia_bds_id @@ -239,7 +239,7 @@ class ExportPgsqlBaseEvaluation { for(def city : citys) { def cityCode = cityCodeMap.get(city) def sql = "select urbn_id,gujia_admndiv_id,gujia_comm_id,gujia_admndiv_nm,comm_nm,gujia_dtl_adr,gujia_bds_ltt,gujia_bds_lgt,gujiabdsocpoldtotarea,gujiabds_bld_tot_area,gujia_bds_grnrate,gujia_bds_vol_rate,gujia_bds_park_num,gujia_bds_bld_yr_rc,gujia_bds_hs_use_val,totunits,gujia_bds_data_tm_val,gujiabds_udt_flag_val,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_comm_tp_val,gujia_comm_fmnm,gujia_str,gujia_hsno,gujiawthr_new_pan_ind,gujia_comm_intd,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,gujia_src_co,gujia_vlgtwn_str_mng,gujia_lgc_admndiv_cd,gujia_lgc_adiv,gujia_bds_bds_st from "+database+".t0831_bigdata_community_gj_h where urbn_id='${cityCode}' ";//where cityId in (350200,150100) - sqlInstance.eachRow(sql,{r-> + pgsqlInstance.eachRow(sql,{r-> def data = [] data << r.urbn_id data << r.gujia_admndiv_id @@ -287,7 +287,7 @@ class ExportPgsqlBaseEvaluation { def allRows = [] def sql = "select gujia_instm_id,gujia_comm_id,gujiacomm_prim_prd_id,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+".t0831_community_staging_gj_h"; - sqlInstance.eachRow(sql, { r -> + pgsqlInstance.eachRow(sql, { r -> def data = [] data << r.gujia_instm_id data << r.gujia_comm_id @@ -310,7 +310,7 @@ class ExportPgsqlBaseEvaluation { for(def city : citys) { def cityCode = cityCodeMap.get(city) def sql = "select urbn_id,gujia_comm_id,gujia_bld_tp_val,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_bds_data_tm_val from "+database+".t0831_bigdata_commbuildtype_gj_h where urbn_id='${cityCode}'"; - sqlInstance.eachRow(sql, { r -> + pgsqlInstance.eachRow(sql, { r -> def data = [] data << r.urbn_id data << r.gujia_comm_id @@ -371,7 +371,7 @@ class ExportPgsqlBaseEvaluation { def getThirdPartyDatas(def sql){ def allRows = [] - sqlInstance.eachRow(sql, { r -> + pgsqlInstance.eachRow(sql, { r -> def data = [] data << r.gujia_trdpt_avprc_id data << r.urbn_id @@ -424,7 +424,7 @@ class ExportPgsqlBaseEvaluation { def dealData(def sql){ def allRows = [] - sqlInstance.eachRow(sql, { r -> + pgsqlInstance.eachRow(sql, { r -> def data = [] data << r.urbn_id data << r.gujia_bds_mdl_id @@ -546,7 +546,7 @@ class ExportPgsqlBaseEvaluation { def getHouseDatas(sql){ def allRows = [] def index = 0 - sqlInstance.eachRow(sql,{r-> + pgsqlInstance.eachRow(sql,{r-> index = index+1 println r.urbn_nm +" "+index def data = [] From b5facb4a87ff12dcdc8bfd792a1d653313aa8092 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 22 Feb 2023 17:59:18 +0800 Subject: [PATCH 93/99] update --- .../pgsql/ExportPgsqlBaseEvaluation.groovy | 35 ++-- .../pgsql/ExportPgsqlEvaluationResult.groovy | 189 +++++++++--------- 2 files changed, 113 insertions(+), 111 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlBaseEvaluation.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlBaseEvaluation.groovy index e1bb0d430..70a2a1041 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlBaseEvaluation.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlBaseEvaluation.groovy @@ -27,29 +27,30 @@ class ExportPgsqlBaseEvaluation { def run(args) { citys = CcbCommonUtils.getRunningCitys(args) + citys = citys.reverse() def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")); def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); def lastMonthStr = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM") def new_dists = [] - exportEvaluationRateData() - exportBigdataPoiData() - exportBigdataDistrictData() - - //µÚÈý·½ÍøÕ¾È¨Öرí - exportWebsiteRate(); - - exportGuidedPrice()//ËæÈ˹¤¸Ä±ä// - exportCommunityGuidedPrice(lastMonth.toString()+"28")//ËæÕþ²ß// - //´¦ÀíÐ¡ÇøÇé¿ö - exportCommunityStaging()//ûcityid - exportBigdataCommunity()// - - exportCommunityBuildType()// - - exportThirdPartyPrice2(lastMonth, new_dists)// +// exportEvaluationRateData() +// exportBigdataPoiData() +// exportBigdataDistrictData() +// +// //µÚÈý·½ÍøÕ¾È¨Öرí +// exportWebsiteRate(); +// +// exportGuidedPrice()//ËæÈ˹¤¸Ä±ä// +// exportCommunityGuidedPrice(lastMonth.toString()+"28")//ËæÕþ²ß// +// //´¦ÀíÐ¡ÇøÇé¿ö +// exportCommunityStaging()//ûcityid +// exportBigdataCommunity()// +// +// exportCommunityBuildType()// +// +// exportThirdPartyPrice2(lastMonth, new_dists)// exportBigdataForsale(lastMonthStr +"-18", new_dists) //ÏÖÔÚ°´updatetimeµ¼³ö - exportBigdataDeal(lastMonthStr + "-01", new_dists) //2022-07-01 +// exportBigdataDeal(lastMonthStr + "-01", new_dists) //2022-07-01 } def exportEvaluationRateData() { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlEvaluationResult.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlEvaluationResult.groovy index 24254df78..a5321a3e8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlEvaluationResult.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlEvaluationResult.groovy @@ -14,7 +14,7 @@ import org.apache.commons.codec.digest.DigestUtils class ExportPgsqlEvaluationResult { - def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/postgres", + def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse", "postgres", "postgres", "org.postgresql.Driver") def citys = null def cityCodeMap = CcbCommonUtils.findCityCodeValue() @@ -34,17 +34,17 @@ class ExportPgsqlEvaluationResult { def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")); //¼ÆËã½á¹ûÏà¹Ø -// exportAvmEvaluationGJNB(curMonth) -// exportAVMEvaluationItemGJNB(curMonth) -// -// exportCityPrice(curMonth) -// exportRegionPrice(curMonth) -// -// exportCommunityBasePrice(curMonth) -// -// exportCommunityPdescribeGj(curMonth) -// exportCommCaseDetailGj(curMonth) -// + exportAvmEvaluationGJNB(curMonth) + exportAVMEvaluationItemGJNB(curMonth) + + exportCityPrice(curMonth) + exportRegionPrice(curMonth) + + exportCommunityBasePrice(curMonth) + + exportCommunityPdescribeGj(curMonth) + exportCommCaseDetailGj(curMonth) + exportInOk() SSHHelperTest jschCommand = new SSHHelperTest(); @@ -57,36 +57,36 @@ class ExportPgsqlEvaluationResult { def allRows = [] for(def city : citys) { def cityCode = cityCodeMap.get(city) - def sql = "select gujia_fsid,urbn_id,urbn_nm,gujia_yrmo,gujia_comm_num,gujia_val_comm_num,gujia_prc_cvr_rate,gujiamnldrcprccommnum,gujia_mnl_drc_prc_pct,gujia_samp_comm_num,gujiaext_samp_clc_pct,gujianervethruncomnum,gujianervethrunclcpct,gujia_tst_case_num,gujia_tst_comm_num,gujiatst_comm_num_pct,gujia_ppe15_cvr_rate,gujia_rto_mdn,gujia_rto_lv_san_ceft,gujiamdl_tst_case_num,gujiamdl_tst_comm_num,gujiamdltstcommnumpct,gujiamdltst_ppe15_pct,gujiamdlrtolvsan_ceft,gujia_mdl_md_bit_rto,gujiatrdpttstcase_num,gujiatrdpttstcomm_num,gujiatrdpttcommnumpct,gujiatrdpttstppe15pct,gujiatrdptrolvsanceft,gujiatrdpt_md_bit_rto,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"avm_evaluation_gj_h where gujia_yrmo = ${yearMonth} and urbn_id='${cityCode}'"; - sqlInstance.eachRow(sql, { r -> + def sql = "select gujia_fsid,urbn_id,urbn_nm,gujia_bds_yrmo,gujia_bds_comm_num,gujiabdsvalmnycommnum,gujia_prc_cvr_rate,gujiabdsmnldpccommnum,gujia_mnl_drc_prc_pct,gujiabdssamp_comm_num,gujiaext_samp_clc_pct,gujiabdsnervetncomnum,gujianervethrunclcpct,gujiabds_tst_case_num,gujiabds_tst_comm_num,gujiatst_comm_num_pct,gujia_ppe15_cvr_rate,gujia_rto_mdn,gujia_rto_lv_san_ceft,gujiabdsmdltstcasenum,gujiabdsmdltstcommnum,gujiamdltstcommnumpct,gujiamdltst_ppe15_pct,gujiamdlrtolvsan_ceft,gujia_mdl_md_bit_rto,gujiabdstrdpttcasenum,gujiabdstrdpttcommnum,gujiatrdpttcommnumpct,gujiatrdpttstppe15pct,gujiatrdptrolvsanceft,gujiatrdpt_md_bit_rto,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"avm_evaluation_gj_h where gujia_bds_yrmo = ${yearMonth} and urbn_id='${cityCode}'"; + pgsqlInstance.eachRow(sql, { r -> def data = [] data << r.gujia_fsid data << r.urbn_id data << r.urbn_nm - data << r.gujia_yrmo - data << r.gujia_comm_num - data << r.gujia_val_comm_num + data << r.gujia_bds_yrmo + data << r.gujia_bds_comm_num + data << r.gujiabdsvalmnycommnum data << r.gujia_prc_cvr_rate - data << r.gujiamnldrcprccommnum + data << r.gujiabdsmnldpccommnum data << r.gujia_mnl_drc_prc_pct - data << r.gujia_samp_comm_num + data << r.gujiabdssamp_comm_num data << r.gujiaext_samp_clc_pct - data << r.gujianervethruncomnum + data << r.gujiabdsnervetncomnum data << r.gujianervethrunclcpct - data << r.gujia_tst_case_num - data << r.gujia_tst_comm_num + data << r.gujiabds_tst_case_num + data << r.gujiabds_tst_comm_num data << r.gujiatst_comm_num_pct data << r.gujia_ppe15_cvr_rate data << r.gujia_rto_mdn data << r.gujia_rto_lv_san_ceft - data << r.gujiamdl_tst_case_num - data << r.gujiamdl_tst_comm_num + data << r.gujiabdsmdltstcasenum + data << r.gujiabdsmdltstcommnum data << r.gujiamdltstcommnumpct data << r.gujiamdltst_ppe15_pct data << r.gujiamdlrtolvsan_ceft data << r.gujia_mdl_md_bit_rto - data << r.gujiatrdpttstcase_num - data << r.gujiatrdpttstcomm_num + data << r.gujiabdstrdpttcasenum + data << r.gujiabdstrdpttcommnum data << r.gujiatrdpttcommnumpct data << r.gujiatrdpttstppe15pct data << r.gujiatrdptrolvsanceft @@ -115,8 +115,8 @@ class ExportPgsqlEvaluationResult { for(def city : citys) { def cityCode = cityCodeMap.get(city) - def sql = "select gujia_fsid,urbn_id,urbn_nm,gujia_rgon_id,gujia_rgon_nm,gujia_comm_id,gujia_comm_nm,gujia_clc_yrmo,gujia_clc_avprc,gujia_avprc,gujia_clc_tp_val,gujia_rlb,gujia_to_mkprc_mny, to_char(mdl_dt,'yyyy-mm-dd') as mdl_dt,gujia_valt_unitprc,gujia_area,gujia_dcrt_val,gujia_twrd_val,gujia_wbt_floor,gujia_tot_floor,gujia_data_tp,gujia_rto,gujia_mns_rto, to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm from "+database+"avm_evaluation_item_gj_h where gujia_clc_yrmo = ${yearMonth} AND urbn_id='${cityCode}'"; - sqlInstance.eachRow(sql, { r -> + def sql = "select gujia_fsid,urbn_id,urbn_nm,gujia_rgon_id,gujia_rgon_nm,gujia_comm_id,gujia_comm_nm,gujiabds_clc_yrmo_val,gujia_bds_clc_avprc,gujia_bds_avprc,gujia_bds_clc_tp,gujia_bds_rlb,gujiabds_to_mkprc_mny,to_char(mdl_dt,'yyyy-mm-dd') as mdl_dt,gujiabds_valt_unitprc,gujia_area,gujia_dcrt_val,gujia_twrd_val,gujia_bds_floor_num,gujiabdstot_floor_num,gujia_data_tp,gujia_rto,gujia_mns_rto,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') AS data_crt_tm from "+database+"avm_evaluation_item_gj_h where gujiabds_clc_yrmo_val = ${yearMonth} AND urbn_id='${cityCode}'"; + pgsqlInstance.eachRow(sql, { r -> def data = [] data << r.gujia_fsid data << r.urbn_id @@ -125,19 +125,19 @@ class ExportPgsqlEvaluationResult { data << r.gujia_rgon_nm data << r.gujia_comm_id data << r.gujia_comm_nm - data << r.gujia_clc_yrmo - data << r.gujia_clc_avprc - data << r.gujia_avprc - data << r.gujia_clc_tp_val - data << r.gujia_rlb - data << r.gujia_to_mkprc_mny + data << r.gujiabds_clc_yrmo_val + data << r.gujia_bds_clc_avprc + data << r.gujia_bds_avprc + data << r.gujia_bds_clc_tp + data << r.gujia_bds_rlb + data << r.gujiabds_to_mkprc_mny data << r.mdl_dt - data << r.gujia_valt_unitprc + data << r.gujiabds_valt_unitprc data << r.gujia_area data << r.gujia_dcrt_val data << r.gujia_twrd_val - data << r.gujia_wbt_floor - data << r.gujia_tot_floor + data << r.gujia_bds_floor_num + data << r.gujiabdstot_floor_num data << r.gujia_data_tp data << r.gujia_rto data << r.gujia_mns_rto @@ -161,15 +161,15 @@ class ExportPgsqlEvaluationResult { def allRows = [] for(def city : citys) { def cityCode = cityCodeMap.get(city) - def sql = "select gujia_trdpt_avprc_id,urbn_id,urbn_nm,gujia_clc_yrmo,gujia_clc_avprc,gujia_urbn_avprc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"avm_city_price_gj_h where gujia_clc_yrmo=${month} AND urbn_id='${cityCode}'"; - sqlInstance.eachRow(sql, { r -> + def sql = "select gujia_trdpt_avprc_id,urbn_id,urbn_nm,gujiabds_clc_yrmo_val,gujia_bds_clc_avprc,gujia_bds_urbn_avprc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"avm_city_price_gj_h where gujiabds_clc_yrmo_val=${month} AND urbn_id='${cityCode}'"; + pgsqlInstance.eachRow(sql, { r -> def data = [] data << r.gujia_trdpt_avprc_id data << r.urbn_id data << r.urbn_nm - data << r.gujia_clc_yrmo - data << r.gujia_clc_avprc - data << r.gujia_urbn_avprc + data << r.gujiabds_clc_yrmo_val + data << r.gujia_bds_clc_avprc + data << r.gujia_bds_urbn_avprc data << r.data_crt_tm data << r.data_udt_tm data << r.gujia_src_co @@ -188,18 +188,18 @@ class ExportPgsqlEvaluationResult { def allRows = [] for(def city : citys) { def cityCode = cityCodeMap.get(city) - def sql = "select gujia_trdpt_avprc_id,urbn_id,urbn_nm,gujia_admndiv_id,gxda_admndiv_nm,gujia_clc_yrmo,gujia_clc_avprc,gujia_urbn_avprc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"avm_district_price_gj_h where gujia_clc_yrmo=${month} AND urbn_id='${cityCode}'"; + def sql = "select gujia_trdpt_avprc_id,urbn_id,urbn_nm,gujia_admndiv_id,gxda_admndiv_nm,gujiabds_clc_yrmo_val,gujia_bds_clc_avprc,gujia_bds_urbn_avprc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"avm_district_price_gj_h where gujiabds_clc_yrmo_val=${month} AND urbn_id='${cityCode}'"; - sqlInstance.eachRow(sql, { r -> + pgsqlInstance.eachRow(sql, { r -> def data = [] data << r.gujia_trdpt_avprc_id data << r.urbn_id data << r.urbn_nm data << r.gujia_admndiv_id data << r.gxda_admndiv_nm - data << r.gujia_clc_yrmo - data << r.gujia_clc_avprc - data << r.gujia_urbn_avprc + data << r.gujiabds_clc_yrmo_val + data << r.gujia_bds_clc_avprc + data << r.gujia_bds_urbn_avprc data << r.data_crt_tm data << r.data_udt_tm data << r.gujia_src_co @@ -219,8 +219,8 @@ class ExportPgsqlEvaluationResult { def allRows = [] for(def city : citys) { def cityCode = cityCodeMap.get(city) - def sql = "select gujia_bss_prc_id,urbn_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_bds_id,comm_nm,gujia_clc_yrmo,gujia_clc_avprc,gujia_comm_basic_prc,gujia_clmtd_val,gujia_fwd_samp_rto,gujia_mns_grd,gujia_cfdrg,gujiaarndcommprc_crlt,gujia_cfdc_dgr_val,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"avm_community_base_price_gj_h where gujia_clc_yrmo=${yearMonth} AND urbn_id='${cityCode}'"; - sqlInstance.eachRow(sql, { r -> + def sql = "select gujia_bss_prc_id,urbn_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_bds_id,comm_nm,gujiabds_clc_yrmo_val,gujia_bds_clc_avprc,gujia_comm_basic_prc,gujia_bds_clmtd,gujiabds_fwd_samp_rto,gujia_bds_mns_grd,gujia_bds_cfdrg,gujiabdsarndcmprccrlt,gujia_bds_cfdc_dgr,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"avm_community_base_price_gj_h where gujiabds_clc_yrmo_val=${yearMonth} AND urbn_id='${cityCode}'"; + pgsqlInstance.eachRow(sql, { r -> def data = [] data << r.gujia_bss_prc_id data << r.urbn_id @@ -229,15 +229,15 @@ class ExportPgsqlEvaluationResult { data << r.gujia_admndiv_nm data << r.gujia_bds_id data << r.comm_nm - data << r.gujia_clc_yrmo - data << r.gujia_clc_avprc + data << r.gujiabds_clc_yrmo_val + data << r.gujia_bds_clc_avprc data << r.gujia_comm_basic_prc - data << r.gujia_clmtd_val - data << r.gujia_fwd_samp_rto - data << r.gujia_mns_grd - data << r.gujia_cfdrg - data << r.gujiaarndcommprc_crlt - data << r.gujia_cfdc_dgr_val + data << r.gujia_bds_clmtd + data << r.gujiabds_fwd_samp_rto + data << r.gujia_bds_mns_grd + data << r.gujia_bds_cfdrg + data << r.gujiabdsarndcmprccrlt + data << r.gujia_bds_cfdc_dgr data << r.data_crt_tm data << r.data_udt_tm data << r.gujia_src_co @@ -264,9 +264,9 @@ class ExportPgsqlEvaluationResult { def allRows = [] for(def city : citys) { def cityCode = cityCodeMap.get(city) - def sql = "select gujia_fsid,gujia_comm_id,urbn_id,urbn_nm,admndiv_ecd,gujia_admndiv_nm,comm_nm,gujia_als_dsc,gujia_bss_prc,gujia_cfdc_dgr_grd,gujiart3mlistgcasenum,gujialistgcaselwstprc,gujialistgcase_md_val,gujia_listg_case_avg,gujialistgcasemax_prc,gujiaavprc_crpnd_s_mo,gujiaavgwithminvalavg,gujiaavgwithmaxvalavg,gujia_rel_err,gujia_dvtdgr_dsc,gujia_als_prc_rng,gujiaprcminandmax_rng,gujia_dsc,gujia_lmth_bss_prc,gujia_wthr_mnl_ind,gujia_lmth_fnl_prc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,gujianervethruntw_prc,gujia_prc_src,gujiaabtjanlcmdbitval,gujiaabtjanligcaseavg,gujia_indx_lbl,gujia_prim_pty_tp,gujia_comm_tp_val,gujia_ltst_mdl_prc,to_char(gujia_ltst_mdl_dt,'yyyy-mm-dd') as gujia_ltst_mdl_dt,gujia_idxn_mo_moonmo,gujiaabt1yiminmmoonmo,gujia_idxn_yr_moonmo,gujiaabt1yr_samp_scvl,gujiaarnd1kscommavprc from "+database+"avm_community_p_describe_gj_h where gujiaavprc_crpnd_s_mo=${month} AND urbn_id='${cityCode}'"; + def sql = "select gujia_fsid,gujia_comm_id,urbn_id,urbn_nm,admndiv_ecd,gujia_admndiv_nm,comm_nm,gujia_als_dsc,gujia_bds_bss_prc,gujiabds_cfdc_dgr_grd,gujiabdsrt3mlgcasenum,gujiabdslistgclwstprc,gujiabdslistgcaemdval,gujiabdslistgcase_avg,gujiabdslistgcemaxprc,gujia_bds_mo_val,gujiabdsavgwminvalavg,gujiabdsavgwmaxvalavg,gujia_bds_rel_err_val,gujia_dvtdgr_dsc,gujia_als_prc_rng,gujiaprcminandmax_rng,gujia_dsc,gujiabds_lmth_bss_prc,gujiabds_wthr_mnl_val,gujiabds_lmth_fnl_prc,to_char(data_crt_tm,'yyyy-mm-dd') as data_crt_tm,gujiabdsnervetuntwprc,gujia_prc_src,gujiabdsabtjlcmbitval,gujiabdsabtjlgcaseavg,gujia_indx_lbl,gujia_prim_pty_tp,gujia_comm_tp_val,gujiabds_ltst_mdl_prc,to_char(gujia_ltst_mdl_dt,'yyyy-mm-dd') as gujia_ltst_mdl_dt,gujiabdsidxnmo_moonmo,gujiabdsa1yiminmmonmo,gujiabdsidxnyr_moonmo,gujiabdsat1yspdgrscor,gujiabdsarnd1kscavprc from "+database+"avm_community_p_describe_gj_h where gujia_bds_mo_val=${month} AND urbn_id='${cityCode}'"; - sqlInstance.eachRow(sql, { r -> + pgsqlInstance.eachRow(sql, { r -> def data = [] data << r.gujia_fsid data << r.gujia_comm_id @@ -276,39 +276,39 @@ class ExportPgsqlEvaluationResult { data << r.gujia_admndiv_nm data << r.comm_nm data << r.gujia_als_dsc - data << r.gujia_bss_prc - data << r.gujia_cfdc_dgr_grd - data << r.gujiart3mlistgcasenum - data << r.gujialistgcaselwstprc - data << r.gujialistgcase_md_val - data << r.gujia_listg_case_avg - data << r.gujialistgcasemax_prc - data << r.gujiaavprc_crpnd_s_mo - data << r.gujiaavgwithminvalavg - data << r.gujiaavgwithmaxvalavg - data << r.gujia_rel_err + data << r.gujia_bds_bss_prc + data << r.gujiabds_cfdc_dgr_grd + data << r.gujiabdsrt3mlgcasenum + data << r.gujiabdslistgclwstprc + data << r.gujiabdslistgcaemdval + data << r.gujiabdslistgcase_avg + data << r.gujiabdslistgcemaxprc + data << r.gujia_bds_mo_val + data << r.gujiabdsavgwminvalavg + data << r.gujiabdsavgwmaxvalavg + data << r.gujia_bds_rel_err_val data << r.gujia_dvtdgr_dsc data << r.gujia_als_prc_rng data << r.gujiaprcminandmax_rng data << r.gujia_dsc - data << r.gujia_lmth_bss_prc - data << r.gujia_wthr_mnl_ind - data << r.gujia_lmth_fnl_prc + data << r.gujiabds_lmth_bss_prc + data << r.gujiabds_wthr_mnl_val + data << r.gujiabds_lmth_fnl_prc data << r.data_crt_tm - data << r.gujianervethruntw_prc + data << r.gujiabdsnervetuntwprc data << r.gujia_prc_src - data << r.gujiaabtjanlcmdbitval - data << r.gujiaabtjanligcaseavg + data << r.gujiabdsabtjlcmbitval + data << r.gujiabdsabtjlgcaseavg data << r.gujia_indx_lbl data << r.gujia_prim_pty_tp data << r.gujia_comm_tp_val - data << r.gujia_ltst_mdl_prc + data << r.gujiabds_ltst_mdl_prc data << r.gujia_ltst_mdl_dt - data << r.gujia_idxn_mo_moonmo - data << r.gujiaabt1yiminmmoonmo - data << r.gujia_idxn_yr_moonmo - data << r.gujiaabt1yr_samp_scvl - data << r.gujiaarnd1kscommavprc + data << r.gujiabdsidxnmo_moonmo + data << r.gujiabdsa1yiminmmonmo + data << r.gujiabdsidxnyr_moonmo + data << r.gujiabdsat1yspdgrscor + data << r.gujiabdsarnd1kscavprc allRows << data }); } @@ -322,27 +322,26 @@ class ExportPgsqlEvaluationResult { } for(def city : citys){ def cityCode = cityCodeMap.get(city) - def sql = "select gujia_fsid, urbn_id, gujia_comm_id, gujia_case_id,crterm,to_char(data_crt_tm,'yyyy-mm-dd') as data_crt_tm from "+database+"avm_comm_case_detail_gj_h where urbn_id='${cityCode}' and crterm=${month}"; + def sql = "select gujia_fsid, urbn_id, gujia_comm_id, gujia_case_id,gujia_bds_crterm,to_char(data_crt_tm,'yyyy-mm-dd') as data_crt_tm from "+database+"avm_comm_case_detail_gj_h where urbn_id='${cityCode}' and gujia_bds_crterm=${month}"; def allRows = [] - sqlInstance.eachRow(sql,{r-> + pgsqlInstance.eachRow(sql,{r-> def data = [] data << r.gujia_fsid data << r.urbn_id data << r.gujia_comm_id data << r.gujia_case_id - data << r.crterm + data << r.gujia_bds_crterm data << r.data_crt_tm allRows << data - if(allRows.size()%200000==0){ - FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", true); - allRows.clear() - } +// if(allRows.size()%200000==0){ +// FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", true); +// allRows.clear() +// } }); - if(allRows.size()>0){ - FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", true); - } + + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); } - CcbFtpUtils.ftpUpload(null, fileName, curDate) + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) } def exportMd5(){ @@ -379,7 +378,9 @@ class ExportPgsqlEvaluationResult { fileName = win_prefix + "${curDate}.ok" } - createFileToFtp("ok txt",fileRows, fileName, curDate, false) + FileUtils.createTxtFile(fileRows, fileName, "utf-8", false); + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "okÎļþÉÏ´«FtpÍê³É......" } From e850461897ad2f6b831bd8cea97ea54b3c1abbab Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 22 Feb 2023 20:32:38 +0800 Subject: [PATCH 94/99] updadte --- .../ccbevaluation/GuiDingPriceBatch.groovy | 4 +- .../fangjia/script/ccbevaluation/Test.groovy | 70 +++++++++---------- .../pgsql/ExportPgsqlBaseEvaluation.groovy | 2 +- 3 files changed, 38 insertions(+), 38 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index 32a919245..b6b171c81 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) //Ìí¼Ó×Ô¼º¿âÖеÄÈ˹¤Ö¸µ¼¼Û - def prices = CcbCommonUtils.findPersonPrice(city, "2023-01-25", "2023-02-21") + def prices = CcbCommonUtils.findPersonPrice(city, "2023-01-25", "2023-02-23") // def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22") if(prices){ def distMap = getDistrictMap(city) @@ -85,7 +85,7 @@ class GuiDingPriceBatch extends BaseBatch{ saveToPgSql(rows) //¸üо²Ì¬Êý¾Ý±í - updateVersion(DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28") +// updateVersion(DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28") } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index 8aca6bac6..f0a1f1448 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -45,28 +45,28 @@ DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_craw def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017"; def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business", "res_esfdealprice", offlineUrl) -def citys = ["ɽÄÏ","»ÆÄÏÖÝ","¸ÊÄÏ","¿Ë×ÎÀÕËÕ","¼ÎÓø¹Ø","º£±±ÖÝ","°¢À­ÉÆÃË","Å­½­ÖÝ","ºÍÌï","ÈÕ¿¦Ôò","²ý¶¼","º£ÄÏÖÝ","ÓñÊ÷ÖÝ","ͳ·¬","°¢ÀÕÌ©","Èýɳ","Ëþ³Ç","¸Ê×ÎÖÝ","¹ûÂåÖÝ","ÁÖÖ¥","ÄÇÇú","°¢°ÓÖÝ","ÉñÅ©¼Ü"] -def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() +//def citys = ["ɽÄÏ","»ÆÄÏÖÝ","¸ÊÄÏ","¿Ë×ÎÀÕËÕ","¼ÎÓø¹Ø","º£±±ÖÝ","°¢À­ÉÆÃË","Å­½­ÖÝ","ºÍÌï","ÈÕ¿¦Ôò","²ý¶¼","º£ÄÏÖÝ","ÓñÊ÷ÖÝ","ͳ·¬","°¢ÀÕÌ©","Èýɳ","Ëþ³Ç","¸Ê×ÎÖÝ","¹ûÂåÖÝ","ÁÖÖ¥","ÄÇÇú","°¢°ÓÖÝ","ÉñÅ©¼Ü"] +//def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() -ExcelUtils.multiThreadingModify("D:\\Êý¾Ý\\Ð¡Çø\\Ð¡ÇøµØÖ·Ãûµ¥_¹À¼Ò¿Æ¼¼V2 (3).xlsx",{d-> - if(d['J']){ - def cityId = cityCodeDataMap.get(d['H']) - def map = findCcbPriceCredit(sqlInstance236, cityId, d['J']) - println d['H'] +" "+d['J']+ map.toString() - if(map){ - d['Q'] = map.credit - d['R'] = map.month - d['U'] = map.price - } -// def thirdMap = findThirdPriceBySource(sqlInstance236, cityId, d['J']) -// if(thirdMap){ -// d['S'] = thirdMap.price -// d['T'] = thirdMap.month +//ExcelUtils.multiThreadingModify("D:\\Êý¾Ý\\Ð¡Çø\\Ð¡ÇøµØÖ·Ãûµ¥_¹À¼Ò¿Æ¼¼V2 (3).xlsx",{d-> +// if(d['J']){ +// def cityId = cityCodeDataMap.get(d['H']) +// def map = findCcbPriceCredit(sqlInstance236, cityId, d['J']) +// println d['H'] +" "+d['J']+ map.toString() +// if(map){ +// d['Q'] = map.credit +// d['R'] = map.month +// d['U'] = map.price // } - } - -}) +//// def thirdMap = findThirdPriceBySource(sqlInstance236, cityId, d['J']) +//// if(thirdMap){ +//// d['S'] = thirdMap.price +//// d['T'] = thirdMap.month +//// } +// } +// +//}) def findCcbPriceCredit(sqlInstance, cityid, commid){ def sql = "select * from (select CONFIDENCE,year_month,avg_price from community_base_price_gj where cityid='${cityid}' and commid='${commid}' order by year_month desc) WHERE ROWNUM <= 1" @@ -94,13 +94,13 @@ def findThirdPriceBySource(sqlInstance, cityid, commid){ -for(def c : citys){ - def cityCode = cityCodeDataMap.get(c) - println c +" "+cityCode - def priceDists = findAllPriceDist(sqlInstance236,cityCode, 202301) -// - checkNoPriceDist(sqlInstance236, priceDists, cityCode) -} +//for(def c : citys){ +// def cityCode = cityCodeDataMap.get(c) +// println c +" "+cityCode +// def priceDists = findAllPriceDist(sqlInstance236,cityCode, 202301) +//// +// checkNoPriceDist(sqlInstance236, priceDists, cityCode) +//} def checkNoPriceDist(sqlInstance236,priceDists, cityid){ def commids = [] @@ -1170,15 +1170,15 @@ def findThirdCommCount(cityid, sqlInstance){ //} -//def fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Ö¸µ¼¼Û\\¹úÐÅ´ï¼Û¸ñ-º¼ÖÝ.xlsx" -//ExcelUtils.multiThreadingModify(fileName, { d -> -// def dist = CommonDatautils.findDistrictById(d['A']) -// if(dist){ -// d['C'] = dist.city -// d['D'] = dist.region -// d['E'] = dist.name -// } -//}) +def fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Ö¸µ¼¼Û\\ÎÞ¼Û¸ñÐ¡Çø(1).xlsx" +ExcelUtils.multiThreadingModify(fileName, { d -> + def dist = CommonDatautils.findDistrictById(d['A']) + if(dist){ + d['G'] = dist.city + d['H'] = dist.region + d['I'] = dist.name + } +}) //def fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Ö¸µ¼¼Û\\0922Èë¿âÊý¾Ý.xlsx" //def datas = ExcelUtils.read(fileName, 0, 1) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlBaseEvaluation.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlBaseEvaluation.groovy index 70a2a1041..ffa8a5b46 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlBaseEvaluation.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlBaseEvaluation.groovy @@ -519,6 +519,7 @@ class ExportPgsqlBaseEvaluation { "from "+database+".t0831_bigdata_forsale_gj_h where urbn_id='${cityCode}' and data_udt_tm>=to_date('${publishDate}','yyyy-MM-dd')"; allRows = getHouseDatas(sql) } + println city +" "+allRows.size() if(allRows.size()>0){ index = index + 1 if(index == 1){ @@ -549,7 +550,6 @@ class ExportPgsqlBaseEvaluation { def index = 0 pgsqlInstance.eachRow(sql,{r-> index = index+1 - println r.urbn_nm +" "+index def data = [] data << r.urbn_id data << r.gujia_sell_id From d6515cce943388b1a730963dc7df89778cbf2dc9 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 22 Feb 2023 21:30:19 +0800 Subject: [PATCH 95/99] =?UTF-8?q?=E6=88=BF=E6=BA=90=E6=97=B6=E9=97=B4?= =?UTF-8?q?=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/ccbevaluation/pgsql/ExportPgsqlBaseEvaluation.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlBaseEvaluation.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlBaseEvaluation.groovy index ffa8a5b46..8b6d6dbd1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlBaseEvaluation.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlBaseEvaluation.groovy @@ -49,7 +49,7 @@ class ExportPgsqlBaseEvaluation { // exportCommunityBuildType()// // // exportThirdPartyPrice2(lastMonth, new_dists)// - exportBigdataForsale(lastMonthStr +"-18", new_dists) //ÏÖÔÚ°´updatetimeµ¼³ö + exportBigdataForsale(curMonth +"-18", new_dists) //ÏÖÔÚ°´updatetimeµ¼³ö // exportBigdataDeal(lastMonthStr + "-01", new_dists) //2022-07-01 } From bf0c8aaf92e67a25b8ea2b8ab91f77445bbfde74 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 22 Feb 2023 21:37:49 +0800 Subject: [PATCH 96/99] update --- .../script/ccbevaluation/pgsql/ExportPgsqlBaseEvaluation.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlBaseEvaluation.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlBaseEvaluation.groovy index 8b6d6dbd1..59d62cfea 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlBaseEvaluation.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlBaseEvaluation.groovy @@ -49,7 +49,7 @@ class ExportPgsqlBaseEvaluation { // exportCommunityBuildType()// // // exportThirdPartyPrice2(lastMonth, new_dists)// - exportBigdataForsale(curMonth +"-18", new_dists) //ÏÖÔÚ°´updatetimeµ¼³ö + exportBigdataForsale(DateUtils.formatDateByPattern(new Date(), "yyyy-MM") +"-18", new_dists) //ÏÖÔÚ°´updatetimeµ¼³ö // exportBigdataDeal(lastMonthStr + "-01", new_dists) //2022-07-01 } From e7e36a652a1ab7f7f2dad893c3d5fb7964308c0b Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 23 Feb 2023 14:10:15 +0800 Subject: [PATCH 97/99] =?UTF-8?q?=E4=B8=B4=E6=97=B6=E8=B7=91=E6=95=B0?= =?UTF-8?q?=E6=8D=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../pgsql/ExportPgsqlEvaluationResult.groovy | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlEvaluationResult.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlEvaluationResult.groovy index a5321a3e8..3f269baaa 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlEvaluationResult.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlEvaluationResult.groovy @@ -34,21 +34,21 @@ class ExportPgsqlEvaluationResult { def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")); //¼ÆËã½á¹ûÏà¹Ø - exportAvmEvaluationGJNB(curMonth) - exportAVMEvaluationItemGJNB(curMonth) +// exportAvmEvaluationGJNB(curMonth) +// exportAVMEvaluationItemGJNB(curMonth) exportCityPrice(curMonth) exportRegionPrice(curMonth) exportCommunityBasePrice(curMonth) - exportCommunityPdescribeGj(curMonth) - exportCommCaseDetailGj(curMonth) - - exportInOk() - - SSHHelperTest jschCommand = new SSHHelperTest(); - System.out.println(jschCommand.executeCommand("sh /home/ccb_evaluation/output/eval_package.sh ${curDate}")); +// exportCommunityPdescribeGj(curMonth) +// exportCommCaseDetailGj(curMonth) +// +// exportInOk() +// +// SSHHelperTest jschCommand = new SSHHelperTest(); +// System.out.println(jschCommand.executeCommand("sh /home/ccb_evaluation/output/eval_package.sh ${curDate}")); } @@ -161,7 +161,7 @@ class ExportPgsqlEvaluationResult { def allRows = [] for(def city : citys) { def cityCode = cityCodeMap.get(city) - def sql = "select gujia_trdpt_avprc_id,urbn_id,urbn_nm,gujiabds_clc_yrmo_val,gujia_bds_clc_avprc,gujia_bds_urbn_avprc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"avm_city_price_gj_h where gujiabds_clc_yrmo_val=${month} AND urbn_id='${cityCode}'"; + def sql = "select gujia_trdpt_avprc_id,urbn_id,urbn_nm,gujiabds_clc_yrmo_val,gujia_bds_clc_avprc,gujia_bds_urbn_avprc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"avm_city_price_gj_h where urbn_id='${cityCode}'";//gujiabds_clc_yrmo_val=${month} AND pgsqlInstance.eachRow(sql, { r -> def data = [] data << r.gujia_trdpt_avprc_id @@ -188,7 +188,7 @@ class ExportPgsqlEvaluationResult { def allRows = [] for(def city : citys) { def cityCode = cityCodeMap.get(city) - def sql = "select gujia_trdpt_avprc_id,urbn_id,urbn_nm,gujia_admndiv_id,gxda_admndiv_nm,gujiabds_clc_yrmo_val,gujia_bds_clc_avprc,gujia_bds_urbn_avprc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"avm_district_price_gj_h where gujiabds_clc_yrmo_val=${month} AND urbn_id='${cityCode}'"; + def sql = "select gujia_trdpt_avprc_id,urbn_id,urbn_nm,gujia_admndiv_id,gxda_admndiv_nm,gujiabds_clc_yrmo_val,gujia_bds_clc_avprc,gujia_bds_urbn_avprc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"avm_district_price_gj_h where urbn_id='${cityCode}'"; //gujiabds_clc_yrmo_val=${month} AND pgsqlInstance.eachRow(sql, { r -> def data = [] @@ -219,7 +219,7 @@ class ExportPgsqlEvaluationResult { def allRows = [] for(def city : citys) { def cityCode = cityCodeMap.get(city) - def sql = "select gujia_bss_prc_id,urbn_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_bds_id,comm_nm,gujiabds_clc_yrmo_val,gujia_bds_clc_avprc,gujia_comm_basic_prc,gujia_bds_clmtd,gujiabds_fwd_samp_rto,gujia_bds_mns_grd,gujia_bds_cfdrg,gujiabdsarndcmprccrlt,gujia_bds_cfdc_dgr,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"avm_community_base_price_gj_h where gujiabds_clc_yrmo_val=${yearMonth} AND urbn_id='${cityCode}'"; + def sql = "select gujia_bss_prc_id,urbn_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_bds_id,comm_nm,gujiabds_clc_yrmo_val,gujia_bds_clc_avprc,gujia_comm_basic_prc,gujia_bds_clmtd,gujiabds_fwd_samp_rto,gujia_bds_mns_grd,gujia_bds_cfdrg,gujiabdsarndcmprccrlt,gujia_bds_cfdc_dgr,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"avm_community_base_price_gj_h where urbn_id='${cityCode}'";//gujiabds_clc_yrmo_val=${yearMonth} AND pgsqlInstance.eachRow(sql, { r -> def data = [] data << r.gujia_bss_prc_id @@ -387,8 +387,8 @@ class ExportPgsqlEvaluationResult { def createFileToFtp(tableName,allRows, fileName, curDate, isAppend){ FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", isAppend); println tableName+" Êý¾ÝÎļþÊä³öÍê³É......" - CcbFtpTestUtils.ftpUpload(null, fileName, curDate) - println tableName+" ÉÏ´«FtpÍê³É......" + //CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + //println tableName+" ÉÏ´«FtpÍê³É......" } } From 6e474ce73266c31da1a005219e39887ff81bb862 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 23 Feb 2023 16:15:20 +0800 Subject: [PATCH 98/99] =?UTF-8?q?=E4=B8=B4=E6=97=B6=E5=A4=84=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../pgsql/ExportPgsqlEvaluationResult.groovy | 55 ++++++++++++++++++- 1 file changed, 52 insertions(+), 3 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlEvaluationResult.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlEvaluationResult.groovy index 3f269baaa..57fc0d413 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlEvaluationResult.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlEvaluationResult.groovy @@ -37,10 +37,11 @@ class ExportPgsqlEvaluationResult { // exportAvmEvaluationGJNB(curMonth) // exportAVMEvaluationItemGJNB(curMonth) - exportCityPrice(curMonth) - exportRegionPrice(curMonth) +// exportCityPrice(curMonth) +// exportRegionPrice(curMonth) - exportCommunityBasePrice(curMonth) +// exportCommunityBasePrice(curMonth) + exportCommunityBasePrice1(curMonth) // exportCommunityPdescribeGj(curMonth) // exportCommCaseDetailGj(curMonth) @@ -242,6 +243,7 @@ class ExportPgsqlEvaluationResult { data << r.data_udt_tm data << r.gujia_src_co allRows << data + }); } @@ -255,6 +257,53 @@ class ExportPgsqlEvaluationResult { createFileToFtp("COMMUNITY_BASE_PRICE_GJ",allRows, fileName1, curDate, false) } + + def exportCommunityBasePrice1(yearMonth){ + + def fileName1 = linux_prefix + "MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName1 = win_prefix + "MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" + } + + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select gujia_bss_prc_id,urbn_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_bds_id,comm_nm,gujiabds_clc_yrmo_val,gujia_bds_clc_avprc,gujia_comm_basic_prc,gujia_bds_clmtd,gujiabds_fwd_samp_rto,gujia_bds_mns_grd,gujia_bds_cfdrg,gujiabdsarndcmprccrlt,gujia_bds_cfdc_dgr,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"avm_community_base_price_gj_h where urbn_id='${cityCode}'";//gujiabds_clc_yrmo_val=${yearMonth} AND + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_bss_prc_id + data << r.urbn_id + data << r.urbn_nm + data << r.gujia_admndiv_id + data << r.gujia_admndiv_nm + data << r.gujia_bds_id + data << r.comm_nm + data << r.gujiabds_clc_yrmo_val + data << r.gujia_bds_clc_avprc + data << r.gujia_comm_basic_prc + data << r.gujia_bds_clmtd + data << r.gujiabds_fwd_samp_rto + data << r.gujia_bds_mns_grd + data << r.gujia_bds_cfdrg + data << r.gujiabdsarndcmprccrlt + data << r.gujia_bds_cfdc_dgr + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + + if(allRows.size()%200000==0){ + FileUtils.batchCreateFile(allRows, fileName1, [], "utf-8", true); + allRows.clear() + } + }); + } + + if(allRows.size()>0){ + FileUtils.batchCreateFile(allRows, fileName1, [], "utf-8", true); + } + } + def exportCommunityPdescribeGj(month){ def fileName = linux_prefix + "MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat" From 6fdfa071a51a587fcba74229118364e0d6d3c80a Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 24 Feb 2023 10:12:06 +0800 Subject: [PATCH 99/99] update --- .../ccbevaluation/pgsql/ExportPgsqlEvaluationResult.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlEvaluationResult.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlEvaluationResult.groovy index 57fc0d413..fc3dfabeb 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlEvaluationResult.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlEvaluationResult.groovy @@ -37,8 +37,8 @@ class ExportPgsqlEvaluationResult { // exportAvmEvaluationGJNB(curMonth) // exportAVMEvaluationItemGJNB(curMonth) -// exportCityPrice(curMonth) -// exportRegionPrice(curMonth) + exportCityPrice(curMonth) + exportRegionPrice(curMonth) // exportCommunityBasePrice(curMonth) exportCommunityBasePrice1(curMonth)