From fda9911b3a7264f9bbef2fd4aa1fd21271e564f3 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 16 May 2024 13:22:24 +0800 Subject: [PATCH] =?UTF-8?q?=E6=88=BF=E6=BA=90=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ccbevaluation/CcbCommonUtils.groovy | 4 +- .../ccbevaluation/PgsqlCommUtils.groovy | 6 + .../FjwBigdataCommbuildtypeBatch.groovy | 5 +- .../FjwBigdataForsaleBatch.groovy | 253 ++---------------- .../fjwPriceBaseData/FjwPgsqlCommUtils.groovy | 25 ++ 5 files changed, 56 insertions(+), 237 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwPgsqlCommUtils.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index cee43a5b9..3e18b6b7f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -16,7 +16,7 @@ import groovy.sql.Sql class CcbCommonUtils { // - public static def citys_fjw_all = [ "上海","长春","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州","白沙","北屯","昌江","澄迈","定安","东方","济源","可克达拉","昆玉","乐东","临高","琼海","琼中","石河子","双河","铁门关","图木舒克","屯昌","万宁","五家渠","五指山","阿拉尔"] + public static def citys_fjw_all = [ "乌鲁木齐","上海","长春","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州","白沙","北屯","昌江","澄迈","定安","东方","济源","可克达拉","昆玉","乐东","临高","琼海","琼中","石河子","双河","铁门关","图木舒克","屯昌","万宁","五家渠","五指山","阿拉尔"] public static def citys_all = [ "上海","长春","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝"] public static def citys_92 = [ "上海","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水"] public static def citys_252 = ["潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州"] @@ -439,7 +439,7 @@ class CcbCommonUtils { q.put("cat", cat); q.put("city", city); // q.put("c_date", new BasicDBObject('$lt', DateUtils.parseFromTightYYYYMMDD("2023-05-01")))//因5月1日新增过多,所以暂且先去掉 此批数据改为name_cat=13 -// q.put("name","汤堡小区") +// q.put("name","新泾三村") // q.put("name_cat", 0); // q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"])) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/PgsqlCommUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/PgsqlCommUtils.groovy index cfbd78a2d..0a1584be9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/PgsqlCommUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/PgsqlCommUtils.groovy @@ -2,6 +2,9 @@ package com.fangjia.script.ccbevaluation import groovy.sql.Sql +/** + * 建行使用 + */ class PgsqlCommUtils { public static def database = "warehouse" @@ -19,6 +22,9 @@ class PgsqlCommUtils { } + + + public static def findAllDistrictCount(def cityid){ def sql = "select count(gujia_comm_id) from base.t0831_bigdata_community_gj_h where urbn_id='${cityid}' and gujia_bds_bds_st=0"; def count = 0 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataCommbuildtypeBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataCommbuildtypeBatch.groovy index b76f4139a..16e99a043 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataCommbuildtypeBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataCommbuildtypeBatch.groovy @@ -3,9 +3,10 @@ package com.fangjia.script.gj.fjwPriceBaseData import com.fangjia.script.BaseBatch import com.fangjia.script.BatchOwner import com.fangjia.script.Watcher +import com.fangjia.script.ccbevaluation.CcbCommonUtils import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.ccb.CcbCodeUtils -import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.gj.fjwPriceBaseData.FjwPgsqlCommUtils import com.fangjia.script.util.DateUtils import groovy.sql.Sql @@ -38,7 +39,7 @@ class FjwBigdataCommbuildtypeBatch extends BaseBatch{ def cityCodeDatas = CcbCommonUtils.findCityCodeValue() for(def city : citys){ - def distIds = PgsqlCommUtils.findAllDistrictIdsByOracle(cityCodeDatas.get(city)) + def distIds = FjwPgsqlCommUtils.findAllDistrictIdsByOracle(cityCodeDatas.get(city)) def rows = createDatas(cityCodeDatas.get(city), distIds) rets.addAll(rows) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataForsaleBatch.groovy index 908bdc375..2b8cde742 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataForsaleBatch.groovy @@ -64,10 +64,10 @@ class FjwBigdataForsaleBatch extends BaseBatch{ public static ExecutorService fixedThreadPool2 = Executors.newFixedThreadPool(200) - def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", - "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") +// def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", +// "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") - def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/fangjia?stringtype=unspecified", + def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified", "postgres", "Goojia123456", "org.postgresql.Driver") @@ -204,64 +204,7 @@ class FjwBigdataForsaleBatch extends BaseBatch{ } - def repeatHouseId(allDatas){ - def totalCount = new AtomicLong() - final CountDownLatch latch = new CountDownLatch(allDatas.size()) - def rets = new CopyOnWriteArrayList() - for(def i=0;i60){ - continue; - } - -// if(!seawater.room){ -// continue; -// } - if(!seawater.room){ - seawater.room = null - } - - if(seawater.room){ - if(seawater.room>=11 || seawater.room<1){ - continue; - } - } - - if(!seawater.hall){ - seawater.hall = null - } - if(seawater.hall && (seawater.hall>=5 || seawater.hall<1)){ - continue; - } - if(!seawater.toilet){ - seawater.toilet = null - } - if(seawater.toilet && (seawater.toilet>=5 || seawater.toilet<1)){ - continue; - } - - - def floor = parseFloor(seawater.floor_info) - if(seawater.height && floor && floor.isNumber()){ - if(floor.toInteger()>seawater.height){ - floor = null - seawater.height = null - } - - } - - if(!floor){ - seawater.floor = null - } - if(floor && floor.toInteger()<1){ - floor = null - } - def vallaFlag = isVilla(seawater.area, seawater.name, seawater.height, seawater.room) - if(vallaFlag){ - continue; - } - - def rowMap = [:]; - if(cat == "rent"){ - rowMap.rent_type = seawater.rent_type - if(seawater.avg_price && seawater.avg_price>1000){ - continue - } - - if(seawater.avg_price && seawater.avg_price<10){ - continue - } - - } - rowMap.houseId = seawater.id - rowMap.city = city - //rowMap.region =seawater.region - //rowMap.address = seawater.address - - - rowMap.area = seawater.area - rowMap.total_price = seawater.total_price - rowMap.avg_price = seawater.avg_price - rowMap.height = seawater.height=="null"?null:seawater.height - if(rowMap.height && rowMap.height<1){ - rowMap.height = null - } - rowMap.floor = floor=="null"?null:floor - rowMap.room = seawater.room=="null"?null:seawater.room - rowMap.hall = seawater.hall=="null"?null:seawater.hall - rowMap.toilet = seawater.toilet=="null"?null:seawater.toilet - if(CcbCommonUtils.rootMap.get(seawater.root)){ - rowMap.root = CcbCommonUtils.rootMap.get(seawater.root) - }else{ - rowMap.root = "99" - } - - def direction = parseDirection(seawater.direction) - rowMap.direction = direction=="null"?null:direction - - def fitment = parseFitment(seawater.fitment) - rowMap.fitment = fitment=="null"?null:fitment - - - - if(seawater.p_date instanceof String){ - seawater.p_date = seawater.p_date.toInteger() - } - if(seawater.p_date<=20200101){ - //seawater.p_date.toString().replace("2019","2020")// - rowMap.p_date = DateUtils.getRandomDate("2020-01-01", "2020-07-31") - rowMap.c_date = rowMap.p_date - rowMap.m_date = rowMap.p_date - }else{ - rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss") - rowMap.c_date = DateUtils.getConverDate(seawater.m_date) - rowMap.m_date = DateUtils.getConverDate(seawater.m_date) - } - - rets << rowMap - } - } - - println commId +" "+houses.size()+" last:"+rets.size() - return rets; - } def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2, def commId, def cityCodeDataMap){ @@ -700,9 +487,9 @@ class FjwBigdataForsaleBatch extends BaseBatch{ def findHousesByMdate(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2, rets1){ - def date = DateUtils.addMonth(endDate.toString(), -3, "yyyyMMdd") - def startMdate = DateUtils.parseFromTightByPattern(date,"yyyyMMdd") - def endMdate = DateUtils.parseFromTightByPattern(endDate.toString(), "yyyyMMdd") + def date = "20200101"//DateUtils.addMonth(endDate.toString(), -3, "yyyyMMdd") + def startMdate = DateUtils.parseFromTightByPattern("20200101","yyyyMMdd")//DateUtils.parseFromTightByPattern(date,"yyyyMMdd") + def endMdate = DateUtils.parseFromTightByPattern("20240518", "yyyyMMdd")//DateUtils.parseFromTightByPattern(endDate.toString(), "yyyyMMdd") QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(date.toInteger()).to(endDate).includeLower(false).includeUpper(false); QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false); @@ -710,9 +497,19 @@ class FjwBigdataForsaleBatch extends BaseBatch{ QueryBuilder date_range_mdate = QueryBuilders.rangeQuery("m_date").from(startMdate.getTime()).to(endMdate.getTime()).includeLower(true).includeUpper(false); + //用以下方法会查出发布时间很久以前的 +// QueryBuilder dateQueryBuilder = QueryBuilders.boolQuery() +// dateQueryBuilder.should(date_range) +// dateQueryBuilder.should(date_range_mdate) +// QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat)) +// .must(area_range).must(dateQueryBuilder) + + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat)) .must(date_range).must(area_range).must(date_range_mdate)//.must(QueryBuilders.termQuery("root", "lianjia.com")) + + if(!alias){ alias = [] } @@ -1094,27 +891,17 @@ class FjwBigdataForsaleBatch extends BaseBatch{ // 创建布隆过滤器对象 BloomFilter filter = BloomFilter.create(Funnels.stringFunnel(StandardCharsets.UTF_8), 200000000, 0.000001); // def houseIds = new HashSet() - pgsqlInstance.eachRow("SELECT gujia_sell_id FROM base.t0831_bigdata_forsale_gj_h",{it-> + pgsqlInstance.eachRow("SELECT gujia_sell_id FROM base.bigdata_forsale",{it-> filter.put(it.gujia_sell_id) }); - pgsqlInstance.eachRow("SELECT gujia_sell_id FROM base.t0831_bigdata_forsale_gj_h_2021",{it-> - filter.put(it.gujia_sell_id) - }); +// pgsqlInstance.eachRow("SELECT gujia_sell_id FROM base.bigdata_forsale_2021",{it-> +// filter.put(it.gujia_sell_id) +// }); return filter; } - def isExistFromHouse(def houseId){ - def sql = "SELECT count(FORSALEID) as count FROM bigdata_forsale_gj WHERE FORSALEID = '"+houseId+"'"; - - def count = null - sqlInstance.eachRow(sql,{it-> - count = it.count - }); - return count; - } - def saveToPgSql(rows){ for(def r : rows) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwPgsqlCommUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwPgsqlCommUtils.groovy new file mode 100644 index 000000000..10c94f0ea --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwPgsqlCommUtils.groovy @@ -0,0 +1,25 @@ +package com.fangjia.script.gj.fjwPriceBaseData + +import groovy.sql.Sql + +/** + * 房价网内部使用 + */ +class FjwPgsqlCommUtils { + + public static def database = "fangjia" + + public static def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/"+database+"?stringtype=unspecified", + "postgres", "Goojia123456", "org.postgresql.Driver") + + public static def findAllDistrictIdsByOracle(def cityid){ + def sql = "select gujia_comm_id from base.bigdata_community where urbn_id='${cityid}' and gujia_bds_bds_st=0"; + def set = new HashSet() + sqlInstance.eachRow(sql,{r-> + set << r.gujia_comm_id + }); + return set + } + + +}