From 4b9f22fd67bdba6f37acac8363f8d266bb32b62c Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Tue, 11 Oct 2022 09:32:52 +0800 Subject: [PATCH 1/3] =?UTF-8?q?=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/temp/shenjian/poi/SeaweedAroundPoi.groovy | 8 ++++---- .../fangjia/script/temp/shenjian/util/GaoDeUtils.groovy | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy index ca74c0184..2bc9b09b2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy @@ -73,11 +73,11 @@ class SeaweedAroundPoi extends BaseBatch { // void run() { def city = obj.city if (obj.lng) { - //def lng = SeaweedUtils.formatLatLng(obj.lng) - //def lat = SeaweedUtils.formatLatLng(obj.lat) + def lng = SeaweedUtils.formatLatLng(obj.lng) + def lat = SeaweedUtils.formatLatLng(obj.lat) // {'lon': 121.810877, 'lat': 31.737642} - def lng = 121.810877 - def lat = 31.737642 +// def lng = 121.810877 +// def lat = 31.737642 List pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "") if (CollectionUtils.isNotEmpty(pois)) { String jsonStr = JSONObject.toJSONString(pois) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy index 50ac505aa..17b02bb6b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -146,13 +146,13 @@ class GaoDeUtils { while (result.size() < count) { params.page = "${page}".toString() page++ - //params.key = this.getGaodeKey("v3/place/around") - params.key = "25b2d459fd133e22f8abcebeac972692" + params.key = this.getGaodeKey("v3/place/around") + //params.key = "25b2d459fd133e22f8abcebeac972692" params.offset = "100" JSONObject json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/around", params); if (json == null) { - //params.key = this.getGaodeKey("v3/place/around") - params.key = "25b2d459fd133e22f8abcebeac972692" + params.key = this.getGaodeKey("v3/place/around") + //params.key = "25b2d459fd133e22f8abcebeac972692" params.offset = "100" json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/around", params); } From 701185070273adec2a7d0e67e7bda107cb12fd7d Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 11 Oct 2022 17:30:29 +0800 Subject: [PATCH 2/3] =?UTF-8?q?=E5=BB=BA=E8=A1=8C=E8=AF=BB=E5=8F=96?= =?UTF-8?q?=E5=9F=8E=E5=B8=82=E6=9B=B4=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../BigdataCommbuildtypeBatch.groovy | 2 +- .../ccbevaluation/CcbCommonUtils.groovy | 22 +++++++++++++++++-- .../ccbevaluation/StagingDistrictBatch.groovy | 2 +- .../ronghe/ExportDistrictRongHe.groovy | 7 +++--- .../fuzhou/DistrictInfoExportBatch.groovy | 1 + .../fuzhou/DistrictPriceExportBatch.groovy | 6 ++++- .../script/fuzhou/FzDataResultStat.groovy | 4 +++- .../com/fangjia/script/fuzhou/Test1.groovy | 2 +- .../gj/paimai/BeiKeCrawlByChrome.groovy | 4 ++-- .../gj/paimai/LianjiaCrawlByChrome.groovy | 5 +++-- .../script/gj/paimai/PaimaiDataSend.groovy | 15 ++++++++----- 11 files changed, 50 insertions(+), 20 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy index 1000b157c..066318e29 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy @@ -28,7 +28,7 @@ class BigdataCommbuildtypeBatch extends BaseBatch{ @Override public void process(Object args) { - def citys = CcbCommonUtils.citys; + def citys = CcbCommonUtils.getRunningCitys(args) citys = citys.unique() def rets = [] def cityCodeDatas = CcbCommonUtils.findCityCodeValue() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index 3ef4cee91..24ee5952f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -17,8 +17,8 @@ class CcbCommonUtils { // public static def citys_all = [ "长春","上海","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州"] - public static def citys = [ "上海","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水"] -// public static def citys = ["武汉"] + public static def citys_92 = [ "上海","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水"] + public static def citys_252 = ["潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州"] public static def provCityMap = ["河北":"石家庄" ,"山西":"太原" ,"陕西":"西安" ,"山东":"济南","河南":"郑州" ,"辽宁":"沈阳" ,"吉林":"长春" ,"黑龙江":"哈尔滨" ,"江苏":"南京" ,"浙江":"杭州" ,"安徽":"合肥","江西":"南昌" ,"福建":"福州" ,"湖北":"武汉","湖南":"长沙" ,"四川":"成都" ,"贵州":"贵阳" ,"云南":"昆明" ,"广东":"广州" ,"海南":"海口" ,"甘肃":"兰州" ,"青海":"西宁" ,"台湾":"台北","内蒙古":"呼和浩特" ,"新疆":"乌鲁木齐" ,"西藏":"拉萨","广西":"南宁","宁夏":"银川","香港":"香港","澳门":"澳门"] @@ -46,6 +46,24 @@ class CcbCommonUtils { "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + public static def getRunningCitys(def args){ + def citys = [] + println "args 参数:"+args.toString() + if(args){ + def countStr = args[0] + if(countStr=="344"){ + citys = CcbCommonUtils.citys_all + }else if(countStr=="92"){ + citys = CcbCommonUtils.citys_92 + }else if(countStr=="252"){ + citys = CcbCommonUtils.citys_252 + } + }else{ + citys = CcbCommonUtils.citys_92 + } + println "运行城市数量:"+citys.size() + return citys; + } public static def findCityCodeValue(){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy index 3ac7f9a28..ed95e4cc6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy @@ -24,7 +24,7 @@ class StagingDistrictBatch extends BaseBatch{ @Override public void process(Object args) { - def citys = CcbCommonUtils.citys; + def citys = CcbCommonUtils.getRunningCitys(args) def rows = [] for(def city : citys){ def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2","name_cat","lng","master_district_name", "c_date", "m_date"]) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy index fa398bbc5..d38ad153b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy @@ -22,17 +22,18 @@ class ExportDistrictRongHe extends BaseBatch{ runBatch(new ExportDistrictRongHe(), args); } - def citys = CcbCommonUtils.citys + def citys = CcbCommonUtils.citys_all def cityCodeMap = CcbCommonUtils.findCityCodeValue() @Override void process(Object args) { + def citys = CcbCommonUtils.getRunningCitys(args) def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd") def new_dists = []//["350604","350605","410307","410308","450181"] //按天导(每天一个文件夹) - exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-1), "yyyy-MM-dd"), new_dists) +// exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-1), "yyyy-MM-dd"), new_dists) - if(curDate.toString().endsWith("11")){ + if(curDate.toString().endsWith("10")){ exportBigdataCommunityGJRH()// exportBigdataBuildingFJW()// exportBigdataCommfeeGj()// diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy index 9ca07aac4..ea6d2717b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy @@ -167,6 +167,7 @@ class DistrictInfoExportBatch extends BaseBatch{ udate.put(k,v) } udate.removeField("_id") + udate.put("m_date", new Date()) FZ_DIST_COLL.update(new BasicDBObject('_id',d.get('_id')), new BasicDBObject('$set', udate)); }else{ FZ_DIST_COLL.save(d) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy index 14a8a6fac..45f3cbe35 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy @@ -154,7 +154,11 @@ class DistrictPriceExportBatch extends BaseBatch{ m << rMap.city m << rMap.region m << rMap.name - m << curPrice + if(curPrice){ + m << curPrice + }else{ + m << 0 + } m << curDate.toInteger(); result << m } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy index a624061d9..78537bc39 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy @@ -98,9 +98,11 @@ class FzDataResultStat { def dist = CommonDatautils.findDistrictById(list[1]) def flag = "有效" - if(dist.status!=0){ + + if(dist && dist.status!=0){ flag = "失效" } + rows << [keys[0], keys[1], keys[2], list[5].toString(), lastPrice.toString(), markup, flag] } ExcelUtils.write("D:\\数据\\福州\\价格比对_${curMonth}.xlsx", rows, "数据") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy index 7cdcc589a..43203f42f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy @@ -14,7 +14,7 @@ import com.mongodb.DBCollection //福州第三方均价统计,发送钉钉消息 //def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1) //def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv"); -def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202208.csv") +def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202209.csv") def count0 = 0 def count1 = 0 def count2 = 0 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy index 42489aafe..bb4f74fdf 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy @@ -390,13 +390,13 @@ public class BeiKeCrawlByChrome { if(cityEles){ for(def e : cityEles){ def href = e.attr("href") -// if(e.text().contains("福州")){ + if(e.text().contains("福州")){ def cityMap = [:] cityMap.city = e.text() cityMap.url = "https:"+href list << cityMap // cityMap.put(e.text(), "https:"+href) -// } + } } } return list diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy index c42cfa5ba..5b8abfc9a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -31,6 +31,7 @@ public class LianjiaCrawlByChrome { def DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol(); def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) + def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -2, "yyyy-MM") static main(args) { def task = new LianjiaCrawlByChrome(); @@ -250,7 +251,7 @@ public class LianjiaCrawlByChrome { def durl = e.select(".title a").attr("href") def dealDate = e.select(".dealDate").text() dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd") - def endDate = DateUtils.parseFromTightByPattern("2022-07-20", "yyyy-MM-dd") + def endDate = DateUtils.parseFromTightByPattern(startMonth + "-20", "yyyy-MM-dd") if(dealDate.getTime()>=endDate.getTime()){ detailUrls << durl parse(city, e, regionText) @@ -469,7 +470,7 @@ public class LianjiaCrawlByChrome { def cityDoc = Jsoup.parse(driver.getPageSource()); def cityEles = JQueryUtils.select(".city_province a", cityDoc) def cityMap = [:] - def citys= ["苏州"] + def citys= ["福州"] if(cityEles){ for(def e : cityEles) { def href = e.attr("href") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataSend.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataSend.groovy index e61f3588a..d202cdd74 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataSend.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataSend.groovy @@ -335,8 +335,9 @@ class PaimaiDataSend { def q = new BasicDBObject() // q.put("city",new BasicDBObject('$in', ["南京","宿迁","苏州"])) q.put("province","江苏") + q.put("sub_type","土地") // q.put("parser_status", new BasicDBObject('$ne',2)) - q.put("url","https://sf-item.taobao.com/sf_item/665795393092.htm") +// q.put("url","https://sf-item.taobao.com/sf_item/668496148492.htm") //https://sf-item.taobao.com/sf_item/657131607486.htm // q.put("parser_status", new BasicDBObject('$exists',false)) // q.put("finishDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-09-01"))) @@ -420,10 +421,11 @@ class PaimaiDataSend { // q.put("city",new BasicDBObject('$in', ["苏州","宿迁","南京"])) // q.put("city", "上海") q.put("province","江苏") + q.put("sub_type","土地") // q.put("parser_status", new BasicDBObject('$ne',1)) // q.put("region","姑苏") - q.put("url","https://susong-item.taobao.com/auction/674736290116.htm") - def dbcursor = ali_zc_coll.find(q).batchSize(5000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT) +// q.put("url","https://susong-item.taobao.com/auction/674736290116.htm") + def dbcursor = ali_zc_coll.find(q).batchSize(1000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT) def rets = [] for(def d : dbcursor){ index++ @@ -438,6 +440,7 @@ class PaimaiDataSend { JSON result = new JSONObject() result.put("content", d.content) + result.put("sub_type", d.sub_type) def bid_content = net.sf.json.JSONObject.fromObject(d.bid_content) result.put("trades", bid_content) result.put("sub_count", d.sub_count) @@ -453,7 +456,7 @@ class PaimaiDataSend { json.put("result", result) rets << json - if(rets.size()==10000){ + if(rets.size()==1000){ println "共:"+rets.size() send(rets) rets.clear() @@ -522,8 +525,8 @@ class PaimaiDataSend { def getPaimaiDatas(){ def q = new BasicDBObject() // q.put("city",new BasicDBObject('$in', ["南京","宿迁","苏州"])) - q.put("city","苏州") - q.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-10-09"))) + q.put("province","江苏") + q.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-10-10"))) def dbcursor = paimai43_COLL.find(q).batchSize(5000) def rets = [] for(def d : dbcursor){ From 83805163b481787e2fcb61c299c8d697633bf3b3 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 11 Oct 2022 17:34:29 +0800 Subject: [PATCH 3/3] marge --- .../script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy index 039531831..229ed763f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy @@ -22,12 +22,12 @@ class ExportDistrictRongHe extends BaseBatch{ runBatch(new ExportDistrictRongHe(), args); } - def citys = CcbCommonUtils.citys_all + def citys = [] def cityCodeMap = CcbCommonUtils.findCityCodeValue() @Override void process(Object args) { - def citys = CcbCommonUtils.getRunningCitys(args) + citys = CcbCommonUtils.getRunningCitys(args) def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd") def new_dists = []//["350604","350605","410307","410308","450181"] //鎸夊ぉ瀵硷紙姣忓ぉ涓�涓枃浠跺す锛�