From 834c271b7d7ba2839384dce0d581df55b8b303c5 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 14 Jul 2023 22:19:33 +0800 Subject: [PATCH] 100-50 --- .../src/main/groovy/com/fangjia/script/fuzhou/Test.groovy | 5 ++++- .../fangjia/script/gj/district/OfficeFieldCleanBatch.groovy | 4 ++-- .../groovy/com/fangjia/script/gj/house/RemoveHouse.groovy | 2 +- .../com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy | 4 ++-- .../com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy | 2 +- 5 files changed, 10 insertions(+), 7 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test.groovy index aa277adde..b0ce64f12 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test.groovy @@ -239,7 +239,10 @@ def findThirdPrice(city, region, name, AVERAGE_STATISTICS){ def files = CommonDatautils.getAllFiles("D:\\数据\\福州\\原始数据") if(files){ for(def filepath : files){ - def rets = read(filepath); + def filename = filepath.getPath() + def rets = read(filename); + def date = filename.substring(filename.toString().lastIndexOf("_")+1, filename.toString().indexOf(".")) + println date +" "+rets.size() } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/OfficeFieldCleanBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/OfficeFieldCleanBatch.groovy index 1db9e589e..44e6641d4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/OfficeFieldCleanBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/OfficeFieldCleanBatch.groovy @@ -73,7 +73,7 @@ class OfficeFieldCleanBatch extends BaseBatch{ // addFieldValue() - updateErrorField() +// updateErrorField() export() test() // addOffice() @@ -120,7 +120,7 @@ class OfficeFieldCleanBatch extends BaseBatch{ } def test(){ - def datas = ExcelUtils.read("D:\\数据\\写字楼\\写字楼价格租金.xlsx",0 , 1) + def datas = ExcelUtils.read("D:\\数据\\写字楼\\写字楼价格 (2).xlsx",0 , 1) def map = [:] for(def d : datas){ map.put(d['A']+"#"+d['B']+'#'+d['C'], d) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/house/RemoveHouse.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/house/RemoveHouse.groovy index c315c404e..315821223 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/house/RemoveHouse.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/house/RemoveHouse.groovy @@ -29,7 +29,7 @@ class RemoveHouse extends BaseBatch{ def rets = [] for(def d : dbcursor){ rets << d._id - if(rets.size()==100){ + if(rets.size()==50){ def delquery = new BasicDBObject() delquery.put("_id", new BasicDBObject('$in', rets)) SEABIRD_COLL.remove(delquery) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy index 788d1703e..0d031c591 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy @@ -465,9 +465,9 @@ public class BeiKeCrawlByChrome { if(cityEles){ for(def e : cityEles){ def href = "https:"+e.attr("href") - if (citys.contains(e.text())){ +// if (citys.contains(e.text())){ list << [e.text(), href] - } +// } } } return list diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy index 6c0ce91f3..c87a55634 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -570,7 +570,7 @@ public class LianjiaCrawlByChrome { if(cityEles){ for(def e : cityEles) { def href = e.attr("href") - if (!href.contains("fang.lianjia") && citys.contains(e.text())){ + if (!href.contains("fang.lianjia")){ cityList << [e.text(), href] } }