From 36c932625b521bee5e6ec4a161f4b3483c87ff3e Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 25 Aug 2022 20:49:31 +0800 Subject: [PATCH] update --- .../script/gj/paimai/PaimaiDataSend.groovy | 150 +++++++++++------- 1 file changed, 91 insertions(+), 59 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataSend.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataSend.groovy index 4f5343c14..4297a52b8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataSend.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataSend.groovy @@ -29,7 +29,7 @@ class PaimaiDataSend { def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; DBCollection paimai43_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "pai_mai", lineUrl) - public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) static main(args) { def task = new PaimaiDataSend() @@ -198,9 +198,10 @@ class PaimaiDataSend { def index = 0 def q = new BasicDBObject() // q.put("city","宿迁") - q.put("url","https://sf-item.taobao.com/sf_item/649014193736.htm") + q.put("url","https://sf-item.taobao.com/sf_item/665877683620.htm") //https://sf-item.taobao.com/sf_item/657131607486.htm // q.put("parser_status", new BasicDBObject('$exists',false)) +// q.put("finishDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-09-01"))) def dbcursor = ali_fp_coll.find(q).batchSize(2000) def rets = [] @@ -263,10 +264,11 @@ class PaimaiDataSend { def aliZcRun2(){ def index = 0 def q = new BasicDBObject() -// q.put("city","宿迁") + q.put("city","苏州") // q.put("region","姑苏") - q.put("url","https://zc-item.taobao.com/auction/672615219601.htm") +// q.put("url","https://zc-item.taobao.com/auction/636884262340.htm") def dbcursor = ali_zc_coll.find(q).batchSize(1000) + def rets = [] for(def d : dbcursor){ index++ println "ali-zc: "+index @@ -290,19 +292,40 @@ class PaimaiDataSend { result.put("region", d.region) json.put("result", result) - byte[] packedMsg = MessageUtils.pack(json) - CommonUtils.sendMessage("result2parse2dev", packedMsg) - - - def udata = new BasicDBObject() - udata.put("parser_status", 4) - udata.put("m_date", new Date()) - ali_zc_coll.update(new BasicDBObject("_id", d.get("_id")), new BasicDBObject('$set', udata)) + rets << result } + println "共:"+rets.size() + + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(rets.size()) + for(def i=0;i