From 700b4fe67a660c43d7489b8a6e28f5357c75adef Mon Sep 17 00:00:00 2001 From: gaojuan Date: Sun, 9 Oct 2022 17:14:40 +0800 Subject: [PATCH] =?UTF-8?q?=E9=93=BE=E6=8E=A5=E6=94=B9=E4=B8=BA=E7=BA=BF?= =?UTF-8?q?=E4=B8=8A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/ccbevaluation/BigdataCommunityBatch.groovy | 2 +- .../com/fangjia/script/gj/paimai/PaimaiDataSend.groovy | 9 +++++---- .../com/fangjia/script/thirdPriceCrawl/DataStat.groovy | 8 ++++---- .../groovy/com/fangjia/script/util/OpenApiClient.groovy | 5 +++-- 4 files changed, 13 insertions(+), 11 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy index a6ab136cb..7bc77893a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -27,7 +27,7 @@ import java.util.regex.Pattern @BatchOwner(Watcher.GaoJuan) class BigdataCommunityBatch extends BaseBatch{ - public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) static main(args) { runBatch(new BigdataCommunityBatch(), args); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataSend.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataSend.groovy index e942aff70..fb2126f0f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataSend.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataSend.groovy @@ -39,12 +39,12 @@ class PaimaiDataSend { // task.aliZcRun() // task.aliZcRun2() -// task.aliFpRun() + task.aliFpRun() // task.rmfysszcRun() // task.jdRun() // task.checkUrl() - task.paimaiSendToEs() +// task.paimaiSendToEs() } @@ -336,7 +336,7 @@ class PaimaiDataSend { // q.put("city",new BasicDBObject('$in', ["南京","宿迁","苏州"])) q.put("province","江苏") // q.put("parser_status", new BasicDBObject('$ne',2)) -// q.put("url","https://sf-item.taobao.com/sf_item/625652638855.htm") + q.put("url","https://sf-item.taobao.com/sf_item/665795393092.htm") //https://sf-item.taobao.com/sf_item/657131607486.htm // q.put("parser_status", new BasicDBObject('$exists',false)) // q.put("finishDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-09-01"))) @@ -522,7 +522,8 @@ class PaimaiDataSend { def getPaimaiDatas(){ def q = new BasicDBObject() // q.put("city",new BasicDBObject('$in', ["南京","宿迁","苏州"])) - q.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-10-01"))) + q.put("city","上海") + q.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-09-01"))) def dbcursor = paimai43_COLL.find(q).batchSize(5000) def rets = [] for(def d : dbcursor){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy index 1ae32a425..e76898c7f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy @@ -40,11 +40,11 @@ class DataStat { def run(){ - def source = "我爱我家"// - def citys = findCrawlCity(source, 202207); + def source = "安居客"// + def citys = findCrawlCity(source, 202208); for(def city : citys){ - def count1 = findCount(source, city, 202207); - def count2 = findCount(source, city, 202208); + def count1 = findCount(source, city, 202208); + def count2 = findCount(source, city, 202209); println city +" "+count1+" "+count2 // println count1 // println "--- ----" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy index 0a02bada0..68cd266c8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy @@ -356,10 +356,11 @@ public class OpenApiClient { params.put("token", TOKEN); // params.put("__refresh_cache","1") - String content = new HttpRequest().get("http://192.168.0.100:30115/address/standard", params, "utf-8") + //192.168.0.100:30115 + String content = new HttpRequest().get("http://open.fangjia.com/address/standard", params, "utf-8") JSONObject result = JSONObject.fromObject(content); while (result == null) { - result = HttpRequestUtil.getJsonContent("http://192.168.0.100:30115/address/standard", params, "utf-8") + result = HttpRequestUtil.getJsonContent("http://open.fangjia.com/address/standard", params, "utf-8") println("${city} ${address} 程序调用异常") Thread.sleep(6000) }