链接改为线上
This commit is contained in:
+1
-1
@@ -27,7 +27,7 @@ import java.util.regex.Pattern
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class BigdataCommunityBatch extends BaseBatch{
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
|
||||
static main(args) {
|
||||
runBatch(new BigdataCommunityBatch(), args);
|
||||
|
||||
+5
-4
@@ -39,12 +39,12 @@ class PaimaiDataSend {
|
||||
|
||||
// task.aliZcRun()
|
||||
// task.aliZcRun2()
|
||||
// task.aliFpRun()
|
||||
task.aliFpRun()
|
||||
// task.rmfysszcRun()
|
||||
// task.jdRun()
|
||||
// task.checkUrl()
|
||||
|
||||
task.paimaiSendToEs()
|
||||
// task.paimaiSendToEs()
|
||||
}
|
||||
|
||||
|
||||
@@ -336,7 +336,7 @@ class PaimaiDataSend {
|
||||
// q.put("city",new BasicDBObject('$in', ["南京","宿迁","苏州"]))
|
||||
q.put("province","江苏")
|
||||
// q.put("parser_status", new BasicDBObject('$ne',2))
|
||||
// q.put("url","https://sf-item.taobao.com/sf_item/625652638855.htm")
|
||||
q.put("url","https://sf-item.taobao.com/sf_item/665795393092.htm")
|
||||
//https://sf-item.taobao.com/sf_item/657131607486.htm
|
||||
// q.put("parser_status", new BasicDBObject('$exists',false))
|
||||
// q.put("finishDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-09-01")))
|
||||
@@ -522,7 +522,8 @@ class PaimaiDataSend {
|
||||
def getPaimaiDatas(){
|
||||
def q = new BasicDBObject()
|
||||
// q.put("city",new BasicDBObject('$in', ["南京","宿迁","苏州"]))
|
||||
q.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-10-01")))
|
||||
q.put("city","上海")
|
||||
q.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-09-01")))
|
||||
def dbcursor = paimai43_COLL.find(q).batchSize(5000)
|
||||
def rets = []
|
||||
for(def d : dbcursor){
|
||||
|
||||
+4
-4
@@ -40,11 +40,11 @@ class DataStat {
|
||||
|
||||
|
||||
def run(){
|
||||
def source = "我爱我家"//
|
||||
def citys = findCrawlCity(source, 202207);
|
||||
def source = "安居客"//
|
||||
def citys = findCrawlCity(source, 202208);
|
||||
for(def city : citys){
|
||||
def count1 = findCount(source, city, 202207);
|
||||
def count2 = findCount(source, city, 202208);
|
||||
def count1 = findCount(source, city, 202208);
|
||||
def count2 = findCount(source, city, 202209);
|
||||
println city +" "+count1+" "+count2
|
||||
// println count1
|
||||
// println "--- ----"
|
||||
|
||||
+3
-2
@@ -356,10 +356,11 @@ public class OpenApiClient {
|
||||
params.put("token", TOKEN);
|
||||
// params.put("__refresh_cache","1")
|
||||
|
||||
String content = new HttpRequest().get("http://192.168.0.100:30115/address/standard", params, "utf-8")
|
||||
//192.168.0.100:30115
|
||||
String content = new HttpRequest().get("http://open.fangjia.com/address/standard", params, "utf-8")
|
||||
JSONObject result = JSONObject.fromObject(content);
|
||||
while (result == null) {
|
||||
result = HttpRequestUtil.getJsonContent("http://192.168.0.100:30115/address/standard", params, "utf-8")
|
||||
result = HttpRequestUtil.getJsonContent("http://open.fangjia.com/address/standard", params, "utf-8")
|
||||
println("${city} ${address} 程序调用异常")
|
||||
Thread.sleep(6000)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user