链接改为线上

This commit is contained in:
2022-10-09 17:14:40 +08:00
parent 660029377d
commit 700b4fe67a
4 changed files with 13 additions and 11 deletions
@@ -27,7 +27,7 @@ import java.util.regex.Pattern
@BatchOwner(Watcher.GaoJuan)
class BigdataCommunityBatch extends BaseBatch{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
static main(args) {
runBatch(new BigdataCommunityBatch(), args);
@@ -39,12 +39,12 @@ class PaimaiDataSend {
// task.aliZcRun()
// task.aliZcRun2()
// task.aliFpRun()
task.aliFpRun()
// task.rmfysszcRun()
// task.jdRun()
// task.checkUrl()
task.paimaiSendToEs()
// task.paimaiSendToEs()
}
@@ -336,7 +336,7 @@ class PaimaiDataSend {
// q.put("city",new BasicDBObject('$in', ["南京","宿迁","苏州"]))
q.put("province","江苏")
// q.put("parser_status", new BasicDBObject('$ne',2))
// q.put("url","https://sf-item.taobao.com/sf_item/625652638855.htm")
q.put("url","https://sf-item.taobao.com/sf_item/665795393092.htm")
//https://sf-item.taobao.com/sf_item/657131607486.htm
// q.put("parser_status", new BasicDBObject('$exists',false))
// q.put("finishDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-09-01")))
@@ -522,7 +522,8 @@ class PaimaiDataSend {
def getPaimaiDatas(){
def q = new BasicDBObject()
// q.put("city",new BasicDBObject('$in', ["南京","宿迁","苏州"]))
q.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-10-01")))
q.put("city","上海")
q.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-09-01")))
def dbcursor = paimai43_COLL.find(q).batchSize(5000)
def rets = []
for(def d : dbcursor){
@@ -40,11 +40,11 @@ class DataStat {
def run(){
def source = "我爱我家"//
def citys = findCrawlCity(source, 202207);
def source = "安居客"//
def citys = findCrawlCity(source, 202208);
for(def city : citys){
def count1 = findCount(source, city, 202207);
def count2 = findCount(source, city, 202208);
def count1 = findCount(source, city, 202208);
def count2 = findCount(source, city, 202209);
println city +" "+count1+" "+count2
// println count1
// println "--- ----"
@@ -356,10 +356,11 @@ public class OpenApiClient {
params.put("token", TOKEN);
// params.put("__refresh_cache","1")
String content = new HttpRequest().get("http://192.168.0.100:30115/address/standard", params, "utf-8")
//192.168.0.100:30115
String content = new HttpRequest().get("http://open.fangjia.com/address/standard", params, "utf-8")
JSONObject result = JSONObject.fromObject(content);
while (result == null) {
result = HttpRequestUtil.getJsonContent("http://192.168.0.100:30115/address/standard", params, "utf-8")
result = HttpRequestUtil.getJsonContent("http://open.fangjia.com/address/standard", params, "utf-8")
println("${city} ${address} 程序调用异常")
Thread.sleep(6000)
}