This commit is contained in:
2022-08-25 20:49:31 +08:00
parent bb869271c9
commit 36c932625b
@@ -29,7 +29,7 @@ class PaimaiDataSend {
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection paimai43_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "pai_mai", lineUrl)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
static main(args) {
def task = new PaimaiDataSend()
@@ -198,9 +198,10 @@ class PaimaiDataSend {
def index = 0
def q = new BasicDBObject()
// q.put("city","宿迁")
q.put("url","https://sf-item.taobao.com/sf_item/649014193736.htm")
q.put("url","https://sf-item.taobao.com/sf_item/665877683620.htm")
//https://sf-item.taobao.com/sf_item/657131607486.htm
// q.put("parser_status", new BasicDBObject('$exists',false))
// q.put("finishDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-09-01")))
def dbcursor = ali_fp_coll.find(q).batchSize(2000)
def rets = []
@@ -263,10 +264,11 @@ class PaimaiDataSend {
def aliZcRun2(){
def index = 0
def q = new BasicDBObject()
// q.put("city","宿迁")
q.put("city","苏州")
// q.put("region","姑苏")
q.put("url","https://zc-item.taobao.com/auction/672615219601.htm")
// q.put("url","https://zc-item.taobao.com/auction/636884262340.htm")
def dbcursor = ali_zc_coll.find(q).batchSize(1000)
def rets = []
for(def d : dbcursor){
index++
println "ali-zc: "+index
@@ -290,19 +292,40 @@ class PaimaiDataSend {
result.put("region", d.region)
json.put("result", result)
byte[] packedMsg = MessageUtils.pack(json)
CommonUtils.sendMessage("result2parse2dev", packedMsg)
def udata = new BasicDBObject()
udata.put("parser_status", 4)
udata.put("m_date", new Date())
ali_zc_coll.update(new BasicDBObject("_id", d.get("_id")), new BasicDBObject('$set', udata))
rets << result
}
println "共:"+rets.size()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(rets.size())
for(def i=0;i<rets.size();i++){
final def json = rets[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
byte[] packedMsg = MessageUtils.pack(json)
CommonUtils.sendMessage("result2parse2dev", packedMsg)//result2parse2dev
def udata = new BasicDBObject()
udata.put("parser_status", 4)
udata.put("m_date", new Date())
ali_zc_coll.update(new BasicDBObject("_id", json.get("_id")), new BasicDBObject('$set', udata))
println "send: "+totalCount.get()
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
}
def paimaiSendToEs(){
def rets = getPaimaiDatas()
println "共:"+rets.size()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(rets.size())
for(def i=0;i<rets.size();i++){
@@ -312,7 +335,7 @@ class PaimaiDataSend {
try{
totalCount.addAndGet(1)
def flag = FangjiaClientUtils.saveToEsPaimai(m)
println totalCount.get()+" "+m.city +" "+m.name +" "+flag
println totalCount.get()+" "+m.city +" "+m.title +" "+flag
} catch (Exception e) {
e.printStackTrace()
} finally {
@@ -326,56 +349,65 @@ class PaimaiDataSend {
def getPaimaiDatas(){
def q = new BasicDBObject()
q.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-08-01")))
q.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-07-01")))
def dbcursor = paimai43_COLL.find(q).batchSize(1000)
def rets = []
for(def d : dbcursor){
def o = [:]
o.addPrice = d.addPrice
o.agencyName = d.agencyName
o.area = d.area
o.buildYear = d.buildYear
o.callCount = d.callCount
o.city = d.city
o.court = d.court
o.delayCount = d.delayCount
o.delayWeekly = d.delayWeekly
o.depositPrice = d.depositPrice
o.enrollCount = d.enrollCount
o.estateType = d.estateType
o.evaluatePrice = d.evaluatePrice
o.finishDate = d.finishDate
o.height = d.height
o.houseNum = d.houseNum
o.id = d.id
o.introduce = d.introduce
if(d.city){
def o = [:]
o.addPrice = d.addPrice
o.agencyName = d.agencyName
o.area = d.area
o.buildYear = d.buildYear
o.callCount = d.callCount
o.city = d.city
o.court = d.court
o.delayCount = d.delayCount
o.delayWeekly = d.delayWeekly
o.depositPrice = d.depositPrice
o.enrollCount = d.enrollCount
o.estateType = d.estateType
o.evaluatePrice = d.evaluatePrice
o.finishDate = d.finishDate
o.height = d.height
o.houseNum = d.houseNum
o.id = d.id
o.introduce = d.introduce
// o.location = d.location
o.lookCount = d.lookCount
o.marketPrice = d.marketPrice
o.matchAddress = d.matchAddress
o.matchCity = d.matchCity
o.matchName = d.matchName
o.matchRegion = d.matchRegion
o.person = d.person
o.preemptivePerson = d.preemptivePerson
o.pricePkWeekly = d.pricePkWeekly
o.province = d.province
o.rateDiscount = d.rateDiscount
o.region = d.region
o.roomNum = d.roomNum
o.sellOffAdvancePay = d.sellOffAdvancePay
o.source = d.source
o.stage = d.stage
o.startDate = d.startDate
o.startPrice = d.startPrice
o.status = d.status
o.tel = d.tel
o.title = d.title
o.tradePrice = d.tradePrice
o.type = d.type
o.unitNum = d.unitNum
o.url = d.url
rets << o
o.lookCount = d.lookCount
o.marketPrice = d.marketPrice
o.matchAddress = d.matchAddress
o.matchCity = d.matchCity
o.matchName = d.matchName
o.matchRegion = d.matchRegion
o.person = d.person
o.preemptivePerson = d.preemptivePerson
o.pricePkWeekly = d.pricePkWeekly
o.province = d.province
o.rateDiscount = d.rateDiscount
o.region = d.region
o.roomNum = d.roomNum
o.sellOffAdvancePay = d.sellOffAdvancePay
o.source = d.source
o.stage = d.stage
o.startDate = d.startDate
o.startPrice = d.startPrice
o.status = d.status
o.tel = d.tel
o.title = d.title
o.tradePrice = d.tradePrice
o.type = d.type
o.unitNum = d.unitNum
o.url = d.url
if(d.url.contains("taobao.com")){
o.source = "淘宝"
}else if(d.url.contains("jd.com")){
o.source = "京东"
}else if(d.url.contains("rmfysszc.gov.cn")){
o.source = "人民法院讼诉网"
}
rets << o
}
}
return rets;
}