update
This commit is contained in:
+91
-59
@@ -29,7 +29,7 @@ class PaimaiDataSend {
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection paimai43_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "pai_mai", lineUrl)
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
|
||||
static main(args) {
|
||||
def task = new PaimaiDataSend()
|
||||
@@ -198,9 +198,10 @@ class PaimaiDataSend {
|
||||
def index = 0
|
||||
def q = new BasicDBObject()
|
||||
// q.put("city","宿迁")
|
||||
q.put("url","https://sf-item.taobao.com/sf_item/649014193736.htm")
|
||||
q.put("url","https://sf-item.taobao.com/sf_item/665877683620.htm")
|
||||
//https://sf-item.taobao.com/sf_item/657131607486.htm
|
||||
// q.put("parser_status", new BasicDBObject('$exists',false))
|
||||
// q.put("finishDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-09-01")))
|
||||
def dbcursor = ali_fp_coll.find(q).batchSize(2000)
|
||||
|
||||
def rets = []
|
||||
@@ -263,10 +264,11 @@ class PaimaiDataSend {
|
||||
def aliZcRun2(){
|
||||
def index = 0
|
||||
def q = new BasicDBObject()
|
||||
// q.put("city","宿迁")
|
||||
q.put("city","苏州")
|
||||
// q.put("region","姑苏")
|
||||
q.put("url","https://zc-item.taobao.com/auction/672615219601.htm")
|
||||
// q.put("url","https://zc-item.taobao.com/auction/636884262340.htm")
|
||||
def dbcursor = ali_zc_coll.find(q).batchSize(1000)
|
||||
def rets = []
|
||||
for(def d : dbcursor){
|
||||
index++
|
||||
println "ali-zc: "+index
|
||||
@@ -290,19 +292,40 @@ class PaimaiDataSend {
|
||||
result.put("region", d.region)
|
||||
json.put("result", result)
|
||||
|
||||
byte[] packedMsg = MessageUtils.pack(json)
|
||||
CommonUtils.sendMessage("result2parse2dev", packedMsg)
|
||||
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("parser_status", 4)
|
||||
udata.put("m_date", new Date())
|
||||
ali_zc_coll.update(new BasicDBObject("_id", d.get("_id")), new BasicDBObject('$set', udata))
|
||||
rets << result
|
||||
}
|
||||
println "共:"+rets.size()
|
||||
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(rets.size())
|
||||
for(def i=0;i<rets.size();i++){
|
||||
final def json = rets[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
byte[] packedMsg = MessageUtils.pack(json)
|
||||
CommonUtils.sendMessage("result2parse2dev", packedMsg)//result2parse2dev
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("parser_status", 4)
|
||||
udata.put("m_date", new Date())
|
||||
ali_zc_coll.update(new BasicDBObject("_id", json.get("_id")), new BasicDBObject('$set', udata))
|
||||
println "send: "+totalCount.get()
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
}
|
||||
|
||||
def paimaiSendToEs(){
|
||||
def rets = getPaimaiDatas()
|
||||
println "共:"+rets.size()
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(rets.size())
|
||||
for(def i=0;i<rets.size();i++){
|
||||
@@ -312,7 +335,7 @@ class PaimaiDataSend {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def flag = FangjiaClientUtils.saveToEsPaimai(m)
|
||||
println totalCount.get()+" "+m.city +" "+m.name +" "+flag
|
||||
println totalCount.get()+" "+m.city +" "+m.title +" "+flag
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
@@ -326,56 +349,65 @@ class PaimaiDataSend {
|
||||
|
||||
def getPaimaiDatas(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-08-01")))
|
||||
q.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-07-01")))
|
||||
def dbcursor = paimai43_COLL.find(q).batchSize(1000)
|
||||
def rets = []
|
||||
for(def d : dbcursor){
|
||||
def o = [:]
|
||||
o.addPrice = d.addPrice
|
||||
o.agencyName = d.agencyName
|
||||
o.area = d.area
|
||||
o.buildYear = d.buildYear
|
||||
o.callCount = d.callCount
|
||||
o.city = d.city
|
||||
o.court = d.court
|
||||
o.delayCount = d.delayCount
|
||||
o.delayWeekly = d.delayWeekly
|
||||
o.depositPrice = d.depositPrice
|
||||
o.enrollCount = d.enrollCount
|
||||
o.estateType = d.estateType
|
||||
o.evaluatePrice = d.evaluatePrice
|
||||
o.finishDate = d.finishDate
|
||||
o.height = d.height
|
||||
o.houseNum = d.houseNum
|
||||
o.id = d.id
|
||||
o.introduce = d.introduce
|
||||
if(d.city){
|
||||
def o = [:]
|
||||
o.addPrice = d.addPrice
|
||||
o.agencyName = d.agencyName
|
||||
o.area = d.area
|
||||
o.buildYear = d.buildYear
|
||||
o.callCount = d.callCount
|
||||
o.city = d.city
|
||||
o.court = d.court
|
||||
o.delayCount = d.delayCount
|
||||
o.delayWeekly = d.delayWeekly
|
||||
o.depositPrice = d.depositPrice
|
||||
o.enrollCount = d.enrollCount
|
||||
o.estateType = d.estateType
|
||||
o.evaluatePrice = d.evaluatePrice
|
||||
o.finishDate = d.finishDate
|
||||
o.height = d.height
|
||||
o.houseNum = d.houseNum
|
||||
o.id = d.id
|
||||
o.introduce = d.introduce
|
||||
// o.location = d.location
|
||||
o.lookCount = d.lookCount
|
||||
o.marketPrice = d.marketPrice
|
||||
o.matchAddress = d.matchAddress
|
||||
o.matchCity = d.matchCity
|
||||
o.matchName = d.matchName
|
||||
o.matchRegion = d.matchRegion
|
||||
o.person = d.person
|
||||
o.preemptivePerson = d.preemptivePerson
|
||||
o.pricePkWeekly = d.pricePkWeekly
|
||||
o.province = d.province
|
||||
o.rateDiscount = d.rateDiscount
|
||||
o.region = d.region
|
||||
o.roomNum = d.roomNum
|
||||
o.sellOffAdvancePay = d.sellOffAdvancePay
|
||||
o.source = d.source
|
||||
o.stage = d.stage
|
||||
o.startDate = d.startDate
|
||||
o.startPrice = d.startPrice
|
||||
o.status = d.status
|
||||
o.tel = d.tel
|
||||
o.title = d.title
|
||||
o.tradePrice = d.tradePrice
|
||||
o.type = d.type
|
||||
o.unitNum = d.unitNum
|
||||
o.url = d.url
|
||||
rets << o
|
||||
o.lookCount = d.lookCount
|
||||
o.marketPrice = d.marketPrice
|
||||
o.matchAddress = d.matchAddress
|
||||
o.matchCity = d.matchCity
|
||||
o.matchName = d.matchName
|
||||
o.matchRegion = d.matchRegion
|
||||
o.person = d.person
|
||||
o.preemptivePerson = d.preemptivePerson
|
||||
o.pricePkWeekly = d.pricePkWeekly
|
||||
o.province = d.province
|
||||
o.rateDiscount = d.rateDiscount
|
||||
o.region = d.region
|
||||
o.roomNum = d.roomNum
|
||||
o.sellOffAdvancePay = d.sellOffAdvancePay
|
||||
o.source = d.source
|
||||
o.stage = d.stage
|
||||
o.startDate = d.startDate
|
||||
o.startPrice = d.startPrice
|
||||
o.status = d.status
|
||||
o.tel = d.tel
|
||||
o.title = d.title
|
||||
o.tradePrice = d.tradePrice
|
||||
o.type = d.type
|
||||
o.unitNum = d.unitNum
|
||||
o.url = d.url
|
||||
if(d.url.contains("taobao.com")){
|
||||
o.source = "淘宝"
|
||||
}else if(d.url.contains("jd.com")){
|
||||
o.source = "京东"
|
||||
}else if(d.url.contains("rmfysszc.gov.cn")){
|
||||
o.source = "人民法院讼诉网"
|
||||
}
|
||||
rets << o
|
||||
}
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user