update
This commit is contained in:
+164
-123
@@ -15,6 +15,7 @@ import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.rabbitmq.client.impl.AMQImpl
|
||||
import groovy.sql.Sql
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
@@ -32,6 +33,9 @@ import java.util.regex.Pattern
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class BigdataDealBatch extends BaseBatch{
|
||||
|
||||
def startDate = 202206
|
||||
def endDate = 202208
|
||||
|
||||
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd"));
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
def sourceMap = ["贝壳找房":"ke.com","我爱我家":"5i5j.com","链家在线":"lianjia.com","房天下":"fang.com","中原地产":"centanet.com","福州不动产":"","搜房":"fang.com"]
|
||||
@@ -41,6 +45,7 @@ class BigdataDealBatch extends BaseBatch{
|
||||
|
||||
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
|
||||
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
@@ -102,6 +107,9 @@ class BigdataDealBatch extends BaseBatch{
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(rets.size())
|
||||
|
||||
// def lianjiaPriceMap = findThirdPartyPrice("上海")
|
||||
def startDate = DateUtils.parseFromTightByPattern(DateUtils.addMonth(new Date(),-2),'yyyy-MM-dd')
|
||||
def lianjiaPriceMap = getDistrictStatByCity("上海", startDate)
|
||||
for(def i=0;i<rets.size();i++){
|
||||
final def trade = rets[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
@@ -109,7 +117,8 @@ class BigdataDealBatch extends BaseBatch{
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
if(source == "youda"){
|
||||
def ret = converYouDaData(trade, cityCodeDatas, distCodeMap)
|
||||
|
||||
def ret = converYouDaData(trade, cityCodeDatas, distCodeMap, lianjiaPriceMap)
|
||||
if(ret){
|
||||
result << ret
|
||||
println totalCount.get() +" "+trade.city+" "+trade.region+" "+trade.fjName+" "+new Date().toLocaleString()
|
||||
@@ -135,147 +144,176 @@ class BigdataDealBatch extends BaseBatch{
|
||||
return result
|
||||
}
|
||||
|
||||
def findThirdPartyPrice(def city) {
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
|
||||
def converYouDaData(def trade, def cityCodeDatas, def distCodeMap){
|
||||
q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte', endDate))
|
||||
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1))
|
||||
def map = [:]
|
||||
for (def db : dbcursor) {
|
||||
def price = null
|
||||
if (db.lianjia_esf_price && db.lianjia_esf_price > 3000) {
|
||||
price = db.lianjia_esf_price
|
||||
}
|
||||
if (!price && db.beike_esf_price && db.beike_esf_price > 3000) {
|
||||
price = db.beike_esf_price
|
||||
}
|
||||
if(price){
|
||||
map.put(db.region+"##"+db.name, price)
|
||||
}
|
||||
|
||||
}
|
||||
return map
|
||||
}
|
||||
|
||||
def converYouDaData(def trade, def cityCodeDatas, def distCodeMap, def lianjiaPriceMap){
|
||||
def ret = []
|
||||
def obj = CcbCommonUtils.findDistrict(trade.fjCity, trade.fjRegion, trade.fjName, "district")
|
||||
if(!obj){
|
||||
return ret;
|
||||
}
|
||||
ret << cityCodeDatas.get(trade.city)
|
||||
ret << trade._id.toString()
|
||||
ret << trade.city
|
||||
def data = distCodeMap.get(trade.fjRegion)
|
||||
if(data){
|
||||
ret << data.district_code
|
||||
ret << data.district
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
}
|
||||
def street = ""
|
||||
def streetNumber = ""
|
||||
if(trade.address){
|
||||
if(trade.address.toString().contains("弄")){
|
||||
String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄"))
|
||||
if(number){
|
||||
street = trade.address.toString().substring(0, trade.address.toString().indexOf(number))
|
||||
streetNumber = number
|
||||
def price = lianjiaPriceMap.get(obj.region+"##"+obj.name)
|
||||
if(price){
|
||||
def mod = java.lang.Math.abs((trade.price-price)*100/price)
|
||||
if(mod<=10){
|
||||
ret << cityCodeDatas.get(trade.city)
|
||||
ret << trade._id.toString()
|
||||
ret << trade.city
|
||||
def data = distCodeMap.get(trade.fjRegion)
|
||||
if(data){
|
||||
ret << data.district_code
|
||||
ret << data.district
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
}
|
||||
}
|
||||
if(!street){
|
||||
def jsonObj = OpenApiClient.addressMatch(trade.city, trade.address, "property", false, trade.region, trade.name)
|
||||
if(jsonObj && jsonObj.searchAddress){
|
||||
if(jsonObj.searchAddress.street && jsonObj.searchAddress.street.contains("弄")){
|
||||
def street = ""
|
||||
def streetNumber = ""
|
||||
if(trade.address){
|
||||
if(trade.address.toString().contains("弄")){
|
||||
String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄"))
|
||||
if(number){
|
||||
street = trade.address.toString().substring(0, trade.address.toString().indexOf(number))
|
||||
streetNumber = number
|
||||
}
|
||||
}else{
|
||||
street = jsonObj.searchAddress.street
|
||||
streetNumber = jsonObj.searchAddress.streetNumber
|
||||
}
|
||||
if(!street){
|
||||
def jsonObj = OpenApiClient.addressMatch(trade.city, trade.address, "property", false, trade.region, trade.name)
|
||||
if(jsonObj && jsonObj.searchAddress){
|
||||
if(jsonObj.searchAddress.street && jsonObj.searchAddress.street.contains("弄")){
|
||||
String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄"))
|
||||
if(number){
|
||||
street = trade.address.toString().substring(0, trade.address.toString().indexOf(number))
|
||||
streetNumber = number
|
||||
}
|
||||
}else{
|
||||
street = jsonObj.searchAddress.street
|
||||
streetNumber = jsonObj.searchAddress.streetNumber
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(!street){
|
||||
if(obj.address){
|
||||
if(obj.address.toString().contains("弄")){
|
||||
String number = DataUtil.match(obj.address, Pattern.compile("\\d+弄"))
|
||||
if(number){
|
||||
street = obj.address.toString().substring(0, obj.address.toString().indexOf(number))
|
||||
streetNumber = number
|
||||
if(!street){
|
||||
if(obj.address){
|
||||
if(obj.address.toString().contains("弄")){
|
||||
String number = DataUtil.match(obj.address, Pattern.compile("\\d+弄"))
|
||||
if(number){
|
||||
street = obj.address.toString().substring(0, obj.address.toString().indexOf(number))
|
||||
streetNumber = number
|
||||
}
|
||||
}else{
|
||||
def jsonObj = OpenApiClient.addressMatch(obj.city, obj.address, "property", false, trade.region, obj.name)
|
||||
if(jsonObj && jsonObj.searchAddress){
|
||||
street = jsonObj.searchAddress.street
|
||||
streetNumber = jsonObj.searchAddress.streetNumber
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ret << street
|
||||
ret << streetNumber
|
||||
|
||||
|
||||
if(obj){
|
||||
ret << obj._id.toString()
|
||||
ret << trade.fjName
|
||||
}
|
||||
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << CcbCodeUtils.parseHouseProperty(obj.house_roperty)//权属
|
||||
ret << CcbCodeUtils.parseHousePropertyYear(obj.property_life)
|
||||
ret << ""//期数
|
||||
String roomNum = DataUtil.match(trade.address, Pattern.compile("\\d+室"))
|
||||
if(roomNum && roomNum.contains("0")){
|
||||
ret << roomNum.substring(0, roomNum.lastIndexOf("0"))
|
||||
}else{
|
||||
def jsonObj = OpenApiClient.addressMatch(obj.city, obj.address, "property", false, trade.region, obj.name)
|
||||
if(jsonObj && jsonObj.searchAddress){
|
||||
street = jsonObj.searchAddress.street
|
||||
streetNumber = jsonObj.searchAddress.streetNumber
|
||||
}
|
||||
ret << trade.floor
|
||||
}
|
||||
|
||||
ret << ""
|
||||
ret << trade.height
|
||||
def arrs = CcbCodeUtils.parseBuildType(obj.build_type)//建筑类型
|
||||
def buildType = ''
|
||||
if(arrs){
|
||||
buildType = arrs[0]
|
||||
}
|
||||
ret << buildType
|
||||
ret << CcbCodeUtils.getBuildStructureCode(obj.build_structure)
|
||||
ret << trade.area
|
||||
ret << ""
|
||||
ret << getHouseRoom(buildType, trade.room, trade.hall)
|
||||
ret << trade.room
|
||||
ret << trade.hall
|
||||
ret << ""
|
||||
ret << trade.toilet
|
||||
ret << CcbCodeUtils.parseHouseUse(obj.estate_type2)
|
||||
ret << ""
|
||||
ret << CcbCodeUtils.getDirectionCode(trade.direction)
|
||||
ret << CcbCodeUtils.getFitmentCode(trade.fitment)
|
||||
ret << ""
|
||||
ret << trade.price
|
||||
ret << trade.totalPrice * 10000
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << DateUtils.formatDateByPattern(trade.contractTime, "yyyy-MM-dd")
|
||||
ret << "02"
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << "99"
|
||||
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << "FJW"
|
||||
ret << trade.totalPrice * 10000
|
||||
ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")
|
||||
}
|
||||
}
|
||||
|
||||
ret << street
|
||||
ret << streetNumber
|
||||
|
||||
|
||||
if(obj){
|
||||
ret << obj._id.toString()
|
||||
ret << trade.fjName
|
||||
}
|
||||
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << CcbCodeUtils.parseHouseProperty(obj.house_roperty)//权属
|
||||
ret << CcbCodeUtils.parseHousePropertyYear(obj.property_life)
|
||||
ret << ""//期数
|
||||
String roomNum = DataUtil.match(trade.address, Pattern.compile("\\d+室"))
|
||||
if(roomNum && roomNum.contains("0")){
|
||||
ret << roomNum.substring(0, roomNum.lastIndexOf("0"))
|
||||
}else{
|
||||
ret << trade.floor
|
||||
}
|
||||
|
||||
ret << ""
|
||||
ret << trade.height
|
||||
def arrs = CcbCodeUtils.parseBuildType(obj.build_type)//建筑类型
|
||||
def buildType = ''
|
||||
if(arrs){
|
||||
buildType = arrs[0]
|
||||
}
|
||||
ret << buildType
|
||||
ret << CcbCodeUtils.getBuildStructureCode(obj.build_structure)
|
||||
ret << trade.area
|
||||
ret << ""
|
||||
ret << getHouseRoom(buildType, trade.room, trade.hall)
|
||||
ret << trade.room
|
||||
ret << trade.hall
|
||||
ret << ""
|
||||
ret << trade.toilet
|
||||
ret << CcbCodeUtils.parseHouseUse(obj.estate_type2)
|
||||
ret << ""
|
||||
ret << CcbCodeUtils.getDirectionCode(trade.direction)
|
||||
ret << CcbCodeUtils.getFitmentCode(trade.fitment)
|
||||
ret << ""
|
||||
ret << trade.price
|
||||
ret << trade.totalPrice * 10000
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << DateUtils.formatDateByPattern(trade.contractTime, "yyyy-MM-dd")
|
||||
ret << "02"
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << "99"
|
||||
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << "FJW"
|
||||
ret << trade.totalPrice * 10000
|
||||
ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")
|
||||
return ret;
|
||||
}
|
||||
|
||||
@@ -543,11 +581,14 @@ class BigdataDealBatch extends BaseBatch{
|
||||
def getDistrictStatByCity(city, weekend){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("weekend", weekend)
|
||||
def dbcursor = DIST_SELL_STAT.find(q)
|
||||
q.put("weekend", new BasicDBObject('$gte', weekend))
|
||||
def dbcursor = DIST_SELL_STAT.find(q).sort(new BasicDBObject("weekend",-1))
|
||||
def map = new ConcurrentHashMap()
|
||||
for(def d : dbcursor){
|
||||
map.put(d.region+"@"+d.district_name, d.avg_price)
|
||||
def price = map.get(d.region+"##"+d.district_name)
|
||||
if(!price && d.value_type==2){
|
||||
map.put(d.region+"##"+d.district_name, d.avg_price)
|
||||
}
|
||||
}
|
||||
return map
|
||||
}
|
||||
|
||||
+1
-1
@@ -44,7 +44,7 @@ class GuiDingPriceBatch extends BaseBatch{
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
|
||||
//添加自己库中的人工指导价
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2022-07-22", "2022-08-24")
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2022-08-24", "2022-09-21")
|
||||
if(prices){
|
||||
def distMap = getDistrictMap(city)
|
||||
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
|
||||
|
||||
+950
-914
File diff suppressed because it is too large
Load Diff
+5
-5
@@ -75,13 +75,13 @@ class TradeStatLog {
|
||||
def run(){
|
||||
def citys = CcbCommonUtils.citys;
|
||||
def rets = []
|
||||
rets << ["城市","5月总量","6月总量","7月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
|
||||
rets << ["城市","6月总量","7月总量","8月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
|
||||
def sources = ["贝壳找房","链家在线", "我爱我家","房天下","中原地产"]
|
||||
//def startDate = "2021-05-01"
|
||||
def startDate = "2022-05-01"
|
||||
def date = "2022-06-01"
|
||||
def endDate = "2022-07-01"
|
||||
def newDate = "2022-08-01"
|
||||
def startDate = "2022-06-01"
|
||||
def date = "2022-07-01"
|
||||
def endDate = "2022-08-01"
|
||||
def newDate = "2022-09-01"
|
||||
|
||||
for(def city : citys){
|
||||
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
|
||||
|
||||
+1
-1
@@ -754,7 +754,7 @@ class CommonDatautils {
|
||||
q.put("visible",0)
|
||||
q.put("cat", cat);
|
||||
q.put("city", city);
|
||||
// q.put("name","森望芳草绿岸")
|
||||
// q.put("name","湖畔佳苑")
|
||||
|
||||
DBObject keys = new BasicDBObject();
|
||||
if(fields){
|
||||
|
||||
+4
-4
@@ -48,11 +48,11 @@ class YouDaDataParser {
|
||||
def final AtomicInteger count_ = new AtomicInteger(0)
|
||||
|
||||
|
||||
static def statDate = "2022-07-01"
|
||||
static def statDate = "2022-08-01"
|
||||
static main(args) {
|
||||
def task = new YouDaDataParser();
|
||||
// task.newHouseparse();//友达一手房存入mongo
|
||||
// task.esfParser();//友达二手房存入mongo
|
||||
// task.esfParser("D:\\数据\\友达\\2022年9月\\查询1.DBF");//友达二手房存入mongo
|
||||
// task.newhouseToSaveEs();//新房保存到es
|
||||
task.esfhouseToSaveEs();//二手房保存到es
|
||||
// task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间
|
||||
@@ -284,8 +284,8 @@ class YouDaDataParser {
|
||||
}
|
||||
|
||||
|
||||
def esfParser(){
|
||||
def files = ["D:\\数据\\友达\\2022年8月\\查询1.DBF"]
|
||||
def esfParser(path){
|
||||
def files = [path]
|
||||
def datas = []
|
||||
for(def fileName : files){
|
||||
def rows = readDBF(fileName);
|
||||
|
||||
+1
-1
@@ -86,7 +86,7 @@ class DistrictIndexBatch extends BaseBatch{
|
||||
city_avg_total_price = 531.2615913
|
||||
}
|
||||
if(city_avg_total_price){
|
||||
calcute(city, dists, code, city_avg_total_price)
|
||||
// calcute(city, dists, code, city_avg_total_price)
|
||||
calcuteIndex(city, dists, code, city_avg_total_price)
|
||||
|
||||
}
|
||||
|
||||
+2
-2
@@ -250,7 +250,7 @@ public class LianjiaCrawlByChrome {
|
||||
def durl = e.select(".title a").attr("href")
|
||||
def dealDate = e.select(".dealDate").text()
|
||||
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
|
||||
def endDate = DateUtils.parseFromTightByPattern("2022-01-01", "yyyy-MM-dd")
|
||||
def endDate = DateUtils.parseFromTightByPattern("2022-07-20", "yyyy-MM-dd")
|
||||
if(dealDate.getTime()>=endDate.getTime()){
|
||||
detailUrls << durl
|
||||
parse(city, e, regionText)
|
||||
@@ -469,7 +469,7 @@ public class LianjiaCrawlByChrome {
|
||||
def cityDoc = Jsoup.parse(driver.getPageSource());
|
||||
def cityEles = JQueryUtils.select(".city_province a", cityDoc)
|
||||
def cityMap = [:]
|
||||
def citys= ["石家庄","贵阳"]
|
||||
def citys= ["苏州"]
|
||||
if(cityEles){
|
||||
for(def e : cityEles) {
|
||||
def href = e.attr("href")
|
||||
|
||||
+53
-16
@@ -36,13 +36,13 @@ class PaimaiDataSend {
|
||||
def task = new PaimaiDataSend()
|
||||
// task.jdRunByMongo()
|
||||
// task.jdRun()
|
||||
// task.aliZcRun()
|
||||
task.aliZcRun2()
|
||||
//// task.aliZcRun()
|
||||
// task.aliZcRun2()
|
||||
// task.aliFpRun()
|
||||
// task.rmfysszcRun()
|
||||
// task.checkUrl()
|
||||
|
||||
// task.paimaiSendToEs()
|
||||
task.paimaiSendToEs()
|
||||
}
|
||||
|
||||
|
||||
@@ -66,7 +66,8 @@ class PaimaiDataSend {
|
||||
def rmfysszcRun(){
|
||||
def index = 0
|
||||
def q = new BasicDBObject()
|
||||
q.put("city",new BasicDBObject('$in', ["南京市", "宿迁市", "苏州市"]))
|
||||
// q.put("city",new BasicDBObject('$in', ["南京市", "宿迁市", "苏州市"]))
|
||||
q.put("province","广东省")
|
||||
// q.put("url", "https://www.rmfysszc.gov.cn/statichtml/rm_obj/88061.shtml")
|
||||
def dbcursor = rmfysszc_coll.find(q).batchSize(3000)
|
||||
for(def d : dbcursor){
|
||||
@@ -149,10 +150,12 @@ class PaimaiDataSend {
|
||||
def jdRun(){
|
||||
def index = 0
|
||||
def q = new BasicDBObject()
|
||||
q.put("city",new BasicDBObject('$in', ["南京市", "宿迁市","苏州市"]))
|
||||
// q.put("city",new BasicDBObject('$in', ["南京市", "宿迁市","苏州市"]))
|
||||
q.put("province","广东")
|
||||
// q.put("parser_status", new BasicDBObject('$ne',3))
|
||||
// q.put("url","https://paimai.jd.com/115669796")
|
||||
def dbcursor = jd_coll.find(q).batchSize(3000)
|
||||
def rets = []
|
||||
for(def d : dbcursor){
|
||||
index++
|
||||
println "jd:"+index
|
||||
@@ -175,16 +178,49 @@ class PaimaiDataSend {
|
||||
result.put("notice_detail_content", d.notice_detail_content.content)
|
||||
json.put("result", result)
|
||||
|
||||
byte[] packedMsg = MessageUtils.pack(json)
|
||||
CommonUtils.sendMessage("result2parse2dev", packedMsg)
|
||||
rets << json
|
||||
if(rets.size()==10000){
|
||||
println "共:"+rets.size()
|
||||
sendJD(rets)
|
||||
rets.clear()
|
||||
}
|
||||
}
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("parser_status", 3)
|
||||
udata.put("m_date", new Date())
|
||||
jd_coll.update(new BasicDBObject("_id", d.get("_id")), new BasicDBObject('$set', udata))
|
||||
if(rets.size()>0){
|
||||
println "共:"+rets.size()
|
||||
sendJD(rets)
|
||||
}
|
||||
}
|
||||
|
||||
def sendJD(def rets){
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(rets.size())
|
||||
for(def i=0;i<rets.size();i++){
|
||||
final def json = rets[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
byte[] packedMsg = MessageUtils.pack(json)
|
||||
CommonUtils.sendMessage("result2parse2dev", packedMsg)
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("parser_status", 3)
|
||||
udata.put("m_date", new Date())
|
||||
jd_coll.update(new BasicDBObject("_id", json.get("url")), new BasicDBObject('$set', udata))
|
||||
println "send: "+totalCount.get()
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
}
|
||||
|
||||
|
||||
def aliZcRun(){
|
||||
def pm_q = new BasicDBObject()
|
||||
pm_q.put("city",new BasicDBObject('$in', ["南京", "宿迁","苏州"]))
|
||||
@@ -294,7 +330,8 @@ class PaimaiDataSend {
|
||||
def aliFpRun(){
|
||||
def index = 0
|
||||
def q = new BasicDBObject()
|
||||
q.put("city",new BasicDBObject('$in', ["南京","宿迁","苏州"]))
|
||||
// q.put("city",new BasicDBObject('$in', ["南京","宿迁","苏州"]))
|
||||
q.put("province","广东")
|
||||
// q.put("parser_status", new BasicDBObject('$ne',2))
|
||||
// q.put("url","https://sf-item.taobao.com/sf_item/631662596036.htm")
|
||||
//https://sf-item.taobao.com/sf_item/657131607486.htm
|
||||
@@ -374,10 +411,10 @@ class PaimaiDataSend {
|
||||
def index = 0
|
||||
def q = new BasicDBObject()
|
||||
// q.put("city",new BasicDBObject('$in', ["苏州","宿迁","南京"]))
|
||||
q.put("city","上海")
|
||||
q.put("province","广东")
|
||||
// q.put("parser_status", new BasicDBObject('$ne',1))
|
||||
// q.put("region","姑苏")
|
||||
q.put("url","https://susong-item.taobao.com/auction/652481594083.htm")
|
||||
// q.put("url","https://susong-item.taobao.com/auction/652481594083.htm")
|
||||
def dbcursor = ali_zc_coll.find(q).batchSize(5000)
|
||||
def rets = []
|
||||
for(def d : dbcursor){
|
||||
@@ -476,8 +513,8 @@ class PaimaiDataSend {
|
||||
|
||||
def getPaimaiDatas(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city",new BasicDBObject('$in', ["南京","宿迁","苏州"]))
|
||||
q.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-08-01")))
|
||||
// q.put("city",new BasicDBObject('$in', ["南京","宿迁","苏州"]))
|
||||
q.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-09-15")))
|
||||
def dbcursor = paimai43_COLL.find(q).batchSize(5000)
|
||||
def rets = []
|
||||
for(def d : dbcursor){
|
||||
|
||||
+1
-1
@@ -25,7 +25,7 @@ ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
|
||||
def q = new BasicDBObject()
|
||||
// q.put("province",new BasicDBObject('$in', ["江苏","江苏省"]))
|
||||
q.put("status",new BasicDBObject('$ne', 1))
|
||||
def dbcursor = school_coll.find(q).batchSize(5gdc,rfyhyerhtrbhjfxcs- vvv000)
|
||||
def dbcursor = school_coll.find(q).batchSize(5000)
|
||||
def rets = []
|
||||
for(def d : dbcursor){
|
||||
JSON json = new JSONObject()
|
||||
|
||||
Reference in New Issue
Block a user