diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/auto/CityInfoEmailSender.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/auto/CityInfoEmailSender.groovy index cb0a3f0f7..323ce415b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/auto/CityInfoEmailSender.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/auto/CityInfoEmailSender.groovy @@ -23,14 +23,7 @@ class CityInfoEmailSender extends BaseBatch{ private static final Jms jms = Jms.create('tcp://monitor.jms.db.fangjia.com:61616') final static DBCollection fsDB = new Mongo("fs.mongo.db.fangjia.com", 27000).getDB('fangjia_fs').getCollection('files') def emails = [ - "wangsheng@fangjia.com" - ,"xuyejun@fangjia.com" - ,"tiki@fangjia.com" - ,"zhangzx@fangjia.com" - ,"panyingjiao@fangjia.com" - ,"yangqixing@fangjia.com" - ,"james.lin@fangjia.com" - ,"jhu@fangjia.com" + "gaojuan@fangjia.com" ] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeHouseCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeHouseCrawl.groovy index 81056b29d..7c6dcf4de 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeHouseCrawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeHouseCrawl.groovy @@ -25,14 +25,14 @@ class AnjukeHouseCrawl { System.setProperty("http.nonProxyHosts", "localhost"); WebDriver driver = new ChromeDriver() - def href = "https://sh.esf.fang.com/loupan/1210043376.htm" + def href = "https://sh.58.com/hezu/50696364798384x.shtml" def doc = getDocument(href, driver) // def doc = ClaABuYun.getResponse(href) // def newUrl = href.substring(0, hrefs.indexOf("?")) - sendMq(href, doc, "district_detail_fang") + sendMq(href, doc, "rent_58") -// for(def i=1;i<50;i++){ -// def url = "https://fz.anjuke.com/sale/guloue/p${i}/" +// for(def i=25;i<50;i++){ +// def url = "https://fz.anjuke.com/sale/jinana/p${i}/" // def doc = getDocument(url, driver) // def eles = JQueryUtils.select(".list-main .list-left .list .property a", doc) // println "页数:"+i+" 记录数:"+eles.size() @@ -203,7 +203,18 @@ class AnjukeHouseCrawl { } if(!els){ - els = JQueryUtils.select(".village_info", doc) + els = JQueryUtils.select(".detail-content", doc) + } + if(!els){ + els = JQueryUtils.select(".content", doc) + } + + if(!els){ + els = JQueryUtils.select(".wrapper", doc) + } + + if(!els){ + els = JQueryUtils.select(".main-wrap", doc) } if(els && els.size()>0){ stop = true; diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/PaimaiDataSend.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/PaimaiDataSend.groovy new file mode 100644 index 000000000..cdbcbc915 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/PaimaiDataSend.groovy @@ -0,0 +1,71 @@ +package com.fangjia.script.gj.crawl + +import com.alibaba.fastjson.JSON +import com.alibaba.fastjson.JSONObject +import com.fangjia.script.temp.shenjian.util.MessageUtils +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.rabbitmq.client.AMQP + +import java.text.SimpleDateFormat + +DBCollection collection = MongoUtils.getCollectionByName("fangjia_crawl", "ali_fapai_detail", "192.168.0.235", 27018) + +def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; +DBCollection paimai43_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "pai_mai", lineUrl) + + +//Date date=new Date(1550455200000); +//SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); +//String str = format.format(date); +//System.out.println(str); + +def pmquery = new BasicDBObject() +//pmquery.put("city","重庆") +//pmquery.put("startDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2019-01-01"))) +//pmquery.put("status","已成交") +//pmquery.put("estateType", null) +pmquery.put("url","https://susong-item.taobao.com/auction/648798773106.htm") + +//def cur = paimai43_COLL.find(pmquery) +//def urls = [] +//for(def o : cur){ +// urls << o.url +//} + +def index = 0 +//for(def url : urls){ + def q = new BasicDBObject() +// q.put("url", url) + def dbcursor = collection.find(q) + + for(def d : dbcursor){ + index++ + println index + JSON json = new JSONObject() + json.put("project", "paimai_taobao") + json.put("url", d.url) + + JSON result = new JSONObject() + result.put("content", d.content) + def bid_content = net.sf.json.JSONObject.fromObject(d.bid_content) + result.put("trades", bid_content) + result.put("condition", d.condition) + result.put("sub_count", d.sub_count) + def data_json = net.sf.json.JSONObject.fromObject(d.json) + result.put("data_json", data_json) + result.put("desc_content", d.desc_content) + result.put("item_notice_content", d.item_notice_content) + result.put("notice_detail_content", d.notice_detail_content) + + json.put("result", result) + + byte[] packedMsg = MessageUtils.pack(json) + CommonUtils.sendMessage("result2parse", packedMsg) + } +//} + + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/hubingliang/PdsUseStatisticBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/hubingliang/PdsUseStatisticBatch.groovy index 98110403e..3579922b1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/hubingliang/PdsUseStatisticBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/hubingliang/PdsUseStatisticBatch.groovy @@ -44,9 +44,7 @@ class PdsUseStatisticBatch extends BaseBatch{ private static final Jms jms = Jms.create('tcp://monitor.jms.db.fangjia.com:61616') final static DBCollection fsDB = MongoUtils.getCollectionByName("fangjia_fs", "files", MongoUtils.MongoDB_Domain_fs_mongo, 27000) def static emails = [ - "hubingliang@fangjia.com", - "james.lin@fangjia.com", - "panyingjiao@fangjia.com", + "duweiliang@fangjia.com", "yaojingmeng@fangjia.com", "xuqi@fangjia.com", diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/hubingliang/pds/DaiQianGuZhiBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/hubingliang/pds/DaiQianGuZhiBatch.groovy index e1f6932eb..49db7393c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/hubingliang/pds/DaiQianGuZhiBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/hubingliang/pds/DaiQianGuZhiBatch.groovy @@ -34,8 +34,7 @@ class DaiQianGuZhiBatch extends BaseBatch{ final static DBCollection fsDB = new Mongo("fs.mongo.db.fangjia.com", 27000).getDB('fangjia_fs').getCollection('files') def static emails = [ "hubingliang@fangjia.com", - "james.lin@fangjia.com", - "panyingjiao@fangjia.com" + "james.lin@fangjia.com" ] static main(args) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/hubingliang/pds/KeHuFenLeiBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/hubingliang/pds/KeHuFenLeiBatch.groovy index 6fdaf20e3..da74f4bd4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/hubingliang/pds/KeHuFenLeiBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/hubingliang/pds/KeHuFenLeiBatch.groovy @@ -34,8 +34,7 @@ class KeHuFenLeiBatch extends BaseBatch{ final static DBCollection fsDB = MongoUtils.getCollectionByName("fangjia_fs", "files", MongoUtils.MongoDB_Domain_fs_mongo, 27000) def static emails = [ "hubingliang@fangjia.com", - "james.lin@fangjia.com", - "panyingjiao@fangjia.com" + "james.lin@fangjia.com" ] static main(args) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/hubingliang/pds/KeHuPingJiBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/hubingliang/pds/KeHuPingJiBatch.groovy index 9aad05cf6..a2219d524 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/hubingliang/pds/KeHuPingJiBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/hubingliang/pds/KeHuPingJiBatch.groovy @@ -33,9 +33,7 @@ class KeHuPingJiBatch extends BaseBatch{ private static final Jms jms = Jms.create('tcp://monitor.jms.db.fangjia.com:61616') final static DBCollection fsDB = new Mongo("fs.mongo.db.fangjia.com", 27000).getDB('fangjia_fs').getCollection('files') def static emails = [ - "hubingliang@fangjia.com", - "james.lin@fangjia.com", - "panyingjiao@fangjia.com" + ] static main(args) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy index 1297ca3ea..8bca701d7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy @@ -11,8 +11,8 @@ import org.bson.Document import static com.mongodb.client.model.Filters.* MongoCollection OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log") -def startDate = DateUtil.parseDate("20220501", "yyyyMMdd") -def endDate = DateUtil.parseDate("20220601", "yyyyMMdd") +def startDate = DateUtil.parseDate("20220601", "yyyyMMdd") +def endDate = DateUtil.parseDate("20220701", "yyyyMMdd") Calendar calendar = Calendar.getInstance() calendar.setTime(startDate) def count = 0 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/batch/ApiProductFieldStatBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/batch/ApiProductFieldStatBatch.groovy index 6f120465c..026ec703f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/batch/ApiProductFieldStatBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/batch/ApiProductFieldStatBatch.groovy @@ -62,7 +62,7 @@ class ApiProductFieldStatBatch extends BaseBatch { private static final Jms jms = Jms.create('tcp://monitor.jms.db.fangjia.com:61616') final static DBCollection fsDB = MongoUtils.getCollectionByName("fangjia_fs", "files", MongoUtils.MongoDB_Domain_fs_mongo, 27000) def emails = [ - "yinjihuan@fangjia.com","panyingjiao@fangjia.com","liyan@fangjia.com", "jiangxiaochun@fangjia.com" + "jiangxiaochun@fangjia.com" ] static def allDis = [] static DecimalFormat df = new DecimalFormat("0.00%")