From 7fe4a709f5d848225d9474fee6e5cc7a6902cd8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Wed, 26 May 2021 17:16:14 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E5=A2=9E58=E5=B0=8F=E5=8C=BA=E8=AF=A6?= =?UTF-8?q?=E6=83=85=E8=84=9A=E6=9C=AC=E6=8A=93=E5=8F=96=E6=96=B9=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- projects/fangjia-script/pom.xml | 5 ++ .../crawl/58/DistrictDetailCrawl.groovy | 46 +++++++++++++++++++ .../temp/shenjian/util/MessageUtils.groovy | 45 ++++++++++++++++++ .../fangjia/script/util/CommonUtils.groovy | 11 ++++- 4 files changed, 106 insertions(+), 1 deletion(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/DistrictDetailCrawl.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/MessageUtils.groovy diff --git a/projects/fangjia-script/pom.xml b/projects/fangjia-script/pom.xml index 0d5adfa99..871548bdf 100644 --- a/projects/fangjia-script/pom.xml +++ b/projects/fangjia-script/pom.xml @@ -487,5 +487,10 @@ 1.18.20 provided + + org.msgpack + msgpack + 0.6.12 + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/DistrictDetailCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/DistrictDetailCrawl.groovy new file mode 100644 index 000000000..095456eed --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/DistrictDetailCrawl.groovy @@ -0,0 +1,46 @@ +package com.fangjia.script.temp.shenjian.crawl + +import com.alibaba.fastjson.JSON +import com.alibaba.fastjson.JSONObject +import com.fangjia.script.temp.shenjian.util.MessageUtils +import com.fangjia.script.util.ABuYunUtil +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +/** + * 58小区详情爬取 + * + * @author shenjian + * @since 2021/5/26 + */ +class DistrictDetailCrawl { + + static DBCollection districtSeedConn = MongoUtils.getCollectionByName("seeddb", "district_seeds", "192.168.0.235", 27018) + + static void main(String[] args) { + districtSeedConn.find(new BasicDBObject("root", "58.com").append("status", new BasicDBObject('$eq', null)), new BasicDBObject("district_id", 1).append("city_short", 1).append("_id", 1)) + .batchSize(1000).limit(1000).each { obj -> + final def URL = "https://" + obj.city_short + ".58.com/xiaoqu/" + obj.district_id + "/" + def res = ABuYunUtil.getReponseContent2(URL, "utf-8") + if (res && res.code == 200) { + String content = res.content + + JSON json = new JSONObject() + json.put("project", "district_detail_58") + json.put("url", URL) + + JSON result = new JSONObject() + result.put("content", (String)content) + json.put("result", result) + + byte[] packedMsg = MessageUtils.pack(json) + CommonUtils.sendMessage("result2parse", packedMsg) + districtSeedConn.update(new BasicDBObject("_id", obj._id), + new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已抓取"))) + println("已抓取" + obj.district_id) + } + } + } +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/MessageUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/MessageUtils.groovy new file mode 100644 index 000000000..93174d945 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/MessageUtils.groovy @@ -0,0 +1,45 @@ +package com.fangjia.script.temp.shenjian.util + +import com.alibaba.fastjson.JSON +import org.msgpack.MessagePack +import org.msgpack.type.Value; + +/** + * msg解析工具类 + */ +public class MessageUtils { + + public static String unpack(byte[] content) { + MessagePack messagePack = new MessagePack(); + try { + Value value = messagePack.read(content); + return value.toString(); + } catch (IOException e) { + e.printStackTrace(); + } + return null; + } + + + public static byte[] pack(String value){ + MessagePack messagePack = new MessagePack(); + try { + byte[] data = messagePack.write(value) + return data; + } catch (IOException e) { + e.printStackTrace(); + } + return null; + } + + public static byte[] pack(JSON json){ + MessagePack messagePack = new MessagePack(); + try { + byte[] data = messagePack.write(json) + return data; + } catch (IOException e) { + e.printStackTrace(); + } + return null; + } +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/CommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/CommonUtils.groovy index 3d1d74c0c..d37680fac 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/CommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/CommonUtils.groovy @@ -167,7 +167,16 @@ public class CommonUtils { } - + static void sendMessage(String queueName, byte[] bytes){ + ConnectionFactory factory = new ConnectionFactory(); + factory.setUri("amqp://guest:guest@192.168.0.19:45672/%2f"); + Connection conn = factory.newConnection(); + Channel channel = conn.createChannel(); + channel.queueDeclare(queueName, true, false, false, null); + channel.basicPublish("", queueName, null, bytes); + channel.close() + conn.close() + } /**