新增58小区详情脚本抓取方式

This commit is contained in:
沈健
2021-05-26 17:16:14 +08:00
parent 8735c5e6f2
commit 7fe4a709f5
4 changed files with 106 additions and 1 deletions
+5
View File
@@ -487,5 +487,10 @@
<version>1.18.20</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.msgpack</groupId>
<artifactId>msgpack</artifactId>
<version>0.6.12</version>
</dependency>
</dependencies>
</project>
@@ -0,0 +1,46 @@
package com.fangjia.script.temp.shenjian.crawl
import com.alibaba.fastjson.JSON
import com.alibaba.fastjson.JSONObject
import com.fangjia.script.temp.shenjian.util.MessageUtils
import com.fangjia.script.util.ABuYunUtil
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
/**
* 58小区详情爬取
*
* @author shenjian
* @since 2021/5/26
*/
class DistrictDetailCrawl {
static DBCollection districtSeedConn = MongoUtils.getCollectionByName("seeddb", "district_seeds", "192.168.0.235", 27018)
static void main(String[] args) {
districtSeedConn.find(new BasicDBObject("root", "58.com").append("status", new BasicDBObject('$eq', null)), new BasicDBObject("district_id", 1).append("city_short", 1).append("_id", 1))
.batchSize(1000).limit(1000).each { obj ->
final def URL = "https://" + obj.city_short + ".58.com/xiaoqu/" + obj.district_id + "/"
def res = ABuYunUtil.getReponseContent2(URL, "utf-8")
if (res && res.code == 200) {
String content = res.content
JSON json = new JSONObject()
json.put("project", "district_detail_58")
json.put("url", URL)
JSON result = new JSONObject()
result.put("content", (String)content)
json.put("result", result)
byte[] packedMsg = MessageUtils.pack(json)
CommonUtils.sendMessage("result2parse", packedMsg)
districtSeedConn.update(new BasicDBObject("_id", obj._id),
new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已抓取")))
println("已抓取" + obj.district_id)
}
}
}
}
@@ -0,0 +1,45 @@
package com.fangjia.script.temp.shenjian.util
import com.alibaba.fastjson.JSON
import org.msgpack.MessagePack
import org.msgpack.type.Value;
/**
* msg½âÎö¹¤¾ßÀà
*/
public class MessageUtils {
public static String unpack(byte[] content) {
MessagePack messagePack = new MessagePack();
try {
Value value = messagePack.read(content);
return value.toString();
} catch (IOException e) {
e.printStackTrace();
}
return null;
}
public static byte[] pack(String value){
MessagePack messagePack = new MessagePack();
try {
byte[] data = messagePack.write(value)
return data;
} catch (IOException e) {
e.printStackTrace();
}
return null;
}
public static byte[] pack(JSON json){
MessagePack messagePack = new MessagePack();
try {
byte[] data = messagePack.write(json)
return data;
} catch (IOException e) {
e.printStackTrace();
}
return null;
}
}
@@ -167,7 +167,16 @@ public class CommonUtils {
}
static void sendMessage(String queueName, byte[] bytes){
ConnectionFactory factory = new ConnectionFactory();
factory.setUri("amqp://guest:guest@192.168.0.19:45672/%2f");
Connection conn = factory.newConnection();
Channel channel = conn.createChannel();
channel.queueDeclare(queueName, true, false, false, null);
channel.basicPublish("", queueName, null, bytes);
channel.close()
conn.close()
}
/**