add
This commit is contained in:
+189
@@ -0,0 +1,189 @@
|
||||
package com.fangjia.script.gj.crawl
|
||||
|
||||
import com.alibaba.fastjson.JSON
|
||||
import com.alibaba.fastjson.JSONObject
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.crawl.JQueryUtils
|
||||
import com.fangjia.script.gj.EsUtils
|
||||
import com.fangjia.script.temp.shenjian.util.MessageUtils
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import org.jsoup.Jsoup
|
||||
import org.openqa.selenium.WebDriver
|
||||
import org.openqa.selenium.chrome.ChromeDriver
|
||||
|
||||
class AnjukeHouseCrawl {
|
||||
def targetUrl = "http://192.168.0.206:9200/amap_poi/_doc"
|
||||
static main(args) {
|
||||
def task = new AnjukeHouseCrawl();
|
||||
task.run();
|
||||
}
|
||||
|
||||
def run(){
|
||||
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
|
||||
System.setProperty("http.nonProxyHosts", "localhost");
|
||||
WebDriver driver = new ChromeDriver()
|
||||
for(def i=24;i<50;i++){
|
||||
def url = "https://chengde.anjuke.com/sale/shuangqiao/p${i}/?from=navigation"
|
||||
def doc = getDocument(url, driver)
|
||||
def eles = JQueryUtils.select(".list-main .list-left .list .property a", doc)
|
||||
println "页数:"+i+" 记录数:"+eles.size()
|
||||
if(eles){
|
||||
parser(eles, driver)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def parser(eles, driver){
|
||||
def index = 0
|
||||
for(def e : eles){
|
||||
index ++
|
||||
def href = e.attr("href")
|
||||
def doc = getDocument(href, driver)
|
||||
// parserHtml(doc)
|
||||
def newUrl = href.substring(0, href.indexOf("?"))
|
||||
sendMq(newUrl, doc)
|
||||
|
||||
println "[ "+index +"] "+ newUrl
|
||||
}
|
||||
}
|
||||
|
||||
def sendMq(href, doc){
|
||||
JSON json = new JSONObject()
|
||||
json.put("project", "sell_anjuke")
|
||||
json.put("url", href)
|
||||
|
||||
JSON result = new JSONObject()
|
||||
result.put("content", doc.toString())
|
||||
json.put("result", result)
|
||||
|
||||
byte[] packedMsg = MessageUtils.pack(json)
|
||||
CommonUtils.sendMessage("result2parse", packedMsg)
|
||||
}
|
||||
|
||||
def parserHtml(doc) {
|
||||
String city = null
|
||||
def result = [:]
|
||||
result.siteCode = "安居客"
|
||||
if (doc) {
|
||||
result.title = JQueryUtils.selectText([".banner-title h1"], doc)
|
||||
def heads = JQueryUtils.select(".crumbs a", doc)
|
||||
|
||||
if (heads && heads.size() >= 5) {
|
||||
result.city = heads[0].text().replace("房产网", "")
|
||||
def orgRegion = heads[2].text().replace("二手房", "")
|
||||
result.region = RegionData.formatRegion(result.city, orgRegion.trim())
|
||||
result.block = heads[3].text().replace("二手房", "")
|
||||
result.district_name = heads[4].text()
|
||||
} else if (heads && heads.size() >= 3) {
|
||||
result.city = heads[0].text().replace("房产网", "")
|
||||
result.district_name = heads[2].text()
|
||||
def eles = JQueryUtils.select("li:contains(所在位置:) a", doc)
|
||||
if (eles && eles.size() > 0) {
|
||||
def orgRegion = eles[0].text()
|
||||
result.region = RegionData.formatRegion(result.city, orgRegion.trim())
|
||||
}
|
||||
}
|
||||
|
||||
if (!result.district_name) {
|
||||
println " [未解析出(城市-区域-名称)信息]"
|
||||
}
|
||||
println result
|
||||
|
||||
|
||||
def priceStr = JQueryUtils.selectText(".maininfo-price-num", doc)
|
||||
if (priceStr) {
|
||||
if (priceStr.isNumber()) {
|
||||
result.total_price = priceStr.toDouble() * 10000
|
||||
}
|
||||
}
|
||||
def roomInfoStr = JQueryUtils.selectText(".maininfo-model-item-1 .maininfo-model-strong", doc)
|
||||
if (roomInfoStr) {
|
||||
Integer room = DataCrawlUtils.parseRoom(roomInfoStr)
|
||||
Integer hall = DataCrawlUtils.parseHall(roomInfoStr)
|
||||
Integer toilet = DataCrawlUtils.parseToilet(roomInfoStr)
|
||||
|
||||
if (room) {
|
||||
result.room = room
|
||||
}
|
||||
if (hall) {
|
||||
result.hall = hall
|
||||
}
|
||||
if (toilet) {
|
||||
result.toilet = toilet
|
||||
}
|
||||
}
|
||||
|
||||
def areaStr = JQueryUtils.selectText(".maininfo-model-item-2 .maininfo-model-strong", doc).replace("㎡", "")
|
||||
if (areaStr) {
|
||||
result.area = areaStr.toDouble()
|
||||
}
|
||||
def directionStr = JQueryUtils.selectText(".maininfo-model-item-3 i", doc)
|
||||
result.direction = DataCrawlUtils.parseDirection(directionStr)
|
||||
|
||||
def buildDateStr = JQueryUtils.selectText(".maininfo-model-item-3 .maininfo-model-weak", doc)
|
||||
if (buildDateStr && buildDateStr.contains("/")) {
|
||||
def arrs = buildDateStr.split("\\/")
|
||||
if (arrs && arrs.size() == 2) {
|
||||
result.build_date = arrs[0]
|
||||
result.estate_type = arrs[1]
|
||||
}
|
||||
}
|
||||
|
||||
result.floor_info = JQueryUtils.selectText(".maininfo-model-item-1 .maininfo-model-weak", doc)
|
||||
if (result.floor_info && result.floor_info.contains("共")) {
|
||||
def height = result.floor_info
|
||||
if (result.floor_info.contains("(")) {
|
||||
height = height.substring(result.floor_info.indexOf("(") + 1, result.floor_info.indexOf(")"))
|
||||
}
|
||||
height = height.replace("共", "").replace("层", "")
|
||||
if (height) {
|
||||
result.height = height.toInteger()
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if (result.area && result.total_price) {
|
||||
result.avg_price = (result.total_price / result.area).toInteger()
|
||||
}
|
||||
|
||||
def fitmitStr = JQueryUtils.selectText(".maininfo-model-item-2 .maininfo-model-weak", doc)
|
||||
result.fitment = DataCrawlUtils.parseFitment(fitmitStr)
|
||||
|
||||
|
||||
def dateStr = JQueryUtils.selectText(["td:contains(发布时间) .houseInfo-main-item-name"], doc)
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
def getDocument(url, driver){
|
||||
def stop = false
|
||||
def error = 0
|
||||
def doc = null
|
||||
while(!stop){
|
||||
try{
|
||||
driver.get(url)
|
||||
doc = Jsoup.parse(driver.getPageSource());
|
||||
Thread.sleep(5000)
|
||||
def els = JQueryUtils.select(".list-main .list-left .list", doc)
|
||||
if(!els){
|
||||
els = JQueryUtils.select(".props-main", doc)
|
||||
}
|
||||
if(els && els.size()>0){
|
||||
stop = true;
|
||||
}else{
|
||||
error++
|
||||
driver.manage().deleteAllCookies()
|
||||
println "[${error}] error: "+url
|
||||
Thread.sleep(5000)
|
||||
}
|
||||
}catch(Exception e){
|
||||
error++
|
||||
}
|
||||
if(error>=20){
|
||||
stop = true;
|
||||
}
|
||||
}
|
||||
return doc
|
||||
}
|
||||
}
|
||||
+250
@@ -0,0 +1,250 @@
|
||||
package com.fangjia.script.gj.crawl
|
||||
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import org.apache.commons.lang.StringUtils
|
||||
|
||||
import java.util.regex.Matcher
|
||||
import java.util.regex.Pattern
|
||||
|
||||
class DataCrawlUtils {
|
||||
static final Map<Pattern, Closure> DEAL_DATE_RESOLVERS = [:]
|
||||
static final Map<Pattern, Closure> ROOM_RESOLVERS = [:]
|
||||
static final Map<Pattern, Closure> HALL_RESOLVERS = [:]
|
||||
static final Map<Pattern, Closure> TOILET_RESOLVERS = [:]
|
||||
static final Map<Pattern, Closure> TOTAL_PRICE_RESOLVERS = [:]
|
||||
static final Map<Pattern, Closure> UNIT_PRICE_RESOLVERS = [:]
|
||||
static final Map<Pattern, Closure> AREA_RESOLVERS = [:]
|
||||
static final Map<Pattern, Closure> HEIGHT_RESOLVERS = [:]
|
||||
static final Map<Pattern, Closure> FLOOR_RESOLVERS = [:]
|
||||
static final List<String> DIRECTIONS = ["东北", "西北", "东南", "西南", "南北", "东西", "东", "南", "西", "北"]
|
||||
private static final HashMap<String, List<String>> FITMENTS_TYPES_MAPS = new HashMap<String, List<String>>()
|
||||
|
||||
static {
|
||||
DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{4}-\\d{2}-\\d{2}"), { String str ->
|
||||
return DateUtils.parseFromTightByPattern(str, "yyyy-MM-dd")
|
||||
})
|
||||
DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{2}-\\d{2}-\\d{2}"), { String str ->
|
||||
str = "20" + str
|
||||
return DateUtils.parseFromTightByPattern(str, "yyyy-MM-dd")
|
||||
})
|
||||
DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{2}/\\d{2}/\\d{2}"), { String str ->
|
||||
str = "20" + str
|
||||
return DateUtils.parseFromTightByPattern(str, "yyyy/MM/dd")
|
||||
})
|
||||
|
||||
DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{4}\\.\\d{2}\\.\\d{2}"), { String str ->
|
||||
return DateUtils.parseFromTightByPattern(str, "yyyy.MM.dd")
|
||||
})
|
||||
DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{4}-\\d{2}"), { String str ->
|
||||
return DateUtils.parseFromTightByPattern(str, "yyyy-MM")
|
||||
})
|
||||
|
||||
ROOM_RESOLVERS.put(Pattern.compile("[\\d+|一|二|三|四|五|两](室|房)"), { String str ->
|
||||
str = str.replaceAll("室|房", '').trim()
|
||||
return str.toInteger()
|
||||
})
|
||||
|
||||
HALL_RESOLVERS.put(Pattern.compile("[\\d+|一|二|三|四|五|两]厅"), { String str ->
|
||||
str = str.replace("厅", '').trim()
|
||||
return str.toInteger()
|
||||
})
|
||||
TOILET_RESOLVERS.put(Pattern.compile("[\\d+|一|二|三|四|五|两]卫"), { String str ->
|
||||
str = str.replace("卫", '').trim()
|
||||
return str.toInteger()
|
||||
})
|
||||
TOTAL_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?万"), { String str ->
|
||||
str = str.replace("万", '').trim()
|
||||
return (str.toDouble() * 10000).toInteger()
|
||||
})
|
||||
|
||||
UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元/㎡"), { String str ->
|
||||
str = str.replace("元/㎡", '').trim()
|
||||
return str.toDouble().toInteger()
|
||||
})
|
||||
UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元/m?"), { String str ->
|
||||
str = str.replace("元/m?", '').trim()
|
||||
return str.toDouble().toInteger()
|
||||
})
|
||||
UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元/平"), { String str ->
|
||||
str = str.replace("元/平", '').trim()
|
||||
return str.toDouble().toInteger()
|
||||
})
|
||||
UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元"), { String str ->
|
||||
str = str.replace("元", '').trim()
|
||||
return str.toDouble().toInteger()
|
||||
})
|
||||
|
||||
AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?平米"), { String str ->
|
||||
str = str.replace("平米", '').trim()
|
||||
return str.toDouble()
|
||||
})
|
||||
|
||||
AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?平方米"), { String str ->
|
||||
str = str.replace("平方米", '').trim()
|
||||
return str.toDouble()
|
||||
})
|
||||
|
||||
AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?平"), { String str ->
|
||||
str = str.replace("平", '').trim()
|
||||
return str.toDouble()
|
||||
})
|
||||
|
||||
AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?㎡"), { String str ->
|
||||
str = str.replace("㎡", '').trim()
|
||||
return str.toDouble()
|
||||
})
|
||||
AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?m?"), { String str ->
|
||||
str = str.replace("m?", '').trim()
|
||||
return str.toDouble()
|
||||
})
|
||||
HEIGHT_RESOLVERS.put(Pattern.compile("共\\d+层"), { String str ->
|
||||
str = str.replace("共", '').replace('层', '')
|
||||
return str.toInteger()
|
||||
})
|
||||
|
||||
HEIGHT_RESOLVERS.put(Pattern.compile("总\\d+层"), { String str ->
|
||||
str = str.replace("总", '').replace('层', '')
|
||||
return str.toInteger()
|
||||
})
|
||||
|
||||
HEIGHT_RESOLVERS.put(Pattern.compile("(?<=\\D)(\\d{1,2}/[1-6]?\\d)(?=\\D)"), { String str ->
|
||||
def split = str.split("/")
|
||||
def n1 = split[0].toInteger()
|
||||
def n2 = split[1].toInteger()
|
||||
if (n1 <= n2) {
|
||||
return n2
|
||||
}
|
||||
return null
|
||||
})
|
||||
|
||||
|
||||
FLOOR_RESOLVERS.put(Pattern.compile("(?<=\\D)(\\d{1,2}/[1-6]?\\d)(?=\\D)"), { String str ->
|
||||
def split = str.split("/")
|
||||
def n1 = split[0].toInteger()
|
||||
def n2 = split[1].toInteger()
|
||||
if (n1 <= n2) {
|
||||
return n1
|
||||
}
|
||||
return n1
|
||||
})
|
||||
|
||||
FLOOR_RESOLVERS.put(Pattern.compile("第\\d+(层|楼)"), { String str ->
|
||||
str = str.replace("第", "").replaceAll("层|楼", "")
|
||||
return str.toInteger()
|
||||
})
|
||||
|
||||
FITMENTS_TYPES_MAPS.put("毛坯", ["毛坯", "清水", "无装修", "独栋毛坯", "毛胚", "白坯", "旧装", "坯", "原房", "毛坏"])
|
||||
FITMENTS_TYPES_MAPS.put("简装", ["简装", "普通装修", "普通", "简单装修", "简易装修", "普装", "一般装修", "一般", "简", "粗装修"])
|
||||
FITMENTS_TYPES_MAPS.put("精装", ["精装", "全装修", "全装", "中等装修", "中装", "中档装修", "精致装修", "中档", "中等", "精", "中修", "优质装修", "时尚装修", "良装", "中高挡"])
|
||||
FITMENTS_TYPES_MAPS.put("豪装", ["豪华装修", "高档", "豪华", "豪装", "高级装修", "高装", "奢侈"])
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* 解析室数
|
||||
* @param content
|
||||
* @return
|
||||
*/
|
||||
static Integer parseRoom(String content) {
|
||||
return parse(content, ROOM_RESOLVERS)
|
||||
}
|
||||
/**
|
||||
* 解析厅数
|
||||
* @param content
|
||||
* @return
|
||||
*/
|
||||
static Integer parseHall(String content) {
|
||||
return parse(content, HALL_RESOLVERS)
|
||||
}
|
||||
/**
|
||||
* 解析厅数
|
||||
* @param content
|
||||
* @return
|
||||
*/
|
||||
static Integer parseToilet(String content) {
|
||||
return parse(content, TOILET_RESOLVERS)
|
||||
}
|
||||
|
||||
static Object parse(String content, Map<Pattern, Closure> resolvers) {
|
||||
if (content != null) {
|
||||
content = content.replaceAll("\\s", "")
|
||||
}
|
||||
for (def resolver : resolvers.entrySet()) {
|
||||
String match = match(content, resolver.getKey())
|
||||
if (StringUtils.isNotBlank(match)) {
|
||||
match = match.replace("一","1").replace("二","2").replace("三","3").replace("四","4").replace("五","5").replace("两","2")
|
||||
return resolver.getValue()(match)
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
static String match(String str, Pattern pattern) {
|
||||
if (str == null) {
|
||||
return null
|
||||
}
|
||||
Matcher matcher = pattern.matcher(str)
|
||||
if (matcher.find()) {
|
||||
return matcher.group()
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
static String parseDirection(String directionDesc) {
|
||||
if (directionDesc == null) {
|
||||
return null
|
||||
}
|
||||
directionDesc = directionDesc.replaceAll("(\\(|(|《|\"|“|【|〔).+?(\\)|》|)|\"|”|】|〕)", "#_#")
|
||||
StringBuffer sb = new StringBuffer()
|
||||
for (int i = 0; i < directionDesc.length(); i++) {
|
||||
String c = directionDesc.substring(i, i + 1)
|
||||
if (DIRECTIONS.contains(c)) {
|
||||
if (directionDesc.substring(i).startsWith(c + "#_#")) {
|
||||
sb.append(c + " ")
|
||||
} else {
|
||||
sb.append(c)
|
||||
}
|
||||
}
|
||||
}
|
||||
for (String d : DIRECTIONS) {
|
||||
if (sb.toString().startsWith(d)) {
|
||||
return d
|
||||
}
|
||||
}
|
||||
for (String d : DIRECTIONS) {
|
||||
if (sb.toString().contains(d)) {
|
||||
return d
|
||||
}
|
||||
}
|
||||
return null
|
||||
|
||||
}
|
||||
|
||||
static String parseFitment(String fitmentDesc) {
|
||||
if (fitmentDesc == null) {
|
||||
return null
|
||||
}
|
||||
fitmentDesc = fitmentDesc.replaceAll("[\\s ]+", "")
|
||||
if (fitmentDesc.equals("非毛坯")) {
|
||||
return null
|
||||
}
|
||||
Iterator<Map.Entry<String, List<String>>> it = FITMENTS_TYPES_MAPS
|
||||
.entrySet().iterator()
|
||||
Map.Entry<String, List<String>> entry
|
||||
while (it.hasNext()) {
|
||||
entry = it.next()
|
||||
if (fitmentDesc.contains(entry.getKey())) {
|
||||
return entry.getKey()
|
||||
}
|
||||
for (String fitmentKey : entry.getValue()) {
|
||||
if (fitmentDesc.contains(fitmentKey)) {
|
||||
return entry.getKey()
|
||||
}
|
||||
}
|
||||
}
|
||||
return null
|
||||
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
Reference in New Issue
Block a user