This commit is contained in:
2022-06-01 13:48:03 +08:00
parent 047467f6c2
commit efad3cb06d
2 changed files with 439 additions and 0 deletions
@@ -0,0 +1,189 @@
package com.fangjia.script.gj.crawl
import com.alibaba.fastjson.JSON
import com.alibaba.fastjson.JSONObject
import com.fangjia.core.RegionData
import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.gj.EsUtils
import com.fangjia.script.temp.shenjian.util.MessageUtils
import com.fangjia.script.util.CommonUtils
import org.jsoup.Jsoup
import org.openqa.selenium.WebDriver
import org.openqa.selenium.chrome.ChromeDriver
class AnjukeHouseCrawl {
def targetUrl = "http://192.168.0.206:9200/amap_poi/_doc"
static main(args) {
def task = new AnjukeHouseCrawl();
task.run();
}
def run(){
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
System.setProperty("http.nonProxyHosts", "localhost");
WebDriver driver = new ChromeDriver()
for(def i=24;i<50;i++){
def url = "https://chengde.anjuke.com/sale/shuangqiao/p${i}/?from=navigation"
def doc = getDocument(url, driver)
def eles = JQueryUtils.select(".list-main .list-left .list .property a", doc)
println "页数:"+i+" 记录数:"+eles.size()
if(eles){
parser(eles, driver)
}
}
}
def parser(eles, driver){
def index = 0
for(def e : eles){
index ++
def href = e.attr("href")
def doc = getDocument(href, driver)
// parserHtml(doc)
def newUrl = href.substring(0, href.indexOf("?"))
sendMq(newUrl, doc)
println "[ "+index +"] "+ newUrl
}
}
def sendMq(href, doc){
JSON json = new JSONObject()
json.put("project", "sell_anjuke")
json.put("url", href)
JSON result = new JSONObject()
result.put("content", doc.toString())
json.put("result", result)
byte[] packedMsg = MessageUtils.pack(json)
CommonUtils.sendMessage("result2parse", packedMsg)
}
def parserHtml(doc) {
String city = null
def result = [:]
result.siteCode = "安居客"
if (doc) {
result.title = JQueryUtils.selectText([".banner-title h1"], doc)
def heads = JQueryUtils.select(".crumbs a", doc)
if (heads && heads.size() >= 5) {
result.city = heads[0].text().replace("房产网", "")
def orgRegion = heads[2].text().replace("二手房", "")
result.region = RegionData.formatRegion(result.city, orgRegion.trim())
result.block = heads[3].text().replace("二手房", "")
result.district_name = heads[4].text()
} else if (heads && heads.size() >= 3) {
result.city = heads[0].text().replace("房产网", "")
result.district_name = heads[2].text()
def eles = JQueryUtils.select("li:contains(所在位置:) a", doc)
if (eles && eles.size() > 0) {
def orgRegion = eles[0].text()
result.region = RegionData.formatRegion(result.city, orgRegion.trim())
}
}
if (!result.district_name) {
println " [未解析出(城市-区域-名称)信息]"
}
println result
def priceStr = JQueryUtils.selectText(".maininfo-price-num", doc)
if (priceStr) {
if (priceStr.isNumber()) {
result.total_price = priceStr.toDouble() * 10000
}
}
def roomInfoStr = JQueryUtils.selectText(".maininfo-model-item-1 .maininfo-model-strong", doc)
if (roomInfoStr) {
Integer room = DataCrawlUtils.parseRoom(roomInfoStr)
Integer hall = DataCrawlUtils.parseHall(roomInfoStr)
Integer toilet = DataCrawlUtils.parseToilet(roomInfoStr)
if (room) {
result.room = room
}
if (hall) {
result.hall = hall
}
if (toilet) {
result.toilet = toilet
}
}
def areaStr = JQueryUtils.selectText(".maininfo-model-item-2 .maininfo-model-strong", doc).replace("㎡", "")
if (areaStr) {
result.area = areaStr.toDouble()
}
def directionStr = JQueryUtils.selectText(".maininfo-model-item-3 i", doc)
result.direction = DataCrawlUtils.parseDirection(directionStr)
def buildDateStr = JQueryUtils.selectText(".maininfo-model-item-3 .maininfo-model-weak", doc)
if (buildDateStr && buildDateStr.contains("/")) {
def arrs = buildDateStr.split("\\/")
if (arrs && arrs.size() == 2) {
result.build_date = arrs[0]
result.estate_type = arrs[1]
}
}
result.floor_info = JQueryUtils.selectText(".maininfo-model-item-1 .maininfo-model-weak", doc)
if (result.floor_info && result.floor_info.contains("共")) {
def height = result.floor_info
if (result.floor_info.contains("(")) {
height = height.substring(result.floor_info.indexOf("(") + 1, result.floor_info.indexOf(")"))
}
height = height.replace("共", "").replace("层", "")
if (height) {
result.height = height.toInteger()
}
}
if (result.area && result.total_price) {
result.avg_price = (result.total_price / result.area).toInteger()
}
def fitmitStr = JQueryUtils.selectText(".maininfo-model-item-2 .maininfo-model-weak", doc)
result.fitment = DataCrawlUtils.parseFitment(fitmitStr)
def dateStr = JQueryUtils.selectText(["td:contains(发布时间) .houseInfo-main-item-name"], doc)
}
}
def getDocument(url, driver){
def stop = false
def error = 0
def doc = null
while(!stop){
try{
driver.get(url)
doc = Jsoup.parse(driver.getPageSource());
Thread.sleep(5000)
def els = JQueryUtils.select(".list-main .list-left .list", doc)
if(!els){
els = JQueryUtils.select(".props-main", doc)
}
if(els && els.size()>0){
stop = true;
}else{
error++
driver.manage().deleteAllCookies()
println "[${error}] error: "+url
Thread.sleep(5000)
}
}catch(Exception e){
error++
}
if(error>=20){
stop = true;
}
}
return doc
}
}
@@ -0,0 +1,250 @@
package com.fangjia.script.gj.crawl
import com.fangjia.script.util.DateUtils
import org.apache.commons.lang.StringUtils
import java.util.regex.Matcher
import java.util.regex.Pattern
class DataCrawlUtils {
static final Map<Pattern, Closure> DEAL_DATE_RESOLVERS = [:]
static final Map<Pattern, Closure> ROOM_RESOLVERS = [:]
static final Map<Pattern, Closure> HALL_RESOLVERS = [:]
static final Map<Pattern, Closure> TOILET_RESOLVERS = [:]
static final Map<Pattern, Closure> TOTAL_PRICE_RESOLVERS = [:]
static final Map<Pattern, Closure> UNIT_PRICE_RESOLVERS = [:]
static final Map<Pattern, Closure> AREA_RESOLVERS = [:]
static final Map<Pattern, Closure> HEIGHT_RESOLVERS = [:]
static final Map<Pattern, Closure> FLOOR_RESOLVERS = [:]
static final List<String> DIRECTIONS = ["东北", "西北", "东南", "西南", "南北", "东西", "东", "南", "西", "北"]
private static final HashMap<String, List<String>> FITMENTS_TYPES_MAPS = new HashMap<String, List<String>>()
static {
DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{4}-\\d{2}-\\d{2}"), { String str ->
return DateUtils.parseFromTightByPattern(str, "yyyy-MM-dd")
})
DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{2}-\\d{2}-\\d{2}"), { String str ->
str = "20" + str
return DateUtils.parseFromTightByPattern(str, "yyyy-MM-dd")
})
DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{2}/\\d{2}/\\d{2}"), { String str ->
str = "20" + str
return DateUtils.parseFromTightByPattern(str, "yyyy/MM/dd")
})
DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{4}\\.\\d{2}\\.\\d{2}"), { String str ->
return DateUtils.parseFromTightByPattern(str, "yyyy.MM.dd")
})
DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{4}-\\d{2}"), { String str ->
return DateUtils.parseFromTightByPattern(str, "yyyy-MM")
})
ROOM_RESOLVERS.put(Pattern.compile("[\\d+|一|二|三|四|五|两](室|房)"), { String str ->
str = str.replaceAll("室|房", '').trim()
return str.toInteger()
})
HALL_RESOLVERS.put(Pattern.compile("[\\d+|一|二|三|四|五|两]厅"), { String str ->
str = str.replace("厅", '').trim()
return str.toInteger()
})
TOILET_RESOLVERS.put(Pattern.compile("[\\d+|一|二|三|四|五|两]卫"), { String str ->
str = str.replace("卫", '').trim()
return str.toInteger()
})
TOTAL_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?万"), { String str ->
str = str.replace("万", '').trim()
return (str.toDouble() * 10000).toInteger()
})
UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元/㎡"), { String str ->
str = str.replace("元/㎡", '').trim()
return str.toDouble().toInteger()
})
UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元/m?"), { String str ->
str = str.replace("元/m?", '').trim()
return str.toDouble().toInteger()
})
UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元/平"), { String str ->
str = str.replace("元/平", '').trim()
return str.toDouble().toInteger()
})
UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元"), { String str ->
str = str.replace("元", '').trim()
return str.toDouble().toInteger()
})
AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?平米"), { String str ->
str = str.replace("平米", '').trim()
return str.toDouble()
})
AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?平方米"), { String str ->
str = str.replace("平方米", '').trim()
return str.toDouble()
})
AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?平"), { String str ->
str = str.replace("平", '').trim()
return str.toDouble()
})
AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?㎡"), { String str ->
str = str.replace("㎡", '').trim()
return str.toDouble()
})
AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?m?"), { String str ->
str = str.replace("m?", '').trim()
return str.toDouble()
})
HEIGHT_RESOLVERS.put(Pattern.compile("共\\d+层"), { String str ->
str = str.replace("共", '').replace('层', '')
return str.toInteger()
})
HEIGHT_RESOLVERS.put(Pattern.compile("总\\d+层"), { String str ->
str = str.replace("总", '').replace('层', '')
return str.toInteger()
})
HEIGHT_RESOLVERS.put(Pattern.compile("(?<=\\D)(\\d{1,2}/[1-6]?\\d)(?=\\D)"), { String str ->
def split = str.split("/")
def n1 = split[0].toInteger()
def n2 = split[1].toInteger()
if (n1 <= n2) {
return n2
}
return null
})
FLOOR_RESOLVERS.put(Pattern.compile("(?<=\\D)(\\d{1,2}/[1-6]?\\d)(?=\\D)"), { String str ->
def split = str.split("/")
def n1 = split[0].toInteger()
def n2 = split[1].toInteger()
if (n1 <= n2) {
return n1
}
return n1
})
FLOOR_RESOLVERS.put(Pattern.compile("第\\d+(层|楼)"), { String str ->
str = str.replace("第", "").replaceAll("层|楼", "")
return str.toInteger()
})
FITMENTS_TYPES_MAPS.put("毛坯", ["毛坯", "清水", "无装修", "独栋毛坯", "毛胚", "白坯", "旧装", "坯", "原房", "毛坏"])
FITMENTS_TYPES_MAPS.put("简装", ["简装", "普通装修", "普通", "简单装修", "简易装修", "普装", "一般装修", "一般", "简", "粗装修"])
FITMENTS_TYPES_MAPS.put("精装", ["精装", "全装修", "全装", "中等装修", "中装", "中档装修", "精致装修", "中档", "中等", "精", "中修", "优质装修", "时尚装修", "良装", "中高挡"])
FITMENTS_TYPES_MAPS.put("豪装", ["豪华装修", "高档", "豪华", "豪装", "高级装修", "高装", "奢侈"])
}
/**
* 解析室数
* @param content
* @return
*/
static Integer parseRoom(String content) {
return parse(content, ROOM_RESOLVERS)
}
/**
* 解析厅数
* @param content
* @return
*/
static Integer parseHall(String content) {
return parse(content, HALL_RESOLVERS)
}
/**
* 解析厅数
* @param content
* @return
*/
static Integer parseToilet(String content) {
return parse(content, TOILET_RESOLVERS)
}
static Object parse(String content, Map<Pattern, Closure> resolvers) {
if (content != null) {
content = content.replaceAll("\\s", "")
}
for (def resolver : resolvers.entrySet()) {
String match = match(content, resolver.getKey())
if (StringUtils.isNotBlank(match)) {
match = match.replace("一","1").replace("二","2").replace("三","3").replace("四","4").replace("五","5").replace("两","2")
return resolver.getValue()(match)
}
}
return null
}
static String match(String str, Pattern pattern) {
if (str == null) {
return null
}
Matcher matcher = pattern.matcher(str)
if (matcher.find()) {
return matcher.group()
}
return null
}
static String parseDirection(String directionDesc) {
if (directionDesc == null) {
return null
}
directionDesc = directionDesc.replaceAll("(\\(|(|《|\"|“|【|〔).+?(\\)|》|)|\"|”|】|〕)", "#_#")
StringBuffer sb = new StringBuffer()
for (int i = 0; i < directionDesc.length(); i++) {
String c = directionDesc.substring(i, i + 1)
if (DIRECTIONS.contains(c)) {
if (directionDesc.substring(i).startsWith(c + "#_#")) {
sb.append(c + " ")
} else {
sb.append(c)
}
}
}
for (String d : DIRECTIONS) {
if (sb.toString().startsWith(d)) {
return d
}
}
for (String d : DIRECTIONS) {
if (sb.toString().contains(d)) {
return d
}
}
return null
}
static String parseFitment(String fitmentDesc) {
if (fitmentDesc == null) {
return null
}
fitmentDesc = fitmentDesc.replaceAll("[\\s ]+", "")
if (fitmentDesc.equals("非毛坯")) {
return null
}
Iterator<Map.Entry<String, List<String>>> it = FITMENTS_TYPES_MAPS
.entrySet().iterator()
Map.Entry<String, List<String>> entry
while (it.hasNext()) {
entry = it.next()
if (fitmentDesc.contains(entry.getKey())) {
return entry.getKey()
}
for (String fitmentKey : entry.getValue()) {
if (fitmentDesc.contains(fitmentKey)) {
return entry.getKey()
}
}
}
return null
}
}