diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeHouseCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeHouseCrawl.groovy new file mode 100644 index 000000000..1773bdd8f --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeHouseCrawl.groovy @@ -0,0 +1,189 @@ +package com.fangjia.script.gj.crawl + +import com.alibaba.fastjson.JSON +import com.alibaba.fastjson.JSONObject +import com.fangjia.core.RegionData +import com.fangjia.script.crawl.JQueryUtils +import com.fangjia.script.gj.EsUtils +import com.fangjia.script.temp.shenjian.util.MessageUtils +import com.fangjia.script.util.CommonUtils +import org.jsoup.Jsoup +import org.openqa.selenium.WebDriver +import org.openqa.selenium.chrome.ChromeDriver + +class AnjukeHouseCrawl { + def targetUrl = "http://192.168.0.206:9200/amap_poi/_doc" + static main(args) { + def task = new AnjukeHouseCrawl(); + task.run(); + } + + def run(){ + System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); + System.setProperty("http.nonProxyHosts", "localhost"); + WebDriver driver = new ChromeDriver() + for(def i=24;i<50;i++){ + def url = "https://chengde.anjuke.com/sale/shuangqiao/p${i}/?from=navigation" + def doc = getDocument(url, driver) + def eles = JQueryUtils.select(".list-main .list-left .list .property a", doc) + println "页数:"+i+" 记录数:"+eles.size() + if(eles){ + parser(eles, driver) + } + } + } + + def parser(eles, driver){ + def index = 0 + for(def e : eles){ + index ++ + def href = e.attr("href") + def doc = getDocument(href, driver) +// parserHtml(doc) + def newUrl = href.substring(0, href.indexOf("?")) + sendMq(newUrl, doc) + + println "[ "+index +"] "+ newUrl + } + } + + def sendMq(href, doc){ + JSON json = new JSONObject() + json.put("project", "sell_anjuke") + json.put("url", href) + + JSON result = new JSONObject() + result.put("content", doc.toString()) + json.put("result", result) + + byte[] packedMsg = MessageUtils.pack(json) + CommonUtils.sendMessage("result2parse", packedMsg) + } + + def parserHtml(doc) { + String city = null + def result = [:] + result.siteCode = "安居客" + if (doc) { + result.title = JQueryUtils.selectText([".banner-title h1"], doc) + def heads = JQueryUtils.select(".crumbs a", doc) + + if (heads && heads.size() >= 5) { + result.city = heads[0].text().replace("房产网", "") + def orgRegion = heads[2].text().replace("二手房", "") + result.region = RegionData.formatRegion(result.city, orgRegion.trim()) + result.block = heads[3].text().replace("二手房", "") + result.district_name = heads[4].text() + } else if (heads && heads.size() >= 3) { + result.city = heads[0].text().replace("房产网", "") + result.district_name = heads[2].text() + def eles = JQueryUtils.select("li:contains(所在位置:) a", doc) + if (eles && eles.size() > 0) { + def orgRegion = eles[0].text() + result.region = RegionData.formatRegion(result.city, orgRegion.trim()) + } + } + + if (!result.district_name) { + println " [未解析出(城市-区域-名称)信息]" + } + println result + + + def priceStr = JQueryUtils.selectText(".maininfo-price-num", doc) + if (priceStr) { + if (priceStr.isNumber()) { + result.total_price = priceStr.toDouble() * 10000 + } + } + def roomInfoStr = JQueryUtils.selectText(".maininfo-model-item-1 .maininfo-model-strong", doc) + if (roomInfoStr) { + Integer room = DataCrawlUtils.parseRoom(roomInfoStr) + Integer hall = DataCrawlUtils.parseHall(roomInfoStr) + Integer toilet = DataCrawlUtils.parseToilet(roomInfoStr) + + if (room) { + result.room = room + } + if (hall) { + result.hall = hall + } + if (toilet) { + result.toilet = toilet + } + } + + def areaStr = JQueryUtils.selectText(".maininfo-model-item-2 .maininfo-model-strong", doc).replace("㎡", "") + if (areaStr) { + result.area = areaStr.toDouble() + } + def directionStr = JQueryUtils.selectText(".maininfo-model-item-3 i", doc) + result.direction = DataCrawlUtils.parseDirection(directionStr) + + def buildDateStr = JQueryUtils.selectText(".maininfo-model-item-3 .maininfo-model-weak", doc) + if (buildDateStr && buildDateStr.contains("/")) { + def arrs = buildDateStr.split("\\/") + if (arrs && arrs.size() == 2) { + result.build_date = arrs[0] + result.estate_type = arrs[1] + } + } + + result.floor_info = JQueryUtils.selectText(".maininfo-model-item-1 .maininfo-model-weak", doc) + if (result.floor_info && result.floor_info.contains("共")) { + def height = result.floor_info + if (result.floor_info.contains("(")) { + height = height.substring(result.floor_info.indexOf("(") + 1, result.floor_info.indexOf(")")) + } + height = height.replace("共", "").replace("层", "") + if (height) { + result.height = height.toInteger() + } + } + + + if (result.area && result.total_price) { + result.avg_price = (result.total_price / result.area).toInteger() + } + + def fitmitStr = JQueryUtils.selectText(".maininfo-model-item-2 .maininfo-model-weak", doc) + result.fitment = DataCrawlUtils.parseFitment(fitmitStr) + + + def dateStr = JQueryUtils.selectText(["td:contains(发布时间) .houseInfo-main-item-name"], doc) + + + } + } + + def getDocument(url, driver){ + def stop = false + def error = 0 + def doc = null + while(!stop){ + try{ + driver.get(url) + doc = Jsoup.parse(driver.getPageSource()); + Thread.sleep(5000) + def els = JQueryUtils.select(".list-main .list-left .list", doc) + if(!els){ + els = JQueryUtils.select(".props-main", doc) + } + if(els && els.size()>0){ + stop = true; + }else{ + error++ + driver.manage().deleteAllCookies() + println "[${error}] error: "+url + Thread.sleep(5000) + } + }catch(Exception e){ + error++ + } + if(error>=20){ + stop = true; + } + } + return doc + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/DataCrawlUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/DataCrawlUtils.groovy new file mode 100644 index 000000000..465ae5f04 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/DataCrawlUtils.groovy @@ -0,0 +1,250 @@ +package com.fangjia.script.gj.crawl + +import com.fangjia.script.util.DateUtils +import org.apache.commons.lang.StringUtils + +import java.util.regex.Matcher +import java.util.regex.Pattern + +class DataCrawlUtils { + static final Map DEAL_DATE_RESOLVERS = [:] + static final Map ROOM_RESOLVERS = [:] + static final Map HALL_RESOLVERS = [:] + static final Map TOILET_RESOLVERS = [:] + static final Map TOTAL_PRICE_RESOLVERS = [:] + static final Map UNIT_PRICE_RESOLVERS = [:] + static final Map AREA_RESOLVERS = [:] + static final Map HEIGHT_RESOLVERS = [:] + static final Map FLOOR_RESOLVERS = [:] + static final List DIRECTIONS = ["东北", "西北", "东南", "西南", "南北", "东西", "东", "南", "西", "北"] + private static final HashMap> FITMENTS_TYPES_MAPS = new HashMap>() + + static { + DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{4}-\\d{2}-\\d{2}"), { String str -> + return DateUtils.parseFromTightByPattern(str, "yyyy-MM-dd") + }) + DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{2}-\\d{2}-\\d{2}"), { String str -> + str = "20" + str + return DateUtils.parseFromTightByPattern(str, "yyyy-MM-dd") + }) + DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{2}/\\d{2}/\\d{2}"), { String str -> + str = "20" + str + return DateUtils.parseFromTightByPattern(str, "yyyy/MM/dd") + }) + + DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{4}\\.\\d{2}\\.\\d{2}"), { String str -> + return DateUtils.parseFromTightByPattern(str, "yyyy.MM.dd") + }) + DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{4}-\\d{2}"), { String str -> + return DateUtils.parseFromTightByPattern(str, "yyyy-MM") + }) + + ROOM_RESOLVERS.put(Pattern.compile("[\\d+|一|二|三|四|五|两](室|房)"), { String str -> + str = str.replaceAll("室|房", '').trim() + return str.toInteger() + }) + + HALL_RESOLVERS.put(Pattern.compile("[\\d+|一|二|三|四|五|两]厅"), { String str -> + str = str.replace("厅", '').trim() + return str.toInteger() + }) + TOILET_RESOLVERS.put(Pattern.compile("[\\d+|一|二|三|四|五|两]卫"), { String str -> + str = str.replace("卫", '').trim() + return str.toInteger() + }) + TOTAL_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?万"), { String str -> + str = str.replace("万", '').trim() + return (str.toDouble() * 10000).toInteger() + }) + + UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元/㎡"), { String str -> + str = str.replace("元/㎡", '').trim() + return str.toDouble().toInteger() + }) + UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元/m?"), { String str -> + str = str.replace("元/m?", '').trim() + return str.toDouble().toInteger() + }) + UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元/平"), { String str -> + str = str.replace("元/平", '').trim() + return str.toDouble().toInteger() + }) + UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元"), { String str -> + str = str.replace("元", '').trim() + return str.toDouble().toInteger() + }) + + AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?平米"), { String str -> + str = str.replace("平米", '').trim() + return str.toDouble() + }) + + AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?平方米"), { String str -> + str = str.replace("平方米", '').trim() + return str.toDouble() + }) + + AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?平"), { String str -> + str = str.replace("平", '').trim() + return str.toDouble() + }) + + AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?㎡"), { String str -> + str = str.replace("㎡", '').trim() + return str.toDouble() + }) + AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?m?"), { String str -> + str = str.replace("m?", '').trim() + return str.toDouble() + }) + HEIGHT_RESOLVERS.put(Pattern.compile("共\\d+层"), { String str -> + str = str.replace("共", '').replace('层', '') + return str.toInteger() + }) + + HEIGHT_RESOLVERS.put(Pattern.compile("总\\d+层"), { String str -> + str = str.replace("总", '').replace('层', '') + return str.toInteger() + }) + + HEIGHT_RESOLVERS.put(Pattern.compile("(?<=\\D)(\\d{1,2}/[1-6]?\\d)(?=\\D)"), { String str -> + def split = str.split("/") + def n1 = split[0].toInteger() + def n2 = split[1].toInteger() + if (n1 <= n2) { + return n2 + } + return null + }) + + + FLOOR_RESOLVERS.put(Pattern.compile("(?<=\\D)(\\d{1,2}/[1-6]?\\d)(?=\\D)"), { String str -> + def split = str.split("/") + def n1 = split[0].toInteger() + def n2 = split[1].toInteger() + if (n1 <= n2) { + return n1 + } + return n1 + }) + + FLOOR_RESOLVERS.put(Pattern.compile("第\\d+(层|楼)"), { String str -> + str = str.replace("第", "").replaceAll("层|楼", "") + return str.toInteger() + }) + + FITMENTS_TYPES_MAPS.put("毛坯", ["毛坯", "清水", "无装修", "独栋毛坯", "毛胚", "白坯", "旧装", "坯", "原房", "毛坏"]) + FITMENTS_TYPES_MAPS.put("简装", ["简装", "普通装修", "普通", "简单装修", "简易装修", "普装", "一般装修", "一般", "简", "粗装修"]) + FITMENTS_TYPES_MAPS.put("精装", ["精装", "全装修", "全装", "中等装修", "中装", "中档装修", "精致装修", "中档", "中等", "精", "中修", "优质装修", "时尚装修", "良装", "中高挡"]) + FITMENTS_TYPES_MAPS.put("豪装", ["豪华装修", "高档", "豪华", "豪装", "高级装修", "高装", "奢侈"]) + + } + + /** + * 解析室数 + * @param content + * @return + */ + static Integer parseRoom(String content) { + return parse(content, ROOM_RESOLVERS) + } + /** + * 解析厅数 + * @param content + * @return + */ + static Integer parseHall(String content) { + return parse(content, HALL_RESOLVERS) + } + /** + * 解析厅数 + * @param content + * @return + */ + static Integer parseToilet(String content) { + return parse(content, TOILET_RESOLVERS) + } + + static Object parse(String content, Map resolvers) { + if (content != null) { + content = content.replaceAll("\\s", "") + } + for (def resolver : resolvers.entrySet()) { + String match = match(content, resolver.getKey()) + if (StringUtils.isNotBlank(match)) { + match = match.replace("一","1").replace("二","2").replace("三","3").replace("四","4").replace("五","5").replace("两","2") + return resolver.getValue()(match) + } + } + return null + } + + static String match(String str, Pattern pattern) { + if (str == null) { + return null + } + Matcher matcher = pattern.matcher(str) + if (matcher.find()) { + return matcher.group() + } + return null + } + + static String parseDirection(String directionDesc) { + if (directionDesc == null) { + return null + } + directionDesc = directionDesc.replaceAll("(\\(|(|《|\"|“|【|〔).+?(\\)|》|)|\"|”|】|〕)", "#_#") + StringBuffer sb = new StringBuffer() + for (int i = 0; i < directionDesc.length(); i++) { + String c = directionDesc.substring(i, i + 1) + if (DIRECTIONS.contains(c)) { + if (directionDesc.substring(i).startsWith(c + "#_#")) { + sb.append(c + " ") + } else { + sb.append(c) + } + } + } + for (String d : DIRECTIONS) { + if (sb.toString().startsWith(d)) { + return d + } + } + for (String d : DIRECTIONS) { + if (sb.toString().contains(d)) { + return d + } + } + return null + + } + + static String parseFitment(String fitmentDesc) { + if (fitmentDesc == null) { + return null + } + fitmentDesc = fitmentDesc.replaceAll("[\\s ]+", "") + if (fitmentDesc.equals("非毛坯")) { + return null + } + Iterator>> it = FITMENTS_TYPES_MAPS + .entrySet().iterator() + Map.Entry> entry + while (it.hasNext()) { + entry = it.next() + if (fitmentDesc.contains(entry.getKey())) { + return entry.getKey() + } + for (String fitmentKey : entry.getValue()) { + if (fitmentDesc.contains(fitmentKey)) { + return entry.getKey() + } + } + } + return null + + } + + +}