diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy index c2f8fe431..6cd17549e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -27,7 +27,7 @@ import java.util.regex.Pattern @BatchOwner(Watcher.GaoJuan) class BigdataCommunityBatch extends BaseBatch{ - public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80) + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) static main(args) { runBatch(new BigdataCommunityBatch(), args); @@ -187,11 +187,22 @@ class BigdataCommunityBatch extends BaseBatch{ }else{ def jsonObj = null if(dist.address){ - jsonObj = OpenApiClient.addressMatch(dist.city, dist.address, "property", false, dist.region, dist.name) +// jsonObj = OpenApiClient.addressMatch(dist.city, dist.address, "property", false, dist.region, dist.name) + jsonObj = OpenApiClient.standard1Action(dist.city, dist.address) } - if(jsonObj && jsonObj.searchAddress){ - ret << jsonObj.searchAddress.street - ret << jsonObj.searchAddress.streetNumber + if(jsonObj && jsonObj.street){ + if(jsonObj.street){ + ret << jsonObj.street + }else{ + ret << "" + } + + if(jsonObj.streetNumber){ + ret << jsonObj.streetNumber + }else{ + ret << "" + } + }else{ ret << "" ret << "" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbDataStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbDataStat.groovy new file mode 100644 index 000000000..1aa5a391f --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbDataStat.groovy @@ -0,0 +1,155 @@ +package com.fangjia.script.ccbevaluation.data + +import com.fangjia.script.ccbevaluation.CcbCommonUtils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import groovy.sql.Sql + +class CcbDataStat { + + public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com") + + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + + static main(args) { + def task = new CcbDataStat() +// task.run() + task.tradeRun() + } + + def tradeRun(){ + def ccbCitys = CcbCommonUtils.citys; + def startDate = DateUtils.parseFromTightByPattern("2021-01-01","yyyy-MM-dd") + def endDate = DateUtils.parseFromTightByPattern("2022-01-01","yyyy-MM-dd") + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\成交案例统计_0531.xlsx", { d -> + if(d['A']){ + if(ccbCitys.contains(d['A'])){ + d['D'] = findOracleTotalCount(d['A']) + }else{ + def totalcount = findAllTradeCount(d['A'], startDate, endDate) + def repeatCount = findRepeatCount(d['A'], startDate, endDate) + def matchTotalCount = findMatchTradeCount(d['A'], startDate, endDate) + def matchRepeatTotalCount = findMatchRepeatTradeCount(d['A'], startDate, endDate) + d['B'] = totalcount + d['C'] = repeatCount + d['D'] = matchTotalCount + d['E'] = matchRepeatTotalCount + } + + } + }) + } + + + + def findRepeatCount(city, startDate, endDate){ + def q = new BasicDBObject() + q.put("city", city) + q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate)) + + def dbcursor = DEAL_PRICE_COLL.find(q) + def set = new HashSet() + for(def d : dbcursor){ + def key = d.region+"##"+d.district_name+"##"+d.avg_price+"##"+d.area+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyyMMdd") + if(!set.contains(key)){ + set << key + } + } + return set.size() + } + + def findMatchRepeatTradeCount(city, startDate, endDate){ + def q = new BasicDBObject() + q.put("city", city) + q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate)) + q.put("match_id", new BasicDBObject('$exists', true)) + def dbcursor = DEAL_PRICE_COLL.find(q) + def set = new HashSet() + for(def d : dbcursor){ + def key = d.match_id+"#"+d.avg_price+"##"+d.area+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyyMMdd") + if(!set.contains(key)){ + set << key + } + } + return set.size() + } + + + def findMatchTradeCount(city, startDate, endDate){ + def q = new BasicDBObject() + q.put("city", city) + q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate)) + q.put("match_id", new BasicDBObject('$exists', true)) + def count = DEAL_PRICE_COLL.count(q) + return count + } + + + public static def findAllTradeCount(city, startDate, endDate){ + def q = new BasicDBObject() + q.put("city", city) + q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate)) +// q.put("match_id", new BasicDBObject('$exists', false)) + def total = DEAL_PRICE_COLL.count(q) + return total + } + + def findOracleTotalCount(city){ + def sql = "select count(*) as count from bigdata_deal_gj where cityname='${city}' and dealdate>=to_date('2021-01-01','yyyy-MM-dd') and dealdate + count = r.count + }) + return count + } + + def run(){ + def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\建行成交\\92个城市字段覆盖率情况.xlsx",{ d-> + if(d['A']) { + def citycode = cityCodeDataMap.get(d['A']) + def total = findTotalCount(citycode) + def f_count = findDistFinishDate(citycode) + def b_count = findBuildTypeCount(citycode) + def f_pre = DataHandleUtils.getDoubleRound(2, f_count*100/total) + def b_pre = DataHandleUtils.getDoubleRound(2, b_count*100/total) + d['C'] = f_pre + d['D'] = b_pre + d['E'] = total + println d['A'] +" "+ total +" "+f_count+" "+b_count + } + }) + } + + def findTotalCount(citycode){ + def sql = "select count(*) as count from bigdata_community_gj where cityid='${citycode}' and status=0";//where cityId in (350200,150100) + def count = 0 + sqlInstance.eachRow(sql, { r -> + count = r.count + }) + return count + } + + def findDistFinishDate(citycode){ + def sql = "select count(*) as count from bigdata_community_gj where cityid='${citycode}' and status=0 and BUILDYEAR>0";//where cityId in (350200,150100) + def count = 0 + sqlInstance.eachRow(sql, { r -> + count = r.count + }) + return count + } + + def findBuildTypeCount(citycode){ + def sql = "select count(*) as count from (select commid, count(commid) as count from bigdata_commbuildtype_gj where cityid='${citycode}' and BUILDTYPE!='0199' group by commid)";//where cityId in (350200,150100) + def count = 0 + sqlInstance.eachRow(sql, { r -> + count = r.count + }) + return count + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy index d554c0376..7e5074c7c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy @@ -137,7 +137,7 @@ ExcelUtils.write("d:\\ def s = "第三方均价总量:${total}\n6区总量:${regionCount}\ncount1: "+ count1+"( "+pre1+"% )\ncount2: "+ count2+"( "+pre2+"% )\ncount3: "+count3+"( "+pre3+"% )\ncount4: "+count4+"( "+pre4+"% )\ncount5: "+count5+"( "+pre5+"% )\ncount6: "+count6+"( "+pre6+"% )\ncount7: "+count7+"( "+pre7+"%)\ncount8: "+count8+"( "+pre8+"% )\ncount9: "+count9+" ("+pre9+"% )" println s def access_token = "065f4a05d5f87eb39a9f9e048eeefb80c1f7560446f8926441de80453c160c60" -//DingDingMessageUtil.sendTextMessage(access_token, s) +DingDingMessageUtil.sendTextMessage(access_token, s) ////def districts = CommonDatautils.findAllDistricts("福州", "district", ["city", "region", "name"]); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy index 77239e3ea..e4b621c99 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy @@ -164,6 +164,7 @@ class OpenApiUtils { if(StringUtils.isEmpty(city) || StringUtils.isEmpty(address)){ return null; } + address = address.trim().replace(" ","") String url = "https://restapi.amap.com/v3/place/text?keywords=" + address + "&types=&city=" + city + "&output=json&offset=20&page=1&key=2af0b8533415a612b98d59bb93d664b9&extensions=all" List poiList = null; diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCityPriceStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCityPriceStat.groovy index c60a811b5..19709dd22 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCityPriceStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCityPriceStat.groovy @@ -18,44 +18,62 @@ class CcbCityPriceStat { task.run() } def run(){ - def citys = ["北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"] + def citys = ["潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","临汾","聊城","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","齐齐哈尔","眉山","郴州","营口","蚌埠","赤峰","大庆","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","朔州","随州","大理","通化","楚雄","巴彦淖尔","乌兰察布","松原","景德镇","新余","辽阳","阜新","黔南","酒泉","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","钦州","内江","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","鹤岗","萍乡","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","湘西","昭通","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝"] def rets = [] + def startDate = DateUtils.parseFromTightByPattern("2022-02-01","yyyy-MM-dd") + def endDate = DateUtils.parseFromTightByPattern("2022-06-01","yyyy-MM-dd") + for(def city : citys){ - def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "alias"]) - def priceMap = CommonDatautils.queryDistrictPrice(city, DateUtils.parseFromTightYYYYMMDD("2021-05-30")) - def cityRets = [] - for(def dist : districts){ - def tradeAvgPrice = findTrades(dist, "2021-02-01", "2021-06-01") - def weekendPrice = priceMap.get(city +"##"+dist.region+"##"+dist.name) - if(tradeAvgPrice && weekendPrice){ - def row = [city, dist.region, dist.name, weekendPrice, tradeAvgPrice.toInteger()] - cityRets << row - println row.toString() + def matchTotalCount = findMatchTradeCount(city, startDate, endDate) + println city +" "+matchTotalCount + if(matchTotalCount){ + def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "alias"]) + def priceMap = CommonDatautils.queryDistrictPrice(city, DateUtils.parseFromTightYYYYMMDD("2022-05-29")) + def cityRets = [] + for(def dist : districts){ + def tradeAvgPrice = findTrades(dist, "2022-02-01", "2022-06-01") + def weekendPrice = priceMap.get(city +"##"+dist.region+"##"+dist.name) + if(tradeAvgPrice && weekendPrice){ + def row = [city, dist.region, dist.name, weekendPrice, tradeAvgPrice.toInteger()] + cityRets << row + println row.toString() + } } + print city +" "+ cityRets.size() + rets.addAll(cityRets) } - print city +" "+ cityRets.size() - rets.addAll(cityRets) + } println "总量:"+rets.size() - ExcelUtils.write("d:\\数据\\92个城市价格数据.xlsx", rets, "所有") + ExcelUtils.write("d:\\数据\\234个城市价格数据.xlsx", rets, "所有") + } + + + def findMatchTradeCount(city, startDate, endDate){ + def q = new BasicDBObject() + q.put("city", city) + q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate)) + q.put("match_id", new BasicDBObject('$exists', true)) + def count = DEAL_PRICE_COLL.count(q) + return count } def findTrades(dist, startDate, endDate){ def q = new BasicDBObject() - q.put("city", dist.city) - q.put("region", dist.region) - - if(dist.alias){ - if(!dist.alias.contains(dist.name)){ - dist.alias << dist.name - } - q.put("district_name", new BasicDBObject('$in', dist.alias)) - }else{ - q.put("district_name", dist.name) - } - +// q.put("city", dist.city) +// q.put("region", dist.region) +// +// if(dist.alias){ +// if(!dist.alias.contains(dist.name)){ +// dist.alias << dist.name +// } +// q.put("district_name", new BasicDBObject('$in', dist.alias)) +// }else{ +// q.put("district_name", dist.name) +// } + q.put("match_id", dist._id.toString()) q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)).append('$lt',DateUtils.parseFromTightYYYYMMDD(endDate))) def dbcorsor = DEAL_PRICE_COLL.find(q) def avg_price = 0 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeHouseCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeHouseCrawl.groovy new file mode 100644 index 000000000..1773bdd8f --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeHouseCrawl.groovy @@ -0,0 +1,189 @@ +package com.fangjia.script.gj.crawl + +import com.alibaba.fastjson.JSON +import com.alibaba.fastjson.JSONObject +import com.fangjia.core.RegionData +import com.fangjia.script.crawl.JQueryUtils +import com.fangjia.script.gj.EsUtils +import com.fangjia.script.temp.shenjian.util.MessageUtils +import com.fangjia.script.util.CommonUtils +import org.jsoup.Jsoup +import org.openqa.selenium.WebDriver +import org.openqa.selenium.chrome.ChromeDriver + +class AnjukeHouseCrawl { + def targetUrl = "http://192.168.0.206:9200/amap_poi/_doc" + static main(args) { + def task = new AnjukeHouseCrawl(); + task.run(); + } + + def run(){ + System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); + System.setProperty("http.nonProxyHosts", "localhost"); + WebDriver driver = new ChromeDriver() + for(def i=24;i<50;i++){ + def url = "https://chengde.anjuke.com/sale/shuangqiao/p${i}/?from=navigation" + def doc = getDocument(url, driver) + def eles = JQueryUtils.select(".list-main .list-left .list .property a", doc) + println "页数:"+i+" 记录数:"+eles.size() + if(eles){ + parser(eles, driver) + } + } + } + + def parser(eles, driver){ + def index = 0 + for(def e : eles){ + index ++ + def href = e.attr("href") + def doc = getDocument(href, driver) +// parserHtml(doc) + def newUrl = href.substring(0, href.indexOf("?")) + sendMq(newUrl, doc) + + println "[ "+index +"] "+ newUrl + } + } + + def sendMq(href, doc){ + JSON json = new JSONObject() + json.put("project", "sell_anjuke") + json.put("url", href) + + JSON result = new JSONObject() + result.put("content", doc.toString()) + json.put("result", result) + + byte[] packedMsg = MessageUtils.pack(json) + CommonUtils.sendMessage("result2parse", packedMsg) + } + + def parserHtml(doc) { + String city = null + def result = [:] + result.siteCode = "安居客" + if (doc) { + result.title = JQueryUtils.selectText([".banner-title h1"], doc) + def heads = JQueryUtils.select(".crumbs a", doc) + + if (heads && heads.size() >= 5) { + result.city = heads[0].text().replace("房产网", "") + def orgRegion = heads[2].text().replace("二手房", "") + result.region = RegionData.formatRegion(result.city, orgRegion.trim()) + result.block = heads[3].text().replace("二手房", "") + result.district_name = heads[4].text() + } else if (heads && heads.size() >= 3) { + result.city = heads[0].text().replace("房产网", "") + result.district_name = heads[2].text() + def eles = JQueryUtils.select("li:contains(所在位置:) a", doc) + if (eles && eles.size() > 0) { + def orgRegion = eles[0].text() + result.region = RegionData.formatRegion(result.city, orgRegion.trim()) + } + } + + if (!result.district_name) { + println " [未解析出(城市-区域-名称)信息]" + } + println result + + + def priceStr = JQueryUtils.selectText(".maininfo-price-num", doc) + if (priceStr) { + if (priceStr.isNumber()) { + result.total_price = priceStr.toDouble() * 10000 + } + } + def roomInfoStr = JQueryUtils.selectText(".maininfo-model-item-1 .maininfo-model-strong", doc) + if (roomInfoStr) { + Integer room = DataCrawlUtils.parseRoom(roomInfoStr) + Integer hall = DataCrawlUtils.parseHall(roomInfoStr) + Integer toilet = DataCrawlUtils.parseToilet(roomInfoStr) + + if (room) { + result.room = room + } + if (hall) { + result.hall = hall + } + if (toilet) { + result.toilet = toilet + } + } + + def areaStr = JQueryUtils.selectText(".maininfo-model-item-2 .maininfo-model-strong", doc).replace("㎡", "") + if (areaStr) { + result.area = areaStr.toDouble() + } + def directionStr = JQueryUtils.selectText(".maininfo-model-item-3 i", doc) + result.direction = DataCrawlUtils.parseDirection(directionStr) + + def buildDateStr = JQueryUtils.selectText(".maininfo-model-item-3 .maininfo-model-weak", doc) + if (buildDateStr && buildDateStr.contains("/")) { + def arrs = buildDateStr.split("\\/") + if (arrs && arrs.size() == 2) { + result.build_date = arrs[0] + result.estate_type = arrs[1] + } + } + + result.floor_info = JQueryUtils.selectText(".maininfo-model-item-1 .maininfo-model-weak", doc) + if (result.floor_info && result.floor_info.contains("共")) { + def height = result.floor_info + if (result.floor_info.contains("(")) { + height = height.substring(result.floor_info.indexOf("(") + 1, result.floor_info.indexOf(")")) + } + height = height.replace("共", "").replace("层", "") + if (height) { + result.height = height.toInteger() + } + } + + + if (result.area && result.total_price) { + result.avg_price = (result.total_price / result.area).toInteger() + } + + def fitmitStr = JQueryUtils.selectText(".maininfo-model-item-2 .maininfo-model-weak", doc) + result.fitment = DataCrawlUtils.parseFitment(fitmitStr) + + + def dateStr = JQueryUtils.selectText(["td:contains(发布时间) .houseInfo-main-item-name"], doc) + + + } + } + + def getDocument(url, driver){ + def stop = false + def error = 0 + def doc = null + while(!stop){ + try{ + driver.get(url) + doc = Jsoup.parse(driver.getPageSource()); + Thread.sleep(5000) + def els = JQueryUtils.select(".list-main .list-left .list", doc) + if(!els){ + els = JQueryUtils.select(".props-main", doc) + } + if(els && els.size()>0){ + stop = true; + }else{ + error++ + driver.manage().deleteAllCookies() + println "[${error}] error: "+url + Thread.sleep(5000) + } + }catch(Exception e){ + error++ + } + if(error>=20){ + stop = true; + } + } + return doc + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/DataCrawlUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/DataCrawlUtils.groovy new file mode 100644 index 000000000..465ae5f04 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/DataCrawlUtils.groovy @@ -0,0 +1,250 @@ +package com.fangjia.script.gj.crawl + +import com.fangjia.script.util.DateUtils +import org.apache.commons.lang.StringUtils + +import java.util.regex.Matcher +import java.util.regex.Pattern + +class DataCrawlUtils { + static final Map DEAL_DATE_RESOLVERS = [:] + static final Map ROOM_RESOLVERS = [:] + static final Map HALL_RESOLVERS = [:] + static final Map TOILET_RESOLVERS = [:] + static final Map TOTAL_PRICE_RESOLVERS = [:] + static final Map UNIT_PRICE_RESOLVERS = [:] + static final Map AREA_RESOLVERS = [:] + static final Map HEIGHT_RESOLVERS = [:] + static final Map FLOOR_RESOLVERS = [:] + static final List DIRECTIONS = ["东北", "西北", "东南", "西南", "南北", "东西", "东", "南", "西", "北"] + private static final HashMap> FITMENTS_TYPES_MAPS = new HashMap>() + + static { + DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{4}-\\d{2}-\\d{2}"), { String str -> + return DateUtils.parseFromTightByPattern(str, "yyyy-MM-dd") + }) + DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{2}-\\d{2}-\\d{2}"), { String str -> + str = "20" + str + return DateUtils.parseFromTightByPattern(str, "yyyy-MM-dd") + }) + DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{2}/\\d{2}/\\d{2}"), { String str -> + str = "20" + str + return DateUtils.parseFromTightByPattern(str, "yyyy/MM/dd") + }) + + DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{4}\\.\\d{2}\\.\\d{2}"), { String str -> + return DateUtils.parseFromTightByPattern(str, "yyyy.MM.dd") + }) + DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{4}-\\d{2}"), { String str -> + return DateUtils.parseFromTightByPattern(str, "yyyy-MM") + }) + + ROOM_RESOLVERS.put(Pattern.compile("[\\d+|一|二|三|四|五|两](室|房)"), { String str -> + str = str.replaceAll("室|房", '').trim() + return str.toInteger() + }) + + HALL_RESOLVERS.put(Pattern.compile("[\\d+|一|二|三|四|五|两]厅"), { String str -> + str = str.replace("厅", '').trim() + return str.toInteger() + }) + TOILET_RESOLVERS.put(Pattern.compile("[\\d+|一|二|三|四|五|两]卫"), { String str -> + str = str.replace("卫", '').trim() + return str.toInteger() + }) + TOTAL_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?万"), { String str -> + str = str.replace("万", '').trim() + return (str.toDouble() * 10000).toInteger() + }) + + UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元/㎡"), { String str -> + str = str.replace("元/㎡", '').trim() + return str.toDouble().toInteger() + }) + UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元/m?"), { String str -> + str = str.replace("元/m?", '').trim() + return str.toDouble().toInteger() + }) + UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元/平"), { String str -> + str = str.replace("元/平", '').trim() + return str.toDouble().toInteger() + }) + UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元"), { String str -> + str = str.replace("元", '').trim() + return str.toDouble().toInteger() + }) + + AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?平米"), { String str -> + str = str.replace("平米", '').trim() + return str.toDouble() + }) + + AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?平方米"), { String str -> + str = str.replace("平方米", '').trim() + return str.toDouble() + }) + + AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?平"), { String str -> + str = str.replace("平", '').trim() + return str.toDouble() + }) + + AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?㎡"), { String str -> + str = str.replace("㎡", '').trim() + return str.toDouble() + }) + AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?m?"), { String str -> + str = str.replace("m?", '').trim() + return str.toDouble() + }) + HEIGHT_RESOLVERS.put(Pattern.compile("共\\d+层"), { String str -> + str = str.replace("共", '').replace('层', '') + return str.toInteger() + }) + + HEIGHT_RESOLVERS.put(Pattern.compile("总\\d+层"), { String str -> + str = str.replace("总", '').replace('层', '') + return str.toInteger() + }) + + HEIGHT_RESOLVERS.put(Pattern.compile("(?<=\\D)(\\d{1,2}/[1-6]?\\d)(?=\\D)"), { String str -> + def split = str.split("/") + def n1 = split[0].toInteger() + def n2 = split[1].toInteger() + if (n1 <= n2) { + return n2 + } + return null + }) + + + FLOOR_RESOLVERS.put(Pattern.compile("(?<=\\D)(\\d{1,2}/[1-6]?\\d)(?=\\D)"), { String str -> + def split = str.split("/") + def n1 = split[0].toInteger() + def n2 = split[1].toInteger() + if (n1 <= n2) { + return n1 + } + return n1 + }) + + FLOOR_RESOLVERS.put(Pattern.compile("第\\d+(层|楼)"), { String str -> + str = str.replace("第", "").replaceAll("层|楼", "") + return str.toInteger() + }) + + FITMENTS_TYPES_MAPS.put("毛坯", ["毛坯", "清水", "无装修", "独栋毛坯", "毛胚", "白坯", "旧装", "坯", "原房", "毛坏"]) + FITMENTS_TYPES_MAPS.put("简装", ["简装", "普通装修", "普通", "简单装修", "简易装修", "普装", "一般装修", "一般", "简", "粗装修"]) + FITMENTS_TYPES_MAPS.put("精装", ["精装", "全装修", "全装", "中等装修", "中装", "中档装修", "精致装修", "中档", "中等", "精", "中修", "优质装修", "时尚装修", "良装", "中高挡"]) + FITMENTS_TYPES_MAPS.put("豪装", ["豪华装修", "高档", "豪华", "豪装", "高级装修", "高装", "奢侈"]) + + } + + /** + * 解析室数 + * @param content + * @return + */ + static Integer parseRoom(String content) { + return parse(content, ROOM_RESOLVERS) + } + /** + * 解析厅数 + * @param content + * @return + */ + static Integer parseHall(String content) { + return parse(content, HALL_RESOLVERS) + } + /** + * 解析厅数 + * @param content + * @return + */ + static Integer parseToilet(String content) { + return parse(content, TOILET_RESOLVERS) + } + + static Object parse(String content, Map resolvers) { + if (content != null) { + content = content.replaceAll("\\s", "") + } + for (def resolver : resolvers.entrySet()) { + String match = match(content, resolver.getKey()) + if (StringUtils.isNotBlank(match)) { + match = match.replace("一","1").replace("二","2").replace("三","3").replace("四","4").replace("五","5").replace("两","2") + return resolver.getValue()(match) + } + } + return null + } + + static String match(String str, Pattern pattern) { + if (str == null) { + return null + } + Matcher matcher = pattern.matcher(str) + if (matcher.find()) { + return matcher.group() + } + return null + } + + static String parseDirection(String directionDesc) { + if (directionDesc == null) { + return null + } + directionDesc = directionDesc.replaceAll("(\\(|(|《|\"|“|【|〔).+?(\\)|》|)|\"|”|】|〕)", "#_#") + StringBuffer sb = new StringBuffer() + for (int i = 0; i < directionDesc.length(); i++) { + String c = directionDesc.substring(i, i + 1) + if (DIRECTIONS.contains(c)) { + if (directionDesc.substring(i).startsWith(c + "#_#")) { + sb.append(c + " ") + } else { + sb.append(c) + } + } + } + for (String d : DIRECTIONS) { + if (sb.toString().startsWith(d)) { + return d + } + } + for (String d : DIRECTIONS) { + if (sb.toString().contains(d)) { + return d + } + } + return null + + } + + static String parseFitment(String fitmentDesc) { + if (fitmentDesc == null) { + return null + } + fitmentDesc = fitmentDesc.replaceAll("[\\s ]+", "") + if (fitmentDesc.equals("非毛坯")) { + return null + } + Iterator>> it = FITMENTS_TYPES_MAPS + .entrySet().iterator() + Map.Entry> entry + while (it.hasNext()) { + entry = it.next() + if (fitmentDesc.contains(entry.getKey())) { + return entry.getKey() + } + for (String fitmentKey : entry.getValue()) { + if (fitmentDesc.contains(fitmentKey)) { + return entry.getKey() + } + } + } + return null + + } + + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/DealPriceUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/DealPriceUtils.groovy new file mode 100644 index 000000000..02fc9458d --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/DealPriceUtils.groovy @@ -0,0 +1,347 @@ +package com.fangjia.script.gj.crawl + +import com.fangjia.script.util.DateUtils +import org.apache.commons.lang.StringUtils + +import java.util.regex.Matcher +import java.util.regex.Pattern + +/** + * 鎴愪氦鏁版嵁瑙f瀽宸ュ叿绫� + */ +class DealPriceUtils { + + static final Map DEAL_DATE_RESOLVERS = [:] + static final Map ROOM_RESOLVERS = [:] + static final Map HALL_RESOLVERS = [:] + static final Map TOILET_RESOLVERS = [:] + static final Map TOTAL_PRICE_RESOLVERS = [:] + static final Map UNIT_PRICE_RESOLVERS = [:] + static final Map AREA_RESOLVERS = [:] + static final Map HEIGHT_RESOLVERS = [:] + static final Map FLOOR_RESOLVERS = [:] + static final List DIRECTIONS = ["涓滃寳", "瑗垮寳", "涓滃崡", "瑗垮崡", "鍗楀寳", "涓滆タ", "涓�", "鍗�", "瑗�", "鍖�"] + private static final HashMap> FITMENTS_TYPES_MAPS = new HashMap>() + + static { + DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{4}-\\d{2}-\\d{2}"), { String str -> + return DateUtils.parseFromTightByPattern(str, "yyyy-MM-dd") + }) + DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{2}-\\d{2}-\\d{2}"), { String str -> + str = "20" + str + return DateUtils.parseFromTightByPattern(str, "yyyy-MM-dd") + }) + DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{2}/\\d{2}/\\d{2}"), { String str -> + str = "20" + str + return DateUtils.parseFromTightByPattern(str, "yyyy/MM/dd") + }) + + DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{4}\\.\\d{2}\\.\\d{2}"), { String str -> + return DateUtils.parseFromTightByPattern(str, "yyyy.MM.dd") + }) + DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{4}-\\d{2}"), { String str -> + return DateUtils.parseFromTightByPattern(str, "yyyy-MM") + }) + + ROOM_RESOLVERS.put(Pattern.compile("[\\d+|涓�|浜寍涓墊鍥泑浜攟涓(瀹鎴�)"), { String str -> + str = str.replaceAll("瀹鎴�", '').trim() + return str.toInteger() + }) + + HALL_RESOLVERS.put(Pattern.compile("[\\d+|涓�|浜寍涓墊鍥泑浜攟涓鍘�"), { String str -> + str = str.replace("鍘�", '').trim() + return str.toInteger() + }) + TOILET_RESOLVERS.put(Pattern.compile("[\\d+|涓�|浜寍涓墊鍥泑浜攟涓鍗�"), { String str -> + str = str.replace("鍗�", '').trim() + return str.toInteger() + }) + TOTAL_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?涓�"), { String str -> + str = str.replace("涓�", '').trim() + return (str.toDouble() * 10000).toInteger() + }) + + UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?鍏�/銕�"), { String str -> + str = str.replace("鍏�/銕�", '').trim() + return str.toDouble().toInteger() + }) + UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?鍏�/m虏"), { String str -> + str = str.replace("鍏�/m虏", '').trim() + return str.toDouble().toInteger() + }) + UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?鍏�/骞�"), { String str -> + str = str.replace("鍏�/骞�", '').trim() + return str.toDouble().toInteger() + }) + UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?鍏�"), { String str -> + str = str.replace("鍏�", '').trim() + return str.toDouble().toInteger() + }) + + AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?骞崇背"), { String str -> + str = str.replace("骞崇背", '').trim() + return str.toDouble() + }) + + AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?骞虫柟绫�"), { String str -> + str = str.replace("骞虫柟绫�", '').trim() + return str.toDouble() + }) + + AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?骞�"), { String str -> + str = str.replace("骞�", '').trim() + return str.toDouble() + }) + + AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?銕�"), { String str -> + str = str.replace("銕�", '').trim() + return str.toDouble() + }) + AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?m虏"), { String str -> + str = str.replace("m虏", '').trim() + return str.toDouble() + }) + HEIGHT_RESOLVERS.put(Pattern.compile("鍏盶\d+灞�"), { String str -> + str = str.replace("鍏�", '').replace('灞�', '') + return str.toInteger() + }) + + HEIGHT_RESOLVERS.put(Pattern.compile("鎬籠\d+灞�"), { String str -> + str = str.replace("鎬�", '').replace('灞�', '') + return str.toInteger() + }) + + HEIGHT_RESOLVERS.put(Pattern.compile("(?<=\\D)(\\d{1,2}/[1-6]?\\d)(?=\\D)"), { String str -> + def split = str.split("/") + def n1 = split[0].toInteger() + def n2 = split[1].toInteger() + if (n1 <= n2) { + return n2 + } + return null + }) + + + FLOOR_RESOLVERS.put(Pattern.compile("(?<=\\D)(\\d{1,2}/[1-6]?\\d)(?=\\D)"), { String str -> + def split = str.split("/") + def n1 = split[0].toInteger() + def n2 = split[1].toInteger() + if (n1 <= n2) { + return n1 + } + return n1 + }) + + FLOOR_RESOLVERS.put(Pattern.compile("绗琝\d+(灞倈妤�)"), { String str -> + str = str.replace("绗�", "").replaceAll("灞倈妤�", "") + return str.toInteger() + }) + + FITMENTS_TYPES_MAPS.put("姣涘澂", ["姣涘澂", "娓呮按", "鏃犺淇�", "鐙爧姣涘澂", "姣涜儦", "鐧藉澂", "鏃ц", "鍧�", "鍘熸埧", "姣涘潖"]) + FITMENTS_TYPES_MAPS.put("绠�瑁�", ["绠�瑁�", "鏅�氳淇�", "鏅��", "绠�鍗曡淇�", "绠�鏄撹淇�", "鏅", "涓�鑸淇�", "涓�鑸�", "绠�", "绮楄淇�"]) + FITMENTS_TYPES_MAPS.put("绮捐", ["绮捐", "鍏ㄨ淇�", "鍏ㄨ", "涓瓑瑁呬慨", "涓", "涓。瑁呬慨", "绮捐嚧瑁呬慨", "涓。", "涓瓑", "绮�", "涓慨", "浼樿川瑁呬慨", "鏃跺皻瑁呬慨", "鑹", "涓珮鎸�"]) + FITMENTS_TYPES_MAPS.put("璞", ["璞崕瑁呬慨", "楂樻。", "璞崕", "璞", "楂樼骇瑁呬慨", "楂樿", "濂緢"]) + + } + /** + * 瑙f瀽鎴愪氦鏃ユ湡 + * @param content + * @return + */ + static Date parseDealDate(String content) { + return parse(content, DEAL_DATE_RESOLVERS) + } + + /** + * 瑙f瀽瀹ゆ暟 + * @param content + * @return + */ + static Integer parseRoom(String content) { + return parse(content, ROOM_RESOLVERS) + } + /** + * 瑙f瀽鍘呮暟 + * @param content + * @return + */ + static Integer parseHall(String content) { + return parse(content, HALL_RESOLVERS) + } + /** + * 瑙f瀽鍘呮暟 + * @param content + * @return + */ + static Integer parseToilet(String content) { + return parse(content, TOILET_RESOLVERS) + } + + /** + * 瑙f瀽鎬讳环 + * @param content + * @return + */ + static Integer parseTotalPrice(String content) { + return parse(content, TOTAL_PRICE_RESOLVERS) + } + /** + * 瑙f瀽鍗曚环 + * @param content + * @return + */ + static Integer parseUnitPrice(String content) { + return parse(content, UNIT_PRICE_RESOLVERS) + } + /** + * 瑙f瀽闈㈢Н + * @param content + * @return + */ + static Double parseArea(String content) { + return parse(content, AREA_RESOLVERS) + } + /** + * 瑙f瀽妤煎眰 + * @param content + * @return + */ + static Integer parseFloor(String content) { + return parse(content, FLOOR_RESOLVERS) + } + /** + * 瑙f瀽妤奸珮 + * @param content + * @return + */ + static Integer parseHeight(String content) { + return parse(content, HEIGHT_RESOLVERS) + } + + /** + * 鏍煎紡鍖栧皬鍖哄悕 + * @param name + * @return + */ + static String formatName(String name) { + name = name.replace("-鍟嗕綇", "").replace("-鍒", "") + //姹囩瀹跺洯骞冲畨閲岋紙杩侊級2015-2017 + name = name.replaceAll("\\d\\d\\d\\d-\\d\\d\\d\\d", "") + //鍥借锤澶╂偊涓�鏈�2017 + name = name.replaceAll("\\d\\d\\d\\d\$", "") + if (name ==~ /.+(\(|锛�)[\d涓�浜屼笁鍥涗簲鍏竷鍏節鍗佷笢鍗楄タ鍖梶\w]+(鏈焲鍖�)(\)|锛�)$/) { + name = name.replaceAll("\\(|锛坾\\)|锛�", '') + } else if (name ==~ /.+(\(|锛�)鍒(\)|锛�)$/) { + name = name.replaceAll("\\(|锛坾\\)|锛�", '') + } else if (name ==~ /.+(\(|锛�).+(\)|锛�)$/) { + name = name.replaceAll("(\\(|锛�).+(\\)|锛�)", '') + } else if (name.contains("路") || (name.contains("-") && !(name ==~ /.*\d+-\d+.*/))) { + println("---------------------------->" + name) + name = name.replaceAll("路|-", '') + println(name) + } + if (name ==~ /.{2,}璺�.{3,}$/ && !(name ==~ /.{2,}璺�.+鍙�/) && !(name ==~ /.{2,}璺�.+寮�/) && !(name ==~ /.*灏忓尯$/)) { + name = name.replaceAll("^.{2,}璺�", "") + } + return name + } + + + static Object parse(String content, Map resolvers) { + if (content != null) { + content = content.replaceAll("\\s", "") + } + for (def resolver : resolvers.entrySet()) { + String match = match(content, resolver.getKey()) + if (StringUtils.isNotBlank(match)) { + match = match.replace("涓�","1").replace("浜�","2").replace("涓�","3").replace("鍥�","4").replace("浜�","5").replace("涓�","2") + return resolver.getValue()(match) + } + } + return null + } + + /** + * 鍖归厤绗﹀悎鏉′欢鐨勫瓧绗︿覆 + * + * @param str + * @param pattern + * @return + */ + static String match(String str, Pattern pattern) { + if (str == null) { + return null + } + Matcher matcher = pattern.matcher(str) + if (matcher.find()) { + return matcher.group() + } + return null + } + + static String parseFitment(String fitmentDesc) { + if (fitmentDesc == null) { + return null + } + fitmentDesc = fitmentDesc.replaceAll("[\\s ]+", "") + if (fitmentDesc.equals("闈炴瘺鍧�")) { + return null + } + Iterator>> it = FITMENTS_TYPES_MAPS + .entrySet().iterator() + Map.Entry> entry + while (it.hasNext()) { + entry = it.next() + if (fitmentDesc.contains(entry.getKey())) { + return entry.getKey() + } + for (String fitmentKey : entry.getValue()) { + if (fitmentDesc.contains(fitmentKey)) { + return entry.getKey() + } + } + } + return null + + } + + static String parseDirection(String directionDesc) { + if (directionDesc == null) { + return null + } + directionDesc = directionDesc.replaceAll("(\\(|锛坾銆妡\"|鈥渱銆恷銆�).+?(\\)|銆媩锛墊\"|鈥潀銆憒銆�)", "#_#") + StringBuffer sb = new StringBuffer() + for (int i = 0; i < directionDesc.length(); i++) { + String c = directionDesc.substring(i, i + 1) + if (DIRECTIONS.contains(c)) { + if (directionDesc.substring(i).startsWith(c + "#_#")) { + sb.append(c + " ") + } else { + sb.append(c) + } + } + } + for (String d : DIRECTIONS) { + if (sb.toString().startsWith(d)) { + return d + } + } + for (String d : DIRECTIONS) { + if (sb.toString().contains(d)) { + return d + } + } + return null + + } + + + static String parsePrice(String priceStr) { + def pattern =Pattern.compile("\\d+(\\.\\d+)?涓�") + String match = match(priceStr, pattern) + } + + static void main(String[] args) { + println parsePrice("椤剁娆e洯瑗胯嫅2瀹�1鍘�1鍗�492.3涓�") + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/ZhugeFangDistCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/ZhugeFangDistCrawl.groovy new file mode 100644 index 000000000..845ab47d0 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/ZhugeFangDistCrawl.groovy @@ -0,0 +1,167 @@ +package com.fangjia.script.gj.crawl + +import com.fangjia.script.crawl.JQueryUtils +import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil +import com.fangjia.script.thirdPriceCrawl.Utils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.jsoup.Jsoup +import org.openqa.selenium.By +import org.openqa.selenium.WebDriver +import org.openqa.selenium.chrome.ChromeDriver + +class ZhugeFangDistCrawl { + + public static final SHORT_DATE = Utils.getLastMonth(); + def static final DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol(); + + static main(args) { + def task = new ZhugeFangDistCrawl(); + task.run(); + } + def run(){ + System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); + System.setProperty("http.nonProxyHosts", "localhost"); + def url = "https://fz.esfxiaoqu.zhuge.com/fuqing/" + WebDriver driver = new ChromeDriver() + driver.manage().deleteAllCookies() + getRegions(driver, url) + } + + def getRegions(driver, url) { + for(def i=1;i<=11;i++){ + def newUrl = url+"page${i}/" + println newUrl + def doc = getDocument(newUrl, driver) + def total = parseList(doc) + +// def next = JQueryUtils.select("a:contains(下一页)", doc) +// if(next){ +// driver.findElement(By.xpath("//a[contains(text(), '下一页')]")).click() +// } + + } + + } + + def parseList(doc){ + def city= "福州" + def els = JQueryUtils.select("#listTableBox li", doc) + if (els) { + for(def e : els){ + def name = e.select(".house-name").text() + def str = e.select(".house-adr").text() + def address = null + if(str){ + if(str.contains("]")){ + address = str.substring(str.indexOf("]")+1).trim() + }else{ + address = str + } + } + def regionStr = null + if(str && str.contains("]") && str.contains("[")){ + regionStr = str.substring(str.indexOf("[")+1, str.indexOf("]")) + } + + def region = null + def block = null + if(regionStr){ + def arrs = regionStr.split("-") + if(arrs && arrs.size()>0){ + if(arrs.size()==2){ + block = arrs[1].trim() + } + if(arrs.size()>=1){ + region = arrs[0].trim() + } + + } + } + def links = e.select(".house-name a") + def href = null + if(links){ + href = links[0].attr("href") + } + def priceStr = e.select(".price-number").text() + def price = null + if(priceStr && !priceStr.contains("暂无")){ + if(priceStr.contains("元")){ + priceStr = priceStr.substring(0, priceStr.indexOf("元")).trim() + } + if(priceStr && priceStr.isNumber()){ + price = priceStr.toInteger() + } + } + def obj = findByParams(city, region, name) + if(obj){ + if(price){ + DBObject udata2 = new BasicDBObject() + udata2.put("avg_price", price) + udata2.put("m_date", new Date()) + println "[upeate] ${region} "+name + thirdPartyColl.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', udata2)); + } + }else{ + def udata = new BasicDBObject() + udata.put("city", city) + udata.put("region", region) + udata.put("block", block) + udata.put("url", href) + udata.put("address", address) + udata.put("name", name) + udata.put("avg_price", price) + udata.put("s_date", SHORT_DATE) + udata.put("c_date", new Date()) + udata.put("m_date", new Date()) + udata.put("source", "诸葛找房") + thirdPartyColl.save(udata) + println "[save] ${region} "+name+" "+priceStr + } + } + } + return els.size() + } + + + def findByParams(city, region, name){ + def q = new BasicDBObject() + q.put("city", city) + q.put("region", region) + q.put("name", name) + q.put("source", "诸葛找房") + q.put("s_date", SHORT_DATE) + return thirdPartyColl.findOne(q); + } + + + def getDocument(url, driver){ + def stop = false + def error = 0 + def doc = null + while(!stop){ + try{ + driver.get(url) + doc = Jsoup.parse(driver.getPageSource()); + Thread.sleep(10000) + def els = JQueryUtils.select("#listTableBox li", doc) + if(els && els.size()>0){ + stop = true; + }else{ + error++ + driver.manage().deleteAllCookies() + println "[${error}] error: "+url + Thread.sleep(5000) + } + }catch(Exception e){ + error++ + } + if(error>=20){ + stop = true; + } + } + return doc + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/DistrictMapMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/DistrictMapMatch.groovy new file mode 100644 index 000000000..47c9fe748 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/DistrictMapMatch.groovy @@ -0,0 +1,123 @@ +package com.fangjia.script.gj.location + +import com.fangjia.core.util.GeoUtils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.OpenApiUtils +import com.fangjia.script.util.CosineSimilarAlgorithm +import com.fangjia.script.util.ExcelUtils + +class DistrictMapMatch { + + static main(args) { + def task = new DistrictMapMatch() + task.mapMatch() + } + + + def mapMatch(){ + def index = 0 + ExcelUtils.modify("D:\\数据\\重庆农商行\\小区\\待审核小区(地图匹配).xlsx",{ d-> + if(!d['H']){ + index++ + def rets = OpenApiUtils.getGaoDeSearchLocation(d['A'],d['B'],d['C']) + def map = findStandardObject2(d['A'],d['B'],d['C'], d['D'], rets); +// if (!map || !map.lat) { +// rets = OpenApiUtils.getBiDuLocation(d['A'],d['B'],d['C']); +// map = findStandardObject2(d['A'],d['B'],d['C'], d['D'], rets); +// } + if(map){ + d['F'] = map.id + if(map.adname){ + d['G'] = map.adname + }else if(map.region){ + d['G'] = map.region + } + d['H'] = map.name + d['I'] = map.address + if(map.type){ + d['J'] = map.type + }else if(map.estateType){ + d['J'] = map.estateType + } + d['K'] = map.lng + d['L'] = map.lat + d['M'] = map.score + d['N'] = map.source + } + } + },0) + } + + + def findStandardObject2(def city, def region, def name, def address, def rets) { + if (rets) { + def bestObj = [:] + if (rets.size() >= 1) { + for (def mapObj : rets) { + def score = 0 + if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){ + score = 0.2 + } + if(mapObj.name && name){ + def newName = mapObj.name; + def names = [] + if(mapObj.name && mapObj.name.indexOf("(")>-1){ + def name1 = mapObj.name.substring(0, mapObj.name.indexOf("(")) + names << name1 + def name2 = mapObj.name.substring(mapObj.name.indexOf("(")+1, mapObj.name.indexOf(")")) + names << name2 + }else{ + names << newName + } + def maxNameScore = 0 + for(def n : names){ + double score1 = CosineSimilarAlgorithm.getSimilarity(n, name); + if(score1 && score1>0){ + def nameScore = (score1 * 4)/10 + if(nameScore > maxNameScore){ + maxNameScore = nameScore + } + } + } + + if(maxNameScore){ + try{ + maxNameScore = DataHandleUtils.getDoubleRound(2, maxNameScore) + }catch(Exception e){ + maxNameScore = 0 + } + + } + score = maxNameScore + score; + } + + if(mapObj.address && address){ + double addressScore = CosineSimilarAlgorithm.getSimilarity(mapObj.address, address); + if(addressScore && addressScore>0){ + try{ + addressScore = DataHandleUtils.getDoubleRound(2, addressScore) + }catch(Exception e){ + addressScore = 0 + } + score = addressScore*4/10 + score + } + } + + if(!bestObj){ + mapObj.score = score + bestObj = mapObj + }else{ + if(score > bestObj.score){ + mapObj.score = score + bestObj = mapObj + } + } + + } + } + return bestObj + } + return null + + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy index 95bfde522..fe522570a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy @@ -945,20 +945,20 @@ class CZDataBatch extends BaseBatch{ if(price){ return price.avg_price; }else{ - //样例估价 - def s = OpenApiClient.evaluateDistrictPriceBySimilarHouse3(city, region, name, 100, false); - try{ - JSONObject jsonObject = JSONObject.fromObject(s); - def map = DataHandleUtils.JsonToMap(jsonObject); - if(map.code==200){ - if(map.result){ - return map.result.avgPrice; - } - - } - }catch(Exception e){ - - } +// //样例估价 +// def s = OpenApiClient.evaluateDistrictPriceBySimilarHouse3(city, region, name, 100, false); +// try{ +// JSONObject jsonObject = JSONObject.fromObject(s); +// def map = DataHandleUtils.JsonToMap(jsonObject); +// if(map.code==200){ +// if(map.result){ +// return map.result.avgPrice; +// } +// +// } +// }catch(Exception e){ +// +// } //分期找总期 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/didata/MatchFromExcel.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/didata/MatchFromExcel.groovy new file mode 100644 index 000000000..6cf7730ee --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/didata/MatchFromExcel.groovy @@ -0,0 +1,28 @@ +package com.fangjia.script.temp.shenjian.didata + +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.OpenApiClient + +/** + * 从excel中匹配出 + */ +ExcelUtils.multiThreadingModify("D:\\Users\\Domi\\Documents\\知识库\\地数\\数据建设\\待审核小区(地图匹配).xlsx", { row -> + def city = row['A'] + def region = row['B'] + def district_name = row['C'] + + def gaodeRegion = row['H'] + def gaodeName = row['I'] + + def jsonObject = OpenApiClient.addressMatch(city, district_name, "district", false, region, district_name, "strict") + if (jsonObject != null && jsonObject.precise != -1) { + row['D'] = jsonObject.get("name") + } + + if (gaodeRegion && gaodeName) { + def jsonObject2 = OpenApiClient.addressMatch(city, gaodeName, "district", false, gaodeRegion, gaodeName, "strict") + if (jsonObject2 != null && jsonObject2.precise != -1) { + row['J'] = jsonObject2.get("name") + } + } +}); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetPoiFromExcel.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetPoiFromExcel.groovy index bd49875b0..235d779e4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetPoiFromExcel.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetPoiFromExcel.groovy @@ -2,6 +2,8 @@ package com.fangjia.script.temp.shenjian.poi import com.alibaba.fastjson.JSONObject import com.fangjia.script.statistics.poi.AmapPoi +import com.fangjia.script.statistics.poi.AmapPoiDetail +import com.fangjia.script.temp.shenjian.util.CommonDtoUtils import com.fangjia.script.temp.shenjian.util.ExcelUtils import com.fangjia.script.temp.shenjian.util.GaoDeUtils import com.fangjia.script.temp.shenjian.util.HttpUtils @@ -14,13 +16,15 @@ import java.util.concurrent.atomic.AtomicLong class GetPoiFromExcel { - static final def URL = "http://192.168.0.216:38080/FangJiaDataService/saveAmapPoi" static final ExecutorService executorService = Executors.newFixedThreadPool(10) + static final def URL = "http://192.168.0.100:38080/FangJiaDataService/saveAmapPoiDetail" + // 住宅类 + static final String typecodes = "120000|120100|120200|120201|120202|120203|120300|120301|120302|120303|120304"; static void main(String[] args) { AtomicLong atomicLong = new AtomicLong(0) //List fileList = FileUtils.listFiles(new File("D:\\Users\\Domi\\Desktop\\房价网\\项目\\建行\\模型级联\\POI"), null, false); - List fileList = FileUtils.listFiles(new File("D:\\Users\\Domi\\Desktop\\房价网\\项目\\建行\\模型级联\\高德反查无ID"), null, false); + List fileList = FileUtils.listFiles(new File("D:\\Users\\Domi\\Documents\\知识库\\建行\\审核小区"), null, false); for (File file : fileList) { List> seaweeds = ExcelUtils.read(file.getPath()) for (List infoTemp : seaweeds) { @@ -31,9 +35,10 @@ class GetPoiFromExcel { String city = info.get(0) String region = info.get(1) String seaweedName = info.get(2) - List pois = GaoDeUtils.searchPoi(city, seaweedName, region, "120300|120301|120302"); + List pois = GaoDeUtils.searchPoi(city, seaweedName, region, typecodes); if (CollectionUtils.isNotEmpty(pois)) { - String jsonStr = JSONObject.toJSONString(pois) + List detailList = CommonDtoUtils.transformListV(AmapPoiDetail.class, pois); + String jsonStr = JSONObject.toJSONString(detailList) HttpUtils.postForJson(URL, jsonStr) atomicLong.incrementAndGet() } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedPoiDetail.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedPoiDetail.groovy index 3ae943007..ea6d9cc67 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedPoiDetail.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedPoiDetail.groovy @@ -81,6 +81,9 @@ class SeaweedPoiDetail { if (CollectionUtils.isEmpty(pois)) { pois = GaoDeUtils.searchPoi(city, name, region, typecodes) } + if (CollectionUtils.isEmpty(pois)) { + pois = GaoDeUtils.searchPoi(city, address, region, typecodes) + } List detailList = CommonDtoUtils.transformListV(AmapPoiDetail.class, pois); // for (AmapPoi amapPoi : pois) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy index ea456aa50..6b880c5ea 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -25,7 +25,7 @@ import java.util.concurrent.atomic.AtomicInteger class GaoDeUtils { // 新获取key - def static GAO_DE_AK = ["4ccfe3c898a9489884bd2bcb1ef6dcde", "4e9592208336433ee1b7aa4d1d434418"] + def static GAO_DE_AK = ["4e9592208336433ee1b7aa4d1d434418"] def static AtomicInteger keyUsed = new AtomicInteger(0) def static codeNameMap = new HashMap(){{ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy index 011f434e3..9a9fd90ad 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy @@ -30,7 +30,7 @@ import java.util.concurrent.ExecutorService import java.util.concurrent.Executors import java.util.concurrent.atomic.AtomicInteger import java.util.regex.Matcher -import java.util.regex.Patter +import java.util.regex.Pattern /** * 第三方数据标准化处理 @@ -514,6 +514,7 @@ class DataStandardBatch extends BaseBatch{ def filterSpacialChar(def s){ def pattern = ~/[()!;\·]\[【】()]/ def matcher = pattern.matcher(s) + def count = matcher.getCount() if(count && count>0){ return matcher.replaceAll('_') diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/CommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/CommonUtils.groovy index fe433f087..21887e3db 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/CommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/CommonUtils.groovy @@ -169,7 +169,7 @@ public class CommonUtils { static void sendMessage(String queueName, byte[] bytes){ ConnectionFactory factory = new ConnectionFactory(); - factory.setUri("amqp://guest:guest@192.168.0.19:45672/%2f"); + factory.setUri("amqp://guest:guest@192.168.0.100:35677/%2f"); Connection conn = factory.newConnection(); Channel channel = conn.createChannel(); channel.queueDeclare(queueName, true, false, false, null); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy index f0186187b..3d4e855f2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy @@ -348,6 +348,34 @@ public class OpenApiClient { } + public static JSONObject standard1Action(String city, String address) { + try { + Map params = new HashMap(); + params.put("city", city); + params.put("address", address); + params.put("token", TOKEN); + params.put("__refresh_cache","1") + + String content = new HttpRequest().get("https://open.fangjia.com/address/standard", params, "utf-8") + JSONObject result = JSONObject.fromObject(content); + while (result == null) { + result = HttpRequestUtil.getJsonContent("https://open.fangjia.com/address/standard", params, "utf-8") + println("${city} ${address} 程序调用异常") + Thread.sleep(6000) + } + if (result.getInt("code") == 200) { + return result.getJSONObject("result"); + } else if (result.getInt("code") !=0) { + println("${city} ${address}") + } + } catch (Exception e) { + println("${city} ${address} ${e.getMessage()}") + e.printStackTrace(); + } + return null + } + + public static JSONObject standardize(String city, String address) { try { Map params = new HashMap();