diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy index f35522733..406d043a8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy @@ -383,7 +383,6 @@ class DealPriceDataProcess { def sql = "delete from BIGDATA_THIRD_PARTY_PRICE_GJ where commid='${m.commid}' and price_date=${month} and community_price=${m.price}" sqlInstance236.executeUpdate(sql) } - } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeAppCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeAppCrawl.groovy index 5354eafe8..21ddf507c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeAppCrawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeAppCrawl.groovy @@ -48,13 +48,14 @@ class AnjukeAppCrawl { // task.run(); // task.run2() - task.addThirdPrice() +// task.addThirdPrice() // task.saveToEs() // task.saveToEs1() // task.groupByCity() // task.selectNoSave() // task.statCount() + task.exportFzDist() } def saveToEs1(){ def lists = getAllAppDatas() @@ -75,6 +76,24 @@ class AnjukeAppCrawl { }) } + def exportFzDist(){ + def q = new BasicDBObject() + q.put("city","福州") + def dbcursor = detail.find(q).batchSize(5000) + def index = 0 + def rets = [] + rets << ["city","region","name","_id","id","sale_num","city_short","source"] + for(def d : dbcursor){ + def count = 0 + if(d.propInfo.saleNum){ + count = d.propInfo.saleNum.toInteger() + } + rets << [d.base.cityName, d.base.areaName, d.base.name, d._id.toString(),d.base.id, count, "fz","安居客"] + } + + ExcelUtils.write("D:\\数据\\福州\\福州小区.xlsx", rets, "数据") + } + def selectNoSave(){ def esData = findAllEsData() def q = new BasicDBObject() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeHouseCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeHouseCrawl.groovy index 7c6dcf4de..bd7567c8e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeHouseCrawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeHouseCrawl.groovy @@ -25,11 +25,11 @@ class AnjukeHouseCrawl { System.setProperty("http.nonProxyHosts", "localhost"); WebDriver driver = new ChromeDriver() - def href = "https://sh.58.com/hezu/50696364798384x.shtml" + def href = "https://fz.anjuke.com/prop/view/S2512363126645770" def doc = getDocument(href, driver) // def doc = ClaABuYun.getResponse(href) // def newUrl = href.substring(0, hrefs.indexOf("?")) - sendMq(href, doc, "rent_58") + sendMq(href, doc, "sell_anjuke") // for(def i=25;i<50;i++){ // def url = "https://fz.anjuke.com/sale/jinana/p${i}/" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/ZhugeFangDistCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/ZhugeFangDistCrawl.groovy index c9ca234ce..a465c984a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/ZhugeFangDistCrawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/ZhugeFangDistCrawl.groovy @@ -23,14 +23,14 @@ class ZhugeFangDistCrawl { def run(){ System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); System.setProperty("http.nonProxyHosts", "localhost"); - def url = "https://fz.esfxiaoqu.zhuge.com/pingtan/" + def url = "https://fz.esfxiaoqu.zhuge.com/changle/" WebDriver driver = new ChromeDriver() driver.manage().deleteAllCookies() getRegions(driver, url) } def getRegions(driver, url) { - for(def i=1;i<=4;i++){ + for(def i=1;i<=7;i++){ def newUrl = url+"page${i}/" println newUrl def doc = getDocument(newUrl, driver) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/ExportPaimai.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/ExportPaimai.groovy new file mode 100644 index 000000000..0cd120438 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/ExportPaimai.groovy @@ -0,0 +1,147 @@ +package com.fangjia.script.gj.paimai + +import com.fangjia.script.linxsh.util.AddressUtil +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +class ExportPaimai { + + + def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; + DBCollection paimai43_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "pai_mai", lineUrl) + + static main(args) { + def task = new ExportPaimai(); + task.run() +// task.addressStandard() +// task.addPrice() +// task.stat() + } + + def stat(){ + def file = "D:\\数据\\重庆农商行\\法拍\\宿迁法拍数据.xlsx" + def datas = ExcelUtils.read(file, 0, 1) + def index1 = 0 + + def rates1 = [] + for(def d : datas){ + if(d['P'] && d['P']=='已成交'){ + if(d['M'] && d['M']=="住宅") +// if(d['K']>=150){ +// index1++ +// def sumTradePrice = d['S'] +// def marketPrice = d['Y'] +// rates1 << sumTradePrice/marketPrice +// } + if(d['S']>2000000){ + index1++ + def sumTradePrice = d['S'] + def marketPrice = d['Y'] + rates1 << sumTradePrice/marketPrice + } + } + } + def rate = rates1.sum()/rates1.size() + println index1 +" "+ rate + } + + def addPrice(){ + ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\法拍\\宿迁法拍数据.xlsx",{d-> + if(!d['Y'] && d['Z']){ + d['Y'] = d['Z'] + } + }) + } + def addressStandard(){ + ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\法拍\\重庆法拍数据.xlsx",{d-> + def formatAddress = AddressUtil.formatAddress(d['C'], d['A']) + if(formatAddress){ + d['F'] = formatAddress + } + }) + } + + def run(){ + def datas = findPaimaiData() + println "总量:"+datas.size() + def rows = [] + def header = ["标地物地址","省","城市","区域","地址","标题提取地址","楼栋号","单元","室号","楼盘名称","建筑面积","竣工日期","房屋用途","总楼层", + "拍卖状态","是否成交","拍卖阶段","拍卖方式","成交价","拍卖开始时间","拍卖结束时间","起拍价","保证金","加价幅度","评估价","市场价", + "变卖预缴款","竞价周期","延时周期","优先购买权人","报名人数","围观人数","设置提醒人数","延迟次数","URL"] + rows << header + def set = new HashSet() + def index = 0 + for(def d : datas){ + if(!set.contains(d.title)){ + index++ + println index + set << d.title + def row = [] + row << d.title + row << d.province + row << d.city + row << d.region + row << d.address + row << "" +// def formatAddress = AddressUtil.formatAddress(d.city, d.title) +// if(formatAddress){ +// row << formatAddress +// }else{ +// row << "" +// } + row << d.houseNum + row << d.unitNum + row << d.roomNum + row << d.matchName + row << d.area + row << d.buildYear + row << d.estateType + row << "" + row << "" + row << d.status + row << d.stage + row << "" + row << d.tradePrice + row << DateUtils.formatDateByPattern(d.startDate, "yyyy-MM-dd") + row << DateUtils.formatDateByPattern(d.finishDate, "yyyy-MM-dd") + row << d.startPrice + row << d.depositPrice + row << d.addPrice + row << d.evalatePrice + row << d.marketPrice + row << d.sellOffAdvancePay + row << d.pricePkWeekly + row << d.delayWeekly + row << d.preemptivePerson + row << d.enrollCount + row << d.lookCount + row << d.callCount + row << d.delayCount + row << d.url + + rows << row + } + } + ExcelUtils.write("D:\\数据\\重庆农商行\\法拍\\重庆法拍原数据.xlsx",rows,"数据") + } + + def findPaimaiData(){ + def pmquery = new BasicDBObject() + pmquery.put("city","重庆") + pmquery.put("startDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2019-01-01"))) + pmquery.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-06-01"))) + pmquery.put("status","已成交") + def dbcursor = paimai43_COLL.find(pmquery).sort(new BasicDBObject("finishDate",-1)) + def rets = [] + for(def db : dbcursor){ +// if(db.estateType && db.estateType=="车库"){ +// continue +// } + rets << db + } + return rets + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/PaimaiDataSend.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataSend.groovy similarity index 79% rename from projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/PaimaiDataSend.groovy rename to projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataSend.groovy index cdbcbc915..2f0d142d2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/PaimaiDataSend.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataSend.groovy @@ -1,4 +1,4 @@ -package com.fangjia.script.gj.crawl +package com.fangjia.script.gj.paimai import com.alibaba.fastjson.JSON import com.alibaba.fastjson.JSONObject @@ -24,27 +24,27 @@ DBCollection paimai43_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "p //System.out.println(str); def pmquery = new BasicDBObject() -//pmquery.put("city","重庆") -//pmquery.put("startDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2019-01-01"))) +pmquery.put("city","宿迁") +pmquery.put("startDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2019-01-01"))) //pmquery.put("status","已成交") //pmquery.put("estateType", null) -pmquery.put("url","https://susong-item.taobao.com/auction/648798773106.htm") - -//def cur = paimai43_COLL.find(pmquery) -//def urls = [] -//for(def o : cur){ -// urls << o.url -//} +pmquery.put("url","https://sf-item.taobao.com/sf_item/665574733237.htm") +def cur = paimai43_COLL.find(pmquery) +def urls = [] +for(def o : cur){ + urls << o.url +} +println "待解析:"+urls def index = 0 -//for(def url : urls){ +for(def url : urls){ def q = new BasicDBObject() -// q.put("url", url) + q.put("url", url) def dbcursor = collection.find(q) - + index++ + println index for(def d : dbcursor){ - index++ - println index + JSON json = new JSONObject() json.put("project", "paimai_taobao") json.put("url", d.url) @@ -64,8 +64,8 @@ def index = 0 json.put("result", result) byte[] packedMsg = MessageUtils.pack(json) - CommonUtils.sendMessage("result2parse", packedMsg) + CommonUtils.sendMessage("result2parse2dev", packedMsg) } -//} +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy index 4ac878217..383911899 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy @@ -14,8 +14,8 @@ class TestCrawlTotalCount { def run(){ def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"] for(def source :sources){ - def count1 = findTotalCount(source, 202204); - def count2 = findTotalCount(source, 202205); + def count1 = findTotalCount(source, 202205); + def count2 = findTotalCount(source, 202206); println source +" "+count1 +" "+count2 } }