测试
This commit is contained in:
-1
@@ -383,7 +383,6 @@ class DealPriceDataProcess {
|
||||
def sql = "delete from BIGDATA_THIRD_PARTY_PRICE_GJ where commid='${m.commid}' and price_date=${month} and community_price=${m.price}"
|
||||
sqlInstance236.executeUpdate(sql)
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+20
-1
@@ -48,13 +48,14 @@ class AnjukeAppCrawl {
|
||||
// task.run();
|
||||
// task.run2()
|
||||
|
||||
task.addThirdPrice()
|
||||
// task.addThirdPrice()
|
||||
// task.saveToEs()
|
||||
// task.saveToEs1()
|
||||
|
||||
// task.groupByCity()
|
||||
// task.selectNoSave()
|
||||
// task.statCount()
|
||||
task.exportFzDist()
|
||||
}
|
||||
def saveToEs1(){
|
||||
def lists = getAllAppDatas()
|
||||
@@ -75,6 +76,24 @@ class AnjukeAppCrawl {
|
||||
})
|
||||
}
|
||||
|
||||
def exportFzDist(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city","福州")
|
||||
def dbcursor = detail.find(q).batchSize(5000)
|
||||
def index = 0
|
||||
def rets = []
|
||||
rets << ["city","region","name","_id","id","sale_num","city_short","source"]
|
||||
for(def d : dbcursor){
|
||||
def count = 0
|
||||
if(d.propInfo.saleNum){
|
||||
count = d.propInfo.saleNum.toInteger()
|
||||
}
|
||||
rets << [d.base.cityName, d.base.areaName, d.base.name, d._id.toString(),d.base.id, count, "fz","安居客"]
|
||||
}
|
||||
|
||||
ExcelUtils.write("D:\\数据\\福州\\福州小区.xlsx", rets, "数据")
|
||||
}
|
||||
|
||||
def selectNoSave(){
|
||||
def esData = findAllEsData()
|
||||
def q = new BasicDBObject()
|
||||
|
||||
+2
-2
@@ -25,11 +25,11 @@ class AnjukeHouseCrawl {
|
||||
System.setProperty("http.nonProxyHosts", "localhost");
|
||||
WebDriver driver = new ChromeDriver()
|
||||
|
||||
def href = "https://sh.58.com/hezu/50696364798384x.shtml"
|
||||
def href = "https://fz.anjuke.com/prop/view/S2512363126645770"
|
||||
def doc = getDocument(href, driver)
|
||||
// def doc = ClaABuYun.getResponse(href)
|
||||
// def newUrl = href.substring(0, hrefs.indexOf("?"))
|
||||
sendMq(href, doc, "rent_58")
|
||||
sendMq(href, doc, "sell_anjuke")
|
||||
|
||||
// for(def i=25;i<50;i++){
|
||||
// def url = "https://fz.anjuke.com/sale/jinana/p${i}/"
|
||||
|
||||
+2
-2
@@ -23,14 +23,14 @@ class ZhugeFangDistCrawl {
|
||||
def run(){
|
||||
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
|
||||
System.setProperty("http.nonProxyHosts", "localhost");
|
||||
def url = "https://fz.esfxiaoqu.zhuge.com/pingtan/"
|
||||
def url = "https://fz.esfxiaoqu.zhuge.com/changle/"
|
||||
WebDriver driver = new ChromeDriver()
|
||||
driver.manage().deleteAllCookies()
|
||||
getRegions(driver, url)
|
||||
}
|
||||
|
||||
def getRegions(driver, url) {
|
||||
for(def i=1;i<=4;i++){
|
||||
for(def i=1;i<=7;i++){
|
||||
def newUrl = url+"page${i}/"
|
||||
println newUrl
|
||||
def doc = getDocument(newUrl, driver)
|
||||
|
||||
+147
@@ -0,0 +1,147 @@
|
||||
package com.fangjia.script.gj.paimai
|
||||
|
||||
import com.fangjia.script.linxsh.util.AddressUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
class ExportPaimai {
|
||||
|
||||
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection paimai43_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "pai_mai", lineUrl)
|
||||
|
||||
static main(args) {
|
||||
def task = new ExportPaimai();
|
||||
task.run()
|
||||
// task.addressStandard()
|
||||
// task.addPrice()
|
||||
// task.stat()
|
||||
}
|
||||
|
||||
def stat(){
|
||||
def file = "D:\\数据\\重庆农商行\\法拍\\宿迁法拍数据.xlsx"
|
||||
def datas = ExcelUtils.read(file, 0, 1)
|
||||
def index1 = 0
|
||||
|
||||
def rates1 = []
|
||||
for(def d : datas){
|
||||
if(d['P'] && d['P']=='已成交'){
|
||||
if(d['M'] && d['M']=="住宅")
|
||||
// if(d['K']>=150){
|
||||
// index1++
|
||||
// def sumTradePrice = d['S']
|
||||
// def marketPrice = d['Y']
|
||||
// rates1 << sumTradePrice/marketPrice
|
||||
// }
|
||||
if(d['S']>2000000){
|
||||
index1++
|
||||
def sumTradePrice = d['S']
|
||||
def marketPrice = d['Y']
|
||||
rates1 << sumTradePrice/marketPrice
|
||||
}
|
||||
}
|
||||
}
|
||||
def rate = rates1.sum()/rates1.size()
|
||||
println index1 +" "+ rate
|
||||
}
|
||||
|
||||
def addPrice(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\法拍\\宿迁法拍数据.xlsx",{d->
|
||||
if(!d['Y'] && d['Z']){
|
||||
d['Y'] = d['Z']
|
||||
}
|
||||
})
|
||||
}
|
||||
def addressStandard(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\法拍\\重庆法拍数据.xlsx",{d->
|
||||
def formatAddress = AddressUtil.formatAddress(d['C'], d['A'])
|
||||
if(formatAddress){
|
||||
d['F'] = formatAddress
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def run(){
|
||||
def datas = findPaimaiData()
|
||||
println "总量:"+datas.size()
|
||||
def rows = []
|
||||
def header = ["标地物地址","省","城市","区域","地址","标题提取地址","楼栋号","单元","室号","楼盘名称","建筑面积","竣工日期","房屋用途","总楼层",
|
||||
"拍卖状态","是否成交","拍卖阶段","拍卖方式","成交价","拍卖开始时间","拍卖结束时间","起拍价","保证金","加价幅度","评估价","市场价",
|
||||
"变卖预缴款","竞价周期","延时周期","优先购买权人","报名人数","围观人数","设置提醒人数","延迟次数","URL"]
|
||||
rows << header
|
||||
def set = new HashSet()
|
||||
def index = 0
|
||||
for(def d : datas){
|
||||
if(!set.contains(d.title)){
|
||||
index++
|
||||
println index
|
||||
set << d.title
|
||||
def row = []
|
||||
row << d.title
|
||||
row << d.province
|
||||
row << d.city
|
||||
row << d.region
|
||||
row << d.address
|
||||
row << ""
|
||||
// def formatAddress = AddressUtil.formatAddress(d.city, d.title)
|
||||
// if(formatAddress){
|
||||
// row << formatAddress
|
||||
// }else{
|
||||
// row << ""
|
||||
// }
|
||||
row << d.houseNum
|
||||
row << d.unitNum
|
||||
row << d.roomNum
|
||||
row << d.matchName
|
||||
row << d.area
|
||||
row << d.buildYear
|
||||
row << d.estateType
|
||||
row << ""
|
||||
row << ""
|
||||
row << d.status
|
||||
row << d.stage
|
||||
row << ""
|
||||
row << d.tradePrice
|
||||
row << DateUtils.formatDateByPattern(d.startDate, "yyyy-MM-dd")
|
||||
row << DateUtils.formatDateByPattern(d.finishDate, "yyyy-MM-dd")
|
||||
row << d.startPrice
|
||||
row << d.depositPrice
|
||||
row << d.addPrice
|
||||
row << d.evalatePrice
|
||||
row << d.marketPrice
|
||||
row << d.sellOffAdvancePay
|
||||
row << d.pricePkWeekly
|
||||
row << d.delayWeekly
|
||||
row << d.preemptivePerson
|
||||
row << d.enrollCount
|
||||
row << d.lookCount
|
||||
row << d.callCount
|
||||
row << d.delayCount
|
||||
row << d.url
|
||||
|
||||
rows << row
|
||||
}
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\重庆农商行\\法拍\\重庆法拍原数据.xlsx",rows,"数据")
|
||||
}
|
||||
|
||||
def findPaimaiData(){
|
||||
def pmquery = new BasicDBObject()
|
||||
pmquery.put("city","重庆")
|
||||
pmquery.put("startDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2019-01-01")))
|
||||
pmquery.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-06-01")))
|
||||
pmquery.put("status","已成交")
|
||||
def dbcursor = paimai43_COLL.find(pmquery).sort(new BasicDBObject("finishDate",-1))
|
||||
def rets = []
|
||||
for(def db : dbcursor){
|
||||
// if(db.estateType && db.estateType=="车库"){
|
||||
// continue
|
||||
// }
|
||||
rets << db
|
||||
}
|
||||
return rets
|
||||
}
|
||||
}
|
||||
+17
-17
@@ -1,4 +1,4 @@
|
||||
package com.fangjia.script.gj.crawl
|
||||
package com.fangjia.script.gj.paimai
|
||||
|
||||
import com.alibaba.fastjson.JSON
|
||||
import com.alibaba.fastjson.JSONObject
|
||||
@@ -24,27 +24,27 @@ DBCollection paimai43_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "p
|
||||
//System.out.println(str);
|
||||
|
||||
def pmquery = new BasicDBObject()
|
||||
//pmquery.put("city","ÖØÇì")
|
||||
//pmquery.put("startDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2019-01-01")))
|
||||
pmquery.put("city","宿迁")
|
||||
pmquery.put("startDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2019-01-01")))
|
||||
//pmquery.put("status","Òѳɽ»")
|
||||
//pmquery.put("estateType", null)
|
||||
pmquery.put("url","https://susong-item.taobao.com/auction/648798773106.htm")
|
||||
|
||||
//def cur = paimai43_COLL.find(pmquery)
|
||||
//def urls = []
|
||||
//for(def o : cur){
|
||||
// urls << o.url
|
||||
//}
|
||||
pmquery.put("url","https://sf-item.taobao.com/sf_item/665574733237.htm")
|
||||
|
||||
def cur = paimai43_COLL.find(pmquery)
|
||||
def urls = []
|
||||
for(def o : cur){
|
||||
urls << o.url
|
||||
}
|
||||
println "待解析:"+urls
|
||||
def index = 0
|
||||
//for(def url : urls){
|
||||
for(def url : urls){
|
||||
def q = new BasicDBObject()
|
||||
// q.put("url", url)
|
||||
q.put("url", url)
|
||||
def dbcursor = collection.find(q)
|
||||
|
||||
index++
|
||||
println index
|
||||
for(def d : dbcursor){
|
||||
index++
|
||||
println index
|
||||
|
||||
JSON json = new JSONObject()
|
||||
json.put("project", "paimai_taobao")
|
||||
json.put("url", d.url)
|
||||
@@ -64,8 +64,8 @@ def index = 0
|
||||
json.put("result", result)
|
||||
|
||||
byte[] packedMsg = MessageUtils.pack(json)
|
||||
CommonUtils.sendMessage("result2parse", packedMsg)
|
||||
CommonUtils.sendMessage("result2parse2dev", packedMsg)
|
||||
}
|
||||
//}
|
||||
}
|
||||
|
||||
|
||||
+2
-2
@@ -14,8 +14,8 @@ class TestCrawlTotalCount {
|
||||
def run(){
|
||||
def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"]
|
||||
for(def source :sources){
|
||||
def count1 = findTotalCount(source, 202204);
|
||||
def count2 = findTotalCount(source, 202205);
|
||||
def count1 = findTotalCount(source, 202205);
|
||||
def count2 = findTotalCount(source, 202206);
|
||||
println source +" "+count1 +" "+count2
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user