Merge branch 'script_gj' into 'master'
Script gj See merge request !284
This commit is contained in:
+71
-24
@@ -225,17 +225,42 @@ def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def
|
|||||||
|
|
||||||
|
|
||||||
//def fileName = "D:\\数据\\建行\\房产估值\\成交\\成交数据_1222 - 副本.xlsx"
|
//def fileName = "D:\\数据\\建行\\房产估值\\成交\\成交数据_1222 - 副本.xlsx"
|
||||||
//def dealMap = findDealCount(sqlInstance236, "2022-01-01", "2022-02-01")
|
//def dealMap = findDealCount(sqlInstance236, "2022-02-01", "2022-03-01")
|
||||||
//def testMap = findCaluateCityPPE(sqlInstance236, 202202)
|
//def testMap = findCaluateCityPPE(sqlInstance236, 202203)
|
||||||
//ExcelUtils.modify(fileName,{d->
|
//ExcelUtils.modify(fileName,{d->
|
||||||
// if(d['C']){
|
// if(d['C']){
|
||||||
// def oldcount = dealMap.get(d['C'])
|
// def oldcount = dealMap.get(d['C'])
|
||||||
// d['V'] = oldcount
|
// d['X'] = oldcount
|
||||||
// def casecount = testMap.get(d['C'])
|
// def casecount = testMap.get(d['C'])
|
||||||
// d['W'] = casecount
|
// d['Y'] = casecount
|
||||||
// println d
|
// println d
|
||||||
// }
|
// }
|
||||||
//},0)
|
//},0)
|
||||||
|
def rows = findTradePpeData(sqlInstance236)
|
||||||
|
// rows.unique()
|
||||||
|
def map=findTradeDetail(sqlInstance236,null)
|
||||||
|
def datas=[]
|
||||||
|
map.each{key,row->
|
||||||
|
if(!rows.contains(key)){
|
||||||
|
datas<<row
|
||||||
|
}
|
||||||
|
}
|
||||||
|
ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交未使用.xlsx", datas, "数据")
|
||||||
|
|
||||||
|
//def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||||
|
//def datas=[]
|
||||||
|
//cityCodeDataMap.each{city,code->
|
||||||
|
// def rows = findTradePpeData(sqlInstance236, 202203,code)
|
||||||
|
// def alltrades=findTradeDetail(sqlInstance236,code)
|
||||||
|
// alltrades.each{it->
|
||||||
|
// def commid=it[0]
|
||||||
|
// if(!rows.contains(commid)){
|
||||||
|
// datas<<it
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
// println datas.size()
|
||||||
|
//}
|
||||||
|
//ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交未使用.xlsx", datas, "数据")
|
||||||
|
|
||||||
//ExcelUtils.multiThreadingModify("D:\\数据\\小区\\5个城市小区列表0723(1).xlsx",{d->
|
//ExcelUtils.multiThreadingModify("D:\\数据\\小区\\5个城市小区列表0723(1).xlsx",{d->
|
||||||
// if(d['I'] && d['K']){
|
// if(d['I'] && d['K']){
|
||||||
@@ -270,24 +295,24 @@ def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def
|
|||||||
// }
|
// }
|
||||||
//},0)
|
//},0)
|
||||||
|
|
||||||
def index = 0
|
//def index = 0
|
||||||
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\人工核实需删除数据\\是否独有及第三方是否存在20220217\\9370300_淄博_是否独有及第三方是否存在(删除383).xlsx",{d->
|
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\人工核实需删除数据\\是否独有及第三方是否存在20220217\\9370300_淄博_是否独有及第三方是否存在(删除383).xlsx",{d->
|
||||||
def flag = d['M']
|
// def flag = d['M']
|
||||||
if(flag==0.0){
|
// if(flag==0.0){
|
||||||
if(d['D']){
|
// if(d['D']){
|
||||||
def udata = new BasicDBObject()
|
// def udata = new BasicDBObject()
|
||||||
udata.put("name_cat", 11)
|
// udata.put("name_cat", 11)
|
||||||
udata.put("m_date",new Date())
|
// udata.put("m_date",new Date())
|
||||||
|
//
|
||||||
def q = new BasicDBObject()
|
// def q = new BasicDBObject()
|
||||||
q.put("_id", new ObjectId(d['D']))
|
// q.put("_id", new ObjectId(d['D']))
|
||||||
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
|
// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
|
||||||
index++
|
// index++
|
||||||
println index
|
// println index
|
||||||
}
|
// }
|
||||||
}
|
// }
|
||||||
|
//
|
||||||
},0)
|
//},0)
|
||||||
|
|
||||||
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\补充删除数据1223_1228.xlsx",{d->
|
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\补充删除数据1223_1228.xlsx",{d->
|
||||||
// if(d['A']){
|
// if(d['A']){
|
||||||
@@ -558,8 +583,30 @@ def findCaluateCityPPE(def sqlInstance, def yearMonth){
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def findTradePpeData(def sqlInstance){
|
||||||
|
def sql = "SELECT commid,buildarea,to_char(dealdate,'yyyy-mm-dd') as dealdate,target_avg_price FROM avm_evaluation_item_gj where year_month=202203 and data_type='成交'";
|
||||||
|
def rows = new HashSet()
|
||||||
|
sqlInstance.eachRow(sql,{r->
|
||||||
|
rows<<r.commid+"##"+r.buildarea+"##"+r.target_avg_price+"##"+r.dealdate
|
||||||
|
});
|
||||||
|
return rows
|
||||||
|
}
|
||||||
|
def findTradeDetail(def sqlInstance, def cityid){
|
||||||
|
def sql = "select commid,cityname,distname,commname,buildarea,dealunitprice,to_char(dealdate,'yyyy-mm-dd') as dealdate from bigdata_deal_gj where dealdate>=to_date('2022-02-01','yyyy-MM-dd')";
|
||||||
|
def map = [:]
|
||||||
|
sqlInstance.eachRow(sql,{r->
|
||||||
|
def key=r.commid+"##"+r.buildarea+"##"+r.dealunitprice+"##"+r.dealdate
|
||||||
|
def row=[]
|
||||||
|
row<<r.commid
|
||||||
|
row<<r.cityname
|
||||||
|
row<<r.distname
|
||||||
|
row<<r.commname
|
||||||
|
row<<r.buildarea
|
||||||
|
row<<r.dealunitprice
|
||||||
|
map.put(key,row)
|
||||||
|
});
|
||||||
|
return map
|
||||||
|
}
|
||||||
def findEstateType(cityid, sqlInstance){
|
def findEstateType(cityid, sqlInstance){
|
||||||
def sql = "select count(*) as count from bigdata_commbuildtype_gj where cityid='${cityid}' and buildtype!='0199'";
|
def sql = "select count(*) as count from bigdata_commbuildtype_gj where cityid='${cityid}' and buildtype!='0199'";
|
||||||
def count = 0
|
def count = 0
|
||||||
|
|||||||
+275
@@ -0,0 +1,275 @@
|
|||||||
|
package com.fangjia.script.chongqing
|
||||||
|
|
||||||
|
import com.fangjia.script.commData.CommonDatautils
|
||||||
|
import com.fangjia.script.common.DataHandleUtils
|
||||||
|
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||||
|
import com.fangjia.script.util.CommonUtils
|
||||||
|
import com.fangjia.script.util.DateUtils
|
||||||
|
import com.fangjia.script.util.DbUtils
|
||||||
|
import com.fangjia.script.util.ElasticUtils
|
||||||
|
import com.fangjia.script.util.ExcelUtils
|
||||||
|
import com.fangjia.script.util.MongoUtils
|
||||||
|
import com.fangjia.script.util.OpenApiClient
|
||||||
|
import com.mongodb.BasicDBObject
|
||||||
|
import com.mongodb.DBCollection
|
||||||
|
import com.mongodb.DBObject
|
||||||
|
import groovy.sql.Sql
|
||||||
|
import org.apache.lucene.queryparser.classic.QueryParser
|
||||||
|
|
||||||
|
|
||||||
|
class EvaluationBatch {
|
||||||
|
|
||||||
|
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||||
|
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||||
|
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||||
|
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||||
|
|
||||||
|
static main(args) {
|
||||||
|
def task = new EvaluationBatch()
|
||||||
|
// task.run()
|
||||||
|
task.run1();
|
||||||
|
// task.run2()
|
||||||
|
}
|
||||||
|
|
||||||
|
def run2(){
|
||||||
|
def startDate = DateUtils.parseFromTightYYYYMMDD("2021-01-01")
|
||||||
|
def endDate = DateUtils.parseFromTightYYYYMMDD("2022-04-07")
|
||||||
|
def weekends = DateUtils.getWeekendBetweenDates(startDate, endDate)
|
||||||
|
ExcelUtils.modify("D:\\数据\\重庆农商行\\2022-04-02\\重庆房屋交易信息V1.xlsx", { row ->
|
||||||
|
if(row['N'] && row['N'].toString().toDouble()>=20){
|
||||||
|
println row['F'] +" "+row['N']
|
||||||
|
def dist = CommonDatautils.findDistrictById(row['C'])
|
||||||
|
for(def weekend : weekends){
|
||||||
|
CommonUtils.recalculationAverage(row['D'], row['E'], row['F'], dist.estate_type2, weekend,false,0)
|
||||||
|
}
|
||||||
|
// FangjiaClientUtils.sendDistrictPriceCalculate(row['D'], row['E'], row['F'], 48);
|
||||||
|
}
|
||||||
|
},0)
|
||||||
|
}
|
||||||
|
|
||||||
|
def run1(){
|
||||||
|
def city = "重庆"
|
||||||
|
ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\2022-04-02\\重庆房屋交易信息V2.xlsx", { row ->
|
||||||
|
def region = row['E']
|
||||||
|
def name = row['F']
|
||||||
|
def id = row['C']
|
||||||
|
def lastWeekend = DateUtils.getFirstDayOfWeek(new Date()-30)
|
||||||
|
def lastYearWeekend = DateUtils.getFirstDayOfWeek(new Date()-365)
|
||||||
|
if (region && name) {
|
||||||
|
row['H'] = CommonDatautils.findNewSqmetreRent(city, region, name)
|
||||||
|
|
||||||
|
def priceObj = CommonDatautils.findDistrictPrice(city, region, name)
|
||||||
|
if(priceObj){
|
||||||
|
row['AB'] = priceObj.avg_price
|
||||||
|
}
|
||||||
|
|
||||||
|
def dist = CommonDatautils.findDistrictById(id)
|
||||||
|
if(dist) {
|
||||||
|
List<DBObject> list = DbUtils.findAroundDistrict(dist)
|
||||||
|
def map = getArroundAvgPriceMap(list)
|
||||||
|
if(map){
|
||||||
|
row['L'] = map.avg_price
|
||||||
|
row['Q'] = map.max_val
|
||||||
|
row['R'] = map.min_val
|
||||||
|
}
|
||||||
|
def lastArroundPrice = getArroundAvgPriceByWeekend(list, lastWeekend)
|
||||||
|
if(lastArroundPrice && priceObj){
|
||||||
|
def mod = (map.avg_price-lastArroundPrice)*100/lastArroundPrice
|
||||||
|
mod = DataHandleUtils.getDoubleRound(2, mod)
|
||||||
|
row['P'] = mod
|
||||||
|
}
|
||||||
|
def lastYearArroundPrice = getArroundAvgPriceByWeekend(list, lastYearWeekend)
|
||||||
|
if(lastYearArroundPrice && priceObj){
|
||||||
|
def mod = (map.avg_price-lastYearArroundPrice)*100/lastYearArroundPrice
|
||||||
|
mod = DataHandleUtils.getDoubleRound(2, mod)
|
||||||
|
row['O'] = mod
|
||||||
|
}
|
||||||
|
}
|
||||||
|
def lastWeekendObj = CommonDatautils.queryDistrictPrice(city, region, name, lastWeekend)
|
||||||
|
if(lastWeekendObj){
|
||||||
|
row['S'] = lastWeekendObj.avg_price
|
||||||
|
if(priceObj){
|
||||||
|
def mod = (priceObj.avg_price-lastWeekendObj.avg_price)*100/lastWeekendObj.avg_price
|
||||||
|
mod = DataHandleUtils.getDoubleRound(2, mod)
|
||||||
|
row['M'] = mod
|
||||||
|
}
|
||||||
|
}
|
||||||
|
def lastYearObj = CommonDatautils.queryDistrictPrice(city, region, name, lastYearWeekend)
|
||||||
|
if(lastYearObj){
|
||||||
|
row['T'] = lastYearObj.avg_price
|
||||||
|
if(priceObj){
|
||||||
|
def mod = (priceObj.avg_price-lastYearObj.avg_price)*100/lastYearObj.avg_price
|
||||||
|
mod = DataHandleUtils.getDoubleRound(2, mod)
|
||||||
|
row['N'] = mod
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
},0)
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def run(){
|
||||||
|
def city = "重庆"
|
||||||
|
ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\2022-04-02\\重庆估值交易信息V2.xlsx", { row ->
|
||||||
|
def region = row['E']
|
||||||
|
def name = row['F']
|
||||||
|
def id = row['C']
|
||||||
|
def area = row['G'].toDouble()
|
||||||
|
if(region && name){
|
||||||
|
def priceObj = CommonDatautils.findDistrictPrice(city, region, name)
|
||||||
|
if(priceObj){
|
||||||
|
row['J'] = priceObj.avg_price
|
||||||
|
if(row['C']){
|
||||||
|
row['I'] = (area*priceObj.avg_price).toInteger()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
def dist = CommonDatautils.findDistrictById(id)
|
||||||
|
if(dist){
|
||||||
|
List<DBObject> list = DbUtils.findAroundDistrict(dist)
|
||||||
|
row['N'] = getArroundAvgPrice(list)
|
||||||
|
|
||||||
|
def alias = dist.alias
|
||||||
|
if(!alias){
|
||||||
|
alias = []
|
||||||
|
}
|
||||||
|
alias << dist.name
|
||||||
|
def houseAvgPrice = findSimAreaHousePrice(dist.city, dist.region, alias,20220101,20220401, area-10, area+10)
|
||||||
|
if(houseAvgPrice){
|
||||||
|
row['K'] = houseAvgPrice
|
||||||
|
}
|
||||||
|
|
||||||
|
def tradeAvgPrice = findTrades(dist, DateUtils.parseFromTightYYYYMMDD("2021-12-01"), area-10, area+10)
|
||||||
|
row['L'] = tradeAvgPrice
|
||||||
|
|
||||||
|
def avg_price = findAvgPrice(dist._id.toString())
|
||||||
|
if(row['J'] && avg_price){
|
||||||
|
def weekendPrice = row['J'].toString().toDouble()
|
||||||
|
def ppe15 = Math.abs((weekendPrice - avg_price) * 100 / avg_price)
|
||||||
|
row['P'] = ppe15
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
def findTrades(dist, startDate, startArea, endArea){
|
||||||
|
def q = new BasicDBObject()
|
||||||
|
q.put("city", dist.city)
|
||||||
|
q.put("region", dist.region)
|
||||||
|
// q.put("source", "链家在线")
|
||||||
|
q.put("source", new BasicDBObject('$in', ["链家在线","贝壳找房"]))
|
||||||
|
|
||||||
|
def alias = []
|
||||||
|
if(dist.alias){
|
||||||
|
if(!dist.alias.contains(dist.name)){
|
||||||
|
dist.alias << dist.name
|
||||||
|
}
|
||||||
|
alias = dist.alias
|
||||||
|
}else{
|
||||||
|
alias << dist.name
|
||||||
|
}
|
||||||
|
|
||||||
|
if(alias){
|
||||||
|
q.put("district_name", new BasicDBObject('$in', alias))
|
||||||
|
}
|
||||||
|
|
||||||
|
q.put("trade_date", new BasicDBObject('$gte', startDate))
|
||||||
|
q.put("area", new BasicDBObject('$gte', startArea).append('$lt',endArea))
|
||||||
|
def dbcorsor = DEAL_PRICE_COLL.find(q)
|
||||||
|
def prices = []
|
||||||
|
for(def d : dbcorsor){
|
||||||
|
if(d.avg_price){
|
||||||
|
prices << d.avg_price
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
def price = null
|
||||||
|
if(prices){
|
||||||
|
price = (prices.sum()/prices.size()).toInteger()
|
||||||
|
}
|
||||||
|
return price
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def getArroundAvgPriceMap(def list){
|
||||||
|
def prices = []
|
||||||
|
for(def l : list){
|
||||||
|
def priceObj = CommonDatautils.findDistrictPrice(l.city, l.region, l.name)
|
||||||
|
if(priceObj){
|
||||||
|
prices << priceObj.avg_price
|
||||||
|
}
|
||||||
|
}
|
||||||
|
prices.sort()
|
||||||
|
def map = [:]
|
||||||
|
map.min_val = prices[0]
|
||||||
|
map.max_val = prices[prices.size()-1]
|
||||||
|
def avg_price = (prices.sum()/prices.size()).toInteger()
|
||||||
|
map.avg_price = avg_price
|
||||||
|
return map;
|
||||||
|
}
|
||||||
|
|
||||||
|
def getArroundAvgPrice(def list){
|
||||||
|
def prices = []
|
||||||
|
for(def l : list){
|
||||||
|
def priceObj = CommonDatautils.findDistrictPrice(l.city, l.region, l.name)
|
||||||
|
if(priceObj){
|
||||||
|
prices << priceObj.avg_price
|
||||||
|
}
|
||||||
|
}
|
||||||
|
def avg_price = (prices.sum()/prices.size()).toInteger()
|
||||||
|
return avg_price;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def getArroundAvgPriceByWeekend(def list, def weekend){
|
||||||
|
def prices = []
|
||||||
|
for(def l : list){
|
||||||
|
def priceObj =CommonDatautils.queryDistrictPrice(l.city, l.region, l.name, weekend)
|
||||||
|
if(priceObj){
|
||||||
|
prices << priceObj.avg_price
|
||||||
|
}
|
||||||
|
}
|
||||||
|
def avg_price = null;
|
||||||
|
if(prices){
|
||||||
|
avg_price = (prices.sum()/prices.size()).toInteger()
|
||||||
|
}
|
||||||
|
return avg_price;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def findSimAreaHousePrice(def city, def region, def alias, def startDate, def endDate, def startArea, def endArea){
|
||||||
|
def aliasStr = null;
|
||||||
|
def total = 0
|
||||||
|
def prices = []
|
||||||
|
try{
|
||||||
|
aliasStr = escapeExprSpecialWord(alias.join("','"))
|
||||||
|
String sql = "select avg_price from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}') and area>=${startArea} and area<=${endArea}"
|
||||||
|
|
||||||
|
ElasticUtils.executeSqlQuery(sql,{ data->
|
||||||
|
prices << data.avg_price
|
||||||
|
})
|
||||||
|
}catch(Exception e){
|
||||||
|
e.printStackTrace()
|
||||||
|
}
|
||||||
|
|
||||||
|
def price = null
|
||||||
|
if(prices){
|
||||||
|
price = (prices.sum()/prices.size()).toInteger()
|
||||||
|
}
|
||||||
|
return price
|
||||||
|
}
|
||||||
|
|
||||||
|
public String escapeExprSpecialWord(String keyword) {
|
||||||
|
return QueryParser.escape(keyword);
|
||||||
|
}
|
||||||
|
|
||||||
|
def findAvgPrice(def commid){
|
||||||
|
def sql = "select year_month,target_avg_price from (select year_month,target_avg_price from avm_evaluation_item_gj where commid='${commid}' order by year_month desc) where rownum<2";
|
||||||
|
def avg_price = null
|
||||||
|
sqlInstance.eachRow(sql, { r ->
|
||||||
|
avg_price = r.target_avg_price
|
||||||
|
});
|
||||||
|
return avg_price
|
||||||
|
}
|
||||||
|
}
|
||||||
+84
@@ -0,0 +1,84 @@
|
|||||||
|
package com.fangjia.script.chongqing
|
||||||
|
|
||||||
|
import com.fangjia.script.commData.CommonDatautils
|
||||||
|
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
|
||||||
|
import com.fangjia.script.util.ExcelUtils
|
||||||
|
import com.fangjia.script.util.OpenApiClient
|
||||||
|
|
||||||
|
class BlankMatch {
|
||||||
|
static main(args) {
|
||||||
|
def task = new BlankMatch()
|
||||||
|
// task.run()
|
||||||
|
// task.run1()
|
||||||
|
// task.run2()
|
||||||
|
task.run3()
|
||||||
|
}
|
||||||
|
|
||||||
|
def run3(){
|
||||||
|
ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配结果V3.xlsx", { row ->
|
||||||
|
if(!row['C']){
|
||||||
|
def dist = CommonDatautils.findDistrict("重庆", row['E'], row['F'], "district")
|
||||||
|
if(dist){
|
||||||
|
row['C'] = dist._id.toString()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},0)
|
||||||
|
}
|
||||||
|
def run2(){
|
||||||
|
def datas = ExcelUtils.read("D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配不上(合并).xlsx",0,1)
|
||||||
|
ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配结果V3.xlsx", { row ->
|
||||||
|
if(!row['C']){
|
||||||
|
for(def d : datas){
|
||||||
|
if(d['B'] == row['B']){
|
||||||
|
row['C'] = d['C']
|
||||||
|
row['D'] = d['D']
|
||||||
|
row['E'] = d['E']
|
||||||
|
row['F'] = d['J']
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},0)
|
||||||
|
}
|
||||||
|
|
||||||
|
def run() {
|
||||||
|
//rMap = SeaweedDataMatchProcess.doMatch(city, region, d['G'], null);
|
||||||
|
ExcelUtils.modify("D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配结果V3.xlsx", { row ->
|
||||||
|
if(!row['C']){
|
||||||
|
def jsonObject = OpenApiClient.addressMatch("重庆", row['B'], "property", false, null, null)
|
||||||
|
if(jsonObject != null && jsonObject.precise!=-1) {
|
||||||
|
row['C'] = jsonObject.get("id")
|
||||||
|
row['D'] = jsonObject.get("city")
|
||||||
|
row['E'] = jsonObject.get("district")
|
||||||
|
row['F'] = jsonObject.get("name")
|
||||||
|
row['G'] = jsonObject.get("address")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
def run1() {
|
||||||
|
//rMap = SeaweedDataMatchProcess.doMatch(city, region, d['G'], null);
|
||||||
|
ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配不上(合并).xlsx", { row ->
|
||||||
|
if(row['F']){
|
||||||
|
def jsonObject = OpenApiClient.addressMatch("重庆", row['F'], "property", false, row['E'], null)
|
||||||
|
if(jsonObject != null && jsonObject.precise!=-1) {
|
||||||
|
row['C'] = jsonObject.get("id")
|
||||||
|
row['D'] = jsonObject.get("city")
|
||||||
|
row['J'] = jsonObject.get("name")
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
// def jsonObject = OpenApiClient.addressMatch("重庆", row['B'], "property", false, null, null)
|
||||||
|
// if(jsonObject != null && jsonObject.precise!=-1) {
|
||||||
|
// row['C'] = jsonObject.get("id")
|
||||||
|
// row['D'] = jsonObject.get("city")
|
||||||
|
// row['E'] = jsonObject.get("district")
|
||||||
|
// row['F'] = jsonObject.get("name")
|
||||||
|
// row['G'] = jsonObject.get("address")
|
||||||
|
// }
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
+77
@@ -0,0 +1,77 @@
|
|||||||
|
package com.fangjia.script.chongqing
|
||||||
|
|
||||||
|
import com.fangjia.script.commData.CommonDatautils
|
||||||
|
import com.fangjia.script.util.DbUtils
|
||||||
|
import com.fangjia.script.util.ElasticUtils
|
||||||
|
import com.mongodb.BasicDBObject
|
||||||
|
import org.apache.lucene.queryparser.classic.QueryParser
|
||||||
|
import org.bson.types.ObjectId
|
||||||
|
|
||||||
|
class FieldAddValue {
|
||||||
|
static main(args) {
|
||||||
|
def task = new FieldAddValue()
|
||||||
|
task.run()
|
||||||
|
}
|
||||||
|
|
||||||
|
def run(){
|
||||||
|
//16991 14360
|
||||||
|
def districts = CommonDatautils.findAllDistrictByCity("ÖØÇì", "district", ["region","name","alias","is_elevator"])
|
||||||
|
def index = 0
|
||||||
|
def updateIndex = 0
|
||||||
|
for(def d : districts){
|
||||||
|
if(!d.is_elevator){
|
||||||
|
index++
|
||||||
|
println index
|
||||||
|
def alias = d.alias
|
||||||
|
if(!d.alias){
|
||||||
|
alias = []
|
||||||
|
}
|
||||||
|
alias << d.name
|
||||||
|
def maxHeight = findHouseHeight("ÖØÇì", d.region, alias, 20220101)
|
||||||
|
if(maxHeight && maxHeight>6){
|
||||||
|
updateIndex++
|
||||||
|
def udata = new BasicDBObject()
|
||||||
|
udata.put("is_elevator", true)
|
||||||
|
udata.put("m_date",new Date())
|
||||||
|
|
||||||
|
def q = new BasicDBObject()
|
||||||
|
q.put("_id", new ObjectId( d._id.toString()))
|
||||||
|
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
println index+" "+updateIndex
|
||||||
|
}
|
||||||
|
|
||||||
|
def findHouseHeight(def city, def region, def alias, def startDate){
|
||||||
|
def aliasStr = null;
|
||||||
|
def total = 0
|
||||||
|
def heights = null
|
||||||
|
try{
|
||||||
|
aliasStr = escapeExprSpecialWord(alias.join("','"))
|
||||||
|
String sql = "select max(height) as height from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and region='${region}' and district_name in ('${aliasStr}') "
|
||||||
|
|
||||||
|
ElasticUtils.executeSqlQuery(sql,{ data->
|
||||||
|
if(data.height){
|
||||||
|
heights = data.height
|
||||||
|
}
|
||||||
|
|
||||||
|
})
|
||||||
|
}catch(Exception e){
|
||||||
|
e.printStackTrace()
|
||||||
|
}
|
||||||
|
|
||||||
|
// def max_height = null
|
||||||
|
// if(heights){
|
||||||
|
// heights.sort()
|
||||||
|
// max_height = heights[heights.size()-1]
|
||||||
|
// }
|
||||||
|
return heights
|
||||||
|
}
|
||||||
|
|
||||||
|
public String escapeExprSpecialWord(String keyword) {
|
||||||
|
return QueryParser.escape(keyword);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
@@ -13,7 +13,7 @@ import org.apache.commons.lang.StringUtils
|
|||||||
|
|
||||||
class OpenApiUtils {
|
class OpenApiUtils {
|
||||||
static def matchDistrict(def city, def address, def suggestDistrict, def preciseLevelflag){
|
static def matchDistrict(def city, def address, def suggestDistrict, def preciseLevelflag){
|
||||||
address = address.replace("(","(").replace(")",")").replace(" ","")
|
address = address.replace("(","(").replace(")",")").replace(" ","").replace("#","").replace("<","").replace(">","")
|
||||||
def url = "http://open.fangjia.com/address/matchV2?city="+city+"&address="+address+"&suggestDistrict="+suggestDistrict+"&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB&category=district&__refresh_cache=1"
|
def url = "http://open.fangjia.com/address/matchV2?city="+city+"&address="+address+"&suggestDistrict="+suggestDistrict+"&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB&category=district&__refresh_cache=1"
|
||||||
if(preciseLevelflag){
|
if(preciseLevelflag){
|
||||||
url = url +"&preciseLevel=strict"
|
url = url +"&preciseLevel=strict"
|
||||||
|
|||||||
+1
-1
@@ -48,7 +48,7 @@ class VagueAddressLocationBatch extends BaseBatch {
|
|||||||
void process(Object args) {
|
void process(Object args) {
|
||||||
//"北京","常州",
|
//"北京","常州",
|
||||||
// def citys =["无锡","重庆","东莞","中山","合肥","苏州","广州","深圳","佛山","长沙","韶关","惠州","清远","珠海","福州","厦门","龙岩","漳州","成都","天津","南京","青岛"] //CommonDatautils.findAllCity();
|
// def citys =["无锡","重庆","东莞","中山","合肥","苏州","广州","深圳","佛山","长沙","韶关","惠州","清远","珠海","福州","厦门","龙岩","漳州","成都","天津","南京","青岛"] //CommonDatautils.findAllCity();
|
||||||
def citys =["北京"] //CommonDatautils.findAllCity();
|
def citys =["重庆"] //CommonDatautils.findAllCity();
|
||||||
//,"济南","青岛","烟台","威海","潍坊","淄博","临沂","泰安","菏泽","东营","德州","济宁","日照","聊城","滨州","枣庄"
|
//,"济南","青岛","烟台","威海","潍坊","淄博","临沂","泰安","菏泽","东营","德州","济宁","日照","聊城","滨州","枣庄"
|
||||||
//市谊大厦 华三新村 阳光家园
|
//市谊大厦 华三新村 阳光家园
|
||||||
for (def city : citys) {
|
for (def city : citys) {
|
||||||
|
|||||||
+77
-52
@@ -55,7 +55,7 @@ class AddDistrictFieldValue {
|
|||||||
// }
|
// }
|
||||||
// })
|
// })
|
||||||
|
|
||||||
// def citys = ["沈阳","哈尔滨","吉林","遵义","淄博","西宁","青岛","大连","宁波"]//CcbCommonUtils.citys;
|
def citys = ["重庆"]//CcbCommonUtils.citys;
|
||||||
//第三方均价数据匹配
|
//第三方均价数据匹配
|
||||||
// for(def city : citys){
|
// for(def city : citys){
|
||||||
// def rets = findAllDataByCity(city)
|
// def rets = findAllDataByCity(city)
|
||||||
@@ -63,11 +63,11 @@ class AddDistrictFieldValue {
|
|||||||
// }
|
// }
|
||||||
|
|
||||||
//第三方均价数据填补
|
//第三方均价数据填补
|
||||||
// for(def city : citys){
|
for(def city : citys){
|
||||||
//// def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","name_cat"])
|
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","name_cat"])
|
||||||
// def rMap = getMatchDistrict(city)
|
def rMap = getMatchDistrict(city)
|
||||||
// createResultValue(dists, rMap)
|
createResultValue(dists, rMap)
|
||||||
// }
|
}
|
||||||
|
|
||||||
// for(def city : citys){
|
// for(def city : citys){
|
||||||
// def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","name_cat","open_date","build_date","live_date", "build_type"])
|
// def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","name_cat","open_date","build_date","live_date", "build_type"])
|
||||||
@@ -103,52 +103,52 @@ class AddDistrictFieldValue {
|
|||||||
// }
|
// }
|
||||||
|
|
||||||
|
|
||||||
def citys = ["上海"]
|
// def citys = ["上海"]
|
||||||
def updateIndex = 0
|
// def updateIndex = 0
|
||||||
def rets = []
|
// def rets = []
|
||||||
rets << ["编号", "城市", "区域", "名称","户数", "房价网楼栋数","建行楼栋数","来源"]
|
// rets << ["编号", "城市", "区域", "名称","户数", "房价网楼栋数","建行楼栋数","来源"]
|
||||||
for(def city : citys){
|
// for(def city : citys){
|
||||||
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "household_count","loudong_count", "alias"])
|
// def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "household_count","loudong_count", "alias"])
|
||||||
def total = dists.size()
|
// def total = dists.size()
|
||||||
|
//
|
||||||
for(def dist : dists){
|
// for(def dist : dists){
|
||||||
if(dist.household_count){
|
// if(dist.household_count){
|
||||||
def ret = []
|
// def ret = []
|
||||||
ret << dist._id.toString()
|
// ret << dist._id.toString()
|
||||||
ret << dist.city
|
// ret << dist.city
|
||||||
ret << dist.region
|
// ret << dist.region
|
||||||
ret << dist.name
|
// ret << dist.name
|
||||||
ret << dist.household_count
|
// ret << dist.household_count
|
||||||
ret << dist.loudong_count
|
// ret << dist.loudong_count
|
||||||
def details = findDistrictDetail(dist)
|
// def details = findDistrictDetail(dist)
|
||||||
if(details){
|
// if(details){
|
||||||
for(def d : details){
|
// for(def d : details){
|
||||||
def region = RegionData.formatRegion(city, d.region)
|
// def region = RegionData.formatRegion(city, d.region)
|
||||||
if(d.region == region){
|
// if(d.region == region){
|
||||||
if(dist.household_count && dist.household_count==d.house_count){
|
// if(dist.household_count && dist.household_count==d.house_count){
|
||||||
ret << d.loudong_count
|
// ret << d.loudong_count
|
||||||
ret << "链家"
|
// ret << "链家"
|
||||||
// updateIndex++
|
//// updateIndex++
|
||||||
// def udata = new BasicDBObject()
|
//// def udata = new BasicDBObject()
|
||||||
// udata.put("household_count", d.house_count.toInteger())
|
//// udata.put("household_count", d.house_count.toInteger())
|
||||||
// udata.put("m_date", new Date())
|
//// udata.put("m_date", new Date())
|
||||||
// def q = new BasicDBObject()
|
//// def q = new BasicDBObject()
|
||||||
// q.put("_id", dist._id)
|
//// q.put("_id", dist._id)
|
||||||
// println updateIndex +"/${total} "+d.city +" "+d.name+" old-vale:"+dist.household_count+" new-value:"+d.house_count
|
//// println updateIndex +"/${total} "+d.city +" "+d.name+" old-vale:"+dist.household_count+" new-value:"+d.house_count
|
||||||
// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
|
//// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
|
||||||
}
|
// }
|
||||||
}
|
// }
|
||||||
}
|
// }
|
||||||
}else{
|
// }else{
|
||||||
ret << ""
|
// ret << ""
|
||||||
ret << ""
|
// ret << ""
|
||||||
}
|
// }
|
||||||
|
//
|
||||||
rets << ret
|
// rets << ret
|
||||||
}
|
// }
|
||||||
}
|
// }
|
||||||
ExcelUtils.write("d:\\数据\\上海楼盘户数.xlsx", rets, "数据")
|
// ExcelUtils.write("d:\\数据\\上海楼盘户数.xlsx", rets, "数据")
|
||||||
}
|
// }
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -303,6 +303,14 @@ class AddDistrictFieldValue {
|
|||||||
def createResultValue(dists, rMap){
|
def createResultValue(dists, rMap){
|
||||||
def updateIndex = 0
|
def updateIndex = 0
|
||||||
def total = dists.size()
|
def total = dists.size()
|
||||||
|
def addressIndex = 0
|
||||||
|
def allAreaIndex = 0
|
||||||
|
def buildAreaIndex = 0
|
||||||
|
def virescence_percent_index = 0
|
||||||
|
def cubage_percent_index = 0
|
||||||
|
def finish_building_date_index = 0
|
||||||
|
def household_count_index = 0
|
||||||
|
def car_site_count_index = 0
|
||||||
for(def d : dists){
|
for(def d : dists){
|
||||||
def key = d.city +"#"+d.region+"#"+d.name
|
def key = d.city +"#"+d.region+"#"+d.name
|
||||||
def sourceMap = rMap.get(key)
|
def sourceMap = rMap.get(key)
|
||||||
@@ -311,12 +319,14 @@ class AddDistrictFieldValue {
|
|||||||
if(!d.address){
|
if(!d.address){
|
||||||
def lastValue = getValuesByField(sourceMap,"address", d.estate_type2)
|
def lastValue = getValuesByField(sourceMap,"address", d.estate_type2)
|
||||||
if(lastValue){
|
if(lastValue){
|
||||||
|
addressIndex++
|
||||||
udata.put("address", lastValue)
|
udata.put("address", lastValue)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if(!d.all_area){
|
if(!d.all_area){
|
||||||
def lastValue = getValuesByField(sourceMap,"all_area", d.estate_type2)
|
def lastValue = getValuesByField(sourceMap,"all_area", d.estate_type2)
|
||||||
if(lastValue){
|
if(lastValue){
|
||||||
|
allAreaIndex++
|
||||||
udata.put("all_area", lastValue)
|
udata.put("all_area", lastValue)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -324,6 +334,7 @@ class AddDistrictFieldValue {
|
|||||||
if(!d.build_area){
|
if(!d.build_area){
|
||||||
def lastValue = getValuesByField(sourceMap,"build_area", d.estate_type2)
|
def lastValue = getValuesByField(sourceMap,"build_area", d.estate_type2)
|
||||||
if(lastValue){
|
if(lastValue){
|
||||||
|
buildAreaIndex++
|
||||||
udata.put("build_area", lastValue)
|
udata.put("build_area", lastValue)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -331,6 +342,7 @@ class AddDistrictFieldValue {
|
|||||||
if(!d.virescence_percent){
|
if(!d.virescence_percent){
|
||||||
def lastValue = getValuesByField(sourceMap,"greening_rate", d.estate_type2)
|
def lastValue = getValuesByField(sourceMap,"greening_rate", d.estate_type2)
|
||||||
if(lastValue){
|
if(lastValue){
|
||||||
|
virescence_percent_index ++
|
||||||
udata.put("virescence_percent", lastValue)
|
udata.put("virescence_percent", lastValue)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -338,6 +350,7 @@ class AddDistrictFieldValue {
|
|||||||
if(!d.cubage_percent){
|
if(!d.cubage_percent){
|
||||||
def lastValue = getValuesByField(sourceMap,"volumetric_rate", d.estate_type2)
|
def lastValue = getValuesByField(sourceMap,"volumetric_rate", d.estate_type2)
|
||||||
if(lastValue){
|
if(lastValue){
|
||||||
|
cubage_percent_index ++
|
||||||
udata.put("cubage_percent", lastValue)
|
udata.put("cubage_percent", lastValue)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -345,6 +358,7 @@ class AddDistrictFieldValue {
|
|||||||
if(!d.finish_building_date){
|
if(!d.finish_building_date){
|
||||||
def lastValue = getValuesByField(sourceMap,"build_date", d.estate_type2)
|
def lastValue = getValuesByField(sourceMap,"build_date", d.estate_type2)
|
||||||
if(lastValue){
|
if(lastValue){
|
||||||
|
finish_building_date_index ++
|
||||||
udata.put("finish_building_date", lastValue)
|
udata.put("finish_building_date", lastValue)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -352,6 +366,7 @@ class AddDistrictFieldValue {
|
|||||||
if(!d.household_count){
|
if(!d.household_count){
|
||||||
def lastValue = getValuesByField(sourceMap,"house_count", d.estate_type2)
|
def lastValue = getValuesByField(sourceMap,"house_count", d.estate_type2)
|
||||||
if(lastValue){
|
if(lastValue){
|
||||||
|
household_count_index ++
|
||||||
udata.put("household_count", lastValue)
|
udata.put("household_count", lastValue)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -359,6 +374,7 @@ class AddDistrictFieldValue {
|
|||||||
if(!d.car_site_count){
|
if(!d.car_site_count){
|
||||||
def lastValue = getValuesByField(sourceMap,"car_site", d.estate_type2)
|
def lastValue = getValuesByField(sourceMap,"car_site", d.estate_type2)
|
||||||
if(lastValue){
|
if(lastValue){
|
||||||
|
car_site_count_index ++
|
||||||
udata.put("car_site_count", lastValue)
|
udata.put("car_site_count", lastValue)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -373,6 +389,15 @@ class AddDistrictFieldValue {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
println "addressIndex "+ addressIndex
|
||||||
|
println "allAreaIndex "+ allAreaIndex
|
||||||
|
println "buildAreaIndex "+ buildAreaIndex
|
||||||
|
println "virescence_percent_index "+ virescence_percent_index
|
||||||
|
println "cubage_percent_index "+ cubage_percent_index
|
||||||
|
println "finish_building_date_index "+ finish_building_date_index
|
||||||
|
println "household_count_index "+ household_count_index
|
||||||
|
println "car_site_count_index "+ car_site_count_index
|
||||||
}
|
}
|
||||||
|
|
||||||
def getValuesByField(sourceMap, field, estateType){
|
def getValuesByField(sourceMap, field, estateType){
|
||||||
|
|||||||
+1
-1
@@ -17,7 +17,7 @@ class TempMatchDistrict {
|
|||||||
}
|
}
|
||||||
|
|
||||||
def stat(){
|
def stat(){
|
||||||
def citys = ["广州","深圳","佛山","东莞","中山","湛江","惠州","江门","珠海","汕头","揭阳","茂名","清远","肇庆","潮州","河源","韶关","阳江","梅州","云浮","汕尾"]
|
def citys = ["哈尔滨","齐齐哈尔","大庆","牡丹江","佳木斯","绥化","鸡西","七台河","双鸭山","鹤岗","黑河","伊春","大兴安岭"]
|
||||||
def total = 0
|
def total = 0
|
||||||
for(def city : citys){
|
for(def city : citys){
|
||||||
def count = CommonDatautils.findAllDistrictCount("district", city)
|
def count = CommonDatautils.findAllDistrictCount("district", city)
|
||||||
|
|||||||
+2
-2
@@ -38,8 +38,8 @@ class FangjiaClientUtils {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public static def sendDistrictPriceCalculate(def city, def region, def name, def count){
|
public static def sendDistrictPriceCalculate(def city, def region, def name, def count){
|
||||||
def url = "http://192.168.0.100:38080/FangJiaDataService/districtPriceCalculate?category=sell&city="+city+"&count="+count+"&name="+name+"®ion="+region+"&retainHistory=true"
|
// def url = "http://192.168.0.100:38080/FangJiaDataService/districtPriceCalculate?category=sell&city="+city+"&count="+count+"&name="+name+"®ion="+region+"&retainHistory=true"
|
||||||
// def url = "http://192.168.10.19:8080/FangJiaDataService/districtPriceCalculate?category=sell&city="+city+"&count="+count+"&name="+name+"®ion="+region+"&retainHistory=true"
|
def url = "http://192.168.10.216:8080/FangJiaDataService/districtPriceCalculate?category=sell&city="+city+"&count="+count+"&name="+name+"®ion="+region+"&retainHistory=true"
|
||||||
def content = null;
|
def content = null;
|
||||||
try{
|
try{
|
||||||
content = new HttpRequest().newGet(url, null, "utf-8")
|
content = new HttpRequest().newGet(url, null, "utf-8")
|
||||||
|
|||||||
+1
-1
@@ -560,7 +560,7 @@ class DistrictIndexBatch extends BaseBatch{
|
|||||||
return datas
|
return datas
|
||||||
}
|
}
|
||||||
|
|
||||||
def sortByDistance(d, list){
|
def sortByDistance(d, list){
|
||||||
for(def nb : list){
|
for(def nb : list){
|
||||||
def distance = GeoHelper.GetDistance(d.lat/10000000000.0, d.lng/10000000000.0, nb.lat/10000000000.0, nb.lng/10000000000.0)
|
def distance = GeoHelper.GetDistance(d.lat/10000000000.0, d.lng/10000000000.0, nb.lat/10000000000.0, nb.lng/10000000000.0)
|
||||||
nb.distance = distance
|
nb.distance = distance
|
||||||
|
|||||||
+3
-3
@@ -34,7 +34,7 @@ public class BeiKeCrawlByChrome {
|
|||||||
|
|
||||||
static main(args) {
|
static main(args) {
|
||||||
def task = new BeiKeCrawlByChrome();
|
def task = new BeiKeCrawlByChrome();
|
||||||
// task.run()
|
task.run()
|
||||||
// task.run1()
|
// task.run1()
|
||||||
task.addDealRegion();
|
task.addDealRegion();
|
||||||
}
|
}
|
||||||
@@ -110,7 +110,7 @@ public class BeiKeCrawlByChrome {
|
|||||||
def durl = e.select(".title a").attr("href")
|
def durl = e.select(".title a").attr("href")
|
||||||
def dealDate = e.select(".dealDate").text()
|
def dealDate = e.select(".dealDate").text()
|
||||||
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
|
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
|
||||||
def endDate = DateUtils.parseFromTightByPattern("2022-01-20", "yyyy-MM-dd")
|
def endDate = DateUtils.parseFromTightByPattern("2022-02-20", "yyyy-MM-dd")
|
||||||
if(dealDate.getTime()>=endDate.getTime()){
|
if(dealDate.getTime()>=endDate.getTime()){
|
||||||
detailUrls << durl
|
detailUrls << durl
|
||||||
}else{
|
}else{
|
||||||
@@ -342,7 +342,7 @@ public class BeiKeCrawlByChrome {
|
|||||||
if(cityEles){
|
if(cityEles){
|
||||||
for(def e : cityEles){
|
for(def e : cityEles){
|
||||||
def href = e.attr("href")
|
def href = e.attr("href")
|
||||||
if(!e.text().contains("南京")){
|
if(e.text().contains("福州")){
|
||||||
cityMap.put(e.text(), "https:"+href)
|
cityMap.put(e.text(), "https:"+href)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+227
-111
@@ -15,6 +15,7 @@ import org.openqa.selenium.chrome.ChromeDriver
|
|||||||
import org.openqa.selenium.support.ui.ExpectedConditions
|
import org.openqa.selenium.support.ui.ExpectedConditions
|
||||||
import org.openqa.selenium.support.ui.WebDriverWait
|
import org.openqa.selenium.support.ui.WebDriverWait
|
||||||
|
|
||||||
|
import javax.swing.JApplet
|
||||||
import java.text.DateFormat
|
import java.text.DateFormat
|
||||||
import java.util.function.Consumer
|
import java.util.function.Consumer
|
||||||
|
|
||||||
@@ -41,45 +42,137 @@ System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Loca
|
|||||||
System.setProperty("http.nonProxyHosts", "localhost");
|
System.setProperty("http.nonProxyHosts", "localhost");
|
||||||
|
|
||||||
|
|
||||||
def doc = getDocument();
|
|
||||||
def detailUrls = getDetailLinks(doc);
|
WebDriver driver = new ChromeDriver()
|
||||||
def rets = batchParse(detailUrls);
|
def proviceMap = getDocument(driver);
|
||||||
saveToDB(rets);
|
def citys = getCity(driver,proviceMap)
|
||||||
|
def detailUrls = getDetailLinks(driver, citys);
|
||||||
|
|
||||||
|
|
||||||
def getDetailLinks(doc){
|
|
||||||
def urls = []
|
def getDetailLinks(driver, cityList){
|
||||||
if(doc){
|
|
||||||
def eles = JQueryUtils.select(".lots-list .lot-li-a", doc)
|
def zxcitys = ["天津市","北京市","上海市","重庆市"]
|
||||||
if(eles){
|
for(def cityMap : cityList){
|
||||||
for(def el :eles){
|
def citys = cityMap.citys
|
||||||
def href = el.attr("href")
|
|
||||||
urls << "https://paimai.caa123.org.cn"+href
|
citys.each{city,id->
|
||||||
|
def urls = []
|
||||||
|
if(!zxcitys.contains(cityMap.provice)){
|
||||||
|
driver.get("https://paimai.caa123.org.cn/pages/lots/list.html?&province=${cityMap.provice_id}&city=${id}&status=&attribute=&term=&startTimeStamp=&endTimeStamp=&standardType=6&secondaryType=&canLoan=&isRestricted=&insuranceSupport=");
|
||||||
|
}else{
|
||||||
|
driver.get("https://paimai.caa123.org.cn/pages/lots/list.html?&province=${cityMap.provice_id}&status=&attribute=&term=&startTimeStamp=&endTimeStamp=&standardType=6&secondaryType=&canLoan=&isRestricted=&insuranceSupport=");
|
||||||
|
}
|
||||||
|
|
||||||
|
def doc = Jsoup.parse(driver.getPageSource());
|
||||||
|
def count = JQueryUtils.selectText(".count", doc)
|
||||||
|
def totalPage = getTotalPage(count, 12)
|
||||||
|
if(totalPage){
|
||||||
|
for(def i=1;i<=totalPage;i++){
|
||||||
|
doc = Jsoup.parse(driver.getPageSource());
|
||||||
|
def eles = JQueryUtils.select(".lots-list .lot-li-a", doc)
|
||||||
|
if(eles){
|
||||||
|
for(def el :eles){
|
||||||
|
def href = el.attr("href")
|
||||||
|
def map = [:]
|
||||||
|
map.province = cityMap.provice
|
||||||
|
map.city = city
|
||||||
|
map.url = "https://paimai.caa123.org.cn"+href
|
||||||
|
urls << map
|
||||||
|
}
|
||||||
|
}
|
||||||
|
try{
|
||||||
|
driver.findElement(By.className("next")).click()
|
||||||
|
}catch(Exception e){
|
||||||
|
Thread.sleep(8000)
|
||||||
|
driver.findElement(By.className("next")).click()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
println city +" "+urls.size()
|
||||||
|
def rets = batchParse(urls);
|
||||||
|
saveToDB(rets);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
println cityMap.provice +" 完成"
|
||||||
|
|
||||||
}
|
}
|
||||||
return urls;
|
|
||||||
|
//return urls;
|
||||||
}
|
}
|
||||||
|
|
||||||
def getDocument(){
|
def getTotalPage(def total, def pageSize){
|
||||||
WebDriver driver = new ChromeDriver()
|
if(total){
|
||||||
|
def mode = total.toInteger()/pageSize
|
||||||
|
if(mode > mode.toInteger()){
|
||||||
|
mode = mode.toInteger() +1
|
||||||
|
}
|
||||||
|
return mode;
|
||||||
|
}
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
def getCity(driver, proviceMap){
|
||||||
|
def rows = []
|
||||||
|
def zxcitys = ["天津市","北京市","上海市","重庆市"]
|
||||||
|
proviceMap.each {k,v->
|
||||||
|
if(!zxcitys.contains(k)){
|
||||||
|
driver.get("https://paimai.caa123.org.cn/pages/lots/list.html?&province=${v}&status=&attribute=&term=&startTimeStamp=&endTimeStamp=&standardType=6&canLoan=&isRestricted=");
|
||||||
|
def doc = Jsoup.parse(driver.getPageSource());
|
||||||
|
def cityEles = JQueryUtils.select("li:contains("+k+") .value-limited li a", doc)
|
||||||
|
def cityMap = [:]
|
||||||
|
for(def c : cityEles){
|
||||||
|
def cityid = c.attr("data-cityid")
|
||||||
|
def city = c.text()
|
||||||
|
if(!["石家庄市","唐山市","秦皇岛市","邯郸市","邢台市","保定市"].contains(city)){
|
||||||
|
cityMap.put(city, cityid)
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
def map = [:]
|
||||||
|
map.provice = k
|
||||||
|
map.provice_id = v
|
||||||
|
map.citys = cityMap
|
||||||
|
rows << map
|
||||||
|
}else{
|
||||||
|
def map = [:]
|
||||||
|
map.provice = k
|
||||||
|
map.provice_id = v
|
||||||
|
def cityMap = [:]
|
||||||
|
cityMap.put(k, v)
|
||||||
|
map.citys = cityMap
|
||||||
|
rows << map
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return rows
|
||||||
|
}
|
||||||
|
|
||||||
|
def getDocument(driver){
|
||||||
|
|
||||||
driver.get("https://paimai.caa123.org.cn/pages/lots/list.html?&province=9&status=&attribute=&term=&startTimeStamp=&endTimeStamp=&standardType=6&canLoan=&isRestricted=");
|
driver.get("https://paimai.caa123.org.cn/pages/lots/list.html?&province=9&status=&attribute=&term=&startTimeStamp=&endTimeStamp=&standardType=6&canLoan=&isRestricted=");
|
||||||
WebDriverWait wait = new WebDriverWait(driver,20)
|
WebDriverWait wait = new WebDriverWait(driver,20)
|
||||||
wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.className("lots-list"))));
|
wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.className("lots-list"))));
|
||||||
|
|
||||||
def doc = Jsoup.parse(driver.getPageSource());
|
def doc = Jsoup.parse(driver.getPageSource());
|
||||||
|
def provinceMap = [:]
|
||||||
|
if(doc){
|
||||||
return doc;
|
def proviceEles = JQueryUtils.select(".auction-locus .limited-value-list", doc)
|
||||||
|
for(def ele : proviceEles){
|
||||||
|
def provice = ele.text()
|
||||||
|
def value = ele.attr("data-proviceid")
|
||||||
|
if(value && provice!="北京市" && provice!="天津市"){
|
||||||
|
provinceMap.put(provice, value)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return provinceMap;
|
||||||
}
|
}
|
||||||
|
|
||||||
def batchParse(def urls){
|
def batchParse(def urls){
|
||||||
WebDriver driver = new ChromeDriver()
|
WebDriver driver = new ChromeDriver()
|
||||||
def ret = []
|
def ret = []
|
||||||
for(def url :urls){
|
for(def map :urls){
|
||||||
driver.get(url);
|
driver.get(map.url);
|
||||||
def doc = Jsoup.parse(driver.getPageSource());
|
def doc = Jsoup.parse(driver.getPageSource());
|
||||||
def res = parse(url, doc);
|
def res = parse(map.province, map.city, map.url, doc);
|
||||||
if(res){
|
if(res){
|
||||||
ret << res
|
ret << res
|
||||||
}
|
}
|
||||||
@@ -87,111 +180,126 @@ def batchParse(def urls){
|
|||||||
return ret;
|
return ret;
|
||||||
}
|
}
|
||||||
|
|
||||||
def parse(url,doc){
|
def parse(province, city, url,doc){
|
||||||
if(doc){
|
if(doc){
|
||||||
def result = [:]
|
def result = [:]
|
||||||
// def address = JQueryUtils.selectText(["#pro_bid_name"], doc)
|
result.province = province
|
||||||
// result.address = address
|
result.city = city
|
||||||
if(result.address){
|
def address = JQueryUtils.selectText(["#pro_bid_name"], doc)
|
||||||
def rMap = OpenApiUtils.matchDistrict(result.address)
|
result.address = address
|
||||||
if(rMap) {
|
if(result.address && !result.address.contains("测试")) {
|
||||||
result.matchCity = rMap.matchCity
|
def rMap = OpenApiUtils.matchDistrict(city, result.address, null, true)
|
||||||
result.matchRegion = rMap.matchRegion
|
if (rMap) {
|
||||||
result.matchName= rMap.matchName
|
result.matchCity = rMap.city
|
||||||
result.matchAddress = rMap.matchAddress
|
result.matchRegion = rMap.region
|
||||||
|
result.matchName = rMap.name
|
||||||
|
result.matchAddress = rMap.address
|
||||||
|
result.houseNum = rMap.house_num
|
||||||
|
result.roomNum = rMap.room_num
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def finishDateStr = JQueryUtils.selectText(["#sf-countdown .time"], doc)
|
result.status = JQueryUtils.selectText([".bd_state"], doc)
|
||||||
if(finishDateStr) {
|
result.companyName = JQueryUtils.selectText("p:contains(拍卖企业) .companyName", doc)
|
||||||
result.finishDate = com.fangjia.script.util.DateUtils.parseFromTightByPattern(finishDateStr, "yyyy年MM月dd日 HH:mm:ss");
|
result.mobile = JQueryUtils.selectText("p:contains(联系方式) .companyTel", doc)
|
||||||
}
|
|
||||||
|
|
||||||
def curPriceStr = JQueryUtils.selectText([".colorred"], doc)
|
def finishDateStr = JQueryUtils.selectText(["#sf-countdown .time"], doc)
|
||||||
if(curPriceStr) {
|
if (finishDateStr) {
|
||||||
curPriceStr = curPriceStr.replace(",","")
|
result.finishDate = com.fangjia.script.util.DateUtils.parseFromTightByPattern(finishDateStr, "yyyy年MM月dd日 HH:mm");
|
||||||
result.tradePrice = curPriceStr.toInteger()
|
|
||||||
}
|
|
||||||
|
|
||||||
def evaluationPriceStr = JQueryUtils.selectText(["#assessPrice"], doc)
|
|
||||||
if(evaluationPriceStr) {
|
|
||||||
evaluationPriceStr = evaluationPriceStr.replace(",", "")
|
|
||||||
if(evaluationPriceStr.isInteger()) {
|
|
||||||
result.evaluationPrice = evaluationPriceStr.toInteger()
|
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
def depositPriceStr = JQueryUtils.selectText(["#cashDeposit"], doc)
|
def curPriceStr = JQueryUtils.selectText([".colorred"], doc)
|
||||||
if(depositPriceStr) {
|
if (curPriceStr) {
|
||||||
depositPriceStr = depositPriceStr.replace(",", "")
|
curPriceStr = curPriceStr.replace(",", "")
|
||||||
if(depositPriceStr.isInteger()) {
|
result.tradePrice = curPriceStr.toDouble().toInteger()
|
||||||
result.depositPrice = depositPriceStr.toInteger()
|
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
def addPriceStr = JQueryUtils.selectText(["#rateLadder"], doc)
|
def evaluationPriceStr = JQueryUtils.selectText(["#assessPrice"], doc)
|
||||||
if(addPriceStr) {
|
if (evaluationPriceStr) {
|
||||||
addPriceStr = addPriceStr.replace(",", "")
|
evaluationPriceStr = evaluationPriceStr.replace(",", "")
|
||||||
if(addPriceStr.isInteger()) {
|
if (evaluationPriceStr.isNumber()) {
|
||||||
result.addPrice = addPriceStr.toInteger()
|
result.evaluationPrice = evaluationPriceStr.toDouble().toInteger()
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
def startPriceStr = JQueryUtils.selectText(["#startPrice"], doc)
|
|
||||||
if(startPriceStr) {
|
|
||||||
startPriceStr = startPriceStr.replace(",", "").replace("元", "")
|
|
||||||
if(startPriceStr.isInteger()) {
|
|
||||||
result.startPrice = startPriceStr.toInteger()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
result.preemptivePerson= JQueryUtils.selectText(["#prior_span"], doc)
|
|
||||||
result.retain = JQueryUtils.selectText(["#retain"], doc)
|
|
||||||
|
|
||||||
def imgs =JQueryUtils.select(".sf-pic-slide img", doc)
|
|
||||||
if(imgs) {
|
|
||||||
def imgLinks = []
|
|
||||||
for(def img : imgs) {
|
|
||||||
def href = img.attr("src")
|
|
||||||
if(href.contains("http")) {
|
|
||||||
imgLinks << href
|
|
||||||
}else {
|
|
||||||
imgLinks << url.substring(0, url.indexOf(":")+1)+href
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
result.imgUrls = imgLinks
|
|
||||||
}
|
|
||||||
|
|
||||||
def desc = JQueryUtils.selectText(["#J_desc"], doc)
|
def depositPriceStr = JQueryUtils.selectText(["#cashDeposit"], doc)
|
||||||
if(!desc){
|
if (depositPriceStr) {
|
||||||
desc = JQueryUtils.selectText(["#pro_bid_name"], doc)
|
depositPriceStr = depositPriceStr.replace(",", "")
|
||||||
}
|
if (depositPriceStr.isNumber()) {
|
||||||
if(desc) {
|
result.depositPrice = depositPriceStr.toDouble().toInteger()
|
||||||
if(desc.contains("别墅")) {
|
}
|
||||||
result.address = desc.substring(0, desc.indexOf("别墅")+2)
|
|
||||||
}else if(desc.contains("号楼")) {
|
|
||||||
result.address = desc.substring(0, desc.indexOf("号楼")+2)
|
|
||||||
}else if(desc.contains("房屋")) {
|
|
||||||
result.address = desc.substring(0, desc.indexOf("房屋")+2)
|
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
def descr = JQueryUtils.selectText(["body"], doc)
|
def addPriceStr = JQueryUtils.selectText(["#rateLadder"], doc)
|
||||||
if(descr) {
|
if (addPriceStr) {
|
||||||
def area = getAreaNumber(descr, 5000);
|
addPriceStr = addPriceStr.replace(",", "")
|
||||||
if(area){
|
if (addPriceStr.isNumber()) {
|
||||||
result.area = area.toDouble()
|
result.addPrice = addPriceStr.toDouble().toInteger()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
if(result.area && result.tradePrice){
|
|
||||||
result.tradeAvgPrice = (int)(result.tradePrice/result.area)
|
|
||||||
}
|
|
||||||
|
|
||||||
result.content = DataUtil.zip(doc.toString())
|
def startPriceStr = JQueryUtils.selectText(["#startPrice"], doc)
|
||||||
result.introduce = JQueryUtils.select([".d-block table:eq(1)"], doc).toString()
|
if (startPriceStr) {
|
||||||
result.url = url
|
startPriceStr = startPriceStr.replace(",", "").replace("元", "")
|
||||||
return result;
|
if (startPriceStr.isNumber()) {
|
||||||
|
result.startPrice = startPriceStr.toDouble().toInteger()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
result.preemptivePerson = JQueryUtils.selectText(["#prior_span"], doc)
|
||||||
|
result.retain = JQueryUtils.selectText(["#retain"], doc)
|
||||||
|
result.lookCount = JQueryUtils.selectText(".look_count span", doc)
|
||||||
|
|
||||||
|
def startDateStr = JQueryUtils.selectText("span:contains(拍卖会定于)", doc)
|
||||||
|
if(startDateStr){
|
||||||
|
startDateStr = startDateStr.toString().substring(startDateStr.indexOf("定于")+2, startDateStr.indexOf("日")+1).trim()
|
||||||
|
}
|
||||||
|
|
||||||
|
result.startDate = com.fangjia.script.util.DateUtils.parseString2Date(startDateStr)
|
||||||
|
def imgs = JQueryUtils.select(".sf-pic-slide img", doc)
|
||||||
|
if (imgs) {
|
||||||
|
def imgLinks = []
|
||||||
|
for (def img : imgs) {
|
||||||
|
def href = img.attr("src")
|
||||||
|
if (href.contains("http")) {
|
||||||
|
imgLinks << href
|
||||||
|
} else {
|
||||||
|
imgLinks << url.substring(0, url.indexOf(":") + 1) + href
|
||||||
|
}
|
||||||
|
}
|
||||||
|
result.imgUrls = imgLinks
|
||||||
|
}
|
||||||
|
|
||||||
|
def desc = JQueryUtils.selectText(["#J_desc"], doc)
|
||||||
|
if (!desc) {
|
||||||
|
desc = JQueryUtils.selectText(["#pro_bid_name"], doc)
|
||||||
|
}
|
||||||
|
if (desc) {
|
||||||
|
if (desc.contains("别墅")) {
|
||||||
|
result.address = desc.substring(0, desc.indexOf("别墅") + 2)
|
||||||
|
} else if (desc.contains("号楼")) {
|
||||||
|
result.address = desc.substring(0, desc.indexOf("号楼") + 2)
|
||||||
|
} else if (desc.contains("房屋")) {
|
||||||
|
result.address = desc.substring(0, desc.indexOf("房屋") + 2)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
def descr = JQueryUtils.selectText(["body"], doc)
|
||||||
|
if (descr) {
|
||||||
|
def area = getAreaNumber(descr, 5000);
|
||||||
|
if (area) {
|
||||||
|
result.area = area.toDouble()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (result.area && result.tradePrice) {
|
||||||
|
result.tradeAvgPrice = (int) (result.tradePrice / result.area)
|
||||||
|
}
|
||||||
|
|
||||||
|
result.content = DataUtil.zip(doc.toString())
|
||||||
|
result.introduce = JQueryUtils.select([".d-block table:eq(1)"], doc).toString()
|
||||||
|
result.url = url
|
||||||
|
return result;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return null
|
return null
|
||||||
}
|
}
|
||||||
@@ -209,9 +317,11 @@ def saveToDB(rets){
|
|||||||
}
|
}
|
||||||
|
|
||||||
if(dbo){
|
if(dbo){
|
||||||
obj.remove("url")
|
obj.m_date = new Date()
|
||||||
paimaiResultColl.update(q, new BasicDBObject('$set', obj))
|
paimaiResultColl.update(q, new BasicDBObject('$set', obj))
|
||||||
}else{
|
}else{
|
||||||
|
obj.c_date = new Date()
|
||||||
|
obj.m_date = new Date()
|
||||||
paimaiResultColl.save(obj)
|
paimaiResultColl.save(obj)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -229,6 +339,12 @@ def getAreaNumber(input, maxArea) {
|
|||||||
def area = null;
|
def area = null;
|
||||||
|
|
||||||
for (def val : list) {
|
for (def val : list) {
|
||||||
|
if(val.toString().contains(" ")){
|
||||||
|
val = val.toString().substring(val.toString().indexOf(" ")+1, val.toString().size())
|
||||||
|
}
|
||||||
|
if(val.contains("室")){
|
||||||
|
val = val.toString().substring(val.toString().indexOf("室")+1, val.toString().size())
|
||||||
|
}
|
||||||
def v = val.replace("平方米", "").replace("㎡", "").replace("平", "").replace("方米", "").replace(",", "").replace("m2", "");
|
def v = val.replace("平方米", "").replace("㎡", "").replace("平", "").replace("方米", "").replace(",", "").replace("m2", "");
|
||||||
if (maxArea) {
|
if (maxArea) {
|
||||||
if (v.toDouble() < maxArea) {
|
if (v.toDouble() < maxArea) {
|
||||||
|
|||||||
+5
-4
@@ -187,7 +187,7 @@ public class LianjiaCrawlByChrome {
|
|||||||
def durl = e.select(".title a").attr("href")
|
def durl = e.select(".title a").attr("href")
|
||||||
def dealDate = e.select(".dealDate").text()
|
def dealDate = e.select(".dealDate").text()
|
||||||
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
|
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
|
||||||
def endDate = DateUtils.parseFromTightByPattern("2022-01-20", "yyyy-MM-dd")
|
def endDate = DateUtils.parseFromTightByPattern("2022-02-20", "yyyy-MM-dd")
|
||||||
if(dealDate.getTime()>=endDate.getTime()){
|
if(dealDate.getTime()>=endDate.getTime()){
|
||||||
detailUrls << durl
|
detailUrls << durl
|
||||||
}else{
|
}else{
|
||||||
@@ -218,7 +218,7 @@ public class LianjiaCrawlByChrome {
|
|||||||
def durl = e.select(".title a").attr("href")
|
def durl = e.select(".title a").attr("href")
|
||||||
def dealDate = e.select(".dealDate").text()
|
def dealDate = e.select(".dealDate").text()
|
||||||
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
|
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
|
||||||
def endDate = DateUtils.parseFromTightByPattern("2022-01-20", "yyyy-MM-dd")
|
def endDate = DateUtils.parseFromTightByPattern("2022-02-20", "yyyy-MM-dd")
|
||||||
if(dealDate.getTime()>=endDate.getTime()){
|
if(dealDate.getTime()>=endDate.getTime()){
|
||||||
detailUrls << durl
|
detailUrls << durl
|
||||||
parse(city, e)
|
parse(city, e)
|
||||||
@@ -417,11 +417,12 @@ public class LianjiaCrawlByChrome {
|
|||||||
def cityEles = JQueryUtils.select(".city_province a", cityDoc)
|
def cityEles = JQueryUtils.select(".city_province a", cityDoc)
|
||||||
def cityMap = [:]
|
def cityMap = [:]
|
||||||
if(cityEles){
|
if(cityEles){
|
||||||
for(def e : cityEles){
|
for(def e : cityEles) {
|
||||||
def href = e.attr("href")
|
def href = e.attr("href")
|
||||||
// if(!href.contains("fang.lianjia") && e.text().contains("福州")){
|
if (!href.contains("fang.lianjia") && e.text().contains("福州")) {
|
||||||
cityMap.put(e.text(), href)
|
cityMap.put(e.text(), href)
|
||||||
}
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return cityMap
|
return cityMap
|
||||||
}
|
}
|
||||||
|
|||||||
+215
-24
@@ -7,34 +7,147 @@ import com.fangjia.script.util.MongoUtils
|
|||||||
import com.mongodb.BasicDBObject
|
import com.mongodb.BasicDBObject
|
||||||
import com.mongodb.DBCollection
|
import com.mongodb.DBCollection
|
||||||
import com.mongodb.DBObject
|
import com.mongodb.DBObject
|
||||||
import org.jsoup.Jsoup;
|
import org.jsoup.Jsoup
|
||||||
import org.jsoup.nodes.Document
|
import org.openqa.selenium.WebDriver
|
||||||
|
import org.openqa.selenium.chrome.ChromeDriver
|
||||||
|
|
||||||
|
import java.text.SimpleDateFormat
|
||||||
import java.util.regex.Matcher
|
import java.util.regex.Matcher
|
||||||
import java.util.regex.Pattern
|
import java.util.regex.Pattern
|
||||||
import java.util.zip.GZIPInputStream
|
|
||||||
import java.util.zip.GZIPOutputStream
|
|
||||||
import org.apache.commons.lang.StringUtils;
|
|
||||||
import java.text.SimpleDateFormat;
|
|
||||||
|
|
||||||
class PaimaiTest {
|
class PaimaiTest {
|
||||||
|
|
||||||
|
DBCollection paimaiResultColl = MongoUtils.getCollectionByAuth20190102("fangjia", "pai_mai")
|
||||||
|
|
||||||
static main(args) {
|
static main(args) {
|
||||||
def task = new PaimaiTest();
|
def task = new PaimaiTest();
|
||||||
task.run();
|
task.run();
|
||||||
}
|
}
|
||||||
def run(){
|
def run(){
|
||||||
def url = "http://www.gpai.net/sf/item2.do?Web_Item_ID=23134"
|
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
|
||||||
def con = find(url);
|
System.setProperty("http.nonProxyHosts", "localhost");
|
||||||
parser(con, url);
|
WebDriver driver = new ChromeDriver()
|
||||||
|
def url = "https://s.gpai.net/sf/search.do?at=376"
|
||||||
|
def provinces = getProvice(url, driver)
|
||||||
|
def cityUrls = getCity(provinces, driver)
|
||||||
|
getDetailUrls(cityUrls, driver)
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
def getDetailUrls(cityUrls, driver){
|
||||||
|
for(def cMap : cityUrls){
|
||||||
|
driver.get("https:"+cMap.url)
|
||||||
|
def doc = Jsoup.parse(driver.getPageSource());
|
||||||
|
def total = JQueryUtils.selectText("span:contains(共有拍品) label", doc)
|
||||||
|
|
||||||
def parser(s, url){
|
if(total){
|
||||||
Document doc = Jsoup.parse(s);
|
def totalPage = getTotalPage(total.toInteger(), 20)
|
||||||
|
|
||||||
|
def detailDatas = []
|
||||||
|
for(def i=1;i<=totalPage;i++){
|
||||||
|
driver.get("https:"+cMap.url+"&page="+i)
|
||||||
|
doc = Jsoup.parse(driver.getPageSource());
|
||||||
|
def detailEles = JQueryUtils.select(".block-wrap:contains(共有拍品) .filt-result-list li .item-tit", doc)
|
||||||
|
for(def ele : detailEles){
|
||||||
|
def dMap = [:]
|
||||||
|
dMap.proince = cMap.proince
|
||||||
|
dMap.city = cMap.city
|
||||||
|
dMap.url = ele.select("a").attr("href")
|
||||||
|
detailDatas << dMap
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
def results = parser(detailDatas, driver, cMap.proince, cMap.city)
|
||||||
|
saveData(results)
|
||||||
|
println cMap.proince+" "+cMap.city+" "+results.size()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def getTotalPage(def total, def pageSize){
|
||||||
|
if(total){
|
||||||
|
def mode = total.toInteger()/pageSize
|
||||||
|
if(mode > mode.toInteger()){
|
||||||
|
mode = mode.toInteger() +1
|
||||||
|
}
|
||||||
|
return mode;
|
||||||
|
}
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
def getCity(datas, driver){
|
||||||
|
def citys = ["上海","北京","天津","重庆"]
|
||||||
|
def cityList = []
|
||||||
|
for(def data : datas){
|
||||||
|
driver.get("https:"+data.href)
|
||||||
|
def doc = Jsoup.parse(driver.getPageSource());
|
||||||
|
if(citys.contains(data.proince)){
|
||||||
|
def cityMap = [:]
|
||||||
|
cityMap.proince = data.proince
|
||||||
|
cityMap.city = data.proince
|
||||||
|
cityMap.url = data.href
|
||||||
|
cityList << cityMap
|
||||||
|
}else{
|
||||||
|
def links = JQueryUtils.select(".condition .s-subtab a", doc)
|
||||||
|
for(def ele : links){
|
||||||
|
if(!["合肥市","芜湖市","蚌埠市","淮南市","马鞍山市","淮北市","铜陵市"].contains(ele.text())){
|
||||||
|
def cityMap = [:]
|
||||||
|
cityMap.proince = data.proince
|
||||||
|
cityMap.city = ele.text()
|
||||||
|
cityMap.url = ele.attr("href")
|
||||||
|
cityList << cityMap
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return cityList
|
||||||
|
}
|
||||||
|
|
||||||
|
def getProvice(url, driver){
|
||||||
|
driver.get(url)
|
||||||
|
def doc = Jsoup.parse(driver.getPageSource());
|
||||||
|
def datas = []
|
||||||
|
def keys = ["上海","天津","重庆","北京","浙江","河北","山西","辽宁","吉林","江苏","福建","江西","山东","河南","湖北","湖南","广东"]
|
||||||
|
if(doc){
|
||||||
|
def eles = JQueryUtils.select(".condition a", doc)
|
||||||
|
for(def e : eles){
|
||||||
|
def data = [:]
|
||||||
|
if(!keys.contains(e.text())){
|
||||||
|
data.proince = e.text()
|
||||||
|
data.href = e.attr("href")
|
||||||
|
datas << data
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return datas
|
||||||
|
}
|
||||||
|
|
||||||
|
def parser(detailDatas, driver, proince, city){
|
||||||
|
def results = []
|
||||||
|
for(def m : detailDatas){
|
||||||
|
try{
|
||||||
|
driver.get("https:"+m.url)
|
||||||
|
Thread.sleep(3000)
|
||||||
|
def doc = Jsoup.parse(driver.getPageSource());
|
||||||
|
def obj = parsering(doc, m.url)
|
||||||
|
if(obj){
|
||||||
|
obj.proince = proince
|
||||||
|
obj.city = city
|
||||||
|
results << obj
|
||||||
|
}
|
||||||
|
}catch(Exception e){
|
||||||
|
println m.url+" 解析有问题!"
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
return results
|
||||||
|
}
|
||||||
|
|
||||||
|
def parsering(doc, url){
|
||||||
def result = [:]
|
def result = [:]
|
||||||
if(doc) {
|
if(doc) {
|
||||||
|
|
||||||
def title = JQueryUtils.selectText([".d-m-title"], doc)
|
def title = JQueryUtils.selectText([".d-m-title"], doc)
|
||||||
if(!title) {
|
if(!title) {
|
||||||
title = JQueryUtils.selectText([".DivItemName"], doc)
|
title = JQueryUtils.selectText([".DivItemName"], doc)
|
||||||
@@ -48,6 +161,9 @@ class PaimaiTest {
|
|||||||
}
|
}
|
||||||
result.title = title
|
result.title = title
|
||||||
result.status = status
|
result.status = status
|
||||||
|
if(title.contains("北京市朝阳区东三环中路7号4号楼28层3201")){
|
||||||
|
println title
|
||||||
|
}
|
||||||
|
|
||||||
if(title) {
|
if(title) {
|
||||||
def newStr =title
|
def newStr =title
|
||||||
@@ -76,7 +192,7 @@ class PaimaiTest {
|
|||||||
if(curPriceStr.contains("成交价:")) {
|
if(curPriceStr.contains("成交价:")) {
|
||||||
def tradePriceStr = JQueryUtils.selectText([".d-m-price .price-red"], doc).replace("元","")
|
def tradePriceStr = JQueryUtils.selectText([".d-m-price .price-red"], doc).replace("元","")
|
||||||
if(tradePriceStr) {
|
if(tradePriceStr) {
|
||||||
result.tradePrice = tradePriceStr.toInteger()
|
result.tradePrice = tradePriceStr.toDouble().toInteger()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -85,23 +201,28 @@ class PaimaiTest {
|
|||||||
curPriceStr = JQueryUtils.selectText(["#ItemArea .red"], doc)
|
curPriceStr = JQueryUtils.selectText(["#ItemArea .red"], doc)
|
||||||
if(curPriceStr) {
|
if(curPriceStr) {
|
||||||
curPriceStr = curPriceStr.replace(",", "").replace("元","").replace("--","")
|
curPriceStr = curPriceStr.replace(",", "").replace("元","").replace("--","")
|
||||||
result.tradePrice = curPriceStr.toInteger()
|
result.tradePrice = curPriceStr.toDouble().toInteger()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
def enrollCount = JQueryUtils.selectText([".peoples-infos span:eq(0) b"], doc)
|
def enrollCount = JQueryUtils.selectText([".peoples-infos span:contains(人报名) b"], doc)
|
||||||
if(enrollCount) {
|
if(enrollCount && enrollCount.isNumber()) {
|
||||||
result.enrollCount = enrollCount.toInteger()
|
result.enrollCount = enrollCount.toInteger()
|
||||||
}
|
}
|
||||||
|
|
||||||
def callCount = JQueryUtils.selectText([".peoples-infos span:eq(1) b"], doc)
|
def callCount = JQueryUtils.selectText([".peoples-infos span:contains(人设置提醒) b"], doc)
|
||||||
if(callCount) {
|
if(callCount && callCount.isNumber()) {
|
||||||
result.callCount = callCount.toInteger()
|
result.callCount = callCount.toInteger()
|
||||||
}
|
}
|
||||||
|
|
||||||
def lookCount = JQueryUtils.selectText([".peoples-infos span:eq(2) b"], doc)
|
def lookCount = JQueryUtils.selectText([".peoples-infos span:contains(人围观) b"], doc)
|
||||||
if(lookCount) {
|
if(lookCount && lookCount.isNumber()) {
|
||||||
result.lookCount = lookCount.toInteger()
|
try{
|
||||||
|
result.lookCount = lookCount.toInteger()
|
||||||
|
}catch(Exception e){
|
||||||
|
println lookCount
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
def finishDateStr = JQueryUtils.selectText(["p:contains(拍卖竞价确认时间:)"], doc)
|
def finishDateStr = JQueryUtils.selectText(["p:contains(拍卖竞价确认时间:)"], doc)
|
||||||
@@ -140,7 +261,7 @@ class PaimaiTest {
|
|||||||
}else if(str.contains("保证金")) {
|
}else if(str.contains("保证金")) {
|
||||||
def v = str.replace("保证金:", "").replace("元", "").replace(",", "").replace("--", "")
|
def v = str.replace("保证金:", "").replace("元", "").replace(",", "").replace("--", "")
|
||||||
if(v) {
|
if(v) {
|
||||||
result.depositPrice = v.toInteger()
|
result.depositPrice = v.toDouble().toInteger()
|
||||||
}
|
}
|
||||||
}else if(str.contains("延时周期")) {
|
}else if(str.contains("延时周期")) {
|
||||||
|
|
||||||
@@ -184,6 +305,47 @@ class PaimaiTest {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if(detailMap && !detailMap.get("拍品名称")){
|
||||||
|
detailMap = [:]
|
||||||
|
for(def tr : trs) {
|
||||||
|
|
||||||
|
if(tr.text().contains("拍品名称")){
|
||||||
|
def td1 = tr.select("td:contains(拍品名称)+td").text()
|
||||||
|
detailMap.put("拍品名称", td1)
|
||||||
|
}
|
||||||
|
|
||||||
|
if(tr.text().contains("权利来源")){
|
||||||
|
def td1 = tr.select("td:contains(权利来源)+td").text()
|
||||||
|
detailMap.put("权利来源", td1)
|
||||||
|
}
|
||||||
|
|
||||||
|
if(tr.text().contains("权证情况")){
|
||||||
|
def td1 = tr.select("td:contains(权证情况)+td").text()
|
||||||
|
detailMap.put("权证情况", td1)
|
||||||
|
}
|
||||||
|
|
||||||
|
if(tr.text().contains("拍品所有人")){
|
||||||
|
def td1 = tr.select("td:contains(拍品所有人)+td").text()
|
||||||
|
detailMap.put("拍品所有人", td1)
|
||||||
|
}
|
||||||
|
|
||||||
|
if(tr.text().contains("房屋情况")){
|
||||||
|
def td1 = tr.select("td:contains(房屋情况)+td").text()
|
||||||
|
detailMap.put("房屋情况", td1)
|
||||||
|
}
|
||||||
|
|
||||||
|
if(tr.text().contains("权利限制情况及瑕疵情况")){
|
||||||
|
def td1 = tr.select("td:contains(权利限制情况及瑕疵情况)+td").text()
|
||||||
|
detailMap.put("权利限制情况及瑕疵情况", td1)
|
||||||
|
}
|
||||||
|
|
||||||
|
if(tr.text().contains("拍品介绍")){
|
||||||
|
def td1 = tr.select("td:contains(拍品介绍)").text()
|
||||||
|
detailMap.put("拍品介绍", td1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
result.detailMap = detailMap
|
result.detailMap = detailMap
|
||||||
if(!result.address) {
|
if(!result.address) {
|
||||||
result.address = detailMap.get("拍品名称")
|
result.address = detailMap.get("拍品名称")
|
||||||
@@ -248,17 +410,18 @@ class PaimaiTest {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
result.content = DataUtil.zip(s)
|
// result.content = DataUtil.zip(s)
|
||||||
|
|
||||||
def el = JQueryUtils.select([".d-block table:eq(1)"], doc)
|
def el = JQueryUtils.select([".d-block table:eq(1)"], doc)
|
||||||
if(!el) {
|
if(!el) {
|
||||||
el = JQueryUtils.select([".d-title:contains(标的物介绍)+div"], doc)
|
el = JQueryUtils.select([".d-title:contains(标的物介绍)+div"], doc)
|
||||||
}
|
}
|
||||||
|
|
||||||
result.content = DataUtil.zip(s)
|
// result.content = DataUtil.zip(s)
|
||||||
result.introduce = JQueryUtils.select(["#J_desc table"], doc).toString()
|
result.introduce = JQueryUtils.select(["#J_desc table"], doc).toString()
|
||||||
result.url = url
|
result.url = url
|
||||||
result.mDate = new Date();
|
result.mDate = new Date();
|
||||||
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
def find(url){
|
def find(url){
|
||||||
@@ -374,4 +537,32 @@ class PaimaiTest {
|
|||||||
}
|
}
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
def saveData(def results){
|
||||||
|
|
||||||
|
for(def ret : results){
|
||||||
|
def q = new BasicDBObject()
|
||||||
|
q.put("url", ret.url)
|
||||||
|
def dbo = paimaiResultColl.findOne(q);
|
||||||
|
|
||||||
|
DBObject obj = new BasicDBObject();
|
||||||
|
ret.each{k,v->
|
||||||
|
obj.put(k,v)
|
||||||
|
}
|
||||||
|
|
||||||
|
if(dbo){
|
||||||
|
obj.m_date = new Date()
|
||||||
|
paimaiResultColl.update(q, new BasicDBObject('$set', obj))
|
||||||
|
}else{
|
||||||
|
obj.c_date = new Date()
|
||||||
|
obj.m_date = new Date()
|
||||||
|
try{
|
||||||
|
paimaiResultColl.save(obj)
|
||||||
|
}catch(Exception e){
|
||||||
|
println obj.url
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+7
-2
@@ -5,10 +5,15 @@ import com.fangjia.core.util.DataUtil
|
|||||||
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
|
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
|
||||||
import com.fangjia.script.util.ExcelUtils
|
import com.fangjia.script.util.ExcelUtils
|
||||||
import com.fangjia.script.util.OpenApiClient
|
import com.fangjia.script.util.OpenApiClient
|
||||||
|
import groovy.sql.Sql
|
||||||
|
|
||||||
|
class BlankEvaluationBatch {
|
||||||
|
|
||||||
|
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||||
|
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||||
|
|
||||||
class EvaluationBatch {
|
|
||||||
static main(args) {
|
static main(args) {
|
||||||
def task = new EvaluationBatch();
|
def task = new BlankEvaluationBatch();
|
||||||
task.run()
|
task.run()
|
||||||
}
|
}
|
||||||
def run(){
|
def run(){
|
||||||
|
|||||||
+205
@@ -0,0 +1,205 @@
|
|||||||
|
package com.fangjia.script.loudong
|
||||||
|
|
||||||
|
import com.fangjia.script.crawl.JQueryUtils
|
||||||
|
import com.fangjia.script.util.MongoUtils
|
||||||
|
import com.mongodb.BasicDBObject
|
||||||
|
import com.mongodb.DBCollection
|
||||||
|
import com.rabbitmq.client.impl.AMQImpl
|
||||||
|
import org.jsoup.Jsoup
|
||||||
|
import org.openqa.selenium.By
|
||||||
|
import org.openqa.selenium.WebDriver
|
||||||
|
import org.openqa.selenium.chrome.ChromeDriver
|
||||||
|
|
||||||
|
class CqfdcChromeCrawl {
|
||||||
|
|
||||||
|
static DBCollection loudongConn = MongoUtils.getCollectionByName("fangjia_crawl", "loudong_cqfdc", "192.168.0.235", 27018)
|
||||||
|
|
||||||
|
|
||||||
|
static main(args) {
|
||||||
|
def task = new CqfdcChromeCrawl();
|
||||||
|
task.run()
|
||||||
|
}
|
||||||
|
|
||||||
|
def run(){
|
||||||
|
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
|
||||||
|
System.setProperty("http.nonProxyHosts", "localhost");
|
||||||
|
|
||||||
|
WebDriver driver = new ChromeDriver()
|
||||||
|
def list = getListDatas(driver)
|
||||||
|
getDetails(driver, list);
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def getDetails(driver, list){
|
||||||
|
def index = 0
|
||||||
|
for(def map_old : list){
|
||||||
|
def datas = []
|
||||||
|
index++
|
||||||
|
def url = "http://cqfdc.cq315house.com/HtmlPage/ShowRooms.html?buildingid=${map_old.building_id}&block=${map_old.house_num_desc}"
|
||||||
|
// driver.get(url)
|
||||||
|
// def doc = driver.getPageSource()
|
||||||
|
def cityDoc = getDocment(url, driver)
|
||||||
|
def trs = JQueryUtils.select(".building_table tr", cityDoc)
|
||||||
|
def unit1 = null
|
||||||
|
def unit2 = null
|
||||||
|
def unit3 = null
|
||||||
|
def unit1_text = null
|
||||||
|
def unit2_text = null
|
||||||
|
def unit3_text = null
|
||||||
|
for(def i=0;i<trs.size();i++){
|
||||||
|
if(i==1){
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
def tr = trs.get(i)
|
||||||
|
def tds = tr.select("td")
|
||||||
|
for(def j=2;j< tds.size();j++){
|
||||||
|
def td = tds[j]
|
||||||
|
if(i==0){
|
||||||
|
def text = td.text()
|
||||||
|
|
||||||
|
if(!unit1){
|
||||||
|
if(text && text.contains("单元")){
|
||||||
|
unit1 = td.attr("colspan").toInteger()
|
||||||
|
unit1_text = text.replace("单元","")
|
||||||
|
}
|
||||||
|
}else if(unit1 && !unit2){
|
||||||
|
if(text && text.contains("单元")) {
|
||||||
|
unit2 = td.attr("colspan").toInteger()
|
||||||
|
unit2_text = text.replace("单元", "")
|
||||||
|
}
|
||||||
|
}else if(unit1 && !unit3){
|
||||||
|
if(text && text.contains("单元")) {
|
||||||
|
unit3 = td.attr("colspan").toInteger()
|
||||||
|
unit3_text = text.replace("单元", "")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}else{
|
||||||
|
def room_num = td.select("a font").text()
|
||||||
|
if(room_num){
|
||||||
|
def map = [:]
|
||||||
|
def a = td.select("a")
|
||||||
|
map.use_type = td.select("span:contains(用途:)").text().replace("用途:","")
|
||||||
|
map.room_type = td.select("span:contains(户型:)").text().replace("户型:","")
|
||||||
|
map.structure_type = td.select("span:contains(结构:)").text().replace("结构:","")
|
||||||
|
map.area = td.select("span:contains(建面:)").text().replace("建面:","")
|
||||||
|
map.siji_area = td.select("span:contains(套内:)").text().replace("套内:","")
|
||||||
|
map.price = td.select("span:contains(建面拟售价格:)").text().replace("建面拟售价格:","")
|
||||||
|
map.siji_price = td.select("span:contains(套内拟售价格:)").text().replace("套内拟售价格:","")
|
||||||
|
map.fitment = td.select("span:contains(装修水平:)").text().replace("装修水平:","").replace("null","")
|
||||||
|
def physical_floor = tds[0]//物理层
|
||||||
|
def nominal_floor = tds[1]//名义层
|
||||||
|
map.nominal_floor = nominal_floor.text()
|
||||||
|
map.physical_floor = physical_floor.text()
|
||||||
|
|
||||||
|
if(unit1 && (j+1 <= unit1+2)){
|
||||||
|
map.unit_num = unit1_text
|
||||||
|
}else if(unit2 && (j+1 <= unit1+unit2+2)){
|
||||||
|
map.unit_num = unit2_text
|
||||||
|
}else if(unit3 && (j+1 <= unit1+unit2+unit3+2)){
|
||||||
|
map.unit_num = unit3_text
|
||||||
|
}
|
||||||
|
map.room_num = room_num
|
||||||
|
map_old.each{k,v->
|
||||||
|
map.put(k,v)
|
||||||
|
}
|
||||||
|
datas << map
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
saveDatas(datas)
|
||||||
|
println index+" "+map_old.name+" "+map_old.house_num_desc +" "+datas.size()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
def getDocment(def url, def driver){
|
||||||
|
def stop = false
|
||||||
|
def doc = null
|
||||||
|
def error = 0
|
||||||
|
while (!stop || error>=10){
|
||||||
|
try{
|
||||||
|
driver.get(url)
|
||||||
|
def docment = driver.getPageSource()
|
||||||
|
if(docment){
|
||||||
|
doc = Jsoup.parse(docment)
|
||||||
|
stop = true;
|
||||||
|
}else{
|
||||||
|
error ++
|
||||||
|
}
|
||||||
|
}catch(Exception e){
|
||||||
|
error ++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return doc;
|
||||||
|
}
|
||||||
|
|
||||||
|
def getListDatas(driver){
|
||||||
|
driver.get("http://cqfdc.cq315house.com/HtmlPage/PresaleDetail.html")
|
||||||
|
def datas = []
|
||||||
|
for(def i=2;i<=208;i++){
|
||||||
|
def docment = Jsoup.parse(driver.getPageSource());
|
||||||
|
def trs = JQueryUtils.select("#searchbody tr", docment)
|
||||||
|
for(def tr : trs){
|
||||||
|
def tds = tr.select("td")
|
||||||
|
|
||||||
|
def hrefs = tds.get(3).select("a")
|
||||||
|
for(def href : hrefs){
|
||||||
|
def map = [:]
|
||||||
|
map.house_num_desc = href.text()
|
||||||
|
def str = href.attr("href")
|
||||||
|
def building_id= str.substring(str.lastIndexOf("['")+2, str.lastIndexOf("','"))
|
||||||
|
map.building_id = building_id
|
||||||
|
|
||||||
|
map.city = "重庆"
|
||||||
|
map.name = tds.get(0).text()
|
||||||
|
map.company = tds.get(1).text()
|
||||||
|
map.license_key = tds.get(2).text()
|
||||||
|
datas << map
|
||||||
|
}
|
||||||
|
}
|
||||||
|
println i +" "+datas.size()
|
||||||
|
try{
|
||||||
|
driver.findElement(By.id("nextpage")).click()
|
||||||
|
}catch(Exception e){
|
||||||
|
Thread.sleep(8000)
|
||||||
|
driver.findElement(By.id("nextpage")).click()
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
println "列表:"+datas.size()
|
||||||
|
return datas;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def saveDatas(def datas){
|
||||||
|
def rows = []
|
||||||
|
def keys = []
|
||||||
|
for(def d : datas){
|
||||||
|
def q = new BasicDBObject()
|
||||||
|
q.put("city", d.city)
|
||||||
|
q.put("name",d.name)
|
||||||
|
q.put("house_num_desc", d.house_num_desc)
|
||||||
|
q.put("room_num", d.room_num)
|
||||||
|
q.put("unit_num",d.unit_num)
|
||||||
|
def obj = loudongConn.findOne(q)
|
||||||
|
if(!obj){
|
||||||
|
def key = d.house_num_desc +"#"+d.unit_num+"#"+d.room_num
|
||||||
|
if(!keys.contains(key)){
|
||||||
|
def udata = new BasicDBObject()
|
||||||
|
d.each {k,v->
|
||||||
|
udata.put(k,v)
|
||||||
|
}
|
||||||
|
udata.put("c_date", new Date())
|
||||||
|
rows << udata
|
||||||
|
keys << key
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if(rows){
|
||||||
|
loudongConn.insert(rows);
|
||||||
|
println "save: "+rows.size()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
+8
-8
@@ -30,8 +30,8 @@ class HouseBookChromeCrawl {
|
|||||||
WebDriver driver = new ChromeDriver()
|
WebDriver driver = new ChromeDriver()
|
||||||
def listPageUrl = getIndexPage(driver)
|
def listPageUrl = getIndexPage(driver)
|
||||||
def keywords = getListPage(driver,"http://www.house-book.com.cn/"+listPageUrl)
|
def keywords = getListPage(driver,"http://www.house-book.com.cn/"+listPageUrl)
|
||||||
keywords.remove("城南家园")
|
// keywords.remove("城南家园")
|
||||||
keywords.remove("康居西城")
|
// keywords.remove("康居西城")
|
||||||
|
|
||||||
println city +"共 "+keywords.size()+" 个小区"
|
println city +"共 "+keywords.size()+" 个小区"
|
||||||
getDistrictIndex(driver, keywords)
|
getDistrictIndex(driver, keywords)
|
||||||
@@ -61,15 +61,15 @@ class HouseBookChromeCrawl {
|
|||||||
|
|
||||||
def getAllDist(driver, url, page){
|
def getAllDist(driver, url, page){
|
||||||
def keywords = []
|
def keywords = []
|
||||||
for(def i=1;i<=page;i++){
|
for(def i=73;i<=page;i++){
|
||||||
driver.get(url+"?selectPages="+i)
|
driver.get(url+"?selectPages="+i)
|
||||||
def doc = Jsoup.parse(driver.getPageSource())
|
def doc = Jsoup.parse(driver.getPageSource())
|
||||||
def eles = JQueryUtils.select(".title-area .title-text", doc)
|
def eles = JQueryUtils.select(".title-area .title-text", doc)
|
||||||
for(def e : eles){
|
for(def e : eles){
|
||||||
def text = e.text()
|
def text = e.text()
|
||||||
if(text=="富州北郡"){
|
// if(text=="富州北郡"){
|
||||||
println i
|
// println i
|
||||||
}
|
// }
|
||||||
keywords << text
|
keywords << text
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -206,7 +206,7 @@ class HouseBookChromeCrawl {
|
|||||||
detail.estate_type = house_type
|
detail.estate_type = house_type
|
||||||
}
|
}
|
||||||
if(detail){
|
if(detail){
|
||||||
=-09ql.floor = floor
|
detail.floor = floor
|
||||||
rows << detail
|
rows << detail
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -214,7 +214,7 @@ class HouseBookChromeCrawl {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
return rows;
|
return rows;
|
||||||
}+*-*
|
}
|
||||||
|
|
||||||
|
|
||||||
def getIndexPage(driver){
|
def getIndexPage(driver){
|
||||||
|
|||||||
+59
@@ -0,0 +1,59 @@
|
|||||||
|
package com.fangjia.script.loudong
|
||||||
|
|
||||||
|
import com.fangjia.script.crawl.JQueryUtils
|
||||||
|
import com.fangjia.script.thirdPriceCrawl.CrawalUtil
|
||||||
|
import com.fangjia.script.thirdPriceCrawl.UserAgents
|
||||||
|
import com.fangjia.script.util.ABuYunUtil
|
||||||
|
import net.sf.json.JSONObject
|
||||||
|
import org.apache.commons.lang.StringUtils
|
||||||
|
import org.jsoup.Jsoup
|
||||||
|
import org.jsoup.select.Elements
|
||||||
|
|
||||||
|
class HouseBookCrawl {
|
||||||
|
|
||||||
|
public def CHARSET = "utf-8";
|
||||||
|
|
||||||
|
static main(args) {
|
||||||
|
def task = new HouseBookCrawl()
|
||||||
|
task.run()
|
||||||
|
}
|
||||||
|
|
||||||
|
// def run(){
|
||||||
|
// goTocitypage()
|
||||||
|
// }
|
||||||
|
// def goTocitypage(){
|
||||||
|
// def url = "http://www.house-book.com.cn/mainAction/mainAction/city/%25E9%2587%258D%25E5%25BA%2586%25E5%25B8%2582.html"
|
||||||
|
// def headers = [:]
|
||||||
|
// headers.put("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8");
|
||||||
|
// headers.put("User-Agent", UserAgents.get())
|
||||||
|
// headers.put("cookCity", "%2525E9%252587%25258D%2525E5%2525BA%252586%2525E5%2525B8%252582")
|
||||||
|
// headers.put("HBJSESSIONID","afb32a63-796e-4bbc-bf39-0f8cb188079e")
|
||||||
|
// def params = [:]
|
||||||
|
// def responseMap = ABuYunUtil.doPostResponse(url, null, headers, CHARSET, false)
|
||||||
|
// def cookie = getCookie(responseMap)
|
||||||
|
// }
|
||||||
|
//
|
||||||
|
// def getCookie(responseMap) {
|
||||||
|
// def cookieAuth = null
|
||||||
|
// try {
|
||||||
|
// def response = responseMap.get("response");
|
||||||
|
// if (StringUtils.isNotEmpty(response)) {
|
||||||
|
// JSONObject reJson = JSONObject.fromObject(response)
|
||||||
|
// //登录成功获取COOKIE
|
||||||
|
// if (reJson.success) {
|
||||||
|
// responseMap.get("headerFields").each { k, v ->
|
||||||
|
// if (k && k.equals("Set-Cookie")) {
|
||||||
|
// if (v) {
|
||||||
|
// cookieAuth = v.join(';')
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
// } catch (Exception e) {
|
||||||
|
// println "goApiLogin(${responseMap})发生异常, 异常信息为:" + e.getMessage()
|
||||||
|
// }
|
||||||
|
// println("最终构造cookieAuth为:" + cookieAuth)
|
||||||
|
// return cookieAuth
|
||||||
|
// }
|
||||||
|
}
|
||||||
@@ -0,0 +1,147 @@
|
|||||||
|
package com.fangjia.script.loudong
|
||||||
|
|
||||||
|
import net.sourceforge.tess4j.ITesseract
|
||||||
|
import net.sourceforge.tess4j.Tesseract
|
||||||
|
import net.sourceforge.tess4j.util.ImageHelper
|
||||||
|
import org.apache.commons.io.FileUtils
|
||||||
|
import org.ehcache.xml.model.TimeUnit
|
||||||
|
import org.jsoup.Jsoup
|
||||||
|
import org.openqa.selenium.By
|
||||||
|
import org.openqa.selenium.OutputType
|
||||||
|
import org.openqa.selenium.Point
|
||||||
|
import org.openqa.selenium.TakesScreenshot
|
||||||
|
import org.openqa.selenium.WebDriver
|
||||||
|
import org.openqa.selenium.WebElement
|
||||||
|
import org.openqa.selenium.chrome.ChromeDriver
|
||||||
|
|
||||||
|
import javax.imageio.ImageIO
|
||||||
|
import java.awt.image.BufferedImage
|
||||||
|
|
||||||
|
public class ImageTest {
|
||||||
|
|
||||||
|
public static void main(String[] args) throws IOException {
|
||||||
|
// File imageFile = new File("D:\\gaojuan\\img\\WaterMark.gif");
|
||||||
|
// Tesseract instance = new Tesseract();
|
||||||
|
//
|
||||||
|
|
||||||
|
// //将验证码图片的内容识别为字符串
|
||||||
|
// String result = instance.doOCR(imageFile);
|
||||||
|
// println result
|
||||||
|
|
||||||
|
//1.获取网页的浏览器driver
|
||||||
|
//(1)定义url
|
||||||
|
String url="http://cqfdc.cq315house.com/HtmlPage/ShowRooms.html?buildingid=2l/xVhsMJPAZKivgt8ciOA&block=3%E5%8F%B7%E6%A5%BC#";
|
||||||
|
//(2)设置driver驱动chromedriver.exe(版本:2.39)路径(相对路径)
|
||||||
|
String path = System.getProperty("user.dir");
|
||||||
|
System.setProperty("webdriver.chrome.driver","C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
|
||||||
|
//(3)建立selenium 驱动
|
||||||
|
//WebDriver driver = new ChromeDriver(options);
|
||||||
|
WebDriver driver = new ChromeDriver();
|
||||||
|
//(4)设置窗口最大化及网页等待时间
|
||||||
|
driver.manage().window().maximize(); //窗口最大化
|
||||||
|
// driver.manage().timeouts().implicitlyWait(10, TimeUnit.SECONDS);
|
||||||
|
//(5)启动driver浏览网页
|
||||||
|
driver.get(url); //输入网址
|
||||||
|
Thread.sleep(1000);
|
||||||
|
|
||||||
|
driver.findElement(By.id("cell1-11-2")).click()
|
||||||
|
Thread.sleep(1000);
|
||||||
|
|
||||||
|
def docment = Jsoup.parse(driver.getPageSource());
|
||||||
|
println docment
|
||||||
|
|
||||||
|
WebElement ele = driver.findElement(By.id("vimg"));
|
||||||
|
|
||||||
|
File screenshot = ((TakesScreenshot)driver).getScreenshotAs(OutputType.FILE);
|
||||||
|
BufferedImage pic1 = ImageIO.read(screenshot);
|
||||||
|
Point point = ele.getLocation();
|
||||||
|
|
||||||
|
//(2)输入验证码
|
||||||
|
File srcfile = ((TakesScreenshot) driver).getScreenshotAs(OutputType.FILE);
|
||||||
|
FileUtils.copyFile(srcfile, new File("picture/source.png"));
|
||||||
|
//原图片
|
||||||
|
//String soursePicPath = "D:\\data_dev\\test\\test.png";
|
||||||
|
String soursePicPath = "picture/source.png";
|
||||||
|
File sourcePic = new File(soursePicPath);
|
||||||
|
int eleWid = ele.getSize().getWidth()
|
||||||
|
int eleHeiht = ele.getSize().getHeight()
|
||||||
|
|
||||||
|
BufferedImage pic2 = pic1.getSubimage(point.getX(), point.getY(), eleWid, eleHeiht);
|
||||||
|
ImageIO.write(pic2, "jpg", sourcePic);
|
||||||
|
|
||||||
|
//File desImage = new File("D:\\data_dev\\test\\test02.png");
|
||||||
|
File desImage = new File("picture/answer.png");
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
//加载待读取图片
|
||||||
|
//File imageFile = new File("D:\\data_dev\\test\\test02.png");
|
||||||
|
File imageFile = new File("picture/answer.png");
|
||||||
|
BufferedImage image = ImageIO.read(imageFile);
|
||||||
|
//对图片进行处理
|
||||||
|
image = convertImage(image);
|
||||||
|
//创建tess对象
|
||||||
|
ITesseract instance = new Tesseract();
|
||||||
|
//设置训练文件目录
|
||||||
|
//instance.setDatapath("E:\\workspace\\workspace_idea\\com_mth_data\\m-kuaidi100-data\\tessdata");
|
||||||
|
String path02 = System.getProperty("user.dir");
|
||||||
|
instance.setDatapath(path02 + "\\tessdata");
|
||||||
|
//设置训练语言
|
||||||
|
//instance.setLanguage("chi_sim");
|
||||||
|
instance.setLanguage("eng");
|
||||||
|
//执行转换
|
||||||
|
String s = instance.doOCR(image);
|
||||||
|
|
||||||
|
System.out.println(s);
|
||||||
|
String[] split = s.replace(" ", "").split("=|:");
|
||||||
|
String a = split[split.length - 1].trim();
|
||||||
|
char[] chars = a.toCharArray();
|
||||||
|
if (chars.length == 2) {
|
||||||
|
if (chars[0] == '7') {
|
||||||
|
chars[0] = '1';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
a = String.valueOf(chars);
|
||||||
|
|
||||||
|
System.out.println(a);
|
||||||
|
String b = s.substring(0, 1);
|
||||||
|
System.out.println(b);
|
||||||
|
String answer = String.valueOf(Integer.parseInt(a.trim()) - Integer.parseInt(b.trim()));
|
||||||
|
|
||||||
|
|
||||||
|
WebElement input2 = driver.findElement(By.xpath("//*[@id=\"ctl00_Content_Body_TextBox2\"]"));
|
||||||
|
input2.sendKeys(answer);
|
||||||
|
Thread.sleep(4000);
|
||||||
|
//(3)点击"查询"按钮
|
||||||
|
driver.findElement(By.xpath("//*[@id=\"ctl00_Content_Body_Button1\"]")).click();
|
||||||
|
|
||||||
|
String SecondtHandle = driver.getWindowHandle(); //首先得到最先的窗口 权柄
|
||||||
|
for (String winHandle1 : driver.getWindowHandles()) { //得到浏览器所有窗口的权柄为Set集合,遍历
|
||||||
|
if (!winHandle1.equals(SecondtHandle)) { //如果为 最先的窗口 权柄跳出
|
||||||
|
driver.close();
|
||||||
|
driver.switchTo().window(winHandle1); //如果不为 最先的窗口 权柄,将 新窗口的操作权柄 给 driver
|
||||||
|
System.out.println(driver.getCurrentUrl()); //打印是否为新窗口
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Thread.sleep(4000);
|
||||||
|
|
||||||
|
//4.关闭浏览器
|
||||||
|
driver.close();
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
//对图片进行处理 - 提高识别度
|
||||||
|
public static BufferedImage convertImage(BufferedImage image) throws Exception {
|
||||||
|
//按指定宽高创建一个图像副本
|
||||||
|
//image = ImageHelper.getSubImage(image, 0, 0, image.getWidth(), image.getHeight());
|
||||||
|
//图像转换成灰度的简单方法 - 黑白处理
|
||||||
|
image = ImageHelper.convertImageToGrayscale(image);
|
||||||
|
//图像缩放 - 放大n倍图像
|
||||||
|
image = ImageHelper.getScaledInstance(image, image.getWidth() * 15, image.getHeight() * 15);
|
||||||
|
return image;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
+55
-69
@@ -10,6 +10,8 @@ import com.fangjia.script.util.MongoUtils
|
|||||||
import com.mongodb.BasicDBObject
|
import com.mongodb.BasicDBObject
|
||||||
import com.mongodb.DBCollection
|
import com.mongodb.DBCollection
|
||||||
import com.mongodb.DBObject
|
import com.mongodb.DBObject
|
||||||
|
import org.apache.commons.lang.StringUtils
|
||||||
|
import org.bson.types.ObjectId
|
||||||
|
|
||||||
import java.util.concurrent.ConcurrentHashMap
|
import java.util.concurrent.ConcurrentHashMap
|
||||||
import java.util.concurrent.CountDownLatch
|
import java.util.concurrent.CountDownLatch
|
||||||
@@ -27,11 +29,13 @@ class SeaweedExport {
|
|||||||
|
|
||||||
static DBCollection cityMapConn = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "192.168.0.43", 28017)
|
static DBCollection cityMapConn = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "192.168.0.43", 28017)
|
||||||
static ExecutorService executorService = Executors.newFixedThreadPool(30)
|
static ExecutorService executorService = Executors.newFixedThreadPool(30)
|
||||||
|
static final DBCollection seaweed = MongoUtils.getCollectionByName("fangjia", "seaweed", "192.168.0.43", 28017)
|
||||||
|
|
||||||
static def map = new ConcurrentHashMap()
|
static def map = new ConcurrentHashMap()
|
||||||
|
|
||||||
static void main(String[] args) {
|
static void main(String[] args) {
|
||||||
def resultMap = new ConcurrentHashMap()
|
def resultMap = new ConcurrentHashMap()
|
||||||
List<List<Object>> data = ExcelUtils.read("F:\\数据\\重庆农商行\\测试数据汇总_2021.xlsx", 9)
|
List<List<Object>> data = ExcelUtils.read("D:\\Users\\Domi\\Documents\\知识库\\重庆农商行\\POC测试\\重庆样本匹配结果V4.xlsx")
|
||||||
CountDownLatch latch = new CountDownLatch(data.size())
|
CountDownLatch latch = new CountDownLatch(data.size())
|
||||||
for (List<Object> districtT : data) {
|
for (List<Object> districtT : data) {
|
||||||
final List<Object> district = districtT
|
final List<Object> district = districtT
|
||||||
@@ -39,72 +43,55 @@ class SeaweedExport {
|
|||||||
@Override
|
@Override
|
||||||
void run() {
|
void run() {
|
||||||
try {
|
try {
|
||||||
def city = district.get(1)
|
def no = district.get(0)
|
||||||
def district_code = district.get(2)
|
def id = district.get(2)
|
||||||
def name = district.get(3)
|
def ret = []
|
||||||
def matchedCity = district.get(22)
|
if (id != null) {
|
||||||
def matchedRegion = district.get(23)
|
DBObject query = new BasicDBObject()
|
||||||
def matchedName = district.get(24)
|
query.put("_id", new ObjectId(id))
|
||||||
def region;
|
//query.put("region", matchedRegion)
|
||||||
|
query.put("cat", "district")
|
||||||
|
query.put("visible", 0)
|
||||||
|
//query.put("name", matchedName)
|
||||||
|
|
||||||
if (map.get("district_code") != null) {
|
DBObject filter = new BasicDBObject()
|
||||||
region = map.get("district_code")
|
filter.put("_id", 1)
|
||||||
} else {
|
filter.put("developer", 1)
|
||||||
cityMapConn.find(new BasicDBObject("district_code", district_code)).each { obj ->
|
filter.put("estate_company", 1)
|
||||||
region = obj.fangjia_district
|
filter.put("cubage_percent", 1) // 容积率
|
||||||
map.put("district_code", region)
|
filter.put("virescence_percent", 1) // 绿化率
|
||||||
|
filter.put("alias", 1)
|
||||||
|
filter.put("car_site_count", 1)
|
||||||
|
filter.put("all_area", 1)
|
||||||
|
filter.put("loudong_count", 1)
|
||||||
|
filter.put("household_count", 1)
|
||||||
|
filter.put("estate_charge", 1)
|
||||||
|
filter.put("open_date", 1) // 开盘时间
|
||||||
|
filter.put("live_date", 1) // 交房时间
|
||||||
|
filter.put("elevator", 1) // 电梯
|
||||||
|
|
||||||
|
DBObject obj = seaweed.findOne(new ObjectId(id))
|
||||||
|
if (obj) {
|
||||||
|
ret[0] = obj.estate_company
|
||||||
|
ret[1] = obj.developer
|
||||||
|
ret[2] = SeaweedUtils.formatCubagePercent(obj.cubage_percent)
|
||||||
|
ret[3] = SeaweedUtils.formatVirescencePercent(obj.virescence_percent)
|
||||||
|
ret[4] = SeaweedUtils.formatAlias(obj.alias)
|
||||||
|
ret[5] = SeaweedUtils.formatCarSiteCount(obj.car_site_count)
|
||||||
|
ret[6] = SeaweedUtils.calcCarAndHouseholdRadio(obj.car_site_count, obj.household_count)
|
||||||
|
ret[7] = ""
|
||||||
|
ret[8] = SeaweedUtils.formatAllArea(obj.all_area)
|
||||||
|
ret[9] = obj.loudong_count
|
||||||
|
ret[10] = SeaweedUtils.formatHouseholdCount(obj.household_count)
|
||||||
|
ret[11] = ""
|
||||||
|
ret[12] = obj.estate_charge
|
||||||
|
ret[13] = DateUtil.formatToYYYYMMDD(obj.open_date)
|
||||||
|
ret[14] = obj.live_date
|
||||||
|
ret[15] = obj.is_elevator == true ? "有" : (obj.is_elevator == false ? "无" : "")
|
||||||
|
ret[16] = no
|
||||||
|
resultMap.put(no, ret)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (matchedRegion == null) {
|
|
||||||
matchedRegion = region
|
|
||||||
matchedName = name
|
|
||||||
}
|
|
||||||
def ret = []
|
|
||||||
DBObject query = new BasicDBObject()
|
|
||||||
query.put("city", city)
|
|
||||||
query.put("region", matchedRegion)
|
|
||||||
query.put("cat", "district")
|
|
||||||
query.put("visible", 0)
|
|
||||||
query.put("name", matchedName)
|
|
||||||
|
|
||||||
DBObject filter = new BasicDBObject()
|
|
||||||
filter.put("_id", 1)
|
|
||||||
filter.put("developer", 1)
|
|
||||||
filter.put("estate_company", 1)
|
|
||||||
filter.put("cubage_percent", 1) // 容积率
|
|
||||||
filter.put("virescence_percent", 1) // 绿化率
|
|
||||||
filter.put("alias", 1)
|
|
||||||
filter.put("car_site_count", 1)
|
|
||||||
filter.put("all_area", 1)
|
|
||||||
filter.put("loudong_count", 1)
|
|
||||||
filter.put("household_count", 1)
|
|
||||||
filter.put("estate_charge", 1)
|
|
||||||
filter.put("open_date", 1) // 开盘时间
|
|
||||||
filter.put("live_date", 1) // 交房时间
|
|
||||||
filter.put("lng", 1)
|
|
||||||
filter.put("lat", 1)
|
|
||||||
|
|
||||||
DbUtils.seaweed.find(query).limit(1).each { obj ->
|
|
||||||
ret[0] = obj.estate_company
|
|
||||||
ret[1] = obj.developer
|
|
||||||
ret[2] = SeaweedUtils.formatCubagePercent(obj.cubage_percent)
|
|
||||||
ret[3] = SeaweedUtils.formatVirescencePercent(obj.virescence_percent)
|
|
||||||
ret[4] = SeaweedUtils.formatAlias(obj.alias)
|
|
||||||
ret[5] = SeaweedUtils.formatCarSiteCount(obj.car_site_count)
|
|
||||||
ret[6] = SeaweedUtils.calcCarAndHouseholdRadio(obj.car_site_count, obj.household_count)
|
|
||||||
ret[7] = ""
|
|
||||||
ret[8] = SeaweedUtils.formatAllArea(obj.all_area)
|
|
||||||
ret[9] = obj.loudong_count
|
|
||||||
ret[10] = SeaweedUtils.formatHouseholdCount(obj.household_count)
|
|
||||||
ret[11] = ""
|
|
||||||
ret[12] = obj.estate_charge
|
|
||||||
ret[13] = DateUtil.formatToYYYYMMDD(obj.open_date)
|
|
||||||
ret[14] = obj.live_date
|
|
||||||
ret[15] = obj.lng
|
|
||||||
ret[16] = obj.lat
|
|
||||||
}
|
|
||||||
ret[17] = name
|
|
||||||
resultMap.put(district_code + name, ret)
|
|
||||||
} catch (Exception ex) {
|
} catch (Exception ex) {
|
||||||
ex.printStackTrace()
|
ex.printStackTrace()
|
||||||
} finally {
|
} finally {
|
||||||
@@ -115,13 +102,12 @@ class SeaweedExport {
|
|||||||
};
|
};
|
||||||
latch.await();
|
latch.await();
|
||||||
def ret = [];
|
def ret = [];
|
||||||
ret << ["物业公司名称","开发商名称","容积率","绿化率","楼盘别名","停车位数量","车位配比率","车位价","总占地面积","总楼栋数","总户数","非住宅总户数","物业费","开盘日期","交付时间", "经度", "纬度", "小区名称"];
|
ret << ["物业公司名称","开发商名称","容积率","绿化率","楼盘别名","停车位数量","车位配比率","车位价","总占地面积","总楼栋数","总户数","非住宅总户数","物业费","开盘日期","交付时间", "是否有电梯", "序号"];
|
||||||
for (List<Object> district : data) {
|
for (List<Object> district : data) {
|
||||||
def district_code = district.get(2)
|
def no = district.get(0)
|
||||||
def name = district.get(3)
|
ret << resultMap.get(no)
|
||||||
ret << resultMap.get(district_code + name)
|
|
||||||
};
|
};
|
||||||
com.fangjia.script.util.ExcelUtils.write("F:\\数据\\重庆农商行\\开发商及物业信息.xlsx", ret, "开发商及物业信息");
|
com.fangjia.script.util.ExcelUtils.write("D:\\Users\\Domi\\Documents\\知识库\\重庆农商行\\POC测试\\开发商及物业信息.xlsx", ret, "开发商及物业信息");
|
||||||
executorService.shutdown()
|
executorService.shutdown()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+1
-1
@@ -83,7 +83,7 @@ class SeaweedUtils {
|
|||||||
static formatCarSiteCount(def car_site_count) {
|
static formatCarSiteCount(def car_site_count) {
|
||||||
if (car_site_count && FieldValidatorUtils.checkValue(car_site_count)) {
|
if (car_site_count && FieldValidatorUtils.checkValue(car_site_count)) {
|
||||||
long val = Long.valueOf(car_site_count)
|
long val = Long.valueOf(car_site_count)
|
||||||
if (val > 2000 || val < 100) {
|
if (val > 2500 || val < 100) {
|
||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
return val;
|
return val;
|
||||||
|
|||||||
+18
-18
@@ -191,24 +191,24 @@ class Esf58Crawl extends BaseBatch{
|
|||||||
void process(Object args) {
|
void process(Object args) {
|
||||||
//addContent("福州","https://fz.58.com/xiaoqu/", 70);
|
//addContent("福州","https://fz.58.com/xiaoqu/", 70);
|
||||||
|
|
||||||
def citys = ["新余","松原","楚雄","吕梁","雅安","本溪","鹤壁","拉萨","黔东南","忻州","兴安盟","乌海","贵港","三门峡","仙桃","安康","白城","平凉","庆阳","潮州","白银","吴忠","防城港","延安","丽江","黔西南","喀什","钦州","鸡西","萍乡","石嘴山","鹰潭","保山","哈密","铜仁","武威","普洱","河池","酒泉","朔州","百色","鹤岗","张家界","定西","湘西","济源","昭通","中卫","张掖","克拉玛依","黑河","铜川","辽源","云浮","博尔塔拉","固原","白山","天门","潜江","贺州","临沧","伊春","临夏","石河子","来宾","双鸭山","嘉峪关","海东"]
|
// def citys = ["新余","松原","楚雄","吕梁","雅安","本溪","鹤壁","拉萨","黔东南","忻州","兴安盟","乌海","贵港","三门峡","仙桃","安康","白城","平凉","庆阳","潮州","白银","吴忠","防城港","延安","丽江","黔西南","喀什","钦州","鸡西","萍乡","石嘴山","鹰潭","保山","哈密","铜仁","武威","普洱","河池","酒泉","朔州","百色","鹤岗","张家界","定西","湘西","济源","昭通","中卫","张掖","克拉玛依","黑河","铜川","辽源","云浮","博尔塔拉","固原","白山","天门","潜江","贺州","临沧","伊春","临夏","石河子","来宾","双鸭山","嘉峪关","海东"]
|
||||||
for(def city : citys){
|
// for(def city : citys){
|
||||||
def str = cityList.get(city)
|
// def str = cityList.get(city)
|
||||||
for(def i=0;i<1;i++){
|
// for(def i=0;i<1;i++){
|
||||||
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]);
|
// def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]);
|
||||||
if(districts){
|
// if(districts){
|
||||||
def arrs = str.split("\\|")
|
// def arrs = str.split("\\|")
|
||||||
def url = "https://"+arrs[0]+".58.com/xiaoqu/"
|
// def url = "https://"+arrs[0]+".58.com/xiaoqu/"
|
||||||
distMap.put(city, districts)
|
// distMap.put(city, districts)
|
||||||
addCrawlKeys(city);
|
// addCrawlKeys(city);
|
||||||
addDetailContent(city, "https://"+arrs[0]+".58.com", arrs[1]);
|
// addDetailContent(city, "https://"+arrs[0]+".58.com", arrs[1]);
|
||||||
}
|
// }
|
||||||
}
|
// }
|
||||||
}
|
// }
|
||||||
|
|
||||||
/*cityList.each{city, str->
|
cityList.each{city, str->
|
||||||
for(def i=0;i<1;i++){
|
for(def i=0;i<1;i++){
|
||||||
if(citys.contains(city)){
|
// if(citys.contains(city)){
|
||||||
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]);
|
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]);
|
||||||
if(districts){
|
if(districts){
|
||||||
def arrs = str.split("\\|")
|
def arrs = str.split("\\|")
|
||||||
@@ -218,9 +218,9 @@ class Esf58Crawl extends BaseBatch{
|
|||||||
addCrawlKeys(city);
|
addCrawlKeys(city);
|
||||||
addDetailContent(city, "https://"+arrs[0]+".58.com", arrs[1]);
|
addDetailContent(city, "https://"+arrs[0]+".58.com", arrs[1]);
|
||||||
}
|
}
|
||||||
}
|
// }
|
||||||
}
|
}
|
||||||
}*/
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+3
-3
@@ -15,10 +15,10 @@ class SynchDistrictPrice {
|
|||||||
def static final SHORT_DATE = Utils.getLastMonth()
|
def static final SHORT_DATE = Utils.getLastMonth()
|
||||||
def thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
|
def thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
|
||||||
def districtDetailColl = CrawalDBUtil.getDistrictDetailColl();
|
def districtDetailColl = CrawalDBUtil.getDistrictDetailColl();
|
||||||
// def sourceMap = ["fang.com":"搜房","jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fangdd.com":"房多多","house365.com":"365淘房",
|
def sourceMap = ["fang.com":"搜房","jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fangdd.com":"房多多","house365.com":"365淘房",
|
||||||
// "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家"]
|
"anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家"]
|
||||||
//"fang.com":"搜房",
|
//"fang.com":"搜房",
|
||||||
def sourceMap = ["fang.com":"搜房"]
|
// def sourceMap = ["fang.com":"搜房"]
|
||||||
static main(args) {
|
static main(args) {
|
||||||
def task = new SynchDistrictPrice()
|
def task = new SynchDistrictPrice()
|
||||||
task.run()
|
task.run()
|
||||||
|
|||||||
+2
-2
@@ -14,8 +14,8 @@ class TestCrawlTotalCount {
|
|||||||
def run(){
|
def run(){
|
||||||
def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"]
|
def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"]
|
||||||
for(def source :sources){
|
for(def source :sources){
|
||||||
def count1 = findTotalCount(source, 202201);
|
def count1 = findTotalCount(source, 202202);
|
||||||
def count2 = findTotalCount(source, 202202);
|
def count2 = findTotalCount(source, 202203);
|
||||||
println source +" "+count1 +" "+count2
|
println source +" "+count1 +" "+count2
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,5 +1,7 @@
|
|||||||
package com.fangjia.script.util
|
package com.fangjia.script.util
|
||||||
|
|
||||||
|
import org.apache.commons.lang.StringUtils
|
||||||
|
|
||||||
import java.text.SimpleDateFormat;
|
import java.text.SimpleDateFormat;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.Calendar;
|
import java.util.Calendar;
|
||||||
@@ -995,4 +997,42 @@ class DateUtils {
|
|||||||
return rtn;
|
return rtn;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static Date parseString2Date(String str){
|
||||||
|
if(StringUtils.isBlank(str)){
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
str = str.replaceAll("^\\s+", "").replaceAll("\\s+\$", "")
|
||||||
|
|
||||||
|
Date now = new Date()
|
||||||
|
if(str.contains("小时")||str.contains("分钟") || str.contains("秒") || str.contains("当前")|| str.contains("今天")){
|
||||||
|
return org.apache.commons.lang3.time.DateUtils.truncate(new Date(), Calendar.DAY_OF_MONTH)
|
||||||
|
}else if(str.contains("昨天")) {
|
||||||
|
return add(now, -1)
|
||||||
|
}else if(str.contains("天")){
|
||||||
|
int days = Integer.parseInt(str.replaceAll("[^\\.0123456789]", ""))
|
||||||
|
return add(now, (-1)*days)
|
||||||
|
}else if(str.contains("月前")){
|
||||||
|
int month = Integer.parseInt(str.replaceAll("[^\\.0123456789]", ""))
|
||||||
|
return add(now, (-1)*month*30)
|
||||||
|
}else if(str.contains("年前")){
|
||||||
|
int years = Integer.parseInt(str.replaceAll("[^\\.0123456789]", ""))
|
||||||
|
return add(now, (-1)*years*365)
|
||||||
|
}else{
|
||||||
|
if(str.size()<10){
|
||||||
|
if(str.contains("-")){
|
||||||
|
str = (now.getYear()+1900)+"-"+str
|
||||||
|
}
|
||||||
|
}else{
|
||||||
|
if(str.contains("月") && str.contains("日") && !str.contains("年")){
|
||||||
|
str = (now.getYear()+1900)+"年"+str.substring(0, str.indexOf("日")+1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if(str.contains("年")){
|
||||||
|
return DateUtils.parseFromTightByPattern(str, "yyyy年MM月dd日")
|
||||||
|
}else{
|
||||||
|
return DateUtils.parseFromTightByPattern(str, "yyyy-MM-dd")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
+2
-2
@@ -11,8 +11,8 @@ import org.bson.Document
|
|||||||
import static com.mongodb.client.model.Filters.*
|
import static com.mongodb.client.model.Filters.*
|
||||||
|
|
||||||
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
|
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
|
||||||
def startDate = DateUtil.parseDate("20220201", "yyyyMMdd")
|
def startDate = DateUtil.parseDate("20220301", "yyyyMMdd")
|
||||||
def endDate = DateUtil.parseDate("20220301", "yyyyMMdd")
|
def endDate = DateUtil.parseDate("20220401", "yyyyMMdd")
|
||||||
Calendar calendar = Calendar.getInstance()
|
Calendar calendar = Calendar.getInstance()
|
||||||
calendar.setTime(startDate)
|
calendar.setTime(startDate)
|
||||||
def count = 0
|
def count = 0
|
||||||
|
|||||||
Reference in New Issue
Block a user