This commit is contained in:
2022-04-08 17:21:57 +08:00
parent bb96843c8d
commit 4f64858d94
24 changed files with 1531 additions and 259 deletions
@@ -225,17 +225,42 @@ def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def
//def fileName = "D:\\数据\\建行\\房产估值\\成交\\成交数据_1222 - 副本.xlsx"
//def dealMap = findDealCount(sqlInstance236, "2022-01-01", "2022-02-01")
//def testMap = findCaluateCityPPE(sqlInstance236, 202202)
//def dealMap = findDealCount(sqlInstance236, "2022-02-01", "2022-03-01")
//def testMap = findCaluateCityPPE(sqlInstance236, 202203)
//ExcelUtils.modify(fileName,{d->
// if(d['C']){
// def oldcount = dealMap.get(d['C'])
// d['V'] = oldcount
// d['X'] = oldcount
// def casecount = testMap.get(d['C'])
// d['W'] = casecount
// d['Y'] = casecount
// println d
// }
//},0)
def rows = findTradePpeData(sqlInstance236)
// rows.unique()
def map=findTradeDetail(sqlInstance236,null)
def datas=[]
map.each{key,row->
if(!rows.contains(key)){
datas<<row
}
}
ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交未使用.xlsx", datas, "数据")
//def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
//def datas=[]
//cityCodeDataMap.each{city,code->
// def rows = findTradePpeData(sqlInstance236, 202203,code)
// def alltrades=findTradeDetail(sqlInstance236,code)
// alltrades.each{it->
// def commid=it[0]
// if(!rows.contains(commid)){
// datas<<it
// }
// }
// println datas.size()
//}
//ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交未使用.xlsx", datas, "数据")
//ExcelUtils.multiThreadingModify("D:\\数据\\小区\\5个城市小区列表0723(1).xlsx",{d->
// if(d['I'] && d['K']){
@@ -270,24 +295,24 @@ def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def
// }
//},0)
def index = 0
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\人工核实需删除数据\\是否独有及第三方是否存在20220217\\9370300_淄博_是否独有及第三方是否存在(删除383).xlsx",{d->
def flag = d['M']
if(flag==0.0){
if(d['D']){
def udata = new BasicDBObject()
udata.put("name_cat", 11)
udata.put("m_date",new Date())
def q = new BasicDBObject()
q.put("_id", new ObjectId(d['D']))
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
index++
println index
}
}
},0)
//def index = 0
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\人工核实需删除数据\\是否独有及第三方是否存在20220217\\9370300_淄博_是否独有及第三方是否存在(删除383).xlsx",{d->
// def flag = d['M']
// if(flag==0.0){
// if(d['D']){
// def udata = new BasicDBObject()
// udata.put("name_cat", 11)
// udata.put("m_date",new Date())
//
// def q = new BasicDBObject()
// q.put("_id", new ObjectId(d['D']))
// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
// index++
// println index
// }
// }
//
//},0)
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\补充删除数据1223_1228.xlsx",{d->
// if(d['A']){
@@ -558,8 +583,30 @@ def findCaluateCityPPE(def sqlInstance, def yearMonth){
}
def findTradePpeData(def sqlInstance){
def sql = "SELECT commid,buildarea,to_char(dealdate,'yyyy-mm-dd') as dealdate,target_avg_price FROM avm_evaluation_item_gj where year_month=202203 and data_type='成交'";
def rows = new HashSet()
sqlInstance.eachRow(sql,{r->
rows<<r.commid+"##"+r.buildarea+"##"+r.target_avg_price+"##"+r.dealdate
});
return rows
}
def findTradeDetail(def sqlInstance, def cityid){
def sql = "select commid,cityname,distname,commname,buildarea,dealunitprice,to_char(dealdate,'yyyy-mm-dd') as dealdate from bigdata_deal_gj where dealdate>=to_date('2022-02-01','yyyy-MM-dd')";
def map = [:]
sqlInstance.eachRow(sql,{r->
def key=r.commid+"##"+r.buildarea+"##"+r.dealunitprice+"##"+r.dealdate
def row=[]
row<<r.commid
row<<r.cityname
row<<r.distname
row<<r.commname
row<<r.buildarea
row<<r.dealunitprice
map.put(key,row)
});
return map
}
def findEstateType(cityid, sqlInstance){
def sql = "select count(*) as count from bigdata_commbuildtype_gj where cityid='${cityid}' and buildtype!='0199'";
def count = 0
@@ -0,0 +1,275 @@
package com.fangjia.script.chongqing
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import groovy.sql.Sql
import org.apache.lucene.queryparser.classic.QueryParser
class EvaluationBatch {
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static main(args) {
def task = new EvaluationBatch()
// task.run()
task.run1();
// task.run2()
}
def run2(){
def startDate = DateUtils.parseFromTightYYYYMMDD("2021-01-01")
def endDate = DateUtils.parseFromTightYYYYMMDD("2022-04-07")
def weekends = DateUtils.getWeekendBetweenDates(startDate, endDate)
ExcelUtils.modify("D:\\数据\\重庆农商行\\2022-04-02\\重庆房屋交易信息V1.xlsx", { row ->
if(row['N'] && row['N'].toString().toDouble()>=20){
println row['F'] +" "+row['N']
def dist = CommonDatautils.findDistrictById(row['C'])
for(def weekend : weekends){
CommonUtils.recalculationAverage(row['D'], row['E'], row['F'], dist.estate_type2, weekend,false,0)
}
// FangjiaClientUtils.sendDistrictPriceCalculate(row['D'], row['E'], row['F'], 48);
}
},0)
}
def run1(){
def city = "重庆"
ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\2022-04-02\\重庆房屋交易信息V2.xlsx", { row ->
def region = row['E']
def name = row['F']
def id = row['C']
def lastWeekend = DateUtils.getFirstDayOfWeek(new Date()-30)
def lastYearWeekend = DateUtils.getFirstDayOfWeek(new Date()-365)
if (region && name) {
row['H'] = CommonDatautils.findNewSqmetreRent(city, region, name)
def priceObj = CommonDatautils.findDistrictPrice(city, region, name)
if(priceObj){
row['AB'] = priceObj.avg_price
}
def dist = CommonDatautils.findDistrictById(id)
if(dist) {
List<DBObject> list = DbUtils.findAroundDistrict(dist)
def map = getArroundAvgPriceMap(list)
if(map){
row['L'] = map.avg_price
row['Q'] = map.max_val
row['R'] = map.min_val
}
def lastArroundPrice = getArroundAvgPriceByWeekend(list, lastWeekend)
if(lastArroundPrice && priceObj){
def mod = (map.avg_price-lastArroundPrice)*100/lastArroundPrice
mod = DataHandleUtils.getDoubleRound(2, mod)
row['P'] = mod
}
def lastYearArroundPrice = getArroundAvgPriceByWeekend(list, lastYearWeekend)
if(lastYearArroundPrice && priceObj){
def mod = (map.avg_price-lastYearArroundPrice)*100/lastYearArroundPrice
mod = DataHandleUtils.getDoubleRound(2, mod)
row['O'] = mod
}
}
def lastWeekendObj = CommonDatautils.queryDistrictPrice(city, region, name, lastWeekend)
if(lastWeekendObj){
row['S'] = lastWeekendObj.avg_price
if(priceObj){
def mod = (priceObj.avg_price-lastWeekendObj.avg_price)*100/lastWeekendObj.avg_price
mod = DataHandleUtils.getDoubleRound(2, mod)
row['M'] = mod
}
}
def lastYearObj = CommonDatautils.queryDistrictPrice(city, region, name, lastYearWeekend)
if(lastYearObj){
row['T'] = lastYearObj.avg_price
if(priceObj){
def mod = (priceObj.avg_price-lastYearObj.avg_price)*100/lastYearObj.avg_price
mod = DataHandleUtils.getDoubleRound(2, mod)
row['N'] = mod
}
}
}
},0)
}
def run(){
def city = "重庆"
ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\2022-04-02\\重庆估值交易信息V2.xlsx", { row ->
def region = row['E']
def name = row['F']
def id = row['C']
def area = row['G'].toDouble()
if(region && name){
def priceObj = CommonDatautils.findDistrictPrice(city, region, name)
if(priceObj){
row['J'] = priceObj.avg_price
if(row['C']){
row['I'] = (area*priceObj.avg_price).toInteger()
}
}
def dist = CommonDatautils.findDistrictById(id)
if(dist){
List<DBObject> list = DbUtils.findAroundDistrict(dist)
row['N'] = getArroundAvgPrice(list)
def alias = dist.alias
if(!alias){
alias = []
}
alias << dist.name
def houseAvgPrice = findSimAreaHousePrice(dist.city, dist.region, alias,20220101,20220401, area-10, area+10)
if(houseAvgPrice){
row['K'] = houseAvgPrice
}
def tradeAvgPrice = findTrades(dist, DateUtils.parseFromTightYYYYMMDD("2021-12-01"), area-10, area+10)
row['L'] = tradeAvgPrice
def avg_price = findAvgPrice(dist._id.toString())
if(row['J'] && avg_price){
def weekendPrice = row['J'].toString().toDouble()
def ppe15 = Math.abs((weekendPrice - avg_price) * 100 / avg_price)
row['P'] = ppe15
}
}
}
});
}
def findTrades(dist, startDate, startArea, endArea){
def q = new BasicDBObject()
q.put("city", dist.city)
q.put("region", dist.region)
// q.put("source", "链家在线")
q.put("source", new BasicDBObject('$in', ["链家在线","贝壳找房"]))
def alias = []
if(dist.alias){
if(!dist.alias.contains(dist.name)){
dist.alias << dist.name
}
alias = dist.alias
}else{
alias << dist.name
}
if(alias){
q.put("district_name", new BasicDBObject('$in', alias))
}
q.put("trade_date", new BasicDBObject('$gte', startDate))
q.put("area", new BasicDBObject('$gte', startArea).append('$lt',endArea))
def dbcorsor = DEAL_PRICE_COLL.find(q)
def prices = []
for(def d : dbcorsor){
if(d.avg_price){
prices << d.avg_price
}
}
def price = null
if(prices){
price = (prices.sum()/prices.size()).toInteger()
}
return price
}
def getArroundAvgPriceMap(def list){
def prices = []
for(def l : list){
def priceObj = CommonDatautils.findDistrictPrice(l.city, l.region, l.name)
if(priceObj){
prices << priceObj.avg_price
}
}
prices.sort()
def map = [:]
map.min_val = prices[0]
map.max_val = prices[prices.size()-1]
def avg_price = (prices.sum()/prices.size()).toInteger()
map.avg_price = avg_price
return map;
}
def getArroundAvgPrice(def list){
def prices = []
for(def l : list){
def priceObj = CommonDatautils.findDistrictPrice(l.city, l.region, l.name)
if(priceObj){
prices << priceObj.avg_price
}
}
def avg_price = (prices.sum()/prices.size()).toInteger()
return avg_price;
}
def getArroundAvgPriceByWeekend(def list, def weekend){
def prices = []
for(def l : list){
def priceObj =CommonDatautils.queryDistrictPrice(l.city, l.region, l.name, weekend)
if(priceObj){
prices << priceObj.avg_price
}
}
def avg_price = null;
if(prices){
avg_price = (prices.sum()/prices.size()).toInteger()
}
return avg_price;
}
def findSimAreaHousePrice(def city, def region, def alias, def startDate, def endDate, def startArea, def endArea){
def aliasStr = null;
def total = 0
def prices = []
try{
aliasStr = escapeExprSpecialWord(alias.join("','"))
String sql = "select avg_price from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}') and area>=${startArea} and area<=${endArea}"
ElasticUtils.executeSqlQuery(sql,{ data->
prices << data.avg_price
})
}catch(Exception e){
e.printStackTrace()
}
def price = null
if(prices){
price = (prices.sum()/prices.size()).toInteger()
}
return price
}
public String escapeExprSpecialWord(String keyword) {
return QueryParser.escape(keyword);
}
def findAvgPrice(def commid){
def sql = "select year_month,target_avg_price from (select year_month,target_avg_price from avm_evaluation_item_gj where commid='${commid}' order by year_month desc) where rownum<2";
def avg_price = null
sqlInstance.eachRow(sql, { r ->
avg_price = r.target_avg_price
});
return avg_price
}
}
@@ -0,0 +1,84 @@
package com.fangjia.script.chongqing
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
class BlankMatch {
static main(args) {
def task = new BlankMatch()
// task.run()
// task.run1()
// task.run2()
task.run3()
}
def run3(){
ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配结果V3.xlsx", { row ->
if(!row['C']){
def dist = CommonDatautils.findDistrict("重庆", row['E'], row['F'], "district")
if(dist){
row['C'] = dist._id.toString()
}
}
},0)
}
def run2(){
def datas = ExcelUtils.read("D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配不上(合并).xlsx",0,1)
ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配结果V3.xlsx", { row ->
if(!row['C']){
for(def d : datas){
if(d['B'] == row['B']){
row['C'] = d['C']
row['D'] = d['D']
row['E'] = d['E']
row['F'] = d['J']
break;
}
}
}
},0)
}
def run() {
//rMap = SeaweedDataMatchProcess.doMatch(city, region, d['G'], null);
ExcelUtils.modify("D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配结果V3.xlsx", { row ->
if(!row['C']){
def jsonObject = OpenApiClient.addressMatch("重庆", row['B'], "property", false, null, null)
if(jsonObject != null && jsonObject.precise!=-1) {
row['C'] = jsonObject.get("id")
row['D'] = jsonObject.get("city")
row['E'] = jsonObject.get("district")
row['F'] = jsonObject.get("name")
row['G'] = jsonObject.get("address")
}
}
});
}
def run1() {
//rMap = SeaweedDataMatchProcess.doMatch(city, region, d['G'], null);
ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配不上(合并).xlsx", { row ->
if(row['F']){
def jsonObject = OpenApiClient.addressMatch("重庆", row['F'], "property", false, row['E'], null)
if(jsonObject != null && jsonObject.precise!=-1) {
row['C'] = jsonObject.get("id")
row['D'] = jsonObject.get("city")
row['J'] = jsonObject.get("name")
}
}
// def jsonObject = OpenApiClient.addressMatch("重庆", row['B'], "property", false, null, null)
// if(jsonObject != null && jsonObject.precise!=-1) {
// row['C'] = jsonObject.get("id")
// row['D'] = jsonObject.get("city")
// row['E'] = jsonObject.get("district")
// row['F'] = jsonObject.get("name")
// row['G'] = jsonObject.get("address")
// }
});
}
}
@@ -0,0 +1,77 @@
package com.fangjia.script.chongqing
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.mongodb.BasicDBObject
import org.apache.lucene.queryparser.classic.QueryParser
import org.bson.types.ObjectId
class FieldAddValue {
static main(args) {
def task = new FieldAddValue()
task.run()
}
def run(){
//16991 14360
def districts = CommonDatautils.findAllDistrictByCity("ÖØÇì", "district", ["region","name","alias","is_elevator"])
def index = 0
def updateIndex = 0
for(def d : districts){
if(!d.is_elevator){
index++
println index
def alias = d.alias
if(!d.alias){
alias = []
}
alias << d.name
def maxHeight = findHouseHeight("ÖØÇì", d.region, alias, 20220101)
if(maxHeight && maxHeight>6){
updateIndex++
def udata = new BasicDBObject()
udata.put("is_elevator", true)
udata.put("m_date",new Date())
def q = new BasicDBObject()
q.put("_id", new ObjectId( d._id.toString()))
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
}
}
}
println index+" "+updateIndex
}
def findHouseHeight(def city, def region, def alias, def startDate){
def aliasStr = null;
def total = 0
def heights = null
try{
aliasStr = escapeExprSpecialWord(alias.join("','"))
String sql = "select max(height) as height from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and region='${region}' and district_name in ('${aliasStr}') "
ElasticUtils.executeSqlQuery(sql,{ data->
if(data.height){
heights = data.height
}
})
}catch(Exception e){
e.printStackTrace()
}
// def max_height = null
// if(heights){
// heights.sort()
// max_height = heights[heights.size()-1]
// }
return heights
}
public String escapeExprSpecialWord(String keyword) {
return QueryParser.escape(keyword);
}
}
@@ -13,7 +13,7 @@ import org.apache.commons.lang.StringUtils
class OpenApiUtils {
static def matchDistrict(def city, def address, def suggestDistrict, def preciseLevelflag){
address = address.replace("(","(").replace(")",")").replace(" ","")
address = address.replace("(","(").replace(")",")").replace(" ","").replace("#","").replace("<","").replace(">","")
def url = "http://open.fangjia.com/address/matchV2?city="+city+"&address="+address+"&suggestDistrict="+suggestDistrict+"&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB&category=district&__refresh_cache=1"
if(preciseLevelflag){
url = url +"&preciseLevel=strict"
@@ -48,7 +48,7 @@ class VagueAddressLocationBatch extends BaseBatch {
void process(Object args) {
//"北京","常州",
// def citys =["无锡","重庆","东莞","中山","合肥","苏州","广州","深圳","佛山","长沙","韶关","惠州","清远","珠海","福州","厦门","龙岩","漳州","成都","天津","南京","青岛"] //CommonDatautils.findAllCity();
def citys =["北京"] //CommonDatautils.findAllCity();
def citys =["重庆"] //CommonDatautils.findAllCity();
//,"济南","青岛","烟台","威海","潍坊","淄博","临沂","泰安","菏泽","东营","德州","济宁","日照","聊城","滨州","枣庄"
//市谊大厦 华三新村 阳光家园
for (def city : citys) {
@@ -55,7 +55,7 @@ class AddDistrictFieldValue {
// }
// })
// def citys = ["沈阳","哈尔滨","吉林","遵义","淄博","西宁","青岛","大连","宁波"]//CcbCommonUtils.citys;
def citys = ["重庆"]//CcbCommonUtils.citys;
//第三方均价数据匹配
// for(def city : citys){
// def rets = findAllDataByCity(city)
@@ -63,11 +63,11 @@ class AddDistrictFieldValue {
// }
//第三方均价数据填补
// for(def city : citys){
//// def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","name_cat"])
// def rMap = getMatchDistrict(city)
// createResultValue(dists, rMap)
// }
for(def city : citys){
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","name_cat"])
def rMap = getMatchDistrict(city)
createResultValue(dists, rMap)
}
// for(def city : citys){
// def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","name_cat","open_date","build_date","live_date", "build_type"])
@@ -103,52 +103,52 @@ class AddDistrictFieldValue {
// }
def citys = ["上海"]
def updateIndex = 0
def rets = []
rets << ["编号", "城市", "区域", "名称","户数", "房价网楼栋数","建行楼栋数","来源"]
for(def city : citys){
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "household_count","loudong_count", "alias"])
def total = dists.size()
for(def dist : dists){
if(dist.household_count){
def ret = []
ret << dist._id.toString()
ret << dist.city
ret << dist.region
ret << dist.name
ret << dist.household_count
ret << dist.loudong_count
def details = findDistrictDetail(dist)
if(details){
for(def d : details){
def region = RegionData.formatRegion(city, d.region)
if(d.region == region){
if(dist.household_count && dist.household_count==d.house_count){
ret << d.loudong_count
ret << "链家"
// updateIndex++
// def udata = new BasicDBObject()
// udata.put("household_count", d.house_count.toInteger())
// udata.put("m_date", new Date())
// def q = new BasicDBObject()
// q.put("_id", dist._id)
// println updateIndex +"/${total} "+d.city +" "+d.name+" old-vale:"+dist.household_count+" new-value:"+d.house_count
// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
}
}
}
}else{
ret << ""
ret << ""
}
rets << ret
}
}
ExcelUtils.write("d:\\数据\\上海楼盘户数.xlsx", rets, "数据")
}
// def citys = ["上海"]
// def updateIndex = 0
// def rets = []
// rets << ["编号", "城市", "区域", "名称","户数", "房价网楼栋数","建行楼栋数","来源"]
// for(def city : citys){
// def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "household_count","loudong_count", "alias"])
// def total = dists.size()
//
// for(def dist : dists){
// if(dist.household_count){
// def ret = []
// ret << dist._id.toString()
// ret << dist.city
// ret << dist.region
// ret << dist.name
// ret << dist.household_count
// ret << dist.loudong_count
// def details = findDistrictDetail(dist)
// if(details){
// for(def d : details){
// def region = RegionData.formatRegion(city, d.region)
// if(d.region == region){
// if(dist.household_count && dist.household_count==d.house_count){
// ret << d.loudong_count
// ret << "链家"
//// updateIndex++
//// def udata = new BasicDBObject()
//// udata.put("household_count", d.house_count.toInteger())
//// udata.put("m_date", new Date())
//// def q = new BasicDBObject()
//// q.put("_id", dist._id)
//// println updateIndex +"/${total} "+d.city +" "+d.name+" old-vale:"+dist.household_count+" new-value:"+d.house_count
//// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
// }
// }
// }
// }else{
// ret << ""
// ret << ""
// }
//
// rets << ret
// }
// }
// ExcelUtils.write("d:\\数据\\上海楼盘户数.xlsx", rets, "数据")
// }
}
@@ -303,6 +303,14 @@ class AddDistrictFieldValue {
def createResultValue(dists, rMap){
def updateIndex = 0
def total = dists.size()
def addressIndex = 0
def allAreaIndex = 0
def buildAreaIndex = 0
def virescence_percent_index = 0
def cubage_percent_index = 0
def finish_building_date_index = 0
def household_count_index = 0
def car_site_count_index = 0
for(def d : dists){
def key = d.city +"#"+d.region+"#"+d.name
def sourceMap = rMap.get(key)
@@ -311,12 +319,14 @@ class AddDistrictFieldValue {
if(!d.address){
def lastValue = getValuesByField(sourceMap,"address", d.estate_type2)
if(lastValue){
addressIndex++
udata.put("address", lastValue)
}
}
if(!d.all_area){
def lastValue = getValuesByField(sourceMap,"all_area", d.estate_type2)
if(lastValue){
allAreaIndex++
udata.put("all_area", lastValue)
}
}
@@ -324,6 +334,7 @@ class AddDistrictFieldValue {
if(!d.build_area){
def lastValue = getValuesByField(sourceMap,"build_area", d.estate_type2)
if(lastValue){
buildAreaIndex++
udata.put("build_area", lastValue)
}
}
@@ -331,6 +342,7 @@ class AddDistrictFieldValue {
if(!d.virescence_percent){
def lastValue = getValuesByField(sourceMap,"greening_rate", d.estate_type2)
if(lastValue){
virescence_percent_index ++
udata.put("virescence_percent", lastValue)
}
}
@@ -338,6 +350,7 @@ class AddDistrictFieldValue {
if(!d.cubage_percent){
def lastValue = getValuesByField(sourceMap,"volumetric_rate", d.estate_type2)
if(lastValue){
cubage_percent_index ++
udata.put("cubage_percent", lastValue)
}
}
@@ -345,6 +358,7 @@ class AddDistrictFieldValue {
if(!d.finish_building_date){
def lastValue = getValuesByField(sourceMap,"build_date", d.estate_type2)
if(lastValue){
finish_building_date_index ++
udata.put("finish_building_date", lastValue)
}
}
@@ -352,6 +366,7 @@ class AddDistrictFieldValue {
if(!d.household_count){
def lastValue = getValuesByField(sourceMap,"house_count", d.estate_type2)
if(lastValue){
household_count_index ++
udata.put("household_count", lastValue)
}
}
@@ -359,6 +374,7 @@ class AddDistrictFieldValue {
if(!d.car_site_count){
def lastValue = getValuesByField(sourceMap,"car_site", d.estate_type2)
if(lastValue){
car_site_count_index ++
udata.put("car_site_count", lastValue)
}
}
@@ -373,6 +389,15 @@ class AddDistrictFieldValue {
}
}
}
println "addressIndex "+ addressIndex
println "allAreaIndex "+ allAreaIndex
println "buildAreaIndex "+ buildAreaIndex
println "virescence_percent_index "+ virescence_percent_index
println "cubage_percent_index "+ cubage_percent_index
println "finish_building_date_index "+ finish_building_date_index
println "household_count_index "+ household_count_index
println "car_site_count_index "+ car_site_count_index
}
def getValuesByField(sourceMap, field, estateType){
@@ -17,7 +17,7 @@ class TempMatchDistrict {
}
def stat(){
def citys = ["广州","深圳","佛山","东莞","中山","湛江","惠州","江门","珠海","汕头","揭阳","茂名","清远","肇庆","潮州","河源","韶关","阳江","梅州","云浮","汕尾"]
def citys = ["哈尔滨","齐齐哈尔","大庆","牡丹江","佳木斯","绥化","鸡西","七台河","双鸭山","鹤岗","黑河","伊春","大兴安岭"]
def total = 0
for(def city : citys){
def count = CommonDatautils.findAllDistrictCount("district", city)
@@ -38,8 +38,8 @@ class FangjiaClientUtils {
}
public static def sendDistrictPriceCalculate(def city, def region, def name, def count){
def url = "http://192.168.0.100:38080/FangJiaDataService/districtPriceCalculate?category=sell&city="+city+"&count="+count+"&name="+name+"&region="+region+"&retainHistory=true"
// def url = "http://192.168.10.19:8080/FangJiaDataService/districtPriceCalculate?category=sell&city="+city+"&count="+count+"&name="+name+"&region="+region+"&retainHistory=true"
// def url = "http://192.168.0.100:38080/FangJiaDataService/districtPriceCalculate?category=sell&city="+city+"&count="+count+"&name="+name+"&region="+region+"&retainHistory=true"
def url = "http://192.168.10.216:8080/FangJiaDataService/districtPriceCalculate?category=sell&city="+city+"&count="+count+"&name="+name+"&region="+region+"&retainHistory=true"
def content = null;
try{
content = new HttpRequest().newGet(url, null, "utf-8")
@@ -560,7 +560,7 @@ class DistrictIndexBatch extends BaseBatch{
return datas
}
def sortByDistance(d, list){
def sortByDistance(d, list){
for(def nb : list){
def distance = GeoHelper.GetDistance(d.lat/10000000000.0, d.lng/10000000000.0, nb.lat/10000000000.0, nb.lng/10000000000.0)
nb.distance = distance
@@ -34,7 +34,7 @@ public class BeiKeCrawlByChrome {
static main(args) {
def task = new BeiKeCrawlByChrome();
// task.run()
task.run()
// task.run1()
task.addDealRegion();
}
@@ -110,7 +110,7 @@ public class BeiKeCrawlByChrome {
def durl = e.select(".title a").attr("href")
def dealDate = e.select(".dealDate").text()
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
def endDate = DateUtils.parseFromTightByPattern("2022-01-20", "yyyy-MM-dd")
def endDate = DateUtils.parseFromTightByPattern("2022-02-20", "yyyy-MM-dd")
if(dealDate.getTime()>=endDate.getTime()){
detailUrls << durl
}else{
@@ -342,7 +342,7 @@ public class BeiKeCrawlByChrome {
if(cityEles){
for(def e : cityEles){
def href = e.attr("href")
if(!e.text().contains("南京")){
if(e.text().contains("福州")){
cityMap.put(e.text(), "https:"+href)
}
}
@@ -15,6 +15,7 @@ import org.openqa.selenium.chrome.ChromeDriver
import org.openqa.selenium.support.ui.ExpectedConditions
import org.openqa.selenium.support.ui.WebDriverWait
import javax.swing.JApplet
import java.text.DateFormat
import java.util.function.Consumer
@@ -41,45 +42,137 @@ System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Loca
System.setProperty("http.nonProxyHosts", "localhost");
def doc = getDocument();
def detailUrls = getDetailLinks(doc);
def rets = batchParse(detailUrls);
saveToDB(rets);
WebDriver driver = new ChromeDriver()
def proviceMap = getDocument(driver);
def citys = getCity(driver,proviceMap)
def detailUrls = getDetailLinks(driver, citys);
def getDetailLinks(doc){
def urls = []
if(doc){
def eles = JQueryUtils.select(".lots-list .lot-li-a", doc)
if(eles){
for(def el :eles){
def href = el.attr("href")
urls << "https://paimai.caa123.org.cn"+href
def getDetailLinks(driver, cityList){
def zxcitys = ["天津市","北京市","上海市","重庆市"]
for(def cityMap : cityList){
def citys = cityMap.citys
citys.each{city,id->
def urls = []
if(!zxcitys.contains(cityMap.provice)){
driver.get("https://paimai.caa123.org.cn/pages/lots/list.html?&province=${cityMap.provice_id}&city=${id}&status=&attribute=&term=&startTimeStamp=&endTimeStamp=&standardType=6&secondaryType=&canLoan=&isRestricted=&insuranceSupport=");
}else{
driver.get("https://paimai.caa123.org.cn/pages/lots/list.html?&province=${cityMap.provice_id}&status=&attribute=&term=&startTimeStamp=&endTimeStamp=&standardType=6&secondaryType=&canLoan=&isRestricted=&insuranceSupport=");
}
def doc = Jsoup.parse(driver.getPageSource());
def count = JQueryUtils.selectText(".count", doc)
def totalPage = getTotalPage(count, 12)
if(totalPage){
for(def i=1;i<=totalPage;i++){
doc = Jsoup.parse(driver.getPageSource());
def eles = JQueryUtils.select(".lots-list .lot-li-a", doc)
if(eles){
for(def el :eles){
def href = el.attr("href")
def map = [:]
map.province = cityMap.provice
map.city = city
map.url = "https://paimai.caa123.org.cn"+href
urls << map
}
}
try{
driver.findElement(By.className("next")).click()
}catch(Exception e){
Thread.sleep(8000)
driver.findElement(By.className("next")).click()
}
}
println city +" "+urls.size()
def rets = batchParse(urls);
saveToDB(rets);
}
}
println cityMap.provice +" 完成"
}
return urls;
//return urls;
}
def getDocument(){
WebDriver driver = new ChromeDriver()
def getTotalPage(def total, def pageSize){
if(total){
def mode = total.toInteger()/pageSize
if(mode > mode.toInteger()){
mode = mode.toInteger() +1
}
return mode;
}
return 0;
}
def getCity(driver, proviceMap){
def rows = []
def zxcitys = ["天津市","北京市","上海市","重庆市"]
proviceMap.each {k,v->
if(!zxcitys.contains(k)){
driver.get("https://paimai.caa123.org.cn/pages/lots/list.html?&province=${v}&status=&attribute=&term=&startTimeStamp=&endTimeStamp=&standardType=6&canLoan=&isRestricted=");
def doc = Jsoup.parse(driver.getPageSource());
def cityEles = JQueryUtils.select("li:contains("+k+") .value-limited li a", doc)
def cityMap = [:]
for(def c : cityEles){
def cityid = c.attr("data-cityid")
def city = c.text()
if(!["石家庄市","唐山市","秦皇岛市","邯郸市","邢台市","保定市"].contains(city)){
cityMap.put(city, cityid)
}
}
def map = [:]
map.provice = k
map.provice_id = v
map.citys = cityMap
rows << map
}else{
def map = [:]
map.provice = k
map.provice_id = v
def cityMap = [:]
cityMap.put(k, v)
map.citys = cityMap
rows << map
}
}
return rows
}
def getDocument(driver){
driver.get("https://paimai.caa123.org.cn/pages/lots/list.html?&province=9&status=&attribute=&term=&startTimeStamp=&endTimeStamp=&standardType=6&canLoan=&isRestricted=");
WebDriverWait wait = new WebDriverWait(driver,20)
wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.className("lots-list"))));
def doc = Jsoup.parse(driver.getPageSource());
return doc;
def provinceMap = [:]
if(doc){
def proviceEles = JQueryUtils.select(".auction-locus .limited-value-list", doc)
for(def ele : proviceEles){
def provice = ele.text()
def value = ele.attr("data-proviceid")
if(value && provice!="北京市" && provice!="天津市"){
provinceMap.put(provice, value)
}
}
}
return provinceMap;
}
def batchParse(def urls){
WebDriver driver = new ChromeDriver()
def ret = []
for(def url :urls){
driver.get(url);
for(def map :urls){
driver.get(map.url);
def doc = Jsoup.parse(driver.getPageSource());
def res = parse(url, doc);
def res = parse(map.province, map.city, map.url, doc);
if(res){
ret << res
}
@@ -87,111 +180,126 @@ def batchParse(def urls){
return ret;
}
def parse(url,doc){
def parse(province, city, url,doc){
if(doc){
def result = [:]
// def address = JQueryUtils.selectText(["#pro_bid_name"], doc)
// result.address = address
if(result.address){
def rMap = OpenApiUtils.matchDistrict(result.address)
if(rMap) {
result.matchCity = rMap.matchCity
result.matchRegion = rMap.matchRegion
result.matchName= rMap.matchName
result.matchAddress = rMap.matchAddress
result.province = province
result.city = city
def address = JQueryUtils.selectText(["#pro_bid_name"], doc)
result.address = address
if(result.address && !result.address.contains("测试")) {
def rMap = OpenApiUtils.matchDistrict(city, result.address, null, true)
if (rMap) {
result.matchCity = rMap.city
result.matchRegion = rMap.region
result.matchName = rMap.name
result.matchAddress = rMap.address
result.houseNum = rMap.house_num
result.roomNum = rMap.room_num
}
}
def finishDateStr = JQueryUtils.selectText(["#sf-countdown .time"], doc)
if(finishDateStr) {
result.finishDate = com.fangjia.script.util.DateUtils.parseFromTightByPattern(finishDateStr, "yyyy年MM月dd日 HH:mm:ss");
}
result.status = JQueryUtils.selectText([".bd_state"], doc)
result.companyName = JQueryUtils.selectText("p:contains(拍卖企业) .companyName", doc)
result.mobile = JQueryUtils.selectText("p:contains(联系方式) .companyTel", doc)
def curPriceStr = JQueryUtils.selectText([".colorred"], doc)
if(curPriceStr) {
curPriceStr = curPriceStr.replace(",","")
result.tradePrice = curPriceStr.toInteger()
}
def evaluationPriceStr = JQueryUtils.selectText(["#assessPrice"], doc)
if(evaluationPriceStr) {
evaluationPriceStr = evaluationPriceStr.replace(",", "")
if(evaluationPriceStr.isInteger()) {
result.evaluationPrice = evaluationPriceStr.toInteger()
def finishDateStr = JQueryUtils.selectText(["#sf-countdown .time"], doc)
if (finishDateStr) {
result.finishDate = com.fangjia.script.util.DateUtils.parseFromTightByPattern(finishDateStr, "yyyy年MM月dd日 HH:mm");
}
}
def depositPriceStr = JQueryUtils.selectText(["#cashDeposit"], doc)
if(depositPriceStr) {
depositPriceStr = depositPriceStr.replace(",", "")
if(depositPriceStr.isInteger()) {
result.depositPrice = depositPriceStr.toInteger()
def curPriceStr = JQueryUtils.selectText([".colorred"], doc)
if (curPriceStr) {
curPriceStr = curPriceStr.replace(",", "")
result.tradePrice = curPriceStr.toDouble().toInteger()
}
}
def addPriceStr = JQueryUtils.selectText(["#rateLadder"], doc)
if(addPriceStr) {
addPriceStr = addPriceStr.replace(",", "")
if(addPriceStr.isInteger()) {
result.addPrice = addPriceStr.toInteger()
}
}
def startPriceStr = JQueryUtils.selectText(["#startPrice"], doc)
if(startPriceStr) {
startPriceStr = startPriceStr.replace(",", "").replace("元", "")
if(startPriceStr.isInteger()) {
result.startPrice = startPriceStr.toInteger()
}
}
result.preemptivePerson= JQueryUtils.selectText(["#prior_span"], doc)
result.retain = JQueryUtils.selectText(["#retain"], doc)
def imgs =JQueryUtils.select(".sf-pic-slide img", doc)
if(imgs) {
def imgLinks = []
for(def img : imgs) {
def href = img.attr("src")
if(href.contains("http")) {
imgLinks << href
}else {
imgLinks << url.substring(0, url.indexOf(":")+1)+href
def evaluationPriceStr = JQueryUtils.selectText(["#assessPrice"], doc)
if (evaluationPriceStr) {
evaluationPriceStr = evaluationPriceStr.replace(",", "")
if (evaluationPriceStr.isNumber()) {
result.evaluationPrice = evaluationPriceStr.toDouble().toInteger()
}
}
result.imgUrls = imgLinks
}
def desc = JQueryUtils.selectText(["#J_desc"], doc)
if(!desc){
desc = JQueryUtils.selectText(["#pro_bid_name"], doc)
}
if(desc) {
if(desc.contains("别墅")) {
result.address = desc.substring(0, desc.indexOf("别墅")+2)
}else if(desc.contains("号楼")) {
result.address = desc.substring(0, desc.indexOf("号楼")+2)
}else if(desc.contains("房屋")) {
result.address = desc.substring(0, desc.indexOf("房屋")+2)
def depositPriceStr = JQueryUtils.selectText(["#cashDeposit"], doc)
if (depositPriceStr) {
depositPriceStr = depositPriceStr.replace(",", "")
if (depositPriceStr.isNumber()) {
result.depositPrice = depositPriceStr.toDouble().toInteger()
}
}
}
def descr = JQueryUtils.selectText(["body"], doc)
if(descr) {
def area = getAreaNumber(descr, 5000);
if(area){
result.area = area.toDouble()
def addPriceStr = JQueryUtils.selectText(["#rateLadder"], doc)
if (addPriceStr) {
addPriceStr = addPriceStr.replace(",", "")
if (addPriceStr.isNumber()) {
result.addPrice = addPriceStr.toDouble().toInteger()
}
}
}
if(result.area && result.tradePrice){
result.tradeAvgPrice = (int)(result.tradePrice/result.area)
}
result.content = DataUtil.zip(doc.toString())
result.introduce = JQueryUtils.select([".d-block table:eq(1)"], doc).toString()
result.url = url
return result;
def startPriceStr = JQueryUtils.selectText(["#startPrice"], doc)
if (startPriceStr) {
startPriceStr = startPriceStr.replace(",", "").replace("元", "")
if (startPriceStr.isNumber()) {
result.startPrice = startPriceStr.toDouble().toInteger()
}
}
result.preemptivePerson = JQueryUtils.selectText(["#prior_span"], doc)
result.retain = JQueryUtils.selectText(["#retain"], doc)
result.lookCount = JQueryUtils.selectText(".look_count span", doc)
def startDateStr = JQueryUtils.selectText("span:contains(拍卖会定于)", doc)
if(startDateStr){
startDateStr = startDateStr.toString().substring(startDateStr.indexOf("定于")+2, startDateStr.indexOf("日")+1).trim()
}
result.startDate = com.fangjia.script.util.DateUtils.parseString2Date(startDateStr)
def imgs = JQueryUtils.select(".sf-pic-slide img", doc)
if (imgs) {
def imgLinks = []
for (def img : imgs) {
def href = img.attr("src")
if (href.contains("http")) {
imgLinks << href
} else {
imgLinks << url.substring(0, url.indexOf(":") + 1) + href
}
}
result.imgUrls = imgLinks
}
def desc = JQueryUtils.selectText(["#J_desc"], doc)
if (!desc) {
desc = JQueryUtils.selectText(["#pro_bid_name"], doc)
}
if (desc) {
if (desc.contains("别墅")) {
result.address = desc.substring(0, desc.indexOf("别墅") + 2)
} else if (desc.contains("号楼")) {
result.address = desc.substring(0, desc.indexOf("号楼") + 2)
} else if (desc.contains("房屋")) {
result.address = desc.substring(0, desc.indexOf("房屋") + 2)
}
}
def descr = JQueryUtils.selectText(["body"], doc)
if (descr) {
def area = getAreaNumber(descr, 5000);
if (area) {
result.area = area.toDouble()
}
}
if (result.area && result.tradePrice) {
result.tradeAvgPrice = (int) (result.tradePrice / result.area)
}
result.content = DataUtil.zip(doc.toString())
result.introduce = JQueryUtils.select([".d-block table:eq(1)"], doc).toString()
result.url = url
return result;
}
}
return null
}
@@ -209,9 +317,11 @@ def saveToDB(rets){
}
if(dbo){
obj.remove("url")
obj.m_date = new Date()
paimaiResultColl.update(q, new BasicDBObject('$set', obj))
}else{
obj.c_date = new Date()
obj.m_date = new Date()
paimaiResultColl.save(obj)
}
}
@@ -229,6 +339,12 @@ def getAreaNumber(input, maxArea) {
def area = null;
for (def val : list) {
if(val.toString().contains(" ")){
val = val.toString().substring(val.toString().indexOf(" ")+1, val.toString().size())
}
if(val.contains("室")){
val = val.toString().substring(val.toString().indexOf("室")+1, val.toString().size())
}
def v = val.replace("平方米", "").replace("㎡", "").replace("平", "").replace("方米", "").replace(",", "").replace("m2", "");
if (maxArea) {
if (v.toDouble() < maxArea) {
@@ -187,7 +187,7 @@ public class LianjiaCrawlByChrome {
def durl = e.select(".title a").attr("href")
def dealDate = e.select(".dealDate").text()
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
def endDate = DateUtils.parseFromTightByPattern("2022-01-20", "yyyy-MM-dd")
def endDate = DateUtils.parseFromTightByPattern("2022-02-20", "yyyy-MM-dd")
if(dealDate.getTime()>=endDate.getTime()){
detailUrls << durl
}else{
@@ -218,7 +218,7 @@ public class LianjiaCrawlByChrome {
def durl = e.select(".title a").attr("href")
def dealDate = e.select(".dealDate").text()
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
def endDate = DateUtils.parseFromTightByPattern("2022-01-20", "yyyy-MM-dd")
def endDate = DateUtils.parseFromTightByPattern("2022-02-20", "yyyy-MM-dd")
if(dealDate.getTime()>=endDate.getTime()){
detailUrls << durl
parse(city, e)
@@ -417,11 +417,12 @@ public class LianjiaCrawlByChrome {
def cityEles = JQueryUtils.select(".city_province a", cityDoc)
def cityMap = [:]
if(cityEles){
for(def e : cityEles){
for(def e : cityEles) {
def href = e.attr("href")
// if(!href.contains("fang.lianjia") && e.text().contains("福州")){
if (!href.contains("fang.lianjia") && e.text().contains("福州")) {
cityMap.put(e.text(), href)
}
}
}
return cityMap
}
@@ -7,34 +7,147 @@ import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document
import org.jsoup.Jsoup
import org.openqa.selenium.WebDriver
import org.openqa.selenium.chrome.ChromeDriver
import java.text.SimpleDateFormat
import java.util.regex.Matcher
import java.util.regex.Pattern
import java.util.zip.GZIPInputStream
import java.util.zip.GZIPOutputStream
import org.apache.commons.lang.StringUtils;
import java.text.SimpleDateFormat;
class PaimaiTest {
DBCollection paimaiResultColl = MongoUtils.getCollectionByAuth20190102("fangjia", "pai_mai")
static main(args) {
def task = new PaimaiTest();
task.run();
}
def run(){
def url = "http://www.gpai.net/sf/item2.do?Web_Item_ID=23134"
def con = find(url);
parser(con, url);
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
System.setProperty("http.nonProxyHosts", "localhost");
WebDriver driver = new ChromeDriver()
def url = "https://s.gpai.net/sf/search.do?at=376"
def provinces = getProvice(url, driver)
def cityUrls = getCity(provinces, driver)
getDetailUrls(cityUrls, driver)
}
def getDetailUrls(cityUrls, driver){
for(def cMap : cityUrls){
driver.get("https:"+cMap.url)
def doc = Jsoup.parse(driver.getPageSource());
def total = JQueryUtils.selectText("span:contains(共有拍品) label", doc)
def parser(s, url){
Document doc = Jsoup.parse(s);
if(total){
def totalPage = getTotalPage(total.toInteger(), 20)
def detailDatas = []
for(def i=1;i<=totalPage;i++){
driver.get("https:"+cMap.url+"&page="+i)
doc = Jsoup.parse(driver.getPageSource());
def detailEles = JQueryUtils.select(".block-wrap:contains(共有拍品) .filt-result-list li .item-tit", doc)
for(def ele : detailEles){
def dMap = [:]
dMap.proince = cMap.proince
dMap.city = cMap.city
dMap.url = ele.select("a").attr("href")
detailDatas << dMap
}
}
def results = parser(detailDatas, driver, cMap.proince, cMap.city)
saveData(results)
println cMap.proince+" "+cMap.city+" "+results.size()
}
}
}
def getTotalPage(def total, def pageSize){
if(total){
def mode = total.toInteger()/pageSize
if(mode > mode.toInteger()){
mode = mode.toInteger() +1
}
return mode;
}
return 0;
}
def getCity(datas, driver){
def citys = ["上海","北京","天津","重庆"]
def cityList = []
for(def data : datas){
driver.get("https:"+data.href)
def doc = Jsoup.parse(driver.getPageSource());
if(citys.contains(data.proince)){
def cityMap = [:]
cityMap.proince = data.proince
cityMap.city = data.proince
cityMap.url = data.href
cityList << cityMap
}else{
def links = JQueryUtils.select(".condition .s-subtab a", doc)
for(def ele : links){
if(!["合肥市","芜湖市","蚌埠市","淮南市","马鞍山市","淮北市","铜陵市"].contains(ele.text())){
def cityMap = [:]
cityMap.proince = data.proince
cityMap.city = ele.text()
cityMap.url = ele.attr("href")
cityList << cityMap
}
}
}
}
return cityList
}
def getProvice(url, driver){
driver.get(url)
def doc = Jsoup.parse(driver.getPageSource());
def datas = []
def keys = ["上海","天津","重庆","北京","浙江","河北","山西","辽宁","吉林","江苏","福建","江西","山东","河南","湖北","湖南","广东"]
if(doc){
def eles = JQueryUtils.select(".condition a", doc)
for(def e : eles){
def data = [:]
if(!keys.contains(e.text())){
data.proince = e.text()
data.href = e.attr("href")
datas << data
}
}
}
return datas
}
def parser(detailDatas, driver, proince, city){
def results = []
for(def m : detailDatas){
try{
driver.get("https:"+m.url)
Thread.sleep(3000)
def doc = Jsoup.parse(driver.getPageSource());
def obj = parsering(doc, m.url)
if(obj){
obj.proince = proince
obj.city = city
results << obj
}
}catch(Exception e){
println m.url+" 解析有问题!"
}
}
return results
}
def parsering(doc, url){
def result = [:]
if(doc) {
def title = JQueryUtils.selectText([".d-m-title"], doc)
if(!title) {
title = JQueryUtils.selectText([".DivItemName"], doc)
@@ -48,6 +161,9 @@ class PaimaiTest {
}
result.title = title
result.status = status
if(title.contains("北京市朝阳区东三环中路7号4号楼28层3201")){
println title
}
if(title) {
def newStr =title
@@ -76,7 +192,7 @@ class PaimaiTest {
if(curPriceStr.contains("成交价:")) {
def tradePriceStr = JQueryUtils.selectText([".d-m-price .price-red"], doc).replace("元","")
if(tradePriceStr) {
result.tradePrice = tradePriceStr.toInteger()
result.tradePrice = tradePriceStr.toDouble().toInteger()
}
}
}
@@ -85,23 +201,28 @@ class PaimaiTest {
curPriceStr = JQueryUtils.selectText(["#ItemArea .red"], doc)
if(curPriceStr) {
curPriceStr = curPriceStr.replace(",", "").replace("元","").replace("--","")
result.tradePrice = curPriceStr.toInteger()
result.tradePrice = curPriceStr.toDouble().toInteger()
}
}
def enrollCount = JQueryUtils.selectText([".peoples-infos span:eq(0) b"], doc)
if(enrollCount) {
def enrollCount = JQueryUtils.selectText([".peoples-infos span:contains(人报名) b"], doc)
if(enrollCount && enrollCount.isNumber()) {
result.enrollCount = enrollCount.toInteger()
}
def callCount = JQueryUtils.selectText([".peoples-infos span:eq(1) b"], doc)
if(callCount) {
def callCount = JQueryUtils.selectText([".peoples-infos span:contains(人设置提醒) b"], doc)
if(callCount && callCount.isNumber()) {
result.callCount = callCount.toInteger()
}
def lookCount = JQueryUtils.selectText([".peoples-infos span:eq(2) b"], doc)
if(lookCount) {
result.lookCount = lookCount.toInteger()
def lookCount = JQueryUtils.selectText([".peoples-infos span:contains(人围观) b"], doc)
if(lookCount && lookCount.isNumber()) {
try{
result.lookCount = lookCount.toInteger()
}catch(Exception e){
println lookCount
}
}
def finishDateStr = JQueryUtils.selectText(["p:contains(拍卖竞价确认时间:)"], doc)
@@ -140,7 +261,7 @@ class PaimaiTest {
}else if(str.contains("保证金")) {
def v = str.replace("保证金:", "").replace("元", "").replace(",", "").replace("--", "")
if(v) {
result.depositPrice = v.toInteger()
result.depositPrice = v.toDouble().toInteger()
}
}else if(str.contains("延时周期")) {
@@ -184,6 +305,47 @@ class PaimaiTest {
}
}
}
if(detailMap && !detailMap.get("拍品名称")){
detailMap = [:]
for(def tr : trs) {
if(tr.text().contains("拍品名称")){
def td1 = tr.select("td:contains(拍品名称)+td").text()
detailMap.put("拍品名称", td1)
}
if(tr.text().contains("权利来源")){
def td1 = tr.select("td:contains(权利来源)+td").text()
detailMap.put("权利来源", td1)
}
if(tr.text().contains("权证情况")){
def td1 = tr.select("td:contains(权证情况)+td").text()
detailMap.put("权证情况", td1)
}
if(tr.text().contains("拍品所有人")){
def td1 = tr.select("td:contains(拍品所有人)+td").text()
detailMap.put("拍品所有人", td1)
}
if(tr.text().contains("房屋情况")){
def td1 = tr.select("td:contains(房屋情况)+td").text()
detailMap.put("房屋情况", td1)
}
if(tr.text().contains("权利限制情况及瑕疵情况")){
def td1 = tr.select("td:contains(权利限制情况及瑕疵情况)+td").text()
detailMap.put("权利限制情况及瑕疵情况", td1)
}
if(tr.text().contains("拍品介绍")){
def td1 = tr.select("td:contains(拍品介绍)").text()
detailMap.put("拍品介绍", td1)
}
}
}
result.detailMap = detailMap
if(!result.address) {
result.address = detailMap.get("拍品名称")
@@ -248,17 +410,18 @@ class PaimaiTest {
}
}
result.content = DataUtil.zip(s)
// result.content = DataUtil.zip(s)
def el = JQueryUtils.select([".d-block table:eq(1)"], doc)
if(!el) {
el = JQueryUtils.select([".d-title:contains(标的物介绍)+div"], doc)
}
result.content = DataUtil.zip(s)
// result.content = DataUtil.zip(s)
result.introduce = JQueryUtils.select(["#J_desc table"], doc).toString()
result.url = url
result.mDate = new Date();
return result;
}
def find(url){
@@ -374,4 +537,32 @@ class PaimaiTest {
}
return null;
}
def saveData(def results){
for(def ret : results){
def q = new BasicDBObject()
q.put("url", ret.url)
def dbo = paimaiResultColl.findOne(q);
DBObject obj = new BasicDBObject();
ret.each{k,v->
obj.put(k,v)
}
if(dbo){
obj.m_date = new Date()
paimaiResultColl.update(q, new BasicDBObject('$set', obj))
}else{
obj.c_date = new Date()
obj.m_date = new Date()
try{
paimaiResultColl.save(obj)
}catch(Exception e){
println obj.url
}
}
}
}
}
@@ -5,10 +5,15 @@ import com.fangjia.core.util.DataUtil
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
import groovy.sql.Sql
class BlankEvaluationBatch {
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
class EvaluationBatch {
static main(args) {
def task = new EvaluationBatch();
def task = new BlankEvaluationBatch();
task.run()
}
def run(){
@@ -0,0 +1,205 @@
package com.fangjia.script.loudong
import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.impl.AMQImpl
import org.jsoup.Jsoup
import org.openqa.selenium.By
import org.openqa.selenium.WebDriver
import org.openqa.selenium.chrome.ChromeDriver
class CqfdcChromeCrawl {
static DBCollection loudongConn = MongoUtils.getCollectionByName("fangjia_crawl", "loudong_cqfdc", "192.168.0.235", 27018)
static main(args) {
def task = new CqfdcChromeCrawl();
task.run()
}
def run(){
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
System.setProperty("http.nonProxyHosts", "localhost");
WebDriver driver = new ChromeDriver()
def list = getListDatas(driver)
getDetails(driver, list);
}
def getDetails(driver, list){
def index = 0
for(def map_old : list){
def datas = []
index++
def url = "http://cqfdc.cq315house.com/HtmlPage/ShowRooms.html?buildingid=${map_old.building_id}&block=${map_old.house_num_desc}"
// driver.get(url)
// def doc = driver.getPageSource()
def cityDoc = getDocment(url, driver)
def trs = JQueryUtils.select(".building_table tr", cityDoc)
def unit1 = null
def unit2 = null
def unit3 = null
def unit1_text = null
def unit2_text = null
def unit3_text = null
for(def i=0;i<trs.size();i++){
if(i==1){
continue
}
def tr = trs.get(i)
def tds = tr.select("td")
for(def j=2;j< tds.size();j++){
def td = tds[j]
if(i==0){
def text = td.text()
if(!unit1){
if(text && text.contains("单元")){
unit1 = td.attr("colspan").toInteger()
unit1_text = text.replace("单元","")
}
}else if(unit1 && !unit2){
if(text && text.contains("单元")) {
unit2 = td.attr("colspan").toInteger()
unit2_text = text.replace("单元", "")
}
}else if(unit1 && !unit3){
if(text && text.contains("单元")) {
unit3 = td.attr("colspan").toInteger()
unit3_text = text.replace("单元", "")
}
}
}else{
def room_num = td.select("a font").text()
if(room_num){
def map = [:]
def a = td.select("a")
map.use_type = td.select("span:contains(用途:)").text().replace("用途:","")
map.room_type = td.select("span:contains(户型:)").text().replace("户型:","")
map.structure_type = td.select("span:contains(结构:)").text().replace("结构:","")
map.area = td.select("span:contains(建面:)").text().replace("建面:","")
map.siji_area = td.select("span:contains(套内:)").text().replace("套内:","")
map.price = td.select("span:contains(建面拟售价格:)").text().replace("建面拟售价格:","")
map.siji_price = td.select("span:contains(套内拟售价格:)").text().replace("套内拟售价格:","")
map.fitment = td.select("span:contains(装修水平:)").text().replace("装修水平:","").replace("null","")
def physical_floor = tds[0]//物理层
def nominal_floor = tds[1]//名义层
map.nominal_floor = nominal_floor.text()
map.physical_floor = physical_floor.text()
if(unit1 && (j+1 <= unit1+2)){
map.unit_num = unit1_text
}else if(unit2 && (j+1 <= unit1+unit2+2)){
map.unit_num = unit2_text
}else if(unit3 && (j+1 <= unit1+unit2+unit3+2)){
map.unit_num = unit3_text
}
map.room_num = room_num
map_old.each{k,v->
map.put(k,v)
}
datas << map
}
}
}
}
saveDatas(datas)
println index+" "+map_old.name+" "+map_old.house_num_desc +" "+datas.size()
}
}
def getDocment(def url, def driver){
def stop = false
def doc = null
def error = 0
while (!stop || error>=10){
try{
driver.get(url)
def docment = driver.getPageSource()
if(docment){
doc = Jsoup.parse(docment)
stop = true;
}else{
error ++
}
}catch(Exception e){
error ++
}
}
return doc;
}
def getListDatas(driver){
driver.get("http://cqfdc.cq315house.com/HtmlPage/PresaleDetail.html")
def datas = []
for(def i=2;i<=208;i++){
def docment = Jsoup.parse(driver.getPageSource());
def trs = JQueryUtils.select("#searchbody tr", docment)
for(def tr : trs){
def tds = tr.select("td")
def hrefs = tds.get(3).select("a")
for(def href : hrefs){
def map = [:]
map.house_num_desc = href.text()
def str = href.attr("href")
def building_id= str.substring(str.lastIndexOf("['")+2, str.lastIndexOf("','"))
map.building_id = building_id
map.city = "重庆"
map.name = tds.get(0).text()
map.company = tds.get(1).text()
map.license_key = tds.get(2).text()
datas << map
}
}
println i +" "+datas.size()
try{
driver.findElement(By.id("nextpage")).click()
}catch(Exception e){
Thread.sleep(8000)
driver.findElement(By.id("nextpage")).click()
}
}
println "列表:"+datas.size()
return datas;
}
def saveDatas(def datas){
def rows = []
def keys = []
for(def d : datas){
def q = new BasicDBObject()
q.put("city", d.city)
q.put("name",d.name)
q.put("house_num_desc", d.house_num_desc)
q.put("room_num", d.room_num)
q.put("unit_num",d.unit_num)
def obj = loudongConn.findOne(q)
if(!obj){
def key = d.house_num_desc +"#"+d.unit_num+"#"+d.room_num
if(!keys.contains(key)){
def udata = new BasicDBObject()
d.each {k,v->
udata.put(k,v)
}
udata.put("c_date", new Date())
rows << udata
keys << key
}
}
}
if(rows){
loudongConn.insert(rows);
println "save: "+rows.size()
}
}
}
@@ -30,8 +30,8 @@ class HouseBookChromeCrawl {
WebDriver driver = new ChromeDriver()
def listPageUrl = getIndexPage(driver)
def keywords = getListPage(driver,"http://www.house-book.com.cn/"+listPageUrl)
keywords.remove("城南家园")
keywords.remove("康居西城")
// keywords.remove("城南家园")
// keywords.remove("康居西城")
println city +"共 "+keywords.size()+" 个小区"
getDistrictIndex(driver, keywords)
@@ -61,15 +61,15 @@ class HouseBookChromeCrawl {
def getAllDist(driver, url, page){
def keywords = []
for(def i=1;i<=page;i++){
for(def i=73;i<=page;i++){
driver.get(url+"?selectPages="+i)
def doc = Jsoup.parse(driver.getPageSource())
def eles = JQueryUtils.select(".title-area .title-text", doc)
for(def e : eles){
def text = e.text()
if(text=="富州北郡"){
println i
}
// if(text=="富州北郡"){
// println i
// }
keywords << text
}
}
@@ -206,7 +206,7 @@ class HouseBookChromeCrawl {
detail.estate_type = house_type
}
if(detail){
=-09ql.floor = floor
detail.floor = floor
rows << detail
}
}
@@ -214,7 +214,7 @@ class HouseBookChromeCrawl {
}
}
return rows;
}+*-*
}
def getIndexPage(driver){
@@ -0,0 +1,59 @@
package com.fangjia.script.loudong
import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.thirdPriceCrawl.CrawalUtil
import com.fangjia.script.thirdPriceCrawl.UserAgents
import com.fangjia.script.util.ABuYunUtil
import net.sf.json.JSONObject
import org.apache.commons.lang.StringUtils
import org.jsoup.Jsoup
import org.jsoup.select.Elements
class HouseBookCrawl {
public def CHARSET = "utf-8";
static main(args) {
def task = new HouseBookCrawl()
task.run()
}
// def run(){
// goTocitypage()
// }
// def goTocitypage(){
// def url = "http://www.house-book.com.cn/mainAction/mainAction/city/%25E9%2587%258D%25E5%25BA%2586%25E5%25B8%2582.html"
// def headers = [:]
// headers.put("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8");
// headers.put("User-Agent", UserAgents.get())
// headers.put("cookCity", "%2525E9%252587%25258D%2525E5%2525BA%252586%2525E5%2525B8%252582")
// headers.put("HBJSESSIONID","afb32a63-796e-4bbc-bf39-0f8cb188079e")
// def params = [:]
// def responseMap = ABuYunUtil.doPostResponse(url, null, headers, CHARSET, false)
// def cookie = getCookie(responseMap)
// }
//
// def getCookie(responseMap) {
// def cookieAuth = null
// try {
// def response = responseMap.get("response");
// if (StringUtils.isNotEmpty(response)) {
// JSONObject reJson = JSONObject.fromObject(response)
// //登录成功获取COOKIE
// if (reJson.success) {
// responseMap.get("headerFields").each { k, v ->
// if (k && k.equals("Set-Cookie")) {
// if (v) {
// cookieAuth = v.join(';')
// }
// }
// }
// }
// }
// } catch (Exception e) {
// println "goApiLogin(${responseMap})发生异常, 异常信息为:" + e.getMessage()
// }
// println("最终构造cookieAuth为:" + cookieAuth)
// return cookieAuth
// }
}
@@ -0,0 +1,147 @@
package com.fangjia.script.loudong
import net.sourceforge.tess4j.ITesseract
import net.sourceforge.tess4j.Tesseract
import net.sourceforge.tess4j.util.ImageHelper
import org.apache.commons.io.FileUtils
import org.ehcache.xml.model.TimeUnit
import org.jsoup.Jsoup
import org.openqa.selenium.By
import org.openqa.selenium.OutputType
import org.openqa.selenium.Point
import org.openqa.selenium.TakesScreenshot
import org.openqa.selenium.WebDriver
import org.openqa.selenium.WebElement
import org.openqa.selenium.chrome.ChromeDriver
import javax.imageio.ImageIO
import java.awt.image.BufferedImage
public class ImageTest {
public static void main(String[] args) throws IOException {
// File imageFile = new File("D:\\gaojuan\\img\\WaterMark.gif");
// Tesseract instance = new Tesseract();
//
// //将验证码图片的内容识别为字符串
// String result = instance.doOCR(imageFile);
// println result
//1.获取网页的浏览器driver
//(1)定义url
String url="http://cqfdc.cq315house.com/HtmlPage/ShowRooms.html?buildingid=2l/xVhsMJPAZKivgt8ciOA&block=3%E5%8F%B7%E6%A5%BC#";
//(2)设置driver驱动chromedriver.exe(版本:2.39)路径(相对路径)
String path = System.getProperty("user.dir");
System.setProperty("webdriver.chrome.driver","C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
//(3)建立selenium 驱动
//WebDriver driver = new ChromeDriver(options);
WebDriver driver = new ChromeDriver();
//(4)设置窗口最大化及网页等待时间
driver.manage().window().maximize(); //窗口最大化
// driver.manage().timeouts().implicitlyWait(10, TimeUnit.SECONDS);
//(5)启动driver浏览网页
driver.get(url); //输入网址
Thread.sleep(1000);
driver.findElement(By.id("cell1-11-2")).click()
Thread.sleep(1000);
def docment = Jsoup.parse(driver.getPageSource());
println docment
WebElement ele = driver.findElement(By.id("vimg"));
File screenshot = ((TakesScreenshot)driver).getScreenshotAs(OutputType.FILE);
BufferedImage pic1 = ImageIO.read(screenshot);
Point point = ele.getLocation();
//(2)输入验证码
File srcfile = ((TakesScreenshot) driver).getScreenshotAs(OutputType.FILE);
FileUtils.copyFile(srcfile, new File("picture/source.png"));
//原图片
//String soursePicPath = "D:\\data_dev\\test\\test.png";
String soursePicPath = "picture/source.png";
File sourcePic = new File(soursePicPath);
int eleWid = ele.getSize().getWidth()
int eleHeiht = ele.getSize().getHeight()
BufferedImage pic2 = pic1.getSubimage(point.getX(), point.getY(), eleWid, eleHeiht);
ImageIO.write(pic2, "jpg", sourcePic);
//File desImage = new File("D:\\data_dev\\test\\test02.png");
File desImage = new File("picture/answer.png");
//加载待读取图片
//File imageFile = new File("D:\\data_dev\\test\\test02.png");
File imageFile = new File("picture/answer.png");
BufferedImage image = ImageIO.read(imageFile);
//对图片进行处理
image = convertImage(image);
//创建tess对象
ITesseract instance = new Tesseract();
//设置训练文件目录
//instance.setDatapath("E:\\workspace\\workspace_idea\\com_mth_data\\m-kuaidi100-data\\tessdata");
String path02 = System.getProperty("user.dir");
instance.setDatapath(path02 + "\\tessdata");
//设置训练语言
//instance.setLanguage("chi_sim");
instance.setLanguage("eng");
//执行转换
String s = instance.doOCR(image);
System.out.println(s);
String[] split = s.replace(" ", "").split("=|:");
String a = split[split.length - 1].trim();
char[] chars = a.toCharArray();
if (chars.length == 2) {
if (chars[0] == '7') {
chars[0] = '1';
}
}
a = String.valueOf(chars);
System.out.println(a);
String b = s.substring(0, 1);
System.out.println(b);
String answer = String.valueOf(Integer.parseInt(a.trim()) - Integer.parseInt(b.trim()));
WebElement input2 = driver.findElement(By.xpath("//*[@id=\"ctl00_Content_Body_TextBox2\"]"));
input2.sendKeys(answer);
Thread.sleep(4000);
//(3)点击"查询"按钮
driver.findElement(By.xpath("//*[@id=\"ctl00_Content_Body_Button1\"]")).click();
String SecondtHandle = driver.getWindowHandle(); //首先得到最先的窗口 权柄
for (String winHandle1 : driver.getWindowHandles()) { //得到浏览器所有窗口的权柄为Set集合,遍历
if (!winHandle1.equals(SecondtHandle)) { //如果为 最先的窗口 权柄跳出
driver.close();
driver.switchTo().window(winHandle1); //如果不为 最先的窗口 权柄,将 新窗口的操作权柄 给 driver
System.out.println(driver.getCurrentUrl()); //打印是否为新窗口
}
}
Thread.sleep(4000);
//4.关闭浏览器
driver.close();
}
//对图片进行处理 - 提高识别度
public static BufferedImage convertImage(BufferedImage image) throws Exception {
//按指定宽高创建一个图像副本
//image = ImageHelper.getSubImage(image, 0, 0, image.getWidth(), image.getHeight());
//图像转换成灰度的简单方法 - 黑白处理
image = ImageHelper.convertImageToGrayscale(image);
//图像缩放 - 放大n倍图像
image = ImageHelper.getScaledInstance(image, image.getWidth() * 15, image.getHeight() * 15);
return image;
}
}
@@ -191,24 +191,24 @@ class Esf58Crawl extends BaseBatch{
void process(Object args) {
//addContent("福州","https://fz.58.com/xiaoqu/", 70);
def citys = ["新余","松原","楚雄","吕梁","雅安","本溪","鹤壁","拉萨","黔东南","忻州","兴安盟","乌海","贵港","三门峡","仙桃","安康","白城","平凉","庆阳","潮州","白银","吴忠","防城港","延安","丽江","黔西南","喀什","钦州","鸡西","萍乡","石嘴山","鹰潭","保山","哈密","铜仁","武威","普洱","河池","酒泉","朔州","百色","鹤岗","张家界","定西","湘西","济源","昭通","中卫","张掖","克拉玛依","黑河","铜川","辽源","云浮","博尔塔拉","固原","白山","天门","潜江","贺州","临沧","伊春","临夏","石河子","来宾","双鸭山","嘉峪关","海东"]
for(def city : citys){
def str = cityList.get(city)
for(def i=0;i<1;i++){
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]);
if(districts){
def arrs = str.split("\\|")
def url = "https://"+arrs[0]+".58.com/xiaoqu/"
distMap.put(city, districts)
addCrawlKeys(city);
addDetailContent(city, "https://"+arrs[0]+".58.com", arrs[1]);
}
}
}
// def citys = ["新余","松原","楚雄","吕梁","雅安","本溪","鹤壁","拉萨","黔东南","忻州","兴安盟","乌海","贵港","三门峡","仙桃","安康","白城","平凉","庆阳","潮州","白银","吴忠","防城港","延安","丽江","黔西南","喀什","钦州","鸡西","萍乡","石嘴山","鹰潭","保山","哈密","铜仁","武威","普洱","河池","酒泉","朔州","百色","鹤岗","张家界","定西","湘西","济源","昭通","中卫","张掖","克拉玛依","黑河","铜川","辽源","云浮","博尔塔拉","固原","白山","天门","潜江","贺州","临沧","伊春","临夏","石河子","来宾","双鸭山","嘉峪关","海东"]
// for(def city : citys){
// def str = cityList.get(city)
// for(def i=0;i<1;i++){
// def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]);
// if(districts){
// def arrs = str.split("\\|")
// def url = "https://"+arrs[0]+".58.com/xiaoqu/"
// distMap.put(city, districts)
// addCrawlKeys(city);
// addDetailContent(city, "https://"+arrs[0]+".58.com", arrs[1]);
// }
// }
// }
/*cityList.each{city, str->
cityList.each{city, str->
for(def i=0;i<1;i++){
if(citys.contains(city)){
// if(citys.contains(city)){
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]);
if(districts){
def arrs = str.split("\\|")
@@ -218,9 +218,9 @@ class Esf58Crawl extends BaseBatch{
addCrawlKeys(city);
addDetailContent(city, "https://"+arrs[0]+".58.com", arrs[1]);
}
}
// }
}
}*/
}
}
@@ -15,10 +15,10 @@ class SynchDistrictPrice {
def static final SHORT_DATE = Utils.getLastMonth()
def thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
def districtDetailColl = CrawalDBUtil.getDistrictDetailColl();
// def sourceMap = ["fang.com":"搜房","jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fangdd.com":"房多多","house365.com":"365淘房",
// "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家"]
def sourceMap = ["fang.com":"搜房","jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fangdd.com":"房多多","house365.com":"365淘房",
"anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家"]
//"fang.com":"搜房",
def sourceMap = ["fang.com":"搜房"]
// def sourceMap = ["fang.com":"搜房"]
static main(args) {
def task = new SynchDistrictPrice()
task.run()
@@ -14,8 +14,8 @@ class TestCrawlTotalCount {
def run(){
def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"]
for(def source :sources){
def count1 = findTotalCount(source, 202201);
def count2 = findTotalCount(source, 202202);
def count1 = findTotalCount(source, 202202);
def count2 = findTotalCount(source, 202203);
println source +" "+count1 +" "+count2
}
}
@@ -1,5 +1,7 @@
package com.fangjia.script.util
import org.apache.commons.lang.StringUtils
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
@@ -995,4 +997,42 @@ class DateUtils {
return rtn;
}
public static Date parseString2Date(String str){
if(StringUtils.isBlank(str)){
return null
}
str = str.replaceAll("^\\s+", "").replaceAll("\\s+\$", "")
Date now = new Date()
if(str.contains("小时")||str.contains("分钟") || str.contains("秒") || str.contains("当前")|| str.contains("今天")){
return org.apache.commons.lang3.time.DateUtils.truncate(new Date(), Calendar.DAY_OF_MONTH)
}else if(str.contains("昨天")) {
return add(now, -1)
}else if(str.contains("天")){
int days = Integer.parseInt(str.replaceAll("[^\\.0123456789]", ""))
return add(now, (-1)*days)
}else if(str.contains("月前")){
int month = Integer.parseInt(str.replaceAll("[^\\.0123456789]", ""))
return add(now, (-1)*month*30)
}else if(str.contains("年前")){
int years = Integer.parseInt(str.replaceAll("[^\\.0123456789]", ""))
return add(now, (-1)*years*365)
}else{
if(str.size()<10){
if(str.contains("-")){
str = (now.getYear()+1900)+"-"+str
}
}else{
if(str.contains("月") && str.contains("日") && !str.contains("年")){
str = (now.getYear()+1900)+"年"+str.substring(0, str.indexOf("日")+1)
}
}
if(str.contains("年")){
return DateUtils.parseFromTightByPattern(str, "yyyy年MM月dd日")
}else{
return DateUtils.parseFromTightByPattern(str, "yyyy-MM-dd")
}
}
}
}
@@ -11,8 +11,8 @@ import org.bson.Document
import static com.mongodb.client.model.Filters.*
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
def startDate = DateUtil.parseDate("20220201", "yyyyMMdd")
def endDate = DateUtil.parseDate("20220301", "yyyyMMdd")
def startDate = DateUtil.parseDate("20220301", "yyyyMMdd")
def endDate = DateUtil.parseDate("20220401", "yyyyMMdd")
Calendar calendar = Calendar.getInstance()
calendar.setTime(startDate)
def count = 0