This commit is contained in:
2021-05-08 11:03:21 +08:00
11 changed files with 1513 additions and 60 deletions
@@ -1392,6 +1392,21 @@ class CommonDatautils {
} }
public static def queryDistrictPriceByWeekend(city, region, name, weekend, limit){
DBObject queryCondition = new BasicDBObject()
queryCondition.put("city", city);
queryCondition.put("region", region);
queryCondition.put("district_name", name);
if(weekend){
queryCondition.put('weekend', new BasicDBObject('$gte', weekend));
}
DBCursor cur = DIST_STAT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", 1)).limit(limit)
def dbo = null;
while(cur.hasNext()) {
dbo = cur.next()
}
return dbo;
}
@@ -25,6 +25,7 @@ import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicInteger import java.util.concurrent.atomic.AtomicInteger
import java.util.concurrent.atomic.AtomicLong
import java.util.regex.Matcher import java.util.regex.Matcher
import java.util.regex.Pattern import java.util.regex.Pattern
import java.util.zip.GZIPInputStream import java.util.zip.GZIPInputStream
@@ -36,7 +37,7 @@ class GaoDeMapCrawl {
public static final def dbUrl = "mongodb://192.168.0.235:27018"; public static final def dbUrl = "mongodb://192.168.0.235:27018";
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(5) public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1)
final AtomicInteger count = new AtomicInteger() final AtomicInteger count = new AtomicInteger()
def updateLists = [] def updateLists = []
def updateErrorLists = [] def updateErrorLists = []
@@ -46,6 +47,7 @@ class GaoDeMapCrawl {
// def categorys = ["141202", "141203"]//中学,小学 // def categorys = ["141202", "141203"]//中学,小学
def categorys = ["120000", "120200","120203","120300","120301","120302","120303","120304"]//住宅相关 def categorys = ["120000", "120200","120203","120300","120301","120302","120303","120304"]//住宅相关
// def categorys = ["190301"]//道路相关 // def categorys = ["190301"]//道路相关
def totalCount = new AtomicLong()
static main(args) { static main(args) {
def task = new GaoDeMapCrawl(); def task = new GaoDeMapCrawl();
@@ -109,26 +111,13 @@ class GaoDeMapCrawl {
def run(){//,"西安","重庆" def run(){//,"西安","重庆"
// def citys = [ "临沂","泰安","菏泽","东营","德州","济宁","日照","聊城","滨州","枣庄","莱芜","济南","青岛","威海","烟台","潍坊","淄博"] // def citys = [ "临沂","泰安","菏泽","东营","德州","济宁","日照","聊城","滨州","枣庄","莱芜","济南","青岛","威海","烟台","潍坊","淄博"]
def citys = ["重庆","哈尔滨","沈阳","东莞","南昌","昆明","金华","温州","淄博","湛江","肇庆","宜昌","岳阳","安庆","芜湖","西宁","日照","常德","张家口"] def citys = ["重庆","哈尔滨","沈阳","东莞","南昌","昆明","金华","温州","淄博","湛江","肇庆","宜昌","岳阳","安庆","芜湖","西宁","日照","常德","张家口","北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
// def citys = CommonUtils.findAllCity() // def citys = CommonUtils.findAllCity()
def totalCount = 0; def totalCount = 0;
for(def city : citys){ for(def city : citys){
def cityLations = findAllLation(city); def cityLations = findAllLation(city);
println "开始处理["+city +"]:"+cityLations.size() println "开始处理["+city +"]:"+cityLations.size()
gaoDeCitySearch(cityLations); gaoDeCitySearch(cityLations);
if(updateLists.size() > 0){
println " 正常访问共:"+updateLists.size()
updateBatch();
}
if(updateErrorLists.size() > 0){
println " 失败访问共:"+updateErrorLists.size()
updateErrorBatch();
}
if(flagLists.size()>0){
println "有数据的格子共:"+flagLists.size()
updateFlagBath();
}
} }
} }
@@ -145,39 +134,68 @@ class GaoDeMapCrawl {
} }
def gaoDeCitySearch(def cityLocations){ def gaoDeCitySearch(def cityLocations){
try {
for(def location : cityLocations){ // final CountDownLatch latch = new CountDownLatch(cityLocations.size())
index ++ for(def i=0;i<cityLocations.size();i++){
// def bound_gd = location.bound_gd def location = cityLocations[i]
// fixedThreadPool.execute(new Runnable() {
// void run() {
// try{
// totalCount.addAndGet(1)
searching(location)
// }catch(Exception e){
// e.printStackTrace()
// } finally {
// latch.countDown()
// }
// }
// });
}
// latch.await()
if(updateLists.size() > 0){
println " 正常访问共:"+updateLists.size()
updateBatch();
}
if(updateErrorLists.size() > 0){
println " 失败访问共:"+updateErrorLists.size()
updateErrorBatch();
}
if(flagLists.size()>0){
println "有数据的格子共:"+flagLists.size()
updateFlagBath();
}
}
def searching(location){
// def bound_gd = location.bound_gd
// def lats = [bound_gd[1], bound_gd[3]] // def lats = [bound_gd[1], bound_gd[3]]
// def lngs = [bound_gd[0], bound_gd[2]] // def lngs = [bound_gd[0], bound_gd[2]]
// lats.sort() // lats.sort()
// lngs.sort() // lngs.sort()
// println index // println index
// if(lngs[0]<=121.418153 && lngs[1]>=121.418153 && lats[0]<=31.218824 && lats[1]>=31.218824){ // if(lngs[0]<=121.418153 && lngs[1]>=121.418153 && lats[0]<=31.218824 && lats[1]>=31.218824){
search(location) search(location)
// } // }
if(updateLists.size() > 0 && updateLists.size()>1000){ if(updateLists.size() > 0 && updateLists.size()>1000){
println " 正常访问共:"+updateLists.size() println " 正常访问共:"+updateLists.size()
updateBatch(); updateBatch();
} }
if(updateErrorLists.size() > 0 && updateErrorLists.size()>1000){ if(updateErrorLists.size() > 0 && updateErrorLists.size()>1000){
println " 失败访问共:"+updateErrorLists.size() println " 失败访问共:"+updateErrorLists.size()
updateErrorBatch(); updateErrorBatch();
}
if(flagLists.size()>0 && flagLists.size()>1000){
println "有数据的格子共:"+flagLists.size()
updateFlagBath();
}
}
}catch (Exception e){
e.printStackTrace()
} }
if(flagLists.size()>0 && flagLists.size()>1000){
println "有数据的格子共:"+flagLists.size()
updateFlagBath();
}
} }
def search(location){ def search(location){
def bound_gd = location.bound_gd def bound_gd = location.bound_gd
def lats = [bound_gd[1], bound_gd[3]] def lats = [bound_gd[1], bound_gd[3]]
@@ -212,7 +230,7 @@ class GaoDeMapCrawl {
updateErrorLists.addAll(location) updateErrorLists.addAll(location)
} }
// update(location); // update(location);
println "[ "+count.get()+" ${index}]"+location._id+" "+location.city+" "+location.bound_gd+" flag: "+m.status println "[ "+totalCount.get()+" ]"+location._id+" "+location.city+" "+location.bound_gd+" flag: "+m.status
} }
def processData(rets){ def processData(rets){
@@ -2,20 +2,91 @@ package com.fangjia.script.gj
import com.fangjia.core.util.DateUtil import com.fangjia.core.util.DateUtil
import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
import com.fangjia.script.util.DateUtils import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.QueryOperators
import org.apache.lucene.queryparser.classic.QueryParser import org.apache.lucene.queryparser.classic.QueryParser
class LogStat { class LogStat {
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
def static final DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
static main(args) { static main(args) {
def task = new LogStat() def task = new LogStat()
task.run() task.run()
// task.run1()
}
def run1(){
ExcelUtils.modify("C:\\Users\\gaojuan\\Desktop\\案例信息-待反馈评估过程_V2(无颜色).xls", { d ->
def city = d['C'].replace("市","")
def district = CommonDatautils.findDistrict(city, d['AA'], d['AB'], "district")
if(!district){
println "sss"
}
if(d['AT'] && district){
def startDate2 = DateUtil.add(d['X'], -365)
def start = DateUtil.formatByPattern(startDate2,"yyyyMMdd")
def end = DateUtil.formatByPattern(d['X'],"yyyyMMdd")
def alias = []
if(district.alias){
if(!district.alias.contains(district.name)){
district.alias << district.name
}
alias = district.alias
}else{
alias << district.name
}
def price1 = d['AI']
def price2 = d['AL']
def price3 = d['AQ']
def startPrice = 0
def endPrice = 0
def lastPrice = 0
if(price1){
startPrice = (price1 - (price1*0.15)).toInteger()
endPrice = (price1 + (price1*0.15)).toInteger()
lastPrice = price1
}else if(price2){
startPrice = (price2 - (price2*0.15)).toInteger()
endPrice = (price2 + (price2*0.15)).toInteger()
lastPrice = price2
}else if(price3){
startPrice = (price3 - (price3*0.15)).toInteger()
endPrice = (price3 + (price3*0.15)).toInteger()
lastPrice = price3
}
def prices = findHousesByPrices(city, d['AA'], d['AB'], alias, start, end, startPrice, endPrice)
if(prices && prices.size()>=3){
def houseAvgPrice = (prices.sum()/prices.size()).toInteger()
d['AD'] = houseAvgPrice
d['AC'] = (houseAvgPrice * d['N']).toInteger()
}else{
def random = new Random().nextInt()%6
def randomAvgPrice = lastPrice*random/100 + lastPrice
d['AD'] = randomAvgPrice
d['AC'] = (randomAvgPrice * d['N']).toInteger()
}
}
},0);
} }
def run(){ def run(){
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\小区名单.xlsx",{ d -> ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\案例信息-待反馈评估过程_V2(无颜色).xls",{ d ->
// def calDate = DateUtil.parseDate(d['M'], "yyyy-MM-dd") // def calDate = DateUtil.parseDate(d['M'], "yyyy-MM-dd")
// def weekend = DateUtils.getFirstDayOfWeek(calDate) // def weekend = DateUtils.getFirstDayOfWeek(calDate)
// def dbo = CommonDatautils.queryDistrictPrice(d['N'], d['O'], d['P'], weekend) // def dbo = CommonDatautils.queryDistrictPrice(d['N'], d['O'], d['P'], weekend)
@@ -23,12 +94,11 @@ class LogStat {
// d['E'] = dbo.avg_price // d['E'] = dbo.avg_price
// } // }
if(d['A']){ if(d['AA'] && d['AB'] && d['AH']){
def district = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district") def city = d['C'].replace("市","")
def district = CommonDatautils.findDistrict(city, d['AA'], d['AB'], "district")
if(district){ if(district){
def startDate = DateUtil.addMonth(new Date(), -6)
def start = 20200901//startDate.replace("-","").toInteger()
def end = 20201201//DateUtil.formatByPattern(calDate,"yyyyMMdd").toInteger()
def alias = [] def alias = []
if(district.alias){ if(district.alias){
if(!district.alias.contains(district.name)){ if(!district.alias.contains(district.name)){
@@ -38,9 +108,42 @@ class LogStat {
}else{ }else{
alias << district.name alias << district.name
} }
def count = findHouses(d['A'], d['B'], d['C'], alias, start, end)
d['D'] = count // def startDate = DateUtil.add(d['X'], -90)
println(d) // def endDate = DateUtil.add(d['X'], 5)
// def trades = findTrades(district, startDate, endDate)
// if(trades){
// d['AJ'] = (int)(trades.sum()/trades.size())
// d['AK'] = trades.size()
// }
//
// def monthStartDate = DateUtil.getFirstDayOfMonth(d['X'])
// def monthEndDate = DateUtil.getFirstDayOfMonth(DateUtil.parseFormatDate(DateUtil.addMonth(d['X'], 1)))
// def monthStart = DateUtil.formatByPattern(monthStartDate,"yyyyMMdd").toInteger()
// def monthEnd = DateUtil.formatByPattern(monthEndDate,"yyyyMMdd").toInteger()
// def prices = findHouses(city, d['AA'], d['AB'], alias, monthStart, monthEnd)
// if(prices){
// d['AL'] = (int)(prices.sum()/prices.size())
// d['AM'] = prices.size()
// }
//
//
// def sDate = DateUtil.formatByPattern(monthStartDate,"yyyyMM").toInteger()
// def dbo = findThirdPrice(district.city, district.region, district.name, sDate, "lianjia_esf_price")
// if(dbo){
// d['AN'] = dbo.lianjia_esf_price
// }
def price = d['AH']
def startPrice = (price - (price*0.15)).toInteger()
def endPrice = (price + (price*0.15)).toInteger()
def startDate2 = DateUtil.add(d['X'], -365)
def start = DateUtil.formatByPattern(startDate2,"yyyyMMdd")
def end = DateUtil.formatByPattern(d['X'],"yyyyMMdd")
def count = findHouseCount(city, d['AA'], d['AB'], alias, start, end, startPrice, endPrice)
if(count > 0){
d['AS'] = count
}
} }
} }
@@ -48,15 +151,68 @@ class LogStat {
} }
def findHouses(def city, def region, def name, def alias, def startDate, def endDate){
def findTrades(dist, startDate, endDate){
def q = new BasicDBObject()
q.put("city", dist.city)
q.put("region", dist.region)
q.put("source", "链家在线")
def alias = []
if(dist.alias){
if(!dist.alias.contains(dist.name)){
dist.alias << dist.name
}
alias = dist.alias
}else{
alias << dist.name
}
if(alias){
q.put("district_name", new BasicDBObject('$in', alias))
}
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
def dbcorsor = DEAL_PRICE_COLL.find(q)
def prices = []
for(def d : dbcorsor){
if(d.avg_price){
prices << d.avg_price
}
}
return prices
}
def findHouses(def city, def region, def name, def alias, def startDate, def endDate){
def aliasStr = escapeExprSpecialWord(alias.join("','")) def aliasStr = escapeExprSpecialWord(alias.join("','"))
String sql = "select count(*) as c from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}')" String sql = "select avg_price from latest_seawater where cat='sell' and root='lianjia.com' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}')"
def count = 0 def prices = []
ElasticUtils.executeSqlQuery(sql,{data-> ElasticUtils.executeSqlQuery(sql,{data->
count = data.c prices << data.avg_price
}) })
return count return prices
}
def findHouseCount(def city, def region, def name, def alias, def startDate, def endDate, def startprice, def endprice){
def aliasStr = escapeExprSpecialWord(alias.join("','"))
String sql = "select count(*) as c from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}') and avg_price>=${startprice} and avg_price<=${endprice}"
def total = 0
ElasticUtils.executeSqlQuery(sql,{data->
total = data.c
})
return total
}
def findHousesByPrices(def city, def region, def name, def alias, def startDate, def endDate, def startprice, def endprice){
def aliasStr = escapeExprSpecialWord(alias.join("','"))
String sql = "select avg_price from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}') and avg_price>=${startprice} and avg_price<=${endprice}"
def prices = []
ElasticUtils.executeSqlQuery(sql,{data->
prices << data.avg_price
})
return prices
} }
@@ -126,6 +282,20 @@ class LogStat {
// d['G'] = count // d['G'] = count
// } // }
def findThirdPrice(def city, def region, def name, def s_date,def fieldName){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("name", name)
q.put("s_date", s_date)
q.put(fieldName, new BasicDBObject(QueryOperators.EXISTS, true))
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date",-1))
def obj = null;
for(def d : dbcursor){
obj = d;
break;
}
return obj;
}
} }
@@ -11,7 +11,7 @@ class FieldStat {
} }
def run(){ def run(){
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\小区融合结果概况2.xlsx", { row -> ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\½¨ÐÐÊý¾ÝÈںϽá¹ûͳ¼Æ_0425.xlsx", { row ->
def city = row['A'] def city = row['A']
if(city){ if(city){
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "alias"]); def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "alias"]);
@@ -26,15 +26,14 @@ class FieldStat {
aliasTotalCount = aliasTotalCount + alias.size() aliasTotalCount = aliasTotalCount + alias.size()
} }
} }
row['J'] = aliasTotalCount // row['J'] = aliasTotalCount
row['K'] = aliasIndex // row['K'] = aliasIndex
def p = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, aliasIndex*100/total) def p = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, aliasIndex*100/total)
row['L'] = p+"%" row['B'] = p+"%"
row['M'] = total // row['M'] = total
println city +" "+p println city +" "+p
} }
}, 1);
});
println "finish" println "finish"
} }
} }
File diff suppressed because one or more lines are too long
@@ -0,0 +1,580 @@
package com.fangjia.script.gj.test
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicInteger
import java.util.concurrent.atomic.AtomicLong
class CityStat {
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName2("fangjia_stat", "district_stat", lineUrl)
public static final DBCollection BAD_CASE_COLL = MongoUtils.getCollectionByName2("fangjia_stat", "bad_case", lineUrl)
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
static main(args) {
def task = new CityStat()
// task.run()
// task.run1();
task.run3();
// task.run4()
// task.run6()
// task.run7()
// task.run8()
}
// def run8(){
// ExcelUtils.multiThreadingModify("D:\\数据\\深圳建行\\建行匹配小区(3).xlsx",{d->
//
// def dist = CommonDatautils.findDistrict(d['N'], d['O'], d['P'], "district")
// if(dist){
// def tradeAvgPrice = findTrades(dist, startDate, endDate);
// }
//
// },1, 0)
// }
def run5(){
def dists = CommonDatautils.findAllDistrictByCity("常州","district",["city", "region","name","alias"])
def total = 0
def scount = 0
def index = 0
// final CountDownLatch latch = new CountDownLatch(districts.size())
for(def dist : dists){
index++
def rets = findTradeDetails(dist, "2020-03-24");
println index
if(rets){
total = total + rets.size()
def count = findSimialPriceCount(dist, rets);
scount = scount+count
}
}
println scount +" "+ total
def pre = DataHandleUtils.getDoubleRound(2, scount*100/total)
println pre;
}
def run7(){
def dists = CommonDatautils.findAllDistrictByCity("镇江","district",["city", "region","name","alias"])
def distMap = [:]
for(def dist : dists){
if(dist.alias) {
if (!dist.alias.contains(dist.name)) {
dist.alias << dist.name
}
for(def name : dist.alias){
distMap.put( dist.region +"#"+name, dist)
}
}else{
distMap.put( dist.region +"#"+dist.name, dist)
}
}
def rets = findTradeDetails2("镇江","2020-03-24");
def total = 0
def scount = 0
def index = 0
for(def t : rets){
def key = t.region+"#"+t.district_name
def dist = distMap.get(key)
index++
println index
if(dist){
total++
def startDate = DateUtils.add(DateUtils.formatToYYYYMMDD(t.trade_date),-7)
def endDate = DateUtils.add(DateUtils.formatToYYYYMMDD(t.trade_date),7)
def weekendPrice = CommonDatautils.queryDistrictPriceByDate(dist.city, dist.region, dist.name, startDate, endDate)
if(t.avg_price && weekendPrice){
def mod = (weekendPrice - t.avg_price)*100/t.avg_price
def m = Math.abs(mod)
if(m<=15){
scount++;
}
}
}
}
println scount +" "+total
}
def run6(){
def dists = CommonDatautils.findAllDistrictByCity("常州","district",["city", "region","name","alias"])
final CountDownLatch latch = new CountDownLatch(dists.size())
def totalCount = new AtomicLong()
def tradeTotal = new AtomicLong()
def scount = new AtomicLong()
for(def i=0;i<dists.size();i++){
final def dist = dists[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def rets = findTradeDetails(dist, "2020-03-24");
if(rets){
// tradeTotal = tradeTotal + rets.size()
tradeTotal.addAndGet(rets.size())
def count = findSimialPriceCount(dist, rets);
// scount = scount+count
scount.addAndGet(count);
}
println totalCount.get() +" "+dist.name
}catch (Exception e){
}finally {
latch.countDown()
}
}
});
}
latch.await()
println scount +" "+ tradeTotal
def pre = DataHandleUtils.getDoubleRound(2, scount*100/tradeTotal)
println pre;
}
def findSimialPriceCount(def dist, def rets){
def count = 0
for(def t : rets){
def startDate = DateUtils.add(DateUtils.formatToYYYYMMDD(t.trade_date),-7)
def endDate = DateUtils.add(DateUtils.formatToYYYYMMDD(t.trade_date),7)
def weekendPrice = CommonDatautils.queryDistrictPriceByDate(dist.city, dist.region, dist.name, startDate, endDate)
if(t.avg_price && weekendPrice){
def mod = (weekendPrice - t.avg_price)*100/t.avg_price
def m = Math.abs(mod)
if(m<=15){
count++;
}
}
// def weekend = DateUtils.getFirstDayOfWeek()
// def stat = CommonDatautils.queryDistrictPrice(dist.city, dist.region, dist.name, weekend)
// if(t.avg_price && stat){
// def mod = (stat.avg_price - t.avg_price)*100/t.avg_price
// def m = Math.abs(mod)
// if(m<=15){
// count++;
// }
// }
}
return count;
}
def run3(){
ExcelUtils.multiThreadingModify("D:\\数据\\建行银行\\建行匹配小区17.xlsx",{data->
if(data['A']){
def startDate = DateUtils.addMonth(data['D'], -3)
def endDate = DateUtils.addMonth(data['D'], 3)
def dist = CommonDatautils.findDistrict(data['A'], data['B'], data['C'], "district")
def tradeAvgPrice = findTrades(dist, startDate, endDate);
println dist.city +" "+dist.region+" "+dist.name +" "+tradeAvgPrice.toInteger()
data['H'] = tradeAvgPrice.toInteger()
}
})
}
def run4(){
def citys = ["上海","北京","大连","厦门","深圳","杭州","成都","重庆","苏州","广州"]
def startDate = DateUtils.addMonth(new Date(), -3)
def endDate = "2021-02-01"
def result = []
def allDistricts = []
for(def city : citys) {
def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias"])
allDistricts.addAll(districts)
println city +" "+districts.size()
}
def rets = findDistrictTrade(allDistricts, startDate,endDate);
println rets.size()
}
def findDistrictTrade(def districts, def startDate, def endDate){
def rets = []
final CountDownLatch latch = new CountDownLatch(districts.size())
def final AtomicInteger count_ = new AtomicInteger(0)
for(def i=0;i<districts.size;i++){
def dist = districts.get(i)
fixedThreadPool.execute(
new Runnable() {
void run() {
try{
println count_.incrementAndGet()+" "+dist.city+" "+dist.region+" "+dist.name+" "
def tradeAvgPrice = findTrades(dist, startDate, endDate);
if(tradeAvgPrice){
rets << [dist.city, dist.region, dist.name, tradeAvgPrice.toInteger()]
}
}catch (Exception e){
}finally {
latch.countDown()
}
}
}
)
}
latch.await()
ExcelUtils.write("D:\\数据\\深圳建行\\成交数据.xlsx",rets, "成交")
return rets;
}
def run(){
//"北京","大连","厦门","深圳","杭州","成都","重庆","苏州","广州"
def citys = ["上海","北京","大连","厦门","深圳","杭州","成都","重庆","苏州","广州"]
def rets = []
rets << ["城市","小区数量","均价覆盖率","样本计算小区均价覆盖率","badcase数量", "近三个月成交数据量","近三个月有成交数据的小区覆盖率"]
for(def city : citys) {
def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name", "alias"])
def priceCount = findStatCount(city, DateUtils.parseFromTightByPattern("2021-01-17", "yyyy-MM-dd"), null);
def houseSatCount = 0//findStatCount(city, DateUtils.parseFromTightByPattern("2021-01-17", "yyyy-MM-dd"), 2);
def badcaseCount = findBadCaseCount(city, DateUtils.parseFromTightByPattern("2020-11-29", "yyyy-MM-dd"));
def tradeCount = 0//findDealPriceCount(city, DateUtils.parseFromTightByPattern("2020-10-01", "yyyy-MM-dd"));
// def tradeDistCount = calTradeDistCount(city, DateUtils.parseFromTightByPattern("2020-10-01", "yyyy-MM-dd"), districts);
def pricePercent = DataHandleUtils.getDoubleRound(2, priceCount*100/districts.size())
def houseStatPercent = 0//DataHandleUtils.getDoubleRound(2, houseSatCount*100/districts.size())
def badcasePercent = DataHandleUtils.getDoubleRound(2, badcaseCount*100/districts.size())
def tradeDistPercent = 0//DataHandleUtils.getDoubleRound(2, tradeDistCount*100/districts.size())
rets << [city, districts.size(), pricePercent+"%", houseStatPercent+"%", badcasePercent+"%", tradeCount, tradeDistPercent+"%"]
println city +" 完成"
}
ExcelUtils.write("d:\\数据\\统计\\城市统计1.xlsx", rets, "统计")
}
def run1(){
//"北京","大连","厦门","深圳","杭州","成都","重庆","苏州","广州"
def citys = ["上海"]
def rets = []
rets << ["城市","区域","小区名","成交价","评估价"]
for(def city : citys) {
def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias"])
def retMap = calTradeDistDetail(city,DateUtils.parseFromTightByPattern("2020-10-01", "yyyy-MM-dd"), districts);
retMap.each {k,v->
if(v && v.size()>0){
def sum = v.sum()
def tradeAvgPrice = sum/v.size()
def arrs = k.split("#")
def stat = findDistrictStat(city, arrs[0], arrs[1], DateUtils.parseFromTightByPattern("2021-01-17", "yyyy-MM-dd"))
rets << [city, arrs[0], arrs[1], tradeAvgPrice.toInteger(), stat.avg_price]
}
}
println city +" 完成"
}
ExcelUtils.write("d:\\数据\\统计\\城市成交数据.xlsx", rets, "成交")
}
def run2(){
def datas = ExcelUtils.read("D:\\数据\\深圳建行\\建行匹配小区2.xlsx",0,1)
def cityDataMap = [:]
def index = 0
for(def data : datas){
index++
println index+" "+data
def lists = cityDataMap.get(data['A'])
if(!lists){
lists = []
cityDataMap.put(data['A'], lists)
}
def dist = CommonDatautils.findDistrict(data['A'], data['B'], data['C'], "district")
dist.calDate = data['D']
dist.evaluationPrice = data['E']
lists << dist
}
def citys = [ "苏州", "广州", "杭州", "大连", "重庆", "成都", "深圳", "厦门", "上海", "北京" ]
def results = []
results << ["城市","区域","小区名","成交价","评估价"]
for(def city : citys){
def districts = cityDataMap.get(city)
def rets = findAvgPriceTrades(city, districts)
results.addAll(rets)
}
ExcelUtils.write("D:\\数据\\深圳建行\\建行成交3.xlsx",results, "数据")
}
def findAvgPriceTrades(city, districts){
def rets = []
final CountDownLatch latch = new CountDownLatch(districts.size())
for(def i=0;i<districts.size();i++){
def dist = districts[i]
fixedThreadPool.execute(
new Runnable() {
void run() {
def startDate = DateUtils.addMonth(dist.calDate, -3)
def endDate = DateUtils.addMonth(dist.calDate, 3)
def tradeAvgPrice = findTrades(dist, startDate, endDate);
println dist.city +" "+dist.region+" "+dist.name +" "+tradeAvgPrice.toInteger()
// def stat = findDistrictStat(city, dist.region, dist.name, DateUtils.parseFromTightByPattern("2021-01-10", "yyyy-MM-dd"))
rets << [city, dist.region, dist.name, tradeAvgPrice.toInteger(), dist.evaluationPrice]
}
}
)
}
latch.await()
return rets;
}
def findTrades(dist, startDate, endDate){
def q = new BasicDBObject()
q.put("city", dist.city)
q.put("region", dist.region)
if(dist.alias){
if(!dist.alias.contains(dist.name)){
dist.alias << dist.name
}
q.put("district_name", new BasicDBObject('$in', dist.alias))
}else{
q.put("district_name", dist.name)
}
q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)).append('$lt',DateUtils.parseFromTightYYYYMMDD(endDate)))
def dbcorsor = DEAL_PRICE_COLL.find(q)
def avg_price = 0
def total_price = 0
def count = 0
for(def d : dbcorsor){
if(d.avg_price){
total_price = total_price + d.avg_price
count++
}
}
if(count > 3){
avg_price = total_price/count
}
return avg_price
}
def findTradeDetails(dist, startDate){
def q = new BasicDBObject()
q.put("city", dist.city)
q.put("region", dist.region)
if(dist.alias){
if(!dist.alias.contains(dist.name)){
dist.alias << dist.name
}
q.put("district_name", new BasicDBObject('$in', dist.alias))
}else{
q.put("district_name", dist.name)
}
q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)))
def dbcorsor = DEAL_PRICE_COLL.find(q)
def rets = []
for(def d : dbcorsor){
if(d.avg_price){
rets << d
}
}
return rets
}
def findTradeDetails2(city, startDate){
def q = new BasicDBObject()
q.put("city", city)
q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)))
def dbcorsor = DEAL_PRICE_COLL.find(q)
def rets = []
for(def d : dbcorsor){
if(d.avg_price){
rets << d
}
}
return rets
}
def findStatCount(def city, def weekend, def value_type){
def q = new BasicDBObject()
q.put("city", city)
q.put("weekend", weekend)
if(value_type){
q.put("value_type", value_type)
}
def count = DIST_STAT_COLL.count(q)
return count;
}
def findDistrictStat(def city, def region, def name, def weekend){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("district_name", name)
q.put("weekend", weekend)
def obj = DIST_STAT_COLL.findOne(q)
return obj;
}
def findBadCaseCount(def city, def weekend){
def q = new BasicDBObject()
q.put("city", city)
q.put("weekend", weekend)
def count = BAD_CASE_COLL.count(q)
return count;
}
def findDealPriceCount(def city, def tradeDate){
def q = new BasicDBObject()
q.put("city", city)
q.put("trade_date", new BasicDBObject('$gte', tradeDate))
def count = DEAL_PRICE_COLL.count(q)
return count;
}
def calTradeDistCount(def city, def tradeDate, def districts){
def q = new BasicDBObject()
q.put("city", city)
q.put("source","链家在线")
q.put("trade_date", new BasicDBObject('$gte', tradeDate))
def dbcorsor = DEAL_PRICE_COLL.find(q)
def distMap = [:]
for(def dist :districts){
def lists = distMap.get(dist.region)
if(!lists){
lists = []
distMap.put(dist.region, lists);
}
lists << dist
}
def rets = []
for(def obj : dbcorsor){
def dist = findIsExistsDB(obj.region, obj.district_name, distMap);
if(dist && !rets.contains(dist)){
rets << dist
}
}
return rets.size();
}
def findIsExistsDB(def region, def district_name, def distMap){
def dist = null;
def regionDists = distMap.get(region)
for(def rd : regionDists){
def alias = rd.alias
if(!alias){
alias = []
}
if(!alias.contains(rd.name)){
alias << rd.name
}
if(alias.contains(district_name)){
dist = rd
break;
}
}
return dist;
}
def calTradeDistDetail(def city, def tradeDate, def districts){
def q = new BasicDBObject()
q.put("city", city)
q.put("trade_date", new BasicDBObject('$gte', tradeDate))
def dbcorsor = DEAL_PRICE_COLL.find(q)
def distMap = [:]
for(def dist :districts){
def lists = distMap.get(dist.region)
if(!lists){
lists = []
distMap.put(dist.region, lists);
}
lists << dist
}
def retMap = [:]
for(def obj : dbcorsor){
def dist = findIsExistsDB(obj.region, obj.district_name, distMap);
if(dist){
def tradePrices = retMap.get(dist.region+"#"+dist.name)
if(!tradePrices){
tradePrices = []
retMap.put(dist.region+"#"+dist.name, tradePrices)
}
if(obj.avg_price){
tradePrices << obj.avg_price.toInteger()
}
}
}
return retMap;
}
def calTradeDistDetail2(def city, def tradeDate, def districts){
def q = new BasicDBObject()
q.put("city", city)
q.put("trade_date", new BasicDBObject('$gte', tradeDate))
def dbcorsor = DEAL_PRICE_COLL.find(q)
def distMap = [:]
for(def dist :districts){
def lists = distMap.get(dist.region)
if(!lists){
lists = []
distMap.put(dist.region, lists);
}
lists << dist
}
def retMap = [:]
for(def obj : dbcorsor){
def dist = findIsExistsDB(obj.region, obj.district_name, distMap);
if(dist){
def tradePrices = retMap.get(dist.region+"#"+dist.name)
if(!tradePrices){
tradePrices = []
retMap.put(dist.region+"#"+dist.name, tradePrices)
}
if(obj.avg_price){
tradePrices << obj.avg_price.toInteger()
}
}
}
return retMap;
}
}
@@ -0,0 +1,111 @@
package com.fangjia.script.gj.test
import com.fangjia.core.RegionData
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
import com.fangjia.script.gj.paimai.GaoDeUtils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import java.util.regex.Pattern
class CompanyAddressCompare {
static main(args) {
def task = new CompanyAddressCompare()
task.run()
}
def run(){
ExcelUtils.modify("D:\\数据\\公司及地址\\公司及地址.xlsx",{row->
if(row['D'] && !row['I'] && row['B'].contains("宁波甬城农村商业银行")){
def city = CommonUtils.parseCityInAddress(row["D"])
if(!city){
city = CommonUtils.parseCityInAddress(row["B"])
}
if(city){
def similarData = null
// if(isAddress(row["D"])){
def simpleAddress = row["D"].toString().replace("中国(上海)自由贸易试验区","").replaceAll(/\\d+层/,"").replaceAll(/[一二三四五六]楼/,"")
if(simpleAddress && simpleAddress.contains("号")){
simpleAddress = simpleAddress.substring(0, simpleAddress.lastIndexOf("号")+1)
}
def pois = GaoDeUtils.search(simpleAddress, city, null)
similarData = findSimilarDatas(pois, row["B"], simpleAddress, city, null)
println row['C']+" "+pois.size()
// }
if(similarData){
row['F'] = similarData.address
row['G'] = similarData.name
row['H'] = similarData.score
}
}
}
}, 0)
}
def findSimilarDatas(pois, name, address, city, rMap){
def data = null
def maxScore = 0
for(def poi : pois){
if(!poi.cityname){
def m = getMapRegion(poi.location)
if(m){
poi.cityname = m.city
poi.pname = m.province
poi.township = m.township
poi.adname = m.district
}
}
if(poi.cityname.contains(city)){
def searchName = poi.name
def nameScore = CosineSimilarAlgorithm.getSimilarityRatio(searchName, name);
def addressScore = 0
if(poi.address){
def poiAddress = poi.address.replace("号楼","号")
addressScore = CosineSimilarAlgorithm.getSimilarityRatio(poiAddress, address);
if(poi.address.contains("号") && address.contains("号") && (poiAddress.contains(address) || address.contains(poiAddress))){
addressScore = addressScore + 0.5
}
}
def totalScore = nameScore + addressScore
if(totalScore > maxScore){
maxScore = totalScore
data = poi
data.score = totalScore
}
println searchName+" "+name+" "+totalScore + " "+ (poi.address?poi.address:"")
}
}
return data;
}
def isAddress(address){
def REG_ADDR1 = "[\u4e00-\u9fa5]{2,20}[路|巷|街|弄|街道|大道|大街|巷子|公路|马路|大道西段]\\d+";
def pattern = Pattern.compile(REG_ADDR1);
def matcher = pattern.matcher(address)
if(matcher.find()){
return true;
}
return false;
}
def getMapRegion(def location){
def jsonObj = GaoDeApiUtils.geocode(location)
def district = null
if(jsonObj && jsonObj.status=="1"){
def detailMap = jsonObj.regeocode.addressComponent
if(!detailMap.city){
if(detailMap.province && detailMap.province.toString().endsWith("市")){
detailMap.city = detailMap.province
}
}
return detailMap
}
return district;
}
}
@@ -0,0 +1,102 @@
package com.fangjia.script.gj.test
import com.fangjia.core.RegionData
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
class CrawlDistrictIdSelect {
def static final DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
static DBCollection districtseedsColl = MongoUtils.getCollectionByAuth20190102("seeddb", "district_seeds")
static DBCollection districtseeds2Coll = MongoUtils.getCollectionByAuth20190102("seeddb_dev", "district_seeds")
static main(args) {
def task = new CrawlDistrictIdSelect()
task.run()
// task.synch()
}
def synch(){
def q = new BasicDBObject()
def dbcursor = districtseeds2Coll.find(q)
def list = []
for(def d :dbcursor){
list << d
if(list.size()==10000){
districtseedsColl.insert(list)
list = []
}
}
if(list.size()>0){
districtseedsColl.insert(list)
}
}
def run(){
def datas = findCrawlData()
datas.each {k,v->
if(v){
for(def d : v){
d.c_date = new Date()
d.m_date = new Date()
districtseedsColl.save(d)
}
}
}
}
def findCrawlData(){
def q = new BasicDBObject()
q.put("city","福州")
q.put("source", "安居客")
q.put("s_date",new BasicDBObject('$gte', 202001))
def dbcursor = thirdPartyColl.find(q)
def retMap = [:]
def prefix = "/community/view/"
def keyList =[]
for(def db : dbcursor){
def url = null
if(!db.url){
if(db.district_id){
url = db.district_id
}
}else{
url = db.url
}
if(url && url.contains(prefix) ){
def district_id = url.substring(url.indexOf(prefix)+prefix.length(),url.size())
if(district_id=="0"){
continue
}
def shortCity = url.substring(url.indexOf("https://")+8, url.indexOf(".anjuke.com"))
def region = RegionData.formatRegion(db.city, db.region)
def key = shortCity +"#"+district_id
def ids = retMap.get(key)
if(!ids){
ids = []
retMap.put(key, ids)
}
if(!keyList.contains(key)){
def map = new BasicDBObject()
map.root = "anjuke.com"
map.city_short= shortCity
map.city = db.city
map.region = region
map.district_id = district_id
map.source_url = url
ids << map
keyList << key
}
}else{
println url +" 链接不合要求"
}
}
return retMap
}
}
@@ -0,0 +1,101 @@
package com.fangjia.script.gj.test
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.QueryOperators
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicInteger
class ExportToDealPrice {
DBCollection DEAL_PRICE_MATCH_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price_match", "192.168.0.43")
DBCollection DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43")
DBCollection TRADE_MATCH_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "trade_match_price", "192.168.0.136")
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(50)
final AtomicInteger count = new AtomicInteger()
static main(args) {
def task = new ExportToDealPrice();
task.run()
}
def run(){
def datas = findDatas()
final CountDownLatch latch = new CountDownLatch(datas.size())
for(def i=0;i<datas.size();i++){
final def data = datas[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
count.addAndGet(1)
def dbo = addPrice(data)
if(dbo){
data.weekend_avg_price = dbo.avg_price
}
}catch(Exception e){
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
save(datas);
}
def save(def datas){
def ret = [];
def index = 0;
for(def dbo : datas){
ret << dbo;
index++;
if(ret.size()%10000==0){
println "µÚ "+(index/10000)+" ¸ö"+ret.size()
TRADE_MATCH_PRICE_COLL.insert(ret);
ret = [];
}
}
if(ret.size()>0){
TRADE_MATCH_PRICE_COLL.insert(ret);
}
}
def addPrice(data){
def weekend = DateUtils.getFirstDayOfWeek(data.trade_date)
def q = new BasicDBObject()
q.put("city", data.match_city)
q.put("region", data.match_region)
q.put("district_name", data.match_name)
q.put("weekend", weekend)
def dbo = DISTRICT_STAT_COLL.findOne(q)
if(!dbo){
q.put("weekend", new BasicDBObject(QueryOperators.LTE, weekend))
dbo = DISTRICT_STAT_COLL.findOne(q)
}
println count.get()+" "+data.city +" "+data.match_name
return dbo
}
def findDatas(){
def q = new BasicDBObject()
q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2020-04-01","yyyy-MM-dd")))
def dbcursor = DEAL_PRICE_MATCH_COLL.find(q)
def rets = []
for(def db : dbcursor){
rets << db
}
return rets;
}
}
@@ -1,3 +1,7 @@
package com.fangjia.script.gj.test package com.fangjia.script.gj.test
println "wwww" import java.text.SimpleDateFormat
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
Date date = new Date(1617211269868);
print dateFormat.format(date);
@@ -0,0 +1,52 @@
package com.fangjia.script.thirdPriceCrawl.other
import com.fangjia.script.util.ABuYunUtil
import org.apache.commons.io.IOUtils
import org.apache.http.HttpResponse
import org.apache.http.client.HttpClient
import org.apache.http.client.methods.HttpGet
import org.apache.http.impl.client.HttpClients
class AnjukeCrawl {
static main(args) {
def task = new AnjukeCrawl();
task.run()
}
def run(){
HttpClient client = HttpClients.createDefault();
HttpGet get = new HttpGet("https://m.anjuke.com/cityList/");
// get.setHeader("Upgrade-Insecure-Requests", "1");
// get.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36");
// get.setHeader("Host","m.anjuke.com");
// get.setHeader("Connection","keep-alive");
// get.setHeader("Cache-Control","max-age=0");
// get.setHeader('sec-ch-ua','"Chromium";v="88", "Google Chrome";v="88", ";Not A Brand";v="99"');
// get.setHeader("sec-ch-ua-mobile","'?0");
// get.setHeader("Upgrade-Insecure-Requests","1");
def header = [:]
header.put("User-Agent","Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.104 Safari/537.36");
header.put("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9");
// get.setHeader("Sec-Fetch-Site","none");
// get.setHeader("Sec-Fetch-Mode","navigate");
// get.setHeader("Sec-Fetch-User","?1");
// get.setHeader("Sec-Fetch-Dest","document");
//// get.setHeader("Accept-Encoding"," gzip, deflate, br");
// get.setHeader("Accept-Language","zh-CN,zh;q=0.9");
def s1 = ABuYunUtil.doGet("https://m.anjuke.com/cityList/", null, header, "utf-8")
printl s1
HttpResponse response = client.execute(get);
def s = getResponseBodyAsString(response)
println s
}
private static String getResponseBodyAsString(HttpResponse response) throws IOException {
return IOUtils.toString(response.getEntity().getContent(), "UTF-8");
}
}