Merge branch 'master' of http://gitlab.fangjia.com/dev0/main
This commit is contained in:
+15
@@ -1392,6 +1392,21 @@ class CommonDatautils {
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
public static def queryDistrictPriceByWeekend(city, region, name, weekend, limit){
|
||||||
|
DBObject queryCondition = new BasicDBObject()
|
||||||
|
queryCondition.put("city", city);
|
||||||
|
queryCondition.put("region", region);
|
||||||
|
queryCondition.put("district_name", name);
|
||||||
|
if(weekend){
|
||||||
|
queryCondition.put('weekend', new BasicDBObject('$gte', weekend));
|
||||||
|
}
|
||||||
|
DBCursor cur = DIST_STAT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", 1)).limit(limit)
|
||||||
|
def dbo = null;
|
||||||
|
while(cur.hasNext()) {
|
||||||
|
dbo = cur.next()
|
||||||
|
}
|
||||||
|
return dbo;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
+55
-37
@@ -25,6 +25,7 @@ import java.util.concurrent.CountDownLatch
|
|||||||
import java.util.concurrent.ExecutorService
|
import java.util.concurrent.ExecutorService
|
||||||
import java.util.concurrent.Executors
|
import java.util.concurrent.Executors
|
||||||
import java.util.concurrent.atomic.AtomicInteger
|
import java.util.concurrent.atomic.AtomicInteger
|
||||||
|
import java.util.concurrent.atomic.AtomicLong
|
||||||
import java.util.regex.Matcher
|
import java.util.regex.Matcher
|
||||||
import java.util.regex.Pattern
|
import java.util.regex.Pattern
|
||||||
import java.util.zip.GZIPInputStream
|
import java.util.zip.GZIPInputStream
|
||||||
@@ -36,7 +37,7 @@ class GaoDeMapCrawl {
|
|||||||
|
|
||||||
public static final def dbUrl = "mongodb://192.168.0.235:27018";
|
public static final def dbUrl = "mongodb://192.168.0.235:27018";
|
||||||
|
|
||||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(5)
|
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1)
|
||||||
final AtomicInteger count = new AtomicInteger()
|
final AtomicInteger count = new AtomicInteger()
|
||||||
def updateLists = []
|
def updateLists = []
|
||||||
def updateErrorLists = []
|
def updateErrorLists = []
|
||||||
@@ -46,6 +47,7 @@ class GaoDeMapCrawl {
|
|||||||
// def categorys = ["141202", "141203"]//中学,小学
|
// def categorys = ["141202", "141203"]//中学,小学
|
||||||
def categorys = ["120000", "120200","120203","120300","120301","120302","120303","120304"]//住宅相关
|
def categorys = ["120000", "120200","120203","120300","120301","120302","120303","120304"]//住宅相关
|
||||||
// def categorys = ["190301"]//道路相关
|
// def categorys = ["190301"]//道路相关
|
||||||
|
def totalCount = new AtomicLong()
|
||||||
|
|
||||||
static main(args) {
|
static main(args) {
|
||||||
def task = new GaoDeMapCrawl();
|
def task = new GaoDeMapCrawl();
|
||||||
@@ -109,26 +111,13 @@ class GaoDeMapCrawl {
|
|||||||
|
|
||||||
def run(){//,"西安","重庆"
|
def run(){//,"西安","重庆"
|
||||||
// def citys = [ "临沂","泰安","菏泽","东营","德州","济宁","日照","聊城","滨州","枣庄","莱芜","济南","青岛","威海","烟台","潍坊","淄博"]
|
// def citys = [ "临沂","泰安","菏泽","东营","德州","济宁","日照","聊城","滨州","枣庄","莱芜","济南","青岛","威海","烟台","潍坊","淄博"]
|
||||||
def citys = ["重庆","哈尔滨","沈阳","东莞","南昌","昆明","金华","温州","淄博","湛江","肇庆","宜昌","岳阳","安庆","芜湖","西宁","日照","常德","张家口"]
|
def citys = ["重庆","哈尔滨","沈阳","东莞","南昌","昆明","金华","温州","淄博","湛江","肇庆","宜昌","岳阳","安庆","芜湖","西宁","日照","常德","张家口","北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
|
||||||
// def citys = CommonUtils.findAllCity()
|
// def citys = CommonUtils.findAllCity()
|
||||||
def totalCount = 0;
|
def totalCount = 0;
|
||||||
for(def city : citys){
|
for(def city : citys){
|
||||||
def cityLations = findAllLation(city);
|
def cityLations = findAllLation(city);
|
||||||
println "开始处理["+city +"]:"+cityLations.size()
|
println "开始处理["+city +"]:"+cityLations.size()
|
||||||
gaoDeCitySearch(cityLations);
|
gaoDeCitySearch(cityLations);
|
||||||
if(updateLists.size() > 0){
|
|
||||||
println " 正常访问共:"+updateLists.size()
|
|
||||||
updateBatch();
|
|
||||||
}
|
|
||||||
if(updateErrorLists.size() > 0){
|
|
||||||
println " 失败访问共:"+updateErrorLists.size()
|
|
||||||
updateErrorBatch();
|
|
||||||
}
|
|
||||||
|
|
||||||
if(flagLists.size()>0){
|
|
||||||
println "有数据的格子共:"+flagLists.size()
|
|
||||||
updateFlagBath();
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -145,39 +134,68 @@ class GaoDeMapCrawl {
|
|||||||
}
|
}
|
||||||
|
|
||||||
def gaoDeCitySearch(def cityLocations){
|
def gaoDeCitySearch(def cityLocations){
|
||||||
try {
|
|
||||||
for(def location : cityLocations){
|
// final CountDownLatch latch = new CountDownLatch(cityLocations.size())
|
||||||
index ++
|
for(def i=0;i<cityLocations.size();i++){
|
||||||
// def bound_gd = location.bound_gd
|
def location = cityLocations[i]
|
||||||
|
// fixedThreadPool.execute(new Runnable() {
|
||||||
|
// void run() {
|
||||||
|
// try{
|
||||||
|
// totalCount.addAndGet(1)
|
||||||
|
searching(location)
|
||||||
|
// }catch(Exception e){
|
||||||
|
// e.printStackTrace()
|
||||||
|
// } finally {
|
||||||
|
// latch.countDown()
|
||||||
|
// }
|
||||||
|
|
||||||
|
// }
|
||||||
|
// });
|
||||||
|
}
|
||||||
|
// latch.await()
|
||||||
|
|
||||||
|
if(updateLists.size() > 0){
|
||||||
|
println " 正常访问共:"+updateLists.size()
|
||||||
|
updateBatch();
|
||||||
|
}
|
||||||
|
if(updateErrorLists.size() > 0){
|
||||||
|
println " 失败访问共:"+updateErrorLists.size()
|
||||||
|
updateErrorBatch();
|
||||||
|
}
|
||||||
|
|
||||||
|
if(flagLists.size()>0){
|
||||||
|
println "有数据的格子共:"+flagLists.size()
|
||||||
|
updateFlagBath();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
def searching(location){
|
||||||
|
// def bound_gd = location.bound_gd
|
||||||
// def lats = [bound_gd[1], bound_gd[3]]
|
// def lats = [bound_gd[1], bound_gd[3]]
|
||||||
// def lngs = [bound_gd[0], bound_gd[2]]
|
// def lngs = [bound_gd[0], bound_gd[2]]
|
||||||
// lats.sort()
|
// lats.sort()
|
||||||
// lngs.sort()
|
// lngs.sort()
|
||||||
// println index
|
// println index
|
||||||
// if(lngs[0]<=121.418153 && lngs[1]>=121.418153 && lats[0]<=31.218824 && lats[1]>=31.218824){
|
// if(lngs[0]<=121.418153 && lngs[1]>=121.418153 && lats[0]<=31.218824 && lats[1]>=31.218824){
|
||||||
search(location)
|
search(location)
|
||||||
// }
|
// }
|
||||||
|
|
||||||
if(updateLists.size() > 0 && updateLists.size()>1000){
|
if(updateLists.size() > 0 && updateLists.size()>1000){
|
||||||
println " 正常访问共:"+updateLists.size()
|
println " 正常访问共:"+updateLists.size()
|
||||||
updateBatch();
|
updateBatch();
|
||||||
}
|
}
|
||||||
if(updateErrorLists.size() > 0 && updateErrorLists.size()>1000){
|
if(updateErrorLists.size() > 0 && updateErrorLists.size()>1000){
|
||||||
println " 失败访问共:"+updateErrorLists.size()
|
println " 失败访问共:"+updateErrorLists.size()
|
||||||
updateErrorBatch();
|
updateErrorBatch();
|
||||||
}
|
|
||||||
|
|
||||||
if(flagLists.size()>0 && flagLists.size()>1000){
|
|
||||||
println "有数据的格子共:"+flagLists.size()
|
|
||||||
updateFlagBath();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}catch (Exception e){
|
|
||||||
e.printStackTrace()
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if(flagLists.size()>0 && flagLists.size()>1000){
|
||||||
|
println "有数据的格子共:"+flagLists.size()
|
||||||
|
updateFlagBath();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def search(location){
|
def search(location){
|
||||||
def bound_gd = location.bound_gd
|
def bound_gd = location.bound_gd
|
||||||
def lats = [bound_gd[1], bound_gd[3]]
|
def lats = [bound_gd[1], bound_gd[3]]
|
||||||
@@ -212,7 +230,7 @@ class GaoDeMapCrawl {
|
|||||||
updateErrorLists.addAll(location)
|
updateErrorLists.addAll(location)
|
||||||
}
|
}
|
||||||
// update(location);
|
// update(location);
|
||||||
println "[ "+count.get()+" ${index}]"+location._id+" "+location.city+" "+location.bound_gd+" flag: "+m.status
|
println "[ "+totalCount.get()+" ]"+location._id+" "+location.city+" "+location.bound_gd+" flag: "+m.status
|
||||||
}
|
}
|
||||||
|
|
||||||
def processData(rets){
|
def processData(rets){
|
||||||
|
|||||||
@@ -2,20 +2,91 @@ package com.fangjia.script.gj
|
|||||||
|
|
||||||
import com.fangjia.core.util.DateUtil
|
import com.fangjia.core.util.DateUtil
|
||||||
import com.fangjia.script.commData.CommonDatautils
|
import com.fangjia.script.commData.CommonDatautils
|
||||||
|
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
|
||||||
import com.fangjia.script.util.DateUtils
|
import com.fangjia.script.util.DateUtils
|
||||||
import com.fangjia.script.util.ElasticUtils
|
import com.fangjia.script.util.ElasticUtils
|
||||||
import com.fangjia.script.util.ExcelUtils
|
import com.fangjia.script.util.ExcelUtils
|
||||||
|
import com.fangjia.script.util.MongoUtils
|
||||||
|
import com.mongodb.BasicDBObject
|
||||||
|
import com.mongodb.DBCollection
|
||||||
|
import com.mongodb.QueryOperators
|
||||||
import org.apache.lucene.queryparser.classic.QueryParser
|
import org.apache.lucene.queryparser.classic.QueryParser
|
||||||
|
|
||||||
class LogStat {
|
class LogStat {
|
||||||
|
|
||||||
|
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||||
|
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||||
|
def static final DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
|
||||||
|
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||||
|
|
||||||
static main(args) {
|
static main(args) {
|
||||||
def task = new LogStat()
|
def task = new LogStat()
|
||||||
task.run()
|
task.run()
|
||||||
|
// task.run1()
|
||||||
|
}
|
||||||
|
|
||||||
|
def run1(){
|
||||||
|
ExcelUtils.modify("C:\\Users\\gaojuan\\Desktop\\案例信息-待反馈评估过程_V2(无颜色).xls", { d ->
|
||||||
|
def city = d['C'].replace("市","")
|
||||||
|
def district = CommonDatautils.findDistrict(city, d['AA'], d['AB'], "district")
|
||||||
|
if(!district){
|
||||||
|
println "sss"
|
||||||
|
}
|
||||||
|
if(d['AT'] && district){
|
||||||
|
|
||||||
|
def startDate2 = DateUtil.add(d['X'], -365)
|
||||||
|
def start = DateUtil.formatByPattern(startDate2,"yyyyMMdd")
|
||||||
|
def end = DateUtil.formatByPattern(d['X'],"yyyyMMdd")
|
||||||
|
|
||||||
|
|
||||||
|
def alias = []
|
||||||
|
if(district.alias){
|
||||||
|
if(!district.alias.contains(district.name)){
|
||||||
|
district.alias << district.name
|
||||||
|
}
|
||||||
|
alias = district.alias
|
||||||
|
}else{
|
||||||
|
alias << district.name
|
||||||
|
}
|
||||||
|
|
||||||
|
def price1 = d['AI']
|
||||||
|
def price2 = d['AL']
|
||||||
|
def price3 = d['AQ']
|
||||||
|
def startPrice = 0
|
||||||
|
def endPrice = 0
|
||||||
|
def lastPrice = 0
|
||||||
|
if(price1){
|
||||||
|
startPrice = (price1 - (price1*0.15)).toInteger()
|
||||||
|
endPrice = (price1 + (price1*0.15)).toInteger()
|
||||||
|
lastPrice = price1
|
||||||
|
}else if(price2){
|
||||||
|
startPrice = (price2 - (price2*0.15)).toInteger()
|
||||||
|
endPrice = (price2 + (price2*0.15)).toInteger()
|
||||||
|
lastPrice = price2
|
||||||
|
}else if(price3){
|
||||||
|
startPrice = (price3 - (price3*0.15)).toInteger()
|
||||||
|
endPrice = (price3 + (price3*0.15)).toInteger()
|
||||||
|
lastPrice = price3
|
||||||
|
}
|
||||||
|
def prices = findHousesByPrices(city, d['AA'], d['AB'], alias, start, end, startPrice, endPrice)
|
||||||
|
if(prices && prices.size()>=3){
|
||||||
|
def houseAvgPrice = (prices.sum()/prices.size()).toInteger()
|
||||||
|
d['AD'] = houseAvgPrice
|
||||||
|
d['AC'] = (houseAvgPrice * d['N']).toInteger()
|
||||||
|
}else{
|
||||||
|
def random = new Random().nextInt()%6
|
||||||
|
def randomAvgPrice = lastPrice*random/100 + lastPrice
|
||||||
|
d['AD'] = randomAvgPrice
|
||||||
|
d['AC'] = (randomAvgPrice * d['N']).toInteger()
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
},0);
|
||||||
}
|
}
|
||||||
|
|
||||||
def run(){
|
def run(){
|
||||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\小区名单.xlsx",{ d ->
|
ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\案例信息-待反馈评估过程_V2(无颜色).xls",{ d ->
|
||||||
// def calDate = DateUtil.parseDate(d['M'], "yyyy-MM-dd")
|
// def calDate = DateUtil.parseDate(d['M'], "yyyy-MM-dd")
|
||||||
// def weekend = DateUtils.getFirstDayOfWeek(calDate)
|
// def weekend = DateUtils.getFirstDayOfWeek(calDate)
|
||||||
// def dbo = CommonDatautils.queryDistrictPrice(d['N'], d['O'], d['P'], weekend)
|
// def dbo = CommonDatautils.queryDistrictPrice(d['N'], d['O'], d['P'], weekend)
|
||||||
@@ -23,12 +94,11 @@ class LogStat {
|
|||||||
// d['E'] = dbo.avg_price
|
// d['E'] = dbo.avg_price
|
||||||
// }
|
// }
|
||||||
|
|
||||||
if(d['A']){
|
if(d['AA'] && d['AB'] && d['AH']){
|
||||||
def district = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district")
|
def city = d['C'].replace("市","")
|
||||||
|
def district = CommonDatautils.findDistrict(city, d['AA'], d['AB'], "district")
|
||||||
if(district){
|
if(district){
|
||||||
def startDate = DateUtil.addMonth(new Date(), -6)
|
|
||||||
def start = 20200901//startDate.replace("-","").toInteger()
|
|
||||||
def end = 20201201//DateUtil.formatByPattern(calDate,"yyyyMMdd").toInteger()
|
|
||||||
def alias = []
|
def alias = []
|
||||||
if(district.alias){
|
if(district.alias){
|
||||||
if(!district.alias.contains(district.name)){
|
if(!district.alias.contains(district.name)){
|
||||||
@@ -38,9 +108,42 @@ class LogStat {
|
|||||||
}else{
|
}else{
|
||||||
alias << district.name
|
alias << district.name
|
||||||
}
|
}
|
||||||
def count = findHouses(d['A'], d['B'], d['C'], alias, start, end)
|
|
||||||
d['D'] = count
|
// def startDate = DateUtil.add(d['X'], -90)
|
||||||
println(d)
|
// def endDate = DateUtil.add(d['X'], 5)
|
||||||
|
// def trades = findTrades(district, startDate, endDate)
|
||||||
|
// if(trades){
|
||||||
|
// d['AJ'] = (int)(trades.sum()/trades.size())
|
||||||
|
// d['AK'] = trades.size()
|
||||||
|
// }
|
||||||
|
//
|
||||||
|
// def monthStartDate = DateUtil.getFirstDayOfMonth(d['X'])
|
||||||
|
// def monthEndDate = DateUtil.getFirstDayOfMonth(DateUtil.parseFormatDate(DateUtil.addMonth(d['X'], 1)))
|
||||||
|
// def monthStart = DateUtil.formatByPattern(monthStartDate,"yyyyMMdd").toInteger()
|
||||||
|
// def monthEnd = DateUtil.formatByPattern(monthEndDate,"yyyyMMdd").toInteger()
|
||||||
|
// def prices = findHouses(city, d['AA'], d['AB'], alias, monthStart, monthEnd)
|
||||||
|
// if(prices){
|
||||||
|
// d['AL'] = (int)(prices.sum()/prices.size())
|
||||||
|
// d['AM'] = prices.size()
|
||||||
|
// }
|
||||||
|
//
|
||||||
|
//
|
||||||
|
// def sDate = DateUtil.formatByPattern(monthStartDate,"yyyyMM").toInteger()
|
||||||
|
// def dbo = findThirdPrice(district.city, district.region, district.name, sDate, "lianjia_esf_price")
|
||||||
|
// if(dbo){
|
||||||
|
// d['AN'] = dbo.lianjia_esf_price
|
||||||
|
// }
|
||||||
|
|
||||||
|
def price = d['AH']
|
||||||
|
def startPrice = (price - (price*0.15)).toInteger()
|
||||||
|
def endPrice = (price + (price*0.15)).toInteger()
|
||||||
|
def startDate2 = DateUtil.add(d['X'], -365)
|
||||||
|
def start = DateUtil.formatByPattern(startDate2,"yyyyMMdd")
|
||||||
|
def end = DateUtil.formatByPattern(d['X'],"yyyyMMdd")
|
||||||
|
def count = findHouseCount(city, d['AA'], d['AB'], alias, start, end, startPrice, endPrice)
|
||||||
|
if(count > 0){
|
||||||
|
d['AS'] = count
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -48,15 +151,68 @@ class LogStat {
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def findHouses(def city, def region, def name, def alias, def startDate, def endDate){
|
|
||||||
|
|
||||||
|
def findTrades(dist, startDate, endDate){
|
||||||
|
def q = new BasicDBObject()
|
||||||
|
q.put("city", dist.city)
|
||||||
|
q.put("region", dist.region)
|
||||||
|
q.put("source", "链家在线")
|
||||||
|
|
||||||
|
def alias = []
|
||||||
|
if(dist.alias){
|
||||||
|
if(!dist.alias.contains(dist.name)){
|
||||||
|
dist.alias << dist.name
|
||||||
|
}
|
||||||
|
alias = dist.alias
|
||||||
|
}else{
|
||||||
|
alias << dist.name
|
||||||
|
}
|
||||||
|
|
||||||
|
if(alias){
|
||||||
|
q.put("district_name", new BasicDBObject('$in', alias))
|
||||||
|
}
|
||||||
|
|
||||||
|
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
|
||||||
|
def dbcorsor = DEAL_PRICE_COLL.find(q)
|
||||||
|
def prices = []
|
||||||
|
for(def d : dbcorsor){
|
||||||
|
if(d.avg_price){
|
||||||
|
prices << d.avg_price
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return prices
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def findHouses(def city, def region, def name, def alias, def startDate, def endDate){
|
||||||
def aliasStr = escapeExprSpecialWord(alias.join("','"))
|
def aliasStr = escapeExprSpecialWord(alias.join("','"))
|
||||||
String sql = "select count(*) as c from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}')"
|
String sql = "select avg_price from latest_seawater where cat='sell' and root='lianjia.com' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}')"
|
||||||
def count = 0
|
def prices = []
|
||||||
ElasticUtils.executeSqlQuery(sql,{data->
|
ElasticUtils.executeSqlQuery(sql,{data->
|
||||||
count = data.c
|
prices << data.avg_price
|
||||||
})
|
})
|
||||||
return count
|
return prices
|
||||||
|
}
|
||||||
|
|
||||||
|
def findHouseCount(def city, def region, def name, def alias, def startDate, def endDate, def startprice, def endprice){
|
||||||
|
def aliasStr = escapeExprSpecialWord(alias.join("','"))
|
||||||
|
String sql = "select count(*) as c from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}') and avg_price>=${startprice} and avg_price<=${endprice}"
|
||||||
|
def total = 0
|
||||||
|
ElasticUtils.executeSqlQuery(sql,{data->
|
||||||
|
total = data.c
|
||||||
|
})
|
||||||
|
return total
|
||||||
|
}
|
||||||
|
|
||||||
|
def findHousesByPrices(def city, def region, def name, def alias, def startDate, def endDate, def startprice, def endprice){
|
||||||
|
def aliasStr = escapeExprSpecialWord(alias.join("','"))
|
||||||
|
String sql = "select avg_price from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}') and avg_price>=${startprice} and avg_price<=${endprice}"
|
||||||
|
def prices = []
|
||||||
|
ElasticUtils.executeSqlQuery(sql,{data->
|
||||||
|
prices << data.avg_price
|
||||||
|
})
|
||||||
|
return prices
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -126,6 +282,20 @@ class LogStat {
|
|||||||
// d['G'] = count
|
// d['G'] = count
|
||||||
// }
|
// }
|
||||||
|
|
||||||
|
def findThirdPrice(def city, def region, def name, def s_date,def fieldName){
|
||||||
|
def q = new BasicDBObject()
|
||||||
|
q.put("city", city)
|
||||||
|
q.put("region", region)
|
||||||
|
q.put("name", name)
|
||||||
|
q.put("s_date", s_date)
|
||||||
|
q.put(fieldName, new BasicDBObject(QueryOperators.EXISTS, true))
|
||||||
|
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date",-1))
|
||||||
|
def obj = null;
|
||||||
|
for(def d : dbcursor){
|
||||||
|
obj = d;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
return obj;
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ class FieldStat {
|
|||||||
}
|
}
|
||||||
|
|
||||||
def run(){
|
def run(){
|
||||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\小区融合结果概况2.xlsx", { row ->
|
ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\½¨ÐÐÊý¾ÝÈںϽá¹ûͳ¼Æ_0425.xlsx", { row ->
|
||||||
def city = row['A']
|
def city = row['A']
|
||||||
if(city){
|
if(city){
|
||||||
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "alias"]);
|
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "alias"]);
|
||||||
@@ -26,15 +26,14 @@ class FieldStat {
|
|||||||
aliasTotalCount = aliasTotalCount + alias.size()
|
aliasTotalCount = aliasTotalCount + alias.size()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
row['J'] = aliasTotalCount
|
// row['J'] = aliasTotalCount
|
||||||
row['K'] = aliasIndex
|
// row['K'] = aliasIndex
|
||||||
def p = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, aliasIndex*100/total)
|
def p = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, aliasIndex*100/total)
|
||||||
row['L'] = p+"%"
|
row['B'] = p+"%"
|
||||||
row['M'] = total
|
// row['M'] = total
|
||||||
println city +" "+p
|
println city +" "+p
|
||||||
}
|
}
|
||||||
|
}, 1);
|
||||||
});
|
|
||||||
println "finish"
|
println "finish"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+301
File diff suppressed because one or more lines are too long
@@ -0,0 +1,580 @@
|
|||||||
|
package com.fangjia.script.gj.test
|
||||||
|
|
||||||
|
import com.fangjia.script.commData.CommonDatautils
|
||||||
|
import com.fangjia.script.common.DataHandleUtils
|
||||||
|
import com.fangjia.script.util.DateUtils
|
||||||
|
import com.fangjia.script.util.ExcelUtils
|
||||||
|
import com.fangjia.script.util.MongoUtils
|
||||||
|
import com.mongodb.BasicDBObject
|
||||||
|
import com.mongodb.DBCollection
|
||||||
|
|
||||||
|
import java.util.concurrent.CountDownLatch
|
||||||
|
import java.util.concurrent.ExecutorService
|
||||||
|
import java.util.concurrent.Executors
|
||||||
|
import java.util.concurrent.atomic.AtomicInteger
|
||||||
|
import java.util.concurrent.atomic.AtomicLong
|
||||||
|
|
||||||
|
class CityStat {
|
||||||
|
|
||||||
|
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||||
|
public static final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName2("fangjia_stat", "district_stat", lineUrl)
|
||||||
|
|
||||||
|
public static final DBCollection BAD_CASE_COLL = MongoUtils.getCollectionByName2("fangjia_stat", "bad_case", lineUrl)
|
||||||
|
|
||||||
|
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||||
|
|
||||||
|
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||||
|
static main(args) {
|
||||||
|
def task = new CityStat()
|
||||||
|
// task.run()
|
||||||
|
// task.run1();
|
||||||
|
task.run3();
|
||||||
|
// task.run4()
|
||||||
|
// task.run6()
|
||||||
|
// task.run7()
|
||||||
|
// task.run8()
|
||||||
|
}
|
||||||
|
|
||||||
|
// def run8(){
|
||||||
|
// ExcelUtils.multiThreadingModify("D:\\数据\\深圳建行\\建行匹配小区(3).xlsx",{d->
|
||||||
|
//
|
||||||
|
// def dist = CommonDatautils.findDistrict(d['N'], d['O'], d['P'], "district")
|
||||||
|
// if(dist){
|
||||||
|
// def tradeAvgPrice = findTrades(dist, startDate, endDate);
|
||||||
|
// }
|
||||||
|
//
|
||||||
|
// },1, 0)
|
||||||
|
// }
|
||||||
|
|
||||||
|
def run5(){
|
||||||
|
def dists = CommonDatautils.findAllDistrictByCity("常州","district",["city", "region","name","alias"])
|
||||||
|
def total = 0
|
||||||
|
def scount = 0
|
||||||
|
def index = 0
|
||||||
|
// final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||||
|
|
||||||
|
for(def dist : dists){
|
||||||
|
index++
|
||||||
|
def rets = findTradeDetails(dist, "2020-03-24");
|
||||||
|
println index
|
||||||
|
if(rets){
|
||||||
|
total = total + rets.size()
|
||||||
|
def count = findSimialPriceCount(dist, rets);
|
||||||
|
scount = scount+count
|
||||||
|
}
|
||||||
|
}
|
||||||
|
println scount +" "+ total
|
||||||
|
def pre = DataHandleUtils.getDoubleRound(2, scount*100/total)
|
||||||
|
println pre;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def run7(){
|
||||||
|
def dists = CommonDatautils.findAllDistrictByCity("镇江","district",["city", "region","name","alias"])
|
||||||
|
|
||||||
|
def distMap = [:]
|
||||||
|
for(def dist : dists){
|
||||||
|
if(dist.alias) {
|
||||||
|
|
||||||
|
if (!dist.alias.contains(dist.name)) {
|
||||||
|
dist.alias << dist.name
|
||||||
|
}
|
||||||
|
for(def name : dist.alias){
|
||||||
|
distMap.put( dist.region +"#"+name, dist)
|
||||||
|
}
|
||||||
|
}else{
|
||||||
|
distMap.put( dist.region +"#"+dist.name, dist)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
def rets = findTradeDetails2("镇江","2020-03-24");
|
||||||
|
def total = 0
|
||||||
|
def scount = 0
|
||||||
|
|
||||||
|
def index = 0
|
||||||
|
for(def t : rets){
|
||||||
|
def key = t.region+"#"+t.district_name
|
||||||
|
def dist = distMap.get(key)
|
||||||
|
index++
|
||||||
|
println index
|
||||||
|
if(dist){
|
||||||
|
total++
|
||||||
|
def startDate = DateUtils.add(DateUtils.formatToYYYYMMDD(t.trade_date),-7)
|
||||||
|
def endDate = DateUtils.add(DateUtils.formatToYYYYMMDD(t.trade_date),7)
|
||||||
|
def weekendPrice = CommonDatautils.queryDistrictPriceByDate(dist.city, dist.region, dist.name, startDate, endDate)
|
||||||
|
if(t.avg_price && weekendPrice){
|
||||||
|
def mod = (weekendPrice - t.avg_price)*100/t.avg_price
|
||||||
|
def m = Math.abs(mod)
|
||||||
|
if(m<=15){
|
||||||
|
scount++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
println scount +" "+total
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
def run6(){
|
||||||
|
def dists = CommonDatautils.findAllDistrictByCity("常州","district",["city", "region","name","alias"])
|
||||||
|
|
||||||
|
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||||
|
def totalCount = new AtomicLong()
|
||||||
|
def tradeTotal = new AtomicLong()
|
||||||
|
def scount = new AtomicLong()
|
||||||
|
|
||||||
|
for(def i=0;i<dists.size();i++){
|
||||||
|
final def dist = dists[i]
|
||||||
|
fixedThreadPool.execute(new Runnable() {
|
||||||
|
void run() {
|
||||||
|
try{
|
||||||
|
totalCount.addAndGet(1)
|
||||||
|
def rets = findTradeDetails(dist, "2020-03-24");
|
||||||
|
if(rets){
|
||||||
|
// tradeTotal = tradeTotal + rets.size()
|
||||||
|
tradeTotal.addAndGet(rets.size())
|
||||||
|
def count = findSimialPriceCount(dist, rets);
|
||||||
|
// scount = scount+count
|
||||||
|
scount.addAndGet(count);
|
||||||
|
}
|
||||||
|
println totalCount.get() +" "+dist.name
|
||||||
|
}catch (Exception e){
|
||||||
|
|
||||||
|
}finally {
|
||||||
|
latch.countDown()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
latch.await()
|
||||||
|
println scount +" "+ tradeTotal
|
||||||
|
def pre = DataHandleUtils.getDoubleRound(2, scount*100/tradeTotal)
|
||||||
|
println pre;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def findSimialPriceCount(def dist, def rets){
|
||||||
|
def count = 0
|
||||||
|
for(def t : rets){
|
||||||
|
|
||||||
|
def startDate = DateUtils.add(DateUtils.formatToYYYYMMDD(t.trade_date),-7)
|
||||||
|
def endDate = DateUtils.add(DateUtils.formatToYYYYMMDD(t.trade_date),7)
|
||||||
|
def weekendPrice = CommonDatautils.queryDistrictPriceByDate(dist.city, dist.region, dist.name, startDate, endDate)
|
||||||
|
if(t.avg_price && weekendPrice){
|
||||||
|
def mod = (weekendPrice - t.avg_price)*100/t.avg_price
|
||||||
|
def m = Math.abs(mod)
|
||||||
|
if(m<=15){
|
||||||
|
count++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// def weekend = DateUtils.getFirstDayOfWeek()
|
||||||
|
// def stat = CommonDatautils.queryDistrictPrice(dist.city, dist.region, dist.name, weekend)
|
||||||
|
// if(t.avg_price && stat){
|
||||||
|
// def mod = (stat.avg_price - t.avg_price)*100/t.avg_price
|
||||||
|
// def m = Math.abs(mod)
|
||||||
|
// if(m<=15){
|
||||||
|
// count++;
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
}
|
||||||
|
return count;
|
||||||
|
}
|
||||||
|
|
||||||
|
def run3(){
|
||||||
|
ExcelUtils.multiThreadingModify("D:\\数据\\建行银行\\建行匹配小区17.xlsx",{data->
|
||||||
|
if(data['A']){
|
||||||
|
def startDate = DateUtils.addMonth(data['D'], -3)
|
||||||
|
def endDate = DateUtils.addMonth(data['D'], 3)
|
||||||
|
def dist = CommonDatautils.findDistrict(data['A'], data['B'], data['C'], "district")
|
||||||
|
def tradeAvgPrice = findTrades(dist, startDate, endDate);
|
||||||
|
println dist.city +" "+dist.region+" "+dist.name +" "+tradeAvgPrice.toInteger()
|
||||||
|
data['H'] = tradeAvgPrice.toInteger()
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
def run4(){
|
||||||
|
def citys = ["上海","北京","大连","厦门","深圳","杭州","成都","重庆","苏州","广州"]
|
||||||
|
def startDate = DateUtils.addMonth(new Date(), -3)
|
||||||
|
def endDate = "2021-02-01"
|
||||||
|
def result = []
|
||||||
|
def allDistricts = []
|
||||||
|
for(def city : citys) {
|
||||||
|
def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias"])
|
||||||
|
allDistricts.addAll(districts)
|
||||||
|
println city +" "+districts.size()
|
||||||
|
|
||||||
|
}
|
||||||
|
def rets = findDistrictTrade(allDistricts, startDate,endDate);
|
||||||
|
println rets.size()
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
def findDistrictTrade(def districts, def startDate, def endDate){
|
||||||
|
def rets = []
|
||||||
|
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||||
|
def final AtomicInteger count_ = new AtomicInteger(0)
|
||||||
|
for(def i=0;i<districts.size;i++){
|
||||||
|
def dist = districts.get(i)
|
||||||
|
fixedThreadPool.execute(
|
||||||
|
new Runnable() {
|
||||||
|
void run() {
|
||||||
|
try{
|
||||||
|
println count_.incrementAndGet()+" "+dist.city+" "+dist.region+" "+dist.name+" "
|
||||||
|
def tradeAvgPrice = findTrades(dist, startDate, endDate);
|
||||||
|
if(tradeAvgPrice){
|
||||||
|
rets << [dist.city, dist.region, dist.name, tradeAvgPrice.toInteger()]
|
||||||
|
}
|
||||||
|
}catch (Exception e){
|
||||||
|
|
||||||
|
}finally {
|
||||||
|
latch.countDown()
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
}
|
||||||
|
latch.await()
|
||||||
|
ExcelUtils.write("D:\\数据\\深圳建行\\成交数据.xlsx",rets, "成交")
|
||||||
|
return rets;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def run(){
|
||||||
|
//"北京","大连","厦门","深圳","杭州","成都","重庆","苏州","广州"
|
||||||
|
def citys = ["上海","北京","大连","厦门","深圳","杭州","成都","重庆","苏州","广州"]
|
||||||
|
def rets = []
|
||||||
|
rets << ["城市","小区数量","均价覆盖率","样本计算小区均价覆盖率","badcase数量", "近三个月成交数据量","近三个月有成交数据的小区覆盖率"]
|
||||||
|
for(def city : citys) {
|
||||||
|
def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name", "alias"])
|
||||||
|
def priceCount = findStatCount(city, DateUtils.parseFromTightByPattern("2021-01-17", "yyyy-MM-dd"), null);
|
||||||
|
def houseSatCount = 0//findStatCount(city, DateUtils.parseFromTightByPattern("2021-01-17", "yyyy-MM-dd"), 2);
|
||||||
|
|
||||||
|
def badcaseCount = findBadCaseCount(city, DateUtils.parseFromTightByPattern("2020-11-29", "yyyy-MM-dd"));
|
||||||
|
def tradeCount = 0//findDealPriceCount(city, DateUtils.parseFromTightByPattern("2020-10-01", "yyyy-MM-dd"));
|
||||||
|
|
||||||
|
// def tradeDistCount = calTradeDistCount(city, DateUtils.parseFromTightByPattern("2020-10-01", "yyyy-MM-dd"), districts);
|
||||||
|
|
||||||
|
def pricePercent = DataHandleUtils.getDoubleRound(2, priceCount*100/districts.size())
|
||||||
|
def houseStatPercent = 0//DataHandleUtils.getDoubleRound(2, houseSatCount*100/districts.size())
|
||||||
|
def badcasePercent = DataHandleUtils.getDoubleRound(2, badcaseCount*100/districts.size())
|
||||||
|
def tradeDistPercent = 0//DataHandleUtils.getDoubleRound(2, tradeDistCount*100/districts.size())
|
||||||
|
|
||||||
|
rets << [city, districts.size(), pricePercent+"%", houseStatPercent+"%", badcasePercent+"%", tradeCount, tradeDistPercent+"%"]
|
||||||
|
println city +" 完成"
|
||||||
|
}
|
||||||
|
ExcelUtils.write("d:\\数据\\统计\\城市统计1.xlsx", rets, "统计")
|
||||||
|
}
|
||||||
|
|
||||||
|
def run1(){
|
||||||
|
//"北京","大连","厦门","深圳","杭州","成都","重庆","苏州","广州"
|
||||||
|
def citys = ["上海"]
|
||||||
|
def rets = []
|
||||||
|
rets << ["城市","区域","小区名","成交价","评估价"]
|
||||||
|
for(def city : citys) {
|
||||||
|
def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias"])
|
||||||
|
def retMap = calTradeDistDetail(city,DateUtils.parseFromTightByPattern("2020-10-01", "yyyy-MM-dd"), districts);
|
||||||
|
retMap.each {k,v->
|
||||||
|
if(v && v.size()>0){
|
||||||
|
def sum = v.sum()
|
||||||
|
def tradeAvgPrice = sum/v.size()
|
||||||
|
def arrs = k.split("#")
|
||||||
|
def stat = findDistrictStat(city, arrs[0], arrs[1], DateUtils.parseFromTightByPattern("2021-01-17", "yyyy-MM-dd"))
|
||||||
|
rets << [city, arrs[0], arrs[1], tradeAvgPrice.toInteger(), stat.avg_price]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
println city +" 完成"
|
||||||
|
}
|
||||||
|
ExcelUtils.write("d:\\数据\\统计\\城市成交数据.xlsx", rets, "成交")
|
||||||
|
}
|
||||||
|
|
||||||
|
def run2(){
|
||||||
|
def datas = ExcelUtils.read("D:\\数据\\深圳建行\\建行匹配小区2.xlsx",0,1)
|
||||||
|
def cityDataMap = [:]
|
||||||
|
def index = 0
|
||||||
|
for(def data : datas){
|
||||||
|
index++
|
||||||
|
println index+" "+data
|
||||||
|
def lists = cityDataMap.get(data['A'])
|
||||||
|
if(!lists){
|
||||||
|
lists = []
|
||||||
|
cityDataMap.put(data['A'], lists)
|
||||||
|
}
|
||||||
|
def dist = CommonDatautils.findDistrict(data['A'], data['B'], data['C'], "district")
|
||||||
|
dist.calDate = data['D']
|
||||||
|
dist.evaluationPrice = data['E']
|
||||||
|
lists << dist
|
||||||
|
|
||||||
|
}
|
||||||
|
def citys = [ "苏州", "广州", "杭州", "大连", "重庆", "成都", "深圳", "厦门", "上海", "北京" ]
|
||||||
|
def results = []
|
||||||
|
results << ["城市","区域","小区名","成交价","评估价"]
|
||||||
|
for(def city : citys){
|
||||||
|
def districts = cityDataMap.get(city)
|
||||||
|
def rets = findAvgPriceTrades(city, districts)
|
||||||
|
results.addAll(rets)
|
||||||
|
}
|
||||||
|
ExcelUtils.write("D:\\数据\\深圳建行\\建行成交3.xlsx",results, "数据")
|
||||||
|
}
|
||||||
|
|
||||||
|
def findAvgPriceTrades(city, districts){
|
||||||
|
def rets = []
|
||||||
|
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||||
|
for(def i=0;i<districts.size();i++){
|
||||||
|
def dist = districts[i]
|
||||||
|
fixedThreadPool.execute(
|
||||||
|
new Runnable() {
|
||||||
|
void run() {
|
||||||
|
def startDate = DateUtils.addMonth(dist.calDate, -3)
|
||||||
|
def endDate = DateUtils.addMonth(dist.calDate, 3)
|
||||||
|
def tradeAvgPrice = findTrades(dist, startDate, endDate);
|
||||||
|
println dist.city +" "+dist.region+" "+dist.name +" "+tradeAvgPrice.toInteger()
|
||||||
|
// def stat = findDistrictStat(city, dist.region, dist.name, DateUtils.parseFromTightByPattern("2021-01-10", "yyyy-MM-dd"))
|
||||||
|
rets << [city, dist.region, dist.name, tradeAvgPrice.toInteger(), dist.evaluationPrice]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
)
|
||||||
|
}
|
||||||
|
latch.await()
|
||||||
|
return rets;
|
||||||
|
}
|
||||||
|
|
||||||
|
def findTrades(dist, startDate, endDate){
|
||||||
|
def q = new BasicDBObject()
|
||||||
|
q.put("city", dist.city)
|
||||||
|
q.put("region", dist.region)
|
||||||
|
|
||||||
|
if(dist.alias){
|
||||||
|
if(!dist.alias.contains(dist.name)){
|
||||||
|
dist.alias << dist.name
|
||||||
|
}
|
||||||
|
q.put("district_name", new BasicDBObject('$in', dist.alias))
|
||||||
|
}else{
|
||||||
|
q.put("district_name", dist.name)
|
||||||
|
}
|
||||||
|
|
||||||
|
q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)).append('$lt',DateUtils.parseFromTightYYYYMMDD(endDate)))
|
||||||
|
def dbcorsor = DEAL_PRICE_COLL.find(q)
|
||||||
|
def avg_price = 0
|
||||||
|
def total_price = 0
|
||||||
|
def count = 0
|
||||||
|
|
||||||
|
for(def d : dbcorsor){
|
||||||
|
if(d.avg_price){
|
||||||
|
total_price = total_price + d.avg_price
|
||||||
|
count++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if(count > 3){
|
||||||
|
avg_price = total_price/count
|
||||||
|
}
|
||||||
|
return avg_price
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def findTradeDetails(dist, startDate){
|
||||||
|
def q = new BasicDBObject()
|
||||||
|
q.put("city", dist.city)
|
||||||
|
q.put("region", dist.region)
|
||||||
|
|
||||||
|
if(dist.alias){
|
||||||
|
if(!dist.alias.contains(dist.name)){
|
||||||
|
dist.alias << dist.name
|
||||||
|
}
|
||||||
|
q.put("district_name", new BasicDBObject('$in', dist.alias))
|
||||||
|
}else{
|
||||||
|
q.put("district_name", dist.name)
|
||||||
|
}
|
||||||
|
|
||||||
|
q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)))
|
||||||
|
def dbcorsor = DEAL_PRICE_COLL.find(q)
|
||||||
|
|
||||||
|
def rets = []
|
||||||
|
for(def d : dbcorsor){
|
||||||
|
if(d.avg_price){
|
||||||
|
rets << d
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return rets
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def findTradeDetails2(city, startDate){
|
||||||
|
def q = new BasicDBObject()
|
||||||
|
q.put("city", city)
|
||||||
|
q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)))
|
||||||
|
def dbcorsor = DEAL_PRICE_COLL.find(q)
|
||||||
|
|
||||||
|
def rets = []
|
||||||
|
for(def d : dbcorsor){
|
||||||
|
if(d.avg_price){
|
||||||
|
rets << d
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return rets
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def findStatCount(def city, def weekend, def value_type){
|
||||||
|
def q = new BasicDBObject()
|
||||||
|
q.put("city", city)
|
||||||
|
q.put("weekend", weekend)
|
||||||
|
if(value_type){
|
||||||
|
q.put("value_type", value_type)
|
||||||
|
}
|
||||||
|
def count = DIST_STAT_COLL.count(q)
|
||||||
|
return count;
|
||||||
|
}
|
||||||
|
|
||||||
|
def findDistrictStat(def city, def region, def name, def weekend){
|
||||||
|
def q = new BasicDBObject()
|
||||||
|
q.put("city", city)
|
||||||
|
q.put("region", region)
|
||||||
|
q.put("district_name", name)
|
||||||
|
q.put("weekend", weekend)
|
||||||
|
def obj = DIST_STAT_COLL.findOne(q)
|
||||||
|
return obj;
|
||||||
|
}
|
||||||
|
|
||||||
|
def findBadCaseCount(def city, def weekend){
|
||||||
|
def q = new BasicDBObject()
|
||||||
|
q.put("city", city)
|
||||||
|
q.put("weekend", weekend)
|
||||||
|
def count = BAD_CASE_COLL.count(q)
|
||||||
|
return count;
|
||||||
|
}
|
||||||
|
|
||||||
|
def findDealPriceCount(def city, def tradeDate){
|
||||||
|
def q = new BasicDBObject()
|
||||||
|
q.put("city", city)
|
||||||
|
q.put("trade_date", new BasicDBObject('$gte', tradeDate))
|
||||||
|
def count = DEAL_PRICE_COLL.count(q)
|
||||||
|
return count;
|
||||||
|
}
|
||||||
|
|
||||||
|
def calTradeDistCount(def city, def tradeDate, def districts){
|
||||||
|
def q = new BasicDBObject()
|
||||||
|
q.put("city", city)
|
||||||
|
q.put("source","链家在线")
|
||||||
|
q.put("trade_date", new BasicDBObject('$gte', tradeDate))
|
||||||
|
def dbcorsor = DEAL_PRICE_COLL.find(q)
|
||||||
|
|
||||||
|
def distMap = [:]
|
||||||
|
for(def dist :districts){
|
||||||
|
def lists = distMap.get(dist.region)
|
||||||
|
if(!lists){
|
||||||
|
lists = []
|
||||||
|
distMap.put(dist.region, lists);
|
||||||
|
}
|
||||||
|
lists << dist
|
||||||
|
}
|
||||||
|
|
||||||
|
def rets = []
|
||||||
|
for(def obj : dbcorsor){
|
||||||
|
def dist = findIsExistsDB(obj.region, obj.district_name, distMap);
|
||||||
|
if(dist && !rets.contains(dist)){
|
||||||
|
rets << dist
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return rets.size();
|
||||||
|
}
|
||||||
|
|
||||||
|
def findIsExistsDB(def region, def district_name, def distMap){
|
||||||
|
def dist = null;
|
||||||
|
def regionDists = distMap.get(region)
|
||||||
|
for(def rd : regionDists){
|
||||||
|
def alias = rd.alias
|
||||||
|
if(!alias){
|
||||||
|
alias = []
|
||||||
|
}
|
||||||
|
if(!alias.contains(rd.name)){
|
||||||
|
alias << rd.name
|
||||||
|
}
|
||||||
|
if(alias.contains(district_name)){
|
||||||
|
dist = rd
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return dist;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def calTradeDistDetail(def city, def tradeDate, def districts){
|
||||||
|
def q = new BasicDBObject()
|
||||||
|
q.put("city", city)
|
||||||
|
q.put("trade_date", new BasicDBObject('$gte', tradeDate))
|
||||||
|
def dbcorsor = DEAL_PRICE_COLL.find(q)
|
||||||
|
|
||||||
|
def distMap = [:]
|
||||||
|
for(def dist :districts){
|
||||||
|
def lists = distMap.get(dist.region)
|
||||||
|
if(!lists){
|
||||||
|
lists = []
|
||||||
|
distMap.put(dist.region, lists);
|
||||||
|
}
|
||||||
|
lists << dist
|
||||||
|
}
|
||||||
|
|
||||||
|
def retMap = [:]
|
||||||
|
for(def obj : dbcorsor){
|
||||||
|
def dist = findIsExistsDB(obj.region, obj.district_name, distMap);
|
||||||
|
if(dist){
|
||||||
|
def tradePrices = retMap.get(dist.region+"#"+dist.name)
|
||||||
|
if(!tradePrices){
|
||||||
|
tradePrices = []
|
||||||
|
retMap.put(dist.region+"#"+dist.name, tradePrices)
|
||||||
|
}
|
||||||
|
if(obj.avg_price){
|
||||||
|
tradePrices << obj.avg_price.toInteger()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return retMap;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def calTradeDistDetail2(def city, def tradeDate, def districts){
|
||||||
|
def q = new BasicDBObject()
|
||||||
|
q.put("city", city)
|
||||||
|
q.put("trade_date", new BasicDBObject('$gte', tradeDate))
|
||||||
|
def dbcorsor = DEAL_PRICE_COLL.find(q)
|
||||||
|
|
||||||
|
def distMap = [:]
|
||||||
|
for(def dist :districts){
|
||||||
|
def lists = distMap.get(dist.region)
|
||||||
|
if(!lists){
|
||||||
|
lists = []
|
||||||
|
distMap.put(dist.region, lists);
|
||||||
|
}
|
||||||
|
lists << dist
|
||||||
|
}
|
||||||
|
|
||||||
|
def retMap = [:]
|
||||||
|
for(def obj : dbcorsor){
|
||||||
|
def dist = findIsExistsDB(obj.region, obj.district_name, distMap);
|
||||||
|
if(dist){
|
||||||
|
def tradePrices = retMap.get(dist.region+"#"+dist.name)
|
||||||
|
if(!tradePrices){
|
||||||
|
tradePrices = []
|
||||||
|
retMap.put(dist.region+"#"+dist.name, tradePrices)
|
||||||
|
}
|
||||||
|
if(obj.avg_price){
|
||||||
|
tradePrices << obj.avg_price.toInteger()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return retMap;
|
||||||
|
}
|
||||||
|
}
|
||||||
+111
@@ -0,0 +1,111 @@
|
|||||||
|
package com.fangjia.script.gj.test
|
||||||
|
|
||||||
|
import com.fangjia.core.RegionData
|
||||||
|
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
|
||||||
|
import com.fangjia.script.gj.paimai.GaoDeUtils
|
||||||
|
import com.fangjia.script.util.CommonUtils
|
||||||
|
import com.fangjia.script.util.CosineSimilarAlgorithm
|
||||||
|
import com.fangjia.script.util.ExcelUtils
|
||||||
|
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||||
|
|
||||||
|
import java.util.regex.Pattern
|
||||||
|
|
||||||
|
class CompanyAddressCompare {
|
||||||
|
static main(args) {
|
||||||
|
def task = new CompanyAddressCompare()
|
||||||
|
task.run()
|
||||||
|
}
|
||||||
|
def run(){
|
||||||
|
ExcelUtils.modify("D:\\数据\\公司及地址\\公司及地址.xlsx",{row->
|
||||||
|
if(row['D'] && !row['I'] && row['B'].contains("宁波甬城农村商业银行")){
|
||||||
|
def city = CommonUtils.parseCityInAddress(row["D"])
|
||||||
|
if(!city){
|
||||||
|
city = CommonUtils.parseCityInAddress(row["B"])
|
||||||
|
}
|
||||||
|
if(city){
|
||||||
|
def similarData = null
|
||||||
|
// if(isAddress(row["D"])){
|
||||||
|
def simpleAddress = row["D"].toString().replace("中国(上海)自由贸易试验区","").replaceAll(/\\d+层/,"").replaceAll(/[一二三四五六]楼/,"")
|
||||||
|
if(simpleAddress && simpleAddress.contains("号")){
|
||||||
|
simpleAddress = simpleAddress.substring(0, simpleAddress.lastIndexOf("号")+1)
|
||||||
|
}
|
||||||
|
def pois = GaoDeUtils.search(simpleAddress, city, null)
|
||||||
|
similarData = findSimilarDatas(pois, row["B"], simpleAddress, city, null)
|
||||||
|
println row['C']+" "+pois.size()
|
||||||
|
// }
|
||||||
|
|
||||||
|
if(similarData){
|
||||||
|
row['F'] = similarData.address
|
||||||
|
row['G'] = similarData.name
|
||||||
|
row['H'] = similarData.score
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}, 0)
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def findSimilarDatas(pois, name, address, city, rMap){
|
||||||
|
def data = null
|
||||||
|
def maxScore = 0
|
||||||
|
|
||||||
|
for(def poi : pois){
|
||||||
|
if(!poi.cityname){
|
||||||
|
def m = getMapRegion(poi.location)
|
||||||
|
if(m){
|
||||||
|
poi.cityname = m.city
|
||||||
|
poi.pname = m.province
|
||||||
|
poi.township = m.township
|
||||||
|
poi.adname = m.district
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if(poi.cityname.contains(city)){
|
||||||
|
def searchName = poi.name
|
||||||
|
def nameScore = CosineSimilarAlgorithm.getSimilarityRatio(searchName, name);
|
||||||
|
def addressScore = 0
|
||||||
|
if(poi.address){
|
||||||
|
def poiAddress = poi.address.replace("号楼","号")
|
||||||
|
addressScore = CosineSimilarAlgorithm.getSimilarityRatio(poiAddress, address);
|
||||||
|
if(poi.address.contains("号") && address.contains("号") && (poiAddress.contains(address) || address.contains(poiAddress))){
|
||||||
|
addressScore = addressScore + 0.5
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
def totalScore = nameScore + addressScore
|
||||||
|
if(totalScore > maxScore){
|
||||||
|
maxScore = totalScore
|
||||||
|
data = poi
|
||||||
|
data.score = totalScore
|
||||||
|
}
|
||||||
|
println searchName+" "+name+" "+totalScore + " "+ (poi.address?poi.address:"")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return data;
|
||||||
|
}
|
||||||
|
|
||||||
|
def isAddress(address){
|
||||||
|
def REG_ADDR1 = "[\u4e00-\u9fa5]{2,20}[路|巷|街|弄|街道|大道|大街|巷子|公路|马路|大道西段]\\d+";
|
||||||
|
def pattern = Pattern.compile(REG_ADDR1);
|
||||||
|
def matcher = pattern.matcher(address)
|
||||||
|
if(matcher.find()){
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
def getMapRegion(def location){
|
||||||
|
def jsonObj = GaoDeApiUtils.geocode(location)
|
||||||
|
def district = null
|
||||||
|
if(jsonObj && jsonObj.status=="1"){
|
||||||
|
def detailMap = jsonObj.regeocode.addressComponent
|
||||||
|
if(!detailMap.city){
|
||||||
|
if(detailMap.province && detailMap.province.toString().endsWith("市")){
|
||||||
|
detailMap.city = detailMap.province
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return detailMap
|
||||||
|
}
|
||||||
|
return district;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
+102
@@ -0,0 +1,102 @@
|
|||||||
|
package com.fangjia.script.gj.test
|
||||||
|
|
||||||
|
import com.fangjia.core.RegionData
|
||||||
|
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
|
||||||
|
import com.fangjia.script.util.MongoUtils
|
||||||
|
import com.mongodb.BasicDBObject
|
||||||
|
import com.mongodb.DBCollection
|
||||||
|
|
||||||
|
class CrawlDistrictIdSelect {
|
||||||
|
def static final DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
|
||||||
|
|
||||||
|
static DBCollection districtseedsColl = MongoUtils.getCollectionByAuth20190102("seeddb", "district_seeds")
|
||||||
|
static DBCollection districtseeds2Coll = MongoUtils.getCollectionByAuth20190102("seeddb_dev", "district_seeds")
|
||||||
|
|
||||||
|
static main(args) {
|
||||||
|
def task = new CrawlDistrictIdSelect()
|
||||||
|
task.run()
|
||||||
|
// task.synch()
|
||||||
|
}
|
||||||
|
|
||||||
|
def synch(){
|
||||||
|
def q = new BasicDBObject()
|
||||||
|
def dbcursor = districtseeds2Coll.find(q)
|
||||||
|
def list = []
|
||||||
|
for(def d :dbcursor){
|
||||||
|
list << d
|
||||||
|
if(list.size()==10000){
|
||||||
|
districtseedsColl.insert(list)
|
||||||
|
list = []
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if(list.size()>0){
|
||||||
|
districtseedsColl.insert(list)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
def run(){
|
||||||
|
def datas = findCrawlData()
|
||||||
|
datas.each {k,v->
|
||||||
|
if(v){
|
||||||
|
for(def d : v){
|
||||||
|
d.c_date = new Date()
|
||||||
|
d.m_date = new Date()
|
||||||
|
districtseedsColl.save(d)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
def findCrawlData(){
|
||||||
|
def q = new BasicDBObject()
|
||||||
|
q.put("city","福州")
|
||||||
|
q.put("source", "安居客")
|
||||||
|
q.put("s_date",new BasicDBObject('$gte', 202001))
|
||||||
|
def dbcursor = thirdPartyColl.find(q)
|
||||||
|
def retMap = [:]
|
||||||
|
def prefix = "/community/view/"
|
||||||
|
def keyList =[]
|
||||||
|
for(def db : dbcursor){
|
||||||
|
def url = null
|
||||||
|
if(!db.url){
|
||||||
|
if(db.district_id){
|
||||||
|
url = db.district_id
|
||||||
|
}
|
||||||
|
}else{
|
||||||
|
url = db.url
|
||||||
|
}
|
||||||
|
if(url && url.contains(prefix) ){
|
||||||
|
def district_id = url.substring(url.indexOf(prefix)+prefix.length(),url.size())
|
||||||
|
if(district_id=="0"){
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
def shortCity = url.substring(url.indexOf("https://")+8, url.indexOf(".anjuke.com"))
|
||||||
|
def region = RegionData.formatRegion(db.city, db.region)
|
||||||
|
def key = shortCity +"#"+district_id
|
||||||
|
|
||||||
|
def ids = retMap.get(key)
|
||||||
|
if(!ids){
|
||||||
|
ids = []
|
||||||
|
retMap.put(key, ids)
|
||||||
|
}
|
||||||
|
|
||||||
|
if(!keyList.contains(key)){
|
||||||
|
def map = new BasicDBObject()
|
||||||
|
map.root = "anjuke.com"
|
||||||
|
map.city_short= shortCity
|
||||||
|
map.city = db.city
|
||||||
|
map.region = region
|
||||||
|
map.district_id = district_id
|
||||||
|
map.source_url = url
|
||||||
|
ids << map
|
||||||
|
|
||||||
|
keyList << key
|
||||||
|
}
|
||||||
|
}else{
|
||||||
|
println url +" 链接不合要求"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return retMap
|
||||||
|
}
|
||||||
|
}
|
||||||
+101
@@ -0,0 +1,101 @@
|
|||||||
|
package com.fangjia.script.gj.test
|
||||||
|
|
||||||
|
import com.fangjia.script.util.DateUtils
|
||||||
|
import com.fangjia.script.util.MongoUtils
|
||||||
|
import com.mongodb.BasicDBObject
|
||||||
|
import com.mongodb.DBCollection
|
||||||
|
import com.mongodb.QueryOperators
|
||||||
|
|
||||||
|
import java.util.concurrent.CountDownLatch
|
||||||
|
import java.util.concurrent.ExecutorService
|
||||||
|
import java.util.concurrent.Executors
|
||||||
|
import java.util.concurrent.atomic.AtomicInteger
|
||||||
|
|
||||||
|
class ExportToDealPrice {
|
||||||
|
|
||||||
|
DBCollection DEAL_PRICE_MATCH_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price_match", "192.168.0.43")
|
||||||
|
DBCollection DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43")
|
||||||
|
|
||||||
|
DBCollection TRADE_MATCH_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "trade_match_price", "192.168.0.136")
|
||||||
|
|
||||||
|
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(50)
|
||||||
|
final AtomicInteger count = new AtomicInteger()
|
||||||
|
|
||||||
|
static main(args) {
|
||||||
|
def task = new ExportToDealPrice();
|
||||||
|
task.run()
|
||||||
|
}
|
||||||
|
|
||||||
|
def run(){
|
||||||
|
def datas = findDatas()
|
||||||
|
|
||||||
|
final CountDownLatch latch = new CountDownLatch(datas.size())
|
||||||
|
for(def i=0;i<datas.size();i++){
|
||||||
|
final def data = datas[i]
|
||||||
|
fixedThreadPool.execute(new Runnable() {
|
||||||
|
void run() {
|
||||||
|
try{
|
||||||
|
count.addAndGet(1)
|
||||||
|
def dbo = addPrice(data)
|
||||||
|
if(dbo){
|
||||||
|
data.weekend_avg_price = dbo.avg_price
|
||||||
|
}
|
||||||
|
}catch(Exception e){
|
||||||
|
e.printStackTrace()
|
||||||
|
} finally {
|
||||||
|
latch.countDown()
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
latch.await()
|
||||||
|
|
||||||
|
save(datas);
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
def save(def datas){
|
||||||
|
def ret = [];
|
||||||
|
def index = 0;
|
||||||
|
for(def dbo : datas){
|
||||||
|
ret << dbo;
|
||||||
|
index++;
|
||||||
|
if(ret.size()%10000==0){
|
||||||
|
println "µÚ "+(index/10000)+" ¸ö"+ret.size()
|
||||||
|
TRADE_MATCH_PRICE_COLL.insert(ret);
|
||||||
|
ret = [];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if(ret.size()>0){
|
||||||
|
TRADE_MATCH_PRICE_COLL.insert(ret);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
def addPrice(data){
|
||||||
|
def weekend = DateUtils.getFirstDayOfWeek(data.trade_date)
|
||||||
|
def q = new BasicDBObject()
|
||||||
|
q.put("city", data.match_city)
|
||||||
|
q.put("region", data.match_region)
|
||||||
|
q.put("district_name", data.match_name)
|
||||||
|
q.put("weekend", weekend)
|
||||||
|
def dbo = DISTRICT_STAT_COLL.findOne(q)
|
||||||
|
if(!dbo){
|
||||||
|
q.put("weekend", new BasicDBObject(QueryOperators.LTE, weekend))
|
||||||
|
dbo = DISTRICT_STAT_COLL.findOne(q)
|
||||||
|
}
|
||||||
|
println count.get()+" "+data.city +" "+data.match_name
|
||||||
|
return dbo
|
||||||
|
}
|
||||||
|
|
||||||
|
def findDatas(){
|
||||||
|
def q = new BasicDBObject()
|
||||||
|
q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2020-04-01","yyyy-MM-dd")))
|
||||||
|
def dbcursor = DEAL_PRICE_MATCH_COLL.find(q)
|
||||||
|
def rets = []
|
||||||
|
for(def db : dbcursor){
|
||||||
|
rets << db
|
||||||
|
}
|
||||||
|
return rets;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,3 +1,7 @@
|
|||||||
package com.fangjia.script.gj.test
|
package com.fangjia.script.gj.test
|
||||||
|
|
||||||
println "wwww"
|
import java.text.SimpleDateFormat
|
||||||
|
|
||||||
|
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
|
||||||
|
Date date = new Date(1617211269868);
|
||||||
|
print dateFormat.format(date);
|
||||||
+52
@@ -0,0 +1,52 @@
|
|||||||
|
package com.fangjia.script.thirdPriceCrawl.other
|
||||||
|
|
||||||
|
import com.fangjia.script.util.ABuYunUtil
|
||||||
|
import org.apache.commons.io.IOUtils
|
||||||
|
import org.apache.http.HttpResponse
|
||||||
|
import org.apache.http.client.HttpClient
|
||||||
|
import org.apache.http.client.methods.HttpGet
|
||||||
|
import org.apache.http.impl.client.HttpClients
|
||||||
|
|
||||||
|
class AnjukeCrawl {
|
||||||
|
|
||||||
|
static main(args) {
|
||||||
|
def task = new AnjukeCrawl();
|
||||||
|
task.run()
|
||||||
|
}
|
||||||
|
|
||||||
|
def run(){
|
||||||
|
HttpClient client = HttpClients.createDefault();
|
||||||
|
HttpGet get = new HttpGet("https://m.anjuke.com/cityList/");
|
||||||
|
// get.setHeader("Upgrade-Insecure-Requests", "1");
|
||||||
|
// get.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36");
|
||||||
|
|
||||||
|
// get.setHeader("Host","m.anjuke.com");
|
||||||
|
// get.setHeader("Connection","keep-alive");
|
||||||
|
// get.setHeader("Cache-Control","max-age=0");
|
||||||
|
// get.setHeader('sec-ch-ua','"Chromium";v="88", "Google Chrome";v="88", ";Not A Brand";v="99"');
|
||||||
|
// get.setHeader("sec-ch-ua-mobile","'?0");
|
||||||
|
// get.setHeader("Upgrade-Insecure-Requests","1");
|
||||||
|
def header = [:]
|
||||||
|
header.put("User-Agent","Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.104 Safari/537.36");
|
||||||
|
header.put("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9");
|
||||||
|
// get.setHeader("Sec-Fetch-Site","none");
|
||||||
|
// get.setHeader("Sec-Fetch-Mode","navigate");
|
||||||
|
// get.setHeader("Sec-Fetch-User","?1");
|
||||||
|
// get.setHeader("Sec-Fetch-Dest","document");
|
||||||
|
//// get.setHeader("Accept-Encoding"," gzip, deflate, br");
|
||||||
|
// get.setHeader("Accept-Language","zh-CN,zh;q=0.9");
|
||||||
|
|
||||||
|
|
||||||
|
def s1 = ABuYunUtil.doGet("https://m.anjuke.com/cityList/", null, header, "utf-8")
|
||||||
|
printl s1
|
||||||
|
|
||||||
|
HttpResponse response = client.execute(get);
|
||||||
|
def s = getResponseBodyAsString(response)
|
||||||
|
println s
|
||||||
|
}
|
||||||
|
|
||||||
|
private static String getResponseBodyAsString(HttpResponse response) throws IOException {
|
||||||
|
return IOUtils.toString(response.getEntity().getContent(), "UTF-8");
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user