导出小区

This commit is contained in:
2022-01-05 11:34:01 +08:00
parent 300530db76
commit be88ca7c64
5 changed files with 158 additions and 30 deletions
@@ -40,7 +40,7 @@ class ExportOracleData {
// exportCommunityStaging() // exportCommunityStaging()
// exportCommunityBuileType() // exportCommunityBuileType()
// exportBigdataDistrictData() // exportBigdataDistrictData()
// exportBigdataCommunity() exportBigdataCommunity()
// exportBigdataDeal("2021-11-01") // exportBigdataDeal("2021-11-01")
// exportAvmEvaluationGJNB(202112) // exportAvmEvaluationGJNB(202112)
@@ -54,7 +54,7 @@ class ExportOracleData {
// exportCommunityPdescribeGj(202112) // exportCommunityPdescribeGj(202112)
// exportCommCaseDetailGj(202112) // exportCommCaseDetailGj(202112)
exportLastFile() // exportLastFile()
} }
@@ -10,7 +10,9 @@ import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.GeoHelper import com.fangjia.script.util.GeoHelper
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject import com.mongodb.DBObject
import org.apache.poi.ss.util.RegionUtil import org.apache.poi.ss.util.RegionUtil
import org.elasticsearch.index.query.BoolQueryBuilder import org.elasticsearch.index.query.BoolQueryBuilder
@@ -21,13 +23,17 @@ class MatchMapData {
def thirdPartyColl = com.fangjia.script.thirdPriceCrawl.CrawalDBUtil.getThirdPartyCol(); def thirdPartyColl = com.fangjia.script.thirdPriceCrawl.CrawalDBUtil.getThirdPartyCol();
public static final def ZS_MONGODB_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
public static final DBCollection ZS_LEVEL_CONN = MongoUtils.getCollectionByName2("fangjia", "district_analy", ZS_MONGODB_URL)
static main(args) { static main(args) {
def task = new MatchMapData() def task = new MatchMapData()
task.run() // task.run()
// task.run1() // task.run1()
// task.run2() // task.run2()
// task.run3() // task.run3()
// task.exportHouse() // task.exportHouse()
task.addDistrictLevel()
} }
@@ -306,7 +312,8 @@ class MatchMapData {
} }
def exportHouse(){ def exportHouse(){
def files = ["合肥","石家庄","北京","广州","杭州","上海","深圳","天津","武汉","西安","重庆","苏州"] //"合肥","石家庄","北京","广州","杭州","上海","深圳","天津","武汉","西安","重庆",
def files = ["苏州","合肥"]
for(def f : files) { for(def f : files) {
def city = f def city = f
@@ -326,7 +333,7 @@ class MatchMapData {
} }
},0) },0)
ExcelUtils.write("D:\\数据\\建行\\房产估值\\小区\\融合\\租房2\\"+city+"_rent.xlsx", rets, "数据") ExcelUtils.write("D:\\数据\\建行\\房产估值\\小区\\融合\\租房3\\"+city+"_rent.xlsx", rets, "数据")
} }
} }
@@ -338,7 +345,8 @@ class MatchMapData {
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false); QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat)) QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat))
.must(QueryBuilders.termQuery("root", "lianjia.com")).must(date_range).must(area_range) .must(date_range).must(area_range)
//.must(QueryBuilders.termQuery("root", "lianjia.com"))
if(!alias){ if(!alias){
alias = [] alias = []
@@ -376,5 +384,116 @@ class MatchMapData {
return rets; return rets;
} }
def addDistrictLevel(){
def files = ["合肥","苏州","上海","北京","广州","杭州","深圳","石家庄","天津","武汉","西安","重庆"]
def dataMap = null
def weekend = DateUtils.getFirstDayOfWeek(new Date());
for(def fn : files){
def dists = CommonDatautils.findAllDistrictByCity(fn, "district", ["city", "region","name"])
def priceMap = CommonDatautils.queryDistrictPrice(fn, weekend)
def datas = addPriceData(dists, priceMap);
def levelMap = calcuteLevel(datas);
dataMap = getDataMap(fn);
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\融合\\租售比\\详情2\\"+fn+".xlsx",{d->
def region = dataMap.get(d['C'])
if(region){
def newRegion = RegionData.formatRegion(fn, region)
if(newRegion){
def level = levelMap.get( fn+"##"+newRegion+"##"+d['D'])
if(level){
d['J'] = level
}
}
}
},0)
}
}
def addPriceData(dists, priceMap){
def newLists = []
for(def d : dists){
def price = priceMap.get(d.city+"##"+d.region+"##"+d.name);
if(price){
d.avg_price = price;
newLists << d;
}
}
if(newLists){
newLists.sort {a,b->
def v1 = a.avg_price
def v2 = b.avg_price
return v1.compareTo(v2)
}
}
return newLists;
}
def calcuteLevel(def datas){
def total = datas.size()
def v = (int)(total/10)
def start = 0;
def end = 0;
def map = [:]
for(int i=1;i<=10;i++){
end = end + v;
if(i == 10){
map.put(i, [start, total])
}else{
map.put(i, [start, end])
}
start = end;
}
for(def d=0;d<datas.size();d++){
def dist = datas[d]
for (def o in map) {
def ranges = o.value
def m = ranges[0]
def n = ranges[1]
if(d==0){
if(d<=n){
dist.level = 10 - o.key + 1
break
}
}else{
if(d>m && d<=n){
dist.level = 10 - o.key + 1
break
}
}
}
}
def dataMap = [:]
for(def d : datas){
dataMap.put(d.city+"##"+d.region+"##"+d.name, d.level)
}
return dataMap;
}
def getDataMap(city){
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\小区\\融合\\租售比\\"+city+".xlsx",0,1)
def map = [:]
for(def d : datas){
map.put(d['D'], d['E'])
}
return map
}
def getDistrictLevel(city, region, name){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("name", name)
return ZS_LEVEL_CONN.findOne(q)
}
} }
@@ -2,6 +2,7 @@ package com.fangjia.script.gj.test
import com.fangjia.core.RegionData import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject import com.mongodb.BasicDBObject
@@ -29,25 +30,33 @@ import java.text.SimpleDateFormat
// } // }
//}) //})
def onLineSqlInstance = Sql.newInstance("jdbc:mysql://mysql.db.fangjia.com:4306/fangjia_youfang?&characterEncoding=utf8", "sa", "G00jia", "com.mysql.jdbc.Driver"); //def onLineSqlInstance = Sql.newInstance("jdbc:mysql://mysql.db.fangjia.com:4306/fangjia_youfang?&characterEncoding=utf8", "sa", "G00jia", "com.mysql.jdbc.Driver");
//
//def sql = "select visit_time,ip from visit_log where visit_time>='2020-09-18'";
//def countMap = [:]
//onLineSqlInstance.eachRow(sql,{r->
// def addTime = r.visit_time
// def month = DateUtils.formatDateByPattern(addTime, "yyyy-MM")
// def ips = countMap.get(month)
// if(!ips){
// ips = new HashSet();
// countMap.put(month, ips)
// }
// ips << r.ip
//});
//
//countMap.each {k,v->
// println k+" "+v.size()
//}
def sql = "select visit_time,ip from visit_log where visit_time>='2020-09-18'";
def countMap = [:]
onLineSqlInstance.eachRow(sql,{r->
def addTime = r.visit_time
def month = DateUtils.formatDateByPattern(addTime, "yyyy-MM")
def ips = countMap.get(month)
if(!ips){
ips = new HashSet();
countMap.put(month, ips)
}
ips << r.ip
});
countMap.each {k,v-> def dists = CommonDatautils.findAllDistrictByCity("上海", "district", ["city","region","name","finish_building_date"])
println k+" "+v.size() def rets = []
for(def d : dists){
def year = CommonUtils.getDistrictAge(d)
rets << [d.city,d.region, d.name, year]
} }
ExcelUtils.write("D:\\数据\\统计房龄.xlsx", rets, "数据")
@@ -37,12 +37,12 @@ class Esf58Crawl extends BaseBatch{
def static final distMap = [:] def static final distMap = [:]
def static useDistKeys = [] def static useDistKeys = []
static def totalCount = new AtomicLong() static def totalCount = new AtomicLong()
//"福州":"fz|304","合肥":"hf|837","芜湖":"wuhu|2045","蚌埠":"bengbu|3470","阜阳":"fy|2325","淮南":"hn|2319","成都":"cd|102","上海":"sh|2", "北京":"bj|1","天津":"tj|18", //"福州":"fz|304","合肥":"hf|837","芜湖":"wuhu|2045","蚌埠":"bengbu|3470","阜阳":"fy|2325","淮南":"hn|2319","成都":"cd|102",
//"重庆":"cq|37","石家庄":"sjz|241","广州":"gz|3","深圳":"sz|4","东莞":"dg|413","佛山":"fs|222","杭州":"hz|79","武汉":"wh|158","青岛":"qd|122","南京":"nj|172", // "上海":"sh|2", "北京":"bj|1","天津":"tj|18",
//"嘉兴":"jx|497","大连":"dl|147","乌鲁木齐":"xj|984","太原":"ty|740","兰州":"lz|952","南昌":"nc|669", // "重庆":"cq|37","石家庄":"sjz|241","广州":"gz|3","深圳":"sz|4","东莞":"dg|413","佛山":"fs|222","杭州":"hz|79","武汉":"wh|158",
def cityList = [ def cityList = ["青岛":"qd|122","南京":"nj|172",
"嘉兴":"jx|497","大连":"dl|147","乌鲁木齐":"xj|984","太原":"ty|740","兰州":"lz|952","南昌":"nc|669","沈阳":"sy|188","西安":"xa|483","苏州":"su|5","无锡":"wx|93","潍坊":"wf|362","常州":"cz|463",
"沈阳":"sy|188","西安":"xa|483","苏州":"su|5","无锡":"wx|93","潍坊":"wf|362","常州":"cz|463", "南通":"nt|394","吉林":"jl|700","中山":"zs|771","台州":"tz|403","东营":"dy|623","三亚":"sanya|2422", "南通":"nt|394","吉林":"jl|700","中山":"zs|771","台州":"tz|403","东营":"dy|623","三亚":"sanya|2422",
"呼和浩特":"hu|811","济南":"jn|265","温州":"wz|330","咸阳":"xianyang|7453","安庆":"anqing|3251","宿州":"suzhou|3359","六安":"la|2328","唐山":"ts|276","商丘":"sq|1029","大同":"dt|6964","威海":"weihai|518","安阳":"ay|1096", "呼和浩特":"hu|811","济南":"jn|265","温州":"wz|330","咸阳":"xianyang|7453","安庆":"anqing|3251","宿州":"suzhou|3359","六安":"la|2328","唐山":"ts|276","商丘":"sq|1029","大同":"dt|6964","威海":"weihai|518","安阳":"ay|1096",
"临汾":"linfen|5669","临沂":"linyi|505","九江":"jj|2247","淄博":"zb|385","盐城":"yancheng|613","烟台":"yt|228","信阳":"xy|8694","南充":"nanchong|2378","洛阳":"luoyang|556","南阳":"ny|592","厦门":"xm|606", "临汾":"linfen|5669","临沂":"linyi|505","九江":"jj|2247","淄博":"zb|385","盐城":"yancheng|613","烟台":"yt|228","信阳":"xy|8694","南充":"nanchong|2378","洛阳":"luoyang|556","南阳":"ny|592","厦门":"xm|606",
"宜昌":"yc|858","张家口":"zjk|3328","昆明":"km|541","柳州":"liuzhou|7133","绵阳":"mianyang|1057", "泉州":"qz|291","济宁":"jining|450","贵阳":"gy|2015","连云港":"lyg|2049","邯郸":"hd|572", "宜昌":"yc|858","张家口":"zjk|3328","昆明":"km|541","柳州":"liuzhou|7133","绵阳":"mianyang|1057", "泉州":"qz|291","济宁":"jining|450","贵阳":"gy|2015","连云港":"lyg|2049","邯郸":"hd|572",
@@ -14,8 +14,8 @@ class TestCrawlTotalCount {
def run(){ def run(){
def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"] def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"]
for(def source :sources){ for(def source :sources){
def count1 = findTotalCount(source, 202110); def count1 = findTotalCount(source, 202111);
def count2 = findTotalCount(source, 202111); def count2 = findTotalCount(source, 202112);
println source +" "+count1 +" "+count2 println source +" "+count1 +" "+count2
} }
} }