导出小区

This commit is contained in:
2022-01-05 11:34:01 +08:00
parent 300530db76
commit be88ca7c64
5 changed files with 158 additions and 30 deletions
@@ -40,7 +40,7 @@ class ExportOracleData {
// exportCommunityStaging()
// exportCommunityBuileType()
// exportBigdataDistrictData()
// exportBigdataCommunity()
exportBigdataCommunity()
// exportBigdataDeal("2021-11-01")
// exportAvmEvaluationGJNB(202112)
@@ -54,7 +54,7 @@ class ExportOracleData {
// exportCommunityPdescribeGj(202112)
// exportCommCaseDetailGj(202112)
exportLastFile()
// exportLastFile()
}
@@ -10,7 +10,9 @@ import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.GeoHelper
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import org.apache.poi.ss.util.RegionUtil
import org.elasticsearch.index.query.BoolQueryBuilder
@@ -21,13 +23,17 @@ class MatchMapData {
def thirdPartyColl = com.fangjia.script.thirdPriceCrawl.CrawalDBUtil.getThirdPartyCol();
public static final def ZS_MONGODB_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
public static final DBCollection ZS_LEVEL_CONN = MongoUtils.getCollectionByName2("fangjia", "district_analy", ZS_MONGODB_URL)
static main(args) {
def task = new MatchMapData()
task.run()
// task.run()
// task.run1()
// task.run2()
// task.run3()
// task.exportHouse()
task.addDistrictLevel()
}
@@ -306,7 +312,8 @@ class MatchMapData {
}
def exportHouse(){
def files = ["合肥","石家庄","北京","广州","杭州","上海","深圳","天津","武汉","西安","重庆","苏州"]
//"合肥","石家庄","北京","广州","杭州","上海","深圳","天津","武汉","西安","重庆",
def files = ["苏州","合肥"]
for(def f : files) {
def city = f
@@ -326,7 +333,7 @@ class MatchMapData {
}
},0)
ExcelUtils.write("D:\\数据\\建行\\房产估值\\小区\\融合\\租房2\\"+city+"_rent.xlsx", rets, "数据")
ExcelUtils.write("D:\\数据\\建行\\房产估值\\小区\\融合\\租房3\\"+city+"_rent.xlsx", rets, "数据")
}
}
@@ -338,7 +345,8 @@ class MatchMapData {
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat))
.must(QueryBuilders.termQuery("root", "lianjia.com")).must(date_range).must(area_range)
.must(date_range).must(area_range)
//.must(QueryBuilders.termQuery("root", "lianjia.com"))
if(!alias){
alias = []
@@ -376,5 +384,116 @@ class MatchMapData {
return rets;
}
def addDistrictLevel(){
def files = ["合肥","苏州","上海","北京","广州","杭州","深圳","石家庄","天津","武汉","西安","重庆"]
def dataMap = null
def weekend = DateUtils.getFirstDayOfWeek(new Date());
for(def fn : files){
def dists = CommonDatautils.findAllDistrictByCity(fn, "district", ["city", "region","name"])
def priceMap = CommonDatautils.queryDistrictPrice(fn, weekend)
def datas = addPriceData(dists, priceMap);
def levelMap = calcuteLevel(datas);
dataMap = getDataMap(fn);
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\融合\\租售比\\详情2\\"+fn+".xlsx",{d->
def region = dataMap.get(d['C'])
if(region){
def newRegion = RegionData.formatRegion(fn, region)
if(newRegion){
def level = levelMap.get( fn+"##"+newRegion+"##"+d['D'])
if(level){
d['J'] = level
}
}
}
},0)
}
}
def addPriceData(dists, priceMap){
def newLists = []
for(def d : dists){
def price = priceMap.get(d.city+"##"+d.region+"##"+d.name);
if(price){
d.avg_price = price;
newLists << d;
}
}
if(newLists){
newLists.sort {a,b->
def v1 = a.avg_price
def v2 = b.avg_price
return v1.compareTo(v2)
}
}
return newLists;
}
def calcuteLevel(def datas){
def total = datas.size()
def v = (int)(total/10)
def start = 0;
def end = 0;
def map = [:]
for(int i=1;i<=10;i++){
end = end + v;
if(i == 10){
map.put(i, [start, total])
}else{
map.put(i, [start, end])
}
start = end;
}
for(def d=0;d<datas.size();d++){
def dist = datas[d]
for (def o in map) {
def ranges = o.value
def m = ranges[0]
def n = ranges[1]
if(d==0){
if(d<=n){
dist.level = 10 - o.key + 1
break
}
}else{
if(d>m && d<=n){
dist.level = 10 - o.key + 1
break
}
}
}
}
def dataMap = [:]
for(def d : datas){
dataMap.put(d.city+"##"+d.region+"##"+d.name, d.level)
}
return dataMap;
}
def getDataMap(city){
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\小区\\融合\\租售比\\"+city+".xlsx",0,1)
def map = [:]
for(def d : datas){
map.put(d['D'], d['E'])
}
return map
}
def getDistrictLevel(city, region, name){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("name", name)
return ZS_LEVEL_CONN.findOne(q)
}
}
@@ -2,6 +2,7 @@ package com.fangjia.script.gj.test
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
@@ -29,25 +30,33 @@ import java.text.SimpleDateFormat
// }
//})
def onLineSqlInstance = Sql.newInstance("jdbc:mysql://mysql.db.fangjia.com:4306/fangjia_youfang?&characterEncoding=utf8", "sa", "G00jia", "com.mysql.jdbc.Driver");
//def onLineSqlInstance = Sql.newInstance("jdbc:mysql://mysql.db.fangjia.com:4306/fangjia_youfang?&characterEncoding=utf8", "sa", "G00jia", "com.mysql.jdbc.Driver");
//
//def sql = "select visit_time,ip from visit_log where visit_time>='2020-09-18'";
//def countMap = [:]
//onLineSqlInstance.eachRow(sql,{r->
// def addTime = r.visit_time
// def month = DateUtils.formatDateByPattern(addTime, "yyyy-MM")
// def ips = countMap.get(month)
// if(!ips){
// ips = new HashSet();
// countMap.put(month, ips)
// }
// ips << r.ip
//});
//
//countMap.each {k,v->
// println k+" "+v.size()
//}
def sql = "select visit_time,ip from visit_log where visit_time>='2020-09-18'";
def countMap = [:]
onLineSqlInstance.eachRow(sql,{r->
def addTime = r.visit_time
def month = DateUtils.formatDateByPattern(addTime, "yyyy-MM")
def ips = countMap.get(month)
if(!ips){
ips = new HashSet();
countMap.put(month, ips)
}
ips << r.ip
});
countMap.each {k,v->
println k+" "+v.size()
def dists = CommonDatautils.findAllDistrictByCity("上海", "district", ["city","region","name","finish_building_date"])
def rets = []
for(def d : dists){
def year = CommonUtils.getDistrictAge(d)
rets << [d.city,d.region, d.name, year]
}
ExcelUtils.write("D:\\数据\\统计房龄.xlsx", rets, "数据")
@@ -37,12 +37,12 @@ class Esf58Crawl extends BaseBatch{
def static final distMap = [:]
def static useDistKeys = []
static def totalCount = new AtomicLong()
//"福州":"fz|304","合肥":"hf|837","芜湖":"wuhu|2045","蚌埠":"bengbu|3470","阜阳":"fy|2325","淮南":"hn|2319","成都":"cd|102","上海":"sh|2", "北京":"bj|1","天津":"tj|18",
//"重庆":"cq|37","石家庄":"sjz|241","广州":"gz|3","深圳":"sz|4","东莞":"dg|413","佛山":"fs|222","杭州":"hz|79","武汉":"wh|158","青岛":"qd|122","南京":"nj|172",
//"嘉兴":"jx|497","大连":"dl|147","乌鲁木齐":"xj|984","太原":"ty|740","兰州":"lz|952","南昌":"nc|669",
def cityList = [
"沈阳":"sy|188","西安":"xa|483","苏州":"su|5","无锡":"wx|93","潍坊":"wf|362","常州":"cz|463", "南通":"nt|394","吉林":"jl|700","中山":"zs|771","台州":"tz|403","东营":"dy|623","三亚":"sanya|2422",
//"福州":"fz|304","合肥":"hf|837","芜湖":"wuhu|2045","蚌埠":"bengbu|3470","阜阳":"fy|2325","淮南":"hn|2319","成都":"cd|102",
// "上海":"sh|2", "北京":"bj|1","天津":"tj|18",
// "重庆":"cq|37","石家庄":"sjz|241","广州":"gz|3","深圳":"sz|4","东莞":"dg|413","佛山":"fs|222","杭州":"hz|79","武汉":"wh|158",
def cityList = ["青岛":"qd|122","南京":"nj|172",
"嘉兴":"jx|497","大连":"dl|147","乌鲁木齐":"xj|984","太原":"ty|740","兰州":"lz|952","南昌":"nc|669","沈阳":"sy|188","西安":"xa|483","苏州":"su|5","无锡":"wx|93","潍坊":"wf|362","常州":"cz|463",
"南通":"nt|394","吉林":"jl|700","中山":"zs|771","台州":"tz|403","东营":"dy|623","三亚":"sanya|2422",
"呼和浩特":"hu|811","济南":"jn|265","温州":"wz|330","咸阳":"xianyang|7453","安庆":"anqing|3251","宿州":"suzhou|3359","六安":"la|2328","唐山":"ts|276","商丘":"sq|1029","大同":"dt|6964","威海":"weihai|518","安阳":"ay|1096",
"临汾":"linfen|5669","临沂":"linyi|505","九江":"jj|2247","淄博":"zb|385","盐城":"yancheng|613","烟台":"yt|228","信阳":"xy|8694","南充":"nanchong|2378","洛阳":"luoyang|556","南阳":"ny|592","厦门":"xm|606",
"宜昌":"yc|858","张家口":"zjk|3328","昆明":"km|541","柳州":"liuzhou|7133","绵阳":"mianyang|1057", "泉州":"qz|291","济宁":"jining|450","贵阳":"gy|2015","连云港":"lyg|2049","邯郸":"hd|572",
@@ -14,8 +14,8 @@ class TestCrawlTotalCount {
def run(){
def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"]
for(def source :sources){
def count1 = findTotalCount(source, 202110);
def count2 = findTotalCount(source, 202111);
def count1 = findTotalCount(source, 202111);
def count2 = findTotalCount(source, 202112);
println source +" "+count1 +" "+count2
}
}