导出小区
This commit is contained in:
+2
-2
@@ -40,7 +40,7 @@ class ExportOracleData {
|
||||
// exportCommunityStaging()
|
||||
// exportCommunityBuileType()
|
||||
// exportBigdataDistrictData()
|
||||
// exportBigdataCommunity()
|
||||
exportBigdataCommunity()
|
||||
// exportBigdataDeal("2021-11-01")
|
||||
|
||||
// exportAvmEvaluationGJNB(202112)
|
||||
@@ -54,7 +54,7 @@ class ExportOracleData {
|
||||
|
||||
// exportCommunityPdescribeGj(202112)
|
||||
// exportCommCaseDetailGj(202112)
|
||||
exportLastFile()
|
||||
// exportLastFile()
|
||||
}
|
||||
|
||||
|
||||
|
||||
+123
-4
@@ -10,7 +10,9 @@ import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.GeoHelper
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import org.apache.poi.ss.util.RegionUtil
|
||||
import org.elasticsearch.index.query.BoolQueryBuilder
|
||||
@@ -21,13 +23,17 @@ class MatchMapData {
|
||||
|
||||
def thirdPartyColl = com.fangjia.script.thirdPriceCrawl.CrawalDBUtil.getThirdPartyCol();
|
||||
|
||||
public static final def ZS_MONGODB_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
public static final DBCollection ZS_LEVEL_CONN = MongoUtils.getCollectionByName2("fangjia", "district_analy", ZS_MONGODB_URL)
|
||||
|
||||
static main(args) {
|
||||
def task = new MatchMapData()
|
||||
task.run()
|
||||
// task.run()
|
||||
// task.run1()
|
||||
// task.run2()
|
||||
// task.run3()
|
||||
// task.exportHouse()
|
||||
task.addDistrictLevel()
|
||||
|
||||
}
|
||||
|
||||
@@ -306,7 +312,8 @@ class MatchMapData {
|
||||
}
|
||||
|
||||
def exportHouse(){
|
||||
def files = ["合肥","石家庄","北京","广州","杭州","上海","深圳","天津","武汉","西安","重庆","苏州"]
|
||||
//"合肥","石家庄","北京","广州","杭州","上海","深圳","天津","武汉","西安","重庆",
|
||||
def files = ["苏州","合肥"]
|
||||
for(def f : files) {
|
||||
def city = f
|
||||
|
||||
@@ -326,7 +333,7 @@ class MatchMapData {
|
||||
}
|
||||
},0)
|
||||
|
||||
ExcelUtils.write("D:\\数据\\建行\\房产估值\\小区\\融合\\租房2\\"+city+"_rent.xlsx", rets, "数据")
|
||||
ExcelUtils.write("D:\\数据\\建行\\房产估值\\小区\\融合\\租房3\\"+city+"_rent.xlsx", rets, "数据")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -338,7 +345,8 @@ class MatchMapData {
|
||||
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
|
||||
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat))
|
||||
.must(QueryBuilders.termQuery("root", "lianjia.com")).must(date_range).must(area_range)
|
||||
.must(date_range).must(area_range)
|
||||
//.must(QueryBuilders.termQuery("root", "lianjia.com"))
|
||||
|
||||
if(!alias){
|
||||
alias = []
|
||||
@@ -376,5 +384,116 @@ class MatchMapData {
|
||||
return rets;
|
||||
}
|
||||
|
||||
def addDistrictLevel(){
|
||||
def files = ["合肥","苏州","上海","北京","广州","杭州","深圳","石家庄","天津","武汉","西安","重庆"]
|
||||
|
||||
def dataMap = null
|
||||
|
||||
def weekend = DateUtils.getFirstDayOfWeek(new Date());
|
||||
|
||||
for(def fn : files){
|
||||
def dists = CommonDatautils.findAllDistrictByCity(fn, "district", ["city", "region","name"])
|
||||
def priceMap = CommonDatautils.queryDistrictPrice(fn, weekend)
|
||||
def datas = addPriceData(dists, priceMap);
|
||||
def levelMap = calcuteLevel(datas);
|
||||
|
||||
dataMap = getDataMap(fn);
|
||||
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\融合\\租售比\\详情2\\"+fn+".xlsx",{d->
|
||||
|
||||
def region = dataMap.get(d['C'])
|
||||
if(region){
|
||||
def newRegion = RegionData.formatRegion(fn, region)
|
||||
if(newRegion){
|
||||
def level = levelMap.get( fn+"##"+newRegion+"##"+d['D'])
|
||||
if(level){
|
||||
d['J'] = level
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
},0)
|
||||
}
|
||||
}
|
||||
|
||||
def addPriceData(dists, priceMap){
|
||||
def newLists = []
|
||||
for(def d : dists){
|
||||
def price = priceMap.get(d.city+"##"+d.region+"##"+d.name);
|
||||
if(price){
|
||||
d.avg_price = price;
|
||||
newLists << d;
|
||||
}
|
||||
}
|
||||
if(newLists){
|
||||
newLists.sort {a,b->
|
||||
def v1 = a.avg_price
|
||||
def v2 = b.avg_price
|
||||
return v1.compareTo(v2)
|
||||
}
|
||||
}
|
||||
return newLists;
|
||||
}
|
||||
|
||||
def calcuteLevel(def datas){
|
||||
def total = datas.size()
|
||||
def v = (int)(total/10)
|
||||
def start = 0;
|
||||
def end = 0;
|
||||
def map = [:]
|
||||
for(int i=1;i<=10;i++){
|
||||
end = end + v;
|
||||
if(i == 10){
|
||||
map.put(i, [start, total])
|
||||
}else{
|
||||
map.put(i, [start, end])
|
||||
}
|
||||
start = end;
|
||||
}
|
||||
for(def d=0;d<datas.size();d++){
|
||||
def dist = datas[d]
|
||||
|
||||
for (def o in map) {
|
||||
def ranges = o.value
|
||||
def m = ranges[0]
|
||||
def n = ranges[1]
|
||||
if(d==0){
|
||||
if(d<=n){
|
||||
dist.level = 10 - o.key + 1
|
||||
break
|
||||
}
|
||||
}else{
|
||||
if(d>m && d<=n){
|
||||
dist.level = 10 - o.key + 1
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
def dataMap = [:]
|
||||
for(def d : datas){
|
||||
dataMap.put(d.city+"##"+d.region+"##"+d.name, d.level)
|
||||
}
|
||||
return dataMap;
|
||||
}
|
||||
|
||||
def getDataMap(city){
|
||||
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\小区\\融合\\租售比\\"+city+".xlsx",0,1)
|
||||
def map = [:]
|
||||
for(def d : datas){
|
||||
map.put(d['D'], d['E'])
|
||||
}
|
||||
return map
|
||||
|
||||
}
|
||||
def getDistrictLevel(city, region, name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
q.put("name", name)
|
||||
return ZS_LEVEL_CONN.findOne(q)
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@ package com.fangjia.script.gj.test
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
@@ -29,25 +30,33 @@ import java.text.SimpleDateFormat
|
||||
// }
|
||||
//})
|
||||
|
||||
def onLineSqlInstance = Sql.newInstance("jdbc:mysql://mysql.db.fangjia.com:4306/fangjia_youfang?&characterEncoding=utf8", "sa", "G00jia", "com.mysql.jdbc.Driver");
|
||||
//def onLineSqlInstance = Sql.newInstance("jdbc:mysql://mysql.db.fangjia.com:4306/fangjia_youfang?&characterEncoding=utf8", "sa", "G00jia", "com.mysql.jdbc.Driver");
|
||||
//
|
||||
//def sql = "select visit_time,ip from visit_log where visit_time>='2020-09-18'";
|
||||
//def countMap = [:]
|
||||
//onLineSqlInstance.eachRow(sql,{r->
|
||||
// def addTime = r.visit_time
|
||||
// def month = DateUtils.formatDateByPattern(addTime, "yyyy-MM")
|
||||
// def ips = countMap.get(month)
|
||||
// if(!ips){
|
||||
// ips = new HashSet();
|
||||
// countMap.put(month, ips)
|
||||
// }
|
||||
// ips << r.ip
|
||||
//});
|
||||
//
|
||||
//countMap.each {k,v->
|
||||
// println k+" "+v.size()
|
||||
//}
|
||||
|
||||
def sql = "select visit_time,ip from visit_log where visit_time>='2020-09-18'";
|
||||
def countMap = [:]
|
||||
onLineSqlInstance.eachRow(sql,{r->
|
||||
def addTime = r.visit_time
|
||||
def month = DateUtils.formatDateByPattern(addTime, "yyyy-MM")
|
||||
def ips = countMap.get(month)
|
||||
if(!ips){
|
||||
ips = new HashSet();
|
||||
countMap.put(month, ips)
|
||||
}
|
||||
ips << r.ip
|
||||
});
|
||||
|
||||
countMap.each {k,v->
|
||||
println k+" "+v.size()
|
||||
def dists = CommonDatautils.findAllDistrictByCity("上海", "district", ["city","region","name","finish_building_date"])
|
||||
def rets = []
|
||||
for(def d : dists){
|
||||
def year = CommonUtils.getDistrictAge(d)
|
||||
rets << [d.city,d.region, d.name, year]
|
||||
}
|
||||
|
||||
ExcelUtils.write("D:\\数据\\统计房龄.xlsx", rets, "数据")
|
||||
|
||||
|
||||
|
||||
|
||||
+6
-6
@@ -37,12 +37,12 @@ class Esf58Crawl extends BaseBatch{
|
||||
def static final distMap = [:]
|
||||
def static useDistKeys = []
|
||||
static def totalCount = new AtomicLong()
|
||||
//"福州":"fz|304","合肥":"hf|837","芜湖":"wuhu|2045","蚌埠":"bengbu|3470","阜阳":"fy|2325","淮南":"hn|2319","成都":"cd|102","上海":"sh|2", "北京":"bj|1","天津":"tj|18",
|
||||
//"重庆":"cq|37","石家庄":"sjz|241","广州":"gz|3","深圳":"sz|4","东莞":"dg|413","佛山":"fs|222","杭州":"hz|79","武汉":"wh|158","青岛":"qd|122","南京":"nj|172",
|
||||
//"嘉兴":"jx|497","大连":"dl|147","乌鲁木齐":"xj|984","太原":"ty|740","兰州":"lz|952","南昌":"nc|669",
|
||||
def cityList = [
|
||||
|
||||
"沈阳":"sy|188","西安":"xa|483","苏州":"su|5","无锡":"wx|93","潍坊":"wf|362","常州":"cz|463", "南通":"nt|394","吉林":"jl|700","中山":"zs|771","台州":"tz|403","东营":"dy|623","三亚":"sanya|2422",
|
||||
//"福州":"fz|304","合肥":"hf|837","芜湖":"wuhu|2045","蚌埠":"bengbu|3470","阜阳":"fy|2325","淮南":"hn|2319","成都":"cd|102",
|
||||
// "上海":"sh|2", "北京":"bj|1","天津":"tj|18",
|
||||
// "重庆":"cq|37","石家庄":"sjz|241","广州":"gz|3","深圳":"sz|4","东莞":"dg|413","佛山":"fs|222","杭州":"hz|79","武汉":"wh|158",
|
||||
def cityList = ["青岛":"qd|122","南京":"nj|172",
|
||||
"嘉兴":"jx|497","大连":"dl|147","乌鲁木齐":"xj|984","太原":"ty|740","兰州":"lz|952","南昌":"nc|669","沈阳":"sy|188","西安":"xa|483","苏州":"su|5","无锡":"wx|93","潍坊":"wf|362","常州":"cz|463",
|
||||
"南通":"nt|394","吉林":"jl|700","中山":"zs|771","台州":"tz|403","东营":"dy|623","三亚":"sanya|2422",
|
||||
"呼和浩特":"hu|811","济南":"jn|265","温州":"wz|330","咸阳":"xianyang|7453","安庆":"anqing|3251","宿州":"suzhou|3359","六安":"la|2328","唐山":"ts|276","商丘":"sq|1029","大同":"dt|6964","威海":"weihai|518","安阳":"ay|1096",
|
||||
"临汾":"linfen|5669","临沂":"linyi|505","九江":"jj|2247","淄博":"zb|385","盐城":"yancheng|613","烟台":"yt|228","信阳":"xy|8694","南充":"nanchong|2378","洛阳":"luoyang|556","南阳":"ny|592","厦门":"xm|606",
|
||||
"宜昌":"yc|858","张家口":"zjk|3328","昆明":"km|541","柳州":"liuzhou|7133","绵阳":"mianyang|1057", "泉州":"qz|291","济宁":"jining|450","贵阳":"gy|2015","连云港":"lyg|2049","邯郸":"hd|572",
|
||||
|
||||
+2
-2
@@ -14,8 +14,8 @@ class TestCrawlTotalCount {
|
||||
def run(){
|
||||
def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"]
|
||||
for(def source :sources){
|
||||
def count1 = findTotalCount(source, 202110);
|
||||
def count2 = findTotalCount(source, 202111);
|
||||
def count1 = findTotalCount(source, 202111);
|
||||
def count2 = findTotalCount(source, 202112);
|
||||
println source +" "+count1 +" "+count2
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user