diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy index 8749dacb7..e86a065cc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -40,7 +40,7 @@ class ExportOracleData { // exportCommunityStaging() // exportCommunityBuileType() // exportBigdataDistrictData() -// exportBigdataCommunity() + exportBigdataCommunity() // exportBigdataDeal("2021-11-01") // exportAvmEvaluationGJNB(202112) @@ -54,7 +54,7 @@ class ExportOracleData { // exportCommunityPdescribeGj(202112) // exportCommCaseDetailGj(202112) - exportLastFile() +// exportLastFile() } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/MatchMapData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/MatchMapData.groovy index 663546757..51ea1deba 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/MatchMapData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/MatchMapData.groovy @@ -10,7 +10,9 @@ import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.GeoHelper +import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject +import com.mongodb.DBCollection import com.mongodb.DBObject import org.apache.poi.ss.util.RegionUtil import org.elasticsearch.index.query.BoolQueryBuilder @@ -21,13 +23,17 @@ class MatchMapData { def thirdPartyColl = com.fangjia.script.thirdPriceCrawl.CrawalDBUtil.getThirdPartyCol(); + public static final def ZS_MONGODB_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; + public static final DBCollection ZS_LEVEL_CONN = MongoUtils.getCollectionByName2("fangjia", "district_analy", ZS_MONGODB_URL) + static main(args) { def task = new MatchMapData() - task.run() +// task.run() // task.run1() // task.run2() // task.run3() // task.exportHouse() + task.addDistrictLevel() } @@ -306,7 +312,8 @@ class MatchMapData { } def exportHouse(){ - def files = ["合肥","石家庄","北京","广州","杭州","上海","深圳","天津","武汉","西安","重庆","苏州"] + //"合肥","石家庄","北京","广州","杭州","上海","深圳","天津","武汉","西安","重庆", + def files = ["苏州","合肥"] for(def f : files) { def city = f @@ -326,7 +333,7 @@ class MatchMapData { } },0) - ExcelUtils.write("D:\\数据\\建行\\房产估值\\小区\\融合\\租房2\\"+city+"_rent.xlsx", rets, "数据") + ExcelUtils.write("D:\\数据\\建行\\房产估值\\小区\\融合\\租房3\\"+city+"_rent.xlsx", rets, "数据") } } @@ -338,7 +345,8 @@ class MatchMapData { QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false); QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat)) - .must(QueryBuilders.termQuery("root", "lianjia.com")).must(date_range).must(area_range) + .must(date_range).must(area_range) + //.must(QueryBuilders.termQuery("root", "lianjia.com")) if(!alias){ alias = [] @@ -376,5 +384,116 @@ class MatchMapData { return rets; } + def addDistrictLevel(){ + def files = ["合肥","苏州","上海","北京","广州","杭州","深圳","石家庄","天津","武汉","西安","重庆"] + + def dataMap = null + + def weekend = DateUtils.getFirstDayOfWeek(new Date()); + + for(def fn : files){ + def dists = CommonDatautils.findAllDistrictByCity(fn, "district", ["city", "region","name"]) + def priceMap = CommonDatautils.queryDistrictPrice(fn, weekend) + def datas = addPriceData(dists, priceMap); + def levelMap = calcuteLevel(datas); + + dataMap = getDataMap(fn); + + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\融合\\租售比\\详情2\\"+fn+".xlsx",{d-> + + def region = dataMap.get(d['C']) + if(region){ + def newRegion = RegionData.formatRegion(fn, region) + if(newRegion){ + def level = levelMap.get( fn+"##"+newRegion+"##"+d['D']) + if(level){ + d['J'] = level + } + } + } + + },0) + } + } + + def addPriceData(dists, priceMap){ + def newLists = [] + for(def d : dists){ + def price = priceMap.get(d.city+"##"+d.region+"##"+d.name); + if(price){ + d.avg_price = price; + newLists << d; + } + } + if(newLists){ + newLists.sort {a,b-> + def v1 = a.avg_price + def v2 = b.avg_price + return v1.compareTo(v2) + } + } + return newLists; + } + + def calcuteLevel(def datas){ + def total = datas.size() + def v = (int)(total/10) + def start = 0; + def end = 0; + def map = [:] + for(int i=1;i<=10;i++){ + end = end + v; + if(i == 10){ + map.put(i, [start, total]) + }else{ + map.put(i, [start, end]) + } + start = end; + } + for(def d=0;dm && d<=n){ + dist.level = 10 - o.key + 1 + break + } + } + } + } + def dataMap = [:] + for(def d : datas){ + dataMap.put(d.city+"##"+d.region+"##"+d.name, d.level) + } + return dataMap; + } + + def getDataMap(city){ + def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\小区\\融合\\租售比\\"+city+".xlsx",0,1) + def map = [:] + for(def d : datas){ + map.put(d['D'], d['E']) + } + return map + + } + def getDistrictLevel(city, region, name){ + def q = new BasicDBObject() + q.put("city", city) + q.put("region", region) + q.put("name", name) + return ZS_LEVEL_CONN.findOne(q) + } + } + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/Test.groovy index 91498750a..728e6510c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/Test.groovy @@ -2,6 +2,7 @@ package com.fangjia.script.gj.test import com.fangjia.core.RegionData import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.CommonUtils import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ExcelUtils import com.mongodb.BasicDBObject @@ -29,25 +30,33 @@ import java.text.SimpleDateFormat // } //}) -def onLineSqlInstance = Sql.newInstance("jdbc:mysql://mysql.db.fangjia.com:4306/fangjia_youfang?&characterEncoding=utf8", "sa", "G00jia", "com.mysql.jdbc.Driver"); +//def onLineSqlInstance = Sql.newInstance("jdbc:mysql://mysql.db.fangjia.com:4306/fangjia_youfang?&characterEncoding=utf8", "sa", "G00jia", "com.mysql.jdbc.Driver"); +// +//def sql = "select visit_time,ip from visit_log where visit_time>='2020-09-18'"; +//def countMap = [:] +//onLineSqlInstance.eachRow(sql,{r-> +// def addTime = r.visit_time +// def month = DateUtils.formatDateByPattern(addTime, "yyyy-MM") +// def ips = countMap.get(month) +// if(!ips){ +// ips = new HashSet(); +// countMap.put(month, ips) +// } +// ips << r.ip +//}); +// +//countMap.each {k,v-> +// println k+" "+v.size() +//} -def sql = "select visit_time,ip from visit_log where visit_time>='2020-09-18'"; -def countMap = [:] -onLineSqlInstance.eachRow(sql,{r-> - def addTime = r.visit_time - def month = DateUtils.formatDateByPattern(addTime, "yyyy-MM") - def ips = countMap.get(month) - if(!ips){ - ips = new HashSet(); - countMap.put(month, ips) - } - ips << r.ip -}); -countMap.each {k,v-> - println k+" "+v.size() +def dists = CommonDatautils.findAllDistrictByCity("上海", "district", ["city","region","name","finish_building_date"]) +def rets = [] +for(def d : dists){ + def year = CommonUtils.getDistrictAge(d) + rets << [d.city,d.region, d.name, year] } - +ExcelUtils.write("D:\\数据\\统计房龄.xlsx", rets, "数据") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy index 2bffdeb62..d369ef53a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy @@ -37,12 +37,12 @@ class Esf58Crawl extends BaseBatch{ def static final distMap = [:] def static useDistKeys = [] static def totalCount = new AtomicLong() - //"福州":"fz|304","合肥":"hf|837","芜湖":"wuhu|2045","蚌埠":"bengbu|3470","阜阳":"fy|2325","淮南":"hn|2319","成都":"cd|102","上海":"sh|2", "北京":"bj|1","天津":"tj|18", - //"重庆":"cq|37","石家庄":"sjz|241","广州":"gz|3","深圳":"sz|4","东莞":"dg|413","佛山":"fs|222","杭州":"hz|79","武汉":"wh|158","青岛":"qd|122","南京":"nj|172", - //"嘉兴":"jx|497","大连":"dl|147","乌鲁木齐":"xj|984","太原":"ty|740","兰州":"lz|952","南昌":"nc|669", - def cityList = [ - - "沈阳":"sy|188","西安":"xa|483","苏州":"su|5","无锡":"wx|93","潍坊":"wf|362","常州":"cz|463", "南通":"nt|394","吉林":"jl|700","中山":"zs|771","台州":"tz|403","东营":"dy|623","三亚":"sanya|2422", + //"福州":"fz|304","合肥":"hf|837","芜湖":"wuhu|2045","蚌埠":"bengbu|3470","阜阳":"fy|2325","淮南":"hn|2319","成都":"cd|102", +// "上海":"sh|2", "北京":"bj|1","天津":"tj|18", +// "重庆":"cq|37","石家庄":"sjz|241","广州":"gz|3","深圳":"sz|4","东莞":"dg|413","佛山":"fs|222","杭州":"hz|79","武汉":"wh|158", + def cityList = ["青岛":"qd|122","南京":"nj|172", + "嘉兴":"jx|497","大连":"dl|147","乌鲁木齐":"xj|984","太原":"ty|740","兰州":"lz|952","南昌":"nc|669","沈阳":"sy|188","西安":"xa|483","苏州":"su|5","无锡":"wx|93","潍坊":"wf|362","常州":"cz|463", + "南通":"nt|394","吉林":"jl|700","中山":"zs|771","台州":"tz|403","东营":"dy|623","三亚":"sanya|2422", "呼和浩特":"hu|811","济南":"jn|265","温州":"wz|330","咸阳":"xianyang|7453","安庆":"anqing|3251","宿州":"suzhou|3359","六安":"la|2328","唐山":"ts|276","商丘":"sq|1029","大同":"dt|6964","威海":"weihai|518","安阳":"ay|1096", "临汾":"linfen|5669","临沂":"linyi|505","九江":"jj|2247","淄博":"zb|385","盐城":"yancheng|613","烟台":"yt|228","信阳":"xy|8694","南充":"nanchong|2378","洛阳":"luoyang|556","南阳":"ny|592","厦门":"xm|606", "宜昌":"yc|858","张家口":"zjk|3328","昆明":"km|541","柳州":"liuzhou|7133","绵阳":"mianyang|1057", "泉州":"qz|291","济宁":"jining|450","贵阳":"gy|2015","连云港":"lyg|2049","邯郸":"hd|572", diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy index d28d0da9d..a03f7ea2a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy @@ -14,8 +14,8 @@ class TestCrawlTotalCount { def run(){ def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"] for(def source :sources){ - def count1 = findTotalCount(source, 202110); - def count2 = findTotalCount(source, 202111); + def count1 = findTotalCount(source, 202111); + def count2 = findTotalCount(source, 202112); println source +" "+count1 +" "+count2 } }