From fbfd9481ed94437a9009059e2d0d502c30fc508c Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 5 Aug 2021 19:45:53 +0800 Subject: [PATCH] update --- .../BigdataCommbuildtypeBatch.groovy | 15 +- .../BigdataCommunityBatch.groovy | 29 +- .../ccbevaluation/BigdataDealBatch.groovy | 46 ++- .../ccbevaluation/BigdataDistrictBatch.groovy | 11 +- .../ccbevaluation/BigdataForsaleBatch.groovy | 42 +-- .../ccbevaluation/CcbCommonUtils.groovy | 71 ++++- .../script/ccbevaluation/CityBatch.groovy | 2 +- .../ccbevaluation/GuiDingPriceBatch.groovy | 37 ++- .../ccbevaluation/StagingDistrictBatch.groovy | 2 +- .../fangjia/script/ccbevaluation/Test.groovy | 95 +++++- .../ccbevaluation/ThirdPartyPriceBatch.groovy | 254 +++++++++------ .../script/ccbevaluation/TradeStatLog.groovy | 290 ++++++++++++++++++ .../script/gj/AddThirdPriceDistrict.groovy | 53 +++- .../com/fangjia/script/gj/RegionPoint.groovy | 2 +- .../script/gj/district/DataProcess.groovy | 4 +- .../gj/district/MapDistrictProcess.groovy | 2 +- .../script/gj/district/NameProcess.groovy | 4 +- .../gj/price/NoPriceCalcutePrice.groovy | 44 +++ .../gj/test/CrawlDistrictIdSelect.groovy | 2 +- .../TestCrawlTotalCount.groovy | 4 +- .../yinjihaun/amap/GaoDeApiUtils.groovy | 1 + 21 files changed, 806 insertions(+), 204 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/price/NoPriceCalcutePrice.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy index 93864962e..2fd71fe46 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy @@ -31,11 +31,10 @@ class BigdataCommbuildtypeBatch extends BaseBatch{ def citys = CcbCommonUtils.citys; def rets = [] for(def city : citys){ - def cityCodeDatas = CcbCommonUtils.findCityCodeValue(city) - def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) - - def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "build_type", "c_date", "m_date"]) - def rows = createDatas(cityCodeDatas, distCodeMap, dists) + def cityCodeDatas = CcbCommonUtils.findCityCodeValue() + // def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) + def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2","build_type", "c_date", "m_date"]) + def rows = createDatas(cityCodeDatas, dists) rets.addAll(rows) println city +" "+rows.size() } @@ -45,7 +44,7 @@ class BigdataCommbuildtypeBatch extends BaseBatch{ } - def createDatas(cityCodeDatas, distCodeMap, dists){ + def createDatas(cityCodeDatas, dists){ def rets = Collections.synchronizedList(new ArrayList()); def totalCount = new AtomicLong() final CountDownLatch latch = new CountDownLatch(dists.size()) @@ -55,7 +54,7 @@ class BigdataCommbuildtypeBatch extends BaseBatch{ void run() { try{ totalCount.addAndGet(1) - def ret = createData(cityCodeDatas, distCodeMap, dist) + def ret = createData(cityCodeDatas, dist) if(ret){ rets << ret } @@ -73,7 +72,7 @@ class BigdataCommbuildtypeBatch extends BaseBatch{ return rets; } - def createData(cityCodeDatas, distCodeMap, dist){ + def createData(cityCodeDatas, dist){ def ret = [] ret << cityCodeDatas.city_code ret << dist._id.toString() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy index 7904da09f..a260ea191 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -26,7 +26,7 @@ import java.util.regex.Pattern @BatchOwner(Watcher.GaoJuan) class BigdataCommunityBatch extends BaseBatch{ - public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40) + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80) static main(args) { runBatch(new BigdataCommunityBatch(), args); @@ -44,12 +44,13 @@ class BigdataCommunityBatch extends BaseBatch{ def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() for(def city : citys){ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) - def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "address", "lat2", "lng2","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date"]) + def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","estate_type2", "lat2", "lng2","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat"]) + println city +" 小区量:"+dists.size() def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists) rows.addAll(rets) println city +" "+rets.size() - // FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community_${city}.txt",headers, "utf-8", false); +// FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community_${city}.txt",headers, "utf-8", false); } def allDatas = [] @@ -90,15 +91,26 @@ class BigdataCommunityBatch extends BaseBatch{ def createData(def cityCodeDataMap, def distCodeMap, def dist){ def ret = [] try{ + def data = distCodeMap.get(dist.region) + if(!data){ + println dist.city +" "+dist.region +" "+dist.name+" 没有找到对应的区域" + return ret; + } + ret << cityCodeDataMap.get(dist.city) - if(dist.region){ - ret << distCodeMap.get(dist.region) + if(data){ + ret << data.district_code }else{ ret << "" } ret << dist._id.toString() - ret << dist.region + if(data){ + ret << data.district + }else{ + ret << "" + } + ret << dist.name ret << dist.address if(dist.lat2){ @@ -141,7 +153,10 @@ class BigdataCommunityBatch extends BaseBatch{ ret << "" } }else{ - def jsonObj = OpenApiClient.addressMatch(dist.city, dist.address, "property", false, dist.region, dist.name) + def jsonObj = null + if(dist.address){ + jsonObj = OpenApiClient.addressMatch(dist.city, dist.address, "property", false, dist.region, dist.name) + } if(jsonObj && jsonObj.searchAddress){ ret << jsonObj.searchAddress.street ret << jsonObj.searchAddress.streetNumber diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 641c643b6..27b702dcc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -59,10 +59,7 @@ class BigdataDealBatch extends BaseBatch{ rows.addAll(youdaDatas) println city +" "+youdaDatas.size() } - - // FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_deal_${city}_1.txt",headers, "utf-8", false); - def cityDatas = [] cityDatas << headers cityDatas.addAll(rows) @@ -95,11 +92,17 @@ class BigdataDealBatch extends BaseBatch{ try { totalCount.addAndGet(1) if(source == "youda"){ - result << converYouDaData(trade, cityCodeDatas, distCodeMap) - println totalCount.get() +" "+trade.city+" "+trade.region+" "+trade.fjName+" "+new Date().toLocaleString() + def ret = converYouDaData(trade, cityCodeDatas, distCodeMap) + if(ret){ + result << ret + println totalCount.get() +" "+trade.city+" "+trade.region+" "+trade.fjName+" "+new Date().toLocaleString() + } }else{ - result << converData(trade, cityCodeDatas, distCodeMap) - println totalCount.get() +" "+trade.city+" "+trade.region+" "+trade.district_name+" "+new Date().toLocaleString() + def ret = converData(trade, cityCodeDatas, distCodeMap) + if(ret){ + result << ret + println totalCount.get() +" "+trade.city+" "+trade.region+" "+trade.district_name+" "+new Date().toLocaleString() + } } @@ -118,20 +121,23 @@ class BigdataDealBatch extends BaseBatch{ def converYouDaData(def trade, def cityCodeDatas, def distCodeMap){ def ret = [] - def obj = CommonDatautils.findDistrict(trade.fjCity, trade.fjRegion, trade.fjName, "district") + def obj = CcbCommonUtils.findDistrict(trade.fjCity, trade.fjRegion, trade.fjName, "district") if(!obj){ return ret; + }else{ + println trade.city +" "+trade.fjRegion+" 没有找到对应的区域" } ret << cityCodeDatas.get(trade.city) ret << trade._id.toString() ret << trade.city - if(trade.fjRegion){ - ret << distCodeMap.get(trade.fjRegion) + def data = distCodeMap.get(trade.fjRegion) + if(data){ + ret << data.district_code + ret << data.district }else{ ret << "" + ret << "" } - ret << trade.region - def street = "" def streetNumber = "" if(trade.address){ @@ -256,15 +262,23 @@ class BigdataDealBatch extends BaseBatch{ def converData(def trade, def cityCodeDatas, def distCodeMap){ def ret = [] + def data = distCodeMap.get(trade.fjRegion) + if(!data){ + return ret + }else{ + println trade.city +" "+trade.fjRegion+" 没有找到对应的区域" + } ret << cityCodeDatas.get(trade.city) ret << trade._id.toString() ret << trade.city - if(trade.region){ - ret << distCodeMap.get(trade.region) + if(data){ + ret << data.district_code + ret << data.district }else{ ret << "" + ret << "" } - ret << trade.region + if(trade.address){ if(trade.address.toString().endsWith("弄")){ String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄")) @@ -376,7 +390,7 @@ class BigdataDealBatch extends BaseBatch{ def getCityData(def city){ - def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias","address","estate_type2","build_type","build_structure", "house_property","property_life"]) + def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias","address","estate_type2","build_type","build_structure", "house_property","property_life"]) def tradesList = Collections.synchronizedList(new ArrayList()); def totalCount = new AtomicLong() final CountDownLatch latch = new CountDownLatch(dists.size()) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDistrictBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDistrictBatch.groovy index 71d367afe..815336ae3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDistrictBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDistrictBatch.groovy @@ -40,8 +40,8 @@ class BigdataDistrictBatch extends BaseBatch{ def rets = [] rets << headers rets.addAll(allDatas) - ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\bigdata_district.xlsx", rets, "数据") - FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\bigdata_district.dat",headers, "utf-8",false); + ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_district.xlsx", rets, "数据") + FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_district.txt",headers, "utf-8",false); } def createDatas(rows){ @@ -73,17 +73,14 @@ class BigdataDistrictBatch extends BaseBatch{ def findDistrictCodeValue(city){ def q = new BasicDBObject() q.put("fangjia_city", city) - q.put("district", new BasicDBObject('$ne',"市辖区")) + // q.put("district", new BasicDBObject('$ne',"市辖区")) def dbcursor = CcbCommonUtils.FANGJIA_OFFICIAL_AREA_MAP.find(q) def lists = [] for(def db : dbcursor){ - if(db.fangjia_district && db.fangjia_district=="二道"){ - println db.fangjia_district+" "+city - } if(db.district_code){ def m = [:] m.id = db._id.toString() - m.dist_name = db.fangjia_district + m.dist_name = db.district m.dist_spell = com.fangjia.core.util.DataUtil.hanyuQuanpin(db.fangjia_district, "") m.city_id = db.city_code m.dist_code = db.district_code diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index 8c3df2f89..52e7de415 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -69,15 +69,14 @@ class BigdataForsaleBatch extends BaseBatch{ def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() for(def city : citys){ index++ - def distIds = getDistrictIds(cityCodeDataMap.get(city)) - +// def distIds = getDistrictIds(cityCodeDataMap.get(city)) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) - def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "estate_type2", "build_structure"]); - exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, distIds,20200701, 20210731); + def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "estate_type2", "build_structure"]); + exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20200701, 20210731); } } - def exportSellCSV(index,city, cityCodeDataMap, distCodeMap,districts, distIds,startDate, endDate){ + def exportSellCSV(index,city, cityCodeDataMap, distCodeMap, districts, startDate, endDate){ def totalCount = new AtomicLong() final CountDownLatch latch = new CountDownLatch(districts.size()) @@ -92,14 +91,16 @@ class BigdataForsaleBatch extends BaseBatch{ try { totalCount.addAndGet(1) def commId = rMap.get("_id").toString() - if(distIds.contains(commId)){ - def ret = findHouses(city, rMap.region, "sell", startDate,endDate, rMap.name, rMap.alias) - if(ret && ret.size()<100){ - ret = findHouses(city, rMap.region, "sell", 20190701, endDate, rMap.name, rMap.alias) - } - if(ret){ + + def ret = findHouses(city, rMap.region, "sell", startDate,endDate, rMap.name, rMap.alias) + if(ret && ret.size()<100){ + ret = findHouses(city, rMap.region, "sell", 20190701, endDate, rMap.name, rMap.alias) + } + if(ret){ + def areaData = distCodeMap.get(rMap.region) + if(areaData){ + def regionCode = areaData.district_code def cityCode = cityCodeDataMap.get(city) - def regionCode = distCodeMap.get(rMap.region) def buildYear = CcbCodeUtils.parseBuildYear(rMap.finish_building_date) def housePro = CcbCodeUtils.parseHouseProperty(rMap.house_property) def propertyYear = CcbCodeUtils.parseHousePropertyYear(rMap.property_life) @@ -107,7 +108,7 @@ class BigdataForsaleBatch extends BaseBatch{ def struCode = CcbCodeUtils.getBuildStructureCode(rMap.build_structure); for(def m : ret){ m.city_code = cityCode - m.region = rMap.region + m.region = areaData.district m.region_code = regionCode m.district_id = commId m.district_name = rMap.name @@ -124,9 +125,13 @@ class BigdataForsaleBatch extends BaseBatch{ m.source = "FJW" retMap.put(m.houseId, converData(m)) } + }else{ + println city +" "+rMap.region+" 没有找到对应的区域" } - println totalCount.get() +" "+city+" "+rMap.region+" "+rMap.name+" size:"+ret.size()+" "+new Date().toLocaleString() + } + println totalCount.get() +" "+city+" "+rMap.region+" "+rMap.name+" size:"+ret.size()+" "+new Date().toLocaleString() + } catch (Exception e) { e.printStackTrace() } finally { @@ -156,7 +161,6 @@ class BigdataForsaleBatch extends BaseBatch{ // rets << headers // rets.addAll(allDatas) // ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\bigdata_forsale_${city}.xlsx", rets, "数据") - } @@ -296,9 +300,9 @@ class BigdataForsaleBatch extends BaseBatch{ if(!seawater.height){ seawater.height = null } - if(seawater.height && seawater.height<1){ - continue; - } +// if(seawater.height && seawater.height<1){ +// continue; +// } if(seawater.height && seawater.height>60){ continue; } @@ -311,7 +315,7 @@ class BigdataForsaleBatch extends BaseBatch{ } if(seawater.room){ - if(seawater.room>=5 || seawater.room<1){ + if(seawater.room>=11 || seawater.room<1){ continue; } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index 5776037e7..1241ab34d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -1,18 +1,22 @@ package com.fangjia.script.ccbevaluation +import com.fangjia.datastore.driver.DSCollection +import com.fangjia.datastore.driver.DSCursor +import com.fangjia.datastore.driver.DataStore import com.fangjia.script.util.DateUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection +import com.mongodb.DBObject class CcbCommonUtils { - public static def citys = ["成都","沈阳","大连","北京", "上海","陵水","安庆","西宁","日照","长春","武汉","西安","苏州","广州","重庆","南京","青岛","天津","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","扬州","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭"] + public static def citys = ["北京", "上海","重庆","天津","成都","沈阳","大连","陵水","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","扬州","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭"] // public static def citys = ["武汉"] public static def provCityMap = ["河北":"石家庄" ,"山西":"太原" ,"陕西":"西安" ,"山东":"济南","河南":"郑州" ,"辽宁":"沈阳" ,"吉林":"长春" ,"黑龙江":"哈尔滨" ,"江苏":"南京" ,"浙江":"杭州" ,"安徽":"合肥","江西":"南昌" ,"福建":"福州" ,"湖北":"武汉","湖南":"长沙" ,"四川":"成都" ,"贵州":"贵阳" ,"云南":"昆明" ,"广东":"广州" ,"海南":"海口" ,"甘肃":"兰州" ,"青海":"西宁" ,"台湾":"台北","内蒙古":"呼和浩特" ,"新疆":"乌鲁木齐" ,"西藏":"拉萨","广西":"南宁","宁夏":"银川","香港":"香港","澳门":"澳门"] - public static final DBCollection FANGJIA_OFFICIAL_AREA_MAP = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "mongo.db.fangjia.com") + public static final DBCollection FANGJIA_OFFICIAL_AREA_MAP = MongoUtils.getCollectionByName("business", "fangjia_official_area_map_2020", "mongo.db.fangjia.com") public static final DBCollection SERVER_AREA_COLL = MongoUtils.getCollectionByName("fangjia", "server_area", "mongo.db.fangjia.com") @@ -23,6 +27,7 @@ class CcbCommonUtils { public static def rootMap = ["lianjia.com":"1","anjuke.com":"2","fang.com":"3","ke.com":"4","58.com":"5","leju.com":"6","fccs.com":"7","c21.com.cn":"8","qfang.com":"9","ganji.com":"10","leyoujia.com":"11","fangdd.com":"12","5i5j.com":"13","centanet.com":"14","zhuge.com":"15","baixing.com":"16","house365.com":"17","izhiliao.com":"18","wkzf.com":"19","jiwu.com":"20","xyfcw.com":"21","917.com":"22","0597house.com":"23","xmfish.com":"24","rongyizhaofang.com":"25","rz0375.com":"26","596fc.com":"27","zhen22.com":"28","taotaoju.vip":"29","kaifc.cn":"30","zhongmou.cc":"31","0391fc.com":"32","hualongxiang.com":"33","fangtan007.com":"34","h0591.com":"35","0598yu.com":"36","zfgou.cn":"37","0597ok.com":"38","second.cc":"39","rent.cc":"40","ssfcw.net":"41","czfcw.com":"42"] + public static DSCollection dataStoreConnection = null; public static def findCityCodeValue(){ def cityCodeMap = ["武汉":"420100","西安":"610100","苏州":"320500","大连":"210200","广州":"440100","重庆":"500000","南京":"320100","沈阳":"210100","青岛":"370200","天津":"120000","济南":"370100","宁波":"330200","长沙":"430100","石家庄":"130100","佛山":"440600","嘉兴":"330400","银川":"640100","南通":"320600","长春":"220100","廊坊":"131000","中山":"442000","郑州":"410100","哈尔滨":"230100","烟台":"370600","徐州":"320300","绍兴":"330600","厦门":"350200","昆明":"530100","南昌":"360100","贵阳":"520100","珠海":"440400","东莞":"441900","镇江":"321100","洛阳":"410300","温州":"330300","兰州":"620100","福州":"350100","惠州":"441300","呼和浩特":"150100","常州":"320400","唐山":"130200","淮安":"320800","湖州":"330500","九江":"360400","太原":"140100","北海":"450500","株洲":"430200","淄博":"370300","盐城":"320900","乌鲁木齐":"650100","海口":"460100","南宁":"450100","常德":"430700","台州":"331000","金华":"330700","吉林":"220200","岳阳":"430600","柳州":"450200","清远":"441800","湛江":"440800","张家口":"130700","保定":"130600","北京":"110000","成都":"510100","深圳":"440300","上海":"310000","杭州":"330100","合肥":"340100","扬州":"321000","无锡":"320200","泉州":"350500","遵义":"520300","肇庆":"441200","漳州":"350600","宜昌":"420500","芜湖":"340200","泰州":"321200","汕头":"440500","三亚":"460200","秦皇岛":"130300","马鞍山":"340500","连云港":"320700","沧州":"130900","安庆":"340800","西宁":"630100","日照":"371100","拉萨":"540100","锦州":"210700","大同":"140200","文昌":"469005","陵水":"469028","保亭":"469029"] @@ -32,11 +37,11 @@ class CcbCommonUtils { public static def findAllDistrictCodeValue(city){ def q = new BasicDBObject() q.put("fangjia_city", city) - q.put("district", new BasicDBObject('$ne',"市辖区")) + //q.put("district", new BasicDBObject('$ne',"市辖区")) def dbcursor = FANGJIA_OFFICIAL_AREA_MAP.find(q).sort(new BasicDBObject("_id", 1)) def map = [:] for(def db : dbcursor){ - map.put(db.fangjia_district, db.district_code) + map.put(db.fangjia_district, db) } return map; } @@ -135,4 +140,62 @@ class CcbCommonUtils { return null } + public static DSCollection getDataStoreConnection(){ + if(dataStoreConnection != null) { + return dataStoreConnection + } + DataStore ds = new DataStore("seaweed.db.fangjia.com", 27007) + dataStoreConnection = ds.getCollection("fangjia.seaweed") + return dataStoreConnection + } + + public static def findAllDistrictByCity(def city, def cat, def fields){ + + DBObject q = new BasicDBObject(); + q.put("visible",0) + q.put("cat", cat); + q.put("city", city); +// q.put("name_cat", 0); +// q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"])) + + DBObject keys = new BasicDBObject(); + if(fields){ + for(def field : fields){ + keys.put(field, 1); + } + } + def dbsour = getDataStoreConnection().find(q, keys).batchSize(2000) + def rets = [] + for(def d : dbsour){ + if(d.name_cat == 0 && ["普通住宅","商住"].contains(d.estate_type2)){ + rets << d + } + } + return rets; + } + + public static def findDistrict(String city, String region, String name, String cat){ + + DBObject q = new BasicDBObject(); + q.put("city", city); + if(region){ + q.put("region", region); + } + + q.put("name", name); + q.put("cat", cat) + q.put("status", 0) + q.put("name_cat", 0); + q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"])) + + + DSCursor cursor = getDataStoreConnection().find(q); + if(cursor){ + for(def obj : cursor){ + return obj; + } + } + return null; + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CityBatch.groovy index ef52b58c6..db72da8b5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CityBatch.groovy @@ -26,7 +26,7 @@ class CityBatch extends BaseBatch{ def rows = [] def index = 1 for(def city : citys){ - def data = CcbCommonUtils.findCityCodeValue(city) + def data = CcbCommonUtils.findCityCodeValue() def map = [:] map.name = city if(data){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index 234d172b5..49c60f702 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -31,7 +31,7 @@ class GuiDingPriceBatch extends BaseBatch{ def rows = [] for (def city : citys) { - def cityCodeDatas = CcbCommonUtils.findCityCodeValue(city) + def cityCodeDatas = CcbCommonUtils.findCityCodeValue() def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def prices = CcbCommonUtils.findPersonPrice(city, DateUtils.addMonth(new Date(), -6)) @@ -61,25 +61,24 @@ class GuiDingPriceBatch extends BaseBatch{ def dist = distMap.get(ps.region+"##"+ps.name) if(dist){ def ret = [] - ret << ps._id.toString() - ret << dist._id.toString() - ret << dist.name - ret << cityCodeDatas.city_code - ret << cityCodeDatas.fangjia_city - if(dist.region){ - ret << distCodeMap.get(dist.region) - }else{ + def areaData = distCodeMap.get(dist.region) + if(areaData){ + ret << ps._id.toString() + ret << dist._id.toString() + ret << dist.name + ret << cityCodeDatas.city_code + ret << cityCodeDatas.fangjia_city + ret << areaData.district_code + ret << areaData.district + ret << DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss") + ret << ps.guiding_value + ret << ps.guiding_person ret << "" + ret << DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss") + ret << DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss") + ret << "FJW" + rets << ret } - ret << dist.region - ret << DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss") - ret << ps.guiding_value - ret << ps.guiding_person - ret << "" - ret << DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss") - ret << DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss") - ret << "FJW" - rets << ret } } } @@ -87,7 +86,7 @@ class GuiDingPriceBatch extends BaseBatch{ } def getDistrictMap(def city){ - def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "estate_type2", "build_structure"]); + def districts = CcbCommonUtils.findAllDistricts(city, "district", ["city", "region", "name", "address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "estate_type2", "build_structure"]); def distMap = [:] for(def d : districts){ distMap.put(d.region+"##"+d.name, d) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy index 8084e319a..c73edb18e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy @@ -27,7 +27,7 @@ class StagingDistrictBatch extends BaseBatch{ def citys = CcbCommonUtils.citys; def rows = [] for(def city : citys){ - def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "master_district_name", "c_date", "m_date"]) + def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2","master_district_name", "c_date", "m_date"]) def fenQiMap = getFenQiMap(dists); def rets = createCityData(dists, fenQiMap) rows.addAll(rets) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index 399e3c53a..5f482e0a6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -1,21 +1,96 @@ package com.fangjia.script.ccbevaluation + +import com.fangjia.core.RegionData +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils +import com.mongodb.BasicDBObject import groovy.sql.Sql def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") -def citys = CcbCommonUtils.citys; -def rows = [] +//def citys = CcbCommonUtils.citys; +//def rows = [] +// +//for(def city : citys) { +// def data = CcbCommonUtils.findCityCodeValue() +// def sql = "SELECT count(*) as count FROM bigdata_forsale WHERE cityid = '${data.city_code}'"; +// +// def count = 0 +// sqlInstance.execute(sql,{ +// count = it.count +// }); +// println city +" "+count +//} -for(def city : citys) { - def data = CcbCommonUtils.findCityCodeValue(city) - def sql = "SELECT count(*) as count FROM bigdata_forsale WHERE cityid = '${data.city_code}'"; +//def fileName = "d:\\数据\\建行\\房产估值\\采购数据交付标准_0802(3).xlsx" +//def datas = ExcelUtils.read("d:\\数据\\建行\\房产估值\\成交统计.xlsx",0,1) +//def dealMap = [:] +//for(def d : datas){ +// dealMap.put(d['A'], d['B']) +//} +// +// +//ExcelUtils.multiThreadingModify(fileName,{d-> +// if(d['C']){ +// d['P'] = dealMap.get(d['C']) +// } +//},2) - def count = 0 - sqlInstance.execute(sql,{ - count = it.count - }); - println city +" "+count + +def city = "中山" +def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lat", "lng"]) +def index =0 +def updateIndex = 0 +def delIndex = 0 +for(def dist : dists){ + if(dist.region=="石岐区街道"){ + def lat = dist.lat/10000000000.0 + def lng = dist.lng/10000000000.0 + def regionStr = getStreetInfo(city, lng.toString()+","+lat.toString()) + if(regionStr){ + def region = RegionData.formatRegion(dist.city, regionStr) + def obj = CommonDatautils.findDistrict(dist.city, region, dist.name, "district") + if(obj){ + delIndex++ + DbUtils.seaweed.update(new BasicDBObject("_id",dist.get("_id")),new BasicDBObject('$set', new BasicDBObject("status", 1))) + println delIndex+" [del]"+region +" "+dist.name + }else{ + updateIndex++ + DbUtils.seaweed.update(new BasicDBObject("_id",dist.get("_id")),new BasicDBObject('$set', new BasicDBObject("region", region))) + println updateIndex +" [update]"+region+" "+dist.name + } + }else{ + index++ + println index+" [没找到]"+dist.name + } + } +} + +def getStreetInfo(def city, def glocation){ + def township = null + if(glocation){ + def jsonObject = GaoDeApiUtils.geocode(glocation) + if(jsonObject && jsonObject.regeocode) { + def addressComponent = jsonObject.regeocode.addressComponent + if(addressComponent && addressComponent.city.contains(city)){ + township = addressComponent.township + } + } + } + return township; } + + + + + + + + + + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy index 920ec307a..c4e81d699 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy @@ -30,15 +30,15 @@ class ThirdPartyPriceBatch extends BaseBatch { @Override public void process(Object args) { - def citys = ["安庆","西宁","日照","长春","上海","武汉","西安","苏州","大连","广州","重庆","南京","沈阳","青岛","天津","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","北京","成都","深圳","杭州","合肥","扬州","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","陵水","保亭"]//CcbCommonUtils.citys; + def citys = CcbCommonUtils.citys; def rows = [] def index = 0 for(def city : citys){ index++ - def cityCodeDatas = CcbCommonUtils.findCityCodeValue(city) + def cityCodeDatas = CcbCommonUtils.findCityCodeValue() def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) - def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name"]) + def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name","name_cat", "estate_type2"]) def thirdPriceMap = findThirdPartyPrice(city) def rets = createCityDatas(cityCodeDatas, distCodeMap, dists, thirdPriceMap) rows.addAll(rets) @@ -55,103 +55,102 @@ class ThirdPartyPriceBatch extends BaseBatch { for(def d : dists){ def third = thirdPriceMap.get(d.region+"##"+d.name) if(third){ - def ret = [] - ret << third._id.toString() - ret << d._id.toString() - ret << d.name - ret << cityCodeDatas.city_code - ret << cityCodeDatas.fangjia_city - if(d.region){ - ret << distCodeMap.get(d.region) - }else{ - ret << "" - } - ret << d.region - ret << third.s_date + def data = distCodeMap.get(d.region) + if(data){ + def ret = [] + ret << third._id.toString() + ret << d._id.toString() + ret << d.name + ret << cityCodeDatas.city_code + ret << cityCodeDatas.fangjia_city + ret << data.district_code + ret << data.district + ret << third.s_date - if(third.zhugezhaofang_esf_price){ - ret << third.zhugezhaofang_esf_price - }else{ - ret << "" - } - if(third.anjuke_esf_price){ - ret << third.anjuke_esf_price - }else{ - ret << "" - } - if(third.soufang_esf_price){ - ret << third.soufang_esf_price - }else{ - ret << "" - } + if(third.zhugezhaofang_esf_price){ + ret << third.zhugezhaofang_esf_price + }else{ + ret << "" + } + if(third.anjuke_esf_price){ + ret << third.anjuke_esf_price + }else{ + ret << "" + } + if(third.soufang_esf_price){ + ret << third.soufang_esf_price + }else{ + ret << "" + } - if(third.lianjia_esf_price){ - ret << third.lianjia_esf_price - }else{ - ret << "" - } + if(third.lianjia_esf_price){ + ret << third.lianjia_esf_price + }else{ + ret << "" + } - if(third.cityhouse_esf_price){ - ret << third.cityhouse_esf_price - }else{ - ret << "" - } + if(third.cityhouse_esf_price){ + ret << third.cityhouse_esf_price + }else{ + ret << "" + } - if(third.maitian_esf_price){ - ret << third.maitian_esf_price - }else{ - ret << "" - } + if(third.maitian_esf_price){ + ret << third.maitian_esf_price + }else{ + ret << "" + } - if(third.beike_esf_price){ - ret << third.beike_esf_price - }else{ - ret << "" - } + if(third.beike_esf_price){ + ret << third.beike_esf_price + }else{ + ret << "" + } - if(third.get("365house_esf_price")){ - ret << third.get("365house_esf_price") - }else{ - ret << "" - } + if(third.get("365house_esf_price")){ + ret << third.get("365house_esf_price") + }else{ + ret << "" + } - if(third.zhongyuan_esf_price){ - ret << third.zhongyuan_esf_price - }else{ - ret << "" - } + if(third.zhongyuan_esf_price){ + ret << third.zhongyuan_esf_price + }else{ + ret << "" + } - if(third.tuboshi_esf_price){ - ret << third.tuboshi_esf_price - }else{ - ret << "" - } - if(third.qfang_esf_price){ - ret << third.qfang_esf_price - }else{ - ret << "" - } + if(third.tuboshi_esf_price){ + ret << third.tuboshi_esf_price + }else{ + ret << "" + } + if(third.qfang_esf_price){ + ret << third.qfang_esf_price + }else{ + ret << "" + } - if(third.fangtan_esf_price){ - ret << third.fangtan_esf_price - }else{ - ret << "" - } + if(third.fangtan_esf_price){ + ret << third.fangtan_esf_price + }else{ + ret << "" + } - if(third.fangduoduo_esf_price){ - ret << third.fangduoduo_esf_price - }else{ - ret << "" + if(third.fangduoduo_esf_price){ + ret << third.fangduoduo_esf_price + }else{ + ret << "" + } + if(third.get("5i5j_esf_price")){ + ret << third.get("5i5j_esf_price") + }else{ + ret << "" + } + ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") + ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") + ret << "FJW" + rets << ret } - if(third.get("5i5j_esf_price")){ - ret << third.get("5i5j_esf_price") - }else{ - ret << "" - } - ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") - ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") - ret << "FJW" - rets << ret } } @@ -162,15 +161,82 @@ class ThirdPartyPriceBatch extends BaseBatch { def findThirdPartyPrice(def city){ def q = new BasicDBObject() q.put("city", city) - q.put("s_date", curDate) - def dbcursor = AVERAGE_STATISTICS.find(q) + q.put("s_date", new BasicDBObject('$gte', curDate)) + def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1)) def map = [:] for(def db : dbcursor){ - if(db.zhugezhaofang_esf_price || db.anjuke_esf_price ||db.soufang_esf_price || db.lianjia_esf_price || db.cityhouse_esf_price || db.maitian_esf_price || db.beike_esf_price || db.get("365house_esf_price") || db.zhongyuan_esf_price || db.tuboshi_esf_price || db.qfang_esf_price || db.fangtan_esf_price || db.fangduoduo_esf_price || db.get("5i5j_esf_price")){ - map.put(db.region+"##"+db.name, db) + def key = db.region+"##"+db.name + def dataMap = map.get(key) + if(!dataMap){ + dataMap = [:] + map.put(key, dataMap) + } + if(!dataMap.containsKey("zhugezhaofang_esf_price") && db.zhugezhaofang_esf_price){ + dataMap.put("zhugezhaofang_esf_price", db.zhugezhaofang_esf_price) + } + if(!dataMap.containsKey("anjuke_esf_price") && db.anjuke_esf_price){ + dataMap.put("anjuke_esf_price", db.anjuke_esf_price) + } + if(!dataMap.containsKey("anjuke_esf_price") && db.get("58tc_esf")){ + dataMap.put("anjuke_esf_price", db.get("58tc_esf")) + } + if(!dataMap.containsKey("soufang_esf_price") && db.soufang_esf_price){ + dataMap.put("soufang_esf_price", db.soufang_esf_price) + } + if(!dataMap.containsKey("lianjia_esf_price") && db.lianjia_esf_price){ + dataMap.put("lianjia_esf_price", db.lianjia_esf_price) + } + if(!dataMap.containsKey("cityhouse_esf_price") && db.cityhouse_esf_price){ + dataMap.put("cityhouse_esf_price", db.cityhouse_esf_price) + } + + if(!dataMap.containsKey("maitian_esf_price") && db.maitian_esf_price){ + dataMap.put("maitian_esf_price", db.maitian_esf_price) + } + + if(!dataMap.containsKey("beike_esf_price") && db.beike_esf_price){ + dataMap.put("beike_esf_price", db.beike_esf_price) + } + + if(!dataMap.get("365house_esf_price") && db.get("365house_esf_price")){ + dataMap.put("365house_esf_price", db.get("365house_esf_price")) + } + + if(!dataMap.containsKey("zhongyuan_esf_price") && db.zhongyuan_esf_price){ + dataMap.put("zhongyuan_esf_price", db.zhongyuan_esf_price) + } + + if(!dataMap.containsKey("tuboshi_esf_price") && db.tuboshi_esf_price){ + dataMap.put("tuboshi_esf_price", db.tuboshi_esf_price) + } + + if(!dataMap.containsKey("qfang_esf_price") && db.qfang_esf_price){ + dataMap.put("qfang_esf_price", db.qfang_esf_price) + } + + if(!dataMap.containsKey("fangtan_esf_price") && db.fangtan_esf_price){ + dataMap.put("fangtan_esf_price", db.fangtan_esf_price) + } + + if(!dataMap.containsKey("fangduoduo_esf_price") && db.fangduoduo_esf_price){ + dataMap.put("fangduoduo_esf_price", db.fangduoduo_esf_price) + } + + if(!dataMap.get("5i5j_esf_price") && db.get("5i5j_esf_price")){ + dataMap.put("5i5j_esf_price", db.get("5i5j_esf_price")) + } +// if(db.zhugezhaofang_esf_price || db.anjuke_esf_price ||db.soufang_esf_price || db.lianjia_esf_price || db.cityhouse_esf_price || db.maitian_esf_price || db.beike_esf_price || db.get("365house_esf_price") || db.zhongyuan_esf_price || db.tuboshi_esf_price || db.qfang_esf_price || db.fangtan_esf_price || db.fangduoduo_esf_price || db.get("5i5j_esf_price")){ +// map.put(key, db) +// } + } + + def resultMap = [:] + map.each {key, value-> + if(value){ + resultMap.put(key, value) } } - return map + return resultMap } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy new file mode 100644 index 000000000..467a3a9f8 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy @@ -0,0 +1,290 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import groovy.sql.Sql + +/*** + * 建行成交抓取统计 + */ +class TradeStatLog { + + public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com") + + public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017"; + public static final DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business", "res_esfdealprice", offlineUrl) + + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + + static main(args) { + def task = new TradeStatLog() + //task.run() + task.thirdPartyStat() + + } + + def run(){ + def citys = CcbCommonUtils.citys; + def rets = [] + rets << ["城市","5月总量","6月总量","贝壳","链家", "我爱我家","房天下","中原"] + def sources = ["贝壳找房","链家在线", "我爱我家","房天下","中原地产"] + def startDate = "2021-05-01" + def date = "2021-06-01" + def endDate = "2021-07-01" + for(def city : citys){ + def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd")) + def curMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd")) + def ret = [] + ret << city + + if(city == "上海"){ + def lastMonthYoudaCount = getYouDaData(DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd")) + def curMonthYoudaCount = getYouDaData(DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd")) + def a = lastMonthCount.toInteger() + lastMonthYoudaCount.toInteger() + def b = curMonthCount.toInteger() + curMonthYoudaCount.toInteger() + ret << a + ret << b + def c = (b-a)*100/a + def pre = DataHandleUtils.getDoubleRound(2, c) + ret << pre + }else{ + ret << lastMonthCount + ret << curMonthCount + + if(lastMonthCount){ + def a = (curMonthCount-lastMonthCount)*100/lastMonthCount + def pre = DataHandleUtils.getDoubleRound(2, a) + ret << pre + }else{ + ret << null + } + + } + + for(def source : sources){ + def sourceLastMonthCount = findTradesCount(city, source, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd")) + def sourceCurMonthCount = findTradesCount(city, source, DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd")) + ret << sourceLastMonthCount + ret << sourceCurMonthCount + } + rets << ret + println ret + } + + ExcelUtils.write("D:\\数据\\建行\\房产估值\\数据文件\\成交统计.xlsx", rets, "数据") + println "表格生成完成" + } + + + def thirdPartyStat(){ + def citys = CcbCommonUtils.citys; + def rets = [] + rets << ["城市","小区总量","有第三方的总量","1家占比","2家占比","3家占比","4家占比","5家占比","6家占比","7家占比","8家占比","9家占比","10家占比","11家占比","12家占比","13家占比","14家占比"] + for(def city : citys){ + def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2"]); + def m = getThirdPartyPricePrecent(city) + rets << [city, districts.size(), m.total, m.count1, m.count2, m.count3, m.count4, m.count5, m.count6, m.count7, m.count8, m.count9, m.count10, m.count11, m.count12, m.count13, m.count14] + } + ExcelUtils.write("D:\\数据\\建行\\房产估值\\数据文件\\第三方均价统计.xlsx", rets, "数据") + println "表格生成完成" + } + + + public static def findTradesCount(city, source, startDate, endDate){ + def q = new BasicDBObject() + q.put("city", city) + if(source){ + q.put("source", source) + } + q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate)) + def count = DEAL_PRICE_COLL.count(q) + return count + } + + + def getYouDaData(startDate, endDate){ + def q = new BasicDBObject() + q.put("contractTime", new BasicDBObject('$gte', startDate).append('$lt', endDate)); + def count = youda_esfhouse_coll.count(q) + return count; + } + + + def getThirdPartyPricePrecent(def cityid){ + def sql = "SELECT * FROM third_party_price WHERE cityid = '"+cityid+"'"; + + def count0 = 0 + def count1 = 0 + def count2 = 0 + def count3 = 0 + def count4 = 0 + def count5 = 0 + def count6 = 0 + def count7 = 0 + def count8 = 0 + def count9 = 0 + def count10 = 0 + def count11 = 0 + def count12 = 0 + def count13 = 0 + def count14 = 0 + def total = 0 + sqlInstance.eachRow(sql,{third-> + total++ + def ret = [] + if(third.zhugezhaofang_esf_price){ + ret << third.zhugezhaofang_esf_price + } + if(third.anjuke_esf_price){ + ret << third.anjuke_esf_price + } + if(third.soufang_esf_price){ + ret << third.soufang_esf_price + } + + if(third.lianjia_esf_price){ + ret << third.lianjia_esf_price + } + + if(third.cityhouse_esf_price){ + ret << third.cityhouse_esf_price + } + + if(third.maitian_esf_price){ + ret << third.maitian_esf_price + } + + if(third.beike_esf_price){ + ret << third.beike_esf_price + } + + if(third.get("365house_esf_price")){ + ret << third.get("365house_esf_price") + } + + if(third.zhongyuan_esf_price){ + ret << third.zhongyuan_esf_price + } + + if(third.tuboshi_esf_price){ + ret << third.tuboshi_esf_price + } + if(third.qfang_esf_price){ + ret << third.qfang_esf_price + } + + if(third.fangtan_esf_price){ + ret << third.fangtan_esf_price + } + + if(third.fangduoduo_esf_price){ + ret << third.fangduoduo_esf_price + } + if(third.get("5i5j_esf_price")){ + ret << third.get("5i5j_esf_price") + } + + if(ret.size()==0){ + count0++ + } + if(ret.size()==1){ + count1++ + } + if(ret.size()==2){ + count2++ + } + if(ret.size()==3){ + count3++ + } + if(ret.size()==4){ + count4++ + } + if(ret.size()==5){ + count5++ + } + + if(ret.size()==6){ + count6++ + } + if(ret.size()==7){ + count7++ + } + if(ret.size()==8){ + count8++ + } + if(ret.size()==9){ + count9++ + } + if(ret.size()==10){ + count10++ + } + if(ret.size()==11){ + count11++ + } + if(ret.size()==12){ + count12++ + } + if(ret.size()==13){ + count13++ + } + if(ret.size()==14){ + count14++ + } + }); + def pre1 = 0 + def pre2 = 0 + def pre3 = 0 + def pre4 = 0 + def pre5 = 0 + def pre6 = 0 + def pre7 = 0 + def pre8 = 0 + def pre9 = 0 + def pre10 = 0 + def pre11 = 0 + def pre12 = 0 + def pre13 = 0 + def pre14 = 0 + if(total){ + pre1 = DataHandleUtils.getDoubleRound(2, count1*100.0/total) + pre2 = DataHandleUtils.getDoubleRound(2, count2*100.0/total) + pre3 = DataHandleUtils.getDoubleRound(2, count3*100.0/total) + pre4 = DataHandleUtils.getDoubleRound(2, count4*100.0/total) + pre5 = DataHandleUtils.getDoubleRound(2, count5*100.0/total) + pre6 = DataHandleUtils.getDoubleRound(2, count6*100.0/total) + pre7 = DataHandleUtils.getDoubleRound(2, count7*100.0/total) + pre8 = DataHandleUtils.getDoubleRound(2, count8*100.0/total) + pre9 = DataHandleUtils.getDoubleRound(2, count9*100.0/total) + pre10 = DataHandleUtils.getDoubleRound(2, count10*100.0/total) + pre11 = DataHandleUtils.getDoubleRound(2, count11*100.0/total) + pre12 = DataHandleUtils.getDoubleRound(2, count12*100.0/total) + pre13 = DataHandleUtils.getDoubleRound(2, count13*100.0/total) + pre14 = DataHandleUtils.getDoubleRound(2, count14*100.0/total) + } + def map = [:] + map.total = total + map.count1 = pre1 + map.count2 = pre2 + map.count3 = pre3 + map.count4 = pre4 + map.count5 = pre5 + map.count6 = pre6 + map.count7 = pre7 + map.count8 = pre8 + map.count9 = pre9 + map.count10 = pre10 + map.count11 = pre11 + map.count12 = pre12 + map.count13 = pre13 + map.count14 = pre14 + return map + } + + + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddThirdPriceDistrict.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddThirdPriceDistrict.groovy index 25e05d428..579e0710e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddThirdPriceDistrict.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddThirdPriceDistrict.groovy @@ -1,13 +1,16 @@ package com.fangjia.script.gj import com.fangjia.core.RegionData +import com.fangjia.core.util.BaiDuApiUtils import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.district.FangjiaClientUtils import com.fangjia.script.gj.district.NameProcess +import com.fangjia.script.gj.district.SeaweedDataMatchProcess import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil import com.fangjia.script.util.CosineSimilarAlgorithm import com.fangjia.script.util.DataUtil import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.GeoHelper import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils import com.mongodb.BasicDBObject import org.bson.types.ObjectId @@ -89,7 +92,8 @@ class AddThirdPriceDistrict { // addInfo("D:\\数据\\小区\\南昌第三方均价.xlsx"); // addMapInfo("D:\\数据\\小区\\南昌第三方均价.xlsx") // addCrawlMapInfo("D:\\数据\\小区\\南昌第三方均价.xlsx") - dataFilter("D:\\数据\\小区\\南昌第三方均价.xlsx") + // dataFilter("D:\\数据\\小区\\南昌第三方均价.xlsx") +// matchDistDatas("D:\\数据\\小区\\南昌第三方均价.xlsx") addNewMapInfo("D:\\数据\\小区\\南昌第三方均价.xlsx") // addFlag("苏州","D:\\数据\\小区\\苏州第三方均价(1)(2).xlsx"); } @@ -104,21 +108,21 @@ class AddThirdPriceDistrict { }else{ if(name.toString().length()<3){ if(row['N'] && row['N']>=10 && !row['J']){ - row['I'] = 25 + row['T'] = 25 }else{ - row['I'] = 18 + row['T'] = 18 } }else{ def s1 = com.fangjia.core.util.DataUtil.match(name, Pattern.compile("(路|街|道)\\d+号\$")) if(s1){ - row['I'] = 19 + row['T'] = 19 } } } }else{ if(!name){ - row['I'] = 1 + row['T'] = 1 } } @@ -578,7 +582,7 @@ class AddThirdPriceDistrict { def index = 0; ExcelUtils.multiThreadingModify(filePath, { row -> - def flag = row['R'] + def flag = row['T'] def dgDatas = row['G'] // if(!flag){ index++ @@ -612,9 +616,9 @@ class AddThirdPriceDistrict { // def estateType = filterEstateTypeMap.get(row['A']+"##"+row['B']+"##"+row['C']) // row['W'] = estateType - if(!row['X']){ - def name = nameProcess(row['B'], row['C']) - row['X'] = EsSearchUtils.findHouse(row['A'], row['B'] , row['C']) + if(!row['Z']){ + //def name = nameProcess(row['B'], row['C']) + row['Z'] = EsSearchUtils.findHouse(row['A'], row['B'] , row['C']) } // @@ -1053,6 +1057,37 @@ class AddThirdPriceDistrict { return null } + def matchDistDatas(def path){ + def index = 0 + ExcelUtils.multiThreadingModify(path,{ List row -> + index ++ + if(!row['I'] && !row['J']){ + def rMap = SeaweedDataMatchProcess.doMatch(row['A'], row['B'], row['C'], row['D']); + if(rMap && (rMap.region==row['B'] || row['B'].contains(rMap.region))){ + //row['J'] = rMap.city + row['U'] = rMap.region + row['V'] = rMap.name + row['W'] = rMap.address + + if(rMap.lat){ +// def baidu = BaiDuApiUtils.geoconv(row['F'].toDouble(), row['G'].toDouble(), 3) +// def maplng = baidu.x +// def maplat = baidu.y +// row['X'] = GeoHelper.GetDistance(maplat, maplng, rMap.lat, rMap.lng) + def score = CosineSimilarAlgorithm.getSimilarity(row['C'], row['V']) + if(score<0.3){ + score = CosineSimilarAlgorithm.getSimilarity(row['D'], row['V']) + } + row['X'] = score + } + } + println index +" "+row + } + },0) + println "5.小区匹配完成" + } + + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/RegionPoint.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/RegionPoint.groovy index 62d5a00dc..d110c793f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/RegionPoint.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/RegionPoint.groovy @@ -29,7 +29,7 @@ class RegionPointCrawl { def task = new RegionPointCrawl(); //河北省、山西省、辽宁省、吉林省、黑龙江省、江苏省、浙江省、 // def provinces = ["安徽省","福建省","江西省","山东省","河南省","湖南省","广东省","海南省","四川省","贵州省","云南省","陕西省","甘肃省","青海省","台湾省","内蒙古自治区","广西壮族自治区","西藏自治区","宁夏回族自治区","新疆维吾尔自治区"] - def provinces = ["江西省"] + def provinces = ["吉林省"] for(def p : provinces){ task.crawlCityAndDistrictCode(p); task.run(p); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DataProcess.groovy index 87efe314a..51632fe04 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DataProcess.groovy @@ -24,7 +24,7 @@ class DataProcess { static main(args) { //建行信用卡城市(临汾--晋城).xlsx - def filePath = "D:\\数据\\小区\\地图\\东莞.xlsx" + def filePath = "D:\\数据\\小区\\地图\\张家口.xlsx" def task = new DataProcess(); // task.batchSaveDataToDB(filePath) task.mapBatchSaveDataToDB(filePath) @@ -121,7 +121,7 @@ class DataProcess { if(datas){ def keys = [] for(def data : datas){//20,21,25, - if(data['I'] && [26,27].contains(data['I'].toString().toDouble().toInteger())){ + if(data['I'] && [20,21,25,26,27].contains(data['I'].toString().toDouble().toInteger())){ def key = data['A']+"##"+data['B']+"##"+data['H'] if(!keys.contains(key) && data['H'] && data['H'].size()>2){ keys << key diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/MapDistrictProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/MapDistrictProcess.groovy index 56fa2adc2..ca4e29905 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/MapDistrictProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/MapDistrictProcess.groovy @@ -28,7 +28,7 @@ class MapDistrictProcess { def run(){ // def files = ["重庆.xlsx","东莞.xlsx","温州.xlsx","绍兴.xlsx","宁波.xlsx","南昌.xlsx","济南.xlsx","金华.xlsx","肇庆.xlsx","湛江.xlsx","岳阳.xlsx","盐城.xlsx","唐山.xlsx","三亚.xlsx","廊坊.xlsx","呼和浩特.xlsx","西宁.xlsx","日照.xlsx","常德.xlsx"] - def files = ["东莞.xlsx"] + def files = ["青岛.xlsx","沈阳.xlsx","哈尔滨.xlsx","张家口.xlsx","大连.xlsx"] for(def fileName : files){ def filePath = "D:\\数据\\小区\\地图\\"+fileName; typeFilter(filePath) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/NameProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/NameProcess.groovy index 92192ed4f..3b2c217fc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/NameProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/NameProcess.groovy @@ -9,7 +9,7 @@ class NameProcess { static def filterName(def name){ def keys = ["散盘","办公楼","写字楼","商务","家居建材","商贸城","服装城","科技园","删除","五金","购物广场","物流园","文化教育","材料城","充电站","产业园","商业中心","办公大楼","工业","机械城","孵化","体验中心","投诉站","接送点","文化站","创业","办公中心","自编","创新","科技","实业园","管理处","百货","展园","展厅","软件园","基地","学生","青年","女生","学生宿舍","单身","办公室","值班室","服务站","酒店宿舍","男生","地块","租房","临时","大门","农业园","装修","摆件","展示","发展中心","商业广场","运营中心","店铺","筹备组","车场","厂家","培训","号门","管理科","招商处","展点","住院楼","家禽","捐赠","群租","招租","软装","生活广场","直购","员工","理赔","互助","赠品","图片","男宿舍","招租","站点","卸货","购物","大学","消火栓","集散地","过渡房","多肉","等候","租","商铺","家庭式","地产","宾馆","共享","人才","创意","轰趴","会所","不动产","美丽","精装","专区","下客点","办公","设计","汽车","垃圾","圆床房","白领","厂房","水果","专家","的","人员","自如","影吧","物业","会展","文明楼","工房","示范","宣传","老年","妇女","沐浴","蔬果","敬老","棋牌","房产","公益","碧波庭","高层","服务","居民","英才","职工公寓","职工之家","风情园","中心","项目","门卫","客栈","底商","美团","空间","无人机","国际广场","智慧社区","食品","排","交口","爱心", "养老","集贸","保安","建筑工人","单位宿舍","工程","演员","二人转","建工","兵营","供应","土产","学员","职工","门口","欢迎","车棚","宝宝","陵园","登记处","植物","驿站","自助","书","女子","健康","营业","扎啤","创客","电子广场","村民安置房","办事处生活区","萌宠","啤酒","高档","回迁楼","品茶","公司宿舍","置业","明星","家庭","老区","新区","单元","居民点","救助站","公司宿舍","梦想","公司职工宿舍","公司职工住宅楼","单位房","平安","文明社区","书院","丝绸","旧区","建材城","栋宿舍","吉宅","女人城","疾控","医院","易初莲花","发展","魔方","轻纺城","合作","外贸","嬉空间","优+","医生","康乃馨","总部","网格站","私人定制","家装","换新家","仓库","防盗","客运站","蔬菜","旁","爱情","菜场","套","商场","公寓房","守望点","硕士","佛庵","组团","区宿舍","妇幼","英才","血站","女儿","环卫","贸易城","平房","公安","物流","管理","理想","商业城","留医部","劳动","医药","团校","聚贤","豪华","电梯","超豪","恋家","地铁","法院","高级公寓","温馨","安全文明","精英","教师村","旧村改造","私宅","楼群","老师楼","敬业","佳缘","保障性住房","自建房","群众","度假","政府住宅","乘务","科教","房改房","豪宅","移民","艺校","运管处","校区","教委楼","学区","教师公寓","记者站","温泉","旅店","防疫站","园区","扶贫","按摩","文化中心","居委会", "限价房","看房", - "合伙人","中学公寓","海景房","儿童","花艺","乐乐家","招收","商业","研究生","教师","旅游","精洗","物管区","律师","果蔬","文化室","休闲广场","集资房","精神家园","红星美凯龙","雅居","高中","球室","世华易居","住院综合楼"] + "合伙人","中学公寓","海景房","儿童","花艺","乐乐家","招收","商业","研究生","教师","旅游","精洗","物管区","律师","果蔬","文化室","休闲广场","集资房","精神家园","红星美凯龙","雅居","高中","球室","世华易居","住院综合楼","养生"] def isAdd = true; for(def key : keys){ if(name.contains(key)){ @@ -19,7 +19,7 @@ class NameProcess { } if(isAdd){ - def endKeys = ["路","街","店","酒店","汽车城","五金城","机电城","装饰城","材料城","农贸","商城","工业区","家居广场","家居生活中心","服务中心","创新中心","科创中心","财智中心","金融中心","创投中心","外包中心","创业大厦","度假","商写楼","塑化城","材料城","礼品城","汽车用品城","汽配城","市场","批发","文创园","交易中心","服装中心","科技中心","科创园","工业园","创业园","度假区","科创谷","旅游城","食品城","工业坊","百货","商贸广场","采购中心","购物中心","小学","教学楼","学区房","有限公司","电力站","模具城","装饰","数码广场","教师楼","茶城","建材家居","物流城","器材广场","贸易中心","商品城","建材广场","商贸街","绣品街","学生","高层","养生谷","度假项目","工程","度假村","度假区","早市","少儿","药铺","温馨","单间","公司住宅","公司宿舍","校宿舍","格格","茶叶","人行门"] + def endKeys = ["路","街","店","酒店","汽车城","五金城","机电城","装饰城","材料城","农贸","商城","工业区","家居广场","家居生活中心","服务中心","创新中心","科创中心","财智中心","金融中心","创投中心","外包中心","创业大厦","度假","商写楼","塑化城","材料城","礼品城","汽车用品城","汽配城","市场","批发","文创园","交易中心","服装中心","科技中心","科创园","工业园","创业园","度假区","科创谷","旅游城","食品城","工业坊","百货","商贸广场","采购中心","购物中心","小学","教学楼","学区房","有限公司","电力站","模具城","装饰","数码广场","教师楼","茶城","建材家居","物流城","器材广场","贸易中心","商品城","建材广场","商贸街","绣品街","学生","高层","养生谷","度假项目","工程","度假村","度假区","早市","少儿","药铺","温馨","单间","公司住宅","公司宿舍","校宿舍","格格","茶叶","人行门","学校","公司"] for(def key : endKeys){ if(name.toString().endsWith(key)){ isAdd = false; diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/price/NoPriceCalcutePrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/price/NoPriceCalcutePrice.groovy new file mode 100644 index 000000000..07bb6b070 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/price/NoPriceCalcutePrice.groovy @@ -0,0 +1,44 @@ +package com.fangjia.script.gj.price + +import com.fangjia.script.ccbevaluation.CcbCommonUtils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +class NoPriceCalcutePrice { + + DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com") + + static main(args) { + def task = new NoPriceCalcutePrice() + task.run() + } + def run(){ + def citys = CcbCommonUtils.citys; + for(def city : citys){ + def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name"]) + def map = getDistrictByCity(city, "2021-08-01") + for(def d : districts){ + if(!map.containsKey(d.region+"@"+d.name)){ + FangjiaClientUtils.sendDistrictPriceCalculate(d.city, d.region, d.name, 12); + println d.city+" "+d.region+" "+d.name + } + } + } + } + + def getDistrictByCity(city, weekend){ + def q = new BasicDBObject() + q.put("city", city) + q.put("weekend", DateUtils.parseFromTightByPattern(weekend, "yyyy-MM-dd")) + def dbcursor = DIST_SELL_STAT.find(q) + def map = [:] + for(def d : dbcursor){ + map.put(d.region+"@"+d.district_name, d.avg_price) + } + return map + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CrawlDistrictIdSelect.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CrawlDistrictIdSelect.groovy index d436aa081..50ffcf9dc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CrawlDistrictIdSelect.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CrawlDistrictIdSelect.groovy @@ -61,7 +61,7 @@ class CrawlDistrictIdSelect { def findCrawlData(){ def q = new BasicDBObject() // q.put("city","福州") - q.put("source", "安居客") +// q.put("source", "安居客") q.put("s_date",202012) def dbcursor = thirdPartyColl.find(q).batchSize(10000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT) def retMap = [:] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy index 24b482d1e..f6887e506 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy @@ -14,8 +14,8 @@ class TestCrawlTotalCount { def run(){ def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房"] for(def source :sources){ - def count1 = findTotalCount(source, 202105); - def count2 = findTotalCount(source, 202106); + def count1 = findTotalCount(source, 202106); + def count2 = findTotalCount(source, 202107); println source +" "+count1 +" "+count2 } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy index 781687012..2f146d11f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy @@ -26,6 +26,7 @@ class GaoDeApiUtils { Map params = new HashMap() params.put("key", GAO_DE_AK) params.put("location", location) + params.put("extensions","all") String content try { content = new HttpRequest().get("http://restapi.amap.com/v3/geocode/regeo", params, "utf-8")