From 7dba105573a6b63ba10b8db3ce2174c9a23daddc Mon Sep 17 00:00:00 2001 From: gaojuan Date: Sat, 7 Aug 2021 20:42:21 +0800 Subject: [PATCH] update --- .../BigdataCommunityBatch.groovy | 4 +- .../ccbevaluation/BigdataDealBatch.groovy | 10 +- .../fangjia/script/ccbevaluation/Test.groovy | 180 ++++++++++++++---- .../script/commData/CommonDatautils.groovy | 2 + .../script/gj/location/LocationTest.groovy | 51 +++-- 5 files changed, 185 insertions(+), 62 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy index dd345f2d3..b56672f1d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -148,11 +148,11 @@ class BigdataCommunityBatch extends BaseBatch{ ret << "pa100" ret << "" if(dist.address){ - if(dist.address.toString().endsWith("弄")){ + if(dist.address.toString().contains("弄")){ String number = DataUtil.match(dist.address, Pattern.compile("\\d+弄")) if(number){ ret << dist.address.toString().substring(0, dist.address.toString().indexOf(number)) - ret << number + ret << dist.address.toString().substring(dist.address.toString().indexOf(number), dist.address.toString().size()) }else{ ret << "" ret << "" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 3a252731e..af0044a94 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -393,14 +393,14 @@ class BigdataDealBatch extends BaseBatch{ def totalCount = new AtomicLong() final CountDownLatch latch = new CountDownLatch(dists.size()) - def citys = ["北京","广州","深圳","重庆","郑州","西安","成都","东莞","佛山","杭州","合肥","南京","长春","扬州","无锡","太原","台州","泉州","宁波","南通","南昌","贵阳","哈尔滨","惠州","济南","金华","昆明","遵义","镇江","肇庆","宜昌","芜湖","泰州","汕头","清远","秦皇岛","马鞍山","洛阳","连云港","九江","淮安","呼和浩特","沧州","保定","安庆"] +// def citys = ["北京","广州","深圳","重庆","郑州","西安","成都","东莞","佛山","杭州","合肥","南京","长春","扬州","无锡","太原","台州","泉州","宁波","南通","南昌","贵阳","哈尔滨","惠州","济南","金华","昆明","遵义","镇江","肇庆","宜昌","芜湖","泰州","汕头","清远","秦皇岛","马鞍山","洛阳","连云港","九江","淮安","呼和浩特","沧州","保定","安庆"] def startDate = null; - if(citys.contains(city)){ +// if(citys.contains(city)){ startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210501", "yyyyMMdd")) - }else{ - startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210601", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd")) - } +// }else{ +// startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210601", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd")) +// } def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20210701", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date()) for(def i=0;i // if(d['C']){ -// d['P'] = dealMap.get(d['C']) +// d['Q'] = dealMap.get(d['C']) // } //},2) -def city = "中山" -def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lat", "lng"]) -def index =0 -def updateIndex = 0 -def delIndex = 0 -for(def dist : dists){ - if(dist.region=="石岐区街道"){ - def lat = dist.lat/10000000000.0 - def lng = dist.lng/10000000000.0 - def regionStr = getStreetInfo(city, lng.toString()+","+lat.toString()) - if(regionStr){ - def region = RegionData.formatRegion(dist.city, regionStr) - def obj = CommonDatautils.findDistrict(dist.city, region, dist.name, "district") - if(obj){ - delIndex++ - DbUtils.seaweed.update(new BasicDBObject("_id",dist.get("_id")),new BasicDBObject('$set', new BasicDBObject("status", 1))) - println delIndex+" [del]"+region +" "+dist.name - }else{ - updateIndex++ - DbUtils.seaweed.update(new BasicDBObject("_id",dist.get("_id")),new BasicDBObject('$set', new BasicDBObject("region", region))) - println updateIndex +" [update]"+region+" "+dist.name - } - }else{ - index++ - println index+" [没找到]"+dist.name - } + + +def fileName = "d:\\数据\\建行\\房产估值\\采购数据交付标准_0804(3).xlsx" +//def datas = ExcelUtils.read("d:\\数据\\建行\\房产估值\\小区字段.xlsx",0,1) +//def dealMap = [:] +//for(def d : datas){ +// def map = [:] +// map.put("land_area",d['D']) +// map.put("build_area",d['F']) +// map.put("greeningrate_area",d['H']) +// map.put("parkingcnt_area",d['J']) +// map.put("zts_area",d['L']) +// map.put("buildyear_area",d['N']) +// map.put("plotratio",d['P']) +// +// dealMap.put(d['A'], map) +//} + +def cityCodeDatas = CcbCommonUtils.findCityCodeValue() +ExcelUtils.multiThreadingModify(fileName,{d-> + def city = d['B'] + if(city){ + def cityid = cityCodeDatas.get(city) + def total = findCount(cityid, sqlInstance) + d['I'] = total +// def a = findEstateType(cityid, sqlInstance) +// +// d['AA'] = total +// d['AB'] = DataHandleUtils.getDoubleRound(2, a*100/total) +// +// def b = findFiledCount(cityid, 'buildyear', sqlInstance) +// d['AC'] = DataHandleUtils.getDoubleRound(2, b*100/total) +// +// def c = findFiledCount(cityid, 'address', sqlInstance) +// d['AD'] = DataHandleUtils.getDoubleRound(2, c*100/total) +// +// def d1 = findFiledCount(cityid, 'ZTS', sqlInstance) +// d['AE'] = DataHandleUtils.getDoubleRound(2, d1*100/total) +// +// def e = findFiledCount(cityid, 'landarea', sqlInstance) +// d['AF'] = DataHandleUtils.getDoubleRound(2, e*100/total) +// +// def f = findFiledCount(cityid, 'buildarea', sqlInstance) +// d['AG'] = DataHandleUtils.getDoubleRound(2, f*100/total) +// +// def g = findFiledCount(cityid, 'plotratio', sqlInstance) +// d['AH'] = DataHandleUtils.getDoubleRound(2, g*100/total) +// +// def h = findFiledCount(cityid, 'greeningrate', sqlInstance) +// d['AI'] = DataHandleUtils.getDoubleRound(2, h*100/total) +// +// def i = findFiledCount(cityid, 'latitude', sqlInstance) +// d['AJ'] = DataHandleUtils.getDoubleRound(2, i*100/total) +// +// def j = findFiledCount(cityid, 'parkingcnt', sqlInstance) +// d['AK'] = DataHandleUtils.getDoubleRound(2, j*100/total) + + println d } +},0) + +def findEstateType(cityid, sqlInstance){ + def sql = "select count(*) as count from bigdata_commbuildtype where cityid='${cityid}' and buildtype!='0199'"; + def count = 0 + + sqlInstance.eachRow(sql,{r-> + count = r.count + }); + return count } -def getStreetInfo(def city, def glocation){ - def township = null - if(glocation){ - def jsonObject = GaoDeApiUtils.geocode(glocation) - if(jsonObject && jsonObject.regeocode) { - def addressComponent = jsonObject.regeocode.addressComponent - if(addressComponent && addressComponent.city.contains(city)){ - township = addressComponent.township - } - } - } - return township; +def findCount(cityid, sqlInstance){ + def sql = "select count(*) as count from bigdata_community where cityid='${cityid}'"; + def count = 0 + + sqlInstance.eachRow(sql,{r-> + count = r.count + }); + return count +} + +def findFiledCount(cityid, field, sqlInstance){ + def sql = "select count(*) as count from bigdata_community where cityid='${cityid}' and "+field+" is not null"; + def count = 0 + sqlInstance.eachRow(sql,{r-> + count = r.count + }); + return count } @@ -88,6 +136,54 @@ def getStreetInfo(def city, def glocation){ +//def city = "中山" +//def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lat", "lng"]) +//def index =0 +//def updateIndex = 0 +//def delIndex = 0 +//for(def dist : dists){ +// if(dist.region=="石岐区街道"){ +// def lat = dist.lat/10000000000.0 +// def lng = dist.lng/10000000000.0 +// def regionStr = getStreetInfo(city, lng.toString()+","+lat.toString()) +// if(regionStr){ +// def region = RegionData.formatRegion(dist.city, regionStr) +// def obj = CommonDatautils.findDistrict(dist.city, region, dist.name, "district") +// if(obj){ +// delIndex++ +// DbUtils.seaweed.update(new BasicDBObject("_id",dist.get("_id")),new BasicDBObject('$set', new BasicDBObject("status", 1))) +// println delIndex+" [del]"+region +" "+dist.name +// }else{ +// updateIndex++ +// DbUtils.seaweed.update(new BasicDBObject("_id",dist.get("_id")),new BasicDBObject('$set', new BasicDBObject("region", region))) +// println updateIndex +" [update]"+region+" "+dist.name +// } +// }else{ +// index++ +// println index+" [没找到]"+dist.name +// } +// } +//} +// +//def getStreetInfo(def city, def glocation){ +// def township = null +// if(glocation){ +// def jsonObject = GaoDeApiUtils.geocode(glocation) +// if(jsonObject && jsonObject.regeocode) { +// def addressComponent = jsonObject.regeocode.addressComponent +// if(addressComponent && addressComponent.city.contains(city)){ +// township = addressComponent.township +// } +// } +// } +// return township; +//} + + + + + + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 3200b95c6..0338ac33b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -780,6 +780,8 @@ class CommonDatautils { keys.put("address", 1); keys.put("build_structure", 1); keys.put("out_pics", 1); + keys.put("lat", 1); + keys.put("lng", 1); DSCursor cursor = getDataStoreConnection().find(q, keys); if(cursor){ for(def obj : cursor){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/LocationTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/LocationTest.groovy index 524120aba..ce1d3ee1a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/LocationTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/LocationTest.groovy @@ -29,7 +29,7 @@ import java.util.regex.Pattern; class LocationTest { static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline") - public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40) + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") @@ -40,9 +40,9 @@ class LocationTest { // System.out.println(str); def task = new LocationTest() - task.run() +// task.run() // task.run2() -// task.run3() + task.run3() // task.run4() // task.run5() } @@ -162,7 +162,9 @@ class LocationTest { def run3(){ - ExcelUtils.modify("d:\\数据\\福州\\区域经纬度错误-南昌.xlsx", {d-> + def index = 0 + ExcelUtils.modify("d:\\数据\\福州\\经纬度错误.xlsx", {d-> + index++ def distance = null if(d['H']){ distance = d['H'].toDouble() @@ -261,6 +263,7 @@ class LocationTest { } } } + println index },0) } @@ -311,7 +314,7 @@ class LocationTest { } } - def baiduRest = FangjiaClientUtils.searchMapData(dist.city, dist.name,"住宅", "百度") + def baiduRest = null;//FangjiaClientUtils.searchMapData(dist.city, dist.name,"住宅", "百度") if(baiduRest){ bdlocationMap = getMapLocation(dist, baiduRest); if(bdlocationMap){ @@ -464,7 +467,11 @@ class LocationTest { void run() { try{ totalCount.addAndGet(1) - mapLocation(map) + def ret = mapLocation(map) + if(ret){ + rets << ret + } + println totalCount.get() } catch (Exception e) { e.printStackTrace() } finally { @@ -474,14 +481,35 @@ class LocationTest { }); } latch.await() - return rets; + ExcelUtils.write("d:\\数据\\福州\\经纬度错误.xlsx", rets, "经纬度错误") } def mapLocation(map){ - def dist = CommonDatautils.findDistrictById(map.commid) + def dist = CommonDatautils.findDistrictById(map[1]) if(dist){ - + def loactionMap = matchMap(dist) + def ret = [] + ret << dist.city + ret << dist.region + ret << dist.name + if (loactionMap) { + ret << loactionMap.source + ret << loactionMap.region + ret << loactionMap.mapName + ret << loactionMap.address + def distance = GeoHelper.GetDistance(dist.lat / 10000000000.0, dist.lng / 10000000000.0, loactionMap.lat, loactionMap.lng) + ret << distance + ret << loactionMap.lng + ret << loactionMap.lat + if (dist.region == loactionMap.region) { + println dist.region + " " + dist.name + " 不在区域范围内,与地图相同" + } else { + println dist.region + " " + dist.name + " 不在区域范围内 地图:" + loactionMap.region + " " + distance + } + } + return ret } + return null } @@ -494,10 +522,7 @@ class LocationTest { sqlInstance.eachRow(sql,{r-> rets << [r.cityid, r.commid] }); - - def map = [:] - - return map + return rets } }