From 1ea267b81fcfec82199630cabd9e18a09ac52005 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 13 Aug 2021 16:29:01 +0800 Subject: [PATCH] update --- .../BigdataCommunityBatch.groovy | 2 +- .../ccbevaluation/EvaluateRateBatch.groovy | 37 ++- .../fangjia/script/ccbevaluation/Test.groovy | 282 ++++++++++++++---- .../ccbevaluation/ThirdPartyPriceBatch.groovy | 4 +- .../script/commData/CommonDatautils.groovy | 4 +- 5 files changed, 250 insertions(+), 79 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy index b56672f1d..560facf40 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -44,7 +44,7 @@ class BigdataCommunityBatch extends BaseBatch{ def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() for(def city : citys){ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) - def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","estate_type2", "lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat"]) + def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat"]) println city +" 小区量:"+dists.size() def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists) rows.addAll(rets) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/EvaluateRateBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/EvaluateRateBatch.groovy index 5a0c7a328..d64c08dd6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/EvaluateRateBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/EvaluateRateBatch.groovy @@ -36,21 +36,36 @@ class EvaluateRateBatch extends BaseBatch{ def createDatas(def datas){ def allRows = [] datas.each { m -> - def data = [] - data << m.id.toString() - data << m.rate_type - data << m.rate - data << m.rank - data << "全国" - data << m.rank_remark - data << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") - data << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") - data << "FJW" - allRows << data + if(m.rank_remark!="开发商装修"){ + def data = [] + data << m.id.toString() + data << m.rate_type + data << m.rate + data << m.rank + data << "全国" + data << converFitment(m.rank_remark) + data << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") + data << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") + data << "FJW" + allRows << data + } } return allRows } + private def converFitment(rank_remark){ + if(rank_remark){ + if(rank_remark.contains("简单装修")){ + return "简装" + }else if(rank_remark.contains("精装修")){ + return "精装" + }else if(rank_remark.contains("豪华装修")){ + return "豪装" + } + } + return rank_remark + } + private def findEvaluationRate(){ def sql = "SELECT id, rate_type, rate, rank, rank_remark,date_created, last_updated FROM rate WHERE TYPE = '小区'"; def rets = [] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index f1e86c6e9..99b243d36 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -3,15 +3,35 @@ package com.fangjia.script.ccbevaluation import com.fangjia.core.RegionData import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.looplineinfo.InsidePolygon +import com.fangjia.script.looplineinfo.Point import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils import com.mongodb.BasicDBObject +import com.mongodb.DBCollection import groovy.sql.Sql +import org.bson.types.ObjectId + +import java.util.regex.*; def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + +DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline") + +//String str="\${ }"; +//String regex="\\\$\\{.*\\}"; +//Matcher m=Pattern.compile(regex).matcher(str); +//System.out.println(m.matches()); + + + + + + //def citys = CcbCommonUtils.citys; //def rows = [] // @@ -43,7 +63,7 @@ def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl", -def fileName = "d:\\数据\\建行\\房产估值\\采购数据交付标准_0804(3).xlsx" +//def fileName = "d:\\数据\\建行\\房产估值\\采购数据交付标准_0804(3).xlsx" //def datas = ExcelUtils.read("d:\\数据\\建行\\房产估值\\小区字段.xlsx",0,1) //def dealMap = [:] //for(def d : datas){ @@ -59,48 +79,102 @@ def fileName = "d:\\ // dealMap.put(d['A'], map) //} -def cityCodeDatas = CcbCommonUtils.findCityCodeValue() -ExcelUtils.multiThreadingModify(fileName,{d-> - def city = d['B'] - if(city){ - def cityid = cityCodeDatas.get(city) - def total = findCount(cityid, sqlInstance) - d['I'] = total -// def a = findEstateType(cityid, sqlInstance) +//def cityCodeDatas = CcbCommonUtils.findCityCodeValue() +//ExcelUtils.multiThreadingModify(fileName,{d-> +// def city = d['B'] +// if(city){ +// def cityid = cityCodeDatas.get(city) +// def total = findCount(cityid, sqlInstance) +//// d['I'] = total +//// def a = findEstateType(cityid, sqlInstance) +//// +//// d['AA'] = total +//// d['AB'] = DataHandleUtils.getDoubleRound(2, a*100/total) +//// +//// def b = findFiledCount(cityid, 'buildyear', sqlInstance) +//// d['AC'] = DataHandleUtils.getDoubleRound(2, b*100/total) +//// +//// def c = findFiledCount(cityid, 'address', sqlInstance) +//// d['AD'] = DataHandleUtils.getDoubleRound(2, c*100/total) +//// +//// def d1 = findFiledCount(cityid, 'ZTS', sqlInstance) +//// d['AE'] = DataHandleUtils.getDoubleRound(2, d1*100/total) +//// +//// def e = findFiledCount(cityid, 'landarea', sqlInstance) +//// d['AF'] = DataHandleUtils.getDoubleRound(2, e*100/total) +//// +//// def f = findFiledCount(cityid, 'buildarea', sqlInstance) +//// d['AG'] = DataHandleUtils.getDoubleRound(2, f*100/total) +//// +//// def g = findFiledCount(cityid, 'plotratio', sqlInstance) +//// d['AH'] = DataHandleUtils.getDoubleRound(2, g*100/total) +//// +//// def h = findFiledCount(cityid, 'greeningrate', sqlInstance) +//// d['AI'] = DataHandleUtils.getDoubleRound(2, h*100/total) +//// +//// def i = findFiledCount(cityid, 'latitude', sqlInstance) +//// d['AJ'] = DataHandleUtils.getDoubleRound(2, i*100/total) +//// +//// def j = findFiledCount(cityid, 'parkingcnt', sqlInstance) +//// d['AK'] = DataHandleUtils.getDoubleRound(2, j*100/total) // -// d['AA'] = total -// d['AB'] = DataHandleUtils.getDoubleRound(2, a*100/total) -// -// def b = findFiledCount(cityid, 'buildyear', sqlInstance) -// d['AC'] = DataHandleUtils.getDoubleRound(2, b*100/total) -// -// def c = findFiledCount(cityid, 'address', sqlInstance) -// d['AD'] = DataHandleUtils.getDoubleRound(2, c*100/total) -// -// def d1 = findFiledCount(cityid, 'ZTS', sqlInstance) -// d['AE'] = DataHandleUtils.getDoubleRound(2, d1*100/total) -// -// def e = findFiledCount(cityid, 'landarea', sqlInstance) -// d['AF'] = DataHandleUtils.getDoubleRound(2, e*100/total) -// -// def f = findFiledCount(cityid, 'buildarea', sqlInstance) -// d['AG'] = DataHandleUtils.getDoubleRound(2, f*100/total) -// -// def g = findFiledCount(cityid, 'plotratio', sqlInstance) -// d['AH'] = DataHandleUtils.getDoubleRound(2, g*100/total) -// -// def h = findFiledCount(cityid, 'greeningrate', sqlInstance) -// d['AI'] = DataHandleUtils.getDoubleRound(2, h*100/total) -// -// def i = findFiledCount(cityid, 'latitude', sqlInstance) -// d['AJ'] = DataHandleUtils.getDoubleRound(2, i*100/total) -// -// def j = findFiledCount(cityid, 'parkingcnt', sqlInstance) -// d['AK'] = DataHandleUtils.getDoubleRound(2, j*100/total) +// println d +// } +//},0) +//def fileName = "d:\\数据\\建行\\房产估值\\采购数据交付标准_0807-1.xlsx" +//def dealMap = findDealCount(sqlInstance) +//ExcelUtils.multiThreadingModify(fileName,{d-> +// if(d['C']){ +// d['Q'] = dealMap.get(d['C']) +// } +//},2) + + +//def fileName = "d:\\数据\\建行\\房产估值\\经纬度错误数据修正.xlsx" +//def updateIndex = 0 +//def deleteIndex = 0 +//ExcelUtils.modify(fileName,{d-> +// if(d['R'] && d['R'].toString().toDouble().toInteger()==1){ +// def dist = CommonDatautils.findDistrictById(d['D']) +// def city = dist.city +// def region = RegionData.formatRegion(city, d['Q']) +// if(region && dist.region!=region){ +// println city +" new:"+region +" old:"+d['C'] +// def udata = new BasicDBObject() +// def obj = CommonDatautils.findDistrict(dist.city, region, dist.name, "district") +// if(obj){ +// deleteIndex++ +// udata.put("status", 1) +// udata.put("m_date", new Date()) +// DbUtils.seaweed.update(new BasicDBObject('_id',dist._id),new BasicDBObject('$set', udata)) +// println deleteIndex +" "+dist.city +" "+ dist.name+" "+dist.region+"("+region+")" +// }else{ +// updateIndex++ +// udata.put("region", region) +// udata.put("m_date", new Date()) +// DbUtils.seaweed.update(new BasicDBObject('_id',dist._id),new BasicDBObject('$set', udata)) +// println updateIndex +" "+dist.city +" "+ dist.name+" "+dist.region+"("+region+")" +// } +// }else{ +// println d['D']+" 不存在" +// } +// } +//},0) + + +def findDealCount(sqlInstance){ + def sql = "select cityname,count(*) as count from bigdata_deal where dealdate>=to_date('2021-06-01','yyyy-mm-dd') and dealdate + map.put(r.cityname, r.count) + }); + return map +} + + - println d - } -},0) def findEstateType(cityid, sqlInstance){ def sql = "select count(*) as count from bigdata_commbuildtype where cityid='${cityid}' and buildtype!='0199'"; @@ -132,22 +206,95 @@ def findFiledCount(cityid, field, sqlInstance){ } +def fileName = "d:\\数据\\建行\\房产估值\\经纬度错误数据修正.xlsx" +def updateIndex = 0 +def delIndex = 0 +def cityMap = [:] +ExcelUtils.multiThreadingModify(fileName,{d-> + if(!d['R']){ + + def obj = CommonDatautils.findDistrictById(d['D']) + if(obj){ + def ranges = cityMap.get(obj.city) + if(!ranges){ + def dbcursor = POLYLINECOLL.find(new BasicDBObject("city",obj.city)) + ranges = [] + for(def r : dbcursor){ + ranges << r + } + cityMap.put(obj.city, ranges) + } + + if(ranges){ + def lng = obj.lng/10000000000.0 + def lat = obj.lat/10000000000.0 + + def region = null + for(def r : ranges){ + try{ + def flag = isInArea(lng, lat, r.polyline) + if(flag){ + def regionStr = r.region + region = RegionData.formatRegion(obj.city, regionStr) + break + } + }catch(Exception e){ + println e.getMessage() + } + + } + if(region && region!=obj.region){ + updateIndex++ + DbUtils.seaweed.update(new BasicDBObject("_id",obj.get("_id")),new BasicDBObject('$set', new BasicDBObject("region", region))) + println updateIndex +" [update]old:${obj.region} new:"+region+" "+obj.name + }else if(region && region==obj.region){ + println "[same] "+obj.city +" "+obj.name + }else{ + DbUtils.seaweed.update(new BasicDBObject("_id",new ObjectId(d['D'])),new BasicDBObject('$set', new BasicDBObject("status", 1))) + delIndex++ + println delIndex +" [delete] "+obj.city +" "+obj.name + } + }else{ + println obj.city +" 没有此城市边框信息" + } + + }else{ + println d['D']+" 没找到该小区" + } + + } +},0); - -//def city = "中山" +//def city = "杭州" //def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lat", "lng"]) +//def dbcursor = POLYLINECOLL.find(new BasicDBObject("city",city)) +//def ranges = [] +//for(def d : dbcursor){ +// ranges << d +//} //def index =0 //def updateIndex = 0 //def delIndex = 0 //for(def dist : dists){ -// if(dist.region=="石岐区街道"){ +//// if(dist.region=="江干"){ // def lat = dist.lat/10000000000.0 // def lng = dist.lng/10000000000.0 -// def regionStr = getStreetInfo(city, lng.toString()+","+lat.toString()) -// if(regionStr){ -// def region = RegionData.formatRegion(dist.city, regionStr) +// +// def regionStr =null +// def region = null +// for(def r : ranges){ +// def flag = isInArea(lng, lat, r.polyline) +// if(flag){ +// regionStr = r.region +// region = RegionData.formatRegion(dist.city, regionStr) +// break +// } +// } +// // def regionStr = getStreetInfo(city, lng.toString()+","+lat.toString()) +// if(regionStr && region!=dist.region){ +// // def obj = CommonDatautils.findDistrict(dist.city, region, dist.name, "district") // if(obj){ // delIndex++ @@ -156,30 +303,39 @@ def findFiledCount(cityid, field, sqlInstance){ // }else{ // updateIndex++ // DbUtils.seaweed.update(new BasicDBObject("_id",dist.get("_id")),new BasicDBObject('$set', new BasicDBObject("region", region))) -// println updateIndex +" [update]"+region+" "+dist.name +// println updateIndex +" [update]old:${dist.region} new:"+region+" "+dist.name // } // }else{ // index++ // println index+" [没找到]"+dist.name // } -// } -//} -// -//def getStreetInfo(def city, def glocation){ -// def township = null -// if(glocation){ -// def jsonObject = GaoDeApiUtils.geocode(glocation) -// if(jsonObject && jsonObject.regeocode) { -// def addressComponent = jsonObject.regeocode.addressComponent -// if(addressComponent && addressComponent.city.contains(city)){ -// township = addressComponent.township -// } -// } -// } -// return township; +//// } //} +def isInArea(lng, lat, points){ + List pointList = InsidePolygon.createPoint2(points); + //Point point = new Point(lng, lat); + //def flag = InsidePolygon.aboutZone(point, pointList) + def flag = InsidePolygon.isPtInPoly(lng, lat, pointList) + return flag; +} + +def getStreetInfo(def city, def glocation){ + def township = null + if(glocation){ + def jsonObject = GaoDeApiUtils.geocode(glocation) + if(jsonObject && jsonObject.regeocode) { + def addressComponent = jsonObject.regeocode.addressComponent + if(addressComponent && addressComponent.city.contains(city)){ + township = addressComponent.township + } + } + } + return township; +} + + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy index 55fb15bd5..179daf27e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy @@ -186,8 +186,8 @@ class ThirdPartyPriceBatch extends BaseBatch { if(!dataMap.containsKey("anjuke_esf_price") && db.anjuke_esf_price){ dataMap.put("anjuke_esf_price", db.anjuke_esf_price) } - if(!dataMap.containsKey("anjuke_esf_price") && db.get("58tc_esf")){ - dataMap.put("anjuke_esf_price", db.get("58tc_esf")) + if(!dataMap.containsKey("anjuke_esf_price") && db.get("58tc_esf_price")){ + dataMap.put("anjuke_esf_price", db.get("58tc_esf_price")) } if(!dataMap.containsKey("soufang_esf_price") && db.soufang_esf_price){ dataMap.put("soufang_esf_price", db.soufang_esf_price) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 0338ac33b..ca7dada90 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -218,7 +218,7 @@ class CommonDatautils { q.put("visible", 0); q.put("cat", cat); q.put("city", city); -// q.put("name","东寓新村") +// q.put("name","金辉伯爵山") DBObject keys = new BasicDBObject(); if(fields){ @@ -1316,7 +1316,7 @@ class CommonDatautils { } public static def getAllDictionary(){ - def dbCursor = DICT_COLL.find(new BasicDBObject()) + def dbCursor = DICT_COLL.find(new BasicDBObject()).batchSize(5000) def rets = [] def filters = [ "estate","hfword","main","metro","station"] for(def obj : dbCursor){