Merge branch 'script_gj' into 'master'

update

See merge request !82
This commit is contained in:
2021-08-13 15:38:58 +08:00
5 changed files with 250 additions and 79 deletions
@@ -44,7 +44,7 @@ class BigdataCommunityBatch extends BaseBatch{
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
for(def city : citys){ for(def city : citys){
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","estate_type2", "lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat"]) def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat"])
println city +" 小区量:"+dists.size() println city +" 小区量:"+dists.size()
def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists) def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists)
rows.addAll(rets) rows.addAll(rets)
@@ -36,21 +36,36 @@ class EvaluateRateBatch extends BaseBatch{
def createDatas(def datas){ def createDatas(def datas){
def allRows = [] def allRows = []
datas.each { m -> datas.each { m ->
def data = [] if(m.rank_remark!="开发商装修"){
data << m.id.toString() def data = []
data << m.rate_type data << m.id.toString()
data << m.rate data << m.rate_type
data << m.rank data << m.rate
data << "全国" data << m.rank
data << m.rank_remark data << "全国"
data << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") data << converFitment(m.rank_remark)
data << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") data << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
data << "FJW" data << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
allRows << data data << "FJW"
allRows << data
}
} }
return allRows return allRows
} }
private def converFitment(rank_remark){
if(rank_remark){
if(rank_remark.contains("简单装修")){
return "简装"
}else if(rank_remark.contains("精装修")){
return "精装"
}else if(rank_remark.contains("豪华装修")){
return "豪装"
}
}
return rank_remark
}
private def findEvaluationRate(){ private def findEvaluationRate(){
def sql = "SELECT id, rate_type, rate, rank, rank_remark,date_created, last_updated FROM rate WHERE TYPE = '小区'"; def sql = "SELECT id, rate_type, rate, rank, rank_remark,date_created, last_updated FROM rate WHERE TYPE = '小区'";
def rets = [] def rets = []
@@ -3,15 +3,35 @@ package com.fangjia.script.ccbevaluation
import com.fangjia.core.RegionData import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.looplineinfo.Point
import com.fangjia.script.util.DbUtils import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import groovy.sql.Sql import groovy.sql.Sql
import org.bson.types.ObjectId
import java.util.regex.*;
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl", def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
//String str="\${ }";
//String regex="\\\$\\{.*\\}";
//Matcher m=Pattern.compile(regex).matcher(str);
//System.out.println(m.matches());
//def citys = CcbCommonUtils.citys; //def citys = CcbCommonUtils.citys;
//def rows = [] //def rows = []
// //
@@ -43,7 +63,7 @@ def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
def fileName = "d:\\数据\\建行\\房产估值\\采购数据交付标准_0804(3).xlsx" //def fileName = "d:\\数据\\建行\\房产估值\\采购数据交付标准_0804(3).xlsx"
//def datas = ExcelUtils.read("d:\\数据\\建行\\房产估值\\小区字段.xlsx",0,1) //def datas = ExcelUtils.read("d:\\数据\\建行\\房产估值\\小区字段.xlsx",0,1)
//def dealMap = [:] //def dealMap = [:]
//for(def d : datas){ //for(def d : datas){
@@ -59,48 +79,102 @@ def fileName = "d:\\
// dealMap.put(d['A'], map) // dealMap.put(d['A'], map)
//} //}
def cityCodeDatas = CcbCommonUtils.findCityCodeValue() //def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
ExcelUtils.multiThreadingModify(fileName,{d-> //ExcelUtils.multiThreadingModify(fileName,{d->
def city = d['B'] // def city = d['B']
if(city){ // if(city){
def cityid = cityCodeDatas.get(city) // def cityid = cityCodeDatas.get(city)
def total = findCount(cityid, sqlInstance) // def total = findCount(cityid, sqlInstance)
d['I'] = total //// d['I'] = total
// def a = findEstateType(cityid, sqlInstance) //// def a = findEstateType(cityid, sqlInstance)
////
//// d['AA'] = total
//// d['AB'] = DataHandleUtils.getDoubleRound(2, a*100/total)
////
//// def b = findFiledCount(cityid, 'buildyear', sqlInstance)
//// d['AC'] = DataHandleUtils.getDoubleRound(2, b*100/total)
////
//// def c = findFiledCount(cityid, 'address', sqlInstance)
//// d['AD'] = DataHandleUtils.getDoubleRound(2, c*100/total)
////
//// def d1 = findFiledCount(cityid, 'ZTS', sqlInstance)
//// d['AE'] = DataHandleUtils.getDoubleRound(2, d1*100/total)
////
//// def e = findFiledCount(cityid, 'landarea', sqlInstance)
//// d['AF'] = DataHandleUtils.getDoubleRound(2, e*100/total)
////
//// def f = findFiledCount(cityid, 'buildarea', sqlInstance)
//// d['AG'] = DataHandleUtils.getDoubleRound(2, f*100/total)
////
//// def g = findFiledCount(cityid, 'plotratio', sqlInstance)
//// d['AH'] = DataHandleUtils.getDoubleRound(2, g*100/total)
////
//// def h = findFiledCount(cityid, 'greeningrate', sqlInstance)
//// d['AI'] = DataHandleUtils.getDoubleRound(2, h*100/total)
////
//// def i = findFiledCount(cityid, 'latitude', sqlInstance)
//// d['AJ'] = DataHandleUtils.getDoubleRound(2, i*100/total)
////
//// def j = findFiledCount(cityid, 'parkingcnt', sqlInstance)
//// d['AK'] = DataHandleUtils.getDoubleRound(2, j*100/total)
// //
// d['AA'] = total // println d
// d['AB'] = DataHandleUtils.getDoubleRound(2, a*100/total) // }
// //},0)
// def b = findFiledCount(cityid, 'buildyear', sqlInstance) //def fileName = "d:\\数据\\建行\\房产估值\\采购数据交付标准_0807-1.xlsx"
// d['AC'] = DataHandleUtils.getDoubleRound(2, b*100/total) //def dealMap = findDealCount(sqlInstance)
// //ExcelUtils.multiThreadingModify(fileName,{d->
// def c = findFiledCount(cityid, 'address', sqlInstance) // if(d['C']){
// d['AD'] = DataHandleUtils.getDoubleRound(2, c*100/total) // d['Q'] = dealMap.get(d['C'])
// // }
// def d1 = findFiledCount(cityid, 'ZTS', sqlInstance) //},2)
// d['AE'] = DataHandleUtils.getDoubleRound(2, d1*100/total)
//
// def e = findFiledCount(cityid, 'landarea', sqlInstance) //def fileName = "d:\\数据\\建行\\房产估值\\经纬度错误数据修正.xlsx"
// d['AF'] = DataHandleUtils.getDoubleRound(2, e*100/total) //def updateIndex = 0
// //def deleteIndex = 0
// def f = findFiledCount(cityid, 'buildarea', sqlInstance) //ExcelUtils.modify(fileName,{d->
// d['AG'] = DataHandleUtils.getDoubleRound(2, f*100/total) // if(d['R'] && d['R'].toString().toDouble().toInteger()==1){
// // def dist = CommonDatautils.findDistrictById(d['D'])
// def g = findFiledCount(cityid, 'plotratio', sqlInstance) // def city = dist.city
// d['AH'] = DataHandleUtils.getDoubleRound(2, g*100/total) // def region = RegionData.formatRegion(city, d['Q'])
// // if(region && dist.region!=region){
// def h = findFiledCount(cityid, 'greeningrate', sqlInstance) // println city +" new:"+region +" old:"+d['C']
// d['AI'] = DataHandleUtils.getDoubleRound(2, h*100/total) // def udata = new BasicDBObject()
// // def obj = CommonDatautils.findDistrict(dist.city, region, dist.name, "district")
// def i = findFiledCount(cityid, 'latitude', sqlInstance) // if(obj){
// d['AJ'] = DataHandleUtils.getDoubleRound(2, i*100/total) // deleteIndex++
// // udata.put("status", 1)
// def j = findFiledCount(cityid, 'parkingcnt', sqlInstance) // udata.put("m_date", new Date())
// d['AK'] = DataHandleUtils.getDoubleRound(2, j*100/total) // DbUtils.seaweed.update(new BasicDBObject('_id',dist._id),new BasicDBObject('$set', udata))
// println deleteIndex +" "+dist.city +" "+ dist.name+" "+dist.region+"("+region+")"
// }else{
// updateIndex++
// udata.put("region", region)
// udata.put("m_date", new Date())
// DbUtils.seaweed.update(new BasicDBObject('_id',dist._id),new BasicDBObject('$set', udata))
// println updateIndex +" "+dist.city +" "+ dist.name+" "+dist.region+"("+region+")"
// }
// }else{
// println d['D']+" 不存在"
// }
// }
//},0)
def findDealCount(sqlInstance){
def sql = "select cityname,count(*) as count from bigdata_deal where dealdate>=to_date('2021-06-01','yyyy-mm-dd') and dealdate<to_date('2021-07-01','yyyy-mm-dd') group by cityname";
def count = 0
def map = [:]
sqlInstance.eachRow(sql,{r->
map.put(r.cityname, r.count)
});
return map
}
println d
}
},0)
def findEstateType(cityid, sqlInstance){ def findEstateType(cityid, sqlInstance){
def sql = "select count(*) as count from bigdata_commbuildtype where cityid='${cityid}' and buildtype!='0199'"; def sql = "select count(*) as count from bigdata_commbuildtype where cityid='${cityid}' and buildtype!='0199'";
@@ -132,22 +206,95 @@ def findFiledCount(cityid, field, sqlInstance){
} }
def fileName = "d:\\数据\\建行\\房产估值\\经纬度错误数据修正.xlsx"
def updateIndex = 0
def delIndex = 0
def cityMap = [:]
ExcelUtils.multiThreadingModify(fileName,{d->
if(!d['R']){
def obj = CommonDatautils.findDistrictById(d['D'])
if(obj){
def ranges = cityMap.get(obj.city)
if(!ranges){
def dbcursor = POLYLINECOLL.find(new BasicDBObject("city",obj.city))
ranges = []
for(def r : dbcursor){
ranges << r
}
cityMap.put(obj.city, ranges)
}
if(ranges){
def lng = obj.lng/10000000000.0
def lat = obj.lat/10000000000.0
def region = null
for(def r : ranges){
try{
def flag = isInArea(lng, lat, r.polyline)
if(flag){
def regionStr = r.region
region = RegionData.formatRegion(obj.city, regionStr)
break
}
}catch(Exception e){
println e.getMessage()
}
}
if(region && region!=obj.region){
updateIndex++
DbUtils.seaweed.update(new BasicDBObject("_id",obj.get("_id")),new BasicDBObject('$set', new BasicDBObject("region", region)))
println updateIndex +" [update]old:${obj.region} new:"+region+" "+obj.name
}else if(region && region==obj.region){
println "[same] "+obj.city +" "+obj.name
}else{
DbUtils.seaweed.update(new BasicDBObject("_id",new ObjectId(d['D'])),new BasicDBObject('$set', new BasicDBObject("status", 1)))
delIndex++
println delIndex +" [delete] "+obj.city +" "+obj.name
}
}else{
println obj.city +" 没有此城市边框信息"
}
}else{
println d['D']+" 没找到该小区"
}
}
},0);
//def city = "杭州"
//def city = "中山"
//def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lat", "lng"]) //def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lat", "lng"])
//def dbcursor = POLYLINECOLL.find(new BasicDBObject("city",city))
//def ranges = []
//for(def d : dbcursor){
// ranges << d
//}
//def index =0 //def index =0
//def updateIndex = 0 //def updateIndex = 0
//def delIndex = 0 //def delIndex = 0
//for(def dist : dists){ //for(def dist : dists){
// if(dist.region=="石岐区街道"){ //// if(dist.region=="江干"){
// def lat = dist.lat/10000000000.0 // def lat = dist.lat/10000000000.0
// def lng = dist.lng/10000000000.0 // def lng = dist.lng/10000000000.0
// def regionStr = getStreetInfo(city, lng.toString()+","+lat.toString()) //
// if(regionStr){ // def regionStr =null
// def region = RegionData.formatRegion(dist.city, regionStr) // def region = null
// for(def r : ranges){
// def flag = isInArea(lng, lat, r.polyline)
// if(flag){
// regionStr = r.region
// region = RegionData.formatRegion(dist.city, regionStr)
// break
// }
// }
// // def regionStr = getStreetInfo(city, lng.toString()+","+lat.toString())
// if(regionStr && region!=dist.region){
//
// def obj = CommonDatautils.findDistrict(dist.city, region, dist.name, "district") // def obj = CommonDatautils.findDistrict(dist.city, region, dist.name, "district")
// if(obj){ // if(obj){
// delIndex++ // delIndex++
@@ -156,30 +303,39 @@ def findFiledCount(cityid, field, sqlInstance){
// }else{ // }else{
// updateIndex++ // updateIndex++
// DbUtils.seaweed.update(new BasicDBObject("_id",dist.get("_id")),new BasicDBObject('$set', new BasicDBObject("region", region))) // DbUtils.seaweed.update(new BasicDBObject("_id",dist.get("_id")),new BasicDBObject('$set', new BasicDBObject("region", region)))
// println updateIndex +" [update]"+region+" "+dist.name // println updateIndex +" [update]old:${dist.region} new:"+region+" "+dist.name
// } // }
// }else{ // }else{
// index++ // index++
// println index+" [没找到]"+dist.name // println index+" [没找到]"+dist.name
// } // }
// } //// }
//}
//
//def getStreetInfo(def city, def glocation){
// def township = null
// if(glocation){
// def jsonObject = GaoDeApiUtils.geocode(glocation)
// if(jsonObject && jsonObject.regeocode) {
// def addressComponent = jsonObject.regeocode.addressComponent
// if(addressComponent && addressComponent.city.contains(city)){
// township = addressComponent.township
// }
// }
// }
// return township;
//} //}
def isInArea(lng, lat, points){
List<Point> pointList = InsidePolygon.createPoint2(points);
//Point point = new Point(lng, lat);
//def flag = InsidePolygon.aboutZone(point, pointList)
def flag = InsidePolygon.isPtInPoly(lng, lat, pointList)
return flag;
}
def getStreetInfo(def city, def glocation){
def township = null
if(glocation){
def jsonObject = GaoDeApiUtils.geocode(glocation)
if(jsonObject && jsonObject.regeocode) {
def addressComponent = jsonObject.regeocode.addressComponent
if(addressComponent && addressComponent.city.contains(city)){
township = addressComponent.township
}
}
}
return township;
}
@@ -186,8 +186,8 @@ class ThirdPartyPriceBatch extends BaseBatch {
if(!dataMap.containsKey("anjuke_esf_price") && db.anjuke_esf_price){ if(!dataMap.containsKey("anjuke_esf_price") && db.anjuke_esf_price){
dataMap.put("anjuke_esf_price", db.anjuke_esf_price) dataMap.put("anjuke_esf_price", db.anjuke_esf_price)
} }
if(!dataMap.containsKey("anjuke_esf_price") && db.get("58tc_esf")){ if(!dataMap.containsKey("anjuke_esf_price") && db.get("58tc_esf_price")){
dataMap.put("anjuke_esf_price", db.get("58tc_esf")) dataMap.put("anjuke_esf_price", db.get("58tc_esf_price"))
} }
if(!dataMap.containsKey("soufang_esf_price") && db.soufang_esf_price){ if(!dataMap.containsKey("soufang_esf_price") && db.soufang_esf_price){
dataMap.put("soufang_esf_price", db.soufang_esf_price) dataMap.put("soufang_esf_price", db.soufang_esf_price)
@@ -218,7 +218,7 @@ class CommonDatautils {
q.put("visible", 0); q.put("visible", 0);
q.put("cat", cat); q.put("cat", cat);
q.put("city", city); q.put("city", city);
// q.put("name","东寓新村") // q.put("name","金辉伯爵山")
DBObject keys = new BasicDBObject(); DBObject keys = new BasicDBObject();
if(fields){ if(fields){
@@ -1316,7 +1316,7 @@ class CommonDatautils {
} }
public static def getAllDictionary(){ public static def getAllDictionary(){
def dbCursor = DICT_COLL.find(new BasicDBObject()) def dbCursor = DICT_COLL.find(new BasicDBObject()).batchSize(5000)
def rets = [] def rets = []
def filters = [ "estate","hfword","main","metro","station"] def filters = [ "estate","hfword","main","metro","station"]
for(def obj : dbCursor){ for(def obj : dbCursor){