Merge branch 'script_gj' into 'master'

update

See merge request !82
This commit is contained in:
2021-08-13 15:38:58 +08:00
5 changed files with 250 additions and 79 deletions
@@ -44,7 +44,7 @@ class BigdataCommunityBatch extends BaseBatch{
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
for(def city : citys){
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","estate_type2", "lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat"])
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat"])
println city +" 小区量:"+dists.size()
def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists)
rows.addAll(rets)
@@ -36,21 +36,36 @@ class EvaluateRateBatch extends BaseBatch{
def createDatas(def datas){
def allRows = []
datas.each { m ->
def data = []
data << m.id.toString()
data << m.rate_type
data << m.rate
data << m.rank
data << "全国"
data << m.rank_remark
data << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
data << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
data << "FJW"
allRows << data
if(m.rank_remark!="开发商装修"){
def data = []
data << m.id.toString()
data << m.rate_type
data << m.rate
data << m.rank
data << "全国"
data << converFitment(m.rank_remark)
data << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
data << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
data << "FJW"
allRows << data
}
}
return allRows
}
private def converFitment(rank_remark){
if(rank_remark){
if(rank_remark.contains("简单装修")){
return "简装"
}else if(rank_remark.contains("精装修")){
return "精装"
}else if(rank_remark.contains("豪华装修")){
return "豪装"
}
}
return rank_remark
}
private def findEvaluationRate(){
def sql = "SELECT id, rate_type, rate, rank, rank_remark,date_created, last_updated FROM rate WHERE TYPE = '小区'";
def rets = []
@@ -3,15 +3,35 @@ package com.fangjia.script.ccbevaluation
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.looplineinfo.Point
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import groovy.sql.Sql
import org.bson.types.ObjectId
import java.util.regex.*;
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
//String str="\${ }";
//String regex="\\\$\\{.*\\}";
//Matcher m=Pattern.compile(regex).matcher(str);
//System.out.println(m.matches());
//def citys = CcbCommonUtils.citys;
//def rows = []
//
@@ -43,7 +63,7 @@ def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
def fileName = "d:\\数据\\建行\\房产估值\\采购数据交付标准_0804(3).xlsx"
//def fileName = "d:\\数据\\建行\\房产估值\\采购数据交付标准_0804(3).xlsx"
//def datas = ExcelUtils.read("d:\\数据\\建行\\房产估值\\小区字段.xlsx",0,1)
//def dealMap = [:]
//for(def d : datas){
@@ -59,48 +79,102 @@ def fileName = "d:\\
// dealMap.put(d['A'], map)
//}
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
ExcelUtils.multiThreadingModify(fileName,{d->
def city = d['B']
if(city){
def cityid = cityCodeDatas.get(city)
def total = findCount(cityid, sqlInstance)
d['I'] = total
// def a = findEstateType(cityid, sqlInstance)
//def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
//ExcelUtils.multiThreadingModify(fileName,{d->
// def city = d['B']
// if(city){
// def cityid = cityCodeDatas.get(city)
// def total = findCount(cityid, sqlInstance)
//// d['I'] = total
//// def a = findEstateType(cityid, sqlInstance)
////
//// d['AA'] = total
//// d['AB'] = DataHandleUtils.getDoubleRound(2, a*100/total)
////
//// def b = findFiledCount(cityid, 'buildyear', sqlInstance)
//// d['AC'] = DataHandleUtils.getDoubleRound(2, b*100/total)
////
//// def c = findFiledCount(cityid, 'address', sqlInstance)
//// d['AD'] = DataHandleUtils.getDoubleRound(2, c*100/total)
////
//// def d1 = findFiledCount(cityid, 'ZTS', sqlInstance)
//// d['AE'] = DataHandleUtils.getDoubleRound(2, d1*100/total)
////
//// def e = findFiledCount(cityid, 'landarea', sqlInstance)
//// d['AF'] = DataHandleUtils.getDoubleRound(2, e*100/total)
////
//// def f = findFiledCount(cityid, 'buildarea', sqlInstance)
//// d['AG'] = DataHandleUtils.getDoubleRound(2, f*100/total)
////
//// def g = findFiledCount(cityid, 'plotratio', sqlInstance)
//// d['AH'] = DataHandleUtils.getDoubleRound(2, g*100/total)
////
//// def h = findFiledCount(cityid, 'greeningrate', sqlInstance)
//// d['AI'] = DataHandleUtils.getDoubleRound(2, h*100/total)
////
//// def i = findFiledCount(cityid, 'latitude', sqlInstance)
//// d['AJ'] = DataHandleUtils.getDoubleRound(2, i*100/total)
////
//// def j = findFiledCount(cityid, 'parkingcnt', sqlInstance)
//// d['AK'] = DataHandleUtils.getDoubleRound(2, j*100/total)
//
// d['AA'] = total
// d['AB'] = DataHandleUtils.getDoubleRound(2, a*100/total)
//
// def b = findFiledCount(cityid, 'buildyear', sqlInstance)
// d['AC'] = DataHandleUtils.getDoubleRound(2, b*100/total)
//
// def c = findFiledCount(cityid, 'address', sqlInstance)
// d['AD'] = DataHandleUtils.getDoubleRound(2, c*100/total)
//
// def d1 = findFiledCount(cityid, 'ZTS', sqlInstance)
// d['AE'] = DataHandleUtils.getDoubleRound(2, d1*100/total)
//
// def e = findFiledCount(cityid, 'landarea', sqlInstance)
// d['AF'] = DataHandleUtils.getDoubleRound(2, e*100/total)
//
// def f = findFiledCount(cityid, 'buildarea', sqlInstance)
// d['AG'] = DataHandleUtils.getDoubleRound(2, f*100/total)
//
// def g = findFiledCount(cityid, 'plotratio', sqlInstance)
// d['AH'] = DataHandleUtils.getDoubleRound(2, g*100/total)
//
// def h = findFiledCount(cityid, 'greeningrate', sqlInstance)
// d['AI'] = DataHandleUtils.getDoubleRound(2, h*100/total)
//
// def i = findFiledCount(cityid, 'latitude', sqlInstance)
// d['AJ'] = DataHandleUtils.getDoubleRound(2, i*100/total)
//
// def j = findFiledCount(cityid, 'parkingcnt', sqlInstance)
// d['AK'] = DataHandleUtils.getDoubleRound(2, j*100/total)
// println d
// }
//},0)
//def fileName = "d:\\数据\\建行\\房产估值\\采购数据交付标准_0807-1.xlsx"
//def dealMap = findDealCount(sqlInstance)
//ExcelUtils.multiThreadingModify(fileName,{d->
// if(d['C']){
// d['Q'] = dealMap.get(d['C'])
// }
//},2)
//def fileName = "d:\\数据\\建行\\房产估值\\经纬度错误数据修正.xlsx"
//def updateIndex = 0
//def deleteIndex = 0
//ExcelUtils.modify(fileName,{d->
// if(d['R'] && d['R'].toString().toDouble().toInteger()==1){
// def dist = CommonDatautils.findDistrictById(d['D'])
// def city = dist.city
// def region = RegionData.formatRegion(city, d['Q'])
// if(region && dist.region!=region){
// println city +" new:"+region +" old:"+d['C']
// def udata = new BasicDBObject()
// def obj = CommonDatautils.findDistrict(dist.city, region, dist.name, "district")
// if(obj){
// deleteIndex++
// udata.put("status", 1)
// udata.put("m_date", new Date())
// DbUtils.seaweed.update(new BasicDBObject('_id',dist._id),new BasicDBObject('$set', udata))
// println deleteIndex +" "+dist.city +" "+ dist.name+" "+dist.region+"("+region+")"
// }else{
// updateIndex++
// udata.put("region", region)
// udata.put("m_date", new Date())
// DbUtils.seaweed.update(new BasicDBObject('_id',dist._id),new BasicDBObject('$set', udata))
// println updateIndex +" "+dist.city +" "+ dist.name+" "+dist.region+"("+region+")"
// }
// }else{
// println d['D']+" 不存在"
// }
// }
//},0)
def findDealCount(sqlInstance){
def sql = "select cityname,count(*) as count from bigdata_deal where dealdate>=to_date('2021-06-01','yyyy-mm-dd') and dealdate<to_date('2021-07-01','yyyy-mm-dd') group by cityname";
def count = 0
def map = [:]
sqlInstance.eachRow(sql,{r->
map.put(r.cityname, r.count)
});
return map
}
println d
}
},0)
def findEstateType(cityid, sqlInstance){
def sql = "select count(*) as count from bigdata_commbuildtype where cityid='${cityid}' and buildtype!='0199'";
@@ -132,22 +206,95 @@ def findFiledCount(cityid, field, sqlInstance){
}
def fileName = "d:\\数据\\建行\\房产估值\\经纬度错误数据修正.xlsx"
def updateIndex = 0
def delIndex = 0
def cityMap = [:]
ExcelUtils.multiThreadingModify(fileName,{d->
if(!d['R']){
def obj = CommonDatautils.findDistrictById(d['D'])
if(obj){
def ranges = cityMap.get(obj.city)
if(!ranges){
def dbcursor = POLYLINECOLL.find(new BasicDBObject("city",obj.city))
ranges = []
for(def r : dbcursor){
ranges << r
}
cityMap.put(obj.city, ranges)
}
if(ranges){
def lng = obj.lng/10000000000.0
def lat = obj.lat/10000000000.0
def region = null
for(def r : ranges){
try{
def flag = isInArea(lng, lat, r.polyline)
if(flag){
def regionStr = r.region
region = RegionData.formatRegion(obj.city, regionStr)
break
}
}catch(Exception e){
println e.getMessage()
}
}
if(region && region!=obj.region){
updateIndex++
DbUtils.seaweed.update(new BasicDBObject("_id",obj.get("_id")),new BasicDBObject('$set', new BasicDBObject("region", region)))
println updateIndex +" [update]old:${obj.region} new:"+region+" "+obj.name
}else if(region && region==obj.region){
println "[same] "+obj.city +" "+obj.name
}else{
DbUtils.seaweed.update(new BasicDBObject("_id",new ObjectId(d['D'])),new BasicDBObject('$set', new BasicDBObject("status", 1)))
delIndex++
println delIndex +" [delete] "+obj.city +" "+obj.name
}
}else{
println obj.city +" 没有此城市边框信息"
}
}else{
println d['D']+" 没找到该小区"
}
}
},0);
//def city = "中山"
//def city = "杭州"
//def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lat", "lng"])
//def dbcursor = POLYLINECOLL.find(new BasicDBObject("city",city))
//def ranges = []
//for(def d : dbcursor){
// ranges << d
//}
//def index =0
//def updateIndex = 0
//def delIndex = 0
//for(def dist : dists){
// if(dist.region=="石岐区街道"){
//// if(dist.region=="江干"){
// def lat = dist.lat/10000000000.0
// def lng = dist.lng/10000000000.0
// def regionStr = getStreetInfo(city, lng.toString()+","+lat.toString())
// if(regionStr){
// def region = RegionData.formatRegion(dist.city, regionStr)
//
// def regionStr =null
// def region = null
// for(def r : ranges){
// def flag = isInArea(lng, lat, r.polyline)
// if(flag){
// regionStr = r.region
// region = RegionData.formatRegion(dist.city, regionStr)
// break
// }
// }
// // def regionStr = getStreetInfo(city, lng.toString()+","+lat.toString())
// if(regionStr && region!=dist.region){
//
// def obj = CommonDatautils.findDistrict(dist.city, region, dist.name, "district")
// if(obj){
// delIndex++
@@ -156,30 +303,39 @@ def findFiledCount(cityid, field, sqlInstance){
// }else{
// updateIndex++
// DbUtils.seaweed.update(new BasicDBObject("_id",dist.get("_id")),new BasicDBObject('$set', new BasicDBObject("region", region)))
// println updateIndex +" [update]"+region+" "+dist.name
// println updateIndex +" [update]old:${dist.region} new:"+region+" "+dist.name
// }
// }else{
// index++
// println index+" [没找到]"+dist.name
// }
// }
//}
//
//def getStreetInfo(def city, def glocation){
// def township = null
// if(glocation){
// def jsonObject = GaoDeApiUtils.geocode(glocation)
// if(jsonObject && jsonObject.regeocode) {
// def addressComponent = jsonObject.regeocode.addressComponent
// if(addressComponent && addressComponent.city.contains(city)){
// township = addressComponent.township
// }
// }
// }
// return township;
//// }
//}
def isInArea(lng, lat, points){
List<Point> pointList = InsidePolygon.createPoint2(points);
//Point point = new Point(lng, lat);
//def flag = InsidePolygon.aboutZone(point, pointList)
def flag = InsidePolygon.isPtInPoly(lng, lat, pointList)
return flag;
}
def getStreetInfo(def city, def glocation){
def township = null
if(glocation){
def jsonObject = GaoDeApiUtils.geocode(glocation)
if(jsonObject && jsonObject.regeocode) {
def addressComponent = jsonObject.regeocode.addressComponent
if(addressComponent && addressComponent.city.contains(city)){
township = addressComponent.township
}
}
}
return township;
}
@@ -186,8 +186,8 @@ class ThirdPartyPriceBatch extends BaseBatch {
if(!dataMap.containsKey("anjuke_esf_price") && db.anjuke_esf_price){
dataMap.put("anjuke_esf_price", db.anjuke_esf_price)
}
if(!dataMap.containsKey("anjuke_esf_price") && db.get("58tc_esf")){
dataMap.put("anjuke_esf_price", db.get("58tc_esf"))
if(!dataMap.containsKey("anjuke_esf_price") && db.get("58tc_esf_price")){
dataMap.put("anjuke_esf_price", db.get("58tc_esf_price"))
}
if(!dataMap.containsKey("soufang_esf_price") && db.soufang_esf_price){
dataMap.put("soufang_esf_price", db.soufang_esf_price)
@@ -218,7 +218,7 @@ class CommonDatautils {
q.put("visible", 0);
q.put("cat", cat);
q.put("city", city);
// q.put("name","东寓新村")
// q.put("name","金辉伯爵山")
DBObject keys = new BasicDBObject();
if(fields){
@@ -1316,7 +1316,7 @@ class CommonDatautils {
}
public static def getAllDictionary(){
def dbCursor = DICT_COLL.find(new BasicDBObject())
def dbCursor = DICT_COLL.find(new BasicDBObject()).batchSize(5000)
def rets = []
def filters = [ "estate","hfword","main","metro","station"]
for(def obj : dbCursor){