Merge branch 'script_gj' into 'master'

poi 修改

See merge request !869
This commit is contained in:
2024-06-21 21:45:20 +08:00
9 changed files with 169 additions and 75 deletions
@@ -389,7 +389,7 @@ class DealPriceDataProcess {
def deleteThirdPartyData(){ def deleteThirdPartyData(){
// ,"陵水","塔城","保亭","长沙","苏州","南昌","沧州","文昌","衡水" // ,"陵水","塔城","保亭","长沙","苏州","南昌","沧州","文昌","衡水"
// def citys = ["310000","440100","500000","410100","430100","330100","340100","320100","370200","320500","120000","442000","320300","350200","360100","320400","210200","530100","469005","469028","469029","230700","610700","533400","650400","511100","131100"]//CcbCommonUtils.citys_all // def citys = ["310000","440100","500000","410100","430100","330100","340100","320100","370200","320500","120000","442000","320300","350200","360100","320400","210200","530100","469005","469028","469029","230700","610700","533400","650400","511100","131100"]//CcbCommonUtils.citys_all
def citys = ["上海","广州"]//CcbCommonUtils.citys_all def citys = CcbCommonUtils.citys_all
def month = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));//上个月日期 def month = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));//上个月日期
def rows = [] def rows = []
def cityCodeDatas = CcbCommonUtils.findCityCodeValue() def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
@@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//添加自己库中的人工指导价 //添加自己库中的人工指导价
def prices = CcbCommonUtils.findPersonPrice(city, "2024-05-24", "2024-06-21") def prices = CcbCommonUtils.findPersonPrice(city, "2024-05-24", "2024-06-22")
// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22") // def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22")
if(prices){ if(prices){
def distMap = getDistrictMap(city) def distMap = getDistrictMap(city)
@@ -1289,7 +1289,7 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20240607人工指导价.xlsx" def fileName = "D:\\数据\\建行\\房产估值\\指导价\\国信达指导价6-21.xlsx"
ExcelUtils.multiThreadingModify(fileName, {d-> ExcelUtils.multiThreadingModify(fileName, {d->
if(d['J']){ if(d['J']){
@@ -1301,7 +1301,7 @@ ExcelUtils.multiThreadingModify(fileName, {d->
// d['H'] = dist._id.toString() // d['H'] = dist._id.toString()
// } // }
// } // }
if(!d['A'] && d['J'] instanceof String){ if(d['J'] instanceof String){
def dist = CommonDatautils.findDistrictById(d['J']) def dist = CommonDatautils.findDistrictById(d['J'])
if(dist){ if(dist){
d['A'] = dist.city d['A'] = dist.city
@@ -1318,6 +1318,19 @@ ExcelUtils.multiThreadingModify(fileName, {d->
},0) },0)
//def file = "D:\\数据\\建行\\房产估值\\指导价\\5城价格翻新6-21.xlsx"
//ExcelUtils.multiThreadingModify(file, { d ->
// if(d['A']){
// def dist = CommonDatautils.findDistrictById(d['A'])
// if(dist){
// d['C'] = dist.city
// d['D'] = dist.region
// d['E'] = dist.name
// def random = RandomUtil.random(-300,50)
// d['F'] = random + d['B']
// }
// }
//})
//def filePath = "D:\\数据\\建行\\房产估值\\小区\\估家独家数据打标签.xlsx" //def filePath = "D:\\数据\\建行\\房产估值\\小区\\估家独家数据打标签.xlsx"
//ExcelUtils.multiThreadingModify(filePath, { d -> //ExcelUtils.multiThreadingModify(filePath, { d ->
// if(d['B']){ // if(d['B']){
@@ -1846,39 +1846,6 @@ class CommonDatautils {
} }
return priceMap; return priceMap;
} }
public static def queryDistrictPrice(city, region, name, weekend, cat){
DBObject queryCondition = new BasicDBObject()
queryCondition.put("city", city);
queryCondition.put("region", region);
queryCondition.put("district_name", name);
queryCondition.put("weekend", weekend);
if(cat == "district"){
BasicDBObject queryObj = new BasicDBObject();
queryObj.putAt(QueryOperators.IN, ["小区","别墅"]);
queryCondition.put("estate_type", queryObj);
}else if(cat == "office"){
queryCondition.put("estate_type", "写字楼");
}else if(cat == "shop"){
queryCondition.put("estate_type", "商铺");
}
DBCursor cur = DIST_STAT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", -1))
cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def price = 0;
while(cur.hasNext()) {
DBObject dbo = cur.next()
price = dbo.get("avg_price");
if(price){
break;
}
}
return price;
}
public static def queryFirstDistrictPrice(city, region, name){ public static def queryFirstDistrictPrice(city, region, name){
@@ -1917,6 +1884,39 @@ class CommonDatautils {
} }
public static def queryDistrictPrice(city, region, name, weekend, cat){
DBObject queryCondition = new BasicDBObject()
queryCondition.put("city", city);
queryCondition.put("region", region);
queryCondition.put("district_name", name);
queryCondition.put("weekend", weekend);
if(cat == "district"){
BasicDBObject queryObj = new BasicDBObject();
queryObj.putAt(QueryOperators.IN, ["小区","别墅"]);
queryCondition.put("estate_type", queryObj);
}else if(cat == "office"){
queryCondition.put("estate_type", "写字楼");
}else if(cat == "shop"){
queryCondition.put("estate_type", "商铺");
}
DBCursor cur = DIST_STAT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", -1))
cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def price = 0;
while(cur.hasNext()) {
DBObject dbo = cur.next()
price = dbo.get("avg_price");
if(price){
break;
}
}
return price;
}
public static def queryDistrictPrice(city, region, name){ public static def queryDistrictPrice(city, region, name){
DBObject queryCondition = new BasicDBObject() DBObject queryCondition = new BasicDBObject()
queryCondition.put("city", city); queryCondition.put("city", city);
@@ -4679,7 +4679,7 @@ class CommonDatautils {
def q = new BasicDBObject() def q = new BasicDBObject()
q.put("type","乡镇街道") q.put("type","乡镇街道")
q.put("province",new BasicDBObject('$in', ['浙江省','江苏省','山东省'])) // q.put("province",new BasicDBObject('$in', ['浙江省','江苏省','山东省']))
def keys = new BasicDBObject() def keys = new BasicDBObject()
keys.put("province",1) keys.put("province",1)
@@ -39,7 +39,7 @@ class DevProcess {
// task.calAddrDistince() // task.calAddrDistince()
// task.updateCityAndDist() // task.updateCityAndDist()
task.addDevAddres1() //按添加地址中包含开发区名称的地址 // task.addDevAddres1() //按添加地址中包含开发区名称的地址
// task.addDevAddres2()//添加相差2公里范围内的地址 // task.addDevAddres2()//添加相差2公里范围内的地址
// task.matchDevAndAddAddress() // task.matchDevAndAddAddress()
@@ -48,7 +48,7 @@ class DevProcess {
// task.createDevMapping() // task.createDevMapping()
// task.deleteAddr() // task.deleteAddr()
// task.deleteAddr2() // task.deleteAddr2()
// task.synchDevTo43() task.synchDevTo43()
// task.addNewDev2() // task.addNewDev2()
// task.addAlias() // task.addAlias()
@@ -644,16 +644,16 @@ class DevProcess {
def matchDevAndAddAddress(){ def matchDevAndAddAddress(){
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\新跑省份\\福建去重地址入库.xlsx",{d-> ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\新跑省份\\福建重庆去重后地址.xlsx",{d->
if(d['J']){ if(d['E']){
def q = new BasicDBObject() def q = new BasicDBObject()
q.put("name", d['J']) q.put("name", d['E'])
def obj = development_zone_CONN.findOne(q) def obj = development_zone_CONN.findOne(q)
if(obj){ if(obj){
d['P'] = obj._id.toString() d['F'] = obj._id.toString()
d['Q'] = obj.name d['G'] = obj.name
def address = d['L']+d['M'] def address = d['D']
q.put("address", address) q.put("address", address)
// def street = d['G'] // def street = d['G']
// q.put("township", street) // q.put("township", street)
@@ -685,14 +685,14 @@ class DevProcess {
def matchDevTownship(){ def matchDevTownship(){
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\区域匹配开发区.xlsx",{d-> ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\新跑省份\\福建重庆去重后地址.xlsx",{d->
if(d['C']){ if(d['D']){
def q = new BasicDBObject() def q = new BasicDBObject()
q.put("name", d['C']) q.put("name", d['D'])
def obj = development_zone_CONN.findOne(q) def obj = development_zone_CONN.findOne(q)
if(obj){ if(obj){
d['E'] = obj._id.toString() d['F'] = obj._id.toString()
def street = d['D'] def street = d['E']
q.put("township", street) q.put("township", street)
def sameObj = DEV_MAPPING_CONN.findOne(q) def sameObj = DEV_MAPPING_CONN.findOne(q)
if(!sameObj){ if(!sameObj){
@@ -706,11 +706,11 @@ class DevProcess {
} }
udata.put("name",obj.name) udata.put("name",obj.name)
// udata.put("type", "乡镇街道") udata.put("type", "乡镇街道")
// udata.put("township", street) udata.put("township", street)
udata.put("type", "区域") // udata.put("type", "区域")
udata.put("dev_district", street) // udata.put("dev_district", street)
udata.put("c_date",new Date()) udata.put("c_date",new Date())
DEV_MAPPING_CONN.save(udata) DEV_MAPPING_CONN.save(udata)
println obj.name +" "+street println obj.name +" "+street
@@ -39,15 +39,15 @@ class PoiLocation {
} }
def run(){ def run(){
def citys = ["温州市","嘉兴市","台州市","湖州市","绍兴市","金华市","舟山市","衢州市","丽水市", "南京市","苏州市","无锡市","扬州市","常州市","徐州市","镇江市","南通市","泰州市","连云港市","盐城市","淮安市","宿迁市","济南市","青岛市","烟台市","威海市","潍坊市","淄博市","临沂市","泰安市","菏泽市","东营市","德州市","济宁市","日照市","聊城市","滨州市","枣庄市"] def citys = ["温州市","嘉兴市","台州市","湖州市","绍兴市","金华市","舟山市","衢州市","丽水市", "南京市","苏州市","无锡市","扬州市","常州市","徐州市","镇江市","南通市","泰州市","连云港市","盐城市","淮安市","宿迁市","济南市","青岛市","烟台市","威海市","潍坊市","淄博市","临沂市","泰安市","菏泽市","东营市","德州市","济宁市","日照市","聊城市","滨州市","枣庄市"]
for(def city : citys){ // for(def city : citys){
findPois(city) findPois()
} // }
} }
def findPois(def city){ def findPois(){
def q = new BasicDBObject() def q = new BasicDBObject()
q.put("city", city) // q.put("city", city)
q.put("run_status", new BasicDBObject('$exists', false)) q.put("run_status", new BasicDBObject('$exists', false))
def keys = new BasicDBObject() def keys = new BasicDBObject()
@@ -115,7 +115,9 @@ class PoiLocation {
def regionFlag = null def regionFlag = null
if(d.district){ if(d.district){
for(def rs : regions){ for(def rs : regions){
regionFlag = InsidePolygon.isInArea(d.location.lon, d.location.lat, InsidePolygon.createPoint4(rs.shape.coordinates)) if(rs.shape && rs.shape.coordinates){
regionFlag = InsidePolygon.isInArea(d.location.lon, d.location.lat, InsidePolygon.createPoint4(rs.shape.coordinates))
}
} }
} }
@@ -50,7 +50,7 @@ class TownMiddleProcess {
//修改乡镇街道别名 //修改乡镇街道别名
// updateTownAlias() // updateTownAlias()
//中间表同步到100 //中间表同步到100
// synchTown() synchTown()
//添加地级市标志 //添加地级市标志
// addCityFlag() // addCityFlag()
// addPinyin() // addPinyin()
@@ -3,9 +3,11 @@ package com.fangjia.script.gj.data_handle
import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.looplineinfo.InsidePolygon import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.util.CosineSimilarAlgorithm import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils import com.fangjia.script.util.MongoUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection import com.mongodb.DBCollection
import java.util.concurrent.CopyOnWriteArrayList import java.util.concurrent.CopyOnWriteArrayList
@@ -27,7 +29,9 @@ class TownshipStrandUpdate {
static main(args) { static main(args) {
def task = new TownshipStrandUpdate() def task = new TownshipStrandUpdate()
task.process() // task.process()
task.updateTown()
// task.stat()
} }
@@ -42,13 +46,14 @@ class TownshipStrandUpdate {
def citys = CommonDatautils.findAllCity() def citys = CommonDatautils.findAllCity()
citys = citys.reverse() citys = citys.reverse()
for(def city : citys){ for(def city : citys){
if(["中山"].contains(city)){ //
if(["中山","东莞"].contains(city)){
continue continue
} }
def fileName = "D:\\数据\\数据建设\\乡镇街道\\小区\\小区乡镇街道格式化-${city}.xlsx" def fileName = "D:\\数据\\数据建设\\乡镇街道\\商铺\\商铺乡镇街道格式化-${city}.xlsx"
File file=new File(fileName); File file=new File(fileName);
if(!file.exists()){ if(!file.exists()){
def dists = CommonDatautils.findAllDistrictsFromMg(city, "district", ["city","region","name","address","cat","estate_type2","status","township","lng","lat"]) def dists = CommonDatautils.findAllDistrictsFromMg(city, "shop", ["city","region","name","address","cat","estate_type2","status","township","lng","lat"])
dists = checkData(dists) dists = checkData(dists)
def datas = createDatas(dists) def datas = createDatas(dists)
ExcelUtils.write(fileName, datas, "${city}") ExcelUtils.write(fileName, datas, "${city}")
@@ -56,6 +61,77 @@ class TownshipStrandUpdate {
} }
} }
// 相同且为统计局乡镇
// 更正
// 不同且不为统计局乡镇
// 区域不同
// 地图无乡镇街道
// 城市不同
def stat(){
def citys = CommonDatautils.findAllCity()
citys = citys.reverse()
def rets = []
rets << ["城市", "小区量", "更正量","相同且为统计局乡镇","不同且不为统计局乡镇","区域不同","地图无乡镇街道","城市不同","无值"]
for(def city : citys){
def fileName = "D:\\数据\\数据建设\\乡镇街道\\商铺\\商铺乡镇街道格式化-${city}.xlsx"
File file=new File(fileName);
def map = [:]
def datas = null
if(file.exists()){
datas = ExcelUtils.read(fileName,0, 1)
for(def d : datas){
def key = "空值"
if(d['S']){
key = d['S']
}
def count = map.get(key)
if(!count){
count = 0
}
map.put(key, count+1)
}
}
println city
map.each {k,v->
println k+" "+v
}
println "--------------------------------------------------------------------------------------------"
if(map){
rets << [city, datas.size(), map.get("更正"),map.get("相同且为统计局乡镇"),map.get("不同且不为统计局乡镇"),map.get("区域不同"),map.get("地图无乡镇街道"),map.get("城市不同"),map.get("空值")]
}
}
ExcelUtils.write("D:\\数据\\数据建设\\乡镇街道\\乡镇街道更新结果统计.xlsx", rets, "商铺")
}
def updateTown(){
def citys = CommonDatautils.findAllCity()
citys = citys.reverse()
for(def city : citys){
def fileName = "D:\\数据\\数据建设\\乡镇街道\\小区\\小区乡镇街道格式化-${city}.xlsx"
File file=new File(fileName);
if(file.exists()){
println fileName
ExcelUtils.multiThreadingModify(fileName,{d->
if(d['S']!='相同且为统计局乡镇' && d['S']){
def q = new BasicDBObject()
q.put("city", d['A'])
q.put("region", d['B'])
q.put("name", d['C'])
q.put("status", 0)
q.put("cat","district")
def udata = new BasicDBObject()
udata.put("township", d['R'])
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
}
})
}
}
}
def checkData(def dists){ def checkData(def dists){
def totalCount = new AtomicLong() def totalCount = new AtomicLong()
def rets = new CopyOnWriteArrayList() def rets = new CopyOnWriteArrayList()
@@ -33,11 +33,11 @@ class AddPriceTest {
task.run(); task.run();
} }
def run(){ def run(){
def filePath = "D:\\数据\\建行\\房产估值\\神经网络计算.xlsx" def filePath = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\0621.xlsx"
addGXDPrice(filePath) // addGXDPrice(filePath)
// addCcbPrice(filePath) // addCcbPrice(filePath)
// addThirdPrice(filePath) // addThirdPrice(filePath)
// addFjwPrice(filePath) addFjwPrice(filePath)
// exportPkPrice() // exportPkPrice()
// addRegion() // addRegion()
// addTradePrice(filePath) // addTradePrice(filePath)
@@ -205,7 +205,7 @@ class AddPriceTest {
def addThirdPrice(filepath){ def addThirdPrice(filepath){
ExcelUtils.multiThreadingModify(filepath, { d -> ExcelUtils.multiThreadingModify(filepath, { d ->
if (!d['X']) { if (!d['X']) {
def dist = CommonDatautils.findDistrictById(d['Y']) def dist = CommonDatautils.findDistrictById(d['A'])
if(dist){ if(dist){
def thirdPrice = findThirdPartyPrice(dist.city, dist.region, dist.name) def thirdPrice = findThirdPartyPrice(dist.city, dist.region, dist.name)
d['X'] = thirdPrice d['X'] = thirdPrice
@@ -220,8 +220,8 @@ class AddPriceTest {
q.put("city", city) q.put("city", city)
q.put("region", region) q.put("region", region)
q.put("name", name) q.put("name", name)
// q.put("s_date", new BasicDBObject('$gte', 202311).append('$lte',202404)) q.put("s_date", new BasicDBObject('$gte', 202403))
q.put("s_date", 202404) // q.put("s_date", 202404)
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1)) def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1))
def price = null def price = null
for(def db : dbcursor){ for(def db : dbcursor){
@@ -271,13 +271,16 @@ class AddPriceTest {
def addFjwPrice(path){ def addFjwPrice(path){
ExcelUtils.multiThreadingModify(path,{d-> ExcelUtils.multiThreadingModify(path,{d->
if(!d['L']){ if(!d['L']){
if(d['B'] instanceof String){ if(d['A'] instanceof String){
def dist = CommonDatautils.findDistrictById(d['B']) def dist = CommonDatautils.findDistrictById(d['A'])
if(dist){ if(dist){
d['A'] = dist.city
d['B'] = dist.region
d['C'] = dist.name
// d['N'] = dist.region // d['N'] = dist.region
def dbo = CommonDatautils.queryFirstDistrictPrice(dist.city, dist.region, dist.name) def dbo = CommonDatautils.queryFirstDistrictPrice(dist.city, dist.region, dist.name)
if(dbo){ if(dbo){
// d['W'] = dbo.avg_price d['W'] = dbo.avg_price
// d['X'] = dbo.cal_type // d['X'] = dbo.cal_type
d['L'] = dbo.avg_price_2b d['L'] = dbo.avg_price_2b
} }