This commit is contained in:
2025-04-02 09:45:32 +08:00
parent f8baadb0fe
commit a094b9ead8
21 changed files with 1264 additions and 231 deletions
@@ -8,7 +8,7 @@ import com.mongodb.DBCollection
/**
* 深圳公众号成交数据录入
*/
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\2024年11月深圳二手房成交.xlsx", 0, 1)
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\2025年01月深圳二手房成交的.xlsx", 0, 1)
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
@@ -1353,13 +1353,14 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){
//},0)
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20241025人工指导价.xlsx"
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\更新人工指导价_0314.xlsx"
ExcelUtils.multiThreadingModify(fileName, { d ->
def dist = CommonDatautils.findDistrictById(d['J'])
def dist = CommonDatautils.findDistrictById(d['A'])
if(dist){
d['A'] = dist.city
d['B'] = dist.region
d['C'] = dist.name
d['C'] = dist.city
d['D'] = dist.region
d['E'] = dist.name
d['F'] = dist.status
}
})
@@ -262,7 +262,7 @@ class CommonDatautils {
q.put("status", 0);
q.put("cat", cat);
q.put("city", city);
// q.put("name","托斯卡纳庄园")
// q.put("name","新泾三村")
DBObject keys = new BasicDBObject();
if(fields){
@@ -821,7 +821,9 @@ class CommonDatautils {
def dbsour = SEAWEED_COLL.find(q, keys).batchSize(5000)
def rets = []
for(def d : dbsour){
rets << d
if(!d.township){
rets << d
}
}
return rets;
}
@@ -4717,6 +4719,14 @@ class CommonDatautils {
}
static def httpSaveData89(def newUrl,def params){
def header = [:]
header.put("Authorization","Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString())
doPost(newUrl,header, params, true, "utf-8")
}
static def doPost(def targetUrl, def headers, def params, def isJson, def charset){
StringBuilder buffer = new StringBuilder();
try{
@@ -1,5 +1,6 @@
package com.fangjia.script.gj
import com.fangjia.script.common.DataHandleUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection;
@@ -12,12 +13,13 @@ import com.fangjia.script.util.ExcelUtils;
class FangjiatongUserStat {
public static final DBCollection FJT_USER_COLL = MongoUtils.getCollectionByName("fangjiatong_log", "utm_gif_log", "222.73.242.42", 52002)
static final def onLineSqlInstance = Sql.newInstance("jdbc:mysql://222.73.242.42:4306/goojia_fangjia?&characterEncoding=utf8", "sa", "G00jia", "com.mysql.jdbc.Driver");
static final def onLineSqlInstance = Sql.newInstance("jdbc:mysql://mysql.db.fangjia.com:4306/goojia_fangjia?&characterEncoding=utf8", "sa", "G00jia", "com.mysql.jdbc.Driver");
public static void main(String[] args) {
def task = new FangjiatongUserStat();
task.run();
// task.run();
task.stat()
}
def run(){
@@ -60,4 +62,59 @@ class FangjiatongUserStat {
return ret;
}
def stat(){
def users = findUses(12)
def months = ["2019-01-01","2019-02-01","2019-03-01","2019-04-01","2019-05-01","2019-06-01","2019-07-01","2019-08-01","2019-09-01","2019-10-01","2019-11-01","2019-12-01","2020-01-01"]
def rets = []
def header = ["用户id","支行名称", "备注名称"]
months.each{m->
header << m.substring(0, 7)
}
rets << header
for(def m : users){
def ret = []
ret << m.user_id
ret << m.name
ret << m.org_name
for(def i=1;i<months.size();i++){
def evalCount = findEvaluationCount(m.user_id, months[i-1], months[i])
ret << evalCount
}
for(def i=1;i<months.size();i++){
def downCount = findTradesCount(m.user_id, months[i-1], months[i])
ret << downCount
}
rets << ret
}
ExcelUtils.write("D:\\数据\\农行统计.xlsx", rets, "农行统计-2019")
}
def findUses(role){
def sql = "select u.id,ud.name,ud.org_name from (SELECT * from users WHERE role=12) u left join users_detail ud on u.id=ud.id ";
def rets = [];
onLineSqlInstance.eachRow(sql, {
rets << ["user_id":it.id, "name":it.name, "org_name":it.org_name]
})
return rets
}
def findEvaluationCount(user_id, startDate, endDate){
def sql = "select count(*) as count from evaluation where user_id=${user_id} and DATE_CREATED>='${startDate}' and DATE_CREATED<'${endDate}'";
def count = 0;
onLineSqlInstance.eachRow(sql, {
count = it.count
})
return count
}
def findTradesCount(user_id, startDate, endDate){
def sql = "select count(*) as count from user_trades where user_id=${user_id} and DATE_CREATED>='${startDate}' and DATE_CREATED<'${endDate}'";
def count = 0;
onLineSqlInstance.eachRow(sql, {
count = it.count
})
return count
}
}
@@ -22,9 +22,6 @@ class FiledStrand {
def updateIndex = 0;
static main(args) {
// def task = new FiledStrand();
// task.run();
// task.updateOpenDate();
@@ -271,7 +268,7 @@ class FiledStrand {
static def processArea(all_area){
if(all_area){
def v = getValue(all_area)
if(v>=5000 && v<=10000000){
if(v>=1000 && v<=10000000){
return v.toString().toDouble().toInteger()
}
}
@@ -288,11 +285,25 @@ class FiledStrand {
return null
}
static def processLobbyHeight(height){
if(height){
def v = getValue(height)
if(v>=2.4 && v<=30){
return v.toString()
}
}
return null
}
static def processElevators(elevators){
if(elevators){
def v = getValue(elevators)
if(v>0 && v<=100){
return v.toString()
if(v instanceof Integer){
if(v>0 && v<=100){
return v.toString()
}
}
}
return null
@@ -317,24 +328,34 @@ class FiledStrand {
static def processStandardArea(area){
if(area){
area = area.toString().replace(",",",").replace("㎡", "").replace("m", "").replace("/","")
area = Pattern.compile("[\\u4e00-\\u9fa5]").matcher(area).replaceAll("");
return area
def v = getValue(area)
if(v>=500 && v<=20000){
return v.toString().toDouble().toInteger()
}
}
return null
}
static def processCarSiteRent(rent){
if(rent){
def v = getNumberRange(rent)
if(v){
if(v=="0"){
if(["-","0元/月","无","不详","未知","待查","免费元/月","/","0","*","待确认","undefined元/月","0元","0元每月","未知元/月","0.00元/月","未定","暂定","测算中","无元/月","未定元/月","另议元/月","面议元/月","面议元/月","有元/月","暂时未确定","0元(无停车位)","&nbsp;元/月"].contains(rent)){
return null
}else if(rent.contains("暂无") || rent.contains("车位不归") || rent.contains("待定") || rent.contains("暂定") || rent.contains("暂完善") || rent.contains("待审批") || rent.contains("不详") || rent.contains("未定价格") || rent.contains("中国电信") || rent.contains("面谈")){
return null
}else{
rent = rent.replace("元/月","元/月").replace(",","").replace("#","").replace(".00","")
if(rent.toString().length()>100){
return null
}else{
return v.toString()
}
if(rent instanceof Double && rent.toDouble()<100){
return null
}
if(rent instanceof Integer && rent.toInteger()<100){
return null
}
return rent
}
return null
}
return null
}
@@ -382,7 +403,7 @@ class FiledStrand {
static def processCarSite(car_site_count){
def v = getValue(car_site_count.toString());
if(v){
if(v>5 && v<=100000){
if(v>5 && v<=10000){
try {
return v.toString().toInteger();
}catch(Exception e){
@@ -533,12 +554,82 @@ class FiledStrand {
return null
}
static def processLevel(val){
def level = null
if (val) {
if(val.contains("超甲") || val.contains("5A") || val.contains("6A") || val.contains("顶级")){
level = "超甲级"
}else if(val.indexOf("甲")>-1 || val.contains("A")){
level = "甲级"
}else if(val.indexOf("乙")>-1 || val.contains("B")){
level = "乙级";
}else if(val.indexOf("丙")>-1 || val.contains("C")){
level = "丙级";
}
}
return level
}
static def processLoopLine(val){
def line = null
if (val) {
if(!val.contains("暂") && !val.contains("无")){
if(val.indexOf("环")>-1){
line = val
}
}
}
return line
}
static def processForeign(val){
def foreign = null
if (val) {
if(val.indexOf("是")>-1){
foreign = "是"
}else if(val.indexOf("否")>-1){
foreign = "否";
}
}
return foreign
}
static def processRegister(val){
def register = null
if (val) {
if(val.contains("不可以注册") || val.contains("不能注册")){
register = "否";
}else if(val.contains("可以注册")){
register = "是"
}else if(val.indexOf("是")>-1){
register = "是"
}else if(val.indexOf("否")>-1){
register = "否";
}
}
return register
}
static def processFitment(val){
def fitment = null
if(val){
if(val.indexOf("精装")>-1){
fitment = "精装"
}else if(val.indexOf("简单")>-1 || val.indexOf("简装")>-1 || val.indexOf("一般")>-1 || val.indexOf("普通")>-1){
fitment = "简装";
}else if(val.indexOf("毛坯")>-1){
fitment = "毛坯";
}
}
return fitment
}
static def processLngAndLat(val) {
if (val) {
if (val && val!="null") {
def map = [:]
def lng = DataHandleUtils.getDoubleRound(6, val.x)
def lat = DataHandleUtils.getDoubleRound(6, val.y)
def lng = DataHandleUtils.getDoubleRound(6, val.lon)
def lat = DataHandleUtils.getDoubleRound(6, val.lat)
Long gd_lat = lat * 10000000000L
Long gd_lng = lng * 10000000000L
map.lat = gd_lat
@@ -618,7 +709,8 @@ class FiledStrand {
if(cubage_percent){
def v = getValue(cubage_percent)
if(v){
if(v>=0.3 && v<=10){
v = DataHandleUtils.getDoubleRound(2, v.toString().toDouble())
if(v>=0.3 && v<=20){
return v.toString()
}
}
@@ -644,7 +736,7 @@ class FiledStrand {
static def processVirescencePercent(virescence_percent){
if(virescence_percent){
def v = getValue(virescence_percent)
if(v && v>15 && v<=70){
if(v && v>=10 && v<=70){
return v.toString()
}
}
@@ -788,6 +880,20 @@ class FiledStrand {
return null
}
static def processUndergroundFloor(floor){
def v = getValue(floor.toString());
if(v){
if(v>=1 && v<=5){
try {
return v.toString();
}catch(Exception e){
println e.getMessage()
}
}
}
return null
}
static def processRealUsePercent(percent){
def v = getValue(percent.toString());
if(v){
@@ -834,15 +940,40 @@ class FiledStrand {
static def processCompany(val){
if(val){
if(val.length()>20){
if(val.length()>20 || val.length()<4){
return null
}else{
if(val.contains("暂无")){
if(["#NA","*"].contains(val)){
return null
}
if(val.contains("暂无") || val.contains("暂定")){
return null
}
if(val.contains("未知") || val.contains("不祥")){
return null
}
if(val.contains("待定") || val.contains("待详情")){
return null
}
if(val.contains("尚未公开")){
return null
}
if(val.contains("在线咨询")){
return null
}
if(val.contains("平米") || val.contains("元平月")){
return null
}
if(val == "无"){
return null
}
if(val == "-"){
return null
}
if(val == "--"){
return null
}
if(val.isNumber()){
return null
}
@@ -863,7 +994,7 @@ class FiledStrand {
if (pattern.matcher(val).matches()) {
return null
}
return val
return val.replace("*","").replace("*","")
}
}
@@ -3,7 +3,11 @@ package com.fangjia.script.gj.data_handle
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.core.util.ExcelUtil
import com.fangjia.core.util.GeoUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.looplineinfo.InPolygon
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
@@ -13,6 +17,7 @@ import groovy.json.JsonSlurper
import groovy.io.FileType
import org.bson.types.ObjectId
import java.awt.geom.Point2D
import java.util.regex.Pattern
class AddDevShapeData {
@@ -22,22 +27,175 @@ class AddDevShapeData {
public static final def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection dev_mapping_CONN = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL)
public static final DBCollection administrative_mapping_coll = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
def devAreaMap = [:]
def fullCityMap = CommonDatautils.queryFullNameCity()
def results = new HashSet()
def existList = new HashSet()
static main(args) {
def task = new AddDevShapeData();
task.run()
// task.streetInfo()
// task.addStreetToMapping()//通过乡镇街道的点是否都在开发区围栏中,判断是否将此乡镇加入映射表中
}
def addStreetToMapping(){
def citys = CommonDatautils.findAllCity()
for(def city : citys){
def data = fullCityMap.get(city)
def fullCityName = null
if(data){
fullCityName = data.city
}
def devShapes = findDevShapeByCity(fullCityName)
if(devShapes){
def streetShapes = findStreetShapeByCity(fullCityName)
addDevStreetToMapping(fullCityName,streetShapes, devShapes)
}
}
results.each {s->
println s
}
existList.each{s->
println s
}
println " 新增映射乡镇:"+results.size()+" 已存在乡镇:"+existList.size()
}
def addDevStreetToMapping(city,streetShapes, devShapes){
def addCount = 0
def existsCount = 0
for(def streetObj : streetShapes){
def coordinates = streetObj.shape.coordinates
for(def pointList : coordinates){
def objList = findDevInStreetRange(streetObj, pointList, devShapes)
if(objList){
for(def obj : objList){
def existsFlag = isExistStreetByDevName(obj.name, streetObj.street)
if(existsFlag){
existsCount++
existList << "[存在] "+obj.name+" "+ streetObj.street
}else{
addCount++
results << "[新增] "+obj.name+"#"+ streetObj.street
if(!streetObj.street){
println streetObj
}
}
}
}
}
}
}
def findDevInStreetRange(streetObj,pointList, devShapes){
def devList = []
def minDis =200
for(def devObj : devShapes){
def flagList = []
def distances = []
for(def pointArr : pointList){
// def flag = InsidePolygon.isInArea(pointArr[0].toString().toDouble(), pointArr[1].toString().toDouble(), InsidePolygon.createPoint4(devObj.shape.coordinates))
Point2D.Double point = new Point2D.Double(DataHandleUtils.getDoubleRound(6,pointArr[0].toString().toDouble()), DataHandleUtils.getDoubleRound(6, pointArr[1].toString().toDouble()));
def flag = InPolygon.isInArea(point, InPolygon.createPoint(devObj.shape.coordinates))
flagList << flag
if(!flag){
distances << findNearestDistances(pointArr, devObj.shape.coordinates)
}
}
def trueCount = statResult(flagList)
def pre = trueCount*100/flagList.size()
println streetObj.district+" "+streetObj.street +" "+devObj.name +" "+flagList.size()+" "+pre+"%"
if(pre>=100){
devList << devObj
}
// else{
// if(distances && pre>=50){
// def minCount = 0
// for(def d : distances){
// if(d <= minDis){
// minCount++
// }
// }
// def simlarPointPre = minCount*100/distances.size()
// if(simlarPointPre>=90){
// devList << devObj
// }
// println "不在多边型范围内的点,距离小于${minDis}米有:"+minCount+" 占比:"+simlarPointPre
// }
// }
}
return devList
}
def findNearestDistances(pointArr, coordinates){
def minDis = Integer.MAX_VALUE
for(def pointlist : coordinates){
for(def devPoint : pointlist){
def distance = GeoUtils.calculateDistance(pointArr[0], pointArr[1], devPoint[0], devPoint[1])
if(distance < minDis){
minDis = distance
}
}
}
return minDis
}
def statResult(rows){
def count = 0
for(def s : rows){
if(s){
count++
}
}
return count
}
def findStreetShapeByCity(city){
def list = []
def dbcurosr = administrative_mapping_coll.find(new BasicDBObject("city", city).append("type", "乡镇街道").append("shape", new BasicDBObject('$exists', true)))
for(def d : dbcurosr){
list << d
}
return list
}
def findDevShapeByCity(city){
def list = []
def dbcurosr = DEVELOPMENT_ZONE_43_COLL.find(new BasicDBObject("city", city).append('shape', new BasicDBObject('$exists',true)))
for(def d : dbcurosr){
list << d
}
return list
}
def isExistStreetByDevName(devName, street){
def list = dev_mapping_CONN.find(new BasicDBObject("name", devName).append("type","乡镇街道").append("township",street)).toList()
if(list){
return true
}
}
def run(){
def directoryPath = 'D:\\数据\\数据建设\\开发区外包\\111开发区\\'
def directoryPath = 'D:\\数据\\数据建设\\开发区外包\\218\\'
def dir = new File(directoryPath)
if (dir.isDirectory()) {
// dir.eachFileRecurse(FileType.FILES) { file ->
// process(file.absolutePath)
// }
addDevInfo("D:\\数据\\数据建设\\开发区外包\\所有开发区.xlsx")
dir.eachFileRecurse(FileType.FILES) { file ->
process(file.absolutePath)
}
// addDevInfo("D:\\数据\\数据建设\\开发区外包\\所有开发区.xlsx")
} else {
println "The specified path is not a directory."
}
@@ -46,7 +204,7 @@ class AddDevShapeData {
def addDevInfo(file){
ExcelUtils.multiThreadingModify(file, {d->
// if(!d['I']){
def name = d['D']
def name = d['E']
def q = new BasicDBObject()
q.put('_id',new ObjectId(d['A']))
def obj = DEVELOPMENT_ZONE_43_COLL.findOne(q)
@@ -57,18 +215,28 @@ class AddDevShapeData {
if(obj.operate_result){
d['I'] = obj.operate_result
}
if(obj.m_date){
d['P'] = DateUtils.formatDateByPattern(obj.m_date, "yyyy-MM-dd")
if(obj.shape){
d['T'] = "有"
}
// if(obj.m_date){
// d['P'] = DateUtils.formatDateByPattern(obj.m_date, "yyyy-MM-dd")
// }
// if(devAreaMap){
// devAreaMap.each {k,v->
// if(obj.name && k && obj.name.contains(k)){
// d['M'] = v.area_info
// d['N'] = v.count
// d['O'] = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd")
// }
// }
//
// }else{
// println name+" 空值"
// }
}
// }
devAreaMap.each {k,v->
if(name.contains(k)){
d['M'] = v.area_info
d['N'] = v.count
d['O'] = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd")
}
}
})
}
@@ -101,7 +269,7 @@ class AddDevShapeData {
DEVELOPMENT_ZONE_43_COLL.save(obj)
}else{
udata.put("shape", resultMap)
udata.put("m_date", new Date())
// udata.put("m_date", new Date())
udata.put("bd_shape",bd_resultMap)
udata.put("operate_result" , "已处理")
udata.put("operator", "外包")
@@ -124,11 +292,11 @@ class AddDevShapeData {
def coordinates = features[0].geometry.coordinates
def index = 0
println name +" 有 "+coordinates.size()+" 块"
println file.toString()+" "+name +" 有 "+coordinates.size()+" 块"
areaMap.count = coordinates.size()
def areas = []
if(name.contains("泉州经济技术开发区")){
// if(name.contains("江苏常熟综合保税区")){
for(def pointList : coordinates){
index ++
def gaode_coordinates = []
@@ -171,7 +339,7 @@ class AddDevShapeData {
}
areaMap.area_info = areas.join("\n")
}
// }
}else{
println file +" 有多个"
@@ -4,43 +4,104 @@ import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import java.util.regex.Pattern
class AddrAlaisProcess {
DBCollection BAIDU_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206",27020)
def fullCityMap = CommonDatautils.queryFullNameCity()
static main(args) {
def task = new AddrAlaisProcess()
// task.run()
task.nineApi()
// task.createResult()
// task.mergeResult()
}
def mergeResult(){
def rets = ExcelUtils.read("D:\\数据\\数据建设\\小区\\ai-结果-0221.xlsx",0,1)
def retMap = [:]
for(def r : rets){
def num = r['B'].toString().toDouble().toInteger()
def list = retMap.get(num)
if(!list){
list = new HashSet()
retMap.put(num, list)
}
if(r['C']){
list << r['C']
}
}
def fileName = "D:\\数据\\数据建设\\小区\\小区别名-待定.xlsx"
ExcelUtils.modify(fileName,{d->
def num = d['A'].toString().toDouble().toInteger()
def aiList = retMap.get(num)
if(aiList){
d['K'] = aiList.join(",")
}
})
}
def createResult(){
def fileName = "D:\\数据\\数据建设\\小区\\小区别名-待定.xlsx"
def results = []
// results << ["ID", "城市", "地址"]
ExcelUtils.modify(fileName,{d->
def str = d['C']
def addressList = str.split(",")
for(def addr : addressList){
results << [d['A'].toString().toDouble().toInteger(), d['B'], addr]
}
})
println results.size()
FileUtils.batchCreateFileByCustomSuffix(results, "D:\\数据\\数据建设\\小区\\ai-待支行.bat", null, "utf-8", false, " ","");
}
def nineApi(){
def fileName = "/home/goojia/gj/district/addr_alias.xlsx"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\数据建设\\小区\\小区地址及地址别名标准化-测试.xlsx"
fileName = "D:\\数据\\数据建设\\写字楼\\写字楼待处理地址.xlsx"
}
ExcelUtils.multiThreadingModify(fileName, {d->
def addrMap = nineAddrSplit(d['B'], d['E'])
def obj = CommonDatautils.findDistrictById(d['A'])
// def addrMap = nineAddrSplit(obj.city, d['B'])
def addrList = []
if(addrMap && addrMap.address){
d['G'] = addrMap.city
d['H'] = addrMap.address
addrList << addrMap.address
}
if(d['F']){
def addrs = d['F'].split(",")
def addr_alias = []
for(def addr : addrs){
def map = nineAddrSplit(d['B'], addr)
if(map && map.address && DataHandleUtils.isStandardRoad(map.address)){
if(!addrList.contains(map.address)){
addrList << map.address
addr_alias << map.address
}
}
}
d['I'] = addr_alias.join(",")
}
// if(addrMap && addrMap.address){
// d['C'] = addrMap.road
// d['D'] = addrMap.road_num
// }
if(d['C']){
def data = fullCityMap.get(obj.city)
if(data){
def roadObj = findBaiduDatas( data.city, d['C'])
if(roadObj){
d['E'] = "百度存在"
}
}
}
// if(d['F']){
// def addrs = d['F'].split(",")
// def addr_alias = []
// for(def addr : addrs){
// def map = nineAddrSplit(d['B'], addr)
// if(map && map.address && DataHandleUtils.isStandardRoad(map.address)){
// if(!addrList.contains(map.address)){
// addrList << map.address
// addr_alias << map.address
// }
// }
// }
// d['I'] = addr_alias.join(",")
// }
})
}
@@ -67,9 +128,11 @@ class AddrAlaisProcess {
}
if (jsonObject.get("street") && jsonObject.get("street") != "null") {
address = jsonObject.get("street")
map.road = address
}
if (jsonObject.get("streetNumber") && jsonObject.get("streetNumber") != "null") {
address = address + jsonObject.get("streetNumber")
map.road_num = jsonObject.get("streetNumber")
}
map.address = address
}
@@ -95,4 +158,13 @@ class AddrAlaisProcess {
// ExcelUtils.write("D:\\数据\\数据建设\\小区\\小区地址及地址别名标准化.xlsx", rows, "小区")
FileUtils.batchCreateFileByCustomSuffix(rows, "D:\\数据\\数据建设\\小区\\小区地址及地址别名标准化.bat", null, "utf-8", false, "#","");
}
def findBaiduDatas(city, name){
def q = new BasicDBObject()
q.put("city", city)
q.put("name", name)
q.put('std_tag', Pattern.compile('^道路'))
def obj = BAIDU_POI_COLL.findOne(q)
return obj;
}
}
@@ -4,6 +4,7 @@ import com.fangjia.script.BaseBatch
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
@@ -30,7 +31,10 @@ class AutoAddDistrictBatch extends BaseBatch{
def mcaDistMap = CommonDatautils.queryDistNameCityByMapping()
static main(args) {
runBatch(new AutoAddDistrictBatch(), args)
// runBatch(new AutoAddDistrictBatch(), args)
def task = new AutoAddDistrictBatch();
task.checkMap()
}
@Override
@@ -52,7 +56,6 @@ class AutoAddDistrictBatch extends BaseBatch{
}
ExcelUtils.write(fileName, list, "待入库数据")
check(fileName)
}
@@ -159,10 +162,10 @@ class AutoAddDistrictBatch extends BaseBatch{
}
def findGaoDeDatas(city, region, name){
String sql = "select name,type from amap_poi_detail where cityname='${city}' and adname='${region}' and MATCH(name, '${name}')"
String sql = "select name,type from amap_poi_2024 where cityname='${city}' and adname='${region}' and MATCH(name, '${name}')"
def obj = null
ElasticUtils.executeSqlQuery(sql,{ d->
if(d.name == name && d.type.contains("住宅") ){ //&& d.shape
if(d.name == name){ //&& d.shape
obj = d
}
})
@@ -255,8 +258,37 @@ class AutoAddDistrictBatch extends BaseBatch{
q.put("city", city)
q.put("district", district)
q.put("name", name)
q.put('std_tag', '房地产;住宅区')
// q.put('std_tag', '房地产;住宅区')
def obj = BAIDU_POI_COLL.findOne(q)
return obj;
}
def checkMap(){
ExcelUtils.modify("D:\\数据\\数据建设\\写字楼\\写字楼存在性判断-test.xlsx",{d->
def gaode_obj = findGaodeCount(d['B'], d['C'], d['D'])
if(gaode_obj){
d['H'] = "是"
d['I'] = gaode_obj.type
}else{
def baidu_obj = findBaiduCount(d['B'], d['C'], d['D'])
if(baidu_obj){
d['E'] = "是"
d['F'] = baidu_obj.std_tag
}
}
// def baidu_obj1 = findBaiduCount(d['B'], d['G'], d['H'])
// if(baidu_obj1){
// d['M'] = "是"
// }
// def gaode_obj1 = findGaodeCount(d['B'], d['G'], d['H'])
// if(gaode_obj1){
// d['N'] = "是"
// }
// def score = CosineSimilarAlgorithm.getSimilarityRatio(d['D'], d['H']);
// d['O'] = score
})
}
}
@@ -4,10 +4,13 @@ import com.fangjia.core.RegionData
import com.fangjia.script.BaseBatch
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.DataUtil
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.GeoHelper
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CopyOnWriteArraySet
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
@@ -16,19 +19,18 @@ import java.util.concurrent.atomic.AtomicLong
class AutoAddOfficeBatch extends BaseBatch{
def fullCityMap = CommonDatautils.queryFullNameByCity()
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
static main(args) {
runBatch(new AutoAddOfficeBatch(), args)
// runBatch(new AutoAddOfficeBatch(), args)
def task = new AutoAddOfficeBatch()
task.repeatOffice()
}
@Override
void process(Object args) {
// def pattern = /[A-Z0-9]座|[0-9a-z]栋|[A-Z]栋|·|\(建设中\)|[0-9]号楼|\(装修中\)|[A-Z]楼|[0-9]幢/
// def newName = "开放广场".replaceAll(pattern, "")
// println newName
def citys = CommonDatautils.findAllCity()
def allAddOffices = []
allAddOffices << ["地图ID","省","市","区","写字楼名称","地址","经度","纬度","匹配城市","匹配区域","匹配名称","匹配方式"]
@@ -39,6 +41,7 @@ class AutoAddOfficeBatch extends BaseBatch{
officeMaps = filterOffice(officeMaps)
if(officeMaps){
def offices = matchOffice(officeMaps)
allAddOffices.addAll(offices)
}
}
@@ -171,4 +174,73 @@ class AutoAddOfficeBatch extends BaseBatch{
}
return null
}
def repeatOffice(){
def citys = CommonDatautils.findAllCity()
def allrets = []
allrets << ["ID","城市","区域","名称","地址","可疑ID","可疑城市","可疑区域","可疑名称","可疑地址","相似度","距离"]
for(def c : citys){
def offices = CommonDatautils.findAllDistricts(c, "office", ["city","region","name","address", "lat", "lng"])
def rets = checkRepeatOfficeBatch(offices)
println c +" "+offices.size()
allrets.addAll(rets)
}
ExcelUtils.write("D:\\数据\\写字楼\\重复可疑写字楼1.xlsx", allrets, "数据")
}
def checkRepeatOfficeBatch(offices){
def addCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(offices.size())
def total = offices.size()
def rets = new CopyOnWriteArrayList()
def sets = new CopyOnWriteArraySet()
for(def i=0;i<total;i++){
final def o1 = offices[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
addCount.addAndGet(1)
def maxObj = findOneSimilarOffice(o1, offices)
if(maxObj){
if(!sets.contains(o1._id.toString())){
sets.add(maxObj._id.toString())
rets << [o1._id.toString(), o1.city, o1.region, o1.name, o1.address, maxObj._id.toString(), maxObj.city, maxObj.region, maxObj.name, maxObj.address, maxObj.score, maxObj.distance]
}
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rets
}
def findOneSimilarOffice(o1, offices){
def maxScore = 0
def maxObj = null
for(def o2 : offices){
if(o1._id.toString()!=o2._id.toString()){
double nameScore = CosineSimilarAlgorithm.getSimilarity(o1.name, o2.name);
double addrScore = CosineSimilarAlgorithm.getSimilarity(o1.address,o2.address);
if(nameScore+addrScore>maxScore){
maxScore = nameScore+addrScore
maxObj = o2
}
}
}
def distance = -1000
if(o1.lat && maxObj && maxObj.lat){
distance = GeoHelper.GetDistance(o1.lat/10000000000.0, o1.lng/10000000000.0, maxObj.lat/10000000000.0, maxObj.lng/10000000000.0)
}
if(maxObj){
maxObj.distance = distance
maxObj.score = maxScore
return maxObj
}
return null
}
}
@@ -33,8 +33,8 @@ class DevUpdateTestBatch {
static main(args) {
def task = new DevUpdateTestBatch()
def path = "D:\\数据\\数据建设\\开发区\\开发区-园区.xlsx"//开发区-园区.xlsx
// task.addDev(path)
task.addDevAddress(path)
task.addDev(path)
// task.addDevAddress(path)
//
// task.updateHasGov("新疆维吾尔自治区","喀什地区","麦盖提县", "麦盖提镇", "是")
@@ -31,12 +31,12 @@ class MinglujiData {
}
def run(){
def prov = "河北省"
def city = "石家庄市"
// exportDatas(city)
def prov = "江苏省"
def city = "无锡"
exportDatas(city)
// matchDev(prov, city)
// matchCommunity(city)
fiveLevelSplit(city)
// fiveLevelSplit(city)
// nineLevelSplit(city)
// tempMatchDev(prov)
@@ -31,8 +31,8 @@ class RegionStandardBatch extends BaseBatch{
@Override
void process(Object args) {
// seaweedRegionStandard()
poiRegionStandard()
seaweedRegionStandard()
// poiRegionStandard()
}
def seaweedRegionStandard(){
@@ -64,7 +64,7 @@ class TownMiddleProcess {
//修改乡镇街道别名
// updateTownAlias()
//中间表同步到100
// synchTown()
synchTown()
//添加地级市标志
// addCityFlag()
// addPinyin()
@@ -95,7 +95,7 @@ class TownMiddleProcess {
// addTownCenter()//百度接口添加中心点
// checkCenterInTownRang()//检测乡镇街道中心点是否在围栏中
addSpecialCityTownAlias()
// addSpecialCityTownAlias()
}
@@ -451,16 +451,16 @@ class TownMiddleProcess {
for(def d : dbcursor){
datas << d
if (datas.size() >= 2000) {
// ADMINISTRATIVE_MAPPING_COLL_100.insert(datas);
ADMINISTRATIVE_MAPPING_43base_COLL.insert(datas)
ADMINISTRATIVE_MAPPING_COLL_100.insert(datas);
// ADMINISTRATIVE_MAPPING_43base_COLL.insert(datas)
println datas.size()
datas.clear();
}
}
if (datas.size() >= 0) {
// ADMINISTRATIVE_MAPPING_COLL_100.insert(datas);
ADMINISTRATIVE_MAPPING_43base_COLL.insert(datas)
ADMINISTRATIVE_MAPPING_COLL_100.insert(datas);
// ADMINISTRATIVE_MAPPING_43base_COLL.insert(datas)
datas.clear();
}
}
@@ -1,15 +1,21 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.core.RegionData
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.BaseBatch
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.xiamen.ElasticUtils89
import com.fangjia.script.lixue.util.DBUtils
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.util.BaiduMapUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import net.sf.json.JSONObject
import org.apache.commons.io.FileUtils
import org.bson.types.ObjectId
import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.bulk.BulkResponse
import org.elasticsearch.action.index.IndexRequest
@@ -52,14 +58,17 @@ class TownStandardBatch extends BaseBatch{
}
def TownStandard(){
// townStandardByCat("office")
townStandardByCat("office")
// townStandardByPoi()
addTownByTownShape()//为没有乡镇街道的poi,补充乡镇街道,忽略区域,并使用地图
// addTownByTownShape()//为没有乡镇街道的poi,补充乡镇街道,忽略区域,并使用地图
// standardTown()
// updateTownData()
}
def townStandardByCat(def cat){
//"三明","龙岩","南平","漳州","福州","厦门","泉州","莆田","宁德",
def citys = ["重庆","龙岩","南平","漳州","福州","厦门","泉州","莆田","三明","宁德"]//CommonDatautils.findAllCity()
def citys = ["天津"]//CommonDatautils.findAllCity()
for(def city : citys){
def data = fullCityMap.get(city)
@@ -70,10 +79,10 @@ class TownStandardBatch extends BaseBatch{
def mzbRegionMap = CommonDatautils.queryMzbRegion(fullCityName)
def shortRegionNameMap = CommonDatautils.getRegionShortName(fullCityName)
def townMap = CommonDatautils.queryTownByCity(fullCityName, ["city","district", "street", "alias"], mzbRegionMap)
def townMap = CommonDatautils.queryTownByCity(fullCityName, ["city","district", "street", "alias","shape"], mzbRegionMap)
if(townMap){
def regionSet = townMap.keySet()
if(regionSet.size()>1){
if(regionSet.size()>=1){
haveRegionProcess(city, fullCityName, townMap, cat, shortRegionNameMap)
}else{
println city +" 无区域"
@@ -139,11 +148,12 @@ class TownStandardBatch extends BaseBatch{
def haveRegionProcess(city, fullCityName, townMap, cat, shortRegionNameMap){
def results = []
if(fullCityName){
def districts = CommonDatautils.findAllDistricts(city, cat, ["city","region","name", "township"])
def districts = CommonDatautils.findAllDistricts(city, cat, ["city","region","name", "township","lng","lat"])
for(def d : districts){
if(d.township){
def towns = null
def updateTown = null
def towns = null
def updateTown = null
if(d.region){
for(def m : townMap){
def fullRegionName = shortRegionNameMap.get(d.region)
if(m.key == fullRegionName){
@@ -151,41 +161,51 @@ class TownStandardBatch extends BaseBatch{
break
}
}
}
if(!towns){
towns = townMap.get(fullCityName)
}
def opt = true
if(towns){
for(def town : towns){
def townAlias = []
if(town.alias){
townAlias = town.alias
}
townAlias << town.street
if(townAlias.contains(d.township)){
if(d.township!=town.street){
updateTown = town.street
}else{
opt = false
}
break
}
def opt = true
if(towns){
for(def town : towns){
def townAlias = []
if(town.alias){
townAlias = town.alias
}
}
if(opt){
if(updateTown){
def udata = new BasicDBObject()
udata.put("township", updateTown)
udata.put("m_date", new Date())
println d.city +" "+d.region+" "+d.name+" "+d.township +" 乡镇修改为:"+updateTown
// DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
townAlias << town.street
if(townAlias.contains(d.township)){
if(d.township!=town.street){
updateTown = town.street
}else{
opt = false
}
break
}else{
def udata = new BasicDBObject()
udata.put("township", "")
udata.put("m_date", new Date())
println d.city +" "+d.region+" "+d.name+" "+d.township +" 乡镇修改为:空值"
// DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
if(town.shape){
def flag = InsidePolygon.isInArea(d.lng/10000000000.0, d.lat/10000000000.0, InsidePolygon.createPoint4(town.shape.coordinates))
if(flag){
updateTown = town.street
break
}
}
}
}
}
if(opt){
if(updateTown){
def udata = new BasicDBObject()
udata.put("township", updateTown)
udata.put("m_date", new Date())
println d.city +" "+d.region+" "+d.name+" "+d.township +" 乡镇修改为:"+updateTown
DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
}else{
def udata = new BasicDBObject()
udata.put("township", "")
udata.put("m_date", new Date())
println d.city +" "+d.region+" "+d.name+" "+d.township +" 乡镇修改为:空值"
DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
}
}
}
}
@@ -365,6 +385,33 @@ class TownStandardBatch extends BaseBatch{
return rets;
}
def readCSV3(filename){
def lines = FileUtils.readLines(new File(filename))
def rets = [];
def index = 0
for(line in lines) {
if(line && index>=1){
String[] arrs = line.split(",")
if(arrs.size()>=11 && arrs[10]=="是" && arrs[6]!=arrs[7]){
def o = new BasicDBObject()
try{
o.put("id", arrs[0])
o.put("township", arrs[7])
o.put("city", arrs[2])
o.put("old_district", arrs[3])
o.put("new_district", arrs[8])
rets << o
}catch(Exception e){
println arrs
}
}
}
index++
}
return rets;
}
def readCSV2(filename){
def lines = FileUtils.readLines(new File(filename))
def rets = [];
@@ -380,10 +427,11 @@ class TownStandardBatch extends BaseBatch{
o.put("district", arrs[4])
o.put("lng", arrs[5])
o.put("lat", arrs[6])
if(arrs.size()<=7){
o.put("data_source","")
}else{
o.put("data_source", arrs[7])
o.put("data_source", arrs[7])
if(arrs.size()>=9){
o.put("township", arrs[8])
o.put("new_district", arrs[9])
o.put("town_source", arrs[10])
}
rets << o
}catch(Exception e){
@@ -395,43 +443,13 @@ class TownStandardBatch extends BaseBatch{
return rets;
}
def readCSV3(filename){
def lines = FileUtils.readLines(new File(filename))
def rets = [];
def index = 0
for(line in lines) {
if(line && index>=1){
String[] arrs = line.split(",")
if(arrs.size()>=10 && arrs[7] && arrs[9]=="同区域"){
def o = new BasicDBObject()
try{
o.put("id", arrs[0])
o.put("township", arrs[7])
if(arrs.size()>=9 && arrs[8]){
o.put("district", arrs[8])
}
if(arrs.size()>=10 && arrs[9]){
o.put("way", arrs[9])
}
rets << o
}catch(Exception e){
println arrs
}
}
}
index++
}
return rets;
}
def addTownByTownShape(){
def i = 0
def headers = ["_id","province","city","district","lng","lat","data_source","乡镇街道", "新区域", "方式", "标准乡镇"]
while(i<50){
while(i<=50){
def fileName = "/home/goojia/gj/new_poi/input/map/no_township_${i}.csv"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\数据建设\\乡镇街道\\待补充乡镇街道\\原始\\no_township_${i}.csv"
fileName = "D:\\数据\\数据建设\\乡镇街道\\no_township_${i}.csv"
}
File file=new File(fileName);
if(file.exists()) {
@@ -451,19 +469,53 @@ class TownStandardBatch extends BaseBatch{
}
}
def standardTown(){
def i = 49
def headers = ["_id","province","city","district","lng","lat","data_source","乡镇街道", "新区域", "方式", "标准乡镇"]
def cityTownMap = CommonDatautils.queryAllTown(["city", "district", "mzb_district_name", "street", "alias"])
while(i<=50){
def fileName = "/home/goojia/gj/new_poi/input/map/no_township_result_${i}.csv"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\数据建设\\乡镇街道\\待补充乡镇街道\\结果-地图\\no_township_result_${i}.csv"
}
File file=new File(fileName);
if(file.exists()) {
def rets = readCSV2(fileName)
println i +" "+ rets.size()
def rows = addTownshipBatch2(rets, cityTownMap)
if (CommonDatautils.isWindow()) {
com.fangjia.script.gj.ccb.FileUtils.batchCreateFileByCustomSuffix(rows, "D:\\数据\\数据建设\\乡镇街道\\待补充乡镇街道\\结果-地图-验证乡镇\\no_township_result_${i}.csv", headers, "utf-8", false, ',', '');
}else{
com.fangjia.script.gj.ccb.FileUtils.batchCreateFileByCustomSuffix(rows, "/home/goojia/gj/new_poi/output/no_township_result_${i}.csv", headers, "utf-8", false, ',', '');
}
println fileName +" 运行完成"
}else{
println fileName +" 不存在"
}
i++
}
}
def addTownshipBatch(def rets){
def results = new CopyOnWriteArrayList()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(rets.size())
for(def i=0;i<rets.size();i++){
final def r = rets[i]
fixedThreadPool.execute(new Runnable() {
void run() {
// fixedThreadPool.execute(new Runnable() {
// void run() {
try{
totalCount.addAndGet(1)
def rMap = regionFullNameMap.get(r.city)
if(!rMap){
def data = fullCityMap.get(r.city)
rMap = regionFullNameMap.get(data.city)
}
if (rMap) {
def townMap = [:]//CommonDatautils.queryTownByCity(r.city, ["city", "district", "mzb_district_name", "street", "alias", "shape"])
// def townMap = CommonDatautils.queryTownByCity(r.city, ["city", "district", "mzb_district_name", "street", "alias", "shape"])
// def set = rMap.keySet()
println totalCount.get() +" "+r.city + " "
// if (set.size() >= 1) {
@@ -476,6 +528,46 @@ class TownStandardBatch extends BaseBatch{
} finally {
latch.countDown()
}
// }
// });
}
latch.await()
return results
}
def addTownshipBatch2(def rets, def cityTownMap){
def results = new CopyOnWriteArrayList()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(rets.size())
for(def i=0;i<rets.size();i++){
final def r = rets[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def rMap = regionFullNameMap.get(r.city)
def fullcity = null
if(!rMap){
def data = fullCityMap.get(r.city)
rMap = regionFullNameMap.get(data.city)
fullcity = data.city
}
if (rMap) {
def townMap = cityTownMap.get(fullcity)
def set = rMap.keySet()
if (set.size() >= 1) {
if(townMap.size()>=1){
r.strand_town = isStandTown(townMap, r.new_district, r.township)
}
results << [r.id, r.province, r.city, r.district, r.lng, r.lat, r.data_source, r.township, r.new_district, r.town_source, r.strand_town]
}
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
@@ -610,20 +702,21 @@ class TownStandardBatch extends BaseBatch{
}
def updateTownData(){
def i = 0
def i = 49
while(i<50) {
def fileName = "/home/goojia/gj/new_poi/output/no_township_result_${i}.csv"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\数据建设\\乡镇街道\\待补充乡镇街道\\结果\\no_township_result_${i}.csv"
fileName = "D:\\数据\\数据建设\\乡镇街道\\待补充乡镇街道\\结果-地图-验证乡镇\\no_township_result_${i}.csv"
}
File file = new File(fileName);
if (file.exists()) {
def rets = readCSV3(fileName)
println file.getAbsoluteFile().getName() +" "+(rets.size()*100/200000)+"%"+" "+rets.size()
// if(rets){
// updateToEs(rets)
// }
if(rets){
// updateToEs(rets)//不能用这种方法修改,否则其它字段都将丢失
// updateToEs2(rets)
updateDistTownship(rets)
}
}
i++
}
@@ -677,4 +770,72 @@ class TownStandardBatch extends BaseBatch{
println "结束时间 "+new Date()
}
def updateToEs2(rows) {
for (int i = 0; i < rows.size(); i++) {
def map = rows[i]
updateMapPoi(map.id, map.township)
}
}
def updateMapPoi(def id, def township){
def header = [:]
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString()
String seawaterUrl = "http://192.168.0.89:9200/map_poi_2024/_update_by_query?conflicts=proceed&scroll_size=1000&wait_for_completion=false"
def body = """
{
"script": {
"source": "ctx._source.township='${township}'",
"lang": "painless"
},
"query": {
"term": {
"_id": "${id}"
}
}
}
"""
HttpRequest request = new HttpRequest()
try{
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
def response = JSONObject.fromObject(res)
}catch(Exception e){
println "error : "+e.getMessage()
}
}
def updateDistTownship(rets){
for(def m : rets){
def obj = CommonDatautils.findDistrictById(m.id)
if(obj.township != m.township){
def udata = new BasicDBObject()
udata.put("township", m.township)
if(!m.new_district.contains(m.old_district)){
def lists = CommonDatautils.findAllDistrictByName(obj.city, null, obj.name, obj.cat)
if(lists.size()<2){
continue
}
def region = RegionData.formatRegion(m.city, m.new_district)
if(region){
def dbo = CommonDatautils.findDistrict(m.city, region, obj.name, obj.cat)
if(dbo){
udata.put("status", 1)
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(m.id)), new BasicDBObject('$set', udata))
println "删除 "+m.city +" "+obj.name+" "+m.id+" "+obj.cat
continue
}else{
udata.put("region", region)
}
}
}
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(m.id)), new BasicDBObject('$set', udata))
}
}
}
}
@@ -1,6 +1,7 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.core.RegionData
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.util.CosineSimilarAlgorithm
@@ -11,6 +12,7 @@ import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.impl.AMQImpl
import net.sf.json.JSONObject
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
@@ -38,8 +40,8 @@ class TownshipStrandUpdate {
static main(args) {
def task = new TownshipStrandUpdate()
task.process()
task.updateTown()
// task.process()
// task.updateTown()
// task.stat()
// task.updateDiffRegionTown()
@@ -49,6 +51,9 @@ class TownshipStrandUpdate {
// task.updateDevMapping()
// println new Date().getTime()
// task.updateSpecCityTown()
task.updateTownshipAlias()
}
@@ -629,4 +634,69 @@ class TownshipStrandUpdate {
}
}
def updateTownshipAlias(){
def fileName = "D:\\数据\\数据建设\\乡镇街道\\20250219--乡镇街道添加别名.xlsx"
ExcelUtils.modify(fileName,{d->
def q = new BasicDBObject("type","乡镇街道").append("city", d["A"]).append("district",d["B"]).append("street",d["C"])
def list = ADMINISTRATIVE_MAPPING_COLL.find(q).toList()
def newAlias = d['D']
if(list.size()==0){
list = q.put("street", d['D'])
newAlias = d['C']
list = ADMINISTRATIVE_MAPPING_COLL.find(q).toList()
}
if(list && list.size()==1){
def obj = list[0]
def alias = obj.alias
if(!alias){
alias = []
}
if(!alias.contains(newAlias)){
alias << newAlias
}
if(alias){
def udata = new BasicDBObject()
udata.put("alias", alias)
udata.put("m_date", new Date())
udata.put("operator","陈杰")
ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',udata))
}
// updateStreetAlias(obj._id.toString(), alias.join("','"))
}else{
println d["C"] +" "+ d["D"]
}
})
}
def updateStreetAlias(def id, def alias){
def header = [:]
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString()
String seawaterUrl = "http://192.168.0.89:9200/administrative_mapping/_update_by_query?conflicts=proceed&scroll_size=1000&wait_for_completion=false"
def body = """
{
"script": {
"source": "ctx._source.alias=['${alias}']",
"lang": "painless"
},
"query": {
"term": {
"_id": "${id}"
}
}
}
"""
HttpRequest request = new HttpRequest()
try{
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
def response = JSONObject.fromObject(res)
}catch(Exception e){
println "error : "+e.getMessage()
}
}
}
@@ -336,7 +336,7 @@ class FangjiaClientUtils {
try {
address = address.replace("{","")
// def url = "http://192.168.0.19:31080/FangJiaNlpService/formatFivelevel?address="+URLEncoder.encode(address,"UTF-8")
def url = "http://192.168.0.93:8080/FangJiaNlpService/formatFivelevel?address="+URLEncoder.encode(address,"UTF-8")
def url = "http://192.168.0.93:8080/FangJiaNlpService/formatFivelevel?address="+URLEncoder.encode(address,"UTF-8")+"&complete=false&useMap=true&addressAssessment=false"
if(city){
url = url +"&city="+URLEncoder.encode(city,"UTF-8")
}
@@ -354,7 +354,7 @@ class FangjiaClientUtils {
def jsonObj = null
def content = null
def url = null
address = address.replace("{","")
address = address.toString().replace("{","")
int retryCount = 0
while (retryCount < 3) {
url = "http://192.168.0.93:8080/FangJiaNlpService/formatNinelevel?address="+URLEncoder.encode(address,"UTF-8")+"&complete=false&useMap=false&addressAssessment=true&fields=dev_confi"
@@ -74,11 +74,11 @@ class OfficeFieldCleanBatch extends BaseBatch{
// addBlock()
// addFieldValue()
// updateErrorField()
updateErrorField()
// export()
// test()
// addOffice()
fileAddBlock()
// fileAddBlock()
}
def export(){
@@ -134,33 +134,196 @@ class OfficeFieldCleanBatch extends BaseBatch{
}
def updateErrorField(){
// def offices = CommonDatautils.findAllDistrict("office", ["city","estate_charge","addr_alias","property_life", "total_floor","finish_building_date","real_use_percent", "land_car_site", "underground_car_site","parking_fee"])
def offices = CommonDatautils.findAllDistricts("ÉϺ£","office", ["city","estate_charge","addr_alias","property_life", "total_floor","finish_building_date","real_use_percent", "land_car_site", "underground_car_site","parking_fee"])
def offices = CommonDatautils.findAllDistrict("office", ["underground_car_site"])
// def offices = CommonDatautils.findAllDistricts("上海","office", ["build_area"])
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(offices.size())
for(def i=0;i<offices.size();i++){
final def o = offices[i]
fixedThreadPool.execute(new Runnable() {
void run() {
if(o._id.toString()=="5954d66f9aed7b45d3785a40"){
println o._id.toString()
}
// fixedThreadPool.execute(new Runnable() {
// void run() {
try{
totalCount.addAndGet(1)
updateDistField(o)
println totalCount.get() +" "+o.city
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
// }
// });
}
latch.await()
}
def updateDistField(o){
def addr_alias = o.addr_alias
def udata = new BasicDBObject()
if(o.build_area){
def newVal = FiledStrand.processArea(o.build_area)
if(newVal && newVal.toString()!=o.build_area){
println o._id.toString()+" old:"+o.build_area +" new:"+newVal
udata.put("build_area", newVal.toString())
}
}
if(o.all_area){
def newVal = FiledStrand.processArea(o.all_area)
if(newVal.toString()!=o.all_area){
println o._id.toString()+" old:"+o.all_area +" new:"+newVal
udata.put("all_area", newVal.toString())
}
}
if(o.standard_area){
def newVal = FiledStrand.processStandardArea(o.standard_area)
if(newVal.toString()!=o.standard_area){
println o._id.toString()+" old:"+o.standard_area +" new:"+newVal
udata.put("standard_area", newVal.toString())
}
}
if(o.standard_height){
def newVal = FiledStrand.processStandardHeight(o.standard_height)
if(newVal.toString()!=o.standard_height){
println o._id.toString()+" old:"+o.standard_height +" new:"+newVal
udata.put("standard_height", newVal.toString())
}
}
if(o.clear_height){
def newVal = FiledStrand.processStandardHeight(o.clear_height)
if(newVal.toString()!=o.clear_height){
println o._id.toString()+" old:"+o.clear_height +" new:"+newVal
if(newVal){
udata.put("clear_height", newVal.toString())
}else{
udata.put("clear_height", "")
}
}
}
if(o.lobby_height){
def newVal = FiledStrand.processLobbyHeight(o.lobby_height)
if(newVal.toString()!=o.lobby_height){
println o._id.toString()+" old:"+o.lobby_height +" new:"+newVal
if(newVal){
udata.put("lobby_height", newVal.toString())
}else{
udata.put("lobby_height", "")
}
}
}
if(o.cubage_percent){
def newVal = FiledStrand.processCubagePercent(o.cubage_percent)
if(newVal && newVal.toString()!=o.cubage_percent){
println o._id.toString()+" old:"+o.cubage_percent +" new:"+newVal
udata.put("cubage_percent", newVal.toString())
}
}
if(o.virescence_percent){
def newVal = FiledStrand.processVirescencePercent(o.virescence_percent)
if(newVal.toString()!=o.virescence_percent){
println o._id.toString()+" old:"+o.virescence_percent +" new:"+newVal
udata.put("virescence_percent", newVal.toString())
}
}
if(o.elevators_count){
def newVal1 = FiledStrand.processElevators(o.elevators_count)
if(newVal1.toString()!=o.elevators_count){
println o._id.toString()+" old:"+o.elevators_count +" new:"+newVal1
udata.put("elevators_count", newVal1.toString())
}
def newVal2 = null
if(o.elevators_count.contains("客梯")){
newVal2 = FiledStrand.processElevators(o.elevators_count.toString().substring(o.elevators_count.indexOf("客梯")))
if(newVal2.toString()!=o.elevators_count){
println o._id.toString()+" old:"+o.elevators_count +" new:"+newVal2
udata.put("person_elevators_count", newVal2.toString())
}
}
def newVal3 = null
if(o.elevators_count.contains("货梯")){
newVal3 = FiledStrand.processElevators(o.elevators_count.toString().substring(o.elevators_count.indexOf("货梯")))
if(newVal3.toString()!=o.elevators_count){
println o._id.toString()+" old:"+o.elevators_count +" new:"+newVal3
udata.put("goods_elevator_count", newVal3.toString())
}
}
if(newVal2 && newVal3){
def total = newVal2.toInteger() + newVal3.toInteger()
udata.put("elevators_count", total.toString())
println o._id.toString()+" 总电梯:"+total
}
}
if(o.goods_elevator_count){
def newVal = FiledStrand.processElevators(o.goods_elevator_count)
if(newVal.toString()!=o.goods_elevator_count){
println o._id.toString()+" old:"+o.goods_elevator_count +" new:"+newVal
udata.put("goods_elevator_count", newVal.toString())
}
}
if(o.person_elevators_count){
def newVal = FiledStrand.processElevators(o.person_elevators_count)
if(newVal.toString()!=o.person_elevators_count){
println o._id.toString()+" old:"+o.person_elevators_count +" new:"+newVal
if(newVal){
udata.put("person_elevators_count", newVal.toString())
}else{
udata.put("person_elevators_count", "")
}
}
}
if(o.land_car_site){
def newVal = FiledStrand.processCarSite(o.land_car_site)
if(newVal.toString()!=o.land_car_site){
println o._id.toString()+" old:"+o.land_car_site +" new:"+newVal
if(newVal){
udata.put("land_car_site", newVal.toString())
}else{
udata.put("land_car_site", "")
}
}
}
if(o.underground_car_site){
def newVal = FiledStrand.processCarSite(o.underground_car_site)
if(newVal.toString()!=o.underground_car_site){
println o._id.toString()+" old:"+o.underground_car_site +" new:"+newVal
if(newVal){
udata.put("underground_car_site", newVal.toString())
}else{
udata.put("underground_car_site", "")
}
}
}
if(o.is_register){
def newVal = FiledStrand.processRegister(o.is_register)
if(newVal.toString()!=o.is_register){
println o._id.toString()+" old:"+o.is_register +" new:"+newVal
udata.put("is_register", newVal.toString())
}
}
if(o.property_life){
def newVal = FiledStrand.processPropertyLife(o.property_life)
udata.put("property_life", newVal)
@@ -173,7 +336,10 @@ class OfficeFieldCleanBatch extends BaseBatch{
if(o.finish_building_date){
def newVal = FiledStrand.processFinishBuildingDate(o.finish_building_date)
udata.put("finish_building_date", newVal.toString())
if(newVal && newVal.toString()!=o.finish_building_date){
println o._id.toString()+" old:"+o.finish_building_date +" new:"+newVal
udata.put("finish_building_date", newVal.toString())
}
}
if(o.real_use_percent){
@@ -181,21 +347,14 @@ class OfficeFieldCleanBatch extends BaseBatch{
udata.put("real_use_percent", newVal.toString())
}
if(o.underground_car_site){
def newVal = FiledStrand.processCarSite(o.underground_car_site)
udata.put("underground_car_site", newVal.toString())
}
if(o.land_car_site){
def newVal = FiledStrand.processCarSite(o.land_car_site)
udata.put("land_car_site", newVal.toString())
}
if(o.parking_fee){
def newVal = FiledStrand.processCarSiteRent(o.parking_fee)
udata.put("parking_fee", newVal)
}
def addr_alias = o.addr_alias
if(addr_alias){
def addrs = []
for(def arr : addr_alias){
@@ -214,13 +373,7 @@ class OfficeFieldCleanBatch extends BaseBatch{
}
}
// if(o.estate_charge && !o.estate_charge.isNumber()){
//// def val = o.estate_charge.toString().toDouble()
//// if(val>60 || val < 0.35){
//// println val
// udata.put("estate_charge", null)
//// }
// }
if(udata){
udata.put("m_date", new Date())
@@ -27,7 +27,7 @@ class EsMonthHouseStat {
// task.statDistHouse()
// task.statRent()
task.statOfficeHouse()
// task.statOfficeHouse()
}
def statRent(){
@@ -40,16 +40,58 @@ class AddPriceTest {
def run(){
def filePath = "D:\\数据\\价格\\惠州.xlsx"
// addGXDPrice(filePath)
addCcbPrice(filePath)
// addCcbPrice(filePath)
// addThirdPrice(filePath)
// addFjwPrice(filePath)
// exportPkPrice()
// addRegion()
// addTradePrice(filePath)
addThirdSamplePrice("D:\\数据\\建行\\房产估值\\指导价\\级联、国信达价格导入_0313.xlsx")
}
def addThirdSamplePrice(filePath){
ExcelUtils.modify(filePath,{d->
def dist = CommonDatautils.findDistrictById(d['A'])
def o = new BasicDBObject()
o.put("city" , dist.city)
o.put("region" , dist.region)
o.put("district_name" , dist.name)
o.put("isGuidancePrice" , 0)
o.put("category", "district")
o.put("c_date" , new Date())
o.put("user_name" , "房价网")
if(d['B']){
o.put("price", d['B'].toString().toDouble().toInteger())
o.put("source", "JL")
if(o.containsKey("_id")){
o.removeField("_id")
}
confirm_price_date.save(o)
}
if(d['C']){
o.put("price", d['C'].toString().toDouble().toInteger())
o.put("source", "GXD")
if(o.containsKey("_id")){
o.removeField("_id")
}
confirm_price_date.save(o)
}
if(d['D']){
o.put("price", d['D'].toString().toDouble().toInteger())
o.put("source", "FXT")
if(o.containsKey("_id")){
o.removeField("_id")
}
confirm_price_date.save(o)
}
})
}
def addTradePrice(filePath){
ExcelUtils.multiThreadingModify(filePath, { d ->
if(d['Y']){
@@ -19,7 +19,7 @@ DBCollection COMMUNITY_CONN = MongoUtils.getCollectionByName2("fangjia", "commun
//def s = getTownshipInfo("118.702988,39.740979")
def filePath = "D:\\数据\\涉农\\县政府(最终).xlsx"
ExcelUtils.modify(filePath, { row ->
if(!row['I'] && row['I']=='±¦Ó¦ÏØ'){
if(!row['I'] && row['I']=='建瓯市'){
def rets = EsSearchUtils.findPois(row['B'], row['C'], row['C']+"人民政府")
if(rets){
def obj = findOneByName(rets, row['C']+"人民政府")
@@ -1,18 +1,23 @@
package com.fangjia.script.gj.xiamen
import com.fangjia.core.util.GeoUtils
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.BaseBatch
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.ProvinceData
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.AMQP
import net.sf.json.JSONObject
import org.bson.types.ObjectId
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
@@ -23,7 +28,7 @@ import com.fangjia.script.common.DataHandleUtils
class SeaweedDataSynch extends BaseBatch{
static def targetUrl = "http://192.168.0.12:9200/seaweed/"
static def targetUrl = "http://192.168.0.89:9200/seaweed/"
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
@@ -132,17 +137,44 @@ class SeaweedDataSynch extends BaseBatch{
mcaDistMap = CommonDatautils.queryDistNameCityByMapping()
// priceMap = findAllDistrictSellPrice()
def citys = CommonDatautils.findAllCity().reverse()
def citys = ["³±ÖÝ"]//CommonDatautils.findAllCity()
def index = 0
for(def c : citys){
index++
updating(c, "district")
def dists = updating(c, "district")
updateEs(c, dists)
updating(c, "office")
updating(c, "shop")
println index +" "+ c + " 完成"
}
}
def updateEs(city, dists){
def data = fullCityMap.get(city)
def distEsDatas = findEsSeaweedDatas(data.city)
def resultMap = [:]
for(def d : dists){
def esObj = distEsDatas.get(d._id.toString())
def map = [:]
d.each {k,v->
if(k!="_id"){
def esVal = esObj.get(k)
if(v!=esVal){
map.put(k, v)
}
}
}
if(map){
resultMap.put(d._id.toString(), map)
}
}
return resultMap
}
def updating(city, cat){
def dists = CommonDatautils.findAllDistrictsFromMg(city, cat, ["city","region","name","address","block","mapping_alias","addr_alias","cat","estate_type2","status","township","lng","lat","build_type","household_count","committee","estate_charge","car_site_count","property_life","house_property","build_area","all_area","build_structure","developer","estate_company","cubage_percent","virescence_percent","loudong_count","schools","main_type","main_type_area","create_year"])
def rets = new CopyOnWriteArrayList()
@@ -154,15 +186,13 @@ class SeaweedDataSynch extends BaseBatch{
void run() {
try{
totalCount.addAndGet(1)
def map = createData(dist)
//往100 上同步
updateData(map)
//Íù12 esÉÏͬ²½
// def newUrl = targetUrl +"_doc/"+map.id
// CommonDatautils.httpSaveData(newUrl, map)
rets << map
// //Íù89 esÉÏͬ²½
// def newUrl = targetUrl +"_doc/"+map._id
// CommonDatautils.httpSaveData89(newUrl, map)
println city +" "+cat +" "+totalCount.get() +"/"+dists.size()+" "+dist.name+" "+map._id
} catch (Exception e) {
@@ -178,6 +208,33 @@ class SeaweedDataSynch extends BaseBatch{
return rets;
}
def updateMapPoi(def id, def township){
def header = [:]
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString()
String seawaterUrl = "http://192.168.0.89:9200/seaweed/_update_by_query?conflicts=proceed&scroll_size=1000&wait_for_completion=false"
def body = """
{
"script": {
"source": "ctx._source.township='${township}'",
"lang": "painless"
},
"query": {
"term": {
"_id": "${id}"
}
}
}
"""
HttpRequest request = new HttpRequest()
try{
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
def response = JSONObject.fromObject(res)
}catch(Exception e){
println "error : "+e.getMessage()
}
}
def updateData(map){
def udata = new BasicDBObject()
map.each{k,v->
@@ -196,14 +253,14 @@ class SeaweedDataSynch extends BaseBatch{
if(province){
map.province = ProvinceData.getFullProvince(province)
}
map.fangjia_province = province
// map.fangjia_province = province
def data = fullCityMap.get(dist.city)
if(data){
map.city = data.city
area_code = data.mca_city_code
}
map.fangjia_city = dist.city
// map.fangjia_city = dist.city
def distData = mcaDistMap.get(dist.city+"#"+dist.region)
if(distData){
@@ -213,9 +270,8 @@ class SeaweedDataSynch extends BaseBatch{
// map.region = dist.region
println dist.city +" "+dist.region+" 民政部中没有找到相应的区域全称"
}
map.area_code = area_code
map.fangjia_region = dist.region
// map.area_code = area_code
// map.fangjia_region = dist.region
map.name = dist.name
@@ -390,6 +446,14 @@ class SeaweedDataSynch extends BaseBatch{
DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(row['A'])), new BasicDBObject('$set', udata))
}
})
}
def findEsSeaweedDatas(city){
def map = new HashMap()
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city))
ElasticUtils89.executeQuery("192.168.0.89", "seaweed", queryBuilder, { String id, doc->
map.put(id, doc)
})
return map;
}
}