Merge branch 'script_gj' into 'master'

Script gj

See merge request !1101
This commit is contained in:
2025-04-07 13:22:40 +08:00
25 changed files with 1356 additions and 607 deletions
@@ -8,7 +8,7 @@ import com.mongodb.DBCollection
/**
* 深圳公众号成交数据录入
*/
def datas = ExcelUtils.read("D:\\数据\\2025年02月深圳二手房成交.xlsx", 0, 1)
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\2025年01月深圳二手房成交的.xlsx", 0, 1)
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
@@ -1353,13 +1353,14 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){
//},0)
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20241025人工指导价.xlsx"
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\更新人工指导价_0314.xlsx"
ExcelUtils.multiThreadingModify(fileName, { d ->
def dist = CommonDatautils.findDistrictById(d['J'])
def dist = CommonDatautils.findDistrictById(d['A'])
if(dist){
d['A'] = dist.city
d['B'] = dist.region
d['C'] = dist.name
d['C'] = dist.city
d['D'] = dist.region
d['E'] = dist.name
d['F'] = dist.status
}
})
@@ -262,7 +262,7 @@ class CommonDatautils {
q.put("status", 0);
q.put("cat", cat);
q.put("city", city);
// q.put("name","托斯卡纳庄园")
// q.put("name","新泾三村")
DBObject keys = new BasicDBObject();
if(fields){
@@ -821,7 +821,9 @@ class CommonDatautils {
def dbsour = SEAWEED_COLL.find(q, keys).batchSize(5000)
def rets = []
for(def d : dbsour){
rets << d
if(!d.township){
rets << d
}
}
return rets;
}
@@ -4717,6 +4719,14 @@ class CommonDatautils {
}
static def httpSaveData89(def newUrl,def params){
def header = [:]
header.put("Authorization","Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString())
doPost(newUrl,header, params, true, "utf-8")
}
static def doPost(def targetUrl, def headers, def params, def isJson, def charset){
StringBuilder buffer = new StringBuilder();
try{
@@ -5094,11 +5104,10 @@ class CommonDatautils {
def keys = new BasicDBObject()
keys.put("street",1)
keys.put("alias",1)
keys.put("shape",1)
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def retMap = [:]
for(def d : dbcursor){
retMap.put(d.street, d)
retMap.put(d.street, d.alias)
}
return retMap
@@ -1,5 +1,6 @@
package com.fangjia.script.gj
import com.fangjia.script.common.DataHandleUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection;
@@ -12,12 +13,13 @@ import com.fangjia.script.util.ExcelUtils;
class FangjiatongUserStat {
public static final DBCollection FJT_USER_COLL = MongoUtils.getCollectionByName("fangjiatong_log", "utm_gif_log", "222.73.242.42", 52002)
static final def onLineSqlInstance = Sql.newInstance("jdbc:mysql://222.73.242.42:4306/goojia_fangjia?&characterEncoding=utf8", "sa", "G00jia", "com.mysql.jdbc.Driver");
static final def onLineSqlInstance = Sql.newInstance("jdbc:mysql://mysql.db.fangjia.com:4306/goojia_fangjia?&characterEncoding=utf8", "sa", "G00jia", "com.mysql.jdbc.Driver");
public static void main(String[] args) {
def task = new FangjiatongUserStat();
task.run();
// task.run();
task.stat()
}
def run(){
@@ -60,4 +62,59 @@ class FangjiatongUserStat {
return ret;
}
def stat(){
def users = findUses(12)
def months = ["2019-01-01","2019-02-01","2019-03-01","2019-04-01","2019-05-01","2019-06-01","2019-07-01","2019-08-01","2019-09-01","2019-10-01","2019-11-01","2019-12-01","2020-01-01"]
def rets = []
def header = ["用户id","支行名称", "备注名称"]
months.each{m->
header << m.substring(0, 7)
}
rets << header
for(def m : users){
def ret = []
ret << m.user_id
ret << m.name
ret << m.org_name
for(def i=1;i<months.size();i++){
def evalCount = findEvaluationCount(m.user_id, months[i-1], months[i])
ret << evalCount
}
for(def i=1;i<months.size();i++){
def downCount = findTradesCount(m.user_id, months[i-1], months[i])
ret << downCount
}
rets << ret
}
ExcelUtils.write("D:\\数据\\农行统计.xlsx", rets, "农行统计-2019")
}
def findUses(role){
def sql = "select u.id,ud.name,ud.org_name from (SELECT * from users WHERE role=12) u left join users_detail ud on u.id=ud.id ";
def rets = [];
onLineSqlInstance.eachRow(sql, {
rets << ["user_id":it.id, "name":it.name, "org_name":it.org_name]
})
return rets
}
def findEvaluationCount(user_id, startDate, endDate){
def sql = "select count(*) as count from evaluation where user_id=${user_id} and DATE_CREATED>='${startDate}' and DATE_CREATED<'${endDate}'";
def count = 0;
onLineSqlInstance.eachRow(sql, {
count = it.count
})
return count
}
def findTradesCount(user_id, startDate, endDate){
def sql = "select count(*) as count from user_trades where user_id=${user_id} and DATE_CREATED>='${startDate}' and DATE_CREATED<'${endDate}'";
def count = 0;
onLineSqlInstance.eachRow(sql, {
count = it.count
})
return count
}
}
@@ -22,9 +22,6 @@ class FiledStrand {
def updateIndex = 0;
static main(args) {
// def task = new FiledStrand();
// task.run();
// task.updateOpenDate();
@@ -271,7 +268,7 @@ class FiledStrand {
static def processArea(all_area){
if(all_area){
def v = getValue(all_area)
if(v>=5000 && v<=10000000){
if(v>=1000 && v<=10000000){
return v.toString().toDouble().toInteger()
}
}
@@ -288,11 +285,25 @@ class FiledStrand {
return null
}
static def processLobbyHeight(height){
if(height){
def v = getValue(height)
if(v>=2.4 && v<=30){
return v.toString()
}
}
return null
}
static def processElevators(elevators){
if(elevators){
def v = getValue(elevators)
if(v>0 && v<=100){
return v.toString()
if(v instanceof Integer){
if(v>0 && v<=100){
return v.toString()
}
}
}
return null
@@ -317,24 +328,34 @@ class FiledStrand {
static def processStandardArea(area){
if(area){
area = area.toString().replace(",",",").replace("㎡", "").replace("m", "").replace("/","")
area = Pattern.compile("[\\u4e00-\\u9fa5]").matcher(area).replaceAll("");
return area
def v = getValue(area)
if(v>=500 && v<=20000){
return v.toString().toDouble().toInteger()
}
}
return null
}
static def processCarSiteRent(rent){
if(rent){
def v = getNumberRange(rent)
if(v){
if(v=="0"){
if(["-","0元/月","无","不详","未知","待查","免费元/月","/","0","*","待确认","undefined元/月","0元","0元每月","未知元/月","0.00元/月","未定","暂定","测算中","无元/月","未定元/月","另议元/月","面议元/月","面议元/月","有元/月","暂时未确定","0元(无停车位)","&nbsp;元/月"].contains(rent)){
return null
}else if(rent.contains("暂无") || rent.contains("车位不归") || rent.contains("待定") || rent.contains("暂定") || rent.contains("暂完善") || rent.contains("待审批") || rent.contains("不详") || rent.contains("未定价格") || rent.contains("中国电信") || rent.contains("面谈")){
return null
}else{
rent = rent.replace("元/月","元/月").replace(",","").replace("#","").replace(".00","")
if(rent.toString().length()>100){
return null
}else{
return v.toString()
}
if(rent instanceof Double && rent.toDouble()<100){
return null
}
if(rent instanceof Integer && rent.toInteger()<100){
return null
}
return rent
}
return null
}
return null
}
@@ -382,7 +403,7 @@ class FiledStrand {
static def processCarSite(car_site_count){
def v = getValue(car_site_count.toString());
if(v){
if(v>5 && v<=100000){
if(v>5 && v<=10000){
try {
return v.toString().toInteger();
}catch(Exception e){
@@ -533,12 +554,82 @@ class FiledStrand {
return null
}
static def processLevel(val){
def level = null
if (val) {
if(val.contains("超甲") || val.contains("5A") || val.contains("6A") || val.contains("顶级")){
level = "超甲级"
}else if(val.indexOf("甲")>-1 || val.contains("A")){
level = "甲级"
}else if(val.indexOf("乙")>-1 || val.contains("B")){
level = "乙级";
}else if(val.indexOf("丙")>-1 || val.contains("C")){
level = "丙级";
}
}
return level
}
static def processLoopLine(val){
def line = null
if (val) {
if(!val.contains("暂") && !val.contains("无")){
if(val.indexOf("环")>-1){
line = val
}
}
}
return line
}
static def processForeign(val){
def foreign = null
if (val) {
if(val.indexOf("是")>-1){
foreign = "是"
}else if(val.indexOf("否")>-1){
foreign = "否";
}
}
return foreign
}
static def processRegister(val){
def register = null
if (val) {
if(val.contains("不可以注册") || val.contains("不能注册")){
register = "否";
}else if(val.contains("可以注册")){
register = "是"
}else if(val.indexOf("是")>-1){
register = "是"
}else if(val.indexOf("否")>-1){
register = "否";
}
}
return register
}
static def processFitment(val){
def fitment = null
if(val){
if(val.indexOf("精装")>-1){
fitment = "精装"
}else if(val.indexOf("简单")>-1 || val.indexOf("简装")>-1 || val.indexOf("一般")>-1 || val.indexOf("普通")>-1){
fitment = "简装";
}else if(val.indexOf("毛坯")>-1){
fitment = "毛坯";
}
}
return fitment
}
static def processLngAndLat(val) {
if (val) {
if (val && val!="null") {
def map = [:]
def lng = DataHandleUtils.getDoubleRound(6, val.x)
def lat = DataHandleUtils.getDoubleRound(6, val.y)
def lng = DataHandleUtils.getDoubleRound(6, val.lon)
def lat = DataHandleUtils.getDoubleRound(6, val.lat)
Long gd_lat = lat * 10000000000L
Long gd_lng = lng * 10000000000L
map.lat = gd_lat
@@ -618,7 +709,8 @@ class FiledStrand {
if(cubage_percent){
def v = getValue(cubage_percent)
if(v){
if(v>=0.3 && v<=10){
v = DataHandleUtils.getDoubleRound(2, v.toString().toDouble())
if(v>=0.3 && v<=20){
return v.toString()
}
}
@@ -644,7 +736,7 @@ class FiledStrand {
static def processVirescencePercent(virescence_percent){
if(virescence_percent){
def v = getValue(virescence_percent)
if(v && v>15 && v<=70){
if(v && v>=10 && v<=70){
return v.toString()
}
}
@@ -788,6 +880,20 @@ class FiledStrand {
return null
}
static def processUndergroundFloor(floor){
def v = getValue(floor.toString());
if(v){
if(v>=1 && v<=5){
try {
return v.toString();
}catch(Exception e){
println e.getMessage()
}
}
}
return null
}
static def processRealUsePercent(percent){
def v = getValue(percent.toString());
if(v){
@@ -834,15 +940,40 @@ class FiledStrand {
static def processCompany(val){
if(val){
if(val.length()>20){
if(val.length()>20 || val.length()<4){
return null
}else{
if(val.contains("暂无")){
if(["#NA","*"].contains(val)){
return null
}
if(val.contains("暂无") || val.contains("暂定")){
return null
}
if(val.contains("未知") || val.contains("不祥")){
return null
}
if(val.contains("待定") || val.contains("待详情")){
return null
}
if(val.contains("尚未公开")){
return null
}
if(val.contains("在线咨询")){
return null
}
if(val.contains("平米") || val.contains("元平月")){
return null
}
if(val == "无"){
return null
}
if(val == "-"){
return null
}
if(val == "--"){
return null
}
if(val.isNumber()){
return null
}
@@ -863,7 +994,7 @@ class FiledStrand {
if (pattern.matcher(val).matches()) {
return null
}
return val
return val.replace("*","").replace("*","")
}
}
@@ -3,7 +3,11 @@ package com.fangjia.script.gj.data_handle
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.core.util.ExcelUtil
import com.fangjia.core.util.GeoUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.looplineinfo.InPolygon
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
@@ -13,6 +17,7 @@ import groovy.json.JsonSlurper
import groovy.io.FileType
import org.bson.types.ObjectId
import java.awt.geom.Point2D
import java.util.regex.Pattern
class AddDevShapeData {
@@ -22,22 +27,175 @@ class AddDevShapeData {
public static final def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection dev_mapping_CONN = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL)
public static final DBCollection administrative_mapping_coll = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
def devAreaMap = [:]
def fullCityMap = CommonDatautils.queryFullNameCity()
def results = new HashSet()
def existList = new HashSet()
static main(args) {
def task = new AddDevShapeData();
task.run()
// task.streetInfo()
// task.addStreetToMapping()//通过乡镇街道的点是否都在开发区围栏中,判断是否将此乡镇加入映射表中
}
def addStreetToMapping(){
def citys = CommonDatautils.findAllCity()
for(def city : citys){
def data = fullCityMap.get(city)
def fullCityName = null
if(data){
fullCityName = data.city
}
def devShapes = findDevShapeByCity(fullCityName)
if(devShapes){
def streetShapes = findStreetShapeByCity(fullCityName)
addDevStreetToMapping(fullCityName,streetShapes, devShapes)
}
}
results.each {s->
println s
}
existList.each{s->
println s
}
println " 新增映射乡镇:"+results.size()+" 已存在乡镇:"+existList.size()
}
def addDevStreetToMapping(city,streetShapes, devShapes){
def addCount = 0
def existsCount = 0
for(def streetObj : streetShapes){
def coordinates = streetObj.shape.coordinates
for(def pointList : coordinates){
def objList = findDevInStreetRange(streetObj, pointList, devShapes)
if(objList){
for(def obj : objList){
def existsFlag = isExistStreetByDevName(obj.name, streetObj.street)
if(existsFlag){
existsCount++
existList << "[存在] "+obj.name+" "+ streetObj.street
}else{
addCount++
results << "[新增] "+obj.name+"#"+ streetObj.street
if(!streetObj.street){
println streetObj
}
}
}
}
}
}
}
def findDevInStreetRange(streetObj,pointList, devShapes){
def devList = []
def minDis =200
for(def devObj : devShapes){
def flagList = []
def distances = []
for(def pointArr : pointList){
// def flag = InsidePolygon.isInArea(pointArr[0].toString().toDouble(), pointArr[1].toString().toDouble(), InsidePolygon.createPoint4(devObj.shape.coordinates))
Point2D.Double point = new Point2D.Double(DataHandleUtils.getDoubleRound(6,pointArr[0].toString().toDouble()), DataHandleUtils.getDoubleRound(6, pointArr[1].toString().toDouble()));
def flag = InPolygon.isInArea(point, InPolygon.createPoint(devObj.shape.coordinates))
flagList << flag
if(!flag){
distances << findNearestDistances(pointArr, devObj.shape.coordinates)
}
}
def trueCount = statResult(flagList)
def pre = trueCount*100/flagList.size()
println streetObj.district+" "+streetObj.street +" "+devObj.name +" "+flagList.size()+" "+pre+"%"
if(pre>=100){
devList << devObj
}
// else{
// if(distances && pre>=50){
// def minCount = 0
// for(def d : distances){
// if(d <= minDis){
// minCount++
// }
// }
// def simlarPointPre = minCount*100/distances.size()
// if(simlarPointPre>=90){
// devList << devObj
// }
// println "不在多边型范围内的点,距离小于${minDis}米有:"+minCount+" 占比:"+simlarPointPre
// }
// }
}
return devList
}
def findNearestDistances(pointArr, coordinates){
def minDis = Integer.MAX_VALUE
for(def pointlist : coordinates){
for(def devPoint : pointlist){
def distance = GeoUtils.calculateDistance(pointArr[0], pointArr[1], devPoint[0], devPoint[1])
if(distance < minDis){
minDis = distance
}
}
}
return minDis
}
def statResult(rows){
def count = 0
for(def s : rows){
if(s){
count++
}
}
return count
}
def findStreetShapeByCity(city){
def list = []
def dbcurosr = administrative_mapping_coll.find(new BasicDBObject("city", city).append("type", "乡镇街道").append("shape", new BasicDBObject('$exists', true)))
for(def d : dbcurosr){
list << d
}
return list
}
def findDevShapeByCity(city){
def list = []
def dbcurosr = DEVELOPMENT_ZONE_43_COLL.find(new BasicDBObject("city", city).append('shape', new BasicDBObject('$exists',true)))
for(def d : dbcurosr){
list << d
}
return list
}
def isExistStreetByDevName(devName, street){
def list = dev_mapping_CONN.find(new BasicDBObject("name", devName).append("type","乡镇街道").append("township",street)).toList()
if(list){
return true
}
}
def run(){
def directoryPath = 'D:\\数据\\数据建设\\开发区外包\\111开发区\\'
def directoryPath = 'D:\\数据\\数据建设\\开发区外包\\218\\'
def dir = new File(directoryPath)
if (dir.isDirectory()) {
// dir.eachFileRecurse(FileType.FILES) { file ->
// process(file.absolutePath)
// }
addDevInfo("D:\\数据\\数据建设\\开发区外包\\所有开发区.xlsx")
dir.eachFileRecurse(FileType.FILES) { file ->
process(file.absolutePath)
}
// addDevInfo("D:\\数据\\数据建设\\开发区外包\\所有开发区.xlsx")
} else {
println "The specified path is not a directory."
}
@@ -46,7 +204,7 @@ class AddDevShapeData {
def addDevInfo(file){
ExcelUtils.multiThreadingModify(file, {d->
// if(!d['I']){
def name = d['D']
def name = d['E']
def q = new BasicDBObject()
q.put('_id',new ObjectId(d['A']))
def obj = DEVELOPMENT_ZONE_43_COLL.findOne(q)
@@ -57,18 +215,28 @@ class AddDevShapeData {
if(obj.operate_result){
d['I'] = obj.operate_result
}
if(obj.m_date){
d['P'] = DateUtils.formatDateByPattern(obj.m_date, "yyyy-MM-dd")
if(obj.shape){
d['T'] = "有"
}
// if(obj.m_date){
// d['P'] = DateUtils.formatDateByPattern(obj.m_date, "yyyy-MM-dd")
// }
// if(devAreaMap){
// devAreaMap.each {k,v->
// if(obj.name && k && obj.name.contains(k)){
// d['M'] = v.area_info
// d['N'] = v.count
// d['O'] = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd")
// }
// }
//
// }else{
// println name+" 空值"
// }
}
// }
devAreaMap.each {k,v->
if(name.contains(k)){
d['M'] = v.area_info
d['N'] = v.count
d['O'] = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd")
}
}
})
}
@@ -101,7 +269,7 @@ class AddDevShapeData {
DEVELOPMENT_ZONE_43_COLL.save(obj)
}else{
udata.put("shape", resultMap)
udata.put("m_date", new Date())
// udata.put("m_date", new Date())
udata.put("bd_shape",bd_resultMap)
udata.put("operate_result" , "已处理")
udata.put("operator", "外包")
@@ -124,11 +292,11 @@ class AddDevShapeData {
def coordinates = features[0].geometry.coordinates
def index = 0
println name +" 有 "+coordinates.size()+" 块"
println file.toString()+" "+name +" 有 "+coordinates.size()+" 块"
areaMap.count = coordinates.size()
def areas = []
if(name.contains("泉州经济技术开发区")){
// if(name.contains("江苏常熟综合保税区")){
for(def pointList : coordinates){
index ++
def gaode_coordinates = []
@@ -171,7 +339,7 @@ class AddDevShapeData {
}
areaMap.area_info = areas.join("\n")
}
// }
}else{
println file +" 有多个"
@@ -4,43 +4,104 @@ import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import java.util.regex.Pattern
class AddrAlaisProcess {
DBCollection BAIDU_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206",27020)
def fullCityMap = CommonDatautils.queryFullNameCity()
static main(args) {
def task = new AddrAlaisProcess()
// task.run()
task.nineApi()
// task.createResult()
// task.mergeResult()
}
def mergeResult(){
def rets = ExcelUtils.read("D:\\数据\\数据建设\\小区\\ai-结果-0221.xlsx",0,1)
def retMap = [:]
for(def r : rets){
def num = r['B'].toString().toDouble().toInteger()
def list = retMap.get(num)
if(!list){
list = new HashSet()
retMap.put(num, list)
}
if(r['C']){
list << r['C']
}
}
def fileName = "D:\\数据\\数据建设\\小区\\小区别名-待定.xlsx"
ExcelUtils.modify(fileName,{d->
def num = d['A'].toString().toDouble().toInteger()
def aiList = retMap.get(num)
if(aiList){
d['K'] = aiList.join(",")
}
})
}
def createResult(){
def fileName = "D:\\数据\\数据建设\\小区\\小区别名-待定.xlsx"
def results = []
// results << ["ID", "城市", "地址"]
ExcelUtils.modify(fileName,{d->
def str = d['C']
def addressList = str.split(",")
for(def addr : addressList){
results << [d['A'].toString().toDouble().toInteger(), d['B'], addr]
}
})
println results.size()
FileUtils.batchCreateFileByCustomSuffix(results, "D:\\数据\\数据建设\\小区\\ai-待支行.bat", null, "utf-8", false, " ","");
}
def nineApi(){
def fileName = "/home/goojia/gj/district/addr_alias.xlsx"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\数据建设\\小区\\小区地址及地址别名标准化-测试.xlsx"
fileName = "D:\\数据\\数据建设\\写字楼\\写字楼待处理地址.xlsx"
}
ExcelUtils.multiThreadingModify(fileName, {d->
def addrMap = nineAddrSplit(d['B'], d['E'])
def obj = CommonDatautils.findDistrictById(d['A'])
// def addrMap = nineAddrSplit(obj.city, d['B'])
def addrList = []
if(addrMap && addrMap.address){
d['G'] = addrMap.city
d['H'] = addrMap.address
addrList << addrMap.address
}
if(d['F']){
def addrs = d['F'].split(",")
def addr_alias = []
for(def addr : addrs){
def map = nineAddrSplit(d['B'], addr)
if(map && map.address && DataHandleUtils.isStandardRoad(map.address)){
if(!addrList.contains(map.address)){
addrList << map.address
addr_alias << map.address
}
}
}
d['I'] = addr_alias.join(",")
}
// if(addrMap && addrMap.address){
// d['C'] = addrMap.road
// d['D'] = addrMap.road_num
// }
if(d['C']){
def data = fullCityMap.get(obj.city)
if(data){
def roadObj = findBaiduDatas( data.city, d['C'])
if(roadObj){
d['E'] = "百度存在"
}
}
}
// if(d['F']){
// def addrs = d['F'].split(",")
// def addr_alias = []
// for(def addr : addrs){
// def map = nineAddrSplit(d['B'], addr)
// if(map && map.address && DataHandleUtils.isStandardRoad(map.address)){
// if(!addrList.contains(map.address)){
// addrList << map.address
// addr_alias << map.address
// }
// }
// }
// d['I'] = addr_alias.join(",")
// }
})
}
@@ -67,9 +128,11 @@ class AddrAlaisProcess {
}
if (jsonObject.get("street") && jsonObject.get("street") != "null") {
address = jsonObject.get("street")
map.road = address
}
if (jsonObject.get("streetNumber") && jsonObject.get("streetNumber") != "null") {
address = address + jsonObject.get("streetNumber")
map.road_num = jsonObject.get("streetNumber")
}
map.address = address
}
@@ -95,4 +158,13 @@ class AddrAlaisProcess {
// ExcelUtils.write("D:\\数据\\数据建设\\小区\\小区地址及地址别名标准化.xlsx", rows, "小区")
FileUtils.batchCreateFileByCustomSuffix(rows, "D:\\数据\\数据建设\\小区\\小区地址及地址别名标准化.bat", null, "utf-8", false, "#","");
}
def findBaiduDatas(city, name){
def q = new BasicDBObject()
q.put("city", city)
q.put("name", name)
q.put('std_tag', Pattern.compile('^道路'))
def obj = BAIDU_POI_COLL.findOne(q)
return obj;
}
}
@@ -4,6 +4,7 @@ import com.fangjia.script.BaseBatch
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
@@ -30,7 +31,10 @@ class AutoAddDistrictBatch extends BaseBatch{
def mcaDistMap = CommonDatautils.queryDistNameCityByMapping()
static main(args) {
runBatch(new AutoAddDistrictBatch(), args)
// runBatch(new AutoAddDistrictBatch(), args)
def task = new AutoAddDistrictBatch();
task.checkMap()
}
@Override
@@ -52,7 +56,6 @@ class AutoAddDistrictBatch extends BaseBatch{
}
ExcelUtils.write(fileName, list, "待入库数据")
check(fileName)
}
@@ -159,10 +162,10 @@ class AutoAddDistrictBatch extends BaseBatch{
}
def findGaoDeDatas(city, region, name){
String sql = "select name,type from amap_poi_detail where cityname='${city}' and adname='${region}' and MATCH(name, '${name}')"
String sql = "select name,type from amap_poi_2024 where cityname='${city}' and adname='${region}' and MATCH(name, '${name}')"
def obj = null
ElasticUtils.executeSqlQuery(sql,{ d->
if(d.name == name && d.type.contains("住宅") ){ //&& d.shape
if(d.name == name){ //&& d.shape
obj = d
}
})
@@ -255,8 +258,37 @@ class AutoAddDistrictBatch extends BaseBatch{
q.put("city", city)
q.put("district", district)
q.put("name", name)
q.put('std_tag', '房地产;住宅区')
// q.put('std_tag', '房地产;住宅区')
def obj = BAIDU_POI_COLL.findOne(q)
return obj;
}
def checkMap(){
ExcelUtils.modify("D:\\数据\\数据建设\\写字楼\\写字楼存在性判断-test.xlsx",{d->
def gaode_obj = findGaodeCount(d['B'], d['C'], d['D'])
if(gaode_obj){
d['H'] = "是"
d['I'] = gaode_obj.type
}else{
def baidu_obj = findBaiduCount(d['B'], d['C'], d['D'])
if(baidu_obj){
d['E'] = "是"
d['F'] = baidu_obj.std_tag
}
}
// def baidu_obj1 = findBaiduCount(d['B'], d['G'], d['H'])
// if(baidu_obj1){
// d['M'] = "是"
// }
// def gaode_obj1 = findGaodeCount(d['B'], d['G'], d['H'])
// if(gaode_obj1){
// d['N'] = "是"
// }
// def score = CosineSimilarAlgorithm.getSimilarityRatio(d['D'], d['H']);
// d['O'] = score
})
}
}
@@ -4,14 +4,13 @@ import com.fangjia.core.RegionData
import com.fangjia.script.BaseBatch
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.DataUtil
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.fangjia.script.util.GeoHelper
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CopyOnWriteArraySet
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
@@ -20,30 +19,18 @@ import java.util.concurrent.atomic.AtomicLong
class AutoAddOfficeBatch extends BaseBatch{
def fullCityMap = CommonDatautils.queryFullNameByCity()
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
DBCollection BAIDU_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206",27020)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
static main(args) {
runBatch(new AutoAddOfficeBatch(), args)
// runBatch(new AutoAddOfficeBatch(), args)
def task = new AutoAddOfficeBatch()
task.repeatOffice()
}
@Override
void process(Object args) {
// def pattern = /[A-Z0-9]座|[0-9a-z]栋|[A-Z]栋|·|\(建设中\)|[0-9]号楼|\(装修中\)|[A-Z]楼|[0-9]幢/
// def newName = "开放广场".replaceAll(pattern, "")
// println newName
// findAddOfficeDatas() //查找需要新增的写字楼数据
// checkInBaiduMapOffice()//查寻在百度地图上有的写字楼数据
addOffice()//新增写字楼
}
def findAddOfficeDatas(){
def citys = CommonDatautils.findAllCity()
def allAddOffices = []
allAddOffices << ["地图ID","省","市","区","写字楼名称","地址","经度","纬度","匹配城市","匹配区域","匹配名称","匹配方式"]
@@ -54,6 +41,7 @@ class AutoAddOfficeBatch extends BaseBatch{
officeMaps = filterOffice(officeMaps)
if(officeMaps){
def offices = matchOffice(officeMaps)
allAddOffices.addAll(offices)
}
}
@@ -61,19 +49,6 @@ class AutoAddOfficeBatch extends BaseBatch{
ExcelUtils.write("D:\\数据\\高德写字楼.xlsx", allAddOffices, "数据")
}
def checkInBaiduMapOffice(){
ExcelUtils.multiThreadingModify("D:\\数据\\高德写字楼1.xlsx",{d->
if(!d['L']){
def obj = findBaiduDatas(d['C'], d['D'], d['E'])
if(obj){
d['L'] = "百度存在"
d['M'] = obj.std_tag.toString()
}
}
},0)
}
def filterOffice(officeMaps){
def offices = []
def keys = []
@@ -200,60 +175,72 @@ class AutoAddOfficeBatch extends BaseBatch{
return null
}
def findBaiduDatas(city, district, name){
def q = new BasicDBObject()
q.put("city", city)
q.put("district", district)
q.put("name", name)
// q.put('std_tag', '房地产;住宅区')
def obj = BAIDU_POI_COLL.findOne(q)
return obj;
def repeatOffice(){
def citys = CommonDatautils.findAllCity()
def allrets = []
allrets << ["ID","城市","区域","名称","地址","可疑ID","可疑城市","可疑区域","可疑名称","可疑地址","相似度","距离"]
for(def c : citys){
def offices = CommonDatautils.findAllDistricts(c, "office", ["city","region","name","address", "lat", "lng"])
def rets = checkRepeatOfficeBatch(offices)
println c +" "+offices.size()
allrets.addAll(rets)
}
ExcelUtils.write("D:\\数据\\写字楼\\重复可疑写字楼1.xlsx", allrets, "数据")
}
def addOffice(){
ExcelUtils.modify("D:\\数据\\高德待入库写字楼.xlsx",{d->
if(d['L']!='人工删除'){
def city = RegionData.formatCity(d['C'])
def region = RegionData.formatRegion(city, d['D'])
if(city && region){
def locationstr = d['G'].toString()+","+d['H'].toString()
def bdpoint = FangjiaClientUtils.convert(locationstr, 1)
def udata = new BasicDBObject()
if(bdpoint){
udata.put("lng2", (bdpoint[0]*10000000000).longValue())
udata.put("lat2", (bdpoint[1]*10000000000).longValue())
def checkRepeatOfficeBatch(offices){
def addCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(offices.size())
def total = offices.size()
def rets = new CopyOnWriteArrayList()
def sets = new CopyOnWriteArraySet()
for(def i=0;i<total;i++){
final def o1 = offices[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
addCount.addAndGet(1)
def maxObj = findOneSimilarOffice(o1, offices)
if(maxObj){
if(!sets.contains(o1._id.toString())){
sets.add(maxObj._id.toString())
rets << [o1._id.toString(), o1.city, o1.region, o1.name, o1.address, maxObj._id.toString(), maxObj.city, maxObj.region, maxObj.name, maxObj.address, maxObj.score, maxObj.distance]
}
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rets
}
udata.put("city", city)
udata.put("region", region)
udata.put("name", d['E'])
udata.put("address", d['I'])
udata.put("cat", "office")
udata.put("status", 0)
udata.put("c_date",new Date())
udata.put("m_date",new Date())
udata.put("lng", (d['G'].toString().toDouble()*10000000000).longValue())
udata.put("lat", (d['H'].toString().toDouble()*10000000000).longValue())
def estate_type = null
if(d['M'] && d['M'].contains("园区")){
estate_type = "创意园区"
}
if(!estate_type && d['E'].contains("创意")){
estate_type = "创意园区"
}
if(estate_type){
udata.put("estate_type2", estate_type)
}
def markMap = [:]
markMap.put("data_source", "机器入库")
udata.put("mark_map", markMap)
DbUtils.seaweed.save(udata)
println d['E'] +" "+udata._id.toString()
def findOneSimilarOffice(o1, offices){
def maxScore = 0
def maxObj = null
for(def o2 : offices){
if(o1._id.toString()!=o2._id.toString()){
double nameScore = CosineSimilarAlgorithm.getSimilarity(o1.name, o2.name);
double addrScore = CosineSimilarAlgorithm.getSimilarity(o1.address,o2.address);
if(nameScore+addrScore>maxScore){
maxScore = nameScore+addrScore
maxObj = o2
}
}
})
}
def distance = -1000
if(o1.lat && maxObj && maxObj.lat){
distance = GeoHelper.GetDistance(o1.lat/10000000000.0, o1.lng/10000000000.0, maxObj.lat/10000000000.0, maxObj.lng/10000000000.0)
}
if(maxObj){
maxObj.distance = distance
maxObj.score = maxScore
return maxObj
}
return null
}
}
@@ -33,8 +33,8 @@ class DevUpdateTestBatch {
static main(args) {
def task = new DevUpdateTestBatch()
def path = "D:\\数据\\数据建设\\开发区\\开发区-园区.xlsx"//开发区-园区.xlsx
// task.addDev(path)
task.addDevAddress(path)
task.addDev(path)
// task.addDevAddress(path)
//
// task.updateHasGov("新疆维吾尔自治区","喀什地区","麦盖提县", "麦盖提镇", "是")
@@ -5,8 +5,10 @@ import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.impl.AMQImpl
import java.util.regex.Pattern
/**
* 名录集
*/
@@ -29,12 +31,12 @@ class MinglujiData {
}
def run(){
def prov = "省"
def city = "石家庄市"
// exportDatas(city)
def prov = "江苏省"
def city = "无锡"
exportDatas(city)
// matchDev(prov, city)
// matchCommunity(city)
fiveLevelSplit(city)
// fiveLevelSplit(city)
// nineLevelSplit(city)
// tempMatchDev(prov)
@@ -4,12 +4,9 @@ import com.fangjia.core.RegionData
import com.fangjia.script.BaseBatch
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.xiamen.ElasticUtils89
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import org.apache.commons.io.FileUtils
import org.bson.types.ObjectId
import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.update.UpdateRequest
import org.elasticsearch.client.RequestOptions
@@ -34,15 +31,12 @@ class RegionStandardBatch extends BaseBatch{
@Override
void process(Object args) {
// seaweedRegionStandard()
seaweedRegionStandard()
// poiRegionStandard()
//通过乡镇街道修改区域 如:东莞市
// updateRegionByTown()
}
def seaweedRegionStandard(){
seaweedRegionStandardByCat("office")
seaweedRegionStandardByCat("district")
}
def poiRegionStandard(){
@@ -99,7 +93,7 @@ class RegionStandardBatch extends BaseBatch{
def seaweedRegionStandardByCat(cat){
def citys = ["伊春"]//CommonDatautils.findAllCity()
def citys = CommonDatautils.findAllCity()
for(def city : citys){
def rMap = regionMap.get(city)
@@ -154,12 +148,8 @@ class RegionStandardBatch extends BaseBatch{
def streetMap = CommonDatautils.queryAllStreetNameByCity(fullCityName)
if(streetMap){
set = streetMap.keySet()
def districts = CommonDatautils.findAllDistricts(city, cat, ["city","region","name","lng","lat"])
println city +" "+ districts.size()
def index = 0
def districts = CommonDatautils.findAllDistricts(city, cat, ["city","region","name"])
for(def d : districts){
index ++
println index +" "+ d.region
if(set.contains(d.region)){
continue
}
@@ -167,12 +157,8 @@ class RegionStandardBatch extends BaseBatch{
def updateRegion = null
def flag = null
for(def key : set){
def obj = streetMap.get(key)
def alias = null
if(obj && obj.alias){
alias = obj.alias
}
if((alias && alias.contains(d.region)) || (!alias && key==d.region)){
def alias = streetMap.get(key)
if((alias && alias.contains(d.region)) || (!alias && key.contains(d.region))){
def newRegion = RegionData.formatRegion(d.city, key)
if(newRegion!=d.region){
updateRegion = newRegion
@@ -180,26 +166,13 @@ class RegionStandardBatch extends BaseBatch{
flag = "pass"
break;
}
if(obj.shape){
def inAreaFlag = InsidePolygon.isInArea(d.lng/10000000000.0, d.lat/10000000000.0, InsidePolygon.createPoint4(obj.shape.coordinates))
if(inAreaFlag){
def newRegion = obj.street
newRegion = RegionData.formatRegion(d.city, newRegion)
if(newRegion!=d.region){
updateRegion = newRegion
}
flag = "pass"
break;
}
}
}
if(updateRegion){
def udata = new BasicDBObject()
udata.put("region", updateRegion)
udata.put("m_date", new Date())
println d.city +" "+d.region+" "+d.name +" 区域修改为:"+updateRegion
DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
// DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
}else{
if(flag!="pass"){
def udata = new BasicDBObject()
@@ -429,27 +402,4 @@ class RegionStandardBatch extends BaseBatch{
return rets;
}
def updateRegionByTown(){
ExcelUtils.modify("d:\\数据\\城市与区域相同写字楼.xlsx",{d->
def town = RegionData.formatRegion(d['B'], d['F'])
if(town){
def obj = CommonDatautils.findDistrict(d['B'], town, d['E'], "office")
if(!obj){
def udata = new BasicDBObject()
udata.put("region", town)
udata.put("m_date", new Date())
println d['F']+" "+" 区域修改为:" + town
DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(d['A'])), new BasicDBObject('$set', udata))
}else{
def udata = new BasicDBObject()
udata.put("m_date", new Date())
udata.put("status", 1)
println d['F']+" "+" 删除:" + town
DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(d['A'])), new BasicDBObject('$set', udata))
}
}
})
}
}
@@ -1,6 +1,5 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
@@ -220,41 +219,34 @@ class TempExportData {
}
def exportOfficeHouseCount(){
ExcelUtils.multiThreadingModify("D:\\数据\\高德写字楼-房源第三方验证.xlsx",{d->
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\写字楼\\待核实写字楼.xlsx",{d->
// def obj = findYuanQu(d['B'], d['D'])
// if(obj){
// d['T'] = "前瞻存在"
// }
// if(!d['T']){
// def city = RegionData.formatCity(d['C'])
// def obj = findYuanQu(city, d['E'])
// def obj = findYuanQu(d['B'], d['D'])
// if(obj){
// d['T'] = "前瞻存在"
// d['U'] = obj.name
// }
// }
// if(!d['T']) {
// def city = RegionData.formatCity(d['C'])
// def region = RegionData.formatRegion(city, d['D'])
//
// def count = findHouseCountByCat(city, region, d['E'], "office_sell")
// def count = findHouseCountByCat(d['B'], d['C'], d['D'], "office_sell")
// if (!count) {
// count = findHouseCountByCat(city, region, d['E'], "office_rent")
// count = findHouseCountByCat(d['B'], d['C'], d['D'], "office_rent")
// }
// d['T'] = count
// d['S'] = count
// }
if(!d['T']){
def city = RegionData.formatCity(d['C'])
def region = RegionData.formatRegion(city, d['D'])
def datas = CommonDatautils.findThirdNetWorkOffices(city, region, d['E'])
if(!d['J']){
def datas = CommonDatautils.findThirdNetWorkOffices(d['B'], d['C'], d['D'])
if(datas){
def sources = []
for(def o : datas){
sources << o.source
}
if(sources){
d['V'] = sources.join(",")
d['K'] = sources.join(",")
}
}
}
@@ -1,57 +1,58 @@
package com.fangjia.script.gj.data_handle
import org.bson.types.ObjectId
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.apache.commons.lang.math.RandomUtils
def mongoDbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
DBCollection REGION_STAT_COLL = MongoUtils.getCollectionByName2("fangjia", "region_stat", mongoDbUrl)
def city = "昌江"
ExcelUtils.modify("D:\\数据\\海南\\海南省城市均价.xlsx",{d->
def date = d['B']
def start = DateUtils.getFirstDayOfMonth(date)
def end = DateUtils.getLastDayOfMonth(date)
def weekends = DateUtils.getWeekendBetweenDates(start, end)
for(def weekend : weekends){
def avg_price = d['I']
def val = random(-200,200)
def new_price = (int)(avg_price*val/10000 + avg_price)
def last_weekend = DateUtils.add(DateUtils.formatDateByPattern(weekend, "yyyy-MM-dd"), -7)
def last_price = CommonDatautils.queryCityPriceByWeekend(city, last_weekend)
def o = new BasicDBObject()
o.put("avg_price", new_price)
o.put("c_date" , new Date())
o.put("city", city)
o.put("m_date" , new Date())
o.put("weekend", weekend)
o.put("cat","城市")
o.put("moving_average" , (int)avg_price)
if(!last_price){
o.put("weekend_markup" , 0)
}else{
def weekend_markup = (int)((new_price-last_price)*100/last_price)
o.put("weekend_markup" , weekend_markup)
}
def q = new BasicDBObject("cat", "城市").append("city", city).append("weekend", weekend)
REGION_STAT_COLL.update(q, new BasicDBObject('$set', o),true, false)
}
},1)
public static final int random(int i1, int i2) {
int max = Math.max(i1, i2);
int min = Math.min(i1, i2);
return min + RandomUtils.nextInt(max - min + 1);
}
ObjectId objectId = new ObjectId("659c9ae92ae7732fd7d54e40");
Date creationDate = objectId.getDate();
System.out.println(creationDate);
//def mongoDbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
//DBCollection REGION_STAT_COLL = MongoUtils.getCollectionByName2("fangjia", "region_stat", mongoDbUrl)
//
//
//def city = "昌江"
//ExcelUtils.modify("D:\\数据\\海南\\海南省城市均价.xlsx",{d->
// def date = d['B']
// def start = DateUtils.getFirstDayOfMonth(date)
// def end = DateUtils.getLastDayOfMonth(date)
// def weekends = DateUtils.getWeekendBetweenDates(start, end)
// for(def weekend : weekends){
// def avg_price = d['I']
// def val = random(-200,200)
// def new_price = (int)(avg_price*val/10000 + avg_price)
// def last_weekend = DateUtils.add(DateUtils.formatDateByPattern(weekend, "yyyy-MM-dd"), -7)
// def last_price = CommonDatautils.queryCityPriceByWeekend(city, last_weekend)
//
// def o = new BasicDBObject()
// o.put("avg_price", new_price)
// o.put("c_date" , new Date())
// o.put("city", city)
// o.put("m_date" , new Date())
// o.put("weekend", weekend)
// o.put("cat","城市")
// o.put("moving_average" , (int)avg_price)
// if(!last_price){
// o.put("weekend_markup" , 0)
// }else{
// def weekend_markup = (int)((new_price-last_price)*100/last_price)
// o.put("weekend_markup" , weekend_markup)
// }
//
// def q = new BasicDBObject("cat", "城市").append("city", city).append("weekend", weekend)
// REGION_STAT_COLL.update(q, new BasicDBObject('$set', o),true, false)
// }
//},1)
//
//
//public static final int random(int i1, int i2) {
// int max = Math.max(i1, i2);
// int min = Math.min(i1, i2);
// return min + RandomUtils.nextInt(max - min + 1);
//}
//
//
//
//
//
@@ -3,22 +3,23 @@ package com.fangjia.script.gj.data_handle
import com.fangjia.core.RegionData
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.core.util.GeocodeUtils
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.util.BaiduMapUtils
import com.fangjia.script.util.DataUtil
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.ProvinceData
import com.mongodb.AggregationOutput
import com.mongodb.BasicDBObject
import com.mongodb.Bytes
import com.mongodb.DBCollection
import com.mongodb.DBObject
import net.sf.json.JSONObject
import com.mongodb.client.AggregateIterable
import org.bson.Document
import org.bson.types.ObjectId
import java.util.regex.Pattern
/**
* 乡镇街道中间表
* 包含市,区,乡镇街道别名,来源 统计局community表
@@ -44,8 +45,6 @@ class TownMiddleProcess {
DBCollection DEVELOPMENT_ZONE_43_COLL = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com")
static DBCollection POLYLINECOLL2025 = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline_2025")
def fullCityMap = CommonDatautils.queryFullNameCity()
static main(args) {
@@ -65,7 +64,7 @@ class TownMiddleProcess {
//修改乡镇街道别名
// updateTownAlias()
//中间表同步到100
// synchTown()
synchTown()
//添加地级市标志
// addCityFlag()
// addPinyin()
@@ -97,10 +96,6 @@ class TownMiddleProcess {
// checkCenterInTownRang()//检测乡镇街道中心点是否在围栏中
// addSpecialCityTownAlias()
// updateNewRegionPoyline()//更新新的区域围栏
updateMappingPolyline()
}
@@ -686,7 +681,7 @@ class TownMiddleProcess {
}
def regionPolyline(){
def dbcursor = POLYLINECOLL2025.find(new BasicDBObject("level","district"))
def dbcursor = REGIONPOLYLINE_COLL.find(new BasicDBObject("level","district"))
def map = [:]
for(def d : dbcursor){
def resultMap = [:]
@@ -1361,90 +1356,4 @@ class TownMiddleProcess {
keys.put("street", 1)
return ADMINISTRATIVE_MAPPING_COLL.find(q, keys).toList()
}
def updateNewRegionPoyline(){
def regionPolylineMap = regionPolyline()
def q = new BasicDBObject()
q.put("type","区域")
// q.put("city", "苏州市")
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
for(def d : dbcursor){
def shapeData = regionPolylineMap.get(d.city+"|@|"+d.district)
println d.city+" "+d.district
if(shapeData){
println "新围栏数:"+shapeData.shape.coordinates.size()
}
if(d.shape){
println "旧围栏数:"+d.shape.coordinates.size()
}
if(d.shape && shapeData && shapeData.shape!=d.shape){
def udata = new BasicDBObject()
udata.shape = shapeData.shape
udata.location = shapeData.location
udata.m_date = new Date()
ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",d._id), new BasicDBObject('$set',udata))
// if(shapeData && d.shape && shapeData.shape.type=="polygon" && d.shape.type=="polygon"){
// double area = com.fangjia.script.util.GeoHelper.calculateArea(shapeData.shape.coordinates[0])
// double area1 = com.fangjia.script.util.GeoHelper.calculateArea(d.shape.coordinates[0])
// if(Math.abs(area1-area)>=1){
// println "新:"+area
// println "旧:"+area1
// def udata = new BasicDBObject()
// udata.shape = shapeData.shape
// udata.location = shapeData.location
// //DMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",d._id), new BasicDBObject('$set',udata))
// }
// }
}
}
}
def updateMappingPolyline(){
def q = new BasicDBObject()
q.put("type","区域")
q.put("shape.type","polygon")
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q)
for(def d : dbcursor){
updateAdministrativeMapping(d._id.toString(), d.shape)
}
}
def updateAdministrativeMapping(def id, def shape){
def header = [:]
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString()
String seawaterUrl = "http://192.168.0.89:9200/administrative_mapping/_update_by_query?conflicts=proceed&scroll_size=1000&wait_for_completion=false"
def body = """
{
"script": {
"source": "ctx._source.shape = params.newshape",
"params":{
"newshape":{
"type":"${shape.type}",
"coordinates":${shape.coordinates}
}
}
},
"query": {
"term": {
"_id": "${id}"
}
}
}
"""
HttpRequest request = new HttpRequest()
try{
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
def response = JSONObject.fromObject(res)
println(response)
}catch(Exception e){
println "error : "+e.getMessage()
}
}
}
@@ -1,14 +1,21 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.core.RegionData
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.BaseBatch
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.xiamen.ElasticUtils89
import com.fangjia.script.lixue.util.DBUtils
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.util.BaiduMapUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import net.sf.json.JSONObject
import org.apache.commons.io.FileUtils
import org.bson.types.ObjectId
import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.bulk.BulkResponse
import org.elasticsearch.action.index.IndexRequest
@@ -24,6 +31,7 @@ import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
/**
* 区域标准化
* 1.小区库乡镇街道标准化,包含小区,商铺,写字楼
@@ -53,11 +61,14 @@ class TownStandardBatch extends BaseBatch{
townStandardByCat("office")
// townStandardByPoi()
// addTownByTownShape()//为没有乡镇街道的poi,补充乡镇街道,忽略区域,并使用地图
// standardTown()
// updateTownData()
}
def townStandardByCat(def cat){
//"三明","龙岩","南平","漳州","福州","厦门","泉州","莆田","宁德",
def citys = ["重庆","龙岩","南平","漳州","福州","厦门","泉州","莆田","三明","宁德"]//CommonDatautils.findAllCity()
def citys = ["天津"]//CommonDatautils.findAllCity()
for(def city : citys){
def data = fullCityMap.get(city)
@@ -68,10 +79,10 @@ class TownStandardBatch extends BaseBatch{
def mzbRegionMap = CommonDatautils.queryMzbRegion(fullCityName)
def shortRegionNameMap = CommonDatautils.getRegionShortName(fullCityName)
def townMap = CommonDatautils.queryTownByCity(fullCityName, ["city","district", "street", "alias"], mzbRegionMap)
def townMap = CommonDatautils.queryTownByCity(fullCityName, ["city","district", "street", "alias","shape"], mzbRegionMap)
if(townMap){
def regionSet = townMap.keySet()
if(regionSet.size()>1){
if(regionSet.size()>=1){
haveRegionProcess(city, fullCityName, townMap, cat, shortRegionNameMap)
}else{
println city +" 无区域"
@@ -137,11 +148,12 @@ class TownStandardBatch extends BaseBatch{
def haveRegionProcess(city, fullCityName, townMap, cat, shortRegionNameMap){
def results = []
if(fullCityName){
def districts = CommonDatautils.findAllDistricts(city, cat, ["city","region","name", "township"])
def districts = CommonDatautils.findAllDistricts(city, cat, ["city","region","name", "township","lng","lat"])
for(def d : districts){
if(d.township){
def towns = null
def updateTown = null
def towns = null
def updateTown = null
if(d.region){
for(def m : townMap){
def fullRegionName = shortRegionNameMap.get(d.region)
if(m.key == fullRegionName){
@@ -149,41 +161,51 @@ class TownStandardBatch extends BaseBatch{
break
}
}
}
if(!towns){
towns = townMap.get(fullCityName)
}
def opt = true
if(towns){
for(def town : towns){
def townAlias = []
if(town.alias){
townAlias = town.alias
}
townAlias << town.street
if(townAlias.contains(d.township)){
if(d.township!=town.street){
updateTown = town.street
}else{
opt = false
}
break
}
def opt = true
if(towns){
for(def town : towns){
def townAlias = []
if(town.alias){
townAlias = town.alias
}
}
if(opt){
if(updateTown){
def udata = new BasicDBObject()
udata.put("township", updateTown)
udata.put("m_date", new Date())
println d.city +" "+d.region+" "+d.name+" "+d.township +" 乡镇修改为:"+updateTown
// DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
townAlias << town.street
if(townAlias.contains(d.township)){
if(d.township!=town.street){
updateTown = town.street
}else{
opt = false
}
break
}else{
def udata = new BasicDBObject()
udata.put("township", "")
udata.put("m_date", new Date())
println d.city +" "+d.region+" "+d.name+" "+d.township +" 乡镇修改为:空值"
// DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
if(town.shape){
def flag = InsidePolygon.isInArea(d.lng/10000000000.0, d.lat/10000000000.0, InsidePolygon.createPoint4(town.shape.coordinates))
if(flag){
updateTown = town.street
break
}
}
}
}
}
if(opt){
if(updateTown){
def udata = new BasicDBObject()
udata.put("township", updateTown)
udata.put("m_date", new Date())
println d.city +" "+d.region+" "+d.name+" "+d.township +" 乡镇修改为:"+updateTown
DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
}else{
def udata = new BasicDBObject()
udata.put("township", "")
udata.put("m_date", new Date())
println d.city +" "+d.region+" "+d.name+" "+d.township +" 乡镇修改为:空值"
DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
}
}
}
}
@@ -363,6 +385,33 @@ class TownStandardBatch extends BaseBatch{
return rets;
}
def readCSV3(filename){
def lines = FileUtils.readLines(new File(filename))
def rets = [];
def index = 0
for(line in lines) {
if(line && index>=1){
String[] arrs = line.split(",")
if(arrs.size()>=11 && arrs[10]=="是" && arrs[6]!=arrs[7]){
def o = new BasicDBObject()
try{
o.put("id", arrs[0])
o.put("township", arrs[7])
o.put("city", arrs[2])
o.put("old_district", arrs[3])
o.put("new_district", arrs[8])
rets << o
}catch(Exception e){
println arrs
}
}
}
index++
}
return rets;
}
def readCSV2(filename){
def lines = FileUtils.readLines(new File(filename))
def rets = [];
@@ -378,10 +427,11 @@ class TownStandardBatch extends BaseBatch{
o.put("district", arrs[4])
o.put("lng", arrs[5])
o.put("lat", arrs[6])
if(arrs.size()<=7){
o.put("data_source","")
}else{
o.put("data_source", arrs[7])
o.put("data_source", arrs[7])
if(arrs.size()>=9){
o.put("township", arrs[8])
o.put("new_district", arrs[9])
o.put("town_source", arrs[10])
}
rets << o
}catch(Exception e){
@@ -393,43 +443,13 @@ class TownStandardBatch extends BaseBatch{
return rets;
}
def readCSV3(filename){
def lines = FileUtils.readLines(new File(filename))
def rets = [];
def index = 0
for(line in lines) {
if(line && index>=1){
String[] arrs = line.split(",")
if(arrs.size()>=10 && arrs[7] && arrs[9]=="同区域"){
def o = new BasicDBObject()
try{
o.put("id", arrs[0])
o.put("township", arrs[7])
if(arrs.size()>=9 && arrs[8]){
o.put("district", arrs[8])
}
if(arrs.size()>=10 && arrs[9]){
o.put("way", arrs[9])
}
rets << o
}catch(Exception e){
println arrs
}
}
}
index++
}
return rets;
}
def addTownByTownShape(){
def i = 0
def headers = ["_id","province","city","district","lng","lat","data_source","乡镇街道", "新区域", "方式", "标准乡镇"]
while(i<50){
while(i<=50){
def fileName = "/home/goojia/gj/new_poi/input/map/no_township_${i}.csv"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\数据建设\\乡镇街道\\待补充乡镇街道\\原始\\no_township_${i}.csv"
fileName = "D:\\数据\\数据建设\\乡镇街道\\no_township_${i}.csv"
}
File file=new File(fileName);
if(file.exists()) {
@@ -449,19 +469,53 @@ class TownStandardBatch extends BaseBatch{
}
}
def standardTown(){
def i = 49
def headers = ["_id","province","city","district","lng","lat","data_source","乡镇街道", "新区域", "方式", "标准乡镇"]
def cityTownMap = CommonDatautils.queryAllTown(["city", "district", "mzb_district_name", "street", "alias"])
while(i<=50){
def fileName = "/home/goojia/gj/new_poi/input/map/no_township_result_${i}.csv"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\数据建设\\乡镇街道\\待补充乡镇街道\\结果-地图\\no_township_result_${i}.csv"
}
File file=new File(fileName);
if(file.exists()) {
def rets = readCSV2(fileName)
println i +" "+ rets.size()
def rows = addTownshipBatch2(rets, cityTownMap)
if (CommonDatautils.isWindow()) {
com.fangjia.script.gj.ccb.FileUtils.batchCreateFileByCustomSuffix(rows, "D:\\数据\\数据建设\\乡镇街道\\待补充乡镇街道\\结果-地图-验证乡镇\\no_township_result_${i}.csv", headers, "utf-8", false, ',', '');
}else{
com.fangjia.script.gj.ccb.FileUtils.batchCreateFileByCustomSuffix(rows, "/home/goojia/gj/new_poi/output/no_township_result_${i}.csv", headers, "utf-8", false, ',', '');
}
println fileName +" 运行完成"
}else{
println fileName +" 不存在"
}
i++
}
}
def addTownshipBatch(def rets){
def results = new CopyOnWriteArrayList()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(rets.size())
for(def i=0;i<rets.size();i++){
final def r = rets[i]
fixedThreadPool.execute(new Runnable() {
void run() {
// fixedThreadPool.execute(new Runnable() {
// void run() {
try{
totalCount.addAndGet(1)
def rMap = regionFullNameMap.get(r.city)
if(!rMap){
def data = fullCityMap.get(r.city)
rMap = regionFullNameMap.get(data.city)
}
if (rMap) {
def townMap = [:]//CommonDatautils.queryTownByCity(r.city, ["city", "district", "mzb_district_name", "street", "alias", "shape"])
// def townMap = CommonDatautils.queryTownByCity(r.city, ["city", "district", "mzb_district_name", "street", "alias", "shape"])
// def set = rMap.keySet()
println totalCount.get() +" "+r.city + " "
// if (set.size() >= 1) {
@@ -474,6 +528,46 @@ class TownStandardBatch extends BaseBatch{
} finally {
latch.countDown()
}
// }
// });
}
latch.await()
return results
}
def addTownshipBatch2(def rets, def cityTownMap){
def results = new CopyOnWriteArrayList()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(rets.size())
for(def i=0;i<rets.size();i++){
final def r = rets[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def rMap = regionFullNameMap.get(r.city)
def fullcity = null
if(!rMap){
def data = fullCityMap.get(r.city)
rMap = regionFullNameMap.get(data.city)
fullcity = data.city
}
if (rMap) {
def townMap = cityTownMap.get(fullcity)
def set = rMap.keySet()
if (set.size() >= 1) {
if(townMap.size()>=1){
r.strand_town = isStandTown(townMap, r.new_district, r.township)
}
results << [r.id, r.province, r.city, r.district, r.lng, r.lat, r.data_source, r.township, r.new_district, r.town_source, r.strand_town]
}
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
@@ -535,7 +629,7 @@ class TownStandardBatch extends BaseBatch{
// //通过调取地图api接口获取乡镇街道
// if(!saveObj.township){
def gaodeMap = BaiduMapUtils.geocoding(location.lat+","+location.lon)
// def gaodeMap = BaiduMapUtils.geocoding(location.lat+","+location.lon)
// if(gaodeMap && gaodeMap.addressComponent) {
// def addressComponent = gaodeMap.addressComponent
// if(addressComponent){
@@ -608,20 +702,21 @@ class TownStandardBatch extends BaseBatch{
}
def updateTownData(){
def i = 0
def i = 49
while(i<50) {
def fileName = "/home/goojia/gj/new_poi/output/no_township_result_${i}.csv"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\数据建设\\乡镇街道\\待补充乡镇街道\\结果\\no_township_result_${i}.csv"
fileName = "D:\\数据\\数据建设\\乡镇街道\\待补充乡镇街道\\结果-地图-验证乡镇\\no_township_result_${i}.csv"
}
File file = new File(fileName);
if (file.exists()) {
def rets = readCSV3(fileName)
println file.getAbsoluteFile().getName() +" "+(rets.size()*100/200000)+"%"+" "+rets.size()
// if(rets){
// updateToEs(rets)
// }
if(rets){
// updateToEs(rets)//不能用这种方法修改,否则其它字段都将丢失
// updateToEs2(rets)
updateDistTownship(rets)
}
}
i++
}
@@ -675,4 +770,72 @@ class TownStandardBatch extends BaseBatch{
println "结束时间 "+new Date()
}
def updateToEs2(rows) {
for (int i = 0; i < rows.size(); i++) {
def map = rows[i]
updateMapPoi(map.id, map.township)
}
}
def updateMapPoi(def id, def township){
def header = [:]
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString()
String seawaterUrl = "http://192.168.0.89:9200/map_poi_2024/_update_by_query?conflicts=proceed&scroll_size=1000&wait_for_completion=false"
def body = """
{
"script": {
"source": "ctx._source.township='${township}'",
"lang": "painless"
},
"query": {
"term": {
"_id": "${id}"
}
}
}
"""
HttpRequest request = new HttpRequest()
try{
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
def response = JSONObject.fromObject(res)
}catch(Exception e){
println "error : "+e.getMessage()
}
}
def updateDistTownship(rets){
for(def m : rets){
def obj = CommonDatautils.findDistrictById(m.id)
if(obj.township != m.township){
def udata = new BasicDBObject()
udata.put("township", m.township)
if(!m.new_district.contains(m.old_district)){
def lists = CommonDatautils.findAllDistrictByName(obj.city, null, obj.name, obj.cat)
if(lists.size()<2){
continue
}
def region = RegionData.formatRegion(m.city, m.new_district)
if(region){
def dbo = CommonDatautils.findDistrict(m.city, region, obj.name, obj.cat)
if(dbo){
udata.put("status", 1)
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(m.id)), new BasicDBObject('$set', udata))
println "删除 "+m.city +" "+obj.name+" "+m.id+" "+obj.cat
continue
}else{
udata.put("region", region)
}
}
}
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(m.id)), new BasicDBObject('$set', udata))
}
}
}
}
@@ -1,6 +1,7 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.core.RegionData
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.util.CosineSimilarAlgorithm
@@ -11,6 +12,7 @@ import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.impl.AMQImpl
import net.sf.json.JSONObject
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
@@ -38,8 +40,8 @@ class TownshipStrandUpdate {
static main(args) {
def task = new TownshipStrandUpdate()
task.process()
task.updateTown()
// task.process()
// task.updateTown()
// task.stat()
// task.updateDiffRegionTown()
@@ -49,6 +51,9 @@ class TownshipStrandUpdate {
// task.updateDevMapping()
// println new Date().getTime()
// task.updateSpecCityTown()
task.updateTownshipAlias()
}
@@ -629,4 +634,69 @@ class TownshipStrandUpdate {
}
}
def updateTownshipAlias(){
def fileName = "D:\\数据\\数据建设\\乡镇街道\\20250219--乡镇街道添加别名.xlsx"
ExcelUtils.modify(fileName,{d->
def q = new BasicDBObject("type","乡镇街道").append("city", d["A"]).append("district",d["B"]).append("street",d["C"])
def list = ADMINISTRATIVE_MAPPING_COLL.find(q).toList()
def newAlias = d['D']
if(list.size()==0){
list = q.put("street", d['D'])
newAlias = d['C']
list = ADMINISTRATIVE_MAPPING_COLL.find(q).toList()
}
if(list && list.size()==1){
def obj = list[0]
def alias = obj.alias
if(!alias){
alias = []
}
if(!alias.contains(newAlias)){
alias << newAlias
}
if(alias){
def udata = new BasicDBObject()
udata.put("alias", alias)
udata.put("m_date", new Date())
udata.put("operator","陈杰")
ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',udata))
}
// updateStreetAlias(obj._id.toString(), alias.join("','"))
}else{
println d["C"] +" "+ d["D"]
}
})
}
def updateStreetAlias(def id, def alias){
def header = [:]
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString()
String seawaterUrl = "http://192.168.0.89:9200/administrative_mapping/_update_by_query?conflicts=proceed&scroll_size=1000&wait_for_completion=false"
def body = """
{
"script": {
"source": "ctx._source.alias=['${alias}']",
"lang": "painless"
},
"query": {
"term": {
"_id": "${id}"
}
}
}
"""
HttpRequest request = new HttpRequest()
try{
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
def response = JSONObject.fromObject(res)
}catch(Exception e){
println "error : "+e.getMessage()
}
}
}
@@ -1,9 +1,16 @@
package com.fangjia.script.gj.district
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.addrepo.InvokeHttpRequest
import com.fangjia.script.statistics.poi.AmapPoi
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
import com.fangjia.script.temp.shenjian.util.HttpUtils
import com.fangjia.script.util.HttpRequestUtil
import com.mongodb.DBObject
import net.sf.json.JSONArray
import net.sf.json.JSONObject
import org.apache.commons.collections.CollectionUtils
import org.apache.commons.lang.StringUtils
class FangjiaClientUtils {
@@ -347,7 +354,7 @@ class FangjiaClientUtils {
def jsonObj = null
def content = null
def url = null
address = address.replace("{","")
address = address.toString().replace("{","")
int retryCount = 0
while (retryCount < 3) {
url = "http://192.168.0.93:8080/FangJiaNlpService/formatNinelevel?address="+URLEncoder.encode(address,"UTF-8")+"&complete=false&useMap=false&addressAssessment=true&fields=dev_confi"
@@ -74,11 +74,11 @@ class OfficeFieldCleanBatch extends BaseBatch{
// addBlock()
// addFieldValue()
// updateErrorField()
updateErrorField()
// export()
// test()
// addOffice()
fileAddBlock()
// fileAddBlock()
}
def export(){
@@ -134,33 +134,196 @@ class OfficeFieldCleanBatch extends BaseBatch{
}
def updateErrorField(){
// def offices = CommonDatautils.findAllDistrict("office", ["city","estate_charge","addr_alias","property_life", "total_floor","finish_building_date","real_use_percent", "land_car_site", "underground_car_site","parking_fee"])
def offices = CommonDatautils.findAllDistricts("ÉϺ£","office", ["city","estate_charge","addr_alias","property_life", "total_floor","finish_building_date","real_use_percent", "land_car_site", "underground_car_site","parking_fee"])
def offices = CommonDatautils.findAllDistrict("office", ["underground_car_site"])
// def offices = CommonDatautils.findAllDistricts("上海","office", ["build_area"])
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(offices.size())
for(def i=0;i<offices.size();i++){
final def o = offices[i]
fixedThreadPool.execute(new Runnable() {
void run() {
if(o._id.toString()=="5954d66f9aed7b45d3785a40"){
println o._id.toString()
}
// fixedThreadPool.execute(new Runnable() {
// void run() {
try{
totalCount.addAndGet(1)
updateDistField(o)
println totalCount.get() +" "+o.city
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
// }
// });
}
latch.await()
}
def updateDistField(o){
def addr_alias = o.addr_alias
def udata = new BasicDBObject()
if(o.build_area){
def newVal = FiledStrand.processArea(o.build_area)
if(newVal && newVal.toString()!=o.build_area){
println o._id.toString()+" old:"+o.build_area +" new:"+newVal
udata.put("build_area", newVal.toString())
}
}
if(o.all_area){
def newVal = FiledStrand.processArea(o.all_area)
if(newVal.toString()!=o.all_area){
println o._id.toString()+" old:"+o.all_area +" new:"+newVal
udata.put("all_area", newVal.toString())
}
}
if(o.standard_area){
def newVal = FiledStrand.processStandardArea(o.standard_area)
if(newVal.toString()!=o.standard_area){
println o._id.toString()+" old:"+o.standard_area +" new:"+newVal
udata.put("standard_area", newVal.toString())
}
}
if(o.standard_height){
def newVal = FiledStrand.processStandardHeight(o.standard_height)
if(newVal.toString()!=o.standard_height){
println o._id.toString()+" old:"+o.standard_height +" new:"+newVal
udata.put("standard_height", newVal.toString())
}
}
if(o.clear_height){
def newVal = FiledStrand.processStandardHeight(o.clear_height)
if(newVal.toString()!=o.clear_height){
println o._id.toString()+" old:"+o.clear_height +" new:"+newVal
if(newVal){
udata.put("clear_height", newVal.toString())
}else{
udata.put("clear_height", "")
}
}
}
if(o.lobby_height){
def newVal = FiledStrand.processLobbyHeight(o.lobby_height)
if(newVal.toString()!=o.lobby_height){
println o._id.toString()+" old:"+o.lobby_height +" new:"+newVal
if(newVal){
udata.put("lobby_height", newVal.toString())
}else{
udata.put("lobby_height", "")
}
}
}
if(o.cubage_percent){
def newVal = FiledStrand.processCubagePercent(o.cubage_percent)
if(newVal && newVal.toString()!=o.cubage_percent){
println o._id.toString()+" old:"+o.cubage_percent +" new:"+newVal
udata.put("cubage_percent", newVal.toString())
}
}
if(o.virescence_percent){
def newVal = FiledStrand.processVirescencePercent(o.virescence_percent)
if(newVal.toString()!=o.virescence_percent){
println o._id.toString()+" old:"+o.virescence_percent +" new:"+newVal
udata.put("virescence_percent", newVal.toString())
}
}
if(o.elevators_count){
def newVal1 = FiledStrand.processElevators(o.elevators_count)
if(newVal1.toString()!=o.elevators_count){
println o._id.toString()+" old:"+o.elevators_count +" new:"+newVal1
udata.put("elevators_count", newVal1.toString())
}
def newVal2 = null
if(o.elevators_count.contains("客梯")){
newVal2 = FiledStrand.processElevators(o.elevators_count.toString().substring(o.elevators_count.indexOf("客梯")))
if(newVal2.toString()!=o.elevators_count){
println o._id.toString()+" old:"+o.elevators_count +" new:"+newVal2
udata.put("person_elevators_count", newVal2.toString())
}
}
def newVal3 = null
if(o.elevators_count.contains("货梯")){
newVal3 = FiledStrand.processElevators(o.elevators_count.toString().substring(o.elevators_count.indexOf("货梯")))
if(newVal3.toString()!=o.elevators_count){
println o._id.toString()+" old:"+o.elevators_count +" new:"+newVal3
udata.put("goods_elevator_count", newVal3.toString())
}
}
if(newVal2 && newVal3){
def total = newVal2.toInteger() + newVal3.toInteger()
udata.put("elevators_count", total.toString())
println o._id.toString()+" 总电梯:"+total
}
}
if(o.goods_elevator_count){
def newVal = FiledStrand.processElevators(o.goods_elevator_count)
if(newVal.toString()!=o.goods_elevator_count){
println o._id.toString()+" old:"+o.goods_elevator_count +" new:"+newVal
udata.put("goods_elevator_count", newVal.toString())
}
}
if(o.person_elevators_count){
def newVal = FiledStrand.processElevators(o.person_elevators_count)
if(newVal.toString()!=o.person_elevators_count){
println o._id.toString()+" old:"+o.person_elevators_count +" new:"+newVal
if(newVal){
udata.put("person_elevators_count", newVal.toString())
}else{
udata.put("person_elevators_count", "")
}
}
}
if(o.land_car_site){
def newVal = FiledStrand.processCarSite(o.land_car_site)
if(newVal.toString()!=o.land_car_site){
println o._id.toString()+" old:"+o.land_car_site +" new:"+newVal
if(newVal){
udata.put("land_car_site", newVal.toString())
}else{
udata.put("land_car_site", "")
}
}
}
if(o.underground_car_site){
def newVal = FiledStrand.processCarSite(o.underground_car_site)
if(newVal.toString()!=o.underground_car_site){
println o._id.toString()+" old:"+o.underground_car_site +" new:"+newVal
if(newVal){
udata.put("underground_car_site", newVal.toString())
}else{
udata.put("underground_car_site", "")
}
}
}
if(o.is_register){
def newVal = FiledStrand.processRegister(o.is_register)
if(newVal.toString()!=o.is_register){
println o._id.toString()+" old:"+o.is_register +" new:"+newVal
udata.put("is_register", newVal.toString())
}
}
if(o.property_life){
def newVal = FiledStrand.processPropertyLife(o.property_life)
udata.put("property_life", newVal)
@@ -173,7 +336,10 @@ class OfficeFieldCleanBatch extends BaseBatch{
if(o.finish_building_date){
def newVal = FiledStrand.processFinishBuildingDate(o.finish_building_date)
udata.put("finish_building_date", newVal.toString())
if(newVal && newVal.toString()!=o.finish_building_date){
println o._id.toString()+" old:"+o.finish_building_date +" new:"+newVal
udata.put("finish_building_date", newVal.toString())
}
}
if(o.real_use_percent){
@@ -181,21 +347,14 @@ class OfficeFieldCleanBatch extends BaseBatch{
udata.put("real_use_percent", newVal.toString())
}
if(o.underground_car_site){
def newVal = FiledStrand.processCarSite(o.underground_car_site)
udata.put("underground_car_site", newVal.toString())
}
if(o.land_car_site){
def newVal = FiledStrand.processCarSite(o.land_car_site)
udata.put("land_car_site", newVal.toString())
}
if(o.parking_fee){
def newVal = FiledStrand.processCarSiteRent(o.parking_fee)
udata.put("parking_fee", newVal)
}
def addr_alias = o.addr_alias
if(addr_alias){
def addrs = []
for(def arr : addr_alias){
@@ -214,13 +373,7 @@ class OfficeFieldCleanBatch extends BaseBatch{
}
}
// if(o.estate_charge && !o.estate_charge.isNumber()){
//// def val = o.estate_charge.toString().toDouble()
//// if(val>60 || val < 0.35){
//// println val
// udata.put("estate_charge", null)
//// }
// }
if(udata){
udata.put("m_date", new Date())
@@ -27,7 +27,7 @@ class EsMonthHouseStat {
// task.statDistHouse()
// task.statRent()
task.statOfficeHouse()
// task.statOfficeHouse()
}
def statRent(){
@@ -5,7 +5,6 @@ import com.fangjia.core.RegionData
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.Bytes
import com.mongodb.DBCollection
import net.sf.json.JSONObject
import org.apache.commons.lang.math.RandomUtils
@@ -18,9 +17,8 @@ import java.util.regex.Pattern
*/
class RegionPointCrawl {
static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline_2025")
static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
static DBCollection GAODECITYCODECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "gaode_city_code")
static DBCollection OLD_POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
static main(args) {
def task = new RegionPointCrawl();
@@ -28,19 +26,16 @@ class RegionPointCrawl {
//4个直辖市 北京市、上海市、天津市、重庆市
//5个自治区 "内蒙古自治区","广西壮族自治区","西藏自治区","宁夏回族自治区","新疆维吾尔自治区"
// def provinces = ["河北省", "山西省","吉林省","辽宁省", "黑龙江省", "陕西省", "甘肃省", "青海省", "山东省", "福建省", "浙江省", "台湾省", "河南省", "湖北省", "湖南省", "江西省", "江苏省", "安徽省", "广东省", "海南省", "四川省", "贵州省", "云南省"]
//已跑:甘肃省,河北省,山西省,"吉林省","辽宁省", "黑龙江省", "陕西省", "青海省", "山东省", "福建省", "浙江省", "台湾省", "河南省", "湖北省", "湖南省", "江西省", "江苏省", "安徽省", "广东省", "海南省", "四川省", "贵州省","云南省"
// def provinces = ["北京市","上海市","天津市","重庆市" ]
// def index = 0
// for(def p : provinces){
// index ++
// println index +"/"+provinces.size() +" "+p
// task.crawlCityAndDistrictCode(p);
// println "城市 区域 code 完成"
// task.run(p);
// println "区域围栏 完成"
// }
task.statSame()
def provinces = ["甘肃省"]
def index = 0
for(def p : provinces){
index ++
println index +"/"+provinces.size() +" "+p
task.crawlCityAndDistrictCode(p);
println "城市 区域 code 完成"
task.run(p);
println "区域围栏 完成"
}
}
@@ -52,10 +47,7 @@ class RegionPointCrawl {
def cityCodes = findCityCode(pro);
if(cityCodes){
for(def cityMap : cityCodes){
def codeMap = findAllDistrictCode(cityMap.city_code, "district");
// if(!codeMap){
// codeMap = findAllDistrictCode(cityMap.city_code, "street");
// }
def codeMap = findAllDistrictCode(cityMap.city_code);
def index = 0
for(def m : codeMap){
index ++
@@ -101,7 +93,7 @@ class RegionPointCrawl {
def saveGaoDeRegionCode(province, cityName,districts){
if(districts){
for(def dist : districts){
if(dist.level=="district" || dist.level=="street" ){
if(dist.level=="district" || cityName=="东莞市"){
saveGaoDeCityRegionCode(province, dist.name, dist.citycode, dist.adcode, dist.center, dist.level);
}
}
@@ -133,10 +125,10 @@ class RegionPointCrawl {
return new BasicDBObject("$regex", pattern);
}
def findAllDistrictCode(def citycode, def level){
def findAllDistrictCode(def citycode){
def q = new BasicDBObject()
q.put("city_code", citycode)
q.put("level", level)
q.put("level", "district")
def dbcursor = GAODECITYCODECOLL.find(q)
def rets = []
for(def d : dbcursor){
@@ -157,7 +149,6 @@ class RegionPointCrawl {
def o = GAODECITYCODECOLL.findOne(q)
if(!o){
println name +" "+adcode+" 新增"
q.put("c_date", new Date())
q.put("center", center)
GAODECITYCODECOLL.save(q);
@@ -242,62 +233,4 @@ class RegionPointCrawl {
def count = RandomUtils.nextInt(KeyList.size())
return KeyList.get(count);
}
def statSame(){
def dbcursor = POLYLINECOLL.find(new BasicDBObject()).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def index = 0
for(def d : dbcursor){
index++
def q = new BasicDBObject()
q.put("city",d.city)
q.put("region", d.region)
def obj = OLD_POLYLINECOLL.findOne(q)
if(obj){
if(obj.polyline!=d.polyline){
println index +" "+ d.city +" "+d.region+" 2次区域不同"
}
}else{
println index +" "+ d.city +" "+d.region+" 没有此区域"
}
}
}
def regionPolyline(){
def dbcursor = POLYLINECOLL.find(new BasicDBObject("level","district"))
def map = [:]
for(def d : dbcursor){
def resultMap = [:]
resultMap.type = "polygon"
def arrs = d.polyline.split("\\|")
def rets = []
for(def s : arrs){
def points = s.split(";")
def pointList = []
for(def pointStr : points){
def pointArr = pointStr.split(",")
pointList << [pointArr[0].toString().toDouble(), pointArr[1].toString().toDouble()]
}
rets << pointList
}
resultMap.coordinates = rets
if(rets.size()>1) {
resultMap.type = "MultiPolygon"
}
println d.city+" "+d.region+" "+rets.size()
def udata = new BasicDBObject()
udata.put("shape", resultMap)
def center = d.center
if(center){
def pointArrs = center.split(",")
udata.put("location", ["lon":pointArrs[0].toString().toDouble(), "lat":pointArrs[1].toString().toDouble()])
}
def fullCity = fullCityMap.get(d.city)
map.put(fullCity.city+"|@|"+d.region, udata)
}
return map;
}
}
@@ -40,16 +40,58 @@ class AddPriceTest {
def run(){
def filePath = "D:\\数据\\价格\\惠州.xlsx"
// addGXDPrice(filePath)
addCcbPrice(filePath)
// addCcbPrice(filePath)
// addThirdPrice(filePath)
// addFjwPrice(filePath)
// exportPkPrice()
// addRegion()
// addTradePrice(filePath)
addThirdSamplePrice("D:\\数据\\建行\\房产估值\\指导价\\级联、国信达价格导入_0313.xlsx")
}
def addThirdSamplePrice(filePath){
ExcelUtils.modify(filePath,{d->
def dist = CommonDatautils.findDistrictById(d['A'])
def o = new BasicDBObject()
o.put("city" , dist.city)
o.put("region" , dist.region)
o.put("district_name" , dist.name)
o.put("isGuidancePrice" , 0)
o.put("category", "district")
o.put("c_date" , new Date())
o.put("user_name" , "房价网")
if(d['B']){
o.put("price", d['B'].toString().toDouble().toInteger())
o.put("source", "JL")
if(o.containsKey("_id")){
o.removeField("_id")
}
confirm_price_date.save(o)
}
if(d['C']){
o.put("price", d['C'].toString().toDouble().toInteger())
o.put("source", "GXD")
if(o.containsKey("_id")){
o.removeField("_id")
}
confirm_price_date.save(o)
}
if(d['D']){
o.put("price", d['D'].toString().toDouble().toInteger())
o.put("source", "FXT")
if(o.containsKey("_id")){
o.removeField("_id")
}
confirm_price_date.save(o)
}
})
}
def addTradePrice(filePath){
ExcelUtils.multiThreadingModify(filePath, { d ->
if(d['Y']){
@@ -19,7 +19,7 @@ DBCollection COMMUNITY_CONN = MongoUtils.getCollectionByName2("fangjia", "commun
//def s = getTownshipInfo("118.702988,39.740979")
def filePath = "D:\\数据\\涉农\\县政府(最终).xlsx"
ExcelUtils.modify(filePath, { row ->
if(!row['I'] && row['I']=='±¦Ó¦ÏØ'){
if(!row['I'] && row['I']=='建瓯市'){
def rets = EsSearchUtils.findPois(row['B'], row['C'], row['C']+"人民政府")
if(rets){
def obj = findOneByName(rets, row['C']+"人民政府")
@@ -1,23 +1,34 @@
package com.fangjia.script.gj.xiamen
import com.fangjia.core.util.GeoUtils
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.BaseBatch
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.util.*
import com.fangjia.script.util.DateUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.ProvinceData
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.AMQP
import net.sf.json.JSONObject
import org.bson.types.ObjectId
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
import com.fangjia.script.common.DataHandleUtils
class SeaweedDataSynch extends BaseBatch{
static def targetUrl = "http://192.168.0.12:9200/seaweed/"
static def targetUrl = "http://192.168.0.89:9200/seaweed/"
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
@@ -126,17 +137,44 @@ class SeaweedDataSynch extends BaseBatch{
mcaDistMap = CommonDatautils.queryDistNameCityByMapping()
// priceMap = findAllDistrictSellPrice()
def citys = CommonDatautils.findAllCity().reverse()
def citys = ["³±ÖÝ"]//CommonDatautils.findAllCity()
def index = 0
for(def c : citys){
index++
updating(c, "district")
def dists = updating(c, "district")
updateEs(c, dists)
updating(c, "office")
updating(c, "shop")
println index +" "+ c + " 完成"
}
}
def updateEs(city, dists){
def data = fullCityMap.get(city)
def distEsDatas = findEsSeaweedDatas(data.city)
def resultMap = [:]
for(def d : dists){
def esObj = distEsDatas.get(d._id.toString())
def map = [:]
d.each {k,v->
if(k!="_id"){
def esVal = esObj.get(k)
if(v!=esVal){
map.put(k, v)
}
}
}
if(map){
resultMap.put(d._id.toString(), map)
}
}
return resultMap
}
def updating(city, cat){
def dists = CommonDatautils.findAllDistrictsFromMg(city, cat, ["city","region","name","address","block","mapping_alias","addr_alias","cat","estate_type2","status","township","lng","lat","build_type","household_count","committee","estate_charge","car_site_count","property_life","house_property","build_area","all_area","build_structure","developer","estate_company","cubage_percent","virescence_percent","loudong_count","schools","main_type","main_type_area","create_year"])
def rets = new CopyOnWriteArrayList()
@@ -148,15 +186,13 @@ class SeaweedDataSynch extends BaseBatch{
void run() {
try{
totalCount.addAndGet(1)
def map = createData(dist)
//往100 上同步
updateData(map)
//Íù12 esÉÏͬ²½
// def newUrl = targetUrl +"_doc/"+map.id
// CommonDatautils.httpSaveData(newUrl, map)
rets << map
// //Íù89 esÉÏͬ²½
// def newUrl = targetUrl +"_doc/"+map._id
// CommonDatautils.httpSaveData89(newUrl, map)
println city +" "+cat +" "+totalCount.get() +"/"+dists.size()+" "+dist.name+" "+map._id
} catch (Exception e) {
@@ -172,6 +208,33 @@ class SeaweedDataSynch extends BaseBatch{
return rets;
}
def updateMapPoi(def id, def township){
def header = [:]
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString()
String seawaterUrl = "http://192.168.0.89:9200/seaweed/_update_by_query?conflicts=proceed&scroll_size=1000&wait_for_completion=false"
def body = """
{
"script": {
"source": "ctx._source.township='${township}'",
"lang": "painless"
},
"query": {
"term": {
"_id": "${id}"
}
}
}
"""
HttpRequest request = new HttpRequest()
try{
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
def response = JSONObject.fromObject(res)
}catch(Exception e){
println "error : "+e.getMessage()
}
}
def updateData(map){
def udata = new BasicDBObject()
map.each{k,v->
@@ -190,14 +253,14 @@ class SeaweedDataSynch extends BaseBatch{
if(province){
map.province = ProvinceData.getFullProvince(province)
}
map.fangjia_province = province
// map.fangjia_province = province
def data = fullCityMap.get(dist.city)
if(data){
map.city = data.city
area_code = data.mca_city_code
}
map.fangjia_city = dist.city
// map.fangjia_city = dist.city
def distData = mcaDistMap.get(dist.city+"#"+dist.region)
if(distData){
@@ -207,9 +270,8 @@ class SeaweedDataSynch extends BaseBatch{
// map.region = dist.region
println dist.city +" "+dist.region+" 民政部中没有找到相应的区域全称"
}
map.area_code = area_code
map.fangjia_region = dist.region
// map.area_code = area_code
// map.fangjia_region = dist.region
map.name = dist.name
@@ -384,6 +446,14 @@ class SeaweedDataSynch extends BaseBatch{
DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(row['A'])), new BasicDBObject('$set', udata))
}
})
}
def findEsSeaweedDatas(city){
def map = new HashMap()
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city))
ElasticUtils89.executeQuery("192.168.0.89", "seaweed", queryBuilder, { String id, doc->
map.put(id, doc)
})
return map;
}
}
@@ -310,7 +310,7 @@ class SMSOutgoingListener extends BaseBatch {
if(StringUtils.isNotBlank(checkCode)){
JSONObject con = new JSONObject()
con.put("code", checkCode);
return AliyunSmsClient.sendSms(mobile, "SMS_135220034", con.toString())
return AliyunSmsClient.sendSms(mobile, "SMS_482955117", con.toString())
}
}
int resCode = SingletonClient.getClient().sendSMS([mobile] as String[],content, 5)