写字楼区域围栏验证

This commit is contained in:
2023-04-21 15:04:21 +08:00
parent 343c3a0cec
commit ae87f3f055
@@ -0,0 +1,515 @@
package com.fangjia.script.gj.crawl
import com.fangjia.core.RegionData
import com.fangjia.core.util.DataUtil
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.looplineinfo.Point
import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import com.mongodb.Bytes
import com.mongodb.DBCollection
import groovy.sql.Sql
import java.sql.Connection
import java.sql.DriverManager
import java.sql.PreparedStatement
import java.sql.Statement
import java.util.regex.Pattern
class OfficeProcess {
static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
def sqlInstance = Sql.newInstance("jdbc:clickhouse://192.168.0.216:8123/etl?max_memory_usage=50000000",
"goojia", "goojia123456", "ru.yandex.clickhouse.ClickHouseDriver")
static main(args) {
def filePath = "D:\\数据\\写字楼\\6城未匹配上的写字楼.xlsx"
def task = new OfficeProcess();
// task.run("D:\\数据\\写字楼\\6城合并写字楼.xlsx")
// task.addPropertyType2("深圳")
// task.standardName(filePath)
// task.nameRepeatFiter(filePath)
// task.addressFiter(filePath)
// task.nameFiter(filePath)
// task.typeFiter(filePath)
// task.nameFlag(filePath)
//// task.updateFlagToDB(filePath)
// task.updateFlagToDB2(filePath)
task.checkRegion()
}
def nameRepeatFiter(filePath){
def keys = []
ExcelUtils.modify(filePath, { row ->
if (!row['L'] && row['B']) {
def city = row['B']
def region = row['C']
def name = ""
if(row['K']){
name = row['K']
}else{
name = row['D']
}
def key = city+"##"+region+"##"+name
if(keys.contains(key)){
row['L'] = "名称去重"
println key
}else{
keys << key
}
}
})
}
def nameFiter(filePath){
ExcelUtils.modify(filePath, { row ->
def name = row['K']
if(!row['L'] && !name){
row['L'] = "名称过滤"
}
if(!row['L'] && name){
if(name.contains("公司")){
row['L'] = "名称过滤"
}
}
if(!row['L'] && name){
if(isStandardRoad(name)){
row['L'] = "名称过滤"
}
}
if(!row['L'] && name){
def fiters = ["厂房","测试", "服务","综合服务","酒店","深圳特专","工业","经济","产业","居委会","宿舍","购物","休闲","文化","训练","出租","浙菜","值班岗","招租","商住","办公室","配套","美食","博物馆","物流","商铺","公社","商城","值班室","暂停","创业社区","招商","宾馆","器材","教育","门诊","妇产科","商贸城","电子城","超市","零售"]
for(def key : fiters){
if(name.contains(key)){
row['L'] = "名称过滤"
break;
}
}
}
if(!row['L'] && name){
def fiters = ["大厦","裙楼",'花园','大楼','大堂','南塔','北京','大道','主楼','中国','中心','广场','鸟巢','小区','单位','办公','北塔'
,'写字楼','检验楼','办公楼','接待区','商务楼','文体楼','市中心','产业园','科技楼','行政楼','病房43、44、45','律己楼','调度楼','辉腾楼','总务楼'
,'7幢货梯','现厂B栋','(马连道南街)','中央电视台','后座','西大楼','B&C','商业用楼','财务中心','地铁','一瓶','外贸楼','业务楼','研发大楼','结算中心','马场','商务']
for(def key : fiters){
if(name == key){
row['L'] = "名称过滤"
break;
}
}
}
if(!row['L'] && row['J'] && name){
if(row['J'].toString().contains("商务写字楼") && name.contains("公寓")){
row['L'] = "名称过滤"
}
}
if(!row['L'] && name){
if(findPattern(name,'^[A-Za-z0-9]+$') && name.toString().length()<=3){
row['L'] = "名称过滤"
}else if(name.toString().length()<2){
row['L'] = "名称过滤"
}
}
if(!row['L'] && name){
def filters = ['筹备组','小区','家属房','家属院','家属楼','自编','市场','项目','批发城','传达室','梯','型材','学院','工地','冷冻厂','机关楼','阅读室','议事厅','电脑城','童装城','派出所','保安室','散盘','店','工作站',"物业楼",'物业','驿站','锅炉房','处理中心',"销售中心",'政府','馆','捐赠点','百货','工厂','工场','检测场','职工之家','移动','中心站','楼门','职场','鲜花基地','医院','装饰城','实训楼','业务楼','科研楼','学校','交易城','管理楼','考场','通道','团购','楼上','运营中心','商业城']
for(def key : filters){
if(name.toString().endsWith(key)){
row['L'] = "名称过滤"
break;
}
}
}
})
}
def typeFiter(filePath){
ExcelUtils.modify(filePath, { row ->
if(!row['L'] && row['J']){
def fiters = ["商住","服务","住宅小区","酒店","商业综合体","住宅底商"]
for(def key : fiters){
if(row['J'].toString().contains(key)){
row['L'] = "类型过滤"
break;
}
}
}
if(!row['L'] && row['B'] && !row['J']){
row['L'] = "类型空白过滤"
}
})
}
def nameFlag(filePath){
ExcelUtils.modify(filePath, { row ->
if(!row['L'] && row['D']) {
def fiters = ["大厦","大楼","商务楼","商务中心","广场","写字楼","办公楼","科技园","国际大厦","商务大厦","商务广场","创园","创意","中心","营销中心"]
for(def key : fiters){
if(row['D'].toString().contains(key)){
if(row['D']=="广场"){
}
row['L'] = "名称关键字[添加]"
break;
}
}
}
if(!row['L'] && row['J']){
if(row['J'].toString().contains("园区") || row['J'].toString().trim()=="纯写字楼" || row['J'].toString().trim()=="标准写字楼"){
row['L'] = "类型[添加]"
}
if(!row['L'] && row['J'].toString().trim()=="纯写字楼 标准写字楼" || row['J'].toString().trim()=="商务中心" || row['J'].toString().trim()=="商务中心 标准写字楼"){
row['L'] = "类型[添加]"
}
}
if(!row['L'] && row['J'] && row['D']){
if(row['J'].toString().contains("商务写字楼") && !row['D'].toString().contains("公寓")){
row['L'] = "类型[添加]"
}
}
if(!row['L'] && row['J'] && row['D']){
if(row['J'].toString().contains("纯写字楼")){
row['L'] = "类型[添加]"
}
}
})
}
def addressFiter(filePath){
def keyMap = [:]
ExcelUtils.modify(filePath, { row ->
if(!row['L'] && row['F']){
def city = row['B']
def region = row['C']
def name = ""
if(row['K']){
name = row['K']
}else{
name = row['D']
}
def address = row['F']
def key = city+"##"+region+"##"+address
def keys = keyMap.keySet()
if(keys.contains(key)){
def values = keyMap.get(key)
if(values){
for(def n : values){
def score = CosineSimilarAlgorithm.getSimilarityRatio(name, n)
if(score>=0.8){
row['L'] = "地址去重"
break;
}
println address +" "+ name+" "+n +" "+score
}
}
}else{
def values = keyMap.get(key)
if(!values){
values = []
keyMap.put(key, values)
}
values << name
}
}
});
}
def standardName(filePath){
def index = 0
ExcelUtils.modify(filePath, { row ->
index++
if(!row['K'] && row['D']){
def name = row['D']
def newName = name.replaceAll("-|·|(|)|//|\\?|“|”","").replaceAll("\\w+幢\\w+座","").replaceAll(" ","").replaceAll("\\.","").replaceAll("丨","")
newName = newName.replaceAll("-?\\w+号商业楼","").replaceAll("\\?","")
newName = newName.replaceAll("\\w+楼\\w+室","")
newName = newName.replaceAll("\\w+号\\w+(座|幢)","")
newName = newName.replaceAll("(一|二|三|四|五|六|七|八|九|十)号楼(一|二|三|四|五|六|七|八|九|十)单元","")
newName = newName.replaceAll("(一|二|三|四|五|六|七|八|九|十|十一)?幢?(一|二|三|四|五|六|七|八|九|十|东|南|西|北)(塔楼|楼)","")
newName = newName.replaceAll("-?(第)?\\w+(号写字楼|号楼|号大楼|楼|座|栋|南楼|北楼|东楼|西楼|区|单元)","")
newName = newName.replaceAll("(一|二|三|四|五|六|七|八|九|十|壹|贰|叁|肆|伍|陆|柒|捌|玖|拾)栋","")
newName = newName.replaceAll('(东|南|西|北)座$',"")
newName = newName.replaceAll("1期","一期").replaceAll("2期","二期").replaceAll("3期","三期").replaceAll("4期","四期").replaceAll("5期","五期").replaceAll("6期","六期").replaceAll("7期","七期").replaceAll("8期","八期").replaceAll("9期","九期").replaceAll("10期","十期").replaceAll("1-5期","一-五期").replaceAll("5-6期","六-七期")
newName = newName.toUpperCase().replaceAll("(装修中)","").replaceAll("\\(\\)","")
if(newName){
def md = DataUtil.match(newName, Pattern.compile("^[0-9|a-z|A-Z|一|二|三|四|五|六|七|八|九|十|零|壹|贰|叁|肆|伍|陆|柒|捌|玖|拾|东|南|西|北|中]{0,1}[写字|办公|楼|座|区|栋|组|幢|0-9|东|南|西|北|中|号]?[0-9|a-z|A-Z|一|二|三|四|五|六|七|八|九|十|零|壹|贰|叁|肆|伍|陆|柒|捌|玖|拾|东|南|西|北|中|办公|写字]?[楼|座|区|栋|组|幢|0-9|东|南|西|北|中]\$"))
if(md){
newName = ''
}
}
row['K'] = newName
}
})
}
def addPropertyType2(runCity){
def map = [:]
def datas = ExcelUtils.read("D:\\数据\\写字楼\\6城未匹配写字楼.xlsx",0,1)
if(datas){
for(def d : datas){
def key = d['B']+"##"+d['C']+"##"+d['D']
def list = map.get(key)
if(!list){
list = []
map.put(key, list)
}
if(d['M'] && !list.contains(d['M'])){
list << d['M']
}
if(d['U'] && !list.contains(d['U'])){
list << d['U']
}
}
}
def index = 0
ExcelUtils.multiThreadingModify("D:\\数据\\写字楼\\6城未匹配上的写字楼.xlsx", { row ->
index++
def city = row['B']
def region = row['C']
def name = row['D']
def types = map.get(city+"##"+region+"##"+name)
if(types){
row['J'] = types.join(" ")
}
})
}
def addPropertyType(){
def index = 0
ExcelUtils.modify("D:\\数据\\写字楼\\未匹配写字楼.xlsx", { row ->
index++
def city = row['B']
def region = row['C']
def name = row['D']
def types = findDatasByDB(city, region, name)
if(types){
row['J'] = types.join(" ")
}
println index
})
}
def findDatasByDB(city, region, name){
def sql = "select * from office where city='${city}' and region='${region}' and name='${name}'"
def types = []
sqlInstance.eachRow(sql, {r->
if(r.property_type && !types.contains(r.property_type)){
types << r.property_type
}
})
return types
}
def run(filepath){
def index = 0
ExcelUtils.multiThreadingModify(filepath,{row->
index++
def city = row['B']
def region = row['C']
def name = row['D']
def address = row['I']
if(city ){
def jsonObject = OpenApiClient.addressMatch(city, name, "office", false, region, null)
// if(jsonObject && jsonObject.precise == -1){
// jsonObject = OpenApiClient.addressMatch(city, address, "office", false, region, name)
// }
if (jsonObject != null && jsonObject.precise != -1) {
row['J'] = jsonObject.get("id")
row['K'] = jsonObject.get("city")
row['L'] = jsonObject.get("district")
row['M'] = jsonObject.get("name")
row['N'] = jsonObject.get("credit")
row['O'] = jsonObject.get("propertyType")
}
}
println index
})
}
def isStandardRoad(def s){
def pattern = Pattern.compile("[路|街|大道|巷]\\d+(号|弄)?");
def matcher = pattern.matcher(s)
if(matcher.find()){
return true;
}
if(s.toString().endsWith("路")||s.toString().endsWith("街")){
return true;
}
return false
}
def findPattern(def address, def regex) {
Pattern pattern = Pattern.compile(regex);
def matcher = pattern.matcher(address)
if (matcher.find()) {
return true
}
return false;
}
def updateFlagToDB(filePath){
def datas = ExcelUtils.read(filePath,0,1)
def total = datas.size()
def index = 0
for(def d : datas){
def sql = "alter table office update option='${d['L']}', format_name='${d['K']}' where city='${d['B']}' and region='${d['C']}' and name='${d['D']}';"
index++
sqlInstance.executeUpdate(sql)
println index+"/"+ total
}
}
def updateOption(city, region, name, source, flag){
def sql = "alter table office update region_fence='${flag}' where city='${city}' and region='${region}' and name='${name}' and source='${source}';"
sqlInstance.executeUpdate(sql)
}
def batchUpdate(rets){
// 初始化驱动
Class.forName("ru.yandex.clickhouse.ClickHouseDriver");
// url
String url = "jdbc:clickhouse://192.168.0.216:8123/etl";
// 用户名密码
String user = "goojia";
String password = "goojia123456";
// 登录
Connection connection = DriverManager.getConnection(url, user, password);
Statement stmt = connection.createStatement();
def count = 0
def sql = "alter table office update region_fence=? where city=? and region=? and name=? and source=?;"
PreparedStatement pstmt = connection.prepareStatement(sql)
for(def d : rets){
count++
pstmt.setString(1, d.flag)
pstmt.setString(2, d.city)
pstmt.setString(3, d.region)
pstmt.setString(4, d.name)
pstmt.setString(5, d.source)
println count+" "+d.city+" "+d.name
pstmt.addBatch()
if(count % 1 ==0) {
try {
pstmt.executeBatch()
} catch (Exception e) {
e.printStackTrace()
}
}
}
pstmt.executeBatch()
}
def updateFlagToDB2(filePath){
// 初始化驱动
Class.forName("ru.yandex.clickhouse.ClickHouseDriver");
// url
String url = "jdbc:clickhouse://192.168.0.216:8123/etl";
// 用户名密码
String user = "goojia";
String password = "goojia123456";
// 登录
Connection connection = DriverManager.getConnection(url, user, password);
Statement stmt = connection.createStatement();
def count = 0
def datas = ExcelUtils.read(filePath,0,1)
def sql = "alter table office update option=?, format_name=? where city=? and region=? and name=?;"
PreparedStatement pstmt = connection.prepareStatement(sql)
for(def d : datas){
if(d['B']){
count++
pstmt.setString(1, d['L'])
pstmt.setString(2, d['K'])
pstmt.setString(3, d['B'])
pstmt.setString(4, d['C'])
pstmt.setString(5, d['D'])
println count+" "+d['L']+" "+d['K']+" "+d['B']+" "+d['C']+" "+d['D']
pstmt.addBatch()
if(count % 100 ==0) {
try {
pstmt.executeBatch()
} catch (Exception e) {
}
}
}
}
pstmt.executeBatch()
}
def checkRegion(){
def citys = ["深圳","北京","广州","重庆","天津","成都","南京"]//"上海",
for(def city : citys){
def regionPointsMap = findRegionRange(city)
def addDatas = findAddDatasByClickhouse(city)
println "需处理的区域围栏:"+addDatas.size()
def index = 0
def rets = []
for(def data : addDatas){
index ++
if(data.lng){
def flag = isInArea(data.lng.toString().toDouble(), data.lat.toString().toDouble(), regionPointsMap.get(data.region))
def mark = flag?'T':'F'
// rets << ["city":data.city, "region":data.region, "name":data.name, "source":data.source, "flag":mark]
updateOption(data.city, data.region, data.name, data.source, mark)
println index+" "+data.region + " "+data.name+" "+ flag
}
}
// batchUpdate(rets)
}
}
def findAddDatasByClickhouse(city){
def sql = "select city,region, name,option,lng,lat,source from office where city='${city}' and option like '%添加%'"
def datas = []
sqlInstance.eachRow(sql, {r->
datas << ["city":r.city,"region":r.region,"name":r.name, "lat":r.lat, "lng":r.lng, "option":r.option,"source":r.source]
})
return datas
}
def findRegionRange(city){
def q = new BasicDBObject()
q.put("city",city)
def dbcursor = POLYLINECOLL.find(q)
def regionMap = [:]
for(def d : dbcursor){
def region = RegionData.formatRegion(city,d.region)
regionMap.put(region, d.polyline)
}
return regionMap
}
def isInArea(lng, lat, points){
List<Point> pointList = InsidePolygon.createPoint2(points);
def flag = InsidePolygon.isPtInPoly(lng, lat, pointList)
return flag;
}
}