写字楼区域围栏验证
This commit is contained in:
+515
@@ -0,0 +1,515 @@
|
||||
package com.fangjia.script.gj.crawl
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||
import com.fangjia.script.looplineinfo.Point
|
||||
import com.fangjia.script.util.CosineSimilarAlgorithm
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.Bytes
|
||||
import com.mongodb.DBCollection
|
||||
import groovy.sql.Sql
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.DriverManager
|
||||
import java.sql.PreparedStatement
|
||||
import java.sql.Statement
|
||||
import java.util.regex.Pattern
|
||||
|
||||
class OfficeProcess {
|
||||
|
||||
static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
|
||||
def sqlInstance = Sql.newInstance("jdbc:clickhouse://192.168.0.216:8123/etl?max_memory_usage=50000000",
|
||||
"goojia", "goojia123456", "ru.yandex.clickhouse.ClickHouseDriver")
|
||||
|
||||
static main(args) {
|
||||
def filePath = "D:\\数据\\写字楼\\6城未匹配上的写字楼.xlsx"
|
||||
def task = new OfficeProcess();
|
||||
|
||||
// task.run("D:\\数据\\写字楼\\6城合并写字楼.xlsx")
|
||||
// task.addPropertyType2("深圳")
|
||||
|
||||
// task.standardName(filePath)
|
||||
// task.nameRepeatFiter(filePath)
|
||||
// task.addressFiter(filePath)
|
||||
// task.nameFiter(filePath)
|
||||
// task.typeFiter(filePath)
|
||||
// task.nameFlag(filePath)
|
||||
//// task.updateFlagToDB(filePath)
|
||||
// task.updateFlagToDB2(filePath)
|
||||
|
||||
task.checkRegion()
|
||||
|
||||
}
|
||||
|
||||
def nameRepeatFiter(filePath){
|
||||
def keys = []
|
||||
ExcelUtils.modify(filePath, { row ->
|
||||
if (!row['L'] && row['B']) {
|
||||
def city = row['B']
|
||||
def region = row['C']
|
||||
def name = ""
|
||||
if(row['K']){
|
||||
name = row['K']
|
||||
}else{
|
||||
name = row['D']
|
||||
}
|
||||
def key = city+"##"+region+"##"+name
|
||||
if(keys.contains(key)){
|
||||
row['L'] = "名称去重"
|
||||
println key
|
||||
}else{
|
||||
keys << key
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def nameFiter(filePath){
|
||||
ExcelUtils.modify(filePath, { row ->
|
||||
def name = row['K']
|
||||
|
||||
if(!row['L'] && !name){
|
||||
row['L'] = "名称过滤"
|
||||
}
|
||||
if(!row['L'] && name){
|
||||
if(name.contains("公司")){
|
||||
row['L'] = "名称过滤"
|
||||
}
|
||||
}
|
||||
|
||||
if(!row['L'] && name){
|
||||
if(isStandardRoad(name)){
|
||||
row['L'] = "名称过滤"
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
if(!row['L'] && name){
|
||||
def fiters = ["厂房","测试", "服务","综合服务","酒店","深圳特专","工业","经济","产业","居委会","宿舍","购物","休闲","文化","训练","出租","浙菜","值班岗","招租","商住","办公室","配套","美食","博物馆","物流","商铺","公社","商城","值班室","暂停","创业社区","招商","宾馆","器材","教育","门诊","妇产科","商贸城","电子城","超市","零售"]
|
||||
for(def key : fiters){
|
||||
if(name.contains(key)){
|
||||
row['L'] = "名称过滤"
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(!row['L'] && name){
|
||||
def fiters = ["大厦","裙楼",'花园','大楼','大堂','南塔','北京','大道','主楼','中国','中心','广场','鸟巢','小区','单位','办公','北塔'
|
||||
,'写字楼','检验楼','办公楼','接待区','商务楼','文体楼','市中心','产业园','科技楼','行政楼','病房43、44、45','律己楼','调度楼','辉腾楼','总务楼'
|
||||
,'7幢货梯','现厂B栋','(马连道南街)','中央电视台','后座','西大楼','B&C','商业用楼','财务中心','地铁','一瓶','外贸楼','业务楼','研发大楼','结算中心','马场','商务']
|
||||
for(def key : fiters){
|
||||
if(name == key){
|
||||
row['L'] = "名称过滤"
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if(!row['L'] && row['J'] && name){
|
||||
if(row['J'].toString().contains("商务写字楼") && name.contains("公寓")){
|
||||
row['L'] = "名称过滤"
|
||||
}
|
||||
}
|
||||
|
||||
if(!row['L'] && name){
|
||||
if(findPattern(name,'^[A-Za-z0-9]+$') && name.toString().length()<=3){
|
||||
row['L'] = "名称过滤"
|
||||
}else if(name.toString().length()<2){
|
||||
row['L'] = "名称过滤"
|
||||
}
|
||||
}
|
||||
|
||||
if(!row['L'] && name){
|
||||
def filters = ['筹备组','小区','家属房','家属院','家属楼','自编','市场','项目','批发城','传达室','梯','型材','学院','工地','冷冻厂','机关楼','阅读室','议事厅','电脑城','童装城','派出所','保安室','散盘','店','工作站',"物业楼",'物业','驿站','锅炉房','处理中心',"销售中心",'政府','馆','捐赠点','百货','工厂','工场','检测场','职工之家','移动','中心站','楼门','职场','鲜花基地','医院','装饰城','实训楼','业务楼','科研楼','学校','交易城','管理楼','考场','通道','团购','楼上','运营中心','商业城']
|
||||
for(def key : filters){
|
||||
if(name.toString().endsWith(key)){
|
||||
row['L'] = "名称过滤"
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def typeFiter(filePath){
|
||||
ExcelUtils.modify(filePath, { row ->
|
||||
if(!row['L'] && row['J']){
|
||||
def fiters = ["商住","服务","住宅小区","酒店","商业综合体","住宅底商"]
|
||||
for(def key : fiters){
|
||||
if(row['J'].toString().contains(key)){
|
||||
row['L'] = "类型过滤"
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if(!row['L'] && row['B'] && !row['J']){
|
||||
row['L'] = "类型空白过滤"
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def nameFlag(filePath){
|
||||
ExcelUtils.modify(filePath, { row ->
|
||||
if(!row['L'] && row['D']) {
|
||||
def fiters = ["大厦","大楼","商务楼","商务中心","广场","写字楼","办公楼","科技园","国际大厦","商务大厦","商务广场","创园","创意","中心","营销中心"]
|
||||
for(def key : fiters){
|
||||
if(row['D'].toString().contains(key)){
|
||||
if(row['D']=="广场"){
|
||||
|
||||
}
|
||||
row['L'] = "名称关键字[添加]"
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(!row['L'] && row['J']){
|
||||
if(row['J'].toString().contains("园区") || row['J'].toString().trim()=="纯写字楼" || row['J'].toString().trim()=="标准写字楼"){
|
||||
row['L'] = "类型[添加]"
|
||||
}
|
||||
|
||||
if(!row['L'] && row['J'].toString().trim()=="纯写字楼 标准写字楼" || row['J'].toString().trim()=="商务中心" || row['J'].toString().trim()=="商务中心 标准写字楼"){
|
||||
row['L'] = "类型[添加]"
|
||||
}
|
||||
}
|
||||
|
||||
if(!row['L'] && row['J'] && row['D']){
|
||||
if(row['J'].toString().contains("商务写字楼") && !row['D'].toString().contains("公寓")){
|
||||
row['L'] = "类型[添加]"
|
||||
}
|
||||
}
|
||||
|
||||
if(!row['L'] && row['J'] && row['D']){
|
||||
if(row['J'].toString().contains("纯写字楼")){
|
||||
row['L'] = "类型[添加]"
|
||||
}
|
||||
}
|
||||
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
def addressFiter(filePath){
|
||||
def keyMap = [:]
|
||||
ExcelUtils.modify(filePath, { row ->
|
||||
if(!row['L'] && row['F']){
|
||||
def city = row['B']
|
||||
def region = row['C']
|
||||
def name = ""
|
||||
if(row['K']){
|
||||
name = row['K']
|
||||
}else{
|
||||
name = row['D']
|
||||
}
|
||||
def address = row['F']
|
||||
def key = city+"##"+region+"##"+address
|
||||
def keys = keyMap.keySet()
|
||||
if(keys.contains(key)){
|
||||
def values = keyMap.get(key)
|
||||
if(values){
|
||||
for(def n : values){
|
||||
def score = CosineSimilarAlgorithm.getSimilarityRatio(name, n)
|
||||
if(score>=0.8){
|
||||
row['L'] = "地址去重"
|
||||
break;
|
||||
}
|
||||
println address +" "+ name+" "+n +" "+score
|
||||
}
|
||||
}
|
||||
|
||||
}else{
|
||||
def values = keyMap.get(key)
|
||||
if(!values){
|
||||
values = []
|
||||
keyMap.put(key, values)
|
||||
}
|
||||
values << name
|
||||
}
|
||||
}
|
||||
|
||||
});
|
||||
}
|
||||
|
||||
def standardName(filePath){
|
||||
def index = 0
|
||||
ExcelUtils.modify(filePath, { row ->
|
||||
index++
|
||||
if(!row['K'] && row['D']){
|
||||
def name = row['D']
|
||||
def newName = name.replaceAll("-|·|(|)|//|\\?|“|”","").replaceAll("\\w+幢\\w+座","").replaceAll(" ","").replaceAll("\\.","").replaceAll("丨","")
|
||||
newName = newName.replaceAll("-?\\w+号商业楼","").replaceAll("\\?","")
|
||||
newName = newName.replaceAll("\\w+楼\\w+室","")
|
||||
newName = newName.replaceAll("\\w+号\\w+(座|幢)","")
|
||||
newName = newName.replaceAll("(一|二|三|四|五|六|七|八|九|十)号楼(一|二|三|四|五|六|七|八|九|十)单元","")
|
||||
newName = newName.replaceAll("(一|二|三|四|五|六|七|八|九|十|十一)?幢?(一|二|三|四|五|六|七|八|九|十|东|南|西|北)(塔楼|楼)","")
|
||||
newName = newName.replaceAll("-?(第)?\\w+(号写字楼|号楼|号大楼|楼|座|栋|南楼|北楼|东楼|西楼|区|单元)","")
|
||||
newName = newName.replaceAll("(一|二|三|四|五|六|七|八|九|十|壹|贰|叁|肆|伍|陆|柒|捌|玖|拾)栋","")
|
||||
newName = newName.replaceAll('(东|南|西|北)座$',"")
|
||||
newName = newName.replaceAll("1期","一期").replaceAll("2期","二期").replaceAll("3期","三期").replaceAll("4期","四期").replaceAll("5期","五期").replaceAll("6期","六期").replaceAll("7期","七期").replaceAll("8期","八期").replaceAll("9期","九期").replaceAll("10期","十期").replaceAll("1-5期","一-五期").replaceAll("5-6期","六-七期")
|
||||
newName = newName.toUpperCase().replaceAll("(装修中)","").replaceAll("\\(\\)","")
|
||||
|
||||
if(newName){
|
||||
def md = DataUtil.match(newName, Pattern.compile("^[0-9|a-z|A-Z|一|二|三|四|五|六|七|八|九|十|零|壹|贰|叁|肆|伍|陆|柒|捌|玖|拾|东|南|西|北|中]{0,1}[写字|办公|楼|座|区|栋|组|幢|0-9|东|南|西|北|中|号]?[0-9|a-z|A-Z|一|二|三|四|五|六|七|八|九|十|零|壹|贰|叁|肆|伍|陆|柒|捌|玖|拾|东|南|西|北|中|办公|写字]?[楼|座|区|栋|组|幢|0-9|东|南|西|北|中]\$"))
|
||||
if(md){
|
||||
newName = ''
|
||||
}
|
||||
}
|
||||
row['K'] = newName
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def addPropertyType2(runCity){
|
||||
def map = [:]
|
||||
def datas = ExcelUtils.read("D:\\数据\\写字楼\\6城未匹配写字楼.xlsx",0,1)
|
||||
if(datas){
|
||||
for(def d : datas){
|
||||
def key = d['B']+"##"+d['C']+"##"+d['D']
|
||||
def list = map.get(key)
|
||||
if(!list){
|
||||
list = []
|
||||
map.put(key, list)
|
||||
}
|
||||
if(d['M'] && !list.contains(d['M'])){
|
||||
list << d['M']
|
||||
}
|
||||
if(d['U'] && !list.contains(d['U'])){
|
||||
list << d['U']
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
def index = 0
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\写字楼\\6城未匹配上的写字楼.xlsx", { row ->
|
||||
index++
|
||||
def city = row['B']
|
||||
def region = row['C']
|
||||
def name = row['D']
|
||||
def types = map.get(city+"##"+region+"##"+name)
|
||||
if(types){
|
||||
row['J'] = types.join(" ")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def addPropertyType(){
|
||||
def index = 0
|
||||
ExcelUtils.modify("D:\\数据\\写字楼\\未匹配写字楼.xlsx", { row ->
|
||||
index++
|
||||
def city = row['B']
|
||||
def region = row['C']
|
||||
def name = row['D']
|
||||
def types = findDatasByDB(city, region, name)
|
||||
if(types){
|
||||
row['J'] = types.join(" ")
|
||||
}
|
||||
println index
|
||||
})
|
||||
}
|
||||
|
||||
def findDatasByDB(city, region, name){
|
||||
def sql = "select * from office where city='${city}' and region='${region}' and name='${name}'"
|
||||
def types = []
|
||||
sqlInstance.eachRow(sql, {r->
|
||||
if(r.property_type && !types.contains(r.property_type)){
|
||||
types << r.property_type
|
||||
}
|
||||
})
|
||||
return types
|
||||
}
|
||||
|
||||
def run(filepath){
|
||||
def index = 0
|
||||
ExcelUtils.multiThreadingModify(filepath,{row->
|
||||
index++
|
||||
def city = row['B']
|
||||
def region = row['C']
|
||||
def name = row['D']
|
||||
def address = row['I']
|
||||
if(city ){
|
||||
def jsonObject = OpenApiClient.addressMatch(city, name, "office", false, region, null)
|
||||
// if(jsonObject && jsonObject.precise == -1){
|
||||
// jsonObject = OpenApiClient.addressMatch(city, address, "office", false, region, name)
|
||||
// }
|
||||
if (jsonObject != null && jsonObject.precise != -1) {
|
||||
row['J'] = jsonObject.get("id")
|
||||
row['K'] = jsonObject.get("city")
|
||||
row['L'] = jsonObject.get("district")
|
||||
row['M'] = jsonObject.get("name")
|
||||
row['N'] = jsonObject.get("credit")
|
||||
row['O'] = jsonObject.get("propertyType")
|
||||
}
|
||||
}
|
||||
println index
|
||||
})
|
||||
}
|
||||
|
||||
def isStandardRoad(def s){
|
||||
def pattern = Pattern.compile("[路|街|大道|巷]\\d+(号|弄)?");
|
||||
def matcher = pattern.matcher(s)
|
||||
if(matcher.find()){
|
||||
return true;
|
||||
}
|
||||
if(s.toString().endsWith("路")||s.toString().endsWith("街")){
|
||||
return true;
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
|
||||
def findPattern(def address, def regex) {
|
||||
Pattern pattern = Pattern.compile(regex);
|
||||
def matcher = pattern.matcher(address)
|
||||
if (matcher.find()) {
|
||||
return true
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
def updateFlagToDB(filePath){
|
||||
def datas = ExcelUtils.read(filePath,0,1)
|
||||
def total = datas.size()
|
||||
def index = 0
|
||||
for(def d : datas){
|
||||
def sql = "alter table office update option='${d['L']}', format_name='${d['K']}' where city='${d['B']}' and region='${d['C']}' and name='${d['D']}';"
|
||||
index++
|
||||
sqlInstance.executeUpdate(sql)
|
||||
println index+"/"+ total
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def updateOption(city, region, name, source, flag){
|
||||
def sql = "alter table office update region_fence='${flag}' where city='${city}' and region='${region}' and name='${name}' and source='${source}';"
|
||||
sqlInstance.executeUpdate(sql)
|
||||
}
|
||||
|
||||
def batchUpdate(rets){
|
||||
// 初始化驱动
|
||||
Class.forName("ru.yandex.clickhouse.ClickHouseDriver");
|
||||
// url
|
||||
String url = "jdbc:clickhouse://192.168.0.216:8123/etl";
|
||||
// 用户名密码
|
||||
String user = "goojia";
|
||||
String password = "goojia123456";
|
||||
// 登录
|
||||
Connection connection = DriverManager.getConnection(url, user, password);
|
||||
Statement stmt = connection.createStatement();
|
||||
def count = 0
|
||||
def sql = "alter table office update region_fence=? where city=? and region=? and name=? and source=?;"
|
||||
PreparedStatement pstmt = connection.prepareStatement(sql)
|
||||
|
||||
for(def d : rets){
|
||||
count++
|
||||
pstmt.setString(1, d.flag)
|
||||
pstmt.setString(2, d.city)
|
||||
pstmt.setString(3, d.region)
|
||||
pstmt.setString(4, d.name)
|
||||
pstmt.setString(5, d.source)
|
||||
println count+" "+d.city+" "+d.name
|
||||
pstmt.addBatch()
|
||||
if(count % 1 ==0) {
|
||||
try {
|
||||
pstmt.executeBatch()
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
}
|
||||
}
|
||||
}
|
||||
pstmt.executeBatch()
|
||||
}
|
||||
def updateFlagToDB2(filePath){
|
||||
// 初始化驱动
|
||||
Class.forName("ru.yandex.clickhouse.ClickHouseDriver");
|
||||
// url
|
||||
String url = "jdbc:clickhouse://192.168.0.216:8123/etl";
|
||||
// 用户名密码
|
||||
String user = "goojia";
|
||||
String password = "goojia123456";
|
||||
// 登录
|
||||
Connection connection = DriverManager.getConnection(url, user, password);
|
||||
Statement stmt = connection.createStatement();
|
||||
def count = 0
|
||||
def datas = ExcelUtils.read(filePath,0,1)
|
||||
def sql = "alter table office update option=?, format_name=? where city=? and region=? and name=?;"
|
||||
PreparedStatement pstmt = connection.prepareStatement(sql)
|
||||
|
||||
for(def d : datas){
|
||||
if(d['B']){
|
||||
count++
|
||||
pstmt.setString(1, d['L'])
|
||||
pstmt.setString(2, d['K'])
|
||||
pstmt.setString(3, d['B'])
|
||||
pstmt.setString(4, d['C'])
|
||||
pstmt.setString(5, d['D'])
|
||||
println count+" "+d['L']+" "+d['K']+" "+d['B']+" "+d['C']+" "+d['D']
|
||||
pstmt.addBatch()
|
||||
if(count % 100 ==0) {
|
||||
try {
|
||||
pstmt.executeBatch()
|
||||
} catch (Exception e) {
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
pstmt.executeBatch()
|
||||
}
|
||||
|
||||
def checkRegion(){
|
||||
def citys = ["深圳","北京","广州","重庆","天津","成都","南京"]//"上海",
|
||||
for(def city : citys){
|
||||
def regionPointsMap = findRegionRange(city)
|
||||
def addDatas = findAddDatasByClickhouse(city)
|
||||
println "需处理的区域围栏:"+addDatas.size()
|
||||
def index = 0
|
||||
def rets = []
|
||||
for(def data : addDatas){
|
||||
index ++
|
||||
if(data.lng){
|
||||
def flag = isInArea(data.lng.toString().toDouble(), data.lat.toString().toDouble(), regionPointsMap.get(data.region))
|
||||
def mark = flag?'T':'F'
|
||||
// rets << ["city":data.city, "region":data.region, "name":data.name, "source":data.source, "flag":mark]
|
||||
updateOption(data.city, data.region, data.name, data.source, mark)
|
||||
println index+" "+data.region + " "+data.name+" "+ flag
|
||||
}
|
||||
}
|
||||
// batchUpdate(rets)
|
||||
}
|
||||
}
|
||||
|
||||
def findAddDatasByClickhouse(city){
|
||||
def sql = "select city,region, name,option,lng,lat,source from office where city='${city}' and option like '%添加%'"
|
||||
def datas = []
|
||||
sqlInstance.eachRow(sql, {r->
|
||||
datas << ["city":r.city,"region":r.region,"name":r.name, "lat":r.lat, "lng":r.lng, "option":r.option,"source":r.source]
|
||||
})
|
||||
return datas
|
||||
}
|
||||
|
||||
def findRegionRange(city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city",city)
|
||||
def dbcursor = POLYLINECOLL.find(q)
|
||||
def regionMap = [:]
|
||||
for(def d : dbcursor){
|
||||
def region = RegionData.formatRegion(city,d.region)
|
||||
regionMap.put(region, d.polyline)
|
||||
}
|
||||
return regionMap
|
||||
}
|
||||
|
||||
def isInArea(lng, lat, points){
|
||||
List<Point> pointList = InsidePolygon.createPoint2(points);
|
||||
def flag = InsidePolygon.isPtInPoly(lng, lat, pointList)
|
||||
return flag;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user