diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/OfficeProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/OfficeProcess.groovy new file mode 100644 index 000000000..0a57c35c6 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/OfficeProcess.groovy @@ -0,0 +1,515 @@ +package com.fangjia.script.gj.crawl + +import com.fangjia.core.RegionData +import com.fangjia.core.util.DataUtil +import com.fangjia.script.looplineinfo.InsidePolygon +import com.fangjia.script.looplineinfo.Point +import com.fangjia.script.util.CosineSimilarAlgorithm +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.util.OpenApiClient +import com.mongodb.BasicDBObject +import com.mongodb.Bytes +import com.mongodb.DBCollection +import groovy.sql.Sql + +import java.sql.Connection +import java.sql.DriverManager +import java.sql.PreparedStatement +import java.sql.Statement +import java.util.regex.Pattern + +class OfficeProcess { + + static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline") + def sqlInstance = Sql.newInstance("jdbc:clickhouse://192.168.0.216:8123/etl?max_memory_usage=50000000", + "goojia", "goojia123456", "ru.yandex.clickhouse.ClickHouseDriver") + + static main(args) { + def filePath = "D:\\数据\\写字楼\\6城未匹配上的写字楼.xlsx" + def task = new OfficeProcess(); + +// task.run("D:\\数据\\写字楼\\6城合并写字楼.xlsx") +// task.addPropertyType2("深圳") + +// task.standardName(filePath) +// task.nameRepeatFiter(filePath) +// task.addressFiter(filePath) +// task.nameFiter(filePath) +// task.typeFiter(filePath) +// task.nameFlag(filePath) +//// task.updateFlagToDB(filePath) +// task.updateFlagToDB2(filePath) + + task.checkRegion() + + } + + def nameRepeatFiter(filePath){ + def keys = [] + ExcelUtils.modify(filePath, { row -> + if (!row['L'] && row['B']) { + def city = row['B'] + def region = row['C'] + def name = "" + if(row['K']){ + name = row['K'] + }else{ + name = row['D'] + } + def key = city+"##"+region+"##"+name + if(keys.contains(key)){ + row['L'] = "名称去重" + println key + }else{ + keys << key + } + } + }) + } + + def nameFiter(filePath){ + ExcelUtils.modify(filePath, { row -> + def name = row['K'] + + if(!row['L'] && !name){ + row['L'] = "名称过滤" + } + if(!row['L'] && name){ + if(name.contains("公司")){ + row['L'] = "名称过滤" + } + } + + if(!row['L'] && name){ + if(isStandardRoad(name)){ + row['L'] = "名称过滤" + } + } + + + + if(!row['L'] && name){ + def fiters = ["厂房","测试", "服务","综合服务","酒店","深圳特专","工业","经济","产业","居委会","宿舍","购物","休闲","文化","训练","出租","浙菜","值班岗","招租","商住","办公室","配套","美食","博物馆","物流","商铺","公社","商城","值班室","暂停","创业社区","招商","宾馆","器材","教育","门诊","妇产科","商贸城","电子城","超市","零售"] + for(def key : fiters){ + if(name.contains(key)){ + row['L'] = "名称过滤" + break; + } + } + } + + if(!row['L'] && name){ + def fiters = ["大厦","裙楼",'花园','大楼','大堂','南塔','北京','大道','主楼','中国','中心','广场','鸟巢','小区','单位','办公','北塔' + ,'写字楼','检验楼','办公楼','接待区','商务楼','文体楼','市中心','产业园','科技楼','行政楼','病房43、44、45','律己楼','调度楼','辉腾楼','总务楼' + ,'7幢货梯','现厂B栋','(马连道南街)','中央电视台','后座','西大楼','B&C','商业用楼','财务中心','地铁','一瓶','外贸楼','业务楼','研发大楼','结算中心','马场','商务'] + for(def key : fiters){ + if(name == key){ + row['L'] = "名称过滤" + break; + } + } + } + + + if(!row['L'] && row['J'] && name){ + if(row['J'].toString().contains("商务写字楼") && name.contains("公寓")){ + row['L'] = "名称过滤" + } + } + + if(!row['L'] && name){ + if(findPattern(name,'^[A-Za-z0-9]+$') && name.toString().length()<=3){ + row['L'] = "名称过滤" + }else if(name.toString().length()<2){ + row['L'] = "名称过滤" + } + } + + if(!row['L'] && name){ + def filters = ['筹备组','小区','家属房','家属院','家属楼','自编','市场','项目','批发城','传达室','梯','型材','学院','工地','冷冻厂','机关楼','阅读室','议事厅','电脑城','童装城','派出所','保安室','散盘','店','工作站',"物业楼",'物业','驿站','锅炉房','处理中心',"销售中心",'政府','馆','捐赠点','百货','工厂','工场','检测场','职工之家','移动','中心站','楼门','职场','鲜花基地','医院','装饰城','实训楼','业务楼','科研楼','学校','交易城','管理楼','考场','通道','团购','楼上','运营中心','商业城'] + for(def key : filters){ + if(name.toString().endsWith(key)){ + row['L'] = "名称过滤" + break; + } + } + } + }) + } + + def typeFiter(filePath){ + ExcelUtils.modify(filePath, { row -> + if(!row['L'] && row['J']){ + def fiters = ["商住","服务","住宅小区","酒店","商业综合体","住宅底商"] + for(def key : fiters){ + if(row['J'].toString().contains(key)){ + row['L'] = "类型过滤" + break; + } + } + } + if(!row['L'] && row['B'] && !row['J']){ + row['L'] = "类型空白过滤" + } + }) + } + + def nameFlag(filePath){ + ExcelUtils.modify(filePath, { row -> + if(!row['L'] && row['D']) { + def fiters = ["大厦","大楼","商务楼","商务中心","广场","写字楼","办公楼","科技园","国际大厦","商务大厦","商务广场","创园","创意","中心","营销中心"] + for(def key : fiters){ + if(row['D'].toString().contains(key)){ + if(row['D']=="广场"){ + + } + row['L'] = "名称关键字[添加]" + break; + } + } + } + + if(!row['L'] && row['J']){ + if(row['J'].toString().contains("园区") || row['J'].toString().trim()=="纯写字楼" || row['J'].toString().trim()=="标准写字楼"){ + row['L'] = "类型[添加]" + } + + if(!row['L'] && row['J'].toString().trim()=="纯写字楼 标准写字楼" || row['J'].toString().trim()=="商务中心" || row['J'].toString().trim()=="商务中心 标准写字楼"){ + row['L'] = "类型[添加]" + } + } + + if(!row['L'] && row['J'] && row['D']){ + if(row['J'].toString().contains("商务写字楼") && !row['D'].toString().contains("公寓")){ + row['L'] = "类型[添加]" + } + } + + if(!row['L'] && row['J'] && row['D']){ + if(row['J'].toString().contains("纯写字楼")){ + row['L'] = "类型[添加]" + } + } + + }) + + } + + def addressFiter(filePath){ + def keyMap = [:] + ExcelUtils.modify(filePath, { row -> + if(!row['L'] && row['F']){ + def city = row['B'] + def region = row['C'] + def name = "" + if(row['K']){ + name = row['K'] + }else{ + name = row['D'] + } + def address = row['F'] + def key = city+"##"+region+"##"+address + def keys = keyMap.keySet() + if(keys.contains(key)){ + def values = keyMap.get(key) + if(values){ + for(def n : values){ + def score = CosineSimilarAlgorithm.getSimilarityRatio(name, n) + if(score>=0.8){ + row['L'] = "地址去重" + break; + } + println address +" "+ name+" "+n +" "+score + } + } + + }else{ + def values = keyMap.get(key) + if(!values){ + values = [] + keyMap.put(key, values) + } + values << name + } + } + + }); + } + + def standardName(filePath){ + def index = 0 + ExcelUtils.modify(filePath, { row -> + index++ + if(!row['K'] && row['D']){ + def name = row['D'] + def newName = name.replaceAll("-|·|(|)|//|\\?|“|”","").replaceAll("\\w+幢\\w+座","").replaceAll(" ","").replaceAll("\\.","").replaceAll("丨","") + newName = newName.replaceAll("-?\\w+号商业楼","").replaceAll("\\?","") + newName = newName.replaceAll("\\w+楼\\w+室","") + newName = newName.replaceAll("\\w+号\\w+(座|幢)","") + newName = newName.replaceAll("(一|二|三|四|五|六|七|八|九|十)号楼(一|二|三|四|五|六|七|八|九|十)单元","") + newName = newName.replaceAll("(一|二|三|四|五|六|七|八|九|十|十一)?幢?(一|二|三|四|五|六|七|八|九|十|东|南|西|北)(塔楼|楼)","") + newName = newName.replaceAll("-?(第)?\\w+(号写字楼|号楼|号大楼|楼|座|栋|南楼|北楼|东楼|西楼|区|单元)","") + newName = newName.replaceAll("(一|二|三|四|五|六|七|八|九|十|壹|贰|叁|肆|伍|陆|柒|捌|玖|拾)栋","") + newName = newName.replaceAll('(东|南|西|北)座$',"") + newName = newName.replaceAll("1期","一期").replaceAll("2期","二期").replaceAll("3期","三期").replaceAll("4期","四期").replaceAll("5期","五期").replaceAll("6期","六期").replaceAll("7期","七期").replaceAll("8期","八期").replaceAll("9期","九期").replaceAll("10期","十期").replaceAll("1-5期","一-五期").replaceAll("5-6期","六-七期") + newName = newName.toUpperCase().replaceAll("(装修中)","").replaceAll("\\(\\)","") + + if(newName){ + def md = DataUtil.match(newName, Pattern.compile("^[0-9|a-z|A-Z|一|二|三|四|五|六|七|八|九|十|零|壹|贰|叁|肆|伍|陆|柒|捌|玖|拾|东|南|西|北|中]{0,1}[写字|办公|楼|座|区|栋|组|幢|0-9|东|南|西|北|中|号]?[0-9|a-z|A-Z|一|二|三|四|五|六|七|八|九|十|零|壹|贰|叁|肆|伍|陆|柒|捌|玖|拾|东|南|西|北|中|办公|写字]?[楼|座|区|栋|组|幢|0-9|东|南|西|北|中]\$")) + if(md){ + newName = '' + } + } + row['K'] = newName + } + }) + } + + def addPropertyType2(runCity){ + def map = [:] + def datas = ExcelUtils.read("D:\\数据\\写字楼\\6城未匹配写字楼.xlsx",0,1) + if(datas){ + for(def d : datas){ + def key = d['B']+"##"+d['C']+"##"+d['D'] + def list = map.get(key) + if(!list){ + list = [] + map.put(key, list) + } + if(d['M'] && !list.contains(d['M'])){ + list << d['M'] + } + if(d['U'] && !list.contains(d['U'])){ + list << d['U'] + } + + } + } + + def index = 0 + ExcelUtils.multiThreadingModify("D:\\数据\\写字楼\\6城未匹配上的写字楼.xlsx", { row -> + index++ + def city = row['B'] + def region = row['C'] + def name = row['D'] + def types = map.get(city+"##"+region+"##"+name) + if(types){ + row['J'] = types.join(" ") + } + }) + } + + def addPropertyType(){ + def index = 0 + ExcelUtils.modify("D:\\数据\\写字楼\\未匹配写字楼.xlsx", { row -> + index++ + def city = row['B'] + def region = row['C'] + def name = row['D'] + def types = findDatasByDB(city, region, name) + if(types){ + row['J'] = types.join(" ") + } + println index + }) + } + + def findDatasByDB(city, region, name){ + def sql = "select * from office where city='${city}' and region='${region}' and name='${name}'" + def types = [] + sqlInstance.eachRow(sql, {r-> + if(r.property_type && !types.contains(r.property_type)){ + types << r.property_type + } + }) + return types + } + + def run(filepath){ + def index = 0 + ExcelUtils.multiThreadingModify(filepath,{row-> + index++ + def city = row['B'] + def region = row['C'] + def name = row['D'] + def address = row['I'] + if(city ){ + def jsonObject = OpenApiClient.addressMatch(city, name, "office", false, region, null) +// if(jsonObject && jsonObject.precise == -1){ +// jsonObject = OpenApiClient.addressMatch(city, address, "office", false, region, name) +// } + if (jsonObject != null && jsonObject.precise != -1) { + row['J'] = jsonObject.get("id") + row['K'] = jsonObject.get("city") + row['L'] = jsonObject.get("district") + row['M'] = jsonObject.get("name") + row['N'] = jsonObject.get("credit") + row['O'] = jsonObject.get("propertyType") + } + } + println index + }) + } + + def isStandardRoad(def s){ + def pattern = Pattern.compile("[路|街|大道|巷]\\d+(号|弄)?"); + def matcher = pattern.matcher(s) + if(matcher.find()){ + return true; + } + if(s.toString().endsWith("路")||s.toString().endsWith("街")){ + return true; + } + return false + } + + + def findPattern(def address, def regex) { + Pattern pattern = Pattern.compile(regex); + def matcher = pattern.matcher(address) + if (matcher.find()) { + return true + } + return false; + } + + def updateFlagToDB(filePath){ + def datas = ExcelUtils.read(filePath,0,1) + def total = datas.size() + def index = 0 + for(def d : datas){ + def sql = "alter table office update option='${d['L']}', format_name='${d['K']}' where city='${d['B']}' and region='${d['C']}' and name='${d['D']}';" + index++ + sqlInstance.executeUpdate(sql) + println index+"/"+ total + } + } + + + def updateOption(city, region, name, source, flag){ + def sql = "alter table office update region_fence='${flag}' where city='${city}' and region='${region}' and name='${name}' and source='${source}';" + sqlInstance.executeUpdate(sql) + } + + def batchUpdate(rets){ + // 初始化驱动 + Class.forName("ru.yandex.clickhouse.ClickHouseDriver"); + // url + String url = "jdbc:clickhouse://192.168.0.216:8123/etl"; + // 用户名密码 + String user = "goojia"; + String password = "goojia123456"; + // 登录 + Connection connection = DriverManager.getConnection(url, user, password); + Statement stmt = connection.createStatement(); + def count = 0 + def sql = "alter table office update region_fence=? where city=? and region=? and name=? and source=?;" + PreparedStatement pstmt = connection.prepareStatement(sql) + + for(def d : rets){ + count++ + pstmt.setString(1, d.flag) + pstmt.setString(2, d.city) + pstmt.setString(3, d.region) + pstmt.setString(4, d.name) + pstmt.setString(5, d.source) + println count+" "+d.city+" "+d.name + pstmt.addBatch() + if(count % 1 ==0) { + try { + pstmt.executeBatch() + } catch (Exception e) { + e.printStackTrace() + } + } + } + pstmt.executeBatch() + } + def updateFlagToDB2(filePath){ + // 初始化驱动 + Class.forName("ru.yandex.clickhouse.ClickHouseDriver"); + // url + String url = "jdbc:clickhouse://192.168.0.216:8123/etl"; + // 用户名密码 + String user = "goojia"; + String password = "goojia123456"; + // 登录 + Connection connection = DriverManager.getConnection(url, user, password); + Statement stmt = connection.createStatement(); + def count = 0 + def datas = ExcelUtils.read(filePath,0,1) + def sql = "alter table office update option=?, format_name=? where city=? and region=? and name=?;" + PreparedStatement pstmt = connection.prepareStatement(sql) + + for(def d : datas){ + if(d['B']){ + count++ + pstmt.setString(1, d['L']) + pstmt.setString(2, d['K']) + pstmt.setString(3, d['B']) + pstmt.setString(4, d['C']) + pstmt.setString(5, d['D']) + println count+" "+d['L']+" "+d['K']+" "+d['B']+" "+d['C']+" "+d['D'] + pstmt.addBatch() + if(count % 100 ==0) { + try { + pstmt.executeBatch() + } catch (Exception e) { + + } + } + } + } + pstmt.executeBatch() + } + + def checkRegion(){ + def citys = ["深圳","北京","广州","重庆","天津","成都","南京"]//"上海", + for(def city : citys){ + def regionPointsMap = findRegionRange(city) + def addDatas = findAddDatasByClickhouse(city) + println "需处理的区域围栏:"+addDatas.size() + def index = 0 + def rets = [] + for(def data : addDatas){ + index ++ + if(data.lng){ + def flag = isInArea(data.lng.toString().toDouble(), data.lat.toString().toDouble(), regionPointsMap.get(data.region)) + def mark = flag?'T':'F' +// rets << ["city":data.city, "region":data.region, "name":data.name, "source":data.source, "flag":mark] + updateOption(data.city, data.region, data.name, data.source, mark) + println index+" "+data.region + " "+data.name+" "+ flag + } + } +// batchUpdate(rets) + } + } + + def findAddDatasByClickhouse(city){ + def sql = "select city,region, name,option,lng,lat,source from office where city='${city}' and option like '%添加%'" + def datas = [] + sqlInstance.eachRow(sql, {r-> + datas << ["city":r.city,"region":r.region,"name":r.name, "lat":r.lat, "lng":r.lng, "option":r.option,"source":r.source] + }) + return datas + } + + def findRegionRange(city){ + def q = new BasicDBObject() + q.put("city",city) + def dbcursor = POLYLINECOLL.find(q) + def regionMap = [:] + for(def d : dbcursor){ + def region = RegionData.formatRegion(city,d.region) + regionMap.put(region, d.polyline) + } + return regionMap + } + + def isInArea(lng, lat, points){ + List pointList = InsidePolygon.createPoint2(points); + def flag = InsidePolygon.isPtInPoly(lng, lat, pointList) + return flag; + } +}