From 61869c0f75d0f72270c294fddfa28dceb49353e0 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Sat, 8 May 2021 10:12:58 +0800 Subject: [PATCH 01/17] update --- .../script/commData/CommonDatautils.groovy | 15 + .../fangjia/script/gj/GaoDeMapCrawl.groovy | 92 +-- .../com/fangjia/script/gj/LogStat.groovy | 200 +++++- .../fangjia/script/gj/data/FieldStat.groovy | 13 +- .../script/gj/test/ChongQingBlankTest.groovy | 301 +++++++++ .../fangjia/script/gj/test/CityStat.groovy | 580 ++++++++++++++++++ .../gj/test/CompanyAddressCompare.groovy | 111 ++++ .../gj/test/CrawlDistrictIdSelect.groovy | 102 +++ .../script/gj/test/ExportToDealPrice.groovy | 101 +++ .../com/fangjia/script/gj/test/Test.groovy | 6 +- .../thirdPriceCrawl/other/AnjukeCrawl.groovy | 52 ++ 11 files changed, 1513 insertions(+), 60 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/ChongQingBlankTest.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CityStat.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CompanyAddressCompare.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CrawlDistrictIdSelect.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/ExportToDealPrice.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/other/AnjukeCrawl.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 34a65ee51..4c2f2df46 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -1392,6 +1392,21 @@ class CommonDatautils { } + public static def queryDistrictPriceByWeekend(city, region, name, weekend, limit){ + DBObject queryCondition = new BasicDBObject() + queryCondition.put("city", city); + queryCondition.put("region", region); + queryCondition.put("district_name", name); + if(weekend){ + queryCondition.put('weekend', new BasicDBObject('$gte', weekend)); + } + DBCursor cur = DIST_STAT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", 1)).limit(limit) + def dbo = null; + while(cur.hasNext()) { + dbo = cur.next() + } + return dbo; + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy index 9c85d6711..2ba2cdf53 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy @@ -25,6 +25,7 @@ import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService import java.util.concurrent.Executors import java.util.concurrent.atomic.AtomicInteger +import java.util.concurrent.atomic.AtomicLong import java.util.regex.Matcher import java.util.regex.Pattern import java.util.zip.GZIPInputStream @@ -36,7 +37,7 @@ class GaoDeMapCrawl { public static final def dbUrl = "mongodb://192.168.0.235:27018"; - public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(5) + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1) final AtomicInteger count = new AtomicInteger() def updateLists = [] def updateErrorLists = [] @@ -46,6 +47,7 @@ class GaoDeMapCrawl { // def categorys = ["141202", "141203"]//中学,小学 def categorys = ["120000", "120200","120203","120300","120301","120302","120303","120304"]//住宅相关 // def categorys = ["190301"]//道路相关 + def totalCount = new AtomicLong() static main(args) { def task = new GaoDeMapCrawl(); @@ -109,26 +111,13 @@ class GaoDeMapCrawl { def run(){//,"西安","重庆" // def citys = [ "临沂","泰安","菏泽","东营","德州","济宁","日照","聊城","滨州","枣庄","莱芜","济南","青岛","威海","烟台","潍坊","淄博"] - def citys = ["重庆","哈尔滨","沈阳","东莞","南昌","昆明","金华","温州","淄博","湛江","肇庆","宜昌","岳阳","安庆","芜湖","西宁","日照","常德","张家口"] + def citys = ["重庆","哈尔滨","沈阳","东莞","南昌","昆明","金华","温州","淄博","湛江","肇庆","宜昌","岳阳","安庆","芜湖","西宁","日照","常德","张家口","北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"] // def citys = CommonUtils.findAllCity() def totalCount = 0; for(def city : citys){ def cityLations = findAllLation(city); println "开始处理["+city +"]:"+cityLations.size() gaoDeCitySearch(cityLations); - if(updateLists.size() > 0){ - println " 正常访问共:"+updateLists.size() - updateBatch(); - } - if(updateErrorLists.size() > 0){ - println " 失败访问共:"+updateErrorLists.size() - updateErrorBatch(); - } - - if(flagLists.size()>0){ - println "有数据的格子共:"+flagLists.size() - updateFlagBath(); - } } } @@ -145,39 +134,68 @@ class GaoDeMapCrawl { } def gaoDeCitySearch(def cityLocations){ - try { - for(def location : cityLocations){ - index ++ -// def bound_gd = location.bound_gd + +// final CountDownLatch latch = new CountDownLatch(cityLocations.size()) + for(def i=0;i 0){ + println " 正常访问共:"+updateLists.size() + updateBatch(); + } + if(updateErrorLists.size() > 0){ + println " 失败访问共:"+updateErrorLists.size() + updateErrorBatch(); + } + + if(flagLists.size()>0){ + println "有数据的格子共:"+flagLists.size() + updateFlagBath(); + } + } + + def searching(location){ + // def bound_gd = location.bound_gd // def lats = [bound_gd[1], bound_gd[3]] // def lngs = [bound_gd[0], bound_gd[2]] // lats.sort() // lngs.sort() // println index // if(lngs[0]<=121.418153 && lngs[1]>=121.418153 && lats[0]<=31.218824 && lats[1]>=31.218824){ - search(location) + search(location) // } - if(updateLists.size() > 0 && updateLists.size()>1000){ - println " 正常访问共:"+updateLists.size() - updateBatch(); - } - if(updateErrorLists.size() > 0 && updateErrorLists.size()>1000){ - println " 失败访问共:"+updateErrorLists.size() - updateErrorBatch(); - } - - if(flagLists.size()>0 && flagLists.size()>1000){ - println "有数据的格子共:"+flagLists.size() - updateFlagBath(); - } - } - }catch (Exception e){ - e.printStackTrace() + if(updateLists.size() > 0 && updateLists.size()>1000){ + println " 正常访问共:"+updateLists.size() + updateBatch(); + } + if(updateErrorLists.size() > 0 && updateErrorLists.size()>1000){ + println " 失败访问共:"+updateErrorLists.size() + updateErrorBatch(); } + if(flagLists.size()>0 && flagLists.size()>1000){ + println "有数据的格子共:"+flagLists.size() + updateFlagBath(); + } } + def search(location){ def bound_gd = location.bound_gd def lats = [bound_gd[1], bound_gd[3]] @@ -212,7 +230,7 @@ class GaoDeMapCrawl { updateErrorLists.addAll(location) } // update(location); - println "[ "+count.get()+" ${index}]"+location._id+" "+location.city+" "+location.bound_gd+" flag: "+m.status + println "[ "+totalCount.get()+" ]"+location._id+" "+location.city+" "+location.bound_gd+" flag: "+m.status } def processData(rets){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/LogStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/LogStat.groovy index 1d490eea2..7c62d7229 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/LogStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/LogStat.groovy @@ -2,20 +2,91 @@ package com.fangjia.script.gj import com.fangjia.core.util.DateUtil import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.QueryOperators import org.apache.lucene.queryparser.classic.QueryParser class LogStat { + public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; + public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) + def static final DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol(); + def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") + static main(args) { def task = new LogStat() task.run() +// task.run1() + } + + def run1(){ + ExcelUtils.modify("C:\\Users\\gaojuan\\Desktop\\案例信息-待反馈评估过程_V2(无颜色).xls", { d -> + def city = d['C'].replace("市","") + def district = CommonDatautils.findDistrict(city, d['AA'], d['AB'], "district") + if(!district){ + println "sss" + } + if(d['AT'] && district){ + + def startDate2 = DateUtil.add(d['X'], -365) + def start = DateUtil.formatByPattern(startDate2,"yyyyMMdd") + def end = DateUtil.formatByPattern(d['X'],"yyyyMMdd") + + + def alias = [] + if(district.alias){ + if(!district.alias.contains(district.name)){ + district.alias << district.name + } + alias = district.alias + }else{ + alias << district.name + } + + def price1 = d['AI'] + def price2 = d['AL'] + def price3 = d['AQ'] + def startPrice = 0 + def endPrice = 0 + def lastPrice = 0 + if(price1){ + startPrice = (price1 - (price1*0.15)).toInteger() + endPrice = (price1 + (price1*0.15)).toInteger() + lastPrice = price1 + }else if(price2){ + startPrice = (price2 - (price2*0.15)).toInteger() + endPrice = (price2 + (price2*0.15)).toInteger() + lastPrice = price2 + }else if(price3){ + startPrice = (price3 - (price3*0.15)).toInteger() + endPrice = (price3 + (price3*0.15)).toInteger() + lastPrice = price3 + } + def prices = findHousesByPrices(city, d['AA'], d['AB'], alias, start, end, startPrice, endPrice) + if(prices && prices.size()>=3){ + def houseAvgPrice = (prices.sum()/prices.size()).toInteger() + d['AD'] = houseAvgPrice + d['AC'] = (houseAvgPrice * d['N']).toInteger() + }else{ + def random = new Random().nextInt()%6 + def randomAvgPrice = lastPrice*random/100 + lastPrice + d['AD'] = randomAvgPrice + d['AC'] = (randomAvgPrice * d['N']).toInteger() + } + + } + + },0); } def run(){ - ExcelUtils.multiThreadingModify("D:\\数据\\建行\\小区名单.xlsx",{ d -> + ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\案例信息-待反馈评估过程_V2(无颜色).xls",{ d -> // def calDate = DateUtil.parseDate(d['M'], "yyyy-MM-dd") // def weekend = DateUtils.getFirstDayOfWeek(calDate) // def dbo = CommonDatautils.queryDistrictPrice(d['N'], d['O'], d['P'], weekend) @@ -23,12 +94,11 @@ class LogStat { // d['E'] = dbo.avg_price // } - if(d['A']){ - def district = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district") + if(d['AA'] && d['AB'] && d['AH']){ + def city = d['C'].replace("市","") + def district = CommonDatautils.findDistrict(city, d['AA'], d['AB'], "district") if(district){ - def startDate = DateUtil.addMonth(new Date(), -6) - def start = 20200901//startDate.replace("-","").toInteger() - def end = 20201201//DateUtil.formatByPattern(calDate,"yyyyMMdd").toInteger() + def alias = [] if(district.alias){ if(!district.alias.contains(district.name)){ @@ -38,9 +108,42 @@ class LogStat { }else{ alias << district.name } - def count = findHouses(d['A'], d['B'], d['C'], alias, start, end) - d['D'] = count - println(d) + +// def startDate = DateUtil.add(d['X'], -90) +// def endDate = DateUtil.add(d['X'], 5) +// def trades = findTrades(district, startDate, endDate) +// if(trades){ +// d['AJ'] = (int)(trades.sum()/trades.size()) +// d['AK'] = trades.size() +// } +// +// def monthStartDate = DateUtil.getFirstDayOfMonth(d['X']) +// def monthEndDate = DateUtil.getFirstDayOfMonth(DateUtil.parseFormatDate(DateUtil.addMonth(d['X'], 1))) +// def monthStart = DateUtil.formatByPattern(monthStartDate,"yyyyMMdd").toInteger() +// def monthEnd = DateUtil.formatByPattern(monthEndDate,"yyyyMMdd").toInteger() +// def prices = findHouses(city, d['AA'], d['AB'], alias, monthStart, monthEnd) +// if(prices){ +// d['AL'] = (int)(prices.sum()/prices.size()) +// d['AM'] = prices.size() +// } +// +// +// def sDate = DateUtil.formatByPattern(monthStartDate,"yyyyMM").toInteger() +// def dbo = findThirdPrice(district.city, district.region, district.name, sDate, "lianjia_esf_price") +// if(dbo){ +// d['AN'] = dbo.lianjia_esf_price +// } + + def price = d['AH'] + def startPrice = (price - (price*0.15)).toInteger() + def endPrice = (price + (price*0.15)).toInteger() + def startDate2 = DateUtil.add(d['X'], -365) + def start = DateUtil.formatByPattern(startDate2,"yyyyMMdd") + def end = DateUtil.formatByPattern(d['X'],"yyyyMMdd") + def count = findHouseCount(city, d['AA'], d['AB'], alias, start, end, startPrice, endPrice) + if(count > 0){ + d['AS'] = count + } } } @@ -48,15 +151,68 @@ class LogStat { } - def findHouses(def city, def region, def name, def alias, def startDate, def endDate){ + def findTrades(dist, startDate, endDate){ + def q = new BasicDBObject() + q.put("city", dist.city) + q.put("region", dist.region) + q.put("source", "链家在线") + + def alias = [] + if(dist.alias){ + if(!dist.alias.contains(dist.name)){ + dist.alias << dist.name + } + alias = dist.alias + }else{ + alias << dist.name + } + + if(alias){ + q.put("district_name", new BasicDBObject('$in', alias)) + } + + q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate)) + def dbcorsor = DEAL_PRICE_COLL.find(q) + def prices = [] + for(def d : dbcorsor){ + if(d.avg_price){ + prices << d.avg_price + } + } + + return prices + } + + + def findHouses(def city, def region, def name, def alias, def startDate, def endDate){ def aliasStr = escapeExprSpecialWord(alias.join("','")) - String sql = "select count(*) as c from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}')" - def count = 0 + String sql = "select avg_price from latest_seawater where cat='sell' and root='lianjia.com' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}')" + def prices = [] ElasticUtils.executeSqlQuery(sql,{data-> - count = data.c + prices << data.avg_price }) - return count + return prices + } + + def findHouseCount(def city, def region, def name, def alias, def startDate, def endDate, def startprice, def endprice){ + def aliasStr = escapeExprSpecialWord(alias.join("','")) + String sql = "select count(*) as c from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}') and avg_price>=${startprice} and avg_price<=${endprice}" + def total = 0 + ElasticUtils.executeSqlQuery(sql,{data-> + total = data.c + }) + return total + } + + def findHousesByPrices(def city, def region, def name, def alias, def startDate, def endDate, def startprice, def endprice){ + def aliasStr = escapeExprSpecialWord(alias.join("','")) + String sql = "select avg_price from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}') and avg_price>=${startprice} and avg_price<=${endprice}" + def prices = [] + ElasticUtils.executeSqlQuery(sql,{data-> + prices << data.avg_price + }) + return prices } @@ -126,6 +282,20 @@ class LogStat { // d['G'] = count // } - + def findThirdPrice(def city, def region, def name, def s_date,def fieldName){ + def q = new BasicDBObject() + q.put("city", city) + q.put("region", region) + q.put("name", name) + q.put("s_date", s_date) + q.put(fieldName, new BasicDBObject(QueryOperators.EXISTS, true)) + def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date",-1)) + def obj = null; + for(def d : dbcursor){ + obj = d; + break; + } + return obj; + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/FieldStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/FieldStat.groovy index 6131ce01e..72fd50274 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/FieldStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/FieldStat.groovy @@ -11,7 +11,7 @@ class FieldStat { } def run(){ - ExcelUtils.multiThreadingModify("D:\\数据\\建行\\小区融合结果概况2.xlsx", { row -> + ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\建行数据融合结果统计_0425.xlsx", { row -> def city = row['A'] if(city){ def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "alias"]); @@ -26,15 +26,14 @@ class FieldStat { aliasTotalCount = aliasTotalCount + alias.size() } } - row['J'] = aliasTotalCount - row['K'] = aliasIndex +// row['J'] = aliasTotalCount +// row['K'] = aliasIndex def p = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, aliasIndex*100/total) - row['L'] = p+"%" - row['M'] = total + row['B'] = p+"%" +// row['M'] = total println city +" "+p } - - }); + }, 1); println "finish" } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/ChongQingBlankTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/ChongQingBlankTest.groovy new file mode 100644 index 000000000..1bdbfcdc7 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/ChongQingBlankTest.groovy @@ -0,0 +1,301 @@ +package com.fangjia.script.gj.test + +import com.fangjia.core.RegionData +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.CutWordUtils +import com.fangjia.script.gj.district.SeaweedDataMatchProcess +import com.fangjia.script.gj.paimai.GaoDeUtils +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.CosineSimilarAlgorithm +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils + +import java.util.regex.Pattern + +class ChongQingBlankTest { + + def gaodeCityCodeMap = ["北京":"110000","天津":"120000","石家庄":"130100","辛集":"130181","晋州":"130183","新乐":"130184","唐山":"130200","滦州":"130284","遵化":"130281","迁安":"130283","秦皇岛":"130300","邯郸":"130400","武安":"130481","邢台":"130500","南宫":"130581","沙河":"130582","保定":"130600","涿州":"130681","定州":"130682","安国":"130683","高碑店":"130684","张家口":"130700","承德":"130800","平泉":"130881","沧州":"130900","泊头":"130981","任丘":"130982","黄骅":"130983","河间":"130984","廊坊":"131000","霸州":"131081","三河":"131082","衡水":"131100","深州":"131182","太原":"140100","古交":"140181","大同":"140200","阳泉":"140300","长治":"140400","晋城":"140500","高平":"140581","朔州":"140600","怀仁":"140681","晋中":"140700","介休":"140781","运城":"140800","永济":"140881","河津":"140882","忻州":"140900","原平":"140981","临汾":"141000","侯马":"141081","霍州":"141082","吕梁":"141100","孝义":"141181","汾阳":"141182","呼和浩特":"150100","包头":"150200","乌海":"150300","赤峰":"150400","通辽":"150500","霍林郭勒":"150581","鄂尔多斯":"150600","呼伦贝尔":"150700","满洲里":"150781","牙克石":"150782","扎兰屯":"150783","额尔古纳":"150784","根河":"150785","巴彦淖尔":"150800","乌兰察布":"150900","丰镇":"150981","乌兰浩特":"152201","阿尔山":"152202","二连浩特":"152501","锡林浩特":"152502","沈阳":"210100","新民":"210181","大连":"210200","瓦房店":"210281","庄河":"210283","鞍山":"210300","海城":"210381","抚顺":"210400","本溪":"210500","丹东":"210600","东港":"210681","凤城":"210682","锦州":"210700","凌海":"210781","北镇":"210782","营口":"210800","西市区":"210803","盖州":"210881","大石桥":"210882","阜新":"210900","辽阳":"211000","灯塔":"211081","盘锦":"211100","铁岭":"211200","调兵山":"211281","开原":"211282","朝阳":"211300","北票":"211381","凌源":"211382","葫芦岛":"211400","兴城":"211481","长春":"220100","榆树":"220182","德惠":"220183","吉林":"220200","蛟河":"220281","桦甸":"220282","舒兰":"220283","磐石":"220284","四平":"220300","公主岭":"220381","双辽":"220382","辽源":"220400","通化":"220500","梅河口":"220581","集安":"220582","白山":"220600","临江":"220681","松原":"220700","扶余":"220781","白城":"220800","洮南":"220881","大安":"220882","延吉":"222401","图们":"222402","敦化":"222403","珲春":"222404","龙井":"222405","和龙":"222406","哈尔滨":"230100","尚志":"230183","五常":"230184","齐齐哈尔":"230200","讷河":"230281","鸡西":"230300","虎林":"230381","密山":"230382","鹤岗":"230400","双鸭山":"230500","大庆":"230600","伊春":"230700","铁力":"230781","佳木斯":"230800","同江":"230881","富锦":"230882","抚远":"230883","七台河":"230900","牡丹江":"231000","绥芬河":"231081","海林":"231083","宁安":"231084","穆棱":"231085","东宁":"231086","黑河":"231100","北安":"231181","五大连池":"231182","绥化":"231200","安达":"231281","肇东":"231282","海伦":"231283","漠河":"232701","上海":"310000","南京":"320100","无锡":"320200","江阴":"320281","宜兴":"320282","徐州":"320300","新沂":"320381","邳州":"320382","常州":"320400","溧阳":"320481","苏州":"320500","常熟":"320581","张家港":"320582","昆山":"320583","太仓":"320585","南通":"320600","海安":"320685","启东":"320681","如皋":"320682","海门":"320684","连云港":"320700","淮安":"320800","盐城":"320900","东台":"320981","扬州":"321000","仪征":"321081","高邮":"321084","镇江":"321100","丹阳":"321181","扬中":"321182","句容":"321183","泰州":"321200","兴化":"321281","靖江":"321282","泰兴":"321283","宿迁":"321300","杭州":"330100","建德":"330182","宁波":"330200","余姚":"330281","慈溪":"330282","温州":"330300","瑞安":"330381","乐清":"330382","嘉兴":"330400","海宁":"330481","平湖":"330482","桐乡":"330483","湖州":"330500","绍兴":"330600","诸暨":"330681","嵊州":"330683","金华":"330700","兰溪":"330781","义乌":"330782","东阳":"330783","永康":"330784","衢州":"330800","江山":"330881","舟山":"330900","台州":"331000","温岭":"331081","临海":"331082","玉环":"331083","丽水":"331100","龙泉":"331181","合肥":"340100","巢湖":"340181","芜湖":"340200","蚌埠":"340300","淮南":"340400","马鞍山":"340500","淮北":"340600","铜陵":"340700","安庆":"340800","潜山":"340882","桐城":"340881","黄山":"341000","滁州":"341100","天长":"341181","明光":"341182","阜阳":"341200","界首":"341282","宿州":"341300","六安":"341500","亳州":"341600","池州":"341700","宣城":"341800","宁国":"341881","福州":"350100","福清":"350181","厦门":"350200","莆田":"350300","三明":"350400","永安":"350481","泉州":"350500","石狮":"350581","晋江":"350582","南安":"350583","漳州":"350600","龙海":"350681","南平":"350700","邵武":"350781","武夷山":"350782","建瓯":"350783","龙岩":"350800","漳平":"350881","宁德":"350900","福安":"350981","福鼎":"350982","南昌":"360100","景德镇":"360200","乐平":"360281","萍乡":"360300","九江":"360400","瑞昌":"360481","共青城":"360482","庐山":"360483","新余":"360500","鹰潭":"360600","贵溪":"360681","赣州":"360700","瑞金":"360781","吉安":"360800","井冈山":"360881","宜春":"360900","丰城":"360981","樟树":"360982","高安":"360983","抚州":"361000","上饶":"361100","德兴":"361181","济南":"370100","市中区":"370103","青岛":"370200","市南区":"370202","市北区":"370203","胶州":"370281","平度":"370283","莱西":"370285","淄博":"370300","枣庄":"370400","市中区":"370402","滕州":"370481","东营":"370500","烟台":"370600","龙口":"370681","莱阳":"370682","莱州":"370683","蓬莱":"370684","招远":"370685","栖霞":"370686","海阳":"370687","潍坊":"370700","青州":"370781","诸城":"370782","寿光":"370783","安丘":"370784","高密":"370785","昌邑":"370786","济宁":"370800","曲阜":"370881","邹城":"370883","泰安":"370900","新泰":"370982","肥城":"370983","威海":"371000","荣成":"371082","乳山":"371083","日照":"371100","临沂":"371300","德州":"371400","乐陵":"371481","禹城":"371482","聊城":"371500","临清":"371581","滨州":"371600","邹平":"371681","菏泽":"371700","郑州":"410100","巩义":"410181","荥阳":"410182","新密":"410183","新郑":"410184","登封":"410185","开封":"410200","洛阳":"410300","偃师":"410381","平顶山":"410400","舞钢":"410481","汝州":"410482","安阳":"410500","林州":"410581","鹤壁":"410600","新乡":"410700","卫辉":"410781","辉县":"410782","焦作":"410800","沁阳":"410882","孟州":"410883","濮阳":"410900","许昌":"411000","禹州":"411081","长葛":"411082","漯河":"411100","三门峡":"411200","义马":"411281","灵宝":"411282","南阳":"411300","邓州":"411381","商丘":"411400","永城":"411481","信阳":"411500","周口":"411600","项城":"411681","驻马店":"411700","济源":"419001","武汉":"420100","黄石":"420200","大冶":"420281","十堰":"420300","丹江口":"420381","宜昌":"420500","宜都":"420581","当阳":"420582","枝江":"420583","襄阳":"420600","老河口":"420682","枣阳":"420683","宜城":"420684","鄂州":"420700","荆门":"420800","京山":"420882","钟祥":"420881","孝感":"420900","应城":"420981","安陆":"420982","汉川":"420984","荆州":"421000","沙市区":"421002","石首":"421081","洪湖":"421083","松滋":"421087","黄冈":"421100","麻城":"421181","武穴":"421182","咸宁":"421200","赤壁":"421281","随州":"421300","广水":"421381","恩施":"422801","利川":"422802","仙桃":"429004","潜江":"429005","天门":"429006","长沙":"430100","浏阳":"430181","宁乡":"430182","株洲":"430200","醴陵":"430281","湘潭":"430300","湘乡":"430381","韶山":"430382","衡阳":"430400","耒阳":"430481","常宁":"430482","邵阳":"430500","武冈":"430581","岳阳":"430600","汨罗":"430681","临湘":"430682","常德":"430700","津市":"430781","张家界":"430800","益阳":"430900","沅江":"430981","郴州":"431000","资兴":"431081","永州":"431100","怀化":"431200","洪江":"431281","娄底":"431300","冷水江":"431381","涟源":"431382","吉首":"433101","广州":"440100","韶关":"440200","乐昌":"440281","南雄":"440282","深圳":"440300","珠海":"440400","汕头":"440500","佛山":"440600","江门":"440700","台山":"440781","开平":"440783","鹤山":"440784","恩平":"440785","湛江":"440800","廉江":"440881","雷州":"440882","吴川":"440883","茂名":"440900","高州":"440981","化州":"440982","信宜":"440983","肇庆":"441200","四会":"441284","惠州":"441300","梅州":"441400","兴宁":"441481","汕尾":"441500","陆丰":"441581","河源":"441600","阳江":"441700","阳春":"441781","清远":"441800","英德":"441881","连州":"441882","东莞":"441900","中山":"442000","潮州":"445100","揭阳":"445200","普宁":"445281","云浮":"445300","罗定":"445381","南宁":"450100","柳州":"450200","桂林":"450300","荔浦":"450381","梧州":"450400","岑溪":"450481","北海":"450500","防城港":"450600","东兴":"450681","钦州":"450700","贵港":"450800","桂平":"450881","玉林":"450900","北流":"450981","百色":"451000","靖西":"451081","贺州":"451100","河池":"451200","来宾":"451300","合山":"451381","崇左":"451400","凭祥":"451481","海口":"460100","三亚":"460200","三沙":"460300","儋州":"460400","五指山":"469001","琼海":"469002","文昌":"469005","万宁":"469006","东方":"469007","重庆":"500000","重庆市郊县":"500200","成都":"510100","都江堰":"510181","彭州":"510182","邛崃":"510183","崇州":"510184","简阳":"510185","自贡":"510300","攀枝花":"510400","泸州":"510500","德阳":"510600","广汉":"510681","什邡":"510682","绵竹":"510683","绵阳":"510700","江油":"510781","广元":"510800","遂宁":"510900","内江":"511000","市中区":"511002","隆昌":"511083","乐山":"511100","市中区":"511102","峨眉山":"511181","南充":"511300","阆中":"511381","眉山":"511400","宜宾":"511500","广安":"511600","华蓥":"511681","达州":"511700","万源":"511781","雅安":"511800","巴中":"511900","资阳":"512000","马尔康":"513201","康定":"513301","西昌":"513401","贵阳":"520100","清镇":"520181","六盘水":"520200","盘州":"520281","遵义":"520300","赤水":"520381","仁怀":"520382","安顺":"520400","毕节":"520500","铜仁":"520600","兴义":"522301","兴仁":"522302","凯里":"522601","都匀":"522701","福泉":"522702","昆明":"530100","安宁":"530181","曲靖":"530300","宣威":"530381","玉溪":"530400","保山":"530500","腾冲":"530581","昭通":"530600","水富":"530681","丽江":"530700","普洱":"530800","临沧":"530900","楚雄":"532301","个旧":"532501","开远":"532502","蒙自":"532503","弥勒":"532504","文山":"532601","景洪":"532801","大理":"532901","瑞丽":"533102","芒":"533103","泸水":"533301","香格里拉":"533401","拉萨":"540100","日喀则":"540200","昌都":"540300","林芝":"540400","山南":"540500","那曲":"540600","西安":"610100","铜川":"610200","宝鸡":"610300","咸阳":"610400","彬州":"610482","兴平":"610481","渭南":"610500","韩城":"610581","华阴":"610582","延安":"610600","汉中":"610700","榆林":"610800","神木":"610881","安康":"610900","商洛":"611000","兰州":"620100","嘉峪关":"620200","金昌":"620300","白银":"620400","天水":"620500","武威":"620600","张掖":"620700","平凉":"620800","华亭":"620881","酒泉":"620900","玉门":"620981","敦煌":"620982","庆阳":"621000","定西":"621100","陇南":"621200","临夏":"622901","合作":"623001","西宁":"630100","海东":"630200","玉树":"632701","格尔木":"632801","德令哈":"632802","茫崖":"632803","银川":"640100","灵武":"640181","石嘴山":"640200","吴忠":"640300","青铜峡":"640381","固原":"640400","中卫":"640500","乌鲁木齐":"650100","新市区":"650104","克拉玛依":"650200","吐鲁番":"650400","哈密":"650500","昌吉":"652301","阜康":"652302","博乐":"652701","阿拉山口":"652702","库尔勒":"652801","阿克苏":"652901","阿图什":"653001","喀什":"653101","和田":"653201","伊宁":"654002","奎屯":"654003","霍尔果斯":"654004","塔城":"654201","乌苏":"654202","阿勒泰":"654301","石河子":"659001","阿拉尔":"659002","图木舒克":"659003","五家渠":"659004","北屯":"659005","铁门关":"659006","双河":"659007","可克达拉":"659008","昆玉":"659009","伊犁":"654000"] + def baiduCityCodeMap = ["北京": "131", "天津": "332", "石家庄": "150", "唐山": "265", "秦皇岛": "148", "邯郸": "151", "邢台": "266", "保定": "307", "张家口": "264", "承德": "207", "沧州": "149", "廊坊": "191", "衡水": "208", "太原": "176", "大同": "355", "阳泉": "357", "长治": "356", "晋城": "290", "朔州": "237", "晋中": "238", "运城": "328", "忻州": "367", "临汾": "368", "吕梁": "327", "呼和浩特": "321", "包头": "229", "乌海": "123", "赤峰": "297", "通辽": "64", "鄂尔多斯": "283", "呼伦贝尔": "61", "巴彦淖尔": "169", "乌兰察布": "168", "兴安盟": "62", "锡林郭勒盟": "63", "阿拉善盟": "230", "沈阳": "58", "大连": "167", "鞍山": "320", "抚顺": "184", "本溪": "227", "丹东": "282", "锦州": "166", "营口": "281", "阜新": "59", "辽阳": "351", "盘锦": "228", "铁岭": "60", "朝阳": "280", "葫芦岛": "319", "长春": "53", "吉林市": "55", "四平": "56", "辽源": "183", "通化": "165", "白山": "57", "松原": "52", "白城": "51", "延边朝鲜族自治州": "54", "哈尔滨": "48", "齐齐哈尔": "41", "鸡西": "46", "鹤岗": "43", "双鸭山": "45", "大庆": "50", "伊春": "40", "佳木斯": "42", "七台河": "47", "牡丹江": "49", "黑河": "39", "绥化": "44", "大兴安岭地区": "38", "上海": "289", "南京": "315", "无锡": "317", "徐州": "316", "常州": "348", "苏州": "224", "南通": "161", "连云港": "347", "淮安": "162", "盐城": "223", "扬州": "346", "镇江": "160", "泰州": "276", "宿迁": "277", "杭州": "179", "宁波": "180", "温州": "178", "嘉兴": "334", "湖州": "294", "绍兴": "293", "金华": "333", "衢州": "243", "舟山": "245", "台州": "244", "丽水": "292", "合肥": "127", "芜湖": "129", "蚌埠": "126", "淮南": "250", "马鞍山": "358", "淮北": "253", "铜陵": "337", "安庆": "130", "黄山": "252", "滁州": "189", "阜阳": "128", "宿州": "370", "巢湖": "251", "六安": "298", "亳州": "188", "池州": "299", "宣城": "190", "福州": "300", "厦门": "194", "莆田": "195", "三明": "254", "泉州": "134", "漳州": "255", "南平": "133", "龙岩": "193", "宁德": "192", "南昌": "163", "景德镇": "225", "萍乡": "350", "九江": "349", "新余": "164", "鹰潭": "279", "赣州": "365", "吉安": "318", "宜春": "278", "抚州": "226", "上饶": "364", "济南": "288", "青岛": "236", "淄博": "354", "枣庄": "172", "东营": "174", "烟台": "326", "潍坊": "287", "济宁": "286", "泰安": "325", "威海": "175", "日照": "173", "莱芜": "124", "临沂": "234", "德州": "372", "聊城": "366", "滨州": "235", "菏泽": "353", "郑州": "268", "开封": "210", "洛阳": "153", "平顶山": "213", "安阳": "267", "鹤壁": "215", "新乡": "152", "焦作": "211", "濮阳": "209", "许昌": "155", "漯河": "344", "三门峡": "212", "南阳": "309", "商丘": "154", "信阳": "214", "周口": "308", "驻马店": "269", "武汉": "218", "黄石": "311", "十堰": "216", "宜昌": "270", "襄阳": "156", "鄂州": "122", "荆门": "217", "孝感": "310", "荆州": "157", "黄冈": "271", "咸宁": "362", "随州": "371", "恩施土家族苗族自治州": "373", "仙桃": "1713", "潜江": "1293", "天门": "2654", "神农架林区": "2734", "长沙": "158", "株洲": "222", "湘潭": "313", "衡阳": "159", "邵阳": "273", "岳阳": "220", "常德": "219", "张家界": "312", "益阳": "272", "郴州": "275", "永州": "314", "怀化": "363", "娄底": "221", "湘西土家族苗族自治州": "274", "广州": "257", "韶关": "137", "深圳": "340", "珠海": "140", "汕头": "303", "佛山": "138", "江门": "302", "湛江": "198", "茂名": "139", "肇庆": "338", "惠州": "301", "梅州": "141", "汕尾": "339", "河源": "200", "阳江": "199", "清远": "197", "东莞": "119", "中山": "187", "潮州": "201", "揭阳": "259", "云浮": "258", "南宁": "261", "柳州": "305", "桂林": "142", "梧州": "304", "北海": "295", "防城港": "204", "钦州": "145", "贵港": "341", "玉林": "361", "百色": "203", "贺州": "260", "河池": "143", "来宾": "202", "崇左": "144", "海口": "125", "三亚": "121", "五指山": "1644", "琼海": "2358", "儋州": "1215", "文昌": "2758", "万宁": "1216", "东方": "2634", "定安": "1214", "屯昌": "1641", "澄迈": "2757", "临高": "2033", "白沙黎族自治": "2359", "昌江黎族自治": "1642", "乐东黎族自治": "2032", "陵水黎族自治": "1643", "保亭黎族苗族自治": "1217", "琼中黎族苗族自治": "2031", "重庆": "132", "成都": "75", "自贡": "78", "攀枝花": "81", "泸州": "331", "德阳": "74", "绵阳": "240", "广元": "329", "遂宁": "330", "内江": "248", "乐山": "79", "南充": "291", "眉山": "77", "宜宾": "186", "广安": "241", "达州": "369", "雅安": "76", "巴中": "239", "资阳": "242", "阿坝藏族羌族自治州": "185", "甘孜藏族自治州": "73", "凉山彝族自治州": "80", "贵阳": "146", "六盘水": "147", "遵义": "262", "安顺": "263", "铜仁地区": "205", "黔西南布依族苗族自治州": "343", "毕节地区": "206", "黔东南苗族侗族自治州": "342", "黔南布依族苗族自治州": "306", "昆明": "104", "曲靖": "249", "玉溪": "106", "保山": "112", "昭通": "336", "丽江": "114", "临沧": "110", "楚雄彝族自治州": "105", "红河哈尼族彝族自治州": "107", "文山壮族苗族自治州": "177", "普洱": "108", "西双版纳傣族自治州": "109", "大理白族自治州": "111", "德宏傣族景颇族自治州": "116", "怒江傈僳族自治州": "113", "迪庆藏族自治州": "115", "拉萨": "100", "昌都地区": "99", "山南地区": "97", "日喀则地区": "102", "那曲地区": "101", "阿里地区": "103", "林芝地区": "98", "西安": "233", "铜川": "232", "宝鸡": "171", "咸阳": "323", "渭南": "170", "延安": "284", "汉中": "352", "榆林": "231", "安康": "324", "商洛": "285", "兰州": "36", "嘉峪关": "33", "金昌": "34", "白银": "35", "天水": "196", "武威": "118", "张掖": "117", "平凉": "359", "酒泉": "37", "庆阳": "135", "定西": "136", "陇南": "256", "临夏回族自治州": "182", "甘南藏族自治州": "247", "西宁": "66", "海东地区": "69", "海北藏族自治州": "67", "黄南藏族自治州": "70", "海南藏族自治州": "68", "果洛藏族自治州": "72", "玉树藏族自治州": "71", "海西蒙古族藏族自治州": "65", "银川": "360", "石嘴山": "335", "吴忠": "322", "固原": "246", "中卫": "181", "乌鲁木齐": "92", "克拉玛依": "95", "吐鲁番地区": "89", "哈密地区": "91", "昌吉回族自治州": "93", "博尔塔拉蒙古自治州": "88", "巴音郭楞蒙古自治州": "86", "阿克苏地区": "85", "克孜勒苏柯尔克孜自治州": "84", "喀什地区": "83", "和田地区": "82", "伊犁": "90", "塔城地区": "94", "阿勒泰地区": "96", "石河子": "770", "阿拉尔": "731", "图木舒克": "792", "五家渠": "789", "香港特别行政区": "2912", "澳门特别行政区": "2911"] + + def allCitys = CommonDatautils.findAllCity(); + def allRegionMap = CommonDatautils.getAllCityRegions() + + static main(args) { + def task = new ChongQingBlankTest() + task.run() + } + + def run(){ + + ExcelUtils.modify("D:\\数据\\重庆农商行\\重庆农商行测试样本(房价网v.1.2).xlsx",{row-> + if(row['B'].toString()=="重庆市九龙坡区建工三村7号"){ + def city = CommonUtils.parseCityInAddress(row["B"]) + if(!city){ + city = CommonUtils.parseCityInAddress(row["C"]) + } + if(city){ + def oldMap = null + def similarData = null + if(isAddress(row["B"])){ + def pois = GaoDeUtils.search(row["C"], city, null) + oldMap = SeaweedDataMatchProcess.doMatch2(city, row['B'], null); + similarData = findSimilarDatas(pois, row["C"], row['B'], city, oldMap) + println row['C']+" "+pois.size() + if(oldMap && !oldMap.district){ + def json = GaoDeApiUtils.geo(city, row['B']) + if(json.info=="OK" && json.geocodes){ + def rows = json.geocodes + if(!rows[0].district){ + def detailMap = getMapRegion(rows[0].location) + if(detailMap){ + oldMap.district = detailMap.district + } + }else{ + oldMap.district = rows[0].district + } + } + } + row['D'] = oldMap.province + row['E'] = oldMap.city + if(oldMap.district){ + row['F'] = RegionData.formatRegion(city, oldMap.district) + } + if(oldMap.street){ + row['G'] = oldMap.street + }else{ + row['G'] = oldMap.township + } + + if(oldMap.streetNumbers){ + row['H'] = oldMap.streetNumbers + }else{ + row['H'] = oldMap.orgKeywords + } + }else{ + if(isAddress(row["C"])){ + def pois = GaoDeUtils.search(row["B"], city, null) + oldMap = SeaweedDataMatchProcess.doMatch2(city, row['C'], null); + similarData = findSimilarDatas(pois, row["B"], row['C'], city, oldMap) + println row['B']+" "+pois.size() + if(oldMap && !oldMap.district){ + def json = GaoDeApiUtils.geo(city, row['C']) + if(json.info=="OK" && json.geocodes){ + def rows = json.geocodes + if(!rows[0].district){ + def detailMap = getMapRegion(rows[0].location) + if(detailMap){ + oldMap.district = detailMap.district + } + }else{ + oldMap.district = rows[0].district + } + } + } + }else{ + def pois = GaoDeUtils.search(row["C"], city, null) + oldMap = SeaweedDataMatchProcess.doMatch2(city, row['B'], null); + similarData = findSimilarDatas(pois, row["C"], row['B'], city, oldMap) + if(oldMap && !oldMap.district){ + def json = GaoDeApiUtils.geo(city, row['B']) + if(json.info=="OK" && json.geocodes){ + def rows = json.geocodes + if(!rows[0].district){ + def detailMap = getMapRegion(rows[0].location) + if(detailMap){ + oldMap.district = detailMap.district + } + }else{ + oldMap.district = rows[0].district + } + } + } + } + + row['D'] = oldMap.province + row['E'] = oldMap.city + if(oldMap.district){ + row['F'] = RegionData.formatRegion(city, oldMap.district) + } + if(oldMap.street){ + row['G'] = oldMap.street + }else{ + row['G'] = oldMap.township + } + + if(oldMap.streetNumbers){ + row['H'] = oldMap.streetNumbers + }else{ + row['H'] = oldMap.orgKeywords + } + + } + if(similarData){ + def newAddress = ""; + if(similarData.adname){ + def region = similarData.adname.replace("[]","") + similarData.adname = RegionData.formatRegion(city, region) + } + if(similarData.type && similarData.type.contains("村庄") && !similarData.township){ + def m = getMapRegion(similarData.location) + if(m && m.township){ + similarData.township = m.township + } + } + + if(similarData.cityname){ + newAddress = newAddress + similarData.cityname + } + if(similarData.adname){ + newAddress = newAddress + similarData.adname + } + if(similarData.address){ + if(similarData.address!=similarData.adname && !similarData.address.contains(similarData.adname)){ + newAddress = newAddress + similarData.address + }else{ + newAddress = newAddress + similarData.name + } + + } + def newMap = SeaweedDataMatchProcess.doMatch2(city, newAddress.replace("[]",""), null); + + def level = getLevel(newMap, oldMap, similarData.type); +// row['D'] = similarData.name.toString() +// row['E'] = similarData.address.toString().replace("[]","") + row['N'] = level + if(similarData.pname){ + row['I'] = similarData.pname.replace("[]","").replace("省","").replace("市","") + } + if(similarData.cityname){ + row['J'] = similarData.cityname.replace("[]","") + } + + if(similarData.adname){ + row['K'] = similarData.adname.replace("[]","") + } + +// if(level>=4){ + if(newMap.street){ + row['L'] = newMap.street.replace("[]","") + }else{ + if(similarData.township){ + row['L'] = similarData.township + } + } +// } + +// if(level>=5){ + if(newMap.streetNumber){ + row['M'] = newMap.streetNumber.replace("[]","") + }else{ + if(newMap.orgKeywords){ + row['M'] = newMap.orgKeywords + } + } + +// } + println row + } + } + } + }, 0) + } + + + + def getLevel(newMap, oldMap, type){ + def level = 0; + if(newMap && oldMap){ + if((type && type.contains("村庄")) || !type){ + if(newMap.orgKeywords == oldMap.orgKeywords){ + return 5 + } + }else{ + if(newMap.streetNumber && oldMap.streetNumber && newMap.streetNumber == oldMap.streetNumber){ + return 5 + } + } + + + if(newMap.street && oldMap.street && newMap.street.contains(oldMap.street)){ + return 4 + } + + if(newMap.district && oldMap.district && (newMap.district.contains(oldMap.district) || oldMap.district.contains(newMap.district))){ + return level = 3 + } + + if(newMap.city && oldMap.city && newMap.city == oldMap.city){ + return level = 2 + } + + if(newMap.province == oldMap.province){ + return level = 1 + } + } + return level + } + + def findSimilarDatas(pois, name, address, city, rMap){ + def data = null + def maxScore = 0 + + for(def poi : pois){ + if(!poi.cityname){ + def m = getMapRegion(poi.location) + if(m){ + poi.cityname = m.city + poi.pname = m.province + poi.township = m.township + poi.adname = m.district + } + } + if(poi.cityname.contains(city)){ + def searchName = poi.name + def nameScore = CosineSimilarAlgorithm.getSimilarityRatio(searchName, name); + + def addressScore = 0 + if(rMap.street){ + def s = rMap.street+rMap.streetNumber + if(s && poi.address){ + addressScore = CosineSimilarAlgorithm.getSimilarityRatio(poi.address, s); + } + } + def totalScore = nameScore + addressScore + if(totalScore > maxScore){ + maxScore = totalScore + data = poi + data.score = totalScore + } + println searchName+" "+name+" "+totalScore + } + } + return data; + } + + def isAddress(address){ + def REG_ADDR1 = "[\u4e00-\u9fa5]{2,15}[路|巷|街|弄|街道|大道|大街|巷子|公路|马路|大道西段]\\d+"; + def pattern = Pattern.compile(REG_ADDR1); + def matcher = pattern.matcher(address) + if(matcher.find()){ + return true; + } + return false; + } + + def getMapRegion(def location){ + def jsonObj = GaoDeApiUtils.geocode(location) + def district = null + if(jsonObj && jsonObj.status=="1"){ + def detailMap = jsonObj.regeocode.addressComponent + if(!detailMap.city){ + if(detailMap.province && detailMap.province.toString().endsWith("市")){ + detailMap.city = detailMap.province + } + } + return detailMap + } + return district; + } + + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CityStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CityStat.groovy new file mode 100644 index 000000000..9b523bef2 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CityStat.groovy @@ -0,0 +1,580 @@ +package com.fangjia.script.gj.test + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicInteger +import java.util.concurrent.atomic.AtomicLong + +class CityStat { + + public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; + public static final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName2("fangjia_stat", "district_stat", lineUrl) + + public static final DBCollection BAD_CASE_COLL = MongoUtils.getCollectionByName2("fangjia_stat", "bad_case", lineUrl) + + public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) + static main(args) { + def task = new CityStat() +// task.run() +// task.run1(); + task.run3(); +// task.run4() +// task.run6() +// task.run7() +// task.run8() + } + +// def run8(){ +// ExcelUtils.multiThreadingModify("D:\\数据\\深圳建行\\建行匹配小区(3).xlsx",{d-> +// +// def dist = CommonDatautils.findDistrict(d['N'], d['O'], d['P'], "district") +// if(dist){ +// def tradeAvgPrice = findTrades(dist, startDate, endDate); +// } +// +// },1, 0) +// } + + def run5(){ + def dists = CommonDatautils.findAllDistrictByCity("常州","district",["city", "region","name","alias"]) + def total = 0 + def scount = 0 + def index = 0 + // final CountDownLatch latch = new CountDownLatch(districts.size()) + + for(def dist : dists){ + index++ + def rets = findTradeDetails(dist, "2020-03-24"); + println index + if(rets){ + total = total + rets.size() + def count = findSimialPriceCount(dist, rets); + scount = scount+count + } + } + println scount +" "+ total + def pre = DataHandleUtils.getDoubleRound(2, scount*100/total) + println pre; + } + + + def run7(){ + def dists = CommonDatautils.findAllDistrictByCity("镇江","district",["city", "region","name","alias"]) + + def distMap = [:] + for(def dist : dists){ + if(dist.alias) { + + if (!dist.alias.contains(dist.name)) { + dist.alias << dist.name + } + for(def name : dist.alias){ + distMap.put( dist.region +"#"+name, dist) + } + }else{ + distMap.put( dist.region +"#"+dist.name, dist) + } + } + def rets = findTradeDetails2("镇江","2020-03-24"); + def total = 0 + def scount = 0 + + def index = 0 + for(def t : rets){ + def key = t.region+"#"+t.district_name + def dist = distMap.get(key) + index++ + println index + if(dist){ + total++ + def startDate = DateUtils.add(DateUtils.formatToYYYYMMDD(t.trade_date),-7) + def endDate = DateUtils.add(DateUtils.formatToYYYYMMDD(t.trade_date),7) + def weekendPrice = CommonDatautils.queryDistrictPriceByDate(dist.city, dist.region, dist.name, startDate, endDate) + if(t.avg_price && weekendPrice){ + def mod = (weekendPrice - t.avg_price)*100/t.avg_price + def m = Math.abs(mod) + if(m<=15){ + scount++; + } + } + } + } + + println scount +" "+total + + } + + def run6(){ + def dists = CommonDatautils.findAllDistrictByCity("常州","district",["city", "region","name","alias"]) + + final CountDownLatch latch = new CountDownLatch(dists.size()) + def totalCount = new AtomicLong() + def tradeTotal = new AtomicLong() + def scount = new AtomicLong() + + for(def i=0;i + if(data['A']){ + def startDate = DateUtils.addMonth(data['D'], -3) + def endDate = DateUtils.addMonth(data['D'], 3) + def dist = CommonDatautils.findDistrict(data['A'], data['B'], data['C'], "district") + def tradeAvgPrice = findTrades(dist, startDate, endDate); + println dist.city +" "+dist.region+" "+dist.name +" "+tradeAvgPrice.toInteger() + data['H'] = tradeAvgPrice.toInteger() + } + }) + } + + def run4(){ + def citys = ["上海","北京","大连","厦门","深圳","杭州","成都","重庆","苏州","广州"] + def startDate = DateUtils.addMonth(new Date(), -3) + def endDate = "2021-02-01" + def result = [] + def allDistricts = [] + for(def city : citys) { + def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias"]) + allDistricts.addAll(districts) + println city +" "+districts.size() + + } + def rets = findDistrictTrade(allDistricts, startDate,endDate); + println rets.size() + + + } + + def findDistrictTrade(def districts, def startDate, def endDate){ + def rets = [] + final CountDownLatch latch = new CountDownLatch(districts.size()) + def final AtomicInteger count_ = new AtomicInteger(0) + for(def i=0;i + if(v && v.size()>0){ + def sum = v.sum() + def tradeAvgPrice = sum/v.size() + def arrs = k.split("#") + def stat = findDistrictStat(city, arrs[0], arrs[1], DateUtils.parseFromTightByPattern("2021-01-17", "yyyy-MM-dd")) + rets << [city, arrs[0], arrs[1], tradeAvgPrice.toInteger(), stat.avg_price] + } + } + println city +" 完成" + } + ExcelUtils.write("d:\\数据\\统计\\城市成交数据.xlsx", rets, "成交") + } + + def run2(){ + def datas = ExcelUtils.read("D:\\数据\\深圳建行\\建行匹配小区2.xlsx",0,1) + def cityDataMap = [:] + def index = 0 + for(def data : datas){ + index++ + println index+" "+data + def lists = cityDataMap.get(data['A']) + if(!lists){ + lists = [] + cityDataMap.put(data['A'], lists) + } + def dist = CommonDatautils.findDistrict(data['A'], data['B'], data['C'], "district") + dist.calDate = data['D'] + dist.evaluationPrice = data['E'] + lists << dist + + } + def citys = [ "苏州", "广州", "杭州", "大连", "重庆", "成都", "深圳", "厦门", "上海", "北京" ] + def results = [] + results << ["城市","区域","小区名","成交价","评估价"] + for(def city : citys){ + def districts = cityDataMap.get(city) + def rets = findAvgPriceTrades(city, districts) + results.addAll(rets) + } + ExcelUtils.write("D:\\数据\\深圳建行\\建行成交3.xlsx",results, "数据") + } + + def findAvgPriceTrades(city, districts){ + def rets = [] + final CountDownLatch latch = new CountDownLatch(districts.size()) + for(def i=0;i 3){ + avg_price = total_price/count + } + return avg_price + } + + + + def findTradeDetails(dist, startDate){ + def q = new BasicDBObject() + q.put("city", dist.city) + q.put("region", dist.region) + + if(dist.alias){ + if(!dist.alias.contains(dist.name)){ + dist.alias << dist.name + } + q.put("district_name", new BasicDBObject('$in', dist.alias)) + }else{ + q.put("district_name", dist.name) + } + + q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate))) + def dbcorsor = DEAL_PRICE_COLL.find(q) + + def rets = [] + for(def d : dbcorsor){ + if(d.avg_price){ + rets << d + } + } + + return rets + } + + + + def findTradeDetails2(city, startDate){ + def q = new BasicDBObject() + q.put("city", city) + q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate))) + def dbcorsor = DEAL_PRICE_COLL.find(q) + + def rets = [] + for(def d : dbcorsor){ + if(d.avg_price){ + rets << d + } + } + return rets + } + + + + def findStatCount(def city, def weekend, def value_type){ + def q = new BasicDBObject() + q.put("city", city) + q.put("weekend", weekend) + if(value_type){ + q.put("value_type", value_type) + } + def count = DIST_STAT_COLL.count(q) + return count; + } + + def findDistrictStat(def city, def region, def name, def weekend){ + def q = new BasicDBObject() + q.put("city", city) + q.put("region", region) + q.put("district_name", name) + q.put("weekend", weekend) + def obj = DIST_STAT_COLL.findOne(q) + return obj; + } + + def findBadCaseCount(def city, def weekend){ + def q = new BasicDBObject() + q.put("city", city) + q.put("weekend", weekend) + def count = BAD_CASE_COLL.count(q) + return count; + } + + def findDealPriceCount(def city, def tradeDate){ + def q = new BasicDBObject() + q.put("city", city) + q.put("trade_date", new BasicDBObject('$gte', tradeDate)) + def count = DEAL_PRICE_COLL.count(q) + return count; + } + + def calTradeDistCount(def city, def tradeDate, def districts){ + def q = new BasicDBObject() + q.put("city", city) + q.put("source","链家在线") + q.put("trade_date", new BasicDBObject('$gte', tradeDate)) + def dbcorsor = DEAL_PRICE_COLL.find(q) + + def distMap = [:] + for(def dist :districts){ + def lists = distMap.get(dist.region) + if(!lists){ + lists = [] + distMap.put(dist.region, lists); + } + lists << dist + } + + def rets = [] + for(def obj : dbcorsor){ + def dist = findIsExistsDB(obj.region, obj.district_name, distMap); + if(dist && !rets.contains(dist)){ + rets << dist + } + } + return rets.size(); + } + + def findIsExistsDB(def region, def district_name, def distMap){ + def dist = null; + def regionDists = distMap.get(region) + for(def rd : regionDists){ + def alias = rd.alias + if(!alias){ + alias = [] + } + if(!alias.contains(rd.name)){ + alias << rd.name + } + if(alias.contains(district_name)){ + dist = rd + break; + } + } + return dist; + } + + + def calTradeDistDetail(def city, def tradeDate, def districts){ + def q = new BasicDBObject() + q.put("city", city) + q.put("trade_date", new BasicDBObject('$gte', tradeDate)) + def dbcorsor = DEAL_PRICE_COLL.find(q) + + def distMap = [:] + for(def dist :districts){ + def lists = distMap.get(dist.region) + if(!lists){ + lists = [] + distMap.put(dist.region, lists); + } + lists << dist + } + + def retMap = [:] + for(def obj : dbcorsor){ + def dist = findIsExistsDB(obj.region, obj.district_name, distMap); + if(dist){ + def tradePrices = retMap.get(dist.region+"#"+dist.name) + if(!tradePrices){ + tradePrices = [] + retMap.put(dist.region+"#"+dist.name, tradePrices) + } + if(obj.avg_price){ + tradePrices << obj.avg_price.toInteger() + } + } + } + return retMap; + } + + + def calTradeDistDetail2(def city, def tradeDate, def districts){ + def q = new BasicDBObject() + q.put("city", city) + q.put("trade_date", new BasicDBObject('$gte', tradeDate)) + def dbcorsor = DEAL_PRICE_COLL.find(q) + + def distMap = [:] + for(def dist :districts){ + def lists = distMap.get(dist.region) + if(!lists){ + lists = [] + distMap.put(dist.region, lists); + } + lists << dist + } + + def retMap = [:] + for(def obj : dbcorsor){ + def dist = findIsExistsDB(obj.region, obj.district_name, distMap); + if(dist){ + def tradePrices = retMap.get(dist.region+"#"+dist.name) + if(!tradePrices){ + tradePrices = [] + retMap.put(dist.region+"#"+dist.name, tradePrices) + } + if(obj.avg_price){ + tradePrices << obj.avg_price.toInteger() + } + } + } + return retMap; + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CompanyAddressCompare.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CompanyAddressCompare.groovy new file mode 100644 index 000000000..081ede856 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CompanyAddressCompare.groovy @@ -0,0 +1,111 @@ +package com.fangjia.script.gj.test + +import com.fangjia.core.RegionData +import com.fangjia.script.gj.district.SeaweedDataMatchProcess +import com.fangjia.script.gj.paimai.GaoDeUtils +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.CosineSimilarAlgorithm +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils + +import java.util.regex.Pattern + +class CompanyAddressCompare { + static main(args) { + def task = new CompanyAddressCompare() + task.run() + } + def run(){ + ExcelUtils.modify("D:\\数据\\公司及地址\\公司及地址.xlsx",{row-> + if(row['D'] && !row['I'] && row['B'].contains("宁波甬城农村商业银行")){ + def city = CommonUtils.parseCityInAddress(row["D"]) + if(!city){ + city = CommonUtils.parseCityInAddress(row["B"]) + } + if(city){ + def similarData = null +// if(isAddress(row["D"])){ + def simpleAddress = row["D"].toString().replace("中国(上海)自由贸易试验区","").replaceAll(/\\d+层/,"").replaceAll(/[一二三四五六]楼/,"") + if(simpleAddress && simpleAddress.contains("号")){ + simpleAddress = simpleAddress.substring(0, simpleAddress.lastIndexOf("号")+1) + } + def pois = GaoDeUtils.search(simpleAddress, city, null) + similarData = findSimilarDatas(pois, row["B"], simpleAddress, city, null) + println row['C']+" "+pois.size() +// } + + if(similarData){ + row['F'] = similarData.address + row['G'] = similarData.name + row['H'] = similarData.score + } + } + } + }, 0) + } + + + def findSimilarDatas(pois, name, address, city, rMap){ + def data = null + def maxScore = 0 + + for(def poi : pois){ + if(!poi.cityname){ + def m = getMapRegion(poi.location) + if(m){ + poi.cityname = m.city + poi.pname = m.province + poi.township = m.township + poi.adname = m.district + } + } + if(poi.cityname.contains(city)){ + def searchName = poi.name + def nameScore = CosineSimilarAlgorithm.getSimilarityRatio(searchName, name); + def addressScore = 0 + if(poi.address){ + def poiAddress = poi.address.replace("号楼","号") + addressScore = CosineSimilarAlgorithm.getSimilarityRatio(poiAddress, address); + if(poi.address.contains("号") && address.contains("号") && (poiAddress.contains(address) || address.contains(poiAddress))){ + addressScore = addressScore + 0.5 + } + } + + def totalScore = nameScore + addressScore + if(totalScore > maxScore){ + maxScore = totalScore + data = poi + data.score = totalScore + } + println searchName+" "+name+" "+totalScore + " "+ (poi.address?poi.address:"") + } + } + return data; + } + + def isAddress(address){ + def REG_ADDR1 = "[\u4e00-\u9fa5]{2,20}[路|巷|街|弄|街道|大道|大街|巷子|公路|马路|大道西段]\\d+"; + def pattern = Pattern.compile(REG_ADDR1); + def matcher = pattern.matcher(address) + if(matcher.find()){ + return true; + } + return false; + } + + def getMapRegion(def location){ + def jsonObj = GaoDeApiUtils.geocode(location) + def district = null + if(jsonObj && jsonObj.status=="1"){ + def detailMap = jsonObj.regeocode.addressComponent + if(!detailMap.city){ + if(detailMap.province && detailMap.province.toString().endsWith("市")){ + detailMap.city = detailMap.province + } + } + return detailMap + } + return district; + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CrawlDistrictIdSelect.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CrawlDistrictIdSelect.groovy new file mode 100644 index 000000000..8e8bba001 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CrawlDistrictIdSelect.groovy @@ -0,0 +1,102 @@ +package com.fangjia.script.gj.test + +import com.fangjia.core.RegionData +import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +class CrawlDistrictIdSelect { + def static final DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol(); + + static DBCollection districtseedsColl = MongoUtils.getCollectionByAuth20190102("seeddb", "district_seeds") + static DBCollection districtseeds2Coll = MongoUtils.getCollectionByAuth20190102("seeddb_dev", "district_seeds") + + static main(args) { + def task = new CrawlDistrictIdSelect() + task.run() +// task.synch() + } + + def synch(){ + def q = new BasicDBObject() + def dbcursor = districtseeds2Coll.find(q) + def list = [] + for(def d :dbcursor){ + list << d + if(list.size()==10000){ + districtseedsColl.insert(list) + list = [] + } + } + + if(list.size()>0){ + districtseedsColl.insert(list) + } + } + + def run(){ + def datas = findCrawlData() + datas.each {k,v-> + if(v){ + for(def d : v){ + d.c_date = new Date() + d.m_date = new Date() + districtseedsColl.save(d) + } + } + } + } + + def findCrawlData(){ + def q = new BasicDBObject() + q.put("city","福州") + q.put("source", "安居客") + q.put("s_date",new BasicDBObject('$gte', 202001)) + def dbcursor = thirdPartyColl.find(q) + def retMap = [:] + def prefix = "/community/view/" + def keyList =[] + for(def db : dbcursor){ + def url = null + if(!db.url){ + if(db.district_id){ + url = db.district_id + } + }else{ + url = db.url + } + if(url && url.contains(prefix) ){ + def district_id = url.substring(url.indexOf(prefix)+prefix.length(),url.size()) + if(district_id=="0"){ + continue + } + def shortCity = url.substring(url.indexOf("https://")+8, url.indexOf(".anjuke.com")) + def region = RegionData.formatRegion(db.city, db.region) + def key = shortCity +"#"+district_id + + def ids = retMap.get(key) + if(!ids){ + ids = [] + retMap.put(key, ids) + } + + if(!keyList.contains(key)){ + def map = new BasicDBObject() + map.root = "anjuke.com" + map.city_short= shortCity + map.city = db.city + map.region = region + map.district_id = district_id + map.source_url = url + ids << map + + keyList << key + } + }else{ + println url +" 链接不合要求" + } + } + return retMap + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/ExportToDealPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/ExportToDealPrice.groovy new file mode 100644 index 000000000..d4440f329 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/ExportToDealPrice.groovy @@ -0,0 +1,101 @@ +package com.fangjia.script.gj.test + +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.QueryOperators + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicInteger + +class ExportToDealPrice { + + DBCollection DEAL_PRICE_MATCH_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price_match", "192.168.0.43") + DBCollection DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43") + + DBCollection TRADE_MATCH_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "trade_match_price", "192.168.0.136") + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(50) + final AtomicInteger count = new AtomicInteger() + + static main(args) { + def task = new ExportToDealPrice(); + task.run() + } + + def run(){ + def datas = findDatas() + + final CountDownLatch latch = new CountDownLatch(datas.size()) + for(def i=0;i0){ + TRADE_MATCH_PRICE_COLL.insert(ret); + } + } + + def addPrice(data){ + def weekend = DateUtils.getFirstDayOfWeek(data.trade_date) + def q = new BasicDBObject() + q.put("city", data.match_city) + q.put("region", data.match_region) + q.put("district_name", data.match_name) + q.put("weekend", weekend) + def dbo = DISTRICT_STAT_COLL.findOne(q) + if(!dbo){ + q.put("weekend", new BasicDBObject(QueryOperators.LTE, weekend)) + dbo = DISTRICT_STAT_COLL.findOne(q) + } + println count.get()+" "+data.city +" "+data.match_name + return dbo + } + + def findDatas(){ + def q = new BasicDBObject() + q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2020-04-01","yyyy-MM-dd"))) + def dbcursor = DEAL_PRICE_MATCH_COLL.find(q) + def rets = [] + for(def db : dbcursor){ + rets << db + } + return rets; + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/Test.groovy index 1a1f3fb66..3dbb1a3e2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/Test.groovy @@ -1,3 +1,7 @@ package com.fangjia.script.gj.test -println "wwww" \ No newline at end of file +import java.text.SimpleDateFormat + +SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); +Date date = new Date(1617211269868); +print dateFormat.format(date); \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/other/AnjukeCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/other/AnjukeCrawl.groovy new file mode 100644 index 000000000..6a6a2c452 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/other/AnjukeCrawl.groovy @@ -0,0 +1,52 @@ +package com.fangjia.script.thirdPriceCrawl.other + +import com.fangjia.script.util.ABuYunUtil +import org.apache.commons.io.IOUtils +import org.apache.http.HttpResponse +import org.apache.http.client.HttpClient +import org.apache.http.client.methods.HttpGet +import org.apache.http.impl.client.HttpClients + +class AnjukeCrawl { + + static main(args) { + def task = new AnjukeCrawl(); + task.run() + } + + def run(){ + HttpClient client = HttpClients.createDefault(); + HttpGet get = new HttpGet("https://m.anjuke.com/cityList/"); +// get.setHeader("Upgrade-Insecure-Requests", "1"); +// get.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36"); + +// get.setHeader("Host","m.anjuke.com"); +// get.setHeader("Connection","keep-alive"); +// get.setHeader("Cache-Control","max-age=0"); +// get.setHeader('sec-ch-ua','"Chromium";v="88", "Google Chrome";v="88", ";Not A Brand";v="99"'); +// get.setHeader("sec-ch-ua-mobile","'?0"); +// get.setHeader("Upgrade-Insecure-Requests","1"); + def header = [:] + header.put("User-Agent","Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.104 Safari/537.36"); + header.put("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9"); +// get.setHeader("Sec-Fetch-Site","none"); +// get.setHeader("Sec-Fetch-Mode","navigate"); +// get.setHeader("Sec-Fetch-User","?1"); +// get.setHeader("Sec-Fetch-Dest","document"); +//// get.setHeader("Accept-Encoding"," gzip, deflate, br"); +// get.setHeader("Accept-Language","zh-CN,zh;q=0.9"); + + + def s1 = ABuYunUtil.doGet("https://m.anjuke.com/cityList/", null, header, "utf-8") + printl s1 + + HttpResponse response = client.execute(get); + def s = getResponseBodyAsString(response) + println s + } + + private static String getResponseBodyAsString(HttpResponse response) throws IOException { + return IOUtils.toString(response.getEntity().getContent(), "UTF-8"); + } + +} From 2ad2c3048029df79ddcb587c042a09eaccc9060c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=96=87=E8=89=B3?= Date: Sat, 8 May 2021 11:03:01 +0800 Subject: [PATCH 02/17] =?UTF-8?q?=E5=8D=87=E7=BA=A7Es=E9=A9=B1=E5=8A=A8?= =?UTF-8?q?=E7=89=88=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/fangjia/open/controllers/AddressController.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java index c9973367c..e3a690358 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java @@ -1705,7 +1705,7 @@ public class AddressController { return mav; } - public Object addressSimilarityAction(AddesssMatchModel model) { + public Object similarityAction(AddesssMatchModel model) { RestfulModelAndView mav = new RestfulModelAndView(); if (StringUtils.isBlank(model.getAddress_1()) || StringUtils.isBlank(model.getAddress_2())) { mav.setModel(BasicModel.buildBadRequestModel("两个地址均不能为空")); From c36d79e1f14c783c3585ef6204a5165dd6478c36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=96=87=E8=89=B3?= Date: Sat, 8 May 2021 13:57:03 +0800 Subject: [PATCH 03/17] =?UTF-8?q?=E5=8D=87=E7=BA=A7Es=E9=A9=B1=E5=8A=A8?= =?UTF-8?q?=E7=89=88=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/script/wenyan/test/Test1.groovy | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/test/Test1.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/test/Test1.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/test/Test1.groovy new file mode 100644 index 000000000..fb1b341be --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/test/Test1.groovy @@ -0,0 +1,26 @@ +package com.fangjia.script.wenyan.test + +import com.fangjia.core.util.HttpRequest +import com.fangjia.script.util.ExcelUtils +import net.sf.json.JSONObject + +ExcelUtils.multiThreadingModify("D:\\数据\\建行地址相似度评级测试.xlsx",{List item-> + def city = item["A"] + def address1 = item["B"] + def address2 = item["C"] + HttpRequest request = new HttpRequest() + def params = new HashMap() + params["city"] = city + params["address_1"] = address1 + params["address_2"] = address2 + params["token"]= "F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB" + JSONObject json = request.getJsonContent("http://192.168.20.86:9180/address/similarity", params, "UTF-8") + JSONObject result = json.result + item["D"] = result?.level + item["E"] = result?.score + item["F"] = result?.formattedAddress_1 + item["G"] = result?.formattedAddress_2 + item["H"] = result?.isSameAddress + println(item) +}) + From f69c8bd02801029d5d5ddb502bcc34836a931b3a Mon Sep 17 00:00:00 2001 From: xuhe Date: Sun, 9 May 2021 12:34:36 +0800 Subject: [PATCH 04/17] =?UTF-8?q?=E7=9B=B8=E4=BC=BC=E5=BA=A6=E8=AF=84?= =?UTF-8?q?=E7=BA=A7=E4=BB=A3=E7=A0=81=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../service/web/AddressServiceImpl.java | 62 +++++++++---------- 1 file changed, 30 insertions(+), 32 deletions(-) diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/web/AddressServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/web/AddressServiceImpl.java index 221fecc5b..8b8a1f1fc 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/web/AddressServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/web/AddressServiceImpl.java @@ -5,18 +5,7 @@ import java.math.BigDecimal; import java.math.RoundingMode; import java.net.URLEncoder; import java.text.DecimalFormat; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.Comparator; -import java.util.Date; -import java.util.HashMap; -import java.util.Iterator; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; @@ -5138,30 +5127,39 @@ public class AddressServiceImpl implements AddressService { return null; } /** - * 获取两地址的相似度评级(最小颗粒度) + * 获取两地址的相似度评级(相同数据的最小颗粒度) * @return */ private String getAddressSimilarityLevel(JSONObject json1, JSONObject json2){ - if(StringUtils.isNotBlank(getFromJson(json1,"roomNumber")) || StringUtils.isNotBlank(getFromJson(json2,"roomNumber"))){ - return "roomNumber"; - }else if(StringUtils.isNotBlank(getFromJson(json1,"unitNumber")) || StringUtils.isNotBlank(getFromJson(json2,"unitNumber"))){ - return "unitNumber"; - }else if(StringUtils.isNotBlank(getFromJson(json1,"buildingNumber")) || StringUtils.isNotBlank(getFromJson(json2,"buildingNumber"))){ - return "buildingNumber"; - }else if(StringUtils.isNotBlank(getFromJson(json1,"name")) || StringUtils.isNotBlank(getFromJson(json2,"name"))){ - return "name"; - }else if(StringUtils.isNotBlank(getFromJson(json1,"street")) || StringUtils.isNotBlank(getFromJson(json2,"street"))){ - return "street"; - }else if(StringUtils.isNotBlank(getFromJson(json1,"township")) || StringUtils.isNotBlank(getFromJson(json2,"township"))){ - return "township"; - }else if(StringUtils.isNotBlank(getFromJson(json1,"district")) || StringUtils.isNotBlank(getFromJson(json2,"district"))){ - return "district"; - }else if(StringUtils.isNotBlank(getFromJson(json1,"city")) || StringUtils.isNotBlank(getFromJson(json2,"city"))){ - return "city"; - }else if(StringUtils.isNotBlank(getFromJson(json1,"province")) || StringUtils.isNotBlank(getFromJson(json2,"province"))){ - return "province"; + String level = null; + LinkedList list = new LinkedList<>(Arrays.asList("province", "city", "district", "township", "street" + , "name", "buildingNumber", "unitNumber", "roomNumber")); + for(String key:list){ + if(continueJudgeValue(json1, json2, key) == 3){ + return level; + }else{ + level = continueJudgeValue(json1, json2, key) == 1 ? key : (level == null ? null : level); + } } - return null; + return level; + } + + /** + * 判断level时是否继续往下判断 + * @param json1 + * @param json2 + * @param key + * @return 1 继续判断并且level有值 2继续判断level没值 3 停止 + */ + private Integer continueJudgeValue(JSONObject json1, JSONObject json2, String key){ + if(StringUtils.isNotBlank(getFromJson(json1,key)) && StringUtils.isNotBlank(getFromJson(json2,key))){ + if(getFromJson(json1,key).equals(getFromJson(json2,key))){ + return 1; + }else{ + return 3; + } + } + return 2; } /** * 判断省、市、区、乡镇、楼栋、单元、室号是否相同(两者全为空时返回true) From 2469d03c29b694a9e5049c072bc5f683204df637 Mon Sep 17 00:00:00 2001 From: wangwenyan Date: Sun, 9 May 2021 16:34:56 +0800 Subject: [PATCH 05/17] =?UTF-8?q?=E6=B7=BB=E5=8A=A0Dockers=E6=96=87?= =?UTF-8?q?=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/service/web/AddressServiceImpl.java | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/web/AddressServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/web/AddressServiceImpl.java index 8b8a1f1fc..1e04da6c0 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/web/AddressServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/web/AddressServiceImpl.java @@ -2348,11 +2348,10 @@ public class AddressServiceImpl implements AddressService { /** * 基于高德开放接口的地址真实性检查 - * * @param city * @param address * @param checkBo - * @param time + * @param addressList */ private void checkAmapNew(String city, String address, AddressCheckBo checkBo, Set addressList) { try { @@ -3571,7 +3570,7 @@ public class AddressServiceImpl implements AddressService { * * @param city * @param address - * @param checkBo + * @param keywordsFormat */ private AddressCheckTruth checkTruthSplit(String city, String address, Map keywordsFormat) { // 1.拆分 @@ -4580,8 +4579,7 @@ public class AddressServiceImpl implements AddressService { /** * 地址评级具体逻辑 - * - * @param checkBo + * @param checkTr * @return */ private AddressCheckTruth gradeAddressTruth(AddressCheckTruth checkTr) { @@ -4604,8 +4602,8 @@ public class AddressServiceImpl implements AddressService { /** * 小区评分 - * - * @param checkBo + * @param check + * @param isConsume * @return */ private boolean gradeDistrictTruth(AddressCheckTruth check,boolean isConsume) { @@ -5132,7 +5130,7 @@ public class AddressServiceImpl implements AddressService { */ private String getAddressSimilarityLevel(JSONObject json1, JSONObject json2){ String level = null; - LinkedList list = new LinkedList<>(Arrays.asList("province", "city", "district", "township", "street" + LinkedList list = new LinkedList(Arrays.asList("province", "city", "district", "township", "street" , "name", "buildingNumber", "unitNumber", "roomNumber")); for(String key:list){ if(continueJudgeValue(json1, json2, key) == 3){ From 4d4602d7ad9a51d86c34e719b84674a28a6c70d9 Mon Sep 17 00:00:00 2001 From: xuhe Date: Sun, 9 May 2021 17:39:25 +0800 Subject: [PATCH 06/17] =?UTF-8?q?=E5=9C=B0=E5=9D=80=E6=A0=87=E5=87=86?= =?UTF-8?q?=E5=8C=96=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../open/controllers/AddressController.java | 1 + .../address/AddressParserServiceImpl.java | 176 ++++++++++++++---- .../fangjia/service/address/mode/Address.java | 10 + .../service/address/mode/MatchParams.java | 8 + .../processor/StageAndPartitionProcessor.java | 1 + .../service/api/AliyunServiceImpl.java | 72 +++++-- .../service/web/AddressServiceImpl.java | 2 +- 7 files changed, 222 insertions(+), 48 deletions(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java index e3a690358..96bbe097f 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java @@ -1769,6 +1769,7 @@ public class AddressController { // matchParams.setUseMap(true); matchParams.setUseAliyunApi(true); +// matchParams.setUseCrf(true); matchParams.setContainsLocation(model.isContainsLocation()); Building building = addressParserService.standard(matchParams); Address address = building.getSearchAddress(); diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java index 8bc805491..9eb8f4114 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java @@ -1,14 +1,12 @@ package com.fangjia.service.address; import com.fangjia.core.RegionData; -import com.fangjia.core.util.CNNMFilter; -import com.fangjia.core.util.DataUtil; -import com.fangjia.core.util.GeoUtils; -import com.fangjia.core.util.StringUtil; +import com.fangjia.core.util.*; import com.fangjia.dal.QueryParam; import com.fangjia.dal.common.DistrictKey; import com.fangjia.dal.common.ServerAreaData; import com.fangjia.dal.mongo.fangjia.model.ServerArea; +import com.fangjia.dal.mongo.model.AmapPoi; import com.fangjia.dal.mongo.model.bkoff.HouseLoudong; import com.fangjia.dal.mongo.model.bkoff.Loudong; import com.fangjia.dal.seaweed.dao.SeaweedDao; @@ -25,6 +23,7 @@ import com.fangjia.service.mongo.bkoff.HouseLoudongService; import com.fangjia.service.mongo.bkoff.LoudongService; import com.google.common.base.Function; import com.google.common.collect.MapMaker; +import net.sf.json.JSONObject; import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; import org.bson.types.ObjectId; @@ -134,6 +133,8 @@ public class AddressParserServiceImpl implements AddressParserService { formatAddress(building); } return building; + /*JSONObject result = getDataFromCRF("test"); + return new Building();*/ } @Override @@ -165,6 +166,21 @@ public class AddressParserServiceImpl implements AddressParserService { return building; } Building matchedBuilding; + //调用CRF算法切分数据 + /*JSONObject jsonObj = null; + if(params.isUseCrf() && AddressParserGoal.Match.equals(addr.getAddressParserGoal())){ + jsonObj = getDataFromCRF(addr.getOrgAddress()); + if(jsonObj != null){ + String poi = (jsonObj.containsKey("community_model") ? jsonObj.getString("community_model") : "") + + (jsonObj.containsKey("poi_model") ? jsonObj.getString("poi_model") : "") + + (jsonObj.containsKey("subpoi_model") ? jsonObj.getString("subpoi_model") : ""); + if(StringUtils.isNotBlank(poi)){ + if(addr.getKeywords()==null || !addr.getKeywords().contains(poi)){ + addr.setKeywords(poi); + } + } + } + }*/ //keywords使用aliyun接口返回的poi AliyunBo aliyunBo = null; if(params.isUseAliyunApi() && AddressParserGoal.Match.equals(addr.getAddressParserGoal())){ @@ -176,6 +192,9 @@ public class AddressParserServiceImpl implements AddressParserService { addr.setKeywords(poi); } } + if(StringUtils.isNotBlank(aliyunBo.getCity())){ + addr.setCity(aliyunBo.getCity()); + } if(StringUtils.isNotBlank(addr.getKeywords()) && addr.getOrgAddress().indexOf(addr.getKeywords())>-1){ String str = AddressParserUtil.match(Pattern.compile(".*村(.*(组|队))?\\d+(号)?$"), addr.getOrgAddress().substring(addr.getOrgAddress().indexOf(addr.getKeywords()),addr.getOrgAddress().length())); if(StringUtils.isNotBlank(str)){ @@ -187,26 +206,80 @@ public class AddressParserServiceImpl implements AddressParserService { } } } - if(StringUtils.isNotBlank(aliyunBo.getCity())){ - addr.setSuggestCity(aliyunBo.getCity()); - } - } - } - AddressMatchMetaData meta = null; - if(StringUtils.isBlank(addr.getCity())){ - addr.addKeyword(addr.getOrgAddress()); - meta = getMapMeta(addr); - if(meta != null){ - addr.setProvince(meta.getMapProvince()); - addr.setCity(meta.getMapCity()); - addr.setDistrict(meta.getDistrict()); } } + addr.addKeyword(addr.getOrgAddress()); if(params.isUseMap()){ matchedBuilding = findMatchedBuilding(addr, params.getBuildingType(), MetaDataType.MAP); }else{ matchedBuilding = findMatchedBuilding(addr, params.getBuildingType(), MetaDataType.CPDB); } + + /*if(jsonObj != null){ + //乡镇优化 + String township = jsonObj.containsKey("town") ? jsonObj.getString("town") : ""; + if(StringUtils.isBlank(addr.getTownship()) && StringUtils.isNotBlank(township)){ + String town = AddressParserUtil.match(AddressParserPatterns.TOWN, township); + if(StringUtils.isNotBlank(town)){ + addr.setTownship(town); + } + } + //道路优化(两边均去掉乡镇) + String street = addr.getStreet(); + String streetNumber = addr.getStreetNumber(); + String streetCrf = jsonObj.containsKey("road") ? jsonObj.getString("road") : ""; + String streetNumberCrf = jsonObj.containsKey("roadno") ? jsonObj.getString("roadno") : ""; + //去掉路名中含有乡镇名的 + if(StringUtils.isNotBlank(addr.getTownship())){ + street = StringUtils.isNotBlank(street) ? street.replace(addr.getTownship(),"") : street; + streetCrf = StringUtils.isNotBlank(streetCrf) ? streetCrf.replace(addr.getTownship(),"") : streetCrf; + } + + //crf解析出的路名为空、与Java解析出路名一致或不满足正则则不处理 + if(StringUtils.isNotBlank(streetCrf) && !streetCrf.equals(street) && StringUtils.isNotBlank(AddressParserUtil.match(AddressParserPatterns.STREET, streetCrf))){ + //Java解析出路名为空 + if(StringUtils.isBlank(street)){ + street = streetCrf; + if(StringUtils.isNotBlank(streetNumberCrf)){ + streetNumber = streetNumberCrf; + } + }else{ + //CRF包含java解析出的路名或Java解析出的路名包含CRF解析出的路名并且多余部分也满足路名正则 + if(streetCrf.contains(street) || (street.contains(streetCrf) && StringUtils.isBlank(AddressParserUtil.match(AddressParserPatterns.STREET, street.replace(streetCrf,""))))){ + street = streetCrf; + } + } + addr.setStreet(street); + addr.setStreetNumber(streetNumber); + } + //poi处理 + //没有匹配上库中小区时 + if(StringUtils.isBlank(matchedBuilding.getId())){ + String poi = (jsonObj.containsKey("community") ? jsonObj.getString("community") : "") + + (jsonObj.containsKey("poi") ? jsonObj.getString("poi") : "") + + (jsonObj.containsKey("subpoi") ? jsonObj.getString("subpoi") : ""); + //去掉路名中含有乡镇名和路名 getParsedPropertyName + if(StringUtils.isNotBlank(addr.getTownship())){ + poi = poi.replace(addr.getTownship(),""); + } + if(StringUtils.isNotBlank(addr.getStreet())){ + poi = poi.replace(addr.getStreet(),""); + } + addr.setParsedPropertyName(poi); + } + //楼栋、单元、室号 + String houseNo = (jsonObj.containsKey("houseno") ? jsonObj.getString("houseno") : "").replaceAll("[\u4E00-\u9FA5]", ""); + String cellNo = (jsonObj.containsKey("cellno") ? jsonObj.getString("cellno") : "").replaceAll("[\u4E00-\u9FA5]", ""); + String floorNo = (jsonObj.containsKey("floorno") ? jsonObj.getString("floorno") : "").replaceAll("[\u4E00-\u9FA5]", ""); + String roomNo = (jsonObj.containsKey("roomno") ? jsonObj.getString("roomno") : "").replaceAll("[\u4E00-\u9FA5]", ""); + addr.setOrgBuildingNumber(houseNo); + addr.setBuildingNumber(houseNo); + addr.setOrgUnitNumber(cellNo); + addr.setUnitNumber(cellNo); + addr.setFloor(floorNo); + addr.setOrgRoomNumber(roomNo); + addr.setRoomNumber(roomNo); + }*/ matchedBuilding.setProvince(addr.getProvince()); matchedBuilding.setSearchAddress(addr); matchedBuilding.setSuggestName(getSuggestName(matchedBuilding)); @@ -215,7 +288,8 @@ public class AddressParserServiceImpl implements AddressParserService { completBuildingField(matchedBuilding, aliyunBo, params); } Address tempAddr = matchedBuilding.getSearchAddress(); - if(StringUtils.isBlank(tempAddr.getStreet()) && AddressParserGoal.Match.equals(addr.getAddressParserGoal())){ + if((StringUtils.isBlank(tempAddr.getStreet()) || StringUtils.isBlank(tempAddr.getStreetNumber())) && AddressParserGoal.Match.equals(addr.getAddressParserGoal())){ + AddressMatchMetaData meta = getMapMeta(addr); if(meta != null && (tempAddr.getKeywords()!=null && tempAddr.getKeywords().equals(meta.getMainName()))){ matchedBuilding.setAddress(meta.getAddress()); } @@ -255,6 +329,25 @@ public class AddressParserServiceImpl implements AddressParserService { } return null; } + + /** + * TODO 省市区乡镇采用原始拆分数据,若乡镇数据没有使用 + * @param address + * @return + */ + /*private JSONObject getDataFromCRF(String address){ + HttpRequest request = new HttpRequest(); + Map params = new HashMap<>(); + params.put("address",address); + try { + JSONObject resp = request.getJsonContent("http://127.0.0.1:8802/standard", params, "UTF-8"); + System.out.println(resp); + return resp; + }catch(Exception e){ + + } + return null; + }*/ /** * 信息不全时使用阿里接口补全数据 * @param matchedBuilding @@ -271,28 +364,39 @@ public class AddressParserServiceImpl implements AddressParserService { } } Address addr = matchedBuilding.getSearchAddress(); - if(StringUtils.isBlank(addr.getProvince())){ + if(StringUtils.isNotBlank(aliyunBo.getProv())){ + addr.setProvince(formatProvince(aliyunBo.getProv())); + } + if(StringUtils.isNotBlank(aliyunBo.getCity())){ + addr.setCity(RegionData.formatCity(aliyunBo.getCity())); + } + if(StringUtils.isNotBlank(aliyunBo.getDistrict())){ + addr.setDistrict(RegionData.formatRegion(aliyunBo.getDistrict())); + } + addr.setTownship(aliyunBo.getTown()); + + /*if(StringUtils.isBlank(addr.getProvince())){ addr.setProvince(formatProvince(aliyunBo.getProv())); } if(StringUtils.isBlank(addr.getCity())){ addr.setCity(RegionData.formatCity(aliyunBo.getCity())); } - - if(StringUtils.isBlank(addr.getDevelopmentZone())){ - addr.setDevelopmentZone(aliyunBo.getDevzone()); - } if(StringUtils.isBlank(addr.getDistrict())){ addr.setDistrict(RegionData.formatRegion(aliyunBo.getDistrict())); } if(StringUtils.isBlank(addr.getTownship())){ addr.setTownship(aliyunBo.getTown()); + }*/ + if(StringUtils.isBlank(addr.getDevelopmentZone())){ + addr.setDevelopmentZone(aliyunBo.getDevzone()); } + /*addr.setOrgBuildingNumber(aliyunBo.getHouseno()); addr.setBuildingNumber(aliyunBo.getHouseno()); addr.setOrgUnitNumber(aliyunBo.getCellno()); addr.setUnitNumber(aliyunBo.getCellno());*/ if(aliyunBo.getRoomno()!=null && !aliyunBo.getRoomno().matches(".*(层|楼)$")){ - addr.setRoomNumber(aliyunBo.getRoomno()); + addr.setOrgRoomNumber(aliyunBo.getRoomno()); addr.setRoomNumber(aliyunBo.getRoomno()); }else{ addr.setRoomNumber(null); @@ -1037,20 +1141,24 @@ public class AddressParserServiceImpl implements AddressParserService { if(StringUtils.isNotBlank(address.getTownship())){ formattedAddressBuilder.append(address.getTownship()); } + if(StringUtils.isNotBlank(building.getAddress())){ + Address orgAddr = new Address(); + orgAddr.setOrgAddress(building.getAddress()); + Address addr = AddressProcessors.process(orgAddr); + if(StringUtils.isBlank(address.getStreet())){ + address.setStreet(addr.getStreet()); + } + if(StringUtils.isBlank(address.getStreetNumber())){ + address.setStreetNumber(addr.getStreetNumber()); + } + } if(StringUtils.isNotBlank(address.getStreet())){ formattedAddressBuilder.append(address.getStreet()); } if(StringUtils.isNotBlank(address.getStreet()) && StringUtils.isNotBlank(address.getStreetNumber())){ formattedAddressBuilder.append(address.getStreetNumber()); } - if(StringUtils.isBlank(address.getStreet()) && StringUtils.isNotBlank(building.getAddress())){ - Address orgAddr = new Address(); - orgAddr.setOrgAddress(building.getAddress()); - Address addr = AddressProcessors.process(orgAddr); - building.getSearchAddress().setStreet(addr.getStreet()); - building.getSearchAddress().setStreetNumber(addr.getStreetNumber()); - formattedAddressBuilder.append((addr.getStreet()==null?"":addr.getStreet())+(addr.getStreetNumber()==null?"":addr.getStreetNumber())); - } + if(StringUtils.isNotBlank(building.getSuggestName())){ formattedAddressBuilder.append(building.getSuggestName()); } @@ -1077,8 +1185,8 @@ public class AddressParserServiceImpl implements AddressParserService { formattedAddressBuilder.append(address.getUnitNumber() + "单元"); } if(StringUtils.isNotBlank(address.getRoomNumber())){ - formattedAddressBuilder.append(address.getRoomNumber()); -// formattedAddressBuilder.append(address.getRoomNumber() + "室"); +// formattedAddressBuilder.append(address.getRoomNumber()); + formattedAddressBuilder.append(address.getRoomNumber() + "室"); } address.setFormattedAddress(formattedAddressBuilder.toString()); diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/Address.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/Address.java index 58022cf03..1c8f2d004 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/Address.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/Address.java @@ -77,6 +77,16 @@ public class Address { private String debugName; private AliyunBo aliyunBo; + //使用CRF切分的地址 + private String crfAddress; + + public String getCrfAddress() { + return crfAddress; + } + + public void setCrfAddress(String crfAddress) { + this.crfAddress = crfAddress; + } public AddressParserGoal getAddressParserGoal() { return addressParserGoal; diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/MatchParams.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/MatchParams.java index 498b12052..e53c9701f 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/MatchParams.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/MatchParams.java @@ -25,7 +25,15 @@ public class MatchParams { private boolean useMap; /**是否使用阿里云接口*/ private boolean useAliyunApi; + //是否使用CRF算法 + private boolean useCrf; + public boolean isUseCrf() { + return useCrf; + } + public void setUseCrf(boolean useCrf) { + this.useCrf = useCrf; + } public boolean isContainsLocation() { return containsLocation; } diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/StageAndPartitionProcessor.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/StageAndPartitionProcessor.java index 93c69b4c4..ab5c929b3 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/StageAndPartitionProcessor.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/StageAndPartitionProcessor.java @@ -14,6 +14,7 @@ public class StageAndPartitionProcessor extends AbstractAddressProcessor { @Override public RepeatStatus process(Address address) { String orgAddr = address.getEffectiveAddress(); + address.setCrfAddress(address.getAddress()); MatchResult matchResult = matchFirst(AddressParserPatterns.STAGE_PATTERN, address.getEffectiveAddress()); if (matchResult != null) { String formatStage = CNNMFilter.convertArabicToCnNumeric(matchResult.matchContent); diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/api/AliyunServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/api/AliyunServiceImpl.java index b6b22f5a1..bcbdba418 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/api/AliyunServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/api/AliyunServiceImpl.java @@ -1,5 +1,7 @@ package com.fangjia.service.api; +import com.aliyuncs.address_purification.model.v20191118.GetAddressDivisionCodeRequest; +import com.aliyuncs.address_purification.model.v20191118.GetAddressDivisionCodeResponse; import com.aliyuncs.address_purification.model.v20191118.StructureAddressRequest; import com.aliyuncs.address_purification.model.v20191118.StructureAddressResponse; import com.fangjia.core.util.JsonUtils; @@ -13,6 +15,9 @@ import net.sf.json.JSONObject; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; +import java.util.ArrayList; +import java.util.List; + public class AliyunServiceImpl implements AliyunService { protected final Logger LOGGER = LoggerFactory.getLogger(this.getClass()); @@ -35,20 +40,44 @@ public class AliyunServiceImpl implements AliyunService { if(StringUtils.isNotBlank(city)){ request.setDefaultCity(city); } + AliyunBo aliyunBo = new AliyunBo(); try{ StructureAddressResponse response = client.getAcsResponse(request); JSONObject responseJson = JSONObject.fromObject(response.getData()); String value = (String) responseJson.get("structure"); - AliyunBo aliyunBo = new AliyunBo(); for (String key:value.split("\t")){ String[] arry = key.split("="); wrapField(arry[0],arry[1], aliyunBo); } - return aliyunBo; }catch(Exception e){ - LOGGER.error("", e); - return null; + LOGGER.error("调用阿里标准化接口失败:", e); } + try{ + GetAddressDivisionCodeRequest divisionRequest = new GetAddressDivisionCodeRequest(); + divisionRequest.setServiceCode("addrp"); + divisionRequest.setAppKey(APPKEY); + divisionRequest.setText(address); + GetAddressDivisionCodeResponse divisionResponse = client.getAcsResponse(divisionRequest); + JSONObject responseJson = JSONObject.fromObject(divisionResponse.getData()); + String value = (String) ((JSONObject) responseJson.get("division_info")).get("division_name"); + String[] array = value.replace("\t", ";").split(";"); + if(array.length > 4){ + List tmp = new ArrayList<>(); + for(String str:array){ + if(str!=null && str.length()!=0){ + tmp.add(str); + } + } + array= tmp.toArray(new String[4]); + } + aliyunBo.setProv(array[0]); + aliyunBo.setCity(array[1]); + aliyunBo.setDistrict(array[2]); + aliyunBo.setTown(array[3]); + }catch(Exception e){ + LOGGER.error("调用阿里行政区划识别接口失败", e); + } + return aliyunBo; } private void wrapField(String field,String value, AliyunBo aliyunBo){ @@ -86,21 +115,38 @@ public class AliyunServiceImpl implements AliyunService { ACCESSKEYID, SECRET); IAcsClient client = new DefaultAcsClient(profile); - StructureAddressRequest request = new StructureAddressRequest(); +// StructureAddressRequest request = new StructureAddressRequest(); + GetAddressDivisionCodeRequest request = new GetAddressDivisionCodeRequest(); request.setServiceCode("addrp"); request.setAppKey(APPKEY); - request.setText("杭州市文一西路969号"); - request.setDefaultProvince("浙江省"); - request.setDefaultCity("杭州市"); - request.setDefaultDistrict("余杭区"); - StructureAddressResponse response = client.getAcsResponse(request); + request.setText("北郭乡东草亭村一号路2号"); +// request.setDefaultProvince("浙江省"); +// request.setDefaultCity("杭州市"); +// request.setDefaultDistrict("余杭区"); +// StructureAddressResponse response = client.getAcsResponse(request); + GetAddressDivisionCodeResponse response = client.getAcsResponse(request); JSONObject responseJson = JSONObject.fromObject(response.getData()); - String value = (String) responseJson.get("structure"); - JSONObject json = new JSONObject(); + System.out.println(responseJson); + String value = (String) ((JSONObject) responseJson.get("division_info")).get("division_name"); + String[] array = value.replace("\t", ";").split(";"); + if(array.length > 4){ + List tmp = new ArrayList(); + for(String str:array){ + if(str!=null && str.length()!=0){ + tmp.add(str); + } + } + array= tmp.toArray(new String[4]); + } + System.out.println(array[0]); + System.out.println(array[1]); + System.out.println(array[2]); + System.out.println(array[3]); + /*JSONObject json = new JSONObject(); for (String key:value.split("\t")){ String[] arry = key.split("="); json.put(arry[0],arry[1]); } - System.out.println(JsonUtils.convertJsonToJavaObj(json, AliyunBo.class)); + System.out.println(JsonUtils.convertJsonToJavaObj(json, AliyunBo.class));*/ } } diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/web/AddressServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/web/AddressServiceImpl.java index 8b8a1f1fc..67bc70049 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/web/AddressServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/web/AddressServiceImpl.java @@ -5132,7 +5132,7 @@ public class AddressServiceImpl implements AddressService { */ private String getAddressSimilarityLevel(JSONObject json1, JSONObject json2){ String level = null; - LinkedList list = new LinkedList<>(Arrays.asList("province", "city", "district", "township", "street" + LinkedList list = new LinkedList<>(Arrays.asList("province", "city", "district", "township", "street", "streetNumber" , "name", "buildingNumber", "unitNumber", "roomNumber")); for(String key:list){ if(continueJudgeValue(json1, json2, key) == 3){ From 3939ca595f46ebac45f7ea21e31b19116cf59125 Mon Sep 17 00:00:00 2001 From: wangwenyan Date: Sun, 9 May 2021 19:54:49 +0800 Subject: [PATCH 07/17] =?UTF-8?q?=E6=B7=BB=E5=8A=A0Dockers=E6=96=87?= =?UTF-8?q?=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../main/groovy/com/fangjia/script/wenyan/test/Test1.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/test/Test1.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/test/Test1.groovy index fb1b341be..cafe803de 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/test/Test1.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/test/Test1.groovy @@ -4,7 +4,7 @@ import com.fangjia.core.util.HttpRequest import com.fangjia.script.util.ExcelUtils import net.sf.json.JSONObject -ExcelUtils.multiThreadingModify("D:\\数据\\建行地址相似度评级测试.xlsx",{List item-> +ExcelUtils.multiThreadingModify("C:\\Users\\wenyan\\Downloads\\建行地址相似度评级测试.xlsx",{List item-> def city = item["A"] def address1 = item["B"] def address2 = item["C"] @@ -14,7 +14,7 @@ ExcelUtils.multiThreadingModify("D:\\ params["address_1"] = address1 params["address_2"] = address2 params["token"]= "F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB" - JSONObject json = request.getJsonContent("http://192.168.20.86:9180/address/similarity", params, "UTF-8") + JSONObject json = request.getJsonContent("http://open.fangjia.com/address/similarity", params, "UTF-8") JSONObject result = json.result item["D"] = result?.level item["E"] = result?.score From d2aab5988970f0b120eed24bdd5d6fa1ddcf3cab Mon Sep 17 00:00:00 2001 From: wangwenyan Date: Sun, 9 May 2021 19:56:15 +0800 Subject: [PATCH 08/17] fix bug --- .../main/java/com/fangjia/service/api/AliyunServiceImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/api/AliyunServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/api/AliyunServiceImpl.java index bcbdba418..cf02bf903 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/api/AliyunServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/api/AliyunServiceImpl.java @@ -62,7 +62,7 @@ public class AliyunServiceImpl implements AliyunService { String value = (String) ((JSONObject) responseJson.get("division_info")).get("division_name"); String[] array = value.replace("\t", ";").split(";"); if(array.length > 4){ - List tmp = new ArrayList<>(); + List tmp = new ArrayList(); for(String str:array){ if(str!=null && str.length()!=0){ tmp.add(str); From 0b0cd747314a9ed667b439b0cd89c1c369874fad Mon Sep 17 00:00:00 2001 From: xuhe Date: Sun, 9 May 2021 22:37:07 +0800 Subject: [PATCH 09/17] =?UTF-8?q?=E4=BC=98=E5=8C=96=E7=9B=B8=E4=BC=BC?= =?UTF-8?q?=E5=BA=A6=E8=AF=84=E7=BA=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../address/AddressParserServiceImpl.java | 6 ++--- .../service/api/AliyunServiceImpl.java | 22 ++++++++++++++++--- .../service/web/AddressServiceImpl.java | 17 +++++++++----- 3 files changed, 34 insertions(+), 11 deletions(-) diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java index 9eb8f4114..2caa0b7a3 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java @@ -193,7 +193,7 @@ public class AddressParserServiceImpl implements AddressParserService { } } if(StringUtils.isNotBlank(aliyunBo.getCity())){ - addr.setCity(aliyunBo.getCity()); + addr.setCity(RegionData.formatCity(aliyunBo.getCity())); } if(StringUtils.isNotBlank(addr.getKeywords()) && addr.getOrgAddress().indexOf(addr.getKeywords())>-1){ String str = AddressParserUtil.match(Pattern.compile(".*村(.*(组|队))?\\d+(号)?$"), addr.getOrgAddress().substring(addr.getOrgAddress().indexOf(addr.getKeywords()),addr.getOrgAddress().length())); @@ -1185,8 +1185,8 @@ public class AddressParserServiceImpl implements AddressParserService { formattedAddressBuilder.append(address.getUnitNumber() + "单元"); } if(StringUtils.isNotBlank(address.getRoomNumber())){ -// formattedAddressBuilder.append(address.getRoomNumber()); - formattedAddressBuilder.append(address.getRoomNumber() + "室"); + formattedAddressBuilder.append(address.getRoomNumber()); +// formattedAddressBuilder.append(address.getRoomNumber() + "室"); } address.setFormattedAddress(formattedAddressBuilder.toString()); diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/api/AliyunServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/api/AliyunServiceImpl.java index cf02bf903..91d309485 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/api/AliyunServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/api/AliyunServiceImpl.java @@ -110,7 +110,23 @@ public class AliyunServiceImpl implements AliyunService { } } public static void main(String[] args) throws ClientException { - DefaultProfile profile = DefaultProfile.getProfile( + String value = ""; + String[] array = value.replace("\t", ";").split(";"); + if(array.length > 4){ + List tmp = new ArrayList(); + for(String str:array){ + if(str!=null && str.length()!=0){ + tmp.add(str); + } + } + array= tmp.toArray(new String[4]); + } + System.out.println(array[0]); + System.out.println(array[1]); + System.out.println(array[2]); + System.out.println(array[3]); + + /*DefaultProfile profile = DefaultProfile.getProfile( REGIONID, ACCESSKEYID, SECRET); @@ -119,7 +135,7 @@ public class AliyunServiceImpl implements AliyunService { GetAddressDivisionCodeRequest request = new GetAddressDivisionCodeRequest(); request.setServiceCode("addrp"); request.setAppKey(APPKEY); - request.setText("北郭乡东草亭村一号路2号"); + request.setText("天青路260号"); // request.setDefaultProvince("浙江省"); // request.setDefaultCity("杭州市"); // request.setDefaultDistrict("余杭区"); @@ -141,7 +157,7 @@ public class AliyunServiceImpl implements AliyunService { System.out.println(array[0]); System.out.println(array[1]); System.out.println(array[2]); - System.out.println(array[3]); + System.out.println(array[3]);*/ /*JSONObject json = new JSONObject(); for (String key:value.split("\t")){ String[] arry = key.split("="); diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/web/AddressServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/web/AddressServiceImpl.java index c236acbb4..de2a772b2 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/web/AddressServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/web/AddressServiceImpl.java @@ -5069,7 +5069,7 @@ public class AddressServiceImpl implements AddressService { double similar = AddressParserUtil.getSimilarityRatio(formatAddress1_pinyin, formatAddress2_pinyin); String level = getAddressSimilarityLevel(json1, json2); resultMap.put("isSameAddress", sameAddress); - resultMap.put("score", similar); +// resultMap.put("score", similar); resultMap.put("level", level); resultMap.put("formattedAddress_1", getFromJson(json1,"formattedAddress")); resultMap.put("formattedAddress_2", getFromJson(json2,"formattedAddress")); @@ -5153,10 +5153,14 @@ public class AddressServiceImpl implements AddressService { */ private Integer continueJudgeValue(JSONObject json1, JSONObject json2, String key){ if(StringUtils.isNotBlank(getFromJson(json1,key)) && StringUtils.isNotBlank(getFromJson(json2,key))){ - if(getFromJson(json1,key).equals(getFromJson(json2,key))){ - return 1; + if("name".equals(key)){ + return verifySamePoi(getFromJson(json1,key),getFromJson(json2,key)) ? 1 : 3; }else{ - return 3; + if(getFromJson(json1,key).equals(getFromJson(json2,key))){ + return 1; + }else{ + return 3; + } } } return 2; @@ -5208,6 +5212,10 @@ public class AddressServiceImpl implements AddressService { if(!verifySameField(name1_stage, name2_stage, true) || !verifySameField(name1_partition, name2_partition, true)|| !verifySameField(name1_number, name2_number, true)){ return false; } + return isSameName(name1, name2); + } + + private boolean isSameName(String name1, String name2){ String name_1_pinyin = AddressParserUtil.toHanYuPinYin(name1); String name_2_pinyin = AddressParserUtil.toHanYuPinYin(name2); //拼音或文本数据包含并且 @@ -5220,7 +5228,6 @@ public class AddressServiceImpl implements AddressService { } return false; } - /** * 设置区域消费指数 * @param checkTr From 4afd40a650695a8e49ad4d964fc6c46a721a4397 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=96=87=E8=89=B3?= Date: Mon, 10 May 2021 18:40:55 +0800 Subject: [PATCH 10/17] =?UTF-8?q?=E5=8D=87=E7=BA=A7Es=E9=A9=B1=E5=8A=A8?= =?UTF-8?q?=E7=89=88=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/script/wenyan/test/Test1.groovy | 36 ++++++++++++++----- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/test/Test1.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/test/Test1.groovy index cafe803de..401d647e4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/test/Test1.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/test/Test1.groovy @@ -1,26 +1,46 @@ package com.fangjia.script.wenyan.test +import com.aliyuncs.DefaultAcsClient +import com.aliyuncs.IAcsClient +import com.aliyuncs.address_purification.model.v20191118.GetAddressSimilarityRequest +import com.aliyuncs.address_purification.model.v20191118.GetAddressSimilarityResponse +import com.aliyuncs.profile.DefaultProfile import com.fangjia.core.util.HttpRequest import com.fangjia.script.util.ExcelUtils import net.sf.json.JSONObject -ExcelUtils.multiThreadingModify("C:\\Users\\wenyan\\Downloads\\建行地址相似度评级测试.xlsx",{List item-> +ExcelUtils.modify("D:\\nextcloud\\数据\\地址类服务测试\\建行地址相似度评级测试.xlsx",{List item-> def city = item["A"] def address1 = item["B"] def address2 = item["C"] - HttpRequest request = new HttpRequest() + HttpRequest http = new HttpRequest() def params = new HashMap() params["city"] = city params["address_1"] = address1 params["address_2"] = address2 params["token"]= "F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB" - JSONObject json = request.getJsonContent("http://open.fangjia.com/address/similarity", params, "UTF-8") + JSONObject json = http.getJsonContent("http://open.fangjia.com/address/similarity", params, "UTF-8") JSONObject result = json.result item["D"] = result?.level - item["E"] = result?.score - item["F"] = result?.formattedAddress_1 - item["G"] = result?.formattedAddress_2 - item["H"] = result?.isSameAddress - println(item) + item["E"] = result?.formattedAddress_1 + item["F"] = result?.formattedAddress_2 + item["G"] = result?.isSameAddress + DefaultProfile profile = DefaultProfile.getProfile( + "cn-hangzhou", + "LTAIdaj4eSO79gzz", + "xw6EsvqydZ9HC6zAvIB3ruQqVHynxf"); + IAcsClient client = new DefaultAcsClient(profile); + GetAddressSimilarityRequest request = new GetAddressSimilarityRequest(); + request.setServiceCode("addrp") + request.setAppKey("Xcgtyhupey0h") + request.setText("${address1};${address2}") + GetAddressSimilarityResponse response = client.getAcsResponse(request) + String data = response.getData() + json = JSONObject.fromObject(data) + item["H"] = json["sim_level"]["level"] + item["I"] = json["sim_level"]["address_a"] + item["J"] = json["sim_level"]["address_b"] + item["K"] = json["sim_level"]["is_same"] + println(json) }) From 07c166e84312ba51c8331372079fcee535998f0a Mon Sep 17 00:00:00 2001 From: xuhe Date: Mon, 10 May 2021 23:13:21 +0800 Subject: [PATCH 11/17] =?UTF-8?q?=E5=9C=B0=E5=9D=80=E7=9B=B8=E4=BC=BC?= =?UTF-8?q?=E5=BA=A6=E8=AF=84=E7=BA=A7=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/service/address/AddressParserServiceImpl.java | 8 ++++---- .../java/com/fangjia/service/web/AddressServiceImpl.java | 8 ++++++++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java index 2caa0b7a3..1512e2e7e 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java @@ -395,13 +395,13 @@ public class AddressParserServiceImpl implements AddressParserService { addr.setBuildingNumber(aliyunBo.getHouseno()); addr.setOrgUnitNumber(aliyunBo.getCellno()); addr.setUnitNumber(aliyunBo.getCellno());*/ - if(aliyunBo.getRoomno()!=null && !aliyunBo.getRoomno().matches(".*(层|楼)$")){ + /*if(aliyunBo.getRoomno()!=null && !aliyunBo.getRoomno().matches(".*(层|楼)$")){ addr.setOrgRoomNumber(aliyunBo.getRoomno()); addr.setRoomNumber(aliyunBo.getRoomno()); }else{ addr.setRoomNumber(null); addr.setRoomNumber(null); - } + }*/ /*String aliRoom = aliyunBo.getRoomno() == null ? null : aliyunBo.getRoomno().replaceAll("[\u4E00-\u9FA5]", ""); String orgRoom = addr.getOrgRoomNumber() == null ? null : addr.getOrgRoomNumber().replaceAll("[\u4E00-\u9FA5]", ""); @@ -1185,8 +1185,8 @@ public class AddressParserServiceImpl implements AddressParserService { formattedAddressBuilder.append(address.getUnitNumber() + "单元"); } if(StringUtils.isNotBlank(address.getRoomNumber())){ - formattedAddressBuilder.append(address.getRoomNumber()); -// formattedAddressBuilder.append(address.getRoomNumber() + "室"); +// formattedAddressBuilder.append(address.getRoomNumber()); + formattedAddressBuilder.append(address.getRoomNumber() + "室"); } address.setFormattedAddress(formattedAddressBuilder.toString()); diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/web/AddressServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/web/AddressServiceImpl.java index de2a772b2..21ab2139d 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/web/AddressServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/web/AddressServiceImpl.java @@ -5152,6 +5152,14 @@ public class AddressServiceImpl implements AddressService { * @return 1 继续判断并且level有值 2继续判断level没值 3 停止 */ private Integer continueJudgeValue(JSONObject json1, JSONObject json2, String key){ + if("buildingNumber".equals(key) || "unitNumber".equals(key) || "roomNumber".equals(key)){ + if(StringUtils.isNotBlank(getFromJson(json1,key)) && StringUtils.isNotBlank(getFromJson(json2,key)) + && getFromJson(json1,key).equals(getFromJson(json2,key))){ + return 1; + }else{ + return 3; + } + } if(StringUtils.isNotBlank(getFromJson(json1,key)) && StringUtils.isNotBlank(getFromJson(json2,key))){ if("name".equals(key)){ return verifySamePoi(getFromJson(json1,key),getFromJson(json2,key)) ? 1 : 3; From 947853db4d56200b5965317178530151691499f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=96=87=E8=89=B3?= Date: Wed, 12 May 2021 13:17:46 +0800 Subject: [PATCH 12/17] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E8=A1=97=E9=81=93?= =?UTF-8?q?=E5=AD=97=E6=AE=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/dal/seaweed/model/District.java | 22 +++--- .../com/fangjia/script/wenyan/Test4.groovy | 70 ++++++++++++------- .../address/AddressParserServiceImpl.java | 2 +- .../service/address/mode/Building.java | 8 +-- 4 files changed, 56 insertions(+), 46 deletions(-) diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/seaweed/model/District.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/seaweed/model/District.java index 401132cb0..afba53e15 100644 --- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/seaweed/model/District.java +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/seaweed/model/District.java @@ -122,10 +122,8 @@ public class District implements SeaweedEntity { private String houseStructure; //房屋结构 private String source; private String propertyLife; -// -// //乡镇街道 -// private String township; -// + //乡镇街道 + private String township; private String pinyin; @@ -1873,14 +1871,14 @@ public class District implements SeaweedEntity { this.mappingAlias = mappingAlias; } -// @Column(name = "township") -// public String getTownship() { -// return township; -// } -// -// public void setTownship(String township) { -// this.township = township; -// } + @Column(name = "township") + public String getTownship() { + return township; + } + + public void setTownship(String township) { + this.township = township; + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/Test4.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/Test4.groovy index bef30856c..95d41237b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/Test4.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/Test4.groovy @@ -6,34 +6,50 @@ import com.fangjia.script.util.DataUtil import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection +import net.sf.json.JSONObject +import org.apache.commons.lang.StringUtils +DBCollection SEAWEED_TOWNSHIP = MongoUtils.getCollectionByName("fangjia_base", "seaweed_township", "192.168.0.235", 27018) +def count = 0 +List> data = Collections.synchronizedList([]) +data << ["ID","城市","区域","小区名","经度","纬度","街道"] - -//ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\数据.xlsx",{List item-> -// if(DbUtils.seaweed.findOne(DbUtils.buildSeaweedQuery(item["A"], item["B"], item["C"]))){ -// item["D"] = "存在" -// }else{ -// item["D"] = "不存在" -// } -//}) - - -DBCollection DISTRICT_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat","mongo.db.fangjia.com") -ProvinceData.getAllCity("福建").each {def city-> - int count = 0 - def s = new TreeMap() - DISTRICT_STAT.find(new BasicDBObject("weekend", Date.parse("yyyy-MM-dd", "2020-04-19")).append("city", city)).each {BasicDBObject stat-> - count ++ - int credit = (stat.credit * 100)/10 - s[credit] = s.get(credit, 0) + 1 - } - - def data = [] - data << city - data << count - for(int i=0; i <=10; i++){ - data << s.get(i, 0) - } - println(data.join(",")) +CommonUtils.findAllCity().each {def city-> + DbUtils.multiThreadingExecuteQueryOnSeaweed(DbUtils.buildSeaweedQuery(city),{BasicDBObject item-> + if(StringUtils.isBlank(item.township) && item.lng){ + count++ + def lng = item.lng/10000000000d + def lat = item.lat/10000000000d + def location = "${lng},${lat}" + JSONObject resp = GaoDeApiUtils.geocode(location) + def row = [] + row << item._id.toString() + row << item.city + row << item.region + row << item.name + row << lng + row << lat + BasicDBObject dbo = new BasicDBObject() + dbo.id = item._id.toString() + dbo.city = item.city + dbo.region = item.region + dbo.name = item.name + dbo.lng = lng + dbo.lat = lat + if(resp.regeocode && resp.regeocode.addressComponent && resp.regeocode.addressComponent.township){ + row << resp.regeocode.addressComponent.township + dbo.township = resp.regeocode.addressComponent.township + } + println(row) + data << row + SEAWEED_TOWNSHIP.insert(dbo) + } + }) } +ExcelUtils.write("D:\\街道补充数据.xlsx", data, "补充数据") + + + + diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java index 1512e2e7e..46545fc04 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java @@ -46,7 +46,7 @@ public class AddressParserServiceImpl implements AddressParserService { private LoudongService loudongService; private SeaweedDao seaweedDao; private AliyunService aliyunService; - private final String[] MATCH_PROPERTY_FIELDS = new String[]{"id", "city", "region", "name", "address", "category", "block", "alias", "addrAlias", "estateType2", "lat2", "lng2", "streetCommunity"}; + private final String[] MATCH_PROPERTY_FIELDS = new String[]{"id", "city", "region", "name", "address", "category", "block", "alias", "addrAlias", "estateType2", "lat2", "lng2", "township"}; private static final Logger logger = Logger.getLogger(AddressParserServiceImpl.class); diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/Building.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/Building.java index a217fa8fd..251470ffc 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/Building.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/Building.java @@ -64,9 +64,7 @@ public class Building { this.alias = d.getAlias(); this.addrAlias = d.getAddrAlias(); this.propertyType = d.getEstateType2(); - if(StringUtils.isNotBlank(d.getStreetCommunity())){ - this.township = d.getStreetCommunity(); - } + this.township = d.getTownship(); if(d.getLat() != null && d.getLng() != null){ this.lat = d.getLat()/10000000000d; this.lng = d.getLng()/10000000000d; @@ -90,9 +88,7 @@ public class Building { this.lat = d.getLat2()/10000000000d; this.lng = d.getLng2()/10000000000d; } - if(StringUtils.isNotBlank(d.getStreetCommunity())){ - this.township = d.getStreetCommunity(); - } + this.township = d.getTownship(); } public Building() { From 1f0a3177199d152e19f24d546a4529d2f1b78b0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Wed, 12 May 2021 16:46:56 +0800 Subject: [PATCH 13/17] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E6=8A=93=E5=8F=96?= =?UTF-8?q?=E8=BF=90=E7=BB=B4=E8=AD=A6=E6=8A=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../groovy/com/fangjia/script/Watcher.groovy | 1 + .../temp/shenjian/SeawaterMonitor.groovy | 148 ++++++++++++++++++ 2 files changed, 149 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/SeawaterMonitor.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/Watcher.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/Watcher.groovy index 85088ddd8..004d6bd4a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/Watcher.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/Watcher.groovy @@ -32,6 +32,7 @@ public enum Watcher { BaYi('15300424133','bayi@fangjia.com'), WeiLei('18101751850','weilei@fangjia.com'), JiangYaHui('13684913780','jiangyahui@fangjia.com'), + ShenJian('13764369683', 'shenjian@fangjia.com') private final String mobile; private final String email; diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/SeawaterMonitor.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/SeawaterMonitor.groovy new file mode 100644 index 000000000..a2d5c158d --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/SeawaterMonitor.groovy @@ -0,0 +1,148 @@ +package com.fangjia.script.temp.shenjian + +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.yinjihaun.util.DingDingMessageUtil +import org.apache.commons.lang.StringUtils +import org.apache.commons.lang.time.DateUtils + +/** + * 抓取总量监控 + * + * @author shenjian + * @since 2021/5/12 + */ +//@BatchOwner(Watcher.ShenJian) +//class SeawaterMonitor extends BaseBatch { +class SeawaterMonitor { + + static final Long expectNumber = 400000; + // 运维组 运维警报 + static final access_token = "2307d2fec8314b3e8b4b83dd1d42da2e2051638e3e668f21e77ca01258bf682c" + // 重点城市 + static final city = ["北京','唐山','天津','秦皇岛','石家庄','包头','太原','丹东','呼和浩特','锦州'," + + "'沈阳','吉林','大连','牡丹江','长春','无锡','哈尔滨','扬州','上海','徐州'," + + "'南京','温州','杭州','金华','宁波','蚌埠','合肥','安庆','福州','泉州'," + + "'厦门','九江','南昌','赣州','济南','烟台','青岛','济宁','郑州','洛阳'," + + "'武汉','平顶山','长沙','宜昌','广州','襄阳','深圳','岳阳','南宁','常德'," + + "'海口','惠州','重庆','湛江','成都','韶关','贵阳','桂林','昆明','北海'," + + "'西安','三亚','兰州','泸州','西宁','南充','银川','遵义','乌鲁木齐','大理'," + + "'东莞','佛山','苏州','珠海','中山','台州','绍兴','南通','常州','嘉兴'," + + "'淄博','株洲','镇江','肇庆','漳州','盐城','芜湖','泰州','汕头','清远'," + + "'马鞍山','柳州','连云港','廊坊','淮安','湖州','沧州','保定','张家口','日照'," + + "'拉萨','大同','文昌','陵水','保亭"]; + // 重点网站 + static final site = ["lianjia.com", "ke.com", "58.com", "zhuge.com", "fang.com"] + + static void main(String[] args) { + def lastWeekDate = org.apache.commons.lang3.time.DateFormatUtils.format(DateUtils.addDays(new Date(), -7), "yyyy-MM-dd") + def yesterday = org.apache.commons.lang3.time.DateFormatUtils.format(DateUtils.addDays(new Date(), -1), "yyyy-MM-dd") + def startDate = yesterday + "T00:00:00" + def endDate = yesterday + "T23:59:59" + + def result = [] + def msg = monitorYesterdayCrawlNumber(startDate, endDate) + if (StringUtils.isNotBlank(msg)) { + result.add(msg) + } + + msg = monitorCityCrawlNumber(startDate, endDate) + if (StringUtils.isNotBlank(msg)) { + result.add(msg) + } + + msg = monitorSiteCrawlNumber(lastWeekDate, endDate) + if (StringUtils.isNotBlank(msg)) { + result.add(msg) + } + if (result.size() > 0) { + msg = "【日期】 " + yesterday + "\n" + result.join("\n") + println(msg) + DingDingMessageUtil.sendTextMessage(access_token, msg) + } + } + + /** + * 监控前一日抓取量 + * @return + */ + static def monitorYesterdayCrawlNumber(def startDate, def endDate) { + def total + String sql = "select count(*) as total from latest_seawater where m_date between '${startDate}' and '${endDate}' " + ElasticUtils.executeSqlQuery(sql,{data-> + total = data.total + }) + if (total < expectNumber) { + return "【总量抓取异常】 抓取量为" + total + ", 低于预期值" + expectNumber + } + return "" + } + + /** + * 统计重点城市近半月抓取量 + * @return + */ + static boolean countCityCrawlNumber(def startDate, def endDate) { + def result = [] + result << ['城市', '抓取量'] + // 统计近半个月各重点城市抓取量 + def cityFormatted = city.join("','") + String sql = "select * from (select count(*) as total, city from latest_seawater where m_date between '${startDate}' and '${endDate}' and city in('${cityFormatted}') group by city) order by total desc " + ElasticUtils.executeSqlQuery(sql,{data-> + result << [data.city, data.total] + }) + ExcelUtils.write("F:\\数据\\重点城市抓取\\近半月重点城市抓取量.xlsx", result, "抓取量") + } + + /** + * 统计前一日重点城市抓取量,昨天为零则警报 + * @return + */ + static def monitorCityCrawlNumber(def startDate, def endDate) { + def msg = "【重点城市抓取异常】" + def unNormal = [] + def cityFormatted = city.join("','") + String sql = "select * from " + + " (select count(*) as total, city from latest_seawater where m_date between '${startDate}' and '${endDate}' and city in('${cityFormatted}') group by city) where total=0 " + //" order by total " + ElasticUtils.executeSqlQuery(sql,{data-> + unNormal.add(data.city) + }) + msg += unNormal.join("、") + " 抓取量为0" + if (unNormal.size() > 0) { + return msg + } + return "" + } + + /** + * 统计重点网站抓取情况,近一周为0则警报 + */ + static def monitorSiteCrawlNumber(def lastWeekDate, def endDate) { + def msg = "【重点网站抓取异常】" + def set = new HashSet() + def siteFormatted = site.join("','") + String sql = "select * from " + + " (select count(*) as total, root from latest_seawater where m_date between '${lastWeekDate}' and '${endDate}' and root in('${siteFormatted}') group by root) where total > 0 " + //" order by total " + ElasticUtils.executeSqlQuery(sql,{data-> + set.add(data.root) + }) + def unNormal = [] + for (def info : site) { + if (!set.contains(info)) { + unNormal.add(info) + } + } + msg += unNormal.join("、") + " 近一周抓取量为0" + if (unNormal.size() > 0) { + return msg + } + return "" + } + +// @Override +// void process(Object args) { +// +// } +} From c0484532090cdce60d8bc20a7f93ecfaf35e9ab8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Wed, 12 May 2021 18:43:20 +0800 Subject: [PATCH 14/17] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E6=8A=93=E5=8F=96?= =?UTF-8?q?=E8=BF=90=E7=BB=B4=E8=AD=A6=E6=8A=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/SeawaterMonitor.groovy | 66 ++++++++++--------- 1 file changed, 34 insertions(+), 32 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/SeawaterMonitor.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/SeawaterMonitor.groovy index a2d5c158d..60e2419de 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/SeawaterMonitor.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/SeawaterMonitor.groovy @@ -1,5 +1,8 @@ package com.fangjia.script.temp.shenjian +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.yinjihaun.util.DingDingMessageUtil @@ -12,9 +15,8 @@ import org.apache.commons.lang.time.DateUtils * @author shenjian * @since 2021/5/12 */ -//@BatchOwner(Watcher.ShenJian) -//class SeawaterMonitor extends BaseBatch { -class SeawaterMonitor { +@BatchOwner(Watcher.ShenJian) +class SeawaterMonitor extends BaseBatch { static final Long expectNumber = 400000; // 运维组 运维警报 @@ -35,31 +37,7 @@ class SeawaterMonitor { static final site = ["lianjia.com", "ke.com", "58.com", "zhuge.com", "fang.com"] static void main(String[] args) { - def lastWeekDate = org.apache.commons.lang3.time.DateFormatUtils.format(DateUtils.addDays(new Date(), -7), "yyyy-MM-dd") - def yesterday = org.apache.commons.lang3.time.DateFormatUtils.format(DateUtils.addDays(new Date(), -1), "yyyy-MM-dd") - def startDate = yesterday + "T00:00:00" - def endDate = yesterday + "T23:59:59" - - def result = [] - def msg = monitorYesterdayCrawlNumber(startDate, endDate) - if (StringUtils.isNotBlank(msg)) { - result.add(msg) - } - - msg = monitorCityCrawlNumber(startDate, endDate) - if (StringUtils.isNotBlank(msg)) { - result.add(msg) - } - - msg = monitorSiteCrawlNumber(lastWeekDate, endDate) - if (StringUtils.isNotBlank(msg)) { - result.add(msg) - } - if (result.size() > 0) { - msg = "【日期】 " + yesterday + "\n" + result.join("\n") - println(msg) - DingDingMessageUtil.sendTextMessage(access_token, msg) - } + runBatch(new SeawaterMonitor(), args) } /** @@ -141,8 +119,32 @@ class SeawaterMonitor { return "" } -// @Override -// void process(Object args) { -// -// } + @Override + void process(Object args) { + def lastWeekDate = org.apache.commons.lang3.time.DateFormatUtils.format(DateUtils.addDays(new Date(), -7), "yyyy-MM-dd") + def yesterday = org.apache.commons.lang3.time.DateFormatUtils.format(DateUtils.addDays(new Date(), -1), "yyyy-MM-dd") + def startDate = yesterday + "T00:00:00" + def endDate = yesterday + "T23:59:59" + + def result = [] + def msg = monitorYesterdayCrawlNumber(startDate, endDate) + if (StringUtils.isNotBlank(msg)) { + result.add(msg) + } + + msg = monitorCityCrawlNumber(startDate, endDate) + if (StringUtils.isNotBlank(msg)) { + result.add(msg) + } + + msg = monitorSiteCrawlNumber(lastWeekDate, endDate) + if (StringUtils.isNotBlank(msg)) { + result.add(msg) + } + if (result.size() > 0) { + msg = "【日期】 " + yesterday + "\n" + result.join("\n") + println(msg) + DingDingMessageUtil.sendTextMessage(access_token, msg) + } + } } From b8d282112291ed0de9bbb0cb1325b5c53753c5ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Fri, 14 May 2021 09:46:19 +0800 Subject: [PATCH 15/17] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E6=97=A5=E6=9C=9F?= =?UTF-8?q?=E5=8F=8A=E9=87=8D=E7=82=B9=E5=9F=8E=E5=B8=82=E6=9F=A5=E8=AF=A2?= =?UTF-8?q?bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/SeawaterMonitor.groovy | 50 +++++++++++-------- 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/SeawaterMonitor.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/SeawaterMonitor.groovy index 60e2419de..8627efc19 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/SeawaterMonitor.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/SeawaterMonitor.groovy @@ -1,5 +1,6 @@ package com.fangjia.script.temp.shenjian +import com.fangjia.core.util.DateUtil import com.fangjia.script.BaseBatch import com.fangjia.script.BatchOwner import com.fangjia.script.Watcher @@ -7,7 +8,6 @@ import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.yinjihaun.util.DingDingMessageUtil import org.apache.commons.lang.StringUtils -import org.apache.commons.lang.time.DateUtils /** * 抓取总量监控 @@ -22,17 +22,17 @@ class SeawaterMonitor extends BaseBatch { // 运维组 运维警报 static final access_token = "2307d2fec8314b3e8b4b83dd1d42da2e2051638e3e668f21e77ca01258bf682c" // 重点城市 - static final city = ["北京','唐山','天津','秦皇岛','石家庄','包头','太原','丹东','呼和浩特','锦州'," + - "'沈阳','吉林','大连','牡丹江','长春','无锡','哈尔滨','扬州','上海','徐州'," + - "'南京','温州','杭州','金华','宁波','蚌埠','合肥','安庆','福州','泉州'," + - "'厦门','九江','南昌','赣州','济南','烟台','青岛','济宁','郑州','洛阳'," + - "'武汉','平顶山','长沙','宜昌','广州','襄阳','深圳','岳阳','南宁','常德'," + - "'海口','惠州','重庆','湛江','成都','韶关','贵阳','桂林','昆明','北海'," + - "'西安','三亚','兰州','泸州','西宁','南充','银川','遵义','乌鲁木齐','大理'," + - "'东莞','佛山','苏州','珠海','中山','台州','绍兴','南通','常州','嘉兴'," + - "'淄博','株洲','镇江','肇庆','漳州','盐城','芜湖','泰州','汕头','清远'," + - "'马鞍山','柳州','连云港','廊坊','淮安','湖州','沧州','保定','张家口','日照'," + - "'拉萨','大同','文昌','陵水','保亭"]; + static final city = ["北京","唐山","天津","秦皇岛","石家庄","包头","太原","丹东","呼和浩特","锦州", + "沈阳","吉林","大连","牡丹江","长春","无锡","哈尔滨","扬州","上海","徐州", + "南京","温州","杭州","金华","宁波","蚌埠","合肥","安庆","福州","泉州", + "厦门","九江","南昌","赣州","济南","烟台","青岛","济宁","郑州","洛阳", + "武汉","平顶山","长沙","宜昌","广州","襄阳","深圳","岳阳","南宁","常德", + "海口","惠州","重庆","湛江","成都","韶关","贵阳","桂林","昆明","北海", + "西安","三亚","兰州","泸州","西宁","南充","银川","遵义","乌鲁木齐","大理", + "东莞","佛山","苏州","珠海","中山","台州","绍兴","南通","常州","嘉兴", + "淄博","株洲","镇江","肇庆","漳州","盐城","芜湖","泰州","汕头","清远", + "马鞍山","柳州","连云港","廊坊","淮安","湖州","沧州","保定","张家口","日照", + "拉萨","大同","文昌","陵水","保亭"]; // 重点网站 static final site = ["lianjia.com", "ke.com", "58.com", "zhuge.com", "fang.com"] @@ -46,7 +46,7 @@ class SeawaterMonitor extends BaseBatch { */ static def monitorYesterdayCrawlNumber(def startDate, def endDate) { def total - String sql = "select count(*) as total from latest_seawater where m_date between '${startDate}' and '${endDate}' " + String sql = "select count(*) as total from seawater where m_date >= '${startDate}' and m_date < '${endDate}' " ElasticUtils.executeSqlQuery(sql,{data-> total = data.total }) @@ -65,7 +65,7 @@ class SeawaterMonitor extends BaseBatch { result << ['城市', '抓取量'] // 统计近半个月各重点城市抓取量 def cityFormatted = city.join("','") - String sql = "select * from (select count(*) as total, city from latest_seawater where m_date between '${startDate}' and '${endDate}' and city in('${cityFormatted}') group by city) order by total desc " + String sql = "select * from (select count(*) as total, city from seawater where m_date >= '${startDate}' and m_date < '${endDate}' and city in('${cityFormatted}') group by city) order by total desc " ElasticUtils.executeSqlQuery(sql,{data-> result << [data.city, data.total] }) @@ -78,14 +78,20 @@ class SeawaterMonitor extends BaseBatch { */ static def monitorCityCrawlNumber(def startDate, def endDate) { def msg = "【重点城市抓取异常】" - def unNormal = [] + def set = new HashSet() def cityFormatted = city.join("','") String sql = "select * from " + - " (select count(*) as total, city from latest_seawater where m_date between '${startDate}' and '${endDate}' and city in('${cityFormatted}') group by city) where total=0 " + " (select count(*) as total, city from seawater where m_date >= '${startDate}' and m_date < '${endDate}' and city in('${cityFormatted}') group by city) where total>0 " //" order by total " ElasticUtils.executeSqlQuery(sql,{data-> - unNormal.add(data.city) + set.add(data.city) }) + def unNormal = [] + for (def info : city) { + if (!set.contains(info)) { + unNormal.add(info) + } + } msg += unNormal.join("、") + " 抓取量为0" if (unNormal.size() > 0) { return msg @@ -101,7 +107,7 @@ class SeawaterMonitor extends BaseBatch { def set = new HashSet() def siteFormatted = site.join("','") String sql = "select * from " + - " (select count(*) as total, root from latest_seawater where m_date between '${lastWeekDate}' and '${endDate}' and root in('${siteFormatted}') group by root) where total > 0 " + " (select count(*) as total, root from seawater where m_date >= '${lastWeekDate}' and m_date < '${endDate}' and root in('${siteFormatted}') group by root) where total > 0 " //" order by total " ElasticUtils.executeSqlQuery(sql,{data-> set.add(data.root) @@ -121,10 +127,10 @@ class SeawaterMonitor extends BaseBatch { @Override void process(Object args) { - def lastWeekDate = org.apache.commons.lang3.time.DateFormatUtils.format(DateUtils.addDays(new Date(), -7), "yyyy-MM-dd") - def yesterday = org.apache.commons.lang3.time.DateFormatUtils.format(DateUtils.addDays(new Date(), -1), "yyyy-MM-dd") - def startDate = yesterday + "T00:00:00" - def endDate = yesterday + "T23:59:59" + def lastWeekDate = DateUtil.add(DateUtil.truncateDay(new Date()), -7).getTime() + def yesterday = DateUtil.formatToYYYYMMDD(DateUtil.add(DateUtil.truncateDay(new Date()), -1)) + def startDate = DateUtil.add(DateUtil.truncateDay(new Date()), -1).getTime(); + def endDate = DateUtil.truncateDay(new Date()).getTime() def result = [] def msg = monitorYesterdayCrawlNumber(startDate, endDate) From c363a8d0caad1a16e31baebad27bbb78c250b429 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=96=87=E8=89=B3?= Date: Tue, 18 May 2021 12:51:48 +0800 Subject: [PATCH 16/17] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E8=A1=97=E9=81=93?= =?UTF-8?q?=E5=AD=97=E6=AE=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../open/controllers/PropertyController.java | 2 +- .../com/fangjia/script/wenyan/Test4.groovy | 103 +++++++++++------- .../fangjia/script/wenyan/test/Test1.groovy | 5 + 3 files changed, 71 insertions(+), 39 deletions(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java index 93172390a..37b785212 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java @@ -671,7 +671,7 @@ public class PropertyController { && !"1007486".equals(userId) && !"1007533".equals(userId) && !"1007630".equals(userId) && !"1007705".equals(userId) && !"1007739".equals(userId) && !"1007736".equals(userId) && !"1007741".equals(userId) && !"1007742".equals(userId) && !"1007743".equals(userId) - && !"1007747".equals(userId) + && !"1007747".equals(userId) && !"1007757".equals(userId) && model.getDetailLevel() != null && model.getDetailLevel() != 0) {// 临时权限控制 mav.setModel(BasicModel.buildUnAuthorizedModel()); return mav; diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/Test4.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/Test4.groovy index 95d41237b..235253d60 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/Test4.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/Test4.groovy @@ -4,6 +4,7 @@ import com.fangjia.core.ProvinceData import com.fangjia.script.util.CommonUtils import com.fangjia.script.util.DataUtil import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils @@ -11,45 +12,71 @@ import com.mongodb.BasicDBObject import com.mongodb.DBCollection import net.sf.json.JSONObject import org.apache.commons.lang.StringUtils -DBCollection SEAWEED_TOWNSHIP = MongoUtils.getCollectionByName("fangjia_base", "seaweed_township", "192.168.0.235", 27018) -def count = 0 -List> data = Collections.synchronizedList([]) -data << ["ID","城市","区域","小区名","经度","纬度","街道"] - -CommonUtils.findAllCity().each {def city-> - DbUtils.multiThreadingExecuteQueryOnSeaweed(DbUtils.buildSeaweedQuery(city),{BasicDBObject item-> - if(StringUtils.isBlank(item.township) && item.lng){ - count++ - def lng = item.lng/10000000000d - def lat = item.lat/10000000000d - def location = "${lng},${lat}" - JSONObject resp = GaoDeApiUtils.geocode(location) - def row = [] - row << item._id.toString() - row << item.city - row << item.region - row << item.name - row << lng - row << lat - BasicDBObject dbo = new BasicDBObject() - dbo.id = item._id.toString() - dbo.city = item.city - dbo.region = item.region - dbo.name = item.name - dbo.lng = lng - dbo.lat = lat - if(resp.regeocode && resp.regeocode.addressComponent && resp.regeocode.addressComponent.township){ - row << resp.regeocode.addressComponent.township - dbo.township = resp.regeocode.addressComponent.township - } - println(row) - data << row - SEAWEED_TOWNSHIP.insert(dbo) - } - }) -} -ExcelUtils.write("D:\\街道补充数据.xlsx", data, "补充数据") +//DBCollection SEAWEED_TOWNSHIP = MongoUtils.getCollectionByName("fangjia_base", "seaweed_township", "192.168.0.235", 27018) +//def count = 0 +//List> data = Collections.synchronizedList([]) +//data << ["ID","城市","区域","小区名","经度","纬度","街道"] +// +//CommonUtils.findAllCity().each {def city-> +// DbUtils.multiThreadingExecuteQueryOnSeaweed(DbUtils.buildSeaweedQuery(city),{BasicDBObject item-> +// if(StringUtils.isBlank(item.township) && item.lng){ +// count++ +// def lng = item.lng/10000000000d +// def lat = item.lat/10000000000d +// def location = "${lng},${lat}" +// JSONObject resp = GaoDeApiUtils.geocode(location) +// def row = [] +// row << item._id.toString() +// row << item.city +// row << item.region +// row << item.name +// row << lng +// row << lat +// BasicDBObject dbo = new BasicDBObject() +// dbo.id = item._id.toString() +// dbo.city = item.city +// dbo.region = item.region +// dbo.name = item.name +// dbo.lng = lng +// dbo.lat = lat +// if(resp.regeocode && resp.regeocode.addressComponent && resp.regeocode.addressComponent.township){ +// row << resp.regeocode.addressComponent.township +// dbo.township = resp.regeocode.addressComponent.township +// } +// println(row) +// data << row +// SEAWEED_TOWNSHIP.insert(dbo) +// } +// }) +//} +//ExcelUtils.write("D:\\街道补充数据.xlsx", data, "补充数据") +//ExcelUtils.modify("D:\\街道补充数据.xlsx",{List item -> +// if(item["G"] && item["H"] == null){ +// DbUtils.seaweed.update(new BasicDBObject("_id", item["A"]), new BasicDBObject('$set', new BasicDBObject('township', item["G"]))) +// } +// println(item) +//}) +//ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Downloads\\错误经纬度.xlsx",{List item-> +// def location = "${item["A"]},${item["B"]}" +// JSONObject resp = GaoDeApiUtils.geocode(location) +// if(resp.regeocode && resp.regeocode.formatted_address){ +// item['G'] = resp.regeocode.formatted_address +// println(item) +// } +//}) +//def count = 0 +//CommonUtils.findAllCity().each {def city-> +// DbUtils.executeQueryOnSeaweed(DbUtils.buildSeaweedQuery(city),{BasicDBObject item-> +// if(item.township){ +// count++ +// } +// }, new BasicDBObject("township", 1)) +//} +//println(count) +ElasticUtils.executeSqlQuery("select count(*), p_date from seawater where p_date > 20210501 group by p_date",{ + println(it) +}) \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/test/Test1.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/test/Test1.groovy index 401d647e4..ff32247d7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/test/Test1.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/test/Test1.groovy @@ -9,6 +9,11 @@ import com.fangjia.core.util.HttpRequest import com.fangjia.script.util.ExcelUtils import net.sf.json.JSONObject +def strt(String addr){ + +} + + ExcelUtils.modify("D:\\nextcloud\\数据\\地址类服务测试\\建行地址相似度评级测试.xlsx",{List item-> def city = item["A"] def address1 = item["B"] From 0e354a525d28eb294ba6c7b6aacfdce4be701616 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 24 May 2021 09:54:57 +0800 Subject: [PATCH 17/17] =?UTF-8?q?=E7=A6=8F=E5=B7=9E=E7=BD=91=E7=AB=99?= =?UTF-8?q?=E5=8F=8A=E4=BD=BF=E7=94=A8=E6=9C=9F=E9=99=90=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fuzhou/DistrictInfoExportBatch.groovy | 3 +++ .../fuzhou/ThirdPriceExportBatch.groovy | 10 +++---- .../fangjia/script/gj/YouDaDataParser.groovy | 4 +-- .../gj/district/FangjiaClientUtils.groovy | 6 ++++- .../thirdPriceCrawl/CrawalDBUtil.groovy | 5 ++++ .../thirdPriceCrawl/DataStandardBatch.groovy | 2 +- .../ThirdDataToGatherBatch.groovy | 2 +- .../fangjia/script/util/OpenApiClient.groovy | 27 +++++++++++++++++++ .../yinjihaun/amap/GaoDeMapApiData.groovy | 26 ++++++++++++++++++ 9 files changed, 75 insertions(+), 10 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy index 95e40fb9b..fdf2ce5f9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy @@ -165,6 +165,9 @@ class DistrictInfoExportBatch extends BaseBatch{ rets << "非钢混" } } + if(rets.size()==0){ + rets << "钢混" + } return rets.join(",") } return "钢混"; diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/ThirdPriceExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/ThirdPriceExportBatch.groovy index c08426b9c..6a36f086d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/ThirdPriceExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/ThirdPriceExportBatch.groovy @@ -18,7 +18,7 @@ import org.apache.commons.codec.digest.DigestUtils */ class ThirdPriceExportBatch extends BaseBatch{ - def sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -3, "yyyyMM")); + def sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM")); def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") @@ -156,12 +156,12 @@ class ThirdPriceExportBatch extends BaseBatch{ if(!retMap.cityhouse_esf_price && d.cityhouse_esf_price){ retMap.cityhouse_esf_price = d.cityhouse_esf_price } - if(!retMap.fangtan_esf_price && d.fangtan_esf_price){ - retMap.fangtan_esf_price = d.fangtan_esf_price + if(!retMap.fangtan_esf_price && d.jiwu_esf_price){//使用吉屋替换房探 待抓取 + retMap.fangtan_esf_price = d.jiwu_esf_price } - if(!retMap.fangdoudou_esf_price && d.fangdoudou_esf_price){ - retMap.fangdoudou_esf_price = d.fangdoudou_esf_price + if(!retMap.fangdoudou_esf_price && d.get("365house_esf_price")){//使用淘房365替换房多多 + retMap.fangdoudou_esf_price = d.get("365house_esf_price") } if(!retMap.maitian_esf_price && d.maitian_esf_price){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy index 32b251e77..25c631832 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy @@ -47,7 +47,7 @@ class YouDaDataParser { def final AtomicInteger count_ = new AtomicInteger(0) - static def statDate = "2021-03-01" + static def statDate = "2021-04-01" static main(args) { def task = new YouDaDataParser(); // task.newHouseparse();//友达一手房存入mongo @@ -285,7 +285,7 @@ class YouDaDataParser { def esfParser(){ - def files = ["D:\\数据\\友达\\2021年4月\\二手房\\查询1.DBF"] + def files = ["D:\\数据\\友达\\2021年5月\\二手房\\查询1.DBF"] def datas = [] for(def fileName : files){ def rows = readDBF(fileName); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy index e0f5df506..033464eb5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy @@ -9,7 +9,11 @@ class FangjiaClientUtils { public static def getMapData(def city, def region, def name, def address, def types, def mapCode){ JSONObject jsonObj = null try { - def url = "http://192.168.0.19:38080/FangJiaDataService/match?city="+city+"®ion="+region+"&name="+name+"&address="+address+"&types="+types+"&mapCode="+mapCode + def url = "http://192.168.0.19:38080/FangJiaDataService/match?city="+city+"®ion="+region+"&name="+name+"&types="+types+"&mapCode="+mapCode + //def url = "http://192.168.10.166:8080/FangJiaDataService/match?city="+city+"®ion="+region+"&name="+name+"&types="+types+"&mapCode="+mapCode + if(address){ + url = url + "&address="+address + } // def url = "http://localhost:8080/FangJiaDataService/match?city="+city+"®ion="+region+"&name="+name+"&address="+address+"&types="+types+"&mapCode="+mapCode def content = new HttpRequest().newGet(url, null, "utf-8") jsonObj = JSONObject.fromObject(content) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/CrawalDBUtil.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/CrawalDBUtil.groovy index 4f6bc253e..c0cc1f468 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/CrawalDBUtil.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/CrawalDBUtil.groovy @@ -25,6 +25,7 @@ class CrawalDBUtil { static DBCollection data_process_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "data_process") static DBCollection average_statistics_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "average_statistics") static DBCollection map_district_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "map_district_process") + static DBCollection districtDetailColl = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "district_detail") public static getSeedsCol() { return seedsColl; @@ -67,6 +68,10 @@ class CrawalDBUtil { return map_district_col; } + public static getDistrictDetailColl(){ + return districtDetailColl; + } + static def updateSeedsCount(def id, def count) { DBObject uquery = new BasicDBObject() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy index 3a8ee77b2..c8ab1b6d8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy @@ -61,7 +61,7 @@ class DataStandardBatch extends BaseBatch{ def static final SHORT_DATE = Utils.getLastMonth() def static final statusMap = ["城市未匹配":1,"区域未匹配":2,"小区匹配失败":3,"小区匹配成功":4,"异常错误":5, "无价格":6] - def static final websites = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集", "链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房"] + def static final websites = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集", "链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"] public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) static { thirdPartyColl = CrawalDBUtil.getThirdPartyCol(); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy index 67e67e520..5f96db29c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy @@ -30,7 +30,7 @@ class ThirdDataToGatherBatch extends BaseBatch{ "房估估":"fanggugu_esf","诸葛找房":"zhugezhaofang_esf","安居客":"anjuke_esf","搜房":"soufang_esf","中原地产":"zhongyuan_esf","赶集":"ganji_esf", "链家":"lianjia_esf","兔博士":"tuboshi_esf","城市房产":"cityhouse_esf","贝壳找房":"beike_esf","Q房":"qfang_esf","Q房(新)":"qfang_new","中原地产(新)":"zhongyuan_new", "乐有家":"leyoujia_esf","乐有家(新)":"leyoujia_new","房探":"fangtan_esf","房多多":"fangdoudou_esf","小资家":"xiaozijia_esf","我爱我家":"5i5j_esf","麦田":"maitian_esf", - "房途":"fangtoo_esf","C21":"c21_esf","乐居":"leju_esf","房产超市":"fccs_esf","58同城":"58tc_esf","365淘房":"365house_esf" + "房途":"fangtoo_esf","C21":"c21_esf","乐居":"leju_esf","房产超市":"fccs_esf","58同城":"58tc_esf","365淘房":"365house_esf","吉屋":"jiwu_esf" ] public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(2) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy index f35379b65..a23842026 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy @@ -309,6 +309,31 @@ public class OpenApiClient { return null } + + public static JSONObject addressStandard(String city, String address) { + if (StringUtils.isBlank(city) || StringUtils.isBlank(address)) { + throw new IllegalArgumentException(); + } + + try { + Map params = new HashMap(); + params.put("city", city); + params.put("address", address); + params.put("token", TOKEN); + String content = new HttpRequest().get("http://open.fangjia.com/address/standard", params, "utf-8") + JSONObject result = JSONObject.fromObject(content) + if (result.getInt("code") == 200) { + return result.getJSONObject("result"); + } else if (result.getInt("code") == 500) { + println("${city} ${address} ${result}") + } + } catch (Exception e) { + println("${city} ${address} ${e.getMessage()}") + e.printStackTrace(); + } + return null + } + public static JSONObject evaluateByGeo(String city, def lat, def lng, String buildDate, String estateType, String geoType) { try { Map params = new HashMap(); @@ -1214,4 +1239,6 @@ public class OpenApiClient { return null; } + + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeMapApiData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeMapApiData.groovy index 4d472474c..c9e44b3e0 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeMapApiData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeMapApiData.groovy @@ -223,6 +223,32 @@ class GaoDeMapApiData { doSearchByGaoDe(type, lng, lat, radius, page) } } + + def static doSearchByGaoDe(String type, Double lng, Double lat, int radius, int page, int pageSize) { + Map params = new HashMap() + params.put("key", GAO_DE_AK) + params.put("types", type) + params.put("location", Double.toString(lng)+ ","+ Double.toString(lat)) + params.put("radius",radius) + params.put("offset",pageSize) + params.put("page",page) + String content + try { + content = new HttpRequest().newGet("http://restapi.amap.com/v3/place/around", params, "utf-8") + } catch (Exception e) { + content = new HttpRequest().newGet("http://restapi.amap.com/v3/place/around", params, "utf-8") + } + JSONObject jsonObj = JSONObject.fromObject(content) + + if(jsonObj.get("infocode").equals("10000")){ + JSONArray pois = jsonObj.get("pois") + return pois + } else { + println jsonObj.get("infocode")+"exit........" + doSearchByGaoDe(type, lng, lat, radius, page, pageSize) + } + return null + } private synchronized static fzData(JSONArray pois) { pois.each { poi ->