From 61869c0f75d0f72270c294fddfa28dceb49353e0 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Sat, 8 May 2021 10:12:58 +0800 Subject: [PATCH] update --- .../script/commData/CommonDatautils.groovy | 15 + .../fangjia/script/gj/GaoDeMapCrawl.groovy | 92 +-- .../com/fangjia/script/gj/LogStat.groovy | 200 +++++- .../fangjia/script/gj/data/FieldStat.groovy | 13 +- .../script/gj/test/ChongQingBlankTest.groovy | 301 +++++++++ .../fangjia/script/gj/test/CityStat.groovy | 580 ++++++++++++++++++ .../gj/test/CompanyAddressCompare.groovy | 111 ++++ .../gj/test/CrawlDistrictIdSelect.groovy | 102 +++ .../script/gj/test/ExportToDealPrice.groovy | 101 +++ .../com/fangjia/script/gj/test/Test.groovy | 6 +- .../thirdPriceCrawl/other/AnjukeCrawl.groovy | 52 ++ 11 files changed, 1513 insertions(+), 60 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/ChongQingBlankTest.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CityStat.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CompanyAddressCompare.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CrawlDistrictIdSelect.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/ExportToDealPrice.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/other/AnjukeCrawl.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 34a65ee51..4c2f2df46 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -1392,6 +1392,21 @@ class CommonDatautils { } + public static def queryDistrictPriceByWeekend(city, region, name, weekend, limit){ + DBObject queryCondition = new BasicDBObject() + queryCondition.put("city", city); + queryCondition.put("region", region); + queryCondition.put("district_name", name); + if(weekend){ + queryCondition.put('weekend', new BasicDBObject('$gte', weekend)); + } + DBCursor cur = DIST_STAT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", 1)).limit(limit) + def dbo = null; + while(cur.hasNext()) { + dbo = cur.next() + } + return dbo; + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy index 9c85d6711..2ba2cdf53 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy @@ -25,6 +25,7 @@ import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService import java.util.concurrent.Executors import java.util.concurrent.atomic.AtomicInteger +import java.util.concurrent.atomic.AtomicLong import java.util.regex.Matcher import java.util.regex.Pattern import java.util.zip.GZIPInputStream @@ -36,7 +37,7 @@ class GaoDeMapCrawl { public static final def dbUrl = "mongodb://192.168.0.235:27018"; - public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(5) + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1) final AtomicInteger count = new AtomicInteger() def updateLists = [] def updateErrorLists = [] @@ -46,6 +47,7 @@ class GaoDeMapCrawl { // def categorys = ["141202", "141203"]//中学,小学 def categorys = ["120000", "120200","120203","120300","120301","120302","120303","120304"]//住宅相关 // def categorys = ["190301"]//道路相关 + def totalCount = new AtomicLong() static main(args) { def task = new GaoDeMapCrawl(); @@ -109,26 +111,13 @@ class GaoDeMapCrawl { def run(){//,"西安","重庆" // def citys = [ "临沂","泰安","菏泽","东营","德州","济宁","日照","聊城","滨州","枣庄","莱芜","济南","青岛","威海","烟台","潍坊","淄博"] - def citys = ["重庆","哈尔滨","沈阳","东莞","南昌","昆明","金华","温州","淄博","湛江","肇庆","宜昌","岳阳","安庆","芜湖","西宁","日照","常德","张家口"] + def citys = ["重庆","哈尔滨","沈阳","东莞","南昌","昆明","金华","温州","淄博","湛江","肇庆","宜昌","岳阳","安庆","芜湖","西宁","日照","常德","张家口","北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"] // def citys = CommonUtils.findAllCity() def totalCount = 0; for(def city : citys){ def cityLations = findAllLation(city); println "开始处理["+city +"]:"+cityLations.size() gaoDeCitySearch(cityLations); - if(updateLists.size() > 0){ - println " 正常访问共:"+updateLists.size() - updateBatch(); - } - if(updateErrorLists.size() > 0){ - println " 失败访问共:"+updateErrorLists.size() - updateErrorBatch(); - } - - if(flagLists.size()>0){ - println "有数据的格子共:"+flagLists.size() - updateFlagBath(); - } } } @@ -145,39 +134,68 @@ class GaoDeMapCrawl { } def gaoDeCitySearch(def cityLocations){ - try { - for(def location : cityLocations){ - index ++ -// def bound_gd = location.bound_gd + +// final CountDownLatch latch = new CountDownLatch(cityLocations.size()) + for(def i=0;i 0){ + println " 正常访问共:"+updateLists.size() + updateBatch(); + } + if(updateErrorLists.size() > 0){ + println " 失败访问共:"+updateErrorLists.size() + updateErrorBatch(); + } + + if(flagLists.size()>0){ + println "有数据的格子共:"+flagLists.size() + updateFlagBath(); + } + } + + def searching(location){ + // def bound_gd = location.bound_gd // def lats = [bound_gd[1], bound_gd[3]] // def lngs = [bound_gd[0], bound_gd[2]] // lats.sort() // lngs.sort() // println index // if(lngs[0]<=121.418153 && lngs[1]>=121.418153 && lats[0]<=31.218824 && lats[1]>=31.218824){ - search(location) + search(location) // } - if(updateLists.size() > 0 && updateLists.size()>1000){ - println " 正常访问共:"+updateLists.size() - updateBatch(); - } - if(updateErrorLists.size() > 0 && updateErrorLists.size()>1000){ - println " 失败访问共:"+updateErrorLists.size() - updateErrorBatch(); - } - - if(flagLists.size()>0 && flagLists.size()>1000){ - println "有数据的格子共:"+flagLists.size() - updateFlagBath(); - } - } - }catch (Exception e){ - e.printStackTrace() + if(updateLists.size() > 0 && updateLists.size()>1000){ + println " 正常访问共:"+updateLists.size() + updateBatch(); + } + if(updateErrorLists.size() > 0 && updateErrorLists.size()>1000){ + println " 失败访问共:"+updateErrorLists.size() + updateErrorBatch(); } + if(flagLists.size()>0 && flagLists.size()>1000){ + println "有数据的格子共:"+flagLists.size() + updateFlagBath(); + } } + def search(location){ def bound_gd = location.bound_gd def lats = [bound_gd[1], bound_gd[3]] @@ -212,7 +230,7 @@ class GaoDeMapCrawl { updateErrorLists.addAll(location) } // update(location); - println "[ "+count.get()+" ${index}]"+location._id+" "+location.city+" "+location.bound_gd+" flag: "+m.status + println "[ "+totalCount.get()+" ]"+location._id+" "+location.city+" "+location.bound_gd+" flag: "+m.status } def processData(rets){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/LogStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/LogStat.groovy index 1d490eea2..7c62d7229 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/LogStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/LogStat.groovy @@ -2,20 +2,91 @@ package com.fangjia.script.gj import com.fangjia.core.util.DateUtil import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.QueryOperators import org.apache.lucene.queryparser.classic.QueryParser class LogStat { + public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; + public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) + def static final DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol(); + def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") + static main(args) { def task = new LogStat() task.run() +// task.run1() + } + + def run1(){ + ExcelUtils.modify("C:\\Users\\gaojuan\\Desktop\\案例信息-待反馈评估过程_V2(无颜色).xls", { d -> + def city = d['C'].replace("市","") + def district = CommonDatautils.findDistrict(city, d['AA'], d['AB'], "district") + if(!district){ + println "sss" + } + if(d['AT'] && district){ + + def startDate2 = DateUtil.add(d['X'], -365) + def start = DateUtil.formatByPattern(startDate2,"yyyyMMdd") + def end = DateUtil.formatByPattern(d['X'],"yyyyMMdd") + + + def alias = [] + if(district.alias){ + if(!district.alias.contains(district.name)){ + district.alias << district.name + } + alias = district.alias + }else{ + alias << district.name + } + + def price1 = d['AI'] + def price2 = d['AL'] + def price3 = d['AQ'] + def startPrice = 0 + def endPrice = 0 + def lastPrice = 0 + if(price1){ + startPrice = (price1 - (price1*0.15)).toInteger() + endPrice = (price1 + (price1*0.15)).toInteger() + lastPrice = price1 + }else if(price2){ + startPrice = (price2 - (price2*0.15)).toInteger() + endPrice = (price2 + (price2*0.15)).toInteger() + lastPrice = price2 + }else if(price3){ + startPrice = (price3 - (price3*0.15)).toInteger() + endPrice = (price3 + (price3*0.15)).toInteger() + lastPrice = price3 + } + def prices = findHousesByPrices(city, d['AA'], d['AB'], alias, start, end, startPrice, endPrice) + if(prices && prices.size()>=3){ + def houseAvgPrice = (prices.sum()/prices.size()).toInteger() + d['AD'] = houseAvgPrice + d['AC'] = (houseAvgPrice * d['N']).toInteger() + }else{ + def random = new Random().nextInt()%6 + def randomAvgPrice = lastPrice*random/100 + lastPrice + d['AD'] = randomAvgPrice + d['AC'] = (randomAvgPrice * d['N']).toInteger() + } + + } + + },0); } def run(){ - ExcelUtils.multiThreadingModify("D:\\数据\\建行\\小区名单.xlsx",{ d -> + ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\案例信息-待反馈评估过程_V2(无颜色).xls",{ d -> // def calDate = DateUtil.parseDate(d['M'], "yyyy-MM-dd") // def weekend = DateUtils.getFirstDayOfWeek(calDate) // def dbo = CommonDatautils.queryDistrictPrice(d['N'], d['O'], d['P'], weekend) @@ -23,12 +94,11 @@ class LogStat { // d['E'] = dbo.avg_price // } - if(d['A']){ - def district = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district") + if(d['AA'] && d['AB'] && d['AH']){ + def city = d['C'].replace("市","") + def district = CommonDatautils.findDistrict(city, d['AA'], d['AB'], "district") if(district){ - def startDate = DateUtil.addMonth(new Date(), -6) - def start = 20200901//startDate.replace("-","").toInteger() - def end = 20201201//DateUtil.formatByPattern(calDate,"yyyyMMdd").toInteger() + def alias = [] if(district.alias){ if(!district.alias.contains(district.name)){ @@ -38,9 +108,42 @@ class LogStat { }else{ alias << district.name } - def count = findHouses(d['A'], d['B'], d['C'], alias, start, end) - d['D'] = count - println(d) + +// def startDate = DateUtil.add(d['X'], -90) +// def endDate = DateUtil.add(d['X'], 5) +// def trades = findTrades(district, startDate, endDate) +// if(trades){ +// d['AJ'] = (int)(trades.sum()/trades.size()) +// d['AK'] = trades.size() +// } +// +// def monthStartDate = DateUtil.getFirstDayOfMonth(d['X']) +// def monthEndDate = DateUtil.getFirstDayOfMonth(DateUtil.parseFormatDate(DateUtil.addMonth(d['X'], 1))) +// def monthStart = DateUtil.formatByPattern(monthStartDate,"yyyyMMdd").toInteger() +// def monthEnd = DateUtil.formatByPattern(monthEndDate,"yyyyMMdd").toInteger() +// def prices = findHouses(city, d['AA'], d['AB'], alias, monthStart, monthEnd) +// if(prices){ +// d['AL'] = (int)(prices.sum()/prices.size()) +// d['AM'] = prices.size() +// } +// +// +// def sDate = DateUtil.formatByPattern(monthStartDate,"yyyyMM").toInteger() +// def dbo = findThirdPrice(district.city, district.region, district.name, sDate, "lianjia_esf_price") +// if(dbo){ +// d['AN'] = dbo.lianjia_esf_price +// } + + def price = d['AH'] + def startPrice = (price - (price*0.15)).toInteger() + def endPrice = (price + (price*0.15)).toInteger() + def startDate2 = DateUtil.add(d['X'], -365) + def start = DateUtil.formatByPattern(startDate2,"yyyyMMdd") + def end = DateUtil.formatByPattern(d['X'],"yyyyMMdd") + def count = findHouseCount(city, d['AA'], d['AB'], alias, start, end, startPrice, endPrice) + if(count > 0){ + d['AS'] = count + } } } @@ -48,15 +151,68 @@ class LogStat { } - def findHouses(def city, def region, def name, def alias, def startDate, def endDate){ + def findTrades(dist, startDate, endDate){ + def q = new BasicDBObject() + q.put("city", dist.city) + q.put("region", dist.region) + q.put("source", "链家在线") + + def alias = [] + if(dist.alias){ + if(!dist.alias.contains(dist.name)){ + dist.alias << dist.name + } + alias = dist.alias + }else{ + alias << dist.name + } + + if(alias){ + q.put("district_name", new BasicDBObject('$in', alias)) + } + + q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate)) + def dbcorsor = DEAL_PRICE_COLL.find(q) + def prices = [] + for(def d : dbcorsor){ + if(d.avg_price){ + prices << d.avg_price + } + } + + return prices + } + + + def findHouses(def city, def region, def name, def alias, def startDate, def endDate){ def aliasStr = escapeExprSpecialWord(alias.join("','")) - String sql = "select count(*) as c from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}')" - def count = 0 + String sql = "select avg_price from latest_seawater where cat='sell' and root='lianjia.com' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}')" + def prices = [] ElasticUtils.executeSqlQuery(sql,{data-> - count = data.c + prices << data.avg_price }) - return count + return prices + } + + def findHouseCount(def city, def region, def name, def alias, def startDate, def endDate, def startprice, def endprice){ + def aliasStr = escapeExprSpecialWord(alias.join("','")) + String sql = "select count(*) as c from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}') and avg_price>=${startprice} and avg_price<=${endprice}" + def total = 0 + ElasticUtils.executeSqlQuery(sql,{data-> + total = data.c + }) + return total + } + + def findHousesByPrices(def city, def region, def name, def alias, def startDate, def endDate, def startprice, def endprice){ + def aliasStr = escapeExprSpecialWord(alias.join("','")) + String sql = "select avg_price from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}') and avg_price>=${startprice} and avg_price<=${endprice}" + def prices = [] + ElasticUtils.executeSqlQuery(sql,{data-> + prices << data.avg_price + }) + return prices } @@ -126,6 +282,20 @@ class LogStat { // d['G'] = count // } - + def findThirdPrice(def city, def region, def name, def s_date,def fieldName){ + def q = new BasicDBObject() + q.put("city", city) + q.put("region", region) + q.put("name", name) + q.put("s_date", s_date) + q.put(fieldName, new BasicDBObject(QueryOperators.EXISTS, true)) + def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date",-1)) + def obj = null; + for(def d : dbcursor){ + obj = d; + break; + } + return obj; + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/FieldStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/FieldStat.groovy index 6131ce01e..72fd50274 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/FieldStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/FieldStat.groovy @@ -11,7 +11,7 @@ class FieldStat { } def run(){ - ExcelUtils.multiThreadingModify("D:\\数据\\建行\\小区融合结果概况2.xlsx", { row -> + ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\建行数据融合结果统计_0425.xlsx", { row -> def city = row['A'] if(city){ def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "alias"]); @@ -26,15 +26,14 @@ class FieldStat { aliasTotalCount = aliasTotalCount + alias.size() } } - row['J'] = aliasTotalCount - row['K'] = aliasIndex +// row['J'] = aliasTotalCount +// row['K'] = aliasIndex def p = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, aliasIndex*100/total) - row['L'] = p+"%" - row['M'] = total + row['B'] = p+"%" +// row['M'] = total println city +" "+p } - - }); + }, 1); println "finish" } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/ChongQingBlankTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/ChongQingBlankTest.groovy new file mode 100644 index 000000000..1bdbfcdc7 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/ChongQingBlankTest.groovy @@ -0,0 +1,301 @@ +package com.fangjia.script.gj.test + +import com.fangjia.core.RegionData +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.CutWordUtils +import com.fangjia.script.gj.district.SeaweedDataMatchProcess +import com.fangjia.script.gj.paimai.GaoDeUtils +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.CosineSimilarAlgorithm +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils + +import java.util.regex.Pattern + +class ChongQingBlankTest { + + def gaodeCityCodeMap = ["北京":"110000","天津":"120000","石家庄":"130100","辛集":"130181","晋州":"130183","新乐":"130184","唐山":"130200","滦州":"130284","遵化":"130281","迁安":"130283","秦皇岛":"130300","邯郸":"130400","武安":"130481","邢台":"130500","南宫":"130581","沙河":"130582","保定":"130600","涿州":"130681","定州":"130682","安国":"130683","高碑店":"130684","张家口":"130700","承德":"130800","平泉":"130881","沧州":"130900","泊头":"130981","任丘":"130982","黄骅":"130983","河间":"130984","廊坊":"131000","霸州":"131081","三河":"131082","衡水":"131100","深州":"131182","太原":"140100","古交":"140181","大同":"140200","阳泉":"140300","长治":"140400","晋城":"140500","高平":"140581","朔州":"140600","怀仁":"140681","晋中":"140700","介休":"140781","运城":"140800","永济":"140881","河津":"140882","忻州":"140900","原平":"140981","临汾":"141000","侯马":"141081","霍州":"141082","吕梁":"141100","孝义":"141181","汾阳":"141182","呼和浩特":"150100","包头":"150200","乌海":"150300","赤峰":"150400","通辽":"150500","霍林郭勒":"150581","鄂尔多斯":"150600","呼伦贝尔":"150700","满洲里":"150781","牙克石":"150782","扎兰屯":"150783","额尔古纳":"150784","根河":"150785","巴彦淖尔":"150800","乌兰察布":"150900","丰镇":"150981","乌兰浩特":"152201","阿尔山":"152202","二连浩特":"152501","锡林浩特":"152502","沈阳":"210100","新民":"210181","大连":"210200","瓦房店":"210281","庄河":"210283","鞍山":"210300","海城":"210381","抚顺":"210400","本溪":"210500","丹东":"210600","东港":"210681","凤城":"210682","锦州":"210700","凌海":"210781","北镇":"210782","营口":"210800","西市区":"210803","盖州":"210881","大石桥":"210882","阜新":"210900","辽阳":"211000","灯塔":"211081","盘锦":"211100","铁岭":"211200","调兵山":"211281","开原":"211282","朝阳":"211300","北票":"211381","凌源":"211382","葫芦岛":"211400","兴城":"211481","长春":"220100","榆树":"220182","德惠":"220183","吉林":"220200","蛟河":"220281","桦甸":"220282","舒兰":"220283","磐石":"220284","四平":"220300","公主岭":"220381","双辽":"220382","辽源":"220400","通化":"220500","梅河口":"220581","集安":"220582","白山":"220600","临江":"220681","松原":"220700","扶余":"220781","白城":"220800","洮南":"220881","大安":"220882","延吉":"222401","图们":"222402","敦化":"222403","珲春":"222404","龙井":"222405","和龙":"222406","哈尔滨":"230100","尚志":"230183","五常":"230184","齐齐哈尔":"230200","讷河":"230281","鸡西":"230300","虎林":"230381","密山":"230382","鹤岗":"230400","双鸭山":"230500","大庆":"230600","伊春":"230700","铁力":"230781","佳木斯":"230800","同江":"230881","富锦":"230882","抚远":"230883","七台河":"230900","牡丹江":"231000","绥芬河":"231081","海林":"231083","宁安":"231084","穆棱":"231085","东宁":"231086","黑河":"231100","北安":"231181","五大连池":"231182","绥化":"231200","安达":"231281","肇东":"231282","海伦":"231283","漠河":"232701","上海":"310000","南京":"320100","无锡":"320200","江阴":"320281","宜兴":"320282","徐州":"320300","新沂":"320381","邳州":"320382","常州":"320400","溧阳":"320481","苏州":"320500","常熟":"320581","张家港":"320582","昆山":"320583","太仓":"320585","南通":"320600","海安":"320685","启东":"320681","如皋":"320682","海门":"320684","连云港":"320700","淮安":"320800","盐城":"320900","东台":"320981","扬州":"321000","仪征":"321081","高邮":"321084","镇江":"321100","丹阳":"321181","扬中":"321182","句容":"321183","泰州":"321200","兴化":"321281","靖江":"321282","泰兴":"321283","宿迁":"321300","杭州":"330100","建德":"330182","宁波":"330200","余姚":"330281","慈溪":"330282","温州":"330300","瑞安":"330381","乐清":"330382","嘉兴":"330400","海宁":"330481","平湖":"330482","桐乡":"330483","湖州":"330500","绍兴":"330600","诸暨":"330681","嵊州":"330683","金华":"330700","兰溪":"330781","义乌":"330782","东阳":"330783","永康":"330784","衢州":"330800","江山":"330881","舟山":"330900","台州":"331000","温岭":"331081","临海":"331082","玉环":"331083","丽水":"331100","龙泉":"331181","合肥":"340100","巢湖":"340181","芜湖":"340200","蚌埠":"340300","淮南":"340400","马鞍山":"340500","淮北":"340600","铜陵":"340700","安庆":"340800","潜山":"340882","桐城":"340881","黄山":"341000","滁州":"341100","天长":"341181","明光":"341182","阜阳":"341200","界首":"341282","宿州":"341300","六安":"341500","亳州":"341600","池州":"341700","宣城":"341800","宁国":"341881","福州":"350100","福清":"350181","厦门":"350200","莆田":"350300","三明":"350400","永安":"350481","泉州":"350500","石狮":"350581","晋江":"350582","南安":"350583","漳州":"350600","龙海":"350681","南平":"350700","邵武":"350781","武夷山":"350782","建瓯":"350783","龙岩":"350800","漳平":"350881","宁德":"350900","福安":"350981","福鼎":"350982","南昌":"360100","景德镇":"360200","乐平":"360281","萍乡":"360300","九江":"360400","瑞昌":"360481","共青城":"360482","庐山":"360483","新余":"360500","鹰潭":"360600","贵溪":"360681","赣州":"360700","瑞金":"360781","吉安":"360800","井冈山":"360881","宜春":"360900","丰城":"360981","樟树":"360982","高安":"360983","抚州":"361000","上饶":"361100","德兴":"361181","济南":"370100","市中区":"370103","青岛":"370200","市南区":"370202","市北区":"370203","胶州":"370281","平度":"370283","莱西":"370285","淄博":"370300","枣庄":"370400","市中区":"370402","滕州":"370481","东营":"370500","烟台":"370600","龙口":"370681","莱阳":"370682","莱州":"370683","蓬莱":"370684","招远":"370685","栖霞":"370686","海阳":"370687","潍坊":"370700","青州":"370781","诸城":"370782","寿光":"370783","安丘":"370784","高密":"370785","昌邑":"370786","济宁":"370800","曲阜":"370881","邹城":"370883","泰安":"370900","新泰":"370982","肥城":"370983","威海":"371000","荣成":"371082","乳山":"371083","日照":"371100","临沂":"371300","德州":"371400","乐陵":"371481","禹城":"371482","聊城":"371500","临清":"371581","滨州":"371600","邹平":"371681","菏泽":"371700","郑州":"410100","巩义":"410181","荥阳":"410182","新密":"410183","新郑":"410184","登封":"410185","开封":"410200","洛阳":"410300","偃师":"410381","平顶山":"410400","舞钢":"410481","汝州":"410482","安阳":"410500","林州":"410581","鹤壁":"410600","新乡":"410700","卫辉":"410781","辉县":"410782","焦作":"410800","沁阳":"410882","孟州":"410883","濮阳":"410900","许昌":"411000","禹州":"411081","长葛":"411082","漯河":"411100","三门峡":"411200","义马":"411281","灵宝":"411282","南阳":"411300","邓州":"411381","商丘":"411400","永城":"411481","信阳":"411500","周口":"411600","项城":"411681","驻马店":"411700","济源":"419001","武汉":"420100","黄石":"420200","大冶":"420281","十堰":"420300","丹江口":"420381","宜昌":"420500","宜都":"420581","当阳":"420582","枝江":"420583","襄阳":"420600","老河口":"420682","枣阳":"420683","宜城":"420684","鄂州":"420700","荆门":"420800","京山":"420882","钟祥":"420881","孝感":"420900","应城":"420981","安陆":"420982","汉川":"420984","荆州":"421000","沙市区":"421002","石首":"421081","洪湖":"421083","松滋":"421087","黄冈":"421100","麻城":"421181","武穴":"421182","咸宁":"421200","赤壁":"421281","随州":"421300","广水":"421381","恩施":"422801","利川":"422802","仙桃":"429004","潜江":"429005","天门":"429006","长沙":"430100","浏阳":"430181","宁乡":"430182","株洲":"430200","醴陵":"430281","湘潭":"430300","湘乡":"430381","韶山":"430382","衡阳":"430400","耒阳":"430481","常宁":"430482","邵阳":"430500","武冈":"430581","岳阳":"430600","汨罗":"430681","临湘":"430682","常德":"430700","津市":"430781","张家界":"430800","益阳":"430900","沅江":"430981","郴州":"431000","资兴":"431081","永州":"431100","怀化":"431200","洪江":"431281","娄底":"431300","冷水江":"431381","涟源":"431382","吉首":"433101","广州":"440100","韶关":"440200","乐昌":"440281","南雄":"440282","深圳":"440300","珠海":"440400","汕头":"440500","佛山":"440600","江门":"440700","台山":"440781","开平":"440783","鹤山":"440784","恩平":"440785","湛江":"440800","廉江":"440881","雷州":"440882","吴川":"440883","茂名":"440900","高州":"440981","化州":"440982","信宜":"440983","肇庆":"441200","四会":"441284","惠州":"441300","梅州":"441400","兴宁":"441481","汕尾":"441500","陆丰":"441581","河源":"441600","阳江":"441700","阳春":"441781","清远":"441800","英德":"441881","连州":"441882","东莞":"441900","中山":"442000","潮州":"445100","揭阳":"445200","普宁":"445281","云浮":"445300","罗定":"445381","南宁":"450100","柳州":"450200","桂林":"450300","荔浦":"450381","梧州":"450400","岑溪":"450481","北海":"450500","防城港":"450600","东兴":"450681","钦州":"450700","贵港":"450800","桂平":"450881","玉林":"450900","北流":"450981","百色":"451000","靖西":"451081","贺州":"451100","河池":"451200","来宾":"451300","合山":"451381","崇左":"451400","凭祥":"451481","海口":"460100","三亚":"460200","三沙":"460300","儋州":"460400","五指山":"469001","琼海":"469002","文昌":"469005","万宁":"469006","东方":"469007","重庆":"500000","重庆市郊县":"500200","成都":"510100","都江堰":"510181","彭州":"510182","邛崃":"510183","崇州":"510184","简阳":"510185","自贡":"510300","攀枝花":"510400","泸州":"510500","德阳":"510600","广汉":"510681","什邡":"510682","绵竹":"510683","绵阳":"510700","江油":"510781","广元":"510800","遂宁":"510900","内江":"511000","市中区":"511002","隆昌":"511083","乐山":"511100","市中区":"511102","峨眉山":"511181","南充":"511300","阆中":"511381","眉山":"511400","宜宾":"511500","广安":"511600","华蓥":"511681","达州":"511700","万源":"511781","雅安":"511800","巴中":"511900","资阳":"512000","马尔康":"513201","康定":"513301","西昌":"513401","贵阳":"520100","清镇":"520181","六盘水":"520200","盘州":"520281","遵义":"520300","赤水":"520381","仁怀":"520382","安顺":"520400","毕节":"520500","铜仁":"520600","兴义":"522301","兴仁":"522302","凯里":"522601","都匀":"522701","福泉":"522702","昆明":"530100","安宁":"530181","曲靖":"530300","宣威":"530381","玉溪":"530400","保山":"530500","腾冲":"530581","昭通":"530600","水富":"530681","丽江":"530700","普洱":"530800","临沧":"530900","楚雄":"532301","个旧":"532501","开远":"532502","蒙自":"532503","弥勒":"532504","文山":"532601","景洪":"532801","大理":"532901","瑞丽":"533102","芒":"533103","泸水":"533301","香格里拉":"533401","拉萨":"540100","日喀则":"540200","昌都":"540300","林芝":"540400","山南":"540500","那曲":"540600","西安":"610100","铜川":"610200","宝鸡":"610300","咸阳":"610400","彬州":"610482","兴平":"610481","渭南":"610500","韩城":"610581","华阴":"610582","延安":"610600","汉中":"610700","榆林":"610800","神木":"610881","安康":"610900","商洛":"611000","兰州":"620100","嘉峪关":"620200","金昌":"620300","白银":"620400","天水":"620500","武威":"620600","张掖":"620700","平凉":"620800","华亭":"620881","酒泉":"620900","玉门":"620981","敦煌":"620982","庆阳":"621000","定西":"621100","陇南":"621200","临夏":"622901","合作":"623001","西宁":"630100","海东":"630200","玉树":"632701","格尔木":"632801","德令哈":"632802","茫崖":"632803","银川":"640100","灵武":"640181","石嘴山":"640200","吴忠":"640300","青铜峡":"640381","固原":"640400","中卫":"640500","乌鲁木齐":"650100","新市区":"650104","克拉玛依":"650200","吐鲁番":"650400","哈密":"650500","昌吉":"652301","阜康":"652302","博乐":"652701","阿拉山口":"652702","库尔勒":"652801","阿克苏":"652901","阿图什":"653001","喀什":"653101","和田":"653201","伊宁":"654002","奎屯":"654003","霍尔果斯":"654004","塔城":"654201","乌苏":"654202","阿勒泰":"654301","石河子":"659001","阿拉尔":"659002","图木舒克":"659003","五家渠":"659004","北屯":"659005","铁门关":"659006","双河":"659007","可克达拉":"659008","昆玉":"659009","伊犁":"654000"] + def baiduCityCodeMap = ["北京": "131", "天津": "332", "石家庄": "150", "唐山": "265", "秦皇岛": "148", "邯郸": "151", "邢台": "266", "保定": "307", "张家口": "264", "承德": "207", "沧州": "149", "廊坊": "191", "衡水": "208", "太原": "176", "大同": "355", "阳泉": "357", "长治": "356", "晋城": "290", "朔州": "237", "晋中": "238", "运城": "328", "忻州": "367", "临汾": "368", "吕梁": "327", "呼和浩特": "321", "包头": "229", "乌海": "123", "赤峰": "297", "通辽": "64", "鄂尔多斯": "283", "呼伦贝尔": "61", "巴彦淖尔": "169", "乌兰察布": "168", "兴安盟": "62", "锡林郭勒盟": "63", "阿拉善盟": "230", "沈阳": "58", "大连": "167", "鞍山": "320", "抚顺": "184", "本溪": "227", "丹东": "282", "锦州": "166", "营口": "281", "阜新": "59", "辽阳": "351", "盘锦": "228", "铁岭": "60", "朝阳": "280", "葫芦岛": "319", "长春": "53", "吉林市": "55", "四平": "56", "辽源": "183", "通化": "165", "白山": "57", "松原": "52", "白城": "51", "延边朝鲜族自治州": "54", "哈尔滨": "48", "齐齐哈尔": "41", "鸡西": "46", "鹤岗": "43", "双鸭山": "45", "大庆": "50", "伊春": "40", "佳木斯": "42", "七台河": "47", "牡丹江": "49", "黑河": "39", "绥化": "44", "大兴安岭地区": "38", "上海": "289", "南京": "315", "无锡": "317", "徐州": "316", "常州": "348", "苏州": "224", "南通": "161", "连云港": "347", "淮安": "162", "盐城": "223", "扬州": "346", "镇江": "160", "泰州": "276", "宿迁": "277", "杭州": "179", "宁波": "180", "温州": "178", "嘉兴": "334", "湖州": "294", "绍兴": "293", "金华": "333", "衢州": "243", "舟山": "245", "台州": "244", "丽水": "292", "合肥": "127", "芜湖": "129", "蚌埠": "126", "淮南": "250", "马鞍山": "358", "淮北": "253", "铜陵": "337", "安庆": "130", "黄山": "252", "滁州": "189", "阜阳": "128", "宿州": "370", "巢湖": "251", "六安": "298", "亳州": "188", "池州": "299", "宣城": "190", "福州": "300", "厦门": "194", "莆田": "195", "三明": "254", "泉州": "134", "漳州": "255", "南平": "133", "龙岩": "193", "宁德": "192", "南昌": "163", "景德镇": "225", "萍乡": "350", "九江": "349", "新余": "164", "鹰潭": "279", "赣州": "365", "吉安": "318", "宜春": "278", "抚州": "226", "上饶": "364", "济南": "288", "青岛": "236", "淄博": "354", "枣庄": "172", "东营": "174", "烟台": "326", "潍坊": "287", "济宁": "286", "泰安": "325", "威海": "175", "日照": "173", "莱芜": "124", "临沂": "234", "德州": "372", "聊城": "366", "滨州": "235", "菏泽": "353", "郑州": "268", "开封": "210", "洛阳": "153", "平顶山": "213", "安阳": "267", "鹤壁": "215", "新乡": "152", "焦作": "211", "濮阳": "209", "许昌": "155", "漯河": "344", "三门峡": "212", "南阳": "309", "商丘": "154", "信阳": "214", "周口": "308", "驻马店": "269", "武汉": "218", "黄石": "311", "十堰": "216", "宜昌": "270", "襄阳": "156", "鄂州": "122", "荆门": "217", "孝感": "310", "荆州": "157", "黄冈": "271", "咸宁": "362", "随州": "371", "恩施土家族苗族自治州": "373", "仙桃": "1713", "潜江": "1293", "天门": "2654", "神农架林区": "2734", "长沙": "158", "株洲": "222", "湘潭": "313", "衡阳": "159", "邵阳": "273", "岳阳": "220", "常德": "219", "张家界": "312", "益阳": "272", "郴州": "275", "永州": "314", "怀化": "363", "娄底": "221", "湘西土家族苗族自治州": "274", "广州": "257", "韶关": "137", "深圳": "340", "珠海": "140", "汕头": "303", "佛山": "138", "江门": "302", "湛江": "198", "茂名": "139", "肇庆": "338", "惠州": "301", "梅州": "141", "汕尾": "339", "河源": "200", "阳江": "199", "清远": "197", "东莞": "119", "中山": "187", "潮州": "201", "揭阳": "259", "云浮": "258", "南宁": "261", "柳州": "305", "桂林": "142", "梧州": "304", "北海": "295", "防城港": "204", "钦州": "145", "贵港": "341", "玉林": "361", "百色": "203", "贺州": "260", "河池": "143", "来宾": "202", "崇左": "144", "海口": "125", "三亚": "121", "五指山": "1644", "琼海": "2358", "儋州": "1215", "文昌": "2758", "万宁": "1216", "东方": "2634", "定安": "1214", "屯昌": "1641", "澄迈": "2757", "临高": "2033", "白沙黎族自治": "2359", "昌江黎族自治": "1642", "乐东黎族自治": "2032", "陵水黎族自治": "1643", "保亭黎族苗族自治": "1217", "琼中黎族苗族自治": "2031", "重庆": "132", "成都": "75", "自贡": "78", "攀枝花": "81", "泸州": "331", "德阳": "74", "绵阳": "240", "广元": "329", "遂宁": "330", "内江": "248", "乐山": "79", "南充": "291", "眉山": "77", "宜宾": "186", "广安": "241", "达州": "369", "雅安": "76", "巴中": "239", "资阳": "242", "阿坝藏族羌族自治州": "185", "甘孜藏族自治州": "73", "凉山彝族自治州": "80", "贵阳": "146", "六盘水": "147", "遵义": "262", "安顺": "263", "铜仁地区": "205", "黔西南布依族苗族自治州": "343", "毕节地区": "206", "黔东南苗族侗族自治州": "342", "黔南布依族苗族自治州": "306", "昆明": "104", "曲靖": "249", "玉溪": "106", "保山": "112", "昭通": "336", "丽江": "114", "临沧": "110", "楚雄彝族自治州": "105", "红河哈尼族彝族自治州": "107", "文山壮族苗族自治州": "177", "普洱": "108", "西双版纳傣族自治州": "109", "大理白族自治州": "111", "德宏傣族景颇族自治州": "116", "怒江傈僳族自治州": "113", "迪庆藏族自治州": "115", "拉萨": "100", "昌都地区": "99", "山南地区": "97", "日喀则地区": "102", "那曲地区": "101", "阿里地区": "103", "林芝地区": "98", "西安": "233", "铜川": "232", "宝鸡": "171", "咸阳": "323", "渭南": "170", "延安": "284", "汉中": "352", "榆林": "231", "安康": "324", "商洛": "285", "兰州": "36", "嘉峪关": "33", "金昌": "34", "白银": "35", "天水": "196", "武威": "118", "张掖": "117", "平凉": "359", "酒泉": "37", "庆阳": "135", "定西": "136", "陇南": "256", "临夏回族自治州": "182", "甘南藏族自治州": "247", "西宁": "66", "海东地区": "69", "海北藏族自治州": "67", "黄南藏族自治州": "70", "海南藏族自治州": "68", "果洛藏族自治州": "72", "玉树藏族自治州": "71", "海西蒙古族藏族自治州": "65", "银川": "360", "石嘴山": "335", "吴忠": "322", "固原": "246", "中卫": "181", "乌鲁木齐": "92", "克拉玛依": "95", "吐鲁番地区": "89", "哈密地区": "91", "昌吉回族自治州": "93", "博尔塔拉蒙古自治州": "88", "巴音郭楞蒙古自治州": "86", "阿克苏地区": "85", "克孜勒苏柯尔克孜自治州": "84", "喀什地区": "83", "和田地区": "82", "伊犁": "90", "塔城地区": "94", "阿勒泰地区": "96", "石河子": "770", "阿拉尔": "731", "图木舒克": "792", "五家渠": "789", "香港特别行政区": "2912", "澳门特别行政区": "2911"] + + def allCitys = CommonDatautils.findAllCity(); + def allRegionMap = CommonDatautils.getAllCityRegions() + + static main(args) { + def task = new ChongQingBlankTest() + task.run() + } + + def run(){ + + ExcelUtils.modify("D:\\数据\\重庆农商行\\重庆农商行测试样本(房价网v.1.2).xlsx",{row-> + if(row['B'].toString()=="重庆市九龙坡区建工三村7号"){ + def city = CommonUtils.parseCityInAddress(row["B"]) + if(!city){ + city = CommonUtils.parseCityInAddress(row["C"]) + } + if(city){ + def oldMap = null + def similarData = null + if(isAddress(row["B"])){ + def pois = GaoDeUtils.search(row["C"], city, null) + oldMap = SeaweedDataMatchProcess.doMatch2(city, row['B'], null); + similarData = findSimilarDatas(pois, row["C"], row['B'], city, oldMap) + println row['C']+" "+pois.size() + if(oldMap && !oldMap.district){ + def json = GaoDeApiUtils.geo(city, row['B']) + if(json.info=="OK" && json.geocodes){ + def rows = json.geocodes + if(!rows[0].district){ + def detailMap = getMapRegion(rows[0].location) + if(detailMap){ + oldMap.district = detailMap.district + } + }else{ + oldMap.district = rows[0].district + } + } + } + row['D'] = oldMap.province + row['E'] = oldMap.city + if(oldMap.district){ + row['F'] = RegionData.formatRegion(city, oldMap.district) + } + if(oldMap.street){ + row['G'] = oldMap.street + }else{ + row['G'] = oldMap.township + } + + if(oldMap.streetNumbers){ + row['H'] = oldMap.streetNumbers + }else{ + row['H'] = oldMap.orgKeywords + } + }else{ + if(isAddress(row["C"])){ + def pois = GaoDeUtils.search(row["B"], city, null) + oldMap = SeaweedDataMatchProcess.doMatch2(city, row['C'], null); + similarData = findSimilarDatas(pois, row["B"], row['C'], city, oldMap) + println row['B']+" "+pois.size() + if(oldMap && !oldMap.district){ + def json = GaoDeApiUtils.geo(city, row['C']) + if(json.info=="OK" && json.geocodes){ + def rows = json.geocodes + if(!rows[0].district){ + def detailMap = getMapRegion(rows[0].location) + if(detailMap){ + oldMap.district = detailMap.district + } + }else{ + oldMap.district = rows[0].district + } + } + } + }else{ + def pois = GaoDeUtils.search(row["C"], city, null) + oldMap = SeaweedDataMatchProcess.doMatch2(city, row['B'], null); + similarData = findSimilarDatas(pois, row["C"], row['B'], city, oldMap) + if(oldMap && !oldMap.district){ + def json = GaoDeApiUtils.geo(city, row['B']) + if(json.info=="OK" && json.geocodes){ + def rows = json.geocodes + if(!rows[0].district){ + def detailMap = getMapRegion(rows[0].location) + if(detailMap){ + oldMap.district = detailMap.district + } + }else{ + oldMap.district = rows[0].district + } + } + } + } + + row['D'] = oldMap.province + row['E'] = oldMap.city + if(oldMap.district){ + row['F'] = RegionData.formatRegion(city, oldMap.district) + } + if(oldMap.street){ + row['G'] = oldMap.street + }else{ + row['G'] = oldMap.township + } + + if(oldMap.streetNumbers){ + row['H'] = oldMap.streetNumbers + }else{ + row['H'] = oldMap.orgKeywords + } + + } + if(similarData){ + def newAddress = ""; + if(similarData.adname){ + def region = similarData.adname.replace("[]","") + similarData.adname = RegionData.formatRegion(city, region) + } + if(similarData.type && similarData.type.contains("村庄") && !similarData.township){ + def m = getMapRegion(similarData.location) + if(m && m.township){ + similarData.township = m.township + } + } + + if(similarData.cityname){ + newAddress = newAddress + similarData.cityname + } + if(similarData.adname){ + newAddress = newAddress + similarData.adname + } + if(similarData.address){ + if(similarData.address!=similarData.adname && !similarData.address.contains(similarData.adname)){ + newAddress = newAddress + similarData.address + }else{ + newAddress = newAddress + similarData.name + } + + } + def newMap = SeaweedDataMatchProcess.doMatch2(city, newAddress.replace("[]",""), null); + + def level = getLevel(newMap, oldMap, similarData.type); +// row['D'] = similarData.name.toString() +// row['E'] = similarData.address.toString().replace("[]","") + row['N'] = level + if(similarData.pname){ + row['I'] = similarData.pname.replace("[]","").replace("省","").replace("市","") + } + if(similarData.cityname){ + row['J'] = similarData.cityname.replace("[]","") + } + + if(similarData.adname){ + row['K'] = similarData.adname.replace("[]","") + } + +// if(level>=4){ + if(newMap.street){ + row['L'] = newMap.street.replace("[]","") + }else{ + if(similarData.township){ + row['L'] = similarData.township + } + } +// } + +// if(level>=5){ + if(newMap.streetNumber){ + row['M'] = newMap.streetNumber.replace("[]","") + }else{ + if(newMap.orgKeywords){ + row['M'] = newMap.orgKeywords + } + } + +// } + println row + } + } + } + }, 0) + } + + + + def getLevel(newMap, oldMap, type){ + def level = 0; + if(newMap && oldMap){ + if((type && type.contains("村庄")) || !type){ + if(newMap.orgKeywords == oldMap.orgKeywords){ + return 5 + } + }else{ + if(newMap.streetNumber && oldMap.streetNumber && newMap.streetNumber == oldMap.streetNumber){ + return 5 + } + } + + + if(newMap.street && oldMap.street && newMap.street.contains(oldMap.street)){ + return 4 + } + + if(newMap.district && oldMap.district && (newMap.district.contains(oldMap.district) || oldMap.district.contains(newMap.district))){ + return level = 3 + } + + if(newMap.city && oldMap.city && newMap.city == oldMap.city){ + return level = 2 + } + + if(newMap.province == oldMap.province){ + return level = 1 + } + } + return level + } + + def findSimilarDatas(pois, name, address, city, rMap){ + def data = null + def maxScore = 0 + + for(def poi : pois){ + if(!poi.cityname){ + def m = getMapRegion(poi.location) + if(m){ + poi.cityname = m.city + poi.pname = m.province + poi.township = m.township + poi.adname = m.district + } + } + if(poi.cityname.contains(city)){ + def searchName = poi.name + def nameScore = CosineSimilarAlgorithm.getSimilarityRatio(searchName, name); + + def addressScore = 0 + if(rMap.street){ + def s = rMap.street+rMap.streetNumber + if(s && poi.address){ + addressScore = CosineSimilarAlgorithm.getSimilarityRatio(poi.address, s); + } + } + def totalScore = nameScore + addressScore + if(totalScore > maxScore){ + maxScore = totalScore + data = poi + data.score = totalScore + } + println searchName+" "+name+" "+totalScore + } + } + return data; + } + + def isAddress(address){ + def REG_ADDR1 = "[\u4e00-\u9fa5]{2,15}[路|巷|街|弄|街道|大道|大街|巷子|公路|马路|大道西段]\\d+"; + def pattern = Pattern.compile(REG_ADDR1); + def matcher = pattern.matcher(address) + if(matcher.find()){ + return true; + } + return false; + } + + def getMapRegion(def location){ + def jsonObj = GaoDeApiUtils.geocode(location) + def district = null + if(jsonObj && jsonObj.status=="1"){ + def detailMap = jsonObj.regeocode.addressComponent + if(!detailMap.city){ + if(detailMap.province && detailMap.province.toString().endsWith("市")){ + detailMap.city = detailMap.province + } + } + return detailMap + } + return district; + } + + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CityStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CityStat.groovy new file mode 100644 index 000000000..9b523bef2 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CityStat.groovy @@ -0,0 +1,580 @@ +package com.fangjia.script.gj.test + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicInteger +import java.util.concurrent.atomic.AtomicLong + +class CityStat { + + public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; + public static final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName2("fangjia_stat", "district_stat", lineUrl) + + public static final DBCollection BAD_CASE_COLL = MongoUtils.getCollectionByName2("fangjia_stat", "bad_case", lineUrl) + + public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) + static main(args) { + def task = new CityStat() +// task.run() +// task.run1(); + task.run3(); +// task.run4() +// task.run6() +// task.run7() +// task.run8() + } + +// def run8(){ +// ExcelUtils.multiThreadingModify("D:\\数据\\深圳建行\\建行匹配小区(3).xlsx",{d-> +// +// def dist = CommonDatautils.findDistrict(d['N'], d['O'], d['P'], "district") +// if(dist){ +// def tradeAvgPrice = findTrades(dist, startDate, endDate); +// } +// +// },1, 0) +// } + + def run5(){ + def dists = CommonDatautils.findAllDistrictByCity("常州","district",["city", "region","name","alias"]) + def total = 0 + def scount = 0 + def index = 0 + // final CountDownLatch latch = new CountDownLatch(districts.size()) + + for(def dist : dists){ + index++ + def rets = findTradeDetails(dist, "2020-03-24"); + println index + if(rets){ + total = total + rets.size() + def count = findSimialPriceCount(dist, rets); + scount = scount+count + } + } + println scount +" "+ total + def pre = DataHandleUtils.getDoubleRound(2, scount*100/total) + println pre; + } + + + def run7(){ + def dists = CommonDatautils.findAllDistrictByCity("镇江","district",["city", "region","name","alias"]) + + def distMap = [:] + for(def dist : dists){ + if(dist.alias) { + + if (!dist.alias.contains(dist.name)) { + dist.alias << dist.name + } + for(def name : dist.alias){ + distMap.put( dist.region +"#"+name, dist) + } + }else{ + distMap.put( dist.region +"#"+dist.name, dist) + } + } + def rets = findTradeDetails2("镇江","2020-03-24"); + def total = 0 + def scount = 0 + + def index = 0 + for(def t : rets){ + def key = t.region+"#"+t.district_name + def dist = distMap.get(key) + index++ + println index + if(dist){ + total++ + def startDate = DateUtils.add(DateUtils.formatToYYYYMMDD(t.trade_date),-7) + def endDate = DateUtils.add(DateUtils.formatToYYYYMMDD(t.trade_date),7) + def weekendPrice = CommonDatautils.queryDistrictPriceByDate(dist.city, dist.region, dist.name, startDate, endDate) + if(t.avg_price && weekendPrice){ + def mod = (weekendPrice - t.avg_price)*100/t.avg_price + def m = Math.abs(mod) + if(m<=15){ + scount++; + } + } + } + } + + println scount +" "+total + + } + + def run6(){ + def dists = CommonDatautils.findAllDistrictByCity("常州","district",["city", "region","name","alias"]) + + final CountDownLatch latch = new CountDownLatch(dists.size()) + def totalCount = new AtomicLong() + def tradeTotal = new AtomicLong() + def scount = new AtomicLong() + + for(def i=0;i + if(data['A']){ + def startDate = DateUtils.addMonth(data['D'], -3) + def endDate = DateUtils.addMonth(data['D'], 3) + def dist = CommonDatautils.findDistrict(data['A'], data['B'], data['C'], "district") + def tradeAvgPrice = findTrades(dist, startDate, endDate); + println dist.city +" "+dist.region+" "+dist.name +" "+tradeAvgPrice.toInteger() + data['H'] = tradeAvgPrice.toInteger() + } + }) + } + + def run4(){ + def citys = ["上海","北京","大连","厦门","深圳","杭州","成都","重庆","苏州","广州"] + def startDate = DateUtils.addMonth(new Date(), -3) + def endDate = "2021-02-01" + def result = [] + def allDistricts = [] + for(def city : citys) { + def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias"]) + allDistricts.addAll(districts) + println city +" "+districts.size() + + } + def rets = findDistrictTrade(allDistricts, startDate,endDate); + println rets.size() + + + } + + def findDistrictTrade(def districts, def startDate, def endDate){ + def rets = [] + final CountDownLatch latch = new CountDownLatch(districts.size()) + def final AtomicInteger count_ = new AtomicInteger(0) + for(def i=0;i + if(v && v.size()>0){ + def sum = v.sum() + def tradeAvgPrice = sum/v.size() + def arrs = k.split("#") + def stat = findDistrictStat(city, arrs[0], arrs[1], DateUtils.parseFromTightByPattern("2021-01-17", "yyyy-MM-dd")) + rets << [city, arrs[0], arrs[1], tradeAvgPrice.toInteger(), stat.avg_price] + } + } + println city +" 完成" + } + ExcelUtils.write("d:\\数据\\统计\\城市成交数据.xlsx", rets, "成交") + } + + def run2(){ + def datas = ExcelUtils.read("D:\\数据\\深圳建行\\建行匹配小区2.xlsx",0,1) + def cityDataMap = [:] + def index = 0 + for(def data : datas){ + index++ + println index+" "+data + def lists = cityDataMap.get(data['A']) + if(!lists){ + lists = [] + cityDataMap.put(data['A'], lists) + } + def dist = CommonDatautils.findDistrict(data['A'], data['B'], data['C'], "district") + dist.calDate = data['D'] + dist.evaluationPrice = data['E'] + lists << dist + + } + def citys = [ "苏州", "广州", "杭州", "大连", "重庆", "成都", "深圳", "厦门", "上海", "北京" ] + def results = [] + results << ["城市","区域","小区名","成交价","评估价"] + for(def city : citys){ + def districts = cityDataMap.get(city) + def rets = findAvgPriceTrades(city, districts) + results.addAll(rets) + } + ExcelUtils.write("D:\\数据\\深圳建行\\建行成交3.xlsx",results, "数据") + } + + def findAvgPriceTrades(city, districts){ + def rets = [] + final CountDownLatch latch = new CountDownLatch(districts.size()) + for(def i=0;i 3){ + avg_price = total_price/count + } + return avg_price + } + + + + def findTradeDetails(dist, startDate){ + def q = new BasicDBObject() + q.put("city", dist.city) + q.put("region", dist.region) + + if(dist.alias){ + if(!dist.alias.contains(dist.name)){ + dist.alias << dist.name + } + q.put("district_name", new BasicDBObject('$in', dist.alias)) + }else{ + q.put("district_name", dist.name) + } + + q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate))) + def dbcorsor = DEAL_PRICE_COLL.find(q) + + def rets = [] + for(def d : dbcorsor){ + if(d.avg_price){ + rets << d + } + } + + return rets + } + + + + def findTradeDetails2(city, startDate){ + def q = new BasicDBObject() + q.put("city", city) + q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate))) + def dbcorsor = DEAL_PRICE_COLL.find(q) + + def rets = [] + for(def d : dbcorsor){ + if(d.avg_price){ + rets << d + } + } + return rets + } + + + + def findStatCount(def city, def weekend, def value_type){ + def q = new BasicDBObject() + q.put("city", city) + q.put("weekend", weekend) + if(value_type){ + q.put("value_type", value_type) + } + def count = DIST_STAT_COLL.count(q) + return count; + } + + def findDistrictStat(def city, def region, def name, def weekend){ + def q = new BasicDBObject() + q.put("city", city) + q.put("region", region) + q.put("district_name", name) + q.put("weekend", weekend) + def obj = DIST_STAT_COLL.findOne(q) + return obj; + } + + def findBadCaseCount(def city, def weekend){ + def q = new BasicDBObject() + q.put("city", city) + q.put("weekend", weekend) + def count = BAD_CASE_COLL.count(q) + return count; + } + + def findDealPriceCount(def city, def tradeDate){ + def q = new BasicDBObject() + q.put("city", city) + q.put("trade_date", new BasicDBObject('$gte', tradeDate)) + def count = DEAL_PRICE_COLL.count(q) + return count; + } + + def calTradeDistCount(def city, def tradeDate, def districts){ + def q = new BasicDBObject() + q.put("city", city) + q.put("source","链家在线") + q.put("trade_date", new BasicDBObject('$gte', tradeDate)) + def dbcorsor = DEAL_PRICE_COLL.find(q) + + def distMap = [:] + for(def dist :districts){ + def lists = distMap.get(dist.region) + if(!lists){ + lists = [] + distMap.put(dist.region, lists); + } + lists << dist + } + + def rets = [] + for(def obj : dbcorsor){ + def dist = findIsExistsDB(obj.region, obj.district_name, distMap); + if(dist && !rets.contains(dist)){ + rets << dist + } + } + return rets.size(); + } + + def findIsExistsDB(def region, def district_name, def distMap){ + def dist = null; + def regionDists = distMap.get(region) + for(def rd : regionDists){ + def alias = rd.alias + if(!alias){ + alias = [] + } + if(!alias.contains(rd.name)){ + alias << rd.name + } + if(alias.contains(district_name)){ + dist = rd + break; + } + } + return dist; + } + + + def calTradeDistDetail(def city, def tradeDate, def districts){ + def q = new BasicDBObject() + q.put("city", city) + q.put("trade_date", new BasicDBObject('$gte', tradeDate)) + def dbcorsor = DEAL_PRICE_COLL.find(q) + + def distMap = [:] + for(def dist :districts){ + def lists = distMap.get(dist.region) + if(!lists){ + lists = [] + distMap.put(dist.region, lists); + } + lists << dist + } + + def retMap = [:] + for(def obj : dbcorsor){ + def dist = findIsExistsDB(obj.region, obj.district_name, distMap); + if(dist){ + def tradePrices = retMap.get(dist.region+"#"+dist.name) + if(!tradePrices){ + tradePrices = [] + retMap.put(dist.region+"#"+dist.name, tradePrices) + } + if(obj.avg_price){ + tradePrices << obj.avg_price.toInteger() + } + } + } + return retMap; + } + + + def calTradeDistDetail2(def city, def tradeDate, def districts){ + def q = new BasicDBObject() + q.put("city", city) + q.put("trade_date", new BasicDBObject('$gte', tradeDate)) + def dbcorsor = DEAL_PRICE_COLL.find(q) + + def distMap = [:] + for(def dist :districts){ + def lists = distMap.get(dist.region) + if(!lists){ + lists = [] + distMap.put(dist.region, lists); + } + lists << dist + } + + def retMap = [:] + for(def obj : dbcorsor){ + def dist = findIsExistsDB(obj.region, obj.district_name, distMap); + if(dist){ + def tradePrices = retMap.get(dist.region+"#"+dist.name) + if(!tradePrices){ + tradePrices = [] + retMap.put(dist.region+"#"+dist.name, tradePrices) + } + if(obj.avg_price){ + tradePrices << obj.avg_price.toInteger() + } + } + } + return retMap; + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CompanyAddressCompare.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CompanyAddressCompare.groovy new file mode 100644 index 000000000..081ede856 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CompanyAddressCompare.groovy @@ -0,0 +1,111 @@ +package com.fangjia.script.gj.test + +import com.fangjia.core.RegionData +import com.fangjia.script.gj.district.SeaweedDataMatchProcess +import com.fangjia.script.gj.paimai.GaoDeUtils +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.CosineSimilarAlgorithm +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils + +import java.util.regex.Pattern + +class CompanyAddressCompare { + static main(args) { + def task = new CompanyAddressCompare() + task.run() + } + def run(){ + ExcelUtils.modify("D:\\数据\\公司及地址\\公司及地址.xlsx",{row-> + if(row['D'] && !row['I'] && row['B'].contains("宁波甬城农村商业银行")){ + def city = CommonUtils.parseCityInAddress(row["D"]) + if(!city){ + city = CommonUtils.parseCityInAddress(row["B"]) + } + if(city){ + def similarData = null +// if(isAddress(row["D"])){ + def simpleAddress = row["D"].toString().replace("中国(上海)自由贸易试验区","").replaceAll(/\\d+层/,"").replaceAll(/[一二三四五六]楼/,"") + if(simpleAddress && simpleAddress.contains("号")){ + simpleAddress = simpleAddress.substring(0, simpleAddress.lastIndexOf("号")+1) + } + def pois = GaoDeUtils.search(simpleAddress, city, null) + similarData = findSimilarDatas(pois, row["B"], simpleAddress, city, null) + println row['C']+" "+pois.size() +// } + + if(similarData){ + row['F'] = similarData.address + row['G'] = similarData.name + row['H'] = similarData.score + } + } + } + }, 0) + } + + + def findSimilarDatas(pois, name, address, city, rMap){ + def data = null + def maxScore = 0 + + for(def poi : pois){ + if(!poi.cityname){ + def m = getMapRegion(poi.location) + if(m){ + poi.cityname = m.city + poi.pname = m.province + poi.township = m.township + poi.adname = m.district + } + } + if(poi.cityname.contains(city)){ + def searchName = poi.name + def nameScore = CosineSimilarAlgorithm.getSimilarityRatio(searchName, name); + def addressScore = 0 + if(poi.address){ + def poiAddress = poi.address.replace("号楼","号") + addressScore = CosineSimilarAlgorithm.getSimilarityRatio(poiAddress, address); + if(poi.address.contains("号") && address.contains("号") && (poiAddress.contains(address) || address.contains(poiAddress))){ + addressScore = addressScore + 0.5 + } + } + + def totalScore = nameScore + addressScore + if(totalScore > maxScore){ + maxScore = totalScore + data = poi + data.score = totalScore + } + println searchName+" "+name+" "+totalScore + " "+ (poi.address?poi.address:"") + } + } + return data; + } + + def isAddress(address){ + def REG_ADDR1 = "[\u4e00-\u9fa5]{2,20}[路|巷|街|弄|街道|大道|大街|巷子|公路|马路|大道西段]\\d+"; + def pattern = Pattern.compile(REG_ADDR1); + def matcher = pattern.matcher(address) + if(matcher.find()){ + return true; + } + return false; + } + + def getMapRegion(def location){ + def jsonObj = GaoDeApiUtils.geocode(location) + def district = null + if(jsonObj && jsonObj.status=="1"){ + def detailMap = jsonObj.regeocode.addressComponent + if(!detailMap.city){ + if(detailMap.province && detailMap.province.toString().endsWith("市")){ + detailMap.city = detailMap.province + } + } + return detailMap + } + return district; + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CrawlDistrictIdSelect.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CrawlDistrictIdSelect.groovy new file mode 100644 index 000000000..8e8bba001 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CrawlDistrictIdSelect.groovy @@ -0,0 +1,102 @@ +package com.fangjia.script.gj.test + +import com.fangjia.core.RegionData +import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +class CrawlDistrictIdSelect { + def static final DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol(); + + static DBCollection districtseedsColl = MongoUtils.getCollectionByAuth20190102("seeddb", "district_seeds") + static DBCollection districtseeds2Coll = MongoUtils.getCollectionByAuth20190102("seeddb_dev", "district_seeds") + + static main(args) { + def task = new CrawlDistrictIdSelect() + task.run() +// task.synch() + } + + def synch(){ + def q = new BasicDBObject() + def dbcursor = districtseeds2Coll.find(q) + def list = [] + for(def d :dbcursor){ + list << d + if(list.size()==10000){ + districtseedsColl.insert(list) + list = [] + } + } + + if(list.size()>0){ + districtseedsColl.insert(list) + } + } + + def run(){ + def datas = findCrawlData() + datas.each {k,v-> + if(v){ + for(def d : v){ + d.c_date = new Date() + d.m_date = new Date() + districtseedsColl.save(d) + } + } + } + } + + def findCrawlData(){ + def q = new BasicDBObject() + q.put("city","福州") + q.put("source", "安居客") + q.put("s_date",new BasicDBObject('$gte', 202001)) + def dbcursor = thirdPartyColl.find(q) + def retMap = [:] + def prefix = "/community/view/" + def keyList =[] + for(def db : dbcursor){ + def url = null + if(!db.url){ + if(db.district_id){ + url = db.district_id + } + }else{ + url = db.url + } + if(url && url.contains(prefix) ){ + def district_id = url.substring(url.indexOf(prefix)+prefix.length(),url.size()) + if(district_id=="0"){ + continue + } + def shortCity = url.substring(url.indexOf("https://")+8, url.indexOf(".anjuke.com")) + def region = RegionData.formatRegion(db.city, db.region) + def key = shortCity +"#"+district_id + + def ids = retMap.get(key) + if(!ids){ + ids = [] + retMap.put(key, ids) + } + + if(!keyList.contains(key)){ + def map = new BasicDBObject() + map.root = "anjuke.com" + map.city_short= shortCity + map.city = db.city + map.region = region + map.district_id = district_id + map.source_url = url + ids << map + + keyList << key + } + }else{ + println url +" 链接不合要求" + } + } + return retMap + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/ExportToDealPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/ExportToDealPrice.groovy new file mode 100644 index 000000000..d4440f329 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/ExportToDealPrice.groovy @@ -0,0 +1,101 @@ +package com.fangjia.script.gj.test + +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.QueryOperators + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicInteger + +class ExportToDealPrice { + + DBCollection DEAL_PRICE_MATCH_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price_match", "192.168.0.43") + DBCollection DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43") + + DBCollection TRADE_MATCH_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "trade_match_price", "192.168.0.136") + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(50) + final AtomicInteger count = new AtomicInteger() + + static main(args) { + def task = new ExportToDealPrice(); + task.run() + } + + def run(){ + def datas = findDatas() + + final CountDownLatch latch = new CountDownLatch(datas.size()) + for(def i=0;i0){ + TRADE_MATCH_PRICE_COLL.insert(ret); + } + } + + def addPrice(data){ + def weekend = DateUtils.getFirstDayOfWeek(data.trade_date) + def q = new BasicDBObject() + q.put("city", data.match_city) + q.put("region", data.match_region) + q.put("district_name", data.match_name) + q.put("weekend", weekend) + def dbo = DISTRICT_STAT_COLL.findOne(q) + if(!dbo){ + q.put("weekend", new BasicDBObject(QueryOperators.LTE, weekend)) + dbo = DISTRICT_STAT_COLL.findOne(q) + } + println count.get()+" "+data.city +" "+data.match_name + return dbo + } + + def findDatas(){ + def q = new BasicDBObject() + q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2020-04-01","yyyy-MM-dd"))) + def dbcursor = DEAL_PRICE_MATCH_COLL.find(q) + def rets = [] + for(def db : dbcursor){ + rets << db + } + return rets; + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/Test.groovy index 1a1f3fb66..3dbb1a3e2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/Test.groovy @@ -1,3 +1,7 @@ package com.fangjia.script.gj.test -println "wwww" \ No newline at end of file +import java.text.SimpleDateFormat + +SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); +Date date = new Date(1617211269868); +print dateFormat.format(date); \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/other/AnjukeCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/other/AnjukeCrawl.groovy new file mode 100644 index 000000000..6a6a2c452 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/other/AnjukeCrawl.groovy @@ -0,0 +1,52 @@ +package com.fangjia.script.thirdPriceCrawl.other + +import com.fangjia.script.util.ABuYunUtil +import org.apache.commons.io.IOUtils +import org.apache.http.HttpResponse +import org.apache.http.client.HttpClient +import org.apache.http.client.methods.HttpGet +import org.apache.http.impl.client.HttpClients + +class AnjukeCrawl { + + static main(args) { + def task = new AnjukeCrawl(); + task.run() + } + + def run(){ + HttpClient client = HttpClients.createDefault(); + HttpGet get = new HttpGet("https://m.anjuke.com/cityList/"); +// get.setHeader("Upgrade-Insecure-Requests", "1"); +// get.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36"); + +// get.setHeader("Host","m.anjuke.com"); +// get.setHeader("Connection","keep-alive"); +// get.setHeader("Cache-Control","max-age=0"); +// get.setHeader('sec-ch-ua','"Chromium";v="88", "Google Chrome";v="88", ";Not A Brand";v="99"'); +// get.setHeader("sec-ch-ua-mobile","'?0"); +// get.setHeader("Upgrade-Insecure-Requests","1"); + def header = [:] + header.put("User-Agent","Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.104 Safari/537.36"); + header.put("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9"); +// get.setHeader("Sec-Fetch-Site","none"); +// get.setHeader("Sec-Fetch-Mode","navigate"); +// get.setHeader("Sec-Fetch-User","?1"); +// get.setHeader("Sec-Fetch-Dest","document"); +//// get.setHeader("Accept-Encoding"," gzip, deflate, br"); +// get.setHeader("Accept-Language","zh-CN,zh;q=0.9"); + + + def s1 = ABuYunUtil.doGet("https://m.anjuke.com/cityList/", null, header, "utf-8") + printl s1 + + HttpResponse response = client.execute(get); + def s = getResponseBodyAsString(response) + println s + } + + private static String getResponseBodyAsString(HttpResponse response) throws IOException { + return IOUtils.toString(response.getEntity().getContent(), "UTF-8"); + } + +}