diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fangjiatong/AccessLogBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fangjiatong/AccessLogBatch.groovy index 0b4d0724b..06e84af84 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fangjiatong/AccessLogBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fangjiatong/AccessLogBatch.groovy @@ -211,23 +211,35 @@ class AccessLogBatch extends BaseBatch{ } def read(){ - String fileName = "C:\\Users\\gaojuan\\Desktop\\localhost_access_log.2019-06-01.txt"; + String fileName = "D:\\数据\\logs\\performance.open.log"; File file = new File(fileName); def rets = [] - file.eachLine{ + def index = 0 + def list =[] + def slowCount = 0 + file.eachLine{it-> - if(it.contains("/fangjiatong2/")){ - try{ - it = it.replace(it, URLDecoder.decode(URLDecoder.decode(it, "utf-8"), "UTF-8")) - }catch(IllegalArgumentException e){} + + if(it.contains("ws/czbank") && it.contains("entity:House")){ +// println it + index++ +// try{ + def s1 = it.substring(it.indexOf("time:")+5, it.length()) + s1 = s1.substring(0, s1.indexOf("\t")) + list << s1.toInteger() + if(s1.isNumber() && s1.toInteger()>=4000){ + println it + slowCount++ + } +// }catch(IllegalArgumentException e){} } - if(!it.contains("漕溪四村")){ - rets << it - } + } - - writeTxt(rets, "C:\\Users\\gaojuan\\Desktop\\fjt-2020-6-1.txt"); + println list.sum()/list.size() + println index + println "slowCount:"+slowCount +// writeTxt(rets, "C:\\Users\\gaojuan\\Desktop\\fjt-2020-6-1.txt"); } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy index 76ef3729b..4ec26d7db 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy @@ -43,6 +43,24 @@ class OpenApiUtils { } + static def matchDetailDistrict(def city, def address, def suggestDistrict, def preciseLevelflag){ + address = address.replace("(","(").replace(")",")").replace(" ","") + def url = "http://open.fangjia.com/address/matchV2?city="+city+"&address="+address+"&suggestDistrict="+suggestDistrict+"&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB&category=district&__refresh_cache=1" + if(preciseLevelflag){ + url = url +"&preciseLevel=strict" + } + String content = new HttpRequest().get(url, "utf-8"); + if(content){ + def doc = new JsonSlurper().parseText(content) + if(doc && doc.code ==200 && doc.result){ + def dto = doc.result + return dto + } + } + return null; + } + + static def matchDistrict2(def city, def address, def preciseLevelflag){ address = address.replace("(","(").replace(")",")").replace(" ","").replace("#","") def url = "http://open.fangjia.com/address/matchV2?city="+city+"&address="+address+"&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB&category=district&__refresh_cache=1" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/FieldStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/FieldStat.groovy index 72fd50274..eb7d4aadd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/FieldStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/FieldStat.groovy @@ -1,39 +1,106 @@ package com.fangjia.script.gj.data import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection class FieldStat { + final static DBCollection CITY_INFO_CONN = MongoUtils.getCollectionByName("fangjia_data_ana", "city_info", MongoUtils.MongoDB_Domain_mongo); + static main(args) { def task = new FieldStat(); - task.run() +// task.run() + task.run1() +// task.run2() + } + + def run1(){ + def citys = ["北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"] + def dist_total = 0 + + def addr_alias_total = 0 + def rets = [] + rets << ["城市","别名总量","有别名小区数量","小区总量","有别名小区占比"] + for(def city : citys){ + def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "alias", "addr_alias"]); + def alias_index = 0 + def alias_total = 0 + + for(def d : districts){ + def alias = d.alias + if(alias){ + alias_index++ + alias_total = alias_total + alias.size() + } + } + def precent = DataHandleUtils.getDoubleRound(2, alias_index*100.0/districts.size()) + rets << [city, alias_total, alias_index, districts.size(), precent+"%"] + + } + ExcelUtils.write("d:\\数据\\92个城市别名统计.xlsx", rets, "统计") +// println dist_total +// println alias_total +// println addr_alias_total } def run(){ - ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\建行数据融合结果统计_0425.xlsx", { row -> - def city = row['A'] + ExcelUtils.multiThreadingModify("D:\\数据\\城市统计.xlsx", { row -> + def city = row['B'] if(city){ - def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "alias"]); + def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]); def total = districts.size() + row['D'] = total - def aliasIndex = 0 - def aliasTotalCount = 0 - for(def d : districts){ - def alias = d.alias - if(alias){ - aliasIndex++ - aliasTotalCount = aliasTotalCount + alias.size() - } - } -// row['J'] = aliasTotalCount -// row['K'] = aliasIndex - def p = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, aliasIndex*100/total) - row['B'] = p+"%" -// row['M'] = total - println city +" "+p +// def aliasIndex = 0 +// def aliasTotalCount = 0 +// for(def d : districts){ +// def alias = d.alias +// if(alias){ +// aliasIndex++ +// aliasTotalCount = aliasTotalCount + alias.size() +// } +// } +//// row['J'] = aliasTotalCount +//// row['K'] = aliasIndex +// def p = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, aliasIndex*100/total) +// row['B'] = p+"%" +//// row['M'] = total +// println city +" "+p } - }, 1); + }, 0); println "finish" } + + def run2(){ + def citys = ["北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"] + def rets = [] + rets << ["城市","建筑年代覆盖率","物业类型覆盖率","建筑类型覆盖率","区域最高小区均价覆盖率","区域最低小区均价覆盖率"] + for(def c : citys){ + def obj = findCoverPrecent(c) + if(obj){ + def datePrecent = DataHandleUtils.getDoubleRound(2, obj.district_detail.finishBuildingDate*100) + def estateTypePrecent = DataHandleUtils.getDoubleRound(2, obj.district_detail.estateType2*100) + def buildTypePrecent = DataHandleUtils.getDoubleRound(2, obj.district_detail.buildType*100) + rets << [c, datePrecent+"%", estateTypePrecent+"%", buildTypePrecent+"%", "100%", "100%"] + } + } + + ExcelUtils.write("d:\\数据\\92个城市覆盖率统计.xlsx", rets, "统计") + + } + + def findCoverPrecent(def city){ + def q = new BasicDBObject() + q.put("city", city) + def dbcursor = CITY_INFO_CONN.find(q).sort(new BasicDBObject("weekend", -1)).limit(1) + def obj = null + for(def d : dbcursor){ + obj = d + } + return obj + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/ExportData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/ExportData.groovy index fd2856af7..6a44d4139 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/ExportData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/ExportData.groovy @@ -24,8 +24,8 @@ class ExportData { static main(args) { def task = new ExportData(); -// task.exportThird() - task.exportMap(); + task.exportThird() +// task.exportMap(); } def exportMap(){ @@ -71,7 +71,7 @@ class ExportData { def exportThird(){ - def citys = ["淮安","昆山","连云港","南京","南通","宿迁","苏州","泰州","无锡","徐州","盐城","扬州","镇江"] + def citys = ["重庆"] for(def city : citys){ def rets = [] rets << ["城市","区域","名称","别名"] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy index 033464eb5..a131a414c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy @@ -3,6 +3,7 @@ package com.fangjia.script.gj.district import com.fangjia.core.util.HttpRequest import net.sf.json.JSONArray import net.sf.json.JSONObject +import org.apache.commons.lang.StringUtils class FangjiaClientUtils { @@ -48,4 +49,34 @@ class FangjiaClientUtils { println content } + + + public static def searchMapData(def city, def keyword, def types, def mapCode){ + def jsonObj = null + try { + def url = "http://192.168.0.19:38080/FangJiaDataService/search?city="+city+"&keyword="+keyword+"&types="+types+"&mapCode="+mapCode+"&limit=10" + def content = new HttpRequest().newGet(url, null, "utf-8") + // def ret = content.replace("[","").replace("]","") + jsonObj = JSONArray.fromObject(content) + } catch (Exception e) { + return null; + } + return jsonObj; + } + + + public static Map JsonToMap(JSONObject json) { + Map columnValMap = new HashMap(); + Set jsonKeys = json.keySet(); + for (Object key : jsonKeys) { + Object JsonValObj = json.get(key); + if (key instanceof JSONObject) { + columnValMap.put((String) key, JsonToMap((JSONObject) JsonValObj)); + } else { + columnValMap.put((String) key, JsonValObj); + } + } + return columnValMap; + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CrawlDistrictIdSelect.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CrawlDistrictIdSelect.groovy index 8e8bba001..b26bd079b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CrawlDistrictIdSelect.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CrawlDistrictIdSelect.groovy @@ -4,6 +4,7 @@ import com.fangjia.core.RegionData import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject +import com.mongodb.Bytes import com.mongodb.DBCollection class CrawlDistrictIdSelect { @@ -14,8 +15,8 @@ class CrawlDistrictIdSelect { static main(args) { def task = new CrawlDistrictIdSelect() - task.run() -// task.synch() +// task.run() + task.synch() } def synch(){ @@ -50,13 +51,13 @@ class CrawlDistrictIdSelect { def findCrawlData(){ def q = new BasicDBObject() - q.put("city","福州") - q.put("source", "安居客") - q.put("s_date",new BasicDBObject('$gte', 202001)) - def dbcursor = thirdPartyColl.find(q) +// q.put("city","福州") + q.put("source", "诸葛找房") + q.put("s_date",202010) + def dbcursor = thirdPartyColl.find(q).batchSize(10000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT) def retMap = [:] - def prefix = "/community/view/" - def keyList =[] + def prefix = "xiaoqu.zhuge.com/" + def keyList =new HashSet() for(def db : dbcursor){ def url = null if(!db.url){ @@ -67,12 +68,11 @@ class CrawlDistrictIdSelect { url = db.url } if(url && url.contains(prefix) ){ - def district_id = url.substring(url.indexOf(prefix)+prefix.length(),url.size()) + def district_id = url.substring(url.indexOf(prefix)+prefix.length(),url.size()-1) if(district_id=="0"){ continue } - def shortCity = url.substring(url.indexOf("https://")+8, url.indexOf(".anjuke.com")) - def region = RegionData.formatRegion(db.city, db.region) + def shortCity = url.substring(url.indexOf("//")+2, url.indexOf(".xiaoqu.zhuge.com")) def key = shortCity +"#"+district_id def ids = retMap.get(key) @@ -83,15 +83,17 @@ class CrawlDistrictIdSelect { if(!keyList.contains(key)){ def map = new BasicDBObject() - map.root = "anjuke.com" + map.root = "zhuge.com" map.city_short= shortCity map.city = db.city - map.region = region + map.region = db.region map.district_id = district_id map.source_url = url ids << map keyList << key + }else{ + println "重复:"+key } }else{ println url +" 链接不合要求" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/looplineinfo/InsidePolygon.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/looplineinfo/InsidePolygon.groovy index f01c92679..5a8586fb8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/looplineinfo/InsidePolygon.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/looplineinfo/InsidePolygon.groovy @@ -32,6 +32,22 @@ class InsidePolygon { return list; } + + + public static List createPoint2(String points){ + String[] array = points.split(";"); + List list = new ArrayList(); + for(int i =0 ;i> getPoints(String points){ String[] array = points.split("-"); @@ -167,4 +183,44 @@ class InsidePolygon { return false; return true; } + + + + public static boolean isPtInPoly (double ALon , double ALat , List ps) { + int iSum, iCount, iIndex; + double dLon1 = 0, dLon2 = 0, dLat1 = 0, dLat2 = 0, dLon; + if (ps.size() < 3) { + return false; + } + iSum = 0; + iCount = ps.size(); + for (iIndex = 0; iIndex= dLat1) && (ALat < dLat2)) || ((ALat >= dLat2) && (ALat < dLat1))) { + if (Math.abs(dLat1 - dLat2) > 0) { + //得到 A点向左射线与边的交点的x坐标: + dLon = dLon1 - ((dLon1 - dLon2) * (dLat1 - ALat) ) / (dLat1 - dLat2); + // 如果交点在A点左侧(说明是做射线与 边的交点),则射线与边的全部交点数加一: + if (dLon < ALon) { + iSum++; + } + } + } + } + if ((iSum % 2) != 0) { + return true; + } + return false; + } }