This commit is contained in:
2021-06-04 11:31:26 +08:00
parent cbbd463d83
commit d667012ec9
7 changed files with 233 additions and 47 deletions
@@ -211,23 +211,35 @@ class AccessLogBatch extends BaseBatch{
}
def read(){
String fileName = "C:\\Users\\gaojuan\\Desktop\\localhost_access_log.2019-06-01.txt";
String fileName = "D:\\数据\\logs\\performance.open.log";
File file = new File(fileName);
def rets = []
file.eachLine{
def index = 0
def list =[]
def slowCount = 0
file.eachLine{it->
if(it.contains("/fangjiatong2/")){
try{
it = it.replace(it, URLDecoder.decode(URLDecoder.decode(it, "utf-8"), "UTF-8"))
}catch(IllegalArgumentException e){}
if(it.contains("ws/czbank") && it.contains("entity:House")){
// println it
index++
// try{
def s1 = it.substring(it.indexOf("time:")+5, it.length())
s1 = s1.substring(0, s1.indexOf("\t"))
list << s1.toInteger()
if(s1.isNumber() && s1.toInteger()>=4000){
println it
slowCount++
}
// }catch(IllegalArgumentException e){}
}
if(!it.contains("漕溪四村")){
rets << it
}
}
writeTxt(rets, "C:\\Users\\gaojuan\\Desktop\\fjt-2020-6-1.txt");
println list.sum()/list.size()
println index
println "slowCount:"+slowCount
// writeTxt(rets, "C:\\Users\\gaojuan\\Desktop\\fjt-2020-6-1.txt");
}
}
@@ -43,6 +43,24 @@ class OpenApiUtils {
}
static def matchDetailDistrict(def city, def address, def suggestDistrict, def preciseLevelflag){
address = address.replace("(","(").replace(")",")").replace(" ","")
def url = "http://open.fangjia.com/address/matchV2?city="+city+"&address="+address+"&suggestDistrict="+suggestDistrict+"&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB&category=district&__refresh_cache=1"
if(preciseLevelflag){
url = url +"&preciseLevel=strict"
}
String content = new HttpRequest().get(url, "utf-8");
if(content){
def doc = new JsonSlurper().parseText(content)
if(doc && doc.code ==200 && doc.result){
def dto = doc.result
return dto
}
}
return null;
}
static def matchDistrict2(def city, def address, def preciseLevelflag){
address = address.replace("(","(").replace(")",")").replace(" ","").replace("#","")
def url = "http://open.fangjia.com/address/matchV2?city="+city+"&address="+address+"&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB&category=district&__refresh_cache=1"
@@ -1,39 +1,106 @@
package com.fangjia.script.gj.data
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
class FieldStat {
final static DBCollection CITY_INFO_CONN = MongoUtils.getCollectionByName("fangjia_data_ana", "city_info", MongoUtils.MongoDB_Domain_mongo);
static main(args) {
def task = new FieldStat();
task.run()
// task.run()
task.run1()
// task.run2()
}
def run1(){
def citys = ["北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
def dist_total = 0
def addr_alias_total = 0
def rets = []
rets << ["城市","别名总量","有别名小区数量","小区总量","有别名小区占比"]
for(def city : citys){
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "alias", "addr_alias"]);
def alias_index = 0
def alias_total = 0
for(def d : districts){
def alias = d.alias
if(alias){
alias_index++
alias_total = alias_total + alias.size()
}
}
def precent = DataHandleUtils.getDoubleRound(2, alias_index*100.0/districts.size())
rets << [city, alias_total, alias_index, districts.size(), precent+"%"]
}
ExcelUtils.write("d:\\数据\\92个城市别名统计.xlsx", rets, "统计")
// println dist_total
// println alias_total
// println addr_alias_total
}
def run(){
ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\½¨ÐÐÊý¾ÝÈںϽá¹ûͳ¼Æ_0425.xlsx", { row ->
def city = row['A']
ExcelUtils.multiThreadingModify("D:\\数据\\城市统计.xlsx", { row ->
def city = row['B']
if(city){
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "alias"]);
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]);
def total = districts.size()
row['D'] = total
def aliasIndex = 0
def aliasTotalCount = 0
for(def d : districts){
def alias = d.alias
if(alias){
aliasIndex++
aliasTotalCount = aliasTotalCount + alias.size()
}
}
// row['J'] = aliasTotalCount
// row['K'] = aliasIndex
def p = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, aliasIndex*100/total)
row['B'] = p+"%"
// row['M'] = total
println city +" "+p
// def aliasIndex = 0
// def aliasTotalCount = 0
// for(def d : districts){
// def alias = d.alias
// if(alias){
// aliasIndex++
// aliasTotalCount = aliasTotalCount + alias.size()
// }
// }
//// row['J'] = aliasTotalCount
//// row['K'] = aliasIndex
// def p = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, aliasIndex*100/total)
// row['B'] = p+"%"
//// row['M'] = total
// println city +" "+p
}
}, 1);
}, 0);
println "finish"
}
def run2(){
def citys = ["北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
def rets = []
rets << ["城市","建筑年代覆盖率","物业类型覆盖率","建筑类型覆盖率","区域最高小区均价覆盖率","区域最低小区均价覆盖率"]
for(def c : citys){
def obj = findCoverPrecent(c)
if(obj){
def datePrecent = DataHandleUtils.getDoubleRound(2, obj.district_detail.finishBuildingDate*100)
def estateTypePrecent = DataHandleUtils.getDoubleRound(2, obj.district_detail.estateType2*100)
def buildTypePrecent = DataHandleUtils.getDoubleRound(2, obj.district_detail.buildType*100)
rets << [c, datePrecent+"%", estateTypePrecent+"%", buildTypePrecent+"%", "100%", "100%"]
}
}
ExcelUtils.write("d:\\数据\\92个城市覆盖率统计.xlsx", rets, "统计")
}
def findCoverPrecent(def city){
def q = new BasicDBObject()
q.put("city", city)
def dbcursor = CITY_INFO_CONN.find(q).sort(new BasicDBObject("weekend", -1)).limit(1)
def obj = null
for(def d : dbcursor){
obj = d
}
return obj
}
}
@@ -24,8 +24,8 @@ class ExportData {
static main(args) {
def task = new ExportData();
// task.exportThird()
task.exportMap();
task.exportThird()
// task.exportMap();
}
def exportMap(){
@@ -71,7 +71,7 @@ class ExportData {
def exportThird(){
def citys = ["淮安","昆山","连云港","南京","南通","宿迁","苏州","泰州","无锡","徐州","盐城","扬州","镇江"]
def citys = ["重庆"]
for(def city : citys){
def rets = []
rets << ["城市","区域","名称","别名"]
@@ -3,6 +3,7 @@ package com.fangjia.script.gj.district
import com.fangjia.core.util.HttpRequest
import net.sf.json.JSONArray
import net.sf.json.JSONObject
import org.apache.commons.lang.StringUtils
class FangjiaClientUtils {
@@ -48,4 +49,34 @@ class FangjiaClientUtils {
println content
}
public static def searchMapData(def city, def keyword, def types, def mapCode){
def jsonObj = null
try {
def url = "http://192.168.0.19:38080/FangJiaDataService/search?city="+city+"&keyword="+keyword+"&types="+types+"&mapCode="+mapCode+"&limit=10"
def content = new HttpRequest().newGet(url, null, "utf-8")
// def ret = content.replace("[","").replace("]","")
jsonObj = JSONArray.fromObject(content)
} catch (Exception e) {
return null;
}
return jsonObj;
}
public static Map<String, Object> JsonToMap(JSONObject json) {
Map<String, Object> columnValMap = new HashMap<String, Object>();
Set<Object> jsonKeys = json.keySet();
for (Object key : jsonKeys) {
Object JsonValObj = json.get(key);
if (key instanceof JSONObject) {
columnValMap.put((String) key, JsonToMap((JSONObject) JsonValObj));
} else {
columnValMap.put((String) key, JsonValObj);
}
}
return columnValMap;
}
}
@@ -4,6 +4,7 @@ import com.fangjia.core.RegionData
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.Bytes
import com.mongodb.DBCollection
class CrawlDistrictIdSelect {
@@ -14,8 +15,8 @@ class CrawlDistrictIdSelect {
static main(args) {
def task = new CrawlDistrictIdSelect()
task.run()
// task.synch()
// task.run()
task.synch()
}
def synch(){
@@ -50,13 +51,13 @@ class CrawlDistrictIdSelect {
def findCrawlData(){
def q = new BasicDBObject()
q.put("city","福州")
q.put("source", "安居客")
q.put("s_date",new BasicDBObject('$gte', 202001))
def dbcursor = thirdPartyColl.find(q)
// q.put("city","福州")
q.put("source", "诸葛找房")
q.put("s_date",202010)
def dbcursor = thirdPartyColl.find(q).batchSize(10000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def retMap = [:]
def prefix = "/community/view/"
def keyList =[]
def prefix = "xiaoqu.zhuge.com/"
def keyList =new HashSet()
for(def db : dbcursor){
def url = null
if(!db.url){
@@ -67,12 +68,11 @@ class CrawlDistrictIdSelect {
url = db.url
}
if(url && url.contains(prefix) ){
def district_id = url.substring(url.indexOf(prefix)+prefix.length(),url.size())
def district_id = url.substring(url.indexOf(prefix)+prefix.length(),url.size()-1)
if(district_id=="0"){
continue
}
def shortCity = url.substring(url.indexOf("https://")+8, url.indexOf(".anjuke.com"))
def region = RegionData.formatRegion(db.city, db.region)
def shortCity = url.substring(url.indexOf("//")+2, url.indexOf(".xiaoqu.zhuge.com"))
def key = shortCity +"#"+district_id
def ids = retMap.get(key)
@@ -83,15 +83,17 @@ class CrawlDistrictIdSelect {
if(!keyList.contains(key)){
def map = new BasicDBObject()
map.root = "anjuke.com"
map.root = "zhuge.com"
map.city_short= shortCity
map.city = db.city
map.region = region
map.region = db.region
map.district_id = district_id
map.source_url = url
ids << map
keyList << key
}else{
println "重复:"+key
}
}else{
println url +" 链接不合要求"
@@ -32,6 +32,22 @@ class InsidePolygon {
return list;
}
public static List<Point> createPoint2(String points){
String[] array = points.split(";");
List<Point> list = new ArrayList<Point>();
for(int i =0 ;i<array.length-1;i++){
def pointArrs = array[i].split("[,|\\|]")
Point point = new Point();
double x = Double.parseDouble(pointArrs[0]);
double y = Double.parseDouble(pointArrs[1]);
point.setLon(x);
point.setLat(y);
list.add(point);
}
return list;
}
public static List<Map<String, Object>> getPoints(String points){
String[] array = points.split("-");
@@ -167,4 +183,44 @@ class InsidePolygon {
return false;
return true;
}
public static boolean isPtInPoly (double ALon , double ALat , List<Point> ps) {
int iSum, iCount, iIndex;
double dLon1 = 0, dLon2 = 0, dLat1 = 0, dLat2 = 0, dLon;
if (ps.size() < 3) {
return false;
}
iSum = 0;
iCount = ps.size();
for (iIndex = 0; iIndex<iCount;iIndex++) {
if (iIndex == iCount - 1) {
dLon1 = ps[iIndex].getLon()
dLat1 = ps[iIndex].getLat()
dLon2 = ps[0].getLon()
dLat2 = ps[0].getLat()
} else {
dLon1 = ps[iIndex].getLon()
dLat1 = ps[iIndex].getLat()
dLon2 = ps[iIndex + 1].getLon()
dLat2 = ps[iIndex + 1].getLat()
}
// 以下语句判断A点是否在边的两端点的水平平行线之间,在则可能有交点,开始判断交点是否在左射线上
if (((ALat >= dLat1) && (ALat < dLat2)) || ((ALat >= dLat2) && (ALat < dLat1))) {
if (Math.abs(dLat1 - dLat2) > 0) {
//得到 A点向左射线与边的交点的x坐标:
dLon = dLon1 - ((dLon1 - dLon2) * (dLat1 - ALat) ) / (dLat1 - dLat2);
// 如果交点在A点左侧(说明是做射线与 边的交点),则射线与边的全部交点数加一:
if (dLon < ALon) {
iSum++;
}
}
}
}
if ((iSum % 2) != 0) {
return true;
}
return false;
}
}