update
This commit is contained in:
+23
-11
@@ -211,23 +211,35 @@ class AccessLogBatch extends BaseBatch{
|
||||
}
|
||||
|
||||
def read(){
|
||||
String fileName = "C:\\Users\\gaojuan\\Desktop\\localhost_access_log.2019-06-01.txt";
|
||||
String fileName = "D:\\数据\\logs\\performance.open.log";
|
||||
File file = new File(fileName);
|
||||
def rets = []
|
||||
file.eachLine{
|
||||
def index = 0
|
||||
def list =[]
|
||||
def slowCount = 0
|
||||
file.eachLine{it->
|
||||
|
||||
if(it.contains("/fangjiatong2/")){
|
||||
try{
|
||||
it = it.replace(it, URLDecoder.decode(URLDecoder.decode(it, "utf-8"), "UTF-8"))
|
||||
}catch(IllegalArgumentException e){}
|
||||
|
||||
if(it.contains("ws/czbank") && it.contains("entity:House")){
|
||||
// println it
|
||||
index++
|
||||
// try{
|
||||
def s1 = it.substring(it.indexOf("time:")+5, it.length())
|
||||
s1 = s1.substring(0, s1.indexOf("\t"))
|
||||
list << s1.toInteger()
|
||||
if(s1.isNumber() && s1.toInteger()>=4000){
|
||||
println it
|
||||
slowCount++
|
||||
}
|
||||
// }catch(IllegalArgumentException e){}
|
||||
}
|
||||
|
||||
if(!it.contains("漕溪四村")){
|
||||
rets << it
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
writeTxt(rets, "C:\\Users\\gaojuan\\Desktop\\fjt-2020-6-1.txt");
|
||||
println list.sum()/list.size()
|
||||
println index
|
||||
println "slowCount:"+slowCount
|
||||
// writeTxt(rets, "C:\\Users\\gaojuan\\Desktop\\fjt-2020-6-1.txt");
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -43,6 +43,24 @@ class OpenApiUtils {
|
||||
}
|
||||
|
||||
|
||||
static def matchDetailDistrict(def city, def address, def suggestDistrict, def preciseLevelflag){
|
||||
address = address.replace("(","(").replace(")",")").replace(" ","")
|
||||
def url = "http://open.fangjia.com/address/matchV2?city="+city+"&address="+address+"&suggestDistrict="+suggestDistrict+"&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB&category=district&__refresh_cache=1"
|
||||
if(preciseLevelflag){
|
||||
url = url +"&preciseLevel=strict"
|
||||
}
|
||||
String content = new HttpRequest().get(url, "utf-8");
|
||||
if(content){
|
||||
def doc = new JsonSlurper().parseText(content)
|
||||
if(doc && doc.code ==200 && doc.result){
|
||||
def dto = doc.result
|
||||
return dto
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
static def matchDistrict2(def city, def address, def preciseLevelflag){
|
||||
address = address.replace("(","(").replace(")",")").replace(" ","").replace("#","")
|
||||
def url = "http://open.fangjia.com/address/matchV2?city="+city+"&address="+address+"&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB&category=district&__refresh_cache=1"
|
||||
|
||||
+87
-20
@@ -1,39 +1,106 @@
|
||||
package com.fangjia.script.gj.data
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
class FieldStat {
|
||||
|
||||
final static DBCollection CITY_INFO_CONN = MongoUtils.getCollectionByName("fangjia_data_ana", "city_info", MongoUtils.MongoDB_Domain_mongo);
|
||||
|
||||
static main(args) {
|
||||
def task = new FieldStat();
|
||||
task.run()
|
||||
// task.run()
|
||||
task.run1()
|
||||
// task.run2()
|
||||
}
|
||||
|
||||
def run1(){
|
||||
def citys = ["北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
|
||||
def dist_total = 0
|
||||
|
||||
def addr_alias_total = 0
|
||||
def rets = []
|
||||
rets << ["城市","别名总量","有别名小区数量","小区总量","有别名小区占比"]
|
||||
for(def city : citys){
|
||||
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "alias", "addr_alias"]);
|
||||
def alias_index = 0
|
||||
def alias_total = 0
|
||||
|
||||
for(def d : districts){
|
||||
def alias = d.alias
|
||||
if(alias){
|
||||
alias_index++
|
||||
alias_total = alias_total + alias.size()
|
||||
}
|
||||
}
|
||||
def precent = DataHandleUtils.getDoubleRound(2, alias_index*100.0/districts.size())
|
||||
rets << [city, alias_total, alias_index, districts.size(), precent+"%"]
|
||||
|
||||
}
|
||||
ExcelUtils.write("d:\\数据\\92个城市别名统计.xlsx", rets, "统计")
|
||||
// println dist_total
|
||||
// println alias_total
|
||||
// println addr_alias_total
|
||||
}
|
||||
|
||||
def run(){
|
||||
ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\½¨ÐÐÊý¾ÝÈںϽá¹ûͳ¼Æ_0425.xlsx", { row ->
|
||||
def city = row['A']
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\城市统计.xlsx", { row ->
|
||||
def city = row['B']
|
||||
if(city){
|
||||
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "alias"]);
|
||||
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]);
|
||||
def total = districts.size()
|
||||
row['D'] = total
|
||||
|
||||
def aliasIndex = 0
|
||||
def aliasTotalCount = 0
|
||||
for(def d : districts){
|
||||
def alias = d.alias
|
||||
if(alias){
|
||||
aliasIndex++
|
||||
aliasTotalCount = aliasTotalCount + alias.size()
|
||||
}
|
||||
}
|
||||
// row['J'] = aliasTotalCount
|
||||
// row['K'] = aliasIndex
|
||||
def p = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, aliasIndex*100/total)
|
||||
row['B'] = p+"%"
|
||||
// row['M'] = total
|
||||
println city +" "+p
|
||||
// def aliasIndex = 0
|
||||
// def aliasTotalCount = 0
|
||||
// for(def d : districts){
|
||||
// def alias = d.alias
|
||||
// if(alias){
|
||||
// aliasIndex++
|
||||
// aliasTotalCount = aliasTotalCount + alias.size()
|
||||
// }
|
||||
// }
|
||||
//// row['J'] = aliasTotalCount
|
||||
//// row['K'] = aliasIndex
|
||||
// def p = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, aliasIndex*100/total)
|
||||
// row['B'] = p+"%"
|
||||
//// row['M'] = total
|
||||
// println city +" "+p
|
||||
}
|
||||
}, 1);
|
||||
}, 0);
|
||||
println "finish"
|
||||
}
|
||||
|
||||
def run2(){
|
||||
def citys = ["北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
|
||||
def rets = []
|
||||
rets << ["城市","建筑年代覆盖率","物业类型覆盖率","建筑类型覆盖率","区域最高小区均价覆盖率","区域最低小区均价覆盖率"]
|
||||
for(def c : citys){
|
||||
def obj = findCoverPrecent(c)
|
||||
if(obj){
|
||||
def datePrecent = DataHandleUtils.getDoubleRound(2, obj.district_detail.finishBuildingDate*100)
|
||||
def estateTypePrecent = DataHandleUtils.getDoubleRound(2, obj.district_detail.estateType2*100)
|
||||
def buildTypePrecent = DataHandleUtils.getDoubleRound(2, obj.district_detail.buildType*100)
|
||||
rets << [c, datePrecent+"%", estateTypePrecent+"%", buildTypePrecent+"%", "100%", "100%"]
|
||||
}
|
||||
}
|
||||
|
||||
ExcelUtils.write("d:\\数据\\92个城市覆盖率统计.xlsx", rets, "统计")
|
||||
|
||||
}
|
||||
|
||||
def findCoverPrecent(def city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
def dbcursor = CITY_INFO_CONN.find(q).sort(new BasicDBObject("weekend", -1)).limit(1)
|
||||
def obj = null
|
||||
for(def d : dbcursor){
|
||||
obj = d
|
||||
}
|
||||
return obj
|
||||
}
|
||||
}
|
||||
|
||||
+3
-3
@@ -24,8 +24,8 @@ class ExportData {
|
||||
|
||||
static main(args) {
|
||||
def task = new ExportData();
|
||||
// task.exportThird()
|
||||
task.exportMap();
|
||||
task.exportThird()
|
||||
// task.exportMap();
|
||||
}
|
||||
|
||||
def exportMap(){
|
||||
@@ -71,7 +71,7 @@ class ExportData {
|
||||
|
||||
|
||||
def exportThird(){
|
||||
def citys = ["淮安","昆山","连云港","南京","南通","宿迁","苏州","泰州","无锡","徐州","盐城","扬州","镇江"]
|
||||
def citys = ["重庆"]
|
||||
for(def city : citys){
|
||||
def rets = []
|
||||
rets << ["城市","区域","名称","别名"]
|
||||
|
||||
+31
@@ -3,6 +3,7 @@ package com.fangjia.script.gj.district
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import net.sf.json.JSONArray
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.lang.StringUtils
|
||||
|
||||
class FangjiaClientUtils {
|
||||
|
||||
@@ -48,4 +49,34 @@ class FangjiaClientUtils {
|
||||
|
||||
println content
|
||||
}
|
||||
|
||||
|
||||
public static def searchMapData(def city, def keyword, def types, def mapCode){
|
||||
def jsonObj = null
|
||||
try {
|
||||
def url = "http://192.168.0.19:38080/FangJiaDataService/search?city="+city+"&keyword="+keyword+"&types="+types+"&mapCode="+mapCode+"&limit=10"
|
||||
def content = new HttpRequest().newGet(url, null, "utf-8")
|
||||
// def ret = content.replace("[","").replace("]","")
|
||||
jsonObj = JSONArray.fromObject(content)
|
||||
} catch (Exception e) {
|
||||
return null;
|
||||
}
|
||||
return jsonObj;
|
||||
}
|
||||
|
||||
|
||||
public static Map<String, Object> JsonToMap(JSONObject json) {
|
||||
Map<String, Object> columnValMap = new HashMap<String, Object>();
|
||||
Set<Object> jsonKeys = json.keySet();
|
||||
for (Object key : jsonKeys) {
|
||||
Object JsonValObj = json.get(key);
|
||||
if (key instanceof JSONObject) {
|
||||
columnValMap.put((String) key, JsonToMap((JSONObject) JsonValObj));
|
||||
} else {
|
||||
columnValMap.put((String) key, JsonValObj);
|
||||
}
|
||||
}
|
||||
return columnValMap;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+15
-13
@@ -4,6 +4,7 @@ import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.Bytes
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
class CrawlDistrictIdSelect {
|
||||
@@ -14,8 +15,8 @@ class CrawlDistrictIdSelect {
|
||||
|
||||
static main(args) {
|
||||
def task = new CrawlDistrictIdSelect()
|
||||
task.run()
|
||||
// task.synch()
|
||||
// task.run()
|
||||
task.synch()
|
||||
}
|
||||
|
||||
def synch(){
|
||||
@@ -50,13 +51,13 @@ class CrawlDistrictIdSelect {
|
||||
|
||||
def findCrawlData(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city","福州")
|
||||
q.put("source", "安居客")
|
||||
q.put("s_date",new BasicDBObject('$gte', 202001))
|
||||
def dbcursor = thirdPartyColl.find(q)
|
||||
// q.put("city","福州")
|
||||
q.put("source", "诸葛找房")
|
||||
q.put("s_date",202010)
|
||||
def dbcursor = thirdPartyColl.find(q).batchSize(10000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def retMap = [:]
|
||||
def prefix = "/community/view/"
|
||||
def keyList =[]
|
||||
def prefix = "xiaoqu.zhuge.com/"
|
||||
def keyList =new HashSet()
|
||||
for(def db : dbcursor){
|
||||
def url = null
|
||||
if(!db.url){
|
||||
@@ -67,12 +68,11 @@ class CrawlDistrictIdSelect {
|
||||
url = db.url
|
||||
}
|
||||
if(url && url.contains(prefix) ){
|
||||
def district_id = url.substring(url.indexOf(prefix)+prefix.length(),url.size())
|
||||
def district_id = url.substring(url.indexOf(prefix)+prefix.length(),url.size()-1)
|
||||
if(district_id=="0"){
|
||||
continue
|
||||
}
|
||||
def shortCity = url.substring(url.indexOf("https://")+8, url.indexOf(".anjuke.com"))
|
||||
def region = RegionData.formatRegion(db.city, db.region)
|
||||
def shortCity = url.substring(url.indexOf("//")+2, url.indexOf(".xiaoqu.zhuge.com"))
|
||||
def key = shortCity +"#"+district_id
|
||||
|
||||
def ids = retMap.get(key)
|
||||
@@ -83,15 +83,17 @@ class CrawlDistrictIdSelect {
|
||||
|
||||
if(!keyList.contains(key)){
|
||||
def map = new BasicDBObject()
|
||||
map.root = "anjuke.com"
|
||||
map.root = "zhuge.com"
|
||||
map.city_short= shortCity
|
||||
map.city = db.city
|
||||
map.region = region
|
||||
map.region = db.region
|
||||
map.district_id = district_id
|
||||
map.source_url = url
|
||||
ids << map
|
||||
|
||||
keyList << key
|
||||
}else{
|
||||
println "重复:"+key
|
||||
}
|
||||
}else{
|
||||
println url +" 链接不合要求"
|
||||
|
||||
+56
@@ -32,6 +32,22 @@ class InsidePolygon {
|
||||
|
||||
return list;
|
||||
}
|
||||
|
||||
|
||||
public static List<Point> createPoint2(String points){
|
||||
String[] array = points.split(";");
|
||||
List<Point> list = new ArrayList<Point>();
|
||||
for(int i =0 ;i<array.length-1;i++){
|
||||
def pointArrs = array[i].split("[,|\\|]")
|
||||
Point point = new Point();
|
||||
double x = Double.parseDouble(pointArrs[0]);
|
||||
double y = Double.parseDouble(pointArrs[1]);
|
||||
point.setLon(x);
|
||||
point.setLat(y);
|
||||
list.add(point);
|
||||
}
|
||||
return list;
|
||||
}
|
||||
|
||||
public static List<Map<String, Object>> getPoints(String points){
|
||||
String[] array = points.split("-");
|
||||
@@ -167,4 +183,44 @@ class InsidePolygon {
|
||||
return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
|
||||
public static boolean isPtInPoly (double ALon , double ALat , List<Point> ps) {
|
||||
int iSum, iCount, iIndex;
|
||||
double dLon1 = 0, dLon2 = 0, dLat1 = 0, dLat2 = 0, dLon;
|
||||
if (ps.size() < 3) {
|
||||
return false;
|
||||
}
|
||||
iSum = 0;
|
||||
iCount = ps.size();
|
||||
for (iIndex = 0; iIndex<iCount;iIndex++) {
|
||||
if (iIndex == iCount - 1) {
|
||||
dLon1 = ps[iIndex].getLon()
|
||||
dLat1 = ps[iIndex].getLat()
|
||||
dLon2 = ps[0].getLon()
|
||||
dLat2 = ps[0].getLat()
|
||||
} else {
|
||||
dLon1 = ps[iIndex].getLon()
|
||||
dLat1 = ps[iIndex].getLat()
|
||||
dLon2 = ps[iIndex + 1].getLon()
|
||||
dLat2 = ps[iIndex + 1].getLat()
|
||||
}
|
||||
// 以下语句判断A点是否在边的两端点的水平平行线之间,在则可能有交点,开始判断交点是否在左射线上
|
||||
if (((ALat >= dLat1) && (ALat < dLat2)) || ((ALat >= dLat2) && (ALat < dLat1))) {
|
||||
if (Math.abs(dLat1 - dLat2) > 0) {
|
||||
//得到 A点向左射线与边的交点的x坐标:
|
||||
dLon = dLon1 - ((dLon1 - dLon2) * (dLat1 - ALat) ) / (dLat1 - dLat2);
|
||||
// 如果交点在A点左侧(说明是做射线与 边的交点),则射线与边的全部交点数加一:
|
||||
if (dLon < ALon) {
|
||||
iSum++;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if ((iSum % 2) != 0) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user