320 lines
9.1 KiB
Python
320 lines
9.1 KiB
Python
import pandas
|
|
import time
|
|
import logging
|
|
import config
|
|
import pytz
|
|
import datetime
|
|
import calendar
|
|
import requests
|
|
import json
|
|
from fangjia import mongoClients as mongoClients
|
|
from fangjia import ElasticsearchClient as es
|
|
|
|
formatter = logging.Formatter('%(processName)s %(asctime)s %(name)-6s %(levelname)-6s %(message)s')
|
|
|
|
|
|
def get_logger(name):
|
|
logger = logging.getLogger(name)
|
|
handler = logging.StreamHandler()
|
|
handler.setFormatter(formatter)
|
|
logger.addHandler(handler)
|
|
|
|
file_handler = logging.FileHandler("/home/goojia/logs/" + name + ".log")
|
|
file_handler.setFormatter(formatter)
|
|
logger.addHandler(file_handler)
|
|
logger.setLevel("INFO")
|
|
|
|
return logger
|
|
|
|
|
|
def get_performance_logger(name):
|
|
logger = logging.getLogger(name)
|
|
handler = logging.StreamHandler()
|
|
handler.setFormatter(formatter)
|
|
logger.addHandler(handler)
|
|
|
|
file_handler = logging.FileHandler("/home/goojia/logs/" + name + ".performance.log")
|
|
file_handler.setFormatter(formatter)
|
|
logger.addHandler(file_handler)
|
|
logger.setLevel("INFO")
|
|
|
|
return logger
|
|
|
|
|
|
performance_logger = get_performance_logger("performance")
|
|
|
|
|
|
def timer(func):
|
|
def _timer(*args, **kw):
|
|
begin = time.clock()
|
|
rs = func(*args, **kw)
|
|
end = time.clock()
|
|
elapsed = end - begin
|
|
if elapsed > config.get_timeout():
|
|
arg_list = list()
|
|
for arg in args:
|
|
arg_list.append(str(arg))
|
|
performance_logger.info("函数名:%-20s\t耗时:%7.4f\t\t参数:(%s)" % (func.__name__, end - begin, arg_list))
|
|
return rs
|
|
|
|
return _timer
|
|
|
|
|
|
def get_collection(db, name, host="192.168.0.136", port=27017):
|
|
return mongoClients.MongoClients(db, name, host, port).collection
|
|
|
|
|
|
def get_seawater_index_name(city, region):
|
|
return es.get_index_name(city, region)
|
|
|
|
|
|
def get_all_city():
|
|
server_area = get_collection("fangjia", "server_area", config.get_mongodb_host(), config.get_mongodb_port())
|
|
city_list = []
|
|
for area in server_area.find().sort([("city_grade", 1), ("sequence", 1)]):
|
|
if "display" in area and area["display"] != "全国":
|
|
city_list.append(area["display"])
|
|
return city_list
|
|
|
|
|
|
def get_region(city):
|
|
region_block = get_collection("fangjia", "region_block", config.get_mongodb_host(), config.get_mongodb_port())
|
|
return [region["name"] for region in region_block.find({"category": "region", "city": city}, {"name": 1})]
|
|
|
|
|
|
def get_weekend():
|
|
now = datetime.datetime.now().replace(hour=0, minute=0, second=0, microsecond=0)
|
|
for day in range(1, 9):
|
|
if calendar.weekday(now.year, now.month, now.day) == 6:
|
|
break
|
|
now = now + datetime.timedelta(days=1)
|
|
pacific = pytz.timezone('Asia/Shanghai')
|
|
return pacific.localize(now)
|
|
|
|
|
|
def send_ding_msg(msg):
|
|
headers = {
|
|
'Content-Type': "application/json",
|
|
}
|
|
url = "https://oapi.dingtalk.com/robot/send" \
|
|
"?access_token=2307d2fec8314b3e8b4b83dd1d42da2e2051638e3e668f21e77ca01258bf682c"
|
|
|
|
text_body = {
|
|
"msgtype": "text",
|
|
"text": {
|
|
"content": msg
|
|
}
|
|
|
|
}
|
|
requests.request("POST", url, data=json.dumps(text_body, ensure_ascii=False).encode('utf8'), headers=headers)
|
|
|
|
|
|
@timer
|
|
def findHouse(query, fields=None):
|
|
seawater105 = get_collection("fangjia", "seawater", config.get_seawater_host(), 27777)
|
|
for err_count in range(0, 10):
|
|
try:
|
|
lt = 1000
|
|
# seawater.find(query,{"p_date":-1}).sort([("p_date",-1)])
|
|
pos = list(seawater105.find(query, fields).limit(lt))
|
|
data = pandas.DataFrame(pos)
|
|
del pos
|
|
# print("==================>", time.time() - s)
|
|
return data
|
|
except:
|
|
lt = seawater105.count(query)
|
|
pos = list(seawater105.find(query).limit(lt))
|
|
data = pandas.DataFrame(pos)
|
|
del pos
|
|
return data
|
|
|
|
raise Exception("样本查询失败!")
|
|
|
|
|
|
def find_house_from_elastic_search(city, region, alias, roots=None, pdate=None, cat="sell", limit=1000):
|
|
index_name = get_seawater_index_name(city, region)
|
|
|
|
routing_set = [cat + "_" + city + "_" + region + "_" + alia for alia in alias]
|
|
must = [
|
|
{
|
|
"term": {"city": city}
|
|
}, {
|
|
"term": {"cat": cat}
|
|
}, {
|
|
"term": {"format_region": region}
|
|
}, {
|
|
"terms": {"district_name.raw": alias}
|
|
}
|
|
|
|
]
|
|
if roots is not None:
|
|
must.append({
|
|
"terms": {"root": roots}
|
|
})
|
|
|
|
if pdate is not None:
|
|
must.append({"range": {"p_date": {"gt": pdate}}})
|
|
|
|
body = {
|
|
|
|
"query": {
|
|
"bool": {
|
|
"must": must
|
|
}
|
|
},
|
|
"from": 0,
|
|
"size": limit,
|
|
"sort": [{"p_date": "desc"}],
|
|
"aggs": {}
|
|
}
|
|
response = es.es.search(index_name, "doc", routing=",".join(routing_set), body=body)
|
|
return pandas.DataFrame([doc["_source"] for doc in response["hits"]["hits"]])
|
|
|
|
|
|
def find_sample_pool(query, fields=None):
|
|
# s = time.time()
|
|
seawater105 = get_collection("fangjia", "seawater", config.get_seawater_host(), 27777)
|
|
for err_count in range(0, 10):
|
|
|
|
try:
|
|
|
|
# lt = seawater105.count(query)
|
|
lt = 300
|
|
# print('flag2')
|
|
pos = list(seawater105.find(query).limit(lt))
|
|
data = pandas.DataFrame(pos)
|
|
del pos
|
|
# print("==================>", time.time() - s)
|
|
return data
|
|
|
|
except:
|
|
|
|
# err_count = err_count + 1
|
|
# lt = seawater105.count(query)
|
|
lt = 300
|
|
pos = list(seawater105.find(query).limit(lt))
|
|
data = pandas.DataFrame(pos)
|
|
del pos
|
|
return data
|
|
|
|
raise Exception("样本查询失败!")
|
|
|
|
|
|
def find_disanganf_price(query, fields=None):
|
|
# client1 = pymongo.MongoClient('192.168.0.136',27017)
|
|
# db1 = client1.fangjia
|
|
# dsf = db1.average_statistics
|
|
# s = time.time()
|
|
seawater105 = get_collection("fangjia", "average_statistics", "192.168.0.136", 27017)
|
|
for err_count in range(0, 10):
|
|
|
|
try:
|
|
# print('flag3')
|
|
lt = seawater105.count(query)
|
|
|
|
pos = list(seawater105.find(query).limit(lt))
|
|
data = pandas.DataFrame(pos)
|
|
del pos
|
|
# print("==================>", time.time() - s)
|
|
return data
|
|
|
|
except:
|
|
|
|
# err_count = err_count + 1
|
|
lt = seawater105.count(query)
|
|
pos = list(seawater105.find(query).limit(lt))
|
|
data = pandas.DataFrame(pos)
|
|
del pos
|
|
return data
|
|
|
|
raise Exception("yang ben cha xun shi bai1")
|
|
|
|
|
|
def dis_feature(query, fields=None):
|
|
# client1 = pymongo.MongoClient('192.168.0.136',27017)
|
|
# db1 = client1.fangjia
|
|
# dsf = db1.average_statistics
|
|
# s = time.time()
|
|
seawater105 = get_collection("luogan", "distric_feature_0119", "192.168.0.136", 27017)
|
|
for err_count in range(0, 10):
|
|
|
|
try:
|
|
# print('flag4')
|
|
lt = seawater105.count(query)
|
|
|
|
pos = list(seawater105.find(query).limit(lt))
|
|
data = pandas.DataFrame(pos)
|
|
del pos
|
|
# print("==================>", time.time() - s)
|
|
return data
|
|
|
|
except:
|
|
|
|
# err_count = err_count + 1
|
|
lt = seawater105.count(query)
|
|
pos = list(seawater105.find(query).limit(lt))
|
|
data = pandas.DataFrame(pos)
|
|
del pos
|
|
return data
|
|
|
|
raise Exception("yang ben cha xun shi bai1")
|
|
|
|
|
|
def find_seaweed(query, fields=None):
|
|
# dirlt=p[0]
|
|
|
|
# client1 = pymongo.MongoClient('192.168.0.136',27017)
|
|
# db1 = client1.fangjia
|
|
# seaweed1 = db1.seaweed
|
|
# s = time.time()
|
|
seawater105 = get_collection("fangjia", "seaweed", "192.168.0.136", 27017)
|
|
for err_count in range(0, 10):
|
|
|
|
try:
|
|
# print('flag5')
|
|
lt = seawater105.count(query)
|
|
|
|
pos = list(seawater105.find(query).limit(lt))
|
|
data = pandas.DataFrame(pos)
|
|
del pos
|
|
# print("==================>", time.time() - s)
|
|
return data
|
|
|
|
except:
|
|
|
|
# err_count = err_count + 1
|
|
lt = seawater105.count(query)
|
|
pos = list(seawater105.find(query).limit(lt))
|
|
data = pandas.DataFrame(pos)
|
|
del pos
|
|
return data
|
|
|
|
raise Exception("yang ben cha xun shi bai1")
|
|
|
|
|
|
def find_region_district_list_name(query, fields=None):
|
|
# import pandas
|
|
# client1 = pymongo.MongoClient('192.168.0.136',27017)
|
|
# db1 = client1.fangjia
|
|
# seaweed1 = db1.seaweed
|
|
seawater105 = get_collection("fangjia", "seaweed", "192.168.0.136", 27017)
|
|
for err_count in range(0, 10):
|
|
|
|
try:
|
|
|
|
lt = seawater105.count(query)
|
|
|
|
pos = list(seawater105.find(query).limit(lt))
|
|
data = pandas.DataFrame(pos)
|
|
del pos
|
|
return data
|
|
|
|
except:
|
|
|
|
# err_count = err_count + 1
|
|
lt = seawater105.count(query)
|
|
pos = list(seawater105.find(query).limit(lt))
|
|
data = pandas.DataFrame(pos)
|
|
del pos
|
|
return data
|