Files
PriceSystem/PriceSystem/fangjia/__init__.py
T
罗干 ad8a960049 修改: PriceSystem/fangjia/__init__.py
修改:     PriceSystem/fix/__pycache__/price_fix.cpython-36.pyc
	修改:     PriceSystem/fix/price_fix.py
	修改:     PriceSystem/test.py
2018-04-11 16:15:43 +08:00

369 lines
11 KiB
Python

import pandas
import time
import logging
import config
import pytz
import datetime
import calendar
import requests
import json
import pickle
import redis
from fangjia import mongoClients as mongoClients
from fangjia import ElasticsearchClient as es
from fangjia.cachedObjectWrapper import CachedObjectWrapper
formatter = logging.Formatter('%(processName)s %(asctime)s %(name)-6s %(levelname)-6s %(message)s')
def get_logger(name):
logger = logging.getLogger(name)
handler = logging.StreamHandler()
handler.setFormatter(formatter)
logger.addHandler(handler)
file_handler = logging.FileHandler("/home/goojia/logs/" + name + ".log")
file_handler.setFormatter(formatter)
logger.addHandler(file_handler)
logger.setLevel("ERROR")
return logger
def get_performance_logger(name):
logger = logging.getLogger(name)
handler = logging.StreamHandler()
handler.setFormatter(formatter)
logger.addHandler(handler)
file_handler = logging.FileHandler("/home/goojia/logs/" + name + ".performance.log")
file_handler.setFormatter(formatter)
logger.addHandler(file_handler)
logger.setLevel("ERROR")
return logger
now = datetime.datetime.now().replace(hour=0, minute=0, second=0, microsecond=0)
for day in range(1, 9):
if calendar.weekday(now.year, now.month, now.day) == 6:
break
now = now + datetime.timedelta(days=1)
performance_logger = get_performance_logger("performance")
def timer(func):
def _timer(*args, **kw):
begin = time.time()
rs = func(*args, **kw)
end = time.time()
elapsed = end - begin
if elapsed > config.get_timeout():
arg_list = list()
for arg in args:
arg_list.append(str(arg))
if elapsed > config.get_timeout() * 10:
performance_logger.info("函数名:%-20s\t耗时:%7.4f\t\t参数:(%s)" % (func.__name__, end - begin, arg_list))
else:
performance_logger.error("函数名:%-20s\t耗时:%7.4f\t\t参数:(%s)" % (func.__name__, end - begin, arg_list))
return rs
return _timer
QUEUE = "price_system"
redis_pool = redis.ConnectionPool(host=config.get_redis_host(), port=6379, db=config.get_redis_cache_db())
client = redis.Redis(connection_pool=redis_pool)
def cache(expire=None):
def _cache(func):
def __cache(*args, **kw):
arg_list = list()
for arg in args:
arg_list.append(str(arg))
key = func.__name__ + "_" + "_".join(arg_list)
try:
val = client.hget(QUEUE, key)
if val is not None:
wrapper = pickle.loads(val)
if not wrapper.is_expire():
return wrapper.val
else:
client.hset(QUEUE, key, None)
except Exception as ex:
print(ex)
rs = func(*args, **kw)
wrapper = CachedObjectWrapper(rs, time.time(), expire)
client.hset(QUEUE, key, pickle.dumps(wrapper))
return rs
return __cache
return _cache
def get_collection(db, name, host="192.168.0.136", port=27017):
return mongoClients.MongoClients(db, name, host, port).collection
def get_seawater_index_name(city, region):
return es.get_index_name(city, region)
def get_all_city():
server_area = get_collection("fangjia", "server_area", config.get_mongodb_host(), config.get_mongodb_port())
city_list = []
for area in server_area.find().sort([("city_grade", 1), ("sequence", 1)]):
if "display" in area and area["display"] != "全国":
city_list.append(area["display"])
return city_list
def get_region(city):
region_block = get_collection("fangjia", "region_block", config.get_mongodb_host(), config.get_mongodb_port())
return [region["name"] for region in region_block.find({"category": "region", "city": city}, {"name": 1})]
def get_weekend():
"""
返回本周周末
:return:
"""
now = datetime.datetime.now().replace(hour=0, minute=0, second=0, microsecond=0)
for day in range(1, 9):
if calendar.weekday(now.year, now.month, now.day) == 6:
break
now = now + datetime.timedelta(days=1)
pacific = pytz.timezone('Asia/Shanghai')
return pacific.localize(now)
def send_ding_msg(msg):
headers = {
'Content-Type': "application/json",
}
url = "https://oapi.dingtalk.com/robot/send" \
"?access_token=2307d2fec8314b3e8b4b83dd1d42da2e2051638e3e668f21e77ca01258bf682c"
text_body = {
"msgtype": "text",
"text": {
"content": msg
}
}
requests.request("POST", url, data=json.dumps(text_body, ensure_ascii=False).encode('utf8'), headers=headers)
@timer
def findHouse(query, fields=None):
seawater105 = get_collection("fangjia", "seawater", config.get_seawater_host(), 27777)
for err_count in range(0, 10):
try:
lt = 100
# seawater.find(query,{"p_date":-1}).sort([("p_date",-1)])
pos = list(seawater105.find(query, fields).sort([("p_date",-1)]).limit(lt))
#pos = list(seawater105.find(query, fields).limit(lt))
data = pandas.DataFrame(pos)
del pos
# print("==================>", time.time() - s)
return data
except:
lt = 100
pos = list(seawater105.find(query, fields).sort([("p_date",-1)]).limit(lt))
data = pandas.DataFrame(pos)
del pos
return data
raise Exception("样本查询失败!")
def find_house_from_elastic_search(city, region, alias, roots=None, pdate=None, cat="sell", limit=1000):
index_name = get_seawater_index_name(city, region)
routing_set = [cat + "_" + city + "_" + region + "_" + alia for alia in alias]
must = [
{
"term": {"city": city}
}, {
"term": {"cat": cat}
}, {
"term": {"format_region": region}
}, {
"terms": {"district_name.raw": alias}
}
]
if roots is not None:
must.append({
"terms": {"root": roots}
})
if pdate is not None:
must.append({"range": {"p_date": {"gt": pdate}}})
body = {
"query": {
"bool": {
"must": must
}
},
"from": 0,
"size": limit,
"sort": [{"p_date": "desc"}],
"aggs": {}
}
response = es.es.search(index_name, "doc", routing=",".join(routing_set), body=body)
return pandas.DataFrame([doc["_source"] for doc in response["hits"]["hits"]])
def find_sample_pool(query, fields=None):
# s = time.time()
seawater105 = get_collection("fangjia", "seawater", config.get_seawater_host(), 27777)
for err_count in range(0, 10):
try:
# lt = seawater105.count(query)
lt = 300
# print('flag2')
pos = list(seawater105.find(query).limit(lt))
data = pandas.DataFrame(pos)
del pos
# print("==================>", time.time() - s)
return data
except:
# err_count = err_count + 1
# lt = seawater105.count(query)
lt = 300
pos = list(seawater105.find(query).limit(lt))
data = pandas.DataFrame(pos)
del pos
return data
raise Exception("样本查询失败!")
def find_disanganf_price(query, fields=None):
# client1 = pymongo.MongoClient('192.168.0.136',27017)
# db1 = client1.fangjia
# dsf = db1.average_statistics
# s = time.time()
seawater105 = get_collection("fangjia", "average_statistics", config.get_mongodb_host(), 27017)
for err_count in range(0, 10):
try:
# print('flag3')
lt = seawater105.count(query)
pos = list(seawater105.find(query).limit(lt))
data = pandas.DataFrame(pos)
del pos
# print("==================>", time.time() - s)
return data
except:
# err_count = err_count + 1
lt = seawater105.count(query)
pos = list(seawater105.find(query).limit(lt))
data = pandas.DataFrame(pos)
del pos
return data
raise Exception("yang ben cha xun shi bai1")
@timer
def dis_feature(query, fields=None):
# client1 = pymongo.MongoClient('192.168.0.136',27017)
# db1 = client1.fangjia
# dsf = db1.average_statistics
# s = time.time()
seawater105 = get_collection("fangjia_stat", "district_feature", config.get_mongodb_host(), config.get_mongodb_port())
#seawater105 = get_collection("luogan", "district_feature", '192.168.0.136',27017)
for err_count in range(0, 10):
try:
# print('flag4')
lt = seawater105.count(query)
pos = list(seawater105.find(query).limit(lt))
data = pandas.DataFrame(pos)
del pos
# print("==================>", time.time() - s)
return data
except:
# err_count = err_count + 1
lt = seawater105.count(query)
pos = list(seawater105.find(query).limit(lt))
data = pandas.DataFrame(pos)
del pos
return data
raise Exception("yang ben cha xun shi bai1")
@timer
def find_seaweed(query, fields=None):
# dirlt=p[0]
# client1 = pymongo.MongoClient('192.168.0.136',27017)
# db1 = client1.fangjia
# seaweed1 = db1.seaweed
# s = time.time()
seawater105 = get_collection("fangjia", "seaweed", config.get_seaweed_host(), config.get_seaweed_port())
for err_count in range(0, 10):
try:
# print('flag5')
pos = list(seawater105.find(query))
data = pandas.DataFrame(pos)
del pos
# print("==================>", time.time() - s)
return data
except:
pos = list(seawater105.find(query))
data = pandas.DataFrame(pos)
del pos
return data
raise Exception("yang ben cha xun shi bai1")
def find_region_district_list_name(query, fields=None):
# import pandas
# client1 = pymongo.MongoClient('192.168.0.136',27017)
# db1 = client1.fangjia
# seaweed1 = db1.seaweed
seawater105 = get_collection("fangjia", "seaweed", config.get_seaweed_host(), config.get_seaweed_port())
for err_count in range(0, 10):
try:
lt = seawater105.count(query)
pos = list(seawater105.find(query).limit(lt))
data = pandas.DataFrame(pos)
del pos
return data
except:
# err_count = err_count + 1
lt = seawater105.count(query)
pos = list(seawater105.find(query).limit(lt))
data = pandas.DataFrame(pos)
del pos
return data