Files
PriceSystem/PriceSystem/fangjia/__init__.py
T
2018-06-14 10:21:49 +08:00

532 lines
15 KiB
Python

import pandas
import time
import logging
import config
import pytz
import datetime
import calendar
import requests
import json
import pickle
import redis
from fangjia import mongoClients as mongoClients
from fangjia import ElasticsearchClient as es
from fangjia.cachedObjectWrapper import CachedObjectWrapper
#
formatter = logging.Formatter('%(processName)s %(asctime)s %(name)-6s %(levelname)-6s %(message)s')
def get_logger(name):
logger = logging.getLogger(name)
handler = logging.StreamHandler()
handler.setFormatter(formatter)
logger.addHandler(handler)
file_handler = logging.FileHandler("/home/goojia/logs/" + name + ".log")
file_handler.setFormatter(formatter)
logger.addHandler(file_handler)
logger.setLevel("INFO")
return logger
def get_performance_logger(name):
logger = logging.getLogger(name)
handler = logging.StreamHandler()
handler.setFormatter(formatter)
logger.addHandler(handler)
file_handler = logging.FileHandler("/home/goojia/logs/" + name + ".performance.log")
file_handler.setFormatter(formatter)
logger.addHandler(file_handler)
logger.setLevel("INFO")
return logger
performance_logger = get_performance_logger("performance")
def timer(func):
def _timer(*args, **kw):
begin = time.clock()
rs = func(*args, **kw)
end = time.clock()
elapsed = end - begin
if elapsed > config.get_timeout():
arg_list = list()
for arg in args:
arg_list.append(str(arg))
performance_logger.info("函数名:%-20s\t耗时:%7.4f\t\t参数:(%s)" % (func.__name__, end - begin, arg_list))
if elapsed > config.get_timeout():
performance_logger.info("函数名:%-20s\t耗时:%7.4f\t\t参数:(%s)" % (func.__name__, end - begin, arg_list))
return rs
return _timer
QUEUE = "price_system"
redis_pool = redis.ConnectionPool(host=config.get_redis_host(), port=6379, db=config.get_redis_cache_db())
client = redis.Redis(connection_pool=redis_pool)
def cache(expire=None):
def _cache(func):
def __cache(*args, **kw):
arg_list = list()
for arg in args:
arg_list.append(str(arg))
key = func.__name__ + "_" + "_".join(arg_list)
try:
val = client.hget(QUEUE, key)
if val is not None:
wrapper = pickle.loads(val)
if not wrapper.is_expire():
return wrapper.val
else:
client.hset(QUEUE, key, None)
except Exception as ex:
print(ex)
rs = func(*args, **kw)
wrapper = CachedObjectWrapper(rs, time.time(), expire)
client.hset(QUEUE, key, pickle.dumps(wrapper))
return rs
return __cache
return _cache
def get_collection(db, name, host="192.168.0.136", port=27017):
return mongoClients.MongoClients(db, name, host, port).collection
def get_seawater_index_name(city, region):
return es.get_index_name(city, region)
def get_all_city():
server_area = get_collection("fangjia", "server_area", config.get_mongodb_host(), config.get_mongodb_port())
city_list = []
for area in server_area.find().sort([("city_grade", 1), ("sequence", 1)]):
if "display" in area and area["display"] != "全国":
city_list.append(area["display"])
return city_list
def get_region(city):
region_block = get_collection("fangjia", "region_block", config.get_mongodb_host(), config.get_mongodb_port())
return [region["name"] for region in region_block.find({"category": "region", "city": city}, {"name": 1})]
def get_weekend():
now = datetime.datetime.now().replace(hour=0, minute=0, second=0, microsecond=0)
for day in range(1, 9):
if calendar.weekday(now.year, now.month, now.day) == 6:
break
now = now + datetime.timedelta(days=1)
pacific = pytz.timezone('Asia/Shanghai')
return pacific.localize(now)
def send_ding_msg(msg):
headers = {
'Content-Type': "application/json",
}
url = "https://oapi.dingtalk.com/robot/send?access_token=35587c9e7accc54ff7ddf2c6ebdc36" \
"a3ba065b4a2bdac78c0e9928596361b430"
text_body = {
"msgtype": "text",
"text": {
"content": msg
}
}
requests.request("POST", url, data=json.dumps(text_body, ensure_ascii=False).encode('utf8'), headers=headers)
@timer
def findHouse(query, fields=None):
seawater105 = get_collection("fangjia", "seawater", config.get_seawater_host(), 27777)
for err_count in range(0, 2):
try:
lt = 100
# seawater.find(query,{"p_date":-1}).sort([("p_date",-1)])
pos = list(seawater105.find(query, fields).sort([("p_date",-1)]).limit(lt))
#pos = list(seawater105.find(query, fields).limit(lt))
data = pandas.DataFrame(pos)
del pos
# print("==================>", time.time() - s)
return data
except:
lt = 100
pos = list(seawater105.find(query, fields).sort([("p_date",-1)]).limit(lt))
data = pandas.DataFrame(pos)
del pos
return data
raise Exception("样本查询失败!")
def find_house_from_elastic_search(city, region, alias, roots=None, pdate=None, cat="sell", limit=1000):
index_name = get_seawater_index_name(city, region)
routing_set = [cat + "_" + city + "_" + region + "_" + alia for alia in alias]
must = [
{
"term": {"city": city}
}, {
"term": {"cat": cat}
}, {
"term": {"format_region": region}
}, {
"terms": {"district_name.raw": alias}
}
]
if roots is not None:
must.append({
"terms": {"root": roots}
})
if pdate is not None:
must.append({"range": {"p_date": {"gt": pdate}}})
body = {
"query": {
"bool": {
"must": must
}
},
"from": 0,
"size": limit,
"sort": [{"p_date": "desc"}],
"aggs": {}
}
response = es.es.search(index_name, "doc", routing=",".join(routing_set), body=body)
return pandas.DataFrame([doc["_source"] for doc in response["hits"]["hits"]])
def find_sample_pool(query, fields=None):
# s = time.time()
seawater105 = get_collection("fangjia", "seawater", config.get_seawater_host(), 27777)
for err_count in range(0, 2):
try:
# lt = seawater105.count(query)
lt = 300
# print('flag2')
pos = list(seawater105.find(query).limit(lt))
data = pandas.DataFrame(pos)
del pos
# print("==================>", time.time() - s)
return data
except:
# err_count = err_count + 1
# lt = seawater105.count(query)
lt = 300
pos = list(seawater105.find(query).limit(lt))
data = pandas.DataFrame(pos)
del pos
return data
raise Exception("样本查询失败!")
def find_disanganf_price(query, fields=None):
# client1 = pymongo.MongoClient('192.168.0.136',27017)
# db1 = client1.fangjia
# dsf = db1.average_statistics
# s = time.time()
seawater105 = get_collection("fangjia", "average_statistics",'mongo.db.fangjia.com', 27017)
for err_count in range(0, 2):
try:
# print('flag3')
lt = seawater105.count(query)
pos = list(seawater105.find(query).limit(lt))
data = pandas.DataFrame(pos)
del pos
# print("==================>", time.time() - s)
return data
except:
# err_count = err_count + 1
lt = seawater105.count(query)
pos = list(seawater105.find(query).limit(lt))
data = pandas.DataFrame(pos)
del pos
return data
raise Exception("yang ben cha xun shi bai1")
def dis_feature(query, fields=None):
# client1 = pymongo.MongoClient('192.168.0.136',27017)
# db1 = client1.fangjia
# dsf = db1.average_statistics
# s = time.time()
seawater105 = get_collection("fangjia_stat", "district_feature", 'mongo.db.fangjia.com', 27017)
for err_count in range(0, 2):
try:
# print('flag4')
lt = seawater105.count(query)
pos = list(seawater105.find(query).limit(lt))
data = pandas.DataFrame(pos)
del pos
# print("==================>", time.time() - s)
return data
except:
# err_count = err_count + 1
lt = seawater105.count(query)
pos = list(seawater105.find(query).limit(lt))
data = pandas.DataFrame(pos)
del pos
return data
raise Exception("yang ben cha xun shi bai1")
def find_seaweed(query, fields=None):
# dirlt=p[0]
# client1 = pymongo.MongoClient('192.168.0.136',27017)
# db1 = client1.fangjia
# seaweed1 = db1.seaweed
# s = time.time()
seawater105 = get_collection("fangjia", "seaweed", config.get_seaweed_host(), 27017)
for err_count in range(0,2):
try:
# print('flag5')
#lt = seawater105.count(query)
pos = list(seawater105.find(query))
data = pandas.DataFrame(pos)
del pos
# print("==================>", time.time() - s)
return data
except:
# err_count = err_count + 1
#lt = seawater105.count(query)
pos = list(seawater105.find(query))
data = pandas.DataFrame(pos)
del pos
return data
raise Exception("yang ben cha xun shi bai1")
def find_region_price(query, fields=None):
# dirlt=p[0]
# client1 = pymongo.MongoClient('192.168.0.136',27017)
# db1 = client1.fangjia
# seaweed1 = db1.seaweed
# s = time.time()
seawater105 = get_collection("fangjia", "region_stat", '192.168.0.136', 27017)
for err_count in range(0,2):
try:
# print('flag5')
#lt = seawater105.count(query)
pos = list(seawater105.find(query))
data = pandas.DataFrame(pos)
del pos
# print("==================>", time.time() - s)
return data
except:
# err_count = err_count + 1
#lt = seawater105.count(query)
pos = list(seawater105.find(query))
data = pandas.DataFrame(pos)
del pos
return data
raise Exception("yang ben cha xun shi bai1")
def find_region_district_list_name(query, fields=None):
# import pandas
# client1 = pymongo.MongoClient('192.168.0.136',27017)
# db1 = client1.fangjia
# seaweed1 = db1.seaweed
seawater105 = get_collection("fangjia", "seaweed", config.get_seaweed_host(), 27017)
for err_count in range(0, 2):
try:
lt = seawater105.count(query)
pos = list(seawater105.find(query).limit(lt))
data = pandas.DataFrame(pos)
del pos
return data
except:
# err_count = err_count + 1
lt = seawater105.count(query)
pos = list(seawater105.find(query).limit(lt))
data = pandas.DataFrame(pos)
del pos
return data
##########################################################################################
# the code below is about computing rent price /m**2
def rent_sample(query, fields):
# s = time.time()
seawater105 = get_collection("fangjia", "seawater", config.get_seawater_host(), 27777)
for err_count in range(0, 2):
try:
# lt = seawater105.count(query)
lt = 1000
# print('flag2')
pos=list(seawater105.find(query,fields).limit(lt))
data=pandas.DataFrame(pos)
if data.shape[0]>10:
#print('数据条数=',data.shape[0])
return data[['district_name','total_price','area','p_date']]
else:
return pandas.DataFrame()
except:
lt = 1000
# print('flag2')
pos=list(seawater105.find(query,fields).limit(lt))
data=pandas.DataFrame(pos)
if data.shape[0]>10:
#print('数据条数=',data.shape[0])
return data[['district_name','total_price','area','p_date']]
else:
return pandas.DataFrame()
raise Exception("样本查询失败!")
def rent_disanganf_series(query, fields):
# s = time.time()
seawater105 = get_collection("fangjia", "district_stat", 'mongo.db.fangjia.com', 27017)
for err_count in range(0, 2):
try:
pos=list(seawater105.find(query,fields))
print('pos=',pos)
data1=pandas.DataFrame(pos)
one_date=list(set(data1['weekend']))[1]
#two_date=list(set(data1['weekend']))[1]
data=data1[data1['weekend']==one_date]
d0=list(data['calc_value'])
d1=list(data['district_name'])
d2=pandas.Series(d0,index=d1)
return d2
except:
# print('flag2')
pos=list(seawater105.find(query,fields))
data1=pandas.DataFrame(pos)
print('data1=',data1)
one_date=list(set(data1['weekend']))[1]
data=data1[data1['weekend']==one_date]
d0=list(data['calc_value'])
d1=list(data['district_name'])
d2=pandas.Series(d0,index=d1)
return d2
raise Exception("样本查询失败!")
def rent_region_feature(query, fields):
rf = get_collection("fangjia", config.get_rent_temp_db(), '192.168.0.136', 27017)
#client182 = pymongo.MongoClient('192.168.10.182',27017)
#db182 = client182.fangjia_stat.rent_1221
for err_count in range(0, 2):
try:
pos=list(rf.find(query,fields))
pos1=pandas.DataFrame(pos)
return pos1
except:
pos=list(rf.find(query,fields))
pos1=pandas.DataFrame(pos)
return pos1
raise Exception("样本查询失败!")
"""
query1 = {"city":ct,"region":rl,'weekend':month_lastday(timm)}
a1=list(db182.find(query1))
a2=pd.DataFrame(a1)
if a2.shape[0]>0:
ratio=a2['ratio'].iloc[0]
price=a2['avg_price'].iloc[0]
return (ratio,price)
else:
return ()
"""
def rent_find_disanganf_price(query, fields=None):
# client1 = pymongo.MongoClient('192.168.0.136',27017)
# db1 = client1.fangjia
# dsf = db1.average_statistics
# s = time.time()
seawater105 = get_collection("fangjia", "district_stat",'mongo.db.fangjia.com', 27017)
for err_count in range(0, 2):
try:
# print('flag3')
lt = seawater105.count(query)
pos = list(seawater105.find(query).limit(lt))
data = pandas.DataFrame(pos)
del pos
# print("==================>", time.time() - s)
return data
except:
# err_count = err_count + 1
lt = seawater105.count(query)
pos = list(seawater105.find(query).limit(lt))
data = pandas.DataFrame(pos)
del pos
return data
raise Exception("yang ben cha xun shi bai1")