Files
PriceSystem/PriceSystem/fangjia/__init__.py
T
2018-03-13 13:34:02 +08:00

320 lines
9.1 KiB
Python

import pandas
import time
import logging
import config
import pytz
import datetime
import calendar
import requests
import json
from fangjia import mongoClients as mongoClients
from fangjia import ElasticsearchClient as es
formatter = logging.Formatter('%(processName)s %(asctime)s %(name)-6s %(levelname)-6s %(message)s')
def get_logger(name):
logger = logging.getLogger(name)
handler = logging.StreamHandler()
handler.setFormatter(formatter)
logger.addHandler(handler)
file_handler = logging.FileHandler("/home/goojia/logs/" + name + ".log")
file_handler.setFormatter(formatter)
logger.addHandler(file_handler)
logger.setLevel("INFO")
return logger
def get_performance_logger(name):
logger = logging.getLogger(name)
handler = logging.StreamHandler()
handler.setFormatter(formatter)
logger.addHandler(handler)
file_handler = logging.FileHandler("/home/goojia/logs/" + name + ".performance.log")
file_handler.setFormatter(formatter)
logger.addHandler(file_handler)
logger.setLevel("INFO")
return logger
performance_logger = get_performance_logger("performance")
def timer(func):
def _timer(*args, **kw):
begin = time.clock()
rs = func(*args, **kw)
end = time.clock()
elapsed = end - begin
if elapsed > config.get_timeout():
arg_list = list()
for arg in args:
arg_list.append(str(arg))
performance_logger.info("函数名:%-20s\t耗时:%7.4f\t\t参数:(%s)" % (func.__name__, end - begin, arg_list))
return rs
return _timer
def get_collection(db, name, host="192.168.0.136", port=27017):
return mongoClients.MongoClients(db, name, host, port).collection
def get_seawater_index_name(city, region):
return es.get_index_name(city, region)
def get_all_city():
server_area = get_collection("fangjia", "server_area", config.get_mongodb_host(), config.get_mongodb_port())
city_list = []
for area in server_area.find().sort([("city_grade", 1), ("sequence", 1)]):
if "display" in area and area["display"] != "全国":
city_list.append(area["display"])
return city_list
def get_region(city):
region_block = get_collection("fangjia", "region_block", config.get_mongodb_host(), config.get_mongodb_port())
return [region["name"] for region in region_block.find({"category": "region", "city": city}, {"name": 1})]
def get_weekend():
now = datetime.datetime.now().replace(hour=0, minute=0, second=0, microsecond=0)
for day in range(1, 9):
if calendar.weekday(now.year, now.month, now.day) == 6:
break
now = now + datetime.timedelta(days=1)
pacific = pytz.timezone('Asia/Shanghai')
return pacific.localize(now)
def send_ding_msg(msg):
headers = {
'Content-Type': "application/json",
}
url = "https://oapi.dingtalk.com/robot/send" \
"?access_token=2307d2fec8314b3e8b4b83dd1d42da2e2051638e3e668f21e77ca01258bf682c"
text_body = {
"msgtype": "text",
"text": {
"content": msg
}
}
requests.request("POST", url, data=json.dumps(text_body, ensure_ascii=False).encode('utf8'), headers=headers)
@timer
def findHouse(query, fields=None):
seawater105 = get_collection("fangjia", "seawater", config.get_seawater_host(), 27777)
for err_count in range(0, 10):
try:
lt = 1000
# seawater.find(query,{"p_date":-1}).sort([("p_date",-1)])
pos = list(seawater105.find(query, fields).limit(lt))
data = pandas.DataFrame(pos)
del pos
# print("==================>", time.time() - s)
return data
except:
lt = seawater105.count(query)
pos = list(seawater105.find(query).limit(lt))
data = pandas.DataFrame(pos)
del pos
return data
raise Exception("样本查询失败!")
def find_house_from_elastic_search(city, region, alias, roots=None, pdate=None, cat="sell", limit=1000):
index_name = get_seawater_index_name(city, region)
routing_set = [cat + "_" + city + "_" + region + "_" + alia for alia in alias]
must = [
{
"term": {"city": city}
}, {
"term": {"cat": cat}
}, {
"term": {"format_region": region}
}, {
"terms": {"district_name.raw": alias}
}
]
if roots is not None:
must.append({
"terms": {"root": roots}
})
if pdate is not None:
must.append({"range": {"p_date": {"gt": pdate}}})
body = {
"query": {
"bool": {
"must": must
}
},
"from": 0,
"size": limit,
"sort": [{"p_date": "desc"}],
"aggs": {}
}
response = es.es.search(index_name, "doc", routing=",".join(routing_set), body=body)
return pandas.DataFrame([doc["_source"] for doc in response["hits"]["hits"]])
def find_sample_pool(query, fields=None):
# s = time.time()
seawater105 = get_collection("fangjia", "seawater", config.get_seawater_host(), 27777)
for err_count in range(0, 10):
try:
# lt = seawater105.count(query)
lt = 300
# print('flag2')
pos = list(seawater105.find(query).limit(lt))
data = pandas.DataFrame(pos)
del pos
# print("==================>", time.time() - s)
return data
except:
# err_count = err_count + 1
# lt = seawater105.count(query)
lt = 300
pos = list(seawater105.find(query).limit(lt))
data = pandas.DataFrame(pos)
del pos
return data
raise Exception("样本查询失败!")
def find_disanganf_price(query, fields=None):
# client1 = pymongo.MongoClient('192.168.0.136',27017)
# db1 = client1.fangjia
# dsf = db1.average_statistics
# s = time.time()
seawater105 = get_collection("fangjia", "average_statistics", "192.168.0.136", 27017)
for err_count in range(0, 10):
try:
# print('flag3')
lt = seawater105.count(query)
pos = list(seawater105.find(query).limit(lt))
data = pandas.DataFrame(pos)
del pos
# print("==================>", time.time() - s)
return data
except:
# err_count = err_count + 1
lt = seawater105.count(query)
pos = list(seawater105.find(query).limit(lt))
data = pandas.DataFrame(pos)
del pos
return data
raise Exception("yang ben cha xun shi bai1")
def dis_feature(query, fields=None):
# client1 = pymongo.MongoClient('192.168.0.136',27017)
# db1 = client1.fangjia
# dsf = db1.average_statistics
# s = time.time()
seawater105 = get_collection("luogan", "distric_feature_0119", "192.168.0.136", 27017)
for err_count in range(0, 10):
try:
# print('flag4')
lt = seawater105.count(query)
pos = list(seawater105.find(query).limit(lt))
data = pandas.DataFrame(pos)
del pos
# print("==================>", time.time() - s)
return data
except:
# err_count = err_count + 1
lt = seawater105.count(query)
pos = list(seawater105.find(query).limit(lt))
data = pandas.DataFrame(pos)
del pos
return data
raise Exception("yang ben cha xun shi bai1")
def find_seaweed(query, fields=None):
# dirlt=p[0]
# client1 = pymongo.MongoClient('192.168.0.136',27017)
# db1 = client1.fangjia
# seaweed1 = db1.seaweed
# s = time.time()
seawater105 = get_collection("fangjia", "seaweed", "192.168.0.136", 27017)
for err_count in range(0, 10):
try:
# print('flag5')
lt = seawater105.count(query)
pos = list(seawater105.find(query).limit(lt))
data = pandas.DataFrame(pos)
del pos
# print("==================>", time.time() - s)
return data
except:
# err_count = err_count + 1
lt = seawater105.count(query)
pos = list(seawater105.find(query).limit(lt))
data = pandas.DataFrame(pos)
del pos
return data
raise Exception("yang ben cha xun shi bai1")
def find_region_district_list_name(query, fields=None):
# import pandas
# client1 = pymongo.MongoClient('192.168.0.136',27017)
# db1 = client1.fangjia
# seaweed1 = db1.seaweed
seawater105 = get_collection("fangjia", "seaweed", "192.168.0.136", 27017)
for err_count in range(0, 10):
try:
lt = seawater105.count(query)
pos = list(seawater105.find(query).limit(lt))
data = pandas.DataFrame(pos)
del pos
return data
except:
# err_count = err_count + 1
lt = seawater105.count(query)
pos = list(seawater105.find(query).limit(lt))
data = pandas.DataFrame(pos)
del pos
return data