初始化均价系统
This commit is contained in:
@@ -0,0 +1,281 @@
|
||||
import pandas
|
||||
import time
|
||||
import logging
|
||||
import config
|
||||
from fangjia import mongoClients as mongoClients
|
||||
from fangjia import ElasticsearchClient as es
|
||||
|
||||
formatter = logging.Formatter('%(processName)s %(asctime)s %(name)-6s %(levelname)-6s %(message)s')
|
||||
|
||||
|
||||
def get_logger(name):
|
||||
logger = logging.getLogger(name)
|
||||
handler = logging.StreamHandler()
|
||||
handler.setFormatter(formatter)
|
||||
logger.addHandler(handler)
|
||||
|
||||
file_handler = logging.FileHandler("/home/goojia/logs/price.log")
|
||||
file_handler.setFormatter(formatter)
|
||||
logger.addHandler(file_handler)
|
||||
logger.setLevel("INFO")
|
||||
|
||||
return logger
|
||||
|
||||
|
||||
def get_performance_logger(name):
|
||||
logger = logging.getLogger(name)
|
||||
handler = logging.StreamHandler()
|
||||
handler.setFormatter(formatter)
|
||||
logger.addHandler(handler)
|
||||
|
||||
file_handler = logging.FileHandler("/home/goojia/logs/price.performance.log")
|
||||
file_handler.setFormatter(formatter)
|
||||
logger.addHandler(file_handler)
|
||||
logger.setLevel("INFO")
|
||||
|
||||
return logger
|
||||
|
||||
|
||||
performance_logger = get_performance_logger("performance")
|
||||
|
||||
|
||||
def timer(func):
|
||||
def _timer(*args, **kw):
|
||||
begin = time.clock()
|
||||
rs = func(*args, **kw)
|
||||
end = time.clock()
|
||||
elapsed = end - begin
|
||||
if elapsed > config.get_timeout():
|
||||
arg_list = list()
|
||||
for arg in args:
|
||||
arg_list.append(str(arg))
|
||||
performance_logger.info("函数名:(%s) 参数:(%s) 耗时:%f" % (func.__name__, arg_list, end - begin))
|
||||
return rs
|
||||
|
||||
return _timer
|
||||
|
||||
|
||||
def get_collection(db, name, host="192.168.0.136", port=27017):
|
||||
return mongoClients.MongoClients(db, name, host, port).collection
|
||||
|
||||
|
||||
def get_seawater_index_name(city, region):
|
||||
return es.get_index_name(city, region)
|
||||
|
||||
|
||||
@timer
|
||||
def findHouse(query, fields=None):
|
||||
# s = time.time()
|
||||
seawater105 = get_collection("fangjia", "seawater", "192.168.0.105", 27777)
|
||||
for err_count in range(0, 10):
|
||||
|
||||
try:
|
||||
|
||||
# lt = seawater105.count(query)
|
||||
|
||||
# print('flag1')
|
||||
lt = 1000
|
||||
# seawater.find(query,{"p_date":-1}).sort([("p_date",-1)])
|
||||
pos = list(seawater105.find(query).limit(lt))
|
||||
data = pandas.DataFrame(pos)
|
||||
del pos
|
||||
# print("==================>", time.time() - s)
|
||||
return data
|
||||
except:
|
||||
lt = seawater105.count(query)
|
||||
pos = list(seawater105.find(query).limit(lt))
|
||||
data = pandas.DataFrame(pos)
|
||||
del pos
|
||||
return data
|
||||
|
||||
raise Exception("样本查询失败!")
|
||||
|
||||
|
||||
@timer
|
||||
def find_house_from_elastic_search(city, region, alias, roots=None, pdate=None, cat="sell", limit=100):
|
||||
index_name = get_seawater_index_name(city, region)
|
||||
|
||||
routing_set = [cat + "_" + city + "_" + region + "_" + alia for alia in alias]
|
||||
must = [
|
||||
{
|
||||
"term": {"city": city}
|
||||
}, {
|
||||
"term": {"cat": cat}
|
||||
}, {
|
||||
"term": {"format_region": region}
|
||||
}, {
|
||||
"terms": {"district_name.raw": alias}
|
||||
}
|
||||
|
||||
]
|
||||
if roots is not None:
|
||||
must.append({
|
||||
"terms": {"root": roots}
|
||||
})
|
||||
|
||||
if pdate is not None:
|
||||
must.append({"range": {"p_date": {"gt": pdate}}})
|
||||
|
||||
body = {
|
||||
|
||||
"query": {
|
||||
"bool": {
|
||||
"must": must
|
||||
}
|
||||
},
|
||||
"from": 0,
|
||||
"size": limit,
|
||||
"sort": [{"p_date": "desc"}],
|
||||
"aggs": {}
|
||||
}
|
||||
response = es.es.search(index_name, "doc", routing=",".join(routing_set), body=body)
|
||||
return pandas.DataFrame([doc["_source"] for doc in response["hits"]["hits"]])
|
||||
|
||||
|
||||
@timer
|
||||
def find_sample_pool(query, fields=None):
|
||||
# s = time.time()
|
||||
seawater105 = get_collection("fangjia", "seawater", "192.168.0.105 ", 27777)
|
||||
for err_count in range(0, 10):
|
||||
|
||||
try:
|
||||
|
||||
# lt = seawater105.count(query)
|
||||
lt = 300
|
||||
# print('flag2')
|
||||
pos = list(seawater105.find(query).limit(lt))
|
||||
data = pandas.DataFrame(pos)
|
||||
del pos
|
||||
# print("==================>", time.time() - s)
|
||||
return data
|
||||
|
||||
except:
|
||||
|
||||
# err_count = err_count + 1
|
||||
# lt = seawater105.count(query)
|
||||
lt = 300
|
||||
pos = list(seawater105.find(query).limit(lt))
|
||||
data = pandas.DataFrame(pos)
|
||||
del pos
|
||||
return data
|
||||
|
||||
raise Exception("样本查询失败!")
|
||||
|
||||
|
||||
def find_disanganf_price(query, fields=None):
|
||||
# client1 = pymongo.MongoClient('192.168.0.136',27017)
|
||||
# db1 = client1.fangjia
|
||||
# dsf = db1.average_statistics
|
||||
# s = time.time()
|
||||
seawater105 = get_collection("fangjia", "average_statistics", "192.168.0.136", 27017)
|
||||
for err_count in range(0, 10):
|
||||
|
||||
try:
|
||||
# print('flag3')
|
||||
lt = seawater105.count(query)
|
||||
|
||||
pos = list(seawater105.find(query).limit(lt))
|
||||
data = pandas.DataFrame(pos)
|
||||
del pos
|
||||
# print("==================>", time.time() - s)
|
||||
return data
|
||||
|
||||
except:
|
||||
|
||||
# err_count = err_count + 1
|
||||
lt = seawater105.count(query)
|
||||
pos = list(seawater105.find(query).limit(lt))
|
||||
data = pandas.DataFrame(pos)
|
||||
del pos
|
||||
return data
|
||||
|
||||
raise Exception("yang ben cha xun shi bai1")
|
||||
|
||||
|
||||
def dis_feature(query, fields=None):
|
||||
# client1 = pymongo.MongoClient('192.168.0.136',27017)
|
||||
# db1 = client1.fangjia
|
||||
# dsf = db1.average_statistics
|
||||
# s = time.time()
|
||||
seawater105 = get_collection("luogan", "distric_feature_0119", "192.168.0.136", 27017)
|
||||
for err_count in range(0, 10):
|
||||
|
||||
try:
|
||||
# print('flag4')
|
||||
lt = seawater105.count(query)
|
||||
|
||||
pos = list(seawater105.find(query).limit(lt))
|
||||
data = pandas.DataFrame(pos)
|
||||
del pos
|
||||
# print("==================>", time.time() - s)
|
||||
return data
|
||||
|
||||
except:
|
||||
|
||||
# err_count = err_count + 1
|
||||
lt = seawater105.count(query)
|
||||
pos = list(seawater105.find(query).limit(lt))
|
||||
data = pandas.DataFrame(pos)
|
||||
del pos
|
||||
return data
|
||||
|
||||
raise Exception("yang ben cha xun shi bai1")
|
||||
|
||||
@timer
|
||||
def find_seaweed(query, fields=None):
|
||||
# dirlt=p[0]
|
||||
|
||||
# client1 = pymongo.MongoClient('192.168.0.136',27017)
|
||||
# db1 = client1.fangjia
|
||||
# seaweed1 = db1.seaweed
|
||||
# s = time.time()
|
||||
seawater105 = get_collection("fangjia", "seaweed", "192.168.0.136", 27017)
|
||||
for err_count in range(0, 10):
|
||||
|
||||
try:
|
||||
# print('flag5')
|
||||
lt = seawater105.count(query)
|
||||
|
||||
pos = list(seawater105.find(query).limit(lt))
|
||||
data = pandas.DataFrame(pos)
|
||||
del pos
|
||||
# print("==================>", time.time() - s)
|
||||
return data
|
||||
|
||||
except:
|
||||
|
||||
# err_count = err_count + 1
|
||||
lt = seawater105.count(query)
|
||||
pos = list(seawater105.find(query).limit(lt))
|
||||
data = pandas.DataFrame(pos)
|
||||
del pos
|
||||
return data
|
||||
|
||||
raise Exception("yang ben cha xun shi bai1")
|
||||
|
||||
|
||||
def find_region_district_list_name(query, fields=None):
|
||||
# import pandas
|
||||
# client1 = pymongo.MongoClient('192.168.0.136',27017)
|
||||
# db1 = client1.fangjia
|
||||
# seaweed1 = db1.seaweed
|
||||
seawater105 = get_collection("fangjia", "seaweed", "192.168.0.136", 27017)
|
||||
for err_count in range(0, 10):
|
||||
|
||||
try:
|
||||
|
||||
lt = seawater105.count(query)
|
||||
|
||||
pos = list(seawater105.find(query).limit(lt))
|
||||
data = pandas.DataFrame(pos)
|
||||
del pos
|
||||
return data
|
||||
|
||||
except:
|
||||
|
||||
# err_count = err_count + 1
|
||||
lt = seawater105.count(query)
|
||||
pos = list(seawater105.find(query).limit(lt))
|
||||
data = pandas.DataFrame(pos)
|
||||
del pos
|
||||
return data
|
||||
Reference in New Issue
Block a user