初始化均价系统

This commit is contained in:
2018-03-07 09:59:22 +08:00
commit 933f2d7aec
16 changed files with 4350 additions and 0 deletions
+281
View File
@@ -0,0 +1,281 @@
import pandas
import time
import logging
import config
from fangjia import mongoClients as mongoClients
from fangjia import ElasticsearchClient as es
formatter = logging.Formatter('%(processName)s %(asctime)s %(name)-6s %(levelname)-6s %(message)s')
def get_logger(name):
logger = logging.getLogger(name)
handler = logging.StreamHandler()
handler.setFormatter(formatter)
logger.addHandler(handler)
file_handler = logging.FileHandler("/home/goojia/logs/price.log")
file_handler.setFormatter(formatter)
logger.addHandler(file_handler)
logger.setLevel("INFO")
return logger
def get_performance_logger(name):
logger = logging.getLogger(name)
handler = logging.StreamHandler()
handler.setFormatter(formatter)
logger.addHandler(handler)
file_handler = logging.FileHandler("/home/goojia/logs/price.performance.log")
file_handler.setFormatter(formatter)
logger.addHandler(file_handler)
logger.setLevel("INFO")
return logger
performance_logger = get_performance_logger("performance")
def timer(func):
def _timer(*args, **kw):
begin = time.clock()
rs = func(*args, **kw)
end = time.clock()
elapsed = end - begin
if elapsed > config.get_timeout():
arg_list = list()
for arg in args:
arg_list.append(str(arg))
performance_logger.info("函数名:(%s) 参数:(%s) 耗时:%f" % (func.__name__, arg_list, end - begin))
return rs
return _timer
def get_collection(db, name, host="192.168.0.136", port=27017):
return mongoClients.MongoClients(db, name, host, port).collection
def get_seawater_index_name(city, region):
return es.get_index_name(city, region)
@timer
def findHouse(query, fields=None):
# s = time.time()
seawater105 = get_collection("fangjia", "seawater", "192.168.0.105", 27777)
for err_count in range(0, 10):
try:
# lt = seawater105.count(query)
# print('flag1')
lt = 1000
# seawater.find(query,{"p_date":-1}).sort([("p_date",-1)])
pos = list(seawater105.find(query).limit(lt))
data = pandas.DataFrame(pos)
del pos
# print("==================>", time.time() - s)
return data
except:
lt = seawater105.count(query)
pos = list(seawater105.find(query).limit(lt))
data = pandas.DataFrame(pos)
del pos
return data
raise Exception("样本查询失败!")
@timer
def find_house_from_elastic_search(city, region, alias, roots=None, pdate=None, cat="sell", limit=100):
index_name = get_seawater_index_name(city, region)
routing_set = [cat + "_" + city + "_" + region + "_" + alia for alia in alias]
must = [
{
"term": {"city": city}
}, {
"term": {"cat": cat}
}, {
"term": {"format_region": region}
}, {
"terms": {"district_name.raw": alias}
}
]
if roots is not None:
must.append({
"terms": {"root": roots}
})
if pdate is not None:
must.append({"range": {"p_date": {"gt": pdate}}})
body = {
"query": {
"bool": {
"must": must
}
},
"from": 0,
"size": limit,
"sort": [{"p_date": "desc"}],
"aggs": {}
}
response = es.es.search(index_name, "doc", routing=",".join(routing_set), body=body)
return pandas.DataFrame([doc["_source"] for doc in response["hits"]["hits"]])
@timer
def find_sample_pool(query, fields=None):
# s = time.time()
seawater105 = get_collection("fangjia", "seawater", "192.168.0.105 ", 27777)
for err_count in range(0, 10):
try:
# lt = seawater105.count(query)
lt = 300
# print('flag2')
pos = list(seawater105.find(query).limit(lt))
data = pandas.DataFrame(pos)
del pos
# print("==================>", time.time() - s)
return data
except:
# err_count = err_count + 1
# lt = seawater105.count(query)
lt = 300
pos = list(seawater105.find(query).limit(lt))
data = pandas.DataFrame(pos)
del pos
return data
raise Exception("样本查询失败!")
def find_disanganf_price(query, fields=None):
# client1 = pymongo.MongoClient('192.168.0.136',27017)
# db1 = client1.fangjia
# dsf = db1.average_statistics
# s = time.time()
seawater105 = get_collection("fangjia", "average_statistics", "192.168.0.136", 27017)
for err_count in range(0, 10):
try:
# print('flag3')
lt = seawater105.count(query)
pos = list(seawater105.find(query).limit(lt))
data = pandas.DataFrame(pos)
del pos
# print("==================>", time.time() - s)
return data
except:
# err_count = err_count + 1
lt = seawater105.count(query)
pos = list(seawater105.find(query).limit(lt))
data = pandas.DataFrame(pos)
del pos
return data
raise Exception("yang ben cha xun shi bai1")
def dis_feature(query, fields=None):
# client1 = pymongo.MongoClient('192.168.0.136',27017)
# db1 = client1.fangjia
# dsf = db1.average_statistics
# s = time.time()
seawater105 = get_collection("luogan", "distric_feature_0119", "192.168.0.136", 27017)
for err_count in range(0, 10):
try:
# print('flag4')
lt = seawater105.count(query)
pos = list(seawater105.find(query).limit(lt))
data = pandas.DataFrame(pos)
del pos
# print("==================>", time.time() - s)
return data
except:
# err_count = err_count + 1
lt = seawater105.count(query)
pos = list(seawater105.find(query).limit(lt))
data = pandas.DataFrame(pos)
del pos
return data
raise Exception("yang ben cha xun shi bai1")
@timer
def find_seaweed(query, fields=None):
# dirlt=p[0]
# client1 = pymongo.MongoClient('192.168.0.136',27017)
# db1 = client1.fangjia
# seaweed1 = db1.seaweed
# s = time.time()
seawater105 = get_collection("fangjia", "seaweed", "192.168.0.136", 27017)
for err_count in range(0, 10):
try:
# print('flag5')
lt = seawater105.count(query)
pos = list(seawater105.find(query).limit(lt))
data = pandas.DataFrame(pos)
del pos
# print("==================>", time.time() - s)
return data
except:
# err_count = err_count + 1
lt = seawater105.count(query)
pos = list(seawater105.find(query).limit(lt))
data = pandas.DataFrame(pos)
del pos
return data
raise Exception("yang ben cha xun shi bai1")
def find_region_district_list_name(query, fields=None):
# import pandas
# client1 = pymongo.MongoClient('192.168.0.136',27017)
# db1 = client1.fangjia
# seaweed1 = db1.seaweed
seawater105 = get_collection("fangjia", "seaweed", "192.168.0.136", 27017)
for err_count in range(0, 10):
try:
lt = seawater105.count(query)
pos = list(seawater105.find(query).limit(lt))
data = pandas.DataFrame(pos)
del pos
return data
except:
# err_count = err_count + 1
lt = seawater105.count(query)
pos = list(seawater105.find(query).limit(lt))
data = pandas.DataFrame(pos)
del pos
return data