初始化均价系统
This commit is contained in:
@@ -0,0 +1,23 @@
|
||||
from elasticsearch import Elasticsearch
|
||||
from pypinyin import lazy_pinyin
|
||||
from fangjia import mongoClients as mongoClients
|
||||
|
||||
city_province_map = {}
|
||||
es = Elasticsearch(['192.168.0.18'])
|
||||
|
||||
server_area = mongoClients.MongoClients("fangjia", "server_area", "mongo.db.fangjia.com").collection
|
||||
for area in server_area.find({"display": {'$ne': "全国"}}, {"display": 1, "province": 1}):
|
||||
city_province_map[area["display"]] = area["province"]
|
||||
|
||||
|
||||
def to_pinyin(hans):
|
||||
return "".join(lazy_pinyin(hans))
|
||||
|
||||
|
||||
def get_index_name(city, region):
|
||||
big_city = ["北京", "上海", "广州", "深圳", "天津", "杭州", "成都", "苏州", "重庆", "苏州", "南京", "武汉", "厦门"]
|
||||
if big_city.count(city) > 0:
|
||||
return "seawater-" + to_pinyin(city) + "-" + to_pinyin(region)
|
||||
if list(city_province_map.keys()).count(city) == 0:
|
||||
return "seawater-unknown"
|
||||
return "seawater-" + to_pinyin(city)
|
||||
@@ -0,0 +1,281 @@
|
||||
import pandas
|
||||
import time
|
||||
import logging
|
||||
import config
|
||||
from fangjia import mongoClients as mongoClients
|
||||
from fangjia import ElasticsearchClient as es
|
||||
|
||||
formatter = logging.Formatter('%(processName)s %(asctime)s %(name)-6s %(levelname)-6s %(message)s')
|
||||
|
||||
|
||||
def get_logger(name):
|
||||
logger = logging.getLogger(name)
|
||||
handler = logging.StreamHandler()
|
||||
handler.setFormatter(formatter)
|
||||
logger.addHandler(handler)
|
||||
|
||||
file_handler = logging.FileHandler("/home/goojia/logs/price.log")
|
||||
file_handler.setFormatter(formatter)
|
||||
logger.addHandler(file_handler)
|
||||
logger.setLevel("INFO")
|
||||
|
||||
return logger
|
||||
|
||||
|
||||
def get_performance_logger(name):
|
||||
logger = logging.getLogger(name)
|
||||
handler = logging.StreamHandler()
|
||||
handler.setFormatter(formatter)
|
||||
logger.addHandler(handler)
|
||||
|
||||
file_handler = logging.FileHandler("/home/goojia/logs/price.performance.log")
|
||||
file_handler.setFormatter(formatter)
|
||||
logger.addHandler(file_handler)
|
||||
logger.setLevel("INFO")
|
||||
|
||||
return logger
|
||||
|
||||
|
||||
performance_logger = get_performance_logger("performance")
|
||||
|
||||
|
||||
def timer(func):
|
||||
def _timer(*args, **kw):
|
||||
begin = time.clock()
|
||||
rs = func(*args, **kw)
|
||||
end = time.clock()
|
||||
elapsed = end - begin
|
||||
if elapsed > config.get_timeout():
|
||||
arg_list = list()
|
||||
for arg in args:
|
||||
arg_list.append(str(arg))
|
||||
performance_logger.info("函数名:(%s) 参数:(%s) 耗时:%f" % (func.__name__, arg_list, end - begin))
|
||||
return rs
|
||||
|
||||
return _timer
|
||||
|
||||
|
||||
def get_collection(db, name, host="192.168.0.136", port=27017):
|
||||
return mongoClients.MongoClients(db, name, host, port).collection
|
||||
|
||||
|
||||
def get_seawater_index_name(city, region):
|
||||
return es.get_index_name(city, region)
|
||||
|
||||
|
||||
@timer
|
||||
def findHouse(query, fields=None):
|
||||
# s = time.time()
|
||||
seawater105 = get_collection("fangjia", "seawater", "192.168.0.105", 27777)
|
||||
for err_count in range(0, 10):
|
||||
|
||||
try:
|
||||
|
||||
# lt = seawater105.count(query)
|
||||
|
||||
# print('flag1')
|
||||
lt = 1000
|
||||
# seawater.find(query,{"p_date":-1}).sort([("p_date",-1)])
|
||||
pos = list(seawater105.find(query).limit(lt))
|
||||
data = pandas.DataFrame(pos)
|
||||
del pos
|
||||
# print("==================>", time.time() - s)
|
||||
return data
|
||||
except:
|
||||
lt = seawater105.count(query)
|
||||
pos = list(seawater105.find(query).limit(lt))
|
||||
data = pandas.DataFrame(pos)
|
||||
del pos
|
||||
return data
|
||||
|
||||
raise Exception("样本查询失败!")
|
||||
|
||||
|
||||
@timer
|
||||
def find_house_from_elastic_search(city, region, alias, roots=None, pdate=None, cat="sell", limit=100):
|
||||
index_name = get_seawater_index_name(city, region)
|
||||
|
||||
routing_set = [cat + "_" + city + "_" + region + "_" + alia for alia in alias]
|
||||
must = [
|
||||
{
|
||||
"term": {"city": city}
|
||||
}, {
|
||||
"term": {"cat": cat}
|
||||
}, {
|
||||
"term": {"format_region": region}
|
||||
}, {
|
||||
"terms": {"district_name.raw": alias}
|
||||
}
|
||||
|
||||
]
|
||||
if roots is not None:
|
||||
must.append({
|
||||
"terms": {"root": roots}
|
||||
})
|
||||
|
||||
if pdate is not None:
|
||||
must.append({"range": {"p_date": {"gt": pdate}}})
|
||||
|
||||
body = {
|
||||
|
||||
"query": {
|
||||
"bool": {
|
||||
"must": must
|
||||
}
|
||||
},
|
||||
"from": 0,
|
||||
"size": limit,
|
||||
"sort": [{"p_date": "desc"}],
|
||||
"aggs": {}
|
||||
}
|
||||
response = es.es.search(index_name, "doc", routing=",".join(routing_set), body=body)
|
||||
return pandas.DataFrame([doc["_source"] for doc in response["hits"]["hits"]])
|
||||
|
||||
|
||||
@timer
|
||||
def find_sample_pool(query, fields=None):
|
||||
# s = time.time()
|
||||
seawater105 = get_collection("fangjia", "seawater", "192.168.0.105 ", 27777)
|
||||
for err_count in range(0, 10):
|
||||
|
||||
try:
|
||||
|
||||
# lt = seawater105.count(query)
|
||||
lt = 300
|
||||
# print('flag2')
|
||||
pos = list(seawater105.find(query).limit(lt))
|
||||
data = pandas.DataFrame(pos)
|
||||
del pos
|
||||
# print("==================>", time.time() - s)
|
||||
return data
|
||||
|
||||
except:
|
||||
|
||||
# err_count = err_count + 1
|
||||
# lt = seawater105.count(query)
|
||||
lt = 300
|
||||
pos = list(seawater105.find(query).limit(lt))
|
||||
data = pandas.DataFrame(pos)
|
||||
del pos
|
||||
return data
|
||||
|
||||
raise Exception("样本查询失败!")
|
||||
|
||||
|
||||
def find_disanganf_price(query, fields=None):
|
||||
# client1 = pymongo.MongoClient('192.168.0.136',27017)
|
||||
# db1 = client1.fangjia
|
||||
# dsf = db1.average_statistics
|
||||
# s = time.time()
|
||||
seawater105 = get_collection("fangjia", "average_statistics", "192.168.0.136", 27017)
|
||||
for err_count in range(0, 10):
|
||||
|
||||
try:
|
||||
# print('flag3')
|
||||
lt = seawater105.count(query)
|
||||
|
||||
pos = list(seawater105.find(query).limit(lt))
|
||||
data = pandas.DataFrame(pos)
|
||||
del pos
|
||||
# print("==================>", time.time() - s)
|
||||
return data
|
||||
|
||||
except:
|
||||
|
||||
# err_count = err_count + 1
|
||||
lt = seawater105.count(query)
|
||||
pos = list(seawater105.find(query).limit(lt))
|
||||
data = pandas.DataFrame(pos)
|
||||
del pos
|
||||
return data
|
||||
|
||||
raise Exception("yang ben cha xun shi bai1")
|
||||
|
||||
|
||||
def dis_feature(query, fields=None):
|
||||
# client1 = pymongo.MongoClient('192.168.0.136',27017)
|
||||
# db1 = client1.fangjia
|
||||
# dsf = db1.average_statistics
|
||||
# s = time.time()
|
||||
seawater105 = get_collection("luogan", "distric_feature_0119", "192.168.0.136", 27017)
|
||||
for err_count in range(0, 10):
|
||||
|
||||
try:
|
||||
# print('flag4')
|
||||
lt = seawater105.count(query)
|
||||
|
||||
pos = list(seawater105.find(query).limit(lt))
|
||||
data = pandas.DataFrame(pos)
|
||||
del pos
|
||||
# print("==================>", time.time() - s)
|
||||
return data
|
||||
|
||||
except:
|
||||
|
||||
# err_count = err_count + 1
|
||||
lt = seawater105.count(query)
|
||||
pos = list(seawater105.find(query).limit(lt))
|
||||
data = pandas.DataFrame(pos)
|
||||
del pos
|
||||
return data
|
||||
|
||||
raise Exception("yang ben cha xun shi bai1")
|
||||
|
||||
@timer
|
||||
def find_seaweed(query, fields=None):
|
||||
# dirlt=p[0]
|
||||
|
||||
# client1 = pymongo.MongoClient('192.168.0.136',27017)
|
||||
# db1 = client1.fangjia
|
||||
# seaweed1 = db1.seaweed
|
||||
# s = time.time()
|
||||
seawater105 = get_collection("fangjia", "seaweed", "192.168.0.136", 27017)
|
||||
for err_count in range(0, 10):
|
||||
|
||||
try:
|
||||
# print('flag5')
|
||||
lt = seawater105.count(query)
|
||||
|
||||
pos = list(seawater105.find(query).limit(lt))
|
||||
data = pandas.DataFrame(pos)
|
||||
del pos
|
||||
# print("==================>", time.time() - s)
|
||||
return data
|
||||
|
||||
except:
|
||||
|
||||
# err_count = err_count + 1
|
||||
lt = seawater105.count(query)
|
||||
pos = list(seawater105.find(query).limit(lt))
|
||||
data = pandas.DataFrame(pos)
|
||||
del pos
|
||||
return data
|
||||
|
||||
raise Exception("yang ben cha xun shi bai1")
|
||||
|
||||
|
||||
def find_region_district_list_name(query, fields=None):
|
||||
# import pandas
|
||||
# client1 = pymongo.MongoClient('192.168.0.136',27017)
|
||||
# db1 = client1.fangjia
|
||||
# seaweed1 = db1.seaweed
|
||||
seawater105 = get_collection("fangjia", "seaweed", "192.168.0.136", 27017)
|
||||
for err_count in range(0, 10):
|
||||
|
||||
try:
|
||||
|
||||
lt = seawater105.count(query)
|
||||
|
||||
pos = list(seawater105.find(query).limit(lt))
|
||||
data = pandas.DataFrame(pos)
|
||||
del pos
|
||||
return data
|
||||
|
||||
except:
|
||||
|
||||
# err_count = err_count + 1
|
||||
lt = seawater105.count(query)
|
||||
pos = list(seawater105.find(query).limit(lt))
|
||||
data = pandas.DataFrame(pos)
|
||||
del pos
|
||||
return data
|
||||
@@ -0,0 +1,33 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Created on Wed Nov 15 22:45:06 2017
|
||||
|
||||
@author: wenyan
|
||||
"""
|
||||
|
||||
from pymongo import MongoClient
|
||||
|
||||
|
||||
class MongoClients:
|
||||
|
||||
cache = {}
|
||||
|
||||
def __init__(self, db, name, host="127.0.0.0", port=27017):
|
||||
|
||||
key = db + "_" + name + "_" + host + "_" + str(port)
|
||||
if key in MongoClients.cache.keys():
|
||||
self.collection = MongoClients.cache[key]
|
||||
else:
|
||||
print("init " + key)
|
||||
collection = MongoClient(host, port).get_database(db).get_collection(name)
|
||||
MongoClients.cache[key] = collection
|
||||
self.collection = MongoClients.cache[key]
|
||||
"""
|
||||
def find(self, *args, **kwargs):
|
||||
return self.collection.find(*args, **kwargs)
|
||||
|
||||
def find2(self, query, fields, call):
|
||||
|
||||
for item in self.collection.find(query):
|
||||
call(item)
|
||||
"""
|
||||
@@ -0,0 +1,8 @@
|
||||
class CalcCommand:
|
||||
|
||||
def __init__(self, city, region, name, weekend):
|
||||
self.city = city
|
||||
self.region = region
|
||||
self.name = name
|
||||
self.weekend = weekend
|
||||
|
||||
Reference in New Issue
Block a user