增加涨跌幅度较大小区
This commit is contained in:
@@ -1,17 +1,18 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Created on Mon Dec 18 15:45:01 2017
|
||||
|
||||
@author: Administrator
|
||||
"""
|
||||
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Created on Thu Dec 14 15:26:31 2017
|
||||
|
||||
@author: Administrator
|
||||
@author: luogan
|
||||
|
||||
1:通过对时间序列进行分析
|
||||
构建七个指标分别对本小区进行评级
|
||||
判断未来涨跌
|
||||
|
||||
"""
|
||||
|
||||
#导入相关包
|
||||
|
||||
|
||||
import pymongo
|
||||
import pandas
|
||||
|
||||
@@ -27,6 +28,7 @@ from matplotlib.finance import quotes_historical_yahoo_ohlc, candlestick_ohlc
|
||||
from matplotlib.pylab import date2num
|
||||
import potential
|
||||
|
||||
#talib 是一个股票技术分析的包
|
||||
import talib
|
||||
|
||||
now_time = datetime.datetime.now()
|
||||
|
||||
Binary file not shown.
@@ -0,0 +1,45 @@
|
||||
import json
|
||||
import multiprocessing
|
||||
import time
|
||||
import redis
|
||||
#文件名
|
||||
import from_calc_to_avg_0601
|
||||
|
||||
QUEUE = "modify_save"
|
||||
redis_pool = redis.ConnectionPool(host='192.168.0.144', port=6379, db=11)
|
||||
|
||||
|
||||
def worker(pname):
|
||||
client = redis.Redis(connection_pool=redis_pool)
|
||||
while True:
|
||||
try:
|
||||
cmd = client.lpop(QUEUE)
|
||||
except:
|
||||
cmd = None
|
||||
if cmd is None:
|
||||
time.sleep(3)
|
||||
print("sleep")
|
||||
else:
|
||||
cmd = format_cmd(cmd)
|
||||
try:
|
||||
#文件名对应的某个方法
|
||||
from_calc_to_avg_0601.modify_and_save(cmd["city"], cmd["region"], cmd["district_name"])
|
||||
print(pname + ":", cmd, "计算成功")
|
||||
except Exception as ex:
|
||||
print(ex)
|
||||
print(pname + ":", cmd, "计算失败")
|
||||
|
||||
|
||||
def format_cmd(cmd):
|
||||
return json.loads(cmd)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
pro_num = 10
|
||||
pool = multiprocessing.Pool(processes=pro_num)
|
||||
for pid in range(1, pro_num):
|
||||
pid = "PROC" + str(pid).zfill(3)
|
||||
pool.apply_async(worker, (pid,))
|
||||
pool.close()
|
||||
pool.join()
|
||||
@@ -0,0 +1,40 @@
|
||||
import json
|
||||
|
||||
import redis
|
||||
|
||||
#import fangjia
|
||||
|
||||
QUEUE = "modify_save"
|
||||
redisPool = redis.ConnectionPool(host='192.168.0.144', port=6379, db=11)
|
||||
client = redis.Redis(connection_pool=redisPool)
|
||||
|
||||
|
||||
def get_district():
|
||||
import pymongo
|
||||
|
||||
client=pymongo.MongoClient('192.168.0.136',27017)
|
||||
db_a=client.fangjia.dj
|
||||
|
||||
seaweeds = []
|
||||
keys = {"city": 1, "region": 1, "district_name": 1, "_id": 0}
|
||||
#for dis in fangjia.get_collection("fangjia", "seaweed").find({"city": "上海", "cat": "district", "status": 0}, keys):
|
||||
for dis in db_a.find({},keys):
|
||||
seaweeds.append(dis)
|
||||
#for dis in fangjia.get_collection("fangjia", "seaweed").find({ "cat": "district", "status": 0}, keys):
|
||||
|
||||
return seaweeds
|
||||
|
||||
|
||||
def send_cmd(seaweed):
|
||||
cmd = {}
|
||||
cmd["city"] = seaweed["city"]
|
||||
cmd["region"] = seaweed["region"]
|
||||
cmd["district_name"] = seaweed["district_name"]
|
||||
|
||||
json_cmd = json.dumps(seaweed, ensure_ascii=False)
|
||||
client.rpush(QUEUE, json_cmd)
|
||||
print("命令发送成功:" + json_cmd)
|
||||
|
||||
|
||||
for seaweed in get_district():
|
||||
send_cmd(seaweed)
|
||||
@@ -0,0 +1,23 @@
|
||||
from elasticsearch import Elasticsearch
|
||||
from pypinyin import lazy_pinyin
|
||||
from fangjia import mongoClients as mongoClients
|
||||
|
||||
city_province_map = {}
|
||||
es = Elasticsearch(['192.168.0.18'])
|
||||
|
||||
server_area = mongoClients.MongoClients("fangjia", "server_area", "mongo.db.fangjia.com").collection
|
||||
for area in server_area.find({"display": {'$ne': "全国"}}, {"display": 1, "province": 1}):
|
||||
city_province_map[area["display"]] = area["province"]
|
||||
|
||||
|
||||
def to_pinyin(hans):
|
||||
return "".join(lazy_pinyin(hans))
|
||||
|
||||
|
||||
def get_index_name(city, region):
|
||||
big_city = ["北京", "上海", "广州", "深圳", "天津", "杭州", "成都", "苏州", "重庆", "苏州", "南京", "武汉", "厦门"]
|
||||
if big_city.count(city) > 0:
|
||||
return "seawater-" + to_pinyin(city) + "-" + to_pinyin(region)
|
||||
if list(city_province_map.keys()).count(city) == 0:
|
||||
return "seawater-unknown"
|
||||
return "seawater-" + to_pinyin(city)
|
||||
@@ -0,0 +1,531 @@
|
||||
import pandas
|
||||
import time
|
||||
import logging
|
||||
import config
|
||||
import pytz
|
||||
import datetime
|
||||
import calendar
|
||||
import requests
|
||||
import json
|
||||
import pickle
|
||||
import redis
|
||||
from fangjia import mongoClients as mongoClients
|
||||
from fangjia import ElasticsearchClient as es
|
||||
from fangjia.cachedObjectWrapper import CachedObjectWrapper
|
||||
#
|
||||
|
||||
formatter = logging.Formatter('%(processName)s %(asctime)s %(name)-6s %(levelname)-6s %(message)s')
|
||||
|
||||
|
||||
def get_logger(name):
|
||||
logger = logging.getLogger(name)
|
||||
handler = logging.StreamHandler()
|
||||
handler.setFormatter(formatter)
|
||||
logger.addHandler(handler)
|
||||
|
||||
file_handler = logging.FileHandler("/home/goojia/logs/" + name + ".log")
|
||||
file_handler.setFormatter(formatter)
|
||||
logger.addHandler(file_handler)
|
||||
logger.setLevel("INFO")
|
||||
|
||||
return logger
|
||||
|
||||
|
||||
def get_performance_logger(name):
|
||||
logger = logging.getLogger(name)
|
||||
handler = logging.StreamHandler()
|
||||
handler.setFormatter(formatter)
|
||||
logger.addHandler(handler)
|
||||
|
||||
file_handler = logging.FileHandler("/home/goojia/logs/" + name + ".performance.log")
|
||||
file_handler.setFormatter(formatter)
|
||||
logger.addHandler(file_handler)
|
||||
logger.setLevel("INFO")
|
||||
|
||||
return logger
|
||||
|
||||
performance_logger = get_performance_logger("performance")
|
||||
|
||||
|
||||
def timer(func):
|
||||
def _timer(*args, **kw):
|
||||
begin = time.clock()
|
||||
rs = func(*args, **kw)
|
||||
end = time.clock()
|
||||
elapsed = end - begin
|
||||
if elapsed > config.get_timeout():
|
||||
arg_list = list()
|
||||
for arg in args:
|
||||
arg_list.append(str(arg))
|
||||
|
||||
performance_logger.info("函数名:%-20s\t耗时:%7.4f\t\t参数:(%s)" % (func.__name__, end - begin, arg_list))
|
||||
|
||||
if elapsed > config.get_timeout():
|
||||
performance_logger.info("函数名:%-20s\t耗时:%7.4f\t\t参数:(%s)" % (func.__name__, end - begin, arg_list))
|
||||
|
||||
return rs
|
||||
|
||||
return _timer
|
||||
|
||||
|
||||
QUEUE = "price_system"
|
||||
redis_pool = redis.ConnectionPool(host=config.get_redis_host(), port=6379, db=config.get_redis_cache_db())
|
||||
client = redis.Redis(connection_pool=redis_pool)
|
||||
|
||||
|
||||
def cache(expire=None):
|
||||
def _cache(func):
|
||||
def __cache(*args, **kw):
|
||||
arg_list = list()
|
||||
for arg in args:
|
||||
arg_list.append(str(arg))
|
||||
key = func.__name__ + "_" + "_".join(arg_list)
|
||||
try:
|
||||
val = client.hget(QUEUE, key)
|
||||
if val is not None:
|
||||
wrapper = pickle.loads(val)
|
||||
if not wrapper.is_expire():
|
||||
return wrapper.val
|
||||
else:
|
||||
client.hset(QUEUE, key, None)
|
||||
except Exception as ex:
|
||||
print(ex)
|
||||
rs = func(*args, **kw)
|
||||
wrapper = CachedObjectWrapper(rs, time.time(), expire)
|
||||
client.hset(QUEUE, key, pickle.dumps(wrapper))
|
||||
return rs
|
||||
return __cache
|
||||
|
||||
return _cache
|
||||
|
||||
|
||||
def get_collection(db, name, host="192.168.0.136", port=27017):
|
||||
return mongoClients.MongoClients(db, name, host, port).collection
|
||||
|
||||
|
||||
def get_seawater_index_name(city, region):
|
||||
return es.get_index_name(city, region)
|
||||
|
||||
|
||||
def get_all_city():
|
||||
server_area = get_collection("fangjia", "server_area", config.get_mongodb_host(), config.get_mongodb_port())
|
||||
city_list = []
|
||||
for area in server_area.find().sort([("city_grade", 1), ("sequence", 1)]):
|
||||
if "display" in area and area["display"] != "全国":
|
||||
city_list.append(area["display"])
|
||||
return city_list
|
||||
|
||||
|
||||
def get_region(city):
|
||||
region_block = get_collection("fangjia", "region_block", config.get_mongodb_host(), config.get_mongodb_port())
|
||||
return [region["name"] for region in region_block.find({"category": "region", "city": city}, {"name": 1})]
|
||||
|
||||
|
||||
def get_weekend():
|
||||
now = datetime.datetime.now().replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
for day in range(1, 9):
|
||||
if calendar.weekday(now.year, now.month, now.day) == 6:
|
||||
break
|
||||
now = now + datetime.timedelta(days=1)
|
||||
pacific = pytz.timezone('Asia/Shanghai')
|
||||
return pacific.localize(now)
|
||||
|
||||
|
||||
def send_ding_msg(msg):
|
||||
headers = {
|
||||
'Content-Type': "application/json",
|
||||
}
|
||||
url = "https://oapi.dingtalk.com/robot/send" \
|
||||
"?access_token=2307d2fec8314b3e8b4b83dd1d42da2e2051638e3e668f21e77ca01258bf682c"
|
||||
|
||||
text_body = {
|
||||
"msgtype": "text",
|
||||
"text": {
|
||||
"content": msg
|
||||
}
|
||||
|
||||
}
|
||||
requests.request("POST", url, data=json.dumps(text_body, ensure_ascii=False).encode('utf8'), headers=headers)
|
||||
|
||||
|
||||
|
||||
@timer
|
||||
def findHouse(query, fields=None):
|
||||
seawater105 = get_collection("fangjia", "seawater", config.get_seawater_host(), 27777)
|
||||
for err_count in range(0, 2):
|
||||
try:
|
||||
|
||||
|
||||
lt = 100
|
||||
# seawater.find(query,{"p_date":-1}).sort([("p_date",-1)])
|
||||
pos = list(seawater105.find(query, fields).sort([("p_date",-1)]).limit(lt))
|
||||
#pos = list(seawater105.find(query, fields).limit(lt))
|
||||
|
||||
|
||||
data = pandas.DataFrame(pos)
|
||||
del pos
|
||||
# print("==================>", time.time() - s)
|
||||
return data
|
||||
except:
|
||||
lt = 100
|
||||
pos = list(seawater105.find(query, fields).sort([("p_date",-1)]).limit(lt))
|
||||
data = pandas.DataFrame(pos)
|
||||
del pos
|
||||
return data
|
||||
|
||||
raise Exception("样本查询失败!")
|
||||
|
||||
|
||||
def find_house_from_elastic_search(city, region, alias, roots=None, pdate=None, cat="sell", limit=1000):
|
||||
index_name = get_seawater_index_name(city, region)
|
||||
|
||||
routing_set = [cat + "_" + city + "_" + region + "_" + alia for alia in alias]
|
||||
must = [
|
||||
{
|
||||
"term": {"city": city}
|
||||
}, {
|
||||
"term": {"cat": cat}
|
||||
}, {
|
||||
"term": {"format_region": region}
|
||||
}, {
|
||||
"terms": {"district_name.raw": alias}
|
||||
}
|
||||
|
||||
]
|
||||
if roots is not None:
|
||||
must.append({
|
||||
"terms": {"root": roots}
|
||||
})
|
||||
|
||||
if pdate is not None:
|
||||
must.append({"range": {"p_date": {"gt": pdate}}})
|
||||
|
||||
body = {
|
||||
|
||||
"query": {
|
||||
"bool": {
|
||||
"must": must
|
||||
}
|
||||
},
|
||||
"from": 0,
|
||||
"size": limit,
|
||||
"sort": [{"p_date": "desc"}],
|
||||
"aggs": {}
|
||||
}
|
||||
response = es.es.search(index_name, "doc", routing=",".join(routing_set), body=body)
|
||||
return pandas.DataFrame([doc["_source"] for doc in response["hits"]["hits"]])
|
||||
|
||||
|
||||
def find_sample_pool(query, fields=None):
|
||||
# s = time.time()
|
||||
seawater105 = get_collection("fangjia", "seawater", config.get_seawater_host(), 27777)
|
||||
for err_count in range(0, 2):
|
||||
|
||||
try:
|
||||
|
||||
# lt = seawater105.count(query)
|
||||
lt = 300
|
||||
# print('flag2')
|
||||
pos = list(seawater105.find(query).limit(lt))
|
||||
data = pandas.DataFrame(pos)
|
||||
del pos
|
||||
# print("==================>", time.time() - s)
|
||||
return data
|
||||
|
||||
except:
|
||||
|
||||
# err_count = err_count + 1
|
||||
# lt = seawater105.count(query)
|
||||
lt = 300
|
||||
pos = list(seawater105.find(query).limit(lt))
|
||||
data = pandas.DataFrame(pos)
|
||||
del pos
|
||||
return data
|
||||
|
||||
raise Exception("样本查询失败!")
|
||||
|
||||
|
||||
def find_disanganf_price(query, fields=None):
|
||||
# client1 = pymongo.MongoClient('192.168.0.136',27017)
|
||||
# db1 = client1.fangjia
|
||||
# dsf = db1.average_statistics
|
||||
# s = time.time()
|
||||
seawater105 = get_collection("fangjia", "average_statistics",'mongo.db.fangjia.com', 27017)
|
||||
for err_count in range(0, 2):
|
||||
|
||||
try:
|
||||
# print('flag3')
|
||||
lt = seawater105.count(query)
|
||||
|
||||
pos = list(seawater105.find(query).limit(lt))
|
||||
data = pandas.DataFrame(pos)
|
||||
del pos
|
||||
# print("==================>", time.time() - s)
|
||||
return data
|
||||
|
||||
except:
|
||||
|
||||
# err_count = err_count + 1
|
||||
lt = seawater105.count(query)
|
||||
pos = list(seawater105.find(query).limit(lt))
|
||||
data = pandas.DataFrame(pos)
|
||||
del pos
|
||||
return data
|
||||
|
||||
raise Exception("yang ben cha xun shi bai1")
|
||||
|
||||
|
||||
def dis_feature(query, fields=None):
|
||||
# client1 = pymongo.MongoClient('192.168.0.136',27017)
|
||||
# db1 = client1.fangjia
|
||||
# dsf = db1.average_statistics
|
||||
# s = time.time()
|
||||
seawater105 = get_collection("fangjia_stat", "district_feature", 'mongo.db.fangjia.com', 27017)
|
||||
for err_count in range(0, 2):
|
||||
|
||||
try:
|
||||
# print('flag4')
|
||||
lt = seawater105.count(query)
|
||||
|
||||
pos = list(seawater105.find(query).limit(lt))
|
||||
data = pandas.DataFrame(pos)
|
||||
del pos
|
||||
# print("==================>", time.time() - s)
|
||||
return data
|
||||
|
||||
except:
|
||||
|
||||
# err_count = err_count + 1
|
||||
lt = seawater105.count(query)
|
||||
pos = list(seawater105.find(query).limit(lt))
|
||||
data = pandas.DataFrame(pos)
|
||||
del pos
|
||||
return data
|
||||
|
||||
raise Exception("yang ben cha xun shi bai1")
|
||||
|
||||
|
||||
def find_seaweed(query, fields=None):
|
||||
# dirlt=p[0]
|
||||
|
||||
# client1 = pymongo.MongoClient('192.168.0.136',27017)
|
||||
# db1 = client1.fangjia
|
||||
# seaweed1 = db1.seaweed
|
||||
# s = time.time()
|
||||
seawater105 = get_collection("fangjia", "seaweed", config.get_seaweed_host(), 27017)
|
||||
for err_count in range(0,2):
|
||||
|
||||
try:
|
||||
# print('flag5')
|
||||
#lt = seawater105.count(query)
|
||||
|
||||
pos = list(seawater105.find(query))
|
||||
data = pandas.DataFrame(pos)
|
||||
del pos
|
||||
# print("==================>", time.time() - s)
|
||||
return data
|
||||
|
||||
except:
|
||||
|
||||
# err_count = err_count + 1
|
||||
#lt = seawater105.count(query)
|
||||
pos = list(seawater105.find(query))
|
||||
data = pandas.DataFrame(pos)
|
||||
del pos
|
||||
return data
|
||||
|
||||
raise Exception("yang ben cha xun shi bai1")
|
||||
|
||||
|
||||
|
||||
|
||||
def find_region_price(query, fields=None):
|
||||
# dirlt=p[0]
|
||||
|
||||
# client1 = pymongo.MongoClient('192.168.0.136',27017)
|
||||
# db1 = client1.fangjia
|
||||
# seaweed1 = db1.seaweed
|
||||
# s = time.time()
|
||||
seawater105 = get_collection("fangjia", "region_stat", '192.168.0.136', 27017)
|
||||
for err_count in range(0,2):
|
||||
|
||||
try:
|
||||
# print('flag5')
|
||||
#lt = seawater105.count(query)
|
||||
|
||||
pos = list(seawater105.find(query))
|
||||
data = pandas.DataFrame(pos)
|
||||
del pos
|
||||
# print("==================>", time.time() - s)
|
||||
return data
|
||||
|
||||
except:
|
||||
|
||||
# err_count = err_count + 1
|
||||
#lt = seawater105.count(query)
|
||||
pos = list(seawater105.find(query))
|
||||
data = pandas.DataFrame(pos)
|
||||
del pos
|
||||
return data
|
||||
|
||||
raise Exception("yang ben cha xun shi bai1")
|
||||
|
||||
|
||||
def find_region_district_list_name(query, fields=None):
|
||||
# import pandas
|
||||
# client1 = pymongo.MongoClient('192.168.0.136',27017)
|
||||
# db1 = client1.fangjia
|
||||
# seaweed1 = db1.seaweed
|
||||
seawater105 = get_collection("fangjia", "seaweed", 'mongo.db.fangjia.com', 27017)
|
||||
for err_count in range(0, 2):
|
||||
|
||||
try:
|
||||
|
||||
lt = seawater105.count(query)
|
||||
|
||||
pos = list(seawater105.find(query).limit(lt))
|
||||
data = pandas.DataFrame(pos)
|
||||
del pos
|
||||
return data
|
||||
|
||||
except:
|
||||
|
||||
# err_count = err_count + 1
|
||||
lt = seawater105.count(query)
|
||||
pos = list(seawater105.find(query).limit(lt))
|
||||
data = pandas.DataFrame(pos)
|
||||
del pos
|
||||
return data
|
||||
##########################################################################################
|
||||
# the code below is about computing rent price /m**2
|
||||
|
||||
|
||||
def rent_sample(query, fields):
|
||||
# s = time.time()
|
||||
seawater105 = get_collection("fangjia", "seawater", config.get_seawater_host(), 27777)
|
||||
for err_count in range(0, 2):
|
||||
|
||||
try:
|
||||
# lt = seawater105.count(query)
|
||||
lt = 1000
|
||||
# print('flag2')
|
||||
pos=list(seawater105.find(query,fields).limit(lt))
|
||||
data=pandas.DataFrame(pos)
|
||||
if data.shape[0]>10:
|
||||
|
||||
#print('数据条数=',data.shape[0])
|
||||
return data[['district_name','total_price','area','p_date']]
|
||||
else:
|
||||
return pandas.DataFrame()
|
||||
except:
|
||||
|
||||
lt = 1000
|
||||
# print('flag2')
|
||||
pos=list(seawater105.find(query,fields).limit(lt))
|
||||
data=pandas.DataFrame(pos)
|
||||
if data.shape[0]>10:
|
||||
|
||||
#print('数据条数=',data.shape[0])
|
||||
return data[['district_name','total_price','area','p_date']]
|
||||
else:
|
||||
return pandas.DataFrame()
|
||||
raise Exception("样本查询失败!")
|
||||
|
||||
|
||||
|
||||
def rent_disanganf_series(query, fields):
|
||||
|
||||
# s = time.time()
|
||||
seawater105 = get_collection("fangjia", "district_stat", 'mongo.db.fangjia.com', 27017)
|
||||
for err_count in range(0, 2):
|
||||
|
||||
try:
|
||||
|
||||
pos=list(seawater105.find(query,fields))
|
||||
|
||||
print('pos=',pos)
|
||||
data1=pandas.DataFrame(pos)
|
||||
one_date=list(set(data1['weekend']))[1]
|
||||
#two_date=list(set(data1['weekend']))[1]
|
||||
data=data1[data1['weekend']==one_date]
|
||||
|
||||
|
||||
d0=list(data['calc_value'])
|
||||
d1=list(data['district_name'])
|
||||
d2=pandas.Series(d0,index=d1)
|
||||
return d2
|
||||
|
||||
except:
|
||||
# print('flag2')
|
||||
pos=list(seawater105.find(query,fields))
|
||||
data1=pandas.DataFrame(pos)
|
||||
print('data1=',data1)
|
||||
one_date=list(set(data1['weekend']))[1]
|
||||
data=data1[data1['weekend']==one_date]
|
||||
d0=list(data['calc_value'])
|
||||
d1=list(data['district_name'])
|
||||
d2=pandas.Series(d0,index=d1)
|
||||
return d2
|
||||
|
||||
raise Exception("样本查询失败!")
|
||||
|
||||
def rent_region_feature(query, fields):
|
||||
rf = get_collection("fangjia", config.get_rent_temp_db(), '192.168.0.136', 27017)
|
||||
#client182 = pymongo.MongoClient('192.168.10.182',27017)
|
||||
#db182 = client182.fangjia_stat.rent_1221
|
||||
for err_count in range(0, 2):
|
||||
|
||||
try:
|
||||
pos=list(rf.find(query,fields))
|
||||
pos1=pandas.DataFrame(pos)
|
||||
return pos1
|
||||
except:
|
||||
pos=list(rf.find(query,fields))
|
||||
pos1=pandas.DataFrame(pos)
|
||||
return pos1
|
||||
raise Exception("样本查询失败!")
|
||||
"""
|
||||
query1 = {"city":ct,"region":rl,'weekend':month_lastday(timm)}
|
||||
a1=list(db182.find(query1))
|
||||
a2=pd.DataFrame(a1)
|
||||
if a2.shape[0]>0:
|
||||
ratio=a2['ratio'].iloc[0]
|
||||
price=a2['avg_price'].iloc[0]
|
||||
return (ratio,price)
|
||||
else:
|
||||
return ()
|
||||
"""
|
||||
|
||||
def rent_find_disanganf_price(query, fields=None):
|
||||
# client1 = pymongo.MongoClient('192.168.0.136',27017)
|
||||
# db1 = client1.fangjia
|
||||
# dsf = db1.average_statistics
|
||||
# s = time.time()
|
||||
seawater105 = get_collection("fangjia", "district_stat",'mongo.db.fangjia.com', 27017)
|
||||
for err_count in range(0, 2):
|
||||
|
||||
try:
|
||||
# print('flag3')
|
||||
lt = seawater105.count(query)
|
||||
|
||||
pos = list(seawater105.find(query).limit(lt))
|
||||
data = pandas.DataFrame(pos)
|
||||
del pos
|
||||
# print("==================>", time.time() - s)
|
||||
return data
|
||||
|
||||
except:
|
||||
|
||||
# err_count = err_count + 1
|
||||
lt = seawater105.count(query)
|
||||
pos = list(seawater105.find(query).limit(lt))
|
||||
data = pandas.DataFrame(pos)
|
||||
del pos
|
||||
return data
|
||||
|
||||
raise Exception("yang ben cha xun shi bai1")
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
import time
|
||||
|
||||
|
||||
class CachedObjectWrapper:
|
||||
|
||||
def __init__(self, val, timestamp, cached_time=None):
|
||||
self.val = val
|
||||
self.timestamp = timestamp
|
||||
self.cached_time = cached_time
|
||||
|
||||
def is_expire(self):
|
||||
return self.cached_time is not None and time.time() > self.cached_time + self.timestamp
|
||||
@@ -0,0 +1,33 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Created on Wed Nov 15 22:45:06 2017
|
||||
|
||||
@author: wenyan
|
||||
"""
|
||||
|
||||
from pymongo import MongoClient
|
||||
|
||||
|
||||
class MongoClients:
|
||||
|
||||
cache = {}
|
||||
|
||||
def __init__(self, db, name, host="127.0.0.0", port=27017):
|
||||
|
||||
key = db + "_" + name + "_" + host + "_" + str(port)
|
||||
if key in MongoClients.cache.keys():
|
||||
self.collection = MongoClients.cache[key]
|
||||
else:
|
||||
print("init " + key)
|
||||
collection = MongoClient(host, port).get_database(db).get_collection(name)
|
||||
MongoClients.cache[key] = collection
|
||||
self.collection = MongoClients.cache[key]
|
||||
"""
|
||||
def find(self, *args, **kwargs):
|
||||
return self.collection.find(*args, **kwargs)
|
||||
|
||||
def find2(self, query, fields, call):
|
||||
|
||||
for item in self.collection.find(query):
|
||||
call(item)
|
||||
"""
|
||||
@@ -0,0 +1,8 @@
|
||||
class CalcCommand:
|
||||
|
||||
def __init__(self, city, region, name, weekend):
|
||||
self.city = city
|
||||
self.region = region
|
||||
self.name = name
|
||||
self.weekend = weekend
|
||||
|
||||
@@ -0,0 +1,210 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Created on Mon May 28 17:26:56 2018
|
||||
|
||||
@author: luogan
|
||||
"""
|
||||
import pymongo
|
||||
import pytz
|
||||
import pandas as pd
|
||||
import numpy
|
||||
import collections
|
||||
import pandas
|
||||
import time
|
||||
|
||||
|
||||
|
||||
def get_weekend(tt):
|
||||
|
||||
from dateutil.parser import parse
|
||||
bb=parse(tt)
|
||||
pacific = pytz.timezone('Asia/Shanghai')
|
||||
return pacific.localize(bb)
|
||||
|
||||
|
||||
weekend = get_weekend('20180309')
|
||||
client=pymongo.MongoClient('192.168.10.182',27017)
|
||||
db=client.fangjia.district_stat_0604
|
||||
|
||||
|
||||
|
||||
def data(ct,rl,lk):
|
||||
|
||||
query={'weekend':{'$gte':weekend},'city':ct,'region':rl,'district_name':lk}
|
||||
#query={'weekend':weekend,'city':'上海'}
|
||||
|
||||
ll=list(db.find(query))
|
||||
|
||||
pl=pd.DataFrame(ll)
|
||||
return pl
|
||||
|
||||
|
||||
|
||||
#a=pd.read_excel('0529.xls')
|
||||
|
||||
#a1=a[a['出现问题次数']>1]
|
||||
|
||||
|
||||
def k_means(pp1):
|
||||
from sklearn.cluster import KMeans
|
||||
pv = list(pp1)
|
||||
if len(set(pv)) > 3:
|
||||
gf = numpy.array([pv]).T
|
||||
estimator = KMeans(n_clusters=3) # 构造聚类器
|
||||
estimator.fit(gf) # 聚类
|
||||
label_pred = estimator.labels_ # 获取聚类标签
|
||||
aa = collections.Counter(label_pred)
|
||||
v = pandas.Series(aa)
|
||||
gg = list(v)
|
||||
index_max = gg.index(max(gg))
|
||||
|
||||
centroids = estimator.cluster_centers_ # 获取聚类中心
|
||||
# inertia = estimator.inertia_ # 获取聚类准则的总和
|
||||
center = centroids[index_max][0]
|
||||
return (float(center))
|
||||
else:
|
||||
return float(pp1.mean())
|
||||
|
||||
|
||||
def modify(df,k,km_price):
|
||||
|
||||
#print(k)
|
||||
|
||||
this_value=df['avg_price'][k]
|
||||
|
||||
rat=abs(this_value-km_price)/min(this_value,km_price)
|
||||
|
||||
if rat>0.08:
|
||||
df['avg_price'][k]=km_price
|
||||
return df
|
||||
else:
|
||||
return df
|
||||
|
||||
def mm(df,j,km_price):
|
||||
if j==0:
|
||||
df1=modify(df,j,km_price)
|
||||
|
||||
df2=modify(df1,j+1,km_price)
|
||||
return df2
|
||||
elif j==df.shape[0]-1:
|
||||
df0=modify(df,j-1,km_price)
|
||||
df1=modify(df0,j,km_price)
|
||||
#df2=modify(df1,j+1,km_price)
|
||||
return df1
|
||||
else:
|
||||
print('#'*10)
|
||||
df=modify(df,j-1,km_price)
|
||||
df=modify(df,j,km_price)
|
||||
df=modify(df,j+1,km_price)
|
||||
return df
|
||||
|
||||
'''
|
||||
cl=pymongo.MongoClient('192.168.10.182',27017)
|
||||
db182=cl.lg.district_stat
|
||||
'''
|
||||
|
||||
'''
|
||||
for i in a.index:
|
||||
|
||||
b1=a.iloc[i]
|
||||
ct=b1['city']
|
||||
rl=b1['region']
|
||||
lk=b1['district_name']
|
||||
print(ct,rl,lk)
|
||||
modify_and_save(ct,rl,lk)
|
||||
'''
|
||||
|
||||
def modify_and_save(ct,rl,lk):
|
||||
|
||||
print(ct,rl,lk)
|
||||
df=data(ct,rl,lk)
|
||||
|
||||
#df_out=df[(df['markup']>1000) |(df['markup']<-1000) ]
|
||||
|
||||
|
||||
|
||||
km_price= k_means(df['calc_value'].dropna())
|
||||
|
||||
|
||||
'''
|
||||
if df_out.shape[0]>0:
|
||||
|
||||
for j in df_out.index:
|
||||
df= mm(df,j,km_price)
|
||||
#print(df)
|
||||
#time.sleep(3)
|
||||
'''
|
||||
for n in df.index:
|
||||
|
||||
df=modify(df,n,km_price)
|
||||
|
||||
df=df.sort_values(by='weekend',ascending=False)
|
||||
|
||||
for m in range(df.shape[0]-1):
|
||||
p1=df.iloc[m]
|
||||
price=p1['avg_price']
|
||||
_id=p1['_id']
|
||||
|
||||
def mark_up(df,m):
|
||||
|
||||
p2=df.iloc[m+1]
|
||||
price_2=p2['avg_price']
|
||||
t1=p1['weekend']
|
||||
t2=p2['weekend']
|
||||
|
||||
delta=t1-t2
|
||||
|
||||
print(delta.days)
|
||||
if abs(delta.days)==7:
|
||||
make_up = 10000 * (price - price_2) / price_2
|
||||
|
||||
return int(make_up)
|
||||
else:
|
||||
return int(0)
|
||||
|
||||
|
||||
mp=mark_up(df,m)
|
||||
|
||||
print(mp)
|
||||
|
||||
if abs(mp)>1000:
|
||||
print(mp)
|
||||
df['avg_price'].iloc[m]=km_price
|
||||
|
||||
p2o=df.iloc[m+1]
|
||||
price_2o=p2o['avg_price']
|
||||
mpo=10000 * (km_price - price_2o) / price_2o
|
||||
df['markup'].iloc[m]=mpo
|
||||
else:
|
||||
df['markup'].iloc[m]=mp
|
||||
|
||||
#print('df=',df)
|
||||
for q in range(df.shape[0]-1):
|
||||
|
||||
|
||||
avg=df['avg_price'].iloc[q]
|
||||
mmp=int(df['markup'].iloc[q])
|
||||
_id=df['_id'].iloc[q]
|
||||
|
||||
db.update(
|
||||
{"_id": _id},
|
||||
{
|
||||
|
||||
"$set": {'avg_price':int(avg),'markup':mmp,
|
||||
|
||||
"weekend_markup":mmp,"changed_markup" :mmp }
|
||||
|
||||
},True
|
||||
|
||||
)
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
import json
|
||||
import multiprocessing
|
||||
import time
|
||||
import redis
|
||||
#文件名
|
||||
import price_fix
|
||||
|
||||
QUEUE = "calc_cmd"
|
||||
redis_pool = redis.ConnectionPool(host='192.168.0.144', port=6379, db=11)
|
||||
|
||||
|
||||
def worker(pname):
|
||||
client = redis.Redis(connection_pool=redis_pool)
|
||||
while True:
|
||||
try:
|
||||
cmd = client.lpop(QUEUE)
|
||||
except:
|
||||
cmd = None
|
||||
if cmd is None:
|
||||
time.sleep(3)
|
||||
print("sleep")
|
||||
else:
|
||||
cmd = format_cmd(cmd)
|
||||
try:
|
||||
#文件名对应的某个方法
|
||||
price_fix.update(cmd["city"], cmd["region"], cmd["name"])
|
||||
print(pname + ":", cmd, "计算成功")
|
||||
except Exception as ex:
|
||||
print(ex)
|
||||
print(pname + ":", cmd, "计算失败")
|
||||
|
||||
|
||||
def format_cmd(cmd):
|
||||
return json.loads(cmd)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
pro_num = 10
|
||||
pool = multiprocessing.Pool(processes=pro_num)
|
||||
for pid in range(1, pro_num):
|
||||
pid = "PROC" + str(pid).zfill(3)
|
||||
pool.apply_async(worker, (pid,))
|
||||
pool.close()
|
||||
pool.join()
|
||||
@@ -0,0 +1,35 @@
|
||||
import json
|
||||
|
||||
import redis
|
||||
|
||||
import fangjia
|
||||
|
||||
QUEUE = "calc_cmd"
|
||||
redisPool = redis.ConnectionPool(host='192.168.0.144', port=6379, db=11)
|
||||
client = redis.Redis(connection_pool=redisPool)
|
||||
|
||||
|
||||
def get_district():
|
||||
seaweeds = []
|
||||
keys = {"city": 1, "region": 1, "name": 1, "_id": 0}
|
||||
for dis in fangjia.get_collection("fangjia", "seaweed").find({"city": "上海", "cat": "district", "status": 0}, keys):
|
||||
|
||||
#for dis in fangjia.get_collection("fangjia", "seaweed").find({ "cat": "district", "status": 0}, keys):
|
||||
seaweeds.append(dis)
|
||||
return seaweeds
|
||||
|
||||
|
||||
def send_cmd(seaweed):
|
||||
cmd = {}
|
||||
cmd["city"] = seaweed["city"]
|
||||
cmd["region"] = seaweed["region"]
|
||||
cmd["name"] = seaweed["name"]
|
||||
|
||||
json_cmd = json.dumps(seaweed, ensure_ascii=False)
|
||||
client.rpush(QUEUE, json_cmd)
|
||||
print("命令发送成功:" + json_cmd)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
for seaweed in get_district():
|
||||
send_cmd(seaweed)
|
||||
Reference in New Issue
Block a user