diff --git a/PriceSystem/potential_index_test/district_index.py b/PriceSystem/potential_index_test/district_index.py index 7d0ac8b..4f10703 100644 --- a/PriceSystem/potential_index_test/district_index.py +++ b/PriceSystem/potential_index_test/district_index.py @@ -1,17 +1,18 @@ -# -*- coding: utf-8 -*- -""" -Created on Mon Dec 18 15:45:01 2017 - -@author: Administrator -""" - # -*- coding: utf-8 -*- """ Created on Thu Dec 14 15:26:31 2017 -@author: Administrator +@author: luogan + +1:通过对时间序列进行分析 +构建七个指标分别对本小区进行评级 +判断未来涨跌 + """ +#导入相关包 + + import pymongo import pandas @@ -27,6 +28,7 @@ from matplotlib.finance import quotes_historical_yahoo_ohlc, candlestick_ohlc from matplotlib.pylab import date2num import potential +#talib 是一个股票技术分析的包 import talib now_time = datetime.datetime.now() diff --git a/PriceSystem/价格涨跌幅大处理/0529.xls b/PriceSystem/价格涨跌幅大处理/0529.xls new file mode 100644 index 0000000..6016b09 Binary files /dev/null and b/PriceSystem/价格涨跌幅大处理/0529.xls differ diff --git a/PriceSystem/价格涨跌幅大处理/CalcCmdReceiver.py b/PriceSystem/价格涨跌幅大处理/CalcCmdReceiver.py new file mode 100644 index 0000000..d068e84 --- /dev/null +++ b/PriceSystem/价格涨跌幅大处理/CalcCmdReceiver.py @@ -0,0 +1,45 @@ +import json +import multiprocessing +import time +import redis +#文件名 +import from_calc_to_avg_0601 + +QUEUE = "modify_save" +redis_pool = redis.ConnectionPool(host='192.168.0.144', port=6379, db=11) + + +def worker(pname): + client = redis.Redis(connection_pool=redis_pool) + while True: + try: + cmd = client.lpop(QUEUE) + except: + cmd = None + if cmd is None: + time.sleep(3) + print("sleep") + else: + cmd = format_cmd(cmd) + try: + #文件名对应的某个方法 + from_calc_to_avg_0601.modify_and_save(cmd["city"], cmd["region"], cmd["district_name"]) + print(pname + ":", cmd, "计算成功") + except Exception as ex: + print(ex) + print(pname + ":", cmd, "计算失败") + + +def format_cmd(cmd): + return json.loads(cmd) + + +if __name__ == "__main__": + + pro_num = 10 + pool = multiprocessing.Pool(processes=pro_num) + for pid in range(1, pro_num): + pid = "PROC" + str(pid).zfill(3) + pool.apply_async(worker, (pid,)) + pool.close() + pool.join() diff --git a/PriceSystem/价格涨跌幅大处理/CalcCmdSender.py b/PriceSystem/价格涨跌幅大处理/CalcCmdSender.py new file mode 100644 index 0000000..585b158 --- /dev/null +++ b/PriceSystem/价格涨跌幅大处理/CalcCmdSender.py @@ -0,0 +1,40 @@ +import json + +import redis + +#import fangjia + +QUEUE = "modify_save" +redisPool = redis.ConnectionPool(host='192.168.0.144', port=6379, db=11) +client = redis.Redis(connection_pool=redisPool) + + +def get_district(): + import pymongo + + client=pymongo.MongoClient('192.168.0.136',27017) + db_a=client.fangjia.dj + + seaweeds = [] + keys = {"city": 1, "region": 1, "district_name": 1, "_id": 0} + #for dis in fangjia.get_collection("fangjia", "seaweed").find({"city": "上海", "cat": "district", "status": 0}, keys): + for dis in db_a.find({},keys): + seaweeds.append(dis) + #for dis in fangjia.get_collection("fangjia", "seaweed").find({ "cat": "district", "status": 0}, keys): + + return seaweeds + + +def send_cmd(seaweed): + cmd = {} + cmd["city"] = seaweed["city"] + cmd["region"] = seaweed["region"] + cmd["district_name"] = seaweed["district_name"] + + json_cmd = json.dumps(seaweed, ensure_ascii=False) + client.rpush(QUEUE, json_cmd) + print("命令发送成功:" + json_cmd) + + +for seaweed in get_district(): + send_cmd(seaweed) diff --git a/PriceSystem/价格涨跌幅大处理/fangjia/ElasticsearchClient.py b/PriceSystem/价格涨跌幅大处理/fangjia/ElasticsearchClient.py new file mode 100644 index 0000000..3cbe793 --- /dev/null +++ b/PriceSystem/价格涨跌幅大处理/fangjia/ElasticsearchClient.py @@ -0,0 +1,23 @@ +from elasticsearch import Elasticsearch +from pypinyin import lazy_pinyin +from fangjia import mongoClients as mongoClients + +city_province_map = {} +es = Elasticsearch(['192.168.0.18']) + +server_area = mongoClients.MongoClients("fangjia", "server_area", "mongo.db.fangjia.com").collection +for area in server_area.find({"display": {'$ne': "全国"}}, {"display": 1, "province": 1}): + city_province_map[area["display"]] = area["province"] + + +def to_pinyin(hans): + return "".join(lazy_pinyin(hans)) + + +def get_index_name(city, region): + big_city = ["北京", "上海", "广州", "深圳", "天津", "杭州", "成都", "苏州", "重庆", "苏州", "南京", "武汉", "厦门"] + if big_city.count(city) > 0: + return "seawater-" + to_pinyin(city) + "-" + to_pinyin(region) + if list(city_province_map.keys()).count(city) == 0: + return "seawater-unknown" + return "seawater-" + to_pinyin(city) diff --git a/PriceSystem/价格涨跌幅大处理/fangjia/__init__.py b/PriceSystem/价格涨跌幅大处理/fangjia/__init__.py new file mode 100644 index 0000000..cf144c7 --- /dev/null +++ b/PriceSystem/价格涨跌幅大处理/fangjia/__init__.py @@ -0,0 +1,531 @@ +import pandas +import time +import logging +import config +import pytz +import datetime +import calendar +import requests +import json +import pickle +import redis +from fangjia import mongoClients as mongoClients +from fangjia import ElasticsearchClient as es +from fangjia.cachedObjectWrapper import CachedObjectWrapper +# + +formatter = logging.Formatter('%(processName)s %(asctime)s %(name)-6s %(levelname)-6s %(message)s') + + +def get_logger(name): + logger = logging.getLogger(name) + handler = logging.StreamHandler() + handler.setFormatter(formatter) + logger.addHandler(handler) + + file_handler = logging.FileHandler("/home/goojia/logs/" + name + ".log") + file_handler.setFormatter(formatter) + logger.addHandler(file_handler) + logger.setLevel("INFO") + + return logger + + +def get_performance_logger(name): + logger = logging.getLogger(name) + handler = logging.StreamHandler() + handler.setFormatter(formatter) + logger.addHandler(handler) + + file_handler = logging.FileHandler("/home/goojia/logs/" + name + ".performance.log") + file_handler.setFormatter(formatter) + logger.addHandler(file_handler) + logger.setLevel("INFO") + + return logger + +performance_logger = get_performance_logger("performance") + + +def timer(func): + def _timer(*args, **kw): + begin = time.clock() + rs = func(*args, **kw) + end = time.clock() + elapsed = end - begin + if elapsed > config.get_timeout(): + arg_list = list() + for arg in args: + arg_list.append(str(arg)) + + performance_logger.info("函数名:%-20s\t耗时:%7.4f\t\t参数:(%s)" % (func.__name__, end - begin, arg_list)) + + if elapsed > config.get_timeout(): + performance_logger.info("函数名:%-20s\t耗时:%7.4f\t\t参数:(%s)" % (func.__name__, end - begin, arg_list)) + + return rs + + return _timer + + +QUEUE = "price_system" +redis_pool = redis.ConnectionPool(host=config.get_redis_host(), port=6379, db=config.get_redis_cache_db()) +client = redis.Redis(connection_pool=redis_pool) + + +def cache(expire=None): + def _cache(func): + def __cache(*args, **kw): + arg_list = list() + for arg in args: + arg_list.append(str(arg)) + key = func.__name__ + "_" + "_".join(arg_list) + try: + val = client.hget(QUEUE, key) + if val is not None: + wrapper = pickle.loads(val) + if not wrapper.is_expire(): + return wrapper.val + else: + client.hset(QUEUE, key, None) + except Exception as ex: + print(ex) + rs = func(*args, **kw) + wrapper = CachedObjectWrapper(rs, time.time(), expire) + client.hset(QUEUE, key, pickle.dumps(wrapper)) + return rs + return __cache + + return _cache + + +def get_collection(db, name, host="192.168.0.136", port=27017): + return mongoClients.MongoClients(db, name, host, port).collection + + +def get_seawater_index_name(city, region): + return es.get_index_name(city, region) + + +def get_all_city(): + server_area = get_collection("fangjia", "server_area", config.get_mongodb_host(), config.get_mongodb_port()) + city_list = [] + for area in server_area.find().sort([("city_grade", 1), ("sequence", 1)]): + if "display" in area and area["display"] != "全国": + city_list.append(area["display"]) + return city_list + + +def get_region(city): + region_block = get_collection("fangjia", "region_block", config.get_mongodb_host(), config.get_mongodb_port()) + return [region["name"] for region in region_block.find({"category": "region", "city": city}, {"name": 1})] + + +def get_weekend(): + now = datetime.datetime.now().replace(hour=0, minute=0, second=0, microsecond=0) + for day in range(1, 9): + if calendar.weekday(now.year, now.month, now.day) == 6: + break + now = now + datetime.timedelta(days=1) + pacific = pytz.timezone('Asia/Shanghai') + return pacific.localize(now) + + +def send_ding_msg(msg): + headers = { + 'Content-Type': "application/json", + } + url = "https://oapi.dingtalk.com/robot/send" \ + "?access_token=2307d2fec8314b3e8b4b83dd1d42da2e2051638e3e668f21e77ca01258bf682c" + + text_body = { + "msgtype": "text", + "text": { + "content": msg + } + + } + requests.request("POST", url, data=json.dumps(text_body, ensure_ascii=False).encode('utf8'), headers=headers) + + + +@timer +def findHouse(query, fields=None): + seawater105 = get_collection("fangjia", "seawater", config.get_seawater_host(), 27777) + for err_count in range(0, 2): + try: + + + lt = 100 + # seawater.find(query,{"p_date":-1}).sort([("p_date",-1)]) + pos = list(seawater105.find(query, fields).sort([("p_date",-1)]).limit(lt)) + #pos = list(seawater105.find(query, fields).limit(lt)) + + + data = pandas.DataFrame(pos) + del pos + # print("==================>", time.time() - s) + return data + except: + lt = 100 + pos = list(seawater105.find(query, fields).sort([("p_date",-1)]).limit(lt)) + data = pandas.DataFrame(pos) + del pos + return data + + raise Exception("样本查询失败!") + + +def find_house_from_elastic_search(city, region, alias, roots=None, pdate=None, cat="sell", limit=1000): + index_name = get_seawater_index_name(city, region) + + routing_set = [cat + "_" + city + "_" + region + "_" + alia for alia in alias] + must = [ + { + "term": {"city": city} + }, { + "term": {"cat": cat} + }, { + "term": {"format_region": region} + }, { + "terms": {"district_name.raw": alias} + } + + ] + if roots is not None: + must.append({ + "terms": {"root": roots} + }) + + if pdate is not None: + must.append({"range": {"p_date": {"gt": pdate}}}) + + body = { + + "query": { + "bool": { + "must": must + } + }, + "from": 0, + "size": limit, + "sort": [{"p_date": "desc"}], + "aggs": {} + } + response = es.es.search(index_name, "doc", routing=",".join(routing_set), body=body) + return pandas.DataFrame([doc["_source"] for doc in response["hits"]["hits"]]) + + +def find_sample_pool(query, fields=None): + # s = time.time() + seawater105 = get_collection("fangjia", "seawater", config.get_seawater_host(), 27777) + for err_count in range(0, 2): + + try: + + # lt = seawater105.count(query) + lt = 300 + # print('flag2') + pos = list(seawater105.find(query).limit(lt)) + data = pandas.DataFrame(pos) + del pos + # print("==================>", time.time() - s) + return data + + except: + + # err_count = err_count + 1 + # lt = seawater105.count(query) + lt = 300 + pos = list(seawater105.find(query).limit(lt)) + data = pandas.DataFrame(pos) + del pos + return data + + raise Exception("样本查询失败!") + + +def find_disanganf_price(query, fields=None): + # client1 = pymongo.MongoClient('192.168.0.136',27017) + # db1 = client1.fangjia + # dsf = db1.average_statistics + # s = time.time() + seawater105 = get_collection("fangjia", "average_statistics",'mongo.db.fangjia.com', 27017) + for err_count in range(0, 2): + + try: + # print('flag3') + lt = seawater105.count(query) + + pos = list(seawater105.find(query).limit(lt)) + data = pandas.DataFrame(pos) + del pos + # print("==================>", time.time() - s) + return data + + except: + + # err_count = err_count + 1 + lt = seawater105.count(query) + pos = list(seawater105.find(query).limit(lt)) + data = pandas.DataFrame(pos) + del pos + return data + + raise Exception("yang ben cha xun shi bai1") + + +def dis_feature(query, fields=None): + # client1 = pymongo.MongoClient('192.168.0.136',27017) + # db1 = client1.fangjia + # dsf = db1.average_statistics + # s = time.time() + seawater105 = get_collection("fangjia_stat", "district_feature", 'mongo.db.fangjia.com', 27017) + for err_count in range(0, 2): + + try: + # print('flag4') + lt = seawater105.count(query) + + pos = list(seawater105.find(query).limit(lt)) + data = pandas.DataFrame(pos) + del pos + # print("==================>", time.time() - s) + return data + + except: + + # err_count = err_count + 1 + lt = seawater105.count(query) + pos = list(seawater105.find(query).limit(lt)) + data = pandas.DataFrame(pos) + del pos + return data + + raise Exception("yang ben cha xun shi bai1") + + +def find_seaweed(query, fields=None): + # dirlt=p[0] + + # client1 = pymongo.MongoClient('192.168.0.136',27017) + # db1 = client1.fangjia + # seaweed1 = db1.seaweed + # s = time.time() + seawater105 = get_collection("fangjia", "seaweed", config.get_seaweed_host(), 27017) + for err_count in range(0,2): + + try: + # print('flag5') + #lt = seawater105.count(query) + + pos = list(seawater105.find(query)) + data = pandas.DataFrame(pos) + del pos + # print("==================>", time.time() - s) + return data + + except: + + # err_count = err_count + 1 + #lt = seawater105.count(query) + pos = list(seawater105.find(query)) + data = pandas.DataFrame(pos) + del pos + return data + + raise Exception("yang ben cha xun shi bai1") + + + + +def find_region_price(query, fields=None): + # dirlt=p[0] + + # client1 = pymongo.MongoClient('192.168.0.136',27017) + # db1 = client1.fangjia + # seaweed1 = db1.seaweed + # s = time.time() + seawater105 = get_collection("fangjia", "region_stat", '192.168.0.136', 27017) + for err_count in range(0,2): + + try: + # print('flag5') + #lt = seawater105.count(query) + + pos = list(seawater105.find(query)) + data = pandas.DataFrame(pos) + del pos + # print("==================>", time.time() - s) + return data + + except: + + # err_count = err_count + 1 + #lt = seawater105.count(query) + pos = list(seawater105.find(query)) + data = pandas.DataFrame(pos) + del pos + return data + + raise Exception("yang ben cha xun shi bai1") + + +def find_region_district_list_name(query, fields=None): + # import pandas + # client1 = pymongo.MongoClient('192.168.0.136',27017) + # db1 = client1.fangjia + # seaweed1 = db1.seaweed + seawater105 = get_collection("fangjia", "seaweed", 'mongo.db.fangjia.com', 27017) + for err_count in range(0, 2): + + try: + + lt = seawater105.count(query) + + pos = list(seawater105.find(query).limit(lt)) + data = pandas.DataFrame(pos) + del pos + return data + + except: + + # err_count = err_count + 1 + lt = seawater105.count(query) + pos = list(seawater105.find(query).limit(lt)) + data = pandas.DataFrame(pos) + del pos + return data +########################################################################################## +# the code below is about computing rent price /m**2 + + +def rent_sample(query, fields): + # s = time.time() + seawater105 = get_collection("fangjia", "seawater", config.get_seawater_host(), 27777) + for err_count in range(0, 2): + + try: + # lt = seawater105.count(query) + lt = 1000 + # print('flag2') + pos=list(seawater105.find(query,fields).limit(lt)) + data=pandas.DataFrame(pos) + if data.shape[0]>10: + + #print('数据条数=',data.shape[0]) + return data[['district_name','total_price','area','p_date']] + else: + return pandas.DataFrame() + except: + + lt = 1000 + # print('flag2') + pos=list(seawater105.find(query,fields).limit(lt)) + data=pandas.DataFrame(pos) + if data.shape[0]>10: + + #print('数据条数=',data.shape[0]) + return data[['district_name','total_price','area','p_date']] + else: + return pandas.DataFrame() + raise Exception("样本查询失败!") + + + +def rent_disanganf_series(query, fields): + + # s = time.time() + seawater105 = get_collection("fangjia", "district_stat", 'mongo.db.fangjia.com', 27017) + for err_count in range(0, 2): + + try: + + pos=list(seawater105.find(query,fields)) + + print('pos=',pos) + data1=pandas.DataFrame(pos) + one_date=list(set(data1['weekend']))[1] + #two_date=list(set(data1['weekend']))[1] + data=data1[data1['weekend']==one_date] + + + d0=list(data['calc_value']) + d1=list(data['district_name']) + d2=pandas.Series(d0,index=d1) + return d2 + + except: + # print('flag2') + pos=list(seawater105.find(query,fields)) + data1=pandas.DataFrame(pos) + print('data1=',data1) + one_date=list(set(data1['weekend']))[1] + data=data1[data1['weekend']==one_date] + d0=list(data['calc_value']) + d1=list(data['district_name']) + d2=pandas.Series(d0,index=d1) + return d2 + + raise Exception("样本查询失败!") + +def rent_region_feature(query, fields): + rf = get_collection("fangjia", config.get_rent_temp_db(), '192.168.0.136', 27017) + #client182 = pymongo.MongoClient('192.168.10.182',27017) + #db182 = client182.fangjia_stat.rent_1221 + for err_count in range(0, 2): + + try: + pos=list(rf.find(query,fields)) + pos1=pandas.DataFrame(pos) + return pos1 + except: + pos=list(rf.find(query,fields)) + pos1=pandas.DataFrame(pos) + return pos1 + raise Exception("样本查询失败!") + """ + query1 = {"city":ct,"region":rl,'weekend':month_lastday(timm)} + a1=list(db182.find(query1)) + a2=pd.DataFrame(a1) + if a2.shape[0]>0: + ratio=a2['ratio'].iloc[0] + price=a2['avg_price'].iloc[0] + return (ratio,price) + else: + return () + """ + +def rent_find_disanganf_price(query, fields=None): + # client1 = pymongo.MongoClient('192.168.0.136',27017) + # db1 = client1.fangjia + # dsf = db1.average_statistics + # s = time.time() + seawater105 = get_collection("fangjia", "district_stat",'mongo.db.fangjia.com', 27017) + for err_count in range(0, 2): + + try: + # print('flag3') + lt = seawater105.count(query) + + pos = list(seawater105.find(query).limit(lt)) + data = pandas.DataFrame(pos) + del pos + # print("==================>", time.time() - s) + return data + + except: + + # err_count = err_count + 1 + lt = seawater105.count(query) + pos = list(seawater105.find(query).limit(lt)) + data = pandas.DataFrame(pos) + del pos + return data + + raise Exception("yang ben cha xun shi bai1") + + + + + diff --git a/PriceSystem/价格涨跌幅大处理/fangjia/cachedObjectWrapper.py b/PriceSystem/价格涨跌幅大处理/fangjia/cachedObjectWrapper.py new file mode 100644 index 0000000..4e9d56f --- /dev/null +++ b/PriceSystem/价格涨跌幅大处理/fangjia/cachedObjectWrapper.py @@ -0,0 +1,12 @@ +import time + + +class CachedObjectWrapper: + + def __init__(self, val, timestamp, cached_time=None): + self.val = val + self.timestamp = timestamp + self.cached_time = cached_time + + def is_expire(self): + return self.cached_time is not None and time.time() > self.cached_time + self.timestamp \ No newline at end of file diff --git a/PriceSystem/价格涨跌幅大处理/fangjia/mongoClients.py b/PriceSystem/价格涨跌幅大处理/fangjia/mongoClients.py new file mode 100644 index 0000000..3b08fce --- /dev/null +++ b/PriceSystem/价格涨跌幅大处理/fangjia/mongoClients.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- +""" +Created on Wed Nov 15 22:45:06 2017 + +@author: wenyan +""" + +from pymongo import MongoClient + + +class MongoClients: + + cache = {} + + def __init__(self, db, name, host="127.0.0.0", port=27017): + + key = db + "_" + name + "_" + host + "_" + str(port) + if key in MongoClients.cache.keys(): + self.collection = MongoClients.cache[key] + else: + print("init " + key) + collection = MongoClient(host, port).get_database(db).get_collection(name) + MongoClients.cache[key] = collection + self.collection = MongoClients.cache[key] + """ + def find(self, *args, **kwargs): + return self.collection.find(*args, **kwargs) + + def find2(self, query, fields, call): + + for item in self.collection.find(query): + call(item) + """ diff --git a/PriceSystem/价格涨跌幅大处理/fangjia/seaweed.py b/PriceSystem/价格涨跌幅大处理/fangjia/seaweed.py new file mode 100644 index 0000000..91e7207 --- /dev/null +++ b/PriceSystem/价格涨跌幅大处理/fangjia/seaweed.py @@ -0,0 +1,8 @@ +class CalcCommand: + + def __init__(self, city, region, name, weekend): + self.city = city + self.region = region + self.name = name + self.weekend = weekend + diff --git a/PriceSystem/价格涨跌幅大处理/from_calc_to_avg_0604.py b/PriceSystem/价格涨跌幅大处理/from_calc_to_avg_0604.py new file mode 100644 index 0000000..e944484 --- /dev/null +++ b/PriceSystem/价格涨跌幅大处理/from_calc_to_avg_0604.py @@ -0,0 +1,210 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +Created on Mon May 28 17:26:56 2018 + +@author: luogan +""" +import pymongo +import pytz +import pandas as pd +import numpy +import collections +import pandas +import time + + + +def get_weekend(tt): + + from dateutil.parser import parse + bb=parse(tt) + pacific = pytz.timezone('Asia/Shanghai') + return pacific.localize(bb) + + +weekend = get_weekend('20180309') +client=pymongo.MongoClient('192.168.10.182',27017) +db=client.fangjia.district_stat_0604 + + + +def data(ct,rl,lk): + + query={'weekend':{'$gte':weekend},'city':ct,'region':rl,'district_name':lk} + #query={'weekend':weekend,'city':'上海'} + + ll=list(db.find(query)) + + pl=pd.DataFrame(ll) + return pl + + + +#a=pd.read_excel('0529.xls') + +#a1=a[a['出现问题次数']>1] + + +def k_means(pp1): + from sklearn.cluster import KMeans + pv = list(pp1) + if len(set(pv)) > 3: + gf = numpy.array([pv]).T + estimator = KMeans(n_clusters=3) # 构造聚类器 + estimator.fit(gf) # 聚类 + label_pred = estimator.labels_ # 获取聚类标签 + aa = collections.Counter(label_pred) + v = pandas.Series(aa) + gg = list(v) + index_max = gg.index(max(gg)) + + centroids = estimator.cluster_centers_ # 获取聚类中心 + # inertia = estimator.inertia_ # 获取聚类准则的总和 + center = centroids[index_max][0] + return (float(center)) + else: + return float(pp1.mean()) + + +def modify(df,k,km_price): + + #print(k) + + this_value=df['avg_price'][k] + + rat=abs(this_value-km_price)/min(this_value,km_price) + + if rat>0.08: + df['avg_price'][k]=km_price + return df + else: + return df + +def mm(df,j,km_price): + if j==0: + df1=modify(df,j,km_price) + + df2=modify(df1,j+1,km_price) + return df2 + elif j==df.shape[0]-1: + df0=modify(df,j-1,km_price) + df1=modify(df0,j,km_price) + #df2=modify(df1,j+1,km_price) + return df1 + else: + print('#'*10) + df=modify(df,j-1,km_price) + df=modify(df,j,km_price) + df=modify(df,j+1,km_price) + return df + +''' +cl=pymongo.MongoClient('192.168.10.182',27017) +db182=cl.lg.district_stat +''' + +''' +for i in a.index: + + b1=a.iloc[i] + ct=b1['city'] + rl=b1['region'] + lk=b1['district_name'] + print(ct,rl,lk) + modify_and_save(ct,rl,lk) +''' + +def modify_and_save(ct,rl,lk): + + print(ct,rl,lk) + df=data(ct,rl,lk) + + #df_out=df[(df['markup']>1000) |(df['markup']<-1000) ] + + + + km_price= k_means(df['calc_value'].dropna()) + + + ''' + if df_out.shape[0]>0: + + for j in df_out.index: + df= mm(df,j,km_price) + #print(df) + #time.sleep(3) + ''' + for n in df.index: + + df=modify(df,n,km_price) + + df=df.sort_values(by='weekend',ascending=False) + + for m in range(df.shape[0]-1): + p1=df.iloc[m] + price=p1['avg_price'] + _id=p1['_id'] + + def mark_up(df,m): + + p2=df.iloc[m+1] + price_2=p2['avg_price'] + t1=p1['weekend'] + t2=p2['weekend'] + + delta=t1-t2 + + print(delta.days) + if abs(delta.days)==7: + make_up = 10000 * (price - price_2) / price_2 + + return int(make_up) + else: + return int(0) + + + mp=mark_up(df,m) + + print(mp) + + if abs(mp)>1000: + print(mp) + df['avg_price'].iloc[m]=km_price + + p2o=df.iloc[m+1] + price_2o=p2o['avg_price'] + mpo=10000 * (km_price - price_2o) / price_2o + df['markup'].iloc[m]=mpo + else: + df['markup'].iloc[m]=mp + + #print('df=',df) + for q in range(df.shape[0]-1): + + + avg=df['avg_price'].iloc[q] + mmp=int(df['markup'].iloc[q]) + _id=df['_id'].iloc[q] + + db.update( + {"_id": _id}, + { + + "$set": {'avg_price':int(avg),'markup':mmp, + + "weekend_markup":mmp,"changed_markup" :mmp } + + },True + + ) + + + + + + + + + + diff --git a/PriceSystem/价格涨跌幅大处理/redis/CalcCmdReceiver.py b/PriceSystem/价格涨跌幅大处理/redis/CalcCmdReceiver.py new file mode 100644 index 0000000..8103063 --- /dev/null +++ b/PriceSystem/价格涨跌幅大处理/redis/CalcCmdReceiver.py @@ -0,0 +1,45 @@ +import json +import multiprocessing +import time +import redis +#文件名 +import price_fix + +QUEUE = "calc_cmd" +redis_pool = redis.ConnectionPool(host='192.168.0.144', port=6379, db=11) + + +def worker(pname): + client = redis.Redis(connection_pool=redis_pool) + while True: + try: + cmd = client.lpop(QUEUE) + except: + cmd = None + if cmd is None: + time.sleep(3) + print("sleep") + else: + cmd = format_cmd(cmd) + try: + #文件名对应的某个方法 + price_fix.update(cmd["city"], cmd["region"], cmd["name"]) + print(pname + ":", cmd, "计算成功") + except Exception as ex: + print(ex) + print(pname + ":", cmd, "计算失败") + + +def format_cmd(cmd): + return json.loads(cmd) + + +if __name__ == "__main__": + + pro_num = 10 + pool = multiprocessing.Pool(processes=pro_num) + for pid in range(1, pro_num): + pid = "PROC" + str(pid).zfill(3) + pool.apply_async(worker, (pid,)) + pool.close() + pool.join() diff --git a/PriceSystem/价格涨跌幅大处理/redis/CalcCmdSender.py b/PriceSystem/价格涨跌幅大处理/redis/CalcCmdSender.py new file mode 100644 index 0000000..3f5bb9e --- /dev/null +++ b/PriceSystem/价格涨跌幅大处理/redis/CalcCmdSender.py @@ -0,0 +1,35 @@ +import json + +import redis + +import fangjia + +QUEUE = "calc_cmd" +redisPool = redis.ConnectionPool(host='192.168.0.144', port=6379, db=11) +client = redis.Redis(connection_pool=redisPool) + + +def get_district(): + seaweeds = [] + keys = {"city": 1, "region": 1, "name": 1, "_id": 0} + for dis in fangjia.get_collection("fangjia", "seaweed").find({"city": "上海", "cat": "district", "status": 0}, keys): + + #for dis in fangjia.get_collection("fangjia", "seaweed").find({ "cat": "district", "status": 0}, keys): + seaweeds.append(dis) + return seaweeds + + +def send_cmd(seaweed): + cmd = {} + cmd["city"] = seaweed["city"] + cmd["region"] = seaweed["region"] + cmd["name"] = seaweed["name"] + + json_cmd = json.dumps(seaweed, ensure_ascii=False) + client.rpush(QUEUE, json_cmd) + print("命令发送成功:" + json_cmd) + + +if __name__ == "__main__": + for seaweed in get_district(): + send_cmd(seaweed)