From a8c95698cdbf5f11ea49d42ced0f40fdf496a597 Mon Sep 17 00:00:00 2001 From: wangwenyan Date: Mon, 12 Mar 2018 12:58:05 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96=E5=9D=87=E4=BB=B7=E7=AE=97?= =?UTF-8?q?=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- PriceSystem/CalcCmdSender.py | 79 ++++++++++++++++++++++------ PriceSystem/CalcSpeedMonitor.py | 3 +- PriceSystem/config/__init__.py | 16 ++++-- PriceSystem/fangjia/__init__.py | 39 ++++++++++---- PriceSystem/sell_price/zhuge0118A.py | 8 ++- PriceSystem/test.py | 4 +- 6 files changed, 111 insertions(+), 38 deletions(-) diff --git a/PriceSystem/CalcCmdSender.py b/PriceSystem/CalcCmdSender.py index 972ef16..bce99f7 100644 --- a/PriceSystem/CalcCmdSender.py +++ b/PriceSystem/CalcCmdSender.py @@ -2,13 +2,16 @@ import json import redis import fangjia import config -from datetime import datetime, date +import datetime +import sys +import time +from datetime import date class CJsonEncoder(json.JSONEncoder): def default(self, obj): - if isinstance(obj, datetime): - return obj.strftime('%Y-%m-%d %H:%M:%S') + if isinstance(obj, datetime.datetime): + return obj.strftime('%Y-%m-%d') elif isinstance(obj, date): return obj.strftime('%Y-%m-%d') else: @@ -18,32 +21,74 @@ class CJsonEncoder(json.JSONEncoder): QUEUE = "calc_cmd" redisPool = redis.ConnectionPool(host=config.get_redis_host(), port=6379, db=config.get_redis_db()) client = redis.Redis(connection_pool=redisPool) +logger = fangjia.get_logger("CalcCmdReceiver") -def get_district(): +def get_district(city): + weekend = fangjia.get_weekend() seaweeds = [] keys = {"city": 1, "region": 1, "name": 1, "_id": 0} - for dis in fangjia.get_collection("fangjia", "seaweed").find({"cat": "district", "status": 0, "city": "合肥"}, keys): - dis["weekend"] = datetime.strptime("2018-03-11", "%Y-%m-%d") + for dis in fangjia.get_collection("fangjia", "seaweed").find({"cat": "district", "status": 0, "city": city}, keys): + dis["weekend"] = weekend seaweeds.append(dis) return seaweeds def send_cmd(seaweed): - """ - cmd = {} - cmd["city"] = seaweed["city"] - cmd["region"] = seaweed["region"] - cmd["name"] = seaweed["name"] - """ - json_cmd = json.dumps(seaweed, ensure_ascii=False, cls=CJsonEncoder) client.rpush(QUEUE, json_cmd) - print("命令发送成功:" + json_cmd) + logger.info("命令发送成功:" + json_cmd) + + +def send_district_calc_cmd(): + for city in fangjia.get_all_city(): + for seaweed in get_district(city): + send_cmd(seaweed) + + +def save_city_calc_command(city, region, weekend): + calc_command = fangjia.get_collection("fangjia_log", "calc_command", config.get_log_mongodb_host(), + config.get_log_mongodb_port()) + cmd = dict() + cmd["city"] = city + cmd["region"] = region + cmd["block"] = None + cmd["calc_date"] = weekend + cmd["calc_type"] = 0 + cmd["category"] = "sell" + cmd["priority"] = 1 + cmd["status"] = 0 + cmd["remark"] = 'week' + cmd["c_date"] = datetime.datetime.now().utcnow() + calc_command.insert_one(cmd) + if region is None: + logger.info("保存城市均价计算命令:" + str(cmd)) + else: + logger.info("保存区域均价计算命令:" + str(cmd)) + + +def send_city_calc_cmd(): + weekend = fangjia.get_weekend() + for city in fangjia.get_all_city(): + for region in fangjia.get_region(city): + save_city_calc_command(city, region, weekend) + save_city_calc_command(city, None, weekend) + + +def district_calc_cmd_finish(): + count = client.llen(QUEUE) + if count == 0: + return True + else: + return False if __name__ == "__main__": - for seaweed in get_district(): - if len(seaweed) > 2: - send_cmd(seaweed) + # send_district_calc_cmd() + while not district_calc_cmd_finish(): + time.sleep(5) + logger.info("等待小区均价计算完成") + send_city_calc_cmd() + logger.info("城市均价计算完成") + sys.exit(0) diff --git a/PriceSystem/CalcSpeedMonitor.py b/PriceSystem/CalcSpeedMonitor.py index 767d0f3..715f423 100644 --- a/PriceSystem/CalcSpeedMonitor.py +++ b/PriceSystem/CalcSpeedMonitor.py @@ -13,7 +13,8 @@ def monitor(pname): while True: pre_counter = client.llen(QUEUE) time.sleep(60) - print(pname + "#####################################:".zfill(100) + str(pre_counter - client.llen(QUEUE)) + "/min") + now_counter = client.llen(QUEUE) + print(pname + "#########:".zfill(100) + str(pre_counter - now_counter) + "/min") def format_cmd(cmd): diff --git a/PriceSystem/config/__init__.py b/PriceSystem/config/__init__.py index c9a8af5..941fa13 100644 --- a/PriceSystem/config/__init__.py +++ b/PriceSystem/config/__init__.py @@ -8,19 +8,27 @@ def get_logging_config(): def get_worker_num(): - return 80 + return 20 def get_redis_host(): return "192.168.0.144" +def get_district_stat_host(): + return "avgprice.mongo.db.fangjia.com" + + +def get_district_stat_port(): + return 27017 + + def get_redis_db(): return 1 def get_mongodb_host(): - return "192.168.10.182" + return "mongo.db.fangjia.com" def get_mongodb_port(): @@ -28,11 +36,11 @@ def get_mongodb_port(): def get_log_mongodb_host(): - return "192.168.0.136" + return "log.mongo.db.fangjia.com" def get_log_mongodb_port(): - return 52005 + return 52002 def get_timeout(): diff --git a/PriceSystem/fangjia/__init__.py b/PriceSystem/fangjia/__init__.py index effb7ef..6e40e56 100644 --- a/PriceSystem/fangjia/__init__.py +++ b/PriceSystem/fangjia/__init__.py @@ -2,6 +2,9 @@ import pandas import time import logging import config +import pytz +import datetime +import calendar from fangjia import mongoClients as mongoClients from fangjia import ElasticsearchClient as es @@ -49,7 +52,7 @@ def timer(func): arg_list = list() for arg in args: arg_list.append(str(arg)) - performance_logger.info("函数名:%-20s\t耗时:%7.4f\t\t参数:(%s)" % (func.__name__, end - begin, arg_list)) + performance_logger.info("函数名:%-20s\t耗时:%7.4f\t\t参数:(%s)" % (func.__name__, end - begin, arg_list)) return rs return _timer @@ -63,17 +66,37 @@ def get_seawater_index_name(city, region): return es.get_index_name(city, region) +def get_all_city(): + server_area = get_collection("fangjia", "server_area", config.get_mongodb_host(), config.get_mongodb_port()) + city_list = [] + for area in server_area.find().sort([("city_grade", 1), ("sequence", 1)]): + if "display" in area and area["display"] != "全国": + city_list.append(area["display"]) + return city_list + +def get_region(city): + region_block = get_collection("fangjia", "region_block", config.get_mongodb_host(), config.get_mongodb_port()) + return [region["name"] for region in region_block.find({"category": "region", "city": city}, {"name": 1})] + + +def get_weekend(): + now = datetime.datetime.now().replace(hour=0, minute=0, second=0, microsecond=0) + for day in range(1, 9): + if calendar.weekday(now.year, now.month, now.day) == 6: + break + now = now + datetime.timedelta(days=1) + pacific = pytz.timezone('Asia/Shanghai') + return pacific.localize(now) + + + + +@timer def findHouse(query, fields=None): - # s = time.time() seawater105 = get_collection("fangjia", "seawater", config.get_seawater_host(), 27777) for err_count in range(0, 10): - try: - - # lt = seawater105.count(query) - - # print('flag1') lt = 1000 # seawater.find(query,{"p_date":-1}).sort([("p_date",-1)]) pos = list(seawater105.find(query, fields).limit(lt)) @@ -91,7 +114,6 @@ def findHouse(query, fields=None): raise Exception("样本查询失败!") - def find_house_from_elastic_search(city, region, alias, roots=None, pdate=None, cat="sell", limit=1000): index_name = get_seawater_index_name(city, region) @@ -132,7 +154,6 @@ def find_house_from_elastic_search(city, region, alias, roots=None, pdate=None, return pandas.DataFrame([doc["_source"] for doc in response["hits"]["hits"]]) - def find_sample_pool(query, fields=None): # s = time.time() seawater105 = get_collection("fangjia", "seawater", config.get_seawater_host(), 27777) diff --git a/PriceSystem/sell_price/zhuge0118A.py b/PriceSystem/sell_price/zhuge0118A.py index e3d2992..43b9788 100644 --- a/PriceSystem/sell_price/zhuge0118A.py +++ b/PriceSystem/sell_price/zhuge0118A.py @@ -25,6 +25,7 @@ import time import fangjia import sklearn.svm import datetime +import config from fangjia import timer @@ -1144,11 +1145,8 @@ def ghhh(kkg): return a -client1 = pymongo.MongoClient('192.168.10.182', 27017) -db1 = client1.fangjia -stat = db1.distrit_stat - -pt = client1.fangjia_log.calc_command +stat = fangjia.get_collection("fangjia", "distrit_stat", config.get_district_stat_host(), config.get_district_stat_port()) +pt = fangjia.get_collection("fangjia_log", "calc_command", config.get_log_mongodb_host(), config.get_log_mongodb_port()) ################################################################# diff --git a/PriceSystem/test.py b/PriceSystem/test.py index c222294..4231af2 100644 --- a/PriceSystem/test.py +++ b/PriceSystem/test.py @@ -6,8 +6,8 @@ import sell_price.zhuge0118A as sellPrice s = time.time() ct = '上海' -rl = '嘉定' -lk = '莱茵北郡' +rl = '杨浦' +lk = '邯郸路60弄小区' weekend = datetime.datetime.strptime("2018-03-11", "%Y-%m-%d") for i in range(0, 100): sellPrice.out_put_price(ct, rl, lk, weekend)