优化均价算法

This commit is contained in:
2018-03-12 12:58:05 +08:00
parent 742bad7de9
commit a8c95698cd
6 changed files with 111 additions and 38 deletions
+62 -17
View File
@@ -2,13 +2,16 @@ import json
import redis import redis
import fangjia import fangjia
import config import config
from datetime import datetime, date import datetime
import sys
import time
from datetime import date
class CJsonEncoder(json.JSONEncoder): class CJsonEncoder(json.JSONEncoder):
def default(self, obj): def default(self, obj):
if isinstance(obj, datetime): if isinstance(obj, datetime.datetime):
return obj.strftime('%Y-%m-%d %H:%M:%S') return obj.strftime('%Y-%m-%d')
elif isinstance(obj, date): elif isinstance(obj, date):
return obj.strftime('%Y-%m-%d') return obj.strftime('%Y-%m-%d')
else: else:
@@ -18,32 +21,74 @@ class CJsonEncoder(json.JSONEncoder):
QUEUE = "calc_cmd" QUEUE = "calc_cmd"
redisPool = redis.ConnectionPool(host=config.get_redis_host(), port=6379, db=config.get_redis_db()) redisPool = redis.ConnectionPool(host=config.get_redis_host(), port=6379, db=config.get_redis_db())
client = redis.Redis(connection_pool=redisPool) client = redis.Redis(connection_pool=redisPool)
logger = fangjia.get_logger("CalcCmdReceiver")
def get_district(): def get_district(city):
weekend = fangjia.get_weekend()
seaweeds = [] seaweeds = []
keys = {"city": 1, "region": 1, "name": 1, "_id": 0} keys = {"city": 1, "region": 1, "name": 1, "_id": 0}
for dis in fangjia.get_collection("fangjia", "seaweed").find({"cat": "district", "status": 0, "city": "合肥"}, keys): for dis in fangjia.get_collection("fangjia", "seaweed").find({"cat": "district", "status": 0, "city": city}, keys):
dis["weekend"] = datetime.strptime("2018-03-11", "%Y-%m-%d") dis["weekend"] = weekend
seaweeds.append(dis) seaweeds.append(dis)
return seaweeds return seaweeds
def send_cmd(seaweed): def send_cmd(seaweed):
"""
cmd = {}
cmd["city"] = seaweed["city"]
cmd["region"] = seaweed["region"]
cmd["name"] = seaweed["name"]
"""
json_cmd = json.dumps(seaweed, ensure_ascii=False, cls=CJsonEncoder) json_cmd = json.dumps(seaweed, ensure_ascii=False, cls=CJsonEncoder)
client.rpush(QUEUE, json_cmd) client.rpush(QUEUE, json_cmd)
print("命令发送成功:" + json_cmd) logger.info("命令发送成功:" + json_cmd)
def send_district_calc_cmd():
for city in fangjia.get_all_city():
for seaweed in get_district(city):
send_cmd(seaweed)
def save_city_calc_command(city, region, weekend):
calc_command = fangjia.get_collection("fangjia_log", "calc_command", config.get_log_mongodb_host(),
config.get_log_mongodb_port())
cmd = dict()
cmd["city"] = city
cmd["region"] = region
cmd["block"] = None
cmd["calc_date"] = weekend
cmd["calc_type"] = 0
cmd["category"] = "sell"
cmd["priority"] = 1
cmd["status"] = 0
cmd["remark"] = 'week'
cmd["c_date"] = datetime.datetime.now().utcnow()
calc_command.insert_one(cmd)
if region is None:
logger.info("保存城市均价计算命令:" + str(cmd))
else:
logger.info("保存区域均价计算命令:" + str(cmd))
def send_city_calc_cmd():
weekend = fangjia.get_weekend()
for city in fangjia.get_all_city():
for region in fangjia.get_region(city):
save_city_calc_command(city, region, weekend)
save_city_calc_command(city, None, weekend)
def district_calc_cmd_finish():
count = client.llen(QUEUE)
if count == 0:
return True
else:
return False
if __name__ == "__main__": if __name__ == "__main__":
for seaweed in get_district(): # send_district_calc_cmd()
if len(seaweed) > 2: while not district_calc_cmd_finish():
send_cmd(seaweed) time.sleep(5)
logger.info("等待小区均价计算完成")
send_city_calc_cmd()
logger.info("城市均价计算完成")
sys.exit(0)
+2 -1
View File
@@ -13,7 +13,8 @@ def monitor(pname):
while True: while True:
pre_counter = client.llen(QUEUE) pre_counter = client.llen(QUEUE)
time.sleep(60) time.sleep(60)
print(pname + "#####################################:".zfill(100) + str(pre_counter - client.llen(QUEUE)) + "/min") now_counter = client.llen(QUEUE)
print(pname + "#########:".zfill(100) + str(pre_counter - now_counter) + "/min")
def format_cmd(cmd): def format_cmd(cmd):
+12 -4
View File
@@ -8,19 +8,27 @@ def get_logging_config():
def get_worker_num(): def get_worker_num():
return 80 return 20
def get_redis_host(): def get_redis_host():
return "192.168.0.144" return "192.168.0.144"
def get_district_stat_host():
return "avgprice.mongo.db.fangjia.com"
def get_district_stat_port():
return 27017
def get_redis_db(): def get_redis_db():
return 1 return 1
def get_mongodb_host(): def get_mongodb_host():
return "192.168.10.182" return "mongo.db.fangjia.com"
def get_mongodb_port(): def get_mongodb_port():
@@ -28,11 +36,11 @@ def get_mongodb_port():
def get_log_mongodb_host(): def get_log_mongodb_host():
return "192.168.0.136" return "log.mongo.db.fangjia.com"
def get_log_mongodb_port(): def get_log_mongodb_port():
return 52005 return 52002
def get_timeout(): def get_timeout():
+30 -9
View File
@@ -2,6 +2,9 @@ import pandas
import time import time
import logging import logging
import config import config
import pytz
import datetime
import calendar
from fangjia import mongoClients as mongoClients from fangjia import mongoClients as mongoClients
from fangjia import ElasticsearchClient as es from fangjia import ElasticsearchClient as es
@@ -49,7 +52,7 @@ def timer(func):
arg_list = list() arg_list = list()
for arg in args: for arg in args:
arg_list.append(str(arg)) arg_list.append(str(arg))
performance_logger.info("函数名:%-20s\t耗时:%7.4f\t\t参数:(%s)" % (func.__name__, end - begin, arg_list)) performance_logger.info("函数名:%-20s\t耗时:%7.4f\t\t参数:(%s)" % (func.__name__, end - begin, arg_list))
return rs return rs
return _timer return _timer
@@ -63,17 +66,37 @@ def get_seawater_index_name(city, region):
return es.get_index_name(city, region) return es.get_index_name(city, region)
def get_all_city():
server_area = get_collection("fangjia", "server_area", config.get_mongodb_host(), config.get_mongodb_port())
city_list = []
for area in server_area.find().sort([("city_grade", 1), ("sequence", 1)]):
if "display" in area and area["display"] != "全国":
city_list.append(area["display"])
return city_list
def get_region(city):
region_block = get_collection("fangjia", "region_block", config.get_mongodb_host(), config.get_mongodb_port())
return [region["name"] for region in region_block.find({"category": "region", "city": city}, {"name": 1})]
def get_weekend():
now = datetime.datetime.now().replace(hour=0, minute=0, second=0, microsecond=0)
for day in range(1, 9):
if calendar.weekday(now.year, now.month, now.day) == 6:
break
now = now + datetime.timedelta(days=1)
pacific = pytz.timezone('Asia/Shanghai')
return pacific.localize(now)
@timer
def findHouse(query, fields=None): def findHouse(query, fields=None):
# s = time.time()
seawater105 = get_collection("fangjia", "seawater", config.get_seawater_host(), 27777) seawater105 = get_collection("fangjia", "seawater", config.get_seawater_host(), 27777)
for err_count in range(0, 10): for err_count in range(0, 10):
try: try:
# lt = seawater105.count(query)
# print('flag1')
lt = 1000 lt = 1000
# seawater.find(query,{"p_date":-1}).sort([("p_date",-1)]) # seawater.find(query,{"p_date":-1}).sort([("p_date",-1)])
pos = list(seawater105.find(query, fields).limit(lt)) pos = list(seawater105.find(query, fields).limit(lt))
@@ -91,7 +114,6 @@ def findHouse(query, fields=None):
raise Exception("样本查询失败!") raise Exception("样本查询失败!")
def find_house_from_elastic_search(city, region, alias, roots=None, pdate=None, cat="sell", limit=1000): def find_house_from_elastic_search(city, region, alias, roots=None, pdate=None, cat="sell", limit=1000):
index_name = get_seawater_index_name(city, region) index_name = get_seawater_index_name(city, region)
@@ -132,7 +154,6 @@ def find_house_from_elastic_search(city, region, alias, roots=None, pdate=None,
return pandas.DataFrame([doc["_source"] for doc in response["hits"]["hits"]]) return pandas.DataFrame([doc["_source"] for doc in response["hits"]["hits"]])
def find_sample_pool(query, fields=None): def find_sample_pool(query, fields=None):
# s = time.time() # s = time.time()
seawater105 = get_collection("fangjia", "seawater", config.get_seawater_host(), 27777) seawater105 = get_collection("fangjia", "seawater", config.get_seawater_host(), 27777)
+3 -5
View File
@@ -25,6 +25,7 @@ import time
import fangjia import fangjia
import sklearn.svm import sklearn.svm
import datetime import datetime
import config
from fangjia import timer from fangjia import timer
@@ -1144,11 +1145,8 @@ def ghhh(kkg):
return a return a
client1 = pymongo.MongoClient('192.168.10.182', 27017) stat = fangjia.get_collection("fangjia", "distrit_stat", config.get_district_stat_host(), config.get_district_stat_port())
db1 = client1.fangjia pt = fangjia.get_collection("fangjia_log", "calc_command", config.get_log_mongodb_host(), config.get_log_mongodb_port())
stat = db1.distrit_stat
pt = client1.fangjia_log.calc_command
################################################################# #################################################################
+2 -2
View File
@@ -6,8 +6,8 @@ import sell_price.zhuge0118A as sellPrice
s = time.time() s = time.time()
ct = '上海' ct = '上海'
rl = '嘉定' rl = '杨浦'
lk = '莱茵北郡' lk = '邯郸路60弄小区'
weekend = datetime.datetime.strptime("2018-03-11", "%Y-%m-%d") weekend = datetime.datetime.strptime("2018-03-11", "%Y-%m-%d")
for i in range(0, 100): for i in range(0, 100):
sellPrice.out_put_price(ct, rl, lk, weekend) sellPrice.out_put_price(ct, rl, lk, weekend)