优化均价算法

This commit is contained in:
2018-03-12 12:58:05 +08:00
parent 742bad7de9
commit a8c95698cd
6 changed files with 111 additions and 38 deletions
+62 -17
View File
@@ -2,13 +2,16 @@ import json
import redis
import fangjia
import config
from datetime import datetime, date
import datetime
import sys
import time
from datetime import date
class CJsonEncoder(json.JSONEncoder):
def default(self, obj):
if isinstance(obj, datetime):
return obj.strftime('%Y-%m-%d %H:%M:%S')
if isinstance(obj, datetime.datetime):
return obj.strftime('%Y-%m-%d')
elif isinstance(obj, date):
return obj.strftime('%Y-%m-%d')
else:
@@ -18,32 +21,74 @@ class CJsonEncoder(json.JSONEncoder):
QUEUE = "calc_cmd"
redisPool = redis.ConnectionPool(host=config.get_redis_host(), port=6379, db=config.get_redis_db())
client = redis.Redis(connection_pool=redisPool)
logger = fangjia.get_logger("CalcCmdReceiver")
def get_district():
def get_district(city):
weekend = fangjia.get_weekend()
seaweeds = []
keys = {"city": 1, "region": 1, "name": 1, "_id": 0}
for dis in fangjia.get_collection("fangjia", "seaweed").find({"cat": "district", "status": 0, "city": "合肥"}, keys):
dis["weekend"] = datetime.strptime("2018-03-11", "%Y-%m-%d")
for dis in fangjia.get_collection("fangjia", "seaweed").find({"cat": "district", "status": 0, "city": city}, keys):
dis["weekend"] = weekend
seaweeds.append(dis)
return seaweeds
def send_cmd(seaweed):
"""
cmd = {}
cmd["city"] = seaweed["city"]
cmd["region"] = seaweed["region"]
cmd["name"] = seaweed["name"]
"""
json_cmd = json.dumps(seaweed, ensure_ascii=False, cls=CJsonEncoder)
client.rpush(QUEUE, json_cmd)
print("命令发送成功:" + json_cmd)
logger.info("命令发送成功:" + json_cmd)
def send_district_calc_cmd():
for city in fangjia.get_all_city():
for seaweed in get_district(city):
send_cmd(seaweed)
def save_city_calc_command(city, region, weekend):
calc_command = fangjia.get_collection("fangjia_log", "calc_command", config.get_log_mongodb_host(),
config.get_log_mongodb_port())
cmd = dict()
cmd["city"] = city
cmd["region"] = region
cmd["block"] = None
cmd["calc_date"] = weekend
cmd["calc_type"] = 0
cmd["category"] = "sell"
cmd["priority"] = 1
cmd["status"] = 0
cmd["remark"] = 'week'
cmd["c_date"] = datetime.datetime.now().utcnow()
calc_command.insert_one(cmd)
if region is None:
logger.info("保存城市均价计算命令:" + str(cmd))
else:
logger.info("保存区域均价计算命令:" + str(cmd))
def send_city_calc_cmd():
weekend = fangjia.get_weekend()
for city in fangjia.get_all_city():
for region in fangjia.get_region(city):
save_city_calc_command(city, region, weekend)
save_city_calc_command(city, None, weekend)
def district_calc_cmd_finish():
count = client.llen(QUEUE)
if count == 0:
return True
else:
return False
if __name__ == "__main__":
for seaweed in get_district():
if len(seaweed) > 2:
send_cmd(seaweed)
# send_district_calc_cmd()
while not district_calc_cmd_finish():
time.sleep(5)
logger.info("等待小区均价计算完成")
send_city_calc_cmd()
logger.info("城市均价计算完成")
sys.exit(0)
+2 -1
View File
@@ -13,7 +13,8 @@ def monitor(pname):
while True:
pre_counter = client.llen(QUEUE)
time.sleep(60)
print(pname + "#####################################:".zfill(100) + str(pre_counter - client.llen(QUEUE)) + "/min")
now_counter = client.llen(QUEUE)
print(pname + "#########:".zfill(100) + str(pre_counter - now_counter) + "/min")
def format_cmd(cmd):
+12 -4
View File
@@ -8,19 +8,27 @@ def get_logging_config():
def get_worker_num():
return 80
return 20
def get_redis_host():
return "192.168.0.144"
def get_district_stat_host():
return "avgprice.mongo.db.fangjia.com"
def get_district_stat_port():
return 27017
def get_redis_db():
return 1
def get_mongodb_host():
return "192.168.10.182"
return "mongo.db.fangjia.com"
def get_mongodb_port():
@@ -28,11 +36,11 @@ def get_mongodb_port():
def get_log_mongodb_host():
return "192.168.0.136"
return "log.mongo.db.fangjia.com"
def get_log_mongodb_port():
return 52005
return 52002
def get_timeout():
+30 -9
View File
@@ -2,6 +2,9 @@ import pandas
import time
import logging
import config
import pytz
import datetime
import calendar
from fangjia import mongoClients as mongoClients
from fangjia import ElasticsearchClient as es
@@ -49,7 +52,7 @@ def timer(func):
arg_list = list()
for arg in args:
arg_list.append(str(arg))
performance_logger.info("函数名:%-20s\t耗时:%7.4f\t\t参数:(%s)" % (func.__name__, end - begin, arg_list))
performance_logger.info("函数名:%-20s\t耗时:%7.4f\t\t参数:(%s)" % (func.__name__, end - begin, arg_list))
return rs
return _timer
@@ -63,17 +66,37 @@ def get_seawater_index_name(city, region):
return es.get_index_name(city, region)
def get_all_city():
server_area = get_collection("fangjia", "server_area", config.get_mongodb_host(), config.get_mongodb_port())
city_list = []
for area in server_area.find().sort([("city_grade", 1), ("sequence", 1)]):
if "display" in area and area["display"] != "全国":
city_list.append(area["display"])
return city_list
def get_region(city):
region_block = get_collection("fangjia", "region_block", config.get_mongodb_host(), config.get_mongodb_port())
return [region["name"] for region in region_block.find({"category": "region", "city": city}, {"name": 1})]
def get_weekend():
now = datetime.datetime.now().replace(hour=0, minute=0, second=0, microsecond=0)
for day in range(1, 9):
if calendar.weekday(now.year, now.month, now.day) == 6:
break
now = now + datetime.timedelta(days=1)
pacific = pytz.timezone('Asia/Shanghai')
return pacific.localize(now)
@timer
def findHouse(query, fields=None):
# s = time.time()
seawater105 = get_collection("fangjia", "seawater", config.get_seawater_host(), 27777)
for err_count in range(0, 10):
try:
# lt = seawater105.count(query)
# print('flag1')
lt = 1000
# seawater.find(query,{"p_date":-1}).sort([("p_date",-1)])
pos = list(seawater105.find(query, fields).limit(lt))
@@ -91,7 +114,6 @@ def findHouse(query, fields=None):
raise Exception("样本查询失败!")
def find_house_from_elastic_search(city, region, alias, roots=None, pdate=None, cat="sell", limit=1000):
index_name = get_seawater_index_name(city, region)
@@ -132,7 +154,6 @@ def find_house_from_elastic_search(city, region, alias, roots=None, pdate=None,
return pandas.DataFrame([doc["_source"] for doc in response["hits"]["hits"]])
def find_sample_pool(query, fields=None):
# s = time.time()
seawater105 = get_collection("fangjia", "seawater", config.get_seawater_host(), 27777)
+3 -5
View File
@@ -25,6 +25,7 @@ import time
import fangjia
import sklearn.svm
import datetime
import config
from fangjia import timer
@@ -1144,11 +1145,8 @@ def ghhh(kkg):
return a
client1 = pymongo.MongoClient('192.168.10.182', 27017)
db1 = client1.fangjia
stat = db1.distrit_stat
pt = client1.fangjia_log.calc_command
stat = fangjia.get_collection("fangjia", "distrit_stat", config.get_district_stat_host(), config.get_district_stat_port())
pt = fangjia.get_collection("fangjia_log", "calc_command", config.get_log_mongodb_host(), config.get_log_mongodb_port())
#################################################################
+2 -2
View File
@@ -6,8 +6,8 @@ import sell_price.zhuge0118A as sellPrice
s = time.time()
ct = '上海'
rl = '嘉定'
lk = '莱茵北郡'
rl = '杨浦'
lk = '邯郸路60弄小区'
weekend = datetime.datetime.strptime("2018-03-11", "%Y-%m-%d")
for i in range(0, 100):
sellPrice.out_put_price(ct, rl, lk, weekend)