新文件: Rent_Receiver.py

新文件:   Rent_Sender.py
	新文件:   Rent_SpeedMonitor.py
	新文件:   rent_price/__init__.py
	新文件:   rent_price/__pycache__/__init__.cpython-36.pyc
	新文件:   rent_price/__pycache__/district_map0413.cpython-36.pyc
	新文件:   rent_price/district_map0413.py
	新文件:   rent_test.py
This commit is contained in:
罗干
2018-04-25 14:48:03 +08:00
parent ff6efd2113
commit 1ab980e5f0
8 changed files with 1418 additions and 0 deletions
+66
View File
@@ -0,0 +1,66 @@
import json
import multiprocessing
import time
import redis
import config
import rent_price.district_map0413 as rentPrice
import fangjia
import datetime
import pytz
import os
QUEUE = "rent_cmd"
#redis_pool = redis.ConnectionPool(host=config.get_redis_host(), port=6379, db=config.get_redis_db())
redis_pool = redis.ConnectionPool(host=config.get_redis_host(), port=6379, db=config.get_rent_redis_db())
logger = fangjia.get_logger("Rent_Receiver.py")
def worker():
client = redis.Redis(connection_pool=redis_pool)
while True:
try:
cmd = client.lpop(QUEUE)
except Exception as ex:
cmd = None
if cmd is None:
time.sleep(60)
logger.info("等待接受计算命令")
else:
cmd = format_cmd(cmd)
start_time = time.time()
try:
rentPrice.save_to_mongo(cmd["city"], cmd["region"], cmd["name"],cmd['timd'],cmd['timm'])
end_time = time.time()
logger.info("Calc SUC:" + str(cmd) + "\t计算成功,耗时:" + str(end_time - start_time))
except BaseException as ex:
logger.exception(ex)
end_time = time.time()
logger.error("Calc ERROR :" + str(cmd) + "\t计算失败,耗时:" + str(end_time - start_time))
def format_cmd(cmd):
cmd = json.loads(cmd)
return cmd
if __name__ == "__main__":
user_home = os.path.expanduser('~')
lock_file = user_home + "/price.sys.stop"
if os.path.exists(lock_file):
os.remove(lock_file)
pro_num = config.get_worker_num()
pool = multiprocessing.Pool(processes=pro_num)
for pid in range(0, pro_num):
pool.apply_async(worker, ())
pool.close()
logger.info("平米租金系统启动成功")
while True:
if os.path.exists(lock_file):
pool.terminate()
break
else:
time.sleep(3)
if os.path.exists(lock_file):
os.remove(lock_file)
pool.join()
+110
View File
@@ -0,0 +1,110 @@
import json
import redis
import fangjia
import config
import datetime
import sys
import time
from datetime import date
import calendar
#import pytz
def input_time(day):
#from dateutil.parser import parse
#today=parse(str(ti))
today=datetime.datetime.now()
#first = datetime.date(day=1, month=today.month, year=today.year)
lastMonth = today - datetime.timedelta(days=day)
def plus(k):
if k<10:
return '0'+str(k)
else:
return str(k)
y=lastMonth.year
m=lastMonth.month
#d=lastMonth.day
day=calendar.monthrange(y,m)[1]
cc=str(y)+plus(m)+plus(day)
#bb=parse(cc)
#pacific = pytz.timezone('Asia/Shanghai')
#return pacific.localize(bb)
return int(cc)
class CJsonEncoder(json.JSONEncoder):
def default(self, obj):
if isinstance(obj, datetime.datetime):
return obj.strftime('%Y-%m-%d')
elif isinstance(obj, date):
return obj.strftime('%Y-%m-%d')
else:
return json.JSONEncoder.default(self, obj)
QUEUE = "rent_cmd"
#redisPool = redis.ConnectionPool(host=config.get_redis_host(), port=6379, db=config.get_redis_db())
redisPool = redis.ConnectionPool(host=config.get_redis_host(), port=6379, db=config.get_rent_redis_db())
client = redis.Redis(connection_pool=redisPool)
logger = fangjia.get_logger("Rent_Sender.py")
def get_district(city):
timm=input_time(0)
timd=input_time(360)
seaweeds = []
keys = {"city": 1, "region": 1, "name": 1, "_id": 0}
for dis in fangjia.get_collection("fangjia", "seaweed").find({"cat": "district", "status": 0, "city": city}, keys):
dis["timm"] = timm
dis['timd']=timd
#dis["timm"] = 20170401
#dis['timd']=20160401
seaweeds.append(dis)
return seaweeds
def send_cmd(seaweed):
json_cmd = json.dumps(seaweed, ensure_ascii=False, cls=CJsonEncoder)
client.rpush(QUEUE, json_cmd)
logger.info("命令发送成功:" + json_cmd)
def send_district_calc_cmd():
for city in fangjia.get_all_city():
for seaweed in get_district(city):
send_cmd(seaweed)
def district_calc_cmd_finish():
count = client.llen(QUEUE)
if count == 0:
return True
else:
return False
if __name__ == "__main__":
start_time = datetime.datetime.utcnow()
send_district_calc_cmd()
while not district_calc_cmd_finish():
time.sleep(5)
logger.info("等待小区均价计算完成")
#send_city_calc_cmd()
logger.info("城市均价计算完成")
#all_calc_cmd_finish(start_time)
sys.exit(0)
+31
View File
@@ -0,0 +1,31 @@
import json
import multiprocessing
import time
import redis
import config
import fangjia
QUEUE = "rent_cmd"
redis_pool = redis.ConnectionPool(host=config.get_redis_host(), port=6379, db=config.get_redis_db())
logger = fangjia.get_logger("Rent_SpeedMonitor")
def monitor(pname):
client = redis.Redis(connection_pool=redis_pool)
while True:
pre_counter = client.llen(QUEUE)
time.sleep(60)
now_counter = client.llen(QUEUE)
logger.info(pname + "#####:".zfill(20) + str(pre_counter - now_counter) + "/min")
def format_cmd(cmd):
return json.loads(cmd)
if __name__ == "__main__":
pool = multiprocessing.Pool(processes=1)
pool.apply_async(monitor, ("Monitor-001",))
pool.close()
pool.join()
+1
View File
@@ -0,0 +1 @@
from rent_price import district_map0413
File diff suppressed because it is too large Load Diff
+84
View File
@@ -0,0 +1,84 @@
import datetime
import fangjia
import pytz
#fangjia.send_dingding_msg("测试")
import time
import rent_price.district_map0413 as rentPrice
def get_weekend(tt):
from dateutil.parser import parse
bb=parse(tt)
pacific = pytz.timezone('Asia/Shanghai')
return pacific.localize(bb)
weekend = get_weekend('20180415')
ct='上海'
rl='黄浦'
lk='伯爵居'#上海闵行闵浦新苑一村普陀曹杨一村上海
#黄浦伯爵居静安泰安里长宁新泾北苑
timm=20180430
timd=20160603
a=time.time()
rentPrice.save_to_mongo(ct,rl,lk,timd,timm)
print(time.time()-a)
'''
s = time.time()
ct='上海'
rl='浦东'
lk='滨江一品苑'
#for i in range(0, 100):深圳福田世纪豪庭
rentPrice.out_put_price(ct, rl, lk, weekend)
e = time.time()
print(e - s)
'''
'''
###
import pandas as pd
a=pd.read_excel('小区调查.xlsx')
for i in a.index:
ct=a['城市'][i]
rl=a['区域'][i]
lk=a['小区名'][i]
sellPrice.out_put_price(ct, rl, lk, weekend)
'''
'''
@fangjia.cache(1000)
def sum1(a, b):
print(111)
return a + b
print(sum1(1,2))
'''