Files
Sklearn/AvgPricePad.py
2019-05-07 11:33:15 +08:00

111 lines
4.9 KiB
Python

import pandas as pd
import common
from common import configs
from datetime import datetime
import numpy as np
from pymongo import UpdateOne
from concurrent.futures import ThreadPoolExecutor, wait, ALL_COMPLETED
def cal(x):
try:
return int(x * 10000)
except Exception as ex:
print(ex)
return 0
district_stat = common.get_mongodb_collection(configs.MONGO_URI, "fangjia_stat", "district_stat")
fill_command = common.get_mongodb_collection(configs.LOG_MONGO_URI, "fangjia_log", "fill_command")
def padding(city):
latest_week_query = {
'weekend': common.get_last_weekend(),
'city': city
}
count = 1
latest_stats = list(district_stat.find(latest_week_query).batch_size(500))
for latest_stat in latest_stats:
print(city + ":" + str(count))
count = count + 1
query = {
'city': latest_stat["city"],
'region': latest_stat["region"],
'district_name': latest_stat["district_name"],
'estate_type': latest_stat["estate_type"]
}
start_date = list(district_stat.find(query).sort([("weekend", 1)]).limit(1))[0]["weekend"]
end_date = list(district_stat.find(query).sort([("weekend", -1)]).limit(1))[0]["weekend"]
weekends = pd.date_range(start_date, end_date, freq='7D')
if len(weekends) < 3:
continue
if len(weekends) == district_stat.count(query):
continue
stats = pd.DataFrame(list(district_stat.find(query).sort([("weekend", 1)])))
stats = stats.set_index('weekend', drop=False)
isUpdate = False
update_count = 0
outer = False
if len(weekends) != len(stats["weekend"]) and len(weekends) > 3:
miss_weekends = [i for i in weekends if i not in list(stats["weekend"])]
for weekend in miss_weekends:
stats = stats.append(pd.Series({"weekend": weekend, "value_type": 10}, name=weekend))
stats.sort_index(inplace=True)
min_avg_price = int(min(stats["avg_price"]))
max_avg_price = int(max(stats["avg_price"]))
try:
stats["avg_price"].interpolate(method='polynomial', order=3, inplace=True, limit_area='inside')
except Exception as ex:
print(str(ex))
print(latest_stat)
continue
for other_field in ["city", "region", "block", "estate_type", "district_name"]:
if latest_stat[other_field] is not None:
stats[other_field].fillna(latest_stat[other_field], inplace=True)
stats["credit"].fillna(0, inplace=True)
stats['new'] = (stats["avg_price"] - stats.shift(1)["avg_price"]) / stats.shift(1)["avg_price"]
stats['new'].fillna(0, inplace=True)
stats['new'] = stats['new'].apply(lambda x: cal(x))
stats['weekend_markup'].fillna(0, inplace=True)
requests = []
for index, row in stats.iterrows():
if row["_id"] is np.nan or row["_id"] is None or row["new"] != int(row["weekend_markup"]):
update = {
"city": row["city"],
"region": row["region"],
"district_name": row["district_name"],
"estate_type": row["estate_type"],
"weekend": row["weekend"]
}
data = {
"credit": row["credit"],
"avg_price": int(row["avg_price"]),
"m_date": datetime.utcnow(),
"weekend_markup": int(row["new"])
}
if row["_id"] is np.nan or row["_id"] is None:
data["c_date"] = datetime.utcnow()
data["calc_value"] = int(row["avg_price"])
data["value_type"] = 8
data["flg"] = 1
requests.append(UpdateOne(update, {'$set': data}, True))
isUpdate = True
update_count = update_count + 1
if row["avg_price"] > max_avg_price or row["avg_price"] < min_avg_price:
outer = True
if isUpdate:
district_stat.bulk_write(requests)
print(latest_stat)
common.send_refitted_trend(latest_stat["city"], latest_stat["region"], latest_stat["district_name"],
latest_stat["estate_type"], latest_stat["weekend"])
latest_stat["update_count"] = update_count
latest_stat["outer"] = outer
fill_command.save(latest_stat)
executor = ThreadPoolExecutor(100)
all_task = [executor.submit(padding, (city)) for city in common.get_all_city()]
wait(all_task, return_when=ALL_COMPLETED)
print("main")