111 lines
4.9 KiB
Python
111 lines
4.9 KiB
Python
import pandas as pd
|
|
import common
|
|
from common import configs
|
|
from datetime import datetime
|
|
import numpy as np
|
|
from pymongo import UpdateOne
|
|
from concurrent.futures import ThreadPoolExecutor, wait, ALL_COMPLETED
|
|
|
|
|
|
def cal(x):
|
|
try:
|
|
return int(x * 10000)
|
|
except Exception as ex:
|
|
print(ex)
|
|
return 0
|
|
|
|
|
|
district_stat = common.get_mongodb_collection(configs.MONGO_URI, "fangjia_stat", "district_stat")
|
|
fill_command = common.get_mongodb_collection(configs.LOG_MONGO_URI, "fangjia_log", "fill_command")
|
|
|
|
|
|
def padding(city):
|
|
latest_week_query = {
|
|
'weekend': common.get_last_weekend(),
|
|
'city': city
|
|
}
|
|
count = 1
|
|
latest_stats = list(district_stat.find(latest_week_query).batch_size(500))
|
|
for latest_stat in latest_stats:
|
|
print(city + ":" + str(count))
|
|
count = count + 1
|
|
query = {
|
|
'city': latest_stat["city"],
|
|
'region': latest_stat["region"],
|
|
'district_name': latest_stat["district_name"],
|
|
'estate_type': latest_stat["estate_type"]
|
|
}
|
|
start_date = list(district_stat.find(query).sort([("weekend", 1)]).limit(1))[0]["weekend"]
|
|
end_date = list(district_stat.find(query).sort([("weekend", -1)]).limit(1))[0]["weekend"]
|
|
weekends = pd.date_range(start_date, end_date, freq='7D')
|
|
if len(weekends) < 3:
|
|
continue
|
|
if len(weekends) == district_stat.count(query):
|
|
continue
|
|
stats = pd.DataFrame(list(district_stat.find(query).sort([("weekend", 1)])))
|
|
stats = stats.set_index('weekend', drop=False)
|
|
isUpdate = False
|
|
update_count = 0
|
|
outer = False
|
|
if len(weekends) != len(stats["weekend"]) and len(weekends) > 3:
|
|
miss_weekends = [i for i in weekends if i not in list(stats["weekend"])]
|
|
for weekend in miss_weekends:
|
|
stats = stats.append(pd.Series({"weekend": weekend, "value_type": 10}, name=weekend))
|
|
stats.sort_index(inplace=True)
|
|
min_avg_price = int(min(stats["avg_price"]))
|
|
max_avg_price = int(max(stats["avg_price"]))
|
|
try:
|
|
stats["avg_price"].interpolate(method='polynomial', order=3, inplace=True, limit_area='inside')
|
|
except Exception as ex:
|
|
print(str(ex))
|
|
print(latest_stat)
|
|
continue
|
|
for other_field in ["city", "region", "block", "estate_type", "district_name"]:
|
|
if latest_stat[other_field] is not None:
|
|
stats[other_field].fillna(latest_stat[other_field], inplace=True)
|
|
stats["credit"].fillna(0, inplace=True)
|
|
stats['new'] = (stats["avg_price"] - stats.shift(1)["avg_price"]) / stats.shift(1)["avg_price"]
|
|
stats['new'].fillna(0, inplace=True)
|
|
stats['new'] = stats['new'].apply(lambda x: cal(x))
|
|
stats['weekend_markup'].fillna(0, inplace=True)
|
|
requests = []
|
|
for index, row in stats.iterrows():
|
|
if row["_id"] is np.nan or row["_id"] is None or row["new"] != int(row["weekend_markup"]):
|
|
update = {
|
|
"city": row["city"],
|
|
"region": row["region"],
|
|
"district_name": row["district_name"],
|
|
"estate_type": row["estate_type"],
|
|
"weekend": row["weekend"]
|
|
}
|
|
data = {
|
|
"credit": row["credit"],
|
|
"avg_price": int(row["avg_price"]),
|
|
"m_date": datetime.utcnow(),
|
|
"weekend_markup": int(row["new"])
|
|
}
|
|
if row["_id"] is np.nan or row["_id"] is None:
|
|
data["c_date"] = datetime.utcnow()
|
|
data["calc_value"] = int(row["avg_price"])
|
|
data["value_type"] = 8
|
|
data["flg"] = 1
|
|
requests.append(UpdateOne(update, {'$set': data}, True))
|
|
isUpdate = True
|
|
update_count = update_count + 1
|
|
if row["avg_price"] > max_avg_price or row["avg_price"] < min_avg_price:
|
|
outer = True
|
|
if isUpdate:
|
|
district_stat.bulk_write(requests)
|
|
print(latest_stat)
|
|
common.send_refitted_trend(latest_stat["city"], latest_stat["region"], latest_stat["district_name"],
|
|
latest_stat["estate_type"], latest_stat["weekend"])
|
|
latest_stat["update_count"] = update_count
|
|
latest_stat["outer"] = outer
|
|
fill_command.save(latest_stat)
|
|
|
|
|
|
executor = ThreadPoolExecutor(100)
|
|
all_task = [executor.submit(padding, (city)) for city in common.get_all_city()]
|
|
wait(all_task, return_when=ALL_COMPLETED)
|
|
print("main")
|