import pandas as pd import common from common import configs from datetime import datetime import numpy as np from pymongo import UpdateOne from concurrent.futures import ThreadPoolExecutor, wait, ALL_COMPLETED def cal(x): try: return int(x * 10000) except Exception as ex: print(ex) return 0 district_stat = common.get_mongodb_collection(configs.MONGO_URI, "fangjia_stat", "district_stat") fill_command = common.get_mongodb_collection(configs.LOG_MONGO_URI, "fangjia_log", "fill_command") def padding(city): latest_week_query = { 'weekend': common.get_last_weekend(), 'city': city } count = 1 latest_stats = list(district_stat.find(latest_week_query).batch_size(500)) for latest_stat in latest_stats: print(city + ":" + str(count)) count = count + 1 query = { 'city': latest_stat["city"], 'region': latest_stat["region"], 'district_name': latest_stat["district_name"], 'estate_type': latest_stat["estate_type"] } start_date = list(district_stat.find(query).sort([("weekend", 1)]).limit(1))[0]["weekend"] end_date = list(district_stat.find(query).sort([("weekend", -1)]).limit(1))[0]["weekend"] weekends = pd.date_range(start_date, end_date, freq='7D') if len(weekends) < 3: continue if len(weekends) == district_stat.count(query): continue stats = pd.DataFrame(list(district_stat.find(query).sort([("weekend", 1)]))) stats = stats.set_index('weekend', drop=False) isUpdate = False update_count = 0 outer = False if len(weekends) != len(stats["weekend"]) and len(weekends) > 3: miss_weekends = [i for i in weekends if i not in list(stats["weekend"])] for weekend in miss_weekends: stats = stats.append(pd.Series({"weekend": weekend, "value_type": 10}, name=weekend)) stats.sort_index(inplace=True) min_avg_price = int(min(stats["avg_price"])) max_avg_price = int(max(stats["avg_price"])) try: stats["avg_price"].interpolate(method='polynomial', order=3, inplace=True, limit_area='inside') except Exception as ex: print(str(ex)) print(latest_stat) continue for other_field in ["city", "region", "block", "estate_type", "district_name"]: if latest_stat[other_field] is not None: stats[other_field].fillna(latest_stat[other_field], inplace=True) stats["credit"].fillna(0, inplace=True) stats['new'] = (stats["avg_price"] - stats.shift(1)["avg_price"]) / stats.shift(1)["avg_price"] stats['new'].fillna(0, inplace=True) stats['new'] = stats['new'].apply(lambda x: cal(x)) stats['weekend_markup'].fillna(0, inplace=True) requests = [] for index, row in stats.iterrows(): if row["_id"] is np.nan or row["_id"] is None or row["new"] != int(row["weekend_markup"]): update = { "city": row["city"], "region": row["region"], "district_name": row["district_name"], "estate_type": row["estate_type"], "weekend": row["weekend"] } data = { "credit": row["credit"], "avg_price": int(row["avg_price"]), "m_date": datetime.utcnow(), "weekend_markup": int(row["new"]) } if row["_id"] is np.nan or row["_id"] is None: data["c_date"] = datetime.utcnow() data["calc_value"] = int(row["avg_price"]) data["value_type"] = 8 data["flg"] = 1 requests.append(UpdateOne(update, {'$set': data}, True)) isUpdate = True update_count = update_count + 1 if row["avg_price"] > max_avg_price or row["avg_price"] < min_avg_price: outer = True if isUpdate: district_stat.bulk_write(requests) print(latest_stat) common.send_refitted_trend(latest_stat["city"], latest_stat["region"], latest_stat["district_name"], latest_stat["estate_type"], latest_stat["weekend"]) latest_stat["update_count"] = update_count latest_stat["outer"] = outer fill_command.save(latest_stat) executor = ThreadPoolExecutor(100) all_task = [executor.submit(padding, (city)) for city in common.get_all_city()] wait(all_task, return_when=ALL_COMPLETED) print("main")