From 109db091e06bb3e329678c3189b674b2bbbacf9e Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Mon, 20 Mar 2023 17:50:34 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=9B=A0=E5=AD=90=E4=B8=8D?= =?UTF-8?q?=E8=B6=B3=E6=B8=85=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../business/community/consumption.py | 46 +++++++++++++++++-- 1 file changed, 41 insertions(+), 5 deletions(-) diff --git a/dags/business/business/community/consumption.py b/dags/business/business/community/consumption.py index b100206..28f6f3a 100644 --- a/dags/business/business/community/consumption.py +++ b/dags/business/business/community/consumption.py @@ -39,6 +39,12 @@ def calc_consumption_potential(city): columns = community_summary_df.columns if 'total_price' not in columns or 'estate_charge' not in columns or 'relative_price_ranking' not in columns or 'avg_cost' not in columns: return [] + # 解决某些城市无修正物业费问题 + if 'estate_charge_fix' not in community_summary_df.columns: + community_summary_df['estate_charge_fix'] = None + # 修正的物业费入模 + community_summary_df['estate_charge_fix'].fillna(community_summary_df['estate_charge'], inplace=True) + community_summary_df = community_summary_df.dropna(how='any').copy() if len(community_summary_df) == 0: log.warning(f"{city}无法评级,缺少因子") @@ -55,11 +61,6 @@ def calc_consumption_potential(city): community_summary_df['avg_cost_fixed'] = QuantileTransformer(output_distribution="normal").fit_transform( community_summary_df[['avg_cost']]) - # 解决某些城市无修正物业费问题 - if 'estate_charge_fix' not in community_summary_df.columns: - community_summary_df['estate_charge_fix'] = None - # 修正的物业费入模 - community_summary_df['estate_charge_fix'].fillna(community_summary_df['estate_charge'], inplace=True) community_summary_df['estate_charge_fixed'] = QuantileTransformer(output_distribution="normal").fit_transform( community_summary_df[['estate_charge_fix']]) @@ -172,12 +173,47 @@ def sync_by_city(city_now): nextcloudutils.up_to_nextcloud(save_png_path + "/" + city_now + ".png", nextcloud_aim_file_path + "/" + city_now + ".png") +def clear_rating(temp_df): + """ + 清空评级信息 + """ + actions = list() + count = 0 + for index, item in temp_df.iterrows(): + _id = str(item["_id"]) + + action = {'_op_type': 'update', # 操作 index update create delete + '_index': index_name, # index + '_id': _id, # 请勿忘记ID + 'doc': {'consumption_rating': None, 'consumption_potential': None}, + "_retry_on_conflict": 5 + } + actions.append(action) + if len(actions) == 5000: + helpers.bulk(client=es206_client.elastic_client, actions=actions) + count += len(actions) + actions.clear() + log.info("清理中: 当前已清理{}".format(count)) + if len(actions) > 0: + helpers.bulk(client=es206_client.elastic_client, actions=actions) + count += len(actions) + actions.clear() + log.info("清理完毕: 共清理{}".format(count)) + + def sync(city=None): + clear_sql = "select city, region, name from community_summary where (total_price is null or estate_charge is null or avg_cost is null and consumption_potential is not null)" if pd.isna(city): + # 清空缺少因子的老潜力 + temp_df = es206_client.scan_by_sql_to_data_frame(clear_sql) + clear_rating(temp_df) for city_now in reasonable_df['city'].unique(): #if pd.isna(reasonable_dict[city_now]): sync_by_city(city_now) else: + clear_sql += f" and city='{city}' " + temp_df = es206_client.scan_by_sql_to_data_frame(clear_sql) + clear_rating(temp_df) sync_by_city(city)