This commit is contained in:
2023-03-20 18:28:19 +08:00
@@ -39,6 +39,12 @@ def calc_consumption_potential(city):
columns = community_summary_df.columns
if 'total_price' not in columns or 'estate_charge' not in columns or 'relative_price_ranking' not in columns or 'avg_cost' not in columns:
return []
# 解决某些城市无修正物业费问题
if 'estate_charge_fix' not in community_summary_df.columns:
community_summary_df['estate_charge_fix'] = None
# 修正的物业费入模
community_summary_df['estate_charge_fix'].fillna(community_summary_df['estate_charge'], inplace=True)
community_summary_df = community_summary_df.dropna(how='any').copy()
if len(community_summary_df) == 0:
log.warning(f"{city}无法评级,缺少因子")
@@ -55,11 +61,6 @@ def calc_consumption_potential(city):
community_summary_df['avg_cost_fixed'] = QuantileTransformer(output_distribution="normal").fit_transform(
community_summary_df[['avg_cost']])
# 解决某些城市无修正物业费问题
if 'estate_charge_fix' not in community_summary_df.columns:
community_summary_df['estate_charge_fix'] = None
# 修正的物业费入模
community_summary_df['estate_charge_fix'].fillna(community_summary_df['estate_charge'], inplace=True)
community_summary_df['estate_charge_fixed'] = QuantileTransformer(output_distribution="normal").fit_transform(
community_summary_df[['estate_charge_fix']])
@@ -172,12 +173,47 @@ def sync_by_city(city_now):
nextcloudutils.up_to_nextcloud(save_png_path + "/" + city_now + ".png", nextcloud_aim_file_path + "/" + city_now + ".png")
def clear_rating(temp_df):
"""
清空评级信息
"""
actions = list()
count = 0
for index, item in temp_df.iterrows():
_id = str(item["_id"])
action = {'_op_type': 'update', # 操作 index update create delete
'_index': index_name, # index
'_id': _id, # 请勿忘记ID
'doc': {'consumption_rating': None, 'consumption_potential': None},
"_retry_on_conflict": 5
}
actions.append(action)
if len(actions) == 5000:
helpers.bulk(client=es206_client.elastic_client, actions=actions)
count += len(actions)
actions.clear()
log.info("清理中: 当前已清理{}".format(count))
if len(actions) > 0:
helpers.bulk(client=es206_client.elastic_client, actions=actions)
count += len(actions)
actions.clear()
log.info("清理完毕: 共清理{}".format(count))
def sync(city=None):
clear_sql = "select city, region, name from community_summary where consumption_potential is not null and (total_price is null or estate_charge is null or avg_cost is null) "
if pd.isna(city):
# 清空缺少因子的老潜力
temp_df = es206_client.scan_by_sql_to_data_frame(clear_sql)
clear_rating(temp_df)
for city_now in reasonable_df['city'].unique():
#if pd.isna(reasonable_dict[city_now]):
sync_by_city(city_now)
else:
clear_sql += f" and city='{city}' "
temp_df = es206_client.scan_by_sql_to_data_frame(clear_sql)
clear_rating(temp_df)
sync_by_city(city)