Merge branch 'master' of http://gitlab.fangjia.com/dev0/airflow
This commit is contained in:
@@ -39,6 +39,12 @@ def calc_consumption_potential(city):
|
||||
columns = community_summary_df.columns
|
||||
if 'total_price' not in columns or 'estate_charge' not in columns or 'relative_price_ranking' not in columns or 'avg_cost' not in columns:
|
||||
return []
|
||||
# 解决某些城市无修正物业费问题
|
||||
if 'estate_charge_fix' not in community_summary_df.columns:
|
||||
community_summary_df['estate_charge_fix'] = None
|
||||
# 修正的物业费入模
|
||||
community_summary_df['estate_charge_fix'].fillna(community_summary_df['estate_charge'], inplace=True)
|
||||
|
||||
community_summary_df = community_summary_df.dropna(how='any').copy()
|
||||
if len(community_summary_df) == 0:
|
||||
log.warning(f"{city}无法评级,缺少因子")
|
||||
@@ -55,11 +61,6 @@ def calc_consumption_potential(city):
|
||||
community_summary_df['avg_cost_fixed'] = QuantileTransformer(output_distribution="normal").fit_transform(
|
||||
community_summary_df[['avg_cost']])
|
||||
|
||||
# 解决某些城市无修正物业费问题
|
||||
if 'estate_charge_fix' not in community_summary_df.columns:
|
||||
community_summary_df['estate_charge_fix'] = None
|
||||
# 修正的物业费入模
|
||||
community_summary_df['estate_charge_fix'].fillna(community_summary_df['estate_charge'], inplace=True)
|
||||
community_summary_df['estate_charge_fixed'] = QuantileTransformer(output_distribution="normal").fit_transform(
|
||||
community_summary_df[['estate_charge_fix']])
|
||||
|
||||
@@ -172,12 +173,47 @@ def sync_by_city(city_now):
|
||||
nextcloudutils.up_to_nextcloud(save_png_path + "/" + city_now + ".png", nextcloud_aim_file_path + "/" + city_now + ".png")
|
||||
|
||||
|
||||
def clear_rating(temp_df):
|
||||
"""
|
||||
清空评级信息
|
||||
"""
|
||||
actions = list()
|
||||
count = 0
|
||||
for index, item in temp_df.iterrows():
|
||||
_id = str(item["_id"])
|
||||
|
||||
action = {'_op_type': 'update', # 操作 index update create delete
|
||||
'_index': index_name, # index
|
||||
'_id': _id, # 请勿忘记ID
|
||||
'doc': {'consumption_rating': None, 'consumption_potential': None},
|
||||
"_retry_on_conflict": 5
|
||||
}
|
||||
actions.append(action)
|
||||
if len(actions) == 5000:
|
||||
helpers.bulk(client=es206_client.elastic_client, actions=actions)
|
||||
count += len(actions)
|
||||
actions.clear()
|
||||
log.info("清理中: 当前已清理{}".format(count))
|
||||
if len(actions) > 0:
|
||||
helpers.bulk(client=es206_client.elastic_client, actions=actions)
|
||||
count += len(actions)
|
||||
actions.clear()
|
||||
log.info("清理完毕: 共清理{}".format(count))
|
||||
|
||||
|
||||
def sync(city=None):
|
||||
clear_sql = "select city, region, name from community_summary where consumption_potential is not null and (total_price is null or estate_charge is null or avg_cost is null) "
|
||||
if pd.isna(city):
|
||||
# 清空缺少因子的老潜力
|
||||
temp_df = es206_client.scan_by_sql_to_data_frame(clear_sql)
|
||||
clear_rating(temp_df)
|
||||
for city_now in reasonable_df['city'].unique():
|
||||
#if pd.isna(reasonable_dict[city_now]):
|
||||
sync_by_city(city_now)
|
||||
else:
|
||||
clear_sql += f" and city='{city}' "
|
||||
temp_df = es206_client.scan_by_sql_to_data_frame(clear_sql)
|
||||
clear_rating(temp_df)
|
||||
sync_by_city(city)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user