修复物业费评级bug

This commit is contained in:
shenjian@fangjia.com
2022-11-22 15:45:23 +08:00
parent 4e159c307a
commit 3bfd8b75be
2 changed files with 73 additions and 58 deletions
+33 -34
View File
@@ -102,37 +102,35 @@ def calc_consumption_rating(community_summary_df):
"""
consumption_aim_df = community_summary_df[['_id', 'city', 'consumption_potential']]
consumption_aim_df = consumption_aim_df.dropna(subset=['consumption_potential'])
cities = consumption_aim_df['city'].unique()
source = "consumption_potential"
target = "consumption_rating"
actions = list()
count = 0
for city in cities:
df = consumption_aim_df[consumption_aim_df['city'] == city].copy()
if len(df) < 5:
continue
df = gmm_train(5, df, source, target)
# 批量更新
for index, item in df.iterrows():
row = dict()
_id = str(item["_id"])
row['consumption_rating'] = item['consumption_rating']
row['consumption_potential'] = item['consumption_potential']
row['m_date'] = dateutils.iso_format()
df = consumption_aim_df.copy()
if len(df) < 5:
return
df = gmm_train(5, df, source, target)
# 批量更新
for index, item in df.iterrows():
row = dict()
_id = str(item["_id"])
row['consumption_rating'] = item['consumption_rating']
row['consumption_potential'] = item['consumption_potential']
row['m_date'] = dateutils.iso_format()
action = {'_op_type': 'update', # 操作 index update create delete
'_index': index_name, # index
'_id': _id, # 请勿忘记ID
'doc': row,
"_retry_on_conflict": 5
}
actions.append(action)
if len(actions) == 5000:
helpers.bulk(client=es206_client.elastic_client, actions=actions)
count += len(actions)
actions.clear()
log.info("评级中: 当前已评级{}".format(count))
action = {'_op_type': 'update', # 操作 index update create delete
'_index': index_name, # index
'_id': _id, # 请勿忘记ID
'doc': row,
"_retry_on_conflict": 5
}
actions.append(action)
if len(actions) == 5000:
helpers.bulk(client=es206_client.elastic_client, actions=actions)
count += len(actions)
actions.clear()
log.info("评级中: 当前已评级{}".format(count))
if len(actions) > 0:
helpers.bulk(client=es206_client.elastic_client, actions=actions)
count += len(actions)
@@ -140,19 +138,20 @@ def calc_consumption_rating(community_summary_df):
log.info("评级完成,共更新: {}".format(count))
def sync_by_city(city_now):
log.info(f"{city_now}开始评级....")
community_summary_df = calc_consumption_potential(city_now)
calc_consumption_rating(community_summary_df)
save_png(city_now)
nextcloudutils.up_to_nextcloud(save_png_path, nextcloud_aim_file_path)
def sync(city=None):
if pd.isna(city):
for city_now in areautils.get_all_cities():
log.info(f"{city_now}开始评级....")
community_summary_df = calc_consumption_potential(city_now)
calc_consumption_rating(community_summary_df)
save_png(city_now)
nextcloudutils.up_to_nextcloud(save_png_path, nextcloud_aim_file_path)
sync_by_city(city_now)
else:
community_summary_df = calc_consumption_potential(city)
calc_consumption_rating(community_summary_df)
save_png(city)
nextcloudutils.up_to_nextcloud(save_png_path, nextcloud_aim_file_path)
sync_by_city(city)
def filter_min_max(city, community_summary_df):
@@ -63,35 +63,33 @@ def calc_estate_charge_rating(community_summary_df):
"""
consumption_aim_df = community_summary_df[['_id', 'city', 'estate_charge']]
consumption_aim_df = consumption_aim_df.dropna(subset=['estate_charge'])
cities = consumption_aim_df['city'].unique()
source = "estate_charge"
target = "estate_charge_ranking"
actions = list()
count = 0
for city in cities:
df = consumption_aim_df[consumption_aim_df['city'] == city].copy()
if len(df) < 5:
continue
df = gmm_train(5, df, source, target)
# 批量更新
for index, item in df.iterrows():
row = dict()
_id = str(item["_id"])
row['estate_charge_ranking'] = item['estate_charge_ranking']
df = consumption_aim_df.copy()
if len(df) < 5:
return
df = gmm_train(5, df, source, target)
# 批量更新
for index, item in df.iterrows():
row = dict()
_id = str(item["_id"])
row['estate_charge_ranking'] = item['estate_charge_ranking']
action = {'_op_type': 'update', # 操作 index update create delete
'_index': index_name, # index
'_id': _id, # 请勿忘记ID
'doc': row,
"_retry_on_conflict": 5
}
actions.append(action)
if len(actions) == 5000:
helpers.bulk(client=es206_client.elastic_client, actions=actions)
count += len(actions)
actions.clear()
log.info("评级中: 当前已评级{}".format(count))
action = {'_op_type': 'update', # 操作 index update create delete
'_index': index_name, # index
'_id': _id, # 请勿忘记ID
'doc': row,
"_retry_on_conflict": 5
}
actions.append(action)
if len(actions) == 5000:
helpers.bulk(client=es206_client.elastic_client, actions=actions)
count += len(actions)
actions.clear()
log.info("评级中: 当前已评级{}".format(count))
if len(actions) > 0:
helpers.bulk(client=es206_client.elastic_client, actions=actions)
count += len(actions)
@@ -108,6 +106,7 @@ def sync_by_city(city_now):
save_png(city_now)
nextcloudutils.up_to_nextcloud(save_png_path, nextcloud_aim_file_path)
def sync(city=None):
if pd.isna(city):
for city_now in areautils.get_all_cities():
@@ -121,7 +120,24 @@ def save_png(city=None):
生成财力评级图表
"""
sql = "select id, city, region, name, consumption_rating from community_summary where city='" + city + "'"
community_summary_df = es206_client.scan_by_sql_to_data_frame(sql)
community_city = es206_client.scan_by_sql_to_data_frame(sql)
plt.subplot(2, 1, 1)
community_city['estate_charge_ranking'].hist(bins=100)
plt.xlabel('estate_charge_ranking')
plt.tight_layout()
plt.subplot(2, 1, 2)
community_city['estate_charge'].hist(bins=100)
plt.xlabel('estate_charge')
plt.tight_layout()
plt.savefig(save_png_path + "/" + city + ".png")
# 清图
plt.clf()
# 清坐标轴
plt.cla()
# 关窗口
plt.close()