From 3bfd8b75be782b22a7b3cf549fd36f21dd3fa4cb Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Tue, 22 Nov 2022 15:45:23 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E7=89=A9=E4=B8=9A=E8=B4=B9?= =?UTF-8?q?=E8=AF=84=E7=BA=A7bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../business/community/consumption.py | 67 +++++++++---------- .../community/estate_charge_ranking.py | 64 +++++++++++------- 2 files changed, 73 insertions(+), 58 deletions(-) diff --git a/dags/business/business/community/consumption.py b/dags/business/business/community/consumption.py index ba16bc8..ee78799 100644 --- a/dags/business/business/community/consumption.py +++ b/dags/business/business/community/consumption.py @@ -102,37 +102,35 @@ def calc_consumption_rating(community_summary_df): """ consumption_aim_df = community_summary_df[['_id', 'city', 'consumption_potential']] consumption_aim_df = consumption_aim_df.dropna(subset=['consumption_potential']) - cities = consumption_aim_df['city'].unique() source = "consumption_potential" target = "consumption_rating" actions = list() count = 0 - for city in cities: - df = consumption_aim_df[consumption_aim_df['city'] == city].copy() - if len(df) < 5: - continue - df = gmm_train(5, df, source, target) - # 批量更新 - for index, item in df.iterrows(): - row = dict() - _id = str(item["_id"]) - row['consumption_rating'] = item['consumption_rating'] - row['consumption_potential'] = item['consumption_potential'] - row['m_date'] = dateutils.iso_format() + df = consumption_aim_df.copy() + if len(df) < 5: + return + df = gmm_train(5, df, source, target) + # 批量更新 + for index, item in df.iterrows(): + row = dict() + _id = str(item["_id"]) + row['consumption_rating'] = item['consumption_rating'] + row['consumption_potential'] = item['consumption_potential'] + row['m_date'] = dateutils.iso_format() - action = {'_op_type': 'update', # 操作 index update create delete - '_index': index_name, # index - '_id': _id, # 请勿忘记ID - 'doc': row, - "_retry_on_conflict": 5 - } - actions.append(action) - if len(actions) == 5000: - helpers.bulk(client=es206_client.elastic_client, actions=actions) - count += len(actions) - actions.clear() - log.info("评级中: 当前已评级{}".format(count)) + action = {'_op_type': 'update', # 操作 index update create delete + '_index': index_name, # index + '_id': _id, # 请勿忘记ID + 'doc': row, + "_retry_on_conflict": 5 + } + actions.append(action) + if len(actions) == 5000: + helpers.bulk(client=es206_client.elastic_client, actions=actions) + count += len(actions) + actions.clear() + log.info("评级中: 当前已评级{}".format(count)) if len(actions) > 0: helpers.bulk(client=es206_client.elastic_client, actions=actions) count += len(actions) @@ -140,19 +138,20 @@ def calc_consumption_rating(community_summary_df): log.info("评级完成,共更新: {}".format(count)) +def sync_by_city(city_now): + log.info(f"{city_now}开始评级....") + community_summary_df = calc_consumption_potential(city_now) + calc_consumption_rating(community_summary_df) + save_png(city_now) + nextcloudutils.up_to_nextcloud(save_png_path, nextcloud_aim_file_path) + + def sync(city=None): if pd.isna(city): for city_now in areautils.get_all_cities(): - log.info(f"{city_now}开始评级....") - community_summary_df = calc_consumption_potential(city_now) - calc_consumption_rating(community_summary_df) - save_png(city_now) - nextcloudutils.up_to_nextcloud(save_png_path, nextcloud_aim_file_path) + sync_by_city(city_now) else: - community_summary_df = calc_consumption_potential(city) - calc_consumption_rating(community_summary_df) - save_png(city) - nextcloudutils.up_to_nextcloud(save_png_path, nextcloud_aim_file_path) + sync_by_city(city) def filter_min_max(city, community_summary_df): diff --git a/dags/business/business/community/estate_charge_ranking.py b/dags/business/business/community/estate_charge_ranking.py index 201d6c2..0339b5e 100644 --- a/dags/business/business/community/estate_charge_ranking.py +++ b/dags/business/business/community/estate_charge_ranking.py @@ -63,35 +63,33 @@ def calc_estate_charge_rating(community_summary_df): """ consumption_aim_df = community_summary_df[['_id', 'city', 'estate_charge']] consumption_aim_df = consumption_aim_df.dropna(subset=['estate_charge']) - cities = consumption_aim_df['city'].unique() source = "estate_charge" target = "estate_charge_ranking" actions = list() count = 0 - for city in cities: - df = consumption_aim_df[consumption_aim_df['city'] == city].copy() - if len(df) < 5: - continue - df = gmm_train(5, df, source, target) - # 批量更新 - for index, item in df.iterrows(): - row = dict() - _id = str(item["_id"]) - row['estate_charge_ranking'] = item['estate_charge_ranking'] + df = consumption_aim_df.copy() + if len(df) < 5: + return + df = gmm_train(5, df, source, target) + # 批量更新 + for index, item in df.iterrows(): + row = dict() + _id = str(item["_id"]) + row['estate_charge_ranking'] = item['estate_charge_ranking'] - action = {'_op_type': 'update', # 操作 index update create delete - '_index': index_name, # index - '_id': _id, # 请勿忘记ID - 'doc': row, - "_retry_on_conflict": 5 - } - actions.append(action) - if len(actions) == 5000: - helpers.bulk(client=es206_client.elastic_client, actions=actions) - count += len(actions) - actions.clear() - log.info("评级中: 当前已评级{}".format(count)) + action = {'_op_type': 'update', # 操作 index update create delete + '_index': index_name, # index + '_id': _id, # 请勿忘记ID + 'doc': row, + "_retry_on_conflict": 5 + } + actions.append(action) + if len(actions) == 5000: + helpers.bulk(client=es206_client.elastic_client, actions=actions) + count += len(actions) + actions.clear() + log.info("评级中: 当前已评级{}".format(count)) if len(actions) > 0: helpers.bulk(client=es206_client.elastic_client, actions=actions) count += len(actions) @@ -108,6 +106,7 @@ def sync_by_city(city_now): save_png(city_now) nextcloudutils.up_to_nextcloud(save_png_path, nextcloud_aim_file_path) + def sync(city=None): if pd.isna(city): for city_now in areautils.get_all_cities(): @@ -121,7 +120,24 @@ def save_png(city=None): 生成财力评级图表 """ sql = "select id, city, region, name, consumption_rating from community_summary where city='" + city + "'" - community_summary_df = es206_client.scan_by_sql_to_data_frame(sql) + community_city = es206_client.scan_by_sql_to_data_frame(sql) + plt.subplot(2, 1, 1) + community_city['estate_charge_ranking'].hist(bins=100) + plt.xlabel('estate_charge_ranking') + plt.tight_layout() + + plt.subplot(2, 1, 2) + community_city['estate_charge'].hist(bins=100) + plt.xlabel('estate_charge') + plt.tight_layout() + + plt.savefig(save_png_path + "/" + city + ".png") + # 清图 + plt.clf() + # 清坐标轴 + plt.cla() + # 关窗口 + plt.close()