diff --git a/dags/business/business/community/consumption.py b/dags/business/business/community/consumption.py index b81d3cd..b6c47c1 100644 --- a/dags/business/business/community/consumption.py +++ b/dags/business/business/community/consumption.py @@ -1,12 +1,13 @@ """ 月消费潜力、评级计算 """ -from goojia_utils import ElasticSearchClient, dateutils, nextcloudutils, dingutils +from goojia_utils import ElasticSearchClient, dateutils, nextcloudutils, dingutils, areautils import pandas as pd import numpy as np import logging import os from elasticsearch import helpers +import matplotlib.pyplot as plt from sklearn.mixture import GaussianMixture from matplotlib import pyplot @@ -36,22 +37,18 @@ for index, item in factor_df.iterrows(): def calc_consumption_potential(city): + if pd.isna(city): + return """ 计算消费潜力 """ - sql = "select id, city, region, name, total_price, estate_charge, relative_price_ranking, avg_cost from community_summary" - if pd.notna(city): - sql += " where city='" + city + "'" + sql = "select id, city, region, name, total_price, estate_charge, relative_price_ranking, avg_cost from community_summary where city='" + city + "'" community_summary_df = es206_client.scan_by_sql_to_data_frame(sql) + + # 极值设置 + community_summary_df = filter_min_max(city, community_summary_df).copy() + # 户均总价计算时候改成万 - ### 极值设置 - if city in factor_dict: - community_summary_df.loc[community_summary_df['total_price'] < factor_dict[city]['min_total_price'], 'total_price'] = factor_dict[city]['min_total_price'] - community_summary_df.loc[community_summary_df['total_price'] > factor_dict[city]['max_total_price'], 'total_price'] = factor_dict[city]['max_total_price'] - community_summary_df.loc[community_summary_df['estate_charge'] < factor_dict[city]['min_estate_charge'], 'estate_charge'] = factor_dict[city]['min_estate_charge'] - community_summary_df.loc[community_summary_df['estate_charge'] > factor_dict[city]['max_estate_charge'], 'estate_charge'] = factor_dict[city]['max_estate_charge'] - community_summary_df.loc[community_summary_df['avg_cost'] < factor_dict[city]['min_avg_cost'], 'avg_cost'] = factor_dict[city]['min_avg_cost'] - community_summary_df.loc[community_summary_df['avg_cost'] > factor_dict[city]['max_avg_cost'], 'avg_cost'] = factor_dict[city]['max_avg_cost'] community_summary_df['total_price'] = community_summary_df['total_price'] / 10000 community_summary_df['total_price_mean'] = community_summary_df.groupby(by=['city'])['total_price'].transform( @@ -145,8 +142,38 @@ def calc_consumption_rating(community_summary_df): def sync(city=None): - community_summary_df = calc_consumption_potential(city) - calc_consumption_rating(community_summary_df) + if pd.isna(city): + for city in areautils.get_all_cities(): + log.info(f"{city}开始评级....") + community_summary_df = calc_consumption_potential(city) + calc_consumption_rating(community_summary_df) + else: + community_summary_df = calc_consumption_potential(city) + calc_consumption_rating(community_summary_df) + + +def filter_min_max(city, community_summary_df): + """ + 过滤极值 + """ + if city in factor_dict: + community_summary_df.loc[ + community_summary_df['total_price'] < factor_dict[city]['min_total_price'], 'total_price'] = \ + factor_dict[city]['min_total_price'] + community_summary_df.loc[ + community_summary_df['total_price'] > factor_dict[city]['max_total_price'], 'total_price'] = \ + factor_dict[city]['max_total_price'] + community_summary_df.loc[ + community_summary_df['estate_charge'] < factor_dict[city]['min_estate_charge'], 'estate_charge'] = \ + factor_dict[city]['min_estate_charge'] + community_summary_df.loc[ + community_summary_df['estate_charge'] > factor_dict[city]['max_estate_charge'], 'estate_charge'] = \ + factor_dict[city]['max_estate_charge'] + community_summary_df.loc[community_summary_df['avg_cost'] < factor_dict[city]['min_avg_cost'], 'avg_cost'] = \ + factor_dict[city]['min_avg_cost'] + community_summary_df.loc[community_summary_df['avg_cost'] > factor_dict[city]['max_avg_cost'], 'avg_cost'] = \ + factor_dict[city]['max_avg_cost'] + return community_summary_df def save_png(city=None): @@ -157,10 +184,39 @@ def save_png(city=None): if pd.notna(city): sql += " where city='" + city + "'" community_summary_df = es206_client.scan_by_sql_to_data_frame(sql) + community_summary_df['total_price'] = community_summary_df['total_price'] / 10000 for city in community_summary_df['city'].unique(): - community_summary_df[(community_summary_df['consumption_rating'].notna()) & (community_summary_df['city'] == city)]['consumption_rating'].hist(bins=100) - pyplot.savefig(save_png_path + "/" + city + ".png") + community_city = community_summary_df[community_summary_df['city'] == city] + community_city = filter_min_max(city, community_city).copy() + + plt.subplot(3, 1, 1) + community_city['consumption_rating'].hist(bins=100) + plt.xlabel('consumption_rating') + plt.tight_layout() + + plt.subplot(3, 2, 3) + community_city['total_price'].hist(bins=100) + plt.xlabel('total_price') + plt.tight_layout() + + plt.subplot(3, 2, 4) + community_city['estate_charge'].hist(bins=100) + plt.xlabel('estate_charge') + plt.tight_layout() + + plt.subplot(3, 2, 5) + community_city['avg_cost'].hist(bins=100) + plt.xlabel('avg_cost') + plt.tight_layout() + + plt.subplot(3, 2, 6) + community_city['relative_price_ranking'].hist(bins=100) + plt.xlabel('relative_price_ranking') + plt.tight_layout() + + plt.savefig(save_png_path + "/" + city + ".png") + plt.show() sync() diff --git a/dags/business/business/community/estate_charge_ranking.py b/dags/business/business/community/estate_charge_ranking.py index 5b1e57e..68953c9 100644 --- a/dags/business/business/community/estate_charge_ranking.py +++ b/dags/business/business/community/estate_charge_ranking.py @@ -6,9 +6,9 @@ import pandas as pd import logging import os from elasticsearch import helpers +import matplotlib.pyplot as plt from sklearn.mixture import GaussianMixture -from matplotlib import pyplot log = logging.getLogger(__name__) @@ -108,6 +108,36 @@ def save_png(city=None): for city in community_summary_df['city'].unique(): community_summary_df[(community_summary_df['estate_charge_ranking'].notna()) & (community_summary_df['city'] == city)]['estate_charge_ranking'].hist(bins=100) pyplot.savefig(save_png_path + "/" + city + ".png") + pyplot.show() + + fig = plt.figure(figsize=(15, 15)) + + plt.subplot(3, 1, 1) + seaweed_origin_df['consumption_rating'].hist(bins=100) + plt.xlabel('consumption_rating') + plt.tight_layout() + + plt.subplot(3, 2, 3) + seaweed_origin_df['total_price'].hist(bins=100) + plt.xlabel('total_price') + plt.tight_layout() + + plt.subplot(3, 2, 4) + seaweed_origin_df['estate_charge'].hist(bins=100) + plt.xlabel('estate_charge') + plt.tight_layout() + + plt.subplot(3, 2, 5) + seaweed_origin_df['avg_cost'].hist(bins=100) + plt.xlabel('avg_cost') + plt.tight_layout() + + plt.subplot(3, 2, 6) + seaweed_origin_df['relative_price_ranking'].hist(bins=100) + plt.xlabel('relative_price_ranking') + plt.tight_layout() + + plt.savefig() sync()