From 390b869d3e2650bf11c13d55ab1dee947dc43279 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Mon, 14 Apr 2025 17:57:04 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96=E6=89=A7=E8=A1=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../business/community/consumption.py | 20 +++++-- dags/shop/sell/business.py | 52 +++++++++---------- dags/shop/sell/price.py | 52 +++++++++---------- 3 files changed, 68 insertions(+), 56 deletions(-) diff --git a/dags/business/business/community/consumption.py b/dags/business/business/community/consumption.py index de648db..6c20960 100644 --- a/dags/business/business/community/consumption.py +++ b/dags/business/business/community/consumption.py @@ -95,7 +95,11 @@ def calc_consumption_potential(city): # 人均消费极大值处理 community_summary_df['avg_cost_fixed'] = np.where(community_summary_df['avg_cost_fixed'] > 250, 250, community_summary_df['avg_cost_fixed']) - community_summary_df['avg_cost_fixed'] = np.log1p(community_summary_df['avg_cost_fixed']) + try: + community_summary_df['avg_cost_fixed'] = np.log1p(community_summary_df['avg_cost_fixed']) + except Exception: + community_summary_df['avg_cost_fixed'] = None + pass #community_summary_df['avg_cost_fixed'] = 1 - np.log(community_summary_df['avg_cost_fixed'] + 1) / np.log(max_avg_cost_fixed + 1) community_summary_df['estate_charge_fixed'] = 1 - np.log(community_summary_df['estate_charge_fixed'] + 1) / np.log(max_estate_charge_fixed + 1) community_summary_df['relative_price_ranking'] = 1 - np.log(community_summary_df['relative_price_ranking'] + 1) / np.log(max_relative_price_ranking + 1) @@ -110,8 +114,12 @@ def calc_consumption_potential(city): community_summary_df['relative_price_ranking_std'] = community_summary_df.groupby(by=['city'])[ 'relative_price_ranking'].transform(np.std) - community_summary_df['avg_cost_mean'] = community_summary_df.groupby(by=['city'])['avg_cost_fixed'].transform(np.mean) - community_summary_df['avg_cost_std'] = community_summary_df.groupby(by=['city'])['avg_cost_fixed'].transform(np.std) + try: + community_summary_df['avg_cost_mean'] = community_summary_df.groupby(by=['city'])['avg_cost_fixed'].transform(np.mean) + community_summary_df['avg_cost_std'] = community_summary_df.groupby(by=['city'])['avg_cost_fixed'].transform(np.std) + except Exception: + community_summary_df['avg_cost_mean'] = None + community_summary_df['avg_cost_std'] = None community_summary_df['total_price_mean'] = community_summary_df.groupby(by=['city'])['total_price_fixed'].transform( np.mean) @@ -314,9 +322,13 @@ def sync(city=None): # 清空缺少因子的老潜力 temp_df = es206_client.scan_by_sql_to_data_frame(clear_sql) clear_rating(temp_df) + run = False for city_now in reasonable_df['city'].unique(): #if pd.isna(reasonable_dict[city_now]): - sync_by_city(city_now) + if city_now == '昌都': + run = True + if run: + sync_by_city(city_now) else: clear_sql += f" and city='{city}' " temp_df = es206_client.scan_by_sql_to_data_frame(clear_sql) diff --git a/dags/shop/sell/business.py b/dags/shop/sell/business.py index 3245182..9004561 100644 --- a/dags/shop/sell/business.py +++ b/dags/shop/sell/business.py @@ -84,32 +84,32 @@ def calc(city, year_month): log.info(f"{city} {year_month} 商圈均价计算完毕!") -# yearmonth = sys.argv[1] if len(sys.argv) > 1 else '' -# -# if yearmonth == '': -# date = dateutils.add_months(datetime.today(), months=-1) -# yearmonth = int(dateutils.format_date(date, '%Y%m')) -# try: -# for city in areautils.get_all_cities(): -# calc(city, yearmonth) -# except Exception as e: -# log.error(e) +yearmonth = sys.argv[1] if len(sys.argv) > 1 else '' + +if yearmonth == '': + date = dateutils.add_months(datetime.today(), months=-1) + yearmonth = int(dateutils.format_date(date, '%Y%m')) +try: + for city in areautils.get_all_cities(): + calc(city, yearmonth) +except Exception as e: + log.error(e) # # 初始化从2024年9月 # 创建从2024年9月到2024年12月的日期范围 -date_range = pd.date_range(start='2024-09-01', end='2025-02-28', freq='M') - -# 创建一个包含这些日期的DataFrame -df = pd.DataFrame({'date': date_range, 'value': range(1, len(date_range) + 1)}) - -# 将日期设置为索引 -df.set_index('date', inplace=True) - -# 遍历DataFrame -for city in areautils.get_all_cities(): - for date, row in df.iterrows(): - try: - date_format = int(dateutils.format_date(date, '%Y%m')) - calc(city, date_format) - except Exception as e: - log.error(e) +# date_range = pd.date_range(start='2024-09-01', end='2025-02-28', freq='M') +# +# # 创建一个包含这些日期的DataFrame +# df = pd.DataFrame({'date': date_range, 'value': range(1, len(date_range) + 1)}) +# +# # 将日期设置为索引 +# df.set_index('date', inplace=True) +# +# # 遍历DataFrame +# for city in areautils.get_all_cities(): +# for date, row in df.iterrows(): +# try: +# date_format = int(dateutils.format_date(date, '%Y%m')) +# calc(city, date_format) +# except Exception as e: +# log.error(e) diff --git a/dags/shop/sell/price.py b/dags/shop/sell/price.py index d6690db..02030bb 100644 --- a/dags/shop/sell/price.py +++ b/dags/shop/sell/price.py @@ -81,32 +81,32 @@ def calc(city, year_month): log.info(f"{city} {year_month} 均价计算完毕!") -# yearmonth = sys.argv[1] if len(sys.argv) > 1 else '' -# -# if yearmonth == '': -# date = dateutils.add_months(datetime.today(), months=-1) -# yearmonth = int(dateutils.format_date(date, '%Y%m')) -# try: -# for city in areautils.get_all_cities(): -# calc(city, yearmonth) -# except Exception as e: -# log.error(e) +yearmonth = sys.argv[1] if len(sys.argv) > 1 else '' + +if yearmonth == '': + date = dateutils.add_months(datetime.today(), months=-1) + yearmonth = int(dateutils.format_date(date, '%Y%m')) +try: + for city in areautils.get_all_cities(): + calc(city, yearmonth) +except Exception as e: + log.error(e) # # 初始化从2024年9月 # 创建从2024年9月到2024年12月的日期范围 -date_range = pd.date_range(start='2024-09-01', end='2025-02-28', freq='M') - -# 创建一个包含这些日期的DataFrame -df = pd.DataFrame({'date': date_range, 'value': range(1, len(date_range) + 1)}) - -# 将日期设置为索引 -df.set_index('date', inplace=True) - -# 遍历DataFrame -for city in areautils.get_all_cities(): - for date, row in df.iterrows(): - try: - date_format = int(dateutils.format_date(date, '%Y%m')) - calc(city, date_format) - except Exception as e: - log.error(e) +# date_range = pd.date_range(start='2024-09-01', end='2025-02-28', freq='M') +# +# # 创建一个包含这些日期的DataFrame +# df = pd.DataFrame({'date': date_range, 'value': range(1, len(date_range) + 1)}) +# +# # 将日期设置为索引 +# df.set_index('date', inplace=True) +# +# # 遍历DataFrame +# for city in areautils.get_all_cities(): +# for date, row in df.iterrows(): +# try: +# date_format = int(dateutils.format_date(date, '%Y%m')) +# calc(city, date_format) +# except Exception as e: +# log.error(e)