From dcce114957d364ab61f6132a73090f1591f55a57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Mon, 31 Mar 2025 14:43:22 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96=E4=BB=B7=E6=A0=BC=E7=B1=BB?= =?UTF-8?q?=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/office/dag_office_sell.py | 7 ++++++- dags/office/sell/base_price.py | 32 ++++++++++++++++------------- dags/office/sell/fill_base_price.py | 4 +--- dags/office/sell/fill_price.py | 20 +++++++++--------- dags/office/sell/price.py | 11 +++++----- 5 files changed, 41 insertions(+), 33 deletions(-) diff --git a/dags/office/dag_office_sell.py b/dags/office/dag_office_sell.py index d1a5ae9..8cbbe79 100644 --- a/dags/office/dag_office_sell.py +++ b/dags/office/dag_office_sell.py @@ -16,5 +16,10 @@ with DAG( task_id='office_sell', bash_command='python /opt/airflow/dags/repo/dags/office/sell/price.py {{dag_run.conf["yearmonth"] if "yearmonth" in dag_run.conf}} ', ) - t1 + + t2 = BashOperator( + task_id='fill_price', + bash_command='python /opt/airflow/dags/repo/dags/office/sell/fill_price.py {{dag_run.conf["yearmonth"] if "yearmonth" in dag_run.conf}}', + ) + t1 >> t2 diff --git a/dags/office/sell/base_price.py b/dags/office/sell/base_price.py index e38f0ed..cf8c0bd 100644 --- a/dags/office/sell/base_price.py +++ b/dags/office/sell/base_price.py @@ -76,6 +76,7 @@ def calc(city, year_month): # 第一步:计算有样本写字楼的基准价 print("有样本数量:" + str(len(merge_df))) actions = list() + count = 0 for index, item in merge_df.iterrows(): city = item['city'] region = item['region'] @@ -96,9 +97,13 @@ def calc(city, year_month): actions.append(action) if len(actions) == 1000: office_sell_stat_conn.bulk_write(actions) + count += len(actions) + print("已计算" + str(count) + "条数据") actions.clear() if len(actions) > 0: office_sell_stat_conn.bulk_write(actions) + count += len(actions) + print("已计算" + str(count) + "条数据") actions.clear() # 第二步 取周边3公里内同等级、同类型的最近10个有样本写字楼的基准价的平均值 @@ -155,9 +160,13 @@ def calc(city, year_month): actions.append(action) if len(actions) == 1000: office_sell_stat_conn.bulk_write(actions) + count += len(actions) + print("已计算" + str(count) + "条数据") actions.clear() if len(actions) > 0: office_sell_stat_conn.bulk_write(actions) + count += len(actions) + print("已计算" + str(count) + "条数据") actions.clear() # 系数计算在Jupyter中 @@ -259,9 +268,13 @@ def calc(city, year_month): actions.append(action) if len(actions) == 1000: office_sell_stat_conn.bulk_write(actions) + count += len(actions) + print("已计算" + str(count) + "条数据") actions.clear() if len(actions) > 0: office_sell_stat_conn.bulk_write(actions) + count += len(actions) + print("已计算" + str(count) + "条数据") actions.clear() # 第五步 query = {'cat': 'office', 'date': year_month, 'calc_type': {'$in': [1, 2, 3]}} @@ -331,23 +344,14 @@ def calc(city, year_month): actions.append(action) if len(actions) == 1000: office_sell_stat_conn.bulk_write(actions) + count += len(actions) + print("已计算" + str(count) + "条数据") actions.clear() if len(actions) > 0: office_sell_stat_conn.bulk_write(actions) + count += len(actions) + print("已计算" + str(count) + "条数据") actions.clear() yearmonth = sys.argv[1] if len(sys.argv) > 1 else None -for yearmonth in [ - "202406", - "202407", - "202408", - "202409", - "202410", - "202411", - "202412", - "202501" -]: - calc(None, yearmonth) -# # calc("广州", yearmonth) -# # calc("武汉", yearmonth) -# calc("深圳", yearmonth) \ No newline at end of file +calc(None, yearmonth) \ No newline at end of file diff --git a/dags/office/sell/fill_base_price.py b/dags/office/sell/fill_base_price.py index 32b08a2..71c759c 100644 --- a/dags/office/sell/fill_base_price.py +++ b/dags/office/sell/fill_base_price.py @@ -60,7 +60,7 @@ def calc(city, year_month): action = UpdateOne({'cat': 'office', 'city': curr_city, 'district': region, 'name': name, 'date': year_month}, {'$set': {'city': curr_city, 'cat': 'office', 'date': year_month, 'district': region, 'name': name, - 'avg_price': sell_price, 'estate_type2': estate_type2, 'bed_kind': bed_kind, + 'avg_price': int(sell_price), 'estate_type2': estate_type2, 'bed_kind': bed_kind, 'location': location, 'calc_type': 5, 'updated_date': datetime.now()}}, upsert=True) @@ -85,8 +85,6 @@ if yearmonth == '': try: if type(yearmonth) is str: yearmonth = int(yearmonth) - # for curr_city in areautils.get_all_cities(): - # print("正在处理城市:{}".format(curr_city)) calc(None, yearmonth) except Exception as e: print(e) diff --git a/dags/office/sell/fill_price.py b/dags/office/sell/fill_price.py index cf4480f..1187d07 100644 --- a/dags/office/sell/fill_price.py +++ b/dags/office/sell/fill_price.py @@ -15,6 +15,7 @@ office_rent_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'office def calc(city, year_month): + print(f"开始处理{city}, {year_month}") query = {'date': year_month, 'cat': 'office'} if pd.notna(city): query['city'] = city @@ -62,7 +63,7 @@ def calc(city, year_month): # 写入MONGODB for index, row in merge2_df.iterrows(): action = UpdateOne({'city': row['city'], 'cat': 'city', 'date': year_month}, - {'$set': {'city': row['city'], 'cat': 'city', 'date': year_month, 'calc_type': 2, 'avg_price': row['sell_price'], + {'$set': {'city': row['city'], 'cat': 'city', 'date': year_month, 'calc_type': 2, 'avg_price': int(row['sell_price']), 'updated_date': datetime.now()}}, upsert=True) actions.append(action) @@ -91,7 +92,7 @@ def calc(city, year_month): # 写入MONGODB for index, row in merge2_df.iterrows(): action = UpdateOne({'city': row['city'], 'district': row['district'], 'cat': 'district', 'date': year_month}, - {'$set': {'city': row['city'], 'district': row['district'], 'cat': 'district', 'date': year_month, 'calc_type': 2, 'avg_price': row['sell_price'], + {'$set': {'city': row['city'], 'district': row['district'], 'cat': 'district', 'date': year_month, 'calc_type': 2, 'avg_price': int(row['sell_price']), 'updated_date': datetime.now()}}, upsert=True) actions.append(action) @@ -114,7 +115,7 @@ except Exception as e: # # 初始化从2024年9月 # 创建从2024年9月到2024年12月的日期范围 -# date_range = pd.date_range(start='2024-09-01', end='2025-01-31', freq='M') +# date_range = pd.date_range(start='2025-01-31', end='2025-01-31', freq='M') # # # 创建一个包含这些日期的DataFrame # df = pd.DataFrame({'date': date_range, 'value': range(1, len(date_range) + 1)}) @@ -123,10 +124,9 @@ except Exception as e: # df.set_index('date', inplace=True) # # # 遍历DataFrame -# for city in areautils.get_all_cities(): -# for date, row in df.iterrows(): -# try: -# date_format = int(dateutils.format_date(date, '%Y%m')) -# calc(city, date_format) -# except Exception as e: -# log.error(e) +# for date, row in df.iterrows(): +# try: +# date_format = int(dateutils.format_date(date, '%Y%m')) +# calc(None, date_format) +# except Exception as e: +# log.error(e) diff --git a/dags/office/sell/price.py b/dags/office/sell/price.py index 9126a94..9b000ee 100644 --- a/dags/office/sell/price.py +++ b/dags/office/sell/price.py @@ -27,7 +27,7 @@ def calc(city, year_month): office_cleaned = office[np.abs(office['Z_Score']) < 0.1] # 至少样本数大于等于10 if len(office_cleaned['day_avg_price']) >= 10: - city_price = round(office_cleaned['day_avg_price'].median(), 0) + city_price = int(office_cleaned['day_avg_price'].median()) region_median_count_prices = office_cleaned.groupby(office_cleaned['region'])['day_avg_price'].agg(['median', 'count']).to_dict() else: city_price = None @@ -52,7 +52,7 @@ def calc(city, year_month): # 写入MONGODB action = UpdateOne({'city': city, 'cat': 'city', 'date': year_month}, - {'$set': {'city': city, 'cat': 'city', 'date': year_month, 'avg_price': city_price, + {'$set': {'city': city, 'cat': 'city', 'date': year_month, 'avg_price': int(city_price), 'updated_date': datetime.now()}}, upsert=True) @@ -74,19 +74,20 @@ def calc(city, year_month): action = UpdateOne({'city': city, 'cat': 'district', 'date': year_month, 'district': region}, {'$set': {'city': city, 'cat': 'district', 'date': year_month, 'district': region, - 'avg_price': round(avg_price, 0), 'updated_date': datetime.now()}}, + 'avg_price': int(avg_price), 'updated_date': datetime.now()}}, upsert=True) actions.append(action) office_sell_stat_conn.bulk_write(actions) log.info(f"{city} {year_month} 均价计算完毕!") - yearmonth = sys.argv[1] if len(sys.argv) > 1 else '' if yearmonth == '': date = dateutils.add_months(datetime.today(), months=-1) yearmonth = int(dateutils.format_date(date, '%Y%m')) try: + if type(yearmonth) is str: + yearmonth = int(yearmonth) for city in areautils.get_all_cities(): calc(city, yearmonth) except Exception as e: @@ -94,7 +95,7 @@ except Exception as e: # # 初始化从2024年9月 # 创建从2024年9月到2024年12月的日期范围 -# date_range = pd.date_range(start='2024-09-01', end='2025-01-31', freq='M') +# date_range = pd.date_range(start='2024-06-01', end='2025-01-31', freq='M') # # # 创建一个包含这些日期的DataFrame # df = pd.DataFrame({'date': date_range, 'value': range(1, len(date_range) + 1)})