diff --git a/dags/office/__init__.py b/dags/office/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dags/office/dag_rent.py b/dags/office/dag_rent.py new file mode 100644 index 0000000..2be459c --- /dev/null +++ b/dags/office/dag_rent.py @@ -0,0 +1,22 @@ +""" +小区周边配套DAG +""" +from airflow.operators.bash import BashOperator + +import pendulum + +from airflow import DAG + +with DAG( + dag_id="rent_dag", + # 每月25日1点运行 + schedule_interval="0 1 25 * *", + start_date=pendulum.datetime(2024, 7, 17, tz="Asia/Shanghai"), + catchup=True +) as dag: + t1 = BashOperator( + task_id='rent', + bash_command='python /opt/airflow/dags/repo/dags/office/rent/RegionPrice.py {{dag_run.conf["yearmonth"] if "yearmonth" in dag_run.conf}} ', + ) + t1 + diff --git a/dags/office/rent/RegionPrice.py b/dags/office/rent/RegionPrice.py new file mode 100644 index 0000000..844bbea --- /dev/null +++ b/dags/office/rent/RegionPrice.py @@ -0,0 +1,79 @@ +import sys + +from goojia_utils import ElasticSearchClient, MAIN_MONGO_CLIENT, dateutils, areautils +import numpy as np +from pymongo import UpdateOne +from datetime import datetime +import logging +import pandas as pd + +log = logging.getLogger(__name__) + +es_client = ElasticSearchClient(host='192.168.0.206') +office_rent_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'office_rent_stat') + + +def calc(city, year_month): + if len(str(year_month)) != 6: + raise Exception('年月格式错误yyyyMM') + office = es_client.scan_by_sql_to_data_frame( + "select * from latest_seawater where cat='office_rent' and city='" + city + "' and p_date>=" + str( + year_month) + "01 and p_date<=" + str(year_month) + "31 ") + if len(office) == 0: + return + office['day_avg_price'] = office['avg_price'] / 30 + # 计算总价的Z-score + office['Z_Score'] = (office['day_avg_price'] - office['day_avg_price'].mean()) / office['day_avg_price'].std() + # 剔除Z-score超过2的异常值 + office_cleaned = office[np.abs(office['Z_Score']) < 0.1] + city_price = round(office_cleaned['day_avg_price'].mean(), 2) + region_avg_prices = office_cleaned.groupby(office_cleaned['region'])['day_avg_price'].mean() + + actions = list() + + # 写入MONGODB + action = UpdateOne({'city': city, 'cat': 'city', 'date': year_month}, + {'$set': {'city': city, 'cat': 'city', 'date': year_month, 'avg_price': city_price, + 'updated_date': datetime.now()}}, + upsert=True) + + actions.append(action) + for region, avg_price in region_avg_prices.items(): + action = UpdateOne({'city': city, 'cat': 'district', 'date': year_month, 'district': region}, + {'$set': {'city': city, 'cat': 'district', 'date': year_month, 'district': region, + 'avg_price': round(avg_price, 2), 'updated_date': datetime.now()}}, + upsert=True) + actions.append(action) + office_rent_stat_conn.bulk_write(actions) + log.info(f"{city} {year_month} 均价计算完毕!") + +yearmonth = sys.argv[1] if len(sys.argv) > 1 else '' + + +if yearmonth == '': + date = dateutils.add_months(datetime.today(), months=-1) + yearmonth = int(dateutils.format_date(date, '%Y%m')) +try: + for city in areautils.get_all_cities(): + calc(city, yearmonth) +except Exception as e: + log.error(e) + +# 初始化跑两年 +# 创建从2024年1月到2024年12月的日期范围 +# date_range = pd.date_range(start='2023-01-01', end='2024-12-31', freq='M') +# +# # 创建一个包含这些日期的DataFrame +# df = pd.DataFrame({'date': date_range, 'value': range(1, len(date_range) + 1)}) +# +# # 将日期设置为索引 +# df.set_index('date', inplace=True) +# +# # 遍历DataFrame +# for date, row in df.iterrows(): +# date_format = int(dateutils.format_date(date, '%Y%m')) +# try: +# for city in areautils.get_all_cities(): +# calc(city, date_format) +# except Exception as e: +# log.error(e) diff --git a/dags/office/rent/__init__.py b/dags/office/rent/__init__.py new file mode 100644 index 0000000..e69de29