diff --git a/dags/shop/__init__.py b/dags/shop/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dags/shop/dag_shop_rent.py b/dags/shop/dag_shop_rent.py new file mode 100644 index 0000000..fd8e6a5 --- /dev/null +++ b/dags/shop/dag_shop_rent.py @@ -0,0 +1,26 @@ +""" +小区周边配套DAG +""" +from airflow.operators.bash import BashOperator + +import pendulum + +from airflow import DAG + +with DAG( + dag_id="shop_rent_dag", + schedule_interval="0 4 25 * *", + start_date=pendulum.datetime(2025, 4, 10, tz="Asia/Shanghai"), + catchup=True, + description="商铺租金城市区域商圈均价计算" +) as dag: + t1 = BashOperator( + task_id='price', + bash_command='python /opt/airflow/dags/repo/dags/shop/rent/price.py {{dag_run.conf["yearmonth"] if "yearmonth" in dag_run.conf}} ', + ) + t2 = BashOperator( + task_id='business', + bash_command='python /opt/airflow/dags/repo/dags/shop/rent/business.py {{dag_run.conf["yearmonth"] if "yearmonth" in dag_run.conf}} ', + ) + t1 >> t2 + diff --git a/dags/shop/rent/__init__.py b/dags/shop/rent/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dags/shop/rent/business.py b/dags/shop/rent/business.py new file mode 100644 index 0000000..6cb31fe --- /dev/null +++ b/dags/shop/rent/business.py @@ -0,0 +1,118 @@ +import sys + +from goojia_utils import ElasticSearchClient, MAIN_MONGO_CLIENT, DIDATA_DEV_ELASTIC_CLIENT, dateutils, areautils +import numpy as np +from pymongo import UpdateOne +from datetime import datetime +import logging +import pandas as pd + +log = logging.getLogger(__name__) + +es_client = ElasticSearchClient(host='192.168.0.206') +shop_rent_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'shop_rent_stat') + + +def calc(city, year_month): + """ + 商圈计算 + """ + if len(str(year_month)) != 6: + raise Exception('年月格式错误yyyyMM') + # shop = es_client.scan_by_sql_to_data_frame( + # "select * from latest_seawater where cat='shop_rent' and city='" + city + "' and p_date>=" + str( + # year_month) + "01 and p_date<=" + str(year_month) + "31 ") + sql = 'select short_province, short_city, name, shape from bk_business' + if pd.notna(city): + sql += f" where short_city='{city}'" + business = DIDATA_DEV_ELASTIC_CLIENT.scan_by_sql_to_data_frame(sql) + + actions = list() + last_month = int(dateutils.format_date(dateutils.add_months(dateutils.parse_date(str(year_month), '%Y%m'), months=-1), '%Y%m')) + + for index, item in business.iterrows(): + curr_city = item['short_city'] + name = item['name'] + shape = item['shape']['coordinates'][0] + query = { + "size": 0, # 不返回具体文档 + "query": { + "bool": { + "filter": [ + {"term": {"city": curr_city}}, + {"term": {"cat": "shop_rent"}}, + {"range": {"p_date": {"gte": str(year_month) + "01","lte": str(year_month) + "31"}}}, + { + "geo_polygon": { + "location": { + "points": shape + } + } + } + ] + } + }, + "aggs": { + "avg_price": { + "percentiles": { + "field": "avg_price", + "percents": [50] + } + } + } + } + + # 执行查询 + res = es_client.elastic_client.search(index="latest_seawater", body=query) + + # 提取聚合结果 + avg_price = res["aggregations"]["avg_price"]["values"]["50.0"] + if pd.isna(avg_price): + # 处理均价为空情况,用上月均价替代 + stat = shop_rent_stat_conn.find_one({'city': city, 'cat': 'business', 'date': last_month}) + if pd.notna(stat) and pd.notna(stat['avg_price']): + avg_price = stat['avg_price'] + if pd.notna(avg_price): + action = UpdateOne({'city': city, 'name': name, 'cat': 'business', 'date': year_month}, + {'$set': {'city': city, 'cat': 'business', 'date': year_month, 'avg_price': avg_price, + 'updated_date': datetime.now()}}, + upsert=True) + actions.append(action) + if len(actions) == 1000: + shop_rent_stat_conn.bulk_write(actions) + actions.clear() + + if len(actions) > 0: + shop_rent_stat_conn.bulk_write(actions) + log.info(f"{city} {year_month} 商圈均价计算完毕!") + + +yearmonth = sys.argv[1] if len(sys.argv) > 1 else '' + +if yearmonth == '': + date = dateutils.add_months(datetime.today(), months=-1) + yearmonth = int(dateutils.format_date(date, '%Y%m')) +try: + for city in areautils.get_all_cities(): + calc(city, yearmonth) +except Exception as e: + log.error(e) + +# # 初始化从2024年9月 +# 创建从2024年9月到2024年12月的日期范围 +# date_range = pd.date_range(start='2024-09-01', end='2025-02-28', freq='M') +# +# # 创建一个包含这些日期的DataFrame +# df = pd.DataFrame({'date': date_range, 'value': range(1, len(date_range) + 1)}) +# +# # 将日期设置为索引 +# df.set_index('date', inplace=True) +# +# # 遍历DataFrame +# for city in areautils.get_all_cities(): +# for date, row in df.iterrows(): +# try: +# date_format = int(dateutils.format_date(date, '%Y%m')) +# calc(city, date_format) +# except Exception as e: +# log.error(e) diff --git a/dags/shop/rent/price.py b/dags/shop/rent/price.py index 13d648c..2b0cfa8 100644 --- a/dags/shop/rent/price.py +++ b/dags/shop/rent/price.py @@ -20,7 +20,7 @@ def calc(city, year_month): "select * from latest_seawater where cat='shop_rent' and city='" + city + "' and p_date>=" + str( year_month) + "01 and p_date<=" + str(year_month) + "31 ") if len(shop) > 0: - shop['day_avg_price'] = shop['day_rent_price'] + shop['day_avg_price'] = shop['avg_price'] / 30 # 计算总价的Z-score shop['Z_Score'] = (shop['day_avg_price'] - shop['day_avg_price'].mean()) / shop['day_avg_price'].std() # 剔除Z-score超过0.1的异常值 @@ -94,7 +94,7 @@ except Exception as e: # # 初始化从2024年9月 # 创建从2024年9月到2024年12月的日期范围 -# date_range = pd.date_range(start='2024-09-01', end='2025-01-31', freq='M') +# date_range = pd.date_range(start='2024-09-01', end='2025-02-28', freq='M') # # # 创建一个包含这些日期的DataFrame # df = pd.DataFrame({'date': date_range, 'value': range(1, len(date_range) + 1)})