新增写字楼均价计算

This commit is contained in:
沈健
2025-01-20 18:17:27 +08:00
parent cb469f2f41
commit a226629f9c
4 changed files with 101 additions and 0 deletions
View File
+22
View File
@@ -0,0 +1,22 @@
"""
小区周边配套DAG
"""
from airflow.operators.bash import BashOperator
import pendulum
from airflow import DAG
with DAG(
dag_id="rent_dag",
# 每月25日1点运行
schedule_interval="0 1 25 * *",
start_date=pendulum.datetime(2024, 7, 17, tz="Asia/Shanghai"),
catchup=True
) as dag:
t1 = BashOperator(
task_id='rent',
bash_command='python /opt/airflow/dags/repo/dags/office/rent/RegionPrice.py {{dag_run.conf["yearmonth"] if "yearmonth" in dag_run.conf}} ',
)
t1
+79
View File
@@ -0,0 +1,79 @@
import sys
from goojia_utils import ElasticSearchClient, MAIN_MONGO_CLIENT, dateutils, areautils
import numpy as np
from pymongo import UpdateOne
from datetime import datetime
import logging
import pandas as pd
log = logging.getLogger(__name__)
es_client = ElasticSearchClient(host='192.168.0.206')
office_rent_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'office_rent_stat')
def calc(city, year_month):
if len(str(year_month)) != 6:
raise Exception('年月格式错误yyyyMM')
office = es_client.scan_by_sql_to_data_frame(
"select * from latest_seawater where cat='office_rent' and city='" + city + "' and p_date>=" + str(
year_month) + "01 and p_date<=" + str(year_month) + "31 ")
if len(office) == 0:
return
office['day_avg_price'] = office['avg_price'] / 30
# 计算总价的Z-score
office['Z_Score'] = (office['day_avg_price'] - office['day_avg_price'].mean()) / office['day_avg_price'].std()
# 剔除Z-score超过2的异常值
office_cleaned = office[np.abs(office['Z_Score']) < 0.1]
city_price = round(office_cleaned['day_avg_price'].mean(), 2)
region_avg_prices = office_cleaned.groupby(office_cleaned['region'])['day_avg_price'].mean()
actions = list()
# 写入MONGODB
action = UpdateOne({'city': city, 'cat': 'city', 'date': year_month},
{'$set': {'city': city, 'cat': 'city', 'date': year_month, 'avg_price': city_price,
'updated_date': datetime.now()}},
upsert=True)
actions.append(action)
for region, avg_price in region_avg_prices.items():
action = UpdateOne({'city': city, 'cat': 'district', 'date': year_month, 'district': region},
{'$set': {'city': city, 'cat': 'district', 'date': year_month, 'district': region,
'avg_price': round(avg_price, 2), 'updated_date': datetime.now()}},
upsert=True)
actions.append(action)
office_rent_stat_conn.bulk_write(actions)
log.info(f"{city} {year_month} 均价计算完毕!")
yearmonth = sys.argv[1] if len(sys.argv) > 1 else ''
if yearmonth == '':
date = dateutils.add_months(datetime.today(), months=-1)
yearmonth = int(dateutils.format_date(date, '%Y%m'))
try:
for city in areautils.get_all_cities():
calc(city, yearmonth)
except Exception as e:
log.error(e)
# 初始化跑两年
# 创建从2024年1月到2024年12月的日期范围
# date_range = pd.date_range(start='2023-01-01', end='2024-12-31', freq='M')
#
# # 创建一个包含这些日期的DataFrame
# df = pd.DataFrame({'date': date_range, 'value': range(1, len(date_range) + 1)})
#
# # 将日期设置为索引
# df.set_index('date', inplace=True)
#
# # 遍历DataFrame
# for date, row in df.iterrows():
# date_format = int(dateutils.format_date(date, '%Y%m'))
# try:
# for city in areautils.get_all_cities():
# calc(city, date_format)
# except Exception as e:
# log.error(e)
View File