新增商铺价格计算

This commit is contained in:
沈健
2025-04-09 17:49:17 +08:00
parent 12f1dcf7cf
commit b61199d0be
5 changed files with 146 additions and 2 deletions
View File
+26
View File
@@ -0,0 +1,26 @@
"""
小区周边配套DAG
"""
from airflow.operators.bash import BashOperator
import pendulum
from airflow import DAG
with DAG(
dag_id="shop_rent_dag",
schedule_interval="0 4 25 * *",
start_date=pendulum.datetime(2025, 4, 10, tz="Asia/Shanghai"),
catchup=True,
description="商铺租金城市区域商圈均价计算"
) as dag:
t1 = BashOperator(
task_id='price',
bash_command='python /opt/airflow/dags/repo/dags/shop/rent/price.py {{dag_run.conf["yearmonth"] if "yearmonth" in dag_run.conf}} ',
)
t2 = BashOperator(
task_id='business',
bash_command='python /opt/airflow/dags/repo/dags/shop/rent/business.py {{dag_run.conf["yearmonth"] if "yearmonth" in dag_run.conf}} ',
)
t1 >> t2
View File
+118
View File
@@ -0,0 +1,118 @@
import sys
from goojia_utils import ElasticSearchClient, MAIN_MONGO_CLIENT, DIDATA_DEV_ELASTIC_CLIENT, dateutils, areautils
import numpy as np
from pymongo import UpdateOne
from datetime import datetime
import logging
import pandas as pd
log = logging.getLogger(__name__)
es_client = ElasticSearchClient(host='192.168.0.206')
shop_rent_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'shop_rent_stat')
def calc(city, year_month):
"""
商圈计算
"""
if len(str(year_month)) != 6:
raise Exception('年月格式错误yyyyMM')
# shop = es_client.scan_by_sql_to_data_frame(
# "select * from latest_seawater where cat='shop_rent' and city='" + city + "' and p_date>=" + str(
# year_month) + "01 and p_date<=" + str(year_month) + "31 ")
sql = 'select short_province, short_city, name, shape from bk_business'
if pd.notna(city):
sql += f" where short_city='{city}'"
business = DIDATA_DEV_ELASTIC_CLIENT.scan_by_sql_to_data_frame(sql)
actions = list()
last_month = int(dateutils.format_date(dateutils.add_months(dateutils.parse_date(str(year_month), '%Y%m'), months=-1), '%Y%m'))
for index, item in business.iterrows():
curr_city = item['short_city']
name = item['name']
shape = item['shape']['coordinates'][0]
query = {
"size": 0, # 不返回具体文档
"query": {
"bool": {
"filter": [
{"term": {"city": curr_city}},
{"term": {"cat": "shop_rent"}},
{"range": {"p_date": {"gte": str(year_month) + "01","lte": str(year_month) + "31"}}},
{
"geo_polygon": {
"location": {
"points": shape
}
}
}
]
}
},
"aggs": {
"avg_price": {
"percentiles": {
"field": "avg_price",
"percents": [50]
}
}
}
}
# 执行查询
res = es_client.elastic_client.search(index="latest_seawater", body=query)
# 提取聚合结果
avg_price = res["aggregations"]["avg_price"]["values"]["50.0"]
if pd.isna(avg_price):
# 处理均价为空情况,用上月均价替代
stat = shop_rent_stat_conn.find_one({'city': city, 'cat': 'business', 'date': last_month})
if pd.notna(stat) and pd.notna(stat['avg_price']):
avg_price = stat['avg_price']
if pd.notna(avg_price):
action = UpdateOne({'city': city, 'name': name, 'cat': 'business', 'date': year_month},
{'$set': {'city': city, 'cat': 'business', 'date': year_month, 'avg_price': avg_price,
'updated_date': datetime.now()}},
upsert=True)
actions.append(action)
if len(actions) == 1000:
shop_rent_stat_conn.bulk_write(actions)
actions.clear()
if len(actions) > 0:
shop_rent_stat_conn.bulk_write(actions)
log.info(f"{city} {year_month} 商圈均价计算完毕!")
yearmonth = sys.argv[1] if len(sys.argv) > 1 else ''
if yearmonth == '':
date = dateutils.add_months(datetime.today(), months=-1)
yearmonth = int(dateutils.format_date(date, '%Y%m'))
try:
for city in areautils.get_all_cities():
calc(city, yearmonth)
except Exception as e:
log.error(e)
# # 初始化从2024年9月
# 创建从2024年9月到2024年12月的日期范围
# date_range = pd.date_range(start='2024-09-01', end='2025-02-28', freq='M')
#
# # 创建一个包含这些日期的DataFrame
# df = pd.DataFrame({'date': date_range, 'value': range(1, len(date_range) + 1)})
#
# # 将日期设置为索引
# df.set_index('date', inplace=True)
#
# # 遍历DataFrame
# for city in areautils.get_all_cities():
# for date, row in df.iterrows():
# try:
# date_format = int(dateutils.format_date(date, '%Y%m'))
# calc(city, date_format)
# except Exception as e:
# log.error(e)
+2 -2
View File
@@ -20,7 +20,7 @@ def calc(city, year_month):
"select * from latest_seawater where cat='shop_rent' and city='" + city + "' and p_date>=" + str(
year_month) + "01 and p_date<=" + str(year_month) + "31 ")
if len(shop) > 0:
shop['day_avg_price'] = shop['day_rent_price']
shop['day_avg_price'] = shop['avg_price'] / 30
# 计算总价的Z-score
shop['Z_Score'] = (shop['day_avg_price'] - shop['day_avg_price'].mean()) / shop['day_avg_price'].std()
# 剔除Z-score超过0.1的异常值
@@ -94,7 +94,7 @@ except Exception as e:
# # 初始化从2024年9月
# 创建从2024年9月到2024年12月的日期范围
# date_range = pd.date_range(start='2024-09-01', end='2025-01-31', freq='M')
# date_range = pd.date_range(start='2024-09-01', end='2025-02-28', freq='M')
#
# # 创建一个包含这些日期的DataFrame
# df = pd.DataFrame({'date': date_range, 'value': range(1, len(date_range) + 1)})