新增商铺出售
This commit is contained in:
@@ -0,0 +1,26 @@
|
||||
"""
|
||||
小区周边配套DAG
|
||||
"""
|
||||
from airflow.operators.bash import BashOperator
|
||||
|
||||
import pendulum
|
||||
|
||||
from airflow import DAG
|
||||
|
||||
with DAG(
|
||||
dag_id="shop_sell_dag",
|
||||
schedule_interval="0 6 25 * *",
|
||||
start_date=pendulum.datetime(2025, 4, 10, tz="Asia/Shanghai"),
|
||||
catchup=True,
|
||||
description="商铺出售城市区域商圈均价计算"
|
||||
) as dag:
|
||||
t1 = BashOperator(
|
||||
task_id='price',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/shop/sell/price.py {{dag_run.conf["yearmonth"] if "yearmonth" in dag_run.conf}} ',
|
||||
)
|
||||
t2 = BashOperator(
|
||||
task_id='business',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/shop/sell/business.py {{dag_run.conf["yearmonth"] if "yearmonth" in dag_run.conf}} ',
|
||||
)
|
||||
t1 >> t2
|
||||
|
||||
@@ -0,0 +1,115 @@
|
||||
import sys
|
||||
|
||||
from goojia_utils import ElasticSearchClient, MAIN_MONGO_CLIENT, DIDATA_DEV_ELASTIC_CLIENT, dateutils, areautils
|
||||
import numpy as np
|
||||
from pymongo import UpdateOne
|
||||
from datetime import datetime
|
||||
import logging
|
||||
import pandas as pd
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
es_client = ElasticSearchClient(host='192.168.0.206')
|
||||
shop_sell_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'shop_sell_stat')
|
||||
|
||||
|
||||
def calc(city, year_month):
|
||||
"""
|
||||
商圈计算
|
||||
"""
|
||||
if len(str(year_month)) != 6:
|
||||
raise Exception('年月格式错误yyyyMM')
|
||||
sql = 'select short_province, short_city, name, shape from bk_business'
|
||||
if pd.notna(city):
|
||||
sql += f" where short_city='{city}'"
|
||||
business = DIDATA_DEV_ELASTIC_CLIENT.scan_by_sql_to_data_frame(sql)
|
||||
|
||||
actions = list()
|
||||
last_month = int(dateutils.format_date(dateutils.add_months(dateutils.parse_date(str(year_month), '%Y%m'), months=-1), '%Y%m'))
|
||||
|
||||
for index, item in business.iterrows():
|
||||
curr_city = item['short_city']
|
||||
name = item['name']
|
||||
shape = item['shape']['coordinates'][0]
|
||||
query = {
|
||||
"size": 0, # 不返回具体文档
|
||||
"query": {
|
||||
"bool": {
|
||||
"filter": [
|
||||
{"term": {"city": curr_city}},
|
||||
{"term": {"cat": "shop_sell"}},
|
||||
{"range": {"p_date": {"gte": str(year_month) + "01","lte": str(year_month) + "31"}}},
|
||||
{
|
||||
"geo_polygon": {
|
||||
"location": {
|
||||
"points": shape
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"aggs": {
|
||||
"avg_price": {
|
||||
"percentiles": {
|
||||
"field": "avg_price",
|
||||
"percents": [50]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# 执行查询
|
||||
res = es_client.elastic_client.search(index="latest_seawater", body=query)
|
||||
|
||||
# 提取聚合结果
|
||||
avg_price = res["aggregations"]["avg_price"]["values"]["50.0"]
|
||||
if pd.isna(avg_price):
|
||||
# 处理均价为空情况,用上月均价替代
|
||||
stat = shop_sell_stat_conn.find_one({'city': city, 'cat': 'business', 'date': last_month})
|
||||
if pd.notna(stat) and pd.notna(stat['avg_price']):
|
||||
avg_price = stat['avg_price']
|
||||
if pd.notna(avg_price):
|
||||
action = UpdateOne({'city': city, 'name': name, 'cat': 'business', 'date': year_month},
|
||||
{'$set': {'city': city, 'name': name, 'cat': 'business', 'date': year_month, 'avg_price': int(avg_price),
|
||||
'updated_date': datetime.now()}},
|
||||
upsert=True)
|
||||
actions.append(action)
|
||||
if len(actions) == 1000:
|
||||
shop_sell_stat_conn.bulk_write(actions)
|
||||
actions.clear()
|
||||
|
||||
if len(actions) > 0:
|
||||
shop_sell_stat_conn.bulk_write(actions)
|
||||
log.info(f"{city} {year_month} 商圈均价计算完毕!")
|
||||
|
||||
|
||||
# yearmonth = sys.argv[1] if len(sys.argv) > 1 else ''
|
||||
#
|
||||
# if yearmonth == '':
|
||||
# date = dateutils.add_months(datetime.today(), months=-1)
|
||||
# yearmonth = int(dateutils.format_date(date, '%Y%m'))
|
||||
# try:
|
||||
# for city in areautils.get_all_cities():
|
||||
# calc(city, yearmonth)
|
||||
# except Exception as e:
|
||||
# log.error(e)
|
||||
|
||||
# # 初始化从2024年9月
|
||||
# 创建从2024年9月到2024年12月的日期范围
|
||||
date_range = pd.date_range(start='2024-09-01', end='2025-02-28', freq='M')
|
||||
|
||||
# 创建一个包含这些日期的DataFrame
|
||||
df = pd.DataFrame({'date': date_range, 'value': range(1, len(date_range) + 1)})
|
||||
|
||||
# 将日期设置为索引
|
||||
df.set_index('date', inplace=True)
|
||||
|
||||
# 遍历DataFrame
|
||||
for city in areautils.get_all_cities():
|
||||
for date, row in df.iterrows():
|
||||
try:
|
||||
date_format = int(dateutils.format_date(date, '%Y%m'))
|
||||
calc(city, date_format)
|
||||
except Exception as e:
|
||||
log.error(e)
|
||||
@@ -0,0 +1,112 @@
|
||||
import sys
|
||||
|
||||
from goojia_utils import ElasticSearchClient, MAIN_MONGO_CLIENT, dateutils, areautils
|
||||
import numpy as np
|
||||
from pymongo import UpdateOne
|
||||
from datetime import datetime
|
||||
import logging
|
||||
import pandas as pd
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
es_client = ElasticSearchClient(host='192.168.0.206')
|
||||
shop_sell_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'shop_sell_stat')
|
||||
|
||||
|
||||
def calc(city, year_month):
|
||||
if len(str(year_month)) != 6:
|
||||
raise Exception('年月格式错误yyyyMM')
|
||||
shop = es_client.scan_by_sql_to_data_frame(
|
||||
"select * from latest_seawater where cat='shop_sell' and city='" + city + "' and p_date>=" + str(
|
||||
year_month) + "01 and p_date<=" + str(year_month) + "31 ")
|
||||
if len(shop) > 0:
|
||||
shop['day_avg_price'] = shop['avg_price'] / 30
|
||||
# 计算总价的Z-score
|
||||
shop['Z_Score'] = (shop['day_avg_price'] - shop['day_avg_price'].mean()) / shop['day_avg_price'].std()
|
||||
# 剔除Z-score超过0.1的异常值
|
||||
shop_cleaned = shop[np.abs(shop['Z_Score']) < 0.1]
|
||||
# 至少样本数大于等于10
|
||||
if len(shop_cleaned['day_avg_price']) >= 10:
|
||||
city_price = round(shop_cleaned['day_avg_price'].median(), 2)
|
||||
region_median_count_prices = shop_cleaned.groupby(shop_cleaned['region'])['day_avg_price'].agg(['median', 'count']).to_dict()
|
||||
else:
|
||||
city_price = None
|
||||
region_median_count_prices = {'median': {}, 'count': {}}
|
||||
else:
|
||||
city_price = None
|
||||
region_median_count_prices = {'median': {}, 'count': {}}
|
||||
|
||||
actions = list()
|
||||
|
||||
last_month = int(
|
||||
dateutils.format_date(dateutils.add_months(dateutils.parse_date(str(year_month), '%Y%m'), months=-1), '%Y%m'))
|
||||
if pd.isna(city_price):
|
||||
# 处理均价为空情况,用上月均价替代
|
||||
stat = shop_sell_stat_conn.find_one({'city': city, 'cat': 'city', 'date': last_month})
|
||||
if pd.isna(stat):
|
||||
return False
|
||||
last_month_city_price = stat['avg_price']
|
||||
if pd.isna(last_month_city_price):
|
||||
return False
|
||||
city_price = last_month_city_price
|
||||
|
||||
# 写入MONGODB
|
||||
action = UpdateOne({'city': city, 'cat': 'city', 'date': year_month},
|
||||
{'$set': {'city': city, 'cat': 'city', 'date': year_month, 'avg_price': int(city_price),
|
||||
'updated_date': datetime.now()}},
|
||||
upsert=True)
|
||||
|
||||
actions.append(action)
|
||||
for region in areautils.get_all_districts(city):
|
||||
region_avg_prices = region_median_count_prices['median']
|
||||
region_count = region_median_count_prices['count']
|
||||
if region in region_avg_prices and region_count[region] >= 10:
|
||||
avg_price = region_avg_prices[region]
|
||||
else:
|
||||
# 处理均价为空情况,用上月均价替代
|
||||
district_stat = shop_sell_stat_conn.find_one({'city': city, 'cat': 'district', 'date': last_month, 'district': region})
|
||||
if pd.isna(district_stat):
|
||||
continue
|
||||
last_month_region_price = district_stat['avg_price']
|
||||
if pd.isna(last_month_region_price):
|
||||
continue
|
||||
avg_price = last_month_region_price
|
||||
|
||||
action = UpdateOne({'city': city, 'cat': 'district', 'date': year_month, 'district': region},
|
||||
{'$set': {'city': city, 'cat': 'district', 'date': year_month, 'district': region,
|
||||
'avg_price': int(avg_price), 'updated_date': datetime.now()}},
|
||||
upsert=True)
|
||||
actions.append(action)
|
||||
shop_sell_stat_conn.bulk_write(actions)
|
||||
log.info(f"{city} {year_month} 均价计算完毕!")
|
||||
|
||||
|
||||
# yearmonth = sys.argv[1] if len(sys.argv) > 1 else ''
|
||||
#
|
||||
# if yearmonth == '':
|
||||
# date = dateutils.add_months(datetime.today(), months=-1)
|
||||
# yearmonth = int(dateutils.format_date(date, '%Y%m'))
|
||||
# try:
|
||||
# for city in areautils.get_all_cities():
|
||||
# calc(city, yearmonth)
|
||||
# except Exception as e:
|
||||
# log.error(e)
|
||||
|
||||
# # 初始化从2024年9月
|
||||
# 创建从2024年9月到2024年12月的日期范围
|
||||
date_range = pd.date_range(start='2024-09-01', end='2025-02-28', freq='M')
|
||||
|
||||
# 创建一个包含这些日期的DataFrame
|
||||
df = pd.DataFrame({'date': date_range, 'value': range(1, len(date_range) + 1)})
|
||||
|
||||
# 将日期设置为索引
|
||||
df.set_index('date', inplace=True)
|
||||
|
||||
# 遍历DataFrame
|
||||
for city in areautils.get_all_cities():
|
||||
for date, row in df.iterrows():
|
||||
try:
|
||||
date_format = int(dateutils.format_date(date, '%Y%m'))
|
||||
calc(city, date_format)
|
||||
except Exception as e:
|
||||
log.error(e)
|
||||
Reference in New Issue
Block a user