新增价格异常检测脚本

This commit is contained in:
Jian Shen
2022-08-23 10:19:35 +08:00
parent b44842e014
commit 65ec6e0243
4 changed files with 91 additions and 0 deletions
+3
View File
@@ -0,0 +1,3 @@
"""
建行脚本
"""
+68
View File
@@ -0,0 +1,68 @@
"""
价格异常检测脚本
"""
from goojia_utils import dateutils, oracle
import pandas as pd
import logging
log = logging.getLogger(__name__)
month_now = dateutils.format_date(dateutils.add_months(dateutils.datetime.datetime.now()), '%Y%m')
month_last = dateutils.format_date(dateutils.add_months(dateutils.datetime.datetime.now(), months=-1), '%Y%m')
month_before = dateutils.format_date(dateutils.add_months(dateutils.datetime.datetime.now(), months=-2), '%Y%m')
df = oracle.scan_by_sql_to_data_frame("SELECT id, cityname, distname, commid, commname, calc_type, confidence, avg_price, year_month FROM community_base_price_gj WHERE year_month IN('"+month_now+"', '"+month_last+"') " +
" ORDER BY cityname, distname, commid, commname, year_month")
six_map = dict()
five_map = dict()
for index, item in df.iterrows():
year_month = item['year_month']
key = item['cityname'] + "," + item['distname'] + "," + item['commid'] + "," + item['commname']
if year_month == month_last:
five_map[key] = str(item['avg_price']) + "|" + str(item['calc_type']) + '|' + str(item['confidence'])
elif year_month == month_now:
six_map[key] = str(item['avg_price']) + "|" + str(item['calc_type']) + '|' + str(item['confidence'])
actions = list()
city_seaweed_dict = dict()
for key in six_map:
if key in five_map and key in six_map:
try:
five_price = float(five_map[key].split("|")[0])
six_price = float(six_map[key].split("|")[0])
if five_price and six_price:
info = key.split(",")
if abs(six_price - five_price) * 100 / five_price > 10:
actions.append({'city': info[0], 'region': info[1], 'commid': info[2], 'commname': info[3],
'five_price': five_price,'five_calc_type': five_map[key].split("|")[1],
'six_price': six_price,'six_calc_type': six_map[key].split("|")[1],
'涨跌幅': (six_price - five_price) * 100 / five_price, 'confidence': six_map[key].split("|")[2]})
if info[0] not in city_seaweed_dict:
city_seaweed_dict[info[0]] = 1
else:
city_seaweed_dict[info[0]] += 1
except Exception as e:
log.error(e)
log.error(five_map[key] + ' ' + six_map[key])
result_df = pd.DataFrame(actions)
result_df['confidence'] = result_df['confidence'].astype(float)
excel_writer = pd.ExcelWriter('data/'+ month_now +'VS'+ month_last +'月价格波动大于10%小区.xlsx')
result_df = result_df.drop_duplicates(subset=['commid'], keep='first')
result_df = result_df.rename(columns={'city': '城市', 'region': '区域', 'commid': '小区ID', 'commname': '小区名', 'five_price': '上一月价格',
'five_calc_type': '上一月计算方式', 'six_price': '当月价格', 'six_calc_type': '当月计算方式', 'confidence': '置信度'})
result_df.to_excel(excel_writer, index=False, sheet_name='价格差异小区')
log.info(len(result_df))
city_count_df = result_df.groupby(by=["城市"])['小区ID'].count().reset_index()
for index, item in city_count_df.iterrows():
city_count_df.loc[index, '占比'] = round(item['小区ID'] * 100 / city_seaweed_dict[item['城市']], 2)
city_count_df = city_count_df.rename(columns={'小区ID': '小区数量'})
city_count_df.to_excel(excel_writer, index=False, sheet_name='城市维度')
excel_writer.save()
excel_writer.close()
+17
View File
@@ -0,0 +1,17 @@
from airflow.operators.bash import BashOperator
import pendulum
from airflow import DAG
with DAG(
dag_id="price_exception_dag",
# 每月23日运行
schedule_interval="0 10 23 * *",
start_date=pendulum.datetime(2022, 8, 23, tz="Asia/Shanghai"),
catchup=True,
description="建行每月价格异常检测脚本,默认检测本月与上月差异"
) as dag:
t1 = BashOperator(
task_id='price_exception',
bash_command='python /opt/airflow/dags/repo/dags/jh/price/price_exception.py {{dag_run.conf["month"] if "month" in dag_run.conf}}',
)
t1
+3
View File
@@ -0,0 +1,3 @@
"""
美团数据供应脚本
"""