新增价格异常检测脚本

This commit is contained in:
Jian Shen
2022-08-23 13:26:47 +08:00
parent 3f2d91e7ef
commit 65aed9d6b9
2 changed files with 54 additions and 16 deletions
+52 -14
View File
@@ -5,6 +5,52 @@ from goojia_utils import dateutils, oracle
import pandas as pd
import logging
import os
import sys
import datetime
def simple_calc(result_df):
"""
新增是否有样本列
:param result_df:
:return:
"""
month_before = dateutils.format_date(dateutils.add_months(dateutils.datetime.datetime.now(), months=-2), '%Y-%m-%d')
simple_df = pd.DataFrame()
actions = list()
count = 0
for index, item in result_df.iterrows():
actions.append(item['commid'])
if len(actions) == 800:
ins = "','".join(actions)
ins = "'" + ins + "'"
simple_temp_df = oracle.scan_by_sql_to_data_frame("select distinct commid from bigdata_forsale_gj where " +
" publishtime> TO_DATE('" + month_before + "','YYYY-MM-DD') and commid in (" + ins + ")")
simple_df = pd.concat([simple_df, simple_temp_df])
actions.clear()
count += 800
if len(actions) > 0:
ins = "','".join(actions)
ins = "'" + ins + "'"
simple_temp_df = oracle.scan_by_sql_to_data_frame("select distinct commid from bigdata_forsale_gj where " +
" publishtime> TO_DATE('" + month_before + "','YYYY-MM-DD') and commid in (" + ins + ")")
simple_df = pd.concat([simple_df, simple_temp_df])
actions.clear()
simple_df['has_sample'] = 1
result_df = pd.merge(result_df, simple_df, on=['commid'], how='left')
return result_df
def up_to_nextcloud():
"""
上传统计信息至nextcloud
"""
from nextcloud import NextCloud
NEXTCLOUD_URL = "https://nextcloud.fangjia.com"
NEXTCLOUD_USERNAME = "shenjian@fangjia.com"
NEXTCLOUD_PASSWORD = "goojia123456"
nextcloud_file_name = '共享/建行/' + file_name
nxc = NextCloud(endpoint=NEXTCLOUD_URL, user=NEXTCLOUD_USERNAME, password=NEXTCLOUD_PASSWORD, json_output=True)
nxc.upload_file(aim_file_name, nextcloud_file_name)
log = logging.getLogger(__name__)
@@ -16,6 +62,9 @@ month_now = dateutils.format_date(dateutils.add_months(dateutils.datetime.dateti
month_last = dateutils.format_date(dateutils.add_months(dateutils.datetime.datetime.now(), months=-1), '%Y%m')
month_before = dateutils.format_date(dateutils.add_months(dateutils.datetime.datetime.now(), months=-2), '%Y%m')
type = sys.argv[1] if len(sys.argv) > 1 else ''
log.info("输出类型" + type)
df = oracle.scan_by_sql_to_data_frame("SELECT id, cityname, distname, commid, commname, calc_type, confidence, avg_price, year_month FROM community_base_price_gj WHERE year_month IN('"+month_now+"', '"+month_last+"') " +
" ORDER BY cityname, distname, commid, commname, year_month")
@@ -59,6 +108,9 @@ aim_file_name = path + '/' + file_name
excel_writer = pd.ExcelWriter(aim_file_name)
if type == 'simple':
result_df = simple_calc(result_df)
result_df = result_df.drop_duplicates(subset=['commid'], keep='first')
result_df = result_df.rename(columns={'city': '城市', 'region': '区域', 'commid': '小区ID', 'commname': '小区名', 'five_price': '上一月价格',
'five_calc_type': '上一月计算方式', 'six_price': '当月价格', 'six_calc_type': '当月计算方式', 'confidence': '置信度'})
@@ -74,18 +126,4 @@ city_count_df.to_excel(excel_writer, index=False, sheet_name='城市维度')
excel_writer.save()
excel_writer.close()
def up_to_nextcloud():
"""
上传统计信息至nextcloud
"""
from nextcloud import NextCloud
NEXTCLOUD_URL = "https://nextcloud.fangjia.com"
NEXTCLOUD_USERNAME = "shenjian@fangjia.com"
NEXTCLOUD_PASSWORD = "goojia123456"
nextcloud_file_name = '共享/建行/' + file_name
nxc = NextCloud(endpoint=NEXTCLOUD_URL, user=NEXTCLOUD_USERNAME, password=NEXTCLOUD_PASSWORD, json_output=True)
nxc.upload_file(aim_file_name, nextcloud_file_name)
up_to_nextcloud()
+2 -2
View File
@@ -8,10 +8,10 @@ with DAG(
schedule_interval="0 10 23 * *",
start_date=pendulum.datetime(2022, 8, 23, tz="Asia/Shanghai"),
catchup=True,
description="建行每月价格异常检测脚本,默认检测本月与上月差异"
description="建行每月价格异常检测脚本,默认检测本月与上月差异, 若输出样本存在与否,设置type=simple"
) as dag:
t1 = BashOperator(
task_id='price_exception',
bash_command='python /opt/airflow/dags/repo/dags/jh/price/price_exception.py {{dag_run.conf["month"] if "month" in dag_run.conf}}',
bash_command='python /opt/airflow/dags/repo/dags/jh/price/price_exception.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
)
t1