新增价格异常检测脚本
This commit is contained in:
@@ -5,6 +5,52 @@ from goojia_utils import dateutils, oracle
|
||||
import pandas as pd
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
import datetime
|
||||
|
||||
def simple_calc(result_df):
|
||||
"""
|
||||
新增是否有样本列
|
||||
:param result_df:
|
||||
:return:
|
||||
"""
|
||||
month_before = dateutils.format_date(dateutils.add_months(dateutils.datetime.datetime.now(), months=-2), '%Y-%m-%d')
|
||||
simple_df = pd.DataFrame()
|
||||
actions = list()
|
||||
count = 0
|
||||
for index, item in result_df.iterrows():
|
||||
actions.append(item['commid'])
|
||||
if len(actions) == 800:
|
||||
ins = "','".join(actions)
|
||||
ins = "'" + ins + "'"
|
||||
simple_temp_df = oracle.scan_by_sql_to_data_frame("select distinct commid from bigdata_forsale_gj where " +
|
||||
" publishtime> TO_DATE('" + month_before + "','YYYY-MM-DD') and commid in (" + ins + ")")
|
||||
|
||||
simple_df = pd.concat([simple_df, simple_temp_df])
|
||||
actions.clear()
|
||||
count += 800
|
||||
if len(actions) > 0:
|
||||
ins = "','".join(actions)
|
||||
ins = "'" + ins + "'"
|
||||
simple_temp_df = oracle.scan_by_sql_to_data_frame("select distinct commid from bigdata_forsale_gj where " +
|
||||
" publishtime> TO_DATE('" + month_before + "','YYYY-MM-DD') and commid in (" + ins + ")")
|
||||
simple_df = pd.concat([simple_df, simple_temp_df])
|
||||
actions.clear()
|
||||
simple_df['has_sample'] = 1
|
||||
result_df = pd.merge(result_df, simple_df, on=['commid'], how='left')
|
||||
return result_df
|
||||
|
||||
def up_to_nextcloud():
|
||||
"""
|
||||
上传统计信息至nextcloud
|
||||
"""
|
||||
from nextcloud import NextCloud
|
||||
NEXTCLOUD_URL = "https://nextcloud.fangjia.com"
|
||||
NEXTCLOUD_USERNAME = "shenjian@fangjia.com"
|
||||
NEXTCLOUD_PASSWORD = "goojia123456"
|
||||
nextcloud_file_name = '共享/建行/' + file_name
|
||||
nxc = NextCloud(endpoint=NEXTCLOUD_URL, user=NEXTCLOUD_USERNAME, password=NEXTCLOUD_PASSWORD, json_output=True)
|
||||
nxc.upload_file(aim_file_name, nextcloud_file_name)
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -16,6 +62,9 @@ month_now = dateutils.format_date(dateutils.add_months(dateutils.datetime.dateti
|
||||
month_last = dateutils.format_date(dateutils.add_months(dateutils.datetime.datetime.now(), months=-1), '%Y%m')
|
||||
month_before = dateutils.format_date(dateutils.add_months(dateutils.datetime.datetime.now(), months=-2), '%Y%m')
|
||||
|
||||
type = sys.argv[1] if len(sys.argv) > 1 else ''
|
||||
log.info("输出类型" + type)
|
||||
|
||||
df = oracle.scan_by_sql_to_data_frame("SELECT id, cityname, distname, commid, commname, calc_type, confidence, avg_price, year_month FROM community_base_price_gj WHERE year_month IN('"+month_now+"', '"+month_last+"') " +
|
||||
" ORDER BY cityname, distname, commid, commname, year_month")
|
||||
|
||||
@@ -59,6 +108,9 @@ aim_file_name = path + '/' + file_name
|
||||
|
||||
excel_writer = pd.ExcelWriter(aim_file_name)
|
||||
|
||||
if type == 'simple':
|
||||
result_df = simple_calc(result_df)
|
||||
|
||||
result_df = result_df.drop_duplicates(subset=['commid'], keep='first')
|
||||
result_df = result_df.rename(columns={'city': '城市', 'region': '区域', 'commid': '小区ID', 'commname': '小区名', 'five_price': '上一月价格',
|
||||
'five_calc_type': '上一月计算方式', 'six_price': '当月价格', 'six_calc_type': '当月计算方式', 'confidence': '置信度'})
|
||||
@@ -74,18 +126,4 @@ city_count_df.to_excel(excel_writer, index=False, sheet_name='城市维度')
|
||||
excel_writer.save()
|
||||
excel_writer.close()
|
||||
|
||||
|
||||
def up_to_nextcloud():
|
||||
"""
|
||||
上传统计信息至nextcloud
|
||||
"""
|
||||
from nextcloud import NextCloud
|
||||
NEXTCLOUD_URL = "https://nextcloud.fangjia.com"
|
||||
NEXTCLOUD_USERNAME = "shenjian@fangjia.com"
|
||||
NEXTCLOUD_PASSWORD = "goojia123456"
|
||||
nextcloud_file_name = '共享/建行/' + file_name
|
||||
nxc = NextCloud(endpoint=NEXTCLOUD_URL, user=NEXTCLOUD_USERNAME, password=NEXTCLOUD_PASSWORD, json_output=True)
|
||||
nxc.upload_file(aim_file_name, nextcloud_file_name)
|
||||
|
||||
|
||||
up_to_nextcloud()
|
||||
|
||||
@@ -8,10 +8,10 @@ with DAG(
|
||||
schedule_interval="0 10 23 * *",
|
||||
start_date=pendulum.datetime(2022, 8, 23, tz="Asia/Shanghai"),
|
||||
catchup=True,
|
||||
description="建行每月价格异常检测脚本,默认检测本月与上月差异"
|
||||
description="建行每月价格异常检测脚本,默认检测本月与上月差异, 若输出样本存在与否,设置type=simple"
|
||||
) as dag:
|
||||
t1 = BashOperator(
|
||||
task_id='price_exception',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/jh/price/price_exception.py {{dag_run.conf["month"] if "month" in dag_run.conf}}',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/jh/price/price_exception.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
)
|
||||
t1
|
||||
|
||||
Reference in New Issue
Block a user