From 65aed9d6b939f0fb96641eeb6671b07e48f7005f Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Tue, 23 Aug 2022 13:26:47 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E4=BB=B7=E6=A0=BC=E5=BC=82?= =?UTF-8?q?=E5=B8=B8=E6=A3=80=E6=B5=8B=E8=84=9A=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/jh/price/price_exception.py | 66 +++++++++++++++++++++++++------- dags/jh/price_exception_dag.py | 4 +- 2 files changed, 54 insertions(+), 16 deletions(-) diff --git a/dags/jh/price/price_exception.py b/dags/jh/price/price_exception.py index e3422c8..0eab33a 100644 --- a/dags/jh/price/price_exception.py +++ b/dags/jh/price/price_exception.py @@ -5,6 +5,52 @@ from goojia_utils import dateutils, oracle import pandas as pd import logging import os +import sys +import datetime + +def simple_calc(result_df): + """ + 新增是否有样本列 + :param result_df: + :return: + """ + month_before = dateutils.format_date(dateutils.add_months(dateutils.datetime.datetime.now(), months=-2), '%Y-%m-%d') + simple_df = pd.DataFrame() + actions = list() + count = 0 + for index, item in result_df.iterrows(): + actions.append(item['commid']) + if len(actions) == 800: + ins = "','".join(actions) + ins = "'" + ins + "'" + simple_temp_df = oracle.scan_by_sql_to_data_frame("select distinct commid from bigdata_forsale_gj where " + + " publishtime> TO_DATE('" + month_before + "','YYYY-MM-DD') and commid in (" + ins + ")") + + simple_df = pd.concat([simple_df, simple_temp_df]) + actions.clear() + count += 800 + if len(actions) > 0: + ins = "','".join(actions) + ins = "'" + ins + "'" + simple_temp_df = oracle.scan_by_sql_to_data_frame("select distinct commid from bigdata_forsale_gj where " + + " publishtime> TO_DATE('" + month_before + "','YYYY-MM-DD') and commid in (" + ins + ")") + simple_df = pd.concat([simple_df, simple_temp_df]) + actions.clear() + simple_df['has_sample'] = 1 + result_df = pd.merge(result_df, simple_df, on=['commid'], how='left') + return result_df + +def up_to_nextcloud(): + """ + 上传统计信息至nextcloud + """ + from nextcloud import NextCloud + NEXTCLOUD_URL = "https://nextcloud.fangjia.com" + NEXTCLOUD_USERNAME = "shenjian@fangjia.com" + NEXTCLOUD_PASSWORD = "goojia123456" + nextcloud_file_name = '共享/建行/' + file_name + nxc = NextCloud(endpoint=NEXTCLOUD_URL, user=NEXTCLOUD_USERNAME, password=NEXTCLOUD_PASSWORD, json_output=True) + nxc.upload_file(aim_file_name, nextcloud_file_name) log = logging.getLogger(__name__) @@ -16,6 +62,9 @@ month_now = dateutils.format_date(dateutils.add_months(dateutils.datetime.dateti month_last = dateutils.format_date(dateutils.add_months(dateutils.datetime.datetime.now(), months=-1), '%Y%m') month_before = dateutils.format_date(dateutils.add_months(dateutils.datetime.datetime.now(), months=-2), '%Y%m') +type = sys.argv[1] if len(sys.argv) > 1 else '' +log.info("输出类型" + type) + df = oracle.scan_by_sql_to_data_frame("SELECT id, cityname, distname, commid, commname, calc_type, confidence, avg_price, year_month FROM community_base_price_gj WHERE year_month IN('"+month_now+"', '"+month_last+"') " + " ORDER BY cityname, distname, commid, commname, year_month") @@ -59,6 +108,9 @@ aim_file_name = path + '/' + file_name excel_writer = pd.ExcelWriter(aim_file_name) +if type == 'simple': + result_df = simple_calc(result_df) + result_df = result_df.drop_duplicates(subset=['commid'], keep='first') result_df = result_df.rename(columns={'city': '城市', 'region': '区域', 'commid': '小区ID', 'commname': '小区名', 'five_price': '上一月价格', 'five_calc_type': '上一月计算方式', 'six_price': '当月价格', 'six_calc_type': '当月计算方式', 'confidence': '置信度'}) @@ -74,18 +126,4 @@ city_count_df.to_excel(excel_writer, index=False, sheet_name='城市维度') excel_writer.save() excel_writer.close() - -def up_to_nextcloud(): - """ - 上传统计信息至nextcloud - """ - from nextcloud import NextCloud - NEXTCLOUD_URL = "https://nextcloud.fangjia.com" - NEXTCLOUD_USERNAME = "shenjian@fangjia.com" - NEXTCLOUD_PASSWORD = "goojia123456" - nextcloud_file_name = '共享/建行/' + file_name - nxc = NextCloud(endpoint=NEXTCLOUD_URL, user=NEXTCLOUD_USERNAME, password=NEXTCLOUD_PASSWORD, json_output=True) - nxc.upload_file(aim_file_name, nextcloud_file_name) - - up_to_nextcloud() diff --git a/dags/jh/price_exception_dag.py b/dags/jh/price_exception_dag.py index 5ba2817..e07159e 100644 --- a/dags/jh/price_exception_dag.py +++ b/dags/jh/price_exception_dag.py @@ -8,10 +8,10 @@ with DAG( schedule_interval="0 10 23 * *", start_date=pendulum.datetime(2022, 8, 23, tz="Asia/Shanghai"), catchup=True, - description="建行每月价格异常检测脚本,默认检测本月与上月差异" + description="建行每月价格异常检测脚本,默认检测本月与上月差异, 若输出样本存在与否,设置type=simple" ) as dag: t1 = BashOperator( task_id='price_exception', - bash_command='python /opt/airflow/dags/repo/dags/jh/price/price_exception.py {{dag_run.conf["month"] if "month" in dag_run.conf}}', + bash_command='python /opt/airflow/dags/repo/dags/jh/price/price_exception.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', ) t1