From 65ec6e0243c7e364652a00090f84f6a3b1a2e69c Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Tue, 23 Aug 2022 10:19:35 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E4=BB=B7=E6=A0=BC=E5=BC=82?= =?UTF-8?q?=E5=B8=B8=E6=A3=80=E6=B5=8B=E8=84=9A=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/jh/__init__.py | 3 ++ dags/jh/price/price_exception.py | 68 ++++++++++++++++++++++++++++++++ dags/jh/price_exception_dag.py | 17 ++++++++ dags/mt/__init__.py | 3 ++ 4 files changed, 91 insertions(+) create mode 100644 dags/jh/__init__.py create mode 100644 dags/jh/price/price_exception.py create mode 100644 dags/jh/price_exception_dag.py diff --git a/dags/jh/__init__.py b/dags/jh/__init__.py new file mode 100644 index 0000000..40e59f2 --- /dev/null +++ b/dags/jh/__init__.py @@ -0,0 +1,3 @@ +""" +建行脚本 +""" \ No newline at end of file diff --git a/dags/jh/price/price_exception.py b/dags/jh/price/price_exception.py new file mode 100644 index 0000000..705d5e0 --- /dev/null +++ b/dags/jh/price/price_exception.py @@ -0,0 +1,68 @@ +""" +价格异常检测脚本 +""" +from goojia_utils import dateutils, oracle +import pandas as pd +import logging + +log = logging.getLogger(__name__) + +month_now = dateutils.format_date(dateutils.add_months(dateutils.datetime.datetime.now()), '%Y%m') +month_last = dateutils.format_date(dateutils.add_months(dateutils.datetime.datetime.now(), months=-1), '%Y%m') +month_before = dateutils.format_date(dateutils.add_months(dateutils.datetime.datetime.now(), months=-2), '%Y%m') + +df = oracle.scan_by_sql_to_data_frame("SELECT id, cityname, distname, commid, commname, calc_type, confidence, avg_price, year_month FROM community_base_price_gj WHERE year_month IN('"+month_now+"', '"+month_last+"') " + + " ORDER BY cityname, distname, commid, commname, year_month") + +six_map = dict() +five_map = dict() +for index, item in df.iterrows(): + year_month = item['year_month'] + key = item['cityname'] + "," + item['distname'] + "," + item['commid'] + "," + item['commname'] + if year_month == month_last: + five_map[key] = str(item['avg_price']) + "|" + str(item['calc_type']) + '|' + str(item['confidence']) + elif year_month == month_now: + six_map[key] = str(item['avg_price']) + "|" + str(item['calc_type']) + '|' + str(item['confidence']) + +actions = list() +city_seaweed_dict = dict() +for key in six_map: + if key in five_map and key in six_map: + try: + five_price = float(five_map[key].split("|")[0]) + six_price = float(six_map[key].split("|")[0]) + if five_price and six_price: + info = key.split(",") + if abs(six_price - five_price) * 100 / five_price > 10: + actions.append({'city': info[0], 'region': info[1], 'commid': info[2], 'commname': info[3], + 'five_price': five_price,'five_calc_type': five_map[key].split("|")[1], + 'six_price': six_price,'six_calc_type': six_map[key].split("|")[1], + '涨跌幅': (six_price - five_price) * 100 / five_price, 'confidence': six_map[key].split("|")[2]}) + if info[0] not in city_seaweed_dict: + city_seaweed_dict[info[0]] = 1 + else: + city_seaweed_dict[info[0]] += 1 + except Exception as e: + log.error(e) + log.error(five_map[key] + ' ' + six_map[key]) + +result_df = pd.DataFrame(actions) +result_df['confidence'] = result_df['confidence'].astype(float) + +excel_writer = pd.ExcelWriter('data/'+ month_now +'VS'+ month_last +'月价格波动大于10%小区.xlsx') + +result_df = result_df.drop_duplicates(subset=['commid'], keep='first') +result_df = result_df.rename(columns={'city': '城市', 'region': '区域', 'commid': '小区ID', 'commname': '小区名', 'five_price': '上一月价格', + 'five_calc_type': '上一月计算方式', 'six_price': '当月价格', 'six_calc_type': '当月计算方式', 'confidence': '置信度'}) +result_df.to_excel(excel_writer, index=False, sheet_name='价格差异小区') + +log.info(len(result_df)) +city_count_df = result_df.groupby(by=["城市"])['小区ID'].count().reset_index() +for index, item in city_count_df.iterrows(): + city_count_df.loc[index, '占比'] = round(item['小区ID'] * 100 / city_seaweed_dict[item['城市']], 2) +city_count_df = city_count_df.rename(columns={'小区ID': '小区数量'}) +city_count_df.to_excel(excel_writer, index=False, sheet_name='城市维度') + +excel_writer.save() +excel_writer.close() + diff --git a/dags/jh/price_exception_dag.py b/dags/jh/price_exception_dag.py new file mode 100644 index 0000000..5ba2817 --- /dev/null +++ b/dags/jh/price_exception_dag.py @@ -0,0 +1,17 @@ +from airflow.operators.bash import BashOperator +import pendulum +from airflow import DAG + +with DAG( + dag_id="price_exception_dag", + # 每月23日运行 + schedule_interval="0 10 23 * *", + start_date=pendulum.datetime(2022, 8, 23, tz="Asia/Shanghai"), + catchup=True, + description="建行每月价格异常检测脚本,默认检测本月与上月差异" +) as dag: + t1 = BashOperator( + task_id='price_exception', + bash_command='python /opt/airflow/dags/repo/dags/jh/price/price_exception.py {{dag_run.conf["month"] if "month" in dag_run.conf}}', + ) + t1 diff --git a/dags/mt/__init__.py b/dags/mt/__init__.py index e69de29..32bbc48 100644 --- a/dags/mt/__init__.py +++ b/dags/mt/__init__.py @@ -0,0 +1,3 @@ +""" +美团数据供应脚本 +""" \ No newline at end of file