From 9d950d6f3e5b09e56739b93a2995b55bf74b8f50 Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Wed, 29 Jun 2022 16:23:18 +0800 Subject: [PATCH] =?UTF-8?q?=E6=95=B0=E6=8D=AE=E5=86=99=E5=85=A5clickhouse?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/mt/supply/data_stat.py | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/dags/mt/supply/data_stat.py b/dags/mt/supply/data_stat.py index 3fc3de7..d4af8e3 100644 --- a/dags/mt/supply/data_stat.py +++ b/dags/mt/supply/data_stat.py @@ -2,12 +2,15 @@ 统计各城市租金、价格等其他字段覆盖率 """ import pandas as pd -from goojia_utils import dateutils +from goojia_utils import dateutils, ClickHouseClient import datetime path = "/home/airflow/data/mt" -result_df = pd.read_csv(path + '/FJW_COMM_' + dateutils.format_date(datetime.datetime.now(), '%Y%m') + '.csv', sep='@', encoding='utf-8') +clickhouse_client = ClickHouseClient(database='fangjia_stat') + +result_df = pd.read_csv(path + '/FJW_COMM_' + dateutils.format_date(datetime.datetime.now(), '%Y%m') + '.csv', sep='@', + encoding='utf-8') city_df = pd.read_excel('/opt/airflow/dags/repo/dags/mt/supply/data/美团19城市列表.xlsx') cities = city_df['城市列表'].values.tolist() @@ -31,6 +34,19 @@ df = pd.DataFrame(actions) aim_file_name = path + "/" + dateutils.format_date(datetime.datetime.now(), '%Y%m') + '覆盖率统计.xlsx' df.to_excel(aim_file_name, index=False) + +def save2_clickhouse(): + curr_month_start = dateutils.format_date(datetime.datetime.now(), '%Y%m') + '01' + result_df.fillna('null', inplace=True) + for city in cities: + temp_df = result_df[result_df['city'] == city] + temp_df['year_month'] = curr_month_start + # 删除旧数据 + clickhouse_client.execute( + "ALTER TABLE fangjia_stat.mt_community_stat DELETE WHERE city='" + city + "' and year_month = toDate('" + curr_month_start + "')") + clickhouse_client.to_sql(data=temp_df.head(1), table='mt_community_stat', engine='MergeTree', if_exists=False, order_by={'year_month'}) + + def up_to_nextcloud(): """ 上传统计信息至nextcloud @@ -43,5 +59,7 @@ def up_to_nextcloud(): nxc = NextCloud(endpoint=NEXTCLOUD_URL, user=NEXTCLOUD_USERNAME, password=NEXTCLOUD_PASSWORD, json_output=True) nxc.upload_file(aim_file_name, aim_file_path) + +save2_clickhouse() up_to_nextcloud() -# https://oapi.dingtalk.com/robot/send?access_token=e022515642e544942f86bbe234448edc3b8bddea8184e63b7edf4129b31c8e79 \ No newline at end of file +# https://oapi.dingtalk.com/robot/send?access_token=e022515642e544942f86bbe234448edc3b8bddea8184e63b7edf4129b31c8e79