数据写入clickhouse

This commit is contained in:
Jian Shen
2022-06-29 16:23:18 +08:00
parent 61153a7a1b
commit 9d950d6f3e
+21 -3
View File
@@ -2,12 +2,15 @@
统计各城市租金、价格等其他字段覆盖率
"""
import pandas as pd
from goojia_utils import dateutils
from goojia_utils import dateutils, ClickHouseClient
import datetime
path = "/home/airflow/data/mt"
result_df = pd.read_csv(path + '/FJW_COMM_' + dateutils.format_date(datetime.datetime.now(), '%Y%m') + '.csv', sep='@', encoding='utf-8')
clickhouse_client = ClickHouseClient(database='fangjia_stat')
result_df = pd.read_csv(path + '/FJW_COMM_' + dateutils.format_date(datetime.datetime.now(), '%Y%m') + '.csv', sep='@',
encoding='utf-8')
city_df = pd.read_excel('/opt/airflow/dags/repo/dags/mt/supply/data/美团19城市列表.xlsx')
cities = city_df['城市列表'].values.tolist()
@@ -31,6 +34,19 @@ df = pd.DataFrame(actions)
aim_file_name = path + "/" + dateutils.format_date(datetime.datetime.now(), '%Y%m') + '覆盖率统计.xlsx'
df.to_excel(aim_file_name, index=False)
def save2_clickhouse():
curr_month_start = dateutils.format_date(datetime.datetime.now(), '%Y%m') + '01'
result_df.fillna('null', inplace=True)
for city in cities:
temp_df = result_df[result_df['city'] == city]
temp_df['year_month'] = curr_month_start
# 删除旧数据
clickhouse_client.execute(
"ALTER TABLE fangjia_stat.mt_community_stat DELETE WHERE city='" + city + "' and year_month = toDate('" + curr_month_start + "')")
clickhouse_client.to_sql(data=temp_df.head(1), table='mt_community_stat', engine='MergeTree', if_exists=False, order_by={'year_month'})
def up_to_nextcloud():
"""
上传统计信息至nextcloud
@@ -43,5 +59,7 @@ def up_to_nextcloud():
nxc = NextCloud(endpoint=NEXTCLOUD_URL, user=NEXTCLOUD_USERNAME, password=NEXTCLOUD_PASSWORD, json_output=True)
nxc.upload_file(aim_file_name, aim_file_path)
save2_clickhouse()
up_to_nextcloud()
# https://oapi.dingtalk.com/robot/send?access_token=e022515642e544942f86bbe234448edc3b8bddea8184e63b7edf4129b31c8e79
# https://oapi.dingtalk.com/robot/send?access_token=e022515642e544942f86bbe234448edc3b8bddea8184e63b7edf4129b31c8e79