数据写入clickhouse
This commit is contained in:
@@ -2,12 +2,15 @@
|
||||
统计各城市租金、价格等其他字段覆盖率
|
||||
"""
|
||||
import pandas as pd
|
||||
from goojia_utils import dateutils
|
||||
from goojia_utils import dateutils, ClickHouseClient
|
||||
import datetime
|
||||
|
||||
path = "/home/airflow/data/mt"
|
||||
|
||||
result_df = pd.read_csv(path + '/FJW_COMM_' + dateutils.format_date(datetime.datetime.now(), '%Y%m') + '.csv', sep='@', encoding='utf-8')
|
||||
clickhouse_client = ClickHouseClient(database='fangjia_stat')
|
||||
|
||||
result_df = pd.read_csv(path + '/FJW_COMM_' + dateutils.format_date(datetime.datetime.now(), '%Y%m') + '.csv', sep='@',
|
||||
encoding='utf-8')
|
||||
|
||||
city_df = pd.read_excel('/opt/airflow/dags/repo/dags/mt/supply/data/美团19城市列表.xlsx')
|
||||
cities = city_df['城市列表'].values.tolist()
|
||||
@@ -31,6 +34,19 @@ df = pd.DataFrame(actions)
|
||||
aim_file_name = path + "/" + dateutils.format_date(datetime.datetime.now(), '%Y%m') + '覆盖率统计.xlsx'
|
||||
df.to_excel(aim_file_name, index=False)
|
||||
|
||||
|
||||
def save2_clickhouse():
|
||||
curr_month_start = dateutils.format_date(datetime.datetime.now(), '%Y%m') + '01'
|
||||
result_df.fillna('null', inplace=True)
|
||||
for city in cities:
|
||||
temp_df = result_df[result_df['city'] == city]
|
||||
temp_df['year_month'] = curr_month_start
|
||||
# 删除旧数据
|
||||
clickhouse_client.execute(
|
||||
"ALTER TABLE fangjia_stat.mt_community_stat DELETE WHERE city='" + city + "' and year_month = toDate('" + curr_month_start + "')")
|
||||
clickhouse_client.to_sql(data=temp_df.head(1), table='mt_community_stat', engine='MergeTree', if_exists=False, order_by={'year_month'})
|
||||
|
||||
|
||||
def up_to_nextcloud():
|
||||
"""
|
||||
上传统计信息至nextcloud
|
||||
@@ -43,5 +59,7 @@ def up_to_nextcloud():
|
||||
nxc = NextCloud(endpoint=NEXTCLOUD_URL, user=NEXTCLOUD_USERNAME, password=NEXTCLOUD_PASSWORD, json_output=True)
|
||||
nxc.upload_file(aim_file_name, aim_file_path)
|
||||
|
||||
|
||||
save2_clickhouse()
|
||||
up_to_nextcloud()
|
||||
# https://oapi.dingtalk.com/robot/send?access_token=e022515642e544942f86bbe234448edc3b8bddea8184e63b7edf4129b31c8e79
|
||||
# https://oapi.dingtalk.com/robot/send?access_token=e022515642e544942f86bbe234448edc3b8bddea8184e63b7edf4129b31c8e79
|
||||
|
||||
Reference in New Issue
Block a user