From 8c439e3201040ebd49672b37f6a13f9b984fe238 Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Wed, 22 Mar 2023 18:15:17 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96=E7=BB=9F=E8=AE=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/mt/supply/data_stat.py | 3 ++ .../community/community_count_price_rent.py | 30 +++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/dags/mt/supply/data_stat.py b/dags/mt/supply/data_stat.py index c1d5227..f3115aa 100644 --- a/dags/mt/supply/data_stat.py +++ b/dags/mt/supply/data_stat.py @@ -6,8 +6,11 @@ from goojia_utils import dateutils, ClickHouseClient import datetime import sys import requests +import os path = "/home/airflow/data/mt" +if not os.path.exists(path): + os.makedirs(path) clickhouse_client = ClickHouseClient(database='fangjia_stat') tmp_ck = ClickHouseClient(database='tmp') diff --git a/dags/stat/community/community_count_price_rent.py b/dags/stat/community/community_count_price_rent.py index 127e72d..4b6bcb3 100644 --- a/dags/stat/community/community_count_price_rent.py +++ b/dags/stat/community/community_count_price_rent.py @@ -4,6 +4,7 @@ from goojia_utils import MAIN_MONGO_CLIENT, dateutils, ClickHouseClient import pandas as pd +import os server_area_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'server_area') seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') @@ -21,6 +22,11 @@ seaweed_df = pd.DataFrame( .batch_size(10000))) path = "/opt/airflow/dags/repo/dags/business/business/community/data" + +aim_path = "/home/airflow/data/stat" +if not os.path.exists(aim_path): + os.makedirs(aim_path) + ret_df = pd.read_excel(path + '/财力评级分布情况20221215.xlsx') for index, item in city_df.iterrows(): @@ -150,4 +156,28 @@ clickHouseClient.execute( "ALTER TABLE fangjia_stat.community_coverage_stat DELETE WHERE weekend = toDate('" + dateutils.format_date(dateutils.datetime.datetime.now(), '%Y-%m-%d') + "')") clickHouseClient.to_sql(df, table='community_coverage_stat', if_exists='insert', order_by='weekend') +# 上传至nextcloud +ret_df = pd.merge(ret_df, df, how='left', on=['city']) +del ret_df['是否合理'] +output_date = dateutils.format_date(dateutils.datetime.datetime.now()) +aim_file_name = aim_path + "/" + output_date + '覆盖率统计.xlsx' +df.to_excel(aim_file_name, index=False) + +# https://nextcloud.fangjia.com/s/3aw663fy9BbiBeL + +def up_to_nextcloud(): + """ + 上传统计信息至nextcloud + """ + from nextcloud import NextCloud + NEXTCLOUD_URL = "https://nextcloud.fangjia.com" + NEXTCLOUD_USERNAME = "shenjian@fangjia.com" + NEXTCLOUD_PASSWORD = "goojia123456" + aim_file_path = '共享/统计/常用覆盖率统计' + output_date + '.xlsx' + nxc = NextCloud(endpoint=NEXTCLOUD_URL, user=NEXTCLOUD_USERNAME, password=NEXTCLOUD_PASSWORD, json_output=True) + nxc.upload_file(aim_file_name, aim_file_path) + # 上传计算结果至cloud + nxc.upload_file(path + '/FJW_COMM_' + output_date + ".csv", '共享/美团/FJW_COMM_' + output_date + ".csv") + +up_to_nextcloud()