优化统计

This commit is contained in:
shenjian@fangjia.com
2023-03-22 18:15:17 +08:00
parent 0fe9d8f99a
commit 8c439e3201
2 changed files with 33 additions and 0 deletions
+3
View File
@@ -6,8 +6,11 @@ from goojia_utils import dateutils, ClickHouseClient
import datetime
import sys
import requests
import os
path = "/home/airflow/data/mt"
if not os.path.exists(path):
os.makedirs(path)
clickhouse_client = ClickHouseClient(database='fangjia_stat')
tmp_ck = ClickHouseClient(database='tmp')
@@ -4,6 +4,7 @@
from goojia_utils import MAIN_MONGO_CLIENT, dateutils, ClickHouseClient
import pandas as pd
import os
server_area_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'server_area')
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
@@ -21,6 +22,11 @@ seaweed_df = pd.DataFrame(
.batch_size(10000)))
path = "/opt/airflow/dags/repo/dags/business/business/community/data"
aim_path = "/home/airflow/data/stat"
if not os.path.exists(aim_path):
os.makedirs(aim_path)
ret_df = pd.read_excel(path + '/财力评级分布情况20221215.xlsx')
for index, item in city_df.iterrows():
@@ -150,4 +156,28 @@ clickHouseClient.execute(
"ALTER TABLE fangjia_stat.community_coverage_stat DELETE WHERE weekend = toDate('" + dateutils.format_date(dateutils.datetime.datetime.now(), '%Y-%m-%d') + "')")
clickHouseClient.to_sql(df, table='community_coverage_stat', if_exists='insert', order_by='weekend')
# 上传至nextcloud
ret_df = pd.merge(ret_df, df, how='left', on=['city'])
del ret_df['是否合理']
output_date = dateutils.format_date(dateutils.datetime.datetime.now())
aim_file_name = aim_path + "/" + output_date + '覆盖率统计.xlsx'
df.to_excel(aim_file_name, index=False)
# https://nextcloud.fangjia.com/s/3aw663fy9BbiBeL
def up_to_nextcloud():
"""
上传统计信息至nextcloud
"""
from nextcloud import NextCloud
NEXTCLOUD_URL = "https://nextcloud.fangjia.com"
NEXTCLOUD_USERNAME = "shenjian@fangjia.com"
NEXTCLOUD_PASSWORD = "goojia123456"
aim_file_path = '共享/统计/常用覆盖率统计' + output_date + '.xlsx'
nxc = NextCloud(endpoint=NEXTCLOUD_URL, user=NEXTCLOUD_USERNAME, password=NEXTCLOUD_PASSWORD, json_output=True)
nxc.upload_file(aim_file_name, aim_file_path)
# 上传计算结果至cloud
nxc.upload_file(path + '/FJW_COMM_' + output_date + ".csv", '共享/美团/FJW_COMM_' + output_date + ".csv")
up_to_nextcloud()