优化统计
This commit is contained in:
@@ -6,8 +6,11 @@ from goojia_utils import dateutils, ClickHouseClient
|
||||
import datetime
|
||||
import sys
|
||||
import requests
|
||||
import os
|
||||
|
||||
path = "/home/airflow/data/mt"
|
||||
if not os.path.exists(path):
|
||||
os.makedirs(path)
|
||||
|
||||
clickhouse_client = ClickHouseClient(database='fangjia_stat')
|
||||
tmp_ck = ClickHouseClient(database='tmp')
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
|
||||
from goojia_utils import MAIN_MONGO_CLIENT, dateutils, ClickHouseClient
|
||||
import pandas as pd
|
||||
import os
|
||||
|
||||
server_area_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'server_area')
|
||||
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
||||
@@ -21,6 +22,11 @@ seaweed_df = pd.DataFrame(
|
||||
.batch_size(10000)))
|
||||
|
||||
path = "/opt/airflow/dags/repo/dags/business/business/community/data"
|
||||
|
||||
aim_path = "/home/airflow/data/stat"
|
||||
if not os.path.exists(aim_path):
|
||||
os.makedirs(aim_path)
|
||||
|
||||
ret_df = pd.read_excel(path + '/财力评级分布情况20221215.xlsx')
|
||||
|
||||
for index, item in city_df.iterrows():
|
||||
@@ -150,4 +156,28 @@ clickHouseClient.execute(
|
||||
"ALTER TABLE fangjia_stat.community_coverage_stat DELETE WHERE weekend = toDate('" + dateutils.format_date(dateutils.datetime.datetime.now(), '%Y-%m-%d') + "')")
|
||||
clickHouseClient.to_sql(df, table='community_coverage_stat', if_exists='insert', order_by='weekend')
|
||||
|
||||
# 上传至nextcloud
|
||||
ret_df = pd.merge(ret_df, df, how='left', on=['city'])
|
||||
del ret_df['是否合理']
|
||||
|
||||
output_date = dateutils.format_date(dateutils.datetime.datetime.now())
|
||||
aim_file_name = aim_path + "/" + output_date + '覆盖率统计.xlsx'
|
||||
df.to_excel(aim_file_name, index=False)
|
||||
|
||||
# https://nextcloud.fangjia.com/s/3aw663fy9BbiBeL
|
||||
|
||||
def up_to_nextcloud():
|
||||
"""
|
||||
上传统计信息至nextcloud
|
||||
"""
|
||||
from nextcloud import NextCloud
|
||||
NEXTCLOUD_URL = "https://nextcloud.fangjia.com"
|
||||
NEXTCLOUD_USERNAME = "shenjian@fangjia.com"
|
||||
NEXTCLOUD_PASSWORD = "goojia123456"
|
||||
aim_file_path = '共享/统计/常用覆盖率统计' + output_date + '.xlsx'
|
||||
nxc = NextCloud(endpoint=NEXTCLOUD_URL, user=NEXTCLOUD_USERNAME, password=NEXTCLOUD_PASSWORD, json_output=True)
|
||||
nxc.upload_file(aim_file_name, aim_file_path)
|
||||
# 上传计算结果至cloud
|
||||
nxc.upload_file(path + '/FJW_COMM_' + output_date + ".csv", '共享/美团/FJW_COMM_' + output_date + ".csv")
|
||||
|
||||
up_to_nextcloud()
|
||||
|
||||
Reference in New Issue
Block a user