""" 统计各城市租金、价格等其他字段覆盖率 """ import pandas as pd from goojia_utils import dateutils, ClickHouseClient import datetime import sys import requests import os path = "/home/airflow/data/mt" if not os.path.exists(path): os.makedirs(path) clickhouse_client = ClickHouseClient(database='fangjia_stat') tmp_ck = ClickHouseClient(database='tmp') output_date = sys.argv[1] if len(sys.argv) > 1 else dateutils.format_date(datetime.datetime.now(), '%Y%m') print("输出日期" + output_date) result_df = tmp_ck.read_sql_query("select * from tmp.mt_community") city_df = pd.read_excel('/opt/airflow/dags/repo/dags/mt/supply/data/美团88城市列表.xlsx') cities = city_df['城市列表'].values.tolist() actions = list() for city in cities: action = dict() temp_df = result_df[result_df['city'] == city] # 小区量 action['curr_city'] = city action['count'] = len(temp_df) for item in result_df.columns: if item == 'deal_num' or item == 'school' or item == 'market' or item == 'subway' \ or item == 'bank' or item == 'hospital': action[item] = round(len(temp_df[temp_df[item] > 0]) * 100 / len(temp_df), 2) else: action[item] = round(len(temp_df[temp_df[item].notna()]) * 100 / len(temp_df), 2) if action['rent_price'] < 90: action['rent_flag'] = '租金覆盖率低于90%' if action['avg_price'] < 90: action['avg_flag'] = '价格覆盖率低于90%' actions.append(action) df = pd.DataFrame(actions) aim_file_name = path + "/" + output_date + '覆盖率统计.xlsx' df.to_excel(aim_file_name, index=False) def save2_clickhouse(): curr_month_start = output_date + '01' result_df['avg_price'].fillna(-99.0, inplace=True) result_df['rent_price'].fillna(-99.0, inplace=True) result_df['lng'].fillna(-99.0, inplace=True) result_df['lat'].fillna(-99.0, inplace=True) result_df['car_site_count'].fillna(-99.0, inplace=True) result_df['households'].fillna(-99.0, inplace=True) result_df['all_area'].fillna(-99.0, inplace=True) result_df['build_area'].fillna(-99.0, inplace=True) result_df['house_area'].fillna(-99.0, inplace=True) result_df['deal_num'].fillna(-99.0, inplace=True) result_df['school'].fillna(-99.0, inplace=True) result_df['market'].fillna(-99.0, inplace=True) result_df['subway'].fillna(-99.0, inplace=True) result_df['bank'].fillna(-99.0, inplace=True) result_df['hospital'].fillna(-99.0, inplace=True) result_df['property_fee'].fillna(-99.0, inplace=True) result_df.fillna('', inplace=True) result_df['plot_ratio'] = result_df['plot_ratio'].astype(str) result_df['landscaping_ratio'] = result_df['landscaping_ratio'].astype(str) result_df['property_life'] = result_df['property_life'].astype(str) result_df['finish_building_date'] = result_df['finish_building_date'].astype(str) result_df['house_type'] = result_df['house_type'].astype(str) result_df['avg_price'] = result_df['avg_price'].astype(int) result_df['rent_price'] = result_df['rent_price'].astype(int) result_df['lng'] = result_df['lng'].astype(float) result_df['lat'] = result_df['lat'].astype(float) result_df['car_site_count'] = result_df['car_site_count'].astype(int) result_df['households'] = result_df['households'].astype(int) result_df['all_area'] = result_df['all_area'].astype(int) result_df['build_area'] = result_df['build_area'].astype(int) result_df['house_area'] = result_df['house_area'].astype(int) result_df['deal_num'] = result_df['deal_num'].astype(int) result_df['school'] = result_df['school'].astype(int) result_df['market'] = result_df['market'].astype(int) result_df['subway'] = result_df['subway'].astype(int) result_df['bank'] = result_df['bank'].astype(int) result_df['hospital'] = result_df['hospital'].astype(int) result_df['property_fee'] = result_df['property_fee'].astype(float) for city in cities: temp_df = result_df[result_df['city'] == city] temp_df['year_month'] = dateutils.parse_date(curr_month_start, fm='%Y%m%d') # 删除旧数据 clickhouse_client.execute( "ALTER TABLE fangjia_stat.mt_community_stat DELETE WHERE city='" + city + "' and year_month = toDate('" + curr_month_start + "')") clickhouse_client.to_sql(data=temp_df, table='mt_community_stat', engine='MergeTree', if_exists=False, order_by={'year_month'}) def up_to_nextcloud(): """ 上传统计信息至nextcloud """ from nextcloud import NextCloud NEXTCLOUD_URL = "https://nextcloud.fangjia.com" NEXTCLOUD_USERNAME = "shenjian@fangjia.com" NEXTCLOUD_PASSWORD = "goojia123456" aim_file_path = '共享/美团/' + output_date + '覆盖率统计.xlsx' nxc = NextCloud(endpoint=NEXTCLOUD_URL, user=NEXTCLOUD_USERNAME, password=NEXTCLOUD_PASSWORD, json_output=True) nxc.upload_file(aim_file_name, aim_file_path) # 上传计算结果至cloud nxc.upload_file(path + '/FJW_COMM_' + output_date + ".csv", '共享/美团/FJW_COMM_' + output_date + ".csv") def send_msg(): infos = "【数据生成完毕】当月数据及覆盖率统计已生成完毕,请点击 https://nextcloud.fangjia.com/s/tMRPHrF8sgGF2F5 查看详情" data = { "msgtype": "text", "text": { "content": infos } } # 发送消息至群 requests.post( url='https://oapi.dingtalk.com/robot/send?access_token=e022515642e544942f86bbe234448edc3b8bddea8184e63b7edf4129b31c8e79', json=data) up_to_nextcloud() save2_clickhouse() send_msg()