144 lines
6.5 KiB
Python
144 lines
6.5 KiB
Python
"""
|
|
统计各城市租金、价格等其他字段覆盖率
|
|
"""
|
|
import pandas as pd
|
|
from goojia_utils import dateutils, ClickHouseClient
|
|
import datetime
|
|
import sys
|
|
import requests
|
|
import os
|
|
|
|
path = "/home/airflow/data/mt"
|
|
if not os.path.exists(path):
|
|
os.makedirs(path)
|
|
|
|
tmp_ck = ClickHouseClient(database='tmp')
|
|
|
|
output_date = sys.argv[1] if len(sys.argv) > 1 else dateutils.format_date(datetime.datetime.now(), '%Y%m')
|
|
print("输出日期" + output_date)
|
|
|
|
result_df = tmp_ck.read_sql_query("select * from tmp.mt_community")
|
|
del result_df['m_date']
|
|
|
|
city_df = pd.read_excel('/opt/airflow/dags/repo/dags/mt/supply/data/美团88城市列表.xlsx')
|
|
cities = city_df['城市列表'].values.tolist()
|
|
|
|
actions = list()
|
|
for city in cities:
|
|
action = dict()
|
|
temp_df = result_df[result_df['city'] == city]
|
|
# 小区量
|
|
action['curr_city'] = city
|
|
action['count'] = len(temp_df)
|
|
number_columns = ['avg_price', 'rent_price', 'lng', 'lat', 'deal_num', 'school', 'market', 'subway', 'bank', 'hospital']
|
|
|
|
for item in result_df.columns:
|
|
if item in number_columns:
|
|
action[item] = round(len(temp_df[temp_df[item] > 0]) * 100 / len(temp_df), 2)
|
|
else:
|
|
action[item] = round(len(temp_df[temp_df[item] != '']) * 100 / len(temp_df), 2)
|
|
if action['rent_price'] < 90:
|
|
action['rent_flag'] = '租金覆盖率低于90%'
|
|
if action['avg_price'] < 90:
|
|
action['avg_flag'] = '价格覆盖率低于90%'
|
|
actions.append(action)
|
|
|
|
df = pd.DataFrame(actions)
|
|
aim_file_name = path + "/" + output_date + '覆盖率统计.xlsx'
|
|
df.to_excel(aim_file_name, index=False)
|
|
|
|
|
|
def save2_clickhouse():
|
|
curr_month_start = output_date + '01'
|
|
result_df['avg_price'].fillna(-99.0, inplace=True)
|
|
result_df['rent_price'].fillna(-99.0, inplace=True)
|
|
result_df['lng'].fillna(-99.0, inplace=True)
|
|
result_df['lat'].fillna(-99.0, inplace=True)
|
|
result_df.loc[result_df['car_site_count'] == '', 'car_site_count'] = '-99'
|
|
result_df.loc[result_df['households'] == '', 'households'] = '-99'
|
|
result_df.loc[result_df['all_area'] == '', 'all_area'] = '-99'
|
|
result_df.loc[result_df['build_area'] == '', 'build_area'] = '-99'
|
|
result_df.loc[result_df['house_area'] == '', 'house_area'] = '-99'
|
|
result_df['house_area'].fillna(-99.0, inplace=True)
|
|
result_df['deal_num'].fillna(-99.0, inplace=True)
|
|
result_df['school'].fillna(-99.0, inplace=True)
|
|
result_df['market'].fillna(-99.0, inplace=True)
|
|
result_df['subway'].fillna(-99.0, inplace=True)
|
|
result_df['bank'].fillna(-99.0, inplace=True)
|
|
result_df['hospital'].fillna(-99.0, inplace=True)
|
|
result_df.loc[result_df['property_fee'] == '', 'property_fee'] = '-99'
|
|
result_df.fillna('', inplace=True)
|
|
result_df['plot_ratio'] = result_df['plot_ratio'].astype(str)
|
|
result_df['landscaping_ratio'] = result_df['landscaping_ratio'].astype(str)
|
|
result_df['property_life'] = result_df['property_life'].astype(str)
|
|
result_df['finish_building_date'] = result_df['finish_building_date'].astype(str)
|
|
result_df['house_type'] = result_df['house_type'].astype(str)
|
|
result_df['avg_price'] = result_df['avg_price'].astype(int)
|
|
result_df['rent_price'] = result_df['rent_price'].astype(int)
|
|
result_df['lng'] = result_df['lng'].astype(float)
|
|
result_df['lat'] = result_df['lat'].astype(float)
|
|
result_df['car_site_count'] = result_df['car_site_count'].astype(int)
|
|
result_df['households'] = result_df['households'].astype(int)
|
|
result_df['all_area'] = result_df['all_area'].astype(int)
|
|
result_df['build_area'] = result_df['build_area'].astype(int)
|
|
result_df['house_area'] = result_df['house_area'].astype(int)
|
|
result_df['deal_num'] = result_df['deal_num'].astype(int)
|
|
result_df['school'] = result_df['school'].astype(int)
|
|
result_df['market'] = result_df['market'].astype(int)
|
|
result_df['subway'] = result_df['subway'].astype(int)
|
|
result_df['bank'] = result_df['bank'].astype(int)
|
|
result_df['hospital'] = result_df['hospital'].astype(int)
|
|
result_df['property_fee'] = result_df['property_fee'].astype(float)
|
|
|
|
clickhouse_client = ClickHouseClient(database='fangjia_stat')
|
|
for city in cities:
|
|
temp_df = result_df[result_df['city'] == city].copy()
|
|
temp_df['year_month'] = dateutils.parse_date(curr_month_start, fm='%Y%m%d')
|
|
# 删除旧数据
|
|
clickhouse_client.execute(
|
|
"ALTER TABLE fangjia_stat.mt_community_stat DELETE WHERE city='" + city + "' and year_month = toDate('" + curr_month_start + "')")
|
|
clickhouse_client.to_sql(data=temp_df, table='mt_community_stat', engine='MergeTree', if_exists=False,
|
|
order_by={'year_month'})
|
|
|
|
|
|
def up_to_nextcloud():
|
|
"""
|
|
上传统计信息至nextcloud
|
|
"""
|
|
from nextcloud import NextCloud
|
|
NEXTCLOUD_URL = "https://nextcloud.fangjia.com"
|
|
NEXTCLOUD_USERNAME = "shenjian@fangjia.com"
|
|
NEXTCLOUD_PASSWORD = "goojia123456"
|
|
aim_file_path = '共享/美团/' + output_date + '覆盖率统计.xlsx'
|
|
nxc = NextCloud(endpoint=NEXTCLOUD_URL, user=NEXTCLOUD_USERNAME, password=NEXTCLOUD_PASSWORD, json_output=True)
|
|
nxc.upload_file(aim_file_name, aim_file_path)
|
|
columns = ['id', 'city', 'region', 'name', 'alias', 'address', 'addr_alias', 'avg_price', 'rent_price', 'lng',
|
|
'lat',
|
|
'block', 'loop_line_info', 'business_quarter', 'households', 'property_fee', 'car_site_count',
|
|
'property_life', 'house_property',
|
|
'all_area', 'build_area', 'plot_ratio', 'landscaping_ratio', 'property_type', 'build_type',
|
|
'build_structure', 'finish_building_date',
|
|
'developer', 'house_type', 'house_area', 'deal_num', 'district_facilitys', 'school', 'market', 'subway',
|
|
'bank', 'hospital', 'status']
|
|
final_result_df = result_df[columns]
|
|
final_result_df.to_csv(path + '/FJW_COMM_' + output_date + '.csv', sep='@', encoding='utf-8', index=False)
|
|
# 上传计算结果至cloud
|
|
nxc.upload_file(path + '/FJW_COMM_' + output_date + ".csv", '共享/美团/FJW_COMM_' + output_date + ".csv")
|
|
|
|
|
|
def send_msg():
|
|
infos = "【数据生成完毕】当月数据及覆盖率统计已生成完毕,请点击 https://nextcloud.fangjia.com/s/tMRPHrF8sgGF2F5 查看详情"
|
|
data = {
|
|
"msgtype": "text",
|
|
"text": {
|
|
"content": infos
|
|
}
|
|
}
|
|
# 发送消息至群
|
|
requests.post(
|
|
url='https://oapi.dingtalk.com/robot/send?access_token=e022515642e544942f86bbe234448edc3b8bddea8184e63b7edf4129b31c8e79',
|
|
json=data)
|
|
|
|
up_to_nextcloud()
|
|
save2_clickhouse()
|
|
send_msg() |