Files
airflow/dags/mt/supply/facilitys.py
T
Jian Shen f203541ac2 优化
2022-06-29 11:44:01 +08:00

95 lines
3.7 KiB
Python

"""
周边配套数量统计
"""
from goojia_utils import MAIN_MONGO_CLIENT, dateutils
import pandas as pd
import datetime
import logging
district_facilitys_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'district_facilitys')
city_df = pd.read_excel('/opt/airflow/dags/repo/dags/mt/supply/data/美团19城市列表.xlsx')
cities = city_df['城市列表'].values.tolist()
# 1. 获取「今天」
today = datetime.date.today()
# 2. 获取当前月的第一天
first = today.replace(day=1)
# 3. 减一天,得到上个月的最后一天
last_month = first - datetime.timedelta(days=1)
month_start = last_month.strftime("%Y%m") + '01'
curr_month_start = dateutils.format_date(datetime.datetime.now(), '%Y%m') + '01'
logging.info(month_start + ' ' + curr_month_start)
month_start_date = dateutils.parse_date(month_start, '%Y%m%d')
curr_month_start_date = dateutils.parse_date(curr_month_start, '%Y%m%d')
path = "/home/airflow/data/mt"
district_facilitys_df = pd.DataFrame()
for city in cities:
temp_df = pd.DataFrame(
list(district_facilitys_conn.find({'city': city}, {'city': 1, 'region': 1, 'district_name': 1, 'school': 1,
'supermarket': 1, 'subway': 1, 'bank': 1, 'hospital': 1}).batch_size(10000)))
district_facilitys_df = pd.concat([district_facilitys_df, temp_df])
def count_facilitys(x):
ret = x
try:
if 'school' in x and x['school'] is not None and str(x['school']) != 'nan':
for item in x['school']:
if item is not None:
ret.school = len(item)
if 'supermarket' in x and x['supermarket'] is not None and str(x['supermarket']) != 'nan':
for item in x['supermarket']:
if item is not None:
ret.supermarket = len(item)
if 'subway' in x and x['subway'] is not None and str(x['subway']) != 'nan':
for item in x['subway']:
if item is not None:
ret.subway = len(item)
if 'bank' in x and x['bank'] is not None and str(x['bank']) != 'nan':
for item in x['bank']:
if item is not None:
ret.bank = len(item)
if 'hospital' in x and x['hospital'] is not None and str(x['hospital']) != 'nan':
for item in x['hospital']:
if item is not None:
ret.hospital = len(item)
return ret
except Exception as e:
print(e)
print(x)
return None
district_facilitys_new_df = district_facilitys_df.apply(lambda x: count_facilitys(x), axis=1)
district_facilitys_new_df = district_facilitys_new_df[['city', 'region', 'district_name', 'school', 'bank', 'hospital', 'supermarket', 'subway']]
district_facilitys_new_df = district_facilitys_new_df.rename(columns={'district_name': 'name', 'supermarket': 'market'})
result_df = pd.read_csv(path + '/FJW_COMM_' + dateutils.format_date(datetime.datetime.now(), '%Y%m') + '.csv', sep='@', encoding='utf-8')
result_df = pd.merge(result_df, district_facilitys_new_df, how='left', on=['city', 'region', 'name'])
result_df.loc[:, 'district_facilitys'] = '学校、商场、地铁、银行、医院'
def format_facilitys(x):
if type(x['school']) is not int:
x.school = None
if type(x['bank']) is not int:
x.bank = None
if type(x['hospital']) is not int:
x.hospital = None
if type(x['market']) is not int:
x.market = None
if type(x['subway']) is not int:
x.subway = None
return x
result_df = result_df.apply(lambda x: format_facilitys(x), axis=1)
result_df.to_csv(path + '/FJW_COMM_' + dateutils.format_date(datetime.datetime.now(), '%Y%m') + '.csv',
sep='@', encoding='utf-8', index=False)