95 lines
3.7 KiB
Python
95 lines
3.7 KiB
Python
"""
|
|
周边配套数量统计
|
|
"""
|
|
from goojia_utils import MAIN_MONGO_CLIENT, dateutils
|
|
import pandas as pd
|
|
import datetime
|
|
import logging
|
|
|
|
district_facilitys_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'district_facilitys')
|
|
|
|
city_df = pd.read_excel('/opt/airflow/dags/repo/dags/mt/supply/data/美团19城市列表.xlsx')
|
|
cities = city_df['城市列表'].values.tolist()
|
|
|
|
# 1. 获取「今天」
|
|
today = datetime.date.today()
|
|
# 2. 获取当前月的第一天
|
|
first = today.replace(day=1)
|
|
# 3. 减一天,得到上个月的最后一天
|
|
last_month = first - datetime.timedelta(days=1)
|
|
|
|
month_start = last_month.strftime("%Y%m") + '01'
|
|
curr_month_start = dateutils.format_date(datetime.datetime.now(), '%Y%m') + '01'
|
|
logging.info(month_start + ' ' + curr_month_start)
|
|
|
|
month_start_date = dateutils.parse_date(month_start, '%Y%m%d')
|
|
curr_month_start_date = dateutils.parse_date(curr_month_start, '%Y%m%d')
|
|
|
|
path = "/home/airflow/data/mt"
|
|
|
|
district_facilitys_df = pd.DataFrame()
|
|
for city in cities:
|
|
temp_df = pd.DataFrame(
|
|
list(district_facilitys_conn.find({'city': city}, {'city': 1, 'region': 1, 'district_name': 1, 'school': 1,
|
|
'supermarket': 1, 'subway': 1, 'bank': 1, 'hospital': 1}).batch_size(10000)))
|
|
district_facilitys_df = pd.concat([district_facilitys_df, temp_df])
|
|
|
|
|
|
def count_facilitys(x):
|
|
ret = x
|
|
try:
|
|
if 'school' in x and x['school'] is not None and str(x['school']) != 'nan':
|
|
for item in x['school']:
|
|
if item is not None:
|
|
ret.school = len(item)
|
|
if 'supermarket' in x and x['supermarket'] is not None and str(x['supermarket']) != 'nan':
|
|
for item in x['supermarket']:
|
|
if item is not None:
|
|
ret.supermarket = len(item)
|
|
if 'subway' in x and x['subway'] is not None and str(x['subway']) != 'nan':
|
|
for item in x['subway']:
|
|
if item is not None:
|
|
ret.subway = len(item)
|
|
if 'bank' in x and x['bank'] is not None and str(x['bank']) != 'nan':
|
|
for item in x['bank']:
|
|
if item is not None:
|
|
ret.bank = len(item)
|
|
if 'hospital' in x and x['hospital'] is not None and str(x['hospital']) != 'nan':
|
|
for item in x['hospital']:
|
|
if item is not None:
|
|
ret.hospital = len(item)
|
|
return ret
|
|
except Exception as e:
|
|
print(e)
|
|
print(x)
|
|
return None
|
|
|
|
|
|
district_facilitys_new_df = district_facilitys_df.apply(lambda x: count_facilitys(x), axis=1)
|
|
|
|
district_facilitys_new_df = district_facilitys_new_df[['city', 'region', 'district_name', 'school', 'bank', 'hospital', 'supermarket', 'subway']]
|
|
district_facilitys_new_df = district_facilitys_new_df.rename(columns={'district_name': 'name', 'supermarket': 'market'})
|
|
|
|
result_df = pd.read_csv(path + '/FJW_COMM_' + dateutils.format_date(datetime.datetime.now(), '%Y%m') + '.csv', sep='@', encoding='utf-8')
|
|
result_df = pd.merge(result_df, district_facilitys_new_df, how='left', on=['city', 'region', 'name'])
|
|
result_df.loc[:, 'district_facilitys'] = '学校、商场、地铁、银行、医院'
|
|
|
|
|
|
def format_facilitys(x):
|
|
if type(x['school']) is not int:
|
|
x.school = None
|
|
if type(x['bank']) is not int:
|
|
x.bank = None
|
|
if type(x['hospital']) is not int:
|
|
x.hospital = None
|
|
if type(x['market']) is not int:
|
|
x.market = None
|
|
if type(x['subway']) is not int:
|
|
x.subway = None
|
|
return x
|
|
|
|
|
|
result_df = result_df.apply(lambda x: format_facilitys(x), axis=1)
|
|
result_df.to_csv(path + '/FJW_COMM_' + dateutils.format_date(datetime.datetime.now(), '%Y%m') + '.csv',
|
|
sep='@', encoding='utf-8', index=False)
|