145 lines
6.0 KiB
Python
145 lines
6.0 KiB
Python
"""
|
|
周边配套数量统计
|
|
"""
|
|
from goojia_utils import MAIN_MONGO_CLIENT, dateutils
|
|
import pandas as pd
|
|
import datetime
|
|
import logging
|
|
import sys
|
|
|
|
district_facilitys_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'district_facilitys')
|
|
|
|
city_df = pd.read_excel('/opt/airflow/dags/repo/dags/mt/supply/data/美团19城市列表.xlsx')
|
|
cities = city_df['城市列表'].values.tolist()
|
|
|
|
path = "/home/airflow/data/mt"
|
|
|
|
output_date = sys.argv[1] if sys.argv[1] else dateutils.format_date(datetime.datetime.now(), '%Y%m')
|
|
|
|
month_start = dateutils.format_date(dateutils.add_months(dateutils.parse_date(output_date, '%Y%m'), months=-1), '%Y%m') + '01'
|
|
curr_month_start = output_date + '01'
|
|
logging.info("日期信息:" + month_start + ' ' + curr_month_start)
|
|
|
|
month_start_date = dateutils.parse_date(month_start, '%Y%m%d')
|
|
curr_month_start_date = dateutils.parse_date(curr_month_start, '%Y%m%d')
|
|
|
|
district_facilitys_df = pd.DataFrame()
|
|
district_facilitys_school_df = pd.DataFrame(list(district_facilitys_conn.aggregate([
|
|
{'$match': {'school': {'$ne': None}, 'city': {'$in': cities}}},
|
|
{'$project': {'city': '$city', 'region': '$region', 'district_name': '$district_name',
|
|
'school': {'$size': '$school'}}}
|
|
]).batch_size(10000)))
|
|
district_facilitys_school_df = district_facilitys_school_df.drop(columns=['_id'])
|
|
|
|
district_facilitys_supermarket_df = pd.DataFrame(list(district_facilitys_conn.aggregate([
|
|
{'$match': {'supermarket': {'$ne': None}, 'city': {'$in': cities}}},
|
|
{'$project': {'city': '$city', 'region': '$region', 'district_name': '$district_name',
|
|
'supermarket': {'$size': '$supermarket'}}}
|
|
]).batch_size(10000)))
|
|
district_facilitys_supermarket_df = district_facilitys_supermarket_df.drop(columns=['_id'])
|
|
|
|
district_facilitys_subway_df = pd.DataFrame(list(district_facilitys_conn.aggregate([
|
|
{'$match': {'subway': {'$ne': None}, 'city': {'$in': cities}}},
|
|
{'$project': {'city': '$city', 'region': '$region', 'district_name': '$district_name',
|
|
'subway': {'$size': '$subway'}}}
|
|
]).batch_size(10000)))
|
|
district_facilitys_subway_df = district_facilitys_subway_df.drop(columns=['_id'])
|
|
|
|
district_facilitys_bank_df = pd.DataFrame(list(district_facilitys_conn.aggregate([
|
|
{'$match': {'bank': {'$ne': None}, 'city': {'$in': cities}}},
|
|
{'$project': {'city': '$city', 'region': '$region', 'district_name': '$district_name',
|
|
'bank': {'$size': '$bank'}}}
|
|
]).batch_size(10000)))
|
|
district_facilitys_bank_df = district_facilitys_bank_df.drop(columns=['_id'])
|
|
|
|
district_facilitys_hospital_df = pd.DataFrame(list(district_facilitys_conn.aggregate([
|
|
{'$match': {'hospital': {'$ne': None}, 'city': {'$in': cities}}},
|
|
{'$project': {'city': '$city', 'region': '$region', 'district_name': '$district_name',
|
|
'hospital': {'$size': '$hospital'}}}
|
|
]).batch_size(10000)))
|
|
district_facilitys_hospital_df = district_facilitys_hospital_df.drop(columns=['_id'])
|
|
|
|
district_facilitys_df = pd.merge(district_facilitys_school_df, district_facilitys_supermarket_df, how='outer',
|
|
on=['city', 'region', 'district_name'])
|
|
district_facilitys_df = pd.merge(district_facilitys_df, district_facilitys_subway_df, how='outer',
|
|
on=['city', 'region', 'district_name'])
|
|
district_facilitys_df = pd.merge(district_facilitys_df, district_facilitys_bank_df, how='outer',
|
|
on=['city', 'region', 'district_name'])
|
|
district_facilitys_df = pd.merge(district_facilitys_df, district_facilitys_hospital_df, how='outer',
|
|
on=['city', 'region', 'district_name'])
|
|
|
|
district_facilitys_new_df = district_facilitys_df[
|
|
['city', 'region', 'district_name', 'school', 'bank', 'hospital', 'supermarket', 'subway']]
|
|
district_facilitys_new_df = district_facilitys_new_df.rename(columns={'district_name': 'name', 'supermarket': 'market'})
|
|
district_facilitys_new_df = district_facilitys_new_df.drop_duplicates(subset=['city', 'region', 'name'], keep='first')
|
|
|
|
result_df = pd.read_csv(path + '/FJW_COMM_' + output_date + '.csv', sep='@', encoding='utf-8')
|
|
if 'school' in result_df.columns:
|
|
result_df = result_df.drop(columns=['school', 'bank', 'hospital', 'market', 'subway'])
|
|
|
|
result_df = pd.merge(result_df, district_facilitys_new_df, how='left', on=['city', 'region', 'name'])
|
|
result_df.loc[:, 'district_facilitys'] = None
|
|
|
|
|
|
# 剔除特别大的
|
|
def format_facilitys(x):
|
|
# 对于存在周边配套的输出字段
|
|
info = list()
|
|
|
|
if 'school' in x:
|
|
if type(x['school']) is not int:
|
|
x.school = 0
|
|
else:
|
|
if x.school > 0:
|
|
info.append('学校')
|
|
if x.school > 10:
|
|
x.school = 10
|
|
if 'bank' in x:
|
|
if type(x['bank']) is not int:
|
|
x.bank = 0
|
|
else:
|
|
if x.bank > 0:
|
|
info.append('银行')
|
|
if x.bank > 30:
|
|
x.bank = 30
|
|
|
|
if 'hospital' in x:
|
|
if type(x['hospital']) is not int:
|
|
x.hospital = 0
|
|
else:
|
|
if x.hospital > 0:
|
|
info.append('医院')
|
|
if x.hospital > 10:
|
|
x.hospital = 10
|
|
|
|
if 'market' in x:
|
|
if type(x['market']) is not int:
|
|
x.market = 0
|
|
else:
|
|
if x.market > 0:
|
|
info.append('商场')
|
|
if x.market > 15:
|
|
x.market = 15
|
|
|
|
if 'subway' in x:
|
|
if type(x['subway']) is not int:
|
|
x.subway = 0
|
|
else:
|
|
if x.subway > 0:
|
|
info.append('地铁')
|
|
if x.subway > 5:
|
|
x.subway = 5
|
|
|
|
x.district_facilitys = '、'.join(info)
|
|
return x
|
|
|
|
|
|
result_df = result_df.apply(lambda x: format_facilitys(x), axis=1)
|
|
result_df['market'].fillna(0, inplace=True)
|
|
result_df['school'].fillna(0, inplace=True)
|
|
result_df['bank'].fillna(0, inplace=True)
|
|
result_df['hospital'].fillna(0, inplace=True)
|
|
result_df['subway'].fillna(0, inplace=True)
|
|
|
|
result_df.to_csv(path + '/FJW_COMM_' + output_date + '.csv', sep='@', encoding='utf-8', index=False)
|