""" 周边配套数量统计 """ from goojia_utils import MAIN_MONGO_CLIENT, dateutils import pandas as pd import datetime import logging district_facilitys_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'district_facilitys') city_df = pd.read_excel('/opt/airflow/dags/repo/dags/mt/supply/data/美团19城市列表.xlsx') cities = city_df['城市列表'].values.tolist() # 1. 获取「今天」 today = datetime.date.today() # 2. 获取当前月的第一天 first = today.replace(day=1) # 3. 减一天,得到上个月的最后一天 last_month = first - datetime.timedelta(days=1) month_start = last_month.strftime("%Y%m") + '01' curr_month_start = dateutils.format_date(datetime.datetime.now(), '%Y%m') + '01' logging.info(month_start + ' ' + curr_month_start) month_start_date = dateutils.parse_date(month_start, '%Y%m%d') curr_month_start_date = dateutils.parse_date(curr_month_start, '%Y%m%d') path = "/home/airflow/data/mt" district_facilitys_df = pd.DataFrame() for city in cities: temp_df = pd.DataFrame( list(district_facilitys_conn.find({'city': city}, {'city': 1, 'region': 1, 'district_name': 1, 'school': 1, 'supermarket': 1, 'subway': 1, 'bank': 1, 'hospital': 1}).batch_size(10000))) district_facilitys_df = pd.concat([district_facilitys_df, temp_df]) def count_facilitys(x): ret = x try: if 'school' in x and x['school'] is not None and str(x['school']) != 'nan': for item in x['school']: if item is not None: ret.school = len(item) if 'supermarket' in x and x['supermarket'] is not None and str(x['supermarket']) != 'nan': for item in x['supermarket']: if item is not None: ret.supermarket = len(item) if 'subway' in x and x['subway'] is not None and str(x['subway']) != 'nan': for item in x['subway']: if item is not None: ret.subway = len(item) if 'bank' in x and x['bank'] is not None and str(x['bank']) != 'nan': for item in x['bank']: if item is not None: ret.bank = len(item) if 'hospital' in x and x['hospital'] is not None and str(x['hospital']) != 'nan': for item in x['hospital']: if item is not None: ret.hospital = len(item) return ret except Exception as e: print(e) print(x) return None district_facilitys_new_df = district_facilitys_df.apply(lambda x: count_facilitys(x), axis=1) district_facilitys_new_df = district_facilitys_new_df[['city', 'region', 'district_name', 'school', 'bank', 'hospital', 'supermarket', 'subway']] district_facilitys_new_df = district_facilitys_new_df.rename(columns={'district_name': 'name', 'supermarket': 'market'}) result_df = pd.read_csv(path + '/FJW_COMM_' + dateutils.format_date(datetime.datetime.now(), '%Y%m') + '.csv', sep='@', encoding='utf-8') result_df = pd.merge(result_df, district_facilitys_new_df, how='left', on=['city', 'region', 'name']) result_df.loc[:, 'district_facilitys'] = '学校、商场、地铁、银行、医院' def format_facilitys(x): if type(x['school']) is not int: x.school = None if type(x['bank']) is not int: x.bank = None if type(x['hospital']) is not int: x.hospital = None if type(x['market']) is not int: x.market = None if type(x['subway']) is not int: x.subway = None return x result_df = result_df.apply(lambda x: format_facilitys(x), axis=1) result_df.to_csv(path + '/FJW_COMM_' + dateutils.format_date(datetime.datetime.now(), '%Y%m') + '.csv', sep='@', encoding='utf-8', index=False)