""" 周边配套数量统计 """ from goojia_utils import MAIN_MONGO_CLIENT, dateutils, ClickHouseClient import pandas as pd import datetime import logging import sys district_facilitys_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'district_facilitys') tmp_ck = ClickHouseClient(database='tmp') city_df = pd.read_excel('/opt/airflow/dags/repo/dags/mt/supply/data/美团88城市列表.xlsx') cities = city_df['城市列表'].values.tolist() path = "/home/airflow/data/mt" output_date = sys.argv[1] if len(sys.argv) > 1 else dateutils.format_date(datetime.datetime.now(), '%Y%m') month_start = dateutils.format_date(dateutils.add_months(dateutils.parse_date(output_date, '%Y%m'), months=-1), '%Y%m') + '01' curr_month_start = output_date + '01' logging.info("日期信息:" + month_start + ' ' + curr_month_start) month_start_date = dateutils.parse_date(month_start, '%Y%m%d') curr_month_start_date = dateutils.parse_date(curr_month_start, '%Y%m%d') district_facilitys_df = pd.DataFrame() district_facilitys_school_df = pd.DataFrame(list(district_facilitys_conn.aggregate([ {'$match': {'school': {'$ne': None}, 'city': {'$in': cities}}}, {'$project': {'city': '$city', 'region': '$region', 'district_name': '$district_name', 'school': {'$size': '$school'}}} ]).batch_size(10000))) district_facilitys_school_df = district_facilitys_school_df.drop(columns=['_id']) district_facilitys_supermarket_df = pd.DataFrame(list(district_facilitys_conn.aggregate([ {'$match': {'supermarket': {'$ne': None}, 'city': {'$in': cities}}}, {'$project': {'city': '$city', 'region': '$region', 'district_name': '$district_name', 'supermarket': {'$size': '$supermarket'}}} ]).batch_size(10000))) district_facilitys_supermarket_df = district_facilitys_supermarket_df.drop(columns=['_id']) district_facilitys_subway_df = pd.DataFrame(list(district_facilitys_conn.aggregate([ {'$match': {'subway': {'$ne': None}, 'city': {'$in': cities}}}, {'$project': {'city': '$city', 'region': '$region', 'district_name': '$district_name', 'subway': {'$size': '$subway'}}} ]).batch_size(10000))) district_facilitys_subway_df = district_facilitys_subway_df.drop(columns=['_id']) district_facilitys_bank_df = pd.DataFrame(list(district_facilitys_conn.aggregate([ {'$match': {'bank': {'$ne': None}, 'city': {'$in': cities}}}, {'$project': {'city': '$city', 'region': '$region', 'district_name': '$district_name', 'bank': {'$size': '$bank'}}} ]).batch_size(10000))) district_facilitys_bank_df = district_facilitys_bank_df.drop(columns=['_id']) district_facilitys_hospital_df = pd.DataFrame(list(district_facilitys_conn.aggregate([ {'$match': {'hospital': {'$ne': None}, 'city': {'$in': cities}}}, {'$project': {'city': '$city', 'region': '$region', 'district_name': '$district_name', 'hospital': {'$size': '$hospital'}}} ]).batch_size(10000))) district_facilitys_hospital_df = district_facilitys_hospital_df.drop(columns=['_id']) district_facilitys_df = pd.merge(district_facilitys_school_df, district_facilitys_supermarket_df, how='outer', on=['city', 'region', 'district_name']) district_facilitys_df = pd.merge(district_facilitys_df, district_facilitys_subway_df, how='outer', on=['city', 'region', 'district_name']) district_facilitys_df = pd.merge(district_facilitys_df, district_facilitys_bank_df, how='outer', on=['city', 'region', 'district_name']) district_facilitys_df = pd.merge(district_facilitys_df, district_facilitys_hospital_df, how='outer', on=['city', 'region', 'district_name']) district_facilitys_new_df = district_facilitys_df[ ['city', 'region', 'district_name', 'school', 'bank', 'hospital', 'supermarket', 'subway']] district_facilitys_new_df = district_facilitys_new_df.rename(columns={'district_name': 'name', 'supermarket': 'market'}) district_facilitys_new_df = district_facilitys_new_df.drop_duplicates(subset=['city', 'region', 'name'], keep='first') result_df = tmp_ck.read_sql_query("select * from mt_community") del result_df['m_date'] if 'school' in result_df.columns: result_df = result_df.drop(columns=['school', 'bank', 'hospital', 'market', 'subway']) result_df = pd.merge(result_df, district_facilitys_new_df, how='left', on=['city', 'region', 'name']) result_df.loc[:, 'district_facilitys'] = None # 剔除特别大的 def format_facilitys(x): # 对于存在周边配套的输出字段 info = list() if 'school' in x: if type(x['school']) is not float: x.school = 0 else: if x.school > 0: info.append('学校') if x.school > 10: x.school = 10 if 'bank' in x: if type(x['bank']) is not float: x.bank = 0 else: if x.bank > 0: info.append('银行') if x.bank > 30: x.bank = 30 if 'hospital' in x: if type(x['hospital']) is not float: x.hospital = 0 else: if x.hospital > 0: info.append('医院') if x.hospital > 10: x.hospital = 10 if 'market' in x: if type(x['market']) is not float: x.market = 0 else: if x.market > 0: info.append('商场') if x.market > 15: x.market = 15 if 'subway' in x: if type(x['subway']) is not float: x.subway = 0 else: if x.subway > 0: info.append('地铁') if x.subway > 5: x.subway = 5 x.district_facilitys = '、'.join(info) return x result_df = result_df.apply(lambda x: format_facilitys(x), axis=1) result_df['market'].fillna(0, inplace=True) result_df['school'].fillna(0, inplace=True) result_df['bank'].fillna(0, inplace=True) result_df['hospital'].fillna(0, inplace=True) result_df['subway'].fillna(0, inplace=True) ck_delete_sql = "ALTER TABLE tmp.mt_community DELETE WHERE 1=1 " tmp_ck.execute(ck_delete_sql) tmp_ck.to_sql_plus(result_df, 'mt_community')