""" 小区属性导出 """ from goojia_utils import MAIN_MONGO_CLIENT, dateutils, commutils import pandas as pd import datetime import os import re seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') path = "/home/airflow/data/mt" if not os.path.exists(path): os.makedirs(path) city_df = pd.read_excel('/opt/airflow/dags/repo/dags/mt/supply/data/美团19城市列表.xlsx') cities = city_df['城市列表'].values.tolist() seaweed_df = pd.DataFrame() for city in cities: temp_df = pd.DataFrame(list(seaweed_conn.find({'cat': 'district', 'status': 0, 'city': city}).batch_size(10000))) seaweed_df = pd.concat([seaweed_df, temp_df]) seaweed_result_df = seaweed_df[ ['_id', 'city', 'region', 'name', 'mapping_alias', 'address', 'addr_alias', 'lng', 'lat', 'block', 'loop_line_info', 'business_quarter', 'household_count', 'estate_charge', 'car_site_count', 'property_life', 'house_property', 'all_area', 'build_area', 'cubage_percent', 'virescence_percent', 'estate_type2', 'build_type', 'build_structure', 'finish_building_date', 'developer', 'main_type', 'main_type_area']] def format_seaweed(x): """ 格式化输出小区属性 :param x: :return: """ try: if str(x.mapping_alias) != 'nan' and x.mapping_alias != None: # 过滤映射别名,防止含有主名 if x.name in x.mapping_alias: x.mapping_alias.remove(x.name) x.mapping_alias = ','.join(x.mapping_alias) if x.addr_alias is not None and str(x.addr_alias) != 'nan': x.addr_alias = ','.join(x.addr_alias) x.virescence_percent = commutils.format_greening_rate(x.virescence_percent) x.all_area = commutils.format_all_area(x.all_area) x.build_area = commutils.format_build_area(x.build_area) x.car_site_count = commutils.format_car_site_count(x.car_site_count) x.household_count = commutils.format_household_count(x.household_count) x.lng = commutils.format_lat_lng(x.lng) x.lat = commutils.format_lat_lng(x.lat) x.cubage_percent = commutils.format_plot_ratio(x.cubage_percent) # 过滤房型,大于5房过滤,剔除厅卫 if 'house_type' in x and str(x['house_type']) != 'nan': if re.match('[1-6]室', x['house_type']) is None: x.house_type = None x.house_area = None else: x.house_type = re.sub('(\d+厅)?(\d+厨)?(\d+卫)?', '', x.house_type) if '1室' in x.house_type: if x.house_area > 125 or x.house_area < 20: x.house_area = None if '2室' in x.house_type: if x.house_area > 200 or x.house_area < 30: x.house_area = None if '3室' in x.house_type: if x.house_area > 260 or x.house_area < 60: x.house_area = None if '4室' in x.house_type: if x.house_area > 260 or x.house_area < 90: x.house_area = None if type(x.alias) == str and '@' in x.alias: x.alias = None return x except Exception as e: print(e) print(x) pass seaweed_new_result_df = seaweed_result_df.apply(lambda x: format_seaweed(x), axis=1) seaweed_new_result_df = seaweed_new_result_df.rename(columns={'_id': 'id', 'mapping_alias': 'alias', 'household_count': 'households', 'estate_charge': 'property_fee', 'cubage_percent' : 'plot_ratio', 'virescence_percent': 'landscaping_ratio', 'estate_type2': 'property_type', 'main_type': 'house_type', 'main_type_area': 'house_area'}) seaweed_new_result_df.to_csv(path + '/FJW_COMM_' + dateutils.format_date(datetime.datetime.now(), '%Y%m') + '.csv', sep='@', encoding='utf-8', index=False)