diff --git a/dags/mt/supply/comm.py b/dags/mt/supply/comm.py index 1802cf6..2e38f30 100644 --- a/dags/mt/supply/comm.py +++ b/dags/mt/supply/comm.py @@ -5,7 +5,6 @@ from goojia_utils import MAIN_MONGO_CLIENT, dateutils, commutils import pandas as pd import datetime import os -import re seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') @@ -37,13 +36,8 @@ def format_seaweed(x): :return: """ try: - if str(x.mapping_alias) != 'nan' and x.mapping_alias != None: - # 过滤映射别名,防止含有主名 - if x.name in x.mapping_alias: - x.mapping_alias.remove(x.name) - x.mapping_alias = ','.join(x.mapping_alias) - if x.addr_alias is not None and str(x.addr_alias) != 'nan': - x.addr_alias = ','.join(x.addr_alias) + x.mapping_alias = commutils.format_mapping_alias(x.mapping_alias, x.name) + x.addr_alias = commutils.format_addr_alias(x.addr_alias) x.virescence_percent = commutils.format_greening_rate(x.virescence_percent) x.all_area = commutils.format_all_area(x.all_area) x.build_area = commutils.format_build_area(x.build_area) @@ -52,28 +46,18 @@ def format_seaweed(x): x.lng = commutils.format_lat_lng(x.lng) x.lat = commutils.format_lat_lng(x.lat) x.cubage_percent = commutils.format_plot_ratio(x.cubage_percent) - # 过滤房型,大于5房过滤,剔除厅卫 - if 'house_type' in x and str(x['house_type']) != 'nan': - if re.match('[1-6]室', x['house_type']) is None: - x.house_type = None - x.house_area = None - else: - x.house_type = re.sub('(\d+厅)?(\d+厨)?(\d+卫)?', '', x.house_type) - if '1室' in x.house_type: - if x.house_area > 125 or x.house_area < 20: - x.house_area = None - if '2室' in x.house_type: - if x.house_area > 200 or x.house_area < 30: - x.house_area = None - if '3室' in x.house_type: - if x.house_area > 260 or x.house_area < 60: - x.house_area = None - if '4室' in x.house_type: - if x.house_area > 260 or x.house_area < 90: - x.house_area = None - if type(x.alias) == str and '@' in x.alias: - x.alias = None - + house_type_area_dic = commutils.format_house_type_area(x.main_type, x.main_type_area) + x.main_type = house_type_area_dic['house_type'] + x.main_type_area = house_type_area_dic['house_area'] + x.household_count = commutils.format_households(x.household_count) + x.estate_charge = commutils.format_property_fee(x.estate_charge) + x.developer = commutils.format_developer(x.developer) + x.finish_building_date = commutils.format_finish_building_date(x.finish_building_date) + x.loop_line_info = commutils.format_loop_line_info(x.loop_line_info) + x.build_type = commutils.format_build_type(x.build_type) + x.property_life = commutils.format_property_life(x.property_life) + x.block = commutils.format_block(x.block) + x.business_quarter = commutils.format_business_quarter(x.business_quarter) return x except Exception as e: print(e) diff --git a/dags/mt/supply/deal.py b/dags/mt/supply/deal.py index 46e544a..695dd93 100644 --- a/dags/mt/supply/deal.py +++ b/dags/mt/supply/deal.py @@ -85,6 +85,11 @@ result_df['deal_num'] = result_df['deal_num'].replace(0.0, None) # 暂时标识所有小区均未删除 result_df['status'] = 0 +# 剔除id为na的 +result_df = result_df[result_df['id'].notna()] +# 在导出时最后一次去重,防止小区重复 +result_df = result_df.drop_duplicates(subset=['id'], keep='first') + columns = ['id', 'city', 'region', 'name', 'alias', 'address', 'addr_alias', 'avg_price', 'rent_price', 'lng', 'lat', 'block', 'loop_line_info', 'business_quarter', 'households', 'property_fee', 'car_site_count', 'property_life', 'house_property', diff --git a/dags/mt/supply/facilitys.py b/dags/mt/supply/facilitys.py index 5949c0a..86198be 100644 --- a/dags/mt/supply/facilitys.py +++ b/dags/mt/supply/facilitys.py @@ -31,50 +31,57 @@ district_facilitys_df = pd.DataFrame() district_facilitys_school_df = pd.DataFrame(list(district_facilitys_conn.aggregate([ {'$match': {'school': {'$ne': None}}}, {'$project': {'city': '$city', 'region': '$region', 'district_name': '$district_name', - 'school': {'$size': '$school'}}} + 'school': {'$size': '$school'}}} ]).batch_size(10000))) district_facilitys_school_df = district_facilitys_school_df.drop(columns=['_id']) district_facilitys_supermarket_df = pd.DataFrame(list(district_facilitys_conn.aggregate([ {'$match': {'supermarket': {'$ne': None}}}, {'$project': {'city': '$city', 'region': '$region', 'district_name': '$district_name', - 'supermarket': {'$size': '$supermarket'}}} + 'supermarket': {'$size': '$supermarket'}}} ]).batch_size(10000))) district_facilitys_supermarket_df = district_facilitys_supermarket_df.drop(columns=['_id']) district_facilitys_subway_df = pd.DataFrame(list(district_facilitys_conn.aggregate([ {'$match': {'subway': {'$ne': None}}}, {'$project': {'city': '$city', 'region': '$region', 'district_name': '$district_name', - 'subway': {'$size': '$subway'}}} + 'subway': {'$size': '$subway'}}} ]).batch_size(10000))) district_facilitys_subway_df = district_facilitys_subway_df.drop(columns=['_id']) district_facilitys_bank_df = pd.DataFrame(list(district_facilitys_conn.aggregate([ {'$match': {'bank': {'$ne': None}}}, {'$project': {'city': '$city', 'region': '$region', 'district_name': '$district_name', - 'bank': {'$size': '$bank'}}} + 'bank': {'$size': '$bank'}}} ]).batch_size(10000))) district_facilitys_bank_df = district_facilitys_bank_df.drop(columns=['_id']) district_facilitys_hospital_df = pd.DataFrame(list(district_facilitys_conn.aggregate([ {'$match': {'hospital': {'$ne': None}}}, {'$project': {'city': '$city', 'region': '$region', 'district_name': '$district_name', - 'hospital': {'$size': '$hospital'}}} + 'hospital': {'$size': '$hospital'}}} ]).batch_size(10000))) district_facilitys_hospital_df = district_facilitys_hospital_df.drop(columns=['_id']) -district_facilitys_df = pd.merge(district_facilitys_school_df,district_facilitys_supermarket_df, how='left', on=['city', 'region', 'district_name']) -district_facilitys_df = pd.merge(district_facilitys_df,district_facilitys_subway_df, how='left', on=['city', 'region', 'district_name']) -district_facilitys_df = pd.merge(district_facilitys_df,district_facilitys_bank_df, how='left', on=['city', 'region', 'district_name']) -district_facilitys_df = pd.merge(district_facilitys_df,district_facilitys_hospital_df, how='left', on=['city', 'region', 'district_name']) +district_facilitys_df = pd.merge(district_facilitys_school_df, district_facilitys_supermarket_df, how='left', + on=['city', 'region', 'district_name']) +district_facilitys_df = pd.merge(district_facilitys_df, district_facilitys_subway_df, how='left', + on=['city', 'region', 'district_name']) +district_facilitys_df = pd.merge(district_facilitys_df, district_facilitys_bank_df, how='left', + on=['city', 'region', 'district_name']) +district_facilitys_df = pd.merge(district_facilitys_df, district_facilitys_hospital_df, how='left', + on=['city', 'region', 'district_name']) -district_facilitys_new_df = district_facilitys_df[['city', 'region', 'district_name', 'school', 'bank', 'hospital', 'supermarket', 'subway']] +district_facilitys_new_df = district_facilitys_df[ + ['city', 'region', 'district_name', 'school', 'bank', 'hospital', 'supermarket', 'subway']] district_facilitys_new_df = district_facilitys_new_df.rename(columns={'district_name': 'name', 'supermarket': 'market'}) district_facilitys_new_df = district_facilitys_new_df.drop_duplicates(subsets=['city', 'region', 'name'], keep='first') -result_df = pd.read_csv(path + '/FJW_COMM_' + dateutils.format_date(datetime.datetime.now(), '%Y%m') + '.csv', sep='@', encoding='utf-8') +result_df = pd.read_csv(path + '/FJW_COMM_' + dateutils.format_date(datetime.datetime.now(), '%Y%m') + '.csv', sep='@', + encoding='utf-8') result_df = pd.merge(result_df, district_facilitys_new_df, how='left', on=['city', 'region', 'name']) -result_df.loc[:, 'district_facilitys'] = '学校、商场、地铁、银行、医院' +result_df.loc[:, 'district_facilitys'] = None + # 剔除特别大的 def format_facilitys(x): @@ -88,19 +95,39 @@ def format_facilitys(x): x.market = None if type(x['subway']) is not int: x.subway = None - if x['school'] > 10: - x.school = None - if x['market'] > 15: - x.market = None - if x['subway'] > 5: - x.subway = None - if x['bank'] > 30: - x.bank = None - if x['hospital'] > 10: - x.hospital = None + if x.school > 10: + x.school = 10 + if x.market > 15: + x.market = 15 + if x.subway > 5: + x.subway = 5 + if x.bank > 30: + x.bank = 30 + if x.hospital > 10: + x.hospital = 10 + + # 对于存在周边配套的输出字段 + info = list() + if x.school > 0: + info.append('学校') + if x.market > 0: + info.append('商场') + if x.subway > 0: + info.append('地铁') + if x.bank > 0: + info.append('银行') + if x.hospital > 0: + info.append('医院') + x.district_facilitys = '、'.join(info) return x result_df = result_df.apply(lambda x: format_facilitys(x), axis=1) +result_df['market'].fillna(0, inplace=True) +result_df['school'].fillna(0, inplace=True) +result_df['bank'].fillna(0, inplace=True) +result_df['hospital'].fillna(0, inplace=True) +result_df['subway'].fillna(0, inplace=True) + result_df.to_csv(path + '/FJW_COMM_' + dateutils.format_date(datetime.datetime.now(), '%Y%m') + '.csv', - sep='@', encoding='utf-8', index=False) + sep='@', encoding='utf-8', index=False)