diff --git a/dags/mt/supply/data/租金和房价设置阀值(69个城市).xlsx b/dags/mt/supply/data/租金和房价设置阀值(69个城市).xlsx new file mode 100644 index 0000000..ce6e3ec Binary files /dev/null and b/dags/mt/supply/data/租金和房价设置阀值(69个城市).xlsx differ diff --git a/dags/mt/supply/data/美团19城市列表.xlsx b/dags/mt/supply/data/美团19城市列表.xlsx index f90c24a..2023c61 100644 Binary files a/dags/mt/supply/data/美团19城市列表.xlsx and b/dags/mt/supply/data/美团19城市列表.xlsx differ diff --git a/dags/mt/supply/data/美团69城市列表.xlsx b/dags/mt/supply/data/美团69城市列表.xlsx new file mode 100644 index 0000000..0e9df00 Binary files /dev/null and b/dags/mt/supply/data/美团69城市列表.xlsx differ diff --git a/dags/mt/supply/data_stat.py b/dags/mt/supply/data_stat.py index 9a19e6b..2f9e10c 100644 --- a/dags/mt/supply/data_stat.py +++ b/dags/mt/supply/data_stat.py @@ -27,7 +27,11 @@ for city in cities: action['curr_city'] = city action['count'] = len(temp_df) for item in result_df.columns: - action[item] = round(len(temp_df[temp_df[item].notna()]) * 100 / len(temp_df), 2) + if item == 'deal_num' or item == 'school' or item == 'market' or item == 'subway' \ + or item == 'bank' or item == 'hospital': + action[item] = round(len(temp_df[temp_df[item] > 0]) * 100 / len(temp_df), 2) + else: + action[item] = round(len(temp_df[temp_df[item].notna()]) * 100 / len(temp_df), 2) if action['rent_price'] < 90: action['rent_flag'] = '租金覆盖率低于90%' if action['avg_price'] < 90: @@ -41,7 +45,7 @@ df.to_excel(aim_file_name, index=False) def deal_nan_val(x): if np.isnan(x['avg_price']): - x['avg_price'] = None + x['avg_price'] = None if np.isnan(x['rent_price']): x['rent_price'] = None if np.isnan(x['lng']): @@ -122,7 +126,8 @@ def save2_clickhouse(): # 删除旧数据 clickhouse_client.execute( "ALTER TABLE fangjia_stat.mt_community_stat DELETE WHERE city='" + city + "' and year_month = toDate('" + curr_month_start + "')") - clickhouse_client.to_sql(data=temp_df, table='mt_community_stat', engine='MergeTree', if_exists=False, order_by={'year_month'}) + clickhouse_client.to_sql(data=temp_df, table='mt_community_stat', engine='MergeTree', if_exists=False, + order_by={'year_month'}) def up_to_nextcloud(): diff --git a/dags/mt/supply/deal.py b/dags/mt/supply/deal.py index 09cfbf9..811470d 100644 --- a/dags/mt/supply/deal.py +++ b/dags/mt/supply/deal.py @@ -79,7 +79,6 @@ result_df['deal_num'].fillna(0, inplace=True) result_df['es_num'].fillna(0, inplace=True) result_df['deal_num'] += result_df['es_num'] result_df = result_df.drop(columns={'es_num'}) -result_df['deal_num'] = result_df['deal_num'].replace(0.0, None) # 暂时标识所有小区均未删除 result_df['status'] = 0 diff --git a/dags/mt/supply/facilitys.py b/dags/mt/supply/facilitys.py index 6231864..219ae8d 100644 --- a/dags/mt/supply/facilitys.py +++ b/dags/mt/supply/facilitys.py @@ -16,8 +16,6 @@ path = "/home/airflow/data/mt" output_date = sys.argv[1] if sys.argv[1] else dateutils.format_date(datetime.datetime.now(), '%Y%m') -seaweed_new_result_df = pd.read_csv(path + '/FJW_COMM_' + output_date + '.csv', sep='@', encoding='utf-8') - month_start = dateutils.format_date(dateutils.add_months(dateutils.parse_date(output_date, '%Y%m'), months=-1), '%Y%m') + '01' curr_month_start = output_date + '01' logging.info("日期信息:" + month_start + ' ' + curr_month_start) @@ -76,53 +74,62 @@ district_facilitys_new_df = district_facilitys_new_df.rename(columns={'district_ district_facilitys_new_df = district_facilitys_new_df.drop_duplicates(subset=['city', 'region', 'name'], keep='first') result_df = pd.read_csv(path + '/FJW_COMM_' + output_date + '.csv', sep='@', encoding='utf-8') +if 'school' in result_df.columns: + result_df = result_df.drop(columns=['school', 'bank', 'hospital', 'market', 'subway']) + result_df = pd.merge(result_df, district_facilitys_new_df, how='left', on=['city', 'region', 'name']) result_df.loc[:, 'district_facilitys'] = None # 剔除特别大的 def format_facilitys(x): - if type(x['school']) is not int: - x.school = 0 - else: - if x.school > 10: - x.school = 10 - - if type(x['bank']) is not int: - x.bank = 0 - else: - if x.bank > 30: - x.bank = 30 - - if type(x['hospital']) is not int: - x.hospital = 0 - else: - if x.hospital > 10: - x.hospital = 10 - if type(x['market']) is not int: - x.market = 0 - else: - if x.market > 15: - x.market = 15 - - if type(x['subway']) is not int: - x.subway = 0 - else: - if x.subway > 5: - x.subway = 5 - # 对于存在周边配套的输出字段 info = list() - if x.school > 0: - info.append('学校') - if x.market > 0: - info.append('商场') - if x.subway > 0: - info.append('地铁') - if x.bank > 0: - info.append('银行') - if x.hospital > 0: - info.append('医院') + + if 'school' in x: + if type(x['school']) is not int: + x.school = 0 + else: + if x.school > 0: + info.append('学校') + if x.school > 10: + x.school = 10 + if 'bank' in x: + if type(x['bank']) is not int: + x.bank = 0 + else: + if x.bank > 0: + info.append('银行') + if x.bank > 30: + x.bank = 30 + + if 'hospital' in x: + if type(x['hospital']) is not int: + x.hospital = 0 + else: + if x.hospital > 0: + info.append('医院') + if x.hospital > 10: + x.hospital = 10 + + if 'market' in x: + if type(x['market']) is not int: + x.market = 0 + else: + if x.market > 0: + info.append('商场') + if x.market > 15: + x.market = 15 + + if 'subway' in x: + if type(x['subway']) is not int: + x.subway = 0 + else: + if x.subway > 0: + info.append('地铁') + if x.subway > 5: + x.subway = 5 + x.district_facilitys = '、'.join(info) return x