优化
This commit is contained in:
+56
-20
@@ -5,6 +5,7 @@ import pandas as pd
|
||||
from goojia_utils import dateutils, ClickHouseClient
|
||||
import datetime
|
||||
import sys
|
||||
import numpy as np
|
||||
|
||||
path = "/home/airflow/data/mt"
|
||||
|
||||
@@ -13,7 +14,7 @@ clickhouse_client = ClickHouseClient(database='fangjia_stat')
|
||||
output_date = sys.argv[1] if sys.argv[1] else dateutils.format_date(datetime.datetime.now(), '%Y%m')
|
||||
print("输出日期" + output_date)
|
||||
|
||||
result_df = pd.read_csv(path + '/FJW_COMM_' + output_date, sep='@', encoding='utf-8')
|
||||
result_df = pd.read_csv(path + '/FJW_COMM_' + output_date + ".csv", sep='@', encoding='utf-8')
|
||||
|
||||
city_df = pd.read_excel('/opt/airflow/dags/repo/dags/mt/supply/data/美团19城市列表.xlsx')
|
||||
cities = city_df['城市列表'].values.tolist()
|
||||
@@ -38,6 +39,42 @@ aim_file_name = path + "/" + output_date + '覆盖率统计.xlsx'
|
||||
df.to_excel(aim_file_name, index=False)
|
||||
|
||||
|
||||
def deal_nan_val(x):
|
||||
if np.isnan(x['avg_price']):
|
||||
x['avg_price'] = None
|
||||
if np.isnan(x['rent_price']):
|
||||
x['rent_price'] = None
|
||||
if np.isnan(x['lng']):
|
||||
x['lng'] = None
|
||||
if np.isnan(x['lat']):
|
||||
x['lat'] = None
|
||||
if np.isnan(x['car_site_count']):
|
||||
x['car_site_count'] = None
|
||||
if np.isnan(x['households']):
|
||||
x['households'] = None
|
||||
if np.isnan(x['all_area']):
|
||||
x['all_area'] = None
|
||||
if np.isnan(x['build_area']):
|
||||
x['build_area'] = None
|
||||
if np.isnan(x['house_area']):
|
||||
x['house_area'] = None
|
||||
if np.isnan(x['deal_num']):
|
||||
x['deal_num'] = None
|
||||
if np.isnan(x['market']):
|
||||
x['market'] = None
|
||||
if np.isnan(x['subway']):
|
||||
x['subway'] = None
|
||||
if np.isnan(x['bank']):
|
||||
x['bank'] = None
|
||||
if np.isnan(x['bank']):
|
||||
x['bank'] = None
|
||||
if np.isnan(x['hospital']):
|
||||
x['hospital'] = None
|
||||
if np.isnan(x['property_fee']):
|
||||
x['property_fee'] = None
|
||||
return x
|
||||
|
||||
|
||||
def save2_clickhouse():
|
||||
curr_month_start = output_date + '01'
|
||||
# result_df['avg_price'].fillna(-99.0, inplace=True)
|
||||
@@ -56,29 +93,28 @@ def save2_clickhouse():
|
||||
# result_df['bank'].fillna(-99.0, inplace=True)
|
||||
# result_df['hospital'].fillna(-99.0, inplace=True)
|
||||
# result_df['property_fee'].fillna(-99.0, inplace=True)
|
||||
result_df.fillna('null', inplace=True)
|
||||
result_df['plot_ratio'] = result_df['plot_ratio'].astype(str)
|
||||
result_df['landscaping_ratio'] = result_df['landscaping_ratio'].astype(str)
|
||||
result_df['property_life'] = result_df['property_life'].astype(str)
|
||||
result_df['finish_building_date'] = result_df['finish_building_date'].astype(str)
|
||||
result_df['house_type'] = result_df['house_type'].astype(str)
|
||||
|
||||
#result_df.fillna('', inplace=True)
|
||||
result_df['avg_price'].astype(int)
|
||||
result_df['rent_price'].astype(int)
|
||||
result_df['lng'].astype(int)
|
||||
result_df['lat'].astype(int)
|
||||
result_df['car_site_count'].astype(int)
|
||||
result_df['households'].astype(int)
|
||||
result_df['all_area'].astype(int)
|
||||
result_df['build_area'].astype(int)
|
||||
result_df['house_area'].astype(int)
|
||||
result_df['deal_num'].astype(int)
|
||||
result_df['school'].astype(int)
|
||||
result_df['market'].astype(int)
|
||||
result_df['subway'].astype(int)
|
||||
result_df['bank'].astype(int)
|
||||
result_df['hospital'].astype(int)
|
||||
result_df['property_fee'].astype(int)
|
||||
result_df['avg_price'] = result_df['avg_price'].astype(int, errors='ignore')
|
||||
result_df['rent_price'] = result_df['rent_price'].astype(int, errors='ignore')
|
||||
result_df['lng'] = result_df['lng'].astype(float, errors='ignore')
|
||||
result_df['lat'] = result_df['lat'].astype(float, errors='ignore')
|
||||
result_df['car_site_count'] = result_df['car_site_count'].astype(int, errors='ignore')
|
||||
result_df['households'] = result_df['households'].astype(int, errors='ignore')
|
||||
result_df['all_area'] = result_df['all_area'].astype(int, errors='ignore')
|
||||
result_df['build_area'] = result_df['build_area'].astype(int, errors='ignore')
|
||||
result_df['house_area'] = result_df['house_area'].astype(int, errors='ignore')
|
||||
result_df['deal_num'] = result_df['deal_num'].astype(int, errors='ignore')
|
||||
result_df['school'] = result_df['school'].astype(int, errors='ignore')
|
||||
result_df['market'] = result_df['market'].astype(int, errors='ignore')
|
||||
result_df['subway'] = result_df['subway'].astype(int, errors='ignore')
|
||||
result_df['bank'] = result_df['bank'].astype(int, errors='ignore')
|
||||
result_df['hospital'] = result_df['hospital'].astype(int, errors='ignore')
|
||||
result_df['property_fee'] = result_df['property_fee'].astype(float, errors='ignore')
|
||||
|
||||
for city in cities:
|
||||
temp_df = result_df[result_df['city'] == city]
|
||||
@@ -102,6 +138,6 @@ def up_to_nextcloud():
|
||||
nxc.upload_file(aim_file_name, aim_file_path)
|
||||
|
||||
|
||||
save2_clickhouse()
|
||||
up_to_nextcloud()
|
||||
save2_clickhouse()
|
||||
# https://oapi.dingtalk.com/robot/send?access_token=e022515642e544942f86bbe234448edc3b8bddea8184e63b7edf4129b31c8e79
|
||||
|
||||
@@ -28,7 +28,7 @@ logging.info("日期信息:" + month_start + ' ' + curr_month_start)
|
||||
month_start_date = dateutils.parse_date(month_start, '%Y%m%d')
|
||||
curr_month_start_date = dateutils.parse_date(curr_month_start, '%Y%m%d')
|
||||
|
||||
result_df = pd.read_csv(path + '/FJW_COMM_' + dateutils.format_date(datetime.datetime.now(), '%Y%m') + '.csv', sep='@', encoding='utf-8')
|
||||
result_df = pd.read_csv(path + '/FJW_COMM_' + output_date + '.csv', sep='@', encoding='utf-8')
|
||||
|
||||
if 'deal_num' in result_df.columns:
|
||||
result_df = result_df.drop(columns=['deal_num'])
|
||||
|
||||
+24
-16
@@ -73,7 +73,7 @@ district_facilitys_df = pd.merge(district_facilitys_df, district_facilitys_hospi
|
||||
district_facilitys_new_df = district_facilitys_df[
|
||||
['city', 'region', 'district_name', 'school', 'bank', 'hospital', 'supermarket', 'subway']]
|
||||
district_facilitys_new_df = district_facilitys_new_df.rename(columns={'district_name': 'name', 'supermarket': 'market'})
|
||||
district_facilitys_new_df = district_facilitys_new_df.drop_duplicates(subsets=['city', 'region', 'name'], keep='first')
|
||||
district_facilitys_new_df = district_facilitys_new_df.drop_duplicates(subset=['city', 'region', 'name'], keep='first')
|
||||
|
||||
result_df = pd.read_csv(path + '/FJW_COMM_' + output_date + '.csv', sep='@', encoding='utf-8')
|
||||
result_df = pd.merge(result_df, district_facilitys_new_df, how='left', on=['city', 'region', 'name'])
|
||||
@@ -83,25 +83,33 @@ result_df.loc[:, 'district_facilitys'] = None
|
||||
# 剔除特别大的
|
||||
def format_facilitys(x):
|
||||
if type(x['school']) is not int:
|
||||
x.school = None
|
||||
x.school = 0
|
||||
else:
|
||||
if x.school > 10:
|
||||
x.school = 10
|
||||
|
||||
if type(x['bank']) is not int:
|
||||
x.bank = None
|
||||
x.bank = 0
|
||||
else:
|
||||
if x.bank > 30:
|
||||
x.bank = 30
|
||||
|
||||
if type(x['hospital']) is not int:
|
||||
x.hospital = None
|
||||
x.hospital = 0
|
||||
else:
|
||||
if x.hospital > 10:
|
||||
x.hospital = 10
|
||||
if type(x['market']) is not int:
|
||||
x.market = None
|
||||
x.market = 0
|
||||
else:
|
||||
if x.market > 15:
|
||||
x.market = 15
|
||||
|
||||
if type(x['subway']) is not int:
|
||||
x.subway = None
|
||||
if x.school > 10:
|
||||
x.school = 10
|
||||
if x.market > 15:
|
||||
x.market = 15
|
||||
if x.subway > 5:
|
||||
x.subway = 5
|
||||
if x.bank > 30:
|
||||
x.bank = 30
|
||||
if x.hospital > 10:
|
||||
x.hospital = 10
|
||||
x.subway = 0
|
||||
else:
|
||||
if x.subway > 5:
|
||||
x.subway = 5
|
||||
|
||||
# 对于存在周边配套的输出字段
|
||||
info = list()
|
||||
|
||||
Reference in New Issue
Block a user