数据写入clickhouse
This commit is contained in:
@@ -37,14 +37,40 @@ df.to_excel(aim_file_name, index=False)
|
||||
|
||||
def save2_clickhouse():
|
||||
curr_month_start = dateutils.format_date(datetime.datetime.now(), '%Y%m') + '01'
|
||||
result_df['avg_price'].fillna(-99.0, inplace=True)
|
||||
result_df['rent_price'].fillna(-99.0, inplace=True)
|
||||
result_df['lng'].fillna(-99.0, inplace=True)
|
||||
result_df['lat'].fillna(-99.0, inplace=True)
|
||||
result_df['car_site_count'].fillna(-99.0, inplace=True)
|
||||
result_df['households'].fillna(-99.0, inplace=True)
|
||||
result_df['all_area'].fillna(-99.0, inplace=True)
|
||||
result_df['build_area'].fillna(-99.0, inplace=True)
|
||||
result_df['house_area'].fillna(-99.0, inplace=True)
|
||||
result_df['deal_num'].fillna(-99.0, inplace=True)
|
||||
result_df['school'].fillna(-99.0, inplace=True)
|
||||
result_df['market'].fillna(-99.0, inplace=True)
|
||||
result_df['subway'].fillna(-99.0, inplace=True)
|
||||
result_df['bank'].fillna(-99.0, inplace=True)
|
||||
result_df['hospital'].fillna(-99.0, inplace=True)
|
||||
result_df['property_fee'] = result_df['property_fee'].astype(str)
|
||||
result_df['property_fee'] = result_df['property_fee'].astype(str)
|
||||
result_df['plot_ratio'] = result_df['plot_ratio'].astype(str)
|
||||
result_df['landscaping_ratio'] = result_df['landscaping_ratio'].astype(str)
|
||||
result_df['property_life'] = result_df['property_life'].astype(str)
|
||||
result_df['finish_building_date'] = result_df['finish_building_date'].astype(str)
|
||||
result_df['house_type'] = result_df['house_type'].astype(str)
|
||||
|
||||
result_df.fillna('null', inplace=True)
|
||||
# 小区临时设置为0
|
||||
result_df['status'] = 0
|
||||
|
||||
for city in cities:
|
||||
temp_df = result_df[result_df['city'] == city]
|
||||
temp_df['year_month'] = curr_month_start
|
||||
temp_df['year_month'] = dateutils.parse_date(curr_month_start, fm='%Y%m%d')
|
||||
# 删除旧数据
|
||||
clickhouse_client.execute(
|
||||
"ALTER TABLE fangjia_stat.mt_community_stat DELETE WHERE city='" + city + "' and year_month = toDate('" + curr_month_start + "')")
|
||||
clickhouse_client.to_sql(data=temp_df.head(1), table='mt_community_stat', engine='MergeTree', if_exists=False, order_by={'year_month'})
|
||||
clickhouse_client.to_sql(data=temp_df, table='mt_community_stat', engine='MergeTree', if_exists=False, order_by={'year_month'})
|
||||
|
||||
|
||||
def up_to_nextcloud():
|
||||
|
||||
@@ -82,6 +82,8 @@ result_df['deal_num'] += result_df['es_num']
|
||||
result_df = result_df.drop(columns={'es_num'})
|
||||
result_df['deal_num'] = result_df['deal_num'].replace(0, None)
|
||||
|
||||
# 暂时标识所有小区均未删除
|
||||
result_df['status'] = 0
|
||||
|
||||
columns = ['id', 'city', 'region', 'name', 'alias', 'address', 'addr_alias', 'avg_price', 'rent_price', 'lng', 'lat',
|
||||
'block', 'loop_line_info', 'business_quarter', 'households', 'property_fee', 'car_site_count',
|
||||
@@ -89,6 +91,6 @@ columns = ['id', 'city', 'region', 'name', 'alias', 'address', 'addr_alias', 'av
|
||||
'all_area', 'build_area', 'plot_ratio', 'landscaping_ratio', 'property_type', 'build_type',
|
||||
'build_structure', 'finish_building_date',
|
||||
'developer', 'house_type', 'house_area', 'deal_num', 'district_facilitys', 'school', 'market', 'subway',
|
||||
'bank', 'hospital']
|
||||
'bank', 'hospital', 'status']
|
||||
result_df.to_csv(path + '/FJW_COMM_' + dateutils.format_date(datetime.datetime.now(), '%Y%m') + '.csv', sep='@',
|
||||
encoding='utf-8', index=False, columns=columns)
|
||||
Reference in New Issue
Block a user