美团中间表改造
This commit is contained in:
@@ -0,0 +1,44 @@
|
||||
CREATE TABLE tmp.mt_community
|
||||
(
|
||||
id String COMMENT '小区ID',
|
||||
city String COMMENT '城市',
|
||||
region String COMMENT '区域',
|
||||
name String COMMENT '名称',
|
||||
alias String COMMENT '别名',
|
||||
address String COMMENT '地址',
|
||||
addr_alias String COMMENT '地址别名',
|
||||
avg_price Nullable(INT) COMMENT '均价',
|
||||
rent_price Nullable(INT) COMMENT '平米租金',
|
||||
lng Nullable(Float32) COMMENT '经度',
|
||||
lat Nullable(Float32) COMMENT '纬度',
|
||||
estate_type2 String COMMENT '物业类型',
|
||||
block String COMMENT '板块',
|
||||
loop_line_info String COMMENT '环线',
|
||||
business_quarter String COMMENT '商圈',
|
||||
households String COMMENT '户数',
|
||||
property_fee String COMMENT '物业费',
|
||||
car_site_count String COMMENT '停车位',
|
||||
property_life String COMMENT '产权年限',
|
||||
house_property String COMMENT '产权性质',
|
||||
all_area String COMMENT '占地面积',
|
||||
build_area String COMMENT '建筑面积',
|
||||
plot_ratio String COMMENT '容积率',
|
||||
landscaping_ratio String COMMENT '绿化率',
|
||||
property_type String COMMENT '物业类型',
|
||||
build_type String COMMENT '建筑类型',
|
||||
build_structure String COMMENT '建筑结构',
|
||||
finish_building_date String COMMENT '建成年代',
|
||||
developer String COMMENT '开发商',
|
||||
house_type String COMMENT '户型',
|
||||
house_area String COMMENT '房型建筑面积',
|
||||
deal_num Nullable(INT) COMMENT '近一年成交量',
|
||||
district_facilitys String COMMENT '周边配套数量字段',
|
||||
school Nullable(INT) COMMENT '学校数量',
|
||||
market Nullable(INT) COMMENT '商场数量',
|
||||
subway Nullable(INT) COMMENT '地铁数量',
|
||||
bank Nullable(INT) COMMENT '银行数量',
|
||||
hospital Nullable(INT) COMMENT '医院数量',
|
||||
status INT COMMENT '0 正常 1 删除',
|
||||
m_date DateTime('Asia/Shanghai') default now() COMMENT '更新时间'
|
||||
) ENGINE ReplacingMergeTree(m_date)
|
||||
PARTITION BY city PRIMARY KEY (city, region, name) ORDER BY (city, region, name);
|
||||
@@ -1,13 +1,14 @@
|
||||
"""
|
||||
小区属性导出
|
||||
"""
|
||||
from goojia_utils import MAIN_MONGO_CLIENT, dateutils, commutils
|
||||
from goojia_utils import MAIN_MONGO_CLIENT, dateutils, commutils, ClickHouseClient
|
||||
import pandas as pd
|
||||
import datetime
|
||||
import os
|
||||
import sys
|
||||
|
||||
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
||||
tmp_ck = ClickHouseClient(database='tmp')
|
||||
|
||||
path = "/home/airflow/data/mt"
|
||||
if not os.path.exists(path):
|
||||
@@ -74,5 +75,9 @@ seaweed_new_result_df = seaweed_new_result_df.rename(columns={'_id': 'id', 'mapp
|
||||
'cubage_percent' : 'plot_ratio', 'virescence_percent': 'landscaping_ratio', 'estate_type2': 'property_type',
|
||||
'main_type': 'house_type', 'main_type_area': 'house_area'})
|
||||
|
||||
seaweed_new_result_df.to_csv(path + '/FJW_COMM_' + output_date + '.csv',
|
||||
sep='@', encoding='utf-8', index=False)
|
||||
seaweed_new_result_df.to_csv(path + '/FJW_COMM_' + output_date + '.csv', sep='@', encoding='utf-8', index=False)
|
||||
|
||||
ck_delete_sql = "ALTER TABLE tmp.mt_community DELETE WHERE 1=1 "
|
||||
tmp_ck.execute(ck_delete_sql)
|
||||
|
||||
tmp_ck.to_sql_plus(seaweed_new_result_df, 'mt_community')
|
||||
|
||||
@@ -10,11 +10,12 @@ import requests
|
||||
path = "/home/airflow/data/mt"
|
||||
|
||||
clickhouse_client = ClickHouseClient(database='fangjia_stat')
|
||||
tmp_ck = ClickHouseClient(database='tmp')
|
||||
|
||||
output_date = sys.argv[1] if len(sys.argv) > 1 else dateutils.format_date(datetime.datetime.now(), '%Y%m')
|
||||
print("输出日期" + output_date)
|
||||
|
||||
result_df = pd.read_csv(path + '/FJW_COMM_' + output_date + ".csv", sep='@', encoding='utf-8')
|
||||
result_df = tmp_ck.read_sql_query("select * from tmp.mt_community")
|
||||
|
||||
city_df = pd.read_excel('/opt/airflow/dags/repo/dags/mt/supply/data/美团88城市列表.xlsx')
|
||||
cities = city_df['城市列表'].values.tolist()
|
||||
|
||||
+10
-3
@@ -1,7 +1,7 @@
|
||||
"""
|
||||
每个小区近一年成交统计
|
||||
"""
|
||||
from goojia_utils import MAIN_MONGO_CLIENT, dateutils, ElasticSearchClient
|
||||
from goojia_utils import MAIN_MONGO_CLIENT, dateutils, ElasticSearchClient, ClickHouseClient
|
||||
import pandas as pd
|
||||
import datetime
|
||||
import logging
|
||||
@@ -11,6 +11,7 @@ import sys
|
||||
es45_client = ElasticSearchClient(host='192.168.0.45')
|
||||
deal_price_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'deal_price')
|
||||
district_facilitys_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'district_facilitys')
|
||||
tmp_ck = ClickHouseClient(database='tmp')
|
||||
|
||||
city_df = pd.read_excel('/opt/airflow/dags/repo/dags/mt/supply/data/美团88城市列表.xlsx')
|
||||
cities = city_df['城市列表'].values.tolist()
|
||||
@@ -26,7 +27,7 @@ logging.info("日期信息:" + month_start + ' ' + curr_month_start)
|
||||
month_start_date = dateutils.parse_date(month_start, '%Y%m%d')
|
||||
curr_month_start_date = dateutils.parse_date(curr_month_start, '%Y%m%d')
|
||||
|
||||
result_df = pd.read_csv(path + '/FJW_COMM_' + output_date + '.csv', sep='@', encoding='utf-8')
|
||||
result_df = tmp_ck.read_sql_query("select * from mt_community")
|
||||
|
||||
if 'deal_num' in result_df.columns:
|
||||
result_df = result_df.drop(columns=['deal_num'])
|
||||
@@ -93,4 +94,10 @@ columns = ['id', 'city', 'region', 'name', 'alias', 'address', 'addr_alias', 'av
|
||||
'build_structure', 'finish_building_date',
|
||||
'developer', 'house_type', 'house_area', 'deal_num', 'district_facilitys', 'school', 'market', 'subway',
|
||||
'bank', 'hospital', 'status']
|
||||
result_df.to_csv(path + '/FJW_COMM_' + output_date + '.csv', sep='@', encoding='utf-8', index=False, columns=columns)
|
||||
|
||||
result_save_df = result_df[columns]
|
||||
|
||||
ck_delete_sql = "ALTER TABLE tmp.mt_community DELETE WHERE 1=1 "
|
||||
tmp_ck.execute(ck_delete_sql)
|
||||
|
||||
tmp_ck.to_sql_plus(result_save_df, 'mt_community')
|
||||
@@ -1,13 +1,14 @@
|
||||
"""
|
||||
周边配套数量统计
|
||||
"""
|
||||
from goojia_utils import MAIN_MONGO_CLIENT, dateutils
|
||||
from goojia_utils import MAIN_MONGO_CLIENT, dateutils, ClickHouseClient
|
||||
import pandas as pd
|
||||
import datetime
|
||||
import logging
|
||||
import sys
|
||||
|
||||
district_facilitys_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'district_facilitys')
|
||||
tmp_ck = ClickHouseClient(database='tmp')
|
||||
|
||||
city_df = pd.read_excel('/opt/airflow/dags/repo/dags/mt/supply/data/美团88城市列表.xlsx')
|
||||
cities = city_df['城市列表'].values.tolist()
|
||||
@@ -73,7 +74,8 @@ district_facilitys_new_df = district_facilitys_df[
|
||||
district_facilitys_new_df = district_facilitys_new_df.rename(columns={'district_name': 'name', 'supermarket': 'market'})
|
||||
district_facilitys_new_df = district_facilitys_new_df.drop_duplicates(subset=['city', 'region', 'name'], keep='first')
|
||||
|
||||
result_df = pd.read_csv(path + '/FJW_COMM_' + output_date + '.csv', sep='@', encoding='utf-8')
|
||||
result_df = tmp_ck.read_sql_query("select * from mt_community")
|
||||
|
||||
if 'school' in result_df.columns:
|
||||
result_df = result_df.drop(columns=['school', 'bank', 'hospital', 'market', 'subway'])
|
||||
|
||||
@@ -141,4 +143,7 @@ result_df['bank'].fillna(0, inplace=True)
|
||||
result_df['hospital'].fillna(0, inplace=True)
|
||||
result_df['subway'].fillna(0, inplace=True)
|
||||
|
||||
result_df.to_csv(path + '/FJW_COMM_' + output_date + '.csv', sep='@', encoding='utf-8', index=False)
|
||||
ck_delete_sql = "ALTER TABLE tmp.mt_community DELETE WHERE 1=1 "
|
||||
tmp_ck.execute(ck_delete_sql)
|
||||
|
||||
tmp_ck.to_sql_plus(result_df, 'mt_community')
|
||||
@@ -2,17 +2,17 @@
|
||||
租金、均价导出
|
||||
"""
|
||||
|
||||
from goojia_utils import MAIN_MONGO_CLIENT, dateutils, commutils
|
||||
from goojia_utils import MAIN_MONGO_CLIENT, dateutils, commutils, ClickHouseClient
|
||||
import pandas as pd
|
||||
import datetime
|
||||
import logging
|
||||
import sys
|
||||
import numpy as np
|
||||
|
||||
#district_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'district_stat')
|
||||
# 月度均价表
|
||||
district_sell_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'district_sell_stat')
|
||||
sqmetre_rent_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'sqmetre_rent')
|
||||
tmp_ck = ClickHouseClient(database='tmp')
|
||||
|
||||
|
||||
city_df = pd.read_excel('/opt/airflow/dags/repo/dags/mt/supply/data/美团88城市列表.xlsx')
|
||||
@@ -22,7 +22,8 @@ path = "/home/airflow/data/mt"
|
||||
|
||||
output_date = sys.argv[1] if len(sys.argv) > 1 else dateutils.format_date(datetime.datetime.now(), '%Y%m')
|
||||
|
||||
seaweed_new_result_df = pd.read_csv(path + '/FJW_COMM_' + output_date + '.csv', sep='@', encoding='utf-8')
|
||||
seaweed_new_result_df = tmp_ck.read_sql_query("select * from mt_community")
|
||||
|
||||
if 'avg_price' in seaweed_new_result_df:
|
||||
seaweed_new_result_df = seaweed_new_result_df.drop(columns=['avg_price', 'rent_price'])
|
||||
|
||||
@@ -111,4 +112,9 @@ result_df['ignore_comm'] = None
|
||||
result_df = result_df.apply(lambda x: format_rent(x), axis=1)
|
||||
result_df = result_df[result_df['ignore_comm'] != 1]
|
||||
|
||||
result_df.to_csv(path + '/FJW_COMM_' + output_date + '.csv', sep='@', encoding='utf-8', index=False)
|
||||
del result_df['ignore_comm']
|
||||
|
||||
ck_delete_sql = "ALTER TABLE tmp.mt_community DELETE WHERE 1=1 "
|
||||
tmp_ck.execute(ck_delete_sql)
|
||||
|
||||
tmp_ck.to_sql_plus(result_df, 'mt_community')
|
||||
|
||||
Reference in New Issue
Block a user