From 7c99f1b0ac958818ae7b61898a236e84b13a97da Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Mon, 20 Mar 2023 11:38:48 +0800 Subject: [PATCH] =?UTF-8?q?=E7=BE=8E=E5=9B=A2=E4=B8=AD=E9=97=B4=E8=A1=A8?= =?UTF-8?q?=E6=94=B9=E9=80=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/db/tmp.mt_community | 44 ++++++++++++++++++++++++++++++++ dags/mt/supply/comm.py | 11 +++++--- dags/mt/supply/data_stat.py | 3 ++- dags/mt/supply/deal.py | 13 +++++++--- dags/mt/supply/facilitys.py | 11 +++++--- dags/mt/supply/rent_avg_price.py | 14 +++++++--- 6 files changed, 82 insertions(+), 14 deletions(-) create mode 100644 dags/db/tmp.mt_community diff --git a/dags/db/tmp.mt_community b/dags/db/tmp.mt_community new file mode 100644 index 0000000..e8a4416 --- /dev/null +++ b/dags/db/tmp.mt_community @@ -0,0 +1,44 @@ +CREATE TABLE tmp.mt_community +( + id String COMMENT '小区ID', + city String COMMENT '城市', + region String COMMENT '区域', + name String COMMENT '名称', + alias String COMMENT '别名', + address String COMMENT '地址', + addr_alias String COMMENT '地址别名', + avg_price Nullable(INT) COMMENT '均价', + rent_price Nullable(INT) COMMENT '平米租金', + lng Nullable(Float32) COMMENT '经度', + lat Nullable(Float32) COMMENT '纬度', + estate_type2 String COMMENT '物业类型', + block String COMMENT '板块', + loop_line_info String COMMENT '环线', + business_quarter String COMMENT '商圈', + households String COMMENT '户数', + property_fee String COMMENT '物业费', + car_site_count String COMMENT '停车位', + property_life String COMMENT '产权年限', + house_property String COMMENT '产权性质', + all_area String COMMENT '占地面积', + build_area String COMMENT '建筑面积', + plot_ratio String COMMENT '容积率', + landscaping_ratio String COMMENT '绿化率', + property_type String COMMENT '物业类型', + build_type String COMMENT '建筑类型', + build_structure String COMMENT '建筑结构', + finish_building_date String COMMENT '建成年代', + developer String COMMENT '开发商', + house_type String COMMENT '户型', + house_area String COMMENT '房型建筑面积', + deal_num Nullable(INT) COMMENT '近一年成交量', + district_facilitys String COMMENT '周边配套数量字段', + school Nullable(INT) COMMENT '学校数量', + market Nullable(INT) COMMENT '商场数量', + subway Nullable(INT) COMMENT '地铁数量', + bank Nullable(INT) COMMENT '银行数量', + hospital Nullable(INT) COMMENT '医院数量', + status INT COMMENT '0 正常 1 删除', + m_date DateTime('Asia/Shanghai') default now() COMMENT '更新时间' +) ENGINE ReplacingMergeTree(m_date) +PARTITION BY city PRIMARY KEY (city, region, name) ORDER BY (city, region, name); \ No newline at end of file diff --git a/dags/mt/supply/comm.py b/dags/mt/supply/comm.py index 65acaf4..50daf40 100644 --- a/dags/mt/supply/comm.py +++ b/dags/mt/supply/comm.py @@ -1,13 +1,14 @@ """ 小区属性导出 """ -from goojia_utils import MAIN_MONGO_CLIENT, dateutils, commutils +from goojia_utils import MAIN_MONGO_CLIENT, dateutils, commutils, ClickHouseClient import pandas as pd import datetime import os import sys seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') +tmp_ck = ClickHouseClient(database='tmp') path = "/home/airflow/data/mt" if not os.path.exists(path): @@ -74,5 +75,9 @@ seaweed_new_result_df = seaweed_new_result_df.rename(columns={'_id': 'id', 'mapp 'cubage_percent' : 'plot_ratio', 'virescence_percent': 'landscaping_ratio', 'estate_type2': 'property_type', 'main_type': 'house_type', 'main_type_area': 'house_area'}) -seaweed_new_result_df.to_csv(path + '/FJW_COMM_' + output_date + '.csv', - sep='@', encoding='utf-8', index=False) +seaweed_new_result_df.to_csv(path + '/FJW_COMM_' + output_date + '.csv', sep='@', encoding='utf-8', index=False) + +ck_delete_sql = "ALTER TABLE tmp.mt_community DELETE WHERE 1=1 " +tmp_ck.execute(ck_delete_sql) + +tmp_ck.to_sql_plus(seaweed_new_result_df, 'mt_community') diff --git a/dags/mt/supply/data_stat.py b/dags/mt/supply/data_stat.py index 71ea78a..c1d5227 100644 --- a/dags/mt/supply/data_stat.py +++ b/dags/mt/supply/data_stat.py @@ -10,11 +10,12 @@ import requests path = "/home/airflow/data/mt" clickhouse_client = ClickHouseClient(database='fangjia_stat') +tmp_ck = ClickHouseClient(database='tmp') output_date = sys.argv[1] if len(sys.argv) > 1 else dateutils.format_date(datetime.datetime.now(), '%Y%m') print("输出日期" + output_date) -result_df = pd.read_csv(path + '/FJW_COMM_' + output_date + ".csv", sep='@', encoding='utf-8') +result_df = tmp_ck.read_sql_query("select * from tmp.mt_community") city_df = pd.read_excel('/opt/airflow/dags/repo/dags/mt/supply/data/美团88城市列表.xlsx') cities = city_df['城市列表'].values.tolist() diff --git a/dags/mt/supply/deal.py b/dags/mt/supply/deal.py index e1f2b14..4ae707f 100644 --- a/dags/mt/supply/deal.py +++ b/dags/mt/supply/deal.py @@ -1,7 +1,7 @@ """ 每个小区近一年成交统计 """ -from goojia_utils import MAIN_MONGO_CLIENT, dateutils, ElasticSearchClient +from goojia_utils import MAIN_MONGO_CLIENT, dateutils, ElasticSearchClient, ClickHouseClient import pandas as pd import datetime import logging @@ -11,6 +11,7 @@ import sys es45_client = ElasticSearchClient(host='192.168.0.45') deal_price_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'deal_price') district_facilitys_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'district_facilitys') +tmp_ck = ClickHouseClient(database='tmp') city_df = pd.read_excel('/opt/airflow/dags/repo/dags/mt/supply/data/美团88城市列表.xlsx') cities = city_df['城市列表'].values.tolist() @@ -26,7 +27,7 @@ logging.info("日期信息:" + month_start + ' ' + curr_month_start) month_start_date = dateutils.parse_date(month_start, '%Y%m%d') curr_month_start_date = dateutils.parse_date(curr_month_start, '%Y%m%d') -result_df = pd.read_csv(path + '/FJW_COMM_' + output_date + '.csv', sep='@', encoding='utf-8') +result_df = tmp_ck.read_sql_query("select * from mt_community") if 'deal_num' in result_df.columns: result_df = result_df.drop(columns=['deal_num']) @@ -93,4 +94,10 @@ columns = ['id', 'city', 'region', 'name', 'alias', 'address', 'addr_alias', 'av 'build_structure', 'finish_building_date', 'developer', 'house_type', 'house_area', 'deal_num', 'district_facilitys', 'school', 'market', 'subway', 'bank', 'hospital', 'status'] -result_df.to_csv(path + '/FJW_COMM_' + output_date + '.csv', sep='@', encoding='utf-8', index=False, columns=columns) \ No newline at end of file + +result_save_df = result_df[columns] + +ck_delete_sql = "ALTER TABLE tmp.mt_community DELETE WHERE 1=1 " +tmp_ck.execute(ck_delete_sql) + +tmp_ck.to_sql_plus(result_save_df, 'mt_community') \ No newline at end of file diff --git a/dags/mt/supply/facilitys.py b/dags/mt/supply/facilitys.py index 409ee42..81ef498 100644 --- a/dags/mt/supply/facilitys.py +++ b/dags/mt/supply/facilitys.py @@ -1,13 +1,14 @@ """ 周边配套数量统计 """ -from goojia_utils import MAIN_MONGO_CLIENT, dateutils +from goojia_utils import MAIN_MONGO_CLIENT, dateutils, ClickHouseClient import pandas as pd import datetime import logging import sys district_facilitys_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'district_facilitys') +tmp_ck = ClickHouseClient(database='tmp') city_df = pd.read_excel('/opt/airflow/dags/repo/dags/mt/supply/data/美团88城市列表.xlsx') cities = city_df['城市列表'].values.tolist() @@ -73,7 +74,8 @@ district_facilitys_new_df = district_facilitys_df[ district_facilitys_new_df = district_facilitys_new_df.rename(columns={'district_name': 'name', 'supermarket': 'market'}) district_facilitys_new_df = district_facilitys_new_df.drop_duplicates(subset=['city', 'region', 'name'], keep='first') -result_df = pd.read_csv(path + '/FJW_COMM_' + output_date + '.csv', sep='@', encoding='utf-8') +result_df = tmp_ck.read_sql_query("select * from mt_community") + if 'school' in result_df.columns: result_df = result_df.drop(columns=['school', 'bank', 'hospital', 'market', 'subway']) @@ -141,4 +143,7 @@ result_df['bank'].fillna(0, inplace=True) result_df['hospital'].fillna(0, inplace=True) result_df['subway'].fillna(0, inplace=True) -result_df.to_csv(path + '/FJW_COMM_' + output_date + '.csv', sep='@', encoding='utf-8', index=False) +ck_delete_sql = "ALTER TABLE tmp.mt_community DELETE WHERE 1=1 " +tmp_ck.execute(ck_delete_sql) + +tmp_ck.to_sql_plus(result_df, 'mt_community') \ No newline at end of file diff --git a/dags/mt/supply/rent_avg_price.py b/dags/mt/supply/rent_avg_price.py index 1ca4200..a9ed0de 100644 --- a/dags/mt/supply/rent_avg_price.py +++ b/dags/mt/supply/rent_avg_price.py @@ -2,17 +2,17 @@ 租金、均价导出 """ -from goojia_utils import MAIN_MONGO_CLIENT, dateutils, commutils +from goojia_utils import MAIN_MONGO_CLIENT, dateutils, commutils, ClickHouseClient import pandas as pd import datetime import logging import sys -import numpy as np #district_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'district_stat') # 月度均价表 district_sell_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'district_sell_stat') sqmetre_rent_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'sqmetre_rent') +tmp_ck = ClickHouseClient(database='tmp') city_df = pd.read_excel('/opt/airflow/dags/repo/dags/mt/supply/data/美团88城市列表.xlsx') @@ -22,7 +22,8 @@ path = "/home/airflow/data/mt" output_date = sys.argv[1] if len(sys.argv) > 1 else dateutils.format_date(datetime.datetime.now(), '%Y%m') -seaweed_new_result_df = pd.read_csv(path + '/FJW_COMM_' + output_date + '.csv', sep='@', encoding='utf-8') +seaweed_new_result_df = tmp_ck.read_sql_query("select * from mt_community") + if 'avg_price' in seaweed_new_result_df: seaweed_new_result_df = seaweed_new_result_df.drop(columns=['avg_price', 'rent_price']) @@ -111,4 +112,9 @@ result_df['ignore_comm'] = None result_df = result_df.apply(lambda x: format_rent(x), axis=1) result_df = result_df[result_df['ignore_comm'] != 1] -result_df.to_csv(path + '/FJW_COMM_' + output_date + '.csv', sep='@', encoding='utf-8', index=False) +del result_df['ignore_comm'] + +ck_delete_sql = "ALTER TABLE tmp.mt_community DELETE WHERE 1=1 " +tmp_ck.execute(ck_delete_sql) + +tmp_ck.to_sql_plus(result_df, 'mt_community')