diff --git a/dags/db/etl.company b/dags/db/etl.company index 95486bf..f813585 100644 --- a/dags/db/etl.company +++ b/dags/db/etl.company @@ -1,4 +1,5 @@ CREATE TABLE etl.company ( + id Int64, district String, ent_name String, reg_addr String, @@ -9,4 +10,31 @@ SETTINGS connection_wait_timeout=9999999, connect_timeout=9999999, read_write_timeout=9999999, - connection_auto_close=false; \ No newline at end of file + connection_auto_close=false; + +CREATE TABLE etl.dwd_company ( + district String, + ent_name String, + reg_addr String, + unified_code String, + authority String, + m_date DateTime('Asia/Shanghai') default now() COMMENT '更新时间' +) ENGINE ReplacingMergeTree(m_date) +PARTITION BY district PRIMARY KEY (unified_code) ORDER BY (unified_code); + +CREATE TABLE etl.dwm_company ( + district String, + ent_name String, + reg_addr String, + unified_code String, + authority String, + m_date DateTime('Asia/Shanghai') default now() COMMENT '更新时间' +) ENGINE ReplacingMergeTree(m_date) +PARTITION BY district PRIMARY KEY (unified_code) ORDER BY (unified_code); + +INSERT INTO etl.dwm_company( + district String, + ent_name String, + reg_addr String, + unified_code String, + authority String) SELECT * FROM etl.company \ No newline at end of file diff --git a/dags/db/etl.dwm_company b/dags/db/etl.dwm_company deleted file mode 100644 index 9e8644e..0000000 --- a/dags/db/etl.dwm_company +++ /dev/null @@ -1,16 +0,0 @@ -CREATE TABLE etl.dwm_company ( - district String, - ent_name String, - reg_addr String, - unified_code String, - authority String, - m_date DateTime('Asia/Shanghai') default now() COMMENT '更新时间' -) ENGINE ReplacingMergeTree(m_date) -PARTITION BY district PRIMARY KEY (unified_code) ORDER BY (unified_code); - -INSERT INTO etl.dwm_company( - district String, - ent_name String, - reg_addr String, - unified_code String, - authority String) SELECT * FROM etl.company \ No newline at end of file diff --git a/dags/stat/community/community_count_price_rent.py b/dags/stat/community/community_count_price_rent.py index 2568756..08ba153 100644 --- a/dags/stat/community/community_count_price_rent.py +++ b/dags/stat/community/community_count_price_rent.py @@ -146,6 +146,10 @@ df['volume_rate_coverage'] = df['volume_rate_count'] * 100 / df['community_count # 户均总价 df['total_price_count'] = None +# 物业费 +df['estate_charge_count'] = None +# 人均消费 +df['avg_cost_count'] = None # 财力评级 df['consumption_rating_count'] = None for index, item in df.iterrows(): @@ -154,12 +158,22 @@ for index, item in df.iterrows(): num = es206_client.search_by_sql(sql)['hits']['total']['value'] df.loc[index, 'total_price_count'] = num + sql = f"select count(*) from community_summary where city='{curr_city}' and estate_charge>0 " + estate_charge_num = es206_client.search_by_sql(sql)['hits']['total']['value'] + df.loc[index, 'estate_charge_count'] = estate_charge_num + + sql = f"select count(*) from community_summary where city='{curr_city}' and avg_cost>0 " + avg_cost_count = es206_client.search_by_sql(sql)['hits']['total']['value'] + df.loc[index, 'avg_cost_count'] = avg_cost_count + sql = f"select count(*) from community_summary where city='{curr_city}' and consumption_rating>0 " consumption_rating_num = es206_client.search_by_sql(sql)['hits']['total']['value'] df.loc[index, 'consumption_rating_count'] = consumption_rating_num df['total_price_coverage'] = df['total_price_count'] * 100 / df['community_count'] df['consumption_rating_coverage'] = df['consumption_rating_count'] * 100 / df['community_count'] +df['estate_charge_coverage'] = df['estate_charge_count'] * 100 / df['community_count'] +df['avg_cost_coverage'] = df['avg_cost_count'] * 100 / df['community_count'] ##### 2. 近一年成交量 @@ -212,10 +226,10 @@ df['addr_alias_count'] = df['addr_alias_count'].astype(int) df['exist_alias_community_count'] = df['exist_alias_community_count'].astype(int) df = df.round({'avg_price_coverage': 2, 'sqmetre_rent_coverage': 2, 'main_type_coverage': 2, 'main_type_area_coverage': 2, 'total_price_coverage': 2, 'estate_type_coverage': 2, - 'finish_building_date_coverage': 2, 'all_area_coverage': 2, 'build_area_coverage': 2, 'consumption_rating_coverage': 2, 'greening_rate_coverage': 2, 'volume_rate_coverage': 2}) + 'finish_building_date_coverage': 2, 'all_area_coverage': 2, 'build_area_coverage': 2, 'estate_charge_coverage': 2, 'avg_cost_coverage': 2, 'consumption_rating_coverage': 2, 'greening_rate_coverage': 2, 'volume_rate_coverage': 2}) df = df[['city', 'community_count', 'city_level', 'avg_price_coverage', 'sqmetre_rent_coverage', 'alias_count', 'addr_alias_count', 'exist_alias_community_count', 'one_year_deal', 'main_type_coverage', 'main_type_area_coverage', 'total_price_coverage', 'estate_type_coverage', - 'finish_building_date_coverage', 'all_area_coverage', 'build_area_coverage', 'consumption_rating_coverage', 'greening_rate_coverage', 'volume_rate_coverage']] + 'finish_building_date_coverage', 'all_area_coverage', 'build_area_coverage', 'estate_charge_coverage', 'avg_cost_coverage', 'consumption_rating_coverage', 'greening_rate_coverage', 'volume_rate_coverage']] df['weekend'] = dateutils.datetime.datetime.now() # 删除本日生成的重复数据,防止手工点击生成 @@ -227,7 +241,7 @@ clickHouseClient.to_sql(df, table='community_coverage_stat', if_exists='insert', ret_df = pd.merge(ret_df, df, how='left', on=['city']) del ret_df['是否合理'] ret_df = ret_df.rename(columns={'city': '城市', 'city_level': '城市级别', 'community_count': "楼盘数量", "alias_count": "别名数量", "addr_alias_count": "地址别名数量", "exist_alias_community_count": "有别名覆盖小区数量", - 'estate_type_coverage': '物业类型覆盖率', 'avg_price_coverage': '均价覆盖率', 'sqmetre_rent_coverage': '平米租金覆盖率', 'total_price_coverage': '户均总价覆盖率', 'main_type_coverage': '主力户型覆盖率', 'main_type_area_coverage': '主力面积覆盖率', 'finish_building_date_coverage': '建成年代覆盖率', 'all_area_coverage': '占地面积覆盖率', 'build_area_coverage': '建筑面积覆盖率', 'consumption_rating_coverage': '财力评级覆盖率', 'greening_rate_coverage': '绿化率覆盖率', 'volume_rate_coverage': '容积率覆盖率'}) + 'estate_type_coverage': '物业类型覆盖率', 'avg_price_coverage': '均价覆盖率', 'sqmetre_rent_coverage': '平米租金覆盖率', 'total_price_coverage': '户均总价覆盖率', 'estate_charge_coverage': '物业费覆盖率', 'avg_cost_coverage': '人均消费覆盖率', 'main_type_coverage': '主力户型覆盖率', 'main_type_area_coverage': '主力面积覆盖率', 'finish_building_date_coverage': '建成年代覆盖率', 'all_area_coverage': '占地面积覆盖率', 'build_area_coverage': '建筑面积覆盖率', 'consumption_rating_coverage': '财力评级覆盖率', 'greening_rate_coverage': '绿化率覆盖率', 'volume_rate_coverage': '容积率覆盖率'}) output_date = dateutils.format_date(dateutils.datetime.datetime.now()) aim_file_name = aim_path + "/" + output_date + '覆盖率统计.xlsx' ret_df.to_excel(aim_file_name, index=False)