From 98017d34f08cd3c43cfcec394f76d7dd67177065 Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Wed, 29 Mar 2023 15:02:18 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=AD=97=E6=AE=B5=E7=BB=9F?= =?UTF-8?q?=E8=AE=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/db/fangjia_stat.community_coverage_stat | 25 +++++ .../community/community_count_price_rent.py | 98 ++++++++++++++++--- 2 files changed, 112 insertions(+), 11 deletions(-) create mode 100644 dags/db/fangjia_stat.community_coverage_stat diff --git a/dags/db/fangjia_stat.community_coverage_stat b/dags/db/fangjia_stat.community_coverage_stat new file mode 100644 index 0000000..16523e9 --- /dev/null +++ b/dags/db/fangjia_stat.community_coverage_stat @@ -0,0 +1,25 @@ +create table fangjia_stat.community_coverage_stat +( + city String, + city_level Int32, + community_count Int32, + avg_price_count Int32, + avg_price_coverage Float32, + sqmetre_rent_count Int32, + sqmetre_rent_coverage Float32, + weekend Date, + alias_count Nullable(Int32), + addr_alias_count Nullable(Int32), + one_year_deal Nullable(Int32), + exist_alias_community_count Nullable(Int32), + main_type_coverage Nullable(Float32), + main_type_area_coverage Nullable(Float32), + estate_charge_coverage Nullable(Float32), + finish_building_date_coverage Nullable(Float32), + all_area_coverage Nullable(Float32), + build_area_coverage Nullable(Float32), + greening_rate_coverage Nullable(Float32), + volume_rate_coverage Nullable(Float32), + consumption_rating_coverage Nullable(Float32) +) + engine = Memory; \ No newline at end of file diff --git a/dags/stat/community/community_count_price_rent.py b/dags/stat/community/community_count_price_rent.py index a4bc549..9138b2d 100644 --- a/dags/stat/community/community_count_price_rent.py +++ b/dags/stat/community/community_count_price_rent.py @@ -2,7 +2,7 @@ 小区数量 租金 均价 及 覆盖率统计 """ -from goojia_utils import MAIN_MONGO_CLIENT, dateutils, ClickHouseClient +from goojia_utils import MAIN_MONGO_CLIENT, dateutils, ClickHouseClient, ElasticSearchClient import pandas as pd import os @@ -13,6 +13,8 @@ sqmetre_rent_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'sqmetre_rent') clickHouseClient = ClickHouseClient(host="192.168.0.216", database='fangjia_stat') deal_price_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'deal_price') +es206_client = ElasticSearchClient(host='192.168.0.206') + city_df = pd.DataFrame(list(server_area_conn.find({}, {'city_grade': 1, 'display': 1}))) city_df = city_df.rename(columns={'city_grade': 'city_level', 'display': 'city'}) city_df = city_df[city_df['city_level'] != 0] @@ -74,12 +76,82 @@ for index, item in city_df.iterrows(): # alias_df = alias_df.rename(columns={'_id': 'city'}) # df = pd.merge(df, alias_df, how='left', on=['city']) -# 有地址别名小区量 -# addr_alias_count = pd.DataFrame(list(seaweed_conn.aggregate([ -# {'$match': {'cat': 'district', 'status': 0, 'addr_alias': {'$ne': None}}}, -# {'$group': {'_id': "$city", 'addr_alias_count': {'$sum': 1}}}]))) -# addr_alias_count = addr_alias_count.rename(columns={'_id': 'city'}) -# df = pd.merge(df, addr_alias_count, how='left', on=['city']) +df = city_df.copy() + +# 主力户型 +main_type_count = pd.DataFrame(list(seaweed_conn.aggregate([ + {'$match': {'cat': 'district', 'status': 0, 'main_type': {'$ne': None}}}, + {'$group': {'_id': "$city", 'main_type_count': {'$sum': 1}}}]))) +main_type_count = main_type_count.rename(columns={'_id': 'city'}) +df = pd.merge(df, main_type_count, how='left', on=['city']) +df['main_type_coverage'] = df['main_type_count'] * 100 / df['community_count'] +# 主力面积 +main_type_area_count = pd.DataFrame(list(seaweed_conn.aggregate([ + {'$match': {'cat': 'district', 'status': 0, 'main_type_area': {'$ne': None}}}, + {'$group': {'_id': "$city", 'main_type_area_count': {'$sum': 1}}}]))) +main_type_area_count = main_type_area_count.rename(columns={'_id': 'city'}) +df = pd.merge(df, main_type_area_count, how='left', on=['city']) +df['main_type_area_coverage'] = df['main_type_area_count'] * 100 / df['community_count'] +# 物业类型 +estate_type_count = pd.DataFrame(list(seaweed_conn.aggregate([ + {'$match': {'cat': 'district', 'status': 0, 'estate_type2': {'$ne': None}}}, + {'$group': {'_id': "$city", 'estate_charge_count': {'$sum': 1}}}]))) +estate_type_count = estate_type_count.rename(columns={'_id': 'city'}) +df = pd.merge(df, estate_type_count, how='left', on=['city']) +df['estate_type_coverage'] = df['estate_type_count'] * 100 / df['community_count'] +# 建成年代 +finish_building_date_count = pd.DataFrame(list(seaweed_conn.aggregate([ + {'$match': {'cat': 'district', 'status': 0, 'finish_building_date': {'$ne': None}}}, + {'$group': {'_id': "$city", 'finish_building_date_count': {'$sum': 1}}}]))) +finish_building_date_count = finish_building_date_count.rename(columns={'_id': 'city'}) +df = pd.merge(df, finish_building_date_count, how='left', on=['city']) +df['finish_building_date_coverage'] = df['finish_building_date_count'] * 100 / df['community_count'] +# 占地面积 +all_area_count = pd.DataFrame(list(seaweed_conn.aggregate([ + {'$match': {'cat': 'district', 'status': 0, 'all_area': {'$ne': None}}}, + {'$group': {'_id': "$city", 'all_area_count': {'$sum': 1}}}]))) +all_area_count = all_area_count.rename(columns={'_id': 'city'}) +df = pd.merge(df, all_area_count, how='left', on=['city']) +df['all_area_coverage'] = df['all_area_count'] * 100 / df['community_count'] +# 建筑面积 +build_area_count = pd.DataFrame(list(seaweed_conn.aggregate([ + {'$match': {'cat': 'district', 'status': 0, 'build_area': {'$ne': None}}}, + {'$group': {'_id': "$city", 'build_area_count': {'$sum': 1}}}]))) +build_area_count = build_area_count.rename(columns={'_id': 'city'}) +df = pd.merge(df, build_area_count, how='left', on=['city']) +df['build_area_coverage'] = df['build_area_count'] * 100 / df['community_count'] + +# 绿化率 +greening_rate_count = pd.DataFrame(list(seaweed_conn.aggregate([ + {'$match': {'cat': 'district', 'status': 0, 'virescence_percent': {'$ne': None}}}, + {'$group': {'_id': "$city", 'greening_rate_count': {'$sum': 1}}}]))) +greening_rate_count = greening_rate_count.rename(columns={'_id': 'city'}) +df = pd.merge(df, greening_rate_count, how='left', on=['city']) +df['greening_rate_coverage'] = df['greening_rate_count'] * 100 / df['community_count'] +# 容积率 +volume_rate_count = pd.DataFrame(list(seaweed_conn.aggregate([ + {'$match': {'cat': 'district', 'status': 0, 'cubage_percent': {'$ne': None}}}, + {'$group': {'_id': "$city", 'volume_rate_count': {'$sum': 1}}}]))) +volume_rate_count = volume_rate_count.rename(columns={'_id': 'city'}) +df = pd.merge(df, volume_rate_count, how='left', on=['city']) +df['volume_rate_coverage'] = df['volume_rate_count'] * 100 / df['community_count'] + +# 户均总价 +df['total_price_count'] = None +# 财力评级 +df['consumption_rating_count'] = None +for index, item in df.iterrows(): + curr_city = item['city'] + sql = f"select count(*) from community_summary where city='{curr_city}' and total_price>0 " + num = es206_client.search_by_sql(sql)['hits']['total']['value'] + df.loc[index, 'total_price_count'] = num + + sql = f"select count(*) from community_summary where city='{curr_city}' and consumption_rating>0 " + consumption_rating_num = es206_client.search_by_sql(sql)['hits']['total']['value'] + df.loc[index, 'consumption_rating_count'] = consumption_rating_num + +df['total_price_coverage'] = df['total_price_count'] * 100 / df['community_count'] +df['consumption_rating_coverage'] = df['consumption_rating_count'] * 100 / df['community_count'] ##### 2. 近一年成交量 @@ -101,7 +173,7 @@ deal_price_seaweed_notnull_df = deal_price_seaweed_df.dropna(subset=['_id']) deal_price_seaweed_notnull_df.loc[:, 'one_year_deal'] = deal_price_seaweed_notnull_df.groupby(['city'])[ 'city'].transform('size') deal_price_half_year_df = deal_price_seaweed_notnull_df[['city', 'one_year_deal']].drop_duplicates(keep='first') -df = pd.merge(city_df, deal_price_half_year_df, how='left', on=['city']) +df = pd.merge(df, deal_price_half_year_df, how='left', on=['city']) # #### 2. 均价数量 @@ -147,8 +219,11 @@ df['alias_count'] = df['alias_count'].astype(int) df['addr_alias_count'] = df['addr_alias_count'].astype(int) df['exist_alias_community_count'] = df['exist_alias_community_count'].astype(int) -df = df.round({'avg_price_coverage': 2, 'sqmetre_rent_coverage': 2}) -df = df[['city', 'community_count', 'city_level', 'avg_price_count', 'avg_price_coverage', 'sqmetre_rent_count', 'sqmetre_rent_coverage', 'alias_count', 'addr_alias_count', 'exist_alias_community_count', 'one_year_deal']] +df = df.round({'avg_price_coverage': 2, 'sqmetre_rent_coverage': 2, 'main_type_coverage': 2, 'main_type_area_coverage': 2, 'total_price_coverage': 2, 'estate_type_coverage': 2, + 'finish_building_date_coverage': 2, 'all_area_coverage': 2, 'build_area_coverage': 2, 'consumption_rating_coverage': 2, 'greening_rate_coverage': 2, 'volume_rate_coverage': 2}) +df = df[['city', 'community_count', 'city_level', 'avg_price_count', 'avg_price_coverage', 'sqmetre_rent_count', 'sqmetre_rent_coverage', 'alias_count', 'addr_alias_count', + 'exist_alias_community_count', 'one_year_deal', 'main_type_coverage', 'main_type_area_coverage', 'total_price_coverage', 'estate_type_coverage', + 'finish_building_date_coverage', 'all_area_coverage', 'build_area_coverage', 'consumption_rating_coverage', 'greening_rate_coverage', 'volume_rate_coverage']] df['weekend'] = dateutils.datetime.datetime.now() # 删除本日生成的重复数据,防止手工点击生成 @@ -159,7 +234,8 @@ clickHouseClient.to_sql(df, table='community_coverage_stat', if_exists='insert', # 上传至nextcloud ret_df = pd.merge(ret_df, df, how='left', on=['city']) del ret_df['是否合理'] -ret_df = ret_df.rename(columns={'city': '城市', 'city_level': '城市级别', 'community_count': "楼盘数量", "alias_count": "别名数量", "addr_alias_count": "地址别名数量", "exist_alias_community_count": "有别名覆盖小区数量"}) +ret_df = ret_df.rename(columns={'city': '城市', 'city_level': '城市级别', 'community_count': "楼盘数量", "alias_count": "别名数量", "addr_alias_count": "地址别名数量", "exist_alias_community_count": "有别名覆盖小区数量", + 'estate_type_coverage': '物业类型覆盖率', 'avg_price_coverage': '均价覆盖率', 'sqmetre_rent_coverage': '平米租金覆盖率', 'total_price_coverage': '户均总价覆盖率', 'main_type_coverage': '主力户型覆盖率', 'main_type_area_coverage': '主力面积覆盖率', 'finish_building_date_coverage': '建成年代覆盖率', 'all_area_coverage': '占地面积覆盖率', 'build_area_coverage': '建筑面积覆盖率', 'consumption_rating_coverage': '财力评级覆盖率', 'greening_rate_coverage': '绿化率覆盖率', 'volume_rate_coverage': '容积率覆盖率'}) output_date = dateutils.format_date(dateutils.datetime.datetime.now()) aim_file_name = aim_path + "/" + output_date + '覆盖率统计.xlsx' ret_df.to_excel(aim_file_name, index=False)