优化统计

This commit is contained in:
shenjian@fangjia.com
2023-03-22 17:56:00 +08:00
parent ea6ae494cd
commit 0fe9d8f99a
@@ -17,33 +17,63 @@ city_df = city_df.rename(columns={'city_grade': 'city_level', 'display': 'city'}
city_df = city_df[city_df['city_level'] != 0]
seaweed_df = pd.DataFrame(
list(seaweed_conn.find({'cat': 'district', 'status': 0}, {'city': 1, 'region': 1, 'name': 1, 'mapping_alias': 1})
list(seaweed_conn.find({'cat': 'district', 'status': 0}, {'city': 1, 'region': 1, 'name': 1, 'alias': 1, 'addr_alias': 1})
.batch_size(10000)))
seaweed_df = seaweed_df.rename(columns={'mapping_alias': 'alias'})
path = "/opt/airflow/dags/repo/dags/business/business/community/data"
ret_df = pd.read_excel(path + '/财力评级分布情况20221215.xlsx')
for index, item in city_df.iterrows():
city = item['city']
region_df = seaweed_df[(seaweed_df['city'] == city)].copy()
alias_count = 0
addr_count = 0
seaweed_count = 0
# 楼盘数量
city_df.loc[index, 'community_count'] = len(region_df)
for index1, item1 in region_df.iterrows():
flag = True
if type(item1['alias']) == list:
if len(item1['alias']) > 0:
alias_count += len(item1['alias'])
seaweed_count += 1
flag = False
if type(item1['addr_alias']) == list:
if len(item1['addr_alias']) > 0:
addr_count += len(item1['addr_alias'])
if len(item1['addr_alias']) > 0 and flag is True:
seaweed_count += 1
# 楼盘别名总数
city_df.loc[index, 'alias_count'] = alias_count
# 地址别名数量
city_df.loc[index, 'addr_alias_count'] = addr_count
# 有别名小区量【存在楼盘别名或地址别名】
city_df.loc[index, 'exist_alias_community_count'] = seaweed_count
# 【已过期】
# #### 1. 小区数量
result_df = pd.DataFrame(list(seaweed_conn.aggregate([
{'$match': {'cat': 'district', 'status': 0}},
{'$group': {'_id': "$city", 'community_count': {'$sum': 1}}}])))
# result_df = pd.DataFrame(list(seaweed_conn.aggregate([
# {'$match': {'cat': 'district', 'status': 0}},
# {'$group': {'_id': "$city", 'community_count': {'$sum': 1}}}])))
#
#
# result_df = result_df.rename(columns={'_id': 'city'})
# df = pd.merge(result_df, city_df, how='right', on='city')
# df.fillna(0, inplace=True)
result_df = result_df.rename(columns={'_id': 'city'})
df = pd.merge(result_df, city_df, how='right', on='city')
df.fillna(0, inplace=True)
# 有别名小区量
alias_df = pd.DataFrame(list(seaweed_conn.aggregate([
{'$match': {'cat': 'district', 'status': 0, 'mapping_alias': {'$ne': None}}},
{'$group': {'_id': "$city", 'alias_count': {'$sum': 1}}}])))
alias_df = alias_df.rename(columns={'_id': 'city'})
df = pd.merge(df, alias_df, how='left', on=['city'])
# 有别名小区量 该方式统计不准,可能存在[]情况也算了
# alias_df = pd.DataFrame(list(seaweed_conn.aggregate([
# {'$match': {'cat': 'district', 'status': 0, 'mapping_alias': {'$ne': None}}},
# {'$group': {'_id': "$city", 'alias_count': {'$sum': 1}}}])))
# alias_df = alias_df.rename(columns={'_id': 'city'})
# df = pd.merge(df, alias_df, how='left', on=['city'])
# 有地址别名小区量
addr_alias_count = pd.DataFrame(list(seaweed_conn.aggregate([
{'$match': {'cat': 'district', 'status': 0, 'addr_alias': {'$ne': None}}},
{'$group': {'_id': "$city", 'addr_alias_count': {'$sum': 1}}}])))
addr_alias_count = addr_alias_count.rename(columns={'_id': 'city'})
df = pd.merge(df, addr_alias_count, how='left', on=['city'])
# addr_alias_count = pd.DataFrame(list(seaweed_conn.aggregate([
# {'$match': {'cat': 'district', 'status': 0, 'addr_alias': {'$ne': None}}},
# {'$group': {'_id': "$city", 'addr_alias_count': {'$sum': 1}}}])))
# addr_alias_count = addr_alias_count.rename(columns={'_id': 'city'})
# df = pd.merge(df, addr_alias_count, how='left', on=['city'])
##### 2. 近一年成交量
@@ -65,7 +95,7 @@ deal_price_seaweed_notnull_df = deal_price_seaweed_df.dropna(subset=['_id'])
deal_price_seaweed_notnull_df.loc[:, 'one_year_deal'] = deal_price_seaweed_notnull_df.groupby(['city'])[
'city'].transform('size')
deal_price_half_year_df = deal_price_seaweed_notnull_df[['city', 'one_year_deal']].drop_duplicates(keep='first')
df = pd.merge(df, deal_price_half_year_df, how='left', on=['city'])
df = pd.merge(city_df, deal_price_half_year_df, how='left', on=['city'])
# #### 2. 均价数量
@@ -109,9 +139,10 @@ df['sqmetre_rent_count'] = df['sqmetre_rent_count'].astype(int)
df['one_year_deal'] = df['one_year_deal'].astype(int)
df['alias_count'] = df['alias_count'].astype(int)
df['addr_alias_count'] = df['addr_alias_count'].astype(int)
df['exist_alias_community_count'] = df['exist_alias_community_count'].astype(int)
df = df.round({'avg_price_coverage': 2, 'sqmetre_rent_coverage': 2})
df = df[['city', 'community_count', 'city_level', 'avg_price_count', 'avg_price_coverage', 'sqmetre_rent_count', 'sqmetre_rent_coverage', 'alias_count', 'addr_alias_count', 'one_year_deal']]
df = df[['city', 'community_count', 'city_level', 'avg_price_count', 'avg_price_coverage', 'sqmetre_rent_count', 'sqmetre_rent_coverage', 'alias_count', 'addr_alias_count', 'exist_alias_community_count', 'one_year_deal']]
df['weekend'] = dateutils.datetime.datetime.now()
# 删除本日生成的重复数据,防止手工点击生成