Files
airflow/dags/stat/community/community_count_price_rent.py
T
2022-08-17 09:30:22 +08:00

123 lines
5.7 KiB
Python

"""
小区数量 租金 均价 及 覆盖率统计
"""
from goojia_utils import MAIN_MONGO_CLIENT, dateutils, ClickHouseClient
import pandas as pd
server_area_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'server_area')
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
district_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'district_stat')
sqmetre_rent_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'sqmetre_rent')
clickHouseClient = ClickHouseClient(host="192.168.0.216", database='fangjia_stat')
deal_price_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'deal_price')
city_df = pd.DataFrame(list(server_area_conn.find({}, {'city_grade': 1, 'display': 1})))
city_df = city_df.rename(columns={'city_grade': 'city_level', 'display': 'city'})
city_df = city_df[city_df['city_level'] != 0]
seaweed_df = pd.DataFrame(
list(seaweed_conn.find({'cat': 'district', 'status': 0}, {'city': 1, 'region': 1, 'name': 1, 'mapping_alias': 1})
.batch_size(10000)))
seaweed_df = seaweed_df.rename(columns={'mapping_alias': 'alias'})
# #### 1. 小区数量
result_df = pd.DataFrame(list(seaweed_conn.aggregate([
{'$match': {'cat': 'district', 'status': 0}},
{'$group': {'_id': "$city", 'community_count': {'$sum': 1}}}])))
result_df = result_df.rename(columns={'_id': 'city'})
df = pd.merge(result_df, city_df, how='right', on='city')
df.fillna(0, inplace=True)
# 有别名小区量
alias_df = pd.DataFrame(list(seaweed_conn.aggregate([
{'$match': {'cat': 'district', 'status': 0, 'mapping_alias': {'$ne': None}}},
{'$group': {'_id': "$city", 'alias_count': {'$sum': 1}}}])))
alias_df = alias_df.rename(columns={'_id': 'city'})
df = pd.merge(df, alias_df, how='left', on=['city'])
# 有地址别名小区量
addr_alias_count = pd.DataFrame(list(seaweed_conn.aggregate([
{'$match': {'cat': 'district', 'status': 0, 'addr_alias': {'$ne': None}}},
{'$group': {'_id': "$city", 'addr_alias_count': {'$sum': 1}}}])))
addr_alias_count = addr_alias_count.rename(columns={'_id': 'city'})
df = pd.merge(df, addr_alias_count, how='left', on=['city'])
##### 2. 近一年成交量
# 小区别名升级为单独df
actions = list()
for index, item in seaweed_df.iterrows():
if type(item['alias']) == list:
for name in item['alias']:
actions.append({'_id': item['_id'], 'city': item['city'], 'name': name, 'region': item['region']})
alias_df = pd.DataFrame(actions)
seaweed_alias_df = pd.concat([seaweed_df[['_id', 'city', 'region', 'name']], alias_df])
deal_price_df = pd.DataFrame(list(deal_price_conn.find(
{'trade_date': {'$gte': dateutils.add_months(dateutils.datetime.datetime.now(), days=-365)}},
{'city': 1, 'region': 1, 'district_name': 1, '_id': 0}).batch_size(10000)))
deal_price_df = deal_price_df.rename(columns={'district_name': 'name'})
deal_price_seaweed_df = pd.merge(deal_price_df, seaweed_alias_df, how='left', on=['city', 'region', 'name'])
# 剔除没有配到的小区
deal_price_seaweed_notnull_df = deal_price_seaweed_df.dropna(subset=['_id'])
deal_price_seaweed_notnull_df.loc[:, 'one_year_deal'] = deal_price_seaweed_notnull_df.groupby(['city'])[
'city'].transform('size')
deal_price_half_year_df = deal_price_seaweed_notnull_df[['city', 'one_year_deal']].drop_duplicates(keep='first')
df = pd.merge(df, deal_price_half_year_df, how='left', on=['city'])
# #### 2. 均价数量
week_df = pd.DataFrame(list(district_stat_conn.distinct('weekend',
{'weekend': {'$gte': dateutils.add_months(
dateutils.datetime.datetime.now(), days=-60)}})))
week = week_df.loc[len(week_df) - 1, 0]
avg_price_df = pd.DataFrame(list(district_stat_conn.aggregate([
{'$match': {'weekend': {'$gte': week}}},
{'$group': {'_id': '$city', 'num': {'$sum': 1}}}
])))
avg_price_df = avg_price_df.rename(columns={'_id': 'city', 'num': 'avg_price_count'})
df = pd.merge(df, avg_price_df, how='left', on=['city'])
df['avg_price_coverage'] = pd.to_numeric(df['avg_price_count'] * 100 / df['community_count'])
# #### 3.租金数量
week_df = pd.DataFrame(list(sqmetre_rent_conn.distinct('weekend',
{'weekend': {'$gte': dateutils.add_months(dateutils.datetime.datetime.now(), days=-60)}})))
week = week_df.loc[len(week_df) - 1, 0]
sqmetre_rent_df = pd.DataFrame(list(sqmetre_rent_conn.aggregate([
{'$match': {'weekend': {'$gte': week}, 'cat': 'district'}},
{'$group': {'_id': '$city', 'num': {'$sum': 1}}}
])))
sqmetre_rent_df = sqmetre_rent_df.rename(columns={'_id': 'city', 'num': 'sqmetre_rent_count'})
df = pd.merge(df, sqmetre_rent_df, how='left', on=['city'])
df['sqmetre_rent_coverage'] = pd.to_numeric(df['sqmetre_rent_count'] * 100 / df['community_count'])
df.fillna(0, inplace=True)
df['community_count'] = df['community_count'].astype(int)
df['avg_price_count'] = df['avg_price_count'].astype(int)
df['sqmetre_rent_count'] = df['sqmetre_rent_count'].astype(int)
df['one_year_deal'] = df['one_year_deal'].astype(int)
df['alias_count'] = df['alias_count'].astype(int)
df['addr_alias_count'] = df['addr_alias_count'].astype(int)
df = df.round({'avg_price_coverage': 2, 'sqmetre_rent_coverage': 2})
df = df[['city', 'community_count', 'city_level', 'avg_price_count', 'avg_price_coverage', 'sqmetre_rent_count', 'sqmetre_rent_coverage', 'alias_count', 'addr_alias_count', 'one_year_deal']]
df['weekend'] = dateutils.datetime.datetime.now()
# 删除本日生成的重复数据,防止手工点击生成
clickHouseClient.execute(
"ALTER TABLE fangjia_stat.community_coverage_stat DELETE WHERE weekend = toDate('" + dateutils.format_date(dateutils.datetime.datetime.now(), '%Y-%m-%d') + "')")
clickHouseClient.to_sql(df, table='community_coverage_stat', if_exists='insert', order_by='weekend')