diff --git a/dags/stat/community/__init__.py b/dags/stat/community/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dags/stat/community/community_count_price_rent.py b/dags/stat/community/community_count_price_rent.py index e94bd21..ec6c2c5 100644 --- a/dags/stat/community/community_count_price_rent.py +++ b/dags/stat/community/community_count_price_rent.py @@ -1,15 +1,5 @@ -#!/usr/bin/env python -# coding: utf-8 - -# In[1]: - - from goojia_utils import MAIN_MONGO_CLIENT, dateutils, ClickHouseClient import pandas as pd -import math - - -# In[2]: server_area_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'server_area') @@ -18,157 +8,66 @@ district_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'district_ sqmetre_rent_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'sqmetre_rent') clickHouseClient = ClickHouseClient(host="192.168.0.216", database='fangjia_stat') - -# In[3]: - - city_df = pd.DataFrame(list(server_area_conn.find({}, {'city_grade': 1, 'display': 1}))) city_df = city_df.rename(columns={'city_grade': 'city_level', 'display': 'city'}) city_df = city_df[city_df['city_level'] != 0] # #### 1. 小区数量 - -# In[4]: - - result_df = pd.DataFrame(list(seaweed_conn.aggregate([ {'$match': {'cat': 'district', 'status': 0}}, {'$group': {'_id': "$city", 'community_count': {'$sum': 1}}}]))) -# In[5]: - - result_df = result_df.rename(columns={'_id': 'city'}) - - -# In[6]: - - df = pd.merge(result_df, city_df, how='right', on='city') - - -# In[7]: - - df.fillna(0, inplace=True) # #### 2. 均价数量 -# In[9]: - - -week_df = pd.DataFrame(list(district_stat_conn.distinct('weekend', - {'weekend': {'$gte': dateutils.add_months(dateutils.datetime.datetime.now(), days=-60)}}))) +week_df = pd.DataFrame(list(district_stat_conn.distinct('weekend', + {'weekend': {'$gte': dateutils.add_months( + dateutils.datetime.datetime.now(), days=-60)}}))) week = week_df.loc[len(week_df) - 1, 0] - -# In[10]: - - avg_price_df = pd.DataFrame(list(district_stat_conn.aggregate([ - {'$match': {'weekend': {'$gte': week }}}, + {'$match': {'weekend': {'$gte': week}}}, {'$group': {'_id': '$city', 'num': {'$sum': 1}}} ]))) - -# In[11]: - - avg_price_df = avg_price_df.rename(columns={'_id': 'city', 'num': 'avg_price_count'}) -# In[12]: - - df = pd.merge(df, avg_price_df, how='left', on=['city']) - - -# In[13]: - - df['avg_price_coverage'] = pd.to_numeric(df['avg_price_count'] * 100 / df['community_count']) # #### 3.租金数量 -# In[14]: - - week_df = pd.DataFrame(list(sqmetre_rent_conn.distinct('weekend', {'weekend': {'$gte': dateutils.add_months(dateutils.datetime.datetime.now(), days=-60)}}))) week = week_df.loc[len(week_df) - 1, 0] - -# In[15]: - - sqmetre_rent_df = pd.DataFrame(list(sqmetre_rent_conn.aggregate([ {'$match': {'weekend': {'$gte': week}}}, {'$group': {'_id': '$city', 'num': {'$sum': 1}}} ]))) -# In[16]: - - sqmetre_rent_df = sqmetre_rent_df.rename(columns={'_id': 'city', 'num': 'sqmetre_rent_count'}) - - -# In[17]: - - df = pd.merge(df, sqmetre_rent_df, how='left', on=['city']) - - -# In[18]: - - df['sqmetre_rent_coverage'] = pd.to_numeric(df['sqmetre_rent_count'] * 100 / df['community_count']) - - -# In[19]: - - df.fillna(0, inplace=True) - -# In[20]: - - df['community_count'] = df['community_count'].astype(int) df['avg_price_count'] = df['avg_price_count'].astype(int) df['sqmetre_rent_count'] = df['sqmetre_rent_count'].astype(int) - -# In[21]: - - df = df.round({'avg_price_coverage': 2, 'sqmetre_rent_coverage': 2}) - - -# In[22]: - - df = df[['city', 'community_count', 'city_level', 'avg_price_count', 'avg_price_coverage', 'sqmetre_rent_count', 'sqmetre_rent_coverage']] - -# In[23]: - - df['weekend'] = dateutils.datetime.datetime.now() - - -# In[24]: - - clickHouseClient.to_sql(df, table='community_coverage_stat', if_exists='insert', order_by='weekend') -# In[ ]: - - - -