diff --git a/.airflowignore b/.airflowignore index 1e7a37b..661b2a7 100644 --- a/.airflowignore +++ b/.airflowignore @@ -1,3 +1,4 @@ poi/business/.* stat/evaluate/* +stat/community/* mt/supply/* \ No newline at end of file diff --git a/dags/stat/community/community_count_price_rent.py b/dags/stat/community/community_count_price_rent.py new file mode 100644 index 0000000..e94bd21 --- /dev/null +++ b/dags/stat/community/community_count_price_rent.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python +# coding: utf-8 + +# In[1]: + + +from goojia_utils import MAIN_MONGO_CLIENT, dateutils, ClickHouseClient +import pandas as pd +import math + + +# In[2]: + + +server_area_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'server_area') +seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') +district_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'district_stat') +sqmetre_rent_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'sqmetre_rent') +clickHouseClient = ClickHouseClient(host="192.168.0.216", database='fangjia_stat') + + +# In[3]: + + +city_df = pd.DataFrame(list(server_area_conn.find({}, {'city_grade': 1, 'display': 1}))) +city_df = city_df.rename(columns={'city_grade': 'city_level', 'display': 'city'}) +city_df = city_df[city_df['city_level'] != 0] + + +# #### 1. 小区数量 + +# In[4]: + + +result_df = pd.DataFrame(list(seaweed_conn.aggregate([ + {'$match': {'cat': 'district', 'status': 0}}, + {'$group': {'_id': "$city", 'community_count': {'$sum': 1}}}]))) + + +# In[5]: + + +result_df = result_df.rename(columns={'_id': 'city'}) + + +# In[6]: + + +df = pd.merge(result_df, city_df, how='right', on='city') + + +# In[7]: + + +df.fillna(0, inplace=True) + + +# #### 2. 均价数量 + +# In[9]: + + +week_df = pd.DataFrame(list(district_stat_conn.distinct('weekend', + {'weekend': {'$gte': dateutils.add_months(dateutils.datetime.datetime.now(), days=-60)}}))) +week = week_df.loc[len(week_df) - 1, 0] + + +# In[10]: + + +avg_price_df = pd.DataFrame(list(district_stat_conn.aggregate([ + {'$match': {'weekend': {'$gte': week }}}, + {'$group': {'_id': '$city', 'num': {'$sum': 1}}} +]))) + + +# In[11]: + + +avg_price_df = avg_price_df.rename(columns={'_id': 'city', 'num': 'avg_price_count'}) + + +# In[12]: + + +df = pd.merge(df, avg_price_df, how='left', on=['city']) + + +# In[13]: + + +df['avg_price_coverage'] = pd.to_numeric(df['avg_price_count'] * 100 / df['community_count']) + + +# #### 3.租金数量 + +# In[14]: + + +week_df = pd.DataFrame(list(sqmetre_rent_conn.distinct('weekend', + {'weekend': {'$gte': dateutils.add_months(dateutils.datetime.datetime.now(), days=-60)}}))) +week = week_df.loc[len(week_df) - 1, 0] + + +# In[15]: + + +sqmetre_rent_df = pd.DataFrame(list(sqmetre_rent_conn.aggregate([ + {'$match': {'weekend': {'$gte': week}}}, + {'$group': {'_id': '$city', 'num': {'$sum': 1}}} +]))) + + +# In[16]: + + +sqmetre_rent_df = sqmetre_rent_df.rename(columns={'_id': 'city', 'num': 'sqmetre_rent_count'}) + + +# In[17]: + + +df = pd.merge(df, sqmetre_rent_df, how='left', on=['city']) + + +# In[18]: + + +df['sqmetre_rent_coverage'] = pd.to_numeric(df['sqmetre_rent_count'] * 100 / df['community_count']) + + +# In[19]: + + +df.fillna(0, inplace=True) + + +# In[20]: + + +df['community_count'] = df['community_count'].astype(int) +df['avg_price_count'] = df['avg_price_count'].astype(int) +df['sqmetre_rent_count'] = df['sqmetre_rent_count'].astype(int) + + +# In[21]: + + +df = df.round({'avg_price_coverage': 2, 'sqmetre_rent_coverage': 2}) + + +# In[22]: + + +df = df[['city', 'community_count', 'city_level', 'avg_price_count', 'avg_price_coverage', 'sqmetre_rent_count', 'sqmetre_rent_coverage']] + + +# In[23]: + + +df['weekend'] = dateutils.datetime.datetime.now() + + +# In[24]: + + +clickHouseClient.to_sql(df, table='community_coverage_stat', if_exists='insert', order_by='weekend') + + +# In[ ]: + + + + diff --git a/dags/stat/community_count_price_rent_dag.py b/dags/stat/community_count_price_rent_dag.py new file mode 100644 index 0000000..3aee802 --- /dev/null +++ b/dags/stat/community_count_price_rent_dag.py @@ -0,0 +1,16 @@ +from airflow.operators.bash import BashOperator +import pendulum +from airflow import DAG + +with DAG( + "community_count_price_rent_dag", + # 每周五运行 + schedule_interval="0 10 * * 5", + start_date=pendulum.datetime(2022, 7, 4, tz="Asia/Shanghai"), + catchup=True, +) as dag: + t1 = BashOperator( + task_id='community_count_price_rent', + bash_command='python /opt/airflow/dags/repo/dags/stat/community/community_count_price_rent.py', + ) + t1