迁移小区周统计定时任务至airflow
This commit is contained in:
@@ -1,3 +1,4 @@
|
||||
poi/business/.*
|
||||
stat/evaluate/*
|
||||
stat/community/*
|
||||
mt/supply/*
|
||||
@@ -0,0 +1,174 @@
|
||||
#!/usr/bin/env python
|
||||
# coding: utf-8
|
||||
|
||||
# In[1]:
|
||||
|
||||
|
||||
from goojia_utils import MAIN_MONGO_CLIENT, dateutils, ClickHouseClient
|
||||
import pandas as pd
|
||||
import math
|
||||
|
||||
|
||||
# In[2]:
|
||||
|
||||
|
||||
server_area_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'server_area')
|
||||
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
||||
district_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'district_stat')
|
||||
sqmetre_rent_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'sqmetre_rent')
|
||||
clickHouseClient = ClickHouseClient(host="192.168.0.216", database='fangjia_stat')
|
||||
|
||||
|
||||
# In[3]:
|
||||
|
||||
|
||||
city_df = pd.DataFrame(list(server_area_conn.find({}, {'city_grade': 1, 'display': 1})))
|
||||
city_df = city_df.rename(columns={'city_grade': 'city_level', 'display': 'city'})
|
||||
city_df = city_df[city_df['city_level'] != 0]
|
||||
|
||||
|
||||
# #### 1. 小区数量
|
||||
|
||||
# In[4]:
|
||||
|
||||
|
||||
result_df = pd.DataFrame(list(seaweed_conn.aggregate([
|
||||
{'$match': {'cat': 'district', 'status': 0}},
|
||||
{'$group': {'_id': "$city", 'community_count': {'$sum': 1}}}])))
|
||||
|
||||
|
||||
# In[5]:
|
||||
|
||||
|
||||
result_df = result_df.rename(columns={'_id': 'city'})
|
||||
|
||||
|
||||
# In[6]:
|
||||
|
||||
|
||||
df = pd.merge(result_df, city_df, how='right', on='city')
|
||||
|
||||
|
||||
# In[7]:
|
||||
|
||||
|
||||
df.fillna(0, inplace=True)
|
||||
|
||||
|
||||
# #### 2. 均价数量
|
||||
|
||||
# In[9]:
|
||||
|
||||
|
||||
week_df = pd.DataFrame(list(district_stat_conn.distinct('weekend',
|
||||
{'weekend': {'$gte': dateutils.add_months(dateutils.datetime.datetime.now(), days=-60)}})))
|
||||
week = week_df.loc[len(week_df) - 1, 0]
|
||||
|
||||
|
||||
# In[10]:
|
||||
|
||||
|
||||
avg_price_df = pd.DataFrame(list(district_stat_conn.aggregate([
|
||||
{'$match': {'weekend': {'$gte': week }}},
|
||||
{'$group': {'_id': '$city', 'num': {'$sum': 1}}}
|
||||
])))
|
||||
|
||||
|
||||
# In[11]:
|
||||
|
||||
|
||||
avg_price_df = avg_price_df.rename(columns={'_id': 'city', 'num': 'avg_price_count'})
|
||||
|
||||
|
||||
# In[12]:
|
||||
|
||||
|
||||
df = pd.merge(df, avg_price_df, how='left', on=['city'])
|
||||
|
||||
|
||||
# In[13]:
|
||||
|
||||
|
||||
df['avg_price_coverage'] = pd.to_numeric(df['avg_price_count'] * 100 / df['community_count'])
|
||||
|
||||
|
||||
# #### 3.租金数量
|
||||
|
||||
# In[14]:
|
||||
|
||||
|
||||
week_df = pd.DataFrame(list(sqmetre_rent_conn.distinct('weekend',
|
||||
{'weekend': {'$gte': dateutils.add_months(dateutils.datetime.datetime.now(), days=-60)}})))
|
||||
week = week_df.loc[len(week_df) - 1, 0]
|
||||
|
||||
|
||||
# In[15]:
|
||||
|
||||
|
||||
sqmetre_rent_df = pd.DataFrame(list(sqmetre_rent_conn.aggregate([
|
||||
{'$match': {'weekend': {'$gte': week}}},
|
||||
{'$group': {'_id': '$city', 'num': {'$sum': 1}}}
|
||||
])))
|
||||
|
||||
|
||||
# In[16]:
|
||||
|
||||
|
||||
sqmetre_rent_df = sqmetre_rent_df.rename(columns={'_id': 'city', 'num': 'sqmetre_rent_count'})
|
||||
|
||||
|
||||
# In[17]:
|
||||
|
||||
|
||||
df = pd.merge(df, sqmetre_rent_df, how='left', on=['city'])
|
||||
|
||||
|
||||
# In[18]:
|
||||
|
||||
|
||||
df['sqmetre_rent_coverage'] = pd.to_numeric(df['sqmetre_rent_count'] * 100 / df['community_count'])
|
||||
|
||||
|
||||
# In[19]:
|
||||
|
||||
|
||||
df.fillna(0, inplace=True)
|
||||
|
||||
|
||||
# In[20]:
|
||||
|
||||
|
||||
df['community_count'] = df['community_count'].astype(int)
|
||||
df['avg_price_count'] = df['avg_price_count'].astype(int)
|
||||
df['sqmetre_rent_count'] = df['sqmetre_rent_count'].astype(int)
|
||||
|
||||
|
||||
# In[21]:
|
||||
|
||||
|
||||
df = df.round({'avg_price_coverage': 2, 'sqmetre_rent_coverage': 2})
|
||||
|
||||
|
||||
# In[22]:
|
||||
|
||||
|
||||
df = df[['city', 'community_count', 'city_level', 'avg_price_count', 'avg_price_coverage', 'sqmetre_rent_count', 'sqmetre_rent_coverage']]
|
||||
|
||||
|
||||
# In[23]:
|
||||
|
||||
|
||||
df['weekend'] = dateutils.datetime.datetime.now()
|
||||
|
||||
|
||||
# In[24]:
|
||||
|
||||
|
||||
clickHouseClient.to_sql(df, table='community_coverage_stat', if_exists='insert', order_by='weekend')
|
||||
|
||||
|
||||
# In[ ]:
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
from airflow.operators.bash import BashOperator
|
||||
import pendulum
|
||||
from airflow import DAG
|
||||
|
||||
with DAG(
|
||||
"community_count_price_rent_dag",
|
||||
# 每周五运行
|
||||
schedule_interval="0 10 * * 5",
|
||||
start_date=pendulum.datetime(2022, 7, 4, tz="Asia/Shanghai"),
|
||||
catchup=True,
|
||||
) as dag:
|
||||
t1 = BashOperator(
|
||||
task_id='community_count_price_rent',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/stat/community/community_count_price_rent.py',
|
||||
)
|
||||
t1
|
||||
Reference in New Issue
Block a user