diff --git a/dags/business/business/evaluate_ratio.py b/dags/business/business/evaluate_ratio.py new file mode 100644 index 0000000..d84b874 --- /dev/null +++ b/dags/business/business/evaluate_ratio.py @@ -0,0 +1,144 @@ +from goojia_utils import MAIN_MONGO_CLIENT,dateutils +from datetime import datetime,date +import pandas as pd +from dateutil.relativedelta import relativedelta + + +deal_price_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'deal_price') +district_sell_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'district_sell_stat') +evaluate_ratio_conn = MAIN_MONGO_CLIENT.get_collection('business', 'evaluate_ratio') + +m_date = dateutils.iso_format() + +datetimeNow = date.today() +datetimeTwoYearAgo = datetimeNow - relativedelta(years=2) + +ago_year = datetimeTwoYearAgo.year +ago_month = datetimeTwoYearAgo.month +ago_day = datetimeTwoYearAgo.day + +year = datetimeNow.year +month = datetimeNow.month +day = datetimeNow.day + +def sync(): + # 查询2年内成交价格 + dear_price_r = deal_price_conn.find({ + 'trade_date': + { + "$gte": datetime(ago_year, ago_month, ago_day, 0, 0), + "$lt": datetime(year, month, day, 0, 0) + }, + 'area': {"$gt": 20}, + 'type': {'$ne': '车位'}, + 'match_id': {'$exists': 'true'}, + 'avg_price': {'$lte': 500000} + }, {'city': 1, 'region': 1, 'district_name': 1, 'avg_price': 1, 'area': 1, 'trade_date': 1, 'match_id': 1}) + + df1 = pd.DataFrame(list(dear_price_r)) + # 去重 + df1.drop_duplicates(subset=['city', 'region', 'district_name', 'avg_price', 'area', 'trade_date', 'match_id'], + keep='first', inplace=True) + df1.reset_index(drop=True, inplace=True) + + # 过滤数据(同小区成交价与小区成交均价偏差较大) + df1['中位值'] = df1.groupby(by=['city', 'region', 'district_name'])['avg_price'].transform("median") + df1['ratio'] = round((df1['avg_price'] - df1['中位值']) / df1['中位值'], 2) + df1 = df1.drop(df1.index[df1['ratio'] > 0.2]) + df1 = df1.drop(df1.index[df1['ratio'] < -0.2]) + + # 从日期中提取年月,进行拼接 + df1['Time'] = pd.to_datetime(df1['trade_date']) + df1['Year'] = df1['Time'].dt.strftime('%Y').map(str) + df1['Month'] = df1['Time'].dt.strftime('%m').map(str) + df1['date'] = df1['Year'] + df1['Month'] + convert_dict = {'date': int} + df1 = df1.astype(convert_dict) + + # 查询2年内挂牌价格 + district_stat_r = district_sell_stat_conn.find({ + 'date': + { + "$gte": 202106, + "$lt": 202306 + } + },{'city': 1, 'region': 1, 'district_name': 1, 'avg_price': 1, 'date': 1}) + + df2 = pd.DataFrame(list(district_stat_r)) + + # 过滤数据(同小区挂牌价与小区挂牌均价偏差较大) + df2['中位值'] = df2.groupby(by=['city', 'region', 'district_name'])['avg_price'].transform("median") + df2['ratio'] = round((df2['avg_price'] - df2['中位值']) / df2['中位值'], 2) + df2 = df2.drop(df2.index[df2['ratio'] > 0.2]) + df2 = df2.drop(df2.index[df2['ratio'] < -0.2]) + + # 合并表 + df1_2 = pd.merge(df1, df2, how='left', on=['city', 'region', 'district_name', 'date']) + + # 过滤数据(成交价与挂牌价偏差较大) + df1_2['ratio'] = round((df1_2['avg_price_x'] - df1_2['avg_price_y']) / df1_2['avg_price_y'], 2) + + df1_2 = df1_2.drop(df1_2.index[df1_2['ratio'] > 0.2]) + df1_2 = df1_2.drop(df1_2.index[df1_2['ratio'] < -0.2]) + df1_2 = df1_2.dropna(axis='index', how='all', subset=['ratio']) + + # 城市挂牌价与成交价计算价格系数 + deal_price_city_avgprice = df1_2.groupby(by=['city'])['avg_price_x'].mean().reset_index() + district_stat_city_avgprice = df1_2.groupby(by=['city'])['avg_price_y'].mean().reset_index() + + df = pd.merge(deal_price_city_avgprice, district_stat_city_avgprice, how='inner', on=['city']) + df['ratio'] = round((df['avg_price_x'] - df['avg_price_y']) / df['avg_price_y'], 2) + + #df.to_excel('data/city_ratio.xlsx', index=False) + + for index in df.index: + city = df['city'].get(index) + ratio = df['ratio'].get(index) + if ratio > 0.15 or ratio < -0.15: + continue + else: + query = {"city": city, "category": "city"} + update = {"$set": {"ratio": ratio, "m_date": m_date}} + update_result = evaluate_ratio_conn.update_one(query, update, upsert=True) + + # 区域挂牌价与成交价计算价格系数 + deal_price_region_avgprice = df1_2.groupby(by=['city', 'region'])['avg_price_x'].mean().reset_index() + district_stat_region_avgprice = df1_2.groupby(by=['city', 'region'])['avg_price_y'].mean().reset_index() + df = pd.merge(deal_price_region_avgprice, district_stat_region_avgprice, how='inner', on=['city', 'region']) + df['ratio'] = round((df['avg_price_x'] - df['avg_price_y']) / df['avg_price_y'], 2) + + for index in df.index: + city = df['city'].get(index) + ratio = df['ratio'].get(index) + region = df['region'].get(index) + if ratio > 0.15 or ratio < -0.15: + continue + else: + query = {"city": city, "region": region, "category": "region"} + update = {"$set": {"ratio": ratio, "region": region, "m_date": m_date}} + update_result = evaluate_ratio_conn.update_one(query, update, upsert=True) + + # 小区挂牌价与成交价计算价格系数 + deal_price_district_avgprice = df1_2.groupby(by=['city', 'region', 'district_name'])[ + 'avg_price_x'].mean().reset_index() + district_stat_district_avgprice = df1_2.groupby(by=['city', 'region', 'district_name'])[ + 'avg_price_y'].mean().reset_index() + df = pd.merge(deal_price_district_avgprice, district_stat_district_avgprice, how='inner', + on=['city', 'region', 'district_name']) + df['ratio'] = round((df['avg_price_x'] - df['avg_price_y']) / df['avg_price_y'], 2) + df[['city', 'region', 'district_name', 'ratio']] + + for index in df.index: + city = df['city'].get(index) + ratio = df['ratio'].get(index) + region = df['region'].get(index) + district_name = df['district_name'].get(index) + if ratio > 0.15 or ratio < -0.15: + continue + else: + query = {"city": city, "region": region, "district_name": district_name, "category": "district"} + update = { + "$set": {"ratio": ratio, "m_date": m_date}} + update_result = evaluate_ratio_conn.update_one(query, update, upsert=True) + +sync() \ No newline at end of file diff --git a/dags/business/dag_evaluate_ratio.py b/dags/business/dag_evaluate_ratio.py new file mode 100644 index 0000000..d05811a --- /dev/null +++ b/dags/business/dag_evaluate_ratio.py @@ -0,0 +1,17 @@ +from airflow.operators.bash import BashOperator +import pendulum +from airflow import DAG + +with DAG( + dag_id="evaluate_ratio_dag", + # 每月1日0点运行 + schedule_interval="0 0 1 * *", + start_date=pendulum.datetime(2023, 7, 5, tz="Asia/Shanghai"), + catchup=True, + description="挂牌价与成交价计算价格系数" +) as dag: + t1 = BashOperator( + task_id='evaluate_ratio', + bash_command='python /opt/airflow/dags/repo/dags/business/business/evaluate_ratio.py', + ) + t1 \ No newline at end of file