This commit is contained in:
zhaojixiang@fangjia.com
2023-07-05 09:39:49 +08:00
parent 6fee8096dd
commit 278206fab5
2 changed files with 161 additions and 0 deletions
+144
View File
@@ -0,0 +1,144 @@
from goojia_utils import MAIN_MONGO_CLIENT,dateutils
from datetime import datetime,date
import pandas as pd
from dateutil.relativedelta import relativedelta
deal_price_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'deal_price')
district_sell_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'district_sell_stat')
evaluate_ratio_conn = MAIN_MONGO_CLIENT.get_collection('business', 'evaluate_ratio')
m_date = dateutils.iso_format()
datetimeNow = date.today()
datetimeTwoYearAgo = datetimeNow - relativedelta(years=2)
ago_year = datetimeTwoYearAgo.year
ago_month = datetimeTwoYearAgo.month
ago_day = datetimeTwoYearAgo.day
year = datetimeNow.year
month = datetimeNow.month
day = datetimeNow.day
def sync():
# 查询2年内成交价格
dear_price_r = deal_price_conn.find({
'trade_date':
{
"$gte": datetime(ago_year, ago_month, ago_day, 0, 0),
"$lt": datetime(year, month, day, 0, 0)
},
'area': {"$gt": 20},
'type': {'$ne': '车位'},
'match_id': {'$exists': 'true'},
'avg_price': {'$lte': 500000}
}, {'city': 1, 'region': 1, 'district_name': 1, 'avg_price': 1, 'area': 1, 'trade_date': 1, 'match_id': 1})
df1 = pd.DataFrame(list(dear_price_r))
# 去重
df1.drop_duplicates(subset=['city', 'region', 'district_name', 'avg_price', 'area', 'trade_date', 'match_id'],
keep='first', inplace=True)
df1.reset_index(drop=True, inplace=True)
# 过滤数据(同小区成交价与小区成交均价偏差较大)
df1['中位值'] = df1.groupby(by=['city', 'region', 'district_name'])['avg_price'].transform("median")
df1['ratio'] = round((df1['avg_price'] - df1['中位值']) / df1['中位值'], 2)
df1 = df1.drop(df1.index[df1['ratio'] > 0.2])
df1 = df1.drop(df1.index[df1['ratio'] < -0.2])
# 从日期中提取年月,进行拼接
df1['Time'] = pd.to_datetime(df1['trade_date'])
df1['Year'] = df1['Time'].dt.strftime('%Y').map(str)
df1['Month'] = df1['Time'].dt.strftime('%m').map(str)
df1['date'] = df1['Year'] + df1['Month']
convert_dict = {'date': int}
df1 = df1.astype(convert_dict)
# 查询2年内挂牌价格
district_stat_r = district_sell_stat_conn.find({
'date':
{
"$gte": 202106,
"$lt": 202306
}
},{'city': 1, 'region': 1, 'district_name': 1, 'avg_price': 1, 'date': 1})
df2 = pd.DataFrame(list(district_stat_r))
# 过滤数据(同小区挂牌价与小区挂牌均价偏差较大)
df2['中位值'] = df2.groupby(by=['city', 'region', 'district_name'])['avg_price'].transform("median")
df2['ratio'] = round((df2['avg_price'] - df2['中位值']) / df2['中位值'], 2)
df2 = df2.drop(df2.index[df2['ratio'] > 0.2])
df2 = df2.drop(df2.index[df2['ratio'] < -0.2])
# 合并表
df1_2 = pd.merge(df1, df2, how='left', on=['city', 'region', 'district_name', 'date'])
# 过滤数据(成交价与挂牌价偏差较大)
df1_2['ratio'] = round((df1_2['avg_price_x'] - df1_2['avg_price_y']) / df1_2['avg_price_y'], 2)
df1_2 = df1_2.drop(df1_2.index[df1_2['ratio'] > 0.2])
df1_2 = df1_2.drop(df1_2.index[df1_2['ratio'] < -0.2])
df1_2 = df1_2.dropna(axis='index', how='all', subset=['ratio'])
# 城市挂牌价与成交价计算价格系数
deal_price_city_avgprice = df1_2.groupby(by=['city'])['avg_price_x'].mean().reset_index()
district_stat_city_avgprice = df1_2.groupby(by=['city'])['avg_price_y'].mean().reset_index()
df = pd.merge(deal_price_city_avgprice, district_stat_city_avgprice, how='inner', on=['city'])
df['ratio'] = round((df['avg_price_x'] - df['avg_price_y']) / df['avg_price_y'], 2)
#df.to_excel('data/city_ratio.xlsx', index=False)
for index in df.index:
city = df['city'].get(index)
ratio = df['ratio'].get(index)
if ratio > 0.15 or ratio < -0.15:
continue
else:
query = {"city": city, "category": "city"}
update = {"$set": {"ratio": ratio, "m_date": m_date}}
update_result = evaluate_ratio_conn.update_one(query, update, upsert=True)
# 区域挂牌价与成交价计算价格系数
deal_price_region_avgprice = df1_2.groupby(by=['city', 'region'])['avg_price_x'].mean().reset_index()
district_stat_region_avgprice = df1_2.groupby(by=['city', 'region'])['avg_price_y'].mean().reset_index()
df = pd.merge(deal_price_region_avgprice, district_stat_region_avgprice, how='inner', on=['city', 'region'])
df['ratio'] = round((df['avg_price_x'] - df['avg_price_y']) / df['avg_price_y'], 2)
for index in df.index:
city = df['city'].get(index)
ratio = df['ratio'].get(index)
region = df['region'].get(index)
if ratio > 0.15 or ratio < -0.15:
continue
else:
query = {"city": city, "region": region, "category": "region"}
update = {"$set": {"ratio": ratio, "region": region, "m_date": m_date}}
update_result = evaluate_ratio_conn.update_one(query, update, upsert=True)
# 小区挂牌价与成交价计算价格系数
deal_price_district_avgprice = df1_2.groupby(by=['city', 'region', 'district_name'])[
'avg_price_x'].mean().reset_index()
district_stat_district_avgprice = df1_2.groupby(by=['city', 'region', 'district_name'])[
'avg_price_y'].mean().reset_index()
df = pd.merge(deal_price_district_avgprice, district_stat_district_avgprice, how='inner',
on=['city', 'region', 'district_name'])
df['ratio'] = round((df['avg_price_x'] - df['avg_price_y']) / df['avg_price_y'], 2)
df[['city', 'region', 'district_name', 'ratio']]
for index in df.index:
city = df['city'].get(index)
ratio = df['ratio'].get(index)
region = df['region'].get(index)
district_name = df['district_name'].get(index)
if ratio > 0.15 or ratio < -0.15:
continue
else:
query = {"city": city, "region": region, "district_name": district_name, "category": "district"}
update = {
"$set": {"ratio": ratio, "m_date": m_date}}
update_result = evaluate_ratio_conn.update_one(query, update, upsert=True)
sync()
+17
View File
@@ -0,0 +1,17 @@
from airflow.operators.bash import BashOperator
import pendulum
from airflow import DAG
with DAG(
dag_id="evaluate_ratio_dag",
# 每月1日0点运行
schedule_interval="0 0 1 * *",
start_date=pendulum.datetime(2023, 7, 5, tz="Asia/Shanghai"),
catchup=True,
description="挂牌价与成交价计算价格系数"
) as dag:
t1 = BashOperator(
task_id='evaluate_ratio',
bash_command='python /opt/airflow/dags/repo/dags/business/business/evaluate_ratio.py',
)
t1