Files
airflow/dags/office/sell/base_price.py
T

586 lines
25 KiB
Python

"""
写字楼基准价
"""
import math
from datetime import datetime
import numpy as np
import pandas as pd
# 第一步:计算有样本写字楼的基准价
# 取近一年,最近100个样本的中位值
# 第二步:计算无样本写字楼的基准价
# 取周边3公里内同等级、同类型的最近10个有样本写字楼的基准价的平均值
# 第三步:根据第一步和第二步有写字楼等级和写字楼类型的写字楼基准化计算出写字楼等级和类型对基准价影响的系数
# 第四步:根据写字楼等级系数和类型系数计算出没有相同特征写字楼的基准化
#
# 第五步:本身没有等级和类型的写字楼
# 最近10个有样本写字楼的基准价的平均值
from goojia_utils import ElasticSearchClient, dateutils, MAIN_MONGO_CLIENT
from pymongo import UpdateOne
es = ElasticSearchClient(host='192.168.0.206')
seaweed_df = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
office_sell_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'office_sell_stat')
# 计算有样本写字楼的基准价 1. 取近一年,最近100个样本的中位值
def calc(**kwargs):
"""
写字楼基准价
"""
# 从 Airflow 的 dag_run.conf 获取参数
conf = kwargs.get('dag_run').conf if kwargs.get('dag_run') else {}
city = conf.get('city', None)
year_month = conf.get('yearmonth', None)
district = conf.get('district', None)
office_name = conf.get('name', None)
if year_month is None:
year_month = dateutils.format_date(dateutils.add_months(datetime.today(), months=-1), "%Y%m")
if pd.isna(office_name):
calcByCity(city, year_month)
else:
# 直接重算出最近一年的数据,保证一年有数据
calcByOffice(city, district, office_name)
def calcByOffice(city, district, office_name):
"""
写字楼价格重算
"""
query = {'cat': 'office', 'status': 0}
if pd.notna(city):
query['city'] = city
if pd.notna(district):
query['region'] = district
if pd.notna(city):
query['name'] = office_name
print(query)
office = seaweed_df.find_one(query, {'city': 1, 'region': 1, 'name': 1, 'alias': 1, 'lng': 1, 'lat': 1, 'estate_type2': 1, 'bed_kind': 1})
if 'estate_type2' not in office:
office['estate_type2'] = None
if 'bed_kind' not in office:
office['bed_kind'] = None
names = list()
names.append(office['name'])
if ('alias' in office) and (type(office['alias']) == list):
for alias_item in office['alias']:
names.append(alias_item)
names_str = "','".join(names)
names_str = "'" + names_str + "'"
# 只查询该小区的样本
year_time = dateutils.format_date(dateutils.datetime.datetime.now())
one_year_time = dateutils.format_date(dateutils.add_months(dateutils.datetime.datetime.now(), days=-365))
sql = f"select * from latest_seawater where cat='office_sell' and city='{city}' and region='{district}' and district_name in ({names_str}) and p_date > {one_year_time}"
sample = es.scan_by_sql_to_data_frame(sql)
location = {
"type": "Point",
"coordinates": [office['lng'] / math.pow(10, 10), office['lat'] / math.pow(10, 10)]
}
if "p_date" not in sample.columns:
sample['p_date'] = None
date_range = pd.date_range(start=one_year_time, end=year_time, freq='M')
for date in date_range:
curr_date = int(dateutils.format_date(date, fm='%Y%m'))
# 筛选出当前月的样本
curr_sample = sample[(sample['p_date']>=int(str(curr_date)+'01')) & (sample['p_date']<=int(str(curr_date)+'31'))]
update_office(city, pd.DataFrame([office]), curr_sample, curr_date)
# 核查当月写字楼是否存在,不存在取最后6个月的平均值
office_db = office_sell_stat_conn.find_one({'cat': 'office', 'city': city, 'district': district, 'name': office_name, 'date': curr_date})
if pd.isna(office_db):
avg_price_ret = office_sell_stat_conn.aggregate([
{
"$match": {
"cat": "office",
"city": city,
"district": district,
"name": office_name,
"date": { "$gte": curr_date - 3, "$lte": curr_date + 3 }
}
},
{
"$group": {
"_id": "$name",
"avg_price": { "$avg": "$avg_price" }
}
}
])
avg_prices = [doc["avg_price"] for doc in avg_price_ret]
# 计算平均值
average_price = np.mean(avg_prices) if len(avg_prices) > 0 else 0
new_avg_price = None
if average_price > 0:
new_avg_price = int(average_price)
elif "avg_price" in sample.columns:
# 取样本的平均值
new_avg_price = int(sample['avg_price'].mean())
if pd.notna(new_avg_price):
action = UpdateOne(
{'cat': 'office', 'city': city, 'district': district, 'name': office_name, 'date': curr_date},
{'$set': {'city': city, 'cat': 'office', 'date': curr_date, 'district': district,
'name': office_name,
'avg_price': new_avg_price, 'estate_type2': office['estate_type2'],
'bed_kind': office['bed_kind'],
'location': location,
'calc_type': 5, 'updated_date': datetime.now()}},
upsert=True)
office_sell_stat_conn.bulk_write([action])
# 🏁 触发 MoM 和 YoY(环比与同比)的自动计算与更新(出售)
calculate_mom_yoy_for_office(city, district, office_name)
def calculate_mom_yoy_for_office(city, district, office_name):
"""
计算特定写字楼出售历史所有月份的 mom (环比) 和 yoy (同比) 并更新
"""
query = {
'cat': 'office',
'city': city,
'district': district,
'name': office_name
}
docs = list(office_sell_stat_conn.find(query).sort('date', 1))
if not docs:
return
doc_dict = {doc['date']: doc for doc in docs}
actions = []
for doc in docs:
curr_date = doc['date']
curr_price = doc.get('avg_price')
if not curr_price:
continue
# 计算前一个月的 YYYYMM
year = curr_date // 100
month = curr_date % 100
if month == 1:
prev_date = (year - 1) * 100 + 12
else:
prev_date = curr_date - 1
# 计算上年同月的 YYYYMM
ly_date = curr_date - 100
# 计算 mom (环比)
mom = None
if prev_date in doc_dict and doc_dict[prev_date].get('avg_price'):
prev_price = doc_dict[prev_date]['avg_price']
mom = round((curr_price - prev_price) / prev_price * 100, 2)
# 计算 yoy (同比)
yoy = None
if ly_date in doc_dict and doc_dict[ly_date].get('avg_price'):
ly_price = doc_dict[ly_date]['avg_price']
yoy = round((curr_price - ly_price) / ly_price * 100, 2)
action = UpdateOne(
{'_id': doc['_id']},
{'$set': {'mom': mom, 'yoy': yoy}}
)
actions.append(action)
if actions:
office_sell_stat_conn.bulk_write(actions)
def calculate_mom_yoy_by_month(year_month, city=None):
"""
批量计算某月所有写字楼出售的 mom (环比) 和 yoy (同比) 并更新
"""
curr_date = int(year_month)
year = curr_date // 100
month = curr_date % 100
prev_date = (year - 1) * 100 + 12 if month == 1 else curr_date - 1
ly_date = curr_date - 100
base_query = {'cat': 'office'}
if pd.notna(city) and city != '':
base_query['city'] = city
proj = {'_id': 1, 'city': 1, 'district': 1, 'name': 1, 'avg_price': 1}
curr_docs = list(office_sell_stat_conn.find({**base_query, 'date': curr_date}, proj))
if not curr_docs:
return
prev_docs = list(office_sell_stat_conn.find({**base_query, 'date': prev_date}, proj))
ly_docs = list(office_sell_stat_conn.find({**base_query, 'date': ly_date}, proj))
prev_map = {(d.get('city'), d.get('district'), d.get('name')): d.get('avg_price') for d in prev_docs if d.get('avg_price')}
ly_map = {(d.get('city'), d.get('district'), d.get('name')): d.get('avg_price') for d in ly_docs if d.get('avg_price')}
actions = []
for doc in curr_docs:
curr_price = doc.get('avg_price')
if not curr_price:
continue
key = (doc.get('city'), doc.get('district'), doc.get('name'))
prev_price = prev_map.get(key)
ly_price = ly_map.get(key)
mom = round((curr_price - prev_price) / prev_price * 100, 2) if prev_price else None
yoy = round((curr_price - ly_price) / ly_price * 100, 2) if ly_price else None
actions.append(UpdateOne({'_id': doc['_id']}, {'$set': {'mom': mom, 'yoy': yoy}}))
if len(actions) == 5000:
office_sell_stat_conn.bulk_write(actions)
actions.clear()
if actions:
office_sell_stat_conn.bulk_write(actions)
def calcByCity(city, year_month):
"""
写字楼基准价
"""
sql = "select * from latest_seawater where cat='office_sell' "
if pd.notna(city) and city != "":
sql += "and city='" + city + "' "
if pd.notna(year_month):
last_year_month = dateutils.format_date(
dateutils.add_months(dateutils.parse_date(str(year_month), '%Y%m'), months=-13), '%Y%m')
last_month = dateutils.format_date(dateutils.add_months(dateutils.parse_date(str(year_month), '%Y%m'), months=-1), '%Y%m')
sql += "and p_date >= '" + last_year_month + "01' "
if isinstance(year_month, str):
year_month = int(year_month)
else:
last_year_month = dateutils.format_date(dateutils.add_months(datetime.now(), months=-13), '%Y%m')
last_month = dateutils.format_date(dateutils.add_months(datetime.now(), months=-1), '%Y%m')
sql += "and p_date >= '" + last_year_month + "01' "
date = dateutils.add_months(datetime.today(), months=-1)
year_month = int(dateutils.format_date(date, '%Y%m'))
print("开始计算年" + str(year_month) + "写字楼基准价")
sample = es.scan_by_sql_to_data_frame(sql)
print("样本SQL:" + sql + "样本量:" +str(len(sample)))
office = getOffice(city)
update_office(city, office, sample, year_month)
calculate_mom_yoy_by_month(year_month, city)
def update_office(city, office, sample, year_month):
target_city = city
if 'bed_kind' not in office.columns:
office['bed_kind'] = None
if 'estate_type2' not in office.columns:
office['estate_type2'] = None
if 'alias' not in office.columns:
office['alias'] = None
merge_df = None
if not sample.empty:
sample_new = sample
if 'alias' not in sample_new.columns:
sample_new['alias'] = None
sample_new['sample_avg_price'] = sample_new.groupby(by=['city', 'region', 'district_name'])['avg_price'].transform(
'median')
sample_new['sample_count'] = sample_new.groupby(by=['city', 'region', 'district_name'])['avg_price'].transform(
'count')
sample_new = sample_new[sample_new['sample_count'] >= 5]
sample_new = sample_new[['city', 'region', 'district_name', 'sample_avg_price']].dropna()
sample_new['day_avg_price'] = sample_new['sample_avg_price']
sample_new = sample_new[['city', 'region', 'district_name', 'day_avg_price']]
sample_new = sample_new.rename(columns={'district_name': 'name'})
sample_new['alias'] = sample_new['name']
# 使用主名与别名查找样本
merge_df1 = pd.merge(office, sample_new[['city', 'region', 'name', 'day_avg_price']], how='left',
on=['city', 'region', 'name'])
if len(sample_new['alias'].unique()) > 0:
merge_df2 = pd.merge(office, sample_new[['city', 'region', 'alias', 'day_avg_price']], how='left',
on=['city', 'region', 'alias'])
merge_df = pd.concat([merge_df1, merge_df2])
else:
merge_df = merge_df1.copy()
merge_df = merge_df[merge_df['day_avg_price'].notna()]
merge_df = merge_df.drop_duplicates(subset=['city', 'region', 'name'], keep='first')
# 第一步:计算有样本写字楼的基准价
print("有样本数量:" + str(len(merge_df)))
actions = list()
for index, item in merge_df.iterrows():
city = item['city']
region = item['region']
name = item['name']
avg_price = int(item['day_avg_price'])
location = {
"type": "Point",
"coordinates": [item['lng'] / math.pow(10, 10), item['lat'] / math.pow(10, 10)]
}
estate_type2 = item['estate_type2'] if pd.notna(item['estate_type2']) else None
bed_kind = item['bed_kind'] if pd.notna(item['bed_kind']) else None
action = UpdateOne({'cat': 'office', 'city': city, 'district': region, 'name': name, 'date': year_month},
{'$set': {'city': city, 'cat': 'office', 'date': year_month, 'district': region,
'name': name,
'avg_price': avg_price, 'estate_type2': estate_type2, 'bed_kind': bed_kind,
'location': location,
'calc_type': 1, 'updated_date': datetime.now()}},
upsert=True)
actions.append(action)
if len(actions) == 1000:
office_sell_stat_conn.bulk_write(actions)
actions.clear()
if len(actions) > 0:
office_sell_stat_conn.bulk_write(actions)
actions.clear()
# 第二步 取周边3公里内同等级、同类型的最近10个有样本写字楼的基准价的平均值
office_info = office[['city', 'region', 'name', 'lng', 'lat', 'estate_type2', 'bed_kind']].drop_duplicates(
keep='first')
combined = office_info
if merge_df is not None and not merge_df.empty:
combined = pd.concat([office_info, merge_df])
diff_df = combined.drop_duplicates(subset=['city', 'region', 'name'], keep=False)
actions = list()
for index, item in diff_df[
(diff_df['estate_type2'].notna()) & (diff_df['bed_kind'].notna()) & (diff_df['lng'].notna())].iterrows():
location = {
"type": "Point",
"coordinates": [item['lng'] / math.pow(10, 10), item['lat'] / math.pow(10, 10)]
}
city = item['city']
region = item['region']
name = item['name']
estate_type2 = item['estate_type2']
bed_kind = item['bed_kind']
if estate_type2 == '' or bed_kind == '':
continue
ret = office_sell_stat_conn.aggregate([
{
'$geoNear': {
'near': location,
'distanceField': "distance",
'maxDistance': 3000,
'spherical': True
}
},
{
'$match': {
'estate_type2': estate_type2,
'bed_kind': bed_kind,
'calc_type': 1,
'date': year_month
}
},
{
'$limit': 5
}
]);
# 提取 avg_price 的值
avg_prices = [doc["avg_price"] for doc in ret]
# 计算平均值
average_price = np.mean(avg_prices) if len(avg_prices) > 0 else 0
if average_price > 0:
avg_price = int(average_price)
action = UpdateOne({'cat': 'office', 'city': city, 'district': region, 'name': name, 'date': year_month},
{'$set': {'city': city, 'cat': 'office', 'date': year_month, 'district': region,
'name': name,
'avg_price': avg_price, 'estate_type2': estate_type2, 'bed_kind': bed_kind,
'location': location,
'calc_type': 2, 'updated_date': datetime.now()}},
upsert=True)
actions.append(action)
if len(actions) == 1000:
office_sell_stat_conn.bulk_write(actions)
actions.clear()
if len(actions) > 0:
office_sell_stat_conn.bulk_write(actions)
actions.clear()
# 系数计算在Jupyter中
bed_kind_dict = {'甲级': 1.111888111888112, '丙级': 0.9143356643356645, '乙级': 1}
radio_dict = {'创业园区': 0.7799999999999999, '商住两用': 1.0466666666666666, '标准写字楼': 1}
# 根据写字楼等级系数和类型系数计算出没有相同特征写字楼的基准化
actions.clear()
for index, item in diff_df[(diff_df['estate_type2'].notna()) | (diff_df['bed_kind'].notna())].iterrows():
location = {
"type": "Point",
"coordinates": [item['lng'] / math.pow(10, 10), item['lat'] / math.pow(10, 10)]
}
city = item['city']
region = item['region']
name = item['name']
estate_type2 = item['estate_type2']
bed_kind = item['bed_kind']
if estate_type2 == '' or bed_kind == '':
continue
if pd.notna(estate_type2) and pd.notna(bed_kind):
continue
if pd.notna(estate_type2):
ret = office_sell_stat_conn.aggregate([
{
'$geoNear': {
'near': location,
'distanceField': "distance",
'maxDistance': 3000,
'spherical': True
}
},
{
'$match': {
'estate_type2': estate_type2,
'date': year_month
}
},
{
'$limit': 5
}
]);
# 提取 avg_price 的值
count = 0
sum_price = 0
if estate_type2 in radio_dict.keys():
for doc in ret:
if doc['estate_type2'] in radio_dict.keys():
new_price = doc['avg_price'] / radio_dict[doc['estate_type2']] * radio_dict[estate_type2]
sum_price += new_price
count += 1
if count > 0:
avg_price = int(sum_price / count)
action = UpdateOne(
{'cat': 'office', 'city': city, 'district': region, 'name': name, 'date': year_month},
{'$set': {'city': city, 'cat': 'office', 'date': year_month, 'district': region,
'name': name,
'avg_price': avg_price, 'estate_type2': estate_type2, 'bed_kind': None,
'location': location,
'calc_type': 3, 'updated_date': datetime.now()}},
upsert=True)
actions.append(action)
elif pd.notna(bed_kind):
ret = office_sell_stat_conn.aggregate([
{
'$geoNear': {
'near': location,
'distanceField': "distance",
'maxDistance': 3000,
'spherical': True
}
},
{
'$match': {
'bed_kind': bed_kind,
'date': year_month
}
},
{
'$limit': 5
}
]);
# 提取 avg_price 的值
count = 0
sum_price = 0
if bed_kind in bed_kind_dict.keys():
for doc in ret:
if doc['bed_kind'] in bed_kind_dict.keys():
new_price = doc['avg_price'] / bed_kind_dict[doc['bed_kind']] * bed_kind_dict[bed_kind]
sum_price += new_price
count += 1
if count > 0:
avg_price = int(sum_price / count)
action = UpdateOne(
{'cat': 'office', 'city': city, 'district': region, 'name': name, 'date': year_month},
{'$set': {'city': city, 'cat': 'office', 'date': year_month, 'district': region,
'name': name,
'avg_price': avg_price, 'estate_type2': None, 'bed_kind': bed_kind,
'location': location,
'calc_type': 3, 'updated_date': datetime.now()}},
upsert=True)
actions.append(action)
if len(actions) == 1000:
office_sell_stat_conn.bulk_write(actions)
actions.clear()
if len(actions) > 0:
office_sell_stat_conn.bulk_write(actions)
actions.clear()
# 第五步
query = {'cat': 'office', 'date': year_month, 'calc_type': {'$in': [1, 2, 3]}}
if pd.notna(target_city) and target_city != '':
query['city'] = target_city
db_df = pd.DataFrame(list(office_sell_stat_conn.find(query)))
office_info = office[['city', 'region', 'name', 'lng', 'lat', 'estate_type2', 'bed_kind']].drop_duplicates(
keep='first')
# 计算价格的写字楼
db_df = db_df.rename(columns={'district': 'region'})
# 使用 merge 操作并结合 indicator 参数
if len(db_df) > 0:
merged = office_info.merge(db_df[['city', 'region', 'name']], on=['city', 'region', 'name'], how='left',
indicator=True)
# 过滤出仅在 office_info 中存在的记录
diff_df = merged[merged['_merge'] == 'left_only'].drop(columns=['_merge'])
# 如果需要保持原始列顺序,可以重新排列列
diff_df = diff_df[office_info.columns]
actions.clear()
for index, item in diff_df.iterrows():
try:
location = {
"type": "Point",
"coordinates": [item['lng'] / math.pow(10, 10), item['lat'] / math.pow(10, 10)]
}
city = item['city']
region = item['region']
name = item['name']
estate_type2 = item['estate_type2'] if pd.notna(item['estate_type2']) else None
bed_kind = item['bed_kind']
ret = office_sell_stat_conn.aggregate([
{
'$geoNear': {
'near': location,
'distanceField': "distance",
'maxDistance': 3000,
'spherical': True
}
},
{
'$match': {
'date': year_month
}
},
{
'$limit': 5
}
]);
except Exception as e:
print("Error occurred:", str(e))
print(item)
# 提取 avg_price 的值
avg_prices = [doc["avg_price"] for doc in ret]
# 计算平均值
average_price = np.mean(avg_prices) if len(avg_prices) > 0 else 0
if average_price > 0:
avg_price = int(average_price)
action = UpdateOne({'cat': 'office', 'city': city, 'district': region, 'name': name, 'date': year_month},
{'$set': {'city': city, 'cat': 'office', 'date': year_month, 'district': region,
'name': name,
'avg_price': avg_price, 'estate_type2': estate_type2, 'bed_kind': None,
'location': location,
'calc_type': 4, 'updated_date': datetime.now()}},
upsert=True)
actions.append(action)
if len(actions) == 1000:
office_sell_stat_conn.bulk_write(actions)
actions.clear()
if len(actions) > 0:
office_sell_stat_conn.bulk_write(actions)
actions.clear()
def getOffice(city, district=None, name=None):
"""
获取写字楼
"""
query = {'cat': 'office', 'status': 0}
if pd.notna(city):
query['city'] = city
if pd.notna(district):
query['region'] = district
if pd.notna(name):
query['name'] = name
office = pd.DataFrame(list(seaweed_df.find(query).batch_size(10000)))
for col in ['city', 'region', 'name', 'alias', 'estate_type2', 'bed_kind', 'lng', 'lat']:
if col not in office:
office[col] = None
office = office[['city', 'region', 'name', 'alias', 'estate_type2', 'bed_kind', 'lng', 'lat']]
office = office.explode('alias')
return office
if __name__ == "__main__":
# 测试脚本:直接运行时模拟 Airflow 的上下文
calc(dag_run=type('DagRun', (), {'conf': {'city': None}}))