优化价格类型
This commit is contained in:
@@ -16,5 +16,10 @@ with DAG(
|
||||
task_id='office_sell',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/office/sell/price.py {{dag_run.conf["yearmonth"] if "yearmonth" in dag_run.conf}} ',
|
||||
)
|
||||
t1
|
||||
|
||||
t2 = BashOperator(
|
||||
task_id='fill_price',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/office/sell/fill_price.py {{dag_run.conf["yearmonth"] if "yearmonth" in dag_run.conf}}',
|
||||
)
|
||||
t1 >> t2
|
||||
|
||||
|
||||
@@ -76,6 +76,7 @@ def calc(city, year_month):
|
||||
# 第一步:计算有样本写字楼的基准价
|
||||
print("有样本数量:" + str(len(merge_df)))
|
||||
actions = list()
|
||||
count = 0
|
||||
for index, item in merge_df.iterrows():
|
||||
city = item['city']
|
||||
region = item['region']
|
||||
@@ -96,9 +97,13 @@ def calc(city, year_month):
|
||||
actions.append(action)
|
||||
if len(actions) == 1000:
|
||||
office_sell_stat_conn.bulk_write(actions)
|
||||
count += len(actions)
|
||||
print("已计算" + str(count) + "条数据")
|
||||
actions.clear()
|
||||
if len(actions) > 0:
|
||||
office_sell_stat_conn.bulk_write(actions)
|
||||
count += len(actions)
|
||||
print("已计算" + str(count) + "条数据")
|
||||
actions.clear()
|
||||
|
||||
# 第二步 取周边3公里内同等级、同类型的最近10个有样本写字楼的基准价的平均值
|
||||
@@ -155,9 +160,13 @@ def calc(city, year_month):
|
||||
actions.append(action)
|
||||
if len(actions) == 1000:
|
||||
office_sell_stat_conn.bulk_write(actions)
|
||||
count += len(actions)
|
||||
print("已计算" + str(count) + "条数据")
|
||||
actions.clear()
|
||||
if len(actions) > 0:
|
||||
office_sell_stat_conn.bulk_write(actions)
|
||||
count += len(actions)
|
||||
print("已计算" + str(count) + "条数据")
|
||||
actions.clear()
|
||||
|
||||
# 系数计算在Jupyter中
|
||||
@@ -259,9 +268,13 @@ def calc(city, year_month):
|
||||
actions.append(action)
|
||||
if len(actions) == 1000:
|
||||
office_sell_stat_conn.bulk_write(actions)
|
||||
count += len(actions)
|
||||
print("已计算" + str(count) + "条数据")
|
||||
actions.clear()
|
||||
if len(actions) > 0:
|
||||
office_sell_stat_conn.bulk_write(actions)
|
||||
count += len(actions)
|
||||
print("已计算" + str(count) + "条数据")
|
||||
actions.clear()
|
||||
# 第五步
|
||||
query = {'cat': 'office', 'date': year_month, 'calc_type': {'$in': [1, 2, 3]}}
|
||||
@@ -331,23 +344,14 @@ def calc(city, year_month):
|
||||
actions.append(action)
|
||||
if len(actions) == 1000:
|
||||
office_sell_stat_conn.bulk_write(actions)
|
||||
count += len(actions)
|
||||
print("已计算" + str(count) + "条数据")
|
||||
actions.clear()
|
||||
if len(actions) > 0:
|
||||
office_sell_stat_conn.bulk_write(actions)
|
||||
count += len(actions)
|
||||
print("已计算" + str(count) + "条数据")
|
||||
actions.clear()
|
||||
|
||||
yearmonth = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
for yearmonth in [
|
||||
"202406",
|
||||
"202407",
|
||||
"202408",
|
||||
"202409",
|
||||
"202410",
|
||||
"202411",
|
||||
"202412",
|
||||
"202501"
|
||||
]:
|
||||
calc(None, yearmonth)
|
||||
# # calc("广州", yearmonth)
|
||||
# # calc("武汉", yearmonth)
|
||||
# calc("深圳", yearmonth)
|
||||
calc(None, yearmonth)
|
||||
@@ -60,7 +60,7 @@ def calc(city, year_month):
|
||||
action = UpdateOne({'cat': 'office', 'city': curr_city, 'district': region, 'name': name, 'date': year_month},
|
||||
{'$set': {'city': curr_city, 'cat': 'office', 'date': year_month, 'district': region,
|
||||
'name': name,
|
||||
'avg_price': sell_price, 'estate_type2': estate_type2, 'bed_kind': bed_kind,
|
||||
'avg_price': int(sell_price), 'estate_type2': estate_type2, 'bed_kind': bed_kind,
|
||||
'location': location,
|
||||
'calc_type': 5, 'updated_date': datetime.now()}},
|
||||
upsert=True)
|
||||
@@ -85,8 +85,6 @@ if yearmonth == '':
|
||||
try:
|
||||
if type(yearmonth) is str:
|
||||
yearmonth = int(yearmonth)
|
||||
# for curr_city in areautils.get_all_cities():
|
||||
# print("正在处理城市:{}".format(curr_city))
|
||||
calc(None, yearmonth)
|
||||
except Exception as e:
|
||||
print(e)
|
||||
|
||||
@@ -15,6 +15,7 @@ office_rent_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'office
|
||||
|
||||
|
||||
def calc(city, year_month):
|
||||
print(f"开始处理{city}, {year_month}")
|
||||
query = {'date': year_month, 'cat': 'office'}
|
||||
if pd.notna(city):
|
||||
query['city'] = city
|
||||
@@ -62,7 +63,7 @@ def calc(city, year_month):
|
||||
# 写入MONGODB
|
||||
for index, row in merge2_df.iterrows():
|
||||
action = UpdateOne({'city': row['city'], 'cat': 'city', 'date': year_month},
|
||||
{'$set': {'city': row['city'], 'cat': 'city', 'date': year_month, 'calc_type': 2, 'avg_price': row['sell_price'],
|
||||
{'$set': {'city': row['city'], 'cat': 'city', 'date': year_month, 'calc_type': 2, 'avg_price': int(row['sell_price']),
|
||||
'updated_date': datetime.now()}},
|
||||
upsert=True)
|
||||
actions.append(action)
|
||||
@@ -91,7 +92,7 @@ def calc(city, year_month):
|
||||
# 写入MONGODB
|
||||
for index, row in merge2_df.iterrows():
|
||||
action = UpdateOne({'city': row['city'], 'district': row['district'], 'cat': 'district', 'date': year_month},
|
||||
{'$set': {'city': row['city'], 'district': row['district'], 'cat': 'district', 'date': year_month, 'calc_type': 2, 'avg_price': row['sell_price'],
|
||||
{'$set': {'city': row['city'], 'district': row['district'], 'cat': 'district', 'date': year_month, 'calc_type': 2, 'avg_price': int(row['sell_price']),
|
||||
'updated_date': datetime.now()}},
|
||||
upsert=True)
|
||||
actions.append(action)
|
||||
@@ -114,7 +115,7 @@ except Exception as e:
|
||||
|
||||
# # 初始化从2024年9月
|
||||
# 创建从2024年9月到2024年12月的日期范围
|
||||
# date_range = pd.date_range(start='2024-09-01', end='2025-01-31', freq='M')
|
||||
# date_range = pd.date_range(start='2025-01-31', end='2025-01-31', freq='M')
|
||||
#
|
||||
# # 创建一个包含这些日期的DataFrame
|
||||
# df = pd.DataFrame({'date': date_range, 'value': range(1, len(date_range) + 1)})
|
||||
@@ -123,10 +124,9 @@ except Exception as e:
|
||||
# df.set_index('date', inplace=True)
|
||||
#
|
||||
# # 遍历DataFrame
|
||||
# for city in areautils.get_all_cities():
|
||||
# for date, row in df.iterrows():
|
||||
# try:
|
||||
# date_format = int(dateutils.format_date(date, '%Y%m'))
|
||||
# calc(city, date_format)
|
||||
# except Exception as e:
|
||||
# log.error(e)
|
||||
# for date, row in df.iterrows():
|
||||
# try:
|
||||
# date_format = int(dateutils.format_date(date, '%Y%m'))
|
||||
# calc(None, date_format)
|
||||
# except Exception as e:
|
||||
# log.error(e)
|
||||
|
||||
@@ -27,7 +27,7 @@ def calc(city, year_month):
|
||||
office_cleaned = office[np.abs(office['Z_Score']) < 0.1]
|
||||
# 至少样本数大于等于10
|
||||
if len(office_cleaned['day_avg_price']) >= 10:
|
||||
city_price = round(office_cleaned['day_avg_price'].median(), 0)
|
||||
city_price = int(office_cleaned['day_avg_price'].median())
|
||||
region_median_count_prices = office_cleaned.groupby(office_cleaned['region'])['day_avg_price'].agg(['median', 'count']).to_dict()
|
||||
else:
|
||||
city_price = None
|
||||
@@ -52,7 +52,7 @@ def calc(city, year_month):
|
||||
|
||||
# 写入MONGODB
|
||||
action = UpdateOne({'city': city, 'cat': 'city', 'date': year_month},
|
||||
{'$set': {'city': city, 'cat': 'city', 'date': year_month, 'avg_price': city_price,
|
||||
{'$set': {'city': city, 'cat': 'city', 'date': year_month, 'avg_price': int(city_price),
|
||||
'updated_date': datetime.now()}},
|
||||
upsert=True)
|
||||
|
||||
@@ -74,19 +74,20 @@ def calc(city, year_month):
|
||||
|
||||
action = UpdateOne({'city': city, 'cat': 'district', 'date': year_month, 'district': region},
|
||||
{'$set': {'city': city, 'cat': 'district', 'date': year_month, 'district': region,
|
||||
'avg_price': round(avg_price, 0), 'updated_date': datetime.now()}},
|
||||
'avg_price': int(avg_price), 'updated_date': datetime.now()}},
|
||||
upsert=True)
|
||||
actions.append(action)
|
||||
office_sell_stat_conn.bulk_write(actions)
|
||||
log.info(f"{city} {year_month} 均价计算完毕!")
|
||||
|
||||
|
||||
yearmonth = sys.argv[1] if len(sys.argv) > 1 else ''
|
||||
|
||||
if yearmonth == '':
|
||||
date = dateutils.add_months(datetime.today(), months=-1)
|
||||
yearmonth = int(dateutils.format_date(date, '%Y%m'))
|
||||
try:
|
||||
if type(yearmonth) is str:
|
||||
yearmonth = int(yearmonth)
|
||||
for city in areautils.get_all_cities():
|
||||
calc(city, yearmonth)
|
||||
except Exception as e:
|
||||
@@ -94,7 +95,7 @@ except Exception as e:
|
||||
|
||||
# # 初始化从2024年9月
|
||||
# 创建从2024年9月到2024年12月的日期范围
|
||||
# date_range = pd.date_range(start='2024-09-01', end='2025-01-31', freq='M')
|
||||
# date_range = pd.date_range(start='2024-06-01', end='2025-01-31', freq='M')
|
||||
#
|
||||
# # 创建一个包含这些日期的DataFrame
|
||||
# df = pd.DataFrame({'date': date_range, 'value': range(1, len(date_range) + 1)})
|
||||
|
||||
Reference in New Issue
Block a user