优化价格类型

This commit is contained in:
沈健
2025-03-31 14:43:22 +08:00
parent 515351d56b
commit dcce114957
5 changed files with 41 additions and 33 deletions
+6 -1
View File
@@ -16,5 +16,10 @@ with DAG(
task_id='office_sell',
bash_command='python /opt/airflow/dags/repo/dags/office/sell/price.py {{dag_run.conf["yearmonth"] if "yearmonth" in dag_run.conf}} ',
)
t1
t2 = BashOperator(
task_id='fill_price',
bash_command='python /opt/airflow/dags/repo/dags/office/sell/fill_price.py {{dag_run.conf["yearmonth"] if "yearmonth" in dag_run.conf}}',
)
t1 >> t2
+18 -14
View File
@@ -76,6 +76,7 @@ def calc(city, year_month):
# 第一步:计算有样本写字楼的基准价
print("有样本数量:" + str(len(merge_df)))
actions = list()
count = 0
for index, item in merge_df.iterrows():
city = item['city']
region = item['region']
@@ -96,9 +97,13 @@ def calc(city, year_month):
actions.append(action)
if len(actions) == 1000:
office_sell_stat_conn.bulk_write(actions)
count += len(actions)
print("已计算" + str(count) + "条数据")
actions.clear()
if len(actions) > 0:
office_sell_stat_conn.bulk_write(actions)
count += len(actions)
print("已计算" + str(count) + "条数据")
actions.clear()
# 第二步 取周边3公里内同等级、同类型的最近10个有样本写字楼的基准价的平均值
@@ -155,9 +160,13 @@ def calc(city, year_month):
actions.append(action)
if len(actions) == 1000:
office_sell_stat_conn.bulk_write(actions)
count += len(actions)
print("已计算" + str(count) + "条数据")
actions.clear()
if len(actions) > 0:
office_sell_stat_conn.bulk_write(actions)
count += len(actions)
print("已计算" + str(count) + "条数据")
actions.clear()
# 系数计算在Jupyter中
@@ -259,9 +268,13 @@ def calc(city, year_month):
actions.append(action)
if len(actions) == 1000:
office_sell_stat_conn.bulk_write(actions)
count += len(actions)
print("已计算" + str(count) + "条数据")
actions.clear()
if len(actions) > 0:
office_sell_stat_conn.bulk_write(actions)
count += len(actions)
print("已计算" + str(count) + "条数据")
actions.clear()
# 第五步
query = {'cat': 'office', 'date': year_month, 'calc_type': {'$in': [1, 2, 3]}}
@@ -331,23 +344,14 @@ def calc(city, year_month):
actions.append(action)
if len(actions) == 1000:
office_sell_stat_conn.bulk_write(actions)
count += len(actions)
print("已计算" + str(count) + "条数据")
actions.clear()
if len(actions) > 0:
office_sell_stat_conn.bulk_write(actions)
count += len(actions)
print("已计算" + str(count) + "条数据")
actions.clear()
yearmonth = sys.argv[1] if len(sys.argv) > 1 else None
for yearmonth in [
"202406",
"202407",
"202408",
"202409",
"202410",
"202411",
"202412",
"202501"
]:
calc(None, yearmonth)
# # calc("广州", yearmonth)
# # calc("武汉", yearmonth)
# calc("深圳", yearmonth)
calc(None, yearmonth)
+1 -3
View File
@@ -60,7 +60,7 @@ def calc(city, year_month):
action = UpdateOne({'cat': 'office', 'city': curr_city, 'district': region, 'name': name, 'date': year_month},
{'$set': {'city': curr_city, 'cat': 'office', 'date': year_month, 'district': region,
'name': name,
'avg_price': sell_price, 'estate_type2': estate_type2, 'bed_kind': bed_kind,
'avg_price': int(sell_price), 'estate_type2': estate_type2, 'bed_kind': bed_kind,
'location': location,
'calc_type': 5, 'updated_date': datetime.now()}},
upsert=True)
@@ -85,8 +85,6 @@ if yearmonth == '':
try:
if type(yearmonth) is str:
yearmonth = int(yearmonth)
# for curr_city in areautils.get_all_cities():
# print("正在处理城市:{}".format(curr_city))
calc(None, yearmonth)
except Exception as e:
print(e)
+10 -10
View File
@@ -15,6 +15,7 @@ office_rent_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'office
def calc(city, year_month):
print(f"开始处理{city}, {year_month}")
query = {'date': year_month, 'cat': 'office'}
if pd.notna(city):
query['city'] = city
@@ -62,7 +63,7 @@ def calc(city, year_month):
# 写入MONGODB
for index, row in merge2_df.iterrows():
action = UpdateOne({'city': row['city'], 'cat': 'city', 'date': year_month},
{'$set': {'city': row['city'], 'cat': 'city', 'date': year_month, 'calc_type': 2, 'avg_price': row['sell_price'],
{'$set': {'city': row['city'], 'cat': 'city', 'date': year_month, 'calc_type': 2, 'avg_price': int(row['sell_price']),
'updated_date': datetime.now()}},
upsert=True)
actions.append(action)
@@ -91,7 +92,7 @@ def calc(city, year_month):
# 写入MONGODB
for index, row in merge2_df.iterrows():
action = UpdateOne({'city': row['city'], 'district': row['district'], 'cat': 'district', 'date': year_month},
{'$set': {'city': row['city'], 'district': row['district'], 'cat': 'district', 'date': year_month, 'calc_type': 2, 'avg_price': row['sell_price'],
{'$set': {'city': row['city'], 'district': row['district'], 'cat': 'district', 'date': year_month, 'calc_type': 2, 'avg_price': int(row['sell_price']),
'updated_date': datetime.now()}},
upsert=True)
actions.append(action)
@@ -114,7 +115,7 @@ except Exception as e:
# # 初始化从2024年9月
# 创建从2024年9月到2024年12月的日期范围
# date_range = pd.date_range(start='2024-09-01', end='2025-01-31', freq='M')
# date_range = pd.date_range(start='2025-01-31', end='2025-01-31', freq='M')
#
# # 创建一个包含这些日期的DataFrame
# df = pd.DataFrame({'date': date_range, 'value': range(1, len(date_range) + 1)})
@@ -123,10 +124,9 @@ except Exception as e:
# df.set_index('date', inplace=True)
#
# # 遍历DataFrame
# for city in areautils.get_all_cities():
# for date, row in df.iterrows():
# try:
# date_format = int(dateutils.format_date(date, '%Y%m'))
# calc(city, date_format)
# except Exception as e:
# log.error(e)
# for date, row in df.iterrows():
# try:
# date_format = int(dateutils.format_date(date, '%Y%m'))
# calc(None, date_format)
# except Exception as e:
# log.error(e)
+6 -5
View File
@@ -27,7 +27,7 @@ def calc(city, year_month):
office_cleaned = office[np.abs(office['Z_Score']) < 0.1]
# 至少样本数大于等于10
if len(office_cleaned['day_avg_price']) >= 10:
city_price = round(office_cleaned['day_avg_price'].median(), 0)
city_price = int(office_cleaned['day_avg_price'].median())
region_median_count_prices = office_cleaned.groupby(office_cleaned['region'])['day_avg_price'].agg(['median', 'count']).to_dict()
else:
city_price = None
@@ -52,7 +52,7 @@ def calc(city, year_month):
# 写入MONGODB
action = UpdateOne({'city': city, 'cat': 'city', 'date': year_month},
{'$set': {'city': city, 'cat': 'city', 'date': year_month, 'avg_price': city_price,
{'$set': {'city': city, 'cat': 'city', 'date': year_month, 'avg_price': int(city_price),
'updated_date': datetime.now()}},
upsert=True)
@@ -74,19 +74,20 @@ def calc(city, year_month):
action = UpdateOne({'city': city, 'cat': 'district', 'date': year_month, 'district': region},
{'$set': {'city': city, 'cat': 'district', 'date': year_month, 'district': region,
'avg_price': round(avg_price, 0), 'updated_date': datetime.now()}},
'avg_price': int(avg_price), 'updated_date': datetime.now()}},
upsert=True)
actions.append(action)
office_sell_stat_conn.bulk_write(actions)
log.info(f"{city} {year_month} 均价计算完毕!")
yearmonth = sys.argv[1] if len(sys.argv) > 1 else ''
if yearmonth == '':
date = dateutils.add_months(datetime.today(), months=-1)
yearmonth = int(dateutils.format_date(date, '%Y%m'))
try:
if type(yearmonth) is str:
yearmonth = int(yearmonth)
for city in areautils.get_all_cities():
calc(city, yearmonth)
except Exception as e:
@@ -94,7 +95,7 @@ except Exception as e:
# # 初始化从2024年9月
# 创建从2024年9月到2024年12月的日期范围
# date_range = pd.date_range(start='2024-09-01', end='2025-01-31', freq='M')
# date_range = pd.date_range(start='2024-06-01', end='2025-01-31', freq='M')
#
# # 创建一个包含这些日期的DataFrame
# df = pd.DataFrame({'date': date_range, 'value': range(1, len(date_range) + 1)})