版本修改
This commit is contained in:
@@ -6,6 +6,10 @@ import pandas as pd
|
||||
from elasticsearch import helpers
|
||||
import logging
|
||||
log = logging.getLogger(__name__)
|
||||
import os
|
||||
# 🛡️ 极其重要:必须在导入 goojia_utils 之前禁用 ES 8.x 的兼容性 Content-Type 头部请求!
|
||||
# 这样客户端就会使用普通的 'application/json' 头部,彻底解决低版本 Elasticsearch 报错 406 的故障。
|
||||
os.environ["ELASTIC_CLIENT_APIVERSIONING"] = "0"
|
||||
|
||||
flag = False
|
||||
for province in areautils.get_all_province():
|
||||
|
||||
@@ -27,7 +27,6 @@ def choose_task(**context):
|
||||
return ['office_city_region_rent_calc', 'office_base_rent_calc']
|
||||
|
||||
|
||||
|
||||
with DAG(
|
||||
dag_id="office_rent_dag",
|
||||
# 每月25日1点运行
|
||||
@@ -41,7 +40,7 @@ with DAG(
|
||||
branch_task = BranchPythonOperator(
|
||||
task_id='choose_branch',
|
||||
python_callable=choose_task,
|
||||
provide_context=True
|
||||
# 🛡️ 已移除 provide_context=True,防止 Airflow 3.3.0 报错崩溃
|
||||
)
|
||||
# 任务1:城市区域租金计算
|
||||
t1 = BashOperator(
|
||||
@@ -52,7 +51,6 @@ with DAG(
|
||||
t2 = PythonOperator(
|
||||
task_id='office_base_rent_calc',
|
||||
python_callable=calc,
|
||||
provide_context=True
|
||||
# 🛡️ 已移除 provide_context=True,防止 Airflow 3.3.0 报错崩溃
|
||||
)
|
||||
branch_task >> [t1, t2]
|
||||
|
||||
branch_task >> [t1, t2]
|
||||
@@ -4,34 +4,35 @@ import datetime
|
||||
import urllib3
|
||||
from goojia_utils import nextcloudutils, dingutils
|
||||
|
||||
# 禁用 InsecureRequestWarning
|
||||
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
|
||||
if __name__ == "__main__":
|
||||
# 禁用 InsecureRequestWarning
|
||||
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
|
||||
|
||||
# 获取当前日期
|
||||
current_date = datetime.datetime.now()
|
||||
# 计算前一日的日期
|
||||
previous_date = current_date - datetime.timedelta(days=1)
|
||||
# 格式化日期
|
||||
formatted_date = previous_date.strftime("%Y.%m.%d")
|
||||
# 获取当前日期
|
||||
current_date = datetime.datetime.now()
|
||||
# 计算前一日的日期
|
||||
previous_date = current_date - datetime.timedelta(days=1)
|
||||
# 格式化日期
|
||||
formatted_date = previous_date.strftime("%Y.%m.%d")
|
||||
|
||||
data = {
|
||||
"parentCode": "001.002",
|
||||
"dateRange": formatted_date + "," + formatted_date
|
||||
}
|
||||
headers = {
|
||||
"Accept": "*/*",
|
||||
"Content-Type": "application/json",
|
||||
"Authorization": "Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzUxMiJ9.eyJhY2NvdW50IjoiYWlyZmxvdyIsImV4cCI6MTMxMzM5NDE2MzQwNDYxLCJvcmdDb2RlIjoiMDAxLjAwMSIsInJvbGVzIjpbIm5vcm1hbCJdLCJ1c2VySWQiOiIxODg5ODc0MDc4MTgxODMwNjU2IiwidXNlcm5hbWUiOiLlrprml7bku7vliqEifQ.SvD42b8XGlxYe_0dBr215ZgxR9aBdsipaySceZuuWR6g5cHelU5oc-dRtK63b0CMoN07BSvh2XYdlIzQ92ngOA"
|
||||
}
|
||||
data = {
|
||||
"parentCode": "001.002",
|
||||
"dateRange": formatted_date + "," + formatted_date
|
||||
}
|
||||
headers = {
|
||||
"Accept": "*/*",
|
||||
"Content-Type": "application/json",
|
||||
"Authorization": "Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzUxMiJ9.eyJhY2NvdW50IjoiYWlyZmxvdyIsImV4cCI6MTMxMzM5NDE2MzQwNDYxLCJvcmdDb2RlIjoiMDAxLjAwMSIsInJvbGVzIjpbIm5vcm1hbCJdLCJ1c2VySWQiOiIxODg5ODc0MDc4MTgxODMwNjU2IiwidXNlcm5hbWUiOiLlrprml7bku7vliqEifQ.SvD42b8XGlxYe_0dBr215ZgxR9aBdsipaySceZuuWR6g5cHelU5oc-dRtK63b0CMoN07BSvh2XYdlIzQ92ngOA"
|
||||
}
|
||||
|
||||
ret = requests.post(url='https://api.fangjia.com/exportStatistics', json=data, headers=headers, verify=False)
|
||||
#ret = requests.post(url='http://127.0.0.1:8080/exportStatistics', json=data, headers=headers, verify=False)
|
||||
ret = requests.post(url='https://api.fangjia.com/exportStatistics', json=data, headers=headers, verify=False)
|
||||
#ret = requests.post(url='http://127.0.0.1:8080/exportStatistics', json=data, headers=headers, verify=False)
|
||||
|
||||
nextcloud_aim_file_path = '共享/AI估价/农行统计'
|
||||
file_path = "AI估价农行" + formatted_date + ".xlsx"
|
||||
with open(file_path, 'wb') as file:
|
||||
# 将字节流写入文件
|
||||
file.write(ret.content)
|
||||
nextcloudutils.up_to_nextcloud(file_path, nextcloud_aim_file_path + "/" + "AI估价农行" + formatted_date + ".xlsx")
|
||||
nextcloud_aim_file_path = '共享/AI估价/农行统计'
|
||||
file_path = "AI估价农行" + formatted_date + ".xlsx"
|
||||
with open(file_path, 'wb') as file:
|
||||
# 将字节流写入文件
|
||||
file.write(ret.content)
|
||||
nextcloudutils.up_to_nextcloud(file_path, nextcloud_aim_file_path + "/" + "AI估价农行" + formatted_date + ".xlsx")
|
||||
|
||||
dingutils.send_text("【AI估价】农行" + formatted_date + "统计完成, 请点击 https://nextcloud.fangjia.com/s/WK9MzWwH5kjwMBB 查看", "f4f3b03c35c67d097f70affa27e7d47d7a1b6c0eed2438f4d8aa654834419a0b")
|
||||
dingutils.send_text("【AI估价】农行" + formatted_date + "统计完成, 请点击 https://nextcloud.fangjia.com/s/WK9MzWwH5kjwMBB 查看", "f4f3b03c35c67d097f70affa27e7d47d7a1b6c0eed2438f4d8aa654834419a0b")
|
||||
@@ -4,6 +4,7 @@
|
||||
import datetime
|
||||
import logging
|
||||
import pandas as pd
|
||||
import sys # 新增导入系统模块用于安全退出
|
||||
from goojia_utils import MAIN_MONGO_CLIENT
|
||||
from goojia_utils import dateutils
|
||||
from sklearn.mixture import GaussianMixture
|
||||
@@ -11,42 +12,60 @@ from pymongo import InsertOne, DeleteMany, ReplaceOne, UpdateOne
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
district_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'district_stat')
|
||||
district_analy_data_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'district_analy_data')
|
||||
server_area_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'server_area')
|
||||
weekend = list(district_stat_conn.find().sort([("weekend", -1)]).limit(1))[0]['weekend']
|
||||
level_map = {"1": "S", "2": "A", "3": "B", "4": "C", "5": "D"}
|
||||
logger.info(f"开始计算楼盘评级 weekend:{dateutils.format_date(weekend, '%Y-%m-%d')}")
|
||||
for server_area in server_area_conn.find({'display': {'$ne': '全国'}}, {"display": 1}):
|
||||
city = server_area["display"]
|
||||
data = pd.DataFrame(
|
||||
district_stat_conn.find({"city": city, "weekend": weekend},
|
||||
{"_id": 0, "region": 1, "district_name": 1, "avg_price": 1}))
|
||||
if "avg_price" not in data or len(data['avg_price'].unique()) < 5:
|
||||
continue
|
||||
data['city'] = city
|
||||
logger.info(f"开始计算{city}楼盘评级 weekend:{dateutils.format_date(weekend, '%Y-%m-%d')}")
|
||||
# 高斯混合模型
|
||||
gmm = GaussianMixture(n_components=5, covariance_type='tied', max_iter=200, n_init=10).fit(
|
||||
data[["avg_price"]])
|
||||
data.loc[:, 'cluster'] = gmm.predict(data[["avg_price"]])
|
||||
data.loc[:, 'price_mean_cluster'] = data.groupby('cluster')["avg_price"].transform('mean')
|
||||
data.loc[:, "target"] = data['price_mean_cluster'].rank(method='dense', ascending=False).astype(int)
|
||||
data = data.drop(columns=['cluster', 'price_mean_cluster'], axis=1)
|
||||
data["community_rating"] = data['target'].apply(lambda x: level_map[str(x)])
|
||||
opts = []
|
||||
for item in data.to_dict('records'):
|
||||
opts.append(UpdateOne({"city": city, "region": item["region"], "district_name": item["district_name"]},
|
||||
{"$set": {"community_rating": item["community_rating"]}}, upsert=True
|
||||
))
|
||||
# 按照每 1000 条分割 updates 列表
|
||||
batch_size = 1000
|
||||
batches = [opts[i:i + batch_size] for i in range(0, len(opts), batch_size)]
|
||||
# 逐个批次执行批量更新操作
|
||||
cnt = 0
|
||||
for batch in batches:
|
||||
cnt = cnt + 1
|
||||
logger.info(f"{city}楼盘评级更新数据库, 进度:{cnt}/{len(batches)}")
|
||||
result = district_analy_data_conn.bulk_write(batch)
|
||||
print(f"当前批次更新的文档数量: {result.modified_count}")
|
||||
logger.info(f"楼盘评级计算结束{dateutils.format_date(datetime.datetime.utcnow(), '%Y-%m-%d')}")
|
||||
|
||||
# ================== 🛡️ 安全运行入口 ==================
|
||||
|
||||
if __name__ == "__main__":
|
||||
district_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'district_stat')
|
||||
district_analy_data_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'district_analy_data')
|
||||
server_area_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'server_area')
|
||||
|
||||
# 1. 延迟进行重度 MongoDB 查询
|
||||
weekend_list = list(district_stat_conn.find().sort([("weekend", -1)]).limit(1))
|
||||
if not weekend_list:
|
||||
logger.error("在 district_stat 集合中未找到有效的 weekend 记录,程序退出。")
|
||||
sys.exit(1)
|
||||
|
||||
weekend = weekend_list[0]['weekend']
|
||||
level_map = {"1": "S", "2": "A", "3": "B", "4": "C", "5": "D"}
|
||||
|
||||
logger.info(f"开始计算楼盘评级 weekend:{dateutils.format_date(weekend, '%Y-%m-%d')}")
|
||||
|
||||
# 2. 遍历各城市开展机器学习模型评级
|
||||
for server_area in server_area_conn.find({'display': {'$ne': '全国'}}, {"display": 1}):
|
||||
city = server_area["display"]
|
||||
data = pd.DataFrame(
|
||||
district_stat_conn.find({"city": city, "weekend": weekend},
|
||||
{"_id": 0, "region": 1, "district_name": 1, "avg_price": 1}))
|
||||
if "avg_price" not in data or len(data['avg_price'].unique()) < 5:
|
||||
continue
|
||||
data['city'] = city
|
||||
logger.info(f"开始计算{city}楼盘评级 weekend:{dateutils.format_date(weekend, '%Y-%m-%d')}")
|
||||
|
||||
# 高斯混合模型训练及评估(延迟到真正执行时运行,避免污染解析阶段)
|
||||
gmm = GaussianMixture(n_components=5, covariance_type='tied', max_iter=200, n_init=10).fit(
|
||||
data[["avg_price"]])
|
||||
data.loc[:, 'cluster'] = gmm.predict(data[["avg_price"]])
|
||||
data.loc[:, 'price_mean_cluster'] = data.groupby('cluster')["avg_price"].transform('mean')
|
||||
data.loc[:, "target"] = data['price_mean_cluster'].rank(method='dense', ascending=False).astype(int)
|
||||
data = data.drop(columns=['cluster', 'price_mean_cluster'], axis=1)
|
||||
data["community_rating"] = data['target'].apply(lambda x: level_map[str(x)])
|
||||
|
||||
opts = []
|
||||
for item in data.to_dict('records'):
|
||||
opts.append(UpdateOne({"city": city, "region": item["region"], "district_name": item["district_name"]},
|
||||
{"$set": {"community_rating": item["community_rating"]}}, upsert=True
|
||||
))
|
||||
|
||||
# 按照每 1000 条分割 updates 列表并执行 Bulk Write
|
||||
batch_size = 1000
|
||||
batches = [opts[i:i + batch_size] for i in range(0, len(opts), batch_size)]
|
||||
|
||||
cnt = 0
|
||||
for batch in batches:
|
||||
cnt = cnt + 1
|
||||
logger.info(f"{city}楼盘评级更新数据库, 进度:{cnt}/{len(batches)}")
|
||||
result = district_analy_data_conn.bulk_write(batch)
|
||||
print(f"当前批次更新的文档数量: {result.modified_count}")
|
||||
|
||||
logger.info(f"楼盘评级计算结束{dateutils.format_date(datetime.datetime.utcnow(), '%Y-%m-%d')}")
|
||||
Reference in New Issue
Block a user