From b7534064aae2a2cef06a5a923134f6b4278d100c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=96=87=E8=89=B3?= Date: Wed, 22 Jul 2026 16:13:44 +0800 Subject: [PATCH] =?UTF-8?q?=E7=89=88=E6=9C=AC=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/company/company/bd2gaode.py | 4 ++ dags/office/dag_office_rent.py | 8 +-- dags/stats/aiprice/nonghang_use.py | 53 ++++++++-------- dags/stats/community/rate.py | 97 ++++++++++++++++++------------ 4 files changed, 92 insertions(+), 70 deletions(-) diff --git a/dags/company/company/bd2gaode.py b/dags/company/company/bd2gaode.py index 82259ac..a9ffed2 100644 --- a/dags/company/company/bd2gaode.py +++ b/dags/company/company/bd2gaode.py @@ -6,6 +6,10 @@ import pandas as pd from elasticsearch import helpers import logging log = logging.getLogger(__name__) +import os +# 🛡️ 极其重要:必须在导入 goojia_utils 之前禁用 ES 8.x 的兼容性 Content-Type 头部请求! +# 这样客户端就会使用普通的 'application/json' 头部,彻底解决低版本 Elasticsearch 报错 406 的故障。 +os.environ["ELASTIC_CLIENT_APIVERSIONING"] = "0" flag = False for province in areautils.get_all_province(): diff --git a/dags/office/dag_office_rent.py b/dags/office/dag_office_rent.py index b202cb5..2516a11 100644 --- a/dags/office/dag_office_rent.py +++ b/dags/office/dag_office_rent.py @@ -27,7 +27,6 @@ def choose_task(**context): return ['office_city_region_rent_calc', 'office_base_rent_calc'] - with DAG( dag_id="office_rent_dag", # 每月25日1点运行 @@ -41,7 +40,7 @@ with DAG( branch_task = BranchPythonOperator( task_id='choose_branch', python_callable=choose_task, - provide_context=True + # 🛡️ 已移除 provide_context=True,防止 Airflow 3.3.0 报错崩溃 ) # 任务1:城市区域租金计算 t1 = BashOperator( @@ -52,7 +51,6 @@ with DAG( t2 = PythonOperator( task_id='office_base_rent_calc', python_callable=calc, - provide_context=True + # 🛡️ 已移除 provide_context=True,防止 Airflow 3.3.0 报错崩溃 ) - branch_task >> [t1, t2] - + branch_task >> [t1, t2] \ No newline at end of file diff --git a/dags/stats/aiprice/nonghang_use.py b/dags/stats/aiprice/nonghang_use.py index c624bbe..975c031 100644 --- a/dags/stats/aiprice/nonghang_use.py +++ b/dags/stats/aiprice/nonghang_use.py @@ -4,34 +4,35 @@ import datetime import urllib3 from goojia_utils import nextcloudutils, dingutils -# 禁用 InsecureRequestWarning -urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) +if __name__ == "__main__": + # 禁用 InsecureRequestWarning + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) -# 获取当前日期 -current_date = datetime.datetime.now() -# 计算前一日的日期 -previous_date = current_date - datetime.timedelta(days=1) -# 格式化日期 -formatted_date = previous_date.strftime("%Y.%m.%d") + # 获取当前日期 + current_date = datetime.datetime.now() + # 计算前一日的日期 + previous_date = current_date - datetime.timedelta(days=1) + # 格式化日期 + formatted_date = previous_date.strftime("%Y.%m.%d") -data = { - "parentCode": "001.002", - "dateRange": formatted_date + "," + formatted_date -} -headers = { - "Accept": "*/*", - "Content-Type": "application/json", - "Authorization": "Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzUxMiJ9.eyJhY2NvdW50IjoiYWlyZmxvdyIsImV4cCI6MTMxMzM5NDE2MzQwNDYxLCJvcmdDb2RlIjoiMDAxLjAwMSIsInJvbGVzIjpbIm5vcm1hbCJdLCJ1c2VySWQiOiIxODg5ODc0MDc4MTgxODMwNjU2IiwidXNlcm5hbWUiOiLlrprml7bku7vliqEifQ.SvD42b8XGlxYe_0dBr215ZgxR9aBdsipaySceZuuWR6g5cHelU5oc-dRtK63b0CMoN07BSvh2XYdlIzQ92ngOA" -} + data = { + "parentCode": "001.002", + "dateRange": formatted_date + "," + formatted_date + } + headers = { + "Accept": "*/*", + "Content-Type": "application/json", + "Authorization": "Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzUxMiJ9.eyJhY2NvdW50IjoiYWlyZmxvdyIsImV4cCI6MTMxMzM5NDE2MzQwNDYxLCJvcmdDb2RlIjoiMDAxLjAwMSIsInJvbGVzIjpbIm5vcm1hbCJdLCJ1c2VySWQiOiIxODg5ODc0MDc4MTgxODMwNjU2IiwidXNlcm5hbWUiOiLlrprml7bku7vliqEifQ.SvD42b8XGlxYe_0dBr215ZgxR9aBdsipaySceZuuWR6g5cHelU5oc-dRtK63b0CMoN07BSvh2XYdlIzQ92ngOA" + } -ret = requests.post(url='https://api.fangjia.com/exportStatistics', json=data, headers=headers, verify=False) -#ret = requests.post(url='http://127.0.0.1:8080/exportStatistics', json=data, headers=headers, verify=False) + ret = requests.post(url='https://api.fangjia.com/exportStatistics', json=data, headers=headers, verify=False) + #ret = requests.post(url='http://127.0.0.1:8080/exportStatistics', json=data, headers=headers, verify=False) -nextcloud_aim_file_path = '共享/AI估价/农行统计' -file_path = "AI估价农行" + formatted_date + ".xlsx" -with open(file_path, 'wb') as file: - # 将字节流写入文件 - file.write(ret.content) -nextcloudutils.up_to_nextcloud(file_path, nextcloud_aim_file_path + "/" + "AI估价农行" + formatted_date + ".xlsx") + nextcloud_aim_file_path = '共享/AI估价/农行统计' + file_path = "AI估价农行" + formatted_date + ".xlsx" + with open(file_path, 'wb') as file: + # 将字节流写入文件 + file.write(ret.content) + nextcloudutils.up_to_nextcloud(file_path, nextcloud_aim_file_path + "/" + "AI估价农行" + formatted_date + ".xlsx") -dingutils.send_text("【AI估价】农行" + formatted_date + "统计完成, 请点击 https://nextcloud.fangjia.com/s/WK9MzWwH5kjwMBB 查看", "f4f3b03c35c67d097f70affa27e7d47d7a1b6c0eed2438f4d8aa654834419a0b") \ No newline at end of file + dingutils.send_text("【AI估价】农行" + formatted_date + "统计完成, 请点击 https://nextcloud.fangjia.com/s/WK9MzWwH5kjwMBB 查看", "f4f3b03c35c67d097f70affa27e7d47d7a1b6c0eed2438f4d8aa654834419a0b") \ No newline at end of file diff --git a/dags/stats/community/rate.py b/dags/stats/community/rate.py index 6f7c0ce..6cc0681 100644 --- a/dags/stats/community/rate.py +++ b/dags/stats/community/rate.py @@ -4,6 +4,7 @@ import datetime import logging import pandas as pd +import sys # 新增导入系统模块用于安全退出 from goojia_utils import MAIN_MONGO_CLIENT from goojia_utils import dateutils from sklearn.mixture import GaussianMixture @@ -11,42 +12,60 @@ from pymongo import InsertOne, DeleteMany, ReplaceOne, UpdateOne logger = logging.getLogger(__name__) -district_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'district_stat') -district_analy_data_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'district_analy_data') -server_area_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'server_area') -weekend = list(district_stat_conn.find().sort([("weekend", -1)]).limit(1))[0]['weekend'] -level_map = {"1": "S", "2": "A", "3": "B", "4": "C", "5": "D"} -logger.info(f"开始计算楼盘评级 weekend:{dateutils.format_date(weekend, '%Y-%m-%d')}") -for server_area in server_area_conn.find({'display': {'$ne': '全国'}}, {"display": 1}): - city = server_area["display"] - data = pd.DataFrame( - district_stat_conn.find({"city": city, "weekend": weekend}, - {"_id": 0, "region": 1, "district_name": 1, "avg_price": 1})) - if "avg_price" not in data or len(data['avg_price'].unique()) < 5: - continue - data['city'] = city - logger.info(f"开始计算{city}楼盘评级 weekend:{dateutils.format_date(weekend, '%Y-%m-%d')}") - # 高斯混合模型 - gmm = GaussianMixture(n_components=5, covariance_type='tied', max_iter=200, n_init=10).fit( - data[["avg_price"]]) - data.loc[:, 'cluster'] = gmm.predict(data[["avg_price"]]) - data.loc[:, 'price_mean_cluster'] = data.groupby('cluster')["avg_price"].transform('mean') - data.loc[:, "target"] = data['price_mean_cluster'].rank(method='dense', ascending=False).astype(int) - data = data.drop(columns=['cluster', 'price_mean_cluster'], axis=1) - data["community_rating"] = data['target'].apply(lambda x: level_map[str(x)]) - opts = [] - for item in data.to_dict('records'): - opts.append(UpdateOne({"city": city, "region": item["region"], "district_name": item["district_name"]}, - {"$set": {"community_rating": item["community_rating"]}}, upsert=True - )) - # 按照每 1000 条分割 updates 列表 - batch_size = 1000 - batches = [opts[i:i + batch_size] for i in range(0, len(opts), batch_size)] - # 逐个批次执行批量更新操作 - cnt = 0 - for batch in batches: - cnt = cnt + 1 - logger.info(f"{city}楼盘评级更新数据库, 进度:{cnt}/{len(batches)}") - result = district_analy_data_conn.bulk_write(batch) - print(f"当前批次更新的文档数量: {result.modified_count}") -logger.info(f"楼盘评级计算结束{dateutils.format_date(datetime.datetime.utcnow(), '%Y-%m-%d')}") + +# ================== 🛡️ 安全运行入口 ================== + +if __name__ == "__main__": + district_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'district_stat') + district_analy_data_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'district_analy_data') + server_area_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'server_area') + + # 1. 延迟进行重度 MongoDB 查询 + weekend_list = list(district_stat_conn.find().sort([("weekend", -1)]).limit(1)) + if not weekend_list: + logger.error("在 district_stat 集合中未找到有效的 weekend 记录,程序退出。") + sys.exit(1) + + weekend = weekend_list[0]['weekend'] + level_map = {"1": "S", "2": "A", "3": "B", "4": "C", "5": "D"} + + logger.info(f"开始计算楼盘评级 weekend:{dateutils.format_date(weekend, '%Y-%m-%d')}") + + # 2. 遍历各城市开展机器学习模型评级 + for server_area in server_area_conn.find({'display': {'$ne': '全国'}}, {"display": 1}): + city = server_area["display"] + data = pd.DataFrame( + district_stat_conn.find({"city": city, "weekend": weekend}, + {"_id": 0, "region": 1, "district_name": 1, "avg_price": 1})) + if "avg_price" not in data or len(data['avg_price'].unique()) < 5: + continue + data['city'] = city + logger.info(f"开始计算{city}楼盘评级 weekend:{dateutils.format_date(weekend, '%Y-%m-%d')}") + + # 高斯混合模型训练及评估(延迟到真正执行时运行,避免污染解析阶段) + gmm = GaussianMixture(n_components=5, covariance_type='tied', max_iter=200, n_init=10).fit( + data[["avg_price"]]) + data.loc[:, 'cluster'] = gmm.predict(data[["avg_price"]]) + data.loc[:, 'price_mean_cluster'] = data.groupby('cluster')["avg_price"].transform('mean') + data.loc[:, "target"] = data['price_mean_cluster'].rank(method='dense', ascending=False).astype(int) + data = data.drop(columns=['cluster', 'price_mean_cluster'], axis=1) + data["community_rating"] = data['target'].apply(lambda x: level_map[str(x)]) + + opts = [] + for item in data.to_dict('records'): + opts.append(UpdateOne({"city": city, "region": item["region"], "district_name": item["district_name"]}, + {"$set": {"community_rating": item["community_rating"]}}, upsert=True + )) + + # 按照每 1000 条分割 updates 列表并执行 Bulk Write + batch_size = 1000 + batches = [opts[i:i + batch_size] for i in range(0, len(opts), batch_size)] + + cnt = 0 + for batch in batches: + cnt = cnt + 1 + logger.info(f"{city}楼盘评级更新数据库, 进度:{cnt}/{len(batches)}") + result = district_analy_data_conn.bulk_write(batch) + print(f"当前批次更新的文档数量: {result.modified_count}") + + logger.info(f"楼盘评级计算结束{dateutils.format_date(datetime.datetime.utcnow(), '%Y-%m-%d')}") \ No newline at end of file