From 9f9c3b6555a9b5de01ccb659c64439e4010eb329 Mon Sep 17 00:00:00 2001 From: wangwenyan Date: Fri, 12 Jul 2024 16:38:33 +0800 Subject: [PATCH] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E6=A5=BC=E7=9B=98=E8=AF=84?= =?UTF-8?q?=E7=BA=A7=E7=AE=97=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/stat/community/rate.py | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/dags/stat/community/rate.py b/dags/stat/community/rate.py index edd1771..7428b6c 100644 --- a/dags/stat/community/rate.py +++ b/dags/stat/community/rate.py @@ -7,6 +7,7 @@ import pandas as pd from goojia_utils import MAIN_MONGO_CLIENT from goojia_utils import dateutils from sklearn.mixture import GaussianMixture +from pymongo import InsertOne, DeleteMany, ReplaceOne, UpdateOne logger = logging.getLogger(__name__) @@ -33,18 +34,9 @@ for server_area in server_area_conn.find({'display': {'$ne': '全国'}}, {"displ data["community_rating"] = data['target'].apply(lambda x: level_map[str(x)]) opts = [] for item in data.to_dict('records'): - opts.append({ - "updateOne": { - "filter": {"city": city, "region": item["region"], "district_name": item["district_name"]}, - "update": {"$set": {"community_rating": item["community_rating"]}} - } - }) - print({ - "updateOne": { - "filter": {"city": city, "region": item["region"], "district_name": item["district_name"]}, - "update": {"$set": {"community_rating": item["community_rating"]}} - } - }) + opts.append(UpdateOne({"city": city, "region": item["region"], "district_name": item["district_name"]}, + {"$set": {"community_rating": item["community_rating"]}}, upsert=True + )) # 按照每 1000 条分割 updates 列表 batch_size = 1000 batches = [opts[i:i + batch_size] for i in range(0, len(opts), batch_size)] @@ -55,4 +47,4 @@ for server_area in server_area_conn.find({'display': {'$ne': '全国'}}, {"displ logger.info(f"{city}楼盘评级更新数据库, 进度:{cnt}/{len(batches)}") result = district_analy_data_conn.bulk_write(batch) print(f"当前批次更新的文档数量: {result.modified_count}") -logger.info(f"楼盘评级计算结束{dateutils.format_date(datetime.datetime.utcnow(), '%Y-%m-%d')}") \ No newline at end of file +logger.info(f"楼盘评级计算结束{dateutils.format_date(datetime.datetime.utcnow(), '%Y-%m-%d')}")