diff --git a/dags/stat/community/rate.py b/dags/stat/community/rate.py index edd1771..7428b6c 100644 --- a/dags/stat/community/rate.py +++ b/dags/stat/community/rate.py @@ -7,6 +7,7 @@ import pandas as pd from goojia_utils import MAIN_MONGO_CLIENT from goojia_utils import dateutils from sklearn.mixture import GaussianMixture +from pymongo import InsertOne, DeleteMany, ReplaceOne, UpdateOne logger = logging.getLogger(__name__) @@ -33,18 +34,9 @@ for server_area in server_area_conn.find({'display': {'$ne': '全国'}}, {"displ data["community_rating"] = data['target'].apply(lambda x: level_map[str(x)]) opts = [] for item in data.to_dict('records'): - opts.append({ - "updateOne": { - "filter": {"city": city, "region": item["region"], "district_name": item["district_name"]}, - "update": {"$set": {"community_rating": item["community_rating"]}} - } - }) - print({ - "updateOne": { - "filter": {"city": city, "region": item["region"], "district_name": item["district_name"]}, - "update": {"$set": {"community_rating": item["community_rating"]}} - } - }) + opts.append(UpdateOne({"city": city, "region": item["region"], "district_name": item["district_name"]}, + {"$set": {"community_rating": item["community_rating"]}}, upsert=True + )) # 按照每 1000 条分割 updates 列表 batch_size = 1000 batches = [opts[i:i + batch_size] for i in range(0, len(opts), batch_size)] @@ -55,4 +47,4 @@ for server_area in server_area_conn.find({'display': {'$ne': '全国'}}, {"displ logger.info(f"{city}楼盘评级更新数据库, 进度:{cnt}/{len(batches)}") result = district_analy_data_conn.bulk_write(batch) print(f"当前批次更新的文档数量: {result.modified_count}") -logger.info(f"楼盘评级计算结束{dateutils.format_date(datetime.datetime.utcnow(), '%Y-%m-%d')}") \ No newline at end of file +logger.info(f"楼盘评级计算结束{dateutils.format_date(datetime.datetime.utcnow(), '%Y-%m-%d')}")