diff --git a/dags/business/business/didata_sync.py b/dags/business/business/didata_sync.py index d8b8b14..b00a59d 100644 --- a/dags/business/business/didata_sync.py +++ b/dags/business/business/didata_sync.py @@ -23,10 +23,13 @@ def sync_district_office(type): for city in cities: actions = list() id_list = list() - for doc in seaweed_conn.find({'city': city, 'cat': type, 'status': 0}, {'city': 1, 'region': 1, 'block': 1, 'cat': 1, 'name': 1, 'address': 1, - 'estate_type2': 1, 'township': 1, 'alias': 1, 'addr_alias': 1, 'lng': 1, 'lat': 1, 'status': 1})\ + for doc in seaweed_conn.find({'city': city, 'cat': type, 'status': 0}, + {'city': 1, 'region': 1, 'block': 1, 'cat': 1, 'name': 1, 'address': 1, + 'estate_type2': 1, 'township': 1, 'alias': 1, 'addr_alias': 1, 'lng': 1, 'lat': 1, + 'status': 1}) \ .batch_size(10000): - updateDict = {'city': doc['city'], 'region': doc['region'], 'category': doc['cat'], 'name': doc['name'], 'status': doc['status']} + updateDict = {'city': doc['city'], 'region': doc['region'], 'category': doc['cat'], 'name': doc['name'], + 'status': doc['status']} if 'estate_type2' in doc: updateDict['estate_type'] = doc['estate_type2'] @@ -42,7 +45,7 @@ def sync_district_office(type): if 'addr_alias' in doc: updateDict['addr_alias'] = doc['addr_alias'] if 'lng' in doc and doc['lng'] is not None and str(doc['lng']) != 'nan': - updateDict['location'] = {'lon': doc['lng'] / 10**10, 'lat': doc['lat'] / 10**10} + updateDict['location'] = {'lon': doc['lng'] / 10 ** 10, 'lat': doc['lat'] / 10 ** 10} updateDict['update_date'] = update_date action = UpdateOne({'_id': doc['_id']}, {'$set': updateDict}, upsert=True) @@ -51,14 +54,16 @@ def sync_district_office(type): seaweed_didata_conn.bulk_write(actions) actions.clear() - didata_id_df = pd.DataFrame(list(seaweed_didata_conn.find({'city': city, 'category': type}, {'_id': 1}).batch_size(10000))) + didata_id_df = pd.DataFrame( + list(seaweed_didata_conn.find({'city': city, 'category': type}, {'_id': 1}).batch_size(10000))) if len(didata_id_df) > 0: didata_id_df = didata_id_df[['_id']] id_df = pd.DataFrame(id_list) deleted_seaweed_df = pd.concat([didata_id_df, id_df, id_df]).drop_duplicates(keep=False) for deleted_seaweed_index, deleted_seaweed_item in deleted_seaweed_df.iterrows(): - action = UpdateOne({'_id': ObjectId(deleted_seaweed_item['_id'])}, {'$set': {'status': 1, 'update_date': update_date}}, upsert=False) + action = UpdateOne({'_id': ObjectId(deleted_seaweed_item['_id'])}, + {'$set': {'status': 1, 'update_date': update_date}}, upsert=False) actions.append(action) if len(actions) == 10000: seaweed_didata_conn.bulk_write(actions) @@ -69,8 +74,9 @@ def sync_district_office(type): actions.clear() log.info(city + "-" + type + "-同步完成") + if type == 'all': sync_district_office("district") sync_district_office("office") else: - sync_district_office(type) \ No newline at end of file + sync_district_office(type) diff --git a/dags/install.bat b/dags/install.bat index 156cf1b..b3137e8 100644 --- a/dags/install.bat +++ b/dags/install.bat @@ -1,2 +1,2 @@ -pip install -r /opt/airflow/dags/repo/dags/requirements.txt -i https://repo.fangjia.com/repository/pypi-public/simple/ -pip install -I goojia_utils -i https://repo.fangjia.com/repository/pypi-public/simple/ \ No newline at end of file +pip install -I goojia_utils -i https://repo.fangjia.com/repository/pypi-public/simple/ +pip install -r /opt/airflow/dags/repo/dags/requirements.txt -i https://repo.fangjia.com/repository/pypi-public/simple/ \ No newline at end of file diff --git a/dags/stat/community/rate.py b/dags/stat/community/rate.py new file mode 100644 index 0000000..edd1771 --- /dev/null +++ b/dags/stat/community/rate.py @@ -0,0 +1,58 @@ +""" +小区周边1公里交通地铁入库业务逻辑 +""" +import datetime +import logging +import pandas as pd +from goojia_utils import MAIN_MONGO_CLIENT +from goojia_utils import dateutils +from sklearn.mixture import GaussianMixture + +logger = logging.getLogger(__name__) + +district_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'district_stat') +district_analy_data_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'district_analy_data') +server_area_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'server_area') +weekend = list(district_stat_conn.find().sort([("weekend", -1)]).limit(1))[0]['weekend'] +level_map = {"1": "S", "2": "A", "3": "B", "4": "C", "5": "D"} +logger.info(f"开始计算楼盘评级 weekend:{dateutils.format_date(weekend, '%Y-%m-%d')}") +for server_area in server_area_conn.find({'display': {'$ne': '全国'}}, {"display": 1}): + city = server_area["display"] + data = pd.DataFrame( + district_stat_conn.find({"city": city, "weekend": weekend}, + {"_id": 0, "region": 1, "district_name": 1, "avg_price": 1})) + data['city'] = city + logger.info(f"开始计算{city}楼盘评级 weekend:{dateutils.format_date(weekend, '%Y-%m-%d')}") + # 高斯混合模型 + gmm = GaussianMixture(n_components=5, covariance_type='tied', max_iter=200, n_init=10).fit( + data[["avg_price"]]) + data.loc[:, 'cluster'] = gmm.predict(data[["avg_price"]]) + data.loc[:, 'price_mean_cluster'] = data.groupby('cluster')["avg_price"].transform('mean') + data.loc[:, "target"] = data['price_mean_cluster'].rank(method='dense', ascending=False).astype(int) + data = data.drop(columns=['cluster', 'price_mean_cluster'], axis=1) + data["community_rating"] = data['target'].apply(lambda x: level_map[str(x)]) + opts = [] + for item in data.to_dict('records'): + opts.append({ + "updateOne": { + "filter": {"city": city, "region": item["region"], "district_name": item["district_name"]}, + "update": {"$set": {"community_rating": item["community_rating"]}} + } + }) + print({ + "updateOne": { + "filter": {"city": city, "region": item["region"], "district_name": item["district_name"]}, + "update": {"$set": {"community_rating": item["community_rating"]}} + } + }) + # 按照每 1000 条分割 updates 列表 + batch_size = 1000 + batches = [opts[i:i + batch_size] for i in range(0, len(opts), batch_size)] + # 逐个批次执行批量更新操作 + cnt = 0 + for batch in batches: + cnt = cnt + 1 + logger.info(f"{city}楼盘评级更新数据库, 进度:{cnt}/{len(batches)}") + result = district_analy_data_conn.bulk_write(batch) + print(f"当前批次更新的文档数量: {result.modified_count}") +logger.info(f"楼盘评级计算结束{dateutils.format_date(datetime.datetime.utcnow(), '%Y-%m-%d')}") \ No newline at end of file diff --git a/dags/stat/dag_community_rating.py b/dags/stat/dag_community_rating.py new file mode 100644 index 0000000..415bb92 --- /dev/null +++ b/dags/stat/dag_community_rating.py @@ -0,0 +1,16 @@ +from airflow.operators.bash import BashOperator +import pendulum +from airflow import DAG + +with DAG( + dag_id="community_rating_dag", + # 每月的最后一天运行 + schedule_interval="0 20 * * 6", + description="小区楼盘评级计算", + catchup=True +) as dag: + t1 = BashOperator( + task_id='community_rating', + bash_command='python /opt/airflow/dags/repo/dags/stat/community/rate.py' + ) + t1 \ No newline at end of file