添加楼盘评级算法
This commit is contained in:
@@ -23,10 +23,13 @@ def sync_district_office(type):
|
||||
for city in cities:
|
||||
actions = list()
|
||||
id_list = list()
|
||||
for doc in seaweed_conn.find({'city': city, 'cat': type, 'status': 0}, {'city': 1, 'region': 1, 'block': 1, 'cat': 1, 'name': 1, 'address': 1,
|
||||
'estate_type2': 1, 'township': 1, 'alias': 1, 'addr_alias': 1, 'lng': 1, 'lat': 1, 'status': 1})\
|
||||
for doc in seaweed_conn.find({'city': city, 'cat': type, 'status': 0},
|
||||
{'city': 1, 'region': 1, 'block': 1, 'cat': 1, 'name': 1, 'address': 1,
|
||||
'estate_type2': 1, 'township': 1, 'alias': 1, 'addr_alias': 1, 'lng': 1, 'lat': 1,
|
||||
'status': 1}) \
|
||||
.batch_size(10000):
|
||||
updateDict = {'city': doc['city'], 'region': doc['region'], 'category': doc['cat'], 'name': doc['name'], 'status': doc['status']}
|
||||
updateDict = {'city': doc['city'], 'region': doc['region'], 'category': doc['cat'], 'name': doc['name'],
|
||||
'status': doc['status']}
|
||||
if 'estate_type2' in doc:
|
||||
updateDict['estate_type'] = doc['estate_type2']
|
||||
|
||||
@@ -42,7 +45,7 @@ def sync_district_office(type):
|
||||
if 'addr_alias' in doc:
|
||||
updateDict['addr_alias'] = doc['addr_alias']
|
||||
if 'lng' in doc and doc['lng'] is not None and str(doc['lng']) != 'nan':
|
||||
updateDict['location'] = {'lon': doc['lng'] / 10**10, 'lat': doc['lat'] / 10**10}
|
||||
updateDict['location'] = {'lon': doc['lng'] / 10 ** 10, 'lat': doc['lat'] / 10 ** 10}
|
||||
updateDict['update_date'] = update_date
|
||||
action = UpdateOne({'_id': doc['_id']}, {'$set': updateDict}, upsert=True)
|
||||
|
||||
@@ -51,14 +54,16 @@ def sync_district_office(type):
|
||||
seaweed_didata_conn.bulk_write(actions)
|
||||
actions.clear()
|
||||
|
||||
didata_id_df = pd.DataFrame(list(seaweed_didata_conn.find({'city': city, 'category': type}, {'_id': 1}).batch_size(10000)))
|
||||
didata_id_df = pd.DataFrame(
|
||||
list(seaweed_didata_conn.find({'city': city, 'category': type}, {'_id': 1}).batch_size(10000)))
|
||||
if len(didata_id_df) > 0:
|
||||
didata_id_df = didata_id_df[['_id']]
|
||||
id_df = pd.DataFrame(id_list)
|
||||
deleted_seaweed_df = pd.concat([didata_id_df, id_df, id_df]).drop_duplicates(keep=False)
|
||||
|
||||
for deleted_seaweed_index, deleted_seaweed_item in deleted_seaweed_df.iterrows():
|
||||
action = UpdateOne({'_id': ObjectId(deleted_seaweed_item['_id'])}, {'$set': {'status': 1, 'update_date': update_date}}, upsert=False)
|
||||
action = UpdateOne({'_id': ObjectId(deleted_seaweed_item['_id'])},
|
||||
{'$set': {'status': 1, 'update_date': update_date}}, upsert=False)
|
||||
actions.append(action)
|
||||
if len(actions) == 10000:
|
||||
seaweed_didata_conn.bulk_write(actions)
|
||||
@@ -69,8 +74,9 @@ def sync_district_office(type):
|
||||
actions.clear()
|
||||
log.info(city + "-" + type + "-同步完成")
|
||||
|
||||
|
||||
if type == 'all':
|
||||
sync_district_office("district")
|
||||
sync_district_office("office")
|
||||
else:
|
||||
sync_district_office(type)
|
||||
sync_district_office(type)
|
||||
|
||||
+2
-2
@@ -1,2 +1,2 @@
|
||||
pip install -r /opt/airflow/dags/repo/dags/requirements.txt -i https://repo.fangjia.com/repository/pypi-public/simple/
|
||||
pip install -I goojia_utils -i https://repo.fangjia.com/repository/pypi-public/simple/
|
||||
pip install -I goojia_utils -i https://repo.fangjia.com/repository/pypi-public/simple/
|
||||
pip install -r /opt/airflow/dags/repo/dags/requirements.txt -i https://repo.fangjia.com/repository/pypi-public/simple/
|
||||
@@ -0,0 +1,58 @@
|
||||
"""
|
||||
小区周边1公里交通地铁入库业务逻辑
|
||||
"""
|
||||
import datetime
|
||||
import logging
|
||||
import pandas as pd
|
||||
from goojia_utils import MAIN_MONGO_CLIENT
|
||||
from goojia_utils import dateutils
|
||||
from sklearn.mixture import GaussianMixture
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
district_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'district_stat')
|
||||
district_analy_data_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'district_analy_data')
|
||||
server_area_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'server_area')
|
||||
weekend = list(district_stat_conn.find().sort([("weekend", -1)]).limit(1))[0]['weekend']
|
||||
level_map = {"1": "S", "2": "A", "3": "B", "4": "C", "5": "D"}
|
||||
logger.info(f"开始计算楼盘评级 weekend:{dateutils.format_date(weekend, '%Y-%m-%d')}")
|
||||
for server_area in server_area_conn.find({'display': {'$ne': '全国'}}, {"display": 1}):
|
||||
city = server_area["display"]
|
||||
data = pd.DataFrame(
|
||||
district_stat_conn.find({"city": city, "weekend": weekend},
|
||||
{"_id": 0, "region": 1, "district_name": 1, "avg_price": 1}))
|
||||
data['city'] = city
|
||||
logger.info(f"开始计算{city}楼盘评级 weekend:{dateutils.format_date(weekend, '%Y-%m-%d')}")
|
||||
# 高斯混合模型
|
||||
gmm = GaussianMixture(n_components=5, covariance_type='tied', max_iter=200, n_init=10).fit(
|
||||
data[["avg_price"]])
|
||||
data.loc[:, 'cluster'] = gmm.predict(data[["avg_price"]])
|
||||
data.loc[:, 'price_mean_cluster'] = data.groupby('cluster')["avg_price"].transform('mean')
|
||||
data.loc[:, "target"] = data['price_mean_cluster'].rank(method='dense', ascending=False).astype(int)
|
||||
data = data.drop(columns=['cluster', 'price_mean_cluster'], axis=1)
|
||||
data["community_rating"] = data['target'].apply(lambda x: level_map[str(x)])
|
||||
opts = []
|
||||
for item in data.to_dict('records'):
|
||||
opts.append({
|
||||
"updateOne": {
|
||||
"filter": {"city": city, "region": item["region"], "district_name": item["district_name"]},
|
||||
"update": {"$set": {"community_rating": item["community_rating"]}}
|
||||
}
|
||||
})
|
||||
print({
|
||||
"updateOne": {
|
||||
"filter": {"city": city, "region": item["region"], "district_name": item["district_name"]},
|
||||
"update": {"$set": {"community_rating": item["community_rating"]}}
|
||||
}
|
||||
})
|
||||
# 按照每 1000 条分割 updates 列表
|
||||
batch_size = 1000
|
||||
batches = [opts[i:i + batch_size] for i in range(0, len(opts), batch_size)]
|
||||
# 逐个批次执行批量更新操作
|
||||
cnt = 0
|
||||
for batch in batches:
|
||||
cnt = cnt + 1
|
||||
logger.info(f"{city}楼盘评级更新数据库, 进度:{cnt}/{len(batches)}")
|
||||
result = district_analy_data_conn.bulk_write(batch)
|
||||
print(f"当前批次更新的文档数量: {result.modified_count}")
|
||||
logger.info(f"楼盘评级计算结束{dateutils.format_date(datetime.datetime.utcnow(), '%Y-%m-%d')}")
|
||||
@@ -0,0 +1,16 @@
|
||||
from airflow.operators.bash import BashOperator
|
||||
import pendulum
|
||||
from airflow import DAG
|
||||
|
||||
with DAG(
|
||||
dag_id="community_rating_dag",
|
||||
# 每月的最后一天运行
|
||||
schedule_interval="0 20 * * 6",
|
||||
description="小区楼盘评级计算",
|
||||
catchup=True
|
||||
) as dag:
|
||||
t1 = BashOperator(
|
||||
task_id='community_rating',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/stat/community/rate.py'
|
||||
)
|
||||
t1
|
||||
Reference in New Issue
Block a user