新增评级

This commit is contained in:
shenjian@fangjia.com
2022-10-25 11:19:27 +08:00
parent b0edc7de17
commit 33174c43f3
6 changed files with 153 additions and 58 deletions
@@ -4,8 +4,15 @@
from goojia_utils import ElasticSearchClient
import pandas as pd
import numpy as np
import logging
from elasticsearch import helpers
from sklearn.mixture import GaussianMixture
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
index_name = "community_summary"
def calc_consumption_potential(city):
@@ -39,13 +46,11 @@ def calc_consumption_potential(city):
community_summary_df['total_price_score'].fillna(0, inplace=True)
community_summary_df['estate_charge_score'] = 139.1374524 * (
community_summary_df['estate_charge'] - community_summary_df['estate_charge_mean']) / \
community_summary_df['estate_charge_std']
community_summary_df['estate_charge'] - community_summary_df['estate_charge_mean']) / community_summary_df['estate_charge_std']
community_summary_df['estate_charge_score'].fillna(0, inplace=True)
community_summary_df['relative_price_ranking_score'] = 21.8427926 * (
community_summary_df['relative_price_ranking'] - community_summary_df['relative_price_ranking_mean']) / \
community_summary_df['relative_price_ranking_std']
community_summary_df['relative_price_ranking'] - community_summary_df['relative_price_ranking_mean']) / community_summary_df['relative_price_ranking_std']
community_summary_df['relative_price_ranking_score'].fillna(0, inplace=True)
community_summary_df['avg_cost_score'] = 67.0195871700 * (
@@ -54,22 +59,64 @@ def calc_consumption_potential(city):
community_summary_df['avg_cost_score'].fillna(0, inplace=True)
community_summary_df['consumption_potential'] = community_summary_df['total_price_score'] + community_summary_df[
'estate_charge_score'] + community_summary_df['relative_price_ranking_score'] \
+ community_summary_df['avg_cost_score'] + community_summary_df[
'avg_cost_score'] + 4099.978248
pass
'estate_charge_score'] + community_summary_df['relative_price_ranking_score'] + community_summary_df['avg_cost_score'] + community_summary_df['avg_cost_score'] + 4099.978248
return community_summary_df
def calc_consumption_rating():
def gmm_train(total_rate, consumption_df, source, target):
# 高斯混合模型
gmm = GaussianMixture(n_components=total_rate).fit(consumption_df[[source]])
consumption_df.loc[:, 'cluster'] = gmm.predict(consumption_df[[source]])
consumption_df.loc[:, 'price_mean_cluster'] = consumption_df.groupby('cluster')[source].transform('mean')
consumption_df.loc[:, target] = consumption_df['price_mean_cluster'].rank(method='dense').astype(int)
consumption_df = consumption_df.drop(columns=['cluster', 'price_mean_cluster'], axis=1)
return consumption_df
def calc_consumption_rating(community_summary_df):
"""
计算消费等级
"""
pass
consumption_aim_df = community_summary_df[['_id', 'city', 'consumption_potential']]
consumption_aim_df = consumption_aim_df.dropna(subset=['consumption_potential'])
cities = consumption_aim_df['city'].unique()
source = "consumption_potential"
target = "consumption_rating"
actions = list()
count = 0
for city in cities:
df = consumption_aim_df[consumption_aim_df['city'] == city]
if len(df) < 10:
continue
df = gmm_train(10, df, source, target)
# 批量更新
for index, item in df.iterrows():
row = dict()
_id = str(item["_id"])
row['consumption_rating'] = item['consumption_rating']
row['consumption_potential'] = item['consumption_potential']
action = {'_op_type': 'update', # 操作 index update create delete
'_index': index_name, # index
'_id': _id, # 请勿忘记ID
'doc': row}
actions.append(action)
if len(actions) == 5000:
helpers.bulk(client=es206_client.elastic_client, actions=actions)
count += len(actions)
actions.clear()
log.info("评级中: 当前已评级{}".format(count))
if len(actions) > 0:
helpers.bulk(client=es206_client.elastic_client, actions=actions)
count += len(actions)
actions.clear()
log.info("评级完成,共更新: {}".format(count))
def sync(city=None):
calc_consumption_potential(city)
pass
community_summary_df = calc_consumption_potential(city)
calc_consumption_rating(community_summary_df)
sync()
@@ -0,0 +1,84 @@
"""
楼盘价格评级
"""
"""
月消费潜力、评级计算
"""
from goojia_utils import ElasticSearchClient
import pandas as pd
import logging
from elasticsearch import helpers
from sklearn.mixture import GaussianMixture
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
index_name = "community_summary"
def gmm_train(total_rate, consumption_df, source, target):
# 高斯混合模型
gmm = GaussianMixture(n_components=total_rate).fit(consumption_df[[source]])
consumption_df.loc[:, 'cluster'] = gmm.predict(consumption_df[[source]])
consumption_df.loc[:, 'price_mean_cluster'] = consumption_df.groupby('cluster')[source].transform('mean')
consumption_df.loc[:, target] = consumption_df['price_mean_cluster'].rank(method='dense').astype(int)
consumption_df = consumption_df.drop(columns=['cluster', 'price_mean_cluster'], axis=1)
return consumption_df
def calc_price_rating(city=None):
"""
计算消费等级
"""
sql = "select id, city, region, name, avg_price from community_summary"
if pd.notna(city):
sql += " where city='" + city + "'"
community_summary_df = es206_client.scan_by_sql_to_data_frame(sql)
consumption_aim_df = community_summary_df[['_id', 'city', 'avg_price']]
consumption_aim_df = consumption_aim_df.dropna(subset=['avg_price'])
cities = consumption_aim_df['city'].unique()
source = "avg_price"
target = "price_rating"
score_target = "price_score"
actions = list()
count = 0
for city in cities:
df = consumption_aim_df[consumption_aim_df['city'] == city]
if len(df) < 10:
continue
df = gmm_train(10, df, source, target)
if len(df) >= 100:
df = gmm_train(100, df, source, score_target)
df['price_score'] /= 10
# 批量更新
for index, item in df.iterrows():
row = dict()
_id = str(item["_id"])
row['price_rating'] = item['price_rating']
row['avg_price'] = item['avg_price']
action = {'_op_type': 'update',
'_index': index_name,
'_id': _id, # 请勿忘记ID
'doc': row}
actions.append(action)
if len(actions) == 5000:
helpers.bulk(client=es206_client.elastic_client, actions=actions)
count += len(actions)
actions.clear()
log.info("评级中: 当前已评级{}".format(count))
if len(actions) > 0:
helpers.bulk(client=es206_client.elastic_client, actions=actions)
count += len(actions)
actions.clear()
log.info("评级完成,共更新: {}".format(count))
def sync(city=None):
calc_price_rating(city)
sync()
-25
View File
@@ -1,25 +0,0 @@
"""
消费或价格评级定时运行
"""
import requests
import logging
import sys
log = logging.getLogger(__name__)
type = sys.argv[1] if len(sys.argv) > 1 else ''
print("输出评级类型" + type)
if type == 'price':
log.info("定时楼盘评级开始。。。。。")
headers = {'Content-Type': 'application/json'}
url = "http://192.168.0.19:31080/FangJiaNlpService/price_rate"
if requests.get(url=url, headers=headers).text == 'success':
log.info("定时楼盘评级调用完毕。。。。。")
else:
log.info("定时消费评级开始。。。。。")
headers = {'Content-Type': 'application/json'}
url = "http://192.168.0.19:31080/FangJiaNlpService/consumption_rate"
if requests.get(url=url, headers=headers).text == 'success':
log.info("定时消费评级调用完毕。。。。。")
+8 -3
View File
@@ -51,8 +51,13 @@ with DAG(
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/relative_price_ranking.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
)
t11 = BashOperator(
task_id='consumption_potential_dag',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/consumption_potential.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
task_id='consumption_dag',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/consumption.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
)
t12 = BashOperator(
task_id='price_rating_dag',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/price_rating.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
)
t1 >> t2 >> t3 >> t4 >> t5 >> t6 >> t7 >> t8 >> t9 >> t10 >> t11
# t1 >> t2 >> t3 >> t4 >> t5 >> t6 >> t7 >> t8 >> t9 >> t10 >> t11 >> t12
t11 >> t12
-17
View File
@@ -1,17 +0,0 @@
from airflow.operators.bash import BashOperator
import pendulum
from airflow import DAG
with DAG(
"rate_dag",
# 每月3日16点整分运行
schedule_interval="0 16 3 * *",
start_date=pendulum.datetime(2022, 7, 20, tz="Asia/Shanghai"),
catchup=True,
description="地数价格与消费评级 指定type: price或consumption, 不指定则只对消费评级"
) as dag:
t1 = BashOperator(
task_id='rate_dag',
bash_command='python /opt/airflow/dags/repo/dags/business/business/rate.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
)
t1
+2 -1
View File
@@ -1,3 +1,4 @@
goojia_utils
nextcloud-api-wrapper
elasticsearch==7.11.0
elasticsearch==7.11.0
scikit-learn==0.24.2