新增评级
This commit is contained in:
+59
-12
@@ -4,8 +4,15 @@
|
||||
from goojia_utils import ElasticSearchClient
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
import logging
|
||||
from elasticsearch import helpers
|
||||
|
||||
from sklearn.mixture import GaussianMixture
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
index_name = "community_summary"
|
||||
|
||||
|
||||
def calc_consumption_potential(city):
|
||||
@@ -39,13 +46,11 @@ def calc_consumption_potential(city):
|
||||
community_summary_df['total_price_score'].fillna(0, inplace=True)
|
||||
|
||||
community_summary_df['estate_charge_score'] = 139.1374524 * (
|
||||
community_summary_df['estate_charge'] - community_summary_df['estate_charge_mean']) / \
|
||||
community_summary_df['estate_charge_std']
|
||||
community_summary_df['estate_charge'] - community_summary_df['estate_charge_mean']) / community_summary_df['estate_charge_std']
|
||||
community_summary_df['estate_charge_score'].fillna(0, inplace=True)
|
||||
|
||||
community_summary_df['relative_price_ranking_score'] = 21.8427926 * (
|
||||
community_summary_df['relative_price_ranking'] - community_summary_df['relative_price_ranking_mean']) / \
|
||||
community_summary_df['relative_price_ranking_std']
|
||||
community_summary_df['relative_price_ranking'] - community_summary_df['relative_price_ranking_mean']) / community_summary_df['relative_price_ranking_std']
|
||||
community_summary_df['relative_price_ranking_score'].fillna(0, inplace=True)
|
||||
|
||||
community_summary_df['avg_cost_score'] = 67.0195871700 * (
|
||||
@@ -54,22 +59,64 @@ def calc_consumption_potential(city):
|
||||
community_summary_df['avg_cost_score'].fillna(0, inplace=True)
|
||||
|
||||
community_summary_df['consumption_potential'] = community_summary_df['total_price_score'] + community_summary_df[
|
||||
'estate_charge_score'] + community_summary_df['relative_price_ranking_score'] \
|
||||
+ community_summary_df['avg_cost_score'] + community_summary_df[
|
||||
'avg_cost_score'] + 4099.978248
|
||||
pass
|
||||
'estate_charge_score'] + community_summary_df['relative_price_ranking_score'] + community_summary_df['avg_cost_score'] + community_summary_df['avg_cost_score'] + 4099.978248
|
||||
return community_summary_df
|
||||
|
||||
|
||||
def calc_consumption_rating():
|
||||
def gmm_train(total_rate, consumption_df, source, target):
|
||||
# 高斯混合模型
|
||||
gmm = GaussianMixture(n_components=total_rate).fit(consumption_df[[source]])
|
||||
consumption_df.loc[:, 'cluster'] = gmm.predict(consumption_df[[source]])
|
||||
consumption_df.loc[:, 'price_mean_cluster'] = consumption_df.groupby('cluster')[source].transform('mean')
|
||||
consumption_df.loc[:, target] = consumption_df['price_mean_cluster'].rank(method='dense').astype(int)
|
||||
consumption_df = consumption_df.drop(columns=['cluster', 'price_mean_cluster'], axis=1)
|
||||
return consumption_df
|
||||
|
||||
|
||||
def calc_consumption_rating(community_summary_df):
|
||||
"""
|
||||
计算消费等级
|
||||
"""
|
||||
pass
|
||||
consumption_aim_df = community_summary_df[['_id', 'city', 'consumption_potential']]
|
||||
consumption_aim_df = consumption_aim_df.dropna(subset=['consumption_potential'])
|
||||
cities = consumption_aim_df['city'].unique()
|
||||
source = "consumption_potential"
|
||||
target = "consumption_rating"
|
||||
actions = list()
|
||||
count = 0
|
||||
|
||||
for city in cities:
|
||||
df = consumption_aim_df[consumption_aim_df['city'] == city]
|
||||
if len(df) < 10:
|
||||
continue
|
||||
df = gmm_train(10, df, source, target)
|
||||
# 批量更新
|
||||
for index, item in df.iterrows():
|
||||
row = dict()
|
||||
_id = str(item["_id"])
|
||||
row['consumption_rating'] = item['consumption_rating']
|
||||
row['consumption_potential'] = item['consumption_potential']
|
||||
|
||||
action = {'_op_type': 'update', # 操作 index update create delete
|
||||
'_index': index_name, # index
|
||||
'_id': _id, # 请勿忘记ID
|
||||
'doc': row}
|
||||
actions.append(action)
|
||||
if len(actions) == 5000:
|
||||
helpers.bulk(client=es206_client.elastic_client, actions=actions)
|
||||
count += len(actions)
|
||||
actions.clear()
|
||||
log.info("评级中: 当前已评级{}".format(count))
|
||||
if len(actions) > 0:
|
||||
helpers.bulk(client=es206_client.elastic_client, actions=actions)
|
||||
count += len(actions)
|
||||
actions.clear()
|
||||
log.info("评级完成,共更新: {}".format(count))
|
||||
|
||||
|
||||
def sync(city=None):
|
||||
calc_consumption_potential(city)
|
||||
pass
|
||||
community_summary_df = calc_consumption_potential(city)
|
||||
calc_consumption_rating(community_summary_df)
|
||||
|
||||
|
||||
sync()
|
||||
@@ -0,0 +1,84 @@
|
||||
"""
|
||||
楼盘价格评级
|
||||
"""
|
||||
"""
|
||||
月消费潜力、评级计算
|
||||
"""
|
||||
from goojia_utils import ElasticSearchClient
|
||||
import pandas as pd
|
||||
import logging
|
||||
from elasticsearch import helpers
|
||||
|
||||
from sklearn.mixture import GaussianMixture
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
index_name = "community_summary"
|
||||
|
||||
def gmm_train(total_rate, consumption_df, source, target):
|
||||
# 高斯混合模型
|
||||
gmm = GaussianMixture(n_components=total_rate).fit(consumption_df[[source]])
|
||||
consumption_df.loc[:, 'cluster'] = gmm.predict(consumption_df[[source]])
|
||||
consumption_df.loc[:, 'price_mean_cluster'] = consumption_df.groupby('cluster')[source].transform('mean')
|
||||
consumption_df.loc[:, target] = consumption_df['price_mean_cluster'].rank(method='dense').astype(int)
|
||||
consumption_df = consumption_df.drop(columns=['cluster', 'price_mean_cluster'], axis=1)
|
||||
return consumption_df
|
||||
|
||||
|
||||
def calc_price_rating(city=None):
|
||||
"""
|
||||
计算消费等级
|
||||
"""
|
||||
sql = "select id, city, region, name, avg_price from community_summary"
|
||||
if pd.notna(city):
|
||||
sql += " where city='" + city + "'"
|
||||
community_summary_df = es206_client.scan_by_sql_to_data_frame(sql)
|
||||
|
||||
consumption_aim_df = community_summary_df[['_id', 'city', 'avg_price']]
|
||||
consumption_aim_df = consumption_aim_df.dropna(subset=['avg_price'])
|
||||
cities = consumption_aim_df['city'].unique()
|
||||
source = "avg_price"
|
||||
target = "price_rating"
|
||||
score_target = "price_score"
|
||||
actions = list()
|
||||
count = 0
|
||||
|
||||
for city in cities:
|
||||
df = consumption_aim_df[consumption_aim_df['city'] == city]
|
||||
if len(df) < 10:
|
||||
continue
|
||||
df = gmm_train(10, df, source, target)
|
||||
if len(df) >= 100:
|
||||
df = gmm_train(100, df, source, score_target)
|
||||
df['price_score'] /= 10
|
||||
|
||||
# 批量更新
|
||||
for index, item in df.iterrows():
|
||||
row = dict()
|
||||
_id = str(item["_id"])
|
||||
row['price_rating'] = item['price_rating']
|
||||
row['avg_price'] = item['avg_price']
|
||||
|
||||
action = {'_op_type': 'update',
|
||||
'_index': index_name,
|
||||
'_id': _id, # 请勿忘记ID
|
||||
'doc': row}
|
||||
actions.append(action)
|
||||
if len(actions) == 5000:
|
||||
helpers.bulk(client=es206_client.elastic_client, actions=actions)
|
||||
count += len(actions)
|
||||
actions.clear()
|
||||
log.info("评级中: 当前已评级{}".format(count))
|
||||
if len(actions) > 0:
|
||||
helpers.bulk(client=es206_client.elastic_client, actions=actions)
|
||||
count += len(actions)
|
||||
actions.clear()
|
||||
log.info("评级完成,共更新: {}".format(count))
|
||||
|
||||
|
||||
def sync(city=None):
|
||||
calc_price_rating(city)
|
||||
|
||||
|
||||
sync()
|
||||
@@ -1,25 +0,0 @@
|
||||
"""
|
||||
消费或价格评级定时运行
|
||||
"""
|
||||
import requests
|
||||
import logging
|
||||
import sys
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
type = sys.argv[1] if len(sys.argv) > 1 else ''
|
||||
print("输出评级类型" + type)
|
||||
|
||||
if type == 'price':
|
||||
log.info("定时楼盘评级开始。。。。。")
|
||||
headers = {'Content-Type': 'application/json'}
|
||||
url = "http://192.168.0.19:31080/FangJiaNlpService/price_rate"
|
||||
if requests.get(url=url, headers=headers).text == 'success':
|
||||
log.info("定时楼盘评级调用完毕。。。。。")
|
||||
|
||||
else:
|
||||
log.info("定时消费评级开始。。。。。")
|
||||
headers = {'Content-Type': 'application/json'}
|
||||
url = "http://192.168.0.19:31080/FangJiaNlpService/consumption_rate"
|
||||
if requests.get(url=url, headers=headers).text == 'success':
|
||||
log.info("定时消费评级调用完毕。。。。。")
|
||||
@@ -51,8 +51,13 @@ with DAG(
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/relative_price_ranking.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
)
|
||||
t11 = BashOperator(
|
||||
task_id='consumption_potential_dag',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/consumption_potential.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
task_id='consumption_dag',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/consumption.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
)
|
||||
t12 = BashOperator(
|
||||
task_id='price_rating_dag',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/price_rating.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
)
|
||||
|
||||
t1 >> t2 >> t3 >> t4 >> t5 >> t6 >> t7 >> t8 >> t9 >> t10 >> t11
|
||||
# t1 >> t2 >> t3 >> t4 >> t5 >> t6 >> t7 >> t8 >> t9 >> t10 >> t11 >> t12
|
||||
t11 >> t12
|
||||
@@ -1,17 +0,0 @@
|
||||
from airflow.operators.bash import BashOperator
|
||||
import pendulum
|
||||
from airflow import DAG
|
||||
|
||||
with DAG(
|
||||
"rate_dag",
|
||||
# 每月3日16点整分运行
|
||||
schedule_interval="0 16 3 * *",
|
||||
start_date=pendulum.datetime(2022, 7, 20, tz="Asia/Shanghai"),
|
||||
catchup=True,
|
||||
description="地数价格与消费评级 指定type: price或consumption, 不指定则只对消费评级"
|
||||
) as dag:
|
||||
t1 = BashOperator(
|
||||
task_id='rate_dag',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/rate.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
)
|
||||
t1
|
||||
+2
-1
@@ -1,3 +1,4 @@
|
||||
goojia_utils
|
||||
nextcloud-api-wrapper
|
||||
elasticsearch==7.11.0
|
||||
elasticsearch==7.11.0
|
||||
scikit-learn==0.24.2
|
||||
Reference in New Issue
Block a user