新增城市等级级价格等级评级

This commit is contained in:
shenjian@fangjia.com
2023-05-15 15:20:49 +08:00
parent 95462de314
commit 100d89e556
2 changed files with 102 additions and 90 deletions
@@ -1,113 +1,121 @@
"""
城市等级及城市价格等级计算
"""
from goojia_utils import ElasticSearchClient, AmapClient
from goojia_utils import ElasticSearchClient
import pandas as pd
from sklearn.mixture import GaussianMixture
from sklearn.preprocessing import QuantileTransformer
import logging
from elasticsearch import helpers
es206_client = ElasticSearchClient(host='192.168.0.206')
amap_client = AmapClient()
log = logging.getLogger(__name__)
path = "/opt/airflow/dags/repo/dags/business/business/community/data"
new_df = pd.read_excel(path + '/城市等级及城市价格等级信息.xlsx')
community_summary_df = es206_client.scan_by_sql_to_data_frame(
'select city, region, name, consumption_potential from community_summary')
merge_df = pd.merge(community_summary_df, new_df, how='left', on=['city'])
city_grade = ['一线', '新一线', '二线', '三四线', '五线']
rating = [1, 2, 3, 4, 5]
index_name = "community_summary"
import sys
def filter_min_max(community_summary_df, source):
community_summary_df['normal_consumption_potential'] = QuantileTransformer(
output_distribution="normal").fit_transform(community_summary_df[[source]])
return community_summary_df
def sync():
es206_client = ElasticSearchClient(host='192.168.0.206')
log = logging.getLogger(__name__)
path = "/opt/airflow/dags/repo/dags/business/business/community/data"
new_df = pd.read_excel(path + '/城市等级及城市价格等级信息.xlsx')
community_summary_df = es206_client.scan_by_sql_to_data_frame(
'select city, region, name, consumption_potential from community_summary')
def gmm_train(total_rate, consumption_df, source, target):
# 高斯混合模型
gmm = GaussianMixture(n_components=total_rate, covariance_type='tied', max_iter=200, n_init=10).fit(
consumption_df[[source]])
consumption_df.loc[:, 'cluster'] = gmm.predict(consumption_df[[source]])
consumption_df.loc[:, 'price_mean_cluster'] = consumption_df.groupby('cluster')[source].transform('mean')
consumption_df.loc[:, target] = consumption_df['price_mean_cluster'].rank(method='dense', ascending=False).astype(
int)
consumption_df = consumption_df.drop(columns=['cluster', 'price_mean_cluster'], axis=1)
return consumption_df
merge_df = pd.merge(community_summary_df, new_df, how='left', on=['city'])
city_grade = ['一线', '新一线', '二线', '三四线', '五线']
rating = [1, 2, 3, 4, 5]
index_name = "community_summary"
def filter_min_max(community_summary_df, source):
community_summary_df['normal_consumption_potential'] = QuantileTransformer(
output_distribution="normal").fit_transform(community_summary_df[[source]])
return community_summary_df
log.info("开始按城市等级评级...")
merge_df = merge_df[merge_df['consumption_potential'].notna()].copy()
merge_df['consumption_potential_city_grade'] = merge_df['consumption_potential'] * merge_df['city_grade_ratio']
merge_df['consumption_potential_city_grade'].fillna(merge_df['consumption_potential'], inplace=True)
def gmm_train(total_rate, consumption_df, source, target):
# 高斯混合模型
gmm = GaussianMixture(n_components=total_rate, covariance_type='tied', max_iter=200, n_init=10).fit(
consumption_df[[source]])
consumption_df.loc[:, 'cluster'] = gmm.predict(consumption_df[[source]])
consumption_df.loc[:, 'price_mean_cluster'] = consumption_df.groupby('cluster')[source].transform('mean')
consumption_df.loc[:, target] = consumption_df['price_mean_cluster'].rank(method='dense',
ascending=False).astype(
int)
consumption_df = consumption_df.drop(columns=['cluster', 'price_mean_cluster'], axis=1)
return consumption_df
city_grade_ret_df = pd.DataFrame()
for grade in city_grade:
temp_df = merge_df[merge_df['city_grade'] == grade].copy()
log.info(f"当前城市等级{grade}, 评级数量{len(temp_df)}")
temp_df = filter_min_max(temp_df, 'consumption_potential_city_grade')
source = "normal_consumption_potential"
target = "city_level_rating"
temp_df = gmm_train(5, temp_df, source, target)
city_grade_ret_df = pd.concat([city_grade_ret_df, temp_df])
log.info("完成按城市等级评级...")
log.info("开始按城市等级评级...")
merge_df = merge_df[merge_df['consumption_potential'].notna()].copy()
merge_df['consumption_potential_city_grade'] = merge_df['consumption_potential'] * merge_df['city_grade_ratio']
merge_df['consumption_potential_city_grade'].fillna(merge_df['consumption_potential'], inplace=True)
log.info("开始按城市价格等级评级")
merge_df = merge_df[merge_df['consumption_potential'].notna()].copy()
merge_df['consumption_potential_rating'] = merge_df['consumption_potential'] * merge_df['rating_ratio']
merge_df['consumption_potential_rating'].fillna(merge_df['consumption_potential'], inplace=True)
city_grade_ret_df = pd.DataFrame()
for grade in city_grade:
temp_df = merge_df[merge_df['city_grade'] == grade].copy()
log.info(f"当前城市等级{grade}, 评级数量{len(temp_df)}")
temp_df = filter_min_max(temp_df, 'consumption_potential_city_grade')
source = "normal_consumption_potential"
target = "city_level_rating"
temp_df = gmm_train(5, temp_df, source, target)
city_grade_ret_df = pd.concat([city_grade_ret_df, temp_df])
log.info("完成按城市等级评级...")
rating_ret_df = pd.DataFrame()
for grade in rating:
temp_df = merge_df[merge_df['rating'] == grade].copy()
log.info(f"当前城市价格等级{grade}, 评级数量{len(temp_df)}")
temp_df = filter_min_max(temp_df, 'consumption_potential_rating')
source = "normal_consumption_potential"
target = "city_price_level_rating"
temp_df = gmm_train(5, temp_df, source, target)
rating_ret_df = pd.concat([rating_ret_df, temp_df])
log.info("完成按城市价格等级评级")
log.info("开始按城市价格等级评级")
merge_df = merge_df[merge_df['consumption_potential'].notna()].copy()
merge_df['consumption_potential_rating'] = merge_df['consumption_potential'] * merge_df['rating_ratio']
merge_df['consumption_potential_rating'].fillna(merge_df['consumption_potential'], inplace=True)
city_grade_ret_df = city_grade_ret_df[['_id', 'city', 'region', 'name', 'city_grade', 'city_level_rating']]
rating_ret_df = rating_ret_df[['_id', 'city', 'region', 'name', 'rating', 'city_price_level_rating']]
rating_ret_df = pd.DataFrame()
for grade in rating:
temp_df = merge_df[merge_df['rating'] == grade].copy()
log.info(f"当前城市价格等级{grade}, 评级数量{len(temp_df)}")
temp_df = filter_min_max(temp_df, 'consumption_potential_rating')
source = "normal_consumption_potential"
target = "city_price_level_rating"
temp_df = gmm_train(5, temp_df, source, target)
rating_ret_df = pd.concat([rating_ret_df, temp_df])
log.info("完成按城市价格等级评级")
aim_df = pd.merge(city_grade_ret_df, rating_ret_df, how='outer', on=['_id', 'city', 'region', 'name'])
city_grade_ret_df = city_grade_ret_df[['_id', 'city', 'region', 'name', 'city_grade', 'city_level_rating']]
rating_ret_df = rating_ret_df[['_id', 'city', 'region', 'name', 'rating', 'city_price_level_rating']]
log.info("开始写入ES206索引{index_name}")
actions = list()
count = 0
for index, item in aim_df.iterrows():
row = dict()
_id = str(item["_id"])
if pd.notna(item['city_grade']):
row['city_level'] = item['city_grade']
if pd.notna(item['city_level_rating']):
row['city_level_rating'] = item['city_level_rating']
if pd.notna(item['rating']):
row['city_price_level'] = item['rating']
if pd.notna(item['city_price_level_rating']):
row['city_price_level_rating'] = item['city_price_level_rating']
aim_df = pd.merge(city_grade_ret_df, rating_ret_df, how='outer', on=['_id', 'city', 'region', 'name'])
action = {'_op_type': 'update',
'_index': index_name,
'_id': _id, # 请勿忘记ID
'doc': row,
"_retry_on_conflict": 5
}
actions.append(action)
if len(actions) == 5000:
log.info("开始写入ES206索引{index_name}")
actions = list()
count = 0
city_grade_dict = {'一线': 1, '新一线': 2, '二线': 3, '三四线': 4, '五线': 5}
for index, item in aim_df.iterrows():
row = dict()
_id = str(item["_id"])
if pd.notna(item['city_grade']):
row['city_level'] = city_grade_dict[item['city_grade']]
if pd.notna(item['city_level_rating']):
row['city_level_rating'] = item['city_level_rating']
if pd.notna(item['rating']):
row['city_price_level'] = item['rating']
if pd.notna(item['city_price_level_rating']):
row['city_price_level_rating'] = item['city_price_level_rating']
action = {'_op_type': 'update',
'_index': index_name,
'_id': _id, # 请勿忘记ID
'doc': row,
"_retry_on_conflict": 5
}
actions.append(action)
if len(actions) == 5000:
helpers.bulk(client=es206_client.elastic_client, actions=actions)
count += len(actions)
actions.clear()
log.info("评级中: 当前已评级{}".format(count))
if len(actions) > 0:
helpers.bulk(client=es206_client.elastic_client, actions=actions)
count += len(actions)
actions.clear()
log.info("评级中: 当前已评级{}".format(count))
if len(actions) > 0:
helpers.bulk(client=es206_client.elastic_client, actions=actions)
count += len(actions)
actions.clear()
log.info("评级完成,共更新: {}".format(count))
log.info("完成写入ES206索引{index_name}")
log.info("评级完成,共更新: {}".format(count))
log.info("完成写入ES206索引{index_name}")
city = sys.argv[1] if len(sys.argv) > 1 else None
# 对于该评级等级来说,不能单个城市处理,只能全量
if pd.isna(city):
sync()
+5 -1
View File
@@ -74,6 +74,10 @@ with DAG(
task_id='consumption',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/consumption.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
)
city_level_and_price_rating = BashOperator(
task_id='city_level_and_price_rating',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/city_level_and_price_rating.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
)
credit = BashOperator(
task_id='credit',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/credit.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
@@ -107,4 +111,4 @@ with DAG(
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/other.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
)
# 人均消费并行,值为空
comm >> price >> [avg_cost, total_price, relative_price_ranking, estate_charge_complete] >> total_price_complete >> relative_price_ranking_fix >> consumption >> credit >> [estate_charge_ranking, price_rating, person_rating, percentile] >> sell >> trade >> [traffic, business_district, medical, life, education] >> education2 >> extra >> other
comm >> price >> [avg_cost, total_price, relative_price_ranking, estate_charge_complete] >> total_price_complete >> relative_price_ranking_fix >> consumption >> city_level_and_price_rating >> credit >> [estate_charge_ranking, price_rating, person_rating, percentile] >> sell >> trade >> [traffic, business_district, medical, life, education] >> education2 >> extra >> other