新增城市等级级价格等级评级
This commit is contained in:
@@ -1,113 +1,121 @@
|
||||
"""
|
||||
城市等级及城市价格等级计算
|
||||
"""
|
||||
from goojia_utils import ElasticSearchClient, AmapClient
|
||||
from goojia_utils import ElasticSearchClient
|
||||
import pandas as pd
|
||||
from sklearn.mixture import GaussianMixture
|
||||
from sklearn.preprocessing import QuantileTransformer
|
||||
import logging
|
||||
from elasticsearch import helpers
|
||||
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
amap_client = AmapClient()
|
||||
log = logging.getLogger(__name__)
|
||||
path = "/opt/airflow/dags/repo/dags/business/business/community/data"
|
||||
|
||||
new_df = pd.read_excel(path + '/城市等级及城市价格等级信息.xlsx')
|
||||
community_summary_df = es206_client.scan_by_sql_to_data_frame(
|
||||
'select city, region, name, consumption_potential from community_summary')
|
||||
|
||||
merge_df = pd.merge(community_summary_df, new_df, how='left', on=['city'])
|
||||
city_grade = ['一线', '新一线', '二线', '三四线', '五线']
|
||||
rating = [1, 2, 3, 4, 5]
|
||||
index_name = "community_summary"
|
||||
import sys
|
||||
|
||||
|
||||
def filter_min_max(community_summary_df, source):
|
||||
community_summary_df['normal_consumption_potential'] = QuantileTransformer(
|
||||
output_distribution="normal").fit_transform(community_summary_df[[source]])
|
||||
return community_summary_df
|
||||
def sync():
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
log = logging.getLogger(__name__)
|
||||
path = "/opt/airflow/dags/repo/dags/business/business/community/data"
|
||||
|
||||
new_df = pd.read_excel(path + '/城市等级及城市价格等级信息.xlsx')
|
||||
community_summary_df = es206_client.scan_by_sql_to_data_frame(
|
||||
'select city, region, name, consumption_potential from community_summary')
|
||||
|
||||
def gmm_train(total_rate, consumption_df, source, target):
|
||||
# 高斯混合模型
|
||||
gmm = GaussianMixture(n_components=total_rate, covariance_type='tied', max_iter=200, n_init=10).fit(
|
||||
consumption_df[[source]])
|
||||
consumption_df.loc[:, 'cluster'] = gmm.predict(consumption_df[[source]])
|
||||
consumption_df.loc[:, 'price_mean_cluster'] = consumption_df.groupby('cluster')[source].transform('mean')
|
||||
consumption_df.loc[:, target] = consumption_df['price_mean_cluster'].rank(method='dense', ascending=False).astype(
|
||||
int)
|
||||
consumption_df = consumption_df.drop(columns=['cluster', 'price_mean_cluster'], axis=1)
|
||||
return consumption_df
|
||||
merge_df = pd.merge(community_summary_df, new_df, how='left', on=['city'])
|
||||
city_grade = ['一线', '新一线', '二线', '三四线', '五线']
|
||||
rating = [1, 2, 3, 4, 5]
|
||||
index_name = "community_summary"
|
||||
|
||||
def filter_min_max(community_summary_df, source):
|
||||
community_summary_df['normal_consumption_potential'] = QuantileTransformer(
|
||||
output_distribution="normal").fit_transform(community_summary_df[[source]])
|
||||
return community_summary_df
|
||||
|
||||
log.info("开始按城市等级评级...")
|
||||
merge_df = merge_df[merge_df['consumption_potential'].notna()].copy()
|
||||
merge_df['consumption_potential_city_grade'] = merge_df['consumption_potential'] * merge_df['city_grade_ratio']
|
||||
merge_df['consumption_potential_city_grade'].fillna(merge_df['consumption_potential'], inplace=True)
|
||||
def gmm_train(total_rate, consumption_df, source, target):
|
||||
# 高斯混合模型
|
||||
gmm = GaussianMixture(n_components=total_rate, covariance_type='tied', max_iter=200, n_init=10).fit(
|
||||
consumption_df[[source]])
|
||||
consumption_df.loc[:, 'cluster'] = gmm.predict(consumption_df[[source]])
|
||||
consumption_df.loc[:, 'price_mean_cluster'] = consumption_df.groupby('cluster')[source].transform('mean')
|
||||
consumption_df.loc[:, target] = consumption_df['price_mean_cluster'].rank(method='dense',
|
||||
ascending=False).astype(
|
||||
int)
|
||||
consumption_df = consumption_df.drop(columns=['cluster', 'price_mean_cluster'], axis=1)
|
||||
return consumption_df
|
||||
|
||||
city_grade_ret_df = pd.DataFrame()
|
||||
for grade in city_grade:
|
||||
temp_df = merge_df[merge_df['city_grade'] == grade].copy()
|
||||
log.info(f"当前城市等级{grade}, 评级数量{len(temp_df)}")
|
||||
temp_df = filter_min_max(temp_df, 'consumption_potential_city_grade')
|
||||
source = "normal_consumption_potential"
|
||||
target = "city_level_rating"
|
||||
temp_df = gmm_train(5, temp_df, source, target)
|
||||
city_grade_ret_df = pd.concat([city_grade_ret_df, temp_df])
|
||||
log.info("完成按城市等级评级...")
|
||||
log.info("开始按城市等级评级...")
|
||||
merge_df = merge_df[merge_df['consumption_potential'].notna()].copy()
|
||||
merge_df['consumption_potential_city_grade'] = merge_df['consumption_potential'] * merge_df['city_grade_ratio']
|
||||
merge_df['consumption_potential_city_grade'].fillna(merge_df['consumption_potential'], inplace=True)
|
||||
|
||||
log.info("开始按城市价格等级评级")
|
||||
merge_df = merge_df[merge_df['consumption_potential'].notna()].copy()
|
||||
merge_df['consumption_potential_rating'] = merge_df['consumption_potential'] * merge_df['rating_ratio']
|
||||
merge_df['consumption_potential_rating'].fillna(merge_df['consumption_potential'], inplace=True)
|
||||
city_grade_ret_df = pd.DataFrame()
|
||||
for grade in city_grade:
|
||||
temp_df = merge_df[merge_df['city_grade'] == grade].copy()
|
||||
log.info(f"当前城市等级{grade}, 评级数量{len(temp_df)}")
|
||||
temp_df = filter_min_max(temp_df, 'consumption_potential_city_grade')
|
||||
source = "normal_consumption_potential"
|
||||
target = "city_level_rating"
|
||||
temp_df = gmm_train(5, temp_df, source, target)
|
||||
city_grade_ret_df = pd.concat([city_grade_ret_df, temp_df])
|
||||
log.info("完成按城市等级评级...")
|
||||
|
||||
rating_ret_df = pd.DataFrame()
|
||||
for grade in rating:
|
||||
temp_df = merge_df[merge_df['rating'] == grade].copy()
|
||||
log.info(f"当前城市价格等级{grade}, 评级数量{len(temp_df)}")
|
||||
temp_df = filter_min_max(temp_df, 'consumption_potential_rating')
|
||||
source = "normal_consumption_potential"
|
||||
target = "city_price_level_rating"
|
||||
temp_df = gmm_train(5, temp_df, source, target)
|
||||
rating_ret_df = pd.concat([rating_ret_df, temp_df])
|
||||
log.info("完成按城市价格等级评级")
|
||||
log.info("开始按城市价格等级评级")
|
||||
merge_df = merge_df[merge_df['consumption_potential'].notna()].copy()
|
||||
merge_df['consumption_potential_rating'] = merge_df['consumption_potential'] * merge_df['rating_ratio']
|
||||
merge_df['consumption_potential_rating'].fillna(merge_df['consumption_potential'], inplace=True)
|
||||
|
||||
city_grade_ret_df = city_grade_ret_df[['_id', 'city', 'region', 'name', 'city_grade', 'city_level_rating']]
|
||||
rating_ret_df = rating_ret_df[['_id', 'city', 'region', 'name', 'rating', 'city_price_level_rating']]
|
||||
rating_ret_df = pd.DataFrame()
|
||||
for grade in rating:
|
||||
temp_df = merge_df[merge_df['rating'] == grade].copy()
|
||||
log.info(f"当前城市价格等级{grade}, 评级数量{len(temp_df)}")
|
||||
temp_df = filter_min_max(temp_df, 'consumption_potential_rating')
|
||||
source = "normal_consumption_potential"
|
||||
target = "city_price_level_rating"
|
||||
temp_df = gmm_train(5, temp_df, source, target)
|
||||
rating_ret_df = pd.concat([rating_ret_df, temp_df])
|
||||
log.info("完成按城市价格等级评级")
|
||||
|
||||
aim_df = pd.merge(city_grade_ret_df, rating_ret_df, how='outer', on=['_id', 'city', 'region', 'name'])
|
||||
city_grade_ret_df = city_grade_ret_df[['_id', 'city', 'region', 'name', 'city_grade', 'city_level_rating']]
|
||||
rating_ret_df = rating_ret_df[['_id', 'city', 'region', 'name', 'rating', 'city_price_level_rating']]
|
||||
|
||||
log.info("开始写入ES206索引{index_name}")
|
||||
actions = list()
|
||||
count = 0
|
||||
for index, item in aim_df.iterrows():
|
||||
row = dict()
|
||||
_id = str(item["_id"])
|
||||
if pd.notna(item['city_grade']):
|
||||
row['city_level'] = item['city_grade']
|
||||
if pd.notna(item['city_level_rating']):
|
||||
row['city_level_rating'] = item['city_level_rating']
|
||||
if pd.notna(item['rating']):
|
||||
row['city_price_level'] = item['rating']
|
||||
if pd.notna(item['city_price_level_rating']):
|
||||
row['city_price_level_rating'] = item['city_price_level_rating']
|
||||
aim_df = pd.merge(city_grade_ret_df, rating_ret_df, how='outer', on=['_id', 'city', 'region', 'name'])
|
||||
|
||||
action = {'_op_type': 'update',
|
||||
'_index': index_name,
|
||||
'_id': _id, # 请勿忘记ID
|
||||
'doc': row,
|
||||
"_retry_on_conflict": 5
|
||||
}
|
||||
actions.append(action)
|
||||
if len(actions) == 5000:
|
||||
log.info("开始写入ES206索引{index_name}")
|
||||
actions = list()
|
||||
count = 0
|
||||
|
||||
city_grade_dict = {'一线': 1, '新一线': 2, '二线': 3, '三四线': 4, '五线': 5}
|
||||
for index, item in aim_df.iterrows():
|
||||
row = dict()
|
||||
_id = str(item["_id"])
|
||||
if pd.notna(item['city_grade']):
|
||||
row['city_level'] = city_grade_dict[item['city_grade']]
|
||||
if pd.notna(item['city_level_rating']):
|
||||
row['city_level_rating'] = item['city_level_rating']
|
||||
if pd.notna(item['rating']):
|
||||
row['city_price_level'] = item['rating']
|
||||
if pd.notna(item['city_price_level_rating']):
|
||||
row['city_price_level_rating'] = item['city_price_level_rating']
|
||||
|
||||
action = {'_op_type': 'update',
|
||||
'_index': index_name,
|
||||
'_id': _id, # 请勿忘记ID
|
||||
'doc': row,
|
||||
"_retry_on_conflict": 5
|
||||
}
|
||||
actions.append(action)
|
||||
if len(actions) == 5000:
|
||||
helpers.bulk(client=es206_client.elastic_client, actions=actions)
|
||||
count += len(actions)
|
||||
actions.clear()
|
||||
log.info("评级中: 当前已评级{}".format(count))
|
||||
if len(actions) > 0:
|
||||
helpers.bulk(client=es206_client.elastic_client, actions=actions)
|
||||
count += len(actions)
|
||||
actions.clear()
|
||||
log.info("评级中: 当前已评级{}".format(count))
|
||||
if len(actions) > 0:
|
||||
helpers.bulk(client=es206_client.elastic_client, actions=actions)
|
||||
count += len(actions)
|
||||
actions.clear()
|
||||
log.info("评级完成,共更新: {}".format(count))
|
||||
log.info("完成写入ES206索引{index_name}")
|
||||
log.info("评级完成,共更新: {}".format(count))
|
||||
log.info("完成写入ES206索引{index_name}")
|
||||
|
||||
|
||||
city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
# 对于该评级等级来说,不能单个城市处理,只能全量
|
||||
if pd.isna(city):
|
||||
sync()
|
||||
|
||||
@@ -74,6 +74,10 @@ with DAG(
|
||||
task_id='consumption',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/consumption.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||
)
|
||||
city_level_and_price_rating = BashOperator(
|
||||
task_id='city_level_and_price_rating',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/city_level_and_price_rating.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||
)
|
||||
credit = BashOperator(
|
||||
task_id='credit',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/credit.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||
@@ -107,4 +111,4 @@ with DAG(
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/other.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||
)
|
||||
# 人均消费并行,值为空
|
||||
comm >> price >> [avg_cost, total_price, relative_price_ranking, estate_charge_complete] >> total_price_complete >> relative_price_ranking_fix >> consumption >> credit >> [estate_charge_ranking, price_rating, person_rating, percentile] >> sell >> trade >> [traffic, business_district, medical, life, education] >> education2 >> extra >> other
|
||||
comm >> price >> [avg_cost, total_price, relative_price_ranking, estate_charge_complete] >> total_price_complete >> relative_price_ranking_fix >> consumption >> city_level_and_price_rating >> credit >> [estate_charge_ranking, price_rating, person_rating, percentile] >> sell >> trade >> [traffic, business_district, medical, life, education] >> education2 >> extra >> other
|
||||
|
||||
Reference in New Issue
Block a user