新增人口规模
This commit is contained in:
@@ -0,0 +1,79 @@
|
||||
"""
|
||||
人口评级
|
||||
"""
|
||||
from goojia_utils import ElasticSearchClient
|
||||
import pandas as pd
|
||||
import logging
|
||||
from elasticsearch import helpers
|
||||
import sys
|
||||
|
||||
from sklearn.mixture import GaussianMixture
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
index_name = "community_summary"
|
||||
|
||||
|
||||
def gmm_train(total_rate, consumption_df, source, target):
|
||||
# 高斯混合模型
|
||||
gmm = GaussianMixture(n_components=total_rate, covariance_type='tied', max_iter=200, n_init=10).fit(consumption_df[[source]])
|
||||
consumption_df.loc[:, 'cluster'] = gmm.predict(consumption_df[[source]])
|
||||
consumption_df.loc[:, 'price_mean_cluster'] = consumption_df.groupby('cluster')[source].transform('mean')
|
||||
consumption_df.loc[:, target] = consumption_df['price_mean_cluster'].rank(method='dense', ascending=False).astype(int)
|
||||
consumption_df = consumption_df.drop(columns=['cluster', 'price_mean_cluster'], axis=1)
|
||||
return consumption_df
|
||||
|
||||
|
||||
def calc_person_rating(city=None):
|
||||
"""
|
||||
计算消费等级
|
||||
"""
|
||||
sql = "select id, city, region, name, household_count from community_summary"
|
||||
if pd.notna(city):
|
||||
sql += " where city='" + city + "'"
|
||||
community_summary_df = es206_client.scan_by_sql_to_data_frame(sql)
|
||||
|
||||
consumption_aim_df = community_summary_df[['_id', 'city', 'household_count']]
|
||||
consumption_aim_df = consumption_aim_df.dropna(subset=['household_count'])
|
||||
cities = consumption_aim_df['city'].unique()
|
||||
source = "household_count"
|
||||
target = "person_rating"
|
||||
actions = list()
|
||||
count = 0
|
||||
|
||||
for city in cities:
|
||||
df = consumption_aim_df[consumption_aim_df['city'] == city].copy()
|
||||
if len(df) < 5:
|
||||
continue
|
||||
df = gmm_train(5, df, source, target)
|
||||
|
||||
# 批量更新
|
||||
for index, item in df.iterrows():
|
||||
row = dict()
|
||||
_id = str(item["_id"])
|
||||
row['person_rating'] = item['person_rating']
|
||||
|
||||
action = {'_op_type': 'update',
|
||||
'_index': index_name,
|
||||
'_id': _id, # 请勿忘记ID
|
||||
'doc': row}
|
||||
actions.append(action)
|
||||
if len(actions) == 5000:
|
||||
helpers.bulk(client=es206_client.elastic_client, actions=actions)
|
||||
count += len(actions)
|
||||
actions.clear()
|
||||
log.info("评级中: 当前已评级{}".format(count))
|
||||
if len(actions) > 0:
|
||||
helpers.bulk(client=es206_client.elastic_client, actions=actions)
|
||||
count += len(actions)
|
||||
actions.clear()
|
||||
log.info("评级完成,共更新: {}".format(count))
|
||||
|
||||
|
||||
def sync(city=None):
|
||||
calc_person_rating(city)
|
||||
|
||||
|
||||
city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
sync(city)
|
||||
@@ -71,6 +71,10 @@ with DAG(
|
||||
task_id='consumption',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/consumption.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||
)
|
||||
person_rating = BashOperator(
|
||||
task_id='person_rating',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/person_rating.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||
)
|
||||
price_rating = BashOperator(
|
||||
task_id='price_rating',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/price_rating.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||
@@ -92,4 +96,4 @@ with DAG(
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/other.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||
)
|
||||
comm >> price >> avg_cost >> total_price >> relative_price_ranking >> estate_charge_complete >> estate_charge_ranking \
|
||||
>> consumption >> price_rating >> sell >> trade >> traffic >> business_district >> medical >> life >> education >> education2 >> extra >> other
|
||||
>> consumption >> price_rating >> person_rating >> sell >> trade >> traffic >> business_district >> medical >> life >> education >> education2 >> extra >> other
|
||||
|
||||
Reference in New Issue
Block a user