新增人口规模

This commit is contained in:
shenjian@fangjia.com
2023-03-06 15:25:47 +08:00
parent 4d6c59d043
commit b00f3a67a1
2 changed files with 84 additions and 1 deletions
@@ -0,0 +1,79 @@
"""
人口评级
"""
from goojia_utils import ElasticSearchClient
import pandas as pd
import logging
from elasticsearch import helpers
import sys
from sklearn.mixture import GaussianMixture
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
index_name = "community_summary"
def gmm_train(total_rate, consumption_df, source, target):
# 高斯混合模型
gmm = GaussianMixture(n_components=total_rate, covariance_type='tied', max_iter=200, n_init=10).fit(consumption_df[[source]])
consumption_df.loc[:, 'cluster'] = gmm.predict(consumption_df[[source]])
consumption_df.loc[:, 'price_mean_cluster'] = consumption_df.groupby('cluster')[source].transform('mean')
consumption_df.loc[:, target] = consumption_df['price_mean_cluster'].rank(method='dense', ascending=False).astype(int)
consumption_df = consumption_df.drop(columns=['cluster', 'price_mean_cluster'], axis=1)
return consumption_df
def calc_person_rating(city=None):
"""
计算消费等级
"""
sql = "select id, city, region, name, household_count from community_summary"
if pd.notna(city):
sql += " where city='" + city + "'"
community_summary_df = es206_client.scan_by_sql_to_data_frame(sql)
consumption_aim_df = community_summary_df[['_id', 'city', 'household_count']]
consumption_aim_df = consumption_aim_df.dropna(subset=['household_count'])
cities = consumption_aim_df['city'].unique()
source = "household_count"
target = "person_rating"
actions = list()
count = 0
for city in cities:
df = consumption_aim_df[consumption_aim_df['city'] == city].copy()
if len(df) < 5:
continue
df = gmm_train(5, df, source, target)
# 批量更新
for index, item in df.iterrows():
row = dict()
_id = str(item["_id"])
row['person_rating'] = item['person_rating']
action = {'_op_type': 'update',
'_index': index_name,
'_id': _id, # 请勿忘记ID
'doc': row}
actions.append(action)
if len(actions) == 5000:
helpers.bulk(client=es206_client.elastic_client, actions=actions)
count += len(actions)
actions.clear()
log.info("评级中: 当前已评级{}".format(count))
if len(actions) > 0:
helpers.bulk(client=es206_client.elastic_client, actions=actions)
count += len(actions)
actions.clear()
log.info("评级完成,共更新: {}".format(count))
def sync(city=None):
calc_person_rating(city)
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
+5 -1
View File
@@ -71,6 +71,10 @@ with DAG(
task_id='consumption',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/consumption.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
)
person_rating = BashOperator(
task_id='person_rating',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/person_rating.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
)
price_rating = BashOperator(
task_id='price_rating',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/price_rating.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
@@ -92,4 +96,4 @@ with DAG(
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/other.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
)
comm >> price >> avg_cost >> total_price >> relative_price_ranking >> estate_charge_complete >> estate_charge_ranking \
>> consumption >> price_rating >> sell >> trade >> traffic >> business_district >> medical >> life >> education >> education2 >> extra >> other
>> consumption >> price_rating >> person_rating >> sell >> trade >> traffic >> business_district >> medical >> life >> education >> education2 >> extra >> other