新增物业费等级

This commit is contained in:
shenjian@fangjia.com
2022-11-21 17:39:30 +08:00
parent b92e5baa2e
commit 0281a2d5f2
3 changed files with 88 additions and 42 deletions
@@ -0,0 +1,79 @@
"""
物业费等级计算
"""
from goojia_utils import ElasticSearchClient
import pandas as pd
import numpy as np
import logging
from elasticsearch import helpers
from sklearn.mixture import GaussianMixture
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
index_name = "community_summary"
path = "/opt/airflow/dags/repo/dags/business/business/community/data"
def gmm_train(total_rate, consumption_df, source, target):
# 高斯混合模型
gmm = GaussianMixture(n_components=total_rate).fit(consumption_df[[source]])
consumption_df.loc[:, 'cluster'] = gmm.predict(consumption_df[[source]])
consumption_df.loc[:, 'price_mean_cluster'] = consumption_df.groupby('cluster')[source].transform('mean')
consumption_df.loc[:, target] = consumption_df['price_mean_cluster'].rank(method='dense', ascending=False).astype(int)
consumption_df = consumption_df.drop(columns=['cluster', 'price_mean_cluster'], axis=1)
return consumption_df
def calc_estate_charge_rating(community_summary_df):
"""
计算物业费等级
"""
consumption_aim_df = community_summary_df[['_id', 'city', 'estate_charge']]
consumption_aim_df = consumption_aim_df.dropna(subset=['estate_charge'])
cities = consumption_aim_df['city'].unique()
source = "estate_charge"
target = "estate_charge_ranking"
actions = list()
count = 0
for city in cities:
df = consumption_aim_df[consumption_aim_df['city'] == city].copy()
if len(df) < 5:
continue
df = gmm_train(5, df, source, target)
# 批量更新
for index, item in df.iterrows():
row = dict()
_id = str(item["_id"])
row['estate_charge_ranking'] = item['estate_charge_ranking']
action = {'_op_type': 'update', # 操作 index update create delete
'_index': index_name, # index
'_id': _id, # 请勿忘记ID
'doc': row,
"_retry_on_conflict": 5
}
actions.append(action)
if len(actions) == 5000:
helpers.bulk(client=es206_client.elastic_client, actions=actions)
count += len(actions)
actions.clear()
log.info("评级中: 当前已评级{}".format(count))
if len(actions) > 0:
helpers.bulk(client=es206_client.elastic_client, actions=actions)
count += len(actions)
actions.clear()
log.info("评级完成,共更新: {}".format(count))
def sync(city=None):
sql = "select id, city, region, name, estate_charge from community_summary"
if pd.notna(city):
sql += " where city='" + city + "'"
community_summary_df = es206_client.scan_by_sql_to_data_frame(sql)
calc_estate_charge_rating(community_summary_df)
sync()
+4 -41
View File
@@ -16,46 +16,9 @@ m_date = dateutils.iso_format()
count = 0
def get_old_or_new(finish_building_date):
"""
或许新旧
"""
if pd.isna(finish_building_date):
return "旧"
now_year = datetime.datetime.now().year
if now_year - int(finish_building_date) < 3:
return "新"
return "旧"
def deal(actions, x):
global count
estate_charge_ranking = None
if pd.notna(x['estate_charge']):
estate_charge = float(x['estate_charge'])
if estate_charge >= 1:
estate_charge_ranking = "一级"
elif estate_charge >= 0.75:
estate_charge_ranking = "二级"
elif estate_charge >= 0.50:
estate_charge_ranking = "三级"
elif estate_charge >= 0.35:
estate_charge_ranking = "四级"
elif estate_charge < 0.35:
estate_charge_ranking = "五级"
else:
old_or_new = get_old_or_new(x['finish_building_date'])
if old_or_new == '新':
if pd.notna(x['build_type']) and '高层' in x['build_type']:
estate_charge_ranking = '三级'
else:
estate_charge_ranking = '四级'
elif old_or_new == '旧':
estate_charge_ranking = '五级'
doc_info = dict()
if pd.notna(estate_charge_ranking):
doc_info['estate_charge_ranking'] = estate_charge_ranking
if pd.notna(x['avg_price']) and pd.notna(x['rent_price']):
doc_info['sell_to_rent'] = int(x['avg_price'] / x['rent_price'])
if pd.notna(x['household_count']):
@@ -72,14 +35,14 @@ def deal(actions, x):
if len(actions) == 5000:
helpers.bulk(es206_client.elastic_client, actions)
count += len(actions)
log.info(f"物业费等级、售租比、人口数已更新{count}")
log.info(f"售租比、人口数已更新{count}")
actions.clear()
def sync(city=None):
log.info(f"开始同步物业费等级、售租比、人口数至206 {index_name}")
log.info(f"开始同步售租比、人口数至206 {index_name}")
sql = "select city, region, name, estate_charge, finish_building_date, build_type, avg_price, rent_price, household_count from community_summary"
sql = "select city, region, name, finish_building_date, build_type, avg_price, rent_price, household_count from community_summary"
if pd.notna(city):
sql += " where city='" + city + "'"
community_summary_df = es206_client.scan_by_sql_to_data_frame(sql)
@@ -91,7 +54,7 @@ def sync(city=None):
helpers.bulk(es206_client.elastic_client, actions)
actions.clear()
log.info(f"完成同步物业费等级、售租比、人口数至206 {index_name}")
log.info(f"完成同步售租比、人口数至206 {index_name}")
sync()
+5 -1
View File
@@ -62,6 +62,10 @@ with DAG(
task_id='consumption',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/consumption.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
)
t11_1 = BashOperator(
task_id='estate_charge_ranking',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/estate_charge_ranking.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
)
t12 = BashOperator(
task_id='price_rating',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/price_rating.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
@@ -82,4 +86,4 @@ with DAG(
task_id='other',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/other.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
)
t1 >> t2 >> t3 >> t4 >> t5 >> t6 >> t6_1 >> t7 >> t7_1 >> t8 >> t9 >> t10 >> t11 >> t12 >> t13 >> t14 >> t15 >> t16
t1 >> t2 >> t3 >> t4 >> t5 >> t6 >> t6_1 >> t7 >> t7_1 >> t8 >> t9 >> t10 >> t11 >> t11_1 >> t12 >> t13 >> t14 >> t15 >> t16