新增物业费等级
This commit is contained in:
@@ -0,0 +1,79 @@
|
||||
"""
|
||||
物业费等级计算
|
||||
"""
|
||||
from goojia_utils import ElasticSearchClient
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
import logging
|
||||
from elasticsearch import helpers
|
||||
|
||||
from sklearn.mixture import GaussianMixture
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
index_name = "community_summary"
|
||||
path = "/opt/airflow/dags/repo/dags/business/business/community/data"
|
||||
|
||||
|
||||
|
||||
def gmm_train(total_rate, consumption_df, source, target):
|
||||
# 高斯混合模型
|
||||
gmm = GaussianMixture(n_components=total_rate).fit(consumption_df[[source]])
|
||||
consumption_df.loc[:, 'cluster'] = gmm.predict(consumption_df[[source]])
|
||||
consumption_df.loc[:, 'price_mean_cluster'] = consumption_df.groupby('cluster')[source].transform('mean')
|
||||
consumption_df.loc[:, target] = consumption_df['price_mean_cluster'].rank(method='dense', ascending=False).astype(int)
|
||||
consumption_df = consumption_df.drop(columns=['cluster', 'price_mean_cluster'], axis=1)
|
||||
return consumption_df
|
||||
|
||||
|
||||
def calc_estate_charge_rating(community_summary_df):
|
||||
"""
|
||||
计算物业费等级
|
||||
"""
|
||||
consumption_aim_df = community_summary_df[['_id', 'city', 'estate_charge']]
|
||||
consumption_aim_df = consumption_aim_df.dropna(subset=['estate_charge'])
|
||||
cities = consumption_aim_df['city'].unique()
|
||||
source = "estate_charge"
|
||||
target = "estate_charge_ranking"
|
||||
actions = list()
|
||||
count = 0
|
||||
|
||||
for city in cities:
|
||||
df = consumption_aim_df[consumption_aim_df['city'] == city].copy()
|
||||
if len(df) < 5:
|
||||
continue
|
||||
df = gmm_train(5, df, source, target)
|
||||
# 批量更新
|
||||
for index, item in df.iterrows():
|
||||
row = dict()
|
||||
_id = str(item["_id"])
|
||||
row['estate_charge_ranking'] = item['estate_charge_ranking']
|
||||
|
||||
action = {'_op_type': 'update', # 操作 index update create delete
|
||||
'_index': index_name, # index
|
||||
'_id': _id, # 请勿忘记ID
|
||||
'doc': row,
|
||||
"_retry_on_conflict": 5
|
||||
}
|
||||
actions.append(action)
|
||||
if len(actions) == 5000:
|
||||
helpers.bulk(client=es206_client.elastic_client, actions=actions)
|
||||
count += len(actions)
|
||||
actions.clear()
|
||||
log.info("评级中: 当前已评级{}".format(count))
|
||||
if len(actions) > 0:
|
||||
helpers.bulk(client=es206_client.elastic_client, actions=actions)
|
||||
count += len(actions)
|
||||
actions.clear()
|
||||
log.info("评级完成,共更新: {}".format(count))
|
||||
|
||||
|
||||
def sync(city=None):
|
||||
sql = "select id, city, region, name, estate_charge from community_summary"
|
||||
if pd.notna(city):
|
||||
sql += " where city='" + city + "'"
|
||||
community_summary_df = es206_client.scan_by_sql_to_data_frame(sql)
|
||||
calc_estate_charge_rating(community_summary_df)
|
||||
|
||||
sync()
|
||||
@@ -16,46 +16,9 @@ m_date = dateutils.iso_format()
|
||||
count = 0
|
||||
|
||||
|
||||
def get_old_or_new(finish_building_date):
|
||||
"""
|
||||
或许新旧
|
||||
"""
|
||||
if pd.isna(finish_building_date):
|
||||
return "旧"
|
||||
now_year = datetime.datetime.now().year
|
||||
if now_year - int(finish_building_date) < 3:
|
||||
return "新"
|
||||
return "旧"
|
||||
|
||||
|
||||
def deal(actions, x):
|
||||
global count
|
||||
estate_charge_ranking = None
|
||||
if pd.notna(x['estate_charge']):
|
||||
estate_charge = float(x['estate_charge'])
|
||||
if estate_charge >= 1:
|
||||
estate_charge_ranking = "一级"
|
||||
elif estate_charge >= 0.75:
|
||||
estate_charge_ranking = "二级"
|
||||
elif estate_charge >= 0.50:
|
||||
estate_charge_ranking = "三级"
|
||||
elif estate_charge >= 0.35:
|
||||
estate_charge_ranking = "四级"
|
||||
elif estate_charge < 0.35:
|
||||
estate_charge_ranking = "五级"
|
||||
else:
|
||||
old_or_new = get_old_or_new(x['finish_building_date'])
|
||||
if old_or_new == '新':
|
||||
if pd.notna(x['build_type']) and '高层' in x['build_type']:
|
||||
estate_charge_ranking = '三级'
|
||||
else:
|
||||
estate_charge_ranking = '四级'
|
||||
elif old_or_new == '旧':
|
||||
estate_charge_ranking = '五级'
|
||||
|
||||
doc_info = dict()
|
||||
if pd.notna(estate_charge_ranking):
|
||||
doc_info['estate_charge_ranking'] = estate_charge_ranking
|
||||
if pd.notna(x['avg_price']) and pd.notna(x['rent_price']):
|
||||
doc_info['sell_to_rent'] = int(x['avg_price'] / x['rent_price'])
|
||||
if pd.notna(x['household_count']):
|
||||
@@ -72,14 +35,14 @@ def deal(actions, x):
|
||||
if len(actions) == 5000:
|
||||
helpers.bulk(es206_client.elastic_client, actions)
|
||||
count += len(actions)
|
||||
log.info(f"物业费等级、售租比、人口数已更新{count}")
|
||||
log.info(f"售租比、人口数已更新{count}")
|
||||
actions.clear()
|
||||
|
||||
|
||||
def sync(city=None):
|
||||
log.info(f"开始同步物业费等级、售租比、人口数至206 {index_name}")
|
||||
log.info(f"开始同步售租比、人口数至206 {index_name}")
|
||||
|
||||
sql = "select city, region, name, estate_charge, finish_building_date, build_type, avg_price, rent_price, household_count from community_summary"
|
||||
sql = "select city, region, name, finish_building_date, build_type, avg_price, rent_price, household_count from community_summary"
|
||||
if pd.notna(city):
|
||||
sql += " where city='" + city + "'"
|
||||
community_summary_df = es206_client.scan_by_sql_to_data_frame(sql)
|
||||
@@ -91,7 +54,7 @@ def sync(city=None):
|
||||
helpers.bulk(es206_client.elastic_client, actions)
|
||||
actions.clear()
|
||||
|
||||
log.info(f"完成同步物业费等级、售租比、人口数至206 {index_name}")
|
||||
log.info(f"完成同步售租比、人口数至206 {index_name}")
|
||||
|
||||
|
||||
sync()
|
||||
@@ -62,6 +62,10 @@ with DAG(
|
||||
task_id='consumption',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/consumption.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
)
|
||||
t11_1 = BashOperator(
|
||||
task_id='estate_charge_ranking',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/estate_charge_ranking.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
)
|
||||
t12 = BashOperator(
|
||||
task_id='price_rating',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/price_rating.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
@@ -82,4 +86,4 @@ with DAG(
|
||||
task_id='other',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/other.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||
)
|
||||
t1 >> t2 >> t3 >> t4 >> t5 >> t6 >> t6_1 >> t7 >> t7_1 >> t8 >> t9 >> t10 >> t11 >> t12 >> t13 >> t14 >> t15 >> t16
|
||||
t1 >> t2 >> t3 >> t4 >> t5 >> t6 >> t6_1 >> t7 >> t7_1 >> t8 >> t9 >> t10 >> t11 >> t11_1 >> t12 >> t13 >> t14 >> t15 >> t16
|
||||
|
||||
Reference in New Issue
Block a user