diff --git a/dags/business/business/community/credit.py b/dags/business/business/community/credit.py new file mode 100644 index 0000000..8aa45ac --- /dev/null +++ b/dags/business/business/community/credit.py @@ -0,0 +1,70 @@ +""" +置信度及总分计算 +""" +import logging +import pandas as pd +from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, dateutils, data_utils +from elasticsearch import helpers +import sys + +log = logging.getLogger(__name__) +seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') +es206_client = ElasticSearchClient(host='192.168.0.206') + +index_name = "community_summary" +path = "/home/airflow/data/community" +m_date = dateutils.iso_format() +avg_price_map = dict() + + +def build_doc(item): + doc_dict = dict() + if 'total_price' in item and pd.notna(item['total_price']): + doc_dict['total_price'] = int(item['total_price']) + return doc_dict + + +def sync(city=None): + log.info(f"开始置信度打分 {index_name}") + + # 加载库中均价数据 + query_sql = "SELECT city, region, name, credit_detail FROM community_summary WHERE consumption_rating>0 " + if pd.notna(city): + query_sql += " AND city='" + city + "' " + community_summary_df = es206_client.scan_by_sql_to_data_frame(query_sql) + + community_summary_df = data_utils.flatten_json(community_summary_df, 'credit_detail') + community_summary_df['total_price_score'].fillna(0, inplace=True) + community_summary_df['relative_price_ranking_score'].fillna(0, inplace=True) + community_summary_df['estate_charge_score'].fillna(0, inplace=True) + + community_summary_df['total_score'] = 0.6 * community_summary_df['total_price_score'] + 0.2 * community_summary_df['relative_price_ranking_score'] + 0.2 * community_summary_df['estate_charge_score'] + community_summary_df['credit'] = round(community_summary_df['total_score'] / 10, 2) + + actions = list() + count = 0 + + for index, item in community_summary_df.iterrows(): + doc_info = build_doc(item) + + action = { + "_op_type": "update", + "_index": index_name, + "_id": item['_id'], + "doc": doc_info, + "_retry_on_conflict": 5 + } + actions.append(action) + if len(actions) == 5000: + helpers.bulk(es206_client.elastic_client, actions) + count += len(actions) + logging.info("已更新:" + str(count)) + actions.clear() + if len(actions) > 0: + helpers.bulk(es206_client.elastic_client, actions) + actions.clear() + log.info(f"完成置信度打分 {index_name}") + + +city = sys.argv[1] if len(sys.argv) > 1 else None +sync(city)