From 9a139d953f1cf522e50bcae308834cbab9905b86 Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Fri, 28 Oct 2022 13:26:30 +0800 Subject: [PATCH] =?UTF-8?q?=E8=B4=A2=E5=8A=9B=E8=AF=84=E7=BA=A7=E4=BF=AE?= =?UTF-8?q?=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../business/community/consumption.py | 7 +- dags/business/business/community/education.py | 100 ++++++++++++++++++ dags/business/community_summary_dag.py | 16 +-- 3 files changed, 111 insertions(+), 12 deletions(-) create mode 100644 dags/business/business/community/education.py diff --git a/dags/business/business/community/consumption.py b/dags/business/business/community/consumption.py index ac41bf5..699dcd7 100644 --- a/dags/business/business/community/consumption.py +++ b/dags/business/business/community/consumption.py @@ -23,6 +23,9 @@ def calc_consumption_potential(city): if pd.notna(city): sql += " where city='" + city + "'" community_summary_df = es206_client.scan_by_sql_to_data_frame(sql) + # 户均总价计算时候改成万 + community_summary_df['total_price'] = community_summary_df['total_price'] / 10000 + community_summary_df['total_price_mean'] = community_summary_df.groupby(by=['city'])['total_price'].transform( np.mean) community_summary_df['total_price_std'] = community_summary_df.groupby(by=['city'])['total_price'].transform(np.std) @@ -43,20 +46,16 @@ def calc_consumption_potential(city): community_summary_df['total_price_score'] = 286.0134196 * ( community_summary_df['total_price'] - community_summary_df['total_price_mean']) / community_summary_df[ 'total_price_std'] - community_summary_df['total_price_score'].fillna(0, inplace=True) community_summary_df['estate_charge_score'] = 139.1374524 * ( community_summary_df['estate_charge'] - community_summary_df['estate_charge_mean']) / community_summary_df['estate_charge_std'] - community_summary_df['estate_charge_score'].fillna(0, inplace=True) community_summary_df['relative_price_ranking_score'] = 21.8427926 * ( community_summary_df['relative_price_ranking'] - community_summary_df['relative_price_ranking_mean']) / community_summary_df['relative_price_ranking_std'] - community_summary_df['relative_price_ranking_score'].fillna(0, inplace=True) community_summary_df['avg_cost_score'] = 67.0195871700 * ( community_summary_df['avg_cost'] - community_summary_df['avg_cost_mean']) / community_summary_df[ 'avg_cost_std'] - community_summary_df['avg_cost_score'].fillna(0, inplace=True) community_summary_df['consumption_potential'] = community_summary_df['total_price_score'] + community_summary_df[ 'estate_charge_score'] + community_summary_df['relative_price_ranking_score'] + community_summary_df['avg_cost_score'] + community_summary_df['avg_cost_score'] + 4099.978248 diff --git a/dags/business/business/community/education.py b/dags/business/business/community/education.py new file mode 100644 index 0000000..faa1623 --- /dev/null +++ b/dags/business/business/community/education.py @@ -0,0 +1,100 @@ +""" +教育资源相关信息同步至ES206 +""" +import logging +import pandas as pd +from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, areautils, poiutils +from elasticsearch import helpers +from concurrent.futures import ThreadPoolExecutor +from threading import Lock + +log = logging.getLogger(__name__) +es206_client = ElasticSearchClient(host='192.168.0.206') + +index_name = "community_summary" +path = "/home/airflow/data/community" +m_date = dateutils.iso_format() + +count = 0 +mutex = Lock() + + +def deal(actions, item): + global count + curr_city = item['city'] + lng = commutils.format_lat_lng(item['lng']) + lat = commutils.format_lat_lng(item['lat']) + + if pd.isna(lng) or pd.isna(lat): + return + + doc_info = dict() + + # 幼儿园 + kindergarten = poiutils.find_around_by_location(curr_city, 1000, [lng, lat], "学校", "幼儿园") + doc_info['kindergarten_num'] = len(kindergarten) + if len(kindergarten) > 0: + doc_info['kindergarten'] = ",".join(kindergarten) + + # 超市 + supermarket = poiutils.find_around_by_location(curr_city, 1000, [lng, lat], "学校", "小学") + doc_info['supermarket_num'] = len(supermarket) + if len(supermarket) > 0: + doc_info['supermarket'] = ",".join(supermarket) + + # 公园 + park = poiutils.find_around_by_location(curr_city, 1000, [lng, lat], "生活配套", "公园") + doc_info['park_num'] = len(park) + if len(park) > 0: + doc_info['park'] = ",".join(park) + + # 餐厅 + catering = poiutils.find_around_by_location(curr_city, 1000, [lng, lat], "生活配套", "餐厅") + doc_info['catering_num'] = len(catering) + if len(catering) > 0: + doc_info['catering'] = ",".join(catering) + + action = { + "_op_type": "update", + "_index": index_name, + "_id": item['_id'], + "doc": doc_info + } + + mutex.acquire(timeout=60) + actions.append(action) + if len(actions) == 5000: + try: + helpers.bulk(es206_client.elastic_client, actions) + count += len(actions) + log.info("更新:" + str(count)) + except Exception as e: + log.error(e) + actions.clear() + mutex.release() + + +def sync(city=None): + log.info(f"开始同步教育资源至206 {index_name}") + seaweed_query = {'status': 0, 'cat': 'district'} + if pd.notna(city): + seaweed_query['city'] = city + + seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8') + + actions = list() + with ThreadPoolExecutor(max_workers=30) as pool: + for index, item in seaweed_df.iterrows(): + pool.submit(deal, actions, item) + + if len(actions) > 0: + try: + helpers.bulk(es206_client.elastic_client, actions) + except Exception as e: + log.error(e) + actions.clear() + + log.info(f"完成同步教育资源至206 {index_name}") + + +sync() \ No newline at end of file diff --git a/dags/business/community_summary_dag.py b/dags/business/community_summary_dag.py index cb884f9..34776b4 100644 --- a/dags/business/community_summary_dag.py +++ b/dags/business/community_summary_dag.py @@ -50,14 +50,14 @@ with DAG( # task_id='relative_price_ranking_dag', # bash_command='python /opt/airflow/dags/repo/dags/business/business/community/relative_price_ranking.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', # ) - # t11 = BashOperator( - # task_id='consumption_dag', - # bash_command='python /opt/airflow/dags/repo/dags/business/business/community/consumption.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', - # ) - t12 = BashOperator( - task_id='price_rating_dag', - bash_command='python /opt/airflow/dags/repo/dags/business/business/community/price_rating.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + t11 = BashOperator( + task_id='consumption_dag', + bash_command='python /opt/airflow/dags/repo/dags/business/business/community/consumption.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', ) + # t12 = BashOperator( + # task_id='price_rating_dag', + # bash_command='python /opt/airflow/dags/repo/dags/business/business/community/price_rating.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + # ) + t11 # t1 >> t2 >> t3 >> t4 >> t5 >> t6 >> t7 >> t8 >> t9 >> t10 >> t11 >> t12 - t12 \ No newline at end of file