From 0b9dbbdf70b72c4022abadb78d5488f2c9398867 Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Mon, 21 Nov 2022 10:30:11 +0800 Subject: [PATCH 1/7] =?UTF-8?q?=E6=B5=8B=E8=AF=95traffic?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/business/community_summary_dag.py | 138 ++++++++++++------------- 1 file changed, 69 insertions(+), 69 deletions(-) diff --git a/dags/business/community_summary_dag.py b/dags/business/community_summary_dag.py index 4777149..342d992 100644 --- a/dags/business/community_summary_dag.py +++ b/dags/business/community_summary_dag.py @@ -10,76 +10,76 @@ with DAG( catchup=True, description="地址输出信息分类数据定时更新" ) as dag: - t1 = BashOperator( - task_id='comm', - bash_command='python /opt/airflow/dags/repo/dags/business/business/community/comm.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', - ) - t2 = BashOperator( - task_id='price', - bash_command='python /opt/airflow/dags/repo/dags/business/business/community/price.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', - ) - t3 = BashOperator( - task_id='sell', - bash_command='python /opt/airflow/dags/repo/dags/business/business/community/sell.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', - ) - t4 = BashOperator( - task_id='avg_cost', - bash_command='python /opt/airflow/dags/repo/dags/business/business/community/avg_cost.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', - ) - t5 = BashOperator( - task_id='trade', - bash_command='python /opt/airflow/dags/repo/dags/business/business/community/trade.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', - ) - t6 = BashOperator( - task_id='total_price', - bash_command='python /opt/airflow/dags/repo/dags/business/business/community/total_price.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', - ) - t6_1 = BashOperator( - task_id='total_price_complete', - bash_command='python /opt/airflow/dags/repo/dags/business/business/community/total_price_complete.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', - ) + # t1 = BashOperator( + # task_id='comm', + # bash_command='python /opt/airflow/dags/repo/dags/business/business/community/comm.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + # ) + # t2 = BashOperator( + # task_id='price', + # bash_command='python /opt/airflow/dags/repo/dags/business/business/community/price.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + # ) + # t3 = BashOperator( + # task_id='sell', + # bash_command='python /opt/airflow/dags/repo/dags/business/business/community/sell.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + # ) + # t4 = BashOperator( + # task_id='avg_cost', + # bash_command='python /opt/airflow/dags/repo/dags/business/business/community/avg_cost.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + # ) + # t5 = BashOperator( + # task_id='trade', + # bash_command='python /opt/airflow/dags/repo/dags/business/business/community/trade.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + # ) + # t6 = BashOperator( + # task_id='total_price', + # bash_command='python /opt/airflow/dags/repo/dags/business/business/community/total_price.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + # ) + # t6_1 = BashOperator( + # task_id='total_price_complete', + # bash_command='python /opt/airflow/dags/repo/dags/business/business/community/total_price_complete.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + # ) t7 = BashOperator( task_id='traffic', bash_command='python /opt/airflow/dags/repo/dags/business/business/community/traffic.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', ) - t7_1 = BashOperator( - task_id='business_district', - bash_command='python /opt/airflow/dags/repo/dags/business/business/community/business_district.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', - ) - t8 = BashOperator( - task_id='medical', - bash_command='python /opt/airflow/dags/repo/dags/business/business/community/medical.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', - ) - t9 = BashOperator( - task_id='life', - bash_command='python /opt/airflow/dags/repo/dags/business/business/community/life.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', - ) - t10 = BashOperator( - task_id='relative_price_ranking', - bash_command='python /opt/airflow/dags/repo/dags/business/business/community/relative_price_ranking.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', - ) - t11 = BashOperator( - task_id='consumption', - bash_command='python /opt/airflow/dags/repo/dags/business/business/community/consumption.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', - ) - t12 = BashOperator( - task_id='price_rating', - bash_command='python /opt/airflow/dags/repo/dags/business/business/community/price_rating.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', - ) - t13 = BashOperator( - task_id='education', - bash_command='python /opt/airflow/dags/repo/dags/business/business/community/education.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', - ) - t14 = BashOperator( - task_id='education2', - bash_command='python /opt/airflow/dags/repo/dags/business/business/community/education2.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', - ) - t15 = BashOperator( - task_id='extra', - bash_command='python /opt/airflow/dags/repo/dags/business/business/community/extra.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', - ) - t16 = BashOperator( - task_id='other', - bash_command='python /opt/airflow/dags/repo/dags/business/business/community/other.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', - ) - t1 >> t2 >> t3 >> t4 >> t5 >> t6 >> t6_1 >> t7 >> t7_1 >> t8 >> t9 >> t10 >> t11 >> t12 >> t13 >> t14 >> t15 >> t16 + # t7_1 = BashOperator( + # task_id='business_district', + # bash_command='python /opt/airflow/dags/repo/dags/business/business/community/business_district.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + # ) + # t8 = BashOperator( + # task_id='medical', + # bash_command='python /opt/airflow/dags/repo/dags/business/business/community/medical.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + # ) + # t9 = BashOperator( + # task_id='life', + # bash_command='python /opt/airflow/dags/repo/dags/business/business/community/life.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + # ) + # t10 = BashOperator( + # task_id='relative_price_ranking', + # bash_command='python /opt/airflow/dags/repo/dags/business/business/community/relative_price_ranking.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + # ) + # t11 = BashOperator( + # task_id='consumption', + # bash_command='python /opt/airflow/dags/repo/dags/business/business/community/consumption.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + # ) + # t12 = BashOperator( + # task_id='price_rating', + # bash_command='python /opt/airflow/dags/repo/dags/business/business/community/price_rating.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + # ) + # t13 = BashOperator( + # task_id='education', + # bash_command='python /opt/airflow/dags/repo/dags/business/business/community/education.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + # ) + # t14 = BashOperator( + # task_id='education2', + # bash_command='python /opt/airflow/dags/repo/dags/business/business/community/education2.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + # ) + # t15 = BashOperator( + # task_id='extra', + # bash_command='python /opt/airflow/dags/repo/dags/business/business/community/extra.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + # ) + # t16 = BashOperator( + # task_id='other', + # bash_command='python /opt/airflow/dags/repo/dags/business/business/community/other.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + # ) + # t1 >> t2 >> t3 >> t4 >> t5 >> t6 >> t6_1 >> t7 >> t7_1 >> t8 >> t9 >> t10 >> t11 >> t12 >> t13 >> t14 >> t15 >> t16 From 48562ba02382250ac2f68e649d2fce741ae4cdc7 Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Mon, 21 Nov 2022 10:48:26 +0800 Subject: [PATCH 2/7] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=85=AC=E4=BA=A4?= =?UTF-8?q?=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/business/business/community/traffic.py | 12 +- dags/business/community_summary_dag.py | 138 ++++++++++---------- 2 files changed, 77 insertions(+), 73 deletions(-) diff --git a/dags/business/business/community/traffic.py b/dags/business/business/community/traffic.py index 57954c0..ad8cfb1 100644 --- a/dags/business/business/community/traffic.py +++ b/dags/business/business/community/traffic.py @@ -36,7 +36,9 @@ def deal(actions, item): doc_info['bus_station'] = ",".join(bus_station) bus_doc = poiutils.find_nearest_by_location(curr_city, None, [lng, lat], "交通", "公交") + log.info(bus_doc) if pd.notna(bus_doc): + log.info("开始计算最近公交") doc_info['nearest_bus_station_distance'] = int(bus_doc['sort'][0]) doc_info['nearest_bus_station'] = bus_doc['_source']['name'] @@ -46,10 +48,12 @@ def deal(actions, item): if len(subway_station) > 0: doc_info['subway_station'] = ",".join(subway_station) - bus_doc = poiutils.find_nearest_by_location(curr_city, None, [lng, lat], "交通", "地铁") - if pd.notna(bus_doc): - doc_info['nearest_subway_station_distance'] = int(bus_doc['sort'][0]) - doc_info['nearest_subway_station'] = bus_doc['_source']['name'] + subway_doc = poiutils.find_nearest_by_location(curr_city, None, [lng, lat], "交通", "地铁") + log.info(subway_doc) + if pd.notna(subway_doc): + log.info("开始计算最近地铁") + doc_info['nearest_subway_station_distance'] = int(subway_doc['sort'][0]) + doc_info['nearest_subway_station'] = subway_doc['_source']['name'] action = { "_op_type": "update", diff --git a/dags/business/community_summary_dag.py b/dags/business/community_summary_dag.py index 342d992..4777149 100644 --- a/dags/business/community_summary_dag.py +++ b/dags/business/community_summary_dag.py @@ -10,76 +10,76 @@ with DAG( catchup=True, description="地址输出信息分类数据定时更新" ) as dag: - # t1 = BashOperator( - # task_id='comm', - # bash_command='python /opt/airflow/dags/repo/dags/business/business/community/comm.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', - # ) - # t2 = BashOperator( - # task_id='price', - # bash_command='python /opt/airflow/dags/repo/dags/business/business/community/price.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', - # ) - # t3 = BashOperator( - # task_id='sell', - # bash_command='python /opt/airflow/dags/repo/dags/business/business/community/sell.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', - # ) - # t4 = BashOperator( - # task_id='avg_cost', - # bash_command='python /opt/airflow/dags/repo/dags/business/business/community/avg_cost.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', - # ) - # t5 = BashOperator( - # task_id='trade', - # bash_command='python /opt/airflow/dags/repo/dags/business/business/community/trade.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', - # ) - # t6 = BashOperator( - # task_id='total_price', - # bash_command='python /opt/airflow/dags/repo/dags/business/business/community/total_price.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', - # ) - # t6_1 = BashOperator( - # task_id='total_price_complete', - # bash_command='python /opt/airflow/dags/repo/dags/business/business/community/total_price_complete.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', - # ) + t1 = BashOperator( + task_id='comm', + bash_command='python /opt/airflow/dags/repo/dags/business/business/community/comm.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + ) + t2 = BashOperator( + task_id='price', + bash_command='python /opt/airflow/dags/repo/dags/business/business/community/price.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + ) + t3 = BashOperator( + task_id='sell', + bash_command='python /opt/airflow/dags/repo/dags/business/business/community/sell.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + ) + t4 = BashOperator( + task_id='avg_cost', + bash_command='python /opt/airflow/dags/repo/dags/business/business/community/avg_cost.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + ) + t5 = BashOperator( + task_id='trade', + bash_command='python /opt/airflow/dags/repo/dags/business/business/community/trade.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + ) + t6 = BashOperator( + task_id='total_price', + bash_command='python /opt/airflow/dags/repo/dags/business/business/community/total_price.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + ) + t6_1 = BashOperator( + task_id='total_price_complete', + bash_command='python /opt/airflow/dags/repo/dags/business/business/community/total_price_complete.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + ) t7 = BashOperator( task_id='traffic', bash_command='python /opt/airflow/dags/repo/dags/business/business/community/traffic.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', ) - # t7_1 = BashOperator( - # task_id='business_district', - # bash_command='python /opt/airflow/dags/repo/dags/business/business/community/business_district.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', - # ) - # t8 = BashOperator( - # task_id='medical', - # bash_command='python /opt/airflow/dags/repo/dags/business/business/community/medical.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', - # ) - # t9 = BashOperator( - # task_id='life', - # bash_command='python /opt/airflow/dags/repo/dags/business/business/community/life.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', - # ) - # t10 = BashOperator( - # task_id='relative_price_ranking', - # bash_command='python /opt/airflow/dags/repo/dags/business/business/community/relative_price_ranking.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', - # ) - # t11 = BashOperator( - # task_id='consumption', - # bash_command='python /opt/airflow/dags/repo/dags/business/business/community/consumption.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', - # ) - # t12 = BashOperator( - # task_id='price_rating', - # bash_command='python /opt/airflow/dags/repo/dags/business/business/community/price_rating.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', - # ) - # t13 = BashOperator( - # task_id='education', - # bash_command='python /opt/airflow/dags/repo/dags/business/business/community/education.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', - # ) - # t14 = BashOperator( - # task_id='education2', - # bash_command='python /opt/airflow/dags/repo/dags/business/business/community/education2.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', - # ) - # t15 = BashOperator( - # task_id='extra', - # bash_command='python /opt/airflow/dags/repo/dags/business/business/community/extra.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', - # ) - # t16 = BashOperator( - # task_id='other', - # bash_command='python /opt/airflow/dags/repo/dags/business/business/community/other.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', - # ) - # t1 >> t2 >> t3 >> t4 >> t5 >> t6 >> t6_1 >> t7 >> t7_1 >> t8 >> t9 >> t10 >> t11 >> t12 >> t13 >> t14 >> t15 >> t16 + t7_1 = BashOperator( + task_id='business_district', + bash_command='python /opt/airflow/dags/repo/dags/business/business/community/business_district.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + ) + t8 = BashOperator( + task_id='medical', + bash_command='python /opt/airflow/dags/repo/dags/business/business/community/medical.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + ) + t9 = BashOperator( + task_id='life', + bash_command='python /opt/airflow/dags/repo/dags/business/business/community/life.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + ) + t10 = BashOperator( + task_id='relative_price_ranking', + bash_command='python /opt/airflow/dags/repo/dags/business/business/community/relative_price_ranking.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + ) + t11 = BashOperator( + task_id='consumption', + bash_command='python /opt/airflow/dags/repo/dags/business/business/community/consumption.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + ) + t12 = BashOperator( + task_id='price_rating', + bash_command='python /opt/airflow/dags/repo/dags/business/business/community/price_rating.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + ) + t13 = BashOperator( + task_id='education', + bash_command='python /opt/airflow/dags/repo/dags/business/business/community/education.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + ) + t14 = BashOperator( + task_id='education2', + bash_command='python /opt/airflow/dags/repo/dags/business/business/community/education2.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + ) + t15 = BashOperator( + task_id='extra', + bash_command='python /opt/airflow/dags/repo/dags/business/business/community/extra.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + ) + t16 = BashOperator( + task_id='other', + bash_command='python /opt/airflow/dags/repo/dags/business/business/community/other.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + ) + t1 >> t2 >> t3 >> t4 >> t5 >> t6 >> t6_1 >> t7 >> t7_1 >> t8 >> t9 >> t10 >> t11 >> t12 >> t13 >> t14 >> t15 >> t16 From 23b8628fe1386607886e2d48f287dba3a766e7ba Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Mon, 21 Nov 2022 10:58:56 +0800 Subject: [PATCH 3/7] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=85=AC=E4=BA=A4?= =?UTF-8?q?=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/business/business/community/traffic.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/dags/business/business/community/traffic.py b/dags/business/business/community/traffic.py index ad8cfb1..8eba31d 100644 --- a/dags/business/business/community/traffic.py +++ b/dags/business/business/community/traffic.py @@ -22,6 +22,7 @@ mutex = Lock() def deal(actions, item): global count curr_city = item['city'] + log.info(curr_city) lng = commutils.format_lat_lng(item['lng']) lat = commutils.format_lat_lng(item['lat']) @@ -31,6 +32,7 @@ def deal(actions, item): doc_info = dict() # 公交 bus_station = poiutils.find_around_by_location(curr_city, 1000, [lng, lat], "交通", "公交") + log.info(bus_station) doc_info['bus_station_num'] = len(bus_station) if len(bus_station) > 0: doc_info['bus_station'] = ",".join(bus_station) @@ -81,9 +83,11 @@ def sync(city=None): seaweed_df = seaweed_df.reset_index(drop=True) actions = list() - with ThreadPoolExecutor(max_workers=30) as pool: - for index, item in seaweed_df.iterrows(): - pool.submit(deal, actions, item) + # with ThreadPoolExecutor(max_workers=30) as pool: + # for index, item in seaweed_df.iterrows(): + # pool.submit(deal, actions, item) + for index, item in seaweed_df.iterrows(): + deal(actions, item) if len(actions) > 0: helpers.bulk(es206_client.elastic_client, actions) actions.clear() From 22150627210315b2c8a4a10aa69136d34048dbc8 Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Mon, 21 Nov 2022 11:01:58 +0800 Subject: [PATCH 4/7] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=85=AC=E4=BA=A4?= =?UTF-8?q?=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/business/business/community/traffic.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dags/business/business/community/traffic.py b/dags/business/business/community/traffic.py index 8eba31d..847866d 100644 --- a/dags/business/business/community/traffic.py +++ b/dags/business/business/community/traffic.py @@ -34,7 +34,7 @@ def deal(actions, item): bus_station = poiutils.find_around_by_location(curr_city, 1000, [lng, lat], "交通", "公交") log.info(bus_station) doc_info['bus_station_num'] = len(bus_station) - if len(bus_station) > 0: + if len(bus_station.all()) > 0: doc_info['bus_station'] = ",".join(bus_station) bus_doc = poiutils.find_nearest_by_location(curr_city, None, [lng, lat], "交通", "公交") @@ -47,7 +47,7 @@ def deal(actions, item): # 地铁 subway_station = poiutils.find_around_by_location(curr_city, 1000, [lng, lat], "交通", "地铁") doc_info['subway_station_num'] = len(subway_station) - if len(subway_station) > 0: + if len(subway_station.all()) > 0: doc_info['subway_station'] = ",".join(subway_station) subway_doc = poiutils.find_nearest_by_location(curr_city, None, [lng, lat], "交通", "地铁") From d15b389fbc9b4ef9a133b22674a12813abe65dc2 Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Mon, 21 Nov 2022 11:11:56 +0800 Subject: [PATCH 5/7] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=85=AC=E4=BA=A4?= =?UTF-8?q?=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/business/business/community/traffic.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/dags/business/business/community/traffic.py b/dags/business/business/community/traffic.py index 847866d..18038e6 100644 --- a/dags/business/business/community/traffic.py +++ b/dags/business/business/community/traffic.py @@ -31,29 +31,31 @@ def deal(actions, item): doc_info = dict() # 公交 + log.info("查找公交站") bus_station = poiutils.find_around_by_location(curr_city, 1000, [lng, lat], "交通", "公交") log.info(bus_station) doc_info['bus_station_num'] = len(bus_station) - if len(bus_station.all()) > 0: + if len(bus_station) > 0: + log.info("写入公交站") doc_info['bus_station'] = ",".join(bus_station) - + log.info("写入公交站完毕") + log.info("查找最近公交站") bus_doc = poiutils.find_nearest_by_location(curr_city, None, [lng, lat], "交通", "公交") - log.info(bus_doc) + log.info("查找最近公交站完毕") if pd.notna(bus_doc): - log.info("开始计算最近公交") + log.info("写入最近公交站") doc_info['nearest_bus_station_distance'] = int(bus_doc['sort'][0]) doc_info['nearest_bus_station'] = bus_doc['_source']['name'] + log.info("写入最近公交站完毕") # 地铁 subway_station = poiutils.find_around_by_location(curr_city, 1000, [lng, lat], "交通", "地铁") doc_info['subway_station_num'] = len(subway_station) - if len(subway_station.all()) > 0: + if len(subway_station) > 0: doc_info['subway_station'] = ",".join(subway_station) subway_doc = poiutils.find_nearest_by_location(curr_city, None, [lng, lat], "交通", "地铁") - log.info(subway_doc) if pd.notna(subway_doc): - log.info("开始计算最近地铁") doc_info['nearest_subway_station_distance'] = int(subway_doc['sort'][0]) doc_info['nearest_subway_station'] = subway_doc['_source']['name'] From de1854cd244c7e1ab4e657645c94fd385d9cead4 Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Mon, 21 Nov 2022 11:20:30 +0800 Subject: [PATCH 6/7] =?UTF-8?q?=E5=85=AC=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E8=BF=98=E5=8E=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/business/business/community/traffic.py | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/dags/business/business/community/traffic.py b/dags/business/business/community/traffic.py index 18038e6..db57785 100644 --- a/dags/business/business/community/traffic.py +++ b/dags/business/business/community/traffic.py @@ -31,22 +31,14 @@ def deal(actions, item): doc_info = dict() # 公交 - log.info("查找公交站") bus_station = poiutils.find_around_by_location(curr_city, 1000, [lng, lat], "交通", "公交") - log.info(bus_station) doc_info['bus_station_num'] = len(bus_station) if len(bus_station) > 0: - log.info("写入公交站") doc_info['bus_station'] = ",".join(bus_station) - log.info("写入公交站完毕") - log.info("查找最近公交站") bus_doc = poiutils.find_nearest_by_location(curr_city, None, [lng, lat], "交通", "公交") - log.info("查找最近公交站完毕") if pd.notna(bus_doc): - log.info("写入最近公交站") doc_info['nearest_bus_station_distance'] = int(bus_doc['sort'][0]) doc_info['nearest_bus_station'] = bus_doc['_source']['name'] - log.info("写入最近公交站完毕") # 地铁 subway_station = poiutils.find_around_by_location(curr_city, 1000, [lng, lat], "交通", "地铁") @@ -85,11 +77,9 @@ def sync(city=None): seaweed_df = seaweed_df.reset_index(drop=True) actions = list() - # with ThreadPoolExecutor(max_workers=30) as pool: - # for index, item in seaweed_df.iterrows(): - # pool.submit(deal, actions, item) - for index, item in seaweed_df.iterrows(): - deal(actions, item) + with ThreadPoolExecutor(max_workers=30) as pool: + for index, item in seaweed_df.iterrows(): + pool.submit(deal, actions, item) if len(actions) > 0: helpers.bulk(es206_client.elastic_client, actions) actions.clear() From f7e434244cc728996942e3c9976799bdea3526db Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Mon, 21 Nov 2022 11:27:22 +0800 Subject: [PATCH 7/7] =?UTF-8?q?=E5=85=AC=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E8=BF=98=E5=8E=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/business/business/community/traffic.py | 1 - 1 file changed, 1 deletion(-) diff --git a/dags/business/business/community/traffic.py b/dags/business/business/community/traffic.py index db57785..dade577 100644 --- a/dags/business/business/community/traffic.py +++ b/dags/business/business/community/traffic.py @@ -22,7 +22,6 @@ mutex = Lock() def deal(actions, item): global count curr_city = item['city'] - log.info(curr_city) lng = commutils.format_lat_lng(item['lng']) lat = commutils.format_lat_lng(item['lat'])