diff --git a/dags/business/business/community/avg_cost.py b/dags/business/business/community/avg_cost.py index 166f21d..b6f72f1 100644 --- a/dags/business/business/community/avg_cost.py +++ b/dags/business/business/community/avg_cost.py @@ -5,13 +5,12 @@ import logging import pandas as pd from goojia_utils import ElasticSearchClient, commutils, dateutils, areautils from elasticsearch import helpers -import numpy as np from concurrent.futures import ThreadPoolExecutor import sys -import threading +from threading import Lock # 创建线程锁 -lock = threading.Lock() +mutex = Lock() log = logging.getLogger(__name__) es206_client = ElasticSearchClient(host='192.168.0.206') @@ -71,12 +70,12 @@ def find_around_price_item(seaweed_df, index, item): lat = commutils.format_lat_lng(item['lat']) city_now = areautils.get_city_display(item['city']) price = find_around_price(city_now, lng, lat) - lock.acquire() + mutex.acquire(timeout=60) if pd.notna(price): seaweed_df.loc[index, 'avg_cost'] = int(price) else: seaweed_df.loc[index, 'avg_cost'] = None - lock.release() + mutex.release() def sync(city=None): diff --git a/dags/business/business/community/estate_charge_complete.py b/dags/business/business/community/estate_charge_complete.py index f1f9947..ed03faa 100644 --- a/dags/business/business/community/estate_charge_complete.py +++ b/dags/business/business/community/estate_charge_complete.py @@ -7,12 +7,14 @@ import logging import sys from elasticsearch import helpers from concurrent.futures import ThreadPoolExecutor +from threading import Lock log = logging.getLogger(__name__) es206_client = ElasticSearchClient(host='192.168.0.206') index_name = "community_summary" +mutex = Lock() def save(community_summary_df): if 'estate_charge' not in community_summary_df.columns: @@ -127,7 +129,9 @@ def sync_by_city(city_now): # 取周边均值 estate_charge = docs['estate_charge'].mean() if pd.notna(estate_charge): + mutex.acquire(timeout=60) community_summary_df.loc[index, "estate_charge"] = estate_charge + mutex.release() for index, item in community_summary_df.iterrows(): pool.submit(deal, index, item) save(community_summary_df) diff --git a/dags/business/business/community/relative_price_ranking.py b/dags/business/business/community/relative_price_ranking.py index effbbd6..8c0e2ac 100644 --- a/dags/business/business/community/relative_price_ranking.py +++ b/dags/business/business/community/relative_price_ranking.py @@ -8,6 +8,7 @@ from goojia_utils import ElasticSearchClient, commutils, dateutils from elasticsearch import helpers from concurrent.futures import ThreadPoolExecutor import sys +from threading import Lock log = logging.getLogger(__name__) es206_client = ElasticSearchClient(host='192.168.0.206') @@ -15,7 +16,7 @@ es206_client = ElasticSearchClient(host='192.168.0.206') index_name = "community_summary" path = "/home/airflow/data/community" m_date = dateutils.iso_format() - +mutex = Lock() def find_around_seaweed(lng, lat): """ @@ -72,11 +73,13 @@ def sync(city=None): lat = commutils.format_lat_lng(item['lat']) if pd.notna(lng): around_seaweed_df = find_around_seaweed(lng, lat) + mutex.acquire(timeout=60) if len(around_seaweed_df) > 1: around_seaweed_df['relative_price_ranking'] = around_seaweed_df['avg_price'].rank(pct=True) seaweed_df.loc[index, 'relative_price_ranking'] = around_seaweed_df.loc[around_seaweed_df['id'] == item['_id']]['relative_price_ranking'].tolist()[0] elif len(around_seaweed_df) == 1: seaweed_df.loc[index, 'relative_price_ranking'] = 0.5 + mutex.release() with ThreadPoolExecutor(max_workers=20) as pool: for index, item in seaweed_df.iterrows(): diff --git a/dags/business/business/community/sell.py b/dags/business/business/community/sell.py index a5cd0c2..704b178 100644 --- a/dags/business/business/community/sell.py +++ b/dags/business/business/community/sell.py @@ -7,6 +7,7 @@ from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, date from elasticsearch import helpers from concurrent.futures import ThreadPoolExecutor import sys +from threading import Lock log = logging.getLogger(__name__) seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') @@ -15,6 +16,7 @@ es206_client = ElasticSearchClient(host='192.168.0.206') index_name = "community_summary" path = "/home/airflow/data/community" m_date = dateutils.iso_format() +mutex = Lock() def build_doc(item): @@ -51,11 +53,14 @@ def sync(city=None): # 近三个月挂牌量 sample_num = commutils.get_sample(curr_city, curr_region, names_str, cat='sell', daysBefore=93) - seaweed_df.loc[index, 'three_months_sell_num'] = sample_num # 近半年挂牌量 - sample_num = commutils.get_sample(curr_city, curr_region, names_str, cat='sell', daysBefore=186) - seaweed_df.loc[index, 'half_on_year_sell_num'] = sample_num + sample_half_year_num = commutils.get_sample(curr_city, curr_region, names_str, cat='sell', daysBefore=186) + + mutex.acquire() + seaweed_df.loc[index, 'three_months_sell_num'] = sample_num + seaweed_df.loc[index, 'half_on_year_sell_num'] = sample_half_year_num + mutex.release() for index, item in seaweed_df.iterrows(): pool.submit(deal, index, item) diff --git a/dags/business/community_summary_dag.py b/dags/business/community_summary_dag.py index f2d734b..f463aef 100644 --- a/dags/business/community_summary_dag.py +++ b/dags/business/community_summary_dag.py @@ -96,4 +96,4 @@ with DAG( bash_command='python /opt/airflow/dags/repo/dags/business/business/community/other.py {{dag_run.conf["city"] if "city" in dag_run.conf}}', ) # 人均消费并行,值为空 - comm >> price >> avg_cost >> [total_price, relative_price_ranking, estate_charge_complete] >> consumption >> [estate_charge_ranking, price_rating, person_rating] >> sell >> trade >> [traffic, business_district, medical, life, education] >> education2 >> extra >> other + comm >> price >> [avg_cost, total_price, relative_price_ranking, estate_charge_complete] >> consumption >> [estate_charge_ranking, price_rating, person_rating] >> sell >> trade >> [traffic, business_district, medical, life, education] >> education2 >> extra >> other