diff --git a/dags/business/business/community/avg_cost.py b/dags/business/business/community/avg_cost.py index 153481d..b9467d9 100644 --- a/dags/business/business/community/avg_cost.py +++ b/dags/business/business/community/avg_cost.py @@ -3,7 +3,7 @@ """ import logging import pandas as pd -from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils +from goojia_utils import ElasticSearchClient, commutils, dateutils from elasticsearch import helpers import numpy as np from concurrent.futures import ThreadPoolExecutor @@ -13,6 +13,7 @@ es206_client = ElasticSearchClient(host='192.168.0.206') index_name = "community_summary" m_date = dateutils.iso_format() +path = "/home/airflow/data/community" def find_around_price(lng, lat): @@ -100,7 +101,7 @@ def sync(): # 按最新的百分位更新库中价格状态 update_status(new_df) - seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@', encoding='UTF-8') + seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8') def find_around_price_item(index, item): if 'lng' not in item: diff --git a/dags/business/business/community/comm.py b/dags/business/business/community/comm.py index 51cf489..c5d61f5 100644 --- a/dags/business/business/community/comm.py +++ b/dags/business/business/community/comm.py @@ -6,6 +6,7 @@ import logging import pandas as pd from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils from elasticsearch import helpers +import os log = logging.getLogger(__name__) seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') @@ -14,6 +15,9 @@ es206_client = ElasticSearchClient(host='192.168.0.206') index_name = "community_summary" m_date = dateutils.iso_format() +path = "/home/airflow/data/community" +if not os.path.exists(path): + os.makedirs(path) def build_doc(item): """ @@ -120,5 +124,5 @@ def sync(city=None): actions.clear() # 写入csv文件,防止小区变动导致的后续过程更新报错 - seaweed_df.to_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@', encoding='UTF-8', index=False) + seaweed_df.to_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8', index=False) log.info(f"完成同步小区基本属性至206 {index_name}") diff --git a/dags/business/business/community/data/seaweed.csv b/dags/business/business/community/data/seaweed.csv deleted file mode 100644 index e69de29..0000000 diff --git a/dags/business/business/community/life.py b/dags/business/business/community/life.py index b09f6aa..12ebde9 100644 --- a/dags/business/business/community/life.py +++ b/dags/business/business/community/life.py @@ -12,6 +12,7 @@ log = logging.getLogger(__name__) es206_client = ElasticSearchClient(host='192.168.0.206') index_name = "community_summary" +path = "/home/airflow/data/community" m_date = dateutils.iso_format() count = 0 @@ -79,7 +80,7 @@ def sync(city=None): if pd.notna(city): seaweed_query['city'] = city - seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@', encoding='UTF-8') + seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8') actions = list() with ThreadPoolExecutor(max_workers=30) as pool: diff --git a/dags/business/business/community/medical.py b/dags/business/business/community/medical.py index cd07ddb..bcb28a0 100644 --- a/dags/business/business/community/medical.py +++ b/dags/business/business/community/medical.py @@ -13,6 +13,7 @@ seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') es206_client = ElasticSearchClient(host='192.168.0.206') index_name = "community_summary" +path = "/home/airflow/data/community" m_date = dateutils.iso_format() count = 0 @@ -67,7 +68,7 @@ def sync(city=None): if pd.notna(city): seaweed_query['city'] = city - seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@', encoding='UTF-8') + seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8') actions = list() with ThreadPoolExecutor(max_workers=30) as pool: diff --git a/dags/business/business/community/price.py b/dags/business/business/community/price.py index 170dae9..f5d306a 100644 --- a/dags/business/business/community/price.py +++ b/dags/business/business/community/price.py @@ -10,6 +10,7 @@ log = logging.getLogger(__name__) es206_client = ElasticSearchClient(host='192.168.0.206') index_name = "community_summary" +path = "/home/airflow/data/community" m_date = dateutils.iso_format() @@ -42,7 +43,7 @@ def sync(city=None): cities = [city] else: cities = None - seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@', encoding='UTF-8') + seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8') # 小区所在区域均价 region_price_df = commutils.get_region_price(city=city) diff --git a/dags/business/business/community/relative_price_ranking.py b/dags/business/business/community/relative_price_ranking.py index 1e70d61..cbc914f 100644 --- a/dags/business/business/community/relative_price_ranking.py +++ b/dags/business/business/community/relative_price_ranking.py @@ -13,6 +13,7 @@ log = logging.getLogger(__name__) es206_client = ElasticSearchClient(host='192.168.0.206') index_name = "community_summary" +path = "/home/airflow/data/community" m_date = dateutils.iso_format() @@ -57,7 +58,7 @@ def build_doc(item): def sync(): log.info("开始同步相对价格排名数据...") - seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@', encoding='UTF-8') + seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8') def find_around_price_item(index, item): if 'lng' not in item: diff --git a/dags/business/business/community/sell.py b/dags/business/business/community/sell.py index 151cbd0..db92878 100644 --- a/dags/business/business/community/sell.py +++ b/dags/business/business/community/sell.py @@ -12,6 +12,7 @@ seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') es206_client = ElasticSearchClient(host='192.168.0.206') index_name = "community_summary" +path = "/home/airflow/data/community" m_date = dateutils.iso_format() @@ -28,7 +29,7 @@ def sync(city=None): seaweed_query = {'cat': 'district', 'status': 0} if pd.notna(city): seaweed_query['city'] = city - seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@', encoding='UTF-8') + seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8') with ThreadPoolExecutor(50) as pool: def deal(index, item): diff --git a/dags/business/business/community/total_price.py b/dags/business/business/community/total_price.py index 280bc58..2b74fdc 100644 --- a/dags/business/business/community/total_price.py +++ b/dags/business/business/community/total_price.py @@ -13,6 +13,7 @@ seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') es206_client = ElasticSearchClient(host='192.168.0.206') index_name = "community_summary" +path = "/home/airflow/data/community" m_date = dateutils.iso_format() @@ -28,7 +29,7 @@ def sync(city=None): seaweed_query = {'cat': 'district', 'status': 0} if pd.notna(city): seaweed_query['city'] = city - seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@', encoding='UTF-8') + seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8') # 加载库中均价数据 query_sql = "SELECT city, region, name, avg_price FROM community_summary " diff --git a/dags/business/business/community/trade.py b/dags/business/business/community/trade.py index bc033d1..85b9582 100644 --- a/dags/business/business/community/trade.py +++ b/dags/business/business/community/trade.py @@ -10,6 +10,7 @@ log = logging.getLogger(__name__) es206_client = ElasticSearchClient(host='192.168.0.206') index_name = "community_summary" +path = "/home/airflow/data/community" m_date = dateutils.iso_format() @@ -28,7 +29,7 @@ def sync(city=None): if pd.notna(city): seaweed_query['city'] = city cities = [city] - seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@', encoding='UTF-8') + seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8') deal_price_df = commutils.get_deal_avg_price(cities, 3) deal_price_df = deal_price_df[['match_city', 'match_region', 'match_name', 'avg_price']] diff --git a/dags/business/business/community/traffic.py b/dags/business/business/community/traffic.py index 728bbaa..9cba530 100644 --- a/dags/business/business/community/traffic.py +++ b/dags/business/business/community/traffic.py @@ -12,6 +12,7 @@ log = logging.getLogger(__name__) es206_client = ElasticSearchClient(host='192.168.0.206') index_name = "community_summary" +path = "/home/airflow/data/community" m_date = dateutils.iso_format() count = 0 @@ -76,8 +77,7 @@ def sync(city=None): if pd.notna(city): seaweed_query['city'] = city - seaweed_df = seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@', - encoding='UTF-8') + seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8') actions = list() with ThreadPoolExecutor(max_workers=30) as pool: