From b8e8d79821a11de5bde6bc145f51a65e3e01306c Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Fri, 17 Mar 2023 11:26:43 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96=E4=B8=AD=E9=97=B4=E8=A1=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/business/business/community/avg_cost.py | 13 ++++---- .../business/community/business_district.py | 13 ++++---- dags/business/business/community/comm.py | 8 ++--- dags/business/business/community/education.py | 13 ++++---- .../business/business/community/education2.py | 13 ++++---- dags/business/business/community/life.py | 13 ++++---- dags/business/business/community/medical.py | 14 ++++---- dags/business/business/community/price.py | 32 +++++++++---------- .../community/relative_price_ranking.py | 13 ++++---- dags/business/business/community/sell.py | 9 +++--- dags/business/business/community/trade.py | 10 +++--- dags/business/business/community/traffic.py | 13 ++++---- 12 files changed, 87 insertions(+), 77 deletions(-) diff --git a/dags/business/business/community/avg_cost.py b/dags/business/business/community/avg_cost.py index 045e064..f7381e2 100644 --- a/dags/business/business/community/avg_cost.py +++ b/dags/business/business/community/avg_cost.py @@ -3,7 +3,7 @@ """ import logging import pandas as pd -from goojia_utils import ElasticSearchClient, commutils, dateutils, areautils +from goojia_utils import ElasticSearchClient, commutils, dateutils, areautils, ClickHouseClient from elasticsearch import helpers from concurrent.futures import ThreadPoolExecutor import sys @@ -12,6 +12,7 @@ from threading import Lock mutex = Lock() log = logging.getLogger(__name__) es206_client = ElasticSearchClient(host='192.168.0.206') +tmp_ck = ClickHouseClient(database='tmp') index_name = "community_summary" m_date = dateutils.iso_format() @@ -111,10 +112,10 @@ def find_around_price_item(seaweed): def sync(city=None): log.info("开始同步消费数据...") - seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8') + tmp_sql = f"select * from community_summary " if pd.notna(city): - seaweed_df = seaweed_df[seaweed_df['city'] == city].copy() - seaweed_df = seaweed_df.reset_index(drop=True) + tmp_sql += f" where city='{city}'" + seaweed_df = tmp_ck.read_sql_query(tmp_sql) seaweed_list = list() seaweed_df.apply(lambda x: seaweed_list.append(dict(x)), axis=1) @@ -153,5 +154,5 @@ def sync(city=None): log.info("完成同步消费数据...") -city = sys.argv[1] if len(sys.argv) > 1 else None -sync(city) +sys_city = sys.argv[1] if len(sys.argv) > 1 else None +sync(sys_city) diff --git a/dags/business/business/community/business_district.py b/dags/business/business/community/business_district.py index ccece89..3c26117 100644 --- a/dags/business/business/community/business_district.py +++ b/dags/business/business/community/business_district.py @@ -3,7 +3,7 @@ """ import logging import pandas as pd -from goojia_utils import ElasticSearchClient, commutils, dateutils +from goojia_utils import ElasticSearchClient, commutils, dateutils, ClickHouseClient from elasticsearch import helpers from concurrent.futures import ThreadPoolExecutor from threading import Lock @@ -11,6 +11,7 @@ import sys log = logging.getLogger(__name__) es206_client = ElasticSearchClient(host='192.168.0.206') +tmp_ck = ClickHouseClient(database="tmp") index_name = "community_summary" path = "/home/airflow/data/community" @@ -89,10 +90,10 @@ def deal(actions, item): def sync(city=None): log.info(f"开始同步商圈至206 {index_name}") - seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8') + tmp_sql = f"select * from community_summary " if pd.notna(city): - seaweed_df = seaweed_df[seaweed_df['city'] == city].copy() - seaweed_df = seaweed_df.reset_index(drop=True) + tmp_sql += f" where city='{city}'" + seaweed_df = tmp_ck.read_sql_query(tmp_sql) actions = list() with ThreadPoolExecutor(max_workers=30) as pool: @@ -105,5 +106,5 @@ def sync(city=None): log.info(f"完成同步商圈至206 {index_name}") -city = sys.argv[1] if len(sys.argv) > 1 else None -sync(city) \ No newline at end of file +sys_city = sys.argv[1] if len(sys.argv) > 1 else None +sync(sys_city) \ No newline at end of file diff --git a/dags/business/business/community/comm.py b/dags/business/business/community/comm.py index a1748fe..2d6ce04 100644 --- a/dags/business/business/community/comm.py +++ b/dags/business/business/community/comm.py @@ -1,10 +1,10 @@ """ 同步小区基本信息至ES206 """ -# import sys -# sys.path.insert(0, '/opt/airflow/dags/repo') -# import dags.install as install -# install.install() +import sys +sys.path.insert(0, '/opt/airflow/dags/repo') +import dags.install as install +install.install() import logging import pandas as pd diff --git a/dags/business/business/community/education.py b/dags/business/business/community/education.py index f00e718..916c9c1 100644 --- a/dags/business/business/community/education.py +++ b/dags/business/business/community/education.py @@ -3,7 +3,7 @@ """ import logging import pandas as pd -from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils +from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils, ClickHouseClient from elasticsearch import helpers from concurrent.futures import ThreadPoolExecutor from threading import Lock @@ -11,6 +11,7 @@ import sys log = logging.getLogger(__name__) es206_client = ElasticSearchClient(host='192.168.0.206') +tmp_ck = ClickHouseClient(database="tmp") index_name = "community_summary" path = "/home/airflow/data/community" @@ -63,10 +64,10 @@ def deal(actions, item): def sync(city=None): log.info(f"开始同步教育资源【幼儿园、大学】至206 {index_name}") - seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8') + tmp_sql = f"select * from community_summary " if pd.notna(city): - seaweed_df = seaweed_df[seaweed_df['city'] == city].copy() - seaweed_df = seaweed_df.reset_index(drop=True) + tmp_sql += f" where city='{city}'" + seaweed_df = tmp_ck.read_sql_query(tmp_sql) actions = list() with ThreadPoolExecutor(max_workers=30) as pool: @@ -83,5 +84,5 @@ def sync(city=None): log.info(f"完成同步教育资源至206 {index_name}") -city = sys.argv[1] if len(sys.argv) > 1 else None -sync(city) \ No newline at end of file +sys_city = sys.argv[1] if len(sys.argv) > 1 else None +sync(sys_city) \ No newline at end of file diff --git a/dags/business/business/community/education2.py b/dags/business/business/community/education2.py index ec37083..7d83350 100644 --- a/dags/business/business/community/education2.py +++ b/dags/business/business/community/education2.py @@ -3,7 +3,7 @@ """ import logging import pandas as pd -from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils +from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils, ClickHouseClient from elasticsearch import helpers from concurrent.futures import ThreadPoolExecutor from threading import Lock @@ -11,6 +11,7 @@ import sys log = logging.getLogger(__name__) es206_client = ElasticSearchClient(host='192.168.0.206') +tmp_ck = ClickHouseClient(database="tmp") index_name = "community_summary" path = "/home/airflow/data/community" @@ -76,10 +77,10 @@ def deal(actions, item): def sync(city=None): log.info(f"开始同步教育资源【小学、中学】至206 {index_name}") - seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8') + tmp_sql = f"select * from community_summary " if pd.notna(city): - seaweed_df = seaweed_df[seaweed_df['city'] == city].copy() - seaweed_df = seaweed_df.reset_index(drop=True) + tmp_sql += f" where city='{city}'" + seaweed_df = tmp_ck.read_sql_query(tmp_sql) actions = list() with ThreadPoolExecutor(max_workers=30) as pool: @@ -96,5 +97,5 @@ def sync(city=None): log.info(f"完成同步教育资源至206 {index_name}") -city = sys.argv[1] if len(sys.argv) > 1 else None -sync(city) \ No newline at end of file +sys_city = sys.argv[1] if len(sys.argv) > 1 else None +sync(sys_city) \ No newline at end of file diff --git a/dags/business/business/community/life.py b/dags/business/business/community/life.py index 0e62d00..7d0ba49 100644 --- a/dags/business/business/community/life.py +++ b/dags/business/business/community/life.py @@ -3,7 +3,7 @@ """ import logging import pandas as pd -from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils +from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils, ClickHouseClient from elasticsearch import helpers from concurrent.futures import ThreadPoolExecutor from threading import Lock @@ -11,6 +11,7 @@ import sys log = logging.getLogger(__name__) es206_client = ElasticSearchClient(host='192.168.0.206') +tmp_ck = ClickHouseClient(database='tmp') index_name = "community_summary" path = "/home/airflow/data/community" @@ -77,10 +78,10 @@ def deal(actions, item): def sync(city=None): log.info(f"开始同步生活配套至206 {index_name}") - seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8') + tmp_sql = f"select * from community_summary " if pd.notna(city): - seaweed_df = seaweed_df[seaweed_df['city'] == city].copy() - seaweed_df = seaweed_df.reset_index(drop=True) + tmp_sql += f" where city='{city}'" + seaweed_df = tmp_ck.read_sql_query(tmp_sql) actions = list() with ThreadPoolExecutor(max_workers=30) as pool: @@ -97,5 +98,5 @@ def sync(city=None): log.info(f"完成同步生活配套至206 {index_name}") -city = sys.argv[1] if len(sys.argv) > 1 else None -sync(city) \ No newline at end of file +sys_city = sys.argv[1] if len(sys.argv) > 1 else None +sync(sys_city) \ No newline at end of file diff --git a/dags/business/business/community/medical.py b/dags/business/business/community/medical.py index 536b871..0de3227 100644 --- a/dags/business/business/community/medical.py +++ b/dags/business/business/community/medical.py @@ -3,7 +3,7 @@ """ import logging import pandas as pd -from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, areautils, poiutils +from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, ClickHouseClient, poiutils from elasticsearch import helpers from concurrent.futures import ThreadPoolExecutor from threading import Lock @@ -12,6 +12,7 @@ import sys log = logging.getLogger(__name__) seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') es206_client = ElasticSearchClient(host='192.168.0.206') +tmp_ck = ClickHouseClient(database='tmp') index_name = "community_summary" path = "/home/airflow/data/community" @@ -63,11 +64,10 @@ def deal(actions, item): def sync(city=None): log.info(f"开始同步医院至206 {index_name}") - seaweed_query = {'status': 0, 'cat': 'district'} + tmp_sql = f"select * from community_summary " if pd.notna(city): - seaweed_query['city'] = city - - seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8') + tmp_sql += f" where city='{city}'" + seaweed_df = tmp_ck.read_sql_query(tmp_sql) actions = list() with ThreadPoolExecutor(max_workers=30) as pool: @@ -84,5 +84,5 @@ def sync(city=None): log.info(f"完成同步医院至206 {index_name}") -city = sys.argv[1] if len(sys.argv) > 1 else None -sync(city) \ No newline at end of file +sys_city = sys.argv[1] if len(sys.argv) > 1 else None +sync(sys_city) \ No newline at end of file diff --git a/dags/business/business/community/price.py b/dags/business/business/community/price.py index 7d967ad..3665536 100644 --- a/dags/business/business/community/price.py +++ b/dags/business/business/community/price.py @@ -3,12 +3,13 @@ """ import logging import pandas as pd -from goojia_utils import ElasticSearchClient, commutils, dateutils, areautils +from goojia_utils import ElasticSearchClient, commutils, dateutils, areautils, ClickHouseClient from elasticsearch import helpers import sys log = logging.getLogger(__name__) es206_client = ElasticSearchClient(host='192.168.0.206') +tmp_ck = ClickHouseClient(database='tmp') index_name = "community_summary" path = "/home/airflow/data/community" @@ -38,11 +39,10 @@ def build_doc(item): return doc_dict -def sync_by_city(city, total_seaweed_df): - seaweed_df = total_seaweed_df[total_seaweed_df['city'] == city].copy() +def sync_by_city(city): + seaweed_df = tmp_ck.read_sql_query(f"select * from community_summary where city='{city}'") if len(seaweed_df) == 0: return - seaweed_df = seaweed_df.reset_index(drop=True) # 小区所在城市均价 city_price_df = commutils.get_city_price(city=city) @@ -67,10 +67,11 @@ def sync_by_city(city, total_seaweed_df): # 小区均价区域排名 seaweed_df['avg_price_region_ranking'] = \ - seaweed_df.groupby(by=['city', 'region']).rank(method='first', na_option='bottom', ascending=False)['avg_price'] + seaweed_df.groupby(by=['city', 'region']).rank(method='first', na_option='bottom', ascending=False)[ + 'avg_price'] # 小区均价城市排名 seaweed_df['avg_price_ranking'] = \ - seaweed_df.groupby(by=['city']).rank(method='first', na_option='bottom', ascending=False)['avg_price'] + seaweed_df.groupby(by=['city']).rank(method='first', na_option='bottom', ascending=False)['avg_price'] # 12个月均价涨跌幅 avg_price_up_and_downs_df = commutils.get_avg_price_up_and_downs(cities=[city], months=-12) @@ -125,20 +126,19 @@ def sync_by_city(city, total_seaweed_df): actions.clear() -def sync(city=None): +def sync(curr_city=None): log.info(f"开始同步价格信息至206 {index_name}") - if pd.notna(city): - cities = [city] + if pd.notna(curr_city): + cities = [curr_city] else: cities = areautils.get_all_cities() - seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8') - for city in cities: - log.info(f"开始同步{city}") - sync_by_city(city, seaweed_df) - log.info(f"完成同步{city}") + for city_now in cities: + log.info(f"开始同步{city_now}") + sync_by_city(city_now) + log.info(f"完成同步{city_now}") log.info(f"完成同步价格信息至206 {index_name}") -city = sys.argv[1] if len(sys.argv) > 1 else None -sync(city) \ No newline at end of file +sys_city = sys.argv[1] if len(sys.argv) > 1 else None +sync(sys_city) diff --git a/dags/business/business/community/relative_price_ranking.py b/dags/business/business/community/relative_price_ranking.py index 5cdf898..bb8323c 100644 --- a/dags/business/business/community/relative_price_ranking.py +++ b/dags/business/business/community/relative_price_ranking.py @@ -4,7 +4,7 @@ """ import logging import pandas as pd -from goojia_utils import ElasticSearchClient, commutils, dateutils +from goojia_utils import ElasticSearchClient, commutils, dateutils, ClickHouseClient from elasticsearch import helpers from concurrent.futures import ThreadPoolExecutor import sys @@ -12,6 +12,7 @@ from threading import Lock log = logging.getLogger(__name__) es206_client = ElasticSearchClient(host='192.168.0.206') +tmp_ck = ClickHouseClient(database='tmp') index_name = "community_summary" path = "/home/airflow/data/community" @@ -92,10 +93,10 @@ def find_around_price_item(seaweed): def sync(city=None): log.info("开始同步相对价格排名数据...") - seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8') + tmp_sql = f"select * from community_summary " if pd.notna(city): - seaweed_df = seaweed_df[seaweed_df['city'] == city].copy() - seaweed_df = seaweed_df.reset_index(drop=True) + tmp_sql += f" where city='{city}'" + seaweed_df = tmp_ck.read_sql_query(tmp_sql) seaweed_list = list() seaweed_df.apply(lambda x: seaweed_list.append(dict(x)), axis=1) @@ -138,5 +139,5 @@ def sync(city=None): log.info("完成同步相对价格排名数据...") -city = sys.argv[1] if len(sys.argv) > 1 else None -sync(city) +sys_city = sys.argv[1] if len(sys.argv) > 1 else None +sync(sys_city) diff --git a/dags/business/business/community/sell.py b/dags/business/business/community/sell.py index 704b178..6de2b7c 100644 --- a/dags/business/business/community/sell.py +++ b/dags/business/business/community/sell.py @@ -3,7 +3,7 @@ """ import logging import pandas as pd -from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils +from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, ClickHouseClient from elasticsearch import helpers from concurrent.futures import ThreadPoolExecutor import sys @@ -12,6 +12,7 @@ from threading import Lock log = logging.getLogger(__name__) seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') es206_client = ElasticSearchClient(host='192.168.0.206') +tmp_ck = ClickHouseClient(database='tmp') index_name = "community_summary" path = "/home/airflow/data/community" @@ -32,10 +33,10 @@ def build_doc(item): def sync(city=None): log.info(f"开始同步出售信息至206 {index_name}") - seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8') + tmp_sql = f"select * from community_summary " if pd.notna(city): - seaweed_df = seaweed_df[seaweed_df['city'] == city].copy() - seaweed_df = seaweed_df.reset_index(drop=True) + tmp_sql += f" where city='{city}'" + seaweed_df = tmp_ck.read_sql_query(tmp_sql) with ThreadPoolExecutor(30) as pool: def deal(index, item): diff --git a/dags/business/business/community/trade.py b/dags/business/business/community/trade.py index d3a22d8..8036ec8 100644 --- a/dags/business/business/community/trade.py +++ b/dags/business/business/community/trade.py @@ -4,11 +4,12 @@ import logging import pandas as pd import sys -from goojia_utils import ElasticSearchClient, commutils, dateutils +from goojia_utils import ElasticSearchClient, commutils, dateutils, ClickHouseClient from elasticsearch import helpers log = logging.getLogger(__name__) es206_client = ElasticSearchClient(host='192.168.0.206') +tmp_ck = ClickHouseClient(database='tmp') index_name = "community_summary" path = "/home/airflow/data/community" @@ -26,11 +27,12 @@ def sync(city=None): log.info(f"开始同步成交信息至206 {index_name}") cities = None - seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8') + + tmp_sql = f"select * from community_summary " if pd.notna(city): cities = [city] - seaweed_df = seaweed_df[seaweed_df['city'] == city].copy() - seaweed_df = seaweed_df.reset_index(drop=True) + tmp_sql += f" where city='{city}'" + seaweed_df = tmp_ck.read_sql_query(tmp_sql) deal_price_df = commutils.get_deal_avg_price(cities, 3) if len(deal_price_df) <= 0: diff --git a/dags/business/business/community/traffic.py b/dags/business/business/community/traffic.py index 34e90ad..7158f68 100644 --- a/dags/business/business/community/traffic.py +++ b/dags/business/business/community/traffic.py @@ -3,7 +3,7 @@ """ import logging import pandas as pd -from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils +from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils, ClickHouseClient from elasticsearch import helpers from concurrent.futures import ThreadPoolExecutor from threading import Lock @@ -11,6 +11,7 @@ import sys log = logging.getLogger(__name__) es206_client = ElasticSearchClient(host='192.168.0.206') +tmp_ck = ClickHouseClient(database='tmp') index_name = "community_summary" path = "/home/airflow/data/community" @@ -71,10 +72,10 @@ def deal(actions, item): def sync(city=None): log.info(f"开始同步交通至206 {index_name}") - seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8') + tmp_sql = f"select * from community_summary " if pd.notna(city): - seaweed_df = seaweed_df[seaweed_df['city'] == city].copy() - seaweed_df = seaweed_df.reset_index(drop=True) + tmp_sql += f" where city='{city}'" + seaweed_df = tmp_ck.read_sql_query(tmp_sql) actions = list() with ThreadPoolExecutor(max_workers=30) as pool: @@ -87,5 +88,5 @@ def sync(city=None): log.info(f"完成同步交通至206 {index_name}") -city = sys.argv[1] if len(sys.argv) > 1 else None -sync(city) \ No newline at end of file +sys_city = sys.argv[1] if len(sys.argv) > 1 else None +sync(sys_city) \ No newline at end of file