From f5909fa36aed14d9133a83f00275f07fad39c0dd Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Fri, 21 Oct 2022 13:19:43 +0800 Subject: [PATCH] =?UTF-8?q?=E6=94=B9=E4=B8=BA=E5=BC=80=E5=8F=91=E5=BA=93?= =?UTF-8?q?=E6=9F=A5=E8=AF=A2=EF=BC=8C=E8=B5=B0=E7=B4=A2=E5=BC=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/business/business/community/avg_cost.py | 4 +- dags/business/business/community/comm.py | 4 +- .../business/business/community/facilities.py | 73 +++++++++++++++++++ dags/business/business/community/price.py | 4 +- dags/business/business/community/sell.py | 4 +- .../business/community/total_price.py | 4 +- dags/business/business/community/trade.py | 5 +- 7 files changed, 85 insertions(+), 13 deletions(-) create mode 100644 dags/business/business/community/facilities.py diff --git a/dags/business/business/community/avg_cost.py b/dags/business/business/community/avg_cost.py index 72561c8..e822f5e 100644 --- a/dags/business/business/community/avg_cost.py +++ b/dags/business/business/community/avg_cost.py @@ -3,13 +3,13 @@ """ import logging import pandas as pd -from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils +from goojia_utils import DEV_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils from elasticsearch import helpers import numpy as np from concurrent.futures import ThreadPoolExecutor log = logging.getLogger(__name__) -seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') +seaweed_conn = DEV_MONGO_CLIENT.get_collection('fangjia', 'seaweed') es206_client = ElasticSearchClient(host='192.168.0.206') index_name = "community_summary" diff --git a/dags/business/business/community/comm.py b/dags/business/business/community/comm.py index b892e43..7d3167e 100644 --- a/dags/business/business/community/comm.py +++ b/dags/business/business/community/comm.py @@ -4,11 +4,11 @@ import logging import pandas as pd -from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils +from goojia_utils import DEV_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils from elasticsearch import helpers log = logging.getLogger(__name__) -seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') +seaweed_conn = DEV_MONGO_CLIENT.get_collection('fangjia', 'seaweed') es206_client = ElasticSearchClient(host='192.168.0.206') index_name = "community_summary" diff --git a/dags/business/business/community/facilities.py b/dags/business/business/community/facilities.py new file mode 100644 index 0000000..fb71fbd --- /dev/null +++ b/dags/business/business/community/facilities.py @@ -0,0 +1,73 @@ +""" +周边配套相关信息同步至ES206 +""" +import logging +import pandas as pd +from goojia_utils import DEV_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, areautils +from elasticsearch import helpers + +log = logging.getLogger(__name__) +seaweed_conn = DEV_MONGO_CLIENT.get_collection('fangjia', 'seaweed') +es206_client = ElasticSearchClient(host='192.168.0.206') + +index_name = "community_summary" +m_date = dateutils.iso_format() + + +def build_doc(item): + doc_dict = dict() + if pd.notna(item['region_avg_price']): + doc_dict['region_avg_price'] = item['region_avg_price'] + return doc_dict + + +def sync(city=None): + log.info(f"开始同步周边配套至206 {index_name}") + seaweed_query = {'cat': 'district', 'status': 0} + if pd.notna(city): + seaweed_query['city'] = city + cities = [city] + else: + cities = None + seaweed_df = pd.DataFrame( + list(seaweed_conn.find(seaweed_query, {'city': 1, 'region': 1, 'name': 1}) + .batch_size(100000))) + + # 小区所在区域均价 + region_price_df = commutils.get_region_price(city=city) + region_price_df = region_price_df[['city', 'region', 'avg_price']] + region_price_df = region_price_df.rename(columns={'avg_price': 'region_avg_price'}) + seaweed_df = pd.merge(seaweed_df, region_price_df, how='left', on=['city', 'region']) + + + + seaweed_df['_id'] = seaweed_df['_id'].astype(str) + + actions = list() + count = 0 + for index, item in seaweed_df.iterrows(): + doc_info = build_doc(item) + + action = { + "_op_type": "update", + "_index": index_name, + "_id": item['_id'], + "doc": doc_info + } + + actions.append(action) + if len(actions) == 5000: + try: + helpers.bulk(es206_client.elastic_client, actions) + count += len(actions) + print("更新:" + str(count)) + except Exception as e: + print(e) + actions.clear() + if len(actions) > 0: + try: + helpers.bulk(es206_client.elastic_client, actions) + except Exception as e: + print(e) + actions.clear() + log.info(f"完成同步周边配套至206 {index_name}") \ No newline at end of file diff --git a/dags/business/business/community/price.py b/dags/business/business/community/price.py index 7585178..56c9c08 100644 --- a/dags/business/business/community/price.py +++ b/dags/business/business/community/price.py @@ -3,11 +3,11 @@ """ import logging import pandas as pd -from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, areautils +from goojia_utils import DEV_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, areautils from elasticsearch import helpers log = logging.getLogger(__name__) -seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') +seaweed_conn = DEV_MONGO_CLIENT.get_collection('fangjia', 'seaweed') es206_client = ElasticSearchClient(host='192.168.0.206') index_name = "community_summary" diff --git a/dags/business/business/community/sell.py b/dags/business/business/community/sell.py index 3729f5a..61101ce 100644 --- a/dags/business/business/community/sell.py +++ b/dags/business/business/community/sell.py @@ -3,12 +3,12 @@ """ import logging import pandas as pd -from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, areautils +from goojia_utils import DEV_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, areautils from elasticsearch import helpers from concurrent.futures import ThreadPoolExecutor log = logging.getLogger(__name__) -seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') +seaweed_conn = DEV_MONGO_CLIENT.get_collection('fangjia', 'seaweed') es206_client = ElasticSearchClient(host='192.168.0.206') index_name = "community_summary" diff --git a/dags/business/business/community/total_price.py b/dags/business/business/community/total_price.py index ce934d9..0a0989f 100644 --- a/dags/business/business/community/total_price.py +++ b/dags/business/business/community/total_price.py @@ -3,13 +3,13 @@ """ import logging import pandas as pd -from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, areautils +from goojia_utils import DEV_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, areautils from elasticsearch import helpers from concurrent.futures import ThreadPoolExecutor import datetime log = logging.getLogger(__name__) -seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') +seaweed_conn = DEV_MONGO_CLIENT.get_collection('fangjia', 'seaweed') es206_client = ElasticSearchClient(host='192.168.0.206') index_name = "community_summary" diff --git a/dags/business/business/community/trade.py b/dags/business/business/community/trade.py index 282f2d7..988abb2 100644 --- a/dags/business/business/community/trade.py +++ b/dags/business/business/community/trade.py @@ -3,12 +3,11 @@ """ import logging import pandas as pd -from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, areautils +from goojia_utils import DEV_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, areautils from elasticsearch import helpers -from concurrent.futures import ThreadPoolExecutor log = logging.getLogger(__name__) -seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') +seaweed_conn = DEV_MONGO_CLIENT.get_collection('fangjia', 'seaweed') es206_client = ElasticSearchClient(host='192.168.0.206') index_name = "community_summary"