diff --git a/dags/business/business/community/price.py b/dags/business/business/community/price.py index 8cbd276..7585178 100644 --- a/dags/business/business/community/price.py +++ b/dags/business/business/community/price.py @@ -95,11 +95,17 @@ def sync(city=None): actions.append(action) if len(actions) == 5000: - helpers.bulk(es206_client.elastic_client, actions) - count += len(actions) - print("更新:" + str(count)) + try: + helpers.bulk(es206_client.elastic_client, actions) + count += len(actions) + print("更新:" + str(count)) + except Exception as e: + print(e) actions.clear() if len(actions) > 0: - helpers.bulk(es206_client.elastic_client, actions) + try: + helpers.bulk(es206_client.elastic_client, actions) + except Exception as e: + print(e) actions.clear() log.info(f"完成同步价格信息至206 {index_name}") diff --git a/dags/business/business/community/sell.py b/dags/business/business/community/sell.py new file mode 100644 index 0000000..2a8f927 --- /dev/null +++ b/dags/business/business/community/sell.py @@ -0,0 +1,82 @@ +""" +同步挂牌、成交信息至206 +""" +import logging +import pandas as pd +from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, areautils +from elasticsearch import helpers +from concurrent.futures import ThreadPoolExecutor + +log = logging.getLogger(__name__) +seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') +es206_client = ElasticSearchClient(host='192.168.0.206') + +index_name = "community_summary" +m_date = dateutils.iso_format() + + +def build_doc(item): + doc_dict = dict() + if pd.notna(item['three_months_sell_num']): + doc_dict['three_months_sell_num'] = item['three_months_sell_num'] + + +def sync(city=None): + log.info(f"开始同步出售信息至206 {index_name}") + + seaweed_query = {'cat': 'district', 'status': 0} + if pd.notna(city): + seaweed_query['city'] = city + seaweed_df = pd.DataFrame( + list(seaweed_conn.find(seaweed_query, {'city': 1, 'region': 1, 'name': 1, 'alias': 1}).batch_size(100000))) + + # 近三个月 + with ThreadPoolExecutor(30) as pool: + def deal(index, item): + curr_city = item['city'] + curr_region = item['region'] + curr_name = item['name'] + names = list() + + if type(item['alias']) == list: + for alias_item in item['alias']: + names.append(alias_item) + names_str = "','".join(names) + names_str = "'" + names_str + "'" + + sample_num = commutils.get_sample(curr_city, curr_region, names_str, cat='sell', daysBefore=93) + seaweed_df.loc[index, 'three_months_sell_num'] = sample_num + + for index, item in seaweed_df.iterrows(): + pool.submit(deal, index, item) + + seaweed_df['_id'] = seaweed_df['_id'].astype(str) + + actions = list() + count = 0 + for index, item in seaweed_df.iterrows(): + doc_info = build_doc(item) + + action = { + "_op_type": "update", + "_index": index_name, + "_id": item['_id'], + "doc": doc_info + } + + actions.append(action) + if len(actions) == 5000: + try: + helpers.bulk(es206_client.elastic_client, actions) + count += len(actions) + print("更新:" + str(count)) + except Exception as e: + print(e) + actions.clear() + if len(actions) > 0: + try: + helpers.bulk(es206_client.elastic_client, actions) + except Exception as e: + print(e) + actions.clear() + log.info(f"完成同步出售信息至206 {index_name}") \ No newline at end of file