From 10425d7d4a5b0380b7f01639bae3a0b9a24058d9 Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Mon, 17 Oct 2022 17:12:42 +0800 Subject: [PATCH] =?UTF-8?q?=E8=B0=83=E9=80=9A=E4=BB=B7=E6=A0=BC=E4=BF=A1?= =?UTF-8?q?=E6=81=AF=E5=90=8C=E6=AD=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../business/community/community_summary.py | 3 +-- dags/business/business/community/price.py | 19 ++++++++++++------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/dags/business/business/community/community_summary.py b/dags/business/business/community/community_summary.py index 2da6762..74dba73 100644 --- a/dags/business/business/community/community_summary.py +++ b/dags/business/business/community/community_summary.py @@ -11,5 +11,4 @@ log.info("同步类型:" + type) if type == 'all': comm.sync() - #price.sync() - pass \ No newline at end of file + price.sync() \ No newline at end of file diff --git a/dags/business/business/community/price.py b/dags/business/business/community/price.py index d7ecb74..8cbd276 100644 --- a/dags/business/business/community/price.py +++ b/dags/business/business/community/price.py @@ -19,19 +19,20 @@ def build_doc(item): if pd.notna(item['region_avg_price']): doc_dict['region_avg_price'] = item['region_avg_price'] if pd.notna(item['avg_price']): - doc_dict['avg_price'] = item['avg_price'] - doc_dict['avg_price_region_ranking'] = item['avg_price_region_ranking'] - doc_dict['avg_price_ranking'] = item['avg_price_ranking'] + doc_dict['avg_price'] = int(item['avg_price']) + doc_dict['avg_price_region_ranking'] = int(item['avg_price_region_ranking']) + doc_dict['avg_price_ranking'] = int(item['avg_price_ranking']) if pd.notna(item['year_on_year']): - doc_dict['year_on_year'] = item['year_on_year'] + doc_dict['year_on_year'] = round(item['year_on_year'], 3) if pd.notna(item['half_on_year']): - doc_dict['half_on_year'] = item['half_on_year'] + doc_dict['half_on_year'] = round(item['half_on_year'], 3) if pd.notna(item['rent_price']): - doc_dict['rent_price'] = item['rent_price'] + doc_dict['rent_price'] = int(item['rent_price']) return doc_dict def sync(city=None): + log.info(f"开始同步价格信息至206 {index_name}") seaweed_query = {'cat': 'district', 'status': 0} if pd.notna(city): seaweed_query['city'] = city @@ -81,6 +82,7 @@ def sync(city=None): seaweed_df['_id'] = seaweed_df['_id'].astype(str) actions = list() + count = 0 for index, item in seaweed_df.iterrows(): doc_info = build_doc(item) @@ -92,9 +94,12 @@ def sync(city=None): } actions.append(action) - if len(actions) == 1000: + if len(actions) == 5000: helpers.bulk(es206_client.elastic_client, actions) + count += len(actions) + print("更新:" + str(count)) actions.clear() if len(actions) > 0: helpers.bulk(es206_client.elastic_client, actions) actions.clear() + log.info(f"完成同步价格信息至206 {index_name}")