diff --git a/dags/business/business/community/community_summary.py b/dags/business/business/community/community_summary.py index 2da6762..74dba73 100644 --- a/dags/business/business/community/community_summary.py +++ b/dags/business/business/community/community_summary.py @@ -11,5 +11,4 @@ log.info("同步类型:" + type) if type == 'all': comm.sync() - #price.sync() - pass \ No newline at end of file + price.sync() \ No newline at end of file diff --git a/dags/business/business/community/price.py b/dags/business/business/community/price.py index d7ecb74..8cbd276 100644 --- a/dags/business/business/community/price.py +++ b/dags/business/business/community/price.py @@ -19,19 +19,20 @@ def build_doc(item): if pd.notna(item['region_avg_price']): doc_dict['region_avg_price'] = item['region_avg_price'] if pd.notna(item['avg_price']): - doc_dict['avg_price'] = item['avg_price'] - doc_dict['avg_price_region_ranking'] = item['avg_price_region_ranking'] - doc_dict['avg_price_ranking'] = item['avg_price_ranking'] + doc_dict['avg_price'] = int(item['avg_price']) + doc_dict['avg_price_region_ranking'] = int(item['avg_price_region_ranking']) + doc_dict['avg_price_ranking'] = int(item['avg_price_ranking']) if pd.notna(item['year_on_year']): - doc_dict['year_on_year'] = item['year_on_year'] + doc_dict['year_on_year'] = round(item['year_on_year'], 3) if pd.notna(item['half_on_year']): - doc_dict['half_on_year'] = item['half_on_year'] + doc_dict['half_on_year'] = round(item['half_on_year'], 3) if pd.notna(item['rent_price']): - doc_dict['rent_price'] = item['rent_price'] + doc_dict['rent_price'] = int(item['rent_price']) return doc_dict def sync(city=None): + log.info(f"开始同步价格信息至206 {index_name}") seaweed_query = {'cat': 'district', 'status': 0} if pd.notna(city): seaweed_query['city'] = city @@ -81,6 +82,7 @@ def sync(city=None): seaweed_df['_id'] = seaweed_df['_id'].astype(str) actions = list() + count = 0 for index, item in seaweed_df.iterrows(): doc_info = build_doc(item) @@ -92,9 +94,12 @@ def sync(city=None): } actions.append(action) - if len(actions) == 1000: + if len(actions) == 5000: helpers.bulk(es206_client.elastic_client, actions) + count += len(actions) + print("更新:" + str(count)) actions.clear() if len(actions) > 0: helpers.bulk(es206_client.elastic_client, actions) actions.clear() + log.info(f"完成同步价格信息至206 {index_name}")