diff --git a/dags/business/business/community/community_summary.py b/dags/business/business/community/community_summary.py index 196a3a2..a4bc10a 100644 --- a/dags/business/business/community/community_summary.py +++ b/dags/business/business/community/community_summary.py @@ -16,9 +16,9 @@ log.info("同步类型:" + type) if type == 'all': #comm.sync() - #price.sync() + price.sync() #sell.sync() #avg_cost.sync() #trade.sync() #total_price.sync() - traffic.sync() \ No newline at end of file + #traffic.sync() diff --git a/dags/business/business/community/price.py b/dags/business/business/community/price.py index 56c9c08..f272721 100644 --- a/dags/business/business/community/price.py +++ b/dags/business/business/community/price.py @@ -16,18 +16,22 @@ m_date = dateutils.iso_format() def build_doc(item): doc_dict = dict() - if pd.notna(item['region_avg_price']): + if 'region_avg_price' in item and pd.notna(item['region_avg_price']): doc_dict['region_avg_price'] = item['region_avg_price'] - if pd.notna(item['avg_price']): + if 'avg_price' in item and pd.notna(item['avg_price']): doc_dict['avg_price'] = int(item['avg_price']) - doc_dict['avg_price_region_ranking'] = int(item['avg_price_region_ranking']) - doc_dict['avg_price_ranking'] = int(item['avg_price_ranking']) - if pd.notna(item['year_on_year']): + if 'avg_price_region_ranking' in item and pd.notna(item['avg_price_region_ranking']): + doc_dict['avg_price_region_ranking'] = int(item['avg_price_region_ranking']) + if 'avg_price_ranking' in item and pd.notna(item['avg_price_ranking']): + doc_dict['avg_price_ranking'] = int(item['avg_price_ranking']) + if 'year_on_year' in item and pd.notna(item['year_on_year']): doc_dict['year_on_year'] = round(item['year_on_year'], 3) - if pd.notna(item['half_on_year']): + if 'half_on_year' in item and pd.notna(item['half_on_year']): doc_dict['half_on_year'] = round(item['half_on_year'], 3) - if pd.notna(item['rent_price']): + if 'rent_price' in item and pd.notna(item['rent_price']): doc_dict['rent_price'] = int(item['rent_price']) + if 'guide_price' in item and pd.notna(item['guide_price']): + doc_dict['guide_price'] = int(item['guide_price']) return doc_dict @@ -79,6 +83,12 @@ def sync(city=None): rent_price_df = rent_price_df.rename(columns={'district_name': 'name', 'avg_price': 'rent_price'}) seaweed_df = pd.merge(seaweed_df, rent_price_df, how='left', on=['city', 'region', 'name']) + # 政府指导价 + gov_price_df = commutils.get_gov_guide_price() + gov_price_df = gov_price_df[['city', 'region', 'name', 'avg_price']] + gov_price_df = gov_price_df.rename(columns={'avg_price': 'guide_price'}) + seaweed_df = pd.merge(seaweed_df, gov_price_df, how='left', on=['city', 'region', 'name']) + seaweed_df['_id'] = seaweed_df['_id'].astype(str) actions = list() diff --git a/dags/business/business/community/traffic.py b/dags/business/business/community/traffic.py index 2eb379b..761569c 100644 --- a/dags/business/business/community/traffic.py +++ b/dags/business/business/community/traffic.py @@ -56,9 +56,9 @@ def deal(actions, item): "doc": doc_info } - mutex.acquire(timeout=10) + mutex.acquire(timeout=60) actions.append(action) - if len(actions) == 1000: + if len(actions) == 5000: try: helpers.bulk(es206_client.elastic_client, actions) count += len(actions)