From 29f827e2cc4d1810d6aba0ea0848eff791125b0c Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Tue, 25 Apr 2023 13:24:21 +0800 Subject: [PATCH] =?UTF-8?q?=E8=A7=A3=E5=86=B3airflow=E8=8B=8F=E5=B7=9E?= =?UTF-8?q?=E5=9F=8E=E5=B8=82price=E5=90=8C=E6=AD=A5bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/business/business/community/price.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/dags/business/business/community/price.py b/dags/business/business/community/price.py index af425b5..11b0b64 100644 --- a/dags/business/business/community/price.py +++ b/dags/business/business/community/price.py @@ -79,16 +79,22 @@ def sync_by_city(city): avg_price_up_and_downs_df = avg_price_up_and_downs_df[['city', 'region', 'district_name', 'up_and_downs']] avg_price_up_and_downs_df = avg_price_up_and_downs_df.rename( columns={'district_name': 'name', 'up_and_downs': 'year_on_year'}) + avg_price_up_and_downs_df = avg_price_up_and_downs_df.drop_duplicates(subset=['city', 'region', 'name'], keep='first').reset_index( + drop=True).copy() seaweed_df = pd.merge(seaweed_df, avg_price_up_and_downs_df, how='left', on=['city', 'region', 'name']) + # 近半年均价涨跌幅 avg_price_up_and_downs_df = commutils.get_avg_price_up_and_downs(cities=[city], months=-6) if len(avg_price_up_and_downs_df) > 0: avg_price_up_and_downs_df = avg_price_up_and_downs_df[['city', 'region', 'district_name', 'up_and_downs']] avg_price_up_and_downs_df = avg_price_up_and_downs_df.rename( columns={'district_name': 'name', 'up_and_downs': 'half_on_year'}) + avg_price_up_and_downs_df = avg_price_up_and_downs_df.drop_duplicates(subset=['city', 'region', 'name'], keep='first').reset_index( + drop=True).copy() seaweed_df = pd.merge(seaweed_df, avg_price_up_and_downs_df, how='left', on=['city', 'region', 'name']) + # 平米租金 rent_price_df = commutils.get_rent_price(city=city) if len(rent_price_df) > 0: @@ -102,7 +108,6 @@ def sync_by_city(city): gov_price_df = gov_price_df[['city', 'region', 'name', 'avg_price']] gov_price_df = gov_price_df.rename(columns={'avg_price': 'guide_price'}) seaweed_df = pd.merge(seaweed_df, gov_price_df, how='left', on=['city', 'region', 'name']) - actions = list() count = 0 for index, item in seaweed_df.iterrows(): @@ -114,9 +119,13 @@ def sync_by_city(city): "_id": item['_id'], "doc": doc_info } - + if index < 6500: + continue actions.append(action) - if len(actions) == 5000: + + if len(actions) == 1: + print(index) + print(item) helpers.bulk(es206_client.elastic_client, actions) count += len(actions) log.info("更新:" + str(count))