From 3d69b5d3a880b0788eb216e3b393a22477d81136 Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Mon, 17 Oct 2022 15:39:12 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96=E5=AD=97=E6=AE=B5=E5=90=8C?= =?UTF-8?q?=E6=AD=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/business/business/community/comm.py | 10 +++++-- .../business/community/community_summary.py | 2 +- dags/business/business/community/price.py | 26 ++++++++++++------- 3 files changed, 25 insertions(+), 13 deletions(-) diff --git a/dags/business/business/community/comm.py b/dags/business/business/community/comm.py index 8fb2019..b892e43 100644 --- a/dags/business/business/community/comm.py +++ b/dags/business/business/community/comm.py @@ -77,6 +77,7 @@ def sync(): actions = list() # 对于存在的更新 + count = 0 for index, item in seaweed_existed_df.iterrows(): doc_info = build_doc(item) @@ -88,11 +89,14 @@ def sync(): } actions.append(action) - if len(actions) == 1000: + if len(actions) == 5000: helpers.bulk(es206_client.elastic_client, actions) + count += len(actions) + print("更新:" + str(count)) actions.clear() # 对于不存在的插入 + count = 0 for index, item in seaweed_not_existed_df.iterrows(): doc_info = build_doc(item) @@ -104,8 +108,10 @@ def sync(): } actions.append(action) - if len(actions) == 1000: + if len(actions) == 5000: helpers.bulk(es206_client.elastic_client, actions) + count += len(actions) + print("插入:" + str(count)) actions.clear() if len(actions) > 0: helpers.bulk(es206_client.elastic_client, actions) diff --git a/dags/business/business/community/community_summary.py b/dags/business/business/community/community_summary.py index 8914b48..2da6762 100644 --- a/dags/business/business/community/community_summary.py +++ b/dags/business/business/community/community_summary.py @@ -11,5 +11,5 @@ log.info("同步类型:" + type) if type == 'all': comm.sync() - price.sync() + #price.sync() pass \ No newline at end of file diff --git a/dags/business/business/community/price.py b/dags/business/business/community/price.py index 13ef831..d7ecb74 100644 --- a/dags/business/business/community/price.py +++ b/dags/business/business/community/price.py @@ -31,19 +31,25 @@ def build_doc(item): return doc_dict -def sync(): +def sync(city=None): + seaweed_query = {'cat': 'district', 'status': 0} + if pd.notna(city): + seaweed_query['city'] = city + cities = [city] + else: + cities = None seaweed_df = pd.DataFrame( - list(seaweed_conn.find({'city': '上海', 'cat': 'district', 'status': 0}, {'city': 1, 'region': 1, 'name': 1}) + list(seaweed_conn.find(seaweed_query, {'city': 1, 'region': 1, 'name': 1}) .batch_size(100000))) # 小区所在区域均价 - region_price_df = commutils.get_region_price() + region_price_df = commutils.get_region_price(city=city) region_price_df = region_price_df[['city', 'region', 'avg_price']] region_price_df = region_price_df.rename(columns={'avg_price': 'region_avg_price'}) seaweed_df = pd.merge(seaweed_df, region_price_df, how='left', on=['city', 'region']) # 小区均价 - avg_price_df = commutils.get_avg_price() + avg_price_df = commutils.get_avg_price(city=city) avg_price_df = avg_price_df[['city', 'district_name', 'region', 'avg_price']] avg_price_df = avg_price_df.rename(columns={'district_name': 'name'}) seaweed_df = pd.merge(seaweed_df, avg_price_df, how='left', on=['city', 'region', 'name']) @@ -54,24 +60,26 @@ def sync(): seaweed_df['avg_price_ranking'] = seaweed_df.groupby(by=['city']).rank(method='first', na_option='bottom', ascending=False)['avg_price'] # 12个月均价涨跌幅 - avg_price_up_and_downs_df = commutils.get_avg_price_up_and_downs(months=-12) + avg_price_up_and_downs_df = commutils.get_avg_price_up_and_downs(cities=cities, months=-12) avg_price_up_and_downs_df = avg_price_up_and_downs_df[['city', 'region', 'district_name', 'up_and_downs']] avg_price_up_and_downs_df = avg_price_up_and_downs_df.rename(columns={'district_name': 'name', 'up_and_downs': 'year_on_year'}) seaweed_df = pd.merge(seaweed_df, avg_price_up_and_downs_df, how='left', on=['city', 'region', 'name']) # 近半年均价涨跌幅 - avg_price_up_and_downs_df = commutils.get_avg_price_up_and_downs(months=-6) + avg_price_up_and_downs_df = commutils.get_avg_price_up_and_downs(cities=cities, months=-6) avg_price_up_and_downs_df = avg_price_up_and_downs_df[['city', 'region', 'district_name', 'up_and_downs']] avg_price_up_and_downs_df = avg_price_up_and_downs_df.rename( columns={'district_name': 'name', 'up_and_downs': 'half_on_year'}) seaweed_df = pd.merge(seaweed_df, avg_price_up_and_downs_df, how='left', on=['city', 'region', 'name']) # 平米租金 - rent_price_df = commutils.get_rent_price() + rent_price_df = commutils.get_rent_price(city=city) rent_price_df = rent_price_df[['city', 'region', 'district_name', 'avg_price']] rent_price_df = rent_price_df.rename(columns={'district_name': 'name', 'avg_price': 'rent_price'}) seaweed_df = pd.merge(seaweed_df, rent_price_df, how='left', on=['city', 'region', 'name']) + seaweed_df['_id'] = seaweed_df['_id'].astype(str) + actions = list() for index, item in seaweed_df.iterrows(): doc_info = build_doc(item) @@ -79,7 +87,7 @@ def sync(): action = { "_op_type": "update", "_index": index_name, - "_id": item['id'], + "_id": item['_id'], "doc": doc_info } @@ -90,5 +98,3 @@ def sync(): if len(actions) > 0: helpers.bulk(es206_client.elastic_client, actions) actions.clear() - -sync() \ No newline at end of file