From 0ab94220448e71e65b13b0ef8e0ac21c1ab5a854 Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Thu, 5 Jan 2023 18:25:13 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96=E5=9F=8E=E5=B8=82=E5=90=8C?= =?UTF-8?q?=E6=AD=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/business/business/community/price.py | 56 +++++++++++------------ 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/dags/business/business/community/price.py b/dags/business/business/community/price.py index 2d2112f..a6f1b49 100644 --- a/dags/business/business/community/price.py +++ b/dags/business/business/community/price.py @@ -3,7 +3,7 @@ """ import logging import pandas as pd -from goojia_utils import ElasticSearchClient, commutils, dateutils +from goojia_utils import ElasticSearchClient, commutils, dateutils, areautils from elasticsearch import helpers log = logging.getLogger(__name__) @@ -37,27 +37,19 @@ def build_doc(item): return doc_dict -def sync(city=None): - log.info(f"开始同步价格信息至206 {index_name}") - if pd.notna(city): - cities = [city] - else: - cities = None - seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8') - if pd.notna(city): - seaweed_df = seaweed_df[seaweed_df['city'] == city].copy() - seaweed_df = seaweed_df.reset_index(drop=True) +def sync_by_city(city, total_seaweed_df): + seaweed_df = total_seaweed_df[total_seaweed_df['city'] == city].copy() + if len(seaweed_df) == 0: + return + seaweed_df = seaweed_df.reset_index(drop=True) # 小区所在城市均价 - log.info('开始导出小区所在城市均价') city_price_df = commutils.get_city_price(city=city) city_price_df = city_price_df[['city', 'avg_price']] city_price_df = city_price_df.rename(columns={'avg_price': 'city_avg_price'}) seaweed_df = pd.merge(seaweed_df, city_price_df, how='left', on=['city']) - log.info('小区所在城市均价导出完毕') # 小区所在区域均价 - log.info('开始导出小区所在区域均价') region_price_df = commutils.get_region_price(city=city) region_price_df = region_price_df[['city', 'region', 'avg_price']] region_price_df = region_price_df.rename(columns={'avg_price': 'region_avg_price'}) @@ -65,7 +57,6 @@ def sync(city=None): log.info('小区所在区域均价导出完毕') # 小区均价 - log.info('开始导出小区均价') avg_price_df = commutils.get_avg_price(city=city) avg_price_df = avg_price_df[['city', 'district_name', 'region', 'avg_price']] avg_price_df = avg_price_df.rename(columns={'district_name': 'name'}) @@ -73,42 +64,37 @@ def sync(city=None): log.info('小区均价导出完毕') # 小区均价区域排名 - seaweed_df['avg_price_region_ranking'] = seaweed_df.groupby(by=['city', 'region']).rank(method='first', na_option='bottom', ascending=False)['avg_price'] + seaweed_df['avg_price_region_ranking'] = \ + seaweed_df.groupby(by=['city', 'region']).rank(method='first', na_option='bottom', ascending=False)['avg_price'] # 小区均价城市排名 - seaweed_df['avg_price_ranking'] = seaweed_df.groupby(by=['city']).rank(method='first', na_option='bottom', ascending=False)['avg_price'] + seaweed_df['avg_price_ranking'] = \ + seaweed_df.groupby(by=['city']).rank(method='first', na_option='bottom', ascending=False)['avg_price'] # 12个月均价涨跌幅 - log.info('开始导出12个月均价涨跌幅') - avg_price_up_and_downs_df = commutils.get_avg_price_up_and_downs(cities=cities, months=-12) + avg_price_up_and_downs_df = commutils.get_avg_price_up_and_downs(cities=[city], months=-12) avg_price_up_and_downs_df = avg_price_up_and_downs_df[['city', 'region', 'district_name', 'up_and_downs']] - avg_price_up_and_downs_df = avg_price_up_and_downs_df.rename(columns={'district_name': 'name', 'up_and_downs': 'year_on_year'}) + avg_price_up_and_downs_df = avg_price_up_and_downs_df.rename( + columns={'district_name': 'name', 'up_and_downs': 'year_on_year'}) seaweed_df = pd.merge(seaweed_df, avg_price_up_and_downs_df, how='left', on=['city', 'region', 'name']) - log.info('12个月均价涨跌幅导出完毕') # 近半年均价涨跌幅 - log.info('开始导出近半年均价涨跌幅') - avg_price_up_and_downs_df = commutils.get_avg_price_up_and_downs(cities=cities, months=-6) + avg_price_up_and_downs_df = commutils.get_avg_price_up_and_downs(cities=[city], months=-6) avg_price_up_and_downs_df = avg_price_up_and_downs_df[['city', 'region', 'district_name', 'up_and_downs']] avg_price_up_and_downs_df = avg_price_up_and_downs_df.rename( columns={'district_name': 'name', 'up_and_downs': 'half_on_year'}) seaweed_df = pd.merge(seaweed_df, avg_price_up_and_downs_df, how='left', on=['city', 'region', 'name']) - log.info('近半年均价涨跌幅导出完毕') # 平米租金 - log.info('开始导出平米租金') rent_price_df = commutils.get_rent_price(city=city) rent_price_df = rent_price_df[['city', 'region', 'district_name', 'avg_price']] rent_price_df = rent_price_df.rename(columns={'district_name': 'name', 'avg_price': 'rent_price'}) seaweed_df = pd.merge(seaweed_df, rent_price_df, how='left', on=['city', 'region', 'name']) - log.info('平米租金导出完毕') # 政府指导价 - log.info('开始导出政府指导价') gov_price_df = commutils.get_gov_guide_price() gov_price_df = gov_price_df[['city', 'region', 'name', 'avg_price']] gov_price_df = gov_price_df.rename(columns={'avg_price': 'guide_price'}) seaweed_df = pd.merge(seaweed_df, gov_price_df, how='left', on=['city', 'region', 'name']) - log.info('政府指导价导出完毕') actions = list() count = 0 @@ -131,6 +117,20 @@ def sync(city=None): if len(actions) > 0: helpers.bulk(es206_client.elastic_client, actions) actions.clear() + + +def sync(city=None): + log.info(f"开始同步价格信息至206 {index_name}") + if pd.notna(city): + cities = [city] + else: + cities = areautils.get_all_cities() + seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8') + for city in cities: + log.info(f"开始同步{city}") + sync_by_city(city, seaweed_df) + log.info(f"完成同步{city}") + log.info(f"完成同步价格信息至206 {index_name}")