From f03e3dd81b28590ee8031897e4d3d55ce15663b1 Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Tue, 25 Oct 2022 09:33:16 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E6=96=87=E4=BB=B6=E8=AF=BB?= =?UTF-8?q?=E5=8F=96bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/business/business/community/avg_cost.py | 5 ++-- dags/business/business/community/comm.py | 6 +++- .../business/community/community_summary.py | 2 ++ .../community/consumption_potential.py | 30 ++++++++++++++----- .../business/community/data/seaweed.csv | 0 dags/business/business/community/life.py | 3 +- dags/business/business/community/medical.py | 3 +- dags/business/business/community/price.py | 3 +- .../community/relative_price_ranking.py | 10 +++++-- dags/business/business/community/sell.py | 3 +- .../business/community/total_price.py | 3 +- dags/business/business/community/trade.py | 3 +- dags/business/business/community/traffic.py | 4 +-- 13 files changed, 53 insertions(+), 22 deletions(-) delete mode 100644 dags/business/business/community/data/seaweed.csv diff --git a/dags/business/business/community/avg_cost.py b/dags/business/business/community/avg_cost.py index 153481d..b9467d9 100644 --- a/dags/business/business/community/avg_cost.py +++ b/dags/business/business/community/avg_cost.py @@ -3,7 +3,7 @@ """ import logging import pandas as pd -from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils +from goojia_utils import ElasticSearchClient, commutils, dateutils from elasticsearch import helpers import numpy as np from concurrent.futures import ThreadPoolExecutor @@ -13,6 +13,7 @@ es206_client = ElasticSearchClient(host='192.168.0.206') index_name = "community_summary" m_date = dateutils.iso_format() +path = "/home/airflow/data/community" def find_around_price(lng, lat): @@ -100,7 +101,7 @@ def sync(): # 按最新的百分位更新库中价格状态 update_status(new_df) - seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@', encoding='UTF-8') + seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8') def find_around_price_item(index, item): if 'lng' not in item: diff --git a/dags/business/business/community/comm.py b/dags/business/business/community/comm.py index 51cf489..c5d61f5 100644 --- a/dags/business/business/community/comm.py +++ b/dags/business/business/community/comm.py @@ -6,6 +6,7 @@ import logging import pandas as pd from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils from elasticsearch import helpers +import os log = logging.getLogger(__name__) seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') @@ -14,6 +15,9 @@ es206_client = ElasticSearchClient(host='192.168.0.206') index_name = "community_summary" m_date = dateutils.iso_format() +path = "/home/airflow/data/community" +if not os.path.exists(path): + os.makedirs(path) def build_doc(item): """ @@ -120,5 +124,5 @@ def sync(city=None): actions.clear() # 写入csv文件,防止小区变动导致的后续过程更新报错 - seaweed_df.to_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@', encoding='UTF-8', index=False) + seaweed_df.to_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8', index=False) log.info(f"完成同步小区基本属性至206 {index_name}") diff --git a/dags/business/business/community/community_summary.py b/dags/business/business/community/community_summary.py index 8d4c643..95b17d8 100644 --- a/dags/business/business/community/community_summary.py +++ b/dags/business/business/community/community_summary.py @@ -10,6 +10,7 @@ import traffic import medical import life import relative_price_ranking +import consumption_potential log = logging.getLogger(__name__) @@ -28,3 +29,4 @@ if type == 'all': medical.sync() life.sync() relative_price_ranking.sync() + consumption_potential.sync() diff --git a/dags/business/business/community/consumption_potential.py b/dags/business/business/community/consumption_potential.py index 968aba6..30f4e8e 100644 --- a/dags/business/business/community/consumption_potential.py +++ b/dags/business/business/community/consumption_potential.py @@ -12,9 +12,9 @@ def calc_consumption_potential(city): """ 计算消费潜力 """ - sql = "select id, city, region, name, total_price, estate_charge, avg_price_region_ranking, avg_cost from community_summary" + sql = "select id, city, region, name, total_price, estate_charge, relative_price_ranking, avg_cost from community_summary" if pd.notna(city): - sql += " where city=" + city + sql += " where city='" + city + "'" community_summary_df = es206_client.scan_by_sql_to_data_frame(sql) community_summary_df['total_price_mean'] = community_summary_df.groupby(by=['city'])['total_price'].transform(np.mean) community_summary_df['total_price_std'] = community_summary_df.groupby(by=['city'])['total_price'].transform(np.std) @@ -22,16 +22,26 @@ def calc_consumption_potential(city): community_summary_df['estate_charge_mean'] = community_summary_df.groupby(by=['city'])['estate_charge'].transform(np.mean) community_summary_df['estate_charge_std'] = community_summary_df.groupby(by=['city'])['estate_charge'].transform(np.std) - community_summary_df['avg_price_region_ranking_mean'] = community_summary_df.groupby(by=['city'])['avg_price_region_ranking'].transform(np.mean) - community_summary_df['avg_price_region_ranking_std'] = community_summary_df.groupby(by=['city'])['avg_price_region_ranking'].transform(np.std) + community_summary_df['relative_price_ranking_mean'] = community_summary_df.groupby(by=['city'])['relative_price_ranking'].transform(np.mean) + community_summary_df['relative_price_ranking_std'] = community_summary_df.groupby(by=['city'])['relative_price_ranking'].transform(np.std) community_summary_df['avg_cost_mean'] = community_summary_df.groupby(by=['city'])['avg_cost'].transform(np.mean) community_summary_df['avg_cost_std'] = community_summary_df.groupby(by=['city'])['avg_cost'].transform(np.std) - community_summary_df['consumption_potential'] = 286.0134196 * (community_summary_df['total_price'] - community_summary_df['total_price_avg']) / community_summary_df['total_price_std'] \ - + 139.1374524 * (community_summary_df['estate_charge'] - community_summary_df['estate_charge_avg']) / community_summary_df['estate_charge_std'] \ - + 21.8427926 * (community_summary_df['avg_price_region_ranking'] - community_summary_df['avg_price_region_ranking_avg']) / community_summary_df['avg_price_region_ranking_std'] \ - + 67.0195871700 * (community_summary_df['avg_cost'] - community_summary_df['avg_cost_mean']) / community_summary_df['avg_cost_std'] + 4099.978248 + community_summary_df['total_price_score'] = 286.0134196 * (community_summary_df['total_price'] - community_summary_df['total_price_mean']) / community_summary_df['total_price_std'] + community_summary_df['total_price_score'].fillna(0, inplace=True) + + community_summary_df['estate_charge_score'] = 139.1374524 * (community_summary_df['estate_charge'] - community_summary_df['estate_charge_mean']) / community_summary_df['estate_charge_std'] + community_summary_df['estate_charge_score'].fillna(0, inplace=True) + + community_summary_df['relative_price_ranking_score'] = 21.8427926 * (community_summary_df['relative_price_ranking'] - community_summary_df['relative_price_ranking_mean']) / community_summary_df['relative_price_ranking_std'] + community_summary_df['relative_price_ranking_score'].fillna(0, inplace=True) + + community_summary_df['avg_cost_score'] = 67.0195871700 * (community_summary_df['avg_cost'] - community_summary_df['avg_cost_mean']) / community_summary_df['avg_cost_std'] + community_summary_df['avg_cost_score'].fillna(0, inplace=True) + + community_summary_df['consumption_potential'] = community_summary_df['total_price_score'] + community_summary_df['estate_charge_score'] + community_summary_df['relative_price_ranking_score'] \ + + community_summary_df['avg_cost_score'] + community_summary_df['avg_cost_score'] + 4099.978248 pass @@ -42,3 +52,7 @@ def calc_consumption_rating(): """ pass + +def sync(city=None): + calc_consumption_potential(city) + pass \ No newline at end of file diff --git a/dags/business/business/community/data/seaweed.csv b/dags/business/business/community/data/seaweed.csv deleted file mode 100644 index e69de29..0000000 diff --git a/dags/business/business/community/life.py b/dags/business/business/community/life.py index b09f6aa..12ebde9 100644 --- a/dags/business/business/community/life.py +++ b/dags/business/business/community/life.py @@ -12,6 +12,7 @@ log = logging.getLogger(__name__) es206_client = ElasticSearchClient(host='192.168.0.206') index_name = "community_summary" +path = "/home/airflow/data/community" m_date = dateutils.iso_format() count = 0 @@ -79,7 +80,7 @@ def sync(city=None): if pd.notna(city): seaweed_query['city'] = city - seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@', encoding='UTF-8') + seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8') actions = list() with ThreadPoolExecutor(max_workers=30) as pool: diff --git a/dags/business/business/community/medical.py b/dags/business/business/community/medical.py index cd07ddb..bcb28a0 100644 --- a/dags/business/business/community/medical.py +++ b/dags/business/business/community/medical.py @@ -13,6 +13,7 @@ seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') es206_client = ElasticSearchClient(host='192.168.0.206') index_name = "community_summary" +path = "/home/airflow/data/community" m_date = dateutils.iso_format() count = 0 @@ -67,7 +68,7 @@ def sync(city=None): if pd.notna(city): seaweed_query['city'] = city - seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@', encoding='UTF-8') + seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8') actions = list() with ThreadPoolExecutor(max_workers=30) as pool: diff --git a/dags/business/business/community/price.py b/dags/business/business/community/price.py index 170dae9..f5d306a 100644 --- a/dags/business/business/community/price.py +++ b/dags/business/business/community/price.py @@ -10,6 +10,7 @@ log = logging.getLogger(__name__) es206_client = ElasticSearchClient(host='192.168.0.206') index_name = "community_summary" +path = "/home/airflow/data/community" m_date = dateutils.iso_format() @@ -42,7 +43,7 @@ def sync(city=None): cities = [city] else: cities = None - seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@', encoding='UTF-8') + seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8') # 小区所在区域均价 region_price_df = commutils.get_region_price(city=city) diff --git a/dags/business/business/community/relative_price_ranking.py b/dags/business/business/community/relative_price_ranking.py index 1e70d61..e87605b 100644 --- a/dags/business/business/community/relative_price_ranking.py +++ b/dags/business/business/community/relative_price_ranking.py @@ -13,6 +13,7 @@ log = logging.getLogger(__name__) es206_client = ElasticSearchClient(host='192.168.0.206') index_name = "community_summary" +path = "/home/airflow/data/community" m_date = dateutils.iso_format() @@ -57,7 +58,7 @@ def build_doc(item): def sync(): log.info("开始同步相对价格排名数据...") - seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@', encoding='UTF-8') + seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8') def find_around_price_item(index, item): if 'lng' not in item: @@ -66,8 +67,11 @@ def sync(): lat = commutils.format_lat_lng(item['lat']) if pd.notna(lng): around_seaweed_df = find_around_seaweed(lng, lat) - around_seaweed_df['relative_price_ranking'] = around_seaweed_df['avg_price'].rank(pct=True) - seaweed_df.loc[index, 'relative_price_ranking'] = around_seaweed_df.loc[around_seaweed_df['id'] == item['_id']]['relative_price_ranking'].tolist()[0] + if len(around_seaweed_df) > 1: + around_seaweed_df['relative_price_ranking'] = around_seaweed_df['avg_price'].rank(pct=True) + seaweed_df.loc[index, 'relative_price_ranking'] = around_seaweed_df.loc[around_seaweed_df['id'] == item['_id']]['relative_price_ranking'].tolist()[0] + elif len(around_seaweed_df) == 1: + seaweed_df.loc[index, 'relative_price_ranking'] = 0.5 with ThreadPoolExecutor(max_workers=50) as pool: for index, item in seaweed_df.iterrows(): diff --git a/dags/business/business/community/sell.py b/dags/business/business/community/sell.py index 151cbd0..db92878 100644 --- a/dags/business/business/community/sell.py +++ b/dags/business/business/community/sell.py @@ -12,6 +12,7 @@ seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') es206_client = ElasticSearchClient(host='192.168.0.206') index_name = "community_summary" +path = "/home/airflow/data/community" m_date = dateutils.iso_format() @@ -28,7 +29,7 @@ def sync(city=None): seaweed_query = {'cat': 'district', 'status': 0} if pd.notna(city): seaweed_query['city'] = city - seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@', encoding='UTF-8') + seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8') with ThreadPoolExecutor(50) as pool: def deal(index, item): diff --git a/dags/business/business/community/total_price.py b/dags/business/business/community/total_price.py index 280bc58..2b74fdc 100644 --- a/dags/business/business/community/total_price.py +++ b/dags/business/business/community/total_price.py @@ -13,6 +13,7 @@ seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') es206_client = ElasticSearchClient(host='192.168.0.206') index_name = "community_summary" +path = "/home/airflow/data/community" m_date = dateutils.iso_format() @@ -28,7 +29,7 @@ def sync(city=None): seaweed_query = {'cat': 'district', 'status': 0} if pd.notna(city): seaweed_query['city'] = city - seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@', encoding='UTF-8') + seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8') # 加载库中均价数据 query_sql = "SELECT city, region, name, avg_price FROM community_summary " diff --git a/dags/business/business/community/trade.py b/dags/business/business/community/trade.py index bc033d1..85b9582 100644 --- a/dags/business/business/community/trade.py +++ b/dags/business/business/community/trade.py @@ -10,6 +10,7 @@ log = logging.getLogger(__name__) es206_client = ElasticSearchClient(host='192.168.0.206') index_name = "community_summary" +path = "/home/airflow/data/community" m_date = dateutils.iso_format() @@ -28,7 +29,7 @@ def sync(city=None): if pd.notna(city): seaweed_query['city'] = city cities = [city] - seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@', encoding='UTF-8') + seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8') deal_price_df = commutils.get_deal_avg_price(cities, 3) deal_price_df = deal_price_df[['match_city', 'match_region', 'match_name', 'avg_price']] diff --git a/dags/business/business/community/traffic.py b/dags/business/business/community/traffic.py index 728bbaa..9cba530 100644 --- a/dags/business/business/community/traffic.py +++ b/dags/business/business/community/traffic.py @@ -12,6 +12,7 @@ log = logging.getLogger(__name__) es206_client = ElasticSearchClient(host='192.168.0.206') index_name = "community_summary" +path = "/home/airflow/data/community" m_date = dateutils.iso_format() count = 0 @@ -76,8 +77,7 @@ def sync(city=None): if pd.notna(city): seaweed_query['city'] = city - seaweed_df = seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@', - encoding='UTF-8') + seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8') actions = list() with ThreadPoolExecutor(max_workers=30) as pool: