diff --git a/dags/business/business/community/community_summary.py b/dags/business/business/community/community_summary.py index de7e283..318837e 100644 --- a/dags/business/business/community/community_summary.py +++ b/dags/business/business/community/community_summary.py @@ -5,6 +5,7 @@ import price import sell import trade import avg_cost +import total_price log = logging.getLogger(__name__) @@ -15,6 +16,7 @@ log.info("同步类型:" + type) if type == 'all': #comm.sync() #price.sync() - sell.sync() + #sell.sync() #avg_cost.sync() - #trade.sync() \ No newline at end of file + #trade.sync() + total_price.sync() \ No newline at end of file diff --git a/dags/business/business/community/total_price.py b/dags/business/business/community/total_price.py index bb744db..ce934d9 100644 --- a/dags/business/business/community/total_price.py +++ b/dags/business/business/community/total_price.py @@ -1,11 +1,12 @@ """ -同步挂牌信息、户均总价至206 +同步户均总价至206 """ import logging import pandas as pd from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, areautils from elasticsearch import helpers from concurrent.futures import ThreadPoolExecutor +import datetime log = logging.getLogger(__name__) seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') @@ -17,15 +18,12 @@ m_date = dateutils.iso_format() def build_doc(item): doc_dict = dict() - if 'three_months_sell_num' in item and pd.notna(item['three_months_sell_num']): - doc_dict['three_months_sell_num'] = int(item['three_months_sell_num']) if 'total_price' in item and pd.notna(item['total_price']): doc_dict['total_price'] = int(item['total_price']) return doc_dict - def sync(city=None): - log.info(f"开始同步出售信息至206 {index_name}") + log.info(f"开始同步户均总价信息至206 {index_name}") seaweed_query = {'cat': 'district', 'status': 0} if pd.notna(city): @@ -34,7 +32,18 @@ def sync(city=None): list(seaweed_conn.find(seaweed_query, {'city': 1, 'region': 1, 'name': 1, 'alias': 1}).batch_size(100000))) # 加载库中均价数据 - community_summary_df = es206_client.scan_by_sql_to_data_frame("SELECT city, region, name, avg_price FROM community_summary") + query_sql = "SELECT city, region, name, avg_price FROM community_summary " + if pd.notna(city): + query_sql += " WHERE city='" + city + "' " + community_summary_df = es206_client.scan_by_sql_to_data_frame(query_sql) + + avg_price_map = dict() + + def avg_price_map_deal(x): + avg_price_map[x['city'] + x['region'] + x['name']] = x['avg_price'] + return x + + community_summary_df.apply(lambda x: avg_price_map_deal(x), axis=1) with ThreadPoolExecutor(50) as pool: def deal(index, item): @@ -47,20 +56,15 @@ def sync(city=None): if type(item['alias']) == list: for alias_item in item['alias']: names.append(alias_item) - names_str = "','".join(names) - names_str = "'" + names_str + "'" - - # 近三个月挂牌量 - sample_num = commutils.get_sample(curr_city, curr_region, names_str, cat='sell', daysBefore=93) - seaweed_df.loc[index, 'three_months_sell_num'] = sample_num # 户均总价 - avg_price = community_summary_df.loc[(community_summary_df['city'] == curr_city) & (community_summary_df['region'] == curr_region) & (community_summary_df['name'] == curr_name)].reset_index().loc[0, 'avg_price'] - if pd.notna(avg_price): - min_avg_price = 0.8 * avg_price - max_avg_rice = 1.2 * avg_price - total_price = commutils.get_total_price(curr_city, curr_region, names, day_before=93, min_avg_price=min_avg_price, max_avg_price=max_avg_rice) - seaweed_df.loc[index, 'total_price'] = total_price + if (curr_city + curr_region + curr_name) in avg_price_map: + avg_price = avg_price_map[curr_city + curr_region + curr_name] + if pd.notna(avg_price): + min_avg_price = 0.8 * avg_price + max_avg_rice = 1.2 * avg_price + total_price = commutils.get_total_price(curr_city, curr_region, names, day_before=93, min_avg_price=min_avg_price, max_avg_price=max_avg_rice) + seaweed_df.loc[index, 'total_price'] = total_price for index, item in seaweed_df.iterrows(): pool.submit(deal, index, item) @@ -94,4 +98,4 @@ def sync(city=None): except Exception as e: print(e) actions.clear() - log.info(f"完成同步出售信息至206 {index_name}") + log.info(f"完成同步户均总价信息至206 {index_name}")