diff --git a/dags/business/business/community/total_price.py b/dags/business/business/community/total_price.py index e2fd513..a4a98ee 100644 --- a/dags/business/business/community/total_price.py +++ b/dags/business/business/community/total_price.py @@ -15,6 +15,7 @@ es206_client = ElasticSearchClient(host='192.168.0.206') index_name = "community_summary" path = "/home/airflow/data/community" m_date = dateutils.iso_format() +avg_price_map = dict() def build_doc(item): @@ -24,13 +25,43 @@ def build_doc(item): return doc_dict +def avg_price_map_deal(x): + avg_price_map[x['city'] + x['region'] + x['name']] = x['avg_price'] + return x + + +def deal(seaweed_df, index, item): + if index % 1000 == 0: + log.info("已处理" + str(index)) + curr_city = item['city'] + curr_region = item['region'] + curr_name = item['name'] + names = [curr_name] + + if pd.notna(item['alias']) and '[]' != item['alias']: + alias_list = re.sub("\[|\]|'", '', item['alias']).split(',') + if len(alias_list) > 0: + for alias_item in alias_list: + names.append(alias_item) + + # 户均总价 + if (curr_city + curr_region + curr_name) in avg_price_map: + avg_price = avg_price_map[curr_city + curr_region + curr_name] + if pd.notna(avg_price): + # 由于覆盖率较低,暂不过滤 + # min_avg_price = 0.8 * avg_price + # max_avg_rice = 1.2 * avg_price + total_price = commutils.get_total_price(curr_city, curr_region, names, day_before=366) + seaweed_df.loc[index, 'total_price'] = total_price + + def sync(city=None): log.info(f"开始同步户均总价信息至206 {index_name}") - seaweed_query = {'cat': 'district', 'status': 0} - if pd.notna(city): - seaweed_query['city'] = city seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8') + if pd.notna(city): + seaweed_df = seaweed_df[seaweed_df['city'] == city].copy() + seaweed_df = seaweed_df.reset_index() # 加载库中均价数据 query_sql = "SELECT city, region, name, avg_price FROM community_summary " @@ -38,41 +69,11 @@ def sync(city=None): query_sql += " WHERE city='" + city + "' " community_summary_df = es206_client.scan_by_sql_to_data_frame(query_sql) - avg_price_map = dict() - - def avg_price_map_deal(x): - avg_price_map[x['city'] + x['region'] + x['name']] = x['avg_price'] - return x - community_summary_df.apply(lambda x: avg_price_map_deal(x), axis=1) - with ThreadPoolExecutor(50) as pool: - def deal(index, item): - if index % 1000 == 0: - log.info("已处理" + str(index)) - curr_city = item['city'] - curr_region = item['region'] - curr_name = item['name'] - names = [curr_name] - - if pd.notna(item['alias']) and '[]' != item['alias']: - alias_list = re.sub("\[|\]|'", '', item['alias']).split(',') - if len(alias_list) > 0: - for alias_item in alias_list: - names.append(alias_item) - - # 户均总价 - if (curr_city + curr_region + curr_name) in avg_price_map: - avg_price = avg_price_map[curr_city + curr_region + curr_name] - if pd.notna(avg_price): - # 由于覆盖率较低,暂不过滤 - # min_avg_price = 0.8 * avg_price - # max_avg_rice = 1.2 * avg_price - total_price = commutils.get_total_price(curr_city, curr_region, names, day_before=366) - seaweed_df.loc[index, 'total_price'] = total_price - + with ThreadPoolExecutor(30) as pool: for index, item in seaweed_df.iterrows(): - pool.submit(deal, index, item) + pool.submit(deal, seaweed_df, index, item) actions = list() count = 0 @@ -91,15 +92,15 @@ def sync(city=None): try: helpers.bulk(es206_client.elastic_client, actions) count += len(actions) - print("已更新:" + str(count)) + logging.info("已更新:" + str(count)) except Exception as e: - print(e) + log.error(e) actions.clear() if len(actions) > 0: try: helpers.bulk(es206_client.elastic_client, actions) except Exception as e: - print(e) + log.error(e) actions.clear() log.info(f"完成同步户均总价信息至206 {index_name}")