diff --git a/dags/business/business/community/relative_price_ranking.py b/dags/business/business/community/relative_price_ranking.py index caf16ea..5cdf898 100644 --- a/dags/business/business/community/relative_price_ranking.py +++ b/dags/business/business/community/relative_price_ranking.py @@ -18,6 +18,7 @@ path = "/home/airflow/data/community" m_date = dateutils.iso_format() mutex = Lock() + def find_around_seaweed(city_now, lng, lat): """ 查找周边均价 @@ -30,11 +31,11 @@ def find_around_seaweed(city_now, lng, lat): "bool": { "filter": [ { - "term": { - "city": { - "value": city_now - } - } + "term": { + "city": { + "value": city_now + } + } }, { "range": { @@ -63,6 +64,31 @@ def build_doc(item): return doc_dict +find_count = 0 + + +def find_around_price_item(seaweed): + global find_count + find_count += 1 + if find_count % 1000 == 0: + log.info("已查找" + str(find_count)) + if 'lng' not in seaweed: + return + lng = commutils.format_lat_lng(seaweed['lng']) + lat = commutils.format_lat_lng(seaweed['lat']) + if pd.notna(lng): + city_now = seaweed['city'] + around_seaweed_df = find_around_seaweed(city_now, lng, lat) + mutex.acquire(timeout=60) + if len(around_seaweed_df) > 1: + around_seaweed_df['relative_price_ranking'] = around_seaweed_df['avg_price'].rank(pct=True) + seaweed['relative_price_ranking'] = around_seaweed_df[around_seaweed_df['id'] == seaweed['_id']].iloc[0][ + 'relative_price_ranking'] + elif len(around_seaweed_df) == 1: + seaweed['relative_price_ranking'] = 0.5 + mutex.release() + + def sync(city=None): log.info("开始同步相对价格排名数据...") @@ -71,30 +97,17 @@ def sync(city=None): seaweed_df = seaweed_df[seaweed_df['city'] == city].copy() seaweed_df = seaweed_df.reset_index(drop=True) - def find_around_price_item(index, item): - if index % 1000 == 0: - log.info("已处理" + str(index)) - if 'lng' not in item: - return - lng = commutils.format_lat_lng(item['lng']) - lat = commutils.format_lat_lng(item['lat']) - if pd.notna(lng): - city_now = item['city'] - around_seaweed_df = find_around_seaweed(city_now, lng, lat) - mutex.acquire(timeout=60) - if len(around_seaweed_df) > 1: - around_seaweed_df['relative_price_ranking'] = around_seaweed_df['avg_price'].rank(pct=True) - seaweed_df.loc[index, 'relative_price_ranking'] = around_seaweed_df.loc[around_seaweed_df['id'] == item['_id']]['relative_price_ranking'].tolist()[0] - elif len(around_seaweed_df) == 1: - seaweed_df.loc[index, 'relative_price_ranking'] = 0.5 - mutex.release() + seaweed_list = list() + seaweed_df.apply(lambda x: seaweed_list.append(dict(x)), axis=1) with ThreadPoolExecutor(max_workers=30) as pool: - for index, item in seaweed_df.iterrows(): - pool.submit(find_around_price_item, index, item) + for seaweed in seaweed_list: + pool.submit(find_around_price_item, seaweed) - # for index, item in seaweed_df.iterrows(): - # find_around_price_item(index, item) + # for seaweed in seaweed_list: + # find_around_price_item(seaweed) + + seaweed_df = pd.DataFrame(seaweed_list) actions = list() count = 0 @@ -105,17 +118,15 @@ def sync(city=None): "_op_type": "update", "_index": index_name, "_id": item['_id'], - "doc": doc_info + "doc": doc_info, + "_retry_on_conflict": 5 } actions.append(action) if len(actions) == 5000: - try: - helpers.bulk(es206_client.elastic_client, actions) - count += len(actions) - log.info("更新:" + str(count)) - except Exception as e: - log.error(e) + helpers.bulk(es206_client.elastic_client, actions) + count += len(actions) + log.info("更新:" + str(count)) actions.clear() if len(actions) > 0: try: @@ -128,4 +139,4 @@ def sync(city=None): city = sys.argv[1] if len(sys.argv) > 1 else None -sync(city) \ No newline at end of file +sync(city)