diff --git a/dags/business/business/community/avg_cost.py b/dags/business/business/community/avg_cost.py index 72561c8..b78c8ed 100644 --- a/dags/business/business/community/avg_cost.py +++ b/dags/business/business/community/avg_cost.py @@ -9,7 +9,6 @@ import numpy as np from concurrent.futures import ThreadPoolExecutor log = logging.getLogger(__name__) -seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') es206_client = ElasticSearchClient(host='192.168.0.206') index_name = "community_summary" @@ -101,8 +100,7 @@ def sync(): # 按最新的百分位更新库中价格状态 update_status(new_df) - seaweed_df = pd.DataFrame( - list(seaweed_conn.find({'cat': 'district', 'status': 0}, {'city': 1, 'lng': 1, 'lat': 1}).batch_size(100000))) + seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/community/data/seaweed.csv', sep='@', encoding='UTF-8') def find_around_price_item(index, item): if 'lng' not in item: @@ -116,8 +114,6 @@ def sync(): for index, item in seaweed_df.iterrows(): pool.submit(find_around_price_item, index, item) - seaweed_df['_id'] = seaweed_df['_id'].astype(str) - actions = list() count = 0 for index, item in seaweed_df.iterrows(): diff --git a/dags/business/business/community/comm.py b/dags/business/business/community/comm.py index b892e43..f426694 100644 --- a/dags/business/business/community/comm.py +++ b/dags/business/business/community/comm.py @@ -22,7 +22,7 @@ def build_doc(item): :return: """ doc_dict = dict() - doc_dict['id'] = item["id"] + doc_dict['id'] = item["_id"] doc_dict['m_date'] = m_date doc_dict['city'] = item["city"] doc_dict['region'] = item["region"] @@ -52,23 +52,25 @@ def build_doc(item): return doc_dict -def sync(): +def sync(city=None): log.info(f"开始同步小区基本属性至206 {index_name}...") + seaweed_query = {'status': 0, 'cat': 'district'} + if pd.notna(city): + seaweed_query['city'] = city + seaweed_df = pd.DataFrame( - list(seaweed_conn.find({'cat': 'district', 'status': 0}, {'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1, - 'estate_type2': 1, 'build_type': 1, + list(seaweed_conn.find(seaweed_query, {'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1, + 'alias': 1, 'estate_type2': 1, 'build_type': 1, 'household_count': 1, 'estate_charge': 1, 'finish_building_date': 1}) .batch_size(100000))) community_summary_df = es206_client.scan_by_sql_to_data_frame(f"select id from {index_name}") seaweed_df['_id'] = seaweed_df['_id'].astype(str) - seaweed_df = seaweed_df.rename(columns={"_id": "id"}) if len(community_summary_df) > 0: community_summary_df['flag'] = 1 - - seaweed_new_df = pd.merge(seaweed_df, community_summary_df, how='left', on=['id']) + seaweed_new_df = pd.merge(seaweed_df, community_summary_df, how='left', on=['_id']) seaweed_existed_df = seaweed_new_df[seaweed_new_df['flag'] == 1] seaweed_not_existed_df = seaweed_new_df[seaweed_new_df['flag'] != 1] else: @@ -84,7 +86,7 @@ def sync(): action = { "_op_type": "update", "_index": index_name, - "_id": item['id'], + "_id": item['_id'], "doc": doc_info } @@ -103,7 +105,7 @@ def sync(): action = { "_op_type": "index", "_index": index_name, - "_id": item['id'], + "_id": item['_id'], "_source": doc_info } @@ -116,4 +118,7 @@ def sync(): if len(actions) > 0: helpers.bulk(es206_client.elastic_client, actions) actions.clear() + + # 写入csv文件,防止小区变动导致的后续过程更新报错 + seaweed_df.to_csv('/opt/airflow/dags/repo/dags/business/community/data/seaweed.csv', sep='@', encoding='UTF-8', index=False) log.info(f"完成同步小区基本属性至206 {index_name}") diff --git a/dags/business/business/community/community_summary.py b/dags/business/business/community/community_summary.py index 2aa7130..9d4e2ad 100644 --- a/dags/business/business/community/community_summary.py +++ b/dags/business/business/community/community_summary.py @@ -17,12 +17,12 @@ log.info("同步类型:" + type) if type == 'all': - #comm.sync() + comm.sync('上海') #price.sync() #sell.sync() #avg_cost.sync() #trade.sync() - #total_price.sync() + total_price.sync() #traffic.sync() #medical.sync() life.sync() diff --git a/dags/business/business/community/life.py b/dags/business/business/community/life.py index cbcf886..7a82f88 100644 --- a/dags/business/business/community/life.py +++ b/dags/business/business/community/life.py @@ -9,7 +9,6 @@ from concurrent.futures import ThreadPoolExecutor from threading import Lock log = logging.getLogger(__name__) -seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') es206_client = ElasticSearchClient(host='192.168.0.206') index_name = "community_summary" @@ -80,11 +79,7 @@ def sync(city=None): if pd.notna(city): seaweed_query['city'] = city - seaweed_df = pd.DataFrame( - list(seaweed_conn.find(seaweed_query, {'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1}) - .batch_size(100000))) - - seaweed_df['_id'] = seaweed_df['_id'].astype(str) + seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/community/data/seaweed.csv', sep='@', encoding='UTF-8') actions = list() with ThreadPoolExecutor(max_workers=30) as pool: diff --git a/dags/business/business/community/medical.py b/dags/business/business/community/medical.py index 269dee6..2271174 100644 --- a/dags/business/business/community/medical.py +++ b/dags/business/business/community/medical.py @@ -67,11 +67,7 @@ def sync(city=None): if pd.notna(city): seaweed_query['city'] = city - seaweed_df = pd.DataFrame( - list(seaweed_conn.find(seaweed_query, {'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1}) - .batch_size(100000))) - - seaweed_df['_id'] = seaweed_df['_id'].astype(str) + seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/community/data/seaweed.csv', sep='@', encoding='UTF-8') actions = list() with ThreadPoolExecutor(max_workers=30) as pool: diff --git a/dags/business/business/community/price.py b/dags/business/business/community/price.py index 6f5bfa6..a4befa1 100644 --- a/dags/business/business/community/price.py +++ b/dags/business/business/community/price.py @@ -3,11 +3,10 @@ """ import logging import pandas as pd -from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, areautils +from goojia_utils import ElasticSearchClient, commutils, dateutils from elasticsearch import helpers log = logging.getLogger(__name__) -seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') es206_client = ElasticSearchClient(host='192.168.0.206') index_name = "community_summary" @@ -43,9 +42,7 @@ def sync(city=None): cities = [city] else: cities = None - seaweed_df = pd.DataFrame( - list(seaweed_conn.find(seaweed_query, {'city': 1, 'region': 1, 'name': 1}) - .batch_size(100000))) + seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/community/data/seaweed.csv', sep='@', encoding='UTF-8') # 小区所在区域均价 region_price_df = commutils.get_region_price(city=city) @@ -89,8 +86,6 @@ def sync(city=None): gov_price_df = gov_price_df.rename(columns={'avg_price': 'guide_price'}) seaweed_df = pd.merge(seaweed_df, gov_price_df, how='left', on=['city', 'region', 'name']) - seaweed_df['_id'] = seaweed_df['_id'].astype(str) - actions = list() count = 0 for index, item in seaweed_df.iterrows(): diff --git a/dags/business/business/community/sell.py b/dags/business/business/community/sell.py index 3729f5a..9c660d8 100644 --- a/dags/business/business/community/sell.py +++ b/dags/business/business/community/sell.py @@ -3,7 +3,7 @@ """ import logging import pandas as pd -from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, areautils +from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils from elasticsearch import helpers from concurrent.futures import ThreadPoolExecutor @@ -28,8 +28,7 @@ def sync(city=None): seaweed_query = {'cat': 'district', 'status': 0} if pd.notna(city): seaweed_query['city'] = city - seaweed_df = pd.DataFrame( - list(seaweed_conn.find(seaweed_query, {'city': 1, 'region': 1, 'name': 1, 'alias': 1}).batch_size(100000))) + seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/community/data/seaweed.csv', sep='@', encoding='UTF-8') with ThreadPoolExecutor(50) as pool: def deal(index, item): @@ -52,7 +51,6 @@ def sync(city=None): for index, item in seaweed_df.iterrows(): pool.submit(deal, index, item) - seaweed_df['_id'] = seaweed_df['_id'].astype(str) actions = list() count = 0 diff --git a/dags/business/business/community/total_price.py b/dags/business/business/community/total_price.py index ce934d9..7f734cb 100644 --- a/dags/business/business/community/total_price.py +++ b/dags/business/business/community/total_price.py @@ -6,7 +6,7 @@ import pandas as pd from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, areautils from elasticsearch import helpers from concurrent.futures import ThreadPoolExecutor -import datetime +import re log = logging.getLogger(__name__) seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') @@ -28,8 +28,7 @@ def sync(city=None): seaweed_query = {'cat': 'district', 'status': 0} if pd.notna(city): seaweed_query['city'] = city - seaweed_df = pd.DataFrame( - list(seaweed_conn.find(seaweed_query, {'city': 1, 'region': 1, 'name': 1, 'alias': 1}).batch_size(100000))) + seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/community/data/seaweed.csv', sep='@', encoding='UTF-8') # 加载库中均价数据 query_sql = "SELECT city, region, name, avg_price FROM community_summary " @@ -53,9 +52,12 @@ def sync(city=None): curr_region = item['region'] curr_name = item['name'] names = [curr_name] - if type(item['alias']) == list: - for alias_item in item['alias']: - names.append(alias_item) + + if pd.notna(item['alias']) and '[]' != item['alias']: + alias_list = re.sub("\[|\]|'", '', item['alias']).split(',') + if len(alias_list) > 0: + for alias_item in alias_list: + names.append(alias_item) # 户均总价 if (curr_city + curr_region + curr_name) in avg_price_map: @@ -63,13 +65,13 @@ def sync(city=None): if pd.notna(avg_price): min_avg_price = 0.8 * avg_price max_avg_rice = 1.2 * avg_price - total_price = commutils.get_total_price(curr_city, curr_region, names, day_before=93, min_avg_price=min_avg_price, max_avg_price=max_avg_rice) + total_price = commutils.get_total_price(curr_city, curr_region, names, day_before=366, min_avg_price=min_avg_price, max_avg_price=max_avg_rice) seaweed_df.loc[index, 'total_price'] = total_price for index, item in seaweed_df.iterrows(): pool.submit(deal, index, item) - seaweed_df['_id'] = seaweed_df['_id'].astype(str) + actions = list() count = 0 diff --git a/dags/business/business/community/trade.py b/dags/business/business/community/trade.py index 0be193c..5e08242 100644 --- a/dags/business/business/community/trade.py +++ b/dags/business/business/community/trade.py @@ -3,11 +3,10 @@ """ import logging import pandas as pd -from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, areautils +from goojia_utils import ElasticSearchClient, commutils, dateutils from elasticsearch import helpers log = logging.getLogger(__name__) -seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') es206_client = ElasticSearchClient(host='192.168.0.206') index_name = "community_summary" @@ -29,9 +28,7 @@ def sync(city=None): if pd.notna(city): seaweed_query['city'] = city cities = [city] - seaweed_df = pd.DataFrame( - list(seaweed_conn.find(seaweed_query, {'city': 1, 'region': 1, 'name': 1, 'alias': 1}).batch_size(100000))) - seaweed_df['_id'] = seaweed_df['_id'].astype(str) + seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/community/data/seaweed.csv', sep='@', encoding='UTF-8') deal_price_df = commutils.get_deal_avg_price(cities, 3) deal_price_df = deal_price_df[['match_city', 'match_region', 'match_name', 'avg_price']] diff --git a/dags/business/business/community/traffic.py b/dags/business/business/community/traffic.py index 5acb3bb..3d87720 100644 --- a/dags/business/business/community/traffic.py +++ b/dags/business/business/community/traffic.py @@ -3,13 +3,12 @@ """ import logging import pandas as pd -from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, areautils, poiutils +from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils from elasticsearch import helpers from concurrent.futures import ThreadPoolExecutor from threading import Lock log = logging.getLogger(__name__) -seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') es206_client = ElasticSearchClient(host='192.168.0.206') index_name = "community_summary" @@ -77,11 +76,8 @@ def sync(city=None): if pd.notna(city): seaweed_query['city'] = city - seaweed_df = pd.DataFrame( - list(seaweed_conn.find(seaweed_query, {'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1}) - .batch_size(100000))) - - seaweed_df['_id'] = seaweed_df['_id'].astype(str) + seaweed_df = seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/community/data/seaweed.csv', sep='@', + encoding='UTF-8') actions = list() with ThreadPoolExecutor(max_workers=30) as pool: