diff --git a/dags/business/business/community/avg_cost.py b/dags/business/business/community/avg_cost.py index 4eaac4a..38f9312 100644 --- a/dags/business/business/community/avg_cost.py +++ b/dags/business/business/community/avg_cost.py @@ -3,7 +3,7 @@ """ import logging import pandas as pd -from goojia_utils import ElasticSearchClient, commutils, dateutils, areautils, ClickHouseClient +from goojia_utils import ElasticSearchClient, commutils, dateutils, areautils, MAIN_MONGO_CLIENT from elasticsearch import helpers from concurrent.futures import ThreadPoolExecutor import sys @@ -13,7 +13,7 @@ mutex = Lock() log = logging.getLogger(__name__) es206_client = ElasticSearchClient(host='192.168.0.206') es89_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch') -tmp_ck = ClickHouseClient(database='tmp') +community_summary_conn = MAIN_MONGO_CLIENT.get_collection('tmp', 'community_summary') index_name = "community_summary" m_date = dateutils.iso_format() @@ -113,10 +113,10 @@ def find_around_price_item(seaweed): def sync(city=None): log.info("开始同步消费数据...") - tmp_sql = f"select * from community_summary " + query = {} if pd.notna(city): - tmp_sql += f" where city='{city}' " - seaweed_df = tmp_ck.read_sql_query(tmp_sql) + query['city'] = city + seaweed_df = pd.DataFrame(list(community_summary_conn.find(query))) seaweed_list = list() seaweed_df.apply(lambda x: seaweed_list.append(dict(x)), axis=1) diff --git a/dags/business/business/community/business_district.py b/dags/business/business/community/business_district.py index 5ebe1a4..b6c0f55 100644 --- a/dags/business/business/community/business_district.py +++ b/dags/business/business/community/business_district.py @@ -3,7 +3,7 @@ """ import logging import pandas as pd -from goojia_utils import ElasticSearchClient, commutils, dateutils, ClickHouseClient +from goojia_utils import ElasticSearchClient, commutils, dateutils, MAIN_MONGO_CLIENT from elasticsearch import helpers from concurrent.futures import ThreadPoolExecutor from threading import Lock @@ -11,7 +11,7 @@ import sys log = logging.getLogger(__name__) es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch') -tmp_ck = ClickHouseClient(database="tmp") +community_summary_conn = MAIN_MONGO_CLIENT.get_collection('tmp', 'community_summary') index_name = "community_summary" path = "/home/airflow/data/community" @@ -90,10 +90,10 @@ def deal(actions, item): def sync(city=None): log.info(f"开始同步商圈至206 {index_name}") - tmp_sql = f"select * from community_summary " + query = {} if pd.notna(city): - tmp_sql += f" where city='{city}'" - seaweed_df = tmp_ck.read_sql_query(tmp_sql) + query['city'] = city + seaweed_df = pd.DataFrame(list(community_summary_conn.find(query))) actions = list() with ThreadPoolExecutor(max_workers=30) as pool: diff --git a/dags/business/business/community/comm.py b/dags/business/business/community/comm.py index 655aac7..4713a8f 100644 --- a/dags/business/business/community/comm.py +++ b/dags/business/business/community/comm.py @@ -3,7 +3,31 @@ """ import logging import pandas as pd -from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, ClickHouseClient +import numpy as np + +# 兼容性补丁:解决 goojia_utils.ClickHouseClient.to_sql_plus 在将 DataFrame 写入 ClickHouse 时, +# 对含有 list/array 类型的列(如 alias)进行 pd.notna(item) 判定时导致 ValueError 的 bug +_orig_notna = pd.notna +def patched_notna(obj): + if isinstance(obj, (list, tuple, np.ndarray, dict, set)): + return True + try: + res = _orig_notna(obj) + if hasattr(res, 'any'): + return bool(res.any()) + elif isinstance(res, (list, np.ndarray)): + return any(res) + return res + except Exception: + return True + +pd.notna = patched_notna +pd.isna = lambda x: not patched_notna(x) +import pandas +pandas.notna = patched_notna +pandas.isna = lambda x: not patched_notna(x) + +from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils from elasticsearch import helpers import os import sys @@ -11,7 +35,7 @@ import sys log = logging.getLogger(__name__) seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch') -tmp_ck = ClickHouseClient(database='tmp') +community_summary_conn = MAIN_MONGO_CLIENT.get_collection('tmp', 'community_summary') index_name = "community_summary" m_date = dateutils.iso_format() @@ -115,11 +139,11 @@ def deal_comm(community_summary_df, seaweed_df): def sync(city=None): log.info(f"开始同步小区基本属性至206 {index_name}...") seaweed_query = {'status': 0, 'cat': 'district'} - ck_sql = "ALTER TABLE tmp.community_summary DELETE WHERE 1=1 " + mongo_delete_query = {} if pd.notna(city): seaweed_query['city'] = city - ck_sql += " AND city='" + city + "'" - tmp_ck.execute(ck_sql) + mongo_delete_query['city'] = city + community_summary_conn.delete_many(mongo_delete_query) seaweed_df = pd.DataFrame( list(seaweed_conn.find(seaweed_query, {'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1, @@ -156,9 +180,16 @@ def sync(city=None): # 写入csv文件,防止小区变动导致的后续过程更新报错, # seaweed_df.to_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8', index=False) - # 写入中间表clickhouse,解决多worker节点问题 - tmp_ck.to_sql_plus(seaweed_df[['_id', 'city', 'region', 'name', 'lng', 'lat', 'alias', 'estate_type2', 'build_type', - 'household_count', 'block', 'estate_charge', 'finish_building_date']], 'community_summary') + # 写入中间集合 MongoDB, 解决分布式 Worker 节点的共享问题 + insert_cols = ['_id', 'city', 'region', 'name', 'lng', 'lat', 'alias', 'estate_type2', 'build_type', + 'household_count', 'block', 'estate_charge', 'finish_building_date'] + records = seaweed_df[insert_cols].to_dict(orient='records') + if len(records) > 0: + log.info(f"正在写入 MongoDB community_summary 中间集合,共 {len(records)} 条记录...") + batch_size = 5000 + for i in range(0, len(records), batch_size): + community_summary_conn.insert_many(records[i:i + batch_size]) + log.info("MongoDB community_summary 写入完毕。") actions = list() # 对于存在的更新 diff --git a/dags/business/business/community/del_comm.py b/dags/business/business/community/del_comm.py index 7020187..3fd6f9d 100644 --- a/dags/business/business/community/del_comm.py +++ b/dags/business/business/community/del_comm.py @@ -19,7 +19,7 @@ except Exception: import logging import pandas as pd -from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, ClickHouseClient +from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils from elasticsearch import helpers import os import sys @@ -27,7 +27,6 @@ import sys log = logging.getLogger(__name__) seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch') -tmp_ck = ClickHouseClient(database='tmp') index_name = "community_summary" m_date = dateutils.iso_format() diff --git a/dags/business/business/community/education.py b/dags/business/business/community/education.py index 7ad695a..9b4ca50 100644 --- a/dags/business/business/community/education.py +++ b/dags/business/business/community/education.py @@ -3,7 +3,7 @@ """ import logging import pandas as pd -from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils, ClickHouseClient +from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils, MAIN_MONGO_CLIENT from elasticsearch import helpers from concurrent.futures import ThreadPoolExecutor from threading import Lock @@ -11,7 +11,7 @@ import sys log = logging.getLogger(__name__) es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch') -tmp_ck = ClickHouseClient(database="tmp") +community_summary_conn = MAIN_MONGO_CLIENT.get_collection('tmp', 'community_summary') index_name = "community_summary" path = "/home/airflow/data/community" @@ -66,10 +66,10 @@ def deal(actions, item): def sync(city=None): log.info(f"开始同步教育资源【幼儿园、大学】至206 {index_name}") - tmp_sql = f"select * from community_summary " + query = {} if pd.notna(city): - tmp_sql += f" where city='{city}'" - seaweed_df = tmp_ck.read_sql_query(tmp_sql) + query['city'] = city + seaweed_df = pd.DataFrame(list(community_summary_conn.find(query))) actions = list() with ThreadPoolExecutor(max_workers=30) as pool: diff --git a/dags/business/business/community/education2.py b/dags/business/business/community/education2.py index b948c04..185d154 100644 --- a/dags/business/business/community/education2.py +++ b/dags/business/business/community/education2.py @@ -3,7 +3,7 @@ """ import logging import pandas as pd -from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils, ClickHouseClient +from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils, MAIN_MONGO_CLIENT from elasticsearch import helpers from concurrent.futures import ThreadPoolExecutor from threading import Lock @@ -11,7 +11,7 @@ import sys log = logging.getLogger(__name__) es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch') -tmp_ck = ClickHouseClient(database="tmp") +community_summary_conn = MAIN_MONGO_CLIENT.get_collection('tmp', 'community_summary') index_name = "community_summary" path = "/home/airflow/data/community" @@ -81,10 +81,10 @@ def deal(actions, item): def sync(city=None): log.info(f"开始同步教育资源【小学、中学】至206 {index_name}") - tmp_sql = f"select * from community_summary " + query = {} if pd.notna(city): - tmp_sql += f" where city='{city}'" - seaweed_df = tmp_ck.read_sql_query(tmp_sql) + query['city'] = city + seaweed_df = pd.DataFrame(list(community_summary_conn.find(query))) actions = list() with ThreadPoolExecutor(max_workers=30) as pool: diff --git a/dags/business/business/community/life.py b/dags/business/business/community/life.py index a26c74a..39588b7 100644 --- a/dags/business/business/community/life.py +++ b/dags/business/business/community/life.py @@ -3,7 +3,7 @@ """ import logging import pandas as pd -from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils, ClickHouseClient +from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils, MAIN_MONGO_CLIENT from elasticsearch import helpers from concurrent.futures import ThreadPoolExecutor from threading import Lock @@ -11,7 +11,7 @@ import sys log = logging.getLogger(__name__) es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch') -tmp_ck = ClickHouseClient(database='tmp') +community_summary_conn = MAIN_MONGO_CLIENT.get_collection('tmp', 'community_summary') index_name = "community_summary" path = "/home/airflow/data/community" @@ -78,10 +78,10 @@ def deal(actions, item): def sync(city=None): log.info(f"开始同步生活配套至206 {index_name}") - tmp_sql = f"select * from community_summary " + query = {} if pd.notna(city): - tmp_sql += f" where city='{city}'" - seaweed_df = tmp_ck.read_sql_query(tmp_sql) + query['city'] = city + seaweed_df = pd.DataFrame(list(community_summary_conn.find(query))) actions = list() with ThreadPoolExecutor(max_workers=30) as pool: diff --git a/dags/business/business/community/medical.py b/dags/business/business/community/medical.py index a1f746c..0fb59fd 100644 --- a/dags/business/business/community/medical.py +++ b/dags/business/business/community/medical.py @@ -3,7 +3,7 @@ """ import logging import pandas as pd -from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, ClickHouseClient, poiutils +from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, poiutils from elasticsearch import helpers from concurrent.futures import ThreadPoolExecutor from threading import Lock @@ -12,7 +12,7 @@ import sys log = logging.getLogger(__name__) seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch') -tmp_ck = ClickHouseClient(database='tmp') +community_summary_conn = MAIN_MONGO_CLIENT.get_collection('tmp', 'community_summary') index_name = "community_summary" path = "/home/airflow/data/community" @@ -66,10 +66,10 @@ def deal(actions, item): def sync(city=None): log.info(f"开始同步医院至206 {index_name}") - tmp_sql = f"select * from community_summary " + query = {} if pd.notna(city): - tmp_sql += f" where city='{city}'" - seaweed_df = tmp_ck.read_sql_query(tmp_sql) + query['city'] = city + seaweed_df = pd.DataFrame(list(community_summary_conn.find(query))) actions = list() with ThreadPoolExecutor(max_workers=30) as pool: diff --git a/dags/business/business/community/price.py b/dags/business/business/community/price.py index ab68a8e..2158679 100644 --- a/dags/business/business/community/price.py +++ b/dags/business/business/community/price.py @@ -3,13 +3,13 @@ """ import logging import pandas as pd -from goojia_utils import ElasticSearchClient, commutils, dateutils, areautils, ClickHouseClient +from goojia_utils import ElasticSearchClient, commutils, dateutils, areautils, MAIN_MONGO_CLIENT from elasticsearch import helpers import sys log = logging.getLogger(__name__) es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch') -tmp_ck = ClickHouseClient(database='tmp') +community_summary_conn = MAIN_MONGO_CLIENT.get_collection('tmp', 'community_summary') index_name = "community_summary" path = "/home/airflow/data/community" @@ -42,7 +42,7 @@ def build_doc(item): def sync_by_city(city): - seaweed_df = tmp_ck.read_sql_query(f"select _id, city, region, name from community_summary where city='{city}'") + seaweed_df = pd.DataFrame(list(community_summary_conn.find({"city": city}, {"_id": 1, "city": 1, "region": 1, "name": 1}))) if len(seaweed_df) == 0: return diff --git a/dags/business/business/community/relative_price_ranking.py b/dags/business/business/community/relative_price_ranking.py index 0b25c4c..9d99626 100644 --- a/dags/business/business/community/relative_price_ranking.py +++ b/dags/business/business/community/relative_price_ranking.py @@ -4,7 +4,7 @@ """ import logging import pandas as pd -from goojia_utils import ElasticSearchClient, commutils, dateutils, ClickHouseClient +from goojia_utils import ElasticSearchClient, commutils, dateutils, MAIN_MONGO_CLIENT from elasticsearch import helpers from concurrent.futures import ThreadPoolExecutor import sys @@ -12,7 +12,7 @@ from threading import Lock log = logging.getLogger(__name__) es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch') -tmp_ck = ClickHouseClient(database='tmp') +community_summary_conn = MAIN_MONGO_CLIENT.get_collection('tmp', 'community_summary') index_name = "community_summary" path = "/home/airflow/data/community" @@ -134,10 +134,10 @@ def find_around_price_item(seaweed): def sync(city=None): log.info("开始同步相对价格排名数据...") - tmp_sql = f"select * from community_summary " + query = {} if pd.notna(city): - tmp_sql += f" where city='{city}'" - seaweed_df = tmp_ck.read_sql_query(tmp_sql) + query['city'] = city + seaweed_df = pd.DataFrame(list(community_summary_conn.find(query))) seaweed_list = list() seaweed_df.apply(lambda x: seaweed_list.append(dict(x)), axis=1) diff --git a/dags/business/business/community/sell.py b/dags/business/business/community/sell.py index 6ad7918..1f1ec5e 100644 --- a/dags/business/business/community/sell.py +++ b/dags/business/business/community/sell.py @@ -3,7 +3,7 @@ """ import logging import pandas as pd -from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, ClickHouseClient +from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils from elasticsearch import helpers from concurrent.futures import ThreadPoolExecutor import sys @@ -12,7 +12,7 @@ from threading import Lock log = logging.getLogger(__name__) seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch') -tmp_ck = ClickHouseClient(database='tmp') +community_summary_conn = MAIN_MONGO_CLIENT.get_collection('tmp', 'community_summary') index_name = "community_summary" path = "/home/airflow/data/community" @@ -33,10 +33,10 @@ def build_doc(item): def sync(city=None): log.info(f"开始同步出售信息至206 {index_name}") - tmp_sql = f"select * from community_summary " + query = {} if pd.notna(city): - tmp_sql += f" where city='{city}'" - seaweed_df = tmp_ck.read_sql_query(tmp_sql) + query['city'] = city + seaweed_df = pd.DataFrame(list(community_summary_conn.find(query))) with ThreadPoolExecutor(30) as pool: def deal(index, item): diff --git a/dags/business/business/community/sync_es_price_to_mongo.py b/dags/business/business/community/sync_es_price_to_mongo.py new file mode 100644 index 0000000..120f240 --- /dev/null +++ b/dags/business/business/community/sync_es_price_to_mongo.py @@ -0,0 +1,223 @@ +import os +import sys +import logging +import pandas as pd +from datetime import datetime +from pymongo import UpdateOne + +# 兼容性补丁:解决 Elasticsearch 8.x/9.x 客户端在老版本 ES 集群上因 compatible-with 头部导致的 ApiError(406) +os.environ["ELASTIC_CLIENT_APIVERSIONING"] = "0" +try: + import elasticsearch + ver = getattr(elasticsearch, '__version__', None) + if ver and not str(ver).startswith('7.'): + original_init = elasticsearch.Elasticsearch.__init__ + def patched_init(self, *args, **kwargs): + headers = kwargs.get('headers', {}) + headers['Accept'] = 'application/json' + headers['Content-Type'] = 'application/json' + kwargs['headers'] = headers + original_init(self, *args, **kwargs) + elasticsearch.Elasticsearch.__init__ = patched_init +except Exception: + pass + +from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient + +logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') +log = logging.getLogger(__name__) + +# 初始化 ES 206 客户端与 85 上的 MongoDB 集合 +es206_client = ElasticSearchClient(host='192.168.0.206', username='elastic', password='elasticsearch') +average_statistics_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'average_statistics') + +index_name = "community" + +# 映射关系:将 ES 中的 root 主域映射为 Mongo 表中的对应二手房价格 (_esf_price) 字段 +ROOT_FIELD_MAP = { + "anjuke.com": "anjuke_esf_price", # 安居客 + "lianjia.com": "lianjia_esf_price", # 链家 + "ke.com": "beike_esf_price", # 贝壳 + "beike.com": "beike_esf_price", # 贝壳 + "fang.com": "soufang_esf_price", # 房天下/搜房 + "soufang.com": "soufang_esf_price", # 搜房 + "5i5j.com": "5i5j_esf_price", # 我爱我家 + "centanet.com": "zhongyuan_esf_price", # 中原地产 + "qfang.com": "qfang_esf_price", # Q房网 + "leyoujia.com": "leyoujia_esf_price", # 乐有家 + "zhuge.com": "zhugezhaofang_esf_price", # 诸葛找房 + "zhugezhaofang.com": "zhugezhaofang_esf_price", + "jiwu.com": "jiwu_esf_price", # 吉屋 + "house365.com": "365house_esf_price", # 365淘房 + "365.com": "365house_esf_price", + "fanggugu.com": "fanggugu_esf_price", # 房估估 + "yungujia.com": "yungujia_esf_price", # 云估价 + "tuboshi.com": "tuboshi_esf_price", # 兔博士 + "xiaozijia.com": "xiaozijia_esf_price", # 小资家 + "fangdoudou.com": "fangdoudou_esf_price",# 房兜兜 + "fangtan.com": "fangtan_esf_price", # 房探 + "guoxinda.com": "gxd_esf_price", # 国信达 + "gxd.com": "gxd_esf_price", + "xingfuli.com": "xingfuli_esf_price", # 幸福里 + "maitian.cn": "maitian_esf_price", # 麦田房产 + "maitian.com": "maitian_esf_price", + "sina.com.cn": "sina_esf_price", # 新浪二手房 + "sina.com": "sina_esf_price", + "cric.com": "cric_esf_price", # 克而瑞 + "ganji.com": "ganji_esf_price", # 赶集网 +} + +def parse_s_date(m_date_str): + """ + 将 ES 抓取时间 '2022-06-17T13:12:47.148Z' 转化为整型 s_date YYYYMM (例如 202206) + """ + try: + if pd.isna(m_date_str) or not isinstance(m_date_str, str): + return None + return int(m_date_str[:7].replace('-', '')) + except Exception: + return None + +def sync_data(target_month=None): + """ + 数据流式拉取、转换与批量 Upsert + """ + log.info("开始从 ES 206 的 community 索引拉取小区价格数据...") + + # 构造 SQL,默认仅同步近 3 个月(90 天)内更新的数据,降低传输和存储开销 + if target_month is not None: + try: + month_str = str(target_month) + year = int(month_str[:4]) + month = int(month_str[4:6]) + start_date = f"{year:04d}-{month:02d}-01T00:00:00" + if month == 12: + end_date = f"{year + 1:04d}-01-01T00:00:00" + else: + end_date = f"{year:04d}-{month + 1:02d}-01T00:00:00" + + date_filter = f"and m_date >= '{start_date}' and m_date < '{end_date}'" + log.info(f"指定历史月份跑数,过滤条件: {start_date} 至 {end_date}") + except Exception as e: + log.error(f"解析指定月份失败,退回到默认的近3个月同步。Error: {e}") + from datetime import datetime, timedelta + three_months_ago = (datetime.now() - timedelta(days=90)).strftime('%Y-%m-%dT%H:%M:%S') + date_filter = f"and m_date >= '{three_months_ago}'" + else: + from datetime import datetime, timedelta + three_months_ago = (datetime.now() - timedelta(days=90)).strftime('%Y-%m-%dT%H:%M:%S') + date_filter = f"and m_date >= '{three_months_ago}'" + log.info(f"增量运行,过滤近3个月内有更新的数据 (m_date >= '{three_months_ago}')") + + sql = f"select city, region, name, avg_price, root, m_date from {index_name} where avg_price is not null {date_filter}" + + # 城市手动过滤(可选,通过外部参数调度) + sys_city = sys.argv[1] if len(sys.argv) > 1 and not sys.argv[1].isdigit() else None + if pd.notna(sys_city): + sql += f" and city = '{sys_city}'" + log.info(f"正在过滤目标城市: {sys_city}") + + df = es206_client.scan_by_sql_to_data_frame(sql) + if df is None or len(df) == 0: + log.info("未从 ES 中获取到可用数据。") + return + + log.info(f"成功读取到 {len(df)} 条原始记录。开始解析字段,并根据唯一键 (city, region, name, s_date) 进行内存合并...") + + # 聚合内存合并,以支持多个不同 root (渠道来源) 的价格在同一个 (城市-行政区-小区名-月份) 唯一键文档中合并设置 + grouped = {} + + for idx, row in df.iterrows(): + city = row.get('city') + region = row.get('region') + name = row.get('name') + avg_price = row.get('avg_price') + root = row.get('root') + m_date = row.get('m_date') + + if pd.isna(city) or pd.isna(region) or pd.isna(name) or pd.isna(avg_price) or pd.isna(root): + continue + + s_date = parse_s_date(m_date) + if s_date is None: + continue + + # 若指定了目标月份过滤(如仅同步指定月份的数据) + if target_month is not None and s_date != int(target_month): + continue + + field_name = ROOT_FIELD_MAP.get(root) + if not field_name: + continue + + key = (city, region, name, s_date) + if key not in grouped: + grouped[key] = {} + + # 多个相同 root 冲突时,此处保留最新或较高价格(以整型存储) + price_val = int(float(avg_price)) + current_val = grouped[key].get(field_name) + if current_val is None or price_val > current_val: + grouped[key][field_name] = price_val + + log.info(f"内存聚合清洗完毕。共获得 {len(grouped)} 组唯一索引项。开始构建批量 MongoDB UpdateOne upsert 任务...") + + # 构建 Bulk Write Upsert 任务 + actions = [] + now = datetime.now() + + for (city, region, name, s_date), fields_dict in grouped.items(): + # 需要 $set 写入的所有第三方价格字段及最近修改时间 + update_doc = {**fields_dict} + update_doc["m_date"] = now # ModificationDate 对应实体属性 + + query = { + "city": city, + "region": region, + "name": name, + "s_date": s_date + } + + # Upsert 动作 + action = UpdateOne( + query, + { + "$set": update_doc, + "$setOnInsert": { + "category": "district", # 默认对应小区类别 district + "date": now # 对应首建/大日期 longDate + } + }, + upsert=True + ) + actions.append(action) + + total_actions = len(actions) + if total_actions == 0: + log.info("过滤处理后无数据需写入 MongoDB。") + return + + log.info(f"开始向 192.168.0.85 数据库的 average_statistics 集合中分批写入 {total_actions} 条大批处理记录...") + + # 5000条一笔批量执行写入,防止网络长连接超时或内存剧增 + batch_size = 5000 + for i in range(0, total_actions, batch_size): + batch_actions = actions[i:i + batch_size] + result = average_statistics_conn.bulk_write(batch_actions) + log.info(f"批次 {i // batch_size + 1} 写入情况: " + f"Matched={result.matched_count}, " + f"Upserted={result.upserted_count}, " + f"Modified={result.modified_count}") + + log.info("数据同步流任务全部成功结束!") + +if __name__ == '__main__': + # 支持传入参数如 `python sync_es_price_to_mongo.py 202401` 进行指定月份跑数 + month_arg = None + if len(sys.argv) > 1: + arg = sys.argv[1] + if arg.isdigit() and len(arg) == 6: + month_arg = int(arg) + log.info(f"读取到指定月份运行参数: {month_arg}") + + sync_data(target_month=month_arg) diff --git a/dags/business/business/community/trade.py b/dags/business/business/community/trade.py index 9726710..afa0783 100644 --- a/dags/business/business/community/trade.py +++ b/dags/business/business/community/trade.py @@ -4,12 +4,12 @@ import logging import pandas as pd import sys -from goojia_utils import ElasticSearchClient, commutils, dateutils, ClickHouseClient +from goojia_utils import ElasticSearchClient, commutils, dateutils, MAIN_MONGO_CLIENT from elasticsearch import helpers log = logging.getLogger(__name__) es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch') -tmp_ck = ClickHouseClient(database='tmp') +community_summary_conn = MAIN_MONGO_CLIENT.get_collection('tmp', 'community_summary') index_name = "community_summary" path = "/home/airflow/data/community" @@ -28,11 +28,10 @@ def sync(city=None): cities = None - tmp_sql = f"select * from community_summary " + query = {} if pd.notna(city): - cities = [city] - tmp_sql += f" where city='{city}'" - seaweed_df = tmp_ck.read_sql_query(tmp_sql) + query['city'] = city + seaweed_df = pd.DataFrame(list(community_summary_conn.find(query))) deal_price_df = commutils.get_deal_avg_price(cities, 3) if len(deal_price_df) <= 0: diff --git a/dags/business/business/community/traffic.py b/dags/business/business/community/traffic.py index 2da6c20..e94737c 100644 --- a/dags/business/business/community/traffic.py +++ b/dags/business/business/community/traffic.py @@ -3,7 +3,7 @@ """ import logging import pandas as pd -from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils, ClickHouseClient +from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils, MAIN_MONGO_CLIENT from elasticsearch import helpers from concurrent.futures import ThreadPoolExecutor from threading import Lock @@ -11,7 +11,7 @@ import sys log = logging.getLogger(__name__) es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch') -tmp_ck = ClickHouseClient(database='tmp') +community_summary_conn = MAIN_MONGO_CLIENT.get_collection('tmp', 'community_summary') index_name = "community_summary" path = "/home/airflow/data/community" @@ -72,10 +72,10 @@ def deal(actions, item): def sync(city=None): log.info(f"开始同步交通至206 {index_name}") - tmp_sql = f"select * from community_summary " + query = {} if pd.notna(city): - tmp_sql += f" where city='{city}'" - seaweed_df = tmp_ck.read_sql_query(tmp_sql) + query['city'] = city + seaweed_df = pd.DataFrame(list(community_summary_conn.find(query))) actions = list() with ThreadPoolExecutor(max_workers=30) as pool: diff --git a/dags/business/dag_sync_es_price_to_mongo.py b/dags/business/dag_sync_es_price_to_mongo.py new file mode 100644 index 0000000..0a3d9b3 --- /dev/null +++ b/dags/business/dag_sync_es_price_to_mongo.py @@ -0,0 +1,17 @@ +from airflow.operators.bash import BashOperator +import pendulum +from airflow import DAG + +with DAG( + dag_id="sync_es_price_to_mongo_dag", + # 每月3日凌晨2点整执行(在大部分上游爬取和ES清洗完毕后进行数据同步归档) + schedule="0 2 3 * *", + start_date=pendulum.datetime(2026, 8, 16, tz="Asia/Shanghai"), + catchup=False, + description="每月定时把 ES 206 上的 community 均价数据同步到 85 Mongo 的 average_statistics 表" +) as dag: + + sync_es_price_to_mongo = BashOperator( + task_id='sync_es_price_to_mongo', + bash_command='python /opt/airflow/dags/repo/dags/business/business/community/sync_es_price_to_mongo.py {{dag_run.conf["month"] if "month" in dag_run.conf else ""}}', + )