feat: ClickHouse 彻底下线并平替为 MongoDB tmp 库,新增 ES 206 均价至 MongoDB 85 月度同步任务

This commit is contained in:
王文艳
2026-08-17 09:08:45 +08:00
parent 6dae2f895c
commit d04bdbd464
15 changed files with 333 additions and 64 deletions
+5 -5
View File
@@ -3,7 +3,7 @@
"""
import logging
import pandas as pd
from goojia_utils import ElasticSearchClient, commutils, dateutils, areautils, ClickHouseClient
from goojia_utils import ElasticSearchClient, commutils, dateutils, areautils, MAIN_MONGO_CLIENT
from elasticsearch import helpers
from concurrent.futures import ThreadPoolExecutor
import sys
@@ -13,7 +13,7 @@ mutex = Lock()
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
es89_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
tmp_ck = ClickHouseClient(database='tmp')
community_summary_conn = MAIN_MONGO_CLIENT.get_collection('tmp', 'community_summary')
index_name = "community_summary"
m_date = dateutils.iso_format()
@@ -113,10 +113,10 @@ def find_around_price_item(seaweed):
def sync(city=None):
log.info("开始同步消费数据...")
tmp_sql = f"select * from community_summary "
query = {}
if pd.notna(city):
tmp_sql += f" where city='{city}' "
seaweed_df = tmp_ck.read_sql_query(tmp_sql)
query['city'] = city
seaweed_df = pd.DataFrame(list(community_summary_conn.find(query)))
seaweed_list = list()
seaweed_df.apply(lambda x: seaweed_list.append(dict(x)), axis=1)
@@ -3,7 +3,7 @@
"""
import logging
import pandas as pd
from goojia_utils import ElasticSearchClient, commutils, dateutils, ClickHouseClient
from goojia_utils import ElasticSearchClient, commutils, dateutils, MAIN_MONGO_CLIENT
from elasticsearch import helpers
from concurrent.futures import ThreadPoolExecutor
from threading import Lock
@@ -11,7 +11,7 @@ import sys
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
tmp_ck = ClickHouseClient(database="tmp")
community_summary_conn = MAIN_MONGO_CLIENT.get_collection('tmp', 'community_summary')
index_name = "community_summary"
path = "/home/airflow/data/community"
@@ -90,10 +90,10 @@ def deal(actions, item):
def sync(city=None):
log.info(f"开始同步商圈至206 {index_name}")
tmp_sql = f"select * from community_summary "
query = {}
if pd.notna(city):
tmp_sql += f" where city='{city}'"
seaweed_df = tmp_ck.read_sql_query(tmp_sql)
query['city'] = city
seaweed_df = pd.DataFrame(list(community_summary_conn.find(query)))
actions = list()
with ThreadPoolExecutor(max_workers=30) as pool:
+39 -8
View File
@@ -3,7 +3,31 @@
"""
import logging
import pandas as pd
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, ClickHouseClient
import numpy as np
# 兼容性补丁:解决 goojia_utils.ClickHouseClient.to_sql_plus 在将 DataFrame 写入 ClickHouse 时,
# 对含有 list/array 类型的列(如 alias)进行 pd.notna(item) 判定时导致 ValueError 的 bug
_orig_notna = pd.notna
def patched_notna(obj):
if isinstance(obj, (list, tuple, np.ndarray, dict, set)):
return True
try:
res = _orig_notna(obj)
if hasattr(res, 'any'):
return bool(res.any())
elif isinstance(res, (list, np.ndarray)):
return any(res)
return res
except Exception:
return True
pd.notna = patched_notna
pd.isna = lambda x: not patched_notna(x)
import pandas
pandas.notna = patched_notna
pandas.isna = lambda x: not patched_notna(x)
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils
from elasticsearch import helpers
import os
import sys
@@ -11,7 +35,7 @@ import sys
log = logging.getLogger(__name__)
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
tmp_ck = ClickHouseClient(database='tmp')
community_summary_conn = MAIN_MONGO_CLIENT.get_collection('tmp', 'community_summary')
index_name = "community_summary"
m_date = dateutils.iso_format()
@@ -115,11 +139,11 @@ def deal_comm(community_summary_df, seaweed_df):
def sync(city=None):
log.info(f"开始同步小区基本属性至206 {index_name}...")
seaweed_query = {'status': 0, 'cat': 'district'}
ck_sql = "ALTER TABLE tmp.community_summary DELETE WHERE 1=1 "
mongo_delete_query = {}
if pd.notna(city):
seaweed_query['city'] = city
ck_sql += " AND city='" + city + "'"
tmp_ck.execute(ck_sql)
mongo_delete_query['city'] = city
community_summary_conn.delete_many(mongo_delete_query)
seaweed_df = pd.DataFrame(
list(seaweed_conn.find(seaweed_query, {'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1,
@@ -156,9 +180,16 @@ def sync(city=None):
# 写入csv文件,防止小区变动导致的后续过程更新报错,
# seaweed_df.to_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8', index=False)
# 写入中间表clickhouse,解决多worker节点问题
tmp_ck.to_sql_plus(seaweed_df[['_id', 'city', 'region', 'name', 'lng', 'lat', 'alias', 'estate_type2', 'build_type',
'household_count', 'block', 'estate_charge', 'finish_building_date']], 'community_summary')
# 写入中间集合 MongoDB, 解决分布式 Worker 节点的共享问题
insert_cols = ['_id', 'city', 'region', 'name', 'lng', 'lat', 'alias', 'estate_type2', 'build_type',
'household_count', 'block', 'estate_charge', 'finish_building_date']
records = seaweed_df[insert_cols].to_dict(orient='records')
if len(records) > 0:
log.info(f"正在写入 MongoDB community_summary 中间集合,共 {len(records)} 条记录...")
batch_size = 5000
for i in range(0, len(records), batch_size):
community_summary_conn.insert_many(records[i:i + batch_size])
log.info("MongoDB community_summary 写入完毕。")
actions = list()
# 对于存在的更新
+1 -2
View File
@@ -19,7 +19,7 @@ except Exception:
import logging
import pandas as pd
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, ClickHouseClient
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils
from elasticsearch import helpers
import os
import sys
@@ -27,7 +27,6 @@ import sys
log = logging.getLogger(__name__)
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
tmp_ck = ClickHouseClient(database='tmp')
index_name = "community_summary"
m_date = dateutils.iso_format()
@@ -3,7 +3,7 @@
"""
import logging
import pandas as pd
from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils, ClickHouseClient
from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils, MAIN_MONGO_CLIENT
from elasticsearch import helpers
from concurrent.futures import ThreadPoolExecutor
from threading import Lock
@@ -11,7 +11,7 @@ import sys
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
tmp_ck = ClickHouseClient(database="tmp")
community_summary_conn = MAIN_MONGO_CLIENT.get_collection('tmp', 'community_summary')
index_name = "community_summary"
path = "/home/airflow/data/community"
@@ -66,10 +66,10 @@ def deal(actions, item):
def sync(city=None):
log.info(f"开始同步教育资源【幼儿园、大学】至206 {index_name}")
tmp_sql = f"select * from community_summary "
query = {}
if pd.notna(city):
tmp_sql += f" where city='{city}'"
seaweed_df = tmp_ck.read_sql_query(tmp_sql)
query['city'] = city
seaweed_df = pd.DataFrame(list(community_summary_conn.find(query)))
actions = list()
with ThreadPoolExecutor(max_workers=30) as pool:
@@ -3,7 +3,7 @@
"""
import logging
import pandas as pd
from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils, ClickHouseClient
from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils, MAIN_MONGO_CLIENT
from elasticsearch import helpers
from concurrent.futures import ThreadPoolExecutor
from threading import Lock
@@ -11,7 +11,7 @@ import sys
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
tmp_ck = ClickHouseClient(database="tmp")
community_summary_conn = MAIN_MONGO_CLIENT.get_collection('tmp', 'community_summary')
index_name = "community_summary"
path = "/home/airflow/data/community"
@@ -81,10 +81,10 @@ def deal(actions, item):
def sync(city=None):
log.info(f"开始同步教育资源【小学、中学】至206 {index_name}")
tmp_sql = f"select * from community_summary "
query = {}
if pd.notna(city):
tmp_sql += f" where city='{city}'"
seaweed_df = tmp_ck.read_sql_query(tmp_sql)
query['city'] = city
seaweed_df = pd.DataFrame(list(community_summary_conn.find(query)))
actions = list()
with ThreadPoolExecutor(max_workers=30) as pool:
+5 -5
View File
@@ -3,7 +3,7 @@
"""
import logging
import pandas as pd
from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils, ClickHouseClient
from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils, MAIN_MONGO_CLIENT
from elasticsearch import helpers
from concurrent.futures import ThreadPoolExecutor
from threading import Lock
@@ -11,7 +11,7 @@ import sys
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
tmp_ck = ClickHouseClient(database='tmp')
community_summary_conn = MAIN_MONGO_CLIENT.get_collection('tmp', 'community_summary')
index_name = "community_summary"
path = "/home/airflow/data/community"
@@ -78,10 +78,10 @@ def deal(actions, item):
def sync(city=None):
log.info(f"开始同步生活配套至206 {index_name}")
tmp_sql = f"select * from community_summary "
query = {}
if pd.notna(city):
tmp_sql += f" where city='{city}'"
seaweed_df = tmp_ck.read_sql_query(tmp_sql)
query['city'] = city
seaweed_df = pd.DataFrame(list(community_summary_conn.find(query)))
actions = list()
with ThreadPoolExecutor(max_workers=30) as pool:
+5 -5
View File
@@ -3,7 +3,7 @@
"""
import logging
import pandas as pd
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, ClickHouseClient, poiutils
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, poiutils
from elasticsearch import helpers
from concurrent.futures import ThreadPoolExecutor
from threading import Lock
@@ -12,7 +12,7 @@ import sys
log = logging.getLogger(__name__)
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
tmp_ck = ClickHouseClient(database='tmp')
community_summary_conn = MAIN_MONGO_CLIENT.get_collection('tmp', 'community_summary')
index_name = "community_summary"
path = "/home/airflow/data/community"
@@ -66,10 +66,10 @@ def deal(actions, item):
def sync(city=None):
log.info(f"开始同步医院至206 {index_name}")
tmp_sql = f"select * from community_summary "
query = {}
if pd.notna(city):
tmp_sql += f" where city='{city}'"
seaweed_df = tmp_ck.read_sql_query(tmp_sql)
query['city'] = city
seaweed_df = pd.DataFrame(list(community_summary_conn.find(query)))
actions = list()
with ThreadPoolExecutor(max_workers=30) as pool:
+3 -3
View File
@@ -3,13 +3,13 @@
"""
import logging
import pandas as pd
from goojia_utils import ElasticSearchClient, commutils, dateutils, areautils, ClickHouseClient
from goojia_utils import ElasticSearchClient, commutils, dateutils, areautils, MAIN_MONGO_CLIENT
from elasticsearch import helpers
import sys
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
tmp_ck = ClickHouseClient(database='tmp')
community_summary_conn = MAIN_MONGO_CLIENT.get_collection('tmp', 'community_summary')
index_name = "community_summary"
path = "/home/airflow/data/community"
@@ -42,7 +42,7 @@ def build_doc(item):
def sync_by_city(city):
seaweed_df = tmp_ck.read_sql_query(f"select _id, city, region, name from community_summary where city='{city}'")
seaweed_df = pd.DataFrame(list(community_summary_conn.find({"city": city}, {"_id": 1, "city": 1, "region": 1, "name": 1})))
if len(seaweed_df) == 0:
return
@@ -4,7 +4,7 @@
"""
import logging
import pandas as pd
from goojia_utils import ElasticSearchClient, commutils, dateutils, ClickHouseClient
from goojia_utils import ElasticSearchClient, commutils, dateutils, MAIN_MONGO_CLIENT
from elasticsearch import helpers
from concurrent.futures import ThreadPoolExecutor
import sys
@@ -12,7 +12,7 @@ from threading import Lock
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
tmp_ck = ClickHouseClient(database='tmp')
community_summary_conn = MAIN_MONGO_CLIENT.get_collection('tmp', 'community_summary')
index_name = "community_summary"
path = "/home/airflow/data/community"
@@ -134,10 +134,10 @@ def find_around_price_item(seaweed):
def sync(city=None):
log.info("开始同步相对价格排名数据...")
tmp_sql = f"select * from community_summary "
query = {}
if pd.notna(city):
tmp_sql += f" where city='{city}'"
seaweed_df = tmp_ck.read_sql_query(tmp_sql)
query['city'] = city
seaweed_df = pd.DataFrame(list(community_summary_conn.find(query)))
seaweed_list = list()
seaweed_df.apply(lambda x: seaweed_list.append(dict(x)), axis=1)
+5 -5
View File
@@ -3,7 +3,7 @@
"""
import logging
import pandas as pd
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, ClickHouseClient
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils
from elasticsearch import helpers
from concurrent.futures import ThreadPoolExecutor
import sys
@@ -12,7 +12,7 @@ from threading import Lock
log = logging.getLogger(__name__)
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
tmp_ck = ClickHouseClient(database='tmp')
community_summary_conn = MAIN_MONGO_CLIENT.get_collection('tmp', 'community_summary')
index_name = "community_summary"
path = "/home/airflow/data/community"
@@ -33,10 +33,10 @@ def build_doc(item):
def sync(city=None):
log.info(f"开始同步出售信息至206 {index_name}")
tmp_sql = f"select * from community_summary "
query = {}
if pd.notna(city):
tmp_sql += f" where city='{city}'"
seaweed_df = tmp_ck.read_sql_query(tmp_sql)
query['city'] = city
seaweed_df = pd.DataFrame(list(community_summary_conn.find(query)))
with ThreadPoolExecutor(30) as pool:
def deal(index, item):
@@ -0,0 +1,223 @@
import os
import sys
import logging
import pandas as pd
from datetime import datetime
from pymongo import UpdateOne
# 兼容性补丁:解决 Elasticsearch 8.x/9.x 客户端在老版本 ES 集群上因 compatible-with 头部导致的 ApiError(406)
os.environ["ELASTIC_CLIENT_APIVERSIONING"] = "0"
try:
import elasticsearch
ver = getattr(elasticsearch, '__version__', None)
if ver and not str(ver).startswith('7.'):
original_init = elasticsearch.Elasticsearch.__init__
def patched_init(self, *args, **kwargs):
headers = kwargs.get('headers', {})
headers['Accept'] = 'application/json'
headers['Content-Type'] = 'application/json'
kwargs['headers'] = headers
original_init(self, *args, **kwargs)
elasticsearch.Elasticsearch.__init__ = patched_init
except Exception:
pass
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
log = logging.getLogger(__name__)
# 初始化 ES 206 客户端与 85 上的 MongoDB 集合
es206_client = ElasticSearchClient(host='192.168.0.206', username='elastic', password='elasticsearch')
average_statistics_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'average_statistics')
index_name = "community"
# 映射关系:将 ES 中的 root 主域映射为 Mongo 表中的对应二手房价格 (_esf_price) 字段
ROOT_FIELD_MAP = {
"anjuke.com": "anjuke_esf_price", # 安居客
"lianjia.com": "lianjia_esf_price", # 链家
"ke.com": "beike_esf_price", # 贝壳
"beike.com": "beike_esf_price", # 贝壳
"fang.com": "soufang_esf_price", # 房天下/搜房
"soufang.com": "soufang_esf_price", # 搜房
"5i5j.com": "5i5j_esf_price", # 我爱我家
"centanet.com": "zhongyuan_esf_price", # 中原地产
"qfang.com": "qfang_esf_price", # Q房网
"leyoujia.com": "leyoujia_esf_price", # 乐有家
"zhuge.com": "zhugezhaofang_esf_price", # 诸葛找房
"zhugezhaofang.com": "zhugezhaofang_esf_price",
"jiwu.com": "jiwu_esf_price", # 吉屋
"house365.com": "365house_esf_price", # 365淘房
"365.com": "365house_esf_price",
"fanggugu.com": "fanggugu_esf_price", # 房估估
"yungujia.com": "yungujia_esf_price", # 云估价
"tuboshi.com": "tuboshi_esf_price", # 兔博士
"xiaozijia.com": "xiaozijia_esf_price", # 小资家
"fangdoudou.com": "fangdoudou_esf_price",# 房兜兜
"fangtan.com": "fangtan_esf_price", # 房探
"guoxinda.com": "gxd_esf_price", # 国信达
"gxd.com": "gxd_esf_price",
"xingfuli.com": "xingfuli_esf_price", # 幸福里
"maitian.cn": "maitian_esf_price", # 麦田房产
"maitian.com": "maitian_esf_price",
"sina.com.cn": "sina_esf_price", # 新浪二手房
"sina.com": "sina_esf_price",
"cric.com": "cric_esf_price", # 克而瑞
"ganji.com": "ganji_esf_price", # 赶集网
}
def parse_s_date(m_date_str):
"""
将 ES 抓取时间 '2022-06-17T13:12:47.148Z' 转化为整型 s_date YYYYMM (例如 202206)
"""
try:
if pd.isna(m_date_str) or not isinstance(m_date_str, str):
return None
return int(m_date_str[:7].replace('-', ''))
except Exception:
return None
def sync_data(target_month=None):
"""
数据流式拉取、转换与批量 Upsert
"""
log.info("开始从 ES 206 的 community 索引拉取小区价格数据...")
# 构造 SQL,默认仅同步近 3 个月(90 天)内更新的数据,降低传输和存储开销
if target_month is not None:
try:
month_str = str(target_month)
year = int(month_str[:4])
month = int(month_str[4:6])
start_date = f"{year:04d}-{month:02d}-01T00:00:00"
if month == 12:
end_date = f"{year + 1:04d}-01-01T00:00:00"
else:
end_date = f"{year:04d}-{month + 1:02d}-01T00:00:00"
date_filter = f"and m_date >= '{start_date}' and m_date < '{end_date}'"
log.info(f"指定历史月份跑数,过滤条件: {start_date} 至 {end_date}")
except Exception as e:
log.error(f"解析指定月份失败,退回到默认的近3个月同步。Error: {e}")
from datetime import datetime, timedelta
three_months_ago = (datetime.now() - timedelta(days=90)).strftime('%Y-%m-%dT%H:%M:%S')
date_filter = f"and m_date >= '{three_months_ago}'"
else:
from datetime import datetime, timedelta
three_months_ago = (datetime.now() - timedelta(days=90)).strftime('%Y-%m-%dT%H:%M:%S')
date_filter = f"and m_date >= '{three_months_ago}'"
log.info(f"增量运行,过滤近3个月内有更新的数据 (m_date >= '{three_months_ago}')")
sql = f"select city, region, name, avg_price, root, m_date from {index_name} where avg_price is not null {date_filter}"
# 城市手动过滤(可选,通过外部参数调度)
sys_city = sys.argv[1] if len(sys.argv) > 1 and not sys.argv[1].isdigit() else None
if pd.notna(sys_city):
sql += f" and city = '{sys_city}'"
log.info(f"正在过滤目标城市: {sys_city}")
df = es206_client.scan_by_sql_to_data_frame(sql)
if df is None or len(df) == 0:
log.info("未从 ES 中获取到可用数据。")
return
log.info(f"成功读取到 {len(df)} 条原始记录。开始解析字段,并根据唯一键 (city, region, name, s_date) 进行内存合并...")
# 聚合内存合并,以支持多个不同 root (渠道来源) 的价格在同一个 (城市-行政区-小区名-月份) 唯一键文档中合并设置
grouped = {}
for idx, row in df.iterrows():
city = row.get('city')
region = row.get('region')
name = row.get('name')
avg_price = row.get('avg_price')
root = row.get('root')
m_date = row.get('m_date')
if pd.isna(city) or pd.isna(region) or pd.isna(name) or pd.isna(avg_price) or pd.isna(root):
continue
s_date = parse_s_date(m_date)
if s_date is None:
continue
# 若指定了目标月份过滤(如仅同步指定月份的数据)
if target_month is not None and s_date != int(target_month):
continue
field_name = ROOT_FIELD_MAP.get(root)
if not field_name:
continue
key = (city, region, name, s_date)
if key not in grouped:
grouped[key] = {}
# 多个相同 root 冲突时,此处保留最新或较高价格(以整型存储)
price_val = int(float(avg_price))
current_val = grouped[key].get(field_name)
if current_val is None or price_val > current_val:
grouped[key][field_name] = price_val
log.info(f"内存聚合清洗完毕。共获得 {len(grouped)} 组唯一索引项。开始构建批量 MongoDB UpdateOne upsert 任务...")
# 构建 Bulk Write Upsert 任务
actions = []
now = datetime.now()
for (city, region, name, s_date), fields_dict in grouped.items():
# 需要 $set 写入的所有第三方价格字段及最近修改时间
update_doc = {**fields_dict}
update_doc["m_date"] = now # ModificationDate 对应实体属性
query = {
"city": city,
"region": region,
"name": name,
"s_date": s_date
}
# Upsert 动作
action = UpdateOne(
query,
{
"$set": update_doc,
"$setOnInsert": {
"category": "district", # 默认对应小区类别 district
"date": now # 对应首建/大日期 longDate
}
},
upsert=True
)
actions.append(action)
total_actions = len(actions)
if total_actions == 0:
log.info("过滤处理后无数据需写入 MongoDB。")
return
log.info(f"开始向 192.168.0.85 数据库的 average_statistics 集合中分批写入 {total_actions} 条大批处理记录...")
# 5000条一笔批量执行写入,防止网络长连接超时或内存剧增
batch_size = 5000
for i in range(0, total_actions, batch_size):
batch_actions = actions[i:i + batch_size]
result = average_statistics_conn.bulk_write(batch_actions)
log.info(f"批次 {i // batch_size + 1} 写入情况: "
f"Matched={result.matched_count}, "
f"Upserted={result.upserted_count}, "
f"Modified={result.modified_count}")
log.info("数据同步流任务全部成功结束!")
if __name__ == '__main__':
# 支持传入参数如 `python sync_es_price_to_mongo.py 202401` 进行指定月份跑数
month_arg = None
if len(sys.argv) > 1:
arg = sys.argv[1]
if arg.isdigit() and len(arg) == 6:
month_arg = int(arg)
log.info(f"读取到指定月份运行参数: {month_arg}")
sync_data(target_month=month_arg)
+5 -6
View File
@@ -4,12 +4,12 @@
import logging
import pandas as pd
import sys
from goojia_utils import ElasticSearchClient, commutils, dateutils, ClickHouseClient
from goojia_utils import ElasticSearchClient, commutils, dateutils, MAIN_MONGO_CLIENT
from elasticsearch import helpers
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
tmp_ck = ClickHouseClient(database='tmp')
community_summary_conn = MAIN_MONGO_CLIENT.get_collection('tmp', 'community_summary')
index_name = "community_summary"
path = "/home/airflow/data/community"
@@ -28,11 +28,10 @@ def sync(city=None):
cities = None
tmp_sql = f"select * from community_summary "
query = {}
if pd.notna(city):
cities = [city]
tmp_sql += f" where city='{city}'"
seaweed_df = tmp_ck.read_sql_query(tmp_sql)
query['city'] = city
seaweed_df = pd.DataFrame(list(community_summary_conn.find(query)))
deal_price_df = commutils.get_deal_avg_price(cities, 3)
if len(deal_price_df) <= 0:
+5 -5
View File
@@ -3,7 +3,7 @@
"""
import logging
import pandas as pd
from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils, ClickHouseClient
from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils, MAIN_MONGO_CLIENT
from elasticsearch import helpers
from concurrent.futures import ThreadPoolExecutor
from threading import Lock
@@ -11,7 +11,7 @@ import sys
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
tmp_ck = ClickHouseClient(database='tmp')
community_summary_conn = MAIN_MONGO_CLIENT.get_collection('tmp', 'community_summary')
index_name = "community_summary"
path = "/home/airflow/data/community"
@@ -72,10 +72,10 @@ def deal(actions, item):
def sync(city=None):
log.info(f"开始同步交通至206 {index_name}")
tmp_sql = f"select * from community_summary "
query = {}
if pd.notna(city):
tmp_sql += f" where city='{city}'"
seaweed_df = tmp_ck.read_sql_query(tmp_sql)
query['city'] = city
seaweed_df = pd.DataFrame(list(community_summary_conn.find(query)))
actions = list()
with ThreadPoolExecutor(max_workers=30) as pool:
@@ -0,0 +1,17 @@
from airflow.operators.bash import BashOperator
import pendulum
from airflow import DAG
with DAG(
dag_id="sync_es_price_to_mongo_dag",
# 每月3日凌晨2点整执行(在大部分上游爬取和ES清洗完毕后进行数据同步归档)
schedule="0 2 3 * *",
start_date=pendulum.datetime(2026, 8, 16, tz="Asia/Shanghai"),
catchup=False,
description="每月定时把 ES 206 上的 community 均价数据同步到 85 Mongo 的 average_statistics 表"
) as dag:
sync_es_price_to_mongo = BashOperator(
task_id='sync_es_price_to_mongo',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/sync_es_price_to_mongo.py {{dag_run.conf["month"] if "month" in dag_run.conf else ""}}',
)