feat: ClickHouse 彻底下线并平替为 MongoDB tmp 库,新增 ES 206 均价至 MongoDB 85 月度同步任务
This commit is contained in:
@@ -3,7 +3,7 @@
|
||||
"""
|
||||
import logging
|
||||
import pandas as pd
|
||||
from goojia_utils import ElasticSearchClient, commutils, dateutils, areautils, ClickHouseClient
|
||||
from goojia_utils import ElasticSearchClient, commutils, dateutils, areautils, MAIN_MONGO_CLIENT
|
||||
from elasticsearch import helpers
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import sys
|
||||
@@ -13,7 +13,7 @@ mutex = Lock()
|
||||
log = logging.getLogger(__name__)
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
es89_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
|
||||
tmp_ck = ClickHouseClient(database='tmp')
|
||||
community_summary_conn = MAIN_MONGO_CLIENT.get_collection('tmp', 'community_summary')
|
||||
|
||||
index_name = "community_summary"
|
||||
m_date = dateutils.iso_format()
|
||||
@@ -113,10 +113,10 @@ def find_around_price_item(seaweed):
|
||||
|
||||
def sync(city=None):
|
||||
log.info("开始同步消费数据...")
|
||||
tmp_sql = f"select * from community_summary "
|
||||
query = {}
|
||||
if pd.notna(city):
|
||||
tmp_sql += f" where city='{city}' "
|
||||
seaweed_df = tmp_ck.read_sql_query(tmp_sql)
|
||||
query['city'] = city
|
||||
seaweed_df = pd.DataFrame(list(community_summary_conn.find(query)))
|
||||
|
||||
seaweed_list = list()
|
||||
seaweed_df.apply(lambda x: seaweed_list.append(dict(x)), axis=1)
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
"""
|
||||
import logging
|
||||
import pandas as pd
|
||||
from goojia_utils import ElasticSearchClient, commutils, dateutils, ClickHouseClient
|
||||
from goojia_utils import ElasticSearchClient, commutils, dateutils, MAIN_MONGO_CLIENT
|
||||
from elasticsearch import helpers
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from threading import Lock
|
||||
@@ -11,7 +11,7 @@ import sys
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
|
||||
tmp_ck = ClickHouseClient(database="tmp")
|
||||
community_summary_conn = MAIN_MONGO_CLIENT.get_collection('tmp', 'community_summary')
|
||||
|
||||
index_name = "community_summary"
|
||||
path = "/home/airflow/data/community"
|
||||
@@ -90,10 +90,10 @@ def deal(actions, item):
|
||||
|
||||
def sync(city=None):
|
||||
log.info(f"开始同步商圈至206 {index_name}")
|
||||
tmp_sql = f"select * from community_summary "
|
||||
query = {}
|
||||
if pd.notna(city):
|
||||
tmp_sql += f" where city='{city}'"
|
||||
seaweed_df = tmp_ck.read_sql_query(tmp_sql)
|
||||
query['city'] = city
|
||||
seaweed_df = pd.DataFrame(list(community_summary_conn.find(query)))
|
||||
|
||||
actions = list()
|
||||
with ThreadPoolExecutor(max_workers=30) as pool:
|
||||
|
||||
@@ -3,7 +3,31 @@
|
||||
"""
|
||||
import logging
|
||||
import pandas as pd
|
||||
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, ClickHouseClient
|
||||
import numpy as np
|
||||
|
||||
# 兼容性补丁:解决 goojia_utils.ClickHouseClient.to_sql_plus 在将 DataFrame 写入 ClickHouse 时,
|
||||
# 对含有 list/array 类型的列(如 alias)进行 pd.notna(item) 判定时导致 ValueError 的 bug
|
||||
_orig_notna = pd.notna
|
||||
def patched_notna(obj):
|
||||
if isinstance(obj, (list, tuple, np.ndarray, dict, set)):
|
||||
return True
|
||||
try:
|
||||
res = _orig_notna(obj)
|
||||
if hasattr(res, 'any'):
|
||||
return bool(res.any())
|
||||
elif isinstance(res, (list, np.ndarray)):
|
||||
return any(res)
|
||||
return res
|
||||
except Exception:
|
||||
return True
|
||||
|
||||
pd.notna = patched_notna
|
||||
pd.isna = lambda x: not patched_notna(x)
|
||||
import pandas
|
||||
pandas.notna = patched_notna
|
||||
pandas.isna = lambda x: not patched_notna(x)
|
||||
|
||||
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils
|
||||
from elasticsearch import helpers
|
||||
import os
|
||||
import sys
|
||||
@@ -11,7 +35,7 @@ import sys
|
||||
log = logging.getLogger(__name__)
|
||||
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
||||
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
|
||||
tmp_ck = ClickHouseClient(database='tmp')
|
||||
community_summary_conn = MAIN_MONGO_CLIENT.get_collection('tmp', 'community_summary')
|
||||
|
||||
index_name = "community_summary"
|
||||
m_date = dateutils.iso_format()
|
||||
@@ -115,11 +139,11 @@ def deal_comm(community_summary_df, seaweed_df):
|
||||
def sync(city=None):
|
||||
log.info(f"开始同步小区基本属性至206 {index_name}...")
|
||||
seaweed_query = {'status': 0, 'cat': 'district'}
|
||||
ck_sql = "ALTER TABLE tmp.community_summary DELETE WHERE 1=1 "
|
||||
mongo_delete_query = {}
|
||||
if pd.notna(city):
|
||||
seaweed_query['city'] = city
|
||||
ck_sql += " AND city='" + city + "'"
|
||||
tmp_ck.execute(ck_sql)
|
||||
mongo_delete_query['city'] = city
|
||||
community_summary_conn.delete_many(mongo_delete_query)
|
||||
|
||||
seaweed_df = pd.DataFrame(
|
||||
list(seaweed_conn.find(seaweed_query, {'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1,
|
||||
@@ -156,9 +180,16 @@ def sync(city=None):
|
||||
|
||||
# 写入csv文件,防止小区变动导致的后续过程更新报错,
|
||||
# seaweed_df.to_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8', index=False)
|
||||
# 写入中间表clickhouse,解决多worker节点问题
|
||||
tmp_ck.to_sql_plus(seaweed_df[['_id', 'city', 'region', 'name', 'lng', 'lat', 'alias', 'estate_type2', 'build_type',
|
||||
'household_count', 'block', 'estate_charge', 'finish_building_date']], 'community_summary')
|
||||
# 写入中间集合 MongoDB, 解决分布式 Worker 节点的共享问题
|
||||
insert_cols = ['_id', 'city', 'region', 'name', 'lng', 'lat', 'alias', 'estate_type2', 'build_type',
|
||||
'household_count', 'block', 'estate_charge', 'finish_building_date']
|
||||
records = seaweed_df[insert_cols].to_dict(orient='records')
|
||||
if len(records) > 0:
|
||||
log.info(f"正在写入 MongoDB community_summary 中间集合,共 {len(records)} 条记录...")
|
||||
batch_size = 5000
|
||||
for i in range(0, len(records), batch_size):
|
||||
community_summary_conn.insert_many(records[i:i + batch_size])
|
||||
log.info("MongoDB community_summary 写入完毕。")
|
||||
|
||||
actions = list()
|
||||
# 对于存在的更新
|
||||
|
||||
@@ -19,7 +19,7 @@ except Exception:
|
||||
|
||||
import logging
|
||||
import pandas as pd
|
||||
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, ClickHouseClient
|
||||
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils
|
||||
from elasticsearch import helpers
|
||||
import os
|
||||
import sys
|
||||
@@ -27,7 +27,6 @@ import sys
|
||||
log = logging.getLogger(__name__)
|
||||
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
||||
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
|
||||
tmp_ck = ClickHouseClient(database='tmp')
|
||||
|
||||
index_name = "community_summary"
|
||||
m_date = dateutils.iso_format()
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
"""
|
||||
import logging
|
||||
import pandas as pd
|
||||
from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils, ClickHouseClient
|
||||
from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils, MAIN_MONGO_CLIENT
|
||||
from elasticsearch import helpers
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from threading import Lock
|
||||
@@ -11,7 +11,7 @@ import sys
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
|
||||
tmp_ck = ClickHouseClient(database="tmp")
|
||||
community_summary_conn = MAIN_MONGO_CLIENT.get_collection('tmp', 'community_summary')
|
||||
|
||||
index_name = "community_summary"
|
||||
path = "/home/airflow/data/community"
|
||||
@@ -66,10 +66,10 @@ def deal(actions, item):
|
||||
|
||||
def sync(city=None):
|
||||
log.info(f"开始同步教育资源【幼儿园、大学】至206 {index_name}")
|
||||
tmp_sql = f"select * from community_summary "
|
||||
query = {}
|
||||
if pd.notna(city):
|
||||
tmp_sql += f" where city='{city}'"
|
||||
seaweed_df = tmp_ck.read_sql_query(tmp_sql)
|
||||
query['city'] = city
|
||||
seaweed_df = pd.DataFrame(list(community_summary_conn.find(query)))
|
||||
|
||||
actions = list()
|
||||
with ThreadPoolExecutor(max_workers=30) as pool:
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
"""
|
||||
import logging
|
||||
import pandas as pd
|
||||
from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils, ClickHouseClient
|
||||
from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils, MAIN_MONGO_CLIENT
|
||||
from elasticsearch import helpers
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from threading import Lock
|
||||
@@ -11,7 +11,7 @@ import sys
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
|
||||
tmp_ck = ClickHouseClient(database="tmp")
|
||||
community_summary_conn = MAIN_MONGO_CLIENT.get_collection('tmp', 'community_summary')
|
||||
|
||||
index_name = "community_summary"
|
||||
path = "/home/airflow/data/community"
|
||||
@@ -81,10 +81,10 @@ def deal(actions, item):
|
||||
def sync(city=None):
|
||||
log.info(f"开始同步教育资源【小学、中学】至206 {index_name}")
|
||||
|
||||
tmp_sql = f"select * from community_summary "
|
||||
query = {}
|
||||
if pd.notna(city):
|
||||
tmp_sql += f" where city='{city}'"
|
||||
seaweed_df = tmp_ck.read_sql_query(tmp_sql)
|
||||
query['city'] = city
|
||||
seaweed_df = pd.DataFrame(list(community_summary_conn.find(query)))
|
||||
|
||||
actions = list()
|
||||
with ThreadPoolExecutor(max_workers=30) as pool:
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
"""
|
||||
import logging
|
||||
import pandas as pd
|
||||
from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils, ClickHouseClient
|
||||
from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils, MAIN_MONGO_CLIENT
|
||||
from elasticsearch import helpers
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from threading import Lock
|
||||
@@ -11,7 +11,7 @@ import sys
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
|
||||
tmp_ck = ClickHouseClient(database='tmp')
|
||||
community_summary_conn = MAIN_MONGO_CLIENT.get_collection('tmp', 'community_summary')
|
||||
|
||||
index_name = "community_summary"
|
||||
path = "/home/airflow/data/community"
|
||||
@@ -78,10 +78,10 @@ def deal(actions, item):
|
||||
|
||||
def sync(city=None):
|
||||
log.info(f"开始同步生活配套至206 {index_name}")
|
||||
tmp_sql = f"select * from community_summary "
|
||||
query = {}
|
||||
if pd.notna(city):
|
||||
tmp_sql += f" where city='{city}'"
|
||||
seaweed_df = tmp_ck.read_sql_query(tmp_sql)
|
||||
query['city'] = city
|
||||
seaweed_df = pd.DataFrame(list(community_summary_conn.find(query)))
|
||||
|
||||
actions = list()
|
||||
with ThreadPoolExecutor(max_workers=30) as pool:
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
"""
|
||||
import logging
|
||||
import pandas as pd
|
||||
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, ClickHouseClient, poiutils
|
||||
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, poiutils
|
||||
from elasticsearch import helpers
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from threading import Lock
|
||||
@@ -12,7 +12,7 @@ import sys
|
||||
log = logging.getLogger(__name__)
|
||||
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
||||
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
|
||||
tmp_ck = ClickHouseClient(database='tmp')
|
||||
community_summary_conn = MAIN_MONGO_CLIENT.get_collection('tmp', 'community_summary')
|
||||
|
||||
index_name = "community_summary"
|
||||
path = "/home/airflow/data/community"
|
||||
@@ -66,10 +66,10 @@ def deal(actions, item):
|
||||
|
||||
def sync(city=None):
|
||||
log.info(f"开始同步医院至206 {index_name}")
|
||||
tmp_sql = f"select * from community_summary "
|
||||
query = {}
|
||||
if pd.notna(city):
|
||||
tmp_sql += f" where city='{city}'"
|
||||
seaweed_df = tmp_ck.read_sql_query(tmp_sql)
|
||||
query['city'] = city
|
||||
seaweed_df = pd.DataFrame(list(community_summary_conn.find(query)))
|
||||
|
||||
actions = list()
|
||||
with ThreadPoolExecutor(max_workers=30) as pool:
|
||||
|
||||
@@ -3,13 +3,13 @@
|
||||
"""
|
||||
import logging
|
||||
import pandas as pd
|
||||
from goojia_utils import ElasticSearchClient, commutils, dateutils, areautils, ClickHouseClient
|
||||
from goojia_utils import ElasticSearchClient, commutils, dateutils, areautils, MAIN_MONGO_CLIENT
|
||||
from elasticsearch import helpers
|
||||
import sys
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
|
||||
tmp_ck = ClickHouseClient(database='tmp')
|
||||
community_summary_conn = MAIN_MONGO_CLIENT.get_collection('tmp', 'community_summary')
|
||||
|
||||
index_name = "community_summary"
|
||||
path = "/home/airflow/data/community"
|
||||
@@ -42,7 +42,7 @@ def build_doc(item):
|
||||
|
||||
|
||||
def sync_by_city(city):
|
||||
seaweed_df = tmp_ck.read_sql_query(f"select _id, city, region, name from community_summary where city='{city}'")
|
||||
seaweed_df = pd.DataFrame(list(community_summary_conn.find({"city": city}, {"_id": 1, "city": 1, "region": 1, "name": 1})))
|
||||
if len(seaweed_df) == 0:
|
||||
return
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
"""
|
||||
import logging
|
||||
import pandas as pd
|
||||
from goojia_utils import ElasticSearchClient, commutils, dateutils, ClickHouseClient
|
||||
from goojia_utils import ElasticSearchClient, commutils, dateutils, MAIN_MONGO_CLIENT
|
||||
from elasticsearch import helpers
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import sys
|
||||
@@ -12,7 +12,7 @@ from threading import Lock
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
|
||||
tmp_ck = ClickHouseClient(database='tmp')
|
||||
community_summary_conn = MAIN_MONGO_CLIENT.get_collection('tmp', 'community_summary')
|
||||
|
||||
index_name = "community_summary"
|
||||
path = "/home/airflow/data/community"
|
||||
@@ -134,10 +134,10 @@ def find_around_price_item(seaweed):
|
||||
def sync(city=None):
|
||||
log.info("开始同步相对价格排名数据...")
|
||||
|
||||
tmp_sql = f"select * from community_summary "
|
||||
query = {}
|
||||
if pd.notna(city):
|
||||
tmp_sql += f" where city='{city}'"
|
||||
seaweed_df = tmp_ck.read_sql_query(tmp_sql)
|
||||
query['city'] = city
|
||||
seaweed_df = pd.DataFrame(list(community_summary_conn.find(query)))
|
||||
|
||||
seaweed_list = list()
|
||||
seaweed_df.apply(lambda x: seaweed_list.append(dict(x)), axis=1)
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
"""
|
||||
import logging
|
||||
import pandas as pd
|
||||
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, ClickHouseClient
|
||||
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils
|
||||
from elasticsearch import helpers
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import sys
|
||||
@@ -12,7 +12,7 @@ from threading import Lock
|
||||
log = logging.getLogger(__name__)
|
||||
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
||||
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
|
||||
tmp_ck = ClickHouseClient(database='tmp')
|
||||
community_summary_conn = MAIN_MONGO_CLIENT.get_collection('tmp', 'community_summary')
|
||||
|
||||
index_name = "community_summary"
|
||||
path = "/home/airflow/data/community"
|
||||
@@ -33,10 +33,10 @@ def build_doc(item):
|
||||
def sync(city=None):
|
||||
log.info(f"开始同步出售信息至206 {index_name}")
|
||||
|
||||
tmp_sql = f"select * from community_summary "
|
||||
query = {}
|
||||
if pd.notna(city):
|
||||
tmp_sql += f" where city='{city}'"
|
||||
seaweed_df = tmp_ck.read_sql_query(tmp_sql)
|
||||
query['city'] = city
|
||||
seaweed_df = pd.DataFrame(list(community_summary_conn.find(query)))
|
||||
|
||||
with ThreadPoolExecutor(30) as pool:
|
||||
def deal(index, item):
|
||||
|
||||
@@ -0,0 +1,223 @@
|
||||
import os
|
||||
import sys
|
||||
import logging
|
||||
import pandas as pd
|
||||
from datetime import datetime
|
||||
from pymongo import UpdateOne
|
||||
|
||||
# 兼容性补丁:解决 Elasticsearch 8.x/9.x 客户端在老版本 ES 集群上因 compatible-with 头部导致的 ApiError(406)
|
||||
os.environ["ELASTIC_CLIENT_APIVERSIONING"] = "0"
|
||||
try:
|
||||
import elasticsearch
|
||||
ver = getattr(elasticsearch, '__version__', None)
|
||||
if ver and not str(ver).startswith('7.'):
|
||||
original_init = elasticsearch.Elasticsearch.__init__
|
||||
def patched_init(self, *args, **kwargs):
|
||||
headers = kwargs.get('headers', {})
|
||||
headers['Accept'] = 'application/json'
|
||||
headers['Content-Type'] = 'application/json'
|
||||
kwargs['headers'] = headers
|
||||
original_init(self, *args, **kwargs)
|
||||
elasticsearch.Elasticsearch.__init__ = patched_init
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
# 初始化 ES 206 客户端与 85 上的 MongoDB 集合
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206', username='elastic', password='elasticsearch')
|
||||
average_statistics_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'average_statistics')
|
||||
|
||||
index_name = "community"
|
||||
|
||||
# 映射关系:将 ES 中的 root 主域映射为 Mongo 表中的对应二手房价格 (_esf_price) 字段
|
||||
ROOT_FIELD_MAP = {
|
||||
"anjuke.com": "anjuke_esf_price", # 安居客
|
||||
"lianjia.com": "lianjia_esf_price", # 链家
|
||||
"ke.com": "beike_esf_price", # 贝壳
|
||||
"beike.com": "beike_esf_price", # 贝壳
|
||||
"fang.com": "soufang_esf_price", # 房天下/搜房
|
||||
"soufang.com": "soufang_esf_price", # 搜房
|
||||
"5i5j.com": "5i5j_esf_price", # 我爱我家
|
||||
"centanet.com": "zhongyuan_esf_price", # 中原地产
|
||||
"qfang.com": "qfang_esf_price", # Q房网
|
||||
"leyoujia.com": "leyoujia_esf_price", # 乐有家
|
||||
"zhuge.com": "zhugezhaofang_esf_price", # 诸葛找房
|
||||
"zhugezhaofang.com": "zhugezhaofang_esf_price",
|
||||
"jiwu.com": "jiwu_esf_price", # 吉屋
|
||||
"house365.com": "365house_esf_price", # 365淘房
|
||||
"365.com": "365house_esf_price",
|
||||
"fanggugu.com": "fanggugu_esf_price", # 房估估
|
||||
"yungujia.com": "yungujia_esf_price", # 云估价
|
||||
"tuboshi.com": "tuboshi_esf_price", # 兔博士
|
||||
"xiaozijia.com": "xiaozijia_esf_price", # 小资家
|
||||
"fangdoudou.com": "fangdoudou_esf_price",# 房兜兜
|
||||
"fangtan.com": "fangtan_esf_price", # 房探
|
||||
"guoxinda.com": "gxd_esf_price", # 国信达
|
||||
"gxd.com": "gxd_esf_price",
|
||||
"xingfuli.com": "xingfuli_esf_price", # 幸福里
|
||||
"maitian.cn": "maitian_esf_price", # 麦田房产
|
||||
"maitian.com": "maitian_esf_price",
|
||||
"sina.com.cn": "sina_esf_price", # 新浪二手房
|
||||
"sina.com": "sina_esf_price",
|
||||
"cric.com": "cric_esf_price", # 克而瑞
|
||||
"ganji.com": "ganji_esf_price", # 赶集网
|
||||
}
|
||||
|
||||
def parse_s_date(m_date_str):
|
||||
"""
|
||||
将 ES 抓取时间 '2022-06-17T13:12:47.148Z' 转化为整型 s_date YYYYMM (例如 202206)
|
||||
"""
|
||||
try:
|
||||
if pd.isna(m_date_str) or not isinstance(m_date_str, str):
|
||||
return None
|
||||
return int(m_date_str[:7].replace('-', ''))
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def sync_data(target_month=None):
|
||||
"""
|
||||
数据流式拉取、转换与批量 Upsert
|
||||
"""
|
||||
log.info("开始从 ES 206 的 community 索引拉取小区价格数据...")
|
||||
|
||||
# 构造 SQL,默认仅同步近 3 个月(90 天)内更新的数据,降低传输和存储开销
|
||||
if target_month is not None:
|
||||
try:
|
||||
month_str = str(target_month)
|
||||
year = int(month_str[:4])
|
||||
month = int(month_str[4:6])
|
||||
start_date = f"{year:04d}-{month:02d}-01T00:00:00"
|
||||
if month == 12:
|
||||
end_date = f"{year + 1:04d}-01-01T00:00:00"
|
||||
else:
|
||||
end_date = f"{year:04d}-{month + 1:02d}-01T00:00:00"
|
||||
|
||||
date_filter = f"and m_date >= '{start_date}' and m_date < '{end_date}'"
|
||||
log.info(f"指定历史月份跑数,过滤条件: {start_date} 至 {end_date}")
|
||||
except Exception as e:
|
||||
log.error(f"解析指定月份失败,退回到默认的近3个月同步。Error: {e}")
|
||||
from datetime import datetime, timedelta
|
||||
three_months_ago = (datetime.now() - timedelta(days=90)).strftime('%Y-%m-%dT%H:%M:%S')
|
||||
date_filter = f"and m_date >= '{three_months_ago}'"
|
||||
else:
|
||||
from datetime import datetime, timedelta
|
||||
three_months_ago = (datetime.now() - timedelta(days=90)).strftime('%Y-%m-%dT%H:%M:%S')
|
||||
date_filter = f"and m_date >= '{three_months_ago}'"
|
||||
log.info(f"增量运行,过滤近3个月内有更新的数据 (m_date >= '{three_months_ago}')")
|
||||
|
||||
sql = f"select city, region, name, avg_price, root, m_date from {index_name} where avg_price is not null {date_filter}"
|
||||
|
||||
# 城市手动过滤(可选,通过外部参数调度)
|
||||
sys_city = sys.argv[1] if len(sys.argv) > 1 and not sys.argv[1].isdigit() else None
|
||||
if pd.notna(sys_city):
|
||||
sql += f" and city = '{sys_city}'"
|
||||
log.info(f"正在过滤目标城市: {sys_city}")
|
||||
|
||||
df = es206_client.scan_by_sql_to_data_frame(sql)
|
||||
if df is None or len(df) == 0:
|
||||
log.info("未从 ES 中获取到可用数据。")
|
||||
return
|
||||
|
||||
log.info(f"成功读取到 {len(df)} 条原始记录。开始解析字段,并根据唯一键 (city, region, name, s_date) 进行内存合并...")
|
||||
|
||||
# 聚合内存合并,以支持多个不同 root (渠道来源) 的价格在同一个 (城市-行政区-小区名-月份) 唯一键文档中合并设置
|
||||
grouped = {}
|
||||
|
||||
for idx, row in df.iterrows():
|
||||
city = row.get('city')
|
||||
region = row.get('region')
|
||||
name = row.get('name')
|
||||
avg_price = row.get('avg_price')
|
||||
root = row.get('root')
|
||||
m_date = row.get('m_date')
|
||||
|
||||
if pd.isna(city) or pd.isna(region) or pd.isna(name) or pd.isna(avg_price) or pd.isna(root):
|
||||
continue
|
||||
|
||||
s_date = parse_s_date(m_date)
|
||||
if s_date is None:
|
||||
continue
|
||||
|
||||
# 若指定了目标月份过滤(如仅同步指定月份的数据)
|
||||
if target_month is not None and s_date != int(target_month):
|
||||
continue
|
||||
|
||||
field_name = ROOT_FIELD_MAP.get(root)
|
||||
if not field_name:
|
||||
continue
|
||||
|
||||
key = (city, region, name, s_date)
|
||||
if key not in grouped:
|
||||
grouped[key] = {}
|
||||
|
||||
# 多个相同 root 冲突时,此处保留最新或较高价格(以整型存储)
|
||||
price_val = int(float(avg_price))
|
||||
current_val = grouped[key].get(field_name)
|
||||
if current_val is None or price_val > current_val:
|
||||
grouped[key][field_name] = price_val
|
||||
|
||||
log.info(f"内存聚合清洗完毕。共获得 {len(grouped)} 组唯一索引项。开始构建批量 MongoDB UpdateOne upsert 任务...")
|
||||
|
||||
# 构建 Bulk Write Upsert 任务
|
||||
actions = []
|
||||
now = datetime.now()
|
||||
|
||||
for (city, region, name, s_date), fields_dict in grouped.items():
|
||||
# 需要 $set 写入的所有第三方价格字段及最近修改时间
|
||||
update_doc = {**fields_dict}
|
||||
update_doc["m_date"] = now # ModificationDate 对应实体属性
|
||||
|
||||
query = {
|
||||
"city": city,
|
||||
"region": region,
|
||||
"name": name,
|
||||
"s_date": s_date
|
||||
}
|
||||
|
||||
# Upsert 动作
|
||||
action = UpdateOne(
|
||||
query,
|
||||
{
|
||||
"$set": update_doc,
|
||||
"$setOnInsert": {
|
||||
"category": "district", # 默认对应小区类别 district
|
||||
"date": now # 对应首建/大日期 longDate
|
||||
}
|
||||
},
|
||||
upsert=True
|
||||
)
|
||||
actions.append(action)
|
||||
|
||||
total_actions = len(actions)
|
||||
if total_actions == 0:
|
||||
log.info("过滤处理后无数据需写入 MongoDB。")
|
||||
return
|
||||
|
||||
log.info(f"开始向 192.168.0.85 数据库的 average_statistics 集合中分批写入 {total_actions} 条大批处理记录...")
|
||||
|
||||
# 5000条一笔批量执行写入,防止网络长连接超时或内存剧增
|
||||
batch_size = 5000
|
||||
for i in range(0, total_actions, batch_size):
|
||||
batch_actions = actions[i:i + batch_size]
|
||||
result = average_statistics_conn.bulk_write(batch_actions)
|
||||
log.info(f"批次 {i // batch_size + 1} 写入情况: "
|
||||
f"Matched={result.matched_count}, "
|
||||
f"Upserted={result.upserted_count}, "
|
||||
f"Modified={result.modified_count}")
|
||||
|
||||
log.info("数据同步流任务全部成功结束!")
|
||||
|
||||
if __name__ == '__main__':
|
||||
# 支持传入参数如 `python sync_es_price_to_mongo.py 202401` 进行指定月份跑数
|
||||
month_arg = None
|
||||
if len(sys.argv) > 1:
|
||||
arg = sys.argv[1]
|
||||
if arg.isdigit() and len(arg) == 6:
|
||||
month_arg = int(arg)
|
||||
log.info(f"读取到指定月份运行参数: {month_arg}")
|
||||
|
||||
sync_data(target_month=month_arg)
|
||||
@@ -4,12 +4,12 @@
|
||||
import logging
|
||||
import pandas as pd
|
||||
import sys
|
||||
from goojia_utils import ElasticSearchClient, commutils, dateutils, ClickHouseClient
|
||||
from goojia_utils import ElasticSearchClient, commutils, dateutils, MAIN_MONGO_CLIENT
|
||||
from elasticsearch import helpers
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
|
||||
tmp_ck = ClickHouseClient(database='tmp')
|
||||
community_summary_conn = MAIN_MONGO_CLIENT.get_collection('tmp', 'community_summary')
|
||||
|
||||
index_name = "community_summary"
|
||||
path = "/home/airflow/data/community"
|
||||
@@ -28,11 +28,10 @@ def sync(city=None):
|
||||
|
||||
cities = None
|
||||
|
||||
tmp_sql = f"select * from community_summary "
|
||||
query = {}
|
||||
if pd.notna(city):
|
||||
cities = [city]
|
||||
tmp_sql += f" where city='{city}'"
|
||||
seaweed_df = tmp_ck.read_sql_query(tmp_sql)
|
||||
query['city'] = city
|
||||
seaweed_df = pd.DataFrame(list(community_summary_conn.find(query)))
|
||||
|
||||
deal_price_df = commutils.get_deal_avg_price(cities, 3)
|
||||
if len(deal_price_df) <= 0:
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
"""
|
||||
import logging
|
||||
import pandas as pd
|
||||
from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils, ClickHouseClient
|
||||
from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils, MAIN_MONGO_CLIENT
|
||||
from elasticsearch import helpers
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from threading import Lock
|
||||
@@ -11,7 +11,7 @@ import sys
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
|
||||
tmp_ck = ClickHouseClient(database='tmp')
|
||||
community_summary_conn = MAIN_MONGO_CLIENT.get_collection('tmp', 'community_summary')
|
||||
|
||||
index_name = "community_summary"
|
||||
path = "/home/airflow/data/community"
|
||||
@@ -72,10 +72,10 @@ def deal(actions, item):
|
||||
|
||||
def sync(city=None):
|
||||
log.info(f"开始同步交通至206 {index_name}")
|
||||
tmp_sql = f"select * from community_summary "
|
||||
query = {}
|
||||
if pd.notna(city):
|
||||
tmp_sql += f" where city='{city}'"
|
||||
seaweed_df = tmp_ck.read_sql_query(tmp_sql)
|
||||
query['city'] = city
|
||||
seaweed_df = pd.DataFrame(list(community_summary_conn.find(query)))
|
||||
|
||||
actions = list()
|
||||
with ThreadPoolExecutor(max_workers=30) as pool:
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
from airflow.operators.bash import BashOperator
|
||||
import pendulum
|
||||
from airflow import DAG
|
||||
|
||||
with DAG(
|
||||
dag_id="sync_es_price_to_mongo_dag",
|
||||
# 每月3日凌晨2点整执行(在大部分上游爬取和ES清洗完毕后进行数据同步归档)
|
||||
schedule="0 2 3 * *",
|
||||
start_date=pendulum.datetime(2026, 8, 16, tz="Asia/Shanghai"),
|
||||
catchup=False,
|
||||
description="每月定时把 ES 206 上的 community 均价数据同步到 85 Mongo 的 average_statistics 表"
|
||||
) as dag:
|
||||
|
||||
sync_es_price_to_mongo = BashOperator(
|
||||
task_id='sync_es_price_to_mongo',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/sync_es_price_to_mongo.py {{dag_run.conf["month"] if "month" in dag_run.conf else ""}}',
|
||||
)
|
||||
Reference in New Issue
Block a user