fix(community): filter non-positive prices when syncing ES prices to MongoDB
This commit is contained in:
@@ -109,7 +109,7 @@ def sync_data(target_month=None):
|
|||||||
date_filter = f"and m_date >= '{three_months_ago}'"
|
date_filter = f"and m_date >= '{three_months_ago}'"
|
||||||
log.info(f"增量运行,过滤近3个月内有更新的数据 (m_date >= '{three_months_ago}')")
|
log.info(f"增量运行,过滤近3个月内有更新的数据 (m_date >= '{three_months_ago}')")
|
||||||
|
|
||||||
sql = f"select city, region, name, avg_price, root, m_date from {index_name} where avg_price is not null {date_filter}"
|
sql = f"select city, region, name, avg_price, root, m_date from {index_name} where avg_price is not null and avg_price > 0 {date_filter}"
|
||||||
|
|
||||||
# 城市手动过滤(可选,通过外部参数调度)
|
# 城市手动过滤(可选,通过外部参数调度)
|
||||||
sys_city = sys.argv[1] if len(sys.argv) > 1 and not sys.argv[1].isdigit() else None
|
sys_city = sys.argv[1] if len(sys.argv) > 1 and not sys.argv[1].isdigit() else None
|
||||||
@@ -137,6 +137,15 @@ def sync_data(target_month=None):
|
|||||||
|
|
||||||
if pd.isna(city) or pd.isna(region) or pd.isna(name) or pd.isna(avg_price) or pd.isna(root):
|
if pd.isna(city) or pd.isna(region) or pd.isna(name) or pd.isna(avg_price) or pd.isna(root):
|
||||||
continue
|
continue
|
||||||
|
|
||||||
|
try:
|
||||||
|
price_val = int(float(avg_price))
|
||||||
|
except (ValueError, TypeError):
|
||||||
|
continue
|
||||||
|
|
||||||
|
# 过滤价格 <= 0 的异常/无效数据
|
||||||
|
if price_val <= 0:
|
||||||
|
continue
|
||||||
|
|
||||||
s_date = parse_s_date(m_date)
|
s_date = parse_s_date(m_date)
|
||||||
if s_date is None:
|
if s_date is None:
|
||||||
@@ -155,7 +164,6 @@ def sync_data(target_month=None):
|
|||||||
grouped[key] = {}
|
grouped[key] = {}
|
||||||
|
|
||||||
# 多个相同 root 冲突时,此处保留最新或较高价格(以整型存储)
|
# 多个相同 root 冲突时,此处保留最新或较高价格(以整型存储)
|
||||||
price_val = int(float(avg_price))
|
|
||||||
current_val = grouped[key].get(field_name)
|
current_val = grouped[key].get(field_name)
|
||||||
if current_val is None or price_val > current_val:
|
if current_val is None or price_val > current_val:
|
||||||
grouped[key][field_name] = price_val
|
grouped[key][field_name] = price_val
|
||||||
|
|||||||
Reference in New Issue
Block a user