fix: ATM/Mesh 标签子查询 IN→关联 EXISTS — 生产 cancer 搜索 15s 超时修复
根因:flat-text ATM 和 [MH] 展开返回 id IN (SELECT literature_id FROM global_literature_tags WHERE tag_id IN (...))。当该子查询与 tsvector/ILIKE 文本条件 OR 组合进 Filter 时,planner 因命中行数过大(生产 'cancer' 展开 51 tag → 47.7 万 literature_id)拒绝哈希为 initplan,改为 Materialize 反复重扫 Bitmap Heap Scan 54-75 次,主查询 15.3s 超时(本地仅 6.4 万行哈希一次 156ms)。 修复:改为关联 EXISTS(literature_id = global_literature.id AND tag_id IN), 每外行一次 PK(literature_id, tag_id) 索引探测。生产 EXPLAIN 验证: IN+OR 15260ms → EXISTS+OR 43.7ms(350×)。三处同模式一并修复: - query_expansion.expand_atm(flat-text ATM) - search_engine._expand_mesh_tag_ids([MH]/[MAJR]) - literature.search_literature 中文搜索标签注入 本地 IN/EXISTS 结果集逐行等价验证;全量 1007 tests passed。
This commit is contained in:
@@ -7,7 +7,7 @@ from datetime import datetime, timezone
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from fastapi.responses import PlainTextResponse
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import case, func, literal, or_, select, text
|
||||
from sqlalchemy import case, exists, func, literal, or_, select, text
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.permissions import get_current_user
|
||||
@@ -289,10 +289,11 @@ async def search_literature(
|
||||
).limit(100)
|
||||
)).scalars().all()
|
||||
if _tag_matches:
|
||||
_tag_lit_subq = select(GlobalLiteratureTag.literature_id).where(
|
||||
GlobalLiteratureTag.tag_id.in_(list(_tag_matches))
|
||||
_tag_exist = exists().where(
|
||||
GlobalLiteratureTag.literature_id == GlobalLiterature.id,
|
||||
GlobalLiteratureTag.tag_id.in_(list(_tag_matches)),
|
||||
)
|
||||
search_cond = or_(search_cond, GlobalLiterature.id.in_(_tag_lit_subq))
|
||||
search_cond = or_(search_cond, _tag_exist)
|
||||
# COUNT:使用 LIMIT 10001 截断,避免大结果集的精确计数
|
||||
# 结果集 ≤10000 时返回精确值,否则返回 10001+
|
||||
MAX_EXACT = 10000
|
||||
|
||||
@@ -20,7 +20,7 @@ import logging
|
||||
import re
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy import or_ as _or_, select
|
||||
from sqlalchemy import exists, or_ as _or_, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.models.literature import GlobalTag, GlobalTagTreeNumber, GlobalLiteratureTag, GlobalLiterature
|
||||
@@ -53,13 +53,11 @@ async def expand_atm(db: AsyncSession, query: str):
|
||||
if not expanded_ids:
|
||||
return None
|
||||
|
||||
# 不用 .distinct():IN (subquery) 语义本身就按值去重,显式 DISTINCT 只会让 PG
|
||||
# 先物化+排序海量 literature_id('cancer' 这类词展开出 150+ 个 tag,命中的
|
||||
# literature_id 可达数百万),再对 id IN (百万集合) 扫描。去掉后可直接走
|
||||
# ix_glt_tag 索引半连接,year_counts 全量扫描和主查询都受益。
|
||||
return GlobalLiterature.id.in_(
|
||||
select(GlobalLiteratureTag.literature_id)
|
||||
.where(GlobalLiteratureTag.tag_id.in_(expanded_ids))
|
||||
# 不用 .distinct():EXISTS 关联子查询按 literature_id 探测 PK 索引(每外行一次
|
||||
# 索引查找),不走 ix_glt_tag 全量物化。
|
||||
return exists().where(
|
||||
GlobalLiteratureTag.literature_id == GlobalLiterature.id,
|
||||
GlobalLiteratureTag.tag_id.in_(expanded_ids),
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -2190,19 +2190,20 @@ class AdvancedSearchEngine:
|
||||
return None
|
||||
|
||||
uids = list(mesh_tag_ids)
|
||||
# 不用 func.distinct:IN (subquery) 语义本身就按值去重,显式 DISTINCT 只让
|
||||
# PG 多一次物化排序;去掉后可直接走 ix_glt_tag 索引半连接(year_counts 等
|
||||
# 全量扫描场景更敏感)。
|
||||
# 不用 .distinct():EXISTS 关联子查询按 literature_id 探测 PK 索引(每外行一次
|
||||
# 索引查找),不走 ix_glt_tag 全量物化。IN (subquery) 在 OR 过滤器里会被 planner
|
||||
# 反复重扫('cancer' 展开命中的 literature_id 达数十万,Materialize 循环数十次
|
||||
# 直接 15s 超时),EXISTS 关联每次外行只做一次 PK 探测。
|
||||
if major_only:
|
||||
subq = select(GlobalLiteratureTag.literature_id).where(
|
||||
return exists().where(
|
||||
GlobalLiteratureTag.literature_id == GlobalLiterature.id,
|
||||
GlobalLiteratureTag.tag_id.in_(uids),
|
||||
GlobalLiteratureTag.is_major == True,
|
||||
)
|
||||
else:
|
||||
subq = select(GlobalLiteratureTag.literature_id).where(
|
||||
GlobalLiteratureTag.tag_id.in_(uids),
|
||||
)
|
||||
return GlobalLiterature.id.in_(subq)
|
||||
return exists().where(
|
||||
GlobalLiteratureTag.literature_id == GlobalLiterature.id,
|
||||
GlobalLiteratureTag.tag_id.in_(uids),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _best_match_order(tsq):
|
||||
|
||||
Reference in New Issue
Block a user