feat: PubMed搜索合规审计 Phase 5-7 全部修复
P1-2: [SB][STAT][UID][DEP][MESH]字段支持 P1-4: [MH:noexp]解析+抑制树展开 P3: 前端date_to/P3-1、CJK高亮/P3-3、extractPlainText/P3-4、precision_mode移除/P3-6、loadMore sort/P3-5 P4-1: 词数上限50→100 P4-2: retracted参数清理
This commit is contained in:
@@ -89,8 +89,8 @@ class AdvancedSearchRequest(BaseModel):
|
||||
@field_validator('query')
|
||||
@classmethod
|
||||
def check_query_complexity(cls, v: str) -> str:
|
||||
if len(v.split()) > 50:
|
||||
raise ValueError('查询词过多(最多 50 个词),请简化搜索条件')
|
||||
if len(v.split()) > 100:
|
||||
raise ValueError('查询词过多(最多 100 个词),请简化搜索条件')
|
||||
return v
|
||||
|
||||
|
||||
|
||||
@@ -23,7 +23,7 @@ from dataclasses import dataclass, field
|
||||
from enum import Enum, auto
|
||||
|
||||
# ─── 查询复杂度限制 ───
|
||||
MAX_TERMS = 50 # 与 API 层 check_query_complexity 的 50 词限制对齐
|
||||
MAX_TERMS = 100 # P4-1: 放宽到 100 词(原 50 词)
|
||||
MAX_PAREN_DEPTH = 10 # 括号嵌套最深层数
|
||||
|
||||
|
||||
@@ -42,6 +42,7 @@ _FIELD_TAG_MAP: dict[str, str] = {
|
||||
"LAU": "author",
|
||||
"TW": "all",
|
||||
"OT": "all",
|
||||
"MESH": "MH", # P1-2: [MESH] 是 [MH] 的别名
|
||||
# 新增标量字段
|
||||
"LA": "language",
|
||||
"VI": "volume",
|
||||
@@ -55,23 +56,30 @@ _SPECIAL_FIELDS = {
|
||||
"MH", "MAJR", "PT", "DP", "PMID", "DOI",
|
||||
# 日期字段
|
||||
"EDAT", "CRDT", "MHDA", "LR", "DCOM",
|
||||
# P1-2: 电子出版日期
|
||||
"DEP",
|
||||
# JSONB 字段
|
||||
"GR", "SH", "RN", "NM", "SI", "PA",
|
||||
# 待补充抽取的字段(标记为特殊以便未来实现)
|
||||
"AUID", "COIS", "ED", "IR", "PS", "PUBN", "TT",
|
||||
# P1-2: 新字段
|
||||
"SB", "STAT", "UID",
|
||||
}
|
||||
|
||||
# 支持日期范围语法的字段
|
||||
_DATE_RANGE_FIELDS = {"DP", "EDAT", "CRDT", "MHDA", "LR", "DCOM"}
|
||||
_DATE_RANGE_FIELDS = {"DP", "EDAT", "CRDT", "MHDA", "LR", "DCOM", "DEP"}
|
||||
|
||||
# 所有合法字段标签(PubMed 全量字段)
|
||||
# P1-1: 移除了 BOOK/FILTER/ISBN(未实现,降级为 plain text 不如报错透明)
|
||||
_ALL_FIELD_TAGS = {
|
||||
"AB", "AD", "AU", "CN", "FAU", "AUID", "LAU", "COIS",
|
||||
"DCOM", "CRDT", "EDAT", "MHDA", "LR", "DP", "DOI",
|
||||
"DEP", # P1-2: Date of Electronic Publication
|
||||
"RN", "ED", "GR", "IR", "IP",
|
||||
"TA", "JT", "LA", "LID", "MAJR", "SH", "MH", "MH:NOEXP", "OT", "PG",
|
||||
"PA", "PT", "PMID", "PUBN", "SI", "PS", "NM", "TW",
|
||||
"SB", "STAT", "UID", # P1-2: Subset, Status, UID
|
||||
"MESH", # P1-2: [MH] 别名
|
||||
"TI", "TIAB", "TT", "VI",
|
||||
}
|
||||
|
||||
@@ -187,6 +195,8 @@ class ParsedPubmedQuery:
|
||||
lr_to: str | None = None
|
||||
dcom_from: str | None = None # [DCOM]
|
||||
dcom_to: str | None = None
|
||||
dep_from: str | None = None # P1-2: [DEP]
|
||||
dep_to: str | None = None
|
||||
# 特殊字段(存储 Term 以保留 is_not 标志)
|
||||
mesh_terms: list[Term] = field(default_factory=list) # [MH] (was list[str])
|
||||
majr_terms: list[Term] = field(default_factory=list) # [MAJR] (was list[str])
|
||||
@@ -207,6 +217,9 @@ class ParsedPubmedQuery:
|
||||
auid_terms: list[Term] = field(default_factory=list) # [AUID]
|
||||
cois_terms: list[Term] = field(default_factory=list) # [COIS]
|
||||
tt_terms: list[Term] = field(default_factory=list) # [TT]
|
||||
sb_terms: list[Term] = field(default_factory=list) # P1-2: [SB] Subset
|
||||
stat_terms: list[Term] = field(default_factory=list) # P1-2: [STAT] Status
|
||||
uid_terms: list[Term] = field(default_factory=list) # P1-2: [UID] PMID/DOI 统一
|
||||
plain_terms: list[Term] = field(default_factory=list) # no field tag
|
||||
boolean_operator: str = "and" # "and" | "or" | "mixed"
|
||||
has_not: bool = False # contains NOT
|
||||
@@ -349,6 +362,8 @@ class PubmedQueryParser:
|
||||
pass
|
||||
elif term.field == "__RANGE_DCOM__":
|
||||
pass
|
||||
elif term.field == "__RANGE_DEP__":
|
||||
pass
|
||||
elif term.field == "ED":
|
||||
result.ed_terms.append(term)
|
||||
elif term.field == "IR":
|
||||
@@ -363,6 +378,12 @@ class PubmedQueryParser:
|
||||
result.cois_terms.append(term)
|
||||
elif term.field == "TT":
|
||||
result.tt_terms.append(term)
|
||||
elif term.field == "SB":
|
||||
result.sb_terms.append(term)
|
||||
elif term.field == "STAT":
|
||||
result.stat_terms.append(term)
|
||||
elif term.field == "UID":
|
||||
result.uid_terms.append(term)
|
||||
else:
|
||||
result.plain_terms.append(term)
|
||||
|
||||
@@ -494,6 +515,7 @@ class PubmedQueryParser:
|
||||
"MHDA": ("mhda_from", "mhda_to", None, None, "__RANGE_MHDA__"),
|
||||
"LR": ("lr_from", "lr_to", None, None, "__RANGE_LR__"),
|
||||
"DCOM": ("dcom_from", "dcom_to", None, None, "__RANGE_DCOM__"),
|
||||
"DEP": ("dep_from", "dep_to", None, None, "__RANGE_DEP__"), # P1-2
|
||||
}
|
||||
date_attr, date_attr_to, yr_from_attr, yr_to_attr, marker_field = attr_map[field]
|
||||
# 反向范围自动交换(如 2026:2024[DP] → 2024:2026[DP])
|
||||
|
||||
@@ -97,7 +97,7 @@ class AdvancedSearchEngine:
|
||||
journal_tiers: list[str] | None = None,
|
||||
pub_types: list[str] | None = None,
|
||||
tag_ids: list[str] | None = None,
|
||||
retracted: str = "", # "yes", "no", "only"
|
||||
retracted: str = "", # "no", "only"
|
||||
negative_result: str = "", # "yes", "no", "only"
|
||||
is_oa: bool | None = None, # True = 仅开放获取
|
||||
language: str | None = None, # 语言代码(en/zh/fr 等,向后兼容)
|
||||
@@ -820,6 +820,42 @@ class AdvancedSearchEngine:
|
||||
neg_conds = [GlobalLiterature.vernacular_title.ilike(f"%{_escape_ilike(t.text)}%") for t in neg]
|
||||
term_conditions.append(not_(or_(*neg_conds)))
|
||||
|
||||
# P1-2: [SB] Subset → nlm_subsets(需 journal JOIN 子查询)
|
||||
if pp.sb_terms:
|
||||
pos = [t for t in pp.sb_terms if not t.is_not]
|
||||
neg = [t for t in pp.sb_terms if t.is_not]
|
||||
for subl, is_neg in [(pos, False), (neg, True)]:
|
||||
for t in subl:
|
||||
subq = select(GlobalJournal.issn).where(
|
||||
GlobalJournal.nlm_subsets.overlap([t.text.upper()])
|
||||
)
|
||||
cond = GlobalLiterature.journal_issn.in_(subq)
|
||||
term_conditions.append(not_(cond) if is_neg else cond)
|
||||
|
||||
# P1-2: [STAT] Status → citation_status
|
||||
if pp.stat_terms:
|
||||
pos = [t for t in pp.stat_terms if not t.is_not]
|
||||
neg = [t for t in pp.stat_terms if t.is_not]
|
||||
if pos:
|
||||
term_conditions.append(or_(*[
|
||||
GlobalLiterature.citation_status == t.text.lower()
|
||||
for t in pos
|
||||
]))
|
||||
if neg:
|
||||
neg_conds = [GlobalLiterature.citation_status == t.text.lower() for t in neg]
|
||||
term_conditions.append(not_(or_(*neg_conds)))
|
||||
|
||||
# P1-2: [UID] → PMID 优先,兜底 DOI
|
||||
if pp.uid_terms:
|
||||
for t in pp.uid_terms:
|
||||
cond = None
|
||||
try:
|
||||
cond = GlobalLiterature.pmid == int(t.text)
|
||||
except ValueError:
|
||||
cond = GlobalLiterature.doi.ilike(f"%{_escape_ilike(t.text)}%")
|
||||
if cond is not None:
|
||||
term_conditions.append(not_(cond) if t.is_not else cond)
|
||||
|
||||
# 处理括号分组的词(保留 OR/AND 嵌套结构,P2-2)
|
||||
if pp.groups:
|
||||
for idx, group in enumerate(pp.groups):
|
||||
@@ -881,13 +917,14 @@ class AdvancedSearchEngine:
|
||||
cond = and_(*dp_conds) if len(dp_conds) > 1 else dp_conds[0]
|
||||
conditions.append(not_(cond) if dp_negated else cond)
|
||||
|
||||
# 6b. [EDAT] [CRDT] [MHDA] [LR] [DCOM] → 日期字段范围
|
||||
# 6b. [EDAT] [CRDT] [MHDA] [LR] [DCOM] [DEP] → 日期字段范围
|
||||
DATE_FIELD_COLS = {
|
||||
"edat": (GlobalLiterature.entrez_date, "EDAT"),
|
||||
"crdt": (GlobalLiterature.create_date, "CRDT"),
|
||||
"mhda": (GlobalLiterature.meshed_date, "MHDA"),
|
||||
"lr": (GlobalLiterature.pubmed_revised, "LR"),
|
||||
"dcom": (GlobalLiterature.date_completed, "DCOM"),
|
||||
"dep": (GlobalLiterature.pub_date, "DEP"), # P1-2: [DEP] → pub_date
|
||||
}
|
||||
for prefix, (col, field_tag) in DATE_FIELD_COLS.items():
|
||||
_from = getattr(pp, f"{prefix}_from", None)
|
||||
@@ -942,7 +979,7 @@ class AdvancedSearchEngine:
|
||||
|
||||
# 特殊字段 — 与 _pubmed_conditions 中 top-level dispatch 一致
|
||||
if field == "MH":
|
||||
return await AdvancedSearchEngine._expand_mesh_tag_ids(db, [term.text], major_only=False)
|
||||
return await AdvancedSearchEngine._expand_mesh_tag_ids(db, [term.text], major_only=False, noexp=term._noexp)
|
||||
if field == "MAJR":
|
||||
return await AdvancedSearchEngine._expand_mesh_tag_ids(db, [term.text], major_only=True)
|
||||
if field == "PT":
|
||||
@@ -980,6 +1017,18 @@ class AdvancedSearchEngine:
|
||||
return GlobalLiterature.cois_statement.ilike(f"%{_escape_ilike(term.text)}%")
|
||||
if field == "TT":
|
||||
return GlobalLiterature.vernacular_title.ilike(f"%{_escape_ilike(term.text)}%")
|
||||
if field == "SB":
|
||||
subq = select(GlobalJournal.issn).where(
|
||||
GlobalJournal.nlm_subsets.overlap([term.text.upper()])
|
||||
)
|
||||
return GlobalLiterature.journal_issn.in_(subq)
|
||||
if field == "STAT":
|
||||
return GlobalLiterature.citation_status == term.text.lower()
|
||||
if field == "UID":
|
||||
try:
|
||||
return GlobalLiterature.pmid == int(term.text)
|
||||
except ValueError:
|
||||
return GlobalLiterature.doi.ilike(f"%{_escape_ilike(term.text)}%")
|
||||
|
||||
# 回退
|
||||
return AdvancedSearchEngine._field_condition("all", term.text, term.exact)
|
||||
@@ -1051,6 +1100,7 @@ class AdvancedSearchEngine:
|
||||
db: AsyncSession,
|
||||
mesh_names: list[str],
|
||||
major_only: bool = False,
|
||||
noexp: bool = False, # P1-4: [MH:noexp] 抑制树展开
|
||||
) -> object | None:
|
||||
"""用 GlobalTagTreeNumber 展开 [MH]/[MAJR]
|
||||
|
||||
@@ -1058,6 +1108,8 @@ class AdvancedSearchEngine:
|
||||
2. 用 tree_number 前缀展开子节点(C04.588 → 所有子树号)
|
||||
3. [MAJR] 额外 AND is_major=True
|
||||
返回 SQLAlchemy condition 或 None(无匹配时)。
|
||||
|
||||
noexp=True 时跳过第 2 步(树展开),只搜精确词。
|
||||
"""
|
||||
import uuid as _uuid
|
||||
mesh_tag_ids: set[_uuid.UUID] = set()
|
||||
@@ -1091,7 +1143,8 @@ class AdvancedSearchEngine:
|
||||
if not mesh_tag_ids:
|
||||
return None
|
||||
|
||||
# tree_number 前缀展开:取匹配 tag 的所有 tree_number,查子节点
|
||||
# tree_number 前缀展开:取匹配 tag 的所有 tree_number,查子节点([MH:noexp] 时跳过)
|
||||
if not noexp:
|
||||
tns = (await db.execute(
|
||||
select(GlobalTagTreeNumber.tree_number).where(
|
||||
GlobalTagTreeNumber.tag_id.in_(list(mesh_tag_ids))
|
||||
|
||||
@@ -301,9 +301,10 @@ class TestAllFrontendQueryFormats:
|
||||
|
||||
def test_field_map_matches_search_engine(self):
|
||||
"""Verify field tag map covers all search engine field types."""
|
||||
from app.services.pubmed_query_parser import _FIELD_TAG_MAP
|
||||
from app.services.pubmed_query_parser import _FIELD_TAG_MAP, _SPECIAL_FIELDS
|
||||
supported_fields = {"title", "abstract", "all", "author", "journal", "affiliation",
|
||||
"language", "volume", "issue", "pages", "lid"}
|
||||
supported_fields.update(_SPECIAL_FIELDS) # 特殊字段由 _single_term_condition 处理
|
||||
for fld in set(_FIELD_TAG_MAP.values()):
|
||||
assert fld in supported_fields, f"Mapped field '{fld}' not supported"
|
||||
|
||||
|
||||
@@ -111,8 +111,11 @@ const nctId = computed(() => {
|
||||
// 从查询中提取纯文本词(去掉 PubMed 字段标签如 [TI]、[AB] 等)
|
||||
function extractPlainText(q: string): string {
|
||||
return q
|
||||
.replace(/\[[\w-]+\]/g, '')
|
||||
.replace(/"?\b(AND|OR|NOT)\b"?/gi, '')
|
||||
.replace(/\[[\w-]+\]/g, '') // [field tags]
|
||||
.replace(/"?\b(AND|OR|NOT)\b"?/gi, '')// boolean operators
|
||||
.replace(/[()]/g, '') // P3-4: 去掉括号
|
||||
.replace(/#\d+/g, '') // P3-4: 去掉 #N 引用标记
|
||||
.replace(/\*/g, '') // P3-4: 去掉通配符
|
||||
.replace(/"/g, '')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim()
|
||||
@@ -127,8 +130,9 @@ const highlightedTitle = computed(() => {
|
||||
const terms = plain.split(/\s+/).filter(t => t.length > 0)
|
||||
if (terms.length === 0) return ''
|
||||
const escaped = terms.map(t => t.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'))
|
||||
// 单字符词添加 \b 词边界避免过匹配("A" 不匹配每个字母 "a")
|
||||
const pattern = escaped.map(e => e.length === 1 ? `\\b${e}\\b` : e).join('|')
|
||||
// P3-3: 单字符词添加 \b 词边界,但 CJK 字符跳过(\b 对 CJK 无效)
|
||||
const isCJK = (c: string) => /[一-鿿㐀-䶿豈-]/.test(c)
|
||||
const pattern = escaped.map(e => e.length === 1 && !isCJK(e) ? `\\b${e}\\b` : e).join('|')
|
||||
const parts = title.split(new RegExp(`(${pattern})`, 'gi'))
|
||||
return parts.map((part: string) =>
|
||||
terms.some(t => part.toLowerCase() === t.toLowerCase())
|
||||
|
||||
@@ -277,7 +277,7 @@ const { page, total, goToPage } = usePagination({
|
||||
query: query.value, page: p, page_size: pageSize.value, sort: sort.value,
|
||||
}
|
||||
if (field.value !== 'all') body.field = field.value
|
||||
if (precisionMode.value !== 'majr') body.precision_mode = precisionMode.value
|
||||
// P3-6: precision_mode 不再发送(后端已忽略)
|
||||
// ── Keyset 游标分页(sort=date 时跳过 COUNT,纯翻页模式) ──
|
||||
const useKeyset = sort.value === 'date'
|
||||
if (useKeyset) {
|
||||
@@ -371,10 +371,11 @@ function restoreFromQuery() {
|
||||
datePreset.value = String(route.query.date_preset)
|
||||
} else if (route.query.date_from) {
|
||||
datePreset.value = null
|
||||
// 尝试解析 date_from/date_to 到 yearFromStr/yearToStr
|
||||
const df = String(route.query.date_from)
|
||||
if (df.length >= 4) yearFromStr.value = df.slice(0, 4)
|
||||
} else if (route.query.date_to) {
|
||||
}
|
||||
if (route.query.date_to) {
|
||||
datePreset.value = null
|
||||
const dt = String(route.query.date_to)
|
||||
if (dt.length >= 4) yearToStr.value = dt.slice(0, 4)
|
||||
}
|
||||
|
||||
@@ -214,7 +214,7 @@ function doLocalSearch() {
|
||||
if (searchParams.value.date_from) query.date_from = searchParams.value.date_from
|
||||
if (searchParams.value.date_to) query.date_to = searchParams.value.date_to
|
||||
if (searchParams.value.sort && searchParams.value.sort !== 'date') query.sort = searchParams.value.sort
|
||||
if (searchParams.value.precision_mode && searchParams.value.precision_mode !== 'majr') query.precision = searchParams.value.precision_mode
|
||||
// P3-6: precision_mode 不再发送
|
||||
if (searchParams.value.retracted) query.retracted = searchParams.value.retracted
|
||||
if (searchParams.value.negative_result) query.negative = searchParams.value.negative_result
|
||||
if (selectedTagIds.value.length > 0) query.tag = selectedTagIds.value.join(',')
|
||||
@@ -319,7 +319,7 @@ async function loadMore() {
|
||||
if (loadingMore.value || !hasMore.value) return
|
||||
loadingMore.value = true
|
||||
try {
|
||||
const body: Record<string, any> = { page_size: searchParams.value.page_size, sort: 'date' }
|
||||
const body: Record<string, any> = { page_size: searchParams.value.page_size, sort: searchParams.value.sort || 'date' }
|
||||
if (searchParams.value.query) body.query = searchParams.value.query
|
||||
if (searchParams.value.field !== 'all') body.field = searchParams.value.field
|
||||
if (searchParams.value.tag_ids.length) body.tag_ids = searchParams.value.tag_ids
|
||||
@@ -327,7 +327,7 @@ async function loadMore() {
|
||||
if (searchParams.value.date_to) body.date_to = searchParams.value.date_to
|
||||
if (searchParams.value.retracted) body.retracted = searchParams.value.retracted
|
||||
if (searchParams.value.negative_result) body.negative_result = searchParams.value.negative_result
|
||||
if (searchParams.value.precision_mode) body.precision_mode = searchParams.value.precision_mode
|
||||
// P3-6: precision_mode 不再发送
|
||||
if (cursorDate.value && cursorId.value) {
|
||||
body.cursor_date = cursorDate.value
|
||||
body.cursor_id = cursorId.value
|
||||
|
||||
Reference in New Issue
Block a user