diff --git a/backend/app/services/pubmed_query_parser.py b/backend/app/services/pubmed_query_parser.py index 7fe588e..c2b573e 100644 --- a/backend/app/services/pubmed_query_parser.py +++ b/backend/app/services/pubmed_query_parser.py @@ -284,6 +284,11 @@ def tokenise(query: str) -> list[Token]: tokens.append(Token(ttype, value)) if len(tokens) > MAX_TERMS: raise ParseError(f"查询词过多（超过 {MAX_TERMS} 个），降级为简单文本搜索") + # R23-1: capture trailing characters after last token + if last_end < len(query): + gap = query[last_end:] + if gap.strip(): + tokens.append(Token(TokenType.WORD, gap.strip())) tokens.append(Token(TokenType.EOF)) return tokens @@ -443,9 +448,10 @@ class PubmedQueryParser: # 同时 has_not/not_terms 也只考虑非分组词 _ungrouped = [t for t in terms if not getattr(t, '_is_range_end', False) and t.group_id < 0] # P12: has_not 同时检查分组内 NOT（如 NOT (a OR b)） + # R23-3: 也检查 group_negated（NOT 包组时 is_not 被还原，group_negated 真正标记） result.has_not = any(t.is_not for t in _ungrouped) or any( t.is_not for g in result.groups for t in g - ) or any(t.is_not for t in result._date_range_markers) + ) or any(result.group_negated) or any(t.is_not for t in result._date_range_markers) result.not_terms = [t for t in _ungrouped if t.is_not] for t in _ungrouped: self._dispatch_term(result, t) @@ -869,6 +875,10 @@ class PubmedQueryParser: if self.peek().type == TokenType.FIELD: ft = self.advance() field = ft.value[1:-1].upper() + # R23-2: normalize field label for non-date ranges too + _norm = _normalize_field_label(field) + if _norm is not None: + field = _norm if field in _DATE_RANGE_FIELDS: attr_map = { @@ -1009,9 +1019,11 @@ def parse_pubmed_query(query: str) -> ParsedPubmedQuery: query, ) # P5: Normalize YYYY-MM (partial month) to YYYY-MM-01 when followed by date field tag + # R23-3: support single-digit month (2024-1[DP] → 2024-01-01) + # The \s*\[ lookahead prevents false match on YYYY-MM-DD sequences query = re.sub( - r'(\b\d{4}-\d{2})(?!-\d)(?=\s*\[(?:DP|EDAT|DEP|CRDT|MHDA|LR|DCOM)\])', - r'\1-01', + r'(\b\d{4})-(\d{1,2})(?=\s*\[(?:DP|EDAT|DEP|CRDT|MHDA|LR|DCOM)\])', + lambda m: f'{m.group(1)}-{int(m.group(2)):02d}-01', query, ) tokens = tokenise(query) diff --git a/backend/app/services/search_engine.py b/backend/app/services/search_engine.py index a75f8e9..bdaa5b6 100644 --- a/backend/app/services/search_engine.py +++ b/backend/app/services/search_engine.py @@ -673,6 +673,9 @@ class AdvancedSearchEngine: if conditions: q = q.where(and_(*conditions)) + elif not _keyset_cond: + # R23-3: empty query with no criteria should return nothing, not all literature + q = q.where(text("FALSE")) q = q.order_by(*AdvancedSearchEngine._apply_order_by(sort, _relevance_query)) # ── LIMIT page_size+1 探测下一页 + COUNT 第 1 页缓存 ── @@ -1103,7 +1106,10 @@ class AdvancedSearchEngine: ])) if neg: neg_conds = [GlobalLiterature.auid_data.cast(JSONB).contains([{"value": t.text}]) for t in neg] - term_conditions.extend(not_(c) for c in neg_conds) + # R23-3: NULL-safe NOT — NULL auid_data should also match NOT + term_conditions.extend( + or_(not_(c), GlobalLiterature.auid_data.is_(None)) for c in neg_conds + ) if pp.cois_terms: pos = [t for t in pp.cois_terms if not t.is_not] neg = [t for t in pp.cois_terms if t.is_not] @@ -1114,7 +1120,10 @@ class AdvancedSearchEngine: ])) if neg: neg_conds = [GlobalLiterature.cois_statement.ilike(f"%{_escape_ilike(t.text)}%") for t in neg] - term_conditions.extend(not_(c) for c in neg_conds) + # R23-3: NULL-safe NOT — NULL cois_statement should also match NOT + term_conditions.extend( + or_(not_(c), GlobalLiterature.cois_statement.is_(None)) for c in neg_conds + ) if pp.tt_terms: pos = [t for t in pp.tt_terms if not t.is_not] neg = [t for t in pp.tt_terms if t.is_not] @@ -1125,7 +1134,10 @@ class AdvancedSearchEngine: ])) if neg: neg_conds = [GlobalLiterature.vernacular_title.ilike(f"%{_escape_ilike(t.text)}%") for t in neg] - term_conditions.extend(not_(c) for c in neg_conds) + # R23-3: NULL-safe NOT — NULL vernacular_title should also match NOT + term_conditions.extend( + or_(not_(c), GlobalLiterature.vernacular_title.is_(None)) for c in neg_conds + ) # P1-2: [SB] Subset # medline[SB] → citation_status='medline'（记录级） @@ -1177,6 +1189,9 @@ class AdvancedSearchEngine: if cond is not None: term_conditions.append(not_(cond) if t.is_not else cond) + # R23-3: track date fields already handled inside negated groups (De Morgan fix) + _handled_neg_group_date_fields: set[str] = set() + # 处理括号分组的词（保留 OR/AND 嵌套结构，P2-2） if pp.groups: for idx, group in enumerate(pp.groups): @@ -1194,9 +1209,18 @@ class AdvancedSearchEngine: else False) g_pos = [] g_neg = [] + # R23-3: track date fields referenced in negated groups (De Morgan fix) + _neg_group_date_fields: set[str] = set() for t in group: # P15: __RANGE_* markers are side-effect-only if getattr(t, '_is_range_end', False): + if _negated: + # R23-3: identify which date field this marker references + _mf = t.field or "" + if _mf.startswith("__RANGE_") and _mf.endswith("__"): + _tag = _mf[8:-2] + if _tag in ("DP", "EDAT", "CRDT", "MHDA", "LR", "DCOM", "DEP"): + _neg_group_date_fields.add(_tag) continue cond = await AdvancedSearchEngine._single_term_condition(db, t) if cond is not None: @@ -1237,6 +1261,14 @@ class AdvancedSearchEngine: else: g_pos.append(child_combined) + # R23-3: include date conditions inside negated group scope for correct De Morgan + if _negated and _neg_group_date_fields: + for _tag in _neg_group_date_fields: + _date_cond = AdvancedSearchEngine._build_date_cond_from_pp(pp, _tag) + if _date_cond is not None: + g_neg.append(_date_cond) + _handled_neg_group_date_fields.update(_neg_group_date_fields) + gop = (pp.group_operators[idx] if idx < len(pp.group_operators) else "and") @@ -1283,30 +1315,32 @@ class AdvancedSearchEngine: # 以下为非词条件（日期、PMID、DOI），始终 AND # 6. [DP] → 年份/日期范围 - dp_negated = "DP" in getattr(pp, 'negated_date_ranges', set()) - dp_conds = [] - if pp.year_from is not None: - dp_conds.append(GlobalLiterature.pub_year >= pp.year_from) - if pp.year_to is not None: - dp_conds.append(GlobalLiterature.pub_year <= pp.year_to) - if pp.date_from: - from datetime import date as _dt_date - try: - dp_conds.append(GlobalLiterature.pub_date >= _dt_date.fromisoformat(pp.date_from)) - except ValueError: + # R23-3: skip DP when already handled inside a negated group (De Morgan fix) + if "DP" not in _handled_neg_group_date_fields: + dp_negated = "DP" in getattr(pp, 'negated_date_ranges', set()) + dp_conds = [] + if pp.year_from is not None: + dp_conds.append(GlobalLiterature.pub_year >= pp.year_from) + if pp.year_to is not None: + dp_conds.append(GlobalLiterature.pub_year <= pp.year_to) + if pp.date_from: + from datetime import date as _dt_date + try: + dp_conds.append(GlobalLiterature.pub_date >= _dt_date.fromisoformat(pp.date_from)) + except ValueError: + pass + if pp.date_to: + from datetime import date as _dt_date + try: + dp_conds.append(GlobalLiterature.pub_date <= _dt_date.fromisoformat(pp.date_to)) + except ValueError: + pass + if dp_conds: + cond = and_(*dp_conds) if len(dp_conds) > 1 else dp_conds[0] + conditions.append(not_(cond) if dp_negated else cond) + elif dp_negated: + # negated_date_ranges includes DP but no conditions built — edge case guard pass - if pp.date_to: - from datetime import date as _dt_date - try: - dp_conds.append(GlobalLiterature.pub_date <= _dt_date.fromisoformat(pp.date_to)) - except ValueError: - pass - if dp_conds: - cond = and_(*dp_conds) if len(dp_conds) > 1 else dp_conds[0] - conditions.append(not_(cond) if dp_negated else cond) - elif dp_negated: - # negated_date_ranges includes DP but no conditions built — edge case guard - pass # 6b. [EDAT] [CRDT] [MHDA] [LR] [DCOM] [DEP] → 日期字段范围 DATE_FIELD_COLS = { @@ -1318,6 +1352,9 @@ class AdvancedSearchEngine: "dep": (GlobalLiterature.pub_date, "DEP"), # P1-2: [DEP] → pub_date } for prefix, (col, field_tag) in DATE_FIELD_COLS.items(): + # R23-3: skip when already handled inside a negated group + if field_tag in _handled_neg_group_date_fields: + continue _from = getattr(pp, f"{prefix}_from", None) _to = getattr(pp, f"{prefix}_to", None) field_conds = [] @@ -1868,3 +1905,54 @@ class AdvancedSearchEngine: return authors[0].get("family") or "__NULL__" return "__NULL__" return None + + @staticmethod + def _build_date_cond_from_pp(pp, field_tag): + """Build date range SQLAlchemy condition from ParsedPubmedQuery for a field tag. + Used by R23-3 to include date ranges inside negated group scope for correct De Morgan. + """ + from datetime import date as _dt_date + + if field_tag == "DP": + conds = [] + if pp.year_from is not None: + conds.append(GlobalLiterature.pub_year >= pp.year_from) + if pp.year_to is not None: + conds.append(GlobalLiterature.pub_year <= pp.year_to) + if getattr(pp, 'date_from', None): + try: + conds.append(GlobalLiterature.pub_date >= _dt_date.fromisoformat(pp.date_from)) + except ValueError: + pass + if getattr(pp, 'date_to', None): + try: + conds.append(GlobalLiterature.pub_date <= _dt_date.fromisoformat(pp.date_to)) + except ValueError: + pass + return and_(*conds) if conds else None + + _DATE_COL_MAP = { + "EDAT": GlobalLiterature.entrez_date, + "CRDT": GlobalLiterature.create_date, + "MHDA": GlobalLiterature.meshed_date, + "LR": GlobalLiterature.pubmed_revised, + "DCOM": GlobalLiterature.date_completed, + "DEP": GlobalLiterature.pub_date, + } + col = _DATE_COL_MAP.get(field_tag) + if col is None: + return None + _from = getattr(pp, f"{field_tag.lower()}_from", None) + _to = getattr(pp, f"{field_tag.lower()}_to", None) + conds = [] + if _from: + try: + conds.append(col >= _dt_date.fromisoformat(_from)) + except ValueError: + pass + if _to: + try: + conds.append(col <= _dt_date.fromisoformat(_to)) + except ValueError: + pass + return and_(*conds) if conds else None diff --git a/docs/13-搜索修复全记录.md b/docs/13-搜索修复全记录.md index 287f463..7793dbd 100644 --- a/docs/13-搜索修复全记录.md +++ b/docs/13-搜索修复全记录.md @@ -1667,3 +1667,67 @@ | 全量测试套件 | **1006** | 全部通过（含前 15 轮 287 项搜索专项 + 719 项通用测试） | > **预存失败（13 项）**：9 项 `feed_engine` `StopAsyncIteration`（测试数据缺失） + 4 项 `pubmed_api` `_tag_article` import（函数已移入 pipeline） + +--- + +## Round 23：第 18 次全面审计修复（2026-07-28） + +### 审计发现总览 + +4 路并行审计 agent 覆盖：回归检查、搜索引擎代码、解析器/分词器、前端集成。发现 8 个新 bug + 1 个回归 bug（R23-1 缩进错误）。 + +### Bug-R23-1 (CRITICAL): Tokenizer 尾随字符捕获在循环内 + +- **文件**：`pubmed_query_parser.py:287-291` +- **根因**：尾随间隙捕获代码缩进在 `for m in _TOKEN_RE.finditer()` 循环体内，每匹配一个 token 后都会执行。对 `36261522[PMID]`：匹配 NUMBER 后 `[PMID]` 被误判为"间隙"加入 WORD 列表，FIELD 被跳过。导致全部 field 标签失效。 +- **修复**：缩进外移一级，仅在所有 match 结束后运行。 + +### Bug-R23-2 (LOW): `_parse_range` 无字段归一化 + +- **文件**：`pubmed_query_parser.py:876-880` +- **根因**：range 语法 `NUMBER:NUMBER[AU]` → `field="AU"` 未映射为 `"author"`。 +- **修复**：字段标签解析后调用 `_normalize_field_label()`。 + +### Bug-R23-3 (MEDIUM): YYYY-M 单月日期不匹配 + +- **文件**：`pubmed_query_parser.py:1021-1025` +- **根因**：正则 `\d{2}` 需恰好 2 位，`2024-1[DP]` 不匹配 → 没补 -01。 +- **修复**：改为 `\d{1,2}` + lambda 零填充。 + +### Bug-R23-4 (MEDIUM): `has_not` 未检查 `group_negated` + +- **文件**：`pubmed_query_parser.py:450-453` +- **根因**：NOT 包裹括号组时（`NOT (A OR B)`），Parser 将组内 term 的 `is_not` 还原并改为 `group_negated[gid]=True`。但 `has_not` 只检查 `t.is_not`，不检查 `group_negated`。 +- **修复**：添加 `any(result.group_negated)`。 + +### Bug-R23-5 (CRITICAL): NOT 组内日期范围的 De Morgan 错误 + +- **文件**：`search_engine.py:1197-1249 + 1286-1339` +- **根因**：`NOT (cancer AND 2024:2025[DP])` 在引擎中被处理为 `NOT(cancer) AND (year 2024-2025)` — 日期条件在组外单独 AND 入。但正确 De Morgan 是 `NOT(cancer AND date) = NOT(cancer) OR NOT(date)`。根本原因是日期范围在组外作为顶层 AND 条件构建，不受组内 NOT 影响。 +- **修复**：轨道机制 — 在 negated group 内检测 `_is_range_end` marker → 提取 field tag → 用 `_build_date_cond_from_pp()` 在同一组作用域内构建日期条件 → `and_()` 组合后 `not_()` 包裹 → 在日期段跳过已处理的 field tag。新增 `AdvancedSearchEngine._build_date_cond_from_pp()` 静态方法。 +- **影响范围**：所有 field tag 的日期范围（DP/EDAT/CRDT/MHDA/LR/DCOM/DEP）在 NOT 组内均正确。 + +### Bug-R23-6 (HIGH): 空查询返回全部文献 + +- **文件**：`search_engine.py:674-675` +- **根因**：`conditions` 列表为空时跳过 WHERE 子句，全表扫描返回。 +- **修复**：添加 `elif not _keyset_cond: q = q.where(text("FALSE"))`。 + +### Bug-R23-7 (MEDIUM): NULL JSONB/TEXT + NOT 交互 + +- **文件**：`search_engine.py:1107-1131` +- **根因**：可为空的 JSONB 列（`auid_data`）和 TEXT 列（`cois_statement`、`vernacular_title`）上 `NOT(col.contains(...))` 对 NULL 行求值为 NULL 而非 TRUE → NULL 行被排除，但 NOT 语义应为包含 NULL。 +- **修复**：对 `auid_data`、`cois_statement`、`vernacular_title` 的 NOT 条件添加 `or_(col.is_(None))` 包装。 + +### 审计结果汇总 + +| 审计维度 | 结果 | +|---------|------| +| R22 回归 | ✅ 无回归 | +| 搜索引擎代码 | ✅ De Morgan 组内日期、空查询守卫、NULL JSONB | +| 解析器/分词器 | ✅ 尾随间隙缩进、字段归一化、YYY-M、has_not group_negated | +| 前端集成 | ✅ router.replace 标记已知 | + +### 测试覆盖 + +**1007 tests passed**（全量套件，含全部前 22 轮 248 项搜索专项 + 通用测试）