From 275a9f65110f32bed36056247083ec69e8b26950 Mon Sep 17 00:00:00 2001 From: "34047007@qq.com" <34047007@qq.com> Date: Mon, 27 Jul 2026 11:55:03 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20=E7=AC=AC=E4=BA=94=E8=BD=AEPubMed?= =?UTF-8?q?=E6=90=9C=E7=B4=A2=E5=AE=A1=E8=AE=A1=20=E2=80=94=204=E9=A1=B9?= =?UTF-8?q?=E8=BE=B9=E7=BC=98=E6=A1=88=E4=BE=8B=E4=BF=AE=E5=A4=8D=20+=20?= =?UTF-8?q?=E7=AC=AC5=E8=BD=AE=E5=AE=A1=E8=AE=A1=E6=96=87=E6=A1=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Parser边缘案例修复: - P5-1: 尾部NOT导致IndexError降级（_parse_not_expr添加EOF守卫） - P5-2: 单数字日期格式不被识别（YYYY-M-D→YYYY-MM-DD归一化） - P5-3: is_pubmed_syntax不处理全角字符（增加NFKC归一化） - P5-4: extract_pubmed_query_for_prisma不能处理[Title/Article] 同时将在前几轮修复的F1-F8（[MH:noexp]顶层支持、NOT De Morgan、 NOT NOT递归、Custom Range日期、HomeView恢复、precision_mode清理、 is_oa注释、N+1批量优化）补充文档核对验证。 --- backend/app/api/v1/literature.py | 6 +- backend/app/services/pubmed_query_parser.py | 65 +++++++++++++++--- backend/app/services/query_expansion.py | 5 +- backend/app/services/search_engine.py | 23 +++++-- docs/13-搜索修复全记录.md | 75 ++++++++++++++++++--- 5 files changed, 146 insertions(+), 28 deletions(-) diff --git a/backend/app/api/v1/literature.py b/backend/app/api/v1/literature.py index cac6c85..0971634 100644 --- a/backend/app/api/v1/literature.py +++ b/backend/app/api/v1/literature.py @@ -16,7 +16,7 @@ from app.models.literature import GlobalLiterature, GlobalLiteratureTag, GlobalT from app.schemas.literature import FeedResponse, LiteratureCard, LiteratureDetail, cap_pub_date from app.core.cache import cache from app.services.cos_client import cached_get_full_text -from app.services.search_engine import AdvancedSearchEngine +from app.services.search_engine import AdvancedSearchEngine, _escape_ilike from app.services.tag_loader import load_tags_for_literature router = APIRouter() @@ -87,7 +87,7 @@ async def personal_feed( # content search 需要 JOIN literature if q.strip(): - like = f"%{q.strip()}%" + like = f"%{_escape_ilike(q.strip())}%" content_filter = select(UserFeed.id).join( GlobalLiterature, UserFeed.literature_id == GlobalLiterature.id ).where( @@ -260,7 +260,7 @@ async def search_literature( if len(q.split()) > 100: return {"items": [], "total": 0, "error": "查询词过多（最多 100 个词），请简化搜索条件"} offset = (page - 1) * page_size - like = f"%{q}%" + like = f"%{_escape_ilike(q)}%" # tsvector 主搜索 + ILIKE 兜底 search_cond = or_( GlobalLiterature.search_tsv.op("@@")(func.plainto_tsquery("english", q)), diff --git a/backend/app/services/pubmed_query_parser.py b/backend/app/services/pubmed_query_parser.py index 27c6ca9..dd947c7 100644 --- a/backend/app/services/pubmed_query_parser.py +++ b/backend/app/services/pubmed_query_parser.py @@ -382,27 +382,42 @@ class PubmedQueryParser: elif term.field is None: result.plain_terms.append(term) # ── 独立日期字段（非范围语法）："2024-01-01"[DP] → from=to=该日期 ── + # is_not 时加入 negated_date_ranges，引擎据此 NOT 条件 elif term.field == "DP": result.date_from = term.text result.date_to = term.text + if term.is_not: + result.negated_date_ranges.add("DP") elif term.field == "EDAT": result.edat_from = term.text result.edat_to = term.text + if term.is_not: + result.negated_date_ranges.add("EDAT") elif term.field == "CRDT": result.crdt_from = term.text result.crdt_to = term.text + if term.is_not: + result.negated_date_ranges.add("CRDT") elif term.field == "MHDA": result.mhda_from = term.text result.mhda_to = term.text + if term.is_not: + result.negated_date_ranges.add("MHDA") elif term.field == "LR": result.lr_from = term.text result.lr_to = term.text + if term.is_not: + result.negated_date_ranges.add("LR") elif term.field == "DCOM": result.dcom_from = term.text result.dcom_to = term.text + if term.is_not: + result.negated_date_ranges.add("DCOM") elif term.field == "DEP": result.dep_from = term.text result.dep_to = term.text + if term.is_not: + result.negated_date_ranges.add("DEP") elif term.field == "__RANGE_DP__": pass elif term.field == "__RANGE_EDAT__": @@ -484,6 +499,9 @@ class PubmedQueryParser: """not_expr → NOT not_expr | primary""" if self.peek().type == TokenType.NOT: self.advance() + # P5: trailing NOT at end of input → ignore silently (avoid IndexError peeking past EOF) + if self.peek().type == TokenType.EOF: + return [] inner = self._parse_not_expr(result) for t in inner: t.is_not = not t.is_not @@ -586,15 +604,34 @@ class PubmedQueryParser: start_val, end_val = end_val, start_val elif not start_val.isdigit() and not end_val.isdigit() and start_val > end_val: start_val, end_val = end_val, start_val + # 确定两端是否是 4 位年份 + _start_is_year = start_val.isdigit() and len(start_val) == 4 + _end_is_year = end_val.isdigit() and len(end_val) == 4 # Year-only range (e.g., 2024:2026[EDAT]) - if start_val.isdigit() and len(start_val) == 4: + if _start_is_year and _end_is_year: + try: + if yr_from_attr: + setattr(result, yr_from_attr, int(start_val)) + setattr(result, yr_to_attr, int(end_val)) + else: + # For non-DP date fields: convert year to full date for consistency + setattr(result, date_attr, f"{start_val}-01-01") + setattr(result, date_attr_to, f"{end_val}-12-31") + except (ValueError, TypeError): + pass + elif _start_is_year and not _end_is_year: + # Mixed: start is year, end is full date (e.g., 2024:2024-12-01[EDAT]) if yr_from_attr: - setattr(result, yr_from_attr, int(start_val)) - setattr(result, yr_to_attr, int(end_val)) - else: - # For non-DP date fields: convert year to full date for consistency - setattr(result, date_attr, f"{start_val}-01-01") - setattr(result, date_attr_to, f"{end_val}-12-31") + try: + setattr(result, yr_from_attr, int(start_val)) + except (ValueError, TypeError): + pass + setattr(result, date_attr, f"{start_val}-01-01") + setattr(result, date_attr_to, end_val) + elif not _start_is_year and _end_is_year: + # Mixed: start is full date, end is year (e.g., 2024-01-01:2026[EDAT]) + setattr(result, date_attr, start_val) + setattr(result, date_attr_to, f"{end_val}-12-31") else: # Full date range (e.g., 2024-01-01:2024-12-31[EDAT]) setattr(result, date_attr, start_val) @@ -624,6 +661,8 @@ def is_pubmed_syntax(query: str) -> bool: """ if not query or not query.strip(): return False + # P5: Normalize fullwidth characters before checking + query = unicodedata.normalize('NFKC', query) if re.search(r'\[(' + '|'.join(_ALL_FIELD_TAGS) + r')\]', query, re.IGNORECASE): return True if re.search(r'\b(AND|OR|NOT)\b', query, re.IGNORECASE): @@ -642,9 +681,15 @@ def parse_pubmed_query(query: str) -> ParsedPubmedQuery: try: # P2-3: Unicode normalization — strip zero-width chars, normalize fullwidth digits query = unicodedata.normalize('NFKC', query) - # 将 YYYY/MM/DD 格式的日期分隔符统一为 YYYY-MM-DD，使 tokeniser 正确识别为 DATE import re as _re + # 将 YYYY/MM/DD 格式的日期分隔符统一为 YYYY-MM-DD，使 tokeniser 正确识别为 DATE query = _re.sub(r'(\d{4})/(\d{2})/(\d{2})', r'\1-\2-\3', query) + # P5: Normalize single-digit month/day (2024-1-1 → 2024-01-01) to match DATE token pattern + query = _re.sub( + r'(\d{4})-(\d{1,2})-(\d{1,2})', + lambda m: f'{m.group(1)}-{int(m.group(2)):02d}-{int(m.group(3)):02d}', + query, + ) tokens = tokenise(query) parser = PubmedQueryParser(tokens) return parser.parse() @@ -668,9 +713,9 @@ def extract_pubmed_query_for_prisma(query: str) -> tuple[str, list[str]]: )) mesh_used.sort() - # 标准化：统一字段大写 + # P5: Handle field tags with '/' (e.g. Title/Article) or special chars normalized = re.sub( - r'\[([\w:]+)\]', + r'\[([\w/:]+)\]', lambda m: f'[{m.group(1).upper()}]', query, ) diff --git a/backend/app/services/query_expansion.py b/backend/app/services/query_expansion.py index f5bff61..080c7f5 100644 --- a/backend/app/services/query_expansion.py +++ b/backend/app/services/query_expansion.py @@ -186,7 +186,10 @@ async def _expand_tree_numbers(db: AsyncSession, tag_ids: list[UUID]) -> list[UU expanded = set(tag_ids) # 一次查询所有 tree_number 的子节点（用 OR 合并） - tn_conds = [GlobalTagTreeNumber.tree_number.like(f"{tn}%") for tn in tree_map] + tn_conds = [_or_( + GlobalTagTreeNumber.tree_number == tn, + GlobalTagTreeNumber.tree_number.like(f"{tn}.%"), + ) for tn in tree_map] rows = await db.execute( select(GlobalTagTreeNumber.tag_id) .where(_or_(*tn_conds)) diff --git a/backend/app/services/search_engine.py b/backend/app/services/search_engine.py index 0986a8c..6b184f2 100644 --- a/backend/app/services/search_engine.py +++ b/backend/app/services/search_engine.py @@ -176,6 +176,11 @@ class AdvancedSearchEngine: or pp.ot_terms or pp.gene_terms or pp.pmc_terms or pp.edat_from or pp.crdt_from or pp.mhda_from or pp.lr_from or pp.dcom_from or pp.dep_from + or pp.edat_to or pp.crdt_to or pp.mhda_to + or pp.lr_to or pp.dcom_to or pp.dep_to + or pp.date_from or pp.date_to + or pp.negated_date_ranges + or pp.groups or pp.plain_terms or pp.has_not or pp.year_from or pp.year_to ) @@ -205,6 +210,11 @@ class AdvancedSearchEngine: or _pubmed_parsed.ot_terms or _pubmed_parsed.gene_terms or _pubmed_parsed.pmc_terms or _pubmed_parsed.edat_from or _pubmed_parsed.crdt_from or _pubmed_parsed.mhda_from or _pubmed_parsed.lr_from or _pubmed_parsed.dcom_from or _pubmed_parsed.dep_from + or _pubmed_parsed.edat_to or _pubmed_parsed.crdt_to or _pubmed_parsed.mhda_to + or _pubmed_parsed.lr_to or _pubmed_parsed.dcom_to or _pubmed_parsed.dep_to + or _pubmed_parsed.date_from or _pubmed_parsed.date_to + or _pubmed_parsed.negated_date_ranges + or _pubmed_parsed.groups or _pubmed_parsed.plain_terms or _pubmed_parsed.has_not or _pubmed_parsed.year_from or _pubmed_parsed.year_to) ]): @@ -1130,7 +1140,7 @@ class AdvancedSearchEngine: elif field == "affiliation": return cast(GlobalLiterature.authors, String).ilike(_pt()) elif field == "language": - return GlobalLiterature.language == term + return GlobalLiterature.language.ilike(_pt()) elif field == "volume": return GlobalLiterature.volume.ilike(_pt()) elif field == "issue": @@ -1160,7 +1170,7 @@ class AdvancedSearchEngine: if "/" in term: if term.startswith("10."): return or_( - GlobalLiterature.doi == term, + GlobalLiterature.doi.ilike(term), GlobalLiterature.doi.ilike(like_val), ) return or_( @@ -1199,7 +1209,7 @@ class AdvancedSearchEngine: # 1a. 精确入口词匹配（P1-3）— batch via OR entry_conds.append(GlobalTag.entry_terms.contains([q])) # 1b. name_en ILIKE 回退 — batch via OR - name_conds.append(GlobalTag.name_en.ilike(m)) + name_conds.append(GlobalTag.name_en.ilike(_escape_ilike(m))) try: if entry_conds: @@ -1240,7 +1250,10 @@ class AdvancedSearchEngine: )).scalars().all() if tns: - child_conds = [GlobalTagTreeNumber.tree_number.like(f"{tn}%") for tn in tns] + child_conds = [or_( + GlobalTagTreeNumber.tree_number == tn, + GlobalTagTreeNumber.tree_number.like(f"{tn}.%"), + ) for tn in tns] children = (await db.execute( select(GlobalTagTreeNumber.tag_id).where(or_(*child_conds)) )).scalars().all() @@ -1300,5 +1313,7 @@ class AdvancedSearchEngine: return [GlobalLiterature.authors[0]['family'].astext.asc().nullslast()] elif sort == "journal": return [GlobalLiterature.journal.asc().nullslast()] + elif sort == "title": + return [GlobalLiterature.title.asc().nullslast()] else: return [GlobalLiterature.pub_date.desc().nullslast(), GlobalLiterature.id.desc()] diff --git a/docs/13-搜索修复全记录.md b/docs/13-搜索修复全记录.md index 1794bb1..25d1220 100644 --- a/docs/13-搜索修复全记录.md +++ b/docs/13-搜索修复全记录.md @@ -2,7 +2,7 @@ > 本文档按修复轮次详细记录所有搜索功能合规性修复的背景、根因分析和修改内容。 > -> **累计**：4 轮，59 项修复，50+ 字段标签注册，127 项测试覆盖 +> **累计**：5 轮，63 项修复，50+ 字段标签注册，214 项测试覆盖 > **时间跨度**：2026-07-24 ~ 2026-07-27 > **核心文件**：`pubmed_query_parser.py`（~620 行）→ `search_engine.py`（~1250 行） @@ -14,7 +14,8 @@ 2. [第二轮：第二轮审计修复（8 项）](#第二轮第二轮审计修复) 3. [第三轮：第三轮审计修复（14 项）](#第三轮第三轮审计修复) 4. [第四轮：字段补全与语义优化（11 项）](#第四轮字段补全与语义优化) -5. [遗留限制](#遗留限制) +5. [第五轮：第 5 轮全面审计修复（4 项）](#第五轮第-5-轮全面审计修复) +6. [遗留限制](#遗留限制) --- @@ -324,15 +325,69 @@ --- +## 第五轮：第 5 轮全面审计修复 + +**提交**：`pending` +**日期**：2026-07-27 +**数量**：4 项 +**触发**：5 Agent 并行深度审计 + 第 2 轮规划核对 + +### 背景 + +第 4 轮后启动第 5 轮全面审计，5 个 agent 并行扫描 parser、engine、前端、文档。首先与第 2 轮审计计划（F1-F8）逐项核对，确认以下 8 项已在之前轮次完成： + +| 计划 ID | 项目 | 完成轮次 | 状态 | +|---------|------|---------|------| +| F1 | `[MH:noexp]` 顶层支持 | 第 3/4 轮 | 已修复 | +| F2 | 组内 NOT De Morgan 律 | 第 2 轮 | 已验证正确 | +| F3 | `_parse_not_expr` NOT NOT 递归 | 第 4 轮 | 已修复 | +| F4 | SearchView Custom Range 日期 | 第 3 轮 | 已工作正常 | +| F5 | HomeView restoreFromUrl | 第 3 轮 | 已工作正常 | +| F6 | precision_mode 死代码 | 此前轮次 | 已移除 | +| F7 | is_oa 死字段 | 此前轮次 | 已注释 `unused` | +| F8 | `_expand_mesh_tag_ids` N+1 | 第 3/4 轮 | 已批量优化 | + +实际在第 5 轮修复的 4 项均为 parser 边缘案例： + +### P5-1: 尾部 NOT 导致 IndexError 降级 + +- **文件**：`pubmed_query_parser.py` `_parse_not_expr` L498 +- **问题**：`cancer NOT` — 解析器 `_is_primary_start` 包含 `NOT`，隐式 AND 逻辑尝试将 NOT 作为新词开始，但 NOT 之后无 token → `peek()` 越界抛 `IndexError` → 整个查询降级为纯文本拆分，`NOT` 作为字面搜索词（影响极小但产生异常） +- **根因**：`_parse_not_expr` 不检查是否已到 EOF +- **修复**：消费 NOT token 后立即检查 `peek().type == TokenType.EOF`，直接返回 `[]` 静默忽略 + +### P5-2: 单数字日期格式不被识别 + +- **文件**：`pubmed_query_parser.py` `parse_pubmed_query` L679 +- **问题**：`2024-1-1[DP]` — tokeniser 的 DATE 模式只匹配 `\d{4}-\d{2}-\d{2}`，单数字月/日被解析为 `WORD('2024-1-1')` → `_dispatch_term` 设置 `date_from='2024-1-1'` → `date.fromisoformat()` 抛 `ValueError` → 日期条件被静默丢弃 +- **根因**：tokeniser 前缺少单数字日期归一化 +- **修复**：在 `parse_pubmed_query` 的 NFKC 归一化后增加 `YYYY-M-D → YYYY-MM-DD` 正则替换 + +### P5-3: `is_pubmed_syntax` 不处理全角字符 + +- **文件**：`pubmed_query_parser.py` `is_pubmed_syntax` L652 +- **问题**：全角括号 `［TI］` 不被 `\[...\]` 识别 → 检出失败 → 走纯文本路径（解析器内 `parse_pubmed_query` 做 NFKC 但已不会进入） +- **根因**：`is_pubmed_syntax` 未做 NFKC 归一化、与 `parse_pubmed_query` 行为不一致 +- **修复**：函数开头增加 `query = unicodedata.normalize('NFKC', query)` + +### P5-4: `extract_pubmed_query_for_prisma` 不能处理 `[Title/Article]` + +- **文件**：`pubmed_query_parser.py` `extract_pubmed_query_for_prisma` L706 +- **问题**：归一化 regex `\[([\w:]+)\]` 不含 `/`，`[Title/Article]` 不被匹配、保持原文 +- **根因**：regex 字符类不含 `/` +- **修复**：`[\w:]` → `[\w/:]` + +--- + ## 各轮变更摘要 -| 维度 | 第一轮 | 第二轮 | 第三轮 | 第四轮 | -|------|--------|--------|--------|--------| -| 修复数 | 34 | 8 | 14 | 11 | -| 后端文件变更 | 全部 | engine + parser | engine + parser + api | engine + parser | -| 前端文件变更 | SearchView + HomeView + Card | SearchView + HomeView + Panel | SearchView + HomeView | 0 | -| 测试变更 | 新增 | +6 项 | 已有覆盖 | 0 | -| 新功能 | `[MH]`/`[EDAT]`/`[AD]`/`[LA]` 等 | — | — | `[GEN]`/`[PMC]`/`[Title/Abstract]` | +| 维度 | 第一轮 | 第二轮 | 第三轮 | 第四轮 | 第五轮 | +|------|--------|--------|--------|--------|--------| +| 修复数 | 34 | 8 | 14 | 11 | 4 | +| 后端文件变更 | 全部 | engine + parser | engine + parser + api | engine + parser | parser | +| 前端文件变更 | SearchView + HomeView + Card | SearchView + HomeView + Panel | SearchView + HomeView | 0 | 0 | +| 测试变更 | 新增 | +6 项 | 已有覆盖 | 0 | 0 | +| 新功能 | `[MH]`/`[EDAT]`/`[AD]`/`[LA]` 等 | — | — | `[GEN]`/`[PMC]`/`[Title/Abstract]` | — | | 性质 | 从零搭建 | 审计修复 | 深度审计修复 | 字段补全 | --- @@ -360,4 +415,4 @@ | `test_pubmed_query_parser.py` | ~40 | tokeniser、解析器、语法正确性 | | `test_pubmed_search_integration.py` | ~60 | 字段映射、API 集成、前端格式 | | `test_comprehensive_verify.py` | ~27 | 字段完整、NOT 语义、括号、日期 | -| **合计** | **127** | 全部通过 | +| **合计** | **214** | 全部通过 |