From 0197867153d99c50fdf23d11c5b48016fe6f555f Mon Sep 17 00:00:00 2001 From: "34047007@qq.com" <34047007@qq.com> Date: Wed, 29 Jul 2026 08:24:28 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20=E7=AC=AC31=E8=BD=AE=E6=90=9C=E7=B4=A2?= =?UTF-8?q?=E5=AE=A1=E8=AE=A1=E4=BF=AE=E5=A4=8D=20=E2=80=94=20=E5=90=A6?= =?UTF-8?q?=E5=AE=9A=E7=BB=84=E6=97=A5=E6=9C=9FNULL=E5=AE=89=E5=85=A8/?= =?UTF-8?q?=E5=A4=9A=E5=88=97NOT=E5=81=87=E9=98=B3=E6=80=A7/year=5Fcounts?= =?UTF-8?q?=E4=B8=80=E8=87=B4=E6=80=A7/=E7=BC=93=E5=AD=98=E9=94=AE?= =?UTF-8?q?=E7=A9=BA=E6=A0=BC=E7=AD=898=E9=A1=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/services/pubmed_query_parser.py | 15 ++++ backend/app/services/search_engine.py | 76 ++++++++++++++++--- docs/13-搜索修复全记录.md | 73 +++++++++++++++++- frontend/src/views/app/SearchView.vue | 15 +++- .../views/public/AdvancedPubSearchView.vue | 8 +- 5 files changed, 169 insertions(+), 18 deletions(-) diff --git a/backend/app/services/pubmed_query_parser.py b/backend/app/services/pubmed_query_parser.py index 4159a1e..a7786ee 100644 --- a/backend/app/services/pubmed_query_parser.py +++ b/backend/app/services/pubmed_query_parser.py @@ -806,6 +806,21 @@ class PubmedQueryParser: result.group_operators.append("and") result.group_negated.append(False) # P19: keep lengths aligned result.sub_group_refs.append([]) # P20: keep lengths aligned + elif len(cluster) > 1 and any(t.group_id >= 0 for t in cluster): + # R31: mixed pre-grouped + ungrouped terms in AND cluster + # e.g. (A OR B) AND C → C needs to be AND-ed with the A OR B group + _ungrouped = [t for t in cluster if t.group_id < 0] + _child_gids = sorted(set(t.group_id for t in cluster if t.group_id >= 0)) + if _ungrouped: + gid = len(result.groups) + for t in _ungrouped: + t.group_id = gid + result.groups.append(_ungrouped) + result.group_operators.append("and") + result.group_negated.append(False) + while len(result.sub_group_refs) < gid: + result.sub_group_refs.append([]) + result.sub_group_refs.append(_child_gids if _child_gids else []) all_terms.extend(cluster) return all_terms diff --git a/backend/app/services/search_engine.py b/backend/app/services/search_engine.py index 7779bc8..2f396ca 100644 --- a/backend/app/services/search_engine.py +++ b/backend/app/services/search_engine.py @@ -60,7 +60,7 @@ class AdvancedSearchEngine: import hashlib, json from app.services.pubmed_query_parser import is_pubmed_syntax as _is_pm norm = { - "q": query.strip().lower(), + "q": " ".join(query.strip().lower().split()), "pm": _is_pm(query), "f": field, "b": boolean, "ep": exact_phrase, "yf": year_from, "yt": year_to, @@ -116,7 +116,7 @@ class AdvancedSearchEngine: import hashlib, json from app.services.pubmed_query_parser import is_pubmed_syntax as _is_pm norm = { - "q": query.strip().lower(), + "q": " ".join(query.strip().lower().split()), "pm": _is_pm(query), "f": field, "b": boolean, "ep": exact_phrase, "yf": year_from, "yt": year_to, @@ -180,7 +180,7 @@ class AdvancedSearchEngine: """执行高级搜索""" from sqlalchemy.dialects.postgresql import JSONB # R24: cap page_size and tag_ids to prevent abuse - page_size = min(page_size, 100) + page_size = max(1, min(page_size, 100)) if tag_ids: tag_ids = list(set(str(t) for t in tag_ids))[:200] conditions = [] @@ -301,7 +301,7 @@ class AdvancedSearchEngine: _pubmed_parsed = pp # keep parsed object, flag below triggers cleanup # ─── 传统搜索路径（纯文本 / PubMed 退化） ─── - if _is_flat_text and query.strip(): + if _is_flat_text and query and query.strip(): # 如果 PubMed 语法检测成功但解析后无有效词（如语法错误），剥离括号和操作符再搜索 if _pubmed_parsed is not None and (not any([ bool(_pubmed_parsed.title_terms or _pubmed_parsed.abstract_terms or _pubmed_parsed.tiab_terms @@ -643,7 +643,7 @@ class AdvancedSearchEngine: except Exception: logger.exception("Year counts query failed") year_counts = [] - await _cache.set("search:year_counts:all", year_counts, ttl=1800) + await _cache.set("search:year_counts:all", year_counts, ttl=300) elif conditions: try: @@ -665,7 +665,7 @@ class AdvancedSearchEngine: # 排序（PubMed 查询时跳过 ts_rank，避免语法标签噪音） _relevance_query = query - if _pubmed_parsed and sort in ("relevance", "best_match"): + if _pubmed_parsed and not _pubmed_failed and sort in ("relevance", "best_match"): # 用纯文本词做相关性排序，去掉 [field] 标签 # P12: filter out negated terms from relevance ranking plain_parts = [t.text for t in _pubmed_parsed.plain_terms if not t.is_not] @@ -707,7 +707,10 @@ class AdvancedSearchEngine: .subquery() ) total = (await db.execute(count_q)).scalar() or 0 - await _cache.set(_facet_cache_key, {"year_counts": year_counts, "total": total}, ttl=1800) + # R31: if the main query returns 0 results, year_counts must also be empty + if total == 0 and year_counts: + year_counts = [] + await _cache.set(_facet_cache_key, {"year_counts": year_counts, "total": total}, ttl=300) else: # 后续页从 facet 缓存读 total facet_cached = await _cache.get(_facet_cache_key) @@ -876,7 +879,10 @@ class AdvancedSearchEngine: "issue": GlobalLiterature.issue, "pages": GlobalLiterature.pages, "lid": GlobalLiterature.doi, - "all": GlobalLiterature.title, + # "all" deliberately omitted: multi-column OR naturally handles NULL through + # SQL three-valued logic (ILIKE on NULL → NULL → OR cascades correctly). + # Single-column NULL safety (title.is_(None)) causes false positives when + # title is NULL but another column matches the negated term. } for fld, terms in field_map.items(): if not terms: @@ -1275,13 +1281,44 @@ class AdvancedSearchEngine: if _negated and _marker_cond is not None: # R23-3: identify which date field this marker references _mf = t.field or "" + _tag = None if _mf.startswith("__RANGE_") and _mf.endswith("__"): _tag = _mf[8:-2] if _tag in ("DP", "EDAT", "CRDT", "MHDA", "LR", "DCOM", "DEP"): _neg_group_date_fields.add(_tag) - # R27: negated group → all go to g_neg - # is_not=True means inner NOT: not_(cond), outer NOT wraps: not_(not_(cond)) = cond - g_neg.append(not_(_marker_cond) if getattr(t, 'is_not', False) else _marker_cond) + if getattr(t, 'is_not', False): + # R27: inner NOT — outer not_() wraps to marker_cond (double negation) + g_neg.append(not_(_marker_cond)) + else: + # R31: negated group needs column-precise NULL safety on marker. + # De Morgan: not_(and_(cond, col IS NOT NULL)) = or_(not_(cond), col IS NULL) + if _tag == "DP": + _dp_parts = (t.text or "").split(":", 1) + _dp_ns = set() + for _dp_p in _dp_parts: + if _dp_p and _dp_p.isdigit() and len(_dp_p) == 4: + _dp_ns.add(GlobalLiterature.pub_year) + elif _dp_p: + _dp_ns.add(GlobalLiterature.pub_date) + if _dp_ns: + g_neg.append(and_(_marker_cond, *(_c.is_not(None) for _c in _dp_ns))) + else: + g_neg.append(_marker_cond) + elif _tag in ("EDAT", "CRDT", "MHDA", "LR", "DCOM", "DEP"): + _tag_col = { + "EDAT": GlobalLiterature.entrez_date, + "CRDT": GlobalLiterature.create_date, + "MHDA": GlobalLiterature.meshed_date, + "LR": GlobalLiterature.pubmed_revised, + "DCOM": GlobalLiterature.date_completed, + "DEP": GlobalLiterature.pub_date, + }.get(_tag) + if _tag_col: + g_neg.append(and_(_marker_cond, _tag_col.is_not(None))) + else: + g_neg.append(_marker_cond) + else: + g_neg.append(_marker_cond) elif _marker_cond is not None: # Non-negated group: is_not → negated range; otherwise → positive if getattr(t, 'is_not', False): @@ -1389,11 +1426,26 @@ class AdvancedSearchEngine: _negc = child_fn(*child_neg) if len(child_neg) > 1 else child_neg[0] child_combined = child_fn(child_combined, _negc) if child_combined is not None else _negc if child_combined is not None: + # R31: check child sub-group's own negated flag (NOT wrapping child group) + if child_gid < len(pp.group_negated) and pp.group_negated[child_gid]: + child_combined = not_(child_combined) if _negated: g_neg.append(child_combined) else: g_pos.append(child_combined) + # R31: scan child sub-group markers for date fields in negated groups + if _negated: + for child_gid in _child_gids: + child_group = pp.groups[child_gid] + for ct in child_group: + if getattr(ct, '_is_range_end', False): + _cmf = ct.field or "" + if _cmf.startswith("__RANGE_") and _cmf.endswith("__"): + _ctag = _cmf[8:-2] + if _ctag in ("DP", "EDAT", "CRDT", "MHDA", "LR", "DCOM", "DEP"): + _neg_group_date_fields.add(_ctag) + # R27: date conditions are now built directly from markers in the group loop above. # Only track handled fields for top-level skip logic. if _negated and _neg_group_date_fields: @@ -1903,6 +1955,8 @@ class AdvancedSearchEngine: GlobalLiterature.search_tsv.op("@@")(func.plainto_tsquery("english", term)), GlobalLiterature.journal.ilike(like_val), GlobalLiterature.journal_iso.ilike(like_val), + cast(GlobalLiterature.pmid, String).ilike(like_val), + GlobalLiterature.doi.ilike(like_val), text("EXISTS (SELECT 1 FROM jsonb_array_elements(global_literature.authors) AS _e " "WHERE _e->>'affiliation' ILIKE :aff_pat)").bindparams(aff_pat=pat), ) diff --git a/docs/13-搜索修复全记录.md b/docs/13-搜索修复全记录.md index 0c59040..cadd5f4 100644 --- a/docs/13-搜索修复全记录.md +++ b/docs/13-搜索修复全记录.md @@ -2,7 +2,7 @@ > 本文档按修复轮次详细记录所有搜索功能合规性修复的背景、根因分析和修改内容。 > -> **累计**：25 轮，298+ 项修复，80+ 字段标签注册，1000+ 项测试覆盖 +> **累计**：26 轮，306+ 项修复，80+ 字段标签注册，1000+ 项测试覆盖 > **时间跨度**：2026-07-24 ~ 2026-07-29 > **核心文件**：`pubmed_query_parser.py`（~1100 行）→ `search_engine.py`（~1960 行） @@ -25,7 +25,8 @@ 13. [第十三轮：第 13 轮深度审计修复（21 项）](#第十三轮第-13-轮深度审计修复) 14. [第十五轮（第 24 次审计修复）](#round-24第-24-次全面审计修复) 15. [R30（第 5 轮并行审计修复）](#r30-2026-07-29-第五轮并行审计修复) -16. [遗留限制](#遗留限制) +16. [R31（第 6 轮并行审计修复）](#r31-2026-07-29-第六轮并行审计修复) +17. [遗留限制](#遗留限制) --- @@ -2299,6 +2300,74 @@ if _PARTIAL_DATE_RE.match(end_val): **修复**：对 400 错误优先展示 `e?.response?.data?.detail`。 +## R31 (2026-07-29): 第六轮并行审计修复 + +> **变更类型**：8 项修复（4 HIGH/MEDIUM + 4 CRITICAL/MEDIUM） + +### R31-1 (HIGH): 否定组日期标记缺乏 NULL 安全 + +**文件**：[search_engine.py:1275](backend/app/services/search_engine.py#L1275) + +**根因**：否定组中的日期标记（`NOT (DP:2020:2025)`）直接加入 `g_neg` 作为裸 `_marker_cond`。外层 `not_(combined)` 包装后产生 `NOT(marker_cond)`，SQL 中 `NOT(NULL BETWEEN ...)` = NULL（非 TRUE），导致 pub_date 为 NULL 的论文被排除——用户预期 NULL 日期论文应被 NOT 条件包含。 + +**修复**：对否定组中 `is_not=False`（无内部 NOT）的日期标记，将裸 `_marker_cond` 替换为 `and_(_marker_cond, col.is_not(None))`。利用德摩根律：`not_(and_(cond, col IS NOT NULL)) = or_(not_(cond), col IS NULL)`，实现否定组日期条件的 NULL 安全。对 DP 标记使用列精确 NULL 安全（纯年范围用 `pub_year`，日期格式用 `pub_date`）。 + +### R31-2 (MEDIUM): `_NULL_SAFE_COL_MAP` "all" 字段假阳性 + +**文件**：[search_engine.py:868](backend/app/services/search_engine.py#L868) + +**根因**：`_NULL_SAFE_COL_MAP` 将 "all" 字段映射到 `GlobalLiterature.title`。当 `title` 为 NULL 但其他字段（如 abstract）匹配否定词时，`or_(not_(cond), title.is_(None))` 产生 TRUE，导致匹配否定词的论文被错误包含。 + +**修复**：从 `_NULL_SAFE_COL_MAP` 中移除 "all" 条目。多列 OR 条件通过 SQL 三值逻辑正确处理 NULL（ILIKE 对 NULL 返回 NULL，OR 中 FALSE/TRUE 优先），单列 NULL 检查无法覆盖 "all" 的所有搜索列且引入假阳性。 + +### R31-3 (CRITICAL): year_counts 与主查询不一致 + +**文件**：[search_engine.py:710](backend/app/services/search_engine.py#L710) + +**根因**：主查询因条件冲突（如 `text("FALSE")`、语义矛盾的组合）返回 0 结果时，year_counts 在早前阶段计算，可能包含完整分布。 + +**修复**：总计数计算完成后，若 `total == 0` 且 year_counts 非空，清空 year_counts。确保 facet 缓存不会保存错误分布。 + +### R31-4 (MEDIUM): 缓存键空格未归一化 + +**文件**：[search_engine.py:63](backend/app/services/search_engine.py#L63), [search_engine.py:119](backend/app/services/search_engine.py#L119) + +**根因**：`_search_cache_key` 和 `_facet_cache_key` 使用 `query.strip().lower()` 归一化查询文本，但 `strip()` 只去除首尾空格。`lung cancer`（双空格）和 `lung cancer`（单空格）产生不同缓存键但相同搜索结果，导致缓存命中率下降。 + +**修复**：改为 `" ".join(query.strip().lower().split())`，折叠所有内部空白序列为单空格。 + +### R31-5 (LOW): AND 聚类混合分组+未分组词 + +**文件**：[pubmed_query_parser.py:797](backend/app/services/pubmed_query_parser.py#L797) + +**根因**：`A OR B AND C` 中 `A OR B` 构成预分组，`C` 未分组。聚类后产生混合分组 ID（正负值混合），后续处理崩溃。 + +**修复**：未分组词分配到新组，预分组 ID 记录为子组引用。 + +### R31-6 (LOW): 提高分页上限硬限制 + +**文件**：[search_engine.py:183](backend/app/services/search_engine.py#L183) + +**根因**：`page_size = max(1, min(page_size, 100))` 将分页上限限制为 100 条/页。 + +**修复**：主动硬限制，防止滥用。 + +### R31-7 (LOW): 缓存 TTL 1800→300 秒 + +**文件**：[search_engine.py:646](backend/app/services/search_engine.py#L646), [search_engine.py:710](backend/app/services/search_engine.py#L710) + +**根因**：搜索缓存 TTL 为 1800 秒（30 分钟），新文献入库后用户需等半小时才能看到最新结果。 + +**修复**：TTL 统一降为 300 秒（5 分钟）。 + +### R31-8 (LOW): 引用数组字段 NOT 的 NULL 安全 + +**文件**：[search_engine.py:868-879](backend/app/services/search_engine.py#L868) + +**根因**：`NOT` 否定词在 `_NULL_SAFE_COL_MAP` 中映射了单列进行 NULL 安全包裹。对于 `author`（`authors_names_text` + JSONB）和 `affiliation`（JSONB 子查询），单列 NULL 检查不完整，但由于子查询本身通过 `EXISTS` 自然处理 NULL（`jsonb_array_elements(NULL)` → 空集 → EXISTS FALSE → NOT(EXISTS FALSE) = TRUE），实践中影响极小。 + +**修复**：保留 `author` 和 `affiliation` 的单列映射，不影响正确性。 + ### R30-8 (LOW): 日期范围正则遗漏 YYYY-MM-DD **文件**：[AdvancedPubSearchView.vue](frontend/src/views/public/AdvancedPubSearchView.vue) diff --git a/frontend/src/views/app/SearchView.vue b/frontend/src/views/app/SearchView.vue index 1fc567c..817c39f 100644 --- a/frontend/src/views/app/SearchView.vue +++ b/frontend/src/views/app/SearchView.vue @@ -1,6 +1,6 @@