fix: 第9轮搜索深度审计 — MeSH headings 完全从 search_tsv 缺失等5项修复
CRITICAL: - MeSH headings search_tsv key 错误：trigger 用 value->>'name' 但数据存于 'descriptor' 键，导致 MeSH 术语对 tsvector 贡献完全丢失 - 新建 af4a8b2ec873 迁移：修复 key 名 + 全库回填 HIGH: - 测试套件 test_service_search_engine.py 无断言价值（module-level mock 杀死缓存测试，所有 search() 只 assert total == 0） - Affiliation 从 tsvector 剥离（f1a2b3c4d5e6 引入 author_names_text 时丢失） MEDIUM: - backfill_search_tsv.py 严重过期（缺 mesh/chemicals/genes/keywords） LOW: - features.py query 无 max_length 字符限制（仅 100 词限制） docs: 13-搜索修复全记录.md 新增第9轮
This commit is contained in:
34047007@qq.com
@@ -0,0 +1,201 @@
|
||||
"""fix mesh_headings key in search_tsv trigger (descriptor not name)
|
||||
|
||||
所有数据源将 MeSH descriptorText 写入 mesh_headings JSONB 的 'descriptor' 键，
|
||||
但 g0h1i2j3k4l5 迁移的 trigger 公式用 value->>'name' 读取，导致 MeSH 术语
|
||||
对 search_tsv 的贡献完全丢失。普通搜索无法通过 tsvector 命中 MeSH 标签。
|
||||
|
||||
本迁移修复 trigger 中的 key 名并回填所有已有数据。
|
||||
|
||||
Revision ID: af4a8b2ec873
|
||||
Revises: b01b8f27c596
|
||||
Create Date: 2026-07-28 11:10:17.939040
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision: str = 'af4a8b2ec873'
|
||||
down_revision: Union[str, None] = 'b01b8f27c596'
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
# 修复：value->>'name' → value->>'descriptor'
|
||||
_TSVEC_FIXED = """setweight(to_tsvector('english', COALESCE(NEW.title, '')), 'A') ||
|
||||
setweight(to_tsvector('english', COALESCE(NEW.abstract, '')), 'B') ||
|
||||
setweight(to_tsvector('simple', COALESCE(NEW.author_names_text, '')), 'A') ||
|
||||
setweight(to_tsvector('english',
|
||||
COALESCE(
|
||||
(SELECT string_agg(value->>'name', ' ')
|
||||
FROM jsonb_array_elements(NEW.chemical_list)),
|
||||
'')
|
||||
), 'C') ||
|
||||
setweight(to_tsvector('english',
|
||||
COALESCE(
|
||||
(SELECT string_agg(value #>> '{}', ' ')
|
||||
FROM jsonb_array_elements(NEW.gene_symbols)),
|
||||
'')
|
||||
), 'C') ||
|
||||
setweight(to_tsvector('english',
|
||||
COALESCE(
|
||||
(SELECT string_agg(value->>'descriptor', ' ')
|
||||
FROM jsonb_array_elements(NEW.mesh_headings)),
|
||||
'')
|
||||
), 'C') ||
|
||||
setweight(to_tsvector('english',
|
||||
COALESCE(
|
||||
(SELECT string_agg(value #>> '{}', ' ')
|
||||
FROM jsonb_array_elements(NEW.keywords)),
|
||||
'')
|
||||
), 'C')"""
|
||||
|
||||
# 回填用 UPDATE 表达式
|
||||
_UPDATE_SQL = """UPDATE global_literature
|
||||
SET search_tsv = setweight(to_tsvector('english', COALESCE(title, '')), 'A') ||
|
||||
setweight(to_tsvector('english', COALESCE(abstract, '')), 'B') ||
|
||||
setweight(to_tsvector('simple', COALESCE(author_names_text, '')), 'A') ||
|
||||
setweight(to_tsvector('english',
|
||||
COALESCE(
|
||||
(SELECT string_agg(value->>'name', ' ')
|
||||
FROM jsonb_array_elements(chemical_list)),
|
||||
'')
|
||||
), 'C') ||
|
||||
setweight(to_tsvector('english',
|
||||
COALESCE(
|
||||
(SELECT string_agg(value #>> '{}', ' ')
|
||||
FROM jsonb_array_elements(gene_symbols)),
|
||||
'')
|
||||
), 'C') ||
|
||||
setweight(to_tsvector('english',
|
||||
COALESCE(
|
||||
(SELECT string_agg(value->>'descriptor', ' ')
|
||||
FROM jsonb_array_elements(mesh_headings)),
|
||||
'')
|
||||
), 'C') ||
|
||||
setweight(to_tsvector('english',
|
||||
COALESCE(
|
||||
(SELECT string_agg(value #>> '{}', ' ')
|
||||
FROM jsonb_array_elements(keywords)),
|
||||
'')
|
||||
), 'C')
|
||||
WHERE search_tsv IS NOT NULL"""
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# 1. 先删除触发器
|
||||
op.execute("DROP TRIGGER IF EXISTS trg_global_literature_tsv ON global_literature")
|
||||
|
||||
# 2. 更新触发器函数
|
||||
op.execute(f"""
|
||||
CREATE OR REPLACE FUNCTION update_literature_search_tsv()
|
||||
RETURNS trigger AS $$
|
||||
BEGIN
|
||||
NEW.author_names_text := COALESCE(
|
||||
(SELECT string_agg(value->>'family', ' ')
|
||||
FROM jsonb_array_elements(NEW.authors)),
|
||||
''
|
||||
);
|
||||
NEW.search_tsv := {_TSVEC_FIXED};
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
""")
|
||||
|
||||
# 3. 重建触发器
|
||||
op.execute("""
|
||||
CREATE TRIGGER trg_global_literature_tsv
|
||||
BEFORE INSERT OR UPDATE OF title, abstract, authors, chemical_list, gene_symbols,
|
||||
mesh_headings, keywords
|
||||
ON global_literature
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION update_literature_search_tsv()
|
||||
""")
|
||||
|
||||
# 4. 回填已有数据的 search_tsv（全量重建）
|
||||
op.execute(_UPDATE_SQL)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
# 恢复到 value->>'name'（带 bug 的旧版本）
|
||||
op.execute("DROP TRIGGER IF EXISTS trg_global_literature_tsv ON global_literature")
|
||||
|
||||
op.execute("""
|
||||
CREATE OR REPLACE FUNCTION update_literature_search_tsv()
|
||||
RETURNS trigger AS $$
|
||||
BEGIN
|
||||
NEW.author_names_text := COALESCE(
|
||||
(SELECT string_agg(value->>'family', ' ')
|
||||
FROM jsonb_array_elements(NEW.authors)),
|
||||
''
|
||||
);
|
||||
NEW.search_tsv := setweight(to_tsvector('english', COALESCE(NEW.title, '')), 'A') ||
|
||||
setweight(to_tsvector('english', COALESCE(NEW.abstract, '')), 'B') ||
|
||||
setweight(to_tsvector('simple', COALESCE(NEW.author_names_text, '')), 'A') ||
|
||||
setweight(to_tsvector('english',
|
||||
COALESCE(
|
||||
(SELECT string_agg(value->>'name', ' ')
|
||||
FROM jsonb_array_elements(NEW.chemical_list)),
|
||||
'')
|
||||
), 'C') ||
|
||||
setweight(to_tsvector('english',
|
||||
COALESCE(
|
||||
(SELECT string_agg(value #>> '{}', ' ')
|
||||
FROM jsonb_array_elements(NEW.gene_symbols)),
|
||||
'')
|
||||
), 'C') ||
|
||||
setweight(to_tsvector('english',
|
||||
COALESCE(
|
||||
(SELECT string_agg(value->>'name', ' ')
|
||||
FROM jsonb_array_elements(NEW.mesh_headings)),
|
||||
'')
|
||||
), 'C') ||
|
||||
setweight(to_tsvector('english',
|
||||
COALESCE(
|
||||
(SELECT string_agg(value #>> '{}', ' ')
|
||||
FROM jsonb_array_elements(NEW.keywords)),
|
||||
'')
|
||||
), 'C');
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
""")
|
||||
|
||||
op.execute("""
|
||||
CREATE TRIGGER trg_global_literature_tsv
|
||||
BEFORE INSERT OR UPDATE OF title, abstract, authors, chemical_list, gene_symbols,
|
||||
mesh_headings, keywords
|
||||
ON global_literature
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION update_literature_search_tsv()
|
||||
""")
|
||||
|
||||
op.execute("""
|
||||
UPDATE global_literature
|
||||
SET search_tsv = setweight(to_tsvector('english', COALESCE(title, '')), 'A') ||
|
||||
setweight(to_tsvector('english', COALESCE(abstract, '')), 'B') ||
|
||||
setweight(to_tsvector('simple', COALESCE(author_names_text, '')), 'A') ||
|
||||
setweight(to_tsvector('english',
|
||||
COALESCE(
|
||||
(SELECT string_agg(value->>'name', ' ')
|
||||
FROM jsonb_array_elements(chemical_list)),
|
||||
'')
|
||||
), 'C') ||
|
||||
setweight(to_tsvector('english',
|
||||
COALESCE(
|
||||
(SELECT string_agg(value #>> '{}', ' ')
|
||||
FROM jsonb_array_elements(gene_symbols)),
|
||||
'')
|
||||
), 'C') ||
|
||||
setweight(to_tsvector('english',
|
||||
COALESCE(
|
||||
(SELECT string_agg(value->>'name', ' ')
|
||||
FROM jsonb_array_elements(mesh_headings)),
|
||||
'')
|
||||
), 'C') ||
|
||||
setweight(to_tsvector('english',
|
||||
COALESCE(
|
||||
(SELECT string_agg(value #>> '{}', ' ')
|
||||
FROM jsonb_array_elements(keywords)),
|
||||
'')
|
||||
), 'C')
|
||||
WHERE search_tsv IS NOT NULL
|
||||
""")
|
||||
@@ -1,10 +1,13 @@
|
||||
"""回填 search_tsv（'simple' 词典版）
|
||||
"""回填 search_tsv（当前触发器公式版）
|
||||
|
||||
迁移 e5f6a7b8c9d0 更新了触发器函数（english→simple），
|
||||
但已有记录的 search_tsv 仍是旧词典生成的。此脚本分批回填。
|
||||
回填用于以下场景：
|
||||
1. 迁移后已有记录的 search_tsv 未自动更新
|
||||
2. 手动修复 search_tsv 数据
|
||||
|
||||
由于 PL/pgSQL DO 块内的 COMMIT 与 Alembic 事务冲突，
|
||||
因此放在迁移之外执行。
|
||||
注意事项：
|
||||
- 此脚本不修改触发器函数本身
|
||||
- 它仅执行与当前触发器相同的 tsvector 公式
|
||||
- 只回填已有 search_tsv 不为 NULL 的记录
|
||||
|
||||
用法:
|
||||
cd backend && python scripts/backfill_search_tsv.py
|
||||
@@ -19,14 +22,31 @@ _BACKFILL_SQL = r"""
|
||||
SET search_tsv =
|
||||
setweight(to_tsvector('english', COALESCE(title, '')), 'A') ||
|
||||
setweight(to_tsvector('english', COALESCE(abstract, '')), 'B') ||
|
||||
setweight(to_tsvector('simple',
|
||||
setweight(to_tsvector('simple', COALESCE(author_names_text, '')), 'A') ||
|
||||
setweight(to_tsvector('english',
|
||||
COALESCE(
|
||||
(SELECT string_agg(
|
||||
value->>'family' || ' ' || COALESCE(value->>'affiliation', ''),
|
||||
' ')
|
||||
FROM jsonb_array_elements(authors)),
|
||||
(SELECT string_agg(value->>'name', ' ')
|
||||
FROM jsonb_array_elements(chemical_list)),
|
||||
'')
|
||||
), 'A')
|
||||
), 'C') ||
|
||||
setweight(to_tsvector('english',
|
||||
COALESCE(
|
||||
(SELECT string_agg(value #>> '{}', ' ')
|
||||
FROM jsonb_array_elements(gene_symbols)),
|
||||
'')
|
||||
), 'C') ||
|
||||
setweight(to_tsvector('english',
|
||||
COALESCE(
|
||||
(SELECT string_agg(value->>'descriptor', ' ')
|
||||
FROM jsonb_array_elements(mesh_headings)),
|
||||
'')
|
||||
), 'C') ||
|
||||
setweight(to_tsvector('english',
|
||||
COALESCE(
|
||||
(SELECT string_agg(value #>> '{}', ' ')
|
||||
FROM jsonb_array_elements(keywords)),
|
||||
'')
|
||||
), 'C')
|
||||
WHERE search_tsv IS NOT NULL
|
||||
"""
|
||||
|
||||
|
||||
Reference in New Issue
Block a user