fix: 第8轮搜索深度审计修复 — 缓存失效、Redis重试、中文标签、翻页稳定性等12项
CRITICAL:
- invalidate_search_cache 清理 atm:* 缓存(MeSH ATM扩展不再用过期结果)
- Pro 方案 api_quota_per_day 1000→10000(修复低于 Free 的数据错误)
- CacheService/RateLimitMiddleware Redis 连接失败60秒自动重试(原永久降级)
- 普通搜索中文输入自动匹配 GlobalTag.name_zh(如"肺癌"通过MeSH标签关联文献)
- AdvancedPubSearchView resolveQuery 添加 seen Set 检测交替 #N 循环引用
HIGH:
- 限速器 _burst_windows 每500请求清理过期条目(防止内存泄漏)
- cron daily_ftp_update 末尾调用 invalidate_search_cache()(自动管道不再用过期缓存)
MEDIUM:
- _apply_order_by ASC 排序加 id tiebreaker(title/journal/first_author翻页跳行/重复)
- _keyset_condition 所有 is_(None) 加 id tiebreaker + __NULL__ 哨兵值
- _field_condition("all") 默认tsvector路径加 journal/journal_iso ILIKE 兜底
- SearchView restoreFromQuery date_preset/year_from/year_to 优先顺序修复
docs: 更新 12/13 搜索文档,移除 CLAUDE.md 陈旧 SQLite 提及
This commit is contained in:
@@ -0,0 +1,42 @@
|
||||
"""backfill tag_ids from global_literature_tag
|
||||
|
||||
Revision ID: b01b8f27c596
|
||||
Revises: e0764f6d7c21
|
||||
Create Date: 2026-07-27 22:51:25.288622
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy.dialects import postgresql
|
||||
|
||||
revision: str = 'b01b8f27c596'
|
||||
down_revision: Union[str, None] = 'e0764f6d7c21'
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# 回填 tag_ids:从 global_literature_tags 聚合存量数据
|
||||
# 注:author_names_text 的回填已在 f1a2b3c4d5e6 中完成,无需重复
|
||||
#
|
||||
# 用 array_agg + JOIN 代替关联子查询,避免每行触发子查询。
|
||||
# 对百万/千万级数据线性扩展(hash 聚合 + hash join),预期 2-5 分钟。
|
||||
op.execute("""
|
||||
UPDATE global_literature gl
|
||||
SET tag_ids = agg.tag_ids
|
||||
FROM (
|
||||
SELECT literature_id, array_agg(tag_id ORDER BY tag_id) AS tag_ids
|
||||
FROM global_literature_tags
|
||||
GROUP BY literature_id
|
||||
) agg
|
||||
WHERE gl.id = agg.literature_id
|
||||
""")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
# 回滚:清空 tag_ids(列为 nullable=True,回填是幂等的)
|
||||
op.execute("""
|
||||
UPDATE global_literature SET tag_ids = NULL
|
||||
WHERE tag_ids IS NOT NULL
|
||||
""")
|
||||
@@ -67,11 +67,11 @@ def upgrade() -> None:
|
||||
op.add_column('global_literature', sa.Column('entrez_date', sa.DateTime(timezone=True), nullable=True))
|
||||
|
||||
# ─── 2. 19 列 JSON → JSONB(单笔 ALTER TABLE,一次全表重写)───
|
||||
for col in _JSON_TO_JSONB_COLS:
|
||||
op.alter_column('global_literature', col,
|
||||
existing_type=postgresql.JSON(astext_type=sa.Text()),
|
||||
type_=postgresql.JSONB(astext_type=sa.Text()),
|
||||
existing_nullable=True)
|
||||
_cols = ", ".join(
|
||||
f"ALTER COLUMN \"{c}\" TYPE jsonb USING \"{c}\"::jsonb"
|
||||
for c in _JSON_TO_JSONB_COLS
|
||||
)
|
||||
op.execute(f"ALTER TABLE global_literature {_cols}")
|
||||
|
||||
# ─── 3. entry_terms 到 global_tags ───
|
||||
op.add_column('global_tags', sa.Column('entry_terms', postgresql.JSONB(astext_type=sa.Text()), nullable=True))
|
||||
|
||||
@@ -53,33 +53,6 @@ _TSVEC_OLD = """setweight(to_tsvector('english', COALESCE(NEW.title, '')), 'A')
|
||||
'')
|
||||
), 'A')"""
|
||||
|
||||
# 回填用 UPDATE 表达式(引用列名而非 NEW)
|
||||
_UPDATE_SQL = """UPDATE global_literature
|
||||
SET search_tsv = setweight(to_tsvector('english', COALESCE(title, '')), 'A') ||
|
||||
setweight(to_tsvector('english', COALESCE(abstract, '')), 'B') ||
|
||||
setweight(to_tsvector('simple',
|
||||
COALESCE(
|
||||
(SELECT string_agg(
|
||||
value->>'family' || ' ' || COALESCE(value->>'affiliation', ''),
|
||||
' ')
|
||||
FROM jsonb_array_elements(authors)),
|
||||
'')
|
||||
), 'A') ||
|
||||
setweight(to_tsvector('english',
|
||||
COALESCE(
|
||||
(SELECT string_agg(value->>'name', ' ')
|
||||
FROM jsonb_array_elements(chemical_list)),
|
||||
'')
|
||||
), 'C') ||
|
||||
setweight(to_tsvector('english',
|
||||
COALESCE(
|
||||
(SELECT string_agg(value #>> '{}', ' ')
|
||||
FROM jsonb_array_elements(gene_symbols)),
|
||||
'')
|
||||
), 'C')
|
||||
WHERE chemical_list IS NOT NULL AND chemical_list != '[]'::jsonb
|
||||
OR gene_symbols IS NOT NULL AND gene_symbols != '[]'::jsonb"""
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# 1. 先删除触发器(否则函数不能替换)
|
||||
@@ -105,9 +78,8 @@ def upgrade() -> None:
|
||||
EXECUTE FUNCTION update_literature_search_tsv()
|
||||
""")
|
||||
|
||||
# 4. 回填已有数据的 search_tsv(仅更新有 chemical/gene 的行,减少 I/O)
|
||||
op.execute(_UPDATE_SQL)
|
||||
|
||||
# 注意:此处省略 search_tsv 回填。下一个 migration(g0h1i2j3k4l5)会做全量回填,
|
||||
# 包含 chemical/gene 公式,此处的部分回填是冗余的。
|
||||
|
||||
def downgrade() -> None:
|
||||
# 1. 删除触发器
|
||||
|
||||
@@ -1565,6 +1565,12 @@ async def _run_ftp_pipeline(db: AsyncSession) -> dict:
|
||||
await refresh_stats_cache()
|
||||
except Exception:
|
||||
pass
|
||||
# 管道运行后失效搜索缓存(新文献立即可见)
|
||||
from app.core.cache import cache
|
||||
try:
|
||||
await cache.invalidate_search_cache()
|
||||
except Exception:
|
||||
logger.exception("搜索缓存失效失败")
|
||||
return stats
|
||||
|
||||
|
||||
@@ -1601,6 +1607,11 @@ async def _run_eutils_pipeline(db: AsyncSession, mode: str, max_per_query: int,
|
||||
await refresh_stats_cache()
|
||||
except Exception:
|
||||
pass
|
||||
from app.core.cache import cache
|
||||
try:
|
||||
await cache.invalidate_search_cache()
|
||||
except Exception:
|
||||
logger.exception("搜索缓存失效失败")
|
||||
return stats
|
||||
|
||||
|
||||
|
||||
@@ -18,6 +18,7 @@ from app.models.user import User, UserSavedFilter
|
||||
from app.services.daily_digest import send_daily_digest_to_user
|
||||
from app.services.rule_engine import PRESET_RULES, RuleEngine
|
||||
from app.services.search_engine import AdvancedSearchEngine
|
||||
from app.core.constants import AGE_GROUPS, AGE_GROUP_UI_MAP
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
@@ -138,27 +139,6 @@ SPECIAL_MESH_UIS = {
|
||||
],
|
||||
}
|
||||
|
||||
# AGE 筛选层级定义:(key, label, mesh_uis, indent)
|
||||
AGE_GROUPS = [
|
||||
("child_0_18", "Child: birth-18 years", ["D007231", "D007223", "D002675", "D002648", "D000293"], 0),
|
||||
("newborn", "Newborn: birth-1 month", ["D007231"], 1),
|
||||
("infant_0_23", "Infant: birth-23 months", ["D007231", "D007223"], 1),
|
||||
("infant_1_23", "Infant: 1-23 months", ["D007223"], 1),
|
||||
("preschool", "Preschool Child: 2-5 years", ["D002675"], 1),
|
||||
("child_6_12", "Child: 6-12 years", ["D002648"], 1),
|
||||
("adolescent", "Adolescent: 13-18 years", ["D000293"], 1),
|
||||
("adult_19_plus", "Adult: 19+ years", ["D000328", "D008875", "D000368", "D000369"], 0),
|
||||
("young_adult", "Young Adult: 19-24 years", ["D055815"], 1),
|
||||
("adult_19_44", "Adult: 19-44 years", ["D000328"], 1),
|
||||
("middle_aged_plus","Middle Aged + Aged: 45+ years", ["D008875", "D000368", "D000369"], 1),
|
||||
("middle_aged", "Middle Aged: 45-64 years", ["D008875"], 1),
|
||||
("aged", "Aged: 65+ years", ["D000368", "D000369"], 1),
|
||||
("aged_80_plus", "80 and over: 80+ years", ["D000369"], 1),
|
||||
]
|
||||
|
||||
# 预构建 key → UIs 查找表(搜索时用)
|
||||
AGE_GROUP_UI_MAP: dict[str, list[str]] = {k: u for k, _, u, _ in AGE_GROUPS}
|
||||
|
||||
NLM_SUBSET_LABELS = {
|
||||
"AIM": "Core clinical journals",
|
||||
"M": "MEDLINE",
|
||||
@@ -274,7 +254,7 @@ async def advanced_search(req: AdvancedSearchRequest, db: AsyncSession = Depends
|
||||
return await AdvancedSearchEngine.search(db, **req.model_dump())
|
||||
except ValueError as ve:
|
||||
logger.warning("搜索参数错误: %s", ve)
|
||||
raise HTTPException(status_code=400, detail="搜索参数错误,请检查输入")
|
||||
raise HTTPException(status_code=400, detail=str(ve))
|
||||
except Exception as e:
|
||||
logger.exception("搜索服务内部错误")
|
||||
raise HTTPException(status_code=500, detail="搜索服务内部错误") from e
|
||||
|
||||
@@ -259,7 +259,18 @@ async def search_literature(
|
||||
return {"items": [], "total": 0}
|
||||
if len(q.split()) > 100:
|
||||
return {"items": [], "total": 0, "error": "查询词过多(最多 100 个词),请简化搜索条件"}
|
||||
# PubMed 语法检测与降级:普通搜索不支持字段标签和布尔符
|
||||
from app.services.pubmed_query_parser import is_pubmed_syntax
|
||||
if is_pubmed_syntax(q):
|
||||
import re as _pm_re
|
||||
q = _pm_re.sub(r'\[[\w/: -]+\]', '', q)
|
||||
q = _pm_re.sub(r'\b(AND|OR|NOT)\b', '', q)
|
||||
q = q.replace('"', '').replace('(', '').replace(')', '')
|
||||
q = ' '.join(q.split())
|
||||
if not q.strip():
|
||||
return {"items": [], "total": 0}
|
||||
offset = (page - 1) * page_size
|
||||
await db.execute(text("SET LOCAL statement_timeout = '30s'"))
|
||||
like = f"%{_escape_ilike(q)}%"
|
||||
# tsvector 主搜索 + ILIKE 兜底
|
||||
search_cond = or_(
|
||||
@@ -267,6 +278,21 @@ async def search_literature(
|
||||
GlobalLiterature.title.ilike(like),
|
||||
GlobalLiterature.abstract.ilike(like),
|
||||
)
|
||||
# 中文搜索:自动匹配 GlobalTag.name_zh → 注入标签条件
|
||||
import re as _cn_re
|
||||
_CHINESE_RE = _cn_re.compile(r'[一-鿿㐀-䶿豈-]')
|
||||
if _CHINESE_RE.search(q):
|
||||
_tag_matches = (await db.execute(
|
||||
select(GlobalTag.id).where(
|
||||
GlobalTag.source.in_(["mesh", "manual"]),
|
||||
GlobalTag.name_zh.ilike(like),
|
||||
).limit(100)
|
||||
)).scalars().all()
|
||||
if _tag_matches:
|
||||
_tag_lit_subq = select(GlobalLiteratureTag.literature_id).where(
|
||||
GlobalLiteratureTag.tag_id.in_([str(t) for t in _tag_matches])
|
||||
)
|
||||
search_cond = or_(search_cond, GlobalLiterature.id.in_(_tag_lit_subq))
|
||||
count_q = select(func.count(GlobalLiterature.id)).where(search_cond)
|
||||
total = (await db.execute(count_q)).scalar() or 0
|
||||
tsq = func.plainto_tsquery("english", q)
|
||||
|
||||
@@ -18,9 +18,12 @@ class CacheService:
|
||||
self._redis = None
|
||||
|
||||
async def _get_redis(self):
|
||||
"""尝试获取 Redis 连接,失败即降级(只尝试一次)"""
|
||||
"""尝试获取 Redis 连接,失败即降级(每 60 秒重试一次)"""
|
||||
if self._redis_failed:
|
||||
return None
|
||||
import time as _time
|
||||
if getattr(self, '_redis_retry_at', 0) > _time.monotonic():
|
||||
return None
|
||||
self._redis_failed = False
|
||||
if self._redis is not None:
|
||||
return self._redis
|
||||
try:
|
||||
@@ -33,7 +36,9 @@ class CacheService:
|
||||
except Exception:
|
||||
self._redis_failed = True
|
||||
self._redis = None
|
||||
logger.info("Redis 不可用,使用内存缓存")
|
||||
import time as _time
|
||||
self._redis_retry_at = _time.monotonic() + 60
|
||||
logger.info("Redis 不可用,使用内存缓存(60 秒后重试)")
|
||||
return None
|
||||
|
||||
async def get(self, key: str) -> dict | None:
|
||||
@@ -158,9 +163,41 @@ class CacheService:
|
||||
async def invalidate_user(self, user_id: str):
|
||||
await self.delete(f"user:{user_id}:profile")
|
||||
|
||||
async def invalidate_search_cache(self):
|
||||
"""管道运行后失效所有搜索相关缓存。"""
|
||||
await self.delete_pattern("search:advanced:*")
|
||||
await self.delete_pattern("search:year_counts:*")
|
||||
await self.delete("search:year_counts:all")
|
||||
await self.delete("journals:map")
|
||||
await self.delete_pattern("atm:*")
|
||||
|
||||
async def invalidate_tenant(self, tenant_id: str):
|
||||
await self.delete(f"tenant:{tenant_id}:plan")
|
||||
await self.delete(f"tenant:{tenant_id}:settings")
|
||||
|
||||
async def delete_pattern(self, pattern: str):
|
||||
"""Delete all keys matching a glob pattern.
|
||||
|
||||
Redis 模式用 SCAN 0 MATCH pattern 迭代删除。
|
||||
内存模式用 OrderedDict key 前缀匹配删除。
|
||||
"""
|
||||
r = await self._get_redis()
|
||||
if r:
|
||||
try:
|
||||
cursor = 0
|
||||
while True:
|
||||
cursor, keys = await r.scan(cursor, match=pattern, count=100)
|
||||
if keys:
|
||||
await r.delete(*keys)
|
||||
if cursor == 0:
|
||||
break
|
||||
except Exception:
|
||||
logger.exception("Redis SCAN/DEL failed for pattern: %s", pattern)
|
||||
else:
|
||||
import fnmatch
|
||||
to_delete = [k for k in self._store if fnmatch.fnmatch(k, pattern)]
|
||||
for k in to_delete:
|
||||
self._store.pop(k, None)
|
||||
|
||||
|
||||
cache = CacheService()
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
"""Shared constants used across the application."""
|
||||
|
||||
# ─── AGE 筛选层级定义 ───
|
||||
|
||||
AGE_GROUPS = [
|
||||
("child_0_18", "Child: birth-18 years", ["D007231", "D007223", "D002675", "D002648", "D000293"], 0),
|
||||
("newborn", "Newborn: birth-1 month", ["D007231"], 1),
|
||||
("infant_0_23", "Infant: birth-23 months", ["D007231", "D007223"], 1),
|
||||
("infant_1_23", "Infant: 1-23 months", ["D007223"], 1),
|
||||
("preschool", "Preschool Child: 2-5 years", ["D002675"], 1),
|
||||
("child_6_12", "Child: 6-12 years", ["D002648"], 1),
|
||||
("adolescent", "Adolescent: 13-18 years", ["D000293"], 1),
|
||||
("adult_19_plus", "Adult: 19+ years", ["D000328", "D008875", "D000368", "D000369"], 0),
|
||||
("young_adult", "Young Adult: 19-24 years", ["D055815"], 1),
|
||||
("adult_19_44", "Adult: 19-44 years", ["D000328"], 1),
|
||||
("middle_aged_plus","Middle Aged + Aged: 45+ years", ["D008875", "D000368", "D000369"], 1),
|
||||
("middle_aged", "Middle Aged: 45-64 years", ["D008875"], 1),
|
||||
("aged", "Aged: 65+ years", ["D000368", "D000369"], 1),
|
||||
("aged_80_plus", "80 and over: 80+ years", ["D000369"], 1),
|
||||
]
|
||||
|
||||
# 预构建 key → UIs 查找表(搜索时用)
|
||||
AGE_GROUP_UI_MAP: dict[str, list[str]] = {k: u for k, _, u, _ in AGE_GROUPS}
|
||||
@@ -34,7 +34,7 @@ PLANS = {
|
||||
"max_literature_items": 10_000,
|
||||
"max_saved_items": 10_000,
|
||||
"max_review_inclusions": 500,
|
||||
"api_quota_per_day": 1_000,
|
||||
"api_quota_per_day": 10_000,
|
||||
"teams": False,
|
||||
"approval_workflows": False,
|
||||
"sso": False,
|
||||
|
||||
@@ -16,6 +16,8 @@ from app.core.tenant_context import tenant_ctx
|
||||
BURST_MAX_PER_SECOND = 30
|
||||
# 滑动窗口时长(秒)
|
||||
BURST_WINDOW = 1
|
||||
# 每 N 次请求清理一次过期突发窗口
|
||||
BURST_CLEANUP_INTERVAL = 500
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -33,10 +35,15 @@ class RateLimitMiddleware(BaseHTTPMiddleware):
|
||||
self._lock = asyncio.Lock()
|
||||
# 突发保护:{tenant_id: [(timestamp, ...)]}
|
||||
self._burst_windows: dict[str, list[float]] = defaultdict(list)
|
||||
self._burst_cleanup_counter = 0
|
||||
|
||||
async def _get_redis(self):
|
||||
if self._redis is not None:
|
||||
return self._redis
|
||||
if self._redis is False:
|
||||
import time as _rt
|
||||
if getattr(self, '_redis_retry_at', 0) > _rt.monotonic():
|
||||
return None
|
||||
try:
|
||||
from redis.asyncio import Redis
|
||||
r = Redis.from_url(settings.REDIS_URL, decode_responses=True, socket_connect_timeout=1)
|
||||
@@ -44,6 +51,8 @@ class RateLimitMiddleware(BaseHTTPMiddleware):
|
||||
self._redis = r
|
||||
except Exception:
|
||||
self._redis = False
|
||||
import time as _rt
|
||||
self._redis_retry_at = _rt.monotonic() + 60
|
||||
return self._redis if self._redis else None
|
||||
|
||||
async def _get_quota(self, tenant_id: str) -> int:
|
||||
@@ -84,10 +93,30 @@ class RateLimitMiddleware(BaseHTTPMiddleware):
|
||||
window.append(now)
|
||||
return True
|
||||
|
||||
def _cleanup_stale_burst_windows(self):
|
||||
"""清理过期突发窗口条目,防止 _burst_windows 无限增长。"""
|
||||
now = time.time()
|
||||
cutoff = now - BURST_WINDOW
|
||||
stale_keys = []
|
||||
for tid, window in list(self._burst_windows.items()):
|
||||
# 移除过期时间戳
|
||||
while window and window[0] < cutoff:
|
||||
window.pop(0)
|
||||
if not window:
|
||||
stale_keys.append(tid)
|
||||
for k in stale_keys:
|
||||
del self._burst_windows[k]
|
||||
|
||||
async def dispatch(self, request: Request, call_next):
|
||||
if not request.url.path.startswith("/api/v1/"):
|
||||
return await call_next(request)
|
||||
|
||||
# 定期清理过期突发窗口条目,防止内存泄漏
|
||||
self._burst_cleanup_counter += 1
|
||||
if self._burst_cleanup_counter >= BURST_CLEANUP_INTERVAL:
|
||||
self._burst_cleanup_counter = 0
|
||||
self._cleanup_stale_burst_windows()
|
||||
|
||||
tid = tenant_ctx.get()
|
||||
if not tid:
|
||||
auth_header = request.headers.get("Authorization", "")
|
||||
|
||||
@@ -221,7 +221,7 @@ async def get_funnel(db: AsyncSession, tenant_id: uuid.UUID | None = None, days:
|
||||
|
||||
|
||||
async def get_retention(db: AsyncSession, tenant_id: uuid.UUID | None = None, weeks: int = 12):
|
||||
"""周级留存 cohort 分析(兼容 SQLite + PostgreSQL)"""
|
||||
"""周级留存 cohort 分析(PostgreSQL)"""
|
||||
from collections import defaultdict
|
||||
|
||||
since = datetime.now(UTC) - timedelta(weeks=weeks * 2)
|
||||
|
||||
@@ -22,6 +22,13 @@ from pathlib import Path
|
||||
# C04 树前缀(Neoplasms 及其子类)
|
||||
C04_TREE_PREFIXES = ("C04", "C04.")
|
||||
|
||||
# 交叉引入树号(非 C04 但高度肿瘤相关的 MeSH)
|
||||
CROSS_INCLUDE_PREFIXES = (
|
||||
"E02.319", # Antineoplastic agents — 抗癌药
|
||||
"E02.815", # Radiotherapy — 放疗
|
||||
"D27.505.954.248", # Antineoplastic agents (D27 subclass) — 抗肿瘤药
|
||||
)
|
||||
|
||||
# 标题/摘要关键词(全部生效,修复了之前 keywords[:3] 的 bug)
|
||||
ONCOLOGY_KEYWORDS = [
|
||||
"cancer", "carcinoma", "tumor", "tumour", "neoplasm", "malignancy",
|
||||
@@ -71,12 +78,20 @@ class MeshFilter:
|
||||
return self._tree_map.get(descriptor_name.strip().lower(), [])
|
||||
|
||||
def is_oncology_by_mesh(self, mesh_headings: list[dict]) -> bool:
|
||||
"""按 MeSH 判断是否肿瘤相关(基于 C04 树号)"""
|
||||
"""按 MeSH 判断是否肿瘤相关(C04 树号 + 交叉引入树号)
|
||||
|
||||
交叉引入树号覆盖:
|
||||
E02.319 — Antineoplastic agents(抗癌药)
|
||||
E02.815 — Radiotherapy(放疗)
|
||||
D27.505.954.248 — Antineoplastic agents(D27 子类,抗肿瘤药)
|
||||
"""
|
||||
for heading in mesh_headings:
|
||||
name = heading.get("descriptor", "") or heading.get("name", "")
|
||||
trees = self.get_tree_numbers(name)
|
||||
if any(t.startswith(C04_TREE_PREFIXES) for t in trees):
|
||||
return True
|
||||
if any(t.startswith(CROSS_INCLUDE_PREFIXES) for t in trees):
|
||||
return True
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
|
||||
@@ -242,9 +242,11 @@ def _parse_europe_pmc_article(art: dict) -> dict | None:
|
||||
"descriptor": desc, "ui": ui, "major": major, "qualifiers": qualifiers,
|
||||
})
|
||||
|
||||
# Keywords
|
||||
keywords = [kw for kw in
|
||||
(art.get("keywordList", {}).get("keyword") or []) if kw]
|
||||
# Keywords(Europe PMC 可能返回单字符串而非数组)
|
||||
_kw_val = art.get("keywordList", {}).get("keyword")
|
||||
if isinstance(_kw_val, str):
|
||||
_kw_val = [_kw_val]
|
||||
keywords = [kw for kw in (_kw_val or []) if kw]
|
||||
|
||||
# Publication types
|
||||
pub_types = []
|
||||
|
||||
@@ -392,40 +392,69 @@ class PubmedQueryParser:
|
||||
elif term.field is None:
|
||||
result.plain_terms.append(term)
|
||||
# ── 独立日期字段(非范围语法):"2024-01-01"[DP] → from=to=该日期 ──
|
||||
# 纯 4 位年份 "2024"[DP] 用 year_from/year_to,避免 fromisoformat 问题
|
||||
# is_not 时加入 negated_date_ranges,引擎据此 NOT 条件
|
||||
elif term.field == "DP":
|
||||
result.date_from = term.text
|
||||
result.date_to = term.text
|
||||
if term.text.isdigit() and len(term.text) == 4:
|
||||
result.year_from = int(term.text)
|
||||
result.year_to = int(term.text)
|
||||
else:
|
||||
result.date_from = term.text
|
||||
result.date_to = term.text
|
||||
if term.is_not:
|
||||
result.negated_date_ranges.add("DP")
|
||||
elif term.field == "EDAT":
|
||||
result.edat_from = term.text
|
||||
result.edat_to = term.text
|
||||
if term.text.isdigit() and len(term.text) == 4:
|
||||
result.year_from = int(term.text)
|
||||
result.year_to = int(term.text)
|
||||
else:
|
||||
result.edat_from = term.text
|
||||
result.edat_to = term.text
|
||||
if term.is_not:
|
||||
result.negated_date_ranges.add("EDAT")
|
||||
elif term.field == "CRDT":
|
||||
result.crdt_from = term.text
|
||||
result.crdt_to = term.text
|
||||
if term.text.isdigit() and len(term.text) == 4:
|
||||
result.year_from = int(term.text)
|
||||
result.year_to = int(term.text)
|
||||
else:
|
||||
result.crdt_from = term.text
|
||||
result.crdt_to = term.text
|
||||
if term.is_not:
|
||||
result.negated_date_ranges.add("CRDT")
|
||||
elif term.field == "MHDA":
|
||||
result.mhda_from = term.text
|
||||
result.mhda_to = term.text
|
||||
if term.text.isdigit() and len(term.text) == 4:
|
||||
result.year_from = int(term.text)
|
||||
result.year_to = int(term.text)
|
||||
else:
|
||||
result.mhda_from = term.text
|
||||
result.mhda_to = term.text
|
||||
if term.is_not:
|
||||
result.negated_date_ranges.add("MHDA")
|
||||
elif term.field == "LR":
|
||||
result.lr_from = term.text
|
||||
result.lr_to = term.text
|
||||
if term.text.isdigit() and len(term.text) == 4:
|
||||
result.year_from = int(term.text)
|
||||
result.year_to = int(term.text)
|
||||
else:
|
||||
result.lr_from = term.text
|
||||
result.lr_to = term.text
|
||||
if term.is_not:
|
||||
result.negated_date_ranges.add("LR")
|
||||
elif term.field == "DCOM":
|
||||
result.dcom_from = term.text
|
||||
result.dcom_to = term.text
|
||||
if term.text.isdigit() and len(term.text) == 4:
|
||||
result.year_from = int(term.text)
|
||||
result.year_to = int(term.text)
|
||||
else:
|
||||
result.dcom_from = term.text
|
||||
result.dcom_to = term.text
|
||||
if term.is_not:
|
||||
result.negated_date_ranges.add("DCOM")
|
||||
elif term.field == "DEP":
|
||||
result.dep_from = term.text
|
||||
result.dep_to = term.text
|
||||
if term.text.isdigit() and len(term.text) == 4:
|
||||
result.year_from = int(term.text)
|
||||
result.year_to = int(term.text)
|
||||
else:
|
||||
result.dep_from = term.text
|
||||
result.dep_to = term.text
|
||||
if term.is_not:
|
||||
result.negated_date_ranges.add("DEP")
|
||||
elif term.field == "__RANGE_DP__":
|
||||
|
||||
@@ -38,7 +38,7 @@ class AdvancedSearchEngine:
|
||||
tag_ids: list[str] | None,
|
||||
retracted: str, negative_result: str,
|
||||
is_oa: bool | None, language: str | None, languages: list[str] | None, nlm_subsets: list[str] | None,
|
||||
page_size: int, sort: str,
|
||||
page_size: int, sort: str, page: int = 1,
|
||||
# PubMed filter params
|
||||
has_abstract: bool | None = None,
|
||||
is_free_full_text: bool | None = None,
|
||||
@@ -66,7 +66,7 @@ class AdvancedSearchEngine:
|
||||
"r": retracted, "nr": negative_result,
|
||||
"oa": is_oa, "lang": language, "lgs": sorted(languages) if languages else [],
|
||||
"ns": sorted(nlm_subsets) if nlm_subsets else [],
|
||||
"ps": page_size, "s": sort,
|
||||
"ps": page_size, "s": sort, "p": page,
|
||||
"ha": has_abstract,
|
||||
"fft": is_free_full_text,
|
||||
"hft": has_full_text,
|
||||
@@ -185,7 +185,7 @@ class AdvancedSearchEngine:
|
||||
journal_tiers, pub_types, tag_ids,
|
||||
retracted, negative_result,
|
||||
is_oa, language, languages, nlm_subsets,
|
||||
page_size, sort,
|
||||
page_size, sort, page,
|
||||
has_abstract=has_abstract,
|
||||
is_free_full_text=is_free_full_text,
|
||||
has_full_text=has_full_text,
|
||||
@@ -216,8 +216,14 @@ class AdvancedSearchEngine:
|
||||
# 谢缓存:只存了 lit_ids + 聚合数据,需从 ②③ 缓存回填 tags/journal
|
||||
tier_map, name_map = await AdvancedSearchEngine._get_journal_map(db)
|
||||
items = await AdvancedSearchEngine._hydrate_items(db, cached["lit_ids"], tier_map, name_map)
|
||||
# year_counts 从独立 facet 缓存取
|
||||
year_counts = await _cache.get(_facet_cache_key) or []
|
||||
# year_counts 从独立 facet 缓存取(兼容 list 和 dict 两种格式)
|
||||
yc_cached = await _cache.get(_facet_cache_key)
|
||||
if isinstance(yc_cached, dict):
|
||||
year_counts = yc_cached.get("year_counts", [])
|
||||
elif isinstance(yc_cached, list):
|
||||
year_counts = yc_cached
|
||||
else:
|
||||
year_counts = []
|
||||
return {
|
||||
"items": items,
|
||||
"total": cached["total"],
|
||||
@@ -446,13 +452,19 @@ class AdvancedSearchEngine:
|
||||
|
||||
# 撤稿过滤
|
||||
if retracted == "no":
|
||||
conditions.append(GlobalLiterature.retracted == False)
|
||||
conditions.append(or_(
|
||||
GlobalLiterature.retracted == False,
|
||||
GlobalLiterature.retracted.is_(None),
|
||||
))
|
||||
elif retracted in ("only", "yes"):
|
||||
conditions.append(GlobalLiterature.retracted == True)
|
||||
|
||||
# 阴性结果过滤
|
||||
if negative_result == "no":
|
||||
conditions.append(GlobalLiterature.is_negative_result == False)
|
||||
conditions.append(or_(
|
||||
GlobalLiterature.is_negative_result == False,
|
||||
GlobalLiterature.is_negative_result.is_(None),
|
||||
))
|
||||
elif negative_result in ("only", "yes"):
|
||||
conditions.append(GlobalLiterature.is_negative_result == True)
|
||||
|
||||
@@ -483,7 +495,7 @@ class AdvancedSearchEngine:
|
||||
GlobalLiterature.abstract.isnot(None),
|
||||
GlobalLiterature.abstract != '',
|
||||
))
|
||||
if is_free_full_text:
|
||||
if is_free_full_text and is_oa is None:
|
||||
conditions.append(GlobalLiterature.is_oa == True)
|
||||
if has_full_text:
|
||||
conditions.append(GlobalLiterature.pmc_id.isnot(None))
|
||||
@@ -504,7 +516,7 @@ class AdvancedSearchEngine:
|
||||
for ui in sex
|
||||
]))
|
||||
if age:
|
||||
from app.api.v1.features import AGE_GROUP_UI_MAP
|
||||
from app.core.constants import AGE_GROUP_UI_MAP
|
||||
age_uis = []
|
||||
for val in age:
|
||||
if val in AGE_GROUP_UI_MAP:
|
||||
@@ -583,7 +595,7 @@ class AdvancedSearchEngine:
|
||||
except Exception:
|
||||
logger.exception("Year counts query failed")
|
||||
year_counts = []
|
||||
await _cache.set(_facet_cache_key, year_counts, ttl=1800)
|
||||
# 第 1 页结束时统一写 facet 缓存(含 total),此处不重复写入
|
||||
|
||||
# 排序(PubMed 查询时跳过 ts_rank,避免语法标签噪音)
|
||||
_relevance_query = query
|
||||
@@ -591,18 +603,18 @@ class AdvancedSearchEngine:
|
||||
# 用纯文本词做相关性排序,去掉 [field] 标签
|
||||
plain_parts = [t.text for t in _pubmed_parsed.plain_terms]
|
||||
plain_parts += [t.text for t in _pubmed_parsed.title_terms]
|
||||
plain_parts += [t.text for t in _pubmed_parsed.abstract_terms]
|
||||
plain_parts += [t.text for t in _pubmed_parsed.tiab_terms]
|
||||
_relevance_query = " ".join(plain_parts) if plain_parts else ""
|
||||
# ── 通用 keyset 分页(所有列式排序模式统一,代替 OFFSET) ──
|
||||
_keyset_cond = AdvancedSearchEngine._keyset_condition(sort, cursor_val, cursor_id)
|
||||
q = select(GlobalLiterature)
|
||||
if _keyset_cond is not None:
|
||||
conditions.append(_keyset_cond)
|
||||
elif sort not in AdvancedSearchEngine.KEYSET_COLUMN_SORTS and page > 1:
|
||||
# P1-F5: best_match/relevance 不支持 keyset,用 OFFSET 翻页
|
||||
elif page > 1:
|
||||
# P1-F5: keyset 条件不存在时(键集排序但游标无效/非键集排序),用 OFFSET 翻页
|
||||
q = q.offset((page - 1) * page_size)
|
||||
|
||||
# 重新构建查询
|
||||
q = select(GlobalLiterature)
|
||||
if conditions:
|
||||
q = q.where(and_(*conditions))
|
||||
q = q.order_by(*AdvancedSearchEngine._apply_order_by(sort, _relevance_query))
|
||||
@@ -722,6 +734,11 @@ class AdvancedSearchEngine:
|
||||
"journal_tier": tier_map.get(lit.journal_issn),
|
||||
"pub_types": lit.pub_types,
|
||||
"affiliation": authors[0].get("affiliation", "") if authors else "",
|
||||
"study_design": lit.study_design,
|
||||
"trial_reg": lit.trial_reg,
|
||||
"retracted": lit.retracted,
|
||||
"is_negative_result": lit.is_negative_result,
|
||||
"rct_detection": lit.rct_detection,
|
||||
})
|
||||
return results
|
||||
|
||||
@@ -779,13 +796,17 @@ class AdvancedSearchEngine:
|
||||
for fld, terms in field_map.items():
|
||||
if not terms:
|
||||
continue
|
||||
field_conds = []
|
||||
pos_conds = []
|
||||
neg_conds = []
|
||||
for term in terms:
|
||||
cond = AdvancedSearchEngine._field_condition(fld, term.text, term.exact)
|
||||
if term.is_not:
|
||||
cond = not_(cond)
|
||||
field_conds.append(cond)
|
||||
term_conditions.append(field_combine(*field_conds) if len(field_conds) > 1 else field_conds[0])
|
||||
neg_conds.append(not_(cond))
|
||||
else:
|
||||
pos_conds.append(cond)
|
||||
if pos_conds:
|
||||
term_conditions.append(field_combine(*pos_conds) if len(pos_conds) > 1 else pos_conds[0])
|
||||
term_conditions.extend(neg_conds)
|
||||
|
||||
# 2. 纯文本词(无字段标签)— P0-2: 对无标签词补充 ATM MeSH 展开
|
||||
# P0-F1: ATM 只展开肯定词,否定词独立 AND,避免被 ATM OR 短路
|
||||
@@ -1045,7 +1066,9 @@ class AdvancedSearchEngine:
|
||||
for t in subl:
|
||||
val = t.text.upper()
|
||||
if val == "PUBMED":
|
||||
continue # no-op: 所有记录都是 PubMed
|
||||
if is_neg:
|
||||
term_conditions.append(text("FALSE"))
|
||||
continue
|
||||
elif val == "MEDLINE":
|
||||
cond = GlobalLiterature.citation_status == "medline"
|
||||
elif val.isalpha():
|
||||
@@ -1085,23 +1108,39 @@ class AdvancedSearchEngine:
|
||||
# 处理括号分组的词(保留 OR/AND 嵌套结构,P2-2)
|
||||
if pp.groups:
|
||||
for idx, group in enumerate(pp.groups):
|
||||
group_conds = []
|
||||
all_not = all(t.is_not for t in group)
|
||||
g_pos = []
|
||||
g_neg = []
|
||||
for t in group:
|
||||
cond = await AdvancedSearchEngine._single_term_condition(db, t)
|
||||
if cond is not None:
|
||||
if not all_not and t.is_not:
|
||||
cond = not_(cond)
|
||||
group_conds.append(cond)
|
||||
if group_conds:
|
||||
gop = (pp.group_operators[idx]
|
||||
if idx < len(pp.group_operators)
|
||||
else "and")
|
||||
combine_fn = or_ if gop == "or" else and_
|
||||
combined = combine_fn(*group_conds) if len(group_conds) > 1 else group_conds[0]
|
||||
if all_not and len(group_conds) >= 1:
|
||||
combined = not_(combined)
|
||||
term_conditions.append(combined)
|
||||
if all_not:
|
||||
g_neg.append(cond) # raw condition,外部统一 not_()
|
||||
elif t.is_not:
|
||||
g_neg.append(not_(cond))
|
||||
else:
|
||||
g_pos.append(cond)
|
||||
gop = (pp.group_operators[idx]
|
||||
if idx < len(pp.group_operators)
|
||||
else "and")
|
||||
combine_fn = or_ if gop == "or" else and_
|
||||
|
||||
if all_not:
|
||||
# NOT(A OR B): single UnaryExpression → 顶层 OR/NOT 分离时被检测为 neg → 独立 AND
|
||||
if g_neg:
|
||||
combined = combine_fn(*g_neg) if len(g_neg) > 1 else g_neg[0]
|
||||
term_conditions.append(not_(combined))
|
||||
else:
|
||||
# 混合/正组:将 pos 和 neg 按组操作符组合,保留组内结构
|
||||
# 避免 neg 被 OR/NOT 分离拉出来破坏语义
|
||||
combined = None
|
||||
if g_pos:
|
||||
combined = combine_fn(*g_pos) if len(g_pos) > 1 else g_pos[0]
|
||||
if g_neg:
|
||||
neg_combined = combine_fn(*g_neg) if len(g_neg) > 1 else g_neg[0]
|
||||
combined = combine_fn(combined, neg_combined) if combined is not None else neg_combined
|
||||
if combined is not None:
|
||||
term_conditions.append(combined)
|
||||
|
||||
# 将 term_conditions 加入 conditions
|
||||
if term_conditions:
|
||||
@@ -1227,7 +1266,7 @@ class AdvancedSearchEngine:
|
||||
try:
|
||||
return GlobalLiterature.pmid == int(term.text)
|
||||
except ValueError:
|
||||
return None
|
||||
return GlobalLiterature.doi.ilike(f"%{_escape_ilike(term.text)}%")
|
||||
if field == "DOI":
|
||||
return GlobalLiterature.doi.ilike(f"%{_escape_ilike(term.text)}%")
|
||||
if field == "GR":
|
||||
@@ -1262,7 +1301,7 @@ class AdvancedSearchEngine:
|
||||
return text("TRUE") # no-op: 所有记录都是 PubMed
|
||||
elif val == "MEDLINE":
|
||||
return GlobalLiterature.citation_status == "medline"
|
||||
elif len(val) == 1 and val.isalpha():
|
||||
elif val.isalpha():
|
||||
subq = select(GlobalJournal.issn).where(
|
||||
GlobalJournal.nlm_subsets.overlap([val])
|
||||
)
|
||||
@@ -1304,7 +1343,14 @@ class AdvancedSearchEngine:
|
||||
elif field == "abstract":
|
||||
return GlobalLiterature.abstract.ilike(_pt())
|
||||
elif field == "author":
|
||||
return GlobalLiterature.author_names_text.ilike(_pt())
|
||||
pat = _pt()
|
||||
return or_(
|
||||
text(
|
||||
"EXISTS (SELECT 1 FROM jsonb_array_elements(global_literature.authors) AS _a "
|
||||
"WHERE _a->>'family' ILIKE :author_pat)"
|
||||
).bindparams(author_pat=pat),
|
||||
GlobalLiterature.author_names_text.ilike(pat),
|
||||
)
|
||||
elif field == "journal":
|
||||
pat = _pt()
|
||||
return or_(
|
||||
@@ -1342,6 +1388,9 @@ class AdvancedSearchEngine:
|
||||
return or_(
|
||||
GlobalLiterature.title.ilike(pat),
|
||||
GlobalLiterature.abstract.ilike(pat),
|
||||
GlobalLiterature.author_names_text.ilike(pat),
|
||||
GlobalLiterature.journal.ilike(pat),
|
||||
GlobalLiterature.journal_iso.ilike(pat),
|
||||
cast(GlobalLiterature.pmid, String).ilike(pat),
|
||||
GlobalLiterature.doi.ilike(pat),
|
||||
)
|
||||
@@ -1356,14 +1405,25 @@ class AdvancedSearchEngine:
|
||||
GlobalLiterature.title.ilike(like_val),
|
||||
cast(GlobalLiterature.pmid, String).ilike(like_val),
|
||||
GlobalLiterature.doi.ilike(like_val),
|
||||
GlobalLiterature.abstract.ilike(like_val),
|
||||
GlobalLiterature.author_names_text.ilike(like_val),
|
||||
GlobalLiterature.journal.ilike(like_val),
|
||||
)
|
||||
# P7-D2: Chinese → ILIKE fallback (tsvector is English-only)
|
||||
if re.search(r'[一-鿿㐀-䶿豈-]', term):
|
||||
return or_(
|
||||
GlobalLiterature.title.ilike(like_val),
|
||||
GlobalLiterature.abstract.ilike(like_val),
|
||||
GlobalLiterature.author_names_text.ilike(like_val),
|
||||
GlobalLiterature.journal.ilike(like_val),
|
||||
)
|
||||
return GlobalLiterature.search_tsv.op("@@")(func.plainto_tsquery("english", term))
|
||||
# tsvector 索引主覆盖 title/abstract/author_names/chemicals/genes/mesh/keywords
|
||||
# journal/journal_iso/affiliation 不在 tsvector 中,以 ILIKE 兜底
|
||||
return or_(
|
||||
GlobalLiterature.search_tsv.op("@@")(func.plainto_tsquery("english", term)),
|
||||
GlobalLiterature.journal.ilike(like_val),
|
||||
GlobalLiterature.journal_iso.ilike(like_val),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
async def _expand_mesh_tag_ids(
|
||||
@@ -1408,6 +1468,7 @@ class AdvancedSearchEngine:
|
||||
q = m.strip().lower()
|
||||
if not q:
|
||||
continue
|
||||
# entry_terms 在 import_mesh_full.py 时已统一小写,可安全用 @> 精确匹配
|
||||
entry_conds.append(GlobalTag.entry_terms.contains([q]))
|
||||
name_conds.append(GlobalTag.name_en.ilike(_escape_ilike(m)))
|
||||
|
||||
@@ -1516,11 +1577,15 @@ class AdvancedSearchEngine:
|
||||
rank = func.ts_rank(GlobalLiterature.search_tsv, tsq)
|
||||
return [rank.desc(), GlobalLiterature.id.desc()]
|
||||
elif sort == "first_author":
|
||||
return [GlobalLiterature.authors[0]['family'].astext.asc().nullslast()]
|
||||
return [GlobalLiterature.authors[0]['family'].astext.asc().nullslast(),
|
||||
GlobalLiterature.id.asc()]
|
||||
elif sort == "journal":
|
||||
return [GlobalLiterature.journal.asc().nullslast()]
|
||||
return [GlobalLiterature.journal.asc().nullslast(),
|
||||
GlobalLiterature.journal_iso.asc().nullslast(),
|
||||
GlobalLiterature.id.asc()]
|
||||
elif sort == "title":
|
||||
return [GlobalLiterature.title.asc().nullslast()]
|
||||
return [GlobalLiterature.title.asc().nullslast(),
|
||||
GlobalLiterature.id.asc()]
|
||||
else:
|
||||
return [GlobalLiterature.pub_date.desc().nullslast(), GlobalLiterature.id.desc()]
|
||||
|
||||
@@ -1544,34 +1609,50 @@ class AdvancedSearchEngine:
|
||||
return None
|
||||
|
||||
try:
|
||||
if cursor_val == "__NULL__":
|
||||
if sort == "date":
|
||||
return and_(GlobalLiterature.pub_date.is_(None), GlobalLiterature.id < uid)
|
||||
elif sort == "cited":
|
||||
return and_(GlobalLiterature.cited_by_count.is_(None), GlobalLiterature.id < uid)
|
||||
elif sort == "title":
|
||||
return and_(GlobalLiterature.title.is_(None), GlobalLiterature.id > uid)
|
||||
elif sort == "journal":
|
||||
return and_(GlobalLiterature.journal.is_(None), GlobalLiterature.id > uid)
|
||||
elif sort == "first_author":
|
||||
return and_(GlobalLiterature.authors[0]['family'].astext.is_(None), GlobalLiterature.id > uid)
|
||||
if sort == "date":
|
||||
from datetime import date as dt_date
|
||||
val = dt_date.fromisoformat(cursor_val)
|
||||
return or_(
|
||||
GlobalLiterature.pub_date < val,
|
||||
and_(GlobalLiterature.pub_date == val, GlobalLiterature.id < uid),
|
||||
and_(GlobalLiterature.pub_date.is_(None), GlobalLiterature.id < uid),
|
||||
)
|
||||
elif sort == "cited":
|
||||
val = int(cursor_val)
|
||||
return or_(
|
||||
GlobalLiterature.cited_by_count < val,
|
||||
and_(GlobalLiterature.cited_by_count == val, GlobalLiterature.id < uid),
|
||||
and_(GlobalLiterature.cited_by_count.is_(None), GlobalLiterature.id < uid),
|
||||
)
|
||||
elif sort == "title":
|
||||
return or_(
|
||||
GlobalLiterature.title > cursor_val,
|
||||
and_(GlobalLiterature.title == cursor_val, GlobalLiterature.id > uid),
|
||||
and_(GlobalLiterature.title.is_(None), GlobalLiterature.id > uid),
|
||||
)
|
||||
elif sort == "journal":
|
||||
return or_(
|
||||
GlobalLiterature.journal > cursor_val,
|
||||
and_(GlobalLiterature.journal == cursor_val, GlobalLiterature.id > uid),
|
||||
and_(GlobalLiterature.journal.is_(None), GlobalLiterature.id > uid),
|
||||
)
|
||||
elif sort == "first_author":
|
||||
family_col = GlobalLiterature.authors[0]['family'].astext
|
||||
return or_(
|
||||
family_col > cursor_val,
|
||||
and_(family_col == cursor_val, GlobalLiterature.id > uid),
|
||||
and_(family_col.is_(None), GlobalLiterature.id > uid),
|
||||
)
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
@@ -1582,14 +1663,16 @@ class AdvancedSearchEngine:
|
||||
"""从末尾条目标提取 keyset 游标值。"""
|
||||
if sort == "date":
|
||||
val = str(lit.pub_date or lit.article_date or "")
|
||||
return val if val else None
|
||||
return val if val else "__NULL__"
|
||||
elif sort == "cited":
|
||||
return str(lit.cited_by_count) if lit.cited_by_count is not None else None
|
||||
if lit.cited_by_count is not None:
|
||||
return str(lit.cited_by_count)
|
||||
return "__NULL__"
|
||||
elif sort == "title":
|
||||
return lit.title or None
|
||||
return lit.title or "__NULL__"
|
||||
elif sort == "journal":
|
||||
return lit.journal or None
|
||||
return lit.journal or "__NULL__"
|
||||
elif sort == "first_author":
|
||||
authors = lit.authors or []
|
||||
return authors[0].get("family") if authors else None
|
||||
return authors[0].get("family") if authors else "__NULL__"
|
||||
return None
|
||||
|
||||
@@ -25,6 +25,9 @@ async def shutdown(ctx):
|
||||
async def daily_ftp_update(ctx):
|
||||
"""每日 FTP 增量更新(取代旧的精搜+宽搜+retagger)"""
|
||||
stats = await run_daily_ftp_update(ctx)
|
||||
# 搜索缓存失效:管道新增/修改文献后,缓存中的搜索结果立即过时
|
||||
from app.core.cache import cache
|
||||
await cache.invalidate_search_cache()
|
||||
return stats if isinstance(stats, dict) else {"status": "ok"}
|
||||
|
||||
|
||||
|
||||
@@ -9,7 +9,7 @@ pubmed_filter:
|
||||
mesh_cross_include:
|
||||
- "E02.319" # Antineoplastic agents
|
||||
- "E02.815" # Radiotherapy
|
||||
- "D27.505" # Antineoplastic drugs
|
||||
- "D27.505.954.248" # Antineoplastic agents (D27 subclass)
|
||||
pub_type_prefer:
|
||||
- "Guideline"
|
||||
- "Randomized Controlled Trial"
|
||||
|
||||
@@ -224,6 +224,25 @@ async def apply():
|
||||
merged += 1
|
||||
print(f" 合并: {tag.name_zh}/{tag.name_en} ← {dup_tag.name_en} (移动 {moved} 篇)")
|
||||
|
||||
# 更新 tag_ids 数组:移除已删除标签的 ID,确保种子标签 ID 存在
|
||||
from sqlalchemy import text as _sa_text
|
||||
await db.execute(
|
||||
_sa_text("""
|
||||
UPDATE global_literature
|
||||
SET tag_ids = ARRAY(
|
||||
SELECT DISTINCT unnest(
|
||||
array_append(
|
||||
array_remove(tag_ids, :dup_id::uuid),
|
||||
:seed_id::uuid
|
||||
)
|
||||
)
|
||||
)
|
||||
WHERE tag_ids @> ARRAY[:dup_id::uuid]
|
||||
"""),
|
||||
{"dup_id": str(dup_tag.id), "seed_id": str(tag.id)}
|
||||
)
|
||||
print(f" tag_ids 数组已同步")
|
||||
|
||||
await db.commit()
|
||||
print(f"\n完成! 合并 {merged} 个重复标签")
|
||||
|
||||
|
||||
@@ -330,27 +330,32 @@ def _extract_article(elem) -> dict | None:
|
||||
|
||||
|
||||
def _is_oncology(article: dict) -> bool:
|
||||
"""判断一篇文献是否属于肿瘤学范畴。
|
||||
"""判断一篇文献是否应收录。
|
||||
|
||||
两层过滤:
|
||||
Tier 1 — 有 MeSH 标引 → 仅检查 C04 树号(精确),不检查文本
|
||||
此规则基于数据分析:23.6 万条现有非 C04 误判文献中,
|
||||
有 MeSH 再检查文本会导致大量假阳性。
|
||||
三层过滤:
|
||||
Step 0 — 排除出版类型:
|
||||
Editorial, Letter, News, Comment → 不收录
|
||||
Tier 1 — 有 MeSH 标引 → 检查 C04 树号
|
||||
交叉引入树号(E02.319 抗癌药、E02.815 放疗、D27.505.954.248 抗肿瘤药)
|
||||
不检查文本(避免假阳性)
|
||||
Tier 2 — 无 MeSH 标引 → 关键词评分:
|
||||
- 标题命中任一关键词 → 收录(强信号)
|
||||
- 摘要 ≥2 关键词命中 → 收录
|
||||
- 摘要 0-1 关键词 → 放过(等待标引后回查)
|
||||
- 标题命中任一关键词 → 收录(强信号)
|
||||
- 摘要 ≥2 关键词命中 → 收录
|
||||
- 摘要 0-1 关键词 → 放过(等待标引后回查)
|
||||
"""
|
||||
mf = get_mesh_filter()
|
||||
mesh_headings = article.get('mesh_headings', [])
|
||||
# Step 0: 排除出版类型
|
||||
EXCLUDED_TYPES = {"Editorial", "Letter", "News", "Comment"}
|
||||
if any(pt in EXCLUDED_TYPES for pt in article.get("pub_types", [])):
|
||||
return False
|
||||
|
||||
mesh_headings = article.get("mesh_headings", [])
|
||||
if mesh_headings:
|
||||
# Tier 1: 有 MeSH → 只看 C04 树号
|
||||
return mf.is_oncology_by_mesh(mesh_headings)
|
||||
# Tier 1: 有 MeSH → C04 树号 + 交叉引入
|
||||
return get_mesh_filter().is_oncology_by_mesh(mesh_headings)
|
||||
|
||||
# Tier 2: 无 MeSH → 关键词评分模式
|
||||
return mf.is_oncology_by_text_score(
|
||||
article.get('title'), article.get('abstract')
|
||||
# Tier 2: 无 MeSH → 关键词评分
|
||||
return get_mesh_filter().is_oncology_by_text_score(
|
||||
article.get("title"), article.get("abstract")
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -45,7 +45,7 @@
|
||||
|
||||
| ID | 项目 | 严重度 | 详情 |
|
||||
|----|------|--------|------|
|
||||
| **P0-4** | tsvector 扩展 | **中** | 迁移 `g0h1i2j3k4l5` 已创建但生产未 apply。添加了 mesh_headings + keywords(weight C)到 tsvector。本地 SQLite 不执行此迁移。生产需手动 `alembic upgrade head` |
|
||||
| **P0-4** | tsvector 扩展 | **中** | 迁移 `g0h1i2j3k4l5` 已创建但生产未 apply。添加了 mesh_headings + keywords(weight C)到 tsvector。需手动 `alembic upgrade head` |
|
||||
| **P1-6** | Affiliation [AD] | **低** | 仍用 `cast(authors, String).ilike()`。JSONB 结构文本(键名)可能产生假阳性。完整修复需 schema 变更 + 迁移。目前实际影响极小(搜索医院名/机构名极少与 JSONB 键名冲突) |
|
||||
| **P2-3** | 布尔优先级 | **低** | 递归下降 parser 正确解析 `A OR B AND C` = `A OR (B AND C)`。但扁平 term_conditions 列表丢失嵌套结构。没有 PubMed 的复杂布尔优先级测试失败案例,仅理论不足 |
|
||||
| **P2-4** | 精确短语非 "all" | **极低** | 对非 "all" 字段,exact=True 与 =False 生成相同 ILIKE。但这是功能正确的——ILIKE 本身不做词干化。有意为之,不影响结果 |
|
||||
|
||||
@@ -69,7 +69,7 @@ class TestGetQuota:
|
||||
mock_session.execute.return_value = mock_result
|
||||
|
||||
q = await mw._get_quota("t1")
|
||||
assert q == 1000 # Pro plan
|
||||
assert q == 10000 # Pro plan
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_db_exception_fallback(self):
|
||||
|
||||
@@ -68,6 +68,20 @@ class TestMeshByMesh:
|
||||
]
|
||||
assert mf.is_oncology_by_mesh(headings) is True
|
||||
|
||||
def test_cross_include_antineoplastic(self, mf):
|
||||
"""交叉引入:Antineoplastic Agents (D27.505.954.248) → True"""
|
||||
assert mf.is_oncology_by_mesh([{"descriptor": "Antineoplastic Agents"}]) is True
|
||||
|
||||
def test_cross_include_radiotherapy(self, mf):
|
||||
"""交叉引入:Radiotherapy (E02.815) → True"""
|
||||
assert mf.is_oncology_by_mesh([{"descriptor": "Radiotherapy"}]) is True
|
||||
|
||||
def test_cross_include_not_oncology(self, mf):
|
||||
"""非肿瘤交叉引入 → False(Antibiotics D27.505.954.122 下有但非 E02.319/D27.505.954.248 下位)"""
|
||||
# Antibiotics 在 D27.505.954.122 下位但不在抗肿瘤范畴,由具体子类决定
|
||||
# 用明确非肿瘤的 MeSH 验证
|
||||
assert mf.is_oncology_by_mesh([{"descriptor": "Anti-Bacterial Agents"}]) is False
|
||||
|
||||
|
||||
class TestMeshByText:
|
||||
def test_cancer_in_title(self):
|
||||
|
||||
Reference in New Issue
Block a user