backend/backend/app/services/pubmed_query_parser.py

"""PubMed 查询语法解析器

将用户输入的 PubMed 风格查询（如 `"lung cancer"[TI] AND "pembrolizumab"[TIAB]`）
解析为结构化数据，供 AdvancedSearchEngine.search() 消费。

支持的语法：
  - 字段标签 (case-insensitive): [TI] [AB] [TIAB] [AU] [TA] [MH] [MAJR] [PT] [DP] [PMID] [DOI]
  - 布尔运算符: AND / OR / NOT（大小写不敏感，前后需词边界）
  - 引号短语: "exact phrase"
  - 括号分组: (a OR b) AND c
  - 日期范围: 2024:2026[DP]

设计原则：
  - 向后兼容：不含特殊语法的纯文本查询原样返回为 plain_terms
  - 容错：语法错误时退化到 plain_terms，不抛异常
  - 组合：解析结果映射到 AdvancedSearchEngine.search() 的已有参数
"""

from __future__ import annotations

import re
import unicodedata
from dataclasses import dataclass, field
from enum import Enum, auto

# P11: partial date YYYY-MM pattern, expanded to full month range
_PARTIAL_DATE_RE = re.compile(r'^\d{4}-\d{2}$')
_LAST_DAY = {1:31, 2:29, 3:31, 4:30, 5:31, 6:30, 7:31, 8:31, 9:30, 10:31, 11:30, 12:31}


def _is_leap_year(year: int) -> bool:
    """Check if year is a leap year."""
    return (year % 4 == 0 and year % 100 != 0) or (year % 400 == 0)


def _validate_date_str(s: str) -> bool:
    """Validate that a string is a valid calendar date: YYYY or YYYY-MM-DD."""
    if s.isdigit() and len(s) == 4:
        return True
    if len(s) == 10 and s[4] == '-' and s[7] == '-' and s.replace('-', '').isdigit():
        try:
            y, m, d = int(s[:4]), int(s[5:7]), int(s[8:10])
            if m < 1 or m > 12 or d < 1:
                return False
            last = _LAST_DAY.get(m, 31)
            if m == 2 and last == 29 and not _is_leap_year(y):
                last = 28
            return d <= last
        except (ValueError, IndexError):
            return False
    return False


def _expand_partial_date(text: str) -> tuple[str, str]:
    """Expand YYYY-MM to full month range (YYYY-MM-01 to YYYY-MM-last_day)."""
    date_from = f"{text}-01"
    y, m = text.split("-")
    month = int(m)
    if month < 1 or month > 12:
        # P12: invalid month → return year-only range
        return f"{y}-01-01", f"{y}-12-31"
    last_day = _LAST_DAY.get(month, 31)
    if month == 2 and last_day == 29 and not _is_leap_year(int(y)):
        last_day = 28
    date_to = f"{y}-{m}-{last_day}"
    return date_from, date_to


def _normalize_field_label(raw: str) -> str | None:
    """Normalize raw PubMed field label to internal field name. (P12)"""
    if raw == "MH:NOEXP":
        return "MH"
    if raw in _FIELD_TAG_MAP:
        return _FIELD_TAG_MAP[raw]
    if raw in _SPECIAL_FIELDS:
        return raw
    return None


# ─── 查询复杂度限制 ───
MAX_TERMS = 100      # P4-1: 放宽到 100 词（原 50 词）
MAX_PAREN_DEPTH = 10  # 括号嵌套最深层数


# ─── 字段标签映射 ───

_FIELD_TAG_MAP: dict[str, str] = {
    "TI": "title",
    "AB": "abstract",
    "TIAB": "all",
    "AU": "author",
    "TA": "journal",
    "JT": "journal",         # P1-2: Journal Title (同 TA)
    "AD": "affiliation",
    "CN": "author",
    "FAU": "author",
    "LAU": "author",
    "TW": "all",
    "ALL": "all",            # P3-7: [ALL] = 全部字段
    "MESH": "MH",            # P1-2: [MESH] 是 [MH] 的别名
    # 新增标量字段
    "LA": "language",
    "VI": "volume",
    "IP": "issue",
    "PG": "pages",
    "LID": "lid",
    # P4 新增字段标签
    "Title/Abstract": "all",   # [Title/Abstract] 长标签 → all
    "TITLE/ABSTRACT": "all",  # 解析器 .upper() 后的大写版本
    "OAB": "all",              # [OAB] Other Abstract → all
    "WORD": "all",             # [WORD] Word in text → all
    "FI": "GR",                # [FI] Funder Identifier → 同 GR（grant_id）
    "SO": "journal",           # [SO] Source → journal（近似）
    "PL": "journal",           # [PL] Place of Publication → journal（近似）
}

# 需要特殊处理的字段（不直接映射到 field 参数）
_SPECIAL_FIELDS = {
    "MH", "MAJR", "PT", "DP", "PMID", "DOI",
    # 日期字段
    "EDAT", "CRDT", "MHDA", "LR", "DCOM",
    # P1-2: 电子出版日期
    "DEP",
    # JSONB 字段
    "GR", "SH", "RN", "NM", "SI", "PA",
    # 待补充抽取的字段（标记为特殊以便未来实现）
    "AUID", "COIS", "ED", "IR", "PS", "PUBN", "TT",
    # P1-2: 新字段
    "SB", "STAT", "UID",
    # P4 新增：独立语义字段
    "OT",              # [OT] → keywords JSONB（不再映射到 all）
    "GEN",             # [GEN] → gene_symbols JSONB
    "PMC",             # [PMC] → pmc_id
}

# 支持日期范围语法的字段
_DATE_RANGE_FIELDS = {"DP", "EDAT", "CRDT", "MHDA", "LR", "DCOM", "DEP"}

# 所有合法字段标签（PubMed 全量字段）
# P1-1: 移除了 BOOK/FILTER/ISBN（未实现，降级为 plain text 不如报错透明）
_ALL_FIELD_TAGS = {
    "AB", "AD", "ALL", "AU", "CN", "FAU", "AUID", "LAU", "COIS",
    "DCOM", "CRDT", "EDAT", "MHDA", "LR", "DP", "DOI",
    "DEP",                               # P1-2: Date of Electronic Publication
    "RN", "ED", "GR", "IR", "IP",
    "TA", "JT", "LA", "LID", "MAJR", "SH", "MH", "MH:NOEXP", "OT", "PG",
    "PA", "PT", "PMID", "PUBN", "SI", "PS", "NM", "TW",
    "SB", "STAT", "UID",                 # P1-2: Subset, Status, UID
    "MESH",                              # P1-2: [MH] 别名
    "TI", "TIAB", "TT", "VI",
    # P4 新增字段标签
    "Title/Abstract",                    # [Title/Abstract] 长标签
    "OAB", "WORD",                       # [OAB] Other Abstract, [WORD] Word in text
    "FI", "GEN", "PMC", "SO", "PL",     # [FI] Funder, [GEN] Gene, [PMC] PMCID, [SO] Source, [PL] Place
}


# ─── Token Types ───

class TokenType(Enum):
    QUOTED = auto()       # "text in quotes"
    FIELD = auto()        # [TI], [MH], etc.
    AND = auto()          # AND operator
    OR = auto()           # OR operator
    NOT = auto()          # NOT operator
    LPAREN = auto()       # (
    RPAREN = auto()       # )
    COLON = auto()        # :
    WORD = auto()         # unquoted text
    NUMBER = auto()       # digits (for PMID, year)
    DATE = auto()         # YYYY-MM-DD (normalized from YYYY/MM/DD)
    UNKNOWN_FIELD = auto() # [...] 但内容不是已知字段标签
    EOF = auto()          # end of input


@dataclass
class Token:
    type: TokenType
    value: str = ""


# ─── Tokeniser ───

_TOKEN_PATTERNS: list[tuple[TokenType, str]] = [
    (TokenType.QUOTED, r'"(?:[^"\\]|\\.)*"'),
    (TokenType.FIELD, r'\[(?:' + '|'.join(_ALL_FIELD_TAGS) + r')\]'),
    (TokenType.UNKNOWN_FIELD, r'\[[^\]]*\]'),  # 未识别的字段标签 → 触发降级
    (TokenType.AND, r'\bAND\b'),
    (TokenType.OR, r'\bOR\b'),
    (TokenType.NOT, r'\bNOT\b'),
    (TokenType.LPAREN, r'\('),
    (TokenType.RPAREN, r'\)'),
    (TokenType.COLON, r':'),
    (TokenType.DATE, r'\d{4}-\d{2}-\d{2}'),
    (TokenType.NUMBER, r'\d+'),
    (TokenType.WORD, r'[^\s"\[\]():]+'),
]

_TOKEN_RE = re.compile(
    '|'.join(f'(?P<{t.name}>{p})' for t, p in _TOKEN_PATTERNS),
    re.IGNORECASE | re.ASCII,
)


def tokenise(query: str) -> list[Token]:
    """将 PubMed 查询字符串分片为 Token 列表。"""
    tokens: list[Token] = []
    last_end = 0
    for m in _TOKEN_RE.finditer(query):
        # P5: 检测未匹配的字符（不在任何 token 模式中的字符被静默丢弃）
        if m.start() > last_end:
            gap = query[last_end:m.start()]
            if gap.strip():
                tokens.append(Token(TokenType.WORD, gap.strip()))
                if len(tokens) > MAX_TERMS:
                    raise ParseError(f"查询词过多（超过 {MAX_TERMS} 个），降级为简单文本搜索")
        last_end = m.end()
        for name, value in m.groupdict().items():
            if value is not None:
                ttype = TokenType[name]
                if ttype == TokenType.UNKNOWN_FIELD:
                    fname = value.strip('[]').upper()
                    # P0-1: 不认识字段标签时降级为 WORD，不终止解析
                    tokens.append(Token(TokenType.WORD, value.strip('[]')))
                    if len(tokens) > MAX_TERMS:
                        raise ParseError(f"查询词过多（超过 {MAX_TERMS} 个），降级为简单文本搜索")
                    continue
                tokens.append(Token(ttype, value))
                if len(tokens) > MAX_TERMS:
                    raise ParseError(f"查询词过多（超过 {MAX_TERMS} 个），降级为简单文本搜索")
    tokens.append(Token(TokenType.EOF))
    return tokens


# ─── AST Nodes ───

@dataclass
class Term:
    """单个搜索词项。"""
    text: str
    exact: bool = False          # True if quoted phrase
    field: str | None = None     # None = plain (no field tag); "title"/"abstract"/etc for mapped; "MH"/"PT"/etc for special
    is_not: bool = False         # True if preceded by NOT
    group_id: int = -1           # -1 = top-level, >= 0 = parenthesized group index
    _noexp: bool = False         # P1-4: [MH:noexp] 抑制树展开


@dataclass
class ParsedPubmedQuery:
    """PubMed 查询解析后的结构化表示。"""
    title_terms: list[Term] = field(default_factory=list)          # [TI]
    abstract_terms: list[Term] = field(default_factory=list)       # [AB]
    tiab_terms: list[Term] = field(default_factory=list)           # [TIAB]
    author_terms: list[Term] = field(default_factory=list)         # [AU]
    journal_terms: list[Term] = field(default_factory=list)        # [TA]
    affiliation_terms: list[Term] = field(default_factory=list)    # [AD]
    language_terms: list[Term] = field(default_factory=list)       # [LA]
    volume_terms: list[Term] = field(default_factory=list)         # [VI]
    issue_terms: list[Term] = field(default_factory=list)          # [IP]
    pages_terms: list[Term] = field(default_factory=list)          # [PG]
    lid_terms: list[Term] = field(default_factory=list)            # [LID]
    date_from: str | None = None                                   # [DP] lower bound (YYYY-MM-DD)
    date_to: str | None = None                                     # [DP] upper bound (YYYY-MM-DD)
    year_from: int | None = None                                   # [DP] year lower
    year_to: int | None = None                                     # [DP] year upper
    # 更多日期字段
    edat_from: str | None = None                                   # [EDAT]
    edat_to: str | None = None
    crdt_from: str | None = None                                   # [CRDT]
    crdt_to: str | None = None
    mhda_from: str | None = None                                   # [MHDA]
    mhda_to: str | None = None
    lr_from: str | None = None                                     # [LR]
    lr_to: str | None = None
    dcom_from: str | None = None                                   # [DCOM]
    dcom_to: str | None = None
    dep_from: str | None = None                                     # P1-2: [DEP]
    dep_to: str | None = None
    # 特殊字段（存储 Term 以保留 is_not 标志）
    mesh_terms: list[Term] = field(default_factory=list)           # [MH] (was list[str])
    majr_terms: list[Term] = field(default_factory=list)           # [MAJR] (was list[str])
    pub_types: list[Term] = field(default_factory=list)            # [PT] (was list[str])
    doi_terms: list[Term] = field(default_factory=list)            # [DOI] (was list[str])
    pmid_terms: list[Term] = field(default_factory=list)           # [PMID] (was list[int])
    # JSONB 字段
    grant_terms: list[Term] = field(default_factory=list)           # [GR]
    subheading_terms: list[Term] = field(default_factory=list)      # [SH]
    registry_terms: list[Term] = field(default_factory=list)        # [RN]
    substance_terms: list[Term] = field(default_factory=list)       # [NM]
    databank_terms: list[Term] = field(default_factory=list)        # [SI]
    pharmaco_terms: list[Term] = field(default_factory=list)         # [PA]
    ed_terms: list[Term] = field(default_factory=list)               # [ED]
    investigator_terms: list[Term] = field(default_factory=list)     # [IR]
    personal_name_terms: list[Term] = field(default_factory=list)    # [PS]
    pubnote_terms: list[Term] = field(default_factory=list)          # [PUBN]
    auid_terms: list[Term] = field(default_factory=list)             # [AUID]
    cois_terms: list[Term] = field(default_factory=list)             # [COIS]
    tt_terms: list[Term] = field(default_factory=list)               # [TT]
    sb_terms: list[Term] = field(default_factory=list)               # P1-2: [SB] Subset
    stat_terms: list[Term] = field(default_factory=list)             # P1-2: [STAT] Status
    uid_terms: list[Term] = field(default_factory=list)              # P1-2: [UID] PMID/DOI 统一
    # P4 新增独立语义字段
    ot_terms: list[Term] = field(default_factory=list)               # [OT] → keywords JSONB
    gene_terms: list[Term] = field(default_factory=list)             # [GEN] → gene_symbols JSONB
    pmc_terms: list[Term] = field(default_factory=list)              # [PMC] → pmc_id
    plain_terms: list[Term] = field(default_factory=list)          # no field tag
    boolean_operator: str = "and"                                  # "and" | "or" | "mixed"
    has_not: bool = False                                          # contains NOT
    not_terms: list[Term] = field(default_factory=list)            # terms under NOT
    groups: list[list[Term]] = field(default_factory=list)         # parenthesized sub-groups
    group_operators: list[str] = field(default_factory=list)       # "and"/"or" per group (P2-2)
    group_negated: list[bool] = field(default_factory=list)        # P16: True if group was wrapped by NOT (external negation)
    negated_date_ranges: set[str] = field(default_factory=set)     # date fields negated by NOT
    _date_range_markers: list[Term] = field(default_factory=list, repr=False)  # internal: date range Term collectors


# ─── Parser ───

class ParseError(ValueError):
    """查询语法解析错误（调用方应退化到 plain text 搜索）。"""


class PubmedQueryParser:
    """Recursive descent parser for PubMed query syntax.

    Grammar:
      query     → or_expr
      or_expr   → and_expr (OR and_expr)*
      and_expr  → not_expr (AND not_expr)*
      not_expr  → NOT not_expr | primary
      primary   → atom FIELD? | LPAREN query RPAREN
      atom      → QUOTED | WORD | number (COLON number)?
    """

    def __init__(self, tokens: list[Token]):
        self.tokens = tokens
        self.pos = 0

    def peek(self) -> Token:
        return self.tokens[self.pos]

    def peek_n(self, n: int) -> Token | None:
        idx = self.pos + n
        return self.tokens[idx] if idx < len(self.tokens) else None

    def advance(self) -> Token:
        t = self.tokens[self.pos]
        self.pos += 1
        return t

    def expect(self, *types: TokenType) -> Token:
        t = self.peek()
        if t.type not in types:
            raise ParseError(
                f"Expected one of {[tt.name for tt in types]}, got {t.type.name}({t.value!r})"
            )
        return self.advance()

    def parse(self) -> ParsedPubmedQuery:
        """入口：解析完整的查询字符串。"""
        result = ParsedPubmedQuery()
        self._depth = 0  # 括号嵌套深度计数器
        terms = self._parse_or_expr(result)
        # 解析错误由 parse_pubmed_query 统一降级处理

        # Detect boolean operator from top-level (depth=0) token stream only
        # P11: 括号内的 AND/OR 不应影响顶层布尔操作符判定
        depth = 0
        has_and = has_or = False
        for t in self.tokens:
            if t.type == TokenType.LPAREN:
                depth += 1
            elif t.type == TokenType.RPAREN:
                depth -= 1
            elif depth == 0:
                if t.type == TokenType.AND:
                    has_and = True
                elif t.type == TokenType.OR:
                    has_or = True
        if has_and and has_or:
            result.boolean_operator = "mixed"
        elif has_or and not has_and:
            result.boolean_operator = "or"

        # 分组词不从 flat lists 走，避免括号内外的词被一起 AND/OR
        # 同时 has_not/not_terms 也只考虑非分组词
        _ungrouped = [t for t in terms if not getattr(t, '_is_range_end', False) and t.group_id < 0]
        # P12: has_not 同时检查分组内 NOT（如 NOT (a OR b)）
        result.has_not = any(t.is_not for t in _ungrouped) or any(
            t.is_not for g in result.groups for t in g
        )
        result.not_terms = [t for t in _ungrouped if t.is_not]
        for t in _ungrouped:
            self._dispatch_term(result, t)

        # P2-1: Handle unconsumed tokens (e.g., orphan text after RPAREN)
        if self.pos < len(self.tokens) - 1:
            for t in self.tokens[self.pos:-1]:  # exclude EOF token
                if t.type in (TokenType.WORD, TokenType.QUOTED, TokenType.NUMBER):
                    text = t.value.strip('"') if t.type == TokenType.QUOTED else t.value
                    result.plain_terms.append(Term(text=text, exact=(t.type == TokenType.QUOTED)))

        # Recompute negated_date_ranges from marker terms (after NOT toggling from recursive _parse_not_expr)
        result.negated_date_ranges = {
            t.field.replace("__RANGE_", "").replace("__", "")
            for t in result._date_range_markers if t.is_not
        }

        return result

    def _dispatch_term(self, result: ParsedPubmedQuery, term: Term) -> None:
        if term.field == "title":
            result.title_terms.append(term)
        elif term.field == "abstract":
            result.abstract_terms.append(term)
        elif term.field == "all":  # TIAB
            result.tiab_terms.append(term)
        elif term.field == "author":
            result.author_terms.append(term)
        elif term.field == "affiliation":
            result.affiliation_terms.append(term)
        elif term.field == "journal":
            result.journal_terms.append(term)
        elif term.field == "language":
            result.language_terms.append(term)
        elif term.field == "volume":
            result.volume_terms.append(term)
        elif term.field == "issue":
            result.issue_terms.append(term)
        elif term.field == "pages":
            result.pages_terms.append(term)
        elif term.field == "lid":
            result.lid_terms.append(term)
        elif term.field == "MH":
            result.mesh_terms.append(term)
        elif term.field == "MAJR":
            result.majr_terms.append(term)
        elif term.field == "PT":
            result.pub_types.append(term)
        elif term.field == "PMID":
            try:
                int(term.text)  # validate
            except ValueError:
                result.plain_terms.append(term)
                return
            result.pmid_terms.append(term)
        elif term.field == "DOI":
            result.doi_terms.append(term)
        elif term.field == "GR":
            result.grant_terms.append(term)
        elif term.field == "SH":
            result.subheading_terms.append(term)
        elif term.field == "RN":
            result.registry_terms.append(term)
        elif term.field == "NM":
            result.substance_terms.append(term)
        elif term.field == "SI":
            result.databank_terms.append(term)
        elif term.field == "PA":
            result.pharmaco_terms.append(term)
        elif term.field is None:
            result.plain_terms.append(term)
        # ── 独立日期字段（非范围语法）："2024-01-01"[DP] → from=to=该日期 ──
        #     纯 4 位年份 "2024"[DP] 用 year_from/year_to，避免 fromisoformat 问题
        #     is_not 时加入 negated_date_ranges，引擎据此 NOT 条件
        elif term.field == "DP":
            if term.text.isdigit() and len(term.text) == 4:
                result.year_from = int(term.text)
                result.year_to = int(term.text)
            elif _PARTIAL_DATE_RE.match(term.text):
                result.date_from, result.date_to = _expand_partial_date(term.text)
            else:
                if _validate_date_str(term.text):
                    result.date_from = term.text
                    result.date_to = term.text
                else:
                    result.plain_terms.append(term)
                    return
            if term.is_not:
                result.negated_date_ranges.add("DP")
        elif term.field == "EDAT":
            if term.text.isdigit() and len(term.text) == 4:
                result.edat_from = f"{term.text}-01-01"
                result.edat_to = f"{term.text}-12-31"
            elif _PARTIAL_DATE_RE.match(term.text):
                result.edat_from, result.edat_to = _expand_partial_date(term.text)
            else:
                if _validate_date_str(term.text):
                    result.edat_from = term.text
                    result.edat_to = term.text
                else:
                    result.plain_terms.append(term)
                    return
            if term.is_not:
                result.negated_date_ranges.add("EDAT")
        elif term.field == "CRDT":
            if term.text.isdigit() and len(term.text) == 4:
                result.crdt_from = f"{term.text}-01-01"
                result.crdt_to = f"{term.text}-12-31"
            elif _PARTIAL_DATE_RE.match(term.text):
                result.crdt_from, result.crdt_to = _expand_partial_date(term.text)
            else:
                if _validate_date_str(term.text):
                    result.crdt_from = term.text
                    result.crdt_to = term.text
                else:
                    result.plain_terms.append(term)
                    return
            if term.is_not:
                result.negated_date_ranges.add("CRDT")
        elif term.field == "MHDA":
            if term.text.isdigit() and len(term.text) == 4:
                result.mhda_from = f"{term.text}-01-01"
                result.mhda_to = f"{term.text}-12-31"
            elif _PARTIAL_DATE_RE.match(term.text):
                result.mhda_from, result.mhda_to = _expand_partial_date(term.text)
            else:
                if _validate_date_str(term.text):
                    result.mhda_from = term.text
                    result.mhda_to = term.text
                else:
                    result.plain_terms.append(term)
                    return
            if term.is_not:
                result.negated_date_ranges.add("MHDA")
        elif term.field == "LR":
            if term.text.isdigit() and len(term.text) == 4:
                result.lr_from = f"{term.text}-01-01"
                result.lr_to = f"{term.text}-12-31"
            elif _PARTIAL_DATE_RE.match(term.text):
                result.lr_from, result.lr_to = _expand_partial_date(term.text)
            else:
                if _validate_date_str(term.text):
                    result.lr_from = term.text
                    result.lr_to = term.text
                else:
                    result.plain_terms.append(term)
                    return
            if term.is_not:
                result.negated_date_ranges.add("LR")
        elif term.field == "DCOM":
            if term.text.isdigit() and len(term.text) == 4:
                result.dcom_from = f"{term.text}-01-01"
                result.dcom_to = f"{term.text}-12-31"
            elif _PARTIAL_DATE_RE.match(term.text):
                result.dcom_from, result.dcom_to = _expand_partial_date(term.text)
            else:
                if _validate_date_str(term.text):
                    result.dcom_from = term.text
                    result.dcom_to = term.text
                else:
                    result.plain_terms.append(term)
                    return
            if term.is_not:
                result.negated_date_ranges.add("DCOM")
        elif term.field == "DEP":
            if term.text.isdigit() and len(term.text) == 4:
                result.dep_from = f"{term.text}-01-01"
                result.dep_to = f"{term.text}-12-31"
            elif _PARTIAL_DATE_RE.match(term.text):
                result.dep_from, result.dep_to = _expand_partial_date(term.text)
            else:
                if _validate_date_str(term.text):
                    result.dep_from = term.text
                    result.dep_to = term.text
                else:
                    result.plain_terms.append(term)
                    return
            if term.is_not:
                result.negated_date_ranges.add("DEP")
        elif term.field == "__RANGE_DP__":
            pass
        elif term.field == "__RANGE_EDAT__":
            pass
        elif term.field == "__RANGE_CRDT__":
            pass
        elif term.field == "__RANGE_MHDA__":
            pass
        elif term.field == "__RANGE_LR__":
            pass
        elif term.field == "__RANGE_DCOM__":
            pass
        elif term.field == "__RANGE_DEP__":
            pass
        elif term.field == "ED":
            result.ed_terms.append(term)
        elif term.field == "IR":
            result.investigator_terms.append(term)
        elif term.field == "PS":
            result.personal_name_terms.append(term)
        elif term.field == "PUBN":
            result.pubnote_terms.append(term)
        elif term.field == "AUID":
            result.auid_terms.append(term)
        elif term.field == "COIS":
            result.cois_terms.append(term)
        elif term.field == "TT":
            result.tt_terms.append(term)
        elif term.field == "SB":
            result.sb_terms.append(term)
        elif term.field == "STAT":
            result.stat_terms.append(term)
        elif term.field == "UID":
            result.uid_terms.append(term)
        # P4 新增独立语义字段
        elif term.field == "OT":
            result.ot_terms.append(term)
        elif term.field == "GEN":
            result.gene_terms.append(term)
        elif term.field == "PMC":
            result.pmc_terms.append(term)
        else:
            result.plain_terms.append(term)

    def _parse_or_expr(self, result: ParsedPubmedQuery) -> list[Term]:
        """or_expr → and_expr (OR and_expr)*

        P7-D3: Preserve AND clusters when OR is mixed (e.g. ``A OR B AND C``).
        Collect each ``_parse_and_expr`` result as a separate cluster. When OR
        is found AND a cluster has >1 term (implicit AND), wrap it in a group
        so the engine produces ``A OR (B AND C)`` instead of ``A OR B OR C``.
        """
        clusters = [self._parse_and_expr(result)]
        had_or = False
        while self.peek().type == TokenType.OR:
            had_or = True
            self.advance()
            if self.peek().type == TokenType.EOF:
                break  # trailing OR, ignore silently
            clusters.append(self._parse_and_expr(result))

        if not had_or:
            return clusters[0] if clusters else []

        # OR present: group any AND-cluster with >1 term
        # P16: Only do this at top level (depth=0). Inside parens, _parse_primary handles the grouping
        # with the correct operator from the token stream scan.
        if self._depth > 0:
            all_terms = [t for cluster in clusters for t in cluster]
            return all_terms

        all_terms: list[Term] = []
        for cluster in clusters:
            if len(cluster) > 1 and not any(t.group_id >= 0 for t in cluster):
                gid = len(result.groups)
                for t in cluster:
                    t.group_id = gid
                result.groups.append(cluster)
                result.group_operators.append("and")
            all_terms.extend(cluster)
        return all_terms

    def _is_primary_start(self, token: Token) -> bool:
        """Check if token could start a primary expression."""
        return token.type in (TokenType.WORD, TokenType.QUOTED, TokenType.NUMBER, TokenType.DATE,
                              TokenType.LPAREN, TokenType.NOT)

    def _parse_and_expr(self, result: ParsedPubmedQuery) -> list[Term]:
        """and_expr → not_expr (AND? not_expr)*

        AND is optional between consecutive primaries (implicit AND).
        e.g. ``smith j[AU]`` → smith AND j[AU]
        """
        left = self._parse_not_expr(result)
        left = list(left)  # P12: copy to prevent mutation of shared group list
        while True:
            tok = self.peek()
            if tok.type == TokenType.AND:
                self.advance()
                if self.peek().type == TokenType.EOF:
                    break
            elif self._is_primary_start(tok):
                pass  # implicit AND — continue without consuming
            else:
                break
            right = self._parse_not_expr(result)
            left.extend(right)
        return left

    def _parse_not_expr(self, result: ParsedPubmedQuery) -> list[Term]:
        """not_expr → NOT not_expr | primary"""
        if self.peek().type == TokenType.NOT:
            self.advance()
            # P5: trailing NOT at end of input → ignore silently (avoid IndexError peeking past EOF)
            if self.peek().type == TokenType.EOF:
                return []
            inner = self._parse_not_expr(result)
            for t in inner:
                t.is_not = not t.is_not
            return inner
        return self._parse_primary(result, negated=False)

    def _parse_primary(self, result: ParsedPubmedQuery, negated: bool = False) -> list[Term]:
        """primary → atom FIELD? | LPAREN query RPAREN"""
        if self.peek().type == TokenType.LPAREN:
            self._depth += 1
            if self._depth > MAX_PAREN_DEPTH:
                raise ParseError(f"括号嵌套过深（超过 {MAX_PAREN_DEPTH} 层），降级为简单文本搜索")
            self.advance()
            start_pos = self.pos  # P2-2: 记录组起始标记位置
            terms = self._parse_or_expr(result)
            end_pos = self.pos    # P2-2: 记录组结束标记位置
            self.expect(TokenType.RPAREN)
            self._depth -= 1
            # P2-F12: (a OR b)[TI] — 组后字段标签应用到组内所有词
            if self.peek().type == TokenType.FIELD:
                ft = self.advance()
                _raw_field = ft.value[1:-1].upper()
                _field = _normalize_field_label(_raw_field)
                for t in terms:
                    t.field = _field
            # P15-PRIMARY: 如果 _parse_or_expr 已为 AND 集群（如 A OR B AND C → [B, C] sub-group）
            # 或嵌套括号创建了子组，则这些子组已正确处理结构。
            # 只对尚未分组的词创建外层组，避免 Term 被放入两个组导致 _pubmed_conditions 重复处理。
            _parent_gid = len(result.groups)
            _ungrouped = [t for t in terms if t.group_id < 0]
            for t in _ungrouped:
                t.group_id = _parent_gid
            if _ungrouped:
                result.groups.append(_ungrouped)
                _has_or = any(t.type == TokenType.OR for t in self.tokens[start_pos:end_pos])
                result.group_operators.append("or" if _has_or else "and")
                result.group_negated.append(negated)  # P16: track external NOT vs internal NOT
            # 已分组的 Term（嵌套括号 OR 子组）不再重复加组
            if negated:
                for t in terms:
                    t.is_not = True
            return terms

        return self._parse_atom(result, negated)

    def _parse_atom(self, result: ParsedPubmedQuery, negated: bool = False) -> list[Term]:
        """Parse a single atom with optional field tag.

        Handles:
          - "quoted phrase"[FIELD]
          - word[FIELD]
          - NUMBER:NUMBER[DP]  (year/date range)
          - DATE:DATE[DP]     (full date range like 2024-01-01:2024-12-31)
          - NUMBER  (bare number)
          - word    (bare word)
        """
        # Look ahead for range pattern (NUMBER:NUMBER or DATE:DATE)
        t0 = self.peek()
        t1 = self.peek_n(1)
        t2 = self.peek_n(2)

        if (t1 is not None and t1.type == TokenType.COLON
                and t2 is not None
                and t0.type in (TokenType.NUMBER, TokenType.DATE, TokenType.WORD)
                and t2.type in (TokenType.NUMBER, TokenType.DATE, TokenType.WORD)):
            return self._parse_range(result, negated)

        # Normal atom
        token = self.advance()
        text = token.value.strip('"') if token.type == TokenType.QUOTED else token.value
        is_exact = (token.type == TokenType.QUOTED)
        field = None
        _noexp = False  # P1-4
        if self.peek().type == TokenType.FIELD:
            ft = self.advance()
            raw = ft.value[1:-1].upper()
            # P1-4: [MH:noexp] → 抑制树展开
            if raw == "MH:NOEXP":
                field = "MH"
                _noexp = True
            else:
                field = raw
                if field in _FIELD_TAG_MAP:
                    field = _FIELD_TAG_MAP[field]

        return [Term(text, exact=is_exact, field=field, is_not=negated, _noexp=_noexp)]

    def _parse_range(self, result: ParsedPubmedQuery, negated: bool = False) -> list[Term]:
        """Parse NUMBER:NUMBER[FIELD] — handles date ranges specially."""
        start_val = self.advance().value  # NUMBER
        self.advance()                     # COLON
        end_val = self.advance().value     # NUMBER

        field = None
        if self.peek().type == TokenType.FIELD:
            ft = self.advance()
            field = ft.value[1:-1].upper()

        if field in _DATE_RANGE_FIELDS:
            attr_map = {
                "DP": ("date_from", "date_to", "year_from", "year_to", "__RANGE_DP__"),
                "EDAT": ("edat_from", "edat_to", None, None, "__RANGE_EDAT__"),
                "CRDT": ("crdt_from", "crdt_to", None, None, "__RANGE_CRDT__"),
                "MHDA": ("mhda_from", "mhda_to", None, None, "__RANGE_MHDA__"),
                "LR": ("lr_from", "lr_to", None, None, "__RANGE_LR__"),
                "DCOM": ("dcom_from", "dcom_to", None, None, "__RANGE_DCOM__"),
                "DEP": ("dep_from", "dep_to", None, None, "__RANGE_DEP__"),  # P1-2
            }
            date_attr, date_attr_to, yr_from_attr, yr_to_attr, marker_field = attr_map[field]
            # 反向范围自动交换（如 2026:2024[DP] → 2024:2026[DP]）
            # P5: also handle mixed types (e.g. 2026:2024-01-01 → 2024-01-01:2026)
            _start_is_digit = start_val.isdigit()
            _end_is_digit = end_val.isdigit()
            if _start_is_digit and _end_is_digit and int(start_val) > int(end_val):
                start_val, end_val = end_val, start_val
            elif not _start_is_digit and not _end_is_digit and start_val > end_val:
                start_val, end_val = end_val, start_val
            elif _start_is_digit and not _end_is_digit:
                try:
                    if int(start_val) > int(end_val[:4]):
                        start_val, end_val = end_val, start_val
                except (ValueError, TypeError):
                    pass
            elif not _start_is_digit and _end_is_digit:
                try:
                    if int(start_val[:4]) > int(end_val):
                        start_val, end_val = end_val, start_val
                except (ValueError, TypeError):
                    pass
            # P12: normalize compact YYYYMMDD dates → YYYY-MM-DD
            if start_val.isdigit() and len(start_val) == 8:
                try:
                    start_val = f"{start_val[:4]}-{start_val[4:6]}-{start_val[6:8]}"
                except IndexError:
                    pass
            if end_val.isdigit() and len(end_val) == 8:
                try:
                    end_val = f"{end_val[:4]}-{end_val[4:6]}-{end_val[6:8]}"
                except IndexError:
                    pass
            # P12: validate date values — non-numeric garbage falls back to plain text
            _valid_date = lambda s: _validate_date_str(s)
            if not _valid_date(start_val) or not _valid_date(end_val):
                txt = f"{start_val}:{end_val}[{field}]"
                return [Term(txt, field=None, is_not=negated)]
            # 确定两端是否是 4 位年份
            _start_is_year = start_val.isdigit() and len(start_val) == 4
            _end_is_year = end_val.isdigit() and len(end_val) == 4
            # Year-only range (e.g., 2024:2026[EDAT])
            if _start_is_year and _end_is_year:
                try:
                    if yr_from_attr:
                        setattr(result, yr_from_attr, int(start_val))
                        setattr(result, yr_to_attr, int(end_val))
                    else:
                        # For non-DP date fields: convert year to full date for consistency
                        setattr(result, date_attr, f"{start_val}-01-01")
                        setattr(result, date_attr_to, f"{end_val}-12-31")
                except (ValueError, TypeError):
                    pass
            elif _start_is_year and not _end_is_year:
                # Mixed: start is year, end is full date (e.g., 2024:2024-12-01[EDAT])
                if yr_from_attr:
                    try:
                        setattr(result, yr_from_attr, int(start_val))
                    except (ValueError, TypeError):
                        pass
                setattr(result, date_attr, f"{start_val}-01-01")
                setattr(result, date_attr_to, end_val)
            elif not _start_is_year and _end_is_year:
                # Mixed: start is full date, end is year (e.g., 2024-01-01:2026[EDAT])
                setattr(result, date_attr, start_val)
                setattr(result, date_attr_to, f"{end_val}-12-31")
            else:
                # Full date range (e.g., 2024-01-01:2024-12-31[EDAT])
                setattr(result, date_attr, start_val)
                setattr(result, date_attr_to, end_val)
            marker = Term(f"{start_val}:{end_val}", field=marker_field, is_not=negated)
            marker._is_range_end = True
            result._date_range_markers.append(marker)
            if negated:
                result.negated_date_ranges.add(field)
            return [marker]

        # Non-date range or no field → plain text
        txt = f"{start_val}:{end_val}"
        if field:
            txt = f"{txt}[{field}]"
        return [Term(txt, field=field, is_not=negated)]


# ─── Public API ───

def is_pubmed_syntax(query: str) -> bool:
    """快速检测查询串是否包含 PubMed 语法特征。

    检测条件（任意满足其一）：
    1. 包含 `[...]`（字段标签）
    2. 包含布尔运算符 AND/OR/NOT（大写，词边界）
    """
    if not query or not query.strip():
        return False
    # P5: Normalize fullwidth characters before checking
    query = unicodedata.normalize('NFKC', query)
    if re.search(r'\[(' + '|'.join(_ALL_FIELD_TAGS) + r')\]', query, re.IGNORECASE):
        return True
    if re.search(r'\b(AND|OR|NOT)\b', query):
        return True
    return False


def parse_pubmed_query(query: str) -> ParsedPubmedQuery:
    """解析 PubMed 查询字符串。

    如果解析失败（非法语法），返回全部内容为 plain_terms 的空结构。
    """
    if not query or not query.strip():
        return ParsedPubmedQuery()

    try:
        # P2-3: Unicode normalization — strip zero-width chars, normalize fullwidth digits
        query = unicodedata.normalize('NFKC', query)
        import re as _re
        # 将 YYYY/MM/DD 或 YYYY/M/D 格式的日期分隔符统一为 YYYY-MM-DD，使 tokeniser 正确识别为 DATE
        # Anchored with context boundaries to avoid over-matching inside URLs/paths
        query = _re.sub(
            r'(^|[\[\s":])(\d{4})/(\d{1,2})/(\d{1,2})(?=\s|$|[\[\]":])',
            lambda m: f'{m.group(1)}{m.group(2)}-{int(m.group(3)):02d}-{int(m.group(4)):02d}',
            query,
        )
        # P5: Normalize single-digit month/day (2024-1-1 → 2024-01-01) to match DATE token pattern
        #     Anchored with context boundaries to avoid over-matching inside non-date text
        query = _re.sub(
            r'(^|[\[\s":])(\d{4})-(\d{1,2})-(\d{1,2})(?=\s|$|[\[\]":])',
            lambda m: f'{m.group(1)}{m.group(2)}-{int(m.group(3)):02d}-{int(m.group(4)):02d}',
            query,
        )
        tokens = tokenise(query)
        parser = PubmedQueryParser(tokens)
        return parser.parse()
    except (ParseError, IndexError, ValueError):
        # P0-1: 降级时清理查询中的 [field] 标签、布尔符、引号和括号
        degraded = ParsedPubmedQuery()
        import re as _degrade_re
        _clean = _degrade_re.sub(r'\[[\w/: -]+\]', '', query)
        _clean = _degrade_re.sub(r'\b(AND|OR|NOT)\b', '', _clean)
        _clean = _clean.replace('"', '').replace('(', '').replace(')', '')
        for t in _clean.split():
            if t.strip():
                degraded.plain_terms.append(Term(text=t.strip()))
        return degraded


def extract_pubmed_query_for_prisma(query: str) -> tuple[str, list[str]]:
    """提取 PubMed 查询和所用 MeSH 词，供 PRISMA 导出。

    Returns:
        (normalized_query, mesh_terms_used)
    """
    parsed = parse_pubmed_query(query)
    mesh_used = list(set(
        [t.text for t in parsed.mesh_terms] + [t.text for t in parsed.majr_terms]
    ))
    mesh_used.sort()

    # P5: Handle field tags with '/' (e.g. Title/Article) or special chars
    normalized = re.sub(
        r'\[([\w/:]+)\]',
        lambda m: f'[{m.group(1).upper()}]',
        query,
    )
    return normalized, mesh_used