init: 初始化 dpb 桃育种系统代码库
前后端 + 后端 FastAPI 全量源码、部署脚本与文档。
This commit is contained in:
34047007@qq.com
@@ -0,0 +1,720 @@
|
||||
"""基因组选择：VanRaden G 矩阵 + GBLUP + ssGBLUP（仅 numpy，无 R）。
|
||||
|
||||
- build_g_matrix：dosage(0/1/2) 样本×标记矩阵 → VanRaden G（method1=ZZ'/2Σp(1-p)、
|
||||
method2=ZDZ'，D=1/[2p(1-p)]），缺格按列均值(2p)填充；MAF 过滤 + blend 岭回归保证可逆。
|
||||
- solve_gblup：仅基因型个体入模型，MME 用 G⁻¹ 替代 A⁻¹；方差组分用剖面 REML
|
||||
（黄金分割最大化精确 REML 对数似然，与 blup._solve_gxe 同模板）。
|
||||
- solve_ssgblup：单步法 H⁻¹ = A⁻¹ + [[0,0],[0,G⁻¹ − A22⁻¹]]（Aguilar et al. 2010；
|
||||
A22 = 基因型个体在 A 中的子阵），基因型+非基因型个体都出 EBV。
|
||||
|
||||
可靠性 = 1 − PEV/σ²a，PEV = C22[ii]·σ²e（C 为 MME 系数矩阵逆的加性块）。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import random
|
||||
|
||||
import numpy as np
|
||||
|
||||
from scripts.breeding_stats import blup
|
||||
|
||||
ENGINE_VERSION = "1.4.0" # 1.4.0: 非线性基因组预测 rrBLUP + BayesB; 1.3.0: kfold split=family; 1.2.0: SSR 多等位 + EMMAX GRM 复用
|
||||
_RIDGE = 1e-6 # A22⁻¹ 病态加小岭
|
||||
|
||||
_GBLUP_KEYS = ["ebv", "reliability", "h2", "sigma_a", "sigma_e",
|
||||
"n_obs", "n_individuals", "n_markers", "converged", "warning"]
|
||||
|
||||
|
||||
def build_g_matrix(genotypes: dict[str, dict[str, float]],
|
||||
markers: list[str], *, maf_min: float = 0.05,
|
||||
method: int = 2, blend: float = 0.02):
|
||||
"""VanRaden G 矩阵。
|
||||
|
||||
genotypes: {sample: {marker: dosage(0/1/2)}}；markers: 有序标记列表。
|
||||
返回 (G 按 sorted(genotypes) 排序, meta{G 统计量})。
|
||||
"""
|
||||
samples = sorted(genotypes)
|
||||
n = len(samples)
|
||||
if n < 2 or not markers:
|
||||
raise ValueError("基因型样本或标记数不足（需 n≥2、m≥1）")
|
||||
m = len(markers)
|
||||
M = np.full((n, m), np.nan)
|
||||
n_missing = 0
|
||||
for i, s in enumerate(samples):
|
||||
g = genotypes[s]
|
||||
for j, mk in enumerate(markers):
|
||||
v = g.get(mk)
|
||||
if v is None or (isinstance(v, float) and math.isnan(v)):
|
||||
n_missing += 1
|
||||
else:
|
||||
M[i, j] = float(v)
|
||||
p = np.zeros(m)
|
||||
for j in range(m):
|
||||
col = M[:, j]
|
||||
valid = col[~np.isnan(col)]
|
||||
if len(valid) == 0:
|
||||
p[j] = 0.5
|
||||
else:
|
||||
p[j] = float(np.mean(valid)) / 2.0
|
||||
M[:, j] = np.where(np.isnan(M[:, j]), 2.0 * p[j], M[:, j])
|
||||
maf = np.minimum(p, 1.0 - p)
|
||||
keep = maf >= maf_min
|
||||
if int(keep.sum()) == 0:
|
||||
raise ValueError(f"MAF≥{maf_min} 的标记为 0，请降低 maf_min 或检查标记等位频率")
|
||||
M = M[:, keep]
|
||||
p = p[keep]
|
||||
m_keep = int(keep.sum())
|
||||
Z = M - 2.0 * p[None, :]
|
||||
if method == 1:
|
||||
denom = 2.0 * float(np.sum(p * (1.0 - p)))
|
||||
G = (Z @ Z.T) / denom if denom > 0 else np.eye(n)
|
||||
else:
|
||||
D = 1.0 / (2.0 * p * (1.0 - p))
|
||||
G = Z @ np.diag(D) @ Z.T
|
||||
if blend and blend > 0:
|
||||
diag_mean = float(np.mean(np.diag(G)))
|
||||
G = (1.0 - blend) * G + blend * diag_mean * np.eye(n)
|
||||
off = G[np.triu_indices(n, 1)] if n > 1 else np.array([0.0])
|
||||
return G, {
|
||||
"n_individuals": n,
|
||||
"n_markers": m,
|
||||
"m_after_maf": m_keep,
|
||||
"n_missing": n_missing,
|
||||
"maf_min": maf_min,
|
||||
"method": method,
|
||||
"blend": blend,
|
||||
"diag_mean": round(float(np.mean(np.diag(G))), 6),
|
||||
"diag_range": [round(float(np.min(np.diag(G))), 6), round(float(np.max(np.diag(G))), 6)],
|
||||
"offdiag_range": [round(float(np.min(off)), 6), round(float(np.max(off)), 6)],
|
||||
}
|
||||
|
||||
|
||||
def _scale_g_to_a22(G: np.ndarray, A22: np.ndarray) -> np.ndarray:
|
||||
"""把基因关系阵 G 缩放到系谱关系阵 A22 的尺度（Christensen et al. 调整）。
|
||||
|
||||
VanRaden method=2 的 G 对角线≈标记数 m（远大于 A22 的 ≈1），若直接组 H，
|
||||
基因型块会被 G 主导、h²/EBV 尺度偏置。用两参数对齐 G*=a·G+b 使
|
||||
对角均值/非对角均值与 A22 一致；G 退化（对角≈非对角）时跳过并原样返回。
|
||||
"""
|
||||
n = len(A22)
|
||||
if n < 2:
|
||||
return G
|
||||
diag_g = np.mean(np.diag(G))
|
||||
diag_a = np.mean(np.diag(A22))
|
||||
off_g = float(np.mean(G[np.triu_indices(n, 1)]))
|
||||
off_a = float(np.mean(A22[np.triu_indices(n, 1)]))
|
||||
denom = diag_g - off_g
|
||||
if abs(denom) < 1e-12:
|
||||
return G
|
||||
a = (diag_a - off_a) / denom
|
||||
b = off_a - a * off_g
|
||||
return a * G + b
|
||||
|
||||
|
||||
def _reml_ll(ratio: float, ZGZ: np.ndarray, X: np.ndarray, y: np.ndarray) -> float:
|
||||
"""V = ZGZ·ratio + I（Ve=1 尺度），精确 REML 对数似然（slogdet + y'Py）。"""
|
||||
m = len(y)
|
||||
V = ZGZ * ratio + np.eye(m)
|
||||
try:
|
||||
VinvX = np.linalg.solve(V, X)
|
||||
Vinvy = np.linalg.solve(V, y)
|
||||
XtVinvX = X.T @ VinvX
|
||||
XtVinvY = X.T @ Vinvy
|
||||
yPy = float(y @ Vinvy) - float(XtVinvY @ np.linalg.solve(XtVinvX, XtVinvY))
|
||||
_, lv = np.linalg.slogdet(V)
|
||||
_, lx = np.linalg.slogdet(XtVinvX)
|
||||
return -0.5 * (float(lv) + float(lx) + yPy)
|
||||
except np.linalg.LinAlgError:
|
||||
return -np.inf
|
||||
|
||||
|
||||
def _profile_ve(V: np.ndarray, X: np.ndarray, y: np.ndarray, df: int) -> float:
|
||||
"""REML 剖面 σ²e = y'Py/df（V 固定尺度下）。"""
|
||||
VinvX = np.linalg.solve(V, X)
|
||||
Vinvy = np.linalg.solve(V, y)
|
||||
XtVinvX = X.T @ VinvX
|
||||
XtVinvY = X.T @ Vinvy
|
||||
yPy = float(y @ Vinvy) - float(XtVinvY @ np.linalg.solve(XtVinvX, XtVinvY))
|
||||
return yPy / df if df > 0 else float("nan")
|
||||
|
||||
|
||||
def _profile_solve(ZGZ: np.ndarray, X: np.ndarray, y: np.ndarray,
|
||||
Ginv: np.ndarray, n_all: int, z: np.ndarray):
|
||||
"""共享剖面 REML + MME 求解。返回 (ebv, rel, h2, sigma_a, sigma_e, n_iter)。"""
|
||||
m = len(y)
|
||||
df = m - X.shape[1]
|
||||
# 剖面参数为 log10(σ²a/σ²e)。h²=ratio/(1+ratio)，故 h²∈[H2_MIN,H2_MAX]
|
||||
# 对应 ratio∈[H2_MIN/(1−H2_MIN), H2_MAX/(1−H2_MAX)]（直接用 log10(H2_MAX)≈−0.0004
|
||||
# 会把 σ²a/σ²e 封顶 0.999，令 h² 上限只有 0.5）。
|
||||
lo, hi = math.log10(blup.H2_MIN / (1.0 - blup.H2_MIN)), math.log10(blup.H2_MAX / (1.0 - blup.H2_MAX))
|
||||
|
||||
def _ll(r: float) -> float:
|
||||
return _reml_ll(10.0 ** r, ZGZ, X, y)
|
||||
|
||||
r_opt, n_iter = blup._golden_max(_ll, lo, hi, tol=1e-4, max_iter=blup.MAX_PROFILE_EVALS)
|
||||
ratio = 10.0 ** r_opt
|
||||
V = ZGZ * ratio + np.eye(m)
|
||||
ve = _profile_ve(V, X, y, df)
|
||||
va = ratio * ve if not math.isnan(ve) else 1.0
|
||||
lam = ve / va if va > 0 else 1.0
|
||||
|
||||
Zmat = np.zeros((n_all, m))
|
||||
Zmat[z, np.arange(m)] = 1.0
|
||||
C = np.block([
|
||||
[X.T @ X, X.T @ Zmat.T],
|
||||
[Zmat @ X, Zmat @ Zmat.T + lam * Ginv],
|
||||
])
|
||||
rhs = np.concatenate([X.T @ y, Zmat @ y])
|
||||
try:
|
||||
sol = np.linalg.solve(C, rhs)
|
||||
except np.linalg.LinAlgError:
|
||||
return {"ebv": {}, "reliability": {}, "h2": None, "sigma_a": None,
|
||||
"sigma_e": None, "n_iter": n_iter, "converged": False,
|
||||
"warning": "MME 系数矩阵奇异（基因型共线/样本不足），求解失败"}
|
||||
C_inv = np.linalg.inv(C)
|
||||
C22inv = C_inv[1:, 1:]
|
||||
rel = np.clip(1.0 - np.diag(C22inv) * ve / va, 0.0, 1.0) if va > 0 else np.zeros(n_all)
|
||||
h2 = va / (va + ve) if (va + ve) > 0 else None
|
||||
return {"ebv": sol[1:], "reliability": rel, "h2": h2, "sigma_a": float(va),
|
||||
"sigma_e": float(ve) if not math.isnan(ve) else None,
|
||||
"n_iter": n_iter, "converged": True, "warning": None}
|
||||
|
||||
|
||||
def solve_gblup(individuals: list[str], phenos: dict[str, float], G: np.ndarray,
|
||||
*, tol: float = blup.TOL):
|
||||
"""GBLUP：仅基因型个体。individuals 为 G 行序（个体 id）；phenos 子集有观测。"""
|
||||
n = len(individuals)
|
||||
obs = [i for i in individuals if i in phenos]
|
||||
m = len(obs)
|
||||
if m < 3:
|
||||
return {"ebv": {}, "reliability": {}, "h2": None, "sigma_a": None, "sigma_e": None,
|
||||
"n_obs": m, "n_individuals": n, "n_markers": None,
|
||||
"converged": False, "warning": "表型观测不足 3，无法估计方差组分"}
|
||||
idx = {i: k for k, i in enumerate(individuals)}
|
||||
z = np.array([idx[i] for i in obs])
|
||||
y = np.array([float(phenos[i]) for i in obs])
|
||||
X = np.ones((m, 1))
|
||||
ZGZ = G[np.ix_(z, z)]
|
||||
try:
|
||||
Ginv = np.linalg.inv(G)
|
||||
except np.linalg.LinAlgError:
|
||||
Ginv = np.linalg.pinv(G)
|
||||
res = _profile_solve(ZGZ, X, y, Ginv, n, z)
|
||||
out = dict(res)
|
||||
ebv = {individuals[i]: float(out["ebv"][i]) for i in range(n)}
|
||||
rel = {individuals[i]: float(out["reliability"][i]) for i in range(n)}
|
||||
out.update(ebv=ebv, reliability=rel, n_obs=m, n_individuals=n,
|
||||
n_markers=None if not isinstance(G, np.ndarray) else G.shape[0])
|
||||
if out["warning"] is None and out.get("h2") is not None and (out["h2"] >= blup.H2_MAX or out["h2"] <= blup.H2_MIN):
|
||||
out["warning"] = "遗传力估计逼近边界，剖面 REML 未完全收敛（样本小/信噪比低）"
|
||||
return out
|
||||
|
||||
|
||||
def _build_h(pedigree: list[dict], genotyped: list[str], G: np.ndarray,
|
||||
*, ridge: float = _RIDGE, extra_base: list[str] | None = None):
|
||||
"""ssGBLUP 合并关系阵（H 协方差 + H⁻¹ 仅进 MME，Aguilar et al. 2010）。
|
||||
|
||||
返回 (order, idx, H, Hinv)。系谱中缺失的基因型个体按 base 补入（与 phenos 兜底一致）。
|
||||
"""
|
||||
base: list[int] = []
|
||||
non_base: list[tuple[int, int | None, int | None]] = []
|
||||
seen: set[int] = set()
|
||||
for rec in pedigree:
|
||||
i = rec["individual"]
|
||||
if i in seen:
|
||||
continue
|
||||
seen.add(i)
|
||||
d, s = rec.get("dam"), rec.get("sire")
|
||||
if d is None and s is None:
|
||||
base.append(i)
|
||||
else:
|
||||
non_base.append((i, d, s))
|
||||
if extra_base:
|
||||
for g in extra_base:
|
||||
if g not in seen:
|
||||
seen.add(g)
|
||||
base.append(g)
|
||||
for g in genotyped: # 无表型/无系谱基因型树也按 base 补入 H（不再 raise）
|
||||
if g not in seen:
|
||||
seen.add(g)
|
||||
base.append(g)
|
||||
if not seen:
|
||||
raise ValueError("系谱与基因型均为空")
|
||||
order, idx = blup._order_pedigree(base, non_base)
|
||||
n = len(order)
|
||||
parent_of = {i: (d, s) for (i, d, s) in non_base}
|
||||
A, Ainv = blup._build_ainv(order, len(base), parent_of)
|
||||
|
||||
gen_indices = [idx[g] for g in genotyped]
|
||||
if len(gen_indices) < 2:
|
||||
raise ValueError("基因型个体不足 2，无法 ssGBLUP")
|
||||
A22 = A[np.ix_(gen_indices, gen_indices)]
|
||||
G = _scale_g_to_a22(G, A22)
|
||||
try:
|
||||
A22inv = np.linalg.inv(A22 + ridge * np.eye(len(gen_indices)))
|
||||
Ginv = np.linalg.inv(G)
|
||||
except np.linalg.LinAlgError:
|
||||
A22inv = np.linalg.pinv(A22)
|
||||
Ginv = np.linalg.pinv(G)
|
||||
Hinv = Ainv.copy()
|
||||
Hinv[np.ix_(gen_indices, gen_indices)] += Ginv - A22inv
|
||||
|
||||
# H 合并关系阵（协方差；H⁻¹=A⁻¹+[[0,0],[0,G⁻¹−A22⁻¹]]，Aguilar et al. 2010）
|
||||
# H = A + [[A12·A22⁻¹·D·A22⁻¹·A21, A12·A22⁻¹·D], [D·A22⁻¹·A21, D]]，D=G−A22
|
||||
gen_set = set(gen_indices)
|
||||
non_gen = [i for i in range(n) if i not in gen_set]
|
||||
if non_gen:
|
||||
D = G - A22
|
||||
L = A[np.ix_(non_gen, gen_indices)] @ A22inv
|
||||
LD = L @ D
|
||||
H = A.copy()
|
||||
H[np.ix_(non_gen, non_gen)] += LD @ L.T
|
||||
H[np.ix_(non_gen, gen_indices)] += LD
|
||||
H[np.ix_(gen_indices, non_gen)] += LD.T
|
||||
H[np.ix_(gen_indices, gen_indices)] = G
|
||||
else:
|
||||
H = G
|
||||
return order, idx, H, Hinv
|
||||
|
||||
|
||||
def solve_ssgblup(pedigree: list[dict], phenos: dict[str, float],
|
||||
G: np.ndarray, genotyped: list[str], *, tol: float = blup.TOL,
|
||||
ridge: float = _RIDGE):
|
||||
"""ssGBLUP：单步法。genotyped 为有序列表（与 G 行对应）。全系谱个体出 EBV。"""
|
||||
if not pedigree and not phenos:
|
||||
return {"ebv": {}, "reliability": {}, "h2": None, "sigma_a": None, "sigma_e": None,
|
||||
"n_obs": 0, "n_individuals": 0, "n_markers": None,
|
||||
"converged": False, "warning": "系谱与表型均为空"}
|
||||
try:
|
||||
order, idx, H, Hinv = _build_h(
|
||||
pedigree, genotyped, G, ridge=ridge, extra_base=list(phenos))
|
||||
except ValueError as e:
|
||||
return {"ebv": {}, "reliability": {}, "h2": None, "sigma_a": None, "sigma_e": None,
|
||||
"n_obs": 0, "n_individuals": len({r["individual"] for r in pedigree}),
|
||||
"n_markers": None, "converged": False, "warning": str(e)}
|
||||
n = len(order)
|
||||
|
||||
obs = [i for i in order if i in phenos]
|
||||
m = len(obs)
|
||||
if m < 3:
|
||||
return {"ebv": {}, "reliability": {}, "h2": None, "sigma_a": None, "sigma_e": None,
|
||||
"n_obs": m, "n_individuals": n, "n_markers": None,
|
||||
"converged": False, "warning": "表型观测不足 3，无法估计方差组分"}
|
||||
z = np.array([idx[i] for i in obs])
|
||||
y = np.array([float(phenos[i]) for i in obs])
|
||||
X = np.ones((m, 1))
|
||||
ZGZ = H[np.ix_(z, z)] # V 用 H 协方差块（H⁻¹ 仅进 MME 系数矩阵）
|
||||
res = _profile_solve(ZGZ, X, y, Hinv, n, z)
|
||||
out = dict(res)
|
||||
ebv = {order[i]: float(out["ebv"][i]) for i in range(n)}
|
||||
rel = {order[i]: float(out["reliability"][i]) for i in range(n)}
|
||||
out.update(ebv=ebv, reliability=rel, n_obs=m, n_individuals=n, n_markers=len(genotyped))
|
||||
if out["warning"] is None and out.get("h2") is not None and (out["h2"] >= blup.H2_MAX or out["h2"] <= blup.H2_MIN):
|
||||
out["warning"] = "遗传力估计逼近边界，剖面 REML 未完全收敛"
|
||||
return out
|
||||
|
||||
|
||||
def _build_dosage_matrix(genotypes: dict[str, dict[str, float]],
|
||||
markers: list[str], *, maf_min: float = 0.05):
|
||||
"""dosage dict → 有序样本×标记剂量矩阵（与 build_g_matrix 同 MAF 过滤口径）。
|
||||
|
||||
返回 (X[n×q], 保留标记列表, 等位频率 p[q])。X 已按列均值(≈2p)填充缺失。
|
||||
rrBLUP/BayesB 直接消费 X（Xc = X − 2p 即 GBLUP 的 Z）。
|
||||
"""
|
||||
samples = sorted(genotypes)
|
||||
n = len(samples)
|
||||
m = len(markers)
|
||||
M = np.full((n, m), np.nan)
|
||||
for i, s in enumerate(samples):
|
||||
g = genotypes[s]
|
||||
for j, mk in enumerate(markers):
|
||||
v = g.get(mk)
|
||||
if v is not None and not (isinstance(v, float) and math.isnan(v)):
|
||||
M[i, j] = float(v)
|
||||
p = np.zeros(m)
|
||||
for j in range(m):
|
||||
col = M[:, j]
|
||||
valid = col[~np.isnan(col)]
|
||||
p[j] = 0.5 if len(valid) == 0 else float(np.mean(valid)) / 2.0
|
||||
M[:, j] = np.where(np.isnan(M[:, j]), 2.0 * p[j], M[:, j])
|
||||
keep = np.minimum(p, 1.0 - p) >= maf_min
|
||||
if int(keep.sum()) == 0:
|
||||
raise ValueError(f"MAF≥{maf_min} 的标记为 0，请降低 maf_min 或检查标记等位频率")
|
||||
return M[:, keep], [mk for mk, k in zip(markers, keep) if k], p[keep]
|
||||
|
||||
|
||||
def solve_rrblup(individuals: list[str], phenos: dict[str, float],
|
||||
X: np.ndarray, marker_order: list[str], p: np.ndarray,
|
||||
*, tol: float = blup.TOL):
|
||||
"""RR-BLUP（岭回归逐标记）：y = μ + Xu + e，u=(Xc'Xc+λI)⁻¹Xc'yc。
|
||||
|
||||
与 GBLUP 数学对偶（G = XcXc'/c 时 EBV 一致）：方差组分仍用剖面 REML
|
||||
（_profile_solve 同一求解器）估 σa²/σe² → σu²=σa²/c（c=2Σp(1-p)）→
|
||||
λ=σe²/σu²。输出 EBV（基因型个体，含表型截距）+ marker effects（GBLUP
|
||||
无的解释能力）+ 与 GBLUP 同构的可靠性（PEV 近似）。
|
||||
"""
|
||||
n = len(individuals)
|
||||
obs = [i for i in individuals if i in phenos]
|
||||
m = len(obs)
|
||||
if m < 3:
|
||||
return {"ebv": {}, "reliability": {}, "h2": None, "sigma_a": None, "sigma_e": None,
|
||||
"n_obs": m, "n_individuals": n, "n_markers": None, "marker_effects": {},
|
||||
"converged": False, "warning": "表型观测不足 3，无法估计方差组分"}
|
||||
idx = {i: k for k, i in enumerate(individuals)}
|
||||
z = np.array([idx[i] for i in obs])
|
||||
y = np.array([float(phenos[i]) for i in obs])
|
||||
Xc = X - 2.0 * p[None, :] # 中心化剂量 = GBLUP 的 Z
|
||||
q = Xc.shape[1]
|
||||
c = 2.0 * float(np.sum(p * (1.0 - p)))
|
||||
if c <= 0:
|
||||
c = 1.0
|
||||
G = Xc @ Xc.T / c
|
||||
try:
|
||||
Ginv = np.linalg.inv(G)
|
||||
except np.linalg.LinAlgError:
|
||||
Ginv = np.linalg.pinv(G)
|
||||
res = _profile_solve(G[np.ix_(z, z)], np.ones((m, 1)), y, Ginv, n, z)
|
||||
va, ve = res.get("sigma_a"), res.get("sigma_e")
|
||||
if not res.get("converged") or va is None or ve is None or va <= 0 or ve <= 0:
|
||||
return {"ebv": {}, "reliability": {}, "h2": res.get("h2"), "sigma_a": va,
|
||||
"sigma_e": ve, "n_obs": m, "n_individuals": n, "n_markers": q,
|
||||
"marker_effects": {},
|
||||
"converged": False, "warning": res.get("warning") or "方差组分估计失败，rrBLUP 未收敛"}
|
||||
lam = ve / (va / c) if va > 0 else 1.0 # λ = σe²/σu² = σe²·c/σa²
|
||||
yc = y - float(y.mean())
|
||||
Xo = Xc[z]
|
||||
try:
|
||||
u = np.linalg.solve(Xo.T @ Xo + lam * np.eye(q), Xo.T @ yc)
|
||||
except np.linalg.LinAlgError:
|
||||
u = np.linalg.pinv(Xo.T @ Xo + lam * np.eye(q)) @ (Xo.T @ yc)
|
||||
mu = float(y.mean())
|
||||
ebv = {individuals[i]: float(Xc[i] @ u + mu) for i in range(n)}
|
||||
rel = res["reliability"]
|
||||
out = {"ebv": ebv, "reliability": {individuals[i]: float(rel[i]) for i in range(n)},
|
||||
"h2": res["h2"], "sigma_a": float(va), "sigma_e": float(ve),
|
||||
"n_obs": m, "n_individuals": n, "n_markers": q,
|
||||
"marker_effects": {marker_order[j]: float(u[j]) for j in range(q)},
|
||||
"converged": True, "warning": res.get("warning")}
|
||||
return out
|
||||
|
||||
|
||||
def _bayesb_gibbs(Xo: np.ndarray, yc: np.ndarray, *, n_iter: int, burn_in: int,
|
||||
seed: int, pi: float, df_s: float, scale0: float,
|
||||
df_e: float, scale_e0: float) -> tuple[np.ndarray, float, float, int]:
|
||||
"""BayesB 逐标记 Gibbs 核心：给定中心化剂量观测子阵 Xo 与中心化表型 yc。
|
||||
|
||||
返回 (u_hat 后验均值, σe² 末段, SSE 末段, 收敛样本数 n_save)。
|
||||
numpy 固定 seed（default_rng）保证同数据同参数逐位可复现。
|
||||
"""
|
||||
q = Xo.shape[1]
|
||||
m = len(yc)
|
||||
rng = np.random.default_rng(seed)
|
||||
u = np.zeros(q)
|
||||
e = yc.astype(float).copy()
|
||||
sigma_e2 = float(np.var(yc))
|
||||
if sigma_e2 <= 0:
|
||||
sigma_e2 = 1.0
|
||||
u_acc = np.zeros(q)
|
||||
n_save = 0
|
||||
last_sse = float("inf")
|
||||
for it in range(n_iter):
|
||||
for j in range(q):
|
||||
e_star = e + Xo[:, j] * u[j]
|
||||
xjxj = float(Xo[:, j] @ Xo[:, j])
|
||||
xjy = float(Xo[:, j] @ e_star)
|
||||
if rng.uniform() > pi: # 1−π 概率标记入模型
|
||||
s2 = (df_s * scale0 + u[j] * u[j]) / (rng.chisquare(df_s + 1))
|
||||
if s2 <= 0 or xjxj <= 0:
|
||||
u[j] = 0.0
|
||||
else:
|
||||
denom = xjxj + sigma_e2 / s2
|
||||
u[j] = rng.normal(xjy / denom, math.sqrt(sigma_e2 / denom))
|
||||
else:
|
||||
u[j] = 0.0
|
||||
e = e_star - Xo[:, j] * u[j]
|
||||
sse = float(e @ e)
|
||||
sigma_e2 = (df_e * scale_e0 + sse) / rng.chisquare(df_e + m)
|
||||
if sigma_e2 <= 0:
|
||||
sigma_e2 = 1e-8
|
||||
if it >= burn_in:
|
||||
u_acc += u
|
||||
n_save += 1
|
||||
last_sse = sse
|
||||
if n_save == 0:
|
||||
raise ValueError("后验样本为空（burn_in 覆盖全部迭代）")
|
||||
return u_acc / n_save, sigma_e2, last_sse, n_save
|
||||
|
||||
|
||||
def solve_bayesb(individuals: list[str], phenos: dict[str, float],
|
||||
X: np.ndarray, marker_order: list[str], p: np.ndarray,
|
||||
*, n_iter: int = 2000, burn_in: int = 500, seed: int = 20260805,
|
||||
pi: float = 0.95, df_s: float = 5.0, scale0: float = 0.001,
|
||||
df_e: float = 5.0, scale_e0: float = 1.0,
|
||||
tol: float = blup.TOL):
|
||||
"""BayesB（Meuwissen et al. 2001）：贝叶斯可变选择，逐标记 Gibbs 抽样。
|
||||
|
||||
模型 y = μ + Σ Xc_j·u_j + e；每标记 δ_j ~ Bernoulli(1−π) 概率入模型，
|
||||
u_j | δ=1 ~ N(0, σ²j)、σ²j ~ inv-χ²(df_s, scale)；σe² ~ inv-χ²(df_e, scale_e0)。
|
||||
固定 seed（numpy default_rng）保证可复现（MLOps 铁律：同数据同参数重跑逐位一致）。
|
||||
输出 EBV（后验均值，基因型个体，含表型截距）+ marker effects + 收敛诊断。
|
||||
可靠性未单独估计（后验方差才是正确不确定性度量），按 0 保守处理（note 说明）。
|
||||
"""
|
||||
n = len(individuals)
|
||||
obs = [i for i in individuals if i in phenos]
|
||||
m = len(obs)
|
||||
if m < 3:
|
||||
return {"ebv": {}, "reliability": {}, "h2": None, "sigma_a": None, "sigma_e": None,
|
||||
"n_obs": m, "n_individuals": n, "n_markers": None, "marker_effects": {},
|
||||
"converged": False, "warning": "表型观测不足 3，无法估计方差组分"}
|
||||
if n_iter <= burn_in:
|
||||
return {"ebv": {}, "reliability": {}, "h2": None, "sigma_a": None, "sigma_e": None,
|
||||
"n_obs": m, "n_individuals": n, "n_markers": None, "marker_effects": {},
|
||||
"converged": False, "warning": "n_iter 必须大于 burn_in"}
|
||||
idx = {i: k for k, i in enumerate(individuals)}
|
||||
z = np.array([idx[i] for i in obs])
|
||||
y = np.array([float(phenos[i]) for i in obs])
|
||||
Xc = X - 2.0 * p[None, :]
|
||||
q = Xc.shape[1]
|
||||
yc = y - float(y.mean())
|
||||
Xo = Xc[z]
|
||||
|
||||
try:
|
||||
u_hat, sigma_e2, sse, n_save = _bayesb_gibbs(
|
||||
Xo, yc, n_iter=n_iter, burn_in=burn_in, seed=seed, pi=pi,
|
||||
df_s=df_s, scale0=scale0, df_e=df_e, scale_e0=scale_e0)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
return {"ebv": {}, "reliability": {}, "h2": None, "sigma_a": None, "sigma_e": None,
|
||||
"n_obs": m, "n_individuals": n, "n_markers": q, "marker_effects": {},
|
||||
"converged": False, "warning": f"BayesB 抽样异常: {exc!s}"}
|
||||
mu = float(yc.mean())
|
||||
ebv = {individuals[i]: float(Xc[i] @ u_hat + mu) for i in range(n)}
|
||||
note = (f"BayesB π={pi}·{n_iter}iter·{burn_in}burn-in·seed={seed}（后验均值，"
|
||||
f"收敛样本 {n_save}；σe² 末段 SSE={sse:.3f}）")
|
||||
return {"ebv": ebv,
|
||||
"reliability": {individuals[i]: 0.0 for i in range(n)},
|
||||
"h2": None, "sigma_a": None, "sigma_e": float(sigma_e2),
|
||||
"n_obs": m, "n_individuals": n, "n_markers": q,
|
||||
"marker_effects": {marker_order[j]: float(u_hat[j]) for j in range(q)},
|
||||
"converged": True, "warning": note}
|
||||
|
||||
|
||||
def kfold_cv_genomic(phenos: dict[str, float], G: np.ndarray, genotyped: list[str],
|
||||
*, k: int = 5, seed: int = 20260804, method: str = "gblup",
|
||||
pedigree: list[dict] | None = None,
|
||||
ridge: float = _RIDGE,
|
||||
split: str = "random",
|
||||
family_of: dict[str, str] | None = None,
|
||||
genotypes: dict[str, dict[str, float]] | None = None,
|
||||
markers: list[str] | None = None,
|
||||
n_iter: int = 1000, burn_in: int = 300, pi: float = 0.95,
|
||||
df_s: float = 5.0, scale0: float = 0.001,
|
||||
df_e: float = 5.0, scale_e0: float = 1.0) -> dict:
|
||||
"""GS k 折交叉验证（实证预测准确度：留出 GEBV vs 表型 pearson/RMSE）。
|
||||
|
||||
与 blup.kfold_cv 区别：关系矩阵为 VanRaden G（GBLUP）或单步 H（ssGBLUP），
|
||||
折划分默认按**基因型个体随机分层**（固定种子；GS 同世代样本无系谱家系树，
|
||||
随机分层即标准做法，区别于 ABLUP 的 sire 家系留出）。若样本确有家系结构
|
||||
（同父半同胞），随机分层会把同家系亲缘个体摊入 train/test → 泄漏亲缘、
|
||||
高估准确度，此时应传 `split="family"` + `family_of` 走**家系阻塞折**
|
||||
（同家系个体同折，照 blup.kfold_cv 的 sire 家系留出模式）；无家系个体自成一族。
|
||||
家系数 < 折数 k 时折叠为 n_families 折并附 warning。
|
||||
|
||||
method 分派：
|
||||
- `gblup`：VanRaden G（折内 _profile_solve，掩蔽留出经 G⁻¹ 交叉关系预测）。
|
||||
- `ssgblup`：单步 H（需 pedigree）。
|
||||
- `rrblup`：岭回归逐标记（需 genotypes+markers 剂量矩阵；折内 REML 定 λ，
|
||||
EBV=Xc·u+μ 直接由标记基因型预测，不经关系矩阵逆）。
|
||||
- `bayesb`：贝叶斯可变选择（需 genotypes+markers；折内 Gibbs，固定 seed 可复现）。
|
||||
rrblup/bayesb 缺 genotypes/markers 时返回含 warning 的空结果。
|
||||
|
||||
掩蔽留出：个体保留在 G/H 中，仅掩蔽其表型（_profile_solve 的 MME 非观测个体
|
||||
经 G⁻¹/H⁻¹ 交叉关系预测）。G/H/Hinv 只建一次，逐折只换 z_train 子阵。
|
||||
剂量矩阵（Xc）同样只建一次，逐折只取观测行。
|
||||
|
||||
输出 dict 与 blup.kfold_cv 逐键对齐 + `n_families`。cand=genotyped∩phenos（可验证个体）。
|
||||
"""
|
||||
method = (method or "gblup").lower()
|
||||
if method == "ssgblup":
|
||||
order, idx, H, Hinv = _build_h(
|
||||
pedigree or [], genotyped, G, ridge=ridge, extra_base=list(phenos))
|
||||
individuals = order
|
||||
zmat_full = H
|
||||
relmat_inv = Hinv
|
||||
elif method in ("rrblup", "bayesb"):
|
||||
if genotypes is None or markers is None:
|
||||
return {
|
||||
"k": k, "n_total": 0, "n_individuals": 0,
|
||||
"folds": [], "mean_pearson": None, "mean_rmse": None,
|
||||
"pooled_pearson": None, "pooled_rmse": None, "cv_accuracy": None,
|
||||
"h2": None, "n_families": None,
|
||||
"warning": f"{method} 需要 genotypes+markers 剂量矩阵，无法交叉验证",
|
||||
}
|
||||
X_full, marker_order, p_full = _build_dosage_matrix(genotypes, markers)
|
||||
Xc_full = X_full - 2.0 * p_full[None, :]
|
||||
marker_c = 2.0 * float(np.sum(p_full * (1.0 - p_full)))
|
||||
if marker_c <= 0:
|
||||
marker_c = 1.0
|
||||
individuals = sorted(genotypes)
|
||||
idx = {i: k for k, i in enumerate(individuals)}
|
||||
# _profile_solve 需全 n×n Ginv（MME 随机效应块为 Zmat@Zmat.T + λ·Ginv_full），
|
||||
# 折内只取 zmat_full[np.ix_(z,z)] 观测子阵（= XoXo'/c），与 GBLUP 分支同构。
|
||||
zmat_full = Xc_full @ Xc_full.T / marker_c
|
||||
try:
|
||||
relmat_inv = np.linalg.inv(zmat_full)
|
||||
except np.linalg.LinAlgError:
|
||||
relmat_inv = np.linalg.pinv(zmat_full)
|
||||
else:
|
||||
individuals = list(genotyped)
|
||||
idx = {i: k for k, i in enumerate(individuals)}
|
||||
zmat_full = G
|
||||
try:
|
||||
relmat_inv = np.linalg.inv(G)
|
||||
except np.linalg.LinAlgError:
|
||||
relmat_inv = np.linalg.pinv(G)
|
||||
n = len(individuals)
|
||||
cand = sorted([i for i in genotyped if i in phenos])
|
||||
n_total = len(cand)
|
||||
if n_total < 2:
|
||||
return {
|
||||
"k": k, "n_total": n_total, "n_individuals": n,
|
||||
"folds": [], "mean_pearson": None, "mean_rmse": None,
|
||||
"pooled_pearson": None, "pooled_rmse": None, "cv_accuracy": None,
|
||||
"h2": None, "warning": "可验证个体（有基因型+表型）不足 2，无法交叉验证",
|
||||
}
|
||||
k = max(2, min(int(k or 5), 10))
|
||||
rng = random.Random(seed)
|
||||
n_families: int | None = None
|
||||
fam_warning: str | None = None
|
||||
split = (split or "random").lower()
|
||||
if split == "family":
|
||||
fam_of = {ind: (family_of or {}).get(ind, ("self", ind)) for ind in cand}
|
||||
fams = sorted({f for f in fam_of.values()})
|
||||
n_families = len(fams)
|
||||
if n_families < 2:
|
||||
return {
|
||||
"k": k, "n_total": n_total, "n_individuals": n,
|
||||
"folds": [], "mean_pearson": None, "mean_rmse": None,
|
||||
"pooled_pearson": None, "pooled_rmse": None, "cv_accuracy": None,
|
||||
"h2": None, "n_families": n_families,
|
||||
"warning": "可验证个体仅构成 1 个家系，无法家系阻塞交叉验证（请改用随机分层）",
|
||||
}
|
||||
if n_families < k:
|
||||
k = n_families
|
||||
fam_warning = f"家系数 {n_families} < 请求折数，折叠为 {k} 折（family-blocked）"
|
||||
rng.shuffle(fams)
|
||||
fam_bin = {f: i % k for i, f in enumerate(fams)}
|
||||
bin_of = {c: fam_bin[fam_of[c]] for c in cand}
|
||||
else:
|
||||
order_shuf = list(cand)
|
||||
rng.shuffle(order_shuf)
|
||||
bin_of = {c: i % k for i, c in enumerate(order_shuf)}
|
||||
|
||||
folds: list[dict] = []
|
||||
all_x: list[float] = []
|
||||
all_y: list[float] = []
|
||||
for fold in range(k):
|
||||
test_set = {c for c, b in bin_of.items() if b == fold}
|
||||
train_phenos = {i: v for i, v in phenos.items() if i not in test_set}
|
||||
train_obs = [i for i in individuals if i in train_phenos]
|
||||
if not test_set or len(train_obs) < 2:
|
||||
folds.append({
|
||||
"fold": fold + 1, "n_train": len(train_obs), "n_test": len(test_set),
|
||||
"n_eval": 0, "pearson": None, "rmse": None,
|
||||
"error": "训练集个体不足 2 或测试集为空",
|
||||
})
|
||||
continue
|
||||
z = np.array([idx[i] for i in train_obs])
|
||||
y = np.array([float(train_phenos[i]) for i in train_obs])
|
||||
ebv: dict[str, float] | None = None
|
||||
fold_h2: float | None = None
|
||||
res: dict | None = None
|
||||
try:
|
||||
if method in ("rrblup", "bayesb"):
|
||||
Xo = Xc_full[z]
|
||||
yc = y - float(y.mean())
|
||||
if method == "rrblup":
|
||||
res = _profile_solve(zmat_full[np.ix_(z, z)], np.ones((len(train_obs), 1)),
|
||||
y, relmat_inv, n, z)
|
||||
va, ve = res.get("sigma_a"), res.get("sigma_e")
|
||||
if (res.get("converged") and va is not None and ve is not None
|
||||
and va > 0 and ve > 0):
|
||||
lam = ve / (va / marker_c)
|
||||
q = Xc_full.shape[1]
|
||||
try:
|
||||
u = np.linalg.solve(Xo.T @ Xo + lam * np.eye(q), Xo.T @ yc)
|
||||
except np.linalg.LinAlgError:
|
||||
u = np.linalg.pinv(Xo.T @ Xo + lam * np.eye(q)) @ (Xo.T @ yc)
|
||||
mu = float(y.mean())
|
||||
ebv = np.array([float(Xc_full[i] @ u + mu) for i in range(n)])
|
||||
fold_h2 = res.get("h2")
|
||||
else:
|
||||
u_hat, _sigma_e2, _sse, _n_save = _bayesb_gibbs(
|
||||
Xo, yc, n_iter=n_iter, burn_in=burn_in, seed=seed, pi=pi,
|
||||
df_s=df_s, scale0=scale0, df_e=df_e, scale_e0=scale_e0)
|
||||
mu = float(y.mean())
|
||||
ebv = np.array([float(Xc_full[i] @ u_hat + mu) for i in range(n)])
|
||||
res = {"h2": None, "converged": True}
|
||||
else:
|
||||
X = np.ones((len(train_obs), 1))
|
||||
res = _profile_solve(zmat_full[np.ix_(z, z)], X, y, relmat_inv, n, z)
|
||||
if res.get("converged"):
|
||||
ebv = res["ebv"]
|
||||
fold_h2 = res.get("h2")
|
||||
except Exception as e: # noqa: BLE001
|
||||
folds.append({
|
||||
"fold": fold + 1, "n_train": len(train_obs), "n_test": len(test_set),
|
||||
"n_eval": 0, "pearson": None, "rmse": None,
|
||||
"error": f"求解失败: {e!s}",
|
||||
})
|
||||
continue
|
||||
if not res or not res.get("converged") or ebv is None:
|
||||
folds.append({
|
||||
"fold": fold + 1, "n_train": len(train_obs), "n_test": len(test_set),
|
||||
"n_eval": 0, "pearson": None, "rmse": None,
|
||||
"error": (res or {}).get("warning") or "求解失败",
|
||||
})
|
||||
continue
|
||||
xs = [float(ebv[idx[c]]) for c in sorted(test_set)]
|
||||
ys = [float(phenos[c]) for c in sorted(test_set)]
|
||||
if len(xs) < 2:
|
||||
folds.append({
|
||||
"fold": fold + 1, "n_train": len(train_obs), "n_test": len(test_set),
|
||||
"n_eval": len(xs), "pearson": None, "rmse": None,
|
||||
"error": "测试集有效个体不足 2，无法计算 pearson",
|
||||
})
|
||||
continue
|
||||
pearson = blup._pearson(list(xs), list(ys))
|
||||
rmse = math.sqrt(sum((a - b) ** 2 for a, b in zip(xs, ys)) / len(xs))
|
||||
all_x.extend(xs)
|
||||
all_y.extend(ys)
|
||||
folds.append({
|
||||
"fold": fold + 1, "n_train": len(train_obs), "n_test": len(test_set),
|
||||
"n_eval": len(xs), "pearson": round(pearson, 4),
|
||||
"rmse": round(rmse, 6),
|
||||
"h2": float(fold_h2) if fold_h2 is not None else None,
|
||||
})
|
||||
|
||||
valid = [f for f in folds if f.get("pearson") is not None]
|
||||
mean_pearson = round(sum(f["pearson"] for f in valid) / len(valid), 4) if valid else None
|
||||
rmse_vals = [f["rmse"] for f in valid if f.get("rmse") is not None]
|
||||
mean_rmse = round(sum(rmse_vals) / len(rmse_vals), 6) if rmse_vals else None
|
||||
pooled_pearson = round(blup._pearson(all_x, all_y), 4) if len(all_x) >= 2 else None
|
||||
pooled_rmse = (round(math.sqrt(sum((a - b) ** 2 for a, b in zip(all_x, all_y)) / len(all_x)), 6)
|
||||
if len(all_x) else None)
|
||||
h2_vals = [f["h2"] for f in valid if f.get("h2") is not None]
|
||||
h2_mean = round(sum(h2_vals) / len(h2_vals), 4) if h2_vals else None
|
||||
errors = [f.get("error") for f in folds if f.get("error")]
|
||||
warning_parts = []
|
||||
if fam_warning:
|
||||
warning_parts.append(fam_warning)
|
||||
if errors:
|
||||
warning_parts.append(f"{len(errors)} 折失败: " + "；".join(errors))
|
||||
warning = "；".join(warning_parts) if warning_parts else None
|
||||
return {
|
||||
"k": k, "n_total": n_total, "n_individuals": n,
|
||||
"folds": folds,
|
||||
"mean_pearson": mean_pearson, "mean_rmse": mean_rmse,
|
||||
"pooled_pearson": pooled_pearson, "pooled_rmse": pooled_rmse,
|
||||
"cv_accuracy": mean_pearson, "h2": h2_mean,
|
||||
"n_families": n_families, "warning": warning,
|
||||
}
|
||||
Reference in New Issue
Block a user