#!/bin/bash
# PubMed 增量更新逐文件下载、验证、导入脚本
# 用法: ssh到服务器后: nohup bash /tmp/pubmed_update_one_by_one.sh &
# 日志: tail -f /data/pubmed/import_update_files.log

set -euo pipefail

STAGING=/data/pubmed/staging
WORKDIR=/data/pubmed/updatefiles
IMPORTED=/data/pubmed/updatefiles/imported
COMPOSE_FILE=/root/scilit/docker-compose.prod.yml
LOG=/data/pubmed/import_update_files.log

mkdir -p "$STAGING" "$WORKDIR" "$IMPORTED"

log() {
    echo "[$(TZ=Asia/Shanghai date '+%Y-%m-%d %H:%M:%S')] $*" | tee -a "$LOG"
}

log "=== 开始逐文件导入：195 个增量文件（1335-1529）==="
log "目录: $STAGING -> $WORKDIR -> $IMPORTED"

GOOD=0
SKIP=0
FAIL=0

for i in $(seq 1335 1529); do
    FILE="pubmed26n${i}.xml.gz"
    URL="https://ftp.ncbi.nlm.nih.gov/pubmed/updatefiles/${FILE}"

    log "--- [$((i-1334))/195] $FILE ---"

    # Step 1: Download (single connection, reliable)
    log "  下载中..."
    if ! aria2c -x1 -s1 --retry-wait=3 --max-tries=3 \
        "$URL" -d "$STAGING" -o "$FILE" \
        >> "$LOG" 2>&1; then
        log "  ERROR: 下载失败，跳过"
        SKIP=$((SKIP+1))
        rm -f "$STAGING/$FILE"
        continue
    fi

    # Step 2: Gzip integrity check
    log "  验证完整性..."
    if ! gzip -t "$STAGING/$FILE" 2>/dev/null; then
        log "  CORRUPT: 文件损坏，重试一次..."
        rm -f "$STAGING/$FILE"
        aria2c -x1 -s1 --retry-wait=3 --max-tries=3 \
            "$URL" -d "$STAGING" -o "$FILE" \
            >> "$LOG" 2>&1 || true
        if ! gzip -t "$STAGING/$FILE" 2>/dev/null; then
            log "  FAILED: 重试后仍损坏，跳过"
            SKIP=$((SKIP+1))
            rm -f "$STAGING/$FILE"
            continue
        fi
    fi

    # Step 3: Prep workdir (only this file)
    rm -f "$WORKDIR"/*.xml.gz
    cp "$STAGING/$FILE" "$WORKDIR/"

    # Step 4: Import via docker (--no-deps 避免每次重新启动 migrate)
    log "  导入中..."
    if docker compose -f "$COMPOSE_FILE" run --no-deps --rm \
        -v /data/pubmed/updatefiles:/data/pubmed_baseline \
        worker python /app/scripts/pubmed_baseline.py \
        --dir /data/pubmed_baseline >> "$LOG" 2>&1; then
        mv "$WORKDIR/$FILE" "$IMPORTED/"
        GOOD=$((GOOD+1))
        log "  ✅ 成功: $FILE"
    else
        rm -f "$WORKDIR/$FILE"
        FAIL=$((FAIL+1))
        log "  ❌ 导入失败: $FILE"
    fi

    # Clean staging
    rm -f "$STAGING/$FILE"
done

log "=== 逐文件导入完成 ==="
log "结果: 成功=$GOOD 跳过=$SKIP 失败=$FAIL"
log "详见: $IMPORTED/"