feat: initial commit - oncology literature search platform

OncoLit: a multi-tenant oncology literature search, feed, and collaboration platform. Built with FastAPI + Vue 3 + PostgreSQL. Includes PubMed pipeline, drug approvals, AI summaries, and systematic review tools.
2026-07-27 07:59:18 +08:00
commit a6cd99a4ca
473 changed files with 151472 additions and 0 deletions
@@ -0,0 +1,85 @@
+#!/bin/bash
+# PubMed 增量更新逐文件下载、验证、导入脚本
+# 用法: ssh到服务器后: nohup bash /tmp/pubmed_update_one_by_one.sh &
+# 日志: tail -f /data/pubmed/import_update_files.log
+
+set -euo pipefail
+
+STAGING=/data/pubmed/staging
+WORKDIR=/data/pubmed/updatefiles
+IMPORTED=/data/pubmed/updatefiles/imported
+COMPOSE_FILE=/root/scilit/docker-compose.prod.yml
+LOG=/data/pubmed/import_update_files.log
+
+mkdir -p "$STAGING" "$WORKDIR" "$IMPORTED"
+
+log() {
+    echo "[$(TZ=Asia/Shanghai date '+%Y-%m-%d %H:%M:%S')] $*" | tee -a "$LOG"
+}
+
+log "=== 开始逐文件导入：195 个增量文件（1335-1529）==="
+log "目录: $STAGING -> $WORKDIR -> $IMPORTED"
+
+GOOD=0
+SKIP=0
+FAIL=0
+
+for i in $(seq 1335 1529); do
+    FILE="pubmed26n${i}.xml.gz"
+    URL="https://ftp.ncbi.nlm.nih.gov/pubmed/updatefiles/${FILE}"
+
+    log "--- [$((i-1334))/195] $FILE ---"
+
+    # Step 1: Download (single connection, reliable)
+    log "  下载中..."
+    if ! aria2c -x1 -s1 --retry-wait=3 --max-tries=3 \
+        "$URL" -d "$STAGING" -o "$FILE" \
+        >> "$LOG" 2>&1; then
+        log "  ERROR: 下载失败，跳过"
+        SKIP=$((SKIP+1))
+        rm -f "$STAGING/$FILE"
+        continue
+    fi
+
+    # Step 2: Gzip integrity check
+    log "  验证完整性..."
+    if ! gzip -t "$STAGING/$FILE" 2>/dev/null; then
+        log "  CORRUPT: 文件损坏，重试一次..."
+        rm -f "$STAGING/$FILE"
+        aria2c -x1 -s1 --retry-wait=3 --max-tries=3 \
+            "$URL" -d "$STAGING" -o "$FILE" \
+            >> "$LOG" 2>&1 || true
+        if ! gzip -t "$STAGING/$FILE" 2>/dev/null; then
+            log "  FAILED: 重试后仍损坏，跳过"
+            SKIP=$((SKIP+1))
+            rm -f "$STAGING/$FILE"
+            continue
+        fi
+    fi
+
+    # Step 3: Prep workdir (only this file)
+    rm -f "$WORKDIR"/*.xml.gz
+    cp "$STAGING/$FILE" "$WORKDIR/"
+
+    # Step 4: Import via docker (--no-deps 避免每次重新启动 migrate)
+    log "  导入中..."
+    if docker compose -f "$COMPOSE_FILE" run --no-deps --rm \
+        -v /data/pubmed/updatefiles:/data/pubmed_baseline \
+        worker python /app/scripts/pubmed_baseline.py \
+        --dir /data/pubmed_baseline >> "$LOG" 2>&1; then
+        mv "$WORKDIR/$FILE" "$IMPORTED/"
+        GOOD=$((GOOD+1))
+        log "  ✅ 成功: $FILE"
+    else
+        rm -f "$WORKDIR/$FILE"
+        FAIL=$((FAIL+1))
+        log "  ❌ 导入失败: $FILE"
+    fi
+
+    # Clean staging
+    rm -f "$STAGING/$FILE"
+done
+
+log "=== 逐文件导入完成 ==="
+log "结果: 成功=$GOOD 跳过=$SKIP 失败=$FAIL"
+log "详见: $IMPORTED/"