From 036494ae3581124be5a80b7d0f3a7e8daf87b577 Mon Sep 17 00:00:00 2001 From: "34047007@qq.com" <34047007@qq.com> Date: Mon, 27 Jul 2026 08:13:53 +0800 Subject: [PATCH] chore: add Gitea service to docker-compose, fix gitignore - Gitea (postgres-backed) on ports 3000/2222 - Exclude .claude/ dir and *.db from git - Update CLAUDE.md with Gitea remote info --- .gitignore | 6 +++++- CLAUDE.md | 29 +++++++++++++++++------------ docker-compose.prod.yml | 25 +++++++++++++++++++++++++ 3 files changed, 47 insertions(+), 13 deletions(-) diff --git a/.gitignore b/.gitignore index 799e622..5138193 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,9 @@ frontend/dist/ .vscode/settings.json .idea/ +# Claude Code local config (per-machine, not for repo) +.claude/ + # OS .DS_Store Thumbs.db @@ -36,7 +39,8 @@ Thumbs.db # Runtime data (token store, etc.) backend/data/ -# Database (PostgreSQL data dir - local only) +# Database +*.db /pgdata/ # Logs diff --git a/CLAUDE.md b/CLAUDE.md index 23e3212..2b89fe4 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -29,7 +29,7 @@ docker compose -f docker-compose.prod.yml up -d # production（自动运行迁 # PubMed pipeline (real data) curl -X POST localhost:8000/api/v1/admin/pipeline/run -H "Authorization: Bearer $(admin_token)" -# FTP bulk import +# FTP baseline import (first-time full data) cd backend && python scripts/pubmed_baseline.py --dir /path/to/pubmed/baseline/ --demo # 批量刷新被引次数（PubMed elink，免费，3 req/s。全库约N分钟/N秒） curl -X POST localhost:8000/api/v1/admin/pipeline/refresh-citations -H "Authorization: Bearer $(admin_token)" @@ -88,13 +88,17 @@ Three layouts: `PublicLayout` (no auth, no sidebar), `AuthLayout` (centered card ### PubMed Pipeline -Two modes: API (`pubmed_api.py` using NCBI E-utilities, 3 req/s) and FTP (`pubmed_baseline.py` for bulk). Both filter articles by MeSH tags defined in `config/specialties/oncology.yaml`. Tagging is bidirectional: MeSH UI → `global_tags` lookup → `global_literature_tags` INSERT. +**FTP 每日更新文件为主数据源**（2026-07-25 决策）。取代旧 E-utilities API 多路搜索策略。 -**两阶段召回策略：** -1. **MAJR 高精度** — `ONCOLOGY_SEARCH_QUERIES`，`[MAJR]` 限定 MeSH Major Topic，仅 indexed 记录 -2. **Title/Abstract 高召回** — `BROAD_ONCOLOGY_QUERIES`，关键词覆盖 in-process + publisher 记录 +日常运行： +1. **FTP 下载** `pubmed25updateNNNN.xml.gz`（每日 ~10MB，含全部新增/修改/删除记录） +2. **解析**：`_extract_article()`（复用 `pubmed_baseline.py` 的 lxml 解析器） +3. **过滤**：`_is_oncology()` 按 MeSH 肿瘤科筛选（复用 `pubmed_baseline.py`） +4. **处理**：`` → `_update_lit_from_article()` upsert；`` → 标记 `retracted=True` +5. **打标**: MeSH UI → `global_tags` lookup → `global_literature_tags` INSERT +6. **检查点**：`pipeline_runs.processed_date` 记录已处理的 EDAT 日期 -`_run_pipeline()` 是共享核心（`pubmed_api.py:498`），通过 `use_majr`/`use_broad` 标志控制执行集合。标题/抽象查询通过 `seen_pmids` 集合自动去重。已存在的 PMID 通过 `_update_lit_from_article()` 原地覆盖更新。 +降级：FTP 不可用时，回退到 `pubmed_api.py`（NCBI E-utilities，3 req/s）的 `reldate=1&datetype=edat` 查询。 ### ARQ Scheduled Tasks (`backend/app/tasks/worker.py`) @@ -102,19 +106,19 @@ Two modes: API (`pubmed_api.py` using NCBI E-utilities, 3 req/s) and FTP (`pubme | 任务 | 函数 | Cron (UTC) | 北京时间 | 说明 | |---|---|---|---|---| -| 每日精搜 | `daily_pubmed_pipeline` | `03:07` 每天 | 11:07 | MAJR MeSH 高精度搜索，20 篇/query | -| 每周宽搜 | `weekly_broad_pipeline` | `03:37` 周日 | 11:37 | Title/Abstract 覆盖 in-process + publisher | +| 每日 FTP 增量 | `daily_ftp_update` | `03:07` 每天 | 11:07 | FTP 下载更新文件，处理新增/修改/删除（取代精搜+宽搜+retagger） | | 引用更新 | `daily_citation_update` | `05:13` 每天 | 13:13 | 刷新最近文献被引次数 | | 摘要邮件 | `daily_digest_task` | `22:30` 每天 | 06:30 (次日) | 每日摘要推送 | -`weekly_broad_pipeline` 与 `daily_pubmed_pipeline` 错开 30 分钟执行，避免 PubMed API 限速竞争。 +> **旧任务已移除：** `daily_pubmed_pipeline`（MAJR 精搜）、`weekly_broad_pipeline`（Title/Abstract 宽搜）和 `mesh_retagger.py`（回查）已被 `daily_ftp_update` 完全取代。FTP 每日更新文件包含所有新/改/删记录，覆盖 MeSH in-process→medline 过渡，不再需要多路搜索和回查。 手动触发： ```bash -# 全量运行（MAJR + 宽搜） +# FTP 增量更新 curl -X POST localhost:8000/api/v1/admin/pipeline/run -H "Authorization: Bearer $(admin_token)" -# 仅宽搜 -curl -X POST "localhost:8000/api/v1/admin/pipeline/run?mode=broad" -H "Authorization: Bearer $(admin_token)" +# 批量刷新被引次数 +curl -X POST localhost:8000/api/v1/admin/pipeline/refresh-citations -H "Authorization: Bearer $(admin_token)" +curl -X POST "localhost:8000/api/v1/admin/pipeline/refresh-citations?limit=500" -H "Authorization: Bearer $(admin_token)" ``` ### Citation Counts (PubMed elink) @@ -166,3 +170,4 @@ curl -X POST "localhost:8000/api/v1/admin/pipeline/run?mode=broad" -H "Authoriza - **SQLite in dev, PostgreSQL in prod.** SQLAlchemy generic types enable this. - **Dev mode password reset** returns the reset link directly in API response (no SMTP needed). - **`user["sub"]` is a string.** Always convert to `uuid.UUID()` before passing to SQLAlchemy queries. +- **搜索功能必须与 PubMed 完全一致。** 这是硬性要求，不是"未来优化"。所有 PubMed 字段标签必须全量支持，已存储数据的立即接通搜索路径，缺失数据的补充 XML 抽取和存储。不允许任何字段退化到纯文本搜索。 diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 18ec592..a3da29d 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -259,8 +259,33 @@ services: max-size: "10m" max-file: "3" + gitea: + image: gitea/gitea:latest-rootless + restart: unless-stopped + networks: + - scilit + volumes: + - gitea_data:/var/lib/gitea + environment: + GITEA__database__DB_TYPE: postgres + GITEA__database__HOST: postgres:5432 + GITEA__database__NAME: gitea + GITEA__database__USER: gitea + GITEA__database__PASSWD: gitea_pass_2026 + GITEA__server__DOMAIN: 123.207.9.209 + GITEA__server__HTTP_PORT: 3000 + GITEA__server__ROOT_URL: http://123.207.9.209:3000 + GITEA__server__START_SSH_SERVER: "true" + GITEA__server__SSH_DOMAIN: 123.207.9.209 + GITEA__server__SSH_PORT: 2222 + GITEA__server__SSH_LISTEN_PORT: 22 + ports: + - "3000:3000" + - "2222:22" + volumes: pgdata: redisdata: esdata: miniodata: + gitea_data: