34047007@qq.com
a6cd99a4ca
OncoLit: a multi-tenant oncology literature search, feed, and collaboration platform. Built with FastAPI + Vue 3 + PostgreSQL. Includes PubMed pipeline, drug approvals, AI summaries, and systematic review tools.
254 lines
10 KiB
Python
254 lines
10 KiB
Python
"""GlobalJournal 种子化：从 global_literature 提取所有不重复期刊
|
||
|
||
从现有数据自动提取期刊并写入 global_journals 表。
|
||
- 有 ISSN 的记录按 ISSN 分组，取出现频率最高的 journal 名称作为 canonical name
|
||
- 无 ISSN 的记录按 journal 名称聚合
|
||
- 已有 curated 记录（is_active=True）保留原有 tier 不动
|
||
- 新增记录默认 tier=4、is_active=True
|
||
- 自动计算 priority_score 和 specialty
|
||
"""
|
||
|
||
import os
|
||
import sys
|
||
import uuid as _uuid
|
||
from collections import Counter
|
||
from datetime import datetime
|
||
|
||
from sqlalchemy import create_engine, text, select, func, String
|
||
from sqlalchemy.orm import Session
|
||
|
||
# ── 数据库连接 ──
|
||
_DEFAULT_URL = "postgresql://scilit:scilit_dev@localhost:5432/scilit"
|
||
|
||
def _get_engine():
|
||
url = os.environ.get("SEED_DB_URL") or _DEFAULT_URL
|
||
return create_engine(url, pool_pre_ping=True)
|
||
|
||
|
||
def _score(tier: str, impact_factor: float | None, article_count: int, specialty: str | None) -> float:
|
||
"""内联版评分，避免依赖 app.services.journal_utils（独立脚本）"""
|
||
tier_base = {"1": 80, "2": 65, "3": 50}.get(tier, 20)
|
||
if_bonus = 0
|
||
if impact_factor is not None:
|
||
if impact_factor >= 20: if_bonus = 15
|
||
elif impact_factor >= 10: if_bonus = 10
|
||
elif impact_factor >= 5: if_bonus = 5
|
||
elif impact_factor >= 2: if_bonus = 2
|
||
count_bonus = 0
|
||
if article_count >= 5000: count_bonus = 10
|
||
elif article_count >= 1000: count_bonus = 5
|
||
elif article_count >= 100: count_bonus = 2
|
||
specialty_bonus = 10 if specialty == "oncology" else 0
|
||
return min(tier_base + if_bonus + count_bonus + specialty_bonus, 100.0)
|
||
|
||
|
||
def _spec(tier: str, name: str | None) -> str | None:
|
||
"""内联版专科判定"""
|
||
if tier in ("1", "2", "3"):
|
||
return "oncology"
|
||
if name:
|
||
nl = name.lower()
|
||
for kw in ("oncol", "cancer", "tumor", "tumour", "neoplas"):
|
||
if kw in nl:
|
||
return "oncology"
|
||
return None
|
||
|
||
|
||
def seed_journals(dry_run: bool = False):
|
||
"""主流程"""
|
||
with Session(_get_engine()) as db:
|
||
# 1. 已有的 GlobalJournal 记录
|
||
existing = db.execute(
|
||
select(text("id, issn, eissn, name, tier, is_active"))
|
||
.select_from(text("global_journals"))
|
||
).all()
|
||
existing_by_issn: dict[str, dict] = {}
|
||
for row in existing:
|
||
if row.issn:
|
||
existing_by_issn[row.issn] = {
|
||
"id": row.id, "name": row.name, "tier": row.tier or "4",
|
||
"is_active": row.is_active,
|
||
}
|
||
if row.eissn:
|
||
existing_by_issn[row.eissn] = {
|
||
"id": row.id, "name": row.name, "tier": row.tier or "4",
|
||
"is_active": row.is_active,
|
||
}
|
||
print(f"已有 GlobalJournal 记录: {len(existing)} 条")
|
||
|
||
# 2. 从 global_literature 提取期刊
|
||
rows = db.execute(
|
||
text("SELECT journal, journal_issn FROM global_literature WHERE journal IS NOT NULL AND journal != ''")
|
||
).all()
|
||
print(f"global_literature 有期刊名称的记录: {len(rows)} 条")
|
||
|
||
# 按 ISSN 分组统计 journal 名称频次
|
||
issn_groups: dict[str, Counter] = {} # issn → Counter(journal_name)
|
||
no_issn_counter: Counter = Counter() # 无 ISSN 的 journal 名称
|
||
issn_journal_map: dict[str, str] = {} # issn → 最常见 journal name
|
||
|
||
for journal, issn in rows:
|
||
if issn:
|
||
issn = issn.strip()
|
||
if issn not in issn_groups:
|
||
issn_groups[issn] = Counter()
|
||
issn_groups[issn][journal.strip()] += 1
|
||
else:
|
||
no_issn_counter[journal.strip()] += 1
|
||
|
||
# 计算每 ISSN 最常见的名称
|
||
for issn, counter in issn_groups.items():
|
||
issn_journal_map[issn] = counter.most_common(1)[0][0]
|
||
|
||
# 3. 决定哪些需要插入/更新
|
||
to_insert = [] # (issn, name, article_count, publisher)
|
||
to_update_count = []
|
||
total_new = 0
|
||
total_existing = 0
|
||
|
||
# 有 ISSN 的记录
|
||
for issn, name in issn_journal_map.items():
|
||
article_count = issn_groups[issn].total()
|
||
if issn in existing_by_issn:
|
||
total_existing += 1
|
||
to_update_count.append((article_count, issn))
|
||
else:
|
||
# 跨 ISSN 去重：检查是否有名称高度相似（trigram > 0.9）的已有期刊
|
||
merged = False
|
||
try:
|
||
sim_check = db.execute(
|
||
text("""
|
||
SELECT id, name, issn, eissn FROM global_journals
|
||
WHERE issn IS NOT NULL AND issn != :issn
|
||
AND similarity(name, :name) > 0.9
|
||
LIMIT 1
|
||
"""),
|
||
{"issn": issn, "name": name},
|
||
).fetchone()
|
||
if sim_check:
|
||
# 名称高度相似 → 视为同刊，将新 ISSN 并入 eissn
|
||
target_eissn = sim_check.eissn or ""
|
||
new_eissn = issn if not target_eissn else target_eissn
|
||
db.execute(
|
||
text("""
|
||
UPDATE global_journals
|
||
SET eissn = :eissn, article_count = article_count + :cnt,
|
||
updated_at = NOW()
|
||
WHERE id = :id
|
||
"""),
|
||
{"eissn": new_eissn, "cnt": article_count, "id": sim_check.id},
|
||
)
|
||
print(f" ↳ 跨 ISSN 合并: {name[:50]} ({issn}) → {sim_check.name[:50]} ({sim_check.issn})")
|
||
total_existing += 1
|
||
merged = True
|
||
except Exception:
|
||
pass # pg_trgm 可能未安装
|
||
|
||
if not merged:
|
||
total_new += 1
|
||
specialty = _spec("4", name)
|
||
priority_score = _score("4", None, article_count, specialty)
|
||
to_insert.append({
|
||
"id": str(_uuid.uuid4()),
|
||
"name": name[:500],
|
||
"issn": issn,
|
||
"eissn": None,
|
||
"tier": "4",
|
||
"impact_factor": None,
|
||
"publisher": None,
|
||
"article_count": article_count,
|
||
"is_active": True,
|
||
"priority_score": priority_score,
|
||
"priority_source": "auto",
|
||
"specialty": specialty,
|
||
})
|
||
|
||
# 无 ISSN 的记录（按名称聚合，article_count >= 3 才收录以避免垃圾）
|
||
no_issn_new = 0
|
||
for name, count in no_issn_counter.most_common():
|
||
if count < 3:
|
||
continue
|
||
# 检查是否已存在同名记录
|
||
exists = db.execute(
|
||
text("SELECT 1 FROM global_journals WHERE name = :name AND issn IS NULL LIMIT 1"),
|
||
{"name": name},
|
||
).scalar()
|
||
if exists:
|
||
db.execute(
|
||
text("UPDATE global_journals SET article_count = article_count + :cnt WHERE name = :name AND issn IS NULL"),
|
||
{"cnt": count, "name": name},
|
||
)
|
||
continue
|
||
to_insert.append({
|
||
"id": str(_uuid.uuid4()),
|
||
"name": name[:500],
|
||
"issn": None,
|
||
"eissn": None,
|
||
"tier": "4",
|
||
"impact_factor": None,
|
||
"publisher": None,
|
||
"article_count": count,
|
||
"is_active": True,
|
||
"priority_score": _score("4", None, count, _spec("4", name)),
|
||
"priority_source": "auto",
|
||
"specialty": _spec("4", name),
|
||
})
|
||
no_issn_new += 1
|
||
|
||
print(f"\n统计:")
|
||
print(f" 已有 ISSN 匹配（更新 article_count）: {total_existing}")
|
||
print(f" 新 ISSN 期刊: {total_new}")
|
||
print(f" 无 ISSN 新期刊: {no_issn_new}")
|
||
print(f" 总计待插入: {len(to_insert)}")
|
||
|
||
if dry_run:
|
||
print("\n[Dry Run] 不执行写入。")
|
||
return
|
||
|
||
# 提交 Session 事务，释放表锁，避免与下面 raw_connection 的 INSERT 冲突
|
||
db.commit()
|
||
|
||
# 4. 批量插入
|
||
if to_insert:
|
||
now_val = datetime.utcnow()
|
||
conn = _get_engine().raw_connection()
|
||
try:
|
||
with conn.cursor() as cur:
|
||
import psycopg2.extras
|
||
psycopg2.extras.execute_values(
|
||
cur,
|
||
"""INSERT INTO global_journals (id, name, issn, eissn, tier, impact_factor, publisher, article_count, is_active, priority_score, priority_source, specialty, created_at, updated_at)
|
||
VALUES %s
|
||
ON CONFLICT DO NOTHING""",
|
||
[(
|
||
r["id"], r["name"][:500], r["issn"], r["eissn"],
|
||
r["tier"], r["impact_factor"], r["publisher"],
|
||
r["article_count"], r["is_active"],
|
||
r["priority_score"], r["priority_source"], r["specialty"],
|
||
now_val, now_val,
|
||
) for r in to_insert],
|
||
template="(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)",
|
||
)
|
||
conn.commit()
|
||
finally:
|
||
conn.close()
|
||
print(f"\n插入 {len(to_insert)} 条记录。")
|
||
|
||
# 5. 更新已有记录的 article_count
|
||
for cnt, issn in to_update_count:
|
||
db.execute(
|
||
text("UPDATE global_journals SET article_count = :cnt WHERE issn = :issn"),
|
||
{"cnt": cnt, "issn": issn},
|
||
)
|
||
db.commit()
|
||
print(f"更新 {len(to_update_count)} 条已有记录的 article_count。")
|
||
|
||
# 6. 最终统计
|
||
final_count = db.execute(text("SELECT COUNT(*) FROM global_journals")).scalar()
|
||
print(f"\n完成！global_journals 总记录数: {final_count}")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
dry = "--dry-run" in sys.argv
|
||
seed_journals(dry_run=dry)
|