feat: initial commit - oncology literature search platform

OncoLit: a multi-tenant oncology literature search, feed, and collaboration platform. Built with FastAPI + Vue 3 + PostgreSQL. Includes PubMed pipeline, drug approvals, AI summaries, and systematic review tools.
2026-07-27 07:59:18 +08:00
commit a6cd99a4ca
473 changed files with 151472 additions and 0 deletions
@@ -0,0 +1,103 @@
+"""
+Test a full batch of 200 YYYY-01-01 records via efetch
+"""
+import asyncio, httpx, lxml.etree as ET
+from datetime import date
+from sqlalchemy import text
+from sqlalchemy.ext.asyncio import create_async_engine
+
+DATABASE_URL = 'postgresql+asyncpg://scilit:scilit_prod_2026@postgres:5432/scilit'
+
+MONTH_MAP = {
+    'jan':1,'january':1,'1':1,'feb':2,'february':2,'2':2,'mar':3,'march':3,'3':3,
+    'apr':4,'april':4,'4':4,'may':5,'5':5,'jun':6,'june':6,'6':6,
+    'jul':7,'july':7,'7':7,'aug':8,'august':8,'8':8,'sep':9,'september':9,'9':9,
+    'oct':10,'october':10,'10':10,'nov':11,'november':11,'11':11,'dec':12,'december':12,'12':12,
+}
+DAYS = [0,31,29,31,30,31,30,31,31,30,31,30,31]
+
+def month_num(m): return MONTH_MAP.get(m.strip().lower(), 1)
+def clamp_day(m, d, y):
+    if d < 1: return 1
+    md = DAYS[m]
+    if m == 2 and d == 29 and not (y%4==0 and (y%100!=0 or y%400==0)): md = 28
+    return min(d, md)
+
+def parse_pub_date(elem):
+    pde = elem.find('.//PubDate')
+    if pde is None: return None
+    ye = pde.find('Year'); me = pde.find('Month'); de = pde.find('Day')
+    if ye is not None and ye.text:
+        y = int(ye.text.strip())
+        hm = me is not None and me.text; hd = de is not None and de.text
+        if not hm and not hd: return None
+        m = month_num(me.text.strip()) if hm else 1
+        d = int(de.text.strip()) if hd else 1
+        d = clamp_day(m, d, y)
+        return date(y, m, d)
+    return None
+
+async def main():
+    engine = create_async_engine(DATABASE_URL)
+    async with engine.connect() as conn:
+        # Get 200 "middle" 2016 PMIDs (not newest, not oldest)
+        r = await conn.execute(text("""
+            SELECT id, pmid, pub_date, pub_year FROM global_literature
+            WHERE pub_date IS NOT NULL AND EXTRACT(MONTH FROM pub_date)=1 AND EXTRACT(DAY FROM pub_date)=1
+            AND pmid IS NOT NULL AND pub_year = 2016
+            ORDER BY pmid ASC
+            OFFSET 5000 LIMIT 200
+        """))
+        rows = r.all()
+    await engine.dispose()
+
+    print(f'Testing {len(rows)} PMIDs (2016, offset=5000) via efetch...')
+    pmids = [str(row.pmid) for row in rows]
+    print(f'PMID range: {pmids[0]} ~ {pmids[-1]}')
+
+    async with httpx.AsyncClient(timeout=60) as c:
+        resp = await c.get(
+            'https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi',
+            params={'db': 'pubmed', 'id': ','.join(pmids), 'retmode': 'xml'}
+        )
+        print(f'HTTP status={resp.status_code} content_len={len(resp.content)}')
+        root = ET.fromstring(resp.content)
+        arts = root.findall('PubmedArticle')
+        print(f'Articles returned: {len(arts)}')
+
+        fixed = 0; year_only = 0; still_jan01 = 0; no_pmid = 0
+        efetch_map = {}
+        for art in arts:
+            pmid_e = art.find('MedlineCitation/PMID')
+            pmid = int(pmid_e.text) if pmid_e is not None else None
+            pd = parse_pub_date(art)
+            if pmid: efetch_map[pmid] = pd
+            else: no_pmid += 1
+
+        analysis = []
+        for row in rows:
+            pd = efetch_map.get(row.pmid)
+            if pd is None:
+                year_only += 1
+                analysis.append((row.pmid, 'year_only'))
+            elif pd.month == 1 and pd.day == 1:
+                still_jan01 += 1
+                analysis.append((row.pmid, 'still_jan01'))
+            else:
+                fixed += 1
+                analysis.append((row.pmid, f'FIXED->{pd}'))
+
+        print(f'\n=== Batch Analysis ===')
+        print(f'  Can fix (efetch has better date): {fixed}')
+        print(f'  Year-only in efetch:              {year_only}')
+        print(f'  Still Jan-01 in efetch:           {still_jan01}')
+        print(f'  Articles without PMID in response: {no_pmid}')
+        if fixed:
+            print(f'\nSample fixes:')
+            for pmid, result in analysis:
+                if result.startswith('FIXED'):
+                    print(f'  PMID={pmid} -> {result}')
+        if year_only:
+            print(f'\nSample year-only PMIDs: {[pmid for pmid, r in analysis[:5] if r == "year_only"]}')
+
+asyncio.run(main())