34047007@qq.com
a6cd99a4ca
OncoLit: a multi-tenant oncology literature search, feed, and collaboration platform. Built with FastAPI + Vue 3 + PostgreSQL. Includes PubMed pipeline, drug approvals, AI summaries, and systematic review tools.
104 lines
4.0 KiB
Python
104 lines
4.0 KiB
Python
"""
|
|
Test a full batch of 200 YYYY-01-01 records via efetch
|
|
"""
|
|
import asyncio, httpx, lxml.etree as ET
|
|
from datetime import date
|
|
from sqlalchemy import text
|
|
from sqlalchemy.ext.asyncio import create_async_engine
|
|
|
|
DATABASE_URL = 'postgresql+asyncpg://scilit:scilit_prod_2026@postgres:5432/scilit'
|
|
|
|
MONTH_MAP = {
|
|
'jan':1,'january':1,'1':1,'feb':2,'february':2,'2':2,'mar':3,'march':3,'3':3,
|
|
'apr':4,'april':4,'4':4,'may':5,'5':5,'jun':6,'june':6,'6':6,
|
|
'jul':7,'july':7,'7':7,'aug':8,'august':8,'8':8,'sep':9,'september':9,'9':9,
|
|
'oct':10,'october':10,'10':10,'nov':11,'november':11,'11':11,'dec':12,'december':12,'12':12,
|
|
}
|
|
DAYS = [0,31,29,31,30,31,30,31,31,30,31,30,31]
|
|
|
|
def month_num(m): return MONTH_MAP.get(m.strip().lower(), 1)
|
|
def clamp_day(m, d, y):
|
|
if d < 1: return 1
|
|
md = DAYS[m]
|
|
if m == 2 and d == 29 and not (y%4==0 and (y%100!=0 or y%400==0)): md = 28
|
|
return min(d, md)
|
|
|
|
def parse_pub_date(elem):
|
|
pde = elem.find('.//PubDate')
|
|
if pde is None: return None
|
|
ye = pde.find('Year'); me = pde.find('Month'); de = pde.find('Day')
|
|
if ye is not None and ye.text:
|
|
y = int(ye.text.strip())
|
|
hm = me is not None and me.text; hd = de is not None and de.text
|
|
if not hm and not hd: return None
|
|
m = month_num(me.text.strip()) if hm else 1
|
|
d = int(de.text.strip()) if hd else 1
|
|
d = clamp_day(m, d, y)
|
|
return date(y, m, d)
|
|
return None
|
|
|
|
async def main():
|
|
engine = create_async_engine(DATABASE_URL)
|
|
async with engine.connect() as conn:
|
|
# Get 200 "middle" 2016 PMIDs (not newest, not oldest)
|
|
r = await conn.execute(text("""
|
|
SELECT id, pmid, pub_date, pub_year FROM global_literature
|
|
WHERE pub_date IS NOT NULL AND EXTRACT(MONTH FROM pub_date)=1 AND EXTRACT(DAY FROM pub_date)=1
|
|
AND pmid IS NOT NULL AND pub_year = 2016
|
|
ORDER BY pmid ASC
|
|
OFFSET 5000 LIMIT 200
|
|
"""))
|
|
rows = r.all()
|
|
await engine.dispose()
|
|
|
|
print(f'Testing {len(rows)} PMIDs (2016, offset=5000) via efetch...')
|
|
pmids = [str(row.pmid) for row in rows]
|
|
print(f'PMID range: {pmids[0]} ~ {pmids[-1]}')
|
|
|
|
async with httpx.AsyncClient(timeout=60) as c:
|
|
resp = await c.get(
|
|
'https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi',
|
|
params={'db': 'pubmed', 'id': ','.join(pmids), 'retmode': 'xml'}
|
|
)
|
|
print(f'HTTP status={resp.status_code} content_len={len(resp.content)}')
|
|
root = ET.fromstring(resp.content)
|
|
arts = root.findall('PubmedArticle')
|
|
print(f'Articles returned: {len(arts)}')
|
|
|
|
fixed = 0; year_only = 0; still_jan01 = 0; no_pmid = 0
|
|
efetch_map = {}
|
|
for art in arts:
|
|
pmid_e = art.find('MedlineCitation/PMID')
|
|
pmid = int(pmid_e.text) if pmid_e is not None else None
|
|
pd = parse_pub_date(art)
|
|
if pmid: efetch_map[pmid] = pd
|
|
else: no_pmid += 1
|
|
|
|
analysis = []
|
|
for row in rows:
|
|
pd = efetch_map.get(row.pmid)
|
|
if pd is None:
|
|
year_only += 1
|
|
analysis.append((row.pmid, 'year_only'))
|
|
elif pd.month == 1 and pd.day == 1:
|
|
still_jan01 += 1
|
|
analysis.append((row.pmid, 'still_jan01'))
|
|
else:
|
|
fixed += 1
|
|
analysis.append((row.pmid, f'FIXED->{pd}'))
|
|
|
|
print(f'\n=== Batch Analysis ===')
|
|
print(f' Can fix (efetch has better date): {fixed}')
|
|
print(f' Year-only in efetch: {year_only}')
|
|
print(f' Still Jan-01 in efetch: {still_jan01}')
|
|
print(f' Articles without PMID in response: {no_pmid}')
|
|
if fixed:
|
|
print(f'\nSample fixes:')
|
|
for pmid, result in analysis:
|
|
if result.startswith('FIXED'):
|
|
print(f' PMID={pmid} -> {result}')
|
|
if year_only:
|
|
print(f'\nSample year-only PMIDs: {[pmid for pmid, r in analysis[:5] if r == "year_only"]}')
|
|
|
|
asyncio.run(main())
|