34047007@qq.com
a6cd99a4ca
OncoLit: a multi-tenant oncology literature search, feed, and collaboration platform. Built with FastAPI + Vue 3 + PostgreSQL. Includes PubMed pipeline, drug approvals, AI summaries, and systematic review tools.
102 lines
4.0 KiB
Python
102 lines
4.0 KiB
Python
"""
|
|
Test efetch date parsing for YYYY-01-01 records — CORRECTED parse
|
|
"""
|
|
import asyncio, httpx, lxml.etree as ET
|
|
from datetime import date
|
|
from sqlalchemy import text
|
|
from sqlalchemy.ext.asyncio import create_async_engine
|
|
|
|
DATABASE_URL = 'postgresql+asyncpg://scilit:scilit_prod_2026@postgres:5432/scilit'
|
|
|
|
MONTH_MAP = {
|
|
'jan':1,'january':1,'1':1,'feb':2,'february':2,'2':2,'mar':3,'march':3,'3':3,
|
|
'apr':4,'april':4,'4':4,'may':5,'5':5,'jun':6,'june':6,'6':6,
|
|
'jul':7,'july':7,'7':7,'aug':8,'august':8,'8':8,'sep':9,'september':9,'9':9,
|
|
'oct':10,'october':10,'10':10,'nov':11,'november':11,'11':11,'dec':12,'december':12,'12':12,
|
|
}
|
|
DAYS = [0,31,29,31,30,31,30,31,31,30,31,30,31]
|
|
|
|
def month_num(m): return MONTH_MAP.get(m.strip().lower(), 1)
|
|
def clamp_day(m, d, y):
|
|
if d < 1: return 1
|
|
md = DAYS[m]
|
|
if m == 2 and d == 29 and not (y%4==0 and (y%100!=0 or y%400==0)): md = 28
|
|
return min(d, md)
|
|
|
|
def parse_pub_date(elem):
|
|
"""Same logic as fix_dates_via_efetch.py"""
|
|
pde = elem.find('.//PubDate')
|
|
if pde is None: return None
|
|
ye = pde.find('Year'); me = pde.find('Month'); de = pde.find('Day')
|
|
if ye is not None and ye.text:
|
|
y = int(ye.text.strip())
|
|
hm = me is not None and me.text; hd = de is not None and de.text
|
|
if not hm and not hd: return None
|
|
m = month_num(me.text.strip()) if hm else 1
|
|
d = int(de.text.strip()) if hd else 1
|
|
d = clamp_day(m, d, y) # clamp, not swap
|
|
return date(y, m, d)
|
|
return None
|
|
|
|
async def main():
|
|
engine = create_async_engine(DATABASE_URL)
|
|
async with engine.connect() as conn:
|
|
# Sample across all years with YYYY-01-01
|
|
r = await conn.execute(text("""
|
|
SELECT pub_year, COUNT(*) as cnt FROM global_literature
|
|
WHERE pub_date IS NOT NULL AND EXTRACT(MONTH FROM pub_date)=1 AND EXTRACT(DAY FROM pub_date)=1
|
|
AND pmid IS NOT NULL
|
|
GROUP BY pub_year ORDER BY pub_year DESC
|
|
"""))
|
|
print('=== YYYY-01-01 records by year ===')
|
|
for row in r:
|
|
print(f' {row.pub_year}: {row.cnt}')
|
|
|
|
# Get 2 PMIDs from each year that has >0 records
|
|
r = await conn.execute(text("""
|
|
SELECT id, pmid, pub_date, pub_year FROM global_literature
|
|
WHERE pub_date IS NOT NULL AND EXTRACT(MONTH FROM pub_date)=1 AND EXTRACT(DAY FROM pub_date)=1
|
|
AND pmid IS NOT NULL
|
|
ORDER BY pub_year DESC, pmid DESC LIMIT 40
|
|
"""))
|
|
samples = r.all()
|
|
await engine.dispose()
|
|
|
|
# Efetch and report which PMIDs have better dates
|
|
test_pmids = [row.pmid for row in samples]
|
|
print(f'\n=== Testing {len(test_pmids)} PMIDs via efetch ===')
|
|
async with httpx.AsyncClient(timeout=30) as c:
|
|
resp = await c.get(
|
|
'https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi',
|
|
params={'db': 'pubmed', 'id': ','.join(str(p) for p in test_pmids), 'retmode': 'xml'}
|
|
)
|
|
print(f'HTTP status={resp.status_code} content_len={len(resp.content)}')
|
|
root = ET.fromstring(resp.content)
|
|
arts = root.findall('PubmedArticle')
|
|
print(f'Articles returned: {len(arts)}')
|
|
|
|
fixed = 0; year_only = 0; still_jan01 = 0
|
|
efetch_map = {}
|
|
for art in arts:
|
|
pmid_e = art.find('MedlineCitation/PMID')
|
|
pmid = int(pmid_e.text) if pmid_e is not None else None
|
|
pd = parse_pub_date(art)
|
|
if pmid: efetch_map[pmid] = pd
|
|
|
|
for row in samples:
|
|
pd = efetch_map.get(row.pmid)
|
|
if pd is None:
|
|
year_only += 1
|
|
elif pd.month == 1 and pd.day == 1:
|
|
still_jan01 += 1
|
|
else:
|
|
fixed += 1
|
|
print(f' CAN FIX: PMID={row.pmid} {row.pub_date} -> {pd}')
|
|
|
|
print(f'\n=== Summary ===')
|
|
print(f' Can fix (better date found): {fixed}')
|
|
print(f' Year-only (no month/day): {year_only}')
|
|
print(f' Still Jan-01: {still_jan01}')
|
|
|
|
asyncio.run(main())
|