56 lines
2.7 KiB
Python
56 lines
2.7 KiB
Python
|
"""Test a batch of 2017 YYYY-01-01 records"""
|
||
|
|
import asyncio, httpx, lxml.etree as ET
|
||
|
|
from datetime import date
|
||
|
|
from sqlalchemy import text
|
||
|
|
from sqlalchemy.ext.asyncio import create_async_engine
|
||
|
|
|
||
|
|
DATABASE_URL = 'postgresql+asyncpg://scilit:scilit_prod_2026@postgres:5432/scilit'
|
||
|
|
MONTH_MAP = {'jan':1,'january':1,'1':1,'feb':2,'february':2,'2':2,'mar':3,'march':3,'3':3,'apr':4,'april':4,'4':4,'may':5,'5':5,'jun':6,'june':6,'6':6,'jul':7,'july':7,'7':7,'aug':8,'august':8,'8':8,'sep':9,'september':9,'9':9,'oct':10,'october':10,'10':10,'nov':11,'november':11,'11':11,'dec':12,'december':12,'12':12,}
|
||
|
|
DAYS = [0,31,29,31,30,31,30,31,31,30,31,30,31]
|
||
|
|
def mn(m): return MONTH_MAP.get(m.strip().lower(), 1)
|
||
|
|
def cd(m,d,y):
|
||
|
|
if d<1: return 1
|
||
|
|
md=DAYS[m]
|
||
|
|
if m==2 and d==29 and not (y%4==0 and (y%100!=0 or y%400==0)): md=28
|
||
|
|
return min(d,md)
|
||
|
|
def ppd(elem):
|
||
|
|
pde=elem.find('.//PubDate')
|
||
|
|
if pde is None: return None
|
||
|
|
ye=pde.find('Year'); me=pde.find('Month'); de=pde.find('Day')
|
||
|
|
if ye is not None and ye.text:
|
||
|
|
y=int(ye.text.strip()); hm=me is not None and me.text; hd=de is not None and de.text
|
||
|
|
if not hm and not hd: return None
|
||
|
|
m=mn(me.text.strip()) if hm else 1; d=int(de.text.strip()) if hd else 1; d=cd(m,d,y)
|
||
|
|
return date(y,m,d)
|
||
|
|
return None
|
||
|
|
|
||
|
|
async def main():
|
||
|
|
engine = create_async_engine(DATABASE_URL)
|
||
|
|
async with engine.connect() as conn:
|
||
|
|
r = await conn.execute(text("""SELECT id, pmid, pub_date, pub_year FROM global_literature
|
||
|
|
WHERE pub_date IS NOT NULL AND EXTRACT(MONTH FROM pub_date)=1 AND EXTRACT(DAY FROM pub_date)=1
|
||
|
|
AND pmid IS NOT NULL AND pub_year = 2017
|
||
|
|
ORDER BY pmid ASC LIMIT 200"""))
|
||
|
|
rows = r.all()
|
||
|
|
await engine.dispose()
|
||
|
|
pmids = [str(r.pmid) for r in rows]
|
||
|
|
print(f'Testing {len(rows)} 2017 PMIDs: {pmids[0]} ~ {pmids[-1]}')
|
||
|
|
async with httpx.AsyncClient(timeout=60) as c:
|
||
|
|
resp = await c.get('https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi',params={'db':'pubmed','id':','.join(pmids),'retmode':'xml'})
|
||
|
|
root = ET.fromstring(resp.content)
|
||
|
|
arts = root.findall('PubmedArticle')
|
||
|
|
fixed=0; year_only=0; still_jan01=0
|
||
|
|
efetch_map = {}
|
||
|
|
for art in arts:
|
||
|
|
pmid_e=art.find('MedlineCitation/PMID'); pmid=int(pmid_e.text) if pmid_e is not None else None
|
||
|
|
pd=ppd(art)
|
||
|
|
if pmid: efetch_map[pmid]=pd
|
||
|
|
for row in rows:
|
||
|
|
pd=efetch_map.get(row.pmid)
|
||
|
|
if pd is None: year_only+=1
|
||
|
|
elif pd.month==1 and pd.day==1: still_jan01+=1
|
||
|
|
else: fixed+=1
|
||
|
|
print(f'Can fix: {fixed}, Year-only: {year_only}, Still Jan-01: {still_jan01}')
|
||
|
|
|
||
|
|
asyncio.run(main())
|