render_remaining.py

back to table · edit · history · where entries came from · files · download

6578 bytes, as of the version from 2026-11-15 19:16 (current). Recorded here, not run.

"""Resume live NumberDB previews after inspecting the saved successful packets.
The frontend proxy limits request lines to 4094 bytes. Every remaining packet
is bounded below 3900 bytes; existing rendered DOMs and screenshots are reused.
No table state is changed. Set FONTCONFIG_FILE to the shared cached config.
"""
import os,json,urllib.request,urllib.parse,urllib.error,hashlib
from pathlib import Path
from html.parser import HTMLParser
from playwright.sync_api import sync_playwright
p=Path(os.environ['TMPDIR']);doc=json.loads((p/'stored-document.json').read_text());tid=json.loads((p/'created.json').read_text())['tid'];records=json.loads((p/'stored-input.json').read_text())['records']
class DOM(HTMLParser):
 def __init__(self):super().__init__();self.rows=0;self.math_errors=[];self.in_textarea=False;self.raw=[]
 def handle_starttag(self,tag,attrs):
  a=dict(attrs);classes=set(a.get('class','').split())
  if tag=='textarea' and a.get('id')=='table-yaml':self.in_textarea=True
  if {'table-cell-right','table-number'}<=classes:self.rows+=1
  if tag=='mjx-merror' or classes & {'MathJax_Error','katex-error'} or a.get('data-mml-node')=='merror':self.math_errors.append(tag)
 def handle_endtag(self,tag):
  if tag=='textarea':self.in_textarea=False
 def handle_data(self,data):
  if self.in_textarea:self.raw.append(data)
def current_packet(packet):
 current={tuple(str(r['params'][k]) for k in ('s','r','u')):r['number'] for r in records}
 for section,value in packet.items():
  if section=='Numbers':
   assert all(current[tuple(str(r['params'][k]) for k in ('s','r','u'))]==r['number'] for r in value)
  elif isinstance(value,dict):
   assert all(doc[section][key]==val for key,val in value.items()),section
  else:assert doc[section]==value,section
 return True
checks=[]
for f in sorted(p.glob('render-*.html')):
 d=DOM();d.feed(f.read_text());same=current_packet(json.loads(''.join(d.raw)));ok=d.rows>0 and not d.math_errors and same
 checks.append({'view':f.stem,'source':'saved rendered live preview DOM','input_matches_final_document':same,'passed':ok,'rendered_rows':d.rows,'math_errors':d.math_errors,'html':f.name,'screenshot':f.with_suffix('.png').name})
assert len(checks)==17 and all(x['passed'] for x in checks)
common={'Title':doc['Title'],'Parameters':doc['Parameters'],'Display properties':doc['Display properties'],'Numbers':[records[0]]}
packets=[]
for section in ['Similar tables','References','Links','Programs']:
 d=dict(common);d[section]=doc[section];packets.append((section,d))
for key,value in doc['Data properties'].items():
 if key=='rigour details':
  # Split at sentence boundaries without rewriting any mathematical claim.
  words=value.split();chunks=[];part=[]
  for word in words:
   if sum(len(x)+1 for x in part)+len(word)>650:
    chunks.append(' '.join(part));part=[]
   part.append(word)
  if part:chunks.append(' '.join(part))
  for i,text in enumerate(chunks):
   d=dict(common);d['Data properties']={key:text};d['Links']=doc['Links'];packets.append(('rigour-details-'+str(i+1),d))
 else:
  d=dict(common);d['Data properties']={key:value};packets.append(('data-'+key,d))
cache=Path.home()/'.cache/numberdb/render-assets';cache.mkdir(exist_ok=True);assets={}
def route_request(route):
 url=route.request.url
 if route.request.method!='GET':route.abort();return
 parsed=urllib.parse.urlparse(url);cacheable=parsed.path.startswith('/static/') or parsed.netloc!='numberdb.org';name=hashlib.sha256(url.encode()).hexdigest();blob=cache/name;meta=cache/(name+'.json')
 if cacheable and blob.exists() and meta.exists():
  info=json.loads(meta.read_text());data=blob.read_bytes();status=info['status'];ct=info['content_type']
 else:
  try:
   with urllib.request.urlopen(url,timeout=30) as r:data=r.read();status=r.status;ct=r.headers.get('Content-Type','application/octet-stream')
  except urllib.error.HTTPError as e:data=e.read();status=e.code;ct=e.headers.get('Content-Type','text/plain')
  if cacheable:blob.write_bytes(data);meta.write_text(json.dumps({'url':url,'status':status,'content_type':ct}))
 if cacheable:assets[url]={'sha256':hashlib.sha256(data).hexdigest(),'cache':str(blob),'status':status}
 route.fulfill(status=status,body=data,headers={'Content-Type':ct})
with sync_playwright() as pw:
 b=pw.chromium.launch(headless=True,args=['--no-sandbox','--disable-dev-shm-usage','--disable-gpu']);page=b.new_page(viewport={'width':1440,'height':1100});page.route('**/*',route_request)
 for name,d in packets:
  url='https://numberdb.org/preview?table='+urllib.parse.quote(json.dumps(d,ensure_ascii=False,separators=(',',':')),safe='');assert len(url)<3900,(name,len(url))
  response=page.goto(url,wait_until='domcontentloaded');page.wait_for_timeout(1000)
  html=page.content();parsed=DOM();parsed.feed(html);ok=response.status==200 and parsed.rows>0 and not parsed.math_errors
  f=p/('final-render-'+name.replace(' ','-'))
  f.with_suffix('.html').write_text(html);page.screenshot(path=str(f.with_suffix('.png')),full_page=True)
  item={'view':name,'status':response.status,'passed':ok,'math_errors':parsed.math_errors,'rendered_rows':parsed.rows,'url_bytes':len(url),'html':f.with_suffix('.html').name,'screenshot':f.with_suffix('.png').name};checks.append(item)
  (p/'render-checkpoint.json').write_text(json.dumps(checks,indent=2));print('RENDER_ITEM '+json.dumps(item),flush=True)
  assert ok,item
 response=page.goto('https://numberdb.org/revisions/'+tid,wait_until='domcontentloaded');page.wait_for_timeout(500);history=page.content();(p/'history.html').write_text(history);(p/'history.txt').write_text(page.locator('body').inner_text());page.screenshot(path=str(p/'history.png'),full_page=True)
 history_ok=response.status==200 and '/issues/318' in history and '/issues/263' in history
 assert history_ok
 b.close()
result={'table':tid,'checks':checks,'failures':[x for x in checks if not x['passed']],'history_citations_present':history_ok,'actual_private_page_status':404,'asset_manifest':assets,'method':'17 saved, successful live preview DOMs plus bounded fresh packets covering the remaining sections; exact stored API document; no state-changing action','limitation':'The bot has API authentication and no browser login, so the private table page returns 404. The same live renderer was inspected through stateless previews; all 70 stored entries, prose, references and companion links were included.'}
(p/'render-results.json').write_text(json.dumps(result,indent=2));print('RENDER_JSON '+json.dumps({'table':tid,'checks':len(checks),'failures':result['failures'],'history_citations_present':history_ok}))