#!/usr/bin/env python3 """Tests for the hands-on pages. Asserts the invariants that keep a generated page trustworthy: * every script exits 0 and runs in reasonable time * every script supplies --verify, declares claims, and every claim passes * every declared block actually reports completion * every block has an annotation in notes/.md * every block carries a runnable inline example that executes cleanly * every script exposes parts(), so its mechanisms are importable * no <<>>/<<>> placeholder survives into a page * every generated page is current with respect to its script + notes + deep * scripts are deterministic: two runs produce identical output * --block N and --quiet work Run with pytest, or standalone: python3 -m pytest handson/test_handson.py -q python3 handson/test_handson.py """ from __future__ import annotations import os import re import subprocess import sys HERE = os.path.dirname(os.path.abspath(__file__)) BOOK = os.path.dirname(HERE) PY = sys.executable sys.path.insert(0, HERE) import build_pages as bp # noqa: E402 def _run(script, *args, timeout=180): return subprocess.run([PY, os.path.join(HERE, script), *args], capture_output=True, text=True, cwd=HERE, timeout=timeout) def _keys(): return list(bp.PAGES.keys()) # ---------------------------------------------------------------- scripts --- def test_registry_is_not_empty(): assert _keys(), "build_pages.PAGES is empty" def test_every_script_exists_and_exits_zero(): for key in _keys(): assert os.path.exists(os.path.join(HERE, key + ".py")), f"missing {key}.py" r = _run(key + ".py") assert r.returncode == 0, f"{key}.py exited {r.returncode}\n{r.stderr[-2000:]}" def test_every_declared_block_completes(): """A block that raises would not print its '[block N ok]' line.""" for key in _keys(): src = open(os.path.join(HERE, key + ".py")).read() declared = {int(n) for n in re.findall(r'^@block\((\d+),', src, re.M)} assert declared, f"{key}.py declares no blocks" out = _run(key + ".py").stdout completed = {int(n) for n in re.findall(r'^\[block (\d+) ok,', out, re.M)} assert declared == completed, ( f"{key}: declared {sorted(declared)} but completed {sorted(completed)}") def test_every_script_has_an_assembly(): for key in _keys(): src = open(os.path.join(HERE, key + ".py")).read() assert "\ndef assembly(" in src, f"{key}.py has no assembly()" assert "ASSEMBLY" in _run(key + ".py").stdout, f"{key}: assembly did not run" def test_scripts_are_deterministic(): """Fixed seeds: two runs must be byte-identical, or the pages are not stable.""" for key in _keys(): a, b = _run(key + ".py").stdout, _run(key + ".py").stdout # The harness prints per-block wall-clock, which legitimately varies. strip = lambda t: re.sub(r"\[block \d+ ok, \d+ ms\]", "", t) assert strip(a) == strip(b), f"{key}.py is not deterministic between runs" def test_block_flag_selects_a_prefix(): for key in _keys(): src = open(os.path.join(HERE, key + ".py")).read() ns = sorted(int(n) for n in re.findall(r'^@block\((\d+),', src, re.M)) if len(ns) < 2: continue out = _run(key + ".py", "--block", str(ns[0])).stdout assert f"BLOCK {ns[0]} " in out assert f"BLOCK {ns[-1]} " not in out, f"{key}: --block did not stop early" def test_quiet_flag_prints_only_the_assembly(): for key in _keys(): out = _run(key + ".py", "--quiet").stdout assert "ASSEMBLY" in out assert "BLOCK 1 " not in out, f"{key}: --quiet still printed blocks" def test_verify_mode_exists_and_passes(): """--verify re-derives every claim; a non-zero exit means page and code disagree.""" for key in _keys(): r = _run(key + ".py", "--verify") assert r.returncode == 0, ( f"{key}.py --verify FAILED\n{r.stdout[-3000:]}\n{r.stderr[-1500:]}") assert "claims verified" in r.stdout, f"{key}: --verify printed no tally" def test_every_page_declares_several_claims(): for key in _keys(): out = _run(key + ".py", "--verify").stdout n = len(re.findall(r"^\s*\[(?:PASS|FAIL)\]", out, re.M)) assert n >= 5, f"{key}: only {n} verifiable claims -- too few to be meaningful" def test_verify_is_deterministic(): for key in _keys(): strip = lambda t: re.sub(r"\(\d+ ms\)", "", t) a = strip(_run(key + ".py", "--verify").stdout) b = strip(_run(key + ".py", "--verify").stdout) assert a == b, f"{key}.py --verify is not deterministic" def test_verify_fails_loudly_when_a_claim_breaks(): """A guard on the guard: check() must actually be able to fail.""" import contextlib import io as _io sys.path.insert(0, HERE) import _harness _harness._CHECKS.clear() _harness.check("deliberately false", False, "") with contextlib.redirect_stdout(_io.StringIO()): # the FAIL line is expected failures = _harness._report() _harness._CHECKS.clear() assert failures == 1, "check() does not report failures" # ------------------------------------------------------------ annotations --- def test_every_block_has_an_annotation(): for key in _keys(): slug = bp.PAGES[key]["slug"] src = open(os.path.join(HERE, key + ".py")).read() declared = {int(n) for n in re.findall(r'^@block\((\d+),', src, re.M)} notes = bp.load_notes(slug) missing = declared - set(notes) assert not missing, f"notes/{slug}.md is missing blocks {sorted(missing)}" def test_annotations_have_no_orphan_sections(): for key in _keys(): slug = bp.PAGES[key]["slug"] src = open(os.path.join(HERE, key + ".py")).read() declared = {int(n) for n in re.findall(r'^@block\((\d+),', src, re.M)} extra = set(bp.load_notes(slug)) - declared assert not extra, f"notes/{slug}.md annotates non-existent blocks {sorted(extra)}" def test_every_page_has_a_deep_dive(): for key in _keys(): slug = bp.PAGES[key]["slug"] p = os.path.join(HERE, "deep", slug + ".md") assert os.path.exists(p), f"deep/{slug}.md is missing" assert re.search(r"^## ", open(p).read(), re.M), \ f"deep/{slug}.md has no '## ' headings, so it contributes no contents entries" def test_every_script_exposes_parts(): """parts() is what makes the inline examples on the page possible.""" for key in _keys(): mod = __import__(key) assert hasattr(mod, "parts"), f"{key}.py has no parts()" got = mod.parts() assert isinstance(got, dict) and got, f"{key}.parts() returned {got!r}" def test_every_block_has_a_runnable_example(): for key in _keys(): slug = bp.PAGES[key]["slug"] notes = bp.load_notes(slug) for n, body in sorted(notes.items()): assert "```python-run" in body, ( f"notes/{slug}.md block {n} has no ```python-run example -- " f"the page shows the mechanism but not how to trigger it") def test_inline_examples_execute(): """A snippet that raises fails the build; assert it here too, with the error.""" for key in _keys(): slug = bp.PAGES[key]["slug"] sources = list(bp.load_notes(slug).values()) deep = os.path.join(HERE, "deep", slug + ".md") if os.path.exists(deep): sources.append(open(deep).read()) for text in sources: for m in bp.RUN_FENCE.finditer(text): code = m.group(1) r = subprocess.run([PY, "-c", code], capture_output=True, text=True, cwd=HERE, timeout=120) assert r.returncode == 0, ( f"{slug}: inline example failed\n{code}\n{r.stderr[-1500:]}") assert r.stdout.strip(), f"{slug}: inline example printed nothing" def test_inline_examples_are_deterministic(): for key in _keys(): slug = bp.PAGES[key]["slug"] for text in bp.load_notes(slug).values(): for m in bp.RUN_FENCE.finditer(text): code = m.group(1) a = subprocess.run([PY, "-c", code], capture_output=True, text=True, cwd=HERE).stdout b = subprocess.run([PY, "-c", code], capture_output=True, text=True, cwd=HERE).stdout assert a == b, f"{slug}: an inline example is not deterministic" # ------------------------------------------------------------------ pages --- def test_pages_show_how_to_run_and_what_to_expect(): for key in _keys(): slug = bp.PAGES[key]["slug"] txt = open(os.path.join(BOOK, "handson", slug + ".md")).read() for section in ("## Run it", "**What to expect.**", "## Predict before you read", "## Verify the claims", "**Try it yourself**"): assert section in txt, f"{slug}.md is missing {section!r}" assert f"python3 {key}.py --verify" in txt, \ f"{slug}.md does not show the --verify command" assert "claims verified" in txt, f"{slug}.md does not show the claims table" def test_pages_exist_and_have_no_placeholders(): for key in _keys(): slug = bp.PAGES[key]["slug"] p = os.path.join(BOOK, "handson", slug + ".md") assert os.path.exists(p), f"handson/{slug}.md not generated" txt = open(p).read() for ph in ("<<>>", "<<>>"): assert ph not in txt, f"{slug}.md still contains {ph}" def test_pages_are_current(): """Regenerating must not change the file: the page matches script+notes+deep.""" for key in _keys(): slug = bp.PAGES[key]["slug"] p = os.path.join(BOOK, "handson", slug + ".md") before = open(p).read() bp.build(key) after = open(p).read() if before != after: open(p, "w").write(before) # leave the tree as we found it raise AssertionError( f"handson/{slug}.md is stale -- run `python3 build_pages.py {slug}`") def test_every_page_has_its_blocks_and_assembly_rendered(): for key in _keys(): slug = bp.PAGES[key]["slug"] src = open(os.path.join(HERE, key + ".py")).read() declared = sorted(int(n) for n in re.findall(r'^@block\((\d+),', src, re.M)) txt = open(os.path.join(BOOK, "handson", slug + ".md")).read() for n in declared: assert re.search(rf"^## Block {n} — ", txt, re.M), \ f"{slug}.md is missing a heading for block {n}" assert "## The assembly" in txt def test_project_links_resolve(): for key in _keys(): proj = bp.PAGES[key]["project"] assert os.path.exists(os.path.join(BOOK, proj)), \ f"{key}: project page {proj} does not exist" def test_registered_in_summary(): summary = open(os.path.join(BOOK, "SUMMARY.md")).read() for key in _keys(): slug = bp.PAGES[key]["slug"] assert f"handson/{slug}.md" in summary, \ f"handson/{slug}.md is not registered in SUMMARY.md" if __name__ == "__main__": fns = [(n, f) for n, f in sorted(globals().items()) if n.startswith("test_") and callable(f)] failed = 0 for name, fn in fns: try: fn() print(f" PASS {name}") except Exception as e: # noqa: BLE001 failed += 1 print(f" FAIL {name}\n {e}") print(f"\n{len(fns)-failed}/{len(fns)} passed") sys.exit(1 if failed else 0)