"""IsoFLOPs Ladder, Law Fitting & the Forecast — reference solution. A complete scaling-law workbench, pure stdlib and fully deterministic: * a synthetic ladder generator with SEEDED noise and an injectable LR-decay bias (so you can reproduce the Kaplan bug on demand) * the IsoFLOPs method: parabola fit -> vertex -> power laws -> a+b consistency check * the parametric surface fit L = E + A/N^a + B/D^b, with a choice of estimator * bootstrap confidence intervals * optimal experimental design for where to place the next ablation * the baseline-vs-candidate decision rule, with a crossover scan Run `python solution.py` for the full Kaplan -> Chinchilla story from synthetic data. """ import math import random # Chinchilla-style parameters, used as the "ground truth" the synthetic ladder samples. CHINCHILLA = (1.69, 406.4, 0.34, 410.7, 0.28) # (E, A, alpha, B, beta) # ====================================================================================== # 1. Numerical spine # ====================================================================================== def fit_line(xs, ys): """Ordinary least-squares fit of y = slope*x + intercept.""" n = len(xs) if n != len(ys): raise ValueError("xs and ys must be the same length") if n < 2: raise ValueError("need at least 2 points to fit a line") mx = sum(xs) / n my = sum(ys) / n sxx = sum((x - mx) ** 2 for x in xs) if sxx == 0: raise ValueError("all x values identical — slope is undefined") sxy = sum((x - mx) * (y - my) for x, y in zip(xs, ys)) slope = sxy / sxx return slope, my - slope * mx def fit_power_law(xs, ys): """Fit y = c * x^k by regressing log y on log x. Returns (c, k).""" if len(xs) != len(ys): raise ValueError("xs and ys must be the same length") if any(x <= 0 for x in xs) or any(y <= 0 for y in ys): raise ValueError("power-law fit requires strictly positive data") k, log_c = fit_line([math.log(x) for x in xs], [math.log(y) for y in ys]) return math.exp(log_c), k def fit_quadratic(xs, ys): """Least-squares fit of y = a*x^2 + b*x + c via Cramer's rule on the normal equations. No libraries, fully deterministic.""" if len(xs) != len(ys): raise ValueError("xs and ys must be the same length") if len(xs) < 3: raise ValueError("need at least 3 points to fit a parabola") s = [sum(x ** p for x in xs) for p in range(5)] t = [sum(y * x ** p for x, y in zip(xs, ys)) for p in range(3)] M = [[s[4], s[3], s[2]], [s[3], s[2], s[1]], [s[2], s[1], s[0]]] rhs = [t[2], t[1], t[0]] def det3(m): return (m[0][0] * (m[1][1] * m[2][2] - m[1][2] * m[2][1]) - m[0][1] * (m[1][0] * m[2][2] - m[1][2] * m[2][0]) + m[0][2] * (m[1][0] * m[2][1] - m[1][1] * m[2][0])) D0 = det3(M) # Scale-aware degeneracy check: collinear x values make the system singular. if abs(D0) <= 1e-12 * max(1.0, abs(s[4])): raise ValueError("degenerate design — x values are collinear or too few distinct") def repl(m, col, v): return [[v[r] if c == col else m[r][c] for c in range(3)] for r in range(3)] return (det3(repl(M, 0, rhs)) / D0, det3(repl(M, 1, rhs)) / D0, det3(repl(M, 2, rhs)) / D0) def parabola_vertex(a, b): """x at the minimum of a*x^2 + b*x + c. Requires a > 0.""" if a <= 0: raise ValueError("parabola does not open upward — no interior minimum") return -b / (2.0 * a) # ====================================================================================== # 2. The loss model and the synthetic ladder # ====================================================================================== def parametric_loss(n_params, n_tokens, E, A, alpha, B, beta): """L = E + A/N^alpha + B/D^beta. E irreducible loss (entropy of the data itself) A/N^a capacity term — shrinks with more parameters B/D^b data term — shrinks with more tokens """ if n_params <= 0 or n_tokens <= 0: raise ValueError("n_params and n_tokens must be positive") if alpha <= 0 or beta <= 0: raise ValueError("exponents must be positive") return E + A / (n_params ** alpha) + B / (n_tokens ** beta) def loss_decomposition(n_params, n_tokens, params=CHINCHILLA): """Split a loss into its three sources, with fractions. Reveals which lever to pull.""" E, A, alpha, B, beta = params cap = A / n_params ** alpha dat = B / n_tokens ** beta total = E + cap + dat return {"irreducible": E, "capacity": cap, "data": dat, "total": total, "irreducible_frac": E / total, "capacity_frac": cap / total, "data_frac": dat / total} def schedule_bias(n_tokens, schedule_tokens, magnitude): """The LR-decay measurement bias, as a function of how far into the schedule you are. A learning-rate schedule decays to ~0 at `schedule_tokens`. Much of the final loss improvement happens in that decay. If you READ the loss at `n_tokens` from a run whose schedule targets `schedule_tokens > n_tokens`, the LR is still high and the measured loss is too HIGH by roughly magnitude * (1 - n_tokens / schedule_tokens) The critical property is that this is NOT a constant: a constant would simply be absorbed into the fitted E and change nothing. It varies across the ladder, and a non-uniform bias TILTS the fitted exponents. That is the whole mechanism behind the Kaplan -> Chinchilla correction. """ if magnitude < 0: raise ValueError("magnitude cannot be negative") if schedule_tokens <= 0 or n_tokens <= 0: raise ValueError("token counts must be positive") frac_remaining = max(0.0, 1.0 - n_tokens / schedule_tokens) return magnitude * frac_remaining def synthetic_ladder(budgets, sizes_per_budget, params=CHINCHILLA, noise_std=0.0, lr_decay_bias=0.0, bias_mode="fixed_schedule", fixed_schedule_tokens=None, seed=0): """Generate a ladder of (C, N, D, L) measurements. `lr_decay_bias` > 0 injects the schedule-mismatch measurement bias described in `schedule_bias`. `bias_mode` selects WHICH points the bias hits hardest, because that is what determines how the fit is distorted: "fixed_schedule" every measurement is read from a run whose LR schedule targets `fixed_schedule_tokens` (default: the largest D in the ladder). This is closest to "one run per model size, read intermediate losses" — points with small D are penalized most. "per_budget" the schedule targets the largest D at that budget. The bias is then driven by position WITHIN each IsoFLOPs curve, so large-N (small-D) configs are penalized most. "none" no bias. Returns [(C, N, D, L), ...]. All randomness goes through a SEEDED generator, so the same seed produces byte-identical output. """ if noise_std < 0: raise ValueError("noise_std cannot be negative") if lr_decay_bias < 0: raise ValueError("lr_decay_bias cannot be negative") if bias_mode not in ("fixed_schedule", "per_budget", "none"): raise ValueError(f"unknown bias_mode {bias_mode!r}") if len(budgets) != len(sizes_per_budget): raise ValueError("need one size list per budget") rng = random.Random(seed) # Precompute the D grid so we can pick schedule horizons. grid = [] for C, sizes in zip(budgets, sizes_per_budget): grid.append([(N, C / (6.0 * N)) for N in sizes]) all_d = [D for row in grid for _, D in row] if not all_d: raise ValueError("empty ladder") global_max_d = max(all_d) out = [] for (C, row) in zip(budgets, grid): budget_max_d = max(D for _, D in row) for N, D in row: L = parametric_loss(N, D, *params) if lr_decay_bias > 0 and bias_mode != "none": horizon = (fixed_schedule_tokens or global_max_d if bias_mode == "fixed_schedule" else budget_max_d) L += schedule_bias(D, horizon, lr_decay_bias) if noise_std > 0: L += rng.gauss(0.0, noise_std) out.append((C, N, D, L)) return out # ====================================================================================== # 3. The IsoFLOPs method # ====================================================================================== def isoflop_optimum(points): """Given [(N, D, L), ...] at ONE fixed budget, fit a parabola in log10(N) and return the vertex. Returns (N_opt, quadratic_coeffs).""" if len(points) < 3: raise ValueError("need at least 3 sizes to locate an IsoFLOPs minimum") xs = [math.log10(N) for N, _, _ in points] ys = [L for _, _, L in points] a, b, c = fit_quadratic(xs, ys) return 10 ** parabola_vertex(a, b), (a, b, c) def isoflops_ladder(ladder): """Group a ladder by budget and locate each budget's optimum. Input: [(C, N, D, L), ...] Output: [(C, N_opt, D_opt), ...] sorted by C. """ by_budget = {} for C, N, D, L in ladder: by_budget.setdefault(C, []).append((N, D, L)) results = [] for C in sorted(by_budget): N_opt, _ = isoflop_optimum(by_budget[C]) results.append((C, N_opt, C / (6.0 * N_opt))) return results def fit_scaling_exponents(isoflops_results): """Steps 5-6: fit N_opt ∝ C^a and D_opt ∝ C^b, and check a + b ≈ 1. That check is free and catches FLOP-accounting bugs — it follows directly from C = 6ND, so any fit violating it is wrong somewhere. """ if len(isoflops_results) < 2: raise ValueError("need at least 2 budgets to fit an exponent") Cs = [C for C, _, _ in isoflops_results] Ns = [N for _, N, _ in isoflops_results] Ds = [D for _, _, D in isoflops_results] cn, a = fit_power_law(Cs, Ns) cd, b = fit_power_law(Cs, Ds) return {"n_coeff": cn, "n_exponent": a, "d_coeff": cd, "d_exponent": b, "exponent_sum": a + b, "consistent": abs(a + b - 1.0) < 0.02} # ====================================================================================== # 4. Fitting the parametric surface # ====================================================================================== def residuals(points, params, space="log"): """points: [(N, D, L_observed), ...]; params: (E, A, alpha, B, beta).""" if space not in ("log", "linear"): raise ValueError(f"unknown residual space {space!r}") E, A, alpha, B, beta = params out = [] for N, D, L_obs in points: L_pred = parametric_loss(N, D, E, A, alpha, B, beta) if space == "linear": out.append(L_pred - L_obs) else: if L_pred <= 0 or L_obs <= 0: raise ValueError("log-space residuals need positive losses") out.append(math.log(L_pred) - math.log(L_obs)) return out def squared_loss(rs): return sum(r * r for r in rs) def huber_loss(rs, delta=1e-3): """Quadratic near zero, LINEAR in the tails, so one diverged run cannot dominate the fit. This is what Chinchilla used, and it is not incidental.""" if delta <= 0: raise ValueError("delta must be positive") total = 0.0 for r in rs: total += (0.5 * r * r if abs(r) <= delta else delta * (abs(r) - 0.5 * delta)) return total def _smart_inits(points): """Data-driven starting points for the fit. Coordinate descent on a 5-parameter nonlinear surface has local minima; a bad start lands in the wrong basin and produces a confidently wrong extrapolation. The single most useful anchor is E: the irreducible loss must be BELOW the smallest observed loss, and typically not far below it. We seed a few candidates around that bound and take the best — deterministically, with no randomness. """ lo = min(L for _, _, L in points) return [ (0.90 * lo, 400.0, 0.35, 400.0, 0.30), (0.75 * lo, 400.0, 0.35, 400.0, 0.30), (0.50 * lo, 200.0, 0.30, 200.0, 0.25), (0.95 * lo, 800.0, 0.40, 800.0, 0.35), ] def _descend(points, init, objective, space, iters, seed_step, shrink): """One coordinate-descent run from a fixed start.""" params = list(init) step = [seed_step * abs(p) if p else seed_step for p in params] def score(p): try: rs = residuals(points, tuple(p), space) except (ValueError, OverflowError, ZeroDivisionError): return float("inf") return huber_loss(rs) if objective == "huber" else squared_loss(rs) best = score(params) for _ in range(iters): improved = False for i in range(5): for direction in (1, -1): trial = list(params) trial[i] = params[i] + direction * step[i] if trial[i] <= 0: # keep E, A, alpha, B, beta positive continue s = score(trial) if s < best: best, params, improved = s, trial, True break if not improved: step = [x * shrink for x in step] if max(step) < 1e-10: break return tuple(params), best def fit_parametric(points, init=None, objective="huber", space="log", iters=300, seed_step=0.5, shrink=0.9): """Fit L = E + A/N^alpha + B/D^beta by deterministic multi-start coordinate descent. No randomness and no library optimizer, so the same input always produces byte-identical output — a property the tests assert. (This is not fussiness: if a library upgrade can move your fitted exponents, it can move a nine-figure flagship recommendation, and your process is not reproducible.) `init=None` uses data-driven starting points; pass an explicit tuple to force one. Returns (params, final_objective_value). """ if objective not in ("huber", "squared"): raise ValueError(f"unknown objective {objective!r}") if space not in ("log", "linear"): raise ValueError(f"unknown residual space {space!r}") if len(points) < 3: raise ValueError("need at least 3 points to fit 5 parameters meaningfully") starts = [init] if init is not None else _smart_inits(points) best_params, best_score = None, float("inf") for start in starts: p, s = _descend(points, start, objective, space, iters, seed_step, shrink) if s < best_score: best_params, best_score = p, s return best_params, best_score def analytic_optimum(total_flops, E, A, alpha, B, beta): """Closed-form compute-optimal (N, D) under the parametric law. Minimizing L(N, C/6N) over N gives N_opt = [ (alpha*A)/(beta*B) * (C/6)^beta ] ^ (1/(alpha+beta)) which is a power law in C with exponent beta/(alpha+beta). """ if total_flops <= 0: raise ValueError("total_flops must be positive") if alpha <= 0 or beta <= 0: raise ValueError("exponents must be positive") N = ((alpha * A) / (beta * B) * (total_flops / 6.0) ** beta) ** (1.0 / (alpha + beta)) return N, total_flops / (6.0 * N) def optimum_exponent(alpha, beta): """The exponent in N_opt ∝ C^k. Chinchilla's ~0.5 corresponds to alpha ≈ beta.""" if alpha <= 0 or beta <= 0: raise ValueError("exponents must be positive") return beta / (alpha + beta) # ====================================================================================== # 5. Uncertainty # ====================================================================================== def bootstrap_forecast(points, target_flops, n_boot=120, seed=0, ci=0.95): """Percentile bootstrap CI for the loss forecast at `target_flops`. Resample the ladder with replacement, refit, predict, repeat. Analytic intervals are unavailable because the model is nonlinear and the noise model is unspecified — which is itself one of Feinberg's stated open problems. """ if not 0 < ci < 1: raise ValueError("ci must be in (0, 1)") if n_boot < 10: raise ValueError("n_boot too small to estimate a percentile interval") rng = random.Random(seed) # SEEDED forecasts = [] for _ in range(n_boot): sample = [points[rng.randrange(len(points))] for _ in range(len(points))] if len({(n, d) for n, d, _ in sample}) < 3: continue # degenerate resample try: params, _ = fit_parametric(sample, iters=200) N, D = analytic_optimum(target_flops, *params) forecasts.append(parametric_loss(N, D, *params)) except (ValueError, OverflowError, ZeroDivisionError): continue if len(forecasts) < 10: raise ValueError("too few successful bootstrap fits — ladder is too small " "or too degenerate") forecasts.sort() tail = (1.0 - ci) / 2.0 lo = forecasts[int(tail * len(forecasts))] hi = forecasts[min(len(forecasts) - 1, int((1.0 - tail) * len(forecasts)))] return {"median": forecasts[len(forecasts) // 2], "ci_low": lo, "ci_high": hi, "width": hi - lo, "n_successful": len(forecasts)} def compute_multiplier_for_loss_delta(delta_nats, loss=1.75, irreducible=1.69, alpha=0.5): """Roughly how much extra compute buys `delta_nats` of improvement. Near the operating point, L - E ~ C^-alpha, so a fractional compute increase of delta / (alpha * (L - E)) buys delta nats. The point of this function is to make 'is my confidence interval narrow enough?' answerable in dollars. """ if loss <= irreducible: raise ValueError("loss must exceed the irreducible floor") return delta_nats / (alpha * (loss - irreducible)) # ====================================================================================== # 6. Experimental design # ====================================================================================== def extrapolation_variance(log_c_points, target_log_c): """Variance of a linear extrapolation to target_log_c, up to a noise constant. OLS: Var(y_hat) ∝ 1/n + (x* - xbar)^2 / sum((x - xbar)^2) The second term is why SPREAD beats DENSITY: adding a point far from the mean grows the denominator quadratically. """ n = len(log_c_points) if n < 2: raise ValueError("need at least 2 design points") xbar = sum(log_c_points) / n sxx = sum((x - xbar) ** 2 for x in log_c_points) if sxx == 0: raise ValueError("all design points identical") return 1.0 / n + (target_log_c - xbar) ** 2 / sxx def design_ladder(flagship_flops, ladder_budget_fraction=0.03, n_budgets=5, min_flops=1e18): """Geometrically spaced ladder budgets consuming a fixed fraction of the flagship. Geometric spacing maximizes spread in log space for a given range, which is exactly what minimizes extrapolation variance. """ if not 0 < ladder_budget_fraction < 1: raise ValueError("ladder_budget_fraction must be in (0, 1)") if n_budgets < 2: raise ValueError("need at least 2 budgets") total = flagship_flops * ladder_budget_fraction top = total / 2.0 # largest run takes half the ladder budget if top <= min_flops: raise ValueError("ladder budget too small for the requested min_flops") ratio = (top / min_flops) ** (1.0 / (n_budgets - 1)) return [min_flops * ratio ** i for i in range(n_budgets)] def extrapolation_risk(ladder_max_flops, target_flops, ci_width): """An honest risk label for a forecast: decades of extrapolation x interval width.""" if ladder_max_flops <= 0 or target_flops <= 0: raise ValueError("FLOP counts must be positive") if ci_width < 0: raise ValueError("ci_width cannot be negative") decades = math.log10(target_flops / ladder_max_flops) if decades <= 0: return {"decades": decades, "score": 0.0, "band": "INTERPOLATION"} score = decades * (1 + 20 * ci_width) band = "LOW" if score < 2 else "MODERATE" if score < 4 else "HIGH" return {"decades": decades, "score": score, "band": band} # ====================================================================================== # 7. The decision # ====================================================================================== def loss_at_budget(params, total_flops): """Compute-optimal loss this recipe achieves at a given budget.""" N, D = analytic_optimum(total_flops, *params) return parametric_loss(N, D, *params) def compare_recipes(baseline_params, candidate_params, target_flops, scan_lo=16.0, scan_hi=28.0, scan_steps=120): """Compare two fitted laws AT THE TARGET, and locate any crossover. The crossover is the deliverable. 'Candidate wins above 1e23' is actionable; 'candidate is better' silently assumes a scale, and curves cross. """ if scan_hi <= scan_lo: raise ValueError("scan_hi must exceed scan_lo") base = loss_at_budget(baseline_params, target_flops) cand = loss_at_budget(candidate_params, target_flops) crossover = None prev = None step = (scan_hi - scan_lo) / scan_steps for i in range(scan_steps + 1): C = 10 ** (scan_lo + i * step) wins = loss_at_budget(candidate_params, C) < loss_at_budget(baseline_params, C) if prev is not None and wins != prev: crossover = C break prev = wins return {"baseline_loss": base, "candidate_loss": cand, "delta": cand - base, "candidate_wins": cand < base, "crossover_flops": crossover} def decision_is_supported(delta_nats, ci_width_nats): """A delta smaller than your uncertainty is not a result. Say so out loud.""" return abs(delta_nats) > ci_width_nats # ====================================================================================== # Worked example — the Kaplan -> Chinchilla story, from synthetic data # ====================================================================================== def main(): print("=" * 78) print("ISOFLOPS LADDER, LAW FITTING & THE FORECAST") print("=" * 78) print("\n[1] The loss decomposition at a realistic operating point (70B, 1.4T)") d = loss_decomposition(70e9, 1.4e12) print(f" irreducible E {d['irreducible']:.4f} ({d['irreducible_frac']:5.1%})") print(f" capacity A/N^a {d['capacity']:.4f} ({d['capacity_frac']:5.1%})") print(f" data B/D^b {d['data']:.4f} ({d['data_frac']:5.1%})") print(f" total {d['total']:.4f}") print(" -> the whole industry is fighting over ~11% of the number.") print("\n[2] How much compute is a loss delta worth?") for delta in (0.001, 0.01, 0.05): m = compute_multiplier_for_loss_delta(delta) print(f" {delta:5.3f} nats ~ {m:6.1%} more compute") print("\n[3] One IsoFLOPs curve at C = 1e21 (the U)") pts = [(N, 1e21 / (6 * N), parametric_loss(N, 1e21 / (6 * N), *CHINCHILLA)) for N in (1e8, 3e8, 1e9, 3e9, 1e10, 3e10)] for N, D, L in pts: print(f" N={N:8.1e} D={D:8.1e} D/N={D/N:9.1f} loss={L:.4f}") N_opt, (a, b, c) = isoflop_optimum(pts) print(f" parabola vertex -> N_opt = {N_opt:.3e} " f"(D = {1e21/(6*N_opt):.3e}, {1e21/(6*N_opt)/N_opt:.1f} tok/param)") print("\n[4] The full ladder -> power laws -> the a+b consistency check") budgets = [1e19, 1e20, 1e21, 1e22] sizes = [[N * m for N in (1e7, 3e7, 1e8, 3e8, 1e9)] for m in (1, 3, 10, 30)] ladder = synthetic_ladder(budgets, sizes) iso = isoflops_ladder(ladder) for C, N, D in iso: print(f" C={C:.0e} N_opt={N:9.3e} D_opt={D:9.3e}") exps = fit_scaling_exponents(iso) print(f" N_opt ∝ C^{exps['n_exponent']:.3f}") print(f" D_opt ∝ C^{exps['d_exponent']:.3f}") print(f" a + b = {exps['exponent_sum']:.4f} consistent={exps['consistent']}") print(" (a + b MUST be 1 — it follows from C = 6ND. Free bug detector.)") print("\n[5] THE MEASUREMENT BIAS, reproduced") print(" Same ground truth. The biased ladders are read from runs whose LR") print(" schedule does not match the token count being measured.") clean = synthetic_ladder(budgets, sizes, noise_std=0.0, bias_mode="none") variants = [("unbiased ", clean)] for mode in ("fixed_schedule", "per_budget"): variants.append((f"biased/{mode:<9s}", synthetic_ladder(budgets, sizes, noise_std=0.0, lr_decay_bias=0.15, bias_mode=mode))) for label, lad in variants: f, _ = fit_parametric([(N, D, L) for _, N, D, L in lad]) N, D = analytic_optimum(1e24, *f) iso = fit_scaling_exponents(isoflops_ladder(lad)) print(f" {label}: alpha={f[2]:.3f} beta={f[4]:.3f} " f"N_opt∝C^{iso['n_exponent']:.3f} " f"flagship N={N:.3e} D/N={D/N:7.1f}") print(" -> a NON-UNIFORM bias tilts the fitted exponents and moves the") print(" flagship recommendation. A uniform one would just be absorbed into E.") print("\n[6] The estimator changes the answer (Feinberg: 'Formalize.')") noisy = [(N, D, L) for _, N, D, L in synthetic_ladder(budgets, sizes, noise_std=0.02, seed=7)] for obj, space in (("squared", "linear"), ("squared", "log"), ("huber", "log")): p, _ = fit_parametric(noisy, objective=obj, space=space) N, D = analytic_optimum(1e24, *p) print(f" {obj:8s}/{space:6s} -> flagship N={N:9.3e} D={D:9.3e} " f"loss={parametric_loss(N, D, *p):.4f}") print(" Identical data. Different recommendations. This is an open problem.") print("\n[7] One diverged run: why Huber and not least squares") clean_pts = [(N, D, L) for _, N, D, L in clean] dirty = clean_pts[:-1] + [(clean_pts[-1][0], clean_pts[-1][1], clean_pts[-1][2] + 1.5)] rs_clean = residuals(clean_pts, CHINCHILLA, "log") rs_dirty = residuals(dirty, CHINCHILLA, "log") print(f" clean : sum-sq {squared_loss(rs_clean):.6f} " f"huber {huber_loss(rs_clean):.6f}") print(f" dirty : sum-sq {squared_loss(rs_dirty):.6f} " f"huber {huber_loss(rs_dirty):.6f}") print(f" one bad run costs least-squares {squared_loss(rs_dirty):.4f} " f"and Huber {huber_loss(rs_dirty):.4f}") print("\n[8] The forecast, WITH error bars") fc = bootstrap_forecast(noisy, 1e24, n_boot=120, seed=3) print(f" median {fc['median']:.4f} nats " f"95% CI [{fc['ci_low']:.4f}, {fc['ci_high']:.4f}] " f"width {fc['width']:.4f}") supported = decision_is_supported(0.01, fc["width"]) print(f" could this ladder decide a 0.01-nat recipe difference? {supported}") print("\n[9] Where to put the next run — spread beats density") target = 25.0 for name, design in ( ("clustered (4 runs ~1e19)", [19.0, 19.2, 19.4, 19.6]), ("spread (4 runs 1e18-1e21)", [18.0, 19.0, 20.0, 21.0]), ("skewed (3 small, 1 big)", [18.0, 18.5, 19.0, 21.5]), ("extremes (2 runs only)", [18.0, 21.0])): print(f" {name:32s} Var ∝ {extrapolation_variance(design, target):8.2f}") print("\n[10] Designing the ladder for a 1e25 flagship") flagship = 1e25 bs = design_ladder(flagship) print(f" ladder total {sum(bs):.3e} FLOPs " f"({sum(bs)/flagship:.2%} of the flagship)") for C in bs: print(f" C={C:.3e}") print(f" log-C spread: " f"{math.log10(bs[-1]) - math.log10(bs[0]):.1f} decades") risk = extrapolation_risk(bs[-1], flagship, fc["width"]) print(f" extrapolation: {risk['decades']:.1f} decades -> {risk['band']} risk") print("\n[11] The decision: baseline vs candidate, at the target") baseline = CHINCHILLA # A candidate that LEARNS faster (better exponents) but has a HIGHER floor. # It must therefore win at small C and lose at large C — so the curves cross, # and the crossover, not the winner, is the deliverable. candidate = (1.85, 300.0, 0.40, 300.0, 0.33) for C in (1e20, 1e22, 1e24, 1e26): r = compare_recipes(baseline, candidate, C) who = "CANDIDATE" if r["candidate_wins"] else "baseline " print(f" C={C:.0e}: base {r['baseline_loss']:.4f} " f"cand {r['candidate_loss']:.4f} delta {r['delta']:+.4f} -> {who}") r = compare_recipes(baseline, candidate, 1e24) if r["crossover_flops"]: print(f" crossover at C ≈ {r['crossover_flops']:.2e} FLOPs " f"<- THIS is the deliverable") else: print(" no crossover in the scanned range") print("=" * 78) if __name__ == "__main__": main()