#!/usr/bin/env python3 """ Lab 05 — Inference sizing calculator (pure Python, no dependencies). Turns a workload + SLO into the load-bearing numbers for a sizing memo: - KV cache per token and per sequence (Knowledge 01 §3) - weight memory after quantization (Knowledge 02) - per-GPU concurrency ceiling (memory-bound) (Knowledge 01 §11) - parallelism hint (fit on one chip?) (Knowledge 05 §9) - $/1M tokens and monthly cost (Knowledge 08 §3) This does NOT replace benchmarking (per-instance *goodput* must be measured — Lab 03). It gives you the memory/parallelism/cost skeleton and forces you to state assumptions, which is exactly the discipline of Knowledge 08 §2. Edit the SCENARIO block and run: python sizing_calculator.py """ BYTES = {"fp16": 2, "bf16": 2, "fp8": 1, "int8": 1, "int4": 0.5} # --------------------------------------------------------------------------- # SCENARIO — edit these to your customer's workload. # Defaults model Llama-3-70B (GQA) on H100-80GB, the Knowledge 01 §11 example. # --------------------------------------------------------------------------- SCENARIO = dict( model_name="Llama-3-70B", params_billions=70.0, n_layers=80, n_kv_heads=8, # GQA. MHA would be n_query_heads (e.g. 64) -> 8x more KV head_dim=128, weight_dtype="int8", # quantize BEFORE you shard (K05 §9) kv_dtype="fp16", # try "int8" to ~2x concurrency # workload prompt_tokens=1000, output_tokens=400, peak_requests_per_sec=200.0, # hardware gpu_name="H100-80GB", gpu_memory_gb=80.0, gpu_dollars_per_hour=3.50, # on-demand-ish; use your real number activation_overhead_gb=10.0, # rough per-GPU runtime/activation/fragmentation headroom # serving assumption you MUST measure in Lab 03 (do not trust this default): per_instance_goodput_rps=12.0, # SLO-meeting requests/sec per model instance headroom_factor=1.4, # peak-vs-average + safety (K08 §4) # an instance must hold weights + overhead + KV for at least this many # concurrent sequences, or it's not a usable instance (drives TP degree): min_concurrency_per_instance=32, ) def kv_bytes_per_token(s): # 2 (K and V) * layers * kv_heads * head_dim * bytes_per_elem (K01 §3) return 2 * s["n_layers"] * s["n_kv_heads"] * s["head_dim"] * BYTES[s["kv_dtype"]] def report(s): ctx = s["prompt_tokens"] + s["output_tokens"] weight_gb = s["params_billions"] * 1e9 * BYTES[s["weight_dtype"]] / 1e9 kv_tok = kv_bytes_per_token(s) kv_seq_gb = kv_tok * ctx / 1e9 print("=" * 68) print(f" SIZING — {s['model_name']} on {s['gpu_name']}") print("=" * 68) print("\n[1] MEMORY (Knowledge 01 §3, 02)") print(f" weights ({s['weight_dtype']}) : {weight_gb:8.1f} GB") print(f" KV per token ({s['kv_dtype']}) : {kv_tok/1024:8.1f} KB") print(f" KV per sequence @ {ctx} ctx : {kv_seq_gb*1024:8.0f} MB ({kv_seq_gb:.2f} GB)") # Pick the smallest TP degree that holds weights + per-GPU overhead + KV for # the minimum usable concurrency. An instance that fits weights but has no KV # room is not a usable instance (the INT8-70B trap). K05 §9: shard only as # much as needed, after quantizing. need_kv_gb = s["min_concurrency_per_instance"] * kv_seq_gb tp = 1 while tp <= 64: inst_mem = tp * s["gpu_memory_gb"] if inst_mem - weight_gb - tp * s["activation_overhead_gb"] >= need_kv_gb: break tp += 1 print(f"\n[2] PARALLELISM (Knowledge 05 §9)") if tp == 1: print(f" weights + KV for {s['min_concurrency_per_instance']} seqs fit on ONE GPU") print(f" -> scale by DATA PARALLEL (replication). Simplest.") else: print(f" one GPU can't hold weights + KV for {s['min_concurrency_per_instance']} seqs") print(f" -> TENSOR PARALLEL across {tp} GPUs (over NVLink), then DP to scale.") print(f" (quantize weights/KV further to reduce TP degree; shard only as needed.)") # KV-bound concurrency on the instance inst_mem = tp * s["gpu_memory_gb"] kv_budget_gb = inst_mem - weight_gb - tp * s["activation_overhead_gb"] max_seqs = int(kv_budget_gb / kv_seq_gb) if kv_seq_gb > 0 else 0 print(f"\n[3] CONCURRENCY CEILING (memory-bound, Knowledge 01 §11)") print(f" instance memory ({tp} GPU) : {inst_mem:8.1f} GB") print(f" KV budget after weights : {kv_budget_gb:8.1f} GB") print(f" max concurrent sequences : {max_seqs:8d} (before PagedAttention recovery)") if max_seqs < 8: print(" ** WARNING: KV memory is the binding constraint. Quantize KV (int8),") print(" shorten context, or shard more. This is the real limit, not FLOPs. **") print(f"\n[4] FLEET SIZE (Knowledge 08 §2) [goodput is an ASSUMPTION — measure in Lab 03]") inst_needed = -(-s["peak_requests_per_sec"] // s["per_instance_goodput_rps"]) inst_needed = int(inst_needed) inst_with_headroom = int(-(-(inst_needed * s["headroom_factor"]) // 1)) total_gpus = inst_with_headroom * tp print(f" per-instance goodput (ASSUMED): {s['per_instance_goodput_rps']:8.1f} rps") print(f" instances for {s['peak_requests_per_sec']:.0f} rps peak : {inst_needed:8d}") print(f" + {s['headroom_factor']}x headroom : {inst_with_headroom:8d} instances") print(f" total accelerators : {total_gpus:8d} ({tp} GPU/instance)") print(f"\n[5] COST (Knowledge 08 §3)") fleet_dollars_hr = total_gpus * s["gpu_dollars_per_hour"] # tokens/hr at peak goodput: requests/s * output_tokens * 3600 served_rps = inst_with_headroom * s["per_instance_goodput_rps"] out_tok_per_hr = served_rps * s["output_tokens"] * 3600 dollars_per_1m = fleet_dollars_hr / out_tok_per_hr * 1e6 if out_tok_per_hr else float("nan") print(f" fleet cost : ${fleet_dollars_hr:8.2f} / hour") print(f" output tokens / hour : {out_tok_per_hr/1e6:8.1f} M") print(f" $ per 1M output tokens : ${dollars_per_1m:8.2f}") print(f" monthly (730 h) : ${fleet_dollars_hr*730:10.0f}") print(f"\n[6] ASSUMPTIONS TO STATE IN THE MEMO (K08 §2):") print(f" - I/O lengths {s['prompt_tokens']}/{s['output_tokens']}; peak {s['peak_requests_per_sec']} rps (peak, not avg)") print(f" - weights {s['weight_dtype']}, KV {s['kv_dtype']}; per-instance goodput MEASURED, not assumed") print(f" - accuracy floor validated on the customer's hardest task (K02 §11)") print(f" - peak/average ratio + autoscaling warm-buffer (K08 §4)") print("=" * 68) print("Now write the 3-option frontier (FP16 / INT8 / rightsized) and recommend one.") if __name__ == "__main__": report(SCENARIO)