#!/usr/bin/env python3
"""
Lab 05 — Deterministic eval scorers + a regression gate (pure Python, no deps).

The "programmatic > judge > human" principle (Phase 12, cheatsheet 16): prefer
deterministic scorers wherever the task is verifiable. This module implements
classification, extraction, and retrieval metrics, plus a weighted scorecard
and a CI-style regression gate.

Run:
    python3 scorers.py            # runs a self-test on a tiny golden set
"""
from __future__ import annotations
import json
import sys

# ---- programmatic scorers --------------------------------------------------

def exact_match(pred: str, gold: str) -> float:
    return 1.0 if pred.strip().lower() == gold.strip().lower() else 0.0


def json_schema_valid(pred: str, required_keys: list[str]) -> float:
    try:
        obj = json.loads(pred)
    except Exception:
        return 0.0
    return 1.0 if all(k in obj for k in required_keys) else 0.0


def classification_metrics(preds: list[str], golds: list[str], positive: str) -> dict:
    tp = sum(1 for p, g in zip(preds, golds) if p == positive and g == positive)
    fp = sum(1 for p, g in zip(preds, golds) if p == positive and g != positive)
    fn = sum(1 for p, g in zip(preds, golds) if p != positive and g == positive)
    acc = sum(1 for p, g in zip(preds, golds) if p == g) / max(len(golds), 1)
    prec = tp / (tp + fp) if (tp + fp) else 0.0
    rec = tp / (tp + fn) if (tp + fn) else 0.0
    f1 = 2 * prec * rec / (prec + rec) if (prec + rec) else 0.0
    return {"accuracy": acc, "precision": prec, "recall": rec, "f1": f1}


def recall_at_k(retrieved_ids: list[int], relevant_ids: set[int], k: int) -> float:
    if not relevant_ids:
        return 0.0
    hits = sum(1 for i in retrieved_ids[:k] if i in relevant_ids)
    return hits / len(relevant_ids)


def mrr(retrieved_ids: list[int], relevant_ids: set[int]) -> float:
    for rank, i in enumerate(retrieved_ids, start=1):
        if i in relevant_ids:
            return 1.0 / rank
    return 0.0


# ---- weighted scorecard + safety gate -------------------------------------

def weighted_score(metrics: dict[str, float], weights: dict[str, float]) -> float:
    total_w = sum(weights.values()) or 1.0
    return sum(metrics.get(k, 0.0) * w for k, w in weights.items()) / total_w


def safety_gate(violation_rate: float, threshold: float = 0.0) -> bool:
    """Safety is a GATE, not a weighted axis. Pass only if at/under threshold."""
    return violation_rate <= threshold


# ---- regression gate (CI) --------------------------------------------------

def regression_gate(new_score: float, baseline: float, tolerance: float = 0.0) -> bool:
    """Fail the build if quality dropped beyond tolerance."""
    return new_score >= baseline - tolerance


# ---- self-test -------------------------------------------------------------

def _selftest() -> int:
    print("== classification ==")
    preds = ["spam", "ham", "spam", "ham", "spam"]
    golds = ["spam", "ham", "ham", "ham", "spam"]
    cm = classification_metrics(preds, golds, positive="spam")
    print(f"  {cm}")
    assert abs(cm["accuracy"] - 0.8) < 1e-9

    print("== extraction (schema) ==")
    assert json_schema_valid('{"name":"A","price":1}', ["name", "price"]) == 1.0
    assert json_schema_valid('{"name":"A"}', ["name", "price"]) == 0.0
    assert json_schema_valid("not json", ["name"]) == 0.0
    print("  schema validation OK")

    print("== retrieval ==")
    retrieved = [3, 1, 7, 2, 9]
    relevant = {1, 2}
    r5 = recall_at_k(retrieved, relevant, 5)
    rr = mrr(retrieved, relevant)
    print(f"  recall@5={r5:.2f}  mrr={rr:.3f}")
    assert abs(r5 - 1.0) < 1e-9
    assert abs(rr - 0.5) < 1e-9   # first relevant (id 1) at rank 2

    print("== weighted scorecard + gates ==")
    metrics = {"accuracy": 0.8, "faithfulness": 0.9}
    score = weighted_score(metrics, {"accuracy": 0.5, "faithfulness": 0.5})
    print(f"  weighted score = {score:.3f}")
    assert abs(score - 0.85) < 1e-9
    assert safety_gate(0.0) is True
    assert safety_gate(0.02, threshold=0.0) is False
    assert regression_gate(0.84, baseline=0.85, tolerance=0.02) is True
    assert regression_gate(0.80, baseline=0.85, tolerance=0.02) is False
    print("  gates OK")

    print("\nALL SELF-TESTS PASSED ✅")
    print("Principle: programmatic scorers are deterministic; safety is a hard gate;")
    print("the regression gate blocks quality drops in CI.")
    return 0


if __name__ == "__main__":
    sys.exit(_selftest())