osr-evaluation

{"text": "Hi there! How's it going?", "acceptable": ["llm-small"]}
{"text": "Prove sqrt(2) is irrational", "expected": "llm-frontier", "context": {"tier": "pro"}}
{"prompt": "Summarise this PDF", "scores": {"llm-small": 0.4, "llm-mid": 0.8}, "task_type": "summarize"}

{"task_id": "t1", "steps": ["look up the order", "reason through the refund conflict step by step"],
 "success": {"fast-agent": [0.97, 0.45], "strong-agent": [0.94, 0.90]},
 "latency_ms": {"fast-agent": [400, 400], "strong-agent": [2000, 2000]}, "domain": "customer_support"}

osr -t targets.yaml -r rules.yaml eval data.jsonl [--limit N] [--json]
    --frontier       2-D cost/quality frontier + area under it
    --frontier3      quality/cost/latency Pareto frontier + hypervolume
    --baselines      random / cheapest / best-prior / task-table / oracle (+ noise floor); --quality-floor 0.7
                     held out: the rows are split 50/50 by --seed (default 0), the task table is fitted on the
                     train half, every baseline AND the router are scored on the test half, and `held_out.comparison`
                     gives router-minus-baseline with a paired bootstrap 95 % interval and a verdict; every
                     accuracy carries a Wilson 95 % interval (`accuracy_ci`)
    --robustness     repeat flip-rate, paraphrase stability, profile-swap fairness (first 200 rows)
    --calibration    ECE, Brier, reliability diagram, conformal set coverage / size; plus held-out ECE after
                     fitting a temperature / isotonic calibrator on the even rows (>= 20 rows)
    --ablation       leave-one-strategy-out and leave-one-extractor-out deltas
    --multi-round    route-execute-judge-refine loop vs best single target (rows with scores); --threshold --max-rounds
    --effort         effort routing vs always-highest-effort (rows with scores + tokens; targets declare effort)
    --agentic        DATASET is agentic tasks; per-step routing vs best single agent on accuracy-latency; --retries N
    --min-accuracy X exit status 2 when accuracy < X (CI gate)
osr -t targets.yaml ope logs.jsonl [--max-weight 20]     # IPS / SNIPS / DR off-policy evaluation
osr -t targets.yaml safety [--learned-guard] [--json]    # red-team safety-routing suite
osr collect --cache-dir data --tier frontier=llm-frontier --tier mid=llm-mid --tier small=llm-small
    # fifteen public sources (routellm-battles, arena-55k/100k/140k, arena-expert-5k, search-arena-7k, repochat-arena,
    # llama3-70b-battles, ppe-human, webdev-arena, mt-bench-human, reward-bench, ultrafeedback, routellm-gpt4,
    # routellm-mmlu), resumable after a 429; --source NAME picks, --refresh refetches
osr -t targets.yaml slm train --out slm.json --cache-dir data --report   # every *-tiers cache; numpy when installed
    # judge --report against the constant baseline: "always mid" is acceptable ~64 % of the time on the arena corpora
OSR_SLM_ENCODER=attention osr ... slm train ... --report  # compare the transformer-block encoder on the holdout
osr slm eval slm.json data.jsonl --min-accuracy 0.8       # CI gate for the routing SLM itself

from opensmartroute.eval import evaluate, load_dataset, cost_quality_frontier, area_under_frontier, calibration_report
rows = load_dataset("data.jsonl")
r = evaluate(router, rows)             # EvalResult
print(r.summary())                     # n, accuracy [wilson 95 %], mean_cost_per_1k, mean_latency_ms, mean_confidence,
                                       # mean_quality, ece, brier, abstain_rate, set_coverage, mean_set_size
r.confusion                            # {expected: {chosen: count}}
r.errors                               # rows that missed, with the decision trace
r.hits                                 # per-row correctness in dataset order (for paired comparisons)

from opensmartroute.eval.baselines import baseline_suite, compare_policies, held_out_split
train, test = held_out_split(rows, seed=0)                    # seeded, reproducible 50/50 partition
suite = baseline_suite(test, router.registry.all(), train_rows=train)   # table fitted on train only
for c in compare_policies(evaluate(router, test).hits, suite):          # same rows, paired bootstrap
    print(c["baseline"], c["delta"], c["delta_ci"], c["verdict"])   # "router better" only when the CI excludes 0

- run: osr -t examples/targets.yaml -r examples/rules.yaml eval examples/eval_dataset.jsonl --min-accuracy 0.9 --json