Fix the Failure

FARunResult
├── summary          → FailureSummary (aggregate stats)
├── failure_tags[]   → FailureTag[] (per-item classifications)
│   ├── bucket_name        → "Hallucination"
│   ├── subcategory_name   → "Entity Fabrication"
│   ├── decision           → "fail"
│   ├── severity           → 4
│   ├── confidence         → 0.95
│   ├── judge_rationale    → "The output states Berlin..."
│   └── scoring_breakdown  → { ... }
├── eval_metrics     → Metric scores (if eval was run)
└── inputs           → Original data you submitted

summary = result.summary

print(f"Total items analyzed:  {summary.total_items}")
print(f"Items with failures:   {summary.items_with_failures}")
print(f"Items all passed:      {summary.items_all_passed}")
print(f"Total failures:        {summary.total_failures_detected}")
print(f"Total passes:          {summary.total_passes}")
print(f"Overall severity:      {summary.overall_severity}/5")
print(f"Overall confidence:    {summary.overall_confidence}")
print(f"Primary failure:       {summary.primary_failure_name}")
print(f"Buckets affected:      {summary.buckets_affected}")

for tag in result.failure_tags:
    print(f"Bucket:       {tag.bucket_name}")
    print(f"Subcategory:  {tag.subcategory_name}")
    print(f"Decision:     {tag.decision}")      # "pass", "fail", or "unsure"
    print(f"Severity:     {tag.severity}/5")     # 0 = no issue, 5 = critical
    print(f"Confidence:   {tag.confidence}")     # 0.0 to 1.0
    print(f"Detector:     {tag.detector_type_used}")
    print(f"Item index:   {tag.item_index}")     # Which dataset item
    print()
    if tag.judge_rationale:
        print(f"Rationale:    {tag.judge_rationale}")
    print(f"Breakdown:    {tag.scoring_breakdown}")
    print(f"Evidence:     {tag.evidence_items}")

Rationale: The output states Berlin is the capital of France,
which directly contradicts the provided context that identifies
Paris as the capital. This is a factual fabrication where the
model generated an entity (Berlin) that is not supported by
any of the provided context passages.

breakdown = tag.scoring_breakdown
# Typically includes:
# - Which context passages were relevant
# - What the correct answer should be
# - Why the output diverges from the context

dataset_failing = [
    {
        "input": "What is the capital of France?",
        "output": "The capital of France is Berlin.",  # ← Wrong
        "context": ["The capital of France is Paris."],
    }
]

dataset_fixed = [
    {
        "input": "What is the capital of France?",
        "output": "The capital of France is Paris.",  # ← Correct
        "context": ["The capital of France is Paris."],
    }
]

result = client.failure_analysis.run(dataset=dataset_fixed)

print(f"Failures: {result.summary.total_failures_detected}")
print(f"Passes:   {result.summary.total_passes}")
# Expected:
#   Failures: 0
#   Passes: 1

# Get only failures (skip passes and unsure)
failures = [t for t in result.failure_tags if t.decision == "fail"]

# Sort by severity (worst first)
failures.sort(key=lambda t: t.severity, reverse=True)

# Group by bucket
from collections import defaultdict
by_bucket = defaultdict(list)
for tag in failures:
    by_bucket[tag.bucket_name].append(tag)

for bucket, tags in by_bucket.items():
    print(f"\n{bucket} ({len(tags)} failures):")
    for tag in tags:
        print(f"  - {tag.subcategory_name} (severity: {tag.severity})")

result = client.failure_analysis.run(dataset=test_cases)

if result.summary.should_gate_ci:
    print("❌ Critical failures detected — blocking deployment")
    exit(1)
else:
    print("✅ All checks passed")