{
    "api": "YAML JSON TOON Database",
    "version": "1.0.0",
    "format": "json",
    "dataset": {
        "id": 1607,
        "slug": "llm-evaluation-failure-modes",
        "title": "LLM Evaluation Failure Modes",
        "description": "Evaluation mistakes that inflate confidence in language-model and agent changes, with corrective checks.",
        "summary": "Evaluation mistakes that inflate confidence in language-model and agent changes, with corrective checks. Each row includes a practical application and a failure to check before using it.",
        "use_when": [
            "Review a model or prompt change before shipping.",
            "Build a held-out test for answers grounded in reference data."
        ],
        "caveats": [
            "This is an evaluation checklist, not a benchmark leaderboard.",
            "The correct test depends on the actual user task and consequences of failure."
        ],
        "related": [
            "rag-retrieval-metrics",
            "ai-ethics-interpretability"
        ],
        "reviewed_at": "2026-10-12",
        "sources": [
            {
                "title": "HELM evaluation framework",
                "url": "https:\/\/crfm.stanford.edu\/helm\/"
            },
            {
                "title": "NIST AI Risk Management Framework",
                "url": "https:\/\/www.nist.gov\/itl\/ai-risk-management-framework"
            }
        ],
        "reading": [
            "llm-evaluation-checklist-for-reference-grounded-answers"
        ],
        "provenance": "Original editorial reference notes by YJTOON, informed by the linked primary documentation. External specifications retain their own licenses.",
        "entry_count": 10,
        "fields": [
            {
                "section": "data",
                "field": "failure",
                "type": "string",
                "example": "Test contamination"
            },
            {
                "section": "data",
                "field": "symptom",
                "type": "string",
                "example": "Strong scores on examples used during developme\u2026"
            },
            {
                "section": "data",
                "field": "better_check",
                "type": "string",
                "example": "Hold out cases and document dataset provenance."
            },
            {
                "section": "data",
                "field": "remaining_limit",
                "type": "string",
                "example": "Unknown pretraining overlap can remain."
            }
        ],
        "category": "AI & ML Terms",
        "category_slug": "ai-ml-terms",
        "tags": "ai-ml-terms,llm evaluation failure modes,reference,engineering",
        "view_count": 13,
        "created_at": 1791798640,
        "updated_at": 1791798640
    },
    "data": [
        {
            "failure": "Test contamination",
            "symptom": "Strong scores on examples used during development.",
            "better_check": "Hold out cases and document dataset provenance.",
            "remaining_limit": "Unknown pretraining overlap can remain."
        },
        {
            "failure": "Prompt overfitting",
            "symptom": "One prompt works on a small familiar set.",
            "better_check": "Use varied held-out cases and a baseline.",
            "remaining_limit": "A larger sample can still miss rare failures."
        },
        {
            "failure": "Judge position bias",
            "symptom": "The judge favors the first or second candidate.",
            "better_check": "Randomize candidate order and audit disagreements.",
            "remaining_limit": "Other judge biases can remain."
        },
        {
            "failure": "Single aggregate score",
            "symptom": "A gain hides a serious subgroup regression.",
            "better_check": "Report task slices and important failure categories.",
            "remaining_limit": "Tiny slices have wide uncertainty."
        },
        {
            "failure": "Uncontrolled sampling",
            "symptom": "Results move after an unrelated change.",
            "better_check": "Fix settings and compare repeated paired runs.",
            "remaining_limit": "Provider-side nondeterminism may remain."
        },
        {
            "failure": "Missing abstention test",
            "symptom": "The system answers even when evidence is absent.",
            "better_check": "Include unanswerable and adversarial cases.",
            "remaining_limit": "A good abstention rate does not prove answer quality."
        },
        {
            "failure": "Tool success proxy",
            "symptom": "A valid tool call is counted as a solved task.",
            "better_check": "Verify the intended state or external artifact.",
            "remaining_limit": "Some external outcomes are hard to observe."
        },
        {
            "failure": "Latency blind spot",
            "symptom": "A quality gain makes the workflow unusably slow.",
            "better_check": "Measure elapsed time and tail behavior under load.",
            "remaining_limit": "Lab traffic may differ from production."
        },
        {
            "failure": "Invalid confidence interval",
            "symptom": "Correlated turns are treated as independent samples.",
            "better_check": "Resample or analyze at the appropriate independent unit.",
            "remaining_limit": "The unit must reflect the data collection process."
        },
        {
            "failure": "No failure review",
            "symptom": "The dashboard improves while users report regressions.",
            "better_check": "Inspect concrete failures with a reproducible rubric.",
            "remaining_limit": "Human review also needs consistency checks."
        }
    ]
}