{
    "api": "YAML JSON TOON Database",
    "version": "1.0.0",
    "format": "json",
    "dataset": {
        "id": 1020,
        "slug": "experimental-design-pitfalls",
        "title": "Experimental Design and Inference Pitfalls",
        "description": "Common threats to experiment validity, with a practical design adjustment and reporting consequence.",
        "summary": "Common threats to experiment validity, with a practical design adjustment and reporting consequence. Each row includes a practical application and a failure to check before using it.",
        "use_when": [
            "Review a product experiment before collecting data.",
            "Interpret a metric change without overstating causality."
        ],
        "caveats": [
            "Design guidance does not replace a task-specific statistical analysis.",
            "Randomization reduces some biases but does not fix poor measurement."
        ],
        "related": [
            "probability-distribution-reference",
            "statistics-basics"
        ],
        "category": "Mathematics",
        "category_slug": "mathematics",
        "tags": "mathematics,experimental design pitfalls,reference,engineering",
        "view_count": 0,
        "created_at": 1791798640,
        "updated_at": 1791798640,
        "reviewed_at": "2026-10-12",
        "sources": [
            {
                "title": "NIST design of experiments handbook",
                "url": "https://www.itl.nist.gov/div898/handbook/pri/section3/pri3.htm"
            },
            {
                "title": "NIST statistical assumptions",
                "url": "https://www.itl.nist.gov/div898/handbook/eda/section2/eda2.htm"
            }
        ],
        "reading": [
            "experiment-checklist-before-trusting-a-metric-change"
        ],
        "provenance": "Original editorial reference notes by YJTOON, informed by the linked primary documentation. External specifications retain their own licenses.",
        "entry_count": 10,
        "fields": [
            {
                "section": "data",
                "field": "pitfall",
                "type": "string",
                "example": "Selection bias"
            },
            {
                "section": "data",
                "field": "example",
                "type": "string",
                "example": "Only enthusiastic early users enter the experim…"
            },
            {
                "section": "data",
                "field": "adjustment",
                "type": "string",
                "example": "Define the target population and recruitment pr…"
            },
            {
                "section": "data",
                "field": "report",
                "type": "string",
                "example": "State who is outside the measured population."
            }
        ]
    },
    "data": [
        {
            "pitfall": "Selection bias",
            "example": "Only enthusiastic early users enter the experiment.",
            "adjustment": "Define the target population and recruitment process.",
            "report": "State who is outside the measured population."
        },
        {
            "pitfall": "Confounding",
            "example": "A new feature and a pricing change launch together.",
            "adjustment": "Randomize assignment or separate the interventions.",
            "report": "Distinguish association from an identified causal effect."
        },
        {
            "pitfall": "Peeking",
            "example": "Stop when a running p-value first falls below a threshold.",
            "adjustment": "Use a planned fixed horizon or valid sequential method.",
            "report": "Describe the stopping rule."
        },
        {
            "pitfall": "Multiple comparisons",
            "example": "Test many metrics and publish only the best.",
            "adjustment": "Predefine primary outcomes and account for multiplicity.",
            "report": "Show the family of analyses performed."
        },
        {
            "pitfall": "Low power",
            "example": "A small test misses a practically important effect.",
            "adjustment": "Plan sample size around a meaningful effect and variance.",
            "report": "Report uncertainty rather than declaring no effect."
        },
        {
            "pitfall": "Attrition",
            "example": "One group loses more participants before measurement.",
            "adjustment": "Measure and investigate missingness by assignment.",
            "report": "Report dropout and sensitivity analyses."
        },
        {
            "pitfall": "Unit mismatch",
            "example": "Randomize teams but analyze users as independent.",
            "adjustment": "Use an analysis that respects the assignment unit.",
            "report": "State the number of independent units."
        },
        {
            "pitfall": "Interference",
            "example": "Treated users change the experience of control users.",
            "adjustment": "Consider cluster assignment or an interference model.",
            "report": "Describe spillover limitations."
        },
        {
            "pitfall": "Metric gaming",
            "example": "A proxy improves while the real user outcome worsens.",
            "adjustment": "Use guardrail metrics and inspect concrete outcomes.",
            "report": "Explain what the primary metric fails to capture."
        },
        {
            "pitfall": "Post-treatment filtering",
            "example": "Remove participants based on behavior affected by treatment.",
            "adjustment": "Define inclusion before assignment or justify the estimand.",
            "report": "Disclose exclusions and their timing."
        }
    ]
}