{
    "api": "YAML JSON TOON Database",
    "version": "1.0.0",
    "format": "json",
    "dataset": {
        "id": 1011,
        "slug": "unicode-text-handling",
        "title": "Unicode Text Handling Reference",
        "description": "Text boundaries, normalization, case handling, and encoding decisions for multilingual applications.",
        "summary": "Text boundaries, normalization, case handling, and encoding decisions for multilingual applications. Each row includes a practical application and a failure to check before using it.",
        "use_when": [
            "Build search that handles composed and decomposed text.",
            "Choose the unit for text length or truncation."
        ],
        "caveats": [
            "Preserve original text when using a lossy search transform.",
            "Identifier security requires more than normalization."
        ],
        "related": [
            "date-time-interchange",
            "json-schema-keywords"
        ],
        "category": "Programming Languages",
        "category_slug": "programming-languages",
        "tags": "programming-languages,unicode text handling,reference,engineering",
        "view_count": 0,
        "created_at": 1791798640,
        "updated_at": 1791798640,
        "reviewed_at": "2026-10-12",
        "sources": [
            {
                "title": "Unicode normalization standard",
                "url": "https://www.unicode.org/reports/tr15/"
            },
            {
                "title": "Unicode text segmentation",
                "url": "https://www.unicode.org/reports/tr29/"
            },
            {
                "title": "Unicode security mechanisms",
                "url": "https://www.unicode.org/reports/tr39/"
            }
        ],
        "reading": [
            "unicode-normalization-for-search-and-identifiers"
        ],
        "provenance": "Original editorial reference notes by YJTOON, informed by the linked primary documentation. External specifications retain their own licenses.",
        "entry_count": 10,
        "fields": [
            {
                "section": "data",
                "field": "concept",
                "type": "string",
                "example": "UTF-8 bytes"
            },
            {
                "section": "data",
                "field": "example",
                "type": "string",
                "example": "é occupies two UTF-8 bytes"
            },
            {
                "section": "data",
                "field": "use",
                "type": "string",
                "example": "Measure network and file size."
            },
            {
                "section": "data",
                "field": "pitfall",
                "type": "string",
                "example": "A byte count is not a character count."
            }
        ]
    },
    "data": [
        {
            "concept": "UTF-8 bytes",
            "example": "é occupies two UTF-8 bytes",
            "use": "Measure network and file size.",
            "pitfall": "A byte count is not a character count."
        },
        {
            "concept": "Code point",
            "example": "U+00E9 is LATIN SMALL LETTER E WITH ACUTE",
            "use": "Identify a Unicode scalar value.",
            "pitfall": "A visible character can contain several code points."
        },
        {
            "concept": "Grapheme cluster",
            "example": "e plus combining acute can display as one character",
            "use": "Move a caret or truncate visible text.",
            "pitfall": "Code-point slicing can split a visible character."
        },
        {
            "concept": "NFC",
            "example": "Compose canonically equivalent sequences where possible",
            "use": "Normalize text used for ordinary comparison.",
            "pitfall": "It does not remove accents."
        },
        {
            "concept": "NFD",
            "example": "Decompose canonical equivalents",
            "use": "Inspect combining marks or build a deliberate search transform.",
            "pitfall": "Removing every mark changes meaning in many languages."
        },
        {
            "concept": "NFKC",
            "example": "Apply compatibility decomposition and composition",
            "use": "Normalize selected identifiers under an explicit policy.",
            "pitfall": "It can change distinctions the original text intended."
        },
        {
            "concept": "Case folding",
            "example": "Treat selected uppercase and lowercase forms as equivalent",
            "use": "Build Unicode-aware caseless matching.",
            "pitfall": "Locale and security requirements still matter."
        },
        {
            "concept": "Turkish dotted I",
            "example": "I, İ, ı, i require deliberate handling",
            "use": "Test Turkish search and casing.",
            "pitfall": "ASCII-only lowercasing is not a complete locale policy."
        },
        {
            "concept": "Bidirectional text",
            "example": "Arabic or Hebrew mixed with Latin identifiers",
            "use": "Render mixed-direction labels safely.",
            "pitfall": "Invisible controls can obscure visual ordering."
        },
        {
            "concept": "Confusable characters",
            "example": "Latin a and Cyrillic а can look alike",
            "use": "Review identifiers and suspicious domains.",
            "pitfall": "Visual similarity does not imply string equality."
        }
    ]
}