#include "tokenizer/utf8.h"
#include <string.h>
#include <stdlib.h>
#include <stdbool.h>

Functions
size_t	ck_tokenizer_utf8_normalize_nfc (const char src, size_t src_len, char dst, size_t dst_size)

size_t	ck_utf8_byte_to_offset (const char *str, size_t len, size_t byte_offset)

int	ck_utf8_char_length (unsigned char c)

size_t	ck_utf8_count_chars (const char *str, size_t len)

int	ck_utf8_from_cp (uint32_t cp, char *out)

bool	ck_utf8_is_valid (const char *str, size_t len)

bool	ck_utf8_is_whitespace (uint32_t cp)

int32_t	ck_utf8_next_char (const char *str, int out_len)

size_t	ck_utf8_offset_to_byte (const char *str, size_t len, size_t n)

size_t	ck_utf8_validate (const char *str, size_t len)

Variables
struct {

uint32_t end

uint32_t start

}	CK_UTF8_WHITESPACE_RANGES []

Function Documentation

◆ ck_tokenizer_utf8_normalize_nfc()

size_t ck_tokenizer_utf8_normalize_nfc	(	const char *	src,
		size_t	src_len,
		char *	dst,
		size_t	dst_size
	)

Definition at line 240 of file utf8.c.

                                                                   {
    if (src_len == (size_t)-1) {
        src_len = strlen(src);
    }
 
    /* For now, just copy (full NFC is complex) */
    if (dst == NULL) {
        return src_len;
    }
 
    size_t to_copy = src_len < dst_size - 1 ? src_len : dst_size - 1;
    memcpy(dst, src, to_copy);
    dst[to_copy] = '\0';
 
    return to_copy;
}

◆ ck_utf8_byte_to_offset()

size_t ck_utf8_byte_to_offset	(	const char *	str,
		size_t	len,
		size_t	byte_offset
	)

Get the character index from byte offset.

Parameters

str	String
len	String length in bytes
byte_offset	Byte offset

Returns: Character index, or total chars if offset beyond end

Definition at line 187 of file utf8.c.

                                                                               {
    if (len == (size_t)-1) {
        len = strlen(str);
    }
 
    if (byte_offset >= len) {
        return ck_utf8_count_chars(str, len);
    }
 
    size_t char_idx = 0;
    size_t i = 0;
 
    while (i < byte_offset) {
        int char_len = ck_utf8_char_length((unsigned char)str[i]);
        if (char_len == 0) {
            i++;
        } else {
            i += char_len;
        }
        char_idx++;
    }
 
    return char_idx;
}

References ck_utf8_char_length(), and ck_utf8_count_chars().

◆ ck_utf8_char_length()

int ck_utf8_char_length ( unsigned char c )

Get the length of a UTF-8 character from its first byte.

Parameters

c	First byte of UTF-8 character

Returns: Number of bytes in character (1-4), or 0 for invalid

Definition at line 12 of file utf8.c.

                                         {
    if ((c & 0x80) == 0) {
        return 1;  /* ASCII */
    } else if ((c & 0xE0) == 0xC0) {
        return 2;  /* 2-byte sequence */
    } else if ((c & 0xF0) == 0xE0) {
        return 3;  /* 3-byte sequence */
    } else if ((c & 0xF8) == 0xF0) {
        return 4;  /* 4-byte sequence */
    }
    return 0;  /* Invalid */
}

Referenced by ck_utf8_byte_to_offset(), ck_utf8_count_chars(), ck_utf8_next_char(), ck_utf8_offset_to_byte(), and ck_utf8_validate().

◆ ck_utf8_count_chars()

size_t ck_utf8_count_chars	(	const char *	str,
		size_t	len
	)

Count UTF-8 characters in a string.

Parameters

str	String to count
len	Length in bytes, or -1 for null-terminated

Returns: Number of Unicode code points

Definition at line 82 of file utf8.c.

                                                        {
    if (len == (size_t)-1) {
        len = strlen(str);
    }
 
    size_t count = 0;
    size_t i = 0;
 
    while (i < len) {
        int char_len = ck_utf8_char_length((unsigned char)str[i]);
        if (char_len == 0) {
            /* Invalid UTF-8, count as 1 byte */
            i++;
        } else {
            i += char_len;
            count++;
        }
    }
 
    return count;
}

References ck_utf8_char_length().

Referenced by ck_utf8_byte_to_offset().

◆ ck_utf8_from_cp()

int ck_utf8_from_cp	(	uint32_t	cp,
		char *	out
	)

Write a Unicode code point as UTF-8.

Parameters

cp	Unicode code point
out	Output buffer (must have 4+ bytes)

Returns: Number of bytes written

Definition at line 144 of file utf8.c.

                                            {
    if (cp < 0x80) {
        out[0] = (char)cp;
        return 1;
    } else if (cp < 0x800) {
        out[0] = (char)(0xC0 | (cp >> 6));
        out[1] = (char)(0x80 | (cp & 0x3F));
        return 2;
    } else if (cp < 0x10000) {
        out[0] = (char)(0xE0 | (cp >> 12));
        out[1] = (char)(0x80 | ((cp >> 6) & 0x3F));
        out[2] = (char)(0x80 | (cp & 0x3F));
        return 3;
    } else {
        out[0] = (char)(0xF0 | (cp >> 18));
        out[1] = (char)(0x80 | ((cp >> 12) & 0x3F));
        out[2] = (char)(0x80 | ((cp >> 6) & 0x3F));
        out[3] = (char)(0x80 | (cp & 0x3F));
        return 4;
    }
}

◆ ck_utf8_is_valid()

bool ck_utf8_is_valid	(	const char *	str,
		size_t	len
	)

Check if a byte sequence is valid UTF-8.

Parameters

str	String to check
len	Length in bytes

Returns: true if all bytes form valid UTF-8

Definition at line 78 of file utf8.c.

                                                   {
    return ck_utf8_validate(str, len) == 0;
}

References ck_utf8_validate().

◆ ck_utf8_is_whitespace()

bool ck_utf8_is_whitespace ( uint32_t cp )

Check if character is whitespace (Unicode White_Space property).

Parameters

cp	Unicode code point

Returns: true if whitespace

Definition at line 229 of file utf8.c.

                                        {
    for (size_t i = 0; i < sizeof(CK_UTF8_WHITESPACE_RANGES) / sizeof(CK_UTF8_WHITESPACE_RANGES[0]); i++) {
        if (cp >= CK_UTF8_WHITESPACE_RANGES[i].start &&
            cp <= CK_UTF8_WHITESPACE_RANGES[i].end) {
            return true;
        }
    }
    return false;
}

References CK_UTF8_WHITESPACE_RANGES, end, and start.

◆ ck_utf8_next_char()

int32_t ck_utf8_next_char	(	const char **	str,
		int *	out_len
	)

Get next UTF-8 character, return its code point.

Parameters

str	String pointer (will be updated)
out_len	Output: length of character in bytes

Returns: Unicode code point, or -1 on error

Definition at line 104 of file utf8.c.

                                                          {
    if (!str || !*str || !out_len) {
        return -1;
    }
 
    unsigned char c = (unsigned char)(**str);
    int char_len = ck_utf8_char_length(c);
 
    if (char_len == 0) {
        *out_len = 1;
        (*str)++;
        return -1;
    }
 
    uint32_t cp;
 
    switch (char_len) {
        case 1:
            cp = c;
            break;
        case 2:
            cp = ck_utf8_decode_2(*str);
            break;
        case 3:
            cp = ck_utf8_decode_3(*str);
            break;
        case 4:
            cp = ck_utf8_decode_4(*str);
            break;
        default:
            cp = -1;
            break;
    }
 
    *out_len = char_len;
    *str += char_len;
 
    return (int32_t)cp;
}

References ck_utf8_char_length(), ck_utf8_decode_2(), ck_utf8_decode_3(), ck_utf8_decode_4(), and out_len.

◆ ck_utf8_offset_to_byte()

size_t ck_utf8_offset_to_byte	(	const char *	str,
		size_t	len,
		size_t	n
	)

Get the byte offset of the N-th character.

Parameters

str	String
len	String length in bytes
n	Character index (0-based)

Returns: Byte offset, or len if n >= char count

Definition at line 166 of file utf8.c.

                                                                     {
    if (len == (size_t)-1) {
        len = strlen(str);
    }
 
    size_t byte_offset = 0;
    size_t char_idx = 0;
 
    while (byte_offset < len && char_idx < n) {
        int char_len = ck_utf8_char_length((unsigned char)str[byte_offset]);
        if (char_len == 0) {
            byte_offset++;
        } else {
            byte_offset += char_len;
        }
        char_idx++;
    }
 
    return byte_offset;
}

References ck_utf8_char_length().

◆ ck_utf8_validate()

size_t ck_utf8_validate	(	const char *	str,
		size_t	len
	)

Validate a UTF-8 string.

Parameters

str	String to validate
len	Length in bytes, or -1 for null-terminated

Returns: 0 if valid, position of first invalid byte otherwise

Definition at line 25 of file utf8.c.

                                                     {
    if (len == (size_t)-1) {
        len = strlen(str);
    }
 
    size_t i = 0;
    while (i < len) {
        unsigned char c = (unsigned char)str[i];
        int char_len = ck_utf8_char_length(c);
 
        if (char_len == 0) {
            return i;  /* Invalid byte */
        }
 
        /* Check continuation bytes */
        for (int j = 1; j < char_len; j++) {
            if (i + j >= len) {
                return i + j;  /* Truncated sequence */
            }
            unsigned char cont = (unsigned char)str[i + j];
            if ((cont & 0xC0) != 0x80) {
                return i + j;  /* Invalid continuation */
            }
        }
 
        /* Validate code point for overlong encodings */
        if (char_len == 2 && c < 0xC2) {
            return i;  /* Overlong 2-byte */
        } else if (char_len == 3) {
            if (c == 0xE0 && (str[i + 1] & 0xE0) == 0x80) {
                return i;  /* Overlong 3-byte */
            }
            if (c == 0xED && (str[i + 1] & 0xE0) == 0xA0) {
                return i;  /* Surrogate */
            }
        } else if (char_len == 4) {
            if (c == 0xF0 && (str[i + 1] & 0xF0) == 0x80) {
                return i;  /* Overlong 4-byte */
            }
            if (c > 0xF4) {
                return i;  /* Invalid */
            }
            if (c == 0xF4 && (str[i + 1] & 0xF0) == 0x80) {
                return i;  /* Out of range */
            }
        }
 
        i += char_len;
    }
 
    return 0;  /* Valid */
}

References ck_utf8_char_length().

Referenced by ck_utf8_is_valid().

Variable Documentation

◆ [struct]

const struct { ... } CK_UTF8_WHITESPACE_RANGES[]

Initial value:

= {
    {0x0009, 0x000D},  
    {0x0020, 0x0020},  
    {0x0085, 0x0085},  
    {0x00A0, 0x00A0},  
    {0x1680, 0x1680},  
    {0x2000, 0x200A},  
    {0x2028, 0x2029},  
    {0x202F, 0x202F},  
    {0x205F, 0x205F},  
    {0x3000, 0x3000},  
}

Referenced by ck_utf8_is_whitespace().

◆ end

uint32_t end

Definition at line 215 of file utf8.c.

Referenced by ck_accum_multi_parallel_work(), ck_accum_parallel_work(), ck_adamw_parallel_work(), ck_attention_f16_split_work(), ck_attention_oracle_exact_dump_layer(), ck_attention_vec_dump_parse_env_int(), ck_audio_conv2d_whc_grouped_f32_range(), ck_audio_glu_split_f32_range(), ck_audio_relative_shift_f32_range(), ck_bf16_convert_work(), ck_bf16_round_work(), ck_env_int_default(), ck_gemm_nt_bf16_exact_rows(), ck_gemm_nt_bf16_storage_exact_rows(), ck_gemm_nt_fp32_exact_rows(), ck_gemma4_embed_range(), ck_gemma4_prepare_bf16_range(), ck_gemma4_prepare_q5_range(), ck_gemv_bf16_rows(), ck_gemv_bf16_storage_rows(), ck_ir_v2_apply_meta(), ck_ir_v2_apply_weight_dtypes(), ck_ir_v2_find_array_end(), ck_ir_v2_find_key(), ck_ir_v2_find_key(), ck_ir_v2_next_object(), ck_ir_v2_parse_bool(), ck_ir_v2_parse_bool(), ck_ir_v2_parse_buffers(), ck_ir_v2_parse_float(), ck_ir_v2_parse_int(), ck_ir_v2_parse_json(), ck_ir_v2_parse_nodes(), ck_ir_v2_parse_string(), ck_ir_v2_parse_string(), ck_ir_v2_parse_string_field(), ck_ir_v2_skip_string(), ck_ir_v2_skip_ws(), ck_moe_q4k_mixed_parallel_work(), ck_moe_q4k_q5k_parallel_work(), ck_moe_q4k_q5k_quantize_work(), ck_moe_shared_q4k_parallel_work(), ck_moe_shared_q8_0_parallel_work(), ck_nvfp4_gemv_rows(), ck_parallel_for_worker(), ck_parse_env_int(), ck_patch_projection_bf16_native_work(), ck_scale_parallel_work(), ck_ssm_conv1d_llama_channel_range(), ck_ssm_conv1d_llama_fma_channel_range(), ck_sum_sq_parallel_work(), ck_threadpool_parallel_for_n(), ck_utf8_is_whitespace(), ds_mla_kv_decompress_bf16_rows(), gemm_q4_packed_vnni_x8_q8k_4m_range_fn(), load_manifest(), load_manifest(), load_manifest(), mamba2_rmsnorm_gate_f32(), parse_eos_ids(), parse_eos_ids(), parse_float_field_in_range(), parse_float_field_in_range(), parse_float_field_in_range(), parse_float_field_in_range(), parse_int_field_in_range(), parse_int_field_in_range(), parse_int_field_in_range(), parse_int_field_in_range(), run_benchmark(), run_benchmark(), topology_discover_numa(), topology_measure_memory_bandwidth_ex(), trim_string(), and vision_mrope_yarn_corr_dims().

◆ start

uint32_t start

Definition at line 214 of file utf8.c.

Referenced by bump_bytes(), ck_accum_multi_parallel_work(), ck_accum_parallel_work(), ck_adamw_parallel_work(), ck_ir_v2_apply_weight_dtypes(), ck_ir_v2_next_object(), ck_ir_v2_parse_bindings(), ck_ir_v2_parse_buffers(), ck_ir_v2_parse_nodes(), ck_ir_v2_parse_shape(), ck_ir_v2_parse_string(), ck_ir_v2_parse_string(), ck_scale_parallel_work(), ck_sum_sq_parallel_work(), ck_utf8_is_whitespace(), encode_text_segment(), find_object_range(), find_object_range(), find_object_range(), find_object_range(), group_limited_topk_router_f32_impl(), load_manifest(), load_manifest(), load_manifest(), mamba2_rmsnorm_gate_f32(), run_benchmark(), run_benchmark(), sample_token(), sample_token(), sample_token(), sample_topk(), sample_topk(), sample_topk(), topology_discover_numa(), topology_measure_memory_bandwidth_ex(), trim_string(), and vision_mrope_yarn_corr_dims().

Functions

Variables

Function Documentation

◆ ck_tokenizer_utf8_normalize_nfc()

◆ ck_utf8_byte_to_offset()

◆ ck_utf8_char_length()

◆ ck_utf8_count_chars()

◆ ck_utf8_from_cp()

◆ ck_utf8_is_valid()

◆ ck_utf8_is_whitespace()

◆ ck_utf8_next_char()

◆ ck_utf8_offset_to_byte()

◆ ck_utf8_validate()

Variable Documentation

◆ [struct]

◆ end

◆ start