#include <stddef.h>
#include <stdint.h>
#include <stdbool.h>
#include "tokenizer/murmurhash3.h"
#include "tokenizer/memory_pool.h"
#include "tokenizer/hash_table.h"
#include "tokenizer/utf8.h"
#include "data_structures/tries/trie.h"

Data Structures
struct	CKTokenizer

struct	CKTokenizerConfig

struct	CKTokenizerToken

Macros
#define	CK_TOKENIZER_API __attribute__((visibility("default")))

#define	CK_TOKENIZER_DEFAULT_HT_SIZE 65536

#define	CK_TOKENIZER_MAX_TOKEN_LEN 256

#define	CK_TOKENIZER_MAX_VOCAB_SIZE 256000

Enumerations
enum	CKSpacePrefixStyle { CK_SPACE_PREFIX_AUTO = 0 , CK_SPACE_PREFIX_GPT2 = 1 , CK_SPACE_PREFIX_SPM = 2 , CK_SPACE_PREFIX_ASCII = 3 }

enum	CKSpmMode { CK_SPM_MODE_UNIGRAM = 0 , CK_SPM_MODE_LLAMA = 1 }

enum	CKTokenizerType { CK_TOKENIZER_BPE = 0 , CK_TOKENIZER_WORDPIECE = 1 , CK_TOKENIZER_SPM = 2 }

Functions
	__attribute__ ((visibility("default"))) CKTokenizer *ck_tokenizer_create(CKTokenizerType type)

int	ck_tokenizer_add_merge (CKTokenizer *tok, int32_t left_id, int32_t right_id, int32_t merged_id, int32_t priority)

int	ck_tokenizer_add_special_token (CKTokenizer tok, const char name, int32_t id)

int	ck_tokenizer_add_token (CKTokenizer tok, const char token, int32_t id, float score)

static CKTokenizer *	ck_tokenizer_create_bpe (void)

static CKTokenizer *	ck_tokenizer_create_spm (void)

static CKTokenizer *	ck_tokenizer_create_wordpiece (void)

int	ck_tokenizer_decode (const CKTokenizer tok, const int32_t ids, int num_ids, char *text, int max_len)

int	ck_tokenizer_encode (const CKTokenizer tok, const char text, int text_len, int32_t *ids, int max_ids)

int	ck_tokenizer_encode_tokens (const CKTokenizer tok, const char text, int text_len, const char **out_tokens, int max_tokens)

int	ck_tokenizer_encode_with_special (CKTokenizer tok, const char text, int text_len, int32_t *ids, int max_ids, bool add_special)

int	ck_tokenizer_load_binary (CKTokenizer tok, int vocab_size, const int32_t offsets, const char strings, int num_merges, const int32_t merges)

int	ck_tokenizer_load_binary_with_scores (CKTokenizer tok, int vocab_size, const int32_t offsets, const char strings, const float scores, const uint8_t types, int num_merges, const int32_t merges)

int	ck_tokenizer_load_gguf (CKTokenizer tok, const char path)

int	ck_tokenizer_load_json (CKTokenizer tok, const char path)

int	ck_tokenizer_load_merges (CKTokenizer tok, const char path)

int	ck_tokenizer_load_text (CKTokenizer tok, const char path)

static size_t	ck_tokenizer_vocab_size (const CKTokenizer *tok)

Variables
bool	add_bos

bool bool	add_eos

bool	add_space_prefix

int32_t int32_t	bos

int32_t int32_t int32_t	eos

int32_t	id

const int32_t *	ids

bool	lowercase

int32_t int32_t int32_t int32_t int32_t	mask

const int32_t int	num_ids

const int32_t int int *	out_len

int32_t int32_t int32_t int32_t	pad

int32_t float *	score

CKSpmMode	spm_mode

CKSpacePrefixStyle	style

const char *	text

const char *	token

int32_t	unk

bool	use_trie

Macro Definition Documentation

◆ CK_TOKENIZER_API

#define CK_TOKENIZER_API __attribute__((visibility("default")))

Definition at line 40 of file tokenizer.h.

◆ CK_TOKENIZER_DEFAULT_HT_SIZE

#define CK_TOKENIZER_DEFAULT_HT_SIZE 65536

Definition at line 50 of file tokenizer.h.

◆ CK_TOKENIZER_MAX_TOKEN_LEN

#define CK_TOKENIZER_MAX_TOKEN_LEN 256

Definition at line 44 of file tokenizer.h.

◆ CK_TOKENIZER_MAX_VOCAB_SIZE

#define CK_TOKENIZER_MAX_VOCAB_SIZE 256000

Definition at line 47 of file tokenizer.h.

Enumeration Type Documentation

◆ CKSpacePrefixStyle

enum CKSpacePrefixStyle

Enumerator
CK_SPACE_PREFIX_AUTO
CK_SPACE_PREFIX_GPT2
CK_SPACE_PREFIX_SPM
CK_SPACE_PREFIX_ASCII

Definition at line 60 of file tokenizer.h.

             {
    CK_SPACE_PREFIX_AUTO = 0,    /* Auto-detect from vocabulary */
    CK_SPACE_PREFIX_GPT2 = 1,    /* GPT-2 style: Ġ (U+0120, bytes 0xC4 0xA0) */
    CK_SPACE_PREFIX_SPM = 2,     /* SentencePiece style: ▁ (U+2581, bytes 0xE2 0x96 0x81) */
    CK_SPACE_PREFIX_ASCII = 3    /* ASCII identity mode: no UTF-8/byte remap */
} CKSpacePrefixStyle;

◆ CKSpmMode

enum CKSpmMode

Enumerator
CK_SPM_MODE_UNIGRAM
CK_SPM_MODE_LLAMA

Definition at line 68 of file tokenizer.h.

             {
    CK_SPM_MODE_UNIGRAM = 0,     /* SentencePiece unigram/Viterbi */
    CK_SPM_MODE_LLAMA = 1        /* llama.cpp merge-style SPM */
} CKSpmMode;

◆ CKTokenizerType

enum CKTokenizerType

Enumerator
CK_TOKENIZER_BPE
CK_TOKENIZER_WORDPIECE
CK_TOKENIZER_SPM

Definition at line 53 of file tokenizer.h.

             {
    CK_TOKENIZER_BPE = 0,        /* Byte-Pair Encoding (GPT-2, LLaMA, Qwen) */
    CK_TOKENIZER_WORDPIECE = 1,  /* WordPiece (BERT, RoBERTa) */
    CK_TOKENIZER_SPM = 2         /* SentencePiece (unigram) */
} CKTokenizerType;

Function Documentation

◆ attribute()

__attribute__ ( (visibility("default")) )

Create a new tokenizer.

Parameters

type	Tokenizer type (BPE, WordPiece, SPM)

Returns: Newly allocated tokenizer, or NULL on error

Free a tokenizer.

Parameters

tok	Tokenizer to free

Reset tokenizer state (clear vocab but keep config).

Parameters

tok	Tokenizer to reset

Set special token IDs.

Parameters

tok	Tokenizer
unk	Unknown token ID
bos	Beginning-of-sequence token ID
eos	End-of-sequence token ID
pad	Padding token ID
mask	Mask token ID

Set whether to add BOS/EOS tokens during encoding.

Parameters

tok	Tokenizer
add_bos	If true, prepend BOS token (if available)
add_eos	If true, append EOS token (if available)

Set whether to add the SentencePiece space prefix (▁) at the start.

This mirrors SentencePiece's add_dummy_prefix behavior.

Parameters

tok	Tokenizer
add_space_prefix	If true, add leading ▁ when appropriate

Set SentencePiece mode.

Parameters

tok	Tokenizer
spm_mode	SPM mode (unigram or llama-style)

Set whether to lowercase input text before tokenizing.

Parameters

tok	Tokenizer
lowercase	If true, convert text to lowercase

Set lookup method (trie vs hash table).

Parameters

tok	Tokenizer
use_trie	If true, use trie (faster for longest-match), false = hash table

Set space prefix style for BPE tokenizers.

GPT-2/Qwen use Ġ (U+0120), LLaMA/SentencePiece use ▁ (U+2581). Default is AUTO which auto-detects from vocabulary.

Parameters

tok	Tokenizer
style	Space prefix style (AUTO, GPT2, or SPM)

Auto-detect space prefix style from vocabulary.

Checks for presence of tokens starting with Ġ vs ▁ to determine style.

Parameters

tok Tokenizer

Returns: Detected style (GPT2 or SPM)

Look up token ID by string.

Parameters

tok	Tokenizer
token	Token string

Returns: Token ID, or unk_id if not found

Get token string by ID.

Parameters

tok	Tokenizer
id	Token ID

Returns: Token string, or NULL if invalid

Get token info by ID.

Parameters

tok	Tokenizer
id	Token ID
score	Output: token score

Returns: Token string, or NULL if invalid

Decode to buffer allocated by caller.

Parameters

tok	Tokenizer
ids	Input token IDs
num_ids	Number of IDs
out_len	Output: length of decoded string

Returns: Newly allocated string, or NULL on error

Get the tokenizer type name.

Parameters

tok Tokenizer

Returns: Type name string

Check if token is special.

Parameters

tok	Tokenizer
id	Token ID

Returns: true if special token

Estimate encoded token count.

Parameters

tok	Tokenizer
text	Input text

Returns: Estimated number of tokens

Get last error message.

Returns: Last error message, or NULL if no error

Free a True BPE tokenizer.

Parameters

bpe	Tokenizer to free

Add a token to the vocabulary.

Parameters

bpe	Tokenizer
token	Token string (UTF-8)
id	Token ID
score	Token score (for unigram models, 0.0 for BPE)

Returns: 0 on success, -1 on error

Add a BPE merge rule by token IDs.

Merge rules define how tokens are combined during encoding. Rules with lower priority numbers are applied first.

Parameters

bpe	Tokenizer
left_id	Left token ID
right_id	Right token ID
merged_id	Resulting merged token ID
priority	Merge priority (lower = applied first)

Returns: 0 on success, -1 on error

Add a BPE merge rule by token strings.

This looks up the token IDs automatically and determines the merged token. The merged token must already exist in the vocabulary.

Parameters

bpe	Tokenizer
left	Left token string
right	Right token string
priority	Merge priority (lower = applied first)

Returns: 0 on success, -1 on error

Set special token IDs.

Parameters

bpe	Tokenizer
unk	Unknown token ID (-1 to disable)
bos	Beginning-of-sequence token ID (-1 to disable)
eos	End-of-sequence token ID (-1 to disable)
pad	Padding token ID (-1 to disable)

Add a special token that should be matched BEFORE BPE encoding.

Parameters

bpe	Tokenizer
token	Token string to match literally (e.g., "<\|im_end\|>")
id	Token ID to output when matched

Returns: 0 on success, -1 on error

Set tokenizer configuration.

Parameters

bpe	Tokenizer
config	Configuration to apply

Load vocabulary + merges from binary buffers.

Parameters

bpe	Tokenizer
vocab_size	Number of tokens
offsets	Offsets array (length vocab_size)
strings	Null-terminated token strings blob
num_merges	Number of merge rules
merges	Merge triples [left_id, right_id, merged_id] (length num_merges*3)

Returns: 0 on success, -1 on error

Look up a token ID by string.

Parameters

bpe	Tokenizer
token	Token string

Returns: Token ID, or unk_id if not found

Get a token string by ID.

Parameters

bpe	Tokenizer
id	Token ID

Returns: Token string, or NULL if invalid

Get vocabulary size.

Parameters

bpe Tokenizer

Returns: Number of tokens in vocabulary

Get number of merge rules.

Parameters

bpe Tokenizer

Returns: Number of merge rules

Auto-detect space prefix style from vocabulary.

Counts tokens starting with Ġ (GPT-2) vs ▁ (SentencePiece) to determine style. The detected style is cached in the config.

Parameters

bpe Tokenizer

Returns: Detected style (GPT2 or SPM)

Encode text to token IDs using true BPE algorithm.

This applies merge rules in priority order (not greedy longest-match).

Parameters

bpe	Tokenizer
text	Input text (UTF-8)
text_len	Text length in bytes, or -1 for null-terminated
ids	Output token IDs array
max_ids	Maximum IDs to write

Returns: Number of tokens written

Decode token IDs to text.

Parameters

bpe	Tokenizer
ids	Input token IDs
num_ids	Number of IDs
text	Output text buffer
max_len	Maximum text length

Returns: Number of bytes written (excluding null terminator)

Referenced by __attribute__(), fused_mlp_swiglu_decode(), fused_mlp_swiglu_decode_tiled(), fused_mlp_swiglu_decode_v2(), gelu_backward_exact(), gelu_backward_fast(), gelu_fast_inplace(), recurrent_dt_gate_forward_pytorch_fp32(), recurrent_sigmoid_forward_pytorch_bf16_input_fp32_output(), recurrent_silu_forward_pytorch_bf16_input_fp32_output(), recurrent_silu_forward_pytorch_bf16_storage(), rmsnorm_qkv_q4k_fused(), swiglu_backward(), swiglu_forward(), and swiglu_forward_pytorch_bf16_storage().

◆ ck_tokenizer_add_merge()

int ck_tokenizer_add_merge	(	CKTokenizer *	tok,
		int32_t	left_id,
		int32_t	right_id,
		int32_t	merged_id,
		int32_t	priority
	)

Add a BPE merge rule.

Parameters

tok	Tokenizer
left_id	Left token ID
right_id	Right token ID
merged_id	Merged token ID
priority	Lower = higher priority (applied first)

Returns: 0 on success, -1 on error

Definition at line 560 of file tokenizer.c.

                                                                                                            {
    (void)tok; (void)left; (void)right; (void)merged; (void)priority; return 0;
}

References left, priority, and right.

◆ ck_tokenizer_add_special_token()

int ck_tokenizer_add_special_token	(	CKTokenizer *	tok,
		const char *	name,
		int32_t	id
	)

Add special token (UNK, BOS, EOS, PAD, MASK).

Parameters

tok	Tokenizer
name	Special token name ("unk", "bos", "eos", "pad", "mask")
id	Token ID

Returns: 0 on success, -1 on error

Definition at line 218 of file tokenizer.c.

                                                                                   {
    if (!tok || !name) return -1;
    if (ck_tokenizer_add_token(tok, name, id, -1e10f) != 0) return -1;
 
    TokenInfo *info = (TokenInfo *)ck_tokenizer_hash_table_lookup(tok->vocab, name);
    if (info) info->is_special = true;
 
    /* Also add to trie as special */
    if (tok->vocab_trie) {
        ck_trie_insert(tok->vocab_trie, name, id, true, 0);
    }
 
    if (strcmp(name, "<unk>") == 0 || strcmp(name, "[UNK]") == 0) tok->unk_id = id;
    else if (strcmp(name, "<s>") == 0 || strcmp(name, "<bos>") == 0 || strcmp(name, "[BOS]") == 0) tok->bos_id = id;
    else if (strcmp(name, "</s>") == 0 || strcmp(name, "<eos>") == 0 || strcmp(name, "[EOS]") == 0) tok->eos_id = id;
    else if (strcmp(name, "<pad>") == 0 || strcmp(name, "[PAD]") == 0) tok->pad_id = id;
 
    return 0;
}

References CKTokenizer::bos_id, ck_tokenizer_add_token(), ck_tokenizer_hash_table_lookup(), ck_trie_insert(), CKTokenizer::eos_id, id, CKTokenizer::pad_id, CKTokenizer::unk_id, CKTokenizer::vocab, and CKTokenizer::vocab_trie.

Referenced by main().

◆ ck_tokenizer_add_token()

int ck_tokenizer_add_token	(	CKTokenizer *	tok,
		const char *	token,
		int32_t	id,
		float	score
	)

Add a token to vocabulary.

Parameters

tok	Tokenizer
token	Token string
id	Token ID
score	Token score (for SPM)

Returns: 0 on success, -1 on error

Definition at line 162 of file tokenizer.c.

                                                                                         {
    if (!tok || !token) {
        return -1;
    }
 
    /* Ensure we have space in reverse vocab */
    if (id >= (int32_t)tok->vocab_capacity) {
        size_t new_cap = tok->vocab_capacity * 2;
        while (new_cap <= (size_t)id) {
            new_cap *= 2;
        }
        char **new_array = (char **)realloc(tok->id_to_token, new_cap * sizeof(char *));
        if (!new_array) {
            return -1;
        }
        memset(new_array + tok->vocab_capacity, 0, (new_cap - tok->vocab_capacity) * sizeof(char *));
        tok->id_to_token = new_array;
        tok->vocab_capacity = new_cap;
    }
 
    /* Check if token already exists */
    TokenInfo *existing = (TokenInfo *)ck_tokenizer_hash_table_lookup(tok->vocab, token);
    if (existing) {
        existing->id = id;
        existing->score = score;
        if (id >= (int32_t)tok->vocab_size) tok->vocab_size = id + 1;
        if (tok->id_to_token[id]) free(tok->id_to_token[id]);
        tok->id_to_token[id] = strdup(token);
        return 0;
    }
 
    /* Create new token info */
    TokenInfo *info = (TokenInfo *)malloc(sizeof(TokenInfo));
    if (!info) return -1;
    info->id = id;
    info->score = score;
    info->is_special = false;
 
    if (ck_tokenizer_hash_table_insert(tok->vocab, token, info) != 0) {
        free(info);
        return -1;
    }
 
    /* Also add to trie for fast longest-match lookups */
    if (tok->vocab_trie) {
        ck_trie_insert(tok->vocab_trie, token, id, false, 0);
    }
 
    if (id >= (int32_t)tok->vocab_size) tok->vocab_size = id + 1;
    if (tok->id_to_token[id]) free(tok->id_to_token[id]);
    tok->id_to_token[id] = strdup(token);
 
    return 0;
}

References ck_tokenizer_hash_table_insert(), ck_tokenizer_hash_table_lookup(), ck_trie_insert(), id, CKTokenizer::id_to_token, score, token, CKTokenizer::vocab, CKTokenizer::vocab_capacity, CKTokenizer::vocab_size, and CKTokenizer::vocab_trie.

Referenced by ck_tokenizer_add_special_token().

◆ ck_tokenizer_create_bpe()

static CKTokenizer * ck_tokenizer_create_bpe ( void )

inlinestatic

Create tokenizer with default BPE config.

Definition at line 157 of file tokenizer.h.

                                                         {
    return ck_tokenizer_create(CK_TOKENIZER_BPE);
}

References CK_TOKENIZER_BPE, and ck_tokenizer_create().

Referenced by main().

◆ ck_tokenizer_create_spm()

static CKTokenizer * ck_tokenizer_create_spm ( void )

inlinestatic

Create tokenizer with default SPM config.

Definition at line 171 of file tokenizer.h.

                                                         {
    return ck_tokenizer_create(CK_TOKENIZER_SPM);
}

References ck_tokenizer_create(), and CK_TOKENIZER_SPM.

◆ ck_tokenizer_create_wordpiece()

static CKTokenizer * ck_tokenizer_create_wordpiece ( void )

inlinestatic

Create tokenizer with default WordPiece config.

Definition at line 164 of file tokenizer.h.

                                                               {
    return ck_tokenizer_create(CK_TOKENIZER_WORDPIECE);
}

References ck_tokenizer_create(), and CK_TOKENIZER_WORDPIECE.

◆ ck_tokenizer_decode()

int ck_tokenizer_decode	(	const CKTokenizer *	tok,
		const int32_t *	ids,
		int	num_ids,
		char *	text,
		int	max_len
	)

Decode token IDs to text.

Parameters

tok	Tokenizer
ids	Input token IDs
num_ids	Number of IDs
text	Output text buffer
max_len	Maximum text length

Returns: Number of bytes written

Definition at line 734 of file ck_tokenizer.c.

                                     {
    int len = 0;
 
    for (int i = 0; i < num_ids; i++) {
        /* Skip special tokens */
        if (ids[i] == tok->bos_id || ids[i] == tok->eos_id || ids[i] == tok->pad_id) {
            continue;
        }
 
        const char *token = ck_tokenizer_id_to_token(tok, ids[i]);
        if (!token) continue;
 
        int token_len = (int)strlen(token);
 
        /* Handle byte tokens <0xXX> */
        if (token_len == 6 && token[0] == '<' && token[1] == '0' && token[2] == 'x') {
            char hex[3] = {token[3], token[4], 0};
            unsigned int byte = (unsigned int)strtol(hex, NULL, 16);
            if (len < max_len - 1) {
                text[len++] = (char)byte;
            }
            continue;
        }
 
        /* Handle tokenizer space prefixes:
         *   GPT/byte-level BPE: Ġ (0xC4 0xA0)
         *   SentencePiece BPE: ▁ (0xE2 0x96 0x81)
         */
        const char *src = token;
        if ((unsigned char)token[0] == 0xC4 && (unsigned char)token[1] == 0xA0) {
            if (len < max_len - 1) {
                text[len++] = ' ';
            }
            src = token + 2;
            token_len -= 2;
        } else if ((unsigned char)token[0] == 0xE2 &&
                   (unsigned char)token[1] == 0x96 &&
                   (unsigned char)token[2] == 0x81) {
            if (len < max_len - 1) {
                text[len++] = ' ';
            }
            src = token + 3;
            token_len -= 3;
        }
 
        /* Copy token, normalizing any embedded SentencePiece markers too.
         * Some Gemma tokenizer pieces contain repeated ▁ markers after
         * indentation or punctuation; those are word-boundary spaces, not
         * literal output characters.
         */
        for (int j = 0; j < token_len && len < max_len - 1; ) {
            if (j + 2 < token_len &&
                (unsigned char)src[j] == 0xE2 &&
                (unsigned char)src[j + 1] == 0x96 &&
                (unsigned char)src[j + 2] == 0x81) {
                text[len++] = ' ';
                j += 3;
                continue;
            }
            text[len++] = src[j++];
        }
    }
 
    text[len] = '\0';
    return len;
}

References ck_tokenizer_id_to_token(), ids, max_len, num_ids, text, and token.

◆ ck_tokenizer_encode()

int ck_tokenizer_encode	(	const CKTokenizer *	tok,
		const char *	text,
		int	text_len,
		int32_t *	ids,
		int	max_ids
	)

Encode text to token IDs using greedy longest-match.

For BPE: applies merge rules iteratively. For WordPiece/SPM: greedy longest-match from vocabulary.

Parameters

tok	Tokenizer
text	Input text
text_len	Text length, or -1 for null-terminated
ids	Output token IDs
max_ids	Maximum IDs to write

Returns: Number of tokens written

Definition at line 635 of file ck_tokenizer.c.

                                     {
    if (text_len < 0) text_len = (int)strlen(text);
 
    /* Pre-tokenize: split on whitespace, keep spaces as tokens */
    /* For simplicity, treat each byte as initial token, then apply BPE */
 
    /* Initial tokens: one per byte */
    int32_t *tokens = (int32_t *)malloc(text_len * sizeof(int32_t));
    int num_tokens = 0;
 
    for (int i = 0; i < text_len; i++) {
        /* Look up single-character token */
        char c[2] = {text[i], '\0'};
        int32_t id = ck_tokenizer_lookup(tok, c, 1);
 
        /* Handle special byte tokens like <0xXX> */
        if (id == tok->unk_id) {
            char byte_token[8];
            snprintf(byte_token, sizeof(byte_token), "<0x%02X>", (unsigned char)text[i]);
            id = ck_tokenizer_lookup(tok, byte_token, -1);
        }
 
        /* Try UTF-8 multi-byte sequences */
        if (id == tok->unk_id && (unsigned char)text[i] >= 0x80) {
            int utf8_len = 1;
            if ((text[i] & 0xE0) == 0xC0) utf8_len = 2;
            else if ((text[i] & 0xF0) == 0xE0) utf8_len = 3;
            else if ((text[i] & 0xF8) == 0xF0) utf8_len = 4;
 
            if (i + utf8_len <= text_len) {
                id = ck_tokenizer_lookup(tok, text + i, utf8_len);
                if (id != tok->unk_id) {
                    tokens[num_tokens++] = id;
                    i += utf8_len - 1;
                    continue;
                }
            }
        }
 
        tokens[num_tokens++] = id;
    }
 
    /* Apply BPE merges iteratively */
    bool changed = true;
    while (changed && num_tokens > 1) {
        changed = false;
 
        /* Find best merge (lowest priority = earliest in merge list) */
        int best_pos = -1;
        int best_priority = tok->num_merges;
 
        for (int i = 0; i < num_tokens - 1; i++) {
            int merge_idx = ck_tokenizer_lookup_merge(tok, tokens[i], tokens[i + 1]);
            if (merge_idx >= 0 && tok->merges[merge_idx].priority < best_priority) {
                best_pos = i;
                best_priority = tok->merges[merge_idx].priority;
            }
        }
 
        if (best_pos >= 0) {
            int merge_idx = ck_tokenizer_lookup_merge(tok, tokens[best_pos], tokens[best_pos + 1]);
            tokens[best_pos] = tok->merges[merge_idx].merged;
 
            /* Shift remaining tokens */
            for (int i = best_pos + 1; i < num_tokens - 1; i++) {
                tokens[i] = tokens[i + 1];
            }
            num_tokens--;
            changed = true;
        }
    }
 
    /* Copy to output */
    int out_len = 0;
 
    if (tok->add_bos && out_len < max_ids) {
        ids[out_len++] = tok->bos_id;
    }
 
    for (int i = 0; i < num_tokens && out_len < max_ids; i++) {
        ids[out_len++] = tokens[i];
    }
 
    if (tok->add_eos && out_len < max_ids) {
        ids[out_len++] = tok->eos_id;
    }
 
    free(tokens);
    return out_len;
}

References CKTokenizerConfig::add_bos, CKTokenizerConfig::add_eos, CKTokenizer::bos_id, CK_TOKENIZER_BPE, ck_tokenizer_detect_space_prefix_style(), ck_tokenizer_encode_spm_dispatch(), CK_TOKENIZER_SPM, CKTokenizer::config, config, CKTokenizer::eos_id, find_longest_match(), id, ids, max_ids, preprocess_bpe_spaces(), style, text, text_len, CKTokenizerConfig::type, and CKTokenizer::unk_id.

◆ ck_tokenizer_encode_tokens()

int ck_tokenizer_encode_tokens	(	const CKTokenizer *	tok,
		const char *	text,
		int	text_len,
		const char **	out_tokens,
		int	max_tokens
	)

Encode and return tokens as array of strings.

Parameters

tok	Tokenizer
text	Input text
text_len	Text length
out_tokens	Output token strings (caller must free each)
max_tokens	Maximum tokens

Returns: Number of tokens written

◆ ck_tokenizer_encode_with_special()

int ck_tokenizer_encode_with_special	(	CKTokenizer *	tok,
		const char *	text,
		int	text_len,
		int32_t *	ids,
		int	max_ids,
		bool	add_special
	)

Encode with special token handling.

Parameters

tok	Tokenizer
text	Input text
text_len	Text length, or -1 for null-terminated
ids	Output token IDs
max_ids	Maximum IDs to write
add_special	Add BOS/EOS tokens

Returns: Number of tokens written

◆ ck_tokenizer_load_binary()

int ck_tokenizer_load_binary	(	CKTokenizer *	tok,
		int	vocab_size,
		const int32_t *	offsets,
		const char *	strings,
		int	num_merges,
		const int32_t *	merges
	)

Load vocabulary from memory-mapped binary data.

Parameters

tok	Tokenizer
vocab_size	Number of tokens
offsets	Array of offsets into strings pool
strings	String pool containing null-terminated tokens
num_merges	Number of BPE merges
merges	Merge rules as (left, right, merged) triplets

Returns: 0 on success, -1 on error

Definition at line 18 of file ck_tokenizer_v2.c.

                                                    {
    if (!tok || !offsets || !strings) return -1;
 
    // We assume ck_tokenizer_init was already called to alloc hash tables
    tok->vocab_size = 0;
    
    for (int i = 0; i < vocab_size; i++) {
        const char *token = strings + offsets[i];
        int len = (int)strlen(token);
        
        CKVocabEntry *entry = (CKVocabEntry *)ck_pool_alloc(&tok->pool, sizeof(CKVocabEntry));
        entry->token = (char *)token; 
        entry->token_len = len;
        entry->id = i;
 
        uint32_t bucket = hash_string(token, len) % tok->vocab_hash_size;
        entry->next = tok->vocab_hash[bucket];
        tok->vocab_hash[bucket] = entry;
 
        tok->id_to_token[i] = entry->token;
        tok->vocab_size++;
    }
 
    if (merges && num_merges > 0) {
        for (int i = 0; i < num_merges; i++) {
            int32_t left = merges[i*3 + 0];
            int32_t right = merges[i*3 + 1];
            int32_t merged = merges[i*3 + 2];
            ck_tokenizer_add_merge(tok, left, right, merged);
        }
    }
 
    return 0;
}

References ck_pool_alloc(), ck_tokenizer_add_merge(), ck_tokenizer_load_binary_with_scores(), hash_string(), hash_string(), hash_string(), hash_string(), hash_string(), CKVocabEntry::id, CKTokenizer::id_to_token, left, merges, CKVocabEntry::next, num_merges, offsets, CKTokenizer::pool, right, strings, CKVocabEntry::token, token, CKVocabEntry::token_len, CKTokenizer::vocab_hash, CKTokenizer::vocab_hash_size, CKTokenizer::vocab_size, and vocab_size.

Referenced by main().

◆ ck_tokenizer_load_binary_with_scores()

int ck_tokenizer_load_binary_with_scores	(	CKTokenizer *	tok,
		int	vocab_size,
		const int32_t *	offsets,
		const char *	strings,
		const float *	scores,
		const uint8_t *	types,
		int	num_merges,
		const int32_t *	merges
	)

Load vocabulary from memory-mapped binary data with scores and types.

This extended version supports SPM (SentencePiece) tokenizers which require token scores for Viterbi/DP encoding.

Parameters

tok	Tokenizer
vocab_size	Number of tokens
offsets	Array of offsets into strings pool
strings	String pool containing null-terminated tokens
scores	Array of token scores (float32), can be NULL
types	Array of token types (uint8), can be NULL
num_merges	Number of BPE merges
merges	Merge rules as (left, right, merged) triplets

Returns: 0 on success, -1 on error

Definition at line 858 of file tokenizer_spm.c.

                                                                {
    if (!tok || !offsets || !strings) return -1;
    ck_tokenizer_reset(tok);
 
    /* Free any existing scores/types arrays before reallocating */
    if (tok->scores) {
        free(tok->scores);
        tok->scores = NULL;
        tok->scores_size = 0;
    }
    if (tok->types) {
        free(tok->types);
        tok->types = NULL;
        tok->types_size = 0;
    }
 
    /* Allocate scores and types arrays if provided */
    if (scores && vocab_size > 0) {
        tok->scores = (float *)malloc(vocab_size * sizeof(float));
        if (!tok->scores) return -1;
        memcpy(tok->scores, scores, vocab_size * sizeof(float));
        tok->scores_size = (size_t)vocab_size;
    }
    if (types && vocab_size > 0) {
        tok->types = (uint8_t *)malloc(vocab_size * sizeof(uint8_t));
        if (!tok->types) {
            if (tok->scores) {
                free(tok->scores);
                tok->scores = NULL;
            }
            return -1;
        }
        memcpy(tok->types, types, vocab_size * sizeof(uint8_t));
        tok->types_size = (size_t)vocab_size;
    }
 
    for (int i = 0; i < vocab_size; i++) {
        const char *token = strings + offsets[i];
        float score = scores ? scores[i] : 0.0f;
        ck_tokenizer_add_token(tok, token, i, score);
    }
 
    /* Build byte token lookup table if types are available */
    if (types && vocab_size > 0) {
        spm_build_byte_lookup(tok, strings, offsets, vocab_size);
 
        /* Log token type statistics */
        int count_normal = 0, count_unknown = 0, count_control = 0, count_byte = 0, count_other = 0;
        int max_type = 0;
        for (int i = 0; i < vocab_size; i++) {
            uint8_t t = tok->types[i];
            if (t > max_type) max_type = t;
            switch (t) {
                case GGUF_TOKEN_NORMAL: count_normal++; break;
                case GGUF_TOKEN_UNKNOWN: count_unknown++; break;
                case GGUF_TOKEN_CONTROL: count_control++; break;
                case GGUF_TOKEN_BYTE: count_byte++; break;
                default: count_other++; break;
            }
        }
        fprintf(stderr, "[TOKENIZER] Loaded %d tokens: normal=%d, unknown=%d, control=%d, byte=%d, other=%d\n",
                vocab_size, count_normal, count_unknown, count_control, count_byte, count_other);
        if (max_type > GGUF_TOKEN_BYTE) {
            fprintf(stderr, "[TOKENIZER] Warning: Unexpected token type %d\n", max_type);
        }
    }
 
    /* TODO: Merges */
    (void)num_merges; (void)merges;
    return 0;
}

References ck_tokenizer_add_token(), ck_tokenizer_reset(), GGUF_TOKEN_BYTE, GGUF_TOKEN_CONTROL, GGUF_TOKEN_NORMAL, GGUF_TOKEN_UNKNOWN, merges, num_merges, offsets, score, CKTokenizer::scores, CKTokenizer::scores_size, spm_build_byte_lookup(), strings, token, CKTokenizer::types, CKTokenizer::types_size, and vocab_size.

Referenced by ck_tokenizer_load_binary().

◆ ck_tokenizer_load_gguf()

int ck_tokenizer_load_gguf	(	CKTokenizer *	tok,
		const char *	path
	)

Load vocabulary from GGUF file.

Parameters

tok	Tokenizer
path	Path to GGUF file

Returns: 0 on success, -1 on error

Definition at line 556 of file tokenizer.c.

556{ (void)tok; (void)path; return -1; }

◆ ck_tokenizer_load_json()

int ck_tokenizer_load_json	(	CKTokenizer *	tok,
		const char *	path
	)

Load vocabulary from JSON file (HuggingFace format).

Parameters

tok	Tokenizer
path	Path to vocab.json or tokenizer.json

Returns: 0 on success, -1 on error

Definition at line 557 of file tokenizer.c.

557{ (void)tok; (void)path; return -1; }

◆ ck_tokenizer_load_merges()

int ck_tokenizer_load_merges	(	CKTokenizer *	tok,
		const char *	path
	)

Load BPE merges from text file.

Format: token1 token2 (one merge per line)

Parameters

tok	Tokenizer
path	Path to merges.txt

Returns: 0 on success, -1 on error

Definition at line 559 of file tokenizer.c.

559{ (void)tok; (void)path; return -1; }

◆ ck_tokenizer_load_text()

int ck_tokenizer_load_text	(	CKTokenizer *	tok,
		const char *	path
	)

Load vocabulary from text file (one token per line).

Format: token_string [id] [score] Lines starting with # are comments.

Parameters

tok	Tokenizer
path	Path to vocabulary file

Returns: 0 on success, -1 on error

Definition at line 558 of file tokenizer.c.

558{ (void)tok; (void)path; return -1; }

◆ ck_tokenizer_vocab_size()

static size_t ck_tokenizer_vocab_size ( const CKTokenizer * tok )

inlinestatic

Get vocabulary size.

Definition at line 333 of file tokenizer.h.

                                                                     {
    return tok ? tok->vocab_size : 0;
}

References CKTokenizer::vocab_size.

Variable Documentation

◆ add_bos

bool add_bos

Definition at line 243 of file tokenizer.h.

Referenced by ck_tokenizer_set_add_bos_eos().

◆ add_eos

bool bool add_eos

Definition at line 243 of file tokenizer.h.

Referenced by ck_tokenizer_set_add_bos_eos().

◆ add_space_prefix

bool add_space_prefix

Definition at line 253 of file tokenizer.h.

Referenced by ck_tokenizer_set_add_space_prefix(), preprocess_spm_llama_text(), and preprocess_spm_text().

◆ bos

int32_t int32_t bos

Definition at line 231 of file tokenizer.h.

Referenced by ck_tokenizer_set_special_ids(), ck_true_bpe_set_special_ids(), load_eos_from_vocab_json(), and load_eos_from_vocab_json().

◆ eos

int32_t int32_t int32_t eos

Definition at line 232 of file tokenizer.h.

Referenced by ck_tokenizer_set_special_ids(), ck_true_bpe_set_special_ids(), load_eos_from_vocab_json(), and load_eos_from_vocab_json().

◆ id

int32_t id

Definition at line 316 of file tokenizer.h.

Referenced by ck_tokenizer_add_special_token(), ck_tokenizer_add_token(), ck_tokenizer_encode(), ck_tokenizer_id_to_token(), ck_tokenizer_load(), ck_tokenizer_lookup_exact_n(), ck_true_bpe_add_special_token(), ck_true_bpe_add_token(), ck_true_bpe_id_to_token(), ck_true_bpe_lookup(), encode_chunk(), quantize_row_q8_0(), quantize_row_q8_0_ref_local(), token_list_append(), and topology_discover_cpu().

◆ ids

const int32_t * ids

Definition at line 444 of file tokenizer.h.

Referenced by ck_tokenizer_decode(), ck_tokenizer_encode(), ck_tokenizer_encode_spm_dispatch(), ck_tokenizer_encode_spm_impl(), ck_tokenizer_encode_spm_llama_impl(), ck_tokenizer_encode_spm_plain_segment(), ck_true_bpe_decode(), ck_true_bpe_encode(), encode_chunk(), encode_text_segment(), main(), run_prompt(), run_prompt(), spm_encode_byte_fallback(), and spm_llama_resegment_node().

◆ lowercase

bool lowercase

Definition at line 269 of file tokenizer.h.

◆ mask

int32_t int32_t int32_t int32_t int32_t mask

Definition at line 234 of file tokenizer.h.

Referenced by ck_deltanet_pytorch_outer_sum(), ck_dtype_supported(), ck_tokenizer_set_special_ids(), recurrent_pytorch_fp32_square_sum(), relu2_backward(), relu_backward(), relu_backward_bf16(), relu_forward_bf16(), relu_forward_inplace_bf16(), rmsnorm_forward_pytorch_bf16_storage_impl(), and topology_discover_affinity().

◆ num_ids

const int32_t int num_ids

Definition at line 445 of file tokenizer.h.

Referenced by ck_tokenizer_decode(), ck_true_bpe_decode(), and main().

◆ out_len

const int32_t int int* out_len

Definition at line 446 of file tokenizer.h.

Referenced by ck_tokenizer_encode(), ck_utf8_next_char(), decode_bpe_token(), decode_bpe_token(), eos_pattern_process(), eos_pattern_process(), find_object_range(), find_object_range(), find_object_range(), find_object_range(), preprocess_bpe_spaces(), preprocess_spm_llama_text(), preprocess_spm_text(), preprocess_text(), run_prompt(), and run_prompt().

◆ pad

int32_t int32_t int32_t int32_t pad

Definition at line 233 of file tokenizer.h.

Referenced by ck_tokenizer_set_special_ids(), and ck_true_bpe_set_special_ids().

◆ score

int32_t float* score

Definition at line 328 of file tokenizer.h.

Referenced by attention_flash_decode_scalar(), attention_flash_query_causal(), attention_flash_query_causal_exact(), attention_flash_query_causal_exact_f16kv(), attention_flash_query_causal_exact_prerounded_f16kv(), attention_flash_query_sliding(), attention_forward_decode_head_major_gqa_bf16cache_pytorch_contract(), attention_forward_decode_head_major_gqa_flash_f16cache(), attention_mlp_fused_fp32(), attention_mlp_fused_q4k(), attention_mlp_separate_fp32(), attention_query_full_exact_regular(), ck_attention_f16_split_work(), ck_attention_query_key_f32_work(), ck_tokenizer_add_token(), ck_tokenizer_encode_spm_llama_impl(), ck_tokenizer_load_binary_with_scores(), ck_true_bpe_add_token(), deepseek_mla_attention_decode_f32_workspace(), ds_mla_attention_f32_query_range(), layer_fused_attn_mlp_qkv_q4k(), qwen4_qsa_index_select_bf16(), simple_attention(), simple_attention(), and simple_attention().

◆ spm_mode

CKSpmMode spm_mode

Definition at line 261 of file tokenizer.h.

Referenced by ck_tokenizer_set_spm_mode().

◆ style

CKSpacePrefixStyle style

Definition at line 288 of file tokenizer.h.

Referenced by ck_tokenizer_encode(), ck_tokenizer_set_space_prefix_style(), ck_true_bpe_decode(), encode_chunk(), encode_text_segment(), init_tokens_from_text(), preprocess_bpe_spaces(), and preprocess_text().

◆ text

const int32_t int char * text

◆ token

const char * token

Definition at line 307 of file tokenizer.h.

Referenced by attention_forward_decode_head_major_gqa_bf16cache_pytorch_contract(), attention_output_index(), ck_attention_llama_regular_impl(), ck_attention_llama_regular_query(), ck_gemma4_prepare_bf16_range(), ck_gemma4_prepare_q5_range(), ck_model_decode(), ck_multimodal_mrope_positions_2d(), ck_tokenizer_add_token(), ck_tokenizer_add_token(), ck_tokenizer_decode(), ck_tokenizer_detect_space_prefix_style(), ck_tokenizer_encode_spm_impl(), ck_tokenizer_load(), ck_tokenizer_load_binary(), ck_tokenizer_load_binary_with_scores(), ck_tokenizer_lookup(), ck_tokenizer_lookup(), ck_tokenizer_lookup_exact(), ck_trie_insert(), ck_true_bpe_add_special_token(), ck_true_bpe_add_token(), ck_true_bpe_decode(), ck_true_bpe_detect_space_style(), ck_true_bpe_load_binary(), ck_true_bpe_lookup(), decode_bpe_token(), decode_bpe_token(), ds_mla_attention_f32_work(), ds_qkv_idx(), eos_is_potential_prefix(), eos_is_potential_prefix(), is_eos_token(), is_eos_token(), lookup_token_exact(), main(), model_decode(), model_decode_token(), model_decode_token(), model_decode_token(), model_decode_token(), output_token(), output_token(), qwen2_0_5b_decode_decode(), qwen2_0_5b_decode_decode_token(), qwen2_0_5b_decode_decode_token(), qwen2_0_5b_decode_decode_token(), qwen2_0_5b_decode_decode_token(), qwen2_0_5b_decode_decode_token(), qwen4_ple_ngram_embed_impl(), qwen4_qsa_index_select_bf16(), rope_forward_qk_pairwise_llama_cpu(), run_benchmark(), run_benchmark(), run_generation_test(), run_generation_test(), run_inference(), run_inference(), run_inference(), spm_build_byte_lookup(), spm_token_is_byte_format(), token_has_gpt2_bytes(), topology_discover_numa(), and yarn_rope_cache_explicit_positions_impl().

◆ unk

int32_t unk

Definition at line 230 of file tokenizer.h.

Referenced by ck_tokenizer_set_special_ids(), and ck_true_bpe_set_special_ids().

◆ use_trie

bool use_trie

Definition at line 277 of file tokenizer.h.

Referenced by ck_tokenizer_set_use_trie().

Data Structures

Macros

Enumerations

Functions

Variables

Macro Definition Documentation

◆ CK_TOKENIZER_API

◆ CK_TOKENIZER_DEFAULT_HT_SIZE

◆ CK_TOKENIZER_MAX_TOKEN_LEN

◆ CK_TOKENIZER_MAX_VOCAB_SIZE

Enumeration Type Documentation

◆ CKSpacePrefixStyle

◆ CKSpmMode

◆ CKTokenizerType

Function Documentation

◆ __attribute__()

◆ ck_tokenizer_add_merge()

◆ ck_tokenizer_add_special_token()

◆ ck_tokenizer_add_token()

◆ ck_tokenizer_create_bpe()

◆ ck_tokenizer_create_spm()

◆ ck_tokenizer_create_wordpiece()

◆ ck_tokenizer_decode()

◆ ck_tokenizer_encode()

◆ ck_tokenizer_encode_tokens()

◆ ck_tokenizer_encode_with_special()

◆ ck_tokenizer_load_binary()

◆ ck_tokenizer_load_binary_with_scores()

◆ ck_tokenizer_load_gguf()

◆ ck_tokenizer_load_json()

◆ ck_tokenizer_load_merges()

◆ ck_tokenizer_load_text()

◆ ck_tokenizer_vocab_size()

Variable Documentation

◆ add_bos

◆ add_eos

◆ add_space_prefix

◆ bos

◆ eos

◆ id

◆ ids

◆ lowercase

◆ mask

◆ num_ids

◆ out_len

◆ pad

◆ score

◆ spm_mode

◆ style

◆ text

◆ token

◆ unk

◆ use_trie

◆ attribute()