tokenizer.h File Reference

#include <stddef.h>
#include <stdint.h>
#include <stdbool.h>
#include "tokenizer/murmurhash3.h"
#include "tokenizer/memory_pool.h"
#include "tokenizer/hash_table.h"
#include "tokenizer/utf8.h"
#include "data_structures/tries/trie.h"

Go to the source code of this file.

Data Structures
struct	CKTokenizer
struct	CKTokenizerConfig
struct	CKTokenizerToken

Macros
#define	CK_TOKENIZER_API   __attribute__((visibility("default")))
#define	CK_TOKENIZER_DEFAULT_HT_SIZE   65536
#define	CK_TOKENIZER_MAX_TOKEN_LEN   256
#define	CK_TOKENIZER_MAX_VOCAB_SIZE   256000

Enumerations
enum	CKSpacePrefixStyle {   CK_SPACE_PREFIX_AUTO = 0 , CK_SPACE_PREFIX_GPT2 = 1 , CK_SPACE_PREFIX_SPM = 2 , CK_SPACE_PREFIX_AUTO = 0 ,   CK_SPACE_PREFIX_GPT2 = 1 , CK_SPACE_PREFIX_SPM = 2 }
enum	CKSpmMode { CK_SPM_MODE_UNIGRAM = 0 , CK_SPM_MODE_LLAMA = 1 }
enum	CKTokenizerType { CK_TOKENIZER_BPE = 0 , CK_TOKENIZER_WORDPIECE = 1 , CK_TOKENIZER_SPM = 2 }

Functions
	__attribute__ ((visibility("default"))) CKTokenizer *ck_tokenizer_create(CKTokenizerType type)
int	ck_tokenizer_add_merge (CKTokenizer *tok, int32_t left_id, int32_t right_id, int32_t merged_id, int32_t priority)
int	ck_tokenizer_add_special_token (CKTokenizer *tok, const char *name, int32_t id)
int	ck_tokenizer_add_token (CKTokenizer *tok, const char *token, int32_t id, float score)
static CKTokenizer *	ck_tokenizer_create_bpe (void)
static CKTokenizer *	ck_tokenizer_create_spm (void)
static CKTokenizer *	ck_tokenizer_create_wordpiece (void)
int	ck_tokenizer_decode (const CKTokenizer *tok, const int32_t *ids, int num_ids, char *text, int max_len)
int	ck_tokenizer_encode (const CKTokenizer *tok, const char *text, int text_len, int32_t *ids, int max_ids)
int	ck_tokenizer_encode_tokens (const CKTokenizer *tok, const char *text, int text_len, const char **out_tokens, int max_tokens)
int	ck_tokenizer_encode_with_special (CKTokenizer *tok, const char *text, int text_len, int32_t *ids, int max_ids, bool add_special)
int	ck_tokenizer_load_binary (CKTokenizer *tok, int vocab_size, const int32_t *offsets, const char *strings, int num_merges, const int32_t *merges)
int	ck_tokenizer_load_binary_with_scores (CKTokenizer *tok, int vocab_size, const int32_t *offsets, const char *strings, const float *scores, const uint8_t *types, int num_merges, const int32_t *merges)
int	ck_tokenizer_load_gguf (CKTokenizer *tok, const char *path)
int	ck_tokenizer_load_json (CKTokenizer *tok, const char *path)
int	ck_tokenizer_load_merges (CKTokenizer *tok, const char *path)
int	ck_tokenizer_load_text (CKTokenizer *tok, const char *path)
static size_t	ck_tokenizer_vocab_size (const CKTokenizer *tok)

Variables
bool	add_bos
bool bool	add_eos
bool	add_space_prefix
int32_t int32_t	bos
int32_t int32_t int32_t	eos
int32_t	id
const int32_t *	ids
bool	lowercase
int32_t int32_t int32_t int32_t int32_t	mask
const int32_t int	num_ids
const int32_t int int *	out_len
int32_t int32_t int32_t int32_t	pad
int32_t float *	score
CKSpmMode	spm_mode
CKSpacePrefixStyle	style
const char *	text
const char *	token
int32_t	unk
bool	use_trie

Macro Definition Documentation

CK_TOKENIZER_API

#define CK_TOKENIZER_API   __attribute__((visibility("default")))

Definition at line 40 of file tokenizer.h.

CK_TOKENIZER_DEFAULT_HT_SIZE

#define CK_TOKENIZER_DEFAULT_HT_SIZE   65536

Definition at line 50 of file tokenizer.h.

CK_TOKENIZER_MAX_TOKEN_LEN

#define CK_TOKENIZER_MAX_TOKEN_LEN   256

Definition at line 44 of file tokenizer.h.

CK_TOKENIZER_MAX_VOCAB_SIZE

#define CK_TOKENIZER_MAX_VOCAB_SIZE   256000

Definition at line 47 of file tokenizer.h.

Enumeration Type Documentation

CKSpacePrefixStyle

enum CKSpacePrefixStyle

Enumerator
CK_SPACE_PREFIX_AUTO
CK_SPACE_PREFIX_GPT2
CK_SPACE_PREFIX_SPM
CK_SPACE_PREFIX_AUTO
CK_SPACE_PREFIX_GPT2
CK_SPACE_PREFIX_SPM

Definition at line 60 of file tokenizer.h.

             {
    CK_SPACE_PREFIX_AUTO = 0,    /* Auto-detect from vocabulary */
    CK_SPACE_PREFIX_GPT2 = 1,    /* GPT-2 style: Ġ (U+0120, bytes 0xC4 0xA0) */
    CK_SPACE_PREFIX_SPM = 2      /* SentencePiece style: ▁ (U+2581, bytes 0xE2 0x96 0x81) */
} CKSpacePrefixStyle;

CKSpmMode

enum CKSpmMode

Enumerator
CK_SPM_MODE_UNIGRAM
CK_SPM_MODE_LLAMA

Definition at line 67 of file tokenizer.h.

             {
    CK_SPM_MODE_UNIGRAM = 0,     /* SentencePiece unigram/Viterbi */
    CK_SPM_MODE_LLAMA = 1        /* llama.cpp merge-style SPM */
} CKSpmMode;

CKTokenizerType

enum CKTokenizerType

Enumerator
CK_TOKENIZER_BPE
CK_TOKENIZER_WORDPIECE
CK_TOKENIZER_SPM

Definition at line 53 of file tokenizer.h.

             {
    CK_TOKENIZER_BPE = 0,        /* Byte-Pair Encoding (GPT-2, LLaMA, Qwen) */
    CK_TOKENIZER_WORDPIECE = 1,  /* WordPiece (BERT, RoBERTa) */
    CK_TOKENIZER_SPM = 2         /* SentencePiece (unigram) */
} CKTokenizerType;

Function Documentation

__attribute__()

__attribute__

(

(visibility("default"))

)

Create a new tokenizer.

Parameters

type	Tokenizer type (BPE, WordPiece, SPM)

Returns

Newly allocated tokenizer, or NULL on error

Free a tokenizer.

Parameters

tok	Tokenizer to free

Reset tokenizer state (clear vocab but keep config).

Parameters

tok	Tokenizer to reset

Set special token IDs.

Parameters

tok	Tokenizer
unk	Unknown token ID
bos	Beginning-of-sequence token ID
eos	End-of-sequence token ID
pad	Padding token ID
mask	Mask token ID

Set whether to add BOS/EOS tokens during encoding.

Parameters

tok	Tokenizer
add_bos	If true, prepend BOS token (if available)
add_eos	If true, append EOS token (if available)

Set whether to add the SentencePiece space prefix (▁) at the start.

This mirrors SentencePiece's add_dummy_prefix behavior.

Parameters

tok	Tokenizer
add_space_prefix	If true, add leading ▁ when appropriate

Set SentencePiece mode.

Parameters

tok	Tokenizer
spm_mode	SPM mode (unigram or llama-style)

Set whether to lowercase input text before tokenizing.

Parameters

tok	Tokenizer
lowercase	If true, convert text to lowercase

Set lookup method (trie vs hash table).

Parameters

tok	Tokenizer
use_trie	If true, use trie (faster for longest-match), false = hash table

Set space prefix style for BPE tokenizers.

GPT-2/Qwen use Ġ (U+0120), LLaMA/SentencePiece use ▁ (U+2581). Default is AUTO which auto-detects from vocabulary.

Parameters

tok	Tokenizer
style	Space prefix style (AUTO, GPT2, or SPM)

Auto-detect space prefix style from vocabulary.

Checks for presence of tokens starting with Ġ vs ▁ to determine style.

Parameters

tok

Tokenizer

Returns

Detected style (GPT2 or SPM)

Look up token ID by string.

Parameters

tok	Tokenizer
token	Token string

Returns

Token ID, or unk_id if not found

Get token string by ID.

Parameters

tok	Tokenizer
id	Token ID

Returns

Token string, or NULL if invalid

Get token info by ID.

Parameters

tok	Tokenizer
id	Token ID
score	Output: token score

Returns

Token string, or NULL if invalid

Decode to buffer allocated by caller.

Parameters

tok	Tokenizer
ids	Input token IDs
num_ids	Number of IDs
out_len	Output: length of decoded string

Returns

Newly allocated string, or NULL on error

Get the tokenizer type name.

Parameters

tok

Tokenizer

Returns

Type name string

Check if token is special.

Parameters

tok	Tokenizer
id	Token ID

Returns

true if special token

Estimate encoded token count.

Parameters

tok	Tokenizer
text	Input text

Returns

Estimated number of tokens

Get last error message.

Returns

Last error message, or NULL if no error

Free a True BPE tokenizer.

Parameters

bpe	Tokenizer to free

Add a token to the vocabulary.

Parameters

bpe	Tokenizer
token	Token string (UTF-8)
id	Token ID
score	Token score (for unigram models, 0.0 for BPE)

Returns

0 on success, -1 on error

Add a BPE merge rule by token IDs.

Merge rules define how tokens are combined during encoding. Rules with lower priority numbers are applied first.

Parameters

bpe	Tokenizer
left_id	Left token ID
right_id	Right token ID
merged_id	Resulting merged token ID
priority	Merge priority (lower = applied first)

Returns

0 on success, -1 on error

Add a BPE merge rule by token strings.

This looks up the token IDs automatically and determines the merged token. The merged token must already exist in the vocabulary.

Parameters

bpe	Tokenizer
left	Left token string
right	Right token string
priority	Merge priority (lower = applied first)

Returns

0 on success, -1 on error

Set special token IDs.

Parameters

bpe	Tokenizer
unk	Unknown token ID (-1 to disable)
bos	Beginning-of-sequence token ID (-1 to disable)
eos	End-of-sequence token ID (-1 to disable)
pad	Padding token ID (-1 to disable)

Add a special token that should be matched BEFORE BPE encoding.

Special tokens like <|im_start|>, <|im_end|>, <|endoftext|> are matched literally in the input text before BPE processing. Without this, BPE would break them into individual characters.

Parameters

bpe	Tokenizer
token	Token string to match literally (e.g., "<|im_end|>")
id	Token ID to output when matched

Returns

0 on success, -1 on error

Set tokenizer configuration.

Parameters

bpe	Tokenizer
config	Configuration to apply

Load vocabulary + merges from binary buffers.

Parameters

bpe	Tokenizer
vocab_size	Number of tokens
offsets	Offsets array (length vocab_size)
strings	Null-terminated token strings blob
num_merges	Number of merge rules
merges	Merge triples [left_id, right_id, merged_id] (length num_merges*3)

Returns

0 on success, -1 on error

Look up a token ID by string.

Parameters

bpe	Tokenizer
token	Token string

Returns

Token ID, or unk_id if not found

Get a token string by ID.

Parameters

bpe	Tokenizer
id	Token ID

Returns

Token string, or NULL if invalid

Get vocabulary size.

Parameters

bpe

Tokenizer

Returns

Number of tokens in vocabulary

Get number of merge rules.

Parameters

bpe

Tokenizer

Returns

Number of merge rules

Auto-detect space prefix style from vocabulary.

Counts tokens starting with Ġ (GPT-2) vs ▁ (SentencePiece) to determine style. The detected style is cached in the config.

Parameters

bpe

Tokenizer

Returns

Detected style (GPT2 or SPM)

Encode text to token IDs using true BPE algorithm.

This applies merge rules in priority order (not greedy longest-match).

Parameters

bpe	Tokenizer
text	Input text (UTF-8)
text_len	Text length in bytes, or -1 for null-terminated
ids	Output token IDs array
max_ids	Maximum IDs to write

Returns

Number of tokens written

Decode token IDs to text.

Parameters

bpe	Tokenizer
ids	Input token IDs
num_ids	Number of IDs
text	Output text buffer
max_len	Maximum text length

Returns

Number of bytes written (excluding null terminator)

Referenced by fused_mlp_swiglu_decode(), fused_mlp_swiglu_decode_tiled(), fused_mlp_swiglu_decode_v2(), geglu_forward_fp32(), gelu_backward_exact(), gelu_backward_fast(), gelu_fast_inplace(), swiglu_backward(), and swiglu_forward().

ck_tokenizer_add_merge()

int ck_tokenizer_add_merge	(	CKTokenizer *	tok,
		int32_t	left_id,
		int32_t	right_id,
		int32_t	merged_id,
		int32_t	priority
	)

Add a BPE merge rule.

Parameters

tok	Tokenizer
left_id	Left token ID
right_id	Right token ID
merged_id	Merged token ID
priority	Lower = higher priority (applied first)

Returns

0 on success, -1 on error

Definition at line 1336 of file tokenizer.c.

                                                                                                            {
    (void)tok; (void)left; (void)right; (void)merged; (void)priority; return 0;
}

References left, priority, and right.

ck_tokenizer_add_special_token()

int ck_tokenizer_add_special_token	(	CKTokenizer *	tok,
		const char *	name,
		int32_t	id
	)

Add special token (UNK, BOS, EOS, PAD, MASK).

Parameters

tok	Tokenizer
name	Special token name ("unk", "bos", "eos", "pad", "mask")
id	Token ID

Returns

0 on success, -1 on error

Definition at line 213 of file tokenizer.c.

                                                                                   {
    if (!tok || !name) return -1;
    if (ck_tokenizer_add_token(tok, name, id, -1e10f) != 0) return -1;
 
    TokenInfo *info = (TokenInfo *)ck_tokenizer_hash_table_lookup(tok->vocab, name);
    if (info) info->is_special = true;
 
    /* Also add to trie as special */
    if (tok->vocab_trie) {
        ck_trie_insert(tok->vocab_trie, name, id, true, 0);
    }
 
    if (strcmp(name, "<unk>") == 0 || strcmp(name, "[UNK]") == 0) tok->unk_id = id;
    else if (strcmp(name, "<s>") == 0 || strcmp(name, "<bos>") == 0 || strcmp(name, "[BOS]") == 0) tok->bos_id = id;
    else if (strcmp(name, "</s>") == 0 || strcmp(name, "<eos>") == 0 || strcmp(name, "[EOS]") == 0) tok->eos_id = id;
    else if (strcmp(name, "<pad>") == 0 || strcmp(name, "[PAD]") == 0) tok->pad_id = id;
 
    return 0;
}

References CKTokenizer::bos_id, ck_tokenizer_add_token(), ck_tokenizer_hash_table_lookup(), ck_trie_insert(), CKTokenizer::eos_id, id, CKTokenizer::pad_id, CKTokenizer::unk_id, CKTokenizer::vocab, and CKTokenizer::vocab_trie.

Referenced by main().

ck_tokenizer_add_token()

int ck_tokenizer_add_token	(	CKTokenizer *	tok,
		const char *	token,
		int32_t	id,
		float	score
	)

Add a token to vocabulary.

Parameters

tok	Tokenizer
token	Token string
id	Token ID
score	Token score (for SPM)

Returns

0 on success, -1 on error

Definition at line 157 of file tokenizer.c.

                                                                                         {
    if (!tok || !token) {
        return -1;
    }
 
    /* Ensure we have space in reverse vocab */
    if (id >= (int32_t)tok->vocab_capacity) {
        size_t new_cap = tok->vocab_capacity * 2;
        while (new_cap <= (size_t)id) {
            new_cap *= 2;
        }
        char **new_array = (char **)realloc(tok->id_to_token, new_cap * sizeof(char *));
        if (!new_array) {
            return -1;
        }
        memset(new_array + tok->vocab_capacity, 0, (new_cap - tok->vocab_capacity) * sizeof(char *));
        tok->id_to_token = new_array;
        tok->vocab_capacity = new_cap;
    }
 
    /* Check if token already exists */
    TokenInfo *existing = (TokenInfo *)ck_tokenizer_hash_table_lookup(tok->vocab, token);
    if (existing) {
        existing->id = id;
        existing->score = score;
        if (id >= (int32_t)tok->vocab_size) tok->vocab_size = id + 1;
        if (tok->id_to_token[id]) free(tok->id_to_token[id]);
        tok->id_to_token[id] = strdup(token);
        return 0;
    }
 
    /* Create new token info */
    TokenInfo *info = (TokenInfo *)malloc(sizeof(TokenInfo));
    if (!info) return -1;
    info->id = id;
    info->score = score;
    info->is_special = false;
 
    if (ck_tokenizer_hash_table_insert(tok->vocab, token, info) != 0) {
        free(info);
        return -1;
    }
 
    /* Also add to trie for fast longest-match lookups */
    if (tok->vocab_trie) {
        ck_trie_insert(tok->vocab_trie, token, id, false, 0);
    }
 
    if (id >= (int32_t)tok->vocab_size) tok->vocab_size = id + 1;
    if (tok->id_to_token[id]) free(tok->id_to_token[id]);
    tok->id_to_token[id] = strdup(token);
 
    return 0;
}

References ck_tokenizer_hash_table_insert(), ck_tokenizer_hash_table_lookup(), ck_trie_insert(), id, CKTokenizer::id_to_token, score, token, CKTokenizer::vocab, CKTokenizer::vocab_capacity, CKTokenizer::vocab_size, and CKTokenizer::vocab_trie.

Referenced by ck_tokenizer_add_special_token(), and ck_tokenizer_load_binary_with_scores().

ck_tokenizer_create_bpe()

static CKTokenizer* ck_tokenizer_create_bpe ( void  )

inlinestatic

Create tokenizer with default BPE config.

Definition at line 156 of file tokenizer.h.

                                                         {
    return ck_tokenizer_create(CK_TOKENIZER_BPE);
}

References CK_TOKENIZER_BPE, and ck_tokenizer_create().

Referenced by main().

ck_tokenizer_create_spm()

static CKTokenizer* ck_tokenizer_create_spm ( void  )

inlinestatic

Create tokenizer with default SPM config.

Definition at line 170 of file tokenizer.h.

                                                         {
    return ck_tokenizer_create(CK_TOKENIZER_SPM);
}

References ck_tokenizer_create(), and CK_TOKENIZER_SPM.

ck_tokenizer_create_wordpiece()

static CKTokenizer* ck_tokenizer_create_wordpiece ( void  )

inlinestatic

Create tokenizer with default WordPiece config.

Definition at line 163 of file tokenizer.h.

                                                               {
    return ck_tokenizer_create(CK_TOKENIZER_WORDPIECE);
}

References ck_tokenizer_create(), and CK_TOKENIZER_WORDPIECE.

ck_tokenizer_decode()

int ck_tokenizer_decode	(	const CKTokenizer *	tok,
		const int32_t *	ids,
		int	num_ids,
		char *	text,
		int	max_len
	)

Decode token IDs to text.

Parameters

tok	Tokenizer
ids	Input token IDs
num_ids	Number of IDs
text	Output text buffer
max_len	Maximum text length

Returns

Number of bytes written

Definition at line 737 of file ck_tokenizer.c.

                                     {
    int len = 0;
 
    for (int i = 0; i < num_ids; i++) {
        /* Skip special tokens */
        if (ids[i] == tok->bos_id || ids[i] == tok->eos_id || ids[i] == tok->pad_id) {
            continue;
        }
 
        const char *token = ck_tokenizer_id_to_token(tok, ids[i]);
        if (!token) continue;
 
        int token_len = (int)strlen(token);
 
        /* Handle byte tokens <0xXX> */
        if (token_len == 6 && token[0] == '<' && token[1] == '0' && token[2] == 'x') {
            char hex[3] = {token[3], token[4], 0};
            unsigned int byte = (unsigned int)strtol(hex, NULL, 16);
            if (len < max_len - 1) {
                text[len++] = (char)byte;
            }
            continue;
        }
 
        /* Handle GPT-style space prefix (Ġ = 0xC4 0xA0 in UTF-8) */
        const char *src = token;
        if ((unsigned char)token[0] == 0xC4 && (unsigned char)token[1] == 0xA0) {
            if (len < max_len - 1) {
                text[len++] = ' ';
            }
            src = token + 2;
            token_len -= 2;
        }
 
        /* Copy token */
        for (int j = 0; j < token_len && len < max_len - 1; j++) {
            text[len++] = src[j];
        }
    }
 
    text[len] = '\0';
    return len;
}

References ck_tokenizer_id_to_token(), ids, max_len, num_ids, text, and token.

ck_tokenizer_encode()

int ck_tokenizer_encode	(	const CKTokenizer *	tok,
		const char *	text,
		int	text_len,
		int32_t *	ids,
		int	max_ids
	)

Encode text to token IDs using greedy longest-match.

For BPE: applies merge rules iteratively. For WordPiece/SPM: greedy longest-match from vocabulary.

Parameters

tok	Tokenizer
text	Input text
text_len	Text length, or -1 for null-terminated
ids	Output token IDs
max_ids	Maximum IDs to write

Returns

Number of tokens written

Definition at line 638 of file ck_tokenizer.c.

                                     {
    if (text_len < 0) text_len = (int)strlen(text);
 
    /* Pre-tokenize: split on whitespace, keep spaces as tokens */
    /* For simplicity, treat each byte as initial token, then apply BPE */
 
    /* Initial tokens: one per byte */
    int32_t *tokens = (int32_t *)malloc(text_len * sizeof(int32_t));
    int num_tokens = 0;
 
    for (int i = 0; i < text_len; i++) {
        /* Look up single-character token */
        char c[2] = {text[i], '\0'};
        int32_t id = ck_tokenizer_lookup(tok, c, 1);
 
        /* Handle special byte tokens like <0xXX> */
        if (id == tok->unk_id) {
            char byte_token[8];
            snprintf(byte_token, sizeof(byte_token), "<0x%02X>", (unsigned char)text[i]);
            id = ck_tokenizer_lookup(tok, byte_token, -1);
        }
 
        /* Try UTF-8 multi-byte sequences */
        if (id == tok->unk_id && (unsigned char)text[i] >= 0x80) {
            int utf8_len = 1;
            if ((text[i] & 0xE0) == 0xC0) utf8_len = 2;
            else if ((text[i] & 0xF0) == 0xE0) utf8_len = 3;
            else if ((text[i] & 0xF8) == 0xF0) utf8_len = 4;
 
            if (i + utf8_len <= text_len) {
                id = ck_tokenizer_lookup(tok, text + i, utf8_len);
                if (id != tok->unk_id) {
                    tokens[num_tokens++] = id;
                    i += utf8_len - 1;
                    continue;
                }
            }
        }
 
        tokens[num_tokens++] = id;
    }
 
    /* Apply BPE merges iteratively */
    bool changed = true;
    while (changed && num_tokens > 1) {
        changed = false;
 
        /* Find best merge (lowest priority = earliest in merge list) */
        int best_pos = -1;
        int best_priority = tok->num_merges;
 
        for (int i = 0; i < num_tokens - 1; i++) {
            int merge_idx = ck_tokenizer_lookup_merge(tok, tokens[i], tokens[i + 1]);
            if (merge_idx >= 0 && tok->merges[merge_idx].priority < best_priority) {
                best_pos = i;
                best_priority = tok->merges[merge_idx].priority;
            }
        }
 
        if (best_pos >= 0) {
            int merge_idx = ck_tokenizer_lookup_merge(tok, tokens[best_pos], tokens[best_pos + 1]);
            tokens[best_pos] = tok->merges[merge_idx].merged;
 
            /* Shift remaining tokens */
            for (int i = best_pos + 1; i < num_tokens - 1; i++) {
                tokens[i] = tokens[i + 1];
            }
            num_tokens--;
            changed = true;
        }
    }
 
    /* Copy to output */
    int out_len = 0;
 
    if (tok->add_bos && out_len < max_ids) {
        ids[out_len++] = tok->bos_id;
    }
 
    for (int i = 0; i < num_tokens && out_len < max_ids; i++) {
        ids[out_len++] = tokens[i];
    }
 
    if (tok->add_eos && out_len < max_ids) {
        ids[out_len++] = tok->eos_id;
    }
 
    free(tokens);
    return out_len;
}

References CKTokenizerConfig::add_bos, CKTokenizerConfig::add_eos, CKTokenizer::bos_id, CK_SPM_MODE_LLAMA, CK_TOKENIZER_BPE, ck_tokenizer_detect_space_prefix_style(), ck_tokenizer_encode_spm_impl(), ck_tokenizer_encode_spm_llama_impl(), CK_TOKENIZER_SPM, CKTokenizer::config, config, CKTokenizer::eos_id, find_longest_match(), id, ids, max_ids, preprocess_bpe_spaces(), CKTokenizerConfig::spm_mode, style, text, text_len, CKTokenizerConfig::type, and CKTokenizer::unk_id.

ck_tokenizer_encode_tokens()

int ck_tokenizer_encode_tokens	(	const CKTokenizer *	tok,
		const char *	text,
		int	text_len,
		const char **	out_tokens,
		int	max_tokens
	)

Encode and return tokens as array of strings.

Parameters

tok	Tokenizer
text	Input text
text_len	Text length
out_tokens	Output token strings (caller must free each)
max_tokens	Maximum tokens

Returns

Number of tokens written

ck_tokenizer_encode_with_special()

int ck_tokenizer_encode_with_special	(	CKTokenizer *	tok,
		const char *	text,
		int	text_len,
		int32_t *	ids,
		int	max_ids,
		bool	add_special
	)

Encode with special token handling.

Parameters

tok	Tokenizer
text	Input text
text_len	Text length, or -1 for null-terminated
ids	Output token IDs
max_ids	Maximum IDs to write
add_special	Add BOS/EOS tokens

Returns

Number of tokens written

ck_tokenizer_load_binary()

int ck_tokenizer_load_binary	(	CKTokenizer *	tok,
		int	vocab_size,
		const int32_t *	offsets,
		const char *	strings,
		int	num_merges,
		const int32_t *	merges
	)

Load vocabulary from memory-mapped binary data.

Parameters

tok	Tokenizer
vocab_size	Number of tokens
offsets	Array of offsets into strings pool
strings	String pool containing null-terminated tokens
num_merges	Number of BPE merges
merges	Merge rules as (left, right, merged) triplets

Returns

0 on success, -1 on error

Definition at line 18 of file ck_tokenizer_v2.c.

                                                    {
    if (!tok || !offsets || !strings) return -1;
 
    // We assume ck_tokenizer_init was already called to alloc hash tables
    tok->vocab_size = 0;
    
    for (int i = 0; i < vocab_size; i++) {
        const char *token = strings + offsets[i];
        int len = (int)strlen(token);
        
        CKVocabEntry *entry = (CKVocabEntry *)ck_pool_alloc(&tok->pool, sizeof(CKVocabEntry));
        entry->token = (char *)token; 
        entry->token_len = len;
        entry->id = i;
 
        uint32_t bucket = hash_string(token, len) % tok->vocab_hash_size;
        entry->next = tok->vocab_hash[bucket];
        tok->vocab_hash[bucket] = entry;
 
        tok->id_to_token[i] = entry->token;
        tok->vocab_size++;
    }
 
    if (merges && num_merges > 0) {
        for (int i = 0; i < num_merges; i++) {
            int32_t left = merges[i*3 + 0];
            int32_t right = merges[i*3 + 1];
            int32_t merged = merges[i*3 + 2];
            ck_tokenizer_add_merge(tok, left, right, merged);
        }
    }
 
    return 0;
}

References ck_pool_alloc(), ck_tokenizer_add_merge(), ck_tokenizer_load_binary_with_scores(), hash_string(), CKVocabEntry::id, CKTokenizer::id_to_token, left, merges, CKVocabEntry::next, num_merges, offsets, CKTokenizer::pool, right, strings, CKVocabEntry::token, token, CKVocabEntry::token_len, CKTokenizer::vocab_hash, CKTokenizer::vocab_hash_size, CKTokenizer::vocab_size, and vocab_size.

Referenced by main().

ck_tokenizer_load_binary_with_scores()

int ck_tokenizer_load_binary_with_scores	(	CKTokenizer *	tok,
		int	vocab_size,
		const int32_t *	offsets,
		const char *	strings,
		const float *	scores,
		const uint8_t *	types,
		int	num_merges,
		const int32_t *	merges
	)

Load vocabulary from memory-mapped binary data with scores and types.

This extended version supports SPM (SentencePiece) tokenizers which require token scores for Viterbi/DP encoding.

Parameters

tok	Tokenizer
vocab_size	Number of tokens
offsets	Array of offsets into strings pool
strings	String pool containing null-terminated tokens
scores	Array of token scores (float32), can be NULL
types	Array of token types (uint8), can be NULL
num_merges	Number of BPE merges
merges	Merge rules as (left, right, merged) triplets

Returns

0 on success, -1 on error

Definition at line 1252 of file tokenizer.c.

                                                                {
    if (!tok || !offsets || !strings) return -1;
    ck_tokenizer_reset(tok);
 
    /* Free any existing scores/types arrays before reallocating */
    if (tok->scores) {
        free(tok->scores);
        tok->scores = NULL;
        tok->scores_size = 0;
    }
    if (tok->types) {
        free(tok->types);
        tok->types = NULL;
        tok->types_size = 0;
    }
 
    /* Allocate scores and types arrays if provided */
    if (scores && vocab_size > 0) {
        tok->scores = (float *)malloc(vocab_size * sizeof(float));
        if (!tok->scores) return -1;
        memcpy(tok->scores, scores, vocab_size * sizeof(float));
        tok->scores_size = (size_t)vocab_size;
    }
    if (types && vocab_size > 0) {
        tok->types = (uint8_t *)malloc(vocab_size * sizeof(uint8_t));
        if (!tok->types) {
            if (tok->scores) {
                free(tok->scores);
                tok->scores = NULL;
            }
            return -1;
        }
        memcpy(tok->types, types, vocab_size * sizeof(uint8_t));
        tok->types_size = (size_t)vocab_size;
    }
 
    for (int i = 0; i < vocab_size; i++) {
        const char *token = strings + offsets[i];
        float score = scores ? scores[i] : 0.0f;
        ck_tokenizer_add_token(tok, token, i, score);
    }
 
    /* Build byte token lookup table if types are available */
    if (types && vocab_size > 0) {
        spm_build_byte_lookup(tok, strings, offsets, vocab_size);
 
        /* Log token type statistics */
        int count_normal = 0, count_unknown = 0, count_control = 0, count_byte = 0, count_other = 0;
        int max_type = 0;
        for (int i = 0; i < vocab_size; i++) {
            uint8_t t = tok->types[i];
            if (t > max_type) max_type = t;
            switch (t) {
                case GGUF_TOKEN_NORMAL: count_normal++; break;
                case GGUF_TOKEN_UNKNOWN: count_unknown++; break;
                case GGUF_TOKEN_CONTROL: count_control++; break;
                case GGUF_TOKEN_BYTE: count_byte++; break;
                default: count_other++; break;
            }
        }
        fprintf(stderr, "[TOKENIZER] Loaded %d tokens: normal=%d, unknown=%d, control=%d, byte=%d, other=%d\n",
                vocab_size, count_normal, count_unknown, count_control, count_byte, count_other);
        if (max_type > GGUF_TOKEN_BYTE) {
            fprintf(stderr, "[TOKENIZER] Warning: Unexpected token type %d\n", max_type);
        }
    }
 
    /* TODO: Merges */
    (void)num_merges; (void)merges;
    return 0;
}

References ck_tokenizer_add_token(), ck_tokenizer_reset(), GGUF_TOKEN_BYTE, GGUF_TOKEN_CONTROL, GGUF_TOKEN_NORMAL, GGUF_TOKEN_UNKNOWN, merges, num_merges, offsets, score, CKTokenizer::scores, CKTokenizer::scores_size, spm_build_byte_lookup(), strings, token, CKTokenizer::types, CKTokenizer::types_size, and vocab_size.

Referenced by ck_tokenizer_load_binary().

ck_tokenizer_load_gguf()

int ck_tokenizer_load_gguf	(	CKTokenizer *	tok,
		const char *	path
	)

Load vocabulary from GGUF file.

Parameters

tok	Tokenizer
path	Path to GGUF file

Returns

0 on success, -1 on error

Definition at line 1332 of file tokenizer.c.

{ (void)tok; (void)path; return -1; }

ck_tokenizer_load_json()

int ck_tokenizer_load_json	(	CKTokenizer *	tok,
		const char *	path
	)

Load vocabulary from JSON file (HuggingFace format).

Parameters

tok	Tokenizer
path	Path to vocab.json or tokenizer.json

Returns

0 on success, -1 on error

Definition at line 1333 of file tokenizer.c.

{ (void)tok; (void)path; return -1; }

ck_tokenizer_load_merges()

int ck_tokenizer_load_merges	(	CKTokenizer *	tok,
		const char *	path
	)

Load BPE merges from text file.

Format: token1 token2 (one merge per line)

Parameters

tok	Tokenizer
path	Path to merges.txt

Returns

0 on success, -1 on error

Definition at line 1335 of file tokenizer.c.

{ (void)tok; (void)path; return -1; }

ck_tokenizer_load_text()

int ck_tokenizer_load_text	(	CKTokenizer *	tok,
		const char *	path
	)

Load vocabulary from text file (one token per line).

Format: token_string [id] [score] Lines starting with # are comments.

Parameters

tok	Tokenizer
path	Path to vocabulary file

Returns

0 on success, -1 on error

Definition at line 1334 of file tokenizer.c.

{ (void)tok; (void)path; return -1; }

ck_tokenizer_vocab_size()

static size_t ck_tokenizer_vocab_size ( const CKTokenizer *  tok )

inlinestatic

Get vocabulary size.

Definition at line 332 of file tokenizer.h.

                                                                     {
    return tok ? tok->vocab_size : 0;
}

References CKTokenizer::vocab_size.

Variable Documentation

add_bos

bool add_bos

Definition at line 242 of file tokenizer.h.

Referenced by ck_tokenizer_set_add_bos_eos().

add_eos

bool bool add_eos

Definition at line 242 of file tokenizer.h.

Referenced by ck_tokenizer_set_add_bos_eos().

add_space_prefix

bool add_space_prefix

Definition at line 252 of file tokenizer.h.

Referenced by ck_tokenizer_set_add_space_prefix(), preprocess_spm_llama_text(), and preprocess_spm_text().

bos

int32_t int32_t bos

Definition at line 230 of file tokenizer.h.

Referenced by ck_tokenizer_set_special_ids(), ck_true_bpe_set_special_ids(), and load_eos_from_vocab_json().

eos

int32_t int32_t int32_t eos

Definition at line 231 of file tokenizer.h.

Referenced by ck_tokenizer_set_special_ids(), ck_true_bpe_set_special_ids(), and load_eos_from_vocab_json().

id

int32_t id

Definition at line 315 of file tokenizer.h.

Referenced by ck_tokenizer_add_special_token(), ck_tokenizer_add_token(), ck_tokenizer_encode(), ck_tokenizer_id_to_token(), ck_tokenizer_load(), ck_tokenizer_lookup_exact_n(), ck_true_bpe_add_special_token(), ck_true_bpe_add_token(), ck_true_bpe_id_to_token(), encode_chunk(), quantize_row_q8_0(), token_list_append(), and topology_discover_cpu().

ids

const int32_t * ids

Definition at line 443 of file tokenizer.h.

Referenced by ck_tokenizer_decode(), ck_tokenizer_encode(), ck_tokenizer_encode_spm_impl(), ck_tokenizer_encode_spm_llama_impl(), ck_true_bpe_decode(), ck_true_bpe_encode(), encode_chunk(), encode_text_segment(), main(), run_prompt(), spm_encode_byte_fallback(), and spm_llama_resegment_node().

lowercase

bool lowercase

Definition at line 268 of file tokenizer.h.

mask

int32_t int32_t int32_t int32_t int32_t mask

Definition at line 233 of file tokenizer.h.

Referenced by ck_dtype_supported(), ck_tokenizer_set_special_ids(), relu_backward(), and topology_discover_affinity().

num_ids

const int32_t int num_ids

Definition at line 444 of file tokenizer.h.

Referenced by ck_tokenizer_decode(), ck_true_bpe_decode(), and main().

out_len

const int32_t int int* out_len

Definition at line 445 of file tokenizer.h.

Referenced by ck_tokenizer_encode(), ck_utf8_next_char(), decode_bpe_token(), eos_pattern_process(), find_object_range(), preprocess_bpe_spaces(), preprocess_spm_llama_text(), preprocess_spm_text(), preprocess_text(), and run_prompt().

pad

int32_t int32_t int32_t int32_t pad

Definition at line 232 of file tokenizer.h.

Referenced by ck_tokenizer_set_special_ids(), and ck_true_bpe_set_special_ids().

score

int32_t float* score

Definition at line 327 of file tokenizer.h.

Referenced by attention_flash_decode_scalar(), attention_flash_query_causal(), attention_flash_query_sliding(), attention_mlp_fused_fp32(), attention_mlp_fused_q4k(), attention_mlp_separate_fp32(), ck_tokenizer_add_token(), ck_tokenizer_encode_spm_llama_impl(), ck_tokenizer_load_binary_with_scores(), ck_true_bpe_add_token(), layer_fused_attn_mlp_qkv_q4k(), and simple_attention().

spm_mode

CKSpmMode spm_mode

Definition at line 260 of file tokenizer.h.

Referenced by ck_tokenizer_set_spm_mode().

style

CKSpacePrefixStyle style

Definition at line 287 of file tokenizer.h.

Referenced by ck_tokenizer_encode(), ck_tokenizer_set_space_prefix_style(), preprocess_bpe_spaces(), and preprocess_text().

text

const int32_t int char * text

Definition at line 563 of file tokenizer.h.

Referenced by ck_tokenizer_decode(), ck_tokenizer_encode(), ck_tokenizer_encode_spm_impl(), ck_tokenizer_encode_spm_llama_impl(), ck_tokenizer_lookup_exact_n(), ck_trie_find_longest(), ck_trie_has_prefix(), ck_true_bpe_decode(), ck_true_bpe_encode(), encode_text_segment(), find_longest_match(), find_longest_match_hash(), find_longest_match_trie(), gpt2_pretokenize(), init_tokens_from_text(), main(), match_special_token(), output_append(), preprocess_bpe_spaces(), preprocess_spm_llama_text(), preprocess_spm_text(), preprocess_text(), spm_count_unknown_run(), spm_encode_byte_fallback(), spm_find_candidates_at_pos(), and tokenize().

token

const char * token

Definition at line 306 of file tokenizer.h.

Referenced by ck_model_decode(), ck_tokenizer_add_token(), ck_tokenizer_decode(), ck_tokenizer_detect_space_prefix_style(), ck_tokenizer_encode_spm_impl(), ck_tokenizer_load(), ck_tokenizer_load_binary(), ck_tokenizer_load_binary_with_scores(), ck_tokenizer_lookup(), ck_tokenizer_lookup_exact(), ck_trie_insert(), ck_true_bpe_add_special_token(), ck_true_bpe_add_token(), ck_true_bpe_decode(), ck_true_bpe_detect_space_style(), ck_true_bpe_load_binary(), ck_true_bpe_lookup(), decode_bpe_token(), eos_is_potential_prefix(), is_eos_token(), main(), model_decode(), model_decode_token(), output_token(), qwen2_0_5b_decode_decode(), qwen2_0_5b_decode_decode_token(), run_benchmark(), run_generation_test(), run_inference(), spm_build_byte_lookup(), spm_token_is_byte_format(), and topology_discover_numa().

unk

int32_t unk

Definition at line 229 of file tokenizer.h.

Referenced by ck_tokenizer_set_special_ids(), and ck_true_bpe_set_special_ids().

use_trie

bool use_trie

Definition at line 276 of file tokenizer.h.

Referenced by ck_tokenizer_set_use_trie().