ck_tokenizer.h File Reference

#include <stddef.h>
#include <stdint.h>
#include <stdbool.h>

Go to the source code of this file.

Data Structures
struct	CKMemPool
struct	CKMergeRule
struct	CKPoolBlock
struct	CKTokenizer
struct	CKVocabEntry

Macros
#define	CK_MAX_TOKEN_LEN   256
#define	CK_MAX_VOCAB_SIZE   256000
#define	CK_POOL_BLOCK_SIZE   (1024 * 1024) /* 1MB */

Functions
void *	ck_pool_alloc (CKMemPool *pool, size_t size)
void	ck_pool_free (CKMemPool *pool)
void	ck_pool_init (CKMemPool *pool)
char *	ck_pool_strdup (CKMemPool *pool, const char *s, int len)
int	ck_tokenizer_add_merge (CKTokenizer *tok, int32_t left, int32_t right, int32_t merged)
int32_t	ck_tokenizer_add_token (CKTokenizer *tok, const char *token, int len)
int	ck_tokenizer_decode (const CKTokenizer *tok, const int32_t *ids, int num_ids, char *text, int max_len)
int	ck_tokenizer_encode (const CKTokenizer *tok, const char *text, int text_len, int32_t *ids, int max_ids)
void	ck_tokenizer_free (CKTokenizer *tok)
const char *	ck_tokenizer_id_to_token (const CKTokenizer *tok, int32_t id)
int	ck_tokenizer_init (CKTokenizer *tok)
int	ck_tokenizer_load (CKTokenizer *tok, const char *path)
int32_t	ck_tokenizer_lookup (const CKTokenizer *tok, const char *token, int len)
int	ck_tokenizer_lookup_merge (const CKTokenizer *tok, int32_t left, int32_t right)
static int	ck_tokenizer_vocab_size (const CKTokenizer *tok)

Macro Definition Documentation

CK_MAX_TOKEN_LEN

#define CK_MAX_TOKEN_LEN   256

Definition at line 25 of file ck_tokenizer.h.

CK_MAX_VOCAB_SIZE

#define CK_MAX_VOCAB_SIZE   256000

Definition at line 28 of file ck_tokenizer.h.

CK_POOL_BLOCK_SIZE

#define CK_POOL_BLOCK_SIZE   (1024 * 1024) /* 1MB */

Definition at line 31 of file ck_tokenizer.h.

Function Documentation

ck_pool_alloc()

void * ck_pool_alloc	(	CKMemPool *	pool,
		size_t	size
	)

Definition at line 69 of file ck_tokenizer.c.

                                                  {
    /* Align to 8 bytes */
    size = (size + 7) & ~7;
 
    /* Check if current block has space */
    if (pool->current && pool->current->used + size <= pool->current->capacity) {
        void *ptr = pool->current->data + pool->current->used;
        pool->current->used += size;
        pool->total_allocated += size;
        return ptr;
    }
 
    /* Need new block */
    size_t block_size = CK_POOL_BLOCK_SIZE;
    if (size > block_size) block_size = size;
 
    CKPoolBlock *block = pool_new_block(block_size);
    if (!block) return NULL;
 
    block->next = pool->head;
    pool->head = block;
    pool->current = block;
 
    void *ptr = block->data;
    block->used = size;
    pool->total_allocated += size;
    return ptr;
}

References CK_POOL_BLOCK_SIZE, CKMemPool::current, CKPoolBlock::data, CKMemPool::head, CKPoolBlock::next, pool_new_block(), CKMemPool::total_allocated, and CKPoolBlock::used.

Referenced by ck_pool_strdup(), ck_tokenizer_add_token(), ck_tokenizer_load(), and ck_tokenizer_load_binary().

ck_pool_free()

void ck_pool_free

(

CKMemPool *

pool

)

Definition at line 107 of file ck_tokenizer.c.

                                   {
    CKPoolBlock *block = pool->head;
    while (block) {
        CKPoolBlock *next = block->next;
        free(block->data);
        free(block);
        block = next;
    }
    memset(pool, 0, sizeof(*pool));
}

References CKPoolBlock::data, CKMemPool::head, and CKPoolBlock::next.

Referenced by ck_tokenizer_free().

ck_pool_init()

void ck_pool_init

(

CKMemPool *

pool

)

Definition at line 51 of file ck_tokenizer.c.

                                   {
    memset(pool, 0, sizeof(*pool));
}

Referenced by ck_tokenizer_init().

ck_pool_strdup()

char * ck_pool_strdup	(	CKMemPool *	pool,
		const char *	s,
		int	len
	)

Definition at line 98 of file ck_tokenizer.c.

                                                              {
    if (len < 0) len = (int)strlen(s);
    char *copy = (char *)ck_pool_alloc(pool, len + 1);
    if (!copy) return NULL;
    memcpy(copy, s, len);
    copy[len] = '\0';
    return copy;
}

References ck_pool_alloc().

Referenced by ck_tokenizer_add_token(), and ck_tokenizer_load().

ck_tokenizer_add_merge()

int ck_tokenizer_add_merge	(	CKTokenizer *	tok,
		int32_t	left,
		int32_t	right,
		int32_t	merged
	)

Definition at line 248 of file ck_tokenizer.c.

                                                                                          {
    int idx = tok->num_merges;
 
    /* Grow merges array if needed */
    if (idx % 4096 == 0) {
        size_t new_cap = (idx + 4096) * sizeof(CKMergeRule);
        CKMergeRule *new_merges = (CKMergeRule *)realloc(tok->merges, new_cap);
        if (!new_merges) return -1;
        tok->merges = new_merges;
    }
 
    tok->merges[idx].left = left;
    tok->merges[idx].right = right;
    tok->merges[idx].merged = merged;
    tok->merges[idx].priority = idx;  /* Earlier = higher priority */
 
    /* Add to hash table */
    uint32_t bucket = hash_pair(left, right) % tok->merge_hash_size;
    /* Linear probing */
    while (tok->merge_hash[bucket] >= 0) {
        bucket = (bucket + 1) % tok->merge_hash_size;
    }
    tok->merge_hash[bucket] = idx;
 
    tok->num_merges++;
    return 0;
}

References hash_pair(), CKMergeRule::left, left, CKTokenizer::merge_hash, CKTokenizer::merge_hash_size, CKMergeRule::merged, CKTokenizer::merges, CKTokenizer::num_merges, CKMergeRule::priority, CKMergeRule::right, and right.

Referenced by ck_tokenizer_load(), and ck_tokenizer_load_binary().

ck_tokenizer_add_token()

int32_t ck_tokenizer_add_token	(	CKTokenizer *	tok,
		const char *	token,
		int	len
	)

Definition at line 196 of file ck_tokenizer.c.

                                                                             {
    if (len < 0) len = (int)strlen(token);
    if (tok->vocab_size >= CK_MAX_VOCAB_SIZE) return -1;
 
    /* Check if already exists */
    int32_t existing = ck_tokenizer_lookup(tok, token, len);
    if (existing != tok->unk_id || (len == 0)) {
        return existing;
    }
 
    /* Create new entry */
    CKVocabEntry *entry = (CKVocabEntry *)ck_pool_alloc(&tok->pool, sizeof(CKVocabEntry));
    if (!entry) return -1;
 
    entry->token = ck_pool_strdup(&tok->pool, token, len);
    if (!entry->token) return -1;
    entry->token_len = len;
    entry->id = tok->vocab_size;
 
    /* Add to hash table */
    uint32_t bucket = hash_string(token, len) % tok->vocab_hash_size;
    entry->next = tok->vocab_hash[bucket];
    tok->vocab_hash[bucket] = entry;
 
    /* Add to reverse lookup */
    tok->id_to_token[tok->vocab_size] = entry->token;
 
    tok->vocab_size++;
    return entry->id;
}

References CK_MAX_VOCAB_SIZE, ck_pool_alloc(), ck_pool_strdup(), ck_tokenizer_lookup(), hash_string(), CKVocabEntry::id, CKTokenizer::id_to_token, CKVocabEntry::next, CKTokenizer::pool, CKVocabEntry::token, token, CKVocabEntry::token_len, CKTokenizer::unk_id, CKTokenizer::vocab_hash, CKTokenizer::vocab_hash_size, and CKTokenizer::vocab_size.

Referenced by ck_tokenizer_load(), ck_tokenizer_load_binary_with_scores(), and main().

ck_tokenizer_decode()

int ck_tokenizer_decode	(	const CKTokenizer *	tok,
		const int32_t *	ids,
		int	num_ids,
		char *	text,
		int	max_len
	)

Definition at line 734 of file ck_tokenizer.c.

                                     {
    int len = 0;
 
    for (int i = 0; i < num_ids; i++) {
        /* Skip special tokens */
        if (ids[i] == tok->bos_id || ids[i] == tok->eos_id || ids[i] == tok->pad_id) {
            continue;
        }
 
        const char *token = ck_tokenizer_id_to_token(tok, ids[i]);
        if (!token) continue;
 
        int token_len = (int)strlen(token);
 
        /* Handle byte tokens <0xXX> */
        if (token_len == 6 && token[0] == '<' && token[1] == '0' && token[2] == 'x') {
            char hex[3] = {token[3], token[4], 0};
            unsigned int byte = (unsigned int)strtol(hex, NULL, 16);
            if (len < max_len - 1) {
                text[len++] = (char)byte;
            }
            continue;
        }
 
        /* Handle tokenizer space prefixes:
         *   GPT/byte-level BPE: Ġ (0xC4 0xA0)
         *   SentencePiece BPE: ▁ (0xE2 0x96 0x81)
         */
        const char *src = token;
        if ((unsigned char)token[0] == 0xC4 && (unsigned char)token[1] == 0xA0) {
            if (len < max_len - 1) {
                text[len++] = ' ';
            }
            src = token + 2;
            token_len -= 2;
        } else if ((unsigned char)token[0] == 0xE2 &&
                   (unsigned char)token[1] == 0x96 &&
                   (unsigned char)token[2] == 0x81) {
            if (len < max_len - 1) {
                text[len++] = ' ';
            }
            src = token + 3;
            token_len -= 3;
        }
 
        /* Copy token, normalizing any embedded SentencePiece markers too.
         * Some Gemma tokenizer pieces contain repeated ▁ markers after
         * indentation or punctuation; those are word-boundary spaces, not
         * literal output characters.
         */
        for (int j = 0; j < token_len && len < max_len - 1; ) {
            if (j + 2 < token_len &&
                (unsigned char)src[j] == 0xE2 &&
                (unsigned char)src[j + 1] == 0x96 &&
                (unsigned char)src[j + 2] == 0x81) {
                text[len++] = ' ';
                j += 3;
                continue;
            }
            text[len++] = src[j++];
        }
    }
 
    text[len] = '\0';
    return len;
}

References CKTokenizer::bos_id, ck_tokenizer_id_to_token(), CKTokenizer::eos_id, ids, max_len, num_ids, CKTokenizer::pad_id, text, and token.

Referenced by main().

ck_tokenizer_encode()

int ck_tokenizer_encode	(	const CKTokenizer *	tok,
		const char *	text,
		int	text_len,
		int32_t *	ids,
		int	max_ids
	)

Definition at line 635 of file ck_tokenizer.c.

                                     {
    if (text_len < 0) text_len = (int)strlen(text);
 
    /* Pre-tokenize: split on whitespace, keep spaces as tokens */
    /* For simplicity, treat each byte as initial token, then apply BPE */
 
    /* Initial tokens: one per byte */
    int32_t *tokens = (int32_t *)malloc(text_len * sizeof(int32_t));
    int num_tokens = 0;
 
    for (int i = 0; i < text_len; i++) {
        /* Look up single-character token */
        char c[2] = {text[i], '\0'};
        int32_t id = ck_tokenizer_lookup(tok, c, 1);
 
        /* Handle special byte tokens like <0xXX> */
        if (id == tok->unk_id) {
            char byte_token[8];
            snprintf(byte_token, sizeof(byte_token), "<0x%02X>", (unsigned char)text[i]);
            id = ck_tokenizer_lookup(tok, byte_token, -1);
        }
 
        /* Try UTF-8 multi-byte sequences */
        if (id == tok->unk_id && (unsigned char)text[i] >= 0x80) {
            int utf8_len = 1;
            if ((text[i] & 0xE0) == 0xC0) utf8_len = 2;
            else if ((text[i] & 0xF0) == 0xE0) utf8_len = 3;
            else if ((text[i] & 0xF8) == 0xF0) utf8_len = 4;
 
            if (i + utf8_len <= text_len) {
                id = ck_tokenizer_lookup(tok, text + i, utf8_len);
                if (id != tok->unk_id) {
                    tokens[num_tokens++] = id;
                    i += utf8_len - 1;
                    continue;
                }
            }
        }
 
        tokens[num_tokens++] = id;
    }
 
    /* Apply BPE merges iteratively */
    bool changed = true;
    while (changed && num_tokens > 1) {
        changed = false;
 
        /* Find best merge (lowest priority = earliest in merge list) */
        int best_pos = -1;
        int best_priority = tok->num_merges;
 
        for (int i = 0; i < num_tokens - 1; i++) {
            int merge_idx = ck_tokenizer_lookup_merge(tok, tokens[i], tokens[i + 1]);
            if (merge_idx >= 0 && tok->merges[merge_idx].priority < best_priority) {
                best_pos = i;
                best_priority = tok->merges[merge_idx].priority;
            }
        }
 
        if (best_pos >= 0) {
            int merge_idx = ck_tokenizer_lookup_merge(tok, tokens[best_pos], tokens[best_pos + 1]);
            tokens[best_pos] = tok->merges[merge_idx].merged;
 
            /* Shift remaining tokens */
            for (int i = best_pos + 1; i < num_tokens - 1; i++) {
                tokens[i] = tokens[i + 1];
            }
            num_tokens--;
            changed = true;
        }
    }
 
    /* Copy to output */
    int out_len = 0;
 
    if (tok->add_bos && out_len < max_ids) {
        ids[out_len++] = tok->bos_id;
    }
 
    for (int i = 0; i < num_tokens && out_len < max_ids; i++) {
        ids[out_len++] = tokens[i];
    }
 
    if (tok->add_eos && out_len < max_ids) {
        ids[out_len++] = tok->eos_id;
    }
 
    free(tokens);
    return out_len;
}

References CKTokenizer::add_bos, CKTokenizer::add_eos, CKTokenizer::bos_id, ck_tokenizer_lookup(), ck_tokenizer_lookup_merge(), CKTokenizer::eos_id, id, ids, max_ids, CKMergeRule::merged, CKTokenizer::merges, CKTokenizer::num_merges, out_len, CKMergeRule::priority, text, text_len, CKTokenizer::unk_id, and utf8_len().

Referenced by main(), and run_inference().

ck_tokenizer_free()

void ck_tokenizer_free

(

CKTokenizer *

tok

)

Definition at line 183 of file ck_tokenizer.c.

                                         {
    ck_pool_free(&tok->pool);
    free(tok->vocab_hash);
    free(tok->id_to_token);
    free(tok->merges);
    free(tok->merge_hash);
    memset(tok, 0, sizeof(*tok));
}

References CKTokenizer::byte_token_id, ck_pool_free(), ck_tokenizer_hash_table_free(), ck_tokenizer_mempool_free(), ck_trie_free(), CKTokenizer::id_to_token, CKTokenizer::merge_hash, CKTokenizer::merges, CKTokenizer::pool, CKTokenizer::scores, CKTokenizer::types, CKTokenizer::vocab, CKTokenizer::vocab_hash, CKTokenizer::vocab_size, and CKTokenizer::vocab_trie.

Referenced by main(), run_inference(), run_inference(), and run_inference().

ck_tokenizer_id_to_token()

const char * ck_tokenizer_id_to_token	(	const CKTokenizer *	tok,
		int32_t	id
	)

Definition at line 239 of file ck_tokenizer.c.

                                                                         {
    if (id < 0 || id >= tok->vocab_size) return NULL;
    return tok->id_to_token[id];
}

References id, CKTokenizer::id_to_token, and CKTokenizer::vocab_size.

Referenced by ck_tokenizer_decode(), ck_tokenizer_encode_spm_impl(), main(), and run_inference().

ck_tokenizer_init()

int ck_tokenizer_init

(

CKTokenizer *

tok

)

Definition at line 148 of file ck_tokenizer.c.

                                        {
    memset(tok, 0, sizeof(*tok));
    ck_pool_init(&tok->pool);
 
    /* Default special tokens */
    tok->unk_id = 0;
    tok->bos_id = 1;
    tok->eos_id = 2;
    tok->pad_id = 3;
 
    /* Allocate vocab hash table */
    tok->vocab_hash_size = 65536;  /* 64K buckets */
    tok->vocab_hash = (CKVocabEntry **)calloc(tok->vocab_hash_size, sizeof(CKVocabEntry *));
    if (!tok->vocab_hash) return -1;
 
    /* Allocate reverse vocab */
    tok->id_to_token = (char **)calloc(CK_MAX_VOCAB_SIZE, sizeof(char *));
    if (!tok->id_to_token) {
        free(tok->vocab_hash);
        return -1;
    }
 
    /* Allocate merge hash table */
    tok->merge_hash_size = 262144;  /* 256K buckets */
    tok->merge_hash = (int *)malloc(tok->merge_hash_size * sizeof(int));
    if (!tok->merge_hash) {
        free(tok->vocab_hash);
        free(tok->id_to_token);
        return -1;
    }
    memset(tok->merge_hash, -1, tok->merge_hash_size * sizeof(int));
 
    return 0;
}

References CKTokenizer::bos_id, CK_MAX_VOCAB_SIZE, ck_pool_init(), CKTokenizer::eos_id, CKTokenizer::id_to_token, CKTokenizer::merge_hash, CKTokenizer::merge_hash_size, CKTokenizer::pad_id, CKTokenizer::pool, CKTokenizer::unk_id, CKTokenizer::vocab_hash, and CKTokenizer::vocab_hash_size.

Referenced by run_inference().

ck_tokenizer_load()

int ck_tokenizer_load	(	CKTokenizer *	tok,
		const char *	path
	)

Definition at line 432 of file ck_tokenizer.c.

                                                          {
    FILE *f = fopen(path, "rb");
    if (!f) {
        fprintf(stderr, "Failed to open tokenizer: %s\n", path);
        return -1;
    }
 
    fseek(f, 0, SEEK_END);
    long size = ftell(f);
    fseek(f, 0, SEEK_SET);
 
    char *data = (char *)malloc(size + 1);
    if (!data) {
        fclose(f);
        return -1;
    }
    fread(data, 1, size, f);
    data[size] = '\0';
    fclose(f);
 
    JSONParser parser = {data, data, data + size};
    JSONParser *p = &parser;
 
    /* Parse top-level object */
    if (!json_match_char(p, '{')) {
        free(data);
        return -1;
    }
 
    char key[256];
    while (p->pos < p->end && *p->pos != '}') {
        if (json_parse_string(p, key, sizeof(key)) < 0) break;
        if (!json_match_char(p, ':')) break;
 
        if (strcmp(key, "model") == 0) {
            /* Parse model object */
            if (!json_match_char(p, '{')) {
                json_skip_value(p);
                json_match_char(p, ',');
                continue;
            }
 
            while (p->pos < p->end && *p->pos != '}') {
                if (json_parse_string(p, key, sizeof(key)) < 0) break;
                if (!json_match_char(p, ':')) break;
 
                if (strcmp(key, "vocab") == 0) {
                    /* Parse vocab object: {"token": id, ...} */
                    if (!json_match_char(p, '{')) {
                        json_skip_value(p);
                        json_match_char(p, ',');
                        continue;
                    }
 
                    char token[CK_MAX_TOKEN_LEN];
                    while (p->pos < p->end && *p->pos != '}') {
                        int token_len = json_parse_string(p, token, sizeof(token));
                        if (token_len < 0) break;
                        if (!json_match_char(p, ':')) break;
 
                        int id;
                        if (json_parse_int(p, &id) < 0) break;
 
                        /* Ensure we have space up to this ID */
                        while (tok->vocab_size <= id) {
                            ck_tokenizer_add_token(tok, "", 0);
                        }
 
                        /* Add/update token */
                        uint32_t bucket = hash_string(token, token_len) % tok->vocab_hash_size;
                        CKVocabEntry *entry = (CKVocabEntry *)ck_pool_alloc(&tok->pool, sizeof(CKVocabEntry));
                        entry->token = ck_pool_strdup(&tok->pool, token, token_len);
                        entry->token_len = token_len;
                        entry->id = id;
                        entry->next = tok->vocab_hash[bucket];
                        tok->vocab_hash[bucket] = entry;
                        tok->id_to_token[id] = entry->token;
                        if (id >= tok->vocab_size) tok->vocab_size = id + 1;
 
                        json_match_char(p, ',');
                    }
                    json_match_char(p, '}');
 
                } else if (strcmp(key, "merges") == 0) {
                    /* Parse merges array: ["tok1 tok2", ...] */
                    if (!json_match_char(p, '[')) {
                        json_skip_value(p);
                        json_match_char(p, ',');
                        continue;
                    }
 
                    char merge_str[512];
                    while (p->pos < p->end && *p->pos != ']') {
                        int merge_len = json_parse_string(p, merge_str, sizeof(merge_str));
                        if (merge_len < 0) break;
 
                        /* Parse "token1 token2" */
                        char *space = strchr(merge_str, ' ');
                        if (space) {
                            *space = '\0';
                            char *tok1 = merge_str;
                            char *tok2 = space + 1;
 
                            int32_t id1 = ck_tokenizer_lookup(tok, tok1, -1);
                            int32_t id2 = ck_tokenizer_lookup(tok, tok2, -1);
 
                            /* Create merged token */
                            char merged[512];
                            snprintf(merged, sizeof(merged), "%s%s", tok1, tok2);
                            int32_t merged_id = ck_tokenizer_lookup(tok, merged, -1);
 
                            if (merged_id == tok->unk_id) {
                                merged_id = ck_tokenizer_add_token(tok, merged, -1);
                            }
 
                            ck_tokenizer_add_merge(tok, id1, id2, merged_id);
                        }
 
                        json_match_char(p, ',');
                    }
                    json_match_char(p, ']');
 
                } else {
                    json_skip_value(p);
                }
 
                json_match_char(p, ',');
            }
            json_match_char(p, '}');
 
        } else if (strcmp(key, "added_tokens") == 0) {
            /* Parse added_tokens array for special tokens */
            if (!json_match_char(p, '[')) {
                json_skip_value(p);
                json_match_char(p, ',');
                continue;
            }
 
            while (p->pos < p->end && *p->pos != ']') {
                if (!json_match_char(p, '{')) {
                    json_skip_value(p);
                    json_match_char(p, ',');
                    continue;
                }
 
                char content[256] = "";
                int id = -1;
                while (p->pos < p->end && *p->pos != '}') {
                    if (json_parse_string(p, key, sizeof(key)) < 0) break;
                    if (!json_match_char(p, ':')) break;
 
                    if (strcmp(key, "content") == 0) {
                        json_parse_string(p, content, sizeof(content));
                    } else if (strcmp(key, "id") == 0) {
                        json_parse_int(p, &id);
                    } else if (strcmp(key, "special") == 0) {
                        json_skip_whitespace(p);
                        json_skip_value(p);
                    } else {
                        json_skip_value(p);
                    }
                    json_match_char(p, ',');
                }
                json_match_char(p, '}');
 
                if (id >= 0 && content[0]) {
                    /* Identify special tokens */
                    if (strcmp(content, "<unk>") == 0 || strcmp(content, "[UNK]") == 0) {
                        tok->unk_id = id;
                    } else if (strcmp(content, "<s>") == 0 || strcmp(content, "<bos>") == 0 ||
                               strcmp(content, "[BOS]") == 0) {
                        tok->bos_id = id;
                    } else if (strcmp(content, "</s>") == 0 || strcmp(content, "<eos>") == 0 ||
                               strcmp(content, "[EOS]") == 0 || strcmp(content, "<|endoftext|>") == 0) {
                        tok->eos_id = id;
                    } else if (strcmp(content, "<pad>") == 0 || strcmp(content, "[PAD]") == 0) {
                        tok->pad_id = id;
                    }
                }
 
                json_match_char(p, ',');
            }
            json_match_char(p, ']');
 
        } else {
            json_skip_value(p);
        }
 
        json_match_char(p, ',');
    }
 
    free(data);
 
    printf("Loaded tokenizer: %d tokens, %d merges\n", tok->vocab_size, tok->num_merges);
    printf("  UNK=%d BOS=%d EOS=%d PAD=%d\n", tok->unk_id, tok->bos_id, tok->eos_id, tok->pad_id);
 
    return 0;
}

References CKTokenizer::bos_id, CK_MAX_TOKEN_LEN, ck_pool_alloc(), ck_pool_strdup(), ck_tokenizer_add_merge(), ck_tokenizer_add_token(), ck_tokenizer_lookup(), CKTokenizer::eos_id, hash_string(), CKVocabEntry::id, id, CKTokenizer::id_to_token, json_match_char(), json_parse_int(), json_parse_string(), json_skip_value(), json_skip_whitespace(), merged_id, CKVocabEntry::next, CKTokenizer::num_merges, CKTokenizer::pad_id, CKTokenizer::pool, CKVocabEntry::token, token, CKVocabEntry::token_len, CKTokenizer::unk_id, CKTokenizer::vocab_hash, CKTokenizer::vocab_hash_size, and CKTokenizer::vocab_size.

Referenced by run_inference().

ck_tokenizer_lookup()

int32_t ck_tokenizer_lookup	(	const CKTokenizer *	tok,
		const char *	token,
		int	len
	)

Definition at line 227 of file ck_tokenizer.c.

                                                                                {
    if (len < 0) len = (int)strlen(token);
    uint32_t bucket = hash_string(token, len) % tok->vocab_hash_size;
 
    for (CKVocabEntry *e = tok->vocab_hash[bucket]; e; e = e->next) {
        if (e->token_len == len && memcmp(e->token, token, len) == 0) {
            return e->id;
        }
    }
    return tok->unk_id;
}

References hash_string(), token, CKTokenizer::unk_id, CKTokenizer::vocab_hash, and CKTokenizer::vocab_hash_size.

Referenced by ck_tokenizer_add_token(), ck_tokenizer_encode(), ck_tokenizer_load(), main(), and spm_get_byte_token().

ck_tokenizer_lookup_merge()

int ck_tokenizer_lookup_merge	(	const CKTokenizer *	tok,
		int32_t	left,
		int32_t	right
	)

Definition at line 276 of file ck_tokenizer.c.

                                                                                   {
    uint32_t bucket = hash_pair(left, right) % tok->merge_hash_size;
 
    /* Linear probing */
    int probes = 0;
    while (tok->merge_hash[bucket] >= 0 && probes < tok->merge_hash_size) {
        int idx = tok->merge_hash[bucket];
        if (tok->merges[idx].left == left && tok->merges[idx].right == right) {
            return idx;
        }
        bucket = (bucket + 1) % tok->merge_hash_size;
        probes++;
    }
    return -1;
}

References hash_pair(), CKMergeRule::left, left, CKTokenizer::merge_hash, CKTokenizer::merge_hash_size, CKTokenizer::merges, CKMergeRule::right, and right.

Referenced by ck_tokenizer_encode().

ck_tokenizer_vocab_size()

static int ck_tokenizer_vocab_size ( const CKTokenizer *  tok )

inlinestatic

Definition at line 196 of file ck_tokenizer.h.

                                                                  {
    return tok->vocab_size;
}

References CKTokenizer::vocab_size.

Referenced by main(), and run_inference().