v2_legacy/ck_tokenizer_v2.c File Reference

#include <stdio.h>
#include <stdlib.h>
#include <stdint.h>
#include <string.h>
#include <ctype.h>
#include "ck_tokenizer.h"

Go to the source code of this file.

Functions
int	ck_tokenizer_load_binary (CKTokenizer *tok, int vocab_size, const int32_t *offsets, const char *strings, int num_merges, const int32_t *merges)
static uint32_t	hash_string (const char *s, int len)

Function Documentation

ck_tokenizer_load_binary()

int ck_tokenizer_load_binary	(	CKTokenizer *	tok,
		int	vocab_size,
		const int32_t *	offsets,
		const char *	strings,
		int	num_merges,
		const int32_t *	merges
	)

Load vocabulary from memory-mapped binary data.

Parameters

tok	Tokenizer
vocab_size	Number of tokens
offsets	Array of offsets into strings pool
strings	String pool containing null-terminated tokens
num_merges	Number of BPE merges
merges	Merge rules as (left, right, merged) triplets

Returns

0 on success, -1 on error

Definition at line 18 of file v2_legacy/ck_tokenizer_v2.c.

                                                    {
    if (!tok || !offsets || !strings) return -1;
 
    // We assume ck_tokenizer_init was already called to alloc hash tables
    tok->vocab_size = 0;
    
    for (int i = 0; i < vocab_size; i++) {
        const char *token = strings + offsets[i];
        int len = (int)strlen(token);
        
        CKVocabEntry *entry = (CKVocabEntry *)ck_pool_alloc(&tok->pool, sizeof(CKVocabEntry));
        entry->token = (char *)token; 
        entry->token_len = len;
        entry->id = i;
 
        uint32_t bucket = hash_string(token, len) % tok->vocab_hash_size;
        entry->next = tok->vocab_hash[bucket];
        tok->vocab_hash[bucket] = entry;
 
        tok->id_to_token[i] = entry->token;
        tok->vocab_size++;
    }
 
    if (merges && num_merges > 0) {
        for (int i = 0; i < num_merges; i++) {
            int32_t left = merges[i*3 + 0];
            int32_t right = merges[i*3 + 1];
            int32_t merged = merges[i*3 + 2];
            ck_tokenizer_add_merge(tok, left, right, merged);
        }
    }
 
    return 0;
}

References ck_pool_alloc(), ck_tokenizer_add_merge(), hash_string(), CKVocabEntry::id, CKTokenizer::id_to_token, left, merges, CKVocabEntry::next, num_merges, offsets, CKTokenizer::pool, right, strings, CKVocabEntry::token, token, CKVocabEntry::token_len, CKTokenizer::vocab_hash, CKTokenizer::vocab_hash_size, CKTokenizer::vocab_size, and vocab_size.

hash_string()

static uint32_t hash_string ( const char *  s, int  len  )

static

Definition at line 9 of file v2_legacy/ck_tokenizer_v2.c.

                                                    {
    uint32_t hash = 2166136261u;
    for (int i = 0; i < len; i++) {
        hash ^= (uint8_t)s[i];
        hash *= 16777619u;
    }
    return hash;
}

Referenced by ck_tokenizer_load_binary().