ckernel_alloc.c File Reference

#include "ckernel_alloc.h"
#include <errno.h>
#include <fcntl.h>
#include <pthread.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <strings.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>

Go to the source code of this file.

Macros
#define	_GNU_SOURCE
#define	HUGE_PAGE_SIZE   (2UL * 1024UL * 1024UL)

Functions
static size_t	align_up_bytes (size_t n, size_t align)
void	ck_bump_alloc_free (ck_bump_alloc_t *alloc)
int	ck_bump_alloc_init (ck_bump_alloc_t *alloc, const char *weights_path, size_t total_size, size_t weights_base, size_t activations_base)
int	ck_bump_alloc_needs_weight_materialization (const ck_bump_alloc_t *alloc)
static void	ck_bump_alloc_reset (ck_bump_alloc_t *alloc)
void *	ck_huge_alloc (size_t bytes)
void	ck_huge_free (void *ptr, size_t bytes)
static ck_huge_alloc_entry_t *	detach_allocation (void *ptr)
static int	env_flag_enabled (const char *name)
static int	record_allocation (void *ptr, size_t len, int was_mmap)

Variables
static ck_huge_alloc_entry_t *	g_alloc_list = NULL
static pthread_mutex_t	g_alloc_mutex = PTHREAD_MUTEX_INITIALIZER

Macro Definition Documentation

_GNU_SOURCE

#define _GNU_SOURCE

Definition at line 1 of file ckernel_alloc.c.

HUGE_PAGE_SIZE

#define HUGE_PAGE_SIZE   (2UL * 1024UL * 1024UL)

Definition at line 18 of file ckernel_alloc.c.

Function Documentation

align_up_bytes()

static size_t align_up_bytes ( size_t  n, size_t  align  )

static

Definition at line 31 of file ckernel_alloc.c.

{
    if (align == 0) return n;
    return (n + align - 1) & ~(align - 1);
}

Referenced by ck_bump_alloc_init(), and ck_huge_alloc().

ck_bump_alloc_free()

void ck_bump_alloc_free

(

ck_bump_alloc_t *

alloc

)

Definition at line 343 of file ckernel_alloc.c.

{
    if (!alloc || !alloc->base) {
        ck_bump_alloc_reset(alloc);
        return;
    }
 
    if (alloc->mode == CK_BUMP_MODE_MIXED_FILE_BACKED) {
        if (alloc->mapped_len > 0) {
            munmap(alloc->base, alloc->mapped_len);
        }
    } else if (alloc->mode == CK_BUMP_MODE_ANON) {
        ck_huge_free(alloc->base, alloc->total_size);
    }
 
    ck_bump_alloc_reset(alloc);
}

References ck_bump_alloc_t::base, ck_bump_alloc_reset(), CK_BUMP_MODE_ANON, CK_BUMP_MODE_MIXED_FILE_BACKED, ck_huge_free(), ck_bump_alloc_t::mapped_len, ck_bump_alloc_t::mode, and ck_bump_alloc_t::total_size.

ck_bump_alloc_init()

int ck_bump_alloc_init	(	ck_bump_alloc_t *	alloc,
		const char *	weights_path,
		size_t	total_size,
		size_t	weights_base,
		size_t	activations_base
	)

Definition at line 286 of file ckernel_alloc.c.

{
    int force_mixed = 0;
    int disable_mixed = 0;
 
    if (!alloc || total_size == 0 ||
        total_size > SIZE_MAX - (HUGE_PAGE_SIZE - 1)) {
        fprintf(stderr, "ck_bump_alloc_init: invalid arguments\n");
        return -1;
    }
 
    ck_bump_alloc_reset(alloc);
    alloc->total_size = total_size;
    alloc->weights_base = weights_base;
    alloc->activations_base = activations_base;
    force_mixed = env_flag_enabled("CK_BUMP_FORCE_MIXED");
    disable_mixed = env_flag_enabled("CK_BUMP_DISABLE_MIXED");
 
    if (weights_base > activations_base || activations_base > total_size) {
        fprintf(stderr,
                "ck_bump_alloc_init: invalid bump layout weights=%zu activations=%zu total=%zu\n",
                weights_base, activations_base, total_size);
        ck_bump_alloc_reset(alloc);
        return -1;
    }
 
    if (!force_mixed) {
        void *anon = ck_huge_alloc(total_size);
        if (anon) {
            alloc->base = (uint8_t *)anon;
            alloc->mapped_len = align_up_bytes(total_size, HUGE_PAGE_SIZE);
            alloc->mode = CK_BUMP_MODE_ANON;
            return 0;
        }
    }
 
#ifdef __linux__
    if (!disable_mixed) {
        if (ck_bump_alloc_try_mixed(alloc, weights_path) == 0) {
            return 0;
        }
    }
#else
    (void)weights_path;
#endif
 
    fprintf(stderr,
            "ck_bump_alloc_init: failed to allocate bump arena (%zu bytes, %.2f MiB)\n",
            total_size, (double)total_size / (1024.0 * 1024.0));
    ck_bump_alloc_reset(alloc);
    return -1;
}

References ck_bump_alloc_t::activations_base, align_up_bytes(), ck_bump_alloc_t::base, ck_bump_alloc_reset(), CK_BUMP_MODE_ANON, ck_huge_alloc(), env_flag_enabled(), HUGE_PAGE_SIZE, ck_bump_alloc_t::mapped_len, ck_bump_alloc_t::mode, ck_bump_alloc_t::total_size, and ck_bump_alloc_t::weights_base.

ck_bump_alloc_needs_weight_materialization()

int ck_bump_alloc_needs_weight_materialization

(

const ck_bump_alloc_t *

alloc

)

Definition at line 361 of file ckernel_alloc.c.

{
    if (!alloc) {
        return 1;
    }
    return alloc->mode != CK_BUMP_MODE_MIXED_FILE_BACKED;
}

References CK_BUMP_MODE_MIXED_FILE_BACKED, and ck_bump_alloc_t::mode.

ck_bump_alloc_reset()

static void ck_bump_alloc_reset ( ck_bump_alloc_t *  alloc )

static

Definition at line 142 of file ckernel_alloc.c.

{
    if (!alloc) {
        return;
    }
    alloc->base = NULL;
    alloc->total_size = 0;
    alloc->weights_base = 0;
    alloc->activations_base = 0;
    alloc->mapped_len = 0;
    alloc->weights_file_size = 0;
    alloc->mode = CK_BUMP_MODE_UNINITIALIZED;
}

References ck_bump_alloc_t::activations_base, ck_bump_alloc_t::base, CK_BUMP_MODE_UNINITIALIZED, ck_bump_alloc_t::mapped_len, ck_bump_alloc_t::mode, ck_bump_alloc_t::total_size, ck_bump_alloc_t::weights_base, and ck_bump_alloc_t::weights_file_size.

Referenced by ck_bump_alloc_free(), and ck_bump_alloc_init().

ck_huge_alloc()

void * ck_huge_alloc

(

size_t

bytes

)

Allocate a large, contiguous memory region for model weights/activations.

Implementation strategy:

• Try to allocate 2MB-aligned memory backed by huge pages where possible.
• Fall back to aligned_alloc + madvise(MADV_HUGEPAGE) when explicit huge pages are not available.

Returns NULL on failure.

Definition at line 70 of file ckernel_alloc.c.

{
    if (bytes == 0 || bytes > SIZE_MAX - (HUGE_PAGE_SIZE - 1)) {
        errno = ENOMEM;
        return NULL;
    }
    size_t len = align_up_bytes(bytes, HUGE_PAGE_SIZE);
 
    /* First, try explicit huge pages via mmap + MAP_HUGETLB. */
    void *p = mmap(NULL, len,
                   PROT_READ | PROT_WRITE,
                   MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB,
                   -1, 0);
    if (p != MAP_FAILED) {
        if (!record_allocation(p, len, 1)) {
            munmap(p, len);
            return NULL;
        }
        return p;
    }
 
    /* Fallback: aligned_alloc with transparent hugepage hint. */
    void *q = aligned_alloc(HUGE_PAGE_SIZE, len);
    if (!q) {
        fprintf(stderr, "ck_huge_alloc: aligned_alloc failed for %zu bytes: %s\n",
                len, strerror(errno));
        return NULL;
    }
 
    /* Best-effort hint; ignore errors. */
    (void)madvise(q, len, MADV_HUGEPAGE);
    if (!record_allocation(q, len, 0)) {
        free(q);
        return NULL;
    }
    return q;
}

Referenced by ck_bump_alloc_init().

ck_huge_free()

void ck_huge_free	(	void *	ptr,
		size_t	bytes
	)

Free memory allocated by ck_huge_alloc.

The bytes parameter should be the same size passed to ck_huge_alloc.

Definition at line 108 of file ckernel_alloc.c.

{
    if (!ptr || bytes == 0) {
        return;
    }
 
    ck_huge_alloc_entry_t *entry = detach_allocation(ptr);
    if (!entry) {
        /* Fall back to malloc/free if the allocation wasn't tracked. */
        free(ptr);
        return;
    }
 
    if (entry->was_mmap) {
        munmap(ptr, entry->len);
    } else {
        free(ptr);
    }
 
    free(entry);
}

Referenced by ck_bump_alloc_free().

detach_allocation()

static ck_huge_alloc_entry_t * detach_allocation ( void *  ptr )

static

Definition at line 53 of file ckernel_alloc.c.

{
    pthread_mutex_lock(&g_alloc_mutex);
    ck_huge_alloc_entry_t **node = &g_alloc_list;
    while (*node) {
        if ((*node)->ptr == ptr) {
            ck_huge_alloc_entry_t *entry = *node;
            *node = entry->next;
            pthread_mutex_unlock(&g_alloc_mutex);
            return entry;
        }
        node = &(*node)->next;
    }
    pthread_mutex_unlock(&g_alloc_mutex);
    return NULL;
}

References g_alloc_list, and g_alloc_mutex.

Referenced by ck_huge_free().

env_flag_enabled()

static int env_flag_enabled ( const char *  name )

static

Definition at line 130 of file ckernel_alloc.c.

{
    const char *value = getenv(name);
    if (!value || !*value) {
        return 0;
    }
    if (strcmp(value, "0") == 0 || strcasecmp(value, "false") == 0 || strcasecmp(value, "no") == 0) {
        return 0;
    }
    return 1;
}

Referenced by ck_bump_alloc_init().

record_allocation()

static int record_allocation ( void *  ptr, size_t  len, int  was_mmap  )

static

Definition at line 37 of file ckernel_alloc.c.

{
    ck_huge_alloc_entry_t *entry = malloc(sizeof(*entry));
    if (!entry) {
        return 0;
    }
    entry->ptr = ptr;
    entry->len = len;
    entry->was_mmap = was_mmap;
    pthread_mutex_lock(&g_alloc_mutex);
    entry->next = g_alloc_list;
    g_alloc_list = entry;
    pthread_mutex_unlock(&g_alloc_mutex);
    return 1;
}

References g_alloc_list, and g_alloc_mutex.

Referenced by ck_huge_alloc().

Variable Documentation

g_alloc_list

ck_huge_alloc_entry_t* g_alloc_list = NULL

static

Definition at line 29 of file ckernel_alloc.c.

Referenced by detach_allocation(), and record_allocation().

g_alloc_mutex

pthread_mutex_t g_alloc_mutex = PTHREAD_MUTEX_INITIALIZER

static

Definition at line 28 of file ckernel_alloc.c.

Referenced by detach_allocation(), and record_allocation().