gemm_kernels_f16.c File Reference

GEMM kernels with FP16 (half-precision) weights. More...

#include <stdint.h>
#include <stddef.h>
#include <stdbool.h>
#include "ckernel_quant.h"
#include "ckernel_engine.h"
#include "ck_threadpool.h"
#include "ggml_runtime_compat.h"
#include <dlfcn.h>
#include <stdlib.h>
#include <string.h>

Go to the source code of this file.

Macros
#define	_GNU_SOURCE
#define	fp16_to_fp32(x)   ggml_fp16_to_fp32(x)
#define	fp32_to_fp16(x)   ggml_fp32_to_fp16(x)

Typedefs
typedef void(*	ck_f16_ggml_build_forward_expand_fn) (struct ggml_cgraph *, struct ggml_tensor *)
typedef void(*	ck_f16_ggml_cpu_init_fn) (void)
typedef void(*	ck_f16_ggml_free_fn) (struct ggml_context *)
typedef float *(*	ck_f16_ggml_get_data_f32_fn) (const struct ggml_tensor *)
typedef void *(*	ck_f16_ggml_get_data_fn) (const struct ggml_tensor *)
typedef enum ggml_status(*	ck_f16_ggml_graph_compute_with_ctx_fn) (struct ggml_context *, struct ggml_cgraph *, int)
typedef struct ggml_context *(*	ck_f16_ggml_init_fn) (struct ggml_init_params)
typedef struct ggml_tensor *(*	ck_f16_ggml_mul_mat_fn) (struct ggml_context *, struct ggml_tensor *, struct ggml_tensor *)
typedef struct ggml_cgraph *(*	ck_f16_ggml_new_graph_fn) (struct ggml_context *)
typedef struct ggml_tensor *(*	ck_f16_ggml_new_tensor_2d_fn) (struct ggml_context *, enum ggml_type, int64_t, int64_t)

Functions
static float	ck_dot_f16_f16_local (const uint16_t *w, const uint16_t *x, int k)
static ck_f16_ggml_build_forward_expand_fn	ck_f16_resolve_ggml_build_forward_expand (void)
static ck_f16_ggml_cpu_init_fn	ck_f16_resolve_ggml_cpu_init (void)
static ck_f16_ggml_free_fn	ck_f16_resolve_ggml_free (void)
static ck_f16_ggml_get_data_fn	ck_f16_resolve_ggml_get_data (void)
static ck_f16_ggml_get_data_f32_fn	ck_f16_resolve_ggml_get_data_f32 (void)
static ck_f16_ggml_graph_compute_with_ctx_fn	ck_f16_resolve_ggml_graph_compute_with_ctx (void)
static ck_f16_ggml_init_fn	ck_f16_resolve_ggml_init (void)
static ck_f16_ggml_mul_mat_fn	ck_f16_resolve_ggml_mul_mat (void)
static ck_f16_ggml_new_graph_fn	ck_f16_resolve_ggml_new_graph (void)
static ck_f16_ggml_new_tensor_2d_fn	ck_f16_resolve_ggml_new_tensor_2d (void)
static void	ck_f32_to_f16_row_local (uint16_t *dst, const float *src, int n)
static void	ck_gemm_f16_input_fp16_work (int ith, int nth, void *opaque)
static int	ck_gemm_f16_pick_active_threads (const ck_threadpool_t *pool, int M, int N, int K)
static int	ck_gemm_f16_threadpool_enabled (int M, int N, int K)
int	ck_gemm_nt_f16_ggml_oracle (const float *A, const void *B, const float *bias, float *C, int M, int N, int K)
int	ck_gemm_nt_f16_simd_lanes (void)
void	convert_f16_to_f32 (float *dst, const uint16_t *src, size_t count)
	Convert FP16 tensor to FP32.
void	convert_f32_to_f16 (uint16_t *dst, const float *src, size_t count)
	Convert FP32 tensor to FP16.
float	dot_f16 (const uint16_t *w_f16, const float *x, int K)
void	gemm_f16 (float *Y, const uint16_t *W, const float *X, int M, int N, int K)
	Auto-dispatch GEMM based on available SIMD.
void	gemm_f16_backward (float *dX, const uint16_t *W, const float *dY, int M, int N, int K)
	Batched backward pass.
static void	gemm_f16_input_fp16_ref (float *Y, const uint16_t *W, const float *X, int M, int N, int K)
static int	gemm_f16_input_fp16_threadpool (float *Y, const uint16_t *W, const float *X, int M, int N, int K)
void	gemm_f16_ref (float *Y, const uint16_t *W, const float *X, int M, int N, int K)
	Matrix-matrix multiply with FP16 weights (scalar reference)
void	gemm_nt_f16 (const float *A, const void *B, const float *bias, float *C, int M, int N, int K)
	NT GEMM wrapper for FP16 weights with the engine's standard ABI.
void	gemm_nt_f16_clipped (const float *A, const void *B, const float *bias, const float *input_min, const float *input_max, const float *output_min, const float *output_max, float *C, int M, int N, int K)
static int	gemm_nt_f16_ggml_strict (const float *A, const void *B, const float *bias, float *C, int M, int N, int K)
void	gemv_f16 (float *y, const uint16_t *W, const float *x, int M, int K)
	Auto-dispatch GEMV based on available SIMD.
void	gemv_f16_backward (float *dX, const uint16_t *W, const float *dY, int M, int K)
	Auto-dispatch backward.
void	gemv_f16_backward_ref (float *dX, const uint16_t *W, const float *dY, int M, int K)
	Backward pass: compute input gradient (scalar reference)
void	gemv_f16_ref (float *y, const uint16_t *W, const float *x, int M, int K)
	Matrix-vector multiply with FP16 weights (scalar reference)

Detailed Description

GEMM kernels with FP16 (half-precision) weights.

CK-ENGINE KERNEL RULES:

1. NO malloc/free - memory via bump allocator, pointers passed in
2. NO OpenMP - parallelization at orchestrator/codegen layer
3. API must define: inputs, outputs, workspace, and memory layouts
4. Pure computation - deterministic, no side effects

After changes: make test && make llamacpp-parity-full

Implements matrix multiplication where:

• Weights: FP16 (IEEE half-precision, used by vision encoders)
• Activations: FP32
• Output: FP32

Used for multimodal projection layers (mmproj-*.gguf files).

Definition in file gemm_kernels_f16.c.

Macro Definition Documentation

_GNU_SOURCE

#define _GNU_SOURCE

Definition at line 23 of file gemm_kernels_f16.c.

fp16_to_fp32

#define fp16_to_fp32

(

x

)

ggml_fp16_to_fp32(x)

Definition at line 279 of file gemm_kernels_f16.c.

fp32_to_fp16

#define fp32_to_fp16

(

x

)

ggml_fp32_to_fp16(x)

Definition at line 280 of file gemm_kernels_f16.c.

Typedef Documentation

ck_f16_ggml_build_forward_expand_fn

typedef void(* ck_f16_ggml_build_forward_expand_fn) (struct ggml_cgraph *, struct ggml_tensor *)

Definition at line 47 of file gemm_kernels_f16.c.

ck_f16_ggml_cpu_init_fn

typedef void(* ck_f16_ggml_cpu_init_fn) (void)

Definition at line 49 of file gemm_kernels_f16.c.

ck_f16_ggml_free_fn

typedef void(* ck_f16_ggml_free_fn) (struct ggml_context *)

Definition at line 43 of file gemm_kernels_f16.c.

ck_f16_ggml_get_data_f32_fn

typedef float *(* ck_f16_ggml_get_data_f32_fn) (const struct ggml_tensor *)

Definition at line 51 of file gemm_kernels_f16.c.

ck_f16_ggml_get_data_fn

typedef void *(* ck_f16_ggml_get_data_fn) (const struct ggml_tensor *)

Definition at line 50 of file gemm_kernels_f16.c.

ck_f16_ggml_graph_compute_with_ctx_fn

typedef enum ggml_status(* ck_f16_ggml_graph_compute_with_ctx_fn) (struct ggml_context *, struct ggml_cgraph *, int)

Definition at line 47 of file gemm_kernels_f16.c.

ck_f16_ggml_init_fn

typedef struct ggml_context *(* ck_f16_ggml_init_fn) (struct ggml_init_params)

Definition at line 1 of file gemm_kernels_f16.c.

ck_f16_ggml_mul_mat_fn

typedef struct ggml_tensor *(* ck_f16_ggml_mul_mat_fn) (struct ggml_context *, struct ggml_tensor *, struct ggml_tensor *)

Definition at line 43 of file gemm_kernels_f16.c.

ck_f16_ggml_new_graph_fn

typedef struct ggml_cgraph *(* ck_f16_ggml_new_graph_fn) (struct ggml_context *)

Definition at line 43 of file gemm_kernels_f16.c.

ck_f16_ggml_new_tensor_2d_fn

typedef struct ggml_tensor *(* ck_f16_ggml_new_tensor_2d_fn) (struct ggml_context *, enum ggml_type, int64_t, int64_t)

Definition at line 43 of file gemm_kernels_f16.c.

Function Documentation

ck_dot_f16_f16_local()

static float ck_dot_f16_f16_local ( const uint16_t *  w, const uint16_t *  x, int  k  )

inlinestatic

Definition at line 568 of file gemm_kernels_f16.c.

{
#if defined(__AVX512F__) && defined(__F16C__)
    return ck_dot_f16_f16_avx512(w, x, k);
#elif defined(__F16C__) && defined(__AVX__)
    return ck_dot_f16_f16_avx(w, x, k);
#else
    float sum = 0.0f;
    for (int i = 0; i < k; ++i) {
        sum += fp16_to_fp32(w[i]) * fp16_to_fp32(x[i]);
    }
    return sum;
#endif
}

References fp16_to_fp32.

Referenced by ck_gemm_f16_input_fp16_work(), and gemm_f16_input_fp16_ref().

ck_f16_resolve_ggml_build_forward_expand()

static ck_f16_ggml_build_forward_expand_fn ck_f16_resolve_ggml_build_forward_expand ( void  )

static

Definition at line 108 of file gemm_kernels_f16.c.

{
    static int tried = 0;
    static ck_f16_ggml_build_forward_expand_fn fn = NULL;
    if (!tried) {
        tried = 1;
        fn = (ck_f16_ggml_build_forward_expand_fn) dlsym(RTLD_DEFAULT, "ggml_build_forward_expand");
    }
    return fn;
}

References RTLD_DEFAULT.

Referenced by gemm_nt_f16_ggml_strict().

ck_f16_resolve_ggml_cpu_init()

static ck_f16_ggml_cpu_init_fn ck_f16_resolve_ggml_cpu_init ( void  )

static

Definition at line 130 of file gemm_kernels_f16.c.

{
    static int tried = 0;
    static ck_f16_ggml_cpu_init_fn fn = NULL;
    if (!tried) {
        tried = 1;
        fn = (ck_f16_ggml_cpu_init_fn) dlsym(RTLD_DEFAULT, "ggml_cpu_init");
    }
    return fn;
}

References RTLD_DEFAULT.

Referenced by gemm_nt_f16_ggml_strict().

ck_f16_resolve_ggml_free()

static ck_f16_ggml_free_fn ck_f16_resolve_ggml_free ( void  )

static

Definition at line 64 of file gemm_kernels_f16.c.

{
    static int tried = 0;
    static ck_f16_ggml_free_fn fn = NULL;
    if (!tried) {
        tried = 1;
        fn = (ck_f16_ggml_free_fn) dlsym(RTLD_DEFAULT, "ggml_free");
    }
    return fn;
}

References RTLD_DEFAULT.

Referenced by gemm_nt_f16_ggml_strict().

ck_f16_resolve_ggml_get_data()

static ck_f16_ggml_get_data_fn ck_f16_resolve_ggml_get_data ( void  )

static

Definition at line 141 of file gemm_kernels_f16.c.

{
    static int tried = 0;
    static ck_f16_ggml_get_data_fn fn = NULL;
    if (!tried) {
        tried = 1;
        fn = (ck_f16_ggml_get_data_fn) dlsym(RTLD_DEFAULT, "ggml_get_data");
    }
    return fn;
}

References RTLD_DEFAULT.

Referenced by gemm_nt_f16_ggml_strict().

ck_f16_resolve_ggml_get_data_f32()

static ck_f16_ggml_get_data_f32_fn ck_f16_resolve_ggml_get_data_f32 ( void  )

static

Definition at line 152 of file gemm_kernels_f16.c.

{
    static int tried = 0;
    static ck_f16_ggml_get_data_f32_fn fn = NULL;
    if (!tried) {
        tried = 1;
        fn = (ck_f16_ggml_get_data_f32_fn) dlsym(RTLD_DEFAULT, "ggml_get_data_f32");
    }
    return fn;
}

References RTLD_DEFAULT.

Referenced by gemm_nt_f16_ggml_strict().

ck_f16_resolve_ggml_graph_compute_with_ctx()

static ck_f16_ggml_graph_compute_with_ctx_fn ck_f16_resolve_ggml_graph_compute_with_ctx ( void  )

static

Definition at line 119 of file gemm_kernels_f16.c.

{
    static int tried = 0;
    static ck_f16_ggml_graph_compute_with_ctx_fn fn = NULL;
    if (!tried) {
        tried = 1;
        fn = (ck_f16_ggml_graph_compute_with_ctx_fn) dlsym(RTLD_DEFAULT, "ggml_graph_compute_with_ctx");
    }
    return fn;
}

References RTLD_DEFAULT.

Referenced by gemm_nt_f16_ggml_strict().

ck_f16_resolve_ggml_init()

static ck_f16_ggml_init_fn ck_f16_resolve_ggml_init ( void  )

static

Definition at line 53 of file gemm_kernels_f16.c.

{
    static int tried = 0;
    static ck_f16_ggml_init_fn fn = NULL;
    if (!tried) {
        tried = 1;
        fn = (ck_f16_ggml_init_fn) dlsym(RTLD_DEFAULT, "ggml_init");
    }
    return fn;
}

References RTLD_DEFAULT.

Referenced by gemm_nt_f16_ggml_strict().

ck_f16_resolve_ggml_mul_mat()

static ck_f16_ggml_mul_mat_fn ck_f16_resolve_ggml_mul_mat ( void  )

static

Definition at line 86 of file gemm_kernels_f16.c.

{
    static int tried = 0;
    static ck_f16_ggml_mul_mat_fn fn = NULL;
    if (!tried) {
        tried = 1;
        fn = (ck_f16_ggml_mul_mat_fn) dlsym(RTLD_DEFAULT, "ggml_mul_mat");
    }
    return fn;
}

References RTLD_DEFAULT.

Referenced by gemm_nt_f16_ggml_strict().

ck_f16_resolve_ggml_new_graph()

static ck_f16_ggml_new_graph_fn ck_f16_resolve_ggml_new_graph ( void  )

static

Definition at line 97 of file gemm_kernels_f16.c.

{
    static int tried = 0;
    static ck_f16_ggml_new_graph_fn fn = NULL;
    if (!tried) {
        tried = 1;
        fn = (ck_f16_ggml_new_graph_fn) dlsym(RTLD_DEFAULT, "ggml_new_graph");
    }
    return fn;
}

References RTLD_DEFAULT.

Referenced by gemm_nt_f16_ggml_strict().

ck_f16_resolve_ggml_new_tensor_2d()

static ck_f16_ggml_new_tensor_2d_fn ck_f16_resolve_ggml_new_tensor_2d ( void  )

static

Definition at line 75 of file gemm_kernels_f16.c.

{
    static int tried = 0;
    static ck_f16_ggml_new_tensor_2d_fn fn = NULL;
    if (!tried) {
        tried = 1;
        fn = (ck_f16_ggml_new_tensor_2d_fn) dlsym(RTLD_DEFAULT, "ggml_new_tensor_2d");
    }
    return fn;
}

References RTLD_DEFAULT.

Referenced by gemm_nt_f16_ggml_strict().

ck_f32_to_f16_row_local()

static void ck_f32_to_f16_row_local ( uint16_t *  dst, const float *  src, int  n  )

inlinestatic

Definition at line 292 of file gemm_kernels_f16.c.

{
#ifdef __AVX512F__
    int i = 0;
    const int n16 = (n / 16) * 16;
    for (; i < n16; i += 16) {
        const __m512 v = _mm512_loadu_ps(src + i);
        const __m256i h = _mm512_cvtps_ph(v, 0);
        _mm256_storeu_si256((__m256i *)(dst + i), h);
    }
    for (; i < n; ++i) {
        dst[i] = fp32_to_fp16(src[i]);
    }
#elif defined(__F16C__) && defined(__AVX__)
    int i = 0;
    const int n8 = (n / 8) * 8;
    for (; i < n8; i += 8) {
        const __m256 v = _mm256_loadu_ps(src + i);
        const __m128i h = _mm256_cvtps_ph(v, 0);
        _mm_storeu_si128((__m128i *)(dst + i), h);
    }
    for (; i < n; ++i) {
        dst[i] = fp32_to_fp16(src[i]);
    }
#else
    for (int i = 0; i < n; ++i) {
        dst[i] = fp32_to_fp16(src[i]);
    }
#endif
}

References fp32_to_fp16.

Referenced by ck_gemm_f16_input_fp16_work(), and gemm_f16_input_fp16_ref().

ck_gemm_f16_input_fp16_work()

static void ck_gemm_f16_input_fp16_work ( int  ith, int  nth, void *  opaque  )

static

Definition at line 818 of file gemm_kernels_f16.c.

{
    ck_gemm_f16_input_fp16_args_t *args = (ck_gemm_f16_input_fp16_args_t *) opaque;
    const int M = args->M;
    const int N = args->N;
    const int K = args->K;
 
    const int token_groups = (N + 3) / 4;
    for (int group = ith; group < token_groups; group += nth) {
        const int n0 = group * 4;
        const int token_count = N - n0 < 4 ? N - n0 : 4;
#if defined(__F16C__) && defined(__AVX__) && !defined(__AVX512F__)
        if (token_count == 4 && ck_gemm_f16_m4n2_enabled()) {
            uint16_t x_f16[4][K];
            for (int t = 0; t < 4; ++t) {
                ck_f32_to_f16_row_local(
                    x_f16[t], args->X + (size_t)(n0 + t) * (size_t)K, K);
            }
 
            int row = 0;
            for (; row + 1 < M; row += 2) {
                float sums[4][2];
                const uint16_t *w0 = args->W + (size_t)row * (size_t)K;
                ck_dot_f16_f16_avx_m4n2(
                    w0, w0 + K,
                    x_f16[0], x_f16[1], x_f16[2], x_f16[3], K, sums);
                for (int t = 0; t < 4; ++t) {
                    float *out = args->Y + (size_t)(n0 + t) * (size_t)M + (size_t)row;
                    out[0] = sums[t][0];
                    out[1] = sums[t][1];
                }
            }
            for (; row < M; ++row) {
                const uint16_t *w = args->W + (size_t)row * (size_t)K;
                for (int t = 0; t < 4; ++t) {
                    args->Y[(size_t)(n0 + t) * (size_t)M + (size_t)row] =
                        ck_dot_f16_f16_local(w, x_f16[t], K);
                }
            }
            continue;
        }
#endif
        for (int n = n0; n < n0 + token_count; ++n) {
        const float *x_row = args->X + (size_t)n * (size_t)K;
        uint16_t x_f16[K];
 
        ck_f32_to_f16_row_local(x_f16, x_row, K);
 
        int row = 0;
#if defined(__AVX512F__) && defined(__F16C__)
        for (; row + 3 < M; row += 4) {
            float sums[4];
            ck_dot_f16_f16_avx512_4(
                args->W + (size_t)row * (size_t)K, x_f16, K, sums);
            args->Y[(size_t)n * (size_t)M + (size_t)row] = sums[0];
            args->Y[(size_t)n * (size_t)M + (size_t)row + 1] = sums[1];
            args->Y[(size_t)n * (size_t)M + (size_t)row + 2] = sums[2];
            args->Y[(size_t)n * (size_t)M + (size_t)row + 3] = sums[3];
        }
#elif defined(__F16C__) && defined(__AVX__)
        for (; row + 3 < M; row += 4) {
            float sums[4];
            ck_dot_f16_f16_avx4(
                args->W + (size_t)row * (size_t)K, x_f16, K, sums);
            args->Y[(size_t)n * (size_t)M + (size_t)row] = sums[0];
            args->Y[(size_t)n * (size_t)M + (size_t)row + 1] = sums[1];
            args->Y[(size_t)n * (size_t)M + (size_t)row + 2] = sums[2];
            args->Y[(size_t)n * (size_t)M + (size_t)row + 3] = sums[3];
        }
#endif
        for (; row < M; ++row) {
            const uint16_t *w_row = args->W + (size_t)row * (size_t)K;
            const float sum = ck_dot_f16_f16_local(w_row, x_f16, K);
            args->Y[(size_t)n * (size_t)M + (size_t)row] = sum;
        }
        }
    }
}

References ck_dot_f16_f16_local(), and ck_f32_to_f16_row_local().

Referenced by gemm_f16_input_fp16_threadpool().

ck_gemm_f16_pick_active_threads()

static int ck_gemm_f16_pick_active_threads ( const ck_threadpool_t *  pool, int  M, int  N, int  K  )

static

Definition at line 913 of file gemm_kernels_f16.c.

{
    int nth = pool ? ck_threadpool_n_threads(pool) : 1;
    if (nth <= 1) return 1;
 
    const char *cap_env = getenv("CK_F16_GEMM_THREAD_CAP");
    int cap = cap_env && cap_env[0] ? atoi(cap_env) : 24;
    if (cap < 1) cap = 1;
    if (cap > nth) cap = nth;
 
    int active = N;
    if (M >= 1024 && K >= 1024 && active < 8) active = 8;
    if (active > cap) active = cap;
    if (active > nth) active = nth;
    return active < 1 ? 1 : active;
}

References ck_threadpool_n_threads().

Referenced by gemm_f16_input_fp16_threadpool().

ck_gemm_f16_threadpool_enabled()

static int ck_gemm_f16_threadpool_enabled ( int  M, int  N, int  K  )

static

Definition at line 897 of file gemm_kernels_f16.c.

{
    const char *disable = getenv("CK_DISABLE_F16_GEMM_THREADPOOL");
    if (disable && disable[0] && strcmp(disable, "0") != 0) return 0;
    if (M < 256 || N < 16 || K < 256) return 0;
    return 1;
}

Referenced by gemm_f16_input_fp16_threadpool().

ck_gemm_nt_f16_ggml_oracle()

int ck_gemm_nt_f16_ggml_oracle	(	const float *	A,
		const void *	B,
		const float *	bias,
		float *	C,
		int	M,
		int	N,
		int	K
	)

Definition at line 262 of file gemm_kernels_f16.c.

{
    return gemm_nt_f16_ggml_strict(A, B, bias, C, M, N, K);
}

References C, and gemm_nt_f16_ggml_strict().

ck_gemm_nt_f16_simd_lanes()

int ck_gemm_nt_f16_simd_lanes

(

void

)

Definition at line 583 of file gemm_kernels_f16.c.

{
#if defined(__AVX512F__) && defined(__F16C__)
    return 16;
#elif defined(__F16C__) && defined(__AVX__)
    return 8;
#else
    return 1;
#endif
}

convert_f16_to_f32()

void convert_f16_to_f32	(	float *	dst,
		const uint16_t *	src,
		size_t	count
	)

Convert FP16 tensor to FP32.

Definition at line 1047 of file gemm_kernels_f16.c.

{
#ifdef __AVX512F__
    const size_t count16 = count / 16 * 16;
 
    for (size_t i = 0; i < count16; i += 16) {
        __m256i f16 = _mm256_loadu_si256((const __m256i *)&src[i]);
        __m512 f32 = _mm512_cvtph_ps(f16);
        _mm512_storeu_ps(&dst[i], f32);
    }
 
    for (size_t i = count16; i < count; i++) {
        dst[i] = fp16_to_fp32(src[i]);
    }
#else
    for (size_t i = 0; i < count; i++) {
        dst[i] = fp16_to_fp32(src[i]);
    }
#endif
}

References fp16_to_fp32.

convert_f32_to_f16()

void convert_f32_to_f16	(	uint16_t *	dst,
		const float *	src,
		size_t	count
	)

Convert FP32 tensor to FP16.

Definition at line 1071 of file gemm_kernels_f16.c.

{
#ifdef __AVX512F__
    const size_t count16 = count / 16 * 16;
 
    for (size_t i = 0; i < count16; i += 16) {
        __m512 f32 = _mm512_loadu_ps(&src[i]);
        __m256i f16 = _mm512_cvtps_ph(f32, 0);
        _mm256_storeu_si256((__m256i *)&dst[i], f16);
    }
 
    for (size_t i = count16; i < count; i++) {
        dst[i] = fp32_to_fp16(src[i]);
    }
#else
    for (size_t i = 0; i < count; i++) {
        dst[i] = fp32_to_fp16(src[i]);
    }
#endif
}

References fp32_to_fp16.

dot_f16()

float dot_f16	(	const uint16_t *	w_f16,
		const float *	x,
		int	K
	)

Definition at line 1208 of file gemm_kernels_f16.c.

{
    float result;
    gemv_f16(&result, w_f16, x, 1, K);
    return result;
}

References gemv_f16().

gemm_f16()

void gemm_f16	(	float *	Y,
		const uint16_t *	W,
		const float *	X,
		int	M,
		int	N,
		int	K
	)

Auto-dispatch GEMM based on available SIMD.

Definition at line 752 of file gemm_kernels_f16.c.

{
#ifdef __AVX512F__
    gemm_f16_avx512(Y, W, X, M, N, K);
#else
    gemm_f16_ref(Y, W, X, M, N, K);
#endif
}

References gemm_f16_ref().

gemm_f16_backward()

void gemm_f16_backward	(	float *	dX,
		const uint16_t *	W,
		const float *	dY,
		int	M,
		int	N,
		int	K
	)

Batched backward pass.

Definition at line 1194 of file gemm_kernels_f16.c.

{
    for (int n = 0; n < N; n++) {
        gemv_f16_backward(&dX[n * K], W, &dY[n * M], M, K);
    }
}

References gemv_f16_backward().

gemm_f16_input_fp16_ref()

static void gemm_f16_input_fp16_ref ( float *  Y, const uint16_t *  W, const float *  X, int  M, int  N, int  K  )

static

Definition at line 764 of file gemm_kernels_f16.c.

{
#pragma omp parallel for schedule(static) if(N > 1)
    for (int n = 0; n < N; ++n) {
        const float *x_row = &X[(size_t)n * (size_t)K];
        uint16_t x_f16[K];
 
        ck_f32_to_f16_row_local(x_f16, x_row, K);
 
        int row = 0;
#if defined(__AVX512F__) && defined(__F16C__)
        for (; row + 3 < M; row += 4) {
            float sums[4];
            ck_dot_f16_f16_avx512_4(
                &W[(size_t)row * (size_t)K], x_f16, K, sums);
            Y[(size_t)n * (size_t)M + (size_t)row] = sums[0];
            Y[(size_t)n * (size_t)M + (size_t)row + 1] = sums[1];
            Y[(size_t)n * (size_t)M + (size_t)row + 2] = sums[2];
            Y[(size_t)n * (size_t)M + (size_t)row + 3] = sums[3];
        }
#elif defined(__F16C__) && defined(__AVX__)
        for (; row + 3 < M; row += 4) {
            float sums[4];
            ck_dot_f16_f16_avx4(&W[(size_t)row * (size_t)K], x_f16, K, sums);
            Y[(size_t)n * (size_t)M + (size_t)row] = sums[0];
            Y[(size_t)n * (size_t)M + (size_t)row + 1] = sums[1];
            Y[(size_t)n * (size_t)M + (size_t)row + 2] = sums[2];
            Y[(size_t)n * (size_t)M + (size_t)row + 3] = sums[3];
        }
#endif
        for (; row < M; ++row) {
            const uint16_t *w_row = &W[(size_t)row * (size_t)K];
            const float sum = ck_dot_f16_f16_local(w_row, x_f16, K);
            Y[(size_t)n * (size_t)M + (size_t)row] = sum;
        }
    }
}

References ck_dot_f16_f16_local(), and ck_f32_to_f16_row_local().

Referenced by gemm_nt_f16().

gemm_f16_input_fp16_threadpool()

static int gemm_f16_input_fp16_threadpool ( float *  Y, const uint16_t *  W, const float *  X, int  M, int  N, int  K  )

static

Definition at line 930 of file gemm_kernels_f16.c.

{
    if (!ck_gemm_f16_threadpool_enabled(M, N, K)) {
        return 0;
    }
 
    ck_threadpool_t *pool = ck_threadpool_global();
    const int active = ck_gemm_f16_pick_active_threads(pool, M, N, K);
    if (!pool || active <= 1) {
        return 0;
    }
 
    ck_gemm_f16_input_fp16_args_t args = {
        .Y = Y,
        .W = W,
        .X = X,
        .M = M,
        .N = N,
        .K = K,
    };
    ck_threadpool_dispatch_n(pool, active, ck_gemm_f16_input_fp16_work, &args);
    return 1;
}

References ck_gemm_f16_input_fp16_work(), ck_gemm_f16_pick_active_threads(), ck_gemm_f16_threadpool_enabled(), ck_threadpool_dispatch_n(), and ck_threadpool_global().

Referenced by gemm_nt_f16().

gemm_f16_ref()

void gemm_f16_ref	(	float *	Y,
		const uint16_t *	W,
		const float *	X,
		int	M,
		int	N,
		int	K
	)

Matrix-matrix multiply with FP16 weights (scalar reference)

Parameters

Y	Output matrix [M x N]
W	Weight matrix in FP16 [M x K]
X	Input matrix [K x N]
M	Number of output rows
N	Batch size
K	Hidden dimension

Definition at line 699 of file gemm_kernels_f16.c.

{
    for (int n = 0; n < N; n++) {
        gemv_f16_ref(&Y[n * M], W, &X[n * K], M, K);
    }
}

References gemv_f16_ref().

Referenced by gemm_f16().

gemm_nt_f16()

void gemm_nt_f16	(	const float *	A,
		const void *	B,
		const float *	bias,
		float *	C,
		int	M,
		int	N,
		int	K
	)

NT GEMM wrapper for FP16 weights with the engine's standard ABI.

Contract: A: [M, K] fp32 activation matrix B: [N, K] fp16 weight matrix stored row-major (transposed layout) C: [M, N] fp32 output matrix

This wrapper follows llama.cpp's CPU F16 mul_mat contract: activation rows are rounded to FP16 first, then the dot runs as F16 x F16 with FP32 output accumulation. The lower-level gemm_f16() helper remains the direct F16-weight x FP32-activation operator for generic use.

Definition at line 970 of file gemm_kernels_f16.c.

{
    if (ck_strict_parity_enabled() &&
        gemm_nt_f16_ggml_strict(A, B, bias, C, M, N, K)) {
        return;
    }
 
    if (!gemm_f16_input_fp16_threadpool(C, (const uint16_t *)B, A, N, M, K)) {
        gemm_f16_input_fp16_ref(C, (const uint16_t *)B, A, N, M, K);
    }
 
    if (!bias) {
        return;
    }
 
#pragma omp parallel for schedule(static) if(M > 1)
    for (int i = 0; i < M; ++i) {
        float *c_row = C + (size_t)i * (size_t)N;
        for (int j = 0; j < N; ++j) {
            c_row[j] += bias[j];
        }
    }
}

References C, ck_strict_parity_enabled(), gemm_f16_input_fp16_ref(), gemm_f16_input_fp16_threadpool(), and gemm_nt_f16_ggml_strict().

gemm_nt_f16_clipped()

void gemm_nt_f16_clipped	(	const float *	A,
		const void *	B,
		const float *	bias,
		const float *	input_min,
		const float *	input_max,
		const float *	output_min,
		const float *	output_max,
		float *	C,
		int	M,
		int	N,
		int	K
	)

Definition at line 998 of file gemm_kernels_f16.c.

{
    const float in_min = input_min ? input_min[0] : -3.4028234663852886e38f;
    const float in_max = input_max ? input_max[0] : 3.4028234663852886e38f;
    const float out_min = output_min ? output_min[0] : -3.4028234663852886e38f;
    const float out_max = output_max ? output_max[0] : 3.4028234663852886e38f;
    const uint16_t *W = (const uint16_t *)B;
 
#pragma omp parallel for schedule(static) if(M > 1)
    for (int m = 0; m < M; ++m) {
        const float *a_row = A + (size_t)m * (size_t)K;
        uint16_t a_f16[K];
 
        for (int k = 0; k < K; ++k) {
            float x = a_row[k];
            if (x < in_min) x = in_min;
            if (x > in_max) x = in_max;
            a_f16[k] = fp32_to_fp16(x);
        }
 
        float *c_row = C + (size_t)m * (size_t)N;
        for (int n = 0; n < N; ++n) {
            const uint16_t *w_row = W + (size_t)n * (size_t)K;
            float sum = bias ? bias[n] : 0.0f;
            for (int k = 0; k < K; ++k) {
                sum += fp16_to_fp32(w_row[k]) * fp16_to_fp32(a_f16[k]);
            }
            if (sum < out_min) sum = out_min;
            if (sum > out_max) sum = out_max;
            c_row[n] = sum;
        }
    }
}

References C, fp16_to_fp32, and fp32_to_fp16.

gemm_nt_f16_ggml_strict()

static int gemm_nt_f16_ggml_strict ( const float *  A, const void *  B, const float *  bias, float *  C, int  M, int  N, int  K  )

static

Definition at line 163 of file gemm_kernels_f16.c.

{
    ck_f16_ggml_cpu_init_fn ggml_cpu_init_fn = ck_f16_resolve_ggml_cpu_init();
    ck_f16_ggml_init_fn ggml_init_fn = ck_f16_resolve_ggml_init();
    ck_f16_ggml_free_fn ggml_free_fn = ck_f16_resolve_ggml_free();
    ck_f16_ggml_new_tensor_2d_fn ggml_new_tensor_2d_fn = ck_f16_resolve_ggml_new_tensor_2d();
    ck_f16_ggml_mul_mat_fn ggml_mul_mat_fn = ck_f16_resolve_ggml_mul_mat();
    ck_f16_ggml_new_graph_fn ggml_new_graph_fn = ck_f16_resolve_ggml_new_graph();
    ck_f16_ggml_build_forward_expand_fn ggml_build_forward_expand_fn = ck_f16_resolve_ggml_build_forward_expand();
    ck_f16_ggml_graph_compute_with_ctx_fn ggml_graph_compute_with_ctx_fn = ck_f16_resolve_ggml_graph_compute_with_ctx();
    ck_f16_ggml_get_data_fn ggml_get_data_fn = ck_f16_resolve_ggml_get_data();
    ck_f16_ggml_get_data_f32_fn ggml_get_data_f32_fn = ck_f16_resolve_ggml_get_data_f32();
 
    if (!ggml_cpu_init_fn || !ggml_init_fn || !ggml_free_fn || !ggml_new_tensor_2d_fn ||
        !ggml_mul_mat_fn || !ggml_new_graph_fn || !ggml_build_forward_expand_fn ||
        !ggml_graph_compute_with_ctx_fn || !ggml_get_data_fn || !ggml_get_data_f32_fn) {
        return 0;
    }
 
    ggml_cpu_init_fn();
 
    const size_t output_bytes = (size_t) M * (size_t) N * sizeof(float);
    const size_t mem_size = ((size_t) 128 * 1024 * 1024) + output_bytes;
 
    struct ggml_init_params params = {
        .mem_size = mem_size,
        .mem_buffer = NULL,
        .no_alloc = false,
    };
    struct ggml_context *ctx = ggml_init_fn(params);
    if (!ctx) {
        return 0;
    }
 
    int ok = 0;
    struct ggml_tensor *w = ggml_new_tensor_2d_fn(ctx, GGML_TYPE_F16, K, N);
    struct ggml_tensor *x = ggml_new_tensor_2d_fn(ctx, GGML_TYPE_F32, K, M);
    if (!w || !x) {
        ggml_free_fn(ctx);
        return 0;
    }
 
    {
        void *w_data = ggml_get_data_fn(w);
        void *x_data = ggml_get_data_fn(x);
        if (!w_data || !x_data) {
            ggml_free_fn(ctx);
            return 0;
        }
        memcpy(w_data, B, (size_t) K * (size_t) N * sizeof(uint16_t));
        memcpy(x_data, A, (size_t) K * (size_t) M * sizeof(float));
    }
 
    struct ggml_tensor *y = ggml_mul_mat_fn(ctx, w, x);
    if (!y) {
        ggml_free_fn(ctx);
        return 0;
    }
 
    struct ggml_cgraph *gf = ggml_new_graph_fn(ctx);
    if (!gf) {
        ggml_free_fn(ctx);
        return 0;
    }
    ggml_build_forward_expand_fn(gf, y);
    if (ggml_graph_compute_with_ctx_fn(ctx, gf, 1) != GGML_STATUS_SUCCESS) {
        ggml_free_fn(ctx);
        return 0;
    }
 
    {
        const float *src = ggml_get_data_f32_fn(y);
        if (!src) {
            ggml_free_fn(ctx);
            return 0;
        }
        for (int m = 0; m < M; ++m) {
            memcpy(C + (size_t) m * (size_t) N,
                   src + (size_t) m * (size_t) N,
                   (size_t) N * sizeof(float));
            if (bias) {
                for (int n = 0; n < N; ++n) {
                    C[(size_t) m * (size_t) N + (size_t) n] += bias[n];
                }
            }
        }
    }
 
    ok = 1;
    ggml_free_fn(ctx);
    return ok;
}

References C, ck_f16_resolve_ggml_build_forward_expand(), ck_f16_resolve_ggml_cpu_init(), ck_f16_resolve_ggml_free(), ck_f16_resolve_ggml_get_data(), ck_f16_resolve_ggml_get_data_f32(), ck_f16_resolve_ggml_graph_compute_with_ctx(), ck_f16_resolve_ggml_init(), ck_f16_resolve_ggml_mul_mat(), ck_f16_resolve_ggml_new_graph(), ck_f16_resolve_ggml_new_tensor_2d(), GGML_STATUS_SUCCESS, GGML_TYPE_F16, GGML_TYPE_F32, and ggml_init_params::mem_size.

Referenced by ck_gemm_nt_f16_ggml_oracle(), and gemm_nt_f16().

gemv_f16()

void gemv_f16	(	float *	y,
		const uint16_t *	W,
		const float *	x,
		int	M,
		int	K
	)

Auto-dispatch GEMV based on available SIMD.

Definition at line 673 of file gemm_kernels_f16.c.

{
#ifdef __AVX512F__
    gemv_f16_avx512(y, W, x, M, K);
#else
    gemv_f16_ref(y, W, x, M, K);
#endif
}

References gemv_f16_ref().

Referenced by dot_f16().

gemv_f16_backward()

void gemv_f16_backward	(	float *	dX,
		const uint16_t *	W,
		const float *	dY,
		int	M,
		int	K
	)

Auto-dispatch backward.

Definition at line 1179 of file gemm_kernels_f16.c.

{
#ifdef __AVX512F__
    gemv_f16_backward_avx512(dX, W, dY, M, K);
#else
    gemv_f16_backward_ref(dX, W, dY, M, K);
#endif
}

References gemv_f16_backward_ref().

Referenced by gemm_f16_backward().

gemv_f16_backward_ref()

void gemv_f16_backward_ref	(	float *	dX,
		const uint16_t *	W,
		const float *	dY,
		int	M,
		int	K
	)

Backward pass: compute input gradient (scalar reference)

Parameters

dX	Output gradient w.r.t. input [K]
W	Weight matrix in FP16 format [M x K]
dY	Gradient w.r.t. output [M]
M	Number of output rows
K	Number of columns (input dimension)

Definition at line 1110 of file gemm_kernels_f16.c.

{
    /* Zero output gradient */
    for (int k = 0; k < K; k++) {
        dX[k] = 0.0f;
    }
 
    /* Accumulate: dX += W^T @ dY */
    for (int row = 0; row < M; row++) {
        const float dy = dY[row];
        const uint16_t *w_row = &W[row * K];
 
        for (int k = 0; k < K; k++) {
            float w = fp16_to_fp32(w_row[k]);
            dX[k] += w * dy;
        }
    }
}

References fp16_to_fp32.

Referenced by gemv_f16_backward().

gemv_f16_ref()

void gemv_f16_ref	(	float *	y,
		const uint16_t *	W,
		const float *	x,
		int	M,
		int	K
	)

Matrix-vector multiply with FP16 weights (scalar reference)

Parameters

y	Output vector [M]
W	Weight matrix in FP16 [M x K]
x	Input vector [K]
M	Number of output rows
K	Number of columns

Definition at line 607 of file gemm_kernels_f16.c.

{
    for (int row = 0; row < M; row++) {
        float sum = 0.0f;
        const uint16_t *w_row = &W[row * K];
 
        for (int k = 0; k < K; k++) {
            float w = fp16_to_fp32(w_row[k]);
            sum += w * x[k];
        }
 
        y[row] = sum;
    }
}

References fp16_to_fp32.

Referenced by gemm_f16_ref(), and gemv_f16().