gemm_kernels_f16.c File Reference

GEMM kernels with FP16 (half-precision) weights. More...

#include <stdint.h>
#include <stddef.h>
#include "ckernel_quant.h"

Go to the source code of this file.

Macros
#define	fp16_to_fp32(x)   ggml_fp16_to_fp32(x)
#define	fp32_to_fp16(x)   ggml_fp32_to_fp16(x)

Functions
void	convert_f16_to_f32 (float *dst, const uint16_t *src, size_t count)
	Convert FP16 tensor to FP32. More...
void	convert_f32_to_f16 (uint16_t *dst, const float *src, size_t count)
	Convert FP32 tensor to FP16. More...
float	dot_f16 (const uint16_t *w_f16, const float *x, int K)
void	gemm_f16 (float *Y, const uint16_t *W, const float *X, int M, int N, int K)
	Auto-dispatch GEMM based on available SIMD. More...
void	gemm_f16_backward (float *dX, const uint16_t *W, const float *dY, int M, int N, int K)
	Batched backward pass. More...
void	gemm_f16_ref (float *Y, const uint16_t *W, const float *X, int M, int N, int K)
	Matrix-matrix multiply with FP16 weights (scalar reference) More...
void	gemv_f16 (float *y, const uint16_t *W, const float *x, int M, int K)
	Auto-dispatch GEMV based on available SIMD. More...
void	gemv_f16_backward (float *dX, const uint16_t *W, const float *dY, int M, int K)
	Auto-dispatch backward. More...
void	gemv_f16_backward_ref (float *dX, const uint16_t *W, const float *dY, int M, int K)
	Backward pass: compute input gradient (scalar reference) More...
void	gemv_f16_ref (float *y, const uint16_t *W, const float *x, int M, int K)
	Matrix-vector multiply with FP16 weights (scalar reference) More...

Detailed Description

GEMM kernels with FP16 (half-precision) weights.

CK-ENGINE KERNEL RULES:

1. NO malloc/free - memory via bump allocator, pointers passed in
2. NO OpenMP - parallelization at orchestrator/codegen layer
3. API must define: inputs, outputs, workspace, and memory layouts
4. Pure computation - deterministic, no side effects

After changes: make test && make llamacpp-parity-full

Implements matrix multiplication where:

• Weights: FP16 (IEEE half-precision, used by vision encoders)
• Activations: FP32
• Output: FP32

Used for multimodal projection layers (mmproj-*.gguf files).

Definition in file gemm_kernels_f16.c.

Macro Definition Documentation

fp16_to_fp32

#define fp16_to_fp32

(

x

)

ggml_fp16_to_fp32(x)

Definition at line 36 of file gemm_kernels_f16.c.

fp32_to_fp16

#define fp32_to_fp16

(

x

)

ggml_fp32_to_fp16(x)

Definition at line 37 of file gemm_kernels_f16.c.

Function Documentation

convert_f16_to_f32()

void convert_f16_to_f32	(	float *	dst,
		const uint16_t *	src,
		size_t	count
	)

Convert FP16 tensor to FP32.

Definition at line 226 of file gemm_kernels_f16.c.

{
#ifdef __AVX512F__
    const size_t count16 = count / 16 * 16;
 
    for (size_t i = 0; i < count16; i += 16) {
        __m256i f16 = _mm256_loadu_si256((const __m256i *)&src[i]);
        __m512 f32 = _mm512_cvtph_ps(f16);
        _mm512_storeu_ps(&dst[i], f32);
    }
 
    for (size_t i = count16; i < count; i++) {
        dst[i] = fp16_to_fp32(src[i]);
    }
#else
    for (size_t i = 0; i < count; i++) {
        dst[i] = fp16_to_fp32(src[i]);
    }
#endif
}

References fp16_to_fp32.

convert_f32_to_f16()

void convert_f32_to_f16	(	uint16_t *	dst,
		const float *	src,
		size_t	count
	)

Convert FP32 tensor to FP16.

Definition at line 250 of file gemm_kernels_f16.c.

{
#ifdef __AVX512F__
    const size_t count16 = count / 16 * 16;
 
    for (size_t i = 0; i < count16; i += 16) {
        __m512 f32 = _mm512_loadu_ps(&src[i]);
        __m256i f16 = _mm512_cvtps_ph(f32, 0);
        _mm256_storeu_si256((__m256i *)&dst[i], f16);
    }
 
    for (size_t i = count16; i < count; i++) {
        dst[i] = fp32_to_fp16(src[i]);
    }
#else
    for (size_t i = 0; i < count; i++) {
        dst[i] = fp32_to_fp16(src[i]);
    }
#endif
}

References fp32_to_fp16.

dot_f16()

float dot_f16	(	const uint16_t *	w_f16,
		const float *	x,
		int	K
	)

Definition at line 387 of file gemm_kernels_f16.c.

{
    float result;
    gemv_f16(&result, w_f16, x, 1, K);
    return result;
}

References gemv_f16().

gemm_f16()

void gemm_f16	(	float *	Y,
		const uint16_t *	W,
		const float *	X,
		int	M,
		int	N,
		int	K
	)

Auto-dispatch GEMM based on available SIMD.

Definition at line 207 of file gemm_kernels_f16.c.

{
#ifdef __AVX512F__
    gemm_f16_avx512(Y, W, X, M, N, K);
#else
    gemm_f16_ref(Y, W, X, M, N, K);
#endif
}

References gemm_f16_ref().

gemm_f16_backward()

void gemm_f16_backward	(	float *	dX,
		const uint16_t *	W,
		const float *	dY,
		int	M,
		int	N,
		int	K
	)

Batched backward pass.

Definition at line 373 of file gemm_kernels_f16.c.

{
    for (int n = 0; n < N; n++) {
        gemv_f16_backward(&dX[n * K], W, &dY[n * M], M, K);
    }
}

References gemv_f16_backward().

gemm_f16_ref()

void gemm_f16_ref	(	float *	Y,
		const uint16_t *	W,
		const float *	X,
		int	M,
		int	N,
		int	K
	)

Matrix-matrix multiply with FP16 weights (scalar reference)

Parameters

Y	Output matrix [M x N]
W	Weight matrix in FP16 [M x K]
X	Input matrix [K x N]
M	Number of output rows
N	Batch size
K	Hidden dimension

Definition at line 154 of file gemm_kernels_f16.c.

{
    for (int n = 0; n < N; n++) {
        gemv_f16_ref(&Y[n * M], W, &X[n * K], M, K);
    }
}

References gemv_f16_ref().

Referenced by gemm_f16().

gemv_f16()

void gemv_f16	(	float *	y,
		const uint16_t *	W,
		const float *	x,
		int	M,
		int	K
	)

Auto-dispatch GEMV based on available SIMD.

Definition at line 128 of file gemm_kernels_f16.c.

{
#ifdef __AVX512F__
    gemv_f16_avx512(y, W, x, M, K);
#else
    gemv_f16_ref(y, W, x, M, K);
#endif
}

References gemv_f16_ref().

Referenced by dot_f16().

gemv_f16_backward()

void gemv_f16_backward	(	float *	dX,
		const uint16_t *	W,
		const float *	dY,
		int	M,
		int	K
	)

Auto-dispatch backward.

Definition at line 358 of file gemm_kernels_f16.c.

{
#ifdef __AVX512F__
    gemv_f16_backward_avx512(dX, W, dY, M, K);
#else
    gemv_f16_backward_ref(dX, W, dY, M, K);
#endif
}

References gemv_f16_backward_ref().

Referenced by gemm_f16_backward().

gemv_f16_backward_ref()

void gemv_f16_backward_ref	(	float *	dX,
		const uint16_t *	W,
		const float *	dY,
		int	M,
		int	K
	)

Backward pass: compute input gradient (scalar reference)

Parameters

dX	Output gradient w.r.t. input [K]
W	Weight matrix in FP16 format [M x K]
dY	Gradient w.r.t. output [M]
M	Number of output rows
K	Number of columns (input dimension)

Definition at line 289 of file gemm_kernels_f16.c.

{
    /* Zero output gradient */
    for (int k = 0; k < K; k++) {
        dX[k] = 0.0f;
    }
 
    /* Accumulate: dX += W^T @ dY */
    for (int row = 0; row < M; row++) {
        const float dy = dY[row];
        const uint16_t *w_row = &W[row * K];
 
        for (int k = 0; k < K; k++) {
            float w = fp16_to_fp32(w_row[k]);
            dX[k] += w * dy;
        }
    }
}

References fp16_to_fp32.

Referenced by gemv_f16_backward().

gemv_f16_ref()

void gemv_f16_ref	(	float *	y,
		const uint16_t *	W,
		const float *	x,
		int	M,
		int	K
	)

Matrix-vector multiply with FP16 weights (scalar reference)

Parameters

y	Output vector [M]
W	Weight matrix in FP16 [M x K]
x	Input vector [K]
M	Number of output rows
K	Number of columns

Definition at line 62 of file gemm_kernels_f16.c.

{
    for (int row = 0; row < M; row++) {
        float sum = 0.0f;
        const uint16_t *w_row = &W[row * K];
 
        for (int k = 0; k < K; k++) {
            float w = fp16_to_fp32(w_row[k]);
            sum += w * x[k];
        }
 
        y[row] = sum;
    }
}

References fp16_to_fp32.

Referenced by gemm_f16_ref(), and gemv_f16().