gemm_kernels_q8_0.c File Reference

GEMM/GEMV kernels with Q8_0 quantized weights. More...

#include <stdint.h>
#include <stddef.h>
#include <string.h>
#include "ckernel_quant.h"
#include "ck_features.h"

Go to the source code of this file.

Functions
float	dot_q8_0 (const void *w_q8_0, const float *x, int K)
void	gemm_nt_q8_0 (const float *A, const void *B, const float *bias, float *C, int M, int N, int K)
	Matrix-matrix multiply: C[M,N] = A[M,K] @ B[N,K]^T + bias. More...
void	gemm_q8_0 (float *Y, const void *W, const float *X, int M, int N, int K)
	Matrix-matrix multiply with Q8_0 weights. More...
void	gemm_q8_0_backward (float *dX, const void *W, const float *dY, int M, int N, int K)
	Batched backward pass. More...
void	gemv_q8_0 (float *y, const void *W, const float *x, int M, int K)
	Auto-dispatch GEMV for Q8_0 weights based on CPU features. More...
void	gemv_q8_0_backward (float *dX, const void *W, const float *dY, int M, int K)
	Auto-dispatch backward. More...
void	gemv_q8_0_backward_ref (float *dX, const void *W, const float *dY, int M, int K)
	Backward pass: compute input gradient (scalar reference) More...
void	gemv_q8_0_parallel_simd (float *y, const void *W, const float *x, int M, int K, int ith, int nth)
	Parallel SIMD GEMV for Q8_0 weights x FP32 input with prefetching. More...
void	gemv_q8_0_q8_0 (float *y, const void *W, const void *x_q8, int M, int K)
	Matrix-vector multiply with Q8_0 weights and Q8_0 input. More...
void	gemv_q8_0_q8_0_parallel (float *y, const void *W, const void *x_q8, int M, int K, int ith, int nth)
	Parallel reference GEMV for Q8_0 x Q8_0. More...
void	gemv_q8_0_q8_0_parallel_simd (float *y, const void *W, const void *x_q8, int M, int K, int ith, int nth)
	Parallel SIMD GEMV for Q8_0 x Q8_0 with prefetching. More...
void	gemv_q8_0_ref (float *y, const void *W, const float *x, int M, int K)
	Matrix-vector multiply with Q8_0 weights (scalar reference) More...
void	quantize_batch_q8_0 (const float *x, void *vy, int num_rows, int k)
	Batch quantize FP32 to Q8_0 format (row-major output) More...
void	quantize_batch_q8_k (const float *x, void *vy, int num_rows, int k)
	Batch quantize FP32 to Q8_K format (row-major output) More...
void	quantize_row_q8_0 (const float *x, void *vy, int k)
	Quantize FP32 to Q8_0 format (scalar reference) More...
void	quantize_row_q8_k (const float *x, void *vy, int k)
void	vec_dot_q8_0_q8_0 (int n, float *s, const void *vx, const void *vy)
	Auto-dispatch quantized dot product Q8_0 x Q8_0. More...
void	vec_dot_q8_0_q8_0_ref (int n, float *s, const void *vx, const void *vy)
	Quantized dot product: Q8_0 weights x Q8_0 input (scalar reference) More...

Detailed Description

GEMM/GEMV kernels with Q8_0 quantized weights.

CK-ENGINE KERNEL RULES:

1. NO malloc/free - memory via bump allocator, pointers passed in
2. NO OpenMP - parallelization at orchestrator/codegen layer
3. API must define: inputs, outputs, workspace, and memory layouts
4. Pure computation - deterministic, no side effects

After changes: make test && make llamacpp-parity-full

Q8_0 Format:

• 32 weights per block
• 1 FP16 scale per block
• 34 bytes per 32 weights = 8.5 bits/weight
• Weights stored as signed 8-bit integers

Operations: Forward: Y = W @ X (W is Q8_0, X and Y are FP32) Backward: dX = W^T @ dY (gradient w.r.t. input)

Note: Q8_0 is often used for activation quantization or as an intermediate format. Higher precision than Q4_0/Q4_K.

Definition in file gemm_kernels_q8_0.c.

Function Documentation

dot_q8_0()

float dot_q8_0	(	const void *	w_q8_0,
		const float *	x,
		int	K
	)

Definition at line 834 of file gemm_kernels_q8_0.c.

{
    float result;
    gemv_q8_0(&result, w_q8_0, x, 1, K);
    return result;
}

References gemv_q8_0().

gemm_nt_q8_0()

void gemm_nt_q8_0	(	const float *	A,
		const void *	B,
		const float *	bias,
		float *	C,
		int	M,
		int	N,
		int	K
	)

Matrix-matrix multiply: C[M,N] = A[M,K] @ B[N,K]^T + bias.

Parameters

A	Input matrix [M x K], row-major FP32
B	Weight matrix in Q8_0 format, [N x K] stored row-major
bias	Optional bias [N], NULL if not used
C	Output [M x N], row-major FP32
M	Batch size (number of tokens)
N	Output dimension (number of rows in B)
K	Input dimension

Definition at line 681 of file gemm_kernels_q8_0.c.

{
    /* Use GEMV dispatch which selects AVX/SSE/scalar based on CPU */
    for (int m = 0; m < M; m++) {
        gemv_q8_0(&C[m * N], B, &A[m * K], N, K);
        if (bias) {
            for (int n = 0; n < N; n++) C[m * N + n] += bias[n];
        }
    }
    return;
 
    const block_q8_0 *blocks = (const block_q8_0 *)B;
    const int blocks_per_row = K / QK8_0;
 
    for (int m = 0; m < M; m++) {
        const float *a_row = &A[m * K];
 
        for (int n = 0; n < N; n++) {
            float sum = 0.0f;
 
            for (int b = 0; b < blocks_per_row; b++) {
                const block_q8_0 *block = &blocks[n * blocks_per_row + b];
                const float d = CK_FP16_TO_FP32(block->d);
                const float *ap = &a_row[b * QK8_0];
 
                for (int i = 0; i < QK8_0; i++) {
                    sum += d * (float)block->qs[i] * ap[i];
                }
            }
 
            C[m * N + n] = sum + (bias ? bias[n] : 0.0f);
        }
    }
}

References C, CK_FP16_TO_FP32, block_q8_0::d, gemv_q8_0(), QK8_0, and block_q8_0::qs.

Referenced by ck_gemm_nt_quant(), qwen2_0_5b_decode_decode_token(), qwen2_0_5b_decode_forward_prefill_impl(), qwen2_0_5b_decode_layer_0_decode(), qwen2_0_5b_decode_layer_10_decode(), qwen2_0_5b_decode_layer_13_decode(), qwen2_0_5b_decode_layer_16_decode(), qwen2_0_5b_decode_layer_19_decode(), qwen2_0_5b_decode_layer_1_decode(), qwen2_0_5b_decode_layer_21_decode(), qwen2_0_5b_decode_layer_3_decode(), qwen2_0_5b_decode_layer_6_decode(), qwen2_0_5b_decode_layer_7_decode(), qwen2_0_5b_decode_layer_8_decode(), and qwen2_0_5b_decode_layer_9_decode().

gemm_q8_0()

void gemm_q8_0	(	float *	Y,
		const void *	W,
		const float *	X,
		int	M,
		int	N,
		int	K
	)

Matrix-matrix multiply with Q8_0 weights.

Definition at line 656 of file gemm_kernels_q8_0.c.

{
    for (int n = 0; n < N; n++) {
        gemv_q8_0(&Y[n * M], W, &X[n * K], M, K);
    }
}

References gemv_q8_0().

gemm_q8_0_backward()

void gemm_q8_0_backward	(	float *	dX,
		const void *	W,
		const float *	dY,
		int	M,
		int	N,
		int	K
	)

Batched backward pass.

Definition at line 820 of file gemm_kernels_q8_0.c.

{
    for (int n = 0; n < N; n++) {
        gemv_q8_0_backward(&dX[n * K], W, &dY[n * M], M, K);
    }
}

References gemv_q8_0_backward().

gemv_q8_0()

void gemv_q8_0	(	float *	y,
		const void *	W,
		const float *	x,
		int	M,
		int	K
	)

Auto-dispatch GEMV for Q8_0 weights based on CPU features.

Dispatch priority (best available):

1. AVX-512 (512-bit vectors) - Intel Skylake-X+
2. AVX2+FMA (256-bit vectors) - Intel Haswell+
3. AVX (256-bit vectors) - Intel Sandy Bridge+
4. SSE4.1 (128-bit vectors) - Intel Nehalem+
5. Reference (scalar) - Fallback

Uses ck_features.h for standardized feature detection.

Parameters

y	Output vector [M]
W	Weight matrix in Q8_0 format [M x K]
x	Input vector [K]
M	Number of output rows
K	Number of input columns (hidden dimension)

Definition at line 630 of file gemm_kernels_q8_0.c.

{
// Dispatch order: AVX512 > AVX2 > AVX > SSE > ref
#if defined(__AVX512F__)
    gemv_q8_0_avx512(y, W, x, M, K);
#elif defined(__AVX2__)
    gemv_q8_0_avx2(y, W, x, M, K);
#elif defined(__AVX__)
    gemv_q8_0_avx(y, W, x, M, K);
#elif defined(__SSE4_1__)
    gemv_q8_0_sse(y, W, x, M, K);
#else
    gemv_q8_0_ref(y, W, x, M, K);
#endif
}

References gemv_q8_0_ref().

Referenced by dot_q8_0(), gemm_nt_q8_0(), and gemm_q8_0().

gemv_q8_0_backward()

void gemv_q8_0_backward	(	float *	dX,
		const void *	W,
		const float *	dY,
		int	M,
		int	K
	)

Auto-dispatch backward.

Definition at line 805 of file gemm_kernels_q8_0.c.

{
#ifdef __AVX512F__
    gemv_q8_0_backward_avx512(dX, W, dY, M, K);
#else
    gemv_q8_0_backward_ref(dX, W, dY, M, K);
#endif
}

References gemv_q8_0_backward_ref().

Referenced by gemm_q8_0_backward().

gemv_q8_0_backward_ref()

void gemv_q8_0_backward_ref	(	float *	dX,
		const void *	W,
		const float *	dY,
		int	M,
		int	K
	)

Backward pass: compute input gradient (scalar reference)

Parameters

dX	Output gradient w.r.t. input [K]
W	Weight matrix in Q8_0 format [M x K]
dY	Gradient w.r.t. output [M]
M	Number of output rows
K	Number of columns (input dimension)

Definition at line 733 of file gemm_kernels_q8_0.c.

{
    const block_q8_0 *blocks = (const block_q8_0 *)W;
    const int blocks_per_row = K / QK8_0;
 
    /* Zero output gradient */
    memset(dX, 0, K * sizeof(float));
 
    /* Accumulate: dX += W^T @ dY */
    for (int row = 0; row < M; row++) {
        const float dy = dY[row];
 
        for (int b = 0; b < blocks_per_row; b++) {
            const block_q8_0 *block = &blocks[row * blocks_per_row + b];
            const float d = CK_FP16_TO_FP32(block->d);
            float *dxp = &dX[b * QK8_0];
 
            for (int i = 0; i < QK8_0; i++) {
                dxp[i] += d * (float)block->qs[i] * dy;
            }
        }
    }
}

References CK_FP16_TO_FP32, block_q8_0::d, QK8_0, and block_q8_0::qs.

Referenced by gemv_q8_0_backward().

gemv_q8_0_parallel_simd()

void gemv_q8_0_parallel_simd	(	float *	y,
		const void *	W,
		const float *	x,
		int	M,
		int	K,
		int	ith,
		int	nth
	)

Parallel SIMD GEMV for Q8_0 weights x FP32 input with prefetching.

Definition at line 1153 of file gemm_kernels_q8_0.c.

{
    if (!y || !W || !x || M <= 0 || K <= 0) return;
    if (ith < 0 || nth <= 0 || ith >= nth) return;
 
    const int dr = (M + nth - 1) / nth;
    const int r0 = dr * ith;
    const int r1 = (r0 + dr < M) ? (r0 + dr) : M;
 
    if (r0 >= M) return;
 
    const block_q8_0 *blocks = (const block_q8_0 *)W;
    const int blocks_per_row = K / QK8_0;
 
#if defined(__AVX__) || defined(__SSE4_1__)
    const int PREFETCH_ROWS = 4;
    for (int p = 0; p < PREFETCH_ROWS && r0 + p < r1; ++p) {
        const char *row_ptr = (const char *)(blocks + (r0 + p) * blocks_per_row);
        _mm_prefetch(row_ptr, _MM_HINT_T0);
        _mm_prefetch(row_ptr + 64, _MM_HINT_T0);
    }
 
    for (int row = r0; row < r1; ++row) {
        if (row + PREFETCH_ROWS < r1) {
            const char *pf = (const char *)(blocks + (row + PREFETCH_ROWS) * blocks_per_row);
            _mm_prefetch(pf, _MM_HINT_T0);
            _mm_prefetch(pf + 64, _MM_HINT_T0);
        }
 
        /* Dispatch to best available SIMD for single row */
#if defined(__AVX512F__)
        gemv_q8_0_avx512(&y[row],
                          (const char *)blocks + row * blocks_per_row * sizeof(block_q8_0),
                          x, 1, K);
#elif defined(__AVX2__)
        gemv_q8_0_avx2(&y[row],
                        (const char *)blocks + row * blocks_per_row * sizeof(block_q8_0),
                        x, 1, K);
#elif defined(__AVX__)
        gemv_q8_0_avx(&y[row],
                       (const char *)blocks + row * blocks_per_row * sizeof(block_q8_0),
                       x, 1, K);
#elif defined(__SSE4_1__)
        gemv_q8_0_sse(&y[row],
                       (const char *)blocks + row * blocks_per_row * sizeof(block_q8_0),
                       x, 1, K);
#else
        gemv_q8_0_ref(&y[row],
                       (const char *)blocks + row * blocks_per_row * sizeof(block_q8_0),
                       x, 1, K);
#endif
    }
#else
    for (int row = r0; row < r1; row++) {
        gemv_q8_0_ref(&y[row],
                       (const char *)blocks + row * blocks_per_row * sizeof(block_q8_0),
                       x, 1, K);
    }
#endif
}

References gemv_q8_0_ref(), and QK8_0.

gemv_q8_0_q8_0()

void gemv_q8_0_q8_0	(	float *	y,
		const void *	W,
		const void *	x_q8,
		int	M,
		int	K
	)

Matrix-vector multiply with Q8_0 weights and Q8_0 input.

Parameters

y	Output vector [M]
W	Weight matrix in Q8_0 format [M x K]
x_q8	Input vector in Q8_0 format [K]
M	Number of output rows
K	Number of columns (must be multiple of 32)

Definition at line 1042 of file gemm_kernels_q8_0.c.

{
    const block_q8_0 *w_blocks = (const block_q8_0 *)W;
    const block_q8_0 *x_blocks = (const block_q8_0 *)x_q8;
    const int blocks_per_row = K / QK8_0;
 
    for (int row = 0; row < M; row++) {
        vec_dot_q8_0_q8_0(K, &y[row],
                          &w_blocks[row * blocks_per_row],
                          x_blocks);
    }
}

References QK8_0, and vec_dot_q8_0_q8_0().

Referenced by ck_test_gemv_q8_0(), and ck_test_gemv_q8_0_q8_0().

gemv_q8_0_q8_0_parallel()

void gemv_q8_0_q8_0_parallel	(	float *	y,
		const void *	W,
		const void *	x_q8,
		int	M,
		int	K,
		int	ith,
		int	nth
	)

Parallel reference GEMV for Q8_0 x Q8_0.

Definition at line 1068 of file gemm_kernels_q8_0.c.

{
    if (!y || !W || !x_q8 || M <= 0 || K <= 0) return;
    if (ith < 0 || nth <= 0 || ith >= nth) return;
 
    const int dr = (M + nth - 1) / nth;
    const int r0 = dr * ith;
    const int r1 = (r0 + dr < M) ? (r0 + dr) : M;
 
    if (r0 >= M) return;
 
    const block_q8_0 *w_blocks = (const block_q8_0 *)W;
    const block_q8_0 *x_blocks = (const block_q8_0 *)x_q8;
    const int blocks_per_row = K / QK8_0;
 
    for (int row = r0; row < r1; row++) {
        vec_dot_q8_0_q8_0(K, &y[row],
                          &w_blocks[row * blocks_per_row],
                          x_blocks);
    }
}

References QK8_0, and vec_dot_q8_0_q8_0().

gemv_q8_0_q8_0_parallel_simd()

void gemv_q8_0_q8_0_parallel_simd	(	float *	y,
		const void *	W,
		const void *	x_q8,
		int	M,
		int	K,
		int	ith,
		int	nth
	)

Parallel SIMD GEMV for Q8_0 x Q8_0 with prefetching.

Each thread processes rows [r0, r1) where r0 = ith * ceil(M/nth). Prefetches upcoming weight rows to hide memory latency.

Definition at line 1100 of file gemm_kernels_q8_0.c.

{
    if (!y || !W || !x_q8 || M <= 0 || K <= 0) return;
    if (ith < 0 || nth <= 0 || ith >= nth) return;
 
    const int dr = (M + nth - 1) / nth;
    const int r0 = dr * ith;
    const int r1 = (r0 + dr < M) ? (r0 + dr) : M;
 
    if (r0 >= M) return;
 
    const block_q8_0 *w_blocks = (const block_q8_0 *)W;
    const block_q8_0 *x_blocks = (const block_q8_0 *)x_q8;
    const int blocks_per_row = K / QK8_0;
 
#if defined(__AVX__) || defined(__SSE4_1__)
    /* Prefetch first few rows */
    const int PREFETCH_ROWS = 4;
    for (int p = 0; p < PREFETCH_ROWS && r0 + p < r1; ++p) {
        const char *row_ptr = (const char *)(w_blocks + (r0 + p) * blocks_per_row);
        _mm_prefetch(row_ptr, _MM_HINT_T0);
        _mm_prefetch(row_ptr + 64, _MM_HINT_T0);
    }
 
    for (int row = r0; row < r1; ++row) {
        /* Prefetch upcoming rows */
        if (row + PREFETCH_ROWS < r1) {
            const char *pf = (const char *)(w_blocks + (row + PREFETCH_ROWS) * blocks_per_row);
            _mm_prefetch(pf, _MM_HINT_T0);
            _mm_prefetch(pf + 64, _MM_HINT_T0);
        }
 
        vec_dot_q8_0_q8_0(K, &y[row],
                          &w_blocks[row * blocks_per_row],
                          x_blocks);
    }
#else
    /* Fallback: no prefetching */
    for (int row = r0; row < r1; row++) {
        vec_dot_q8_0_q8_0(K, &y[row],
                          &w_blocks[row * blocks_per_row],
                          x_blocks);
    }
#endif
}

References QK8_0, and vec_dot_q8_0_q8_0().

gemv_q8_0_ref()

void gemv_q8_0_ref	(	float *	y,
		const void *	W,
		const float *	x,
		int	M,
		int	K
	)

Matrix-vector multiply with Q8_0 weights (scalar reference)

Parameters

y	Output vector [M]
W	Weight matrix in Q8_0 format [M x K]
x	Input vector [K]
M	Number of output rows
K	Number of columns (must be multiple of 32)

Definition at line 252 of file gemm_kernels_q8_0.c.

{
    const block_q8_0 *blocks = (const block_q8_0 *)W;
    const int blocks_per_row = K / QK8_0;
 
    for (int row = 0; row < M; row++) {
        float sum = 0.0f;
 
        for (int b = 0; b < blocks_per_row; b++) {
            const block_q8_0 *block = &blocks[row * blocks_per_row + b];
            const float d = CK_FP16_TO_FP32(block->d);
            const float *xp = &x[b * QK8_0];
 
            for (int i = 0; i < QK8_0; i++) {
                sum += d * (float)block->qs[i] * xp[i];
            }
        }
 
        y[row] = sum;
    }
}

References CK_FP16_TO_FP32, block_q8_0::d, QK8_0, and block_q8_0::qs.

Referenced by gemv_q8_0(), and gemv_q8_0_parallel_simd().

quantize_batch_q8_0()

void quantize_batch_q8_0	(	const float *	x,
		void *	vy,
		int	num_rows,
		int	k
	)

Batch quantize FP32 to Q8_0 format (row-major output)

Quantizes multiple rows of FP32 data to Q8_0 format, placing each row's Q8_0 output at the correct byte offset for GEMM compatibility.

Memory layout: Input: [num_rows, k] FP32, row-major (stride = k * sizeof(float)) Output: [num_rows, q8_row_bytes] Q8_0, row-major (stride = q8_row_bytes)

where q8_row_bytes = (k / 32) * sizeof(block_q8_0) = (k / 32) * 34

Parameters

x	Input FP32 values [num_rows * k]
vy	Output Q8_0 blocks [num_rows * (k/32) blocks]
num_rows	Number of rows (batch size / tokens)
k	Elements per row (must be multiple of 32)

Definition at line 192 of file gemm_kernels_q8_0.c.

{
    const size_t row_bytes_in = (size_t)k * sizeof(float);
    const size_t row_bytes_out = (size_t)(k / QK8_0) * sizeof(block_q8_0);
 
    uint8_t *out = (uint8_t *)vy;
    const uint8_t *in = (const uint8_t *)x;
 
    for (int row = 0; row < num_rows; ++row) {
        quantize_row_q8_0(
            (const float *)(in + row * row_bytes_in),
            (void *)(out + row * row_bytes_out),
            k
        );
    }
}

References QK8_0, and quantize_row_q8_0().

quantize_batch_q8_k()

void quantize_batch_q8_k	(	const float *	x,
		void *	vy,
		int	num_rows,
		int	k
	)

Batch quantize FP32 to Q8_K format (row-major output)

Same as quantize_batch_q8_0 but for Q8_K format (super-blocks).

Parameters

x	Input FP32 values [num_rows * k]
vy	Output Q8_K blocks
num_rows	Number of rows (batch size / tokens)
k	Elements per row (must be multiple of 256)

Definition at line 219 of file gemm_kernels_q8_0.c.

{
    /* Q8_K: 256 elements per super-block, each block is larger */
    const size_t row_bytes_in = (size_t)k * sizeof(float);
    /* Q8_K block size = 2 (d) + 256 (qs) + 32 (bsums/2) = ~274 bytes for 256 elements */
    /* Actual: sizeof(block_q8_K) from ckernel_quant.h */
    const size_t row_bytes_out = (size_t)(k / 256) * sizeof(block_q8_K);
 
    uint8_t *out = (uint8_t *)vy;
    const uint8_t *in = (const uint8_t *)x;
 
    for (int row = 0; row < num_rows; ++row) {
        quantize_row_q8_k(
            (const float *)(in + row * row_bytes_in),
            (void *)(out + row * row_bytes_out),
            k
        );
    }
}

References quantize_row_q8_k().

quantize_row_q8_0()

void quantize_row_q8_0	(	const float *	x,
		void *	vy,
		int	k
	)

Quantize FP32 to Q8_0 format (scalar reference)

Parameters

x	Input FP32 values
vy	Output Q8_0 blocks
k	Number of elements (must be multiple of 32)

Definition at line 59 of file gemm_kernels_q8_0.c.

{
    block_q8_0 *y = (block_q8_0 *)vy;
    const int nb = k / QK8_0;  /* QK8_0 = 32 */
 
#if defined(__AVX__)
    const __m256 sign_bit = _mm256_set1_ps(-0.0f);
    const __m256 v_half = _mm256_set1_ps(0.5f);
    const __m256 v_min = _mm256_set1_ps(-127.0f);
    const __m256 v_max = _mm256_set1_ps(127.0f);
 
    for (int i = 0; i < nb; i++) {
        __m256 v0 = _mm256_loadu_ps(x + 0);
        __m256 v1 = _mm256_loadu_ps(x + 8);
        __m256 v2 = _mm256_loadu_ps(x + 16);
        __m256 v3 = _mm256_loadu_ps(x + 24);
        x += QK8_0;
 
        __m256 max_abs = _mm256_andnot_ps(sign_bit, v0);
        max_abs = _mm256_max_ps(max_abs, _mm256_andnot_ps(sign_bit, v1));
        max_abs = _mm256_max_ps(max_abs, _mm256_andnot_ps(sign_bit, v2));
        max_abs = _mm256_max_ps(max_abs, _mm256_andnot_ps(sign_bit, v3));
 
        __m128 max4 = _mm_max_ps(_mm256_extractf128_ps(max_abs, 1),
                                 _mm256_castps256_ps128(max_abs));
        max4 = _mm_max_ps(max4, _mm_movehl_ps(max4, max4));
        max4 = _mm_max_ss(max4, _mm_movehdup_ps(max4));
        const float max_scalar = _mm_cvtss_f32(max4);
 
        const float d = max_scalar / 127.0f;
        const float id = max_scalar != 0.0f ? 127.0f / max_scalar : 0.0f;
        y[i].d = CK_FP32_TO_FP16(d);
 
        const __m256 mul = _mm256_set1_ps(id);
        v0 = _mm256_mul_ps(v0, mul);
        v1 = _mm256_mul_ps(v1, mul);
        v2 = _mm256_mul_ps(v2, mul);
        v3 = _mm256_mul_ps(v3, mul);
 
        v0 = _mm256_min_ps(_mm256_max_ps(v0, v_min), v_max);
        v1 = _mm256_min_ps(_mm256_max_ps(v1, v_min), v_max);
        v2 = _mm256_min_ps(_mm256_max_ps(v2, v_min), v_max);
        v3 = _mm256_min_ps(_mm256_max_ps(v3, v_min), v_max);
 
        /* Round half away from zero to match the scalar path */
        v0 = _mm256_add_ps(v0, _mm256_or_ps(_mm256_and_ps(v0, sign_bit), v_half));
        v1 = _mm256_add_ps(v1, _mm256_or_ps(_mm256_and_ps(v1, sign_bit), v_half));
        v2 = _mm256_add_ps(v2, _mm256_or_ps(_mm256_and_ps(v2, sign_bit), v_half));
        v3 = _mm256_add_ps(v3, _mm256_or_ps(_mm256_and_ps(v3, sign_bit), v_half));
 
        __m256i i0 = _mm256_cvttps_epi32(v0);
        __m256i i1 = _mm256_cvttps_epi32(v1);
        __m256i i2 = _mm256_cvttps_epi32(v2);
        __m256i i3 = _mm256_cvttps_epi32(v3);
 
#if defined(__AVX2__)
        i0 = _mm256_packs_epi32(i0, i1);
        i2 = _mm256_packs_epi32(i2, i3);
        i0 = _mm256_packs_epi16(i0, i2);
 
        const __m256i perm = _mm256_setr_epi32(0, 4, 1, 5, 2, 6, 3, 7);
        i0 = _mm256_permutevar8x32_epi32(i0, perm);
        _mm256_storeu_si256((__m256i *)y[i].qs, i0);
#else
        __m128i ni0 = _mm256_castsi256_si128(i0);
        __m128i ni1 = _mm256_extractf128_si256(i0, 1);
        __m128i ni2 = _mm256_castsi256_si128(i1);
        __m128i ni3 = _mm256_extractf128_si256(i1, 1);
        __m128i ni4 = _mm256_castsi256_si128(i2);
        __m128i ni5 = _mm256_extractf128_si256(i2, 1);
        __m128i ni6 = _mm256_castsi256_si128(i3);
        __m128i ni7 = _mm256_extractf128_si256(i3, 1);
 
        ni0 = _mm_packs_epi32(ni0, ni1);
        ni2 = _mm_packs_epi32(ni2, ni3);
        ni4 = _mm_packs_epi32(ni4, ni5);
        ni6 = _mm_packs_epi32(ni6, ni7);
 
        ni0 = _mm_packs_epi16(ni0, ni2);
        ni4 = _mm_packs_epi16(ni4, ni6);
 
        _mm_storeu_si128((__m128i *)(y[i].qs + 0), ni0);
        _mm_storeu_si128((__m128i *)(y[i].qs + 16), ni4);
#endif
    }
#else
    for (int i = 0; i < nb; i++) {
        const float *xb = x + i * QK8_0;
 
        /* Find max absolute value in block */
        float amax = 0.0f;
        for (int j = 0; j < QK8_0; j++) {
            float av = xb[j] >= 0 ? xb[j] : -xb[j];
            if (av > amax) amax = av;
        }
 
        /* Compute scale: d = max / 127 */
        float d = amax / 127.0f;
        float id = d != 0.0f ? 127.0f / amax : 0.0f;
 
        /* Store scale as FP16 */
        y[i].d = CK_FP32_TO_FP16(d);
 
        /* Quantize values */
        for (int j = 0; j < QK8_0; j++) {
            float v = xb[j] * id;
            /* Round to nearest int and clamp to [-127, 127] */
            int q = (int)(v + (v >= 0 ? 0.5f : -0.5f));
            if (q > 127) q = 127;
            if (q < -127) q = -127;
            y[i].qs[j] = (int8_t)q;
        }
    }
#endif
}

References CK_FP32_TO_FP16, block_q8_0::d, id, QK8_0, and block_q8_0::qs.

Referenced by ck_test_gemm_q5_0(), ck_test_gemm_q8_0(), ck_test_gemv_q5_0(), ck_test_gemv_q5_0_q8_0(), ck_test_gemv_q8_0(), ck_test_gemv_q8_0_q8_0(), fused_mlp_swiglu_prefill_w1w2_quant(), fused_rmsnorm_qkv_prefill_head_major_quant(), gemv_fused_q5_0_bias_parallel_omp(), gemv_q5_0_from_fp32(), gemv_q8_0_from_fp32(), mega_fused_attention_decode_q5_0(), mega_fused_attention_decode_q5_0_parallel_simd(), quantize_attn_out_head_major_q8_0(), and quantize_batch_q8_0().

quantize_row_q8_k()

void quantize_row_q8_k	(	const float *	x,
		void *	vy,
		int	k
	)

Definition at line 107 of file gemm_kernels_q4k_q8k.c.

                                                        {
#if defined(__SSE4_1__)
    quantize_row_q8_k_sse(x, vy, k);
#else
    quantize_row_q8_k_ref(x, vy, k);
#endif
}

Referenced by quantize_batch_q8_k().

vec_dot_q8_0_q8_0()

void vec_dot_q8_0_q8_0	(	int	n,
		float *	s,
		const void *	vx,
		const void *	vy
	)

Auto-dispatch quantized dot product Q8_0 x Q8_0.

Definition at line 1013 of file gemm_kernels_q8_0.c.

{
#ifdef __AVX512F__
    vec_dot_q8_0_q8_0_avx512(n, s, vx, vy);
#elif defined(__AVX__)
    vec_dot_q8_0_q8_0_avx(n, s, vx, vy);
#elif defined(__SSE4_1__)
    vec_dot_q8_0_q8_0_sse(n, s, vx, vy);
#else
    vec_dot_q8_0_q8_0_ref(n, s, vx, vy);
#endif
}

References vec_dot_q8_0_q8_0_ref().

Referenced by ck_test_vec_dot_q8_0_q8_0(), gemv_q8_0_from_fp32(), gemv_q8_0_q8_0(), gemv_q8_0_q8_0_parallel(), gemv_q8_0_q8_0_parallel_omp(), gemv_q8_0_q8_0_parallel_simd(), and out_proj_head_major_q8_0_q8_0().

vec_dot_q8_0_q8_0_ref()

void vec_dot_q8_0_q8_0_ref	(	int	n,
		float *	s,
		const void *	vx,
		const void *	vy
	)

Quantized dot product: Q8_0 weights x Q8_0 input (scalar reference)

Parameters

n	Number of elements (must be multiple of 32)
s	Output: scalar dot product result
vx	Q8_0 quantized weights
vy	Q8_0 quantized input

Definition at line 863 of file gemm_kernels_q8_0.c.

{
    const int qk = QK8_0;  /* 32 */
    const int nb = n / qk;
 
    const block_q8_0 *x = (const block_q8_0 *)vx;
    const block_q8_0 *y = (const block_q8_0 *)vy;
 
    float sumf = 0.0f;
 
    for (int ib = 0; ib < nb; ib++) {
        int sumi = 0;
 
        for (int j = 0; j < qk; j++) {
            sumi += x[ib].qs[j] * y[ib].qs[j];
        }
 
        sumf += sumi * (CK_FP16_TO_FP32(x[ib].d) * CK_FP16_TO_FP32(y[ib].d));
    }
 
    *s = sumf;
}

References CK_FP16_TO_FP32, QK8_0, and block_q8_0::qs.

Referenced by vec_dot_q8_0_q8_0().