ckernel_quant.h File Reference

Quantization block structures for weight-only quantization. More...

#include <stdint.h>
#include <stddef.h>
#include "ckernel_dtype.h"

Go to the source code of this file.

Data Structures
struct	block_q4_0
struct	block_q4_1
struct	block_q4_K
struct	block_q5_0
struct	block_q5_1
struct	block_q5_K
struct	block_q6_K
struct	block_q8_0
struct	block_q8_K

Macros
#define	CK_FP16_TO_FP32(x)   ck_fp16_to_fp32(x)
#define	CK_FP16_TO_FP32_SIMD(x)   ck_fp16_to_fp32_simd(x)
#define	CK_FP16_TO_FP32_SOFT(x)   ck_fp16_to_fp32_soft(x)
#define	CK_FP32_TO_FP16(x)   ck_fp32_to_fp16(x)
#define	CK_FP32_TO_FP16_SIMD(x)   ck_fp32_to_fp16_simd(x)
#define	CK_FP32_TO_FP16_SOFT(x)   ck_fp32_to_fp16_soft(x)
#define	ggml_fp16_to_fp32   ck_fp16_to_fp32
#define	GGML_FP16_TO_FP32   CK_FP16_TO_FP32
#define	ggml_fp32_to_fp16   ck_fp32_to_fp16
#define	GGML_FP32_TO_FP16   CK_FP32_TO_FP16
#define	K_SCALE_SIZE   12
#define	QK4_0   32
#define	QK4_1   32
#define	QK5_0   32
#define	QK5_1   32
#define	QK8_0   32
#define	QK_K   256

Typedefs
typedef uint16_t	ck_half
typedef ck_half	ggml_half

Functions
static float	ck_fp16_to_fp32 (ck_half h)
static float	ck_fp16_to_fp32_soft (ck_half h)
	Convert FP16 (ck_half) to FP32 — software implementation. More...
static ck_half	ck_fp32_to_fp16 (float f)
static ck_half	ck_fp32_to_fp16_soft (float f)
	Convert FP32 to FP16 (ck_half) — software implementation. More...
static size_t	ck_quant_block_size (int type)
	Get the block size (number of weights per block) for a quant type. More...
static size_t	ck_quant_row_size (int type, int64_t n_elements)
	Calculate total bytes needed for n_elements with given quant type. More...
static size_t	ck_quant_type_size (int type)
	Get the byte size per block for a quant type. More...
void	gemm_nt_q5_0_q8_0 (const void *A_q8, const void *B_q5, const float *bias, float *C, int M, int N, int K)
	Batch GEMM with Q5_0 weights and Q8_0 activations for prefill. More...
void	gemm_nt_q5_0_q8_0_unroll_avx (const void *A_q8, const void *B_q5, const float *bias, float *C, int M, int N, int K)
void	gemm_nt_q5_0_sse_v2 (const float *A, const void *B, const float *bias, float *C, int M, int N, int K)
void	gemm_nt_q5_k (const float *A, const void *B, const float *bias, float *C, int M, int N, int K)
void	gemm_nt_q5_k_ref (const float *A, const void *B, const float *bias, float *C, int M, int N, int K)
void	gemm_nt_q6_k_ref (const float *A, const void *B, const float *bias, float *C, int M, int N, int K)
void	gemm_nt_q6_k_sse (const float *A, const void *B, const float *bias, float *C, int M, int N, int K)
void	gemv_q4_k_q8_k_sse (float *y, const void *W, const void *x_q8, int M, int K)
void	gemv_q5_k (float *y, const void *W, const float *x, int M, int K)
void	gemv_q5_k_ref (float *y, const void *W, const float *x, int M, int K)
void	quantize_row_q8_0 (const float *x, void *vy, int k)
	Quantize FP32 to Q8_0 format (scalar reference) More...
void	quantize_row_q8_k_sse (const float *x, void *vy, int k)
void	rmsnorm_q8_k_fused (const float *input, const float *gamma, void *vy, int tokens, int d_model, int aligned_embed_dim, float eps)
static void	unpack_q4_k_scales (const uint8_t *scales, uint8_t *sc, uint8_t *m)
	Unpack Q4_K sub-block scales and mins. More...
static void	unpack_q5_k_scales (const uint8_t *scales, uint8_t *sc, uint8_t *m)
	Unpack Q5_K sub-block scales and mins. More...
void	vec_dot_q5_0_q8_0 (int n, float *s, const void *vx, const void *vy)
	Auto-dispatch quantized dot product Q5_0 x Q8_0. More...
void	vec_dot_q8_0_q8_0 (int n, float *s, const void *vx, const void *vy)
	Auto-dispatch quantized dot product Q8_0 x Q8_0. More...

Detailed Description

Quantization block structures for weight-only quantization.

Defines block structures for various quantization formats used in LLM inference. Primary focus on Q4_K_M which is commonly used for LLM weight compression.

Block structures are compatible with llama.cpp/GGML for model loading.

Definition in file ckernel_quant.h.

Macro Definition Documentation

CK_FP16_TO_FP32

#define CK_FP16_TO_FP32

(

x

)

ck_fp16_to_fp32(x)

Definition at line 400 of file ckernel_quant.h.

CK_FP16_TO_FP32_SIMD

#define CK_FP16_TO_FP32_SIMD

(

x

)

ck_fp16_to_fp32_simd(x)

Definition at line 402 of file ckernel_quant.h.

CK_FP16_TO_FP32_SOFT

#define CK_FP16_TO_FP32_SOFT

(

x

)

ck_fp16_to_fp32_soft(x)

Definition at line 404 of file ckernel_quant.h.

CK_FP32_TO_FP16

#define CK_FP32_TO_FP16

(

x

)

ck_fp32_to_fp16(x)

Definition at line 401 of file ckernel_quant.h.

CK_FP32_TO_FP16_SIMD

#define CK_FP32_TO_FP16_SIMD

(

x

)

ck_fp32_to_fp16_simd(x)

Definition at line 403 of file ckernel_quant.h.

CK_FP32_TO_FP16_SOFT

#define CK_FP32_TO_FP16_SOFT

(

x

)

ck_fp32_to_fp16_soft(x)

Definition at line 405 of file ckernel_quant.h.

ggml_fp16_to_fp32

#define ggml_fp16_to_fp32   ck_fp16_to_fp32

Definition at line 409 of file ckernel_quant.h.

GGML_FP16_TO_FP32

#define GGML_FP16_TO_FP32   CK_FP16_TO_FP32

Definition at line 411 of file ckernel_quant.h.

ggml_fp32_to_fp16

#define ggml_fp32_to_fp16   ck_fp32_to_fp16

Definition at line 410 of file ckernel_quant.h.

GGML_FP32_TO_FP16

#define GGML_FP32_TO_FP16   CK_FP32_TO_FP16

Definition at line 412 of file ckernel_quant.h.

K_SCALE_SIZE

#define K_SCALE_SIZE   12

Definition at line 121 of file ckernel_quant.h.

QK4_0

#define QK4_0   32

Definition at line 35 of file ckernel_quant.h.

QK4_1

#define QK4_1   32

Definition at line 50 of file ckernel_quant.h.

QK5_0

#define QK5_0   32

Definition at line 67 of file ckernel_quant.h.

QK5_1

#define QK5_1   32

Definition at line 84 of file ckernel_quant.h.

QK8_0

#define QK8_0   32

Definition at line 101 of file ckernel_quant.h.

QK_K

#define QK_K   256

Definition at line 120 of file ckernel_quant.h.

Typedef Documentation

ck_half

typedef uint16_t ck_half

Definition at line 26 of file ckernel_quant.h.

ggml_half

typedef ck_half ggml_half

Definition at line 408 of file ckernel_quant.h.

Function Documentation

ck_fp16_to_fp32()

static float ck_fp16_to_fp32 ( ck_half  h )

inlinestatic

Definition at line 383 of file ckernel_quant.h.

                                               {
#if defined(__F16C__)
    return ck_fp16_to_fp32_simd(h);
#else
    return ck_fp16_to_fp32_soft(h);
#endif
}

References ck_fp16_to_fp32_soft().

ck_fp16_to_fp32_soft()

static float ck_fp16_to_fp32_soft ( ck_half  h )

inlinestatic

Convert FP16 (ck_half) to FP32 — software implementation.

Definition at line 303 of file ckernel_quant.h.

                                                    {
    uint32_t sign = (h & 0x8000) << 16;
    uint32_t exp = (h >> 10) & 0x1F;
    uint32_t mant = h & 0x3FF;
 
    uint32_t result;
 
    if (exp == 0) {
        if (mant == 0) {
            result = sign;
        } else {
            /* Denormalized - convert to normalized FP32 */
            exp = 1;
            while ((mant & 0x400) == 0) {
                mant <<= 1;
                exp--;
            }
            mant &= 0x3FF;
            result = sign | ((exp + 127 - 15) << 23) | (mant << 13);
        }
    } else if (exp == 31) {
        result = sign | 0x7F800000 | (mant << 13);
    } else {
        result = sign | ((exp + 127 - 15) << 23) | (mant << 13);
    }
 
    union { uint32_t u; float f; } u;
    u.u = result;
    return u.f;
}

Referenced by ck_fp16_to_fp32().

ck_fp32_to_fp16()

static ck_half ck_fp32_to_fp16 ( float  f )

inlinestatic

Definition at line 391 of file ckernel_quant.h.

                                               {
#if defined(__F16C__)
    return ck_fp32_to_fp16_simd(f);
#else
    return ck_fp32_to_fp16_soft(f);
#endif
}

References ck_fp32_to_fp16_soft().

ck_fp32_to_fp16_soft()

static ck_half ck_fp32_to_fp16_soft ( float  f )

inlinestatic

Convert FP32 to FP16 (ck_half) — software implementation.

Definition at line 337 of file ckernel_quant.h.

                                                    {
    union { uint32_t u; float f; } u;
    u.f = f;
 
    uint32_t sign = (u.u >> 16) & 0x8000;
    int32_t exp = ((u.u >> 23) & 0xFF) - 127 + 15;
    uint32_t mant = (u.u >> 13) & 0x3FF;
 
    if (exp <= 0) {
        if (exp < -10) {
            return sign;
        }
        mant = (mant | 0x400) >> (1 - exp);
        return sign | mant;
    } else if (exp >= 31) {
        return sign | 0x7C00;
    }
 
    return sign | (exp << 10) | mant;
}

Referenced by ck_fp32_to_fp16().

ck_quant_block_size()

static size_t ck_quant_block_size ( int  type )

inlinestatic

Get the block size (number of weights per block) for a quant type.

Definition at line 184 of file ckernel_quant.h.

                                                   {
    switch (type) {
        case 0: return QK4_0;    /* Q4_0 */
        case 1: return QK8_0;    /* Q8_0 */
        case 2: return QK_K;     /* Q4_K */
        case 3: return QK_K;     /* Q8_K */
        case CK_DT_Q4_1: return QK4_1;
        case CK_DT_Q5_0: return QK5_0;
        case CK_DT_Q5_1: return QK5_1;
        case CK_DT_Q5_K: return QK_K;
        case CK_DT_Q6_K: return QK_K;
        default: return 1;
    }
}

References CK_DT_Q4_1, CK_DT_Q5_0, CK_DT_Q5_1, CK_DT_Q5_K, CK_DT_Q6_K, QK4_0, QK4_1, QK5_0, QK5_1, QK8_0, and QK_K.

Referenced by ck_quant_row_size().

ck_quant_row_size()

static size_t ck_quant_row_size ( int  type, int64_t  n_elements  )

inlinestatic

Calculate total bytes needed for n_elements with given quant type.

Definition at line 220 of file ckernel_quant.h.

                                                                     {
    size_t block_size = ck_quant_block_size(type);
    size_t type_size = ck_quant_type_size(type);
    return (n_elements / block_size) * type_size;
}

References ck_quant_block_size(), and ck_quant_type_size().

ck_quant_type_size()

static size_t ck_quant_type_size ( int  type )

inlinestatic

Get the byte size per block for a quant type.

Definition at line 202 of file ckernel_quant.h.

                                                  {
    switch (type) {
        case 0: return sizeof(block_q4_0);
        case 1: return sizeof(block_q8_0);
        case 2: return sizeof(block_q4_K);
        case 3: return sizeof(block_q8_K);
        case CK_DT_Q4_1: return sizeof(block_q4_1);
        case CK_DT_Q5_0: return sizeof(block_q5_0);
        case CK_DT_Q5_1: return sizeof(block_q5_1);
        case CK_DT_Q5_K: return sizeof(block_q5_K);
        case CK_DT_Q6_K: return sizeof(block_q6_K);
        default: return 4; /* FP32 */
    }
}

References CK_DT_Q4_1, CK_DT_Q5_0, CK_DT_Q5_1, CK_DT_Q5_K, and CK_DT_Q6_K.

Referenced by ck_quant_row_size().

gemm_nt_q5_0_q8_0()

void gemm_nt_q5_0_q8_0	(	const void *	A_q8,
		const void *	B_q5,
		const float *	bias,
		float *	C,
		int	M,
		int	N,
		int	K
	)

Batch GEMM with Q5_0 weights and Q8_0 activations for prefill.

Computes C = A @ B^T + bias where: A: [M x K] Q8_0 quantized activations (M tokens, K features) B: [N x K] Q5_0 quantized weights (N outputs, K features) C: [M x N] FP32 output

This is the INT8 batch kernel for prefill, using pre-quantized activations to avoid FP32->Q8_0 conversion overhead per operation.

Parameters

A_q8	Input activations in Q8_0 format [M rows of K/32 blocks each]
B_q5	Weights in Q5_0 format [N rows of K/32 blocks each]
bias	Optional bias vector [N], NULL if not used
C	Output matrix [M x N], row-major FP32
M	Batch size (number of tokens)
N	Output dimension (number of output features)
K	Input dimension (must be multiple of 32)

Definition at line 1617 of file gemm_kernels_q5_0.c.

{
    const block_q5_0 *weights = (const block_q5_0 *)B_q5;
    const block_q8_0 *inputs = (const block_q8_0 *)A_q8;
    const int blocks_per_row = K / QK5_0;
 
    for (int m = 0; m < M; m++) {
        const block_q8_0 *input_row = &inputs[m * blocks_per_row];
 
        for (int n = 0; n < N; n++) {
            const block_q5_0 *weight_row = &weights[n * blocks_per_row];
            float *out = &C[m * N + n];
 
            /* Dispatches to vec_dot_q5_0_q8_0_avx (2x block unrolled) on AVX */
            vec_dot_q5_0_q8_0(K, out, weight_row, input_row);
 
            if (bias) {
                *out += bias[n];
            }
        }
    }
}

References C, QK5_0, and vec_dot_q5_0_q8_0().

Referenced by gemm_nt_q8_0_dispatch(), and gemm_nt_q8_0_mlp_dispatch().

gemm_nt_q5_0_q8_0_unroll_avx()

void gemm_nt_q5_0_q8_0_unroll_avx	(	const void *	A_q8,
		const void *	B_q5,
		const float *	bias,
		float *	C,
		int	M,
		int	N,
		int	K
	)

gemm_nt_q5_0_sse_v2()

void gemm_nt_q5_0_sse_v2	(	const float *	A,
		const void *	B,
		const float *	bias,
		float *	C,
		int	M,
		int	N,
		int	K
	)

Definition at line 77 of file gemm_kernels_q5_0_sse_v2.c.

{
    if (K % QK_K != 0) {
        gemm_nt_q5_0_ref(A, B, bias, C, M, N, K);
        return;
    }
 
    size_t q8_size = (K / QK_K) * sizeof(block_q8_K);
    block_q8_K *A_q8 = (block_q8_K *)alloca(q8_size);
 
    const block_q5_0 *weights = (const block_q5_0 *)B;
    const int blocks_per_row = K / 32;
 
    for (int m = 0; m < M; m++) {
        quantize_row_q8_k(&A[m * K], A_q8, K);
 
        for (int n = 0; n < N; n++) {
            float sumf = 0.0f;
            const block_q5_0 *w_row = weights + n * blocks_per_row;
 
            for (int b = 0; b < blocks_per_row; b++) {
                int q8_block_idx = (b * 32) / QK_K;
                int q8_offset = (b * 32) % QK_K;
                sumf += dot_q5_0_q8_k_32_sse(&w_row[b], &A_q8[q8_block_idx], q8_offset);
            }
 
            C[m * N + n] = sumf + (bias ? bias[n] : 0.0f);
        }
    }
}

References C, dot_q5_0_q8_k_32_sse(), gemm_nt_q5_0_ref(), QK_K, and quantize_row_q8_k().

gemm_nt_q5_k()

void gemm_nt_q5_k	(	const float *	A,
		const void *	B,
		const float *	bias,
		float *	C,
		int	M,
		int	N,
		int	K
	)

Definition at line 218 of file gemm_kernels_q5_k.c.

{
#if defined(__AVX512F__)
    /* TODO: AVX-512 implementation */
    gemm_nt_q5_k_ref(A, B, bias, C, M, N, K);
#elif defined(__AVX2__)
    /* TODO: AVX-2 implementation */
    gemm_nt_q5_k_ref(A, B, bias, C, M, N, K);
#elif defined(__AVX__)
    /* TODO: AVX implementation */
    gemm_nt_q5_k_ref(A, B, bias, C, M, N, K);
#elif defined(__SSE4_1__)
    /* TODO: SSE4.1 implementation */
    gemm_nt_q5_k_ref(A, B, bias, C, M, N, K);
#else
    gemm_nt_q5_k_ref(A, B, bias, C, M, N, K);
#endif
}

gemm_nt_q5_k_ref()

void gemm_nt_q5_k_ref	(	const float *	A,
		const void *	B,
		const float *	bias,
		float *	C,
		int	M,
		int	N,
		int	K
	)

Definition at line 145 of file gemm_kernels_q5_k.c.

{
    const block_q5_K *blocks = (const block_q5_K *)B;
    const int blocks_per_col = K / QK_K;
 
    for (int m = 0; m < M; m++) {
        const float *a_row = &A[m * K];
 
        for (int n = 0; n < N; n++) {
            float sum = 0.0f;
 
            for (int b = 0; b < blocks_per_col; b++) {
                const block_q5_K *block = &blocks[n * blocks_per_col + b];
                const float d = CK_FP16_TO_FP32(block->d);
                const float dmin = CK_FP16_TO_FP32(block->dmin);
                const uint8_t *scales = block->scales;
                const uint8_t *qh = block->qh;
                const uint8_t *qs = block->qs;
 
                /* Process 8 sub-blocks of 32 weights each */
                for (int sb = 0; sb < 8; sb++) {
                    uint8_t sc, m;
                    get_q5_k_scale_min(sb, scales, &sc, &m);
 
                    const float d_sub = d * (float)sc / 64.0f;
                    const float m_sub = dmin * (float)m / 64.0f;
 
                    const int qs_offset = sb * 16;
                    const int qh_offset = sb * 4;
 
                    for (int i = 0; i < 32; i++) {
                        uint8_t qs_val = (qs[qs_offset + i/2] >> (4 * (i % 2))) & 0xF;
                        uint8_t qh_bit = (qh[qh_offset + i/8] >> (i % 8)) & 1;
                        uint8_t q = qs_val | (qh_bit << 4);
 
                        float w = d_sub * (float)q - m_sub;
                        sum += w * a_row[b * QK_K + sb * 32 + i];
                    }
                }
            }
 
            C[m * N + n] = sum + (bias ? bias[n] : 0.0f);
        }
    }
}

References C, CK_FP16_TO_FP32, block_q5_K::d, block_q5_K::dmin, get_q5_k_scale_min(), block_q5_K::qh, QK_K, block_q5_K::qs, and block_q5_K::scales.

Referenced by gemm_nt_q5_k().

gemm_nt_q6_k_ref()

void gemm_nt_q6_k_ref	(	const float *	A,
		const void *	B,
		const float *	bias,
		float *	C,
		int	M,
		int	N,
		int	K
	)

Definition at line 243 of file gemm_kernels_q6k.c.

{
    gemm_nt_q6_k(A, B, bias, C, M, N, K);
}

References C, and gemm_nt_q6_k().

Referenced by gemm_nt_q6_k_sse().

gemm_nt_q6_k_sse()

void gemm_nt_q6_k_sse	(	const float *	A,
		const void *	B,
		const float *	bias,
		float *	C,
		int	M,
		int	N,
		int	K
	)

Definition at line 66 of file gemm_kernels_q6k_sse.c.

{
    if (K % QK_K != 0) {
        gemm_nt_q6_k_ref(A, B, bias, C, M, N, K);
        return;
    }
 
    size_t q8_size = (K / QK_K) * sizeof(block_q8_K);
    block_q8_K *A_q8 = (block_q8_K *)alloca(q8_size);
 
    const block_q6_K *weights = (const block_q6_K *)B;
    const int blocks_per_row = K / QK_K;
 
    for (int m = 0; m < M; m++) {
        quantize_row_q8_k(&A[m * K], A_q8, K);
 
        for (int n = 0; n < N; n++) {
            float sumf = 0.0f;
            const block_q6_K *w_row = weights + n * blocks_per_row;
 
            for (int b = 0; b < blocks_per_row; b++) {
                sumf += dot_q6_k_q8_k_256_sse(&w_row[b], &A_q8[b]);
            }
 
            C[m * N + n] = sumf + (bias ? bias[n] : 0.0f);
        }
    }
}

References C, dot_q6_k_q8_k_256_sse(), gemm_nt_q6_k_ref(), QK_K, and quantize_row_q8_k().

gemv_q4_k_q8_k_sse()

void gemv_q4_k_q8_k_sse	(	float *	y,
		const void *	W,
		const void *	x_q8,
		int	M,
		int	K
	)

Definition at line 33 of file gemm_kernels_q4k_sse.c.

{
    const block_q4_K *blocks = (const block_q4_K *)W;
    const block_q8_K *x = (const block_q8_K *)x_q8;
    const int blocks_per_row = K / QK_K;
 
    const __m128i mask_low = _mm_set1_epi8(0x0F);
 
    for (int row = 0; row < M; ++row) {
        float sumf = 0.0f;
        const block_q4_K *w_row = blocks + row * blocks_per_row;
 
        for (int i = 0; i < blocks_per_row; ++i) {
            const block_q4_K *b4 = &w_row[i];
            const block_q8_K *b8 = &x[i];
 
            // Unpack scales (same as ref)
            uint8_t sc[8], m_val[8];
            unpack_q4_k_scales(b4->scales, sc, m_val);
 
            float d = CK_FP16_TO_FP32(b4->d) * b8->d;
            float dmin = CK_FP16_TO_FP32(b4->dmin) * b8->d;
 
            int is = 0;
            int q_offset = 0;
 
            // Process 4 chunks of 64 elements (256 total)
            for (int j = 0; j < QK_K; j += 64) {
                // We process 32 bytes of qs (covering 64 elements via low/high nibbles)
                // We access qs[0..31] relative to q_offset
 
                // Accumulators for this 64-element chunk
                __m128i acc_lo = _mm_setzero_si128();
                __m128i acc_hi = _mm_setzero_si128();
 
                // Inner loop: 2 iters of 16 bytes (32 elements)
                for (int l = 0; l < 32; l += 16) {
                    // Load 16 bytes of Q4
                    __m128i q4_vec = _mm_loadu_si128((const __m128i *)(b4->qs + q_offset + l));
 
                    // Low nibbles -> correspond to q8_lo (elements j+l .. j+l+15)
                    __m128i q4_lo = _mm_and_si128(q4_vec, mask_low);
                    
                    // High nibbles -> correspond to q8_hi (elements j+32+l .. j+32+l+15)
                    __m128i q4_hi = _mm_and_si128(_mm_srli_epi16(q4_vec, 4), mask_low);
 
                    // Load Q8
                    __m128i q8_lo_vec = _mm_loadu_si128((const __m128i *)(b8->qs + j + l));
                    __m128i q8_hi_vec = _mm_loadu_si128((const __m128i *)(b8->qs + j + 32 + l));
 
                    // Expand and Multiply-Add: Q4(u8) * Q8(s8) -> i32
                    // Since Q4 is u8 and Q8 is s8, we use intermediate i16
                    
                    // LO PART
                    __m128i q4_lo_16_L = _mm_cvtepu8_epi16(q4_lo); // lower 8 -> 16
                    __m128i q8_lo_16_L = _mm_cvtepi8_epi16(q8_lo_vec);
                    __m128i prod_lo_L = _mm_madd_epi16(q4_lo_16_L, q8_lo_16_L); // i32
                    acc_lo = _mm_add_epi32(acc_lo, prod_lo_L);
 
                    __m128i q4_lo_16_H = _mm_cvtepu8_epi16(_mm_srli_si128(q4_lo, 8)); // upper 8 -> 16
                    __m128i q8_lo_16_H = _mm_cvtepi8_epi16(_mm_srli_si128(q8_lo_vec, 8));
                    __m128i prod_lo_H = _mm_madd_epi16(q4_lo_16_H, q8_lo_16_H); // i32
                    acc_lo = _mm_add_epi32(acc_lo, prod_lo_H);
 
                    // HI PART
                    __m128i q4_hi_16_L = _mm_cvtepu8_epi16(q4_hi);
                    __m128i q8_hi_16_L = _mm_cvtepi8_epi16(q8_hi_vec);
                    __m128i prod_hi_L = _mm_madd_epi16(q4_hi_16_L, q8_hi_16_L);
                    acc_hi = _mm_add_epi32(acc_hi, prod_hi_L);
 
                    __m128i q4_hi_16_H = _mm_cvtepu8_epi16(_mm_srli_si128(q4_hi, 8));
                    __m128i q8_hi_16_H = _mm_cvtepi8_epi16(_mm_srli_si128(q8_hi_vec, 8));
                    __m128i prod_hi_H = _mm_madd_epi16(q4_hi_16_H, q8_hi_16_H);
                    acc_hi = _mm_add_epi32(acc_hi, prod_hi_H);
                }
 
                int32_t sum_q4q8_lo = hsum_epi32_sse(acc_lo);
                int32_t sum_q4q8_hi = hsum_epi32_sse(acc_hi);
 
                /* bsums: each bsum is 16 elements */
                int32_t bsum_lo = (int32_t)b8->bsums[j / 16] +
                                  (int32_t)b8->bsums[j / 16 + 1];
                int32_t bsum_hi = (int32_t)b8->bsums[(j + 32) / 16] +
                                  (int32_t)b8->bsums[(j + 32) / 16 + 1];
 
                sumf += d * (float)sc[is] * (float)sum_q4q8_lo;
                sumf -= dmin * (float)m_val[is] * (float)bsum_lo;
                sumf += d * (float)sc[is + 1] * (float)sum_q4q8_hi;
                sumf -= dmin * (float)m_val[is + 1] * (float)bsum_hi;
 
                q_offset += 32;
                is += 2;
            }
        }
        y[row] = sumf;
    }
}

References block_q8_K::bsums, CK_FP16_TO_FP32, block_q4_K::d, block_q8_K::d, block_q4_K::dmin, hsum_epi32_sse(), QK_K, block_q4_K::qs, block_q8_K::qs, block_q4_K::scales, and unpack_q4_k_scales().

gemv_q5_k()

void gemv_q5_k	(	float *	y,
		const void *	W,
		const float *	x,
		int	M,
		int	K
	)

Definition at line 199 of file gemm_kernels_q5_k.c.

{
#if defined(__AVX512F__)
    /* TODO: AVX-512 implementation */
    gemv_q5_k_ref(y, W, x, M, K);
#elif defined(__AVX2__)
    /* TODO: AVX-2 implementation */
    gemv_q5_k_ref(y, W, x, M, K);
#elif defined(__AVX__)
    /* TODO: AVX implementation */
    gemv_q5_k_ref(y, W, x, M, K);
#elif defined(__SSE4_1__)
    /* TODO: SSE4.1 implementation */
    gemv_q5_k_ref(y, W, x, M, K);
#else
    gemv_q5_k_ref(y, W, x, M, K);
#endif
}

gemv_q5_k_ref()

void gemv_q5_k_ref	(	float *	y,
		const void *	W,
		const float *	x,
		int	M,
		int	K
	)

Definition at line 92 of file gemm_kernels_q5_k.c.

{
    const block_q5_K *blocks = (const block_q5_K *)W;
    const int blocks_per_row = K / QK_K;
 
    for (int m = 0; m < M; m++) {
        const float *x_row = x;
        float sum = 0.0f;
 
        for (int b = 0; b < blocks_per_row; b++) {
            const block_q5_K *block = &blocks[m * blocks_per_row + b];
            const float d = CK_FP16_TO_FP32(block->d);
            const float dmin = CK_FP16_TO_FP32(block->dmin);
            const uint8_t *scales = block->scales;
            const uint8_t *qh = block->qh;
            const uint8_t *qs = block->qs;
 
            /* Process 8 sub-blocks of 32 weights each */
            for (int sb = 0; sb < 8; sb++) {
                uint8_t sc, m;
                get_q5_k_scale_min(sb, scales, &sc, &m);
 
                const float d_sub = d * (float)sc / 64.0f;
                const float m_sub = dmin * (float)m / 64.0f;
 
                /* Each sub-block has 32 weights: low 4 bits in qs, high 1 bit in qh */
                const int qs_offset = sb * 16;  /* 16 bytes per sub-block */
                const int qh_offset = sb * 4;   /* 4 bytes per sub-block */
 
                for (int i = 0; i < 32; i++) {
                    uint8_t qs_val = (qs[qs_offset + i/2] >> (4 * (i % 2))) & 0xF;
                    uint8_t qh_bit = (qh[qh_offset + i/8] >> (i % 8)) & 1;
                    uint8_t q = qs_val | (qh_bit << 4);
 
                    /* Q5_K dequantization: w = d * sc/64 * q - dmin * m/64 */
                    float w = d_sub * (float)q - m_sub;
                    sum += w * x_row[b * QK_K + sb * 32 + i];
                }
            }
        }
 
        y[m] = sum;
    }
}

References CK_FP16_TO_FP32, block_q5_K::d, block_q5_K::dmin, get_q5_k_scale_min(), block_q5_K::qh, QK_K, block_q5_K::qs, and block_q5_K::scales.

Referenced by gemv_q5_k().

quantize_row_q8_0()

void quantize_row_q8_0	(	const float *	x,
		void *	vy,
		int	k
	)

Quantize FP32 to Q8_0 format (scalar reference)

Parameters

x	Input FP32 values
vy	Output Q8_0 blocks
k	Number of elements (must be multiple of 32)

Definition at line 59 of file gemm_kernels_q8_0.c.

{
    block_q8_0 *y = (block_q8_0 *)vy;
    const int nb = k / QK8_0;  /* QK8_0 = 32 */
 
#if defined(__AVX__)
    const __m256 sign_bit = _mm256_set1_ps(-0.0f);
    const __m256 v_half = _mm256_set1_ps(0.5f);
    const __m256 v_min = _mm256_set1_ps(-127.0f);
    const __m256 v_max = _mm256_set1_ps(127.0f);
 
    for (int i = 0; i < nb; i++) {
        __m256 v0 = _mm256_loadu_ps(x + 0);
        __m256 v1 = _mm256_loadu_ps(x + 8);
        __m256 v2 = _mm256_loadu_ps(x + 16);
        __m256 v3 = _mm256_loadu_ps(x + 24);
        x += QK8_0;
 
        __m256 max_abs = _mm256_andnot_ps(sign_bit, v0);
        max_abs = _mm256_max_ps(max_abs, _mm256_andnot_ps(sign_bit, v1));
        max_abs = _mm256_max_ps(max_abs, _mm256_andnot_ps(sign_bit, v2));
        max_abs = _mm256_max_ps(max_abs, _mm256_andnot_ps(sign_bit, v3));
 
        __m128 max4 = _mm_max_ps(_mm256_extractf128_ps(max_abs, 1),
                                 _mm256_castps256_ps128(max_abs));
        max4 = _mm_max_ps(max4, _mm_movehl_ps(max4, max4));
        max4 = _mm_max_ss(max4, _mm_movehdup_ps(max4));
        const float max_scalar = _mm_cvtss_f32(max4);
 
        const float d = max_scalar / 127.0f;
        const float id = max_scalar != 0.0f ? 127.0f / max_scalar : 0.0f;
        y[i].d = CK_FP32_TO_FP16(d);
 
        const __m256 mul = _mm256_set1_ps(id);
        v0 = _mm256_mul_ps(v0, mul);
        v1 = _mm256_mul_ps(v1, mul);
        v2 = _mm256_mul_ps(v2, mul);
        v3 = _mm256_mul_ps(v3, mul);
 
        v0 = _mm256_min_ps(_mm256_max_ps(v0, v_min), v_max);
        v1 = _mm256_min_ps(_mm256_max_ps(v1, v_min), v_max);
        v2 = _mm256_min_ps(_mm256_max_ps(v2, v_min), v_max);
        v3 = _mm256_min_ps(_mm256_max_ps(v3, v_min), v_max);
 
        /* Round half away from zero to match the scalar path */
        v0 = _mm256_add_ps(v0, _mm256_or_ps(_mm256_and_ps(v0, sign_bit), v_half));
        v1 = _mm256_add_ps(v1, _mm256_or_ps(_mm256_and_ps(v1, sign_bit), v_half));
        v2 = _mm256_add_ps(v2, _mm256_or_ps(_mm256_and_ps(v2, sign_bit), v_half));
        v3 = _mm256_add_ps(v3, _mm256_or_ps(_mm256_and_ps(v3, sign_bit), v_half));
 
        __m256i i0 = _mm256_cvttps_epi32(v0);
        __m256i i1 = _mm256_cvttps_epi32(v1);
        __m256i i2 = _mm256_cvttps_epi32(v2);
        __m256i i3 = _mm256_cvttps_epi32(v3);
 
#if defined(__AVX2__)
        i0 = _mm256_packs_epi32(i0, i1);
        i2 = _mm256_packs_epi32(i2, i3);
        i0 = _mm256_packs_epi16(i0, i2);
 
        const __m256i perm = _mm256_setr_epi32(0, 4, 1, 5, 2, 6, 3, 7);
        i0 = _mm256_permutevar8x32_epi32(i0, perm);
        _mm256_storeu_si256((__m256i *)y[i].qs, i0);
#else
        __m128i ni0 = _mm256_castsi256_si128(i0);
        __m128i ni1 = _mm256_extractf128_si256(i0, 1);
        __m128i ni2 = _mm256_castsi256_si128(i1);
        __m128i ni3 = _mm256_extractf128_si256(i1, 1);
        __m128i ni4 = _mm256_castsi256_si128(i2);
        __m128i ni5 = _mm256_extractf128_si256(i2, 1);
        __m128i ni6 = _mm256_castsi256_si128(i3);
        __m128i ni7 = _mm256_extractf128_si256(i3, 1);
 
        ni0 = _mm_packs_epi32(ni0, ni1);
        ni2 = _mm_packs_epi32(ni2, ni3);
        ni4 = _mm_packs_epi32(ni4, ni5);
        ni6 = _mm_packs_epi32(ni6, ni7);
 
        ni0 = _mm_packs_epi16(ni0, ni2);
        ni4 = _mm_packs_epi16(ni4, ni6);
 
        _mm_storeu_si128((__m128i *)(y[i].qs + 0), ni0);
        _mm_storeu_si128((__m128i *)(y[i].qs + 16), ni4);
#endif
    }
#else
    for (int i = 0; i < nb; i++) {
        const float *xb = x + i * QK8_0;
 
        /* Find max absolute value in block */
        float amax = 0.0f;
        for (int j = 0; j < QK8_0; j++) {
            float av = xb[j] >= 0 ? xb[j] : -xb[j];
            if (av > amax) amax = av;
        }
 
        /* Compute scale: d = max / 127 */
        float d = amax / 127.0f;
        float id = d != 0.0f ? 127.0f / amax : 0.0f;
 
        /* Store scale as FP16 */
        y[i].d = CK_FP32_TO_FP16(d);
 
        /* Quantize values */
        for (int j = 0; j < QK8_0; j++) {
            float v = xb[j] * id;
            /* Round to nearest int and clamp to [-127, 127] */
            int q = (int)(v + (v >= 0 ? 0.5f : -0.5f));
            if (q > 127) q = 127;
            if (q < -127) q = -127;
            y[i].qs[j] = (int8_t)q;
        }
    }
#endif
}

quantize_row_q8_k_sse()

void quantize_row_q8_k_sse	(	const float *	x,
		void *	vy,
		int	k
	)

Definition at line 29 of file quantize_row_q8_k_sse.c.

                                                            {
    if (!x || !vy || k <= 0) {
        return;
    }
    assert(k % QK_K == 0);
    const int nb = k / QK_K;
    block_q8_K *y = (block_q8_K *)vy;
 
    for (int i = 0; i < nb; ++i) {
        float max = 0.0f;
        
        // SSE max absolute value
        __m128 v_max = _mm_setzero_ps();
        for (int j = 0; j < QK_K; j += 4) {
            __m128 v = _mm_loadu_ps(x + j);
            __m128 v_abs = _mm_andnot_ps(_mm_set1_ps(-0.0f), v);
            v_max = _mm_max_ps(v_max, v_abs);
        }
        
        // Horizontal max
        v_max = _mm_max_ps(v_max, _mm_shuffle_ps(v_max, v_max, _MM_SHUFFLE(1, 0, 3, 2)));
        v_max = _mm_max_ps(v_max, _mm_shuffle_ps(v_max, v_max, _MM_SHUFFLE(0, 1, 0, 1)));
        _mm_store_ss(&max, v_max);
 
        if (max == 0.0f) {
            y[i].d = 0.0f;
            memset(y[i].qs, 0, sizeof(y[i].qs));
            memset(y[i].bsums, 0, sizeof(y[i].bsums));
            x += QK_K;
            continue;
        }
 
        const float iscale = -127.0f / max;
        __m128 v_iscale = _mm_set1_ps(iscale);
        
        // Quantize and compute bsums in SSE
        for (int j = 0; j < QK_K; j += 16) {
            __m128 x0 = _mm_loadu_ps(x + j + 0);
            __m128 x1 = _mm_loadu_ps(x + j + 4);
            __m128 x2 = _mm_loadu_ps(x + j + 8);
            __m128 x3 = _mm_loadu_ps(x + j + 12);
 
            __m128i q0 = _mm_cvtps_epi32(_mm_mul_ps(x0, v_iscale));
            __m128i q1 = _mm_cvtps_epi32(_mm_mul_ps(x1, v_iscale));
            __m128i q2 = _mm_cvtps_epi32(_mm_mul_ps(x2, v_iscale));
            __m128i q3 = _mm_cvtps_epi32(_mm_mul_ps(x3, v_iscale));
 
            // Pack i32 -> i16 -> i8
            __m128i q01 = _mm_packs_epi32(q0, q1);
            __m128i q23 = _mm_packs_epi32(q2, q3);
            __m128i q0123 = _mm_packs_epi16(q01, q23);
 
            _mm_storeu_si128((__m128i *)(y[i].qs + j), q0123);
 
            // Compute bsum for these 16 elements
            // Each bsum[j/16] covers 16 elements
            __m128i p01 = _mm_add_epi16(q01, q23);
            p01 = _mm_add_epi16(p01, _mm_shuffle_epi32(p01, _MM_SHUFFLE(1, 0, 3, 2)));
            p01 = _mm_add_epi16(p01, _mm_shufflelo_epi16(p01, _MM_SHUFFLE(1, 0, 3, 2)));
            int16_t bsum = (int16_t)_mm_extract_epi16(p01, 0) + (int16_t)_mm_extract_epi16(p01, 1);
            y[i].bsums[j / 16] = bsum;
        }
 
        y[i].d = 1.0f / iscale;
        x += QK_K;
    }
}

References block_q8_K::bsums, block_q8_K::d, and QK_K.

rmsnorm_q8_k_fused()

void rmsnorm_q8_k_fused	(	const float *	input,
		const float *	gamma,
		void *	vy,
		int	tokens,
		int	d_model,
		int	aligned_embed_dim,
		float	eps
	)

Fused RMSNorm + Q8_K Quantization

Benefits:

1. Single pass over input data (reduces DRAM pressure)
2. Normalization results stay in registers for quantization
3. Keeps hot data in L1/L2 cache

Definition at line 54 of file rmsnorm_q8_k_fused.c.

{
    const int T = tokens;
    const int D = d_model;
    block_q8_K *y = (block_q8_K *)vy;
 
    for (int t = 0; t < T; ++t) {
        const float *x = input + (size_t)t * aligned_embed_dim;
        
        // 1. Compute sum of squares using AVX
        __m256 sum_sq_vec = _mm256_setzero_ps();
        for (int d = 0; d < D; d += 8) {
            __m256 xv = _mm256_loadu_ps(&x[d]);
            sum_sq_vec = _mm256_add_ps(sum_sq_vec, _mm256_mul_ps(xv, xv));
        }
        float sum_sq = hsum256_ps_fused(sum_sq_vec);
        float rstd = 1.0f / sqrtf(sum_sq / (float)D + eps);
        __m256 vrstd = _mm256_set1_ps(rstd);
 
        // 2. We need the max absolute value of the NORMALIZED data for quantization
        // y_i = gamma_i * (x_i * rstd)
        // We do this in blocks of QK_K (256) to match Q8_K layout
        for (int b = 0; b < D / QK_K; ++b) {
            const float *xb = x + b * QK_K;
            const float *gb = gamma + b * QK_K;
            block_q8_K *out_block = &y[t * (D / QK_K) + b];
 
            // Local normalization and max search
            __m256 v_max_abs = _mm256_setzero_ps();
            float norm_buf[QK_K];
 
            for (int d = 0; d < QK_K; d += 8) {
                __m256 xv = _mm256_loadu_ps(&xb[d]);
                __m256 gv = _mm256_loadu_ps(&gb[d]);
                __m256 normalized = _mm256_mul_ps(_mm256_mul_ps(xv, vrstd), gv);
                
                _mm256_storeu_ps(&norm_buf[d], normalized);
                
                __m256 v_abs = _mm256_andnot_ps(_mm256_set1_ps(-0.0f), normalized);
                v_max_abs = _mm256_max_ps(v_max_abs, v_abs);
            }
 
            float max_val = hmax256_ps_fused(v_max_abs);
            if (max_val == 0.0f) {
                out_block->d = 0.0f;
                memset(out_block->qs, 0, QK_K);
                memset(out_block->bsums, 0, sizeof(out_block->bsums));
                continue;
            }
 
            // 3. Quantize to Q8_K
            float iscale = -127.0f / max_val;
            __m256 v_iscale = _mm256_set1_ps(iscale);
            out_block->d = 1.0f / iscale;
 
            for (int j = 0; j < QK_K; j += 16) {
                // AVX1 doesn't have 256-bit integer conversion, so we use 128-bit SSE for packing
                __m128 n0 = _mm_loadu_ps(&norm_buf[j + 0]);
                __m128 n1 = _mm_loadu_ps(&norm_buf[j + 4]);
                __m128 n2 = _mm_loadu_ps(&norm_buf[j + 8]);
                __m128 n3 = _mm_loadu_ps(&norm_buf[j + 12]);
 
                __m128i q0 = _mm_cvtps_epi32(_mm_mul_ps(n0, _mm256_castps256_ps128(v_iscale)));
                __m128i q1 = _mm_cvtps_epi32(_mm_mul_ps(n1, _mm256_castps256_ps128(v_iscale)));
                __m128i q2 = _mm_cvtps_epi32(_mm_mul_ps(n2, _mm256_castps256_ps128(v_iscale)));
                __m128i q3 = _mm_cvtps_epi32(_mm_mul_ps(n3, _mm256_castps256_ps128(v_iscale)));
 
                __m128i q01 = _mm_packs_epi32(q0, q1);
                __m128i q23 = _mm_packs_epi32(q2, q3);
                __m128i q0123 = _mm_packs_epi16(q01, q23);
 
                _mm_storeu_si128((__m128i *)(out_block->qs + j), q0123);
 
                // Compute bsum for 16 elements
                __m128i p01 = _mm_add_epi16(q01, q23);
                p01 = _mm_add_epi16(p01, _mm_shuffle_epi32(p01, _MM_SHUFFLE(1, 0, 3, 2)));
                p01 = _mm_add_epi16(p01, _mm_shufflelo_epi16(p01, _MM_SHUFFLE(1, 0, 3, 2)));
                int16_t bsum = (int16_t)_mm_extract_epi16(p01, 0) + (int16_t)_mm_extract_epi16(p01, 1);
                out_block->bsums[j / 16] = bsum;
            }
        }
    }
}

References block_q8_K::bsums, block_q8_K::d, hmax256_ps_fused(), hsum256_ps_fused(), QK_K, and block_q8_K::qs.

unpack_q4_k_scales()

static void unpack_q4_k_scales ( const uint8_t *  scales, uint8_t *  sc, uint8_t *  m  )

inlinestatic

Unpack Q4_K sub-block scales and mins.

Parameters

scales	The packed scales[12] array from block_q4_K
sc	Output: 8 unpacked scale values (multiply by super-block d)
m	Output: 8 unpacked min values (multiply by super-block dmin)

This matches llama.cpp's get_scale_min_k4() function exactly. The 12-byte scales array layout:

• bytes 0-3: 6-bit scales[0-3] (high 2 bits used for scales[4-7])
• bytes 4-7: 6-bit mins[0-3] (high 2 bits used for mins[4-7])
• bytes 8-11: low 4 bits for scales[4-7], high 4 bits for mins[4-7]

Definition at line 246 of file ckernel_quant.h.

                                                                {
    /* Direct 6-bit values for indices 0-3 */
    sc[0] = scales[0] & 0x3F;
    sc[1] = scales[1] & 0x3F;
    sc[2] = scales[2] & 0x3F;
    sc[3] = scales[3] & 0x3F;
 
    m[0] = scales[4] & 0x3F;
    m[1] = scales[5] & 0x3F;
    m[2] = scales[6] & 0x3F;
    m[3] = scales[7] & 0x3F;
 
    /* 6-bit values for indices 4-7: low 4 bits from bytes 8-11,
     * high 2 bits from upper bits of bytes 0-3 (scales) and 4-7 (mins) */
    sc[4] = (scales[8]  & 0x0F) | ((scales[0] >> 6) << 4);
    sc[5] = (scales[9]  & 0x0F) | ((scales[1] >> 6) << 4);
    sc[6] = (scales[10] & 0x0F) | ((scales[2] >> 6) << 4);
    sc[7] = (scales[11] & 0x0F) | ((scales[3] >> 6) << 4);
 
    m[4] = (scales[8]  >> 4) | ((scales[4] >> 6) << 4);
    m[5] = (scales[9]  >> 4) | ((scales[5] >> 6) << 4);
    m[6] = (scales[10] >> 4) | ((scales[6] >> 6) << 4);
    m[7] = (scales[11] >> 4) | ((scales[7] >> 6) << 4);
}

Referenced by dequant_q4_k_block(), dot_q4_k_q8_k_ref(), gemv_q4_k_backward_ref(), gemv_q4_k_q8_k_sse(), gemv_q4_k_ref(), and unpack_q5_k_scales().

unpack_q5_k_scales()

static void unpack_q5_k_scales ( const uint8_t *  scales, uint8_t *  sc, uint8_t *  m  )

inlinestatic

Unpack Q5_K sub-block scales and mins.

Parameters

scales	The packed scales[12] array from block_q5_K
sc	Output: 8 unpacked scale values (multiply by super-block d)
m	Output: 8 unpacked min values (multiply by super-block dmin)

Q5_K uses the same 6-bit packed format as Q4_K for scales/mins. The 12-byte scales array layout is identical:

• bytes 0-3: 6-bit scales[0-3] (high 2 bits used for scales[4-7])
• bytes 4-7: 6-bit mins[0-3] (high 2 bits used for mins[4-7])
• bytes 8-11: low 4 bits for scales[4-7], high 4 bits for mins[4-7]

Definition at line 285 of file ckernel_quant.h.

                                                              {
    /* Q5_K uses identical packing as Q4_K for scales/mins */
    unpack_q4_k_scales(scales, sc, m);
}

References unpack_q4_k_scales().

vec_dot_q5_0_q8_0()

void vec_dot_q5_0_q8_0	(	int	n,
		float *	s,
		const void *	vx,
		const void *	vy
	)

Auto-dispatch quantized dot product Q5_0 x Q8_0.

Dispatch priority:

1. AVX512 (best performance on modern Intel/AMD)
2. AVX (256-bit float ops, works on Sandy/Ivy Bridge and newer)
3. SSSE3 (128-bit fallback)
4. Reference scalar (last resort)

Definition at line 1498 of file gemm_kernels_q5_0.c.

{
#if defined(__AVX512F__)
    vec_dot_q5_0_q8_0_avx512(n, s, vx, vy);
#elif defined(__AVX__)
    /* AVX for 256-bit float ops (works on Ivy Bridge and newer) */
    vec_dot_q5_0_q8_0_avx(n, s, vx, vy);
#elif defined(__SSSE3__)
    /* SSSE3 - most efficient on older CPUs */
    vec_dot_q5_0_q8_0_sse(n, s, vx, vy);
#else
    vec_dot_q5_0_q8_0_ref(n, s, vx, vy);
#endif
}

References vec_dot_q5_0_q8_0_ref().

Referenced by out_proj_head_major_q5_0_q8_0().

vec_dot_q8_0_q8_0()

void vec_dot_q8_0_q8_0	(	int	n,
		float *	s,
		const void *	vx,
		const void *	vy
	)

Auto-dispatch quantized dot product Q8_0 x Q8_0.

Definition at line 1013 of file gemm_kernels_q8_0.c.

{
#ifdef __AVX512F__
    vec_dot_q8_0_q8_0_avx512(n, s, vx, vy);
#elif defined(__AVX__)
    vec_dot_q8_0_q8_0_avx(n, s, vx, vy);
#elif defined(__SSE4_1__)
    vec_dot_q8_0_q8_0_sse(n, s, vx, vy);
#else
    vec_dot_q8_0_q8_0_ref(n, s, vx, vy);
#endif
}

References vec_dot_q8_0_q8_0_ref().

Referenced by out_proj_head_major_q8_0_q8_0().