ckernel_quant.h File Reference

Quantization block structures for weight-only quantization. More...

#include <stdint.h>
#include <stddef.h>
#include <math.h>
#include "ckernel_dtype.h"

Go to the source code of this file.

Data Structures
struct	block_nvfp4
struct	block_q4_0
struct	block_q4_1
struct	block_q4_K
struct	block_q5_0
struct	block_q5_1
struct	block_q6_K
struct	block_q8_0
struct	block_q8_K

Macros
#define	CK_FP16_TO_FP32(x)   ck_fp16_to_fp32(x)
#define	CK_FP16_TO_FP32_SIMD(x)   ck_fp16_to_fp32_simd(x)
#define	CK_FP16_TO_FP32_SOFT(x)   ck_fp16_to_fp32_soft(x)
#define	CK_FP32_TO_FP16(x)   ck_fp32_to_fp16(x)
#define	CK_FP32_TO_FP16_SIMD(x)   ck_fp32_to_fp16_simd(x)
#define	CK_FP32_TO_FP16_SOFT(x)   ck_fp32_to_fp16_soft(x)
#define	ggml_fp16_to_fp32   ck_fp16_to_fp32
#define	GGML_FP16_TO_FP32   CK_FP16_TO_FP32
#define	ggml_fp32_to_fp16   ck_fp32_to_fp16
#define	GGML_FP32_TO_FP16   CK_FP32_TO_FP16
#define	K_SCALE_SIZE   12
#define	QK4_0   32
#define	QK4_1   32
#define	QK5_0   32
#define	QK5_1   32
#define	QK8_0   32
#define	QK_K   256
#define	QK_NVFP4   64
#define	QK_NVFP4_SUB   16

Typedefs
typedef uint16_t	ck_half
typedef ck_half	ggml_half

Functions
static float	ck_fp16_to_fp32 (ck_half h)
static float	ck_fp16_to_fp32_soft (ck_half h)
	Convert FP16 (ck_half) to FP32 — software implementation.
static float	ck_fp32_from_bits (uint32_t u32)
static uint32_t	ck_fp32_to_bits (float f)
static ck_half	ck_fp32_to_fp16 (float f)
static ck_half	ck_fp32_to_fp16_soft (float f)
	Convert FP32 to FP16 (ck_half) — software implementation.
static size_t	ck_quant_block_size (int type)
	Get the block size (number of weights per block) for a quant type.
static size_t	ck_quant_row_size (int type, int64_t n_elements)
	Calculate total bytes needed for n_elements with given quant type.
static size_t	ck_quant_type_size (int type)
	Get the byte size per block for a quant type.
float	ck_ue4m3_to_fp32 (uint8_t value)
void	dequantize_row_nvfp4 (const void *weights, float *output, int k, float weight_scale)
void	gemm_nt_q5_0_q8_0 (const void *A_q8, const void *B_q5, const float *bias, float *C, int M, int N, int K)
	Batch GEMM with Q5_0 weights and Q8_0 activations for prefill.
void	gemm_nt_q5_0_q8_0_unroll_avx (const void *A_q8, const void *B_q5, const float *bias, float *C, int M, int N, int K)
void	gemm_nt_q5_0_sse_v2 (const float *A, const void *B, const float *bias, float *C, int M, int N, int K)
void	gemm_nt_q6_k_ref (const float *A, const void *B, const float *bias, float *C, int M, int N, int K)
void	gemm_nt_q6_k_sse (const float *A, const void *B, const float *bias, float *C, int M, int N, int K)
void	gemv_nvfp4_q8_0 (float *output, const void *weights, const float *weight_scales, const void *activations, int rows, int cols)
void	gemv_q4_k_q8_k_sse (float *y, const void *W, const void *x_q8, int M, int K)
int	moe_swiglu_expert_forward_nvfp4_workspace (const float *hidden, const int *indices, const float *routing_weights, const void *expert_gate, const float *expert_gate_scales, const void *expert_up, const float *expert_up_scales, const void *expert_down, const float *expert_down_scales, float *output, int rows, int hidden_dim, int intermediate_dim, int n_experts, int top_k, void *workspace, size_t workspace_bytes)
size_t	moe_swiglu_nvfp4_workspace_bytes (int hidden_dim, int intermediate_dim)
int	moe_swiglu_shared_forward_nvfp4_workspace (const float *hidden, const float *routed, const void *shared_gate, const float *shared_gate_scale, const void *shared_up, const float *shared_up_scale, const void *shared_down, const float *shared_down_scale, float *output, int rows, int hidden_dim, int intermediate_dim, float combination_scale, void *workspace, size_t workspace_bytes)
void	quantize_row_q8_0 (const float *x, void *vy, int k)
	Quantize FP32 to Q8_0 format (scalar reference)
void	quantize_row_q8_k_sse (const float *x, void *vy, int k)
void	rmsnorm_q8_k_fused (const float *input, const float *gamma, void *vy, int tokens, int d_model, int aligned_embed_dim, float eps)
static void	unpack_q4_k_scales (const uint8_t *scales, uint8_t *sc, uint8_t *m)
	Unpack Q4_K sub-block scales and mins.
void	vec_dot_nvfp4_q8_0 (int n, float *output, const void *weights, const void *activations, float weight_scale)
void	vec_dot_nvfp4_q8_0_ref (int n, float *output, const void *weights, const void *activations, float weight_scale)
void	vec_dot_q5_0_q8_0 (int n, float *s, const void *vx, const void *vy)
	Auto-dispatch quantized dot product Q5_0 x Q8_0.
void	vec_dot_q8_0_q8_0 (int n, float *s, const void *vx, const void *vy)
	Auto-dispatch quantized dot product Q8_0 x Q8_0.

Detailed Description

Quantization block structures for weight-only quantization.

Defines block structures for various quantization formats used in LLM inference. Primary focus on Q4_K_M which is commonly used for LLM weight compression.

Block structures are compatible with llama.cpp/GGML for model loading.

Definition in file ckernel_quant.h.

Macro Definition Documentation

CK_FP16_TO_FP32

#define CK_FP16_TO_FP32

(

x

)

ck_fp16_to_fp32(x)

Definition at line 440 of file ckernel_quant.h.

CK_FP16_TO_FP32_SIMD

#define CK_FP16_TO_FP32_SIMD

(

x

)

ck_fp16_to_fp32_simd(x)

Definition at line 442 of file ckernel_quant.h.

CK_FP16_TO_FP32_SOFT

#define CK_FP16_TO_FP32_SOFT

(

x

)

ck_fp16_to_fp32_soft(x)

Definition at line 444 of file ckernel_quant.h.

CK_FP32_TO_FP16

#define CK_FP32_TO_FP16

(

x

)

ck_fp32_to_fp16(x)

Definition at line 441 of file ckernel_quant.h.

CK_FP32_TO_FP16_SIMD

#define CK_FP32_TO_FP16_SIMD

(

x

)

ck_fp32_to_fp16_simd(x)

Definition at line 443 of file ckernel_quant.h.

CK_FP32_TO_FP16_SOFT

#define CK_FP32_TO_FP16_SOFT

(

x

)

ck_fp32_to_fp16_soft(x)

Definition at line 445 of file ckernel_quant.h.

ggml_fp16_to_fp32

#define ggml_fp16_to_fp32   ck_fp16_to_fp32

Definition at line 449 of file ckernel_quant.h.

GGML_FP16_TO_FP32

#define GGML_FP16_TO_FP32   CK_FP16_TO_FP32

Definition at line 451 of file ckernel_quant.h.

ggml_fp32_to_fp16

#define ggml_fp32_to_fp16   ck_fp32_to_fp16

Definition at line 450 of file ckernel_quant.h.

GGML_FP32_TO_FP16

#define GGML_FP32_TO_FP16   CK_FP32_TO_FP16

Definition at line 452 of file ckernel_quant.h.

K_SCALE_SIZE

#define K_SCALE_SIZE   12

Definition at line 176 of file ckernel_quant.h.

QK4_0

#define QK4_0   32

Definition at line 36 of file ckernel_quant.h.

QK4_1

#define QK4_1   32

Definition at line 51 of file ckernel_quant.h.

QK5_0

#define QK5_0   32

Definition at line 68 of file ckernel_quant.h.

QK5_1

#define QK5_1   32

Definition at line 85 of file ckernel_quant.h.

QK8_0

#define QK8_0   32

Definition at line 102 of file ckernel_quant.h.

QK_K

#define QK_K   256

Definition at line 175 of file ckernel_quant.h.

QK_NVFP4

#define QK_NVFP4   64

Definition at line 124 of file ckernel_quant.h.

QK_NVFP4_SUB

#define QK_NVFP4_SUB   16

Definition at line 125 of file ckernel_quant.h.

Typedef Documentation

ck_half

typedef uint16_t ck_half

Definition at line 27 of file ckernel_quant.h.

ggml_half

typedef ck_half ggml_half

Definition at line 448 of file ckernel_quant.h.

Function Documentation

ck_fp16_to_fp32()

static float ck_fp16_to_fp32 ( ck_half  h )

inlinestatic

Definition at line 423 of file ckernel_quant.h.

                                               {
#if defined(__F16C__)
    return ck_fp16_to_fp32_simd(h);
#else
    return ck_fp16_to_fp32_soft(h);
#endif
}

References ck_fp16_to_fp32_soft().

Referenced by ck_mrope_round_storage(), and qwen4_ple_gate_conv_inject_impl().

ck_fp16_to_fp32_soft()

static float ck_fp16_to_fp32_soft ( ck_half  h )

inlinestatic

Convert FP16 (ck_half) to FP32 — software implementation.

Definition at line 325 of file ckernel_quant.h.

                                                    {
    uint32_t sign = (h & 0x8000) << 16;
    uint32_t exp = (h >> 10) & 0x1F;
    uint32_t mant = h & 0x3FF;
 
    uint32_t result;
 
    if (exp == 0) {
        if (mant == 0) {
            result = sign;
        } else {
            /* Denormalized - convert to normalized FP32 */
            exp = 1;
            while ((mant & 0x400) == 0) {
                mant <<= 1;
                exp--;
            }
            mant &= 0x3FF;
            result = sign | ((exp + 127 - 15) << 23) | (mant << 13);
        }
    } else if (exp == 31) {
        result = sign | 0x7F800000 | (mant << 13);
    } else {
        result = sign | ((exp + 127 - 15) << 23) | (mant << 13);
    }
 
    union { uint32_t u; float f; } u;
    u.u = result;
    return u.f;
}

Referenced by ck_fp16_to_fp32().

ck_fp32_from_bits()

static float ck_fp32_from_bits ( uint32_t  u32 )

inlinestatic

Definition at line 362 of file ckernel_quant.h.

                                                    {
    union { uint32_t u; float f; } u;
    u.u = u32;
    return u.f;
}

Referenced by ck_fp32_to_fp16_soft().

ck_fp32_to_bits()

static uint32_t ck_fp32_to_bits ( float  f )

inlinestatic

Definition at line 356 of file ckernel_quant.h.

                                                {
    union { float f; uint32_t u; } u;
    u.f = f;
    return u.u;
}

Referenced by ck_fp32_to_fp16_soft().

ck_fp32_to_fp16()

static ck_half ck_fp32_to_fp16 ( float  f )

inlinestatic

Definition at line 431 of file ckernel_quant.h.

                                               {
#if defined(__F16C__)
    return ck_fp32_to_fp16_simd(f);
#else
    return ck_fp32_to_fp16_soft(f);
#endif
}

References ck_fp32_to_fp16_soft().

Referenced by ck_mrope_round_storage().

ck_fp32_to_fp16_soft()

static ck_half ck_fp32_to_fp16_soft ( float  f )

inlinestatic

Convert FP32 to FP16 (ck_half) — software implementation.

Definition at line 371 of file ckernel_quant.h.

                                                    {
#if (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) || defined(__GNUC__) && !defined(__STRICT_ANSI__)) && (!defined(__cplusplus) || __cplusplus >= 201703L)
    const float scale_to_inf = 0x1.0p+112f;
    const float scale_to_zero = 0x1.0p-110f;
#else
    const float scale_to_inf = ck_fp32_from_bits(UINT32_C(0x77800000));
    const float scale_to_zero = ck_fp32_from_bits(UINT32_C(0x08800000));
#endif
    float base = (fabsf(f) * scale_to_inf) * scale_to_zero;
 
    const uint32_t w = ck_fp32_to_bits(f);
    const uint32_t shl1_w = w + w;
    const uint32_t sign = w & UINT32_C(0x80000000);
    uint32_t bias = shl1_w & UINT32_C(0xFF000000);
    if (bias < UINT32_C(0x71000000)) {
        bias = UINT32_C(0x71000000);
    }
 
    base = ck_fp32_from_bits((bias >> 1) + UINT32_C(0x07800000)) + base;
    const uint32_t bits = ck_fp32_to_bits(base);
    const uint32_t exp_bits = (bits >> 13) & UINT32_C(0x00007C00);
    const uint32_t mantissa_bits = bits & UINT32_C(0x00000FFF);
    const uint32_t nonsign = exp_bits + mantissa_bits;
 
    return (ck_half) ((sign >> 16) | (shl1_w > UINT32_C(0xFF000000) ? UINT16_C(0x7E00) : nonsign));
}

References ck_fp32_from_bits(), and ck_fp32_to_bits().

Referenced by ck_fp32_to_fp16().

ck_quant_block_size()

static size_t ck_quant_block_size ( int  type )

inlinestatic

Get the block size (number of weights per block) for a quant type.

Definition at line 227 of file ckernel_quant.h.

                                                   {
    switch (type) {
        case 0: return QK4_0;    /* Q4_0 */
        case 1: return QK8_0;    /* Q8_0 */
        case 2: return QK_K;     /* Q4_K */
        case 3: return QK_K;     /* Q8_K */
        case CK_DT_Q4_1: return QK4_1;
        case CK_DT_Q5_0: return QK5_0;
        case CK_DT_Q5_1: return QK5_1;
        case CK_DT_Q6_K: return QK_K;
        default: return 1;
    }
}

References CK_DT_Q4_1, CK_DT_Q5_0, CK_DT_Q5_1, CK_DT_Q6_K, QK4_0, QK4_1, QK5_0, QK5_1, QK8_0, and QK_K.

Referenced by ck_quant_row_size().

ck_quant_row_size()

static size_t ck_quant_row_size ( int  type, int64_t  n_elements  )

inlinestatic

Calculate total bytes needed for n_elements with given quant type.

Definition at line 261 of file ckernel_quant.h.

                                                                     {
    size_t block_size = ck_quant_block_size(type);
    size_t type_size = ck_quant_type_size(type);
    return (n_elements / block_size) * type_size;
}

References ck_quant_block_size(), and ck_quant_type_size().

ck_quant_type_size()

static size_t ck_quant_type_size ( int  type )

inlinestatic

Get the byte size per block for a quant type.

Definition at line 244 of file ckernel_quant.h.

                                                  {
    switch (type) {
        case 0: return sizeof(block_q4_0);
        case 1: return sizeof(block_q8_0);
        case 2: return sizeof(block_q4_K);
        case 3: return sizeof(block_q8_K);
        case CK_DT_Q4_1: return sizeof(block_q4_1);
        case CK_DT_Q5_0: return sizeof(block_q5_0);
        case CK_DT_Q5_1: return sizeof(block_q5_1);
        case CK_DT_Q6_K: return sizeof(block_q6_K);
        default: return 4; /* FP32 */
    }
}

References CK_DT_Q4_1, CK_DT_Q5_0, CK_DT_Q5_1, and CK_DT_Q6_K.

Referenced by ck_quant_row_size().

ck_ue4m3_to_fp32()

float ck_ue4m3_to_fp32

(

uint8_t

value

)

Definition at line 53 of file gemm_kernels_nvfp4.c.

{
    return ck_ue4m3_to_fp32_inline(value);
}

References ck_ue4m3_to_fp32_inline().

dequantize_row_nvfp4()

void dequantize_row_nvfp4	(	const void *	weights,
		float *	output,
		int	k,
		float	weight_scale
	)

Definition at line 58 of file gemm_kernels_nvfp4.c.

{
    assert(k >= 0 && k % QK_NVFP4 == 0);
    const block_nvfp4 *blocks = (const block_nvfp4 *)weights;
    const int block_count = k / QK_NVFP4;
 
    for (int block_index = 0; block_index < block_count; ++block_index) {
        const block_nvfp4 *block = &blocks[block_index];
        for (int sub = 0; sub < QK_NVFP4 / QK_NVFP4_SUB; ++sub) {
            const float scale = ck_ue4m3_to_fp32_inline(block->d[sub]) * weight_scale;
            const uint8_t *packed = &block->qs[sub * (QK_NVFP4_SUB / 2)];
            float *dst = &output[block_index * QK_NVFP4 + sub * QK_NVFP4_SUB];
            for (int lane = 0; lane < QK_NVFP4_SUB / 2; ++lane) {
                const uint8_t pair = packed[lane];
                dst[lane] = scale * (float)ck_nvfp4_e2m1_x2[pair & 0x0f];
                dst[lane + QK_NVFP4_SUB / 2] =
                    scale * (float)ck_nvfp4_e2m1_x2[pair >> 4];
            }
        }
    }
}

References ck_nvfp4_e2m1_x2, ck_ue4m3_to_fp32_inline(), block_nvfp4::d, QK_NVFP4, QK_NVFP4_SUB, and block_nvfp4::qs.

gemm_nt_q5_0_q8_0()

void gemm_nt_q5_0_q8_0	(	const void *	A_q8,
		const void *	B_q5,
		const float *	bias,
		float *	C,
		int	M,
		int	N,
		int	K
	)

Batch GEMM with Q5_0 weights and Q8_0 activations for prefill.

Computes C = A @ B^T + bias where: A: [M x K] Q8_0 quantized activations (M tokens, K features) B: [N x K] Q5_0 quantized weights (N outputs, K features) C: [M x N] FP32 output

This is the INT8 batch kernel for prefill, using pre-quantized activations to avoid FP32->Q8_0 conversion overhead per operation.

Parameters

A_q8	Input activations in Q8_0 format [M rows of K/32 blocks each]
B_q5	Weights in Q5_0 format [N rows of K/32 blocks each]
bias	Optional bias vector [N], NULL if not used
C	Output matrix [M x N], row-major FP32
M	Batch size (number of tokens)
N	Output dimension (number of output features)
K	Input dimension (must be multiple of 32)

Definition at line 1728 of file gemm_kernels_q5_0.c.

{
    const block_q5_0 *weights = (const block_q5_0 *)B_q5;
    const block_q8_0 *inputs = (const block_q8_0 *)A_q8;
    const int blocks_per_row = K / QK5_0;
 
    for (int m = 0; m < M; m++) {
        const block_q8_0 *input_row = &inputs[m * blocks_per_row];
 
        for (int n = 0; n < N; n++) {
            const block_q5_0 *weight_row = &weights[n * blocks_per_row];
            float *out = &C[m * N + n];
 
            /* Dispatches to vec_dot_q5_0_q8_0_avx (2x block unrolled) on AVX */
            vec_dot_q5_0_q8_0(K, out, weight_row, input_row);
 
            if (bias) {
                *out += bias[n];
            }
        }
    }
}

References C, QK5_0, and vec_dot_q5_0_q8_0().

Referenced by gemm_nt_q8_0_dispatch(), and gemm_nt_q8_0_mlp_dispatch().

gemm_nt_q5_0_q8_0_unroll_avx()

void gemm_nt_q5_0_q8_0_unroll_avx	(	const void *	A_q8,
		const void *	B_q5,
		const float *	bias,
		float *	C,
		int	M,
		int	N,
		int	K
	)

gemm_nt_q5_0_sse_v2()

void gemm_nt_q5_0_sse_v2	(	const float *	A,
		const void *	B,
		const float *	bias,
		float *	C,
		int	M,
		int	N,
		int	K
	)

Definition at line 77 of file gemm_kernels_q5_0_sse_v2.c.

{
    if (K % QK_K != 0) {
        gemm_nt_q5_0_ref(A, B, bias, C, M, N, K);
        return;
    }
 
    size_t q8_size = (K / QK_K) * sizeof(block_q8_K);
    block_q8_K *A_q8 = (block_q8_K *)alloca(q8_size);
 
    const block_q5_0 *weights = (const block_q5_0 *)B;
    const int blocks_per_row = K / 32;
 
    for (int m = 0; m < M; m++) {
        quantize_row_q8_k(&A[m * K], A_q8, K);
 
        for (int n = 0; n < N; n++) {
            float sumf = 0.0f;
            const block_q5_0 *w_row = weights + n * blocks_per_row;
 
            for (int b = 0; b < blocks_per_row; b++) {
                int q8_block_idx = (b * 32) / QK_K;
                int q8_offset = (b * 32) % QK_K;
                sumf += dot_q5_0_q8_k_32_sse(&w_row[b], &A_q8[q8_block_idx], q8_offset);
            }
 
            C[m * N + n] = sumf + (bias ? bias[n] : 0.0f);
        }
    }
}

References C, dot_q5_0_q8_k_32_sse(), gemm_nt_q5_0_ref(), QK_K, and quantize_row_q8_k().

gemm_nt_q6_k_ref()

void gemm_nt_q6_k_ref	(	const float *	A,
		const void *	B,
		const float *	bias,
		float *	C,
		int	M,
		int	N,
		int	K
	)

Definition at line 243 of file gemm_kernels_q6k.c.

{
    gemm_nt_q6_k(A, B, bias, C, M, N, K);
}

References C, and gemm_nt_q6_k().

Referenced by gemm_nt_q6_k_sse().

gemm_nt_q6_k_sse()

void gemm_nt_q6_k_sse	(	const float *	A,
		const void *	B,
		const float *	bias,
		float *	C,
		int	M,
		int	N,
		int	K
	)

Definition at line 66 of file gemm_kernels_q6k_sse.c.

{
    if (K % QK_K != 0) {
        gemm_nt_q6_k_ref(A, B, bias, C, M, N, K);
        return;
    }
 
    size_t q8_size = (K / QK_K) * sizeof(block_q8_K);
    block_q8_K *A_q8 = (block_q8_K *)alloca(q8_size);
 
    const block_q6_K *weights = (const block_q6_K *)B;
    const int blocks_per_row = K / QK_K;
 
    for (int m = 0; m < M; m++) {
        quantize_row_q8_k(&A[m * K], A_q8, K);
 
        for (int n = 0; n < N; n++) {
            float sumf = 0.0f;
            const block_q6_K *w_row = weights + n * blocks_per_row;
 
            for (int b = 0; b < blocks_per_row; b++) {
                sumf += dot_q6_k_q8_k_256_sse(&w_row[b], &A_q8[b]);
            }
 
            C[m * N + n] = sumf + (bias ? bias[n] : 0.0f);
        }
    }
}

References C, dot_q6_k_q8_k_256_sse(), gemm_nt_q6_k_ref(), QK_K, and quantize_row_q8_k().

gemv_nvfp4_q8_0()

void gemv_nvfp4_q8_0	(	float *	output,
		const void *	weights,
		const float *	weight_scales,
		const void *	activations,
		int	rows,
		int	cols
	)

Definition at line 203 of file gemm_kernels_nvfp4.c.

{
    assert(rows >= 0 && cols >= 0 && cols % QK_NVFP4 == 0);
    const size_t row_bytes = (size_t)(cols / QK_NVFP4) * sizeof(block_nvfp4);
    const uint8_t *weight_bytes = (const uint8_t *)weights;
    for (int row = 0; row < rows; ++row) {
        const float scale = weight_scales ? weight_scales[row] : 1.0f;
        vec_dot_nvfp4_q8_0(cols, &output[row],
                          weight_bytes + (size_t)row * row_bytes,
                          activations, scale);
    }
}

References QK_NVFP4, and vec_dot_nvfp4_q8_0().

gemv_q4_k_q8_k_sse()

void gemv_q4_k_q8_k_sse	(	float *	y,
		const void *	W,
		const void *	x_q8,
		int	M,
		int	K
	)

Definition at line 33 of file gemm_kernels_q4k_sse.c.

{
    const block_q4_K *blocks = (const block_q4_K *)W;
    const block_q8_K *x = (const block_q8_K *)x_q8;
    const int blocks_per_row = K / QK_K;
 
    const __m128i mask_low = _mm_set1_epi8(0x0F);
 
    for (int row = 0; row < M; ++row) {
        float sumf = 0.0f;
        const block_q4_K *w_row = blocks + row * blocks_per_row;
 
        for (int i = 0; i < blocks_per_row; ++i) {
            const block_q4_K *b4 = &w_row[i];
            const block_q8_K *b8 = &x[i];
 
            // Unpack scales (same as ref)
            uint8_t sc[8], m_val[8];
            unpack_q4_k_scales(b4->scales, sc, m_val);
 
            float d = CK_FP16_TO_FP32(b4->d) * b8->d;
            float dmin = CK_FP16_TO_FP32(b4->dmin) * b8->d;
 
            int is = 0;
            int q_offset = 0;
 
            // Process 4 chunks of 64 elements (256 total)
            for (int j = 0; j < QK_K; j += 64) {
                // We process 32 bytes of qs (covering 64 elements via low/high nibbles)
                // We access qs[0..31] relative to q_offset
 
                // Accumulators for this 64-element chunk
                __m128i acc_lo = _mm_setzero_si128();
                __m128i acc_hi = _mm_setzero_si128();
 
                // Inner loop: 2 iters of 16 bytes (32 elements)
                for (int l = 0; l < 32; l += 16) {
                    // Load 16 bytes of Q4
                    __m128i q4_vec = _mm_loadu_si128((const __m128i *)(b4->qs + q_offset + l));
 
                    // Low nibbles -> correspond to q8_lo (elements j+l .. j+l+15)
                    __m128i q4_lo = _mm_and_si128(q4_vec, mask_low);
                    
                    // High nibbles -> correspond to q8_hi (elements j+32+l .. j+32+l+15)
                    __m128i q4_hi = _mm_and_si128(_mm_srli_epi16(q4_vec, 4), mask_low);
 
                    // Load Q8
                    __m128i q8_lo_vec = _mm_loadu_si128((const __m128i *)(b8->qs + j + l));
                    __m128i q8_hi_vec = _mm_loadu_si128((const __m128i *)(b8->qs + j + 32 + l));
 
                    // Expand and Multiply-Add: Q4(u8) * Q8(s8) -> i32
                    // Since Q4 is u8 and Q8 is s8, we use intermediate i16
                    
                    // LO PART
                    __m128i q4_lo_16_L = _mm_cvtepu8_epi16(q4_lo); // lower 8 -> 16
                    __m128i q8_lo_16_L = _mm_cvtepi8_epi16(q8_lo_vec);
                    __m128i prod_lo_L = _mm_madd_epi16(q4_lo_16_L, q8_lo_16_L); // i32
                    acc_lo = _mm_add_epi32(acc_lo, prod_lo_L);
 
                    __m128i q4_lo_16_H = _mm_cvtepu8_epi16(_mm_srli_si128(q4_lo, 8)); // upper 8 -> 16
                    __m128i q8_lo_16_H = _mm_cvtepi8_epi16(_mm_srli_si128(q8_lo_vec, 8));
                    __m128i prod_lo_H = _mm_madd_epi16(q4_lo_16_H, q8_lo_16_H); // i32
                    acc_lo = _mm_add_epi32(acc_lo, prod_lo_H);
 
                    // HI PART
                    __m128i q4_hi_16_L = _mm_cvtepu8_epi16(q4_hi);
                    __m128i q8_hi_16_L = _mm_cvtepi8_epi16(q8_hi_vec);
                    __m128i prod_hi_L = _mm_madd_epi16(q4_hi_16_L, q8_hi_16_L);
                    acc_hi = _mm_add_epi32(acc_hi, prod_hi_L);
 
                    __m128i q4_hi_16_H = _mm_cvtepu8_epi16(_mm_srli_si128(q4_hi, 8));
                    __m128i q8_hi_16_H = _mm_cvtepi8_epi16(_mm_srli_si128(q8_hi_vec, 8));
                    __m128i prod_hi_H = _mm_madd_epi16(q4_hi_16_H, q8_hi_16_H);
                    acc_hi = _mm_add_epi32(acc_hi, prod_hi_H);
                }
 
                int32_t sum_q4q8_lo = hsum_epi32_sse(acc_lo);
                int32_t sum_q4q8_hi = hsum_epi32_sse(acc_hi);
 
                /* bsums: each bsum is 16 elements */
                int32_t bsum_lo = (int32_t)b8->bsums[j / 16] +
                                  (int32_t)b8->bsums[j / 16 + 1];
                int32_t bsum_hi = (int32_t)b8->bsums[(j + 32) / 16] +
                                  (int32_t)b8->bsums[(j + 32) / 16 + 1];
 
                sumf += d * (float)sc[is] * (float)sum_q4q8_lo;
                sumf -= dmin * (float)m_val[is] * (float)bsum_lo;
                sumf += d * (float)sc[is + 1] * (float)sum_q4q8_hi;
                sumf -= dmin * (float)m_val[is + 1] * (float)bsum_hi;
 
                q_offset += 32;
                is += 2;
            }
        }
        y[row] = sumf;
    }
}

References block_q8_K::bsums, CK_FP16_TO_FP32, block_q4_K::d, block_q8_K::d, block_q4_K::dmin, hsum_epi32_sse(), QK_K, block_q4_K::qs, block_q8_K::qs, block_q4_K::scales, and unpack_q4_k_scales().

moe_swiglu_expert_forward_nvfp4_workspace()

int moe_swiglu_expert_forward_nvfp4_workspace	(	const float *	hidden,
		const int *	indices,
		const float *	routing_weights,
		const void *	expert_gate,
		const float *	expert_gate_scales,
		const void *	expert_up,
		const float *	expert_up_scales,
		const void *	expert_down,
		const float *	expert_down_scales,
		float *	output,
		int	rows,
		int	hidden_dim,
		int	intermediate_dim,
		int	n_experts,
		int	top_k,
		void *	workspace,
		size_t	workspace_bytes
	)

Definition at line 318 of file gemm_kernels_nvfp4.c.

{
    const size_t required = moe_swiglu_nvfp4_workspace_bytes(
        hidden_dim, intermediate_dim);
    if (!hidden || !indices || !routing_weights || !expert_gate ||
        !expert_gate_scales || !expert_up || !expert_up_scales ||
        !expert_down || !expert_down_scales || !output || !workspace ||
        required == 0 || workspace_bytes < required || rows <= 0 ||
        n_experts <= 0 || top_k <= 0 || top_k > n_experts) {
        return -1;
    }
 
    const size_t hidden_q8_bytes = ck_nvfp4_align64(
        ck_dtype_row_bytes(CK_DT_Q8_0, (size_t)hidden_dim));
    const size_t gate_up_bytes = ck_nvfp4_align64(
        2u * (size_t)intermediate_dim * sizeof(float));
    const size_t act_q8_bytes = ck_nvfp4_align64(
        ck_dtype_row_bytes(CK_DT_Q8_0, (size_t)intermediate_dim));
    float *expert_output = (float *)((uint8_t *)workspace + hidden_q8_bytes +
                                     gate_up_bytes + act_q8_bytes);
    const size_t up_expert_bytes = (size_t)intermediate_dim *
        ck_dtype_row_bytes(CK_DT_NVFP4, (size_t)hidden_dim);
    const size_t down_expert_bytes = (size_t)hidden_dim *
        ck_dtype_row_bytes(CK_DT_NVFP4, (size_t)intermediate_dim);
    memset(output, 0, (size_t)rows * (size_t)hidden_dim * sizeof(float));
 
    for (int row = 0; row < rows; ++row) {
        const float *x = hidden + (size_t)row * (size_t)hidden_dim;
        float *y = output + (size_t)row * (size_t)hidden_dim;
        for (int slot = 0; slot < top_k; ++slot) {
            const size_t route = (size_t)row * (size_t)top_k + (size_t)slot;
            const int expert = indices[route];
            if (expert < 0 || expert >= n_experts) {
                return -2;
            }
            ck_moe_swiglu_nvfp4_projection(
                x,
                (const uint8_t *)expert_gate + (size_t)expert * up_expert_bytes,
                expert_gate_scales[expert],
                (const uint8_t *)expert_up + (size_t)expert * up_expert_bytes,
                expert_up_scales[expert],
                (const uint8_t *)expert_down + (size_t)expert * down_expert_bytes,
                expert_down_scales[expert], expert_output,
                hidden_dim, intermediate_dim, workspace);
            const float route_weight = routing_weights[route];
            for (int h = 0; h < hidden_dim; ++h) {
                y[h] += route_weight * expert_output[h];
            }
        }
    }
    return 0;
}

References CK_DT_NVFP4, CK_DT_Q8_0, ck_dtype_row_bytes(), ck_moe_swiglu_nvfp4_projection(), ck_nvfp4_align64(), and moe_swiglu_nvfp4_workspace_bytes().

moe_swiglu_nvfp4_workspace_bytes()

size_t moe_swiglu_nvfp4_workspace_bytes	(	int	hidden_dim,
		int	intermediate_dim
	)

Definition at line 274 of file gemm_kernels_nvfp4.c.

{
    if (hidden_dim <= 0 || intermediate_dim <= 0 || hidden_dim % 64 != 0 ||
        intermediate_dim % 64 != 0) {
        return 0;
    }
    size_t bytes = ck_nvfp4_align64(
        ck_dtype_row_bytes(CK_DT_Q8_0, (size_t)hidden_dim));
    bytes += ck_nvfp4_align64(2u * (size_t)intermediate_dim * sizeof(float));
    bytes += ck_nvfp4_align64(
        ck_dtype_row_bytes(CK_DT_Q8_0, (size_t)intermediate_dim));
    bytes += ck_nvfp4_align64((size_t)hidden_dim * sizeof(float));
    return bytes;
}

References CK_DT_Q8_0, ck_dtype_row_bytes(), and ck_nvfp4_align64().

Referenced by moe_swiglu_expert_forward_nvfp4_workspace(), and moe_swiglu_shared_forward_nvfp4_workspace().

moe_swiglu_shared_forward_nvfp4_workspace()

int moe_swiglu_shared_forward_nvfp4_workspace	(	const float *	hidden,
		const float *	routed,
		const void *	shared_gate,
		const float *	shared_gate_scale,
		const void *	shared_up,
		const float *	shared_up_scale,
		const void *	shared_down,
		const float *	shared_down_scale,
		float *	output,
		int	rows,
		int	hidden_dim,
		int	intermediate_dim,
		float	combination_scale,
		void *	workspace,
		size_t	workspace_bytes
	)

Definition at line 377 of file gemm_kernels_nvfp4.c.

{
    const size_t required = moe_swiglu_nvfp4_workspace_bytes(
        hidden_dim, intermediate_dim);
    if (!hidden || !shared_gate || !shared_gate_scale || !shared_up ||
        !shared_up_scale || !shared_down || !shared_down_scale || !output ||
        !workspace || required == 0 || workspace_bytes < required || rows <= 0) {
        return -1;
    }
    const size_t hidden_q8_bytes = ck_nvfp4_align64(
        ck_dtype_row_bytes(CK_DT_Q8_0, (size_t)hidden_dim));
    const size_t gate_up_bytes = ck_nvfp4_align64(
        2u * (size_t)intermediate_dim * sizeof(float));
    const size_t act_q8_bytes = ck_nvfp4_align64(
        ck_dtype_row_bytes(CK_DT_Q8_0, (size_t)intermediate_dim));
    float *shared_output = (float *)((uint8_t *)workspace + hidden_q8_bytes +
                                     gate_up_bytes + act_q8_bytes);
    for (int row = 0; row < rows; ++row) {
        ck_moe_swiglu_nvfp4_projection(
            hidden + (size_t)row * (size_t)hidden_dim,
            shared_gate, shared_gate_scale[0], shared_up, shared_up_scale[0],
            shared_down, shared_down_scale[0], shared_output,
            hidden_dim, intermediate_dim, workspace);
        float *y = output + (size_t)row * (size_t)hidden_dim;
        const float *route = routed ? routed + (size_t)row * (size_t)hidden_dim : NULL;
        for (int h = 0; h < hidden_dim; ++h) {
            y[h] = combination_scale *
                   (shared_output[h] + (route ? route[h] : 0.0f));
        }
    }
    return 0;
}

References CK_DT_Q8_0, ck_dtype_row_bytes(), ck_moe_swiglu_nvfp4_projection(), ck_nvfp4_align64(), and moe_swiglu_nvfp4_workspace_bytes().

quantize_row_q8_0()

void quantize_row_q8_0	(	const float *	x,
		void *	vy,
		int	k
	)

Quantize FP32 to Q8_0 format (scalar reference)

Parameters

x	Input FP32 values
vy	Output Q8_0 blocks
k	Number of elements (must be multiple of 32)

Definition at line 125 of file gemm_kernels_q8_0.c.

{
    block_q8_0 *y = (block_q8_0 *)vy;
    const int nb = k / QK8_0;  /* QK8_0 = 32 */
 
#if defined(__AVX__)
    const __m256 sign_bit = _mm256_set1_ps(-0.0f);
 
    for (int i = 0; i < nb; i++) {
        __m256 v0 = _mm256_loadu_ps(x + 0);
        __m256 v1 = _mm256_loadu_ps(x + 8);
        __m256 v2 = _mm256_loadu_ps(x + 16);
        __m256 v3 = _mm256_loadu_ps(x + 24);
        x += QK8_0;
 
        __m256 max_abs = _mm256_andnot_ps(sign_bit, v0);
        max_abs = _mm256_max_ps(max_abs, _mm256_andnot_ps(sign_bit, v1));
        max_abs = _mm256_max_ps(max_abs, _mm256_andnot_ps(sign_bit, v2));
        max_abs = _mm256_max_ps(max_abs, _mm256_andnot_ps(sign_bit, v3));
 
        __m128 max4 = _mm_max_ps(_mm256_extractf128_ps(max_abs, 1),
                                 _mm256_castps256_ps128(max_abs));
        max4 = _mm_max_ps(max4, _mm_movehl_ps(max4, max4));
        max4 = _mm_max_ss(max4, _mm_movehdup_ps(max4));
        const float max_scalar = _mm_cvtss_f32(max4);
 
#if defined(__INTEL_LLVM_COMPILER)
        const float d = ck_q8_0_div_rounded_f32(max_scalar, 127.0f);
        const float id = max_scalar != 0.0f
            ? ck_q8_0_div_rounded_f32(127.0f, max_scalar)
            : 0.0f;
#else
        const float d = max_scalar / 127.0f;
        const float id = max_scalar != 0.0f ? 127.0f / max_scalar : 0.0f;
#endif
        y[i].d = CK_FP32_TO_FP16(d);
 
        const __m256 mul = _mm256_set1_ps(id);
        v0 = _mm256_mul_ps(v0, mul);
        v1 = _mm256_mul_ps(v1, mul);
        v2 = _mm256_mul_ps(v2, mul);
        v3 = _mm256_mul_ps(v3, mul);
 
        /* Match llama.cpp x86 Q8 quantization: nearest-even rounding. */
        v0 = _mm256_round_ps(v0, _MM_ROUND_NEAREST);
        v1 = _mm256_round_ps(v1, _MM_ROUND_NEAREST);
        v2 = _mm256_round_ps(v2, _MM_ROUND_NEAREST);
        v3 = _mm256_round_ps(v3, _MM_ROUND_NEAREST);
 
        __m256i i0 = _mm256_cvtps_epi32(v0);
        __m256i i1 = _mm256_cvtps_epi32(v1);
        __m256i i2 = _mm256_cvtps_epi32(v2);
        __m256i i3 = _mm256_cvtps_epi32(v3);
 
#if defined(__AVX2__)
        i0 = _mm256_packs_epi32(i0, i1);
        i2 = _mm256_packs_epi32(i2, i3);
        i0 = _mm256_packs_epi16(i0, i2);
 
        const __m256i perm = _mm256_setr_epi32(0, 4, 1, 5, 2, 6, 3, 7);
        i0 = _mm256_permutevar8x32_epi32(i0, perm);
        _mm256_storeu_si256((__m256i *)y[i].qs, i0);
#else
        __m128i ni0 = _mm256_castsi256_si128(i0);
        __m128i ni1 = _mm256_extractf128_si256(i0, 1);
        __m128i ni2 = _mm256_castsi256_si128(i1);
        __m128i ni3 = _mm256_extractf128_si256(i1, 1);
        __m128i ni4 = _mm256_castsi256_si128(i2);
        __m128i ni5 = _mm256_extractf128_si256(i2, 1);
        __m128i ni6 = _mm256_castsi256_si128(i3);
        __m128i ni7 = _mm256_extractf128_si256(i3, 1);
 
        ni0 = _mm_packs_epi32(ni0, ni1);
        ni2 = _mm_packs_epi32(ni2, ni3);
        ni4 = _mm_packs_epi32(ni4, ni5);
        ni6 = _mm_packs_epi32(ni6, ni7);
 
        ni0 = _mm_packs_epi16(ni0, ni2);
        ni4 = _mm_packs_epi16(ni4, ni6);
 
        _mm_storeu_si128((__m128i *)(y[i].qs + 0), ni0);
        _mm_storeu_si128((__m128i *)(y[i].qs + 16), ni4);
#endif
    }
#else
    for (int i = 0; i < nb; i++) {
        const float *xb = x + i * QK8_0;
 
        /* Find max absolute value in block */
        float amax = 0.0f;
        for (int j = 0; j < QK8_0; j++) {
            float av = xb[j] >= 0 ? xb[j] : -xb[j];
            if (av > amax) amax = av;
        }
 
        /* Compute scale: d = max / 127 */
        float d = amax / 127.0f;
        float id = d != 0.0f ? 127.0f / amax : 0.0f;
 
        /* Store scale as FP16 */
        y[i].d = CK_FP32_TO_FP16(d);
 
        /* Quantize values */
        for (int j = 0; j < QK8_0; j++) {
            float v = xb[j] * id;
            int q = ck_nearest_int_q8_0(v);
            if (q > 127) q = 127;
            if (q < -127) q = -127;
            y[i].qs[j] = (int8_t)q;
        }
    }
#endif
}

quantize_row_q8_k_sse()

void quantize_row_q8_k_sse	(	const float *	x,
		void *	vy,
		int	k
	)

Definition at line 22 of file quantize_row_q8_k_sse.c.

                                                            {
    if (!x || !vy || k <= 0) {
        return;
    }
    assert(k % QK_K == 0);
 
    const int nb = k / QK_K;
    block_q8_K *y = (block_q8_K *)vy;
 
    for (int i = 0; i < nb; ++i) {
        /* Keep the exact signed-max selection contract from llama.cpp/ref. */
        float max = 0.0f;
        float amax = 0.0f;
        for (int j = 0; j < QK_K; ++j) {
            const float xv = x[j];
            const float ax = fabsf(xv);
            if (ax > amax) {
                amax = ax;
                max = xv;
            }
        }
 
        if (amax == 0.0f) {
            y[i].d = 0.0f;
            memset(y[i].qs, 0, sizeof(y[i].qs));
            memset(y[i].bsums, 0, sizeof(y[i].bsums));
            x += QK_K;
            continue;
        }
 
        const float iscale = -127.0f / max;
        const __m128 v_iscale = _mm_set1_ps(iscale);
        const __m128 v_magic = _mm_set1_ps(12582912.0f);
        const __m128i v_mantissa = _mm_set1_epi32(0x007fffff);
        const __m128i v_bias = _mm_set1_epi32(0x00400000);
        const __m128i v_min = _mm_set1_epi32(-128);
        const __m128i v_max = _mm_set1_epi32(127);
 
        for (int j = 0; j < QK_K; j += 16) {
            const __m128 x0 = _mm_loadu_ps(x + j + 0);
            const __m128 x1 = _mm_loadu_ps(x + j + 4);
            const __m128 x2 = _mm_loadu_ps(x + j + 8);
            const __m128 x3 = _mm_loadu_ps(x + j + 12);
 
            __m128i q0 = _mm_sub_epi32(
                _mm_and_si128(_mm_castps_si128(_mm_add_ps(_mm_mul_ps(x0, v_iscale), v_magic)), v_mantissa),
                v_bias);
            __m128i q1 = _mm_sub_epi32(
                _mm_and_si128(_mm_castps_si128(_mm_add_ps(_mm_mul_ps(x1, v_iscale), v_magic)), v_mantissa),
                v_bias);
            __m128i q2 = _mm_sub_epi32(
                _mm_and_si128(_mm_castps_si128(_mm_add_ps(_mm_mul_ps(x2, v_iscale), v_magic)), v_mantissa),
                v_bias);
            __m128i q3 = _mm_sub_epi32(
                _mm_and_si128(_mm_castps_si128(_mm_add_ps(_mm_mul_ps(x3, v_iscale), v_magic)), v_mantissa),
                v_bias);
 
            q0 = _mm_min_epi32(_mm_max_epi32(q0, v_min), v_max);
            q1 = _mm_min_epi32(_mm_max_epi32(q1, v_min), v_max);
            q2 = _mm_min_epi32(_mm_max_epi32(q2, v_min), v_max);
            q3 = _mm_min_epi32(_mm_max_epi32(q3, v_min), v_max);
 
            const __m128i q01 = _mm_packs_epi32(q0, q1);
            const __m128i q23 = _mm_packs_epi32(q2, q3);
            const __m128i q0123 = _mm_packs_epi16(q01, q23);
 
            _mm_storeu_si128((__m128i *)(y[i].qs + j), q0123);
 
            int sum = 0;
            for (int ii = 0; ii < 16; ++ii) {
                sum += y[i].qs[j + ii];
            }
            y[i].bsums[j / 16] = (int16_t)sum;
        }
 
        y[i].d = 1.0f / iscale;
        x += QK_K;
    }
}

References block_q8_K::bsums, block_q8_K::d, QK_K, and block_q8_K::qs.

rmsnorm_q8_k_fused()

void rmsnorm_q8_k_fused	(	const float *	input,
		const float *	gamma,
		void *	vy,
		int	tokens,
		int	d_model,
		int	aligned_embed_dim,
		float	eps
	)

Definition at line 144 of file rmsnorm_q8_k_fused.c.

{
    const int T = tokens;
    const int D = d_model;
    block_q8_K *y = (block_q8_K *)vy;
 
    for (int t = 0; t < T; ++t) {
        const float *x = input + (size_t)t * (size_t)aligned_embed_dim;
 
        float sum_sq = 0.0f;
        for (int d = 0; d < D; ++d) {
            sum_sq += x[d] * x[d];
        }
        const float rstd = 1.0f / sqrtf(sum_sq / (float)D + eps);
 
        for (int b = 0; b < D / QK_K; ++b) {
            const float *xb = x + b * QK_K;
            const float *gb = gamma + b * QK_K;
            block_q8_K *out_block = &y[t * (D / QK_K) + b];
 
            float norm_buf[QK_K];
            float max_val = 0.0f;
            for (int d = 0; d < QK_K; ++d) {
                const float normalized = xb[d] * rstd * gb[d];
                norm_buf[d] = normalized;
                const float abs_val = fabsf(normalized);
                if (abs_val > max_val) {
                    max_val = abs_val;
                }
            }
 
            if (max_val == 0.0f) {
                out_block->d = 0.0f;
                memset(out_block->qs, 0, QK_K);
                memset(out_block->bsums, 0, sizeof(out_block->bsums));
                continue;
            }
 
            const float iscale = -127.0f / max_val;
            out_block->d = 1.0f / iscale;
            for (int j = 0; j < QK_K; j += 16) {
                int bsum = 0;
                for (int k = 0; k < 16; ++k) {
                    int q = (int)lrintf(norm_buf[j + k] * iscale);
                    if (q < -128) q = -128;
                    if (q > 127) q = 127;
                    out_block->qs[j + k] = (int8_t)q;
                    bsum += q;
                }
                out_block->bsums[j / 16] = (int16_t)bsum;
            }
        }
    }
}

References block_q8_K::bsums, block_q8_K::d, QK_K, and block_q8_K::qs.

unpack_q4_k_scales()

static void unpack_q4_k_scales ( const uint8_t *  scales, uint8_t *  sc, uint8_t *  m  )

inlinestatic

Unpack Q4_K sub-block scales and mins.

Parameters

scales	The packed scales[12] array from block_q4_K
sc	Output: 8 unpacked scale values (multiply by super-block d)
m	Output: 8 unpacked min values (multiply by super-block dmin)

This matches llama.cpp's get_scale_min_k4() function exactly. The 12-byte scales array layout:

• bytes 0-3: 6-bit scales[0-3] (high 2 bits used for scales[4-7])
• bytes 4-7: 6-bit mins[0-3] (high 2 bits used for mins[4-7])
• bytes 8-11: low 4 bits for scales[4-7], high 4 bits for mins[4-7]

Definition at line 287 of file ckernel_quant.h.

                                                                {
    /* Direct 6-bit values for indices 0-3 */
    sc[0] = scales[0] & 0x3F;
    sc[1] = scales[1] & 0x3F;
    sc[2] = scales[2] & 0x3F;
    sc[3] = scales[3] & 0x3F;
 
    m[0] = scales[4] & 0x3F;
    m[1] = scales[5] & 0x3F;
    m[2] = scales[6] & 0x3F;
    m[3] = scales[7] & 0x3F;
 
    /* 6-bit values for indices 4-7: low 4 bits from bytes 8-11,
     * high 2 bits from upper bits of bytes 0-3 (scales) and 4-7 (mins) */
    sc[4] = (scales[8]  & 0x0F) | ((scales[0] >> 6) << 4);
    sc[5] = (scales[9]  & 0x0F) | ((scales[1] >> 6) << 4);
    sc[6] = (scales[10] & 0x0F) | ((scales[2] >> 6) << 4);
    sc[7] = (scales[11] & 0x0F) | ((scales[3] >> 6) << 4);
 
    m[4] = (scales[8]  >> 4) | ((scales[4] >> 6) << 4);
    m[5] = (scales[9]  >> 4) | ((scales[5] >> 6) << 4);
    m[6] = (scales[10] >> 4) | ((scales[6] >> 6) << 4);
    m[7] = (scales[11] >> 4) | ((scales[7] >> 6) << 4);
}

Referenced by dequant_q4_k_block(), dot_q4_k_q8_k_ref(), gemv_q4_k_backward_ref(), gemv_q4_k_q8_k_sse(), gemv_q4_k_ref(), pack_q4_k_to_packed_meta(), pack_q4_k_to_packed_meta_x16(), pack_q4_k_to_packed_meta_x8(), pack_q4_k_to_packed_u8(), pack_q4_k_to_packed_u8_x16(), pack_q4_k_to_packed_vnni_x16(), and pack_q4_k_to_packed_vnni_x8().

vec_dot_nvfp4_q8_0()

void vec_dot_nvfp4_q8_0	(	int	n,
		float *	output,
		const void *	weights,
		const void *	activations,
		float	weight_scale
	)

Definition at line 125 of file gemm_kernels_nvfp4.c.

{
#if defined(__AVX2__)
    assert(n >= 0 && n % QK_NVFP4 == 0);
    const block_nvfp4 *w = (const block_nvfp4 *)weights;
    const block_q8_0 *x = (const block_q8_0 *)activations;
    const int block_count = n / QK_NVFP4;
    const __m128i lut = _mm_loadu_si128(
        (const __m128i *)ck_nvfp4_e2m1_x2);
    const __m128i nibble_mask = _mm_set1_epi8(0x0f);
    const __m256i ones = _mm256_set1_epi16(1);
    __m256 accumulated = _mm256_setzero_ps();
 
    for (int block_index = 0; block_index < block_count; ++block_index) {
        const block_nvfp4 *block = &w[block_index];
        const __m128i packed01 = _mm_loadu_si128(
            (const __m128i *)(block->qs + 0));
        const __m128i packed23 = _mm_loadu_si128(
            (const __m128i *)(block->qs + 16));
        const __m128i low01 = _mm_shuffle_epi8(
            lut, _mm_and_si128(packed01, nibble_mask));
        const __m128i high01 = _mm_shuffle_epi8(
            lut, _mm_and_si128(_mm_srli_epi16(packed01, 4), nibble_mask));
        const __m128i low23 = _mm_shuffle_epi8(
            lut, _mm_and_si128(packed23, nibble_mask));
        const __m128i high23 = _mm_shuffle_epi8(
            lut, _mm_and_si128(_mm_srli_epi16(packed23, 4), nibble_mask));
 
        __m256i values01 = _mm256_castsi128_si256(
            _mm_unpacklo_epi64(low01, high01));
        values01 = _mm256_inserti128_si256(
            values01, _mm_unpackhi_epi64(low01, high01), 1);
        __m256i values23 = _mm256_castsi128_si256(
            _mm_unpacklo_epi64(low23, high23));
        values23 = _mm256_inserti128_si256(
            values23, _mm_unpackhi_epi64(low23, high23), 1);
 
        const __m256i q8_01 = _mm256_loadu_si256(
            (const __m256i *)x[2 * block_index + 0].qs);
        const __m256i q8_23 = _mm256_loadu_si256(
            (const __m256i *)x[2 * block_index + 1].qs);
        const __m256i dot01 = _mm256_madd_epi16(
            ck_nvfp4_mul_add_i8_avx2(values01, q8_01), ones);
        const __m256i dot23 = _mm256_madd_epi16(
            ck_nvfp4_mul_add_i8_avx2(values23, q8_23), ones);
 
        const float q8_scale0 =
            CK_FP16_TO_FP32(x[2 * block_index + 0].d);
        const float q8_scale1 =
            CK_FP16_TO_FP32(x[2 * block_index + 1].d);
        const float scale0 = ck_ue4m3_to_fp32_inline(block->d[0]) * q8_scale0;
        const float scale1 = ck_ue4m3_to_fp32_inline(block->d[1]) * q8_scale0;
        const float scale2 = ck_ue4m3_to_fp32_inline(block->d[2]) * q8_scale1;
        const float scale3 = ck_ue4m3_to_fp32_inline(block->d[3]) * q8_scale1;
        const __m256 scales01 = _mm256_insertf128_ps(
            _mm256_castps128_ps256(_mm_set1_ps(scale0)),
            _mm_set1_ps(scale1), 1);
        const __m256 scales23 = _mm256_insertf128_ps(
            _mm256_castps128_ps256(_mm_set1_ps(scale2)),
            _mm_set1_ps(scale3), 1);
        accumulated = _mm256_fmadd_ps(
            scales01, _mm256_cvtepi32_ps(dot01), accumulated);
        accumulated = _mm256_fmadd_ps(
            scales23, _mm256_cvtepi32_ps(dot23), accumulated);
    }
 
    __m128 sum4 = _mm_add_ps(
        _mm256_castps256_ps128(accumulated),
        _mm256_extractf128_ps(accumulated, 1));
    sum4 = _mm_hadd_ps(sum4, sum4);
    sum4 = _mm_hadd_ps(sum4, sum4);
    *output = _mm_cvtss_f32(sum4) * weight_scale;
#else
    vec_dot_nvfp4_q8_0_ref(n, output, weights, activations, weight_scale);
#endif
}

References CK_FP16_TO_FP32, ck_nvfp4_e2m1_x2, ck_ue4m3_to_fp32_inline(), block_nvfp4::d, QK_NVFP4, block_nvfp4::qs, and vec_dot_nvfp4_q8_0_ref().

Referenced by ck_nvfp4_gemv_rows(), and gemv_nvfp4_q8_0().

vec_dot_nvfp4_q8_0_ref()

void vec_dot_nvfp4_q8_0_ref	(	int	n,
		float *	output,
		const void *	weights,
		const void *	activations,
		float	weight_scale
	)

Definition at line 81 of file gemm_kernels_nvfp4.c.

{
    assert(n >= 0 && n % QK_NVFP4 == 0);
    const block_nvfp4 *w = (const block_nvfp4 *)weights;
    const block_q8_0 *x = (const block_q8_0 *)activations;
    const int block_count = n / QK_NVFP4;
    float sum = 0.0f;
 
    for (int block_index = 0; block_index < block_count; ++block_index) {
        for (int sub = 0; sub < QK_NVFP4 / QK_NVFP4_SUB; ++sub) {
            const int q8_block = sub / 2;
            const int q8_offset = (sub % 2) * QK_NVFP4_SUB;
            const float scale = ck_ue4m3_to_fp32_inline(w[block_index].d[sub]) *
                                CK_FP16_TO_FP32(x[2 * block_index + q8_block].d) *
                                weight_scale;
            const uint8_t *packed =
                &w[block_index].qs[sub * (QK_NVFP4_SUB / 2)];
            const int8_t *q8 = &x[2 * block_index + q8_block].qs[q8_offset];
            int integer_sum = 0;
            for (int lane = 0; lane < QK_NVFP4_SUB / 2; ++lane) {
                const uint8_t pair = packed[lane];
                integer_sum += (int)q8[lane] *
                               (int)ck_nvfp4_e2m1_x2[pair & 0x0f];
                integer_sum += (int)q8[lane + QK_NVFP4_SUB / 2] *
                               (int)ck_nvfp4_e2m1_x2[pair >> 4];
            }
            sum += scale * (float)integer_sum;
        }
    }
    *output = sum;
}

References CK_FP16_TO_FP32, ck_nvfp4_e2m1_x2, ck_ue4m3_to_fp32_inline(), QK_NVFP4, QK_NVFP4_SUB, block_q8_0::qs, and block_nvfp4::qs.

Referenced by vec_dot_nvfp4_q8_0().

vec_dot_q5_0_q8_0()

void vec_dot_q5_0_q8_0	(	int	n,
		float *	s,
		const void *	vx,
		const void *	vy
	)

Auto-dispatch quantized dot product Q5_0 x Q8_0.

Dispatch priority:

1. AVX512 (best performance on modern Intel/AMD)
2. AVX (256-bit float ops, works on Sandy/Ivy Bridge and newer)
3. SSSE3 (128-bit fallback)
4. Reference scalar (last resort)

Definition at line 1602 of file gemm_kernels_q5_0.c.

{
#if defined(__AVX2__)
    /* llama.cpp uses the packed AVX2 dot on AVX-512 hosts as well. It keeps
     * Q5/Q8 data in byte lanes and avoids the per-block 32-bit lane expansion
     * overhead of the baseline AVX-512 path. */
    vec_dot_q5_0_q8_0_avx2(n, s, vx, vy);
#elif defined(__AVX512F__)
    vec_dot_q5_0_q8_0_avx512(n, s, vx, vy);
#elif defined(__ARM_NEON) || defined(__aarch64__)
    vec_dot_q5_0_q8_0_neon(n, s, vx, vy);
#elif defined(__AVX__)
    /* AVX for 256-bit float ops (works on Ivy Bridge and newer) */
    vec_dot_q5_0_q8_0_avx(n, s, vx, vy);
#elif defined(__SSSE3__)
    /* SSSE3 - most efficient on older CPUs */
    vec_dot_q5_0_q8_0_sse(n, s, vx, vy);
#else
    vec_dot_q5_0_q8_0_ref(n, s, vx, vy);
#endif
}

References vec_dot_q5_0_q8_0_ref().

Referenced by out_proj_head_major_q5_0_q8_0(), and out_proj_head_major_q5_0_q8_0().

vec_dot_q8_0_q8_0()

void vec_dot_q8_0_q8_0	(	int	n,
		float *	s,
		const void *	vx,
		const void *	vy
	)

Auto-dispatch quantized dot product Q8_0 x Q8_0.

Definition at line 1368 of file gemm_kernels_q8_0.c.

{
    if (ck_q8_0_q8_0_debug_ref()) {
        vec_dot_q8_0_q8_0_ref(n, s, vx, vy);
        return;
    }
#ifdef __AVX512F__
    vec_dot_q8_0_q8_0_avx512(n, s, vx, vy);
#elif defined(__AVX2__)
    vec_dot_q8_0_q8_0_avx2(n, s, vx, vy);
#elif defined(__ARM_NEON) || defined(__aarch64__)
    vec_dot_q8_0_q8_0_neon(n, s, vx, vy);
#elif defined(__AVX__)
    vec_dot_q8_0_q8_0_avx(n, s, vx, vy);
#elif defined(__SSE4_1__)
    vec_dot_q8_0_q8_0_sse(n, s, vx, vy);
#else
    vec_dot_q8_0_q8_0_ref(n, s, vx, vy);
#endif
}

References ck_q8_0_q8_0_debug_ref(), and vec_dot_q8_0_q8_0_ref().

Referenced by out_proj_head_major_q8_0_q8_0(), and out_proj_head_major_q8_0_q8_0().