gemm_kernels_q4k_q8k.c File Reference

Q4_K (weights) x Q8_K (activations) kernels for inference. More...

#include <assert.h>
#include <immintrin.h>
#include <math.h>
#include <stdlib.h>
#include <string.h>
#include "ck_threadpool.h"
#include "ckernel_quant.h"

Go to the source code of this file.

Functions
static int	ck_nearest_int (float fval)
static int	ck_q4k_q8k_force_ref (void)
static float	dot_q4_k_q8_k_ref (const block_q4_K *w, const block_q8_K *x, int k)
void	gemm_nt_q4_k_q8_k (const void *A_q8, const void *B, const float *bias, float *C, int M, int N, int K)
void	gemm_q4_k_q8_k (float *Y, const void *W, const void *X_q8, int M, int N, int K)
void	gemm_q4_k_q8_k_ref (float *Y, const void *W, const void *X_q8, int M, int N, int K)
static void	gemm_q4_k_q8_k_thread_fn (int ith, int nth, void *args)
void	gemv_q4_k_q8_k (float *y, const void *W, const void *x_q8, int M, int K)
void	gemv_q4_k_q8_k_avx (float *y, const void *W, const void *x_q8, int M, int K)
void	gemv_q4_k_q8_k_avx2 (float *y, const void *W, const void *x_q8, int M, int K)
void	gemv_q4_k_q8_k_parallel (float *y, const void *W, const void *x_q8, int M, int K, int ith, int nth)
void	gemv_q4_k_q8_k_ref (float *y, const void *W, const void *x_q8, int M, int K)
void	gemv_q4_k_q8_k_sse (float *y, const void *W, const void *x_q8, int M, int K)
void	gemv_q4_k_q8_k_vnni (float *y, const void *W, const void *x_q8, int M, int K)
void	quantize_batch_q8_k_4row_nearest_even (const float *x, void *vy, int num_rows, int k)
void	quantize_row_q8_k (const float *x, void *vy, int k)
void	quantize_row_q8_k_avx (const float *x, void *vy, int k)
void	quantize_row_q8_k_avx2 (const float *x, void *vy, int k)
void	quantize_row_q8_k_avx512 (const float *x, void *vy, int k)
void	quantize_row_q8_k_ref (const float *x, void *vy, int k)
void	quantize_row_q8_k_sse (const float *x, void *vy, int k)

Detailed Description

Q4_K (weights) x Q8_K (activations) kernels for inference.

CK-ENGINE KERNEL RULES:

1. NO malloc/free - memory via bump allocator, pointers passed in
2. NO OpenMP - parallelization at orchestrator/codegen layer
3. API must define: inputs, outputs, workspace, and memory layouts
4. Pure computation - deterministic, no side effects

After changes: make test && make llamacpp-parity-full

Implements decode-style matvec/matmul where weights are Q4_K and the activations are quantized on-the-fly to Q8_K. This is inference-only; no backward pass is provided here.

Definition in file gemm_kernels_q4k_q8k.c.

Function Documentation

ck_nearest_int()

static int ck_nearest_int ( float  fval )

inlinestatic

Definition at line 48 of file gemm_kernels_q4k_q8k.c.

                                             {
    /* Bit-level round-to-nearest from llama.cpp (fast + deterministic). */
    float val = fval + 12582912.f;
    int i;
    memcpy(&i, &val, sizeof(int));
    return (i & 0x007fffff) - 0x00400000;
}

Referenced by quantize_row_q8_k_ref().

ck_q4k_q8k_force_ref()

static int ck_q4k_q8k_force_ref ( void  )

static

Definition at line 220 of file gemm_kernels_q4k_q8k.c.

{
    static int cached = -1;
    if (cached < 0) {
        const char *env = getenv("CK_DEBUG_Q4K_Q8_REF");
        cached = (env && env[0] && env[0] != '0') ? 1 : 0;
    }
    return cached;
}

Referenced by gemv_q4_k_q8_k().

dot_q4_k_q8_k_ref()

static float dot_q4_k_q8_k_ref ( const block_q4_K *  w, const block_q8_K *  x, int  k  )

static

Definition at line 163 of file gemm_kernels_q4k_q8k.c.

{
    const int nb = k / QK_K;
    float sumf = 0.0f;
 
    for (int i = 0; i < nb; ++i) {
        uint8_t sc[8], m_val[8];
        unpack_q4_k_scales(w[i].scales, sc, m_val);
 
        const float d = CK_FP16_TO_FP32(w[i].d) * x[i].d;
        const float dmin = CK_FP16_TO_FP32(w[i].dmin) * x[i].d;
 
        int sumi = 0;
        for (int j = 0; j < QK_K / 16; ++j) {
            sumi += (int)x[i].bsums[j] * (int)m_val[j / 2];
        }
 
        int32_t scaled_sum = 0;
        for (int group = 0; group < 4; ++group) {
            const uint8_t *qs = &w[i].qs[group * 32];
            const int8_t *q8_lo = &x[i].qs[group * 64];
            const int8_t *q8_hi = q8_lo + 32;
            int32_t lo = 0;
            int32_t hi = 0;
            for (int l = 0; l < 32; ++l) {
                lo += (int32_t)(qs[l] & 0x0F) * (int32_t)q8_lo[l];
                hi += (int32_t)(qs[l] >> 4) * (int32_t)q8_hi[l];
            }
            scaled_sum += (int32_t)sc[2 * group] * lo;
            scaled_sum += (int32_t)sc[2 * group + 1] * hi;
        }
        sumf += d * (float)scaled_sum - dmin * (float)sumi;
    }
    return sumf;
}

References CK_FP16_TO_FP32, block_q8_K::d, QK_K, block_q4_K::qs, block_q8_K::qs, and unpack_q4_k_scales().

Referenced by gemv_q4_k_q8_k_parallel(), and gemv_q4_k_q8_k_ref().

gemm_nt_q4_k_q8_k()

void gemm_nt_q4_k_q8_k	(	const void *	A_q8,
		const void *	B,
		const float *	bias,
		float *	C,
		int	M,
		int	N,
		int	K
	)

Definition at line 397 of file gemm_kernels_q4k_q8k.c.

{
    if (!A_q8 || !B || !C) {
        return;
    }
    if (M <= 0 || N <= 0 || K <= 0) {
        return;
    }
 
    gemm_q4_k_q8_k(C, B, A_q8, /*M_out=*/N, /*N_batch=*/M, K);
 
    if (!bias) {
        return;
    }
 
    for (int i = 0; i < M; ++i) {
        float *row = C + (size_t)i * (size_t)N;
        for (int j = 0; j < N; ++j) {
            row[j] += bias[j];
        }
    }
}

References C, and gemm_q4_k_q8_k().

Referenced by ck_attention_project_head_major_q4_k_q8_k(), ck_layer_forward_rmsnorm_swiglu_decode_q4_k(), ck_mlp_swiglu_forward_q4_k_q8_k(), ck_mlp_swiglu_forward_q4_k_q8_k_prefill(), ck_qkv_project_head_major_token_q4_k_q8_k(), ck_test_gemm_q4_k(), gemm_nt_q8_k_mlp_dispatch(), gemm_nt_q8_k_qkv_dispatch(), model_forward_prefill_impl(), model_forward_prefill_impl(), model_forward_prefill_impl(), model_forward_prefill_impl(), qwen2_0_5b_decode_forward_prefill_impl(), qwen2_0_5b_decode_forward_prefill_impl(), qwen2_0_5b_decode_forward_prefill_impl(), and qwen2_0_5b_decode_forward_prefill_impl().

gemm_q4_k_q8_k()

void gemm_q4_k_q8_k	(	float *	Y,
		const void *	W,
		const void *	X_q8,
		int	M,
		int	N,
		int	K
	)

Definition at line 354 of file gemm_kernels_q4k_q8k.c.

{
    if (!Y || !W || !X_q8 || M <= 0 || N <= 0 || K <= 0) {
        return;
    }
 
    const block_q8_K *X = (const block_q8_K *)X_q8;
    const int blocks_per_vec = K / QK_K;
    const int blocks_per_row = K / QK_K;
    const size_t work_items = (size_t)M * (size_t)N;
 
    if (work_items >= 4096u && M >= 512 && N > 1) {
        ck_threadpool_t *pool = ck_threadpool_global();
        const int pool_threads = pool ? ck_threadpool_n_threads(pool) : 1;
        int active_threads = pool_threads;
        if (active_threads > M) {
            active_threads = M;
        }
        if (active_threads > 1) {
            gemm_q4_k_q8_k_work_t work = {
                .Y = Y,
                .W = W,
                .X = X,
                .M_out = M,
                .N_batch = N,
                .K = K,
                .blocks_per_vec = blocks_per_vec,
                .blocks_per_row = blocks_per_row,
            };
            ck_threadpool_dispatch_n(pool, active_threads, gemm_q4_k_q8_k_thread_fn, &work);
            return;
        }
    }
 
    for (int n = 0; n < N; ++n) {
        const block_q8_K *x_row = X + (size_t)n * (size_t)blocks_per_vec;
        gemv_q4_k_q8_k(&Y[n * M], W, x_row, M, K);
    }
}

References ck_threadpool_dispatch_n(), ck_threadpool_global(), ck_threadpool_n_threads(), gemm_q4_k_q8_k_thread_fn(), gemv_q4_k_q8_k(), and QK_K.

Referenced by gemm_nt_q4_k_q8_k().

gemm_q4_k_q8_k_ref()

void gemm_q4_k_q8_k_ref	(	float *	Y,
		const void *	W,
		const void *	X_q8,
		int	M,
		int	N,
		int	K
	)

Definition at line 297 of file gemm_kernels_q4k_q8k.c.

{
    if (!Y || !W || !X_q8 || M <= 0 || N <= 0 || K <= 0) {
        return;
    }
 
    const block_q8_K *X = (const block_q8_K *)X_q8;
    const int blocks_per_vec = K / QK_K;
 
    for (int n = 0; n < N; ++n) {
        const block_q8_K *x_row = X + (size_t)n * (size_t)blocks_per_vec;
        gemv_q4_k_q8_k_ref(&Y[n * M], W, x_row, M, K);
    }
}

References gemv_q4_k_q8_k_ref(), and QK_K.

gemm_q4_k_q8_k_thread_fn()

static void gemm_q4_k_q8_k_thread_fn ( int  ith, int  nth, void *  args  )

static

Definition at line 326 of file gemm_kernels_q4k_q8k.c.

{
    gemm_q4_k_q8_k_work_t *a = (gemm_q4_k_q8_k_work_t *)args;
    if (!a || ith < 0 || nth <= 0 || ith >= nth) {
        return;
    }
 
    const int dr = (a->M_out + nth - 1) / nth;
    const int r0 = dr * ith;
    const int r1 = (r0 + dr < a->M_out) ? (r0 + dr) : a->M_out;
    if (r0 >= a->M_out) {
        return;
    }
 
    const block_q4_K *blocks = (const block_q4_K *)a->W;
    const block_q4_K *w_start = blocks + (size_t)r0 * (size_t)a->blocks_per_row;
    const int rows = r1 - r0;
 
    for (int n = 0; n < a->N_batch; ++n) {
        const block_q8_K *x_row = a->X + (size_t)n * (size_t)a->blocks_per_vec;
        gemv_q4_k_q8_k(a->Y + (size_t)n * (size_t)a->M_out + (size_t)r0,
                       w_start,
                       x_row,
                       rows,
                       a->K);
    }
}

References gemv_q4_k_q8_k().

Referenced by gemm_q4_k_q8_k().

gemv_q4_k_q8_k()

void gemv_q4_k_q8_k	(	float *	y,
		const void *	W,
		const void *	x_q8,
		int	M,
		int	K
	)

Definition at line 273 of file gemm_kernels_q4k_q8k.c.

{
    if (ck_q4k_q8k_force_ref()) {
        gemv_q4_k_q8_k_ref(y, W, x_q8, M, K);
        return;
    }
#if defined(__AVX512VNNI__) && defined(__AVX512VL__) && !defined(CK_NO_AVX512_VNNI)
    /* VNNI: Best for decode (single token) - INT8 dot product acceleration */
    gemv_q4_k_q8_k_vnni(y, W, x_q8, M, K);
#elif defined(__AVX2__)
    gemv_q4_k_q8_k_avx2(y, W, x_q8, M, K);
#elif defined(__AVX__)
    /* AVX version uses maddubs_epi16 (more efficient than SSE) */
    gemv_q4_k_q8_k_avx(y, W, x_q8, M, K);
#elif defined(__SSE4_1__)
    gemv_q4_k_q8_k_sse(y, W, x_q8, M, K);
#else
    gemv_q4_k_q8_k_ref(y, W, x_q8, M, K);
#endif
}

References ck_q4k_q8k_force_ref(), gemv_q4_k_q8_k_avx(), gemv_q4_k_q8_k_avx2(), gemv_q4_k_q8_k_ref(), gemv_q4_k_q8_k_sse(), and gemv_q4_k_q8_k_vnni().

Referenced by ck_moe_q4k_llama_projection(), ck_moe_q4k_mixed_route_work(), ck_moe_q4k_q5k_route_work(), ck_test_gemv_q4_k(), ck_test_vec_dot_q4_k_q8_k(), fused_rmsnorm_linear_q4k(), gemm_q4_k_q8_k(), gemm_q4_k_q8_k_compact_rows4(), gemm_q4_k_q8_k_thread_fn(), gemv_q4_k(), model_decode_token(), model_decode_token(), model_layer_0_decode(), model_layer_0_decode(), model_layer_10_decode(), model_layer_10_decode(), model_layer_11_decode(), model_layer_11_decode(), model_layer_12_decode(), model_layer_12_decode(), model_layer_13_decode(), model_layer_13_decode(), model_layer_14_decode(), model_layer_14_decode(), model_layer_15_decode(), model_layer_15_decode(), model_layer_16_decode(), model_layer_16_decode(), model_layer_17_decode(), model_layer_17_decode(), model_layer_18_decode(), model_layer_18_decode(), model_layer_19_decode(), model_layer_19_decode(), model_layer_1_decode(), model_layer_1_decode(), model_layer_20_decode(), model_layer_20_decode(), model_layer_21_decode(), model_layer_21_decode(), model_layer_22_decode(), model_layer_22_decode(), model_layer_23_decode(), model_layer_23_decode(), model_layer_2_decode(), model_layer_2_decode(), model_layer_3_decode(), model_layer_3_decode(), model_layer_4_decode(), model_layer_4_decode(), model_layer_5_decode(), model_layer_5_decode(), model_layer_6_decode(), model_layer_6_decode(), model_layer_7_decode(), model_layer_7_decode(), model_layer_8_decode(), model_layer_8_decode(), model_layer_9_decode(), model_layer_9_decode(), moe_swiglu_expert_forward_q4k_q4k_workspace(), moe_swiglu_expert_forward_q4k_q5k_workspace(), moe_swiglu_expert_forward_q4k_q6k_workspace(), moe_swiglu_shared_forward_q4k_q4k_workspace(), moe_swiglu_shared_forward_q4k_q6k_workspace(), qwen2_0_5b_decode_decode_token(), qwen2_0_5b_decode_decode_token(), qwen2_0_5b_decode_layer_0_decode(), qwen2_0_5b_decode_layer_0_decode(), qwen2_0_5b_decode_layer_10_decode(), qwen2_0_5b_decode_layer_10_decode(), qwen2_0_5b_decode_layer_11_decode(), qwen2_0_5b_decode_layer_11_decode(), qwen2_0_5b_decode_layer_12_decode(), qwen2_0_5b_decode_layer_12_decode(), qwen2_0_5b_decode_layer_13_decode(), qwen2_0_5b_decode_layer_13_decode(), qwen2_0_5b_decode_layer_14_decode(), qwen2_0_5b_decode_layer_14_decode(), qwen2_0_5b_decode_layer_15_decode(), qwen2_0_5b_decode_layer_15_decode(), qwen2_0_5b_decode_layer_16_decode(), qwen2_0_5b_decode_layer_16_decode(), qwen2_0_5b_decode_layer_17_decode(), qwen2_0_5b_decode_layer_17_decode(), qwen2_0_5b_decode_layer_18_decode(), qwen2_0_5b_decode_layer_18_decode(), qwen2_0_5b_decode_layer_19_decode(), qwen2_0_5b_decode_layer_19_decode(), qwen2_0_5b_decode_layer_1_decode(), qwen2_0_5b_decode_layer_1_decode(), qwen2_0_5b_decode_layer_20_decode(), qwen2_0_5b_decode_layer_20_decode(), qwen2_0_5b_decode_layer_21_decode(), qwen2_0_5b_decode_layer_21_decode(), qwen2_0_5b_decode_layer_22_decode(), qwen2_0_5b_decode_layer_22_decode(), qwen2_0_5b_decode_layer_23_decode(), qwen2_0_5b_decode_layer_23_decode(), qwen2_0_5b_decode_layer_2_decode(), qwen2_0_5b_decode_layer_2_decode(), qwen2_0_5b_decode_layer_3_decode(), qwen2_0_5b_decode_layer_3_decode(), qwen2_0_5b_decode_layer_4_decode(), qwen2_0_5b_decode_layer_4_decode(), qwen2_0_5b_decode_layer_5_decode(), qwen2_0_5b_decode_layer_5_decode(), qwen2_0_5b_decode_layer_6_decode(), qwen2_0_5b_decode_layer_6_decode(), qwen2_0_5b_decode_layer_7_decode(), qwen2_0_5b_decode_layer_7_decode(), qwen2_0_5b_decode_layer_8_decode(), qwen2_0_5b_decode_layer_8_decode(), qwen2_0_5b_decode_layer_9_decode(), qwen2_0_5b_decode_layer_9_decode(), and unfused_rmsnorm_linear_q4k_ref().

gemv_q4_k_q8_k_avx()

void gemv_q4_k_q8_k_avx	(	float *	y,
		const void *	W,
		const void *	x_q8,
		int	M,
		int	K
	)

Definition at line 251 of file gemm_kernels_q4k_avx.c.

{
    gemv_q4_k_q8_k_ref(y, W, x_q8, M, K);
}

Referenced by gemv_q4_k_q8_k().

gemv_q4_k_q8_k_avx2()

void gemv_q4_k_q8_k_avx2	(	float *	y,
		const void *	W,
		const void *	x_q8,
		int	M,
		int	K
	)

Definition at line 118 of file gemm_kernels_q4k_q8k_avx2.c.

{
#if defined(__AVX2__)
    if (!y || !W || !x_q8 || M <= 0 || K <= 0) {
        return;
    }
 
    const block_q4_K *blocks = (const block_q4_K *)W;
    const block_q8_K *x = (const block_q8_K *)x_q8;
    const int blocks_per_row = K / QK_K;
 
    for (int row = 0; row < M; ++row) {
        const block_q4_K *w_row = blocks + (size_t)row * (size_t)blocks_per_row;
        y[row] = dot_q4_k_q8_k_avx2(w_row, x, K);
    }
#else
    gemv_q4_k_q8_k_ref(y, W, x_q8, M, K);
#endif
}

Referenced by gemv_q4_k_q8_k().

gemv_q4_k_q8_k_parallel()

void gemv_q4_k_q8_k_parallel	(	float *	y,
		const void *	W,
		const void *	x_q8,
		int	M,
		int	K,
		int	ith,
		int	nth
	)

Definition at line 240 of file gemm_kernels_q4k_q8k.c.

{
    if (!y || !W || !x_q8 || M <= 0 || K <= 0) {
        return;
    }
    if (ith < 0 || nth <= 0 || ith >= nth) {
        return;
    }
 
    /* Compute row range for this thread */
    const int dr = (M + nth - 1) / nth;
    const int r0 = dr * ith;
    const int r1 = (r0 + dr < M) ? (r0 + dr) : M;
 
    if (r0 >= M) {
        return;  /* This thread has no work */
    }
 
    const block_q4_K *blocks = (const block_q4_K *)W;
    const block_q8_K *x = (const block_q8_K *)x_q8;
    const int blocks_per_row = K / QK_K;
 
    /* Only process rows [r0, r1) */
    for (int row = r0; row < r1; ++row) {
        const block_q4_K *w_row = blocks + (size_t)row * (size_t)blocks_per_row;
        y[row] = dot_q4_k_q8_k_ref(w_row, x, K);
    }
}

References dot_q4_k_q8_k_ref(), and QK_K.

Referenced by gemv_q4_k_q8_k_parallel_simd().

gemv_q4_k_q8_k_ref()

void gemv_q4_k_q8_k_ref	(	float *	y,
		const void *	W,
		const void *	x_q8,
		int	M,
		int	K
	)

Definition at line 201 of file gemm_kernels_q4k_q8k.c.

{
    if (!y || !W || !x_q8 || M <= 0 || K <= 0) {
        return;
    }
 
    const block_q4_K *blocks = (const block_q4_K *)W;
    const block_q8_K *x = (const block_q8_K *)x_q8;
    const int blocks_per_row = K / QK_K;
 
    for (int row = 0; row < M; ++row) {
        const block_q4_K *w_row = blocks + (size_t)row * (size_t)blocks_per_row;
        y[row] = dot_q4_k_q8_k_ref(w_row, x, K);
    }
}

References dot_q4_k_q8_k_ref(), and QK_K.

Referenced by gemm_q4_k_q8_k_ref(), gemv_q4_k_q8_k(), gemv_q4_k_q8_k_amx(), gemv_q4_k_q8_k_avx(), gemv_q4_k_q8_k_avx2(), and gemv_q4_k_q8_k_vnni().

gemv_q4_k_q8_k_sse()

void gemv_q4_k_q8_k_sse	(	float *	y,
		const void *	W,
		const void *	x_q8,
		int	M,
		int	K
	)

Definition at line 33 of file gemm_kernels_q4k_sse.c.

{
    const block_q4_K *blocks = (const block_q4_K *)W;
    const block_q8_K *x = (const block_q8_K *)x_q8;
    const int blocks_per_row = K / QK_K;
 
    const __m128i mask_low = _mm_set1_epi8(0x0F);
 
    for (int row = 0; row < M; ++row) {
        float sumf = 0.0f;
        const block_q4_K *w_row = blocks + row * blocks_per_row;
 
        for (int i = 0; i < blocks_per_row; ++i) {
            const block_q4_K *b4 = &w_row[i];
            const block_q8_K *b8 = &x[i];
 
            // Unpack scales (same as ref)
            uint8_t sc[8], m_val[8];
            unpack_q4_k_scales(b4->scales, sc, m_val);
 
            float d = CK_FP16_TO_FP32(b4->d) * b8->d;
            float dmin = CK_FP16_TO_FP32(b4->dmin) * b8->d;
 
            int is = 0;
            int q_offset = 0;
 
            // Process 4 chunks of 64 elements (256 total)
            for (int j = 0; j < QK_K; j += 64) {
                // We process 32 bytes of qs (covering 64 elements via low/high nibbles)
                // We access qs[0..31] relative to q_offset
 
                // Accumulators for this 64-element chunk
                __m128i acc_lo = _mm_setzero_si128();
                __m128i acc_hi = _mm_setzero_si128();
 
                // Inner loop: 2 iters of 16 bytes (32 elements)
                for (int l = 0; l < 32; l += 16) {
                    // Load 16 bytes of Q4
                    __m128i q4_vec = _mm_loadu_si128((const __m128i *)(b4->qs + q_offset + l));
 
                    // Low nibbles -> correspond to q8_lo (elements j+l .. j+l+15)
                    __m128i q4_lo = _mm_and_si128(q4_vec, mask_low);
                    
                    // High nibbles -> correspond to q8_hi (elements j+32+l .. j+32+l+15)
                    __m128i q4_hi = _mm_and_si128(_mm_srli_epi16(q4_vec, 4), mask_low);
 
                    // Load Q8
                    __m128i q8_lo_vec = _mm_loadu_si128((const __m128i *)(b8->qs + j + l));
                    __m128i q8_hi_vec = _mm_loadu_si128((const __m128i *)(b8->qs + j + 32 + l));
 
                    // Expand and Multiply-Add: Q4(u8) * Q8(s8) -> i32
                    // Since Q4 is u8 and Q8 is s8, we use intermediate i16
                    
                    // LO PART
                    __m128i q4_lo_16_L = _mm_cvtepu8_epi16(q4_lo); // lower 8 -> 16
                    __m128i q8_lo_16_L = _mm_cvtepi8_epi16(q8_lo_vec);
                    __m128i prod_lo_L = _mm_madd_epi16(q4_lo_16_L, q8_lo_16_L); // i32
                    acc_lo = _mm_add_epi32(acc_lo, prod_lo_L);
 
                    __m128i q4_lo_16_H = _mm_cvtepu8_epi16(_mm_srli_si128(q4_lo, 8)); // upper 8 -> 16
                    __m128i q8_lo_16_H = _mm_cvtepi8_epi16(_mm_srli_si128(q8_lo_vec, 8));
                    __m128i prod_lo_H = _mm_madd_epi16(q4_lo_16_H, q8_lo_16_H); // i32
                    acc_lo = _mm_add_epi32(acc_lo, prod_lo_H);
 
                    // HI PART
                    __m128i q4_hi_16_L = _mm_cvtepu8_epi16(q4_hi);
                    __m128i q8_hi_16_L = _mm_cvtepi8_epi16(q8_hi_vec);
                    __m128i prod_hi_L = _mm_madd_epi16(q4_hi_16_L, q8_hi_16_L);
                    acc_hi = _mm_add_epi32(acc_hi, prod_hi_L);
 
                    __m128i q4_hi_16_H = _mm_cvtepu8_epi16(_mm_srli_si128(q4_hi, 8));
                    __m128i q8_hi_16_H = _mm_cvtepi8_epi16(_mm_srli_si128(q8_hi_vec, 8));
                    __m128i prod_hi_H = _mm_madd_epi16(q4_hi_16_H, q8_hi_16_H);
                    acc_hi = _mm_add_epi32(acc_hi, prod_hi_H);
                }
 
                int32_t sum_q4q8_lo = hsum_epi32_sse(acc_lo);
                int32_t sum_q4q8_hi = hsum_epi32_sse(acc_hi);
 
                /* bsums: each bsum is 16 elements */
                int32_t bsum_lo = (int32_t)b8->bsums[j / 16] +
                                  (int32_t)b8->bsums[j / 16 + 1];
                int32_t bsum_hi = (int32_t)b8->bsums[(j + 32) / 16] +
                                  (int32_t)b8->bsums[(j + 32) / 16 + 1];
 
                sumf += d * (float)sc[is] * (float)sum_q4q8_lo;
                sumf -= dmin * (float)m_val[is] * (float)bsum_lo;
                sumf += d * (float)sc[is + 1] * (float)sum_q4q8_hi;
                sumf -= dmin * (float)m_val[is + 1] * (float)bsum_hi;
 
                q_offset += 32;
                is += 2;
            }
        }
        y[row] = sumf;
    }
}

Referenced by gemv_q4_k_q8_k().

gemv_q4_k_q8_k_vnni()

void gemv_q4_k_q8_k_vnni	(	float *	y,
		const void *	W,
		const void *	x_q8,
		int	M,
		int	K
	)

Definition at line 3534 of file gemm_kernels_q4k_q8k_vnni.c.

{
#if defined(__AVX512VNNI__) && defined(__AVX512VL__)
    const char *fast_env = getenv("CK_ENABLE_Q4K_Q8K_VNNI_FAST");
    const int fast_disabled = fast_env && fast_env[0] && fast_env[0] == '0';
    if (!fast_disabled && !ck_strict_parity_enabled()) {
        if (!y || !W || !x_q8 || M <= 0 || K <= 0) {
            return;
        }
 
        const block_q4_K *blocks = (const block_q4_K *)W;
        const block_q8_K *x = (const block_q8_K *)x_q8;
        const int blocks_per_row = K / QK_K;
 
        for (int row = 0; row < M; ++row) {
            const block_q4_K *w_row = blocks + (size_t)row * (size_t)blocks_per_row;
            float sum = 0.0f;
            for (int b = 0; b < blocks_per_row; ++b) {
                sum += dot_q4_k_q8_k_vnni_block(&w_row[b], &x[b]);
            }
            y[row] = sum;
        }
        return;
    }
#endif
 
    /* Strict/debug parity keeps the llama-style scalar accumulation path.
     * Production AVX-512 hosts use VNNI by default; set
     * CK_ENABLE_Q4K_Q8K_VNNI_FAST=0 or CK_DEBUG_Q4K_Q8_REF=1 when attributing
     * borderline logit movement against scalar/reference behavior.
     */
    gemv_q4_k_q8_k_ref(y, W, x_q8, M, K);
}

Referenced by gemv_q4_k_q8_k().

quantize_batch_q8_k_4row_nearest_even()

void quantize_batch_q8_k_4row_nearest_even	(	const float *	x,
		void *	vy,
		int	num_rows,
		int	k
	)

Definition at line 140 of file gemm_kernels_q4k_q8k.c.

                                                                 {
    if (!x || !vy || num_rows <= 0 || k <= 0) {
        return;
    }
    assert(k % QK_K == 0);
 
    block_q8_K *y = (block_q8_K *)vy;
    const int blocks_per_row = k / QK_K;
 
    /* Q8_K bytes are a numerical ABI. The previous four-row AVX2 path used
     * a different max/scale evaluation order and changed real Qwen3-VL
     * visual-prefix scales by one FP32 ULP. Keep this public grouped ABI on
     * the canonical row provider until an optimized implementation is
     * byte-exact against llama.cpp on both synthetic and production inputs. */
    for (int row = 0; row < num_rows; ++row) {
        quantize_row_q8_k(
                x + (size_t)row * (size_t)k,
                y + (size_t)row * (size_t)blocks_per_row,
                k);
    }
}

References QK_K, and quantize_row_q8_k().

quantize_row_q8_k()

void quantize_row_q8_k	(	const float *	x,
		void *	vy,
		int	k
	)

Definition at line 121 of file gemm_kernels_q4k_q8k.c.

                                                        {
    const char *ref_env = getenv("CK_DEBUG_Q8K_REF");
    if (ref_env && atoi(ref_env) != 0) {
        quantize_row_q8_k_ref(x, vy, k);
        return;
    }
#if defined(__AVX512F__) && defined(__AVX512BW__)
    quantize_row_q8_k_avx512(x, vy, k);
#elif defined(__AVX2__)
    quantize_row_q8_k_avx2(x, vy, k);
#elif defined(__AVX__)
    quantize_row_q8_k_avx(x, vy, k);
#elif defined(__SSE4_1__)
    quantize_row_q8_k_sse(x, vy, k);
#else
    quantize_row_q8_k_ref(x, vy, k);
#endif
}

References quantize_row_q8_k_avx(), quantize_row_q8_k_avx2(), quantize_row_q8_k_avx512(), quantize_row_q8_k_ref(), and quantize_row_q8_k_sse().

Referenced by ck_attention_project_head_major_q4_k_q8_k(), ck_layer_forward_rmsnorm_swiglu_decode_q4_k(), ck_mlp_swiglu_forward_q4_k_q8_k(), ck_mlp_swiglu_forward_q4_k_q8_k_prefill(), ck_moe_q4k_mixed_route_parallel(), ck_moe_q4k_mixed_route_work(), ck_moe_q4k_q5k_bucket_work(), ck_moe_q4k_q5k_quantize_work(), ck_moe_q4k_q5k_route_parallel(), ck_moe_q4k_q5k_route_work(), ck_moe_shared_q4k_gated_workspace(), ck_qkv_project_head_major_q4_k_q8_k(), ck_test_gemm_q4_k(), ck_test_gemm_q6_k(), ck_test_gemv_q4_k(), ck_test_gemv_q6_k(), ck_test_quantize_q8_k(), decode_layer_parallel(), fused_mlp_swiglu_prefill_w1w2_quant(), fused_rmsnorm_qkv_prefill_head_major_quant(), gemm_nt_q5_0_sse_v2(), gemm_nt_q5_k_prepared(), gemm_nt_q5_k_prepared_m4(), gemm_nt_q5_k_ref(), gemm_nt_q6_k_sse(), gemv_q4_k(), gemv_q5_k_ref(), hyper_connection_mix_quantized(), mlp_parallel(), model_decode_token(), model_decode_token(), model_forward_prefill_impl(), model_forward_prefill_impl(), model_forward_prefill_impl(), model_forward_prefill_impl(), model_layer_0_decode(), model_layer_0_decode(), model_layer_10_decode(), model_layer_10_decode(), model_layer_11_decode(), model_layer_11_decode(), model_layer_12_decode(), model_layer_12_decode(), model_layer_13_decode(), model_layer_13_decode(), model_layer_14_decode(), model_layer_14_decode(), model_layer_15_decode(), model_layer_15_decode(), model_layer_16_decode(), model_layer_16_decode(), model_layer_17_decode(), model_layer_17_decode(), model_layer_18_decode(), model_layer_18_decode(), model_layer_19_decode(), model_layer_19_decode(), model_layer_1_decode(), model_layer_1_decode(), model_layer_20_decode(), model_layer_20_decode(), model_layer_21_decode(), model_layer_21_decode(), model_layer_22_decode(), model_layer_22_decode(), model_layer_23_decode(), model_layer_23_decode(), model_layer_2_decode(), model_layer_2_decode(), model_layer_3_decode(), model_layer_3_decode(), model_layer_4_decode(), model_layer_4_decode(), model_layer_5_decode(), model_layer_5_decode(), model_layer_6_decode(), model_layer_6_decode(), model_layer_7_decode(), model_layer_7_decode(), model_layer_8_decode(), model_layer_8_decode(), model_layer_9_decode(), model_layer_9_decode(), moe_swiglu_expert_forward_q4k_q4k_workspace(), moe_swiglu_expert_forward_q4k_q5_0_workspace(), moe_swiglu_expert_forward_q4k_q5k_workspace(), moe_swiglu_expert_forward_q4k_q6k_workspace(), moe_swiglu_expert_forward_q4k_q8_0_workspace(), moe_swiglu_shared_forward_q4k_q4k_workspace(), moe_swiglu_shared_forward_q4k_q6k_workspace(), quantize_batch_q8_k(), quantize_batch_q8_k_4row_nearest_even(), qwen2_0_5b_decode_decode_token(), qwen2_0_5b_decode_decode_token(), qwen2_0_5b_decode_forward_prefill_impl(), qwen2_0_5b_decode_forward_prefill_impl(), qwen2_0_5b_decode_forward_prefill_impl(), qwen2_0_5b_decode_forward_prefill_impl(), qwen2_0_5b_decode_layer_0_decode(), qwen2_0_5b_decode_layer_0_decode(), qwen2_0_5b_decode_layer_10_decode(), qwen2_0_5b_decode_layer_10_decode(), qwen2_0_5b_decode_layer_11_decode(), qwen2_0_5b_decode_layer_11_decode(), qwen2_0_5b_decode_layer_12_decode(), qwen2_0_5b_decode_layer_12_decode(), qwen2_0_5b_decode_layer_13_decode(), qwen2_0_5b_decode_layer_13_decode(), qwen2_0_5b_decode_layer_14_decode(), qwen2_0_5b_decode_layer_14_decode(), qwen2_0_5b_decode_layer_15_decode(), qwen2_0_5b_decode_layer_15_decode(), qwen2_0_5b_decode_layer_16_decode(), qwen2_0_5b_decode_layer_16_decode(), qwen2_0_5b_decode_layer_17_decode(), qwen2_0_5b_decode_layer_17_decode(), qwen2_0_5b_decode_layer_18_decode(), qwen2_0_5b_decode_layer_18_decode(), qwen2_0_5b_decode_layer_19_decode(), qwen2_0_5b_decode_layer_19_decode(), qwen2_0_5b_decode_layer_1_decode(), qwen2_0_5b_decode_layer_1_decode(), qwen2_0_5b_decode_layer_20_decode(), qwen2_0_5b_decode_layer_20_decode(), qwen2_0_5b_decode_layer_21_decode(), qwen2_0_5b_decode_layer_21_decode(), qwen2_0_5b_decode_layer_22_decode(), qwen2_0_5b_decode_layer_22_decode(), qwen2_0_5b_decode_layer_23_decode(), qwen2_0_5b_decode_layer_23_decode(), qwen2_0_5b_decode_layer_2_decode(), qwen2_0_5b_decode_layer_2_decode(), qwen2_0_5b_decode_layer_3_decode(), qwen2_0_5b_decode_layer_3_decode(), qwen2_0_5b_decode_layer_4_decode(), qwen2_0_5b_decode_layer_4_decode(), qwen2_0_5b_decode_layer_5_decode(), qwen2_0_5b_decode_layer_5_decode(), qwen2_0_5b_decode_layer_6_decode(), qwen2_0_5b_decode_layer_6_decode(), qwen2_0_5b_decode_layer_7_decode(), qwen2_0_5b_decode_layer_7_decode(), qwen2_0_5b_decode_layer_8_decode(), qwen2_0_5b_decode_layer_8_decode(), qwen2_0_5b_decode_layer_9_decode(), qwen2_0_5b_decode_layer_9_decode(), swiglu_forward_q8_k(), and unfused_rmsnorm_linear_q4k_ref().

quantize_row_q8_k_avx()

void quantize_row_q8_k_avx	(	const float *	x,
		void *	vy,
		int	k
	)

Definition at line 20 of file quantize_row_q8_k_avx.c.

                                                            {
    /* Reuse the parity-clean SIMD implementation until a wider AVX variant is
     * worth maintaining separately. */
#if defined(__SSE4_1__)
    quantize_row_q8_k_sse(x, vy, k);
#else
    quantize_row_q8_k_ref(x, vy, k);
#endif
}

References quantize_row_q8_k_ref(), and quantize_row_q8_k_sse().

Referenced by quantize_row_q8_k().

quantize_row_q8_k_avx2()

void quantize_row_q8_k_avx2	(	const float *	x,
		void *	vy,
		int	k
	)

Definition at line 14 of file quantize_row_q8_k_avx2.c.

                                                             {
    quantize_row_q8_k_ref(x, vy, k);
}

References quantize_row_q8_k_ref().

Referenced by quantize_row_q8_k().

quantize_row_q8_k_avx512()

void quantize_row_q8_k_avx512	(	const float *	x,
		void *	vy,
		int	k
	)

Definition at line 8 of file quantize_row_q8_k_avx512.c.

                                                               {
    /*
     * Q8_K bytes are part of the numerical ABI consumed by Q4_K/Q6_K dots.
     * Keep AVX-512 on the reference contract until a vector implementation is
     * byte-exact for every block, including multi-block activation rows.
     */
    quantize_row_q8_k_ref(x, vy, k);
}

References quantize_row_q8_k_ref().

Referenced by quantize_row_q8_k().

quantize_row_q8_k_ref()

void quantize_row_q8_k_ref	(	const float *	x,
		void *	vy,
		int	k
	)

Definition at line 61 of file gemm_kernels_q4k_q8k.c.

                                                            {
    if (!x || !vy || k <= 0) {
        return;
    }
    assert(k % QK_K == 0);
    const int nb = k / QK_K;
    block_q8_K *y = (block_q8_K *)vy;
 
    for (int i = 0; i < nb; ++i) {
        float max = 0.0f;
        float amax = 0.0f;
        for (int j = 0; j < QK_K; ++j) {
            float ax = fabsf(x[j]);
            if (ax > amax) {
                amax = ax;
                max = x[j];
            }
        }
        if (!amax) {
            y[i].d = 0.0f;
            memset(y[i].qs, 0, sizeof(y[i].qs));
            memset(y[i].bsums, 0, sizeof(y[i].bsums));
            x += QK_K;
            continue;
        }
 
        const float iscale = -127.0f / max;
        for (int j = 0; j < QK_K; ++j) {
            /* llama.cpp rounds the multiply before adding nearest_int's magic
             * constant. Contracting both operations changes Q8_K tie cases. */
            float scaled = iscale * x[j];
            int v = ck_nearest_int(scaled);
            if (v > 127) {
                v = 127;
            }
            if (v < -128) {
                v = -128;
            }
            y[i].qs[j] = (int8_t)v;
        }
 
        for (int j = 0; j < QK_K / 16; ++j) {
            int sum = 0;
            const int8_t *qs = &y[i].qs[j * 16];
            for (int ii = 0; ii < 16; ++ii) {
                sum += qs[ii];
            }
            y[i].bsums[j] = (int16_t)sum;
        }
 
        y[i].d = 1.0f / iscale;
        x += QK_K;
    }
}

References block_q8_K::bsums, ck_nearest_int(), block_q8_K::d, QK_K, and block_q8_K::qs.

Referenced by quantize_row_q8_k(), quantize_row_q8_k_avx(), quantize_row_q8_k_avx2(), and quantize_row_q8_k_avx512().

quantize_row_q8_k_sse()

void quantize_row_q8_k_sse	(	const float *	x,
		void *	vy,
		int	k
	)

Definition at line 22 of file quantize_row_q8_k_sse.c.

                                                            {
    if (!x || !vy || k <= 0) {
        return;
    }
    assert(k % QK_K == 0);
 
    const int nb = k / QK_K;
    block_q8_K *y = (block_q8_K *)vy;
 
    for (int i = 0; i < nb; ++i) {
        /* Keep the exact signed-max selection contract from llama.cpp/ref. */
        float max = 0.0f;
        float amax = 0.0f;
        for (int j = 0; j < QK_K; ++j) {
            const float xv = x[j];
            const float ax = fabsf(xv);
            if (ax > amax) {
                amax = ax;
                max = xv;
            }
        }
 
        if (amax == 0.0f) {
            y[i].d = 0.0f;
            memset(y[i].qs, 0, sizeof(y[i].qs));
            memset(y[i].bsums, 0, sizeof(y[i].bsums));
            x += QK_K;
            continue;
        }
 
        const float iscale = -127.0f / max;
        const __m128 v_iscale = _mm_set1_ps(iscale);
        const __m128 v_magic = _mm_set1_ps(12582912.0f);
        const __m128i v_mantissa = _mm_set1_epi32(0x007fffff);
        const __m128i v_bias = _mm_set1_epi32(0x00400000);
        const __m128i v_min = _mm_set1_epi32(-128);
        const __m128i v_max = _mm_set1_epi32(127);
 
        for (int j = 0; j < QK_K; j += 16) {
            const __m128 x0 = _mm_loadu_ps(x + j + 0);
            const __m128 x1 = _mm_loadu_ps(x + j + 4);
            const __m128 x2 = _mm_loadu_ps(x + j + 8);
            const __m128 x3 = _mm_loadu_ps(x + j + 12);
 
            __m128i q0 = _mm_sub_epi32(
                _mm_and_si128(_mm_castps_si128(_mm_add_ps(_mm_mul_ps(x0, v_iscale), v_magic)), v_mantissa),
                v_bias);
            __m128i q1 = _mm_sub_epi32(
                _mm_and_si128(_mm_castps_si128(_mm_add_ps(_mm_mul_ps(x1, v_iscale), v_magic)), v_mantissa),
                v_bias);
            __m128i q2 = _mm_sub_epi32(
                _mm_and_si128(_mm_castps_si128(_mm_add_ps(_mm_mul_ps(x2, v_iscale), v_magic)), v_mantissa),
                v_bias);
            __m128i q3 = _mm_sub_epi32(
                _mm_and_si128(_mm_castps_si128(_mm_add_ps(_mm_mul_ps(x3, v_iscale), v_magic)), v_mantissa),
                v_bias);
 
            q0 = _mm_min_epi32(_mm_max_epi32(q0, v_min), v_max);
            q1 = _mm_min_epi32(_mm_max_epi32(q1, v_min), v_max);
            q2 = _mm_min_epi32(_mm_max_epi32(q2, v_min), v_max);
            q3 = _mm_min_epi32(_mm_max_epi32(q3, v_min), v_max);
 
            const __m128i q01 = _mm_packs_epi32(q0, q1);
            const __m128i q23 = _mm_packs_epi32(q2, q3);
            const __m128i q0123 = _mm_packs_epi16(q01, q23);
 
            _mm_storeu_si128((__m128i *)(y[i].qs + j), q0123);
 
            int sum = 0;
            for (int ii = 0; ii < 16; ++ii) {
                sum += y[i].qs[j + ii];
            }
            y[i].bsums[j / 16] = (int16_t)sum;
        }
 
        y[i].d = 1.0f / iscale;
        x += QK_K;
    }
}

Referenced by quantize_row_q8_k().