gemm_kernels_q6k_q8k.c File Reference

Q6_K (weights) x Q8_K (activations) kernels for inference. More...

#include <assert.h>
#include <math.h>
#include <string.h>
#include <stdint.h>
#include <stddef.h>
#include <stdlib.h>
#include "ckernel_engine.h"
#include "ckernel_quant.h"

Go to the source code of this file.

Functions
static float	ck_dot_q6_k_q8_k_fast_or_ref (const block_q6_K *w, const block_q8_K *x, int K)
void	ck_q6_k_prepare_weight (const void *src, void *dst, int N, int K)
size_t	ck_q6_k_prepared_block_size (void)
const char *	ck_q6_k_prepared_provider_name (void)
const char *	ck_q6_k_q8_k_provider_name (void)
static int	ck_q6k_q8k_force_ref (void)
static float	dot_q6_k_q8_k_ref (const block_q6_K *w, const block_q8_K *x, int K)
	Scalar dot product for Q6_K x Q8_K.
void	gemm_nt_q6_k_q8_k (const void *A_q8, const void *B, const float *bias, float *C, int M, int N, int K)
void	gemm_nt_q6_k_q8_k_m2 (const void *A_q8, const void *B, const float *bias, float *C, int M, int N, int K)
void	gemm_nt_q6_k_q8_k_m4_tile (const void *A_q8, const void *B, const float *bias, float *C, int M, int N, int K, int m0, int m1, int n0, int n1)
void	gemm_nt_q6_k_q8_k_prepared (const void *A_q8, const void *B_prepared, const float *bias, float *C, int M, int N, int K)
void	gemm_nt_q6_k_q8_k_prepared_avx512_vnni (const void *A_q8, const void *B_prepared, const float *bias, float *C, int M, int N, int K)
void	gemm_nt_q6_k_q8_k_prepared_tile (const void *A_q8, const void *B_prepared, const float *bias, float *C, int M, int N, int K, int m0, int m1, int n0, int n1)
static void	gemm_nt_q6_k_q8_k_prepared_tile_impl (const void *A_q8, const void *B_prepared, const float *bias, float *C, int M, int N, int K, int m0, int m1, int n0, int n1, int use_avx512_vnni)
void	gemm_nt_q6_k_q8_k_tile (const void *A_q8, const void *B, const float *bias, float *C, int M, int N, int K, int m0, int m1, int n0, int n1)
	Compute one C[m0:m1, n0:n1] tile for Q8_K activations x Q6_K weights.
void	gemm_nt_q6_k_q8_k_tiled (const void *A_q8, const void *B, const float *bias, float *C, int M, int N, int K)
	Experimental single-thread tiled NT GEMM wrapper.
void	gemm_nt_q6_k_q8_k_tiled_parity (const void *A_q8, const void *B, const float *bias, float *C, int M, int N, int K)
	NT GEMM: C = A @ B^T where A is Q8_K and B is Q6_K.
void	gemm_nt_q6_k_q8_k_tiled_parity_tile (const void *A_q8, const void *B, const float *bias, float *C, int M, int N, int K, int m0, int m1, int n0, int n1)
void	gemm_q6_k_q8_k (float *Y, const void *W, const void *X_q8, int M, int N, int K)
	GEMM: Y = W @ X^T where W is Q6_K and X is Q8_K.
void	gemv_q6_k_q8_k (float *y, const void *W, const void *x_q8, int M, int K)
	GEMV: y = W @ x where W is Q6_K and x is Q8_K.
void	gemv_q6_k_q8_k_avx (float *y, const void *W, const void *x_q8, int M, int K)
void	gemv_q6_k_q8_k_avx2 (float *y, const void *W, const void *x_q8, int M, int K)
void	gemv_q6_k_q8_k_avx512 (float *y, const void *W, const void *x_q8, int M, int K)
void	gemv_q6_k_q8_k_avx512_vbmi (float *y, const void *W, const void *x_q8, int M, int K)
void	gemv_q6_k_q8_k_parallel (float *y, const void *W, const void *x_q8, int M, int K, int ith, int nth)
	Parallel reference GEMV for Q6_K × Q8_K.
void	gemv_q6_k_q8_k_parallel_simd (float *y, const void *W, const void *x_q8, int M, int K, int ith, int nth)
	Parallel SIMD GEMV for Q6_K × Q8_K.
void	gemv_q6_k_q8_k_ref (float *y, const void *W, const void *x_q8, int M, int K)
void	gemv_q6_k_q8_k_sse (float *y, const void *W, const void *x_q8, int M, int K)
void	vec_dot_q6_k_q8_k (int n, float *s, const void *vx, const void *vy)
	Q6_K x Q8_K dot product (single row)

Detailed Description

Q6_K (weights) x Q8_K (activations) kernels for inference.

CK-ENGINE KERNEL RULES:

1. NO malloc/free - memory via bump allocator, pointers passed in
2. NO OpenMP - parallelization at orchestrator/codegen layer
3. API must define: inputs, outputs, workspace, and memory layouts
4. Pure computation - deterministic, no side effects

After changes: make test && make llamacpp-parity-full

Implements decode-style matvec/matmul where weights are Q6_K and the activations are quantized on-the-fly to Q8_K. This is inference-only; no backward pass is provided here.

Q6_K Format (256 weights per block):

• d: FP16 super-block scale
• ql: 128 bytes (low 4 bits of each weight)
• qh: 64 bytes (high 2 bits of each weight)
• scales: 16 int8 sub-block scales

Q8_K Format (256 weights per block):

• d: FP32 scale
• qs: 256 int8 values
• bsums: 16 int16 block sums

Definition in file gemm_kernels_q6k_q8k.c.

Function Documentation

ck_dot_q6_k_q8_k_fast_or_ref()

static float ck_dot_q6_k_q8_k_fast_or_ref ( const block_q6_K *  w, const block_q8_K *  x, int  K  )

inlinestatic

Definition at line 1729 of file gemm_kernels_q6k_q8k.c.

{
    if (ck_q6k_q8k_force_ref()) {
        return dot_q6_k_q8_k_ref(w, x, K);
    }
#if defined(__AVX2__)
    /* Short strict prefill uses the pinned x86 graph's AVX2 leaf reduction.
     * Batched strict prefill uses the separate tiled integer-slab kernel. */
    return dot_q6_k_q8_k_avx2(w, x, K);
#elif defined(__AVX__)
    if (ck_strict_parity_enabled()) return dot_q6_k_q8_k_ref(w, x, K);
    return dot_q6_k_q8_k_avx(w, x, K);
#elif defined(__SSE4_1__)
    if (ck_strict_parity_enabled()) return dot_q6_k_q8_k_ref(w, x, K);
    return dot_q6_k_q8_k_sse(w, x, K);
#elif defined(__ARM_NEON) || defined(__aarch64__)
    if (ck_strict_parity_enabled()) return dot_q6_k_q8_k_ref(w, x, K);
    return dot_q6_k_q8_k_neon(w, x, K);
#else
    return dot_q6_k_q8_k_ref(w, x, K);
#endif
}

References ck_q6k_q8k_force_ref(), ck_strict_parity_enabled(), and dot_q6_k_q8_k_ref().

Referenced by gemm_nt_q6_k_q8_k(), and gemm_nt_q6_k_q8_k_tile().

ck_q6_k_prepare_weight()

void ck_q6_k_prepare_weight	(	const void *	src,
		void *	dst,
		int	N,
		int	K
	)

Definition at line 74 of file gemm_kernels_q6k_q8k.c.

{
    if (!src || !dst || N <= 0 || K <= 0 || (K % QK_K) != 0) return;
    const block_q6_K *input = (const block_q6_K *)src;
    block_q6_K_prepared *output = (block_q6_K_prepared *)dst;
    const size_t blocks = (size_t)N * (size_t)(K / QK_K);
 
    for (size_t b = 0; b < blocks; ++b) {
        output[b].d = input[b].d;
        memcpy(output[b].scales, input[b].scales, sizeof(output[b].scales));
        for (int n = 0; n < QK_K; n += 128) {
            const uint8_t *ql = input[b].ql + n / 2;
            const uint8_t *qh = input[b].qh + n / 4;
            for (int l = 0; l < 32; ++l) {
                output[b].qs[n + l + 0] =
                    (uint8_t)((ql[l] & 0x0f) | (((qh[l] >> 0) & 3) << 4));
                output[b].qs[n + l + 32] =
                    (uint8_t)((ql[l + 32] & 0x0f) | (((qh[l] >> 2) & 3) << 4));
                output[b].qs[n + l + 64] =
                    (uint8_t)((ql[l] >> 4) | (((qh[l] >> 4) & 3) << 4));
                output[b].qs[n + l + 96] =
                    (uint8_t)((ql[l + 32] >> 4) | (((qh[l] >> 6) & 3) << 4));
            }
        }
    }
}

References block_q6_K::d, block_q6_K::qh, QK_K, and block_q6_K::ql.

ck_q6_k_prepared_block_size()

size_t ck_q6_k_prepared_block_size

(

void

)

Definition at line 57 of file gemm_kernels_q6k_q8k.c.

{
    return sizeof(block_q6_K_prepared);
}

ck_q6_k_prepared_provider_name()

const char * ck_q6_k_prepared_provider_name

(

void

)

Definition at line 62 of file gemm_kernels_q6k_q8k.c.

{
#if defined(__AVX512F__) && defined(__AVX512BW__) && \
    defined(__AVX512VNNI__)
    return "q6_k_prepared_avx512_vnni_exact";
#elif defined(__AVX2__)
    return "q6_k_prepared_avx2_exact";
#else
    return "q6_k_prepared_unavailable";
#endif
}

ck_q6_k_q8_k_provider_name()

const char * ck_q6_k_q8_k_provider_name

(

void

)

Definition at line 1389 of file gemm_kernels_q6k_q8k.c.

{
    if (ck_strict_parity_enabled() || ck_q6k_q8k_force_ref()) {
        return "q6_k_q8_k_ref";
    }
#if defined(__AVX2__)
    return "q6_k_q8_k_avx2";
#elif defined(__AVX__)
    return "q6_k_q8_k_avx";
#elif defined(__SSE4_1__)
    return "q6_k_q8_k_sse";
#elif defined(__ARM_NEON) || defined(__aarch64__)
    return "q6_k_q8_k_neon";
#else
    return "q6_k_q8_k_ref";
#endif
}

References ck_q6k_q8k_force_ref(), and ck_strict_parity_enabled().

ck_q6k_q8k_force_ref()

static int ck_q6k_q8k_force_ref ( void  )

static

Definition at line 108 of file gemm_kernels_q6k_q8k.c.

{
    static int cached = -1;
    if (cached < 0) {
        const char *env = getenv("CK_DEBUG_Q6K_Q8K_REF");
        cached = (env && env[0] && env[0] != '0') ? 1 : 0;
    }
    return cached;
}

Referenced by ck_dot_q6_k_q8_k_fast_or_ref(), ck_q6_k_q8_k_provider_name(), gemm_nt_q6_k_q8_k_m4_tile(), gemv_q6_k_q8_k(), and gemv_q6_k_q8_k_parallel_simd().

dot_q6_k_q8_k_ref()

static float dot_q6_k_q8_k_ref ( const block_q6_K *  w, const block_q8_K *  x, int  K  )

static

Scalar dot product for Q6_K x Q8_K.

Q6_K layout: 256 weights per block

• ql[0..127]: low 4 bits for all 256 weights (packed 2 per byte)
• qh[0..63]: high 2 bits for all 256 weights (packed 4 per byte)
• scales[0..15]: int8 scale for each 16-weight sub-block
• d: FP16 super-block scale

The dequantization formula for each weight is: weight = d * scale[sub] * (q6_value - 32) where q6_value is the 6-bit unsigned value (0..63).

Definition at line 135 of file gemm_kernels_q6k_q8k.c.

{
    const int nb = K / QK_K;
    float sumf = 0.0f;
    float sums[8] = {0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f};
 
    for (int i = 0; i < nb; ++i) {
        const float d = GGML_FP16_TO_FP32(w[i].d) * x[i].d;
 
        const uint8_t *ql = w[i].ql;
        const uint8_t *qh = w[i].qh;
        const int8_t *sc = w[i].scales;
        const int8_t *q8 = x[i].qs;
        int32_t aux32[8] = {0, 0, 0, 0, 0, 0, 0, 0};
 
        /* Process 256 weights in 2 iterations of 128 */
        for (int n = 0; n < QK_K; n += 128) {
            /* Each iteration processes 128 weights:
             * - ql[0..63] contains low 4 bits
             * - qh[0..31] contains high 2 bits
             * - Interleaved pattern: weights 0-31, 32-63, 64-95, 96-127
             */
            for (int l = 0; l < 32; ++l) {
                /* Sub-block index: each scale covers 16 weights */
                const int is = l / 16;
 
                /* Extract 6-bit values from packed format */
                /* q1: weights l+0 (low nibble of ql[l], bits 0-1 of qh[l]) */
                const int8_t q1 = (int8_t)((ql[l + 0] & 0xF) | (((qh[l] >> 0) & 3) << 4)) - 32;
                /* q2: weights l+32 (low nibble of ql[l+32], bits 2-3 of qh[l]) */
                const int8_t q2 = (int8_t)((ql[l + 32] & 0xF) | (((qh[l] >> 2) & 3) << 4)) - 32;
                /* q3: weights l+64 (high nibble of ql[l], bits 4-5 of qh[l]) */
                const int8_t q3 = (int8_t)((ql[l + 0] >> 4) | (((qh[l] >> 4) & 3) << 4)) - 32;
                /* q4: weights l+96 (high nibble of ql[l+32], bits 6-7 of qh[l]) */
                const int8_t q4 = (int8_t)((ql[l + 32] >> 4) | (((qh[l] >> 6) & 3) << 4)) - 32;
 
                aux32[l & 7] += (int)sc[is + 0] * (int)q1 * (int)q8[l + 0];
                aux32[l & 7] += (int)sc[is + 2] * (int)q2 * (int)q8[l + 32];
                aux32[l & 7] += (int)sc[is + 4] * (int)q3 * (int)q8[l + 64];
                aux32[l & 7] += (int)sc[is + 6] * (int)q4 * (int)q8[l + 96];
            }
            q8 += 128;
            ql += 64;
            qh += 32;
            sc += 8;
        }
 
        for (int l = 0; l < 8; ++l) {
            sums[l] += d * (float)aux32[l];
        }
    }
 
    for (int l = 0; l < 8; ++l) {
        sumf += sums[l];
    }
    return sumf;
}

References block_q8_K::d, GGML_FP16_TO_FP32, block_q6_K::qh, QK_K, block_q6_K::ql, block_q8_K::qs, and block_q6_K::scales.

Referenced by ck_dot_q6_k_q8_k_fast_or_ref(), gemv_q6_k_q8_k_parallel(), gemv_q6_k_q8_k_parallel_simd(), gemv_q6_k_q8_k_ref(), and vec_dot_q6_k_q8_k().

gemm_nt_q6_k_q8_k()

void gemm_nt_q6_k_q8_k	(	const void *	A_q8,
		const void *	B,
		const float *	bias,
		float *	C,
		int	M,
		int	N,
		int	K
	)

Definition at line 1548 of file gemm_kernels_q6k_q8k.c.

{
    if (!A_q8 || !B || !C) {
        return;
    }
    if (M <= 0 || N <= 0 || K <= 0) {
        return;
    }
#if defined(__AVX2__) && defined(__FMA__)
    if (M >= 8 && ck_strict_parity_enabled()) {
        gemm_nt_q6_k_q8_k_tiled_parity(A_q8, B, bias, C, M, N, K);
        return;
    }
#endif
 
    /* Prefill GEMM uses the pinned x86 leaf reduction for short row groups
     * and the tiled reduction for strict batches of at least eight rows.
     * CK_DEBUG_Q6K_Q8K_REF=1 still forces the scalar reference leaf. */
    const block_q8_K *A = (const block_q8_K *)A_q8;
    const block_q6_K *W = (const block_q6_K *)B;
    const int blocks_per_vec = K / QK_K;
    const int blocks_per_row = K / QK_K;
 
    for (int m = 0; m < M; ++m) {
        const block_q8_K *a_row = A + (size_t)m * (size_t)blocks_per_vec;
        float *c_row = C + (size_t)m * (size_t)N;
        for (int n = 0; n < N; ++n) {
            const block_q6_K *w_row = W + (size_t)n * (size_t)blocks_per_row;
            const float b = bias ? bias[n] : 0.0f;
            c_row[n] = ck_dot_q6_k_q8_k_fast_or_ref(w_row, a_row, K) + b;
        }
    }
}

References C, ck_dot_q6_k_q8_k_fast_or_ref(), ck_strict_parity_enabled(), gemm_nt_q6_k_q8_k_tiled_parity(), and QK_K.

Referenced by ck_test_gemm_q6_k(), gemm_nt_q8_k_mlp_dispatch(), and gemm_nt_q8_k_qkv_dispatch().

gemm_nt_q6_k_q8_k_m2()

void gemm_nt_q6_k_q8_k_m2	(	const void *	A_q8,
		const void *	B,
		const float *	bias,
		float *	C,
		int	M,
		int	N,
		int	K
	)

Definition at line 1854 of file gemm_kernels_q6k_q8k.c.

{
    if (!A_q8 || !B || !C || M != 2 || N <= 0 || K <= 0 ||
        (K % QK_K) != 0) return;
    gemm_nt_q6_k_q8_k_m4_tile(A_q8, B, bias, C, M, N, K,
                                0, M, 0, N);
}

References C, gemm_nt_q6_k_q8_k_m4_tile(), and QK_K.

gemm_nt_q6_k_q8_k_m4_tile()

void gemm_nt_q6_k_q8_k_m4_tile	(	const void *	A_q8,
		const void *	B,
		const float *	bias,
		float *	C,
		int	M,
		int	N,
		int	K,
		int	m0,
		int	m1,
		int	n0,
		int	n1
	)

Definition at line 1797 of file gemm_kernels_q6k_q8k.c.

{
    if (!A_q8 || !B || !C || M <= 0 || N <= 0 || K <= 0 ||
        K % QK_K != 0) {
        return;
    }
    if (m0 < 0) m0 = 0;
    if (n0 < 0) n0 = 0;
    if (m1 > M) m1 = M;
    if (n1 > N) n1 = N;
    if (m0 >= m1 || n0 >= n1) return;
 
#if defined(__AVX2__)
    if (!ck_strict_parity_enabled() && !ck_q6k_q8k_force_ref()) {
        const block_q8_K *A = (const block_q8_K *)A_q8;
        const block_q6_K *W = (const block_q6_K *)B;
        const int blocks_per_vec = K / QK_K;
        for (int n = n0; n < n1; ++n) {
            const block_q6_K *w_row =
                W + (size_t)n * (size_t)blocks_per_vec;
            const float b = bias ? bias[n] : 0.0f;
            int m = m0;
            for (; m + 4 <= m1; m += 4) {
                float values[4];
                dot_q6_k_q8_k_avx2_m4(
                    w_row, A + (size_t)m * (size_t)blocks_per_vec,
                    blocks_per_vec, 4, K, values);
                for (int r = 0; r < 4; ++r) {
                    C[(size_t)(m + r) * (size_t)N + (size_t)n] = values[r] + b;
                }
            }
            if (m < m1) {
                float values[4];
                const int rows = m1 - m;
                dot_q6_k_q8_k_avx2_m4(
                    w_row, A + (size_t)m * (size_t)blocks_per_vec,
                    blocks_per_vec, rows, K, values);
                for (int r = 0; r < rows; ++r) {
                    C[(size_t)(m + r) * (size_t)N + (size_t)n] = values[r] + b;
                }
            }
        }
        return;
    }
#endif
    gemm_nt_q6_k_q8_k_tile(
        A_q8, B, bias, C, M, N, K, m0, m1, n0, n1);
}

References C, ck_q6k_q8k_force_ref(), ck_strict_parity_enabled(), gemm_nt_q6_k_q8_k_tile(), and QK_K.

Referenced by gemm_nt_q6_k_q8_k_m2().

gemm_nt_q6_k_q8_k_prepared()

void gemm_nt_q6_k_q8_k_prepared	(	const void *	A_q8,
		const void *	B_prepared,
		const float *	bias,
		float *	C,
		int	M,
		int	N,
		int	K
	)

Definition at line 1719 of file gemm_kernels_q6k_q8k.c.

{
    gemm_nt_q6_k_q8_k_prepared_tile(
        A_q8, B_prepared, bias, C, M, N, K, 0, M, 0, N);
}

References C, and gemm_nt_q6_k_q8_k_prepared_tile().

gemm_nt_q6_k_q8_k_prepared_avx512_vnni()

void gemm_nt_q6_k_q8_k_prepared_avx512_vnni	(	const void *	A_q8,
		const void *	B_prepared,
		const float *	bias,
		float *	C,
		int	M,
		int	N,
		int	K
	)

Definition at line 1707 of file gemm_kernels_q6k_q8k.c.

{
    gemm_nt_q6_k_q8_k_prepared_tile_impl(
        A_q8, B_prepared, bias, C, M, N, K,
        0, M, 0, N, 1);
}

References C, and gemm_nt_q6_k_q8_k_prepared_tile_impl().

gemm_nt_q6_k_q8_k_prepared_tile()

void gemm_nt_q6_k_q8_k_prepared_tile	(	const void *	A_q8,
		const void *	B_prepared,
		const float *	bias,
		float *	C,
		int	M,
		int	N,
		int	K,
		int	m0,
		int	m1,
		int	n0,
		int	n1
	)

Definition at line 1689 of file gemm_kernels_q6k_q8k.c.

{
    int use_avx512_vnni = 0;
#if defined(__AVX512F__) && defined(__AVX512BW__) && \
    defined(__AVX512VNNI__)
    use_avx512_vnni = 1;
#endif
    gemm_nt_q6_k_q8_k_prepared_tile_impl(
        A_q8, B_prepared, bias, C, M, N, K,
        m0, m1, n0, n1, use_avx512_vnni);
}

References C, and gemm_nt_q6_k_q8_k_prepared_tile_impl().

Referenced by gemm_nt_q6_k_q8_k_prepared().

gemm_nt_q6_k_q8_k_prepared_tile_impl()

static void gemm_nt_q6_k_q8_k_prepared_tile_impl ( const void *  A_q8, const void *  B_prepared, const float *  bias, float *  C, int  M, int  N, int  K, int  m0, int  m1, int  n0, int  n1, int  use_avx512_vnni  )

static

Definition at line 1636 of file gemm_kernels_q6k_q8k.c.

{
#if !defined(__AVX2__)
    (void)A_q8; (void)B_prepared; (void)bias; (void)C;
    (void)M; (void)N; (void)K; (void)m0; (void)m1; (void)n0; (void)n1;
    (void)use_avx512_vnni;
#else
#if !defined(__AVX512F__) || !defined(__AVX512BW__) || \
    !defined(__AVX512VNNI__)
    (void)use_avx512_vnni;
#endif
    if (!A_q8 || !B_prepared || !C || M <= 0 || N <= 0 || K <= 0 ||
        (K % QK_K) != 0) return;
    if (m0 < 0) m0 = 0;
    if (n0 < 0) n0 = 0;
    if (m1 > M) m1 = M;
    if (n1 > N) n1 = N;
    if (m0 >= m1 || n0 >= n1) return;
 
    const block_q8_K *A = (const block_q8_K *)A_q8;
    const block_q6_K_prepared *W =
        (const block_q6_K_prepared *)B_prepared;
    const int blocks_per_row = K / QK_K;
    for (int n = n0; n < n1; ++n) {
        const block_q6_K_prepared *w_row =
            W + (size_t)n * (size_t)blocks_per_row;
        const float b = bias ? bias[n] : 0.0f;
        for (int m = m0; m < m1; ++m) {
            const block_q8_K *a_row =
                A + (size_t)m * (size_t)blocks_per_row;
#if defined(__AVX512F__) && defined(__AVX512BW__) && \
    defined(__AVX512VNNI__)
            const float dot = use_avx512_vnni
                ? dot_q6_k_prepared_q8_k_avx512_vnni(w_row, a_row, K)
                : dot_q6_k_prepared_q8_k_avx2(w_row, a_row, K);
#else
            const float dot = dot_q6_k_prepared_q8_k_avx2(w_row, a_row, K);
#endif
            C[(size_t)m * (size_t)N + (size_t)n] =
                dot + b;
        }
    }
#endif
}

References C, and QK_K.

Referenced by gemm_nt_q6_k_q8_k_prepared_avx512_vnni(), and gemm_nt_q6_k_q8_k_prepared_tile().

gemm_nt_q6_k_q8_k_tile()

void gemm_nt_q6_k_q8_k_tile	(	const void *	A_q8,
		const void *	B,
		const float *	bias,
		float *	C,
		int	M,
		int	N,
		int	K,
		int	m0,
		int	m1,
		int	n0,
		int	n1
	)

Compute one C[m0:m1, n0:n1] tile for Q8_K activations x Q6_K weights.

Pure tile math only: no threadpool, no global scheduling, no allocation. The orchestrator decides how to split tile jobs across cores.

Definition at line 1760 of file gemm_kernels_q6k_q8k.c.

{
    if (!A_q8 || !B || !C) {
        return;
    }
    if (M <= 0 || N <= 0 || K <= 0 || K % QK_K != 0) {
        return;
    }
    if (m0 < 0) m0 = 0;
    if (n0 < 0) n0 = 0;
    if (m1 > M) m1 = M;
    if (n1 > N) n1 = N;
    if (m0 >= m1 || n0 >= n1) {
        return;
    }
 
    const block_q8_K *A = (const block_q8_K *)A_q8;
    const block_q6_K *W = (const block_q6_K *)B;
    const int blocks_per_vec = K / QK_K;
    const int blocks_per_row = K / QK_K;
 
    for (int n = n0; n < n1; ++n) {
        const block_q6_K *w_row = W + (size_t)n * (size_t)blocks_per_row;
        const float b = bias ? bias[n] : 0.0f;
        for (int m = m0; m < m1; ++m) {
            const block_q8_K *a_row = A + (size_t)m * (size_t)blocks_per_vec;
            C[(size_t)m * (size_t)N + (size_t)n] = ck_dot_q6_k_q8_k_fast_or_ref(w_row, a_row, K) + b;
        }
    }
}

References C, ck_dot_q6_k_q8_k_fast_or_ref(), and QK_K.

Referenced by gemm_nt_q6_k_q8_k_m4_tile(), and gemm_nt_q6_k_q8_k_tiled().

gemm_nt_q6_k_q8_k_tiled()

void gemm_nt_q6_k_q8_k_tiled	(	const void *	A_q8,
		const void *	B,
		const float *	bias,
		float *	C,
		int	M,
		int	N,
		int	K
	)

Experimental single-thread tiled NT GEMM wrapper.

Kept as a separate symbol from gemm_nt_q6_k_q8_k for benchmarks and parity. Production prefill should prefer the v8 2D tile scheduler when enabled.

Definition at line 1870 of file gemm_kernels_q6k_q8k.c.

{
    enum { TILE_M = 8, TILE_N = 16 };
    for (int n0 = 0; n0 < N; n0 += TILE_N) {
        const int n1 = (n0 + TILE_N < N) ? (n0 + TILE_N) : N;
        for (int m0 = 0; m0 < M; m0 += TILE_M) {
            const int m1 = (m0 + TILE_M < M) ? (m0 + TILE_M) : M;
            gemm_nt_q6_k_q8_k_tile(A_q8, B, bias, C, M, N, K, m0, m1, n0, n1);
        }
    }
}

References C, and gemm_nt_q6_k_q8_k_tile().

gemm_nt_q6_k_q8_k_tiled_parity()

void gemm_nt_q6_k_q8_k_tiled_parity	(	const void *	A_q8,
		const void *	B,
		const float *	bias,
		float *	C,
		int	M,
		int	N,
		int	K
	)

NT GEMM: C = A @ B^T where A is Q8_K and B is Q6_K.

This is the typical inference pattern:

• A: Activations in Q8_K format [M x K]
• B: Weights in Q6_K format [N x K]
• C: Output [M x N]

Parameters

A_q8	Input activations in Q8_K format
B	Weight matrix in Q6_K format
bias	Optional bias vector [N]
C	Output matrix
M	Batch size (number of tokens)
N	Output dimension
K	Input dimension

Definition at line 1626 of file gemm_kernels_q6k_q8k.c.

{
    gemm_nt_q6_k_q8_k_tiled_parity_tile(A_q8, B, bias, C,
                                        M, N, K, 0, M, 0, N);
}

References C, and gemm_nt_q6_k_q8_k_tiled_parity_tile().

Referenced by gemm_nt_q6_k_q8_k().

gemm_nt_q6_k_q8_k_tiled_parity_tile()

void gemm_nt_q6_k_q8_k_tiled_parity_tile	(	const void *	A_q8,
		const void *	B,
		const float *	bias,
		float *	C,
		int	M,
		int	N,
		int	K,
		int	m0,
		int	m1,
		int	n0,
		int	n1
	)

Definition at line 1588 of file gemm_kernels_q6k_q8k.c.

{
    if (!A_q8 || !B || !C || M <= 0 || N <= 0 || K <= 0 || K % QK_K) return;
    if (m0 < 0 || n0 < 0 || m1 > M || n1 > N || m0 >= m1 || n0 >= n1) return;
    const block_q8_K *A = (const block_q8_K *) A_q8;
    const block_q6_K *W = (const block_q6_K *) B;
    const int nb = K / QK_K;
    for (int m = m0; m < m1; ++m) {
        for (int n = n0; n < n1; ++n) {
            float acc = 0.0f;
            for (int b = 0; b < nb; ++b) {
                const block_q6_K *w = W + (size_t)n * (size_t)nb + b;
                const block_q8_K *a = A + (size_t)m * (size_t)nb + b;
                int32_t dot = 0;
                for (int e = 0; e < QK_K; ++e) {
                    const int group = (e / 128) * 128;
                    const int pos = e % 128;
                    const int lane = pos % 32;
                    const int ql_off = group / 2 + lane + ((pos / 32) & 1) * 32;
                    const int qh_off = group / 4 + lane;
                    const int ql_shift = (pos >= 64) ? 4 : 0;
                    const int qh_shift = (pos / 32) * 2;
                    const int code = ((w->ql[ql_off] >> ql_shift) & 15) |
                                     (((w->qh[qh_off] >> qh_shift) & 3) << 4);
                    dot += (code - 32) * (int)w->scales[e / 16] * (int)a->qs[e];
                }
                acc = fmaf(GGML_FP16_TO_FP32(w->d) * (float)dot, a->d, acc);
            }
            C[(size_t)m * (size_t)N + n] = acc + (bias ? bias[n] : 0.0f);
        }
    }
}

References C, block_q6_K::d, block_q8_K::d, GGML_FP16_TO_FP32, block_q6_K::qh, QK_K, block_q6_K::ql, block_q8_K::qs, and block_q6_K::scales.

Referenced by gemm_nt_q6_k_q8_k_tiled_parity().

gemm_q6_k_q8_k()

void gemm_q6_k_q8_k	(	float *	Y,
		const void *	W,
		const void *	X_q8,
		int	M,
		int	N,
		int	K
	)

GEMM: Y = W @ X^T where W is Q6_K and X is Q8_K.

Parameters

Y	Output matrix [N x M] in row-major
W	Weight matrix in Q6_K format [M x K]
X_q8	Input matrix in Q8_K format [N x K]
M	Number of output rows (output dim)
N	Number of input vectors (batch size)
K	Input dimension

Definition at line 1508 of file gemm_kernels_q6k_q8k.c.

{
    if (!Y || !W || !X_q8 || M <= 0 || N <= 0 || K <= 0) {
        return;
    }
 
    const block_q8_K *X = (const block_q8_K *)X_q8;
    const int blocks_per_vec = K / QK_K;
 
    for (int n = 0; n < N; ++n) {
        const block_q8_K *x_row = X + (size_t)n * (size_t)blocks_per_vec;
        gemv_q6_k_q8_k(&Y[n * M], W, x_row, M, K);
    }
}

References gemv_q6_k_q8_k(), and QK_K.

gemv_q6_k_q8_k()

void gemv_q6_k_q8_k	(	float *	y,
		const void *	W,
		const void *	x_q8,
		int	M,
		int	K
	)

GEMV: y = W @ x where W is Q6_K and x is Q8_K.

Definition at line 1356 of file gemm_kernels_q6k_q8k.c.

{
    if (ck_strict_parity_enabled() || ck_q6k_q8k_force_ref()) {
        gemv_q6_k_q8_k_ref(y, W, x_q8, M, K);
        return;
    }
 
#if defined(__AVX2__)
    /* llama.cpp's x86 Q6_K production graph keeps the AVX2 reduction order
     * even when AVX-512 is available. Wider ISA availability is not a license
     * to change this numerical contract. */
    gemv_q6_k_q8_k_avx2(y, W, x_q8, M, K);
    return;
#elif defined(__AVX__)
    gemv_q6_k_q8_k_avx(y, W, x_q8, M, K);
    return;
#elif defined(__SSE4_1__)
    gemv_q6_k_q8_k_sse(y, W, x_q8, M, K);
    return;
#elif defined(__ARM_NEON) || defined(__aarch64__)
    /* NEON keeps the same integer-exact per-block dot as the scalar
     * reference but accumulates the FP32 block contributions sequentially
     * instead of through eight partial lanes, so strict parity mode (handled
     * above) still uses the reference reduction order. */
    gemv_q6_k_q8_k_neon(y, W, x_q8, M, K);
    return;
#endif
    gemv_q6_k_q8_k_ref(y, W, x_q8, M, K);
}

References ck_q6k_q8k_force_ref(), ck_strict_parity_enabled(), gemv_q6_k_q8_k_avx(), gemv_q6_k_q8_k_avx2(), gemv_q6_k_q8_k_ref(), and gemv_q6_k_q8_k_sse().

Referenced by ck_moe_q4k_mixed_route_work(), ck_test_gemv_q6_k(), ck_test_vec_dot_q6_k_q8_k(), gemm_q6_k_q8_k(), moe_swiglu_expert_forward_q4k_q6k_workspace(), and moe_swiglu_shared_forward_q4k_q6k_workspace().

gemv_q6_k_q8_k_avx()

void gemv_q6_k_q8_k_avx	(	float *	y,
		const void *	W,
		const void *	x_q8,
		int	M,
		int	K
	)

Referenced by gemv_q6_k_q8_k().

gemv_q6_k_q8_k_avx2()

void gemv_q6_k_q8_k_avx2	(	float *	y,
		const void *	W,
		const void *	x_q8,
		int	M,
		int	K
	)

Referenced by gemv_q6_k_q8_k().

gemv_q6_k_q8_k_avx512()

void gemv_q6_k_q8_k_avx512	(	float *	y,
		const void *	W,
		const void *	x_q8,
		int	M,
		int	K
	)

gemv_q6_k_q8_k_avx512_vbmi()

void gemv_q6_k_q8_k_avx512_vbmi	(	float *	y,
		const void *	W,
		const void *	x_q8,
		int	M,
		int	K
	)

gemv_q6_k_q8_k_parallel()

void gemv_q6_k_q8_k_parallel	(	float *	y,
		const void *	W,
		const void *	x_q8,
		int	M,
		int	K,
		int	ith,
		int	nth
	)

Parallel reference GEMV for Q6_K × Q8_K.

Caller provides ith (thread index) and nth (total threads). Each thread processes rows [r0, r1).

Definition at line 1422 of file gemm_kernels_q6k_q8k.c.

{
    if (!y || !W || !x_q8 || M <= 0 || K <= 0) return;
    if (ith < 0 || nth <= 0 || ith >= nth) return;
 
    /* Compute row range for this thread */
    const int dr = (M + nth - 1) / nth;
    const int r0 = dr * ith;
    const int r1 = (r0 + dr < M) ? (r0 + dr) : M;
 
    if (r0 >= M) return;
 
    const block_q6_K *blocks = (const block_q6_K *)W;
    const block_q8_K *x = (const block_q8_K *)x_q8;
    const int blocks_per_row = K / QK_K;
 
    for (int row = r0; row < r1; ++row) {
        const block_q6_K *w_row = blocks + (size_t)row * (size_t)blocks_per_row;
        y[row] = dot_q6_k_q8_k_ref(w_row, x, K);
    }
}

References dot_q6_k_q8_k_ref(), and QK_K.

gemv_q6_k_q8_k_parallel_simd()

void gemv_q6_k_q8_k_parallel_simd	(	float *	y,
		const void *	W,
		const void *	x_q8,
		int	M,
		int	K,
		int	ith,
		int	nth
	)

Parallel SIMD GEMV for Q6_K × Q8_K.

Uses best available SIMD (AVX/SSE) with row prefetching. Caller provides ith/nth from OpenMP region.

Definition at line 1454 of file gemm_kernels_q6k_q8k.c.

{
    if (!y || !W || !x_q8 || M <= 0 || K <= 0) return;
    if (ith < 0 || nth <= 0 || ith >= nth) return;
 
    const int dr = (M + nth - 1) / nth;
    const int r0 = dr * ith;
    const int r1 = (r0 + dr < M) ? (r0 + dr) : M;
 
    if (r0 >= M) return;
 
    const block_q6_K *blocks = (const block_q6_K *)W;
    const block_q8_K *x = (const block_q8_K *)x_q8;
    const int blocks_per_row = K / QK_K;
    const int strict = ck_strict_parity_enabled() || ck_q6k_q8k_force_ref();
 
    for (int row = r0; row < r1; ++row) {
        const block_q6_K *w_row = blocks + (size_t)row * (size_t)blocks_per_row;
#if defined(__AVX2__)
        y[row] = strict ? dot_q6_k_q8_k_ref(w_row, x, K)
                        : dot_q6_k_q8_k_avx2(w_row, x, K);
#elif defined(__AVX__)
        y[row] = strict ? dot_q6_k_q8_k_ref(w_row, x, K)
                        : dot_q6_k_q8_k_avx(w_row, x, K);
#elif defined(__SSE4_1__)
        y[row] = strict ? dot_q6_k_q8_k_ref(w_row, x, K)
                        : dot_q6_k_q8_k_sse(w_row, x, K);
#elif defined(__ARM_NEON) || defined(__aarch64__)
        y[row] = strict ? dot_q6_k_q8_k_ref(w_row, x, K)
                        : dot_q6_k_q8_k_neon(w_row, x, K);
#else
        y[row] = dot_q6_k_q8_k_ref(w_row, x, K);
#endif
    }
}

References ck_q6k_q8k_force_ref(), ck_strict_parity_enabled(), dot_q6_k_q8_k_ref(), and QK_K.

gemv_q6_k_q8_k_ref()

void gemv_q6_k_q8_k_ref	(	float *	y,
		const void *	W,
		const void *	x_q8,
		int	M,
		int	K
	)

Definition at line 285 of file gemm_kernels_q6k_q8k.c.

{
    if (!y || !W || !x_q8 || M <= 0 || K <= 0) {
        return;
    }
 
    const block_q6_K *blocks = (const block_q6_K *)W;
    const block_q8_K *x = (const block_q8_K *)x_q8;
    const int blocks_per_row = K / QK_K;
 
    for (int row = 0; row < M; ++row) {
        const block_q6_K *w_row = blocks + (size_t)row * (size_t)blocks_per_row;
        y[row] = dot_q6_k_q8_k_ref(w_row, x, K);
    }
}

References dot_q6_k_q8_k_ref(), and QK_K.

Referenced by gemv_q6_k_q8_k().

gemv_q6_k_q8_k_sse()

void gemv_q6_k_q8_k_sse	(	float *	y,
		const void *	W,
		const void *	x_q8,
		int	M,
		int	K
	)

Referenced by gemv_q6_k_q8_k().

vec_dot_q6_k_q8_k()

void vec_dot_q6_k_q8_k	(	int	n,
		float *	s,
		const void *	vx,
		const void *	vy
	)

Q6_K x Q8_K dot product (single row)

Definition at line 1339 of file gemm_kernels_q6k_q8k.c.

{
    if (!s || !vx || !vy || n <= 0) {
        return;
    }
 
    const block_q6_K *x = (const block_q6_K *)vx;
    const block_q8_K *y = (const block_q8_K *)vy;
 
    /* This is the architecture-neutral scalar oracle. Production x86 dispatch
     * uses the separately parity-tested SIMD reduction tree. */
    *s = dot_q6_k_q8_k_ref(x, y, n);
}

References dot_q6_k_q8_k_ref().