gemm_kernels_q4k_q8k.c File Reference

Q4_K (weights) x Q8_K (activations) kernels for inference. More...

#include <assert.h>
#include <math.h>
#include <string.h>
#include "ckernel_quant.h"

Go to the source code of this file.

Functions
static int	ck_nearest_int (float fval)
static float	dot_q4_k_q8_k_ref (const block_q4_K *w, const block_q8_K *x, int k)
void	gemm_nt_q4_k_q8_k (const void *A_q8, const void *B, const float *bias, float *C, int M, int N, int K)
void	gemm_q4_k_q8_k (float *Y, const void *W, const void *X_q8, int M, int N, int K)
void	gemm_q4_k_q8_k_ref (float *Y, const void *W, const void *X_q8, int M, int N, int K)
void	gemv_q4_k_q8_k (float *y, const void *W, const void *x_q8, int M, int K)
void	gemv_q4_k_q8_k_avx (float *y, const void *W, const void *x_q8, int M, int K)
void	gemv_q4_k_q8_k_avx2 (float *y, const void *W, const void *x_q8, int M, int K)
void	gemv_q4_k_q8_k_parallel (float *y, const void *W, const void *x_q8, int M, int K, int ith, int nth)
void	gemv_q4_k_q8_k_ref (float *y, const void *W, const void *x_q8, int M, int K)
void	gemv_q4_k_q8_k_sse (float *y, const void *W, const void *x_q8, int M, int K)
void	gemv_q4_k_q8_k_vnni (float *y, const void *W, const void *x_q8, int M, int K)
void	quantize_row_q8_k (const float *x, void *vy, int k)
void	quantize_row_q8_k_ref (const float *x, void *vy, int k)
void	quantize_row_q8_k_sse (const float *x, void *vy, int k)

Detailed Description

Q4_K (weights) x Q8_K (activations) kernels for inference.

CK-ENGINE KERNEL RULES:

1. NO malloc/free - memory via bump allocator, pointers passed in
2. NO OpenMP - parallelization at orchestrator/codegen layer
3. API must define: inputs, outputs, workspace, and memory layouts
4. Pure computation - deterministic, no side effects

After changes: make test && make llamacpp-parity-full

Implements decode-style matvec/matmul where weights are Q4_K and the activations are quantized on-the-fly to Q8_K. This is inference-only; no backward pass is provided here.

Definition in file gemm_kernels_q4k_q8k.c.

Function Documentation

ck_nearest_int()

static int ck_nearest_int ( float  fval )

inlinestatic

Definition at line 45 of file gemm_kernels_q4k_q8k.c.

                                             {
    /* Bit-level round-to-nearest from llama.cpp (fast + deterministic). */
    float val = fval + 12582912.f;
    int i;
    memcpy(&i, &val, sizeof(int));
    return (i & 0x007fffff) - 0x00400000;
}

Referenced by quantize_row_q8_k_ref().

dot_q4_k_q8_k_ref()

static float dot_q4_k_q8_k_ref ( const block_q4_K *  w, const block_q8_K *  x, int  k  )

static

Definition at line 115 of file gemm_kernels_q4k_q8k.c.

{
    const int nb = k / QK_K;
    float sumf = 0.0f;
 
    for (int i = 0; i < nb; ++i) {
        uint8_t sc[8], m_val[8];
        unpack_q4_k_scales(w[i].scales, sc, m_val);
 
        const float d = CK_FP16_TO_FP32(w[i].d) * x[i].d;
        const float dmin = CK_FP16_TO_FP32(w[i].dmin) * x[i].d;
 
        /* Q4_K layout: process 64 elements at a time
         * - Low nibbles of qs[0..31] → elements 0..31 → uses sc[0], m[0]
         * - High nibbles of qs[0..31] → elements 32..63 → uses sc[1], m[1]
         * - Low nibbles of qs[32..63] → elements 64..95 → uses sc[2], m[2]
         * - etc.
         */
        int is = 0;
        int q_offset = 0;
 
        for (int j = 0; j < QK_K; j += 64) {
            const uint8_t *qs = &w[i].qs[q_offset];
            const int8_t *q8_lo = &x[i].qs[j];       /* Elements j to j+31 */
            const int8_t *q8_hi = &x[i].qs[j + 32];  /* Elements j+32 to j+63 */
 
            /* Sum for low nibbles (elements j to j+31) */
            int32_t sum_q4q8_lo = 0;
            for (int l = 0; l < 32; ++l) {
                int q4_val = qs[l] & 0x0F;
                sum_q4q8_lo += q4_val * q8_lo[l];
            }
 
            /* Sum for high nibbles (elements j+32 to j+63) */
            int32_t sum_q4q8_hi = 0;
            for (int l = 0; l < 32; ++l) {
                int q4_val = qs[l] >> 4;
                sum_q4q8_hi += q4_val * q8_hi[l];
            }
 
            /* bsums: each bsum is 16 elements */
            int32_t bsum_lo = (int32_t)x[i].bsums[j / 16] +
                              (int32_t)x[i].bsums[j / 16 + 1];
            int32_t bsum_hi = (int32_t)x[i].bsums[(j + 32) / 16] +
                              (int32_t)x[i].bsums[(j + 32) / 16 + 1];
 
            /* Accumulate: d * sc * sum(q4*q8) - dmin * m * sum(q8) */
            sumf += d * (float)sc[is] * (float)sum_q4q8_lo;
            sumf -= dmin * (float)m_val[is] * (float)bsum_lo;
            sumf += d * (float)sc[is + 1] * (float)sum_q4q8_hi;
            sumf -= dmin * (float)m_val[is + 1] * (float)bsum_hi;
 
            q_offset += 32;
            is += 2;
        }
    }
 
    return sumf;
}

References block_q8_K::bsums, CK_FP16_TO_FP32, block_q8_K::d, QK_K, block_q4_K::qs, block_q8_K::qs, and unpack_q4_k_scales().

Referenced by gemv_q4_k_q8_k_parallel(), and gemv_q4_k_q8_k_ref().

gemm_nt_q4_k_q8_k()

void gemm_nt_q4_k_q8_k	(	const void *	A_q8,
		const void *	B,
		const float *	bias,
		float *	C,
		int	M,
		int	N,
		int	K
	)

Definition at line 295 of file gemm_kernels_q4k_q8k.c.

{
    if (!A_q8 || !B || !C) {
        return;
    }
    if (M <= 0 || N <= 0 || K <= 0) {
        return;
    }
 
    gemm_q4_k_q8_k(C, B, A_q8, /*M_out=*/N, /*N_batch=*/M, K);
 
    if (!bias) {
        return;
    }
 
    for (int i = 0; i < M; ++i) {
        float *row = C + (size_t)i * (size_t)N;
        for (int j = 0; j < N; ++j) {
            row[j] += bias[j];
        }
    }
}

References C, and gemm_q4_k_q8_k().

Referenced by ck_attention_project_head_major_q4_k_q8_k(), ck_layer_forward_rmsnorm_swiglu_decode_q4_k(), ck_mlp_swiglu_forward_q4_k_q8_k(), ck_mlp_swiglu_forward_q4_k_q8_k_prefill(), ck_qkv_project_head_major_token_q4_k_q8_k(), ck_test_gemm_q4_k(), gemm_nt_q8_k_mlp_dispatch(), gemm_nt_q8_k_qkv_dispatch(), model_forward_prefill_impl(), and qwen2_0_5b_decode_forward_prefill_impl().

gemm_q4_k_q8_k()

void gemm_q4_k_q8_k	(	float *	Y,
		const void *	W,
		const void *	X_q8,
		int	M,
		int	N,
		int	K
	)

Definition at line 277 of file gemm_kernels_q4k_q8k.c.

{
    if (!Y || !W || !X_q8 || M <= 0 || N <= 0 || K <= 0) {
        return;
    }
 
    const block_q8_K *X = (const block_q8_K *)X_q8;
    const int blocks_per_vec = K / QK_K;
 
    for (int n = 0; n < N; ++n) {
        const block_q8_K *x_row = X + (size_t)n * (size_t)blocks_per_vec;
        gemv_q4_k_q8_k(&Y[n * M], W, x_row, M, K);
    }
}

References gemv_q4_k_q8_k(), and QK_K.

Referenced by gemm_nt_q4_k_q8_k().

gemm_q4_k_q8_k_ref()

void gemm_q4_k_q8_k_ref	(	float *	Y,
		const void *	W,
		const void *	X_q8,
		int	M,
		int	N,
		int	K
	)

Definition at line 259 of file gemm_kernels_q4k_q8k.c.

{
    if (!Y || !W || !X_q8 || M <= 0 || N <= 0 || K <= 0) {
        return;
    }
 
    const block_q8_K *X = (const block_q8_K *)X_q8;
    const int blocks_per_vec = K / QK_K;
 
    for (int n = 0; n < N; ++n) {
        const block_q8_K *x_row = X + (size_t)n * (size_t)blocks_per_vec;
        gemv_q4_k_q8_k_ref(&Y[n * M], W, x_row, M, K);
    }
}

References gemv_q4_k_q8_k_ref(), and QK_K.

gemv_q4_k_q8_k()

void gemv_q4_k_q8_k	(	float *	y,
		const void *	W,
		const void *	x_q8,
		int	M,
		int	K
	)

Definition at line 239 of file gemm_kernels_q4k_q8k.c.

{
#if defined(__AVX512VNNI__) && defined(__AVX512VL__)
    /* VNNI: Best for decode (single token) - INT8 dot product acceleration */
    gemv_q4_k_q8_k_vnni(y, W, x_q8, M, K);
#elif defined(__AVX2__)
    gemv_q4_k_q8_k_avx2(y, W, x_q8, M, K);
#elif defined(__AVX__)
    /* AVX version uses maddubs_epi16 (more efficient than SSE) */
    gemv_q4_k_q8_k_avx(y, W, x_q8, M, K);
#elif defined(__SSE4_1__)
    gemv_q4_k_q8_k_sse(y, W, x_q8, M, K);
#else
    gemv_q4_k_q8_k_ref(y, W, x_q8, M, K);
#endif
}

References gemv_q4_k_q8_k_avx(), gemv_q4_k_q8_k_avx2(), gemv_q4_k_q8_k_ref(), gemv_q4_k_q8_k_sse(), and gemv_q4_k_q8_k_vnni().

Referenced by ck_test_gemv_q4_k(), fused_rmsnorm_linear_q4k(), gemm_q4_k_q8_k(), model_decode_token(), model_layer_0_decode(), model_layer_10_decode(), model_layer_11_decode(), model_layer_12_decode(), model_layer_13_decode(), model_layer_14_decode(), model_layer_15_decode(), model_layer_16_decode(), model_layer_17_decode(), model_layer_18_decode(), model_layer_19_decode(), model_layer_1_decode(), model_layer_20_decode(), model_layer_21_decode(), model_layer_22_decode(), model_layer_23_decode(), model_layer_2_decode(), model_layer_3_decode(), model_layer_4_decode(), model_layer_5_decode(), model_layer_6_decode(), model_layer_7_decode(), model_layer_8_decode(), model_layer_9_decode(), qwen2_0_5b_decode_decode_token(), qwen2_0_5b_decode_layer_0_decode(), qwen2_0_5b_decode_layer_10_decode(), qwen2_0_5b_decode_layer_11_decode(), qwen2_0_5b_decode_layer_12_decode(), qwen2_0_5b_decode_layer_13_decode(), qwen2_0_5b_decode_layer_14_decode(), qwen2_0_5b_decode_layer_15_decode(), qwen2_0_5b_decode_layer_16_decode(), qwen2_0_5b_decode_layer_17_decode(), qwen2_0_5b_decode_layer_18_decode(), qwen2_0_5b_decode_layer_19_decode(), qwen2_0_5b_decode_layer_1_decode(), qwen2_0_5b_decode_layer_20_decode(), qwen2_0_5b_decode_layer_21_decode(), qwen2_0_5b_decode_layer_22_decode(), qwen2_0_5b_decode_layer_23_decode(), qwen2_0_5b_decode_layer_2_decode(), qwen2_0_5b_decode_layer_3_decode(), qwen2_0_5b_decode_layer_4_decode(), qwen2_0_5b_decode_layer_5_decode(), qwen2_0_5b_decode_layer_6_decode(), qwen2_0_5b_decode_layer_7_decode(), qwen2_0_5b_decode_layer_8_decode(), qwen2_0_5b_decode_layer_9_decode(), and unfused_rmsnorm_linear_q4k_ref().

gemv_q4_k_q8_k_avx()

void gemv_q4_k_q8_k_avx	(	float *	y,
		const void *	W,
		const void *	x_q8,
		int	M,
		int	K
	)

Definition at line 251 of file gemm_kernels_q4k_avx.c.

{
    gemv_q4_k_q8_k_ref(y, W, x_q8, M, K);
}

Referenced by gemv_q4_k_q8_k().

gemv_q4_k_q8_k_avx2()

void gemv_q4_k_q8_k_avx2	(	float *	y,
		const void *	W,
		const void *	x_q8,
		int	M,
		int	K
	)

Definition at line 89 of file gemm_kernels_q4k_q8k_avx2.c.

{
    /* TODO: Implement AVX2 version with correct Q4_K memory layout.
     * For now, fall back to reference implementation which has been
     * fixed to use the correct layout.
     */
    gemv_q4_k_q8_k_ref(y, W, x_q8, M, K);
}

Referenced by gemv_q4_k_q8_k().

gemv_q4_k_q8_k_parallel()

void gemv_q4_k_q8_k_parallel	(	float *	y,
		const void *	W,
		const void *	x_q8,
		int	M,
		int	K,
		int	ith,
		int	nth
	)

Definition at line 206 of file gemm_kernels_q4k_q8k.c.

{
    if (!y || !W || !x_q8 || M <= 0 || K <= 0) {
        return;
    }
    if (ith < 0 || nth <= 0 || ith >= nth) {
        return;
    }
 
    /* Compute row range for this thread */
    const int dr = (M + nth - 1) / nth;
    const int r0 = dr * ith;
    const int r1 = (r0 + dr < M) ? (r0 + dr) : M;
 
    if (r0 >= M) {
        return;  /* This thread has no work */
    }
 
    const block_q4_K *blocks = (const block_q4_K *)W;
    const block_q8_K *x = (const block_q8_K *)x_q8;
    const int blocks_per_row = K / QK_K;
 
    /* Only process rows [r0, r1) */
    for (int row = r0; row < r1; ++row) {
        const block_q4_K *w_row = blocks + (size_t)row * (size_t)blocks_per_row;
        y[row] = dot_q4_k_q8_k_ref(w_row, x, K);
    }
}

References dot_q4_k_q8_k_ref(), and QK_K.

Referenced by gemv_q4_k_q8_k_parallel_simd().

gemv_q4_k_q8_k_ref()

void gemv_q4_k_q8_k_ref	(	float *	y,
		const void *	W,
		const void *	x_q8,
		int	M,
		int	K
	)

Definition at line 177 of file gemm_kernels_q4k_q8k.c.

{
    if (!y || !W || !x_q8 || M <= 0 || K <= 0) {
        return;
    }
 
    const block_q4_K *blocks = (const block_q4_K *)W;
    const block_q8_K *x = (const block_q8_K *)x_q8;
    const int blocks_per_row = K / QK_K;
 
    for (int row = 0; row < M; ++row) {
        const block_q4_K *w_row = blocks + (size_t)row * (size_t)blocks_per_row;
        y[row] = dot_q4_k_q8_k_ref(w_row, x, K);
    }
}

References dot_q4_k_q8_k_ref(), and QK_K.

Referenced by gemm_q4_k_q8_k_ref(), gemv_q4_k_q8_k(), gemv_q4_k_q8_k_amx(), gemv_q4_k_q8_k_avx(), gemv_q4_k_q8_k_avx2(), and gemv_q4_k_q8_k_vnni().

gemv_q4_k_q8_k_sse()

void gemv_q4_k_q8_k_sse	(	float *	y,
		const void *	W,
		const void *	x_q8,
		int	M,
		int	K
	)

Definition at line 33 of file gemm_kernels_q4k_sse.c.

{
    const block_q4_K *blocks = (const block_q4_K *)W;
    const block_q8_K *x = (const block_q8_K *)x_q8;
    const int blocks_per_row = K / QK_K;
 
    const __m128i mask_low = _mm_set1_epi8(0x0F);
 
    for (int row = 0; row < M; ++row) {
        float sumf = 0.0f;
        const block_q4_K *w_row = blocks + row * blocks_per_row;
 
        for (int i = 0; i < blocks_per_row; ++i) {
            const block_q4_K *b4 = &w_row[i];
            const block_q8_K *b8 = &x[i];
 
            // Unpack scales (same as ref)
            uint8_t sc[8], m_val[8];
            unpack_q4_k_scales(b4->scales, sc, m_val);
 
            float d = CK_FP16_TO_FP32(b4->d) * b8->d;
            float dmin = CK_FP16_TO_FP32(b4->dmin) * b8->d;
 
            int is = 0;
            int q_offset = 0;
 
            // Process 4 chunks of 64 elements (256 total)
            for (int j = 0; j < QK_K; j += 64) {
                // We process 32 bytes of qs (covering 64 elements via low/high nibbles)
                // We access qs[0..31] relative to q_offset
 
                // Accumulators for this 64-element chunk
                __m128i acc_lo = _mm_setzero_si128();
                __m128i acc_hi = _mm_setzero_si128();
 
                // Inner loop: 2 iters of 16 bytes (32 elements)
                for (int l = 0; l < 32; l += 16) {
                    // Load 16 bytes of Q4
                    __m128i q4_vec = _mm_loadu_si128((const __m128i *)(b4->qs + q_offset + l));
 
                    // Low nibbles -> correspond to q8_lo (elements j+l .. j+l+15)
                    __m128i q4_lo = _mm_and_si128(q4_vec, mask_low);
                    
                    // High nibbles -> correspond to q8_hi (elements j+32+l .. j+32+l+15)
                    __m128i q4_hi = _mm_and_si128(_mm_srli_epi16(q4_vec, 4), mask_low);
 
                    // Load Q8
                    __m128i q8_lo_vec = _mm_loadu_si128((const __m128i *)(b8->qs + j + l));
                    __m128i q8_hi_vec = _mm_loadu_si128((const __m128i *)(b8->qs + j + 32 + l));
 
                    // Expand and Multiply-Add: Q4(u8) * Q8(s8) -> i32
                    // Since Q4 is u8 and Q8 is s8, we use intermediate i16
                    
                    // LO PART
                    __m128i q4_lo_16_L = _mm_cvtepu8_epi16(q4_lo); // lower 8 -> 16
                    __m128i q8_lo_16_L = _mm_cvtepi8_epi16(q8_lo_vec);
                    __m128i prod_lo_L = _mm_madd_epi16(q4_lo_16_L, q8_lo_16_L); // i32
                    acc_lo = _mm_add_epi32(acc_lo, prod_lo_L);
 
                    __m128i q4_lo_16_H = _mm_cvtepu8_epi16(_mm_srli_si128(q4_lo, 8)); // upper 8 -> 16
                    __m128i q8_lo_16_H = _mm_cvtepi8_epi16(_mm_srli_si128(q8_lo_vec, 8));
                    __m128i prod_lo_H = _mm_madd_epi16(q4_lo_16_H, q8_lo_16_H); // i32
                    acc_lo = _mm_add_epi32(acc_lo, prod_lo_H);
 
                    // HI PART
                    __m128i q4_hi_16_L = _mm_cvtepu8_epi16(q4_hi);
                    __m128i q8_hi_16_L = _mm_cvtepi8_epi16(q8_hi_vec);
                    __m128i prod_hi_L = _mm_madd_epi16(q4_hi_16_L, q8_hi_16_L);
                    acc_hi = _mm_add_epi32(acc_hi, prod_hi_L);
 
                    __m128i q4_hi_16_H = _mm_cvtepu8_epi16(_mm_srli_si128(q4_hi, 8));
                    __m128i q8_hi_16_H = _mm_cvtepi8_epi16(_mm_srli_si128(q8_hi_vec, 8));
                    __m128i prod_hi_H = _mm_madd_epi16(q4_hi_16_H, q8_hi_16_H);
                    acc_hi = _mm_add_epi32(acc_hi, prod_hi_H);
                }
 
                int32_t sum_q4q8_lo = hsum_epi32_sse(acc_lo);
                int32_t sum_q4q8_hi = hsum_epi32_sse(acc_hi);
 
                /* bsums: each bsum is 16 elements */
                int32_t bsum_lo = (int32_t)b8->bsums[j / 16] +
                                  (int32_t)b8->bsums[j / 16 + 1];
                int32_t bsum_hi = (int32_t)b8->bsums[(j + 32) / 16] +
                                  (int32_t)b8->bsums[(j + 32) / 16 + 1];
 
                sumf += d * (float)sc[is] * (float)sum_q4q8_lo;
                sumf -= dmin * (float)m_val[is] * (float)bsum_lo;
                sumf += d * (float)sc[is + 1] * (float)sum_q4q8_hi;
                sumf -= dmin * (float)m_val[is + 1] * (float)bsum_hi;
 
                q_offset += 32;
                is += 2;
            }
        }
        y[row] = sumf;
    }
}

Referenced by gemv_q4_k_q8_k().

gemv_q4_k_q8_k_vnni()

void gemv_q4_k_q8_k_vnni	(	float *	y,
		const void *	W,
		const void *	x_q8,
		int	M,
		int	K
	)

Definition at line 95 of file gemm_kernels_q4k_q8k_vnni.c.

{
    /* TODO: Implement VNNI version with correct Q4_K memory layout.
     * For now, fall back to reference implementation which has been
     * fixed to use the correct layout.
     */
    gemv_q4_k_q8_k_ref(y, W, x_q8, M, K);
}

Referenced by gemv_q4_k_q8_k().

quantize_row_q8_k()

void quantize_row_q8_k	(	const float *	x,
		void *	vy,
		int	k
	)

Definition at line 107 of file gemm_kernels_q4k_q8k.c.

                                                        {
#if defined(__SSE4_1__)
    quantize_row_q8_k_sse(x, vy, k);
#else
    quantize_row_q8_k_ref(x, vy, k);
#endif
}

References quantize_row_q8_k_ref(), and quantize_row_q8_k_sse().

Referenced by ck_attention_project_head_major_q4_k_q8_k(), ck_layer_forward_rmsnorm_swiglu_decode_q4_k(), ck_mlp_swiglu_forward_q4_k_q8_k(), ck_mlp_swiglu_forward_q4_k_q8_k_prefill(), ck_qkv_project_head_major_q4_k_q8_k(), ck_test_gemm_q4_k(), ck_test_gemm_q6_k(), ck_test_gemv_q4_k(), ck_test_quantize_q8_k(), decode_layer_parallel(), fused_mlp_swiglu_prefill_w1w2_quant(), fused_rmsnorm_qkv_prefill_head_major_quant(), gemm_nt_q5_0_sse_v2(), gemm_nt_q6_k_sse(), mlp_parallel(), model_decode_token(), model_forward_prefill_impl(), model_layer_0_decode(), model_layer_10_decode(), model_layer_11_decode(), model_layer_12_decode(), model_layer_13_decode(), model_layer_14_decode(), model_layer_15_decode(), model_layer_16_decode(), model_layer_17_decode(), model_layer_18_decode(), model_layer_19_decode(), model_layer_1_decode(), model_layer_20_decode(), model_layer_21_decode(), model_layer_22_decode(), model_layer_23_decode(), model_layer_2_decode(), model_layer_3_decode(), model_layer_4_decode(), model_layer_5_decode(), model_layer_6_decode(), model_layer_7_decode(), model_layer_8_decode(), model_layer_9_decode(), quantize_batch_q8_k(), qwen2_0_5b_decode_decode_token(), qwen2_0_5b_decode_forward_prefill_impl(), qwen2_0_5b_decode_layer_0_decode(), qwen2_0_5b_decode_layer_10_decode(), qwen2_0_5b_decode_layer_11_decode(), qwen2_0_5b_decode_layer_12_decode(), qwen2_0_5b_decode_layer_13_decode(), qwen2_0_5b_decode_layer_14_decode(), qwen2_0_5b_decode_layer_15_decode(), qwen2_0_5b_decode_layer_16_decode(), qwen2_0_5b_decode_layer_17_decode(), qwen2_0_5b_decode_layer_18_decode(), qwen2_0_5b_decode_layer_19_decode(), qwen2_0_5b_decode_layer_1_decode(), qwen2_0_5b_decode_layer_20_decode(), qwen2_0_5b_decode_layer_21_decode(), qwen2_0_5b_decode_layer_22_decode(), qwen2_0_5b_decode_layer_23_decode(), qwen2_0_5b_decode_layer_2_decode(), qwen2_0_5b_decode_layer_3_decode(), qwen2_0_5b_decode_layer_4_decode(), qwen2_0_5b_decode_layer_5_decode(), qwen2_0_5b_decode_layer_6_decode(), qwen2_0_5b_decode_layer_7_decode(), qwen2_0_5b_decode_layer_8_decode(), qwen2_0_5b_decode_layer_9_decode(), and unfused_rmsnorm_linear_q4k_ref().

quantize_row_q8_k_ref()

void quantize_row_q8_k_ref	(	const float *	x,
		void *	vy,
		int	k
	)

Definition at line 53 of file gemm_kernels_q4k_q8k.c.

                                                            {
    if (!x || !vy || k <= 0) {
        return;
    }
    assert(k % QK_K == 0);
    const int nb = k / QK_K;
    block_q8_K *y = (block_q8_K *)vy;
 
    for (int i = 0; i < nb; ++i) {
        float max = 0.0f;
        float amax = 0.0f;
        for (int j = 0; j < QK_K; ++j) {
            float ax = fabsf(x[j]);
            if (ax > amax) {
                amax = ax;
                max = x[j];
            }
        }
        if (!amax) {
            y[i].d = 0.0f;
            memset(y[i].qs, 0, sizeof(y[i].qs));
            memset(y[i].bsums, 0, sizeof(y[i].bsums));
            x += QK_K;
            continue;
        }
 
        const float iscale = -127.0f / max;
        for (int j = 0; j < QK_K; ++j) {
            int v = ck_nearest_int(iscale * x[j]);
            if (v > 127) {
                v = 127;
            }
            if (v < -128) {
                v = -128;
            }
            y[i].qs[j] = (int8_t)v;
        }
 
        for (int j = 0; j < QK_K / 16; ++j) {
            int sum = 0;
            const int8_t *qs = &y[i].qs[j * 16];
            for (int ii = 0; ii < 16; ++ii) {
                sum += qs[ii];
            }
            y[i].bsums[j] = (int16_t)sum;
        }
 
        y[i].d = 1.0f / iscale;
        x += QK_K;
    }
}

References block_q8_K::bsums, ck_nearest_int(), block_q8_K::d, QK_K, and block_q8_K::qs.

Referenced by quantize_row_q8_k().

quantize_row_q8_k_sse()

void quantize_row_q8_k_sse	(	const float *	x,
		void *	vy,
		int	k
	)

Definition at line 29 of file quantize_row_q8_k_sse.c.

                                                            {
    if (!x || !vy || k <= 0) {
        return;
    }
    assert(k % QK_K == 0);
    const int nb = k / QK_K;
    block_q8_K *y = (block_q8_K *)vy;
 
    for (int i = 0; i < nb; ++i) {
        float max = 0.0f;
        
        // SSE max absolute value
        __m128 v_max = _mm_setzero_ps();
        for (int j = 0; j < QK_K; j += 4) {
            __m128 v = _mm_loadu_ps(x + j);
            __m128 v_abs = _mm_andnot_ps(_mm_set1_ps(-0.0f), v);
            v_max = _mm_max_ps(v_max, v_abs);
        }
        
        // Horizontal max
        v_max = _mm_max_ps(v_max, _mm_shuffle_ps(v_max, v_max, _MM_SHUFFLE(1, 0, 3, 2)));
        v_max = _mm_max_ps(v_max, _mm_shuffle_ps(v_max, v_max, _MM_SHUFFLE(0, 1, 0, 1)));
        _mm_store_ss(&max, v_max);
 
        if (max == 0.0f) {
            y[i].d = 0.0f;
            memset(y[i].qs, 0, sizeof(y[i].qs));
            memset(y[i].bsums, 0, sizeof(y[i].bsums));
            x += QK_K;
            continue;
        }
 
        const float iscale = -127.0f / max;
        __m128 v_iscale = _mm_set1_ps(iscale);
        
        // Quantize and compute bsums in SSE
        for (int j = 0; j < QK_K; j += 16) {
            __m128 x0 = _mm_loadu_ps(x + j + 0);
            __m128 x1 = _mm_loadu_ps(x + j + 4);
            __m128 x2 = _mm_loadu_ps(x + j + 8);
            __m128 x3 = _mm_loadu_ps(x + j + 12);
 
            __m128i q0 = _mm_cvtps_epi32(_mm_mul_ps(x0, v_iscale));
            __m128i q1 = _mm_cvtps_epi32(_mm_mul_ps(x1, v_iscale));
            __m128i q2 = _mm_cvtps_epi32(_mm_mul_ps(x2, v_iscale));
            __m128i q3 = _mm_cvtps_epi32(_mm_mul_ps(x3, v_iscale));
 
            // Pack i32 -> i16 -> i8
            __m128i q01 = _mm_packs_epi32(q0, q1);
            __m128i q23 = _mm_packs_epi32(q2, q3);
            __m128i q0123 = _mm_packs_epi16(q01, q23);
 
            _mm_storeu_si128((__m128i *)(y[i].qs + j), q0123);
 
            // Compute bsum for these 16 elements
            // Each bsum[j/16] covers 16 elements
            __m128i p01 = _mm_add_epi16(q01, q23);
            p01 = _mm_add_epi16(p01, _mm_shuffle_epi32(p01, _MM_SHUFFLE(1, 0, 3, 2)));
            p01 = _mm_add_epi16(p01, _mm_shufflelo_epi16(p01, _MM_SHUFFLE(1, 0, 3, 2)));
            int16_t bsum = (int16_t)_mm_extract_epi16(p01, 0) + (int16_t)_mm_extract_epi16(p01, 1);
            y[i].bsums[j / 16] = bsum;
        }
 
        y[i].d = 1.0f / iscale;
        x += QK_K;
    }
}

Referenced by quantize_row_q8_k().