AVX entrypoint for exact Q8_K row quantization. More...

#include "ckernel_quant.h"

Functions
void	quantize_row_q8_k_avx (const float x, void vy, int k)

void	quantize_row_q8_k_ref (const float x, void vy, int k)

void	quantize_row_q8_k_sse (const float x, void vy, int k)

Detailed Description

AVX entrypoint for exact Q8_K row quantization.

CK-ENGINE KERNEL RULES:

NO malloc/free - memory via bump allocator, pointers passed in
NO OpenMP - parallelization at orchestrator/codegen layer
API must define: inputs, outputs, workspace, and memory layouts
Pure computation - deterministic, no side effects

After changes: make test && make llamacpp-parity-full

Definition in file quantize_row_q8_k_avx.c.

Function Documentation

◆ quantize_row_q8_k_avx()

void quantize_row_q8_k_avx	(	const float *	x,
		void *	vy,
		int	k
	)

Definition at line 20 of file quantize_row_q8_k_avx.c.

                                                            {
    /* Reuse the parity-clean SIMD implementation until a wider AVX variant is
     * worth maintaining separately. */
#if defined(__SSE4_1__)
    quantize_row_q8_k_sse(x, vy, k);
#else
    quantize_row_q8_k_ref(x, vy, k);
#endif
}

References quantize_row_q8_k_ref(), and quantize_row_q8_k_sse().

Referenced by quantize_row_q8_k().

◆ quantize_row_q8_k_ref()

void quantize_row_q8_k_ref	(	const float *	x,
		void *	vy,
		int	k
	)

Definition at line 61 of file gemm_kernels_q4k_q8k.c.

                                                            {
    if (!x || !vy || k <= 0) {
        return;
    }
    assert(k % QK_K == 0);
    const int nb = k / QK_K;
    block_q8_K *y = (block_q8_K *)vy;
 
    for (int i = 0; i < nb; ++i) {
        float max = 0.0f;
        float amax = 0.0f;
        for (int j = 0; j < QK_K; ++j) {
            float ax = fabsf(x[j]);
            if (ax > amax) {
                amax = ax;
                max = x[j];
            }
        }
        if (!amax) {
            y[i].d = 0.0f;
            memset(y[i].qs, 0, sizeof(y[i].qs));
            memset(y[i].bsums, 0, sizeof(y[i].bsums));
            x += QK_K;
            continue;
        }
 
        const float iscale = -127.0f / max;
        for (int j = 0; j < QK_K; ++j) {
            /* llama.cpp rounds the multiply before adding nearest_int's magic
             * constant. Contracting both operations changes Q8_K tie cases. */
            float scaled = iscale * x[j];
            int v = ck_nearest_int(scaled);
            if (v > 127) {
                v = 127;
            }
            if (v < -128) {
                v = -128;
            }
            y[i].qs[j] = (int8_t)v;
        }
 
        for (int j = 0; j < QK_K / 16; ++j) {
            int sum = 0;
            const int8_t *qs = &y[i].qs[j * 16];
            for (int ii = 0; ii < 16; ++ii) {
                sum += qs[ii];
            }
            y[i].bsums[j] = (int16_t)sum;
        }
 
        y[i].d = 1.0f / iscale;
        x += QK_K;
    }
}

Referenced by quantize_row_q8_k_avx().

◆ quantize_row_q8_k_sse()

void quantize_row_q8_k_sse	(	const float *	x,
		void *	vy,
		int	k
	)

Definition at line 22 of file quantize_row_q8_k_sse.c.

                                                            {
    if (!x || !vy || k <= 0) {
        return;
    }
    assert(k % QK_K == 0);
 
    const int nb = k / QK_K;
    block_q8_K *y = (block_q8_K *)vy;
 
    for (int i = 0; i < nb; ++i) {
        /* Keep the exact signed-max selection contract from llama.cpp/ref. */
        float max = 0.0f;
        float amax = 0.0f;
        for (int j = 0; j < QK_K; ++j) {
            const float xv = x[j];
            const float ax = fabsf(xv);
            if (ax > amax) {
                amax = ax;
                max = xv;
            }
        }
 
        if (amax == 0.0f) {
            y[i].d = 0.0f;
            memset(y[i].qs, 0, sizeof(y[i].qs));
            memset(y[i].bsums, 0, sizeof(y[i].bsums));
            x += QK_K;
            continue;
        }
 
        const float iscale = -127.0f / max;
        const __m128 v_iscale = _mm_set1_ps(iscale);
        const __m128 v_magic = _mm_set1_ps(12582912.0f);
        const __m128i v_mantissa = _mm_set1_epi32(0x007fffff);
        const __m128i v_bias = _mm_set1_epi32(0x00400000);
        const __m128i v_min = _mm_set1_epi32(-128);
        const __m128i v_max = _mm_set1_epi32(127);
 
        for (int j = 0; j < QK_K; j += 16) {
            const __m128 x0 = _mm_loadu_ps(x + j + 0);
            const __m128 x1 = _mm_loadu_ps(x + j + 4);
            const __m128 x2 = _mm_loadu_ps(x + j + 8);
            const __m128 x3 = _mm_loadu_ps(x + j + 12);
 
            __m128i q0 = _mm_sub_epi32(
                _mm_and_si128(_mm_castps_si128(_mm_add_ps(_mm_mul_ps(x0, v_iscale), v_magic)), v_mantissa),
                v_bias);
            __m128i q1 = _mm_sub_epi32(
                _mm_and_si128(_mm_castps_si128(_mm_add_ps(_mm_mul_ps(x1, v_iscale), v_magic)), v_mantissa),
                v_bias);
            __m128i q2 = _mm_sub_epi32(
                _mm_and_si128(_mm_castps_si128(_mm_add_ps(_mm_mul_ps(x2, v_iscale), v_magic)), v_mantissa),
                v_bias);
            __m128i q3 = _mm_sub_epi32(
                _mm_and_si128(_mm_castps_si128(_mm_add_ps(_mm_mul_ps(x3, v_iscale), v_magic)), v_mantissa),
                v_bias);
 
            q0 = _mm_min_epi32(_mm_max_epi32(q0, v_min), v_max);
            q1 = _mm_min_epi32(_mm_max_epi32(q1, v_min), v_max);
            q2 = _mm_min_epi32(_mm_max_epi32(q2, v_min), v_max);
            q3 = _mm_min_epi32(_mm_max_epi32(q3, v_min), v_max);
 
            const __m128i q01 = _mm_packs_epi32(q0, q1);
            const __m128i q23 = _mm_packs_epi32(q2, q3);
            const __m128i q0123 = _mm_packs_epi16(q01, q23);
 
            _mm_storeu_si128((__m128i *)(y[i].qs + j), q0123);
 
            int sum = 0;
            for (int ii = 0; ii < 16; ++ii) {
                sum += y[i].qs[j + ii];
            }
            y[i].bsums[j / 16] = (int16_t)sum;
        }
 
        y[i].d = 1.0f / iscale;
        x += QK_K;
    }
}