gemv_omp.c File Reference

#include "ckernel_quant.h"

Go to the source code of this file.

Functions
void	gemv_fused_q5_0_bias_parallel_omp (float *y, const void *W, const float *x, const float *bias, int M, int K)
void	gemv_q5_0_q8_0_parallel_omp (float *y, const void *W, const void *x_q8, int M, int K)
void	gemv_q8_0_q8_0_parallel_omp (float *y, const void *W, const void *x_q8, int M, int K)
void	quantize_row_q8_0 (const float *x, void *y, int k)
	Quantize FP32 to Q8_0 format (scalar reference)
void	vec_dot_q5_0_q8_0 (int n, float *s, const void *vx, const void *vy)
	Auto-dispatch quantized dot product Q5_0 x Q8_0.
void	vec_dot_q8_0_q8_0 (int n, float *s, const void *vx, const void *vy)
	Auto-dispatch quantized dot product Q8_0 x Q8_0.

Function Documentation

gemv_fused_q5_0_bias_parallel_omp()

void gemv_fused_q5_0_bias_parallel_omp	(	float *	y,
		const void *	W,
		const float *	x,
		const float *	bias,
		int	M,
		int	K
	)

Definition at line 98 of file gemv_omp.c.

{
    const block_q5_0 *w_blocks = (const block_q5_0 *)W;
    const int blocks_per_row = K / QK5_0;
 
    /* Quantize input ONCE (serial, fast — K=896 → 28 blocks = 952 bytes) */
    block_q8_0 x_q8[K / QK8_0];
    quantize_row_q8_0(x, (void *)x_q8, K);
 
    /* Parallel GEMV over output rows */
    #pragma omp parallel for schedule(static)
    for (int row = 0; row < M; row++) {
        vec_dot_q5_0_q8_0(K, &y[row],
                          &w_blocks[row * blocks_per_row],
                          x_q8);
        if (bias) y[row] += bias[row];
    }
}

References QK5_0, QK8_0, quantize_row_q8_0(), and vec_dot_q5_0_q8_0().

gemv_q5_0_q8_0_parallel_omp()

void gemv_q5_0_q8_0_parallel_omp	(	float *	y,
		const void *	W,
		const void *	x_q8,
		int	M,
		int	K
	)

Definition at line 74 of file gemv_omp.c.

{
    const block_q5_0 *w_blocks = (const block_q5_0 *)W;
    const block_q8_0 *x_blocks = (const block_q8_0 *)x_q8;
    const int blocks_per_row = K / QK5_0;
 
    #pragma omp parallel for schedule(static)
    for (int row = 0; row < M; row++) {
        vec_dot_q5_0_q8_0(K, &y[row],
                          &w_blocks[row * blocks_per_row],
                          x_blocks);
    }
}

References QK5_0, and vec_dot_q5_0_q8_0().

gemv_q8_0_q8_0_parallel_omp()

void gemv_q8_0_q8_0_parallel_omp	(	float *	y,
		const void *	W,
		const void *	x_q8,
		int	M,
		int	K
	)

Definition at line 52 of file gemv_omp.c.

{
    const block_q8_0 *w_blocks = (const block_q8_0 *)W;
    const block_q8_0 *x_blocks = (const block_q8_0 *)x_q8;
    const int blocks_per_row = K / QK8_0;
 
    #pragma omp parallel for schedule(static)
    for (int row = 0; row < M; row++) {
        vec_dot_q8_0_q8_0(K, &y[row],
                          &w_blocks[row * blocks_per_row],
                          x_blocks);
    }
}

References QK8_0, and vec_dot_q8_0_q8_0().

quantize_row_q8_0()

void quantize_row_q8_0 ( const float *  x, void *  vy, int  k  )

extern

Quantize FP32 to Q8_0 format (scalar reference)

Parameters

x	Input FP32 values
vy	Output Q8_0 blocks
k	Number of elements (must be multiple of 32)

Definition at line 125 of file gemm_kernels_q8_0.c.

{
    block_q8_0 *y = (block_q8_0 *)vy;
    const int nb = k / QK8_0;  /* QK8_0 = 32 */
 
#if defined(__AVX__)
    const __m256 sign_bit = _mm256_set1_ps(-0.0f);
 
    for (int i = 0; i < nb; i++) {
        __m256 v0 = _mm256_loadu_ps(x + 0);
        __m256 v1 = _mm256_loadu_ps(x + 8);
        __m256 v2 = _mm256_loadu_ps(x + 16);
        __m256 v3 = _mm256_loadu_ps(x + 24);
        x += QK8_0;
 
        __m256 max_abs = _mm256_andnot_ps(sign_bit, v0);
        max_abs = _mm256_max_ps(max_abs, _mm256_andnot_ps(sign_bit, v1));
        max_abs = _mm256_max_ps(max_abs, _mm256_andnot_ps(sign_bit, v2));
        max_abs = _mm256_max_ps(max_abs, _mm256_andnot_ps(sign_bit, v3));
 
        __m128 max4 = _mm_max_ps(_mm256_extractf128_ps(max_abs, 1),
                                 _mm256_castps256_ps128(max_abs));
        max4 = _mm_max_ps(max4, _mm_movehl_ps(max4, max4));
        max4 = _mm_max_ss(max4, _mm_movehdup_ps(max4));
        const float max_scalar = _mm_cvtss_f32(max4);
 
#if defined(__INTEL_LLVM_COMPILER)
        const float d = ck_q8_0_div_rounded_f32(max_scalar, 127.0f);
        const float id = max_scalar != 0.0f
            ? ck_q8_0_div_rounded_f32(127.0f, max_scalar)
            : 0.0f;
#else
        const float d = max_scalar / 127.0f;
        const float id = max_scalar != 0.0f ? 127.0f / max_scalar : 0.0f;
#endif
        y[i].d = CK_FP32_TO_FP16(d);
 
        const __m256 mul = _mm256_set1_ps(id);
        v0 = _mm256_mul_ps(v0, mul);
        v1 = _mm256_mul_ps(v1, mul);
        v2 = _mm256_mul_ps(v2, mul);
        v3 = _mm256_mul_ps(v3, mul);
 
        /* Match llama.cpp x86 Q8 quantization: nearest-even rounding. */
        v0 = _mm256_round_ps(v0, _MM_ROUND_NEAREST);
        v1 = _mm256_round_ps(v1, _MM_ROUND_NEAREST);
        v2 = _mm256_round_ps(v2, _MM_ROUND_NEAREST);
        v3 = _mm256_round_ps(v3, _MM_ROUND_NEAREST);
 
        __m256i i0 = _mm256_cvtps_epi32(v0);
        __m256i i1 = _mm256_cvtps_epi32(v1);
        __m256i i2 = _mm256_cvtps_epi32(v2);
        __m256i i3 = _mm256_cvtps_epi32(v3);
 
#if defined(__AVX2__)
        i0 = _mm256_packs_epi32(i0, i1);
        i2 = _mm256_packs_epi32(i2, i3);
        i0 = _mm256_packs_epi16(i0, i2);
 
        const __m256i perm = _mm256_setr_epi32(0, 4, 1, 5, 2, 6, 3, 7);
        i0 = _mm256_permutevar8x32_epi32(i0, perm);
        _mm256_storeu_si256((__m256i *)y[i].qs, i0);
#else
        __m128i ni0 = _mm256_castsi256_si128(i0);
        __m128i ni1 = _mm256_extractf128_si256(i0, 1);
        __m128i ni2 = _mm256_castsi256_si128(i1);
        __m128i ni3 = _mm256_extractf128_si256(i1, 1);
        __m128i ni4 = _mm256_castsi256_si128(i2);
        __m128i ni5 = _mm256_extractf128_si256(i2, 1);
        __m128i ni6 = _mm256_castsi256_si128(i3);
        __m128i ni7 = _mm256_extractf128_si256(i3, 1);
 
        ni0 = _mm_packs_epi32(ni0, ni1);
        ni2 = _mm_packs_epi32(ni2, ni3);
        ni4 = _mm_packs_epi32(ni4, ni5);
        ni6 = _mm_packs_epi32(ni6, ni7);
 
        ni0 = _mm_packs_epi16(ni0, ni2);
        ni4 = _mm_packs_epi16(ni4, ni6);
 
        _mm_storeu_si128((__m128i *)(y[i].qs + 0), ni0);
        _mm_storeu_si128((__m128i *)(y[i].qs + 16), ni4);
#endif
    }
#else
    for (int i = 0; i < nb; i++) {
        const float *xb = x + i * QK8_0;
 
        /* Find max absolute value in block */
        float amax = 0.0f;
        for (int j = 0; j < QK8_0; j++) {
            float av = xb[j] >= 0 ? xb[j] : -xb[j];
            if (av > amax) amax = av;
        }
 
        /* Compute scale: d = max / 127 */
        float d = amax / 127.0f;
        float id = d != 0.0f ? 127.0f / amax : 0.0f;
 
        /* Store scale as FP16 */
        y[i].d = CK_FP32_TO_FP16(d);
 
        /* Quantize values */
        for (int j = 0; j < QK8_0; j++) {
            float v = xb[j] * id;
            int q = ck_nearest_int_q8_0(v);
            if (q > 127) q = 127;
            if (q < -127) q = -127;
            y[i].qs[j] = (int8_t)q;
        }
    }
#endif
}

Referenced by gemv_fused_q5_0_bias_parallel_omp(), and quantize_batch_q8_0().

vec_dot_q5_0_q8_0()

void vec_dot_q5_0_q8_0 ( int  n, float *  s, const void *  vx, const void *  vy  )

extern

Auto-dispatch quantized dot product Q5_0 x Q8_0.

Dispatch priority:

1. AVX512 (best performance on modern Intel/AMD)
2. AVX (256-bit float ops, works on Sandy/Ivy Bridge and newer)
3. SSSE3 (128-bit fallback)
4. Reference scalar (last resort)

Definition at line 1602 of file gemm_kernels_q5_0.c.

{
#if defined(__AVX2__)
    /* llama.cpp uses the packed AVX2 dot on AVX-512 hosts as well. It keeps
     * Q5/Q8 data in byte lanes and avoids the per-block 32-bit lane expansion
     * overhead of the baseline AVX-512 path. */
    vec_dot_q5_0_q8_0_avx2(n, s, vx, vy);
#elif defined(__AVX512F__)
    vec_dot_q5_0_q8_0_avx512(n, s, vx, vy);
#elif defined(__ARM_NEON) || defined(__aarch64__)
    vec_dot_q5_0_q8_0_neon(n, s, vx, vy);
#elif defined(__AVX__)
    /* AVX for 256-bit float ops (works on Ivy Bridge and newer) */
    vec_dot_q5_0_q8_0_avx(n, s, vx, vy);
#elif defined(__SSSE3__)
    /* SSSE3 - most efficient on older CPUs */
    vec_dot_q5_0_q8_0_sse(n, s, vx, vy);
#else
    vec_dot_q5_0_q8_0_ref(n, s, vx, vy);
#endif
}

Referenced by gemm_nt_q5_0_q8_0(), gemm_nt_q5_0_q8_0_m2n4_tile(), gemm_nt_q5_0_q8_0_m4n2_tile(), gemv_fused_q5_0_bias_parallel_omp(), gemv_q5_0_q8_0(), gemv_q5_0_q8_0_parallel_omp(), and gemv_q5_0_q8_0_parallel_simd().

vec_dot_q8_0_q8_0()

void vec_dot_q8_0_q8_0 ( int  n, float *  s, const void *  vx, const void *  vy  )

extern

Auto-dispatch quantized dot product Q8_0 x Q8_0.

Definition at line 1368 of file gemm_kernels_q8_0.c.

{
    if (ck_q8_0_q8_0_debug_ref()) {
        vec_dot_q8_0_q8_0_ref(n, s, vx, vy);
        return;
    }
#ifdef __AVX512F__
    vec_dot_q8_0_q8_0_avx512(n, s, vx, vy);
#elif defined(__AVX2__)
    vec_dot_q8_0_q8_0_avx2(n, s, vx, vy);
#elif defined(__ARM_NEON) || defined(__aarch64__)
    vec_dot_q8_0_q8_0_neon(n, s, vx, vy);
#elif defined(__AVX__)
    vec_dot_q8_0_q8_0_avx(n, s, vx, vy);
#elif defined(__SSE4_1__)
    vec_dot_q8_0_q8_0_sse(n, s, vx, vy);
#else
    vec_dot_q8_0_q8_0_ref(n, s, vx, vy);
#endif
}

Referenced by gemv_q8_0_q8_0(), gemv_q8_0_q8_0_parallel(), gemv_q8_0_q8_0_parallel_omp(), and gemv_q8_0_q8_0_parallel_simd().