Q5_1 x Q8_1 contract kernels used for ggml parity (Gemma-sensitive path) More...

#include <stdint.h>
#include <string.h>
#include <math.h>
#include "ckernel_quant.h"

Macros
#define	CK_Q51_STACK_Q8_BLOCKS 256

#define	QK8_1 32

Functions
static float	dot_q5_1_q8_1_block (const block_q5_1 w, const block_q8_1 x)

void	gemm_nt_q5_1_q8_1 (const float A, const void B, const float bias, float C, int M, int N, int K)

void	gemm_nt_q5_1_q8_1_m2 (const float A, const void B, const float bias, float C, int M, int N, int K)

void	gemm_nt_q5_1_q8_1_m4 (const float A, const void B, const float bias, float C, int M, int N, int K)

void	gemm_nt_q5_1_q8_1_m8 (const float A, const void B, const float bias, float C, int M, int N, int K)

void	gemm_nt_q5_1_q8_1_ref (const void A_q8, const void B, const float bias, float C, int M, int N, int K)

void	gemv_q5_1_q8_1 (float y, const void W, const float *x, int M, int K)

void	gemv_q5_1_q8_1_ref (float y, const void W, const void *x_q8, int M, int K)

static void	quantize_row_q8_1_scalar (const float x, block_q8_1 y, int k)

Detailed Description

Q5_1 x Q8_1 contract kernels used for ggml parity (Gemma-sensitive path)

Definition in file gemm_kernels_q5_1_q8_1.c.

Macro Definition Documentation

◆ CK_Q51_STACK_Q8_BLOCKS

#define CK_Q51_STACK_Q8_BLOCKS 256

Definition at line 16 of file gemm_kernels_q5_1_q8_1.c.

◆ QK8_1

#define QK8_1 32

Definition at line 22 of file gemm_kernels_q5_1_q8_1.c.

Function Documentation

◆ dot_q5_1_q8_1_block()

static float dot_q5_1_q8_1_block	(	const block_q5_1 *	w,
		const block_q8_1 *	x
	)

static

Definition at line 224 of file gemm_kernels_q5_1_q8_1.c.

                                                                           {
#if defined(__AVX2__)
    return dot_q5_1_q8_1_block_avx2(w, x);
#else
    uint32_t qh;
    memcpy(&qh, w->qh, sizeof(qh));
 
    int sumi0 = 0;
    int sumi1 = 0;
    for (int j = 0; j < QK5_1 / 2; ++j) {
        const uint8_t xh0 = (uint8_t)(((qh >> (j + 0)) << 4) & 0x10);
        const uint8_t xh1 = (uint8_t)(((qh >> (j + 12))     ) & 0x10);
        const int32_t q0 = (int32_t)((w->qs[j] & 0x0F) | xh0);
        const int32_t q1 = (int32_t)((w->qs[j] >> 4) | xh1);
        sumi0 += q0 * (int32_t)x->qs[j];
        sumi1 += q1 * (int32_t)x->qs[j + QK5_1 / 2];
    }
 
    const float wd = CK_FP16_TO_FP32(w->d);
    const float wm = CK_FP16_TO_FP32(w->m);
    const float xd = CK_FP16_TO_FP32(x->d);
    const float xs = CK_FP16_TO_FP32(x->s);
    return (wd * xd) * (float)(sumi0 + sumi1) + wm * xs;
#endif
}

References CK_FP16_TO_FP32, block_q5_1::d, block_q5_1::m, block_q5_1::qh, QK5_1, and block_q5_1::qs.

Referenced by gemm_nt_q5_1_q8_1(), gemm_nt_q5_1_q8_1_m2(), gemm_nt_q5_1_q8_1_ref(), and gemv_q5_1_q8_1_ref().

◆ gemm_nt_q5_1_q8_1()

void gemm_nt_q5_1_q8_1	(	const float *	A,
		const void *	B,
		const float *	bias,
		float *	C,
		int	M,
		int	N,
		int	K
	)

Definition at line 320 of file gemm_kernels_q5_1_q8_1.c.

{
    if (!A || !B || !C || M <= 0 || N <= 0 || K <= 0 || (K % QK5_1) != 0) {
        return;
    }
 
    const int blocks_per_row = K / QK5_1;
    if (blocks_per_row > CK_Q51_STACK_Q8_BLOCKS) {
        return;
    }
 
    const block_q5_1 *W = (const block_q5_1 *)B;
 
    for (int m = 0; m < M; ++m) {
        block_q8_1 a_q8[CK_Q51_STACK_Q8_BLOCKS];
        quantize_row_q8_1_scalar(&A[m * K], a_q8, K);
        float *c_row = &C[(size_t)m * (size_t)N];
 
        int n = 0;
        for (; n + 7 < N; n += 8) {
            const block_q5_1 *w0 = &W[(size_t)(n + 0) * (size_t)blocks_per_row];
            const block_q5_1 *w1 = &W[(size_t)(n + 1) * (size_t)blocks_per_row];
            const block_q5_1 *w2 = &W[(size_t)(n + 2) * (size_t)blocks_per_row];
            const block_q5_1 *w3 = &W[(size_t)(n + 3) * (size_t)blocks_per_row];
            const block_q5_1 *w4 = &W[(size_t)(n + 4) * (size_t)blocks_per_row];
            const block_q5_1 *w5 = &W[(size_t)(n + 5) * (size_t)blocks_per_row];
            const block_q5_1 *w6 = &W[(size_t)(n + 6) * (size_t)blocks_per_row];
            const block_q5_1 *w7 = &W[(size_t)(n + 7) * (size_t)blocks_per_row];
            float s0 = 0.0f;
            float s1 = 0.0f;
            float s2 = 0.0f;
            float s3 = 0.0f;
            float s4 = 0.0f;
            float s5 = 0.0f;
            float s6 = 0.0f;
            float s7 = 0.0f;
 
            for (int b = 0; b < blocks_per_row; ++b) {
                const block_q8_1 *x = &a_q8[b];
                s0 += dot_q5_1_q8_1_block(&w0[b], x);
                s1 += dot_q5_1_q8_1_block(&w1[b], x);
                s2 += dot_q5_1_q8_1_block(&w2[b], x);
                s3 += dot_q5_1_q8_1_block(&w3[b], x);
                s4 += dot_q5_1_q8_1_block(&w4[b], x);
                s5 += dot_q5_1_q8_1_block(&w5[b], x);
                s6 += dot_q5_1_q8_1_block(&w6[b], x);
                s7 += dot_q5_1_q8_1_block(&w7[b], x);
            }
 
            c_row[n + 0] = s0 + (bias ? bias[n + 0] : 0.0f);
            c_row[n + 1] = s1 + (bias ? bias[n + 1] : 0.0f);
            c_row[n + 2] = s2 + (bias ? bias[n + 2] : 0.0f);
            c_row[n + 3] = s3 + (bias ? bias[n + 3] : 0.0f);
            c_row[n + 4] = s4 + (bias ? bias[n + 4] : 0.0f);
            c_row[n + 5] = s5 + (bias ? bias[n + 5] : 0.0f);
            c_row[n + 6] = s6 + (bias ? bias[n + 6] : 0.0f);
            c_row[n + 7] = s7 + (bias ? bias[n + 7] : 0.0f);
        }
 
        for (; n < N; ++n) {
            const block_q5_1 *w_row = &W[(size_t)n * (size_t)blocks_per_row];
            float sum = 0.0f;
            for (int b = 0; b < blocks_per_row; ++b) {
                sum += dot_q5_1_q8_1_block(&w_row[b], &a_q8[b]);
            }
            c_row[n] = sum + (bias ? bias[n] : 0.0f);
        }
    }
}

References C, CK_Q51_STACK_Q8_BLOCKS, dot_q5_1_q8_1_block(), QK5_1, and quantize_row_q8_1_scalar().

Referenced by gemm_nt_q5_1_q8_1_m4().

◆ gemm_nt_q5_1_q8_1_m2()

void gemm_nt_q5_1_q8_1_m2	(	const float *	A,
		const void *	B,
		const float *	bias,
		float *	C,
		int	M,
		int	N,
		int	K
	)

Definition at line 396 of file gemm_kernels_q5_1_q8_1.c.

{
    if (!A || !B || !C || M != 2 || N <= 0 || K <= 0 || (K % QK5_1) != 0)
        return;
    const int blocks_per_row = K / QK5_1;
    if (blocks_per_row > CK_Q51_STACK_Q8_BLOCKS) return;
    block_q8_1 x[2][CK_Q51_STACK_Q8_BLOCKS];
    quantize_row_q8_1_scalar(A, x[0], K);
    quantize_row_q8_1_scalar(A + K, x[1], K);
    const block_q5_1 *weights = (const block_q5_1 *)B;
    for (int n = 0; n < N; ++n) {
        const block_q5_1 *w = weights + (size_t)n * (size_t)blocks_per_row;
        float sums[2] = {0.0f, 0.0f};
        for (int b = 0; b < blocks_per_row; ++b) {
#if defined(__AVX2__)
            float partial[2];
            dot_q5_1_q8_1_block_m2_avx2(w + b, x[0] + b, x[1] + b, partial);
            sums[0] += partial[0];
            sums[1] += partial[1];
#else
            sums[0] += dot_q5_1_q8_1_block(w + b, x[0] + b);
            sums[1] += dot_q5_1_q8_1_block(w + b, x[1] + b);
#endif
        }
        const float add = bias ? bias[n] : 0.0f;
        C[n] = sums[0] + add;
        C[(size_t)N + n] = sums[1] + add;
    }
}

References C, CK_Q51_STACK_Q8_BLOCKS, dot_q5_1_q8_1_block(), QK5_1, and quantize_row_q8_1_scalar().

◆ gemm_nt_q5_1_q8_1_m4()

void gemm_nt_q5_1_q8_1_m4	(	const float *	A,
		const void *	B,
		const float *	bias,
		float *	C,
		int	M,
		int	N,
		int	K
	)

Definition at line 427 of file gemm_kernels_q5_1_q8_1.c.

{
#if defined(__AVX2__)
    if (!A || !B || !C || M <= 0 || N <= 0 || K <= 0 || (K % QK5_1) != 0) {
        return;
    }
 
    const int blocks_per_row = K / QK5_1;
    if (blocks_per_row > CK_Q51_STACK_Q8_BLOCKS) {
        gemm_nt_q5_1_q8_1(A, B, bias, C, M, N, K);
        return;
    }
 
    const block_q5_1 *weights = (const block_q5_1 *)B;
    int m = 0;
    for (; m + 3 < M; m += 4) {
        block_q8_1 activation_q8[4][CK_Q51_STACK_Q8_BLOCKS];
        for (int row = 0; row < 4; ++row) {
            quantize_row_q8_1_scalar(
                &A[(size_t)(m + row) * (size_t)K], activation_q8[row], K);
        }
 
        for (int n = 0; n < N; ++n) {
            const block_q5_1 *weight_row =
                &weights[(size_t)n * (size_t)blocks_per_row];
            float sums[4] = {0.0f, 0.0f, 0.0f, 0.0f};
            for (int block = 0; block < blocks_per_row; ++block) {
                float partial[4];
                dot_q5_1_q8_1_block_m4_avx2(
                    &weight_row[block],
                    &activation_q8[0][block], &activation_q8[1][block],
                    &activation_q8[2][block], &activation_q8[3][block], partial);
                sums[0] += partial[0];
                sums[1] += partial[1];
                sums[2] += partial[2];
                sums[3] += partial[3];
            }
            const float add = bias ? bias[n] : 0.0f;
            C[(size_t)(m + 0) * (size_t)N + n] = sums[0] + add;
            C[(size_t)(m + 1) * (size_t)N + n] = sums[1] + add;
            C[(size_t)(m + 2) * (size_t)N + n] = sums[2] + add;
            C[(size_t)(m + 3) * (size_t)N + n] = sums[3] + add;
        }
    }
 
    if (m < M) {
        gemm_nt_q5_1_q8_1(
            A + (size_t)m * (size_t)K, B, bias,
            C + (size_t)m * (size_t)N, M - m, N, K);
    }
#else
    gemm_nt_q5_1_q8_1(A, B, bias, C, M, N, K);
#endif
}

References C, CK_Q51_STACK_Q8_BLOCKS, gemm_nt_q5_1_q8_1(), QK5_1, and quantize_row_q8_1_scalar().

Referenced by gemm_nt_q5_1_q8_1_m8().

◆ gemm_nt_q5_1_q8_1_m8()

void gemm_nt_q5_1_q8_1_m8	(	const float *	A,
		const void *	B,
		const float *	bias,
		float *	C,
		int	M,
		int	N,
		int	K
	)

Definition at line 488 of file gemm_kernels_q5_1_q8_1.c.

{
#if defined(__AVX2__)
    if (!A || !B || !C || M <= 0 || N <= 0 || K <= 0 || (K % QK5_1) != 0) {
        return;
    }
 
    const int blocks_per_row = K / QK5_1;
    if (blocks_per_row > CK_Q51_STACK_Q8_BLOCKS) {
        gemm_nt_q5_1_q8_1_m4(A, B, bias, C, M, N, K);
        return;
    }
 
    const block_q5_1 *weights = (const block_q5_1 *)B;
    int m = 0;
    for (; m + 7 < M; m += 8) {
        block_q8_1 activation_q8[8][CK_Q51_STACK_Q8_BLOCKS];
        for (int row = 0; row < 8; ++row) {
            quantize_row_q8_1_scalar(
                &A[(size_t)(m + row) * (size_t)K], activation_q8[row], K);
        }
 
        for (int n = 0; n < N; ++n) {
            const block_q5_1 *weight_row =
                &weights[(size_t)n * (size_t)blocks_per_row];
            float sums[8] = {0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f};
            for (int block = 0; block < blocks_per_row; ++block) {
                const block_q8_1 *rows[8] = {
                    &activation_q8[0][block], &activation_q8[1][block],
                    &activation_q8[2][block], &activation_q8[3][block],
                    &activation_q8[4][block], &activation_q8[5][block],
                    &activation_q8[6][block], &activation_q8[7][block],
                };
                float partial[8];
                dot_q5_1_q8_1_block_m8_avx2(&weight_row[block], rows, partial);
                for (int row = 0; row < 8; ++row) {
                    sums[row] += partial[row];
                }
            }
            const float add = bias ? bias[n] : 0.0f;
            for (int row = 0; row < 8; ++row) {
                C[(size_t)(m + row) * (size_t)N + n] = sums[row] + add;
            }
        }
    }
 
    if (m < M) {
        gemm_nt_q5_1_q8_1_m4(
            A + (size_t)m * (size_t)K, B, bias,
            C + (size_t)m * (size_t)N, M - m, N, K);
    }
#else
    gemm_nt_q5_1_q8_1_m4(A, B, bias, C, M, N, K);
#endif
}

References C, CK_Q51_STACK_Q8_BLOCKS, gemm_nt_q5_1_q8_1_m4(), QK5_1, and quantize_row_q8_1_scalar().

◆ gemm_nt_q5_1_q8_1_ref()

void gemm_nt_q5_1_q8_1_ref	(	const void *	A_q8,
		const void *	B,
		const float *	bias,
		float *	C,
		int	M,
		int	N,
		int	K
	)

Definition at line 273 of file gemm_kernels_q5_1_q8_1.c.

{
    if (!A_q8 || !B || !C || M <= 0 || N <= 0 || K <= 0 || (K % QK5_1) != 0) {
        return;
    }
 
    const block_q8_1 *A = (const block_q8_1 *)A_q8;
    const block_q5_1 *W = (const block_q5_1 *)B;
    const int blocks_per_row = K / QK5_1;
 
    for (int m = 0; m < M; ++m) {
        const block_q8_1 *a_row = &A[m * blocks_per_row];
        for (int n = 0; n < N; ++n) {
            const block_q5_1 *w_row = &W[n * blocks_per_row];
            float sum = 0.0f;
            for (int b = 0; b < blocks_per_row; ++b) {
                sum += dot_q5_1_q8_1_block(&w_row[b], &a_row[b]);
            }
            C[m * N + n] = sum + (bias ? bias[n] : 0.0f);
        }
    }
}

References C, dot_q5_1_q8_1_block(), and QK5_1.

◆ gemv_q5_1_q8_1()

void gemv_q5_1_q8_1	(	float *	y,
		const void *	W,
		const float *	x,
		int	M,
		int	K
	)

Definition at line 300 of file gemm_kernels_q5_1_q8_1.c.

{
    if (!y || !W || !x || M <= 0 || K <= 0 || (K % QK5_1) != 0) {
        return;
    }
 
    const int blocks_per_row = K / QK5_1;
    if (blocks_per_row > CK_Q51_STACK_Q8_BLOCKS) {
        return;
    }
 
    block_q8_1 x_q8[CK_Q51_STACK_Q8_BLOCKS];
    quantize_row_q8_1_scalar(x, x_q8, K);
    gemv_q5_1_q8_1_ref(y, W, x_q8, M, K);
}

References CK_Q51_STACK_Q8_BLOCKS, gemv_q5_1_q8_1_ref(), QK5_1, and quantize_row_q8_1_scalar().

◆ gemv_q5_1_q8_1_ref()

void gemv_q5_1_q8_1_ref	(	float *	y,
		const void *	W,
		const void *	x_q8,
		int	M,
		int	K
	)

Definition at line 250 of file gemm_kernels_q5_1_q8_1.c.

{
    if (!y || !W || !x_q8 || M <= 0 || K <= 0 || (K % QK5_1) != 0) {
        return;
    }
 
    const block_q5_1 *blocks = (const block_q5_1 *)W;
    const block_q8_1 *x = (const block_q8_1 *)x_q8;
    const int blocks_per_row = K / QK5_1;
 
    for (int row = 0; row < M; ++row) {
        const block_q5_1 *w_row = &blocks[row * blocks_per_row];
        float sum = 0.0f;
        for (int b = 0; b < blocks_per_row; ++b) {
            sum += dot_q5_1_q8_1_block(&w_row[b], &x[b]);
        }
        y[row] = sum;
    }
}

References dot_q5_1_q8_1_block(), and QK5_1.

Referenced by gemv_q5_1_q8_1().

◆ quantize_row_q8_1_scalar()

static void quantize_row_q8_1_scalar	(	const float *	x,
		block_q8_1 *	y,
		int	k
	)