FP32 API adapters that enforce Q8_0 x Q8_0 activation contract. More...

#include "ckernel_engine.h"
#include "ckernel_quant.h"
#include "ggml_runtime_compat.h"
#include <dlfcn.h>
#include <stdint.h>
#include <math.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

Macros
#define	_GNU_SOURCE

#define	CK_Q80_STACK_Q8_BLOCKS 1024

Typedefs
typedef void(*	ck_q80_ggml_build_forward_expand_fn) (struct ggml_cgraph , struct ggml_tensor )

typedef void(*	ck_q80_ggml_cpu_init_fn) (void)

typedef void(*	ck_q80_ggml_free_fn) (struct ggml_context *)

typedef float (	ck_q80_ggml_get_data_f32_fn) (const struct ggml_tensor *)

typedef void (	ck_q80_ggml_get_data_fn) (const struct ggml_tensor *)

typedef enum ggml_status(*	ck_q80_ggml_graph_compute_with_ctx_fn) (struct ggml_context , struct ggml_cgraph , int)

typedef struct ggml_context (	ck_q80_ggml_init_fn) (struct ggml_init_params)

typedef struct ggml_tensor (	ck_q80_ggml_mul_mat_fn) (struct ggml_context , struct ggml_tensor , struct ggml_tensor *)

typedef size_t(*	ck_q80_ggml_nbytes_fn) (const struct ggml_tensor *)

typedef struct ggml_cgraph (	ck_q80_ggml_new_graph_fn) (struct ggml_context *)

typedef struct ggml_tensor (	ck_q80_ggml_new_tensor_2d_fn) (struct ggml_context *, enum ggml_type, int64_t, int64_t)

Functions
static int	ck_nearest_int_q8_0_ref (float fval)

static int	ck_q80_contract_cached_input_enabled (void)

static int	ck_q80_contract_dump_enabled (void)

static void	ck_q80_contract_dump_tensor (const char name, int layer_id, const float data, size_t elem_count)

static ck_q80_ggml_build_forward_expand_fn	ck_q80_resolve_ggml_build_forward_expand (void)

static ck_q80_ggml_cpu_init_fn	ck_q80_resolve_ggml_cpu_init (void)

static ck_q80_ggml_free_fn	ck_q80_resolve_ggml_free (void)

static ck_q80_ggml_get_data_fn	ck_q80_resolve_ggml_get_data (void)

static ck_q80_ggml_get_data_f32_fn	ck_q80_resolve_ggml_get_data_f32 (void)

static ck_q80_ggml_graph_compute_with_ctx_fn	ck_q80_resolve_ggml_graph_compute_with_ctx (void)

static ck_q80_ggml_init_fn	ck_q80_resolve_ggml_init (void)

static ck_q80_ggml_mul_mat_fn	ck_q80_resolve_ggml_mul_mat (void)

static ck_q80_ggml_nbytes_fn	ck_q80_resolve_ggml_nbytes (void)

static ck_q80_ggml_new_graph_fn	ck_q80_resolve_ggml_new_graph (void)

static ck_q80_ggml_new_tensor_2d_fn	ck_q80_resolve_ggml_new_tensor_2d (void)

void	gemm_nt_q8_0_q8_0_contract (const float A, const void B, const float bias, float C, int M, int N, int K)

void	gemm_nt_q8_0_q8_0_contract_m2 (const float A, const void B, const float bias, float C, int M, int N, int K)

static int	gemm_nt_q8_0_q8_0_ggml_strict (const float A, const void B, const float bias, float C, int M, int N, int K)

void	gemm_q8_0_q8_0_m2n4_strided (float C, int ldc, const void W, const void *A_q8, int M, int N, int K)

void	gemv_q8_0_q8_0_contract (float y, const void W, const float *x, int M, int K)

static void	gemv_q8_0_q8_0_ref_rows (float y, const void W, const void *x_q8, int M, int K)

static void	quantize_row_q8_0_ref_local (const float x, block_q8_0 y, int k)

void	vec_dot_q8_0_q8_0_ref (int n, float s, const void vx, const void *vy)
	Quantized dot product: Q8_0 weights x Q8_0 input (scalar reference)

Variables
static int	ck_q80_contract_cached_gemm_seq = 0

static const char	ck_q80_contract_magic [8] = {'C', 'K', 'D', 'M', 'P', '\0', '\0', '\0'}

static const uint32_t	ck_q80_contract_version = 1u

Detailed Description

FP32 API adapters that enforce Q8_0 x Q8_0 activation contract.

Definition in file gemm_kernels_q8_0_q8_0_contract.c.

Macro Definition Documentation

◆ _GNU_SOURCE

#define _GNU_SOURCE

Definition at line 7 of file gemm_kernels_q8_0_q8_0_contract.c.

◆ CK_Q80_STACK_Q8_BLOCKS

#define CK_Q80_STACK_Q8_BLOCKS 1024

Definition at line 21 of file gemm_kernels_q8_0_q8_0_contract.c.

Typedef Documentation

◆ ck_q80_ggml_build_forward_expand_fn

typedef void(* ck_q80_ggml_build_forward_expand_fn) (struct ggml_cgraph *, struct ggml_tensor *)

Definition at line 91 of file gemm_kernels_q8_0_q8_0_contract.c.

◆ ck_q80_ggml_cpu_init_fn

typedef void(* ck_q80_ggml_cpu_init_fn) (void)

Definition at line 93 of file gemm_kernels_q8_0_q8_0_contract.c.

◆ ck_q80_ggml_free_fn

typedef void(* ck_q80_ggml_free_fn) (struct ggml_context *)

Definition at line 87 of file gemm_kernels_q8_0_q8_0_contract.c.

◆ ck_q80_ggml_get_data_f32_fn

typedef float *(* ck_q80_ggml_get_data_f32_fn) (const struct ggml_tensor *)

Definition at line 95 of file gemm_kernels_q8_0_q8_0_contract.c.

◆ ck_q80_ggml_get_data_fn

typedef void *(* ck_q80_ggml_get_data_fn) (const struct ggml_tensor *)

Definition at line 94 of file gemm_kernels_q8_0_q8_0_contract.c.

◆ ck_q80_ggml_graph_compute_with_ctx_fn

typedef enum ggml_status(* ck_q80_ggml_graph_compute_with_ctx_fn) (struct ggml_context *, struct ggml_cgraph *, int)

Definition at line 91 of file gemm_kernels_q8_0_q8_0_contract.c.

◆ ck_q80_ggml_init_fn

typedef struct ggml_context *(* ck_q80_ggml_init_fn) (struct ggml_init_params)

Definition at line 52 of file gemm_kernels_q8_0_q8_0_contract.c.

◆ ck_q80_ggml_mul_mat_fn

typedef struct ggml_tensor *(* ck_q80_ggml_mul_mat_fn) (struct ggml_context *, struct ggml_tensor *, struct ggml_tensor *)

Definition at line 87 of file gemm_kernels_q8_0_q8_0_contract.c.

◆ ck_q80_ggml_nbytes_fn

typedef size_t(* ck_q80_ggml_nbytes_fn) (const struct ggml_tensor *)

Definition at line 96 of file gemm_kernels_q8_0_q8_0_contract.c.

◆ ck_q80_ggml_new_graph_fn

typedef struct ggml_cgraph *(* ck_q80_ggml_new_graph_fn) (struct ggml_context *)

Definition at line 87 of file gemm_kernels_q8_0_q8_0_contract.c.

◆ ck_q80_ggml_new_tensor_2d_fn

typedef struct ggml_tensor *(* ck_q80_ggml_new_tensor_2d_fn) (struct ggml_context *, enum ggml_type, int64_t, int64_t)

Definition at line 87 of file gemm_kernels_q8_0_q8_0_contract.c.

Function Documentation

◆ ck_nearest_int_q8_0_ref()

static int ck_nearest_int_q8_0_ref ( float fval )

inlinestatic

Definition at line 324 of file gemm_kernels_q8_0_q8_0_contract.c.

{
    float val = fval + 12582912.f;
    int i;
    memcpy(&i, &val, sizeof(int));
    return (i & 0x007fffff) - 0x00400000;
}

Referenced by quantize_row_q8_0_ref_local().

◆ ck_q80_contract_cached_input_enabled()

static int ck_q80_contract_cached_input_enabled ( void )

static

Definition at line 46 of file gemm_kernels_q8_0_q8_0_contract.c.

{
    const char *v = getenv("CK_STRICT_GEMM_USE_CACHED_A");
    return v && v[0] && strcmp(v, "0") != 0;
}

Referenced by gemm_nt_q8_0_q8_0_contract().

◆ ck_q80_contract_dump_enabled()

static int ck_q80_contract_dump_enabled ( void )

static

Definition at line 40 of file gemm_kernels_q8_0_q8_0_contract.c.

{
    const char *v = getenv("CK_STRICT_GEMM_DUMP");
    return v && v[0] && strcmp(v, "0") != 0;
}

Referenced by gemm_nt_q8_0_q8_0_contract().

◆ ck_q80_contract_dump_tensor()

static void ck_q80_contract_dump_tensor	(	const char *	name,
		int	layer_id,
		const float *	data,
		size_t	elem_count
	)

static

Definition at line 52 of file gemm_kernels_q8_0_q8_0_contract.c.

{
    const char *dir = getenv("CK_PARITY_DIR");
    if (!dir || !dir[0] || !data || elem_count == 0 || !name || !name[0]) {
        return;
    }
 
    char path[4096];
    snprintf(path, sizeof(path), "%s/%s", dir, "strict_internal.bin");
    FILE *f = fopen(path, "ab");
    if (!f) {
        return;
    }
 
    ck_q80_contract_dump_header_t h;
    memset(&h, 0, sizeof(h));
    memcpy(h.magic, ck_q80_contract_magic, sizeof(ck_q80_contract_magic));
    h.version = ck_q80_contract_version;
    h.layer_id = layer_id;
    strncpy(h.op_name, name, sizeof(h.op_name) - 1);
    h.dtype = 0u;
    h.rank = 1u;
    h.shape[0] = (int64_t) elem_count;
    h.elem_count = (uint32_t) elem_count;
    h.token_id = 0;
 
    fwrite(&h, sizeof(h), 1, f);
    fwrite(data, sizeof(float), elem_count, f);
    fclose(f);
}

References ck_q80_contract_magic, and ck_q80_contract_version.

Referenced by gemm_nt_q8_0_q8_0_contract().

◆ ck_q80_resolve_ggml_build_forward_expand()

static ck_q80_ggml_build_forward_expand_fn ck_q80_resolve_ggml_build_forward_expand ( void )

static

Definition at line 153 of file gemm_kernels_q8_0_q8_0_contract.c.

{
    static int tried = 0;
    static ck_q80_ggml_build_forward_expand_fn fn = NULL;
    if (!tried) {
        tried = 1;
        fn = (ck_q80_ggml_build_forward_expand_fn) dlsym(RTLD_DEFAULT, "ggml_build_forward_expand");
    }
    return fn;
}

References RTLD_DEFAULT.

Referenced by gemm_nt_q8_0_q8_0_ggml_strict().

◆ ck_q80_resolve_ggml_cpu_init()

static ck_q80_ggml_cpu_init_fn ck_q80_resolve_ggml_cpu_init ( void )

static

Definition at line 175 of file gemm_kernels_q8_0_q8_0_contract.c.

{
    static int tried = 0;
    static ck_q80_ggml_cpu_init_fn fn = NULL;
    if (!tried) {
        tried = 1;
        fn = (ck_q80_ggml_cpu_init_fn) dlsym(RTLD_DEFAULT, "ggml_cpu_init");
    }
    return fn;
}

References RTLD_DEFAULT.

Referenced by gemm_nt_q8_0_q8_0_ggml_strict().

◆ ck_q80_resolve_ggml_free()

static ck_q80_ggml_free_fn ck_q80_resolve_ggml_free ( void )

static

Definition at line 109 of file gemm_kernels_q8_0_q8_0_contract.c.

{
    static int tried = 0;
    static ck_q80_ggml_free_fn fn = NULL;
    if (!tried) {
        tried = 1;
        fn = (ck_q80_ggml_free_fn) dlsym(RTLD_DEFAULT, "ggml_free");
    }
    return fn;
}

References RTLD_DEFAULT.

Referenced by gemm_nt_q8_0_q8_0_ggml_strict().

◆ ck_q80_resolve_ggml_get_data()

static ck_q80_ggml_get_data_fn ck_q80_resolve_ggml_get_data ( void )

static

Definition at line 186 of file gemm_kernels_q8_0_q8_0_contract.c.

{
    static int tried = 0;
    static ck_q80_ggml_get_data_fn fn = NULL;
    if (!tried) {
        tried = 1;
        fn = (ck_q80_ggml_get_data_fn) dlsym(RTLD_DEFAULT, "ggml_get_data");
    }
    return fn;
}

References RTLD_DEFAULT.

Referenced by gemm_nt_q8_0_q8_0_ggml_strict().

◆ ck_q80_resolve_ggml_get_data_f32()

static ck_q80_ggml_get_data_f32_fn ck_q80_resolve_ggml_get_data_f32 ( void )

static

Definition at line 197 of file gemm_kernels_q8_0_q8_0_contract.c.

{
    static int tried = 0;
    static ck_q80_ggml_get_data_f32_fn fn = NULL;
    if (!tried) {
        tried = 1;
        fn = (ck_q80_ggml_get_data_f32_fn) dlsym(RTLD_DEFAULT, "ggml_get_data_f32");
    }
    return fn;
}

References RTLD_DEFAULT.

Referenced by gemm_nt_q8_0_q8_0_ggml_strict().

◆ ck_q80_resolve_ggml_graph_compute_with_ctx()

static ck_q80_ggml_graph_compute_with_ctx_fn ck_q80_resolve_ggml_graph_compute_with_ctx ( void )

static

Definition at line 164 of file gemm_kernels_q8_0_q8_0_contract.c.

{
    static int tried = 0;
    static ck_q80_ggml_graph_compute_with_ctx_fn fn = NULL;
    if (!tried) {
        tried = 1;
        fn = (ck_q80_ggml_graph_compute_with_ctx_fn) dlsym(RTLD_DEFAULT, "ggml_graph_compute_with_ctx");
    }
    return fn;
}

References RTLD_DEFAULT.

Referenced by gemm_nt_q8_0_q8_0_ggml_strict().

◆ ck_q80_resolve_ggml_init()

static ck_q80_ggml_init_fn ck_q80_resolve_ggml_init ( void )

static

Definition at line 98 of file gemm_kernels_q8_0_q8_0_contract.c.

{
    static int tried = 0;
    static ck_q80_ggml_init_fn fn = NULL;
    if (!tried) {
        tried = 1;
        fn = (ck_q80_ggml_init_fn) dlsym(RTLD_DEFAULT, "ggml_init");
    }
    return fn;
}

References RTLD_DEFAULT.

Referenced by gemm_nt_q8_0_q8_0_ggml_strict().

◆ ck_q80_resolve_ggml_mul_mat()

static ck_q80_ggml_mul_mat_fn ck_q80_resolve_ggml_mul_mat ( void )

static

Definition at line 131 of file gemm_kernels_q8_0_q8_0_contract.c.

{
    static int tried = 0;
    static ck_q80_ggml_mul_mat_fn fn = NULL;
    if (!tried) {
        tried = 1;
        fn = (ck_q80_ggml_mul_mat_fn) dlsym(RTLD_DEFAULT, "ggml_mul_mat");
    }
    return fn;
}

References RTLD_DEFAULT.

Referenced by gemm_nt_q8_0_q8_0_ggml_strict().

◆ ck_q80_resolve_ggml_nbytes()

static ck_q80_ggml_nbytes_fn ck_q80_resolve_ggml_nbytes ( void )

static

Definition at line 208 of file gemm_kernels_q8_0_q8_0_contract.c.

{
    static int tried = 0;
    static ck_q80_ggml_nbytes_fn fn = NULL;
    if (!tried) {
        tried = 1;
        fn = (ck_q80_ggml_nbytes_fn) dlsym(RTLD_DEFAULT, "ggml_nbytes");
    }
    return fn;
}

References RTLD_DEFAULT.

Referenced by gemm_nt_q8_0_q8_0_ggml_strict().

◆ ck_q80_resolve_ggml_new_graph()

static ck_q80_ggml_new_graph_fn ck_q80_resolve_ggml_new_graph ( void )

static

Definition at line 142 of file gemm_kernels_q8_0_q8_0_contract.c.

{
    static int tried = 0;
    static ck_q80_ggml_new_graph_fn fn = NULL;
    if (!tried) {
        tried = 1;
        fn = (ck_q80_ggml_new_graph_fn) dlsym(RTLD_DEFAULT, "ggml_new_graph");
    }
    return fn;
}

References RTLD_DEFAULT.

Referenced by gemm_nt_q8_0_q8_0_ggml_strict().

◆ ck_q80_resolve_ggml_new_tensor_2d()

static ck_q80_ggml_new_tensor_2d_fn ck_q80_resolve_ggml_new_tensor_2d ( void )

static

Definition at line 120 of file gemm_kernels_q8_0_q8_0_contract.c.

{
    static int tried = 0;
    static ck_q80_ggml_new_tensor_2d_fn fn = NULL;
    if (!tried) {
        tried = 1;
        fn = (ck_q80_ggml_new_tensor_2d_fn) dlsym(RTLD_DEFAULT, "ggml_new_tensor_2d");
    }
    return fn;
}

References RTLD_DEFAULT.

Referenced by gemm_nt_q8_0_q8_0_ggml_strict().

◆ gemm_nt_q8_0_q8_0_contract()

void gemm_nt_q8_0_q8_0_contract	(	const float *	A,
		const void *	B,
		const float *	bias,
		float *	C,
		int	M,
		int	N,
		int	K
	)

Definition at line 447 of file gemm_kernels_q8_0_q8_0_contract.c.

{
    if (!A || !B || !C || M <= 0 || N <= 0 || K <= 0) {
        return;
    }
 
    const float *A_use = A;
    const int strict = ck_strict_parity_enabled();
    const int dump_enabled = strict && ck_q80_contract_dump_enabled();
    int strict_cached_layer = -1;
    int strict_dump_layer = -1;
    if (ck_strict_parity_enabled() && ck_q80_contract_cached_input_enabled()) {
        const float *cached = ck_strict_consume_next_gemm_a((size_t) M * (size_t) K);
        if (cached) {
            A_use = cached;
            strict_cached_layer = ck_q80_contract_cached_gemm_seq++;
        }
    }
    if (dump_enabled) {
        strict_dump_layer = strict_cached_layer >= 0
            ? strict_cached_layer
            : ck_q80_contract_cached_gemm_seq++;
    }
 
    if (dump_enabled && strict_dump_layer >= 0) {
        ck_q80_contract_dump_tensor(strict_cached_layer >= 0
                                        ? "strict_out_proj_input_cached"
                                        : "strict_out_proj_input_live",
                                    strict_dump_layer,
                                    A_use,
                                    (size_t) M * (size_t) K);
    }
 
    if (strict &&
        gemm_nt_q8_0_q8_0_ggml_strict(A_use, B, bias, C, M, N, K)) {
        if (dump_enabled && strict_dump_layer >= 0) {
            ck_q80_contract_dump_tensor("strict_out_proj_output_ggml",
                                        strict_dump_layer,
                                        C,
                                        (size_t) M * (size_t) N);
        }
        return;
    }
 
    if (!strict) {
#pragma omp parallel for schedule(static) if(M > 1)
        for (int m = 0; m < M; ++m) {
            gemv_q8_0_q8_0_contract(&C[m * N], B, &A_use[m * K], N, K);
            if (bias) {
                for (int n = 0; n < N; ++n) {
                    C[m * N + n] += bias[n];
                }
            }
        }
        return;
    }
 
    for (int m = 0; m < M; ++m) {
        gemv_q8_0_q8_0_contract(&C[m * N], B, &A_use[m * K], N, K);
        if (bias) {
            for (int n = 0; n < N; ++n) {
                C[m * N + n] += bias[n];
            }
        }
    }
 
    if (dump_enabled && strict_dump_layer >= 0) {
        ck_q80_contract_dump_tensor("strict_out_proj_output_fallback",
                                    strict_dump_layer,
                                    C,
                                    (size_t) M * (size_t) N);
    }
}

References C, ck_q80_contract_cached_gemm_seq, ck_q80_contract_cached_input_enabled(), ck_q80_contract_dump_enabled(), ck_q80_contract_dump_tensor(), ck_strict_consume_next_gemm_a(), ck_strict_parity_enabled(), gemm_nt_q8_0_q8_0_ggml_strict(), and gemv_q8_0_q8_0_contract().

◆ gemm_nt_q8_0_q8_0_contract_m2()

void gemm_nt_q8_0_q8_0_contract_m2	(	const float *	A,
		const void *	B,
		const float *	bias,
		float *	C,
		int	M,
		int	N,
		int	K
	)

Definition at line 418 of file gemm_kernels_q8_0_q8_0_contract.c.

{
    if (!A || !B || !C || M != 2 || N <= 0 || K <= 0 ||
        (K % QK8_0) != 0 || K / QK8_0 > CK_Q80_STACK_Q8_BLOCKS) return;
    const int nb = K / QK8_0;
    block_q8_0 x[2 * CK_Q80_STACK_Q8_BLOCKS];
    if (ck_strict_parity_enabled()) {
        quantize_row_q8_0_ref_local(A, x, K);
        quantize_row_q8_0_ref_local(A + K, x + nb, K);
    } else {
        quantize_row_q8_0(A, x, K);
        quantize_row_q8_0(A + K, x + nb, K);
    }
    if (ck_strict_parity_enabled()) {
        gemv_q8_0_q8_0_ref_rows(C, B, x, N, K);
        gemv_q8_0_q8_0_ref_rows(C + N, B, x + nb, N, K);
    } else {
        gemm_q8_0_q8_0_m2n4_strided(C, N, B, x, 2, N, K);
    }
    if (bias) {
        for (int n = 0; n < N; ++n) {
            C[n] += bias[n];
            C[(size_t)N + n] += bias[n];
        }
    }
}

References C, CK_Q80_STACK_Q8_BLOCKS, ck_strict_parity_enabled(), gemm_q8_0_q8_0_m2n4_strided(), gemv_q8_0_q8_0_ref_rows(), QK8_0, quantize_row_q8_0(), and quantize_row_q8_0_ref_local().

◆ gemm_nt_q8_0_q8_0_ggml_strict()

static int gemm_nt_q8_0_q8_0_ggml_strict	(	const float *	A,
		const void *	B,
		const float *	bias,
		float *	C,
		int	M,
		int	N,
		int	K
	)

static

Definition at line 219 of file gemm_kernels_q8_0_q8_0_contract.c.

{
    ck_q80_ggml_cpu_init_fn ggml_cpu_init_fn = ck_q80_resolve_ggml_cpu_init();
    ck_q80_ggml_init_fn ggml_init_fn = ck_q80_resolve_ggml_init();
    ck_q80_ggml_free_fn ggml_free_fn = ck_q80_resolve_ggml_free();
    ck_q80_ggml_new_tensor_2d_fn ggml_new_tensor_2d_fn = ck_q80_resolve_ggml_new_tensor_2d();
    ck_q80_ggml_mul_mat_fn ggml_mul_mat_fn = ck_q80_resolve_ggml_mul_mat();
    ck_q80_ggml_new_graph_fn ggml_new_graph_fn = ck_q80_resolve_ggml_new_graph();
    ck_q80_ggml_build_forward_expand_fn ggml_build_forward_expand_fn = ck_q80_resolve_ggml_build_forward_expand();
    ck_q80_ggml_graph_compute_with_ctx_fn ggml_graph_compute_with_ctx_fn = ck_q80_resolve_ggml_graph_compute_with_ctx();
    ck_q80_ggml_get_data_fn ggml_get_data_fn = ck_q80_resolve_ggml_get_data();
    ck_q80_ggml_get_data_f32_fn ggml_get_data_f32_fn = ck_q80_resolve_ggml_get_data_f32();
    ck_q80_ggml_nbytes_fn ggml_nbytes_fn = ck_q80_resolve_ggml_nbytes();
 
    if (!ggml_cpu_init_fn || !ggml_init_fn || !ggml_free_fn || !ggml_new_tensor_2d_fn ||
        !ggml_mul_mat_fn || !ggml_new_graph_fn || !ggml_build_forward_expand_fn ||
        !ggml_graph_compute_with_ctx_fn || !ggml_get_data_fn || !ggml_get_data_f32_fn ||
        !ggml_nbytes_fn) {
        return 0;
    }
 
    ggml_cpu_init_fn();
 
    const size_t output_bytes = (size_t) M * (size_t) N * sizeof(float);
    const size_t mem_size = ((size_t) 128 * 1024 * 1024) + output_bytes;
 
    struct ggml_init_params params = {
        .mem_size = mem_size,
        .mem_buffer = NULL,
        .no_alloc = false,
    };
    struct ggml_context *ctx = ggml_init_fn(params);
    if (!ctx) {
        return 0;
    }
 
    int ok = 0;
    struct ggml_tensor *w = ggml_new_tensor_2d_fn(ctx, GGML_TYPE_Q8_0, K, N);
    struct ggml_tensor *x = ggml_new_tensor_2d_fn(ctx, GGML_TYPE_F32, K, M);
    if (!w || !x) {
        ggml_free_fn(ctx);
        return 0;
    }
 
    void *w_data = ggml_get_data_fn(w);
    void *x_data = ggml_get_data_fn(x);
    const size_t w_nbytes = ggml_nbytes_fn(w);
    if (!w_data || !x_data || w_nbytes == 0) {
        ggml_free_fn(ctx);
        return 0;
    }
 
    memcpy(w_data, B, w_nbytes);
    memcpy(x_data, A, (size_t) M * (size_t) K * sizeof(float));
 
    struct ggml_tensor *y = ggml_mul_mat_fn(ctx, w, x);
    if (!y) {
        ggml_free_fn(ctx);
        return 0;
    }
 
    struct ggml_cgraph *gf = ggml_new_graph_fn(ctx);
    if (!gf) {
        ggml_free_fn(ctx);
        return 0;
    }
    ggml_build_forward_expand_fn(gf, y);
    if (ggml_graph_compute_with_ctx_fn(ctx, gf, 1) != GGML_STATUS_SUCCESS) {
        ggml_free_fn(ctx);
        return 0;
    }
 
    {
        const float *src = ggml_get_data_f32_fn(y);
        if (!src) {
            ggml_free_fn(ctx);
            return 0;
        }
        for (int m = 0; m < M; ++m) {
            memcpy(C + (size_t) m * (size_t) N,
                   src + (size_t) m * (size_t) N,
                   (size_t) N * sizeof(float));
            if (bias) {
                for (int n = 0; n < N; ++n) {
                    C[(size_t) m * (size_t) N + (size_t) n] += bias[n];
                }
            }
        }
    }
 
    ok = 1;
    ggml_free_fn(ctx);
    return ok;
}

References C, ck_q80_resolve_ggml_build_forward_expand(), ck_q80_resolve_ggml_cpu_init(), ck_q80_resolve_ggml_free(), ck_q80_resolve_ggml_get_data(), ck_q80_resolve_ggml_get_data_f32(), ck_q80_resolve_ggml_graph_compute_with_ctx(), ck_q80_resolve_ggml_init(), ck_q80_resolve_ggml_mul_mat(), ck_q80_resolve_ggml_nbytes(), ck_q80_resolve_ggml_new_graph(), ck_q80_resolve_ggml_new_tensor_2d(), GGML_STATUS_SUCCESS, GGML_TYPE_F32, GGML_TYPE_Q8_0, and ggml_init_params::mem_size.

Referenced by gemm_nt_q8_0_q8_0_contract().

◆ gemm_q8_0_q8_0_m2n4_strided()

void gemm_q8_0_q8_0_m2n4_strided	(	float *	C,
		int	ldc,
		const void *	W,
		const void *	A_q8,
		int	M,
		int	N,
		int	K
	)

Definition at line 1502 of file gemm_kernels_q8_0.c.

{
#if defined(__AVX2__) || defined(__AVX512F__)
    if (ck_q8_0_q8_0_debug_ref() || (K % QK8_0) != 0) {
        const int nb = K / QK8_0;
        const block_q8_0 *a = (const block_q8_0 *)A_q8;
        for (int m = 0; m < M; ++m) {
            gemv_q8_0_q8_0(C + (size_t)m * (size_t)ldc, W,
                           a + (size_t)m * (size_t)nb, N, K);
        }
        return;
    }
 
    const block_q8_0 *w = (const block_q8_0 *)W;
    const block_q8_0 *a = (const block_q8_0 *)A_q8;
    const int nb = K / QK8_0;
    int m = 0;
    for (; m + 1 < M; m += 2) {
        const block_q8_0 *a0 = a + (size_t)(m + 0) * (size_t)nb;
        const block_q8_0 *a1 = a + (size_t)(m + 1) * (size_t)nb;
        int n = 0;
        for (; n + 3 < N; n += 4) {
            const block_q8_0 *w0 = w + (size_t)(n + 0) * (size_t)nb;
            const block_q8_0 *w1 = w + (size_t)(n + 1) * (size_t)nb;
            const block_q8_0 *w2 = w + (size_t)(n + 2) * (size_t)nb;
            const block_q8_0 *w3 = w + (size_t)(n + 3) * (size_t)nb;
            __m256 acc00 = _mm256_setzero_ps();
            __m256 acc01 = _mm256_setzero_ps();
            __m256 acc02 = _mm256_setzero_ps();
            __m256 acc03 = _mm256_setzero_ps();
            __m256 acc10 = _mm256_setzero_ps();
            __m256 acc11 = _mm256_setzero_ps();
            __m256 acc12 = _mm256_setzero_ps();
            __m256 acc13 = _mm256_setzero_ps();
 
            for (int ib = 0; ib < nb; ++ib) {
                const __m256i qa0 = _mm256_loadu_si256((const __m256i *)a0[ib].qs);
                const __m256i qa1 = _mm256_loadu_si256((const __m256i *)a1[ib].qs);
                const __m256i qw0 = _mm256_loadu_si256((const __m256i *)w0[ib].qs);
                const __m256i qw1 = _mm256_loadu_si256((const __m256i *)w1[ib].qs);
                const __m256i qw2 = _mm256_loadu_si256((const __m256i *)w2[ib].qs);
                const __m256i qw3 = _mm256_loadu_si256((const __m256i *)w3[ib].qs);
                const float da0 = CK_FP16_TO_FP32(a0[ib].d);
                const float da1 = CK_FP16_TO_FP32(a1[ib].d);
                const float dw0 = CK_FP16_TO_FP32(w0[ib].d);
                const float dw1 = CK_FP16_TO_FP32(w1[ib].d);
                const float dw2 = CK_FP16_TO_FP32(w2[ib].d);
                const float dw3 = CK_FP16_TO_FP32(w3[ib].d);
                const __m256 p00 = mul_sum_i8_pairs_float_q8_0_avx2(qw0, qa0);
                const __m256 p01 = mul_sum_i8_pairs_float_q8_0_avx2(qw1, qa0);
                const __m256 p02 = mul_sum_i8_pairs_float_q8_0_avx2(qw2, qa0);
                const __m256 p03 = mul_sum_i8_pairs_float_q8_0_avx2(qw3, qa0);
                const __m256 p10 = mul_sum_i8_pairs_float_q8_0_avx2(qw0, qa1);
                const __m256 p11 = mul_sum_i8_pairs_float_q8_0_avx2(qw1, qa1);
                const __m256 p12 = mul_sum_i8_pairs_float_q8_0_avx2(qw2, qa1);
                const __m256 p13 = mul_sum_i8_pairs_float_q8_0_avx2(qw3, qa1);
#if defined(__FMA__)
                acc00 = _mm256_fmadd_ps(_mm256_set1_ps(dw0 * da0), p00, acc00);
                acc01 = _mm256_fmadd_ps(_mm256_set1_ps(dw1 * da0), p01, acc01);
                acc02 = _mm256_fmadd_ps(_mm256_set1_ps(dw2 * da0), p02, acc02);
                acc03 = _mm256_fmadd_ps(_mm256_set1_ps(dw3 * da0), p03, acc03);
                acc10 = _mm256_fmadd_ps(_mm256_set1_ps(dw0 * da1), p10, acc10);
                acc11 = _mm256_fmadd_ps(_mm256_set1_ps(dw1 * da1), p11, acc11);
                acc12 = _mm256_fmadd_ps(_mm256_set1_ps(dw2 * da1), p12, acc12);
                acc13 = _mm256_fmadd_ps(_mm256_set1_ps(dw3 * da1), p13, acc13);
#else
                acc00 = _mm256_add_ps(_mm256_mul_ps(_mm256_set1_ps(dw0 * da0), p00), acc00);
                acc01 = _mm256_add_ps(_mm256_mul_ps(_mm256_set1_ps(dw1 * da0), p01), acc01);
                acc02 = _mm256_add_ps(_mm256_mul_ps(_mm256_set1_ps(dw2 * da0), p02), acc02);
                acc03 = _mm256_add_ps(_mm256_mul_ps(_mm256_set1_ps(dw3 * da0), p03), acc03);
                acc10 = _mm256_add_ps(_mm256_mul_ps(_mm256_set1_ps(dw0 * da1), p10), acc10);
                acc11 = _mm256_add_ps(_mm256_mul_ps(_mm256_set1_ps(dw1 * da1), p11), acc11);
                acc12 = _mm256_add_ps(_mm256_mul_ps(_mm256_set1_ps(dw2 * da1), p12), acc12);
                acc13 = _mm256_add_ps(_mm256_mul_ps(_mm256_set1_ps(dw3 * da1), p13), acc13);
#endif
            }
 
            C[(size_t)(m + 0) * (size_t)ldc + (n + 0)] = hsum_float_8_q8_0(acc00);
            C[(size_t)(m + 0) * (size_t)ldc + (n + 1)] = hsum_float_8_q8_0(acc01);
            C[(size_t)(m + 0) * (size_t)ldc + (n + 2)] = hsum_float_8_q8_0(acc02);
            C[(size_t)(m + 0) * (size_t)ldc + (n + 3)] = hsum_float_8_q8_0(acc03);
            C[(size_t)(m + 1) * (size_t)ldc + (n + 0)] = hsum_float_8_q8_0(acc10);
            C[(size_t)(m + 1) * (size_t)ldc + (n + 1)] = hsum_float_8_q8_0(acc11);
            C[(size_t)(m + 1) * (size_t)ldc + (n + 2)] = hsum_float_8_q8_0(acc12);
            C[(size_t)(m + 1) * (size_t)ldc + (n + 3)] = hsum_float_8_q8_0(acc13);
        }
        if (n < N) {
            gemv_q8_0_q8_0(C + (size_t)(m + 0) * (size_t)ldc + n,
                           w + (size_t)n * (size_t)nb, a0, N - n, K);
            gemv_q8_0_q8_0(C + (size_t)(m + 1) * (size_t)ldc + n,
                           w + (size_t)n * (size_t)nb, a1, N - n, K);
        }
    }
    if (m < M) {
        gemv_q8_0_q8_0_x4(C + (size_t)m * (size_t)ldc, W,
                          a + (size_t)m * (size_t)nb, N, K);
    }
#else
    const int nb = K / QK8_0;
    const block_q8_0 *a = (const block_q8_0 *)A_q8;
    for (int m = 0; m < M; ++m) {
        gemv_q8_0_q8_0(C + (size_t)m * (size_t)ldc, W,
                       a + (size_t)m * (size_t)nb, N, K);
    }
#endif
}

Referenced by gemm_nt_q8_0_q8_0_contract_m2(), and gemm_q8_0_q8_0_m2n4().

◆ gemv_q8_0_q8_0_contract()

void gemv_q8_0_q8_0_contract	(	float *	y,
		const void *	W,
		const float *	x,
		int	M,
		int	K
	)

Definition at line 385 of file gemm_kernels_q8_0_q8_0_contract.c.

{
    if (!y || !W || !x || M <= 0 || K <= 0) {
        return;
    }
 
    if ((K % QK8_0) != 0) {
        gemv_q8_0(y, W, x, M, K);
        return;
    }
 
    const int blocks_per_row = K / QK8_0;
    if (blocks_per_row > CK_Q80_STACK_Q8_BLOCKS) {
        gemv_q8_0(y, W, x, M, K);
        return;
    }
 
    block_q8_0 x_q8[CK_Q80_STACK_Q8_BLOCKS];
    if (ck_strict_parity_enabled()) {
        quantize_row_q8_0_ref_local(x, x_q8, K);
        gemv_q8_0_q8_0_ref_rows(y, W, x_q8, M, K);
        return;
    }
    quantize_row_q8_0(x, x_q8, K);
    gemv_q8_0_q8_0_x4(y, W, x_q8, M, K);
}

References CK_Q80_STACK_Q8_BLOCKS, ck_strict_parity_enabled(), gemv_q8_0(), gemv_q8_0_q8_0_ref_rows(), gemv_q8_0_q8_0_x4(), QK8_0, quantize_row_q8_0(), and quantize_row_q8_0_ref_local().

Referenced by gemm_nt_q8_0_q8_0_contract().

◆ gemv_q8_0_q8_0_ref_rows()

static void gemv_q8_0_q8_0_ref_rows	(	float *	y,
		const void *	W,
		const void *	x_q8,
		int	M,
		int	K
	)

static

Definition at line 366 of file gemm_kernels_q8_0_q8_0_contract.c.

{
    const block_q8_0 *w_blocks = (const block_q8_0 *)W;
    const int blocks_per_row = K / QK8_0;
 
    for (int row = 0; row < M; ++row) {
        vec_dot_q8_0_q8_0_ref(
            K,
            &y[row],
            &w_blocks[row * blocks_per_row],
            x_q8
        );
    }
}

References QK8_0, and vec_dot_q8_0_q8_0_ref().

Referenced by gemm_nt_q8_0_q8_0_contract_m2(), and gemv_q8_0_q8_0_contract().

◆ quantize_row_q8_0_ref_local()

static void quantize_row_q8_0_ref_local	(	const float *	x,
		block_q8_0 *	y,
		int	k
	)

static

Definition at line 332 of file gemm_kernels_q8_0_q8_0_contract.c.

{
    const int nb = k / QK8_0;
 
    for (int i = 0; i < nb; ++i) {
        float amax = 0.0f;
        for (int j = 0; j < QK8_0; ++j) {
            const float v = x[i * QK8_0 + j];
            const float av = fabsf(v);
            if (av > amax) {
                amax = av;
            }
        }
 
        const float d = amax / 127.0f;
        const float id = d != 0.0f ? 1.0f / d : 0.0f;
        y[i].d = CK_FP32_TO_FP16(d);
 
        for (int j = 0; j < QK8_0; ++j) {
            const float x0 = x[i * QK8_0 + j] * id;
            int q = ck_nearest_int_q8_0_ref(x0);
            if (q > 127) {
                q = 127;
            }
            if (q < -127) {
                q = -127;
            }
            y[i].qs[j] = (int8_t) q;
        }
    }
}

References CK_FP32_TO_FP16, ck_nearest_int_q8_0_ref(), block_q8_0::d, id, QK8_0, and block_q8_0::qs.

Referenced by gemm_nt_q8_0_q8_0_contract_m2(), and gemv_q8_0_q8_0_contract().

◆ vec_dot_q8_0_q8_0_ref()

void vec_dot_q8_0_q8_0_ref	(	int	n,
		float *	s,
		const void *	vx,
		const void *	vy
	)

Quantized dot product: Q8_0 weights x Q8_0 input (scalar reference)

Parameters

n	Number of elements (must be multiple of 32)
s	Output: scalar dot product result
vx	Q8_0 quantized weights
vy	Q8_0 quantized input

Definition at line 1140 of file gemm_kernels_q8_0.c.

{
    const int qk = QK8_0;  /* 32 */
    const int nb = n / qk;
 
    const block_q8_0 *x = (const block_q8_0 *)vx;
    const block_q8_0 *y = (const block_q8_0 *)vy;
 
    float sumf = 0.0f;
 
    for (int ib = 0; ib < nb; ib++) {
        int sumi = 0;
 
        for (int j = 0; j < qk; j++) {
            sumi += x[ib].qs[j] * y[ib].qs[j];
        }
 
        sumf += sumi * (CK_FP16_TO_FP32(x[ib].d) * CK_FP16_TO_FP32(y[ib].d));
    }
 
    *s = sumf;
}

References CK_FP16_TO_FP32, QK8_0, and block_q8_0::qs.

Referenced by gemv_q8_0_q8_0_ref_rows(), and vec_dot_q8_0_q8_0().

Variable Documentation

◆ ck_q80_contract_cached_gemm_seq

int ck_q80_contract_cached_gemm_seq = 0

static

Definition at line 38 of file gemm_kernels_q8_0_q8_0_contract.c.

Referenced by gemm_nt_q8_0_q8_0_contract().

◆ ck_q80_contract_magic

const char ck_q80_contract_magic[8] = {'C', 'K', 'D', 'M', 'P', '\0', '\0', '\0'}

static

Definition at line 36 of file gemm_kernels_q8_0_q8_0_contract.c.

36{'C', 'K', 'D', 'M', 'P', '\0', '\0', '\0'};

Referenced by ck_q80_contract_dump_tensor().

◆ ck_q80_contract_version

const uint32_t ck_q80_contract_version = 1u

static

Definition at line 37 of file gemm_kernels_q8_0_q8_0_contract.c.

Referenced by ck_q80_contract_dump_tensor().

Macros

Typedefs

Functions

Variables

Detailed Description

Macro Definition Documentation

◆ _GNU_SOURCE

◆ CK_Q80_STACK_Q8_BLOCKS

Typedef Documentation

◆ ck_q80_ggml_build_forward_expand_fn

◆ ck_q80_ggml_cpu_init_fn

◆ ck_q80_ggml_free_fn

◆ ck_q80_ggml_get_data_f32_fn

◆ ck_q80_ggml_get_data_fn

◆ ck_q80_ggml_graph_compute_with_ctx_fn

◆ ck_q80_ggml_init_fn

◆ ck_q80_ggml_mul_mat_fn

◆ ck_q80_ggml_nbytes_fn

◆ ck_q80_ggml_new_graph_fn

◆ ck_q80_ggml_new_tensor_2d_fn

Function Documentation

◆ ck_nearest_int_q8_0_ref()

◆ ck_q80_contract_cached_input_enabled()

◆ ck_q80_contract_dump_enabled()

◆ ck_q80_contract_dump_tensor()

◆ ck_q80_resolve_ggml_build_forward_expand()

◆ ck_q80_resolve_ggml_cpu_init()

◆ ck_q80_resolve_ggml_free()

◆ ck_q80_resolve_ggml_get_data()

◆ ck_q80_resolve_ggml_get_data_f32()

◆ ck_q80_resolve_ggml_graph_compute_with_ctx()

◆ ck_q80_resolve_ggml_init()

◆ ck_q80_resolve_ggml_mul_mat()

◆ ck_q80_resolve_ggml_nbytes()

◆ ck_q80_resolve_ggml_new_graph()

◆ ck_q80_resolve_ggml_new_tensor_2d()

◆ gemm_nt_q8_0_q8_0_contract()

◆ gemm_nt_q8_0_q8_0_contract_m2()

◆ gemm_nt_q8_0_q8_0_ggml_strict()

◆ gemm_q8_0_q8_0_m2n4_strided()

◆ gemv_q8_0_q8_0_contract()

◆ gemv_q8_0_q8_0_ref_rows()

◆ quantize_row_q8_0_ref_local()

◆ vec_dot_q8_0_q8_0_ref()

Variable Documentation

◆ ck_q80_contract_cached_gemm_seq

◆ ck_q80_contract_magic

◆ ck_q80_contract_version