gemm_kernels_q8_0_q8_0_contract.c File Reference

FP32 API adapters that enforce Q8_0 x Q8_0 activation contract. More...

#include "ckernel_engine.h"
#include "ckernel_quant.h"
#include "ggml_runtime_compat.h"
#include <dlfcn.h>
#include <stdint.h>
#include <math.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

Go to the source code of this file.

Macros
#define	_GNU_SOURCE
#define	CK_Q80_STACK_Q8_BLOCKS   1024

Typedefs
typedef void(*	ck_q80_ggml_build_forward_expand_fn) (struct ggml_cgraph *, struct ggml_tensor *)
typedef void(*	ck_q80_ggml_cpu_init_fn) (void)
typedef void(*	ck_q80_ggml_free_fn) (struct ggml_context *)
typedef float *(*	ck_q80_ggml_get_data_f32_fn) (const struct ggml_tensor *)
typedef void *(*	ck_q80_ggml_get_data_fn) (const struct ggml_tensor *)
typedef enum ggml_status(*	ck_q80_ggml_graph_compute_with_ctx_fn) (struct ggml_context *, struct ggml_cgraph *, int)
typedef struct ggml_context *(*	ck_q80_ggml_init_fn) (struct ggml_init_params)
typedef struct ggml_tensor *(*	ck_q80_ggml_mul_mat_fn) (struct ggml_context *, struct ggml_tensor *, struct ggml_tensor *)
typedef size_t(*	ck_q80_ggml_nbytes_fn) (const struct ggml_tensor *)
typedef struct ggml_cgraph *(*	ck_q80_ggml_new_graph_fn) (struct ggml_context *)
typedef struct ggml_tensor *(*	ck_q80_ggml_new_tensor_2d_fn) (struct ggml_context *, enum ggml_type, int64_t, int64_t)

Functions
static int	ck_nearest_int_q8_0_ref (float fval)
static int	ck_q80_contract_cached_input_enabled (void)
static int	ck_q80_contract_dump_enabled (void)
static void	ck_q80_contract_dump_tensor (const char *name, int layer_id, const float *data, size_t elem_count)
static ck_q80_ggml_build_forward_expand_fn	ck_q80_resolve_ggml_build_forward_expand (void)
static ck_q80_ggml_cpu_init_fn	ck_q80_resolve_ggml_cpu_init (void)
static ck_q80_ggml_free_fn	ck_q80_resolve_ggml_free (void)
static ck_q80_ggml_get_data_fn	ck_q80_resolve_ggml_get_data (void)
static ck_q80_ggml_get_data_f32_fn	ck_q80_resolve_ggml_get_data_f32 (void)
static ck_q80_ggml_graph_compute_with_ctx_fn	ck_q80_resolve_ggml_graph_compute_with_ctx (void)
static ck_q80_ggml_init_fn	ck_q80_resolve_ggml_init (void)
static ck_q80_ggml_mul_mat_fn	ck_q80_resolve_ggml_mul_mat (void)
static ck_q80_ggml_nbytes_fn	ck_q80_resolve_ggml_nbytes (void)
static ck_q80_ggml_new_graph_fn	ck_q80_resolve_ggml_new_graph (void)
static ck_q80_ggml_new_tensor_2d_fn	ck_q80_resolve_ggml_new_tensor_2d (void)
void	gemm_nt_q8_0_q8_0_contract (const float *A, const void *B, const float *bias, float *C, int M, int N, int K)
static int	gemm_nt_q8_0_q8_0_ggml_strict (const float *A, const void *B, const float *bias, float *C, int M, int N, int K)
void	gemv_q8_0_q8_0_contract (float *y, const void *W, const float *x, int M, int K)
static void	gemv_q8_0_q8_0_ref_rows (float *y, const void *W, const void *x_q8, int M, int K)
static void	quantize_row_q8_0_ref_local (const float *x, block_q8_0 *y, int k)
void	vec_dot_q8_0_q8_0_ref (int n, float *s, const void *vx, const void *vy)
	Quantized dot product: Q8_0 weights x Q8_0 input (scalar reference)

Variables
static int	ck_q80_contract_cached_gemm_seq = 0
static const char	ck_q80_contract_magic [8] = {'C', 'K', 'D', 'M', 'P', '\0', '\0', '\0'}
static const uint32_t	ck_q80_contract_version = 1u

Detailed Description

FP32 API adapters that enforce Q8_0 x Q8_0 activation contract.

Definition in file gemm_kernels_q8_0_q8_0_contract.c.

Macro Definition Documentation

_GNU_SOURCE

#define _GNU_SOURCE

Definition at line 7 of file gemm_kernels_q8_0_q8_0_contract.c.

CK_Q80_STACK_Q8_BLOCKS

#define CK_Q80_STACK_Q8_BLOCKS   1024

Definition at line 21 of file gemm_kernels_q8_0_q8_0_contract.c.

Typedef Documentation

ck_q80_ggml_build_forward_expand_fn

typedef void(* ck_q80_ggml_build_forward_expand_fn) (struct ggml_cgraph *, struct ggml_tensor *)

Definition at line 91 of file gemm_kernels_q8_0_q8_0_contract.c.

ck_q80_ggml_cpu_init_fn

typedef void(* ck_q80_ggml_cpu_init_fn) (void)

Definition at line 93 of file gemm_kernels_q8_0_q8_0_contract.c.

ck_q80_ggml_free_fn

typedef void(* ck_q80_ggml_free_fn) (struct ggml_context *)

Definition at line 87 of file gemm_kernels_q8_0_q8_0_contract.c.

ck_q80_ggml_get_data_f32_fn

typedef float *(* ck_q80_ggml_get_data_f32_fn) (const struct ggml_tensor *)

Definition at line 95 of file gemm_kernels_q8_0_q8_0_contract.c.

ck_q80_ggml_get_data_fn

typedef void *(* ck_q80_ggml_get_data_fn) (const struct ggml_tensor *)

Definition at line 94 of file gemm_kernels_q8_0_q8_0_contract.c.

ck_q80_ggml_graph_compute_with_ctx_fn

typedef enum ggml_status(* ck_q80_ggml_graph_compute_with_ctx_fn) (struct ggml_context *, struct ggml_cgraph *, int)

Definition at line 91 of file gemm_kernels_q8_0_q8_0_contract.c.

ck_q80_ggml_init_fn

typedef struct ggml_context *(* ck_q80_ggml_init_fn) (struct ggml_init_params)

Definition at line 52 of file gemm_kernels_q8_0_q8_0_contract.c.

ck_q80_ggml_mul_mat_fn

typedef struct ggml_tensor *(* ck_q80_ggml_mul_mat_fn) (struct ggml_context *, struct ggml_tensor *, struct ggml_tensor *)

Definition at line 87 of file gemm_kernels_q8_0_q8_0_contract.c.

ck_q80_ggml_nbytes_fn

typedef size_t(* ck_q80_ggml_nbytes_fn) (const struct ggml_tensor *)

Definition at line 96 of file gemm_kernels_q8_0_q8_0_contract.c.

ck_q80_ggml_new_graph_fn

typedef struct ggml_cgraph *(* ck_q80_ggml_new_graph_fn) (struct ggml_context *)

Definition at line 87 of file gemm_kernels_q8_0_q8_0_contract.c.

ck_q80_ggml_new_tensor_2d_fn

typedef struct ggml_tensor *(* ck_q80_ggml_new_tensor_2d_fn) (struct ggml_context *, enum ggml_type, int64_t, int64_t)

Definition at line 87 of file gemm_kernels_q8_0_q8_0_contract.c.

Function Documentation

ck_nearest_int_q8_0_ref()

static int ck_nearest_int_q8_0_ref ( float  fval )

inlinestatic

Definition at line 322 of file gemm_kernels_q8_0_q8_0_contract.c.

{
    float val = fval + 12582912.f;
    int i;
    memcpy(&i, &val, sizeof(int));
    return (i & 0x007fffff) - 0x00400000;
}

Referenced by quantize_row_q8_0_ref_local().

ck_q80_contract_cached_input_enabled()

static int ck_q80_contract_cached_input_enabled ( void  )

static

Definition at line 46 of file gemm_kernels_q8_0_q8_0_contract.c.

{
    const char *v = getenv("CK_STRICT_GEMM_USE_CACHED_A");
    return v && v[0] && strcmp(v, "0") != 0;
}

Referenced by gemm_nt_q8_0_q8_0_contract().

ck_q80_contract_dump_enabled()

static int ck_q80_contract_dump_enabled ( void  )

static

Definition at line 40 of file gemm_kernels_q8_0_q8_0_contract.c.

{
    const char *v = getenv("CK_STRICT_GEMM_DUMP");
    return v && v[0] && strcmp(v, "0") != 0;
}

Referenced by gemm_nt_q8_0_q8_0_contract().

ck_q80_contract_dump_tensor()

static void ck_q80_contract_dump_tensor ( const char *  name, int  layer_id, const float *  data, size_t  elem_count  )

static

Definition at line 52 of file gemm_kernels_q8_0_q8_0_contract.c.

{
    const char *dir = getenv("CK_PARITY_DIR");
    if (!dir || !dir[0] || !data || elem_count == 0 || !name || !name[0]) {
        return;
    }
 
    char path[4096];
    snprintf(path, sizeof(path), "%s/%s", dir, "strict_internal.bin");
    FILE *f = fopen(path, "ab");
    if (!f) {
        return;
    }
 
    ck_q80_contract_dump_header_t h;
    memset(&h, 0, sizeof(h));
    memcpy(h.magic, ck_q80_contract_magic, sizeof(ck_q80_contract_magic));
    h.version = ck_q80_contract_version;
    h.layer_id = layer_id;
    strncpy(h.op_name, name, sizeof(h.op_name) - 1);
    h.dtype = 0u;
    h.rank = 1u;
    h.shape[0] = (int64_t) elem_count;
    h.elem_count = (uint32_t) elem_count;
    h.token_id = 0;
 
    fwrite(&h, sizeof(h), 1, f);
    fwrite(data, sizeof(float), elem_count, f);
    fclose(f);
}

References ck_q80_contract_magic, and ck_q80_contract_version.

Referenced by gemm_nt_q8_0_q8_0_contract().

ck_q80_resolve_ggml_build_forward_expand()

static ck_q80_ggml_build_forward_expand_fn ck_q80_resolve_ggml_build_forward_expand ( void  )

static

Definition at line 153 of file gemm_kernels_q8_0_q8_0_contract.c.

{
    static int tried = 0;
    static ck_q80_ggml_build_forward_expand_fn fn = NULL;
    if (!tried) {
        tried = 1;
        fn = (ck_q80_ggml_build_forward_expand_fn) dlsym(RTLD_DEFAULT, "ggml_build_forward_expand");
    }
    return fn;
}

References RTLD_DEFAULT.

Referenced by gemm_nt_q8_0_q8_0_ggml_strict().

ck_q80_resolve_ggml_cpu_init()

static ck_q80_ggml_cpu_init_fn ck_q80_resolve_ggml_cpu_init ( void  )

static

Definition at line 175 of file gemm_kernels_q8_0_q8_0_contract.c.

{
    static int tried = 0;
    static ck_q80_ggml_cpu_init_fn fn = NULL;
    if (!tried) {
        tried = 1;
        fn = (ck_q80_ggml_cpu_init_fn) dlsym(RTLD_DEFAULT, "ggml_cpu_init");
    }
    return fn;
}

References RTLD_DEFAULT.

Referenced by gemm_nt_q8_0_q8_0_ggml_strict().

ck_q80_resolve_ggml_free()

static ck_q80_ggml_free_fn ck_q80_resolve_ggml_free ( void  )

static

Definition at line 109 of file gemm_kernels_q8_0_q8_0_contract.c.

{
    static int tried = 0;
    static ck_q80_ggml_free_fn fn = NULL;
    if (!tried) {
        tried = 1;
        fn = (ck_q80_ggml_free_fn) dlsym(RTLD_DEFAULT, "ggml_free");
    }
    return fn;
}

References RTLD_DEFAULT.

Referenced by gemm_nt_q8_0_q8_0_ggml_strict().

ck_q80_resolve_ggml_get_data()

static ck_q80_ggml_get_data_fn ck_q80_resolve_ggml_get_data ( void  )

static

Definition at line 186 of file gemm_kernels_q8_0_q8_0_contract.c.

{
    static int tried = 0;
    static ck_q80_ggml_get_data_fn fn = NULL;
    if (!tried) {
        tried = 1;
        fn = (ck_q80_ggml_get_data_fn) dlsym(RTLD_DEFAULT, "ggml_get_data");
    }
    return fn;
}

References RTLD_DEFAULT.

Referenced by gemm_nt_q8_0_q8_0_ggml_strict().

ck_q80_resolve_ggml_get_data_f32()

static ck_q80_ggml_get_data_f32_fn ck_q80_resolve_ggml_get_data_f32 ( void  )

static

Definition at line 197 of file gemm_kernels_q8_0_q8_0_contract.c.

{
    static int tried = 0;
    static ck_q80_ggml_get_data_f32_fn fn = NULL;
    if (!tried) {
        tried = 1;
        fn = (ck_q80_ggml_get_data_f32_fn) dlsym(RTLD_DEFAULT, "ggml_get_data_f32");
    }
    return fn;
}

References RTLD_DEFAULT.

Referenced by gemm_nt_q8_0_q8_0_ggml_strict().

ck_q80_resolve_ggml_graph_compute_with_ctx()

static ck_q80_ggml_graph_compute_with_ctx_fn ck_q80_resolve_ggml_graph_compute_with_ctx ( void  )

static

Definition at line 164 of file gemm_kernels_q8_0_q8_0_contract.c.

{
    static int tried = 0;
    static ck_q80_ggml_graph_compute_with_ctx_fn fn = NULL;
    if (!tried) {
        tried = 1;
        fn = (ck_q80_ggml_graph_compute_with_ctx_fn) dlsym(RTLD_DEFAULT, "ggml_graph_compute_with_ctx");
    }
    return fn;
}

References RTLD_DEFAULT.

Referenced by gemm_nt_q8_0_q8_0_ggml_strict().

ck_q80_resolve_ggml_init()

static ck_q80_ggml_init_fn ck_q80_resolve_ggml_init ( void  )

static

Definition at line 98 of file gemm_kernels_q8_0_q8_0_contract.c.

{
    static int tried = 0;
    static ck_q80_ggml_init_fn fn = NULL;
    if (!tried) {
        tried = 1;
        fn = (ck_q80_ggml_init_fn) dlsym(RTLD_DEFAULT, "ggml_init");
    }
    return fn;
}

References RTLD_DEFAULT.

Referenced by gemm_nt_q8_0_q8_0_ggml_strict().

ck_q80_resolve_ggml_mul_mat()

static ck_q80_ggml_mul_mat_fn ck_q80_resolve_ggml_mul_mat ( void  )

static

Definition at line 131 of file gemm_kernels_q8_0_q8_0_contract.c.

{
    static int tried = 0;
    static ck_q80_ggml_mul_mat_fn fn = NULL;
    if (!tried) {
        tried = 1;
        fn = (ck_q80_ggml_mul_mat_fn) dlsym(RTLD_DEFAULT, "ggml_mul_mat");
    }
    return fn;
}

References RTLD_DEFAULT.

Referenced by gemm_nt_q8_0_q8_0_ggml_strict().

ck_q80_resolve_ggml_nbytes()

static ck_q80_ggml_nbytes_fn ck_q80_resolve_ggml_nbytes ( void  )

static

Definition at line 208 of file gemm_kernels_q8_0_q8_0_contract.c.

{
    static int tried = 0;
    static ck_q80_ggml_nbytes_fn fn = NULL;
    if (!tried) {
        tried = 1;
        fn = (ck_q80_ggml_nbytes_fn) dlsym(RTLD_DEFAULT, "ggml_nbytes");
    }
    return fn;
}

References RTLD_DEFAULT.

Referenced by gemm_nt_q8_0_q8_0_ggml_strict().

ck_q80_resolve_ggml_new_graph()

static ck_q80_ggml_new_graph_fn ck_q80_resolve_ggml_new_graph ( void  )

static

Definition at line 142 of file gemm_kernels_q8_0_q8_0_contract.c.

{
    static int tried = 0;
    static ck_q80_ggml_new_graph_fn fn = NULL;
    if (!tried) {
        tried = 1;
        fn = (ck_q80_ggml_new_graph_fn) dlsym(RTLD_DEFAULT, "ggml_new_graph");
    }
    return fn;
}

References RTLD_DEFAULT.

Referenced by gemm_nt_q8_0_q8_0_ggml_strict().

ck_q80_resolve_ggml_new_tensor_2d()

static ck_q80_ggml_new_tensor_2d_fn ck_q80_resolve_ggml_new_tensor_2d ( void  )

static

Definition at line 120 of file gemm_kernels_q8_0_q8_0_contract.c.

{
    static int tried = 0;
    static ck_q80_ggml_new_tensor_2d_fn fn = NULL;
    if (!tried) {
        tried = 1;
        fn = (ck_q80_ggml_new_tensor_2d_fn) dlsym(RTLD_DEFAULT, "ggml_new_tensor_2d");
    }
    return fn;
}

References RTLD_DEFAULT.

Referenced by gemm_nt_q8_0_q8_0_ggml_strict().

gemm_nt_q8_0_q8_0_contract()

void gemm_nt_q8_0_q8_0_contract	(	const float *	A,
		const void *	B,
		const float *	bias,
		float *	C,
		int	M,
		int	N,
		int	K
	)

Definition at line 414 of file gemm_kernels_q8_0_q8_0_contract.c.

{
    if (!A || !B || !C || M <= 0 || N <= 0 || K <= 0) {
        return;
    }
 
    const float *A_use = A;
    const int strict = ck_strict_parity_enabled();
    const int dump_enabled = strict && ck_q80_contract_dump_enabled();
    int strict_cached_layer = -1;
    int strict_dump_layer = -1;
    if (ck_strict_parity_enabled() && ck_q80_contract_cached_input_enabled()) {
        const float *cached = ck_strict_consume_next_gemm_a((size_t) M * (size_t) K);
        if (cached) {
            A_use = cached;
            strict_cached_layer = ck_q80_contract_cached_gemm_seq++;
        }
    }
    if (dump_enabled) {
        strict_dump_layer = strict_cached_layer >= 0
            ? strict_cached_layer
            : ck_q80_contract_cached_gemm_seq++;
    }
 
    if (dump_enabled && strict_dump_layer >= 0) {
        ck_q80_contract_dump_tensor(strict_cached_layer >= 0
                                        ? "strict_out_proj_input_cached"
                                        : "strict_out_proj_input_live",
                                    strict_dump_layer,
                                    A_use,
                                    (size_t) M * (size_t) K);
    }
 
    if (strict &&
        gemm_nt_q8_0_q8_0_ggml_strict(A_use, B, bias, C, M, N, K)) {
        if (dump_enabled && strict_dump_layer >= 0) {
            ck_q80_contract_dump_tensor("strict_out_proj_output_ggml",
                                        strict_dump_layer,
                                        C,
                                        (size_t) M * (size_t) N);
        }
        return;
    }
 
    if (!strict) {
#pragma omp parallel for schedule(static) if(M > 1)
        for (int m = 0; m < M; ++m) {
            gemv_q8_0_q8_0_contract(&C[m * N], B, &A_use[m * K], N, K);
            if (bias) {
                for (int n = 0; n < N; ++n) {
                    C[m * N + n] += bias[n];
                }
            }
        }
        return;
    }
 
    for (int m = 0; m < M; ++m) {
        gemv_q8_0_q8_0_contract(&C[m * N], B, &A_use[m * K], N, K);
        if (bias) {
            for (int n = 0; n < N; ++n) {
                C[m * N + n] += bias[n];
            }
        }
    }
 
    if (dump_enabled && strict_dump_layer >= 0) {
        ck_q80_contract_dump_tensor("strict_out_proj_output_fallback",
                                    strict_dump_layer,
                                    C,
                                    (size_t) M * (size_t) N);
    }
}

References C, ck_q80_contract_cached_gemm_seq, ck_q80_contract_cached_input_enabled(), ck_q80_contract_dump_enabled(), ck_q80_contract_dump_tensor(), ck_strict_consume_next_gemm_a(), ck_strict_parity_enabled(), gemm_nt_q8_0_q8_0_ggml_strict(), and gemv_q8_0_q8_0_contract().

gemm_nt_q8_0_q8_0_ggml_strict()

static int gemm_nt_q8_0_q8_0_ggml_strict ( const float *  A, const void *  B, const float *  bias, float *  C, int  M, int  N, int  K  )

static

Definition at line 219 of file gemm_kernels_q8_0_q8_0_contract.c.

{
    ck_q80_ggml_cpu_init_fn ggml_cpu_init_fn = ck_q80_resolve_ggml_cpu_init();
    ck_q80_ggml_init_fn ggml_init_fn = ck_q80_resolve_ggml_init();
    ck_q80_ggml_free_fn ggml_free_fn = ck_q80_resolve_ggml_free();
    ck_q80_ggml_new_tensor_2d_fn ggml_new_tensor_2d_fn = ck_q80_resolve_ggml_new_tensor_2d();
    ck_q80_ggml_mul_mat_fn ggml_mul_mat_fn = ck_q80_resolve_ggml_mul_mat();
    ck_q80_ggml_new_graph_fn ggml_new_graph_fn = ck_q80_resolve_ggml_new_graph();
    ck_q80_ggml_build_forward_expand_fn ggml_build_forward_expand_fn = ck_q80_resolve_ggml_build_forward_expand();
    ck_q80_ggml_graph_compute_with_ctx_fn ggml_graph_compute_with_ctx_fn = ck_q80_resolve_ggml_graph_compute_with_ctx();
    ck_q80_ggml_get_data_fn ggml_get_data_fn = ck_q80_resolve_ggml_get_data();
    ck_q80_ggml_get_data_f32_fn ggml_get_data_f32_fn = ck_q80_resolve_ggml_get_data_f32();
    ck_q80_ggml_nbytes_fn ggml_nbytes_fn = ck_q80_resolve_ggml_nbytes();
 
    if (!ggml_cpu_init_fn || !ggml_init_fn || !ggml_free_fn || !ggml_new_tensor_2d_fn ||
        !ggml_mul_mat_fn || !ggml_new_graph_fn || !ggml_build_forward_expand_fn ||
        !ggml_graph_compute_with_ctx_fn || !ggml_get_data_fn || !ggml_get_data_f32_fn ||
        !ggml_nbytes_fn) {
        return 0;
    }
 
    ggml_cpu_init_fn();
 
    const size_t output_bytes = (size_t) M * (size_t) N * sizeof(float);
    const size_t mem_size = ((size_t) 128 * 1024 * 1024) + output_bytes;
 
    struct ggml_init_params params = {
        .mem_size = mem_size,
        .mem_buffer = NULL,
        .no_alloc = false,
    };
    struct ggml_context *ctx = ggml_init_fn(params);
    if (!ctx) {
        return 0;
    }
 
    int ok = 0;
    struct ggml_tensor *w = ggml_new_tensor_2d_fn(ctx, GGML_TYPE_Q8_0, K, N);
    struct ggml_tensor *x = ggml_new_tensor_2d_fn(ctx, GGML_TYPE_F32, K, M);
    if (!w || !x) {
        ggml_free_fn(ctx);
        return 0;
    }
 
    void *w_data = ggml_get_data_fn(w);
    void *x_data = ggml_get_data_fn(x);
    const size_t w_nbytes = ggml_nbytes_fn(w);
    if (!w_data || !x_data || w_nbytes == 0) {
        ggml_free_fn(ctx);
        return 0;
    }
 
    memcpy(w_data, B, w_nbytes);
    memcpy(x_data, A, (size_t) M * (size_t) K * sizeof(float));
 
    struct ggml_tensor *y = ggml_mul_mat_fn(ctx, w, x);
    if (!y) {
        ggml_free_fn(ctx);
        return 0;
    }
 
    struct ggml_cgraph *gf = ggml_new_graph_fn(ctx);
    if (!gf) {
        ggml_free_fn(ctx);
        return 0;
    }
    ggml_build_forward_expand_fn(gf, y);
    if (ggml_graph_compute_with_ctx_fn(ctx, gf, 1) != GGML_STATUS_SUCCESS) {
        ggml_free_fn(ctx);
        return 0;
    }
 
    {
        const float *src = ggml_get_data_f32_fn(y);
        if (!src) {
            ggml_free_fn(ctx);
            return 0;
        }
        for (int m = 0; m < M; ++m) {
            memcpy(C + (size_t) m * (size_t) N,
                   src + (size_t) m * (size_t) N,
                   (size_t) N * sizeof(float));
            if (bias) {
                for (int n = 0; n < N; ++n) {
                    C[(size_t) m * (size_t) N + (size_t) n] += bias[n];
                }
            }
        }
    }
 
    ok = 1;
    ggml_free_fn(ctx);
    return ok;
}

References C, ck_q80_resolve_ggml_build_forward_expand(), ck_q80_resolve_ggml_cpu_init(), ck_q80_resolve_ggml_free(), ck_q80_resolve_ggml_get_data(), ck_q80_resolve_ggml_get_data_f32(), ck_q80_resolve_ggml_graph_compute_with_ctx(), ck_q80_resolve_ggml_init(), ck_q80_resolve_ggml_mul_mat(), ck_q80_resolve_ggml_nbytes(), ck_q80_resolve_ggml_new_graph(), ck_q80_resolve_ggml_new_tensor_2d(), GGML_STATUS_SUCCESS, GGML_TYPE_F32, GGML_TYPE_Q8_0, and ggml_init_params::mem_size.

Referenced by gemm_nt_q8_0_q8_0_contract().

gemv_q8_0_q8_0_contract()

void gemv_q8_0_q8_0_contract	(	float *	y,
		const void *	W,
		const float *	x,
		int	M,
		int	K
	)

Definition at line 383 of file gemm_kernels_q8_0_q8_0_contract.c.

{
    if (!y || !W || !x || M <= 0 || K <= 0) {
        return;
    }
 
    if ((K % QK8_0) != 0) {
        gemv_q8_0(y, W, x, M, K);
        return;
    }
 
    const int blocks_per_row = K / QK8_0;
    if (blocks_per_row > CK_Q80_STACK_Q8_BLOCKS) {
        gemv_q8_0(y, W, x, M, K);
        return;
    }
 
    block_q8_0 x_q8[CK_Q80_STACK_Q8_BLOCKS];
    if (ck_strict_parity_enabled()) {
        quantize_row_q8_0_ref_local(x, x_q8, K);
        gemv_q8_0_q8_0_ref_rows(y, W, x_q8, M, K);
        return;
    }
    quantize_row_q8_0(x, x_q8, K);
    gemv_q8_0_q8_0_x4(y, W, x_q8, M, K);
}

References CK_Q80_STACK_Q8_BLOCKS, ck_strict_parity_enabled(), gemv_q8_0(), gemv_q8_0_q8_0_ref_rows(), gemv_q8_0_q8_0_x4(), QK8_0, quantize_row_q8_0(), and quantize_row_q8_0_ref_local().

Referenced by gemm_nt_q8_0_q8_0_contract().

gemv_q8_0_q8_0_ref_rows()

static void gemv_q8_0_q8_0_ref_rows ( float *  y, const void *  W, const void *  x_q8, int  M, int  K  )

static

Definition at line 364 of file gemm_kernels_q8_0_q8_0_contract.c.

{
    const block_q8_0 *w_blocks = (const block_q8_0 *)W;
    const int blocks_per_row = K / QK8_0;
 
    for (int row = 0; row < M; ++row) {
        vec_dot_q8_0_q8_0_ref(
            K,
            &y[row],
            &w_blocks[row * blocks_per_row],
            x_q8
        );
    }
}

References QK8_0, and vec_dot_q8_0_q8_0_ref().

Referenced by gemv_q8_0_q8_0_contract().

quantize_row_q8_0_ref_local()

static void quantize_row_q8_0_ref_local ( const float *  x, block_q8_0 *  y, int  k  )

static

Definition at line 330 of file gemm_kernels_q8_0_q8_0_contract.c.

{
    const int nb = k / QK8_0;
 
    for (int i = 0; i < nb; ++i) {
        float amax = 0.0f;
        for (int j = 0; j < QK8_0; ++j) {
            const float v = x[i * QK8_0 + j];
            const float av = fabsf(v);
            if (av > amax) {
                amax = av;
            }
        }
 
        const float d = amax / 127.0f;
        const float id = d != 0.0f ? 1.0f / d : 0.0f;
        y[i].d = CK_FP32_TO_FP16(d);
 
        for (int j = 0; j < QK8_0; ++j) {
            const float x0 = x[i * QK8_0 + j] * id;
            int q = ck_nearest_int_q8_0_ref(x0);
            if (q > 127) {
                q = 127;
            }
            if (q < -127) {
                q = -127;
            }
            y[i].qs[j] = (int8_t) q;
        }
    }
}

References CK_FP32_TO_FP16, ck_nearest_int_q8_0_ref(), block_q8_0::d, id, QK8_0, and block_q8_0::qs.

Referenced by gemv_q8_0_q8_0_contract().

vec_dot_q8_0_q8_0_ref()

void vec_dot_q8_0_q8_0_ref	(	int	n,
		float *	s,
		const void *	vx,
		const void *	vy
	)

Quantized dot product: Q8_0 weights x Q8_0 input (scalar reference)

Parameters

n	Number of elements (must be multiple of 32)
s	Output: scalar dot product result
vx	Q8_0 quantized weights
vy	Q8_0 quantized input

Definition at line 1140 of file gemm_kernels_q8_0.c.

{
    const int qk = QK8_0;  /* 32 */
    const int nb = n / qk;
 
    const block_q8_0 *x = (const block_q8_0 *)vx;
    const block_q8_0 *y = (const block_q8_0 *)vy;
 
    float sumf = 0.0f;
 
    for (int ib = 0; ib < nb; ib++) {
        int sumi = 0;
 
        for (int j = 0; j < qk; j++) {
            sumi += x[ib].qs[j] * y[ib].qs[j];
        }
 
        sumf += sumi * (CK_FP16_TO_FP32(x[ib].d) * CK_FP16_TO_FP32(y[ib].d));
    }
 
    *s = sumf;
}

References CK_FP16_TO_FP32, QK8_0, and block_q8_0::qs.

Referenced by gemv_q8_0_q8_0_ref_rows(), and vec_dot_q8_0_q8_0().

Variable Documentation

ck_q80_contract_cached_gemm_seq

int ck_q80_contract_cached_gemm_seq = 0

static

Definition at line 38 of file gemm_kernels_q8_0_q8_0_contract.c.

Referenced by gemm_nt_q8_0_q8_0_contract().

ck_q80_contract_magic

const char ck_q80_contract_magic[8] = {'C', 'K', 'D', 'M', 'P', '\0', '\0', '\0'}

static

Definition at line 36 of file gemm_kernels_q8_0_q8_0_contract.c.

{'C', 'K', 'D', 'M', 'P', '\0', '\0', '\0'};

Referenced by ck_q80_contract_dump_tensor().

ck_q80_contract_version

const uint32_t ck_q80_contract_version = 1u

static

Definition at line 37 of file gemm_kernels_q8_0_q8_0_contract.c.

Referenced by ck_q80_contract_dump_tensor().