hyper_connection_kernels.c File Reference

#include "ckernel_engine.h"
#include "bf16_utils.h"
#include <math.h>
#include <stddef.h>
#include <stdint.h>
#include <string.h>

Go to the source code of this file.

Typedefs
typedef void(*	ck_hyper_q8k_gemm_fn) (const void *, const void *, const float *, float *, int, int, int)

Functions
static float	ck_bf16_round (float value)
static float	ck_sigmoid_bf16 (float value)
void	gemm_nt_q4_k_q8_k_pairwise_split_min_parallel_dispatch (const void *A, const void *B, const float *bias, float *C, int M, int N, int K)
void	gemm_nt_q5_0_q8_0_parallel_dispatch (const void *A, const void *B, const float *bias, float *C, int M, int N, int K)
void	gemm_nt_q6_k_q8_k_parallel_dispatch (const void *A, const void *B, const float *bias, float *C, int M, int N, int K)
void	gemv_q4_k_q8_k_avx2 (float *y, const void *W, const void *x_q8, int M, int K)
void	hyper_connection_mix_bf16 (const float *hyper_input, const float *norm_weight, const uint16_t *mix_down_weight, const uint16_t *mix_up_weight, const uint16_t *inject_weight, float *mixed_output, float *injection_output, float *normalized_scratch, float *dynamic_scratch, float *mix_scratch, int rows, int streams, int hidden_dim, int dynamic_dim, float eps, int emit_injection)
void	hyper_connection_mix_q4k_q5_0_q4k (const float *hyper_input, const float *norm_weight, const void *mix_down_weight, const void *mix_up_weight, const void *inject_weight, float *mixed_output, float *injection_output, float *normalized_scratch, float *dynamic_scratch, float *mix_scratch, int rows, int streams, int hidden_dim, int dynamic_dim, float eps, int emit_injection)
void	hyper_connection_mix_q6k_q5_0_q4k (const float *hyper_input, const float *norm_weight, const void *mix_down_weight, const void *mix_up_weight, const void *inject_weight, float *mixed_output, float *injection_output, float *normalized_scratch, float *dynamic_scratch, float *mix_scratch, int rows, int streams, int hidden_dim, int dynamic_dim, float eps, int emit_injection)
static void	hyper_connection_mix_quantized (const float *hyper_input, const float *norm_weight, const void *mix_down_weight, const void *mix_up_weight, const void *inject_weight, float *mixed_output, float *injection_output, float *normalized_scratch, float *dynamic_scratch, float *mix_scratch, int rows, int streams, int hidden_dim, int dynamic_dim, float eps, int emit_injection, ck_hyper_q8k_gemm_fn injection_gemm, ck_hyper_q8k_gemm_fn down_gemm)
static void	hyper_injection_q4k_q8k_llama_dispatch (const void *input, const void *weight, const float *bias, float *output, int rows, int output_dim, int input_dim)
void	hyper_stream_expand_bf16 (const float *input, float *output, int rows, int streams, int hidden_dim)
void	hyper_stream_expand_f32 (const float *input, float *output, int rows, int streams, int hidden_dim)
void	hyper_stream_inject_bf16 (const float *hyper_input, const float *block_output, const float *injection_weight, float *output, int rows, int streams, int hidden_dim)
void	hyper_stream_inject_f32 (const float *hyper_input, const float *block_output, const float *injection_weight, float *output, int rows, int streams, int hidden_dim)

Typedef Documentation

ck_hyper_q8k_gemm_fn

typedef void(* ck_hyper_q8k_gemm_fn) (const void *, const void *, const float *, float *, int, int, int)

Definition at line 174 of file hyper_connection_kernels.c.

Function Documentation

ck_bf16_round()

static float ck_bf16_round ( float  value )

inlinestatic

Definition at line 26 of file hyper_connection_kernels.c.

                                               {
    return bf16_to_float(float_to_bf16(value));
}

References bf16_to_float(), and float_to_bf16().

Referenced by ck_sigmoid_bf16(), hyper_connection_mix_bf16(), hyper_stream_expand_bf16(), and hyper_stream_inject_bf16().

ck_sigmoid_bf16()

static float ck_sigmoid_bf16 ( float  value )

inlinestatic

Definition at line 30 of file hyper_connection_kernels.c.

                                                 {
    return ck_bf16_round(1.0f / (1.0f + expf(-value)));
}

References ck_bf16_round().

Referenced by hyper_connection_mix_bf16().

gemm_nt_q4_k_q8_k_pairwise_split_min_parallel_dispatch()

void gemm_nt_q4_k_q8_k_pairwise_split_min_parallel_dispatch	(	const void *	A,
		const void *	B,
		const float *	bias,
		float *	C,
		int	M,
		int	N,
		int	K
	)

Referenced by hyper_connection_mix_q4k_q5_0_q4k().

gemm_nt_q5_0_q8_0_parallel_dispatch()

void gemm_nt_q5_0_q8_0_parallel_dispatch	(	const void *	A,
		const void *	B,
		const float *	bias,
		float *	C,
		int	M,
		int	N,
		int	K
	)

Referenced by hyper_connection_mix_quantized().

gemm_nt_q6_k_q8_k_parallel_dispatch()

void gemm_nt_q6_k_q8_k_parallel_dispatch	(	const void *	A,
		const void *	B,
		const float *	bias,
		float *	C,
		int	M,
		int	N,
		int	K
	)

Referenced by hyper_connection_mix_q6k_q5_0_q4k().

gemv_q4_k_q8_k_avx2()

void gemv_q4_k_q8_k_avx2	(	float *	y,
		const void *	W,
		const void *	x_q8,
		int	M,
		int	K
	)

Definition at line 118 of file gemm_kernels_q4k_q8k_avx2.c.

{
#if defined(__AVX2__)
    if (!y || !W || !x_q8 || M <= 0 || K <= 0) {
        return;
    }
 
    const block_q4_K *blocks = (const block_q4_K *)W;
    const block_q8_K *x = (const block_q8_K *)x_q8;
    const int blocks_per_row = K / QK_K;
 
    for (int row = 0; row < M; ++row) {
        const block_q4_K *w_row = blocks + (size_t)row * (size_t)blocks_per_row;
        y[row] = dot_q4_k_q8_k_avx2(w_row, x, K);
    }
#else
    gemv_q4_k_q8_k_ref(y, W, x_q8, M, K);
#endif
}

Referenced by hyper_injection_q4k_q8k_llama_dispatch().

hyper_connection_mix_bf16()

void hyper_connection_mix_bf16	(	const float *	hyper_input,
		const float *	norm_weight,
		const uint16_t *	mix_down_weight,
		const uint16_t *	mix_up_weight,
		const uint16_t *	inject_weight,
		float *	mixed_output,
		float *	injection_output,
		float *	normalized_scratch,
		float *	dynamic_scratch,
		float *	mix_scratch,
		int	rows,
		int	streams,
		int	hidden_dim,
		int	dynamic_dim,
		float	eps,
		int	emit_injection
	)

Definition at line 73 of file hyper_connection_kernels.c.

                                                   {
    if (!hyper_input || !norm_weight || !mix_down_weight || !mix_up_weight ||
        !mixed_output || !normalized_scratch || !dynamic_scratch || !mix_scratch ||
        rows <= 0 || streams <= 0 || hidden_dim <= 0 || dynamic_dim <= 0) {
        return;
    }
    if (emit_injection && (!inject_weight || !injection_output)) {
        return;
    }
 
    const int hyper_dim = streams * hidden_dim;
    const float inv_streams = 1.0f / (float)streams;
 
    for (int row = 0; row < rows; ++row) {
        const float *input_row =
            hyper_input + (size_t)row * (size_t)hyper_dim;
        float *norm_row =
            normalized_scratch + (size_t)row * (size_t)hyper_dim;
        float *dynamic_row =
            dynamic_scratch + (size_t)row * (size_t)dynamic_dim;
        float *mix_row = mix_scratch + (size_t)row * (size_t)hyper_dim;
 
        for (int stream = 0; stream < streams; ++stream) {
            const int base = stream * hidden_dim;
            rmsnorm_forward_qwen3next_pytorch_bf16_storage(
                input_row + base,
                norm_weight + base,
                norm_row + base,
                NULL,
                1,
                hidden_dim,
                hidden_dim,
                eps
            );
        }
 
        for (int out = 0; out < dynamic_dim; ++out) {
            const uint16_t *weight_row =
                mix_down_weight + (size_t)out * (size_t)hyper_dim;
            float sum = 0.0f;
            for (int col = 0; col < hyper_dim; ++col) {
                sum += norm_row[col] * bf16_to_float(weight_row[col]);
            }
            const float projected = ck_bf16_round(sum * inv_streams);
            dynamic_row[out] = ck_bf16_round(
                projected / (1.0f + expf(-projected)));
        }
 
        for (int out = 0; out < hyper_dim; ++out) {
            const uint16_t *weight_row =
                mix_up_weight + (size_t)out * (size_t)dynamic_dim;
            float sum = 0.0f;
            for (int col = 0; col < dynamic_dim; ++col) {
                sum += dynamic_row[col] * bf16_to_float(weight_row[col]);
            }
            mix_row[out] = ck_sigmoid_bf16(ck_bf16_round(sum));
        }
 
        float *mixed_row =
            mixed_output + (size_t)row * (size_t)hidden_dim;
        for (int col = 0; col < hidden_dim; ++col) {
            float sum = 0.0f;
            for (int stream = 0; stream < streams; ++stream) {
                const int index = stream * hidden_dim + col;
                sum += ck_bf16_round(norm_row[index] * mix_row[index]);
            }
            mixed_row[col] = ck_bf16_round(sum * inv_streams);
        }
 
        if (emit_injection) {
            float *injection_row =
                injection_output + (size_t)row * (size_t)streams;
            for (int stream = 0; stream < streams; ++stream) {
                const uint16_t *weight_row =
                    inject_weight + (size_t)stream * (size_t)hyper_dim;
                float sum = 0.0f;
                for (int col = 0; col < hyper_dim; ++col) {
                    sum += norm_row[col] * bf16_to_float(weight_row[col]);
                }
                injection_row[stream] = ck_bf16_round(
                    2.0f * ck_sigmoid_bf16(ck_bf16_round(sum * inv_streams)));
            }
        }
    }
}

References bf16_to_float(), ck_bf16_round(), ck_sigmoid_bf16(), and rmsnorm_forward_qwen3next_pytorch_bf16_storage().

hyper_connection_mix_q4k_q5_0_q4k()

void hyper_connection_mix_q4k_q5_0_q4k	(	const float *	hyper_input,
		const float *	norm_weight,
		const void *	mix_down_weight,
		const void *	mix_up_weight,
		const void *	inject_weight,
		float *	mixed_output,
		float *	injection_output,
		float *	normalized_scratch,
		float *	dynamic_scratch,
		float *	mix_scratch,
		int	rows,
		int	streams,
		int	hidden_dim,
		int	dynamic_dim,
		float	eps,
		int	emit_injection
	)

Definition at line 350 of file hyper_connection_kernels.c.

                                                           {
    hyper_connection_mix_quantized(
        hyper_input, norm_weight, mix_down_weight, mix_up_weight, inject_weight,
        mixed_output, injection_output, normalized_scratch, dynamic_scratch,
        mix_scratch, rows, streams, hidden_dim, dynamic_dim, eps,
        emit_injection, hyper_injection_q4k_q8k_llama_dispatch,
        gemm_nt_q4_k_q8_k_pairwise_split_min_parallel_dispatch);
}

References gemm_nt_q4_k_q8_k_pairwise_split_min_parallel_dispatch(), hyper_connection_mix_quantized(), and hyper_injection_q4k_q8k_llama_dispatch().

hyper_connection_mix_q6k_q5_0_q4k()

void hyper_connection_mix_q6k_q5_0_q4k	(	const float *	hyper_input,
		const float *	norm_weight,
		const void *	mix_down_weight,
		const void *	mix_up_weight,
		const void *	inject_weight,
		float *	mixed_output,
		float *	injection_output,
		float *	normalized_scratch,
		float *	dynamic_scratch,
		float *	mix_scratch,
		int	rows,
		int	streams,
		int	hidden_dim,
		int	dynamic_dim,
		float	eps,
		int	emit_injection
	)

Definition at line 374 of file hyper_connection_kernels.c.

                                                           {
    hyper_connection_mix_quantized(
        hyper_input, norm_weight, mix_down_weight, mix_up_weight, inject_weight,
        mixed_output, injection_output, normalized_scratch, dynamic_scratch,
        mix_scratch, rows, streams, hidden_dim, dynamic_dim, eps,
        emit_injection, hyper_injection_q4k_q8k_llama_dispatch,
        gemm_nt_q6_k_q8_k_parallel_dispatch);
}

References gemm_nt_q6_k_q8_k_parallel_dispatch(), hyper_connection_mix_quantized(), and hyper_injection_q4k_q8k_llama_dispatch().

hyper_connection_mix_quantized()

static void hyper_connection_mix_quantized ( const float *  hyper_input, const float *  norm_weight, const void *  mix_down_weight, const void *  mix_up_weight, const void *  inject_weight, float *  mixed_output, float *  injection_output, float *  normalized_scratch, float *  dynamic_scratch, float *  mix_scratch, int  rows, int  streams, int  hidden_dim, int  dynamic_dim, float  eps, int  emit_injection, ck_hyper_q8k_gemm_fn  injection_gemm, ck_hyper_q8k_gemm_fn  down_gemm  )

static

Definition at line 210 of file hyper_connection_kernels.c.

                                                                           {
    if (!hyper_input || !norm_weight || !mix_down_weight || !mix_up_weight ||
        !mixed_output || !normalized_scratch || !dynamic_scratch || !mix_scratch ||
        !injection_gemm || !down_gemm || rows <= 0 || streams <= 0 || hidden_dim <= 0 ||
        dynamic_dim <= 0) {
        return;
    }
    if (emit_injection && (!inject_weight || !injection_output)) {
        return;
    }
 
    const int hyper_dim = streams * hidden_dim;
    const float inv_streams = 1.0f / (float)streams;
    if (hyper_dim % QK_K != 0 || dynamic_dim % QK8_0 != 0) {
        return;
    }
 
    const size_t normalized_q8_row_bytes =
        (size_t)(hyper_dim / QK_K) * sizeof(block_q8_K);
    const size_t dynamic_q8_row_bytes =
        (size_t)(dynamic_dim / QK8_0) * sizeof(block_q8_0);
    block_q8_K *normalized_q8 = (block_q8_K *)mix_scratch;
    block_q8_0 dynamic_q8[dynamic_dim / QK8_0];
 
    for (int row = 0; row < rows; ++row) {
        const float *input_row = hyper_input + (size_t)row * (size_t)hyper_dim;
        float *norm_row = normalized_scratch + (size_t)row * (size_t)hyper_dim;
        for (int stream = 0; stream < streams; ++stream) {
            const int base = stream * hidden_dim;
            double sum_sq = 0.0;
            for (int col = 0; col < hidden_dim; ++col) {
                const float value = input_row[base + col];
                sum_sq += (double)(value * value);
            }
            const float mean = (float)(sum_sq / (double)hidden_dim);
            const float rstd = 1.0f / sqrtf(mean + eps);
            for (int col = 0; col < hidden_dim; ++col) {
                const int index = base + col;
                norm_row[index] = input_row[index] * rstd * norm_weight[index];
            }
        }
 
        quantize_row_q8_k(
            norm_row,
            (uint8_t *)normalized_q8 + (size_t)row * normalized_q8_row_bytes,
            hyper_dim);
    }
 
    down_gemm(
        normalized_q8,
        mix_down_weight,
        NULL,
        dynamic_scratch,
        rows,
        dynamic_dim,
        hyper_dim);
 
    if (emit_injection) {
        injection_gemm(
            normalized_q8,
            inject_weight,
            NULL,
            injection_output,
            rows,
            streams,
            hyper_dim);
        for (int row = 0; row < rows; ++row) {
            float *injection_row = injection_output + (size_t)row * (size_t)streams;
            for (int stream = 0; stream < streams; ++stream) {
                injection_row[stream] *= inv_streams;
            }
            recurrent_sigmoid_forward_ggml(
                injection_row, injection_row, 1, streams);
            for (int stream = 0; stream < streams; ++stream) {
                injection_row[stream] *= 2.0f;
            }
        }
    }
 
    for (int row = 0; row < rows; ++row) {
        float *dynamic_row = dynamic_scratch + (size_t)row * (size_t)dynamic_dim;
        for (int col = 0; col < dynamic_dim; ++col) {
            dynamic_row[col] *= inv_streams;
        }
        recurrent_silu_forward_ggml(
            dynamic_row, dynamic_row, 1, dynamic_dim);
 
        quantize_row_q8_0(dynamic_row, dynamic_q8, dynamic_dim);
        memcpy(
            (uint8_t *)dynamic_scratch + (size_t)row * dynamic_q8_row_bytes,
            dynamic_q8,
            dynamic_q8_row_bytes);
    }
 
    gemm_nt_q5_0_q8_0_parallel_dispatch(
        dynamic_scratch,
        mix_up_weight,
        NULL,
        mix_scratch,
        rows,
        hyper_dim,
        dynamic_dim);
 
    for (int row = 0; row < rows; ++row) {
        const float *norm_row =
            normalized_scratch + (size_t)row * (size_t)hyper_dim;
        float *mix_row = mix_scratch + (size_t)row * (size_t)hyper_dim;
        recurrent_sigmoid_forward_ggml(
            mix_row, mix_row, 1, hyper_dim);
 
        float *mixed_row = mixed_output + (size_t)row * (size_t)hidden_dim;
        for (int col = 0; col < hidden_dim; ++col) {
            float sum = 0.0f;
            for (int stream = 0; stream < streams; ++stream) {
                const int index = stream * hidden_dim + col;
                sum += norm_row[index] * mix_row[index];
            }
            mixed_row[col] = sum * inv_streams;
        }
 
    }
}

References gemm_nt_q5_0_q8_0_parallel_dispatch(), QK8_0, QK_K, quantize_row_q8_0(), quantize_row_q8_k(), recurrent_sigmoid_forward_ggml(), and recurrent_silu_forward_ggml().

Referenced by hyper_connection_mix_q4k_q5_0_q4k(), and hyper_connection_mix_q6k_q5_0_q4k().

hyper_injection_q4k_q8k_llama_dispatch()

static void hyper_injection_q4k_q8k_llama_dispatch ( const void *  input, const void *  weight, const float *  bias, float *  output, int  rows, int  output_dim, int  input_dim  )

static

Definition at line 177 of file hyper_connection_kernels.c.

                   {
    if (!input || !weight || !output || rows <= 0 || output_dim <= 0 ||
        input_dim <= 0 || input_dim % QK_K != 0) {
        return;
    }
 
    const size_t input_row_bytes =
        (size_t)(input_dim / QK_K) * sizeof(block_q8_K);
    for (int row = 0; row < rows; ++row) {
        float *output_row = output + (size_t)row * (size_t)output_dim;
        const void *input_row =
            (const uint8_t *)input + (size_t)row * input_row_bytes;
#if defined(CK_TARGET_ARM)
        gemv_q4_k_q8_k(
#else
        gemv_q4_k_q8_k_avx2(
#endif
            output_row, weight, input_row, output_dim, input_dim);
        if (bias) {
            for (int col = 0; col < output_dim; ++col) {
                output_row[col] += bias[col];
            }
        }
    }
}

References gemv_q4_k_q8_k(), gemv_q4_k_q8_k_avx2(), and QK_K.

Referenced by hyper_connection_mix_q4k_q5_0_q4k(), and hyper_connection_mix_q6k_q5_0_q4k().

hyper_stream_expand_bf16()

void hyper_stream_expand_bf16	(	const float *	input,
		float *	output,
		int	rows,
		int	streams,
		int	hidden_dim
	)

Definition at line 53 of file hyper_connection_kernels.c.

                                              {
    if (!input || !output || rows <= 0 || streams <= 0 || hidden_dim <= 0) {
        return;
    }
    for (int row = 0; row < rows; ++row) {
        const float *src = input + (size_t)row * (size_t)hidden_dim;
        float *dst = output + (size_t)row * (size_t)streams * (size_t)hidden_dim;
        for (int stream = 0; stream < streams; ++stream) {
            for (int col = 0; col < hidden_dim; ++col) {
                dst[(size_t)stream * (size_t)hidden_dim + (size_t)col] =
                    ck_bf16_round(src[col]);
            }
        }
    }
}

References ck_bf16_round().

hyper_stream_expand_f32()

void hyper_stream_expand_f32	(	const float *	input,
		float *	output,
		int	rows,
		int	streams,
		int	hidden_dim
	)

Definition at line 34 of file hyper_connection_kernels.c.

                                             {
    if (!input || !output || rows <= 0 || streams <= 0 || hidden_dim <= 0) {
        return;
    }
    for (int row = 0; row < rows; ++row) {
        const float *src = input + (size_t)row * (size_t)hidden_dim;
        float *dst = output + (size_t)row * (size_t)streams * (size_t)hidden_dim;
        for (int stream = 0; stream < streams; ++stream) {
            for (int col = 0; col < hidden_dim; ++col) {
                dst[(size_t)stream * (size_t)hidden_dim + (size_t)col] = src[col];
            }
        }
    }
}

hyper_stream_inject_bf16()

void hyper_stream_inject_bf16	(	const float *	hyper_input,
		const float *	block_output,
		const float *	injection_weight,
		float *	output,
		int	rows,
		int	streams,
		int	hidden_dim
	)

Definition at line 398 of file hyper_connection_kernels.c.

                                              {
    if (!hyper_input || !block_output || !injection_weight || !output ||
        rows <= 0 || streams <= 0 || hidden_dim <= 0) {
        return;
    }
    const int hyper_dim = streams * hidden_dim;
    for (int row = 0; row < rows; ++row) {
        const float *hyper_row =
            hyper_input + (size_t)row * (size_t)hyper_dim;
        const float *block_row =
            block_output + (size_t)row * (size_t)hidden_dim;
        const float *inject_row =
            injection_weight + (size_t)row * (size_t)streams;
        float *output_row = output + (size_t)row * (size_t)hyper_dim;
        for (int stream = 0; stream < streams; ++stream) {
            for (int col = 0; col < hidden_dim; ++col) {
                const int index = stream * hidden_dim + col;
                output_row[index] = ck_bf16_round(
                    hyper_row[index] +
                    ck_bf16_round(block_row[col] * inject_row[stream]));
            }
        }
    }
}

References ck_bf16_round().

hyper_stream_inject_f32()

void hyper_stream_inject_f32	(	const float *	hyper_input,
		const float *	block_output,
		const float *	injection_weight,
		float *	output,
		int	rows,
		int	streams,
		int	hidden_dim
	)

Definition at line 429 of file hyper_connection_kernels.c.

                                             {
    if (!hyper_input || !block_output || !injection_weight || !output ||
        rows <= 0 || streams <= 0 || hidden_dim <= 0) {
        return;
    }
    const int hyper_dim = streams * hidden_dim;
    for (int row = 0; row < rows; ++row) {
        const float *hyper_row =
            hyper_input + (size_t)row * (size_t)hyper_dim;
        const float *block_row =
            block_output + (size_t)row * (size_t)hidden_dim;
        const float *inject_row =
            injection_weight + (size_t)row * (size_t)streams;
        float *output_row = output + (size_t)row * (size_t)hyper_dim;
        for (int stream = 0; stream < streams; ++stream) {
            for (int col = 0; col < hidden_dim; ++col) {
                const int index = stream * hidden_dim + col;
                volatile const float weighted =
                    block_row[col] * inject_row[stream];
                output_row[index] = hyper_row[index] + weighted;
            }
        }
    }
}