ckernel_audio.h File Reference

#include <stdint.h>
#include <stddef.h>

Go to the source code of this file.

Data Structures
struct	CKAudioWavInfo

Macros
#define	CK_AUDIO_WHISPER_HOP_LENGTH   160
#define	CK_AUDIO_WHISPER_N_FFT   400
#define	CK_AUDIO_WHISPER_POWER_BINS   201
#define	CK_AUDIO_WHISPER_SAMPLE_RATE   16000

Functions
int	audio_adain_instance_norm_f32 (const float *input, size_t input_elements, size_t input_stride, const float *norm_weight, size_t norm_weight_elements, const float *norm_bias, size_t norm_bias_elements, const float *style_affine, size_t style_affine_elements, float *output, size_t output_elements, size_t output_stride, size_t channels, size_t frames, float epsilon)
int	audio_adaptive_layer_norm_f32 (const float *input, size_t input_elements, const float *style, size_t style_elements, const float *projection_weight, size_t projection_weight_elements, const float *projection_bias, size_t projection_bias_elements, float *output, size_t output_elements, float *projection_scratch, size_t projection_scratch_bytes, int tokens, int channels, int style_dim, size_t input_stride, size_t output_stride, float epsilon)
int	audio_argmax_first_f32 (const float *values, int elements, int *selected)
int	audio_batch_norm_inference_channel_major_f32 (const float *input, const float *running_mean, const float *running_variance, const float *weight, const float *bias, float *output, int channels, int frames, float epsilon)
int	audio_conformer_relative_attention_f32 (const float *query, const float *key, const float *value, const float *relative_key, const float *bias_u, const float *bias_v, float *output, int frames, int heads, int head_dim, float scale, float *scores_scratch, size_t scores_scratch_bytes)
int	audio_conv1d_channel_major_f32 (const float *input, const float *weight, const float *bias, float *output, int input_channels, int output_channels, int input_frames, int kernel_size, int stride, int padding, int output_frames)
int	audio_conv1d_channel_major_grouped_f32 (const float *input, const float *weight, const float *bias, float *output, int input_channels, int output_channels, int input_frames, int kernel_size, int stride, int padding, int groups, int output_frames)
int	audio_conv2d_whc_grouped_f32 (const float *input, const float *weight, const float *bias, float *output, int input_width, int input_height, int input_channels, int output_channels, int kernel_width, int kernel_height, int stride_width, int stride_height, int padding_width, int padding_height, int groups, int output_width, int output_height)
int	audio_fastconformer_block_f32 (const float *input, const float *relative_positions, const float *ff1_norm_weight, const float *ff1_norm_bias, const float *ff1_up_weight, const float *ff1_up_bias, const float *ff1_down_weight, const float *ff1_down_bias, const float *attn_norm_weight, const float *attn_norm_bias, const float *q_weight, const float *q_bias, const float *k_weight, const float *k_bias, const float *v_weight, const float *v_bias, const float *relative_weight, const float *attn_bias_u, const float *attn_bias_v, const float *attn_out_weight, const float *attn_out_bias, const float *conv_norm_weight, const float *conv_norm_bias, const float *conv_pw1_weight, const float *conv_pw1_bias, const float *conv_dw_weight, const float *conv_dw_bias, const float *conv_bn_mean, const float *conv_bn_variance, const float *conv_bn_weight, const float *conv_bn_bias, const float *conv_pw2_weight, const float *conv_pw2_bias, const float *ff2_norm_weight, const float *ff2_norm_bias, const float *ff2_up_weight, const float *ff2_up_bias, const float *ff2_down_weight, const float *ff2_down_bias, const float *out_norm_weight, const float *out_norm_bias, float *output, void *workspace, size_t workspace_bytes, int frames, int hidden_size, int intermediate_size, int heads, int head_dim, int conv_kernel_size, float layer_norm_epsilon, float batch_norm_epsilon)
size_t	audio_fastconformer_block_workspace_bytes (int frames, int hidden_size, int intermediate_size, int heads)
int	audio_fastconformer_subsampling_f32 (const float *features, const float *conv0_weight, const float *conv0_bias, const float *depthwise1_weight, const float *depthwise1_bias, const float *pointwise1_weight, const float *pointwise1_bias, const float *depthwise2_weight, const float *depthwise2_bias, const float *pointwise2_weight, const float *pointwise2_bias, const float *linear_weight, const float *linear_bias, float *output, void *workspace, size_t workspace_bytes, int feature_frames, int live_frames, int feature_channels, int conv_channels, int hidden_size, int kernel_size, int stride, int output_capacity_frames, int *output_frames)
size_t	audio_fastconformer_subsampling_workspace_bytes (int feature_frames, int feature_channels, int conv_channels, int kernel_size, int stride)
int	audio_feature_normalize_per_feature_f32 (const float *input, float *output, int channels, int frames, float epsilon)
int	audio_glu_split_channel_major_f32 (const float *input, float *output, int channels, int frames)
int	audio_hann_window_f32 (float *output, int frames, int periodic)
int	audio_log_mel_time_major_f32 (const float *power, const float *mel_filters, float *log_mel, int frames, int bins, int channels, float epsilon)
int	audio_lstm_bidirectional_scan_f32 (const float *input, size_t input_elements, const float *weight_ih, size_t weight_ih_elements, const float *weight_hh, size_t weight_hh_elements, const float *bias_ih, size_t bias_ih_elements, const float *bias_hh, size_t bias_hh_elements, float *output, size_t output_elements, float *hidden_state, size_t hidden_state_elements, float *cell_state, size_t cell_state_elements, float *gates_scratch, size_t gates_scratch_bytes, int tokens, int input_size, int hidden_size, size_t input_stride, size_t output_stride)
int	audio_lstm_step_f32 (const float *input, const float *weight_ih, const float *weight_hh, const float *bias_ih, const float *bias_hh, float *hidden_state, float *cell_state, float *output, float *gates_scratch, size_t gates_scratch_bytes, int input_size, int hidden_size)
int	audio_pad_or_truncate_f32 (const float *input, int input_frames, float *output, int output_frames)
int	audio_pcm_s16_to_mono_f32 (const int16_t *interleaved, int n_frames, int n_channels, float *mono)
int	audio_preemphasis_f32 (const float *input, float *output, int frames, float coefficient)
int	audio_relative_shift_f32 (const float *raw_scores, float *scores, int heads, int query_frames)
int	audio_relative_sinusoidal_position_f32 (float *output, int frames, int channels)
int	audio_resample_linear_f32 (const float *input, int input_frames, int input_rate, float *output, int output_frames, int output_rate)
int	audio_resample_windowed_sinc_f32 (const float *input, int input_frames, int input_rate, float *output, int output_frames, int output_rate, int radius)
int	audio_resampled_frame_count (int input_frames, int input_rate, int output_rate)
int	audio_scaled_residual_add_f32 (const float *residual, const float *branch, float scale, float *output, size_t elements)
int	audio_stft_power_centered_window_f32 (const float *samples, int n_samples, const float *window, int window_length, const float *cos_table, const float *sin_table, int n_fft, int hop_length, int reflect_padding, float *power, int n_frames)
int	audio_stft_power_fft400_f32 (const float *samples, int n_samples, const float *window, const float *cos_table, const float *sin_table, int hop_length, float *power, int n_frames, float *fft_scratch)
int	audio_stft_power_precomputed_f32 (const float *samples, int n_samples, const float *window, const float *cos_table, const float *sin_table, int n_fft, int hop_length, float *power, int n_frames)
int	audio_stft_precompute_tables_f32 (int n_fft, float *window, float *cos_table, float *sin_table)
int	audio_transpose_channel_to_token_f32 (const float *input, float *output, int channels, int frames)
int	audio_wav_decode_memory_pcm16_mono_f32 (const uint8_t *bytes, size_t byte_count, float *mono, int mono_capacity, CKAudioWavInfo *info)
int	audio_wav_decode_memory_pcm16_mono_window_f32 (const uint8_t *bytes, size_t byte_count, int start_frame, float *mono, int mono_capacity, CKAudioWavInfo *info)
int	audio_wav_decode_pcm16_mono_f32 (const uint8_t *bytes, size_t byte_count, const CKAudioWavInfo *info, float *mono, int mono_capacity)
int	audio_wav_parse_memory (const uint8_t *bytes, size_t byte_count, CKAudioWavInfo *info)
int	audio_whisper_log_mel_from_power_reference_f32 (const float *power, const float *mel_filters, int n_mels, int n_frames, float *log_mel)
int	audio_whisper_log_mel_reference_f32 (const float *samples, int n_samples, const float *mel_filters, int n_mels, float *power_scratch, float *log_mel, int n_frames)
int	audio_whisper_log_mel_window_wav_pcm16_f32 (const uint8_t *bytes, size_t byte_count, int start_frame, int target_sample_rate, const float *window, const float *cos_table, const float *sin_table, const float *mel_filters, int n_mels, int output_frames, float *log_mel)
int	audio_whisper_mel_filters_slaney_f32 (int sample_rate, int n_fft, int n_mels, float *mel_filters)
int	audio_whisper_stft_power_reference_f32 (const float *samples, int n_samples, float *power, int n_frames)

Macro Definition Documentation

CK_AUDIO_WHISPER_HOP_LENGTH

#define CK_AUDIO_WHISPER_HOP_LENGTH   160

Definition at line 13 of file ckernel_audio.h.

CK_AUDIO_WHISPER_N_FFT

#define CK_AUDIO_WHISPER_N_FFT   400

Definition at line 12 of file ckernel_audio.h.

CK_AUDIO_WHISPER_POWER_BINS

#define CK_AUDIO_WHISPER_POWER_BINS   201

Definition at line 14 of file ckernel_audio.h.

CK_AUDIO_WHISPER_SAMPLE_RATE

#define CK_AUDIO_WHISPER_SAMPLE_RATE   16000

Definition at line 11 of file ckernel_audio.h.

Function Documentation

audio_adain_instance_norm_f32()

int audio_adain_instance_norm_f32	(	const float *	input,
		size_t	input_elements,
		size_t	input_stride,
		const float *	norm_weight,
		size_t	norm_weight_elements,
		const float *	norm_bias,
		size_t	norm_bias_elements,
		const float *	style_affine,
		size_t	style_affine_elements,
		float *	output,
		size_t	output_elements,
		size_t	output_stride,
		size_t	channels,
		size_t	frames,
		float	epsilon
	)

Definition at line 38 of file audio_adain_instance_norm.c.

{
    if (input == NULL || norm_weight == NULL || norm_bias == NULL ||
        style_affine == NULL || output == NULL ||
        !isfinite(epsilon) || epsilon <= 0.0f ||
        channels == 0u || channels > SIZE_MAX / 2u) return -1;
    size_t input_required, output_required;
    if (!checked_span(channels, input_stride, frames, &input_required) ||
        !checked_span(channels, output_stride, frames, &output_required))
        return -1;
    if (input_elements < input_required || output_elements < output_required ||
        norm_weight_elements < channels || norm_bias_elements < channels ||
        style_affine_elements < 2u * channels) return -2;
 
    for (size_t channel = 0u; channel < channels; ++channel) {
        if (!isfinite(norm_weight[channel]) || !isfinite(norm_bias[channel]) ||
            !isfinite(style_affine[channel]) ||
            !isfinite(style_affine[channels + channel])) return -3;
        const float *row = input + channel * input_stride;
        for (size_t frame = 0u; frame < frames; ++frame)
            if (!isfinite(row[frame])) return -3;
    }
 
    /* Dry-run every result before the first output write. The repeated scalar
     * pass trades throughput for a small, allocation-free reference kernel. */
    for (int pass = 0; pass < 2; ++pass) {
        for (size_t channel = 0u; channel < channels; ++channel) {
            const float *row = input + channel * input_stride;
            float *output_row = output + channel * output_stride;
            double mean, inverse_std;
            if (!channel_stats(row, frames, epsilon, &mean, &inverse_std))
                return -3;
            const float weight = norm_weight[channel];
            const float bias = norm_bias[channel];
            const float gamma = style_affine[channel];
            const float beta = style_affine[channels + channel];
            for (size_t frame = 0u; frame < frames; ++frame) {
                const float normalized = (float)(
                    ((double)row[frame] - mean) * inverse_std);
                const float affine = normalized * weight + bias;
                const float value = (1.0f + gamma) * affine + beta;
                if (!isfinite(value)) return -3;
                if (pass != 0) output_row[frame] = value;
            }
        }
    }
    return 0;
}

References channel_stats(), and checked_span().

audio_adaptive_layer_norm_f32()

int audio_adaptive_layer_norm_f32	(	const float *	input,
		size_t	input_elements,
		const float *	style,
		size_t	style_elements,
		const float *	projection_weight,
		size_t	projection_weight_elements,
		const float *	projection_bias,
		size_t	projection_bias_elements,
		float *	output,
		size_t	output_elements,
		float *	projection_scratch,
		size_t	projection_scratch_bytes,
		int	tokens,
		int	channels,
		int	style_dim,
		size_t	input_stride,
		size_t	output_stride,
		float	epsilon
	)

Definition at line 57 of file audio_adaptive_layer_norm.c.

{
    if (input == NULL || style == NULL || projection_weight == NULL ||
        projection_bias == NULL || output == NULL ||
        projection_scratch == NULL) return -1;
    if (tokens <= 0 || channels <= 0 || style_dim <= 0 ||
        !isfinite(epsilon) || epsilon <= 0.0f) return -2;
 
    const size_t rows = (size_t)tokens;
    const size_t width = (size_t)channels;
    const size_t style_width = (size_t)style_dim;
    size_t input_required, output_required, projection_width;
    size_t weight_required, scratch_required;
    if (input_stride < width || output_stride < width ||
        !checked_span(rows, input_stride, width, &input_required) ||
        !checked_span(rows, output_stride, width, &output_required) ||
        !checked_product(width, 2u, &projection_width) ||
        !checked_product(projection_width, style_width, &weight_required) ||
        !checked_product(projection_width, sizeof(float), &scratch_required) ||
        !checked_float_extent(input_required) ||
        !checked_float_extent(output_required) ||
        !checked_float_extent(style_width) ||
        !checked_float_extent(weight_required) ||
        !checked_float_extent(projection_width))
        return -2;
    if (input_elements < input_required || output_elements < output_required ||
        style_elements < style_width ||
        projection_weight_elements < weight_required ||
        projection_bias_elements < projection_width ||
        projection_scratch_bytes < scratch_required) return -3;
 
    /* Validate all external values before changing output or scratch. Padding
     * is deliberately not read; only the declared logical columns are valid.
     */
    for (size_t index = 0u; index < style_width; ++index)
        if (!isfinite(style[index])) return -4;
    for (size_t index = 0u; index < weight_required; ++index)
        if (!isfinite(projection_weight[index])) return -4;
    for (size_t index = 0u; index < projection_width; ++index)
        if (!isfinite(projection_bias[index])) return -4;
    for (size_t row = 0u; row < rows; ++row)
        for (size_t channel = 0u; channel < width; ++channel)
            if (!isfinite(input[row * input_stride + channel])) return -4;
 
    for (size_t channel = 0u; channel < projection_width; ++channel) {
        float value = projection_bias[channel];
        const float *weight = projection_weight + channel * style_width;
        for (size_t index = 0u; index < style_width; ++index)
            value += weight[index] * style[index];
        if (!isfinite(value)) return -4;
        projection_scratch[channel] = value;
    }
 
    /* Validate every produced value before the first output write. The second
     * pass repeats cheap normalization arithmetic without a hidden T*C buffer.
     */
    for (int pass = 0; pass < 2; ++pass) {
        for (size_t row = 0u; row < rows; ++row) {
            const float *input_row = input + row * input_stride;
            float *output_row = output + row * output_stride;
            double mean, inverse_std;
            if (!row_stats(input_row, width, epsilon, &mean, &inverse_std))
                return -4;
            for (size_t channel = 0u; channel < width; ++channel) {
                const float normalized = (float)(
                    ((double)input_row[channel] - mean) * inverse_std);
                const float gamma = projection_scratch[channel];
                const float beta = projection_scratch[width + channel];
                const float value = (1.0f + gamma) * normalized + beta;
                if (!isfinite(value)) return -4;
                if (pass != 0) output_row[channel] = value;
            }
        }
    }
    return 0;
}

References checked_float_extent(), checked_product(), checked_span(), row_stats(), and style.

audio_argmax_first_f32()

int audio_argmax_first_f32	(	const float *	values,
		int	elements,
		int *	selected
	)

Definition at line 2147 of file audio_kernels.c.

{
    if (values == NULL || selected == NULL) {
        return -1;
    }
    if (elements <= 0) {
        return -2;
    }
    if (!isfinite(values[0])) {
        return -3;
    }
    int best = 0;
    float maximum = values[0];
    for (int index = 1; index < elements; ++index) {
        if (!isfinite(values[index])) {
            return -3;
        }
        if (values[index] > maximum) {
            maximum = values[index];
            best = index;
        }
    }
    *selected = best;
    return 0;
}

audio_batch_norm_inference_channel_major_f32()

int audio_batch_norm_inference_channel_major_f32	(	const float *	input,
		const float *	running_mean,
		const float *	running_variance,
		const float *	weight,
		const float *	bias,
		float *	output,
		int	channels,
		int	frames,
		float	epsilon
	)

Definition at line 469 of file audio_kernels.c.

{
    if (input == NULL || running_mean == NULL || running_variance == NULL ||
        weight == NULL || bias == NULL || output == NULL) {
        return -1;
    }
    if (channels <= 0 || frames <= 0 || !isfinite(epsilon) || epsilon < 0.0f) {
        return -2;
    }
    for (int channel = 0; channel < channels; ++channel) {
        const float variance = running_variance[channel];
        if (!isfinite(variance) || variance + epsilon <= 0.0f) {
            return -3;
        }
    }
    for (int channel = 0; channel < channels; ++channel) {
        const float variance = running_variance[channel];
        const float scale = weight[channel] / sqrtf(variance + epsilon);
        const float offset = bias[channel] - running_mean[channel] * scale;
        for (int frame = 0; frame < frames; ++frame) {
            const size_t index = (size_t)channel * frames + frame;
            output[index] = input[index] * scale + offset;
        }
    }
    return 0;
}

Referenced by audio_fastconformer_block_f32().

audio_conformer_relative_attention_f32()

int audio_conformer_relative_attention_f32	(	const float *	query,
		const float *	key,
		const float *	value,
		const float *	relative_key,
		const float *	bias_u,
		const float *	bias_v,
		float *	output,
		int	frames,
		int	heads,
		int	head_dim,
		float	scale,
		float *	scores_scratch,
		size_t	scores_scratch_bytes
	)

Definition at line 1766 of file audio_kernels.c.

{
    if (query == NULL || key == NULL || value == NULL || relative_key == NULL ||
        bias_u == NULL || bias_v == NULL || output == NULL ||
        scores_scratch == NULL) {
        return -1;
    }
    if (frames <= 0 || frames > (INT_MAX / 2) + 1 || heads <= 0 ||
        head_dim <= 0 || heads > INT_MAX / head_dim ||
        heads > INT_MAX / frames || !isfinite(scale) || scale <= 0.0f) {
        return -2;
    }
    if ((size_t)frames > SIZE_MAX / (size_t)heads ||
        (size_t)frames * (size_t)heads > SIZE_MAX / sizeof(float) ||
        scores_scratch_bytes <
            (size_t)frames * (size_t)heads * sizeof(float)) {
        return -3;
    }
    const int channels = heads * head_dim;
    ck_threadpool_t *pool = ck_threadpool_global();
    ck_audio_conformer_relative_attention_f32_args_t args = {
        .query = query,
        .key = key,
        .value = value,
        .relative_key = relative_key,
        .bias_u = bias_u,
        .bias_v = bias_v,
        .output = output,
        .scores_scratch = scores_scratch,
        .pool = pool,
        .frames = frames,
        .heads = heads,
        .head_dim = head_dim,
        .channels = channels,
        .scale = scale,
    };
    int active = pool ? ck_threadpool_n_threads(pool) : 1;
    const size_t scratch_slots = scores_scratch_bytes /
        ((size_t)frames * sizeof(float));
    const int rows = heads * frames;
    if ((size_t)active > scratch_slots) active = (int)scratch_slots;
    if (active > rows) active = rows;
    if (pool != NULL && active > 1) {
        ck_threadpool_parallel_for_n(
            pool, active, 0, rows, 1,
            ck_audio_conformer_relative_attention_f32_row_range, &args);
    } else {
        ck_audio_conformer_relative_attention_f32_row_range(0, rows, &args);
    }
    return 0;
}

References ck_audio_conformer_relative_attention_f32_row_range(), ck_threadpool_global(), ck_threadpool_n_threads(), and ck_threadpool_parallel_for_n().

Referenced by audio_fastconformer_block_f32().

audio_conv1d_channel_major_f32()

int audio_conv1d_channel_major_f32	(	const float *	input,
		const float *	weight,
		const float *	bias,
		float *	output,
		int	input_channels,
		int	output_channels,
		int	input_frames,
		int	kernel_size,
		int	stride,
		int	padding,
		int	output_frames
	)

Definition at line 1309 of file audio_kernels.c.

{
    return audio_conv1d_channel_major_grouped_f32(
        input, weight, bias, output, input_channels, output_channels,
        input_frames, kernel_size, stride, padding, 1, output_frames);
}

References audio_conv1d_channel_major_grouped_f32().

audio_conv1d_channel_major_grouped_f32()

int audio_conv1d_channel_major_grouped_f32	(	const float *	input,
		const float *	weight,
		const float *	bias,
		float *	output,
		int	input_channels,
		int	output_channels,
		int	input_frames,
		int	kernel_size,
		int	stride,
		int	padding,
		int	groups,
		int	output_frames
	)

Definition at line 1327 of file audio_kernels.c.

{
    if (input == NULL || weight == NULL || output == NULL) {
        return -1;
    }
    if (input_channels <= 0 || output_channels <= 0 || input_frames <= 0 ||
        kernel_size <= 0 || stride <= 0 || padding < 0 || groups <= 0 ||
        output_frames <= 0 || input_channels % groups != 0 ||
        output_channels % groups != 0) {
        return -2;
    }
    const int64_t padded_frames =
        (int64_t)input_frames + 2 * (int64_t)padding - kernel_size;
    if (padded_frames < 0 || padded_frames / stride + 1 > INT_MAX ||
        output_frames != (int)(padded_frames / stride + 1)) {
        return -3;
    }
    const char *disable_stride2 =
        getenv("CK_DISABLE_AUDIO_CONV_STRIDE2_CONTIGUOUS");
    ck_audio_conv1d_f32_args_t args = {
        .input = input,
        .weight = weight,
        .bias = bias,
        .output = output,
        .input_channels = input_channels,
        .output_channels = output_channels,
        .input_frames = input_frames,
        .kernel_size = kernel_size,
        .stride = stride,
        .padding = padding,
        .output_frames = output_frames,
        .groups = groups,
        .input_channels_per_group = input_channels / groups,
        .output_channels_per_group = output_channels / groups,
        .use_stride2_contiguous = !(
            disable_stride2 && disable_stride2[0] &&
            strcmp(disable_stride2, "0") != 0),
    };
    ck_threadpool_t *pool = ck_threadpool_global();
    int active = pool ? ck_threadpool_n_threads(pool) : 1;
    if (active > output_channels) active = output_channels;
    if (pool != NULL && active > 1) {
        ck_threadpool_dispatch_n(
            pool, active, ck_audio_conv1d_channel_major_f32_work, &args);
    } else {
        ck_audio_conv1d_channel_major_f32_work(0, 1, &args);
    }
    return 0;
}

References ck_audio_conv1d_channel_major_f32_work(), ck_threadpool_dispatch_n(), ck_threadpool_global(), and ck_threadpool_n_threads().

Referenced by audio_conv1d_channel_major_f32(), and audio_fastconformer_block_f32().

audio_conv2d_whc_grouped_f32()

int audio_conv2d_whc_grouped_f32	(	const float *	input,
		const float *	weight,
		const float *	bias,
		float *	output,
		int	input_width,
		int	input_height,
		int	input_channels,
		int	output_channels,
		int	kernel_width,
		int	kernel_height,
		int	stride_width,
		int	stride_height,
		int	padding_width,
		int	padding_height,
		int	groups,
		int	output_width,
		int	output_height
	)

Definition at line 1500 of file audio_kernels.c.

{
    if (input == NULL || weight == NULL || output == NULL) {
        return -1;
    }
    if (input_width <= 0 || input_height <= 0 || input_channels <= 0 ||
        output_channels <= 0 || kernel_width <= 0 || kernel_height <= 0 ||
        stride_width <= 0 || stride_height <= 0 || padding_width < 0 ||
        padding_height < 0 || groups <= 0 || output_width <= 0 ||
        output_height <= 0 || input_channels % groups != 0 ||
        output_channels % groups != 0) {
        return -2;
    }
    const int expected_width =
        (input_width + 2 * padding_width - kernel_width) / stride_width + 1;
    const int expected_height =
        (input_height + 2 * padding_height - kernel_height) / stride_height + 1;
    if (output_width != expected_width || output_height != expected_height) {
        return -3;
    }
    ck_audio_conv2d_whc_f32_args_t args = {
        .input = input,
        .weight = weight,
        .bias = bias,
        .output = output,
        .input_width = input_width,
        .input_height = input_height,
        .input_channels = input_channels,
        .output_channels = output_channels,
        .kernel_width = kernel_width,
        .kernel_height = kernel_height,
        .stride_width = stride_width,
        .stride_height = stride_height,
        .padding_width = padding_width,
        .padding_height = padding_height,
        .groups = groups,
        .output_width = output_width,
        .output_height = output_height,
        .use_width_simd = 1,
    };
    const char *disable_width_simd = getenv("CK_AUDIO_DISABLE_CONV2D_WIDTH_SIMD");
    if (disable_width_simd != NULL && disable_width_simd[0] != '\0' &&
        strcmp(disable_width_simd, "0") != 0) {
        args.use_width_simd = 0;
    }
    const int output_elements = output_channels * output_height * output_width;
    ck_threadpool_t *pool = ck_threadpool_global();
    int active = pool ? ck_threadpool_n_threads(pool) : 1;
    if (active > output_elements) {
        active = output_elements;
    }
    if (pool != NULL && active > 1) {
        const int grain = output_width > 0 ? output_width : 1;
        ck_threadpool_parallel_for_n(
            pool, active, 0, output_elements, grain,
            ck_audio_conv2d_whc_grouped_f32_range, &args);
    } else {
        ck_audio_conv2d_whc_grouped_f32_range(0, output_elements, &args);
    }
    return 0;
}

References ck_audio_conv2d_whc_grouped_f32_range(), ck_threadpool_global(), ck_threadpool_n_threads(), and ck_threadpool_parallel_for_n().

Referenced by audio_fastconformer_subsampling_f32().

audio_fastconformer_block_f32()

int audio_fastconformer_block_f32	(	const float *	input,
		const float *	relative_positions,
		const float *	ff1_norm_weight,
		const float *	ff1_norm_bias,
		const float *	ff1_up_weight,
		const float *	ff1_up_bias,
		const float *	ff1_down_weight,
		const float *	ff1_down_bias,
		const float *	attn_norm_weight,
		const float *	attn_norm_bias,
		const float *	q_weight,
		const float *	q_bias,
		const float *	k_weight,
		const float *	k_bias,
		const float *	v_weight,
		const float *	v_bias,
		const float *	relative_weight,
		const float *	attn_bias_u,
		const float *	attn_bias_v,
		const float *	attn_out_weight,
		const float *	attn_out_bias,
		const float *	conv_norm_weight,
		const float *	conv_norm_bias,
		const float *	conv_pw1_weight,
		const float *	conv_pw1_bias,
		const float *	conv_dw_weight,
		const float *	conv_dw_bias,
		const float *	conv_bn_mean,
		const float *	conv_bn_variance,
		const float *	conv_bn_weight,
		const float *	conv_bn_bias,
		const float *	conv_pw2_weight,
		const float *	conv_pw2_bias,
		const float *	ff2_norm_weight,
		const float *	ff2_norm_bias,
		const float *	ff2_up_weight,
		const float *	ff2_up_bias,
		const float *	ff2_down_weight,
		const float *	ff2_down_bias,
		const float *	out_norm_weight,
		const float *	out_norm_bias,
		float *	output,
		void *	workspace,
		size_t	workspace_bytes,
		int	frames,
		int	hidden_size,
		int	intermediate_size,
		int	heads,
		int	head_dim,
		int	conv_kernel_size,
		float	layer_norm_epsilon,
		float	batch_norm_epsilon
	)

Definition at line 1928 of file audio_kernels.c.

{
    if (input == NULL || relative_positions == NULL ||
        ff1_norm_weight == NULL || ff1_norm_bias == NULL ||
        ff1_up_weight == NULL || ff1_down_weight == NULL ||
        attn_norm_weight == NULL || attn_norm_bias == NULL ||
        q_weight == NULL || k_weight == NULL || v_weight == NULL ||
        relative_weight == NULL || attn_bias_u == NULL || attn_bias_v == NULL ||
        attn_out_weight == NULL || conv_norm_weight == NULL ||
        conv_norm_bias == NULL || conv_pw1_weight == NULL ||
        conv_dw_weight == NULL || conv_bn_mean == NULL ||
        conv_bn_variance == NULL || conv_bn_weight == NULL ||
        conv_bn_bias == NULL || conv_pw2_weight == NULL ||
        ff2_norm_weight == NULL || ff2_norm_bias == NULL ||
        ff2_up_weight == NULL || ff2_down_weight == NULL ||
        out_norm_weight == NULL || out_norm_bias == NULL || output == NULL ||
        workspace == NULL) {
        return -1;
    }
    if (frames <= 0 || hidden_size <= 0 || intermediate_size <= 0 ||
        heads <= 0 || head_dim <= 0 || heads > INT_MAX / head_dim ||
        heads * head_dim != hidden_size || conv_kernel_size <= 0 ||
        (conv_kernel_size & 1) == 0 || !isfinite(layer_norm_epsilon) ||
        layer_norm_epsilon <= 0.0f || !isfinite(batch_norm_epsilon) ||
        batch_norm_epsilon <= 0.0f) {
        return -2;
    }
    const size_t minimum_workspace =
        audio_fastconformer_block_workspace_bytes_for_score_slots(
            frames, hidden_size, intermediate_size, heads, (size_t)heads);
    if (minimum_workspace == 0 || workspace_bytes < minimum_workspace) {
        return -3;
    }
 
    const size_t token_elements = (size_t)frames * (size_t)hidden_size;
    const size_t ff_elements = (size_t)frames * (size_t)intermediate_size;
    const size_t relative_elements =
        (size_t)(2 * frames - 1) * (size_t)hidden_size;
    const size_t doubled_token_elements = 2u * token_elements;
    size_t large_elements = ff_elements;
    if (relative_elements > large_elements) large_elements = relative_elements;
    if (doubled_token_elements > large_elements) {
        large_elements = doubled_token_elements;
    }
 
    float *cursor = (float *)workspace;
    float *norm = cursor;
    cursor += token_elements;
    float *large = cursor;
    cursor += large_elements;
    float *buffer2 = cursor;
    cursor += token_elements;
    float *buffer3 = cursor;
    cursor += token_elements;
    float *buffer4 = cursor;
    cursor += token_elements;
    float *buffer5 = cursor;
    cursor += token_elements;
    float *scores = cursor;
    const size_t scores_offset = (size_t)(
        (unsigned char *)scores - (unsigned char *)workspace);
 
    memcpy(output, input, token_elements * sizeof(float));
 
    layernorm_naive_serial_matched_precision(
        output, ff1_norm_weight, ff1_norm_bias, norm, NULL, NULL,
        frames, hidden_size, layer_norm_epsilon);
    gemm_nt_f32_llama_production_parallel_dispatch(
        norm, ff1_up_weight, ff1_up_bias, large,
        frames, intermediate_size, hidden_size);
    recurrent_silu_forward(large, large, frames, intermediate_size);
    gemm_nt_f32_llama_production_parallel_dispatch(
        large, ff1_down_weight, ff1_down_bias, buffer2,
        frames, hidden_size, intermediate_size);
    if (audio_scaled_residual_add_f32(
            output, buffer2, 0.5f, output, token_elements) != 0) {
        return -10;
    }
 
    layernorm_naive_serial_matched_precision(
        output, attn_norm_weight, attn_norm_bias, norm, NULL, NULL,
        frames, hidden_size, layer_norm_epsilon);
    gemm_nt_f32_llama_production_parallel_dispatch(
        norm, q_weight, q_bias, buffer2, frames, hidden_size, hidden_size);
    gemm_nt_f32_llama_production_parallel_dispatch(
        norm, k_weight, k_bias, buffer3, frames, hidden_size, hidden_size);
    gemm_nt_f32_llama_production_parallel_dispatch(
        norm, v_weight, v_bias, buffer4, frames, hidden_size, hidden_size);
    gemm_nt_f32_llama_production_parallel_dispatch(
        relative_positions, relative_weight, NULL, large,
        2 * frames - 1, hidden_size, hidden_size);
    if (audio_conformer_relative_attention_f32(
            buffer2, buffer3, buffer4, large, attn_bias_u, attn_bias_v,
            buffer5, frames, heads, head_dim, 1.0f / sqrtf((float)head_dim),
            scores, workspace_bytes - scores_offset) != 0) {
        return -11;
    }
    gemm_nt_f32_llama_production_parallel_dispatch(
        buffer5, attn_out_weight, attn_out_bias, norm,
        frames, hidden_size, hidden_size);
    if (audio_scaled_residual_add_f32(
            output, norm, 1.0f, output, token_elements) != 0) {
        return -12;
    }
 
    layernorm_naive_serial_matched_precision(
        output, conv_norm_weight, conv_norm_bias, norm, NULL, NULL,
        frames, hidden_size, layer_norm_epsilon);
    for (int frame = 0; frame < frames; ++frame) {
        for (int channel = 0; channel < hidden_size; ++channel) {
            buffer2[(size_t)channel * (size_t)frames + (size_t)frame] =
                norm[(size_t)frame * (size_t)hidden_size + (size_t)channel];
        }
    }
    if (audio_conv1d_channel_major_grouped_f32(
            buffer2, conv_pw1_weight, conv_pw1_bias, large,
            hidden_size, 2 * hidden_size, frames, 1, 1, 0, 1, frames) != 0 ||
        audio_glu_split_channel_major_f32(
            large, buffer3, hidden_size, frames) != 0 ||
        audio_conv1d_channel_major_grouped_f32(
            buffer3, conv_dw_weight, conv_dw_bias, buffer4,
            hidden_size, hidden_size, frames, conv_kernel_size, 1,
            conv_kernel_size / 2, hidden_size, frames) != 0 ||
        audio_batch_norm_inference_channel_major_f32(
            buffer4, conv_bn_mean, conv_bn_variance, conv_bn_weight,
            conv_bn_bias, buffer5, hidden_size, frames,
            batch_norm_epsilon) != 0) {
        return -13;
    }
    recurrent_silu_forward(buffer5, buffer5, hidden_size, frames);
    if (audio_conv1d_channel_major_grouped_f32(
            buffer5, conv_pw2_weight, conv_pw2_bias, buffer3,
            hidden_size, hidden_size, frames, 1, 1, 0, 1, frames) != 0) {
        return -14;
    }
    for (int frame = 0; frame < frames; ++frame) {
        for (int channel = 0; channel < hidden_size; ++channel) {
            norm[(size_t)frame * (size_t)hidden_size + (size_t)channel] =
                buffer3[(size_t)channel * (size_t)frames + (size_t)frame];
        }
    }
    if (audio_scaled_residual_add_f32(
            output, norm, 1.0f, output, token_elements) != 0) {
        return -15;
    }
 
    layernorm_naive_serial_matched_precision(
        output, ff2_norm_weight, ff2_norm_bias, norm, NULL, NULL,
        frames, hidden_size, layer_norm_epsilon);
    gemm_nt_f32_llama_production_parallel_dispatch(
        norm, ff2_up_weight, ff2_up_bias, large,
        frames, intermediate_size, hidden_size);
    recurrent_silu_forward(large, large, frames, intermediate_size);
    gemm_nt_f32_llama_production_parallel_dispatch(
        large, ff2_down_weight, ff2_down_bias, buffer2,
        frames, hidden_size, intermediate_size);
    if (audio_scaled_residual_add_f32(
            output, buffer2, 0.5f, output, token_elements) != 0) {
        return -16;
    }
    layernorm_naive_serial_matched_precision(
        output, out_norm_weight, out_norm_bias, norm, NULL, NULL,
        frames, hidden_size, layer_norm_epsilon);
    memcpy(output, norm, token_elements * sizeof(float));
    return 0;
}

References audio_batch_norm_inference_channel_major_f32(), audio_conformer_relative_attention_f32(), audio_conv1d_channel_major_grouped_f32(), audio_fastconformer_block_workspace_bytes_for_score_slots(), audio_glu_split_channel_major_f32(), audio_scaled_residual_add_f32(), gemm_nt_f32_llama_production_parallel_dispatch(), layernorm_naive_serial_matched_precision(), and recurrent_silu_forward().

audio_fastconformer_block_workspace_bytes()

size_t audio_fastconformer_block_workspace_bytes	(	int	frames,
		int	hidden_size,
		int	intermediate_size,
		int	heads
	)

Definition at line 1914 of file audio_kernels.c.

{
    ck_threadpool_t *pool = ck_threadpool_global();
    const int requested_threads = pool ? ck_threadpool_n_threads(pool) : 1;
    const size_t score_slots = (size_t)(requested_threads > heads
        ? requested_threads : heads);
    return audio_fastconformer_block_workspace_bytes_for_score_slots(
        frames, hidden_size, intermediate_size, heads, score_slots);
}

References audio_fastconformer_block_workspace_bytes_for_score_slots(), ck_threadpool_global(), and ck_threadpool_n_threads().

audio_fastconformer_subsampling_f32()

int audio_fastconformer_subsampling_f32	(	const float *	features,
		const float *	conv0_weight,
		const float *	conv0_bias,
		const float *	depthwise1_weight,
		const float *	depthwise1_bias,
		const float *	pointwise1_weight,
		const float *	pointwise1_bias,
		const float *	depthwise2_weight,
		const float *	depthwise2_bias,
		const float *	pointwise2_weight,
		const float *	pointwise2_bias,
		const float *	linear_weight,
		const float *	linear_bias,
		float *	output,
		void *	workspace,
		size_t	workspace_bytes,
		int	feature_frames,
		int	live_frames,
		int	feature_channels,
		int	conv_channels,
		int	hidden_size,
		int	kernel_size,
		int	stride,
		int	output_capacity_frames,
		int *	output_frames
	)

Definition at line 1024 of file audio_kernels.c.

{
    if (features == NULL || conv0_weight == NULL || conv0_bias == NULL ||
        depthwise1_weight == NULL || depthwise1_bias == NULL ||
        pointwise1_weight == NULL || pointwise1_bias == NULL ||
        depthwise2_weight == NULL || depthwise2_bias == NULL ||
        pointwise2_weight == NULL || pointwise2_bias == NULL ||
        linear_weight == NULL || linear_bias == NULL || output == NULL ||
        workspace == NULL || output_frames == NULL || feature_frames <= 0 ||
        live_frames <= 0 || live_frames > feature_frames || feature_channels <= 0 ||
        conv_channels <= 0 || hidden_size <= 0 || kernel_size != 3 || stride != 2 ||
        output_capacity_frames <= 0) {
        return -1;
    }
 
    const size_t required_workspace = audio_fastconformer_subsampling_workspace_bytes(
        feature_frames, feature_channels, conv_channels, kernel_size, stride);
    if (required_workspace == 0 || workspace_bytes < required_workspace) {
        return -2;
    }
 
    const int padding = kernel_size / 2;
    int heights[3];
    int widths[3];
    int live_heights[3];
    heights[0] = conv_output_extent(feature_frames, kernel_size, stride, padding);
    widths[0] = conv_output_extent(feature_channels, kernel_size, stride, padding);
    live_heights[0] = conv_output_extent(live_frames, kernel_size, stride, padding);
    for (int stage = 1; stage < 3; ++stage) {
        heights[stage] = conv_output_extent(heights[stage - 1], kernel_size, stride, padding);
        widths[stage] = conv_output_extent(widths[stage - 1], kernel_size, stride, padding);
        live_heights[stage] = conv_output_extent(
            live_heights[stage - 1], kernel_size, stride, padding);
    }
    if (heights[2] <= 0 || widths[2] <= 0 || live_heights[2] <= 0 ||
        output_capacity_frames < heights[2]) {
        return -3;
    }
 
    size_t stage0_elements = 0;
    if (checked_mul_size((size_t)conv_channels, (size_t)heights[0], &stage0_elements) != 0 ||
        checked_mul_size(stage0_elements, (size_t)widths[0], &stage0_elements) != 0) {
        return -4;
    }
    float *ping = (float *)workspace;
    float *pong = ping + stage0_elements;
 
    int status = audio_conv2d_whc_grouped_f32(
        features, conv0_weight, conv0_bias, ping,
        feature_channels, feature_frames, 1, conv_channels,
        kernel_size, kernel_size, stride, stride, padding, padding, 1,
        widths[0], heights[0]);
    if (status != 0) return -10;
    zero_subsampling_padding(ping, conv_channels, heights[0], widths[0], live_heights[0]);
    relu_subsampling_inplace(ping, stage0_elements);
 
    status = audio_conv2d_whc_grouped_f32(
        ping, depthwise1_weight, depthwise1_bias, pong,
        widths[0], heights[0], conv_channels, conv_channels,
        kernel_size, kernel_size, stride, stride, padding, padding, conv_channels,
        widths[1], heights[1]);
    if (status != 0) return -11;
    zero_subsampling_padding(pong, conv_channels, heights[1], widths[1], live_heights[1]);
 
    status = audio_conv2d_whc_grouped_f32(
        pong, pointwise1_weight, pointwise1_bias, ping,
        widths[1], heights[1], conv_channels, conv_channels,
        1, 1, 1, 1, 0, 0, 1, widths[1], heights[1]);
    if (status != 0) return -12;
    size_t stage1_elements = 0;
    if (checked_mul_size((size_t)conv_channels, (size_t)heights[1], &stage1_elements) != 0 ||
        checked_mul_size(stage1_elements, (size_t)widths[1], &stage1_elements) != 0) {
        return -4;
    }
    zero_subsampling_padding(ping, conv_channels, heights[1], widths[1], live_heights[1]);
    relu_subsampling_inplace(ping, stage1_elements);
 
    status = audio_conv2d_whc_grouped_f32(
        ping, depthwise2_weight, depthwise2_bias, pong,
        widths[1], heights[1], conv_channels, conv_channels,
        kernel_size, kernel_size, stride, stride, padding, padding, conv_channels,
        widths[2], heights[2]);
    if (status != 0) return -13;
    zero_subsampling_padding(pong, conv_channels, heights[2], widths[2], live_heights[2]);
 
    status = audio_conv2d_whc_grouped_f32(
        pong, pointwise2_weight, pointwise2_bias, ping,
        widths[2], heights[2], conv_channels, conv_channels,
        1, 1, 1, 1, 0, 0, 1, widths[2], heights[2]);
    if (status != 0) return -14;
    size_t final_elements = 0;
    if (checked_mul_size((size_t)conv_channels, (size_t)heights[2], &final_elements) != 0 ||
        checked_mul_size(final_elements, (size_t)widths[2], &final_elements) != 0) {
        return -4;
    }
    zero_subsampling_padding(ping, conv_channels, heights[2], widths[2], live_heights[2]);
    relu_subsampling_inplace(ping, final_elements);
 
    if (widths[2] > INT_MAX / conv_channels) return -4;
    const int flattened_width = conv_channels * widths[2];
    for (int frame = 0; frame < heights[2]; ++frame) {
        float *row = pong + (size_t)frame * (size_t)flattened_width;
        for (int channel = 0; channel < conv_channels; ++channel) {
            const float *source = ping +
                ((size_t)channel * (size_t)heights[2] + (size_t)frame) * (size_t)widths[2];
            memcpy(row + (size_t)channel * (size_t)widths[2], source,
                   (size_t)widths[2] * sizeof(float));
        }
    }
 
    gemm_nt_f32_llama_production_parallel_dispatch(
        pong, linear_weight, linear_bias, output,
        heights[2], hidden_size, flattened_width);
    *output_frames = heights[2];
    return 0;
}

References audio_conv2d_whc_grouped_f32(), audio_fastconformer_subsampling_workspace_bytes(), checked_mul_size(), conv_output_extent(), gemm_nt_f32_llama_production_parallel_dispatch(), relu_subsampling_inplace(), and zero_subsampling_padding().

audio_fastconformer_subsampling_workspace_bytes()

size_t audio_fastconformer_subsampling_workspace_bytes	(	int	feature_frames,
		int	feature_channels,
		int	conv_channels,
		int	kernel_size,
		int	stride
	)

Definition at line 967 of file audio_kernels.c.

{
    if (feature_frames <= 0 || feature_channels <= 0 || conv_channels <= 0 ||
        kernel_size != 3 || stride != 2) {
        return 0;
    }
    const int padding = kernel_size / 2;
    const int stage_frames = conv_output_extent(
        feature_frames, kernel_size, stride, padding);
    const int stage_width = conv_output_extent(
        feature_channels, kernel_size, stride, padding);
    if (stage_frames <= 0 || stage_width <= 0) {
        return 0;
    }
    size_t elements = 0;
    size_t bytes = 0;
    if (checked_mul_size((size_t)stage_frames, (size_t)stage_width, &elements) != 0 ||
        checked_mul_size(elements, (size_t)conv_channels, &elements) != 0 ||
        checked_mul_size(elements, 2u * sizeof(float), &bytes) != 0) {
        return 0;
    }
    return bytes;
}

References checked_mul_size(), and conv_output_extent().

Referenced by audio_fastconformer_subsampling_f32().

audio_feature_normalize_per_feature_f32()

int audio_feature_normalize_per_feature_f32	(	const float *	input,
		float *	output,
		int	channels,
		int	frames,
		float	epsilon
	)

Definition at line 398 of file audio_kernels.c.

{
    if (input == NULL || output == NULL) {
        return -1;
    }
    if (channels <= 0 || frames <= 0 || !isfinite(epsilon) || epsilon < 0.0f) {
        return -2;
    }
    const int denominator = frames > 1 ? frames - 1 : 1;
    for (int channel = 0; channel < channels; ++channel) {
        double sum = 0.0;
        double sum_squared_difference = 0.0;
        for (int frame = 0; frame < frames; ++frame) {
            sum += (double)input[(size_t)frame * channels + channel];
        }
        const double mean = sum / (double)frames;
        for (int frame = 0; frame < frames; ++frame) {
            const double difference =
                (double)input[(size_t)frame * channels + channel] - mean;
            sum_squared_difference += difference * difference;
        }
        float standard_deviation = sqrtf(
            (float)(sum_squared_difference / (double)denominator));
        if (isnan(standard_deviation)) {
            standard_deviation = 0.0f;
        }
        const float inverse_standard_deviation =
            1.0f / (standard_deviation + epsilon);
        for (int frame = 0; frame < frames; ++frame) {
            const size_t index = (size_t)frame * channels + channel;
            output[index] = (float)((double)input[index] - mean) *
                inverse_standard_deviation;
        }
    }
    return 0;
}

audio_glu_split_channel_major_f32()

int audio_glu_split_channel_major_f32	(	const float *	input,
		float *	output,
		int	channels,
		int	frames
	)

Definition at line 1596 of file audio_kernels.c.

{
    if (input == NULL || output == NULL) {
        return -1;
    }
    if (channels <= 0 || frames <= 0) {
        return -2;
    }
    const int elements = channels * frames;
    ck_audio_glu_split_f32_args_t args = {
        .value = input,
        .gate = input + elements,
        .output = output,
    };
    ck_threadpool_t *pool = ck_threadpool_global();
    int active = pool ? ck_threadpool_n_threads(pool) : 1;
    if (active > elements) {
        active = elements;
    }
    if (pool != NULL && active > 1) {
        ck_threadpool_parallel_for_n(
            pool, active, 0, elements, 256,
            ck_audio_glu_split_f32_range, &args);
    } else {
        ck_audio_glu_split_f32_range(0, elements, &args);
    }
    return 0;
}

References ck_audio_glu_split_f32_range(), ck_threadpool_global(), ck_threadpool_n_threads(), and ck_threadpool_parallel_for_n().

Referenced by audio_fastconformer_block_f32().

audio_hann_window_f32()

int audio_hann_window_f32	(	float *	output,
		int	frames,
		int	periodic
	)

Definition at line 357 of file audio_kernels.c.

{
    if (output == NULL) {
        return -1;
    }
    if (frames <= 1 || (periodic != 0 && periodic != 1)) {
        return -2;
    }
    const double denominator = (double)(periodic ? frames : frames - 1);
    for (int frame = 0; frame < frames; ++frame) {
        const double phase = 2.0 * CK_AUDIO_PI_D * (double)frame / denominator;
        output[frame] = (float)(0.5 - 0.5 * cos(phase));
    }
    return 0;
}

References CK_AUDIO_PI_D.

audio_log_mel_time_major_f32()

int audio_log_mel_time_major_f32	(	const float *	power,
		const float *	mel_filters,
		float *	log_mel,
		int	frames,
		int	bins,
		int	channels,
		float	epsilon
	)

Definition at line 750 of file audio_kernels.c.

{
    if (power == NULL || mel_filters == NULL || log_mel == NULL) {
        return -1;
    }
    if (frames <= 0 || bins <= 0 || channels <= 0 || epsilon <= 0.0f) {
        return -2;
    }
    for (int frame = 0; frame < frames; ++frame) {
        const float *spectrum = power + (size_t)frame * bins;
        float *output = log_mel + (size_t)frame * channels;
        for (int channel = 0; channel < channels; ++channel) {
            const float *filter = mel_filters + (size_t)channel * bins;
            float sum = 0.0f;
            for (int bin = 0; bin < bins; ++bin) {
                sum = fmaf(spectrum[bin], filter[bin], sum);
            }
            output[channel] = logf(sum + epsilon);
        }
    }
    return 0;
}

audio_lstm_bidirectional_scan_f32()

int audio_lstm_bidirectional_scan_f32	(	const float *	input,
		size_t	input_elements,
		const float *	weight_ih,
		size_t	weight_ih_elements,
		const float *	weight_hh,
		size_t	weight_hh_elements,
		const float *	bias_ih,
		size_t	bias_ih_elements,
		const float *	bias_hh,
		size_t	bias_hh_elements,
		float *	output,
		size_t	output_elements,
		float *	hidden_state,
		size_t	hidden_state_elements,
		float *	cell_state,
		size_t	cell_state_elements,
		float *	gates_scratch,
		size_t	gates_scratch_bytes,
		int	tokens,
		int	input_size,
		int	hidden_size,
		size_t	input_stride,
		size_t	output_stride
	)

Definition at line 31 of file audio_lstm_scan.c.

{
    if (input == NULL || weight_ih == NULL || weight_hh == NULL ||
        bias_ih == NULL || bias_hh == NULL || output == NULL ||
        hidden_state == NULL || cell_state == NULL || gates_scratch == NULL) {
        return -1;
    }
    if (tokens <= 0 || input_size <= 0 || hidden_size <= 0 ||
        hidden_size > INT_MAX / 4) {
        return -2;
    }
 
    const size_t token_count = (size_t)tokens;
    const size_t input_width = (size_t)input_size;
    const size_t hidden_width = (size_t)hidden_size;
    const size_t output_width = hidden_width * 2u;
    size_t input_required, output_required, gates_count;
    size_t ih_per_direction, hh_per_direction, bias_required;
    size_t ih_required, hh_required, gates_bytes, state_bytes;
    if (input_stride < input_width || output_stride < output_width ||
        !checked_span(token_count, input_stride, input_width, &input_required) ||
        !checked_span(token_count, output_stride, output_width, &output_required) ||
        !checked_product(hidden_width, 4u, &gates_count) ||
        !checked_product(gates_count, input_width, &ih_per_direction) ||
        !checked_product(gates_count, hidden_width, &hh_per_direction) ||
        !checked_product(ih_per_direction, 2u, &ih_required) ||
        !checked_product(hh_per_direction, 2u, &hh_required) ||
        !checked_product(gates_count, 2u, &bias_required) ||
        !checked_product(gates_count, sizeof(float), &gates_bytes) ||
        !checked_product(output_width, sizeof(float), &state_bytes)) {
        return -2;
    }
    if (input_elements < input_required || output_elements < output_required ||
        weight_ih_elements < ih_required || weight_hh_elements < hh_required ||
        bias_ih_elements < bias_required || bias_hh_elements < bias_required ||
        hidden_state_elements < output_width ||
        cell_state_elements < output_width ||
        gates_scratch_bytes < gates_bytes) {
        return -3;
    }
 
    /* Every invocation starts from zero state, as a batch-one PyTorch LSTM
     * called without initial (h, c) does. Padding outside valid rows is not touched.
     */
    memset(hidden_state, 0, state_bytes);
    memset(cell_state, 0, state_bytes);
    for (size_t index = 0u; index < token_count; ++index) {
        const int status = audio_lstm_step_f32(
            input + index * input_stride,
            weight_ih, weight_hh, bias_ih, bias_hh,
            hidden_state, cell_state,
            output + index * output_stride,
            gates_scratch, gates_scratch_bytes, input_size, hidden_size);
        if (status != 0) {
            return status;
        }
    }
    for (size_t reverse_index = 0u; reverse_index < token_count;
         ++reverse_index) {
        const size_t index = token_count - 1u - reverse_index;
        const int status = audio_lstm_step_f32(
            input + index * input_stride,
            weight_ih + ih_per_direction,
            weight_hh + hh_per_direction,
            bias_ih + gates_count,
            bias_hh + gates_count,
            hidden_state + hidden_width,
            cell_state + hidden_width,
            output + index * output_stride + hidden_width,
            gates_scratch, gates_scratch_bytes, input_size, hidden_size);
        if (status != 0) {
            return status;
        }
    }
    return 0;
}

References audio_lstm_step_f32(), checked_product(), and checked_span().

audio_lstm_step_f32()

int audio_lstm_step_f32	(	const float *	input,
		const float *	weight_ih,
		const float *	weight_hh,
		const float *	bias_ih,
		const float *	bias_hh,
		float *	hidden_state,
		float *	cell_state,
		float *	output,
		float *	gates_scratch,
		size_t	gates_scratch_bytes,
		int	input_size,
		int	hidden_size
	)

Definition at line 505 of file audio_kernels.c.

{
    if (input == NULL || weight_ih == NULL || weight_hh == NULL ||
        bias_ih == NULL || bias_hh == NULL || hidden_state == NULL ||
        cell_state == NULL || output == NULL || gates_scratch == NULL) {
        return -1;
    }
    /* The IFGO update indexes gates[3 * hidden_size + index] with signed int. */
    if (input_size <= 0 || hidden_size <= 0 ||
        hidden_size > INT_MAX / 4 ||
        (size_t)hidden_size > SIZE_MAX / 4u) {
        return -2;
    }
    const size_t gate_count = (size_t)hidden_size * 4u;
    if (gate_count > SIZE_MAX / sizeof(float) ||
        gates_scratch_bytes < gate_count * sizeof(float)) {
        return -3;
    }
 
    for (size_t gate = 0; gate < gate_count; ++gate) {
        float sum = bias_ih[gate] + bias_hh[gate];
        const float *input_weight = weight_ih + gate * (size_t)input_size;
        const float *hidden_weight = weight_hh + gate * (size_t)hidden_size;
        for (int index = 0; index < input_size; ++index) {
            sum += input_weight[index] * input[index];
        }
        for (int index = 0; index < hidden_size; ++index) {
            sum += hidden_weight[index] * hidden_state[index];
        }
        gates_scratch[gate] = sum;
    }
 
    for (int index = 0; index < hidden_size; ++index) {
        const float input_gate = 1.0f /
            (1.0f + expf(-gates_scratch[index]));
        const float forget_gate = 1.0f /
            (1.0f + expf(-gates_scratch[hidden_size + index]));
        const float cell_gate = tanhf(gates_scratch[2 * hidden_size + index]);
        const float output_gate = 1.0f /
            (1.0f + expf(-gates_scratch[3 * hidden_size + index]));
        const float cell = forget_gate * cell_state[index] +
            input_gate * cell_gate;
        const float hidden = output_gate * tanhf(cell);
        cell_state[index] = cell;
        hidden_state[index] = hidden;
        output[index] = hidden;
    }
    return 0;
}

Referenced by audio_lstm_bidirectional_scan_f32().

audio_pad_or_truncate_f32()

int audio_pad_or_truncate_f32	(	const float *	input,
		int	input_frames,
		float *	output,
		int	output_frames
	)

Definition at line 334 of file audio_kernels.c.

{
    if (input == NULL || output == NULL) {
        return -1;
    }
    if (input_frames <= 0 || output_frames <= 0) {
        return -2;
    }
    const int copied = input_frames < output_frames ? input_frames : output_frames;
    memmove(output, input, (size_t)copied * sizeof(float));
    if (copied < output_frames) {
        memset(
            output + copied,
            0,
            (size_t)(output_frames - copied) * sizeof(float));
    }
    return copied;
}

audio_pcm_s16_to_mono_f32()

int audio_pcm_s16_to_mono_f32	(	const int16_t *	interleaved,
		int	n_frames,
		int	n_channels,
		float *	mono
	)

Definition at line 230 of file audio_kernels.c.

{
    if (interleaved == NULL || mono == NULL) {
        return -1;
    }
    if (n_frames <= 0 || n_channels <= 0) {
        return -2;
    }
    const float scale = 1.0f / 32768.0f;
    for (int frame = 0; frame < n_frames; ++frame) {
        float sum = 0.0f;
        for (int channel = 0; channel < n_channels; ++channel) {
            sum += (float)interleaved[(size_t)frame * n_channels + channel];
        }
        mono[frame] = (sum / (float)n_channels) * scale;
    }
    return 0;
}

audio_preemphasis_f32()

int audio_preemphasis_f32	(	const float *	input,
		float *	output,
		int	frames,
		float	coefficient
	)

Definition at line 373 of file audio_kernels.c.

{
    if (input == NULL || output == NULL) {
        return -1;
    }
    if (frames <= 0 || !isfinite(coefficient)) {
        return -2;
    }
    if (input == output) {
        for (int frame = frames - 1; frame > 0; --frame) {
            output[frame] = input[frame] - coefficient * input[frame - 1];
        }
    } else {
        output[0] = input[0];
        for (int frame = 1; frame < frames; ++frame) {
            output[frame] = input[frame] - coefficient * input[frame - 1];
        }
    }
    return 0;
}

audio_relative_shift_f32()

int audio_relative_shift_f32	(	const float *	raw_scores,
		float *	scores,
		int	heads,
		int	query_frames
	)

Definition at line 1654 of file audio_kernels.c.

{
    if (raw_scores == NULL || scores == NULL) {
        return -1;
    }
    if (heads <= 0 || query_frames <= 0) {
        return -2;
    }
    const int rows = heads * query_frames;
    ck_audio_relative_shift_f32_args_t args = {
        .raw_scores = raw_scores,
        .scores = scores,
        .query_frames = query_frames,
        .raw_key_frames = 2 * query_frames - 1,
    };
    ck_threadpool_t *pool = ck_threadpool_global();
    int active = pool ? ck_threadpool_n_threads(pool) : 1;
    if (active > rows) {
        active = rows;
    }
    if (pool != NULL && active > 1) {
        ck_threadpool_parallel_for_n(
            pool, active, 0, rows, 1,
            ck_audio_relative_shift_f32_range, &args);
    } else {
        ck_audio_relative_shift_f32_range(0, rows, &args);
    }
    return 0;
}

References ck_audio_relative_shift_f32_range(), ck_threadpool_global(), ck_threadpool_n_threads(), and ck_threadpool_parallel_for_n().

audio_relative_sinusoidal_position_f32()

int audio_relative_sinusoidal_position_f32	(	float *	output,
		int	frames,
		int	channels
	)

Definition at line 440 of file audio_kernels.c.

{
    if (output == NULL) {
        return -1;
    }
    if (frames <= 0 || frames > (INT_MAX / 2) + 1 ||
        channels <= 0 || (channels & 1) != 0) {
        return -2;
    }
    if ((size_t)(2 * frames - 1) > SIZE_MAX / (size_t)channels) {
        return -3;
    }
    const int positions = 2 * frames - 1;
    for (int row = 0; row < positions; ++row) {
        const float position = (float)(frames - 1 - row);
        for (int channel = 0; channel < channels; channel += 2) {
            const float exponent = (float)channel / (float)channels;
            const float inverse_frequency = 1.0f / powf(10000.0f, exponent);
            const float frequency = inverse_frequency * position;
            output[(size_t)row * channels + channel] = sinf(frequency);
            output[(size_t)row * channels + channel + 1] = cosf(frequency);
        }
    }
    return 0;
}

audio_resample_linear_f32()

int audio_resample_linear_f32	(	const float *	input,
		int	input_frames,
		int	input_rate,
		float *	output,
		int	output_frames,
		int	output_rate
	)

Definition at line 264 of file audio_kernels.c.

{
    if (input == NULL || output == NULL) {
        return -1;
    }
    const int expected = audio_resampled_frame_count(input_frames, input_rate, output_rate);
    if (expected <= 0 || output_frames != expected) {
        return -2;
    }
    for (int frame = 0; frame < output_frames; ++frame) {
        const long long numerator = (long long)frame * input_rate;
        const int left = (int)(numerator / output_rate);
        const int right = left + 1 < input_frames ? left + 1 : left;
        const float fraction = (float)(numerator % output_rate) / (float)output_rate;
        output[frame] = fmaf(input[right] - input[left], fraction, input[left]);
    }
    return 0;
}

References audio_resampled_frame_count(), left, and right.

audio_resample_windowed_sinc_f32()

int audio_resample_windowed_sinc_f32	(	const float *	input,
		int	input_frames,
		int	input_rate,
		float *	output,
		int	output_frames,
		int	output_rate,
		int	radius
	)

Definition at line 289 of file audio_kernels.c.

{
    if (input == NULL || output == NULL) {
        return -1;
    }
    const int expected = audio_resampled_frame_count(input_frames, input_rate, output_rate);
    if (expected <= 0 || output_frames != expected || radius < 2 || radius > 128) {
        return -2;
    }
    const double ratio = (double)output_rate / (double)input_rate;
    const double cutoff = ratio < 1.0 ? ratio : 1.0;
    for (int frame = 0; frame < output_frames; ++frame) {
        const double source = (double)frame * (double)input_rate / (double)output_rate;
        const int center = (int)floor(source);
        double weighted = 0.0;
        double weight_sum = 0.0;
        for (int tap = center - radius + 1; tap <= center + radius; ++tap) {
            if (tap < 0 || tap >= input_frames) {
                continue;
            }
            const double distance = source - (double)tap;
            const double scaled = cutoff * distance;
            const double sinc = fabs(scaled) < 1.0e-12 ? 1.0 :
                sin(CK_AUDIO_PI_D * scaled) / (CK_AUDIO_PI_D * scaled);
            const double window_x = distance / (double)radius;
            if (fabs(window_x) >= 1.0) {
                continue;
            }
            const double window = 0.5 * (1.0 + cos(CK_AUDIO_PI_D * window_x));
            const double weight = cutoff * sinc * window;
            weighted += (double)input[tap] * weight;
            weight_sum += weight;
        }
        output[frame] = weight_sum != 0.0 ? (float)(weighted / weight_sum) : 0.0f;
    }
    return 0;
}

References audio_resampled_frame_count(), and CK_AUDIO_PI_D.

audio_resampled_frame_count()

int audio_resampled_frame_count	(	int	input_frames,
		int	input_rate,
		int	output_rate
	)

Definition at line 253 of file audio_kernels.c.

{
    if (input_frames <= 0 || input_rate <= 0 || output_rate <= 0) {
        return -1;
    }
    return 1 + (int)(((long long)(input_frames - 1) * output_rate) / input_rate);
}

Referenced by audio_resample_linear_f32(), and audio_resample_windowed_sinc_f32().

audio_scaled_residual_add_f32()

int audio_scaled_residual_add_f32	(	const float *	residual,
		const float *	branch,
		float	scale,
		float *	output,
		size_t	elements
	)

Definition at line 1852 of file audio_kernels.c.

{
    if (residual == NULL || branch == NULL || output == NULL) {
        return -1;
    }
    if (elements == 0 || !isfinite(scale)) {
        return -2;
    }
    for (size_t index = 0; index < elements; ++index) {
        /* Preserve the pinned elementwise multiply then add cast boundary. */
        volatile float scaled = branch[index] * scale;
        output[index] = residual[index] + scaled;
    }
    return 0;
}

Referenced by audio_fastconformer_block_f32().

audio_stft_power_centered_window_f32()

int audio_stft_power_centered_window_f32	(	const float *	samples,
		int	n_samples,
		const float *	window,
		int	window_length,
		const float *	cos_table,
		const float *	sin_table,
		int	n_fft,
		int	hop_length,
		int	reflect_padding,
		float *	power,
		int	n_frames
	)

Definition at line 695 of file audio_kernels.c.

{
    if (samples == NULL || window == NULL || cos_table == NULL ||
        sin_table == NULL || power == NULL) {
        return -1;
    }
    if (n_samples <= 0 || window_length <= 0 || n_fft <= 0 ||
        window_length > n_fft || (n_fft & 1) != 0 || hop_length <= 0 ||
        n_frames <= 0 || (reflect_padding != 0 && reflect_padding != 1)) {
        return -2;
    }
    if (n_frames != n_samples / hop_length + 1) {
        return -3;
    }
 
    const int bins = n_fft / 2 + 1;
    const int center = n_fft / 2;
    const int window_start = (n_fft - window_length) / 2;
    for (int frame = 0; frame < n_frames; ++frame) {
        for (int bin = 0; bin < bins; ++bin) {
            const float *cos_row = cos_table + (size_t)bin * n_fft;
            const float *sin_row = sin_table + (size_t)bin * n_fft;
            float real = 0.0f;
            float imag = 0.0f;
            for (int sample = 0; sample < window_length; ++sample) {
                const int fft_sample = window_start + sample;
                int source = frame * hop_length + fft_sample - center;
                if (source < 0 || source >= n_samples) {
                    if (!reflect_padding) {
                        continue;
                    }
                    source = reflect_index(source, n_samples);
                }
                const float value = samples[source] * window[sample];
                real = fmaf(value, cos_row[fft_sample], real);
                imag = fmaf(value, sin_row[fft_sample], imag);
            }
            power[(size_t)frame * bins + bin] =
                fmaf(real, real, imag * imag);
        }
    }
    return 0;
}

References reflect_index().

audio_stft_power_fft400_f32()

int audio_stft_power_fft400_f32	(	const float *	samples,
		int	n_samples,
		const float *	window,
		const float *	cos_table,
		const float *	sin_table,
		int	hop_length,
		float *	power,
		int	n_frames,
		float *	fft_scratch
	)

Definition at line 790 of file audio_kernels.c.

{
    const int n_fft = CK_AUDIO_WHISPER_N_FFT;
    if (samples == NULL || window == NULL || cos_table == NULL ||
        sin_table == NULL || power == NULL || fft_scratch == NULL) {
        return -1;
    }
    if (hop_length != CK_AUDIO_WHISPER_HOP_LENGTH ||
        n_samples <= n_fft / 2 || n_frames <= 0 ||
        n_frames != n_samples / hop_length) {
        return -2;
    }
    for (int frame = 0; frame < n_frames; ++frame) {
        audio_stft_power_fft400_frame_f32(
            samples,
            n_samples,
            frame,
            window,
            cos_table,
            sin_table,
            power + (size_t)frame * CK_AUDIO_WHISPER_POWER_BINS,
            fft_scratch);
    }
    return 0;
}

References audio_stft_power_fft400_frame_f32(), CK_AUDIO_WHISPER_HOP_LENGTH, CK_AUDIO_WHISPER_N_FFT, and CK_AUDIO_WHISPER_POWER_BINS.

audio_stft_power_precomputed_f32()

int audio_stft_power_precomputed_f32	(	const float *	samples,
		int	n_samples,
		const float *	window,
		const float *	cos_table,
		const float *	sin_table,
		int	n_fft,
		int	hop_length,
		float *	power,
		int	n_frames
	)

Definition at line 651 of file audio_kernels.c.

{
    if (samples == NULL || window == NULL || cos_table == NULL ||
        sin_table == NULL || power == NULL) {
        return -1;
    }
    if (n_fft <= 0 || hop_length <= 0 || n_samples <= n_fft / 2 ||
        (n_fft & 1) != 0 || n_frames <= 0) {
        return -2;
    }
    if (n_frames != n_samples / hop_length) {
        return -3;
    }
    const int bins = n_fft / 2 + 1;
    const int center = n_fft / 2;
    for (int frame = 0; frame < n_frames; ++frame) {
        for (int bin = 0; bin < bins; ++bin) {
            const float *cos_row = cos_table + (size_t)bin * n_fft;
            const float *sin_row = sin_table + (size_t)bin * n_fft;
            float real = 0.0f;
            float imag = 0.0f;
            for (int sample = 0; sample < n_fft; ++sample) {
                const int source = reflect_index(
                    frame * hop_length + sample - center, n_samples);
                const float value = samples[source] * window[sample];
                real = fmaf(value, cos_row[sample], real);
                imag = fmaf(value, sin_row[sample], imag);
            }
            power[(size_t)frame * bins + bin] =
                fmaf(real, real, imag * imag);
        }
    }
    return 0;
}

References reflect_index().

audio_stft_precompute_tables_f32()

int audio_stft_precompute_tables_f32	(	int	n_fft,
		float *	window,
		float *	cos_table,
		float *	sin_table
	)

Definition at line 567 of file audio_kernels.c.

{
    if (window == NULL || cos_table == NULL || sin_table == NULL) {
        return -1;
    }
    if (n_fft <= 0 || (n_fft & 1) != 0) {
        return -2;
    }
    const int bins = n_fft / 2 + 1;
    for (int sample = 0; sample < n_fft; ++sample) {
        window[sample] = 0.5f - 0.5f * cosf(
            2.0f * CK_AUDIO_PI_F * (float)sample / (float)n_fft);
    }
    for (int bin = 0; bin < bins; ++bin) {
        for (int sample = 0; sample < n_fft; ++sample) {
            const float angle = -2.0f * CK_AUDIO_PI_F *
                (float)(bin * sample) / (float)n_fft;
            const size_t index = (size_t)bin * n_fft + sample;
            cos_table[index] = cosf(angle);
            sin_table[index] = sinf(angle);
        }
    }
    return 0;
}

References CK_AUDIO_PI_F.

audio_transpose_channel_to_token_f32()

int audio_transpose_channel_to_token_f32	(	const float *	input,
		float *	output,
		int	channels,
		int	frames
	)

Definition at line 1831 of file audio_kernels.c.

{
    if (input == NULL || output == NULL) {
        return -1;
    }
    if (channels <= 0 || frames <= 0) {
        return -2;
    }
    for (int frame = 0; frame < frames; ++frame) {
        for (int channel = 0; channel < channels; ++channel) {
            output[(size_t)frame * channels + channel] =
                input[(size_t)channel * frames + frame];
        }
    }
    return 0;
}

audio_wav_decode_memory_pcm16_mono_f32()

int audio_wav_decode_memory_pcm16_mono_f32	(	const uint8_t *	bytes,
		size_t	byte_count,
		float *	mono,
		int	mono_capacity,
		CKAudioWavInfo *	info
	)

Definition at line 187 of file audio_kernels.c.

{
    return audio_wav_decode_memory_pcm16_mono_window_f32(
        bytes, byte_count, 0, mono, mono_capacity, info);
}

References audio_wav_decode_memory_pcm16_mono_window_f32().

audio_wav_decode_memory_pcm16_mono_window_f32()

int audio_wav_decode_memory_pcm16_mono_window_f32	(	const uint8_t *	bytes,
		size_t	byte_count,
		int	start_frame,
		float *	mono,
		int	mono_capacity,
		CKAudioWavInfo *	info
	)

Definition at line 198 of file audio_kernels.c.

{
    const int status = audio_wav_parse_memory(bytes, byte_count, info);
    if (status != 0) {
        return status;
    }
    if (start_frame < 0 || start_frame >= info->frames || mono_capacity <= 0) {
        return -7;
    }
    const int available = info->frames - start_frame;
    const int decoded = available < mono_capacity ? available : mono_capacity;
    const uint8_t *pcm = bytes + info->data_offset;
    const float scale = 1.0f / 32768.0f;
    for (int frame = 0; frame < decoded; ++frame) {
        float sum = 0.0f;
        const size_t source_frame = (size_t)start_frame + (size_t)frame;
        for (int channel = 0; channel < info->channels; ++channel) {
            const size_t index =
                (source_frame * (size_t)info->channels + (size_t)channel) * 2u;
            sum += (float)(int16_t)read_u16_le(pcm + index);
        }
        mono[frame] = (sum / (float)info->channels) * scale;
    }
    return decoded;
}

References audio_wav_parse_memory(), CKAudioWavInfo::channels, CKAudioWavInfo::data_offset, CKAudioWavInfo::frames, and read_u16_le().

Referenced by audio_wav_decode_memory_pcm16_mono_f32().

audio_wav_decode_pcm16_mono_f32()

int audio_wav_decode_pcm16_mono_f32	(	const uint8_t *	bytes,
		size_t	byte_count,
		const CKAudioWavInfo *	info,
		float *	mono,
		int	mono_capacity
	)

Definition at line 159 of file audio_kernels.c.

{
    if (bytes == NULL || info == NULL || mono == NULL) {
        return -1;
    }
    if (info->format_tag != 1 || info->bits_per_sample != 16 ||
        info->channels <= 0 || info->frames <= 0 || mono_capacity < info->frames ||
        info->data_offset > byte_count || info->data_bytes > byte_count - info->data_offset) {
        return -2;
    }
    const uint8_t *pcm = bytes + info->data_offset;
    const float scale = 1.0f / 32768.0f;
    for (int frame = 0; frame < info->frames; ++frame) {
        float sum = 0.0f;
        for (int channel = 0; channel < info->channels; ++channel) {
            const size_t index = ((size_t)frame * info->channels + channel) * 2u;
            sum += (float)(int16_t)read_u16_le(pcm + index);
        }
        mono[frame] = (sum / (float)info->channels) * scale;
    }
    return info->frames;
}

References CKAudioWavInfo::bits_per_sample, CKAudioWavInfo::channels, CKAudioWavInfo::data_bytes, CKAudioWavInfo::data_offset, CKAudioWavInfo::format_tag, CKAudioWavInfo::frames, and read_u16_le().

Referenced by audio_whisper_log_mel_window_wav_pcm16_f32().

audio_wav_parse_memory()

int audio_wav_parse_memory	(	const uint8_t *	bytes,
		size_t	byte_count,
		CKAudioWavInfo *	info
	)

Definition at line 98 of file audio_kernels.c.

{
    if (bytes == NULL || info == NULL) {
        return -1;
    }
    if (byte_count < 12 || memcmp(bytes, "RIFF", 4) != 0 ||
        memcmp(bytes + 8, "WAVE", 4) != 0) {
        return -2;
    }
    const size_t riff_end = (size_t)read_u32_le(bytes + 4) + 8u;
    if (riff_end < 12 || riff_end > byte_count) {
        return -3;
    }
    memset(info, 0, sizeof(*info));
    int found_format = 0;
    int found_data = 0;
    size_t offset = 12;
    while (offset + 8 <= riff_end) {
        const uint8_t *chunk = bytes + offset;
        const uint32_t chunk_bytes = read_u32_le(chunk + 4);
        const size_t payload = offset + 8;
        if ((size_t)chunk_bytes > riff_end - payload) {
            return -3;
        }
        if (!found_format && memcmp(chunk, "fmt ", 4) == 0) {
            if (chunk_bytes < 16) {
                return -4;
            }
            info->format_tag = (int)read_u16_le(bytes + payload);
            info->channels = (int)read_u16_le(bytes + payload + 2);
            info->sample_rate = (int)read_u32_le(bytes + payload + 4);
            info->bits_per_sample = (int)read_u16_le(bytes + payload + 14);
            found_format = 1;
        } else if (!found_data && memcmp(chunk, "data", 4) == 0) {
            info->data_offset = payload;
            info->data_bytes = chunk_bytes;
            found_data = 1;
        }
        const size_t padded = (size_t)chunk_bytes + ((size_t)chunk_bytes & 1u);
        if (padded > SIZE_MAX - payload) {
            return -3;
        }
        offset = payload + padded;
    }
    if (!found_format || !found_data || info->format_tag != 1 ||
        info->channels <= 0 || info->sample_rate <= 0 ||
        info->bits_per_sample != 16) {
        return -5;
    }
    const size_t bytes_per_frame = (size_t)info->channels * 2u;
    if (bytes_per_frame == 0 || info->data_bytes % bytes_per_frame != 0 ||
        info->data_bytes / bytes_per_frame > (size_t)INT_MAX) {
        return -6;
    }
    info->frames = (int)(info->data_bytes / bytes_per_frame);
    return info->frames > 0 ? 0 : -6;
}

References CKAudioWavInfo::bits_per_sample, CKAudioWavInfo::channels, CKAudioWavInfo::data_bytes, CKAudioWavInfo::data_offset, CKAudioWavInfo::format_tag, CKAudioWavInfo::frames, read_u16_le(), read_u32_le(), and CKAudioWavInfo::sample_rate.

Referenced by audio_wav_decode_memory_pcm16_mono_window_f32(), and audio_whisper_log_mel_window_wav_pcm16_f32().

audio_whisper_log_mel_from_power_reference_f32()

int audio_whisper_log_mel_from_power_reference_f32	(	const float *	power,
		const float *	mel_filters,
		int	n_mels,
		int	n_frames,
		float *	log_mel
	)

Definition at line 2214 of file audio_kernels.c.

{
    if (power == NULL || mel_filters == NULL || log_mel == NULL) {
        return -1;
    }
    if (n_mels <= 0 || n_frames <= 0) {
        return -2;
    }
 
    float maximum = -INFINITY;
    for (int mel = 0; mel < n_mels; ++mel) {
        const float *filter = mel_filters + (size_t)mel * CK_AUDIO_WHISPER_POWER_BINS;
        float *output = log_mel + (size_t)mel * n_frames;
        for (int frame = 0; frame < n_frames; ++frame) {
            const float *spectrum = power + (size_t)frame * CK_AUDIO_WHISPER_POWER_BINS;
            float sum = 0.0f;
            for (int bin = 0; bin < CK_AUDIO_WHISPER_POWER_BINS; ++bin) {
                sum = fmaf(filter[bin], spectrum[bin], sum);
            }
            const float value = log10f(fmaxf(sum, 1.0e-10f));
            output[frame] = value;
            maximum = fmaxf(maximum, value);
        }
    }
 
    const float floor = maximum - 8.0f;
    for (int mel = 0; mel < n_mels; ++mel) {
        float *output = log_mel + (size_t)mel * n_frames;
        for (int frame = 0; frame < n_frames; ++frame) {
            output[frame] = (fmaxf(output[frame], floor) + 4.0f) / 4.0f;
        }
    }
    return 0;
}

References CK_AUDIO_WHISPER_POWER_BINS.

Referenced by audio_whisper_log_mel_reference_f32().

audio_whisper_log_mel_reference_f32()

int audio_whisper_log_mel_reference_f32	(	const float *	samples,
		int	n_samples,
		const float *	mel_filters,
		int	n_mels,
		float *	power_scratch,
		float *	log_mel,
		int	n_frames
	)

Definition at line 2254 of file audio_kernels.c.

{
    const int stft_status = audio_whisper_stft_power_reference_f32(
        samples, n_samples, power_scratch, n_frames);
    if (stft_status != 0) {
        return stft_status;
    }
    return audio_whisper_log_mel_from_power_reference_f32(
        power_scratch, mel_filters, n_mels, n_frames, log_mel);
}

References audio_whisper_log_mel_from_power_reference_f32(), and audio_whisper_stft_power_reference_f32().

audio_whisper_log_mel_window_wav_pcm16_f32()

int audio_whisper_log_mel_window_wav_pcm16_f32	(	const uint8_t *	bytes,
		size_t	byte_count,
		int	start_frame,
		int	target_sample_rate,
		const float *	window,
		const float *	cos_table,
		const float *	sin_table,
		const float *	mel_filters,
		int	n_mels,
		int	output_frames,
		float *	log_mel
	)

Definition at line 878 of file audio_kernels.c.

{
    if (bytes == NULL || window == NULL || cos_table == NULL ||
        sin_table == NULL || mel_filters == NULL || log_mel == NULL) {
        return -1;
    }
    CKAudioWavInfo info;
    if (audio_wav_parse_memory(bytes, byte_count, &info) != 0 ||
        info.sample_rate != target_sample_rate ||
        target_sample_rate != CK_AUDIO_WHISPER_SAMPLE_RATE ||
        start_frame < 0 ||
        start_frame % CK_AUDIO_WHISPER_HOP_LENGTH != 0 ||
        n_mels <= 0 || output_frames <= 0) {
        return -2;
    }
    float *samples = (float *)malloc((size_t)info.frames * sizeof(float));
    if (samples == NULL) {
        return -3;
    }
    const int decoded = audio_wav_decode_pcm16_mono_f32(
        bytes, byte_count, &info, samples, info.frames);
    if (decoded != info.frames) {
        free(samples);
        return -4;
    }
 
    memset(
        log_mel,
        0,
        (size_t)n_mels * (size_t)output_frames * sizeof(float));
    const int global_frames = info.frames / CK_AUDIO_WHISPER_HOP_LENGTH;
    const int start_feature = start_frame / CK_AUDIO_WHISPER_HOP_LENGTH;
    float maximum = -INFINITY;
    float power[CK_AUDIO_WHISPER_POWER_BINS];
    float fft_scratch[2 * CK_AUDIO_WHISPER_N_FFT];
    for (int frame = 0; frame < global_frames; ++frame) {
        audio_stft_power_fft400_frame_f32(
            samples,
            info.frames,
            frame,
            window,
            cos_table,
            sin_table,
            power,
            fft_scratch);
        for (int mel = 0; mel < n_mels; ++mel) {
            const float *filter =
                mel_filters + (size_t)mel * CK_AUDIO_WHISPER_POWER_BINS;
            float sum = 0.0f;
            for (int bin = 0; bin < CK_AUDIO_WHISPER_POWER_BINS; ++bin) {
                sum = fmaf(filter[bin], power[bin], sum);
            }
            const float value = log10f(fmaxf(sum, 1.0e-10f));
            maximum = fmaxf(maximum, value);
            const int output_frame = frame - start_feature;
            if (output_frame >= 0 && output_frame < output_frames) {
                log_mel[(size_t)mel * output_frames + output_frame] = value;
            }
        }
    }
    free(samples);
    if (!isfinite(maximum)) {
        return -5;
    }
 
    const int available = global_frames - start_feature;
    const int valid_frames =
        available < output_frames ? (available > 0 ? available : 0) : output_frames;
    const float floor = maximum - 8.0f;
    for (int mel = 0; mel < n_mels; ++mel) {
        float *output = log_mel + (size_t)mel * output_frames;
        for (int frame = 0; frame < valid_frames; ++frame) {
            output[frame] = (fmaxf(output[frame], floor) + 4.0f) / 4.0f;
        }
    }
    return valid_frames;
}

References audio_stft_power_fft400_frame_f32(), audio_wav_decode_pcm16_mono_f32(), audio_wav_parse_memory(), CK_AUDIO_WHISPER_HOP_LENGTH, CK_AUDIO_WHISPER_N_FFT, CK_AUDIO_WHISPER_POWER_BINS, CK_AUDIO_WHISPER_SAMPLE_RATE, CKAudioWavInfo::frames, and CKAudioWavInfo::sample_rate.

audio_whisper_mel_filters_slaney_f32()

int audio_whisper_mel_filters_slaney_f32	(	int	sample_rate,
		int	n_fft,
		int	n_mels,
		float *	mel_filters
	)

Definition at line 612 of file audio_kernels.c.

{
    if (mel_filters == NULL) {
        return -1;
    }
    if (sample_rate <= 0 || n_fft <= 0 || (n_fft & 1) != 0 || n_mels <= 0) {
        return -2;
    }
    const int bins = n_fft / 2 + 1;
    const double mel_min = audio_hz_to_mel_slaney(0.0);
    const double mel_max = audio_hz_to_mel_slaney((double)sample_rate / 2.0);
    for (int mel = 0; mel < n_mels; ++mel) {
        const double left_mel =
            mel_min + (mel_max - mel_min) * (double)mel / (double)(n_mels + 1);
        const double center_mel =
            mel_min + (mel_max - mel_min) * (double)(mel + 1) / (double)(n_mels + 1);
        const double right_mel =
            mel_min + (mel_max - mel_min) * (double)(mel + 2) / (double)(n_mels + 1);
        const double left = audio_mel_to_hz_slaney(left_mel);
        const double center = audio_mel_to_hz_slaney(center_mel);
        const double right = audio_mel_to_hz_slaney(right_mel);
        const double normalization = 2.0 / (right - left);
        for (int bin = 0; bin < bins; ++bin) {
            const double hz =
                ((double)sample_rate / 2.0) * (double)bin / (double)(bins - 1);
            const double lower = (hz - left) / (center - left);
            const double upper = (right - hz) / (right - center);
            const double triangle = fmax(0.0, fmin(lower, upper));
            mel_filters[(size_t)mel * bins + bin] =
                (float)(triangle * normalization);
        }
    }
    return 0;
}

References audio_hz_to_mel_slaney(), audio_mel_to_hz_slaney(), left, and right.

audio_whisper_stft_power_reference_f32()

int audio_whisper_stft_power_reference_f32	(	const float *	samples,
		int	n_samples,
		float *	power,
		int	n_frames
	)

Definition at line 2173 of file audio_kernels.c.

{
    if (samples == NULL || power == NULL) {
        return -1;
    }
    if (n_samples <= CK_AUDIO_WHISPER_N_FFT / 2 || n_frames <= 0) {
        return -2;
    }
    if (n_frames != n_samples / CK_AUDIO_WHISPER_HOP_LENGTH) {
        return -3;
    }
 
    const int center = CK_AUDIO_WHISPER_N_FFT / 2;
    for (int frame = 0; frame < n_frames; ++frame) {
        for (int bin = 0; bin < CK_AUDIO_WHISPER_POWER_BINS; ++bin) {
            float real = 0.0f;
            float imag = 0.0f;
            for (int sample = 0; sample < CK_AUDIO_WHISPER_N_FFT; ++sample) {
                const int source = reflect_index(
                    frame * CK_AUDIO_WHISPER_HOP_LENGTH + sample - center,
                    n_samples);
                const float window = 0.5f - 0.5f * cosf(
                    2.0f * CK_AUDIO_PI_F * (float)sample /
                    (float)CK_AUDIO_WHISPER_N_FFT);
                const float value = samples[source] * window;
                const float angle = -2.0f * CK_AUDIO_PI_F *
                    (float)(bin * sample) / (float)CK_AUDIO_WHISPER_N_FFT;
                real = fmaf(value, cosf(angle), real);
                imag = fmaf(value, sinf(angle), imag);
            }
            power[(size_t)frame * CK_AUDIO_WHISPER_POWER_BINS + bin] =
                fmaf(real, real, imag * imag);
        }
    }
    return 0;
}

References CK_AUDIO_PI_F, CK_AUDIO_WHISPER_HOP_LENGTH, CK_AUDIO_WHISPER_N_FFT, CK_AUDIO_WHISPER_POWER_BINS, and reflect_index().

Referenced by audio_whisper_log_mel_reference_f32().