← Back to C-Kernel-Engine Docs Doxygen Source Documentation
 
Loading...
Searching...
No Matches
Functions
bf16_utils.h File Reference
#include <stdint.h>
#include <stddef.h>

Go to the source code of this file.

Functions

static void bf16_tensor_to_float (const uint16_t *src, float *dst, size_t count)
 
static float bf16_to_float (uint16_t v)
 
static void float_tensor_to_bf16 (const float *src, uint16_t *dst, size_t count)
 
static uint16_t float_to_bf16 (float f)
 

Function Documentation

◆ bf16_tensor_to_float()

static void bf16_tensor_to_float ( const uint16_t *  src,
float *  dst,
size_t  count 
)
inlinestatic

Definition at line 250 of file bf16_utils.h.

251{
252#if defined(__AVX512F__)
253 size_t i = 0;
254 for (; i + 16 <= count; i += 16) {
255 __m512 fp32_vec = bf16_loadu_cvt_fp32(&src[i]);
256 _mm512_storeu_ps(&dst[i], fp32_vec);
257 }
258 for (; i < count; ++i) {
259 dst[i] = bf16_to_float(src[i]);
260 }
261#else
262 for (size_t i = 0; i < count; ++i) {
263 dst[i] = bf16_to_float(src[i]);
264 }
265#endif
266}
bf16_to_float
static float bf16_to_float(uint16_t v)
Definition bf16_utils.h:38

References bf16_to_float().

Referenced by backward_causal_softmax_head_major_bf16(), causal_softmax_head_major_bf16(), convert_bf16_tensor_to_buf(), geglu_forward_bf16(), gelu_backward_exact_bf16(), gelu_backward_fast_bf16(), gelu_fast_inplace_bf16(), layernorm_backward_kernel_bf16(), layernorm_forward_rolled_slice_bf16(), layernorm_forward_unrolled_slice_bf16(), mlp_token_parallel_bf16_fp32act(), rope_backward_bf16(), rope_forward_bf16_with_rotary_dim(), sigmoid_backward_bf16(), sigmoid_forward_bf16(), and softmax_cross_entropy_loss_bf16().

◆ bf16_to_float()

static float bf16_to_float ( uint16_t  v)
inlinestatic

Definition at line 38 of file bf16_utils.h.

39{
40 union {
41 uint32_t u;
42 float f;
43 } tmp;
44 tmp.u = (uint32_t)v << 16; // Place BF16 in upper 16 bits, lower bits = 0
45 return tmp.f;
46}

Referenced by __attribute__(), adamw_update_bf16(), add_forward_2d_bf16(), add_forward_bf16(), add_inplace_bf16(), add_scaled_forward_bf16(), add_scaled_inplace_bf16(), attention_forward_causal_head_major_gqa_prefill_append_bf16cache_pytorch_contract_workspace(), attention_forward_decode_head_major_gqa_bf16cache_pytorch_contract(), attention_forward_full_head_major_gqa_flash_strided_bf16_storage(), attention_forward_full_head_major_gqa_sdpa_bf16_storage(), attn_gate_sigmoid_mul_pytorch_bf16_storage(), bf16_tensor_to_float(), ck_attention_full_bf16_sdpa_amx_range(), ck_attention_full_bf16_sdpa_tiled_range(), ck_attention_u16_cache_to_f32(), ck_bf16_dot_contract(), ck_bf16_round(), ck_bf16_round_work(), ck_deltanet_pytorch_gate_values(), ck_gemm_bf16_native_work(), ck_moe_bf16_round(), ck_mrope_round_storage(), ck_patch_projection_bf16_native_work(), ck_residual_add_token_major_bf16_storage(), deepseek_mla_kv_decompress_bf16_token_range(), ds_mla_bf16_round(), embedding_backward_bf16(), embedding_backward_bf16_mixed(), embedding_forward_bf16(), embedding_forward_bf16_fp32(), farskip_swiglu_shared_combine_bf16_row_range(), gated_deltanet_llama_avx2_grouped_forward_impl(), gated_deltanet_pytorch_grouped_bf16_forward_impl(), geglu_backward_bf16_mixed(), gelu_erf_bf16_storage(), gelu_pytorch_erf_sleef_bf16_storage(), gelu_pytorch_tanh_bf16_storage(), gemm_backward_bf16_mixed(), gemm_bf16_fp32out(), gemm_nn_bf16(), gemm_nt_bf16_bf16_storage_row_range(), gemm_nt_bf16_pytorch_onednn_brgemm_bf16_storage_impl(), gemm_nt_bf16_row_range(), gemm_tn_bf16(), gemv_bf16_bf16_storage_row_range(), gemv_bf16_row_range(), gradient_accumulate_bf16(), gradient_clip_norm_bf16(), gradient_scale_bf16(), hyper_connection_mix_bf16(), layernorm_naive_serial_bf16_storage(), layernorm_pytorch_welford_bf16_storage(), mlp_token_parallel_bf16(), mlp_token_parallel_bf16_backward_mixed(), moe_softmax_topk_router_pytorch_bf16_workspace(), moe_swiglu_expert_forward_bf16_row_range(), moe_swiglu_packed_expert_forward_bf16(), moe_swiglu_shared_forward_bf16_gated_row_range(), moe_swiglu_shared_forward_bf16_row_range(), patch2im_bf16(), patch_projection_bf16_pytorch_onednn_conv3d_storage(), position_embeddings_add_tiled_2d_align_corners_bf16(), position_embeddings_add_tiled_2d_align_corners_fp32_interp_bf16(), recurrent_norm_gate_pytorch_bf16_storage(), recurrent_norm_sigmoid_gate_pytorch_bf16_storage(), recurrent_pytorch_bf16_l2_rows(), recurrent_pytorch_bf16_square_sum(), recurrent_sigmoid_forward_pytorch_bf16_input_fp32_output(), recurrent_silu_forward_pytorch_bf16_input_fp32_output(), recurrent_silu_forward_pytorch_bf16_storage(), rmsnorm_backward_bf16(), rmsnorm_forward_bf16(), rmsnorm_forward_pytorch_bf16_storage_impl(), sgd_momentum_update_bf16(), ssm_conv1d_forward_pytorch_bf16_storage(), swiglu_backward_bf16(), swiglu_forward_bf16(), swiglu_forward_pytorch_bf16_storage(), text_mrope_apply_positions_pytorch_bf16_storage(), and text_mrope_apply_pytorch_bf16_storage().

◆ float_tensor_to_bf16()

static void float_tensor_to_bf16 ( const float *  src,
uint16_t *  dst,
size_t  count 
)
inlinestatic

Definition at line 271 of file bf16_utils.h.

272{
273#if defined(__AVX512F__)
274 size_t i = 0;
275 for (; i + 16 <= count; i += 16) {
276 __m512 fp32_vec = _mm512_loadu_ps(&src[i]);
277 fp32_cvt_storeu_bf16(&dst[i], fp32_vec);
278 }
279 for (; i < count; ++i) {
280 dst[i] = float_to_bf16(src[i]);
281 }
282#else
283 for (size_t i = 0; i < count; ++i) {
284 dst[i] = float_to_bf16(src[i]);
285 }
286#endif
287}
float_to_bf16
static uint16_t float_to_bf16(float f)
Definition bf16_utils.h:90

References float_to_bf16().

Referenced by backward_causal_softmax_head_major_bf16(), causal_softmax_head_major_bf16(), geglu_forward_bf16(), gelu_backward_exact_bf16(), gelu_backward_fast_bf16(), gelu_fast_inplace_bf16(), layernorm_backward_kernel_bf16(), layernorm_forward_rolled_slice_bf16(), layernorm_forward_unrolled_slice_bf16(), mlp_token_parallel_bf16_fp32act(), rope_backward_bf16(), rope_forward_bf16_with_rotary_dim(), sigmoid_backward_bf16(), sigmoid_forward_bf16(), and softmax_cross_entropy_loss_bf16().

◆ float_to_bf16()

static uint16_t float_to_bf16 ( float  f)
inlinestatic

Definition at line 90 of file bf16_utils.h.

91{
92 union {
93 uint32_t u;
94 float f;
95 } tmp;
96 tmp.f = f;
97 // Extract bit 16 (will be the LSB of the BF16 result after truncation)
98 uint32_t lsb = (tmp.u >> 16) & 1u;
99 // Add rounding bias: 0x7FFF normally, 0x8000 if LSB=1 (rounds ties to even)
100 tmp.u += 0x7FFFu + lsb;
101 // Truncate lower 16 bits
102 return (uint16_t)(tmp.u >> 16);
103}

Referenced by __attribute__(), adamw_update_bf16(), add_forward_2d_bf16(), add_forward_bf16(), add_inplace_bf16(), add_scaled_forward_bf16(), add_scaled_inplace_bf16(), attention_forward_causal_head_major_gqa_prefill_append_bf16cache_pytorch_contract_workspace(), attention_forward_decode_head_major_gqa_bf16cache_pytorch_contract(), attention_forward_full_head_major_gqa_flash_strided_bf16_storage(), attention_forward_full_head_major_gqa_sdpa_bf16_storage(), attn_gate_sigmoid_mul_pytorch_bf16_storage(), ck_attention_full_bf16_sdpa_amx_range(), ck_attention_full_bf16_sdpa_tiled_range(), ck_bf16_convert_work(), ck_bf16_dot_contract(), ck_bf16_round(), ck_bf16_round_work(), ck_deltanet_pytorch_gate_values(), ck_gemm_bf16_native_work(), ck_local_fp32_to_bf16_row(), ck_moe_bf16_round(), ck_mrope_round_storage(), ck_patch_projection_bf16_native_work(), ck_residual_add_token_major_bf16_storage(), deepseek_mla_kv_decompress_bf16_token_range(), ds_mla_bf16_round(), embedding_backward_bf16(), embedding_forward_bf16(), float_tensor_to_bf16(), gated_deltanet_llama_avx2_grouped_forward_impl(), gated_deltanet_pytorch_grouped_bf16_forward_impl(), gelu_erf_bf16_storage(), gelu_pytorch_erf_sleef_bf16_storage(), gelu_pytorch_tanh_bf16_storage(), gemm_nn_bf16(), gemm_nt_bf16_bf16_storage_row_range(), gemm_nt_bf16_pytorch_onednn_brgemm_bf16_storage_impl(), gemm_nt_bf16_row_range(), gemm_tn_bf16(), gemv_bf16_bf16_storage_row_range(), gemv_bf16_row_range(), gradient_accumulate_bf16(), gradient_scale_bf16(), layernorm_naive_serial_bf16_storage(), layernorm_pytorch_welford_bf16_storage(), mlp_token_parallel_bf16(), mlp_token_parallel_bf16_backward_mixed(), moe_softmax_topk_router_pytorch_bf16_workspace(), patch2im_bf16(), patch_projection_bf16_pytorch_onednn_conv3d_storage(), position_embeddings_add_tiled_2d_align_corners_bf16(), position_embeddings_add_tiled_2d_align_corners_fp32_interp_bf16(), recurrent_norm_gate_pytorch_bf16_storage(), recurrent_norm_sigmoid_gate_pytorch_bf16_storage(), recurrent_pytorch_bf16_l2_rows(), recurrent_pytorch_bf16_square_sum(), recurrent_sigmoid_forward_pytorch_bf16_input_fp32_output(), recurrent_silu_forward_pytorch_bf16_input_fp32_output(), recurrent_silu_forward_pytorch_bf16_storage(), rmsnorm_backward_bf16(), rmsnorm_forward_bf16(), rmsnorm_forward_pytorch_bf16_storage_impl(), sgd_momentum_update_bf16(), ssm_conv1d_forward_pytorch_bf16_storage(), swiglu_backward_bf16(), swiglu_forward_bf16(), swiglu_forward_pytorch_bf16_storage(), text_mrope_apply_positions_pytorch_bf16_storage(), text_mrope_apply_pytorch_bf16_storage(), and yarn_rope_cache_explicit_positions_impl().