v6.5/test_generated/qwen2_int8.c File Reference

AUTO-GENERATED: qwen2_0.5b_decode Implementation (IR v6.5 - Explicit Unrolled) More...

#include "ck-kernel-inference.h"
#include "ckernel_engine.h"
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <stdint.h>
#include <math.h>
#include "ck_model_api.h"

Go to the source code of this file.

Macros
#define	_GNU_SOURCE   /* For MAP_ANONYMOUS, MAP_HUGETLB */

Functions
struct	__attribute__ ((packed))
	_Static_assert (sizeof(MagicHeader)==64, "MagicHeader must be 64 bytes")
void *	ck_model_create (void)
void	ck_model_decode (void *model, const int *token, int token_index)
void	ck_model_forward (void *model, const int *tokens, int num_tokens)
void	ck_model_free (void *model)
void *	ck_model_get_base (void *model)
const CKModelConfig *	ck_model_get_config (void)
float *	ck_model_get_logits (void *model)
size_t	ck_model_get_total_bytes (void *model)
void	ck_model_precompute_rope (void *model)
int	ck_model_verify_canaries (void *model)
static int	qwen2_0_5b_decode_align_elems (int elems, int elem_bytes, int align_bytes)
void	qwen2_0_5b_decode_decode (QWEN2_0_5B_DECODEModel *model, const int *token, int token_index)
static void	qwen2_0_5b_decode_decode_token (QWEN2_0_5B_DECODEModel *model, const int *token, int token_index)
void	qwen2_0_5b_decode_forward (QWEN2_0_5B_DECODEModel *model, const int *tokens, int num_tokens)
static void	qwen2_0_5b_decode_forward_prefill_impl (QWEN2_0_5B_DECODEModel *model, const int *tokens, int num_tokens)
static void	qwen2_0_5b_decode_layer_0_decode (QWEN2_0_5B_DECODEModel *model, int token_index, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
static void	qwen2_0_5b_decode_layer_0_prefill (QWEN2_0_5B_DECODEModel *model, int num_tokens, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
static void	qwen2_0_5b_decode_layer_10_decode (QWEN2_0_5B_DECODEModel *model, int token_index, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
static void	qwen2_0_5b_decode_layer_10_prefill (QWEN2_0_5B_DECODEModel *model, int num_tokens, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
static void	qwen2_0_5b_decode_layer_11_decode (QWEN2_0_5B_DECODEModel *model, int token_index, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
static void	qwen2_0_5b_decode_layer_11_prefill (QWEN2_0_5B_DECODEModel *model, int num_tokens, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
static void	qwen2_0_5b_decode_layer_12_decode (QWEN2_0_5B_DECODEModel *model, int token_index, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
static void	qwen2_0_5b_decode_layer_12_prefill (QWEN2_0_5B_DECODEModel *model, int num_tokens, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
static void	qwen2_0_5b_decode_layer_13_decode (QWEN2_0_5B_DECODEModel *model, int token_index, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
static void	qwen2_0_5b_decode_layer_13_prefill (QWEN2_0_5B_DECODEModel *model, int num_tokens, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
static void	qwen2_0_5b_decode_layer_14_decode (QWEN2_0_5B_DECODEModel *model, int token_index, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
static void	qwen2_0_5b_decode_layer_14_prefill (QWEN2_0_5B_DECODEModel *model, int num_tokens, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
static void	qwen2_0_5b_decode_layer_15_decode (QWEN2_0_5B_DECODEModel *model, int token_index, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
static void	qwen2_0_5b_decode_layer_15_prefill (QWEN2_0_5B_DECODEModel *model, int num_tokens, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
static void	qwen2_0_5b_decode_layer_16_decode (QWEN2_0_5B_DECODEModel *model, int token_index, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
static void	qwen2_0_5b_decode_layer_16_prefill (QWEN2_0_5B_DECODEModel *model, int num_tokens, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
static void	qwen2_0_5b_decode_layer_17_decode (QWEN2_0_5B_DECODEModel *model, int token_index, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
static void	qwen2_0_5b_decode_layer_17_prefill (QWEN2_0_5B_DECODEModel *model, int num_tokens, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
static void	qwen2_0_5b_decode_layer_18_decode (QWEN2_0_5B_DECODEModel *model, int token_index, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
static void	qwen2_0_5b_decode_layer_18_prefill (QWEN2_0_5B_DECODEModel *model, int num_tokens, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
static void	qwen2_0_5b_decode_layer_19_decode (QWEN2_0_5B_DECODEModel *model, int token_index, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
static void	qwen2_0_5b_decode_layer_19_prefill (QWEN2_0_5B_DECODEModel *model, int num_tokens, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
static void	qwen2_0_5b_decode_layer_1_decode (QWEN2_0_5B_DECODEModel *model, int token_index, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
static void	qwen2_0_5b_decode_layer_1_prefill (QWEN2_0_5B_DECODEModel *model, int num_tokens, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
static void	qwen2_0_5b_decode_layer_20_decode (QWEN2_0_5B_DECODEModel *model, int token_index, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
static void	qwen2_0_5b_decode_layer_20_prefill (QWEN2_0_5B_DECODEModel *model, int num_tokens, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
static void	qwen2_0_5b_decode_layer_21_decode (QWEN2_0_5B_DECODEModel *model, int token_index, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
static void	qwen2_0_5b_decode_layer_21_prefill (QWEN2_0_5B_DECODEModel *model, int num_tokens, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
static void	qwen2_0_5b_decode_layer_22_decode (QWEN2_0_5B_DECODEModel *model, int token_index, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
static void	qwen2_0_5b_decode_layer_22_prefill (QWEN2_0_5B_DECODEModel *model, int num_tokens, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
static void	qwen2_0_5b_decode_layer_23_decode (QWEN2_0_5B_DECODEModel *model, int token_index, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
static void	qwen2_0_5b_decode_layer_23_prefill (QWEN2_0_5B_DECODEModel *model, int num_tokens, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
static void	qwen2_0_5b_decode_layer_2_decode (QWEN2_0_5B_DECODEModel *model, int token_index, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
static void	qwen2_0_5b_decode_layer_2_prefill (QWEN2_0_5B_DECODEModel *model, int num_tokens, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
static void	qwen2_0_5b_decode_layer_3_decode (QWEN2_0_5B_DECODEModel *model, int token_index, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
static void	qwen2_0_5b_decode_layer_3_prefill (QWEN2_0_5B_DECODEModel *model, int num_tokens, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
static void	qwen2_0_5b_decode_layer_4_decode (QWEN2_0_5B_DECODEModel *model, int token_index, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
static void	qwen2_0_5b_decode_layer_4_prefill (QWEN2_0_5B_DECODEModel *model, int num_tokens, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
static void	qwen2_0_5b_decode_layer_5_decode (QWEN2_0_5B_DECODEModel *model, int token_index, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
static void	qwen2_0_5b_decode_layer_5_prefill (QWEN2_0_5B_DECODEModel *model, int num_tokens, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
static void	qwen2_0_5b_decode_layer_6_decode (QWEN2_0_5B_DECODEModel *model, int token_index, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
static void	qwen2_0_5b_decode_layer_6_prefill (QWEN2_0_5B_DECODEModel *model, int num_tokens, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
static void	qwen2_0_5b_decode_layer_7_decode (QWEN2_0_5B_DECODEModel *model, int token_index, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
static void	qwen2_0_5b_decode_layer_7_prefill (QWEN2_0_5B_DECODEModel *model, int num_tokens, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
static void	qwen2_0_5b_decode_layer_8_decode (QWEN2_0_5B_DECODEModel *model, int token_index, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
static void	qwen2_0_5b_decode_layer_8_prefill (QWEN2_0_5B_DECODEModel *model, int num_tokens, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
static void	qwen2_0_5b_decode_layer_9_decode (QWEN2_0_5B_DECODEModel *model, int token_index, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
static void	qwen2_0_5b_decode_layer_9_prefill (QWEN2_0_5B_DECODEModel *model, int num_tokens, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
int	qwen2_0_5b_decode_model_allocate (QWEN2_0_5B_DECODEModel *model)
void	qwen2_0_5b_decode_model_free (QWEN2_0_5B_DECODEModel *model)
void	qwen2_0_5b_decode_precompute_rope (QWEN2_0_5B_DECODEModel *model)
static void	qwen2_0_5b_decode_residual_add_token_major (const float *a, const float *b, float *out, int tokens, int aligned_embed_dim)
int	qwen2_0_5b_decode_verify_canaries (QWEN2_0_5B_DECODEModel *model)

Variables
static CKModelConfig	g_model_config
	MagicHeader

Detailed Description

AUTO-GENERATED: qwen2_0.5b_decode Implementation (IR v6.5 - Explicit Unrolled)

Generated: 2026-01-12T11:58:55.212793 UTC Total Memory: 3.57 GB Mode: decode Layers: 24 (fully unrolled)

Per-layer quant types: Layer 0: wq=q4_k wk=q4_k wv=q4_k wo=q4_k w1=q4_k w2=q4_k Layer 1: wq=q4_k wk=q4_k wv=q4_k wo=q4_k w1=q4_k w2=q4_k Layer 2: wq=q4_k wk=q4_k wv=q4_k wo=q4_k w1=q4_k w2=q4_k ... (21 more layers)

DO NOT EDIT - Regenerate with build_ir_v6.5.py or codegen_v6.5.py

Definition in file v6.5/test_generated/qwen2_int8.c.

Macro Definition Documentation

_GNU_SOURCE

#define _GNU_SOURCE   /* For MAP_ANONYMOUS, MAP_HUGETLB */

Definition at line 19 of file v6.5/test_generated/qwen2_int8.c.

Function Documentation

__attribute__()

struct __attribute__

(

(packed)

)

Definition at line 43 of file v6.5/test_generated/qwen2_int8.c.

                                       {
    uint32_t magic;           /* 0x434B454E */
    uint32_t version;          /* IR version */
    uint64_t total_bytes;
    uint64_t weight_bytes;
    uint64_t activation_bytes;
    uint32_t num_layers;
    uint32_t embed_dim;
    uint32_t num_heads;
    uint32_t vocab_size;
    uint32_t max_seq_len;
    uint32_t canary_count;
    uint8_t  reserved[8];       /* Pad to 64 bytes */
} MagicHeader;

_Static_assert()

_Static_assert	(	sizeof(MagicHeader)	= =64,
		"MagicHeader must be 64 bytes"
	)

ck_model_create()

void* ck_model_create

(

void

)

Create and allocate model memory. Returns opaque model pointer, or NULL on failure.

Definition at line 8873 of file v6.5/test_generated/qwen2_int8.c.

                            {
    QWEN2_0_5B_DECODEModel *model = malloc(sizeof(QWEN2_0_5B_DECODEModel));
    if (!model) return NULL;
    if (qwen2_0_5b_decode_model_allocate(model) != 0) {
        free(model);
        return NULL;
    }
    return model;
}

References qwen2_0_5b_decode_model_allocate().

ck_model_decode()

void ck_model_decode	(	void *	model,
		const int *	token,
		int	token_index
	)

Decode single token at position token_index. Used for autoregressive generation.

Definition at line 8897 of file v6.5/test_generated/qwen2_int8.c.

                                                                     {
    qwen2_0_5b_decode_decode((QWEN2_0_5B_DECODEModel *)model, token, token_index);
}

References qwen2_0_5b_decode_decode(), and token.

ck_model_forward()

void ck_model_forward	(	void *	model,
		const int *	tokens,
		int	num_tokens
	)

Forward pass (prefill) - process multiple tokens. Used for initial prompt processing.

Definition at line 8893 of file v6.5/test_generated/qwen2_int8.c.

                                                                      {
    qwen2_0_5b_decode_forward((QWEN2_0_5B_DECODEModel *)model, tokens, num_tokens);
}

References qwen2_0_5b_decode_forward().

ck_model_free()

void ck_model_free

(

void *

model

)

Free model memory.

Definition at line 8883 of file v6.5/test_generated/qwen2_int8.c.

                                {
    if (!model) return;
    qwen2_0_5b_decode_model_free((QWEN2_0_5B_DECODEModel *)model);
    free(model);
}

References qwen2_0_5b_decode_model_free().

ck_model_get_base()

void* ck_model_get_base

(

void *

model

)

Get model base pointer (for weight loading).

Definition at line 8910 of file v6.5/test_generated/qwen2_int8.c.

                                     {
    return ((QWEN2_0_5B_DECODEModel *)model)->base;
}

ck_model_get_config()

const CKModelConfig* ck_model_get_config

(

void

)

Get model configuration (dimensions, sizes, etc.) This is available before allocation.

Definition at line 8869 of file v6.5/test_generated/qwen2_int8.c.

                                               {
    return &g_model_config;
}

References g_model_config.

ck_model_get_logits()

float* ck_model_get_logits

(

void *

model

)

Get pointer to output logits buffer. Size is vocab_size floats.

Definition at line 8901 of file v6.5/test_generated/qwen2_int8.c.

                                        {
    QWEN2_0_5B_DECODEModel *m = (QWEN2_0_5B_DECODEModel *)model;
    return QWEN2_0_5B_DECODE_PTR(m, QWEN2_0_5B_DECODE_FOOTER.logits);
}

References QWEN2_0_5B_DECODEFooterOffsets::logits, QWEN2_0_5B_DECODE_FOOTER, and QWEN2_0_5B_DECODE_PTR.

ck_model_get_total_bytes()

size_t ck_model_get_total_bytes

(

void *

model

)

Get total model size in bytes.

Definition at line 8914 of file v6.5/test_generated/qwen2_int8.c.

                                             {
    return ((QWEN2_0_5B_DECODEModel *)model)->total_bytes;
}

ck_model_precompute_rope()

void ck_model_precompute_rope

(

void *

model

)

Precompute RoPE cos/sin caches. Call once after allocation, before inference.

Definition at line 8889 of file v6.5/test_generated/qwen2_int8.c.

                                           {
    qwen2_0_5b_decode_precompute_rope((QWEN2_0_5B_DECODEModel *)model);
}

References qwen2_0_5b_decode_precompute_rope().

ck_model_verify_canaries()

int ck_model_verify_canaries

(

void *

model

)

Verify memory canaries (debug). Returns number of corrupted canaries (0 = OK).

Definition at line 8906 of file v6.5/test_generated/qwen2_int8.c.

                                          {
    return qwen2_0_5b_decode_verify_canaries((QWEN2_0_5B_DECODEModel *)model);
}

References qwen2_0_5b_decode_verify_canaries().

qwen2_0_5b_decode_align_elems()

static int qwen2_0_5b_decode_align_elems ( int  elems, int  elem_bytes, int  align_bytes  )

static

Definition at line 176 of file v6.5/test_generated/qwen2_int8.c.

                                                                                     {
    int bytes = elems * elem_bytes;
    int aligned = (bytes + align_bytes - 1) / align_bytes * align_bytes;
    return aligned / elem_bytes;
}

qwen2_0_5b_decode_decode()

void qwen2_0_5b_decode_decode	(	QWEN2_0_5B_DECODEModel *	model,
		const int *	token,
		int	token_index
	)

Definition at line 8841 of file v6.5/test_generated/qwen2_int8.c.

                                                                                                {
    qwen2_0_5b_decode_decode_token(model, token, token_index);
}

References qwen2_0_5b_decode_decode_token(), and token.

Referenced by ck_model_decode().

qwen2_0_5b_decode_decode_token()

static void qwen2_0_5b_decode_decode_token ( QWEN2_0_5B_DECODEModel *  model, const int *  token, int  token_index  )

static

Definition at line 8750 of file v6.5/test_generated/qwen2_int8.c.

  {
    if (!model || !token) return;
 
    const int aligned_embed_dim = 1024;
    const int aligned_head_dim = 64;
    const int aligned_intermediate_dim = 4864;
    const int aligned_context_window = 131072;
 
    if (token_index < 0 || token_index >= aligned_context_window) return;
 
    /* Embedding lookup */
    float *embed_out = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_HEADER.embedded_input);
    const void *embed_weight = (const void *)QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_HEADER.token_emb);
    /* Embedding: Q4_K -> embedding_forward_q4_k */
    embedding_forward_q4_k((const int32_t *)token,
                          1,
                          QWEN2_0_5B_DECODE_VOCAB_SIZE,
                          embed_weight,
                          NULL,
                          embed_out,
                          QWEN2_0_5B_DECODE_EMBED_DIM,
                          aligned_embed_dim,
                          1,
                          0);
 
    /* Process each layer explicitly */
    qwen2_0_5b_decode_layer_0_decode(model, token_index, aligned_embed_dim, aligned_head_dim, aligned_intermediate_dim, aligned_context_window);
    qwen2_0_5b_decode_layer_1_decode(model, token_index, aligned_embed_dim, aligned_head_dim, aligned_intermediate_dim, aligned_context_window);
    qwen2_0_5b_decode_layer_2_decode(model, token_index, aligned_embed_dim, aligned_head_dim, aligned_intermediate_dim, aligned_context_window);
    qwen2_0_5b_decode_layer_3_decode(model, token_index, aligned_embed_dim, aligned_head_dim, aligned_intermediate_dim, aligned_context_window);
    qwen2_0_5b_decode_layer_4_decode(model, token_index, aligned_embed_dim, aligned_head_dim, aligned_intermediate_dim, aligned_context_window);
    qwen2_0_5b_decode_layer_5_decode(model, token_index, aligned_embed_dim, aligned_head_dim, aligned_intermediate_dim, aligned_context_window);
    qwen2_0_5b_decode_layer_6_decode(model, token_index, aligned_embed_dim, aligned_head_dim, aligned_intermediate_dim, aligned_context_window);
    qwen2_0_5b_decode_layer_7_decode(model, token_index, aligned_embed_dim, aligned_head_dim, aligned_intermediate_dim, aligned_context_window);
    qwen2_0_5b_decode_layer_8_decode(model, token_index, aligned_embed_dim, aligned_head_dim, aligned_intermediate_dim, aligned_context_window);
    qwen2_0_5b_decode_layer_9_decode(model, token_index, aligned_embed_dim, aligned_head_dim, aligned_intermediate_dim, aligned_context_window);
    qwen2_0_5b_decode_layer_10_decode(model, token_index, aligned_embed_dim, aligned_head_dim, aligned_intermediate_dim, aligned_context_window);
    qwen2_0_5b_decode_layer_11_decode(model, token_index, aligned_embed_dim, aligned_head_dim, aligned_intermediate_dim, aligned_context_window);
    qwen2_0_5b_decode_layer_12_decode(model, token_index, aligned_embed_dim, aligned_head_dim, aligned_intermediate_dim, aligned_context_window);
    qwen2_0_5b_decode_layer_13_decode(model, token_index, aligned_embed_dim, aligned_head_dim, aligned_intermediate_dim, aligned_context_window);
    qwen2_0_5b_decode_layer_14_decode(model, token_index, aligned_embed_dim, aligned_head_dim, aligned_intermediate_dim, aligned_context_window);
    qwen2_0_5b_decode_layer_15_decode(model, token_index, aligned_embed_dim, aligned_head_dim, aligned_intermediate_dim, aligned_context_window);
    qwen2_0_5b_decode_layer_16_decode(model, token_index, aligned_embed_dim, aligned_head_dim, aligned_intermediate_dim, aligned_context_window);
    qwen2_0_5b_decode_layer_17_decode(model, token_index, aligned_embed_dim, aligned_head_dim, aligned_intermediate_dim, aligned_context_window);
    qwen2_0_5b_decode_layer_18_decode(model, token_index, aligned_embed_dim, aligned_head_dim, aligned_intermediate_dim, aligned_context_window);
    qwen2_0_5b_decode_layer_19_decode(model, token_index, aligned_embed_dim, aligned_head_dim, aligned_intermediate_dim, aligned_context_window);
    qwen2_0_5b_decode_layer_20_decode(model, token_index, aligned_embed_dim, aligned_head_dim, aligned_intermediate_dim, aligned_context_window);
    qwen2_0_5b_decode_layer_21_decode(model, token_index, aligned_embed_dim, aligned_head_dim, aligned_intermediate_dim, aligned_context_window);
    qwen2_0_5b_decode_layer_22_decode(model, token_index, aligned_embed_dim, aligned_head_dim, aligned_intermediate_dim, aligned_context_window);
    qwen2_0_5b_decode_layer_23_decode(model, token_index, aligned_embed_dim, aligned_head_dim, aligned_intermediate_dim, aligned_context_window);
 
    /* Final RMSNorm */
    float *last_hidden = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[23].output);
    float *final_ln_weight = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_FOOTER.final_ln_weight);
    float *final_out = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_FOOTER.final_output);
    rmsnorm_forward(last_hidden,
                    final_ln_weight,
                    final_out,
                    NULL,
                    1,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    /* LM head projection */
    float *logits = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_FOOTER.logits);
    const void *lm_head = (const void *)QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_FOOTER.lm_head_weight);
    /* LM head (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t final_q8_bytes = ((((aligned_embed_dim) + 255) / 256) * 292);
    uint8_t final_q8[final_q8_bytes];
    quantize_row_q8_k(final_out, final_q8, aligned_embed_dim);
    gemv_q4_k_q8_k(logits, lm_head, final_q8, QWEN2_0_5B_DECODE_VOCAB_SIZE, aligned_embed_dim);
}

References QWEN2_0_5B_DECODEHeaderOffsets::embedded_input, embedding_forward_q4_k(), QWEN2_0_5B_DECODEFooterOffsets::final_ln_weight, QWEN2_0_5B_DECODEFooterOffsets::final_output, gemv_q4_k_q8_k(), QWEN2_0_5B_DECODEFooterOffsets::lm_head_weight, QWEN2_0_5B_DECODEFooterOffsets::logits, quantize_row_q8_k(), QWEN2_0_5B_DECODE_EMBED_DIM, QWEN2_0_5B_DECODE_FOOTER, QWEN2_0_5B_DECODE_HEADER, qwen2_0_5b_decode_layer_0_decode(), qwen2_0_5b_decode_layer_10_decode(), qwen2_0_5b_decode_layer_11_decode(), qwen2_0_5b_decode_layer_12_decode(), qwen2_0_5b_decode_layer_13_decode(), qwen2_0_5b_decode_layer_14_decode(), qwen2_0_5b_decode_layer_15_decode(), qwen2_0_5b_decode_layer_16_decode(), qwen2_0_5b_decode_layer_17_decode(), qwen2_0_5b_decode_layer_18_decode(), qwen2_0_5b_decode_layer_19_decode(), qwen2_0_5b_decode_layer_1_decode(), qwen2_0_5b_decode_layer_20_decode(), qwen2_0_5b_decode_layer_21_decode(), qwen2_0_5b_decode_layer_22_decode(), qwen2_0_5b_decode_layer_23_decode(), qwen2_0_5b_decode_layer_2_decode(), qwen2_0_5b_decode_layer_3_decode(), qwen2_0_5b_decode_layer_4_decode(), qwen2_0_5b_decode_layer_5_decode(), qwen2_0_5b_decode_layer_6_decode(), qwen2_0_5b_decode_layer_7_decode(), qwen2_0_5b_decode_layer_8_decode(), qwen2_0_5b_decode_layer_9_decode(), QWEN2_0_5B_DECODE_LAYERS, QWEN2_0_5B_DECODE_PTR, QWEN2_0_5B_DECODE_VOCAB_SIZE, rmsnorm_forward(), token, and QWEN2_0_5B_DECODEHeaderOffsets::token_emb.

Referenced by qwen2_0_5b_decode_decode().

qwen2_0_5b_decode_forward()

void qwen2_0_5b_decode_forward	(	QWEN2_0_5B_DECODEModel *	model,
		const int *	tokens,
		int	num_tokens
	)

Definition at line 8832 of file v6.5/test_generated/qwen2_int8.c.

  {
    if (!model || !tokens || num_tokens <= 0) return;
    qwen2_0_5b_decode_forward_prefill_impl(model, tokens, num_tokens);
}

References qwen2_0_5b_decode_forward_prefill_impl().

Referenced by ck_model_forward().

qwen2_0_5b_decode_forward_prefill_impl()

static void qwen2_0_5b_decode_forward_prefill_impl ( QWEN2_0_5B_DECODEModel *  model, const int *  tokens, int  num_tokens  )

static

Definition at line 4148 of file v6.5/test_generated/qwen2_int8.c.

  {
    if (!model || !tokens || num_tokens <= 0) {
        return;
    }
 
    const int elem_bytes = QWEN2_0_5B_DECODE_DTYPE_BYTES;
    const int aligned_embed_dim = 1024;
    const int aligned_head_dim = 64;
    const int aligned_intermediate_dim = 4864;
    const int aligned_context_window = 131072;
 
    float *embed_out = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_HEADER.embedded_input);
    const void *embed_weight = (const void *)QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_HEADER.token_emb);
    embedding_forward_q4_k((const int32_t *)tokens,
                          num_tokens,
                          QWEN2_0_5B_DECODE_VOCAB_SIZE,
                          embed_weight,
                          NULL,
                          embed_out,
                          QWEN2_0_5B_DECODE_EMBED_DIM,
                          aligned_embed_dim,
                          num_tokens,
                          0);
 
    qwen2_0_5b_decode_layer_0_prefill(
        model,
        num_tokens,
        aligned_embed_dim,
        aligned_head_dim,
        aligned_intermediate_dim,
        aligned_context_window);
 
    qwen2_0_5b_decode_layer_1_prefill(
        model,
        num_tokens,
        aligned_embed_dim,
        aligned_head_dim,
        aligned_intermediate_dim,
        aligned_context_window);
 
    qwen2_0_5b_decode_layer_2_prefill(
        model,
        num_tokens,
        aligned_embed_dim,
        aligned_head_dim,
        aligned_intermediate_dim,
        aligned_context_window);
 
    qwen2_0_5b_decode_layer_3_prefill(
        model,
        num_tokens,
        aligned_embed_dim,
        aligned_head_dim,
        aligned_intermediate_dim,
        aligned_context_window);
 
    qwen2_0_5b_decode_layer_4_prefill(
        model,
        num_tokens,
        aligned_embed_dim,
        aligned_head_dim,
        aligned_intermediate_dim,
        aligned_context_window);
 
    qwen2_0_5b_decode_layer_5_prefill(
        model,
        num_tokens,
        aligned_embed_dim,
        aligned_head_dim,
        aligned_intermediate_dim,
        aligned_context_window);
 
    qwen2_0_5b_decode_layer_6_prefill(
        model,
        num_tokens,
        aligned_embed_dim,
        aligned_head_dim,
        aligned_intermediate_dim,
        aligned_context_window);
 
    qwen2_0_5b_decode_layer_7_prefill(
        model,
        num_tokens,
        aligned_embed_dim,
        aligned_head_dim,
        aligned_intermediate_dim,
        aligned_context_window);
 
    qwen2_0_5b_decode_layer_8_prefill(
        model,
        num_tokens,
        aligned_embed_dim,
        aligned_head_dim,
        aligned_intermediate_dim,
        aligned_context_window);
 
    qwen2_0_5b_decode_layer_9_prefill(
        model,
        num_tokens,
        aligned_embed_dim,
        aligned_head_dim,
        aligned_intermediate_dim,
        aligned_context_window);
 
    qwen2_0_5b_decode_layer_10_prefill(
        model,
        num_tokens,
        aligned_embed_dim,
        aligned_head_dim,
        aligned_intermediate_dim,
        aligned_context_window);
 
    qwen2_0_5b_decode_layer_11_prefill(
        model,
        num_tokens,
        aligned_embed_dim,
        aligned_head_dim,
        aligned_intermediate_dim,
        aligned_context_window);
 
    qwen2_0_5b_decode_layer_12_prefill(
        model,
        num_tokens,
        aligned_embed_dim,
        aligned_head_dim,
        aligned_intermediate_dim,
        aligned_context_window);
 
    qwen2_0_5b_decode_layer_13_prefill(
        model,
        num_tokens,
        aligned_embed_dim,
        aligned_head_dim,
        aligned_intermediate_dim,
        aligned_context_window);
 
    qwen2_0_5b_decode_layer_14_prefill(
        model,
        num_tokens,
        aligned_embed_dim,
        aligned_head_dim,
        aligned_intermediate_dim,
        aligned_context_window);
 
    qwen2_0_5b_decode_layer_15_prefill(
        model,
        num_tokens,
        aligned_embed_dim,
        aligned_head_dim,
        aligned_intermediate_dim,
        aligned_context_window);
 
    qwen2_0_5b_decode_layer_16_prefill(
        model,
        num_tokens,
        aligned_embed_dim,
        aligned_head_dim,
        aligned_intermediate_dim,
        aligned_context_window);
 
    qwen2_0_5b_decode_layer_17_prefill(
        model,
        num_tokens,
        aligned_embed_dim,
        aligned_head_dim,
        aligned_intermediate_dim,
        aligned_context_window);
 
    qwen2_0_5b_decode_layer_18_prefill(
        model,
        num_tokens,
        aligned_embed_dim,
        aligned_head_dim,
        aligned_intermediate_dim,
        aligned_context_window);
 
    qwen2_0_5b_decode_layer_19_prefill(
        model,
        num_tokens,
        aligned_embed_dim,
        aligned_head_dim,
        aligned_intermediate_dim,
        aligned_context_window);
 
    qwen2_0_5b_decode_layer_20_prefill(
        model,
        num_tokens,
        aligned_embed_dim,
        aligned_head_dim,
        aligned_intermediate_dim,
        aligned_context_window);
 
    qwen2_0_5b_decode_layer_21_prefill(
        model,
        num_tokens,
        aligned_embed_dim,
        aligned_head_dim,
        aligned_intermediate_dim,
        aligned_context_window);
 
    qwen2_0_5b_decode_layer_22_prefill(
        model,
        num_tokens,
        aligned_embed_dim,
        aligned_head_dim,
        aligned_intermediate_dim,
        aligned_context_window);
 
    qwen2_0_5b_decode_layer_23_prefill(
        model,
        num_tokens,
        aligned_embed_dim,
        aligned_head_dim,
        aligned_intermediate_dim,
        aligned_context_window);
 
    float *last_hidden = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[QWEN2_0_5B_DECODE_NUM_LAYERS - 1].output);
    float *final_ln_weight = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_FOOTER.final_ln_weight);
    float *final_out = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_FOOTER.final_output);
    rmsnorm_forward(last_hidden,
                   final_ln_weight,
                   final_out,
                   NULL,
                   num_tokens,
                   QWEN2_0_5B_DECODE_EMBED_DIM,
                   aligned_embed_dim,
                   1e-06f);
 
    float *logits = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_FOOTER.logits);
    const void *lm_head = (const void *)QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_FOOTER.lm_head_weight);
    const size_t q8_bytes = ck_dtype_row_bytes(CK_DT_Q8_K, (size_t)aligned_embed_dim);
    for (int t = 0; t < num_tokens; ++t) {
        uint8_t q8_buf[q8_bytes];
        const float *row = final_out + (size_t)t * (size_t)aligned_embed_dim;
        float *logits_row = logits + (size_t)t * (size_t)QWEN2_0_5B_DECODE_VOCAB_SIZE;
        quantize_row_q8_k(row, q8_buf, aligned_embed_dim);
        gemm_nt_q4_k_q8_k(q8_buf,
                          lm_head,
                          NULL,
                          logits_row,
                          1,
                          QWEN2_0_5B_DECODE_VOCAB_SIZE,
                          aligned_embed_dim);
    }
}

References CK_DT_Q8_K, ck_dtype_row_bytes(), QWEN2_0_5B_DECODEHeaderOffsets::embedded_input, embedding_forward_q4_k(), QWEN2_0_5B_DECODEFooterOffsets::final_ln_weight, QWEN2_0_5B_DECODEFooterOffsets::final_output, gemm_nt_q4_k_q8_k(), QWEN2_0_5B_DECODEFooterOffsets::lm_head_weight, QWEN2_0_5B_DECODEFooterOffsets::logits, quantize_row_q8_k(), QWEN2_0_5B_DECODE_DTYPE_BYTES, QWEN2_0_5B_DECODE_EMBED_DIM, QWEN2_0_5B_DECODE_FOOTER, QWEN2_0_5B_DECODE_HEADER, qwen2_0_5b_decode_layer_0_prefill(), qwen2_0_5b_decode_layer_10_prefill(), qwen2_0_5b_decode_layer_11_prefill(), qwen2_0_5b_decode_layer_12_prefill(), qwen2_0_5b_decode_layer_13_prefill(), qwen2_0_5b_decode_layer_14_prefill(), qwen2_0_5b_decode_layer_15_prefill(), qwen2_0_5b_decode_layer_16_prefill(), qwen2_0_5b_decode_layer_17_prefill(), qwen2_0_5b_decode_layer_18_prefill(), qwen2_0_5b_decode_layer_19_prefill(), qwen2_0_5b_decode_layer_1_prefill(), qwen2_0_5b_decode_layer_20_prefill(), qwen2_0_5b_decode_layer_21_prefill(), qwen2_0_5b_decode_layer_22_prefill(), qwen2_0_5b_decode_layer_23_prefill(), qwen2_0_5b_decode_layer_2_prefill(), qwen2_0_5b_decode_layer_3_prefill(), qwen2_0_5b_decode_layer_4_prefill(), qwen2_0_5b_decode_layer_5_prefill(), qwen2_0_5b_decode_layer_6_prefill(), qwen2_0_5b_decode_layer_7_prefill(), qwen2_0_5b_decode_layer_8_prefill(), qwen2_0_5b_decode_layer_9_prefill(), QWEN2_0_5B_DECODE_LAYERS, QWEN2_0_5B_DECODE_NUM_LAYERS, QWEN2_0_5B_DECODE_PTR, QWEN2_0_5B_DECODE_VOCAB_SIZE, rmsnorm_forward(), and QWEN2_0_5B_DECODEHeaderOffsets::token_emb.

Referenced by qwen2_0_5b_decode_forward().

qwen2_0_5b_decode_layer_0_decode()

static void qwen2_0_5b_decode_layer_0_decode ( QWEN2_0_5B_DECODEModel *  model, int  token_index, int  aligned_embed_dim, int  aligned_head_dim, int  aligned_intermediate_dim, int  aligned_context_window  )

static

Definition at line 4405 of file v6.5/test_generated/qwen2_int8.c.

  {
    const QWEN2_0_5B_DECODELayerOffsets *L = &QWEN2_0_5B_DECODE_LAYERS[0];
 
    float *input = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_HEADER.embedded_input);
 
    float *ln1_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln1_gamma);
    float *ln1_out = QWEN2_0_5B_DECODE_PTR(model, L->ln1_out);
    float *ln2_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln2_gamma);
    float *ln2_out = QWEN2_0_5B_DECODE_PTR(model, L->ln2_out);
    float *k_cache = QWEN2_0_5B_DECODE_PTR(model, L->k);
    float *v_cache = QWEN2_0_5B_DECODE_PTR(model, L->v);
    float *proj_tmp = QWEN2_0_5B_DECODE_PTR(model, L->proj_tmp);
    float *proj_scratch = QWEN2_0_5B_DECODE_PTR(model, L->proj_scratch);
    float *residual1 = QWEN2_0_5B_DECODE_PTR(model, L->residual1);
    float *mlp_out = QWEN2_0_5B_DECODE_PTR(model, L->mlp_out);
    float *output = QWEN2_0_5B_DECODE_PTR(model, L->output);
 
    /* Weights (explicit types for layer 0) */
    const void *WQ = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wq);  /* Q4_K */
    const void *WK = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wk);  /* Q4_K */
    const void *WV = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wv);  /* Q4_K */
    const void *WO = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wo);  /* Q4_K */
    const void *W1 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w1);  /* Q4_K (gate+up) */
    const void *W2 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w2);  /* Q4_K (down) */
 
    float *rope_cos = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_cos_cache);
    float *rope_sin = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_sin_cache);
 
    const int H = QWEN2_0_5B_DECODE_NUM_HEADS;
    const int H_kv = QWEN2_0_5B_DECODE_NUM_KV_HEADS;
    const int head_dim = QWEN2_0_5B_DECODE_HEAD_DIM;
 
    float q_token[H * aligned_head_dim];
    float k_token[H_kv * aligned_head_dim];
    float v_token[H_kv * aligned_head_dim];
    float attn_token[H * aligned_head_dim];
 
    /* Local MLP buffers (avoid layout dependencies for intermediate values) */
    float fc1_out[2 * aligned_intermediate_dim];
    float swiglu_out[aligned_intermediate_dim];
 
    /* Step 1: RMSNorm before attention */
    rmsnorm_forward(input,
                    ln1_gamma,
                    ln1_out,
                    NULL,
                    1,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    const size_t kv_head_stride = (size_t)aligned_context_window * (size_t)aligned_head_dim;
 
    /* Step 2: QKV projection */
    /* Q projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t ln1_q8_bytes = ((((aligned_embed_dim) + 255) / 256) * 292);
    uint8_t ln1_q8[ln1_q8_bytes];
    quantize_row_q8_k(ln1_out, ln1_q8, aligned_embed_dim);
    gemv_q4_k_q8_k(q_token, WQ, ln1_q8, H * head_dim, aligned_embed_dim);
    if (aligned_head_dim > head_dim) {
        for (int h = 0; h < H; ++h) {
            float *q_head = q_token + (size_t)h * (size_t)aligned_head_dim;
            for (int d = head_dim; d < aligned_head_dim; ++d) {
                q_head[d] = 0.0f;
            }
        }
    }
 
    /* K projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k (direct-to-cache) */
    const size_t wk_head_elems = (size_t)aligned_head_dim * (size_t)aligned_embed_dim;
    const size_t wk_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, wk_head_elems);
    const uint8_t *WK_bytes = (const uint8_t *)WK;
    /* ln1_q8 already quantized above */
    for (int h = 0; h < H_kv; ++h) {
        const void *wk_h = (const void *)(WK_bytes + (size_t)h * wk_head_bytes);
        float *k_head = k_cache + (size_t)h * kv_head_stride + (size_t)token_index * (size_t)aligned_head_dim;
        gemv_q4_k_q8_k(k_head, wk_h, ln1_q8, head_dim, aligned_embed_dim);
        for (int d = head_dim; d < aligned_head_dim; ++d) {
            k_head[d] = 0.0f;
        }
    }
 
    /* V projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k (direct-to-cache) */
    const size_t wv_head_elems = (size_t)aligned_head_dim * (size_t)aligned_embed_dim;
    const size_t wv_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, wv_head_elems);
    const uint8_t *WV_bytes = (const uint8_t *)WV;
    /* ln1_q8 already quantized above */
    for (int h = 0; h < H_kv; ++h) {
        const void *wv_h = (const void *)(WV_bytes + (size_t)h * wv_head_bytes);
        float *v_head = v_cache + (size_t)h * kv_head_stride + (size_t)token_index * (size_t)aligned_head_dim;
        gemv_q4_k_q8_k(v_head, wv_h, ln1_q8, head_dim, aligned_embed_dim);
        for (int d = head_dim; d < aligned_head_dim; ++d) {
            v_head[d] = 0.0f;
        }
    }
 
    /* Step 3: RoPE */
    rope_forward(q_token,
                 rope_cos,
                 rope_sin,
                 H,
                 1,
                 head_dim,
                 aligned_head_dim,
                 token_index);
    for (int h = 0; h < H_kv; ++h) {
        float *k_head = k_cache + (size_t)h * kv_head_stride + (size_t)token_index * (size_t)aligned_head_dim;
        rope_forward(k_head,
                     rope_cos,
                     rope_sin,
                     1,
                     1,
                     head_dim,
                     aligned_head_dim,
                     token_index);
    }
 
    /* Step 4: KV cache write (direct-to-cache) */
 
    /* Step 5: Attention (decode, flash) */
    attention_forward_decode_head_major_gqa_flash(q_token,
                                                   k_cache,
                                                   v_cache,
                                                   attn_token,
                                                   H,
                                                   H_kv,
                                                   token_index + 1,
                                                   aligned_context_window,
                                                   head_dim,
                                                   aligned_head_dim);
 
    /* Step 6: Output projection */
    /* WO projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t attn_q8_bytes = ((((H * head_dim) + 255) / 256) * 292);
    uint8_t attn_q8[attn_q8_bytes];
    quantize_row_q8_k(attn_token, attn_q8, H * head_dim);
    gemv_q4_k_q8_k(proj_tmp, WO, attn_q8, aligned_embed_dim, H * head_dim);
 
    /* Step 7: Residual add */
    qwen2_0_5b_decode_residual_add_token_major(input, proj_tmp, residual1, 1, aligned_embed_dim);
 
    /* Step 8: RMSNorm before MLP */
    rmsnorm_forward(residual1,
                    ln2_gamma,
                    ln2_out,
                    NULL,
                    1,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    /* Step 9: MLP (SwiGLU) */
    /* Gate+Up projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t ln2_q8_bytes = ((((aligned_embed_dim) + 255) / 256) * 292);
    uint8_t ln2_q8[ln2_q8_bytes];
    quantize_row_q8_k(ln2_out, ln2_q8, aligned_embed_dim);
    gemv_q4_k_q8_k(fc1_out, W1, ln2_q8, 2 * aligned_intermediate_dim, aligned_embed_dim);
 
    /* SwiGLU activation */
    swiglu_forward(fc1_out, swiglu_out, 1, aligned_intermediate_dim);
 
    /* Down projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t swiglu_q8_bytes = ((((aligned_intermediate_dim) + 255) / 256) * 292);
    uint8_t swiglu_q8[swiglu_q8_bytes];
    quantize_row_q8_k(swiglu_out, swiglu_q8, aligned_intermediate_dim);
    gemv_q4_k_q8_k(mlp_out, W2, swiglu_q8, aligned_embed_dim, aligned_intermediate_dim);
 
    /* Step 10: Final residual add */
    qwen2_0_5b_decode_residual_add_token_major(residual1, mlp_out, output, 1, aligned_embed_dim);
}

References attention_forward_decode_head_major_gqa_flash(), CK_DT_Q4_K, ck_dtype_row_bytes(), QWEN2_0_5B_DECODEHeaderOffsets::embedded_input, gemv_q4_k_q8_k(), QWEN2_0_5B_DECODELayerOffsets::k, QWEN2_0_5B_DECODELayerOffsets::ln1_gamma, QWEN2_0_5B_DECODELayerOffsets::ln1_out, QWEN2_0_5B_DECODELayerOffsets::ln2_gamma, QWEN2_0_5B_DECODELayerOffsets::ln2_out, QWEN2_0_5B_DECODELayerOffsets::mlp_out, QWEN2_0_5B_DECODELayerOffsets::output, QWEN2_0_5B_DECODELayerOffsets::proj_scratch, QWEN2_0_5B_DECODELayerOffsets::proj_tmp, quantize_row_q8_k(), QWEN2_0_5B_DECODE_EMBED_DIM, QWEN2_0_5B_DECODE_GLOBALS, QWEN2_0_5B_DECODE_HEAD_DIM, QWEN2_0_5B_DECODE_HEADER, QWEN2_0_5B_DECODE_LAYERS, QWEN2_0_5B_DECODE_NUM_HEADS, QWEN2_0_5B_DECODE_NUM_KV_HEADS, QWEN2_0_5B_DECODE_PTR, qwen2_0_5b_decode_residual_add_token_major(), QWEN2_0_5B_DECODELayerOffsets::residual1, rmsnorm_forward(), QWEN2_0_5B_DECODEGlobalOffsets::rope_cos_cache, rope_forward(), QWEN2_0_5B_DECODEGlobalOffsets::rope_sin_cache, swiglu_forward(), QWEN2_0_5B_DECODELayerOffsets::v, QWEN2_0_5B_DECODELayerOffsets::w1, QWEN2_0_5B_DECODELayerOffsets::w2, QWEN2_0_5B_DECODELayerOffsets::wk, QWEN2_0_5B_DECODELayerOffsets::wo, QWEN2_0_5B_DECODELayerOffsets::wq, and QWEN2_0_5B_DECODELayerOffsets::wv.

Referenced by qwen2_0_5b_decode_decode_token().

qwen2_0_5b_decode_layer_0_prefill()

static void qwen2_0_5b_decode_layer_0_prefill ( QWEN2_0_5B_DECODEModel *  model, int  num_tokens, int  aligned_embed_dim, int  aligned_head_dim, int  aligned_intermediate_dim, int  aligned_context_window  )

static

Definition at line 211 of file v6.5/test_generated/qwen2_int8.c.

  {
    const QWEN2_0_5B_DECODELayerOffsets *L = &QWEN2_0_5B_DECODE_LAYERS[0];
 
    float *input = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_HEADER.embedded_input);
    float *ln1_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln1_gamma);
    float *ln1_out = QWEN2_0_5B_DECODE_PTR(model, L->ln1_out);
    float *ln2_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln2_gamma);
    float *ln2_out = QWEN2_0_5B_DECODE_PTR(model, L->ln2_out);
    float *q = QWEN2_0_5B_DECODE_PTR(model, L->q);
    float *k = QWEN2_0_5B_DECODE_PTR(model, L->k);
    float *v = QWEN2_0_5B_DECODE_PTR(model, L->v);
    float *attn_out = QWEN2_0_5B_DECODE_PTR(model, L->attn_out);
    float *proj_tmp = QWEN2_0_5B_DECODE_PTR(model, L->proj_tmp);
    float *proj_scratch = QWEN2_0_5B_DECODE_PTR(model, L->proj_scratch);
    float *residual1 = QWEN2_0_5B_DECODE_PTR(model, L->residual1);
    float *fc1_out = QWEN2_0_5B_DECODE_PTR(model, L->fc1_out);
    float *swiglu_out = QWEN2_0_5B_DECODE_PTR(model, L->swiglu_out);
    float *mlp_out = QWEN2_0_5B_DECODE_PTR(model, L->mlp_out);
    float *output = QWEN2_0_5B_DECODE_PTR(model, L->output);
 
    const void *WQ = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wq);
    const void *WK = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wk);
    const void *WV = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wv);
    const void *WO = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wo);
    const void *W1 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w1);
    const void *W2 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w2);
    const float *BQ = NULL;
    const float *BK = NULL;
    const float *BV = NULL;
    const float *BO = NULL;
    const float *B1 = NULL;
    const float *B2 = NULL;
 
    float *rope_cos = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_cos_cache);
    float *rope_sin = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_sin_cache);
 
    const int H = QWEN2_0_5B_DECODE_NUM_HEADS;
    const int H_kv = QWEN2_0_5B_DECODE_NUM_KV_HEADS;
    const int head_dim = QWEN2_0_5B_DECODE_HEAD_DIM;
    const size_t head_w_elems = (size_t)aligned_head_dim * (size_t)aligned_embed_dim;
    const size_t q_head_stride = (size_t)num_tokens * (size_t)aligned_head_dim;
    const size_t kv_head_stride = (size_t)aligned_context_window * (size_t)aligned_head_dim;
 
    /* RMSNorm before attention */
    rmsnorm_forward(input,
                    ln1_gamma,
                    ln1_out,
                    NULL,
                    num_tokens,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    /* Q projection (head-major) */
    const size_t wq_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, head_w_elems);
    const uint8_t *WQ_bytes = (const uint8_t *)WQ;
    for (int h = 0; h < H; ++h) {
        const void *wq_h = (const void *)(WQ_bytes + (size_t)h * wq_head_bytes);
        const float *bq_h = BQ ? (BQ + (size_t)h * (size_t)aligned_head_dim) : NULL;
        float *q_h = q + (size_t)h * q_head_stride;
        gemm_nt_q4_k(ln1_out, wq_h, bq_h, q_h, num_tokens, aligned_head_dim, aligned_embed_dim);
    }
 
    /* K projection (head-major) */
    const size_t wk_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, head_w_elems);
    const uint8_t *WK_bytes = (const uint8_t *)WK;
    for (int h = 0; h < H_kv; ++h) {
        const void *wk_h = (const void *)(WK_bytes + (size_t)h * wk_head_bytes);
        const float *bk_h = BK ? (BK + (size_t)h * (size_t)aligned_head_dim) : NULL;
        float *k_h = k + (size_t)h * kv_head_stride;
        gemm_nt_q4_k(ln1_out, wk_h, bk_h, k_h, num_tokens, aligned_head_dim, aligned_embed_dim);
    }
 
    /* V projection (head-major) */
    const size_t wv_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, head_w_elems);
    const uint8_t *WV_bytes = (const uint8_t *)WV;
    for (int h = 0; h < H_kv; ++h) {
        const void *wv_h = (const void *)(WV_bytes + (size_t)h * wv_head_bytes);
        const float *bv_h = BV ? (BV + (size_t)h * (size_t)aligned_head_dim) : NULL;
        float *v_h = v + (size_t)h * kv_head_stride;
        gemm_nt_q4_k(ln1_out, wv_h, bv_h, v_h, num_tokens, aligned_head_dim, aligned_embed_dim);
    }
 
    /* RoPE */
    rope_forward_qk_strided(q,
                            k,
                            rope_cos,
                            rope_sin,
                            H,
                            H_kv,
                            num_tokens,
                            head_dim,
                            aligned_head_dim,
                            0,
                            num_tokens,
                            aligned_context_window);
 
    /* Attention (prefill, causal) */
    attention_forward_causal_head_major_gqa_flash_strided(q,
                                                           k,
                                                           v,
                                                           attn_out,
                                                           H,
                                                           H_kv,
                                                           num_tokens,
                                                           head_dim,
                                                           aligned_head_dim,
                                                           aligned_context_window);
 
    /* Output projection (flatten head-major to token-major) */
    const int K = H * aligned_head_dim;
    if (K != aligned_embed_dim) {
        return;
    }
    const float *proj_in = attn_out;
    if (H > 1) {
        if (!proj_scratch) {
            return;
        }
        for (int t = 0; t < num_tokens; ++t) {
            float *dst = proj_scratch + (size_t)t * (size_t)aligned_embed_dim;
            for (int h = 0; h < H; ++h) {
                const float *src = attn_out + (size_t)h * q_head_stride + (size_t)t * (size_t)aligned_head_dim;
                memcpy(dst + (size_t)h * (size_t)aligned_head_dim,
                       src,
                       (size_t)aligned_head_dim * sizeof(float));
            }
        }
        proj_in = proj_scratch;
    }
    gemm_nt_q4_k(proj_in, WO, BO, proj_tmp, num_tokens, aligned_embed_dim, K);
 
    /* Residual add */
    qwen2_0_5b_decode_residual_add_token_major(input, proj_tmp, residual1, num_tokens, aligned_embed_dim);
 
    /* RMSNorm before MLP */
    rmsnorm_forward(residual1,
                    ln2_gamma,
                    ln2_out,
                    NULL,
                    num_tokens,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    /* MLP (SwiGLU) */
    gemm_nt_q4_k(ln2_out, W1, B1, fc1_out, num_tokens, 2 * aligned_intermediate_dim, aligned_embed_dim);
    swiglu_forward(fc1_out, swiglu_out, num_tokens, aligned_intermediate_dim);
    gemm_nt_q4_k(swiglu_out, W2, B2, mlp_out, num_tokens, aligned_embed_dim, aligned_intermediate_dim);
 
    /* Final residual add */
    qwen2_0_5b_decode_residual_add_token_major(residual1, mlp_out, output, num_tokens, aligned_embed_dim);
}

References attention_forward_causal_head_major_gqa_flash_strided(), CK_DT_Q4_K, ck_dtype_row_bytes(), QWEN2_0_5B_DECODEHeaderOffsets::embedded_input, gemm_nt_q4_k(), QWEN2_0_5B_DECODELayerOffsets::k, QWEN2_0_5B_DECODELayerOffsets::ln1_gamma, QWEN2_0_5B_DECODELayerOffsets::ln1_out, QWEN2_0_5B_DECODELayerOffsets::ln2_gamma, QWEN2_0_5B_DECODELayerOffsets::ln2_out, QWEN2_0_5B_DECODELayerOffsets::mlp_out, QWEN2_0_5B_DECODELayerOffsets::output, QWEN2_0_5B_DECODELayerOffsets::proj_scratch, QWEN2_0_5B_DECODELayerOffsets::proj_tmp, QWEN2_0_5B_DECODELayerOffsets::q, QWEN2_0_5B_DECODE_EMBED_DIM, QWEN2_0_5B_DECODE_GLOBALS, QWEN2_0_5B_DECODE_HEAD_DIM, QWEN2_0_5B_DECODE_HEADER, QWEN2_0_5B_DECODE_LAYERS, QWEN2_0_5B_DECODE_NUM_HEADS, QWEN2_0_5B_DECODE_NUM_KV_HEADS, QWEN2_0_5B_DECODE_PTR, qwen2_0_5b_decode_residual_add_token_major(), QWEN2_0_5B_DECODELayerOffsets::residual1, rmsnorm_forward(), QWEN2_0_5B_DECODEGlobalOffsets::rope_cos_cache, rope_forward_qk_strided(), QWEN2_0_5B_DECODEGlobalOffsets::rope_sin_cache, swiglu_forward(), QWEN2_0_5B_DECODELayerOffsets::v, QWEN2_0_5B_DECODELayerOffsets::w1, QWEN2_0_5B_DECODELayerOffsets::w2, QWEN2_0_5B_DECODELayerOffsets::wk, QWEN2_0_5B_DECODELayerOffsets::wo, QWEN2_0_5B_DECODELayerOffsets::wq, and QWEN2_0_5B_DECODELayerOffsets::wv.

Referenced by qwen2_0_5b_decode_forward_prefill_impl().

qwen2_0_5b_decode_layer_10_decode()

static void qwen2_0_5b_decode_layer_10_decode ( QWEN2_0_5B_DECODEModel *  model, int  token_index, int  aligned_embed_dim, int  aligned_head_dim, int  aligned_intermediate_dim, int  aligned_context_window  )

static

Definition at line 6215 of file v6.5/test_generated/qwen2_int8.c.

  {
    const QWEN2_0_5B_DECODELayerOffsets *L = &QWEN2_0_5B_DECODE_LAYERS[10];
 
    float *input = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[9].output);
 
    float *ln1_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln1_gamma);
    float *ln1_out = QWEN2_0_5B_DECODE_PTR(model, L->ln1_out);
    float *ln2_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln2_gamma);
    float *ln2_out = QWEN2_0_5B_DECODE_PTR(model, L->ln2_out);
    float *k_cache = QWEN2_0_5B_DECODE_PTR(model, L->k);
    float *v_cache = QWEN2_0_5B_DECODE_PTR(model, L->v);
    float *proj_tmp = QWEN2_0_5B_DECODE_PTR(model, L->proj_tmp);
    float *proj_scratch = QWEN2_0_5B_DECODE_PTR(model, L->proj_scratch);
    float *residual1 = QWEN2_0_5B_DECODE_PTR(model, L->residual1);
    float *mlp_out = QWEN2_0_5B_DECODE_PTR(model, L->mlp_out);
    float *output = QWEN2_0_5B_DECODE_PTR(model, L->output);
 
    /* Weights (explicit types for layer 10) */
    const void *WQ = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wq);  /* Q4_K */
    const void *WK = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wk);  /* Q4_K */
    const void *WV = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wv);  /* Q4_K */
    const void *WO = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wo);  /* Q4_K */
    const void *W1 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w1);  /* Q4_K (gate+up) */
    const void *W2 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w2);  /* Q4_K (down) */
 
    float *rope_cos = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_cos_cache);
    float *rope_sin = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_sin_cache);
 
    const int H = QWEN2_0_5B_DECODE_NUM_HEADS;
    const int H_kv = QWEN2_0_5B_DECODE_NUM_KV_HEADS;
    const int head_dim = QWEN2_0_5B_DECODE_HEAD_DIM;
 
    float q_token[H * aligned_head_dim];
    float k_token[H_kv * aligned_head_dim];
    float v_token[H_kv * aligned_head_dim];
    float attn_token[H * aligned_head_dim];
 
    /* Local MLP buffers (avoid layout dependencies for intermediate values) */
    float fc1_out[2 * aligned_intermediate_dim];
    float swiglu_out[aligned_intermediate_dim];
 
    /* Step 1: RMSNorm before attention */
    rmsnorm_forward(input,
                    ln1_gamma,
                    ln1_out,
                    NULL,
                    1,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    const size_t kv_head_stride = (size_t)aligned_context_window * (size_t)aligned_head_dim;
 
    /* Step 2: QKV projection */
    /* Q projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t ln1_q8_bytes = ((((aligned_embed_dim) + 255) / 256) * 292);
    uint8_t ln1_q8[ln1_q8_bytes];
    quantize_row_q8_k(ln1_out, ln1_q8, aligned_embed_dim);
    gemv_q4_k_q8_k(q_token, WQ, ln1_q8, H * head_dim, aligned_embed_dim);
    if (aligned_head_dim > head_dim) {
        for (int h = 0; h < H; ++h) {
            float *q_head = q_token + (size_t)h * (size_t)aligned_head_dim;
            for (int d = head_dim; d < aligned_head_dim; ++d) {
                q_head[d] = 0.0f;
            }
        }
    }
 
    /* K projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k (direct-to-cache) */
    const size_t wk_head_elems = (size_t)aligned_head_dim * (size_t)aligned_embed_dim;
    const size_t wk_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, wk_head_elems);
    const uint8_t *WK_bytes = (const uint8_t *)WK;
    /* ln1_q8 already quantized above */
    for (int h = 0; h < H_kv; ++h) {
        const void *wk_h = (const void *)(WK_bytes + (size_t)h * wk_head_bytes);
        float *k_head = k_cache + (size_t)h * kv_head_stride + (size_t)token_index * (size_t)aligned_head_dim;
        gemv_q4_k_q8_k(k_head, wk_h, ln1_q8, head_dim, aligned_embed_dim);
        for (int d = head_dim; d < aligned_head_dim; ++d) {
            k_head[d] = 0.0f;
        }
    }
 
    /* V projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k (direct-to-cache) */
    const size_t wv_head_elems = (size_t)aligned_head_dim * (size_t)aligned_embed_dim;
    const size_t wv_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, wv_head_elems);
    const uint8_t *WV_bytes = (const uint8_t *)WV;
    /* ln1_q8 already quantized above */
    for (int h = 0; h < H_kv; ++h) {
        const void *wv_h = (const void *)(WV_bytes + (size_t)h * wv_head_bytes);
        float *v_head = v_cache + (size_t)h * kv_head_stride + (size_t)token_index * (size_t)aligned_head_dim;
        gemv_q4_k_q8_k(v_head, wv_h, ln1_q8, head_dim, aligned_embed_dim);
        for (int d = head_dim; d < aligned_head_dim; ++d) {
            v_head[d] = 0.0f;
        }
    }
 
    /* Step 3: RoPE */
    rope_forward(q_token,
                 rope_cos,
                 rope_sin,
                 H,
                 1,
                 head_dim,
                 aligned_head_dim,
                 token_index);
    for (int h = 0; h < H_kv; ++h) {
        float *k_head = k_cache + (size_t)h * kv_head_stride + (size_t)token_index * (size_t)aligned_head_dim;
        rope_forward(k_head,
                     rope_cos,
                     rope_sin,
                     1,
                     1,
                     head_dim,
                     aligned_head_dim,
                     token_index);
    }
 
    /* Step 4: KV cache write (direct-to-cache) */
 
    /* Step 5: Attention (decode, flash) */
    attention_forward_decode_head_major_gqa_flash(q_token,
                                                   k_cache,
                                                   v_cache,
                                                   attn_token,
                                                   H,
                                                   H_kv,
                                                   token_index + 1,
                                                   aligned_context_window,
                                                   head_dim,
                                                   aligned_head_dim);
 
    /* Step 6: Output projection */
    /* WO projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t attn_q8_bytes = ((((H * head_dim) + 255) / 256) * 292);
    uint8_t attn_q8[attn_q8_bytes];
    quantize_row_q8_k(attn_token, attn_q8, H * head_dim);
    gemv_q4_k_q8_k(proj_tmp, WO, attn_q8, aligned_embed_dim, H * head_dim);
 
    /* Step 7: Residual add */
    qwen2_0_5b_decode_residual_add_token_major(input, proj_tmp, residual1, 1, aligned_embed_dim);
 
    /* Step 8: RMSNorm before MLP */
    rmsnorm_forward(residual1,
                    ln2_gamma,
                    ln2_out,
                    NULL,
                    1,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    /* Step 9: MLP (SwiGLU) */
    /* Gate+Up projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t ln2_q8_bytes = ((((aligned_embed_dim) + 255) / 256) * 292);
    uint8_t ln2_q8[ln2_q8_bytes];
    quantize_row_q8_k(ln2_out, ln2_q8, aligned_embed_dim);
    gemv_q4_k_q8_k(fc1_out, W1, ln2_q8, 2 * aligned_intermediate_dim, aligned_embed_dim);
 
    /* SwiGLU activation */
    swiglu_forward(fc1_out, swiglu_out, 1, aligned_intermediate_dim);
 
    /* Down projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t swiglu_q8_bytes = ((((aligned_intermediate_dim) + 255) / 256) * 292);
    uint8_t swiglu_q8[swiglu_q8_bytes];
    quantize_row_q8_k(swiglu_out, swiglu_q8, aligned_intermediate_dim);
    gemv_q4_k_q8_k(mlp_out, W2, swiglu_q8, aligned_embed_dim, aligned_intermediate_dim);
 
    /* Step 10: Final residual add */
    qwen2_0_5b_decode_residual_add_token_major(residual1, mlp_out, output, 1, aligned_embed_dim);
}

References attention_forward_decode_head_major_gqa_flash(), CK_DT_Q4_K, ck_dtype_row_bytes(), gemv_q4_k_q8_k(), QWEN2_0_5B_DECODELayerOffsets::k, QWEN2_0_5B_DECODELayerOffsets::ln1_gamma, QWEN2_0_5B_DECODELayerOffsets::ln1_out, QWEN2_0_5B_DECODELayerOffsets::ln2_gamma, QWEN2_0_5B_DECODELayerOffsets::ln2_out, QWEN2_0_5B_DECODELayerOffsets::mlp_out, QWEN2_0_5B_DECODELayerOffsets::output, QWEN2_0_5B_DECODELayerOffsets::proj_scratch, QWEN2_0_5B_DECODELayerOffsets::proj_tmp, quantize_row_q8_k(), QWEN2_0_5B_DECODE_EMBED_DIM, QWEN2_0_5B_DECODE_GLOBALS, QWEN2_0_5B_DECODE_HEAD_DIM, QWEN2_0_5B_DECODE_LAYERS, QWEN2_0_5B_DECODE_NUM_HEADS, QWEN2_0_5B_DECODE_NUM_KV_HEADS, QWEN2_0_5B_DECODE_PTR, qwen2_0_5b_decode_residual_add_token_major(), QWEN2_0_5B_DECODELayerOffsets::residual1, rmsnorm_forward(), QWEN2_0_5B_DECODEGlobalOffsets::rope_cos_cache, rope_forward(), QWEN2_0_5B_DECODEGlobalOffsets::rope_sin_cache, swiglu_forward(), QWEN2_0_5B_DECODELayerOffsets::v, QWEN2_0_5B_DECODELayerOffsets::w1, QWEN2_0_5B_DECODELayerOffsets::w2, QWEN2_0_5B_DECODELayerOffsets::wk, QWEN2_0_5B_DECODELayerOffsets::wo, QWEN2_0_5B_DECODELayerOffsets::wq, and QWEN2_0_5B_DECODELayerOffsets::wv.

Referenced by qwen2_0_5b_decode_decode_token().

qwen2_0_5b_decode_layer_10_prefill()

static void qwen2_0_5b_decode_layer_10_prefill ( QWEN2_0_5B_DECODEModel *  model, int  num_tokens, int  aligned_embed_dim, int  aligned_head_dim, int  aligned_intermediate_dim, int  aligned_context_window  )

static

Definition at line 1851 of file v6.5/test_generated/qwen2_int8.c.

  {
    const QWEN2_0_5B_DECODELayerOffsets *L = &QWEN2_0_5B_DECODE_LAYERS[10];
 
    float *input = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[9].output);
    float *ln1_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln1_gamma);
    float *ln1_out = QWEN2_0_5B_DECODE_PTR(model, L->ln1_out);
    float *ln2_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln2_gamma);
    float *ln2_out = QWEN2_0_5B_DECODE_PTR(model, L->ln2_out);
    float *q = QWEN2_0_5B_DECODE_PTR(model, L->q);
    float *k = QWEN2_0_5B_DECODE_PTR(model, L->k);
    float *v = QWEN2_0_5B_DECODE_PTR(model, L->v);
    float *attn_out = QWEN2_0_5B_DECODE_PTR(model, L->attn_out);
    float *proj_tmp = QWEN2_0_5B_DECODE_PTR(model, L->proj_tmp);
    float *proj_scratch = QWEN2_0_5B_DECODE_PTR(model, L->proj_scratch);
    float *residual1 = QWEN2_0_5B_DECODE_PTR(model, L->residual1);
    float *fc1_out = QWEN2_0_5B_DECODE_PTR(model, L->fc1_out);
    float *swiglu_out = QWEN2_0_5B_DECODE_PTR(model, L->swiglu_out);
    float *mlp_out = QWEN2_0_5B_DECODE_PTR(model, L->mlp_out);
    float *output = QWEN2_0_5B_DECODE_PTR(model, L->output);
 
    const void *WQ = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wq);
    const void *WK = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wk);
    const void *WV = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wv);
    const void *WO = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wo);
    const void *W1 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w1);
    const void *W2 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w2);
    const float *BQ = NULL;
    const float *BK = NULL;
    const float *BV = NULL;
    const float *BO = NULL;
    const float *B1 = NULL;
    const float *B2 = NULL;
 
    float *rope_cos = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_cos_cache);
    float *rope_sin = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_sin_cache);
 
    const int H = QWEN2_0_5B_DECODE_NUM_HEADS;
    const int H_kv = QWEN2_0_5B_DECODE_NUM_KV_HEADS;
    const int head_dim = QWEN2_0_5B_DECODE_HEAD_DIM;
    const size_t head_w_elems = (size_t)aligned_head_dim * (size_t)aligned_embed_dim;
    const size_t q_head_stride = (size_t)num_tokens * (size_t)aligned_head_dim;
    const size_t kv_head_stride = (size_t)aligned_context_window * (size_t)aligned_head_dim;
 
    /* RMSNorm before attention */
    rmsnorm_forward(input,
                    ln1_gamma,
                    ln1_out,
                    NULL,
                    num_tokens,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    /* Q projection (head-major) */
    const size_t wq_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, head_w_elems);
    const uint8_t *WQ_bytes = (const uint8_t *)WQ;
    for (int h = 0; h < H; ++h) {
        const void *wq_h = (const void *)(WQ_bytes + (size_t)h * wq_head_bytes);
        const float *bq_h = BQ ? (BQ + (size_t)h * (size_t)aligned_head_dim) : NULL;
        float *q_h = q + (size_t)h * q_head_stride;
        gemm_nt_q4_k(ln1_out, wq_h, bq_h, q_h, num_tokens, aligned_head_dim, aligned_embed_dim);
    }
 
    /* K projection (head-major) */
    const size_t wk_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, head_w_elems);
    const uint8_t *WK_bytes = (const uint8_t *)WK;
    for (int h = 0; h < H_kv; ++h) {
        const void *wk_h = (const void *)(WK_bytes + (size_t)h * wk_head_bytes);
        const float *bk_h = BK ? (BK + (size_t)h * (size_t)aligned_head_dim) : NULL;
        float *k_h = k + (size_t)h * kv_head_stride;
        gemm_nt_q4_k(ln1_out, wk_h, bk_h, k_h, num_tokens, aligned_head_dim, aligned_embed_dim);
    }
 
    /* V projection (head-major) */
    const size_t wv_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, head_w_elems);
    const uint8_t *WV_bytes = (const uint8_t *)WV;
    for (int h = 0; h < H_kv; ++h) {
        const void *wv_h = (const void *)(WV_bytes + (size_t)h * wv_head_bytes);
        const float *bv_h = BV ? (BV + (size_t)h * (size_t)aligned_head_dim) : NULL;
        float *v_h = v + (size_t)h * kv_head_stride;
        gemm_nt_q4_k(ln1_out, wv_h, bv_h, v_h, num_tokens, aligned_head_dim, aligned_embed_dim);
    }
 
    /* RoPE */
    rope_forward_qk_strided(q,
                            k,
                            rope_cos,
                            rope_sin,
                            H,
                            H_kv,
                            num_tokens,
                            head_dim,
                            aligned_head_dim,
                            0,
                            num_tokens,
                            aligned_context_window);
 
    /* Attention (prefill, causal) */
    attention_forward_causal_head_major_gqa_flash_strided(q,
                                                           k,
                                                           v,
                                                           attn_out,
                                                           H,
                                                           H_kv,
                                                           num_tokens,
                                                           head_dim,
                                                           aligned_head_dim,
                                                           aligned_context_window);
 
    /* Output projection (flatten head-major to token-major) */
    const int K = H * aligned_head_dim;
    if (K != aligned_embed_dim) {
        return;
    }
    const float *proj_in = attn_out;
    if (H > 1) {
        if (!proj_scratch) {
            return;
        }
        for (int t = 0; t < num_tokens; ++t) {
            float *dst = proj_scratch + (size_t)t * (size_t)aligned_embed_dim;
            for (int h = 0; h < H; ++h) {
                const float *src = attn_out + (size_t)h * q_head_stride + (size_t)t * (size_t)aligned_head_dim;
                memcpy(dst + (size_t)h * (size_t)aligned_head_dim,
                       src,
                       (size_t)aligned_head_dim * sizeof(float));
            }
        }
        proj_in = proj_scratch;
    }
    gemm_nt_q4_k(proj_in, WO, BO, proj_tmp, num_tokens, aligned_embed_dim, K);
 
    /* Residual add */
    qwen2_0_5b_decode_residual_add_token_major(input, proj_tmp, residual1, num_tokens, aligned_embed_dim);
 
    /* RMSNorm before MLP */
    rmsnorm_forward(residual1,
                    ln2_gamma,
                    ln2_out,
                    NULL,
                    num_tokens,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    /* MLP (SwiGLU) */
    gemm_nt_q4_k(ln2_out, W1, B1, fc1_out, num_tokens, 2 * aligned_intermediate_dim, aligned_embed_dim);
    swiglu_forward(fc1_out, swiglu_out, num_tokens, aligned_intermediate_dim);
    gemm_nt_q4_k(swiglu_out, W2, B2, mlp_out, num_tokens, aligned_embed_dim, aligned_intermediate_dim);
 
    /* Final residual add */
    qwen2_0_5b_decode_residual_add_token_major(residual1, mlp_out, output, num_tokens, aligned_embed_dim);
}

References attention_forward_causal_head_major_gqa_flash_strided(), CK_DT_Q4_K, ck_dtype_row_bytes(), gemm_nt_q4_k(), QWEN2_0_5B_DECODELayerOffsets::k, QWEN2_0_5B_DECODELayerOffsets::ln1_gamma, QWEN2_0_5B_DECODELayerOffsets::ln1_out, QWEN2_0_5B_DECODELayerOffsets::ln2_gamma, QWEN2_0_5B_DECODELayerOffsets::ln2_out, QWEN2_0_5B_DECODELayerOffsets::mlp_out, QWEN2_0_5B_DECODELayerOffsets::output, QWEN2_0_5B_DECODELayerOffsets::proj_scratch, QWEN2_0_5B_DECODELayerOffsets::proj_tmp, QWEN2_0_5B_DECODELayerOffsets::q, QWEN2_0_5B_DECODE_EMBED_DIM, QWEN2_0_5B_DECODE_GLOBALS, QWEN2_0_5B_DECODE_HEAD_DIM, QWEN2_0_5B_DECODE_LAYERS, QWEN2_0_5B_DECODE_NUM_HEADS, QWEN2_0_5B_DECODE_NUM_KV_HEADS, QWEN2_0_5B_DECODE_PTR, qwen2_0_5b_decode_residual_add_token_major(), QWEN2_0_5B_DECODELayerOffsets::residual1, rmsnorm_forward(), QWEN2_0_5B_DECODEGlobalOffsets::rope_cos_cache, rope_forward_qk_strided(), QWEN2_0_5B_DECODEGlobalOffsets::rope_sin_cache, swiglu_forward(), QWEN2_0_5B_DECODELayerOffsets::v, QWEN2_0_5B_DECODELayerOffsets::w1, QWEN2_0_5B_DECODELayerOffsets::w2, QWEN2_0_5B_DECODELayerOffsets::wk, QWEN2_0_5B_DECODELayerOffsets::wo, QWEN2_0_5B_DECODELayerOffsets::wq, and QWEN2_0_5B_DECODELayerOffsets::wv.

Referenced by qwen2_0_5b_decode_forward_prefill_impl().

qwen2_0_5b_decode_layer_11_decode()

static void qwen2_0_5b_decode_layer_11_decode ( QWEN2_0_5B_DECODEModel *  model, int  token_index, int  aligned_embed_dim, int  aligned_head_dim, int  aligned_intermediate_dim, int  aligned_context_window  )

static

Definition at line 6396 of file v6.5/test_generated/qwen2_int8.c.

  {
    const QWEN2_0_5B_DECODELayerOffsets *L = &QWEN2_0_5B_DECODE_LAYERS[11];
 
    float *input = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[10].output);
 
    float *ln1_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln1_gamma);
    float *ln1_out = QWEN2_0_5B_DECODE_PTR(model, L->ln1_out);
    float *ln2_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln2_gamma);
    float *ln2_out = QWEN2_0_5B_DECODE_PTR(model, L->ln2_out);
    float *k_cache = QWEN2_0_5B_DECODE_PTR(model, L->k);
    float *v_cache = QWEN2_0_5B_DECODE_PTR(model, L->v);
    float *proj_tmp = QWEN2_0_5B_DECODE_PTR(model, L->proj_tmp);
    float *proj_scratch = QWEN2_0_5B_DECODE_PTR(model, L->proj_scratch);
    float *residual1 = QWEN2_0_5B_DECODE_PTR(model, L->residual1);
    float *mlp_out = QWEN2_0_5B_DECODE_PTR(model, L->mlp_out);
    float *output = QWEN2_0_5B_DECODE_PTR(model, L->output);
 
    /* Weights (explicit types for layer 11) */
    const void *WQ = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wq);  /* Q4_K */
    const void *WK = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wk);  /* Q4_K */
    const void *WV = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wv);  /* Q4_K */
    const void *WO = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wo);  /* Q4_K */
    const void *W1 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w1);  /* Q4_K (gate+up) */
    const void *W2 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w2);  /* Q4_K (down) */
 
    float *rope_cos = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_cos_cache);
    float *rope_sin = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_sin_cache);
 
    const int H = QWEN2_0_5B_DECODE_NUM_HEADS;
    const int H_kv = QWEN2_0_5B_DECODE_NUM_KV_HEADS;
    const int head_dim = QWEN2_0_5B_DECODE_HEAD_DIM;
 
    float q_token[H * aligned_head_dim];
    float k_token[H_kv * aligned_head_dim];
    float v_token[H_kv * aligned_head_dim];
    float attn_token[H * aligned_head_dim];
 
    /* Local MLP buffers (avoid layout dependencies for intermediate values) */
    float fc1_out[2 * aligned_intermediate_dim];
    float swiglu_out[aligned_intermediate_dim];
 
    /* Step 1: RMSNorm before attention */
    rmsnorm_forward(input,
                    ln1_gamma,
                    ln1_out,
                    NULL,
                    1,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    const size_t kv_head_stride = (size_t)aligned_context_window * (size_t)aligned_head_dim;
 
    /* Step 2: QKV projection */
    /* Q projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t ln1_q8_bytes = ((((aligned_embed_dim) + 255) / 256) * 292);
    uint8_t ln1_q8[ln1_q8_bytes];
    quantize_row_q8_k(ln1_out, ln1_q8, aligned_embed_dim);
    gemv_q4_k_q8_k(q_token, WQ, ln1_q8, H * head_dim, aligned_embed_dim);
    if (aligned_head_dim > head_dim) {
        for (int h = 0; h < H; ++h) {
            float *q_head = q_token + (size_t)h * (size_t)aligned_head_dim;
            for (int d = head_dim; d < aligned_head_dim; ++d) {
                q_head[d] = 0.0f;
            }
        }
    }
 
    /* K projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k (direct-to-cache) */
    const size_t wk_head_elems = (size_t)aligned_head_dim * (size_t)aligned_embed_dim;
    const size_t wk_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, wk_head_elems);
    const uint8_t *WK_bytes = (const uint8_t *)WK;
    /* ln1_q8 already quantized above */
    for (int h = 0; h < H_kv; ++h) {
        const void *wk_h = (const void *)(WK_bytes + (size_t)h * wk_head_bytes);
        float *k_head = k_cache + (size_t)h * kv_head_stride + (size_t)token_index * (size_t)aligned_head_dim;
        gemv_q4_k_q8_k(k_head, wk_h, ln1_q8, head_dim, aligned_embed_dim);
        for (int d = head_dim; d < aligned_head_dim; ++d) {
            k_head[d] = 0.0f;
        }
    }
 
    /* V projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k (direct-to-cache) */
    const size_t wv_head_elems = (size_t)aligned_head_dim * (size_t)aligned_embed_dim;
    const size_t wv_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, wv_head_elems);
    const uint8_t *WV_bytes = (const uint8_t *)WV;
    /* ln1_q8 already quantized above */
    for (int h = 0; h < H_kv; ++h) {
        const void *wv_h = (const void *)(WV_bytes + (size_t)h * wv_head_bytes);
        float *v_head = v_cache + (size_t)h * kv_head_stride + (size_t)token_index * (size_t)aligned_head_dim;
        gemv_q4_k_q8_k(v_head, wv_h, ln1_q8, head_dim, aligned_embed_dim);
        for (int d = head_dim; d < aligned_head_dim; ++d) {
            v_head[d] = 0.0f;
        }
    }
 
    /* Step 3: RoPE */
    rope_forward(q_token,
                 rope_cos,
                 rope_sin,
                 H,
                 1,
                 head_dim,
                 aligned_head_dim,
                 token_index);
    for (int h = 0; h < H_kv; ++h) {
        float *k_head = k_cache + (size_t)h * kv_head_stride + (size_t)token_index * (size_t)aligned_head_dim;
        rope_forward(k_head,
                     rope_cos,
                     rope_sin,
                     1,
                     1,
                     head_dim,
                     aligned_head_dim,
                     token_index);
    }
 
    /* Step 4: KV cache write (direct-to-cache) */
 
    /* Step 5: Attention (decode, flash) */
    attention_forward_decode_head_major_gqa_flash(q_token,
                                                   k_cache,
                                                   v_cache,
                                                   attn_token,
                                                   H,
                                                   H_kv,
                                                   token_index + 1,
                                                   aligned_context_window,
                                                   head_dim,
                                                   aligned_head_dim);
 
    /* Step 6: Output projection */
    /* WO projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t attn_q8_bytes = ((((H * head_dim) + 255) / 256) * 292);
    uint8_t attn_q8[attn_q8_bytes];
    quantize_row_q8_k(attn_token, attn_q8, H * head_dim);
    gemv_q4_k_q8_k(proj_tmp, WO, attn_q8, aligned_embed_dim, H * head_dim);
 
    /* Step 7: Residual add */
    qwen2_0_5b_decode_residual_add_token_major(input, proj_tmp, residual1, 1, aligned_embed_dim);
 
    /* Step 8: RMSNorm before MLP */
    rmsnorm_forward(residual1,
                    ln2_gamma,
                    ln2_out,
                    NULL,
                    1,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    /* Step 9: MLP (SwiGLU) */
    /* Gate+Up projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t ln2_q8_bytes = ((((aligned_embed_dim) + 255) / 256) * 292);
    uint8_t ln2_q8[ln2_q8_bytes];
    quantize_row_q8_k(ln2_out, ln2_q8, aligned_embed_dim);
    gemv_q4_k_q8_k(fc1_out, W1, ln2_q8, 2 * aligned_intermediate_dim, aligned_embed_dim);
 
    /* SwiGLU activation */
    swiglu_forward(fc1_out, swiglu_out, 1, aligned_intermediate_dim);
 
    /* Down projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t swiglu_q8_bytes = ((((aligned_intermediate_dim) + 255) / 256) * 292);
    uint8_t swiglu_q8[swiglu_q8_bytes];
    quantize_row_q8_k(swiglu_out, swiglu_q8, aligned_intermediate_dim);
    gemv_q4_k_q8_k(mlp_out, W2, swiglu_q8, aligned_embed_dim, aligned_intermediate_dim);
 
    /* Step 10: Final residual add */
    qwen2_0_5b_decode_residual_add_token_major(residual1, mlp_out, output, 1, aligned_embed_dim);
}

References attention_forward_decode_head_major_gqa_flash(), CK_DT_Q4_K, ck_dtype_row_bytes(), gemv_q4_k_q8_k(), QWEN2_0_5B_DECODELayerOffsets::k, QWEN2_0_5B_DECODELayerOffsets::ln1_gamma, QWEN2_0_5B_DECODELayerOffsets::ln1_out, QWEN2_0_5B_DECODELayerOffsets::ln2_gamma, QWEN2_0_5B_DECODELayerOffsets::ln2_out, QWEN2_0_5B_DECODELayerOffsets::mlp_out, QWEN2_0_5B_DECODELayerOffsets::output, QWEN2_0_5B_DECODELayerOffsets::proj_scratch, QWEN2_0_5B_DECODELayerOffsets::proj_tmp, quantize_row_q8_k(), QWEN2_0_5B_DECODE_EMBED_DIM, QWEN2_0_5B_DECODE_GLOBALS, QWEN2_0_5B_DECODE_HEAD_DIM, QWEN2_0_5B_DECODE_LAYERS, QWEN2_0_5B_DECODE_NUM_HEADS, QWEN2_0_5B_DECODE_NUM_KV_HEADS, QWEN2_0_5B_DECODE_PTR, qwen2_0_5b_decode_residual_add_token_major(), QWEN2_0_5B_DECODELayerOffsets::residual1, rmsnorm_forward(), QWEN2_0_5B_DECODEGlobalOffsets::rope_cos_cache, rope_forward(), QWEN2_0_5B_DECODEGlobalOffsets::rope_sin_cache, swiglu_forward(), QWEN2_0_5B_DECODELayerOffsets::v, QWEN2_0_5B_DECODELayerOffsets::w1, QWEN2_0_5B_DECODELayerOffsets::w2, QWEN2_0_5B_DECODELayerOffsets::wk, QWEN2_0_5B_DECODELayerOffsets::wo, QWEN2_0_5B_DECODELayerOffsets::wq, and QWEN2_0_5B_DECODELayerOffsets::wv.

Referenced by qwen2_0_5b_decode_decode_token().

qwen2_0_5b_decode_layer_11_prefill()

static void qwen2_0_5b_decode_layer_11_prefill ( QWEN2_0_5B_DECODEModel *  model, int  num_tokens, int  aligned_embed_dim, int  aligned_head_dim, int  aligned_intermediate_dim, int  aligned_context_window  )

static

Definition at line 2015 of file v6.5/test_generated/qwen2_int8.c.

  {
    const QWEN2_0_5B_DECODELayerOffsets *L = &QWEN2_0_5B_DECODE_LAYERS[11];
 
    float *input = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[10].output);
    float *ln1_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln1_gamma);
    float *ln1_out = QWEN2_0_5B_DECODE_PTR(model, L->ln1_out);
    float *ln2_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln2_gamma);
    float *ln2_out = QWEN2_0_5B_DECODE_PTR(model, L->ln2_out);
    float *q = QWEN2_0_5B_DECODE_PTR(model, L->q);
    float *k = QWEN2_0_5B_DECODE_PTR(model, L->k);
    float *v = QWEN2_0_5B_DECODE_PTR(model, L->v);
    float *attn_out = QWEN2_0_5B_DECODE_PTR(model, L->attn_out);
    float *proj_tmp = QWEN2_0_5B_DECODE_PTR(model, L->proj_tmp);
    float *proj_scratch = QWEN2_0_5B_DECODE_PTR(model, L->proj_scratch);
    float *residual1 = QWEN2_0_5B_DECODE_PTR(model, L->residual1);
    float *fc1_out = QWEN2_0_5B_DECODE_PTR(model, L->fc1_out);
    float *swiglu_out = QWEN2_0_5B_DECODE_PTR(model, L->swiglu_out);
    float *mlp_out = QWEN2_0_5B_DECODE_PTR(model, L->mlp_out);
    float *output = QWEN2_0_5B_DECODE_PTR(model, L->output);
 
    const void *WQ = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wq);
    const void *WK = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wk);
    const void *WV = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wv);
    const void *WO = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wo);
    const void *W1 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w1);
    const void *W2 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w2);
    const float *BQ = NULL;
    const float *BK = NULL;
    const float *BV = NULL;
    const float *BO = NULL;
    const float *B1 = NULL;
    const float *B2 = NULL;
 
    float *rope_cos = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_cos_cache);
    float *rope_sin = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_sin_cache);
 
    const int H = QWEN2_0_5B_DECODE_NUM_HEADS;
    const int H_kv = QWEN2_0_5B_DECODE_NUM_KV_HEADS;
    const int head_dim = QWEN2_0_5B_DECODE_HEAD_DIM;
    const size_t head_w_elems = (size_t)aligned_head_dim * (size_t)aligned_embed_dim;
    const size_t q_head_stride = (size_t)num_tokens * (size_t)aligned_head_dim;
    const size_t kv_head_stride = (size_t)aligned_context_window * (size_t)aligned_head_dim;
 
    /* RMSNorm before attention */
    rmsnorm_forward(input,
                    ln1_gamma,
                    ln1_out,
                    NULL,
                    num_tokens,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    /* Q projection (head-major) */
    const size_t wq_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, head_w_elems);
    const uint8_t *WQ_bytes = (const uint8_t *)WQ;
    for (int h = 0; h < H; ++h) {
        const void *wq_h = (const void *)(WQ_bytes + (size_t)h * wq_head_bytes);
        const float *bq_h = BQ ? (BQ + (size_t)h * (size_t)aligned_head_dim) : NULL;
        float *q_h = q + (size_t)h * q_head_stride;
        gemm_nt_q4_k(ln1_out, wq_h, bq_h, q_h, num_tokens, aligned_head_dim, aligned_embed_dim);
    }
 
    /* K projection (head-major) */
    const size_t wk_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, head_w_elems);
    const uint8_t *WK_bytes = (const uint8_t *)WK;
    for (int h = 0; h < H_kv; ++h) {
        const void *wk_h = (const void *)(WK_bytes + (size_t)h * wk_head_bytes);
        const float *bk_h = BK ? (BK + (size_t)h * (size_t)aligned_head_dim) : NULL;
        float *k_h = k + (size_t)h * kv_head_stride;
        gemm_nt_q4_k(ln1_out, wk_h, bk_h, k_h, num_tokens, aligned_head_dim, aligned_embed_dim);
    }
 
    /* V projection (head-major) */
    const size_t wv_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, head_w_elems);
    const uint8_t *WV_bytes = (const uint8_t *)WV;
    for (int h = 0; h < H_kv; ++h) {
        const void *wv_h = (const void *)(WV_bytes + (size_t)h * wv_head_bytes);
        const float *bv_h = BV ? (BV + (size_t)h * (size_t)aligned_head_dim) : NULL;
        float *v_h = v + (size_t)h * kv_head_stride;
        gemm_nt_q4_k(ln1_out, wv_h, bv_h, v_h, num_tokens, aligned_head_dim, aligned_embed_dim);
    }
 
    /* RoPE */
    rope_forward_qk_strided(q,
                            k,
                            rope_cos,
                            rope_sin,
                            H,
                            H_kv,
                            num_tokens,
                            head_dim,
                            aligned_head_dim,
                            0,
                            num_tokens,
                            aligned_context_window);
 
    /* Attention (prefill, causal) */
    attention_forward_causal_head_major_gqa_flash_strided(q,
                                                           k,
                                                           v,
                                                           attn_out,
                                                           H,
                                                           H_kv,
                                                           num_tokens,
                                                           head_dim,
                                                           aligned_head_dim,
                                                           aligned_context_window);
 
    /* Output projection (flatten head-major to token-major) */
    const int K = H * aligned_head_dim;
    if (K != aligned_embed_dim) {
        return;
    }
    const float *proj_in = attn_out;
    if (H > 1) {
        if (!proj_scratch) {
            return;
        }
        for (int t = 0; t < num_tokens; ++t) {
            float *dst = proj_scratch + (size_t)t * (size_t)aligned_embed_dim;
            for (int h = 0; h < H; ++h) {
                const float *src = attn_out + (size_t)h * q_head_stride + (size_t)t * (size_t)aligned_head_dim;
                memcpy(dst + (size_t)h * (size_t)aligned_head_dim,
                       src,
                       (size_t)aligned_head_dim * sizeof(float));
            }
        }
        proj_in = proj_scratch;
    }
    gemm_nt_q4_k(proj_in, WO, BO, proj_tmp, num_tokens, aligned_embed_dim, K);
 
    /* Residual add */
    qwen2_0_5b_decode_residual_add_token_major(input, proj_tmp, residual1, num_tokens, aligned_embed_dim);
 
    /* RMSNorm before MLP */
    rmsnorm_forward(residual1,
                    ln2_gamma,
                    ln2_out,
                    NULL,
                    num_tokens,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    /* MLP (SwiGLU) */
    gemm_nt_q4_k(ln2_out, W1, B1, fc1_out, num_tokens, 2 * aligned_intermediate_dim, aligned_embed_dim);
    swiglu_forward(fc1_out, swiglu_out, num_tokens, aligned_intermediate_dim);
    gemm_nt_q4_k(swiglu_out, W2, B2, mlp_out, num_tokens, aligned_embed_dim, aligned_intermediate_dim);
 
    /* Final residual add */
    qwen2_0_5b_decode_residual_add_token_major(residual1, mlp_out, output, num_tokens, aligned_embed_dim);
}

References attention_forward_causal_head_major_gqa_flash_strided(), CK_DT_Q4_K, ck_dtype_row_bytes(), gemm_nt_q4_k(), QWEN2_0_5B_DECODELayerOffsets::k, QWEN2_0_5B_DECODELayerOffsets::ln1_gamma, QWEN2_0_5B_DECODELayerOffsets::ln1_out, QWEN2_0_5B_DECODELayerOffsets::ln2_gamma, QWEN2_0_5B_DECODELayerOffsets::ln2_out, QWEN2_0_5B_DECODELayerOffsets::mlp_out, QWEN2_0_5B_DECODELayerOffsets::output, QWEN2_0_5B_DECODELayerOffsets::proj_scratch, QWEN2_0_5B_DECODELayerOffsets::proj_tmp, QWEN2_0_5B_DECODELayerOffsets::q, QWEN2_0_5B_DECODE_EMBED_DIM, QWEN2_0_5B_DECODE_GLOBALS, QWEN2_0_5B_DECODE_HEAD_DIM, QWEN2_0_5B_DECODE_LAYERS, QWEN2_0_5B_DECODE_NUM_HEADS, QWEN2_0_5B_DECODE_NUM_KV_HEADS, QWEN2_0_5B_DECODE_PTR, qwen2_0_5b_decode_residual_add_token_major(), QWEN2_0_5B_DECODELayerOffsets::residual1, rmsnorm_forward(), QWEN2_0_5B_DECODEGlobalOffsets::rope_cos_cache, rope_forward_qk_strided(), QWEN2_0_5B_DECODEGlobalOffsets::rope_sin_cache, swiglu_forward(), QWEN2_0_5B_DECODELayerOffsets::v, QWEN2_0_5B_DECODELayerOffsets::w1, QWEN2_0_5B_DECODELayerOffsets::w2, QWEN2_0_5B_DECODELayerOffsets::wk, QWEN2_0_5B_DECODELayerOffsets::wo, QWEN2_0_5B_DECODELayerOffsets::wq, and QWEN2_0_5B_DECODELayerOffsets::wv.

Referenced by qwen2_0_5b_decode_forward_prefill_impl().

qwen2_0_5b_decode_layer_12_decode()

static void qwen2_0_5b_decode_layer_12_decode ( QWEN2_0_5B_DECODEModel *  model, int  token_index, int  aligned_embed_dim, int  aligned_head_dim, int  aligned_intermediate_dim, int  aligned_context_window  )

static

Definition at line 6577 of file v6.5/test_generated/qwen2_int8.c.

  {
    const QWEN2_0_5B_DECODELayerOffsets *L = &QWEN2_0_5B_DECODE_LAYERS[12];
 
    float *input = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[11].output);
 
    float *ln1_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln1_gamma);
    float *ln1_out = QWEN2_0_5B_DECODE_PTR(model, L->ln1_out);
    float *ln2_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln2_gamma);
    float *ln2_out = QWEN2_0_5B_DECODE_PTR(model, L->ln2_out);
    float *k_cache = QWEN2_0_5B_DECODE_PTR(model, L->k);
    float *v_cache = QWEN2_0_5B_DECODE_PTR(model, L->v);
    float *proj_tmp = QWEN2_0_5B_DECODE_PTR(model, L->proj_tmp);
    float *proj_scratch = QWEN2_0_5B_DECODE_PTR(model, L->proj_scratch);
    float *residual1 = QWEN2_0_5B_DECODE_PTR(model, L->residual1);
    float *mlp_out = QWEN2_0_5B_DECODE_PTR(model, L->mlp_out);
    float *output = QWEN2_0_5B_DECODE_PTR(model, L->output);
 
    /* Weights (explicit types for layer 12) */
    const void *WQ = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wq);  /* Q4_K */
    const void *WK = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wk);  /* Q4_K */
    const void *WV = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wv);  /* Q4_K */
    const void *WO = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wo);  /* Q4_K */
    const void *W1 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w1);  /* Q4_K (gate+up) */
    const void *W2 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w2);  /* Q4_K (down) */
 
    float *rope_cos = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_cos_cache);
    float *rope_sin = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_sin_cache);
 
    const int H = QWEN2_0_5B_DECODE_NUM_HEADS;
    const int H_kv = QWEN2_0_5B_DECODE_NUM_KV_HEADS;
    const int head_dim = QWEN2_0_5B_DECODE_HEAD_DIM;
 
    float q_token[H * aligned_head_dim];
    float k_token[H_kv * aligned_head_dim];
    float v_token[H_kv * aligned_head_dim];
    float attn_token[H * aligned_head_dim];
 
    /* Local MLP buffers (avoid layout dependencies for intermediate values) */
    float fc1_out[2 * aligned_intermediate_dim];
    float swiglu_out[aligned_intermediate_dim];
 
    /* Step 1: RMSNorm before attention */
    rmsnorm_forward(input,
                    ln1_gamma,
                    ln1_out,
                    NULL,
                    1,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    const size_t kv_head_stride = (size_t)aligned_context_window * (size_t)aligned_head_dim;
 
    /* Step 2: QKV projection */
    /* Q projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t ln1_q8_bytes = ((((aligned_embed_dim) + 255) / 256) * 292);
    uint8_t ln1_q8[ln1_q8_bytes];
    quantize_row_q8_k(ln1_out, ln1_q8, aligned_embed_dim);
    gemv_q4_k_q8_k(q_token, WQ, ln1_q8, H * head_dim, aligned_embed_dim);
    if (aligned_head_dim > head_dim) {
        for (int h = 0; h < H; ++h) {
            float *q_head = q_token + (size_t)h * (size_t)aligned_head_dim;
            for (int d = head_dim; d < aligned_head_dim; ++d) {
                q_head[d] = 0.0f;
            }
        }
    }
 
    /* K projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k (direct-to-cache) */
    const size_t wk_head_elems = (size_t)aligned_head_dim * (size_t)aligned_embed_dim;
    const size_t wk_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, wk_head_elems);
    const uint8_t *WK_bytes = (const uint8_t *)WK;
    /* ln1_q8 already quantized above */
    for (int h = 0; h < H_kv; ++h) {
        const void *wk_h = (const void *)(WK_bytes + (size_t)h * wk_head_bytes);
        float *k_head = k_cache + (size_t)h * kv_head_stride + (size_t)token_index * (size_t)aligned_head_dim;
        gemv_q4_k_q8_k(k_head, wk_h, ln1_q8, head_dim, aligned_embed_dim);
        for (int d = head_dim; d < aligned_head_dim; ++d) {
            k_head[d] = 0.0f;
        }
    }
 
    /* V projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k (direct-to-cache) */
    const size_t wv_head_elems = (size_t)aligned_head_dim * (size_t)aligned_embed_dim;
    const size_t wv_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, wv_head_elems);
    const uint8_t *WV_bytes = (const uint8_t *)WV;
    /* ln1_q8 already quantized above */
    for (int h = 0; h < H_kv; ++h) {
        const void *wv_h = (const void *)(WV_bytes + (size_t)h * wv_head_bytes);
        float *v_head = v_cache + (size_t)h * kv_head_stride + (size_t)token_index * (size_t)aligned_head_dim;
        gemv_q4_k_q8_k(v_head, wv_h, ln1_q8, head_dim, aligned_embed_dim);
        for (int d = head_dim; d < aligned_head_dim; ++d) {
            v_head[d] = 0.0f;
        }
    }
 
    /* Step 3: RoPE */
    rope_forward(q_token,
                 rope_cos,
                 rope_sin,
                 H,
                 1,
                 head_dim,
                 aligned_head_dim,
                 token_index);
    for (int h = 0; h < H_kv; ++h) {
        float *k_head = k_cache + (size_t)h * kv_head_stride + (size_t)token_index * (size_t)aligned_head_dim;
        rope_forward(k_head,
                     rope_cos,
                     rope_sin,
                     1,
                     1,
                     head_dim,
                     aligned_head_dim,
                     token_index);
    }
 
    /* Step 4: KV cache write (direct-to-cache) */
 
    /* Step 5: Attention (decode, flash) */
    attention_forward_decode_head_major_gqa_flash(q_token,
                                                   k_cache,
                                                   v_cache,
                                                   attn_token,
                                                   H,
                                                   H_kv,
                                                   token_index + 1,
                                                   aligned_context_window,
                                                   head_dim,
                                                   aligned_head_dim);
 
    /* Step 6: Output projection */
    /* WO projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t attn_q8_bytes = ((((H * head_dim) + 255) / 256) * 292);
    uint8_t attn_q8[attn_q8_bytes];
    quantize_row_q8_k(attn_token, attn_q8, H * head_dim);
    gemv_q4_k_q8_k(proj_tmp, WO, attn_q8, aligned_embed_dim, H * head_dim);
 
    /* Step 7: Residual add */
    qwen2_0_5b_decode_residual_add_token_major(input, proj_tmp, residual1, 1, aligned_embed_dim);
 
    /* Step 8: RMSNorm before MLP */
    rmsnorm_forward(residual1,
                    ln2_gamma,
                    ln2_out,
                    NULL,
                    1,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    /* Step 9: MLP (SwiGLU) */
    /* Gate+Up projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t ln2_q8_bytes = ((((aligned_embed_dim) + 255) / 256) * 292);
    uint8_t ln2_q8[ln2_q8_bytes];
    quantize_row_q8_k(ln2_out, ln2_q8, aligned_embed_dim);
    gemv_q4_k_q8_k(fc1_out, W1, ln2_q8, 2 * aligned_intermediate_dim, aligned_embed_dim);
 
    /* SwiGLU activation */
    swiglu_forward(fc1_out, swiglu_out, 1, aligned_intermediate_dim);
 
    /* Down projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t swiglu_q8_bytes = ((((aligned_intermediate_dim) + 255) / 256) * 292);
    uint8_t swiglu_q8[swiglu_q8_bytes];
    quantize_row_q8_k(swiglu_out, swiglu_q8, aligned_intermediate_dim);
    gemv_q4_k_q8_k(mlp_out, W2, swiglu_q8, aligned_embed_dim, aligned_intermediate_dim);
 
    /* Step 10: Final residual add */
    qwen2_0_5b_decode_residual_add_token_major(residual1, mlp_out, output, 1, aligned_embed_dim);
}

References attention_forward_decode_head_major_gqa_flash(), CK_DT_Q4_K, ck_dtype_row_bytes(), gemv_q4_k_q8_k(), QWEN2_0_5B_DECODELayerOffsets::k, QWEN2_0_5B_DECODELayerOffsets::ln1_gamma, QWEN2_0_5B_DECODELayerOffsets::ln1_out, QWEN2_0_5B_DECODELayerOffsets::ln2_gamma, QWEN2_0_5B_DECODELayerOffsets::ln2_out, QWEN2_0_5B_DECODELayerOffsets::mlp_out, QWEN2_0_5B_DECODELayerOffsets::output, QWEN2_0_5B_DECODELayerOffsets::proj_scratch, QWEN2_0_5B_DECODELayerOffsets::proj_tmp, quantize_row_q8_k(), QWEN2_0_5B_DECODE_EMBED_DIM, QWEN2_0_5B_DECODE_GLOBALS, QWEN2_0_5B_DECODE_HEAD_DIM, QWEN2_0_5B_DECODE_LAYERS, QWEN2_0_5B_DECODE_NUM_HEADS, QWEN2_0_5B_DECODE_NUM_KV_HEADS, QWEN2_0_5B_DECODE_PTR, qwen2_0_5b_decode_residual_add_token_major(), QWEN2_0_5B_DECODELayerOffsets::residual1, rmsnorm_forward(), QWEN2_0_5B_DECODEGlobalOffsets::rope_cos_cache, rope_forward(), QWEN2_0_5B_DECODEGlobalOffsets::rope_sin_cache, swiglu_forward(), QWEN2_0_5B_DECODELayerOffsets::v, QWEN2_0_5B_DECODELayerOffsets::w1, QWEN2_0_5B_DECODELayerOffsets::w2, QWEN2_0_5B_DECODELayerOffsets::wk, QWEN2_0_5B_DECODELayerOffsets::wo, QWEN2_0_5B_DECODELayerOffsets::wq, and QWEN2_0_5B_DECODELayerOffsets::wv.

Referenced by qwen2_0_5b_decode_decode_token().

qwen2_0_5b_decode_layer_12_prefill()

static void qwen2_0_5b_decode_layer_12_prefill ( QWEN2_0_5B_DECODEModel *  model, int  num_tokens, int  aligned_embed_dim, int  aligned_head_dim, int  aligned_intermediate_dim, int  aligned_context_window  )

static

Definition at line 2179 of file v6.5/test_generated/qwen2_int8.c.

  {
    const QWEN2_0_5B_DECODELayerOffsets *L = &QWEN2_0_5B_DECODE_LAYERS[12];
 
    float *input = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[11].output);
    float *ln1_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln1_gamma);
    float *ln1_out = QWEN2_0_5B_DECODE_PTR(model, L->ln1_out);
    float *ln2_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln2_gamma);
    float *ln2_out = QWEN2_0_5B_DECODE_PTR(model, L->ln2_out);
    float *q = QWEN2_0_5B_DECODE_PTR(model, L->q);
    float *k = QWEN2_0_5B_DECODE_PTR(model, L->k);
    float *v = QWEN2_0_5B_DECODE_PTR(model, L->v);
    float *attn_out = QWEN2_0_5B_DECODE_PTR(model, L->attn_out);
    float *proj_tmp = QWEN2_0_5B_DECODE_PTR(model, L->proj_tmp);
    float *proj_scratch = QWEN2_0_5B_DECODE_PTR(model, L->proj_scratch);
    float *residual1 = QWEN2_0_5B_DECODE_PTR(model, L->residual1);
    float *fc1_out = QWEN2_0_5B_DECODE_PTR(model, L->fc1_out);
    float *swiglu_out = QWEN2_0_5B_DECODE_PTR(model, L->swiglu_out);
    float *mlp_out = QWEN2_0_5B_DECODE_PTR(model, L->mlp_out);
    float *output = QWEN2_0_5B_DECODE_PTR(model, L->output);
 
    const void *WQ = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wq);
    const void *WK = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wk);
    const void *WV = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wv);
    const void *WO = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wo);
    const void *W1 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w1);
    const void *W2 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w2);
    const float *BQ = NULL;
    const float *BK = NULL;
    const float *BV = NULL;
    const float *BO = NULL;
    const float *B1 = NULL;
    const float *B2 = NULL;
 
    float *rope_cos = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_cos_cache);
    float *rope_sin = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_sin_cache);
 
    const int H = QWEN2_0_5B_DECODE_NUM_HEADS;
    const int H_kv = QWEN2_0_5B_DECODE_NUM_KV_HEADS;
    const int head_dim = QWEN2_0_5B_DECODE_HEAD_DIM;
    const size_t head_w_elems = (size_t)aligned_head_dim * (size_t)aligned_embed_dim;
    const size_t q_head_stride = (size_t)num_tokens * (size_t)aligned_head_dim;
    const size_t kv_head_stride = (size_t)aligned_context_window * (size_t)aligned_head_dim;
 
    /* RMSNorm before attention */
    rmsnorm_forward(input,
                    ln1_gamma,
                    ln1_out,
                    NULL,
                    num_tokens,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    /* Q projection (head-major) */
    const size_t wq_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, head_w_elems);
    const uint8_t *WQ_bytes = (const uint8_t *)WQ;
    for (int h = 0; h < H; ++h) {
        const void *wq_h = (const void *)(WQ_bytes + (size_t)h * wq_head_bytes);
        const float *bq_h = BQ ? (BQ + (size_t)h * (size_t)aligned_head_dim) : NULL;
        float *q_h = q + (size_t)h * q_head_stride;
        gemm_nt_q4_k(ln1_out, wq_h, bq_h, q_h, num_tokens, aligned_head_dim, aligned_embed_dim);
    }
 
    /* K projection (head-major) */
    const size_t wk_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, head_w_elems);
    const uint8_t *WK_bytes = (const uint8_t *)WK;
    for (int h = 0; h < H_kv; ++h) {
        const void *wk_h = (const void *)(WK_bytes + (size_t)h * wk_head_bytes);
        const float *bk_h = BK ? (BK + (size_t)h * (size_t)aligned_head_dim) : NULL;
        float *k_h = k + (size_t)h * kv_head_stride;
        gemm_nt_q4_k(ln1_out, wk_h, bk_h, k_h, num_tokens, aligned_head_dim, aligned_embed_dim);
    }
 
    /* V projection (head-major) */
    const size_t wv_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, head_w_elems);
    const uint8_t *WV_bytes = (const uint8_t *)WV;
    for (int h = 0; h < H_kv; ++h) {
        const void *wv_h = (const void *)(WV_bytes + (size_t)h * wv_head_bytes);
        const float *bv_h = BV ? (BV + (size_t)h * (size_t)aligned_head_dim) : NULL;
        float *v_h = v + (size_t)h * kv_head_stride;
        gemm_nt_q4_k(ln1_out, wv_h, bv_h, v_h, num_tokens, aligned_head_dim, aligned_embed_dim);
    }
 
    /* RoPE */
    rope_forward_qk_strided(q,
                            k,
                            rope_cos,
                            rope_sin,
                            H,
                            H_kv,
                            num_tokens,
                            head_dim,
                            aligned_head_dim,
                            0,
                            num_tokens,
                            aligned_context_window);
 
    /* Attention (prefill, causal) */
    attention_forward_causal_head_major_gqa_flash_strided(q,
                                                           k,
                                                           v,
                                                           attn_out,
                                                           H,
                                                           H_kv,
                                                           num_tokens,
                                                           head_dim,
                                                           aligned_head_dim,
                                                           aligned_context_window);
 
    /* Output projection (flatten head-major to token-major) */
    const int K = H * aligned_head_dim;
    if (K != aligned_embed_dim) {
        return;
    }
    const float *proj_in = attn_out;
    if (H > 1) {
        if (!proj_scratch) {
            return;
        }
        for (int t = 0; t < num_tokens; ++t) {
            float *dst = proj_scratch + (size_t)t * (size_t)aligned_embed_dim;
            for (int h = 0; h < H; ++h) {
                const float *src = attn_out + (size_t)h * q_head_stride + (size_t)t * (size_t)aligned_head_dim;
                memcpy(dst + (size_t)h * (size_t)aligned_head_dim,
                       src,
                       (size_t)aligned_head_dim * sizeof(float));
            }
        }
        proj_in = proj_scratch;
    }
    gemm_nt_q4_k(proj_in, WO, BO, proj_tmp, num_tokens, aligned_embed_dim, K);
 
    /* Residual add */
    qwen2_0_5b_decode_residual_add_token_major(input, proj_tmp, residual1, num_tokens, aligned_embed_dim);
 
    /* RMSNorm before MLP */
    rmsnorm_forward(residual1,
                    ln2_gamma,
                    ln2_out,
                    NULL,
                    num_tokens,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    /* MLP (SwiGLU) */
    gemm_nt_q4_k(ln2_out, W1, B1, fc1_out, num_tokens, 2 * aligned_intermediate_dim, aligned_embed_dim);
    swiglu_forward(fc1_out, swiglu_out, num_tokens, aligned_intermediate_dim);
    gemm_nt_q4_k(swiglu_out, W2, B2, mlp_out, num_tokens, aligned_embed_dim, aligned_intermediate_dim);
 
    /* Final residual add */
    qwen2_0_5b_decode_residual_add_token_major(residual1, mlp_out, output, num_tokens, aligned_embed_dim);
}

References attention_forward_causal_head_major_gqa_flash_strided(), CK_DT_Q4_K, ck_dtype_row_bytes(), gemm_nt_q4_k(), QWEN2_0_5B_DECODELayerOffsets::k, QWEN2_0_5B_DECODELayerOffsets::ln1_gamma, QWEN2_0_5B_DECODELayerOffsets::ln1_out, QWEN2_0_5B_DECODELayerOffsets::ln2_gamma, QWEN2_0_5B_DECODELayerOffsets::ln2_out, QWEN2_0_5B_DECODELayerOffsets::mlp_out, QWEN2_0_5B_DECODELayerOffsets::output, QWEN2_0_5B_DECODELayerOffsets::proj_scratch, QWEN2_0_5B_DECODELayerOffsets::proj_tmp, QWEN2_0_5B_DECODELayerOffsets::q, QWEN2_0_5B_DECODE_EMBED_DIM, QWEN2_0_5B_DECODE_GLOBALS, QWEN2_0_5B_DECODE_HEAD_DIM, QWEN2_0_5B_DECODE_LAYERS, QWEN2_0_5B_DECODE_NUM_HEADS, QWEN2_0_5B_DECODE_NUM_KV_HEADS, QWEN2_0_5B_DECODE_PTR, qwen2_0_5b_decode_residual_add_token_major(), QWEN2_0_5B_DECODELayerOffsets::residual1, rmsnorm_forward(), QWEN2_0_5B_DECODEGlobalOffsets::rope_cos_cache, rope_forward_qk_strided(), QWEN2_0_5B_DECODEGlobalOffsets::rope_sin_cache, swiglu_forward(), QWEN2_0_5B_DECODELayerOffsets::v, QWEN2_0_5B_DECODELayerOffsets::w1, QWEN2_0_5B_DECODELayerOffsets::w2, QWEN2_0_5B_DECODELayerOffsets::wk, QWEN2_0_5B_DECODELayerOffsets::wo, QWEN2_0_5B_DECODELayerOffsets::wq, and QWEN2_0_5B_DECODELayerOffsets::wv.

Referenced by qwen2_0_5b_decode_forward_prefill_impl().

qwen2_0_5b_decode_layer_13_decode()

static void qwen2_0_5b_decode_layer_13_decode ( QWEN2_0_5B_DECODEModel *  model, int  token_index, int  aligned_embed_dim, int  aligned_head_dim, int  aligned_intermediate_dim, int  aligned_context_window  )

static

Definition at line 6758 of file v6.5/test_generated/qwen2_int8.c.

  {
    const QWEN2_0_5B_DECODELayerOffsets *L = &QWEN2_0_5B_DECODE_LAYERS[13];
 
    float *input = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[12].output);
 
    float *ln1_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln1_gamma);
    float *ln1_out = QWEN2_0_5B_DECODE_PTR(model, L->ln1_out);
    float *ln2_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln2_gamma);
    float *ln2_out = QWEN2_0_5B_DECODE_PTR(model, L->ln2_out);
    float *k_cache = QWEN2_0_5B_DECODE_PTR(model, L->k);
    float *v_cache = QWEN2_0_5B_DECODE_PTR(model, L->v);
    float *proj_tmp = QWEN2_0_5B_DECODE_PTR(model, L->proj_tmp);
    float *proj_scratch = QWEN2_0_5B_DECODE_PTR(model, L->proj_scratch);
    float *residual1 = QWEN2_0_5B_DECODE_PTR(model, L->residual1);
    float *mlp_out = QWEN2_0_5B_DECODE_PTR(model, L->mlp_out);
    float *output = QWEN2_0_5B_DECODE_PTR(model, L->output);
 
    /* Weights (explicit types for layer 13) */
    const void *WQ = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wq);  /* Q4_K */
    const void *WK = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wk);  /* Q4_K */
    const void *WV = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wv);  /* Q4_K */
    const void *WO = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wo);  /* Q4_K */
    const void *W1 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w1);  /* Q4_K (gate+up) */
    const void *W2 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w2);  /* Q4_K (down) */
 
    float *rope_cos = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_cos_cache);
    float *rope_sin = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_sin_cache);
 
    const int H = QWEN2_0_5B_DECODE_NUM_HEADS;
    const int H_kv = QWEN2_0_5B_DECODE_NUM_KV_HEADS;
    const int head_dim = QWEN2_0_5B_DECODE_HEAD_DIM;
 
    float q_token[H * aligned_head_dim];
    float k_token[H_kv * aligned_head_dim];
    float v_token[H_kv * aligned_head_dim];
    float attn_token[H * aligned_head_dim];
 
    /* Local MLP buffers (avoid layout dependencies for intermediate values) */
    float fc1_out[2 * aligned_intermediate_dim];
    float swiglu_out[aligned_intermediate_dim];
 
    /* Step 1: RMSNorm before attention */
    rmsnorm_forward(input,
                    ln1_gamma,
                    ln1_out,
                    NULL,
                    1,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    const size_t kv_head_stride = (size_t)aligned_context_window * (size_t)aligned_head_dim;
 
    /* Step 2: QKV projection */
    /* Q projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t ln1_q8_bytes = ((((aligned_embed_dim) + 255) / 256) * 292);
    uint8_t ln1_q8[ln1_q8_bytes];
    quantize_row_q8_k(ln1_out, ln1_q8, aligned_embed_dim);
    gemv_q4_k_q8_k(q_token, WQ, ln1_q8, H * head_dim, aligned_embed_dim);
    if (aligned_head_dim > head_dim) {
        for (int h = 0; h < H; ++h) {
            float *q_head = q_token + (size_t)h * (size_t)aligned_head_dim;
            for (int d = head_dim; d < aligned_head_dim; ++d) {
                q_head[d] = 0.0f;
            }
        }
    }
 
    /* K projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k (direct-to-cache) */
    const size_t wk_head_elems = (size_t)aligned_head_dim * (size_t)aligned_embed_dim;
    const size_t wk_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, wk_head_elems);
    const uint8_t *WK_bytes = (const uint8_t *)WK;
    /* ln1_q8 already quantized above */
    for (int h = 0; h < H_kv; ++h) {
        const void *wk_h = (const void *)(WK_bytes + (size_t)h * wk_head_bytes);
        float *k_head = k_cache + (size_t)h * kv_head_stride + (size_t)token_index * (size_t)aligned_head_dim;
        gemv_q4_k_q8_k(k_head, wk_h, ln1_q8, head_dim, aligned_embed_dim);
        for (int d = head_dim; d < aligned_head_dim; ++d) {
            k_head[d] = 0.0f;
        }
    }
 
    /* V projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k (direct-to-cache) */
    const size_t wv_head_elems = (size_t)aligned_head_dim * (size_t)aligned_embed_dim;
    const size_t wv_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, wv_head_elems);
    const uint8_t *WV_bytes = (const uint8_t *)WV;
    /* ln1_q8 already quantized above */
    for (int h = 0; h < H_kv; ++h) {
        const void *wv_h = (const void *)(WV_bytes + (size_t)h * wv_head_bytes);
        float *v_head = v_cache + (size_t)h * kv_head_stride + (size_t)token_index * (size_t)aligned_head_dim;
        gemv_q4_k_q8_k(v_head, wv_h, ln1_q8, head_dim, aligned_embed_dim);
        for (int d = head_dim; d < aligned_head_dim; ++d) {
            v_head[d] = 0.0f;
        }
    }
 
    /* Step 3: RoPE */
    rope_forward(q_token,
                 rope_cos,
                 rope_sin,
                 H,
                 1,
                 head_dim,
                 aligned_head_dim,
                 token_index);
    for (int h = 0; h < H_kv; ++h) {
        float *k_head = k_cache + (size_t)h * kv_head_stride + (size_t)token_index * (size_t)aligned_head_dim;
        rope_forward(k_head,
                     rope_cos,
                     rope_sin,
                     1,
                     1,
                     head_dim,
                     aligned_head_dim,
                     token_index);
    }
 
    /* Step 4: KV cache write (direct-to-cache) */
 
    /* Step 5: Attention (decode, flash) */
    attention_forward_decode_head_major_gqa_flash(q_token,
                                                   k_cache,
                                                   v_cache,
                                                   attn_token,
                                                   H,
                                                   H_kv,
                                                   token_index + 1,
                                                   aligned_context_window,
                                                   head_dim,
                                                   aligned_head_dim);
 
    /* Step 6: Output projection */
    /* WO projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t attn_q8_bytes = ((((H * head_dim) + 255) / 256) * 292);
    uint8_t attn_q8[attn_q8_bytes];
    quantize_row_q8_k(attn_token, attn_q8, H * head_dim);
    gemv_q4_k_q8_k(proj_tmp, WO, attn_q8, aligned_embed_dim, H * head_dim);
 
    /* Step 7: Residual add */
    qwen2_0_5b_decode_residual_add_token_major(input, proj_tmp, residual1, 1, aligned_embed_dim);
 
    /* Step 8: RMSNorm before MLP */
    rmsnorm_forward(residual1,
                    ln2_gamma,
                    ln2_out,
                    NULL,
                    1,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    /* Step 9: MLP (SwiGLU) */
    /* Gate+Up projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t ln2_q8_bytes = ((((aligned_embed_dim) + 255) / 256) * 292);
    uint8_t ln2_q8[ln2_q8_bytes];
    quantize_row_q8_k(ln2_out, ln2_q8, aligned_embed_dim);
    gemv_q4_k_q8_k(fc1_out, W1, ln2_q8, 2 * aligned_intermediate_dim, aligned_embed_dim);
 
    /* SwiGLU activation */
    swiglu_forward(fc1_out, swiglu_out, 1, aligned_intermediate_dim);
 
    /* Down projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t swiglu_q8_bytes = ((((aligned_intermediate_dim) + 255) / 256) * 292);
    uint8_t swiglu_q8[swiglu_q8_bytes];
    quantize_row_q8_k(swiglu_out, swiglu_q8, aligned_intermediate_dim);
    gemv_q4_k_q8_k(mlp_out, W2, swiglu_q8, aligned_embed_dim, aligned_intermediate_dim);
 
    /* Step 10: Final residual add */
    qwen2_0_5b_decode_residual_add_token_major(residual1, mlp_out, output, 1, aligned_embed_dim);
}

References attention_forward_decode_head_major_gqa_flash(), CK_DT_Q4_K, ck_dtype_row_bytes(), gemv_q4_k_q8_k(), QWEN2_0_5B_DECODELayerOffsets::k, QWEN2_0_5B_DECODELayerOffsets::ln1_gamma, QWEN2_0_5B_DECODELayerOffsets::ln1_out, QWEN2_0_5B_DECODELayerOffsets::ln2_gamma, QWEN2_0_5B_DECODELayerOffsets::ln2_out, QWEN2_0_5B_DECODELayerOffsets::mlp_out, QWEN2_0_5B_DECODELayerOffsets::output, QWEN2_0_5B_DECODELayerOffsets::proj_scratch, QWEN2_0_5B_DECODELayerOffsets::proj_tmp, quantize_row_q8_k(), QWEN2_0_5B_DECODE_EMBED_DIM, QWEN2_0_5B_DECODE_GLOBALS, QWEN2_0_5B_DECODE_HEAD_DIM, QWEN2_0_5B_DECODE_LAYERS, QWEN2_0_5B_DECODE_NUM_HEADS, QWEN2_0_5B_DECODE_NUM_KV_HEADS, QWEN2_0_5B_DECODE_PTR, qwen2_0_5b_decode_residual_add_token_major(), QWEN2_0_5B_DECODELayerOffsets::residual1, rmsnorm_forward(), QWEN2_0_5B_DECODEGlobalOffsets::rope_cos_cache, rope_forward(), QWEN2_0_5B_DECODEGlobalOffsets::rope_sin_cache, swiglu_forward(), QWEN2_0_5B_DECODELayerOffsets::v, QWEN2_0_5B_DECODELayerOffsets::w1, QWEN2_0_5B_DECODELayerOffsets::w2, QWEN2_0_5B_DECODELayerOffsets::wk, QWEN2_0_5B_DECODELayerOffsets::wo, QWEN2_0_5B_DECODELayerOffsets::wq, and QWEN2_0_5B_DECODELayerOffsets::wv.

Referenced by qwen2_0_5b_decode_decode_token().

qwen2_0_5b_decode_layer_13_prefill()

static void qwen2_0_5b_decode_layer_13_prefill ( QWEN2_0_5B_DECODEModel *  model, int  num_tokens, int  aligned_embed_dim, int  aligned_head_dim, int  aligned_intermediate_dim, int  aligned_context_window  )

static

Definition at line 2343 of file v6.5/test_generated/qwen2_int8.c.

  {
    const QWEN2_0_5B_DECODELayerOffsets *L = &QWEN2_0_5B_DECODE_LAYERS[13];
 
    float *input = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[12].output);
    float *ln1_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln1_gamma);
    float *ln1_out = QWEN2_0_5B_DECODE_PTR(model, L->ln1_out);
    float *ln2_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln2_gamma);
    float *ln2_out = QWEN2_0_5B_DECODE_PTR(model, L->ln2_out);
    float *q = QWEN2_0_5B_DECODE_PTR(model, L->q);
    float *k = QWEN2_0_5B_DECODE_PTR(model, L->k);
    float *v = QWEN2_0_5B_DECODE_PTR(model, L->v);
    float *attn_out = QWEN2_0_5B_DECODE_PTR(model, L->attn_out);
    float *proj_tmp = QWEN2_0_5B_DECODE_PTR(model, L->proj_tmp);
    float *proj_scratch = QWEN2_0_5B_DECODE_PTR(model, L->proj_scratch);
    float *residual1 = QWEN2_0_5B_DECODE_PTR(model, L->residual1);
    float *fc1_out = QWEN2_0_5B_DECODE_PTR(model, L->fc1_out);
    float *swiglu_out = QWEN2_0_5B_DECODE_PTR(model, L->swiglu_out);
    float *mlp_out = QWEN2_0_5B_DECODE_PTR(model, L->mlp_out);
    float *output = QWEN2_0_5B_DECODE_PTR(model, L->output);
 
    const void *WQ = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wq);
    const void *WK = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wk);
    const void *WV = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wv);
    const void *WO = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wo);
    const void *W1 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w1);
    const void *W2 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w2);
    const float *BQ = NULL;
    const float *BK = NULL;
    const float *BV = NULL;
    const float *BO = NULL;
    const float *B1 = NULL;
    const float *B2 = NULL;
 
    float *rope_cos = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_cos_cache);
    float *rope_sin = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_sin_cache);
 
    const int H = QWEN2_0_5B_DECODE_NUM_HEADS;
    const int H_kv = QWEN2_0_5B_DECODE_NUM_KV_HEADS;
    const int head_dim = QWEN2_0_5B_DECODE_HEAD_DIM;
    const size_t head_w_elems = (size_t)aligned_head_dim * (size_t)aligned_embed_dim;
    const size_t q_head_stride = (size_t)num_tokens * (size_t)aligned_head_dim;
    const size_t kv_head_stride = (size_t)aligned_context_window * (size_t)aligned_head_dim;
 
    /* RMSNorm before attention */
    rmsnorm_forward(input,
                    ln1_gamma,
                    ln1_out,
                    NULL,
                    num_tokens,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    /* Q projection (head-major) */
    const size_t wq_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, head_w_elems);
    const uint8_t *WQ_bytes = (const uint8_t *)WQ;
    for (int h = 0; h < H; ++h) {
        const void *wq_h = (const void *)(WQ_bytes + (size_t)h * wq_head_bytes);
        const float *bq_h = BQ ? (BQ + (size_t)h * (size_t)aligned_head_dim) : NULL;
        float *q_h = q + (size_t)h * q_head_stride;
        gemm_nt_q4_k(ln1_out, wq_h, bq_h, q_h, num_tokens, aligned_head_dim, aligned_embed_dim);
    }
 
    /* K projection (head-major) */
    const size_t wk_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, head_w_elems);
    const uint8_t *WK_bytes = (const uint8_t *)WK;
    for (int h = 0; h < H_kv; ++h) {
        const void *wk_h = (const void *)(WK_bytes + (size_t)h * wk_head_bytes);
        const float *bk_h = BK ? (BK + (size_t)h * (size_t)aligned_head_dim) : NULL;
        float *k_h = k + (size_t)h * kv_head_stride;
        gemm_nt_q4_k(ln1_out, wk_h, bk_h, k_h, num_tokens, aligned_head_dim, aligned_embed_dim);
    }
 
    /* V projection (head-major) */
    const size_t wv_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, head_w_elems);
    const uint8_t *WV_bytes = (const uint8_t *)WV;
    for (int h = 0; h < H_kv; ++h) {
        const void *wv_h = (const void *)(WV_bytes + (size_t)h * wv_head_bytes);
        const float *bv_h = BV ? (BV + (size_t)h * (size_t)aligned_head_dim) : NULL;
        float *v_h = v + (size_t)h * kv_head_stride;
        gemm_nt_q4_k(ln1_out, wv_h, bv_h, v_h, num_tokens, aligned_head_dim, aligned_embed_dim);
    }
 
    /* RoPE */
    rope_forward_qk_strided(q,
                            k,
                            rope_cos,
                            rope_sin,
                            H,
                            H_kv,
                            num_tokens,
                            head_dim,
                            aligned_head_dim,
                            0,
                            num_tokens,
                            aligned_context_window);
 
    /* Attention (prefill, causal) */
    attention_forward_causal_head_major_gqa_flash_strided(q,
                                                           k,
                                                           v,
                                                           attn_out,
                                                           H,
                                                           H_kv,
                                                           num_tokens,
                                                           head_dim,
                                                           aligned_head_dim,
                                                           aligned_context_window);
 
    /* Output projection (flatten head-major to token-major) */
    const int K = H * aligned_head_dim;
    if (K != aligned_embed_dim) {
        return;
    }
    const float *proj_in = attn_out;
    if (H > 1) {
        if (!proj_scratch) {
            return;
        }
        for (int t = 0; t < num_tokens; ++t) {
            float *dst = proj_scratch + (size_t)t * (size_t)aligned_embed_dim;
            for (int h = 0; h < H; ++h) {
                const float *src = attn_out + (size_t)h * q_head_stride + (size_t)t * (size_t)aligned_head_dim;
                memcpy(dst + (size_t)h * (size_t)aligned_head_dim,
                       src,
                       (size_t)aligned_head_dim * sizeof(float));
            }
        }
        proj_in = proj_scratch;
    }
    gemm_nt_q4_k(proj_in, WO, BO, proj_tmp, num_tokens, aligned_embed_dim, K);
 
    /* Residual add */
    qwen2_0_5b_decode_residual_add_token_major(input, proj_tmp, residual1, num_tokens, aligned_embed_dim);
 
    /* RMSNorm before MLP */
    rmsnorm_forward(residual1,
                    ln2_gamma,
                    ln2_out,
                    NULL,
                    num_tokens,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    /* MLP (SwiGLU) */
    gemm_nt_q4_k(ln2_out, W1, B1, fc1_out, num_tokens, 2 * aligned_intermediate_dim, aligned_embed_dim);
    swiglu_forward(fc1_out, swiglu_out, num_tokens, aligned_intermediate_dim);
    gemm_nt_q4_k(swiglu_out, W2, B2, mlp_out, num_tokens, aligned_embed_dim, aligned_intermediate_dim);
 
    /* Final residual add */
    qwen2_0_5b_decode_residual_add_token_major(residual1, mlp_out, output, num_tokens, aligned_embed_dim);
}

References attention_forward_causal_head_major_gqa_flash_strided(), CK_DT_Q4_K, ck_dtype_row_bytes(), gemm_nt_q4_k(), QWEN2_0_5B_DECODELayerOffsets::k, QWEN2_0_5B_DECODELayerOffsets::ln1_gamma, QWEN2_0_5B_DECODELayerOffsets::ln1_out, QWEN2_0_5B_DECODELayerOffsets::ln2_gamma, QWEN2_0_5B_DECODELayerOffsets::ln2_out, QWEN2_0_5B_DECODELayerOffsets::mlp_out, QWEN2_0_5B_DECODELayerOffsets::output, QWEN2_0_5B_DECODELayerOffsets::proj_scratch, QWEN2_0_5B_DECODELayerOffsets::proj_tmp, QWEN2_0_5B_DECODELayerOffsets::q, QWEN2_0_5B_DECODE_EMBED_DIM, QWEN2_0_5B_DECODE_GLOBALS, QWEN2_0_5B_DECODE_HEAD_DIM, QWEN2_0_5B_DECODE_LAYERS, QWEN2_0_5B_DECODE_NUM_HEADS, QWEN2_0_5B_DECODE_NUM_KV_HEADS, QWEN2_0_5B_DECODE_PTR, qwen2_0_5b_decode_residual_add_token_major(), QWEN2_0_5B_DECODELayerOffsets::residual1, rmsnorm_forward(), QWEN2_0_5B_DECODEGlobalOffsets::rope_cos_cache, rope_forward_qk_strided(), QWEN2_0_5B_DECODEGlobalOffsets::rope_sin_cache, swiglu_forward(), QWEN2_0_5B_DECODELayerOffsets::v, QWEN2_0_5B_DECODELayerOffsets::w1, QWEN2_0_5B_DECODELayerOffsets::w2, QWEN2_0_5B_DECODELayerOffsets::wk, QWEN2_0_5B_DECODELayerOffsets::wo, QWEN2_0_5B_DECODELayerOffsets::wq, and QWEN2_0_5B_DECODELayerOffsets::wv.

Referenced by qwen2_0_5b_decode_forward_prefill_impl().

qwen2_0_5b_decode_layer_14_decode()

static void qwen2_0_5b_decode_layer_14_decode ( QWEN2_0_5B_DECODEModel *  model, int  token_index, int  aligned_embed_dim, int  aligned_head_dim, int  aligned_intermediate_dim, int  aligned_context_window  )

static

Definition at line 6939 of file v6.5/test_generated/qwen2_int8.c.

  {
    const QWEN2_0_5B_DECODELayerOffsets *L = &QWEN2_0_5B_DECODE_LAYERS[14];
 
    float *input = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[13].output);
 
    float *ln1_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln1_gamma);
    float *ln1_out = QWEN2_0_5B_DECODE_PTR(model, L->ln1_out);
    float *ln2_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln2_gamma);
    float *ln2_out = QWEN2_0_5B_DECODE_PTR(model, L->ln2_out);
    float *k_cache = QWEN2_0_5B_DECODE_PTR(model, L->k);
    float *v_cache = QWEN2_0_5B_DECODE_PTR(model, L->v);
    float *proj_tmp = QWEN2_0_5B_DECODE_PTR(model, L->proj_tmp);
    float *proj_scratch = QWEN2_0_5B_DECODE_PTR(model, L->proj_scratch);
    float *residual1 = QWEN2_0_5B_DECODE_PTR(model, L->residual1);
    float *mlp_out = QWEN2_0_5B_DECODE_PTR(model, L->mlp_out);
    float *output = QWEN2_0_5B_DECODE_PTR(model, L->output);
 
    /* Weights (explicit types for layer 14) */
    const void *WQ = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wq);  /* Q4_K */
    const void *WK = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wk);  /* Q4_K */
    const void *WV = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wv);  /* Q4_K */
    const void *WO = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wo);  /* Q4_K */
    const void *W1 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w1);  /* Q4_K (gate+up) */
    const void *W2 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w2);  /* Q4_K (down) */
 
    float *rope_cos = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_cos_cache);
    float *rope_sin = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_sin_cache);
 
    const int H = QWEN2_0_5B_DECODE_NUM_HEADS;
    const int H_kv = QWEN2_0_5B_DECODE_NUM_KV_HEADS;
    const int head_dim = QWEN2_0_5B_DECODE_HEAD_DIM;
 
    float q_token[H * aligned_head_dim];
    float k_token[H_kv * aligned_head_dim];
    float v_token[H_kv * aligned_head_dim];
    float attn_token[H * aligned_head_dim];
 
    /* Local MLP buffers (avoid layout dependencies for intermediate values) */
    float fc1_out[2 * aligned_intermediate_dim];
    float swiglu_out[aligned_intermediate_dim];
 
    /* Step 1: RMSNorm before attention */
    rmsnorm_forward(input,
                    ln1_gamma,
                    ln1_out,
                    NULL,
                    1,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    const size_t kv_head_stride = (size_t)aligned_context_window * (size_t)aligned_head_dim;
 
    /* Step 2: QKV projection */
    /* Q projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t ln1_q8_bytes = ((((aligned_embed_dim) + 255) / 256) * 292);
    uint8_t ln1_q8[ln1_q8_bytes];
    quantize_row_q8_k(ln1_out, ln1_q8, aligned_embed_dim);
    gemv_q4_k_q8_k(q_token, WQ, ln1_q8, H * head_dim, aligned_embed_dim);
    if (aligned_head_dim > head_dim) {
        for (int h = 0; h < H; ++h) {
            float *q_head = q_token + (size_t)h * (size_t)aligned_head_dim;
            for (int d = head_dim; d < aligned_head_dim; ++d) {
                q_head[d] = 0.0f;
            }
        }
    }
 
    /* K projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k (direct-to-cache) */
    const size_t wk_head_elems = (size_t)aligned_head_dim * (size_t)aligned_embed_dim;
    const size_t wk_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, wk_head_elems);
    const uint8_t *WK_bytes = (const uint8_t *)WK;
    /* ln1_q8 already quantized above */
    for (int h = 0; h < H_kv; ++h) {
        const void *wk_h = (const void *)(WK_bytes + (size_t)h * wk_head_bytes);
        float *k_head = k_cache + (size_t)h * kv_head_stride + (size_t)token_index * (size_t)aligned_head_dim;
        gemv_q4_k_q8_k(k_head, wk_h, ln1_q8, head_dim, aligned_embed_dim);
        for (int d = head_dim; d < aligned_head_dim; ++d) {
            k_head[d] = 0.0f;
        }
    }
 
    /* V projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k (direct-to-cache) */
    const size_t wv_head_elems = (size_t)aligned_head_dim * (size_t)aligned_embed_dim;
    const size_t wv_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, wv_head_elems);
    const uint8_t *WV_bytes = (const uint8_t *)WV;
    /* ln1_q8 already quantized above */
    for (int h = 0; h < H_kv; ++h) {
        const void *wv_h = (const void *)(WV_bytes + (size_t)h * wv_head_bytes);
        float *v_head = v_cache + (size_t)h * kv_head_stride + (size_t)token_index * (size_t)aligned_head_dim;
        gemv_q4_k_q8_k(v_head, wv_h, ln1_q8, head_dim, aligned_embed_dim);
        for (int d = head_dim; d < aligned_head_dim; ++d) {
            v_head[d] = 0.0f;
        }
    }
 
    /* Step 3: RoPE */
    rope_forward(q_token,
                 rope_cos,
                 rope_sin,
                 H,
                 1,
                 head_dim,
                 aligned_head_dim,
                 token_index);
    for (int h = 0; h < H_kv; ++h) {
        float *k_head = k_cache + (size_t)h * kv_head_stride + (size_t)token_index * (size_t)aligned_head_dim;
        rope_forward(k_head,
                     rope_cos,
                     rope_sin,
                     1,
                     1,
                     head_dim,
                     aligned_head_dim,
                     token_index);
    }
 
    /* Step 4: KV cache write (direct-to-cache) */
 
    /* Step 5: Attention (decode, flash) */
    attention_forward_decode_head_major_gqa_flash(q_token,
                                                   k_cache,
                                                   v_cache,
                                                   attn_token,
                                                   H,
                                                   H_kv,
                                                   token_index + 1,
                                                   aligned_context_window,
                                                   head_dim,
                                                   aligned_head_dim);
 
    /* Step 6: Output projection */
    /* WO projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t attn_q8_bytes = ((((H * head_dim) + 255) / 256) * 292);
    uint8_t attn_q8[attn_q8_bytes];
    quantize_row_q8_k(attn_token, attn_q8, H * head_dim);
    gemv_q4_k_q8_k(proj_tmp, WO, attn_q8, aligned_embed_dim, H * head_dim);
 
    /* Step 7: Residual add */
    qwen2_0_5b_decode_residual_add_token_major(input, proj_tmp, residual1, 1, aligned_embed_dim);
 
    /* Step 8: RMSNorm before MLP */
    rmsnorm_forward(residual1,
                    ln2_gamma,
                    ln2_out,
                    NULL,
                    1,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    /* Step 9: MLP (SwiGLU) */
    /* Gate+Up projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t ln2_q8_bytes = ((((aligned_embed_dim) + 255) / 256) * 292);
    uint8_t ln2_q8[ln2_q8_bytes];
    quantize_row_q8_k(ln2_out, ln2_q8, aligned_embed_dim);
    gemv_q4_k_q8_k(fc1_out, W1, ln2_q8, 2 * aligned_intermediate_dim, aligned_embed_dim);
 
    /* SwiGLU activation */
    swiglu_forward(fc1_out, swiglu_out, 1, aligned_intermediate_dim);
 
    /* Down projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t swiglu_q8_bytes = ((((aligned_intermediate_dim) + 255) / 256) * 292);
    uint8_t swiglu_q8[swiglu_q8_bytes];
    quantize_row_q8_k(swiglu_out, swiglu_q8, aligned_intermediate_dim);
    gemv_q4_k_q8_k(mlp_out, W2, swiglu_q8, aligned_embed_dim, aligned_intermediate_dim);
 
    /* Step 10: Final residual add */
    qwen2_0_5b_decode_residual_add_token_major(residual1, mlp_out, output, 1, aligned_embed_dim);
}

References attention_forward_decode_head_major_gqa_flash(), CK_DT_Q4_K, ck_dtype_row_bytes(), gemv_q4_k_q8_k(), QWEN2_0_5B_DECODELayerOffsets::k, QWEN2_0_5B_DECODELayerOffsets::ln1_gamma, QWEN2_0_5B_DECODELayerOffsets::ln1_out, QWEN2_0_5B_DECODELayerOffsets::ln2_gamma, QWEN2_0_5B_DECODELayerOffsets::ln2_out, QWEN2_0_5B_DECODELayerOffsets::mlp_out, QWEN2_0_5B_DECODELayerOffsets::output, QWEN2_0_5B_DECODELayerOffsets::proj_scratch, QWEN2_0_5B_DECODELayerOffsets::proj_tmp, quantize_row_q8_k(), QWEN2_0_5B_DECODE_EMBED_DIM, QWEN2_0_5B_DECODE_GLOBALS, QWEN2_0_5B_DECODE_HEAD_DIM, QWEN2_0_5B_DECODE_LAYERS, QWEN2_0_5B_DECODE_NUM_HEADS, QWEN2_0_5B_DECODE_NUM_KV_HEADS, QWEN2_0_5B_DECODE_PTR, qwen2_0_5b_decode_residual_add_token_major(), QWEN2_0_5B_DECODELayerOffsets::residual1, rmsnorm_forward(), QWEN2_0_5B_DECODEGlobalOffsets::rope_cos_cache, rope_forward(), QWEN2_0_5B_DECODEGlobalOffsets::rope_sin_cache, swiglu_forward(), QWEN2_0_5B_DECODELayerOffsets::v, QWEN2_0_5B_DECODELayerOffsets::w1, QWEN2_0_5B_DECODELayerOffsets::w2, QWEN2_0_5B_DECODELayerOffsets::wk, QWEN2_0_5B_DECODELayerOffsets::wo, QWEN2_0_5B_DECODELayerOffsets::wq, and QWEN2_0_5B_DECODELayerOffsets::wv.

Referenced by qwen2_0_5b_decode_decode_token().

qwen2_0_5b_decode_layer_14_prefill()

static void qwen2_0_5b_decode_layer_14_prefill ( QWEN2_0_5B_DECODEModel *  model, int  num_tokens, int  aligned_embed_dim, int  aligned_head_dim, int  aligned_intermediate_dim, int  aligned_context_window  )

static

Definition at line 2507 of file v6.5/test_generated/qwen2_int8.c.

  {
    const QWEN2_0_5B_DECODELayerOffsets *L = &QWEN2_0_5B_DECODE_LAYERS[14];
 
    float *input = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[13].output);
    float *ln1_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln1_gamma);
    float *ln1_out = QWEN2_0_5B_DECODE_PTR(model, L->ln1_out);
    float *ln2_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln2_gamma);
    float *ln2_out = QWEN2_0_5B_DECODE_PTR(model, L->ln2_out);
    float *q = QWEN2_0_5B_DECODE_PTR(model, L->q);
    float *k = QWEN2_0_5B_DECODE_PTR(model, L->k);
    float *v = QWEN2_0_5B_DECODE_PTR(model, L->v);
    float *attn_out = QWEN2_0_5B_DECODE_PTR(model, L->attn_out);
    float *proj_tmp = QWEN2_0_5B_DECODE_PTR(model, L->proj_tmp);
    float *proj_scratch = QWEN2_0_5B_DECODE_PTR(model, L->proj_scratch);
    float *residual1 = QWEN2_0_5B_DECODE_PTR(model, L->residual1);
    float *fc1_out = QWEN2_0_5B_DECODE_PTR(model, L->fc1_out);
    float *swiglu_out = QWEN2_0_5B_DECODE_PTR(model, L->swiglu_out);
    float *mlp_out = QWEN2_0_5B_DECODE_PTR(model, L->mlp_out);
    float *output = QWEN2_0_5B_DECODE_PTR(model, L->output);
 
    const void *WQ = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wq);
    const void *WK = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wk);
    const void *WV = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wv);
    const void *WO = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wo);
    const void *W1 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w1);
    const void *W2 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w2);
    const float *BQ = NULL;
    const float *BK = NULL;
    const float *BV = NULL;
    const float *BO = NULL;
    const float *B1 = NULL;
    const float *B2 = NULL;
 
    float *rope_cos = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_cos_cache);
    float *rope_sin = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_sin_cache);
 
    const int H = QWEN2_0_5B_DECODE_NUM_HEADS;
    const int H_kv = QWEN2_0_5B_DECODE_NUM_KV_HEADS;
    const int head_dim = QWEN2_0_5B_DECODE_HEAD_DIM;
    const size_t head_w_elems = (size_t)aligned_head_dim * (size_t)aligned_embed_dim;
    const size_t q_head_stride = (size_t)num_tokens * (size_t)aligned_head_dim;
    const size_t kv_head_stride = (size_t)aligned_context_window * (size_t)aligned_head_dim;
 
    /* RMSNorm before attention */
    rmsnorm_forward(input,
                    ln1_gamma,
                    ln1_out,
                    NULL,
                    num_tokens,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    /* Q projection (head-major) */
    const size_t wq_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, head_w_elems);
    const uint8_t *WQ_bytes = (const uint8_t *)WQ;
    for (int h = 0; h < H; ++h) {
        const void *wq_h = (const void *)(WQ_bytes + (size_t)h * wq_head_bytes);
        const float *bq_h = BQ ? (BQ + (size_t)h * (size_t)aligned_head_dim) : NULL;
        float *q_h = q + (size_t)h * q_head_stride;
        gemm_nt_q4_k(ln1_out, wq_h, bq_h, q_h, num_tokens, aligned_head_dim, aligned_embed_dim);
    }
 
    /* K projection (head-major) */
    const size_t wk_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, head_w_elems);
    const uint8_t *WK_bytes = (const uint8_t *)WK;
    for (int h = 0; h < H_kv; ++h) {
        const void *wk_h = (const void *)(WK_bytes + (size_t)h * wk_head_bytes);
        const float *bk_h = BK ? (BK + (size_t)h * (size_t)aligned_head_dim) : NULL;
        float *k_h = k + (size_t)h * kv_head_stride;
        gemm_nt_q4_k(ln1_out, wk_h, bk_h, k_h, num_tokens, aligned_head_dim, aligned_embed_dim);
    }
 
    /* V projection (head-major) */
    const size_t wv_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, head_w_elems);
    const uint8_t *WV_bytes = (const uint8_t *)WV;
    for (int h = 0; h < H_kv; ++h) {
        const void *wv_h = (const void *)(WV_bytes + (size_t)h * wv_head_bytes);
        const float *bv_h = BV ? (BV + (size_t)h * (size_t)aligned_head_dim) : NULL;
        float *v_h = v + (size_t)h * kv_head_stride;
        gemm_nt_q4_k(ln1_out, wv_h, bv_h, v_h, num_tokens, aligned_head_dim, aligned_embed_dim);
    }
 
    /* RoPE */
    rope_forward_qk_strided(q,
                            k,
                            rope_cos,
                            rope_sin,
                            H,
                            H_kv,
                            num_tokens,
                            head_dim,
                            aligned_head_dim,
                            0,
                            num_tokens,
                            aligned_context_window);
 
    /* Attention (prefill, causal) */
    attention_forward_causal_head_major_gqa_flash_strided(q,
                                                           k,
                                                           v,
                                                           attn_out,
                                                           H,
                                                           H_kv,
                                                           num_tokens,
                                                           head_dim,
                                                           aligned_head_dim,
                                                           aligned_context_window);
 
    /* Output projection (flatten head-major to token-major) */
    const int K = H * aligned_head_dim;
    if (K != aligned_embed_dim) {
        return;
    }
    const float *proj_in = attn_out;
    if (H > 1) {
        if (!proj_scratch) {
            return;
        }
        for (int t = 0; t < num_tokens; ++t) {
            float *dst = proj_scratch + (size_t)t * (size_t)aligned_embed_dim;
            for (int h = 0; h < H; ++h) {
                const float *src = attn_out + (size_t)h * q_head_stride + (size_t)t * (size_t)aligned_head_dim;
                memcpy(dst + (size_t)h * (size_t)aligned_head_dim,
                       src,
                       (size_t)aligned_head_dim * sizeof(float));
            }
        }
        proj_in = proj_scratch;
    }
    gemm_nt_q4_k(proj_in, WO, BO, proj_tmp, num_tokens, aligned_embed_dim, K);
 
    /* Residual add */
    qwen2_0_5b_decode_residual_add_token_major(input, proj_tmp, residual1, num_tokens, aligned_embed_dim);
 
    /* RMSNorm before MLP */
    rmsnorm_forward(residual1,
                    ln2_gamma,
                    ln2_out,
                    NULL,
                    num_tokens,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    /* MLP (SwiGLU) */
    gemm_nt_q4_k(ln2_out, W1, B1, fc1_out, num_tokens, 2 * aligned_intermediate_dim, aligned_embed_dim);
    swiglu_forward(fc1_out, swiglu_out, num_tokens, aligned_intermediate_dim);
    gemm_nt_q4_k(swiglu_out, W2, B2, mlp_out, num_tokens, aligned_embed_dim, aligned_intermediate_dim);
 
    /* Final residual add */
    qwen2_0_5b_decode_residual_add_token_major(residual1, mlp_out, output, num_tokens, aligned_embed_dim);
}

References attention_forward_causal_head_major_gqa_flash_strided(), CK_DT_Q4_K, ck_dtype_row_bytes(), gemm_nt_q4_k(), QWEN2_0_5B_DECODELayerOffsets::k, QWEN2_0_5B_DECODELayerOffsets::ln1_gamma, QWEN2_0_5B_DECODELayerOffsets::ln1_out, QWEN2_0_5B_DECODELayerOffsets::ln2_gamma, QWEN2_0_5B_DECODELayerOffsets::ln2_out, QWEN2_0_5B_DECODELayerOffsets::mlp_out, QWEN2_0_5B_DECODELayerOffsets::output, QWEN2_0_5B_DECODELayerOffsets::proj_scratch, QWEN2_0_5B_DECODELayerOffsets::proj_tmp, QWEN2_0_5B_DECODELayerOffsets::q, QWEN2_0_5B_DECODE_EMBED_DIM, QWEN2_0_5B_DECODE_GLOBALS, QWEN2_0_5B_DECODE_HEAD_DIM, QWEN2_0_5B_DECODE_LAYERS, QWEN2_0_5B_DECODE_NUM_HEADS, QWEN2_0_5B_DECODE_NUM_KV_HEADS, QWEN2_0_5B_DECODE_PTR, qwen2_0_5b_decode_residual_add_token_major(), QWEN2_0_5B_DECODELayerOffsets::residual1, rmsnorm_forward(), QWEN2_0_5B_DECODEGlobalOffsets::rope_cos_cache, rope_forward_qk_strided(), QWEN2_0_5B_DECODEGlobalOffsets::rope_sin_cache, swiglu_forward(), QWEN2_0_5B_DECODELayerOffsets::v, QWEN2_0_5B_DECODELayerOffsets::w1, QWEN2_0_5B_DECODELayerOffsets::w2, QWEN2_0_5B_DECODELayerOffsets::wk, QWEN2_0_5B_DECODELayerOffsets::wo, QWEN2_0_5B_DECODELayerOffsets::wq, and QWEN2_0_5B_DECODELayerOffsets::wv.

Referenced by qwen2_0_5b_decode_forward_prefill_impl().

qwen2_0_5b_decode_layer_15_decode()

static void qwen2_0_5b_decode_layer_15_decode ( QWEN2_0_5B_DECODEModel *  model, int  token_index, int  aligned_embed_dim, int  aligned_head_dim, int  aligned_intermediate_dim, int  aligned_context_window  )

static

Definition at line 7120 of file v6.5/test_generated/qwen2_int8.c.

  {
    const QWEN2_0_5B_DECODELayerOffsets *L = &QWEN2_0_5B_DECODE_LAYERS[15];
 
    float *input = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[14].output);
 
    float *ln1_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln1_gamma);
    float *ln1_out = QWEN2_0_5B_DECODE_PTR(model, L->ln1_out);
    float *ln2_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln2_gamma);
    float *ln2_out = QWEN2_0_5B_DECODE_PTR(model, L->ln2_out);
    float *k_cache = QWEN2_0_5B_DECODE_PTR(model, L->k);
    float *v_cache = QWEN2_0_5B_DECODE_PTR(model, L->v);
    float *proj_tmp = QWEN2_0_5B_DECODE_PTR(model, L->proj_tmp);
    float *proj_scratch = QWEN2_0_5B_DECODE_PTR(model, L->proj_scratch);
    float *residual1 = QWEN2_0_5B_DECODE_PTR(model, L->residual1);
    float *mlp_out = QWEN2_0_5B_DECODE_PTR(model, L->mlp_out);
    float *output = QWEN2_0_5B_DECODE_PTR(model, L->output);
 
    /* Weights (explicit types for layer 15) */
    const void *WQ = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wq);  /* Q4_K */
    const void *WK = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wk);  /* Q4_K */
    const void *WV = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wv);  /* Q4_K */
    const void *WO = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wo);  /* Q4_K */
    const void *W1 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w1);  /* Q4_K (gate+up) */
    const void *W2 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w2);  /* Q4_K (down) */
 
    float *rope_cos = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_cos_cache);
    float *rope_sin = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_sin_cache);
 
    const int H = QWEN2_0_5B_DECODE_NUM_HEADS;
    const int H_kv = QWEN2_0_5B_DECODE_NUM_KV_HEADS;
    const int head_dim = QWEN2_0_5B_DECODE_HEAD_DIM;
 
    float q_token[H * aligned_head_dim];
    float k_token[H_kv * aligned_head_dim];
    float v_token[H_kv * aligned_head_dim];
    float attn_token[H * aligned_head_dim];
 
    /* Local MLP buffers (avoid layout dependencies for intermediate values) */
    float fc1_out[2 * aligned_intermediate_dim];
    float swiglu_out[aligned_intermediate_dim];
 
    /* Step 1: RMSNorm before attention */
    rmsnorm_forward(input,
                    ln1_gamma,
                    ln1_out,
                    NULL,
                    1,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    const size_t kv_head_stride = (size_t)aligned_context_window * (size_t)aligned_head_dim;
 
    /* Step 2: QKV projection */
    /* Q projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t ln1_q8_bytes = ((((aligned_embed_dim) + 255) / 256) * 292);
    uint8_t ln1_q8[ln1_q8_bytes];
    quantize_row_q8_k(ln1_out, ln1_q8, aligned_embed_dim);
    gemv_q4_k_q8_k(q_token, WQ, ln1_q8, H * head_dim, aligned_embed_dim);
    if (aligned_head_dim > head_dim) {
        for (int h = 0; h < H; ++h) {
            float *q_head = q_token + (size_t)h * (size_t)aligned_head_dim;
            for (int d = head_dim; d < aligned_head_dim; ++d) {
                q_head[d] = 0.0f;
            }
        }
    }
 
    /* K projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k (direct-to-cache) */
    const size_t wk_head_elems = (size_t)aligned_head_dim * (size_t)aligned_embed_dim;
    const size_t wk_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, wk_head_elems);
    const uint8_t *WK_bytes = (const uint8_t *)WK;
    /* ln1_q8 already quantized above */
    for (int h = 0; h < H_kv; ++h) {
        const void *wk_h = (const void *)(WK_bytes + (size_t)h * wk_head_bytes);
        float *k_head = k_cache + (size_t)h * kv_head_stride + (size_t)token_index * (size_t)aligned_head_dim;
        gemv_q4_k_q8_k(k_head, wk_h, ln1_q8, head_dim, aligned_embed_dim);
        for (int d = head_dim; d < aligned_head_dim; ++d) {
            k_head[d] = 0.0f;
        }
    }
 
    /* V projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k (direct-to-cache) */
    const size_t wv_head_elems = (size_t)aligned_head_dim * (size_t)aligned_embed_dim;
    const size_t wv_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, wv_head_elems);
    const uint8_t *WV_bytes = (const uint8_t *)WV;
    /* ln1_q8 already quantized above */
    for (int h = 0; h < H_kv; ++h) {
        const void *wv_h = (const void *)(WV_bytes + (size_t)h * wv_head_bytes);
        float *v_head = v_cache + (size_t)h * kv_head_stride + (size_t)token_index * (size_t)aligned_head_dim;
        gemv_q4_k_q8_k(v_head, wv_h, ln1_q8, head_dim, aligned_embed_dim);
        for (int d = head_dim; d < aligned_head_dim; ++d) {
            v_head[d] = 0.0f;
        }
    }
 
    /* Step 3: RoPE */
    rope_forward(q_token,
                 rope_cos,
                 rope_sin,
                 H,
                 1,
                 head_dim,
                 aligned_head_dim,
                 token_index);
    for (int h = 0; h < H_kv; ++h) {
        float *k_head = k_cache + (size_t)h * kv_head_stride + (size_t)token_index * (size_t)aligned_head_dim;
        rope_forward(k_head,
                     rope_cos,
                     rope_sin,
                     1,
                     1,
                     head_dim,
                     aligned_head_dim,
                     token_index);
    }
 
    /* Step 4: KV cache write (direct-to-cache) */
 
    /* Step 5: Attention (decode, flash) */
    attention_forward_decode_head_major_gqa_flash(q_token,
                                                   k_cache,
                                                   v_cache,
                                                   attn_token,
                                                   H,
                                                   H_kv,
                                                   token_index + 1,
                                                   aligned_context_window,
                                                   head_dim,
                                                   aligned_head_dim);
 
    /* Step 6: Output projection */
    /* WO projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t attn_q8_bytes = ((((H * head_dim) + 255) / 256) * 292);
    uint8_t attn_q8[attn_q8_bytes];
    quantize_row_q8_k(attn_token, attn_q8, H * head_dim);
    gemv_q4_k_q8_k(proj_tmp, WO, attn_q8, aligned_embed_dim, H * head_dim);
 
    /* Step 7: Residual add */
    qwen2_0_5b_decode_residual_add_token_major(input, proj_tmp, residual1, 1, aligned_embed_dim);
 
    /* Step 8: RMSNorm before MLP */
    rmsnorm_forward(residual1,
                    ln2_gamma,
                    ln2_out,
                    NULL,
                    1,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    /* Step 9: MLP (SwiGLU) */
    /* Gate+Up projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t ln2_q8_bytes = ((((aligned_embed_dim) + 255) / 256) * 292);
    uint8_t ln2_q8[ln2_q8_bytes];
    quantize_row_q8_k(ln2_out, ln2_q8, aligned_embed_dim);
    gemv_q4_k_q8_k(fc1_out, W1, ln2_q8, 2 * aligned_intermediate_dim, aligned_embed_dim);
 
    /* SwiGLU activation */
    swiglu_forward(fc1_out, swiglu_out, 1, aligned_intermediate_dim);
 
    /* Down projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t swiglu_q8_bytes = ((((aligned_intermediate_dim) + 255) / 256) * 292);
    uint8_t swiglu_q8[swiglu_q8_bytes];
    quantize_row_q8_k(swiglu_out, swiglu_q8, aligned_intermediate_dim);
    gemv_q4_k_q8_k(mlp_out, W2, swiglu_q8, aligned_embed_dim, aligned_intermediate_dim);
 
    /* Step 10: Final residual add */
    qwen2_0_5b_decode_residual_add_token_major(residual1, mlp_out, output, 1, aligned_embed_dim);
}

References attention_forward_decode_head_major_gqa_flash(), CK_DT_Q4_K, ck_dtype_row_bytes(), gemv_q4_k_q8_k(), QWEN2_0_5B_DECODELayerOffsets::k, QWEN2_0_5B_DECODELayerOffsets::ln1_gamma, QWEN2_0_5B_DECODELayerOffsets::ln1_out, QWEN2_0_5B_DECODELayerOffsets::ln2_gamma, QWEN2_0_5B_DECODELayerOffsets::ln2_out, QWEN2_0_5B_DECODELayerOffsets::mlp_out, QWEN2_0_5B_DECODELayerOffsets::output, QWEN2_0_5B_DECODELayerOffsets::proj_scratch, QWEN2_0_5B_DECODELayerOffsets::proj_tmp, quantize_row_q8_k(), QWEN2_0_5B_DECODE_EMBED_DIM, QWEN2_0_5B_DECODE_GLOBALS, QWEN2_0_5B_DECODE_HEAD_DIM, QWEN2_0_5B_DECODE_LAYERS, QWEN2_0_5B_DECODE_NUM_HEADS, QWEN2_0_5B_DECODE_NUM_KV_HEADS, QWEN2_0_5B_DECODE_PTR, qwen2_0_5b_decode_residual_add_token_major(), QWEN2_0_5B_DECODELayerOffsets::residual1, rmsnorm_forward(), QWEN2_0_5B_DECODEGlobalOffsets::rope_cos_cache, rope_forward(), QWEN2_0_5B_DECODEGlobalOffsets::rope_sin_cache, swiglu_forward(), QWEN2_0_5B_DECODELayerOffsets::v, QWEN2_0_5B_DECODELayerOffsets::w1, QWEN2_0_5B_DECODELayerOffsets::w2, QWEN2_0_5B_DECODELayerOffsets::wk, QWEN2_0_5B_DECODELayerOffsets::wo, QWEN2_0_5B_DECODELayerOffsets::wq, and QWEN2_0_5B_DECODELayerOffsets::wv.

Referenced by qwen2_0_5b_decode_decode_token().

qwen2_0_5b_decode_layer_15_prefill()

static void qwen2_0_5b_decode_layer_15_prefill ( QWEN2_0_5B_DECODEModel *  model, int  num_tokens, int  aligned_embed_dim, int  aligned_head_dim, int  aligned_intermediate_dim, int  aligned_context_window  )

static

Definition at line 2671 of file v6.5/test_generated/qwen2_int8.c.

  {
    const QWEN2_0_5B_DECODELayerOffsets *L = &QWEN2_0_5B_DECODE_LAYERS[15];
 
    float *input = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[14].output);
    float *ln1_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln1_gamma);
    float *ln1_out = QWEN2_0_5B_DECODE_PTR(model, L->ln1_out);
    float *ln2_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln2_gamma);
    float *ln2_out = QWEN2_0_5B_DECODE_PTR(model, L->ln2_out);
    float *q = QWEN2_0_5B_DECODE_PTR(model, L->q);
    float *k = QWEN2_0_5B_DECODE_PTR(model, L->k);
    float *v = QWEN2_0_5B_DECODE_PTR(model, L->v);
    float *attn_out = QWEN2_0_5B_DECODE_PTR(model, L->attn_out);
    float *proj_tmp = QWEN2_0_5B_DECODE_PTR(model, L->proj_tmp);
    float *proj_scratch = QWEN2_0_5B_DECODE_PTR(model, L->proj_scratch);
    float *residual1 = QWEN2_0_5B_DECODE_PTR(model, L->residual1);
    float *fc1_out = QWEN2_0_5B_DECODE_PTR(model, L->fc1_out);
    float *swiglu_out = QWEN2_0_5B_DECODE_PTR(model, L->swiglu_out);
    float *mlp_out = QWEN2_0_5B_DECODE_PTR(model, L->mlp_out);
    float *output = QWEN2_0_5B_DECODE_PTR(model, L->output);
 
    const void *WQ = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wq);
    const void *WK = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wk);
    const void *WV = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wv);
    const void *WO = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wo);
    const void *W1 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w1);
    const void *W2 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w2);
    const float *BQ = NULL;
    const float *BK = NULL;
    const float *BV = NULL;
    const float *BO = NULL;
    const float *B1 = NULL;
    const float *B2 = NULL;
 
    float *rope_cos = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_cos_cache);
    float *rope_sin = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_sin_cache);
 
    const int H = QWEN2_0_5B_DECODE_NUM_HEADS;
    const int H_kv = QWEN2_0_5B_DECODE_NUM_KV_HEADS;
    const int head_dim = QWEN2_0_5B_DECODE_HEAD_DIM;
    const size_t head_w_elems = (size_t)aligned_head_dim * (size_t)aligned_embed_dim;
    const size_t q_head_stride = (size_t)num_tokens * (size_t)aligned_head_dim;
    const size_t kv_head_stride = (size_t)aligned_context_window * (size_t)aligned_head_dim;
 
    /* RMSNorm before attention */
    rmsnorm_forward(input,
                    ln1_gamma,
                    ln1_out,
                    NULL,
                    num_tokens,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    /* Q projection (head-major) */
    const size_t wq_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, head_w_elems);
    const uint8_t *WQ_bytes = (const uint8_t *)WQ;
    for (int h = 0; h < H; ++h) {
        const void *wq_h = (const void *)(WQ_bytes + (size_t)h * wq_head_bytes);
        const float *bq_h = BQ ? (BQ + (size_t)h * (size_t)aligned_head_dim) : NULL;
        float *q_h = q + (size_t)h * q_head_stride;
        gemm_nt_q4_k(ln1_out, wq_h, bq_h, q_h, num_tokens, aligned_head_dim, aligned_embed_dim);
    }
 
    /* K projection (head-major) */
    const size_t wk_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, head_w_elems);
    const uint8_t *WK_bytes = (const uint8_t *)WK;
    for (int h = 0; h < H_kv; ++h) {
        const void *wk_h = (const void *)(WK_bytes + (size_t)h * wk_head_bytes);
        const float *bk_h = BK ? (BK + (size_t)h * (size_t)aligned_head_dim) : NULL;
        float *k_h = k + (size_t)h * kv_head_stride;
        gemm_nt_q4_k(ln1_out, wk_h, bk_h, k_h, num_tokens, aligned_head_dim, aligned_embed_dim);
    }
 
    /* V projection (head-major) */
    const size_t wv_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, head_w_elems);
    const uint8_t *WV_bytes = (const uint8_t *)WV;
    for (int h = 0; h < H_kv; ++h) {
        const void *wv_h = (const void *)(WV_bytes + (size_t)h * wv_head_bytes);
        const float *bv_h = BV ? (BV + (size_t)h * (size_t)aligned_head_dim) : NULL;
        float *v_h = v + (size_t)h * kv_head_stride;
        gemm_nt_q4_k(ln1_out, wv_h, bv_h, v_h, num_tokens, aligned_head_dim, aligned_embed_dim);
    }
 
    /* RoPE */
    rope_forward_qk_strided(q,
                            k,
                            rope_cos,
                            rope_sin,
                            H,
                            H_kv,
                            num_tokens,
                            head_dim,
                            aligned_head_dim,
                            0,
                            num_tokens,
                            aligned_context_window);
 
    /* Attention (prefill, causal) */
    attention_forward_causal_head_major_gqa_flash_strided(q,
                                                           k,
                                                           v,
                                                           attn_out,
                                                           H,
                                                           H_kv,
                                                           num_tokens,
                                                           head_dim,
                                                           aligned_head_dim,
                                                           aligned_context_window);
 
    /* Output projection (flatten head-major to token-major) */
    const int K = H * aligned_head_dim;
    if (K != aligned_embed_dim) {
        return;
    }
    const float *proj_in = attn_out;
    if (H > 1) {
        if (!proj_scratch) {
            return;
        }
        for (int t = 0; t < num_tokens; ++t) {
            float *dst = proj_scratch + (size_t)t * (size_t)aligned_embed_dim;
            for (int h = 0; h < H; ++h) {
                const float *src = attn_out + (size_t)h * q_head_stride + (size_t)t * (size_t)aligned_head_dim;
                memcpy(dst + (size_t)h * (size_t)aligned_head_dim,
                       src,
                       (size_t)aligned_head_dim * sizeof(float));
            }
        }
        proj_in = proj_scratch;
    }
    gemm_nt_q4_k(proj_in, WO, BO, proj_tmp, num_tokens, aligned_embed_dim, K);
 
    /* Residual add */
    qwen2_0_5b_decode_residual_add_token_major(input, proj_tmp, residual1, num_tokens, aligned_embed_dim);
 
    /* RMSNorm before MLP */
    rmsnorm_forward(residual1,
                    ln2_gamma,
                    ln2_out,
                    NULL,
                    num_tokens,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    /* MLP (SwiGLU) */
    gemm_nt_q4_k(ln2_out, W1, B1, fc1_out, num_tokens, 2 * aligned_intermediate_dim, aligned_embed_dim);
    swiglu_forward(fc1_out, swiglu_out, num_tokens, aligned_intermediate_dim);
    gemm_nt_q4_k(swiglu_out, W2, B2, mlp_out, num_tokens, aligned_embed_dim, aligned_intermediate_dim);
 
    /* Final residual add */
    qwen2_0_5b_decode_residual_add_token_major(residual1, mlp_out, output, num_tokens, aligned_embed_dim);
}

References attention_forward_causal_head_major_gqa_flash_strided(), CK_DT_Q4_K, ck_dtype_row_bytes(), gemm_nt_q4_k(), QWEN2_0_5B_DECODELayerOffsets::k, QWEN2_0_5B_DECODELayerOffsets::ln1_gamma, QWEN2_0_5B_DECODELayerOffsets::ln1_out, QWEN2_0_5B_DECODELayerOffsets::ln2_gamma, QWEN2_0_5B_DECODELayerOffsets::ln2_out, QWEN2_0_5B_DECODELayerOffsets::mlp_out, QWEN2_0_5B_DECODELayerOffsets::output, QWEN2_0_5B_DECODELayerOffsets::proj_scratch, QWEN2_0_5B_DECODELayerOffsets::proj_tmp, QWEN2_0_5B_DECODELayerOffsets::q, QWEN2_0_5B_DECODE_EMBED_DIM, QWEN2_0_5B_DECODE_GLOBALS, QWEN2_0_5B_DECODE_HEAD_DIM, QWEN2_0_5B_DECODE_LAYERS, QWEN2_0_5B_DECODE_NUM_HEADS, QWEN2_0_5B_DECODE_NUM_KV_HEADS, QWEN2_0_5B_DECODE_PTR, qwen2_0_5b_decode_residual_add_token_major(), QWEN2_0_5B_DECODELayerOffsets::residual1, rmsnorm_forward(), QWEN2_0_5B_DECODEGlobalOffsets::rope_cos_cache, rope_forward_qk_strided(), QWEN2_0_5B_DECODEGlobalOffsets::rope_sin_cache, swiglu_forward(), QWEN2_0_5B_DECODELayerOffsets::v, QWEN2_0_5B_DECODELayerOffsets::w1, QWEN2_0_5B_DECODELayerOffsets::w2, QWEN2_0_5B_DECODELayerOffsets::wk, QWEN2_0_5B_DECODELayerOffsets::wo, QWEN2_0_5B_DECODELayerOffsets::wq, and QWEN2_0_5B_DECODELayerOffsets::wv.

Referenced by qwen2_0_5b_decode_forward_prefill_impl().

qwen2_0_5b_decode_layer_16_decode()

static void qwen2_0_5b_decode_layer_16_decode ( QWEN2_0_5B_DECODEModel *  model, int  token_index, int  aligned_embed_dim, int  aligned_head_dim, int  aligned_intermediate_dim, int  aligned_context_window  )

static

Definition at line 7301 of file v6.5/test_generated/qwen2_int8.c.

  {
    const QWEN2_0_5B_DECODELayerOffsets *L = &QWEN2_0_5B_DECODE_LAYERS[16];
 
    float *input = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[15].output);
 
    float *ln1_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln1_gamma);
    float *ln1_out = QWEN2_0_5B_DECODE_PTR(model, L->ln1_out);
    float *ln2_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln2_gamma);
    float *ln2_out = QWEN2_0_5B_DECODE_PTR(model, L->ln2_out);
    float *k_cache = QWEN2_0_5B_DECODE_PTR(model, L->k);
    float *v_cache = QWEN2_0_5B_DECODE_PTR(model, L->v);
    float *proj_tmp = QWEN2_0_5B_DECODE_PTR(model, L->proj_tmp);
    float *proj_scratch = QWEN2_0_5B_DECODE_PTR(model, L->proj_scratch);
    float *residual1 = QWEN2_0_5B_DECODE_PTR(model, L->residual1);
    float *mlp_out = QWEN2_0_5B_DECODE_PTR(model, L->mlp_out);
    float *output = QWEN2_0_5B_DECODE_PTR(model, L->output);
 
    /* Weights (explicit types for layer 16) */
    const void *WQ = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wq);  /* Q4_K */
    const void *WK = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wk);  /* Q4_K */
    const void *WV = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wv);  /* Q4_K */
    const void *WO = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wo);  /* Q4_K */
    const void *W1 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w1);  /* Q4_K (gate+up) */
    const void *W2 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w2);  /* Q4_K (down) */
 
    float *rope_cos = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_cos_cache);
    float *rope_sin = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_sin_cache);
 
    const int H = QWEN2_0_5B_DECODE_NUM_HEADS;
    const int H_kv = QWEN2_0_5B_DECODE_NUM_KV_HEADS;
    const int head_dim = QWEN2_0_5B_DECODE_HEAD_DIM;
 
    float q_token[H * aligned_head_dim];
    float k_token[H_kv * aligned_head_dim];
    float v_token[H_kv * aligned_head_dim];
    float attn_token[H * aligned_head_dim];
 
    /* Local MLP buffers (avoid layout dependencies for intermediate values) */
    float fc1_out[2 * aligned_intermediate_dim];
    float swiglu_out[aligned_intermediate_dim];
 
    /* Step 1: RMSNorm before attention */
    rmsnorm_forward(input,
                    ln1_gamma,
                    ln1_out,
                    NULL,
                    1,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    const size_t kv_head_stride = (size_t)aligned_context_window * (size_t)aligned_head_dim;
 
    /* Step 2: QKV projection */
    /* Q projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t ln1_q8_bytes = ((((aligned_embed_dim) + 255) / 256) * 292);
    uint8_t ln1_q8[ln1_q8_bytes];
    quantize_row_q8_k(ln1_out, ln1_q8, aligned_embed_dim);
    gemv_q4_k_q8_k(q_token, WQ, ln1_q8, H * head_dim, aligned_embed_dim);
    if (aligned_head_dim > head_dim) {
        for (int h = 0; h < H; ++h) {
            float *q_head = q_token + (size_t)h * (size_t)aligned_head_dim;
            for (int d = head_dim; d < aligned_head_dim; ++d) {
                q_head[d] = 0.0f;
            }
        }
    }
 
    /* K projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k (direct-to-cache) */
    const size_t wk_head_elems = (size_t)aligned_head_dim * (size_t)aligned_embed_dim;
    const size_t wk_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, wk_head_elems);
    const uint8_t *WK_bytes = (const uint8_t *)WK;
    /* ln1_q8 already quantized above */
    for (int h = 0; h < H_kv; ++h) {
        const void *wk_h = (const void *)(WK_bytes + (size_t)h * wk_head_bytes);
        float *k_head = k_cache + (size_t)h * kv_head_stride + (size_t)token_index * (size_t)aligned_head_dim;
        gemv_q4_k_q8_k(k_head, wk_h, ln1_q8, head_dim, aligned_embed_dim);
        for (int d = head_dim; d < aligned_head_dim; ++d) {
            k_head[d] = 0.0f;
        }
    }
 
    /* V projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k (direct-to-cache) */
    const size_t wv_head_elems = (size_t)aligned_head_dim * (size_t)aligned_embed_dim;
    const size_t wv_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, wv_head_elems);
    const uint8_t *WV_bytes = (const uint8_t *)WV;
    /* ln1_q8 already quantized above */
    for (int h = 0; h < H_kv; ++h) {
        const void *wv_h = (const void *)(WV_bytes + (size_t)h * wv_head_bytes);
        float *v_head = v_cache + (size_t)h * kv_head_stride + (size_t)token_index * (size_t)aligned_head_dim;
        gemv_q4_k_q8_k(v_head, wv_h, ln1_q8, head_dim, aligned_embed_dim);
        for (int d = head_dim; d < aligned_head_dim; ++d) {
            v_head[d] = 0.0f;
        }
    }
 
    /* Step 3: RoPE */
    rope_forward(q_token,
                 rope_cos,
                 rope_sin,
                 H,
                 1,
                 head_dim,
                 aligned_head_dim,
                 token_index);
    for (int h = 0; h < H_kv; ++h) {
        float *k_head = k_cache + (size_t)h * kv_head_stride + (size_t)token_index * (size_t)aligned_head_dim;
        rope_forward(k_head,
                     rope_cos,
                     rope_sin,
                     1,
                     1,
                     head_dim,
                     aligned_head_dim,
                     token_index);
    }
 
    /* Step 4: KV cache write (direct-to-cache) */
 
    /* Step 5: Attention (decode, flash) */
    attention_forward_decode_head_major_gqa_flash(q_token,
                                                   k_cache,
                                                   v_cache,
                                                   attn_token,
                                                   H,
                                                   H_kv,
                                                   token_index + 1,
                                                   aligned_context_window,
                                                   head_dim,
                                                   aligned_head_dim);
 
    /* Step 6: Output projection */
    /* WO projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t attn_q8_bytes = ((((H * head_dim) + 255) / 256) * 292);
    uint8_t attn_q8[attn_q8_bytes];
    quantize_row_q8_k(attn_token, attn_q8, H * head_dim);
    gemv_q4_k_q8_k(proj_tmp, WO, attn_q8, aligned_embed_dim, H * head_dim);
 
    /* Step 7: Residual add */
    qwen2_0_5b_decode_residual_add_token_major(input, proj_tmp, residual1, 1, aligned_embed_dim);
 
    /* Step 8: RMSNorm before MLP */
    rmsnorm_forward(residual1,
                    ln2_gamma,
                    ln2_out,
                    NULL,
                    1,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    /* Step 9: MLP (SwiGLU) */
    /* Gate+Up projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t ln2_q8_bytes = ((((aligned_embed_dim) + 255) / 256) * 292);
    uint8_t ln2_q8[ln2_q8_bytes];
    quantize_row_q8_k(ln2_out, ln2_q8, aligned_embed_dim);
    gemv_q4_k_q8_k(fc1_out, W1, ln2_q8, 2 * aligned_intermediate_dim, aligned_embed_dim);
 
    /* SwiGLU activation */
    swiglu_forward(fc1_out, swiglu_out, 1, aligned_intermediate_dim);
 
    /* Down projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t swiglu_q8_bytes = ((((aligned_intermediate_dim) + 255) / 256) * 292);
    uint8_t swiglu_q8[swiglu_q8_bytes];
    quantize_row_q8_k(swiglu_out, swiglu_q8, aligned_intermediate_dim);
    gemv_q4_k_q8_k(mlp_out, W2, swiglu_q8, aligned_embed_dim, aligned_intermediate_dim);
 
    /* Step 10: Final residual add */
    qwen2_0_5b_decode_residual_add_token_major(residual1, mlp_out, output, 1, aligned_embed_dim);
}

References attention_forward_decode_head_major_gqa_flash(), CK_DT_Q4_K, ck_dtype_row_bytes(), gemv_q4_k_q8_k(), QWEN2_0_5B_DECODELayerOffsets::k, QWEN2_0_5B_DECODELayerOffsets::ln1_gamma, QWEN2_0_5B_DECODELayerOffsets::ln1_out, QWEN2_0_5B_DECODELayerOffsets::ln2_gamma, QWEN2_0_5B_DECODELayerOffsets::ln2_out, QWEN2_0_5B_DECODELayerOffsets::mlp_out, QWEN2_0_5B_DECODELayerOffsets::output, QWEN2_0_5B_DECODELayerOffsets::proj_scratch, QWEN2_0_5B_DECODELayerOffsets::proj_tmp, quantize_row_q8_k(), QWEN2_0_5B_DECODE_EMBED_DIM, QWEN2_0_5B_DECODE_GLOBALS, QWEN2_0_5B_DECODE_HEAD_DIM, QWEN2_0_5B_DECODE_LAYERS, QWEN2_0_5B_DECODE_NUM_HEADS, QWEN2_0_5B_DECODE_NUM_KV_HEADS, QWEN2_0_5B_DECODE_PTR, qwen2_0_5b_decode_residual_add_token_major(), QWEN2_0_5B_DECODELayerOffsets::residual1, rmsnorm_forward(), QWEN2_0_5B_DECODEGlobalOffsets::rope_cos_cache, rope_forward(), QWEN2_0_5B_DECODEGlobalOffsets::rope_sin_cache, swiglu_forward(), QWEN2_0_5B_DECODELayerOffsets::v, QWEN2_0_5B_DECODELayerOffsets::w1, QWEN2_0_5B_DECODELayerOffsets::w2, QWEN2_0_5B_DECODELayerOffsets::wk, QWEN2_0_5B_DECODELayerOffsets::wo, QWEN2_0_5B_DECODELayerOffsets::wq, and QWEN2_0_5B_DECODELayerOffsets::wv.

Referenced by qwen2_0_5b_decode_decode_token().

qwen2_0_5b_decode_layer_16_prefill()

static void qwen2_0_5b_decode_layer_16_prefill ( QWEN2_0_5B_DECODEModel *  model, int  num_tokens, int  aligned_embed_dim, int  aligned_head_dim, int  aligned_intermediate_dim, int  aligned_context_window  )

static

Definition at line 2835 of file v6.5/test_generated/qwen2_int8.c.

  {
    const QWEN2_0_5B_DECODELayerOffsets *L = &QWEN2_0_5B_DECODE_LAYERS[16];
 
    float *input = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[15].output);
    float *ln1_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln1_gamma);
    float *ln1_out = QWEN2_0_5B_DECODE_PTR(model, L->ln1_out);
    float *ln2_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln2_gamma);
    float *ln2_out = QWEN2_0_5B_DECODE_PTR(model, L->ln2_out);
    float *q = QWEN2_0_5B_DECODE_PTR(model, L->q);
    float *k = QWEN2_0_5B_DECODE_PTR(model, L->k);
    float *v = QWEN2_0_5B_DECODE_PTR(model, L->v);
    float *attn_out = QWEN2_0_5B_DECODE_PTR(model, L->attn_out);
    float *proj_tmp = QWEN2_0_5B_DECODE_PTR(model, L->proj_tmp);
    float *proj_scratch = QWEN2_0_5B_DECODE_PTR(model, L->proj_scratch);
    float *residual1 = QWEN2_0_5B_DECODE_PTR(model, L->residual1);
    float *fc1_out = QWEN2_0_5B_DECODE_PTR(model, L->fc1_out);
    float *swiglu_out = QWEN2_0_5B_DECODE_PTR(model, L->swiglu_out);
    float *mlp_out = QWEN2_0_5B_DECODE_PTR(model, L->mlp_out);
    float *output = QWEN2_0_5B_DECODE_PTR(model, L->output);
 
    const void *WQ = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wq);
    const void *WK = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wk);
    const void *WV = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wv);
    const void *WO = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wo);
    const void *W1 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w1);
    const void *W2 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w2);
    const float *BQ = NULL;
    const float *BK = NULL;
    const float *BV = NULL;
    const float *BO = NULL;
    const float *B1 = NULL;
    const float *B2 = NULL;
 
    float *rope_cos = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_cos_cache);
    float *rope_sin = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_sin_cache);
 
    const int H = QWEN2_0_5B_DECODE_NUM_HEADS;
    const int H_kv = QWEN2_0_5B_DECODE_NUM_KV_HEADS;
    const int head_dim = QWEN2_0_5B_DECODE_HEAD_DIM;
    const size_t head_w_elems = (size_t)aligned_head_dim * (size_t)aligned_embed_dim;
    const size_t q_head_stride = (size_t)num_tokens * (size_t)aligned_head_dim;
    const size_t kv_head_stride = (size_t)aligned_context_window * (size_t)aligned_head_dim;
 
    /* RMSNorm before attention */
    rmsnorm_forward(input,
                    ln1_gamma,
                    ln1_out,
                    NULL,
                    num_tokens,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    /* Q projection (head-major) */
    const size_t wq_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, head_w_elems);
    const uint8_t *WQ_bytes = (const uint8_t *)WQ;
    for (int h = 0; h < H; ++h) {
        const void *wq_h = (const void *)(WQ_bytes + (size_t)h * wq_head_bytes);
        const float *bq_h = BQ ? (BQ + (size_t)h * (size_t)aligned_head_dim) : NULL;
        float *q_h = q + (size_t)h * q_head_stride;
        gemm_nt_q4_k(ln1_out, wq_h, bq_h, q_h, num_tokens, aligned_head_dim, aligned_embed_dim);
    }
 
    /* K projection (head-major) */
    const size_t wk_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, head_w_elems);
    const uint8_t *WK_bytes = (const uint8_t *)WK;
    for (int h = 0; h < H_kv; ++h) {
        const void *wk_h = (const void *)(WK_bytes + (size_t)h * wk_head_bytes);
        const float *bk_h = BK ? (BK + (size_t)h * (size_t)aligned_head_dim) : NULL;
        float *k_h = k + (size_t)h * kv_head_stride;
        gemm_nt_q4_k(ln1_out, wk_h, bk_h, k_h, num_tokens, aligned_head_dim, aligned_embed_dim);
    }
 
    /* V projection (head-major) */
    const size_t wv_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, head_w_elems);
    const uint8_t *WV_bytes = (const uint8_t *)WV;
    for (int h = 0; h < H_kv; ++h) {
        const void *wv_h = (const void *)(WV_bytes + (size_t)h * wv_head_bytes);
        const float *bv_h = BV ? (BV + (size_t)h * (size_t)aligned_head_dim) : NULL;
        float *v_h = v + (size_t)h * kv_head_stride;
        gemm_nt_q4_k(ln1_out, wv_h, bv_h, v_h, num_tokens, aligned_head_dim, aligned_embed_dim);
    }
 
    /* RoPE */
    rope_forward_qk_strided(q,
                            k,
                            rope_cos,
                            rope_sin,
                            H,
                            H_kv,
                            num_tokens,
                            head_dim,
                            aligned_head_dim,
                            0,
                            num_tokens,
                            aligned_context_window);
 
    /* Attention (prefill, causal) */
    attention_forward_causal_head_major_gqa_flash_strided(q,
                                                           k,
                                                           v,
                                                           attn_out,
                                                           H,
                                                           H_kv,
                                                           num_tokens,
                                                           head_dim,
                                                           aligned_head_dim,
                                                           aligned_context_window);
 
    /* Output projection (flatten head-major to token-major) */
    const int K = H * aligned_head_dim;
    if (K != aligned_embed_dim) {
        return;
    }
    const float *proj_in = attn_out;
    if (H > 1) {
        if (!proj_scratch) {
            return;
        }
        for (int t = 0; t < num_tokens; ++t) {
            float *dst = proj_scratch + (size_t)t * (size_t)aligned_embed_dim;
            for (int h = 0; h < H; ++h) {
                const float *src = attn_out + (size_t)h * q_head_stride + (size_t)t * (size_t)aligned_head_dim;
                memcpy(dst + (size_t)h * (size_t)aligned_head_dim,
                       src,
                       (size_t)aligned_head_dim * sizeof(float));
            }
        }
        proj_in = proj_scratch;
    }
    gemm_nt_q4_k(proj_in, WO, BO, proj_tmp, num_tokens, aligned_embed_dim, K);
 
    /* Residual add */
    qwen2_0_5b_decode_residual_add_token_major(input, proj_tmp, residual1, num_tokens, aligned_embed_dim);
 
    /* RMSNorm before MLP */
    rmsnorm_forward(residual1,
                    ln2_gamma,
                    ln2_out,
                    NULL,
                    num_tokens,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    /* MLP (SwiGLU) */
    gemm_nt_q4_k(ln2_out, W1, B1, fc1_out, num_tokens, 2 * aligned_intermediate_dim, aligned_embed_dim);
    swiglu_forward(fc1_out, swiglu_out, num_tokens, aligned_intermediate_dim);
    gemm_nt_q4_k(swiglu_out, W2, B2, mlp_out, num_tokens, aligned_embed_dim, aligned_intermediate_dim);
 
    /* Final residual add */
    qwen2_0_5b_decode_residual_add_token_major(residual1, mlp_out, output, num_tokens, aligned_embed_dim);
}

References attention_forward_causal_head_major_gqa_flash_strided(), CK_DT_Q4_K, ck_dtype_row_bytes(), gemm_nt_q4_k(), QWEN2_0_5B_DECODELayerOffsets::k, QWEN2_0_5B_DECODELayerOffsets::ln1_gamma, QWEN2_0_5B_DECODELayerOffsets::ln1_out, QWEN2_0_5B_DECODELayerOffsets::ln2_gamma, QWEN2_0_5B_DECODELayerOffsets::ln2_out, QWEN2_0_5B_DECODELayerOffsets::mlp_out, QWEN2_0_5B_DECODELayerOffsets::output, QWEN2_0_5B_DECODELayerOffsets::proj_scratch, QWEN2_0_5B_DECODELayerOffsets::proj_tmp, QWEN2_0_5B_DECODELayerOffsets::q, QWEN2_0_5B_DECODE_EMBED_DIM, QWEN2_0_5B_DECODE_GLOBALS, QWEN2_0_5B_DECODE_HEAD_DIM, QWEN2_0_5B_DECODE_LAYERS, QWEN2_0_5B_DECODE_NUM_HEADS, QWEN2_0_5B_DECODE_NUM_KV_HEADS, QWEN2_0_5B_DECODE_PTR, qwen2_0_5b_decode_residual_add_token_major(), QWEN2_0_5B_DECODELayerOffsets::residual1, rmsnorm_forward(), QWEN2_0_5B_DECODEGlobalOffsets::rope_cos_cache, rope_forward_qk_strided(), QWEN2_0_5B_DECODEGlobalOffsets::rope_sin_cache, swiglu_forward(), QWEN2_0_5B_DECODELayerOffsets::v, QWEN2_0_5B_DECODELayerOffsets::w1, QWEN2_0_5B_DECODELayerOffsets::w2, QWEN2_0_5B_DECODELayerOffsets::wk, QWEN2_0_5B_DECODELayerOffsets::wo, QWEN2_0_5B_DECODELayerOffsets::wq, and QWEN2_0_5B_DECODELayerOffsets::wv.

Referenced by qwen2_0_5b_decode_forward_prefill_impl().

qwen2_0_5b_decode_layer_17_decode()

static void qwen2_0_5b_decode_layer_17_decode ( QWEN2_0_5B_DECODEModel *  model, int  token_index, int  aligned_embed_dim, int  aligned_head_dim, int  aligned_intermediate_dim, int  aligned_context_window  )

static

Definition at line 7482 of file v6.5/test_generated/qwen2_int8.c.

  {
    const QWEN2_0_5B_DECODELayerOffsets *L = &QWEN2_0_5B_DECODE_LAYERS[17];
 
    float *input = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[16].output);
 
    float *ln1_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln1_gamma);
    float *ln1_out = QWEN2_0_5B_DECODE_PTR(model, L->ln1_out);
    float *ln2_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln2_gamma);
    float *ln2_out = QWEN2_0_5B_DECODE_PTR(model, L->ln2_out);
    float *k_cache = QWEN2_0_5B_DECODE_PTR(model, L->k);
    float *v_cache = QWEN2_0_5B_DECODE_PTR(model, L->v);
    float *proj_tmp = QWEN2_0_5B_DECODE_PTR(model, L->proj_tmp);
    float *proj_scratch = QWEN2_0_5B_DECODE_PTR(model, L->proj_scratch);
    float *residual1 = QWEN2_0_5B_DECODE_PTR(model, L->residual1);
    float *mlp_out = QWEN2_0_5B_DECODE_PTR(model, L->mlp_out);
    float *output = QWEN2_0_5B_DECODE_PTR(model, L->output);
 
    /* Weights (explicit types for layer 17) */
    const void *WQ = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wq);  /* Q4_K */
    const void *WK = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wk);  /* Q4_K */
    const void *WV = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wv);  /* Q4_K */
    const void *WO = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wo);  /* Q4_K */
    const void *W1 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w1);  /* Q4_K (gate+up) */
    const void *W2 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w2);  /* Q4_K (down) */
 
    float *rope_cos = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_cos_cache);
    float *rope_sin = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_sin_cache);
 
    const int H = QWEN2_0_5B_DECODE_NUM_HEADS;
    const int H_kv = QWEN2_0_5B_DECODE_NUM_KV_HEADS;
    const int head_dim = QWEN2_0_5B_DECODE_HEAD_DIM;
 
    float q_token[H * aligned_head_dim];
    float k_token[H_kv * aligned_head_dim];
    float v_token[H_kv * aligned_head_dim];
    float attn_token[H * aligned_head_dim];
 
    /* Local MLP buffers (avoid layout dependencies for intermediate values) */
    float fc1_out[2 * aligned_intermediate_dim];
    float swiglu_out[aligned_intermediate_dim];
 
    /* Step 1: RMSNorm before attention */
    rmsnorm_forward(input,
                    ln1_gamma,
                    ln1_out,
                    NULL,
                    1,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    const size_t kv_head_stride = (size_t)aligned_context_window * (size_t)aligned_head_dim;
 
    /* Step 2: QKV projection */
    /* Q projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t ln1_q8_bytes = ((((aligned_embed_dim) + 255) / 256) * 292);
    uint8_t ln1_q8[ln1_q8_bytes];
    quantize_row_q8_k(ln1_out, ln1_q8, aligned_embed_dim);
    gemv_q4_k_q8_k(q_token, WQ, ln1_q8, H * head_dim, aligned_embed_dim);
    if (aligned_head_dim > head_dim) {
        for (int h = 0; h < H; ++h) {
            float *q_head = q_token + (size_t)h * (size_t)aligned_head_dim;
            for (int d = head_dim; d < aligned_head_dim; ++d) {
                q_head[d] = 0.0f;
            }
        }
    }
 
    /* K projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k (direct-to-cache) */
    const size_t wk_head_elems = (size_t)aligned_head_dim * (size_t)aligned_embed_dim;
    const size_t wk_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, wk_head_elems);
    const uint8_t *WK_bytes = (const uint8_t *)WK;
    /* ln1_q8 already quantized above */
    for (int h = 0; h < H_kv; ++h) {
        const void *wk_h = (const void *)(WK_bytes + (size_t)h * wk_head_bytes);
        float *k_head = k_cache + (size_t)h * kv_head_stride + (size_t)token_index * (size_t)aligned_head_dim;
        gemv_q4_k_q8_k(k_head, wk_h, ln1_q8, head_dim, aligned_embed_dim);
        for (int d = head_dim; d < aligned_head_dim; ++d) {
            k_head[d] = 0.0f;
        }
    }
 
    /* V projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k (direct-to-cache) */
    const size_t wv_head_elems = (size_t)aligned_head_dim * (size_t)aligned_embed_dim;
    const size_t wv_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, wv_head_elems);
    const uint8_t *WV_bytes = (const uint8_t *)WV;
    /* ln1_q8 already quantized above */
    for (int h = 0; h < H_kv; ++h) {
        const void *wv_h = (const void *)(WV_bytes + (size_t)h * wv_head_bytes);
        float *v_head = v_cache + (size_t)h * kv_head_stride + (size_t)token_index * (size_t)aligned_head_dim;
        gemv_q4_k_q8_k(v_head, wv_h, ln1_q8, head_dim, aligned_embed_dim);
        for (int d = head_dim; d < aligned_head_dim; ++d) {
            v_head[d] = 0.0f;
        }
    }
 
    /* Step 3: RoPE */
    rope_forward(q_token,
                 rope_cos,
                 rope_sin,
                 H,
                 1,
                 head_dim,
                 aligned_head_dim,
                 token_index);
    for (int h = 0; h < H_kv; ++h) {
        float *k_head = k_cache + (size_t)h * kv_head_stride + (size_t)token_index * (size_t)aligned_head_dim;
        rope_forward(k_head,
                     rope_cos,
                     rope_sin,
                     1,
                     1,
                     head_dim,
                     aligned_head_dim,
                     token_index);
    }
 
    /* Step 4: KV cache write (direct-to-cache) */
 
    /* Step 5: Attention (decode, flash) */
    attention_forward_decode_head_major_gqa_flash(q_token,
                                                   k_cache,
                                                   v_cache,
                                                   attn_token,
                                                   H,
                                                   H_kv,
                                                   token_index + 1,
                                                   aligned_context_window,
                                                   head_dim,
                                                   aligned_head_dim);
 
    /* Step 6: Output projection */
    /* WO projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t attn_q8_bytes = ((((H * head_dim) + 255) / 256) * 292);
    uint8_t attn_q8[attn_q8_bytes];
    quantize_row_q8_k(attn_token, attn_q8, H * head_dim);
    gemv_q4_k_q8_k(proj_tmp, WO, attn_q8, aligned_embed_dim, H * head_dim);
 
    /* Step 7: Residual add */
    qwen2_0_5b_decode_residual_add_token_major(input, proj_tmp, residual1, 1, aligned_embed_dim);
 
    /* Step 8: RMSNorm before MLP */
    rmsnorm_forward(residual1,
                    ln2_gamma,
                    ln2_out,
                    NULL,
                    1,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    /* Step 9: MLP (SwiGLU) */
    /* Gate+Up projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t ln2_q8_bytes = ((((aligned_embed_dim) + 255) / 256) * 292);
    uint8_t ln2_q8[ln2_q8_bytes];
    quantize_row_q8_k(ln2_out, ln2_q8, aligned_embed_dim);
    gemv_q4_k_q8_k(fc1_out, W1, ln2_q8, 2 * aligned_intermediate_dim, aligned_embed_dim);
 
    /* SwiGLU activation */
    swiglu_forward(fc1_out, swiglu_out, 1, aligned_intermediate_dim);
 
    /* Down projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t swiglu_q8_bytes = ((((aligned_intermediate_dim) + 255) / 256) * 292);
    uint8_t swiglu_q8[swiglu_q8_bytes];
    quantize_row_q8_k(swiglu_out, swiglu_q8, aligned_intermediate_dim);
    gemv_q4_k_q8_k(mlp_out, W2, swiglu_q8, aligned_embed_dim, aligned_intermediate_dim);
 
    /* Step 10: Final residual add */
    qwen2_0_5b_decode_residual_add_token_major(residual1, mlp_out, output, 1, aligned_embed_dim);
}

References attention_forward_decode_head_major_gqa_flash(), CK_DT_Q4_K, ck_dtype_row_bytes(), gemv_q4_k_q8_k(), QWEN2_0_5B_DECODELayerOffsets::k, QWEN2_0_5B_DECODELayerOffsets::ln1_gamma, QWEN2_0_5B_DECODELayerOffsets::ln1_out, QWEN2_0_5B_DECODELayerOffsets::ln2_gamma, QWEN2_0_5B_DECODELayerOffsets::ln2_out, QWEN2_0_5B_DECODELayerOffsets::mlp_out, QWEN2_0_5B_DECODELayerOffsets::output, QWEN2_0_5B_DECODELayerOffsets::proj_scratch, QWEN2_0_5B_DECODELayerOffsets::proj_tmp, quantize_row_q8_k(), QWEN2_0_5B_DECODE_EMBED_DIM, QWEN2_0_5B_DECODE_GLOBALS, QWEN2_0_5B_DECODE_HEAD_DIM, QWEN2_0_5B_DECODE_LAYERS, QWEN2_0_5B_DECODE_NUM_HEADS, QWEN2_0_5B_DECODE_NUM_KV_HEADS, QWEN2_0_5B_DECODE_PTR, qwen2_0_5b_decode_residual_add_token_major(), QWEN2_0_5B_DECODELayerOffsets::residual1, rmsnorm_forward(), QWEN2_0_5B_DECODEGlobalOffsets::rope_cos_cache, rope_forward(), QWEN2_0_5B_DECODEGlobalOffsets::rope_sin_cache, swiglu_forward(), QWEN2_0_5B_DECODELayerOffsets::v, QWEN2_0_5B_DECODELayerOffsets::w1, QWEN2_0_5B_DECODELayerOffsets::w2, QWEN2_0_5B_DECODELayerOffsets::wk, QWEN2_0_5B_DECODELayerOffsets::wo, QWEN2_0_5B_DECODELayerOffsets::wq, and QWEN2_0_5B_DECODELayerOffsets::wv.

Referenced by qwen2_0_5b_decode_decode_token().

qwen2_0_5b_decode_layer_17_prefill()

static void qwen2_0_5b_decode_layer_17_prefill ( QWEN2_0_5B_DECODEModel *  model, int  num_tokens, int  aligned_embed_dim, int  aligned_head_dim, int  aligned_intermediate_dim, int  aligned_context_window  )

static

Definition at line 2999 of file v6.5/test_generated/qwen2_int8.c.

  {
    const QWEN2_0_5B_DECODELayerOffsets *L = &QWEN2_0_5B_DECODE_LAYERS[17];
 
    float *input = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[16].output);
    float *ln1_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln1_gamma);
    float *ln1_out = QWEN2_0_5B_DECODE_PTR(model, L->ln1_out);
    float *ln2_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln2_gamma);
    float *ln2_out = QWEN2_0_5B_DECODE_PTR(model, L->ln2_out);
    float *q = QWEN2_0_5B_DECODE_PTR(model, L->q);
    float *k = QWEN2_0_5B_DECODE_PTR(model, L->k);
    float *v = QWEN2_0_5B_DECODE_PTR(model, L->v);
    float *attn_out = QWEN2_0_5B_DECODE_PTR(model, L->attn_out);
    float *proj_tmp = QWEN2_0_5B_DECODE_PTR(model, L->proj_tmp);
    float *proj_scratch = QWEN2_0_5B_DECODE_PTR(model, L->proj_scratch);
    float *residual1 = QWEN2_0_5B_DECODE_PTR(model, L->residual1);
    float *fc1_out = QWEN2_0_5B_DECODE_PTR(model, L->fc1_out);
    float *swiglu_out = QWEN2_0_5B_DECODE_PTR(model, L->swiglu_out);
    float *mlp_out = QWEN2_0_5B_DECODE_PTR(model, L->mlp_out);
    float *output = QWEN2_0_5B_DECODE_PTR(model, L->output);
 
    const void *WQ = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wq);
    const void *WK = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wk);
    const void *WV = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wv);
    const void *WO = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wo);
    const void *W1 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w1);
    const void *W2 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w2);
    const float *BQ = NULL;
    const float *BK = NULL;
    const float *BV = NULL;
    const float *BO = NULL;
    const float *B1 = NULL;
    const float *B2 = NULL;
 
    float *rope_cos = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_cos_cache);
    float *rope_sin = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_sin_cache);
 
    const int H = QWEN2_0_5B_DECODE_NUM_HEADS;
    const int H_kv = QWEN2_0_5B_DECODE_NUM_KV_HEADS;
    const int head_dim = QWEN2_0_5B_DECODE_HEAD_DIM;
    const size_t head_w_elems = (size_t)aligned_head_dim * (size_t)aligned_embed_dim;
    const size_t q_head_stride = (size_t)num_tokens * (size_t)aligned_head_dim;
    const size_t kv_head_stride = (size_t)aligned_context_window * (size_t)aligned_head_dim;
 
    /* RMSNorm before attention */
    rmsnorm_forward(input,
                    ln1_gamma,
                    ln1_out,
                    NULL,
                    num_tokens,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    /* Q projection (head-major) */
    const size_t wq_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, head_w_elems);
    const uint8_t *WQ_bytes = (const uint8_t *)WQ;
    for (int h = 0; h < H; ++h) {
        const void *wq_h = (const void *)(WQ_bytes + (size_t)h * wq_head_bytes);
        const float *bq_h = BQ ? (BQ + (size_t)h * (size_t)aligned_head_dim) : NULL;
        float *q_h = q + (size_t)h * q_head_stride;
        gemm_nt_q4_k(ln1_out, wq_h, bq_h, q_h, num_tokens, aligned_head_dim, aligned_embed_dim);
    }
 
    /* K projection (head-major) */
    const size_t wk_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, head_w_elems);
    const uint8_t *WK_bytes = (const uint8_t *)WK;
    for (int h = 0; h < H_kv; ++h) {
        const void *wk_h = (const void *)(WK_bytes + (size_t)h * wk_head_bytes);
        const float *bk_h = BK ? (BK + (size_t)h * (size_t)aligned_head_dim) : NULL;
        float *k_h = k + (size_t)h * kv_head_stride;
        gemm_nt_q4_k(ln1_out, wk_h, bk_h, k_h, num_tokens, aligned_head_dim, aligned_embed_dim);
    }
 
    /* V projection (head-major) */
    const size_t wv_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, head_w_elems);
    const uint8_t *WV_bytes = (const uint8_t *)WV;
    for (int h = 0; h < H_kv; ++h) {
        const void *wv_h = (const void *)(WV_bytes + (size_t)h * wv_head_bytes);
        const float *bv_h = BV ? (BV + (size_t)h * (size_t)aligned_head_dim) : NULL;
        float *v_h = v + (size_t)h * kv_head_stride;
        gemm_nt_q4_k(ln1_out, wv_h, bv_h, v_h, num_tokens, aligned_head_dim, aligned_embed_dim);
    }
 
    /* RoPE */
    rope_forward_qk_strided(q,
                            k,
                            rope_cos,
                            rope_sin,
                            H,
                            H_kv,
                            num_tokens,
                            head_dim,
                            aligned_head_dim,
                            0,
                            num_tokens,
                            aligned_context_window);
 
    /* Attention (prefill, causal) */
    attention_forward_causal_head_major_gqa_flash_strided(q,
                                                           k,
                                                           v,
                                                           attn_out,
                                                           H,
                                                           H_kv,
                                                           num_tokens,
                                                           head_dim,
                                                           aligned_head_dim,
                                                           aligned_context_window);
 
    /* Output projection (flatten head-major to token-major) */
    const int K = H * aligned_head_dim;
    if (K != aligned_embed_dim) {
        return;
    }
    const float *proj_in = attn_out;
    if (H > 1) {
        if (!proj_scratch) {
            return;
        }
        for (int t = 0; t < num_tokens; ++t) {
            float *dst = proj_scratch + (size_t)t * (size_t)aligned_embed_dim;
            for (int h = 0; h < H; ++h) {
                const float *src = attn_out + (size_t)h * q_head_stride + (size_t)t * (size_t)aligned_head_dim;
                memcpy(dst + (size_t)h * (size_t)aligned_head_dim,
                       src,
                       (size_t)aligned_head_dim * sizeof(float));
            }
        }
        proj_in = proj_scratch;
    }
    gemm_nt_q4_k(proj_in, WO, BO, proj_tmp, num_tokens, aligned_embed_dim, K);
 
    /* Residual add */
    qwen2_0_5b_decode_residual_add_token_major(input, proj_tmp, residual1, num_tokens, aligned_embed_dim);
 
    /* RMSNorm before MLP */
    rmsnorm_forward(residual1,
                    ln2_gamma,
                    ln2_out,
                    NULL,
                    num_tokens,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    /* MLP (SwiGLU) */
    gemm_nt_q4_k(ln2_out, W1, B1, fc1_out, num_tokens, 2 * aligned_intermediate_dim, aligned_embed_dim);
    swiglu_forward(fc1_out, swiglu_out, num_tokens, aligned_intermediate_dim);
    gemm_nt_q4_k(swiglu_out, W2, B2, mlp_out, num_tokens, aligned_embed_dim, aligned_intermediate_dim);
 
    /* Final residual add */
    qwen2_0_5b_decode_residual_add_token_major(residual1, mlp_out, output, num_tokens, aligned_embed_dim);
}

References attention_forward_causal_head_major_gqa_flash_strided(), CK_DT_Q4_K, ck_dtype_row_bytes(), gemm_nt_q4_k(), QWEN2_0_5B_DECODELayerOffsets::k, QWEN2_0_5B_DECODELayerOffsets::ln1_gamma, QWEN2_0_5B_DECODELayerOffsets::ln1_out, QWEN2_0_5B_DECODELayerOffsets::ln2_gamma, QWEN2_0_5B_DECODELayerOffsets::ln2_out, QWEN2_0_5B_DECODELayerOffsets::mlp_out, QWEN2_0_5B_DECODELayerOffsets::output, QWEN2_0_5B_DECODELayerOffsets::proj_scratch, QWEN2_0_5B_DECODELayerOffsets::proj_tmp, QWEN2_0_5B_DECODELayerOffsets::q, QWEN2_0_5B_DECODE_EMBED_DIM, QWEN2_0_5B_DECODE_GLOBALS, QWEN2_0_5B_DECODE_HEAD_DIM, QWEN2_0_5B_DECODE_LAYERS, QWEN2_0_5B_DECODE_NUM_HEADS, QWEN2_0_5B_DECODE_NUM_KV_HEADS, QWEN2_0_5B_DECODE_PTR, qwen2_0_5b_decode_residual_add_token_major(), QWEN2_0_5B_DECODELayerOffsets::residual1, rmsnorm_forward(), QWEN2_0_5B_DECODEGlobalOffsets::rope_cos_cache, rope_forward_qk_strided(), QWEN2_0_5B_DECODEGlobalOffsets::rope_sin_cache, swiglu_forward(), QWEN2_0_5B_DECODELayerOffsets::v, QWEN2_0_5B_DECODELayerOffsets::w1, QWEN2_0_5B_DECODELayerOffsets::w2, QWEN2_0_5B_DECODELayerOffsets::wk, QWEN2_0_5B_DECODELayerOffsets::wo, QWEN2_0_5B_DECODELayerOffsets::wq, and QWEN2_0_5B_DECODELayerOffsets::wv.

Referenced by qwen2_0_5b_decode_forward_prefill_impl().

qwen2_0_5b_decode_layer_18_decode()

static void qwen2_0_5b_decode_layer_18_decode ( QWEN2_0_5B_DECODEModel *  model, int  token_index, int  aligned_embed_dim, int  aligned_head_dim, int  aligned_intermediate_dim, int  aligned_context_window  )

static

Definition at line 7663 of file v6.5/test_generated/qwen2_int8.c.

  {
    const QWEN2_0_5B_DECODELayerOffsets *L = &QWEN2_0_5B_DECODE_LAYERS[18];
 
    float *input = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[17].output);
 
    float *ln1_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln1_gamma);
    float *ln1_out = QWEN2_0_5B_DECODE_PTR(model, L->ln1_out);
    float *ln2_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln2_gamma);
    float *ln2_out = QWEN2_0_5B_DECODE_PTR(model, L->ln2_out);
    float *k_cache = QWEN2_0_5B_DECODE_PTR(model, L->k);
    float *v_cache = QWEN2_0_5B_DECODE_PTR(model, L->v);
    float *proj_tmp = QWEN2_0_5B_DECODE_PTR(model, L->proj_tmp);
    float *proj_scratch = QWEN2_0_5B_DECODE_PTR(model, L->proj_scratch);
    float *residual1 = QWEN2_0_5B_DECODE_PTR(model, L->residual1);
    float *mlp_out = QWEN2_0_5B_DECODE_PTR(model, L->mlp_out);
    float *output = QWEN2_0_5B_DECODE_PTR(model, L->output);
 
    /* Weights (explicit types for layer 18) */
    const void *WQ = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wq);  /* Q4_K */
    const void *WK = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wk);  /* Q4_K */
    const void *WV = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wv);  /* Q4_K */
    const void *WO = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wo);  /* Q4_K */
    const void *W1 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w1);  /* Q4_K (gate+up) */
    const void *W2 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w2);  /* Q4_K (down) */
 
    float *rope_cos = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_cos_cache);
    float *rope_sin = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_sin_cache);
 
    const int H = QWEN2_0_5B_DECODE_NUM_HEADS;
    const int H_kv = QWEN2_0_5B_DECODE_NUM_KV_HEADS;
    const int head_dim = QWEN2_0_5B_DECODE_HEAD_DIM;
 
    float q_token[H * aligned_head_dim];
    float k_token[H_kv * aligned_head_dim];
    float v_token[H_kv * aligned_head_dim];
    float attn_token[H * aligned_head_dim];
 
    /* Local MLP buffers (avoid layout dependencies for intermediate values) */
    float fc1_out[2 * aligned_intermediate_dim];
    float swiglu_out[aligned_intermediate_dim];
 
    /* Step 1: RMSNorm before attention */
    rmsnorm_forward(input,
                    ln1_gamma,
                    ln1_out,
                    NULL,
                    1,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    const size_t kv_head_stride = (size_t)aligned_context_window * (size_t)aligned_head_dim;
 
    /* Step 2: QKV projection */
    /* Q projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t ln1_q8_bytes = ((((aligned_embed_dim) + 255) / 256) * 292);
    uint8_t ln1_q8[ln1_q8_bytes];
    quantize_row_q8_k(ln1_out, ln1_q8, aligned_embed_dim);
    gemv_q4_k_q8_k(q_token, WQ, ln1_q8, H * head_dim, aligned_embed_dim);
    if (aligned_head_dim > head_dim) {
        for (int h = 0; h < H; ++h) {
            float *q_head = q_token + (size_t)h * (size_t)aligned_head_dim;
            for (int d = head_dim; d < aligned_head_dim; ++d) {
                q_head[d] = 0.0f;
            }
        }
    }
 
    /* K projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k (direct-to-cache) */
    const size_t wk_head_elems = (size_t)aligned_head_dim * (size_t)aligned_embed_dim;
    const size_t wk_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, wk_head_elems);
    const uint8_t *WK_bytes = (const uint8_t *)WK;
    /* ln1_q8 already quantized above */
    for (int h = 0; h < H_kv; ++h) {
        const void *wk_h = (const void *)(WK_bytes + (size_t)h * wk_head_bytes);
        float *k_head = k_cache + (size_t)h * kv_head_stride + (size_t)token_index * (size_t)aligned_head_dim;
        gemv_q4_k_q8_k(k_head, wk_h, ln1_q8, head_dim, aligned_embed_dim);
        for (int d = head_dim; d < aligned_head_dim; ++d) {
            k_head[d] = 0.0f;
        }
    }
 
    /* V projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k (direct-to-cache) */
    const size_t wv_head_elems = (size_t)aligned_head_dim * (size_t)aligned_embed_dim;
    const size_t wv_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, wv_head_elems);
    const uint8_t *WV_bytes = (const uint8_t *)WV;
    /* ln1_q8 already quantized above */
    for (int h = 0; h < H_kv; ++h) {
        const void *wv_h = (const void *)(WV_bytes + (size_t)h * wv_head_bytes);
        float *v_head = v_cache + (size_t)h * kv_head_stride + (size_t)token_index * (size_t)aligned_head_dim;
        gemv_q4_k_q8_k(v_head, wv_h, ln1_q8, head_dim, aligned_embed_dim);
        for (int d = head_dim; d < aligned_head_dim; ++d) {
            v_head[d] = 0.0f;
        }
    }
 
    /* Step 3: RoPE */
    rope_forward(q_token,
                 rope_cos,
                 rope_sin,
                 H,
                 1,
                 head_dim,
                 aligned_head_dim,
                 token_index);
    for (int h = 0; h < H_kv; ++h) {
        float *k_head = k_cache + (size_t)h * kv_head_stride + (size_t)token_index * (size_t)aligned_head_dim;
        rope_forward(k_head,
                     rope_cos,
                     rope_sin,
                     1,
                     1,
                     head_dim,
                     aligned_head_dim,
                     token_index);
    }
 
    /* Step 4: KV cache write (direct-to-cache) */
 
    /* Step 5: Attention (decode, flash) */
    attention_forward_decode_head_major_gqa_flash(q_token,
                                                   k_cache,
                                                   v_cache,
                                                   attn_token,
                                                   H,
                                                   H_kv,
                                                   token_index + 1,
                                                   aligned_context_window,
                                                   head_dim,
                                                   aligned_head_dim);
 
    /* Step 6: Output projection */
    /* WO projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t attn_q8_bytes = ((((H * head_dim) + 255) / 256) * 292);
    uint8_t attn_q8[attn_q8_bytes];
    quantize_row_q8_k(attn_token, attn_q8, H * head_dim);
    gemv_q4_k_q8_k(proj_tmp, WO, attn_q8, aligned_embed_dim, H * head_dim);
 
    /* Step 7: Residual add */
    qwen2_0_5b_decode_residual_add_token_major(input, proj_tmp, residual1, 1, aligned_embed_dim);
 
    /* Step 8: RMSNorm before MLP */
    rmsnorm_forward(residual1,
                    ln2_gamma,
                    ln2_out,
                    NULL,
                    1,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    /* Step 9: MLP (SwiGLU) */
    /* Gate+Up projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t ln2_q8_bytes = ((((aligned_embed_dim) + 255) / 256) * 292);
    uint8_t ln2_q8[ln2_q8_bytes];
    quantize_row_q8_k(ln2_out, ln2_q8, aligned_embed_dim);
    gemv_q4_k_q8_k(fc1_out, W1, ln2_q8, 2 * aligned_intermediate_dim, aligned_embed_dim);
 
    /* SwiGLU activation */
    swiglu_forward(fc1_out, swiglu_out, 1, aligned_intermediate_dim);
 
    /* Down projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t swiglu_q8_bytes = ((((aligned_intermediate_dim) + 255) / 256) * 292);
    uint8_t swiglu_q8[swiglu_q8_bytes];
    quantize_row_q8_k(swiglu_out, swiglu_q8, aligned_intermediate_dim);
    gemv_q4_k_q8_k(mlp_out, W2, swiglu_q8, aligned_embed_dim, aligned_intermediate_dim);
 
    /* Step 10: Final residual add */
    qwen2_0_5b_decode_residual_add_token_major(residual1, mlp_out, output, 1, aligned_embed_dim);
}

References attention_forward_decode_head_major_gqa_flash(), CK_DT_Q4_K, ck_dtype_row_bytes(), gemv_q4_k_q8_k(), QWEN2_0_5B_DECODELayerOffsets::k, QWEN2_0_5B_DECODELayerOffsets::ln1_gamma, QWEN2_0_5B_DECODELayerOffsets::ln1_out, QWEN2_0_5B_DECODELayerOffsets::ln2_gamma, QWEN2_0_5B_DECODELayerOffsets::ln2_out, QWEN2_0_5B_DECODELayerOffsets::mlp_out, QWEN2_0_5B_DECODELayerOffsets::output, QWEN2_0_5B_DECODELayerOffsets::proj_scratch, QWEN2_0_5B_DECODELayerOffsets::proj_tmp, quantize_row_q8_k(), QWEN2_0_5B_DECODE_EMBED_DIM, QWEN2_0_5B_DECODE_GLOBALS, QWEN2_0_5B_DECODE_HEAD_DIM, QWEN2_0_5B_DECODE_LAYERS, QWEN2_0_5B_DECODE_NUM_HEADS, QWEN2_0_5B_DECODE_NUM_KV_HEADS, QWEN2_0_5B_DECODE_PTR, qwen2_0_5b_decode_residual_add_token_major(), QWEN2_0_5B_DECODELayerOffsets::residual1, rmsnorm_forward(), QWEN2_0_5B_DECODEGlobalOffsets::rope_cos_cache, rope_forward(), QWEN2_0_5B_DECODEGlobalOffsets::rope_sin_cache, swiglu_forward(), QWEN2_0_5B_DECODELayerOffsets::v, QWEN2_0_5B_DECODELayerOffsets::w1, QWEN2_0_5B_DECODELayerOffsets::w2, QWEN2_0_5B_DECODELayerOffsets::wk, QWEN2_0_5B_DECODELayerOffsets::wo, QWEN2_0_5B_DECODELayerOffsets::wq, and QWEN2_0_5B_DECODELayerOffsets::wv.

Referenced by qwen2_0_5b_decode_decode_token().

qwen2_0_5b_decode_layer_18_prefill()

static void qwen2_0_5b_decode_layer_18_prefill ( QWEN2_0_5B_DECODEModel *  model, int  num_tokens, int  aligned_embed_dim, int  aligned_head_dim, int  aligned_intermediate_dim, int  aligned_context_window  )

static

Definition at line 3163 of file v6.5/test_generated/qwen2_int8.c.

  {
    const QWEN2_0_5B_DECODELayerOffsets *L = &QWEN2_0_5B_DECODE_LAYERS[18];
 
    float *input = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[17].output);
    float *ln1_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln1_gamma);
    float *ln1_out = QWEN2_0_5B_DECODE_PTR(model, L->ln1_out);
    float *ln2_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln2_gamma);
    float *ln2_out = QWEN2_0_5B_DECODE_PTR(model, L->ln2_out);
    float *q = QWEN2_0_5B_DECODE_PTR(model, L->q);
    float *k = QWEN2_0_5B_DECODE_PTR(model, L->k);
    float *v = QWEN2_0_5B_DECODE_PTR(model, L->v);
    float *attn_out = QWEN2_0_5B_DECODE_PTR(model, L->attn_out);
    float *proj_tmp = QWEN2_0_5B_DECODE_PTR(model, L->proj_tmp);
    float *proj_scratch = QWEN2_0_5B_DECODE_PTR(model, L->proj_scratch);
    float *residual1 = QWEN2_0_5B_DECODE_PTR(model, L->residual1);
    float *fc1_out = QWEN2_0_5B_DECODE_PTR(model, L->fc1_out);
    float *swiglu_out = QWEN2_0_5B_DECODE_PTR(model, L->swiglu_out);
    float *mlp_out = QWEN2_0_5B_DECODE_PTR(model, L->mlp_out);
    float *output = QWEN2_0_5B_DECODE_PTR(model, L->output);
 
    const void *WQ = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wq);
    const void *WK = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wk);
    const void *WV = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wv);
    const void *WO = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wo);
    const void *W1 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w1);
    const void *W2 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w2);
    const float *BQ = NULL;
    const float *BK = NULL;
    const float *BV = NULL;
    const float *BO = NULL;
    const float *B1 = NULL;
    const float *B2 = NULL;
 
    float *rope_cos = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_cos_cache);
    float *rope_sin = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_sin_cache);
 
    const int H = QWEN2_0_5B_DECODE_NUM_HEADS;
    const int H_kv = QWEN2_0_5B_DECODE_NUM_KV_HEADS;
    const int head_dim = QWEN2_0_5B_DECODE_HEAD_DIM;
    const size_t head_w_elems = (size_t)aligned_head_dim * (size_t)aligned_embed_dim;
    const size_t q_head_stride = (size_t)num_tokens * (size_t)aligned_head_dim;
    const size_t kv_head_stride = (size_t)aligned_context_window * (size_t)aligned_head_dim;
 
    /* RMSNorm before attention */
    rmsnorm_forward(input,
                    ln1_gamma,
                    ln1_out,
                    NULL,
                    num_tokens,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    /* Q projection (head-major) */
    const size_t wq_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, head_w_elems);
    const uint8_t *WQ_bytes = (const uint8_t *)WQ;
    for (int h = 0; h < H; ++h) {
        const void *wq_h = (const void *)(WQ_bytes + (size_t)h * wq_head_bytes);
        const float *bq_h = BQ ? (BQ + (size_t)h * (size_t)aligned_head_dim) : NULL;
        float *q_h = q + (size_t)h * q_head_stride;
        gemm_nt_q4_k(ln1_out, wq_h, bq_h, q_h, num_tokens, aligned_head_dim, aligned_embed_dim);
    }
 
    /* K projection (head-major) */
    const size_t wk_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, head_w_elems);
    const uint8_t *WK_bytes = (const uint8_t *)WK;
    for (int h = 0; h < H_kv; ++h) {
        const void *wk_h = (const void *)(WK_bytes + (size_t)h * wk_head_bytes);
        const float *bk_h = BK ? (BK + (size_t)h * (size_t)aligned_head_dim) : NULL;
        float *k_h = k + (size_t)h * kv_head_stride;
        gemm_nt_q4_k(ln1_out, wk_h, bk_h, k_h, num_tokens, aligned_head_dim, aligned_embed_dim);
    }
 
    /* V projection (head-major) */
    const size_t wv_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, head_w_elems);
    const uint8_t *WV_bytes = (const uint8_t *)WV;
    for (int h = 0; h < H_kv; ++h) {
        const void *wv_h = (const void *)(WV_bytes + (size_t)h * wv_head_bytes);
        const float *bv_h = BV ? (BV + (size_t)h * (size_t)aligned_head_dim) : NULL;
        float *v_h = v + (size_t)h * kv_head_stride;
        gemm_nt_q4_k(ln1_out, wv_h, bv_h, v_h, num_tokens, aligned_head_dim, aligned_embed_dim);
    }
 
    /* RoPE */
    rope_forward_qk_strided(q,
                            k,
                            rope_cos,
                            rope_sin,
                            H,
                            H_kv,
                            num_tokens,
                            head_dim,
                            aligned_head_dim,
                            0,
                            num_tokens,
                            aligned_context_window);
 
    /* Attention (prefill, causal) */
    attention_forward_causal_head_major_gqa_flash_strided(q,
                                                           k,
                                                           v,
                                                           attn_out,
                                                           H,
                                                           H_kv,
                                                           num_tokens,
                                                           head_dim,
                                                           aligned_head_dim,
                                                           aligned_context_window);
 
    /* Output projection (flatten head-major to token-major) */
    const int K = H * aligned_head_dim;
    if (K != aligned_embed_dim) {
        return;
    }
    const float *proj_in = attn_out;
    if (H > 1) {
        if (!proj_scratch) {
            return;
        }
        for (int t = 0; t < num_tokens; ++t) {
            float *dst = proj_scratch + (size_t)t * (size_t)aligned_embed_dim;
            for (int h = 0; h < H; ++h) {
                const float *src = attn_out + (size_t)h * q_head_stride + (size_t)t * (size_t)aligned_head_dim;
                memcpy(dst + (size_t)h * (size_t)aligned_head_dim,
                       src,
                       (size_t)aligned_head_dim * sizeof(float));
            }
        }
        proj_in = proj_scratch;
    }
    gemm_nt_q4_k(proj_in, WO, BO, proj_tmp, num_tokens, aligned_embed_dim, K);
 
    /* Residual add */
    qwen2_0_5b_decode_residual_add_token_major(input, proj_tmp, residual1, num_tokens, aligned_embed_dim);
 
    /* RMSNorm before MLP */
    rmsnorm_forward(residual1,
                    ln2_gamma,
                    ln2_out,
                    NULL,
                    num_tokens,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    /* MLP (SwiGLU) */
    gemm_nt_q4_k(ln2_out, W1, B1, fc1_out, num_tokens, 2 * aligned_intermediate_dim, aligned_embed_dim);
    swiglu_forward(fc1_out, swiglu_out, num_tokens, aligned_intermediate_dim);
    gemm_nt_q4_k(swiglu_out, W2, B2, mlp_out, num_tokens, aligned_embed_dim, aligned_intermediate_dim);
 
    /* Final residual add */
    qwen2_0_5b_decode_residual_add_token_major(residual1, mlp_out, output, num_tokens, aligned_embed_dim);
}

References attention_forward_causal_head_major_gqa_flash_strided(), CK_DT_Q4_K, ck_dtype_row_bytes(), gemm_nt_q4_k(), QWEN2_0_5B_DECODELayerOffsets::k, QWEN2_0_5B_DECODELayerOffsets::ln1_gamma, QWEN2_0_5B_DECODELayerOffsets::ln1_out, QWEN2_0_5B_DECODELayerOffsets::ln2_gamma, QWEN2_0_5B_DECODELayerOffsets::ln2_out, QWEN2_0_5B_DECODELayerOffsets::mlp_out, QWEN2_0_5B_DECODELayerOffsets::output, QWEN2_0_5B_DECODELayerOffsets::proj_scratch, QWEN2_0_5B_DECODELayerOffsets::proj_tmp, QWEN2_0_5B_DECODELayerOffsets::q, QWEN2_0_5B_DECODE_EMBED_DIM, QWEN2_0_5B_DECODE_GLOBALS, QWEN2_0_5B_DECODE_HEAD_DIM, QWEN2_0_5B_DECODE_LAYERS, QWEN2_0_5B_DECODE_NUM_HEADS, QWEN2_0_5B_DECODE_NUM_KV_HEADS, QWEN2_0_5B_DECODE_PTR, qwen2_0_5b_decode_residual_add_token_major(), QWEN2_0_5B_DECODELayerOffsets::residual1, rmsnorm_forward(), QWEN2_0_5B_DECODEGlobalOffsets::rope_cos_cache, rope_forward_qk_strided(), QWEN2_0_5B_DECODEGlobalOffsets::rope_sin_cache, swiglu_forward(), QWEN2_0_5B_DECODELayerOffsets::v, QWEN2_0_5B_DECODELayerOffsets::w1, QWEN2_0_5B_DECODELayerOffsets::w2, QWEN2_0_5B_DECODELayerOffsets::wk, QWEN2_0_5B_DECODELayerOffsets::wo, QWEN2_0_5B_DECODELayerOffsets::wq, and QWEN2_0_5B_DECODELayerOffsets::wv.

Referenced by qwen2_0_5b_decode_forward_prefill_impl().

qwen2_0_5b_decode_layer_19_decode()

static void qwen2_0_5b_decode_layer_19_decode ( QWEN2_0_5B_DECODEModel *  model, int  token_index, int  aligned_embed_dim, int  aligned_head_dim, int  aligned_intermediate_dim, int  aligned_context_window  )

static

Definition at line 7844 of file v6.5/test_generated/qwen2_int8.c.

  {
    const QWEN2_0_5B_DECODELayerOffsets *L = &QWEN2_0_5B_DECODE_LAYERS[19];
 
    float *input = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[18].output);
 
    float *ln1_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln1_gamma);
    float *ln1_out = QWEN2_0_5B_DECODE_PTR(model, L->ln1_out);
    float *ln2_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln2_gamma);
    float *ln2_out = QWEN2_0_5B_DECODE_PTR(model, L->ln2_out);
    float *k_cache = QWEN2_0_5B_DECODE_PTR(model, L->k);
    float *v_cache = QWEN2_0_5B_DECODE_PTR(model, L->v);
    float *proj_tmp = QWEN2_0_5B_DECODE_PTR(model, L->proj_tmp);
    float *proj_scratch = QWEN2_0_5B_DECODE_PTR(model, L->proj_scratch);
    float *residual1 = QWEN2_0_5B_DECODE_PTR(model, L->residual1);
    float *mlp_out = QWEN2_0_5B_DECODE_PTR(model, L->mlp_out);
    float *output = QWEN2_0_5B_DECODE_PTR(model, L->output);
 
    /* Weights (explicit types for layer 19) */
    const void *WQ = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wq);  /* Q4_K */
    const void *WK = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wk);  /* Q4_K */
    const void *WV = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wv);  /* Q4_K */
    const void *WO = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wo);  /* Q4_K */
    const void *W1 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w1);  /* Q4_K (gate+up) */
    const void *W2 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w2);  /* Q4_K (down) */
 
    float *rope_cos = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_cos_cache);
    float *rope_sin = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_sin_cache);
 
    const int H = QWEN2_0_5B_DECODE_NUM_HEADS;
    const int H_kv = QWEN2_0_5B_DECODE_NUM_KV_HEADS;
    const int head_dim = QWEN2_0_5B_DECODE_HEAD_DIM;
 
    float q_token[H * aligned_head_dim];
    float k_token[H_kv * aligned_head_dim];
    float v_token[H_kv * aligned_head_dim];
    float attn_token[H * aligned_head_dim];
 
    /* Local MLP buffers (avoid layout dependencies for intermediate values) */
    float fc1_out[2 * aligned_intermediate_dim];
    float swiglu_out[aligned_intermediate_dim];
 
    /* Step 1: RMSNorm before attention */
    rmsnorm_forward(input,
                    ln1_gamma,
                    ln1_out,
                    NULL,
                    1,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    const size_t kv_head_stride = (size_t)aligned_context_window * (size_t)aligned_head_dim;
 
    /* Step 2: QKV projection */
    /* Q projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t ln1_q8_bytes = ((((aligned_embed_dim) + 255) / 256) * 292);
    uint8_t ln1_q8[ln1_q8_bytes];
    quantize_row_q8_k(ln1_out, ln1_q8, aligned_embed_dim);
    gemv_q4_k_q8_k(q_token, WQ, ln1_q8, H * head_dim, aligned_embed_dim);
    if (aligned_head_dim > head_dim) {
        for (int h = 0; h < H; ++h) {
            float *q_head = q_token + (size_t)h * (size_t)aligned_head_dim;
            for (int d = head_dim; d < aligned_head_dim; ++d) {
                q_head[d] = 0.0f;
            }
        }
    }
 
    /* K projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k (direct-to-cache) */
    const size_t wk_head_elems = (size_t)aligned_head_dim * (size_t)aligned_embed_dim;
    const size_t wk_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, wk_head_elems);
    const uint8_t *WK_bytes = (const uint8_t *)WK;
    /* ln1_q8 already quantized above */
    for (int h = 0; h < H_kv; ++h) {
        const void *wk_h = (const void *)(WK_bytes + (size_t)h * wk_head_bytes);
        float *k_head = k_cache + (size_t)h * kv_head_stride + (size_t)token_index * (size_t)aligned_head_dim;
        gemv_q4_k_q8_k(k_head, wk_h, ln1_q8, head_dim, aligned_embed_dim);
        for (int d = head_dim; d < aligned_head_dim; ++d) {
            k_head[d] = 0.0f;
        }
    }
 
    /* V projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k (direct-to-cache) */
    const size_t wv_head_elems = (size_t)aligned_head_dim * (size_t)aligned_embed_dim;
    const size_t wv_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, wv_head_elems);
    const uint8_t *WV_bytes = (const uint8_t *)WV;
    /* ln1_q8 already quantized above */
    for (int h = 0; h < H_kv; ++h) {
        const void *wv_h = (const void *)(WV_bytes + (size_t)h * wv_head_bytes);
        float *v_head = v_cache + (size_t)h * kv_head_stride + (size_t)token_index * (size_t)aligned_head_dim;
        gemv_q4_k_q8_k(v_head, wv_h, ln1_q8, head_dim, aligned_embed_dim);
        for (int d = head_dim; d < aligned_head_dim; ++d) {
            v_head[d] = 0.0f;
        }
    }
 
    /* Step 3: RoPE */
    rope_forward(q_token,
                 rope_cos,
                 rope_sin,
                 H,
                 1,
                 head_dim,
                 aligned_head_dim,
                 token_index);
    for (int h = 0; h < H_kv; ++h) {
        float *k_head = k_cache + (size_t)h * kv_head_stride + (size_t)token_index * (size_t)aligned_head_dim;
        rope_forward(k_head,
                     rope_cos,
                     rope_sin,
                     1,
                     1,
                     head_dim,
                     aligned_head_dim,
                     token_index);
    }
 
    /* Step 4: KV cache write (direct-to-cache) */
 
    /* Step 5: Attention (decode, flash) */
    attention_forward_decode_head_major_gqa_flash(q_token,
                                                   k_cache,
                                                   v_cache,
                                                   attn_token,
                                                   H,
                                                   H_kv,
                                                   token_index + 1,
                                                   aligned_context_window,
                                                   head_dim,
                                                   aligned_head_dim);
 
    /* Step 6: Output projection */
    /* WO projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t attn_q8_bytes = ((((H * head_dim) + 255) / 256) * 292);
    uint8_t attn_q8[attn_q8_bytes];
    quantize_row_q8_k(attn_token, attn_q8, H * head_dim);
    gemv_q4_k_q8_k(proj_tmp, WO, attn_q8, aligned_embed_dim, H * head_dim);
 
    /* Step 7: Residual add */
    qwen2_0_5b_decode_residual_add_token_major(input, proj_tmp, residual1, 1, aligned_embed_dim);
 
    /* Step 8: RMSNorm before MLP */
    rmsnorm_forward(residual1,
                    ln2_gamma,
                    ln2_out,
                    NULL,
                    1,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    /* Step 9: MLP (SwiGLU) */
    /* Gate+Up projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t ln2_q8_bytes = ((((aligned_embed_dim) + 255) / 256) * 292);
    uint8_t ln2_q8[ln2_q8_bytes];
    quantize_row_q8_k(ln2_out, ln2_q8, aligned_embed_dim);
    gemv_q4_k_q8_k(fc1_out, W1, ln2_q8, 2 * aligned_intermediate_dim, aligned_embed_dim);
 
    /* SwiGLU activation */
    swiglu_forward(fc1_out, swiglu_out, 1, aligned_intermediate_dim);
 
    /* Down projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t swiglu_q8_bytes = ((((aligned_intermediate_dim) + 255) / 256) * 292);
    uint8_t swiglu_q8[swiglu_q8_bytes];
    quantize_row_q8_k(swiglu_out, swiglu_q8, aligned_intermediate_dim);
    gemv_q4_k_q8_k(mlp_out, W2, swiglu_q8, aligned_embed_dim, aligned_intermediate_dim);
 
    /* Step 10: Final residual add */
    qwen2_0_5b_decode_residual_add_token_major(residual1, mlp_out, output, 1, aligned_embed_dim);
}

References attention_forward_decode_head_major_gqa_flash(), CK_DT_Q4_K, ck_dtype_row_bytes(), gemv_q4_k_q8_k(), QWEN2_0_5B_DECODELayerOffsets::k, QWEN2_0_5B_DECODELayerOffsets::ln1_gamma, QWEN2_0_5B_DECODELayerOffsets::ln1_out, QWEN2_0_5B_DECODELayerOffsets::ln2_gamma, QWEN2_0_5B_DECODELayerOffsets::ln2_out, QWEN2_0_5B_DECODELayerOffsets::mlp_out, QWEN2_0_5B_DECODELayerOffsets::output, QWEN2_0_5B_DECODELayerOffsets::proj_scratch, QWEN2_0_5B_DECODELayerOffsets::proj_tmp, quantize_row_q8_k(), QWEN2_0_5B_DECODE_EMBED_DIM, QWEN2_0_5B_DECODE_GLOBALS, QWEN2_0_5B_DECODE_HEAD_DIM, QWEN2_0_5B_DECODE_LAYERS, QWEN2_0_5B_DECODE_NUM_HEADS, QWEN2_0_5B_DECODE_NUM_KV_HEADS, QWEN2_0_5B_DECODE_PTR, qwen2_0_5b_decode_residual_add_token_major(), QWEN2_0_5B_DECODELayerOffsets::residual1, rmsnorm_forward(), QWEN2_0_5B_DECODEGlobalOffsets::rope_cos_cache, rope_forward(), QWEN2_0_5B_DECODEGlobalOffsets::rope_sin_cache, swiglu_forward(), QWEN2_0_5B_DECODELayerOffsets::v, QWEN2_0_5B_DECODELayerOffsets::w1, QWEN2_0_5B_DECODELayerOffsets::w2, QWEN2_0_5B_DECODELayerOffsets::wk, QWEN2_0_5B_DECODELayerOffsets::wo, QWEN2_0_5B_DECODELayerOffsets::wq, and QWEN2_0_5B_DECODELayerOffsets::wv.

Referenced by qwen2_0_5b_decode_decode_token().

qwen2_0_5b_decode_layer_19_prefill()

static void qwen2_0_5b_decode_layer_19_prefill ( QWEN2_0_5B_DECODEModel *  model, int  num_tokens, int  aligned_embed_dim, int  aligned_head_dim, int  aligned_intermediate_dim, int  aligned_context_window  )

static

Definition at line 3327 of file v6.5/test_generated/qwen2_int8.c.

  {
    const QWEN2_0_5B_DECODELayerOffsets *L = &QWEN2_0_5B_DECODE_LAYERS[19];
 
    float *input = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[18].output);
    float *ln1_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln1_gamma);
    float *ln1_out = QWEN2_0_5B_DECODE_PTR(model, L->ln1_out);
    float *ln2_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln2_gamma);
    float *ln2_out = QWEN2_0_5B_DECODE_PTR(model, L->ln2_out);
    float *q = QWEN2_0_5B_DECODE_PTR(model, L->q);
    float *k = QWEN2_0_5B_DECODE_PTR(model, L->k);
    float *v = QWEN2_0_5B_DECODE_PTR(model, L->v);
    float *attn_out = QWEN2_0_5B_DECODE_PTR(model, L->attn_out);
    float *proj_tmp = QWEN2_0_5B_DECODE_PTR(model, L->proj_tmp);
    float *proj_scratch = QWEN2_0_5B_DECODE_PTR(model, L->proj_scratch);
    float *residual1 = QWEN2_0_5B_DECODE_PTR(model, L->residual1);
    float *fc1_out = QWEN2_0_5B_DECODE_PTR(model, L->fc1_out);
    float *swiglu_out = QWEN2_0_5B_DECODE_PTR(model, L->swiglu_out);
    float *mlp_out = QWEN2_0_5B_DECODE_PTR(model, L->mlp_out);
    float *output = QWEN2_0_5B_DECODE_PTR(model, L->output);
 
    const void *WQ = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wq);
    const void *WK = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wk);
    const void *WV = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wv);
    const void *WO = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wo);
    const void *W1 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w1);
    const void *W2 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w2);
    const float *BQ = NULL;
    const float *BK = NULL;
    const float *BV = NULL;
    const float *BO = NULL;
    const float *B1 = NULL;
    const float *B2 = NULL;
 
    float *rope_cos = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_cos_cache);
    float *rope_sin = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_sin_cache);
 
    const int H = QWEN2_0_5B_DECODE_NUM_HEADS;
    const int H_kv = QWEN2_0_5B_DECODE_NUM_KV_HEADS;
    const int head_dim = QWEN2_0_5B_DECODE_HEAD_DIM;
    const size_t head_w_elems = (size_t)aligned_head_dim * (size_t)aligned_embed_dim;
    const size_t q_head_stride = (size_t)num_tokens * (size_t)aligned_head_dim;
    const size_t kv_head_stride = (size_t)aligned_context_window * (size_t)aligned_head_dim;
 
    /* RMSNorm before attention */
    rmsnorm_forward(input,
                    ln1_gamma,
                    ln1_out,
                    NULL,
                    num_tokens,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    /* Q projection (head-major) */
    const size_t wq_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, head_w_elems);
    const uint8_t *WQ_bytes = (const uint8_t *)WQ;
    for (int h = 0; h < H; ++h) {
        const void *wq_h = (const void *)(WQ_bytes + (size_t)h * wq_head_bytes);
        const float *bq_h = BQ ? (BQ + (size_t)h * (size_t)aligned_head_dim) : NULL;
        float *q_h = q + (size_t)h * q_head_stride;
        gemm_nt_q4_k(ln1_out, wq_h, bq_h, q_h, num_tokens, aligned_head_dim, aligned_embed_dim);
    }
 
    /* K projection (head-major) */
    const size_t wk_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, head_w_elems);
    const uint8_t *WK_bytes = (const uint8_t *)WK;
    for (int h = 0; h < H_kv; ++h) {
        const void *wk_h = (const void *)(WK_bytes + (size_t)h * wk_head_bytes);
        const float *bk_h = BK ? (BK + (size_t)h * (size_t)aligned_head_dim) : NULL;
        float *k_h = k + (size_t)h * kv_head_stride;
        gemm_nt_q4_k(ln1_out, wk_h, bk_h, k_h, num_tokens, aligned_head_dim, aligned_embed_dim);
    }
 
    /* V projection (head-major) */
    const size_t wv_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, head_w_elems);
    const uint8_t *WV_bytes = (const uint8_t *)WV;
    for (int h = 0; h < H_kv; ++h) {
        const void *wv_h = (const void *)(WV_bytes + (size_t)h * wv_head_bytes);
        const float *bv_h = BV ? (BV + (size_t)h * (size_t)aligned_head_dim) : NULL;
        float *v_h = v + (size_t)h * kv_head_stride;
        gemm_nt_q4_k(ln1_out, wv_h, bv_h, v_h, num_tokens, aligned_head_dim, aligned_embed_dim);
    }
 
    /* RoPE */
    rope_forward_qk_strided(q,
                            k,
                            rope_cos,
                            rope_sin,
                            H,
                            H_kv,
                            num_tokens,
                            head_dim,
                            aligned_head_dim,
                            0,
                            num_tokens,
                            aligned_context_window);
 
    /* Attention (prefill, causal) */
    attention_forward_causal_head_major_gqa_flash_strided(q,
                                                           k,
                                                           v,
                                                           attn_out,
                                                           H,
                                                           H_kv,
                                                           num_tokens,
                                                           head_dim,
                                                           aligned_head_dim,
                                                           aligned_context_window);
 
    /* Output projection (flatten head-major to token-major) */
    const int K = H * aligned_head_dim;
    if (K != aligned_embed_dim) {
        return;
    }
    const float *proj_in = attn_out;
    if (H > 1) {
        if (!proj_scratch) {
            return;
        }
        for (int t = 0; t < num_tokens; ++t) {
            float *dst = proj_scratch + (size_t)t * (size_t)aligned_embed_dim;
            for (int h = 0; h < H; ++h) {
                const float *src = attn_out + (size_t)h * q_head_stride + (size_t)t * (size_t)aligned_head_dim;
                memcpy(dst + (size_t)h * (size_t)aligned_head_dim,
                       src,
                       (size_t)aligned_head_dim * sizeof(float));
            }
        }
        proj_in = proj_scratch;
    }
    gemm_nt_q4_k(proj_in, WO, BO, proj_tmp, num_tokens, aligned_embed_dim, K);
 
    /* Residual add */
    qwen2_0_5b_decode_residual_add_token_major(input, proj_tmp, residual1, num_tokens, aligned_embed_dim);
 
    /* RMSNorm before MLP */
    rmsnorm_forward(residual1,
                    ln2_gamma,
                    ln2_out,
                    NULL,
                    num_tokens,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    /* MLP (SwiGLU) */
    gemm_nt_q4_k(ln2_out, W1, B1, fc1_out, num_tokens, 2 * aligned_intermediate_dim, aligned_embed_dim);
    swiglu_forward(fc1_out, swiglu_out, num_tokens, aligned_intermediate_dim);
    gemm_nt_q4_k(swiglu_out, W2, B2, mlp_out, num_tokens, aligned_embed_dim, aligned_intermediate_dim);
 
    /* Final residual add */
    qwen2_0_5b_decode_residual_add_token_major(residual1, mlp_out, output, num_tokens, aligned_embed_dim);
}

References attention_forward_causal_head_major_gqa_flash_strided(), CK_DT_Q4_K, ck_dtype_row_bytes(), gemm_nt_q4_k(), QWEN2_0_5B_DECODELayerOffsets::k, QWEN2_0_5B_DECODELayerOffsets::ln1_gamma, QWEN2_0_5B_DECODELayerOffsets::ln1_out, QWEN2_0_5B_DECODELayerOffsets::ln2_gamma, QWEN2_0_5B_DECODELayerOffsets::ln2_out, QWEN2_0_5B_DECODELayerOffsets::mlp_out, QWEN2_0_5B_DECODELayerOffsets::output, QWEN2_0_5B_DECODELayerOffsets::proj_scratch, QWEN2_0_5B_DECODELayerOffsets::proj_tmp, QWEN2_0_5B_DECODELayerOffsets::q, QWEN2_0_5B_DECODE_EMBED_DIM, QWEN2_0_5B_DECODE_GLOBALS, QWEN2_0_5B_DECODE_HEAD_DIM, QWEN2_0_5B_DECODE_LAYERS, QWEN2_0_5B_DECODE_NUM_HEADS, QWEN2_0_5B_DECODE_NUM_KV_HEADS, QWEN2_0_5B_DECODE_PTR, qwen2_0_5b_decode_residual_add_token_major(), QWEN2_0_5B_DECODELayerOffsets::residual1, rmsnorm_forward(), QWEN2_0_5B_DECODEGlobalOffsets::rope_cos_cache, rope_forward_qk_strided(), QWEN2_0_5B_DECODEGlobalOffsets::rope_sin_cache, swiglu_forward(), QWEN2_0_5B_DECODELayerOffsets::v, QWEN2_0_5B_DECODELayerOffsets::w1, QWEN2_0_5B_DECODELayerOffsets::w2, QWEN2_0_5B_DECODELayerOffsets::wk, QWEN2_0_5B_DECODELayerOffsets::wo, QWEN2_0_5B_DECODELayerOffsets::wq, and QWEN2_0_5B_DECODELayerOffsets::wv.

Referenced by qwen2_0_5b_decode_forward_prefill_impl().

qwen2_0_5b_decode_layer_1_decode()

static void qwen2_0_5b_decode_layer_1_decode ( QWEN2_0_5B_DECODEModel *  model, int  token_index, int  aligned_embed_dim, int  aligned_head_dim, int  aligned_intermediate_dim, int  aligned_context_window  )

static

Definition at line 4586 of file v6.5/test_generated/qwen2_int8.c.

  {
    const QWEN2_0_5B_DECODELayerOffsets *L = &QWEN2_0_5B_DECODE_LAYERS[1];
 
    float *input = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[0].output);
 
    float *ln1_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln1_gamma);
    float *ln1_out = QWEN2_0_5B_DECODE_PTR(model, L->ln1_out);
    float *ln2_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln2_gamma);
    float *ln2_out = QWEN2_0_5B_DECODE_PTR(model, L->ln2_out);
    float *k_cache = QWEN2_0_5B_DECODE_PTR(model, L->k);
    float *v_cache = QWEN2_0_5B_DECODE_PTR(model, L->v);
    float *proj_tmp = QWEN2_0_5B_DECODE_PTR(model, L->proj_tmp);
    float *proj_scratch = QWEN2_0_5B_DECODE_PTR(model, L->proj_scratch);
    float *residual1 = QWEN2_0_5B_DECODE_PTR(model, L->residual1);
    float *mlp_out = QWEN2_0_5B_DECODE_PTR(model, L->mlp_out);
    float *output = QWEN2_0_5B_DECODE_PTR(model, L->output);
 
    /* Weights (explicit types for layer 1) */
    const void *WQ = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wq);  /* Q4_K */
    const void *WK = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wk);  /* Q4_K */
    const void *WV = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wv);  /* Q4_K */
    const void *WO = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wo);  /* Q4_K */
    const void *W1 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w1);  /* Q4_K (gate+up) */
    const void *W2 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w2);  /* Q4_K (down) */
 
    float *rope_cos = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_cos_cache);
    float *rope_sin = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_sin_cache);
 
    const int H = QWEN2_0_5B_DECODE_NUM_HEADS;
    const int H_kv = QWEN2_0_5B_DECODE_NUM_KV_HEADS;
    const int head_dim = QWEN2_0_5B_DECODE_HEAD_DIM;
 
    float q_token[H * aligned_head_dim];
    float k_token[H_kv * aligned_head_dim];
    float v_token[H_kv * aligned_head_dim];
    float attn_token[H * aligned_head_dim];
 
    /* Local MLP buffers (avoid layout dependencies for intermediate values) */
    float fc1_out[2 * aligned_intermediate_dim];
    float swiglu_out[aligned_intermediate_dim];
 
    /* Step 1: RMSNorm before attention */
    rmsnorm_forward(input,
                    ln1_gamma,
                    ln1_out,
                    NULL,
                    1,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    const size_t kv_head_stride = (size_t)aligned_context_window * (size_t)aligned_head_dim;
 
    /* Step 2: QKV projection */
    /* Q projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t ln1_q8_bytes = ((((aligned_embed_dim) + 255) / 256) * 292);
    uint8_t ln1_q8[ln1_q8_bytes];
    quantize_row_q8_k(ln1_out, ln1_q8, aligned_embed_dim);
    gemv_q4_k_q8_k(q_token, WQ, ln1_q8, H * head_dim, aligned_embed_dim);
    if (aligned_head_dim > head_dim) {
        for (int h = 0; h < H; ++h) {
            float *q_head = q_token + (size_t)h * (size_t)aligned_head_dim;
            for (int d = head_dim; d < aligned_head_dim; ++d) {
                q_head[d] = 0.0f;
            }
        }
    }
 
    /* K projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k (direct-to-cache) */
    const size_t wk_head_elems = (size_t)aligned_head_dim * (size_t)aligned_embed_dim;
    const size_t wk_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, wk_head_elems);
    const uint8_t *WK_bytes = (const uint8_t *)WK;
    /* ln1_q8 already quantized above */
    for (int h = 0; h < H_kv; ++h) {
        const void *wk_h = (const void *)(WK_bytes + (size_t)h * wk_head_bytes);
        float *k_head = k_cache + (size_t)h * kv_head_stride + (size_t)token_index * (size_t)aligned_head_dim;
        gemv_q4_k_q8_k(k_head, wk_h, ln1_q8, head_dim, aligned_embed_dim);
        for (int d = head_dim; d < aligned_head_dim; ++d) {
            k_head[d] = 0.0f;
        }
    }
 
    /* V projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k (direct-to-cache) */
    const size_t wv_head_elems = (size_t)aligned_head_dim * (size_t)aligned_embed_dim;
    const size_t wv_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, wv_head_elems);
    const uint8_t *WV_bytes = (const uint8_t *)WV;
    /* ln1_q8 already quantized above */
    for (int h = 0; h < H_kv; ++h) {
        const void *wv_h = (const void *)(WV_bytes + (size_t)h * wv_head_bytes);
        float *v_head = v_cache + (size_t)h * kv_head_stride + (size_t)token_index * (size_t)aligned_head_dim;
        gemv_q4_k_q8_k(v_head, wv_h, ln1_q8, head_dim, aligned_embed_dim);
        for (int d = head_dim; d < aligned_head_dim; ++d) {
            v_head[d] = 0.0f;
        }
    }
 
    /* Step 3: RoPE */
    rope_forward(q_token,
                 rope_cos,
                 rope_sin,
                 H,
                 1,
                 head_dim,
                 aligned_head_dim,
                 token_index);
    for (int h = 0; h < H_kv; ++h) {
        float *k_head = k_cache + (size_t)h * kv_head_stride + (size_t)token_index * (size_t)aligned_head_dim;
        rope_forward(k_head,
                     rope_cos,
                     rope_sin,
                     1,
                     1,
                     head_dim,
                     aligned_head_dim,
                     token_index);
    }
 
    /* Step 4: KV cache write (direct-to-cache) */
 
    /* Step 5: Attention (decode, flash) */
    attention_forward_decode_head_major_gqa_flash(q_token,
                                                   k_cache,
                                                   v_cache,
                                                   attn_token,
                                                   H,
                                                   H_kv,
                                                   token_index + 1,
                                                   aligned_context_window,
                                                   head_dim,
                                                   aligned_head_dim);
 
    /* Step 6: Output projection */
    /* WO projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t attn_q8_bytes = ((((H * head_dim) + 255) / 256) * 292);
    uint8_t attn_q8[attn_q8_bytes];
    quantize_row_q8_k(attn_token, attn_q8, H * head_dim);
    gemv_q4_k_q8_k(proj_tmp, WO, attn_q8, aligned_embed_dim, H * head_dim);
 
    /* Step 7: Residual add */
    qwen2_0_5b_decode_residual_add_token_major(input, proj_tmp, residual1, 1, aligned_embed_dim);
 
    /* Step 8: RMSNorm before MLP */
    rmsnorm_forward(residual1,
                    ln2_gamma,
                    ln2_out,
                    NULL,
                    1,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    /* Step 9: MLP (SwiGLU) */
    /* Gate+Up projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t ln2_q8_bytes = ((((aligned_embed_dim) + 255) / 256) * 292);
    uint8_t ln2_q8[ln2_q8_bytes];
    quantize_row_q8_k(ln2_out, ln2_q8, aligned_embed_dim);
    gemv_q4_k_q8_k(fc1_out, W1, ln2_q8, 2 * aligned_intermediate_dim, aligned_embed_dim);
 
    /* SwiGLU activation */
    swiglu_forward(fc1_out, swiglu_out, 1, aligned_intermediate_dim);
 
    /* Down projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t swiglu_q8_bytes = ((((aligned_intermediate_dim) + 255) / 256) * 292);
    uint8_t swiglu_q8[swiglu_q8_bytes];
    quantize_row_q8_k(swiglu_out, swiglu_q8, aligned_intermediate_dim);
    gemv_q4_k_q8_k(mlp_out, W2, swiglu_q8, aligned_embed_dim, aligned_intermediate_dim);
 
    /* Step 10: Final residual add */
    qwen2_0_5b_decode_residual_add_token_major(residual1, mlp_out, output, 1, aligned_embed_dim);
}

References attention_forward_decode_head_major_gqa_flash(), CK_DT_Q4_K, ck_dtype_row_bytes(), gemv_q4_k_q8_k(), QWEN2_0_5B_DECODELayerOffsets::k, QWEN2_0_5B_DECODELayerOffsets::ln1_gamma, QWEN2_0_5B_DECODELayerOffsets::ln1_out, QWEN2_0_5B_DECODELayerOffsets::ln2_gamma, QWEN2_0_5B_DECODELayerOffsets::ln2_out, QWEN2_0_5B_DECODELayerOffsets::mlp_out, QWEN2_0_5B_DECODELayerOffsets::output, QWEN2_0_5B_DECODELayerOffsets::proj_scratch, QWEN2_0_5B_DECODELayerOffsets::proj_tmp, quantize_row_q8_k(), QWEN2_0_5B_DECODE_EMBED_DIM, QWEN2_0_5B_DECODE_GLOBALS, QWEN2_0_5B_DECODE_HEAD_DIM, QWEN2_0_5B_DECODE_LAYERS, QWEN2_0_5B_DECODE_NUM_HEADS, QWEN2_0_5B_DECODE_NUM_KV_HEADS, QWEN2_0_5B_DECODE_PTR, qwen2_0_5b_decode_residual_add_token_major(), QWEN2_0_5B_DECODELayerOffsets::residual1, rmsnorm_forward(), QWEN2_0_5B_DECODEGlobalOffsets::rope_cos_cache, rope_forward(), QWEN2_0_5B_DECODEGlobalOffsets::rope_sin_cache, swiglu_forward(), QWEN2_0_5B_DECODELayerOffsets::v, QWEN2_0_5B_DECODELayerOffsets::w1, QWEN2_0_5B_DECODELayerOffsets::w2, QWEN2_0_5B_DECODELayerOffsets::wk, QWEN2_0_5B_DECODELayerOffsets::wo, QWEN2_0_5B_DECODELayerOffsets::wq, and QWEN2_0_5B_DECODELayerOffsets::wv.

Referenced by qwen2_0_5b_decode_decode_token().

qwen2_0_5b_decode_layer_1_prefill()

static void qwen2_0_5b_decode_layer_1_prefill ( QWEN2_0_5B_DECODEModel *  model, int  num_tokens, int  aligned_embed_dim, int  aligned_head_dim, int  aligned_intermediate_dim, int  aligned_context_window  )

static

Definition at line 375 of file v6.5/test_generated/qwen2_int8.c.

  {
    const QWEN2_0_5B_DECODELayerOffsets *L = &QWEN2_0_5B_DECODE_LAYERS[1];
 
    float *input = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[0].output);
    float *ln1_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln1_gamma);
    float *ln1_out = QWEN2_0_5B_DECODE_PTR(model, L->ln1_out);
    float *ln2_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln2_gamma);
    float *ln2_out = QWEN2_0_5B_DECODE_PTR(model, L->ln2_out);
    float *q = QWEN2_0_5B_DECODE_PTR(model, L->q);
    float *k = QWEN2_0_5B_DECODE_PTR(model, L->k);
    float *v = QWEN2_0_5B_DECODE_PTR(model, L->v);
    float *attn_out = QWEN2_0_5B_DECODE_PTR(model, L->attn_out);
    float *proj_tmp = QWEN2_0_5B_DECODE_PTR(model, L->proj_tmp);
    float *proj_scratch = QWEN2_0_5B_DECODE_PTR(model, L->proj_scratch);
    float *residual1 = QWEN2_0_5B_DECODE_PTR(model, L->residual1);
    float *fc1_out = QWEN2_0_5B_DECODE_PTR(model, L->fc1_out);
    float *swiglu_out = QWEN2_0_5B_DECODE_PTR(model, L->swiglu_out);
    float *mlp_out = QWEN2_0_5B_DECODE_PTR(model, L->mlp_out);
    float *output = QWEN2_0_5B_DECODE_PTR(model, L->output);
 
    const void *WQ = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wq);
    const void *WK = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wk);
    const void *WV = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wv);
    const void *WO = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wo);
    const void *W1 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w1);
    const void *W2 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w2);
    const float *BQ = NULL;
    const float *BK = NULL;
    const float *BV = NULL;
    const float *BO = NULL;
    const float *B1 = NULL;
    const float *B2 = NULL;
 
    float *rope_cos = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_cos_cache);
    float *rope_sin = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_sin_cache);
 
    const int H = QWEN2_0_5B_DECODE_NUM_HEADS;
    const int H_kv = QWEN2_0_5B_DECODE_NUM_KV_HEADS;
    const int head_dim = QWEN2_0_5B_DECODE_HEAD_DIM;
    const size_t head_w_elems = (size_t)aligned_head_dim * (size_t)aligned_embed_dim;
    const size_t q_head_stride = (size_t)num_tokens * (size_t)aligned_head_dim;
    const size_t kv_head_stride = (size_t)aligned_context_window * (size_t)aligned_head_dim;
 
    /* RMSNorm before attention */
    rmsnorm_forward(input,
                    ln1_gamma,
                    ln1_out,
                    NULL,
                    num_tokens,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    /* Q projection (head-major) */
    const size_t wq_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, head_w_elems);
    const uint8_t *WQ_bytes = (const uint8_t *)WQ;
    for (int h = 0; h < H; ++h) {
        const void *wq_h = (const void *)(WQ_bytes + (size_t)h * wq_head_bytes);
        const float *bq_h = BQ ? (BQ + (size_t)h * (size_t)aligned_head_dim) : NULL;
        float *q_h = q + (size_t)h * q_head_stride;
        gemm_nt_q4_k(ln1_out, wq_h, bq_h, q_h, num_tokens, aligned_head_dim, aligned_embed_dim);
    }
 
    /* K projection (head-major) */
    const size_t wk_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, head_w_elems);
    const uint8_t *WK_bytes = (const uint8_t *)WK;
    for (int h = 0; h < H_kv; ++h) {
        const void *wk_h = (const void *)(WK_bytes + (size_t)h * wk_head_bytes);
        const float *bk_h = BK ? (BK + (size_t)h * (size_t)aligned_head_dim) : NULL;
        float *k_h = k + (size_t)h * kv_head_stride;
        gemm_nt_q4_k(ln1_out, wk_h, bk_h, k_h, num_tokens, aligned_head_dim, aligned_embed_dim);
    }
 
    /* V projection (head-major) */
    const size_t wv_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, head_w_elems);
    const uint8_t *WV_bytes = (const uint8_t *)WV;
    for (int h = 0; h < H_kv; ++h) {
        const void *wv_h = (const void *)(WV_bytes + (size_t)h * wv_head_bytes);
        const float *bv_h = BV ? (BV + (size_t)h * (size_t)aligned_head_dim) : NULL;
        float *v_h = v + (size_t)h * kv_head_stride;
        gemm_nt_q4_k(ln1_out, wv_h, bv_h, v_h, num_tokens, aligned_head_dim, aligned_embed_dim);
    }
 
    /* RoPE */
    rope_forward_qk_strided(q,
                            k,
                            rope_cos,
                            rope_sin,
                            H,
                            H_kv,
                            num_tokens,
                            head_dim,
                            aligned_head_dim,
                            0,
                            num_tokens,
                            aligned_context_window);
 
    /* Attention (prefill, causal) */
    attention_forward_causal_head_major_gqa_flash_strided(q,
                                                           k,
                                                           v,
                                                           attn_out,
                                                           H,
                                                           H_kv,
                                                           num_tokens,
                                                           head_dim,
                                                           aligned_head_dim,
                                                           aligned_context_window);
 
    /* Output projection (flatten head-major to token-major) */
    const int K = H * aligned_head_dim;
    if (K != aligned_embed_dim) {
        return;
    }
    const float *proj_in = attn_out;
    if (H > 1) {
        if (!proj_scratch) {
            return;
        }
        for (int t = 0; t < num_tokens; ++t) {
            float *dst = proj_scratch + (size_t)t * (size_t)aligned_embed_dim;
            for (int h = 0; h < H; ++h) {
                const float *src = attn_out + (size_t)h * q_head_stride + (size_t)t * (size_t)aligned_head_dim;
                memcpy(dst + (size_t)h * (size_t)aligned_head_dim,
                       src,
                       (size_t)aligned_head_dim * sizeof(float));
            }
        }
        proj_in = proj_scratch;
    }
    gemm_nt_q4_k(proj_in, WO, BO, proj_tmp, num_tokens, aligned_embed_dim, K);
 
    /* Residual add */
    qwen2_0_5b_decode_residual_add_token_major(input, proj_tmp, residual1, num_tokens, aligned_embed_dim);
 
    /* RMSNorm before MLP */
    rmsnorm_forward(residual1,
                    ln2_gamma,
                    ln2_out,
                    NULL,
                    num_tokens,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    /* MLP (SwiGLU) */
    gemm_nt_q4_k(ln2_out, W1, B1, fc1_out, num_tokens, 2 * aligned_intermediate_dim, aligned_embed_dim);
    swiglu_forward(fc1_out, swiglu_out, num_tokens, aligned_intermediate_dim);
    gemm_nt_q4_k(swiglu_out, W2, B2, mlp_out, num_tokens, aligned_embed_dim, aligned_intermediate_dim);
 
    /* Final residual add */
    qwen2_0_5b_decode_residual_add_token_major(residual1, mlp_out, output, num_tokens, aligned_embed_dim);
}

References attention_forward_causal_head_major_gqa_flash_strided(), CK_DT_Q4_K, ck_dtype_row_bytes(), gemm_nt_q4_k(), QWEN2_0_5B_DECODELayerOffsets::k, QWEN2_0_5B_DECODELayerOffsets::ln1_gamma, QWEN2_0_5B_DECODELayerOffsets::ln1_out, QWEN2_0_5B_DECODELayerOffsets::ln2_gamma, QWEN2_0_5B_DECODELayerOffsets::ln2_out, QWEN2_0_5B_DECODELayerOffsets::mlp_out, QWEN2_0_5B_DECODELayerOffsets::output, QWEN2_0_5B_DECODELayerOffsets::proj_scratch, QWEN2_0_5B_DECODELayerOffsets::proj_tmp, QWEN2_0_5B_DECODELayerOffsets::q, QWEN2_0_5B_DECODE_EMBED_DIM, QWEN2_0_5B_DECODE_GLOBALS, QWEN2_0_5B_DECODE_HEAD_DIM, QWEN2_0_5B_DECODE_LAYERS, QWEN2_0_5B_DECODE_NUM_HEADS, QWEN2_0_5B_DECODE_NUM_KV_HEADS, QWEN2_0_5B_DECODE_PTR, qwen2_0_5b_decode_residual_add_token_major(), QWEN2_0_5B_DECODELayerOffsets::residual1, rmsnorm_forward(), QWEN2_0_5B_DECODEGlobalOffsets::rope_cos_cache, rope_forward_qk_strided(), QWEN2_0_5B_DECODEGlobalOffsets::rope_sin_cache, swiglu_forward(), QWEN2_0_5B_DECODELayerOffsets::v, QWEN2_0_5B_DECODELayerOffsets::w1, QWEN2_0_5B_DECODELayerOffsets::w2, QWEN2_0_5B_DECODELayerOffsets::wk, QWEN2_0_5B_DECODELayerOffsets::wo, QWEN2_0_5B_DECODELayerOffsets::wq, and QWEN2_0_5B_DECODELayerOffsets::wv.

Referenced by qwen2_0_5b_decode_forward_prefill_impl().

qwen2_0_5b_decode_layer_20_decode()

static void qwen2_0_5b_decode_layer_20_decode ( QWEN2_0_5B_DECODEModel *  model, int  token_index, int  aligned_embed_dim, int  aligned_head_dim, int  aligned_intermediate_dim, int  aligned_context_window  )

static

Definition at line 8025 of file v6.5/test_generated/qwen2_int8.c.

  {
    const QWEN2_0_5B_DECODELayerOffsets *L = &QWEN2_0_5B_DECODE_LAYERS[20];
 
    float *input = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[19].output);
 
    float *ln1_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln1_gamma);
    float *ln1_out = QWEN2_0_5B_DECODE_PTR(model, L->ln1_out);
    float *ln2_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln2_gamma);
    float *ln2_out = QWEN2_0_5B_DECODE_PTR(model, L->ln2_out);
    float *k_cache = QWEN2_0_5B_DECODE_PTR(model, L->k);
    float *v_cache = QWEN2_0_5B_DECODE_PTR(model, L->v);
    float *proj_tmp = QWEN2_0_5B_DECODE_PTR(model, L->proj_tmp);
    float *proj_scratch = QWEN2_0_5B_DECODE_PTR(model, L->proj_scratch);
    float *residual1 = QWEN2_0_5B_DECODE_PTR(model, L->residual1);
    float *mlp_out = QWEN2_0_5B_DECODE_PTR(model, L->mlp_out);
    float *output = QWEN2_0_5B_DECODE_PTR(model, L->output);
 
    /* Weights (explicit types for layer 20) */
    const void *WQ = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wq);  /* Q4_K */
    const void *WK = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wk);  /* Q4_K */
    const void *WV = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wv);  /* Q4_K */
    const void *WO = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wo);  /* Q4_K */
    const void *W1 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w1);  /* Q4_K (gate+up) */
    const void *W2 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w2);  /* Q4_K (down) */
 
    float *rope_cos = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_cos_cache);
    float *rope_sin = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_sin_cache);
 
    const int H = QWEN2_0_5B_DECODE_NUM_HEADS;
    const int H_kv = QWEN2_0_5B_DECODE_NUM_KV_HEADS;
    const int head_dim = QWEN2_0_5B_DECODE_HEAD_DIM;
 
    float q_token[H * aligned_head_dim];
    float k_token[H_kv * aligned_head_dim];
    float v_token[H_kv * aligned_head_dim];
    float attn_token[H * aligned_head_dim];
 
    /* Local MLP buffers (avoid layout dependencies for intermediate values) */
    float fc1_out[2 * aligned_intermediate_dim];
    float swiglu_out[aligned_intermediate_dim];
 
    /* Step 1: RMSNorm before attention */
    rmsnorm_forward(input,
                    ln1_gamma,
                    ln1_out,
                    NULL,
                    1,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    const size_t kv_head_stride = (size_t)aligned_context_window * (size_t)aligned_head_dim;
 
    /* Step 2: QKV projection */
    /* Q projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t ln1_q8_bytes = ((((aligned_embed_dim) + 255) / 256) * 292);
    uint8_t ln1_q8[ln1_q8_bytes];
    quantize_row_q8_k(ln1_out, ln1_q8, aligned_embed_dim);
    gemv_q4_k_q8_k(q_token, WQ, ln1_q8, H * head_dim, aligned_embed_dim);
    if (aligned_head_dim > head_dim) {
        for (int h = 0; h < H; ++h) {
            float *q_head = q_token + (size_t)h * (size_t)aligned_head_dim;
            for (int d = head_dim; d < aligned_head_dim; ++d) {
                q_head[d] = 0.0f;
            }
        }
    }
 
    /* K projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k (direct-to-cache) */
    const size_t wk_head_elems = (size_t)aligned_head_dim * (size_t)aligned_embed_dim;
    const size_t wk_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, wk_head_elems);
    const uint8_t *WK_bytes = (const uint8_t *)WK;
    /* ln1_q8 already quantized above */
    for (int h = 0; h < H_kv; ++h) {
        const void *wk_h = (const void *)(WK_bytes + (size_t)h * wk_head_bytes);
        float *k_head = k_cache + (size_t)h * kv_head_stride + (size_t)token_index * (size_t)aligned_head_dim;
        gemv_q4_k_q8_k(k_head, wk_h, ln1_q8, head_dim, aligned_embed_dim);
        for (int d = head_dim; d < aligned_head_dim; ++d) {
            k_head[d] = 0.0f;
        }
    }
 
    /* V projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k (direct-to-cache) */
    const size_t wv_head_elems = (size_t)aligned_head_dim * (size_t)aligned_embed_dim;
    const size_t wv_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, wv_head_elems);
    const uint8_t *WV_bytes = (const uint8_t *)WV;
    /* ln1_q8 already quantized above */
    for (int h = 0; h < H_kv; ++h) {
        const void *wv_h = (const void *)(WV_bytes + (size_t)h * wv_head_bytes);
        float *v_head = v_cache + (size_t)h * kv_head_stride + (size_t)token_index * (size_t)aligned_head_dim;
        gemv_q4_k_q8_k(v_head, wv_h, ln1_q8, head_dim, aligned_embed_dim);
        for (int d = head_dim; d < aligned_head_dim; ++d) {
            v_head[d] = 0.0f;
        }
    }
 
    /* Step 3: RoPE */
    rope_forward(q_token,
                 rope_cos,
                 rope_sin,
                 H,
                 1,
                 head_dim,
                 aligned_head_dim,
                 token_index);
    for (int h = 0; h < H_kv; ++h) {
        float *k_head = k_cache + (size_t)h * kv_head_stride + (size_t)token_index * (size_t)aligned_head_dim;
        rope_forward(k_head,
                     rope_cos,
                     rope_sin,
                     1,
                     1,
                     head_dim,
                     aligned_head_dim,
                     token_index);
    }
 
    /* Step 4: KV cache write (direct-to-cache) */
 
    /* Step 5: Attention (decode, flash) */
    attention_forward_decode_head_major_gqa_flash(q_token,
                                                   k_cache,
                                                   v_cache,
                                                   attn_token,
                                                   H,
                                                   H_kv,
                                                   token_index + 1,
                                                   aligned_context_window,
                                                   head_dim,
                                                   aligned_head_dim);
 
    /* Step 6: Output projection */
    /* WO projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t attn_q8_bytes = ((((H * head_dim) + 255) / 256) * 292);
    uint8_t attn_q8[attn_q8_bytes];
    quantize_row_q8_k(attn_token, attn_q8, H * head_dim);
    gemv_q4_k_q8_k(proj_tmp, WO, attn_q8, aligned_embed_dim, H * head_dim);
 
    /* Step 7: Residual add */
    qwen2_0_5b_decode_residual_add_token_major(input, proj_tmp, residual1, 1, aligned_embed_dim);
 
    /* Step 8: RMSNorm before MLP */
    rmsnorm_forward(residual1,
                    ln2_gamma,
                    ln2_out,
                    NULL,
                    1,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    /* Step 9: MLP (SwiGLU) */
    /* Gate+Up projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t ln2_q8_bytes = ((((aligned_embed_dim) + 255) / 256) * 292);
    uint8_t ln2_q8[ln2_q8_bytes];
    quantize_row_q8_k(ln2_out, ln2_q8, aligned_embed_dim);
    gemv_q4_k_q8_k(fc1_out, W1, ln2_q8, 2 * aligned_intermediate_dim, aligned_embed_dim);
 
    /* SwiGLU activation */
    swiglu_forward(fc1_out, swiglu_out, 1, aligned_intermediate_dim);
 
    /* Down projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t swiglu_q8_bytes = ((((aligned_intermediate_dim) + 255) / 256) * 292);
    uint8_t swiglu_q8[swiglu_q8_bytes];
    quantize_row_q8_k(swiglu_out, swiglu_q8, aligned_intermediate_dim);
    gemv_q4_k_q8_k(mlp_out, W2, swiglu_q8, aligned_embed_dim, aligned_intermediate_dim);
 
    /* Step 10: Final residual add */
    qwen2_0_5b_decode_residual_add_token_major(residual1, mlp_out, output, 1, aligned_embed_dim);
}

References attention_forward_decode_head_major_gqa_flash(), CK_DT_Q4_K, ck_dtype_row_bytes(), gemv_q4_k_q8_k(), QWEN2_0_5B_DECODELayerOffsets::k, QWEN2_0_5B_DECODELayerOffsets::ln1_gamma, QWEN2_0_5B_DECODELayerOffsets::ln1_out, QWEN2_0_5B_DECODELayerOffsets::ln2_gamma, QWEN2_0_5B_DECODELayerOffsets::ln2_out, QWEN2_0_5B_DECODELayerOffsets::mlp_out, QWEN2_0_5B_DECODELayerOffsets::output, QWEN2_0_5B_DECODELayerOffsets::proj_scratch, QWEN2_0_5B_DECODELayerOffsets::proj_tmp, quantize_row_q8_k(), QWEN2_0_5B_DECODE_EMBED_DIM, QWEN2_0_5B_DECODE_GLOBALS, QWEN2_0_5B_DECODE_HEAD_DIM, QWEN2_0_5B_DECODE_LAYERS, QWEN2_0_5B_DECODE_NUM_HEADS, QWEN2_0_5B_DECODE_NUM_KV_HEADS, QWEN2_0_5B_DECODE_PTR, qwen2_0_5b_decode_residual_add_token_major(), QWEN2_0_5B_DECODELayerOffsets::residual1, rmsnorm_forward(), QWEN2_0_5B_DECODEGlobalOffsets::rope_cos_cache, rope_forward(), QWEN2_0_5B_DECODEGlobalOffsets::rope_sin_cache, swiglu_forward(), QWEN2_0_5B_DECODELayerOffsets::v, QWEN2_0_5B_DECODELayerOffsets::w1, QWEN2_0_5B_DECODELayerOffsets::w2, QWEN2_0_5B_DECODELayerOffsets::wk, QWEN2_0_5B_DECODELayerOffsets::wo, QWEN2_0_5B_DECODELayerOffsets::wq, and QWEN2_0_5B_DECODELayerOffsets::wv.

Referenced by qwen2_0_5b_decode_decode_token().

qwen2_0_5b_decode_layer_20_prefill()

static void qwen2_0_5b_decode_layer_20_prefill ( QWEN2_0_5B_DECODEModel *  model, int  num_tokens, int  aligned_embed_dim, int  aligned_head_dim, int  aligned_intermediate_dim, int  aligned_context_window  )

static

Definition at line 3491 of file v6.5/test_generated/qwen2_int8.c.

  {
    const QWEN2_0_5B_DECODELayerOffsets *L = &QWEN2_0_5B_DECODE_LAYERS[20];
 
    float *input = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[19].output);
    float *ln1_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln1_gamma);
    float *ln1_out = QWEN2_0_5B_DECODE_PTR(model, L->ln1_out);
    float *ln2_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln2_gamma);
    float *ln2_out = QWEN2_0_5B_DECODE_PTR(model, L->ln2_out);
    float *q = QWEN2_0_5B_DECODE_PTR(model, L->q);
    float *k = QWEN2_0_5B_DECODE_PTR(model, L->k);
    float *v = QWEN2_0_5B_DECODE_PTR(model, L->v);
    float *attn_out = QWEN2_0_5B_DECODE_PTR(model, L->attn_out);
    float *proj_tmp = QWEN2_0_5B_DECODE_PTR(model, L->proj_tmp);
    float *proj_scratch = QWEN2_0_5B_DECODE_PTR(model, L->proj_scratch);
    float *residual1 = QWEN2_0_5B_DECODE_PTR(model, L->residual1);
    float *fc1_out = QWEN2_0_5B_DECODE_PTR(model, L->fc1_out);
    float *swiglu_out = QWEN2_0_5B_DECODE_PTR(model, L->swiglu_out);
    float *mlp_out = QWEN2_0_5B_DECODE_PTR(model, L->mlp_out);
    float *output = QWEN2_0_5B_DECODE_PTR(model, L->output);
 
    const void *WQ = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wq);
    const void *WK = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wk);
    const void *WV = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wv);
    const void *WO = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wo);
    const void *W1 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w1);
    const void *W2 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w2);
    const float *BQ = NULL;
    const float *BK = NULL;
    const float *BV = NULL;
    const float *BO = NULL;
    const float *B1 = NULL;
    const float *B2 = NULL;
 
    float *rope_cos = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_cos_cache);
    float *rope_sin = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_sin_cache);
 
    const int H = QWEN2_0_5B_DECODE_NUM_HEADS;
    const int H_kv = QWEN2_0_5B_DECODE_NUM_KV_HEADS;
    const int head_dim = QWEN2_0_5B_DECODE_HEAD_DIM;
    const size_t head_w_elems = (size_t)aligned_head_dim * (size_t)aligned_embed_dim;
    const size_t q_head_stride = (size_t)num_tokens * (size_t)aligned_head_dim;
    const size_t kv_head_stride = (size_t)aligned_context_window * (size_t)aligned_head_dim;
 
    /* RMSNorm before attention */
    rmsnorm_forward(input,
                    ln1_gamma,
                    ln1_out,
                    NULL,
                    num_tokens,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    /* Q projection (head-major) */
    const size_t wq_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, head_w_elems);
    const uint8_t *WQ_bytes = (const uint8_t *)WQ;
    for (int h = 0; h < H; ++h) {
        const void *wq_h = (const void *)(WQ_bytes + (size_t)h * wq_head_bytes);
        const float *bq_h = BQ ? (BQ + (size_t)h * (size_t)aligned_head_dim) : NULL;
        float *q_h = q + (size_t)h * q_head_stride;
        gemm_nt_q4_k(ln1_out, wq_h, bq_h, q_h, num_tokens, aligned_head_dim, aligned_embed_dim);
    }
 
    /* K projection (head-major) */
    const size_t wk_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, head_w_elems);
    const uint8_t *WK_bytes = (const uint8_t *)WK;
    for (int h = 0; h < H_kv; ++h) {
        const void *wk_h = (const void *)(WK_bytes + (size_t)h * wk_head_bytes);
        const float *bk_h = BK ? (BK + (size_t)h * (size_t)aligned_head_dim) : NULL;
        float *k_h = k + (size_t)h * kv_head_stride;
        gemm_nt_q4_k(ln1_out, wk_h, bk_h, k_h, num_tokens, aligned_head_dim, aligned_embed_dim);
    }
 
    /* V projection (head-major) */
    const size_t wv_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, head_w_elems);
    const uint8_t *WV_bytes = (const uint8_t *)WV;
    for (int h = 0; h < H_kv; ++h) {
        const void *wv_h = (const void *)(WV_bytes + (size_t)h * wv_head_bytes);
        const float *bv_h = BV ? (BV + (size_t)h * (size_t)aligned_head_dim) : NULL;
        float *v_h = v + (size_t)h * kv_head_stride;
        gemm_nt_q4_k(ln1_out, wv_h, bv_h, v_h, num_tokens, aligned_head_dim, aligned_embed_dim);
    }
 
    /* RoPE */
    rope_forward_qk_strided(q,
                            k,
                            rope_cos,
                            rope_sin,
                            H,
                            H_kv,
                            num_tokens,
                            head_dim,
                            aligned_head_dim,
                            0,
                            num_tokens,
                            aligned_context_window);
 
    /* Attention (prefill, causal) */
    attention_forward_causal_head_major_gqa_flash_strided(q,
                                                           k,
                                                           v,
                                                           attn_out,
                                                           H,
                                                           H_kv,
                                                           num_tokens,
                                                           head_dim,
                                                           aligned_head_dim,
                                                           aligned_context_window);
 
    /* Output projection (flatten head-major to token-major) */
    const int K = H * aligned_head_dim;
    if (K != aligned_embed_dim) {
        return;
    }
    const float *proj_in = attn_out;
    if (H > 1) {
        if (!proj_scratch) {
            return;
        }
        for (int t = 0; t < num_tokens; ++t) {
            float *dst = proj_scratch + (size_t)t * (size_t)aligned_embed_dim;
            for (int h = 0; h < H; ++h) {
                const float *src = attn_out + (size_t)h * q_head_stride + (size_t)t * (size_t)aligned_head_dim;
                memcpy(dst + (size_t)h * (size_t)aligned_head_dim,
                       src,
                       (size_t)aligned_head_dim * sizeof(float));
            }
        }
        proj_in = proj_scratch;
    }
    gemm_nt_q4_k(proj_in, WO, BO, proj_tmp, num_tokens, aligned_embed_dim, K);
 
    /* Residual add */
    qwen2_0_5b_decode_residual_add_token_major(input, proj_tmp, residual1, num_tokens, aligned_embed_dim);
 
    /* RMSNorm before MLP */
    rmsnorm_forward(residual1,
                    ln2_gamma,
                    ln2_out,
                    NULL,
                    num_tokens,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    /* MLP (SwiGLU) */
    gemm_nt_q4_k(ln2_out, W1, B1, fc1_out, num_tokens, 2 * aligned_intermediate_dim, aligned_embed_dim);
    swiglu_forward(fc1_out, swiglu_out, num_tokens, aligned_intermediate_dim);
    gemm_nt_q4_k(swiglu_out, W2, B2, mlp_out, num_tokens, aligned_embed_dim, aligned_intermediate_dim);
 
    /* Final residual add */
    qwen2_0_5b_decode_residual_add_token_major(residual1, mlp_out, output, num_tokens, aligned_embed_dim);
}

References attention_forward_causal_head_major_gqa_flash_strided(), CK_DT_Q4_K, ck_dtype_row_bytes(), gemm_nt_q4_k(), QWEN2_0_5B_DECODELayerOffsets::k, QWEN2_0_5B_DECODELayerOffsets::ln1_gamma, QWEN2_0_5B_DECODELayerOffsets::ln1_out, QWEN2_0_5B_DECODELayerOffsets::ln2_gamma, QWEN2_0_5B_DECODELayerOffsets::ln2_out, QWEN2_0_5B_DECODELayerOffsets::mlp_out, QWEN2_0_5B_DECODELayerOffsets::output, QWEN2_0_5B_DECODELayerOffsets::proj_scratch, QWEN2_0_5B_DECODELayerOffsets::proj_tmp, QWEN2_0_5B_DECODELayerOffsets::q, QWEN2_0_5B_DECODE_EMBED_DIM, QWEN2_0_5B_DECODE_GLOBALS, QWEN2_0_5B_DECODE_HEAD_DIM, QWEN2_0_5B_DECODE_LAYERS, QWEN2_0_5B_DECODE_NUM_HEADS, QWEN2_0_5B_DECODE_NUM_KV_HEADS, QWEN2_0_5B_DECODE_PTR, qwen2_0_5b_decode_residual_add_token_major(), QWEN2_0_5B_DECODELayerOffsets::residual1, rmsnorm_forward(), QWEN2_0_5B_DECODEGlobalOffsets::rope_cos_cache, rope_forward_qk_strided(), QWEN2_0_5B_DECODEGlobalOffsets::rope_sin_cache, swiglu_forward(), QWEN2_0_5B_DECODELayerOffsets::v, QWEN2_0_5B_DECODELayerOffsets::w1, QWEN2_0_5B_DECODELayerOffsets::w2, QWEN2_0_5B_DECODELayerOffsets::wk, QWEN2_0_5B_DECODELayerOffsets::wo, QWEN2_0_5B_DECODELayerOffsets::wq, and QWEN2_0_5B_DECODELayerOffsets::wv.

Referenced by qwen2_0_5b_decode_forward_prefill_impl().

qwen2_0_5b_decode_layer_21_decode()

static void qwen2_0_5b_decode_layer_21_decode ( QWEN2_0_5B_DECODEModel *  model, int  token_index, int  aligned_embed_dim, int  aligned_head_dim, int  aligned_intermediate_dim, int  aligned_context_window  )

static

Definition at line 8206 of file v6.5/test_generated/qwen2_int8.c.

  {
    const QWEN2_0_5B_DECODELayerOffsets *L = &QWEN2_0_5B_DECODE_LAYERS[21];
 
    float *input = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[20].output);
 
    float *ln1_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln1_gamma);
    float *ln1_out = QWEN2_0_5B_DECODE_PTR(model, L->ln1_out);
    float *ln2_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln2_gamma);
    float *ln2_out = QWEN2_0_5B_DECODE_PTR(model, L->ln2_out);
    float *k_cache = QWEN2_0_5B_DECODE_PTR(model, L->k);
    float *v_cache = QWEN2_0_5B_DECODE_PTR(model, L->v);
    float *proj_tmp = QWEN2_0_5B_DECODE_PTR(model, L->proj_tmp);
    float *proj_scratch = QWEN2_0_5B_DECODE_PTR(model, L->proj_scratch);
    float *residual1 = QWEN2_0_5B_DECODE_PTR(model, L->residual1);
    float *mlp_out = QWEN2_0_5B_DECODE_PTR(model, L->mlp_out);
    float *output = QWEN2_0_5B_DECODE_PTR(model, L->output);
 
    /* Weights (explicit types for layer 21) */
    const void *WQ = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wq);  /* Q4_K */
    const void *WK = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wk);  /* Q4_K */
    const void *WV = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wv);  /* Q4_K */
    const void *WO = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wo);  /* Q4_K */
    const void *W1 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w1);  /* Q4_K (gate+up) */
    const void *W2 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w2);  /* Q4_K (down) */
 
    float *rope_cos = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_cos_cache);
    float *rope_sin = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_sin_cache);
 
    const int H = QWEN2_0_5B_DECODE_NUM_HEADS;
    const int H_kv = QWEN2_0_5B_DECODE_NUM_KV_HEADS;
    const int head_dim = QWEN2_0_5B_DECODE_HEAD_DIM;
 
    float q_token[H * aligned_head_dim];
    float k_token[H_kv * aligned_head_dim];
    float v_token[H_kv * aligned_head_dim];
    float attn_token[H * aligned_head_dim];
 
    /* Local MLP buffers (avoid layout dependencies for intermediate values) */
    float fc1_out[2 * aligned_intermediate_dim];
    float swiglu_out[aligned_intermediate_dim];
 
    /* Step 1: RMSNorm before attention */
    rmsnorm_forward(input,
                    ln1_gamma,
                    ln1_out,
                    NULL,
                    1,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    const size_t kv_head_stride = (size_t)aligned_context_window * (size_t)aligned_head_dim;
 
    /* Step 2: QKV projection */
    /* Q projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t ln1_q8_bytes = ((((aligned_embed_dim) + 255) / 256) * 292);
    uint8_t ln1_q8[ln1_q8_bytes];
    quantize_row_q8_k(ln1_out, ln1_q8, aligned_embed_dim);
    gemv_q4_k_q8_k(q_token, WQ, ln1_q8, H * head_dim, aligned_embed_dim);
    if (aligned_head_dim > head_dim) {
        for (int h = 0; h < H; ++h) {
            float *q_head = q_token + (size_t)h * (size_t)aligned_head_dim;
            for (int d = head_dim; d < aligned_head_dim; ++d) {
                q_head[d] = 0.0f;
            }
        }
    }
 
    /* K projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k (direct-to-cache) */
    const size_t wk_head_elems = (size_t)aligned_head_dim * (size_t)aligned_embed_dim;
    const size_t wk_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, wk_head_elems);
    const uint8_t *WK_bytes = (const uint8_t *)WK;
    /* ln1_q8 already quantized above */
    for (int h = 0; h < H_kv; ++h) {
        const void *wk_h = (const void *)(WK_bytes + (size_t)h * wk_head_bytes);
        float *k_head = k_cache + (size_t)h * kv_head_stride + (size_t)token_index * (size_t)aligned_head_dim;
        gemv_q4_k_q8_k(k_head, wk_h, ln1_q8, head_dim, aligned_embed_dim);
        for (int d = head_dim; d < aligned_head_dim; ++d) {
            k_head[d] = 0.0f;
        }
    }
 
    /* V projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k (direct-to-cache) */
    const size_t wv_head_elems = (size_t)aligned_head_dim * (size_t)aligned_embed_dim;
    const size_t wv_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, wv_head_elems);
    const uint8_t *WV_bytes = (const uint8_t *)WV;
    /* ln1_q8 already quantized above */
    for (int h = 0; h < H_kv; ++h) {
        const void *wv_h = (const void *)(WV_bytes + (size_t)h * wv_head_bytes);
        float *v_head = v_cache + (size_t)h * kv_head_stride + (size_t)token_index * (size_t)aligned_head_dim;
        gemv_q4_k_q8_k(v_head, wv_h, ln1_q8, head_dim, aligned_embed_dim);
        for (int d = head_dim; d < aligned_head_dim; ++d) {
            v_head[d] = 0.0f;
        }
    }
 
    /* Step 3: RoPE */
    rope_forward(q_token,
                 rope_cos,
                 rope_sin,
                 H,
                 1,
                 head_dim,
                 aligned_head_dim,
                 token_index);
    for (int h = 0; h < H_kv; ++h) {
        float *k_head = k_cache + (size_t)h * kv_head_stride + (size_t)token_index * (size_t)aligned_head_dim;
        rope_forward(k_head,
                     rope_cos,
                     rope_sin,
                     1,
                     1,
                     head_dim,
                     aligned_head_dim,
                     token_index);
    }
 
    /* Step 4: KV cache write (direct-to-cache) */
 
    /* Step 5: Attention (decode, flash) */
    attention_forward_decode_head_major_gqa_flash(q_token,
                                                   k_cache,
                                                   v_cache,
                                                   attn_token,
                                                   H,
                                                   H_kv,
                                                   token_index + 1,
                                                   aligned_context_window,
                                                   head_dim,
                                                   aligned_head_dim);
 
    /* Step 6: Output projection */
    /* WO projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t attn_q8_bytes = ((((H * head_dim) + 255) / 256) * 292);
    uint8_t attn_q8[attn_q8_bytes];
    quantize_row_q8_k(attn_token, attn_q8, H * head_dim);
    gemv_q4_k_q8_k(proj_tmp, WO, attn_q8, aligned_embed_dim, H * head_dim);
 
    /* Step 7: Residual add */
    qwen2_0_5b_decode_residual_add_token_major(input, proj_tmp, residual1, 1, aligned_embed_dim);
 
    /* Step 8: RMSNorm before MLP */
    rmsnorm_forward(residual1,
                    ln2_gamma,
                    ln2_out,
                    NULL,
                    1,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    /* Step 9: MLP (SwiGLU) */
    /* Gate+Up projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t ln2_q8_bytes = ((((aligned_embed_dim) + 255) / 256) * 292);
    uint8_t ln2_q8[ln2_q8_bytes];
    quantize_row_q8_k(ln2_out, ln2_q8, aligned_embed_dim);
    gemv_q4_k_q8_k(fc1_out, W1, ln2_q8, 2 * aligned_intermediate_dim, aligned_embed_dim);
 
    /* SwiGLU activation */
    swiglu_forward(fc1_out, swiglu_out, 1, aligned_intermediate_dim);
 
    /* Down projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t swiglu_q8_bytes = ((((aligned_intermediate_dim) + 255) / 256) * 292);
    uint8_t swiglu_q8[swiglu_q8_bytes];
    quantize_row_q8_k(swiglu_out, swiglu_q8, aligned_intermediate_dim);
    gemv_q4_k_q8_k(mlp_out, W2, swiglu_q8, aligned_embed_dim, aligned_intermediate_dim);
 
    /* Step 10: Final residual add */
    qwen2_0_5b_decode_residual_add_token_major(residual1, mlp_out, output, 1, aligned_embed_dim);
}

References attention_forward_decode_head_major_gqa_flash(), CK_DT_Q4_K, ck_dtype_row_bytes(), gemv_q4_k_q8_k(), QWEN2_0_5B_DECODELayerOffsets::k, QWEN2_0_5B_DECODELayerOffsets::ln1_gamma, QWEN2_0_5B_DECODELayerOffsets::ln1_out, QWEN2_0_5B_DECODELayerOffsets::ln2_gamma, QWEN2_0_5B_DECODELayerOffsets::ln2_out, QWEN2_0_5B_DECODELayerOffsets::mlp_out, QWEN2_0_5B_DECODELayerOffsets::output, QWEN2_0_5B_DECODELayerOffsets::proj_scratch, QWEN2_0_5B_DECODELayerOffsets::proj_tmp, quantize_row_q8_k(), QWEN2_0_5B_DECODE_EMBED_DIM, QWEN2_0_5B_DECODE_GLOBALS, QWEN2_0_5B_DECODE_HEAD_DIM, QWEN2_0_5B_DECODE_LAYERS, QWEN2_0_5B_DECODE_NUM_HEADS, QWEN2_0_5B_DECODE_NUM_KV_HEADS, QWEN2_0_5B_DECODE_PTR, qwen2_0_5b_decode_residual_add_token_major(), QWEN2_0_5B_DECODELayerOffsets::residual1, rmsnorm_forward(), QWEN2_0_5B_DECODEGlobalOffsets::rope_cos_cache, rope_forward(), QWEN2_0_5B_DECODEGlobalOffsets::rope_sin_cache, swiglu_forward(), QWEN2_0_5B_DECODELayerOffsets::v, QWEN2_0_5B_DECODELayerOffsets::w1, QWEN2_0_5B_DECODELayerOffsets::w2, QWEN2_0_5B_DECODELayerOffsets::wk, QWEN2_0_5B_DECODELayerOffsets::wo, QWEN2_0_5B_DECODELayerOffsets::wq, and QWEN2_0_5B_DECODELayerOffsets::wv.

Referenced by qwen2_0_5b_decode_decode_token().

qwen2_0_5b_decode_layer_21_prefill()

static void qwen2_0_5b_decode_layer_21_prefill ( QWEN2_0_5B_DECODEModel *  model, int  num_tokens, int  aligned_embed_dim, int  aligned_head_dim, int  aligned_intermediate_dim, int  aligned_context_window  )

static

Definition at line 3655 of file v6.5/test_generated/qwen2_int8.c.

  {
    const QWEN2_0_5B_DECODELayerOffsets *L = &QWEN2_0_5B_DECODE_LAYERS[21];
 
    float *input = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[20].output);
    float *ln1_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln1_gamma);
    float *ln1_out = QWEN2_0_5B_DECODE_PTR(model, L->ln1_out);
    float *ln2_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln2_gamma);
    float *ln2_out = QWEN2_0_5B_DECODE_PTR(model, L->ln2_out);
    float *q = QWEN2_0_5B_DECODE_PTR(model, L->q);
    float *k = QWEN2_0_5B_DECODE_PTR(model, L->k);
    float *v = QWEN2_0_5B_DECODE_PTR(model, L->v);
    float *attn_out = QWEN2_0_5B_DECODE_PTR(model, L->attn_out);
    float *proj_tmp = QWEN2_0_5B_DECODE_PTR(model, L->proj_tmp);
    float *proj_scratch = QWEN2_0_5B_DECODE_PTR(model, L->proj_scratch);
    float *residual1 = QWEN2_0_5B_DECODE_PTR(model, L->residual1);
    float *fc1_out = QWEN2_0_5B_DECODE_PTR(model, L->fc1_out);
    float *swiglu_out = QWEN2_0_5B_DECODE_PTR(model, L->swiglu_out);
    float *mlp_out = QWEN2_0_5B_DECODE_PTR(model, L->mlp_out);
    float *output = QWEN2_0_5B_DECODE_PTR(model, L->output);
 
    const void *WQ = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wq);
    const void *WK = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wk);
    const void *WV = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wv);
    const void *WO = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wo);
    const void *W1 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w1);
    const void *W2 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w2);
    const float *BQ = NULL;
    const float *BK = NULL;
    const float *BV = NULL;
    const float *BO = NULL;
    const float *B1 = NULL;
    const float *B2 = NULL;
 
    float *rope_cos = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_cos_cache);
    float *rope_sin = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_sin_cache);
 
    const int H = QWEN2_0_5B_DECODE_NUM_HEADS;
    const int H_kv = QWEN2_0_5B_DECODE_NUM_KV_HEADS;
    const int head_dim = QWEN2_0_5B_DECODE_HEAD_DIM;
    const size_t head_w_elems = (size_t)aligned_head_dim * (size_t)aligned_embed_dim;
    const size_t q_head_stride = (size_t)num_tokens * (size_t)aligned_head_dim;
    const size_t kv_head_stride = (size_t)aligned_context_window * (size_t)aligned_head_dim;
 
    /* RMSNorm before attention */
    rmsnorm_forward(input,
                    ln1_gamma,
                    ln1_out,
                    NULL,
                    num_tokens,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    /* Q projection (head-major) */
    const size_t wq_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, head_w_elems);
    const uint8_t *WQ_bytes = (const uint8_t *)WQ;
    for (int h = 0; h < H; ++h) {
        const void *wq_h = (const void *)(WQ_bytes + (size_t)h * wq_head_bytes);
        const float *bq_h = BQ ? (BQ + (size_t)h * (size_t)aligned_head_dim) : NULL;
        float *q_h = q + (size_t)h * q_head_stride;
        gemm_nt_q4_k(ln1_out, wq_h, bq_h, q_h, num_tokens, aligned_head_dim, aligned_embed_dim);
    }
 
    /* K projection (head-major) */
    const size_t wk_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, head_w_elems);
    const uint8_t *WK_bytes = (const uint8_t *)WK;
    for (int h = 0; h < H_kv; ++h) {
        const void *wk_h = (const void *)(WK_bytes + (size_t)h * wk_head_bytes);
        const float *bk_h = BK ? (BK + (size_t)h * (size_t)aligned_head_dim) : NULL;
        float *k_h = k + (size_t)h * kv_head_stride;
        gemm_nt_q4_k(ln1_out, wk_h, bk_h, k_h, num_tokens, aligned_head_dim, aligned_embed_dim);
    }
 
    /* V projection (head-major) */
    const size_t wv_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, head_w_elems);
    const uint8_t *WV_bytes = (const uint8_t *)WV;
    for (int h = 0; h < H_kv; ++h) {
        const void *wv_h = (const void *)(WV_bytes + (size_t)h * wv_head_bytes);
        const float *bv_h = BV ? (BV + (size_t)h * (size_t)aligned_head_dim) : NULL;
        float *v_h = v + (size_t)h * kv_head_stride;
        gemm_nt_q4_k(ln1_out, wv_h, bv_h, v_h, num_tokens, aligned_head_dim, aligned_embed_dim);
    }
 
    /* RoPE */
    rope_forward_qk_strided(q,
                            k,
                            rope_cos,
                            rope_sin,
                            H,
                            H_kv,
                            num_tokens,
                            head_dim,
                            aligned_head_dim,
                            0,
                            num_tokens,
                            aligned_context_window);
 
    /* Attention (prefill, causal) */
    attention_forward_causal_head_major_gqa_flash_strided(q,
                                                           k,
                                                           v,
                                                           attn_out,
                                                           H,
                                                           H_kv,
                                                           num_tokens,
                                                           head_dim,
                                                           aligned_head_dim,
                                                           aligned_context_window);
 
    /* Output projection (flatten head-major to token-major) */
    const int K = H * aligned_head_dim;
    if (K != aligned_embed_dim) {
        return;
    }
    const float *proj_in = attn_out;
    if (H > 1) {
        if (!proj_scratch) {
            return;
        }
        for (int t = 0; t < num_tokens; ++t) {
            float *dst = proj_scratch + (size_t)t * (size_t)aligned_embed_dim;
            for (int h = 0; h < H; ++h) {
                const float *src = attn_out + (size_t)h * q_head_stride + (size_t)t * (size_t)aligned_head_dim;
                memcpy(dst + (size_t)h * (size_t)aligned_head_dim,
                       src,
                       (size_t)aligned_head_dim * sizeof(float));
            }
        }
        proj_in = proj_scratch;
    }
    gemm_nt_q4_k(proj_in, WO, BO, proj_tmp, num_tokens, aligned_embed_dim, K);
 
    /* Residual add */
    qwen2_0_5b_decode_residual_add_token_major(input, proj_tmp, residual1, num_tokens, aligned_embed_dim);
 
    /* RMSNorm before MLP */
    rmsnorm_forward(residual1,
                    ln2_gamma,
                    ln2_out,
                    NULL,
                    num_tokens,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    /* MLP (SwiGLU) */
    gemm_nt_q4_k(ln2_out, W1, B1, fc1_out, num_tokens, 2 * aligned_intermediate_dim, aligned_embed_dim);
    swiglu_forward(fc1_out, swiglu_out, num_tokens, aligned_intermediate_dim);
    gemm_nt_q4_k(swiglu_out, W2, B2, mlp_out, num_tokens, aligned_embed_dim, aligned_intermediate_dim);
 
    /* Final residual add */
    qwen2_0_5b_decode_residual_add_token_major(residual1, mlp_out, output, num_tokens, aligned_embed_dim);
}

References attention_forward_causal_head_major_gqa_flash_strided(), CK_DT_Q4_K, ck_dtype_row_bytes(), gemm_nt_q4_k(), QWEN2_0_5B_DECODELayerOffsets::k, QWEN2_0_5B_DECODELayerOffsets::ln1_gamma, QWEN2_0_5B_DECODELayerOffsets::ln1_out, QWEN2_0_5B_DECODELayerOffsets::ln2_gamma, QWEN2_0_5B_DECODELayerOffsets::ln2_out, QWEN2_0_5B_DECODELayerOffsets::mlp_out, QWEN2_0_5B_DECODELayerOffsets::output, QWEN2_0_5B_DECODELayerOffsets::proj_scratch, QWEN2_0_5B_DECODELayerOffsets::proj_tmp, QWEN2_0_5B_DECODELayerOffsets::q, QWEN2_0_5B_DECODE_EMBED_DIM, QWEN2_0_5B_DECODE_GLOBALS, QWEN2_0_5B_DECODE_HEAD_DIM, QWEN2_0_5B_DECODE_LAYERS, QWEN2_0_5B_DECODE_NUM_HEADS, QWEN2_0_5B_DECODE_NUM_KV_HEADS, QWEN2_0_5B_DECODE_PTR, qwen2_0_5b_decode_residual_add_token_major(), QWEN2_0_5B_DECODELayerOffsets::residual1, rmsnorm_forward(), QWEN2_0_5B_DECODEGlobalOffsets::rope_cos_cache, rope_forward_qk_strided(), QWEN2_0_5B_DECODEGlobalOffsets::rope_sin_cache, swiglu_forward(), QWEN2_0_5B_DECODELayerOffsets::v, QWEN2_0_5B_DECODELayerOffsets::w1, QWEN2_0_5B_DECODELayerOffsets::w2, QWEN2_0_5B_DECODELayerOffsets::wk, QWEN2_0_5B_DECODELayerOffsets::wo, QWEN2_0_5B_DECODELayerOffsets::wq, and QWEN2_0_5B_DECODELayerOffsets::wv.

Referenced by qwen2_0_5b_decode_forward_prefill_impl().

qwen2_0_5b_decode_layer_22_decode()

static void qwen2_0_5b_decode_layer_22_decode ( QWEN2_0_5B_DECODEModel *  model, int  token_index, int  aligned_embed_dim, int  aligned_head_dim, int  aligned_intermediate_dim, int  aligned_context_window  )

static

Definition at line 8387 of file v6.5/test_generated/qwen2_int8.c.

  {
    const QWEN2_0_5B_DECODELayerOffsets *L = &QWEN2_0_5B_DECODE_LAYERS[22];
 
    float *input = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[21].output);
 
    float *ln1_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln1_gamma);
    float *ln1_out = QWEN2_0_5B_DECODE_PTR(model, L->ln1_out);
    float *ln2_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln2_gamma);
    float *ln2_out = QWEN2_0_5B_DECODE_PTR(model, L->ln2_out);
    float *k_cache = QWEN2_0_5B_DECODE_PTR(model, L->k);
    float *v_cache = QWEN2_0_5B_DECODE_PTR(model, L->v);
    float *proj_tmp = QWEN2_0_5B_DECODE_PTR(model, L->proj_tmp);
    float *proj_scratch = QWEN2_0_5B_DECODE_PTR(model, L->proj_scratch);
    float *residual1 = QWEN2_0_5B_DECODE_PTR(model, L->residual1);
    float *mlp_out = QWEN2_0_5B_DECODE_PTR(model, L->mlp_out);
    float *output = QWEN2_0_5B_DECODE_PTR(model, L->output);
 
    /* Weights (explicit types for layer 22) */
    const void *WQ = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wq);  /* Q4_K */
    const void *WK = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wk);  /* Q4_K */
    const void *WV = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wv);  /* Q4_K */
    const void *WO = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wo);  /* Q4_K */
    const void *W1 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w1);  /* Q4_K (gate+up) */
    const void *W2 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w2);  /* Q4_K (down) */
 
    float *rope_cos = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_cos_cache);
    float *rope_sin = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_sin_cache);
 
    const int H = QWEN2_0_5B_DECODE_NUM_HEADS;
    const int H_kv = QWEN2_0_5B_DECODE_NUM_KV_HEADS;
    const int head_dim = QWEN2_0_5B_DECODE_HEAD_DIM;
 
    float q_token[H * aligned_head_dim];
    float k_token[H_kv * aligned_head_dim];
    float v_token[H_kv * aligned_head_dim];
    float attn_token[H * aligned_head_dim];
 
    /* Local MLP buffers (avoid layout dependencies for intermediate values) */
    float fc1_out[2 * aligned_intermediate_dim];
    float swiglu_out[aligned_intermediate_dim];
 
    /* Step 1: RMSNorm before attention */
    rmsnorm_forward(input,
                    ln1_gamma,
                    ln1_out,
                    NULL,
                    1,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    const size_t kv_head_stride = (size_t)aligned_context_window * (size_t)aligned_head_dim;
 
    /* Step 2: QKV projection */
    /* Q projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t ln1_q8_bytes = ((((aligned_embed_dim) + 255) / 256) * 292);
    uint8_t ln1_q8[ln1_q8_bytes];
    quantize_row_q8_k(ln1_out, ln1_q8, aligned_embed_dim);
    gemv_q4_k_q8_k(q_token, WQ, ln1_q8, H * head_dim, aligned_embed_dim);
    if (aligned_head_dim > head_dim) {
        for (int h = 0; h < H; ++h) {
            float *q_head = q_token + (size_t)h * (size_t)aligned_head_dim;
            for (int d = head_dim; d < aligned_head_dim; ++d) {
                q_head[d] = 0.0f;
            }
        }
    }
 
    /* K projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k (direct-to-cache) */
    const size_t wk_head_elems = (size_t)aligned_head_dim * (size_t)aligned_embed_dim;
    const size_t wk_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, wk_head_elems);
    const uint8_t *WK_bytes = (const uint8_t *)WK;
    /* ln1_q8 already quantized above */
    for (int h = 0; h < H_kv; ++h) {
        const void *wk_h = (const void *)(WK_bytes + (size_t)h * wk_head_bytes);
        float *k_head = k_cache + (size_t)h * kv_head_stride + (size_t)token_index * (size_t)aligned_head_dim;
        gemv_q4_k_q8_k(k_head, wk_h, ln1_q8, head_dim, aligned_embed_dim);
        for (int d = head_dim; d < aligned_head_dim; ++d) {
            k_head[d] = 0.0f;
        }
    }
 
    /* V projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k (direct-to-cache) */
    const size_t wv_head_elems = (size_t)aligned_head_dim * (size_t)aligned_embed_dim;
    const size_t wv_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, wv_head_elems);
    const uint8_t *WV_bytes = (const uint8_t *)WV;
    /* ln1_q8 already quantized above */
    for (int h = 0; h < H_kv; ++h) {
        const void *wv_h = (const void *)(WV_bytes + (size_t)h * wv_head_bytes);
        float *v_head = v_cache + (size_t)h * kv_head_stride + (size_t)token_index * (size_t)aligned_head_dim;
        gemv_q4_k_q8_k(v_head, wv_h, ln1_q8, head_dim, aligned_embed_dim);
        for (int d = head_dim; d < aligned_head_dim; ++d) {
            v_head[d] = 0.0f;
        }
    }
 
    /* Step 3: RoPE */
    rope_forward(q_token,
                 rope_cos,
                 rope_sin,
                 H,
                 1,
                 head_dim,
                 aligned_head_dim,
                 token_index);
    for (int h = 0; h < H_kv; ++h) {
        float *k_head = k_cache + (size_t)h * kv_head_stride + (size_t)token_index * (size_t)aligned_head_dim;
        rope_forward(k_head,
                     rope_cos,
                     rope_sin,
                     1,
                     1,
                     head_dim,
                     aligned_head_dim,
                     token_index);
    }
 
    /* Step 4: KV cache write (direct-to-cache) */
 
    /* Step 5: Attention (decode, flash) */
    attention_forward_decode_head_major_gqa_flash(q_token,
                                                   k_cache,
                                                   v_cache,
                                                   attn_token,
                                                   H,
                                                   H_kv,
                                                   token_index + 1,
                                                   aligned_context_window,
                                                   head_dim,
                                                   aligned_head_dim);
 
    /* Step 6: Output projection */
    /* WO projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t attn_q8_bytes = ((((H * head_dim) + 255) / 256) * 292);
    uint8_t attn_q8[attn_q8_bytes];
    quantize_row_q8_k(attn_token, attn_q8, H * head_dim);
    gemv_q4_k_q8_k(proj_tmp, WO, attn_q8, aligned_embed_dim, H * head_dim);
 
    /* Step 7: Residual add */
    qwen2_0_5b_decode_residual_add_token_major(input, proj_tmp, residual1, 1, aligned_embed_dim);
 
    /* Step 8: RMSNorm before MLP */
    rmsnorm_forward(residual1,
                    ln2_gamma,
                    ln2_out,
                    NULL,
                    1,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    /* Step 9: MLP (SwiGLU) */
    /* Gate+Up projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t ln2_q8_bytes = ((((aligned_embed_dim) + 255) / 256) * 292);
    uint8_t ln2_q8[ln2_q8_bytes];
    quantize_row_q8_k(ln2_out, ln2_q8, aligned_embed_dim);
    gemv_q4_k_q8_k(fc1_out, W1, ln2_q8, 2 * aligned_intermediate_dim, aligned_embed_dim);
 
    /* SwiGLU activation */
    swiglu_forward(fc1_out, swiglu_out, 1, aligned_intermediate_dim);
 
    /* Down projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t swiglu_q8_bytes = ((((aligned_intermediate_dim) + 255) / 256) * 292);
    uint8_t swiglu_q8[swiglu_q8_bytes];
    quantize_row_q8_k(swiglu_out, swiglu_q8, aligned_intermediate_dim);
    gemv_q4_k_q8_k(mlp_out, W2, swiglu_q8, aligned_embed_dim, aligned_intermediate_dim);
 
    /* Step 10: Final residual add */
    qwen2_0_5b_decode_residual_add_token_major(residual1, mlp_out, output, 1, aligned_embed_dim);
}

References attention_forward_decode_head_major_gqa_flash(), CK_DT_Q4_K, ck_dtype_row_bytes(), gemv_q4_k_q8_k(), QWEN2_0_5B_DECODELayerOffsets::k, QWEN2_0_5B_DECODELayerOffsets::ln1_gamma, QWEN2_0_5B_DECODELayerOffsets::ln1_out, QWEN2_0_5B_DECODELayerOffsets::ln2_gamma, QWEN2_0_5B_DECODELayerOffsets::ln2_out, QWEN2_0_5B_DECODELayerOffsets::mlp_out, QWEN2_0_5B_DECODELayerOffsets::output, QWEN2_0_5B_DECODELayerOffsets::proj_scratch, QWEN2_0_5B_DECODELayerOffsets::proj_tmp, quantize_row_q8_k(), QWEN2_0_5B_DECODE_EMBED_DIM, QWEN2_0_5B_DECODE_GLOBALS, QWEN2_0_5B_DECODE_HEAD_DIM, QWEN2_0_5B_DECODE_LAYERS, QWEN2_0_5B_DECODE_NUM_HEADS, QWEN2_0_5B_DECODE_NUM_KV_HEADS, QWEN2_0_5B_DECODE_PTR, qwen2_0_5b_decode_residual_add_token_major(), QWEN2_0_5B_DECODELayerOffsets::residual1, rmsnorm_forward(), QWEN2_0_5B_DECODEGlobalOffsets::rope_cos_cache, rope_forward(), QWEN2_0_5B_DECODEGlobalOffsets::rope_sin_cache, swiglu_forward(), QWEN2_0_5B_DECODELayerOffsets::v, QWEN2_0_5B_DECODELayerOffsets::w1, QWEN2_0_5B_DECODELayerOffsets::w2, QWEN2_0_5B_DECODELayerOffsets::wk, QWEN2_0_5B_DECODELayerOffsets::wo, QWEN2_0_5B_DECODELayerOffsets::wq, and QWEN2_0_5B_DECODELayerOffsets::wv.

Referenced by qwen2_0_5b_decode_decode_token().

qwen2_0_5b_decode_layer_22_prefill()

static void qwen2_0_5b_decode_layer_22_prefill ( QWEN2_0_5B_DECODEModel *  model, int  num_tokens, int  aligned_embed_dim, int  aligned_head_dim, int  aligned_intermediate_dim, int  aligned_context_window  )

static

Definition at line 3819 of file v6.5/test_generated/qwen2_int8.c.

  {
    const QWEN2_0_5B_DECODELayerOffsets *L = &QWEN2_0_5B_DECODE_LAYERS[22];
 
    float *input = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[21].output);
    float *ln1_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln1_gamma);
    float *ln1_out = QWEN2_0_5B_DECODE_PTR(model, L->ln1_out);
    float *ln2_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln2_gamma);
    float *ln2_out = QWEN2_0_5B_DECODE_PTR(model, L->ln2_out);
    float *q = QWEN2_0_5B_DECODE_PTR(model, L->q);
    float *k = QWEN2_0_5B_DECODE_PTR(model, L->k);
    float *v = QWEN2_0_5B_DECODE_PTR(model, L->v);
    float *attn_out = QWEN2_0_5B_DECODE_PTR(model, L->attn_out);
    float *proj_tmp = QWEN2_0_5B_DECODE_PTR(model, L->proj_tmp);
    float *proj_scratch = QWEN2_0_5B_DECODE_PTR(model, L->proj_scratch);
    float *residual1 = QWEN2_0_5B_DECODE_PTR(model, L->residual1);
    float *fc1_out = QWEN2_0_5B_DECODE_PTR(model, L->fc1_out);
    float *swiglu_out = QWEN2_0_5B_DECODE_PTR(model, L->swiglu_out);
    float *mlp_out = QWEN2_0_5B_DECODE_PTR(model, L->mlp_out);
    float *output = QWEN2_0_5B_DECODE_PTR(model, L->output);
 
    const void *WQ = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wq);
    const void *WK = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wk);
    const void *WV = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wv);
    const void *WO = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wo);
    const void *W1 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w1);
    const void *W2 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w2);
    const float *BQ = NULL;
    const float *BK = NULL;
    const float *BV = NULL;
    const float *BO = NULL;
    const float *B1 = NULL;
    const float *B2 = NULL;
 
    float *rope_cos = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_cos_cache);
    float *rope_sin = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_sin_cache);
 
    const int H = QWEN2_0_5B_DECODE_NUM_HEADS;
    const int H_kv = QWEN2_0_5B_DECODE_NUM_KV_HEADS;
    const int head_dim = QWEN2_0_5B_DECODE_HEAD_DIM;
    const size_t head_w_elems = (size_t)aligned_head_dim * (size_t)aligned_embed_dim;
    const size_t q_head_stride = (size_t)num_tokens * (size_t)aligned_head_dim;
    const size_t kv_head_stride = (size_t)aligned_context_window * (size_t)aligned_head_dim;
 
    /* RMSNorm before attention */
    rmsnorm_forward(input,
                    ln1_gamma,
                    ln1_out,
                    NULL,
                    num_tokens,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    /* Q projection (head-major) */
    const size_t wq_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, head_w_elems);
    const uint8_t *WQ_bytes = (const uint8_t *)WQ;
    for (int h = 0; h < H; ++h) {
        const void *wq_h = (const void *)(WQ_bytes + (size_t)h * wq_head_bytes);
        const float *bq_h = BQ ? (BQ + (size_t)h * (size_t)aligned_head_dim) : NULL;
        float *q_h = q + (size_t)h * q_head_stride;
        gemm_nt_q4_k(ln1_out, wq_h, bq_h, q_h, num_tokens, aligned_head_dim, aligned_embed_dim);
    }
 
    /* K projection (head-major) */
    const size_t wk_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, head_w_elems);
    const uint8_t *WK_bytes = (const uint8_t *)WK;
    for (int h = 0; h < H_kv; ++h) {
        const void *wk_h = (const void *)(WK_bytes + (size_t)h * wk_head_bytes);
        const float *bk_h = BK ? (BK + (size_t)h * (size_t)aligned_head_dim) : NULL;
        float *k_h = k + (size_t)h * kv_head_stride;
        gemm_nt_q4_k(ln1_out, wk_h, bk_h, k_h, num_tokens, aligned_head_dim, aligned_embed_dim);
    }
 
    /* V projection (head-major) */
    const size_t wv_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, head_w_elems);
    const uint8_t *WV_bytes = (const uint8_t *)WV;
    for (int h = 0; h < H_kv; ++h) {
        const void *wv_h = (const void *)(WV_bytes + (size_t)h * wv_head_bytes);
        const float *bv_h = BV ? (BV + (size_t)h * (size_t)aligned_head_dim) : NULL;
        float *v_h = v + (size_t)h * kv_head_stride;
        gemm_nt_q4_k(ln1_out, wv_h, bv_h, v_h, num_tokens, aligned_head_dim, aligned_embed_dim);
    }
 
    /* RoPE */
    rope_forward_qk_strided(q,
                            k,
                            rope_cos,
                            rope_sin,
                            H,
                            H_kv,
                            num_tokens,
                            head_dim,
                            aligned_head_dim,
                            0,
                            num_tokens,
                            aligned_context_window);
 
    /* Attention (prefill, causal) */
    attention_forward_causal_head_major_gqa_flash_strided(q,
                                                           k,
                                                           v,
                                                           attn_out,
                                                           H,
                                                           H_kv,
                                                           num_tokens,
                                                           head_dim,
                                                           aligned_head_dim,
                                                           aligned_context_window);
 
    /* Output projection (flatten head-major to token-major) */
    const int K = H * aligned_head_dim;
    if (K != aligned_embed_dim) {
        return;
    }
    const float *proj_in = attn_out;
    if (H > 1) {
        if (!proj_scratch) {
            return;
        }
        for (int t = 0; t < num_tokens; ++t) {
            float *dst = proj_scratch + (size_t)t * (size_t)aligned_embed_dim;
            for (int h = 0; h < H; ++h) {
                const float *src = attn_out + (size_t)h * q_head_stride + (size_t)t * (size_t)aligned_head_dim;
                memcpy(dst + (size_t)h * (size_t)aligned_head_dim,
                       src,
                       (size_t)aligned_head_dim * sizeof(float));
            }
        }
        proj_in = proj_scratch;
    }
    gemm_nt_q4_k(proj_in, WO, BO, proj_tmp, num_tokens, aligned_embed_dim, K);
 
    /* Residual add */
    qwen2_0_5b_decode_residual_add_token_major(input, proj_tmp, residual1, num_tokens, aligned_embed_dim);
 
    /* RMSNorm before MLP */
    rmsnorm_forward(residual1,
                    ln2_gamma,
                    ln2_out,
                    NULL,
                    num_tokens,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    /* MLP (SwiGLU) */
    gemm_nt_q4_k(ln2_out, W1, B1, fc1_out, num_tokens, 2 * aligned_intermediate_dim, aligned_embed_dim);
    swiglu_forward(fc1_out, swiglu_out, num_tokens, aligned_intermediate_dim);
    gemm_nt_q4_k(swiglu_out, W2, B2, mlp_out, num_tokens, aligned_embed_dim, aligned_intermediate_dim);
 
    /* Final residual add */
    qwen2_0_5b_decode_residual_add_token_major(residual1, mlp_out, output, num_tokens, aligned_embed_dim);
}

References attention_forward_causal_head_major_gqa_flash_strided(), CK_DT_Q4_K, ck_dtype_row_bytes(), gemm_nt_q4_k(), QWEN2_0_5B_DECODELayerOffsets::k, QWEN2_0_5B_DECODELayerOffsets::ln1_gamma, QWEN2_0_5B_DECODELayerOffsets::ln1_out, QWEN2_0_5B_DECODELayerOffsets::ln2_gamma, QWEN2_0_5B_DECODELayerOffsets::ln2_out, QWEN2_0_5B_DECODELayerOffsets::mlp_out, QWEN2_0_5B_DECODELayerOffsets::output, QWEN2_0_5B_DECODELayerOffsets::proj_scratch, QWEN2_0_5B_DECODELayerOffsets::proj_tmp, QWEN2_0_5B_DECODELayerOffsets::q, QWEN2_0_5B_DECODE_EMBED_DIM, QWEN2_0_5B_DECODE_GLOBALS, QWEN2_0_5B_DECODE_HEAD_DIM, QWEN2_0_5B_DECODE_LAYERS, QWEN2_0_5B_DECODE_NUM_HEADS, QWEN2_0_5B_DECODE_NUM_KV_HEADS, QWEN2_0_5B_DECODE_PTR, qwen2_0_5b_decode_residual_add_token_major(), QWEN2_0_5B_DECODELayerOffsets::residual1, rmsnorm_forward(), QWEN2_0_5B_DECODEGlobalOffsets::rope_cos_cache, rope_forward_qk_strided(), QWEN2_0_5B_DECODEGlobalOffsets::rope_sin_cache, swiglu_forward(), QWEN2_0_5B_DECODELayerOffsets::v, QWEN2_0_5B_DECODELayerOffsets::w1, QWEN2_0_5B_DECODELayerOffsets::w2, QWEN2_0_5B_DECODELayerOffsets::wk, QWEN2_0_5B_DECODELayerOffsets::wo, QWEN2_0_5B_DECODELayerOffsets::wq, and QWEN2_0_5B_DECODELayerOffsets::wv.

Referenced by qwen2_0_5b_decode_forward_prefill_impl().

qwen2_0_5b_decode_layer_23_decode()

static void qwen2_0_5b_decode_layer_23_decode ( QWEN2_0_5B_DECODEModel *  model, int  token_index, int  aligned_embed_dim, int  aligned_head_dim, int  aligned_intermediate_dim, int  aligned_context_window  )

static

Definition at line 8568 of file v6.5/test_generated/qwen2_int8.c.

  {
    const QWEN2_0_5B_DECODELayerOffsets *L = &QWEN2_0_5B_DECODE_LAYERS[23];
 
    float *input = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[22].output);
 
    float *ln1_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln1_gamma);
    float *ln1_out = QWEN2_0_5B_DECODE_PTR(model, L->ln1_out);
    float *ln2_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln2_gamma);
    float *ln2_out = QWEN2_0_5B_DECODE_PTR(model, L->ln2_out);
    float *k_cache = QWEN2_0_5B_DECODE_PTR(model, L->k);
    float *v_cache = QWEN2_0_5B_DECODE_PTR(model, L->v);
    float *proj_tmp = QWEN2_0_5B_DECODE_PTR(model, L->proj_tmp);
    float *proj_scratch = QWEN2_0_5B_DECODE_PTR(model, L->proj_scratch);
    float *residual1 = QWEN2_0_5B_DECODE_PTR(model, L->residual1);
    float *mlp_out = QWEN2_0_5B_DECODE_PTR(model, L->mlp_out);
    float *output = QWEN2_0_5B_DECODE_PTR(model, L->output);
 
    /* Weights (explicit types for layer 23) */
    const void *WQ = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wq);  /* Q4_K */
    const void *WK = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wk);  /* Q4_K */
    const void *WV = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wv);  /* Q4_K */
    const void *WO = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wo);  /* Q4_K */
    const void *W1 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w1);  /* Q4_K (gate+up) */
    const void *W2 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w2);  /* Q4_K (down) */
 
    float *rope_cos = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_cos_cache);
    float *rope_sin = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_sin_cache);
 
    const int H = QWEN2_0_5B_DECODE_NUM_HEADS;
    const int H_kv = QWEN2_0_5B_DECODE_NUM_KV_HEADS;
    const int head_dim = QWEN2_0_5B_DECODE_HEAD_DIM;
 
    float q_token[H * aligned_head_dim];
    float k_token[H_kv * aligned_head_dim];
    float v_token[H_kv * aligned_head_dim];
    float attn_token[H * aligned_head_dim];
 
    /* Local MLP buffers (avoid layout dependencies for intermediate values) */
    float fc1_out[2 * aligned_intermediate_dim];
    float swiglu_out[aligned_intermediate_dim];
 
    /* Step 1: RMSNorm before attention */
    rmsnorm_forward(input,
                    ln1_gamma,
                    ln1_out,
                    NULL,
                    1,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    const size_t kv_head_stride = (size_t)aligned_context_window * (size_t)aligned_head_dim;
 
    /* Step 2: QKV projection */
    /* Q projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t ln1_q8_bytes = ((((aligned_embed_dim) + 255) / 256) * 292);
    uint8_t ln1_q8[ln1_q8_bytes];
    quantize_row_q8_k(ln1_out, ln1_q8, aligned_embed_dim);
    gemv_q4_k_q8_k(q_token, WQ, ln1_q8, H * head_dim, aligned_embed_dim);
    if (aligned_head_dim > head_dim) {
        for (int h = 0; h < H; ++h) {
            float *q_head = q_token + (size_t)h * (size_t)aligned_head_dim;
            for (int d = head_dim; d < aligned_head_dim; ++d) {
                q_head[d] = 0.0f;
            }
        }
    }
 
    /* K projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k (direct-to-cache) */
    const size_t wk_head_elems = (size_t)aligned_head_dim * (size_t)aligned_embed_dim;
    const size_t wk_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, wk_head_elems);
    const uint8_t *WK_bytes = (const uint8_t *)WK;
    /* ln1_q8 already quantized above */
    for (int h = 0; h < H_kv; ++h) {
        const void *wk_h = (const void *)(WK_bytes + (size_t)h * wk_head_bytes);
        float *k_head = k_cache + (size_t)h * kv_head_stride + (size_t)token_index * (size_t)aligned_head_dim;
        gemv_q4_k_q8_k(k_head, wk_h, ln1_q8, head_dim, aligned_embed_dim);
        for (int d = head_dim; d < aligned_head_dim; ++d) {
            k_head[d] = 0.0f;
        }
    }
 
    /* V projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k (direct-to-cache) */
    const size_t wv_head_elems = (size_t)aligned_head_dim * (size_t)aligned_embed_dim;
    const size_t wv_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, wv_head_elems);
    const uint8_t *WV_bytes = (const uint8_t *)WV;
    /* ln1_q8 already quantized above */
    for (int h = 0; h < H_kv; ++h) {
        const void *wv_h = (const void *)(WV_bytes + (size_t)h * wv_head_bytes);
        float *v_head = v_cache + (size_t)h * kv_head_stride + (size_t)token_index * (size_t)aligned_head_dim;
        gemv_q4_k_q8_k(v_head, wv_h, ln1_q8, head_dim, aligned_embed_dim);
        for (int d = head_dim; d < aligned_head_dim; ++d) {
            v_head[d] = 0.0f;
        }
    }
 
    /* Step 3: RoPE */
    rope_forward(q_token,
                 rope_cos,
                 rope_sin,
                 H,
                 1,
                 head_dim,
                 aligned_head_dim,
                 token_index);
    for (int h = 0; h < H_kv; ++h) {
        float *k_head = k_cache + (size_t)h * kv_head_stride + (size_t)token_index * (size_t)aligned_head_dim;
        rope_forward(k_head,
                     rope_cos,
                     rope_sin,
                     1,
                     1,
                     head_dim,
                     aligned_head_dim,
                     token_index);
    }
 
    /* Step 4: KV cache write (direct-to-cache) */
 
    /* Step 5: Attention (decode, flash) */
    attention_forward_decode_head_major_gqa_flash(q_token,
                                                   k_cache,
                                                   v_cache,
                                                   attn_token,
                                                   H,
                                                   H_kv,
                                                   token_index + 1,
                                                   aligned_context_window,
                                                   head_dim,
                                                   aligned_head_dim);
 
    /* Step 6: Output projection */
    /* WO projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t attn_q8_bytes = ((((H * head_dim) + 255) / 256) * 292);
    uint8_t attn_q8[attn_q8_bytes];
    quantize_row_q8_k(attn_token, attn_q8, H * head_dim);
    gemv_q4_k_q8_k(proj_tmp, WO, attn_q8, aligned_embed_dim, H * head_dim);
 
    /* Step 7: Residual add */
    qwen2_0_5b_decode_residual_add_token_major(input, proj_tmp, residual1, 1, aligned_embed_dim);
 
    /* Step 8: RMSNorm before MLP */
    rmsnorm_forward(residual1,
                    ln2_gamma,
                    ln2_out,
                    NULL,
                    1,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    /* Step 9: MLP (SwiGLU) */
    /* Gate+Up projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t ln2_q8_bytes = ((((aligned_embed_dim) + 255) / 256) * 292);
    uint8_t ln2_q8[ln2_q8_bytes];
    quantize_row_q8_k(ln2_out, ln2_q8, aligned_embed_dim);
    gemv_q4_k_q8_k(fc1_out, W1, ln2_q8, 2 * aligned_intermediate_dim, aligned_embed_dim);
 
    /* SwiGLU activation */
    swiglu_forward(fc1_out, swiglu_out, 1, aligned_intermediate_dim);
 
    /* Down projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t swiglu_q8_bytes = ((((aligned_intermediate_dim) + 255) / 256) * 292);
    uint8_t swiglu_q8[swiglu_q8_bytes];
    quantize_row_q8_k(swiglu_out, swiglu_q8, aligned_intermediate_dim);
    gemv_q4_k_q8_k(mlp_out, W2, swiglu_q8, aligned_embed_dim, aligned_intermediate_dim);
 
    /* Step 10: Final residual add */
    qwen2_0_5b_decode_residual_add_token_major(residual1, mlp_out, output, 1, aligned_embed_dim);
}

References attention_forward_decode_head_major_gqa_flash(), CK_DT_Q4_K, ck_dtype_row_bytes(), gemv_q4_k_q8_k(), QWEN2_0_5B_DECODELayerOffsets::k, QWEN2_0_5B_DECODELayerOffsets::ln1_gamma, QWEN2_0_5B_DECODELayerOffsets::ln1_out, QWEN2_0_5B_DECODELayerOffsets::ln2_gamma, QWEN2_0_5B_DECODELayerOffsets::ln2_out, QWEN2_0_5B_DECODELayerOffsets::mlp_out, QWEN2_0_5B_DECODELayerOffsets::output, QWEN2_0_5B_DECODELayerOffsets::proj_scratch, QWEN2_0_5B_DECODELayerOffsets::proj_tmp, quantize_row_q8_k(), QWEN2_0_5B_DECODE_EMBED_DIM, QWEN2_0_5B_DECODE_GLOBALS, QWEN2_0_5B_DECODE_HEAD_DIM, QWEN2_0_5B_DECODE_LAYERS, QWEN2_0_5B_DECODE_NUM_HEADS, QWEN2_0_5B_DECODE_NUM_KV_HEADS, QWEN2_0_5B_DECODE_PTR, qwen2_0_5b_decode_residual_add_token_major(), QWEN2_0_5B_DECODELayerOffsets::residual1, rmsnorm_forward(), QWEN2_0_5B_DECODEGlobalOffsets::rope_cos_cache, rope_forward(), QWEN2_0_5B_DECODEGlobalOffsets::rope_sin_cache, swiglu_forward(), QWEN2_0_5B_DECODELayerOffsets::v, QWEN2_0_5B_DECODELayerOffsets::w1, QWEN2_0_5B_DECODELayerOffsets::w2, QWEN2_0_5B_DECODELayerOffsets::wk, QWEN2_0_5B_DECODELayerOffsets::wo, QWEN2_0_5B_DECODELayerOffsets::wq, and QWEN2_0_5B_DECODELayerOffsets::wv.

Referenced by qwen2_0_5b_decode_decode_token().

qwen2_0_5b_decode_layer_23_prefill()

static void qwen2_0_5b_decode_layer_23_prefill ( QWEN2_0_5B_DECODEModel *  model, int  num_tokens, int  aligned_embed_dim, int  aligned_head_dim, int  aligned_intermediate_dim, int  aligned_context_window  )

static

Definition at line 3983 of file v6.5/test_generated/qwen2_int8.c.

  {
    const QWEN2_0_5B_DECODELayerOffsets *L = &QWEN2_0_5B_DECODE_LAYERS[23];
 
    float *input = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[22].output);
    float *ln1_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln1_gamma);
    float *ln1_out = QWEN2_0_5B_DECODE_PTR(model, L->ln1_out);
    float *ln2_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln2_gamma);
    float *ln2_out = QWEN2_0_5B_DECODE_PTR(model, L->ln2_out);
    float *q = QWEN2_0_5B_DECODE_PTR(model, L->q);
    float *k = QWEN2_0_5B_DECODE_PTR(model, L->k);
    float *v = QWEN2_0_5B_DECODE_PTR(model, L->v);
    float *attn_out = QWEN2_0_5B_DECODE_PTR(model, L->attn_out);
    float *proj_tmp = QWEN2_0_5B_DECODE_PTR(model, L->proj_tmp);
    float *proj_scratch = QWEN2_0_5B_DECODE_PTR(model, L->proj_scratch);
    float *residual1 = QWEN2_0_5B_DECODE_PTR(model, L->residual1);
    float *fc1_out = QWEN2_0_5B_DECODE_PTR(model, L->fc1_out);
    float *swiglu_out = QWEN2_0_5B_DECODE_PTR(model, L->swiglu_out);
    float *mlp_out = QWEN2_0_5B_DECODE_PTR(model, L->mlp_out);
    float *output = QWEN2_0_5B_DECODE_PTR(model, L->output);
 
    const void *WQ = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wq);
    const void *WK = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wk);
    const void *WV = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wv);
    const void *WO = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wo);
    const void *W1 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w1);
    const void *W2 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w2);
    const float *BQ = NULL;
    const float *BK = NULL;
    const float *BV = NULL;
    const float *BO = NULL;
    const float *B1 = NULL;
    const float *B2 = NULL;
 
    float *rope_cos = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_cos_cache);
    float *rope_sin = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_sin_cache);
 
    const int H = QWEN2_0_5B_DECODE_NUM_HEADS;
    const int H_kv = QWEN2_0_5B_DECODE_NUM_KV_HEADS;
    const int head_dim = QWEN2_0_5B_DECODE_HEAD_DIM;
    const size_t head_w_elems = (size_t)aligned_head_dim * (size_t)aligned_embed_dim;
    const size_t q_head_stride = (size_t)num_tokens * (size_t)aligned_head_dim;
    const size_t kv_head_stride = (size_t)aligned_context_window * (size_t)aligned_head_dim;
 
    /* RMSNorm before attention */
    rmsnorm_forward(input,
                    ln1_gamma,
                    ln1_out,
                    NULL,
                    num_tokens,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    /* Q projection (head-major) */
    const size_t wq_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, head_w_elems);
    const uint8_t *WQ_bytes = (const uint8_t *)WQ;
    for (int h = 0; h < H; ++h) {
        const void *wq_h = (const void *)(WQ_bytes + (size_t)h * wq_head_bytes);
        const float *bq_h = BQ ? (BQ + (size_t)h * (size_t)aligned_head_dim) : NULL;
        float *q_h = q + (size_t)h * q_head_stride;
        gemm_nt_q4_k(ln1_out, wq_h, bq_h, q_h, num_tokens, aligned_head_dim, aligned_embed_dim);
    }
 
    /* K projection (head-major) */
    const size_t wk_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, head_w_elems);
    const uint8_t *WK_bytes = (const uint8_t *)WK;
    for (int h = 0; h < H_kv; ++h) {
        const void *wk_h = (const void *)(WK_bytes + (size_t)h * wk_head_bytes);
        const float *bk_h = BK ? (BK + (size_t)h * (size_t)aligned_head_dim) : NULL;
        float *k_h = k + (size_t)h * kv_head_stride;
        gemm_nt_q4_k(ln1_out, wk_h, bk_h, k_h, num_tokens, aligned_head_dim, aligned_embed_dim);
    }
 
    /* V projection (head-major) */
    const size_t wv_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, head_w_elems);
    const uint8_t *WV_bytes = (const uint8_t *)WV;
    for (int h = 0; h < H_kv; ++h) {
        const void *wv_h = (const void *)(WV_bytes + (size_t)h * wv_head_bytes);
        const float *bv_h = BV ? (BV + (size_t)h * (size_t)aligned_head_dim) : NULL;
        float *v_h = v + (size_t)h * kv_head_stride;
        gemm_nt_q4_k(ln1_out, wv_h, bv_h, v_h, num_tokens, aligned_head_dim, aligned_embed_dim);
    }
 
    /* RoPE */
    rope_forward_qk_strided(q,
                            k,
                            rope_cos,
                            rope_sin,
                            H,
                            H_kv,
                            num_tokens,
                            head_dim,
                            aligned_head_dim,
                            0,
                            num_tokens,
                            aligned_context_window);
 
    /* Attention (prefill, causal) */
    attention_forward_causal_head_major_gqa_flash_strided(q,
                                                           k,
                                                           v,
                                                           attn_out,
                                                           H,
                                                           H_kv,
                                                           num_tokens,
                                                           head_dim,
                                                           aligned_head_dim,
                                                           aligned_context_window);
 
    /* Output projection (flatten head-major to token-major) */
    const int K = H * aligned_head_dim;
    if (K != aligned_embed_dim) {
        return;
    }
    const float *proj_in = attn_out;
    if (H > 1) {
        if (!proj_scratch) {
            return;
        }
        for (int t = 0; t < num_tokens; ++t) {
            float *dst = proj_scratch + (size_t)t * (size_t)aligned_embed_dim;
            for (int h = 0; h < H; ++h) {
                const float *src = attn_out + (size_t)h * q_head_stride + (size_t)t * (size_t)aligned_head_dim;
                memcpy(dst + (size_t)h * (size_t)aligned_head_dim,
                       src,
                       (size_t)aligned_head_dim * sizeof(float));
            }
        }
        proj_in = proj_scratch;
    }
    gemm_nt_q4_k(proj_in, WO, BO, proj_tmp, num_tokens, aligned_embed_dim, K);
 
    /* Residual add */
    qwen2_0_5b_decode_residual_add_token_major(input, proj_tmp, residual1, num_tokens, aligned_embed_dim);
 
    /* RMSNorm before MLP */
    rmsnorm_forward(residual1,
                    ln2_gamma,
                    ln2_out,
                    NULL,
                    num_tokens,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    /* MLP (SwiGLU) */
    gemm_nt_q4_k(ln2_out, W1, B1, fc1_out, num_tokens, 2 * aligned_intermediate_dim, aligned_embed_dim);
    swiglu_forward(fc1_out, swiglu_out, num_tokens, aligned_intermediate_dim);
    gemm_nt_q4_k(swiglu_out, W2, B2, mlp_out, num_tokens, aligned_embed_dim, aligned_intermediate_dim);
 
    /* Final residual add */
    qwen2_0_5b_decode_residual_add_token_major(residual1, mlp_out, output, num_tokens, aligned_embed_dim);
}

References attention_forward_causal_head_major_gqa_flash_strided(), CK_DT_Q4_K, ck_dtype_row_bytes(), gemm_nt_q4_k(), QWEN2_0_5B_DECODELayerOffsets::k, QWEN2_0_5B_DECODELayerOffsets::ln1_gamma, QWEN2_0_5B_DECODELayerOffsets::ln1_out, QWEN2_0_5B_DECODELayerOffsets::ln2_gamma, QWEN2_0_5B_DECODELayerOffsets::ln2_out, QWEN2_0_5B_DECODELayerOffsets::mlp_out, QWEN2_0_5B_DECODELayerOffsets::output, QWEN2_0_5B_DECODELayerOffsets::proj_scratch, QWEN2_0_5B_DECODELayerOffsets::proj_tmp, QWEN2_0_5B_DECODELayerOffsets::q, QWEN2_0_5B_DECODE_EMBED_DIM, QWEN2_0_5B_DECODE_GLOBALS, QWEN2_0_5B_DECODE_HEAD_DIM, QWEN2_0_5B_DECODE_LAYERS, QWEN2_0_5B_DECODE_NUM_HEADS, QWEN2_0_5B_DECODE_NUM_KV_HEADS, QWEN2_0_5B_DECODE_PTR, qwen2_0_5b_decode_residual_add_token_major(), QWEN2_0_5B_DECODELayerOffsets::residual1, rmsnorm_forward(), QWEN2_0_5B_DECODEGlobalOffsets::rope_cos_cache, rope_forward_qk_strided(), QWEN2_0_5B_DECODEGlobalOffsets::rope_sin_cache, swiglu_forward(), QWEN2_0_5B_DECODELayerOffsets::v, QWEN2_0_5B_DECODELayerOffsets::w1, QWEN2_0_5B_DECODELayerOffsets::w2, QWEN2_0_5B_DECODELayerOffsets::wk, QWEN2_0_5B_DECODELayerOffsets::wo, QWEN2_0_5B_DECODELayerOffsets::wq, and QWEN2_0_5B_DECODELayerOffsets::wv.

Referenced by qwen2_0_5b_decode_forward_prefill_impl().

qwen2_0_5b_decode_layer_2_decode()

static void qwen2_0_5b_decode_layer_2_decode ( QWEN2_0_5B_DECODEModel *  model, int  token_index, int  aligned_embed_dim, int  aligned_head_dim, int  aligned_intermediate_dim, int  aligned_context_window  )

static

Definition at line 4767 of file v6.5/test_generated/qwen2_int8.c.

  {
    const QWEN2_0_5B_DECODELayerOffsets *L = &QWEN2_0_5B_DECODE_LAYERS[2];
 
    float *input = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[1].output);
 
    float *ln1_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln1_gamma);
    float *ln1_out = QWEN2_0_5B_DECODE_PTR(model, L->ln1_out);
    float *ln2_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln2_gamma);
    float *ln2_out = QWEN2_0_5B_DECODE_PTR(model, L->ln2_out);
    float *k_cache = QWEN2_0_5B_DECODE_PTR(model, L->k);
    float *v_cache = QWEN2_0_5B_DECODE_PTR(model, L->v);
    float *proj_tmp = QWEN2_0_5B_DECODE_PTR(model, L->proj_tmp);
    float *proj_scratch = QWEN2_0_5B_DECODE_PTR(model, L->proj_scratch);
    float *residual1 = QWEN2_0_5B_DECODE_PTR(model, L->residual1);
    float *mlp_out = QWEN2_0_5B_DECODE_PTR(model, L->mlp_out);
    float *output = QWEN2_0_5B_DECODE_PTR(model, L->output);
 
    /* Weights (explicit types for layer 2) */
    const void *WQ = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wq);  /* Q4_K */
    const void *WK = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wk);  /* Q4_K */
    const void *WV = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wv);  /* Q4_K */
    const void *WO = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wo);  /* Q4_K */
    const void *W1 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w1);  /* Q4_K (gate+up) */
    const void *W2 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w2);  /* Q4_K (down) */
 
    float *rope_cos = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_cos_cache);
    float *rope_sin = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_sin_cache);
 
    const int H = QWEN2_0_5B_DECODE_NUM_HEADS;
    const int H_kv = QWEN2_0_5B_DECODE_NUM_KV_HEADS;
    const int head_dim = QWEN2_0_5B_DECODE_HEAD_DIM;
 
    float q_token[H * aligned_head_dim];
    float k_token[H_kv * aligned_head_dim];
    float v_token[H_kv * aligned_head_dim];
    float attn_token[H * aligned_head_dim];
 
    /* Local MLP buffers (avoid layout dependencies for intermediate values) */
    float fc1_out[2 * aligned_intermediate_dim];
    float swiglu_out[aligned_intermediate_dim];
 
    /* Step 1: RMSNorm before attention */
    rmsnorm_forward(input,
                    ln1_gamma,
                    ln1_out,
                    NULL,
                    1,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    const size_t kv_head_stride = (size_t)aligned_context_window * (size_t)aligned_head_dim;
 
    /* Step 2: QKV projection */
    /* Q projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t ln1_q8_bytes = ((((aligned_embed_dim) + 255) / 256) * 292);
    uint8_t ln1_q8[ln1_q8_bytes];
    quantize_row_q8_k(ln1_out, ln1_q8, aligned_embed_dim);
    gemv_q4_k_q8_k(q_token, WQ, ln1_q8, H * head_dim, aligned_embed_dim);
    if (aligned_head_dim > head_dim) {
        for (int h = 0; h < H; ++h) {
            float *q_head = q_token + (size_t)h * (size_t)aligned_head_dim;
            for (int d = head_dim; d < aligned_head_dim; ++d) {
                q_head[d] = 0.0f;
            }
        }
    }
 
    /* K projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k (direct-to-cache) */
    const size_t wk_head_elems = (size_t)aligned_head_dim * (size_t)aligned_embed_dim;
    const size_t wk_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, wk_head_elems);
    const uint8_t *WK_bytes = (const uint8_t *)WK;
    /* ln1_q8 already quantized above */
    for (int h = 0; h < H_kv; ++h) {
        const void *wk_h = (const void *)(WK_bytes + (size_t)h * wk_head_bytes);
        float *k_head = k_cache + (size_t)h * kv_head_stride + (size_t)token_index * (size_t)aligned_head_dim;
        gemv_q4_k_q8_k(k_head, wk_h, ln1_q8, head_dim, aligned_embed_dim);
        for (int d = head_dim; d < aligned_head_dim; ++d) {
            k_head[d] = 0.0f;
        }
    }
 
    /* V projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k (direct-to-cache) */
    const size_t wv_head_elems = (size_t)aligned_head_dim * (size_t)aligned_embed_dim;
    const size_t wv_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, wv_head_elems);
    const uint8_t *WV_bytes = (const uint8_t *)WV;
    /* ln1_q8 already quantized above */
    for (int h = 0; h < H_kv; ++h) {
        const void *wv_h = (const void *)(WV_bytes + (size_t)h * wv_head_bytes);
        float *v_head = v_cache + (size_t)h * kv_head_stride + (size_t)token_index * (size_t)aligned_head_dim;
        gemv_q4_k_q8_k(v_head, wv_h, ln1_q8, head_dim, aligned_embed_dim);
        for (int d = head_dim; d < aligned_head_dim; ++d) {
            v_head[d] = 0.0f;
        }
    }
 
    /* Step 3: RoPE */
    rope_forward(q_token,
                 rope_cos,
                 rope_sin,
                 H,
                 1,
                 head_dim,
                 aligned_head_dim,
                 token_index);
    for (int h = 0; h < H_kv; ++h) {
        float *k_head = k_cache + (size_t)h * kv_head_stride + (size_t)token_index * (size_t)aligned_head_dim;
        rope_forward(k_head,
                     rope_cos,
                     rope_sin,
                     1,
                     1,
                     head_dim,
                     aligned_head_dim,
                     token_index);
    }
 
    /* Step 4: KV cache write (direct-to-cache) */
 
    /* Step 5: Attention (decode, flash) */
    attention_forward_decode_head_major_gqa_flash(q_token,
                                                   k_cache,
                                                   v_cache,
                                                   attn_token,
                                                   H,
                                                   H_kv,
                                                   token_index + 1,
                                                   aligned_context_window,
                                                   head_dim,
                                                   aligned_head_dim);
 
    /* Step 6: Output projection */
    /* WO projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t attn_q8_bytes = ((((H * head_dim) + 255) / 256) * 292);
    uint8_t attn_q8[attn_q8_bytes];
    quantize_row_q8_k(attn_token, attn_q8, H * head_dim);
    gemv_q4_k_q8_k(proj_tmp, WO, attn_q8, aligned_embed_dim, H * head_dim);
 
    /* Step 7: Residual add */
    qwen2_0_5b_decode_residual_add_token_major(input, proj_tmp, residual1, 1, aligned_embed_dim);
 
    /* Step 8: RMSNorm before MLP */
    rmsnorm_forward(residual1,
                    ln2_gamma,
                    ln2_out,
                    NULL,
                    1,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    /* Step 9: MLP (SwiGLU) */
    /* Gate+Up projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t ln2_q8_bytes = ((((aligned_embed_dim) + 255) / 256) * 292);
    uint8_t ln2_q8[ln2_q8_bytes];
    quantize_row_q8_k(ln2_out, ln2_q8, aligned_embed_dim);
    gemv_q4_k_q8_k(fc1_out, W1, ln2_q8, 2 * aligned_intermediate_dim, aligned_embed_dim);
 
    /* SwiGLU activation */
    swiglu_forward(fc1_out, swiglu_out, 1, aligned_intermediate_dim);
 
    /* Down projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t swiglu_q8_bytes = ((((aligned_intermediate_dim) + 255) / 256) * 292);
    uint8_t swiglu_q8[swiglu_q8_bytes];
    quantize_row_q8_k(swiglu_out, swiglu_q8, aligned_intermediate_dim);
    gemv_q4_k_q8_k(mlp_out, W2, swiglu_q8, aligned_embed_dim, aligned_intermediate_dim);
 
    /* Step 10: Final residual add */
    qwen2_0_5b_decode_residual_add_token_major(residual1, mlp_out, output, 1, aligned_embed_dim);
}

References attention_forward_decode_head_major_gqa_flash(), CK_DT_Q4_K, ck_dtype_row_bytes(), gemv_q4_k_q8_k(), QWEN2_0_5B_DECODELayerOffsets::k, QWEN2_0_5B_DECODELayerOffsets::ln1_gamma, QWEN2_0_5B_DECODELayerOffsets::ln1_out, QWEN2_0_5B_DECODELayerOffsets::ln2_gamma, QWEN2_0_5B_DECODELayerOffsets::ln2_out, QWEN2_0_5B_DECODELayerOffsets::mlp_out, QWEN2_0_5B_DECODELayerOffsets::output, QWEN2_0_5B_DECODELayerOffsets::proj_scratch, QWEN2_0_5B_DECODELayerOffsets::proj_tmp, quantize_row_q8_k(), QWEN2_0_5B_DECODE_EMBED_DIM, QWEN2_0_5B_DECODE_GLOBALS, QWEN2_0_5B_DECODE_HEAD_DIM, QWEN2_0_5B_DECODE_LAYERS, QWEN2_0_5B_DECODE_NUM_HEADS, QWEN2_0_5B_DECODE_NUM_KV_HEADS, QWEN2_0_5B_DECODE_PTR, qwen2_0_5b_decode_residual_add_token_major(), QWEN2_0_5B_DECODELayerOffsets::residual1, rmsnorm_forward(), QWEN2_0_5B_DECODEGlobalOffsets::rope_cos_cache, rope_forward(), QWEN2_0_5B_DECODEGlobalOffsets::rope_sin_cache, swiglu_forward(), QWEN2_0_5B_DECODELayerOffsets::v, QWEN2_0_5B_DECODELayerOffsets::w1, QWEN2_0_5B_DECODELayerOffsets::w2, QWEN2_0_5B_DECODELayerOffsets::wk, QWEN2_0_5B_DECODELayerOffsets::wo, QWEN2_0_5B_DECODELayerOffsets::wq, and QWEN2_0_5B_DECODELayerOffsets::wv.

Referenced by qwen2_0_5b_decode_decode_token().

qwen2_0_5b_decode_layer_2_prefill()

static void qwen2_0_5b_decode_layer_2_prefill ( QWEN2_0_5B_DECODEModel *  model, int  num_tokens, int  aligned_embed_dim, int  aligned_head_dim, int  aligned_intermediate_dim, int  aligned_context_window  )

static

Definition at line 539 of file v6.5/test_generated/qwen2_int8.c.

  {
    const QWEN2_0_5B_DECODELayerOffsets *L = &QWEN2_0_5B_DECODE_LAYERS[2];
 
    float *input = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[1].output);
    float *ln1_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln1_gamma);
    float *ln1_out = QWEN2_0_5B_DECODE_PTR(model, L->ln1_out);
    float *ln2_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln2_gamma);
    float *ln2_out = QWEN2_0_5B_DECODE_PTR(model, L->ln2_out);
    float *q = QWEN2_0_5B_DECODE_PTR(model, L->q);
    float *k = QWEN2_0_5B_DECODE_PTR(model, L->k);
    float *v = QWEN2_0_5B_DECODE_PTR(model, L->v);
    float *attn_out = QWEN2_0_5B_DECODE_PTR(model, L->attn_out);
    float *proj_tmp = QWEN2_0_5B_DECODE_PTR(model, L->proj_tmp);
    float *proj_scratch = QWEN2_0_5B_DECODE_PTR(model, L->proj_scratch);
    float *residual1 = QWEN2_0_5B_DECODE_PTR(model, L->residual1);
    float *fc1_out = QWEN2_0_5B_DECODE_PTR(model, L->fc1_out);
    float *swiglu_out = QWEN2_0_5B_DECODE_PTR(model, L->swiglu_out);
    float *mlp_out = QWEN2_0_5B_DECODE_PTR(model, L->mlp_out);
    float *output = QWEN2_0_5B_DECODE_PTR(model, L->output);
 
    const void *WQ = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wq);
    const void *WK = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wk);
    const void *WV = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wv);
    const void *WO = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wo);
    const void *W1 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w1);
    const void *W2 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w2);
    const float *BQ = NULL;
    const float *BK = NULL;
    const float *BV = NULL;
    const float *BO = NULL;
    const float *B1 = NULL;
    const float *B2 = NULL;
 
    float *rope_cos = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_cos_cache);
    float *rope_sin = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_sin_cache);
 
    const int H = QWEN2_0_5B_DECODE_NUM_HEADS;
    const int H_kv = QWEN2_0_5B_DECODE_NUM_KV_HEADS;
    const int head_dim = QWEN2_0_5B_DECODE_HEAD_DIM;
    const size_t head_w_elems = (size_t)aligned_head_dim * (size_t)aligned_embed_dim;
    const size_t q_head_stride = (size_t)num_tokens * (size_t)aligned_head_dim;
    const size_t kv_head_stride = (size_t)aligned_context_window * (size_t)aligned_head_dim;
 
    /* RMSNorm before attention */
    rmsnorm_forward(input,
                    ln1_gamma,
                    ln1_out,
                    NULL,
                    num_tokens,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    /* Q projection (head-major) */
    const size_t wq_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, head_w_elems);
    const uint8_t *WQ_bytes = (const uint8_t *)WQ;
    for (int h = 0; h < H; ++h) {
        const void *wq_h = (const void *)(WQ_bytes + (size_t)h * wq_head_bytes);
        const float *bq_h = BQ ? (BQ + (size_t)h * (size_t)aligned_head_dim) : NULL;
        float *q_h = q + (size_t)h * q_head_stride;
        gemm_nt_q4_k(ln1_out, wq_h, bq_h, q_h, num_tokens, aligned_head_dim, aligned_embed_dim);
    }
 
    /* K projection (head-major) */
    const size_t wk_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, head_w_elems);
    const uint8_t *WK_bytes = (const uint8_t *)WK;
    for (int h = 0; h < H_kv; ++h) {
        const void *wk_h = (const void *)(WK_bytes + (size_t)h * wk_head_bytes);
        const float *bk_h = BK ? (BK + (size_t)h * (size_t)aligned_head_dim) : NULL;
        float *k_h = k + (size_t)h * kv_head_stride;
        gemm_nt_q4_k(ln1_out, wk_h, bk_h, k_h, num_tokens, aligned_head_dim, aligned_embed_dim);
    }
 
    /* V projection (head-major) */
    const size_t wv_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, head_w_elems);
    const uint8_t *WV_bytes = (const uint8_t *)WV;
    for (int h = 0; h < H_kv; ++h) {
        const void *wv_h = (const void *)(WV_bytes + (size_t)h * wv_head_bytes);
        const float *bv_h = BV ? (BV + (size_t)h * (size_t)aligned_head_dim) : NULL;
        float *v_h = v + (size_t)h * kv_head_stride;
        gemm_nt_q4_k(ln1_out, wv_h, bv_h, v_h, num_tokens, aligned_head_dim, aligned_embed_dim);
    }
 
    /* RoPE */
    rope_forward_qk_strided(q,
                            k,
                            rope_cos,
                            rope_sin,
                            H,
                            H_kv,
                            num_tokens,
                            head_dim,
                            aligned_head_dim,
                            0,
                            num_tokens,
                            aligned_context_window);
 
    /* Attention (prefill, causal) */
    attention_forward_causal_head_major_gqa_flash_strided(q,
                                                           k,
                                                           v,
                                                           attn_out,
                                                           H,
                                                           H_kv,
                                                           num_tokens,
                                                           head_dim,
                                                           aligned_head_dim,
                                                           aligned_context_window);
 
    /* Output projection (flatten head-major to token-major) */
    const int K = H * aligned_head_dim;
    if (K != aligned_embed_dim) {
        return;
    }
    const float *proj_in = attn_out;
    if (H > 1) {
        if (!proj_scratch) {
            return;
        }
        for (int t = 0; t < num_tokens; ++t) {
            float *dst = proj_scratch + (size_t)t * (size_t)aligned_embed_dim;
            for (int h = 0; h < H; ++h) {
                const float *src = attn_out + (size_t)h * q_head_stride + (size_t)t * (size_t)aligned_head_dim;
                memcpy(dst + (size_t)h * (size_t)aligned_head_dim,
                       src,
                       (size_t)aligned_head_dim * sizeof(float));
            }
        }
        proj_in = proj_scratch;
    }
    gemm_nt_q4_k(proj_in, WO, BO, proj_tmp, num_tokens, aligned_embed_dim, K);
 
    /* Residual add */
    qwen2_0_5b_decode_residual_add_token_major(input, proj_tmp, residual1, num_tokens, aligned_embed_dim);
 
    /* RMSNorm before MLP */
    rmsnorm_forward(residual1,
                    ln2_gamma,
                    ln2_out,
                    NULL,
                    num_tokens,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    /* MLP (SwiGLU) */
    gemm_nt_q4_k(ln2_out, W1, B1, fc1_out, num_tokens, 2 * aligned_intermediate_dim, aligned_embed_dim);
    swiglu_forward(fc1_out, swiglu_out, num_tokens, aligned_intermediate_dim);
    gemm_nt_q4_k(swiglu_out, W2, B2, mlp_out, num_tokens, aligned_embed_dim, aligned_intermediate_dim);
 
    /* Final residual add */
    qwen2_0_5b_decode_residual_add_token_major(residual1, mlp_out, output, num_tokens, aligned_embed_dim);
}

References attention_forward_causal_head_major_gqa_flash_strided(), CK_DT_Q4_K, ck_dtype_row_bytes(), gemm_nt_q4_k(), QWEN2_0_5B_DECODELayerOffsets::k, QWEN2_0_5B_DECODELayerOffsets::ln1_gamma, QWEN2_0_5B_DECODELayerOffsets::ln1_out, QWEN2_0_5B_DECODELayerOffsets::ln2_gamma, QWEN2_0_5B_DECODELayerOffsets::ln2_out, QWEN2_0_5B_DECODELayerOffsets::mlp_out, QWEN2_0_5B_DECODELayerOffsets::output, QWEN2_0_5B_DECODELayerOffsets::proj_scratch, QWEN2_0_5B_DECODELayerOffsets::proj_tmp, QWEN2_0_5B_DECODELayerOffsets::q, QWEN2_0_5B_DECODE_EMBED_DIM, QWEN2_0_5B_DECODE_GLOBALS, QWEN2_0_5B_DECODE_HEAD_DIM, QWEN2_0_5B_DECODE_LAYERS, QWEN2_0_5B_DECODE_NUM_HEADS, QWEN2_0_5B_DECODE_NUM_KV_HEADS, QWEN2_0_5B_DECODE_PTR, qwen2_0_5b_decode_residual_add_token_major(), QWEN2_0_5B_DECODELayerOffsets::residual1, rmsnorm_forward(), QWEN2_0_5B_DECODEGlobalOffsets::rope_cos_cache, rope_forward_qk_strided(), QWEN2_0_5B_DECODEGlobalOffsets::rope_sin_cache, swiglu_forward(), QWEN2_0_5B_DECODELayerOffsets::v, QWEN2_0_5B_DECODELayerOffsets::w1, QWEN2_0_5B_DECODELayerOffsets::w2, QWEN2_0_5B_DECODELayerOffsets::wk, QWEN2_0_5B_DECODELayerOffsets::wo, QWEN2_0_5B_DECODELayerOffsets::wq, and QWEN2_0_5B_DECODELayerOffsets::wv.

Referenced by qwen2_0_5b_decode_forward_prefill_impl().

qwen2_0_5b_decode_layer_3_decode()

static void qwen2_0_5b_decode_layer_3_decode ( QWEN2_0_5B_DECODEModel *  model, int  token_index, int  aligned_embed_dim, int  aligned_head_dim, int  aligned_intermediate_dim, int  aligned_context_window  )

static

Definition at line 4948 of file v6.5/test_generated/qwen2_int8.c.

  {
    const QWEN2_0_5B_DECODELayerOffsets *L = &QWEN2_0_5B_DECODE_LAYERS[3];
 
    float *input = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[2].output);
 
    float *ln1_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln1_gamma);
    float *ln1_out = QWEN2_0_5B_DECODE_PTR(model, L->ln1_out);
    float *ln2_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln2_gamma);
    float *ln2_out = QWEN2_0_5B_DECODE_PTR(model, L->ln2_out);
    float *k_cache = QWEN2_0_5B_DECODE_PTR(model, L->k);
    float *v_cache = QWEN2_0_5B_DECODE_PTR(model, L->v);
    float *proj_tmp = QWEN2_0_5B_DECODE_PTR(model, L->proj_tmp);
    float *proj_scratch = QWEN2_0_5B_DECODE_PTR(model, L->proj_scratch);
    float *residual1 = QWEN2_0_5B_DECODE_PTR(model, L->residual1);
    float *mlp_out = QWEN2_0_5B_DECODE_PTR(model, L->mlp_out);
    float *output = QWEN2_0_5B_DECODE_PTR(model, L->output);
 
    /* Weights (explicit types for layer 3) */
    const void *WQ = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wq);  /* Q4_K */
    const void *WK = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wk);  /* Q4_K */
    const void *WV = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wv);  /* Q4_K */
    const void *WO = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wo);  /* Q4_K */
    const void *W1 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w1);  /* Q4_K (gate+up) */
    const void *W2 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w2);  /* Q4_K (down) */
 
    float *rope_cos = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_cos_cache);
    float *rope_sin = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_sin_cache);
 
    const int H = QWEN2_0_5B_DECODE_NUM_HEADS;
    const int H_kv = QWEN2_0_5B_DECODE_NUM_KV_HEADS;
    const int head_dim = QWEN2_0_5B_DECODE_HEAD_DIM;
 
    float q_token[H * aligned_head_dim];
    float k_token[H_kv * aligned_head_dim];
    float v_token[H_kv * aligned_head_dim];
    float attn_token[H * aligned_head_dim];
 
    /* Local MLP buffers (avoid layout dependencies for intermediate values) */
    float fc1_out[2 * aligned_intermediate_dim];
    float swiglu_out[aligned_intermediate_dim];
 
    /* Step 1: RMSNorm before attention */
    rmsnorm_forward(input,
                    ln1_gamma,
                    ln1_out,
                    NULL,
                    1,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    const size_t kv_head_stride = (size_t)aligned_context_window * (size_t)aligned_head_dim;
 
    /* Step 2: QKV projection */
    /* Q projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t ln1_q8_bytes = ((((aligned_embed_dim) + 255) / 256) * 292);
    uint8_t ln1_q8[ln1_q8_bytes];
    quantize_row_q8_k(ln1_out, ln1_q8, aligned_embed_dim);
    gemv_q4_k_q8_k(q_token, WQ, ln1_q8, H * head_dim, aligned_embed_dim);
    if (aligned_head_dim > head_dim) {
        for (int h = 0; h < H; ++h) {
            float *q_head = q_token + (size_t)h * (size_t)aligned_head_dim;
            for (int d = head_dim; d < aligned_head_dim; ++d) {
                q_head[d] = 0.0f;
            }
        }
    }
 
    /* K projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k (direct-to-cache) */
    const size_t wk_head_elems = (size_t)aligned_head_dim * (size_t)aligned_embed_dim;
    const size_t wk_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, wk_head_elems);
    const uint8_t *WK_bytes = (const uint8_t *)WK;
    /* ln1_q8 already quantized above */
    for (int h = 0; h < H_kv; ++h) {
        const void *wk_h = (const void *)(WK_bytes + (size_t)h * wk_head_bytes);
        float *k_head = k_cache + (size_t)h * kv_head_stride + (size_t)token_index * (size_t)aligned_head_dim;
        gemv_q4_k_q8_k(k_head, wk_h, ln1_q8, head_dim, aligned_embed_dim);
        for (int d = head_dim; d < aligned_head_dim; ++d) {
            k_head[d] = 0.0f;
        }
    }
 
    /* V projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k (direct-to-cache) */
    const size_t wv_head_elems = (size_t)aligned_head_dim * (size_t)aligned_embed_dim;
    const size_t wv_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, wv_head_elems);
    const uint8_t *WV_bytes = (const uint8_t *)WV;
    /* ln1_q8 already quantized above */
    for (int h = 0; h < H_kv; ++h) {
        const void *wv_h = (const void *)(WV_bytes + (size_t)h * wv_head_bytes);
        float *v_head = v_cache + (size_t)h * kv_head_stride + (size_t)token_index * (size_t)aligned_head_dim;
        gemv_q4_k_q8_k(v_head, wv_h, ln1_q8, head_dim, aligned_embed_dim);
        for (int d = head_dim; d < aligned_head_dim; ++d) {
            v_head[d] = 0.0f;
        }
    }
 
    /* Step 3: RoPE */
    rope_forward(q_token,
                 rope_cos,
                 rope_sin,
                 H,
                 1,
                 head_dim,
                 aligned_head_dim,
                 token_index);
    for (int h = 0; h < H_kv; ++h) {
        float *k_head = k_cache + (size_t)h * kv_head_stride + (size_t)token_index * (size_t)aligned_head_dim;
        rope_forward(k_head,
                     rope_cos,
                     rope_sin,
                     1,
                     1,
                     head_dim,
                     aligned_head_dim,
                     token_index);
    }
 
    /* Step 4: KV cache write (direct-to-cache) */
 
    /* Step 5: Attention (decode, flash) */
    attention_forward_decode_head_major_gqa_flash(q_token,
                                                   k_cache,
                                                   v_cache,
                                                   attn_token,
                                                   H,
                                                   H_kv,
                                                   token_index + 1,
                                                   aligned_context_window,
                                                   head_dim,
                                                   aligned_head_dim);
 
    /* Step 6: Output projection */
    /* WO projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t attn_q8_bytes = ((((H * head_dim) + 255) / 256) * 292);
    uint8_t attn_q8[attn_q8_bytes];
    quantize_row_q8_k(attn_token, attn_q8, H * head_dim);
    gemv_q4_k_q8_k(proj_tmp, WO, attn_q8, aligned_embed_dim, H * head_dim);
 
    /* Step 7: Residual add */
    qwen2_0_5b_decode_residual_add_token_major(input, proj_tmp, residual1, 1, aligned_embed_dim);
 
    /* Step 8: RMSNorm before MLP */
    rmsnorm_forward(residual1,
                    ln2_gamma,
                    ln2_out,
                    NULL,
                    1,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    /* Step 9: MLP (SwiGLU) */
    /* Gate+Up projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t ln2_q8_bytes = ((((aligned_embed_dim) + 255) / 256) * 292);
    uint8_t ln2_q8[ln2_q8_bytes];
    quantize_row_q8_k(ln2_out, ln2_q8, aligned_embed_dim);
    gemv_q4_k_q8_k(fc1_out, W1, ln2_q8, 2 * aligned_intermediate_dim, aligned_embed_dim);
 
    /* SwiGLU activation */
    swiglu_forward(fc1_out, swiglu_out, 1, aligned_intermediate_dim);
 
    /* Down projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t swiglu_q8_bytes = ((((aligned_intermediate_dim) + 255) / 256) * 292);
    uint8_t swiglu_q8[swiglu_q8_bytes];
    quantize_row_q8_k(swiglu_out, swiglu_q8, aligned_intermediate_dim);
    gemv_q4_k_q8_k(mlp_out, W2, swiglu_q8, aligned_embed_dim, aligned_intermediate_dim);
 
    /* Step 10: Final residual add */
    qwen2_0_5b_decode_residual_add_token_major(residual1, mlp_out, output, 1, aligned_embed_dim);
}

References attention_forward_decode_head_major_gqa_flash(), CK_DT_Q4_K, ck_dtype_row_bytes(), gemv_q4_k_q8_k(), QWEN2_0_5B_DECODELayerOffsets::k, QWEN2_0_5B_DECODELayerOffsets::ln1_gamma, QWEN2_0_5B_DECODELayerOffsets::ln1_out, QWEN2_0_5B_DECODELayerOffsets::ln2_gamma, QWEN2_0_5B_DECODELayerOffsets::ln2_out, QWEN2_0_5B_DECODELayerOffsets::mlp_out, QWEN2_0_5B_DECODELayerOffsets::output, QWEN2_0_5B_DECODELayerOffsets::proj_scratch, QWEN2_0_5B_DECODELayerOffsets::proj_tmp, quantize_row_q8_k(), QWEN2_0_5B_DECODE_EMBED_DIM, QWEN2_0_5B_DECODE_GLOBALS, QWEN2_0_5B_DECODE_HEAD_DIM, QWEN2_0_5B_DECODE_LAYERS, QWEN2_0_5B_DECODE_NUM_HEADS, QWEN2_0_5B_DECODE_NUM_KV_HEADS, QWEN2_0_5B_DECODE_PTR, qwen2_0_5b_decode_residual_add_token_major(), QWEN2_0_5B_DECODELayerOffsets::residual1, rmsnorm_forward(), QWEN2_0_5B_DECODEGlobalOffsets::rope_cos_cache, rope_forward(), QWEN2_0_5B_DECODEGlobalOffsets::rope_sin_cache, swiglu_forward(), QWEN2_0_5B_DECODELayerOffsets::v, QWEN2_0_5B_DECODELayerOffsets::w1, QWEN2_0_5B_DECODELayerOffsets::w2, QWEN2_0_5B_DECODELayerOffsets::wk, QWEN2_0_5B_DECODELayerOffsets::wo, QWEN2_0_5B_DECODELayerOffsets::wq, and QWEN2_0_5B_DECODELayerOffsets::wv.

Referenced by qwen2_0_5b_decode_decode_token().

qwen2_0_5b_decode_layer_3_prefill()

static void qwen2_0_5b_decode_layer_3_prefill ( QWEN2_0_5B_DECODEModel *  model, int  num_tokens, int  aligned_embed_dim, int  aligned_head_dim, int  aligned_intermediate_dim, int  aligned_context_window  )

static

Definition at line 703 of file v6.5/test_generated/qwen2_int8.c.

  {
    const QWEN2_0_5B_DECODELayerOffsets *L = &QWEN2_0_5B_DECODE_LAYERS[3];
 
    float *input = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[2].output);
    float *ln1_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln1_gamma);
    float *ln1_out = QWEN2_0_5B_DECODE_PTR(model, L->ln1_out);
    float *ln2_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln2_gamma);
    float *ln2_out = QWEN2_0_5B_DECODE_PTR(model, L->ln2_out);
    float *q = QWEN2_0_5B_DECODE_PTR(model, L->q);
    float *k = QWEN2_0_5B_DECODE_PTR(model, L->k);
    float *v = QWEN2_0_5B_DECODE_PTR(model, L->v);
    float *attn_out = QWEN2_0_5B_DECODE_PTR(model, L->attn_out);
    float *proj_tmp = QWEN2_0_5B_DECODE_PTR(model, L->proj_tmp);
    float *proj_scratch = QWEN2_0_5B_DECODE_PTR(model, L->proj_scratch);
    float *residual1 = QWEN2_0_5B_DECODE_PTR(model, L->residual1);
    float *fc1_out = QWEN2_0_5B_DECODE_PTR(model, L->fc1_out);
    float *swiglu_out = QWEN2_0_5B_DECODE_PTR(model, L->swiglu_out);
    float *mlp_out = QWEN2_0_5B_DECODE_PTR(model, L->mlp_out);
    float *output = QWEN2_0_5B_DECODE_PTR(model, L->output);
 
    const void *WQ = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wq);
    const void *WK = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wk);
    const void *WV = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wv);
    const void *WO = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wo);
    const void *W1 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w1);
    const void *W2 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w2);
    const float *BQ = NULL;
    const float *BK = NULL;
    const float *BV = NULL;
    const float *BO = NULL;
    const float *B1 = NULL;
    const float *B2 = NULL;
 
    float *rope_cos = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_cos_cache);
    float *rope_sin = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_sin_cache);
 
    const int H = QWEN2_0_5B_DECODE_NUM_HEADS;
    const int H_kv = QWEN2_0_5B_DECODE_NUM_KV_HEADS;
    const int head_dim = QWEN2_0_5B_DECODE_HEAD_DIM;
    const size_t head_w_elems = (size_t)aligned_head_dim * (size_t)aligned_embed_dim;
    const size_t q_head_stride = (size_t)num_tokens * (size_t)aligned_head_dim;
    const size_t kv_head_stride = (size_t)aligned_context_window * (size_t)aligned_head_dim;
 
    /* RMSNorm before attention */
    rmsnorm_forward(input,
                    ln1_gamma,
                    ln1_out,
                    NULL,
                    num_tokens,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    /* Q projection (head-major) */
    const size_t wq_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, head_w_elems);
    const uint8_t *WQ_bytes = (const uint8_t *)WQ;
    for (int h = 0; h < H; ++h) {
        const void *wq_h = (const void *)(WQ_bytes + (size_t)h * wq_head_bytes);
        const float *bq_h = BQ ? (BQ + (size_t)h * (size_t)aligned_head_dim) : NULL;
        float *q_h = q + (size_t)h * q_head_stride;
        gemm_nt_q4_k(ln1_out, wq_h, bq_h, q_h, num_tokens, aligned_head_dim, aligned_embed_dim);
    }
 
    /* K projection (head-major) */
    const size_t wk_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, head_w_elems);
    const uint8_t *WK_bytes = (const uint8_t *)WK;
    for (int h = 0; h < H_kv; ++h) {
        const void *wk_h = (const void *)(WK_bytes + (size_t)h * wk_head_bytes);
        const float *bk_h = BK ? (BK + (size_t)h * (size_t)aligned_head_dim) : NULL;
        float *k_h = k + (size_t)h * kv_head_stride;
        gemm_nt_q4_k(ln1_out, wk_h, bk_h, k_h, num_tokens, aligned_head_dim, aligned_embed_dim);
    }
 
    /* V projection (head-major) */
    const size_t wv_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, head_w_elems);
    const uint8_t *WV_bytes = (const uint8_t *)WV;
    for (int h = 0; h < H_kv; ++h) {
        const void *wv_h = (const void *)(WV_bytes + (size_t)h * wv_head_bytes);
        const float *bv_h = BV ? (BV + (size_t)h * (size_t)aligned_head_dim) : NULL;
        float *v_h = v + (size_t)h * kv_head_stride;
        gemm_nt_q4_k(ln1_out, wv_h, bv_h, v_h, num_tokens, aligned_head_dim, aligned_embed_dim);
    }
 
    /* RoPE */
    rope_forward_qk_strided(q,
                            k,
                            rope_cos,
                            rope_sin,
                            H,
                            H_kv,
                            num_tokens,
                            head_dim,
                            aligned_head_dim,
                            0,
                            num_tokens,
                            aligned_context_window);
 
    /* Attention (prefill, causal) */
    attention_forward_causal_head_major_gqa_flash_strided(q,
                                                           k,
                                                           v,
                                                           attn_out,
                                                           H,
                                                           H_kv,
                                                           num_tokens,
                                                           head_dim,
                                                           aligned_head_dim,
                                                           aligned_context_window);
 
    /* Output projection (flatten head-major to token-major) */
    const int K = H * aligned_head_dim;
    if (K != aligned_embed_dim) {
        return;
    }
    const float *proj_in = attn_out;
    if (H > 1) {
        if (!proj_scratch) {
            return;
        }
        for (int t = 0; t < num_tokens; ++t) {
            float *dst = proj_scratch + (size_t)t * (size_t)aligned_embed_dim;
            for (int h = 0; h < H; ++h) {
                const float *src = attn_out + (size_t)h * q_head_stride + (size_t)t * (size_t)aligned_head_dim;
                memcpy(dst + (size_t)h * (size_t)aligned_head_dim,
                       src,
                       (size_t)aligned_head_dim * sizeof(float));
            }
        }
        proj_in = proj_scratch;
    }
    gemm_nt_q4_k(proj_in, WO, BO, proj_tmp, num_tokens, aligned_embed_dim, K);
 
    /* Residual add */
    qwen2_0_5b_decode_residual_add_token_major(input, proj_tmp, residual1, num_tokens, aligned_embed_dim);
 
    /* RMSNorm before MLP */
    rmsnorm_forward(residual1,
                    ln2_gamma,
                    ln2_out,
                    NULL,
                    num_tokens,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    /* MLP (SwiGLU) */
    gemm_nt_q4_k(ln2_out, W1, B1, fc1_out, num_tokens, 2 * aligned_intermediate_dim, aligned_embed_dim);
    swiglu_forward(fc1_out, swiglu_out, num_tokens, aligned_intermediate_dim);
    gemm_nt_q4_k(swiglu_out, W2, B2, mlp_out, num_tokens, aligned_embed_dim, aligned_intermediate_dim);
 
    /* Final residual add */
    qwen2_0_5b_decode_residual_add_token_major(residual1, mlp_out, output, num_tokens, aligned_embed_dim);
}

References attention_forward_causal_head_major_gqa_flash_strided(), CK_DT_Q4_K, ck_dtype_row_bytes(), gemm_nt_q4_k(), QWEN2_0_5B_DECODELayerOffsets::k, QWEN2_0_5B_DECODELayerOffsets::ln1_gamma, QWEN2_0_5B_DECODELayerOffsets::ln1_out, QWEN2_0_5B_DECODELayerOffsets::ln2_gamma, QWEN2_0_5B_DECODELayerOffsets::ln2_out, QWEN2_0_5B_DECODELayerOffsets::mlp_out, QWEN2_0_5B_DECODELayerOffsets::output, QWEN2_0_5B_DECODELayerOffsets::proj_scratch, QWEN2_0_5B_DECODELayerOffsets::proj_tmp, QWEN2_0_5B_DECODELayerOffsets::q, QWEN2_0_5B_DECODE_EMBED_DIM, QWEN2_0_5B_DECODE_GLOBALS, QWEN2_0_5B_DECODE_HEAD_DIM, QWEN2_0_5B_DECODE_LAYERS, QWEN2_0_5B_DECODE_NUM_HEADS, QWEN2_0_5B_DECODE_NUM_KV_HEADS, QWEN2_0_5B_DECODE_PTR, qwen2_0_5b_decode_residual_add_token_major(), QWEN2_0_5B_DECODELayerOffsets::residual1, rmsnorm_forward(), QWEN2_0_5B_DECODEGlobalOffsets::rope_cos_cache, rope_forward_qk_strided(), QWEN2_0_5B_DECODEGlobalOffsets::rope_sin_cache, swiglu_forward(), QWEN2_0_5B_DECODELayerOffsets::v, QWEN2_0_5B_DECODELayerOffsets::w1, QWEN2_0_5B_DECODELayerOffsets::w2, QWEN2_0_5B_DECODELayerOffsets::wk, QWEN2_0_5B_DECODELayerOffsets::wo, QWEN2_0_5B_DECODELayerOffsets::wq, and QWEN2_0_5B_DECODELayerOffsets::wv.

Referenced by qwen2_0_5b_decode_forward_prefill_impl().

qwen2_0_5b_decode_layer_4_decode()

static void qwen2_0_5b_decode_layer_4_decode ( QWEN2_0_5B_DECODEModel *  model, int  token_index, int  aligned_embed_dim, int  aligned_head_dim, int  aligned_intermediate_dim, int  aligned_context_window  )

static

Definition at line 5129 of file v6.5/test_generated/qwen2_int8.c.

  {
    const QWEN2_0_5B_DECODELayerOffsets *L = &QWEN2_0_5B_DECODE_LAYERS[4];
 
    float *input = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[3].output);
 
    float *ln1_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln1_gamma);
    float *ln1_out = QWEN2_0_5B_DECODE_PTR(model, L->ln1_out);
    float *ln2_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln2_gamma);
    float *ln2_out = QWEN2_0_5B_DECODE_PTR(model, L->ln2_out);
    float *k_cache = QWEN2_0_5B_DECODE_PTR(model, L->k);
    float *v_cache = QWEN2_0_5B_DECODE_PTR(model, L->v);
    float *proj_tmp = QWEN2_0_5B_DECODE_PTR(model, L->proj_tmp);
    float *proj_scratch = QWEN2_0_5B_DECODE_PTR(model, L->proj_scratch);
    float *residual1 = QWEN2_0_5B_DECODE_PTR(model, L->residual1);
    float *mlp_out = QWEN2_0_5B_DECODE_PTR(model, L->mlp_out);
    float *output = QWEN2_0_5B_DECODE_PTR(model, L->output);
 
    /* Weights (explicit types for layer 4) */
    const void *WQ = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wq);  /* Q4_K */
    const void *WK = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wk);  /* Q4_K */
    const void *WV = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wv);  /* Q4_K */
    const void *WO = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wo);  /* Q4_K */
    const void *W1 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w1);  /* Q4_K (gate+up) */
    const void *W2 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w2);  /* Q4_K (down) */
 
    float *rope_cos = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_cos_cache);
    float *rope_sin = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_sin_cache);
 
    const int H = QWEN2_0_5B_DECODE_NUM_HEADS;
    const int H_kv = QWEN2_0_5B_DECODE_NUM_KV_HEADS;
    const int head_dim = QWEN2_0_5B_DECODE_HEAD_DIM;
 
    float q_token[H * aligned_head_dim];
    float k_token[H_kv * aligned_head_dim];
    float v_token[H_kv * aligned_head_dim];
    float attn_token[H * aligned_head_dim];
 
    /* Local MLP buffers (avoid layout dependencies for intermediate values) */
    float fc1_out[2 * aligned_intermediate_dim];
    float swiglu_out[aligned_intermediate_dim];
 
    /* Step 1: RMSNorm before attention */
    rmsnorm_forward(input,
                    ln1_gamma,
                    ln1_out,
                    NULL,
                    1,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    const size_t kv_head_stride = (size_t)aligned_context_window * (size_t)aligned_head_dim;
 
    /* Step 2: QKV projection */
    /* Q projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t ln1_q8_bytes = ((((aligned_embed_dim) + 255) / 256) * 292);
    uint8_t ln1_q8[ln1_q8_bytes];
    quantize_row_q8_k(ln1_out, ln1_q8, aligned_embed_dim);
    gemv_q4_k_q8_k(q_token, WQ, ln1_q8, H * head_dim, aligned_embed_dim);
    if (aligned_head_dim > head_dim) {
        for (int h = 0; h < H; ++h) {
            float *q_head = q_token + (size_t)h * (size_t)aligned_head_dim;
            for (int d = head_dim; d < aligned_head_dim; ++d) {
                q_head[d] = 0.0f;
            }
        }
    }
 
    /* K projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k (direct-to-cache) */
    const size_t wk_head_elems = (size_t)aligned_head_dim * (size_t)aligned_embed_dim;
    const size_t wk_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, wk_head_elems);
    const uint8_t *WK_bytes = (const uint8_t *)WK;
    /* ln1_q8 already quantized above */
    for (int h = 0; h < H_kv; ++h) {
        const void *wk_h = (const void *)(WK_bytes + (size_t)h * wk_head_bytes);
        float *k_head = k_cache + (size_t)h * kv_head_stride + (size_t)token_index * (size_t)aligned_head_dim;
        gemv_q4_k_q8_k(k_head, wk_h, ln1_q8, head_dim, aligned_embed_dim);
        for (int d = head_dim; d < aligned_head_dim; ++d) {
            k_head[d] = 0.0f;
        }
    }
 
    /* V projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k (direct-to-cache) */
    const size_t wv_head_elems = (size_t)aligned_head_dim * (size_t)aligned_embed_dim;
    const size_t wv_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, wv_head_elems);
    const uint8_t *WV_bytes = (const uint8_t *)WV;
    /* ln1_q8 already quantized above */
    for (int h = 0; h < H_kv; ++h) {
        const void *wv_h = (const void *)(WV_bytes + (size_t)h * wv_head_bytes);
        float *v_head = v_cache + (size_t)h * kv_head_stride + (size_t)token_index * (size_t)aligned_head_dim;
        gemv_q4_k_q8_k(v_head, wv_h, ln1_q8, head_dim, aligned_embed_dim);
        for (int d = head_dim; d < aligned_head_dim; ++d) {
            v_head[d] = 0.0f;
        }
    }
 
    /* Step 3: RoPE */
    rope_forward(q_token,
                 rope_cos,
                 rope_sin,
                 H,
                 1,
                 head_dim,
                 aligned_head_dim,
                 token_index);
    for (int h = 0; h < H_kv; ++h) {
        float *k_head = k_cache + (size_t)h * kv_head_stride + (size_t)token_index * (size_t)aligned_head_dim;
        rope_forward(k_head,
                     rope_cos,
                     rope_sin,
                     1,
                     1,
                     head_dim,
                     aligned_head_dim,
                     token_index);
    }
 
    /* Step 4: KV cache write (direct-to-cache) */
 
    /* Step 5: Attention (decode, flash) */
    attention_forward_decode_head_major_gqa_flash(q_token,
                                                   k_cache,
                                                   v_cache,
                                                   attn_token,
                                                   H,
                                                   H_kv,
                                                   token_index + 1,
                                                   aligned_context_window,
                                                   head_dim,
                                                   aligned_head_dim);
 
    /* Step 6: Output projection */
    /* WO projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t attn_q8_bytes = ((((H * head_dim) + 255) / 256) * 292);
    uint8_t attn_q8[attn_q8_bytes];
    quantize_row_q8_k(attn_token, attn_q8, H * head_dim);
    gemv_q4_k_q8_k(proj_tmp, WO, attn_q8, aligned_embed_dim, H * head_dim);
 
    /* Step 7: Residual add */
    qwen2_0_5b_decode_residual_add_token_major(input, proj_tmp, residual1, 1, aligned_embed_dim);
 
    /* Step 8: RMSNorm before MLP */
    rmsnorm_forward(residual1,
                    ln2_gamma,
                    ln2_out,
                    NULL,
                    1,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    /* Step 9: MLP (SwiGLU) */
    /* Gate+Up projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t ln2_q8_bytes = ((((aligned_embed_dim) + 255) / 256) * 292);
    uint8_t ln2_q8[ln2_q8_bytes];
    quantize_row_q8_k(ln2_out, ln2_q8, aligned_embed_dim);
    gemv_q4_k_q8_k(fc1_out, W1, ln2_q8, 2 * aligned_intermediate_dim, aligned_embed_dim);
 
    /* SwiGLU activation */
    swiglu_forward(fc1_out, swiglu_out, 1, aligned_intermediate_dim);
 
    /* Down projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t swiglu_q8_bytes = ((((aligned_intermediate_dim) + 255) / 256) * 292);
    uint8_t swiglu_q8[swiglu_q8_bytes];
    quantize_row_q8_k(swiglu_out, swiglu_q8, aligned_intermediate_dim);
    gemv_q4_k_q8_k(mlp_out, W2, swiglu_q8, aligned_embed_dim, aligned_intermediate_dim);
 
    /* Step 10: Final residual add */
    qwen2_0_5b_decode_residual_add_token_major(residual1, mlp_out, output, 1, aligned_embed_dim);
}

References attention_forward_decode_head_major_gqa_flash(), CK_DT_Q4_K, ck_dtype_row_bytes(), gemv_q4_k_q8_k(), QWEN2_0_5B_DECODELayerOffsets::k, QWEN2_0_5B_DECODELayerOffsets::ln1_gamma, QWEN2_0_5B_DECODELayerOffsets::ln1_out, QWEN2_0_5B_DECODELayerOffsets::ln2_gamma, QWEN2_0_5B_DECODELayerOffsets::ln2_out, QWEN2_0_5B_DECODELayerOffsets::mlp_out, QWEN2_0_5B_DECODELayerOffsets::output, QWEN2_0_5B_DECODELayerOffsets::proj_scratch, QWEN2_0_5B_DECODELayerOffsets::proj_tmp, quantize_row_q8_k(), QWEN2_0_5B_DECODE_EMBED_DIM, QWEN2_0_5B_DECODE_GLOBALS, QWEN2_0_5B_DECODE_HEAD_DIM, QWEN2_0_5B_DECODE_LAYERS, QWEN2_0_5B_DECODE_NUM_HEADS, QWEN2_0_5B_DECODE_NUM_KV_HEADS, QWEN2_0_5B_DECODE_PTR, qwen2_0_5b_decode_residual_add_token_major(), QWEN2_0_5B_DECODELayerOffsets::residual1, rmsnorm_forward(), QWEN2_0_5B_DECODEGlobalOffsets::rope_cos_cache, rope_forward(), QWEN2_0_5B_DECODEGlobalOffsets::rope_sin_cache, swiglu_forward(), QWEN2_0_5B_DECODELayerOffsets::v, QWEN2_0_5B_DECODELayerOffsets::w1, QWEN2_0_5B_DECODELayerOffsets::w2, QWEN2_0_5B_DECODELayerOffsets::wk, QWEN2_0_5B_DECODELayerOffsets::wo, QWEN2_0_5B_DECODELayerOffsets::wq, and QWEN2_0_5B_DECODELayerOffsets::wv.

Referenced by qwen2_0_5b_decode_decode_token().

qwen2_0_5b_decode_layer_4_prefill()

static void qwen2_0_5b_decode_layer_4_prefill ( QWEN2_0_5B_DECODEModel *  model, int  num_tokens, int  aligned_embed_dim, int  aligned_head_dim, int  aligned_intermediate_dim, int  aligned_context_window  )

static

Definition at line 867 of file v6.5/test_generated/qwen2_int8.c.

  {
    const QWEN2_0_5B_DECODELayerOffsets *L = &QWEN2_0_5B_DECODE_LAYERS[4];
 
    float *input = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[3].output);
    float *ln1_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln1_gamma);
    float *ln1_out = QWEN2_0_5B_DECODE_PTR(model, L->ln1_out);
    float *ln2_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln2_gamma);
    float *ln2_out = QWEN2_0_5B_DECODE_PTR(model, L->ln2_out);
    float *q = QWEN2_0_5B_DECODE_PTR(model, L->q);
    float *k = QWEN2_0_5B_DECODE_PTR(model, L->k);
    float *v = QWEN2_0_5B_DECODE_PTR(model, L->v);
    float *attn_out = QWEN2_0_5B_DECODE_PTR(model, L->attn_out);
    float *proj_tmp = QWEN2_0_5B_DECODE_PTR(model, L->proj_tmp);
    float *proj_scratch = QWEN2_0_5B_DECODE_PTR(model, L->proj_scratch);
    float *residual1 = QWEN2_0_5B_DECODE_PTR(model, L->residual1);
    float *fc1_out = QWEN2_0_5B_DECODE_PTR(model, L->fc1_out);
    float *swiglu_out = QWEN2_0_5B_DECODE_PTR(model, L->swiglu_out);
    float *mlp_out = QWEN2_0_5B_DECODE_PTR(model, L->mlp_out);
    float *output = QWEN2_0_5B_DECODE_PTR(model, L->output);
 
    const void *WQ = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wq);
    const void *WK = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wk);
    const void *WV = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wv);
    const void *WO = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wo);
    const void *W1 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w1);
    const void *W2 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w2);
    const float *BQ = NULL;
    const float *BK = NULL;
    const float *BV = NULL;
    const float *BO = NULL;
    const float *B1 = NULL;
    const float *B2 = NULL;
 
    float *rope_cos = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_cos_cache);
    float *rope_sin = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_sin_cache);
 
    const int H = QWEN2_0_5B_DECODE_NUM_HEADS;
    const int H_kv = QWEN2_0_5B_DECODE_NUM_KV_HEADS;
    const int head_dim = QWEN2_0_5B_DECODE_HEAD_DIM;
    const size_t head_w_elems = (size_t)aligned_head_dim * (size_t)aligned_embed_dim;
    const size_t q_head_stride = (size_t)num_tokens * (size_t)aligned_head_dim;
    const size_t kv_head_stride = (size_t)aligned_context_window * (size_t)aligned_head_dim;
 
    /* RMSNorm before attention */
    rmsnorm_forward(input,
                    ln1_gamma,
                    ln1_out,
                    NULL,
                    num_tokens,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    /* Q projection (head-major) */
    const size_t wq_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, head_w_elems);
    const uint8_t *WQ_bytes = (const uint8_t *)WQ;
    for (int h = 0; h < H; ++h) {
        const void *wq_h = (const void *)(WQ_bytes + (size_t)h * wq_head_bytes);
        const float *bq_h = BQ ? (BQ + (size_t)h * (size_t)aligned_head_dim) : NULL;
        float *q_h = q + (size_t)h * q_head_stride;
        gemm_nt_q4_k(ln1_out, wq_h, bq_h, q_h, num_tokens, aligned_head_dim, aligned_embed_dim);
    }
 
    /* K projection (head-major) */
    const size_t wk_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, head_w_elems);
    const uint8_t *WK_bytes = (const uint8_t *)WK;
    for (int h = 0; h < H_kv; ++h) {
        const void *wk_h = (const void *)(WK_bytes + (size_t)h * wk_head_bytes);
        const float *bk_h = BK ? (BK + (size_t)h * (size_t)aligned_head_dim) : NULL;
        float *k_h = k + (size_t)h * kv_head_stride;
        gemm_nt_q4_k(ln1_out, wk_h, bk_h, k_h, num_tokens, aligned_head_dim, aligned_embed_dim);
    }
 
    /* V projection (head-major) */
    const size_t wv_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, head_w_elems);
    const uint8_t *WV_bytes = (const uint8_t *)WV;
    for (int h = 0; h < H_kv; ++h) {
        const void *wv_h = (const void *)(WV_bytes + (size_t)h * wv_head_bytes);
        const float *bv_h = BV ? (BV + (size_t)h * (size_t)aligned_head_dim) : NULL;
        float *v_h = v + (size_t)h * kv_head_stride;
        gemm_nt_q4_k(ln1_out, wv_h, bv_h, v_h, num_tokens, aligned_head_dim, aligned_embed_dim);
    }
 
    /* RoPE */
    rope_forward_qk_strided(q,
                            k,
                            rope_cos,
                            rope_sin,
                            H,
                            H_kv,
                            num_tokens,
                            head_dim,
                            aligned_head_dim,
                            0,
                            num_tokens,
                            aligned_context_window);
 
    /* Attention (prefill, causal) */
    attention_forward_causal_head_major_gqa_flash_strided(q,
                                                           k,
                                                           v,
                                                           attn_out,
                                                           H,
                                                           H_kv,
                                                           num_tokens,
                                                           head_dim,
                                                           aligned_head_dim,
                                                           aligned_context_window);
 
    /* Output projection (flatten head-major to token-major) */
    const int K = H * aligned_head_dim;
    if (K != aligned_embed_dim) {
        return;
    }
    const float *proj_in = attn_out;
    if (H > 1) {
        if (!proj_scratch) {
            return;
        }
        for (int t = 0; t < num_tokens; ++t) {
            float *dst = proj_scratch + (size_t)t * (size_t)aligned_embed_dim;
            for (int h = 0; h < H; ++h) {
                const float *src = attn_out + (size_t)h * q_head_stride + (size_t)t * (size_t)aligned_head_dim;
                memcpy(dst + (size_t)h * (size_t)aligned_head_dim,
                       src,
                       (size_t)aligned_head_dim * sizeof(float));
            }
        }
        proj_in = proj_scratch;
    }
    gemm_nt_q4_k(proj_in, WO, BO, proj_tmp, num_tokens, aligned_embed_dim, K);
 
    /* Residual add */
    qwen2_0_5b_decode_residual_add_token_major(input, proj_tmp, residual1, num_tokens, aligned_embed_dim);
 
    /* RMSNorm before MLP */
    rmsnorm_forward(residual1,
                    ln2_gamma,
                    ln2_out,
                    NULL,
                    num_tokens,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    /* MLP (SwiGLU) */
    gemm_nt_q4_k(ln2_out, W1, B1, fc1_out, num_tokens, 2 * aligned_intermediate_dim, aligned_embed_dim);
    swiglu_forward(fc1_out, swiglu_out, num_tokens, aligned_intermediate_dim);
    gemm_nt_q4_k(swiglu_out, W2, B2, mlp_out, num_tokens, aligned_embed_dim, aligned_intermediate_dim);
 
    /* Final residual add */
    qwen2_0_5b_decode_residual_add_token_major(residual1, mlp_out, output, num_tokens, aligned_embed_dim);
}

References attention_forward_causal_head_major_gqa_flash_strided(), CK_DT_Q4_K, ck_dtype_row_bytes(), gemm_nt_q4_k(), QWEN2_0_5B_DECODELayerOffsets::k, QWEN2_0_5B_DECODELayerOffsets::ln1_gamma, QWEN2_0_5B_DECODELayerOffsets::ln1_out, QWEN2_0_5B_DECODELayerOffsets::ln2_gamma, QWEN2_0_5B_DECODELayerOffsets::ln2_out, QWEN2_0_5B_DECODELayerOffsets::mlp_out, QWEN2_0_5B_DECODELayerOffsets::output, QWEN2_0_5B_DECODELayerOffsets::proj_scratch, QWEN2_0_5B_DECODELayerOffsets::proj_tmp, QWEN2_0_5B_DECODELayerOffsets::q, QWEN2_0_5B_DECODE_EMBED_DIM, QWEN2_0_5B_DECODE_GLOBALS, QWEN2_0_5B_DECODE_HEAD_DIM, QWEN2_0_5B_DECODE_LAYERS, QWEN2_0_5B_DECODE_NUM_HEADS, QWEN2_0_5B_DECODE_NUM_KV_HEADS, QWEN2_0_5B_DECODE_PTR, qwen2_0_5b_decode_residual_add_token_major(), QWEN2_0_5B_DECODELayerOffsets::residual1, rmsnorm_forward(), QWEN2_0_5B_DECODEGlobalOffsets::rope_cos_cache, rope_forward_qk_strided(), QWEN2_0_5B_DECODEGlobalOffsets::rope_sin_cache, swiglu_forward(), QWEN2_0_5B_DECODELayerOffsets::v, QWEN2_0_5B_DECODELayerOffsets::w1, QWEN2_0_5B_DECODELayerOffsets::w2, QWEN2_0_5B_DECODELayerOffsets::wk, QWEN2_0_5B_DECODELayerOffsets::wo, QWEN2_0_5B_DECODELayerOffsets::wq, and QWEN2_0_5B_DECODELayerOffsets::wv.

Referenced by qwen2_0_5b_decode_forward_prefill_impl().

qwen2_0_5b_decode_layer_5_decode()

static void qwen2_0_5b_decode_layer_5_decode ( QWEN2_0_5B_DECODEModel *  model, int  token_index, int  aligned_embed_dim, int  aligned_head_dim, int  aligned_intermediate_dim, int  aligned_context_window  )

static

Definition at line 5310 of file v6.5/test_generated/qwen2_int8.c.

  {
    const QWEN2_0_5B_DECODELayerOffsets *L = &QWEN2_0_5B_DECODE_LAYERS[5];
 
    float *input = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[4].output);
 
    float *ln1_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln1_gamma);
    float *ln1_out = QWEN2_0_5B_DECODE_PTR(model, L->ln1_out);
    float *ln2_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln2_gamma);
    float *ln2_out = QWEN2_0_5B_DECODE_PTR(model, L->ln2_out);
    float *k_cache = QWEN2_0_5B_DECODE_PTR(model, L->k);
    float *v_cache = QWEN2_0_5B_DECODE_PTR(model, L->v);
    float *proj_tmp = QWEN2_0_5B_DECODE_PTR(model, L->proj_tmp);
    float *proj_scratch = QWEN2_0_5B_DECODE_PTR(model, L->proj_scratch);
    float *residual1 = QWEN2_0_5B_DECODE_PTR(model, L->residual1);
    float *mlp_out = QWEN2_0_5B_DECODE_PTR(model, L->mlp_out);
    float *output = QWEN2_0_5B_DECODE_PTR(model, L->output);
 
    /* Weights (explicit types for layer 5) */
    const void *WQ = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wq);  /* Q4_K */
    const void *WK = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wk);  /* Q4_K */
    const void *WV = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wv);  /* Q4_K */
    const void *WO = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wo);  /* Q4_K */
    const void *W1 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w1);  /* Q4_K (gate+up) */
    const void *W2 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w2);  /* Q4_K (down) */
 
    float *rope_cos = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_cos_cache);
    float *rope_sin = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_sin_cache);
 
    const int H = QWEN2_0_5B_DECODE_NUM_HEADS;
    const int H_kv = QWEN2_0_5B_DECODE_NUM_KV_HEADS;
    const int head_dim = QWEN2_0_5B_DECODE_HEAD_DIM;
 
    float q_token[H * aligned_head_dim];
    float k_token[H_kv * aligned_head_dim];
    float v_token[H_kv * aligned_head_dim];
    float attn_token[H * aligned_head_dim];
 
    /* Local MLP buffers (avoid layout dependencies for intermediate values) */
    float fc1_out[2 * aligned_intermediate_dim];
    float swiglu_out[aligned_intermediate_dim];
 
    /* Step 1: RMSNorm before attention */
    rmsnorm_forward(input,
                    ln1_gamma,
                    ln1_out,
                    NULL,
                    1,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    const size_t kv_head_stride = (size_t)aligned_context_window * (size_t)aligned_head_dim;
 
    /* Step 2: QKV projection */
    /* Q projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t ln1_q8_bytes = ((((aligned_embed_dim) + 255) / 256) * 292);
    uint8_t ln1_q8[ln1_q8_bytes];
    quantize_row_q8_k(ln1_out, ln1_q8, aligned_embed_dim);
    gemv_q4_k_q8_k(q_token, WQ, ln1_q8, H * head_dim, aligned_embed_dim);
    if (aligned_head_dim > head_dim) {
        for (int h = 0; h < H; ++h) {
            float *q_head = q_token + (size_t)h * (size_t)aligned_head_dim;
            for (int d = head_dim; d < aligned_head_dim; ++d) {
                q_head[d] = 0.0f;
            }
        }
    }
 
    /* K projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k (direct-to-cache) */
    const size_t wk_head_elems = (size_t)aligned_head_dim * (size_t)aligned_embed_dim;
    const size_t wk_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, wk_head_elems);
    const uint8_t *WK_bytes = (const uint8_t *)WK;
    /* ln1_q8 already quantized above */
    for (int h = 0; h < H_kv; ++h) {
        const void *wk_h = (const void *)(WK_bytes + (size_t)h * wk_head_bytes);
        float *k_head = k_cache + (size_t)h * kv_head_stride + (size_t)token_index * (size_t)aligned_head_dim;
        gemv_q4_k_q8_k(k_head, wk_h, ln1_q8, head_dim, aligned_embed_dim);
        for (int d = head_dim; d < aligned_head_dim; ++d) {
            k_head[d] = 0.0f;
        }
    }
 
    /* V projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k (direct-to-cache) */
    const size_t wv_head_elems = (size_t)aligned_head_dim * (size_t)aligned_embed_dim;
    const size_t wv_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, wv_head_elems);
    const uint8_t *WV_bytes = (const uint8_t *)WV;
    /* ln1_q8 already quantized above */
    for (int h = 0; h < H_kv; ++h) {
        const void *wv_h = (const void *)(WV_bytes + (size_t)h * wv_head_bytes);
        float *v_head = v_cache + (size_t)h * kv_head_stride + (size_t)token_index * (size_t)aligned_head_dim;
        gemv_q4_k_q8_k(v_head, wv_h, ln1_q8, head_dim, aligned_embed_dim);
        for (int d = head_dim; d < aligned_head_dim; ++d) {
            v_head[d] = 0.0f;
        }
    }
 
    /* Step 3: RoPE */
    rope_forward(q_token,
                 rope_cos,
                 rope_sin,
                 H,
                 1,
                 head_dim,
                 aligned_head_dim,
                 token_index);
    for (int h = 0; h < H_kv; ++h) {
        float *k_head = k_cache + (size_t)h * kv_head_stride + (size_t)token_index * (size_t)aligned_head_dim;
        rope_forward(k_head,
                     rope_cos,
                     rope_sin,
                     1,
                     1,
                     head_dim,
                     aligned_head_dim,
                     token_index);
    }
 
    /* Step 4: KV cache write (direct-to-cache) */
 
    /* Step 5: Attention (decode, flash) */
    attention_forward_decode_head_major_gqa_flash(q_token,
                                                   k_cache,
                                                   v_cache,
                                                   attn_token,
                                                   H,
                                                   H_kv,
                                                   token_index + 1,
                                                   aligned_context_window,
                                                   head_dim,
                                                   aligned_head_dim);
 
    /* Step 6: Output projection */
    /* WO projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t attn_q8_bytes = ((((H * head_dim) + 255) / 256) * 292);
    uint8_t attn_q8[attn_q8_bytes];
    quantize_row_q8_k(attn_token, attn_q8, H * head_dim);
    gemv_q4_k_q8_k(proj_tmp, WO, attn_q8, aligned_embed_dim, H * head_dim);
 
    /* Step 7: Residual add */
    qwen2_0_5b_decode_residual_add_token_major(input, proj_tmp, residual1, 1, aligned_embed_dim);
 
    /* Step 8: RMSNorm before MLP */
    rmsnorm_forward(residual1,
                    ln2_gamma,
                    ln2_out,
                    NULL,
                    1,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    /* Step 9: MLP (SwiGLU) */
    /* Gate+Up projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t ln2_q8_bytes = ((((aligned_embed_dim) + 255) / 256) * 292);
    uint8_t ln2_q8[ln2_q8_bytes];
    quantize_row_q8_k(ln2_out, ln2_q8, aligned_embed_dim);
    gemv_q4_k_q8_k(fc1_out, W1, ln2_q8, 2 * aligned_intermediate_dim, aligned_embed_dim);
 
    /* SwiGLU activation */
    swiglu_forward(fc1_out, swiglu_out, 1, aligned_intermediate_dim);
 
    /* Down projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t swiglu_q8_bytes = ((((aligned_intermediate_dim) + 255) / 256) * 292);
    uint8_t swiglu_q8[swiglu_q8_bytes];
    quantize_row_q8_k(swiglu_out, swiglu_q8, aligned_intermediate_dim);
    gemv_q4_k_q8_k(mlp_out, W2, swiglu_q8, aligned_embed_dim, aligned_intermediate_dim);
 
    /* Step 10: Final residual add */
    qwen2_0_5b_decode_residual_add_token_major(residual1, mlp_out, output, 1, aligned_embed_dim);
}

References attention_forward_decode_head_major_gqa_flash(), CK_DT_Q4_K, ck_dtype_row_bytes(), gemv_q4_k_q8_k(), QWEN2_0_5B_DECODELayerOffsets::k, QWEN2_0_5B_DECODELayerOffsets::ln1_gamma, QWEN2_0_5B_DECODELayerOffsets::ln1_out, QWEN2_0_5B_DECODELayerOffsets::ln2_gamma, QWEN2_0_5B_DECODELayerOffsets::ln2_out, QWEN2_0_5B_DECODELayerOffsets::mlp_out, QWEN2_0_5B_DECODELayerOffsets::output, QWEN2_0_5B_DECODELayerOffsets::proj_scratch, QWEN2_0_5B_DECODELayerOffsets::proj_tmp, quantize_row_q8_k(), QWEN2_0_5B_DECODE_EMBED_DIM, QWEN2_0_5B_DECODE_GLOBALS, QWEN2_0_5B_DECODE_HEAD_DIM, QWEN2_0_5B_DECODE_LAYERS, QWEN2_0_5B_DECODE_NUM_HEADS, QWEN2_0_5B_DECODE_NUM_KV_HEADS, QWEN2_0_5B_DECODE_PTR, qwen2_0_5b_decode_residual_add_token_major(), QWEN2_0_5B_DECODELayerOffsets::residual1, rmsnorm_forward(), QWEN2_0_5B_DECODEGlobalOffsets::rope_cos_cache, rope_forward(), QWEN2_0_5B_DECODEGlobalOffsets::rope_sin_cache, swiglu_forward(), QWEN2_0_5B_DECODELayerOffsets::v, QWEN2_0_5B_DECODELayerOffsets::w1, QWEN2_0_5B_DECODELayerOffsets::w2, QWEN2_0_5B_DECODELayerOffsets::wk, QWEN2_0_5B_DECODELayerOffsets::wo, QWEN2_0_5B_DECODELayerOffsets::wq, and QWEN2_0_5B_DECODELayerOffsets::wv.

Referenced by qwen2_0_5b_decode_decode_token().

qwen2_0_5b_decode_layer_5_prefill()

static void qwen2_0_5b_decode_layer_5_prefill ( QWEN2_0_5B_DECODEModel *  model, int  num_tokens, int  aligned_embed_dim, int  aligned_head_dim, int  aligned_intermediate_dim, int  aligned_context_window  )

static

Definition at line 1031 of file v6.5/test_generated/qwen2_int8.c.

  {
    const QWEN2_0_5B_DECODELayerOffsets *L = &QWEN2_0_5B_DECODE_LAYERS[5];
 
    float *input = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[4].output);
    float *ln1_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln1_gamma);
    float *ln1_out = QWEN2_0_5B_DECODE_PTR(model, L->ln1_out);
    float *ln2_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln2_gamma);
    float *ln2_out = QWEN2_0_5B_DECODE_PTR(model, L->ln2_out);
    float *q = QWEN2_0_5B_DECODE_PTR(model, L->q);
    float *k = QWEN2_0_5B_DECODE_PTR(model, L->k);
    float *v = QWEN2_0_5B_DECODE_PTR(model, L->v);
    float *attn_out = QWEN2_0_5B_DECODE_PTR(model, L->attn_out);
    float *proj_tmp = QWEN2_0_5B_DECODE_PTR(model, L->proj_tmp);
    float *proj_scratch = QWEN2_0_5B_DECODE_PTR(model, L->proj_scratch);
    float *residual1 = QWEN2_0_5B_DECODE_PTR(model, L->residual1);
    float *fc1_out = QWEN2_0_5B_DECODE_PTR(model, L->fc1_out);
    float *swiglu_out = QWEN2_0_5B_DECODE_PTR(model, L->swiglu_out);
    float *mlp_out = QWEN2_0_5B_DECODE_PTR(model, L->mlp_out);
    float *output = QWEN2_0_5B_DECODE_PTR(model, L->output);
 
    const void *WQ = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wq);
    const void *WK = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wk);
    const void *WV = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wv);
    const void *WO = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wo);
    const void *W1 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w1);
    const void *W2 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w2);
    const float *BQ = NULL;
    const float *BK = NULL;
    const float *BV = NULL;
    const float *BO = NULL;
    const float *B1 = NULL;
    const float *B2 = NULL;
 
    float *rope_cos = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_cos_cache);
    float *rope_sin = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_sin_cache);
 
    const int H = QWEN2_0_5B_DECODE_NUM_HEADS;
    const int H_kv = QWEN2_0_5B_DECODE_NUM_KV_HEADS;
    const int head_dim = QWEN2_0_5B_DECODE_HEAD_DIM;
    const size_t head_w_elems = (size_t)aligned_head_dim * (size_t)aligned_embed_dim;
    const size_t q_head_stride = (size_t)num_tokens * (size_t)aligned_head_dim;
    const size_t kv_head_stride = (size_t)aligned_context_window * (size_t)aligned_head_dim;
 
    /* RMSNorm before attention */
    rmsnorm_forward(input,
                    ln1_gamma,
                    ln1_out,
                    NULL,
                    num_tokens,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    /* Q projection (head-major) */
    const size_t wq_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, head_w_elems);
    const uint8_t *WQ_bytes = (const uint8_t *)WQ;
    for (int h = 0; h < H; ++h) {
        const void *wq_h = (const void *)(WQ_bytes + (size_t)h * wq_head_bytes);
        const float *bq_h = BQ ? (BQ + (size_t)h * (size_t)aligned_head_dim) : NULL;
        float *q_h = q + (size_t)h * q_head_stride;
        gemm_nt_q4_k(ln1_out, wq_h, bq_h, q_h, num_tokens, aligned_head_dim, aligned_embed_dim);
    }
 
    /* K projection (head-major) */
    const size_t wk_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, head_w_elems);
    const uint8_t *WK_bytes = (const uint8_t *)WK;
    for (int h = 0; h < H_kv; ++h) {
        const void *wk_h = (const void *)(WK_bytes + (size_t)h * wk_head_bytes);
        const float *bk_h = BK ? (BK + (size_t)h * (size_t)aligned_head_dim) : NULL;
        float *k_h = k + (size_t)h * kv_head_stride;
        gemm_nt_q4_k(ln1_out, wk_h, bk_h, k_h, num_tokens, aligned_head_dim, aligned_embed_dim);
    }
 
    /* V projection (head-major) */
    const size_t wv_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, head_w_elems);
    const uint8_t *WV_bytes = (const uint8_t *)WV;
    for (int h = 0; h < H_kv; ++h) {
        const void *wv_h = (const void *)(WV_bytes + (size_t)h * wv_head_bytes);
        const float *bv_h = BV ? (BV + (size_t)h * (size_t)aligned_head_dim) : NULL;
        float *v_h = v + (size_t)h * kv_head_stride;
        gemm_nt_q4_k(ln1_out, wv_h, bv_h, v_h, num_tokens, aligned_head_dim, aligned_embed_dim);
    }
 
    /* RoPE */
    rope_forward_qk_strided(q,
                            k,
                            rope_cos,
                            rope_sin,
                            H,
                            H_kv,
                            num_tokens,
                            head_dim,
                            aligned_head_dim,
                            0,
                            num_tokens,
                            aligned_context_window);
 
    /* Attention (prefill, causal) */
    attention_forward_causal_head_major_gqa_flash_strided(q,
                                                           k,
                                                           v,
                                                           attn_out,
                                                           H,
                                                           H_kv,
                                                           num_tokens,
                                                           head_dim,
                                                           aligned_head_dim,
                                                           aligned_context_window);
 
    /* Output projection (flatten head-major to token-major) */
    const int K = H * aligned_head_dim;
    if (K != aligned_embed_dim) {
        return;
    }
    const float *proj_in = attn_out;
    if (H > 1) {
        if (!proj_scratch) {
            return;
        }
        for (int t = 0; t < num_tokens; ++t) {
            float *dst = proj_scratch + (size_t)t * (size_t)aligned_embed_dim;
            for (int h = 0; h < H; ++h) {
                const float *src = attn_out + (size_t)h * q_head_stride + (size_t)t * (size_t)aligned_head_dim;
                memcpy(dst + (size_t)h * (size_t)aligned_head_dim,
                       src,
                       (size_t)aligned_head_dim * sizeof(float));
            }
        }
        proj_in = proj_scratch;
    }
    gemm_nt_q4_k(proj_in, WO, BO, proj_tmp, num_tokens, aligned_embed_dim, K);
 
    /* Residual add */
    qwen2_0_5b_decode_residual_add_token_major(input, proj_tmp, residual1, num_tokens, aligned_embed_dim);
 
    /* RMSNorm before MLP */
    rmsnorm_forward(residual1,
                    ln2_gamma,
                    ln2_out,
                    NULL,
                    num_tokens,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    /* MLP (SwiGLU) */
    gemm_nt_q4_k(ln2_out, W1, B1, fc1_out, num_tokens, 2 * aligned_intermediate_dim, aligned_embed_dim);
    swiglu_forward(fc1_out, swiglu_out, num_tokens, aligned_intermediate_dim);
    gemm_nt_q4_k(swiglu_out, W2, B2, mlp_out, num_tokens, aligned_embed_dim, aligned_intermediate_dim);
 
    /* Final residual add */
    qwen2_0_5b_decode_residual_add_token_major(residual1, mlp_out, output, num_tokens, aligned_embed_dim);
}

References attention_forward_causal_head_major_gqa_flash_strided(), CK_DT_Q4_K, ck_dtype_row_bytes(), gemm_nt_q4_k(), QWEN2_0_5B_DECODELayerOffsets::k, QWEN2_0_5B_DECODELayerOffsets::ln1_gamma, QWEN2_0_5B_DECODELayerOffsets::ln1_out, QWEN2_0_5B_DECODELayerOffsets::ln2_gamma, QWEN2_0_5B_DECODELayerOffsets::ln2_out, QWEN2_0_5B_DECODELayerOffsets::mlp_out, QWEN2_0_5B_DECODELayerOffsets::output, QWEN2_0_5B_DECODELayerOffsets::proj_scratch, QWEN2_0_5B_DECODELayerOffsets::proj_tmp, QWEN2_0_5B_DECODELayerOffsets::q, QWEN2_0_5B_DECODE_EMBED_DIM, QWEN2_0_5B_DECODE_GLOBALS, QWEN2_0_5B_DECODE_HEAD_DIM, QWEN2_0_5B_DECODE_LAYERS, QWEN2_0_5B_DECODE_NUM_HEADS, QWEN2_0_5B_DECODE_NUM_KV_HEADS, QWEN2_0_5B_DECODE_PTR, qwen2_0_5b_decode_residual_add_token_major(), QWEN2_0_5B_DECODELayerOffsets::residual1, rmsnorm_forward(), QWEN2_0_5B_DECODEGlobalOffsets::rope_cos_cache, rope_forward_qk_strided(), QWEN2_0_5B_DECODEGlobalOffsets::rope_sin_cache, swiglu_forward(), QWEN2_0_5B_DECODELayerOffsets::v, QWEN2_0_5B_DECODELayerOffsets::w1, QWEN2_0_5B_DECODELayerOffsets::w2, QWEN2_0_5B_DECODELayerOffsets::wk, QWEN2_0_5B_DECODELayerOffsets::wo, QWEN2_0_5B_DECODELayerOffsets::wq, and QWEN2_0_5B_DECODELayerOffsets::wv.

Referenced by qwen2_0_5b_decode_forward_prefill_impl().

qwen2_0_5b_decode_layer_6_decode()

static void qwen2_0_5b_decode_layer_6_decode ( QWEN2_0_5B_DECODEModel *  model, int  token_index, int  aligned_embed_dim, int  aligned_head_dim, int  aligned_intermediate_dim, int  aligned_context_window  )

static

Definition at line 5491 of file v6.5/test_generated/qwen2_int8.c.

  {
    const QWEN2_0_5B_DECODELayerOffsets *L = &QWEN2_0_5B_DECODE_LAYERS[6];
 
    float *input = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[5].output);
 
    float *ln1_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln1_gamma);
    float *ln1_out = QWEN2_0_5B_DECODE_PTR(model, L->ln1_out);
    float *ln2_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln2_gamma);
    float *ln2_out = QWEN2_0_5B_DECODE_PTR(model, L->ln2_out);
    float *k_cache = QWEN2_0_5B_DECODE_PTR(model, L->k);
    float *v_cache = QWEN2_0_5B_DECODE_PTR(model, L->v);
    float *proj_tmp = QWEN2_0_5B_DECODE_PTR(model, L->proj_tmp);
    float *proj_scratch = QWEN2_0_5B_DECODE_PTR(model, L->proj_scratch);
    float *residual1 = QWEN2_0_5B_DECODE_PTR(model, L->residual1);
    float *mlp_out = QWEN2_0_5B_DECODE_PTR(model, L->mlp_out);
    float *output = QWEN2_0_5B_DECODE_PTR(model, L->output);
 
    /* Weights (explicit types for layer 6) */
    const void *WQ = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wq);  /* Q4_K */
    const void *WK = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wk);  /* Q4_K */
    const void *WV = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wv);  /* Q4_K */
    const void *WO = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wo);  /* Q4_K */
    const void *W1 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w1);  /* Q4_K (gate+up) */
    const void *W2 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w2);  /* Q4_K (down) */
 
    float *rope_cos = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_cos_cache);
    float *rope_sin = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_sin_cache);
 
    const int H = QWEN2_0_5B_DECODE_NUM_HEADS;
    const int H_kv = QWEN2_0_5B_DECODE_NUM_KV_HEADS;
    const int head_dim = QWEN2_0_5B_DECODE_HEAD_DIM;
 
    float q_token[H * aligned_head_dim];
    float k_token[H_kv * aligned_head_dim];
    float v_token[H_kv * aligned_head_dim];
    float attn_token[H * aligned_head_dim];
 
    /* Local MLP buffers (avoid layout dependencies for intermediate values) */
    float fc1_out[2 * aligned_intermediate_dim];
    float swiglu_out[aligned_intermediate_dim];
 
    /* Step 1: RMSNorm before attention */
    rmsnorm_forward(input,
                    ln1_gamma,
                    ln1_out,
                    NULL,
                    1,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    const size_t kv_head_stride = (size_t)aligned_context_window * (size_t)aligned_head_dim;
 
    /* Step 2: QKV projection */
    /* Q projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t ln1_q8_bytes = ((((aligned_embed_dim) + 255) / 256) * 292);
    uint8_t ln1_q8[ln1_q8_bytes];
    quantize_row_q8_k(ln1_out, ln1_q8, aligned_embed_dim);
    gemv_q4_k_q8_k(q_token, WQ, ln1_q8, H * head_dim, aligned_embed_dim);
    if (aligned_head_dim > head_dim) {
        for (int h = 0; h < H; ++h) {
            float *q_head = q_token + (size_t)h * (size_t)aligned_head_dim;
            for (int d = head_dim; d < aligned_head_dim; ++d) {
                q_head[d] = 0.0f;
            }
        }
    }
 
    /* K projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k (direct-to-cache) */
    const size_t wk_head_elems = (size_t)aligned_head_dim * (size_t)aligned_embed_dim;
    const size_t wk_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, wk_head_elems);
    const uint8_t *WK_bytes = (const uint8_t *)WK;
    /* ln1_q8 already quantized above */
    for (int h = 0; h < H_kv; ++h) {
        const void *wk_h = (const void *)(WK_bytes + (size_t)h * wk_head_bytes);
        float *k_head = k_cache + (size_t)h * kv_head_stride + (size_t)token_index * (size_t)aligned_head_dim;
        gemv_q4_k_q8_k(k_head, wk_h, ln1_q8, head_dim, aligned_embed_dim);
        for (int d = head_dim; d < aligned_head_dim; ++d) {
            k_head[d] = 0.0f;
        }
    }
 
    /* V projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k (direct-to-cache) */
    const size_t wv_head_elems = (size_t)aligned_head_dim * (size_t)aligned_embed_dim;
    const size_t wv_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, wv_head_elems);
    const uint8_t *WV_bytes = (const uint8_t *)WV;
    /* ln1_q8 already quantized above */
    for (int h = 0; h < H_kv; ++h) {
        const void *wv_h = (const void *)(WV_bytes + (size_t)h * wv_head_bytes);
        float *v_head = v_cache + (size_t)h * kv_head_stride + (size_t)token_index * (size_t)aligned_head_dim;
        gemv_q4_k_q8_k(v_head, wv_h, ln1_q8, head_dim, aligned_embed_dim);
        for (int d = head_dim; d < aligned_head_dim; ++d) {
            v_head[d] = 0.0f;
        }
    }
 
    /* Step 3: RoPE */
    rope_forward(q_token,
                 rope_cos,
                 rope_sin,
                 H,
                 1,
                 head_dim,
                 aligned_head_dim,
                 token_index);
    for (int h = 0; h < H_kv; ++h) {
        float *k_head = k_cache + (size_t)h * kv_head_stride + (size_t)token_index * (size_t)aligned_head_dim;
        rope_forward(k_head,
                     rope_cos,
                     rope_sin,
                     1,
                     1,
                     head_dim,
                     aligned_head_dim,
                     token_index);
    }
 
    /* Step 4: KV cache write (direct-to-cache) */
 
    /* Step 5: Attention (decode, flash) */
    attention_forward_decode_head_major_gqa_flash(q_token,
                                                   k_cache,
                                                   v_cache,
                                                   attn_token,
                                                   H,
                                                   H_kv,
                                                   token_index + 1,
                                                   aligned_context_window,
                                                   head_dim,
                                                   aligned_head_dim);
 
    /* Step 6: Output projection */
    /* WO projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t attn_q8_bytes = ((((H * head_dim) + 255) / 256) * 292);
    uint8_t attn_q8[attn_q8_bytes];
    quantize_row_q8_k(attn_token, attn_q8, H * head_dim);
    gemv_q4_k_q8_k(proj_tmp, WO, attn_q8, aligned_embed_dim, H * head_dim);
 
    /* Step 7: Residual add */
    qwen2_0_5b_decode_residual_add_token_major(input, proj_tmp, residual1, 1, aligned_embed_dim);
 
    /* Step 8: RMSNorm before MLP */
    rmsnorm_forward(residual1,
                    ln2_gamma,
                    ln2_out,
                    NULL,
                    1,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    /* Step 9: MLP (SwiGLU) */
    /* Gate+Up projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t ln2_q8_bytes = ((((aligned_embed_dim) + 255) / 256) * 292);
    uint8_t ln2_q8[ln2_q8_bytes];
    quantize_row_q8_k(ln2_out, ln2_q8, aligned_embed_dim);
    gemv_q4_k_q8_k(fc1_out, W1, ln2_q8, 2 * aligned_intermediate_dim, aligned_embed_dim);
 
    /* SwiGLU activation */
    swiglu_forward(fc1_out, swiglu_out, 1, aligned_intermediate_dim);
 
    /* Down projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t swiglu_q8_bytes = ((((aligned_intermediate_dim) + 255) / 256) * 292);
    uint8_t swiglu_q8[swiglu_q8_bytes];
    quantize_row_q8_k(swiglu_out, swiglu_q8, aligned_intermediate_dim);
    gemv_q4_k_q8_k(mlp_out, W2, swiglu_q8, aligned_embed_dim, aligned_intermediate_dim);
 
    /* Step 10: Final residual add */
    qwen2_0_5b_decode_residual_add_token_major(residual1, mlp_out, output, 1, aligned_embed_dim);
}

References attention_forward_decode_head_major_gqa_flash(), CK_DT_Q4_K, ck_dtype_row_bytes(), gemv_q4_k_q8_k(), QWEN2_0_5B_DECODELayerOffsets::k, QWEN2_0_5B_DECODELayerOffsets::ln1_gamma, QWEN2_0_5B_DECODELayerOffsets::ln1_out, QWEN2_0_5B_DECODELayerOffsets::ln2_gamma, QWEN2_0_5B_DECODELayerOffsets::ln2_out, QWEN2_0_5B_DECODELayerOffsets::mlp_out, QWEN2_0_5B_DECODELayerOffsets::output, QWEN2_0_5B_DECODELayerOffsets::proj_scratch, QWEN2_0_5B_DECODELayerOffsets::proj_tmp, quantize_row_q8_k(), QWEN2_0_5B_DECODE_EMBED_DIM, QWEN2_0_5B_DECODE_GLOBALS, QWEN2_0_5B_DECODE_HEAD_DIM, QWEN2_0_5B_DECODE_LAYERS, QWEN2_0_5B_DECODE_NUM_HEADS, QWEN2_0_5B_DECODE_NUM_KV_HEADS, QWEN2_0_5B_DECODE_PTR, qwen2_0_5b_decode_residual_add_token_major(), QWEN2_0_5B_DECODELayerOffsets::residual1, rmsnorm_forward(), QWEN2_0_5B_DECODEGlobalOffsets::rope_cos_cache, rope_forward(), QWEN2_0_5B_DECODEGlobalOffsets::rope_sin_cache, swiglu_forward(), QWEN2_0_5B_DECODELayerOffsets::v, QWEN2_0_5B_DECODELayerOffsets::w1, QWEN2_0_5B_DECODELayerOffsets::w2, QWEN2_0_5B_DECODELayerOffsets::wk, QWEN2_0_5B_DECODELayerOffsets::wo, QWEN2_0_5B_DECODELayerOffsets::wq, and QWEN2_0_5B_DECODELayerOffsets::wv.

Referenced by qwen2_0_5b_decode_decode_token().

qwen2_0_5b_decode_layer_6_prefill()

static void qwen2_0_5b_decode_layer_6_prefill ( QWEN2_0_5B_DECODEModel *  model, int  num_tokens, int  aligned_embed_dim, int  aligned_head_dim, int  aligned_intermediate_dim, int  aligned_context_window  )

static

Definition at line 1195 of file v6.5/test_generated/qwen2_int8.c.

  {
    const QWEN2_0_5B_DECODELayerOffsets *L = &QWEN2_0_5B_DECODE_LAYERS[6];
 
    float *input = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[5].output);
    float *ln1_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln1_gamma);
    float *ln1_out = QWEN2_0_5B_DECODE_PTR(model, L->ln1_out);
    float *ln2_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln2_gamma);
    float *ln2_out = QWEN2_0_5B_DECODE_PTR(model, L->ln2_out);
    float *q = QWEN2_0_5B_DECODE_PTR(model, L->q);
    float *k = QWEN2_0_5B_DECODE_PTR(model, L->k);
    float *v = QWEN2_0_5B_DECODE_PTR(model, L->v);
    float *attn_out = QWEN2_0_5B_DECODE_PTR(model, L->attn_out);
    float *proj_tmp = QWEN2_0_5B_DECODE_PTR(model, L->proj_tmp);
    float *proj_scratch = QWEN2_0_5B_DECODE_PTR(model, L->proj_scratch);
    float *residual1 = QWEN2_0_5B_DECODE_PTR(model, L->residual1);
    float *fc1_out = QWEN2_0_5B_DECODE_PTR(model, L->fc1_out);
    float *swiglu_out = QWEN2_0_5B_DECODE_PTR(model, L->swiglu_out);
    float *mlp_out = QWEN2_0_5B_DECODE_PTR(model, L->mlp_out);
    float *output = QWEN2_0_5B_DECODE_PTR(model, L->output);
 
    const void *WQ = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wq);
    const void *WK = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wk);
    const void *WV = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wv);
    const void *WO = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wo);
    const void *W1 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w1);
    const void *W2 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w2);
    const float *BQ = NULL;
    const float *BK = NULL;
    const float *BV = NULL;
    const float *BO = NULL;
    const float *B1 = NULL;
    const float *B2 = NULL;
 
    float *rope_cos = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_cos_cache);
    float *rope_sin = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_sin_cache);
 
    const int H = QWEN2_0_5B_DECODE_NUM_HEADS;
    const int H_kv = QWEN2_0_5B_DECODE_NUM_KV_HEADS;
    const int head_dim = QWEN2_0_5B_DECODE_HEAD_DIM;
    const size_t head_w_elems = (size_t)aligned_head_dim * (size_t)aligned_embed_dim;
    const size_t q_head_stride = (size_t)num_tokens * (size_t)aligned_head_dim;
    const size_t kv_head_stride = (size_t)aligned_context_window * (size_t)aligned_head_dim;
 
    /* RMSNorm before attention */
    rmsnorm_forward(input,
                    ln1_gamma,
                    ln1_out,
                    NULL,
                    num_tokens,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    /* Q projection (head-major) */
    const size_t wq_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, head_w_elems);
    const uint8_t *WQ_bytes = (const uint8_t *)WQ;
    for (int h = 0; h < H; ++h) {
        const void *wq_h = (const void *)(WQ_bytes + (size_t)h * wq_head_bytes);
        const float *bq_h = BQ ? (BQ + (size_t)h * (size_t)aligned_head_dim) : NULL;
        float *q_h = q + (size_t)h * q_head_stride;
        gemm_nt_q4_k(ln1_out, wq_h, bq_h, q_h, num_tokens, aligned_head_dim, aligned_embed_dim);
    }
 
    /* K projection (head-major) */
    const size_t wk_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, head_w_elems);
    const uint8_t *WK_bytes = (const uint8_t *)WK;
    for (int h = 0; h < H_kv; ++h) {
        const void *wk_h = (const void *)(WK_bytes + (size_t)h * wk_head_bytes);
        const float *bk_h = BK ? (BK + (size_t)h * (size_t)aligned_head_dim) : NULL;
        float *k_h = k + (size_t)h * kv_head_stride;
        gemm_nt_q4_k(ln1_out, wk_h, bk_h, k_h, num_tokens, aligned_head_dim, aligned_embed_dim);
    }
 
    /* V projection (head-major) */
    const size_t wv_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, head_w_elems);
    const uint8_t *WV_bytes = (const uint8_t *)WV;
    for (int h = 0; h < H_kv; ++h) {
        const void *wv_h = (const void *)(WV_bytes + (size_t)h * wv_head_bytes);
        const float *bv_h = BV ? (BV + (size_t)h * (size_t)aligned_head_dim) : NULL;
        float *v_h = v + (size_t)h * kv_head_stride;
        gemm_nt_q4_k(ln1_out, wv_h, bv_h, v_h, num_tokens, aligned_head_dim, aligned_embed_dim);
    }
 
    /* RoPE */
    rope_forward_qk_strided(q,
                            k,
                            rope_cos,
                            rope_sin,
                            H,
                            H_kv,
                            num_tokens,
                            head_dim,
                            aligned_head_dim,
                            0,
                            num_tokens,
                            aligned_context_window);
 
    /* Attention (prefill, causal) */
    attention_forward_causal_head_major_gqa_flash_strided(q,
                                                           k,
                                                           v,
                                                           attn_out,
                                                           H,
                                                           H_kv,
                                                           num_tokens,
                                                           head_dim,
                                                           aligned_head_dim,
                                                           aligned_context_window);
 
    /* Output projection (flatten head-major to token-major) */
    const int K = H * aligned_head_dim;
    if (K != aligned_embed_dim) {
        return;
    }
    const float *proj_in = attn_out;
    if (H > 1) {
        if (!proj_scratch) {
            return;
        }
        for (int t = 0; t < num_tokens; ++t) {
            float *dst = proj_scratch + (size_t)t * (size_t)aligned_embed_dim;
            for (int h = 0; h < H; ++h) {
                const float *src = attn_out + (size_t)h * q_head_stride + (size_t)t * (size_t)aligned_head_dim;
                memcpy(dst + (size_t)h * (size_t)aligned_head_dim,
                       src,
                       (size_t)aligned_head_dim * sizeof(float));
            }
        }
        proj_in = proj_scratch;
    }
    gemm_nt_q4_k(proj_in, WO, BO, proj_tmp, num_tokens, aligned_embed_dim, K);
 
    /* Residual add */
    qwen2_0_5b_decode_residual_add_token_major(input, proj_tmp, residual1, num_tokens, aligned_embed_dim);
 
    /* RMSNorm before MLP */
    rmsnorm_forward(residual1,
                    ln2_gamma,
                    ln2_out,
                    NULL,
                    num_tokens,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    /* MLP (SwiGLU) */
    gemm_nt_q4_k(ln2_out, W1, B1, fc1_out, num_tokens, 2 * aligned_intermediate_dim, aligned_embed_dim);
    swiglu_forward(fc1_out, swiglu_out, num_tokens, aligned_intermediate_dim);
    gemm_nt_q4_k(swiglu_out, W2, B2, mlp_out, num_tokens, aligned_embed_dim, aligned_intermediate_dim);
 
    /* Final residual add */
    qwen2_0_5b_decode_residual_add_token_major(residual1, mlp_out, output, num_tokens, aligned_embed_dim);
}

References attention_forward_causal_head_major_gqa_flash_strided(), CK_DT_Q4_K, ck_dtype_row_bytes(), gemm_nt_q4_k(), QWEN2_0_5B_DECODELayerOffsets::k, QWEN2_0_5B_DECODELayerOffsets::ln1_gamma, QWEN2_0_5B_DECODELayerOffsets::ln1_out, QWEN2_0_5B_DECODELayerOffsets::ln2_gamma, QWEN2_0_5B_DECODELayerOffsets::ln2_out, QWEN2_0_5B_DECODELayerOffsets::mlp_out, QWEN2_0_5B_DECODELayerOffsets::output, QWEN2_0_5B_DECODELayerOffsets::proj_scratch, QWEN2_0_5B_DECODELayerOffsets::proj_tmp, QWEN2_0_5B_DECODELayerOffsets::q, QWEN2_0_5B_DECODE_EMBED_DIM, QWEN2_0_5B_DECODE_GLOBALS, QWEN2_0_5B_DECODE_HEAD_DIM, QWEN2_0_5B_DECODE_LAYERS, QWEN2_0_5B_DECODE_NUM_HEADS, QWEN2_0_5B_DECODE_NUM_KV_HEADS, QWEN2_0_5B_DECODE_PTR, qwen2_0_5b_decode_residual_add_token_major(), QWEN2_0_5B_DECODELayerOffsets::residual1, rmsnorm_forward(), QWEN2_0_5B_DECODEGlobalOffsets::rope_cos_cache, rope_forward_qk_strided(), QWEN2_0_5B_DECODEGlobalOffsets::rope_sin_cache, swiglu_forward(), QWEN2_0_5B_DECODELayerOffsets::v, QWEN2_0_5B_DECODELayerOffsets::w1, QWEN2_0_5B_DECODELayerOffsets::w2, QWEN2_0_5B_DECODELayerOffsets::wk, QWEN2_0_5B_DECODELayerOffsets::wo, QWEN2_0_5B_DECODELayerOffsets::wq, and QWEN2_0_5B_DECODELayerOffsets::wv.

Referenced by qwen2_0_5b_decode_forward_prefill_impl().

qwen2_0_5b_decode_layer_7_decode()

static void qwen2_0_5b_decode_layer_7_decode ( QWEN2_0_5B_DECODEModel *  model, int  token_index, int  aligned_embed_dim, int  aligned_head_dim, int  aligned_intermediate_dim, int  aligned_context_window  )

static

Definition at line 5672 of file v6.5/test_generated/qwen2_int8.c.

  {
    const QWEN2_0_5B_DECODELayerOffsets *L = &QWEN2_0_5B_DECODE_LAYERS[7];
 
    float *input = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[6].output);
 
    float *ln1_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln1_gamma);
    float *ln1_out = QWEN2_0_5B_DECODE_PTR(model, L->ln1_out);
    float *ln2_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln2_gamma);
    float *ln2_out = QWEN2_0_5B_DECODE_PTR(model, L->ln2_out);
    float *k_cache = QWEN2_0_5B_DECODE_PTR(model, L->k);
    float *v_cache = QWEN2_0_5B_DECODE_PTR(model, L->v);
    float *proj_tmp = QWEN2_0_5B_DECODE_PTR(model, L->proj_tmp);
    float *proj_scratch = QWEN2_0_5B_DECODE_PTR(model, L->proj_scratch);
    float *residual1 = QWEN2_0_5B_DECODE_PTR(model, L->residual1);
    float *mlp_out = QWEN2_0_5B_DECODE_PTR(model, L->mlp_out);
    float *output = QWEN2_0_5B_DECODE_PTR(model, L->output);
 
    /* Weights (explicit types for layer 7) */
    const void *WQ = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wq);  /* Q4_K */
    const void *WK = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wk);  /* Q4_K */
    const void *WV = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wv);  /* Q4_K */
    const void *WO = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wo);  /* Q4_K */
    const void *W1 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w1);  /* Q4_K (gate+up) */
    const void *W2 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w2);  /* Q4_K (down) */
 
    float *rope_cos = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_cos_cache);
    float *rope_sin = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_sin_cache);
 
    const int H = QWEN2_0_5B_DECODE_NUM_HEADS;
    const int H_kv = QWEN2_0_5B_DECODE_NUM_KV_HEADS;
    const int head_dim = QWEN2_0_5B_DECODE_HEAD_DIM;
 
    float q_token[H * aligned_head_dim];
    float k_token[H_kv * aligned_head_dim];
    float v_token[H_kv * aligned_head_dim];
    float attn_token[H * aligned_head_dim];
 
    /* Local MLP buffers (avoid layout dependencies for intermediate values) */
    float fc1_out[2 * aligned_intermediate_dim];
    float swiglu_out[aligned_intermediate_dim];
 
    /* Step 1: RMSNorm before attention */
    rmsnorm_forward(input,
                    ln1_gamma,
                    ln1_out,
                    NULL,
                    1,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    const size_t kv_head_stride = (size_t)aligned_context_window * (size_t)aligned_head_dim;
 
    /* Step 2: QKV projection */
    /* Q projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t ln1_q8_bytes = ((((aligned_embed_dim) + 255) / 256) * 292);
    uint8_t ln1_q8[ln1_q8_bytes];
    quantize_row_q8_k(ln1_out, ln1_q8, aligned_embed_dim);
    gemv_q4_k_q8_k(q_token, WQ, ln1_q8, H * head_dim, aligned_embed_dim);
    if (aligned_head_dim > head_dim) {
        for (int h = 0; h < H; ++h) {
            float *q_head = q_token + (size_t)h * (size_t)aligned_head_dim;
            for (int d = head_dim; d < aligned_head_dim; ++d) {
                q_head[d] = 0.0f;
            }
        }
    }
 
    /* K projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k (direct-to-cache) */
    const size_t wk_head_elems = (size_t)aligned_head_dim * (size_t)aligned_embed_dim;
    const size_t wk_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, wk_head_elems);
    const uint8_t *WK_bytes = (const uint8_t *)WK;
    /* ln1_q8 already quantized above */
    for (int h = 0; h < H_kv; ++h) {
        const void *wk_h = (const void *)(WK_bytes + (size_t)h * wk_head_bytes);
        float *k_head = k_cache + (size_t)h * kv_head_stride + (size_t)token_index * (size_t)aligned_head_dim;
        gemv_q4_k_q8_k(k_head, wk_h, ln1_q8, head_dim, aligned_embed_dim);
        for (int d = head_dim; d < aligned_head_dim; ++d) {
            k_head[d] = 0.0f;
        }
    }
 
    /* V projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k (direct-to-cache) */
    const size_t wv_head_elems = (size_t)aligned_head_dim * (size_t)aligned_embed_dim;
    const size_t wv_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, wv_head_elems);
    const uint8_t *WV_bytes = (const uint8_t *)WV;
    /* ln1_q8 already quantized above */
    for (int h = 0; h < H_kv; ++h) {
        const void *wv_h = (const void *)(WV_bytes + (size_t)h * wv_head_bytes);
        float *v_head = v_cache + (size_t)h * kv_head_stride + (size_t)token_index * (size_t)aligned_head_dim;
        gemv_q4_k_q8_k(v_head, wv_h, ln1_q8, head_dim, aligned_embed_dim);
        for (int d = head_dim; d < aligned_head_dim; ++d) {
            v_head[d] = 0.0f;
        }
    }
 
    /* Step 3: RoPE */
    rope_forward(q_token,
                 rope_cos,
                 rope_sin,
                 H,
                 1,
                 head_dim,
                 aligned_head_dim,
                 token_index);
    for (int h = 0; h < H_kv; ++h) {
        float *k_head = k_cache + (size_t)h * kv_head_stride + (size_t)token_index * (size_t)aligned_head_dim;
        rope_forward(k_head,
                     rope_cos,
                     rope_sin,
                     1,
                     1,
                     head_dim,
                     aligned_head_dim,
                     token_index);
    }
 
    /* Step 4: KV cache write (direct-to-cache) */
 
    /* Step 5: Attention (decode, flash) */
    attention_forward_decode_head_major_gqa_flash(q_token,
                                                   k_cache,
                                                   v_cache,
                                                   attn_token,
                                                   H,
                                                   H_kv,
                                                   token_index + 1,
                                                   aligned_context_window,
                                                   head_dim,
                                                   aligned_head_dim);
 
    /* Step 6: Output projection */
    /* WO projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t attn_q8_bytes = ((((H * head_dim) + 255) / 256) * 292);
    uint8_t attn_q8[attn_q8_bytes];
    quantize_row_q8_k(attn_token, attn_q8, H * head_dim);
    gemv_q4_k_q8_k(proj_tmp, WO, attn_q8, aligned_embed_dim, H * head_dim);
 
    /* Step 7: Residual add */
    qwen2_0_5b_decode_residual_add_token_major(input, proj_tmp, residual1, 1, aligned_embed_dim);
 
    /* Step 8: RMSNorm before MLP */
    rmsnorm_forward(residual1,
                    ln2_gamma,
                    ln2_out,
                    NULL,
                    1,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    /* Step 9: MLP (SwiGLU) */
    /* Gate+Up projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t ln2_q8_bytes = ((((aligned_embed_dim) + 255) / 256) * 292);
    uint8_t ln2_q8[ln2_q8_bytes];
    quantize_row_q8_k(ln2_out, ln2_q8, aligned_embed_dim);
    gemv_q4_k_q8_k(fc1_out, W1, ln2_q8, 2 * aligned_intermediate_dim, aligned_embed_dim);
 
    /* SwiGLU activation */
    swiglu_forward(fc1_out, swiglu_out, 1, aligned_intermediate_dim);
 
    /* Down projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t swiglu_q8_bytes = ((((aligned_intermediate_dim) + 255) / 256) * 292);
    uint8_t swiglu_q8[swiglu_q8_bytes];
    quantize_row_q8_k(swiglu_out, swiglu_q8, aligned_intermediate_dim);
    gemv_q4_k_q8_k(mlp_out, W2, swiglu_q8, aligned_embed_dim, aligned_intermediate_dim);
 
    /* Step 10: Final residual add */
    qwen2_0_5b_decode_residual_add_token_major(residual1, mlp_out, output, 1, aligned_embed_dim);
}

References attention_forward_decode_head_major_gqa_flash(), CK_DT_Q4_K, ck_dtype_row_bytes(), gemv_q4_k_q8_k(), QWEN2_0_5B_DECODELayerOffsets::k, QWEN2_0_5B_DECODELayerOffsets::ln1_gamma, QWEN2_0_5B_DECODELayerOffsets::ln1_out, QWEN2_0_5B_DECODELayerOffsets::ln2_gamma, QWEN2_0_5B_DECODELayerOffsets::ln2_out, QWEN2_0_5B_DECODELayerOffsets::mlp_out, QWEN2_0_5B_DECODELayerOffsets::output, QWEN2_0_5B_DECODELayerOffsets::proj_scratch, QWEN2_0_5B_DECODELayerOffsets::proj_tmp, quantize_row_q8_k(), QWEN2_0_5B_DECODE_EMBED_DIM, QWEN2_0_5B_DECODE_GLOBALS, QWEN2_0_5B_DECODE_HEAD_DIM, QWEN2_0_5B_DECODE_LAYERS, QWEN2_0_5B_DECODE_NUM_HEADS, QWEN2_0_5B_DECODE_NUM_KV_HEADS, QWEN2_0_5B_DECODE_PTR, qwen2_0_5b_decode_residual_add_token_major(), QWEN2_0_5B_DECODELayerOffsets::residual1, rmsnorm_forward(), QWEN2_0_5B_DECODEGlobalOffsets::rope_cos_cache, rope_forward(), QWEN2_0_5B_DECODEGlobalOffsets::rope_sin_cache, swiglu_forward(), QWEN2_0_5B_DECODELayerOffsets::v, QWEN2_0_5B_DECODELayerOffsets::w1, QWEN2_0_5B_DECODELayerOffsets::w2, QWEN2_0_5B_DECODELayerOffsets::wk, QWEN2_0_5B_DECODELayerOffsets::wo, QWEN2_0_5B_DECODELayerOffsets::wq, and QWEN2_0_5B_DECODELayerOffsets::wv.

Referenced by qwen2_0_5b_decode_decode_token().

qwen2_0_5b_decode_layer_7_prefill()

static void qwen2_0_5b_decode_layer_7_prefill ( QWEN2_0_5B_DECODEModel *  model, int  num_tokens, int  aligned_embed_dim, int  aligned_head_dim, int  aligned_intermediate_dim, int  aligned_context_window  )

static

Definition at line 1359 of file v6.5/test_generated/qwen2_int8.c.

  {
    const QWEN2_0_5B_DECODELayerOffsets *L = &QWEN2_0_5B_DECODE_LAYERS[7];
 
    float *input = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[6].output);
    float *ln1_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln1_gamma);
    float *ln1_out = QWEN2_0_5B_DECODE_PTR(model, L->ln1_out);
    float *ln2_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln2_gamma);
    float *ln2_out = QWEN2_0_5B_DECODE_PTR(model, L->ln2_out);
    float *q = QWEN2_0_5B_DECODE_PTR(model, L->q);
    float *k = QWEN2_0_5B_DECODE_PTR(model, L->k);
    float *v = QWEN2_0_5B_DECODE_PTR(model, L->v);
    float *attn_out = QWEN2_0_5B_DECODE_PTR(model, L->attn_out);
    float *proj_tmp = QWEN2_0_5B_DECODE_PTR(model, L->proj_tmp);
    float *proj_scratch = QWEN2_0_5B_DECODE_PTR(model, L->proj_scratch);
    float *residual1 = QWEN2_0_5B_DECODE_PTR(model, L->residual1);
    float *fc1_out = QWEN2_0_5B_DECODE_PTR(model, L->fc1_out);
    float *swiglu_out = QWEN2_0_5B_DECODE_PTR(model, L->swiglu_out);
    float *mlp_out = QWEN2_0_5B_DECODE_PTR(model, L->mlp_out);
    float *output = QWEN2_0_5B_DECODE_PTR(model, L->output);
 
    const void *WQ = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wq);
    const void *WK = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wk);
    const void *WV = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wv);
    const void *WO = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wo);
    const void *W1 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w1);
    const void *W2 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w2);
    const float *BQ = NULL;
    const float *BK = NULL;
    const float *BV = NULL;
    const float *BO = NULL;
    const float *B1 = NULL;
    const float *B2 = NULL;
 
    float *rope_cos = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_cos_cache);
    float *rope_sin = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_sin_cache);
 
    const int H = QWEN2_0_5B_DECODE_NUM_HEADS;
    const int H_kv = QWEN2_0_5B_DECODE_NUM_KV_HEADS;
    const int head_dim = QWEN2_0_5B_DECODE_HEAD_DIM;
    const size_t head_w_elems = (size_t)aligned_head_dim * (size_t)aligned_embed_dim;
    const size_t q_head_stride = (size_t)num_tokens * (size_t)aligned_head_dim;
    const size_t kv_head_stride = (size_t)aligned_context_window * (size_t)aligned_head_dim;
 
    /* RMSNorm before attention */
    rmsnorm_forward(input,
                    ln1_gamma,
                    ln1_out,
                    NULL,
                    num_tokens,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    /* Q projection (head-major) */
    const size_t wq_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, head_w_elems);
    const uint8_t *WQ_bytes = (const uint8_t *)WQ;
    for (int h = 0; h < H; ++h) {
        const void *wq_h = (const void *)(WQ_bytes + (size_t)h * wq_head_bytes);
        const float *bq_h = BQ ? (BQ + (size_t)h * (size_t)aligned_head_dim) : NULL;
        float *q_h = q + (size_t)h * q_head_stride;
        gemm_nt_q4_k(ln1_out, wq_h, bq_h, q_h, num_tokens, aligned_head_dim, aligned_embed_dim);
    }
 
    /* K projection (head-major) */
    const size_t wk_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, head_w_elems);
    const uint8_t *WK_bytes = (const uint8_t *)WK;
    for (int h = 0; h < H_kv; ++h) {
        const void *wk_h = (const void *)(WK_bytes + (size_t)h * wk_head_bytes);
        const float *bk_h = BK ? (BK + (size_t)h * (size_t)aligned_head_dim) : NULL;
        float *k_h = k + (size_t)h * kv_head_stride;
        gemm_nt_q4_k(ln1_out, wk_h, bk_h, k_h, num_tokens, aligned_head_dim, aligned_embed_dim);
    }
 
    /* V projection (head-major) */
    const size_t wv_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, head_w_elems);
    const uint8_t *WV_bytes = (const uint8_t *)WV;
    for (int h = 0; h < H_kv; ++h) {
        const void *wv_h = (const void *)(WV_bytes + (size_t)h * wv_head_bytes);
        const float *bv_h = BV ? (BV + (size_t)h * (size_t)aligned_head_dim) : NULL;
        float *v_h = v + (size_t)h * kv_head_stride;
        gemm_nt_q4_k(ln1_out, wv_h, bv_h, v_h, num_tokens, aligned_head_dim, aligned_embed_dim);
    }
 
    /* RoPE */
    rope_forward_qk_strided(q,
                            k,
                            rope_cos,
                            rope_sin,
                            H,
                            H_kv,
                            num_tokens,
                            head_dim,
                            aligned_head_dim,
                            0,
                            num_tokens,
                            aligned_context_window);
 
    /* Attention (prefill, causal) */
    attention_forward_causal_head_major_gqa_flash_strided(q,
                                                           k,
                                                           v,
                                                           attn_out,
                                                           H,
                                                           H_kv,
                                                           num_tokens,
                                                           head_dim,
                                                           aligned_head_dim,
                                                           aligned_context_window);
 
    /* Output projection (flatten head-major to token-major) */
    const int K = H * aligned_head_dim;
    if (K != aligned_embed_dim) {
        return;
    }
    const float *proj_in = attn_out;
    if (H > 1) {
        if (!proj_scratch) {
            return;
        }
        for (int t = 0; t < num_tokens; ++t) {
            float *dst = proj_scratch + (size_t)t * (size_t)aligned_embed_dim;
            for (int h = 0; h < H; ++h) {
                const float *src = attn_out + (size_t)h * q_head_stride + (size_t)t * (size_t)aligned_head_dim;
                memcpy(dst + (size_t)h * (size_t)aligned_head_dim,
                       src,
                       (size_t)aligned_head_dim * sizeof(float));
            }
        }
        proj_in = proj_scratch;
    }
    gemm_nt_q4_k(proj_in, WO, BO, proj_tmp, num_tokens, aligned_embed_dim, K);
 
    /* Residual add */
    qwen2_0_5b_decode_residual_add_token_major(input, proj_tmp, residual1, num_tokens, aligned_embed_dim);
 
    /* RMSNorm before MLP */
    rmsnorm_forward(residual1,
                    ln2_gamma,
                    ln2_out,
                    NULL,
                    num_tokens,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    /* MLP (SwiGLU) */
    gemm_nt_q4_k(ln2_out, W1, B1, fc1_out, num_tokens, 2 * aligned_intermediate_dim, aligned_embed_dim);
    swiglu_forward(fc1_out, swiglu_out, num_tokens, aligned_intermediate_dim);
    gemm_nt_q4_k(swiglu_out, W2, B2, mlp_out, num_tokens, aligned_embed_dim, aligned_intermediate_dim);
 
    /* Final residual add */
    qwen2_0_5b_decode_residual_add_token_major(residual1, mlp_out, output, num_tokens, aligned_embed_dim);
}

References attention_forward_causal_head_major_gqa_flash_strided(), CK_DT_Q4_K, ck_dtype_row_bytes(), gemm_nt_q4_k(), QWEN2_0_5B_DECODELayerOffsets::k, QWEN2_0_5B_DECODELayerOffsets::ln1_gamma, QWEN2_0_5B_DECODELayerOffsets::ln1_out, QWEN2_0_5B_DECODELayerOffsets::ln2_gamma, QWEN2_0_5B_DECODELayerOffsets::ln2_out, QWEN2_0_5B_DECODELayerOffsets::mlp_out, QWEN2_0_5B_DECODELayerOffsets::output, QWEN2_0_5B_DECODELayerOffsets::proj_scratch, QWEN2_0_5B_DECODELayerOffsets::proj_tmp, QWEN2_0_5B_DECODELayerOffsets::q, QWEN2_0_5B_DECODE_EMBED_DIM, QWEN2_0_5B_DECODE_GLOBALS, QWEN2_0_5B_DECODE_HEAD_DIM, QWEN2_0_5B_DECODE_LAYERS, QWEN2_0_5B_DECODE_NUM_HEADS, QWEN2_0_5B_DECODE_NUM_KV_HEADS, QWEN2_0_5B_DECODE_PTR, qwen2_0_5b_decode_residual_add_token_major(), QWEN2_0_5B_DECODELayerOffsets::residual1, rmsnorm_forward(), QWEN2_0_5B_DECODEGlobalOffsets::rope_cos_cache, rope_forward_qk_strided(), QWEN2_0_5B_DECODEGlobalOffsets::rope_sin_cache, swiglu_forward(), QWEN2_0_5B_DECODELayerOffsets::v, QWEN2_0_5B_DECODELayerOffsets::w1, QWEN2_0_5B_DECODELayerOffsets::w2, QWEN2_0_5B_DECODELayerOffsets::wk, QWEN2_0_5B_DECODELayerOffsets::wo, QWEN2_0_5B_DECODELayerOffsets::wq, and QWEN2_0_5B_DECODELayerOffsets::wv.

Referenced by qwen2_0_5b_decode_forward_prefill_impl().

qwen2_0_5b_decode_layer_8_decode()

static void qwen2_0_5b_decode_layer_8_decode ( QWEN2_0_5B_DECODEModel *  model, int  token_index, int  aligned_embed_dim, int  aligned_head_dim, int  aligned_intermediate_dim, int  aligned_context_window  )

static

Definition at line 5853 of file v6.5/test_generated/qwen2_int8.c.

  {
    const QWEN2_0_5B_DECODELayerOffsets *L = &QWEN2_0_5B_DECODE_LAYERS[8];
 
    float *input = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[7].output);
 
    float *ln1_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln1_gamma);
    float *ln1_out = QWEN2_0_5B_DECODE_PTR(model, L->ln1_out);
    float *ln2_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln2_gamma);
    float *ln2_out = QWEN2_0_5B_DECODE_PTR(model, L->ln2_out);
    float *k_cache = QWEN2_0_5B_DECODE_PTR(model, L->k);
    float *v_cache = QWEN2_0_5B_DECODE_PTR(model, L->v);
    float *proj_tmp = QWEN2_0_5B_DECODE_PTR(model, L->proj_tmp);
    float *proj_scratch = QWEN2_0_5B_DECODE_PTR(model, L->proj_scratch);
    float *residual1 = QWEN2_0_5B_DECODE_PTR(model, L->residual1);
    float *mlp_out = QWEN2_0_5B_DECODE_PTR(model, L->mlp_out);
    float *output = QWEN2_0_5B_DECODE_PTR(model, L->output);
 
    /* Weights (explicit types for layer 8) */
    const void *WQ = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wq);  /* Q4_K */
    const void *WK = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wk);  /* Q4_K */
    const void *WV = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wv);  /* Q4_K */
    const void *WO = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wo);  /* Q4_K */
    const void *W1 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w1);  /* Q4_K (gate+up) */
    const void *W2 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w2);  /* Q4_K (down) */
 
    float *rope_cos = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_cos_cache);
    float *rope_sin = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_sin_cache);
 
    const int H = QWEN2_0_5B_DECODE_NUM_HEADS;
    const int H_kv = QWEN2_0_5B_DECODE_NUM_KV_HEADS;
    const int head_dim = QWEN2_0_5B_DECODE_HEAD_DIM;
 
    float q_token[H * aligned_head_dim];
    float k_token[H_kv * aligned_head_dim];
    float v_token[H_kv * aligned_head_dim];
    float attn_token[H * aligned_head_dim];
 
    /* Local MLP buffers (avoid layout dependencies for intermediate values) */
    float fc1_out[2 * aligned_intermediate_dim];
    float swiglu_out[aligned_intermediate_dim];
 
    /* Step 1: RMSNorm before attention */
    rmsnorm_forward(input,
                    ln1_gamma,
                    ln1_out,
                    NULL,
                    1,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    const size_t kv_head_stride = (size_t)aligned_context_window * (size_t)aligned_head_dim;
 
    /* Step 2: QKV projection */
    /* Q projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t ln1_q8_bytes = ((((aligned_embed_dim) + 255) / 256) * 292);
    uint8_t ln1_q8[ln1_q8_bytes];
    quantize_row_q8_k(ln1_out, ln1_q8, aligned_embed_dim);
    gemv_q4_k_q8_k(q_token, WQ, ln1_q8, H * head_dim, aligned_embed_dim);
    if (aligned_head_dim > head_dim) {
        for (int h = 0; h < H; ++h) {
            float *q_head = q_token + (size_t)h * (size_t)aligned_head_dim;
            for (int d = head_dim; d < aligned_head_dim; ++d) {
                q_head[d] = 0.0f;
            }
        }
    }
 
    /* K projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k (direct-to-cache) */
    const size_t wk_head_elems = (size_t)aligned_head_dim * (size_t)aligned_embed_dim;
    const size_t wk_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, wk_head_elems);
    const uint8_t *WK_bytes = (const uint8_t *)WK;
    /* ln1_q8 already quantized above */
    for (int h = 0; h < H_kv; ++h) {
        const void *wk_h = (const void *)(WK_bytes + (size_t)h * wk_head_bytes);
        float *k_head = k_cache + (size_t)h * kv_head_stride + (size_t)token_index * (size_t)aligned_head_dim;
        gemv_q4_k_q8_k(k_head, wk_h, ln1_q8, head_dim, aligned_embed_dim);
        for (int d = head_dim; d < aligned_head_dim; ++d) {
            k_head[d] = 0.0f;
        }
    }
 
    /* V projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k (direct-to-cache) */
    const size_t wv_head_elems = (size_t)aligned_head_dim * (size_t)aligned_embed_dim;
    const size_t wv_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, wv_head_elems);
    const uint8_t *WV_bytes = (const uint8_t *)WV;
    /* ln1_q8 already quantized above */
    for (int h = 0; h < H_kv; ++h) {
        const void *wv_h = (const void *)(WV_bytes + (size_t)h * wv_head_bytes);
        float *v_head = v_cache + (size_t)h * kv_head_stride + (size_t)token_index * (size_t)aligned_head_dim;
        gemv_q4_k_q8_k(v_head, wv_h, ln1_q8, head_dim, aligned_embed_dim);
        for (int d = head_dim; d < aligned_head_dim; ++d) {
            v_head[d] = 0.0f;
        }
    }
 
    /* Step 3: RoPE */
    rope_forward(q_token,
                 rope_cos,
                 rope_sin,
                 H,
                 1,
                 head_dim,
                 aligned_head_dim,
                 token_index);
    for (int h = 0; h < H_kv; ++h) {
        float *k_head = k_cache + (size_t)h * kv_head_stride + (size_t)token_index * (size_t)aligned_head_dim;
        rope_forward(k_head,
                     rope_cos,
                     rope_sin,
                     1,
                     1,
                     head_dim,
                     aligned_head_dim,
                     token_index);
    }
 
    /* Step 4: KV cache write (direct-to-cache) */
 
    /* Step 5: Attention (decode, flash) */
    attention_forward_decode_head_major_gqa_flash(q_token,
                                                   k_cache,
                                                   v_cache,
                                                   attn_token,
                                                   H,
                                                   H_kv,
                                                   token_index + 1,
                                                   aligned_context_window,
                                                   head_dim,
                                                   aligned_head_dim);
 
    /* Step 6: Output projection */
    /* WO projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t attn_q8_bytes = ((((H * head_dim) + 255) / 256) * 292);
    uint8_t attn_q8[attn_q8_bytes];
    quantize_row_q8_k(attn_token, attn_q8, H * head_dim);
    gemv_q4_k_q8_k(proj_tmp, WO, attn_q8, aligned_embed_dim, H * head_dim);
 
    /* Step 7: Residual add */
    qwen2_0_5b_decode_residual_add_token_major(input, proj_tmp, residual1, 1, aligned_embed_dim);
 
    /* Step 8: RMSNorm before MLP */
    rmsnorm_forward(residual1,
                    ln2_gamma,
                    ln2_out,
                    NULL,
                    1,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    /* Step 9: MLP (SwiGLU) */
    /* Gate+Up projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t ln2_q8_bytes = ((((aligned_embed_dim) + 255) / 256) * 292);
    uint8_t ln2_q8[ln2_q8_bytes];
    quantize_row_q8_k(ln2_out, ln2_q8, aligned_embed_dim);
    gemv_q4_k_q8_k(fc1_out, W1, ln2_q8, 2 * aligned_intermediate_dim, aligned_embed_dim);
 
    /* SwiGLU activation */
    swiglu_forward(fc1_out, swiglu_out, 1, aligned_intermediate_dim);
 
    /* Down projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t swiglu_q8_bytes = ((((aligned_intermediate_dim) + 255) / 256) * 292);
    uint8_t swiglu_q8[swiglu_q8_bytes];
    quantize_row_q8_k(swiglu_out, swiglu_q8, aligned_intermediate_dim);
    gemv_q4_k_q8_k(mlp_out, W2, swiglu_q8, aligned_embed_dim, aligned_intermediate_dim);
 
    /* Step 10: Final residual add */
    qwen2_0_5b_decode_residual_add_token_major(residual1, mlp_out, output, 1, aligned_embed_dim);
}

References attention_forward_decode_head_major_gqa_flash(), CK_DT_Q4_K, ck_dtype_row_bytes(), gemv_q4_k_q8_k(), QWEN2_0_5B_DECODELayerOffsets::k, QWEN2_0_5B_DECODELayerOffsets::ln1_gamma, QWEN2_0_5B_DECODELayerOffsets::ln1_out, QWEN2_0_5B_DECODELayerOffsets::ln2_gamma, QWEN2_0_5B_DECODELayerOffsets::ln2_out, QWEN2_0_5B_DECODELayerOffsets::mlp_out, QWEN2_0_5B_DECODELayerOffsets::output, QWEN2_0_5B_DECODELayerOffsets::proj_scratch, QWEN2_0_5B_DECODELayerOffsets::proj_tmp, quantize_row_q8_k(), QWEN2_0_5B_DECODE_EMBED_DIM, QWEN2_0_5B_DECODE_GLOBALS, QWEN2_0_5B_DECODE_HEAD_DIM, QWEN2_0_5B_DECODE_LAYERS, QWEN2_0_5B_DECODE_NUM_HEADS, QWEN2_0_5B_DECODE_NUM_KV_HEADS, QWEN2_0_5B_DECODE_PTR, qwen2_0_5b_decode_residual_add_token_major(), QWEN2_0_5B_DECODELayerOffsets::residual1, rmsnorm_forward(), QWEN2_0_5B_DECODEGlobalOffsets::rope_cos_cache, rope_forward(), QWEN2_0_5B_DECODEGlobalOffsets::rope_sin_cache, swiglu_forward(), QWEN2_0_5B_DECODELayerOffsets::v, QWEN2_0_5B_DECODELayerOffsets::w1, QWEN2_0_5B_DECODELayerOffsets::w2, QWEN2_0_5B_DECODELayerOffsets::wk, QWEN2_0_5B_DECODELayerOffsets::wo, QWEN2_0_5B_DECODELayerOffsets::wq, and QWEN2_0_5B_DECODELayerOffsets::wv.

Referenced by qwen2_0_5b_decode_decode_token().

qwen2_0_5b_decode_layer_8_prefill()

static void qwen2_0_5b_decode_layer_8_prefill ( QWEN2_0_5B_DECODEModel *  model, int  num_tokens, int  aligned_embed_dim, int  aligned_head_dim, int  aligned_intermediate_dim, int  aligned_context_window  )

static

Definition at line 1523 of file v6.5/test_generated/qwen2_int8.c.

  {
    const QWEN2_0_5B_DECODELayerOffsets *L = &QWEN2_0_5B_DECODE_LAYERS[8];
 
    float *input = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[7].output);
    float *ln1_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln1_gamma);
    float *ln1_out = QWEN2_0_5B_DECODE_PTR(model, L->ln1_out);
    float *ln2_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln2_gamma);
    float *ln2_out = QWEN2_0_5B_DECODE_PTR(model, L->ln2_out);
    float *q = QWEN2_0_5B_DECODE_PTR(model, L->q);
    float *k = QWEN2_0_5B_DECODE_PTR(model, L->k);
    float *v = QWEN2_0_5B_DECODE_PTR(model, L->v);
    float *attn_out = QWEN2_0_5B_DECODE_PTR(model, L->attn_out);
    float *proj_tmp = QWEN2_0_5B_DECODE_PTR(model, L->proj_tmp);
    float *proj_scratch = QWEN2_0_5B_DECODE_PTR(model, L->proj_scratch);
    float *residual1 = QWEN2_0_5B_DECODE_PTR(model, L->residual1);
    float *fc1_out = QWEN2_0_5B_DECODE_PTR(model, L->fc1_out);
    float *swiglu_out = QWEN2_0_5B_DECODE_PTR(model, L->swiglu_out);
    float *mlp_out = QWEN2_0_5B_DECODE_PTR(model, L->mlp_out);
    float *output = QWEN2_0_5B_DECODE_PTR(model, L->output);
 
    const void *WQ = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wq);
    const void *WK = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wk);
    const void *WV = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wv);
    const void *WO = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wo);
    const void *W1 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w1);
    const void *W2 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w2);
    const float *BQ = NULL;
    const float *BK = NULL;
    const float *BV = NULL;
    const float *BO = NULL;
    const float *B1 = NULL;
    const float *B2 = NULL;
 
    float *rope_cos = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_cos_cache);
    float *rope_sin = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_sin_cache);
 
    const int H = QWEN2_0_5B_DECODE_NUM_HEADS;
    const int H_kv = QWEN2_0_5B_DECODE_NUM_KV_HEADS;
    const int head_dim = QWEN2_0_5B_DECODE_HEAD_DIM;
    const size_t head_w_elems = (size_t)aligned_head_dim * (size_t)aligned_embed_dim;
    const size_t q_head_stride = (size_t)num_tokens * (size_t)aligned_head_dim;
    const size_t kv_head_stride = (size_t)aligned_context_window * (size_t)aligned_head_dim;
 
    /* RMSNorm before attention */
    rmsnorm_forward(input,
                    ln1_gamma,
                    ln1_out,
                    NULL,
                    num_tokens,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    /* Q projection (head-major) */
    const size_t wq_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, head_w_elems);
    const uint8_t *WQ_bytes = (const uint8_t *)WQ;
    for (int h = 0; h < H; ++h) {
        const void *wq_h = (const void *)(WQ_bytes + (size_t)h * wq_head_bytes);
        const float *bq_h = BQ ? (BQ + (size_t)h * (size_t)aligned_head_dim) : NULL;
        float *q_h = q + (size_t)h * q_head_stride;
        gemm_nt_q4_k(ln1_out, wq_h, bq_h, q_h, num_tokens, aligned_head_dim, aligned_embed_dim);
    }
 
    /* K projection (head-major) */
    const size_t wk_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, head_w_elems);
    const uint8_t *WK_bytes = (const uint8_t *)WK;
    for (int h = 0; h < H_kv; ++h) {
        const void *wk_h = (const void *)(WK_bytes + (size_t)h * wk_head_bytes);
        const float *bk_h = BK ? (BK + (size_t)h * (size_t)aligned_head_dim) : NULL;
        float *k_h = k + (size_t)h * kv_head_stride;
        gemm_nt_q4_k(ln1_out, wk_h, bk_h, k_h, num_tokens, aligned_head_dim, aligned_embed_dim);
    }
 
    /* V projection (head-major) */
    const size_t wv_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, head_w_elems);
    const uint8_t *WV_bytes = (const uint8_t *)WV;
    for (int h = 0; h < H_kv; ++h) {
        const void *wv_h = (const void *)(WV_bytes + (size_t)h * wv_head_bytes);
        const float *bv_h = BV ? (BV + (size_t)h * (size_t)aligned_head_dim) : NULL;
        float *v_h = v + (size_t)h * kv_head_stride;
        gemm_nt_q4_k(ln1_out, wv_h, bv_h, v_h, num_tokens, aligned_head_dim, aligned_embed_dim);
    }
 
    /* RoPE */
    rope_forward_qk_strided(q,
                            k,
                            rope_cos,
                            rope_sin,
                            H,
                            H_kv,
                            num_tokens,
                            head_dim,
                            aligned_head_dim,
                            0,
                            num_tokens,
                            aligned_context_window);
 
    /* Attention (prefill, causal) */
    attention_forward_causal_head_major_gqa_flash_strided(q,
                                                           k,
                                                           v,
                                                           attn_out,
                                                           H,
                                                           H_kv,
                                                           num_tokens,
                                                           head_dim,
                                                           aligned_head_dim,
                                                           aligned_context_window);
 
    /* Output projection (flatten head-major to token-major) */
    const int K = H * aligned_head_dim;
    if (K != aligned_embed_dim) {
        return;
    }
    const float *proj_in = attn_out;
    if (H > 1) {
        if (!proj_scratch) {
            return;
        }
        for (int t = 0; t < num_tokens; ++t) {
            float *dst = proj_scratch + (size_t)t * (size_t)aligned_embed_dim;
            for (int h = 0; h < H; ++h) {
                const float *src = attn_out + (size_t)h * q_head_stride + (size_t)t * (size_t)aligned_head_dim;
                memcpy(dst + (size_t)h * (size_t)aligned_head_dim,
                       src,
                       (size_t)aligned_head_dim * sizeof(float));
            }
        }
        proj_in = proj_scratch;
    }
    gemm_nt_q4_k(proj_in, WO, BO, proj_tmp, num_tokens, aligned_embed_dim, K);
 
    /* Residual add */
    qwen2_0_5b_decode_residual_add_token_major(input, proj_tmp, residual1, num_tokens, aligned_embed_dim);
 
    /* RMSNorm before MLP */
    rmsnorm_forward(residual1,
                    ln2_gamma,
                    ln2_out,
                    NULL,
                    num_tokens,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    /* MLP (SwiGLU) */
    gemm_nt_q4_k(ln2_out, W1, B1, fc1_out, num_tokens, 2 * aligned_intermediate_dim, aligned_embed_dim);
    swiglu_forward(fc1_out, swiglu_out, num_tokens, aligned_intermediate_dim);
    gemm_nt_q4_k(swiglu_out, W2, B2, mlp_out, num_tokens, aligned_embed_dim, aligned_intermediate_dim);
 
    /* Final residual add */
    qwen2_0_5b_decode_residual_add_token_major(residual1, mlp_out, output, num_tokens, aligned_embed_dim);
}

References attention_forward_causal_head_major_gqa_flash_strided(), CK_DT_Q4_K, ck_dtype_row_bytes(), gemm_nt_q4_k(), QWEN2_0_5B_DECODELayerOffsets::k, QWEN2_0_5B_DECODELayerOffsets::ln1_gamma, QWEN2_0_5B_DECODELayerOffsets::ln1_out, QWEN2_0_5B_DECODELayerOffsets::ln2_gamma, QWEN2_0_5B_DECODELayerOffsets::ln2_out, QWEN2_0_5B_DECODELayerOffsets::mlp_out, QWEN2_0_5B_DECODELayerOffsets::output, QWEN2_0_5B_DECODELayerOffsets::proj_scratch, QWEN2_0_5B_DECODELayerOffsets::proj_tmp, QWEN2_0_5B_DECODELayerOffsets::q, QWEN2_0_5B_DECODE_EMBED_DIM, QWEN2_0_5B_DECODE_GLOBALS, QWEN2_0_5B_DECODE_HEAD_DIM, QWEN2_0_5B_DECODE_LAYERS, QWEN2_0_5B_DECODE_NUM_HEADS, QWEN2_0_5B_DECODE_NUM_KV_HEADS, QWEN2_0_5B_DECODE_PTR, qwen2_0_5b_decode_residual_add_token_major(), QWEN2_0_5B_DECODELayerOffsets::residual1, rmsnorm_forward(), QWEN2_0_5B_DECODEGlobalOffsets::rope_cos_cache, rope_forward_qk_strided(), QWEN2_0_5B_DECODEGlobalOffsets::rope_sin_cache, swiglu_forward(), QWEN2_0_5B_DECODELayerOffsets::v, QWEN2_0_5B_DECODELayerOffsets::w1, QWEN2_0_5B_DECODELayerOffsets::w2, QWEN2_0_5B_DECODELayerOffsets::wk, QWEN2_0_5B_DECODELayerOffsets::wo, QWEN2_0_5B_DECODELayerOffsets::wq, and QWEN2_0_5B_DECODELayerOffsets::wv.

Referenced by qwen2_0_5b_decode_forward_prefill_impl().

qwen2_0_5b_decode_layer_9_decode()

static void qwen2_0_5b_decode_layer_9_decode ( QWEN2_0_5B_DECODEModel *  model, int  token_index, int  aligned_embed_dim, int  aligned_head_dim, int  aligned_intermediate_dim, int  aligned_context_window  )

static

Definition at line 6034 of file v6.5/test_generated/qwen2_int8.c.

  {
    const QWEN2_0_5B_DECODELayerOffsets *L = &QWEN2_0_5B_DECODE_LAYERS[9];
 
    float *input = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[8].output);
 
    float *ln1_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln1_gamma);
    float *ln1_out = QWEN2_0_5B_DECODE_PTR(model, L->ln1_out);
    float *ln2_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln2_gamma);
    float *ln2_out = QWEN2_0_5B_DECODE_PTR(model, L->ln2_out);
    float *k_cache = QWEN2_0_5B_DECODE_PTR(model, L->k);
    float *v_cache = QWEN2_0_5B_DECODE_PTR(model, L->v);
    float *proj_tmp = QWEN2_0_5B_DECODE_PTR(model, L->proj_tmp);
    float *proj_scratch = QWEN2_0_5B_DECODE_PTR(model, L->proj_scratch);
    float *residual1 = QWEN2_0_5B_DECODE_PTR(model, L->residual1);
    float *mlp_out = QWEN2_0_5B_DECODE_PTR(model, L->mlp_out);
    float *output = QWEN2_0_5B_DECODE_PTR(model, L->output);
 
    /* Weights (explicit types for layer 9) */
    const void *WQ = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wq);  /* Q4_K */
    const void *WK = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wk);  /* Q4_K */
    const void *WV = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wv);  /* Q4_K */
    const void *WO = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wo);  /* Q4_K */
    const void *W1 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w1);  /* Q4_K (gate+up) */
    const void *W2 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w2);  /* Q4_K (down) */
 
    float *rope_cos = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_cos_cache);
    float *rope_sin = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_sin_cache);
 
    const int H = QWEN2_0_5B_DECODE_NUM_HEADS;
    const int H_kv = QWEN2_0_5B_DECODE_NUM_KV_HEADS;
    const int head_dim = QWEN2_0_5B_DECODE_HEAD_DIM;
 
    float q_token[H * aligned_head_dim];
    float k_token[H_kv * aligned_head_dim];
    float v_token[H_kv * aligned_head_dim];
    float attn_token[H * aligned_head_dim];
 
    /* Local MLP buffers (avoid layout dependencies for intermediate values) */
    float fc1_out[2 * aligned_intermediate_dim];
    float swiglu_out[aligned_intermediate_dim];
 
    /* Step 1: RMSNorm before attention */
    rmsnorm_forward(input,
                    ln1_gamma,
                    ln1_out,
                    NULL,
                    1,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    const size_t kv_head_stride = (size_t)aligned_context_window * (size_t)aligned_head_dim;
 
    /* Step 2: QKV projection */
    /* Q projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t ln1_q8_bytes = ((((aligned_embed_dim) + 255) / 256) * 292);
    uint8_t ln1_q8[ln1_q8_bytes];
    quantize_row_q8_k(ln1_out, ln1_q8, aligned_embed_dim);
    gemv_q4_k_q8_k(q_token, WQ, ln1_q8, H * head_dim, aligned_embed_dim);
    if (aligned_head_dim > head_dim) {
        for (int h = 0; h < H; ++h) {
            float *q_head = q_token + (size_t)h * (size_t)aligned_head_dim;
            for (int d = head_dim; d < aligned_head_dim; ++d) {
                q_head[d] = 0.0f;
            }
        }
    }
 
    /* K projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k (direct-to-cache) */
    const size_t wk_head_elems = (size_t)aligned_head_dim * (size_t)aligned_embed_dim;
    const size_t wk_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, wk_head_elems);
    const uint8_t *WK_bytes = (const uint8_t *)WK;
    /* ln1_q8 already quantized above */
    for (int h = 0; h < H_kv; ++h) {
        const void *wk_h = (const void *)(WK_bytes + (size_t)h * wk_head_bytes);
        float *k_head = k_cache + (size_t)h * kv_head_stride + (size_t)token_index * (size_t)aligned_head_dim;
        gemv_q4_k_q8_k(k_head, wk_h, ln1_q8, head_dim, aligned_embed_dim);
        for (int d = head_dim; d < aligned_head_dim; ++d) {
            k_head[d] = 0.0f;
        }
    }
 
    /* V projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k (direct-to-cache) */
    const size_t wv_head_elems = (size_t)aligned_head_dim * (size_t)aligned_embed_dim;
    const size_t wv_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, wv_head_elems);
    const uint8_t *WV_bytes = (const uint8_t *)WV;
    /* ln1_q8 already quantized above */
    for (int h = 0; h < H_kv; ++h) {
        const void *wv_h = (const void *)(WV_bytes + (size_t)h * wv_head_bytes);
        float *v_head = v_cache + (size_t)h * kv_head_stride + (size_t)token_index * (size_t)aligned_head_dim;
        gemv_q4_k_q8_k(v_head, wv_h, ln1_q8, head_dim, aligned_embed_dim);
        for (int d = head_dim; d < aligned_head_dim; ++d) {
            v_head[d] = 0.0f;
        }
    }
 
    /* Step 3: RoPE */
    rope_forward(q_token,
                 rope_cos,
                 rope_sin,
                 H,
                 1,
                 head_dim,
                 aligned_head_dim,
                 token_index);
    for (int h = 0; h < H_kv; ++h) {
        float *k_head = k_cache + (size_t)h * kv_head_stride + (size_t)token_index * (size_t)aligned_head_dim;
        rope_forward(k_head,
                     rope_cos,
                     rope_sin,
                     1,
                     1,
                     head_dim,
                     aligned_head_dim,
                     token_index);
    }
 
    /* Step 4: KV cache write (direct-to-cache) */
 
    /* Step 5: Attention (decode, flash) */
    attention_forward_decode_head_major_gqa_flash(q_token,
                                                   k_cache,
                                                   v_cache,
                                                   attn_token,
                                                   H,
                                                   H_kv,
                                                   token_index + 1,
                                                   aligned_context_window,
                                                   head_dim,
                                                   aligned_head_dim);
 
    /* Step 6: Output projection */
    /* WO projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t attn_q8_bytes = ((((H * head_dim) + 255) / 256) * 292);
    uint8_t attn_q8[attn_q8_bytes];
    quantize_row_q8_k(attn_token, attn_q8, H * head_dim);
    gemv_q4_k_q8_k(proj_tmp, WO, attn_q8, aligned_embed_dim, H * head_dim);
 
    /* Step 7: Residual add */
    qwen2_0_5b_decode_residual_add_token_major(input, proj_tmp, residual1, 1, aligned_embed_dim);
 
    /* Step 8: RMSNorm before MLP */
    rmsnorm_forward(residual1,
                    ln2_gamma,
                    ln2_out,
                    NULL,
                    1,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    /* Step 9: MLP (SwiGLU) */
    /* Gate+Up projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t ln2_q8_bytes = ((((aligned_embed_dim) + 255) / 256) * 292);
    uint8_t ln2_q8[ln2_q8_bytes];
    quantize_row_q8_k(ln2_out, ln2_q8, aligned_embed_dim);
    gemv_q4_k_q8_k(fc1_out, W1, ln2_q8, 2 * aligned_intermediate_dim, aligned_embed_dim);
 
    /* SwiGLU activation */
    swiglu_forward(fc1_out, swiglu_out, 1, aligned_intermediate_dim);
 
    /* Down projection (INT8): Q4_K x Q8_K -> gemv_q4_k_q8_k */
    const size_t swiglu_q8_bytes = ((((aligned_intermediate_dim) + 255) / 256) * 292);
    uint8_t swiglu_q8[swiglu_q8_bytes];
    quantize_row_q8_k(swiglu_out, swiglu_q8, aligned_intermediate_dim);
    gemv_q4_k_q8_k(mlp_out, W2, swiglu_q8, aligned_embed_dim, aligned_intermediate_dim);
 
    /* Step 10: Final residual add */
    qwen2_0_5b_decode_residual_add_token_major(residual1, mlp_out, output, 1, aligned_embed_dim);
}

References attention_forward_decode_head_major_gqa_flash(), CK_DT_Q4_K, ck_dtype_row_bytes(), gemv_q4_k_q8_k(), QWEN2_0_5B_DECODELayerOffsets::k, QWEN2_0_5B_DECODELayerOffsets::ln1_gamma, QWEN2_0_5B_DECODELayerOffsets::ln1_out, QWEN2_0_5B_DECODELayerOffsets::ln2_gamma, QWEN2_0_5B_DECODELayerOffsets::ln2_out, QWEN2_0_5B_DECODELayerOffsets::mlp_out, QWEN2_0_5B_DECODELayerOffsets::output, QWEN2_0_5B_DECODELayerOffsets::proj_scratch, QWEN2_0_5B_DECODELayerOffsets::proj_tmp, quantize_row_q8_k(), QWEN2_0_5B_DECODE_EMBED_DIM, QWEN2_0_5B_DECODE_GLOBALS, QWEN2_0_5B_DECODE_HEAD_DIM, QWEN2_0_5B_DECODE_LAYERS, QWEN2_0_5B_DECODE_NUM_HEADS, QWEN2_0_5B_DECODE_NUM_KV_HEADS, QWEN2_0_5B_DECODE_PTR, qwen2_0_5b_decode_residual_add_token_major(), QWEN2_0_5B_DECODELayerOffsets::residual1, rmsnorm_forward(), QWEN2_0_5B_DECODEGlobalOffsets::rope_cos_cache, rope_forward(), QWEN2_0_5B_DECODEGlobalOffsets::rope_sin_cache, swiglu_forward(), QWEN2_0_5B_DECODELayerOffsets::v, QWEN2_0_5B_DECODELayerOffsets::w1, QWEN2_0_5B_DECODELayerOffsets::w2, QWEN2_0_5B_DECODELayerOffsets::wk, QWEN2_0_5B_DECODELayerOffsets::wo, QWEN2_0_5B_DECODELayerOffsets::wq, and QWEN2_0_5B_DECODELayerOffsets::wv.

Referenced by qwen2_0_5b_decode_decode_token().

qwen2_0_5b_decode_layer_9_prefill()

static void qwen2_0_5b_decode_layer_9_prefill ( QWEN2_0_5B_DECODEModel *  model, int  num_tokens, int  aligned_embed_dim, int  aligned_head_dim, int  aligned_intermediate_dim, int  aligned_context_window  )

static

Definition at line 1687 of file v6.5/test_generated/qwen2_int8.c.

  {
    const QWEN2_0_5B_DECODELayerOffsets *L = &QWEN2_0_5B_DECODE_LAYERS[9];
 
    float *input = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[8].output);
    float *ln1_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln1_gamma);
    float *ln1_out = QWEN2_0_5B_DECODE_PTR(model, L->ln1_out);
    float *ln2_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln2_gamma);
    float *ln2_out = QWEN2_0_5B_DECODE_PTR(model, L->ln2_out);
    float *q = QWEN2_0_5B_DECODE_PTR(model, L->q);
    float *k = QWEN2_0_5B_DECODE_PTR(model, L->k);
    float *v = QWEN2_0_5B_DECODE_PTR(model, L->v);
    float *attn_out = QWEN2_0_5B_DECODE_PTR(model, L->attn_out);
    float *proj_tmp = QWEN2_0_5B_DECODE_PTR(model, L->proj_tmp);
    float *proj_scratch = QWEN2_0_5B_DECODE_PTR(model, L->proj_scratch);
    float *residual1 = QWEN2_0_5B_DECODE_PTR(model, L->residual1);
    float *fc1_out = QWEN2_0_5B_DECODE_PTR(model, L->fc1_out);
    float *swiglu_out = QWEN2_0_5B_DECODE_PTR(model, L->swiglu_out);
    float *mlp_out = QWEN2_0_5B_DECODE_PTR(model, L->mlp_out);
    float *output = QWEN2_0_5B_DECODE_PTR(model, L->output);
 
    const void *WQ = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wq);
    const void *WK = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wk);
    const void *WV = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wv);
    const void *WO = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wo);
    const void *W1 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w1);
    const void *W2 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w2);
    const float *BQ = NULL;
    const float *BK = NULL;
    const float *BV = NULL;
    const float *BO = NULL;
    const float *B1 = NULL;
    const float *B2 = NULL;
 
    float *rope_cos = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_cos_cache);
    float *rope_sin = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_sin_cache);
 
    const int H = QWEN2_0_5B_DECODE_NUM_HEADS;
    const int H_kv = QWEN2_0_5B_DECODE_NUM_KV_HEADS;
    const int head_dim = QWEN2_0_5B_DECODE_HEAD_DIM;
    const size_t head_w_elems = (size_t)aligned_head_dim * (size_t)aligned_embed_dim;
    const size_t q_head_stride = (size_t)num_tokens * (size_t)aligned_head_dim;
    const size_t kv_head_stride = (size_t)aligned_context_window * (size_t)aligned_head_dim;
 
    /* RMSNorm before attention */
    rmsnorm_forward(input,
                    ln1_gamma,
                    ln1_out,
                    NULL,
                    num_tokens,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    /* Q projection (head-major) */
    const size_t wq_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, head_w_elems);
    const uint8_t *WQ_bytes = (const uint8_t *)WQ;
    for (int h = 0; h < H; ++h) {
        const void *wq_h = (const void *)(WQ_bytes + (size_t)h * wq_head_bytes);
        const float *bq_h = BQ ? (BQ + (size_t)h * (size_t)aligned_head_dim) : NULL;
        float *q_h = q + (size_t)h * q_head_stride;
        gemm_nt_q4_k(ln1_out, wq_h, bq_h, q_h, num_tokens, aligned_head_dim, aligned_embed_dim);
    }
 
    /* K projection (head-major) */
    const size_t wk_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, head_w_elems);
    const uint8_t *WK_bytes = (const uint8_t *)WK;
    for (int h = 0; h < H_kv; ++h) {
        const void *wk_h = (const void *)(WK_bytes + (size_t)h * wk_head_bytes);
        const float *bk_h = BK ? (BK + (size_t)h * (size_t)aligned_head_dim) : NULL;
        float *k_h = k + (size_t)h * kv_head_stride;
        gemm_nt_q4_k(ln1_out, wk_h, bk_h, k_h, num_tokens, aligned_head_dim, aligned_embed_dim);
    }
 
    /* V projection (head-major) */
    const size_t wv_head_bytes = ck_dtype_row_bytes(CK_DT_Q4_K, head_w_elems);
    const uint8_t *WV_bytes = (const uint8_t *)WV;
    for (int h = 0; h < H_kv; ++h) {
        const void *wv_h = (const void *)(WV_bytes + (size_t)h * wv_head_bytes);
        const float *bv_h = BV ? (BV + (size_t)h * (size_t)aligned_head_dim) : NULL;
        float *v_h = v + (size_t)h * kv_head_stride;
        gemm_nt_q4_k(ln1_out, wv_h, bv_h, v_h, num_tokens, aligned_head_dim, aligned_embed_dim);
    }
 
    /* RoPE */
    rope_forward_qk_strided(q,
                            k,
                            rope_cos,
                            rope_sin,
                            H,
                            H_kv,
                            num_tokens,
                            head_dim,
                            aligned_head_dim,
                            0,
                            num_tokens,
                            aligned_context_window);
 
    /* Attention (prefill, causal) */
    attention_forward_causal_head_major_gqa_flash_strided(q,
                                                           k,
                                                           v,
                                                           attn_out,
                                                           H,
                                                           H_kv,
                                                           num_tokens,
                                                           head_dim,
                                                           aligned_head_dim,
                                                           aligned_context_window);
 
    /* Output projection (flatten head-major to token-major) */
    const int K = H * aligned_head_dim;
    if (K != aligned_embed_dim) {
        return;
    }
    const float *proj_in = attn_out;
    if (H > 1) {
        if (!proj_scratch) {
            return;
        }
        for (int t = 0; t < num_tokens; ++t) {
            float *dst = proj_scratch + (size_t)t * (size_t)aligned_embed_dim;
            for (int h = 0; h < H; ++h) {
                const float *src = attn_out + (size_t)h * q_head_stride + (size_t)t * (size_t)aligned_head_dim;
                memcpy(dst + (size_t)h * (size_t)aligned_head_dim,
                       src,
                       (size_t)aligned_head_dim * sizeof(float));
            }
        }
        proj_in = proj_scratch;
    }
    gemm_nt_q4_k(proj_in, WO, BO, proj_tmp, num_tokens, aligned_embed_dim, K);
 
    /* Residual add */
    qwen2_0_5b_decode_residual_add_token_major(input, proj_tmp, residual1, num_tokens, aligned_embed_dim);
 
    /* RMSNorm before MLP */
    rmsnorm_forward(residual1,
                    ln2_gamma,
                    ln2_out,
                    NULL,
                    num_tokens,
                    QWEN2_0_5B_DECODE_EMBED_DIM,
                    aligned_embed_dim,
                    1e-06f);
 
    /* MLP (SwiGLU) */
    gemm_nt_q4_k(ln2_out, W1, B1, fc1_out, num_tokens, 2 * aligned_intermediate_dim, aligned_embed_dim);
    swiglu_forward(fc1_out, swiglu_out, num_tokens, aligned_intermediate_dim);
    gemm_nt_q4_k(swiglu_out, W2, B2, mlp_out, num_tokens, aligned_embed_dim, aligned_intermediate_dim);
 
    /* Final residual add */
    qwen2_0_5b_decode_residual_add_token_major(residual1, mlp_out, output, num_tokens, aligned_embed_dim);
}

References attention_forward_causal_head_major_gqa_flash_strided(), CK_DT_Q4_K, ck_dtype_row_bytes(), gemm_nt_q4_k(), QWEN2_0_5B_DECODELayerOffsets::k, QWEN2_0_5B_DECODELayerOffsets::ln1_gamma, QWEN2_0_5B_DECODELayerOffsets::ln1_out, QWEN2_0_5B_DECODELayerOffsets::ln2_gamma, QWEN2_0_5B_DECODELayerOffsets::ln2_out, QWEN2_0_5B_DECODELayerOffsets::mlp_out, QWEN2_0_5B_DECODELayerOffsets::output, QWEN2_0_5B_DECODELayerOffsets::proj_scratch, QWEN2_0_5B_DECODELayerOffsets::proj_tmp, QWEN2_0_5B_DECODELayerOffsets::q, QWEN2_0_5B_DECODE_EMBED_DIM, QWEN2_0_5B_DECODE_GLOBALS, QWEN2_0_5B_DECODE_HEAD_DIM, QWEN2_0_5B_DECODE_LAYERS, QWEN2_0_5B_DECODE_NUM_HEADS, QWEN2_0_5B_DECODE_NUM_KV_HEADS, QWEN2_0_5B_DECODE_PTR, qwen2_0_5b_decode_residual_add_token_major(), QWEN2_0_5B_DECODELayerOffsets::residual1, rmsnorm_forward(), QWEN2_0_5B_DECODEGlobalOffsets::rope_cos_cache, rope_forward_qk_strided(), QWEN2_0_5B_DECODEGlobalOffsets::rope_sin_cache, swiglu_forward(), QWEN2_0_5B_DECODELayerOffsets::v, QWEN2_0_5B_DECODELayerOffsets::w1, QWEN2_0_5B_DECODELayerOffsets::w2, QWEN2_0_5B_DECODELayerOffsets::wk, QWEN2_0_5B_DECODELayerOffsets::wo, QWEN2_0_5B_DECODELayerOffsets::wq, and QWEN2_0_5B_DECODELayerOffsets::wv.

Referenced by qwen2_0_5b_decode_forward_prefill_impl().

qwen2_0_5b_decode_model_allocate()

int qwen2_0_5b_decode_model_allocate

(

QWEN2_0_5B_DECODEModel *

model

)

Definition at line 88 of file v6.5/test_generated/qwen2_int8.c.

                                                                    {
    size_t total = QWEN2_0_5B_DECODE_TOTAL_BYTES;
 
#ifdef __linux__
    model->base = mmap(NULL, total,
                       PROT_READ | PROT_WRITE,
                       MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB,
                       -1, 0);
    if (model->base == MAP_FAILED) {
        model->base = mmap(NULL, total,
                           PROT_READ | PROT_WRITE,
                           MAP_PRIVATE | MAP_ANONYMOUS,
                           -1, 0);
    }
    if (model->base == MAP_FAILED) {
        perror("mmap failed");
        return -1;
    }
#else
    model->base = aligned_alloc(64, total);
    if (!model->base) {
        perror("aligned_alloc failed");
        return -1;
    }
#endif
 
    model->total_bytes = total;
 
    /* Initialize magic header */
    MagicHeader *header = (MagicHeader *)model->base;
    header->magic = QWEN2_0_5B_DECODE_MAGIC;
    header->version = 5;
    header->total_bytes = QWEN2_0_5B_DECODE_TOTAL_BYTES;
    header->weight_bytes = QWEN2_0_5B_DECODE_WEIGHT_BYTES;
    header->activation_bytes = QWEN2_0_5B_DECODE_ACTIVATION_BYTES;
    header->num_layers = QWEN2_0_5B_DECODE_NUM_LAYERS;
    header->embed_dim = QWEN2_0_5B_DECODE_EMBED_DIM;
    header->num_heads = QWEN2_0_5B_DECODE_NUM_HEADS;
    header->vocab_size = QWEN2_0_5B_DECODE_VOCAB_SIZE;
    header->max_seq_len = QWEN2_0_5B_DECODE_MAX_SEQ_LEN;
    header->canary_count = QWEN2_0_5B_DECODE_CANARY_COUNT;
 
    /* Initialize canary guards */
    for (int i = 0; i < QWEN2_0_5B_DECODE_CANARY_COUNT; i++) {
        uint32_t *ptr = (uint32_t*)((char*)model->base + QWEN2_0_5B_DECODE_CANARIES[i].offset);
        for (int j = 0; j < (QWEN2_0_5B_DECODE_CANARY_SIZE / 4); j++) {
            ptr[j] = QWEN2_0_5B_DECODE_CANARY_VALUE;
        }
    }
 
    return 0;
}

References QWEN2_0_5B_DECODEModel::base, MagicHeader, QWEN2_0_5B_DECODECanary::offset, QWEN2_0_5B_DECODE_ACTIVATION_BYTES, QWEN2_0_5B_DECODE_CANARIES, QWEN2_0_5B_DECODE_CANARY_COUNT, QWEN2_0_5B_DECODE_CANARY_SIZE, QWEN2_0_5B_DECODE_CANARY_VALUE, QWEN2_0_5B_DECODE_EMBED_DIM, QWEN2_0_5B_DECODE_MAGIC, QWEN2_0_5B_DECODE_MAX_SEQ_LEN, QWEN2_0_5B_DECODE_NUM_HEADS, QWEN2_0_5B_DECODE_NUM_LAYERS, QWEN2_0_5B_DECODE_TOTAL_BYTES, QWEN2_0_5B_DECODE_VOCAB_SIZE, QWEN2_0_5B_DECODE_WEIGHT_BYTES, and QWEN2_0_5B_DECODEModel::total_bytes.

Referenced by ck_model_create().

qwen2_0_5b_decode_model_free()

void qwen2_0_5b_decode_model_free

(

QWEN2_0_5B_DECODEModel *

model

)

Definition at line 141 of file v6.5/test_generated/qwen2_int8.c.

                                                                 {
    if (!model || !model->base) return;
#ifdef __linux__
    munmap(model->base, model->total_bytes);
#else
    free(model->base);
#endif
    model->base = NULL;
    model->total_bytes = 0;
}

References QWEN2_0_5B_DECODEModel::base, and QWEN2_0_5B_DECODEModel::total_bytes.

Referenced by ck_model_free().

qwen2_0_5b_decode_precompute_rope()

void qwen2_0_5b_decode_precompute_rope

(

QWEN2_0_5B_DECODEModel *

model

)

Definition at line 186 of file v6.5/test_generated/qwen2_int8.c.

                                                                      {
    const int T = QWEN2_0_5B_DECODE_MAX_SEQ_LEN;
    const int D = QWEN2_0_5B_DECODE_HEAD_DIM / 2;
    const float theta = 1000000.0f;
 
    float *cos_ptr = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_cos_cache);
    float *sin_ptr = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_sin_cache);
 
    for (int pos = 0; pos < T; pos++) {
        for (int i = 0; i < D; i++) {
            float freq = 1.0f / powf(theta, (float)(2 * i) / (float)(D * 2));
            float angle = (float)pos * freq;
            cos_ptr[pos * D + i] = cosf(angle);
            sin_ptr[pos * D + i] = sinf(angle);
        }
    }
}

References QWEN2_0_5B_DECODE_GLOBALS, QWEN2_0_5B_DECODE_HEAD_DIM, QWEN2_0_5B_DECODE_MAX_SEQ_LEN, QWEN2_0_5B_DECODE_PTR, QWEN2_0_5B_DECODEGlobalOffsets::rope_cos_cache, and QWEN2_0_5B_DECODEGlobalOffsets::rope_sin_cache.

Referenced by ck_model_precompute_rope().

qwen2_0_5b_decode_residual_add_token_major()

static void qwen2_0_5b_decode_residual_add_token_major ( const float *  a, const float *  b, float *  out, int  tokens, int  aligned_embed_dim  )

static

Definition at line 43 of file v6.5/test_generated/qwen2_int8.c.

  {
    if (!a || !b || !out) {
        return;
    }
    for (int t = 0; t < tokens; ++t) {
        const float *pa = a + (size_t)t * (size_t)aligned_embed_dim;
        const float *pb = b + (size_t)t * (size_t)aligned_embed_dim;
        float *pc = out + (size_t)t * (size_t)aligned_embed_dim;
        for (int d = 0; d < aligned_embed_dim; ++d) {
            pc[d] = pa[d] + pb[d];
        }
    }
}

Referenced by qwen2_0_5b_decode_layer_0_decode(), qwen2_0_5b_decode_layer_0_prefill(), qwen2_0_5b_decode_layer_10_decode(), qwen2_0_5b_decode_layer_10_prefill(), qwen2_0_5b_decode_layer_11_decode(), qwen2_0_5b_decode_layer_11_prefill(), qwen2_0_5b_decode_layer_12_decode(), qwen2_0_5b_decode_layer_12_prefill(), qwen2_0_5b_decode_layer_13_decode(), qwen2_0_5b_decode_layer_13_prefill(), qwen2_0_5b_decode_layer_14_decode(), qwen2_0_5b_decode_layer_14_prefill(), qwen2_0_5b_decode_layer_15_decode(), qwen2_0_5b_decode_layer_15_prefill(), qwen2_0_5b_decode_layer_16_decode(), qwen2_0_5b_decode_layer_16_prefill(), qwen2_0_5b_decode_layer_17_decode(), qwen2_0_5b_decode_layer_17_prefill(), qwen2_0_5b_decode_layer_18_decode(), qwen2_0_5b_decode_layer_18_prefill(), qwen2_0_5b_decode_layer_19_decode(), qwen2_0_5b_decode_layer_19_prefill(), qwen2_0_5b_decode_layer_1_decode(), qwen2_0_5b_decode_layer_1_prefill(), qwen2_0_5b_decode_layer_20_decode(), qwen2_0_5b_decode_layer_20_prefill(), qwen2_0_5b_decode_layer_21_decode(), qwen2_0_5b_decode_layer_21_prefill(), qwen2_0_5b_decode_layer_22_decode(), qwen2_0_5b_decode_layer_22_prefill(), qwen2_0_5b_decode_layer_23_decode(), qwen2_0_5b_decode_layer_23_prefill(), qwen2_0_5b_decode_layer_2_decode(), qwen2_0_5b_decode_layer_2_prefill(), qwen2_0_5b_decode_layer_3_decode(), qwen2_0_5b_decode_layer_3_prefill(), qwen2_0_5b_decode_layer_4_decode(), qwen2_0_5b_decode_layer_4_prefill(), qwen2_0_5b_decode_layer_5_decode(), qwen2_0_5b_decode_layer_5_prefill(), qwen2_0_5b_decode_layer_6_decode(), qwen2_0_5b_decode_layer_6_prefill(), qwen2_0_5b_decode_layer_7_decode(), qwen2_0_5b_decode_layer_7_prefill(), qwen2_0_5b_decode_layer_8_decode(), qwen2_0_5b_decode_layer_8_prefill(), qwen2_0_5b_decode_layer_9_decode(), and qwen2_0_5b_decode_layer_9_prefill().

qwen2_0_5b_decode_verify_canaries()

int qwen2_0_5b_decode_verify_canaries

(

QWEN2_0_5B_DECODEModel *

model

)

Definition at line 152 of file v6.5/test_generated/qwen2_int8.c.

                                                                     {
    int errors = 0;
    uint32_t *ptr;
 
    for (int i = 0; i < QWEN2_0_5B_DECODE_CANARY_COUNT; i++) {
        ptr = (uint32_t*)((char*)model->base + QWEN2_0_5B_DECODE_CANARIES[i].offset);
        for (int j = 0; j < 4; j++) {
            if (ptr[j] != QWEN2_0_5B_DECODE_CANARY_VALUE) {
                fprintf(stderr, "CANARY CORRUPTION: %s at offset 0x%lX\n",
                        QWEN2_0_5B_DECODE_CANARIES[i].name,
                        QWEN2_0_5B_DECODE_CANARIES[i].offset);
                errors++;
                break;
            }
        }
    }
 
    return errors;
}

References QWEN2_0_5B_DECODEModel::base, QWEN2_0_5B_DECODECanary::offset, QWEN2_0_5B_DECODE_CANARIES, QWEN2_0_5B_DECODE_CANARY_COUNT, and QWEN2_0_5B_DECODE_CANARY_VALUE.

Referenced by ck_model_verify_canaries().

Variable Documentation

g_model_config

CKModelConfig g_model_config

static

Initial value:

= {
    .embed_dim =  896 ,
    .num_heads =  14 ,
    .num_kv_heads =  2 ,
    .head_dim =  64 ,
    .intermediate_size =  4864 ,
    .num_layers =  24 ,
    .vocab_size =  151936 ,
    .max_seq_len =  131072 ,
    .total_bytes =  3573889600ULL ,
    .weight_bytes =  317683328ULL ,
    .activation_bytes =  3256169984ULL ,
    .model_name = "qwen2_0.5b_decode",
    .model_family = "qwen2",
}

Definition at line 8853 of file v6.5/test_generated/qwen2_int8.c.

Referenced by ck_model_get_config().

MagicHeader

MagicHeader

Definition at line 80 of file v6.5/test_generated/qwen2_int8.c.

Referenced by qwen2_0_5b_decode_model_allocate().