← Back to C-Kernel-Engine Docs Doxygen Source Documentation
v6.6/generated/ck-kernel-inference.c
Go to the documentation of this file.
1 /**
2  * @file ck-kernel-inference.c
3  * @brief AUTO-GENERATED: qwen2_0.5b_decode Implementation (IR v6 - Explicit Unrolled)
4  *
5  * Generated: 2026-01-15T19:23:13.600148 UTC
6  * Total Memory: 3.65 GB
7  * Mode: decode
8  * Layers: 24 (fully unrolled)
9  *
10  * Per-layer quant types:
11  * Layer 0: wq=q5_0 wk=q5_0 wv=q8_0 wo=q5_0 w1=q5_0 w2=q6_k
12  * Layer 1: wq=q5_0 wk=q5_0 wv=q8_0 wo=q5_0 w1=q5_0 w2=q6_k
13  * Layer 2: wq=q5_0 wk=q5_0 wv=q5_0 wo=q5_0 w1=q5_0 w2=q4_k
14  * ... (21 more layers)
15  *
16  * ═══════════════════════════════════════════════════════════════════════════
17  * MANIFEST VALIDATION (from weights_manifest.json)
18  * ═══════════════════════════════════════════════════════════════════════════
19  *
20  * Layer | WQ | WK | WV | WO | W1 | W2 | BQ | BK | BV | BO
21  * ------|-------|-------|-------|-------|-------|-------|----|----|----|----|
22  * 0 | q5_0 | q5_0 | q8_0 | q5_0 | q5_0 | q6_k | ✓ | ✓ | ✓ | ○
23  * 1 | q5_0 | q5_0 | q8_0 | q5_0 | q5_0 | q6_k | ✓ | ✓ | ✓ | ○
24  * 2 | q5_0 | q5_0 | q5_0 | q5_0 | q5_0 | q4_k | ✓ | ✓ | ✓ | ○
25  * 3 | q5_0 | q5_0 | q8_0 | q5_0 | q5_0 | q6_k | ✓ | ✓ | ✓ | ○
26  * 4 | q5_0 | q5_0 | q5_0 | q5_0 | q5_0 | q4_k | ✓ | ✓ | ✓ | ○
27  * 5 | q5_0 | q5_0 | q5_0 | q5_0 | q5_0 | q4_k | ✓ | ✓ | ✓ | ○
28  * 6 | q5_0 | q5_0 | q8_0 | q5_0 | q5_0 | q6_k | ✓ | ✓ | ✓ | ○
29  * 7 | q5_0 | q5_0 | q8_0 | q5_0 | q5_0 | q6_k | ✓ | ✓ | ✓ | ○
30  * 8 | q5_0 | q5_0 | q8_0 | q5_0 | q5_0 | q6_k | ✓ | ✓ | ✓ | ○
31  * 9 | q5_0 | q5_0 | q8_0 | q5_0 | q5_0 | q6_k | ✓ | ✓ | ✓ | ○
32  * 10 | q5_0 | q5_0 | q8_0 | q5_0 | q5_0 | q6_k | ✓ | ✓ | ✓ | ○
33  * 11 | q5_0 | q5_0 | q5_0 | q5_0 | q5_0 | q4_k | ✓ | ✓ | ✓ | ○
34  * 12 | q5_0 | q5_0 | q5_0 | q5_0 | q5_0 | q4_k | ✓ | ✓ | ✓ | ○
35  * 13 | q5_0 | q5_0 | q8_0 | q5_0 | q5_0 | q6_k | ✓ | ✓ | ✓ | ○
36  * 14 | q5_0 | q5_0 | q5_0 | q5_0 | q5_0 | q4_k | ✓ | ✓ | ✓ | ○
37  * 15 | q5_0 | q5_0 | q5_0 | q5_0 | q5_0 | q4_k | ✓ | ✓ | ✓ | ○
38  * 16 | q5_0 | q5_0 | q8_0 | q5_0 | q5_0 | q6_k | ✓ | ✓ | ✓ | ○
39  * 17 | q5_0 | q5_0 | q5_0 | q5_0 | q5_0 | q4_k | ✓ | ✓ | ✓ | ○
40  * 18 | q5_0 | q5_0 | q5_0 | q5_0 | q5_0 | q4_k | ✓ | ✓ | ✓ | ○
41  * 19 | q5_0 | q5_0 | q8_0 | q5_0 | q5_0 | q6_k | ✓ | ✓ | ✓ | ○
42  * 20 | q5_0 | q5_0 | q5_0 | q5_0 | q5_0 | q4_k | ✓ | ✓ | ✓ | ○
43  * 21 | q5_0 | q5_0 | q8_0 | q5_0 | q5_0 | q6_k | ✓ | ✓ | ✓ | ○
44  * 22 | q5_0 | q5_0 | q5_0 | q5_0 | q5_0 | q4_k | ✓ | ✓ | ✓ | ○
45  * 23 | q5_0 | q5_0 | q5_0 | q5_0 | q5_0 | q4_k | ✓ | ✓ | ✓ | ○
46  *
47  * Total manifest entries: 269
48  * Attention biases: PRESENT (Qwen2-style)
49  * ═══════════════════════════════════════════════════════════════════════════
50  *
51  *
52  * DO NOT EDIT - Regenerate with build_ir_v6.py or codegen_v6.py
53  */
54 
55 #define _GNU_SOURCE /* For MAP_ANONYMOUS, MAP_HUGETLB */
56 
57 #include "ck-kernel-inference.h"
58 
59 #include "ckernel_engine.h"
60 
61 #include <stdio.h>
62 #include <stdlib.h>
63 #include <string.h>
64 #include <stdint.h>
65 #include <math.h>
66 
67 #ifdef __linux__
68 #include <sys/mman.h>
69 #endif
70 
71 #if QWEN2_0_5B_DECODE_DTYPE_BYTES != 4
72 #error "qwen2_0.5b_decode: v6 codegen currently supports fp32 only. Use --dtype=fp32."
73 #endif
74 
75 /* ============================================================================
76  * LOCAL HELPERS (no orchestration dependency)
77  * ============================================================================ */
78 
79 static void qwen2_0_5b_decode_residual_add_token_major(
80  const float *a,
81  const float *b,
82  float *out,
83  int tokens,
84  int aligned_embed_dim
85 ) {
86  if (!a || !b || !out) {
87  return;
88  }
89  for (int t = 0; t < tokens; ++t) {
90  const float *pa = a + (size_t)t * (size_t)aligned_embed_dim;
91  const float *pb = b + (size_t)t * (size_t)aligned_embed_dim;
92  float *pc = out + (size_t)t * (size_t)aligned_embed_dim;
93  for (int d = 0; d < aligned_embed_dim; ++d) {
94  pc[d] = pa[d] + pb[d];
95  }
96  }
97 }
98 
99 /* ============================================================================
100  * MAGIC HEADER
101  * ============================================================================ */
102 
103 typedef struct __attribute__((packed)) {
104  uint32_t magic; /* 0x434B454E */
105  uint32_t version; /* IR version */
106  uint64_t total_bytes;
107  uint64_t weight_bytes;
108  uint64_t activation_bytes;
109  uint32_t num_layers;
110  uint32_t embed_dim;
111  uint32_t num_heads;
112  uint32_t vocab_size;
113  uint32_t max_seq_len;
114  uint32_t canary_count;
115  uint8_t reserved[8]; /* Pad to 64 bytes */
116 } MagicHeader;
117 
118 _Static_assert(sizeof(MagicHeader) == 64, "MagicHeader must be 64 bytes");
119 
120 /* ============================================================================
121  * ALLOCATION
122  * ============================================================================ */
123 
124 int qwen2_0_5b_decode_model_allocate(QWEN2_0_5B_DECODEModel *model) {
125  size_t total = QWEN2_0_5B_DECODE_TOTAL_BYTES;
126 
127 #ifdef __linux__
128  model->base = mmap(NULL, total,
129  PROT_READ | PROT_WRITE,
130  MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB,
131  -1, 0);
132  if (model->base == MAP_FAILED) {
133  model->base = mmap(NULL, total,
134  PROT_READ | PROT_WRITE,
135  MAP_PRIVATE | MAP_ANONYMOUS,
136  -1, 0);
137  }
138  if (model->base == MAP_FAILED) {
139  perror("mmap failed");
140  return -1;
141  }
142 #else
143  model->base = aligned_alloc(64, total);
144  if (!model->base) {
145  perror("aligned_alloc failed");
146  return -1;
147  }
148 #endif
149 
150  model->total_bytes = total;
151 
152  /* Initialize magic header */
153  MagicHeader *header = (MagicHeader *)model->base;
154  header->magic = QWEN2_0_5B_DECODE_MAGIC;
155  header->version = 5;
156  header->total_bytes = QWEN2_0_5B_DECODE_TOTAL_BYTES;
157  header->weight_bytes = QWEN2_0_5B_DECODE_WEIGHT_BYTES;
158  header->activation_bytes = QWEN2_0_5B_DECODE_ACTIVATION_BYTES;
159  header->num_layers = QWEN2_0_5B_DECODE_NUM_LAYERS;
160  header->embed_dim = QWEN2_0_5B_DECODE_EMBED_DIM;
161  header->num_heads = QWEN2_0_5B_DECODE_NUM_HEADS;
162  header->vocab_size = QWEN2_0_5B_DECODE_VOCAB_SIZE;
163  header->max_seq_len = QWEN2_0_5B_DECODE_MAX_SEQ_LEN;
164  header->canary_count = QWEN2_0_5B_DECODE_CANARY_COUNT;
165 
166  /* Initialize canary guards */
167  for (int i = 0; i < QWEN2_0_5B_DECODE_CANARY_COUNT; i++) {
168  uint32_t *ptr = (uint32_t*)((char*)model->base + QWEN2_0_5B_DECODE_CANARIES[i].offset);
169  for (int j = 0; j < (QWEN2_0_5B_DECODE_CANARY_SIZE / 4); j++) {
170  ptr[j] = QWEN2_0_5B_DECODE_CANARY_VALUE;
171  }
172  }
173 
174  return 0;
175 }
176 
177 void qwen2_0_5b_decode_model_free(QWEN2_0_5B_DECODEModel *model) {
178  if (!model || !model->base) return;
179 #ifdef __linux__
180  munmap(model->base, model->total_bytes);
181 #else
182  free(model->base);
183 #endif
184  model->base = NULL;
185  model->total_bytes = 0;
186 }
187 
188 int qwen2_0_5b_decode_verify_canaries(QWEN2_0_5B_DECODEModel *model) {
189  int errors = 0;
190  uint32_t *ptr;
191 
192  for (int i = 0; i < QWEN2_0_5B_DECODE_CANARY_COUNT; i++) {
193  ptr = (uint32_t*)((char*)model->base + QWEN2_0_5B_DECODE_CANARIES[i].offset);
194  for (int j = 0; j < 4; j++) {
195  if (ptr[j] != QWEN2_0_5B_DECODE_CANARY_VALUE) {
196  fprintf(stderr, "CANARY CORRUPTION: %s at offset 0x%lX\n",
197  QWEN2_0_5B_DECODE_CANARIES[i].name,
198  QWEN2_0_5B_DECODE_CANARIES[i].offset);
199  errors++;
200  break;
201  }
202  }
203  }
204 
205  return errors;
206 }
207 
208 /* ============================================================================
209  * ALIGNMENT HELPERS
210  * ============================================================================ */
211 
212 static int qwen2_0_5b_decode_align_elems(int elems, int elem_bytes, int align_bytes) {
213  int bytes = elems * elem_bytes;
214  int aligned = (bytes + align_bytes - 1) / align_bytes * align_bytes;
215  return aligned / elem_bytes;
216 }
217 
218 /* ============================================================================
219  * ROPE PRECOMPUTE
220  * ============================================================================ */
221 
222 void qwen2_0_5b_decode_precompute_rope(QWEN2_0_5B_DECODEModel *model) {
223  const int T = QWEN2_0_5B_DECODE_MAX_SEQ_LEN;
224  const int D = QWEN2_0_5B_DECODE_HEAD_DIM / 2;
225  const float theta = 1000000.0f;
226 
227  float *cos_ptr = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_cos_cache);
228  float *sin_ptr = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_sin_cache);
229 
230  for (int pos = 0; pos < T; pos++) {
231  for (int i = 0; i < D; i++) {
232  float freq = 1.0f / powf(theta, (float)(2 * i) / (float)(D * 2));
233  float angle = (float)pos * freq;
234  cos_ptr[pos * D + i] = cosf(angle);
235  sin_ptr[pos * D + i] = sinf(angle);
236  }
237  }
238 }
239 
240 /* ============================================================================
241  * EXPLICIT PER-LAYER PREFILL FUNCTIONS (v6 unrolled)
242  * ============================================================================ */
243 
244 /*
245  * Layer 0: wq=q5_0 wk=q5_0 wv=q8_0 wo=q5_0 w1=q5_0 w2=q6_k
246  */
247 
248 static void qwen2_0_5b_decode_forward_prefill_impl(
249  QWEN2_0_5B_DECODEModel *model,
250  const int *tokens,
251  int num_tokens
252 ) {
253  if (!model || !tokens || num_tokens <= 0) {
254  return;
255  }
256 
257  const int elem_bytes = QWEN2_0_5B_DECODE_DTYPE_BYTES;
258  const int aligned_embed_dim = 896;
259  const int aligned_head_dim = 64;
260  const int aligned_intermediate_dim = 4864;
261  const int aligned_context_window = 131072;
262 
263  float *embed_out = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_HEADER.embedded_input);
264  const void *embed_weight = (const void *)QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_HEADER.token_emb);
265  embedding_forward_q8_0((const int32_t *)tokens,
266  num_tokens,
267  QWEN2_0_5B_DECODE_VOCAB_SIZE,
268  embed_weight,
269  NULL,
270  embed_out,
271  QWEN2_0_5B_DECODE_EMBED_DIM,
272  aligned_embed_dim,
273  num_tokens,
274  0);
275 
276  qwen2_0_5b_decode_layer_0_prefill(
277  model,
278  num_tokens,
279  aligned_embed_dim,
280  aligned_head_dim,
281  aligned_intermediate_dim,
282  aligned_context_window);
283 
284  kv_cache_repack_head_major_inplace(
285  QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[0].k),
286  QWEN2_0_5B_DECODE_NUM_KV_HEADS,
287  num_tokens,
288  aligned_context_window,
289  aligned_head_dim);
290  kv_cache_repack_head_major_inplace(
291  QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[0].v),
292  QWEN2_0_5B_DECODE_NUM_KV_HEADS,
293  num_tokens,
294  aligned_context_window,
295  aligned_head_dim);
296 
297  qwen2_0_5b_decode_layer_1_prefill(
298  model,
299  num_tokens,
300  aligned_embed_dim,
301  aligned_head_dim,
302  aligned_intermediate_dim,
303  aligned_context_window);
304 
305  kv_cache_repack_head_major_inplace(
306  QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[1].k),
307  QWEN2_0_5B_DECODE_NUM_KV_HEADS,
308  num_tokens,
309  aligned_context_window,
310  aligned_head_dim);
311  kv_cache_repack_head_major_inplace(
312  QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[1].v),
313  QWEN2_0_5B_DECODE_NUM_KV_HEADS,
314  num_tokens,
315  aligned_context_window,
316  aligned_head_dim);
317 
318  qwen2_0_5b_decode_layer_2_prefill(
319  model,
320  num_tokens,
321  aligned_embed_dim,
322  aligned_head_dim,
323  aligned_intermediate_dim,
324  aligned_context_window);
325 
326  kv_cache_repack_head_major_inplace(
327  QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[2].k),
328  QWEN2_0_5B_DECODE_NUM_KV_HEADS,
329  num_tokens,
330  aligned_context_window,
331  aligned_head_dim);
332  kv_cache_repack_head_major_inplace(
333  QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[2].v),
334  QWEN2_0_5B_DECODE_NUM_KV_HEADS,
335  num_tokens,
336  aligned_context_window,
337  aligned_head_dim);
338 
339  qwen2_0_5b_decode_layer_3_prefill(
340  model,
341  num_tokens,
342  aligned_embed_dim,
343  aligned_head_dim,
344  aligned_intermediate_dim,
345  aligned_context_window);
346 
347  kv_cache_repack_head_major_inplace(
348  QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[3].k),
349  QWEN2_0_5B_DECODE_NUM_KV_HEADS,
350  num_tokens,
351  aligned_context_window,
352  aligned_head_dim);
353  kv_cache_repack_head_major_inplace(
354  QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[3].v),
355  QWEN2_0_5B_DECODE_NUM_KV_HEADS,
356  num_tokens,
357  aligned_context_window,
358  aligned_head_dim);
359 
360  qwen2_0_5b_decode_layer_4_prefill(
361  model,
362  num_tokens,
363  aligned_embed_dim,
364  aligned_head_dim,
365  aligned_intermediate_dim,
366  aligned_context_window);
367 
368  kv_cache_repack_head_major_inplace(
369  QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[4].k),
370  QWEN2_0_5B_DECODE_NUM_KV_HEADS,
371  num_tokens,
372  aligned_context_window,
373  aligned_head_dim);
374  kv_cache_repack_head_major_inplace(
375  QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[4].v),
376  QWEN2_0_5B_DECODE_NUM_KV_HEADS,
377  num_tokens,
378  aligned_context_window,
379  aligned_head_dim);
380 
381  qwen2_0_5b_decode_layer_5_prefill(
382  model,
383  num_tokens,
384  aligned_embed_dim,
385  aligned_head_dim,
386  aligned_intermediate_dim,
387  aligned_context_window);
388 
389  kv_cache_repack_head_major_inplace(
390  QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[5].k),
391  QWEN2_0_5B_DECODE_NUM_KV_HEADS,
392  num_tokens,
393  aligned_context_window,
394  aligned_head_dim);
395  kv_cache_repack_head_major_inplace(
396  QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[5].v),
397  QWEN2_0_5B_DECODE_NUM_KV_HEADS,
398  num_tokens,
399  aligned_context_window,
400  aligned_head_dim);
401 
402  qwen2_0_5b_decode_layer_6_prefill(
403  model,
404  num_tokens,
405  aligned_embed_dim,
406  aligned_head_dim,
407  aligned_intermediate_dim,
408  aligned_context_window);
409 
410  kv_cache_repack_head_major_inplace(
411  QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[6].k),
412  QWEN2_0_5B_DECODE_NUM_KV_HEADS,
413  num_tokens,
414  aligned_context_window,
415  aligned_head_dim);
416  kv_cache_repack_head_major_inplace(
417  QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[6].v),
418  QWEN2_0_5B_DECODE_NUM_KV_HEADS,
419  num_tokens,
420  aligned_context_window,
421  aligned_head_dim);
422 
423  qwen2_0_5b_decode_layer_7_prefill(
424  model,
425  num_tokens,
426  aligned_embed_dim,
427  aligned_head_dim,
428  aligned_intermediate_dim,
429  aligned_context_window);
430 
431  kv_cache_repack_head_major_inplace(
432  QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[7].k),
433  QWEN2_0_5B_DECODE_NUM_KV_HEADS,
434  num_tokens,
435  aligned_context_window,
436  aligned_head_dim);
437  kv_cache_repack_head_major_inplace(
438  QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[7].v),
439  QWEN2_0_5B_DECODE_NUM_KV_HEADS,
440  num_tokens,
441  aligned_context_window,
442  aligned_head_dim);
443 
444  qwen2_0_5b_decode_layer_8_prefill(
445  model,
446  num_tokens,
447  aligned_embed_dim,
448  aligned_head_dim,
449  aligned_intermediate_dim,
450  aligned_context_window);
451 
452  kv_cache_repack_head_major_inplace(
453  QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[8].k),
454  QWEN2_0_5B_DECODE_NUM_KV_HEADS,
455  num_tokens,
456  aligned_context_window,
457  aligned_head_dim);
458  kv_cache_repack_head_major_inplace(
459  QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[8].v),
460  QWEN2_0_5B_DECODE_NUM_KV_HEADS,
461  num_tokens,
462  aligned_context_window,
463  aligned_head_dim);
464 
465  qwen2_0_5b_decode_layer_9_prefill(
466  model,
467  num_tokens,
468  aligned_embed_dim,
469  aligned_head_dim,
470  aligned_intermediate_dim,
471  aligned_context_window);
472 
473  kv_cache_repack_head_major_inplace(
474  QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[9].k),
475  QWEN2_0_5B_DECODE_NUM_KV_HEADS,
476  num_tokens,
477  aligned_context_window,
478  aligned_head_dim);
479  kv_cache_repack_head_major_inplace(
480  QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[9].v),
481  QWEN2_0_5B_DECODE_NUM_KV_HEADS,
482  num_tokens,
483  aligned_context_window,
484  aligned_head_dim);
485 
486  qwen2_0_5b_decode_layer_10_prefill(
487  model,
488  num_tokens,
489  aligned_embed_dim,
490  aligned_head_dim,
491  aligned_intermediate_dim,
492  aligned_context_window);
493 
494  kv_cache_repack_head_major_inplace(
495  QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[10].k),
496  QWEN2_0_5B_DECODE_NUM_KV_HEADS,
497  num_tokens,
498  aligned_context_window,
499  aligned_head_dim);
500  kv_cache_repack_head_major_inplace(
501  QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[10].v),
502  QWEN2_0_5B_DECODE_NUM_KV_HEADS,
503  num_tokens,
504  aligned_context_window,
505  aligned_head_dim);
506 
507  qwen2_0_5b_decode_layer_11_prefill(
508  model,
509  num_tokens,
510  aligned_embed_dim,
511  aligned_head_dim,
512  aligned_intermediate_dim,
513  aligned_context_window);
514 
515  kv_cache_repack_head_major_inplace(
516  QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[11].k),
517  QWEN2_0_5B_DECODE_NUM_KV_HEADS,
518  num_tokens,
519  aligned_context_window,
520  aligned_head_dim);
521  kv_cache_repack_head_major_inplace(
522  QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[11].v),
523  QWEN2_0_5B_DECODE_NUM_KV_HEADS,
524  num_tokens,
525  aligned_context_window,
526  aligned_head_dim);
527 
528  qwen2_0_5b_decode_layer_12_prefill(
529  model,
530  num_tokens,
531  aligned_embed_dim,
532  aligned_head_dim,
533  aligned_intermediate_dim,
534  aligned_context_window);
535 
536  kv_cache_repack_head_major_inplace(
537  QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[12].k),
538  QWEN2_0_5B_DECODE_NUM_KV_HEADS,
539  num_tokens,
540  aligned_context_window,
541  aligned_head_dim);
542  kv_cache_repack_head_major_inplace(
543  QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[12].v),
544  QWEN2_0_5B_DECODE_NUM_KV_HEADS,
545  num_tokens,
546  aligned_context_window,
547  aligned_head_dim);
548 
549  qwen2_0_5b_decode_layer_13_prefill(
550  model,
551  num_tokens,
552  aligned_embed_dim,
553  aligned_head_dim,
554  aligned_intermediate_dim,
555  aligned_context_window);
556 
557  kv_cache_repack_head_major_inplace(
558  QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[13].k),
559  QWEN2_0_5B_DECODE_NUM_KV_HEADS,
560  num_tokens,
561  aligned_context_window,
562  aligned_head_dim);
563  kv_cache_repack_head_major_inplace(
564  QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[13].v),
565  QWEN2_0_5B_DECODE_NUM_KV_HEADS,
566  num_tokens,
567  aligned_context_window,
568  aligned_head_dim);
569 
570  qwen2_0_5b_decode_layer_14_prefill(
571  model,
572  num_tokens,
573  aligned_embed_dim,
574  aligned_head_dim,
575  aligned_intermediate_dim,
576  aligned_context_window);
577 
578  kv_cache_repack_head_major_inplace(
579  QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[14].k),
580  QWEN2_0_5B_DECODE_NUM_KV_HEADS,
581  num_tokens,
582  aligned_context_window,
583  aligned_head_dim);
584  kv_cache_repack_head_major_inplace(
585  QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[14].v),
586  QWEN2_0_5B_DECODE_NUM_KV_HEADS,
587  num_tokens,
588  aligned_context_window,
589  aligned_head_dim);
590 
591  qwen2_0_5b_decode_layer_15_prefill(
592  model,
593  num_tokens,
594  aligned_embed_dim,
595  aligned_head_dim,
596  aligned_intermediate_dim,
597  aligned_context_window);
598 
599  kv_cache_repack_head_major_inplace(
600  QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[15].k),
601  QWEN2_0_5B_DECODE_NUM_KV_HEADS,
602  num_tokens,
603  aligned_context_window,
604  aligned_head_dim);
605  kv_cache_repack_head_major_inplace(
606  QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[15].v),
607  QWEN2_0_5B_DECODE_NUM_KV_HEADS,
608  num_tokens,
609  aligned_context_window,
610  aligned_head_dim);
611 
612  qwen2_0_5b_decode_layer_16_prefill(
613  model,
614  num_tokens,
615  aligned_embed_dim,
616  aligned_head_dim,
617  aligned_intermediate_dim,
618  aligned_context_window);
619 
620  kv_cache_repack_head_major_inplace(
621  QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[16].k),
622  QWEN2_0_5B_DECODE_NUM_KV_HEADS,
623  num_tokens,
624  aligned_context_window,
625  aligned_head_dim);
626  kv_cache_repack_head_major_inplace(
627  QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[16].v),
628  QWEN2_0_5B_DECODE_NUM_KV_HEADS,
629  num_tokens,
630  aligned_context_window,
631  aligned_head_dim);
632 
633  qwen2_0_5b_decode_layer_17_prefill(
634  model,
635  num_tokens,
636  aligned_embed_dim,
637  aligned_head_dim,
638  aligned_intermediate_dim,
639  aligned_context_window);
640 
641  kv_cache_repack_head_major_inplace(
642  QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[17].k),
643  QWEN2_0_5B_DECODE_NUM_KV_HEADS,
644  num_tokens,
645  aligned_context_window,
646  aligned_head_dim);
647  kv_cache_repack_head_major_inplace(
648  QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[17].v),
649  QWEN2_0_5B_DECODE_NUM_KV_HEADS,
650  num_tokens,
651  aligned_context_window,
652  aligned_head_dim);
653 
654  qwen2_0_5b_decode_layer_18_prefill(
655  model,
656  num_tokens,
657  aligned_embed_dim,
658  aligned_head_dim,
659  aligned_intermediate_dim,
660  aligned_context_window);
661 
662  kv_cache_repack_head_major_inplace(
663  QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[18].k),
664  QWEN2_0_5B_DECODE_NUM_KV_HEADS,
665  num_tokens,
666  aligned_context_window,
667  aligned_head_dim);
668  kv_cache_repack_head_major_inplace(
669  QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[18].v),
670  QWEN2_0_5B_DECODE_NUM_KV_HEADS,
671  num_tokens,
672  aligned_context_window,
673  aligned_head_dim);
674 
675  qwen2_0_5b_decode_layer_19_prefill(
676  model,
677  num_tokens,
678  aligned_embed_dim,
679  aligned_head_dim,
680  aligned_intermediate_dim,
681  aligned_context_window);
682 
683  kv_cache_repack_head_major_inplace(
684  QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[19].k),
685  QWEN2_0_5B_DECODE_NUM_KV_HEADS,
686  num_tokens,
687  aligned_context_window,
688  aligned_head_dim);
689  kv_cache_repack_head_major_inplace(
690  QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[19].v),
691  QWEN2_0_5B_DECODE_NUM_KV_HEADS,
692  num_tokens,
693  aligned_context_window,
694  aligned_head_dim);
695 
696  qwen2_0_5b_decode_layer_20_prefill(
697  model,
698  num_tokens,
699  aligned_embed_dim,
700  aligned_head_dim,
701  aligned_intermediate_dim,
702  aligned_context_window);
703 
704  kv_cache_repack_head_major_inplace(
705  QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[20].k),
706  QWEN2_0_5B_DECODE_NUM_KV_HEADS,
707  num_tokens,
708  aligned_context_window,
709  aligned_head_dim);
710  kv_cache_repack_head_major_inplace(
711  QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[20].v),
712  QWEN2_0_5B_DECODE_NUM_KV_HEADS,
713  num_tokens,
714  aligned_context_window,
715  aligned_head_dim);
716 
717  qwen2_0_5b_decode_layer_21_prefill(
718  model,
719  num_tokens,
720  aligned_embed_dim,
721  aligned_head_dim,
722  aligned_intermediate_dim,
723  aligned_context_window);
724 
725  kv_cache_repack_head_major_inplace(
726  QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[21].k),
727  QWEN2_0_5B_DECODE_NUM_KV_HEADS,
728  num_tokens,
729  aligned_context_window,
730  aligned_head_dim);
731  kv_cache_repack_head_major_inplace(
732  QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[21].v),
733  QWEN2_0_5B_DECODE_NUM_KV_HEADS,
734  num_tokens,
735  aligned_context_window,
736  aligned_head_dim);
737 
738  qwen2_0_5b_decode_layer_22_prefill(
739  model,
740  num_tokens,
741  aligned_embed_dim,
742  aligned_head_dim,
743  aligned_intermediate_dim,
744  aligned_context_window);
745 
746  kv_cache_repack_head_major_inplace(
747  QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[22].k),
748  QWEN2_0_5B_DECODE_NUM_KV_HEADS,
749  num_tokens,
750  aligned_context_window,
751  aligned_head_dim);
752  kv_cache_repack_head_major_inplace(
753  QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[22].v),
754  QWEN2_0_5B_DECODE_NUM_KV_HEADS,
755  num_tokens,
756  aligned_context_window,
757  aligned_head_dim);
758 
759  qwen2_0_5b_decode_layer_23_prefill(
760  model,
761  num_tokens,
762  aligned_embed_dim,
763  aligned_head_dim,
764  aligned_intermediate_dim,
765  aligned_context_window);
766 
767  kv_cache_repack_head_major_inplace(
768  QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[23].k),
769  QWEN2_0_5B_DECODE_NUM_KV_HEADS,
770  num_tokens,
771  aligned_context_window,
772  aligned_head_dim);
773  kv_cache_repack_head_major_inplace(
774  QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[23].v),
775  QWEN2_0_5B_DECODE_NUM_KV_HEADS,
776  num_tokens,
777  aligned_context_window,
778  aligned_head_dim);
779 
780  float *last_hidden = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[QWEN2_0_5B_DECODE_NUM_LAYERS - 1].output);
781  float *final_ln_weight = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_FOOTER.final_ln_weight);
782  float *final_out = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_FOOTER.final_output);
783  rmsnorm_forward(last_hidden,
784  final_ln_weight,
785  final_out,
786  NULL,
787  num_tokens,
788  QWEN2_0_5B_DECODE_EMBED_DIM,
789  aligned_embed_dim,
790  1e-06f);
791 
792  float *logits = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_FOOTER.logits);
793  const void *lm_head = (const void *)QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_FOOTER.lm_head_weight);
794  for (int t = 0; t < num_tokens; ++t) {
795  const float *row = final_out + (size_t)t * (size_t)aligned_embed_dim;
796  float *logits_row = logits + (size_t)t * (size_t)QWEN2_0_5B_DECODE_VOCAB_SIZE;
797  gemm_nt_q8_0(row,
798  lm_head,
799  NULL,
800  logits_row,
801  1,
802  QWEN2_0_5B_DECODE_VOCAB_SIZE,
803  aligned_embed_dim);
804  }
805 }
806 
807 /* ============================================================================
808  * EXPLICIT PER-LAYER DECODE FUNCTIONS (v6 unrolled)
809  * ============================================================================ */
810 
811 /*
812  * Layer 0: wq=q5_0 wk=q5_0 wv=q8_0 wo=q5_0 w1=q5_0 w2=q6_k
813  */
814 static void qwen2_0_5b_decode_layer_0_decode(
815  QWEN2_0_5B_DECODEModel *model,
816  int token_index,
817  int aligned_embed_dim,
818  int aligned_head_dim,
819  int aligned_intermediate_dim,
820  int aligned_context_window
821 ) {
822  const QWEN2_0_5B_DECODELayerOffsets *L = &QWEN2_0_5B_DECODE_LAYERS[0];
823 
824  float *input = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_HEADER.embedded_input);
825 
826  float *ln1_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln1_gamma);
827  float *ln1_out = QWEN2_0_5B_DECODE_PTR(model, L->ln1_out);
828  float *ln2_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln2_gamma);
829  float *ln2_out = QWEN2_0_5B_DECODE_PTR(model, L->ln2_out);
830  float *k_cache = QWEN2_0_5B_DECODE_PTR(model, L->k);
831  float *v_cache = QWEN2_0_5B_DECODE_PTR(model, L->v);
832  float *proj_tmp = QWEN2_0_5B_DECODE_PTR(model, L->proj_tmp);
833  float *proj_scratch = QWEN2_0_5B_DECODE_PTR(model, L->proj_scratch);
834  float *residual1 = QWEN2_0_5B_DECODE_PTR(model, L->residual1);
835  float *mlp_out = QWEN2_0_5B_DECODE_PTR(model, L->mlp_out);
836  float *output = QWEN2_0_5B_DECODE_PTR(model, L->output);
837 
838  /* Weights (explicit types for layer 0) */
839  const void *WQ = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wq); /* Q5_0 */
840  const void *WK = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wk); /* Q5_0 */
841  const void *WV = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wv); /* Q8_0 */
842  const void *WO = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wo); /* Q5_0 */
843  const void *W1 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w1); /* Q5_0 (gate+up) */
844  const void *W2 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w2); /* Q6_K (down) */
845 
846  /* Attention biases (Qwen2-style) */
847  const float *BQ = (const float *)QWEN2_0_5B_DECODE_PTR(model, L->bq);
848  const float *BK = (const float *)QWEN2_0_5B_DECODE_PTR(model, L->bk);
849  const float *BV = (const float *)QWEN2_0_5B_DECODE_PTR(model, L->bv);
850 
851  float *rope_cos = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_cos_cache);
852  float *rope_sin = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_sin_cache);
853 
854  const int H = QWEN2_0_5B_DECODE_NUM_HEADS;
855  const int H_kv = QWEN2_0_5B_DECODE_NUM_KV_HEADS;
856  const int head_dim = QWEN2_0_5B_DECODE_HEAD_DIM;
857 
858  float q_token[H * aligned_head_dim];
859  float k_token[H_kv * aligned_head_dim];
860  float v_token[H_kv * aligned_head_dim];
861  float attn_token[H * aligned_head_dim];
862 
863  /* Local MLP buffers (avoid layout dependencies for intermediate values) */
864  float fc1_out[2 * aligned_intermediate_dim];
865  float swiglu_out[aligned_intermediate_dim];
866 
867  /* Step 1: RMSNorm before attention */
868  rmsnorm_forward(input,
869  ln1_gamma,
870  ln1_out,
871  NULL,
872  1,
873  QWEN2_0_5B_DECODE_EMBED_DIM,
874  aligned_embed_dim,
875  1e-06f);
876 
877  /* Step 2: QKV projection */
878  /* Q projection: Q5_0 -> gemm_nt_q5_0 */
879  gemm_nt_q5_0(ln1_out, WQ, BQ, q_token, 1, H * head_dim, aligned_embed_dim);
880 
881  /* K projection: Q5_0 -> gemm_nt_q5_0 */
882  gemm_nt_q5_0(ln1_out, WK, BK, k_token, 1, H_kv * head_dim, aligned_embed_dim);
883 
884  /* V projection: Q8_0 -> gemm_nt_q8_0 */
885  gemm_nt_q8_0(ln1_out, WV, BV, v_token, 1, H_kv * head_dim, aligned_embed_dim);
886 
887  /* Step 3: RoPE */
888  rope_forward_qk(q_token,
889  k_token,
890  rope_cos,
891  rope_sin,
892  H,
893  H_kv,
894  1,
895  head_dim,
896  aligned_head_dim,
897  token_index);
898 
899  /* Step 4: KV cache write */
900  kv_cache_write_head_major(k_token,
901  v_token,
902  k_cache,
903  v_cache,
904  H_kv,
905  token_index,
906  aligned_context_window,
907  head_dim,
908  aligned_head_dim);
909 
910  /* Step 5: Attention (decode) */
911  attention_forward_decode_head_major_gqa_regular(q_token,
912  k_cache,
913  v_cache,
914  attn_token,
915  H,
916  H_kv,
917  token_index + 1,
918  aligned_context_window,
919  head_dim,
920  aligned_head_dim);
921 
922  /* Step 6: Output projection */
923  /* WO projection: Q5_0 -> gemm_nt_q5_0 */
924  gemm_nt_q5_0(attn_token, WO, NULL, proj_tmp, 1, aligned_embed_dim, H * head_dim);
925 
926  /* Step 7: Residual add */
927  qwen2_0_5b_decode_residual_add_token_major(input, proj_tmp, residual1, 1, aligned_embed_dim);
928 
929  /* Step 8: RMSNorm before MLP */
930  rmsnorm_forward(residual1,
931  ln2_gamma,
932  ln2_out,
933  NULL,
934  1,
935  QWEN2_0_5B_DECODE_EMBED_DIM,
936  aligned_embed_dim,
937  1e-06f);
938 
939  /* Step 9: MLP (SwiGLU) */
940  /* Gate+Up projection: Q5_0 -> gemm_nt_q5_0 */
941  gemm_nt_q5_0(ln2_out, W1, NULL, fc1_out, 1, 2 * aligned_intermediate_dim, aligned_embed_dim);
942 
943  /* SwiGLU activation */
944  swiglu_forward(fc1_out, swiglu_out, 1, aligned_intermediate_dim);
945 
946  /* Down projection: Q6_K -> gemm_nt_q6_k */
947  gemm_nt_q6_k(swiglu_out, W2, NULL, mlp_out, 1, aligned_embed_dim, aligned_intermediate_dim);
948 
949  /* Step 10: Final residual add */
950  qwen2_0_5b_decode_residual_add_token_major(residual1, mlp_out, output, 1, aligned_embed_dim);
951 }
952 
953 /*
954  * Layer 1: wq=q5_0 wk=q5_0 wv=q8_0 wo=q5_0 w1=q5_0 w2=q6_k
955  */
956 static void qwen2_0_5b_decode_layer_1_decode(
957  QWEN2_0_5B_DECODEModel *model,
958  int token_index,
959  int aligned_embed_dim,
960  int aligned_head_dim,
961  int aligned_intermediate_dim,
962  int aligned_context_window
963 ) {
964  const QWEN2_0_5B_DECODELayerOffsets *L = &QWEN2_0_5B_DECODE_LAYERS[1];
965 
966  float *input = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[0].output);
967 
968  float *ln1_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln1_gamma);
969  float *ln1_out = QWEN2_0_5B_DECODE_PTR(model, L->ln1_out);
970  float *ln2_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln2_gamma);
971  float *ln2_out = QWEN2_0_5B_DECODE_PTR(model, L->ln2_out);
972  float *k_cache = QWEN2_0_5B_DECODE_PTR(model, L->k);
973  float *v_cache = QWEN2_0_5B_DECODE_PTR(model, L->v);
974  float *proj_tmp = QWEN2_0_5B_DECODE_PTR(model, L->proj_tmp);
975  float *proj_scratch = QWEN2_0_5B_DECODE_PTR(model, L->proj_scratch);
976  float *residual1 = QWEN2_0_5B_DECODE_PTR(model, L->residual1);
977  float *mlp_out = QWEN2_0_5B_DECODE_PTR(model, L->mlp_out);
978  float *output = QWEN2_0_5B_DECODE_PTR(model, L->output);
979 
980  /* Weights (explicit types for layer 1) */
981  const void *WQ = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wq); /* Q5_0 */
982  const void *WK = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wk); /* Q5_0 */
983  const void *WV = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wv); /* Q8_0 */
984  const void *WO = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wo); /* Q5_0 */
985  const void *W1 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w1); /* Q5_0 (gate+up) */
986  const void *W2 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w2); /* Q6_K (down) */
987 
988  /* Attention biases (Qwen2-style) */
989  const float *BQ = (const float *)QWEN2_0_5B_DECODE_PTR(model, L->bq);
990  const float *BK = (const float *)QWEN2_0_5B_DECODE_PTR(model, L->bk);
991  const float *BV = (const float *)QWEN2_0_5B_DECODE_PTR(model, L->bv);
992 
993  float *rope_cos = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_cos_cache);
994  float *rope_sin = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_sin_cache);
995 
996  const int H = QWEN2_0_5B_DECODE_NUM_HEADS;
997  const int H_kv = QWEN2_0_5B_DECODE_NUM_KV_HEADS;
998  const int head_dim = QWEN2_0_5B_DECODE_HEAD_DIM;
999 
1000  float q_token[H * aligned_head_dim];
1001  float k_token[H_kv * aligned_head_dim];
1002  float v_token[H_kv * aligned_head_dim];
1003  float attn_token[H * aligned_head_dim];
1004 
1005  /* Local MLP buffers (avoid layout dependencies for intermediate values) */
1006  float fc1_out[2 * aligned_intermediate_dim];
1007  float swiglu_out[aligned_intermediate_dim];
1008 
1009  /* Step 1: RMSNorm before attention */
1010  rmsnorm_forward(input,
1011  ln1_gamma,
1012  ln1_out,
1013  NULL,
1014  1,
1015  QWEN2_0_5B_DECODE_EMBED_DIM,
1016  aligned_embed_dim,
1017  1e-06f);
1018 
1019  /* Step 2: QKV projection */
1020  /* Q projection: Q5_0 -> gemm_nt_q5_0 */
1021  gemm_nt_q5_0(ln1_out, WQ, BQ, q_token, 1, H * head_dim, aligned_embed_dim);
1022 
1023  /* K projection: Q5_0 -> gemm_nt_q5_0 */
1024  gemm_nt_q5_0(ln1_out, WK, BK, k_token, 1, H_kv * head_dim, aligned_embed_dim);
1025 
1026  /* V projection: Q8_0 -> gemm_nt_q8_0 */
1027  gemm_nt_q8_0(ln1_out, WV, BV, v_token, 1, H_kv * head_dim, aligned_embed_dim);
1028 
1029  /* Step 3: RoPE */
1030  rope_forward_qk(q_token,
1031  k_token,
1032  rope_cos,
1033  rope_sin,
1034  H,
1035  H_kv,
1036  1,
1037  head_dim,
1038  aligned_head_dim,
1039  token_index);
1040 
1041  /* Step 4: KV cache write */
1042  kv_cache_write_head_major(k_token,
1043  v_token,
1044  k_cache,
1045  v_cache,
1046  H_kv,
1047  token_index,
1048  aligned_context_window,
1049  head_dim,
1050  aligned_head_dim);
1051 
1052  /* Step 5: Attention (decode) */
1053  attention_forward_decode_head_major_gqa_regular(q_token,
1054  k_cache,
1055  v_cache,
1056  attn_token,
1057  H,
1058  H_kv,
1059  token_index + 1,
1060  aligned_context_window,
1061  head_dim,
1062  aligned_head_dim);
1063 
1064  /* Step 6: Output projection */
1065  /* WO projection: Q5_0 -> gemm_nt_q5_0 */
1066  gemm_nt_q5_0(attn_token, WO, NULL, proj_tmp, 1, aligned_embed_dim, H * head_dim);
1067 
1068  /* Step 7: Residual add */
1069  qwen2_0_5b_decode_residual_add_token_major(input, proj_tmp, residual1, 1, aligned_embed_dim);
1070 
1071  /* Step 8: RMSNorm before MLP */
1072  rmsnorm_forward(residual1,
1073  ln2_gamma,
1074  ln2_out,
1075  NULL,
1076  1,
1077  QWEN2_0_5B_DECODE_EMBED_DIM,
1078  aligned_embed_dim,
1079  1e-06f);
1080 
1081  /* Step 9: MLP (SwiGLU) */
1082  /* Gate+Up projection: Q5_0 -> gemm_nt_q5_0 */
1083  gemm_nt_q5_0(ln2_out, W1, NULL, fc1_out, 1, 2 * aligned_intermediate_dim, aligned_embed_dim);
1084 
1085  /* SwiGLU activation */
1086  swiglu_forward(fc1_out, swiglu_out, 1, aligned_intermediate_dim);
1087 
1088  /* Down projection: Q6_K -> gemm_nt_q6_k */
1089  gemm_nt_q6_k(swiglu_out, W2, NULL, mlp_out, 1, aligned_embed_dim, aligned_intermediate_dim);
1090 
1091  /* Step 10: Final residual add */
1092  qwen2_0_5b_decode_residual_add_token_major(residual1, mlp_out, output, 1, aligned_embed_dim);
1093 }
1094 
1095 /*
1096  * Layer 2: wq=q5_0 wk=q5_0 wv=q5_0 wo=q5_0 w1=q5_0 w2=q4_k
1097  */
1098 static void qwen2_0_5b_decode_layer_2_decode(
1099  QWEN2_0_5B_DECODEModel *model,
1100  int token_index,
1101  int aligned_embed_dim,
1102  int aligned_head_dim,
1103  int aligned_intermediate_dim,
1104  int aligned_context_window
1105 ) {
1106  const QWEN2_0_5B_DECODELayerOffsets *L = &QWEN2_0_5B_DECODE_LAYERS[2];
1107 
1108  float *input = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[1].output);
1109 
1110  float *ln1_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln1_gamma);
1111  float *ln1_out = QWEN2_0_5B_DECODE_PTR(model, L->ln1_out);
1112  float *ln2_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln2_gamma);
1113  float *ln2_out = QWEN2_0_5B_DECODE_PTR(model, L->ln2_out);
1114  float *k_cache = QWEN2_0_5B_DECODE_PTR(model, L->k);
1115  float *v_cache = QWEN2_0_5B_DECODE_PTR(model, L->v);
1116  float *proj_tmp = QWEN2_0_5B_DECODE_PTR(model, L->proj_tmp);
1117  float *proj_scratch = QWEN2_0_5B_DECODE_PTR(model, L->proj_scratch);
1118  float *residual1 = QWEN2_0_5B_DECODE_PTR(model, L->residual1);
1119  float *mlp_out = QWEN2_0_5B_DECODE_PTR(model, L->mlp_out);
1120  float *output = QWEN2_0_5B_DECODE_PTR(model, L->output);
1121 
1122  /* Weights (explicit types for layer 2) */
1123  const void *WQ = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wq); /* Q5_0 */
1124  const void *WK = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wk); /* Q5_0 */
1125  const void *WV = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wv); /* Q5_0 */
1126  const void *WO = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wo); /* Q5_0 */
1127  const void *W1 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w1); /* Q5_0 (gate+up) */
1128  const void *W2 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w2); /* Q4_K (down) */
1129 
1130  /* Attention biases (Qwen2-style) */
1131  const float *BQ = (const float *)QWEN2_0_5B_DECODE_PTR(model, L->bq);
1132  const float *BK = (const float *)QWEN2_0_5B_DECODE_PTR(model, L->bk);
1133  const float *BV = (const float *)QWEN2_0_5B_DECODE_PTR(model, L->bv);
1134 
1135  float *rope_cos = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_cos_cache);
1136  float *rope_sin = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_sin_cache);
1137 
1138  const int H = QWEN2_0_5B_DECODE_NUM_HEADS;
1139  const int H_kv = QWEN2_0_5B_DECODE_NUM_KV_HEADS;
1140  const int head_dim = QWEN2_0_5B_DECODE_HEAD_DIM;
1141 
1142  float q_token[H * aligned_head_dim];
1143  float k_token[H_kv * aligned_head_dim];
1144  float v_token[H_kv * aligned_head_dim];
1145  float attn_token[H * aligned_head_dim];
1146 
1147  /* Local MLP buffers (avoid layout dependencies for intermediate values) */
1148  float fc1_out[2 * aligned_intermediate_dim];
1149  float swiglu_out[aligned_intermediate_dim];
1150 
1151  /* Step 1: RMSNorm before attention */
1152  rmsnorm_forward(input,
1153  ln1_gamma,
1154  ln1_out,
1155  NULL,
1156  1,
1157  QWEN2_0_5B_DECODE_EMBED_DIM,
1158  aligned_embed_dim,
1159  1e-06f);
1160 
1161  /* Step 2: QKV projection */
1162  /* Q projection: Q5_0 -> gemm_nt_q5_0 */
1163  gemm_nt_q5_0(ln1_out, WQ, BQ, q_token, 1, H * head_dim, aligned_embed_dim);
1164 
1165  /* K projection: Q5_0 -> gemm_nt_q5_0 */
1166  gemm_nt_q5_0(ln1_out, WK, BK, k_token, 1, H_kv * head_dim, aligned_embed_dim);
1167 
1168  /* V projection: Q5_0 -> gemm_nt_q5_0 */
1169  gemm_nt_q5_0(ln1_out, WV, BV, v_token, 1, H_kv * head_dim, aligned_embed_dim);
1170 
1171  /* Step 3: RoPE */
1172  rope_forward_qk(q_token,
1173  k_token,
1174  rope_cos,
1175  rope_sin,
1176  H,
1177  H_kv,
1178  1,
1179  head_dim,
1180  aligned_head_dim,
1181  token_index);
1182 
1183  /* Step 4: KV cache write */
1184  kv_cache_write_head_major(k_token,
1185  v_token,
1186  k_cache,
1187  v_cache,
1188  H_kv,
1189  token_index,
1190  aligned_context_window,
1191  head_dim,
1192  aligned_head_dim);
1193 
1194  /* Step 5: Attention (decode) */
1195  attention_forward_decode_head_major_gqa_regular(q_token,
1196  k_cache,
1197  v_cache,
1198  attn_token,
1199  H,
1200  H_kv,
1201  token_index + 1,
1202  aligned_context_window,
1203  head_dim,
1204  aligned_head_dim);
1205 
1206  /* Step 6: Output projection */
1207  /* WO projection: Q5_0 -> gemm_nt_q5_0 */
1208  gemm_nt_q5_0(attn_token, WO, NULL, proj_tmp, 1, aligned_embed_dim, H * head_dim);
1209 
1210  /* Step 7: Residual add */
1211  qwen2_0_5b_decode_residual_add_token_major(input, proj_tmp, residual1, 1, aligned_embed_dim);
1212 
1213  /* Step 8: RMSNorm before MLP */
1214  rmsnorm_forward(residual1,
1215  ln2_gamma,
1216  ln2_out,
1217  NULL,
1218  1,
1219  QWEN2_0_5B_DECODE_EMBED_DIM,
1220  aligned_embed_dim,
1221  1e-06f);
1222 
1223  /* Step 9: MLP (SwiGLU) */
1224  /* Gate+Up projection: Q5_0 -> gemm_nt_q5_0 */
1225  gemm_nt_q5_0(ln2_out, W1, NULL, fc1_out, 1, 2 * aligned_intermediate_dim, aligned_embed_dim);
1226 
1227  /* SwiGLU activation */
1228  swiglu_forward(fc1_out, swiglu_out, 1, aligned_intermediate_dim);
1229 
1230  /* Down projection: Q4_K -> gemm_nt_q4_k */
1231  gemm_nt_q4_k(swiglu_out, W2, NULL, mlp_out, 1, aligned_embed_dim, aligned_intermediate_dim);
1232 
1233  /* Step 10: Final residual add */
1234  qwen2_0_5b_decode_residual_add_token_major(residual1, mlp_out, output, 1, aligned_embed_dim);
1235 }
1236 
1237 /*
1238  * Layer 3: wq=q5_0 wk=q5_0 wv=q8_0 wo=q5_0 w1=q5_0 w2=q6_k
1239  */
1240 static void qwen2_0_5b_decode_layer_3_decode(
1241  QWEN2_0_5B_DECODEModel *model,
1242  int token_index,
1243  int aligned_embed_dim,
1244  int aligned_head_dim,
1245  int aligned_intermediate_dim,
1246  int aligned_context_window
1247 ) {
1248  const QWEN2_0_5B_DECODELayerOffsets *L = &QWEN2_0_5B_DECODE_LAYERS[3];
1249 
1250  float *input = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[2].output);
1251 
1252  float *ln1_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln1_gamma);
1253  float *ln1_out = QWEN2_0_5B_DECODE_PTR(model, L->ln1_out);
1254  float *ln2_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln2_gamma);
1255  float *ln2_out = QWEN2_0_5B_DECODE_PTR(model, L->ln2_out);
1256  float *k_cache = QWEN2_0_5B_DECODE_PTR(model, L->k);
1257  float *v_cache = QWEN2_0_5B_DECODE_PTR(model, L->v);
1258  float *proj_tmp = QWEN2_0_5B_DECODE_PTR(model, L->proj_tmp);
1259  float *proj_scratch = QWEN2_0_5B_DECODE_PTR(model, L->proj_scratch);
1260  float *residual1 = QWEN2_0_5B_DECODE_PTR(model, L->residual1);
1261  float *mlp_out = QWEN2_0_5B_DECODE_PTR(model, L->mlp_out);
1262  float *output = QWEN2_0_5B_DECODE_PTR(model, L->output);
1263 
1264  /* Weights (explicit types for layer 3) */
1265  const void *WQ = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wq); /* Q5_0 */
1266  const void *WK = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wk); /* Q5_0 */
1267  const void *WV = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wv); /* Q8_0 */
1268  const void *WO = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wo); /* Q5_0 */
1269  const void *W1 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w1); /* Q5_0 (gate+up) */
1270  const void *W2 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w2); /* Q6_K (down) */
1271 
1272  /* Attention biases (Qwen2-style) */
1273  const float *BQ = (const float *)QWEN2_0_5B_DECODE_PTR(model, L->bq);
1274  const float *BK = (const float *)QWEN2_0_5B_DECODE_PTR(model, L->bk);
1275  const float *BV = (const float *)QWEN2_0_5B_DECODE_PTR(model, L->bv);
1276 
1277  float *rope_cos = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_cos_cache);
1278  float *rope_sin = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_sin_cache);
1279 
1280  const int H = QWEN2_0_5B_DECODE_NUM_HEADS;
1281  const int H_kv = QWEN2_0_5B_DECODE_NUM_KV_HEADS;
1282  const int head_dim = QWEN2_0_5B_DECODE_HEAD_DIM;
1283 
1284  float q_token[H * aligned_head_dim];
1285  float k_token[H_kv * aligned_head_dim];
1286  float v_token[H_kv * aligned_head_dim];
1287  float attn_token[H * aligned_head_dim];
1288 
1289  /* Local MLP buffers (avoid layout dependencies for intermediate values) */
1290  float fc1_out[2 * aligned_intermediate_dim];
1291  float swiglu_out[aligned_intermediate_dim];
1292 
1293  /* Step 1: RMSNorm before attention */
1294  rmsnorm_forward(input,
1295  ln1_gamma,
1296  ln1_out,
1297  NULL,
1298  1,
1299  QWEN2_0_5B_DECODE_EMBED_DIM,
1300  aligned_embed_dim,
1301  1e-06f);
1302 
1303  /* Step 2: QKV projection */
1304  /* Q projection: Q5_0 -> gemm_nt_q5_0 */
1305  gemm_nt_q5_0(ln1_out, WQ, BQ, q_token, 1, H * head_dim, aligned_embed_dim);
1306 
1307  /* K projection: Q5_0 -> gemm_nt_q5_0 */
1308  gemm_nt_q5_0(ln1_out, WK, BK, k_token, 1, H_kv * head_dim, aligned_embed_dim);
1309 
1310  /* V projection: Q8_0 -> gemm_nt_q8_0 */
1311  gemm_nt_q8_0(ln1_out, WV, BV, v_token, 1, H_kv * head_dim, aligned_embed_dim);
1312 
1313  /* Step 3: RoPE */
1314  rope_forward_qk(q_token,
1315  k_token,
1316  rope_cos,
1317  rope_sin,
1318  H,
1319  H_kv,
1320  1,
1321  head_dim,
1322  aligned_head_dim,
1323  token_index);
1324 
1325  /* Step 4: KV cache write */
1326  kv_cache_write_head_major(k_token,
1327  v_token,
1328  k_cache,
1329  v_cache,
1330  H_kv,
1331  token_index,
1332  aligned_context_window,
1333  head_dim,
1334  aligned_head_dim);
1335 
1336  /* Step 5: Attention (decode) */
1337  attention_forward_decode_head_major_gqa_regular(q_token,
1338  k_cache,
1339  v_cache,
1340  attn_token,
1341  H,
1342  H_kv,
1343  token_index + 1,
1344  aligned_context_window,
1345  head_dim,
1346  aligned_head_dim);
1347 
1348  /* Step 6: Output projection */
1349  /* WO projection: Q5_0 -> gemm_nt_q5_0 */
1350  gemm_nt_q5_0(attn_token, WO, NULL, proj_tmp, 1, aligned_embed_dim, H * head_dim);
1351 
1352  /* Step 7: Residual add */
1353  qwen2_0_5b_decode_residual_add_token_major(input, proj_tmp, residual1, 1, aligned_embed_dim);
1354 
1355  /* Step 8: RMSNorm before MLP */
1356  rmsnorm_forward(residual1,
1357  ln2_gamma,
1358  ln2_out,
1359  NULL,
1360  1,
1361  QWEN2_0_5B_DECODE_EMBED_DIM,
1362  aligned_embed_dim,
1363  1e-06f);
1364 
1365  /* Step 9: MLP (SwiGLU) */
1366  /* Gate+Up projection: Q5_0 -> gemm_nt_q5_0 */
1367  gemm_nt_q5_0(ln2_out, W1, NULL, fc1_out, 1, 2 * aligned_intermediate_dim, aligned_embed_dim);
1368 
1369  /* SwiGLU activation */
1370  swiglu_forward(fc1_out, swiglu_out, 1, aligned_intermediate_dim);
1371 
1372  /* Down projection: Q6_K -> gemm_nt_q6_k */
1373  gemm_nt_q6_k(swiglu_out, W2, NULL, mlp_out, 1, aligned_embed_dim, aligned_intermediate_dim);
1374 
1375  /* Step 10: Final residual add */
1376  qwen2_0_5b_decode_residual_add_token_major(residual1, mlp_out, output, 1, aligned_embed_dim);
1377 }
1378 
1379 /*
1380  * Layer 4: wq=q5_0 wk=q5_0 wv=q5_0 wo=q5_0 w1=q5_0 w2=q4_k
1381  */
1382 static void qwen2_0_5b_decode_layer_4_decode(
1383  QWEN2_0_5B_DECODEModel *model,
1384  int token_index,
1385  int aligned_embed_dim,
1386  int aligned_head_dim,
1387  int aligned_intermediate_dim,
1388  int aligned_context_window
1389 ) {
1390  const QWEN2_0_5B_DECODELayerOffsets *L = &QWEN2_0_5B_DECODE_LAYERS[4];
1391 
1392  float *input = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[3].output);
1393 
1394  float *ln1_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln1_gamma);
1395  float *ln1_out = QWEN2_0_5B_DECODE_PTR(model, L->ln1_out);
1396  float *ln2_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln2_gamma);
1397  float *ln2_out = QWEN2_0_5B_DECODE_PTR(model, L->ln2_out);
1398  float *k_cache = QWEN2_0_5B_DECODE_PTR(model, L->k);
1399  float *v_cache = QWEN2_0_5B_DECODE_PTR(model, L->v);
1400  float *proj_tmp = QWEN2_0_5B_DECODE_PTR(model, L->proj_tmp);
1401  float *proj_scratch = QWEN2_0_5B_DECODE_PTR(model, L->proj_scratch);
1402  float *residual1 = QWEN2_0_5B_DECODE_PTR(model, L->residual1);
1403  float *mlp_out = QWEN2_0_5B_DECODE_PTR(model, L->mlp_out);
1404  float *output = QWEN2_0_5B_DECODE_PTR(model, L->output);
1405 
1406  /* Weights (explicit types for layer 4) */
1407  const void *WQ = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wq); /* Q5_0 */
1408  const void *WK = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wk); /* Q5_0 */
1409  const void *WV = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wv); /* Q5_0 */
1410  const void *WO = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wo); /* Q5_0 */
1411  const void *W1 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w1); /* Q5_0 (gate+up) */
1412  const void *W2 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w2); /* Q4_K (down) */
1413 
1414  /* Attention biases (Qwen2-style) */
1415  const float *BQ = (const float *)QWEN2_0_5B_DECODE_PTR(model, L->bq);
1416  const float *BK = (const float *)QWEN2_0_5B_DECODE_PTR(model, L->bk);
1417  const float *BV = (const float *)QWEN2_0_5B_DECODE_PTR(model, L->bv);
1418 
1419  float *rope_cos = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_cos_cache);
1420  float *rope_sin = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_sin_cache);
1421 
1422  const int H = QWEN2_0_5B_DECODE_NUM_HEADS;
1423  const int H_kv = QWEN2_0_5B_DECODE_NUM_KV_HEADS;
1424  const int head_dim = QWEN2_0_5B_DECODE_HEAD_DIM;
1425 
1426  float q_token[H * aligned_head_dim];
1427  float k_token[H_kv * aligned_head_dim];
1428  float v_token[H_kv * aligned_head_dim];
1429  float attn_token[H * aligned_head_dim];
1430 
1431  /* Local MLP buffers (avoid layout dependencies for intermediate values) */
1432  float fc1_out[2 * aligned_intermediate_dim];
1433  float swiglu_out[aligned_intermediate_dim];
1434 
1435  /* Step 1: RMSNorm before attention */
1436  rmsnorm_forward(input,
1437  ln1_gamma,
1438  ln1_out,
1439  NULL,
1440  1,
1441  QWEN2_0_5B_DECODE_EMBED_DIM,
1442  aligned_embed_dim,
1443  1e-06f);
1444 
1445  /* Step 2: QKV projection */
1446  /* Q projection: Q5_0 -> gemm_nt_q5_0 */
1447  gemm_nt_q5_0(ln1_out, WQ, BQ, q_token, 1, H * head_dim, aligned_embed_dim);
1448 
1449  /* K projection: Q5_0 -> gemm_nt_q5_0 */
1450  gemm_nt_q5_0(ln1_out, WK, BK, k_token, 1, H_kv * head_dim, aligned_embed_dim);
1451 
1452  /* V projection: Q5_0 -> gemm_nt_q5_0 */
1453  gemm_nt_q5_0(ln1_out, WV, BV, v_token, 1, H_kv * head_dim, aligned_embed_dim);
1454 
1455  /* Step 3: RoPE */
1456  rope_forward_qk(q_token,
1457  k_token,
1458  rope_cos,
1459  rope_sin,
1460  H,
1461  H_kv,
1462  1,
1463  head_dim,
1464  aligned_head_dim,
1465  token_index);
1466 
1467  /* Step 4: KV cache write */
1468  kv_cache_write_head_major(k_token,
1469  v_token,
1470  k_cache,
1471  v_cache,
1472  H_kv,
1473  token_index,
1474  aligned_context_window,
1475  head_dim,
1476  aligned_head_dim);
1477 
1478  /* Step 5: Attention (decode) */
1479  attention_forward_decode_head_major_gqa_regular(q_token,
1480  k_cache,
1481  v_cache,
1482  attn_token,
1483  H,
1484  H_kv,
1485  token_index + 1,
1486  aligned_context_window,
1487  head_dim,
1488  aligned_head_dim);
1489 
1490  /* Step 6: Output projection */
1491  /* WO projection: Q5_0 -> gemm_nt_q5_0 */
1492  gemm_nt_q5_0(attn_token, WO, NULL, proj_tmp, 1, aligned_embed_dim, H * head_dim);
1493 
1494  /* Step 7: Residual add */
1495  qwen2_0_5b_decode_residual_add_token_major(input, proj_tmp, residual1, 1, aligned_embed_dim);
1496 
1497  /* Step 8: RMSNorm before MLP */
1498  rmsnorm_forward(residual1,
1499  ln2_gamma,
1500  ln2_out,
1501  NULL,
1502  1,
1503  QWEN2_0_5B_DECODE_EMBED_DIM,
1504  aligned_embed_dim,
1505  1e-06f);
1506 
1507  /* Step 9: MLP (SwiGLU) */
1508  /* Gate+Up projection: Q5_0 -> gemm_nt_q5_0 */
1509  gemm_nt_q5_0(ln2_out, W1, NULL, fc1_out, 1, 2 * aligned_intermediate_dim, aligned_embed_dim);
1510 
1511  /* SwiGLU activation */
1512  swiglu_forward(fc1_out, swiglu_out, 1, aligned_intermediate_dim);
1513 
1514  /* Down projection: Q4_K -> gemm_nt_q4_k */
1515  gemm_nt_q4_k(swiglu_out, W2, NULL, mlp_out, 1, aligned_embed_dim, aligned_intermediate_dim);
1516 
1517  /* Step 10: Final residual add */
1518  qwen2_0_5b_decode_residual_add_token_major(residual1, mlp_out, output, 1, aligned_embed_dim);
1519 }
1520 
1521 /*
1522  * Layer 5: wq=q5_0 wk=q5_0 wv=q5_0 wo=q5_0 w1=q5_0 w2=q4_k
1523  */
1524 static void qwen2_0_5b_decode_layer_5_decode(
1525  QWEN2_0_5B_DECODEModel *model,
1526  int token_index,
1527  int aligned_embed_dim,
1528  int aligned_head_dim,
1529  int aligned_intermediate_dim,
1530  int aligned_context_window
1531 ) {
1532  const QWEN2_0_5B_DECODELayerOffsets *L = &QWEN2_0_5B_DECODE_LAYERS[5];
1533 
1534  float *input = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[4].output);
1535 
1536  float *ln1_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln1_gamma);
1537  float *ln1_out = QWEN2_0_5B_DECODE_PTR(model, L->ln1_out);
1538  float *ln2_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln2_gamma);
1539  float *ln2_out = QWEN2_0_5B_DECODE_PTR(model, L->ln2_out);
1540  float *k_cache = QWEN2_0_5B_DECODE_PTR(model, L->k);
1541  float *v_cache = QWEN2_0_5B_DECODE_PTR(model, L->v);
1542  float *proj_tmp = QWEN2_0_5B_DECODE_PTR(model, L->proj_tmp);
1543  float *proj_scratch = QWEN2_0_5B_DECODE_PTR(model, L->proj_scratch);
1544  float *residual1 = QWEN2_0_5B_DECODE_PTR(model, L->residual1);
1545  float *mlp_out = QWEN2_0_5B_DECODE_PTR(model, L->mlp_out);
1546  float *output = QWEN2_0_5B_DECODE_PTR(model, L->output);
1547 
1548  /* Weights (explicit types for layer 5) */
1549  const void *WQ = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wq); /* Q5_0 */
1550  const void *WK = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wk); /* Q5_0 */
1551  const void *WV = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wv); /* Q5_0 */
1552  const void *WO = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wo); /* Q5_0 */
1553  const void *W1 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w1); /* Q5_0 (gate+up) */
1554  const void *W2 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w2); /* Q4_K (down) */
1555 
1556  /* Attention biases (Qwen2-style) */
1557  const float *BQ = (const float *)QWEN2_0_5B_DECODE_PTR(model, L->bq);
1558  const float *BK = (const float *)QWEN2_0_5B_DECODE_PTR(model, L->bk);
1559  const float *BV = (const float *)QWEN2_0_5B_DECODE_PTR(model, L->bv);
1560 
1561  float *rope_cos = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_cos_cache);
1562  float *rope_sin = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_sin_cache);
1563 
1564  const int H = QWEN2_0_5B_DECODE_NUM_HEADS;
1565  const int H_kv = QWEN2_0_5B_DECODE_NUM_KV_HEADS;
1566  const int head_dim = QWEN2_0_5B_DECODE_HEAD_DIM;
1567 
1568  float q_token[H * aligned_head_dim];
1569  float k_token[H_kv * aligned_head_dim];
1570  float v_token[H_kv * aligned_head_dim];
1571  float attn_token[H * aligned_head_dim];
1572 
1573  /* Local MLP buffers (avoid layout dependencies for intermediate values) */
1574  float fc1_out[2 * aligned_intermediate_dim];
1575  float swiglu_out[aligned_intermediate_dim];
1576 
1577  /* Step 1: RMSNorm before attention */
1578  rmsnorm_forward(input,
1579  ln1_gamma,
1580  ln1_out,
1581  NULL,
1582  1,
1583  QWEN2_0_5B_DECODE_EMBED_DIM,
1584  aligned_embed_dim,
1585  1e-06f);
1586 
1587  /* Step 2: QKV projection */
1588  /* Q projection: Q5_0 -> gemm_nt_q5_0 */
1589  gemm_nt_q5_0(ln1_out, WQ, BQ, q_token, 1, H * head_dim, aligned_embed_dim);
1590 
1591  /* K projection: Q5_0 -> gemm_nt_q5_0 */
1592  gemm_nt_q5_0(ln1_out, WK, BK, k_token, 1, H_kv * head_dim, aligned_embed_dim);
1593 
1594  /* V projection: Q5_0 -> gemm_nt_q5_0 */
1595  gemm_nt_q5_0(ln1_out, WV, BV, v_token, 1, H_kv * head_dim, aligned_embed_dim);
1596 
1597  /* Step 3: RoPE */
1598  rope_forward_qk(q_token,
1599  k_token,
1600  rope_cos,
1601  rope_sin,
1602  H,
1603  H_kv,
1604  1,
1605  head_dim,
1606  aligned_head_dim,
1607  token_index);
1608 
1609  /* Step 4: KV cache write */
1610  kv_cache_write_head_major(k_token,
1611  v_token,
1612  k_cache,
1613  v_cache,
1614  H_kv,
1615  token_index,
1616  aligned_context_window,
1617  head_dim,
1618  aligned_head_dim);
1619 
1620  /* Step 5: Attention (decode) */
1621  attention_forward_decode_head_major_gqa_regular(q_token,
1622  k_cache,
1623  v_cache,
1624  attn_token,
1625  H,
1626  H_kv,
1627  token_index + 1,
1628  aligned_context_window,
1629  head_dim,
1630  aligned_head_dim);
1631 
1632  /* Step 6: Output projection */
1633  /* WO projection: Q5_0 -> gemm_nt_q5_0 */
1634  gemm_nt_q5_0(attn_token, WO, NULL, proj_tmp, 1, aligned_embed_dim, H * head_dim);
1635 
1636  /* Step 7: Residual add */
1637  qwen2_0_5b_decode_residual_add_token_major(input, proj_tmp, residual1, 1, aligned_embed_dim);
1638 
1639  /* Step 8: RMSNorm before MLP */
1640  rmsnorm_forward(residual1,
1641  ln2_gamma,
1642  ln2_out,
1643  NULL,
1644  1,
1645  QWEN2_0_5B_DECODE_EMBED_DIM,
1646  aligned_embed_dim,
1647  1e-06f);
1648 
1649  /* Step 9: MLP (SwiGLU) */
1650  /* Gate+Up projection: Q5_0 -> gemm_nt_q5_0 */
1651  gemm_nt_q5_0(ln2_out, W1, NULL, fc1_out, 1, 2 * aligned_intermediate_dim, aligned_embed_dim);
1652 
1653  /* SwiGLU activation */
1654  swiglu_forward(fc1_out, swiglu_out, 1, aligned_intermediate_dim);
1655 
1656  /* Down projection: Q4_K -> gemm_nt_q4_k */
1657  gemm_nt_q4_k(swiglu_out, W2, NULL, mlp_out, 1, aligned_embed_dim, aligned_intermediate_dim);
1658 
1659  /* Step 10: Final residual add */
1660  qwen2_0_5b_decode_residual_add_token_major(residual1, mlp_out, output, 1, aligned_embed_dim);
1661 }
1662 
1663 /*
1664  * Layer 6: wq=q5_0 wk=q5_0 wv=q8_0 wo=q5_0 w1=q5_0 w2=q6_k
1665  */
1666 static void qwen2_0_5b_decode_layer_6_decode(
1667  QWEN2_0_5B_DECODEModel *model,
1668  int token_index,
1669  int aligned_embed_dim,
1670  int aligned_head_dim,
1671  int aligned_intermediate_dim,
1672  int aligned_context_window
1673 ) {
1674  const QWEN2_0_5B_DECODELayerOffsets *L = &QWEN2_0_5B_DECODE_LAYERS[6];
1675 
1676  float *input = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[5].output);
1677 
1678  float *ln1_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln1_gamma);
1679  float *ln1_out = QWEN2_0_5B_DECODE_PTR(model, L->ln1_out);
1680  float *ln2_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln2_gamma);
1681  float *ln2_out = QWEN2_0_5B_DECODE_PTR(model, L->ln2_out);
1682  float *k_cache = QWEN2_0_5B_DECODE_PTR(model, L->k);
1683  float *v_cache = QWEN2_0_5B_DECODE_PTR(model, L->v);
1684  float *proj_tmp = QWEN2_0_5B_DECODE_PTR(model, L->proj_tmp);
1685  float *proj_scratch = QWEN2_0_5B_DECODE_PTR(model, L->proj_scratch);
1686  float *residual1 = QWEN2_0_5B_DECODE_PTR(model, L->residual1);
1687  float *mlp_out = QWEN2_0_5B_DECODE_PTR(model, L->mlp_out);
1688  float *output = QWEN2_0_5B_DECODE_PTR(model, L->output);
1689 
1690  /* Weights (explicit types for layer 6) */
1691  const void *WQ = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wq); /* Q5_0 */
1692  const void *WK = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wk); /* Q5_0 */
1693  const void *WV = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wv); /* Q8_0 */
1694  const void *WO = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wo); /* Q5_0 */
1695  const void *W1 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w1); /* Q5_0 (gate+up) */
1696  const void *W2 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w2); /* Q6_K (down) */
1697 
1698  /* Attention biases (Qwen2-style) */
1699  const float *BQ = (const float *)QWEN2_0_5B_DECODE_PTR(model, L->bq);
1700  const float *BK = (const float *)QWEN2_0_5B_DECODE_PTR(model, L->bk);
1701  const float *BV = (const float *)QWEN2_0_5B_DECODE_PTR(model, L->bv);
1702 
1703  float *rope_cos = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_cos_cache);
1704  float *rope_sin = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_sin_cache);
1705 
1706  const int H = QWEN2_0_5B_DECODE_NUM_HEADS;
1707  const int H_kv = QWEN2_0_5B_DECODE_NUM_KV_HEADS;
1708  const int head_dim = QWEN2_0_5B_DECODE_HEAD_DIM;
1709 
1710  float q_token[H * aligned_head_dim];
1711  float k_token[H_kv * aligned_head_dim];
1712  float v_token[H_kv * aligned_head_dim];
1713  float attn_token[H * aligned_head_dim];
1714 
1715  /* Local MLP buffers (avoid layout dependencies for intermediate values) */
1716  float fc1_out[2 * aligned_intermediate_dim];
1717  float swiglu_out[aligned_intermediate_dim];
1718 
1719  /* Step 1: RMSNorm before attention */
1720  rmsnorm_forward(input,
1721  ln1_gamma,
1722  ln1_out,
1723  NULL,
1724  1,
1725  QWEN2_0_5B_DECODE_EMBED_DIM,
1726  aligned_embed_dim,
1727  1e-06f);
1728 
1729  /* Step 2: QKV projection */
1730  /* Q projection: Q5_0 -> gemm_nt_q5_0 */
1731  gemm_nt_q5_0(ln1_out, WQ, BQ, q_token, 1, H * head_dim, aligned_embed_dim);
1732 
1733  /* K projection: Q5_0 -> gemm_nt_q5_0 */
1734  gemm_nt_q5_0(ln1_out, WK, BK, k_token, 1, H_kv * head_dim, aligned_embed_dim);
1735 
1736  /* V projection: Q8_0 -> gemm_nt_q8_0 */
1737  gemm_nt_q8_0(ln1_out, WV, BV, v_token, 1, H_kv * head_dim, aligned_embed_dim);
1738 
1739  /* Step 3: RoPE */
1740  rope_forward_qk(q_token,
1741  k_token,
1742  rope_cos,
1743  rope_sin,
1744  H,
1745  H_kv,
1746  1,
1747  head_dim,
1748  aligned_head_dim,
1749  token_index);
1750 
1751  /* Step 4: KV cache write */
1752  kv_cache_write_head_major(k_token,
1753  v_token,
1754  k_cache,
1755  v_cache,
1756  H_kv,
1757  token_index,
1758  aligned_context_window,
1759  head_dim,
1760  aligned_head_dim);
1761 
1762  /* Step 5: Attention (decode) */
1763  attention_forward_decode_head_major_gqa_regular(q_token,
1764  k_cache,
1765  v_cache,
1766  attn_token,
1767  H,
1768  H_kv,
1769  token_index + 1,
1770  aligned_context_window,
1771  head_dim,
1772  aligned_head_dim);
1773 
1774  /* Step 6: Output projection */
1775  /* WO projection: Q5_0 -> gemm_nt_q5_0 */
1776  gemm_nt_q5_0(attn_token, WO, NULL, proj_tmp, 1, aligned_embed_dim, H * head_dim);
1777 
1778  /* Step 7: Residual add */
1779  qwen2_0_5b_decode_residual_add_token_major(input, proj_tmp, residual1, 1, aligned_embed_dim);
1780 
1781  /* Step 8: RMSNorm before MLP */
1782  rmsnorm_forward(residual1,
1783  ln2_gamma,
1784  ln2_out,
1785  NULL,
1786  1,
1787  QWEN2_0_5B_DECODE_EMBED_DIM,
1788  aligned_embed_dim,
1789  1e-06f);
1790 
1791  /* Step 9: MLP (SwiGLU) */
1792  /* Gate+Up projection: Q5_0 -> gemm_nt_q5_0 */
1793  gemm_nt_q5_0(ln2_out, W1, NULL, fc1_out, 1, 2 * aligned_intermediate_dim, aligned_embed_dim);
1794 
1795  /* SwiGLU activation */
1796  swiglu_forward(fc1_out, swiglu_out, 1, aligned_intermediate_dim);
1797 
1798  /* Down projection: Q6_K -> gemm_nt_q6_k */
1799  gemm_nt_q6_k(swiglu_out, W2, NULL, mlp_out, 1, aligned_embed_dim, aligned_intermediate_dim);
1800 
1801  /* Step 10: Final residual add */
1802  qwen2_0_5b_decode_residual_add_token_major(residual1, mlp_out, output, 1, aligned_embed_dim);
1803 }
1804 
1805 /*
1806  * Layer 7: wq=q5_0 wk=q5_0 wv=q8_0 wo=q5_0 w1=q5_0 w2=q6_k
1807  */
1808 static void qwen2_0_5b_decode_layer_7_decode(
1809  QWEN2_0_5B_DECODEModel *model,
1810  int token_index,
1811  int aligned_embed_dim,
1812  int aligned_head_dim,
1813  int aligned_intermediate_dim,
1814  int aligned_context_window
1815 ) {
1816  const QWEN2_0_5B_DECODELayerOffsets *L = &QWEN2_0_5B_DECODE_LAYERS[7];
1817 
1818  float *input = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[6].output);
1819 
1820  float *ln1_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln1_gamma);
1821  float *ln1_out = QWEN2_0_5B_DECODE_PTR(model, L->ln1_out);
1822  float *ln2_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln2_gamma);
1823  float *ln2_out = QWEN2_0_5B_DECODE_PTR(model, L->ln2_out);
1824  float *k_cache = QWEN2_0_5B_DECODE_PTR(model, L->k);
1825  float *v_cache = QWEN2_0_5B_DECODE_PTR(model, L->v);
1826  float *proj_tmp = QWEN2_0_5B_DECODE_PTR(model, L->proj_tmp);
1827  float *proj_scratch = QWEN2_0_5B_DECODE_PTR(model, L->proj_scratch);
1828  float *residual1 = QWEN2_0_5B_DECODE_PTR(model, L->residual1);
1829  float *mlp_out = QWEN2_0_5B_DECODE_PTR(model, L->mlp_out);
1830  float *output = QWEN2_0_5B_DECODE_PTR(model, L->output);
1831 
1832  /* Weights (explicit types for layer 7) */
1833  const void *WQ = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wq); /* Q5_0 */
1834  const void *WK = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wk); /* Q5_0 */
1835  const void *WV = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wv); /* Q8_0 */
1836  const void *WO = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wo); /* Q5_0 */
1837  const void *W1 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w1); /* Q5_0 (gate+up) */
1838  const void *W2 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w2); /* Q6_K (down) */
1839 
1840  /* Attention biases (Qwen2-style) */
1841  const float *BQ = (const float *)QWEN2_0_5B_DECODE_PTR(model, L->bq);
1842  const float *BK = (const float *)QWEN2_0_5B_DECODE_PTR(model, L->bk);
1843  const float *BV = (const float *)QWEN2_0_5B_DECODE_PTR(model, L->bv);
1844 
1845  float *rope_cos = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_cos_cache);
1846  float *rope_sin = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_sin_cache);
1847 
1848  const int H = QWEN2_0_5B_DECODE_NUM_HEADS;
1849  const int H_kv = QWEN2_0_5B_DECODE_NUM_KV_HEADS;
1850  const int head_dim = QWEN2_0_5B_DECODE_HEAD_DIM;
1851 
1852  float q_token[H * aligned_head_dim];
1853  float k_token[H_kv * aligned_head_dim];
1854  float v_token[H_kv * aligned_head_dim];
1855  float attn_token[H * aligned_head_dim];
1856 
1857  /* Local MLP buffers (avoid layout dependencies for intermediate values) */
1858  float fc1_out[2 * aligned_intermediate_dim];
1859  float swiglu_out[aligned_intermediate_dim];
1860 
1861  /* Step 1: RMSNorm before attention */
1862  rmsnorm_forward(input,
1863  ln1_gamma,
1864  ln1_out,
1865  NULL,
1866  1,
1867  QWEN2_0_5B_DECODE_EMBED_DIM,
1868  aligned_embed_dim,
1869  1e-06f);
1870 
1871  /* Step 2: QKV projection */
1872  /* Q projection: Q5_0 -> gemm_nt_q5_0 */
1873  gemm_nt_q5_0(ln1_out, WQ, BQ, q_token, 1, H * head_dim, aligned_embed_dim);
1874 
1875  /* K projection: Q5_0 -> gemm_nt_q5_0 */
1876  gemm_nt_q5_0(ln1_out, WK, BK, k_token, 1, H_kv * head_dim, aligned_embed_dim);
1877 
1878  /* V projection: Q8_0 -> gemm_nt_q8_0 */
1879  gemm_nt_q8_0(ln1_out, WV, BV, v_token, 1, H_kv * head_dim, aligned_embed_dim);
1880 
1881  /* Step 3: RoPE */
1882  rope_forward_qk(q_token,
1883  k_token,
1884  rope_cos,
1885  rope_sin,
1886  H,
1887  H_kv,
1888  1,
1889  head_dim,
1890  aligned_head_dim,
1891  token_index);
1892 
1893  /* Step 4: KV cache write */
1894  kv_cache_write_head_major(k_token,
1895  v_token,
1896  k_cache,
1897  v_cache,
1898  H_kv,
1899  token_index,
1900  aligned_context_window,
1901  head_dim,
1902  aligned_head_dim);
1903 
1904  /* Step 5: Attention (decode) */
1905  attention_forward_decode_head_major_gqa_regular(q_token,
1906  k_cache,
1907  v_cache,
1908  attn_token,
1909  H,
1910  H_kv,
1911  token_index + 1,
1912  aligned_context_window,
1913  head_dim,
1914  aligned_head_dim);
1915 
1916  /* Step 6: Output projection */
1917  /* WO projection: Q5_0 -> gemm_nt_q5_0 */
1918  gemm_nt_q5_0(attn_token, WO, NULL, proj_tmp, 1, aligned_embed_dim, H * head_dim);
1919 
1920  /* Step 7: Residual add */
1921  qwen2_0_5b_decode_residual_add_token_major(input, proj_tmp, residual1, 1, aligned_embed_dim);
1922 
1923  /* Step 8: RMSNorm before MLP */
1924  rmsnorm_forward(residual1,
1925  ln2_gamma,
1926  ln2_out,
1927  NULL,
1928  1,
1929  QWEN2_0_5B_DECODE_EMBED_DIM,
1930  aligned_embed_dim,
1931  1e-06f);
1932 
1933  /* Step 9: MLP (SwiGLU) */
1934  /* Gate+Up projection: Q5_0 -> gemm_nt_q5_0 */
1935  gemm_nt_q5_0(ln2_out, W1, NULL, fc1_out, 1, 2 * aligned_intermediate_dim, aligned_embed_dim);
1936 
1937  /* SwiGLU activation */
1938  swiglu_forward(fc1_out, swiglu_out, 1, aligned_intermediate_dim);
1939 
1940  /* Down projection: Q6_K -> gemm_nt_q6_k */
1941  gemm_nt_q6_k(swiglu_out, W2, NULL, mlp_out, 1, aligned_embed_dim, aligned_intermediate_dim);
1942 
1943  /* Step 10: Final residual add */
1944  qwen2_0_5b_decode_residual_add_token_major(residual1, mlp_out, output, 1, aligned_embed_dim);
1945 }
1946 
1947 /*
1948  * Layer 8: wq=q5_0 wk=q5_0 wv=q8_0 wo=q5_0 w1=q5_0 w2=q6_k
1949  */
1950 static void qwen2_0_5b_decode_layer_8_decode(
1951  QWEN2_0_5B_DECODEModel *model,
1952  int token_index,
1953  int aligned_embed_dim,
1954  int aligned_head_dim,
1955  int aligned_intermediate_dim,
1956  int aligned_context_window
1957 ) {
1958  const QWEN2_0_5B_DECODELayerOffsets *L = &QWEN2_0_5B_DECODE_LAYERS[8];
1959 
1960  float *input = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[7].output);
1961 
1962  float *ln1_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln1_gamma);
1963  float *ln1_out = QWEN2_0_5B_DECODE_PTR(model, L->ln1_out);
1964  float *ln2_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln2_gamma);
1965  float *ln2_out = QWEN2_0_5B_DECODE_PTR(model, L->ln2_out);
1966  float *k_cache = QWEN2_0_5B_DECODE_PTR(model, L->k);
1967  float *v_cache = QWEN2_0_5B_DECODE_PTR(model, L->v);
1968  float *proj_tmp = QWEN2_0_5B_DECODE_PTR(model, L->proj_tmp);
1969  float *proj_scratch = QWEN2_0_5B_DECODE_PTR(model, L->proj_scratch);
1970  float *residual1 = QWEN2_0_5B_DECODE_PTR(model, L->residual1);
1971  float *mlp_out = QWEN2_0_5B_DECODE_PTR(model, L->mlp_out);
1972  float *output = QWEN2_0_5B_DECODE_PTR(model, L->output);
1973 
1974  /* Weights (explicit types for layer 8) */
1975  const void *WQ = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wq); /* Q5_0 */
1976  const void *WK = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wk); /* Q5_0 */
1977  const void *WV = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wv); /* Q8_0 */
1978  const void *WO = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wo); /* Q5_0 */
1979  const void *W1 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w1); /* Q5_0 (gate+up) */
1980  const void *W2 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w2); /* Q6_K (down) */
1981 
1982  /* Attention biases (Qwen2-style) */
1983  const float *BQ = (const float *)QWEN2_0_5B_DECODE_PTR(model, L->bq);
1984  const float *BK = (const float *)QWEN2_0_5B_DECODE_PTR(model, L->bk);
1985  const float *BV = (const float *)QWEN2_0_5B_DECODE_PTR(model, L->bv);
1986 
1987  float *rope_cos = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_cos_cache);
1988  float *rope_sin = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_sin_cache);
1989 
1990  const int H = QWEN2_0_5B_DECODE_NUM_HEADS;
1991  const int H_kv = QWEN2_0_5B_DECODE_NUM_KV_HEADS;
1992  const int head_dim = QWEN2_0_5B_DECODE_HEAD_DIM;
1993 
1994  float q_token[H * aligned_head_dim];
1995  float k_token[H_kv * aligned_head_dim];
1996  float v_token[H_kv * aligned_head_dim];
1997  float attn_token[H * aligned_head_dim];
1998 
1999  /* Local MLP buffers (avoid layout dependencies for intermediate values) */
2000  float fc1_out[2 * aligned_intermediate_dim];
2001  float swiglu_out[aligned_intermediate_dim];
2002 
2003  /* Step 1: RMSNorm before attention */
2004  rmsnorm_forward(input,
2005  ln1_gamma,
2006  ln1_out,
2007  NULL,
2008  1,
2009  QWEN2_0_5B_DECODE_EMBED_DIM,
2010  aligned_embed_dim,
2011  1e-06f);
2012 
2013  /* Step 2: QKV projection */
2014  /* Q projection: Q5_0 -> gemm_nt_q5_0 */
2015  gemm_nt_q5_0(ln1_out, WQ, BQ, q_token, 1, H * head_dim, aligned_embed_dim);
2016 
2017  /* K projection: Q5_0 -> gemm_nt_q5_0 */
2018  gemm_nt_q5_0(ln1_out, WK, BK, k_token, 1, H_kv * head_dim, aligned_embed_dim);
2019 
2020  /* V projection: Q8_0 -> gemm_nt_q8_0 */
2021  gemm_nt_q8_0(ln1_out, WV, BV, v_token, 1, H_kv * head_dim, aligned_embed_dim);
2022 
2023  /* Step 3: RoPE */
2024  rope_forward_qk(q_token,
2025  k_token,
2026  rope_cos,
2027  rope_sin,
2028  H,
2029  H_kv,
2030  1,
2031  head_dim,
2032  aligned_head_dim,
2033  token_index);
2034 
2035  /* Step 4: KV cache write */
2036  kv_cache_write_head_major(k_token,
2037  v_token,
2038  k_cache,
2039  v_cache,
2040  H_kv,
2041  token_index,
2042  aligned_context_window,
2043  head_dim,
2044  aligned_head_dim);
2045 
2046  /* Step 5: Attention (decode) */
2047  attention_forward_decode_head_major_gqa_regular(q_token,
2048  k_cache,
2049  v_cache,
2050  attn_token,
2051  H,
2052  H_kv,
2053  token_index + 1,
2054  aligned_context_window,
2055  head_dim,
2056  aligned_head_dim);
2057 
2058  /* Step 6: Output projection */
2059  /* WO projection: Q5_0 -> gemm_nt_q5_0 */
2060  gemm_nt_q5_0(attn_token, WO, NULL, proj_tmp, 1, aligned_embed_dim, H * head_dim);
2061 
2062  /* Step 7: Residual add */
2063  qwen2_0_5b_decode_residual_add_token_major(input, proj_tmp, residual1, 1, aligned_embed_dim);
2064 
2065  /* Step 8: RMSNorm before MLP */
2066  rmsnorm_forward(residual1,
2067  ln2_gamma,
2068  ln2_out,
2069  NULL,
2070  1,
2071  QWEN2_0_5B_DECODE_EMBED_DIM,
2072  aligned_embed_dim,
2073  1e-06f);
2074 
2075  /* Step 9: MLP (SwiGLU) */
2076  /* Gate+Up projection: Q5_0 -> gemm_nt_q5_0 */
2077  gemm_nt_q5_0(ln2_out, W1, NULL, fc1_out, 1, 2 * aligned_intermediate_dim, aligned_embed_dim);
2078 
2079  /* SwiGLU activation */
2080  swiglu_forward(fc1_out, swiglu_out, 1, aligned_intermediate_dim);
2081 
2082  /* Down projection: Q6_K -> gemm_nt_q6_k */
2083  gemm_nt_q6_k(swiglu_out, W2, NULL, mlp_out, 1, aligned_embed_dim, aligned_intermediate_dim);
2084 
2085  /* Step 10: Final residual add */
2086  qwen2_0_5b_decode_residual_add_token_major(residual1, mlp_out, output, 1, aligned_embed_dim);
2087 }
2088 
2089 /*
2090  * Layer 9: wq=q5_0 wk=q5_0 wv=q8_0 wo=q5_0 w1=q5_0 w2=q6_k
2091  */
2092 static void qwen2_0_5b_decode_layer_9_decode(
2093  QWEN2_0_5B_DECODEModel *model,
2094  int token_index,
2095  int aligned_embed_dim,
2096  int aligned_head_dim,
2097  int aligned_intermediate_dim,
2098  int aligned_context_window
2099 ) {
2100  const QWEN2_0_5B_DECODELayerOffsets *L = &QWEN2_0_5B_DECODE_LAYERS[9];
2101 
2102  float *input = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[8].output);
2103 
2104  float *ln1_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln1_gamma);
2105  float *ln1_out = QWEN2_0_5B_DECODE_PTR(model, L->ln1_out);
2106  float *ln2_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln2_gamma);
2107  float *ln2_out = QWEN2_0_5B_DECODE_PTR(model, L->ln2_out);
2108  float *k_cache = QWEN2_0_5B_DECODE_PTR(model, L->k);
2109  float *v_cache = QWEN2_0_5B_DECODE_PTR(model, L->v);
2110  float *proj_tmp = QWEN2_0_5B_DECODE_PTR(model, L->proj_tmp);
2111  float *proj_scratch = QWEN2_0_5B_DECODE_PTR(model, L->proj_scratch);
2112  float *residual1 = QWEN2_0_5B_DECODE_PTR(model, L->residual1);
2113  float *mlp_out = QWEN2_0_5B_DECODE_PTR(model, L->mlp_out);
2114  float *output = QWEN2_0_5B_DECODE_PTR(model, L->output);
2115 
2116  /* Weights (explicit types for layer 9) */
2117  const void *WQ = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wq); /* Q5_0 */
2118  const void *WK = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wk); /* Q5_0 */
2119  const void *WV = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wv); /* Q8_0 */
2120  const void *WO = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wo); /* Q5_0 */
2121  const void *W1 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w1); /* Q5_0 (gate+up) */
2122  const void *W2 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w2); /* Q6_K (down) */
2123 
2124  /* Attention biases (Qwen2-style) */
2125  const float *BQ = (const float *)QWEN2_0_5B_DECODE_PTR(model, L->bq);
2126  const float *BK = (const float *)QWEN2_0_5B_DECODE_PTR(model, L->bk);
2127  const float *BV = (const float *)QWEN2_0_5B_DECODE_PTR(model, L->bv);
2128 
2129  float *rope_cos = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_cos_cache);
2130  float *rope_sin = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_sin_cache);
2131 
2132  const int H = QWEN2_0_5B_DECODE_NUM_HEADS;
2133  const int H_kv = QWEN2_0_5B_DECODE_NUM_KV_HEADS;
2134  const int head_dim = QWEN2_0_5B_DECODE_HEAD_DIM;
2135 
2136  float q_token[H * aligned_head_dim];
2137  float k_token[H_kv * aligned_head_dim];
2138  float v_token[H_kv * aligned_head_dim];
2139  float attn_token[H * aligned_head_dim];
2140 
2141  /* Local MLP buffers (avoid layout dependencies for intermediate values) */
2142  float fc1_out[2 * aligned_intermediate_dim];
2143  float swiglu_out[aligned_intermediate_dim];
2144 
2145  /* Step 1: RMSNorm before attention */
2146  rmsnorm_forward(input,
2147  ln1_gamma,
2148  ln1_out,
2149  NULL,
2150  1,
2151  QWEN2_0_5B_DECODE_EMBED_DIM,
2152  aligned_embed_dim,
2153  1e-06f);
2154 
2155  /* Step 2: QKV projection */
2156  /* Q projection: Q5_0 -> gemm_nt_q5_0 */
2157  gemm_nt_q5_0(ln1_out, WQ, BQ, q_token, 1, H * head_dim, aligned_embed_dim);
2158 
2159  /* K projection: Q5_0 -> gemm_nt_q5_0 */
2160  gemm_nt_q5_0(ln1_out, WK, BK, k_token, 1, H_kv * head_dim, aligned_embed_dim);
2161 
2162  /* V projection: Q8_0 -> gemm_nt_q8_0 */
2163  gemm_nt_q8_0(ln1_out, WV, BV, v_token, 1, H_kv * head_dim, aligned_embed_dim);
2164 
2165  /* Step 3: RoPE */
2166  rope_forward_qk(q_token,
2167  k_token,
2168  rope_cos,
2169  rope_sin,
2170  H,
2171  H_kv,
2172  1,
2173  head_dim,
2174  aligned_head_dim,
2175  token_index);
2176 
2177  /* Step 4: KV cache write */
2178  kv_cache_write_head_major(k_token,
2179  v_token,
2180  k_cache,
2181  v_cache,
2182  H_kv,
2183  token_index,
2184  aligned_context_window,
2185  head_dim,
2186  aligned_head_dim);
2187 
2188  /* Step 5: Attention (decode) */
2189  attention_forward_decode_head_major_gqa_regular(q_token,
2190  k_cache,
2191  v_cache,
2192  attn_token,
2193  H,
2194  H_kv,
2195  token_index + 1,
2196  aligned_context_window,
2197  head_dim,
2198  aligned_head_dim);
2199 
2200  /* Step 6: Output projection */
2201  /* WO projection: Q5_0 -> gemm_nt_q5_0 */
2202  gemm_nt_q5_0(attn_token, WO, NULL, proj_tmp, 1, aligned_embed_dim, H * head_dim);
2203 
2204  /* Step 7: Residual add */
2205  qwen2_0_5b_decode_residual_add_token_major(input, proj_tmp, residual1, 1, aligned_embed_dim);
2206 
2207  /* Step 8: RMSNorm before MLP */
2208  rmsnorm_forward(residual1,
2209  ln2_gamma,
2210  ln2_out,
2211  NULL,
2212  1,
2213  QWEN2_0_5B_DECODE_EMBED_DIM,
2214  aligned_embed_dim,
2215  1e-06f);
2216 
2217  /* Step 9: MLP (SwiGLU) */
2218  /* Gate+Up projection: Q5_0 -> gemm_nt_q5_0 */
2219  gemm_nt_q5_0(ln2_out, W1, NULL, fc1_out, 1, 2 * aligned_intermediate_dim, aligned_embed_dim);
2220 
2221  /* SwiGLU activation */
2222  swiglu_forward(fc1_out, swiglu_out, 1, aligned_intermediate_dim);
2223 
2224  /* Down projection: Q6_K -> gemm_nt_q6_k */
2225  gemm_nt_q6_k(swiglu_out, W2, NULL, mlp_out, 1, aligned_embed_dim, aligned_intermediate_dim);
2226 
2227  /* Step 10: Final residual add */
2228  qwen2_0_5b_decode_residual_add_token_major(residual1, mlp_out, output, 1, aligned_embed_dim);
2229 }
2230 
2231 /*
2232  * Layer 10: wq=q5_0 wk=q5_0 wv=q8_0 wo=q5_0 w1=q5_0 w2=q6_k
2233  */
2234 static void qwen2_0_5b_decode_layer_10_decode(
2235  QWEN2_0_5B_DECODEModel *model,
2236  int token_index,
2237  int aligned_embed_dim,
2238  int aligned_head_dim,
2239  int aligned_intermediate_dim,
2240  int aligned_context_window
2241 ) {
2242  const QWEN2_0_5B_DECODELayerOffsets *L = &QWEN2_0_5B_DECODE_LAYERS[10];
2243 
2244  float *input = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[9].output);
2245 
2246  float *ln1_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln1_gamma);
2247  float *ln1_out = QWEN2_0_5B_DECODE_PTR(model, L->ln1_out);
2248  float *ln2_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln2_gamma);
2249  float *ln2_out = QWEN2_0_5B_DECODE_PTR(model, L->ln2_out);
2250  float *k_cache = QWEN2_0_5B_DECODE_PTR(model, L->k);
2251  float *v_cache = QWEN2_0_5B_DECODE_PTR(model, L->v);
2252  float *proj_tmp = QWEN2_0_5B_DECODE_PTR(model, L->proj_tmp);
2253  float *proj_scratch = QWEN2_0_5B_DECODE_PTR(model, L->proj_scratch);
2254  float *residual1 = QWEN2_0_5B_DECODE_PTR(model, L->residual1);
2255  float *mlp_out = QWEN2_0_5B_DECODE_PTR(model, L->mlp_out);
2256  float *output = QWEN2_0_5B_DECODE_PTR(model, L->output);
2257 
2258  /* Weights (explicit types for layer 10) */
2259  const void *WQ = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wq); /* Q5_0 */
2260  const void *WK = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wk); /* Q5_0 */
2261  const void *WV = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wv); /* Q8_0 */
2262  const void *WO = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wo); /* Q5_0 */
2263  const void *W1 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w1); /* Q5_0 (gate+up) */
2264  const void *W2 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w2); /* Q6_K (down) */
2265 
2266  /* Attention biases (Qwen2-style) */
2267  const float *BQ = (const float *)QWEN2_0_5B_DECODE_PTR(model, L->bq);
2268  const float *BK = (const float *)QWEN2_0_5B_DECODE_PTR(model, L->bk);
2269  const float *BV = (const float *)QWEN2_0_5B_DECODE_PTR(model, L->bv);
2270 
2271  float *rope_cos = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_cos_cache);
2272  float *rope_sin = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_sin_cache);
2273 
2274  const int H = QWEN2_0_5B_DECODE_NUM_HEADS;
2275  const int H_kv = QWEN2_0_5B_DECODE_NUM_KV_HEADS;
2276  const int head_dim = QWEN2_0_5B_DECODE_HEAD_DIM;
2277 
2278  float q_token[H * aligned_head_dim];
2279  float k_token[H_kv * aligned_head_dim];
2280  float v_token[H_kv * aligned_head_dim];
2281  float attn_token[H * aligned_head_dim];
2282 
2283  /* Local MLP buffers (avoid layout dependencies for intermediate values) */
2284  float fc1_out[2 * aligned_intermediate_dim];
2285  float swiglu_out[aligned_intermediate_dim];
2286 
2287  /* Step 1: RMSNorm before attention */
2288  rmsnorm_forward(input,
2289  ln1_gamma,
2290  ln1_out,
2291  NULL,
2292  1,
2293  QWEN2_0_5B_DECODE_EMBED_DIM,
2294  aligned_embed_dim,
2295  1e-06f);
2296 
2297  /* Step 2: QKV projection */
2298  /* Q projection: Q5_0 -> gemm_nt_q5_0 */
2299  gemm_nt_q5_0(ln1_out, WQ, BQ, q_token, 1, H * head_dim, aligned_embed_dim);
2300 
2301  /* K projection: Q5_0 -> gemm_nt_q5_0 */
2302  gemm_nt_q5_0(ln1_out, WK, BK, k_token, 1, H_kv * head_dim, aligned_embed_dim);
2303 
2304  /* V projection: Q8_0 -> gemm_nt_q8_0 */
2305  gemm_nt_q8_0(ln1_out, WV, BV, v_token, 1, H_kv * head_dim, aligned_embed_dim);
2306 
2307  /* Step 3: RoPE */
2308  rope_forward_qk(q_token,
2309  k_token,
2310  rope_cos,
2311  rope_sin,
2312  H,
2313  H_kv,
2314  1,
2315  head_dim,
2316  aligned_head_dim,
2317  token_index);
2318 
2319  /* Step 4: KV cache write */
2320  kv_cache_write_head_major(k_token,
2321  v_token,
2322  k_cache,
2323  v_cache,
2324  H_kv,
2325  token_index,
2326  aligned_context_window,
2327  head_dim,
2328  aligned_head_dim);
2329 
2330  /* Step 5: Attention (decode) */
2331  attention_forward_decode_head_major_gqa_regular(q_token,
2332  k_cache,
2333  v_cache,
2334  attn_token,
2335  H,
2336  H_kv,
2337  token_index + 1,
2338  aligned_context_window,
2339  head_dim,
2340  aligned_head_dim);
2341 
2342  /* Step 6: Output projection */
2343  /* WO projection: Q5_0 -> gemm_nt_q5_0 */
2344  gemm_nt_q5_0(attn_token, WO, NULL, proj_tmp, 1, aligned_embed_dim, H * head_dim);
2345 
2346  /* Step 7: Residual add */
2347  qwen2_0_5b_decode_residual_add_token_major(input, proj_tmp, residual1, 1, aligned_embed_dim);
2348 
2349  /* Step 8: RMSNorm before MLP */
2350  rmsnorm_forward(residual1,
2351  ln2_gamma,
2352  ln2_out,
2353  NULL,
2354  1,
2355  QWEN2_0_5B_DECODE_EMBED_DIM,
2356  aligned_embed_dim,
2357  1e-06f);
2358 
2359  /* Step 9: MLP (SwiGLU) */
2360  /* Gate+Up projection: Q5_0 -> gemm_nt_q5_0 */
2361  gemm_nt_q5_0(ln2_out, W1, NULL, fc1_out, 1, 2 * aligned_intermediate_dim, aligned_embed_dim);
2362 
2363  /* SwiGLU activation */
2364  swiglu_forward(fc1_out, swiglu_out, 1, aligned_intermediate_dim);
2365 
2366  /* Down projection: Q6_K -> gemm_nt_q6_k */
2367  gemm_nt_q6_k(swiglu_out, W2, NULL, mlp_out, 1, aligned_embed_dim, aligned_intermediate_dim);
2368 
2369  /* Step 10: Final residual add */
2370  qwen2_0_5b_decode_residual_add_token_major(residual1, mlp_out, output, 1, aligned_embed_dim);
2371 }
2372 
2373 /*
2374  * Layer 11: wq=q5_0 wk=q5_0 wv=q5_0 wo=q5_0 w1=q5_0 w2=q4_k
2375  */
2376 static void qwen2_0_5b_decode_layer_11_decode(
2377  QWEN2_0_5B_DECODEModel *model,
2378  int token_index,
2379  int aligned_embed_dim,
2380  int aligned_head_dim,
2381  int aligned_intermediate_dim,
2382  int aligned_context_window
2383 ) {
2384  const QWEN2_0_5B_DECODELayerOffsets *L = &QWEN2_0_5B_DECODE_LAYERS[11];
2385 
2386  float *input = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[10].output);
2387 
2388  float *ln1_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln1_gamma);
2389  float *ln1_out = QWEN2_0_5B_DECODE_PTR(model, L->ln1_out);
2390  float *ln2_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln2_gamma);
2391  float *ln2_out = QWEN2_0_5B_DECODE_PTR(model, L->ln2_out);
2392  float *k_cache = QWEN2_0_5B_DECODE_PTR(model, L->k);
2393  float *v_cache = QWEN2_0_5B_DECODE_PTR(model, L->v);
2394  float *proj_tmp = QWEN2_0_5B_DECODE_PTR(model, L->proj_tmp);
2395  float *proj_scratch = QWEN2_0_5B_DECODE_PTR(model, L->proj_scratch);
2396  float *residual1 = QWEN2_0_5B_DECODE_PTR(model, L->residual1);
2397  float *mlp_out = QWEN2_0_5B_DECODE_PTR(model, L->mlp_out);
2398  float *output = QWEN2_0_5B_DECODE_PTR(model, L->output);
2399 
2400  /* Weights (explicit types for layer 11) */
2401  const void *WQ = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wq); /* Q5_0 */
2402  const void *WK = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wk); /* Q5_0 */
2403  const void *WV = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wv); /* Q5_0 */
2404  const void *WO = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wo); /* Q5_0 */
2405  const void *W1 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w1); /* Q5_0 (gate+up) */
2406  const void *W2 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w2); /* Q4_K (down) */
2407 
2408  /* Attention biases (Qwen2-style) */
2409  const float *BQ = (const float *)QWEN2_0_5B_DECODE_PTR(model, L->bq);
2410  const float *BK = (const float *)QWEN2_0_5B_DECODE_PTR(model, L->bk);
2411  const float *BV = (const float *)QWEN2_0_5B_DECODE_PTR(model, L->bv);
2412 
2413  float *rope_cos = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_cos_cache);
2414  float *rope_sin = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_sin_cache);
2415 
2416  const int H = QWEN2_0_5B_DECODE_NUM_HEADS;
2417  const int H_kv = QWEN2_0_5B_DECODE_NUM_KV_HEADS;
2418  const int head_dim = QWEN2_0_5B_DECODE_HEAD_DIM;
2419 
2420  float q_token[H * aligned_head_dim];
2421  float k_token[H_kv * aligned_head_dim];
2422  float v_token[H_kv * aligned_head_dim];
2423  float attn_token[H * aligned_head_dim];
2424 
2425  /* Local MLP buffers (avoid layout dependencies for intermediate values) */
2426  float fc1_out[2 * aligned_intermediate_dim];
2427  float swiglu_out[aligned_intermediate_dim];
2428 
2429  /* Step 1: RMSNorm before attention */
2430  rmsnorm_forward(input,
2431  ln1_gamma,
2432  ln1_out,
2433  NULL,
2434  1,
2435  QWEN2_0_5B_DECODE_EMBED_DIM,
2436  aligned_embed_dim,
2437  1e-06f);
2438 
2439  /* Step 2: QKV projection */
2440  /* Q projection: Q5_0 -> gemm_nt_q5_0 */
2441  gemm_nt_q5_0(ln1_out, WQ, BQ, q_token, 1, H * head_dim, aligned_embed_dim);
2442 
2443  /* K projection: Q5_0 -> gemm_nt_q5_0 */
2444  gemm_nt_q5_0(ln1_out, WK, BK, k_token, 1, H_kv * head_dim, aligned_embed_dim);
2445 
2446  /* V projection: Q5_0 -> gemm_nt_q5_0 */
2447  gemm_nt_q5_0(ln1_out, WV, BV, v_token, 1, H_kv * head_dim, aligned_embed_dim);
2448 
2449  /* Step 3: RoPE */
2450  rope_forward_qk(q_token,
2451  k_token,
2452  rope_cos,
2453  rope_sin,
2454  H,
2455  H_kv,
2456  1,
2457  head_dim,
2458  aligned_head_dim,
2459  token_index);
2460 
2461  /* Step 4: KV cache write */
2462  kv_cache_write_head_major(k_token,
2463  v_token,
2464  k_cache,
2465  v_cache,
2466  H_kv,
2467  token_index,
2468  aligned_context_window,
2469  head_dim,
2470  aligned_head_dim);
2471 
2472  /* Step 5: Attention (decode) */
2473  attention_forward_decode_head_major_gqa_regular(q_token,
2474  k_cache,
2475  v_cache,
2476  attn_token,
2477  H,
2478  H_kv,
2479  token_index + 1,
2480  aligned_context_window,
2481  head_dim,
2482  aligned_head_dim);
2483 
2484  /* Step 6: Output projection */
2485  /* WO projection: Q5_0 -> gemm_nt_q5_0 */
2486  gemm_nt_q5_0(attn_token, WO, NULL, proj_tmp, 1, aligned_embed_dim, H * head_dim);
2487 
2488  /* Step 7: Residual add */
2489  qwen2_0_5b_decode_residual_add_token_major(input, proj_tmp, residual1, 1, aligned_embed_dim);
2490 
2491  /* Step 8: RMSNorm before MLP */
2492  rmsnorm_forward(residual1,
2493  ln2_gamma,
2494  ln2_out,
2495  NULL,
2496  1,
2497  QWEN2_0_5B_DECODE_EMBED_DIM,
2498  aligned_embed_dim,
2499  1e-06f);
2500 
2501  /* Step 9: MLP (SwiGLU) */
2502  /* Gate+Up projection: Q5_0 -> gemm_nt_q5_0 */
2503  gemm_nt_q5_0(ln2_out, W1, NULL, fc1_out, 1, 2 * aligned_intermediate_dim, aligned_embed_dim);
2504 
2505  /* SwiGLU activation */
2506  swiglu_forward(fc1_out, swiglu_out, 1, aligned_intermediate_dim);
2507 
2508  /* Down projection: Q4_K -> gemm_nt_q4_k */
2509  gemm_nt_q4_k(swiglu_out, W2, NULL, mlp_out, 1, aligned_embed_dim, aligned_intermediate_dim);
2510 
2511  /* Step 10: Final residual add */
2512  qwen2_0_5b_decode_residual_add_token_major(residual1, mlp_out, output, 1, aligned_embed_dim);
2513 }
2514 
2515 /*
2516  * Layer 12: wq=q5_0 wk=q5_0 wv=q5_0 wo=q5_0 w1=q5_0 w2=q4_k
2517  */
2518 static void qwen2_0_5b_decode_layer_12_decode(
2519  QWEN2_0_5B_DECODEModel *model,
2520  int token_index,
2521  int aligned_embed_dim,
2522  int aligned_head_dim,
2523  int aligned_intermediate_dim,
2524  int aligned_context_window
2525 ) {
2526  const QWEN2_0_5B_DECODELayerOffsets *L = &QWEN2_0_5B_DECODE_LAYERS[12];
2527 
2528  float *input = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[11].output);
2529 
2530  float *ln1_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln1_gamma);
2531  float *ln1_out = QWEN2_0_5B_DECODE_PTR(model, L->ln1_out);
2532  float *ln2_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln2_gamma);
2533  float *ln2_out = QWEN2_0_5B_DECODE_PTR(model, L->ln2_out);
2534  float *k_cache = QWEN2_0_5B_DECODE_PTR(model, L->k);
2535  float *v_cache = QWEN2_0_5B_DECODE_PTR(model, L->v);
2536  float *proj_tmp = QWEN2_0_5B_DECODE_PTR(model, L->proj_tmp);
2537  float *proj_scratch = QWEN2_0_5B_DECODE_PTR(model, L->proj_scratch);
2538  float *residual1 = QWEN2_0_5B_DECODE_PTR(model, L->residual1);
2539  float *mlp_out = QWEN2_0_5B_DECODE_PTR(model, L->mlp_out);
2540  float *output = QWEN2_0_5B_DECODE_PTR(model, L->output);
2541 
2542  /* Weights (explicit types for layer 12) */
2543  const void *WQ = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wq); /* Q5_0 */
2544  const void *WK = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wk); /* Q5_0 */
2545  const void *WV = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wv); /* Q5_0 */
2546  const void *WO = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wo); /* Q5_0 */
2547  const void *W1 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w1); /* Q5_0 (gate+up) */
2548  const void *W2 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w2); /* Q4_K (down) */
2549 
2550  /* Attention biases (Qwen2-style) */
2551  const float *BQ = (const float *)QWEN2_0_5B_DECODE_PTR(model, L->bq);
2552  const float *BK = (const float *)QWEN2_0_5B_DECODE_PTR(model, L->bk);
2553  const float *BV = (const float *)QWEN2_0_5B_DECODE_PTR(model, L->bv);
2554 
2555  float *rope_cos = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_cos_cache);
2556  float *rope_sin = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_sin_cache);
2557 
2558  const int H = QWEN2_0_5B_DECODE_NUM_HEADS;
2559  const int H_kv = QWEN2_0_5B_DECODE_NUM_KV_HEADS;
2560  const int head_dim = QWEN2_0_5B_DECODE_HEAD_DIM;
2561 
2562  float q_token[H * aligned_head_dim];
2563  float k_token[H_kv * aligned_head_dim];
2564  float v_token[H_kv * aligned_head_dim];
2565  float attn_token[H * aligned_head_dim];
2566 
2567  /* Local MLP buffers (avoid layout dependencies for intermediate values) */
2568  float fc1_out[2 * aligned_intermediate_dim];
2569  float swiglu_out[aligned_intermediate_dim];
2570 
2571  /* Step 1: RMSNorm before attention */
2572  rmsnorm_forward(input,
2573  ln1_gamma,
2574  ln1_out,
2575  NULL,
2576  1,
2577  QWEN2_0_5B_DECODE_EMBED_DIM,
2578  aligned_embed_dim,
2579  1e-06f);
2580 
2581  /* Step 2: QKV projection */
2582  /* Q projection: Q5_0 -> gemm_nt_q5_0 */
2583  gemm_nt_q5_0(ln1_out, WQ, BQ, q_token, 1, H * head_dim, aligned_embed_dim);
2584 
2585  /* K projection: Q5_0 -> gemm_nt_q5_0 */
2586  gemm_nt_q5_0(ln1_out, WK, BK, k_token, 1, H_kv * head_dim, aligned_embed_dim);
2587 
2588  /* V projection: Q5_0 -> gemm_nt_q5_0 */
2589  gemm_nt_q5_0(ln1_out, WV, BV, v_token, 1, H_kv * head_dim, aligned_embed_dim);
2590 
2591  /* Step 3: RoPE */
2592  rope_forward_qk(q_token,
2593  k_token,
2594  rope_cos,
2595  rope_sin,
2596  H,
2597  H_kv,
2598  1,
2599  head_dim,
2600  aligned_head_dim,
2601  token_index);
2602 
2603  /* Step 4: KV cache write */
2604  kv_cache_write_head_major(k_token,
2605  v_token,
2606  k_cache,
2607  v_cache,
2608  H_kv,
2609  token_index,
2610  aligned_context_window,
2611  head_dim,
2612  aligned_head_dim);
2613 
2614  /* Step 5: Attention (decode) */
2615  attention_forward_decode_head_major_gqa_regular(q_token,
2616  k_cache,
2617  v_cache,
2618  attn_token,
2619  H,
2620  H_kv,
2621  token_index + 1,
2622  aligned_context_window,
2623  head_dim,
2624  aligned_head_dim);
2625 
2626  /* Step 6: Output projection */
2627  /* WO projection: Q5_0 -> gemm_nt_q5_0 */
2628  gemm_nt_q5_0(attn_token, WO, NULL, proj_tmp, 1, aligned_embed_dim, H * head_dim);
2629 
2630  /* Step 7: Residual add */
2631  qwen2_0_5b_decode_residual_add_token_major(input, proj_tmp, residual1, 1, aligned_embed_dim);
2632 
2633  /* Step 8: RMSNorm before MLP */
2634  rmsnorm_forward(residual1,
2635  ln2_gamma,
2636  ln2_out,
2637  NULL,
2638  1,
2639  QWEN2_0_5B_DECODE_EMBED_DIM,
2640  aligned_embed_dim,
2641  1e-06f);
2642 
2643  /* Step 9: MLP (SwiGLU) */
2644  /* Gate+Up projection: Q5_0 -> gemm_nt_q5_0 */
2645  gemm_nt_q5_0(ln2_out, W1, NULL, fc1_out, 1, 2 * aligned_intermediate_dim, aligned_embed_dim);
2646 
2647  /* SwiGLU activation */
2648  swiglu_forward(fc1_out, swiglu_out, 1, aligned_intermediate_dim);
2649 
2650  /* Down projection: Q4_K -> gemm_nt_q4_k */
2651  gemm_nt_q4_k(swiglu_out, W2, NULL, mlp_out, 1, aligned_embed_dim, aligned_intermediate_dim);
2652 
2653  /* Step 10: Final residual add */
2654  qwen2_0_5b_decode_residual_add_token_major(residual1, mlp_out, output, 1, aligned_embed_dim);
2655 }
2656 
2657 /*
2658  * Layer 13: wq=q5_0 wk=q5_0 wv=q8_0 wo=q5_0 w1=q5_0 w2=q6_k
2659  */
2660 static void qwen2_0_5b_decode_layer_13_decode(
2661  QWEN2_0_5B_DECODEModel *model,
2662  int token_index,
2663  int aligned_embed_dim,
2664  int aligned_head_dim,
2665  int aligned_intermediate_dim,
2666  int aligned_context_window
2667 ) {
2668  const QWEN2_0_5B_DECODELayerOffsets *L = &QWEN2_0_5B_DECODE_LAYERS[13];
2669 
2670  float *input = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[12].output);
2671 
2672  float *ln1_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln1_gamma);
2673  float *ln1_out = QWEN2_0_5B_DECODE_PTR(model, L->ln1_out);
2674  float *ln2_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln2_gamma);
2675  float *ln2_out = QWEN2_0_5B_DECODE_PTR(model, L->ln2_out);
2676  float *k_cache = QWEN2_0_5B_DECODE_PTR(model, L->k);
2677  float *v_cache = QWEN2_0_5B_DECODE_PTR(model, L->v);
2678  float *proj_tmp = QWEN2_0_5B_DECODE_PTR(model, L->proj_tmp);
2679  float *proj_scratch = QWEN2_0_5B_DECODE_PTR(model, L->proj_scratch);
2680  float *residual1 = QWEN2_0_5B_DECODE_PTR(model, L->residual1);
2681  float *mlp_out = QWEN2_0_5B_DECODE_PTR(model, L->mlp_out);
2682  float *output = QWEN2_0_5B_DECODE_PTR(model, L->output);
2683 
2684  /* Weights (explicit types for layer 13) */
2685  const void *WQ = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wq); /* Q5_0 */
2686  const void *WK = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wk); /* Q5_0 */
2687  const void *WV = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wv); /* Q8_0 */
2688  const void *WO = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wo); /* Q5_0 */
2689  const void *W1 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w1); /* Q5_0 (gate+up) */
2690  const void *W2 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w2); /* Q6_K (down) */
2691 
2692  /* Attention biases (Qwen2-style) */
2693  const float *BQ = (const float *)QWEN2_0_5B_DECODE_PTR(model, L->bq);
2694  const float *BK = (const float *)QWEN2_0_5B_DECODE_PTR(model, L->bk);
2695  const float *BV = (const float *)QWEN2_0_5B_DECODE_PTR(model, L->bv);
2696 
2697  float *rope_cos = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_cos_cache);
2698  float *rope_sin = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_sin_cache);
2699 
2700  const int H = QWEN2_0_5B_DECODE_NUM_HEADS;
2701  const int H_kv = QWEN2_0_5B_DECODE_NUM_KV_HEADS;
2702  const int head_dim = QWEN2_0_5B_DECODE_HEAD_DIM;
2703 
2704  float q_token[H * aligned_head_dim];
2705  float k_token[H_kv * aligned_head_dim];
2706  float v_token[H_kv * aligned_head_dim];
2707  float attn_token[H * aligned_head_dim];
2708 
2709  /* Local MLP buffers (avoid layout dependencies for intermediate values) */
2710  float fc1_out[2 * aligned_intermediate_dim];
2711  float swiglu_out[aligned_intermediate_dim];
2712 
2713  /* Step 1: RMSNorm before attention */
2714  rmsnorm_forward(input,
2715  ln1_gamma,
2716  ln1_out,
2717  NULL,
2718  1,
2719  QWEN2_0_5B_DECODE_EMBED_DIM,
2720  aligned_embed_dim,
2721  1e-06f);
2722 
2723  /* Step 2: QKV projection */
2724  /* Q projection: Q5_0 -> gemm_nt_q5_0 */
2725  gemm_nt_q5_0(ln1_out, WQ, BQ, q_token, 1, H * head_dim, aligned_embed_dim);
2726 
2727  /* K projection: Q5_0 -> gemm_nt_q5_0 */
2728  gemm_nt_q5_0(ln1_out, WK, BK, k_token, 1, H_kv * head_dim, aligned_embed_dim);
2729 
2730  /* V projection: Q8_0 -> gemm_nt_q8_0 */
2731  gemm_nt_q8_0(ln1_out, WV, BV, v_token, 1, H_kv * head_dim, aligned_embed_dim);
2732 
2733  /* Step 3: RoPE */
2734  rope_forward_qk(q_token,
2735  k_token,
2736  rope_cos,
2737  rope_sin,
2738  H,
2739  H_kv,
2740  1,
2741  head_dim,
2742  aligned_head_dim,
2743  token_index);
2744 
2745  /* Step 4: KV cache write */
2746  kv_cache_write_head_major(k_token,
2747  v_token,
2748  k_cache,
2749  v_cache,
2750  H_kv,
2751  token_index,
2752  aligned_context_window,
2753  head_dim,
2754  aligned_head_dim);
2755 
2756  /* Step 5: Attention (decode) */
2757  attention_forward_decode_head_major_gqa_regular(q_token,
2758  k_cache,
2759  v_cache,
2760  attn_token,
2761  H,
2762  H_kv,
2763  token_index + 1,
2764  aligned_context_window,
2765  head_dim,
2766  aligned_head_dim);
2767 
2768  /* Step 6: Output projection */
2769  /* WO projection: Q5_0 -> gemm_nt_q5_0 */
2770  gemm_nt_q5_0(attn_token, WO, NULL, proj_tmp, 1, aligned_embed_dim, H * head_dim);
2771 
2772  /* Step 7: Residual add */
2773  qwen2_0_5b_decode_residual_add_token_major(input, proj_tmp, residual1, 1, aligned_embed_dim);
2774 
2775  /* Step 8: RMSNorm before MLP */
2776  rmsnorm_forward(residual1,
2777  ln2_gamma,
2778  ln2_out,
2779  NULL,
2780  1,
2781  QWEN2_0_5B_DECODE_EMBED_DIM,
2782  aligned_embed_dim,
2783  1e-06f);
2784 
2785  /* Step 9: MLP (SwiGLU) */
2786  /* Gate+Up projection: Q5_0 -> gemm_nt_q5_0 */
2787  gemm_nt_q5_0(ln2_out, W1, NULL, fc1_out, 1, 2 * aligned_intermediate_dim, aligned_embed_dim);
2788 
2789  /* SwiGLU activation */
2790  swiglu_forward(fc1_out, swiglu_out, 1, aligned_intermediate_dim);
2791 
2792  /* Down projection: Q6_K -> gemm_nt_q6_k */
2793  gemm_nt_q6_k(swiglu_out, W2, NULL, mlp_out, 1, aligned_embed_dim, aligned_intermediate_dim);
2794 
2795  /* Step 10: Final residual add */
2796  qwen2_0_5b_decode_residual_add_token_major(residual1, mlp_out, output, 1, aligned_embed_dim);
2797 }
2798 
2799 /*
2800  * Layer 14: wq=q5_0 wk=q5_0 wv=q5_0 wo=q5_0 w1=q5_0 w2=q4_k
2801  */
2802 static void qwen2_0_5b_decode_layer_14_decode(
2803  QWEN2_0_5B_DECODEModel *model,
2804  int token_index,
2805  int aligned_embed_dim,
2806  int aligned_head_dim,
2807  int aligned_intermediate_dim,
2808  int aligned_context_window
2809 ) {
2810  const QWEN2_0_5B_DECODELayerOffsets *L = &QWEN2_0_5B_DECODE_LAYERS[14];
2811 
2812  float *input = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[13].output);
2813 
2814  float *ln1_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln1_gamma);
2815  float *ln1_out = QWEN2_0_5B_DECODE_PTR(model, L->ln1_out);
2816  float *ln2_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln2_gamma);
2817  float *ln2_out = QWEN2_0_5B_DECODE_PTR(model, L->ln2_out);
2818  float *k_cache = QWEN2_0_5B_DECODE_PTR(model, L->k);
2819  float *v_cache = QWEN2_0_5B_DECODE_PTR(model, L->v);
2820  float *proj_tmp = QWEN2_0_5B_DECODE_PTR(model, L->proj_tmp);
2821  float *proj_scratch = QWEN2_0_5B_DECODE_PTR(model, L->proj_scratch);
2822  float *residual1 = QWEN2_0_5B_DECODE_PTR(model, L->residual1);
2823  float *mlp_out = QWEN2_0_5B_DECODE_PTR(model, L->mlp_out);
2824  float *output = QWEN2_0_5B_DECODE_PTR(model, L->output);
2825 
2826  /* Weights (explicit types for layer 14) */
2827  const void *WQ = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wq); /* Q5_0 */
2828  const void *WK = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wk); /* Q5_0 */
2829  const void *WV = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wv); /* Q5_0 */
2830  const void *WO = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wo); /* Q5_0 */
2831  const void *W1 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w1); /* Q5_0 (gate+up) */
2832  const void *W2 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w2); /* Q4_K (down) */
2833 
2834  /* Attention biases (Qwen2-style) */
2835  const float *BQ = (const float *)QWEN2_0_5B_DECODE_PTR(model, L->bq);
2836  const float *BK = (const float *)QWEN2_0_5B_DECODE_PTR(model, L->bk);
2837  const float *BV = (const float *)QWEN2_0_5B_DECODE_PTR(model, L->bv);
2838 
2839  float *rope_cos = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_cos_cache);
2840  float *rope_sin = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_sin_cache);
2841 
2842  const int H = QWEN2_0_5B_DECODE_NUM_HEADS;
2843  const int H_kv = QWEN2_0_5B_DECODE_NUM_KV_HEADS;
2844  const int head_dim = QWEN2_0_5B_DECODE_HEAD_DIM;
2845 
2846  float q_token[H * aligned_head_dim];
2847  float k_token[H_kv * aligned_head_dim];
2848  float v_token[H_kv * aligned_head_dim];
2849  float attn_token[H * aligned_head_dim];
2850 
2851  /* Local MLP buffers (avoid layout dependencies for intermediate values) */
2852  float fc1_out[2 * aligned_intermediate_dim];
2853  float swiglu_out[aligned_intermediate_dim];
2854 
2855  /* Step 1: RMSNorm before attention */
2856  rmsnorm_forward(input,
2857  ln1_gamma,
2858  ln1_out,
2859  NULL,
2860  1,
2861  QWEN2_0_5B_DECODE_EMBED_DIM,
2862  aligned_embed_dim,
2863  1e-06f);
2864 
2865  /* Step 2: QKV projection */
2866  /* Q projection: Q5_0 -> gemm_nt_q5_0 */
2867  gemm_nt_q5_0(ln1_out, WQ, BQ, q_token, 1, H * head_dim, aligned_embed_dim);
2868 
2869  /* K projection: Q5_0 -> gemm_nt_q5_0 */
2870  gemm_nt_q5_0(ln1_out, WK, BK, k_token, 1, H_kv * head_dim, aligned_embed_dim);
2871 
2872  /* V projection: Q5_0 -> gemm_nt_q5_0 */
2873  gemm_nt_q5_0(ln1_out, WV, BV, v_token, 1, H_kv * head_dim, aligned_embed_dim);
2874 
2875  /* Step 3: RoPE */
2876  rope_forward_qk(q_token,
2877  k_token,
2878  rope_cos,
2879  rope_sin,
2880  H,
2881  H_kv,
2882  1,
2883  head_dim,
2884  aligned_head_dim,
2885  token_index);
2886 
2887  /* Step 4: KV cache write */
2888  kv_cache_write_head_major(k_token,
2889  v_token,
2890  k_cache,
2891  v_cache,
2892  H_kv,
2893  token_index,
2894  aligned_context_window,
2895  head_dim,
2896  aligned_head_dim);
2897 
2898  /* Step 5: Attention (decode) */
2899  attention_forward_decode_head_major_gqa_regular(q_token,
2900  k_cache,
2901  v_cache,
2902  attn_token,
2903  H,
2904  H_kv,
2905  token_index + 1,
2906  aligned_context_window,
2907  head_dim,
2908  aligned_head_dim);
2909 
2910  /* Step 6: Output projection */
2911  /* WO projection: Q5_0 -> gemm_nt_q5_0 */
2912  gemm_nt_q5_0(attn_token, WO, NULL, proj_tmp, 1, aligned_embed_dim, H * head_dim);
2913 
2914  /* Step 7: Residual add */
2915  qwen2_0_5b_decode_residual_add_token_major(input, proj_tmp, residual1, 1, aligned_embed_dim);
2916 
2917  /* Step 8: RMSNorm before MLP */
2918  rmsnorm_forward(residual1,
2919  ln2_gamma,
2920  ln2_out,
2921  NULL,
2922  1,
2923  QWEN2_0_5B_DECODE_EMBED_DIM,
2924  aligned_embed_dim,
2925  1e-06f);
2926 
2927  /* Step 9: MLP (SwiGLU) */
2928  /* Gate+Up projection: Q5_0 -> gemm_nt_q5_0 */
2929  gemm_nt_q5_0(ln2_out, W1, NULL, fc1_out, 1, 2 * aligned_intermediate_dim, aligned_embed_dim);
2930 
2931  /* SwiGLU activation */
2932  swiglu_forward(fc1_out, swiglu_out, 1, aligned_intermediate_dim);
2933 
2934  /* Down projection: Q4_K -> gemm_nt_q4_k */
2935  gemm_nt_q4_k(swiglu_out, W2, NULL, mlp_out, 1, aligned_embed_dim, aligned_intermediate_dim);
2936 
2937  /* Step 10: Final residual add */
2938  qwen2_0_5b_decode_residual_add_token_major(residual1, mlp_out, output, 1, aligned_embed_dim);
2939 }
2940 
2941 /*
2942  * Layer 15: wq=q5_0 wk=q5_0 wv=q5_0 wo=q5_0 w1=q5_0 w2=q4_k
2943  */
2944 static void qwen2_0_5b_decode_layer_15_decode(
2945  QWEN2_0_5B_DECODEModel *model,
2946  int token_index,
2947  int aligned_embed_dim,
2948  int aligned_head_dim,
2949  int aligned_intermediate_dim,
2950  int aligned_context_window
2951 ) {
2952  const QWEN2_0_5B_DECODELayerOffsets *L = &QWEN2_0_5B_DECODE_LAYERS[15];
2953 
2954  float *input = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[14].output);
2955 
2956  float *ln1_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln1_gamma);
2957  float *ln1_out = QWEN2_0_5B_DECODE_PTR(model, L->ln1_out);
2958  float *ln2_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln2_gamma);
2959  float *ln2_out = QWEN2_0_5B_DECODE_PTR(model, L->ln2_out);
2960  float *k_cache = QWEN2_0_5B_DECODE_PTR(model, L->k);
2961  float *v_cache = QWEN2_0_5B_DECODE_PTR(model, L->v);
2962  float *proj_tmp = QWEN2_0_5B_DECODE_PTR(model, L->proj_tmp);
2963  float *proj_scratch = QWEN2_0_5B_DECODE_PTR(model, L->proj_scratch);
2964  float *residual1 = QWEN2_0_5B_DECODE_PTR(model, L->residual1);
2965  float *mlp_out = QWEN2_0_5B_DECODE_PTR(model, L->mlp_out);
2966  float *output = QWEN2_0_5B_DECODE_PTR(model, L->output);
2967 
2968  /* Weights (explicit types for layer 15) */
2969  const void *WQ = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wq); /* Q5_0 */
2970  const void *WK = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wk); /* Q5_0 */
2971  const void *WV = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wv); /* Q5_0 */
2972  const void *WO = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wo); /* Q5_0 */
2973  const void *W1 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w1); /* Q5_0 (gate+up) */
2974  const void *W2 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w2); /* Q4_K (down) */
2975 
2976  /* Attention biases (Qwen2-style) */
2977  const float *BQ = (const float *)QWEN2_0_5B_DECODE_PTR(model, L->bq);
2978  const float *BK = (const float *)QWEN2_0_5B_DECODE_PTR(model, L->bk);
2979  const float *BV = (const float *)QWEN2_0_5B_DECODE_PTR(model, L->bv);
2980 
2981  float *rope_cos = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_cos_cache);
2982  float *rope_sin = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_sin_cache);
2983 
2984  const int H = QWEN2_0_5B_DECODE_NUM_HEADS;
2985  const int H_kv = QWEN2_0_5B_DECODE_NUM_KV_HEADS;
2986  const int head_dim = QWEN2_0_5B_DECODE_HEAD_DIM;
2987 
2988  float q_token[H * aligned_head_dim];
2989  float k_token[H_kv * aligned_head_dim];
2990  float v_token[H_kv * aligned_head_dim];
2991  float attn_token[H * aligned_head_dim];
2992 
2993  /* Local MLP buffers (avoid layout dependencies for intermediate values) */
2994  float fc1_out[2 * aligned_intermediate_dim];
2995  float swiglu_out[aligned_intermediate_dim];
2996 
2997  /* Step 1: RMSNorm before attention */
2998  rmsnorm_forward(input,
2999  ln1_gamma,
3000  ln1_out,
3001  NULL,
3002  1,
3003  QWEN2_0_5B_DECODE_EMBED_DIM,
3004  aligned_embed_dim,
3005  1e-06f);
3006 
3007  /* Step 2: QKV projection */
3008  /* Q projection: Q5_0 -> gemm_nt_q5_0 */
3009  gemm_nt_q5_0(ln1_out, WQ, BQ, q_token, 1, H * head_dim, aligned_embed_dim);
3010 
3011  /* K projection: Q5_0 -> gemm_nt_q5_0 */
3012  gemm_nt_q5_0(ln1_out, WK, BK, k_token, 1, H_kv * head_dim, aligned_embed_dim);
3013 
3014  /* V projection: Q5_0 -> gemm_nt_q5_0 */
3015  gemm_nt_q5_0(ln1_out, WV, BV, v_token, 1, H_kv * head_dim, aligned_embed_dim);
3016 
3017  /* Step 3: RoPE */
3018  rope_forward_qk(q_token,
3019  k_token,
3020  rope_cos,
3021  rope_sin,
3022  H,
3023  H_kv,
3024  1,
3025  head_dim,
3026  aligned_head_dim,
3027  token_index);
3028 
3029  /* Step 4: KV cache write */
3030  kv_cache_write_head_major(k_token,
3031  v_token,
3032  k_cache,
3033  v_cache,
3034  H_kv,
3035  token_index,
3036  aligned_context_window,
3037  head_dim,
3038  aligned_head_dim);
3039 
3040  /* Step 5: Attention (decode) */
3041  attention_forward_decode_head_major_gqa_regular(q_token,
3042  k_cache,
3043  v_cache,
3044  attn_token,
3045  H,
3046  H_kv,
3047  token_index + 1,
3048  aligned_context_window,
3049  head_dim,
3050  aligned_head_dim);
3051 
3052  /* Step 6: Output projection */
3053  /* WO projection: Q5_0 -> gemm_nt_q5_0 */
3054  gemm_nt_q5_0(attn_token, WO, NULL, proj_tmp, 1, aligned_embed_dim, H * head_dim);
3055 
3056  /* Step 7: Residual add */
3057  qwen2_0_5b_decode_residual_add_token_major(input, proj_tmp, residual1, 1, aligned_embed_dim);
3058 
3059  /* Step 8: RMSNorm before MLP */
3060  rmsnorm_forward(residual1,
3061  ln2_gamma,
3062  ln2_out,
3063  NULL,
3064  1,
3065  QWEN2_0_5B_DECODE_EMBED_DIM,
3066  aligned_embed_dim,
3067  1e-06f);
3068 
3069  /* Step 9: MLP (SwiGLU) */
3070  /* Gate+Up projection: Q5_0 -> gemm_nt_q5_0 */
3071  gemm_nt_q5_0(ln2_out, W1, NULL, fc1_out, 1, 2 * aligned_intermediate_dim, aligned_embed_dim);
3072 
3073  /* SwiGLU activation */
3074  swiglu_forward(fc1_out, swiglu_out, 1, aligned_intermediate_dim);
3075 
3076  /* Down projection: Q4_K -> gemm_nt_q4_k */
3077  gemm_nt_q4_k(swiglu_out, W2, NULL, mlp_out, 1, aligned_embed_dim, aligned_intermediate_dim);
3078 
3079  /* Step 10: Final residual add */
3080  qwen2_0_5b_decode_residual_add_token_major(residual1, mlp_out, output, 1, aligned_embed_dim);
3081 }
3082 
3083 /*
3084  * Layer 16: wq=q5_0 wk=q5_0 wv=q8_0 wo=q5_0 w1=q5_0 w2=q6_k
3085  */
3086 static void qwen2_0_5b_decode_layer_16_decode(
3087  QWEN2_0_5B_DECODEModel *model,
3088  int token_index,
3089  int aligned_embed_dim,
3090  int aligned_head_dim,
3091  int aligned_intermediate_dim,
3092  int aligned_context_window
3093 ) {
3094  const QWEN2_0_5B_DECODELayerOffsets *L = &QWEN2_0_5B_DECODE_LAYERS[16];
3095 
3096  float *input = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[15].output);
3097 
3098  float *ln1_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln1_gamma);
3099  float *ln1_out = QWEN2_0_5B_DECODE_PTR(model, L->ln1_out);
3100  float *ln2_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln2_gamma);
3101  float *ln2_out = QWEN2_0_5B_DECODE_PTR(model, L->ln2_out);
3102  float *k_cache = QWEN2_0_5B_DECODE_PTR(model, L->k);
3103  float *v_cache = QWEN2_0_5B_DECODE_PTR(model, L->v);
3104  float *proj_tmp = QWEN2_0_5B_DECODE_PTR(model, L->proj_tmp);
3105  float *proj_scratch = QWEN2_0_5B_DECODE_PTR(model, L->proj_scratch);
3106  float *residual1 = QWEN2_0_5B_DECODE_PTR(model, L->residual1);
3107  float *mlp_out = QWEN2_0_5B_DECODE_PTR(model, L->mlp_out);
3108  float *output = QWEN2_0_5B_DECODE_PTR(model, L->output);
3109 
3110  /* Weights (explicit types for layer 16) */
3111  const void *WQ = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wq); /* Q5_0 */
3112  const void *WK = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wk); /* Q5_0 */
3113  const void *WV = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wv); /* Q8_0 */
3114  const void *WO = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wo); /* Q5_0 */
3115  const void *W1 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w1); /* Q5_0 (gate+up) */
3116  const void *W2 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w2); /* Q6_K (down) */
3117 
3118  /* Attention biases (Qwen2-style) */
3119  const float *BQ = (const float *)QWEN2_0_5B_DECODE_PTR(model, L->bq);
3120  const float *BK = (const float *)QWEN2_0_5B_DECODE_PTR(model, L->bk);
3121  const float *BV = (const float *)QWEN2_0_5B_DECODE_PTR(model, L->bv);
3122 
3123  float *rope_cos = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_cos_cache);
3124  float *rope_sin = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_sin_cache);
3125 
3126  const int H = QWEN2_0_5B_DECODE_NUM_HEADS;
3127  const int H_kv = QWEN2_0_5B_DECODE_NUM_KV_HEADS;
3128  const int head_dim = QWEN2_0_5B_DECODE_HEAD_DIM;
3129 
3130  float q_token[H * aligned_head_dim];
3131  float k_token[H_kv * aligned_head_dim];
3132  float v_token[H_kv * aligned_head_dim];
3133  float attn_token[H * aligned_head_dim];
3134 
3135  /* Local MLP buffers (avoid layout dependencies for intermediate values) */
3136  float fc1_out[2 * aligned_intermediate_dim];
3137  float swiglu_out[aligned_intermediate_dim];
3138 
3139  /* Step 1: RMSNorm before attention */
3140  rmsnorm_forward(input,
3141  ln1_gamma,
3142  ln1_out,
3143  NULL,
3144  1,
3145  QWEN2_0_5B_DECODE_EMBED_DIM,
3146  aligned_embed_dim,
3147  1e-06f);
3148 
3149  /* Step 2: QKV projection */
3150  /* Q projection: Q5_0 -> gemm_nt_q5_0 */
3151  gemm_nt_q5_0(ln1_out, WQ, BQ, q_token, 1, H * head_dim, aligned_embed_dim);
3152 
3153  /* K projection: Q5_0 -> gemm_nt_q5_0 */
3154  gemm_nt_q5_0(ln1_out, WK, BK, k_token, 1, H_kv * head_dim, aligned_embed_dim);
3155 
3156  /* V projection: Q8_0 -> gemm_nt_q8_0 */
3157  gemm_nt_q8_0(ln1_out, WV, BV, v_token, 1, H_kv * head_dim, aligned_embed_dim);
3158 
3159  /* Step 3: RoPE */
3160  rope_forward_qk(q_token,
3161  k_token,
3162  rope_cos,
3163  rope_sin,
3164  H,
3165  H_kv,
3166  1,
3167  head_dim,
3168  aligned_head_dim,
3169  token_index);
3170 
3171  /* Step 4: KV cache write */
3172  kv_cache_write_head_major(k_token,
3173  v_token,
3174  k_cache,
3175  v_cache,
3176  H_kv,
3177  token_index,
3178  aligned_context_window,
3179  head_dim,
3180  aligned_head_dim);
3181 
3182  /* Step 5: Attention (decode) */
3183  attention_forward_decode_head_major_gqa_regular(q_token,
3184  k_cache,
3185  v_cache,
3186  attn_token,
3187  H,
3188  H_kv,
3189  token_index + 1,
3190  aligned_context_window,
3191  head_dim,
3192  aligned_head_dim);
3193 
3194  /* Step 6: Output projection */
3195  /* WO projection: Q5_0 -> gemm_nt_q5_0 */
3196  gemm_nt_q5_0(attn_token, WO, NULL, proj_tmp, 1, aligned_embed_dim, H * head_dim);
3197 
3198  /* Step 7: Residual add */
3199  qwen2_0_5b_decode_residual_add_token_major(input, proj_tmp, residual1, 1, aligned_embed_dim);
3200 
3201  /* Step 8: RMSNorm before MLP */
3202  rmsnorm_forward(residual1,
3203  ln2_gamma,
3204  ln2_out,
3205  NULL,
3206  1,
3207  QWEN2_0_5B_DECODE_EMBED_DIM,
3208  aligned_embed_dim,
3209  1e-06f);
3210 
3211  /* Step 9: MLP (SwiGLU) */
3212  /* Gate+Up projection: Q5_0 -> gemm_nt_q5_0 */
3213  gemm_nt_q5_0(ln2_out, W1, NULL, fc1_out, 1, 2 * aligned_intermediate_dim, aligned_embed_dim);
3214 
3215  /* SwiGLU activation */
3216  swiglu_forward(fc1_out, swiglu_out, 1, aligned_intermediate_dim);
3217 
3218  /* Down projection: Q6_K -> gemm_nt_q6_k */
3219  gemm_nt_q6_k(swiglu_out, W2, NULL, mlp_out, 1, aligned_embed_dim, aligned_intermediate_dim);
3220 
3221  /* Step 10: Final residual add */
3222  qwen2_0_5b_decode_residual_add_token_major(residual1, mlp_out, output, 1, aligned_embed_dim);
3223 }
3224 
3225 /*
3226  * Layer 17: wq=q5_0 wk=q5_0 wv=q5_0 wo=q5_0 w1=q5_0 w2=q4_k
3227  */
3228 static void qwen2_0_5b_decode_layer_17_decode(
3229  QWEN2_0_5B_DECODEModel *model,
3230  int token_index,
3231  int aligned_embed_dim,
3232  int aligned_head_dim,
3233  int aligned_intermediate_dim,
3234  int aligned_context_window
3235 ) {
3236  const QWEN2_0_5B_DECODELayerOffsets *L = &QWEN2_0_5B_DECODE_LAYERS[17];
3237 
3238  float *input = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[16].output);
3239 
3240  float *ln1_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln1_gamma);
3241  float *ln1_out = QWEN2_0_5B_DECODE_PTR(model, L->ln1_out);
3242  float *ln2_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln2_gamma);
3243  float *ln2_out = QWEN2_0_5B_DECODE_PTR(model, L->ln2_out);
3244  float *k_cache = QWEN2_0_5B_DECODE_PTR(model, L->k);
3245  float *v_cache = QWEN2_0_5B_DECODE_PTR(model, L->v);
3246  float *proj_tmp = QWEN2_0_5B_DECODE_PTR(model, L->proj_tmp);
3247  float *proj_scratch = QWEN2_0_5B_DECODE_PTR(model, L->proj_scratch);
3248  float *residual1 = QWEN2_0_5B_DECODE_PTR(model, L->residual1);
3249  float *mlp_out = QWEN2_0_5B_DECODE_PTR(model, L->mlp_out);
3250  float *output = QWEN2_0_5B_DECODE_PTR(model, L->output);
3251 
3252  /* Weights (explicit types for layer 17) */
3253  const void *WQ = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wq); /* Q5_0 */
3254  const void *WK = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wk); /* Q5_0 */
3255  const void *WV = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wv); /* Q5_0 */
3256  const void *WO = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wo); /* Q5_0 */
3257  const void *W1 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w1); /* Q5_0 (gate+up) */
3258  const void *W2 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w2); /* Q4_K (down) */
3259 
3260  /* Attention biases (Qwen2-style) */
3261  const float *BQ = (const float *)QWEN2_0_5B_DECODE_PTR(model, L->bq);
3262  const float *BK = (const float *)QWEN2_0_5B_DECODE_PTR(model, L->bk);
3263  const float *BV = (const float *)QWEN2_0_5B_DECODE_PTR(model, L->bv);
3264 
3265  float *rope_cos = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_cos_cache);
3266  float *rope_sin = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_sin_cache);
3267 
3268  const int H = QWEN2_0_5B_DECODE_NUM_HEADS;
3269  const int H_kv = QWEN2_0_5B_DECODE_NUM_KV_HEADS;
3270  const int head_dim = QWEN2_0_5B_DECODE_HEAD_DIM;
3271 
3272  float q_token[H * aligned_head_dim];
3273  float k_token[H_kv * aligned_head_dim];
3274  float v_token[H_kv * aligned_head_dim];
3275  float attn_token[H * aligned_head_dim];
3276 
3277  /* Local MLP buffers (avoid layout dependencies for intermediate values) */
3278  float fc1_out[2 * aligned_intermediate_dim];
3279  float swiglu_out[aligned_intermediate_dim];
3280 
3281  /* Step 1: RMSNorm before attention */
3282  rmsnorm_forward(input,
3283  ln1_gamma,
3284  ln1_out,
3285  NULL,
3286  1,
3287  QWEN2_0_5B_DECODE_EMBED_DIM,
3288  aligned_embed_dim,
3289  1e-06f);
3290 
3291  /* Step 2: QKV projection */
3292  /* Q projection: Q5_0 -> gemm_nt_q5_0 */
3293  gemm_nt_q5_0(ln1_out, WQ, BQ, q_token, 1, H * head_dim, aligned_embed_dim);
3294 
3295  /* K projection: Q5_0 -> gemm_nt_q5_0 */
3296  gemm_nt_q5_0(ln1_out, WK, BK, k_token, 1, H_kv * head_dim, aligned_embed_dim);
3297 
3298  /* V projection: Q5_0 -> gemm_nt_q5_0 */
3299  gemm_nt_q5_0(ln1_out, WV, BV, v_token, 1, H_kv * head_dim, aligned_embed_dim);
3300 
3301  /* Step 3: RoPE */
3302  rope_forward_qk(q_token,
3303  k_token,
3304  rope_cos,
3305  rope_sin,
3306  H,
3307  H_kv,
3308  1,
3309  head_dim,
3310  aligned_head_dim,
3311  token_index);
3312 
3313  /* Step 4: KV cache write */
3314  kv_cache_write_head_major(k_token,
3315  v_token,
3316  k_cache,
3317  v_cache,
3318  H_kv,
3319  token_index,
3320  aligned_context_window,
3321  head_dim,
3322  aligned_head_dim);
3323 
3324  /* Step 5: Attention (decode) */
3325  attention_forward_decode_head_major_gqa_regular(q_token,
3326  k_cache,
3327  v_cache,
3328  attn_token,
3329  H,
3330  H_kv,
3331  token_index + 1,
3332  aligned_context_window,
3333  head_dim,
3334  aligned_head_dim);
3335 
3336  /* Step 6: Output projection */
3337  /* WO projection: Q5_0 -> gemm_nt_q5_0 */
3338  gemm_nt_q5_0(attn_token, WO, NULL, proj_tmp, 1, aligned_embed_dim, H * head_dim);
3339 
3340  /* Step 7: Residual add */
3341  qwen2_0_5b_decode_residual_add_token_major(input, proj_tmp, residual1, 1, aligned_embed_dim);
3342 
3343  /* Step 8: RMSNorm before MLP */
3344  rmsnorm_forward(residual1,
3345  ln2_gamma,
3346  ln2_out,
3347  NULL,
3348  1,
3349  QWEN2_0_5B_DECODE_EMBED_DIM,
3350  aligned_embed_dim,
3351  1e-06f);
3352 
3353  /* Step 9: MLP (SwiGLU) */
3354  /* Gate+Up projection: Q5_0 -> gemm_nt_q5_0 */
3355  gemm_nt_q5_0(ln2_out, W1, NULL, fc1_out, 1, 2 * aligned_intermediate_dim, aligned_embed_dim);
3356 
3357  /* SwiGLU activation */
3358  swiglu_forward(fc1_out, swiglu_out, 1, aligned_intermediate_dim);
3359 
3360  /* Down projection: Q4_K -> gemm_nt_q4_k */
3361  gemm_nt_q4_k(swiglu_out, W2, NULL, mlp_out, 1, aligned_embed_dim, aligned_intermediate_dim);
3362 
3363  /* Step 10: Final residual add */
3364  qwen2_0_5b_decode_residual_add_token_major(residual1, mlp_out, output, 1, aligned_embed_dim);
3365 }
3366 
3367 /*
3368  * Layer 18: wq=q5_0 wk=q5_0 wv=q5_0 wo=q5_0 w1=q5_0 w2=q4_k
3369  */
3370 static void qwen2_0_5b_decode_layer_18_decode(
3371  QWEN2_0_5B_DECODEModel *model,
3372  int token_index,
3373  int aligned_embed_dim,
3374  int aligned_head_dim,
3375  int aligned_intermediate_dim,
3376  int aligned_context_window
3377 ) {
3378  const QWEN2_0_5B_DECODELayerOffsets *L = &QWEN2_0_5B_DECODE_LAYERS[18];
3379 
3380  float *input = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[17].output);
3381 
3382  float *ln1_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln1_gamma);
3383  float *ln1_out = QWEN2_0_5B_DECODE_PTR(model, L->ln1_out);
3384  float *ln2_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln2_gamma);
3385  float *ln2_out = QWEN2_0_5B_DECODE_PTR(model, L->ln2_out);
3386  float *k_cache = QWEN2_0_5B_DECODE_PTR(model, L->k);
3387  float *v_cache = QWEN2_0_5B_DECODE_PTR(model, L->v);
3388  float *proj_tmp = QWEN2_0_5B_DECODE_PTR(model, L->proj_tmp);
3389  float *proj_scratch = QWEN2_0_5B_DECODE_PTR(model, L->proj_scratch);
3390  float *residual1 = QWEN2_0_5B_DECODE_PTR(model, L->residual1);
3391  float *mlp_out = QWEN2_0_5B_DECODE_PTR(model, L->mlp_out);
3392  float *output = QWEN2_0_5B_DECODE_PTR(model, L->output);
3393 
3394  /* Weights (explicit types for layer 18) */
3395  const void *WQ = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wq); /* Q5_0 */
3396  const void *WK = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wk); /* Q5_0 */
3397  const void *WV = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wv); /* Q5_0 */
3398  const void *WO = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wo); /* Q5_0 */
3399  const void *W1 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w1); /* Q5_0 (gate+up) */
3400  const void *W2 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w2); /* Q4_K (down) */
3401 
3402  /* Attention biases (Qwen2-style) */
3403  const float *BQ = (const float *)QWEN2_0_5B_DECODE_PTR(model, L->bq);
3404  const float *BK = (const float *)QWEN2_0_5B_DECODE_PTR(model, L->bk);
3405  const float *BV = (const float *)QWEN2_0_5B_DECODE_PTR(model, L->bv);
3406 
3407  float *rope_cos = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_cos_cache);
3408  float *rope_sin = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_sin_cache);
3409 
3410  const int H = QWEN2_0_5B_DECODE_NUM_HEADS;
3411  const int H_kv = QWEN2_0_5B_DECODE_NUM_KV_HEADS;
3412  const int head_dim = QWEN2_0_5B_DECODE_HEAD_DIM;
3413 
3414  float q_token[H * aligned_head_dim];
3415  float k_token[H_kv * aligned_head_dim];
3416  float v_token[H_kv * aligned_head_dim];
3417  float attn_token[H * aligned_head_dim];
3418 
3419  /* Local MLP buffers (avoid layout dependencies for intermediate values) */
3420  float fc1_out[2 * aligned_intermediate_dim];
3421  float swiglu_out[aligned_intermediate_dim];
3422 
3423  /* Step 1: RMSNorm before attention */
3424  rmsnorm_forward(input,
3425  ln1_gamma,
3426  ln1_out,
3427  NULL,
3428  1,
3429  QWEN2_0_5B_DECODE_EMBED_DIM,
3430  aligned_embed_dim,
3431  1e-06f);
3432 
3433  /* Step 2: QKV projection */
3434  /* Q projection: Q5_0 -> gemm_nt_q5_0 */
3435  gemm_nt_q5_0(ln1_out, WQ, BQ, q_token, 1, H * head_dim, aligned_embed_dim);
3436 
3437  /* K projection: Q5_0 -> gemm_nt_q5_0 */
3438  gemm_nt_q5_0(ln1_out, WK, BK, k_token, 1, H_kv * head_dim, aligned_embed_dim);
3439 
3440  /* V projection: Q5_0 -> gemm_nt_q5_0 */
3441  gemm_nt_q5_0(ln1_out, WV, BV, v_token, 1, H_kv * head_dim, aligned_embed_dim);
3442 
3443  /* Step 3: RoPE */
3444  rope_forward_qk(q_token,
3445  k_token,
3446  rope_cos,
3447  rope_sin,
3448  H,
3449  H_kv,
3450  1,
3451  head_dim,
3452  aligned_head_dim,
3453  token_index);
3454 
3455  /* Step 4: KV cache write */
3456  kv_cache_write_head_major(k_token,
3457  v_token,
3458  k_cache,
3459  v_cache,
3460  H_kv,
3461  token_index,
3462  aligned_context_window,
3463  head_dim,
3464  aligned_head_dim);
3465 
3466  /* Step 5: Attention (decode) */
3467  attention_forward_decode_head_major_gqa_regular(q_token,
3468  k_cache,
3469  v_cache,
3470  attn_token,
3471  H,
3472  H_kv,
3473  token_index + 1,
3474  aligned_context_window,
3475  head_dim,
3476  aligned_head_dim);
3477 
3478  /* Step 6: Output projection */
3479  /* WO projection: Q5_0 -> gemm_nt_q5_0 */
3480  gemm_nt_q5_0(attn_token, WO, NULL, proj_tmp, 1, aligned_embed_dim, H * head_dim);
3481 
3482  /* Step 7: Residual add */
3483  qwen2_0_5b_decode_residual_add_token_major(input, proj_tmp, residual1, 1, aligned_embed_dim);
3484 
3485  /* Step 8: RMSNorm before MLP */
3486  rmsnorm_forward(residual1,
3487  ln2_gamma,
3488  ln2_out,
3489  NULL,
3490  1,
3491  QWEN2_0_5B_DECODE_EMBED_DIM,
3492  aligned_embed_dim,
3493  1e-06f);
3494 
3495  /* Step 9: MLP (SwiGLU) */
3496  /* Gate+Up projection: Q5_0 -> gemm_nt_q5_0 */
3497  gemm_nt_q5_0(ln2_out, W1, NULL, fc1_out, 1, 2 * aligned_intermediate_dim, aligned_embed_dim);
3498 
3499  /* SwiGLU activation */
3500  swiglu_forward(fc1_out, swiglu_out, 1, aligned_intermediate_dim);
3501 
3502  /* Down projection: Q4_K -> gemm_nt_q4_k */
3503  gemm_nt_q4_k(swiglu_out, W2, NULL, mlp_out, 1, aligned_embed_dim, aligned_intermediate_dim);
3504 
3505  /* Step 10: Final residual add */
3506  qwen2_0_5b_decode_residual_add_token_major(residual1, mlp_out, output, 1, aligned_embed_dim);
3507 }
3508 
3509 /*
3510  * Layer 19: wq=q5_0 wk=q5_0 wv=q8_0 wo=q5_0 w1=q5_0 w2=q6_k
3511  */
3512 static void qwen2_0_5b_decode_layer_19_decode(
3513  QWEN2_0_5B_DECODEModel *model,
3514  int token_index,
3515  int aligned_embed_dim,
3516  int aligned_head_dim,
3517  int aligned_intermediate_dim,
3518  int aligned_context_window
3519 ) {
3520  const QWEN2_0_5B_DECODELayerOffsets *L = &QWEN2_0_5B_DECODE_LAYERS[19];
3521 
3522  float *input = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[18].output);
3523 
3524  float *ln1_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln1_gamma);
3525  float *ln1_out = QWEN2_0_5B_DECODE_PTR(model, L->ln1_out);
3526  float *ln2_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln2_gamma);
3527  float *ln2_out = QWEN2_0_5B_DECODE_PTR(model, L->ln2_out);
3528  float *k_cache = QWEN2_0_5B_DECODE_PTR(model, L->k);
3529  float *v_cache = QWEN2_0_5B_DECODE_PTR(model, L->v);
3530  float *proj_tmp = QWEN2_0_5B_DECODE_PTR(model, L->proj_tmp);
3531  float *proj_scratch = QWEN2_0_5B_DECODE_PTR(model, L->proj_scratch);
3532  float *residual1 = QWEN2_0_5B_DECODE_PTR(model, L->residual1);
3533  float *mlp_out = QWEN2_0_5B_DECODE_PTR(model, L->mlp_out);
3534  float *output = QWEN2_0_5B_DECODE_PTR(model, L->output);
3535 
3536  /* Weights (explicit types for layer 19) */
3537  const void *WQ = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wq); /* Q5_0 */
3538  const void *WK = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wk); /* Q5_0 */
3539  const void *WV = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wv); /* Q8_0 */
3540  const void *WO = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wo); /* Q5_0 */
3541  const void *W1 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w1); /* Q5_0 (gate+up) */
3542  const void *W2 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w2); /* Q6_K (down) */
3543 
3544  /* Attention biases (Qwen2-style) */
3545  const float *BQ = (const float *)QWEN2_0_5B_DECODE_PTR(model, L->bq);
3546  const float *BK = (const float *)QWEN2_0_5B_DECODE_PTR(model, L->bk);
3547  const float *BV = (const float *)QWEN2_0_5B_DECODE_PTR(model, L->bv);
3548 
3549  float *rope_cos = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_cos_cache);
3550  float *rope_sin = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_sin_cache);
3551 
3552  const int H = QWEN2_0_5B_DECODE_NUM_HEADS;
3553  const int H_kv = QWEN2_0_5B_DECODE_NUM_KV_HEADS;
3554  const int head_dim = QWEN2_0_5B_DECODE_HEAD_DIM;
3555 
3556  float q_token[H * aligned_head_dim];
3557  float k_token[H_kv * aligned_head_dim];
3558  float v_token[H_kv * aligned_head_dim];
3559  float attn_token[H * aligned_head_dim];
3560 
3561  /* Local MLP buffers (avoid layout dependencies for intermediate values) */
3562  float fc1_out[2 * aligned_intermediate_dim];
3563  float swiglu_out[aligned_intermediate_dim];
3564 
3565  /* Step 1: RMSNorm before attention */
3566  rmsnorm_forward(input,
3567  ln1_gamma,
3568  ln1_out,
3569  NULL,
3570  1,
3571  QWEN2_0_5B_DECODE_EMBED_DIM,
3572  aligned_embed_dim,
3573  1e-06f);
3574 
3575  /* Step 2: QKV projection */
3576  /* Q projection: Q5_0 -> gemm_nt_q5_0 */
3577  gemm_nt_q5_0(ln1_out, WQ, BQ, q_token, 1, H * head_dim, aligned_embed_dim);
3578 
3579  /* K projection: Q5_0 -> gemm_nt_q5_0 */
3580  gemm_nt_q5_0(ln1_out, WK, BK, k_token, 1, H_kv * head_dim, aligned_embed_dim);
3581 
3582  /* V projection: Q8_0 -> gemm_nt_q8_0 */
3583  gemm_nt_q8_0(ln1_out, WV, BV, v_token, 1, H_kv * head_dim, aligned_embed_dim);
3584 
3585  /* Step 3: RoPE */
3586  rope_forward_qk(q_token,
3587  k_token,
3588  rope_cos,
3589  rope_sin,
3590  H,
3591  H_kv,
3592  1,
3593  head_dim,
3594  aligned_head_dim,
3595  token_index);
3596 
3597  /* Step 4: KV cache write */
3598  kv_cache_write_head_major(k_token,
3599  v_token,
3600  k_cache,
3601  v_cache,
3602  H_kv,
3603  token_index,
3604  aligned_context_window,
3605  head_dim,
3606  aligned_head_dim);
3607 
3608  /* Step 5: Attention (decode) */
3609  attention_forward_decode_head_major_gqa_regular(q_token,
3610  k_cache,
3611  v_cache,
3612  attn_token,
3613  H,
3614  H_kv,
3615  token_index + 1,
3616  aligned_context_window,
3617  head_dim,
3618  aligned_head_dim);
3619 
3620  /* Step 6: Output projection */
3621  /* WO projection: Q5_0 -> gemm_nt_q5_0 */
3622  gemm_nt_q5_0(attn_token, WO, NULL, proj_tmp, 1, aligned_embed_dim, H * head_dim);
3623 
3624  /* Step 7: Residual add */
3625  qwen2_0_5b_decode_residual_add_token_major(input, proj_tmp, residual1, 1, aligned_embed_dim);
3626 
3627  /* Step 8: RMSNorm before MLP */
3628  rmsnorm_forward(residual1,
3629  ln2_gamma,
3630  ln2_out,
3631  NULL,
3632  1,
3633  QWEN2_0_5B_DECODE_EMBED_DIM,
3634  aligned_embed_dim,
3635  1e-06f);
3636 
3637  /* Step 9: MLP (SwiGLU) */
3638  /* Gate+Up projection: Q5_0 -> gemm_nt_q5_0 */
3639  gemm_nt_q5_0(ln2_out, W1, NULL, fc1_out, 1, 2 * aligned_intermediate_dim, aligned_embed_dim);
3640 
3641  /* SwiGLU activation */
3642  swiglu_forward(fc1_out, swiglu_out, 1, aligned_intermediate_dim);
3643 
3644  /* Down projection: Q6_K -> gemm_nt_q6_k */
3645  gemm_nt_q6_k(swiglu_out, W2, NULL, mlp_out, 1, aligned_embed_dim, aligned_intermediate_dim);
3646 
3647  /* Step 10: Final residual add */
3648  qwen2_0_5b_decode_residual_add_token_major(residual1, mlp_out, output, 1, aligned_embed_dim);
3649 }
3650 
3651 /*
3652  * Layer 20: wq=q5_0 wk=q5_0 wv=q5_0 wo=q5_0 w1=q5_0 w2=q4_k
3653  */
3654 static void qwen2_0_5b_decode_layer_20_decode(
3655  QWEN2_0_5B_DECODEModel *model,
3656  int token_index,
3657  int aligned_embed_dim,
3658  int aligned_head_dim,
3659  int aligned_intermediate_dim,
3660  int aligned_context_window
3661 ) {
3662  const QWEN2_0_5B_DECODELayerOffsets *L = &QWEN2_0_5B_DECODE_LAYERS[20];
3663 
3664  float *input = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[19].output);
3665 
3666  float *ln1_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln1_gamma);
3667  float *ln1_out = QWEN2_0_5B_DECODE_PTR(model, L->ln1_out);
3668  float *ln2_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln2_gamma);
3669  float *ln2_out = QWEN2_0_5B_DECODE_PTR(model, L->ln2_out);
3670  float *k_cache = QWEN2_0_5B_DECODE_PTR(model, L->k);
3671  float *v_cache = QWEN2_0_5B_DECODE_PTR(model, L->v);
3672  float *proj_tmp = QWEN2_0_5B_DECODE_PTR(model, L->proj_tmp);
3673  float *proj_scratch = QWEN2_0_5B_DECODE_PTR(model, L->proj_scratch);
3674  float *residual1 = QWEN2_0_5B_DECODE_PTR(model, L->residual1);
3675  float *mlp_out = QWEN2_0_5B_DECODE_PTR(model, L->mlp_out);
3676  float *output = QWEN2_0_5B_DECODE_PTR(model, L->output);
3677 
3678  /* Weights (explicit types for layer 20) */
3679  const void *WQ = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wq); /* Q5_0 */
3680  const void *WK = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wk); /* Q5_0 */
3681  const void *WV = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wv); /* Q5_0 */
3682  const void *WO = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wo); /* Q5_0 */
3683  const void *W1 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w1); /* Q5_0 (gate+up) */
3684  const void *W2 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w2); /* Q4_K (down) */
3685 
3686  /* Attention biases (Qwen2-style) */
3687  const float *BQ = (const float *)QWEN2_0_5B_DECODE_PTR(model, L->bq);
3688  const float *BK = (const float *)QWEN2_0_5B_DECODE_PTR(model, L->bk);
3689  const float *BV = (const float *)QWEN2_0_5B_DECODE_PTR(model, L->bv);
3690 
3691  float *rope_cos = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_cos_cache);
3692  float *rope_sin = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_sin_cache);
3693 
3694  const int H = QWEN2_0_5B_DECODE_NUM_HEADS;
3695  const int H_kv = QWEN2_0_5B_DECODE_NUM_KV_HEADS;
3696  const int head_dim = QWEN2_0_5B_DECODE_HEAD_DIM;
3697 
3698  float q_token[H * aligned_head_dim];
3699  float k_token[H_kv * aligned_head_dim];
3700  float v_token[H_kv * aligned_head_dim];
3701  float attn_token[H * aligned_head_dim];
3702 
3703  /* Local MLP buffers (avoid layout dependencies for intermediate values) */
3704  float fc1_out[2 * aligned_intermediate_dim];
3705  float swiglu_out[aligned_intermediate_dim];
3706 
3707  /* Step 1: RMSNorm before attention */
3708  rmsnorm_forward(input,
3709  ln1_gamma,
3710  ln1_out,
3711  NULL,
3712  1,
3713  QWEN2_0_5B_DECODE_EMBED_DIM,
3714  aligned_embed_dim,
3715  1e-06f);
3716 
3717  /* Step 2: QKV projection */
3718  /* Q projection: Q5_0 -> gemm_nt_q5_0 */
3719  gemm_nt_q5_0(ln1_out, WQ, BQ, q_token, 1, H * head_dim, aligned_embed_dim);
3720 
3721  /* K projection: Q5_0 -> gemm_nt_q5_0 */
3722  gemm_nt_q5_0(ln1_out, WK, BK, k_token, 1, H_kv * head_dim, aligned_embed_dim);
3723 
3724  /* V projection: Q5_0 -> gemm_nt_q5_0 */
3725  gemm_nt_q5_0(ln1_out, WV, BV, v_token, 1, H_kv * head_dim, aligned_embed_dim);
3726 
3727  /* Step 3: RoPE */
3728  rope_forward_qk(q_token,
3729  k_token,
3730  rope_cos,
3731  rope_sin,
3732  H,
3733  H_kv,
3734  1,
3735  head_dim,
3736  aligned_head_dim,
3737  token_index);
3738 
3739  /* Step 4: KV cache write */
3740  kv_cache_write_head_major(k_token,
3741  v_token,
3742  k_cache,
3743  v_cache,
3744  H_kv,
3745  token_index,
3746  aligned_context_window,
3747  head_dim,
3748  aligned_head_dim);
3749 
3750  /* Step 5: Attention (decode) */
3751  attention_forward_decode_head_major_gqa_regular(q_token,
3752  k_cache,
3753  v_cache,
3754  attn_token,
3755  H,
3756  H_kv,
3757  token_index + 1,
3758  aligned_context_window,
3759  head_dim,
3760  aligned_head_dim);
3761 
3762  /* Step 6: Output projection */
3763  /* WO projection: Q5_0 -> gemm_nt_q5_0 */
3764  gemm_nt_q5_0(attn_token, WO, NULL, proj_tmp, 1, aligned_embed_dim, H * head_dim);
3765 
3766  /* Step 7: Residual add */
3767  qwen2_0_5b_decode_residual_add_token_major(input, proj_tmp, residual1, 1, aligned_embed_dim);
3768 
3769  /* Step 8: RMSNorm before MLP */
3770  rmsnorm_forward(residual1,
3771  ln2_gamma,
3772  ln2_out,
3773  NULL,
3774  1,
3775  QWEN2_0_5B_DECODE_EMBED_DIM,
3776  aligned_embed_dim,
3777  1e-06f);
3778 
3779  /* Step 9: MLP (SwiGLU) */
3780  /* Gate+Up projection: Q5_0 -> gemm_nt_q5_0 */
3781  gemm_nt_q5_0(ln2_out, W1, NULL, fc1_out, 1, 2 * aligned_intermediate_dim, aligned_embed_dim);
3782 
3783  /* SwiGLU activation */
3784  swiglu_forward(fc1_out, swiglu_out, 1, aligned_intermediate_dim);
3785 
3786  /* Down projection: Q4_K -> gemm_nt_q4_k */
3787  gemm_nt_q4_k(swiglu_out, W2, NULL, mlp_out, 1, aligned_embed_dim, aligned_intermediate_dim);
3788 
3789  /* Step 10: Final residual add */
3790  qwen2_0_5b_decode_residual_add_token_major(residual1, mlp_out, output, 1, aligned_embed_dim);
3791 }
3792 
3793 /*
3794  * Layer 21: wq=q5_0 wk=q5_0 wv=q8_0 wo=q5_0 w1=q5_0 w2=q6_k
3795  */
3796 static void qwen2_0_5b_decode_layer_21_decode(
3797  QWEN2_0_5B_DECODEModel *model,
3798  int token_index,
3799  int aligned_embed_dim,
3800  int aligned_head_dim,
3801  int aligned_intermediate_dim,
3802  int aligned_context_window
3803 ) {
3804  const QWEN2_0_5B_DECODELayerOffsets *L = &QWEN2_0_5B_DECODE_LAYERS[21];
3805 
3806  float *input = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[20].output);
3807 
3808  float *ln1_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln1_gamma);
3809  float *ln1_out = QWEN2_0_5B_DECODE_PTR(model, L->ln1_out);
3810  float *ln2_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln2_gamma);
3811  float *ln2_out = QWEN2_0_5B_DECODE_PTR(model, L->ln2_out);
3812  float *k_cache = QWEN2_0_5B_DECODE_PTR(model, L->k);
3813  float *v_cache = QWEN2_0_5B_DECODE_PTR(model, L->v);
3814  float *proj_tmp = QWEN2_0_5B_DECODE_PTR(model, L->proj_tmp);
3815  float *proj_scratch = QWEN2_0_5B_DECODE_PTR(model, L->proj_scratch);
3816  float *residual1 = QWEN2_0_5B_DECODE_PTR(model, L->residual1);
3817  float *mlp_out = QWEN2_0_5B_DECODE_PTR(model, L->mlp_out);
3818  float *output = QWEN2_0_5B_DECODE_PTR(model, L->output);
3819 
3820  /* Weights (explicit types for layer 21) */
3821  const void *WQ = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wq); /* Q5_0 */
3822  const void *WK = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wk); /* Q5_0 */
3823  const void *WV = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wv); /* Q8_0 */
3824  const void *WO = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wo); /* Q5_0 */
3825  const void *W1 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w1); /* Q5_0 (gate+up) */
3826  const void *W2 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w2); /* Q6_K (down) */
3827 
3828  /* Attention biases (Qwen2-style) */
3829  const float *BQ = (const float *)QWEN2_0_5B_DECODE_PTR(model, L->bq);
3830  const float *BK = (const float *)QWEN2_0_5B_DECODE_PTR(model, L->bk);
3831  const float *BV = (const float *)QWEN2_0_5B_DECODE_PTR(model, L->bv);
3832 
3833  float *rope_cos = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_cos_cache);
3834  float *rope_sin = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_sin_cache);
3835 
3836  const int H = QWEN2_0_5B_DECODE_NUM_HEADS;
3837  const int H_kv = QWEN2_0_5B_DECODE_NUM_KV_HEADS;
3838  const int head_dim = QWEN2_0_5B_DECODE_HEAD_DIM;
3839 
3840  float q_token[H * aligned_head_dim];
3841  float k_token[H_kv * aligned_head_dim];
3842  float v_token[H_kv * aligned_head_dim];
3843  float attn_token[H * aligned_head_dim];
3844 
3845  /* Local MLP buffers (avoid layout dependencies for intermediate values) */
3846  float fc1_out[2 * aligned_intermediate_dim];
3847  float swiglu_out[aligned_intermediate_dim];
3848 
3849  /* Step 1: RMSNorm before attention */
3850  rmsnorm_forward(input,
3851  ln1_gamma,
3852  ln1_out,
3853  NULL,
3854  1,
3855  QWEN2_0_5B_DECODE_EMBED_DIM,
3856  aligned_embed_dim,
3857  1e-06f);
3858 
3859  /* Step 2: QKV projection */
3860  /* Q projection: Q5_0 -> gemm_nt_q5_0 */
3861  gemm_nt_q5_0(ln1_out, WQ, BQ, q_token, 1, H * head_dim, aligned_embed_dim);
3862 
3863  /* K projection: Q5_0 -> gemm_nt_q5_0 */
3864  gemm_nt_q5_0(ln1_out, WK, BK, k_token, 1, H_kv * head_dim, aligned_embed_dim);
3865 
3866  /* V projection: Q8_0 -> gemm_nt_q8_0 */
3867  gemm_nt_q8_0(ln1_out, WV, BV, v_token, 1, H_kv * head_dim, aligned_embed_dim);
3868 
3869  /* Step 3: RoPE */
3870  rope_forward_qk(q_token,
3871  k_token,
3872  rope_cos,
3873  rope_sin,
3874  H,
3875  H_kv,
3876  1,
3877  head_dim,
3878  aligned_head_dim,
3879  token_index);
3880 
3881  /* Step 4: KV cache write */
3882  kv_cache_write_head_major(k_token,
3883  v_token,
3884  k_cache,
3885  v_cache,
3886  H_kv,
3887  token_index,
3888  aligned_context_window,
3889  head_dim,
3890  aligned_head_dim);
3891 
3892  /* Step 5: Attention (decode) */
3893  attention_forward_decode_head_major_gqa_regular(q_token,
3894  k_cache,
3895  v_cache,
3896  attn_token,
3897  H,
3898  H_kv,
3899  token_index + 1,
3900  aligned_context_window,
3901  head_dim,
3902  aligned_head_dim);
3903 
3904  /* Step 6: Output projection */
3905  /* WO projection: Q5_0 -> gemm_nt_q5_0 */
3906  gemm_nt_q5_0(attn_token, WO, NULL, proj_tmp, 1, aligned_embed_dim, H * head_dim);
3907 
3908  /* Step 7: Residual add */
3909  qwen2_0_5b_decode_residual_add_token_major(input, proj_tmp, residual1, 1, aligned_embed_dim);
3910 
3911  /* Step 8: RMSNorm before MLP */
3912  rmsnorm_forward(residual1,
3913  ln2_gamma,
3914  ln2_out,
3915  NULL,
3916  1,
3917  QWEN2_0_5B_DECODE_EMBED_DIM,
3918  aligned_embed_dim,
3919  1e-06f);
3920 
3921  /* Step 9: MLP (SwiGLU) */
3922  /* Gate+Up projection: Q5_0 -> gemm_nt_q5_0 */
3923  gemm_nt_q5_0(ln2_out, W1, NULL, fc1_out, 1, 2 * aligned_intermediate_dim, aligned_embed_dim);
3924 
3925  /* SwiGLU activation */
3926  swiglu_forward(fc1_out, swiglu_out, 1, aligned_intermediate_dim);
3927 
3928  /* Down projection: Q6_K -> gemm_nt_q6_k */
3929  gemm_nt_q6_k(swiglu_out, W2, NULL, mlp_out, 1, aligned_embed_dim, aligned_intermediate_dim);
3930 
3931  /* Step 10: Final residual add */
3932  qwen2_0_5b_decode_residual_add_token_major(residual1, mlp_out, output, 1, aligned_embed_dim);
3933 }
3934 
3935 /*
3936  * Layer 22: wq=q5_0 wk=q5_0 wv=q5_0 wo=q5_0 w1=q5_0 w2=q4_k
3937  */
3938 static void qwen2_0_5b_decode_layer_22_decode(
3939  QWEN2_0_5B_DECODEModel *model,
3940  int token_index,
3941  int aligned_embed_dim,
3942  int aligned_head_dim,
3943  int aligned_intermediate_dim,
3944  int aligned_context_window
3945 ) {
3946  const QWEN2_0_5B_DECODELayerOffsets *L = &QWEN2_0_5B_DECODE_LAYERS[22];
3947 
3948  float *input = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[21].output);
3949 
3950  float *ln1_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln1_gamma);
3951  float *ln1_out = QWEN2_0_5B_DECODE_PTR(model, L->ln1_out);
3952  float *ln2_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln2_gamma);
3953  float *ln2_out = QWEN2_0_5B_DECODE_PTR(model, L->ln2_out);
3954  float *k_cache = QWEN2_0_5B_DECODE_PTR(model, L->k);
3955  float *v_cache = QWEN2_0_5B_DECODE_PTR(model, L->v);
3956  float *proj_tmp = QWEN2_0_5B_DECODE_PTR(model, L->proj_tmp);
3957  float *proj_scratch = QWEN2_0_5B_DECODE_PTR(model, L->proj_scratch);
3958  float *residual1 = QWEN2_0_5B_DECODE_PTR(model, L->residual1);
3959  float *mlp_out = QWEN2_0_5B_DECODE_PTR(model, L->mlp_out);
3960  float *output = QWEN2_0_5B_DECODE_PTR(model, L->output);
3961 
3962  /* Weights (explicit types for layer 22) */
3963  const void *WQ = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wq); /* Q5_0 */
3964  const void *WK = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wk); /* Q5_0 */
3965  const void *WV = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wv); /* Q5_0 */
3966  const void *WO = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wo); /* Q5_0 */
3967  const void *W1 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w1); /* Q5_0 (gate+up) */
3968  const void *W2 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w2); /* Q4_K (down) */
3969 
3970  /* Attention biases (Qwen2-style) */
3971  const float *BQ = (const float *)QWEN2_0_5B_DECODE_PTR(model, L->bq);
3972  const float *BK = (const float *)QWEN2_0_5B_DECODE_PTR(model, L->bk);
3973  const float *BV = (const float *)QWEN2_0_5B_DECODE_PTR(model, L->bv);
3974 
3975  float *rope_cos = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_cos_cache);
3976  float *rope_sin = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_sin_cache);
3977 
3978  const int H = QWEN2_0_5B_DECODE_NUM_HEADS;
3979  const int H_kv = QWEN2_0_5B_DECODE_NUM_KV_HEADS;
3980  const int head_dim = QWEN2_0_5B_DECODE_HEAD_DIM;
3981 
3982  float q_token[H * aligned_head_dim];
3983  float k_token[H_kv * aligned_head_dim];
3984  float v_token[H_kv * aligned_head_dim];
3985  float attn_token[H * aligned_head_dim];
3986 
3987  /* Local MLP buffers (avoid layout dependencies for intermediate values) */
3988  float fc1_out[2 * aligned_intermediate_dim];
3989  float swiglu_out[aligned_intermediate_dim];
3990 
3991  /* Step 1: RMSNorm before attention */
3992  rmsnorm_forward(input,
3993  ln1_gamma,
3994  ln1_out,
3995  NULL,
3996  1,
3997  QWEN2_0_5B_DECODE_EMBED_DIM,
3998  aligned_embed_dim,
3999  1e-06f);
4000 
4001  /* Step 2: QKV projection */
4002  /* Q projection: Q5_0 -> gemm_nt_q5_0 */
4003  gemm_nt_q5_0(ln1_out, WQ, BQ, q_token, 1, H * head_dim, aligned_embed_dim);
4004 
4005  /* K projection: Q5_0 -> gemm_nt_q5_0 */
4006  gemm_nt_q5_0(ln1_out, WK, BK, k_token, 1, H_kv * head_dim, aligned_embed_dim);
4007 
4008  /* V projection: Q5_0 -> gemm_nt_q5_0 */
4009  gemm_nt_q5_0(ln1_out, WV, BV, v_token, 1, H_kv * head_dim, aligned_embed_dim);
4010 
4011  /* Step 3: RoPE */
4012  rope_forward_qk(q_token,
4013  k_token,
4014  rope_cos,
4015  rope_sin,
4016  H,
4017  H_kv,
4018  1,
4019  head_dim,
4020  aligned_head_dim,
4021  token_index);
4022 
4023  /* Step 4: KV cache write */
4024  kv_cache_write_head_major(k_token,
4025  v_token,
4026  k_cache,
4027  v_cache,
4028  H_kv,
4029  token_index,
4030  aligned_context_window,
4031  head_dim,
4032  aligned_head_dim);
4033 
4034  /* Step 5: Attention (decode) */
4035  attention_forward_decode_head_major_gqa_regular(q_token,
4036  k_cache,
4037  v_cache,
4038  attn_token,
4039  H,
4040  H_kv,
4041  token_index + 1,
4042  aligned_context_window,
4043  head_dim,
4044  aligned_head_dim);
4045 
4046  /* Step 6: Output projection */
4047  /* WO projection: Q5_0 -> gemm_nt_q5_0 */
4048  gemm_nt_q5_0(attn_token, WO, NULL, proj_tmp, 1, aligned_embed_dim, H * head_dim);
4049 
4050  /* Step 7: Residual add */
4051  qwen2_0_5b_decode_residual_add_token_major(input, proj_tmp, residual1, 1, aligned_embed_dim);
4052 
4053  /* Step 8: RMSNorm before MLP */
4054  rmsnorm_forward(residual1,
4055  ln2_gamma,
4056  ln2_out,
4057  NULL,
4058  1,
4059  QWEN2_0_5B_DECODE_EMBED_DIM,
4060  aligned_embed_dim,
4061  1e-06f);
4062 
4063  /* Step 9: MLP (SwiGLU) */
4064  /* Gate+Up projection: Q5_0 -> gemm_nt_q5_0 */
4065  gemm_nt_q5_0(ln2_out, W1, NULL, fc1_out, 1, 2 * aligned_intermediate_dim, aligned_embed_dim);
4066 
4067  /* SwiGLU activation */
4068  swiglu_forward(fc1_out, swiglu_out, 1, aligned_intermediate_dim);
4069 
4070  /* Down projection: Q4_K -> gemm_nt_q4_k */
4071  gemm_nt_q4_k(swiglu_out, W2, NULL, mlp_out, 1, aligned_embed_dim, aligned_intermediate_dim);
4072 
4073  /* Step 10: Final residual add */
4074  qwen2_0_5b_decode_residual_add_token_major(residual1, mlp_out, output, 1, aligned_embed_dim);
4075 }
4076 
4077 /*
4078  * Layer 23: wq=q5_0 wk=q5_0 wv=q5_0 wo=q5_0 w1=q5_0 w2=q4_k
4079  */
4080 static void qwen2_0_5b_decode_layer_23_decode(
4081  QWEN2_0_5B_DECODEModel *model,
4082  int token_index,
4083  int aligned_embed_dim,
4084  int aligned_head_dim,
4085  int aligned_intermediate_dim,
4086  int aligned_context_window
4087 ) {
4088  const QWEN2_0_5B_DECODELayerOffsets *L = &QWEN2_0_5B_DECODE_LAYERS[23];
4089 
4090  float *input = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[22].output);
4091 
4092  float *ln1_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln1_gamma);
4093  float *ln1_out = QWEN2_0_5B_DECODE_PTR(model, L->ln1_out);
4094  float *ln2_gamma = QWEN2_0_5B_DECODE_PTR(model, L->ln2_gamma);
4095  float *ln2_out = QWEN2_0_5B_DECODE_PTR(model, L->ln2_out);
4096  float *k_cache = QWEN2_0_5B_DECODE_PTR(model, L->k);
4097  float *v_cache = QWEN2_0_5B_DECODE_PTR(model, L->v);
4098  float *proj_tmp = QWEN2_0_5B_DECODE_PTR(model, L->proj_tmp);
4099  float *proj_scratch = QWEN2_0_5B_DECODE_PTR(model, L->proj_scratch);
4100  float *residual1 = QWEN2_0_5B_DECODE_PTR(model, L->residual1);
4101  float *mlp_out = QWEN2_0_5B_DECODE_PTR(model, L->mlp_out);
4102  float *output = QWEN2_0_5B_DECODE_PTR(model, L->output);
4103 
4104  /* Weights (explicit types for layer 23) */
4105  const void *WQ = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wq); /* Q5_0 */
4106  const void *WK = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wk); /* Q5_0 */
4107  const void *WV = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wv); /* Q5_0 */
4108  const void *WO = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->wo); /* Q5_0 */
4109  const void *W1 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w1); /* Q5_0 (gate+up) */
4110  const void *W2 = (const void *)QWEN2_0_5B_DECODE_PTR(model, L->w2); /* Q4_K (down) */
4111 
4112  /* Attention biases (Qwen2-style) */
4113  const float *BQ = (const float *)QWEN2_0_5B_DECODE_PTR(model, L->bq);
4114  const float *BK = (const float *)QWEN2_0_5B_DECODE_PTR(model, L->bk);
4115  const float *BV = (const float *)QWEN2_0_5B_DECODE_PTR(model, L->bv);
4116 
4117  float *rope_cos = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_cos_cache);
4118  float *rope_sin = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_GLOBALS.rope_sin_cache);
4119 
4120  const int H = QWEN2_0_5B_DECODE_NUM_HEADS;
4121  const int H_kv = QWEN2_0_5B_DECODE_NUM_KV_HEADS;
4122  const int head_dim = QWEN2_0_5B_DECODE_HEAD_DIM;
4123 
4124  float q_token[H * aligned_head_dim];
4125  float k_token[H_kv * aligned_head_dim];
4126  float v_token[H_kv * aligned_head_dim];
4127  float attn_token[H * aligned_head_dim];
4128 
4129  /* Local MLP buffers (avoid layout dependencies for intermediate values) */
4130  float fc1_out[2 * aligned_intermediate_dim];
4131  float swiglu_out[aligned_intermediate_dim];
4132 
4133  /* Step 1: RMSNorm before attention */
4134  rmsnorm_forward(input,
4135  ln1_gamma,
4136  ln1_out,
4137  NULL,
4138  1,
4139  QWEN2_0_5B_DECODE_EMBED_DIM,
4140  aligned_embed_dim,
4141  1e-06f);
4142 
4143  /* Step 2: QKV projection */
4144  /* Q projection: Q5_0 -> gemm_nt_q5_0 */
4145  gemm_nt_q5_0(ln1_out, WQ, BQ, q_token, 1, H * head_dim, aligned_embed_dim);
4146 
4147  /* K projection: Q5_0 -> gemm_nt_q5_0 */
4148  gemm_nt_q5_0(ln1_out, WK, BK, k_token, 1, H_kv * head_dim, aligned_embed_dim);
4149 
4150  /* V projection: Q5_0 -> gemm_nt_q5_0 */
4151  gemm_nt_q5_0(ln1_out, WV, BV, v_token, 1, H_kv * head_dim, aligned_embed_dim);
4152 
4153  /* Step 3: RoPE */
4154  rope_forward_qk(q_token,
4155  k_token,
4156  rope_cos,
4157  rope_sin,
4158  H,
4159  H_kv,
4160  1,
4161  head_dim,
4162  aligned_head_dim,
4163  token_index);
4164 
4165  /* Step 4: KV cache write */
4166  kv_cache_write_head_major(k_token,
4167  v_token,
4168  k_cache,
4169  v_cache,
4170  H_kv,
4171  token_index,
4172  aligned_context_window,
4173  head_dim,
4174  aligned_head_dim);
4175 
4176  /* Step 5: Attention (decode) */
4177  attention_forward_decode_head_major_gqa_regular(q_token,
4178  k_cache,
4179  v_cache,
4180  attn_token,
4181  H,
4182  H_kv,
4183  token_index + 1,
4184  aligned_context_window,
4185  head_dim,
4186  aligned_head_dim);
4187 
4188  /* Step 6: Output projection */
4189  /* WO projection: Q5_0 -> gemm_nt_q5_0 */
4190  gemm_nt_q5_0(attn_token, WO, NULL, proj_tmp, 1, aligned_embed_dim, H * head_dim);
4191 
4192  /* Step 7: Residual add */
4193  qwen2_0_5b_decode_residual_add_token_major(input, proj_tmp, residual1, 1, aligned_embed_dim);
4194 
4195  /* Step 8: RMSNorm before MLP */
4196  rmsnorm_forward(residual1,
4197  ln2_gamma,
4198  ln2_out,
4199  NULL,
4200  1,
4201  QWEN2_0_5B_DECODE_EMBED_DIM,
4202  aligned_embed_dim,
4203  1e-06f);
4204 
4205  /* Step 9: MLP (SwiGLU) */
4206  /* Gate+Up projection: Q5_0 -> gemm_nt_q5_0 */
4207  gemm_nt_q5_0(ln2_out, W1, NULL, fc1_out, 1, 2 * aligned_intermediate_dim, aligned_embed_dim);
4208 
4209  /* SwiGLU activation */
4210  swiglu_forward(fc1_out, swiglu_out, 1, aligned_intermediate_dim);
4211 
4212  /* Down projection: Q4_K -> gemm_nt_q4_k */
4213  gemm_nt_q4_k(swiglu_out, W2, NULL, mlp_out, 1, aligned_embed_dim, aligned_intermediate_dim);
4214 
4215  /* Step 10: Final residual add */
4216  qwen2_0_5b_decode_residual_add_token_major(residual1, mlp_out, output, 1, aligned_embed_dim);
4217 }
4218 
4219 /* ============================================================================
4220  * DECODE TOKEN (calls each layer explicitly)
4221  * ============================================================================ */
4222 
4223 static void qwen2_0_5b_decode_decode_token(
4224  QWEN2_0_5B_DECODEModel *model,
4225  const int *token,
4226  int token_index
4227 ) {
4228  if (!model || !token) return;
4229 
4230  const int aligned_embed_dim = 896;
4231  const int aligned_head_dim = 64;
4232  const int aligned_intermediate_dim = 4864;
4233  const int aligned_context_window = 131072;
4234 
4235  if (token_index < 0 || token_index >= aligned_context_window) return;
4236 
4237  /* Embedding lookup */
4238  float *embed_out = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_HEADER.embedded_input);
4239  const void *embed_weight = (const void *)QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_HEADER.token_emb);
4240  /* Embedding: Q8_0 -> embedding_forward_q8_0 */
4241  embedding_forward_q8_0((const int32_t *)token,
4242  1,
4243  QWEN2_0_5B_DECODE_VOCAB_SIZE,
4244  embed_weight,
4245  NULL,
4246  embed_out,
4247  QWEN2_0_5B_DECODE_EMBED_DIM,
4248  aligned_embed_dim,
4249  1,
4250  0);
4251 
4252  /* Process each layer explicitly */
4253  qwen2_0_5b_decode_layer_0_decode(model, token_index, aligned_embed_dim, aligned_head_dim, aligned_intermediate_dim, aligned_context_window);
4254  qwen2_0_5b_decode_layer_1_decode(model, token_index, aligned_embed_dim, aligned_head_dim, aligned_intermediate_dim, aligned_context_window);
4255  qwen2_0_5b_decode_layer_2_decode(model, token_index, aligned_embed_dim, aligned_head_dim, aligned_intermediate_dim, aligned_context_window);
4256  qwen2_0_5b_decode_layer_3_decode(model, token_index, aligned_embed_dim, aligned_head_dim, aligned_intermediate_dim, aligned_context_window);
4257  qwen2_0_5b_decode_layer_4_decode(model, token_index, aligned_embed_dim, aligned_head_dim, aligned_intermediate_dim, aligned_context_window);
4258  qwen2_0_5b_decode_layer_5_decode(model, token_index, aligned_embed_dim, aligned_head_dim, aligned_intermediate_dim, aligned_context_window);
4259  qwen2_0_5b_decode_layer_6_decode(model, token_index, aligned_embed_dim, aligned_head_dim, aligned_intermediate_dim, aligned_context_window);
4260  qwen2_0_5b_decode_layer_7_decode(model, token_index, aligned_embed_dim, aligned_head_dim, aligned_intermediate_dim, aligned_context_window);
4261  qwen2_0_5b_decode_layer_8_decode(model, token_index, aligned_embed_dim, aligned_head_dim, aligned_intermediate_dim, aligned_context_window);
4262  qwen2_0_5b_decode_layer_9_decode(model, token_index, aligned_embed_dim, aligned_head_dim, aligned_intermediate_dim, aligned_context_window);
4263  qwen2_0_5b_decode_layer_10_decode(model, token_index, aligned_embed_dim, aligned_head_dim, aligned_intermediate_dim, aligned_context_window);
4264  qwen2_0_5b_decode_layer_11_decode(model, token_index, aligned_embed_dim, aligned_head_dim, aligned_intermediate_dim, aligned_context_window);
4265  qwen2_0_5b_decode_layer_12_decode(model, token_index, aligned_embed_dim, aligned_head_dim, aligned_intermediate_dim, aligned_context_window);
4266  qwen2_0_5b_decode_layer_13_decode(model, token_index, aligned_embed_dim, aligned_head_dim, aligned_intermediate_dim, aligned_context_window);
4267  qwen2_0_5b_decode_layer_14_decode(model, token_index, aligned_embed_dim, aligned_head_dim, aligned_intermediate_dim, aligned_context_window);
4268  qwen2_0_5b_decode_layer_15_decode(model, token_index, aligned_embed_dim, aligned_head_dim, aligned_intermediate_dim, aligned_context_window);
4269  qwen2_0_5b_decode_layer_16_decode(model, token_index, aligned_embed_dim, aligned_head_dim, aligned_intermediate_dim, aligned_context_window);
4270  qwen2_0_5b_decode_layer_17_decode(model, token_index, aligned_embed_dim, aligned_head_dim, aligned_intermediate_dim, aligned_context_window);
4271  qwen2_0_5b_decode_layer_18_decode(model, token_index, aligned_embed_dim, aligned_head_dim, aligned_intermediate_dim, aligned_context_window);
4272  qwen2_0_5b_decode_layer_19_decode(model, token_index, aligned_embed_dim, aligned_head_dim, aligned_intermediate_dim, aligned_context_window);
4273  qwen2_0_5b_decode_layer_20_decode(model, token_index, aligned_embed_dim, aligned_head_dim, aligned_intermediate_dim, aligned_context_window);
4274  qwen2_0_5b_decode_layer_21_decode(model, token_index, aligned_embed_dim, aligned_head_dim, aligned_intermediate_dim, aligned_context_window);
4275  qwen2_0_5b_decode_layer_22_decode(model, token_index, aligned_embed_dim, aligned_head_dim, aligned_intermediate_dim, aligned_context_window);
4276  qwen2_0_5b_decode_layer_23_decode(model, token_index, aligned_embed_dim, aligned_head_dim, aligned_intermediate_dim, aligned_context_window);
4277 
4278  /* Final RMSNorm */
4279  float *last_hidden = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_LAYERS[23].output);
4280  float *final_ln_weight = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_FOOTER.final_ln_weight);
4281  float *final_out = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_FOOTER.final_output);
4282  rmsnorm_forward(last_hidden,
4283  final_ln_weight,
4284  final_out,
4285  NULL,
4286  1,
4287  QWEN2_0_5B_DECODE_EMBED_DIM,
4288  aligned_embed_dim,
4289  1e-06f);
4290 
4291  /* LM head projection */
4292  float *logits = QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_FOOTER.logits);
4293  const void *lm_head = (const void *)QWEN2_0_5B_DECODE_PTR(model, QWEN2_0_5B_DECODE_FOOTER.lm_head_weight);
4294  /* LM head: Q8_0 -> gemm_nt_q8_0 */
4295  gemm_nt_q8_0(final_out, lm_head, NULL, logits, 1, QWEN2_0_5B_DECODE_VOCAB_SIZE, aligned_embed_dim);
4296 }
4297 
4298 /* ============================================================================
4299  * PUBLIC API
4300  * ============================================================================ */
4301 
4302 void qwen2_0_5b_decode_forward(
4303  QWEN2_0_5B_DECODEModel *model,
4304  const int *tokens,
4305  int num_tokens
4306 ) {
4307  if (!model || !tokens || num_tokens <= 0) return;
4308  qwen2_0_5b_decode_forward_prefill_impl(model, tokens, num_tokens);
4309 }
4310 
4311 void qwen2_0_5b_decode_decode(QWEN2_0_5B_DECODEModel *model, const int *token, int token_index) {
4312  qwen2_0_5b_decode_decode_token(model, token, token_index);
4313 }
ckernel_engine.h
swiglu_forward
void swiglu_forward(const float *input, float *output, int tokens, int dim)
Definition: swiglu_kernels.c:131
gemm_nt_q4_k
void gemm_nt_q4_k(const float *A, const void *B, const float *bias, float *C, int M, int N, int K)
Definition: gemm_kernels_q4k.c:683
kv_cache_repack_head_major_inplace
void kv_cache_repack_head_major_inplace(float *buf, int num_heads, int tokens, int cache_capacity, int aligned_head_dim)
Definition: kv_cache_kernels.c:28
kv_cache_write_head_major
void kv_cache_write_head_major(const float *__restrict k_token, const float *__restrict v_token, float *__restrict k_cache, float *__restrict v_cache, int num_kv_heads, int token_index, int cache_capacity, int head_dim, int aligned_head_dim)
Definition: kv_cache_kernels.c:60
gemm_nt_q5_0
void gemm_nt_q5_0(const float *A, const void *B, const float *bias, float *C, int M, int N, int K)
Definition: gemm_kernels_q5_0.c:831
gemm_nt_q6_k
void gemm_nt_q6_k(const float *A, const void *B, const float *bias, float *C, int M, int N, int K)
Definition: gemm_kernels_q6k.c:212
gemm_nt_q8_0
void gemm_nt_q8_0(const float *A, const void *B, const float *bias, float *C, int M, int N, int K)
Matrix-matrix multiply: C[M,N] = A[M,K] @ B[N,K]^T + bias.
Definition: gemm_kernels_q8_0.c:681
attention_forward_decode_head_major_gqa_regular
void attention_forward_decode_head_major_gqa_regular(const float *q_token, const float *k_cache, const float *v_cache, float *out_token, int num_heads, int num_kv_heads, int kv_tokens, int cache_capacity, int head_dim, int aligned_head_dim)
WARNING: This is NOT true flash attention!
Definition: attention_kernels.c:1524
rmsnorm_forward
void rmsnorm_forward(const float *input, const float *gamma, float *output, float *rstd_cache, int tokens, int d_model, int aligned_embed_dim, float eps)
Definition: rmsnorm_kernels.c:50
rope_forward_qk
void rope_forward_qk(float *q, float *k, const float *cos_cache, const float *sin_cache, int num_heads, int num_kv_heads, int num_tokens, int head_dim, int aligned_head_dim, int pos_offset)
Definition: rope_kernels.c:448
embedding_forward_q8_0
void embedding_forward_q8_0(const int32_t *token_ids, int token_count, int vocab_size, const void *token_embeddings, const float *pos_embeddings, float *output, int embed_dim, int aligned_embed_dim, int context_window, int add_pos)
Definition: embedding_kernels.c:131
QWEN2_0_5B_DECODECanary::offset
size_t offset
Definition: v6.5/test_generated/ck-kernel-inference.h:687
QWEN2_0_5B_DECODEFooterOffsets::logits
size_t logits
Definition: v6.5/test_generated/ck-kernel-inference.h:658
QWEN2_0_5B_DECODEFooterOffsets::final_output
size_t final_output
Definition: v6.5/test_generated/ck-kernel-inference.h:656
QWEN2_0_5B_DECODEFooterOffsets::lm_head_weight
size_t lm_head_weight
Definition: v6.5/test_generated/ck-kernel-inference.h:657
QWEN2_0_5B_DECODEFooterOffsets::final_ln_weight
size_t final_ln_weight
Definition: v6.5/test_generated/ck-kernel-inference.h:655
QWEN2_0_5B_DECODEGlobalOffsets::rope_cos_cache
size_t rope_cos_cache
Definition: v6.5/test_generated/ck-kernel-inference.h:673
QWEN2_0_5B_DECODEGlobalOffsets::rope_sin_cache
size_t rope_sin_cache
Definition: v6.5/test_generated/ck-kernel-inference.h:674
QWEN2_0_5B_DECODEHeaderOffsets::token_emb
size_t token_emb
Definition: v6.5/test_generated/ck-kernel-inference.h:50
QWEN2_0_5B_DECODEHeaderOffsets::embedded_input
size_t embedded_input
Definition: v6.5/test_generated/ck-kernel-inference.h:54
QWEN2_0_5B_DECODELayerOffsets
Definition: v6.5/test_generated/ck-kernel-inference.h:69
QWEN2_0_5B_DECODELayerOffsets::ln2_gamma
size_t ln2_gamma
Definition: v6.5/test_generated/ck-kernel-inference.h:85
QWEN2_0_5B_DECODELayerOffsets::bk
size_t bk
Definition: v6.5/test_generated/ck-kernel-inference.h:75
QWEN2_0_5B_DECODELayerOffsets::w1
size_t w1
Definition: v6.5/test_generated/ck-kernel-inference.h:87
QWEN2_0_5B_DECODELayerOffsets::proj_tmp
size_t proj_tmp
Definition: v6.5/test_generated/ck-kernel-inference.h:82
QWEN2_0_5B_DECODELayerOffsets::ln1_out
size_t ln1_out
Definition: v6.5/test_generated/ck-kernel-inference.h:71
QWEN2_0_5B_DECODELayerOffsets::mlp_out
size_t mlp_out
Definition: v6.5/test_generated/ck-kernel-inference.h:89
QWEN2_0_5B_DECODELayerOffsets::k
size_t k
Definition: v6.5/test_generated/ck-kernel-inference.h:79
QWEN2_0_5B_DECODELayerOffsets::wv
size_t wv
Definition: v6.5/test_generated/ck-kernel-inference.h:76
QWEN2_0_5B_DECODELayerOffsets::ln2_out
size_t ln2_out
Definition: v6.5/test_generated/ck-kernel-inference.h:86
QWEN2_0_5B_DECODELayerOffsets::ln1_gamma
size_t ln1_gamma
Definition: v6.5/test_generated/ck-kernel-inference.h:70
QWEN2_0_5B_DECODELayerOffsets::residual1
size_t residual1
Definition: v6.5/test_generated/ck-kernel-inference.h:84
QWEN2_0_5B_DECODELayerOffsets::v
size_t v
Definition: v6.5/test_generated/ck-kernel-inference.h:80
QWEN2_0_5B_DECODELayerOffsets::wk
size_t wk
Definition: v6.5/test_generated/ck-kernel-inference.h:74
QWEN2_0_5B_DECODELayerOffsets::wq
size_t wq
Definition: v6.5/test_generated/ck-kernel-inference.h:72
QWEN2_0_5B_DECODELayerOffsets::wo
size_t wo
Definition: v6.5/test_generated/ck-kernel-inference.h:81
QWEN2_0_5B_DECODELayerOffsets::output
size_t output
Definition: v6.5/test_generated/ck-kernel-inference.h:90
QWEN2_0_5B_DECODELayerOffsets::bv
size_t bv
Definition: v6.5/test_generated/ck-kernel-inference.h:77
QWEN2_0_5B_DECODELayerOffsets::proj_scratch
size_t proj_scratch
Definition: v6.5/test_generated/ck-kernel-inference.h:83
QWEN2_0_5B_DECODELayerOffsets::w2
size_t w2
Definition: v6.5/test_generated/ck-kernel-inference.h:88
QWEN2_0_5B_DECODELayerOffsets::bq
size_t bq
Definition: v6.5/test_generated/ck-kernel-inference.h:73
QWEN2_0_5B_DECODEModel
Definition: v6.5/test_generated/ck-kernel-inference.h:1265
QWEN2_0_5B_DECODEModel::base
void * base
Definition: v6.5/test_generated/ck-kernel-inference.h:1266
QWEN2_0_5B_DECODEModel::total_bytes
size_t total_bytes
Definition: v6.5/test_generated/ck-kernel-inference.h:1267
token
const char * token
Definition: tokenizer.h:306
vocab_size
int vocab_size
Definition: true_bpe.h:185
qwen2_0_5b_decode_layer_18_prefill
static void qwen2_0_5b_decode_layer_18_prefill(QWEN2_0_5B_DECODEModel *model, int num_tokens, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
Definition: v6.5/test_generated/ck-kernel-inference.c:3109
qwen2_0_5b_decode_layer_2_prefill
static void qwen2_0_5b_decode_layer_2_prefill(QWEN2_0_5B_DECODEModel *model, int num_tokens, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
Definition: v6.5/test_generated/ck-kernel-inference.c:533
qwen2_0_5b_decode_layer_1_prefill
static void qwen2_0_5b_decode_layer_1_prefill(QWEN2_0_5B_DECODEModel *model, int num_tokens, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
Definition: v6.5/test_generated/ck-kernel-inference.c:372
qwen2_0_5b_decode_layer_8_prefill
static void qwen2_0_5b_decode_layer_8_prefill(QWEN2_0_5B_DECODEModel *model, int num_tokens, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
Definition: v6.5/test_generated/ck-kernel-inference.c:1499
qwen2_0_5b_decode_layer_20_prefill
static void qwen2_0_5b_decode_layer_20_prefill(QWEN2_0_5B_DECODEModel *model, int num_tokens, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
Definition: v6.5/test_generated/ck-kernel-inference.c:3431
qwen2_0_5b_decode_layer_0_prefill
static void qwen2_0_5b_decode_layer_0_prefill(QWEN2_0_5B_DECODEModel *model, int num_tokens, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
Definition: v6.5/test_generated/ck-kernel-inference.c:211
qwen2_0_5b_decode_layer_7_prefill
static void qwen2_0_5b_decode_layer_7_prefill(QWEN2_0_5B_DECODEModel *model, int num_tokens, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
Definition: v6.5/test_generated/ck-kernel-inference.c:1338
qwen2_0_5b_decode_layer_6_prefill
static void qwen2_0_5b_decode_layer_6_prefill(QWEN2_0_5B_DECODEModel *model, int num_tokens, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
Definition: v6.5/test_generated/ck-kernel-inference.c:1177
qwen2_0_5b_decode_layer_19_prefill
static void qwen2_0_5b_decode_layer_19_prefill(QWEN2_0_5B_DECODEModel *model, int num_tokens, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
Definition: v6.5/test_generated/ck-kernel-inference.c:3270
qwen2_0_5b_decode_layer_15_prefill
static void qwen2_0_5b_decode_layer_15_prefill(QWEN2_0_5B_DECODEModel *model, int num_tokens, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
Definition: v6.5/test_generated/ck-kernel-inference.c:2626
qwen2_0_5b_decode_layer_17_prefill
static void qwen2_0_5b_decode_layer_17_prefill(QWEN2_0_5B_DECODEModel *model, int num_tokens, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
Definition: v6.5/test_generated/ck-kernel-inference.c:2948
qwen2_0_5b_decode_layer_22_prefill
static void qwen2_0_5b_decode_layer_22_prefill(QWEN2_0_5B_DECODEModel *model, int num_tokens, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
Definition: v6.5/test_generated/ck-kernel-inference.c:3753
qwen2_0_5b_decode_layer_3_prefill
static void qwen2_0_5b_decode_layer_3_prefill(QWEN2_0_5B_DECODEModel *model, int num_tokens, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
Definition: v6.5/test_generated/ck-kernel-inference.c:694
qwen2_0_5b_decode_layer_13_prefill
static void qwen2_0_5b_decode_layer_13_prefill(QWEN2_0_5B_DECODEModel *model, int num_tokens, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
Definition: v6.5/test_generated/ck-kernel-inference.c:2304
qwen2_0_5b_decode_layer_21_prefill
static void qwen2_0_5b_decode_layer_21_prefill(QWEN2_0_5B_DECODEModel *model, int num_tokens, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
Definition: v6.5/test_generated/ck-kernel-inference.c:3592
qwen2_0_5b_decode_layer_4_prefill
static void qwen2_0_5b_decode_layer_4_prefill(QWEN2_0_5B_DECODEModel *model, int num_tokens, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
Definition: v6.5/test_generated/ck-kernel-inference.c:855
qwen2_0_5b_decode_layer_12_prefill
static void qwen2_0_5b_decode_layer_12_prefill(QWEN2_0_5B_DECODEModel *model, int num_tokens, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
Definition: v6.5/test_generated/ck-kernel-inference.c:2143
qwen2_0_5b_decode_layer_16_prefill
static void qwen2_0_5b_decode_layer_16_prefill(QWEN2_0_5B_DECODEModel *model, int num_tokens, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
Definition: v6.5/test_generated/ck-kernel-inference.c:2787
qwen2_0_5b_decode_layer_5_prefill
static void qwen2_0_5b_decode_layer_5_prefill(QWEN2_0_5B_DECODEModel *model, int num_tokens, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
Definition: v6.5/test_generated/ck-kernel-inference.c:1016
qwen2_0_5b_decode_layer_11_prefill
static void qwen2_0_5b_decode_layer_11_prefill(QWEN2_0_5B_DECODEModel *model, int num_tokens, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
Definition: v6.5/test_generated/ck-kernel-inference.c:1982
qwen2_0_5b_decode_layer_14_prefill
static void qwen2_0_5b_decode_layer_14_prefill(QWEN2_0_5B_DECODEModel *model, int num_tokens, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
Definition: v6.5/test_generated/ck-kernel-inference.c:2465
qwen2_0_5b_decode_layer_10_prefill
static void qwen2_0_5b_decode_layer_10_prefill(QWEN2_0_5B_DECODEModel *model, int num_tokens, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
Definition: v6.5/test_generated/ck-kernel-inference.c:1821
qwen2_0_5b_decode_layer_9_prefill
static void qwen2_0_5b_decode_layer_9_prefill(QWEN2_0_5B_DECODEModel *model, int num_tokens, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
Definition: v6.5/test_generated/ck-kernel-inference.c:1660
qwen2_0_5b_decode_layer_23_prefill
static void qwen2_0_5b_decode_layer_23_prefill(QWEN2_0_5B_DECODEModel *model, int num_tokens, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
Definition: v6.5/test_generated/ck-kernel-inference.c:3914
QWEN2_0_5B_DECODE_TOTAL_BYTES
#define QWEN2_0_5B_DECODE_TOTAL_BYTES
Definition: v6.5/test_generated/ck-kernel-inference.h:37
QWEN2_0_5B_DECODE_HEAD_DIM
#define QWEN2_0_5B_DECODE_HEAD_DIM
Definition: v6.5/test_generated/ck-kernel-inference.h:28
QWEN2_0_5B_DECODE_PTR
#define QWEN2_0_5B_DECODE_PTR(model, offset)
Definition: v6.5/test_generated/ck-kernel-inference.h:1274
QWEN2_0_5B_DECODE_ACTIVATION_BYTES
#define QWEN2_0_5B_DECODE_ACTIVATION_BYTES
Definition: v6.5/test_generated/ck-kernel-inference.h:39
QWEN2_0_5B_DECODE_FOOTER
static const QWEN2_0_5B_DECODEFooterOffsets QWEN2_0_5B_DECODE_FOOTER
Definition: v6.5/test_generated/ck-kernel-inference.h:661
QWEN2_0_5B_DECODE_LAYERS
static const QWEN2_0_5B_DECODELayerOffsets QWEN2_0_5B_DECODE_LAYERS[24]
Definition: v6.5/test_generated/ck-kernel-inference.h:93
QWEN2_0_5B_DECODE_DTYPE_BYTES
#define QWEN2_0_5B_DECODE_DTYPE_BYTES
Definition: v6.5/test_generated/ck-kernel-inference.h:35
QWEN2_0_5B_DECODE_EMBED_DIM
#define QWEN2_0_5B_DECODE_EMBED_DIM
Definition: v6.5/test_generated/ck-kernel-inference.h:25
QWEN2_0_5B_DECODE_MAGIC
#define QWEN2_0_5B_DECODE_MAGIC
Definition: v6.5/test_generated/ck-kernel-inference.h:41
QWEN2_0_5B_DECODE_CANARY_COUNT
#define QWEN2_0_5B_DECODE_CANARY_COUNT
Definition: v6.5/test_generated/ck-kernel-inference.h:1259
QWEN2_0_5B_DECODE_MAX_SEQ_LEN
#define QWEN2_0_5B_DECODE_MAX_SEQ_LEN
Definition: v6.5/test_generated/ck-kernel-inference.h:32
QWEN2_0_5B_DECODE_NUM_LAYERS
#define QWEN2_0_5B_DECODE_NUM_LAYERS
Definition: v6.5/test_generated/ck-kernel-inference.h:30
QWEN2_0_5B_DECODE_WEIGHT_BYTES
#define QWEN2_0_5B_DECODE_WEIGHT_BYTES
Definition: v6.5/test_generated/ck-kernel-inference.h:38
QWEN2_0_5B_DECODE_CANARY_VALUE
#define QWEN2_0_5B_DECODE_CANARY_VALUE
Definition: v6.5/test_generated/ck-kernel-inference.h:42
QWEN2_0_5B_DECODE_GLOBALS
static const QWEN2_0_5B_DECODEGlobalOffsets QWEN2_0_5B_DECODE_GLOBALS
Definition: v6.5/test_generated/ck-kernel-inference.h:677
QWEN2_0_5B_DECODE_NUM_KV_HEADS
#define QWEN2_0_5B_DECODE_NUM_KV_HEADS
Definition: v6.5/test_generated/ck-kernel-inference.h:27
QWEN2_0_5B_DECODE_NUM_HEADS
#define QWEN2_0_5B_DECODE_NUM_HEADS
Definition: v6.5/test_generated/ck-kernel-inference.h:26
QWEN2_0_5B_DECODE_CANARY_SIZE
#define QWEN2_0_5B_DECODE_CANARY_SIZE
Definition: v6.5/test_generated/ck-kernel-inference.h:43
QWEN2_0_5B_DECODE_VOCAB_SIZE
#define QWEN2_0_5B_DECODE_VOCAB_SIZE
Definition: v6.5/test_generated/ck-kernel-inference.h:31
QWEN2_0_5B_DECODE_HEADER
static const QWEN2_0_5B_DECODEHeaderOffsets QWEN2_0_5B_DECODE_HEADER
Definition: v6.5/test_generated/ck-kernel-inference.h:57
QWEN2_0_5B_DECODE_CANARIES
static const QWEN2_0_5B_DECODECanary QWEN2_0_5B_DECODE_CANARIES[]
Definition: v6.5/test_generated/ck-kernel-inference.h:691
qwen2_0_5b_decode_layer_3_decode
static void qwen2_0_5b_decode_layer_3_decode(QWEN2_0_5B_DECODEModel *model, int token_index, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
Definition: v6.6/generated/ck-kernel-inference.c:1240
qwen2_0_5b_decode_layer_18_decode
static void qwen2_0_5b_decode_layer_18_decode(QWEN2_0_5B_DECODEModel *model, int token_index, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
Definition: v6.6/generated/ck-kernel-inference.c:3370
qwen2_0_5b_decode_layer_16_decode
static void qwen2_0_5b_decode_layer_16_decode(QWEN2_0_5B_DECODEModel *model, int token_index, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
Definition: v6.6/generated/ck-kernel-inference.c:3086
qwen2_0_5b_decode_precompute_rope
void qwen2_0_5b_decode_precompute_rope(QWEN2_0_5B_DECODEModel *model)
Definition: v6.6/generated/ck-kernel-inference.c:222
qwen2_0_5b_decode_verify_canaries
int qwen2_0_5b_decode_verify_canaries(QWEN2_0_5B_DECODEModel *model)
Definition: v6.6/generated/ck-kernel-inference.c:188
qwen2_0_5b_decode_residual_add_token_major
static void qwen2_0_5b_decode_residual_add_token_major(const float *a, const float *b, float *out, int tokens, int aligned_embed_dim)
Definition: v6.6/generated/ck-kernel-inference.c:79
qwen2_0_5b_decode_layer_4_decode
static void qwen2_0_5b_decode_layer_4_decode(QWEN2_0_5B_DECODEModel *model, int token_index, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
Definition: v6.6/generated/ck-kernel-inference.c:1382
qwen2_0_5b_decode_align_elems
static int qwen2_0_5b_decode_align_elems(int elems, int elem_bytes, int align_bytes)
Definition: v6.6/generated/ck-kernel-inference.c:212
qwen2_0_5b_decode_layer_8_decode
static void qwen2_0_5b_decode_layer_8_decode(QWEN2_0_5B_DECODEModel *model, int token_index, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
Definition: v6.6/generated/ck-kernel-inference.c:1950
qwen2_0_5b_decode_layer_5_decode
static void qwen2_0_5b_decode_layer_5_decode(QWEN2_0_5B_DECODEModel *model, int token_index, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
Definition: v6.6/generated/ck-kernel-inference.c:1524
qwen2_0_5b_decode_layer_13_decode
static void qwen2_0_5b_decode_layer_13_decode(QWEN2_0_5B_DECODEModel *model, int token_index, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
Definition: v6.6/generated/ck-kernel-inference.c:2660
qwen2_0_5b_decode_model_allocate
int qwen2_0_5b_decode_model_allocate(QWEN2_0_5B_DECODEModel *model)
Definition: v6.6/generated/ck-kernel-inference.c:124
qwen2_0_5b_decode_layer_0_decode
static void qwen2_0_5b_decode_layer_0_decode(QWEN2_0_5B_DECODEModel *model, int token_index, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
Definition: v6.6/generated/ck-kernel-inference.c:814
qwen2_0_5b_decode_layer_19_decode
static void qwen2_0_5b_decode_layer_19_decode(QWEN2_0_5B_DECODEModel *model, int token_index, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
Definition: v6.6/generated/ck-kernel-inference.c:3512
qwen2_0_5b_decode_layer_6_decode
static void qwen2_0_5b_decode_layer_6_decode(QWEN2_0_5B_DECODEModel *model, int token_index, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
Definition: v6.6/generated/ck-kernel-inference.c:1666
qwen2_0_5b_decode_layer_7_decode
static void qwen2_0_5b_decode_layer_7_decode(QWEN2_0_5B_DECODEModel *model, int token_index, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
Definition: v6.6/generated/ck-kernel-inference.c:1808
qwen2_0_5b_decode_decode
void qwen2_0_5b_decode_decode(QWEN2_0_5B_DECODEModel *model, const int *token, int token_index)
Definition: v6.6/generated/ck-kernel-inference.c:4311
qwen2_0_5b_decode_layer_2_decode
static void qwen2_0_5b_decode_layer_2_decode(QWEN2_0_5B_DECODEModel *model, int token_index, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
Definition: v6.6/generated/ck-kernel-inference.c:1098
qwen2_0_5b_decode_forward_prefill_impl
static void qwen2_0_5b_decode_forward_prefill_impl(QWEN2_0_5B_DECODEModel *model, const int *tokens, int num_tokens)
Definition: v6.6/generated/ck-kernel-inference.c:248
qwen2_0_5b_decode_layer_9_decode
static void qwen2_0_5b_decode_layer_9_decode(QWEN2_0_5B_DECODEModel *model, int token_index, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
Definition: v6.6/generated/ck-kernel-inference.c:2092
qwen2_0_5b_decode_layer_10_decode
static void qwen2_0_5b_decode_layer_10_decode(QWEN2_0_5B_DECODEModel *model, int token_index, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
Definition: v6.6/generated/ck-kernel-inference.c:2234
qwen2_0_5b_decode_decode_token
static void qwen2_0_5b_decode_decode_token(QWEN2_0_5B_DECODEModel *model, const int *token, int token_index)
Definition: v6.6/generated/ck-kernel-inference.c:4223
qwen2_0_5b_decode_layer_21_decode
static void qwen2_0_5b_decode_layer_21_decode(QWEN2_0_5B_DECODEModel *model, int token_index, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
Definition: v6.6/generated/ck-kernel-inference.c:3796
qwen2_0_5b_decode_layer_22_decode
static void qwen2_0_5b_decode_layer_22_decode(QWEN2_0_5B_DECODEModel *model, int token_index, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
Definition: v6.6/generated/ck-kernel-inference.c:3938
qwen2_0_5b_decode_layer_11_decode
static void qwen2_0_5b_decode_layer_11_decode(QWEN2_0_5B_DECODEModel *model, int token_index, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
Definition: v6.6/generated/ck-kernel-inference.c:2376
qwen2_0_5b_decode_layer_23_decode
static void qwen2_0_5b_decode_layer_23_decode(QWEN2_0_5B_DECODEModel *model, int token_index, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
Definition: v6.6/generated/ck-kernel-inference.c:4080
__attribute__
struct __attribute__((packed))
Definition: v6.6/generated/ck-kernel-inference.c:103
qwen2_0_5b_decode_layer_1_decode
static void qwen2_0_5b_decode_layer_1_decode(QWEN2_0_5B_DECODEModel *model, int token_index, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
Definition: v6.6/generated/ck-kernel-inference.c:956
qwen2_0_5b_decode_forward
void qwen2_0_5b_decode_forward(QWEN2_0_5B_DECODEModel *model, const int *tokens, int num_tokens)
Definition: v6.6/generated/ck-kernel-inference.c:4302
qwen2_0_5b_decode_layer_14_decode
static void qwen2_0_5b_decode_layer_14_decode(QWEN2_0_5B_DECODEModel *model, int token_index, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
Definition: v6.6/generated/ck-kernel-inference.c:2802
qwen2_0_5b_decode_layer_15_decode
static void qwen2_0_5b_decode_layer_15_decode(QWEN2_0_5B_DECODEModel *model, int token_index, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
Definition: v6.6/generated/ck-kernel-inference.c:2944
qwen2_0_5b_decode_layer_12_decode
static void qwen2_0_5b_decode_layer_12_decode(QWEN2_0_5B_DECODEModel *model, int token_index, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
Definition: v6.6/generated/ck-kernel-inference.c:2518
qwen2_0_5b_decode_model_free
void qwen2_0_5b_decode_model_free(QWEN2_0_5B_DECODEModel *model)
Definition: v6.6/generated/ck-kernel-inference.c:177
qwen2_0_5b_decode_layer_20_decode
static void qwen2_0_5b_decode_layer_20_decode(QWEN2_0_5B_DECODEModel *model, int token_index, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
Definition: v6.6/generated/ck-kernel-inference.c:3654
_Static_assert
_Static_assert(sizeof(MagicHeader)==64, "MagicHeader must be 64 bytes")
qwen2_0_5b_decode_layer_17_decode
static void qwen2_0_5b_decode_layer_17_decode(QWEN2_0_5B_DECODEModel *model, int token_index, int aligned_embed_dim, int aligned_head_dim, int aligned_intermediate_dim, int aligned_context_window)
Definition: v6.6/generated/ck-kernel-inference.c:3228
ck-kernel-inference.h
AUTO-GENERATED: qwen2_0.5b_decode Memory Layout.