Quantization Deep Dive

Memory Bandwidth is the Bottleneck

Modern CPUs can do billions of FLOPs/second, but memory bandwidth is limited:

Matrix-vector multiply (inference) has arithmetic intensity of ~2 FLOPS/byte. We're memory-bound, so smaller weights = faster inference.

FP32 Format (32 bits)

Value = (-1)^S x 2^(E-127) x 1.Mantissa

Example: 0.15625
  Sign: 0 (positive)
  Exponent: 01111100 (124 - 127 = -3)
  Mantissa: 01000000000000000000000 (1.25)
  = 1 x 2^(-3) x 1.25 = 0.15625

Precision: ~7 significant decimal digits (relative error ~1.2e-7)

BF16 Format (16 bits)

Same exponent range as FP32, but reduced precision (7 bits = ~2 decimal digits).

Relative error: ~7.8e-3 (~0.78%). This is why our BF16 kernel tests use 1e-2 tolerance.

Why Group Size Matters

Trade-off: Smaller groups = better precision but more scale storage overhead.

Format Comparison

Region Separation

// Bump allocator layout for quantized inference
typedef struct {
    // Region 1: Quantized weights (read-only after load)
    uint8_t *weights_q4;      // Q4_0 or Q4_K blocks
    size_t weights_size;

    // Region 2: Activations (FP32, read-write)
    float *activations;       // Input/output tensors
    size_t act_size;

    // Region 3: Scratch (FP32, temporary)
    float *scratch;           // Intermediate buffers
    size_t scratch_size;

    // Region 4: Dequantized cache (optional)
    float *dequant_cache;     // Hot weights kept dequantized
    size_t cache_size;
} BumpAllocator;
    

Type-Tagged Tensors

typedef enum {
    DTYPE_FP32,
    DTYPE_FP16,
    DTYPE_BF16,
    DTYPE_Q4_0,
    DTYPE_Q4_1,
    DTYPE_Q5_0,
    DTYPE_Q5_1,
    DTYPE_Q4_K,
    DTYPE_Q6_K,
    DTYPE_Q8_0,
    DTYPE_Q8_K
} DType;

typedef struct {
    void *data;
    size_t shape[4];
    DType dtype;
    // ... other metadata
} Tensor;
    

Cache Line Basics

Cache line size: 64 bytes (typical x86)

Q4_0 block: 18 bytes -> ~3.5 blocks per cache line
Q4_K block: 144 bytes -> 2.25 cache lines per block

Key insight: Scale and weights should be in SAME cache line
when possible for minimal memory traffic.
    

Complete Quantized GEMV Example

// Quantized matrix-vector multiply: y = A_q4 @ x
void gemv_q4_0(float* y, const block_q4_0* A, const float* x,
               int M, int K) {
    const int blocks_per_row = K / 32;

    for (int row = 0; row < M; row++) {
        __m512 acc = _mm512_setzero_ps();

        for (int b = 0; b < blocks_per_row; b++) {
            // Load quantized block (18 bytes from RAM)
            const block_q4_0* block = &A[row * blocks_per_row + b];

            // Dequantize in registers (no memory write!)
            __m512 w_lo = dequant_q4_0_lower(block);  // weights 0-15
            __m512 w_hi = dequant_q4_0_upper(block);  // weights 16-31

            // Load corresponding input (32 floats)
            __m512 x_lo = _mm512_loadu_ps(&x[b * 32]);
            __m512 x_hi = _mm512_loadu_ps(&x[b * 32 + 16]);

            // FMA: accumulate dot product
            acc = _mm512_fmadd_ps(w_lo, x_lo, acc);
            acc = _mm512_fmadd_ps(w_hi, x_hi, acc);
        }

        // Horizontal sum and store single result
        y[row] = _mm512_reduce_add_ps(acc);
    }
}
    

// Dequant INT8 -> BF16 using AVX-512
// This is "wasted" work before AMX compute

__m512i w_int8 = _mm512_loadu_si512(weights);     // 64 INT8 values
__m512 scale = _mm512_set1_ps(group_scale);

// Unpack to 32-bit, convert to float, scale, convert to BF16
__m512i w_lo = _mm512_cvtepi8_epi32(_mm512_extracti32x4_epi32(w_int8, 0));
__m512 w_f32 = _mm512_mul_ps(_mm512_cvtepi32_ps(w_lo), scale);
__m256i w_bf16 = _mm512_cvtneps_pbh(w_f32);  // FP32 -> BF16

// Now feed to AMX-BF16...
    

This dequant step eats cycles. The question is: does memory bandwidth savings outweigh dequant overhead?

Q4_K Structure Definition

// Q4_K block: 256 weights per block

typedef struct {
    ck_half d;             // 2 bytes: super-block scale (FP16)
    ck_half dmin;          // 2 bytes: super-block minimum (FP16)
    uint8_t scales[12];    // 12 bytes: sub-block scales (6-bit each, packed)
    uint8_t qs[128];       // 128 bytes: 256 INT4 weights (4 bits x 256)
} block_q4_K;              // Total: 144 bytes for 256 weights = 4.5 bpw
    

K-Quant Variants

Fused Dequant + Compute

Critical insight: do not dequantize the whole tensor, then matmul. Dequantize-and-compute per block:

for each block (256 weights):
  load (d, dmin, sub-scales)
  unpack 4-bit nibbles
  convert to fp32 and apply scales
  fused multiply-add with fp32 activations
  accumulate into fp32
    

Tensor Structure for Quantized Types

typedef enum {
    CK_DT_FP32,
    CK_DT_FP16,
    CK_DT_BF16,
    CK_DT_Q4_0,    // Simple: 32 weights, 1 scale
    CK_DT_Q4_K,    // K-quant: 256 weights, hierarchical scales
    CK_DT_Q8_0,
} CKDataType;

typedef struct {
    CKDataType dtype;
    uint32_t ne[4];      // Dimensions
    size_t nb[4];        // Strides in bytes
    size_t block_size;   // Weights per block (32 for Q4_0, 256 for Q4_K)
    size_t type_size;    // Bytes per block
    void* data;          // Points into bump allocator
} tensor;
    

Realistic LLM Inference on Xeon

void matmul_q4(float* out, const float* x, const tensor* W) {
    if (batch_size == 1 && seq_len <= 32) {
        // Memory-bound: token generation
        // INT4 weights + dequant + AVX-512 FP32
        // (AMX setup overhead not worth it for small tiles)
        matmul_q4_avx512(out, x, W);
    }
    else {
        // Compute-bound: prefill or batched
        // Dequant to BF16 + AMX-BF16
        dequant_q4_to_bf16(W_bf16, W_q4);  // AVX-512
        amx_matmul_bf16(out, x, W_bf16);   // AMX
    }
}