4.1 Testing layers against references
Chapter 1 checked each layer once, on one
input. Real tests check many shapes against an independent reference, with
metrics that tell a rounding difference from a bug. This section builds the
pattern MAX uses for its own layers, in
max/python/layer_benchmarks,
and applies it to your decoder layer, with Hugging Face’s PyTorch Llama as the
reference.
import logging
import warnings
import numpy as np
import plotly.graph_objects as go
import torch
from max import functional as F
from max.driver import CPU
from max.dtype import DType
from max.tensor import Tensor, default_device, default_dtype
from max.tensor.tensor import TensorType
from transformers import AutoConfig
from transformers.models.llama.modeling_llama import (
LlamaDecoderLayer,
LlamaRotaryEmbedding,
)
from minillm import viz
from minillm.device import pick_device
from minillm.config import MODEL_ID, LlamaConfig
from minillm.model import DecoderLayer
from minillm.weights import load_checkpoint, select
logging.getLogger("max").setLevel(logging.ERROR)
warnings.filterwarnings("ignore", message="The eager interpreter failed")
DEVICE = pick_device() # a GPU if MAX finds one, otherwise the CPU
config = LlamaConfig.from_pretrained()
layer0 = select(load_checkpoint(), "model.layers.0.")
How to compare outputs
One number is not enough. MAX’s test harness reports three:
- The maximum absolute error catches any single element that is far off.
- The mean absolute error is the typical error, which tells a few outliers from a systematic drift.
- The cosine distance, \(1 - \cos\theta\) between the flattened outputs, is near 0 when the outputs point the same way even if their scale differs slightly, and large for a structural bug such as a wrong head layout.
A test passes when the outputs are allclose, within an absolute and a
relative tolerance, and the cosine distance is below a threshold.
from __future__ import annotations
from dataclasses import dataclass
import numpy as np
@dataclass
class CorrectnessResult:
"""How far an output is from its reference, as MAX's layer tests report it."""
label: str
mae: float
max_abs_err: float
cos_dist: float
passed: bool
def __str__(self) -> str:
status = "pass" if self.passed else "FAIL"
return (
f"{self.label:16} mae {self.mae:.1e} max {self.max_abs_err:.1e} "
f"cos dist {self.cos_dist:.1e} {status}"
)
def compare_outputs(
actual: np.ndarray,
expected: np.ndarray,
label: str = "",
atol: float = 1e-3,
rtol: float = 1e-3,
cos_threshold: float = 1e-5,
) -> CorrectnessResult:
"""Compares ``actual`` with ``expected``; passes when they are allclose
and their cosine distance is below ``cos_threshold``."""
a = actual.astype(np.float64).ravel()
e = expected.astype(np.float64).ravel()
cos_dist = 1.0 - float(
a @ e / (np.linalg.norm(a) * np.linalg.norm(e) + 1e-12)
)
allclose = bool(np.allclose(actual, expected, atol=atol, rtol=rtol))
return CorrectnessResult(
label=label,
mae=float(np.mean(np.abs(a - e))),
max_abs_err=float(np.max(np.abs(a - e))),
cos_dist=cos_dist,
passed=allclose and cos_dist < cos_threshold,
)
Saved to minillm/testing.py.
The reference
Hugging Face’s LlamaDecoderLayer is an independent implementation of the
same layer, in PyTorch. Its parameter names match yours, so the same
checkpoint entries load into both. "eager" selects its plain attention
code instead of a fused kernel, so the reference stays simple.
hf_config = AutoConfig.from_pretrained(MODEL_ID)
hf_config._attn_implementation = "eager"
reference = LlamaDecoderLayer(hf_config, layer_idx=0).float().eval()
reference.load_state_dict(
{name: torch.from_numpy(w.copy()) for name, w in layer0.items()}
)
rotary = LlamaRotaryEmbedding(hf_config)
def reference_layer(x: np.ndarray) -> np.ndarray:
batch, seq, _ = x.shape
hidden = torch.from_numpy(x)
positions = torch.arange(seq).expand(batch, seq)
mask = torch.full((seq, seq), float("-inf")).triu(1)[None, None]
with torch.no_grad():
out = reference(
hidden,
attention_mask=mask,
position_ids=positions,
position_embeddings=rotary(hidden, positions),
)
return out.numpy()
Compile once, test many shapes
MAX’s harness separates static parameters, which fix the graph (here, the layer’s sizes), from dynamic ones, which vary per run (batch size and sequence length). Compile once with symbolic dimensions, then run every shape through the same compiled layer. The shapes cover the edges: a single token (a decode step), odd lengths, a batch, and a long prefill.
def compiled_layer(layer_class=DecoderLayer, weights=layer0, cfg=config):
with F.lazy(), default_device(DEVICE), default_dtype(DType.float32):
layer = layer_class(cfg)
layer.load_state_dict(weights)
return layer.compile(
TensorType(
DType.float32, ("batch", "seq_len", cfg.hidden_size), device=DEVICE
)
)
SHAPES = [(1, 1), (1, 7), (2, 33), (1, 512)]
def check(run, shapes=SHAPES) -> list:
rng = np.random.default_rng(0)
results = []
for batch, seq in shapes:
x = rng.standard_normal((batch, seq, config.hidden_size)).astype(
np.float32
)
actual = run(Tensor(x, device=DEVICE)).to_numpy()
results.append(
compare_outputs(
actual, reference_layer(x), label=f"batch {batch}, seq {seq}"
)
)
return results
correct = check(compiled_layer())
for result in correct:
print(result)
assert all(r.passed for r in correct)
batch 1, seq 1 mae 2.7e-07 max 2.9e-06 cos dist 2.4e-14 pass
batch 1, seq 7 mae 6.5e-07 max 1.7e-05 cos dist 1.9e-13 pass
batch 2, seq 33 mae 7.0e-07 max 2.1e-05 cos dist 1.9e-13 pass
batch 1, seq 512 mae 7.1e-07 max 3.8e-05 cos dist 2.0e-13 pass
Every shape passes, with maximum errors of 4e-5 or less and cosine
distances near zero.
A test must be able to fail
A test that never fails proves nothing, so check that this one catches a real
bug. A plausible one is a config read wrong, with the base of RoPE
(Section 1.4) at Llama 2’s
10000 instead of SmolLM2’s 100000:
from dataclasses import replace
buggy = compiled_layer(cfg=replace(config, rope_theta=10000.0))
buggy_results = check(buggy)
for result in buggy_results:
print(result)
assert not all(r.passed for r in buggy_results)
batch 1, seq 1 mae 2.7e-07 max 2.9e-06 cos dist 2.4e-14 pass
batch 1, seq 7 mae 4.9e-04 max 6.6e-03 cos dist 1.2e-07 FAIL
batch 2, seq 33 mae 2.2e-03 max 1.3e-01 cos dist 2.3e-06 FAIL
batch 1, seq 512 mae 5.0e-03 max 2.3e-01 cos dist 1.1e-05 FAIL
labels = [r.label for r in correct]
fig = go.Figure(
[
go.Bar(
x=labels,
y=[r.max_abs_err for r in results],
name=name,
marker_color=color,
hovertemplate="%{x}: max error %{y:.1e}<extra>" + name + "</extra>",
)
for name, results, color in [
("correct layer", correct, viz.BLUE),
("rope_theta = 10000", buggy_results, viz.ORANGE),
]
]
)
fig.update_layout(
title="Max absolute error against Hugging Face, per shape",
barmode="group",
height=400,
yaxis=dict(title="max |error| (log scale)", type="log", dtick=1),
)
viz.show(fig)
The single-token case still passes: at position 0, RoPE doesn’t rotate at
all, so the bug can’t show there. Every longer input fails. The maximum
error grows by two to four orders of magnitude and breaks the allclose
check. The cosine distance grows by six to eight orders of magnitude, but it
crosses its 1e-5 threshold only on the 512-token input. The two checks
catch different bugs, which is why the test uses both, and why the shape
list must include sequences, not just one token.
Where in the sequence does the bug show? Run the 512-token input through both layers and plot the largest error at each position:
x = (
np.random.default_rng(1)
.standard_normal((1, 512, config.hidden_size))
.astype(np.float32)
)
expected = reference_layer(x)[0]
per_position = {
name: np.abs(run(Tensor(x, device=DEVICE)).to_numpy()[0] - expected).max(-1)
for name, run in [
("correct layer", compiled_layer()),
("rope_theta = 10000", buggy),
]
}
fig = go.Figure(
[
go.Scatter(
x=np.arange(512),
y=np.maximum(err, 1e-9),
name=name,
mode="lines",
line_color=color,
hovertemplate="position %{x}: %{y:.1e}<extra>" + name + "</extra>",
)
for (name, err), color in zip(
per_position.items(), [viz.BLUE, viz.ORANGE]
)
]
)
fig.update_layout(
title="Largest error at each position of a 512-token input",
height=400,
xaxis_title="position",
yaxis=dict(title="max |error| (log scale)", type="log", dtick=1),
)
viz.show(fig)
At position 0 the two layers agree, because RoPE doesn’t rotate there.
Within a few positions the buggy layer’s error is three to four orders of
magnitude larger, and it stays there for the whole sequence, while the
correct layer’s error stays near 1e-5. A test that only ran one token
would have passed both.
As a pytest file
In a repository, these checks belong in a test file that runs on every
change. %%writefile saves a cell as a file. The test compiles the layer
once per module, as MAX’s own tests do, and checks every shape:
%%writefile test_decoder_layer.py
import numpy as np
import pytest
import torch
from max.dtype import DType
from max.tensor import Tensor, default_device, default_dtype
from max.tensor.tensor import TensorType
from transformers import AutoConfig
from transformers.models.llama.modeling_llama import LlamaDecoderLayer, LlamaRotaryEmbedding
from minillm.config import MODEL_ID, LlamaConfig
from minillm.device import pick_device
from minillm.model import DecoderLayer
from minillm.testing import compare_outputs
from minillm.weights import load_checkpoint, select
CONFIG = LlamaConfig.from_pretrained()
@pytest.fixture(scope="module")
def layers():
weights = select(load_checkpoint(), "model.layers.0.")
with default_device(pick_device()), default_dtype(DType.float32):
ours = DecoderLayer(CONFIG)
ours.load_state_dict(weights)
compiled = ours.compile(
TensorType(DType.float32, ("batch", "seq_len", CONFIG.hidden_size), device=pick_device())
)
hf_config = AutoConfig.from_pretrained(MODEL_ID)
hf_config._attn_implementation = "eager"
reference = LlamaDecoderLayer(hf_config, layer_idx=0).float().eval()
reference.load_state_dict({k: torch.from_numpy(v.copy()) for k, v in weights.items()})
return compiled, reference, LlamaRotaryEmbedding(hf_config)
@pytest.mark.parametrize("batch, seq", [(1, 1), (1, 7), (2, 33)])
def test_decoder_layer_matches_reference(layers, batch, seq):
compiled, reference, rotary = layers
x = np.random.default_rng(seq).standard_normal((batch, seq, CONFIG.hidden_size)).astype(np.float32)
hidden, positions = torch.from_numpy(x), torch.arange(seq).expand(batch, seq)
mask = torch.full((seq, seq), float("-inf")).triu(1)[None, None]
with torch.no_grad():
expected = reference(hidden, attention_mask=mask, position_ids=positions,
position_embeddings=rotary(hidden, positions)).numpy()
result = compare_outputs(compiled(Tensor(x, device=pick_device())).to_numpy(), expected)
assert result.passed, result
Overwriting test_decoder_layer.py
!python -m pytest -q -p no:cacheprovider test_decoder_layer.py 2>&1 | tail -n 3
... [100%]
3 passed in 4.34s
MAX’s own layer tests follow the same shape, with one more step: each harness also benchmarks the layer on the same shapes, so correctness and speed are measured together. Section 4.3 covers benchmarking.
Exercises
- Add a
bfloat16run: cast the input and the weights, and compare with thefloat32reference. Which tolerances do you need, and which metric moves most? - Test
CachedAttentionfrom Section 1.8: compare a prefill of 8 tokens followed by one decode step with a single 9-token pass. - Plant a different bug, such as swapping
gate_projandup_proj. Which metric catches it first?
Next
Section 4.2 tests the whole model: its logits on real prompts, against the same reference.