7.1 Serve your architecture
In Section 1.1 you served
SmolLM2 with MAX’s own implementation. This section serves yours: the
LlamaForCausalLM you wrote in
Section 1.6, behind the same
AutoPipeline API and the same OpenAI-compatible server. MAX loads model code
from outside its own package through custom architectures: a Python package
that describes the model, which max serve imports with
--custom-architectures.
import logging
import subprocess
import time
import urllib.request
import warnings
from max.pipelines import AutoPipeline
from max.pipelines.lib.arch_lookup import import_custom_architectures
from openai import OpenAI
from minillm.config import MODEL_ID
logging.getLogger("max").setLevel(logging.ERROR)
warnings.filterwarnings("ignore", message="The eager interpreter failed")
PROMPT = ["The capital of France is"]
First, the reference: SmolLM2 as MAX implements it, loaded with
AutoPipeline.
reference = AutoPipeline(MODEL_ID, quantization_encoding="float32")
expected = reference(PROMPT, max_new_tokens=16, top_k=1)
reference.close()
expected
['The capital of France is Paris. Paris is the largest city in France and the capital of the French department']
A custom architecture package
A MAX architecture has three parts, all in the package below:
- A config class tells MAX the sizes it needs to plan memory before it loads
anything: the number of layers, key/value heads, and head size. It also
declares the supported encodings; this model computes in
float32. - A pipeline model loads the weights and runs the model. MAX gives it the
checkpoint’s weights and, at each step, the request’s tokens, and it returns
the next-token logits. Here it builds your
LlamaForCausalLMwithload_modelfrom Section 1.6, casts the checkpoint’sbfloat16weights tofloat32, and compiles it. - A
SupportedArchitectureties them together under the name in the checkpoint’sconfig.json, hereLlamaForCausalLM, along with the tokenizer and the weight format.ARCHITECTURESlists it for--custom-architecturesto find.
To keep the package short, this pipeline model reruns the whole sequence at each
step and serves one request at a time (max_batch_size is 1), like the
generation loop in Section 1.7.
MAX’s own architectures use its paged KV cache and continuous batching instead:
they implement a decoder module that reads keys and values from the cache. MAX’s
OLMo 3 architecture is a compact example.
from __future__ import annotations
from collections.abc import Sequence
from dataclasses import dataclass
from typing import Any, ClassVar
import numpy as np
from max.driver import Buffer, Device
from max.dtype import DType
from max.engine import InferenceSession
from max.graph import DeviceRef
from max.graph.weights import Weights, WeightsAdapter, WeightsFormat
from max.nn.kv_cache import KVCacheInputs, KVCacheParams
from max.nn.outputs import ReturnHiddenStates, ReturnLogits
from max.pipelines.context import TextContext
from max.pipelines.lib import (
KVCacheConfig,
MAXModelConfig,
MemoryPlan,
ModelInputs,
ModelOutputs,
PipelineConfig,
PipelineModelWithKVCache,
SupportedArchitecture,
SupportedEncoding,
TextTokenizer,
)
from max.pipelines.lib.interfaces.arch_config import (
ArchConfigWithAttentionKVCache,
)
from max.pipelines.lib.utils import (
parse_state_dict_from_weights,
upper_bounded_default,
)
from max.pipelines.modeling.types import PipelineTask
from max.tensor import Tensor
from max.tensor.tensor import TensorType
from transformers import AutoConfig
from minillm.config import LlamaConfig
from minillm.model import load_model
@dataclass
class MinillmArchConfig(ArchConfigWithAttentionKVCache):
DEFAULT_ENCODING: ClassVar[SupportedEncoding] = "float32"
SUPPORTED_ENCODINGS: ClassVar[set[SupportedEncoding]] = {"float32"}
@property
def num_key_value_heads(self) -> int:
return self.huggingface_config.num_key_value_heads
@property
def head_dim(self) -> int:
hf = self.huggingface_config
return (
getattr(hf, "head_dim", None)
or hf.hidden_size // hf.num_attention_heads
)
@property
def num_layers(self) -> int:
return self.huggingface_config.num_hidden_layers
@classmethod
def calculate_max_seq_len(
cls, huggingface_config: AutoConfig, model_config: MAXModelConfig
) -> int:
return upper_bounded_default(
upper_bound=huggingface_config.max_position_embeddings,
default=model_config.max_length,
)
@dataclass
class TokenInputs(ModelInputs):
tokens: Buffer
@property
def buffers(self) -> tuple[Buffer, ...]:
return (self.tokens,)
class MinillmPipelineModel(PipelineModelWithKVCache[TextContext]):
def __init__(
self,
pipeline_config: PipelineConfig,
session: InferenceSession,
devices: list[Device],
kv_cache_config: KVCacheConfig,
weights: Weights,
adapter: WeightsAdapter | None,
return_logits: ReturnLogits,
*,
memory_plan: MemoryPlan,
return_hidden_states: ReturnHiddenStates = ReturnHiddenStates.NONE,
max_batch_size: int = 1,
) -> None:
super().__init__(
pipeline_config=pipeline_config,
session=session,
devices=devices,
kv_cache_config=kv_cache_config,
weights=weights,
adapter=adapter,
return_logits=return_logits,
return_hidden_states=return_hidden_states,
max_batch_size=max_batch_size,
memory_plan=memory_plan,
)
hf = self.huggingface_config
config = LlamaConfig(
**{
k: getattr(hf, k)
for k in LlamaConfig.__dataclass_fields__
if hasattr(hf, k)
}
)
state = parse_state_dict_from_weights(
self.pipeline_config, weights, adapter
)
state = {name: w.astype(DType.float32) for name, w in state.items()}
model = load_model(config, state, self.devices[0])
self.model = model.compile(
TensorType(
DType.int64, ("batch", "seq_len"), device=self.devices[0]
)
)
@classmethod
def get_kv_params(
cls,
huggingface_config: Any,
pipeline_config: PipelineConfig,
devices: list[DeviceRef],
kv_cache_config: KVCacheConfig,
cache_dtype: DType,
) -> KVCacheParams:
hf = huggingface_config
return kv_cache_config.to_params(
dtype=cache_dtype,
n_kv_heads=hf.num_key_value_heads,
head_dim=getattr(hf, "head_dim", None)
or hf.hidden_size // hf.num_attention_heads,
num_layers=hf.num_hidden_layers,
devices=devices,
data_parallel_degree=pipeline_config.model.data_parallel_degree,
)
def execute(self, model_inputs: ModelInputs) -> ModelOutputs:
assert isinstance(model_inputs, TokenInputs)
logits = self.model(Tensor.from_dlpack(model_inputs.tokens))
last = Buffer.from_numpy(
np.ascontiguousarray(logits.to_numpy()[0, -1:, :])
).to(self.devices[0])
return ModelOutputs(logits=last, next_token_logits=last)
def prepare_initial_token_inputs(
self,
replica_batches: Sequence[Sequence[TextContext]],
kv_cache_inputs: KVCacheInputs[Buffer, Buffer] | None = None,
return_n_logits: int = 1,
) -> TokenInputs:
ids = replica_batches[0][0].tokens.all
inputs = TokenInputs(
tokens=Buffer.from_numpy(np.array([ids], dtype=np.int64)).to(
self.devices[0]
)
)
inputs.kv_cache_inputs = kv_cache_inputs
return inputs
def _identity_adapter(state_dict: dict[str, Weights], **_: object) -> dict:
return {name: w.data() for name, w in state_dict.items()}
minillm_arch = SupportedArchitecture(
name="LlamaForCausalLM",
task=PipelineTask.TEXT_GENERATION,
example_repo_ids=["HuggingFaceTB/SmolLM2-135M-Instruct"],
default_weights_format=WeightsFormat.safetensors,
default_encoding=MinillmArchConfig.DEFAULT_ENCODING,
supported_encodings=MinillmArchConfig.SUPPORTED_ENCODINGS,
pipeline_model=MinillmPipelineModel,
tokenizer=TextTokenizer,
context_type=TextContext,
multi_gpu_supported=False,
weight_adapters={WeightsFormat.safetensors: _identity_adapter},
config=MinillmArchConfig,
required_arguments={"enable_prefix_caching": False, "max_batch_size": 1},
supports_overlap_scheduler=False,
supports_device_graph_capture=False,
)
ARCHITECTURES = [minillm_arch]
Saved to minillm/serve/__init__.py.
Use it in Python
import_custom_architectures registers the package’s architectures, as
--custom-architectures does for the CLI. A registered architecture with
the same name as one of MAX’s replaces it in this process. When an
architecture has a newer module-based implementation, MAX prefers that one,
so prefer_module_v3=False selects the registered LlamaForCausalLM:
import_custom_architectures(["minillm.serve"])
ours = AutoPipeline(
MODEL_ID, quantization_encoding="float32", prefer_module_v3=False
)
print("pipeline model:", type(ours.model.pipeline._pipeline_model).__name__)
result = ours(PROMPT, max_new_tokens=16, top_k=1)
ours.close()
print(result)
print("same as MAX's implementation:", result == expected)
assert result == expected
pipeline model: MinillmPipelineModel
['The capital of France is Paris. Paris is the largest city in France and the capital of the French department']
same as MAX's implementation: True
The pipeline model is yours, and its greedy output matches MAX’s implementation token for token.
Serve it
The same package serves over HTTP. --custom-architectures minillm.serve
imports it, and --no-prefer-module-v3 selects it over MAX’s own
implementation:
server = subprocess.Popen(
[
"max",
"serve",
"--model",
MODEL_ID,
"--port",
"8000",
"--quantization-encoding",
"float32",
"--custom-architectures",
"minillm.serve",
"--no-prefer-module-v3",
],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
start = time.perf_counter()
while True:
try:
urllib.request.urlopen("http://localhost:8000/health")
break
except OSError:
if server.poll() is not None:
raise RuntimeError(
"max serve exited; run it in a terminal to see why"
)
time.sleep(1)
print(f"server ready in {time.perf_counter() - start:.0f} s")
client = OpenAI(base_url="http://localhost:8000/v1", api_key="unused")
completion = client.completions.create(
model=MODEL_ID, prompt=PROMPT[0], max_tokens=16, temperature=0
)
print(PROMPT[0] + completion.choices[0].text)
assert PROMPT[0] + completion.choices[0].text == expected[0]
server.terminate()
_ = server.wait()
server ready in 14 s
The capital of France is Paris. Paris is the largest city in France and the capital of the French department
Your model now answers an OpenAI client exactly as MAX’s implementation does. The handbook’s OpenAI-compatible API page covers the request format. MAX’s guide to serving custom model architectures covers the production version: a decoder module on MAX’s paged KV cache, with batching and GPU kernels.
Exercises
- Register
Qwen3ForCausalLMfrom Section 6.1 the same way, and serveQwen/Qwen3-0.6Bthrough it. Which parts of the package depend on the model class, and which work unchanged? - Make the pipeline model serve more than one request per step: read every
context in
replica_batches[0], pad them asrun_stepfrom Section 5.1 does, and return one row of logits per request. - Replace the full recompute with
KVCacheDecoderfrom Section 1.8, keeping one cache per request ID. What happens when a request finishes?
Next
Section 7.2 measures the
server under load with max benchmark.