Materialization writes an intermediate computational result to memory so a later operation can read it. In GPU kernels, storing these intermediates in global memory can turn a computation that might stay local into one dominated by memory traffic. section 3.3.1 section 4.2.2
Concrete cases show where intermediate results are materialized:
- In a naive SwiGLU implementation, separate operations write intermediate tensors to global memory before later operations read them. section 3.3.1
- In an unfused softmax, exponentials and partial reduction results are written to and read from global memory across multiple kernels. section 4.2.2
- Fusion keeps softmax intermediates local to a program instance, reducing memory traffic and kernel launches. section 4.2.3
Listing 4.6 Unfused softmax via global reductions and intermediate materialization
import torch
import triton
import triton.language as tl
@triton.jit
def reduce_max_kernel(x_ptr, out_ptr, N, BLOCK_SIZE: tl.constexpr):
pid = tl.program_id(0)
offsets = pid * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
mask = offsets < N
x = tl.load(x_ptr + offsets, mask=mask, other=-float("inf")).to(
tl.float32)
m = tl.max(x, axis=0)
tl.store(out_ptr + pid, m)
@triton.jit
def transform_exp_kernel(x_ptr, z_ptr, max_ptr, N, BLOCK_SIZE: tl.constexpr):
pid = tl.program_id(0)
offsets = pid * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
mask = offsets < N
m = tl.load(max_ptr).to(tl.float32)
x = tl.load(x_ptr + offsets, mask=mask, other=-float("inf")).to(
tl.float32)
z = tl.exp(x - m)
tl.store(z_ptr + offsets, z, mask=mask)
@triton.jit
def reduce_sum_kernel(z_ptr, out_ptr, N, BLOCK_SIZE: tl.constexpr):
pid = tl.program_id(0)
offsets = pid * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
mask = offsets < N
z = tl.load(z_ptr + offsets, mask=mask, other=0.0).to(tl.float32)
s = tl.sum(z, axis=0)
tl.store(out_ptr + pid, s)
@triton.jit
def div_store_kernel(z_ptr, y_ptr, sum_ptr, N, BLOCK_SIZE: tl.constexpr):
pid = tl.program_id(0)
offsets = pid * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
mask = offsets < N
s = tl.load(sum_ptr).to(tl.float32)
z = tl.load(z_ptr + offsets, mask=mask, other=0.0).to(tl.float32)
y = z / s
tl.store(y_ptr + offsets, y, mask=mask)
def reduce_global(x, N, reduce_kernel, BLOCK_SIZE=1024): #1
cur = x
cur_N = N
while cur_N > 1:
out_N = (cur_N + BLOCK_SIZE - 1) // BLOCK_SIZE
out = torch.empty((out_N,), device=x.device, dtype=torch.float32)
reduce_kernel[(out_N,)](cur, out, cur_N, BLOCK_SIZE=BLOCK_SIZE)
cur = out
cur_N = out_N
return cur
def softmax_unfused(x: torch.Tensor, y: torch.Tensor, N: int):
BLOCK_SIZE = 1024
grid = ((N + BLOCK_SIZE - 1) // BLOCK_SIZE,)
max_val = reduce_global(x, N, reduce_max_kernel) #2
z = torch.empty_like(x) #3
transform_exp_kernel[grid](x, z, max_val, N, BLOCK_SIZE=BLOCK_SIZE)
sum_val = reduce_global(z, N, reduce_sum_kernel) #4
div_store_kernel[grid](z, y, sum_val, N, BLOCK_SIZE=BLOCK_SIZE)
The unfused softmax listing makes the sequence of intermediate writes and reads visible. section 4.2.2
