A beginning-of-sequence (BOS) token is a special marker automatically inserted at the start of a tokenized sequence. In Llama-family models, it acts as an attention sink: its abnormally large activations provide a reference point for other tokens but contain no useful semantic information for the visualization. Excluding BOS prevents those activations from distorting the heatmap. section 10.2.1
Listing 10.4 Auxiliary functions for tokenization, binning and rendering
def get_token_labels(tokenizer, prompt):
input_ids = tokenizer(prompt, return_tensors="pt")["input_ids"][0]
start = 1 if input_ids[
0] == tokenizer.bos_token_id else 0 #A
labels = [
tokenizer.convert_ids_to_tokens(
int(t)).replace("▁", "").replace("Ġ", "") #B
for t in input_ids[start:]
]
return labels, start
def bin_neurons(tensor, bin_size):
n_rows, n_neurons = tensor.shape
n_bins = n_neurons // bin_size
trimmed = tensor[:, : n_bins * bin_size] #C
return trimmed.reshape(
n_rows,
n_bins,
bin_size).mean(axis=2) #1
def plot_heatmap_panel(ax, matrix, bin_size, cmap, vmax_percentile, title, y_labels=None):
abs_bound = float(np.percentile(np.abs(matrix),
vmax_percentile)) #2
norm = TwoSlopeNorm(vmin=-abs_bound,
vcenter=0.0,
vmax=abs_bound) #D
im = ax.imshow(matrix, aspect="auto",
cmap=cmap,
norm=norm, interpolation="nearest")
n_bins = matrix.shape[1]
xtick_bins = np.linspace(0,
n_bins - 1, min(9, n_bins)).astype(int)
ax.set_xticks(xtick_bins)
ax.set_xticklabels(
[str(int(b * bin_size)) for b in xtick_bins],
fontsize=8, rotation=30)
ax.set_xlabel("Neuron index", fontsize=10)
if y_labels is not None:
ax.set_yticks(range(len(y_labels)))
ax.set_yticklabels(y_labels, fontsize=8)
fig = ax.get_figure()
cbar = fig.colorbar(im, ax=ax, fraction=0.03, pad=0.02)
cbar.set_label(
f"[-{abs_bound:.3f}, +{abs_bound:.3f}] (p{int(vmax_percentile)})",
fontsize=8)
ax.set_title(title, fontsize=11, fontweight="bold")
return im
The token-label helper checks whether the first input ID is the BOS token ID and returns a start index that skips it. section 10.2.1



