A special token is a reserved token used for syntax or control in tokenized model input. When training a tokenizer, define these tokens alongside vocabulary size and minimum frequency so important training markers are represented explicitly. section 4.4.1
A practical tokenizer configuration commonly includes:
- <s> and </s> for sequence boundaries, <pad> for padding, <unk> for unknown tokens, and <mask> for masked-token training. section 4.4.1
- Choose the set with the tokenizer and model being trained or fine-tuned; BPE and SentencePiece are both common alternatives. section 4.4.1
Listing 4.10 Training your own subword tokenizers
import os
from pathlib import Path
import transformers
from tokenizers import ByteLevelBPETokenizer, SentencePieceBPETokenizer
from tokenizers.processors import BertProcessing
paths = [str(x) for x in Path("./data/").glob("**/*.txt")] #1
bpe_tokenizer = ByteLevelBPETokenizer() #2
bpe_tokenizer.train( #2
files=paths, #2
vocab_size=52_000, #2
min_frequency=2, #2
show_progress=True, #2
special_tokens=[ #2
"<s>", #2
"<pad>", #2
"</s>", #2
"<unk>", #2
"<mask>", #2
], #2
) #2
token_dir = "./chapters/chapter_4/tokenizers/bytelevelbpe/"
if not os.path.exists(token_dir):
os.makedirs(token_dir)
bpe_tokenizer.save_model(token_dir)
bpe_tokenizer = ByteLevelBPETokenizer(
f"{token_dir}vocab.json",
f"{token_dir}merges.txt",
)
example_text = "This sentence is getting encoded by a tokenizer."
print(bpe_tokenizer.encode(example_text).tokens)
# ['This', 'Ġsentence', 'Ġis', 'Ġgetting', 'Ġenc', \
# 'oded', 'Ġby', 'Ġa', 'Ġto', 'ken', 'izer', '.']
print(bpe_tokenizer.encode(example_text).ids)
# [2666, 5651, 342, 1875, 4650, 10010, 504, 265, \
# 285, 1507, 13035, 18]
bpe_tokenizer._tokenizer.post_processor = BertProcessing(
("</s>", bpe_tokenizer.token_to_id("</s>")),
("<s>", bpe_tokenizer.token_to_id("<s>")),
)
bpe_tokenizer.enable_truncation(max_length=512)
special_tokens = [
"<s>",
"<pad>",
"</s>",
"<unk>",
"<cls>",
"<sep>",
"<mask>",
]
sentencepiece_tokenizer = SentencePieceBPETokenizer() #3
sentencepiece_tokenizer.train( #3
files=paths, #3
vocab_size=4000, #3
min_frequency=2, #3
show_progress=True, #3
special_tokens=special_tokens, #3
) #3
token_dir = "./chapters/chapter_4/tokenizers/sentencepiece/"
if not os.path.exists(token_dir):
os.makedirs(token_dir)
sentencepiece_tokenizer.save_model(token_dir)
tokenizer = transformers.PreTrainedTokenizerFast(
tokenizer_object=sentencepiece_tokenizer,
model_max_length=512,
special_tokens=special_tokens,
) #4
tokenizer.bos_token = "<s>"
tokenizer.bos_token_id = sentencepiece_tokenizer.token_to_id("<s>")
tokenizer.pad_token = "<pad>"
tokenizer.pad_token_id = sentencepiece_tokenizer.token_to_id("<pad>")
tokenizer.eos_token = "</s>"
tokenizer.eos_token_id = sentencepiece_tokenizer.token_to_id("</s>")
tokenizer.unk_token = "<unk>"
tokenizer.unk_token_id = sentencepiece_tokenizer.token_to_id("<unk>")
tokenizer.cls_token = "<cls>"
tokenizer.cls_token_id = sentencepiece_tokenizer.token_to_id("<cls>")
tokenizer.sep_token = "<sep>"
tokenizer.sep_token_id = sentencepiece_tokenizer.token_to_id("<sep>")
tokenizer.mask_token = "<mask>"
tokenizer.mask_token_id = sentencepiece_tokenizer.token_to_id("<mask>")
tokenizer.save_pretrained(token_dir) #5
print(tokenizer.tokenize(example_text))
# ['_This', '_s', 'ent', 'ence', '_is', '_', 'g', 'et', 'tin', 'g', '_'
# 'en', 'co', 'd', 'ed', '_', 'b', 'y', '_a', '_', 't', 'ok', 'en',
# 'iz', 'er', '.']
print(tokenizer.encode(example_text))
# [814, 1640, 609, 203, 1810, 623, 70, \
# 351, 148, 371, 125, 146, 2402, 959, 632]
The tokenizer-training listing shows special_tokens configured directly in a ByteLevelBPE training call, alongside vocab_size and min_frequency. section 4.4.1
