Minimum token frequency sets how many times a subword must appear in the training dataset before entering the tokenizer vocabulary. Raising it keeps rare tokens from consuming vocabulary space, but can also discard rare tokens that matter. section 4.4.1
A practical setting
A custom byte-level BPE tokenizer can require two occurrences before adding a subword: min_frequency=2. Tune this alongside vocab_size and special_tokens rather than treating it as an isolated setting. section 4.4.1
Listing 4.10 Training your own subword tokenizers
import os
from pathlib import Path
import transformers
from tokenizers import ByteLevelBPETokenizer, SentencePieceBPETokenizer
from tokenizers.processors import BertProcessing
paths = [str(x) for x in Path("./data/").glob("**/*.txt")] #1
bpe_tokenizer = ByteLevelBPETokenizer() #2
bpe_tokenizer.train( #2
files=paths, #2
vocab_size=52_000, #2
min_frequency=2, #2
show_progress=True, #2
special_tokens=[ #2
"<s>", #2
"<pad>", #2
"</s>", #2
"<unk>", #2
"<mask>", #2
], #2
) #2
token_dir = "./chapters/chapter_4/tokenizers/bytelevelbpe/"
if not os.path.exists(token_dir):
os.makedirs(token_dir)
bpe_tokenizer.save_model(token_dir)
bpe_tokenizer = ByteLevelBPETokenizer(
f"{token_dir}vocab.json",
f"{token_dir}merges.txt",
)
example_text = "This sentence is getting encoded by a tokenizer."
print(bpe_tokenizer.encode(example_text).tokens)
# ['This', 'Ġsentence', 'Ġis', 'Ġgetting', 'Ġenc', \
# 'oded', 'Ġby', 'Ġa', 'Ġto', 'ken', 'izer', '.']
print(bpe_tokenizer.encode(example_text).ids)
# [2666, 5651, 342, 1875, 4650, 10010, 504, 265, \
# 285, 1507, 13035, 18]
bpe_tokenizer._tokenizer.post_processor = BertProcessing(
("</s>", bpe_tokenizer.token_to_id("</s>")),
("<s>", bpe_tokenizer.token_to_id("<s>")),
)
bpe_tokenizer.enable_truncation(max_length=512)
special_tokens = [
"<s>",
"<pad>",
"</s>",
"<unk>",
"<cls>",
"<sep>",
"<mask>",
]
sentencepiece_tokenizer = SentencePieceBPETokenizer() #3
sentencepiece_tokenizer.train( #3
files=paths, #3
vocab_size=4000, #3
min_frequency=2, #3
show_progress=True, #3
special_tokens=special_tokens, #3
) #3
token_dir = "./chapters/chapter_4/tokenizers/sentencepiece/"
if not os.path.exists(token_dir):
os.makedirs(token_dir)
sentencepiece_tokenizer.save_model(token_dir)
tokenizer = transformers.PreTrainedTokenizerFast(
tokenizer_object=sentencepiece_tokenizer,
model_max_length=512,
special_tokens=special_tokens,
) #4
tokenizer.bos_token = "<s>"
tokenizer.bos_token_id = sentencepiece_tokenizer.token_to_id("<s>")
tokenizer.pad_token = "<pad>"
tokenizer.pad_token_id = sentencepiece_tokenizer.token_to_id("<pad>")
tokenizer.eos_token = "</s>"
tokenizer.eos_token_id = sentencepiece_tokenizer.token_to_id("</s>")
tokenizer.unk_token = "<unk>"
tokenizer.unk_token_id = sentencepiece_tokenizer.token_to_id("<unk>")
tokenizer.cls_token = "<cls>"
tokenizer.cls_token_id = sentencepiece_tokenizer.token_to_id("<cls>")
tokenizer.sep_token = "<sep>"
tokenizer.sep_token_id = sentencepiece_tokenizer.token_to_id("<sep>")
tokenizer.mask_token = "<mask>"
tokenizer.mask_token_id = sentencepiece_tokenizer.token_to_id("<mask>")
tokenizer.save_pretrained(token_dir) #5
print(tokenizer.tokenize(example_text))
# ['_This', '_s', 'ent', 'ence', '_is', '_', 'g', 'et', 'tin', 'g', '_'
# 'en', 'co', 'd', 'ed', '_', 'b', 'y', '_a', '_', 't', 'ok', 'en',
# 'iz', 'er', '.']
print(tokenizer.encode(example_text))
# [814, 1640, 609, 203, 1810, 623, 70, \
# 351, 148, 371, 125, 146, 2402, 959, 632]
Inspect the complete tokenizer-training example, including min_frequency=2 for a custom ByteLevelBPE tokenizer. section 4.4.1
