A naive Bayes classifier turns text into class probabilities rather than generated language. It applies Bayes' theorem to choose among hypotheses, making it useful for classification and as an augmentation component around generative language systems. section 2.2.2
The naive version is deliberately simple: it treats observed word sequences as disconnected, much like bag-of-words models. That simplicity can make the implementation clear, but it discards sequence relationships and can limit effectiveness; multinomial extensions and other improvements may perform better for particular problems. section 2.2.2
Classifying text with word-label counts
A categorical implementation processes utterances, pairs each word with its class label, and counts those pairs. The listing uses sentences from Hamlet and binary sentiment labels, illustrating how class-associated word frequencies provide the evidence for classification. section 2.2.2
Listing 2.2 Categorical naive Bayes language model implementation
from utils import process_utt, lookup
from nltk.corpus.reader import PlaintextCorpusReader
import numpy as np
my_corpus = PlaintextCorpusReader("./", ".*\.txt")
sents = my_corpus.sents(fileids="hamlet.txt")
def count_utts(result, utts, ys):
"""
Input:
result: a dictionary that is used to map each pair to its frequency
utts: a list of utts
ys: a list of the sentiment of each utt (either 0 or 1)
Output:
result: a dictionary mapping each pair to its frequency
"""
for y, utt in zip(ys, utts):
for word in process_utt(utt):
pair = (word, y) #1
if pair in result:
result[pair] += 1 #2
else:
result[pair] = 1 #3
return result
result = {}
utts = [" ".join(sent) for sent in sents]
ys = [sent.count("be") > 0 for sent in sents]
count_utts(result, utts, ys)
freqs = count_utts({}, utts, ys)
lookup(freqs, "be", True)
for k, v in freqs.items():
if "be" in k:
print(f"{k}:{v}")
def train_naive_bayes(freqs, train_x, train_y):
"""
Input:
freqs: dictionary from (word, label) to how often the word appears
train_x: a list of utts
train_y: a list of labels correponding to the utts (0,1)
Output:
logprior: the log prior.
loglikelihood: the log likelihood of you Naive bayes equation.
"""
loglikelihood = {}
logprior = 0
vocab = set([pair[0] for pair in freqs.keys()]) #4
V = len(vocab)
N_pos = N_neg = 0 #5
for pair in freqs.keys():
if pair[1] > 0: #6
N_pos += lookup(freqs, pair[0], True) #7
else: #8
N_neg += lookup(freqs, pair[0], False) #9
D = len(train_y) #10
D_pos = sum(train_y) #11
D_neg = D - D_pos #12
logprior = np.log(D_pos) - np.log(D_neg) #13
for word in vocab: #14
freq_pos = lookup(freqs, word, 1)
freq_neg = lookup(freqs, word, 0)
p_w_pos = (freq_pos + 1) / (N_pos + V) #15
p_w_neg = (freq_neg + 1) / (N_neg + V)
loglikelihood[word] = np.log(p_w_pos / p_w_neg) #16
return logprior, loglikelihood
def naive_bayes_predict(utt, logprior, loglikelihood):
"""
Input:
utt: a string
logprior: a number
loglikelihood: a dictionary of words mapping to numbers
Output:
p: the sum of all the logliklihoods + logprior
"""
word_l = process_utt(utt) #17
p = 0 #18
p += logprior #19
for word in word_l:
if word in loglikelihood: #20
p += loglikelihood[word] #21
return p
def test_naive_bayes(test_x, test_y, logprior, loglikelihood):
"""
Input:
test_x: A list of utts
test_y: the corresponding labels for the list of utts
logprior: the logprior
loglikelihood: a dictionary with the loglikelihoods for each word
Output:
accuracy: (# of utts classified correctly)/(total # of utts)
"""
accuracy = 0 #22
y_hats = []
for utt in test_x:
if naive_bayes_predict(utt, logprior, loglikelihood) > 0: #23
y_hat_i = 1 #24
else:
y_hat_i = 0 #25
y_hats.append(y_hat_i) #26
error = sum(
[abs(y_hat - test) for y_hat, test in zip(y_hats, test_y)]
) / len(y_hats) #27
accuracy = 1 - error #28
return accuracy
if __name__ == "__main__":
logprior, loglikelihood = train_naive_bayes(freqs, utts, ys)
print(logprior)
print(len(loglikelihood))
my_utt = "To be or not to be, that is the question."
p = naive_bayes_predict(my_utt, logprior, loglikelihood)
print("The expected output is", p)
print(
f"Naive Bayes accuracy = {test_naive_bayes(utts, ys, logprior, loglikelihood):0.4f}
)
Inspect the hand-written categorical implementation to follow corpus loading, utterance processing, label pairing, and frequency counting without hiding the mechanics behind a machine-learning package. section 2.2.2
