Continuous language modeling predicts a target word from distributed context representations instead of relying only on discrete word counts. A CBoW example uses surrounding words to guess “linguistics,” and continuous models can use words after the target as well as before it. section 2.2.4
The technique matters more than any single component choice:
- Pair a bag-of-words context representation with a one-layer neural network to estimate embeddings. section 2.2.4
- Treat model components as replaceable: N-grams can replace bag-of-words, and naive Bayes can replace the neural network. section 2.2.4
- Expect generation weaknesses when context is incomplete, especially at sentence boundaries where the model may favor previously observed endings. section 2.2.4
A small generative CBoW model
Given “Learning about linguistics makes me happy,” a context window can provide “learning,” “about,” “makes,” and “me” to predict “linguistics.” A minimal implementation then initializes weights, applies ReLU and softmax, and performs forward and backpropagation. section 2.2.4
Listing 2.4 Generative CBoW language model implementation
import nltk
import numpy as np
from utils import get_batches, compute_pca, get_dict
import re
from matplotlib import pyplot
with open("hamlet.txt", "r", encoding="utf-8") as f:
data = f.read() #1
data = re.sub(r"[,!?;-]", ".", data) #2
data = nltk.word_tokenize(data)
data = [ch.lower() for ch in data if ch.isalpha() or ch == "."]
print("Number of tokens:", len(data), "\n", data[500:515])
fdist = nltk.FreqDist(word for word in data) #3
print("Size of vocabulary:", len(fdist))
print("Most Frequent Tokens:", fdist.most_common(20))
word2Ind, Ind2word = get_dict(data) #4
V = len(word2Ind)
print("Size of vocabulary:", V)
print("Index of the word 'king':", word2Ind["king"])
print("Word which has index 2743:", Ind2word[2743])
def initialize_model(N, V, random_seed=1): #5
"""
Inputs:
N: dimension of hidden vector
V: dimension of vocabulary
random_seed: seed for consistent results in tests
Outputs:
W1, W2, b1, b2: initialized weights and biases
"""
np.random.seed(random_seed)
W1 = np.random.rand(N, V)
W2 = np.random.rand(V, N)
b1 = np.random.rand(N, 1)
b2 = np.random.rand(V, 1)
return W1, W2, b1, b2
def softmax(z): #6
"""
Inputs:
z: output scores from the hidden layer
Outputs:
yhat: prediction (estimate of y)
"""
yhat = np.exp(z) / np.sum(np.exp(z), axis=0)
return yhat
def forward_prop(x, W1, W2, b1, b2): #7
"""
Inputs:
x: average one-hot vector for the context
W1,W2,b1,b2: weights and biases to be learned
Outputs:
z: output score vector
"""
h = W1 @ x + b1
h = np.maximum(0, h)
z = W2 @ h + b2
return z, h
def compute_cost(y, yhat, batch_size): #8
logprobs = np.multiply(np.log(yhat), y) + np.multiply(
np.log(1 - yhat), 1 - y
)
cost = -1 / batch_size * np.sum(logprobs)
cost = np.squeeze(cost)
return cost
def back_prop(x, yhat, y, h, W1, W2, b1, b2, batch_size): #9
"""
Inputs:
x: average one hot vector for the context
yhat: prediction (estimate of y)
y: target vector
h: hidden vector (see eq. 1)
W1, W2, b1, b2: weights and biases
batch_size: batch size
Outputs:
grad_W1, grad_W2, grad_b1, grad_b2: gradients of weights and biases
"""
l1 = np.dot(W2.T, yhat - y)
l1 = np.maximum(0, l1)
grad_W1 = np.dot(l1, x.T) / batch_size
grad_W2 = np.dot(yhat - y, h.T) / batch_size
grad_b1 = np.sum(l1, axis=1, keepdims=True) / batch_size
grad_b2 = np.sum(yhat - y, axis=1, keepdims=True) / batch_size
return grad_W1, grad_W2, grad_b1, grad_b2
def gradient_descent(data, word2Ind, N, V, num_iters, alpha=0.03): #10
"""
This is the gradient_descent function
Inputs:
data: text
word2Ind: words to Indices
N: dimension of hidden vector
V: dimension of vocabulary
num_iters: number of iterations
Outputs:
W1, W2, b1, b2: updated matrices and biases
"""
W1, W2, b1, b2 = initialize_model(N, V, random_seed=8855)
batch_size = 128
iters = 0
C = 2
for x, y in get_batches(data, word2Ind, V, C, batch_size):
z, h = forward_prop(x, W1, W2, b1, b2)
yhat = softmax(z)
cost = compute_cost(y, yhat, batch_size)
if (iters + 1) % 10 == 0:
print(f"iters: {iters+1} cost: {cost:.6f}")
grad_W1, grad_W2, grad_b1, grad_b2 = back_prop(
x, yhat, y, h, W1, W2, b1, b2, batch_size
)
W1 = W1 - alpha * grad_W1
W2 = W2 - alpha * grad_W2
b1 = b1 - alpha * grad_b1
b2 = b2 - alpha * grad_b2
iters += 1
if iters == num_iters:
break
if iters % 100 == 0:
alpha *= 0.66
return W1, W2, b1, b2
C = 2 #11
N = 50
word2Ind, Ind2word = get_dict(data)
V = len(word2Ind)
num_iters = 150
print("Call gradient_descent")
W1, W2, b1, b2 = gradient_descent(data, word2Ind, N, V, num_iters)
# Call gradient descent
# Iters: 10 loss: 0.525015
# Iters: 20 loss: 0.092373
# Iters: 30 loss: 0.050474
# Iters: 40 loss: 0.034724
# Iters: 50 loss: 0.026468
# Iters: 60 loss: 0.021385
# Iters: 70 loss: 0.017941
# Iters: 80 loss: 0.015453
# Iters: 90 loss: 0.012099
# Iters: 100 loss: 0.012099
# Iters: 110 loss: 0.011253
# Iters: 120 loss: 0.010551
# Iters: 130 loss: 0.009932
# Iters: 140 loss: 0.009382
# Iters: 150 loss: 0.008889
Inspect the complete NumPy-based generative CBoW implementation, including Hamlet preprocessing, vocabulary construction, model initialization, and training structure. section 2.2.4
