A learning rate schedule specifies how the optimization step size changes across iterations. It balances progress toward an optimum against unstable updates or unnecessarily long training. section 10.4 section 5.4
A commonly used schedule is
- ηk = (τ0 + k)^−κ, with τ0 ≥ 0 and κ ∈ (0.5, 1]. section 5.4 section 5.7
- τ0 slows early iterations, while κ controls how quickly older step sizes are forgotten. section 5.4
- Robbins–Monro conditions require learning rates to support convergence while controlling oscillations. section 10.4
- For neural networks, a practical starting point is 1e−3, followed by reduction as iterations increase. section 10.4
SGD logistic regression
An SGD implementation stores τ0, κ, and an η array, then computes η[i] = (τ0 + i)^−κ before updating parameters opposite to the gradient. A synthetic Gaussian-mixture dataset supplies the training example. section 5.4
Listing 10.6 Neural network optimizers
import numpy as np
import pandas as pd
import tensorflow as tf
from tensorflow import keras
from keras import backend as K
from keras.models import Sequential
from keras.layers import Dense, Dropout, Flatten
from keras.layers import Conv2D, MaxPooling2D, Activation
from keras.callbacks import ModelCheckpoint
from keras.callbacks import TensorBoard
from keras.callbacks import LearningRateScheduler
from keras.callbacks import EarlyStopping
import math
import matplotlib.pyplot as plt
tf.keras.utils.set_random_seed(42)
SAVE_PATH = "/content/drive/MyDrive/Colab Notebooks/data/"
def scheduler(epoch, lr):
if epoch < 4:
return lr
else:
return lr * tf.math.exp(-0.1)
if __name__ == "__main__":
img_rows, img_cols = 32, 32
(x_train, y_train), (x_test, y_test) =
keras.datasets.cifar100.load_data() #1
x_train = x_train.reshape(x_train.shape[0], img_rows, img_cols,
3).astype("float32") / 255
x_test = x_test.reshape(x_test.shape[0], img_rows, img_cols,
3).astype("float32") / 255
y_train_label = keras.utils.to_categorical(y_train)
y_test_label = keras.utils.to_categorical(y_test)
num_classes = y_train_label.shape[1]
batch_size = 256
num_epochs = 32 #2
num_filters_l1 = 64
num_filters_l2 = 128 #3
#CNN architecture
cnn = Sequential()
cnn.add(Conv2D(num_filters_l1, kernel_size = (5, 5),
input_shape=(img_rows, img_cols, 3), padding='same'))
cnn.add(Activation('relu'))
cnn.add(MaxPooling2D(pool_size=(2,2), strides=(2,2)))
cnn.add(Conv2D(num_filters_l2, kernel_size = (5, 5), padding='same'))
cnn.add(Activation('relu'))
cnn.add(MaxPooling2D(pool_size=(2,2), strides=(2,2)))
cnn.add(Flatten())
cnn.add(Dense(128))
cnn.add(Activation('relu'))
cnn.add(Dense(num_classes))
cnn.add(Activation('softmax'))
opt1 = tf.keras.optimizers.SGD() #4
opt2 = tf.keras.optimizers.SGD(momentum=0.9, #4
nesterov=True) #4
opt3 = tf.keras.optimizers.RMSprop() #4
opt4 = tf.keras.optimizers.Adam() #4
optimizer_list = [opt1, opt2, opt3, opt4]
history_list = []
for idx in range(len(optimizer_list)):
K.clear_session()
opt = optimizer_list[idx]
cnn.compile(
loss=keras.losses.CategoricalCrossentropy(),
optimizer=opt,
metrics=["accuracy"]
)
#define callbacks
reduce_lr = LearningRateScheduler(scheduler, verbose=1)
callbacks_list = [reduce_lr]
#training loop
hist = cnn.fit(x_train, y_train_label, batch_size=batch_size,
epochs=num_epochs, callbacks=callbacks_list, validation_split=0.2)
history_list.append(hist)
#end for
plt.figure()
plt.plot(history_list[0].history['loss'], 'b', lw=2.0, label='SGD')
plt.plot(history_list[1].history['loss'], '--r', lw=2.0, label='SGD
Nesterov')
plt.plot(history_list[2].history['loss'], ':g', lw=2.0, label='RMSProp')
plt.plot(history_list[3].history['loss'], '-.k', lw=2.0, label='ADAM')
plt.title('LeNet, CIFAR-100, Optimizers')
plt.xlabel('Epochs')
plt.ylabel('Cross-Entropy Training Loss')
plt.legend(loc='upper right')
plt.show()
#plt.savefig('./figures/lenet_loss.png')
plt.figure()
plt.plot(history_list[0].history['val_accuracy'], 'b', lw=2.0,
label='SGD')
plt.plot(history_list[1].history['val_accuracy'], '--r', lw=2.0,
label='SGD Nesterov')
plt.plot(history_list[2].history['val_accuracy'], ':g', lw=2.0,
label='RMSProp')
plt.plot(history_list[3].history['val_accuracy'], '-.k', lw=2.0,
label='ADAM')
plt.title('LeNet, CIFAR-100, Optimizers')
plt.xlabel('Epochs')
plt.ylabel('Validation Accuracy')
plt.legend(loc='upper right')
plt.show()
#plt.savefig('./figures/lenet_loss.png')
plt.figure()
plt.plot(history_list[0].history['lr'], 'b', lw=2.0, label='SGD')
plt.plot(history_list[1].history['lr'], '--r', lw=2.0, label='SGD
Nesterov')
plt.plot(history_list[2].history['lr'], ':g', lw=2.0, label='RMSProp')
plt.plot(history_list[3].history['lr'], '-.k', lw=2.0, label='ADAM')
plt.title('LeNet, CIFAR-100, Optimizers')
plt.xlabel('Epochs')
plt.ylabel('Learning Rate Schedule')
plt.legend(loc='upper right')
plt.show()
#plt.savefig('./figures/lenet_loss.png')
The TensorFlow/Keras implementation defines a LearningRateScheduler callback that keeps the rate unchanged for four epochs and then multiplies it by exp(−0.1). section 10.4
