RMSprop is an adaptive gradient optimizer that normalizes parameter updates with an exponentially decaying average of squared gradients. It reduces effective learning rates for parameters receiving large gradients and increases progress in gently sloped directions. Unlike AdaGrad, it discounts distant history, avoiding continual learning-rate decay when gradients remain constant. section 10.4
The optimization logic proceeds in three conceptual steps:
- Identify the problem with a learning rate set independently of gradient magnitude: large gradients can cause oscillating, oversized steps, while small gradients can make progress slow. section 10.4
- Accumulate squared gradient information with an exponentially weighted moving average rather than AdaGrad’s sum over the entire training history. section 10.4
- Scale updates using this recent gradient-history estimate, making updates cautious in steep directions and comparatively larger in gently sloped directions. section 10.4
Optimizer comparison on image classification
RMSprop can be evaluated alongside stochastic gradient descent, momentum, Nesterov momentum, AdaGrad, and Adam when training a LeNet network on CIFAR-100 with Keras/TensorFlow. Loss and validation accuracy provide the comparison measures; the reported experiment identifies Adam and Nesterov momentum as producing the highest validation accuracy. section 10.4
Listing 10.6 Neural network optimizers
import numpy as np
import pandas as pd
import tensorflow as tf
from tensorflow import keras
from keras import backend as K
from keras.models import Sequential
from keras.layers import Dense, Dropout, Flatten
from keras.layers import Conv2D, MaxPooling2D, Activation
from keras.callbacks import ModelCheckpoint
from keras.callbacks import TensorBoard
from keras.callbacks import LearningRateScheduler
from keras.callbacks import EarlyStopping
import math
import matplotlib.pyplot as plt
tf.keras.utils.set_random_seed(42)
SAVE_PATH = "/content/drive/MyDrive/Colab Notebooks/data/"
def scheduler(epoch, lr):
if epoch < 4:
return lr
else:
return lr * tf.math.exp(-0.1)
if __name__ == "__main__":
img_rows, img_cols = 32, 32
(x_train, y_train), (x_test, y_test) =
keras.datasets.cifar100.load_data() #1
x_train = x_train.reshape(x_train.shape[0], img_rows, img_cols,
3).astype("float32") / 255
x_test = x_test.reshape(x_test.shape[0], img_rows, img_cols,
3).astype("float32") / 255
y_train_label = keras.utils.to_categorical(y_train)
y_test_label = keras.utils.to_categorical(y_test)
num_classes = y_train_label.shape[1]
batch_size = 256
num_epochs = 32 #2
num_filters_l1 = 64
num_filters_l2 = 128 #3
#CNN architecture
cnn = Sequential()
cnn.add(Conv2D(num_filters_l1, kernel_size = (5, 5),
input_shape=(img_rows, img_cols, 3), padding='same'))
cnn.add(Activation('relu'))
cnn.add(MaxPooling2D(pool_size=(2,2), strides=(2,2)))
cnn.add(Conv2D(num_filters_l2, kernel_size = (5, 5), padding='same'))
cnn.add(Activation('relu'))
cnn.add(MaxPooling2D(pool_size=(2,2), strides=(2,2)))
cnn.add(Flatten())
cnn.add(Dense(128))
cnn.add(Activation('relu'))
cnn.add(Dense(num_classes))
cnn.add(Activation('softmax'))
opt1 = tf.keras.optimizers.SGD() #4
opt2 = tf.keras.optimizers.SGD(momentum=0.9, #4
nesterov=True) #4
opt3 = tf.keras.optimizers.RMSprop() #4
opt4 = tf.keras.optimizers.Adam() #4
optimizer_list = [opt1, opt2, opt3, opt4]
history_list = []
for idx in range(len(optimizer_list)):
K.clear_session()
opt = optimizer_list[idx]
cnn.compile(
loss=keras.losses.CategoricalCrossentropy(),
optimizer=opt,
metrics=["accuracy"]
)
#define callbacks
reduce_lr = LearningRateScheduler(scheduler, verbose=1)
callbacks_list = [reduce_lr]
#training loop
hist = cnn.fit(x_train, y_train_label, batch_size=batch_size,
epochs=num_epochs, callbacks=callbacks_list, validation_split=0.2)
history_list.append(hist)
#end for
plt.figure()
plt.plot(history_list[0].history['loss'], 'b', lw=2.0, label='SGD')
plt.plot(history_list[1].history['loss'], '--r', lw=2.0, label='SGD
Nesterov')
plt.plot(history_list[2].history['loss'], ':g', lw=2.0, label='RMSProp')
plt.plot(history_list[3].history['loss'], '-.k', lw=2.0, label='ADAM')
plt.title('LeNet, CIFAR-100, Optimizers')
plt.xlabel('Epochs')
plt.ylabel('Cross-Entropy Training Loss')
plt.legend(loc='upper right')
plt.show()
#plt.savefig('./figures/lenet_loss.png')
plt.figure()
plt.plot(history_list[0].history['val_accuracy'], 'b', lw=2.0,
label='SGD')
plt.plot(history_list[1].history['val_accuracy'], '--r', lw=2.0,
label='SGD Nesterov')
plt.plot(history_list[2].history['val_accuracy'], ':g', lw=2.0,
label='RMSProp')
plt.plot(history_list[3].history['val_accuracy'], '-.k', lw=2.0,
label='ADAM')
plt.title('LeNet, CIFAR-100, Optimizers')
plt.xlabel('Epochs')
plt.ylabel('Validation Accuracy')
plt.legend(loc='upper right')
plt.show()
#plt.savefig('./figures/lenet_loss.png')
plt.figure()
plt.plot(history_list[0].history['lr'], 'b', lw=2.0, label='SGD')
plt.plot(history_list[1].history['lr'], '--r', lw=2.0, label='SGD
Nesterov')
plt.plot(history_list[2].history['lr'], ':g', lw=2.0, label='RMSProp')
plt.plot(history_list[3].history['lr'], '-.k', lw=2.0, label='ADAM')
plt.title('LeNet, CIFAR-100, Optimizers')
plt.xlabel('Epochs')
plt.ylabel('Learning Rate Schedule')
plt.legend(loc='upper right')
plt.show()
#plt.savefig('./figures/lenet_loss.png')
Use the neural-network optimizer listing to inspect the Keras/TensorFlow training setup used for comparing optimizers. section 10.4
