An ensemble combines several machine-learning techniques into one predictive model. Its purpose may be to reduce variance through bagging, reduce bias through boosting, or improve predictions through stacking. section 7.5
Ensemble methods differ in how they generate and combine base learners:
- Sequential ensembles generate learners one after another and exploit dependence between them; AdaBoost is an example. section 7.5
- Parallel ensembles generate learners independently and reduce error through averaging; random forests are an example. section 7.5
- Homogeneous ensembles use one type of base learner, whereas heterogeneous ensembles combine different learner types. section 7.5
- Accurate and diverse base learners are needed for an ensemble to outperform its individual members. section 7.5
Bagging, or bootstrap aggregation, trains multiple base models on randomly sampled subsets of the data, with replacement. It combines classification outputs by voting and regression outputs by averaging. On iris data, decision-tree bagging improved accuracy as the ensemble grew, then plateaued at approximately 20 estimators; adding more increased computational complexity without improving accuracy. section 7.5.1
Stacking on iris data
A stacking classifier combines KNN, random forest, and naive Bayes predictions with logistic regression as a meta-classifier. The base models provide features for the meta-model, producing blended decision boundaries and higher accuracy than the individual classifiers in the illustrated experiment. section 7.5.3
Listing 7.8 Bagging ensemble
import itertools
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
import matplotlib.gridspec as gridspec
from sklearn import datasets
from sklearn.tree import DecisionTreeClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble import BaggingClassifier
from sklearn.model_selection import cross_val_score, train_test_split
from mlxtend.plotting import plot_learning_curves
from mlxtend.plotting import plot_decision_regions
def main():
iris = datasets.load_iris()
X, y = iris.data[:, 0:2], iris.target
clf1 = DecisionTreeClassifier(criterion='entropy',
max_depth=None) #1
clf2 = KNeighborsClassifier(n_neighbors=1) #2
bagging1 = BaggingClassifier(base_estimator=clf1, n_estimators=10,
max_samples=0.8, max_features=0.8)
bagging2 = BaggingClassifier(base_estimator=clf2, n_estimators=10,
max_samples=0.8, max_features=0.8)
label = ['Decision Tree', 'K-NN', 'Bagging Tree', 'Bagging K-NN']
clf_list = [clf1, clf2, bagging1, bagging2]
fig = plt.figure(figsize=(10, 8))
gs = gridspec.GridSpec(2, 2)
grid = itertools.product([0,1],repeat=2)
for clf, label, grd in zip(clf_list, label, grid):
scores = cross_val_score(clf, X, y, cv=3, scoring='accuracy')
print("Accuracy: %.2f (+/- %.2f) [%s]" %(scores.mean(),
scores.std(), label))
clf.fit(X, y)
ax = plt.subplot(gs[grd[0], grd[1]])
fig = plot_decision_regions(X=X, y=y, clf=clf, legend=2)
plt.title(label)
plt.show()
#plt.savefig('./figures/bagging_ensemble.png')
#plot learning curves
X_train, X_test, y_train, y_test = train_test_split(X, y,
test_size=0.3, random_state=0)
plt.figure()
plot_learning_curves(X_train, y_train, X_test, y_test, bagging1,
print_model=False, style='ggplot')
plt.show()
#plt.savefig('./figures/bagging_ensemble_learning_curve.png')
#Ensemble Size
num_est = list(map(int, np.linspace(1,100,20)))
bg_clf_cv_mean = []
bg_clf_cv_std = []
for n_est in num_est:
print("num_est: ", n_est)
bg_clf = BaggingClassifier(base_estimator=clf1, n_estimators=n_est,
max_samples=0.8, max_features=0.8)
scores = cross_val_score(bg_clf, X, y, cv=3, scoring='accuracy')
bg_clf_cv_mean.append(scores.mean())
bg_clf_cv_std.append(scores.std())
plt.figure()
(_, caps, _) = plt.errorbar(num_est, bg_clf_cv_mean,
yerr=bg_clf_cv_std, c='blue', fmt='-o', capsize=5)
for cap in caps:
cap.set_markeredgewidth(1)
plt.ylabel('Accuracy'); plt.xlabel('Ensemble Size'); plt.title('Bagging
Tree Ensemble');
plt.show()
#plt.savefig('./figures/bagging_ensemble_size.png')
if __name__ == "__main__":
main()
Inspect the bagging implementation and its iris experiments, including base estimators, subsampling, cross-validation, decision regions, and learning curves. section 7.5.1
Listing 7.10 Stacking ensemble
import itertools
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
import matplotlib.gridspec as gridspec
from sklearn import datasets
from sklearn.linear_model import LogisticRegression
from sklearn.neighbors import KNeighborsClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.ensemble import RandomForestClassifier
from mlxtend.classifier import StackingClassifier
from sklearn.model_selection import cross_val_score, train_test_split
from mlxtend.plotting import plot_learning_curves
from mlxtend.plotting import plot_decision_regions
def main():
iris = datasets.load_iris()
X, y = iris.data[:, 1:3], iris.target
clf1 = KNeighborsClassifier(n_neighbors=1)
clf2 = RandomForestClassifier(random_state=1)
clf3 = GaussianNB()
lr = LogisticRegression()
sclf = StackingClassifier(classifiers=[clf1, clf2, clf3],
meta_classifier=lr) #1
label = ['KNN', 'Random Forest', 'Naive Bayes', 'Stacking Classifier']
clf_list = [clf1, clf2, clf3, sclf]
fig = plt.figure(figsize=(10,8))
gs = gridspec.GridSpec(2, 2)
grid = itertools.product([0,1],repeat=2)
clf_cv_mean = []
clf_cv_std = []
for clf, label, grd in zip(clf_list, label, grid):
scores = cross_val_score(clf, X, y, cv=3, scoring='accuracy')
print("Accuracy: %.2f (+/- %.2f) [%s]" %(scores.mean(),
scores.std(), label))
clf_cv_mean.append(scores.mean())
clf_cv_std.append(scores.std())
clf.fit(X, y)
ax = plt.subplot(gs[grd[0], grd[1]])
fig = plot_decision_regions(X=X, y=y, clf=clf)
plt.title(label)
plt.show()
#plt.savefig("./figures/ensemble_stacking.png")
#plot classifier accuracy
plt.figure()
(_, caps, _) = plt.errorbar(range(4), clf_cv_mean, yerr=clf_cv_std,
c='blue', fmt='-o', capsize=5)
for cap in caps:
cap.set_markeredgewidth(1)
plt.xticks(range(4), ['KNN', 'RF', 'NB', 'Stacking'], rotation='vertical')
plt.ylabel('Accuracy'); plt.xlabel('Classifier');
plt.title('Stacking Ensemble');
plt.show()
#plt.savefig('./figures/stacking_ensemble_size.png')
#plot learning curves
X_train, X_test, y_train, y_test = train_test_split(X, y,
test_size=0.3, random_state=0)
plt.figure()
plot_learning_curves(X_train, y_train, X_test, y_test, sclf,
print_model=False, style='ggplot')
plt.show()
#plt.savefig('./figures/stacking_ensemble_learning_curve.png')
if __name__ == "__main__":
main()
Inspect the stacking implementation combining KNN, random forest, naive Bayes, and logistic regression. section 7.5.3
