307 lines
12 KiB
Python
307 lines
12 KiB
Python
# Soft Voting/Majority Rule classifier
|
|
|
|
# Sebastian Raschka 2014-2020
|
|
# mlxtend Machine Learning Library Extensions
|
|
#
|
|
# Implementation of an meta-classification algorithm for majority voting.
|
|
# Author: Sebastian Raschka <sebastianraschka.com>
|
|
#
|
|
# License: BSD 3 clause
|
|
|
|
import numpy as np
|
|
import warnings
|
|
from sklearn.base import (BaseEstimator, ClassifierMixin, TransformerMixin,
|
|
clone)
|
|
from sklearn.exceptions import NotFittedError
|
|
from sklearn.preprocessing import LabelEncoder
|
|
|
|
from ..externals import six
|
|
from ..externals.name_estimators import _name_estimators
|
|
|
|
|
|
class EnsembleVoteClassifier(BaseEstimator, ClassifierMixin, TransformerMixin):
|
|
|
|
"""Soft Voting/Majority Rule classifier for scikit-learn estimators.
|
|
|
|
Parameters
|
|
----------
|
|
clfs : array-like, shape = [n_classifiers]
|
|
A list of classifiers.
|
|
Invoking the `fit` method on the `VotingClassifier` will fit clones
|
|
of those original classifiers
|
|
be stored in the class attribute
|
|
if `use_clones=True` (default) and
|
|
`fit_base_estimators=True` (default).
|
|
voting : str, {'hard', 'soft'} (default='hard')
|
|
If 'hard', uses predicted class labels for majority rule voting.
|
|
Else if 'soft', predicts the class label based on the argmax of
|
|
the sums of the predicted probalities, which is recommended for
|
|
an ensemble of well-calibrated classifiers.
|
|
weights : array-like, shape = [n_classifiers], optional (default=`None`)
|
|
Sequence of weights (`float` or `int`) to weight the occurances of
|
|
predicted class labels (`hard` voting) or class probabilities
|
|
before averaging (`soft` voting). Uses uniform weights if `None`.
|
|
verbose : int, optional (default=0)
|
|
Controls the verbosity of the building process.
|
|
- `verbose=0` (default): Prints nothing
|
|
- `verbose=1`: Prints the number & name of the clf being fitted
|
|
- `verbose=2`: Prints info about the parameters of the clf being fitted
|
|
- `verbose>2`: Changes `verbose` param of the underlying clf to
|
|
self.verbose - 2
|
|
use_clones : bool (default: True)
|
|
Clones the classifiers for stacking classification if True (default)
|
|
or else uses the original ones, which will be refitted on the dataset
|
|
upon calling the `fit` method. Hence, if use_clones=True, the original
|
|
input classifiers will remain unmodified upon using the
|
|
StackingClassifier's `fit` method.
|
|
Setting `use_clones=False` is
|
|
recommended if you are working with estimators that are supporting
|
|
the scikit-learn fit/predict API interface but are not compatible
|
|
to scikit-learn's `clone` function.
|
|
fit_base_estimators : bool (default: True)
|
|
Refits classifiers in `clfs` if True; uses references to the `clfs`,
|
|
otherwise (assumes that the classifiers were already fit).
|
|
Note: fit_base_estimators=False will enforce use_clones to be False,
|
|
and is incompatible to most scikit-learn wrappers!
|
|
For instance, if any form of cross-validation is performed
|
|
this would require the re-fitting classifiers to training folds, which
|
|
would raise a NotFitterError if fit_base_estimators=False.
|
|
(New in mlxtend v0.6.)
|
|
|
|
Attributes
|
|
----------
|
|
classes_ : array-like, shape = [n_predictions]
|
|
clf : array-like, shape = [n_predictions]
|
|
The input classifiers; may be overwritten if `use_clones=False`
|
|
clf_ : array-like, shape = [n_predictions]
|
|
Fitted input classifiers; clones if `use_clones=True`
|
|
|
|
Examples
|
|
--------
|
|
|
|
>>> import numpy as np
|
|
>>> from sklearn.linear_model import LogisticRegression
|
|
>>> from sklearn.naive_bayes import GaussianNB
|
|
>>> from sklearn.ensemble import RandomForestClassifier
|
|
>>> from mlxtend.sklearn import EnsembleVoteClassifier
|
|
>>> clf1 = LogisticRegression(random_seed=1)
|
|
>>> clf2 = RandomForestClassifier(random_seed=1)
|
|
>>> clf3 = GaussianNB()
|
|
>>> X = np.array([[-1, -1], [-2, -1], [-3, -2], [1, 1], [2, 1], [3, 2]])
|
|
>>> y = np.array([1, 1, 1, 2, 2, 2])
|
|
>>> eclf1 = EnsembleVoteClassifier(clfs=[clf1, clf2, clf3],
|
|
... voting='hard', verbose=1)
|
|
>>> eclf1 = eclf1.fit(X, y)
|
|
>>> print(eclf1.predict(X))
|
|
[1 1 1 2 2 2]
|
|
>>> eclf2 = EnsembleVoteClassifier(clfs=[clf1, clf2, clf3], voting='soft')
|
|
>>> eclf2 = eclf2.fit(X, y)
|
|
>>> print(eclf2.predict(X))
|
|
[1 1 1 2 2 2]
|
|
>>> eclf3 = EnsembleVoteClassifier(clfs=[clf1, clf2, clf3],
|
|
... voting='soft', weights=[2,1,1])
|
|
>>> eclf3 = eclf3.fit(X, y)
|
|
>>> print(eclf3.predict(X))
|
|
[1 1 1 2 2 2]
|
|
>>>
|
|
|
|
For more usage examples, please see
|
|
http://rasbt.github.io/mlxtend/user_guide/classifier/EnsembleVoteClassifier/
|
|
"""
|
|
|
|
def __init__(self, clfs, voting='hard',
|
|
weights=None, verbose=0,
|
|
use_clones=True,
|
|
fit_base_estimators=True):
|
|
|
|
self.clfs = clfs
|
|
self.named_clfs = {key: value for key, value in _name_estimators(clfs)}
|
|
self.voting = voting
|
|
self.weights = weights
|
|
self.verbose = verbose
|
|
self.use_clones = use_clones
|
|
self.fit_base_estimators = fit_base_estimators
|
|
|
|
def fit(self, X, y, sample_weight=None):
|
|
"""Learn weight coefficients from training data for each classifier.
|
|
|
|
Parameters
|
|
----------
|
|
X : {array-like, sparse matrix}, shape = [n_samples, n_features]
|
|
Training vectors, where n_samples is the number of samples and
|
|
n_features is the number of features.
|
|
|
|
y : array-like, shape = [n_samples]
|
|
Target values.
|
|
|
|
sample_weight : array-like, shape = [n_samples], optional
|
|
Sample weights passed as sample_weights to each regressor
|
|
in the regressors list as well as the meta_regressor.
|
|
Raises error if some regressor does not support
|
|
sample_weight in the fit() method.
|
|
|
|
Returns
|
|
-------
|
|
self : object
|
|
|
|
"""
|
|
if isinstance(y, np.ndarray) and len(y.shape) > 1 and y.shape[1] > 1:
|
|
raise NotImplementedError('Multilabel and multi-output'
|
|
' classification is not supported.')
|
|
|
|
if self.voting not in ('soft', 'hard'):
|
|
raise ValueError("Voting must be 'soft' or 'hard'; got (voting=%r)"
|
|
% self.voting)
|
|
|
|
if self.weights and len(self.weights) != len(self.clfs):
|
|
raise ValueError('Number of classifiers and weights must be equal'
|
|
'; got %d weights, %d clfs'
|
|
% (len(self.weights), len(self.clfs)))
|
|
|
|
self.le_ = LabelEncoder()
|
|
self.le_.fit(y)
|
|
self.classes_ = self.le_.classes_
|
|
|
|
if not self.fit_base_estimators:
|
|
warnings.warn("fit_base_estimators=False "
|
|
"enforces use_clones to be `False`")
|
|
self.use_clones = False
|
|
|
|
if self.use_clones:
|
|
self.clfs_ = clone(self.clfs)
|
|
else:
|
|
self.clfs_ = self.clfs
|
|
|
|
if self.fit_base_estimators:
|
|
if self.verbose > 0:
|
|
print("Fitting %d classifiers..." % (len(self.clfs)))
|
|
|
|
for clf in self.clfs_:
|
|
|
|
if self.verbose > 0:
|
|
i = self.clfs_.index(clf) + 1
|
|
print("Fitting clf%d: %s (%d/%d)" %
|
|
(i, _name_estimators((clf,))[0][0], i,
|
|
len(self.clfs_)))
|
|
|
|
if self.verbose > 2:
|
|
if hasattr(clf, 'verbose'):
|
|
clf.set_params(verbose=self.verbose - 2)
|
|
|
|
if self.verbose > 1:
|
|
print(_name_estimators((clf,))[0][1])
|
|
|
|
if sample_weight is None:
|
|
clf.fit(X, self.le_.transform(y))
|
|
else:
|
|
clf.fit(X, self.le_.transform(y),
|
|
sample_weight=sample_weight)
|
|
return self
|
|
|
|
def predict(self, X):
|
|
""" Predict class labels for X.
|
|
|
|
Parameters
|
|
----------
|
|
X : {array-like, sparse matrix}, shape = [n_samples, n_features]
|
|
Training vectors, where n_samples is the number of samples and
|
|
n_features is the number of features.
|
|
|
|
Returns
|
|
----------
|
|
maj : array-like, shape = [n_samples]
|
|
Predicted class labels.
|
|
|
|
"""
|
|
if not hasattr(self, 'clfs_'):
|
|
raise NotFittedError("Estimator not fitted, "
|
|
"call `fit` before exploiting the model.")
|
|
|
|
if self.voting == 'soft':
|
|
|
|
maj = np.argmax(self.predict_proba(X), axis=1)
|
|
|
|
else: # 'hard' voting
|
|
predictions = self._predict(X)
|
|
|
|
maj = np.apply_along_axis(lambda x:
|
|
np.argmax(np.bincount(
|
|
x, weights=self.weights)),
|
|
axis=1,
|
|
arr=predictions)
|
|
|
|
maj = self.le_.inverse_transform(maj)
|
|
return maj
|
|
|
|
def predict_proba(self, X):
|
|
""" Predict class probabilities for X.
|
|
|
|
Parameters
|
|
----------
|
|
X : {array-like, sparse matrix}, shape = [n_samples, n_features]
|
|
Training vectors, where n_samples is the number of samples and
|
|
n_features is the number of features.
|
|
|
|
Returns
|
|
----------
|
|
avg : array-like, shape = [n_samples, n_classes]
|
|
Weighted average probability for each class per sample.
|
|
|
|
"""
|
|
if not hasattr(self, 'clfs_'):
|
|
raise NotFittedError("Estimator not fitted, "
|
|
"call `fit` before exploiting the model.")
|
|
|
|
avg = np.average(self._predict_probas(X), axis=0, weights=self.weights)
|
|
return avg
|
|
|
|
def transform(self, X):
|
|
""" Return class labels or probabilities for X for each estimator.
|
|
|
|
Parameters
|
|
----------
|
|
X : {array-like, sparse matrix}, shape = [n_samples, n_features]
|
|
Training vectors, where n_samples is the number of samples and
|
|
n_features is the number of features.
|
|
|
|
Returns
|
|
-------
|
|
If `voting='soft'` : array-like = [n_classifiers, n_samples, n_classes]
|
|
Class probabilties calculated by each classifier.
|
|
If `voting='hard'` : array-like = [n_classifiers, n_samples]
|
|
Class labels predicted by each classifier.
|
|
|
|
"""
|
|
if self.voting == 'soft':
|
|
return self._predict_probas(X)
|
|
else:
|
|
return self._predict(X)
|
|
|
|
def get_params(self, deep=True):
|
|
"""Return estimator parameter names for GridSearch support."""
|
|
if not deep:
|
|
return super(EnsembleVoteClassifier, self).get_params(deep=False)
|
|
else:
|
|
out = self.named_clfs.copy()
|
|
for name, step in six.iteritems(self.named_clfs):
|
|
for key, value in six.iteritems(step.get_params(deep=True)):
|
|
out['%s__%s' % (name, key)] = value
|
|
|
|
for key, value in six.iteritems(
|
|
super(EnsembleVoteClassifier, self).get_params(deep=False)):
|
|
out['%s' % key] = value
|
|
return out
|
|
|
|
def _predict(self, X):
|
|
"""Collect results from clf.predict calls."""
|
|
|
|
if self.fit_base_estimators:
|
|
return np.asarray([clf.predict(X) for clf in self.clfs_]).T
|
|
else:
|
|
return np.asarray([self.le_.transform(clf.predict(X))
|
|
for clf in self.clfs_]).T
|
|
|
|
def _predict_probas(self, X):
|
|
"""Collect results from clf.predict_proba calls."""
|
|
return np.asarray([clf.predict_proba(X) for clf in self.clfs_])
|