# Authors: Gilles Louppe, Mathieu Blondel, Maheshakya Wijewardena # License: BSD 3 clause from copy import deepcopy import numpy as np from numbers import Integral, Real from ._base import SelectorMixin from ._base import _get_feature_importances from ..base import BaseEstimator, clone, MetaEstimatorMixin from ..utils._tags import _safe_tags from ..utils.validation import check_is_fitted, check_scalar, _num_features from ..utils._param_validation import HasMethods, Interval, Options from ..exceptions import NotFittedError from ..utils.metaestimators import available_if def _calculate_threshold(estimator, importances, threshold): """Interpret the threshold value""" if threshold is None: # determine default from estimator est_name = estimator.__class__.__name__ is_l1_penalized = hasattr(estimator, "penalty") and estimator.penalty == "l1" is_lasso = "Lasso" in est_name is_elasticnet_l1_penalized = "ElasticNet" in est_name and ( (hasattr(estimator, "l1_ratio_") and np.isclose(estimator.l1_ratio_, 1.0)) or (hasattr(estimator, "l1_ratio") and np.isclose(estimator.l1_ratio, 1.0)) ) if is_l1_penalized or is_lasso or is_elasticnet_l1_penalized: # the natural default threshold is 0 when l1 penalty was used threshold = 1e-5 else: threshold = "mean" if isinstance(threshold, str): if "*" in threshold: scale, reference = threshold.split("*") scale = float(scale.strip()) reference = reference.strip() if reference == "median": reference = np.median(importances) elif reference == "mean": reference = np.mean(importances) else: raise ValueError("Unknown reference: " + reference) threshold = scale * reference elif threshold == "median": threshold = np.median(importances) elif threshold == "mean": threshold = np.mean(importances) else: raise ValueError( "Expected threshold='mean' or threshold='median' got %s" % threshold ) else: threshold = float(threshold) return threshold def _estimator_has(attr): """Check if we can delegate a method to the underlying estimator. First, we check the fitted estimator if available, otherwise we check the unfitted estimator. """ return lambda self: ( hasattr(self.estimator_, attr) if hasattr(self, "estimator_") else hasattr(self.estimator, attr) ) class SelectFromModel(MetaEstimatorMixin, SelectorMixin, BaseEstimator): """Meta-transformer for selecting features based on importance weights. .. versionadded:: 0.17 Read more in the :ref:`User Guide `. Parameters ---------- estimator : object The base estimator from which the transformer is built. This can be both a fitted (if ``prefit`` is set to True) or a non-fitted estimator. The estimator should have a ``feature_importances_`` or ``coef_`` attribute after fitting. Otherwise, the ``importance_getter`` parameter should be used. threshold : str or float, default=None The threshold value to use for feature selection. Features whose absolute importance value is greater or equal are kept while the others are discarded. If "median" (resp. "mean"), then the ``threshold`` value is the median (resp. the mean) of the feature importances. A scaling factor (e.g., "1.25*mean") may also be used. If None and if the estimator has a parameter penalty set to l1, either explicitly or implicitly (e.g, Lasso), the threshold used is 1e-5. Otherwise, "mean" is used by default. prefit : bool, default=False Whether a prefit model is expected to be passed into the constructor directly or not. If `True`, `estimator` must be a fitted estimator. If `False`, `estimator` is fitted and updated by calling `fit` and `partial_fit`, respectively. norm_order : non-zero int, inf, -inf, default=1 Order of the norm used to filter the vectors of coefficients below ``threshold`` in the case where the ``coef_`` attribute of the estimator is of dimension 2. max_features : int, callable, default=None The maximum number of features to select. - If an integer, then it specifies the maximum number of features to allow. - If a callable, then it specifies how to calculate the maximum number of features allowed by using the output of `max_features(X)`. - If `None`, then all features are kept. To only select based on ``max_features``, set ``threshold=-np.inf``. .. versionadded:: 0.20 .. versionchanged:: 1.1 `max_features` accepts a callable. importance_getter : str or callable, default='auto' If 'auto', uses the feature importance either through a ``coef_`` attribute or ``feature_importances_`` attribute of estimator. Also accepts a string that specifies an attribute name/path for extracting feature importance (implemented with `attrgetter`). For example, give `regressor_.coef_` in case of :class:`~sklearn.compose.TransformedTargetRegressor` or `named_steps.clf.feature_importances_` in case of :class:`~sklearn.pipeline.Pipeline` with its last step named `clf`. If `callable`, overrides the default feature importance getter. The callable is passed with the fitted estimator and it should return importance for each feature. .. versionadded:: 0.24 Attributes ---------- estimator_ : estimator The base estimator from which the transformer is built. This attribute exist only when `fit` has been called. - If `prefit=True`, it is a deep copy of `estimator`. - If `prefit=False`, it is a clone of `estimator` and fit on the data passed to `fit` or `partial_fit`. n_features_in_ : int Number of features seen during :term:`fit`. Only defined if the underlying estimator exposes such an attribute when fit. .. versionadded:: 0.24 max_features_ : int Maximum number of features calculated during :term:`fit`. Only defined if the ``max_features`` is not `None`. - If `max_features` is an `int`, then `max_features_ = max_features`. - If `max_features` is a callable, then `max_features_ = max_features(X)`. .. versionadded:: 1.1 feature_names_in_ : ndarray of shape (`n_features_in_`,) Names of features seen during :term:`fit`. Defined only when `X` has feature names that are all strings. .. versionadded:: 1.0 threshold_ : float The threshold value used for feature selection. See Also -------- RFE : Recursive feature elimination based on importance weights. RFECV : Recursive feature elimination with built-in cross-validated selection of the best number of features. SequentialFeatureSelector : Sequential cross-validation based feature selection. Does not rely on importance weights. Notes ----- Allows NaN/Inf in the input if the underlying estimator does as well. Examples -------- >>> from sklearn.feature_selection import SelectFromModel >>> from sklearn.linear_model import LogisticRegression >>> X = [[ 0.87, -1.34, 0.31 ], ... [-2.79, -0.02, -0.85 ], ... [-1.34, -0.48, -2.55 ], ... [ 1.92, 1.48, 0.65 ]] >>> y = [0, 1, 0, 1] >>> selector = SelectFromModel(estimator=LogisticRegression()).fit(X, y) >>> selector.estimator_.coef_ array([[-0.3252302 , 0.83462377, 0.49750423]]) >>> selector.threshold_ 0.55245... >>> selector.get_support() array([False, True, False]) >>> selector.transform(X) array([[-1.34], [-0.02], [-0.48], [ 1.48]]) Using a callable to create a selector that can use no more than half of the input features. >>> def half_callable(X): ... return round(len(X[0]) / 2) >>> half_selector = SelectFromModel(estimator=LogisticRegression(), ... max_features=half_callable) >>> _ = half_selector.fit(X, y) >>> half_selector.max_features_ 2 """ _parameter_constraints: dict = { "estimator": [HasMethods("fit")], "threshold": [Interval(Real, None, None, closed="both"), str, None], "prefit": ["boolean"], "norm_order": [ Interval(Integral, None, -1, closed="right"), Interval(Integral, 1, None, closed="left"), Options(Real, {np.inf, -np.inf}), ], "max_features": [Interval(Integral, 0, None, closed="left"), callable, None], "importance_getter": [str, callable], } def __init__( self, estimator, *, threshold=None, prefit=False, norm_order=1, max_features=None, importance_getter="auto", ): self.estimator = estimator self.threshold = threshold self.prefit = prefit self.importance_getter = importance_getter self.norm_order = norm_order self.max_features = max_features def _get_support_mask(self): estimator = getattr(self, "estimator_", self.estimator) max_features = getattr(self, "max_features_", self.max_features) if self.prefit: try: check_is_fitted(self.estimator) except NotFittedError as exc: raise NotFittedError( "When `prefit=True`, `estimator` is expected to be a fitted " "estimator." ) from exc if callable(max_features): # This branch is executed when `transform` is called directly and thus # `max_features_` is not set and we fallback using `self.max_features` # that is not validated raise NotFittedError( "When `prefit=True` and `max_features` is a callable, call `fit` " "before calling `transform`." ) elif max_features is not None and not isinstance(max_features, Integral): raise ValueError( f"`max_features` must be an integer. Got `max_features={max_features}` " "instead." ) scores = _get_feature_importances( estimator=estimator, getter=self.importance_getter, transform_func="norm", norm_order=self.norm_order, ) threshold = _calculate_threshold(estimator, scores, self.threshold) if self.max_features is not None: mask = np.zeros_like(scores, dtype=bool) candidate_indices = np.argsort(-scores, kind="mergesort")[:max_features] mask[candidate_indices] = True else: mask = np.ones_like(scores, dtype=bool) mask[scores < threshold] = False return mask def _check_max_features(self, X): if self.max_features is not None: n_features = _num_features(X) if callable(self.max_features): max_features = self.max_features(X) else: # int max_features = self.max_features check_scalar( max_features, "max_features", Integral, min_val=0, max_val=n_features, ) self.max_features_ = max_features def fit(self, X, y=None, **fit_params): """Fit the SelectFromModel meta-transformer. Parameters ---------- X : array-like of shape (n_samples, n_features) The training input samples. y : array-like of shape (n_samples,), default=None The target values (integers that correspond to classes in classification, real numbers in regression). **fit_params : dict Other estimator specific parameters. Returns ------- self : object Fitted estimator. """ self._validate_params() self._check_max_features(X) if self.prefit: try: check_is_fitted(self.estimator) except NotFittedError as exc: raise NotFittedError( "When `prefit=True`, `estimator` is expected to be a fitted " "estimator." ) from exc self.estimator_ = deepcopy(self.estimator) else: self.estimator_ = clone(self.estimator) self.estimator_.fit(X, y, **fit_params) if hasattr(self.estimator_, "feature_names_in_"): self.feature_names_in_ = self.estimator_.feature_names_in_ else: self._check_feature_names(X, reset=True) return self @property def threshold_(self): """Threshold value used for feature selection.""" scores = _get_feature_importances( estimator=self.estimator_, getter=self.importance_getter, transform_func="norm", norm_order=self.norm_order, ) return _calculate_threshold(self.estimator, scores, self.threshold) @available_if(_estimator_has("partial_fit")) def partial_fit(self, X, y=None, **fit_params): """Fit the SelectFromModel meta-transformer only once. Parameters ---------- X : array-like of shape (n_samples, n_features) The training input samples. y : array-like of shape (n_samples,), default=None The target values (integers that correspond to classes in classification, real numbers in regression). **fit_params : dict Other estimator specific parameters. Returns ------- self : object Fitted estimator. """ first_call = not hasattr(self, "estimator_") if first_call: self._validate_params() self._check_max_features(X) if self.prefit: if first_call: try: check_is_fitted(self.estimator) except NotFittedError as exc: raise NotFittedError( "When `prefit=True`, `estimator` is expected to be a fitted " "estimator." ) from exc self.estimator_ = deepcopy(self.estimator) return self if first_call: self.estimator_ = clone(self.estimator) self.estimator_.partial_fit(X, y, **fit_params) if hasattr(self.estimator_, "feature_names_in_"): self.feature_names_in_ = self.estimator_.feature_names_in_ else: self._check_feature_names(X, reset=first_call) return self @property def n_features_in_(self): """Number of features seen during `fit`.""" # For consistency with other estimators we raise a AttributeError so # that hasattr() fails if the estimator isn't fitted. try: check_is_fitted(self) except NotFittedError as nfe: raise AttributeError( "{} object has no n_features_in_ attribute.".format( self.__class__.__name__ ) ) from nfe return self.estimator_.n_features_in_ def _more_tags(self): return {"allow_nan": _safe_tags(self.estimator, key="allow_nan")}