# Authors: Gilles Louppe, Mathieu Blondel, Maheshakya Wijewardena # License: BSD 3 clause import numpy as np from .base import SelectorMixin from ..base import TransformerMixin, BaseEstimator, clone from ..externals import six from ..utils import safe_mask, check_array, deprecated from ..utils.validation import check_is_fitted from ..exceptions import NotFittedError def _get_feature_importances(estimator): """Retrieve or aggregate feature importances from estimator""" if hasattr(estimator, "feature_importances_"): importances = estimator.feature_importances_ elif hasattr(estimator, "coef_"): if estimator.coef_.ndim == 1: importances = np.abs(estimator.coef_) else: importances = np.sum(np.abs(estimator.coef_), axis=0) else: raise ValueError( "The underlying estimator %s has no `coef_` or " "`feature_importances_` attribute. Either pass a fitted estimator" " to SelectFromModel or call fit before calling transform." % estimator.__class__.__name__) return importances def _calculate_threshold(estimator, importances, threshold): """Interpret the threshold value""" if threshold is None: # determine default from estimator est_name = estimator.__class__.__name__ if ((hasattr(estimator, "penalty") and estimator.penalty == "l1") or "Lasso" in est_name): # the natural default threshold is 0 when l1 penalty was used threshold = 1e-5 else: threshold = "mean" if isinstance(threshold, six.string_types): if "*" in threshold: scale, reference = threshold.split("*") scale = float(scale.strip()) reference = reference.strip() if reference == "median": reference = np.median(importances) elif reference == "mean": reference = np.mean(importances) else: raise ValueError("Unknown reference: " + reference) threshold = scale * reference elif threshold == "median": threshold = np.median(importances) elif threshold == "mean": threshold = np.mean(importances) else: raise ValueError("Expected threshold='mean' or threshold='median' " "got %s" % threshold) else: threshold = float(threshold) return threshold class _LearntSelectorMixin(TransformerMixin): # Note because of the extra threshold parameter in transform, this does # not naturally extend from SelectorMixin """Transformer mixin selecting features based on importance weights. This implementation can be mixin on any estimator that exposes a ``feature_importances_`` or ``coef_`` attribute to evaluate the relative importance of individual features for feature selection. """ @deprecated('Support to use estimators as feature selectors will be ' 'removed in version 0.19. Use SelectFromModel instead.') def transform(self, X, threshold=None): """Reduce X to its most important features. Uses ``coef_`` or ``feature_importances_`` to determine the most important features. For models with a ``coef_`` for each class, the absolute sum over the classes is used. Parameters ---------- X : array or scipy sparse matrix of shape [n_samples, n_features] The input samples. threshold : string, float or None, optional (default=None) The threshold value to use for feature selection. Features whose importance is greater or equal are kept while the others are discarded. If "median" (resp. "mean"), then the threshold value is the median (resp. the mean) of the feature importances. A scaling factor (e.g., "1.25*mean") may also be used. If None and if available, the object attribute ``threshold`` is used. Otherwise, "mean" is used by default. Returns ------- X_r : array of shape [n_samples, n_selected_features] The input samples with only the selected features. """ check_is_fitted(self, ('coef_', 'feature_importances_'), all_or_any=any) X = check_array(X, 'csc') importances = _get_feature_importances(self) if len(importances) != X.shape[1]: raise ValueError("X has different number of features than" " during model fitting.") if threshold is None: threshold = getattr(self, 'threshold', None) threshold = _calculate_threshold(self, importances, threshold) # Selection try: mask = importances >= threshold except TypeError: # Fails in Python 3.x when threshold is str; # result is array of True raise ValueError("Invalid threshold: all features are discarded.") if np.any(mask): mask = safe_mask(X, mask) return X[:, mask] else: raise ValueError("Invalid threshold: all features are discarded.") class SelectFromModel(BaseEstimator, SelectorMixin): """Meta-transformer for selecting features based on importance weights. .. versionadded:: 0.17 Parameters ---------- estimator : object The base estimator from which the transformer is built. This can be both a fitted (if ``prefit`` is set to True) or a non-fitted estimator. threshold : string, float, optional default None The threshold value to use for feature selection. Features whose importance is greater or equal are kept while the others are discarded. If "median" (resp. "mean"), then the ``threshold`` value is the median (resp. the mean) of the feature importances. A scaling factor (e.g., "1.25*mean") may also be used. If None and if the estimator has a parameter penalty set to l1, either explicitly or implicitly (e.g, Lasso), the threshold used is 1e-5. Otherwise, "mean" is used by default. prefit : bool, default False Whether a prefit model is expected to be passed into the constructor directly or not. If True, ``transform`` must be called directly and SelectFromModel cannot be used with ``cross_val_score``, ``GridSearchCV`` and similar utilities that clone the estimator. Otherwise train the model using ``fit`` and then ``transform`` to do feature selection. Attributes ---------- `estimator_`: an estimator The base estimator from which the transformer is built. This is stored only when a non-fitted estimator is passed to the ``SelectFromModel``, i.e when prefit is False. `threshold_`: float The threshold value used for feature selection. """ def __init__(self, estimator, threshold=None, prefit=False): self.estimator = estimator self.threshold = threshold self.prefit = prefit def _get_support_mask(self): # SelectFromModel can directly call on transform. if self.prefit: estimator = self.estimator elif hasattr(self, 'estimator_'): estimator = self.estimator_ else: raise ValueError( 'Either fit the model before transform or set "prefit=True"' ' while passing the fitted estimator to the constructor.') scores = _get_feature_importances(estimator) self.threshold_ = _calculate_threshold(estimator, scores, self.threshold) return scores >= self.threshold_ def fit(self, X, y=None, **fit_params): """Fit the SelectFromModel meta-transformer. Parameters ---------- X : array-like of shape (n_samples, n_features) The training input samples. y : array-like, shape (n_samples,) The target values (integers that correspond to classes in classification, real numbers in regression). **fit_params : Other estimator specific parameters Returns ------- self : object Returns self. """ if self.prefit: raise NotFittedError( "Since 'prefit=True', call transform directly") if not hasattr(self, "estimator_"): self.estimator_ = clone(self.estimator) self.estimator_.fit(X, y, **fit_params) return self def partial_fit(self, X, y=None, **fit_params): """Fit the SelectFromModel meta-transformer only once. Parameters ---------- X : array-like of shape (n_samples, n_features) The training input samples. y : array-like, shape (n_samples,) The target values (integers that correspond to classes in classification, real numbers in regression). **fit_params : Other estimator specific parameters Returns ------- self : object Returns self. """ if self.prefit: raise NotFittedError( "Since 'prefit=True', call transform directly") if not hasattr(self, "estimator_"): self.estimator_ = clone(self.estimator) self.estimator_.partial_fit(X, y, **fit_params) return self