""" The :mod:`sklearn.kernel_approximation` module implements several approximate kernel feature maps based on Fourier transforms and Count Sketches. """ # Author: Andreas Mueller # Daniel Lopez-Sanchez (TensorSketch) # License: BSD 3 clause import warnings import numpy as np import scipy.sparse as sp from scipy.linalg import svd try: from scipy.fft import fft, ifft except ImportError: # scipy < 1.4 from scipy.fftpack import fft, ifft from .base import BaseEstimator from .base import TransformerMixin from .utils import check_random_state, as_float_array from .utils.extmath import safe_sparse_dot from .utils.validation import check_is_fitted from .metrics.pairwise import pairwise_kernels, KERNEL_PARAMS from .utils.validation import check_non_negative, _deprecate_positional_args class PolynomialCountSketch(BaseEstimator, TransformerMixin): """Polynomial kernel approximation via Tensor Sketch. Implements Tensor Sketch, which approximates the feature map of the polynomial kernel:: K(X, Y) = (gamma * + coef0)^degree by efficiently computing a Count Sketch of the outer product of a vector with itself using Fast Fourier Transforms (FFT). Read more in the :ref:`User Guide `. .. versionadded:: 0.24 Parameters ---------- gamma : float, default=1.0 Parameter of the polynomial kernel whose feature map will be approximated. degree : int, default=2 Degree of the polynomial kernel whose feature map will be approximated. coef0 : int, default=0 Constant term of the polynomial kernel whose feature map will be approximated. n_components : int, default=100 Dimensionality of the output feature space. Usually, n_components should be greater than the number of features in input samples in order to achieve good performance. The optimal score / run time balance is typically achieved around n_components = 10 * n_features, but this depends on the specific dataset being used. random_state : int, RandomState instance, default=None Determines random number generation for indexHash and bitHash initialization. Pass an int for reproducible results across multiple function calls. See :term:`Glossary `. Attributes ---------- indexHash_ : ndarray of shape (degree, n_features), dtype=int64 Array of indexes in range [0, n_components) used to represent the 2-wise independent hash functions for Count Sketch computation. bitHash_ : ndarray of shape (degree, n_features), dtype=float32 Array with random entries in {+1, -1}, used to represent the 2-wise independent hash functions for Count Sketch computation. Examples -------- >>> from sklearn.kernel_approximation import PolynomialCountSketch >>> from sklearn.linear_model import SGDClassifier >>> X = [[0, 0], [1, 1], [1, 0], [0, 1]] >>> y = [0, 0, 1, 1] >>> ps = PolynomialCountSketch(degree=3, random_state=1) >>> X_features = ps.fit_transform(X) >>> clf = SGDClassifier(max_iter=10, tol=1e-3) >>> clf.fit(X_features, y) SGDClassifier(max_iter=10) >>> clf.score(X_features, y) 1.0 """ def __init__(self, *, gamma=1., degree=2, coef0=0, n_components=100, random_state=None): self.gamma = gamma self.degree = degree self.coef0 = coef0 self.n_components = n_components self.random_state = random_state def fit(self, X, y=None): """Fit the model with X. Initializes the internal variables. The method needs no information about the distribution of data, so we only care about n_features in X. Parameters ---------- X : {array-like, sparse matrix} of shape (n_samples, n_features) Training data, where n_samples in the number of samples and n_features is the number of features. Returns ------- self : object Returns the transformer. """ if not self.degree >= 1: raise ValueError(f"degree={self.degree} should be >=1.") X = self._validate_data(X, accept_sparse="csc") random_state = check_random_state(self.random_state) n_features = X.shape[1] if self.coef0 != 0: n_features += 1 self.indexHash_ = random_state.randint(0, high=self.n_components, size=(self.degree, n_features)) self.bitHash_ = random_state.choice(a=[-1, 1], size=(self.degree, n_features)) return self def transform(self, X): """Generate the feature map approximation for X. Parameters ---------- X : {array-like}, shape (n_samples, n_features) New data, where n_samples in the number of samples and n_features is the number of features. Returns ------- X_new : array-like, shape (n_samples, n_components) """ check_is_fitted(self) X = self._validate_data(X, accept_sparse="csc", reset=False) X_gamma = np.sqrt(self.gamma) * X if sp.issparse(X_gamma) and self.coef0 != 0: X_gamma = sp.hstack([X_gamma, np.sqrt(self.coef0) * np.ones((X_gamma.shape[0], 1))], format="csc") elif not sp.issparse(X_gamma) and self.coef0 != 0: X_gamma = np.hstack([X_gamma, np.sqrt(self.coef0) * np.ones((X_gamma.shape[0], 1))]) if X_gamma.shape[1] != self.indexHash_.shape[1]: raise ValueError("Number of features of test samples does not" " match that of training samples.") count_sketches = np.zeros( (X_gamma.shape[0], self.degree, self.n_components)) if sp.issparse(X_gamma): for j in range(X_gamma.shape[1]): for d in range(self.degree): iHashIndex = self.indexHash_[d, j] iHashBit = self.bitHash_[d, j] count_sketches[:, d, iHashIndex] += \ (iHashBit * X_gamma[:, j]).toarray().ravel() else: for j in range(X_gamma.shape[1]): for d in range(self.degree): iHashIndex = self.indexHash_[d, j] iHashBit = self.bitHash_[d, j] count_sketches[:, d, iHashIndex] += \ iHashBit * X_gamma[:, j] # For each same, compute a count sketch of phi(x) using the polynomial # multiplication (via FFT) of p count sketches of x. count_sketches_fft = fft(count_sketches, axis=2, overwrite_x=True) count_sketches_fft_prod = np.prod(count_sketches_fft, axis=1) data_sketch = np.real(ifft(count_sketches_fft_prod, overwrite_x=True)) return data_sketch class RBFSampler(TransformerMixin, BaseEstimator): """Approximates feature map of an RBF kernel by Monte Carlo approximation of its Fourier transform. It implements a variant of Random Kitchen Sinks.[1] Read more in the :ref:`User Guide `. Parameters ---------- gamma : float, default=1.0 Parameter of RBF kernel: exp(-gamma * x^2) n_components : int, default=100 Number of Monte Carlo samples per original feature. Equals the dimensionality of the computed feature space. random_state : int, RandomState instance or None, default=None Pseudo-random number generator to control the generation of the random weights and random offset when fitting the training data. Pass an int for reproducible output across multiple function calls. See :term:`Glossary `. Attributes ---------- random_offset_ : ndarray of shape (n_components,), dtype=float64 Random offset used to compute the projection in the `n_components` dimensions of the feature space. random_weights_ : ndarray of shape (n_features, n_components),\ dtype=float64 Random projection directions drawn from the Fourier transform of the RBF kernel. Examples -------- >>> from sklearn.kernel_approximation import RBFSampler >>> from sklearn.linear_model import SGDClassifier >>> X = [[0, 0], [1, 1], [1, 0], [0, 1]] >>> y = [0, 0, 1, 1] >>> rbf_feature = RBFSampler(gamma=1, random_state=1) >>> X_features = rbf_feature.fit_transform(X) >>> clf = SGDClassifier(max_iter=5, tol=1e-3) >>> clf.fit(X_features, y) SGDClassifier(max_iter=5) >>> clf.score(X_features, y) 1.0 Notes ----- See "Random Features for Large-Scale Kernel Machines" by A. Rahimi and Benjamin Recht. [1] "Weighted Sums of Random Kitchen Sinks: Replacing minimization with randomization in learning" by A. Rahimi and Benjamin Recht. (https://people.eecs.berkeley.edu/~brecht/papers/08.rah.rec.nips.pdf) """ @_deprecate_positional_args def __init__(self, *, gamma=1., n_components=100, random_state=None): self.gamma = gamma self.n_components = n_components self.random_state = random_state def fit(self, X, y=None): """Fit the model with X. Samples random projection according to n_features. Parameters ---------- X : {array-like, sparse matrix}, shape (n_samples, n_features) Training data, where n_samples in the number of samples and n_features is the number of features. Returns ------- self : object Returns the transformer. """ X = self._validate_data(X, accept_sparse='csr') random_state = check_random_state(self.random_state) n_features = X.shape[1] self.random_weights_ = (np.sqrt(2 * self.gamma) * random_state.normal( size=(n_features, self.n_components))) self.random_offset_ = random_state.uniform(0, 2 * np.pi, size=self.n_components) return self def transform(self, X): """Apply the approximate feature map to X. Parameters ---------- X : {array-like, sparse matrix}, shape (n_samples, n_features) New data, where n_samples in the number of samples and n_features is the number of features. Returns ------- X_new : array-like, shape (n_samples, n_components) """ check_is_fitted(self) X = self._validate_data(X, accept_sparse='csr', reset=False) projection = safe_sparse_dot(X, self.random_weights_) projection += self.random_offset_ np.cos(projection, projection) projection *= np.sqrt(2.) / np.sqrt(self.n_components) return projection class SkewedChi2Sampler(TransformerMixin, BaseEstimator): """Approximates feature map of the "skewed chi-squared" kernel by Monte Carlo approximation of its Fourier transform. Read more in the :ref:`User Guide `. Parameters ---------- skewedness : float, default=1.0 "skewedness" parameter of the kernel. Needs to be cross-validated. n_components : int, default=100 number of Monte Carlo samples per original feature. Equals the dimensionality of the computed feature space. random_state : int, RandomState instance or None, default=None Pseudo-random number generator to control the generation of the random weights and random offset when fitting the training data. Pass an int for reproducible output across multiple function calls. See :term:`Glossary `. Attributes ---------- random_weights_ : ndarray of shape (n_features, n_components) Weight array, sampled from a secant hyperbolic distribution, which will be used to linearly transform the log of the data. random_offset_ : ndarray of shape (n_features, n_components) Bias term, which will be added to the data. It is uniformly distributed between 0 and 2*pi. Examples -------- >>> from sklearn.kernel_approximation import SkewedChi2Sampler >>> from sklearn.linear_model import SGDClassifier >>> X = [[0, 0], [1, 1], [1, 0], [0, 1]] >>> y = [0, 0, 1, 1] >>> chi2_feature = SkewedChi2Sampler(skewedness=.01, ... n_components=10, ... random_state=0) >>> X_features = chi2_feature.fit_transform(X, y) >>> clf = SGDClassifier(max_iter=10, tol=1e-3) >>> clf.fit(X_features, y) SGDClassifier(max_iter=10) >>> clf.score(X_features, y) 1.0 References ---------- See "Random Fourier Approximations for Skewed Multiplicative Histogram Kernels" by Fuxin Li, Catalin Ionescu and Cristian Sminchisescu. See Also -------- AdditiveChi2Sampler : A different approach for approximating an additive variant of the chi squared kernel. sklearn.metrics.pairwise.chi2_kernel : The exact chi squared kernel. """ @_deprecate_positional_args def __init__(self, *, skewedness=1., n_components=100, random_state=None): self.skewedness = skewedness self.n_components = n_components self.random_state = random_state def fit(self, X, y=None): """Fit the model with X. Samples random projection according to n_features. Parameters ---------- X : array-like, shape (n_samples, n_features) Training data, where n_samples in the number of samples and n_features is the number of features. Returns ------- self : object Returns the transformer. """ X = self._validate_data(X) random_state = check_random_state(self.random_state) n_features = X.shape[1] uniform = random_state.uniform(size=(n_features, self.n_components)) # transform by inverse CDF of sech self.random_weights_ = (1. / np.pi * np.log(np.tan(np.pi / 2. * uniform))) self.random_offset_ = random_state.uniform(0, 2 * np.pi, size=self.n_components) return self def transform(self, X): """Apply the approximate feature map to X. Parameters ---------- X : array-like, shape (n_samples, n_features) New data, where n_samples in the number of samples and n_features is the number of features. All values of X must be strictly greater than "-skewedness". Returns ------- X_new : array-like, shape (n_samples, n_components) """ check_is_fitted(self) X = as_float_array(X, copy=True) X = self._validate_data(X, copy=False, reset=False) if (X <= -self.skewedness).any(): raise ValueError("X may not contain entries smaller than" " -skewedness.") X += self.skewedness np.log(X, X) projection = safe_sparse_dot(X, self.random_weights_) projection += self.random_offset_ np.cos(projection, projection) projection *= np.sqrt(2.) / np.sqrt(self.n_components) return projection class AdditiveChi2Sampler(TransformerMixin, BaseEstimator): """Approximate feature map for additive chi2 kernel. Uses sampling the fourier transform of the kernel characteristic at regular intervals. Since the kernel that is to be approximated is additive, the components of the input vectors can be treated separately. Each entry in the original space is transformed into 2*sample_steps+1 features, where sample_steps is a parameter of the method. Typical values of sample_steps include 1, 2 and 3. Optimal choices for the sampling interval for certain data ranges can be computed (see the reference). The default values should be reasonable. Read more in the :ref:`User Guide `. Parameters ---------- sample_steps : int, default=2 Gives the number of (complex) sampling points. sample_interval : float, default=None Sampling interval. Must be specified when sample_steps not in {1,2,3}. Attributes ---------- sample_interval_ : float Stored sampling interval. Specified as a parameter if sample_steps not in {1,2,3}. Examples -------- >>> from sklearn.datasets import load_digits >>> from sklearn.linear_model import SGDClassifier >>> from sklearn.kernel_approximation import AdditiveChi2Sampler >>> X, y = load_digits(return_X_y=True) >>> chi2sampler = AdditiveChi2Sampler(sample_steps=2) >>> X_transformed = chi2sampler.fit_transform(X, y) >>> clf = SGDClassifier(max_iter=5, random_state=0, tol=1e-3) >>> clf.fit(X_transformed, y) SGDClassifier(max_iter=5, random_state=0) >>> clf.score(X_transformed, y) 0.9499... Notes ----- This estimator approximates a slightly different version of the additive chi squared kernel then ``metric.additive_chi2`` computes. See Also -------- SkewedChi2Sampler : A Fourier-approximation to a non-additive variant of the chi squared kernel. sklearn.metrics.pairwise.chi2_kernel : The exact chi squared kernel. sklearn.metrics.pairwise.additive_chi2_kernel : The exact additive chi squared kernel. References ---------- See `"Efficient additive kernels via explicit feature maps" `_ A. Vedaldi and A. Zisserman, Pattern Analysis and Machine Intelligence, 2011 """ @_deprecate_positional_args def __init__(self, *, sample_steps=2, sample_interval=None): self.sample_steps = sample_steps self.sample_interval = sample_interval def fit(self, X, y=None): """Set the parameters Parameters ---------- X : array-like, shape (n_samples, n_features) Training data, where n_samples in the number of samples and n_features is the number of features. Returns ------- self : object Returns the transformer. """ X = self._validate_data(X, accept_sparse='csr') check_non_negative(X, 'X in AdditiveChi2Sampler.fit') if self.sample_interval is None: # See reference, figure 2 c) if self.sample_steps == 1: self.sample_interval_ = 0.8 elif self.sample_steps == 2: self.sample_interval_ = 0.5 elif self.sample_steps == 3: self.sample_interval_ = 0.4 else: raise ValueError("If sample_steps is not in [1, 2, 3]," " you need to provide sample_interval") else: self.sample_interval_ = self.sample_interval return self def transform(self, X): """Apply approximate feature map to X. Parameters ---------- X : {array-like, sparse matrix} of shape (n_samples, n_features) Returns ------- X_new : {ndarray, sparse matrix}, \ shape = (n_samples, n_features * (2*sample_steps + 1)) Whether the return value is an array of sparse matrix depends on the type of the input X. """ msg = ("%(name)s is not fitted. Call fit to set the parameters before" " calling transform") check_is_fitted(self, msg=msg) X = self._validate_data(X, accept_sparse='csr', reset=False) check_non_negative(X, 'X in AdditiveChi2Sampler.transform') sparse = sp.issparse(X) # zeroth component # 1/cosh = sech # cosh(0) = 1.0 transf = self._transform_sparse if sparse else self._transform_dense return transf(X) def _transform_dense(self, X): non_zero = (X != 0.0) X_nz = X[non_zero] X_step = np.zeros_like(X) X_step[non_zero] = np.sqrt(X_nz * self.sample_interval_) X_new = [X_step] log_step_nz = self.sample_interval_ * np.log(X_nz) step_nz = 2 * X_nz * self.sample_interval_ for j in range(1, self.sample_steps): factor_nz = np.sqrt(step_nz / np.cosh(np.pi * j * self.sample_interval_)) X_step = np.zeros_like(X) X_step[non_zero] = factor_nz * np.cos(j * log_step_nz) X_new.append(X_step) X_step = np.zeros_like(X) X_step[non_zero] = factor_nz * np.sin(j * log_step_nz) X_new.append(X_step) return np.hstack(X_new) def _transform_sparse(self, X): indices = X.indices.copy() indptr = X.indptr.copy() data_step = np.sqrt(X.data * self.sample_interval_) X_step = sp.csr_matrix((data_step, indices, indptr), shape=X.shape, dtype=X.dtype, copy=False) X_new = [X_step] log_step_nz = self.sample_interval_ * np.log(X.data) step_nz = 2 * X.data * self.sample_interval_ for j in range(1, self.sample_steps): factor_nz = np.sqrt(step_nz / np.cosh(np.pi * j * self.sample_interval_)) data_step = factor_nz * np.cos(j * log_step_nz) X_step = sp.csr_matrix((data_step, indices, indptr), shape=X.shape, dtype=X.dtype, copy=False) X_new.append(X_step) data_step = factor_nz * np.sin(j * log_step_nz) X_step = sp.csr_matrix((data_step, indices, indptr), shape=X.shape, dtype=X.dtype, copy=False) X_new.append(X_step) return sp.hstack(X_new) def _more_tags(self): return {'stateless': True, 'requires_positive_X': True} class Nystroem(TransformerMixin, BaseEstimator): """Approximate a kernel map using a subset of the training data. Constructs an approximate feature map for an arbitrary kernel using a subset of the data as basis. Read more in the :ref:`User Guide `. .. versionadded:: 0.13 Parameters ---------- kernel : string or callable, default='rbf' Kernel map to be approximated. A callable should accept two arguments and the keyword arguments passed to this object as kernel_params, and should return a floating point number. gamma : float, default=None Gamma parameter for the RBF, laplacian, polynomial, exponential chi2 and sigmoid kernels. Interpretation of the default value is left to the kernel; see the documentation for sklearn.metrics.pairwise. Ignored by other kernels. coef0 : float, default=None Zero coefficient for polynomial and sigmoid kernels. Ignored by other kernels. degree : float, default=None Degree of the polynomial kernel. Ignored by other kernels. kernel_params : dict, default=None Additional parameters (keyword arguments) for kernel function passed as callable object. n_components : int, default=100 Number of features to construct. How many data points will be used to construct the mapping. random_state : int, RandomState instance or None, default=None Pseudo-random number generator to control the uniform sampling without replacement of n_components of the training data to construct the basis kernel. Pass an int for reproducible output across multiple function calls. See :term:`Glossary `. n_jobs : int, default=None The number of jobs to use for the computation. This works by breaking down the kernel matrix into n_jobs even slices and computing them in parallel. ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context. ``-1`` means using all processors. See :term:`Glossary ` for more details. .. versionadded:: 0.24 Attributes ---------- components_ : ndarray of shape (n_components, n_features) Subset of training points used to construct the feature map. component_indices_ : ndarray of shape (n_components) Indices of ``components_`` in the training set. normalization_ : ndarray of shape (n_components, n_components) Normalization matrix needed for embedding. Square root of the kernel matrix on ``components_``. Examples -------- >>> from sklearn import datasets, svm >>> from sklearn.kernel_approximation import Nystroem >>> X, y = datasets.load_digits(n_class=9, return_X_y=True) >>> data = X / 16. >>> clf = svm.LinearSVC() >>> feature_map_nystroem = Nystroem(gamma=.2, ... random_state=1, ... n_components=300) >>> data_transformed = feature_map_nystroem.fit_transform(data) >>> clf.fit(data_transformed, y) LinearSVC() >>> clf.score(data_transformed, y) 0.9987... References ---------- * Williams, C.K.I. and Seeger, M. "Using the Nystroem method to speed up kernel machines", Advances in neural information processing systems 2001 * T. Yang, Y. Li, M. Mahdavi, R. Jin and Z. Zhou "Nystroem Method vs Random Fourier Features: A Theoretical and Empirical Comparison", Advances in Neural Information Processing Systems 2012 See Also -------- RBFSampler : An approximation to the RBF kernel using random Fourier features. sklearn.metrics.pairwise.kernel_metrics : List of built-in kernels. """ @_deprecate_positional_args def __init__(self, kernel="rbf", *, gamma=None, coef0=None, degree=None, kernel_params=None, n_components=100, random_state=None, n_jobs=None): self.kernel = kernel self.gamma = gamma self.coef0 = coef0 self.degree = degree self.kernel_params = kernel_params self.n_components = n_components self.random_state = random_state self.n_jobs = n_jobs def fit(self, X, y=None): """Fit estimator to data. Samples a subset of training points, computes kernel on these and computes normalization matrix. Parameters ---------- X : array-like of shape (n_samples, n_features) Training data. """ X = self._validate_data(X, accept_sparse='csr') rnd = check_random_state(self.random_state) n_samples = X.shape[0] # get basis vectors if self.n_components > n_samples: # XXX should we just bail? n_components = n_samples warnings.warn("n_components > n_samples. This is not possible.\n" "n_components was set to n_samples, which results" " in inefficient evaluation of the full kernel.") else: n_components = self.n_components n_components = min(n_samples, n_components) inds = rnd.permutation(n_samples) basis_inds = inds[:n_components] basis = X[basis_inds] basis_kernel = pairwise_kernels(basis, metric=self.kernel, filter_params=True, n_jobs=self.n_jobs, **self._get_kernel_params()) # sqrt of kernel matrix on basis vectors U, S, V = svd(basis_kernel) S = np.maximum(S, 1e-12) self.normalization_ = np.dot(U / np.sqrt(S), V) self.components_ = basis self.component_indices_ = inds return self def transform(self, X): """Apply feature map to X. Computes an approximate feature map using the kernel between some training points and X. Parameters ---------- X : array-like of shape (n_samples, n_features) Data to transform. Returns ------- X_transformed : ndarray of shape (n_samples, n_components) Transformed data. """ check_is_fitted(self) X = self._validate_data(X, accept_sparse='csr', reset=False) kernel_params = self._get_kernel_params() embedded = pairwise_kernels(X, self.components_, metric=self.kernel, filter_params=True, n_jobs=self.n_jobs, **kernel_params) return np.dot(embedded, self.normalization_.T) def _get_kernel_params(self): params = self.kernel_params if params is None: params = {} if not callable(self.kernel) and self.kernel != 'precomputed': for param in (KERNEL_PARAMS[self.kernel]): if getattr(self, param) is not None: params[param] = getattr(self, param) else: if (self.gamma is not None or self.coef0 is not None or self.degree is not None): raise ValueError("Don't pass gamma, coef0 or degree to " "Nystroem if using a callable " "or precomputed kernel") return params def _more_tags(self): return { '_xfail_checks': { 'check_transformer_preserve_dtypes': 'dtypes are preserved but not at a close enough precision', }, 'preserves_dtype': [np.float64, np.float32] }