2162 lines
72 KiB
Python
2162 lines
72 KiB
Python
"""
|
|
Test the ColumnTransformer.
|
|
"""
|
|
import re
|
|
import pickle
|
|
|
|
import numpy as np
|
|
from scipy import sparse
|
|
import pytest
|
|
|
|
from numpy.testing import assert_allclose
|
|
from sklearn.utils._testing import assert_array_equal
|
|
from sklearn.utils._testing import assert_allclose_dense_sparse
|
|
from sklearn.utils._testing import assert_almost_equal
|
|
|
|
from sklearn.base import BaseEstimator, TransformerMixin
|
|
from sklearn.compose import (
|
|
ColumnTransformer,
|
|
make_column_transformer,
|
|
make_column_selector,
|
|
)
|
|
from sklearn.exceptions import NotFittedError
|
|
from sklearn.preprocessing import FunctionTransformer
|
|
from sklearn.preprocessing import StandardScaler, Normalizer, OneHotEncoder
|
|
|
|
|
|
class Trans(TransformerMixin, BaseEstimator):
|
|
def fit(self, X, y=None):
|
|
return self
|
|
|
|
def transform(self, X, y=None):
|
|
# 1D Series -> 2D DataFrame
|
|
if hasattr(X, "to_frame"):
|
|
return X.to_frame()
|
|
# 1D array -> 2D array
|
|
if X.ndim == 1:
|
|
return np.atleast_2d(X).T
|
|
return X
|
|
|
|
|
|
class DoubleTrans(BaseEstimator):
|
|
def fit(self, X, y=None):
|
|
return self
|
|
|
|
def transform(self, X):
|
|
return 2 * X
|
|
|
|
|
|
class SparseMatrixTrans(BaseEstimator):
|
|
def fit(self, X, y=None):
|
|
return self
|
|
|
|
def transform(self, X, y=None):
|
|
n_samples = len(X)
|
|
return sparse.eye(n_samples, n_samples).tocsr()
|
|
|
|
|
|
class TransNo2D(BaseEstimator):
|
|
def fit(self, X, y=None):
|
|
return self
|
|
|
|
def transform(self, X, y=None):
|
|
return X
|
|
|
|
|
|
class TransRaise(BaseEstimator):
|
|
def fit(self, X, y=None):
|
|
raise ValueError("specific message")
|
|
|
|
def transform(self, X, y=None):
|
|
raise ValueError("specific message")
|
|
|
|
|
|
def test_column_transformer():
|
|
X_array = np.array([[0, 1, 2], [2, 4, 6]]).T
|
|
|
|
X_res_first1D = np.array([0, 1, 2])
|
|
X_res_second1D = np.array([2, 4, 6])
|
|
X_res_first = X_res_first1D.reshape(-1, 1)
|
|
X_res_both = X_array
|
|
|
|
cases = [
|
|
# single column 1D / 2D
|
|
(0, X_res_first),
|
|
([0], X_res_first),
|
|
# list-like
|
|
([0, 1], X_res_both),
|
|
(np.array([0, 1]), X_res_both),
|
|
# slice
|
|
(slice(0, 1), X_res_first),
|
|
(slice(0, 2), X_res_both),
|
|
# boolean mask
|
|
(np.array([True, False]), X_res_first),
|
|
([True, False], X_res_first),
|
|
(np.array([True, True]), X_res_both),
|
|
([True, True], X_res_both),
|
|
]
|
|
|
|
for selection, res in cases:
|
|
ct = ColumnTransformer([("trans", Trans(), selection)], remainder="drop")
|
|
assert_array_equal(ct.fit_transform(X_array), res)
|
|
assert_array_equal(ct.fit(X_array).transform(X_array), res)
|
|
|
|
# callable that returns any of the allowed specifiers
|
|
ct = ColumnTransformer(
|
|
[("trans", Trans(), lambda x: selection)], remainder="drop"
|
|
)
|
|
assert_array_equal(ct.fit_transform(X_array), res)
|
|
assert_array_equal(ct.fit(X_array).transform(X_array), res)
|
|
|
|
ct = ColumnTransformer([("trans1", Trans(), [0]), ("trans2", Trans(), [1])])
|
|
assert_array_equal(ct.fit_transform(X_array), X_res_both)
|
|
assert_array_equal(ct.fit(X_array).transform(X_array), X_res_both)
|
|
assert len(ct.transformers_) == 2
|
|
|
|
# test with transformer_weights
|
|
transformer_weights = {"trans1": 0.1, "trans2": 10}
|
|
both = ColumnTransformer(
|
|
[("trans1", Trans(), [0]), ("trans2", Trans(), [1])],
|
|
transformer_weights=transformer_weights,
|
|
)
|
|
res = np.vstack(
|
|
[
|
|
transformer_weights["trans1"] * X_res_first1D,
|
|
transformer_weights["trans2"] * X_res_second1D,
|
|
]
|
|
).T
|
|
assert_array_equal(both.fit_transform(X_array), res)
|
|
assert_array_equal(both.fit(X_array).transform(X_array), res)
|
|
assert len(both.transformers_) == 2
|
|
|
|
both = ColumnTransformer(
|
|
[("trans", Trans(), [0, 1])], transformer_weights={"trans": 0.1}
|
|
)
|
|
assert_array_equal(both.fit_transform(X_array), 0.1 * X_res_both)
|
|
assert_array_equal(both.fit(X_array).transform(X_array), 0.1 * X_res_both)
|
|
assert len(both.transformers_) == 1
|
|
|
|
|
|
def test_column_transformer_dataframe():
|
|
pd = pytest.importorskip("pandas")
|
|
|
|
X_array = np.array([[0, 1, 2], [2, 4, 6]]).T
|
|
X_df = pd.DataFrame(X_array, columns=["first", "second"])
|
|
|
|
X_res_first = np.array([0, 1, 2]).reshape(-1, 1)
|
|
X_res_both = X_array
|
|
|
|
cases = [
|
|
# String keys: label based
|
|
# scalar
|
|
("first", X_res_first),
|
|
# list
|
|
(["first"], X_res_first),
|
|
(["first", "second"], X_res_both),
|
|
# slice
|
|
(slice("first", "second"), X_res_both),
|
|
# int keys: positional
|
|
# scalar
|
|
(0, X_res_first),
|
|
# list
|
|
([0], X_res_first),
|
|
([0, 1], X_res_both),
|
|
(np.array([0, 1]), X_res_both),
|
|
# slice
|
|
(slice(0, 1), X_res_first),
|
|
(slice(0, 2), X_res_both),
|
|
# boolean mask
|
|
(np.array([True, False]), X_res_first),
|
|
(pd.Series([True, False], index=["first", "second"]), X_res_first),
|
|
([True, False], X_res_first),
|
|
]
|
|
|
|
for selection, res in cases:
|
|
ct = ColumnTransformer([("trans", Trans(), selection)], remainder="drop")
|
|
assert_array_equal(ct.fit_transform(X_df), res)
|
|
assert_array_equal(ct.fit(X_df).transform(X_df), res)
|
|
|
|
# callable that returns any of the allowed specifiers
|
|
ct = ColumnTransformer(
|
|
[("trans", Trans(), lambda X: selection)], remainder="drop"
|
|
)
|
|
assert_array_equal(ct.fit_transform(X_df), res)
|
|
assert_array_equal(ct.fit(X_df).transform(X_df), res)
|
|
|
|
ct = ColumnTransformer(
|
|
[("trans1", Trans(), ["first"]), ("trans2", Trans(), ["second"])]
|
|
)
|
|
assert_array_equal(ct.fit_transform(X_df), X_res_both)
|
|
assert_array_equal(ct.fit(X_df).transform(X_df), X_res_both)
|
|
assert len(ct.transformers_) == 2
|
|
assert ct.transformers_[-1][0] != "remainder"
|
|
|
|
ct = ColumnTransformer([("trans1", Trans(), [0]), ("trans2", Trans(), [1])])
|
|
assert_array_equal(ct.fit_transform(X_df), X_res_both)
|
|
assert_array_equal(ct.fit(X_df).transform(X_df), X_res_both)
|
|
assert len(ct.transformers_) == 2
|
|
assert ct.transformers_[-1][0] != "remainder"
|
|
|
|
# test with transformer_weights
|
|
transformer_weights = {"trans1": 0.1, "trans2": 10}
|
|
both = ColumnTransformer(
|
|
[("trans1", Trans(), ["first"]), ("trans2", Trans(), ["second"])],
|
|
transformer_weights=transformer_weights,
|
|
)
|
|
res = np.vstack(
|
|
[
|
|
transformer_weights["trans1"] * X_df["first"],
|
|
transformer_weights["trans2"] * X_df["second"],
|
|
]
|
|
).T
|
|
assert_array_equal(both.fit_transform(X_df), res)
|
|
assert_array_equal(both.fit(X_df).transform(X_df), res)
|
|
assert len(both.transformers_) == 2
|
|
assert both.transformers_[-1][0] != "remainder"
|
|
|
|
# test multiple columns
|
|
both = ColumnTransformer(
|
|
[("trans", Trans(), ["first", "second"])], transformer_weights={"trans": 0.1}
|
|
)
|
|
assert_array_equal(both.fit_transform(X_df), 0.1 * X_res_both)
|
|
assert_array_equal(both.fit(X_df).transform(X_df), 0.1 * X_res_both)
|
|
assert len(both.transformers_) == 1
|
|
assert both.transformers_[-1][0] != "remainder"
|
|
|
|
both = ColumnTransformer(
|
|
[("trans", Trans(), [0, 1])], transformer_weights={"trans": 0.1}
|
|
)
|
|
assert_array_equal(both.fit_transform(X_df), 0.1 * X_res_both)
|
|
assert_array_equal(both.fit(X_df).transform(X_df), 0.1 * X_res_both)
|
|
assert len(both.transformers_) == 1
|
|
assert both.transformers_[-1][0] != "remainder"
|
|
|
|
# ensure pandas object is passed through
|
|
|
|
class TransAssert(BaseEstimator):
|
|
def fit(self, X, y=None):
|
|
return self
|
|
|
|
def transform(self, X, y=None):
|
|
assert isinstance(X, (pd.DataFrame, pd.Series))
|
|
if isinstance(X, pd.Series):
|
|
X = X.to_frame()
|
|
return X
|
|
|
|
ct = ColumnTransformer([("trans", TransAssert(), "first")], remainder="drop")
|
|
ct.fit_transform(X_df)
|
|
ct = ColumnTransformer([("trans", TransAssert(), ["first", "second"])])
|
|
ct.fit_transform(X_df)
|
|
|
|
# integer column spec + integer column names -> still use positional
|
|
X_df2 = X_df.copy()
|
|
X_df2.columns = [1, 0]
|
|
ct = ColumnTransformer([("trans", Trans(), 0)], remainder="drop")
|
|
assert_array_equal(ct.fit_transform(X_df2), X_res_first)
|
|
assert_array_equal(ct.fit(X_df2).transform(X_df2), X_res_first)
|
|
|
|
assert len(ct.transformers_) == 2
|
|
assert ct.transformers_[-1][0] == "remainder"
|
|
assert ct.transformers_[-1][1] == "drop"
|
|
assert_array_equal(ct.transformers_[-1][2], [1])
|
|
|
|
|
|
@pytest.mark.parametrize("pandas", [True, False], ids=["pandas", "numpy"])
|
|
@pytest.mark.parametrize(
|
|
"column_selection",
|
|
[[], np.array([False, False]), [False, False]],
|
|
ids=["list", "bool", "bool_int"],
|
|
)
|
|
@pytest.mark.parametrize("callable_column", [False, True])
|
|
def test_column_transformer_empty_columns(pandas, column_selection, callable_column):
|
|
# test case that ensures that the column transformer does also work when
|
|
# a given transformer doesn't have any columns to work on
|
|
X_array = np.array([[0, 1, 2], [2, 4, 6]]).T
|
|
X_res_both = X_array
|
|
|
|
if pandas:
|
|
pd = pytest.importorskip("pandas")
|
|
X = pd.DataFrame(X_array, columns=["first", "second"])
|
|
else:
|
|
X = X_array
|
|
|
|
if callable_column:
|
|
column = lambda X: column_selection # noqa
|
|
else:
|
|
column = column_selection
|
|
|
|
ct = ColumnTransformer(
|
|
[("trans1", Trans(), [0, 1]), ("trans2", TransRaise(), column)]
|
|
)
|
|
assert_array_equal(ct.fit_transform(X), X_res_both)
|
|
assert_array_equal(ct.fit(X).transform(X), X_res_both)
|
|
assert len(ct.transformers_) == 2
|
|
assert isinstance(ct.transformers_[1][1], TransRaise)
|
|
|
|
ct = ColumnTransformer(
|
|
[("trans1", TransRaise(), column), ("trans2", Trans(), [0, 1])]
|
|
)
|
|
assert_array_equal(ct.fit_transform(X), X_res_both)
|
|
assert_array_equal(ct.fit(X).transform(X), X_res_both)
|
|
assert len(ct.transformers_) == 2
|
|
assert isinstance(ct.transformers_[0][1], TransRaise)
|
|
|
|
ct = ColumnTransformer([("trans", TransRaise(), column)], remainder="passthrough")
|
|
assert_array_equal(ct.fit_transform(X), X_res_both)
|
|
assert_array_equal(ct.fit(X).transform(X), X_res_both)
|
|
assert len(ct.transformers_) == 2 # including remainder
|
|
assert isinstance(ct.transformers_[0][1], TransRaise)
|
|
|
|
fixture = np.array([[], [], []])
|
|
ct = ColumnTransformer([("trans", TransRaise(), column)], remainder="drop")
|
|
assert_array_equal(ct.fit_transform(X), fixture)
|
|
assert_array_equal(ct.fit(X).transform(X), fixture)
|
|
assert len(ct.transformers_) == 2 # including remainder
|
|
assert isinstance(ct.transformers_[0][1], TransRaise)
|
|
|
|
|
|
def test_column_transformer_output_indices():
|
|
# Checks for the output_indices_ attribute
|
|
X_array = np.arange(6).reshape(3, 2)
|
|
|
|
ct = ColumnTransformer([("trans1", Trans(), [0]), ("trans2", Trans(), [1])])
|
|
X_trans = ct.fit_transform(X_array)
|
|
assert ct.output_indices_ == {
|
|
"trans1": slice(0, 1),
|
|
"trans2": slice(1, 2),
|
|
"remainder": slice(0, 0),
|
|
}
|
|
assert_array_equal(X_trans[:, [0]], X_trans[:, ct.output_indices_["trans1"]])
|
|
assert_array_equal(X_trans[:, [1]], X_trans[:, ct.output_indices_["trans2"]])
|
|
|
|
# test with transformer_weights and multiple columns
|
|
ct = ColumnTransformer(
|
|
[("trans", Trans(), [0, 1])], transformer_weights={"trans": 0.1}
|
|
)
|
|
X_trans = ct.fit_transform(X_array)
|
|
assert ct.output_indices_ == {"trans": slice(0, 2), "remainder": slice(0, 0)}
|
|
assert_array_equal(X_trans[:, [0, 1]], X_trans[:, ct.output_indices_["trans"]])
|
|
assert_array_equal(X_trans[:, []], X_trans[:, ct.output_indices_["remainder"]])
|
|
|
|
# test case that ensures that the attribute does also work when
|
|
# a given transformer doesn't have any columns to work on
|
|
ct = ColumnTransformer([("trans1", Trans(), [0, 1]), ("trans2", TransRaise(), [])])
|
|
X_trans = ct.fit_transform(X_array)
|
|
assert ct.output_indices_ == {
|
|
"trans1": slice(0, 2),
|
|
"trans2": slice(0, 0),
|
|
"remainder": slice(0, 0),
|
|
}
|
|
assert_array_equal(X_trans[:, [0, 1]], X_trans[:, ct.output_indices_["trans1"]])
|
|
assert_array_equal(X_trans[:, []], X_trans[:, ct.output_indices_["trans2"]])
|
|
assert_array_equal(X_trans[:, []], X_trans[:, ct.output_indices_["remainder"]])
|
|
|
|
ct = ColumnTransformer([("trans", TransRaise(), [])], remainder="passthrough")
|
|
X_trans = ct.fit_transform(X_array)
|
|
assert ct.output_indices_ == {"trans": slice(0, 0), "remainder": slice(0, 2)}
|
|
assert_array_equal(X_trans[:, []], X_trans[:, ct.output_indices_["trans"]])
|
|
assert_array_equal(X_trans[:, [0, 1]], X_trans[:, ct.output_indices_["remainder"]])
|
|
|
|
|
|
def test_column_transformer_output_indices_df():
|
|
# Checks for the output_indices_ attribute with data frames
|
|
pd = pytest.importorskip("pandas")
|
|
|
|
X_df = pd.DataFrame(np.arange(6).reshape(3, 2), columns=["first", "second"])
|
|
|
|
ct = ColumnTransformer(
|
|
[("trans1", Trans(), ["first"]), ("trans2", Trans(), ["second"])]
|
|
)
|
|
X_trans = ct.fit_transform(X_df)
|
|
assert ct.output_indices_ == {
|
|
"trans1": slice(0, 1),
|
|
"trans2": slice(1, 2),
|
|
"remainder": slice(0, 0),
|
|
}
|
|
assert_array_equal(X_trans[:, [0]], X_trans[:, ct.output_indices_["trans1"]])
|
|
assert_array_equal(X_trans[:, [1]], X_trans[:, ct.output_indices_["trans2"]])
|
|
assert_array_equal(X_trans[:, []], X_trans[:, ct.output_indices_["remainder"]])
|
|
|
|
ct = ColumnTransformer([("trans1", Trans(), [0]), ("trans2", Trans(), [1])])
|
|
X_trans = ct.fit_transform(X_df)
|
|
assert ct.output_indices_ == {
|
|
"trans1": slice(0, 1),
|
|
"trans2": slice(1, 2),
|
|
"remainder": slice(0, 0),
|
|
}
|
|
assert_array_equal(X_trans[:, [0]], X_trans[:, ct.output_indices_["trans1"]])
|
|
assert_array_equal(X_trans[:, [1]], X_trans[:, ct.output_indices_["trans2"]])
|
|
assert_array_equal(X_trans[:, []], X_trans[:, ct.output_indices_["remainder"]])
|
|
|
|
|
|
def test_column_transformer_sparse_array():
|
|
X_sparse = sparse.eye(3, 2).tocsr()
|
|
|
|
# no distinction between 1D and 2D
|
|
X_res_first = X_sparse[:, 0]
|
|
X_res_both = X_sparse
|
|
|
|
for col in [0, [0], slice(0, 1)]:
|
|
for remainder, res in [("drop", X_res_first), ("passthrough", X_res_both)]:
|
|
ct = ColumnTransformer(
|
|
[("trans", Trans(), col)], remainder=remainder, sparse_threshold=0.8
|
|
)
|
|
assert sparse.issparse(ct.fit_transform(X_sparse))
|
|
assert_allclose_dense_sparse(ct.fit_transform(X_sparse), res)
|
|
assert_allclose_dense_sparse(ct.fit(X_sparse).transform(X_sparse), res)
|
|
|
|
for col in [[0, 1], slice(0, 2)]:
|
|
ct = ColumnTransformer([("trans", Trans(), col)], sparse_threshold=0.8)
|
|
assert sparse.issparse(ct.fit_transform(X_sparse))
|
|
assert_allclose_dense_sparse(ct.fit_transform(X_sparse), X_res_both)
|
|
assert_allclose_dense_sparse(ct.fit(X_sparse).transform(X_sparse), X_res_both)
|
|
|
|
|
|
def test_column_transformer_list():
|
|
X_list = [[1, float("nan"), "a"], [0, 0, "b"]]
|
|
expected_result = np.array(
|
|
[
|
|
[1, float("nan"), 1, 0],
|
|
[-1, 0, 0, 1],
|
|
]
|
|
)
|
|
|
|
ct = ColumnTransformer(
|
|
[
|
|
("numerical", StandardScaler(), [0, 1]),
|
|
("categorical", OneHotEncoder(), [2]),
|
|
]
|
|
)
|
|
|
|
assert_array_equal(ct.fit_transform(X_list), expected_result)
|
|
assert_array_equal(ct.fit(X_list).transform(X_list), expected_result)
|
|
|
|
|
|
def test_column_transformer_sparse_stacking():
|
|
X_array = np.array([[0, 1, 2], [2, 4, 6]]).T
|
|
col_trans = ColumnTransformer(
|
|
[("trans1", Trans(), [0]), ("trans2", SparseMatrixTrans(), 1)],
|
|
sparse_threshold=0.8,
|
|
)
|
|
col_trans.fit(X_array)
|
|
X_trans = col_trans.transform(X_array)
|
|
assert sparse.issparse(X_trans)
|
|
assert X_trans.shape == (X_trans.shape[0], X_trans.shape[0] + 1)
|
|
assert_array_equal(X_trans.toarray()[:, 1:], np.eye(X_trans.shape[0]))
|
|
assert len(col_trans.transformers_) == 2
|
|
assert col_trans.transformers_[-1][0] != "remainder"
|
|
|
|
col_trans = ColumnTransformer(
|
|
[("trans1", Trans(), [0]), ("trans2", SparseMatrixTrans(), 1)],
|
|
sparse_threshold=0.1,
|
|
)
|
|
col_trans.fit(X_array)
|
|
X_trans = col_trans.transform(X_array)
|
|
assert not sparse.issparse(X_trans)
|
|
assert X_trans.shape == (X_trans.shape[0], X_trans.shape[0] + 1)
|
|
assert_array_equal(X_trans[:, 1:], np.eye(X_trans.shape[0]))
|
|
|
|
|
|
def test_column_transformer_mixed_cols_sparse():
|
|
df = np.array([["a", 1, True], ["b", 2, False]], dtype="O")
|
|
|
|
ct = make_column_transformer(
|
|
(OneHotEncoder(), [0]), ("passthrough", [1, 2]), sparse_threshold=1.0
|
|
)
|
|
|
|
# this shouldn't fail, since boolean can be coerced into a numeric
|
|
# See: https://github.com/scikit-learn/scikit-learn/issues/11912
|
|
X_trans = ct.fit_transform(df)
|
|
assert X_trans.getformat() == "csr"
|
|
assert_array_equal(X_trans.toarray(), np.array([[1, 0, 1, 1], [0, 1, 2, 0]]))
|
|
|
|
ct = make_column_transformer(
|
|
(OneHotEncoder(), [0]), ("passthrough", [0]), sparse_threshold=1.0
|
|
)
|
|
with pytest.raises(ValueError, match="For a sparse output, all columns should"):
|
|
# this fails since strings `a` and `b` cannot be
|
|
# coerced into a numeric.
|
|
ct.fit_transform(df)
|
|
|
|
|
|
def test_column_transformer_sparse_threshold():
|
|
X_array = np.array([["a", "b"], ["A", "B"]], dtype=object).T
|
|
# above data has sparsity of 4 / 8 = 0.5
|
|
|
|
# apply threshold even if all sparse
|
|
col_trans = ColumnTransformer(
|
|
[("trans1", OneHotEncoder(), [0]), ("trans2", OneHotEncoder(), [1])],
|
|
sparse_threshold=0.2,
|
|
)
|
|
res = col_trans.fit_transform(X_array)
|
|
assert not sparse.issparse(res)
|
|
assert not col_trans.sparse_output_
|
|
|
|
# mixed -> sparsity of (4 + 2) / 8 = 0.75
|
|
for thres in [0.75001, 1]:
|
|
col_trans = ColumnTransformer(
|
|
[
|
|
("trans1", OneHotEncoder(sparse_output=True), [0]),
|
|
("trans2", OneHotEncoder(sparse_output=False), [1]),
|
|
],
|
|
sparse_threshold=thres,
|
|
)
|
|
res = col_trans.fit_transform(X_array)
|
|
assert sparse.issparse(res)
|
|
assert col_trans.sparse_output_
|
|
|
|
for thres in [0.75, 0]:
|
|
col_trans = ColumnTransformer(
|
|
[
|
|
("trans1", OneHotEncoder(sparse_output=True), [0]),
|
|
("trans2", OneHotEncoder(sparse_output=False), [1]),
|
|
],
|
|
sparse_threshold=thres,
|
|
)
|
|
res = col_trans.fit_transform(X_array)
|
|
assert not sparse.issparse(res)
|
|
assert not col_trans.sparse_output_
|
|
|
|
# if nothing is sparse -> no sparse
|
|
for thres in [0.33, 0, 1]:
|
|
col_trans = ColumnTransformer(
|
|
[
|
|
("trans1", OneHotEncoder(sparse_output=False), [0]),
|
|
("trans2", OneHotEncoder(sparse_output=False), [1]),
|
|
],
|
|
sparse_threshold=thres,
|
|
)
|
|
res = col_trans.fit_transform(X_array)
|
|
assert not sparse.issparse(res)
|
|
assert not col_trans.sparse_output_
|
|
|
|
|
|
def test_column_transformer_error_msg_1D():
|
|
X_array = np.array([[0.0, 1.0, 2.0], [2.0, 4.0, 6.0]]).T
|
|
|
|
col_trans = ColumnTransformer([("trans", StandardScaler(), 0)])
|
|
msg = "1D data passed to a transformer"
|
|
with pytest.raises(ValueError, match=msg):
|
|
col_trans.fit(X_array)
|
|
|
|
with pytest.raises(ValueError, match=msg):
|
|
col_trans.fit_transform(X_array)
|
|
|
|
col_trans = ColumnTransformer([("trans", TransRaise(), 0)])
|
|
for func in [col_trans.fit, col_trans.fit_transform]:
|
|
with pytest.raises(ValueError, match="specific message"):
|
|
func(X_array)
|
|
|
|
|
|
def test_2D_transformer_output():
|
|
X_array = np.array([[0, 1, 2], [2, 4, 6]]).T
|
|
|
|
# if one transformer is dropped, test that name is still correct
|
|
ct = ColumnTransformer([("trans1", "drop", 0), ("trans2", TransNo2D(), 1)])
|
|
|
|
msg = "the 'trans2' transformer should be 2D"
|
|
with pytest.raises(ValueError, match=msg):
|
|
ct.fit_transform(X_array)
|
|
# because fit is also doing transform, this raises already on fit
|
|
with pytest.raises(ValueError, match=msg):
|
|
ct.fit(X_array)
|
|
|
|
|
|
def test_2D_transformer_output_pandas():
|
|
pd = pytest.importorskip("pandas")
|
|
|
|
X_array = np.array([[0, 1, 2], [2, 4, 6]]).T
|
|
X_df = pd.DataFrame(X_array, columns=["col1", "col2"])
|
|
|
|
# if one transformer is dropped, test that name is still correct
|
|
ct = ColumnTransformer([("trans1", TransNo2D(), "col1")])
|
|
msg = "the 'trans1' transformer should be 2D"
|
|
with pytest.raises(ValueError, match=msg):
|
|
ct.fit_transform(X_df)
|
|
# because fit is also doing transform, this raises already on fit
|
|
with pytest.raises(ValueError, match=msg):
|
|
ct.fit(X_df)
|
|
|
|
|
|
@pytest.mark.parametrize("remainder", ["drop", "passthrough"])
|
|
def test_column_transformer_invalid_columns(remainder):
|
|
X_array = np.array([[0, 1, 2], [2, 4, 6]]).T
|
|
|
|
# general invalid
|
|
for col in [1.5, ["string", 1], slice(1, "s"), np.array([1.0])]:
|
|
ct = ColumnTransformer([("trans", Trans(), col)], remainder=remainder)
|
|
with pytest.raises(ValueError, match="No valid specification"):
|
|
ct.fit(X_array)
|
|
|
|
# invalid for arrays
|
|
for col in ["string", ["string", "other"], slice("a", "b")]:
|
|
ct = ColumnTransformer([("trans", Trans(), col)], remainder=remainder)
|
|
with pytest.raises(ValueError, match="Specifying the columns"):
|
|
ct.fit(X_array)
|
|
|
|
# transformed n_features does not match fitted n_features
|
|
col = [0, 1]
|
|
ct = ColumnTransformer([("trans", Trans(), col)], remainder=remainder)
|
|
ct.fit(X_array)
|
|
X_array_more = np.array([[0, 1, 2], [2, 4, 6], [3, 6, 9]]).T
|
|
msg = "X has 3 features, but ColumnTransformer is expecting 2 features as input."
|
|
with pytest.raises(ValueError, match=msg):
|
|
ct.transform(X_array_more)
|
|
X_array_fewer = np.array(
|
|
[
|
|
[0, 1, 2],
|
|
]
|
|
).T
|
|
err_msg = (
|
|
"X has 1 features, but ColumnTransformer is expecting 2 features as input."
|
|
)
|
|
with pytest.raises(ValueError, match=err_msg):
|
|
ct.transform(X_array_fewer)
|
|
|
|
|
|
def test_column_transformer_invalid_transformer():
|
|
class NoTrans(BaseEstimator):
|
|
def fit(self, X, y=None):
|
|
return self
|
|
|
|
def predict(self, X):
|
|
return X
|
|
|
|
X_array = np.array([[0, 1, 2], [2, 4, 6]]).T
|
|
ct = ColumnTransformer([("trans", NoTrans(), [0])])
|
|
msg = "All estimators should implement fit and transform"
|
|
with pytest.raises(TypeError, match=msg):
|
|
ct.fit(X_array)
|
|
|
|
|
|
def test_make_column_transformer():
|
|
scaler = StandardScaler()
|
|
norm = Normalizer()
|
|
ct = make_column_transformer((scaler, "first"), (norm, ["second"]))
|
|
names, transformers, columns = zip(*ct.transformers)
|
|
assert names == ("standardscaler", "normalizer")
|
|
assert transformers == (scaler, norm)
|
|
assert columns == ("first", ["second"])
|
|
|
|
|
|
def test_make_column_transformer_pandas():
|
|
pd = pytest.importorskip("pandas")
|
|
X_array = np.array([[0, 1, 2], [2, 4, 6]]).T
|
|
X_df = pd.DataFrame(X_array, columns=["first", "second"])
|
|
norm = Normalizer()
|
|
ct1 = ColumnTransformer([("norm", Normalizer(), X_df.columns)])
|
|
ct2 = make_column_transformer((norm, X_df.columns))
|
|
assert_almost_equal(ct1.fit_transform(X_df), ct2.fit_transform(X_df))
|
|
|
|
|
|
def test_make_column_transformer_kwargs():
|
|
scaler = StandardScaler()
|
|
norm = Normalizer()
|
|
ct = make_column_transformer(
|
|
(scaler, "first"),
|
|
(norm, ["second"]),
|
|
n_jobs=3,
|
|
remainder="drop",
|
|
sparse_threshold=0.5,
|
|
)
|
|
assert (
|
|
ct.transformers
|
|
== make_column_transformer((scaler, "first"), (norm, ["second"])).transformers
|
|
)
|
|
assert ct.n_jobs == 3
|
|
assert ct.remainder == "drop"
|
|
assert ct.sparse_threshold == 0.5
|
|
# invalid keyword parameters should raise an error message
|
|
msg = re.escape(
|
|
"make_column_transformer() got an unexpected "
|
|
"keyword argument 'transformer_weights'"
|
|
)
|
|
with pytest.raises(TypeError, match=msg):
|
|
make_column_transformer(
|
|
(scaler, "first"),
|
|
(norm, ["second"]),
|
|
transformer_weights={"pca": 10, "Transf": 1},
|
|
)
|
|
|
|
|
|
def test_make_column_transformer_remainder_transformer():
|
|
scaler = StandardScaler()
|
|
norm = Normalizer()
|
|
remainder = StandardScaler()
|
|
ct = make_column_transformer(
|
|
(scaler, "first"), (norm, ["second"]), remainder=remainder
|
|
)
|
|
assert ct.remainder == remainder
|
|
|
|
|
|
def test_column_transformer_get_set_params():
|
|
ct = ColumnTransformer(
|
|
[("trans1", StandardScaler(), [0]), ("trans2", StandardScaler(), [1])]
|
|
)
|
|
|
|
exp = {
|
|
"n_jobs": None,
|
|
"remainder": "drop",
|
|
"sparse_threshold": 0.3,
|
|
"trans1": ct.transformers[0][1],
|
|
"trans1__copy": True,
|
|
"trans1__with_mean": True,
|
|
"trans1__with_std": True,
|
|
"trans2": ct.transformers[1][1],
|
|
"trans2__copy": True,
|
|
"trans2__with_mean": True,
|
|
"trans2__with_std": True,
|
|
"transformers": ct.transformers,
|
|
"transformer_weights": None,
|
|
"verbose_feature_names_out": True,
|
|
"verbose": False,
|
|
}
|
|
|
|
assert ct.get_params() == exp
|
|
|
|
ct.set_params(trans1__with_mean=False)
|
|
assert not ct.get_params()["trans1__with_mean"]
|
|
|
|
ct.set_params(trans1="passthrough")
|
|
exp = {
|
|
"n_jobs": None,
|
|
"remainder": "drop",
|
|
"sparse_threshold": 0.3,
|
|
"trans1": "passthrough",
|
|
"trans2": ct.transformers[1][1],
|
|
"trans2__copy": True,
|
|
"trans2__with_mean": True,
|
|
"trans2__with_std": True,
|
|
"transformers": ct.transformers,
|
|
"transformer_weights": None,
|
|
"verbose_feature_names_out": True,
|
|
"verbose": False,
|
|
}
|
|
|
|
assert ct.get_params() == exp
|
|
|
|
|
|
def test_column_transformer_named_estimators():
|
|
X_array = np.array([[0.0, 1.0, 2.0], [2.0, 4.0, 6.0]]).T
|
|
ct = ColumnTransformer(
|
|
[
|
|
("trans1", StandardScaler(), [0]),
|
|
("trans2", StandardScaler(with_std=False), [1]),
|
|
]
|
|
)
|
|
assert not hasattr(ct, "transformers_")
|
|
ct.fit(X_array)
|
|
assert hasattr(ct, "transformers_")
|
|
assert isinstance(ct.named_transformers_["trans1"], StandardScaler)
|
|
assert isinstance(ct.named_transformers_.trans1, StandardScaler)
|
|
assert isinstance(ct.named_transformers_["trans2"], StandardScaler)
|
|
assert isinstance(ct.named_transformers_.trans2, StandardScaler)
|
|
assert not ct.named_transformers_.trans2.with_std
|
|
# check it are fitted transformers
|
|
assert ct.named_transformers_.trans1.mean_ == 1.0
|
|
|
|
|
|
def test_column_transformer_cloning():
|
|
X_array = np.array([[0.0, 1.0, 2.0], [2.0, 4.0, 6.0]]).T
|
|
|
|
ct = ColumnTransformer([("trans", StandardScaler(), [0])])
|
|
ct.fit(X_array)
|
|
assert not hasattr(ct.transformers[0][1], "mean_")
|
|
assert hasattr(ct.transformers_[0][1], "mean_")
|
|
|
|
ct = ColumnTransformer([("trans", StandardScaler(), [0])])
|
|
ct.fit_transform(X_array)
|
|
assert not hasattr(ct.transformers[0][1], "mean_")
|
|
assert hasattr(ct.transformers_[0][1], "mean_")
|
|
|
|
|
|
def test_column_transformer_get_feature_names():
|
|
X_array = np.array([[0.0, 1.0, 2.0], [2.0, 4.0, 6.0]]).T
|
|
ct = ColumnTransformer([("trans", Trans(), [0, 1])])
|
|
# raise correct error when not fitted
|
|
with pytest.raises(NotFittedError):
|
|
ct.get_feature_names_out()
|
|
# raise correct error when no feature names are available
|
|
ct.fit(X_array)
|
|
msg = re.escape(
|
|
"Transformer trans (type Trans) does not provide get_feature_names_out"
|
|
)
|
|
with pytest.raises(AttributeError, match=msg):
|
|
ct.get_feature_names_out()
|
|
|
|
|
|
def test_column_transformer_special_strings():
|
|
|
|
# one 'drop' -> ignore
|
|
X_array = np.array([[0.0, 1.0, 2.0], [2.0, 4.0, 6.0]]).T
|
|
ct = ColumnTransformer([("trans1", Trans(), [0]), ("trans2", "drop", [1])])
|
|
exp = np.array([[0.0], [1.0], [2.0]])
|
|
assert_array_equal(ct.fit_transform(X_array), exp)
|
|
assert_array_equal(ct.fit(X_array).transform(X_array), exp)
|
|
assert len(ct.transformers_) == 2
|
|
assert ct.transformers_[-1][0] != "remainder"
|
|
|
|
# all 'drop' -> return shape 0 array
|
|
ct = ColumnTransformer([("trans1", "drop", [0]), ("trans2", "drop", [1])])
|
|
assert_array_equal(ct.fit(X_array).transform(X_array).shape, (3, 0))
|
|
assert_array_equal(ct.fit_transform(X_array).shape, (3, 0))
|
|
assert len(ct.transformers_) == 2
|
|
assert ct.transformers_[-1][0] != "remainder"
|
|
|
|
# 'passthrough'
|
|
X_array = np.array([[0.0, 1.0, 2.0], [2.0, 4.0, 6.0]]).T
|
|
ct = ColumnTransformer([("trans1", Trans(), [0]), ("trans2", "passthrough", [1])])
|
|
exp = X_array
|
|
assert_array_equal(ct.fit_transform(X_array), exp)
|
|
assert_array_equal(ct.fit(X_array).transform(X_array), exp)
|
|
assert len(ct.transformers_) == 2
|
|
assert ct.transformers_[-1][0] != "remainder"
|
|
|
|
# None itself / other string is not valid
|
|
for val in [None, "other"]:
|
|
ct = ColumnTransformer([("trans1", Trans(), [0]), ("trans2", None, [1])])
|
|
msg = "All estimators should implement"
|
|
with pytest.raises(TypeError, match=msg):
|
|
ct.fit_transform(X_array)
|
|
with pytest.raises(TypeError, match=msg):
|
|
ct.fit(X_array)
|
|
|
|
|
|
def test_column_transformer_remainder():
|
|
X_array = np.array([[0, 1, 2], [2, 4, 6]]).T
|
|
|
|
X_res_first = np.array([0, 1, 2]).reshape(-1, 1)
|
|
X_res_second = np.array([2, 4, 6]).reshape(-1, 1)
|
|
X_res_both = X_array
|
|
|
|
# default drop
|
|
ct = ColumnTransformer([("trans1", Trans(), [0])])
|
|
assert_array_equal(ct.fit_transform(X_array), X_res_first)
|
|
assert_array_equal(ct.fit(X_array).transform(X_array), X_res_first)
|
|
assert len(ct.transformers_) == 2
|
|
assert ct.transformers_[-1][0] == "remainder"
|
|
assert ct.transformers_[-1][1] == "drop"
|
|
assert_array_equal(ct.transformers_[-1][2], [1])
|
|
|
|
# specify passthrough
|
|
ct = ColumnTransformer([("trans", Trans(), [0])], remainder="passthrough")
|
|
assert_array_equal(ct.fit_transform(X_array), X_res_both)
|
|
assert_array_equal(ct.fit(X_array).transform(X_array), X_res_both)
|
|
assert len(ct.transformers_) == 2
|
|
assert ct.transformers_[-1][0] == "remainder"
|
|
assert ct.transformers_[-1][1] == "passthrough"
|
|
assert_array_equal(ct.transformers_[-1][2], [1])
|
|
|
|
# column order is not preserved (passed through added to end)
|
|
ct = ColumnTransformer([("trans1", Trans(), [1])], remainder="passthrough")
|
|
assert_array_equal(ct.fit_transform(X_array), X_res_both[:, ::-1])
|
|
assert_array_equal(ct.fit(X_array).transform(X_array), X_res_both[:, ::-1])
|
|
assert len(ct.transformers_) == 2
|
|
assert ct.transformers_[-1][0] == "remainder"
|
|
assert ct.transformers_[-1][1] == "passthrough"
|
|
assert_array_equal(ct.transformers_[-1][2], [0])
|
|
|
|
# passthrough when all actual transformers are skipped
|
|
ct = ColumnTransformer([("trans1", "drop", [0])], remainder="passthrough")
|
|
assert_array_equal(ct.fit_transform(X_array), X_res_second)
|
|
assert_array_equal(ct.fit(X_array).transform(X_array), X_res_second)
|
|
assert len(ct.transformers_) == 2
|
|
assert ct.transformers_[-1][0] == "remainder"
|
|
assert ct.transformers_[-1][1] == "passthrough"
|
|
assert_array_equal(ct.transformers_[-1][2], [1])
|
|
|
|
# error on invalid arg
|
|
ct = ColumnTransformer([("trans1", Trans(), [0])], remainder=1)
|
|
msg = "remainder keyword needs to be one of 'drop', 'passthrough', or estimator."
|
|
with pytest.raises(ValueError, match=msg):
|
|
ct.fit(X_array)
|
|
|
|
with pytest.raises(ValueError, match=msg):
|
|
ct.fit_transform(X_array)
|
|
|
|
# check default for make_column_transformer
|
|
ct = make_column_transformer((Trans(), [0]))
|
|
assert ct.remainder == "drop"
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"key", [[0], np.array([0]), slice(0, 1), np.array([True, False])]
|
|
)
|
|
def test_column_transformer_remainder_numpy(key):
|
|
# test different ways that columns are specified with passthrough
|
|
X_array = np.array([[0, 1, 2], [2, 4, 6]]).T
|
|
X_res_both = X_array
|
|
|
|
ct = ColumnTransformer([("trans1", Trans(), key)], remainder="passthrough")
|
|
assert_array_equal(ct.fit_transform(X_array), X_res_both)
|
|
assert_array_equal(ct.fit(X_array).transform(X_array), X_res_both)
|
|
assert len(ct.transformers_) == 2
|
|
assert ct.transformers_[-1][0] == "remainder"
|
|
assert ct.transformers_[-1][1] == "passthrough"
|
|
assert_array_equal(ct.transformers_[-1][2], [1])
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"key",
|
|
[
|
|
[0],
|
|
slice(0, 1),
|
|
np.array([True, False]),
|
|
["first"],
|
|
"pd-index",
|
|
np.array(["first"]),
|
|
np.array(["first"], dtype=object),
|
|
slice(None, "first"),
|
|
slice("first", "first"),
|
|
],
|
|
)
|
|
def test_column_transformer_remainder_pandas(key):
|
|
# test different ways that columns are specified with passthrough
|
|
pd = pytest.importorskip("pandas")
|
|
if isinstance(key, str) and key == "pd-index":
|
|
key = pd.Index(["first"])
|
|
|
|
X_array = np.array([[0, 1, 2], [2, 4, 6]]).T
|
|
X_df = pd.DataFrame(X_array, columns=["first", "second"])
|
|
X_res_both = X_array
|
|
|
|
ct = ColumnTransformer([("trans1", Trans(), key)], remainder="passthrough")
|
|
assert_array_equal(ct.fit_transform(X_df), X_res_both)
|
|
assert_array_equal(ct.fit(X_df).transform(X_df), X_res_both)
|
|
assert len(ct.transformers_) == 2
|
|
assert ct.transformers_[-1][0] == "remainder"
|
|
assert ct.transformers_[-1][1] == "passthrough"
|
|
assert_array_equal(ct.transformers_[-1][2], [1])
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"key", [[0], np.array([0]), slice(0, 1), np.array([True, False, False])]
|
|
)
|
|
def test_column_transformer_remainder_transformer(key):
|
|
X_array = np.array([[0, 1, 2], [2, 4, 6], [8, 6, 4]]).T
|
|
X_res_both = X_array.copy()
|
|
|
|
# second and third columns are doubled when remainder = DoubleTrans
|
|
X_res_both[:, 1:3] *= 2
|
|
|
|
ct = ColumnTransformer([("trans1", Trans(), key)], remainder=DoubleTrans())
|
|
|
|
assert_array_equal(ct.fit_transform(X_array), X_res_both)
|
|
assert_array_equal(ct.fit(X_array).transform(X_array), X_res_both)
|
|
assert len(ct.transformers_) == 2
|
|
assert ct.transformers_[-1][0] == "remainder"
|
|
assert isinstance(ct.transformers_[-1][1], DoubleTrans)
|
|
assert_array_equal(ct.transformers_[-1][2], [1, 2])
|
|
|
|
|
|
def test_column_transformer_no_remaining_remainder_transformer():
|
|
X_array = np.array([[0, 1, 2], [2, 4, 6], [8, 6, 4]]).T
|
|
|
|
ct = ColumnTransformer([("trans1", Trans(), [0, 1, 2])], remainder=DoubleTrans())
|
|
|
|
assert_array_equal(ct.fit_transform(X_array), X_array)
|
|
assert_array_equal(ct.fit(X_array).transform(X_array), X_array)
|
|
assert len(ct.transformers_) == 1
|
|
assert ct.transformers_[-1][0] != "remainder"
|
|
|
|
|
|
def test_column_transformer_drops_all_remainder_transformer():
|
|
X_array = np.array([[0, 1, 2], [2, 4, 6], [8, 6, 4]]).T
|
|
|
|
# columns are doubled when remainder = DoubleTrans
|
|
X_res_both = 2 * X_array.copy()[:, 1:3]
|
|
|
|
ct = ColumnTransformer([("trans1", "drop", [0])], remainder=DoubleTrans())
|
|
|
|
assert_array_equal(ct.fit_transform(X_array), X_res_both)
|
|
assert_array_equal(ct.fit(X_array).transform(X_array), X_res_both)
|
|
assert len(ct.transformers_) == 2
|
|
assert ct.transformers_[-1][0] == "remainder"
|
|
assert isinstance(ct.transformers_[-1][1], DoubleTrans)
|
|
assert_array_equal(ct.transformers_[-1][2], [1, 2])
|
|
|
|
|
|
def test_column_transformer_sparse_remainder_transformer():
|
|
X_array = np.array([[0, 1, 2], [2, 4, 6], [8, 6, 4]]).T
|
|
|
|
ct = ColumnTransformer(
|
|
[("trans1", Trans(), [0])], remainder=SparseMatrixTrans(), sparse_threshold=0.8
|
|
)
|
|
|
|
X_trans = ct.fit_transform(X_array)
|
|
assert sparse.issparse(X_trans)
|
|
# SparseMatrixTrans creates 3 features for each column. There is
|
|
# one column in ``transformers``, thus:
|
|
assert X_trans.shape == (3, 3 + 1)
|
|
|
|
exp_array = np.hstack((X_array[:, 0].reshape(-1, 1), np.eye(3)))
|
|
assert_array_equal(X_trans.toarray(), exp_array)
|
|
assert len(ct.transformers_) == 2
|
|
assert ct.transformers_[-1][0] == "remainder"
|
|
assert isinstance(ct.transformers_[-1][1], SparseMatrixTrans)
|
|
assert_array_equal(ct.transformers_[-1][2], [1, 2])
|
|
|
|
|
|
def test_column_transformer_drop_all_sparse_remainder_transformer():
|
|
X_array = np.array([[0, 1, 2], [2, 4, 6], [8, 6, 4]]).T
|
|
ct = ColumnTransformer(
|
|
[("trans1", "drop", [0])], remainder=SparseMatrixTrans(), sparse_threshold=0.8
|
|
)
|
|
|
|
X_trans = ct.fit_transform(X_array)
|
|
assert sparse.issparse(X_trans)
|
|
|
|
# SparseMatrixTrans creates 3 features for each column, thus:
|
|
assert X_trans.shape == (3, 3)
|
|
assert_array_equal(X_trans.toarray(), np.eye(3))
|
|
assert len(ct.transformers_) == 2
|
|
assert ct.transformers_[-1][0] == "remainder"
|
|
assert isinstance(ct.transformers_[-1][1], SparseMatrixTrans)
|
|
assert_array_equal(ct.transformers_[-1][2], [1, 2])
|
|
|
|
|
|
def test_column_transformer_get_set_params_with_remainder():
|
|
ct = ColumnTransformer(
|
|
[("trans1", StandardScaler(), [0])], remainder=StandardScaler()
|
|
)
|
|
|
|
exp = {
|
|
"n_jobs": None,
|
|
"remainder": ct.remainder,
|
|
"remainder__copy": True,
|
|
"remainder__with_mean": True,
|
|
"remainder__with_std": True,
|
|
"sparse_threshold": 0.3,
|
|
"trans1": ct.transformers[0][1],
|
|
"trans1__copy": True,
|
|
"trans1__with_mean": True,
|
|
"trans1__with_std": True,
|
|
"transformers": ct.transformers,
|
|
"transformer_weights": None,
|
|
"verbose_feature_names_out": True,
|
|
"verbose": False,
|
|
}
|
|
|
|
assert ct.get_params() == exp
|
|
|
|
ct.set_params(remainder__with_std=False)
|
|
assert not ct.get_params()["remainder__with_std"]
|
|
|
|
ct.set_params(trans1="passthrough")
|
|
exp = {
|
|
"n_jobs": None,
|
|
"remainder": ct.remainder,
|
|
"remainder__copy": True,
|
|
"remainder__with_mean": True,
|
|
"remainder__with_std": False,
|
|
"sparse_threshold": 0.3,
|
|
"trans1": "passthrough",
|
|
"transformers": ct.transformers,
|
|
"transformer_weights": None,
|
|
"verbose_feature_names_out": True,
|
|
"verbose": False,
|
|
}
|
|
assert ct.get_params() == exp
|
|
|
|
|
|
def test_column_transformer_no_estimators():
|
|
X_array = np.array([[0, 1, 2], [2, 4, 6], [8, 6, 4]]).astype("float").T
|
|
ct = ColumnTransformer([], remainder=StandardScaler())
|
|
|
|
params = ct.get_params()
|
|
assert params["remainder__with_mean"]
|
|
|
|
X_trans = ct.fit_transform(X_array)
|
|
assert X_trans.shape == X_array.shape
|
|
assert len(ct.transformers_) == 1
|
|
assert ct.transformers_[-1][0] == "remainder"
|
|
assert ct.transformers_[-1][2] == [0, 1, 2]
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
["est", "pattern"],
|
|
[
|
|
(
|
|
ColumnTransformer(
|
|
[("trans1", Trans(), [0]), ("trans2", Trans(), [1])],
|
|
remainder=DoubleTrans(),
|
|
),
|
|
(
|
|
r"\[ColumnTransformer\].*\(1 of 3\) Processing trans1.* total=.*\n"
|
|
r"\[ColumnTransformer\].*\(2 of 3\) Processing trans2.* total=.*\n"
|
|
r"\[ColumnTransformer\].*\(3 of 3\) Processing remainder.* total=.*\n$"
|
|
),
|
|
),
|
|
(
|
|
ColumnTransformer(
|
|
[("trans1", Trans(), [0]), ("trans2", Trans(), [1])],
|
|
remainder="passthrough",
|
|
),
|
|
(
|
|
r"\[ColumnTransformer\].*\(1 of 3\) Processing trans1.* total=.*\n"
|
|
r"\[ColumnTransformer\].*\(2 of 3\) Processing trans2.* total=.*\n"
|
|
r"\[ColumnTransformer\].*\(3 of 3\) Processing remainder.* total=.*\n$"
|
|
),
|
|
),
|
|
(
|
|
ColumnTransformer(
|
|
[("trans1", Trans(), [0]), ("trans2", "drop", [1])],
|
|
remainder="passthrough",
|
|
),
|
|
(
|
|
r"\[ColumnTransformer\].*\(1 of 2\) Processing trans1.* total=.*\n"
|
|
r"\[ColumnTransformer\].*\(2 of 2\) Processing remainder.* total=.*\n$"
|
|
),
|
|
),
|
|
(
|
|
ColumnTransformer(
|
|
[("trans1", Trans(), [0]), ("trans2", "passthrough", [1])],
|
|
remainder="passthrough",
|
|
),
|
|
(
|
|
r"\[ColumnTransformer\].*\(1 of 3\) Processing trans1.* total=.*\n"
|
|
r"\[ColumnTransformer\].*\(2 of 3\) Processing trans2.* total=.*\n"
|
|
r"\[ColumnTransformer\].*\(3 of 3\) Processing remainder.* total=.*\n$"
|
|
),
|
|
),
|
|
(
|
|
ColumnTransformer([("trans1", Trans(), [0])], remainder="passthrough"),
|
|
(
|
|
r"\[ColumnTransformer\].*\(1 of 2\) Processing trans1.* total=.*\n"
|
|
r"\[ColumnTransformer\].*\(2 of 2\) Processing remainder.* total=.*\n$"
|
|
),
|
|
),
|
|
(
|
|
ColumnTransformer(
|
|
[("trans1", Trans(), [0]), ("trans2", Trans(), [1])], remainder="drop"
|
|
),
|
|
(
|
|
r"\[ColumnTransformer\].*\(1 of 2\) Processing trans1.* total=.*\n"
|
|
r"\[ColumnTransformer\].*\(2 of 2\) Processing trans2.* total=.*\n$"
|
|
),
|
|
),
|
|
(
|
|
ColumnTransformer([("trans1", Trans(), [0])], remainder="drop"),
|
|
r"\[ColumnTransformer\].*\(1 of 1\) Processing trans1.* total=.*\n$",
|
|
),
|
|
],
|
|
)
|
|
@pytest.mark.parametrize("method", ["fit", "fit_transform"])
|
|
def test_column_transformer_verbose(est, pattern, method, capsys):
|
|
X_array = np.array([[0, 1, 2], [2, 4, 6], [8, 6, 4]]).T
|
|
|
|
func = getattr(est, method)
|
|
est.set_params(verbose=False)
|
|
func(X_array)
|
|
assert not capsys.readouterr().out, "Got output for verbose=False"
|
|
|
|
est.set_params(verbose=True)
|
|
func(X_array)
|
|
assert re.match(pattern, capsys.readouterr()[0])
|
|
|
|
|
|
def test_column_transformer_no_estimators_set_params():
|
|
ct = ColumnTransformer([]).set_params(n_jobs=2)
|
|
assert ct.n_jobs == 2
|
|
|
|
|
|
def test_column_transformer_callable_specifier():
|
|
# assert that function gets the full array
|
|
X_array = np.array([[0, 1, 2], [2, 4, 6]]).T
|
|
X_res_first = np.array([[0, 1, 2]]).T
|
|
|
|
def func(X):
|
|
assert_array_equal(X, X_array)
|
|
return [0]
|
|
|
|
ct = ColumnTransformer([("trans", Trans(), func)], remainder="drop")
|
|
assert_array_equal(ct.fit_transform(X_array), X_res_first)
|
|
assert_array_equal(ct.fit(X_array).transform(X_array), X_res_first)
|
|
assert callable(ct.transformers[0][2])
|
|
assert ct.transformers_[0][2] == [0]
|
|
|
|
|
|
def test_column_transformer_callable_specifier_dataframe():
|
|
# assert that function gets the full dataframe
|
|
pd = pytest.importorskip("pandas")
|
|
X_array = np.array([[0, 1, 2], [2, 4, 6]]).T
|
|
X_res_first = np.array([[0, 1, 2]]).T
|
|
|
|
X_df = pd.DataFrame(X_array, columns=["first", "second"])
|
|
|
|
def func(X):
|
|
assert_array_equal(X.columns, X_df.columns)
|
|
assert_array_equal(X.values, X_df.values)
|
|
return ["first"]
|
|
|
|
ct = ColumnTransformer([("trans", Trans(), func)], remainder="drop")
|
|
assert_array_equal(ct.fit_transform(X_df), X_res_first)
|
|
assert_array_equal(ct.fit(X_df).transform(X_df), X_res_first)
|
|
assert callable(ct.transformers[0][2])
|
|
assert ct.transformers_[0][2] == ["first"]
|
|
|
|
|
|
def test_column_transformer_negative_column_indexes():
|
|
X = np.random.randn(2, 2)
|
|
X_categories = np.array([[1], [2]])
|
|
X = np.concatenate([X, X_categories], axis=1)
|
|
|
|
ohe = OneHotEncoder()
|
|
|
|
tf_1 = ColumnTransformer([("ohe", ohe, [-1])], remainder="passthrough")
|
|
tf_2 = ColumnTransformer([("ohe", ohe, [2])], remainder="passthrough")
|
|
assert_array_equal(tf_1.fit_transform(X), tf_2.fit_transform(X))
|
|
|
|
|
|
@pytest.mark.parametrize("array_type", [np.asarray, sparse.csr_matrix])
|
|
def test_column_transformer_mask_indexing(array_type):
|
|
# Regression test for #14510
|
|
# Boolean array-like does not behave as boolean array with sparse matrices.
|
|
X = np.transpose([[1, 2, 3], [4, 5, 6], [5, 6, 7], [8, 9, 10]])
|
|
X = array_type(X)
|
|
column_transformer = ColumnTransformer(
|
|
[("identity", FunctionTransformer(), [False, True, False, True])]
|
|
)
|
|
X_trans = column_transformer.fit_transform(X)
|
|
assert X_trans.shape == (3, 2)
|
|
|
|
|
|
def test_n_features_in():
|
|
# make sure n_features_in is what is passed as input to the column
|
|
# transformer.
|
|
|
|
X = [[1, 2], [3, 4], [5, 6]]
|
|
ct = ColumnTransformer([("a", DoubleTrans(), [0]), ("b", DoubleTrans(), [1])])
|
|
assert not hasattr(ct, "n_features_in_")
|
|
ct.fit(X)
|
|
assert ct.n_features_in_ == 2
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"cols, pattern, include, exclude",
|
|
[
|
|
(["col_int", "col_float"], None, np.number, None),
|
|
(["col_int", "col_float"], None, None, object),
|
|
(["col_int", "col_float"], None, [int, float], None),
|
|
(["col_str"], None, [object], None),
|
|
(["col_str"], None, object, None),
|
|
(["col_float"], None, float, None),
|
|
(["col_float"], "at$", [np.number], None),
|
|
(["col_int"], None, [int], None),
|
|
(["col_int"], "^col_int", [np.number], None),
|
|
(["col_float", "col_str"], "float|str", None, None),
|
|
(["col_str"], "^col_s", None, [int]),
|
|
([], "str$", float, None),
|
|
(["col_int", "col_float", "col_str"], None, [np.number, object], None),
|
|
],
|
|
)
|
|
def test_make_column_selector_with_select_dtypes(cols, pattern, include, exclude):
|
|
pd = pytest.importorskip("pandas")
|
|
|
|
X_df = pd.DataFrame(
|
|
{
|
|
"col_int": np.array([0, 1, 2], dtype=int),
|
|
"col_float": np.array([0.0, 1.0, 2.0], dtype=float),
|
|
"col_str": ["one", "two", "three"],
|
|
},
|
|
columns=["col_int", "col_float", "col_str"],
|
|
)
|
|
|
|
selector = make_column_selector(
|
|
dtype_include=include, dtype_exclude=exclude, pattern=pattern
|
|
)
|
|
|
|
assert_array_equal(selector(X_df), cols)
|
|
|
|
|
|
def test_column_transformer_with_make_column_selector():
|
|
# Functional test for column transformer + column selector
|
|
pd = pytest.importorskip("pandas")
|
|
X_df = pd.DataFrame(
|
|
{
|
|
"col_int": np.array([0, 1, 2], dtype=int),
|
|
"col_float": np.array([0.0, 1.0, 2.0], dtype=float),
|
|
"col_cat": ["one", "two", "one"],
|
|
"col_str": ["low", "middle", "high"],
|
|
},
|
|
columns=["col_int", "col_float", "col_cat", "col_str"],
|
|
)
|
|
X_df["col_str"] = X_df["col_str"].astype("category")
|
|
|
|
cat_selector = make_column_selector(dtype_include=["category", object])
|
|
num_selector = make_column_selector(dtype_include=np.number)
|
|
|
|
ohe = OneHotEncoder()
|
|
scaler = StandardScaler()
|
|
|
|
ct_selector = make_column_transformer((ohe, cat_selector), (scaler, num_selector))
|
|
ct_direct = make_column_transformer(
|
|
(ohe, ["col_cat", "col_str"]), (scaler, ["col_float", "col_int"])
|
|
)
|
|
|
|
X_selector = ct_selector.fit_transform(X_df)
|
|
X_direct = ct_direct.fit_transform(X_df)
|
|
|
|
assert_allclose(X_selector, X_direct)
|
|
|
|
|
|
def test_make_column_selector_error():
|
|
selector = make_column_selector(dtype_include=np.number)
|
|
X = np.array([[0.1, 0.2]])
|
|
msg = "make_column_selector can only be applied to pandas dataframes"
|
|
with pytest.raises(ValueError, match=msg):
|
|
selector(X)
|
|
|
|
|
|
def test_make_column_selector_pickle():
|
|
pd = pytest.importorskip("pandas")
|
|
|
|
X_df = pd.DataFrame(
|
|
{
|
|
"col_int": np.array([0, 1, 2], dtype=int),
|
|
"col_float": np.array([0.0, 1.0, 2.0], dtype=float),
|
|
"col_str": ["one", "two", "three"],
|
|
},
|
|
columns=["col_int", "col_float", "col_str"],
|
|
)
|
|
|
|
selector = make_column_selector(dtype_include=[object])
|
|
selector_picked = pickle.loads(pickle.dumps(selector))
|
|
|
|
assert_array_equal(selector(X_df), selector_picked(X_df))
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"empty_col",
|
|
[[], np.array([], dtype=int), lambda x: []],
|
|
ids=["list", "array", "callable"],
|
|
)
|
|
def test_feature_names_empty_columns(empty_col):
|
|
pd = pytest.importorskip("pandas")
|
|
|
|
df = pd.DataFrame({"col1": ["a", "a", "b"], "col2": ["z", "z", "z"]})
|
|
|
|
ct = ColumnTransformer(
|
|
transformers=[
|
|
("ohe", OneHotEncoder(), ["col1", "col2"]),
|
|
("empty_features", OneHotEncoder(), empty_col),
|
|
],
|
|
)
|
|
|
|
ct.fit(df)
|
|
assert_array_equal(
|
|
ct.get_feature_names_out(), ["ohe__col1_a", "ohe__col1_b", "ohe__col2_z"]
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"selector",
|
|
[
|
|
[1],
|
|
lambda x: [1],
|
|
["col2"],
|
|
lambda x: ["col2"],
|
|
[False, True],
|
|
lambda x: [False, True],
|
|
],
|
|
)
|
|
def test_feature_names_out_pandas(selector):
|
|
"""Checks name when selecting only the second column"""
|
|
pd = pytest.importorskip("pandas")
|
|
df = pd.DataFrame({"col1": ["a", "a", "b"], "col2": ["z", "z", "z"]})
|
|
ct = ColumnTransformer([("ohe", OneHotEncoder(), selector)])
|
|
ct.fit(df)
|
|
|
|
assert_array_equal(ct.get_feature_names_out(), ["ohe__col2_z"])
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"selector", [[1], lambda x: [1], [False, True], lambda x: [False, True]]
|
|
)
|
|
def test_feature_names_out_non_pandas(selector):
|
|
"""Checks name when selecting the second column with numpy array"""
|
|
X = [["a", "z"], ["a", "z"], ["b", "z"]]
|
|
ct = ColumnTransformer([("ohe", OneHotEncoder(), selector)])
|
|
ct.fit(X)
|
|
|
|
assert_array_equal(ct.get_feature_names_out(), ["ohe__x1_z"])
|
|
|
|
|
|
@pytest.mark.parametrize("remainder", ["passthrough", StandardScaler()])
|
|
def test_sk_visual_block_remainder(remainder):
|
|
# remainder='passthrough' or an estimator will be shown in repr_html
|
|
ohe = OneHotEncoder()
|
|
ct = ColumnTransformer(
|
|
transformers=[("ohe", ohe, ["col1", "col2"])], remainder=remainder
|
|
)
|
|
visual_block = ct._sk_visual_block_()
|
|
assert visual_block.names == ("ohe", "remainder")
|
|
assert visual_block.name_details == (["col1", "col2"], "")
|
|
assert visual_block.estimators == (ohe, remainder)
|
|
|
|
|
|
def test_sk_visual_block_remainder_drop():
|
|
# remainder='drop' is not shown in repr_html
|
|
ohe = OneHotEncoder()
|
|
ct = ColumnTransformer(transformers=[("ohe", ohe, ["col1", "col2"])])
|
|
visual_block = ct._sk_visual_block_()
|
|
assert visual_block.names == ("ohe",)
|
|
assert visual_block.name_details == (["col1", "col2"],)
|
|
assert visual_block.estimators == (ohe,)
|
|
|
|
|
|
@pytest.mark.parametrize("remainder", ["passthrough", StandardScaler()])
|
|
def test_sk_visual_block_remainder_fitted_pandas(remainder):
|
|
# Remainder shows the columns after fitting
|
|
pd = pytest.importorskip("pandas")
|
|
ohe = OneHotEncoder()
|
|
ct = ColumnTransformer(
|
|
transformers=[("ohe", ohe, ["col1", "col2"])], remainder=remainder
|
|
)
|
|
df = pd.DataFrame(
|
|
{
|
|
"col1": ["a", "b", "c"],
|
|
"col2": ["z", "z", "z"],
|
|
"col3": [1, 2, 3],
|
|
"col4": [3, 4, 5],
|
|
}
|
|
)
|
|
ct.fit(df)
|
|
visual_block = ct._sk_visual_block_()
|
|
assert visual_block.names == ("ohe", "remainder")
|
|
assert visual_block.name_details == (["col1", "col2"], ["col3", "col4"])
|
|
assert visual_block.estimators == (ohe, remainder)
|
|
|
|
|
|
@pytest.mark.parametrize("remainder", ["passthrough", StandardScaler()])
|
|
def test_sk_visual_block_remainder_fitted_numpy(remainder):
|
|
# Remainder shows the indices after fitting
|
|
X = np.array([[1, 2, 3], [4, 5, 6]], dtype=float)
|
|
scaler = StandardScaler()
|
|
ct = ColumnTransformer(
|
|
transformers=[("scale", scaler, [0, 2])], remainder=remainder
|
|
)
|
|
ct.fit(X)
|
|
visual_block = ct._sk_visual_block_()
|
|
assert visual_block.names == ("scale", "remainder")
|
|
assert visual_block.name_details == ([0, 2], [1])
|
|
assert visual_block.estimators == (scaler, remainder)
|
|
|
|
|
|
@pytest.mark.parametrize("explicit_colname", ["first", "second", 0, 1])
|
|
@pytest.mark.parametrize("remainder", [Trans(), "passthrough", "drop"])
|
|
def test_column_transformer_reordered_column_names_remainder(
|
|
explicit_colname, remainder
|
|
):
|
|
"""Test the interaction between remainder and column transformer"""
|
|
pd = pytest.importorskip("pandas")
|
|
|
|
X_fit_array = np.array([[0, 1, 2], [2, 4, 6]]).T
|
|
X_fit_df = pd.DataFrame(X_fit_array, columns=["first", "second"])
|
|
|
|
X_trans_array = np.array([[2, 4, 6], [0, 1, 2]]).T
|
|
X_trans_df = pd.DataFrame(X_trans_array, columns=["second", "first"])
|
|
|
|
tf = ColumnTransformer([("bycol", Trans(), explicit_colname)], remainder=remainder)
|
|
|
|
tf.fit(X_fit_df)
|
|
X_fit_trans = tf.transform(X_fit_df)
|
|
|
|
# Changing the order still works
|
|
X_trans = tf.transform(X_trans_df)
|
|
assert_allclose(X_trans, X_fit_trans)
|
|
|
|
# extra columns are ignored
|
|
X_extended_df = X_fit_df.copy()
|
|
X_extended_df["third"] = [3, 6, 9]
|
|
X_trans = tf.transform(X_extended_df)
|
|
assert_allclose(X_trans, X_fit_trans)
|
|
|
|
if isinstance(explicit_colname, str):
|
|
# Raise error if columns are specified by names but input only allows
|
|
# to specify by position, e.g. numpy array instead of a pandas df.
|
|
X_array = X_fit_array.copy()
|
|
err_msg = "Specifying the columns"
|
|
with pytest.raises(ValueError, match=err_msg):
|
|
tf.transform(X_array)
|
|
|
|
|
|
def test_feature_name_validation_missing_columns_drop_passthough():
|
|
"""Test the interaction between {'drop', 'passthrough'} and
|
|
missing column names."""
|
|
pd = pytest.importorskip("pandas")
|
|
|
|
X = np.ones(shape=(3, 4))
|
|
df = pd.DataFrame(X, columns=["a", "b", "c", "d"])
|
|
|
|
df_dropped = df.drop("c", axis=1)
|
|
|
|
# with remainder='passthrough', all columns seen during `fit` must be
|
|
# present
|
|
tf = ColumnTransformer([("bycol", Trans(), [1])], remainder="passthrough")
|
|
tf.fit(df)
|
|
msg = r"columns are missing: {'c'}"
|
|
with pytest.raises(ValueError, match=msg):
|
|
tf.transform(df_dropped)
|
|
|
|
# with remainder='drop', it is allowed to have column 'c' missing
|
|
tf = ColumnTransformer([("bycol", Trans(), [1])], remainder="drop")
|
|
tf.fit(df)
|
|
|
|
df_dropped_trans = tf.transform(df_dropped)
|
|
df_fit_trans = tf.transform(df)
|
|
assert_allclose(df_dropped_trans, df_fit_trans)
|
|
|
|
# bycol drops 'c', thus it is allowed for 'c' to be missing
|
|
tf = ColumnTransformer([("bycol", "drop", ["c"])], remainder="passthrough")
|
|
tf.fit(df)
|
|
df_dropped_trans = tf.transform(df_dropped)
|
|
df_fit_trans = tf.transform(df)
|
|
assert_allclose(df_dropped_trans, df_fit_trans)
|
|
|
|
|
|
def test_feature_names_in_():
|
|
"""Feature names are stored in column transformer.
|
|
|
|
Column transformer deliberately does not check for column name consistency.
|
|
It only checks that the non-dropped names seen in `fit` are seen
|
|
in `transform`. This behavior is already tested in
|
|
`test_feature_name_validation_missing_columns_drop_passthough`"""
|
|
|
|
pd = pytest.importorskip("pandas")
|
|
|
|
feature_names = ["a", "c", "d"]
|
|
df = pd.DataFrame([[1, 2, 3]], columns=feature_names)
|
|
ct = ColumnTransformer([("bycol", Trans(), ["a", "d"])], remainder="passthrough")
|
|
|
|
ct.fit(df)
|
|
assert_array_equal(ct.feature_names_in_, feature_names)
|
|
assert isinstance(ct.feature_names_in_, np.ndarray)
|
|
assert ct.feature_names_in_.dtype == object
|
|
|
|
|
|
class TransWithNames(Trans):
|
|
def __init__(self, feature_names_out=None):
|
|
self.feature_names_out = feature_names_out
|
|
|
|
def get_feature_names_out(self, input_features=None):
|
|
if self.feature_names_out is not None:
|
|
return np.asarray(self.feature_names_out, dtype=object)
|
|
return input_features
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"transformers, remainder, expected_names",
|
|
[
|
|
(
|
|
[
|
|
("bycol1", TransWithNames(), ["d", "c"]),
|
|
("bycol2", "passthrough", ["d"]),
|
|
],
|
|
"passthrough",
|
|
["bycol1__d", "bycol1__c", "bycol2__d", "remainder__a", "remainder__b"],
|
|
),
|
|
(
|
|
[
|
|
("bycol1", TransWithNames(), ["d", "c"]),
|
|
("bycol2", "passthrough", ["d"]),
|
|
],
|
|
"drop",
|
|
["bycol1__d", "bycol1__c", "bycol2__d"],
|
|
),
|
|
(
|
|
[
|
|
("bycol1", TransWithNames(), ["b"]),
|
|
("bycol2", "drop", ["d"]),
|
|
],
|
|
"passthrough",
|
|
["bycol1__b", "remainder__a", "remainder__c"],
|
|
),
|
|
(
|
|
[
|
|
("bycol1", TransWithNames(["pca1", "pca2"]), ["a", "b", "d"]),
|
|
],
|
|
"passthrough",
|
|
["bycol1__pca1", "bycol1__pca2", "remainder__c"],
|
|
),
|
|
(
|
|
[
|
|
("bycol1", TransWithNames(["a", "b"]), ["d"]),
|
|
("bycol2", "passthrough", ["b"]),
|
|
],
|
|
"drop",
|
|
["bycol1__a", "bycol1__b", "bycol2__b"],
|
|
),
|
|
(
|
|
[
|
|
("bycol1", TransWithNames([f"pca{i}" for i in range(2)]), ["b"]),
|
|
("bycol2", TransWithNames([f"pca{i}" for i in range(2)]), ["b"]),
|
|
],
|
|
"passthrough",
|
|
[
|
|
"bycol1__pca0",
|
|
"bycol1__pca1",
|
|
"bycol2__pca0",
|
|
"bycol2__pca1",
|
|
"remainder__a",
|
|
"remainder__c",
|
|
"remainder__d",
|
|
],
|
|
),
|
|
(
|
|
[
|
|
("bycol1", "drop", ["d"]),
|
|
],
|
|
"drop",
|
|
[],
|
|
),
|
|
(
|
|
[
|
|
("bycol1", TransWithNames(), slice(1, 3)),
|
|
],
|
|
"drop",
|
|
["bycol1__b", "bycol1__c"],
|
|
),
|
|
(
|
|
[
|
|
("bycol1", TransWithNames(), ["b"]),
|
|
("bycol2", "drop", slice(3, 4)),
|
|
],
|
|
"passthrough",
|
|
["bycol1__b", "remainder__a", "remainder__c"],
|
|
),
|
|
(
|
|
[
|
|
("bycol1", TransWithNames(), ["d", "c"]),
|
|
("bycol2", "passthrough", slice(3, 4)),
|
|
],
|
|
"passthrough",
|
|
["bycol1__d", "bycol1__c", "bycol2__d", "remainder__a", "remainder__b"],
|
|
),
|
|
(
|
|
[
|
|
("bycol1", TransWithNames(), slice("b", "c")),
|
|
],
|
|
"drop",
|
|
["bycol1__b", "bycol1__c"],
|
|
),
|
|
(
|
|
[
|
|
("bycol1", TransWithNames(), ["b"]),
|
|
("bycol2", "drop", slice("c", "d")),
|
|
],
|
|
"passthrough",
|
|
["bycol1__b", "remainder__a"],
|
|
),
|
|
(
|
|
[
|
|
("bycol1", TransWithNames(), ["d", "c"]),
|
|
("bycol2", "passthrough", slice("c", "d")),
|
|
],
|
|
"passthrough",
|
|
[
|
|
"bycol1__d",
|
|
"bycol1__c",
|
|
"bycol2__c",
|
|
"bycol2__d",
|
|
"remainder__a",
|
|
"remainder__b",
|
|
],
|
|
),
|
|
],
|
|
)
|
|
def test_verbose_feature_names_out_true(transformers, remainder, expected_names):
|
|
"""Check feature_names_out for verbose_feature_names_out=True (default)"""
|
|
pd = pytest.importorskip("pandas")
|
|
df = pd.DataFrame([[1, 2, 3, 4]], columns=["a", "b", "c", "d"])
|
|
ct = ColumnTransformer(
|
|
transformers,
|
|
remainder=remainder,
|
|
)
|
|
ct.fit(df)
|
|
|
|
names = ct.get_feature_names_out()
|
|
assert isinstance(names, np.ndarray)
|
|
assert names.dtype == object
|
|
assert_array_equal(names, expected_names)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"transformers, remainder, expected_names",
|
|
[
|
|
(
|
|
[
|
|
("bycol1", TransWithNames(), ["d", "c"]),
|
|
("bycol2", "passthrough", ["a"]),
|
|
],
|
|
"passthrough",
|
|
["d", "c", "a", "b"],
|
|
),
|
|
(
|
|
[
|
|
("bycol1", TransWithNames(["a"]), ["d", "c"]),
|
|
("bycol2", "passthrough", ["d"]),
|
|
],
|
|
"drop",
|
|
["a", "d"],
|
|
),
|
|
(
|
|
[
|
|
("bycol1", TransWithNames(), ["b"]),
|
|
("bycol2", "drop", ["d"]),
|
|
],
|
|
"passthrough",
|
|
["b", "a", "c"],
|
|
),
|
|
(
|
|
[
|
|
("bycol1", TransWithNames(["pca1", "pca2"]), ["a", "b", "d"]),
|
|
],
|
|
"passthrough",
|
|
["pca1", "pca2", "c"],
|
|
),
|
|
(
|
|
[
|
|
("bycol1", TransWithNames(["a", "c"]), ["d"]),
|
|
("bycol2", "passthrough", ["d"]),
|
|
],
|
|
"drop",
|
|
["a", "c", "d"],
|
|
),
|
|
(
|
|
[
|
|
("bycol1", TransWithNames([f"pca{i}" for i in range(2)]), ["b"]),
|
|
("bycol2", TransWithNames([f"kpca{i}" for i in range(2)]), ["b"]),
|
|
],
|
|
"passthrough",
|
|
["pca0", "pca1", "kpca0", "kpca1", "a", "c", "d"],
|
|
),
|
|
(
|
|
[
|
|
("bycol1", "drop", ["d"]),
|
|
],
|
|
"drop",
|
|
[],
|
|
),
|
|
(
|
|
[
|
|
("bycol1", TransWithNames(), slice(1, 2)),
|
|
("bycol2", "drop", ["d"]),
|
|
],
|
|
"passthrough",
|
|
["b", "a", "c"],
|
|
),
|
|
(
|
|
[
|
|
("bycol1", TransWithNames(), ["b"]),
|
|
("bycol2", "drop", slice(3, 4)),
|
|
],
|
|
"passthrough",
|
|
["b", "a", "c"],
|
|
),
|
|
(
|
|
[
|
|
("bycol1", TransWithNames(), ["d", "c"]),
|
|
("bycol2", "passthrough", slice(0, 2)),
|
|
],
|
|
"drop",
|
|
["d", "c", "a", "b"],
|
|
),
|
|
(
|
|
[
|
|
("bycol1", TransWithNames(), slice("a", "b")),
|
|
("bycol2", "drop", ["d"]),
|
|
],
|
|
"passthrough",
|
|
["a", "b", "c"],
|
|
),
|
|
(
|
|
[
|
|
("bycol1", TransWithNames(), ["b"]),
|
|
("bycol2", "drop", slice("c", "d")),
|
|
],
|
|
"passthrough",
|
|
["b", "a"],
|
|
),
|
|
(
|
|
[
|
|
("bycol1", TransWithNames(), ["d", "c"]),
|
|
("bycol2", "passthrough", slice("a", "b")),
|
|
],
|
|
"drop",
|
|
["d", "c", "a", "b"],
|
|
),
|
|
(
|
|
[
|
|
("bycol1", TransWithNames(), ["d", "c"]),
|
|
("bycol2", "passthrough", slice("b", "b")),
|
|
],
|
|
"drop",
|
|
["d", "c", "b"],
|
|
),
|
|
],
|
|
)
|
|
def test_verbose_feature_names_out_false(transformers, remainder, expected_names):
|
|
"""Check feature_names_out for verbose_feature_names_out=False"""
|
|
pd = pytest.importorskip("pandas")
|
|
df = pd.DataFrame([[1, 2, 3, 4]], columns=["a", "b", "c", "d"])
|
|
ct = ColumnTransformer(
|
|
transformers,
|
|
remainder=remainder,
|
|
verbose_feature_names_out=False,
|
|
)
|
|
ct.fit(df)
|
|
|
|
names = ct.get_feature_names_out()
|
|
assert isinstance(names, np.ndarray)
|
|
assert names.dtype == object
|
|
assert_array_equal(names, expected_names)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"transformers, remainder, colliding_columns",
|
|
[
|
|
(
|
|
[
|
|
("bycol1", TransWithNames(), ["b"]),
|
|
("bycol2", "passthrough", ["b"]),
|
|
],
|
|
"drop",
|
|
"['b']",
|
|
),
|
|
(
|
|
[
|
|
("bycol1", TransWithNames(["c", "d"]), ["c"]),
|
|
("bycol2", "passthrough", ["c"]),
|
|
],
|
|
"drop",
|
|
"['c']",
|
|
),
|
|
(
|
|
[
|
|
("bycol1", TransWithNames(["a"]), ["b"]),
|
|
("bycol2", "passthrough", ["b"]),
|
|
],
|
|
"passthrough",
|
|
"['a']",
|
|
),
|
|
(
|
|
[
|
|
("bycol1", TransWithNames(["a"]), ["b"]),
|
|
("bycol2", "drop", ["b"]),
|
|
],
|
|
"passthrough",
|
|
"['a']",
|
|
),
|
|
(
|
|
[
|
|
("bycol1", TransWithNames(["c", "b"]), ["b"]),
|
|
("bycol2", "passthrough", ["c", "b"]),
|
|
],
|
|
"drop",
|
|
"['b', 'c']",
|
|
),
|
|
(
|
|
[
|
|
("bycol1", TransWithNames(["a"]), ["b"]),
|
|
("bycol2", "passthrough", ["a"]),
|
|
("bycol3", TransWithNames(["a"]), ["b"]),
|
|
],
|
|
"passthrough",
|
|
"['a']",
|
|
),
|
|
(
|
|
[
|
|
("bycol1", TransWithNames(["a", "b"]), ["b"]),
|
|
("bycol2", "passthrough", ["a"]),
|
|
("bycol3", TransWithNames(["b"]), ["c"]),
|
|
],
|
|
"passthrough",
|
|
"['a', 'b']",
|
|
),
|
|
(
|
|
[
|
|
("bycol1", TransWithNames([f"pca{i}" for i in range(6)]), ["b"]),
|
|
("bycol2", TransWithNames([f"pca{i}" for i in range(6)]), ["b"]),
|
|
],
|
|
"passthrough",
|
|
"['pca0', 'pca1', 'pca2', 'pca3', 'pca4', ...]",
|
|
),
|
|
(
|
|
[
|
|
("bycol1", TransWithNames(["a", "b"]), slice(1, 2)),
|
|
("bycol2", "passthrough", ["a"]),
|
|
("bycol3", TransWithNames(["b"]), ["c"]),
|
|
],
|
|
"passthrough",
|
|
"['a', 'b']",
|
|
),
|
|
(
|
|
[
|
|
("bycol1", TransWithNames(["a", "b"]), ["b"]),
|
|
("bycol2", "passthrough", slice(0, 1)),
|
|
("bycol3", TransWithNames(["b"]), ["c"]),
|
|
],
|
|
"passthrough",
|
|
"['a', 'b']",
|
|
),
|
|
(
|
|
[
|
|
("bycol1", TransWithNames(["a", "b"]), slice("b", "c")),
|
|
("bycol2", "passthrough", ["a"]),
|
|
("bycol3", TransWithNames(["b"]), ["c"]),
|
|
],
|
|
"passthrough",
|
|
"['a', 'b']",
|
|
),
|
|
(
|
|
[
|
|
("bycol1", TransWithNames(["a", "b"]), ["b"]),
|
|
("bycol2", "passthrough", slice("a", "a")),
|
|
("bycol3", TransWithNames(["b"]), ["c"]),
|
|
],
|
|
"passthrough",
|
|
"['a', 'b']",
|
|
),
|
|
],
|
|
)
|
|
def test_verbose_feature_names_out_false_errors(
|
|
transformers, remainder, colliding_columns
|
|
):
|
|
"""Check feature_names_out for verbose_feature_names_out=False"""
|
|
|
|
pd = pytest.importorskip("pandas")
|
|
df = pd.DataFrame([[1, 2, 3, 4]], columns=["a", "b", "c", "d"])
|
|
ct = ColumnTransformer(
|
|
transformers,
|
|
remainder=remainder,
|
|
verbose_feature_names_out=False,
|
|
)
|
|
ct.fit(df)
|
|
|
|
msg = re.escape(
|
|
f"Output feature names: {colliding_columns} are not unique. Please set "
|
|
"verbose_feature_names_out=True to add prefixes to feature names"
|
|
)
|
|
with pytest.raises(ValueError, match=msg):
|
|
ct.get_feature_names_out()
|
|
|
|
|
|
@pytest.mark.parametrize("verbose_feature_names_out", [True, False])
|
|
@pytest.mark.parametrize("remainder", ["drop", "passthrough"])
|
|
def test_column_transformer_set_output(verbose_feature_names_out, remainder):
|
|
"""Check column transformer behavior with set_output."""
|
|
pd = pytest.importorskip("pandas")
|
|
df = pd.DataFrame([[1, 2, 3, 4]], columns=["a", "b", "c", "d"], index=[10])
|
|
ct = ColumnTransformer(
|
|
[("first", TransWithNames(), ["a", "c"]), ("second", TransWithNames(), ["d"])],
|
|
remainder=remainder,
|
|
verbose_feature_names_out=verbose_feature_names_out,
|
|
)
|
|
X_trans = ct.fit_transform(df)
|
|
assert isinstance(X_trans, np.ndarray)
|
|
|
|
ct.set_output(transform="pandas")
|
|
|
|
df_test = pd.DataFrame([[1, 2, 3, 4]], columns=df.columns, index=[20])
|
|
X_trans = ct.transform(df_test)
|
|
assert isinstance(X_trans, pd.DataFrame)
|
|
|
|
feature_names_out = ct.get_feature_names_out()
|
|
assert_array_equal(X_trans.columns, feature_names_out)
|
|
assert_array_equal(X_trans.index, df_test.index)
|
|
|
|
|
|
@pytest.mark.parametrize("remainder", ["drop", "passthrough"])
|
|
@pytest.mark.parametrize("fit_transform", [True, False])
|
|
def test_column_transform_set_output_mixed(remainder, fit_transform):
|
|
"""Check ColumnTransformer outputs mixed types correctly."""
|
|
pd = pytest.importorskip("pandas")
|
|
df = pd.DataFrame(
|
|
{
|
|
"pet": pd.Series(["dog", "cat", "snake"], dtype="category"),
|
|
"color": pd.Series(["green", "blue", "red"], dtype="object"),
|
|
"age": [1.4, 2.1, 4.4],
|
|
"height": [20, 40, 10],
|
|
"distance": pd.Series([20, pd.NA, 100], dtype="Int32"),
|
|
}
|
|
)
|
|
ct = ColumnTransformer(
|
|
[
|
|
(
|
|
"color_encode",
|
|
OneHotEncoder(sparse_output=False, dtype="int8"),
|
|
["color"],
|
|
),
|
|
("age", StandardScaler(), ["age"]),
|
|
],
|
|
remainder=remainder,
|
|
verbose_feature_names_out=False,
|
|
).set_output(transform="pandas")
|
|
if fit_transform:
|
|
X_trans = ct.fit_transform(df)
|
|
else:
|
|
X_trans = ct.fit(df).transform(df)
|
|
|
|
assert isinstance(X_trans, pd.DataFrame)
|
|
assert_array_equal(X_trans.columns, ct.get_feature_names_out())
|
|
|
|
expected_dtypes = {
|
|
"color_blue": "int8",
|
|
"color_green": "int8",
|
|
"color_red": "int8",
|
|
"age": "float64",
|
|
"pet": "category",
|
|
"height": "int64",
|
|
"distance": "Int32",
|
|
}
|
|
for col, dtype in X_trans.dtypes.items():
|
|
assert dtype == expected_dtypes[col]
|
|
|
|
|
|
@pytest.mark.parametrize("remainder", ["drop", "passthrough"])
|
|
def test_column_transform_set_output_after_fitting(remainder):
|
|
pd = pytest.importorskip("pandas")
|
|
df = pd.DataFrame(
|
|
{
|
|
"pet": pd.Series(["dog", "cat", "snake"], dtype="category"),
|
|
"age": [1.4, 2.1, 4.4],
|
|
"height": [20, 40, 10],
|
|
}
|
|
)
|
|
ct = ColumnTransformer(
|
|
[
|
|
(
|
|
"color_encode",
|
|
OneHotEncoder(sparse_output=False, dtype="int16"),
|
|
["pet"],
|
|
),
|
|
("age", StandardScaler(), ["age"]),
|
|
],
|
|
remainder=remainder,
|
|
verbose_feature_names_out=False,
|
|
)
|
|
|
|
# fit without calling set_output
|
|
X_trans = ct.fit_transform(df)
|
|
assert isinstance(X_trans, np.ndarray)
|
|
assert X_trans.dtype == "float64"
|
|
|
|
ct.set_output(transform="pandas")
|
|
X_trans_df = ct.transform(df)
|
|
expected_dtypes = {
|
|
"pet_cat": "int16",
|
|
"pet_dog": "int16",
|
|
"pet_snake": "int16",
|
|
"height": "int64",
|
|
"age": "float64",
|
|
}
|
|
for col, dtype in X_trans_df.dtypes.items():
|
|
assert dtype == expected_dtypes[col]
|
|
|
|
|
|
# PandasOutTransformer that does not define get_feature_names_out and always expects
|
|
# the input to be a DataFrame.
|
|
class PandasOutTransformer(BaseEstimator):
|
|
def __init__(self, offset=1.0):
|
|
self.offset = offset
|
|
|
|
def fit(self, X, y=None):
|
|
pd = pytest.importorskip("pandas")
|
|
assert isinstance(X, pd.DataFrame)
|
|
return self
|
|
|
|
def transform(self, X, y=None):
|
|
pd = pytest.importorskip("pandas")
|
|
assert isinstance(X, pd.DataFrame)
|
|
return X - self.offset
|
|
|
|
def set_output(self, transform=None):
|
|
# This transformer will always output a DataFrame regardless of the
|
|
# configuration.
|
|
return self
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"trans_1, expected_verbose_names, expected_non_verbose_names",
|
|
[
|
|
(
|
|
PandasOutTransformer(offset=2.0),
|
|
["trans_0__feat1", "trans_1__feat0"],
|
|
["feat1", "feat0"],
|
|
),
|
|
(
|
|
"drop",
|
|
["trans_0__feat1"],
|
|
["feat1"],
|
|
),
|
|
(
|
|
"passthrough",
|
|
["trans_0__feat1", "trans_1__feat0"],
|
|
["feat1", "feat0"],
|
|
),
|
|
],
|
|
)
|
|
def test_transformers_with_pandas_out_but_not_feature_names_out(
|
|
trans_1, expected_verbose_names, expected_non_verbose_names
|
|
):
|
|
"""Check that set_config(transform="pandas") is compatible with more transformers.
|
|
|
|
Specifically, if transformers returns a DataFrame, but does not define
|
|
`get_feature_names_out`.
|
|
"""
|
|
pd = pytest.importorskip("pandas")
|
|
|
|
X_df = pd.DataFrame({"feat0": [1.0, 2.0, 3.0], "feat1": [2.0, 3.0, 4.0]})
|
|
ct = ColumnTransformer(
|
|
[
|
|
("trans_0", PandasOutTransformer(offset=3.0), ["feat1"]),
|
|
("trans_1", trans_1, ["feat0"]),
|
|
]
|
|
)
|
|
X_trans_np = ct.fit_transform(X_df)
|
|
assert isinstance(X_trans_np, np.ndarray)
|
|
|
|
# `ct` does not have `get_feature_names_out` because `PandasOutTransformer` does
|
|
# not define the method.
|
|
with pytest.raises(AttributeError, match="not provide get_feature_names_out"):
|
|
ct.get_feature_names_out()
|
|
|
|
# The feature names are prefixed because verbose_feature_names_out=True is default
|
|
ct.set_output(transform="pandas")
|
|
X_trans_df0 = ct.fit_transform(X_df)
|
|
assert_array_equal(X_trans_df0.columns, expected_verbose_names)
|
|
|
|
ct.set_params(verbose_feature_names_out=False)
|
|
X_trans_df1 = ct.fit_transform(X_df)
|
|
assert_array_equal(X_trans_df1.columns, expected_non_verbose_names)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"empty_selection",
|
|
[[], np.array([False, False]), [False, False]],
|
|
ids=["list", "bool", "bool_int"],
|
|
)
|
|
def test_empty_selection_pandas_output(empty_selection):
|
|
"""Check that pandas output works when there is an empty selection.
|
|
|
|
Non-regression test for gh-25487
|
|
"""
|
|
pd = pytest.importorskip("pandas")
|
|
|
|
X = pd.DataFrame([[1.0, 2.2], [3.0, 1.0]], columns=["a", "b"])
|
|
ct = ColumnTransformer(
|
|
[
|
|
("categorical", "passthrough", empty_selection),
|
|
("numerical", StandardScaler(), ["a", "b"]),
|
|
],
|
|
verbose_feature_names_out=True,
|
|
)
|
|
ct.set_output(transform="pandas")
|
|
X_out = ct.fit_transform(X)
|
|
assert_array_equal(X_out.columns, ["numerical__a", "numerical__b"])
|
|
|
|
ct.set_params(verbose_feature_names_out=False)
|
|
X_out = ct.fit_transform(X)
|
|
assert_array_equal(X_out.columns, ["a", "b"])
|