2117 lines
77 KiB
Python
2117 lines
77 KiB
Python
"""
|
|
Testing for the tree module (sklearn.tree).
|
|
"""
|
|
import copy
|
|
import pickle
|
|
from itertools import product
|
|
import struct
|
|
|
|
import pytest
|
|
import numpy as np
|
|
from numpy.testing import assert_allclose
|
|
from scipy.sparse import csc_matrix
|
|
from scipy.sparse import csr_matrix
|
|
from scipy.sparse import coo_matrix
|
|
|
|
from sklearn.random_projection import _sparse_random_matrix
|
|
|
|
from sklearn.dummy import DummyRegressor
|
|
|
|
from sklearn.metrics import accuracy_score
|
|
from sklearn.metrics import mean_squared_error
|
|
from sklearn.metrics import mean_poisson_deviance
|
|
|
|
from sklearn.model_selection import train_test_split
|
|
|
|
from sklearn.utils._testing import assert_array_equal
|
|
from sklearn.utils._testing import assert_array_almost_equal
|
|
from sklearn.utils._testing import assert_almost_equal
|
|
from sklearn.utils._testing import assert_warns
|
|
from sklearn.utils._testing import assert_warns_message
|
|
from sklearn.utils._testing import create_memmap_backed_data
|
|
from sklearn.utils._testing import ignore_warnings
|
|
from sklearn.utils._testing import skip_if_32bit
|
|
|
|
from sklearn.utils.estimator_checks import check_sample_weights_invariance
|
|
from sklearn.utils.validation import check_random_state
|
|
|
|
from sklearn.exceptions import NotFittedError
|
|
|
|
from sklearn.tree import DecisionTreeClassifier
|
|
from sklearn.tree import DecisionTreeRegressor
|
|
from sklearn.tree import ExtraTreeClassifier
|
|
from sklearn.tree import ExtraTreeRegressor
|
|
|
|
from sklearn import tree
|
|
from sklearn.tree._tree import TREE_LEAF, TREE_UNDEFINED
|
|
from sklearn.tree._classes import CRITERIA_CLF
|
|
from sklearn.tree._classes import CRITERIA_REG
|
|
from sklearn import datasets
|
|
|
|
from sklearn.utils import compute_sample_weight
|
|
|
|
CLF_CRITERIONS = ("gini", "entropy")
|
|
REG_CRITERIONS = ("mse", "mae", "friedman_mse", "poisson")
|
|
|
|
CLF_TREES = {
|
|
"DecisionTreeClassifier": DecisionTreeClassifier,
|
|
"ExtraTreeClassifier": ExtraTreeClassifier,
|
|
}
|
|
|
|
REG_TREES = {
|
|
"DecisionTreeRegressor": DecisionTreeRegressor,
|
|
"ExtraTreeRegressor": ExtraTreeRegressor,
|
|
}
|
|
|
|
ALL_TREES = dict()
|
|
ALL_TREES.update(CLF_TREES)
|
|
ALL_TREES.update(REG_TREES)
|
|
|
|
SPARSE_TREES = ["DecisionTreeClassifier", "DecisionTreeRegressor",
|
|
"ExtraTreeClassifier", "ExtraTreeRegressor"]
|
|
|
|
|
|
X_small = np.array([
|
|
[0, 0, 4, 0, 0, 0, 1, -14, 0, -4, 0, 0, 0, 0, ],
|
|
[0, 0, 5, 3, 0, -4, 0, 0, 1, -5, 0.2, 0, 4, 1, ],
|
|
[-1, -1, 0, 0, -4.5, 0, 0, 2.1, 1, 0, 0, -4.5, 0, 1, ],
|
|
[-1, -1, 0, -1.2, 0, 0, 0, 0, 0, 0, 0.2, 0, 0, 1, ],
|
|
[-1, -1, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, ],
|
|
[-1, -2, 0, 4, -3, 10, 4, 0, -3.2, 0, 4, 3, -4, 1, ],
|
|
[2.11, 0, -6, -0.5, 0, 11, 0, 0, -3.2, 6, 0.5, 0, -3, 1, ],
|
|
[2.11, 0, -6, -0.5, 0, 11, 0, 0, -3.2, 6, 0, 0, -2, 1, ],
|
|
[2.11, 8, -6, -0.5, 0, 11, 0, 0, -3.2, 6, 0, 0, -2, 1, ],
|
|
[2.11, 8, -6, -0.5, 0, 11, 0, 0, -3.2, 6, 0.5, 0, -1, 0, ],
|
|
[2, 8, 5, 1, 0.5, -4, 10, 0, 1, -5, 3, 0, 2, 0, ],
|
|
[2, 0, 1, 1, 1, -1, 1, 0, 0, -2, 3, 0, 1, 0, ],
|
|
[2, 0, 1, 2, 3, -1, 10, 2, 0, -1, 1, 2, 2, 0, ],
|
|
[1, 1, 0, 2, 2, -1, 1, 2, 0, -5, 1, 2, 3, 0, ],
|
|
[3, 1, 0, 3, 0, -4, 10, 0, 1, -5, 3, 0, 3, 1, ],
|
|
[2.11, 8, -6, -0.5, 0, 1, 0, 0, -3.2, 6, 0.5, 0, -3, 1, ],
|
|
[2.11, 8, -6, -0.5, 0, 1, 0, 0, -3.2, 6, 1.5, 1, -1, -1, ],
|
|
[2.11, 8, -6, -0.5, 0, 10, 0, 0, -3.2, 6, 0.5, 0, -1, -1, ],
|
|
[2, 0, 5, 1, 0.5, -2, 10, 0, 1, -5, 3, 1, 0, -1, ],
|
|
[2, 0, 1, 1, 1, -2, 1, 0, 0, -2, 0, 0, 0, 1, ],
|
|
[2, 1, 1, 1, 2, -1, 10, 2, 0, -1, 0, 2, 1, 1, ],
|
|
[1, 1, 0, 0, 1, -3, 1, 2, 0, -5, 1, 2, 1, 1, ],
|
|
[3, 1, 0, 1, 0, -4, 1, 0, 1, -2, 0, 0, 1, 0, ]])
|
|
|
|
y_small = [1, 1, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, 1, 0, 0,
|
|
0, 0]
|
|
y_small_reg = [1.0, 2.1, 1.2, 0.05, 10, 2.4, 3.1, 1.01, 0.01, 2.98, 3.1, 1.1,
|
|
0.0, 1.2, 2, 11, 0, 0, 4.5, 0.201, 1.06, 0.9, 0]
|
|
|
|
# toy sample
|
|
X = [[-2, -1], [-1, -1], [-1, -2], [1, 1], [1, 2], [2, 1]]
|
|
y = [-1, -1, -1, 1, 1, 1]
|
|
T = [[-1, -1], [2, 2], [3, 2]]
|
|
true_result = [-1, 1, 1]
|
|
|
|
# also load the iris dataset
|
|
# and randomly permute it
|
|
iris = datasets.load_iris()
|
|
rng = np.random.RandomState(1)
|
|
perm = rng.permutation(iris.target.size)
|
|
iris.data = iris.data[perm]
|
|
iris.target = iris.target[perm]
|
|
|
|
# also load the diabetes dataset
|
|
# and randomly permute it
|
|
diabetes = datasets.load_diabetes()
|
|
perm = rng.permutation(diabetes.target.size)
|
|
diabetes.data = diabetes.data[perm]
|
|
diabetes.target = diabetes.target[perm]
|
|
|
|
digits = datasets.load_digits()
|
|
perm = rng.permutation(digits.target.size)
|
|
digits.data = digits.data[perm]
|
|
digits.target = digits.target[perm]
|
|
|
|
random_state = check_random_state(0)
|
|
X_multilabel, y_multilabel = datasets.make_multilabel_classification(
|
|
random_state=0, n_samples=30, n_features=10)
|
|
|
|
# NB: despite their names X_sparse_* are numpy arrays (and not sparse matrices)
|
|
X_sparse_pos = random_state.uniform(size=(20, 5))
|
|
X_sparse_pos[X_sparse_pos <= 0.8] = 0.
|
|
y_random = random_state.randint(0, 4, size=(20, ))
|
|
X_sparse_mix = _sparse_random_matrix(20, 10, density=0.25,
|
|
random_state=0).toarray()
|
|
|
|
|
|
DATASETS = {
|
|
"iris": {"X": iris.data, "y": iris.target},
|
|
"diabetes": {"X": diabetes.data, "y": diabetes.target},
|
|
"digits": {"X": digits.data, "y": digits.target},
|
|
"toy": {"X": X, "y": y},
|
|
"clf_small": {"X": X_small, "y": y_small},
|
|
"reg_small": {"X": X_small, "y": y_small_reg},
|
|
"multilabel": {"X": X_multilabel, "y": y_multilabel},
|
|
"sparse-pos": {"X": X_sparse_pos, "y": y_random},
|
|
"sparse-neg": {"X": - X_sparse_pos, "y": y_random},
|
|
"sparse-mix": {"X": X_sparse_mix, "y": y_random},
|
|
"zeros": {"X": np.zeros((20, 3)), "y": y_random}
|
|
}
|
|
|
|
for name in DATASETS:
|
|
DATASETS[name]["X_sparse"] = csc_matrix(DATASETS[name]["X"])
|
|
|
|
|
|
def assert_tree_equal(d, s, message):
|
|
assert s.node_count == d.node_count, (
|
|
"{0}: inequal number of node ({1} != {2})"
|
|
"".format(message, s.node_count, d.node_count))
|
|
|
|
assert_array_equal(d.children_right, s.children_right,
|
|
message + ": inequal children_right")
|
|
assert_array_equal(d.children_left, s.children_left,
|
|
message + ": inequal children_left")
|
|
|
|
external = d.children_right == TREE_LEAF
|
|
internal = np.logical_not(external)
|
|
|
|
assert_array_equal(d.feature[internal], s.feature[internal],
|
|
message + ": inequal features")
|
|
assert_array_equal(d.threshold[internal], s.threshold[internal],
|
|
message + ": inequal threshold")
|
|
assert_array_equal(d.n_node_samples.sum(), s.n_node_samples.sum(),
|
|
message + ": inequal sum(n_node_samples)")
|
|
assert_array_equal(d.n_node_samples, s.n_node_samples,
|
|
message + ": inequal n_node_samples")
|
|
|
|
assert_almost_equal(d.impurity, s.impurity,
|
|
err_msg=message + ": inequal impurity")
|
|
|
|
assert_array_almost_equal(d.value[external], s.value[external],
|
|
err_msg=message + ": inequal value")
|
|
|
|
|
|
def test_classification_toy():
|
|
# Check classification on a toy dataset.
|
|
for name, Tree in CLF_TREES.items():
|
|
clf = Tree(random_state=0)
|
|
clf.fit(X, y)
|
|
assert_array_equal(clf.predict(T), true_result,
|
|
"Failed with {0}".format(name))
|
|
|
|
clf = Tree(max_features=1, random_state=1)
|
|
clf.fit(X, y)
|
|
assert_array_equal(clf.predict(T), true_result,
|
|
"Failed with {0}".format(name))
|
|
|
|
|
|
def test_weighted_classification_toy():
|
|
# Check classification on a weighted toy dataset.
|
|
for name, Tree in CLF_TREES.items():
|
|
clf = Tree(random_state=0)
|
|
|
|
clf.fit(X, y, sample_weight=np.ones(len(X)))
|
|
assert_array_equal(clf.predict(T), true_result,
|
|
"Failed with {0}".format(name))
|
|
|
|
clf.fit(X, y, sample_weight=np.full(len(X), 0.5))
|
|
assert_array_equal(clf.predict(T), true_result,
|
|
"Failed with {0}".format(name))
|
|
|
|
|
|
@pytest.mark.parametrize("Tree", REG_TREES.values())
|
|
@pytest.mark.parametrize("criterion", REG_CRITERIONS)
|
|
def test_regression_toy(Tree, criterion):
|
|
# Check regression on a toy dataset.
|
|
if criterion == "poisson":
|
|
# make target positive while not touching the original y and
|
|
# true_result
|
|
a = np.abs(np.min(y)) + 1
|
|
y_train = np.array(y) + a
|
|
y_test = np.array(true_result) + a
|
|
else:
|
|
y_train = y
|
|
y_test = true_result
|
|
|
|
reg = Tree(criterion=criterion, random_state=1)
|
|
reg.fit(X, y_train)
|
|
assert_allclose(reg.predict(T), y_test)
|
|
|
|
clf = Tree(criterion=criterion, max_features=1, random_state=1)
|
|
clf.fit(X, y_train)
|
|
assert_allclose(reg.predict(T), y_test)
|
|
|
|
|
|
def test_xor():
|
|
# Check on a XOR problem
|
|
y = np.zeros((10, 10))
|
|
y[:5, :5] = 1
|
|
y[5:, 5:] = 1
|
|
|
|
gridx, gridy = np.indices(y.shape)
|
|
|
|
X = np.vstack([gridx.ravel(), gridy.ravel()]).T
|
|
y = y.ravel()
|
|
|
|
for name, Tree in CLF_TREES.items():
|
|
clf = Tree(random_state=0)
|
|
clf.fit(X, y)
|
|
assert clf.score(X, y) == 1.0, "Failed with {0}".format(name)
|
|
|
|
clf = Tree(random_state=0, max_features=1)
|
|
clf.fit(X, y)
|
|
assert clf.score(X, y) == 1.0, "Failed with {0}".format(name)
|
|
|
|
|
|
def test_iris():
|
|
# Check consistency on dataset iris.
|
|
for (name, Tree), criterion in product(CLF_TREES.items(), CLF_CRITERIONS):
|
|
clf = Tree(criterion=criterion, random_state=0)
|
|
clf.fit(iris.data, iris.target)
|
|
score = accuracy_score(clf.predict(iris.data), iris.target)
|
|
assert score > 0.9, (
|
|
"Failed with {0}, criterion = {1} and score = {2}"
|
|
"".format(name, criterion, score))
|
|
|
|
clf = Tree(criterion=criterion, max_features=2, random_state=0)
|
|
clf.fit(iris.data, iris.target)
|
|
score = accuracy_score(clf.predict(iris.data), iris.target)
|
|
assert score > 0.5, (
|
|
"Failed with {0}, criterion = {1} and score = {2}"
|
|
"".format(name, criterion, score))
|
|
|
|
|
|
@pytest.mark.parametrize("name, Tree", REG_TREES.items())
|
|
@pytest.mark.parametrize("criterion", REG_CRITERIONS)
|
|
def test_diabetes_overfit(name, Tree, criterion):
|
|
# check consistency of overfitted trees on the diabetes dataset
|
|
# since the trees will overfit, we expect an MSE of 0
|
|
reg = Tree(criterion=criterion, random_state=0)
|
|
reg.fit(diabetes.data, diabetes.target)
|
|
score = mean_squared_error(diabetes.target, reg.predict(diabetes.data))
|
|
assert score == pytest.approx(0), (
|
|
f"Failed with {name}, criterion = {criterion} and score = {score}"
|
|
)
|
|
|
|
|
|
@skip_if_32bit
|
|
@pytest.mark.parametrize("name, Tree", REG_TREES.items())
|
|
@pytest.mark.parametrize(
|
|
"criterion, max_depth, metric, max_loss",
|
|
[("mse", 15, mean_squared_error, 60),
|
|
("mae", 20, mean_squared_error, 60),
|
|
("friedman_mse", 15, mean_squared_error, 60),
|
|
("poisson", 15, mean_poisson_deviance, 30)]
|
|
)
|
|
def test_diabetes_underfit(name, Tree, criterion, max_depth, metric, max_loss):
|
|
# check consistency of trees when the depth and the number of features are
|
|
# limited
|
|
|
|
reg = Tree(
|
|
criterion=criterion, max_depth=max_depth,
|
|
max_features=6, random_state=0
|
|
)
|
|
reg.fit(diabetes.data, diabetes.target)
|
|
loss = metric(diabetes.target, reg.predict(diabetes.data))
|
|
assert 0 < loss < max_loss
|
|
|
|
|
|
def test_probability():
|
|
# Predict probabilities using DecisionTreeClassifier.
|
|
|
|
for name, Tree in CLF_TREES.items():
|
|
clf = Tree(max_depth=1, max_features=1, random_state=42)
|
|
clf.fit(iris.data, iris.target)
|
|
|
|
prob_predict = clf.predict_proba(iris.data)
|
|
assert_array_almost_equal(np.sum(prob_predict, 1),
|
|
np.ones(iris.data.shape[0]),
|
|
err_msg="Failed with {0}".format(name))
|
|
assert_array_equal(np.argmax(prob_predict, 1),
|
|
clf.predict(iris.data),
|
|
err_msg="Failed with {0}".format(name))
|
|
assert_almost_equal(clf.predict_proba(iris.data),
|
|
np.exp(clf.predict_log_proba(iris.data)), 8,
|
|
err_msg="Failed with {0}".format(name))
|
|
|
|
|
|
def test_arrayrepr():
|
|
# Check the array representation.
|
|
# Check resize
|
|
X = np.arange(10000)[:, np.newaxis]
|
|
y = np.arange(10000)
|
|
|
|
for name, Tree in REG_TREES.items():
|
|
reg = Tree(max_depth=None, random_state=0)
|
|
reg.fit(X, y)
|
|
|
|
|
|
def test_pure_set():
|
|
# Check when y is pure.
|
|
X = [[-2, -1], [-1, -1], [-1, -2], [1, 1], [1, 2], [2, 1]]
|
|
y = [1, 1, 1, 1, 1, 1]
|
|
|
|
for name, TreeClassifier in CLF_TREES.items():
|
|
clf = TreeClassifier(random_state=0)
|
|
clf.fit(X, y)
|
|
assert_array_equal(clf.predict(X), y,
|
|
err_msg="Failed with {0}".format(name))
|
|
|
|
for name, TreeRegressor in REG_TREES.items():
|
|
reg = TreeRegressor(random_state=0)
|
|
reg.fit(X, y)
|
|
assert_almost_equal(reg.predict(X), y,
|
|
err_msg="Failed with {0}".format(name))
|
|
|
|
|
|
def test_numerical_stability():
|
|
# Check numerical stability.
|
|
X = np.array([
|
|
[152.08097839, 140.40744019, 129.75102234, 159.90493774],
|
|
[142.50700378, 135.81935120, 117.82884979, 162.75781250],
|
|
[127.28772736, 140.40744019, 129.75102234, 159.90493774],
|
|
[132.37025452, 143.71923828, 138.35694885, 157.84558105],
|
|
[103.10237122, 143.71928406, 138.35696411, 157.84559631],
|
|
[127.71276855, 143.71923828, 138.35694885, 157.84558105],
|
|
[120.91514587, 140.40744019, 129.75102234, 159.90493774]])
|
|
|
|
y = np.array(
|
|
[1., 0.70209277, 0.53896582, 0., 0.90914464, 0.48026916, 0.49622521])
|
|
|
|
with np.errstate(all="raise"):
|
|
for name, Tree in REG_TREES.items():
|
|
reg = Tree(random_state=0)
|
|
reg.fit(X, y)
|
|
reg.fit(X, -y)
|
|
reg.fit(-X, y)
|
|
reg.fit(-X, -y)
|
|
|
|
|
|
def test_importances():
|
|
# Check variable importances.
|
|
X, y = datasets.make_classification(n_samples=5000,
|
|
n_features=10,
|
|
n_informative=3,
|
|
n_redundant=0,
|
|
n_repeated=0,
|
|
shuffle=False,
|
|
random_state=0)
|
|
|
|
for name, Tree in CLF_TREES.items():
|
|
clf = Tree(random_state=0)
|
|
|
|
clf.fit(X, y)
|
|
importances = clf.feature_importances_
|
|
n_important = np.sum(importances > 0.1)
|
|
|
|
assert importances.shape[0] == 10, "Failed with {0}".format(name)
|
|
assert n_important == 3, "Failed with {0}".format(name)
|
|
|
|
# Check on iris that importances are the same for all builders
|
|
clf = DecisionTreeClassifier(random_state=0)
|
|
clf.fit(iris.data, iris.target)
|
|
clf2 = DecisionTreeClassifier(random_state=0,
|
|
max_leaf_nodes=len(iris.data))
|
|
clf2.fit(iris.data, iris.target)
|
|
|
|
assert_array_equal(clf.feature_importances_,
|
|
clf2.feature_importances_)
|
|
|
|
|
|
def test_importances_raises():
|
|
# Check if variable importance before fit raises ValueError.
|
|
clf = DecisionTreeClassifier()
|
|
with pytest.raises(ValueError):
|
|
getattr(clf, 'feature_importances_')
|
|
|
|
|
|
def test_importances_gini_equal_mse():
|
|
# Check that gini is equivalent to mse for binary output variable
|
|
|
|
X, y = datasets.make_classification(n_samples=2000,
|
|
n_features=10,
|
|
n_informative=3,
|
|
n_redundant=0,
|
|
n_repeated=0,
|
|
shuffle=False,
|
|
random_state=0)
|
|
|
|
# The gini index and the mean square error (variance) might differ due
|
|
# to numerical instability. Since those instabilities mainly occurs at
|
|
# high tree depth, we restrict this maximal depth.
|
|
clf = DecisionTreeClassifier(criterion="gini", max_depth=5,
|
|
random_state=0).fit(X, y)
|
|
reg = DecisionTreeRegressor(criterion="mse", max_depth=5,
|
|
random_state=0).fit(X, y)
|
|
|
|
assert_almost_equal(clf.feature_importances_, reg.feature_importances_)
|
|
assert_array_equal(clf.tree_.feature, reg.tree_.feature)
|
|
assert_array_equal(clf.tree_.children_left, reg.tree_.children_left)
|
|
assert_array_equal(clf.tree_.children_right, reg.tree_.children_right)
|
|
assert_array_equal(clf.tree_.n_node_samples, reg.tree_.n_node_samples)
|
|
|
|
|
|
def test_max_features():
|
|
# Check max_features.
|
|
for name, TreeRegressor in REG_TREES.items():
|
|
reg = TreeRegressor(max_features="auto")
|
|
reg.fit(diabetes.data, diabetes.target)
|
|
assert reg.max_features_ == diabetes.data.shape[1]
|
|
|
|
for name, TreeClassifier in CLF_TREES.items():
|
|
clf = TreeClassifier(max_features="auto")
|
|
clf.fit(iris.data, iris.target)
|
|
assert clf.max_features_ == 2
|
|
|
|
for name, TreeEstimator in ALL_TREES.items():
|
|
est = TreeEstimator(max_features="sqrt")
|
|
est.fit(iris.data, iris.target)
|
|
assert (est.max_features_ ==
|
|
int(np.sqrt(iris.data.shape[1])))
|
|
|
|
est = TreeEstimator(max_features="log2")
|
|
est.fit(iris.data, iris.target)
|
|
assert (est.max_features_ ==
|
|
int(np.log2(iris.data.shape[1])))
|
|
|
|
est = TreeEstimator(max_features=1)
|
|
est.fit(iris.data, iris.target)
|
|
assert est.max_features_ == 1
|
|
|
|
est = TreeEstimator(max_features=3)
|
|
est.fit(iris.data, iris.target)
|
|
assert est.max_features_ == 3
|
|
|
|
est = TreeEstimator(max_features=0.01)
|
|
est.fit(iris.data, iris.target)
|
|
assert est.max_features_ == 1
|
|
|
|
est = TreeEstimator(max_features=0.5)
|
|
est.fit(iris.data, iris.target)
|
|
assert (est.max_features_ ==
|
|
int(0.5 * iris.data.shape[1]))
|
|
|
|
est = TreeEstimator(max_features=1.0)
|
|
est.fit(iris.data, iris.target)
|
|
assert est.max_features_ == iris.data.shape[1]
|
|
|
|
est = TreeEstimator(max_features=None)
|
|
est.fit(iris.data, iris.target)
|
|
assert est.max_features_ == iris.data.shape[1]
|
|
|
|
# use values of max_features that are invalid
|
|
est = TreeEstimator(max_features=10)
|
|
with pytest.raises(ValueError):
|
|
est.fit(X, y)
|
|
|
|
est = TreeEstimator(max_features=-1)
|
|
with pytest.raises(ValueError):
|
|
est.fit(X, y)
|
|
|
|
est = TreeEstimator(max_features=0.0)
|
|
with pytest.raises(ValueError):
|
|
est.fit(X, y)
|
|
|
|
est = TreeEstimator(max_features=1.5)
|
|
with pytest.raises(ValueError):
|
|
est.fit(X, y)
|
|
|
|
est = TreeEstimator(max_features="foobar")
|
|
with pytest.raises(ValueError):
|
|
est.fit(X, y)
|
|
|
|
|
|
def test_error():
|
|
# Test that it gives proper exception on deficient input.
|
|
for name, TreeEstimator in CLF_TREES.items():
|
|
# predict before fit
|
|
est = TreeEstimator()
|
|
with pytest.raises(NotFittedError):
|
|
est.predict_proba(X)
|
|
|
|
est.fit(X, y)
|
|
X2 = [[-2, -1, 1]] # wrong feature shape for sample
|
|
with pytest.raises(ValueError):
|
|
est.predict_proba(X2)
|
|
|
|
for name, TreeEstimator in ALL_TREES.items():
|
|
with pytest.raises(ValueError):
|
|
TreeEstimator(min_samples_leaf=-1).fit(X, y)
|
|
with pytest.raises(ValueError):
|
|
TreeEstimator(min_samples_leaf=.6).fit(X, y)
|
|
with pytest.raises(ValueError):
|
|
TreeEstimator(min_samples_leaf=0.).fit(X, y)
|
|
with pytest.raises(ValueError):
|
|
TreeEstimator(min_samples_leaf=3.).fit(X, y)
|
|
with pytest.raises(ValueError):
|
|
TreeEstimator(min_weight_fraction_leaf=-1).fit(X, y)
|
|
with pytest.raises(ValueError):
|
|
TreeEstimator(min_weight_fraction_leaf=0.51).fit(X, y)
|
|
with pytest.raises(ValueError):
|
|
TreeEstimator(min_samples_split=-1).fit(X, y)
|
|
with pytest.raises(ValueError):
|
|
TreeEstimator(min_samples_split=0.0).fit(X, y)
|
|
with pytest.raises(ValueError):
|
|
TreeEstimator(min_samples_split=1.1).fit(X, y)
|
|
with pytest.raises(ValueError):
|
|
TreeEstimator(min_samples_split=2.5).fit(X, y)
|
|
with pytest.raises(ValueError):
|
|
TreeEstimator(max_depth=-1).fit(X, y)
|
|
with pytest.raises(ValueError):
|
|
TreeEstimator(max_features=42).fit(X, y)
|
|
# min_impurity_split warning
|
|
with ignore_warnings(category=FutureWarning):
|
|
with pytest.raises(ValueError):
|
|
TreeEstimator(min_impurity_split=-1.0).fit(X, y)
|
|
with pytest.raises(ValueError):
|
|
TreeEstimator(min_impurity_decrease=-1.0).fit(X, y)
|
|
|
|
# Wrong dimensions
|
|
est = TreeEstimator()
|
|
y2 = y[:-1]
|
|
with pytest.raises(ValueError):
|
|
est.fit(X, y2)
|
|
|
|
# Test with arrays that are non-contiguous.
|
|
Xf = np.asfortranarray(X)
|
|
est = TreeEstimator()
|
|
est.fit(Xf, y)
|
|
assert_almost_equal(est.predict(T), true_result)
|
|
|
|
# predict before fitting
|
|
est = TreeEstimator()
|
|
with pytest.raises(NotFittedError):
|
|
est.predict(T)
|
|
|
|
# predict on vector with different dims
|
|
est.fit(X, y)
|
|
t = np.asarray(T)
|
|
with pytest.raises(ValueError):
|
|
est.predict(t[:, 1:])
|
|
|
|
# wrong sample shape
|
|
Xt = np.array(X).T
|
|
|
|
est = TreeEstimator()
|
|
est.fit(np.dot(X, Xt), y)
|
|
with pytest.raises(ValueError):
|
|
est.predict(X)
|
|
with pytest.raises(ValueError):
|
|
est.apply(X)
|
|
|
|
clf = TreeEstimator()
|
|
clf.fit(X, y)
|
|
with pytest.raises(ValueError):
|
|
clf.predict(Xt)
|
|
with pytest.raises(ValueError):
|
|
clf.apply(Xt)
|
|
|
|
# apply before fitting
|
|
est = TreeEstimator()
|
|
with pytest.raises(NotFittedError):
|
|
est.apply(T)
|
|
|
|
# non positive target for Poisson splitting Criterion
|
|
est = DecisionTreeRegressor(criterion="poisson")
|
|
with pytest.raises(ValueError, match="y is not positive.*Poisson"):
|
|
est.fit([[0, 1, 2]], [0, 0, 0])
|
|
with pytest.raises(ValueError, match="Some.*y are negative.*Poisson"):
|
|
est.fit([[0, 1, 2]], [5, -0.1, 2])
|
|
|
|
|
|
def test_min_samples_split():
|
|
"""Test min_samples_split parameter"""
|
|
X = np.asfortranarray(iris.data, dtype=tree._tree.DTYPE)
|
|
y = iris.target
|
|
|
|
# test both DepthFirstTreeBuilder and BestFirstTreeBuilder
|
|
# by setting max_leaf_nodes
|
|
for max_leaf_nodes, name in product((None, 1000), ALL_TREES.keys()):
|
|
TreeEstimator = ALL_TREES[name]
|
|
|
|
# test for integer parameter
|
|
est = TreeEstimator(min_samples_split=10,
|
|
max_leaf_nodes=max_leaf_nodes,
|
|
random_state=0)
|
|
est.fit(X, y)
|
|
# count samples on nodes, -1 means it is a leaf
|
|
node_samples = est.tree_.n_node_samples[est.tree_.children_left != -1]
|
|
|
|
assert np.min(node_samples) > 9, "Failed with {0}".format(name)
|
|
|
|
# test for float parameter
|
|
est = TreeEstimator(min_samples_split=0.2,
|
|
max_leaf_nodes=max_leaf_nodes,
|
|
random_state=0)
|
|
est.fit(X, y)
|
|
# count samples on nodes, -1 means it is a leaf
|
|
node_samples = est.tree_.n_node_samples[est.tree_.children_left != -1]
|
|
|
|
assert np.min(node_samples) > 9, "Failed with {0}".format(name)
|
|
|
|
|
|
def test_min_samples_leaf():
|
|
# Test if leaves contain more than leaf_count training examples
|
|
X = np.asfortranarray(iris.data, dtype=tree._tree.DTYPE)
|
|
y = iris.target
|
|
|
|
# test both DepthFirstTreeBuilder and BestFirstTreeBuilder
|
|
# by setting max_leaf_nodes
|
|
for max_leaf_nodes, name in product((None, 1000), ALL_TREES.keys()):
|
|
TreeEstimator = ALL_TREES[name]
|
|
|
|
# test integer parameter
|
|
est = TreeEstimator(min_samples_leaf=5,
|
|
max_leaf_nodes=max_leaf_nodes,
|
|
random_state=0)
|
|
est.fit(X, y)
|
|
out = est.tree_.apply(X)
|
|
node_counts = np.bincount(out)
|
|
# drop inner nodes
|
|
leaf_count = node_counts[node_counts != 0]
|
|
assert np.min(leaf_count) > 4, "Failed with {0}".format(name)
|
|
|
|
# test float parameter
|
|
est = TreeEstimator(min_samples_leaf=0.1,
|
|
max_leaf_nodes=max_leaf_nodes,
|
|
random_state=0)
|
|
est.fit(X, y)
|
|
out = est.tree_.apply(X)
|
|
node_counts = np.bincount(out)
|
|
# drop inner nodes
|
|
leaf_count = node_counts[node_counts != 0]
|
|
assert np.min(leaf_count) > 4, "Failed with {0}".format(name)
|
|
|
|
|
|
def check_min_weight_fraction_leaf(name, datasets, sparse=False):
|
|
"""Test if leaves contain at least min_weight_fraction_leaf of the
|
|
training set"""
|
|
if sparse:
|
|
X = DATASETS[datasets]["X_sparse"].astype(np.float32)
|
|
else:
|
|
X = DATASETS[datasets]["X"].astype(np.float32)
|
|
y = DATASETS[datasets]["y"]
|
|
|
|
weights = rng.rand(X.shape[0])
|
|
total_weight = np.sum(weights)
|
|
|
|
TreeEstimator = ALL_TREES[name]
|
|
|
|
# test both DepthFirstTreeBuilder and BestFirstTreeBuilder
|
|
# by setting max_leaf_nodes
|
|
for max_leaf_nodes, frac in product((None, 1000), np.linspace(0, 0.5, 6)):
|
|
est = TreeEstimator(min_weight_fraction_leaf=frac,
|
|
max_leaf_nodes=max_leaf_nodes,
|
|
random_state=0)
|
|
est.fit(X, y, sample_weight=weights)
|
|
|
|
if sparse:
|
|
out = est.tree_.apply(X.tocsr())
|
|
|
|
else:
|
|
out = est.tree_.apply(X)
|
|
|
|
node_weights = np.bincount(out, weights=weights)
|
|
# drop inner nodes
|
|
leaf_weights = node_weights[node_weights != 0]
|
|
assert (
|
|
np.min(leaf_weights) >=
|
|
total_weight * est.min_weight_fraction_leaf), (
|
|
"Failed with {0} min_weight_fraction_leaf={1}".format(
|
|
name, est.min_weight_fraction_leaf))
|
|
|
|
# test case with no weights passed in
|
|
total_weight = X.shape[0]
|
|
|
|
for max_leaf_nodes, frac in product((None, 1000), np.linspace(0, 0.5, 6)):
|
|
est = TreeEstimator(min_weight_fraction_leaf=frac,
|
|
max_leaf_nodes=max_leaf_nodes,
|
|
random_state=0)
|
|
est.fit(X, y)
|
|
|
|
if sparse:
|
|
out = est.tree_.apply(X.tocsr())
|
|
else:
|
|
out = est.tree_.apply(X)
|
|
|
|
node_weights = np.bincount(out)
|
|
# drop inner nodes
|
|
leaf_weights = node_weights[node_weights != 0]
|
|
assert (
|
|
np.min(leaf_weights) >=
|
|
total_weight * est.min_weight_fraction_leaf), (
|
|
"Failed with {0} min_weight_fraction_leaf={1}".format(
|
|
name, est.min_weight_fraction_leaf))
|
|
|
|
|
|
@pytest.mark.parametrize("name", ALL_TREES)
|
|
def test_min_weight_fraction_leaf_on_dense_input(name):
|
|
check_min_weight_fraction_leaf(name, "iris")
|
|
|
|
|
|
@pytest.mark.parametrize("name", SPARSE_TREES)
|
|
def test_min_weight_fraction_leaf_on_sparse_input(name):
|
|
check_min_weight_fraction_leaf(name, "multilabel", True)
|
|
|
|
|
|
def check_min_weight_fraction_leaf_with_min_samples_leaf(name, datasets,
|
|
sparse=False):
|
|
"""Test the interaction between min_weight_fraction_leaf and
|
|
min_samples_leaf when sample_weights is not provided in fit."""
|
|
if sparse:
|
|
X = DATASETS[datasets]["X_sparse"].astype(np.float32)
|
|
else:
|
|
X = DATASETS[datasets]["X"].astype(np.float32)
|
|
y = DATASETS[datasets]["y"]
|
|
|
|
total_weight = X.shape[0]
|
|
TreeEstimator = ALL_TREES[name]
|
|
for max_leaf_nodes, frac in product((None, 1000), np.linspace(0, 0.5, 3)):
|
|
# test integer min_samples_leaf
|
|
est = TreeEstimator(min_weight_fraction_leaf=frac,
|
|
max_leaf_nodes=max_leaf_nodes,
|
|
min_samples_leaf=5,
|
|
random_state=0)
|
|
est.fit(X, y)
|
|
|
|
if sparse:
|
|
out = est.tree_.apply(X.tocsr())
|
|
else:
|
|
out = est.tree_.apply(X)
|
|
|
|
node_weights = np.bincount(out)
|
|
# drop inner nodes
|
|
leaf_weights = node_weights[node_weights != 0]
|
|
assert (
|
|
np.min(leaf_weights) >=
|
|
max((total_weight *
|
|
est.min_weight_fraction_leaf), 5)), (
|
|
"Failed with {0} min_weight_fraction_leaf={1}, "
|
|
"min_samples_leaf={2}".format(
|
|
name, est.min_weight_fraction_leaf,
|
|
est.min_samples_leaf))
|
|
for max_leaf_nodes, frac in product((None, 1000), np.linspace(0, 0.5, 3)):
|
|
# test float min_samples_leaf
|
|
est = TreeEstimator(min_weight_fraction_leaf=frac,
|
|
max_leaf_nodes=max_leaf_nodes,
|
|
min_samples_leaf=.1,
|
|
random_state=0)
|
|
est.fit(X, y)
|
|
|
|
if sparse:
|
|
out = est.tree_.apply(X.tocsr())
|
|
else:
|
|
out = est.tree_.apply(X)
|
|
|
|
node_weights = np.bincount(out)
|
|
# drop inner nodes
|
|
leaf_weights = node_weights[node_weights != 0]
|
|
assert (
|
|
np.min(leaf_weights) >=
|
|
max((total_weight * est.min_weight_fraction_leaf),
|
|
(total_weight * est.min_samples_leaf))), (
|
|
"Failed with {0} min_weight_fraction_leaf={1}, "
|
|
"min_samples_leaf={2}".format(name,
|
|
est.min_weight_fraction_leaf,
|
|
est.min_samples_leaf))
|
|
|
|
|
|
@pytest.mark.parametrize("name", ALL_TREES)
|
|
def test_min_weight_fraction_leaf_with_min_samples_leaf_on_dense_input(name):
|
|
check_min_weight_fraction_leaf_with_min_samples_leaf(name, "iris")
|
|
|
|
|
|
@pytest.mark.parametrize("name", SPARSE_TREES)
|
|
def test_min_weight_fraction_leaf_with_min_samples_leaf_on_sparse_input(name):
|
|
check_min_weight_fraction_leaf_with_min_samples_leaf(
|
|
name, "multilabel", True)
|
|
|
|
|
|
def test_min_impurity_split():
|
|
# test if min_impurity_split creates leaves with impurity
|
|
# [0, min_impurity_split) when min_samples_leaf = 1 and
|
|
# min_samples_split = 2.
|
|
X = np.asfortranarray(iris.data, dtype=tree._tree.DTYPE)
|
|
y = iris.target
|
|
|
|
# test both DepthFirstTreeBuilder and BestFirstTreeBuilder
|
|
# by setting max_leaf_nodes
|
|
for max_leaf_nodes, name in product((None, 1000), ALL_TREES.keys()):
|
|
TreeEstimator = ALL_TREES[name]
|
|
min_impurity_split = .5
|
|
|
|
# verify leaf nodes without min_impurity_split less than
|
|
# impurity 1e-7
|
|
est = TreeEstimator(max_leaf_nodes=max_leaf_nodes,
|
|
random_state=0)
|
|
assert est.min_impurity_split is None, (
|
|
"Failed, min_impurity_split = {0} != None".format(
|
|
est.min_impurity_split))
|
|
try:
|
|
assert_warns(FutureWarning, est.fit, X, y)
|
|
except AssertionError:
|
|
pass
|
|
for node in range(est.tree_.node_count):
|
|
if (est.tree_.children_left[node] == TREE_LEAF or
|
|
est.tree_.children_right[node] == TREE_LEAF):
|
|
assert est.tree_.impurity[node] == 0., (
|
|
"Failed with {0} min_impurity_split={1}".format(
|
|
est.tree_.impurity[node],
|
|
est.min_impurity_split))
|
|
|
|
# verify leaf nodes have impurity [0,min_impurity_split] when using
|
|
# min_impurity_split
|
|
est = TreeEstimator(max_leaf_nodes=max_leaf_nodes,
|
|
min_impurity_split=min_impurity_split,
|
|
random_state=0)
|
|
assert_warns_message(FutureWarning,
|
|
"Use the min_impurity_decrease",
|
|
est.fit, X, y)
|
|
for node in range(est.tree_.node_count):
|
|
if (est.tree_.children_left[node] == TREE_LEAF or
|
|
est.tree_.children_right[node] == TREE_LEAF):
|
|
assert est.tree_.impurity[node] >= 0, (
|
|
"Failed with {0}, min_impurity_split={1}".format(
|
|
est.tree_.impurity[node],
|
|
est.min_impurity_split))
|
|
assert est.tree_.impurity[node] <= min_impurity_split, (
|
|
"Failed with {0}, min_impurity_split={1}".format(
|
|
est.tree_.impurity[node],
|
|
est.min_impurity_split))
|
|
|
|
|
|
def test_min_impurity_decrease():
|
|
# test if min_impurity_decrease ensure that a split is made only if
|
|
# if the impurity decrease is atleast that value
|
|
X, y = datasets.make_classification(n_samples=10000, random_state=42)
|
|
|
|
# test both DepthFirstTreeBuilder and BestFirstTreeBuilder
|
|
# by setting max_leaf_nodes
|
|
for max_leaf_nodes, name in product((None, 1000), ALL_TREES.keys()):
|
|
TreeEstimator = ALL_TREES[name]
|
|
|
|
# Check default value of min_impurity_decrease, 1e-7
|
|
est1 = TreeEstimator(max_leaf_nodes=max_leaf_nodes, random_state=0)
|
|
# Check with explicit value of 0.05
|
|
est2 = TreeEstimator(max_leaf_nodes=max_leaf_nodes,
|
|
min_impurity_decrease=0.05, random_state=0)
|
|
# Check with a much lower value of 0.0001
|
|
est3 = TreeEstimator(max_leaf_nodes=max_leaf_nodes,
|
|
min_impurity_decrease=0.0001, random_state=0)
|
|
# Check with a much lower value of 0.1
|
|
est4 = TreeEstimator(max_leaf_nodes=max_leaf_nodes,
|
|
min_impurity_decrease=0.1, random_state=0)
|
|
|
|
for est, expected_decrease in ((est1, 1e-7), (est2, 0.05),
|
|
(est3, 0.0001), (est4, 0.1)):
|
|
assert est.min_impurity_decrease <= expected_decrease, (
|
|
"Failed, min_impurity_decrease = {0} > {1}".format(
|
|
est.min_impurity_decrease,
|
|
expected_decrease))
|
|
est.fit(X, y)
|
|
for node in range(est.tree_.node_count):
|
|
# If current node is a not leaf node, check if the split was
|
|
# justified w.r.t the min_impurity_decrease
|
|
if est.tree_.children_left[node] != TREE_LEAF:
|
|
imp_parent = est.tree_.impurity[node]
|
|
wtd_n_node = est.tree_.weighted_n_node_samples[node]
|
|
|
|
left = est.tree_.children_left[node]
|
|
wtd_n_left = est.tree_.weighted_n_node_samples[left]
|
|
imp_left = est.tree_.impurity[left]
|
|
wtd_imp_left = wtd_n_left * imp_left
|
|
|
|
right = est.tree_.children_right[node]
|
|
wtd_n_right = est.tree_.weighted_n_node_samples[right]
|
|
imp_right = est.tree_.impurity[right]
|
|
wtd_imp_right = wtd_n_right * imp_right
|
|
|
|
wtd_avg_left_right_imp = wtd_imp_right + wtd_imp_left
|
|
wtd_avg_left_right_imp /= wtd_n_node
|
|
|
|
fractional_node_weight = (
|
|
est.tree_.weighted_n_node_samples[node] / X.shape[0])
|
|
|
|
actual_decrease = fractional_node_weight * (
|
|
imp_parent - wtd_avg_left_right_imp)
|
|
|
|
assert actual_decrease >= expected_decrease, (
|
|
"Failed with {0} expected min_impurity_decrease={1}"
|
|
.format(actual_decrease,
|
|
expected_decrease))
|
|
|
|
for name, TreeEstimator in ALL_TREES.items():
|
|
if "Classifier" in name:
|
|
X, y = iris.data, iris.target
|
|
else:
|
|
X, y = diabetes.data, diabetes.target
|
|
|
|
est = TreeEstimator(random_state=0)
|
|
est.fit(X, y)
|
|
score = est.score(X, y)
|
|
fitted_attribute = dict()
|
|
for attribute in ["max_depth", "node_count", "capacity"]:
|
|
fitted_attribute[attribute] = getattr(est.tree_, attribute)
|
|
|
|
serialized_object = pickle.dumps(est)
|
|
est2 = pickle.loads(serialized_object)
|
|
assert type(est2) == est.__class__
|
|
score2 = est2.score(X, y)
|
|
assert score == score2, (
|
|
"Failed to generate same score after pickling "
|
|
"with {0}".format(name))
|
|
|
|
for attribute in fitted_attribute:
|
|
assert (getattr(est2.tree_, attribute) ==
|
|
fitted_attribute[attribute]), (
|
|
"Failed to generate same attribute {0} after "
|
|
"pickling with {1}".format(attribute, name))
|
|
|
|
|
|
def test_multioutput():
|
|
# Check estimators on multi-output problems.
|
|
X = [[-2, -1],
|
|
[-1, -1],
|
|
[-1, -2],
|
|
[1, 1],
|
|
[1, 2],
|
|
[2, 1],
|
|
[-2, 1],
|
|
[-1, 1],
|
|
[-1, 2],
|
|
[2, -1],
|
|
[1, -1],
|
|
[1, -2]]
|
|
|
|
y = [[-1, 0],
|
|
[-1, 0],
|
|
[-1, 0],
|
|
[1, 1],
|
|
[1, 1],
|
|
[1, 1],
|
|
[-1, 2],
|
|
[-1, 2],
|
|
[-1, 2],
|
|
[1, 3],
|
|
[1, 3],
|
|
[1, 3]]
|
|
|
|
T = [[-1, -1], [1, 1], [-1, 1], [1, -1]]
|
|
y_true = [[-1, 0], [1, 1], [-1, 2], [1, 3]]
|
|
|
|
# toy classification problem
|
|
for name, TreeClassifier in CLF_TREES.items():
|
|
clf = TreeClassifier(random_state=0)
|
|
y_hat = clf.fit(X, y).predict(T)
|
|
assert_array_equal(y_hat, y_true)
|
|
assert y_hat.shape == (4, 2)
|
|
|
|
proba = clf.predict_proba(T)
|
|
assert len(proba) == 2
|
|
assert proba[0].shape == (4, 2)
|
|
assert proba[1].shape == (4, 4)
|
|
|
|
log_proba = clf.predict_log_proba(T)
|
|
assert len(log_proba) == 2
|
|
assert log_proba[0].shape == (4, 2)
|
|
assert log_proba[1].shape == (4, 4)
|
|
|
|
# toy regression problem
|
|
for name, TreeRegressor in REG_TREES.items():
|
|
reg = TreeRegressor(random_state=0)
|
|
y_hat = reg.fit(X, y).predict(T)
|
|
assert_almost_equal(y_hat, y_true)
|
|
assert y_hat.shape == (4, 2)
|
|
|
|
|
|
def test_classes_shape():
|
|
# Test that n_classes_ and classes_ have proper shape.
|
|
for name, TreeClassifier in CLF_TREES.items():
|
|
# Classification, single output
|
|
clf = TreeClassifier(random_state=0)
|
|
clf.fit(X, y)
|
|
|
|
assert clf.n_classes_ == 2
|
|
assert_array_equal(clf.classes_, [-1, 1])
|
|
|
|
# Classification, multi-output
|
|
_y = np.vstack((y, np.array(y) * 2)).T
|
|
clf = TreeClassifier(random_state=0)
|
|
clf.fit(X, _y)
|
|
assert len(clf.n_classes_) == 2
|
|
assert len(clf.classes_) == 2
|
|
assert_array_equal(clf.n_classes_, [2, 2])
|
|
assert_array_equal(clf.classes_, [[-1, 1], [-2, 2]])
|
|
|
|
|
|
def test_unbalanced_iris():
|
|
# Check class rebalancing.
|
|
unbalanced_X = iris.data[:125]
|
|
unbalanced_y = iris.target[:125]
|
|
sample_weight = compute_sample_weight("balanced", unbalanced_y)
|
|
|
|
for name, TreeClassifier in CLF_TREES.items():
|
|
clf = TreeClassifier(random_state=0)
|
|
clf.fit(unbalanced_X, unbalanced_y, sample_weight=sample_weight)
|
|
assert_almost_equal(clf.predict(unbalanced_X), unbalanced_y)
|
|
|
|
|
|
def test_memory_layout():
|
|
# Check that it works no matter the memory layout
|
|
for (name, TreeEstimator), dtype in product(ALL_TREES.items(),
|
|
[np.float64, np.float32]):
|
|
est = TreeEstimator(random_state=0)
|
|
|
|
# Nothing
|
|
X = np.asarray(iris.data, dtype=dtype)
|
|
y = iris.target
|
|
assert_array_equal(est.fit(X, y).predict(X), y)
|
|
|
|
# C-order
|
|
X = np.asarray(iris.data, order="C", dtype=dtype)
|
|
y = iris.target
|
|
assert_array_equal(est.fit(X, y).predict(X), y)
|
|
|
|
# F-order
|
|
X = np.asarray(iris.data, order="F", dtype=dtype)
|
|
y = iris.target
|
|
assert_array_equal(est.fit(X, y).predict(X), y)
|
|
|
|
# Contiguous
|
|
X = np.ascontiguousarray(iris.data, dtype=dtype)
|
|
y = iris.target
|
|
assert_array_equal(est.fit(X, y).predict(X), y)
|
|
|
|
# csr matrix
|
|
X = csr_matrix(iris.data, dtype=dtype)
|
|
y = iris.target
|
|
assert_array_equal(est.fit(X, y).predict(X), y)
|
|
|
|
# csc_matrix
|
|
X = csc_matrix(iris.data, dtype=dtype)
|
|
y = iris.target
|
|
assert_array_equal(est.fit(X, y).predict(X), y)
|
|
|
|
# Strided
|
|
X = np.asarray(iris.data[::3], dtype=dtype)
|
|
y = iris.target[::3]
|
|
assert_array_equal(est.fit(X, y).predict(X), y)
|
|
|
|
|
|
def test_sample_weight():
|
|
# Check sample weighting.
|
|
# Test that zero-weighted samples are not taken into account
|
|
X = np.arange(100)[:, np.newaxis]
|
|
y = np.ones(100)
|
|
y[:50] = 0.0
|
|
|
|
sample_weight = np.ones(100)
|
|
sample_weight[y == 0] = 0.0
|
|
|
|
clf = DecisionTreeClassifier(random_state=0)
|
|
clf.fit(X, y, sample_weight=sample_weight)
|
|
assert_array_equal(clf.predict(X), np.ones(100))
|
|
|
|
# Test that low weighted samples are not taken into account at low depth
|
|
X = np.arange(200)[:, np.newaxis]
|
|
y = np.zeros(200)
|
|
y[50:100] = 1
|
|
y[100:200] = 2
|
|
X[100:200, 0] = 200
|
|
|
|
sample_weight = np.ones(200)
|
|
|
|
sample_weight[y == 2] = .51 # Samples of class '2' are still weightier
|
|
clf = DecisionTreeClassifier(max_depth=1, random_state=0)
|
|
clf.fit(X, y, sample_weight=sample_weight)
|
|
assert clf.tree_.threshold[0] == 149.5
|
|
|
|
sample_weight[y == 2] = .5 # Samples of class '2' are no longer weightier
|
|
clf = DecisionTreeClassifier(max_depth=1, random_state=0)
|
|
clf.fit(X, y, sample_weight=sample_weight)
|
|
assert clf.tree_.threshold[0] == 49.5 # Threshold should have moved
|
|
|
|
# Test that sample weighting is the same as having duplicates
|
|
X = iris.data
|
|
y = iris.target
|
|
|
|
duplicates = rng.randint(0, X.shape[0], 100)
|
|
|
|
clf = DecisionTreeClassifier(random_state=1)
|
|
clf.fit(X[duplicates], y[duplicates])
|
|
|
|
sample_weight = np.bincount(duplicates, minlength=X.shape[0])
|
|
clf2 = DecisionTreeClassifier(random_state=1)
|
|
clf2.fit(X, y, sample_weight=sample_weight)
|
|
|
|
internal = clf.tree_.children_left != tree._tree.TREE_LEAF
|
|
assert_array_almost_equal(clf.tree_.threshold[internal],
|
|
clf2.tree_.threshold[internal])
|
|
|
|
|
|
def test_sample_weight_invalid():
|
|
# Check sample weighting raises errors.
|
|
X = np.arange(100)[:, np.newaxis]
|
|
y = np.ones(100)
|
|
y[:50] = 0.0
|
|
|
|
clf = DecisionTreeClassifier(random_state=0)
|
|
|
|
sample_weight = np.random.rand(100, 1)
|
|
with pytest.raises(ValueError):
|
|
clf.fit(X, y, sample_weight=sample_weight)
|
|
|
|
sample_weight = np.array(0)
|
|
expected_err = r"Singleton.* cannot be considered a valid collection"
|
|
with pytest.raises(TypeError, match=expected_err):
|
|
clf.fit(X, y, sample_weight=sample_weight)
|
|
|
|
|
|
def check_class_weights(name):
|
|
"""Check class_weights resemble sample_weights behavior."""
|
|
TreeClassifier = CLF_TREES[name]
|
|
|
|
# Iris is balanced, so no effect expected for using 'balanced' weights
|
|
clf1 = TreeClassifier(random_state=0)
|
|
clf1.fit(iris.data, iris.target)
|
|
clf2 = TreeClassifier(class_weight='balanced', random_state=0)
|
|
clf2.fit(iris.data, iris.target)
|
|
assert_almost_equal(clf1.feature_importances_, clf2.feature_importances_)
|
|
|
|
# Make a multi-output problem with three copies of Iris
|
|
iris_multi = np.vstack((iris.target, iris.target, iris.target)).T
|
|
# Create user-defined weights that should balance over the outputs
|
|
clf3 = TreeClassifier(class_weight=[{0: 2., 1: 2., 2: 1.},
|
|
{0: 2., 1: 1., 2: 2.},
|
|
{0: 1., 1: 2., 2: 2.}],
|
|
random_state=0)
|
|
clf3.fit(iris.data, iris_multi)
|
|
assert_almost_equal(clf2.feature_importances_, clf3.feature_importances_)
|
|
# Check against multi-output "auto" which should also have no effect
|
|
clf4 = TreeClassifier(class_weight='balanced', random_state=0)
|
|
clf4.fit(iris.data, iris_multi)
|
|
assert_almost_equal(clf3.feature_importances_, clf4.feature_importances_)
|
|
|
|
# Inflate importance of class 1, check against user-defined weights
|
|
sample_weight = np.ones(iris.target.shape)
|
|
sample_weight[iris.target == 1] *= 100
|
|
class_weight = {0: 1., 1: 100., 2: 1.}
|
|
clf1 = TreeClassifier(random_state=0)
|
|
clf1.fit(iris.data, iris.target, sample_weight)
|
|
clf2 = TreeClassifier(class_weight=class_weight, random_state=0)
|
|
clf2.fit(iris.data, iris.target)
|
|
assert_almost_equal(clf1.feature_importances_, clf2.feature_importances_)
|
|
|
|
# Check that sample_weight and class_weight are multiplicative
|
|
clf1 = TreeClassifier(random_state=0)
|
|
clf1.fit(iris.data, iris.target, sample_weight ** 2)
|
|
clf2 = TreeClassifier(class_weight=class_weight, random_state=0)
|
|
clf2.fit(iris.data, iris.target, sample_weight)
|
|
assert_almost_equal(clf1.feature_importances_, clf2.feature_importances_)
|
|
|
|
|
|
@pytest.mark.parametrize("name", CLF_TREES)
|
|
def test_class_weights(name):
|
|
check_class_weights(name)
|
|
|
|
|
|
def check_class_weight_errors(name):
|
|
# Test if class_weight raises errors and warnings when expected.
|
|
TreeClassifier = CLF_TREES[name]
|
|
_y = np.vstack((y, np.array(y) * 2)).T
|
|
|
|
# Invalid preset string
|
|
clf = TreeClassifier(class_weight='the larch', random_state=0)
|
|
with pytest.raises(ValueError):
|
|
clf.fit(X, y)
|
|
with pytest.raises(ValueError):
|
|
clf.fit(X, _y)
|
|
|
|
# Not a list or preset for multi-output
|
|
clf = TreeClassifier(class_weight=1, random_state=0)
|
|
with pytest.raises(ValueError):
|
|
clf.fit(X, _y)
|
|
|
|
# Incorrect length list for multi-output
|
|
clf = TreeClassifier(class_weight=[{-1: 0.5, 1: 1.}], random_state=0)
|
|
with pytest.raises(ValueError):
|
|
clf.fit(X, _y)
|
|
|
|
|
|
@pytest.mark.parametrize("name", CLF_TREES)
|
|
def test_class_weight_errors(name):
|
|
check_class_weight_errors(name)
|
|
|
|
|
|
def test_max_leaf_nodes():
|
|
# Test greedy trees with max_depth + 1 leafs.
|
|
X, y = datasets.make_hastie_10_2(n_samples=100, random_state=1)
|
|
k = 4
|
|
for name, TreeEstimator in ALL_TREES.items():
|
|
est = TreeEstimator(max_depth=None, max_leaf_nodes=k + 1).fit(X, y)
|
|
assert est.get_n_leaves() == k + 1
|
|
|
|
# max_leaf_nodes in (0, 1) should raise ValueError
|
|
est = TreeEstimator(max_depth=None, max_leaf_nodes=0)
|
|
with pytest.raises(ValueError):
|
|
est.fit(X, y)
|
|
est = TreeEstimator(max_depth=None, max_leaf_nodes=1)
|
|
with pytest.raises(ValueError):
|
|
est.fit(X, y)
|
|
est = TreeEstimator(max_depth=None, max_leaf_nodes=0.1)
|
|
with pytest.raises(ValueError):
|
|
est.fit(X, y)
|
|
|
|
|
|
def test_max_leaf_nodes_max_depth():
|
|
# Test precedence of max_leaf_nodes over max_depth.
|
|
X, y = datasets.make_hastie_10_2(n_samples=100, random_state=1)
|
|
k = 4
|
|
for name, TreeEstimator in ALL_TREES.items():
|
|
est = TreeEstimator(max_depth=1, max_leaf_nodes=k).fit(X, y)
|
|
assert est.get_depth() == 1
|
|
|
|
|
|
def test_arrays_persist():
|
|
# Ensure property arrays' memory stays alive when tree disappears
|
|
# non-regression for #2726
|
|
for attr in ['n_classes', 'value', 'children_left', 'children_right',
|
|
'threshold', 'impurity', 'feature', 'n_node_samples']:
|
|
value = getattr(DecisionTreeClassifier().fit([[0], [1]],
|
|
[0, 1]).tree_, attr)
|
|
# if pointing to freed memory, contents may be arbitrary
|
|
assert -3 <= value.flat[0] < 3, \
|
|
'Array points to arbitrary memory'
|
|
|
|
|
|
def test_only_constant_features():
|
|
random_state = check_random_state(0)
|
|
X = np.zeros((10, 20))
|
|
y = random_state.randint(0, 2, (10, ))
|
|
for name, TreeEstimator in ALL_TREES.items():
|
|
est = TreeEstimator(random_state=0)
|
|
est.fit(X, y)
|
|
assert est.tree_.max_depth == 0
|
|
|
|
|
|
def test_behaviour_constant_feature_after_splits():
|
|
X = np.transpose(np.vstack(([[0, 0, 0, 0, 0, 1, 2, 4, 5, 6, 7]],
|
|
np.zeros((4, 11)))))
|
|
y = [0, 0, 0, 1, 1, 2, 2, 2, 3, 3, 3]
|
|
for name, TreeEstimator in ALL_TREES.items():
|
|
# do not check extra random trees
|
|
if "ExtraTree" not in name:
|
|
est = TreeEstimator(random_state=0, max_features=1)
|
|
est.fit(X, y)
|
|
assert est.tree_.max_depth == 2
|
|
assert est.tree_.node_count == 5
|
|
|
|
|
|
def test_with_only_one_non_constant_features():
|
|
X = np.hstack([np.array([[1.], [1.], [0.], [0.]]),
|
|
np.zeros((4, 1000))])
|
|
|
|
y = np.array([0., 1., 0., 1.0])
|
|
for name, TreeEstimator in CLF_TREES.items():
|
|
est = TreeEstimator(random_state=0, max_features=1)
|
|
est.fit(X, y)
|
|
assert est.tree_.max_depth == 1
|
|
assert_array_equal(est.predict_proba(X), np.full((4, 2), 0.5))
|
|
|
|
for name, TreeEstimator in REG_TREES.items():
|
|
est = TreeEstimator(random_state=0, max_features=1)
|
|
est.fit(X, y)
|
|
assert est.tree_.max_depth == 1
|
|
assert_array_equal(est.predict(X), np.full((4, ), 0.5))
|
|
|
|
|
|
def test_big_input():
|
|
# Test if the warning for too large inputs is appropriate.
|
|
X = np.repeat(10 ** 40., 4).astype(np.float64).reshape(-1, 1)
|
|
clf = DecisionTreeClassifier()
|
|
try:
|
|
clf.fit(X, [0, 1, 0, 1])
|
|
except ValueError as e:
|
|
assert "float32" in str(e)
|
|
|
|
|
|
def test_realloc():
|
|
from sklearn.tree._utils import _realloc_test
|
|
with pytest.raises(MemoryError):
|
|
_realloc_test()
|
|
|
|
|
|
def test_huge_allocations():
|
|
n_bits = 8 * struct.calcsize("P")
|
|
|
|
X = np.random.randn(10, 2)
|
|
y = np.random.randint(0, 2, 10)
|
|
|
|
# Sanity check: we cannot request more memory than the size of the address
|
|
# space. Currently raises OverflowError.
|
|
huge = 2 ** (n_bits + 1)
|
|
clf = DecisionTreeClassifier(splitter='best', max_leaf_nodes=huge)
|
|
with pytest.raises(Exception):
|
|
clf.fit(X, y)
|
|
|
|
# Non-regression test: MemoryError used to be dropped by Cython
|
|
# because of missing "except *".
|
|
huge = 2 ** (n_bits - 1) - 1
|
|
clf = DecisionTreeClassifier(splitter='best', max_leaf_nodes=huge)
|
|
with pytest.raises(MemoryError):
|
|
clf.fit(X, y)
|
|
|
|
|
|
def check_sparse_input(tree, dataset, max_depth=None):
|
|
TreeEstimator = ALL_TREES[tree]
|
|
X = DATASETS[dataset]["X"]
|
|
X_sparse = DATASETS[dataset]["X_sparse"]
|
|
y = DATASETS[dataset]["y"]
|
|
|
|
# Gain testing time
|
|
if dataset in ["digits", "diabetes"]:
|
|
n_samples = X.shape[0] // 5
|
|
X = X[:n_samples]
|
|
X_sparse = X_sparse[:n_samples]
|
|
y = y[:n_samples]
|
|
|
|
for sparse_format in (csr_matrix, csc_matrix, coo_matrix):
|
|
X_sparse = sparse_format(X_sparse)
|
|
|
|
# Check the default (depth first search)
|
|
d = TreeEstimator(random_state=0, max_depth=max_depth).fit(X, y)
|
|
s = TreeEstimator(random_state=0, max_depth=max_depth).fit(X_sparse, y)
|
|
|
|
assert_tree_equal(d.tree_, s.tree_,
|
|
"{0} with dense and sparse format gave different "
|
|
"trees".format(tree))
|
|
|
|
y_pred = d.predict(X)
|
|
if tree in CLF_TREES:
|
|
y_proba = d.predict_proba(X)
|
|
y_log_proba = d.predict_log_proba(X)
|
|
|
|
for sparse_matrix in (csr_matrix, csc_matrix, coo_matrix):
|
|
X_sparse_test = sparse_matrix(X_sparse, dtype=np.float32)
|
|
|
|
assert_array_almost_equal(s.predict(X_sparse_test), y_pred)
|
|
|
|
if tree in CLF_TREES:
|
|
assert_array_almost_equal(s.predict_proba(X_sparse_test),
|
|
y_proba)
|
|
assert_array_almost_equal(s.predict_log_proba(X_sparse_test),
|
|
y_log_proba)
|
|
|
|
|
|
@pytest.mark.parametrize("tree_type", SPARSE_TREES)
|
|
@pytest.mark.parametrize(
|
|
"dataset",
|
|
("clf_small", "toy", "digits", "multilabel",
|
|
"sparse-pos", "sparse-neg", "sparse-mix",
|
|
"zeros")
|
|
)
|
|
def test_sparse_input(tree_type, dataset):
|
|
max_depth = 3 if dataset == "digits" else None
|
|
check_sparse_input(tree_type, dataset, max_depth)
|
|
|
|
|
|
@pytest.mark.parametrize("tree_type",
|
|
sorted(set(SPARSE_TREES).intersection(REG_TREES)))
|
|
@pytest.mark.parametrize("dataset", ["diabetes", "reg_small"])
|
|
def test_sparse_input_reg_trees(tree_type, dataset):
|
|
# Due to numerical instability of MSE and too strict test, we limit the
|
|
# maximal depth
|
|
check_sparse_input(tree_type, dataset, 2)
|
|
|
|
|
|
def check_sparse_parameters(tree, dataset):
|
|
TreeEstimator = ALL_TREES[tree]
|
|
X = DATASETS[dataset]["X"]
|
|
X_sparse = DATASETS[dataset]["X_sparse"]
|
|
y = DATASETS[dataset]["y"]
|
|
|
|
# Check max_features
|
|
d = TreeEstimator(random_state=0, max_features=1, max_depth=2).fit(X, y)
|
|
s = TreeEstimator(random_state=0, max_features=1,
|
|
max_depth=2).fit(X_sparse, y)
|
|
assert_tree_equal(d.tree_, s.tree_,
|
|
"{0} with dense and sparse format gave different "
|
|
"trees".format(tree))
|
|
assert_array_almost_equal(s.predict(X), d.predict(X))
|
|
|
|
# Check min_samples_split
|
|
d = TreeEstimator(random_state=0, max_features=1,
|
|
min_samples_split=10).fit(X, y)
|
|
s = TreeEstimator(random_state=0, max_features=1,
|
|
min_samples_split=10).fit(X_sparse, y)
|
|
assert_tree_equal(d.tree_, s.tree_,
|
|
"{0} with dense and sparse format gave different "
|
|
"trees".format(tree))
|
|
assert_array_almost_equal(s.predict(X), d.predict(X))
|
|
|
|
# Check min_samples_leaf
|
|
d = TreeEstimator(random_state=0,
|
|
min_samples_leaf=X_sparse.shape[0] // 2).fit(X, y)
|
|
s = TreeEstimator(random_state=0,
|
|
min_samples_leaf=X_sparse.shape[0] // 2).fit(X_sparse, y)
|
|
assert_tree_equal(d.tree_, s.tree_,
|
|
"{0} with dense and sparse format gave different "
|
|
"trees".format(tree))
|
|
assert_array_almost_equal(s.predict(X), d.predict(X))
|
|
|
|
# Check best-first search
|
|
d = TreeEstimator(random_state=0, max_leaf_nodes=3).fit(X, y)
|
|
s = TreeEstimator(random_state=0, max_leaf_nodes=3).fit(X_sparse, y)
|
|
assert_tree_equal(d.tree_, s.tree_,
|
|
"{0} with dense and sparse format gave different "
|
|
"trees".format(tree))
|
|
assert_array_almost_equal(s.predict(X), d.predict(X))
|
|
|
|
|
|
def check_sparse_criterion(tree, dataset):
|
|
TreeEstimator = ALL_TREES[tree]
|
|
X = DATASETS[dataset]["X"]
|
|
X_sparse = DATASETS[dataset]["X_sparse"]
|
|
y = DATASETS[dataset]["y"]
|
|
|
|
# Check various criterion
|
|
CRITERIONS = REG_CRITERIONS if tree in REG_TREES else CLF_CRITERIONS
|
|
for criterion in CRITERIONS:
|
|
d = TreeEstimator(random_state=0, max_depth=3,
|
|
criterion=criterion).fit(X, y)
|
|
s = TreeEstimator(random_state=0, max_depth=3,
|
|
criterion=criterion).fit(X_sparse, y)
|
|
|
|
assert_tree_equal(d.tree_, s.tree_,
|
|
"{0} with dense and sparse format gave different "
|
|
"trees".format(tree))
|
|
assert_array_almost_equal(s.predict(X), d.predict(X))
|
|
|
|
|
|
@pytest.mark.parametrize("tree_type", SPARSE_TREES)
|
|
@pytest.mark.parametrize("dataset",
|
|
["sparse-pos", "sparse-neg", "sparse-mix", "zeros"])
|
|
@pytest.mark.parametrize("check",
|
|
[check_sparse_parameters, check_sparse_criterion])
|
|
def test_sparse(tree_type, dataset, check):
|
|
check(tree_type, dataset)
|
|
|
|
|
|
def check_explicit_sparse_zeros(tree, max_depth=3,
|
|
n_features=10):
|
|
TreeEstimator = ALL_TREES[tree]
|
|
|
|
# n_samples set n_feature to ease construction of a simultaneous
|
|
# construction of a csr and csc matrix
|
|
n_samples = n_features
|
|
samples = np.arange(n_samples)
|
|
|
|
# Generate X, y
|
|
random_state = check_random_state(0)
|
|
indices = []
|
|
data = []
|
|
offset = 0
|
|
indptr = [offset]
|
|
for i in range(n_features):
|
|
n_nonzero_i = random_state.binomial(n_samples, 0.5)
|
|
indices_i = random_state.permutation(samples)[:n_nonzero_i]
|
|
indices.append(indices_i)
|
|
data_i = random_state.binomial(3, 0.5, size=(n_nonzero_i, )) - 1
|
|
data.append(data_i)
|
|
offset += n_nonzero_i
|
|
indptr.append(offset)
|
|
|
|
indices = np.concatenate(indices)
|
|
data = np.array(np.concatenate(data), dtype=np.float32)
|
|
X_sparse = csc_matrix((data, indices, indptr),
|
|
shape=(n_samples, n_features))
|
|
X = X_sparse.toarray()
|
|
X_sparse_test = csr_matrix((data, indices, indptr),
|
|
shape=(n_samples, n_features))
|
|
X_test = X_sparse_test.toarray()
|
|
y = random_state.randint(0, 3, size=(n_samples, ))
|
|
|
|
# Ensure that X_sparse_test owns its data, indices and indptr array
|
|
X_sparse_test = X_sparse_test.copy()
|
|
|
|
# Ensure that we have explicit zeros
|
|
assert (X_sparse.data == 0.).sum() > 0
|
|
assert (X_sparse_test.data == 0.).sum() > 0
|
|
|
|
# Perform the comparison
|
|
d = TreeEstimator(random_state=0, max_depth=max_depth).fit(X, y)
|
|
s = TreeEstimator(random_state=0, max_depth=max_depth).fit(X_sparse, y)
|
|
|
|
assert_tree_equal(d.tree_, s.tree_,
|
|
"{0} with dense and sparse format gave different "
|
|
"trees".format(tree))
|
|
|
|
Xs = (X_test, X_sparse_test)
|
|
for X1, X2 in product(Xs, Xs):
|
|
assert_array_almost_equal(s.tree_.apply(X1), d.tree_.apply(X2))
|
|
assert_array_almost_equal(s.apply(X1), d.apply(X2))
|
|
assert_array_almost_equal(s.apply(X1), s.tree_.apply(X1))
|
|
|
|
assert_array_almost_equal(s.tree_.decision_path(X1).toarray(),
|
|
d.tree_.decision_path(X2).toarray())
|
|
assert_array_almost_equal(s.decision_path(X1).toarray(),
|
|
d.decision_path(X2).toarray())
|
|
assert_array_almost_equal(s.decision_path(X1).toarray(),
|
|
s.tree_.decision_path(X1).toarray())
|
|
|
|
assert_array_almost_equal(s.predict(X1), d.predict(X2))
|
|
|
|
if tree in CLF_TREES:
|
|
assert_array_almost_equal(s.predict_proba(X1),
|
|
d.predict_proba(X2))
|
|
|
|
|
|
@pytest.mark.parametrize("tree_type", SPARSE_TREES)
|
|
def test_explicit_sparse_zeros(tree_type):
|
|
check_explicit_sparse_zeros(tree_type)
|
|
|
|
|
|
@ignore_warnings
|
|
def check_raise_error_on_1d_input(name):
|
|
TreeEstimator = ALL_TREES[name]
|
|
|
|
X = iris.data[:, 0].ravel()
|
|
X_2d = iris.data[:, 0].reshape((-1, 1))
|
|
y = iris.target
|
|
|
|
with pytest.raises(ValueError):
|
|
TreeEstimator(random_state=0).fit(X, y)
|
|
|
|
est = TreeEstimator(random_state=0)
|
|
est.fit(X_2d, y)
|
|
with pytest.raises(ValueError):
|
|
est.predict([X])
|
|
|
|
|
|
@pytest.mark.parametrize("name", ALL_TREES)
|
|
def test_1d_input(name):
|
|
with ignore_warnings():
|
|
check_raise_error_on_1d_input(name)
|
|
|
|
|
|
def _check_min_weight_leaf_split_level(TreeEstimator, X, y, sample_weight):
|
|
est = TreeEstimator(random_state=0)
|
|
est.fit(X, y, sample_weight=sample_weight)
|
|
assert est.tree_.max_depth == 1
|
|
|
|
est = TreeEstimator(random_state=0, min_weight_fraction_leaf=0.4)
|
|
est.fit(X, y, sample_weight=sample_weight)
|
|
assert est.tree_.max_depth == 0
|
|
|
|
|
|
def check_min_weight_leaf_split_level(name):
|
|
TreeEstimator = ALL_TREES[name]
|
|
|
|
X = np.array([[0], [0], [0], [0], [1]])
|
|
y = [0, 0, 0, 0, 1]
|
|
sample_weight = [0.2, 0.2, 0.2, 0.2, 0.2]
|
|
_check_min_weight_leaf_split_level(TreeEstimator, X, y, sample_weight)
|
|
|
|
_check_min_weight_leaf_split_level(TreeEstimator, csc_matrix(X), y,
|
|
sample_weight)
|
|
|
|
|
|
@pytest.mark.parametrize("name", ALL_TREES)
|
|
def test_min_weight_leaf_split_level(name):
|
|
check_min_weight_leaf_split_level(name)
|
|
|
|
|
|
def check_public_apply(name):
|
|
X_small32 = X_small.astype(tree._tree.DTYPE, copy=False)
|
|
|
|
est = ALL_TREES[name]()
|
|
est.fit(X_small, y_small)
|
|
assert_array_equal(est.apply(X_small),
|
|
est.tree_.apply(X_small32))
|
|
|
|
|
|
def check_public_apply_sparse(name):
|
|
X_small32 = csr_matrix(X_small.astype(tree._tree.DTYPE, copy=False))
|
|
|
|
est = ALL_TREES[name]()
|
|
est.fit(X_small, y_small)
|
|
assert_array_equal(est.apply(X_small),
|
|
est.tree_.apply(X_small32))
|
|
|
|
|
|
@pytest.mark.parametrize("name", ALL_TREES)
|
|
def test_public_apply_all_trees(name):
|
|
check_public_apply(name)
|
|
|
|
|
|
@pytest.mark.parametrize("name", SPARSE_TREES)
|
|
def test_public_apply_sparse_trees(name):
|
|
check_public_apply_sparse(name)
|
|
|
|
|
|
def test_decision_path_hardcoded():
|
|
X = iris.data
|
|
y = iris.target
|
|
est = DecisionTreeClassifier(random_state=0, max_depth=1).fit(X, y)
|
|
node_indicator = est.decision_path(X[:2]).toarray()
|
|
assert_array_equal(node_indicator, [[1, 1, 0], [1, 0, 1]])
|
|
|
|
|
|
def check_decision_path(name):
|
|
X = iris.data
|
|
y = iris.target
|
|
n_samples = X.shape[0]
|
|
|
|
TreeEstimator = ALL_TREES[name]
|
|
est = TreeEstimator(random_state=0, max_depth=2)
|
|
est.fit(X, y)
|
|
|
|
node_indicator_csr = est.decision_path(X)
|
|
node_indicator = node_indicator_csr.toarray()
|
|
assert node_indicator.shape == (n_samples, est.tree_.node_count)
|
|
|
|
# Assert that leaves index are correct
|
|
leaves = est.apply(X)
|
|
leave_indicator = [node_indicator[i, j] for i, j in enumerate(leaves)]
|
|
assert_array_almost_equal(leave_indicator, np.ones(shape=n_samples))
|
|
|
|
# Ensure only one leave node per sample
|
|
all_leaves = est.tree_.children_left == TREE_LEAF
|
|
assert_array_almost_equal(np.dot(node_indicator, all_leaves),
|
|
np.ones(shape=n_samples))
|
|
|
|
# Ensure max depth is consistent with sum of indicator
|
|
max_depth = node_indicator.sum(axis=1).max()
|
|
assert est.tree_.max_depth <= max_depth
|
|
|
|
|
|
@pytest.mark.parametrize("name", ALL_TREES)
|
|
def test_decision_path(name):
|
|
check_decision_path(name)
|
|
|
|
|
|
def check_no_sparse_y_support(name):
|
|
X, y = X_multilabel, csr_matrix(y_multilabel)
|
|
TreeEstimator = ALL_TREES[name]
|
|
with pytest.raises(TypeError):
|
|
TreeEstimator(random_state=0).fit(X, y)
|
|
|
|
|
|
@pytest.mark.parametrize("name", ALL_TREES)
|
|
def test_no_sparse_y_support(name):
|
|
# Currently we don't support sparse y
|
|
check_no_sparse_y_support(name)
|
|
|
|
|
|
def test_mae():
|
|
"""Check MAE criterion produces correct results on small toy dataset:
|
|
|
|
------------------
|
|
| X | y | weight |
|
|
------------------
|
|
| 3 | 3 | 0.1 |
|
|
| 5 | 3 | 0.3 |
|
|
| 8 | 4 | 1.0 |
|
|
| 3 | 6 | 0.6 |
|
|
| 5 | 7 | 0.3 |
|
|
------------------
|
|
|sum wt:| 2.3 |
|
|
------------------
|
|
|
|
Because we are dealing with sample weights, we cannot find the median by
|
|
simply choosing/averaging the centre value(s), instead we consider the
|
|
median where 50% of the cumulative weight is found (in a y sorted data set)
|
|
. Therefore with regards to this test data, the cumulative weight is >= 50%
|
|
when y = 4. Therefore:
|
|
Median = 4
|
|
|
|
For all the samples, we can get the total error by summing:
|
|
Absolute(Median - y) * weight
|
|
|
|
I.e., total error = (Absolute(4 - 3) * 0.1)
|
|
+ (Absolute(4 - 3) * 0.3)
|
|
+ (Absolute(4 - 4) * 1.0)
|
|
+ (Absolute(4 - 6) * 0.6)
|
|
+ (Absolute(4 - 7) * 0.3)
|
|
= 2.5
|
|
|
|
Impurity = Total error / total weight
|
|
= 2.5 / 2.3
|
|
= 1.08695652173913
|
|
------------------
|
|
|
|
From this root node, the next best split is between X values of 3 and 5.
|
|
Thus, we have left and right child nodes:
|
|
|
|
LEFT RIGHT
|
|
------------------ ------------------
|
|
| X | y | weight | | X | y | weight |
|
|
------------------ ------------------
|
|
| 3 | 3 | 0.1 | | 5 | 3 | 0.3 |
|
|
| 3 | 6 | 0.6 | | 8 | 4 | 1.0 |
|
|
------------------ | 5 | 7 | 0.3 |
|
|
|sum wt:| 0.7 | ------------------
|
|
------------------ |sum wt:| 1.6 |
|
|
------------------
|
|
|
|
Impurity is found in the same way:
|
|
Left node Median = 6
|
|
Total error = (Absolute(6 - 3) * 0.1)
|
|
+ (Absolute(6 - 6) * 0.6)
|
|
= 0.3
|
|
|
|
Left Impurity = Total error / total weight
|
|
= 0.3 / 0.7
|
|
= 0.428571428571429
|
|
-------------------
|
|
|
|
Likewise for Right node:
|
|
Right node Median = 4
|
|
Total error = (Absolute(4 - 3) * 0.3)
|
|
+ (Absolute(4 - 4) * 1.0)
|
|
+ (Absolute(4 - 7) * 0.3)
|
|
= 1.2
|
|
|
|
Right Impurity = Total error / total weight
|
|
= 1.2 / 1.6
|
|
= 0.75
|
|
------
|
|
"""
|
|
dt_mae = DecisionTreeRegressor(random_state=0, criterion="mae",
|
|
max_leaf_nodes=2)
|
|
|
|
# Test MAE where sample weights are non-uniform (as illustrated above):
|
|
dt_mae.fit(X=[[3], [5], [3], [8], [5]], y=[6, 7, 3, 4, 3],
|
|
sample_weight=[0.6, 0.3, 0.1, 1.0, 0.3])
|
|
assert_allclose(dt_mae.tree_.impurity, [2.5 / 2.3, 0.3 / 0.7, 1.2 / 1.6])
|
|
assert_array_equal(dt_mae.tree_.value.flat, [4.0, 6.0, 4.0])
|
|
|
|
# Test MAE where all sample weights are uniform:
|
|
dt_mae.fit(X=[[3], [5], [3], [8], [5]], y=[6, 7, 3, 4, 3],
|
|
sample_weight=np.ones(5))
|
|
assert_array_equal(dt_mae.tree_.impurity, [1.4, 1.5, 4.0 / 3.0])
|
|
assert_array_equal(dt_mae.tree_.value.flat, [4, 4.5, 4.0])
|
|
|
|
# Test MAE where a `sample_weight` is not explicitly provided.
|
|
# This is equivalent to providing uniform sample weights, though
|
|
# the internal logic is different:
|
|
dt_mae.fit(X=[[3], [5], [3], [8], [5]], y=[6, 7, 3, 4, 3])
|
|
assert_array_equal(dt_mae.tree_.impurity, [1.4, 1.5, 4.0 / 3.0])
|
|
assert_array_equal(dt_mae.tree_.value.flat, [4, 4.5, 4.0])
|
|
|
|
|
|
def test_criterion_copy():
|
|
# Let's check whether copy of our criterion has the same type
|
|
# and properties as original
|
|
n_outputs = 3
|
|
n_classes = np.arange(3, dtype=np.intp)
|
|
n_samples = 100
|
|
|
|
def _pickle_copy(obj):
|
|
return pickle.loads(pickle.dumps(obj))
|
|
for copy_func in [copy.copy, copy.deepcopy, _pickle_copy]:
|
|
for _, typename in CRITERIA_CLF.items():
|
|
criteria = typename(n_outputs, n_classes)
|
|
result = copy_func(criteria).__reduce__()
|
|
typename_, (n_outputs_, n_classes_), _ = result
|
|
assert typename == typename_
|
|
assert n_outputs == n_outputs_
|
|
assert_array_equal(n_classes, n_classes_)
|
|
|
|
for _, typename in CRITERIA_REG.items():
|
|
criteria = typename(n_outputs, n_samples)
|
|
result = copy_func(criteria).__reduce__()
|
|
typename_, (n_outputs_, n_samples_), _ = result
|
|
assert typename == typename_
|
|
assert n_outputs == n_outputs_
|
|
assert n_samples == n_samples_
|
|
|
|
|
|
def test_empty_leaf_infinite_threshold():
|
|
# try to make empty leaf by using near infinite value.
|
|
data = np.random.RandomState(0).randn(100, 11) * 2e38
|
|
data = np.nan_to_num(data.astype('float32'))
|
|
X_full = data[:, :-1]
|
|
X_sparse = csc_matrix(X_full)
|
|
y = data[:, -1]
|
|
for X in [X_full, X_sparse]:
|
|
tree = DecisionTreeRegressor(random_state=0).fit(X, y)
|
|
terminal_regions = tree.apply(X)
|
|
left_leaf = set(np.where(tree.tree_.children_left == TREE_LEAF)[0])
|
|
empty_leaf = left_leaf.difference(terminal_regions)
|
|
infinite_threshold = np.where(~np.isfinite(tree.tree_.threshold))[0]
|
|
assert len(infinite_threshold) == 0
|
|
assert len(empty_leaf) == 0
|
|
|
|
|
|
@pytest.mark.parametrize("criterion", CLF_CRITERIONS)
|
|
@pytest.mark.parametrize(
|
|
"dataset", sorted(set(DATASETS.keys()) - {"reg_small", "diabetes"}))
|
|
@pytest.mark.parametrize(
|
|
"tree_cls", [DecisionTreeClassifier, ExtraTreeClassifier])
|
|
def test_prune_tree_classifier_are_subtrees(criterion, dataset, tree_cls):
|
|
dataset = DATASETS[dataset]
|
|
X, y = dataset["X"], dataset["y"]
|
|
est = tree_cls(max_leaf_nodes=20, random_state=0)
|
|
info = est.cost_complexity_pruning_path(X, y)
|
|
|
|
pruning_path = info.ccp_alphas
|
|
impurities = info.impurities
|
|
assert np.all(np.diff(pruning_path) >= 0)
|
|
assert np.all(np.diff(impurities) >= 0)
|
|
|
|
assert_pruning_creates_subtree(tree_cls, X, y, pruning_path)
|
|
|
|
|
|
@pytest.mark.parametrize("criterion", REG_CRITERIONS)
|
|
@pytest.mark.parametrize("dataset", DATASETS.keys())
|
|
@pytest.mark.parametrize(
|
|
"tree_cls", [DecisionTreeRegressor, ExtraTreeRegressor])
|
|
def test_prune_tree_regression_are_subtrees(criterion, dataset, tree_cls):
|
|
dataset = DATASETS[dataset]
|
|
X, y = dataset["X"], dataset["y"]
|
|
|
|
est = tree_cls(max_leaf_nodes=20, random_state=0)
|
|
info = est.cost_complexity_pruning_path(X, y)
|
|
|
|
pruning_path = info.ccp_alphas
|
|
impurities = info.impurities
|
|
assert np.all(np.diff(pruning_path) >= 0)
|
|
assert np.all(np.diff(impurities) >= 0)
|
|
|
|
assert_pruning_creates_subtree(tree_cls, X, y, pruning_path)
|
|
|
|
|
|
def test_prune_single_node_tree():
|
|
# single node tree
|
|
clf1 = DecisionTreeClassifier(random_state=0)
|
|
clf1.fit([[0], [1]], [0, 0])
|
|
|
|
# pruned single node tree
|
|
clf2 = DecisionTreeClassifier(random_state=0, ccp_alpha=10)
|
|
clf2.fit([[0], [1]], [0, 0])
|
|
|
|
assert_is_subtree(clf1.tree_, clf2.tree_)
|
|
|
|
|
|
def assert_pruning_creates_subtree(estimator_cls, X, y, pruning_path):
|
|
# generate trees with increasing alphas
|
|
estimators = []
|
|
for ccp_alpha in pruning_path:
|
|
est = estimator_cls(
|
|
max_leaf_nodes=20, ccp_alpha=ccp_alpha, random_state=0).fit(X, y)
|
|
estimators.append(est)
|
|
|
|
# A pruned tree must be a subtree of the previous tree (which had a
|
|
# smaller ccp_alpha)
|
|
for prev_est, next_est in zip(estimators, estimators[1:]):
|
|
assert_is_subtree(prev_est.tree_, next_est.tree_)
|
|
|
|
|
|
def assert_is_subtree(tree, subtree):
|
|
assert tree.node_count >= subtree.node_count
|
|
assert tree.max_depth >= subtree.max_depth
|
|
|
|
tree_c_left = tree.children_left
|
|
tree_c_right = tree.children_right
|
|
subtree_c_left = subtree.children_left
|
|
subtree_c_right = subtree.children_right
|
|
|
|
stack = [(0, 0)]
|
|
while stack:
|
|
tree_node_idx, subtree_node_idx = stack.pop()
|
|
assert_array_almost_equal(tree.value[tree_node_idx],
|
|
subtree.value[subtree_node_idx])
|
|
assert_almost_equal(tree.impurity[tree_node_idx],
|
|
subtree.impurity[subtree_node_idx])
|
|
assert_almost_equal(tree.n_node_samples[tree_node_idx],
|
|
subtree.n_node_samples[subtree_node_idx])
|
|
assert_almost_equal(tree.weighted_n_node_samples[tree_node_idx],
|
|
subtree.weighted_n_node_samples[subtree_node_idx])
|
|
|
|
if (subtree_c_left[subtree_node_idx] ==
|
|
subtree_c_right[subtree_node_idx]):
|
|
# is a leaf
|
|
assert_almost_equal(TREE_UNDEFINED,
|
|
subtree.threshold[subtree_node_idx])
|
|
else:
|
|
# not a leaf
|
|
assert_almost_equal(tree.threshold[tree_node_idx],
|
|
subtree.threshold[subtree_node_idx])
|
|
stack.append((tree_c_left[tree_node_idx],
|
|
subtree_c_left[subtree_node_idx]))
|
|
stack.append((tree_c_right[tree_node_idx],
|
|
subtree_c_right[subtree_node_idx]))
|
|
|
|
|
|
def test_prune_tree_raises_negative_ccp_alpha():
|
|
clf = DecisionTreeClassifier()
|
|
msg = "ccp_alpha must be greater than or equal to 0"
|
|
|
|
with pytest.raises(ValueError, match=msg):
|
|
clf.set_params(ccp_alpha=-1.0)
|
|
clf.fit(X, y)
|
|
|
|
clf.set_params(ccp_alpha=0.0)
|
|
clf.fit(X, y)
|
|
|
|
with pytest.raises(ValueError, match=msg):
|
|
clf.set_params(ccp_alpha=-1.0)
|
|
clf._prune_tree()
|
|
|
|
|
|
def check_apply_path_readonly(name):
|
|
X_readonly = create_memmap_backed_data(X_small.astype(tree._tree.DTYPE,
|
|
copy=False))
|
|
y_readonly = create_memmap_backed_data(np.array(y_small,
|
|
dtype=tree._tree.DTYPE))
|
|
est = ALL_TREES[name]()
|
|
est.fit(X_readonly, y_readonly)
|
|
assert_array_equal(est.predict(X_readonly),
|
|
est.predict(X_small))
|
|
assert_array_equal(est.decision_path(X_readonly).todense(),
|
|
est.decision_path(X_small).todense())
|
|
|
|
|
|
@pytest.mark.parametrize("name", ALL_TREES)
|
|
def test_apply_path_readonly_all_trees(name):
|
|
check_apply_path_readonly(name)
|
|
|
|
|
|
@pytest.mark.parametrize("criterion", ["mse", "friedman_mse", "poisson"])
|
|
@pytest.mark.parametrize("Tree", REG_TREES.values())
|
|
def test_balance_property(criterion, Tree):
|
|
# Test that sum(y_pred)=sum(y_true) on training set.
|
|
# This works if the mean is predicted (should even be true for each leaf).
|
|
# MAE predicts the median and is therefore excluded from this test.
|
|
|
|
# Choose a training set with non-negative targets (for poisson)
|
|
X, y = diabetes.data, diabetes.target
|
|
reg = Tree(criterion=criterion)
|
|
reg.fit(X, y)
|
|
assert np.sum(reg.predict(X)) == pytest.approx(np.sum(y))
|
|
|
|
|
|
@pytest.mark.parametrize("seed", range(3))
|
|
def test_poisson_zero_nodes(seed):
|
|
# Test that sum(y)=0 and therefore y_pred=0 is forbidden on nodes.
|
|
X = [[0, 0], [0, 1], [0, 2], [0, 3],
|
|
[1, 0], [1, 2], [1, 2], [1, 3]]
|
|
y = [0, 0, 0, 0, 1, 2, 3, 4]
|
|
# Note that X[:, 0] == 0 is a 100% indicator for y == 0. The tree can
|
|
# easily learn that:
|
|
reg = DecisionTreeRegressor(criterion="mse", random_state=seed)
|
|
reg.fit(X, y)
|
|
assert np.amin(reg.predict(X)) == 0
|
|
# whereas Poisson must predict strictly positive numbers
|
|
reg = DecisionTreeRegressor(criterion="poisson", random_state=seed)
|
|
reg.fit(X, y)
|
|
assert np.all(reg.predict(X) > 0)
|
|
|
|
# Test additional dataset where something could go wrong.
|
|
n_features = 10
|
|
X, y = datasets.make_regression(
|
|
effective_rank=n_features * 2 // 3, tail_strength=0.6,
|
|
n_samples=1_000,
|
|
n_features=n_features,
|
|
n_informative=n_features * 2 // 3,
|
|
random_state=seed,
|
|
)
|
|
# some excess zeros
|
|
y[(-1 < y) & (y < 0)] = 0
|
|
# make sure the target is positive
|
|
y = np.abs(y)
|
|
reg = DecisionTreeRegressor(criterion='poisson', random_state=seed)
|
|
reg.fit(X, y)
|
|
assert np.all(reg.predict(X) > 0)
|
|
|
|
|
|
def test_poisson_vs_mse():
|
|
# For a Poisson distributed target, Poisson loss should give better results
|
|
# than least squares measured in Poisson deviance as metric.
|
|
# We have a similar test, test_poisson(), in
|
|
# sklearn/ensemble/_hist_gradient_boosting/tests/test_gradient_boosting.py
|
|
# Note: Some fine tuning was needed to have metric_poi < metric_dummy on
|
|
# the test set!
|
|
rng = np.random.RandomState(42)
|
|
n_train, n_test, n_features = 500, 500, 10
|
|
X = datasets.make_low_rank_matrix(n_samples=n_train + n_test,
|
|
n_features=n_features, random_state=rng)
|
|
# We create a log-linear Poisson model and downscale coef as it will get
|
|
# exponentiated.
|
|
coef = rng.uniform(low=-2, high=2, size=n_features) / np.max(X, axis=0)
|
|
y = rng.poisson(lam=np.exp(X @ coef))
|
|
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=n_test,
|
|
random_state=rng)
|
|
# We prevent some overfitting by setting min_samples_split=10.
|
|
tree_poi = DecisionTreeRegressor(criterion="poisson",
|
|
min_samples_split=10,
|
|
random_state=rng)
|
|
tree_mse = DecisionTreeRegressor(criterion="mse",
|
|
min_samples_split=10,
|
|
random_state=rng)
|
|
|
|
tree_poi.fit(X_train, y_train)
|
|
tree_mse.fit(X_train, y_train)
|
|
dummy = DummyRegressor(strategy="mean").fit(X_train, y_train)
|
|
|
|
for X, y, val in [(X_train, y_train, "train"), (X_test, y_test, "test")]:
|
|
metric_poi = mean_poisson_deviance(y, tree_poi.predict(X))
|
|
# mse might produce non-positive predictions => clip
|
|
metric_mse = mean_poisson_deviance(y, np.clip(tree_mse.predict(X),
|
|
1e-15, None))
|
|
metric_dummy = mean_poisson_deviance(y, dummy.predict(X))
|
|
# As MSE might correctly predict 0 in train set, its train score can
|
|
# be better than Poisson. This is no longer the case for the test set.
|
|
if val == "test":
|
|
assert metric_poi < metric_mse
|
|
assert metric_poi < metric_dummy
|
|
|
|
|
|
@pytest.mark.parametrize('criterion', REG_CRITERIONS)
|
|
def test_decision_tree_regressor_sample_weight_consistentcy(
|
|
criterion):
|
|
"""Test that the impact of sample_weight is consistent."""
|
|
tree_params = dict(criterion=criterion)
|
|
tree = DecisionTreeRegressor(**tree_params, random_state=42)
|
|
for kind in ['zeros', 'ones']:
|
|
check_sample_weights_invariance("DecisionTreeRegressor_" + criterion,
|
|
tree, kind='zeros')
|
|
|
|
rng = np.random.RandomState(0)
|
|
n_samples, n_features = 10, 5
|
|
|
|
X = rng.rand(n_samples, n_features)
|
|
y = np.mean(X, axis=1) + rng.rand(n_samples)
|
|
# make it positive in order to work also for poisson criterion
|
|
y += np.min(y) + 0.1
|
|
|
|
# check that multiplying sample_weight by 2 is equivalent
|
|
# to repeating corresponding samples twice
|
|
X2 = np.concatenate([X, X[:n_samples//2]], axis=0)
|
|
y2 = np.concatenate([y, y[:n_samples//2]])
|
|
sample_weight_1 = np.ones(len(y))
|
|
sample_weight_1[:n_samples//2] = 2
|
|
|
|
tree1 = DecisionTreeRegressor(**tree_params).fit(
|
|
X, y, sample_weight=sample_weight_1
|
|
)
|
|
|
|
tree2 = DecisionTreeRegressor(**tree_params).fit(
|
|
X2, y2, sample_weight=None
|
|
)
|
|
|
|
assert tree1.tree_.node_count == tree2.tree_.node_count
|
|
# Thresholds, tree.tree_.threshold, and values, tree.tree_.value, are not
|
|
# exactly the same, but on the training set, those differences do not
|
|
# matter and thus predictions are the same.
|
|
assert_allclose(tree1.predict(X), tree2.predict(X))
|
|
|
|
|
|
# TODO: Remove in v1.1
|
|
@pytest.mark.parametrize("TreeEstimator", [DecisionTreeClassifier,
|
|
DecisionTreeRegressor])
|
|
def test_X_idx_sorted_deprecated(TreeEstimator):
|
|
X_idx_sorted = np.argsort(X, axis=0)
|
|
|
|
tree = TreeEstimator()
|
|
|
|
with pytest.warns(FutureWarning,
|
|
match="The parameter 'X_idx_sorted' is deprecated"):
|
|
tree.fit(X, y, X_idx_sorted=X_idx_sorted)
|