# Authors: Danny Sullivan # Tom Dupre la Tour # # License: BSD 3 clause import math import pytest import numpy as np import scipy.sparse as sp from scipy.special import logsumexp from sklearn.linear_model._sag import get_auto_step_size from sklearn.linear_model._sag_fast import _multinomial_grad_loss_all_samples from sklearn.linear_model import LogisticRegression, Ridge from sklearn.linear_model._base import make_dataset from sklearn.linear_model._logistic import _multinomial_loss_grad from sklearn.utils.extmath import row_norms from sklearn.utils._testing import assert_almost_equal from sklearn.utils._testing import assert_array_almost_equal from sklearn.utils._testing import assert_allclose from sklearn.utils._testing import assert_raise_message from sklearn.utils import compute_class_weight from sklearn.utils import check_random_state from sklearn.preprocessing import LabelEncoder, LabelBinarizer from sklearn.datasets import make_blobs, load_iris, make_classification from sklearn.base import clone iris = load_iris() # this is used for sag classification def log_dloss(p, y): z = p * y # approximately equal and saves the computation of the log if z > 18.0: return math.exp(-z) * -y if z < -18.0: return -y return -y / (math.exp(z) + 1.0) def log_loss(p, y): return np.mean(np.log(1. + np.exp(-y * p))) # this is used for sag regression def squared_dloss(p, y): return p - y def squared_loss(p, y): return np.mean(0.5 * (p - y) * (p - y)) # function for measuring the log loss def get_pobj(w, alpha, myX, myy, loss): w = w.ravel() pred = np.dot(myX, w) p = loss(pred, myy) p += alpha * w.dot(w) / 2. return p def sag(X, y, step_size, alpha, n_iter=1, dloss=None, sparse=False, sample_weight=None, fit_intercept=True, saga=False): n_samples, n_features = X.shape[0], X.shape[1] weights = np.zeros(X.shape[1]) sum_gradient = np.zeros(X.shape[1]) gradient_memory = np.zeros((n_samples, n_features)) intercept = 0.0 intercept_sum_gradient = 0.0 intercept_gradient_memory = np.zeros(n_samples) rng = np.random.RandomState(77) decay = 1.0 seen = set() # sparse data has a fixed decay of .01 if sparse: decay = .01 for epoch in range(n_iter): for k in range(n_samples): idx = int(rng.rand(1) * n_samples) # idx = k entry = X[idx] seen.add(idx) p = np.dot(entry, weights) + intercept gradient = dloss(p, y[idx]) if sample_weight is not None: gradient *= sample_weight[idx] update = entry * gradient + alpha * weights gradient_correction = update - gradient_memory[idx] sum_gradient += gradient_correction gradient_memory[idx] = update if saga: weights -= (gradient_correction * step_size * (1 - 1. / len(seen))) if fit_intercept: gradient_correction = (gradient - intercept_gradient_memory[idx]) intercept_gradient_memory[idx] = gradient intercept_sum_gradient += gradient_correction gradient_correction *= step_size * (1. - 1. / len(seen)) if saga: intercept -= (step_size * intercept_sum_gradient / len(seen) * decay) + gradient_correction else: intercept -= (step_size * intercept_sum_gradient / len(seen) * decay) weights -= step_size * sum_gradient / len(seen) return weights, intercept def sag_sparse(X, y, step_size, alpha, n_iter=1, dloss=None, sample_weight=None, sparse=False, fit_intercept=True, saga=False, random_state=0): if step_size * alpha == 1.: raise ZeroDivisionError("Sparse sag does not handle the case " "step_size * alpha == 1") n_samples, n_features = X.shape[0], X.shape[1] weights = np.zeros(n_features) sum_gradient = np.zeros(n_features) last_updated = np.zeros(n_features, dtype=int) gradient_memory = np.zeros(n_samples) rng = check_random_state(random_state) intercept = 0.0 intercept_sum_gradient = 0.0 wscale = 1.0 decay = 1.0 seen = set() c_sum = np.zeros(n_iter * n_samples) # sparse data has a fixed decay of .01 if sparse: decay = .01 counter = 0 for epoch in range(n_iter): for k in range(n_samples): # idx = k idx = int(rng.rand(1) * n_samples) entry = X[idx] seen.add(idx) if counter >= 1: for j in range(n_features): if last_updated[j] == 0: weights[j] -= c_sum[counter - 1] * sum_gradient[j] else: weights[j] -= ((c_sum[counter - 1] - c_sum[last_updated[j] - 1]) * sum_gradient[j]) last_updated[j] = counter p = (wscale * np.dot(entry, weights)) + intercept gradient = dloss(p, y[idx]) if sample_weight is not None: gradient *= sample_weight[idx] update = entry * gradient gradient_correction = update - (gradient_memory[idx] * entry) sum_gradient += gradient_correction if saga: for j in range(n_features): weights[j] -= (gradient_correction[j] * step_size * (1 - 1. / len(seen)) / wscale) if fit_intercept: gradient_correction = gradient - gradient_memory[idx] intercept_sum_gradient += gradient_correction gradient_correction *= step_size * (1. - 1. / len(seen)) if saga: intercept -= ((step_size * intercept_sum_gradient / len(seen) * decay) + gradient_correction) else: intercept -= (step_size * intercept_sum_gradient / len(seen) * decay) gradient_memory[idx] = gradient wscale *= (1.0 - alpha * step_size) if counter == 0: c_sum[0] = step_size / (wscale * len(seen)) else: c_sum[counter] = (c_sum[counter - 1] + step_size / (wscale * len(seen))) if counter >= 1 and wscale < 1e-9: for j in range(n_features): if last_updated[j] == 0: weights[j] -= c_sum[counter] * sum_gradient[j] else: weights[j] -= ((c_sum[counter] - c_sum[last_updated[j] - 1]) * sum_gradient[j]) last_updated[j] = counter + 1 c_sum[counter] = 0 weights *= wscale wscale = 1.0 counter += 1 for j in range(n_features): if last_updated[j] == 0: weights[j] -= c_sum[counter - 1] * sum_gradient[j] else: weights[j] -= ((c_sum[counter - 1] - c_sum[last_updated[j] - 1]) * sum_gradient[j]) weights *= wscale return weights, intercept def get_step_size(X, alpha, fit_intercept, classification=True): if classification: return (4.0 / (np.max(np.sum(X * X, axis=1)) + fit_intercept + 4.0 * alpha)) else: return 1.0 / (np.max(np.sum(X * X, axis=1)) + fit_intercept + alpha) def test_classifier_matching(): n_samples = 20 X, y = make_blobs(n_samples=n_samples, centers=2, random_state=0, cluster_std=0.1) y[y == 0] = -1 alpha = 1.1 fit_intercept = True step_size = get_step_size(X, alpha, fit_intercept) for solver in ['sag', 'saga']: if solver == 'sag': n_iter = 80 else: # SAGA variance w.r.t. stream order is higher n_iter = 300 clf = LogisticRegression(solver=solver, fit_intercept=fit_intercept, tol=1e-11, C=1. / alpha / n_samples, max_iter=n_iter, random_state=10, multi_class='ovr') clf.fit(X, y) weights, intercept = sag_sparse(X, y, step_size, alpha, n_iter=n_iter, dloss=log_dloss, fit_intercept=fit_intercept, saga=solver == 'saga') weights2, intercept2 = sag(X, y, step_size, alpha, n_iter=n_iter, dloss=log_dloss, fit_intercept=fit_intercept, saga=solver == 'saga') weights = np.atleast_2d(weights) intercept = np.atleast_1d(intercept) weights2 = np.atleast_2d(weights2) intercept2 = np.atleast_1d(intercept2) assert_array_almost_equal(weights, clf.coef_, decimal=9) assert_array_almost_equal(intercept, clf.intercept_, decimal=9) assert_array_almost_equal(weights2, clf.coef_, decimal=9) assert_array_almost_equal(intercept2, clf.intercept_, decimal=9) def test_regressor_matching(): n_samples = 10 n_features = 5 rng = np.random.RandomState(10) X = rng.normal(size=(n_samples, n_features)) true_w = rng.normal(size=n_features) y = X.dot(true_w) alpha = 1. n_iter = 100 fit_intercept = True step_size = get_step_size(X, alpha, fit_intercept, classification=False) clf = Ridge(fit_intercept=fit_intercept, tol=.00000000001, solver='sag', alpha=alpha * n_samples, max_iter=n_iter) clf.fit(X, y) weights1, intercept1 = sag_sparse(X, y, step_size, alpha, n_iter=n_iter, dloss=squared_dloss, fit_intercept=fit_intercept) weights2, intercept2 = sag(X, y, step_size, alpha, n_iter=n_iter, dloss=squared_dloss, fit_intercept=fit_intercept) assert_allclose(weights1, clf.coef_) assert_allclose(intercept1, clf.intercept_) assert_allclose(weights2, clf.coef_) assert_allclose(intercept2, clf.intercept_) @pytest.mark.filterwarnings('ignore:The max_iter was reached') def test_sag_pobj_matches_logistic_regression(): """tests if the sag pobj matches log reg""" n_samples = 100 alpha = 1.0 max_iter = 20 X, y = make_blobs(n_samples=n_samples, centers=2, random_state=0, cluster_std=0.1) clf1 = LogisticRegression(solver='sag', fit_intercept=False, tol=.0000001, C=1. / alpha / n_samples, max_iter=max_iter, random_state=10, multi_class='ovr') clf2 = clone(clf1) clf3 = LogisticRegression(fit_intercept=False, tol=.0000001, C=1. / alpha / n_samples, max_iter=max_iter, random_state=10, multi_class='ovr') clf1.fit(X, y) clf2.fit(sp.csr_matrix(X), y) clf3.fit(X, y) pobj1 = get_pobj(clf1.coef_, alpha, X, y, log_loss) pobj2 = get_pobj(clf2.coef_, alpha, X, y, log_loss) pobj3 = get_pobj(clf3.coef_, alpha, X, y, log_loss) assert_array_almost_equal(pobj1, pobj2, decimal=4) assert_array_almost_equal(pobj2, pobj3, decimal=4) assert_array_almost_equal(pobj3, pobj1, decimal=4) @pytest.mark.filterwarnings('ignore:The max_iter was reached') def test_sag_pobj_matches_ridge_regression(): """tests if the sag pobj matches ridge reg""" n_samples = 100 n_features = 10 alpha = 1.0 n_iter = 100 fit_intercept = False rng = np.random.RandomState(10) X = rng.normal(size=(n_samples, n_features)) true_w = rng.normal(size=n_features) y = X.dot(true_w) clf1 = Ridge(fit_intercept=fit_intercept, tol=.00000000001, solver='sag', alpha=alpha, max_iter=n_iter, random_state=42) clf2 = clone(clf1) clf3 = Ridge(fit_intercept=fit_intercept, tol=.00001, solver='lsqr', alpha=alpha, max_iter=n_iter, random_state=42) clf1.fit(X, y) clf2.fit(sp.csr_matrix(X), y) clf3.fit(X, y) pobj1 = get_pobj(clf1.coef_, alpha, X, y, squared_loss) pobj2 = get_pobj(clf2.coef_, alpha, X, y, squared_loss) pobj3 = get_pobj(clf3.coef_, alpha, X, y, squared_loss) assert_array_almost_equal(pobj1, pobj2, decimal=4) assert_array_almost_equal(pobj1, pobj3, decimal=4) assert_array_almost_equal(pobj3, pobj2, decimal=4) @pytest.mark.filterwarnings('ignore:The max_iter was reached') def test_sag_regressor_computed_correctly(): """tests if the sag regressor is computed correctly""" alpha = .1 n_features = 10 n_samples = 40 max_iter = 100 tol = .000001 fit_intercept = True rng = np.random.RandomState(0) X = rng.normal(size=(n_samples, n_features)) w = rng.normal(size=n_features) y = np.dot(X, w) + 2. step_size = get_step_size(X, alpha, fit_intercept, classification=False) clf1 = Ridge(fit_intercept=fit_intercept, tol=tol, solver='sag', alpha=alpha * n_samples, max_iter=max_iter, random_state=rng) clf2 = clone(clf1) clf1.fit(X, y) clf2.fit(sp.csr_matrix(X), y) spweights1, spintercept1 = sag_sparse(X, y, step_size, alpha, n_iter=max_iter, dloss=squared_dloss, fit_intercept=fit_intercept, random_state=rng) spweights2, spintercept2 = sag_sparse(X, y, step_size, alpha, n_iter=max_iter, dloss=squared_dloss, sparse=True, fit_intercept=fit_intercept, random_state=rng) assert_array_almost_equal(clf1.coef_.ravel(), spweights1.ravel(), decimal=3) assert_almost_equal(clf1.intercept_, spintercept1, decimal=1) # TODO: uncomment when sparse Ridge with intercept will be fixed (#4710) # assert_array_almost_equal(clf2.coef_.ravel(), # spweights2.ravel(), # decimal=3) # assert_almost_equal(clf2.intercept_, spintercept2, decimal=1)''' def test_get_auto_step_size(): X = np.array([[1, 2, 3], [2, 3, 4], [2, 3, 2]], dtype=np.float64) alpha = 1.2 fit_intercept = False # sum the squares of the second sample because that's the largest max_squared_sum = 4 + 9 + 16 max_squared_sum_ = row_norms(X, squared=True).max() n_samples = X.shape[0] assert_almost_equal(max_squared_sum, max_squared_sum_, decimal=4) for saga in [True, False]: for fit_intercept in (True, False): if saga: L_sqr = (max_squared_sum + alpha + int(fit_intercept)) L_log = (max_squared_sum + 4.0 * alpha + int(fit_intercept)) / 4.0 mun_sqr = min(2 * n_samples * alpha, L_sqr) mun_log = min(2 * n_samples * alpha, L_log) step_size_sqr = 1 / (2 * L_sqr + mun_sqr) step_size_log = 1 / (2 * L_log + mun_log) else: step_size_sqr = 1.0 / (max_squared_sum + alpha + int(fit_intercept)) step_size_log = 4.0 / (max_squared_sum + 4.0 * alpha + int(fit_intercept)) step_size_sqr_ = get_auto_step_size(max_squared_sum_, alpha, "squared", fit_intercept, n_samples=n_samples, is_saga=saga) step_size_log_ = get_auto_step_size(max_squared_sum_, alpha, "log", fit_intercept, n_samples=n_samples, is_saga=saga) assert_almost_equal(step_size_sqr, step_size_sqr_, decimal=4) assert_almost_equal(step_size_log, step_size_log_, decimal=4) msg = 'Unknown loss function for SAG solver, got wrong instead of' assert_raise_message(ValueError, msg, get_auto_step_size, max_squared_sum_, alpha, "wrong", fit_intercept) @pytest.mark.parametrize("seed", range(3)) # locally tested with 1000 seeds def test_sag_regressor(seed): """tests if the sag regressor performs well""" xmin, xmax = -5, 5 n_samples = 300 tol = .001 max_iter = 100 alpha = 0.1 rng = np.random.RandomState(seed) X = np.linspace(xmin, xmax, n_samples).reshape(n_samples, 1) # simple linear function without noise y = 0.5 * X.ravel() clf1 = Ridge(tol=tol, solver='sag', max_iter=max_iter, alpha=alpha * n_samples, random_state=rng) clf2 = clone(clf1) clf1.fit(X, y) clf2.fit(sp.csr_matrix(X), y) score1 = clf1.score(X, y) score2 = clf2.score(X, y) assert score1 > 0.98 assert score2 > 0.98 # simple linear function with noise y = 0.5 * X.ravel() + rng.randn(n_samples, 1).ravel() clf1 = Ridge(tol=tol, solver='sag', max_iter=max_iter, alpha=alpha * n_samples) clf2 = clone(clf1) clf1.fit(X, y) clf2.fit(sp.csr_matrix(X), y) score1 = clf1.score(X, y) score2 = clf2.score(X, y) assert score1 > 0.45 assert score2 > 0.45 @pytest.mark.filterwarnings('ignore:The max_iter was reached') def test_sag_classifier_computed_correctly(): """tests if the binary classifier is computed correctly""" alpha = .1 n_samples = 50 n_iter = 50 tol = .00001 fit_intercept = True X, y = make_blobs(n_samples=n_samples, centers=2, random_state=0, cluster_std=0.1) step_size = get_step_size(X, alpha, fit_intercept, classification=True) classes = np.unique(y) y_tmp = np.ones(n_samples) y_tmp[y != classes[1]] = -1 y = y_tmp clf1 = LogisticRegression(solver='sag', C=1. / alpha / n_samples, max_iter=n_iter, tol=tol, random_state=77, fit_intercept=fit_intercept, multi_class='ovr') clf2 = clone(clf1) clf1.fit(X, y) clf2.fit(sp.csr_matrix(X), y) spweights, spintercept = sag_sparse(X, y, step_size, alpha, n_iter=n_iter, dloss=log_dloss, fit_intercept=fit_intercept) spweights2, spintercept2 = sag_sparse(X, y, step_size, alpha, n_iter=n_iter, dloss=log_dloss, sparse=True, fit_intercept=fit_intercept) assert_array_almost_equal(clf1.coef_.ravel(), spweights.ravel(), decimal=2) assert_almost_equal(clf1.intercept_, spintercept, decimal=1) assert_array_almost_equal(clf2.coef_.ravel(), spweights2.ravel(), decimal=2) assert_almost_equal(clf2.intercept_, spintercept2, decimal=1) @pytest.mark.filterwarnings('ignore:The max_iter was reached') def test_sag_multiclass_computed_correctly(): """tests if the multiclass classifier is computed correctly""" alpha = .1 n_samples = 20 tol = .00001 max_iter = 40 fit_intercept = True X, y = make_blobs(n_samples=n_samples, centers=3, random_state=0, cluster_std=0.1) step_size = get_step_size(X, alpha, fit_intercept, classification=True) classes = np.unique(y) clf1 = LogisticRegression(solver='sag', C=1. / alpha / n_samples, max_iter=max_iter, tol=tol, random_state=77, fit_intercept=fit_intercept, multi_class='ovr') clf2 = clone(clf1) clf1.fit(X, y) clf2.fit(sp.csr_matrix(X), y) coef1 = [] intercept1 = [] coef2 = [] intercept2 = [] for cl in classes: y_encoded = np.ones(n_samples) y_encoded[y != cl] = -1 spweights1, spintercept1 = sag_sparse(X, y_encoded, step_size, alpha, dloss=log_dloss, n_iter=max_iter, fit_intercept=fit_intercept) spweights2, spintercept2 = sag_sparse(X, y_encoded, step_size, alpha, dloss=log_dloss, n_iter=max_iter, sparse=True, fit_intercept=fit_intercept) coef1.append(spweights1) intercept1.append(spintercept1) coef2.append(spweights2) intercept2.append(spintercept2) coef1 = np.vstack(coef1) intercept1 = np.array(intercept1) coef2 = np.vstack(coef2) intercept2 = np.array(intercept2) for i, cl in enumerate(classes): assert_array_almost_equal(clf1.coef_[i].ravel(), coef1[i].ravel(), decimal=2) assert_almost_equal(clf1.intercept_[i], intercept1[i], decimal=1) assert_array_almost_equal(clf2.coef_[i].ravel(), coef2[i].ravel(), decimal=2) assert_almost_equal(clf2.intercept_[i], intercept2[i], decimal=1) def test_classifier_results(): """tests if classifier results match target""" alpha = .1 n_features = 20 n_samples = 10 tol = .01 max_iter = 200 rng = np.random.RandomState(0) X = rng.normal(size=(n_samples, n_features)) w = rng.normal(size=n_features) y = np.dot(X, w) y = np.sign(y) clf1 = LogisticRegression(solver='sag', C=1. / alpha / n_samples, max_iter=max_iter, tol=tol, random_state=77) clf2 = clone(clf1) clf1.fit(X, y) clf2.fit(sp.csr_matrix(X), y) pred1 = clf1.predict(X) pred2 = clf2.predict(X) assert_almost_equal(pred1, y, decimal=12) assert_almost_equal(pred2, y, decimal=12) @pytest.mark.filterwarnings('ignore:The max_iter was reached') def test_binary_classifier_class_weight(): """tests binary classifier with classweights for each class""" alpha = .1 n_samples = 50 n_iter = 20 tol = .00001 fit_intercept = True X, y = make_blobs(n_samples=n_samples, centers=2, random_state=10, cluster_std=0.1) step_size = get_step_size(X, alpha, fit_intercept, classification=True) classes = np.unique(y) y_tmp = np.ones(n_samples) y_tmp[y != classes[1]] = -1 y = y_tmp class_weight = {1: .45, -1: .55} clf1 = LogisticRegression(solver='sag', C=1. / alpha / n_samples, max_iter=n_iter, tol=tol, random_state=77, fit_intercept=fit_intercept, multi_class='ovr', class_weight=class_weight) clf2 = clone(clf1) clf1.fit(X, y) clf2.fit(sp.csr_matrix(X), y) le = LabelEncoder() class_weight_ = compute_class_weight(class_weight, classes=np.unique(y), y=y) sample_weight = class_weight_[le.fit_transform(y)] spweights, spintercept = sag_sparse(X, y, step_size, alpha, n_iter=n_iter, dloss=log_dloss, sample_weight=sample_weight, fit_intercept=fit_intercept) spweights2, spintercept2 = sag_sparse(X, y, step_size, alpha, n_iter=n_iter, dloss=log_dloss, sparse=True, sample_weight=sample_weight, fit_intercept=fit_intercept) assert_array_almost_equal(clf1.coef_.ravel(), spweights.ravel(), decimal=2) assert_almost_equal(clf1.intercept_, spintercept, decimal=1) assert_array_almost_equal(clf2.coef_.ravel(), spweights2.ravel(), decimal=2) assert_almost_equal(clf2.intercept_, spintercept2, decimal=1) @pytest.mark.filterwarnings('ignore:The max_iter was reached') def test_multiclass_classifier_class_weight(): """tests multiclass with classweights for each class""" alpha = .1 n_samples = 20 tol = .00001 max_iter = 50 class_weight = {0: .45, 1: .55, 2: .75} fit_intercept = True X, y = make_blobs(n_samples=n_samples, centers=3, random_state=0, cluster_std=0.1) step_size = get_step_size(X, alpha, fit_intercept, classification=True) classes = np.unique(y) clf1 = LogisticRegression(solver='sag', C=1. / alpha / n_samples, max_iter=max_iter, tol=tol, random_state=77, fit_intercept=fit_intercept, multi_class='ovr', class_weight=class_weight) clf2 = clone(clf1) clf1.fit(X, y) clf2.fit(sp.csr_matrix(X), y) le = LabelEncoder() class_weight_ = compute_class_weight(class_weight, classes=np.unique(y), y=y) sample_weight = class_weight_[le.fit_transform(y)] coef1 = [] intercept1 = [] coef2 = [] intercept2 = [] for cl in classes: y_encoded = np.ones(n_samples) y_encoded[y != cl] = -1 spweights1, spintercept1 = sag_sparse(X, y_encoded, step_size, alpha, n_iter=max_iter, dloss=log_dloss, sample_weight=sample_weight) spweights2, spintercept2 = sag_sparse(X, y_encoded, step_size, alpha, n_iter=max_iter, dloss=log_dloss, sample_weight=sample_weight, sparse=True) coef1.append(spweights1) intercept1.append(spintercept1) coef2.append(spweights2) intercept2.append(spintercept2) coef1 = np.vstack(coef1) intercept1 = np.array(intercept1) coef2 = np.vstack(coef2) intercept2 = np.array(intercept2) for i, cl in enumerate(classes): assert_array_almost_equal(clf1.coef_[i].ravel(), coef1[i].ravel(), decimal=2) assert_almost_equal(clf1.intercept_[i], intercept1[i], decimal=1) assert_array_almost_equal(clf2.coef_[i].ravel(), coef2[i].ravel(), decimal=2) assert_almost_equal(clf2.intercept_[i], intercept2[i], decimal=1) def test_classifier_single_class(): """tests if ValueError is thrown with only one class""" X = [[1, 2], [3, 4]] y = [1, 1] assert_raise_message(ValueError, "This solver needs samples of at least 2 classes " "in the data", LogisticRegression(solver='sag').fit, X, y) def test_step_size_alpha_error(): X = [[0, 0], [0, 0]] y = [1, -1] fit_intercept = False alpha = 1. msg = ("Current sag implementation does not handle the case" " step_size * alpha_scaled == 1") clf1 = LogisticRegression(solver='sag', C=1. / alpha, fit_intercept=fit_intercept) assert_raise_message(ZeroDivisionError, msg, clf1.fit, X, y) clf2 = Ridge(fit_intercept=fit_intercept, solver='sag', alpha=alpha) assert_raise_message(ZeroDivisionError, msg, clf2.fit, X, y) def test_multinomial_loss(): # test if the multinomial loss and gradient computations are consistent X, y = iris.data, iris.target.astype(np.float64) n_samples, n_features = X.shape n_classes = len(np.unique(y)) rng = check_random_state(42) weights = rng.randn(n_features, n_classes) intercept = rng.randn(n_classes) sample_weights = rng.randn(n_samples) np.abs(sample_weights, sample_weights) # compute loss and gradient like in multinomial SAG dataset, _ = make_dataset(X, y, sample_weights, random_state=42) loss_1, grad_1 = _multinomial_grad_loss_all_samples(dataset, weights, intercept, n_samples, n_features, n_classes) # compute loss and gradient like in multinomial LogisticRegression lbin = LabelBinarizer() Y_bin = lbin.fit_transform(y) weights_intercept = np.vstack((weights, intercept)).T.ravel() loss_2, grad_2, _ = _multinomial_loss_grad(weights_intercept, X, Y_bin, 0.0, sample_weights) grad_2 = grad_2.reshape(n_classes, -1) grad_2 = grad_2[:, :-1].T # comparison assert_array_almost_equal(grad_1, grad_2) assert_almost_equal(loss_1, loss_2) def test_multinomial_loss_ground_truth(): # n_samples, n_features, n_classes = 4, 2, 3 n_classes = 3 X = np.array([[1.1, 2.2], [2.2, -4.4], [3.3, -2.2], [1.1, 1.1]]) y = np.array([0, 1, 2, 0]) lbin = LabelBinarizer() Y_bin = lbin.fit_transform(y) weights = np.array([[0.1, 0.2, 0.3], [1.1, 1.2, -1.3]]) intercept = np.array([1., 0, -.2]) sample_weights = np.array([0.8, 1, 1, 0.8]) prediction = np.dot(X, weights) + intercept logsumexp_prediction = logsumexp(prediction, axis=1) p = prediction - logsumexp_prediction[:, np.newaxis] loss_1 = -(sample_weights[:, np.newaxis] * p * Y_bin).sum() diff = sample_weights[:, np.newaxis] * (np.exp(p) - Y_bin) grad_1 = np.dot(X.T, diff) weights_intercept = np.vstack((weights, intercept)).T.ravel() loss_2, grad_2, _ = _multinomial_loss_grad(weights_intercept, X, Y_bin, 0.0, sample_weights) grad_2 = grad_2.reshape(n_classes, -1) grad_2 = grad_2[:, :-1].T assert_almost_equal(loss_1, loss_2) assert_array_almost_equal(grad_1, grad_2) # ground truth loss_gt = 11.680360354325961 grad_gt = np.array([[-0.557487, -1.619151, +2.176638], [-0.903942, +5.258745, -4.354803]]) assert_almost_equal(loss_1, loss_gt) assert_array_almost_equal(grad_1, grad_gt) @pytest.mark.parametrize("solver", ["sag", "saga"]) def test_sag_classifier_raises_error(solver): # Following #13316, the error handling behavior changed in cython sag. This # is simply a non-regression test to make sure numerical errors are # properly raised. # Train a classifier on a simple problem rng = np.random.RandomState(42) X, y = make_classification(random_state=rng) clf = LogisticRegression(solver=solver, random_state=rng, warm_start=True) clf.fit(X, y) # Trigger a numerical error by: # - corrupting the fitted coefficients of the classifier # - fit it again starting from its current state thanks to warm_start clf.coef_[:] = np.nan with pytest.raises(ValueError, match="Floating-point under-/overflow"): clf.fit(X, y)