from collections.abc import Mapping import re import pytest import warnings from scipy import sparse from sklearn.feature_extraction.text import strip_tags from sklearn.feature_extraction.text import strip_accents_unicode from sklearn.feature_extraction.text import strip_accents_ascii from sklearn.feature_extraction.text import HashingVectorizer from sklearn.feature_extraction.text import CountVectorizer from sklearn.feature_extraction.text import TfidfTransformer from sklearn.feature_extraction.text import TfidfVectorizer from sklearn.feature_extraction.text import ENGLISH_STOP_WORDS from sklearn.model_selection import train_test_split from sklearn.model_selection import cross_val_score from sklearn.model_selection import GridSearchCV from sklearn.pipeline import Pipeline from sklearn.svm import LinearSVC from sklearn.base import clone import numpy as np from numpy.testing import assert_array_almost_equal from numpy.testing import assert_array_equal from sklearn.utils import IS_PYPY from sklearn.utils._testing import ( assert_almost_equal, fails_if_pypy, assert_allclose_dense_sparse, skip_if_32bit, ) from collections import defaultdict from functools import partial import pickle from io import StringIO JUNK_FOOD_DOCS = ( "the pizza pizza beer copyright", "the pizza burger beer copyright", "the the pizza beer beer copyright", "the burger beer beer copyright", "the coke burger coke copyright", "the coke burger burger", ) NOTJUNK_FOOD_DOCS = ( "the salad celeri copyright", "the salad salad sparkling water copyright", "the the celeri celeri copyright", "the tomato tomato salad water", "the tomato salad water copyright", ) ALL_FOOD_DOCS = JUNK_FOOD_DOCS + NOTJUNK_FOOD_DOCS def uppercase(s): return strip_accents_unicode(s).upper() def strip_eacute(s): return s.replace("é", "e") def split_tokenize(s): return s.split() def lazy_analyze(s): return ["the_ultimate_feature"] def test_strip_accents(): # check some classical latin accentuated symbols a = "àáâãäåçèéêë" expected = "aaaaaaceeee" assert strip_accents_unicode(a) == expected a = "ìíîïñòóôõöùúûüý" expected = "iiiinooooouuuuy" assert strip_accents_unicode(a) == expected # check some arabic a = "\u0625" # alef with a hamza below: إ expected = "\u0627" # simple alef: ا assert strip_accents_unicode(a) == expected # mix letters accentuated and not a = "this is à test" expected = "this is a test" assert strip_accents_unicode(a) == expected # strings that are already decomposed a = "o\u0308" # o with diaeresis expected = "o" assert strip_accents_unicode(a) == expected # combining marks by themselves a = "\u0300\u0301\u0302\u0303" expected = "" assert strip_accents_unicode(a) == expected # Multiple combining marks on one character a = "o\u0308\u0304" expected = "o" assert strip_accents_unicode(a) == expected def test_to_ascii(): # check some classical latin accentuated symbols a = "àáâãäåçèéêë" expected = "aaaaaaceeee" assert strip_accents_ascii(a) == expected a = "ìíîïñòóôõöùúûüý" expected = "iiiinooooouuuuy" assert strip_accents_ascii(a) == expected # check some arabic a = "\u0625" # halef with a hamza below expected = "" # halef has no direct ascii match assert strip_accents_ascii(a) == expected # mix letters accentuated and not a = "this is à test" expected = "this is a test" assert strip_accents_ascii(a) == expected @pytest.mark.parametrize("Vectorizer", (CountVectorizer, HashingVectorizer)) def test_word_analyzer_unigrams(Vectorizer): wa = Vectorizer(strip_accents="ascii").build_analyzer() text = "J'ai mangé du kangourou ce midi, c'était pas très bon." expected = [ "ai", "mange", "du", "kangourou", "ce", "midi", "etait", "pas", "tres", "bon", ] assert wa(text) == expected text = "This is a test, really.\n\n I met Harry yesterday." expected = ["this", "is", "test", "really", "met", "harry", "yesterday"] assert wa(text) == expected wa = Vectorizer(input="file").build_analyzer() text = StringIO("This is a test with a file-like object!") expected = ["this", "is", "test", "with", "file", "like", "object"] assert wa(text) == expected # with custom preprocessor wa = Vectorizer(preprocessor=uppercase).build_analyzer() text = "J'ai mangé du kangourou ce midi, c'était pas très bon." expected = [ "AI", "MANGE", "DU", "KANGOUROU", "CE", "MIDI", "ETAIT", "PAS", "TRES", "BON", ] assert wa(text) == expected # with custom tokenizer wa = Vectorizer(tokenizer=split_tokenize, strip_accents="ascii").build_analyzer() text = "J'ai mangé du kangourou ce midi, c'était pas très bon." expected = [ "j'ai", "mange", "du", "kangourou", "ce", "midi,", "c'etait", "pas", "tres", "bon.", ] assert wa(text) == expected def test_word_analyzer_unigrams_and_bigrams(): wa = CountVectorizer( analyzer="word", strip_accents="unicode", ngram_range=(1, 2) ).build_analyzer() text = "J'ai mangé du kangourou ce midi, c'était pas très bon." expected = [ "ai", "mange", "du", "kangourou", "ce", "midi", "etait", "pas", "tres", "bon", "ai mange", "mange du", "du kangourou", "kangourou ce", "ce midi", "midi etait", "etait pas", "pas tres", "tres bon", ] assert wa(text) == expected def test_unicode_decode_error(): # decode_error default to strict, so this should fail # First, encode (as bytes) a unicode string. text = "J'ai mangé du kangourou ce midi, c'était pas très bon." text_bytes = text.encode("utf-8") # Then let the Analyzer try to decode it as ascii. It should fail, # because we have given it an incorrect encoding. wa = CountVectorizer(ngram_range=(1, 2), encoding="ascii").build_analyzer() with pytest.raises(UnicodeDecodeError): wa(text_bytes) ca = CountVectorizer( analyzer="char", ngram_range=(3, 6), encoding="ascii" ).build_analyzer() with pytest.raises(UnicodeDecodeError): ca(text_bytes) def test_char_ngram_analyzer(): cnga = CountVectorizer( analyzer="char", strip_accents="unicode", ngram_range=(3, 6) ).build_analyzer() text = "J'ai mangé du kangourou ce midi, c'était pas très bon" expected = ["j'a", "'ai", "ai ", "i m", " ma"] assert cnga(text)[:5] == expected expected = ["s tres", " tres ", "tres b", "res bo", "es bon"] assert cnga(text)[-5:] == expected text = "This \n\tis a test, really.\n\n I met Harry yesterday" expected = ["thi", "his", "is ", "s i", " is"] assert cnga(text)[:5] == expected expected = [" yeste", "yester", "esterd", "sterda", "terday"] assert cnga(text)[-5:] == expected cnga = CountVectorizer( input="file", analyzer="char", ngram_range=(3, 6) ).build_analyzer() text = StringIO("This is a test with a file-like object!") expected = ["thi", "his", "is ", "s i", " is"] assert cnga(text)[:5] == expected def test_char_wb_ngram_analyzer(): cnga = CountVectorizer( analyzer="char_wb", strip_accents="unicode", ngram_range=(3, 6) ).build_analyzer() text = "This \n\tis a test, really.\n\n I met Harry yesterday" expected = [" th", "thi", "his", "is ", " thi"] assert cnga(text)[:5] == expected expected = ["yester", "esterd", "sterda", "terday", "erday "] assert cnga(text)[-5:] == expected cnga = CountVectorizer( input="file", analyzer="char_wb", ngram_range=(3, 6) ).build_analyzer() text = StringIO("A test with a file-like object!") expected = [" a ", " te", "tes", "est", "st ", " tes"] assert cnga(text)[:6] == expected def test_word_ngram_analyzer(): cnga = CountVectorizer( analyzer="word", strip_accents="unicode", ngram_range=(3, 6) ).build_analyzer() text = "This \n\tis a test, really.\n\n I met Harry yesterday" expected = ["this is test", "is test really", "test really met"] assert cnga(text)[:3] == expected expected = [ "test really met harry yesterday", "this is test really met harry", "is test really met harry yesterday", ] assert cnga(text)[-3:] == expected cnga_file = CountVectorizer( input="file", analyzer="word", ngram_range=(3, 6) ).build_analyzer() file = StringIO(text) assert cnga_file(file) == cnga(text) def test_countvectorizer_custom_vocabulary(): vocab = {"pizza": 0, "beer": 1} terms = set(vocab.keys()) # Try a few of the supported types. for typ in [dict, list, iter, partial(defaultdict, int)]: v = typ(vocab) vect = CountVectorizer(vocabulary=v) vect.fit(JUNK_FOOD_DOCS) if isinstance(v, Mapping): assert vect.vocabulary_ == vocab else: assert set(vect.vocabulary_) == terms X = vect.transform(JUNK_FOOD_DOCS) assert X.shape[1] == len(terms) v = typ(vocab) vect = CountVectorizer(vocabulary=v) inv = vect.inverse_transform(X) assert len(inv) == X.shape[0] def test_countvectorizer_custom_vocabulary_pipeline(): what_we_like = ["pizza", "beer"] pipe = Pipeline( [ ("count", CountVectorizer(vocabulary=what_we_like)), ("tfidf", TfidfTransformer()), ] ) X = pipe.fit_transform(ALL_FOOD_DOCS) assert set(pipe.named_steps["count"].vocabulary_) == set(what_we_like) assert X.shape[1] == len(what_we_like) def test_countvectorizer_custom_vocabulary_repeated_indices(): vocab = {"pizza": 0, "beer": 0} msg = "Vocabulary contains repeated indices" with pytest.raises(ValueError, match=msg): vect = CountVectorizer(vocabulary=vocab) vect.fit(["pasta_siziliana"]) def test_countvectorizer_custom_vocabulary_gap_index(): vocab = {"pizza": 1, "beer": 2} with pytest.raises(ValueError, match="doesn't contain index"): vect = CountVectorizer(vocabulary=vocab) vect.fit(["pasta_verdura"]) def test_countvectorizer_stop_words(): cv = CountVectorizer() cv.set_params(stop_words="english") assert cv.get_stop_words() == ENGLISH_STOP_WORDS cv.set_params(stop_words="_bad_str_stop_") with pytest.raises(ValueError): cv.get_stop_words() cv.set_params(stop_words="_bad_unicode_stop_") with pytest.raises(ValueError): cv.get_stop_words() stoplist = ["some", "other", "words"] cv.set_params(stop_words=stoplist) assert cv.get_stop_words() == set(stoplist) def test_countvectorizer_empty_vocabulary(): with pytest.raises(ValueError, match="empty vocabulary"): vect = CountVectorizer(vocabulary=[]) vect.fit(["foo"]) with pytest.raises(ValueError, match="empty vocabulary"): v = CountVectorizer(max_df=1.0, stop_words="english") # fit on stopwords only v.fit(["to be or not to be", "and me too", "and so do you"]) def test_fit_countvectorizer_twice(): cv = CountVectorizer() X1 = cv.fit_transform(ALL_FOOD_DOCS[:5]) X2 = cv.fit_transform(ALL_FOOD_DOCS[5:]) assert X1.shape[1] != X2.shape[1] def test_countvectorizer_custom_token_pattern(): """Check `get_feature_names_out()` when a custom token pattern is passed. Non-regression test for: https://github.com/scikit-learn/scikit-learn/issues/12971 """ corpus = [ "This is the 1st document in my corpus.", "This document is the 2nd sample.", "And this is the 3rd one.", "Is this the 4th document?", ] token_pattern = r"[0-9]{1,3}(?:st|nd|rd|th)\s\b(\w{2,})\b" vectorizer = CountVectorizer(token_pattern=token_pattern) vectorizer.fit_transform(corpus) expected = ["document", "one", "sample"] feature_names_out = vectorizer.get_feature_names_out() assert_array_equal(feature_names_out, expected) def test_countvectorizer_custom_token_pattern_with_several_group(): """Check that we raise an error if token pattern capture several groups. Non-regression test for: https://github.com/scikit-learn/scikit-learn/issues/12971 """ corpus = [ "This is the 1st document in my corpus.", "This document is the 2nd sample.", "And this is the 3rd one.", "Is this the 4th document?", ] token_pattern = r"([0-9]{1,3}(?:st|nd|rd|th))\s\b(\w{2,})\b" err_msg = "More than 1 capturing group in token pattern" vectorizer = CountVectorizer(token_pattern=token_pattern) with pytest.raises(ValueError, match=err_msg): vectorizer.fit(corpus) def test_countvectorizer_uppercase_in_vocab(): # Check that the check for uppercase in the provided vocabulary is only done at fit # time and not at transform time (#21251) vocabulary = ["Sample", "Upper", "Case", "Vocabulary"] message = ( "Upper case characters found in" " vocabulary while 'lowercase'" " is True. These entries will not" " be matched with any documents" ) vectorizer = CountVectorizer(lowercase=True, vocabulary=vocabulary) with pytest.warns(UserWarning, match=message): vectorizer.fit(vocabulary) with warnings.catch_warnings(): warnings.simplefilter("error", UserWarning) vectorizer.transform(vocabulary) def test_tf_transformer_feature_names_out(): """Check get_feature_names_out for TfidfTransformer""" X = [[1, 1, 1], [1, 1, 0], [1, 0, 0]] tr = TfidfTransformer(smooth_idf=True, norm="l2").fit(X) feature_names_in = ["a", "c", "b"] feature_names_out = tr.get_feature_names_out(feature_names_in) assert_array_equal(feature_names_in, feature_names_out) def test_tf_idf_smoothing(): X = [[1, 1, 1], [1, 1, 0], [1, 0, 0]] tr = TfidfTransformer(smooth_idf=True, norm="l2") tfidf = tr.fit_transform(X).toarray() assert (tfidf >= 0).all() # check normalization assert_array_almost_equal((tfidf**2).sum(axis=1), [1.0, 1.0, 1.0]) # this is robust to features with only zeros X = [[1, 1, 0], [1, 1, 0], [1, 0, 0]] tr = TfidfTransformer(smooth_idf=True, norm="l2") tfidf = tr.fit_transform(X).toarray() assert (tfidf >= 0).all() def test_tfidf_no_smoothing(): X = [[1, 1, 1], [1, 1, 0], [1, 0, 0]] tr = TfidfTransformer(smooth_idf=False, norm="l2") tfidf = tr.fit_transform(X).toarray() assert (tfidf >= 0).all() # check normalization assert_array_almost_equal((tfidf**2).sum(axis=1), [1.0, 1.0, 1.0]) # the lack of smoothing make IDF fragile in the presence of feature with # only zeros X = [[1, 1, 0], [1, 1, 0], [1, 0, 0]] tr = TfidfTransformer(smooth_idf=False, norm="l2") in_warning_message = "divide by zero" with pytest.warns(RuntimeWarning, match=in_warning_message): tr.fit_transform(X).toarray() def test_sublinear_tf(): X = [[1], [2], [3]] tr = TfidfTransformer(sublinear_tf=True, use_idf=False, norm=None) tfidf = tr.fit_transform(X).toarray() assert tfidf[0] == 1 assert tfidf[1] > tfidf[0] assert tfidf[2] > tfidf[1] assert tfidf[1] < 2 assert tfidf[2] < 3 def test_vectorizer(): # raw documents as an iterator train_data = iter(ALL_FOOD_DOCS[:-1]) test_data = [ALL_FOOD_DOCS[-1]] n_train = len(ALL_FOOD_DOCS) - 1 # test without vocabulary v1 = CountVectorizer(max_df=0.5) counts_train = v1.fit_transform(train_data) if hasattr(counts_train, "tocsr"): counts_train = counts_train.tocsr() assert counts_train[0, v1.vocabulary_["pizza"]] == 2 # build a vectorizer v1 with the same vocabulary as the one fitted by v1 v2 = CountVectorizer(vocabulary=v1.vocabulary_) # compare that the two vectorizer give the same output on the test sample for v in (v1, v2): counts_test = v.transform(test_data) if hasattr(counts_test, "tocsr"): counts_test = counts_test.tocsr() vocabulary = v.vocabulary_ assert counts_test[0, vocabulary["salad"]] == 1 assert counts_test[0, vocabulary["tomato"]] == 1 assert counts_test[0, vocabulary["water"]] == 1 # stop word from the fixed list assert "the" not in vocabulary # stop word found automatically by the vectorizer DF thresholding # words that are high frequent across the complete corpus are likely # to be not informative (either real stop words of extraction # artifacts) assert "copyright" not in vocabulary # not present in the sample assert counts_test[0, vocabulary["coke"]] == 0 assert counts_test[0, vocabulary["burger"]] == 0 assert counts_test[0, vocabulary["beer"]] == 0 assert counts_test[0, vocabulary["pizza"]] == 0 # test tf-idf t1 = TfidfTransformer(norm="l1") tfidf = t1.fit(counts_train).transform(counts_train).toarray() assert len(t1.idf_) == len(v1.vocabulary_) assert tfidf.shape == (n_train, len(v1.vocabulary_)) # test tf-idf with new data tfidf_test = t1.transform(counts_test).toarray() assert tfidf_test.shape == (len(test_data), len(v1.vocabulary_)) # test tf alone t2 = TfidfTransformer(norm="l1", use_idf=False) tf = t2.fit(counts_train).transform(counts_train).toarray() assert not hasattr(t2, "idf_") # test idf transform with unlearned idf vector t3 = TfidfTransformer(use_idf=True) with pytest.raises(ValueError): t3.transform(counts_train) # L1-normalized term frequencies sum to one assert_array_almost_equal(np.sum(tf, axis=1), [1.0] * n_train) # test the direct tfidf vectorizer # (equivalent to term count vectorizer + tfidf transformer) train_data = iter(ALL_FOOD_DOCS[:-1]) tv = TfidfVectorizer(norm="l1") tv.max_df = v1.max_df tfidf2 = tv.fit_transform(train_data).toarray() assert not tv.fixed_vocabulary_ assert_array_almost_equal(tfidf, tfidf2) # test the direct tfidf vectorizer with new data tfidf_test2 = tv.transform(test_data).toarray() assert_array_almost_equal(tfidf_test, tfidf_test2) # test transform on unfitted vectorizer with empty vocabulary v3 = CountVectorizer(vocabulary=None) with pytest.raises(ValueError): v3.transform(train_data) # ascii preprocessor? v3.set_params(strip_accents="ascii", lowercase=False) processor = v3.build_preprocessor() text = "J'ai mangé du kangourou ce midi, c'était pas très bon." expected = strip_accents_ascii(text) result = processor(text) assert expected == result # error on bad strip_accents param v3.set_params(strip_accents="_gabbledegook_", preprocessor=None) with pytest.raises(ValueError): v3.build_preprocessor() # error with bad analyzer type v3.set_params = "_invalid_analyzer_type_" with pytest.raises(ValueError): v3.build_analyzer() def test_tfidf_vectorizer_setters(): norm, use_idf, smooth_idf, sublinear_tf = "l2", False, False, False tv = TfidfVectorizer( norm=norm, use_idf=use_idf, smooth_idf=smooth_idf, sublinear_tf=sublinear_tf ) tv.fit(JUNK_FOOD_DOCS) assert tv._tfidf.norm == norm assert tv._tfidf.use_idf == use_idf assert tv._tfidf.smooth_idf == smooth_idf assert tv._tfidf.sublinear_tf == sublinear_tf # assigning value to `TfidfTransformer` should not have any effect until # fitting tv.norm = "l1" tv.use_idf = True tv.smooth_idf = True tv.sublinear_tf = True assert tv._tfidf.norm == norm assert tv._tfidf.use_idf == use_idf assert tv._tfidf.smooth_idf == smooth_idf assert tv._tfidf.sublinear_tf == sublinear_tf tv.fit(JUNK_FOOD_DOCS) assert tv._tfidf.norm == tv.norm assert tv._tfidf.use_idf == tv.use_idf assert tv._tfidf.smooth_idf == tv.smooth_idf assert tv._tfidf.sublinear_tf == tv.sublinear_tf @fails_if_pypy def test_hashing_vectorizer(): v = HashingVectorizer() X = v.transform(ALL_FOOD_DOCS) token_nnz = X.nnz assert X.shape == (len(ALL_FOOD_DOCS), v.n_features) assert X.dtype == v.dtype # By default the hashed values receive a random sign and l2 normalization # makes the feature values bounded assert np.min(X.data) > -1 assert np.min(X.data) < 0 assert np.max(X.data) > 0 assert np.max(X.data) < 1 # Check that the rows are normalized for i in range(X.shape[0]): assert_almost_equal(np.linalg.norm(X[0].data, 2), 1.0) # Check vectorization with some non-default parameters v = HashingVectorizer(ngram_range=(1, 2), norm="l1") X = v.transform(ALL_FOOD_DOCS) assert X.shape == (len(ALL_FOOD_DOCS), v.n_features) assert X.dtype == v.dtype # ngrams generate more non zeros ngrams_nnz = X.nnz assert ngrams_nnz > token_nnz assert ngrams_nnz < 2 * token_nnz # makes the feature values bounded assert np.min(X.data) > -1 assert np.max(X.data) < 1 # Check that the rows are normalized for i in range(X.shape[0]): assert_almost_equal(np.linalg.norm(X[0].data, 1), 1.0) def test_feature_names(): cv = CountVectorizer(max_df=0.5) # test for Value error on unfitted/empty vocabulary with pytest.raises(ValueError): cv.get_feature_names_out() assert not cv.fixed_vocabulary_ # test for vocabulary learned from data X = cv.fit_transform(ALL_FOOD_DOCS) n_samples, n_features = X.shape assert len(cv.vocabulary_) == n_features feature_names = cv.get_feature_names_out() assert isinstance(feature_names, np.ndarray) assert feature_names.dtype == object assert len(feature_names) == n_features assert_array_equal( [ "beer", "burger", "celeri", "coke", "pizza", "salad", "sparkling", "tomato", "water", ], feature_names, ) for idx, name in enumerate(feature_names): assert idx == cv.vocabulary_.get(name) # test for custom vocabulary vocab = [ "beer", "burger", "celeri", "coke", "pizza", "salad", "sparkling", "tomato", "water", ] cv = CountVectorizer(vocabulary=vocab) feature_names = cv.get_feature_names_out() assert_array_equal( [ "beer", "burger", "celeri", "coke", "pizza", "salad", "sparkling", "tomato", "water", ], feature_names, ) assert cv.fixed_vocabulary_ for idx, name in enumerate(feature_names): assert idx == cv.vocabulary_.get(name) @pytest.mark.parametrize("Vectorizer", (CountVectorizer, TfidfVectorizer)) def test_vectorizer_max_features(Vectorizer): expected_vocabulary = {"burger", "beer", "salad", "pizza"} expected_stop_words = { "celeri", "tomato", "copyright", "coke", "sparkling", "water", "the", } # test bounded number of extracted features vectorizer = Vectorizer(max_df=0.6, max_features=4) vectorizer.fit(ALL_FOOD_DOCS) assert set(vectorizer.vocabulary_) == expected_vocabulary assert vectorizer.stop_words_ == expected_stop_words def test_count_vectorizer_max_features(): # Regression test: max_features didn't work correctly in 0.14. cv_1 = CountVectorizer(max_features=1) cv_3 = CountVectorizer(max_features=3) cv_None = CountVectorizer(max_features=None) counts_1 = cv_1.fit_transform(JUNK_FOOD_DOCS).sum(axis=0) counts_3 = cv_3.fit_transform(JUNK_FOOD_DOCS).sum(axis=0) counts_None = cv_None.fit_transform(JUNK_FOOD_DOCS).sum(axis=0) features_1 = cv_1.get_feature_names_out() features_3 = cv_3.get_feature_names_out() features_None = cv_None.get_feature_names_out() # The most common feature is "the", with frequency 7. assert 7 == counts_1.max() assert 7 == counts_3.max() assert 7 == counts_None.max() # The most common feature should be the same assert "the" == features_1[np.argmax(counts_1)] assert "the" == features_3[np.argmax(counts_3)] assert "the" == features_None[np.argmax(counts_None)] def test_vectorizer_max_df(): test_data = ["abc", "dea", "eat"] vect = CountVectorizer(analyzer="char", max_df=1.0) vect.fit(test_data) assert "a" in vect.vocabulary_.keys() assert len(vect.vocabulary_.keys()) == 6 assert len(vect.stop_words_) == 0 vect.max_df = 0.5 # 0.5 * 3 documents -> max_doc_count == 1.5 vect.fit(test_data) assert "a" not in vect.vocabulary_.keys() # {ae} ignored assert len(vect.vocabulary_.keys()) == 4 # {bcdt} remain assert "a" in vect.stop_words_ assert len(vect.stop_words_) == 2 vect.max_df = 1 vect.fit(test_data) assert "a" not in vect.vocabulary_.keys() # {ae} ignored assert len(vect.vocabulary_.keys()) == 4 # {bcdt} remain assert "a" in vect.stop_words_ assert len(vect.stop_words_) == 2 def test_vectorizer_min_df(): test_data = ["abc", "dea", "eat"] vect = CountVectorizer(analyzer="char", min_df=1) vect.fit(test_data) assert "a" in vect.vocabulary_.keys() assert len(vect.vocabulary_.keys()) == 6 assert len(vect.stop_words_) == 0 vect.min_df = 2 vect.fit(test_data) assert "c" not in vect.vocabulary_.keys() # {bcdt} ignored assert len(vect.vocabulary_.keys()) == 2 # {ae} remain assert "c" in vect.stop_words_ assert len(vect.stop_words_) == 4 vect.min_df = 0.8 # 0.8 * 3 documents -> min_doc_count == 2.4 vect.fit(test_data) assert "c" not in vect.vocabulary_.keys() # {bcdet} ignored assert len(vect.vocabulary_.keys()) == 1 # {a} remains assert "c" in vect.stop_words_ assert len(vect.stop_words_) == 5 def test_count_binary_occurrences(): # by default multiple occurrences are counted as longs test_data = ["aaabc", "abbde"] vect = CountVectorizer(analyzer="char", max_df=1.0) X = vect.fit_transform(test_data).toarray() assert_array_equal(["a", "b", "c", "d", "e"], vect.get_feature_names_out()) assert_array_equal([[3, 1, 1, 0, 0], [1, 2, 0, 1, 1]], X) # using boolean features, we can fetch the binary occurrence info # instead. vect = CountVectorizer(analyzer="char", max_df=1.0, binary=True) X = vect.fit_transform(test_data).toarray() assert_array_equal([[1, 1, 1, 0, 0], [1, 1, 0, 1, 1]], X) # check the ability to change the dtype vect = CountVectorizer(analyzer="char", max_df=1.0, binary=True, dtype=np.float32) X_sparse = vect.fit_transform(test_data) assert X_sparse.dtype == np.float32 @fails_if_pypy def test_hashed_binary_occurrences(): # by default multiple occurrences are counted as longs test_data = ["aaabc", "abbde"] vect = HashingVectorizer(alternate_sign=False, analyzer="char", norm=None) X = vect.transform(test_data) assert np.max(X[0:1].data) == 3 assert np.max(X[1:2].data) == 2 assert X.dtype == np.float64 # using boolean features, we can fetch the binary occurrence info # instead. vect = HashingVectorizer( analyzer="char", alternate_sign=False, binary=True, norm=None ) X = vect.transform(test_data) assert np.max(X.data) == 1 assert X.dtype == np.float64 # check the ability to change the dtype vect = HashingVectorizer( analyzer="char", alternate_sign=False, binary=True, norm=None, dtype=np.float64 ) X = vect.transform(test_data) assert X.dtype == np.float64 @pytest.mark.parametrize("Vectorizer", (CountVectorizer, TfidfVectorizer)) def test_vectorizer_inverse_transform(Vectorizer): # raw documents data = ALL_FOOD_DOCS vectorizer = Vectorizer() transformed_data = vectorizer.fit_transform(data) inversed_data = vectorizer.inverse_transform(transformed_data) assert isinstance(inversed_data, list) analyze = vectorizer.build_analyzer() for doc, inversed_terms in zip(data, inversed_data): terms = np.sort(np.unique(analyze(doc))) inversed_terms = np.sort(np.unique(inversed_terms)) assert_array_equal(terms, inversed_terms) assert sparse.issparse(transformed_data) assert transformed_data.format == "csr" # Test that inverse_transform also works with numpy arrays and # scipy transformed_data2 = transformed_data.toarray() inversed_data2 = vectorizer.inverse_transform(transformed_data2) for terms, terms2 in zip(inversed_data, inversed_data2): assert_array_equal(np.sort(terms), np.sort(terms2)) # Check that inverse_transform also works on non CSR sparse data: transformed_data3 = transformed_data.tocsc() inversed_data3 = vectorizer.inverse_transform(transformed_data3) for terms, terms3 in zip(inversed_data, inversed_data3): assert_array_equal(np.sort(terms), np.sort(terms3)) def test_count_vectorizer_pipeline_grid_selection(): # raw documents data = JUNK_FOOD_DOCS + NOTJUNK_FOOD_DOCS # label junk food as -1, the others as +1 target = [-1] * len(JUNK_FOOD_DOCS) + [1] * len(NOTJUNK_FOOD_DOCS) # split the dataset for model development and final evaluation train_data, test_data, target_train, target_test = train_test_split( data, target, test_size=0.2, random_state=0 ) pipeline = Pipeline([("vect", CountVectorizer()), ("svc", LinearSVC())]) parameters = { "vect__ngram_range": [(1, 1), (1, 2)], "svc__loss": ("hinge", "squared_hinge"), } # find the best parameters for both the feature extraction and the # classifier grid_search = GridSearchCV(pipeline, parameters, n_jobs=1, cv=3) # Check that the best model found by grid search is 100% correct on the # held out evaluation set. pred = grid_search.fit(train_data, target_train).predict(test_data) assert_array_equal(pred, target_test) # on this toy dataset bigram representation which is used in the last of # the grid_search is considered the best estimator since they all converge # to 100% accuracy models assert grid_search.best_score_ == 1.0 best_vectorizer = grid_search.best_estimator_.named_steps["vect"] assert best_vectorizer.ngram_range == (1, 1) def test_vectorizer_pipeline_grid_selection(): # raw documents data = JUNK_FOOD_DOCS + NOTJUNK_FOOD_DOCS # label junk food as -1, the others as +1 target = [-1] * len(JUNK_FOOD_DOCS) + [1] * len(NOTJUNK_FOOD_DOCS) # split the dataset for model development and final evaluation train_data, test_data, target_train, target_test = train_test_split( data, target, test_size=0.1, random_state=0 ) pipeline = Pipeline([("vect", TfidfVectorizer()), ("svc", LinearSVC())]) parameters = { "vect__ngram_range": [(1, 1), (1, 2)], "vect__norm": ("l1", "l2"), "svc__loss": ("hinge", "squared_hinge"), } # find the best parameters for both the feature extraction and the # classifier grid_search = GridSearchCV(pipeline, parameters, n_jobs=1) # Check that the best model found by grid search is 100% correct on the # held out evaluation set. pred = grid_search.fit(train_data, target_train).predict(test_data) assert_array_equal(pred, target_test) # on this toy dataset bigram representation which is used in the last of # the grid_search is considered the best estimator since they all converge # to 100% accuracy models assert grid_search.best_score_ == 1.0 best_vectorizer = grid_search.best_estimator_.named_steps["vect"] assert best_vectorizer.ngram_range == (1, 1) assert best_vectorizer.norm == "l2" assert not best_vectorizer.fixed_vocabulary_ def test_vectorizer_pipeline_cross_validation(): # raw documents data = JUNK_FOOD_DOCS + NOTJUNK_FOOD_DOCS # label junk food as -1, the others as +1 target = [-1] * len(JUNK_FOOD_DOCS) + [1] * len(NOTJUNK_FOOD_DOCS) pipeline = Pipeline([("vect", TfidfVectorizer()), ("svc", LinearSVC())]) cv_scores = cross_val_score(pipeline, data, target, cv=3) assert_array_equal(cv_scores, [1.0, 1.0, 1.0]) @fails_if_pypy def test_vectorizer_unicode(): # tests that the count vectorizer works with cyrillic. document = ( "Машинное обучение — обширный подраздел искусственного " "интеллекта, изучающий методы построения алгоритмов, " "способных обучаться." ) vect = CountVectorizer() X_counted = vect.fit_transform([document]) assert X_counted.shape == (1, 12) vect = HashingVectorizer(norm=None, alternate_sign=False) X_hashed = vect.transform([document]) assert X_hashed.shape == (1, 2**20) # No collisions on such a small dataset assert X_counted.nnz == X_hashed.nnz # When norm is None and not alternate_sign, the tokens are counted up to # collisions assert_array_equal(np.sort(X_counted.data), np.sort(X_hashed.data)) def test_tfidf_vectorizer_with_fixed_vocabulary(): # non regression smoke test for inheritance issues vocabulary = ["pizza", "celeri"] vect = TfidfVectorizer(vocabulary=vocabulary) X_1 = vect.fit_transform(ALL_FOOD_DOCS) X_2 = vect.transform(ALL_FOOD_DOCS) assert_array_almost_equal(X_1.toarray(), X_2.toarray()) assert vect.fixed_vocabulary_ def test_pickling_vectorizer(): instances = [ HashingVectorizer(), HashingVectorizer(norm="l1"), HashingVectorizer(binary=True), HashingVectorizer(ngram_range=(1, 2)), CountVectorizer(), CountVectorizer(preprocessor=strip_tags), CountVectorizer(analyzer=lazy_analyze), CountVectorizer(preprocessor=strip_tags).fit(JUNK_FOOD_DOCS), CountVectorizer(strip_accents=strip_eacute).fit(JUNK_FOOD_DOCS), TfidfVectorizer(), TfidfVectorizer(analyzer=lazy_analyze), TfidfVectorizer().fit(JUNK_FOOD_DOCS), ] for orig in instances: s = pickle.dumps(orig) copy = pickle.loads(s) assert type(copy) == orig.__class__ assert copy.get_params() == orig.get_params() if IS_PYPY and isinstance(orig, HashingVectorizer): continue else: assert_allclose_dense_sparse( copy.fit_transform(JUNK_FOOD_DOCS), orig.fit_transform(JUNK_FOOD_DOCS), ) @pytest.mark.parametrize( "factory", [ CountVectorizer.build_analyzer, CountVectorizer.build_preprocessor, CountVectorizer.build_tokenizer, ], ) def test_pickling_built_processors(factory): """Tokenizers cannot be pickled https://github.com/scikit-learn/scikit-learn/issues/12833 """ vec = CountVectorizer() function = factory(vec) text = "J'ai mangé du kangourou ce midi, c'était pas très bon." roundtripped_function = pickle.loads(pickle.dumps(function)) expected = function(text) result = roundtripped_function(text) assert result == expected def test_countvectorizer_vocab_sets_when_pickling(): # ensure that vocabulary of type set is coerced to a list to # preserve iteration ordering after deserialization rng = np.random.RandomState(0) vocab_words = np.array( [ "beer", "burger", "celeri", "coke", "pizza", "salad", "sparkling", "tomato", "water", ] ) for x in range(0, 100): vocab_set = set(rng.choice(vocab_words, size=5, replace=False)) cv = CountVectorizer(vocabulary=vocab_set) unpickled_cv = pickle.loads(pickle.dumps(cv)) cv.fit(ALL_FOOD_DOCS) unpickled_cv.fit(ALL_FOOD_DOCS) assert_array_equal( cv.get_feature_names_out(), unpickled_cv.get_feature_names_out() ) def test_countvectorizer_vocab_dicts_when_pickling(): rng = np.random.RandomState(0) vocab_words = np.array( [ "beer", "burger", "celeri", "coke", "pizza", "salad", "sparkling", "tomato", "water", ] ) for x in range(0, 100): vocab_dict = dict() words = rng.choice(vocab_words, size=5, replace=False) for y in range(0, 5): vocab_dict[words[y]] = y cv = CountVectorizer(vocabulary=vocab_dict) unpickled_cv = pickle.loads(pickle.dumps(cv)) cv.fit(ALL_FOOD_DOCS) unpickled_cv.fit(ALL_FOOD_DOCS) assert_array_equal( cv.get_feature_names_out(), unpickled_cv.get_feature_names_out() ) def test_stop_words_removal(): # Ensure that deleting the stop_words_ attribute doesn't affect transform fitted_vectorizers = ( TfidfVectorizer().fit(JUNK_FOOD_DOCS), CountVectorizer(preprocessor=strip_tags).fit(JUNK_FOOD_DOCS), CountVectorizer(strip_accents=strip_eacute).fit(JUNK_FOOD_DOCS), ) for vect in fitted_vectorizers: vect_transform = vect.transform(JUNK_FOOD_DOCS).toarray() vect.stop_words_ = None stop_None_transform = vect.transform(JUNK_FOOD_DOCS).toarray() delattr(vect, "stop_words_") stop_del_transform = vect.transform(JUNK_FOOD_DOCS).toarray() assert_array_equal(stop_None_transform, vect_transform) assert_array_equal(stop_del_transform, vect_transform) def test_pickling_transformer(): X = CountVectorizer().fit_transform(JUNK_FOOD_DOCS) orig = TfidfTransformer().fit(X) s = pickle.dumps(orig) copy = pickle.loads(s) assert type(copy) == orig.__class__ assert_array_equal(copy.fit_transform(X).toarray(), orig.fit_transform(X).toarray()) def test_transformer_idf_setter(): X = CountVectorizer().fit_transform(JUNK_FOOD_DOCS) orig = TfidfTransformer().fit(X) copy = TfidfTransformer() copy.idf_ = orig.idf_ assert_array_equal(copy.transform(X).toarray(), orig.transform(X).toarray()) def test_tfidf_vectorizer_setter(): orig = TfidfVectorizer(use_idf=True) orig.fit(JUNK_FOOD_DOCS) copy = TfidfVectorizer(vocabulary=orig.vocabulary_, use_idf=True) copy.idf_ = orig.idf_ assert_array_equal( copy.transform(JUNK_FOOD_DOCS).toarray(), orig.transform(JUNK_FOOD_DOCS).toarray(), ) # `idf_` cannot be set with `use_idf=False` copy = TfidfVectorizer(vocabulary=orig.vocabulary_, use_idf=False) err_msg = "`idf_` cannot be set when `user_idf=False`." with pytest.raises(ValueError, match=err_msg): copy.idf_ = orig.idf_ def test_tfidfvectorizer_invalid_idf_attr(): vect = TfidfVectorizer(use_idf=True) vect.fit(JUNK_FOOD_DOCS) copy = TfidfVectorizer(vocabulary=vect.vocabulary_, use_idf=True) expected_idf_len = len(vect.idf_) invalid_idf = [1.0] * (expected_idf_len + 1) with pytest.raises(ValueError): setattr(copy, "idf_", invalid_idf) def test_non_unique_vocab(): vocab = ["a", "b", "c", "a", "a"] vect = CountVectorizer(vocabulary=vocab) with pytest.raises(ValueError): vect.fit([]) @fails_if_pypy def test_hashingvectorizer_nan_in_docs(): # np.nan can appear when using pandas to load text fields from a csv file # with missing values. message = "np.nan is an invalid document, expected byte or unicode string." exception = ValueError def func(): hv = HashingVectorizer() hv.fit_transform(["hello world", np.nan, "hello hello"]) with pytest.raises(exception, match=message): func() def test_tfidfvectorizer_binary(): # Non-regression test: TfidfVectorizer used to ignore its "binary" param. v = TfidfVectorizer(binary=True, use_idf=False, norm=None) assert v.binary X = v.fit_transform(["hello world", "hello hello"]).toarray() assert_array_equal(X.ravel(), [1, 1, 1, 0]) X2 = v.transform(["hello world", "hello hello"]).toarray() assert_array_equal(X2.ravel(), [1, 1, 1, 0]) def test_tfidfvectorizer_export_idf(): vect = TfidfVectorizer(use_idf=True) vect.fit(JUNK_FOOD_DOCS) assert_array_almost_equal(vect.idf_, vect._tfidf.idf_) def test_vectorizer_vocab_clone(): vect_vocab = TfidfVectorizer(vocabulary=["the"]) vect_vocab_clone = clone(vect_vocab) vect_vocab.fit(ALL_FOOD_DOCS) vect_vocab_clone.fit(ALL_FOOD_DOCS) assert vect_vocab_clone.vocabulary_ == vect_vocab.vocabulary_ @pytest.mark.parametrize( "Vectorizer", (CountVectorizer, TfidfVectorizer, HashingVectorizer) ) def test_vectorizer_string_object_as_input(Vectorizer): message = "Iterable over raw text documents expected, string object received." vec = Vectorizer() with pytest.raises(ValueError, match=message): vec.fit_transform("hello world!") with pytest.raises(ValueError, match=message): vec.fit("hello world!") vec.fit(["some text", "some other text"]) with pytest.raises(ValueError, match=message): vec.transform("hello world!") @pytest.mark.parametrize("X_dtype", [np.float32, np.float64]) def test_tfidf_transformer_type(X_dtype): X = sparse.rand(10, 20000, dtype=X_dtype, random_state=42) X_trans = TfidfTransformer().fit_transform(X) assert X_trans.dtype == X.dtype def test_tfidf_transformer_sparse(): X = sparse.rand(10, 20000, dtype=np.float64, random_state=42) X_csc = sparse.csc_matrix(X) X_csr = sparse.csr_matrix(X) X_trans_csc = TfidfTransformer().fit_transform(X_csc) X_trans_csr = TfidfTransformer().fit_transform(X_csr) assert_allclose_dense_sparse(X_trans_csc, X_trans_csr) assert X_trans_csc.format == X_trans_csr.format @pytest.mark.parametrize( "vectorizer_dtype, output_dtype, warning_expected", [ (np.int32, np.float64, True), (np.int64, np.float64, True), (np.float32, np.float32, False), (np.float64, np.float64, False), ], ) def test_tfidf_vectorizer_type(vectorizer_dtype, output_dtype, warning_expected): X = np.array(["numpy", "scipy", "sklearn"]) vectorizer = TfidfVectorizer(dtype=vectorizer_dtype) warning_msg_match = "'dtype' should be used." if warning_expected: with pytest.warns(UserWarning, match=warning_msg_match): X_idf = vectorizer.fit_transform(X) else: with warnings.catch_warnings(): warnings.simplefilter("error", UserWarning) X_idf = vectorizer.fit_transform(X) assert X_idf.dtype == output_dtype @pytest.mark.parametrize( "vec", [ HashingVectorizer(ngram_range=(2, 1)), CountVectorizer(ngram_range=(2, 1)), TfidfVectorizer(ngram_range=(2, 1)), ], ) def test_vectorizers_invalid_ngram_range(vec): # vectorizers could be initialized with invalid ngram range # test for raising error message invalid_range = vec.ngram_range message = re.escape( f"Invalid value for ngram_range={invalid_range} " "lower boundary larger than the upper boundary." ) if isinstance(vec, HashingVectorizer) and IS_PYPY: pytest.xfail(reason="HashingVectorizer is not supported on PyPy") with pytest.raises(ValueError, match=message): vec.fit(["good news everyone"]) with pytest.raises(ValueError, match=message): vec.fit_transform(["good news everyone"]) if isinstance(vec, HashingVectorizer): with pytest.raises(ValueError, match=message): vec.transform(["good news everyone"]) def _check_stop_words_consistency(estimator): stop_words = estimator.get_stop_words() tokenize = estimator.build_tokenizer() preprocess = estimator.build_preprocessor() return estimator._check_stop_words_consistency(stop_words, preprocess, tokenize) @fails_if_pypy def test_vectorizer_stop_words_inconsistent(): lstr = r"\['and', 'll', 've'\]" message = ( "Your stop_words may be inconsistent with your " "preprocessing. Tokenizing the stop words generated " "tokens %s not in stop_words." % lstr ) for vec in [CountVectorizer(), TfidfVectorizer(), HashingVectorizer()]: vec.set_params(stop_words=["you've", "you", "you'll", "AND"]) with pytest.warns(UserWarning, match=message): vec.fit_transform(["hello world"]) # reset stop word validation del vec._stop_words_id assert _check_stop_words_consistency(vec) is False # Only one warning per stop list with warnings.catch_warnings(): warnings.simplefilter("error", UserWarning) vec.fit_transform(["hello world"]) assert _check_stop_words_consistency(vec) is None # Test caching of inconsistency assessment vec.set_params(stop_words=["you've", "you", "you'll", "blah", "AND"]) with pytest.warns(UserWarning, match=message): vec.fit_transform(["hello world"]) @skip_if_32bit def test_countvectorizer_sort_features_64bit_sparse_indices(): """ Check that CountVectorizer._sort_features preserves the dtype of its sparse feature matrix. This test is skipped on 32bit platforms, see: https://github.com/scikit-learn/scikit-learn/pull/11295 for more details. """ X = sparse.csr_matrix((5, 5), dtype=np.int64) # force indices and indptr to int64. INDICES_DTYPE = np.int64 X.indices = X.indices.astype(INDICES_DTYPE) X.indptr = X.indptr.astype(INDICES_DTYPE) vocabulary = {"scikit-learn": 0, "is": 1, "great!": 2} Xs = CountVectorizer()._sort_features(X, vocabulary) assert INDICES_DTYPE == Xs.indices.dtype @fails_if_pypy @pytest.mark.parametrize( "Estimator", [CountVectorizer, TfidfVectorizer, HashingVectorizer] ) def test_stop_word_validation_custom_preprocessor(Estimator): data = [{"text": "some text"}] vec = Estimator() assert _check_stop_words_consistency(vec) is True vec = Estimator(preprocessor=lambda x: x["text"], stop_words=["and"]) assert _check_stop_words_consistency(vec) == "error" # checks are cached assert _check_stop_words_consistency(vec) is None vec.fit_transform(data) class CustomEstimator(Estimator): def build_preprocessor(self): return lambda x: x["text"] vec = CustomEstimator(stop_words=["and"]) assert _check_stop_words_consistency(vec) == "error" vec = Estimator( tokenizer=lambda doc: re.compile(r"\w{1,}").findall(doc), stop_words=["and"] ) assert _check_stop_words_consistency(vec) is True @pytest.mark.parametrize( "Estimator", [CountVectorizer, TfidfVectorizer, HashingVectorizer] ) @pytest.mark.parametrize( "input_type, err_type, err_msg", [ ("filename", FileNotFoundError, ""), ("file", AttributeError, "'str' object has no attribute 'read'"), ], ) def test_callable_analyzer_error(Estimator, input_type, err_type, err_msg): if issubclass(Estimator, HashingVectorizer) and IS_PYPY: pytest.xfail("HashingVectorizer is not supported on PyPy") data = ["this is text, not file or filename"] with pytest.raises(err_type, match=err_msg): Estimator(analyzer=lambda x: x.split(), input=input_type).fit_transform(data) @pytest.mark.parametrize( "Estimator", [ CountVectorizer, TfidfVectorizer, pytest.param(HashingVectorizer, marks=fails_if_pypy), ], ) @pytest.mark.parametrize( "analyzer", [lambda doc: open(doc, "r"), lambda doc: doc.read()] ) @pytest.mark.parametrize("input_type", ["file", "filename"]) def test_callable_analyzer_change_behavior(Estimator, analyzer, input_type): data = ["this is text, not file or filename"] with pytest.raises((FileNotFoundError, AttributeError)): Estimator(analyzer=analyzer, input=input_type).fit_transform(data) @pytest.mark.parametrize( "Estimator", [CountVectorizer, TfidfVectorizer, HashingVectorizer] ) def test_callable_analyzer_reraise_error(tmpdir, Estimator): # check if a custom exception from the analyzer is shown to the user def analyzer(doc): raise Exception("testing") if issubclass(Estimator, HashingVectorizer) and IS_PYPY: pytest.xfail("HashingVectorizer is not supported on PyPy") f = tmpdir.join("file.txt") f.write("sample content\n") with pytest.raises(Exception, match="testing"): Estimator(analyzer=analyzer, input="file").fit_transform([f]) @pytest.mark.parametrize( "Vectorizer", [CountVectorizer, HashingVectorizer, TfidfVectorizer] ) @pytest.mark.parametrize( "stop_words, tokenizer, preprocessor, ngram_range, token_pattern," "analyzer, unused_name, ovrd_name, ovrd_msg", [ ( ["you've", "you'll"], None, None, (1, 1), None, "char", "'stop_words'", "'analyzer'", "!= 'word'", ), ( None, lambda s: s.split(), None, (1, 1), None, "char", "'tokenizer'", "'analyzer'", "!= 'word'", ), ( None, lambda s: s.split(), None, (1, 1), r"\w+", "word", "'token_pattern'", "'tokenizer'", "is not None", ), ( None, None, lambda s: s.upper(), (1, 1), r"\w+", lambda s: s.upper(), "'preprocessor'", "'analyzer'", "is callable", ), ( None, None, None, (1, 2), None, lambda s: s.upper(), "'ngram_range'", "'analyzer'", "is callable", ), ( None, None, None, (1, 1), r"\w+", "char", "'token_pattern'", "'analyzer'", "!= 'word'", ), ], ) def test_unused_parameters_warn( Vectorizer, stop_words, tokenizer, preprocessor, ngram_range, token_pattern, analyzer, unused_name, ovrd_name, ovrd_msg, ): train_data = JUNK_FOOD_DOCS # setting parameter and checking for corresponding warning messages vect = Vectorizer() vect.set_params( stop_words=stop_words, tokenizer=tokenizer, preprocessor=preprocessor, ngram_range=ngram_range, token_pattern=token_pattern, analyzer=analyzer, ) msg = "The parameter %s will not be used since %s %s" % ( unused_name, ovrd_name, ovrd_msg, ) with pytest.warns(UserWarning, match=msg): vect.fit(train_data) @pytest.mark.parametrize( "Vectorizer, X", ( (HashingVectorizer, [{"foo": 1, "bar": 2}, {"foo": 3, "baz": 1}]), (CountVectorizer, JUNK_FOOD_DOCS), ), ) def test_n_features_in(Vectorizer, X): # For vectorizers, n_features_in_ does not make sense vectorizer = Vectorizer() assert not hasattr(vectorizer, "n_features_in_") vectorizer.fit(X) assert not hasattr(vectorizer, "n_features_in_") def test_tie_breaking_sample_order_invariance(): # Checks the sample order invariance when setting max_features # non-regression test for #17939 vec = CountVectorizer(max_features=1) vocab1 = vec.fit(["hello", "world"]).vocabulary_ vocab2 = vec.fit(["world", "hello"]).vocabulary_ assert vocab1 == vocab2 @fails_if_pypy def test_nonnegative_hashing_vectorizer_result_indices(): # add test for pr 19035 hashing = HashingVectorizer(n_features=1000000, ngram_range=(2, 3)) indices = hashing.transform(["22pcs efuture"]).indices assert indices[0] >= 0 @pytest.mark.parametrize( "Estimator", [CountVectorizer, TfidfVectorizer, TfidfTransformer, HashingVectorizer] ) def test_vectorizers_do_not_have_set_output(Estimator): """Check that vectorizers do not define set_output.""" est = Estimator() assert not hasattr(est, "set_output")