366 lines
15 KiB
Python
366 lines
15 KiB
Python
|
import numpy as np
|
||
|
|
import pytest
|
||
|
|
|
||
|
|
from pandas.core.dtypes.concat import union_categoricals
|
||
|
|
|
||
|
|
import pandas as pd
|
||
|
|
from pandas import (
|
||
|
|
Categorical,
|
||
|
|
CategoricalIndex,
|
||
|
|
Series,
|
||
|
|
)
|
||
|
|
import pandas._testing as tm
|
||
|
|
|
||
|
|
|
||
|
|
class TestUnionCategoricals:
|
||
|
|
@pytest.mark.parametrize(
|
||
|
|
"a, b, combined",
|
||
|
|
[
|
||
|
|
(list("abc"), list("abd"), list("abcabd")),
|
||
|
|
([0, 1, 2], [2, 3, 4], [0, 1, 2, 2, 3, 4]),
|
||
|
|
([0, 1.2, 2], [2, 3.4, 4], [0, 1.2, 2, 2, 3.4, 4]),
|
||
|
|
(
|
||
|
|
["b", "b", np.nan, "a"],
|
||
|
|
["a", np.nan, "c"],
|
||
|
|
["b", "b", np.nan, "a", "a", np.nan, "c"],
|
||
|
|
),
|
||
|
|
(
|
||
|
|
pd.date_range("2014-01-01", "2014-01-05"),
|
||
|
|
pd.date_range("2014-01-06", "2014-01-07"),
|
||
|
|
pd.date_range("2014-01-01", "2014-01-07"),
|
||
|
|
),
|
||
|
|
(
|
||
|
|
pd.date_range("2014-01-01", "2014-01-05", tz="US/Central"),
|
||
|
|
pd.date_range("2014-01-06", "2014-01-07", tz="US/Central"),
|
||
|
|
pd.date_range("2014-01-01", "2014-01-07", tz="US/Central"),
|
||
|
|
),
|
||
|
|
(
|
||
|
|
pd.period_range("2014-01-01", "2014-01-05"),
|
||
|
|
pd.period_range("2014-01-06", "2014-01-07"),
|
||
|
|
pd.period_range("2014-01-01", "2014-01-07"),
|
||
|
|
),
|
||
|
|
],
|
||
|
|
)
|
||
|
|
@pytest.mark.parametrize("box", [Categorical, CategoricalIndex, Series])
|
||
|
|
def test_union_categorical(self, a, b, combined, box):
|
||
|
|
# GH 13361
|
||
|
|
result = union_categoricals([box(Categorical(a)), box(Categorical(b))])
|
||
|
|
expected = Categorical(combined)
|
||
|
|
tm.assert_categorical_equal(result, expected)
|
||
|
|
|
||
|
|
def test_union_categorical_ordered_appearance(self):
|
||
|
|
# new categories ordered by appearance
|
||
|
|
s = Categorical(["x", "y", "z"])
|
||
|
|
s2 = Categorical(["a", "b", "c"])
|
||
|
|
result = union_categoricals([s, s2])
|
||
|
|
expected = Categorical(
|
||
|
|
["x", "y", "z", "a", "b", "c"], categories=["x", "y", "z", "a", "b", "c"]
|
||
|
|
)
|
||
|
|
tm.assert_categorical_equal(result, expected)
|
||
|
|
|
||
|
|
def test_union_categorical_ordered_true(self):
|
||
|
|
s = Categorical([0, 1.2, 2], ordered=True)
|
||
|
|
s2 = Categorical([0, 1.2, 2], ordered=True)
|
||
|
|
result = union_categoricals([s, s2])
|
||
|
|
expected = Categorical([0, 1.2, 2, 0, 1.2, 2], ordered=True)
|
||
|
|
tm.assert_categorical_equal(result, expected)
|
||
|
|
|
||
|
|
def test_union_categorical_match_types(self):
|
||
|
|
# must exactly match types
|
||
|
|
s = Categorical([0, 1.2, 2])
|
||
|
|
s2 = Categorical([2, 3, 4])
|
||
|
|
msg = "dtype of categories must be the same"
|
||
|
|
with pytest.raises(TypeError, match=msg):
|
||
|
|
union_categoricals([s, s2])
|
||
|
|
|
||
|
|
def test_union_categorical_empty(self):
|
||
|
|
msg = "No Categoricals to union"
|
||
|
|
with pytest.raises(ValueError, match=msg):
|
||
|
|
union_categoricals([])
|
||
|
|
|
||
|
|
def test_union_categoricals_nan(self):
|
||
|
|
# GH 13759
|
||
|
|
res = union_categoricals(
|
||
|
|
[Categorical([1, 2, np.nan]), Categorical([3, 2, np.nan])]
|
||
|
|
)
|
||
|
|
exp = Categorical([1, 2, np.nan, 3, 2, np.nan])
|
||
|
|
tm.assert_categorical_equal(res, exp)
|
||
|
|
|
||
|
|
res = union_categoricals(
|
||
|
|
[Categorical(["A", "B"]), Categorical(["B", "B", np.nan])]
|
||
|
|
)
|
||
|
|
exp = Categorical(["A", "B", "B", "B", np.nan])
|
||
|
|
tm.assert_categorical_equal(res, exp)
|
||
|
|
|
||
|
|
val1 = [pd.Timestamp("2011-01-01"), pd.Timestamp("2011-03-01"), pd.NaT]
|
||
|
|
val2 = [pd.NaT, pd.Timestamp("2011-01-01"), pd.Timestamp("2011-02-01")]
|
||
|
|
|
||
|
|
res = union_categoricals([Categorical(val1), Categorical(val2)])
|
||
|
|
exp = Categorical(
|
||
|
|
val1 + val2,
|
||
|
|
categories=[
|
||
|
|
pd.Timestamp("2011-01-01"),
|
||
|
|
pd.Timestamp("2011-03-01"),
|
||
|
|
pd.Timestamp("2011-02-01"),
|
||
|
|
],
|
||
|
|
)
|
||
|
|
tm.assert_categorical_equal(res, exp)
|
||
|
|
|
||
|
|
# all NaN
|
||
|
|
res = union_categoricals(
|
||
|
|
[
|
||
|
|
Categorical(np.array([np.nan, np.nan], dtype=object)),
|
||
|
|
Categorical(["X"], categories=pd.Index(["X"], dtype=object)),
|
||
|
|
]
|
||
|
|
)
|
||
|
|
exp = Categorical([np.nan, np.nan, "X"])
|
||
|
|
tm.assert_categorical_equal(res, exp)
|
||
|
|
|
||
|
|
res = union_categoricals(
|
||
|
|
[Categorical([np.nan, np.nan]), Categorical([np.nan, np.nan])]
|
||
|
|
)
|
||
|
|
exp = Categorical([np.nan, np.nan, np.nan, np.nan])
|
||
|
|
tm.assert_categorical_equal(res, exp)
|
||
|
|
|
||
|
|
@pytest.mark.parametrize("val", [[], ["1"]])
|
||
|
|
def test_union_categoricals_empty(self, val, request, using_infer_string):
|
||
|
|
# GH 13759
|
||
|
|
if using_infer_string and val == ["1"]:
|
||
|
|
request.applymarker(pytest.mark.xfail("object and strings dont match"))
|
||
|
|
res = union_categoricals([Categorical([]), Categorical(val)])
|
||
|
|
exp = Categorical(val)
|
||
|
|
tm.assert_categorical_equal(res, exp)
|
||
|
|
|
||
|
|
def test_union_categorical_same_category(self):
|
||
|
|
# check fastpath
|
||
|
|
c1 = Categorical([1, 2, 3, 4], categories=[1, 2, 3, 4])
|
||
|
|
c2 = Categorical([3, 2, 1, np.nan], categories=[1, 2, 3, 4])
|
||
|
|
res = union_categoricals([c1, c2])
|
||
|
|
exp = Categorical([1, 2, 3, 4, 3, 2, 1, np.nan], categories=[1, 2, 3, 4])
|
||
|
|
tm.assert_categorical_equal(res, exp)
|
||
|
|
|
||
|
|
def test_union_categorical_same_category_str(self):
|
||
|
|
c1 = Categorical(["z", "z", "z"], categories=["x", "y", "z"])
|
||
|
|
c2 = Categorical(["x", "x", "x"], categories=["x", "y", "z"])
|
||
|
|
res = union_categoricals([c1, c2])
|
||
|
|
exp = Categorical(["z", "z", "z", "x", "x", "x"], categories=["x", "y", "z"])
|
||
|
|
tm.assert_categorical_equal(res, exp)
|
||
|
|
|
||
|
|
def test_union_categorical_same_categories_different_order(self):
|
||
|
|
# https://github.com/pandas-dev/pandas/issues/19096
|
||
|
|
c1 = Categorical(["a", "b", "c"], categories=["a", "b", "c"])
|
||
|
|
c2 = Categorical(["a", "b", "c"], categories=["b", "a", "c"])
|
||
|
|
result = union_categoricals([c1, c2])
|
||
|
|
expected = Categorical(
|
||
|
|
["a", "b", "c", "a", "b", "c"], categories=["a", "b", "c"]
|
||
|
|
)
|
||
|
|
tm.assert_categorical_equal(result, expected)
|
||
|
|
|
||
|
|
def test_union_categoricals_ordered(self):
|
||
|
|
c1 = Categorical([1, 2, 3], ordered=True)
|
||
|
|
c2 = Categorical([1, 2, 3], ordered=False)
|
||
|
|
|
||
|
|
msg = "Categorical.ordered must be the same"
|
||
|
|
with pytest.raises(TypeError, match=msg):
|
||
|
|
union_categoricals([c1, c2])
|
||
|
|
|
||
|
|
res = union_categoricals([c1, c1])
|
||
|
|
exp = Categorical([1, 2, 3, 1, 2, 3], ordered=True)
|
||
|
|
tm.assert_categorical_equal(res, exp)
|
||
|
|
|
||
|
|
c1 = Categorical([1, 2, 3, np.nan], ordered=True)
|
||
|
|
c2 = Categorical([3, 2], categories=[1, 2, 3], ordered=True)
|
||
|
|
|
||
|
|
res = union_categoricals([c1, c2])
|
||
|
|
exp = Categorical([1, 2, 3, np.nan, 3, 2], ordered=True)
|
||
|
|
tm.assert_categorical_equal(res, exp)
|
||
|
|
|
||
|
|
c1 = Categorical([1, 2, 3], ordered=True)
|
||
|
|
c2 = Categorical([1, 2, 3], categories=[3, 2, 1], ordered=True)
|
||
|
|
|
||
|
|
msg = "to union ordered Categoricals, all categories must be the same"
|
||
|
|
with pytest.raises(TypeError, match=msg):
|
||
|
|
union_categoricals([c1, c2])
|
||
|
|
|
||
|
|
def test_union_categoricals_ignore_order(self):
|
||
|
|
# GH 15219
|
||
|
|
c1 = Categorical([1, 2, 3], ordered=True)
|
||
|
|
c2 = Categorical([1, 2, 3], ordered=False)
|
||
|
|
|
||
|
|
res = union_categoricals([c1, c2], ignore_order=True)
|
||
|
|
exp = Categorical([1, 2, 3, 1, 2, 3])
|
||
|
|
tm.assert_categorical_equal(res, exp)
|
||
|
|
|
||
|
|
msg = "Categorical.ordered must be the same"
|
||
|
|
with pytest.raises(TypeError, match=msg):
|
||
|
|
union_categoricals([c1, c2], ignore_order=False)
|
||
|
|
|
||
|
|
res = union_categoricals([c1, c1], ignore_order=True)
|
||
|
|
exp = Categorical([1, 2, 3, 1, 2, 3])
|
||
|
|
tm.assert_categorical_equal(res, exp)
|
||
|
|
|
||
|
|
res = union_categoricals([c1, c1], ignore_order=False)
|
||
|
|
exp = Categorical([1, 2, 3, 1, 2, 3], categories=[1, 2, 3], ordered=True)
|
||
|
|
tm.assert_categorical_equal(res, exp)
|
||
|
|
|
||
|
|
c1 = Categorical([1, 2, 3, np.nan], ordered=True)
|
||
|
|
c2 = Categorical([3, 2], categories=[1, 2, 3], ordered=True)
|
||
|
|
|
||
|
|
res = union_categoricals([c1, c2], ignore_order=True)
|
||
|
|
exp = Categorical([1, 2, 3, np.nan, 3, 2])
|
||
|
|
tm.assert_categorical_equal(res, exp)
|
||
|
|
|
||
|
|
c1 = Categorical([1, 2, 3], ordered=True)
|
||
|
|
c2 = Categorical([1, 2, 3], categories=[3, 2, 1], ordered=True)
|
||
|
|
|
||
|
|
res = union_categoricals([c1, c2], ignore_order=True)
|
||
|
|
exp = Categorical([1, 2, 3, 1, 2, 3])
|
||
|
|
tm.assert_categorical_equal(res, exp)
|
||
|
|
|
||
|
|
res = union_categoricals([c2, c1], ignore_order=True, sort_categories=True)
|
||
|
|
exp = Categorical([1, 2, 3, 1, 2, 3], categories=[1, 2, 3])
|
||
|
|
tm.assert_categorical_equal(res, exp)
|
||
|
|
|
||
|
|
c1 = Categorical([1, 2, 3], ordered=True)
|
||
|
|
c2 = Categorical([4, 5, 6], ordered=True)
|
||
|
|
result = union_categoricals([c1, c2], ignore_order=True)
|
||
|
|
expected = Categorical([1, 2, 3, 4, 5, 6])
|
||
|
|
tm.assert_categorical_equal(result, expected)
|
||
|
|
|
||
|
|
msg = "to union ordered Categoricals, all categories must be the same"
|
||
|
|
with pytest.raises(TypeError, match=msg):
|
||
|
|
union_categoricals([c1, c2], ignore_order=False)
|
||
|
|
|
||
|
|
with pytest.raises(TypeError, match=msg):
|
||
|
|
union_categoricals([c1, c2])
|
||
|
|
|
||
|
|
def test_union_categoricals_sort(self):
|
||
|
|
# GH 13846
|
||
|
|
c1 = Categorical(["x", "y", "z"])
|
||
|
|
c2 = Categorical(["a", "b", "c"])
|
||
|
|
result = union_categoricals([c1, c2], sort_categories=True)
|
||
|
|
expected = Categorical(
|
||
|
|
["x", "y", "z", "a", "b", "c"], categories=["a", "b", "c", "x", "y", "z"]
|
||
|
|
)
|
||
|
|
tm.assert_categorical_equal(result, expected)
|
||
|
|
|
||
|
|
# fastpath
|
||
|
|
c1 = Categorical(["a", "b"], categories=["b", "a", "c"])
|
||
|
|
c2 = Categorical(["b", "c"], categories=["b", "a", "c"])
|
||
|
|
result = union_categoricals([c1, c2], sort_categories=True)
|
||
|
|
expected = Categorical(["a", "b", "b", "c"], categories=["a", "b", "c"])
|
||
|
|
tm.assert_categorical_equal(result, expected)
|
||
|
|
|
||
|
|
c1 = Categorical(["a", "b"], categories=["c", "a", "b"])
|
||
|
|
c2 = Categorical(["b", "c"], categories=["c", "a", "b"])
|
||
|
|
result = union_categoricals([c1, c2], sort_categories=True)
|
||
|
|
expected = Categorical(["a", "b", "b", "c"], categories=["a", "b", "c"])
|
||
|
|
tm.assert_categorical_equal(result, expected)
|
||
|
|
|
||
|
|
# fastpath - skip resort
|
||
|
|
c1 = Categorical(["a", "b"], categories=["a", "b", "c"])
|
||
|
|
c2 = Categorical(["b", "c"], categories=["a", "b", "c"])
|
||
|
|
result = union_categoricals([c1, c2], sort_categories=True)
|
||
|
|
expected = Categorical(["a", "b", "b", "c"], categories=["a", "b", "c"])
|
||
|
|
tm.assert_categorical_equal(result, expected)
|
||
|
|
|
||
|
|
c1 = Categorical(["x", np.nan])
|
||
|
|
c2 = Categorical([np.nan, "b"])
|
||
|
|
result = union_categoricals([c1, c2], sort_categories=True)
|
||
|
|
expected = Categorical(["x", np.nan, np.nan, "b"], categories=["b", "x"])
|
||
|
|
tm.assert_categorical_equal(result, expected)
|
||
|
|
|
||
|
|
c1 = Categorical([np.nan])
|
||
|
|
c2 = Categorical([np.nan])
|
||
|
|
result = union_categoricals([c1, c2], sort_categories=True)
|
||
|
|
expected = Categorical([np.nan, np.nan])
|
||
|
|
tm.assert_categorical_equal(result, expected)
|
||
|
|
|
||
|
|
c1 = Categorical([])
|
||
|
|
c2 = Categorical([])
|
||
|
|
result = union_categoricals([c1, c2], sort_categories=True)
|
||
|
|
expected = Categorical([])
|
||
|
|
tm.assert_categorical_equal(result, expected)
|
||
|
|
|
||
|
|
c1 = Categorical(["b", "a"], categories=["b", "a", "c"], ordered=True)
|
||
|
|
c2 = Categorical(["a", "c"], categories=["b", "a", "c"], ordered=True)
|
||
|
|
msg = "Cannot use sort_categories=True with ordered Categoricals"
|
||
|
|
with pytest.raises(TypeError, match=msg):
|
||
|
|
union_categoricals([c1, c2], sort_categories=True)
|
||
|
|
|
||
|
|
def test_union_categoricals_sort_false(self):
|
||
|
|
# GH 13846
|
||
|
|
c1 = Categorical(["x", "y", "z"])
|
||
|
|
c2 = Categorical(["a", "b", "c"])
|
||
|
|
result = union_categoricals([c1, c2], sort_categories=False)
|
||
|
|
expected = Categorical(
|
||
|
|
["x", "y", "z", "a", "b", "c"], categories=["x", "y", "z", "a", "b", "c"]
|
||
|
|
)
|
||
|
|
tm.assert_categorical_equal(result, expected)
|
||
|
|
|
||
|
|
def test_union_categoricals_sort_false_fastpath(self):
|
||
|
|
# fastpath
|
||
|
|
c1 = Categorical(["a", "b"], categories=["b", "a", "c"])
|
||
|
|
c2 = Categorical(["b", "c"], categories=["b", "a", "c"])
|
||
|
|
result = union_categoricals([c1, c2], sort_categories=False)
|
||
|
|
expected = Categorical(["a", "b", "b", "c"], categories=["b", "a", "c"])
|
||
|
|
tm.assert_categorical_equal(result, expected)
|
||
|
|
|
||
|
|
def test_union_categoricals_sort_false_skipresort(self):
|
||
|
|
# fastpath - skip resort
|
||
|
|
c1 = Categorical(["a", "b"], categories=["a", "b", "c"])
|
||
|
|
c2 = Categorical(["b", "c"], categories=["a", "b", "c"])
|
||
|
|
result = union_categoricals([c1, c2], sort_categories=False)
|
||
|
|
expected = Categorical(["a", "b", "b", "c"], categories=["a", "b", "c"])
|
||
|
|
tm.assert_categorical_equal(result, expected)
|
||
|
|
|
||
|
|
def test_union_categoricals_sort_false_one_nan(self):
|
||
|
|
c1 = Categorical(["x", np.nan])
|
||
|
|
c2 = Categorical([np.nan, "b"])
|
||
|
|
result = union_categoricals([c1, c2], sort_categories=False)
|
||
|
|
expected = Categorical(["x", np.nan, np.nan, "b"], categories=["x", "b"])
|
||
|
|
tm.assert_categorical_equal(result, expected)
|
||
|
|
|
||
|
|
def test_union_categoricals_sort_false_only_nan(self):
|
||
|
|
c1 = Categorical([np.nan])
|
||
|
|
c2 = Categorical([np.nan])
|
||
|
|
result = union_categoricals([c1, c2], sort_categories=False)
|
||
|
|
expected = Categorical([np.nan, np.nan])
|
||
|
|
tm.assert_categorical_equal(result, expected)
|
||
|
|
|
||
|
|
def test_union_categoricals_sort_false_empty(self):
|
||
|
|
c1 = Categorical([])
|
||
|
|
c2 = Categorical([])
|
||
|
|
result = union_categoricals([c1, c2], sort_categories=False)
|
||
|
|
expected = Categorical([])
|
||
|
|
tm.assert_categorical_equal(result, expected)
|
||
|
|
|
||
|
|
def test_union_categoricals_sort_false_ordered_true(self):
|
||
|
|
c1 = Categorical(["b", "a"], categories=["b", "a", "c"], ordered=True)
|
||
|
|
c2 = Categorical(["a", "c"], categories=["b", "a", "c"], ordered=True)
|
||
|
|
result = union_categoricals([c1, c2], sort_categories=False)
|
||
|
|
expected = Categorical(
|
||
|
|
["b", "a", "a", "c"], categories=["b", "a", "c"], ordered=True
|
||
|
|
)
|
||
|
|
tm.assert_categorical_equal(result, expected)
|
||
|
|
|
||
|
|
def test_union_categorical_unwrap(self):
|
||
|
|
# GH 14173
|
||
|
|
c1 = Categorical(["a", "b"])
|
||
|
|
c2 = Series(["b", "c"], dtype="category")
|
||
|
|
result = union_categoricals([c1, c2])
|
||
|
|
expected = Categorical(["a", "b", "b", "c"])
|
||
|
|
tm.assert_categorical_equal(result, expected)
|
||
|
|
|
||
|
|
c2 = CategoricalIndex(c2)
|
||
|
|
result = union_categoricals([c1, c2])
|
||
|
|
tm.assert_categorical_equal(result, expected)
|
||
|
|
|
||
|
|
c1 = Series(c1)
|
||
|
|
result = union_categoricals([c1, c2])
|
||
|
|
tm.assert_categorical_equal(result, expected)
|
||
|
|
|
||
|
|
msg = "all components to combine must be Categorical"
|
||
|
|
with pytest.raises(TypeError, match=msg):
|
||
|
|
union_categoricals([c1, ["a", "b", "c"]])
|