1237 lines
45 KiB
Python
1237 lines
45 KiB
Python
|
"""
|
||
|
|
test where we are determining what we are grouping, or getting groups
|
||
|
|
"""
|
||
|
|
from datetime import (
|
||
|
|
date,
|
||
|
|
timedelta,
|
||
|
|
)
|
||
|
|
|
||
|
|
import numpy as np
|
||
|
|
import pytest
|
||
|
|
|
||
|
|
import pandas as pd
|
||
|
|
from pandas import (
|
||
|
|
CategoricalIndex,
|
||
|
|
DataFrame,
|
||
|
|
Grouper,
|
||
|
|
Index,
|
||
|
|
MultiIndex,
|
||
|
|
Series,
|
||
|
|
Timestamp,
|
||
|
|
date_range,
|
||
|
|
period_range,
|
||
|
|
)
|
||
|
|
import pandas._testing as tm
|
||
|
|
from pandas.core.groupby.grouper import Grouping
|
||
|
|
|
||
|
|
# selection
|
||
|
|
# --------------------------------
|
||
|
|
|
||
|
|
|
||
|
|
class TestSelection:
|
||
|
|
def test_select_bad_cols(self):
|
||
|
|
df = DataFrame([[1, 2]], columns=["A", "B"])
|
||
|
|
g = df.groupby("A")
|
||
|
|
with pytest.raises(KeyError, match="\"Columns not found: 'C'\""):
|
||
|
|
g[["C"]]
|
||
|
|
|
||
|
|
with pytest.raises(KeyError, match="^[^A]+$"):
|
||
|
|
# A should not be referenced as a bad column...
|
||
|
|
# will have to rethink regex if you change message!
|
||
|
|
g[["A", "C"]]
|
||
|
|
|
||
|
|
def test_groupby_duplicated_column_errormsg(self):
|
||
|
|
# GH7511
|
||
|
|
df = DataFrame(
|
||
|
|
columns=["A", "B", "A", "C"], data=[range(4), range(2, 6), range(0, 8, 2)]
|
||
|
|
)
|
||
|
|
|
||
|
|
msg = "Grouper for 'A' not 1-dimensional"
|
||
|
|
with pytest.raises(ValueError, match=msg):
|
||
|
|
df.groupby("A")
|
||
|
|
with pytest.raises(ValueError, match=msg):
|
||
|
|
df.groupby(["A", "B"])
|
||
|
|
|
||
|
|
grouped = df.groupby("B")
|
||
|
|
c = grouped.count()
|
||
|
|
assert c.columns.nlevels == 1
|
||
|
|
assert c.columns.size == 3
|
||
|
|
|
||
|
|
def test_column_select_via_attr(self, df):
|
||
|
|
result = df.groupby("A").C.sum()
|
||
|
|
expected = df.groupby("A")["C"].sum()
|
||
|
|
tm.assert_series_equal(result, expected)
|
||
|
|
|
||
|
|
df["mean"] = 1.5
|
||
|
|
result = df.groupby("A").mean(numeric_only=True)
|
||
|
|
expected = df.groupby("A")[["C", "D", "mean"]].agg("mean")
|
||
|
|
tm.assert_frame_equal(result, expected)
|
||
|
|
|
||
|
|
def test_getitem_list_of_columns(self):
|
||
|
|
df = DataFrame(
|
||
|
|
{
|
||
|
|
"A": ["foo", "bar", "foo", "bar", "foo", "bar", "foo", "foo"],
|
||
|
|
"B": ["one", "one", "two", "three", "two", "two", "one", "three"],
|
||
|
|
"C": np.random.default_rng(2).standard_normal(8),
|
||
|
|
"D": np.random.default_rng(2).standard_normal(8),
|
||
|
|
"E": np.random.default_rng(2).standard_normal(8),
|
||
|
|
}
|
||
|
|
)
|
||
|
|
|
||
|
|
result = df.groupby("A")[["C", "D"]].mean()
|
||
|
|
result2 = df.groupby("A")[df.columns[2:4]].mean()
|
||
|
|
|
||
|
|
expected = df.loc[:, ["A", "C", "D"]].groupby("A").mean()
|
||
|
|
|
||
|
|
tm.assert_frame_equal(result, expected)
|
||
|
|
tm.assert_frame_equal(result2, expected)
|
||
|
|
|
||
|
|
def test_getitem_numeric_column_names(self):
|
||
|
|
# GH #13731
|
||
|
|
df = DataFrame(
|
||
|
|
{
|
||
|
|
0: list("abcd") * 2,
|
||
|
|
2: np.random.default_rng(2).standard_normal(8),
|
||
|
|
4: np.random.default_rng(2).standard_normal(8),
|
||
|
|
6: np.random.default_rng(2).standard_normal(8),
|
||
|
|
}
|
||
|
|
)
|
||
|
|
result = df.groupby(0)[df.columns[1:3]].mean()
|
||
|
|
result2 = df.groupby(0)[[2, 4]].mean()
|
||
|
|
|
||
|
|
expected = df.loc[:, [0, 2, 4]].groupby(0).mean()
|
||
|
|
|
||
|
|
tm.assert_frame_equal(result, expected)
|
||
|
|
tm.assert_frame_equal(result2, expected)
|
||
|
|
|
||
|
|
# per GH 23566 enforced deprecation raises a ValueError
|
||
|
|
with pytest.raises(ValueError, match="Cannot subset columns with a tuple"):
|
||
|
|
df.groupby(0)[2, 4].mean()
|
||
|
|
|
||
|
|
def test_getitem_single_tuple_of_columns_raises(self, df):
|
||
|
|
# per GH 23566 enforced deprecation raises a ValueError
|
||
|
|
with pytest.raises(ValueError, match="Cannot subset columns with a tuple"):
|
||
|
|
df.groupby("A")["C", "D"].mean()
|
||
|
|
|
||
|
|
def test_getitem_single_column(self):
|
||
|
|
df = DataFrame(
|
||
|
|
{
|
||
|
|
"A": ["foo", "bar", "foo", "bar", "foo", "bar", "foo", "foo"],
|
||
|
|
"B": ["one", "one", "two", "three", "two", "two", "one", "three"],
|
||
|
|
"C": np.random.default_rng(2).standard_normal(8),
|
||
|
|
"D": np.random.default_rng(2).standard_normal(8),
|
||
|
|
"E": np.random.default_rng(2).standard_normal(8),
|
||
|
|
}
|
||
|
|
)
|
||
|
|
|
||
|
|
result = df.groupby("A")["C"].mean()
|
||
|
|
|
||
|
|
as_frame = df.loc[:, ["A", "C"]].groupby("A").mean()
|
||
|
|
as_series = as_frame.iloc[:, 0]
|
||
|
|
expected = as_series
|
||
|
|
|
||
|
|
tm.assert_series_equal(result, expected)
|
||
|
|
|
||
|
|
@pytest.mark.parametrize(
|
||
|
|
"func", [lambda x: x.sum(), lambda x: x.agg(lambda y: y.sum())]
|
||
|
|
)
|
||
|
|
def test_getitem_from_grouper(self, func):
|
||
|
|
# GH 50383
|
||
|
|
df = DataFrame({"a": [1, 1, 2], "b": 3, "c": 4, "d": 5})
|
||
|
|
gb = df.groupby(["a", "b"])[["a", "c"]]
|
||
|
|
|
||
|
|
idx = MultiIndex.from_tuples([(1, 3), (2, 3)], names=["a", "b"])
|
||
|
|
expected = DataFrame({"a": [2, 2], "c": [8, 4]}, index=idx)
|
||
|
|
result = func(gb)
|
||
|
|
|
||
|
|
tm.assert_frame_equal(result, expected)
|
||
|
|
|
||
|
|
def test_indices_grouped_by_tuple_with_lambda(self):
|
||
|
|
# GH 36158
|
||
|
|
df = DataFrame(
|
||
|
|
{
|
||
|
|
"Tuples": (
|
||
|
|
(x, y)
|
||
|
|
for x in [0, 1]
|
||
|
|
for y in np.random.default_rng(2).integers(3, 5, 5)
|
||
|
|
)
|
||
|
|
}
|
||
|
|
)
|
||
|
|
|
||
|
|
gb = df.groupby("Tuples")
|
||
|
|
gb_lambda = df.groupby(lambda x: df.iloc[x, 0])
|
||
|
|
|
||
|
|
expected = gb.indices
|
||
|
|
result = gb_lambda.indices
|
||
|
|
|
||
|
|
tm.assert_dict_equal(result, expected)
|
||
|
|
|
||
|
|
|
||
|
|
# grouping
|
||
|
|
# --------------------------------
|
||
|
|
|
||
|
|
|
||
|
|
class TestGrouping:
|
||
|
|
@pytest.mark.parametrize(
|
||
|
|
"index",
|
||
|
|
[
|
||
|
|
Index(list("abcde")),
|
||
|
|
Index(np.arange(5)),
|
||
|
|
Index(np.arange(5, dtype=float)),
|
||
|
|
date_range("2020-01-01", periods=5),
|
||
|
|
period_range("2020-01-01", periods=5),
|
||
|
|
],
|
||
|
|
)
|
||
|
|
def test_grouper_index_types(self, index):
|
||
|
|
# related GH5375
|
||
|
|
# groupby misbehaving when using a Floatlike index
|
||
|
|
df = DataFrame(np.arange(10).reshape(5, 2), columns=list("AB"), index=index)
|
||
|
|
|
||
|
|
df.groupby(list("abcde"), group_keys=False).apply(lambda x: x)
|
||
|
|
|
||
|
|
df.index = df.index[::-1]
|
||
|
|
df.groupby(list("abcde"), group_keys=False).apply(lambda x: x)
|
||
|
|
|
||
|
|
def test_grouper_multilevel_freq(self):
|
||
|
|
# GH 7885
|
||
|
|
# with level and freq specified in a Grouper
|
||
|
|
d0 = date.today() - timedelta(days=14)
|
||
|
|
dates = date_range(d0, date.today())
|
||
|
|
date_index = MultiIndex.from_product([dates, dates], names=["foo", "bar"])
|
||
|
|
df = DataFrame(np.random.default_rng(2).integers(0, 100, 225), index=date_index)
|
||
|
|
|
||
|
|
# Check string level
|
||
|
|
expected = (
|
||
|
|
df.reset_index()
|
||
|
|
.groupby([Grouper(key="foo", freq="W"), Grouper(key="bar", freq="W")])
|
||
|
|
.sum()
|
||
|
|
)
|
||
|
|
# reset index changes columns dtype to object
|
||
|
|
expected.columns = Index([0], dtype="int64")
|
||
|
|
|
||
|
|
result = df.groupby(
|
||
|
|
[Grouper(level="foo", freq="W"), Grouper(level="bar", freq="W")]
|
||
|
|
).sum()
|
||
|
|
tm.assert_frame_equal(result, expected)
|
||
|
|
|
||
|
|
# Check integer level
|
||
|
|
result = df.groupby(
|
||
|
|
[Grouper(level=0, freq="W"), Grouper(level=1, freq="W")]
|
||
|
|
).sum()
|
||
|
|
tm.assert_frame_equal(result, expected)
|
||
|
|
|
||
|
|
def test_grouper_creation_bug(self):
|
||
|
|
# GH 8795
|
||
|
|
df = DataFrame({"A": [0, 0, 1, 1, 2, 2], "B": [1, 2, 3, 4, 5, 6]})
|
||
|
|
g = df.groupby("A")
|
||
|
|
expected = g.sum()
|
||
|
|
|
||
|
|
g = df.groupby(Grouper(key="A"))
|
||
|
|
result = g.sum()
|
||
|
|
tm.assert_frame_equal(result, expected)
|
||
|
|
|
||
|
|
msg = "Grouper axis keyword is deprecated and will be removed"
|
||
|
|
with tm.assert_produces_warning(FutureWarning, match=msg):
|
||
|
|
gpr = Grouper(key="A", axis=0)
|
||
|
|
g = df.groupby(gpr)
|
||
|
|
result = g.sum()
|
||
|
|
tm.assert_frame_equal(result, expected)
|
||
|
|
|
||
|
|
msg = "DataFrameGroupBy.apply operated on the grouping columns"
|
||
|
|
with tm.assert_produces_warning(DeprecationWarning, match=msg):
|
||
|
|
result = g.apply(lambda x: x.sum())
|
||
|
|
expected["A"] = [0, 2, 4]
|
||
|
|
expected = expected.loc[:, ["A", "B"]]
|
||
|
|
tm.assert_frame_equal(result, expected)
|
||
|
|
|
||
|
|
def test_grouper_creation_bug2(self):
|
||
|
|
# GH14334
|
||
|
|
# Grouper(key=...) may be passed in a list
|
||
|
|
df = DataFrame(
|
||
|
|
{"A": [0, 0, 0, 1, 1, 1], "B": [1, 1, 2, 2, 3, 3], "C": [1, 2, 3, 4, 5, 6]}
|
||
|
|
)
|
||
|
|
# Group by single column
|
||
|
|
expected = df.groupby("A").sum()
|
||
|
|
g = df.groupby([Grouper(key="A")])
|
||
|
|
result = g.sum()
|
||
|
|
tm.assert_frame_equal(result, expected)
|
||
|
|
|
||
|
|
# Group by two columns
|
||
|
|
# using a combination of strings and Grouper objects
|
||
|
|
expected = df.groupby(["A", "B"]).sum()
|
||
|
|
|
||
|
|
# Group with two Grouper objects
|
||
|
|
g = df.groupby([Grouper(key="A"), Grouper(key="B")])
|
||
|
|
result = g.sum()
|
||
|
|
tm.assert_frame_equal(result, expected)
|
||
|
|
|
||
|
|
# Group with a string and a Grouper object
|
||
|
|
g = df.groupby(["A", Grouper(key="B")])
|
||
|
|
result = g.sum()
|
||
|
|
tm.assert_frame_equal(result, expected)
|
||
|
|
|
||
|
|
# Group with a Grouper object and a string
|
||
|
|
g = df.groupby([Grouper(key="A"), "B"])
|
||
|
|
result = g.sum()
|
||
|
|
tm.assert_frame_equal(result, expected)
|
||
|
|
|
||
|
|
def test_grouper_creation_bug3(self, unit):
|
||
|
|
# GH8866
|
||
|
|
dti = date_range("20130101", periods=2, unit=unit)
|
||
|
|
mi = MultiIndex.from_product(
|
||
|
|
[list("ab"), range(2), dti],
|
||
|
|
names=["one", "two", "three"],
|
||
|
|
)
|
||
|
|
ser = Series(
|
||
|
|
np.arange(8, dtype="int64"),
|
||
|
|
index=mi,
|
||
|
|
)
|
||
|
|
result = ser.groupby(Grouper(level="three", freq="ME")).sum()
|
||
|
|
exp_dti = pd.DatetimeIndex(
|
||
|
|
[Timestamp("2013-01-31")], freq="ME", name="three"
|
||
|
|
).as_unit(unit)
|
||
|
|
expected = Series(
|
||
|
|
[28],
|
||
|
|
index=exp_dti,
|
||
|
|
)
|
||
|
|
tm.assert_series_equal(result, expected)
|
||
|
|
|
||
|
|
# just specifying a level breaks
|
||
|
|
result = ser.groupby(Grouper(level="one")).sum()
|
||
|
|
expected = ser.groupby(level="one").sum()
|
||
|
|
tm.assert_series_equal(result, expected)
|
||
|
|
|
||
|
|
@pytest.mark.parametrize("func", [False, True])
|
||
|
|
def test_grouper_returning_tuples(self, func):
|
||
|
|
# GH 22257 , both with dict and with callable
|
||
|
|
df = DataFrame({"X": ["A", "B", "A", "B"], "Y": [1, 4, 3, 2]})
|
||
|
|
mapping = dict(zip(range(4), [("C", 5), ("D", 6)] * 2))
|
||
|
|
|
||
|
|
if func:
|
||
|
|
gb = df.groupby(by=lambda idx: mapping[idx], sort=False)
|
||
|
|
else:
|
||
|
|
gb = df.groupby(by=mapping, sort=False)
|
||
|
|
|
||
|
|
name, expected = next(iter(gb))
|
||
|
|
assert name == ("C", 5)
|
||
|
|
result = gb.get_group(name)
|
||
|
|
|
||
|
|
tm.assert_frame_equal(result, expected)
|
||
|
|
|
||
|
|
def test_grouper_column_and_index(self):
|
||
|
|
# GH 14327
|
||
|
|
|
||
|
|
# Grouping a multi-index frame by a column and an index level should
|
||
|
|
# be equivalent to resetting the index and grouping by two columns
|
||
|
|
idx = MultiIndex.from_tuples(
|
||
|
|
[("a", 1), ("a", 2), ("a", 3), ("b", 1), ("b", 2), ("b", 3)]
|
||
|
|
)
|
||
|
|
idx.names = ["outer", "inner"]
|
||
|
|
df_multi = DataFrame(
|
||
|
|
{"A": np.arange(6), "B": ["one", "one", "two", "two", "one", "one"]},
|
||
|
|
index=idx,
|
||
|
|
)
|
||
|
|
result = df_multi.groupby(["B", Grouper(level="inner")]).mean(numeric_only=True)
|
||
|
|
expected = (
|
||
|
|
df_multi.reset_index().groupby(["B", "inner"]).mean(numeric_only=True)
|
||
|
|
)
|
||
|
|
tm.assert_frame_equal(result, expected)
|
||
|
|
|
||
|
|
# Test the reverse grouping order
|
||
|
|
result = df_multi.groupby([Grouper(level="inner"), "B"]).mean(numeric_only=True)
|
||
|
|
expected = (
|
||
|
|
df_multi.reset_index().groupby(["inner", "B"]).mean(numeric_only=True)
|
||
|
|
)
|
||
|
|
tm.assert_frame_equal(result, expected)
|
||
|
|
|
||
|
|
# Grouping a single-index frame by a column and the index should
|
||
|
|
# be equivalent to resetting the index and grouping by two columns
|
||
|
|
df_single = df_multi.reset_index("outer")
|
||
|
|
result = df_single.groupby(["B", Grouper(level="inner")]).mean(
|
||
|
|
numeric_only=True
|
||
|
|
)
|
||
|
|
expected = (
|
||
|
|
df_single.reset_index().groupby(["B", "inner"]).mean(numeric_only=True)
|
||
|
|
)
|
||
|
|
tm.assert_frame_equal(result, expected)
|
||
|
|
|
||
|
|
# Test the reverse grouping order
|
||
|
|
result = df_single.groupby([Grouper(level="inner"), "B"]).mean(
|
||
|
|
numeric_only=True
|
||
|
|
)
|
||
|
|
expected = (
|
||
|
|
df_single.reset_index().groupby(["inner", "B"]).mean(numeric_only=True)
|
||
|
|
)
|
||
|
|
tm.assert_frame_equal(result, expected)
|
||
|
|
|
||
|
|
def test_groupby_levels_and_columns(self):
|
||
|
|
# GH9344, GH9049
|
||
|
|
idx_names = ["x", "y"]
|
||
|
|
idx = MultiIndex.from_tuples([(1, 1), (1, 2), (3, 4), (5, 6)], names=idx_names)
|
||
|
|
df = DataFrame(np.arange(12).reshape(-1, 3), index=idx)
|
||
|
|
|
||
|
|
by_levels = df.groupby(level=idx_names).mean()
|
||
|
|
# reset_index changes columns dtype to object
|
||
|
|
by_columns = df.reset_index().groupby(idx_names).mean()
|
||
|
|
|
||
|
|
# without casting, by_columns.columns is object-dtype
|
||
|
|
by_columns.columns = by_columns.columns.astype(np.int64)
|
||
|
|
tm.assert_frame_equal(by_levels, by_columns)
|
||
|
|
|
||
|
|
def test_groupby_categorical_index_and_columns(self, observed):
|
||
|
|
# GH18432, adapted for GH25871
|
||
|
|
columns = ["A", "B", "A", "B"]
|
||
|
|
categories = ["B", "A"]
|
||
|
|
data = np.array(
|
||
|
|
[[1, 2, 1, 2], [1, 2, 1, 2], [1, 2, 1, 2], [1, 2, 1, 2], [1, 2, 1, 2]], int
|
||
|
|
)
|
||
|
|
cat_columns = CategoricalIndex(columns, categories=categories, ordered=True)
|
||
|
|
df = DataFrame(data=data, columns=cat_columns)
|
||
|
|
depr_msg = "DataFrame.groupby with axis=1 is deprecated"
|
||
|
|
with tm.assert_produces_warning(FutureWarning, match=depr_msg):
|
||
|
|
result = df.groupby(axis=1, level=0, observed=observed).sum()
|
||
|
|
expected_data = np.array([[4, 2], [4, 2], [4, 2], [4, 2], [4, 2]], int)
|
||
|
|
expected_columns = CategoricalIndex(
|
||
|
|
categories, categories=categories, ordered=True
|
||
|
|
)
|
||
|
|
expected = DataFrame(data=expected_data, columns=expected_columns)
|
||
|
|
tm.assert_frame_equal(result, expected)
|
||
|
|
|
||
|
|
# test transposed version
|
||
|
|
df = DataFrame(data.T, index=cat_columns)
|
||
|
|
msg = "The 'axis' keyword in DataFrame.groupby is deprecated"
|
||
|
|
with tm.assert_produces_warning(FutureWarning, match=msg):
|
||
|
|
result = df.groupby(axis=0, level=0, observed=observed).sum()
|
||
|
|
expected = DataFrame(data=expected_data.T, index=expected_columns)
|
||
|
|
tm.assert_frame_equal(result, expected)
|
||
|
|
|
||
|
|
def test_grouper_getting_correct_binner(self):
|
||
|
|
# GH 10063
|
||
|
|
# using a non-time-based grouper and a time-based grouper
|
||
|
|
# and specifying levels
|
||
|
|
df = DataFrame(
|
||
|
|
{"A": 1},
|
||
|
|
index=MultiIndex.from_product(
|
||
|
|
[list("ab"), date_range("20130101", periods=80)], names=["one", "two"]
|
||
|
|
),
|
||
|
|
)
|
||
|
|
result = df.groupby(
|
||
|
|
[Grouper(level="one"), Grouper(level="two", freq="ME")]
|
||
|
|
).sum()
|
||
|
|
expected = DataFrame(
|
||
|
|
{"A": [31, 28, 21, 31, 28, 21]},
|
||
|
|
index=MultiIndex.from_product(
|
||
|
|
[list("ab"), date_range("20130101", freq="ME", periods=3)],
|
||
|
|
names=["one", "two"],
|
||
|
|
),
|
||
|
|
)
|
||
|
|
tm.assert_frame_equal(result, expected)
|
||
|
|
|
||
|
|
def test_grouper_iter(self, df):
|
||
|
|
gb = df.groupby("A")
|
||
|
|
msg = "DataFrameGroupBy.grouper is deprecated"
|
||
|
|
with tm.assert_produces_warning(FutureWarning, match=msg):
|
||
|
|
grouper = gb.grouper
|
||
|
|
result = sorted(grouper)
|
||
|
|
expected = ["bar", "foo"]
|
||
|
|
assert result == expected
|
||
|
|
|
||
|
|
def test_empty_groups(self, df):
|
||
|
|
# see gh-1048
|
||
|
|
with pytest.raises(ValueError, match="No group keys passed!"):
|
||
|
|
df.groupby([])
|
||
|
|
|
||
|
|
def test_groupby_grouper(self, df):
|
||
|
|
grouped = df.groupby("A")
|
||
|
|
msg = "DataFrameGroupBy.grouper is deprecated"
|
||
|
|
with tm.assert_produces_warning(FutureWarning, match=msg):
|
||
|
|
grouper = grouped.grouper
|
||
|
|
result = df.groupby(grouper).mean(numeric_only=True)
|
||
|
|
expected = grouped.mean(numeric_only=True)
|
||
|
|
tm.assert_frame_equal(result, expected)
|
||
|
|
|
||
|
|
def test_groupby_dict_mapping(self):
|
||
|
|
# GH #679
|
||
|
|
s = Series({"T1": 5})
|
||
|
|
result = s.groupby({"T1": "T2"}).agg("sum")
|
||
|
|
expected = s.groupby(["T2"]).agg("sum")
|
||
|
|
tm.assert_series_equal(result, expected)
|
||
|
|
|
||
|
|
s = Series([1.0, 2.0, 3.0, 4.0], index=list("abcd"))
|
||
|
|
mapping = {"a": 0, "b": 0, "c": 1, "d": 1}
|
||
|
|
|
||
|
|
result = s.groupby(mapping).mean()
|
||
|
|
result2 = s.groupby(mapping).agg("mean")
|
||
|
|
exp_key = np.array([0, 0, 1, 1], dtype=np.int64)
|
||
|
|
expected = s.groupby(exp_key).mean()
|
||
|
|
expected2 = s.groupby(exp_key).mean()
|
||
|
|
tm.assert_series_equal(result, expected)
|
||
|
|
tm.assert_series_equal(result, result2)
|
||
|
|
tm.assert_series_equal(result, expected2)
|
||
|
|
|
||
|
|
@pytest.mark.parametrize(
|
||
|
|
"index",
|
||
|
|
[
|
||
|
|
[0, 1, 2, 3],
|
||
|
|
["a", "b", "c", "d"],
|
||
|
|
[Timestamp(2021, 7, 28 + i) for i in range(4)],
|
||
|
|
],
|
||
|
|
)
|
||
|
|
def test_groupby_series_named_with_tuple(self, frame_or_series, index):
|
||
|
|
# GH 42731
|
||
|
|
obj = frame_or_series([1, 2, 3, 4], index=index)
|
||
|
|
groups = Series([1, 0, 1, 0], index=index, name=("a", "a"))
|
||
|
|
result = obj.groupby(groups).last()
|
||
|
|
expected = frame_or_series([4, 3])
|
||
|
|
expected.index.name = ("a", "a")
|
||
|
|
tm.assert_equal(result, expected)
|
||
|
|
|
||
|
|
def test_groupby_grouper_f_sanity_checked(self):
|
||
|
|
dates = date_range("01-Jan-2013", periods=12, freq="MS")
|
||
|
|
ts = Series(np.random.default_rng(2).standard_normal(12), index=dates)
|
||
|
|
|
||
|
|
# GH51979
|
||
|
|
# simple check that the passed function doesn't operates on the whole index
|
||
|
|
msg = "'Timestamp' object is not subscriptable"
|
||
|
|
with pytest.raises(TypeError, match=msg):
|
||
|
|
ts.groupby(lambda key: key[0:6])
|
||
|
|
|
||
|
|
result = ts.groupby(lambda x: x).sum()
|
||
|
|
expected = ts.groupby(ts.index).sum()
|
||
|
|
expected.index.freq = None
|
||
|
|
tm.assert_series_equal(result, expected)
|
||
|
|
|
||
|
|
def test_groupby_with_datetime_key(self):
|
||
|
|
# GH 51158
|
||
|
|
df = DataFrame(
|
||
|
|
{
|
||
|
|
"id": ["a", "b"] * 3,
|
||
|
|
"b": date_range("2000-01-01", "2000-01-03", freq="9h"),
|
||
|
|
}
|
||
|
|
)
|
||
|
|
grouper = Grouper(key="b", freq="D")
|
||
|
|
gb = df.groupby([grouper, "id"])
|
||
|
|
|
||
|
|
# test number of groups
|
||
|
|
expected = {
|
||
|
|
(Timestamp("2000-01-01"), "a"): [0, 2],
|
||
|
|
(Timestamp("2000-01-01"), "b"): [1],
|
||
|
|
(Timestamp("2000-01-02"), "a"): [4],
|
||
|
|
(Timestamp("2000-01-02"), "b"): [3, 5],
|
||
|
|
}
|
||
|
|
tm.assert_dict_equal(gb.groups, expected)
|
||
|
|
|
||
|
|
# test number of group keys
|
||
|
|
assert len(gb.groups.keys()) == 4
|
||
|
|
|
||
|
|
def test_grouping_error_on_multidim_input(self, df):
|
||
|
|
msg = "Grouper for '<class 'pandas.core.frame.DataFrame'>' not 1-dimensional"
|
||
|
|
with pytest.raises(ValueError, match=msg):
|
||
|
|
Grouping(df.index, df[["A", "A"]])
|
||
|
|
|
||
|
|
def test_multiindex_passthru(self):
|
||
|
|
# GH 7997
|
||
|
|
# regression from 0.14.1
|
||
|
|
df = DataFrame([[1, 2, 3], [4, 5, 6], [7, 8, 9]])
|
||
|
|
df.columns = MultiIndex.from_tuples([(0, 1), (1, 1), (2, 1)])
|
||
|
|
|
||
|
|
depr_msg = "DataFrame.groupby with axis=1 is deprecated"
|
||
|
|
with tm.assert_produces_warning(FutureWarning, match=depr_msg):
|
||
|
|
gb = df.groupby(axis=1, level=[0, 1])
|
||
|
|
result = gb.first()
|
||
|
|
tm.assert_frame_equal(result, df)
|
||
|
|
|
||
|
|
def test_multiindex_negative_level(self, multiindex_dataframe_random_data):
|
||
|
|
# GH 13901
|
||
|
|
result = multiindex_dataframe_random_data.groupby(level=-1).sum()
|
||
|
|
expected = multiindex_dataframe_random_data.groupby(level="second").sum()
|
||
|
|
tm.assert_frame_equal(result, expected)
|
||
|
|
|
||
|
|
result = multiindex_dataframe_random_data.groupby(level=-2).sum()
|
||
|
|
expected = multiindex_dataframe_random_data.groupby(level="first").sum()
|
||
|
|
tm.assert_frame_equal(result, expected)
|
||
|
|
|
||
|
|
result = multiindex_dataframe_random_data.groupby(level=[-2, -1]).sum()
|
||
|
|
expected = multiindex_dataframe_random_data.sort_index()
|
||
|
|
tm.assert_frame_equal(result, expected)
|
||
|
|
|
||
|
|
result = multiindex_dataframe_random_data.groupby(level=[-1, "first"]).sum()
|
||
|
|
expected = multiindex_dataframe_random_data.groupby(
|
||
|
|
level=["second", "first"]
|
||
|
|
).sum()
|
||
|
|
tm.assert_frame_equal(result, expected)
|
||
|
|
|
||
|
|
def test_multifunc_select_col_integer_cols(self, df):
|
||
|
|
df.columns = np.arange(len(df.columns))
|
||
|
|
|
||
|
|
# it works!
|
||
|
|
msg = "Passing a dictionary to SeriesGroupBy.agg is deprecated"
|
||
|
|
with tm.assert_produces_warning(FutureWarning, match=msg):
|
||
|
|
df.groupby(1, as_index=False)[2].agg({"Q": np.mean})
|
||
|
|
|
||
|
|
def test_multiindex_columns_empty_level(self):
|
||
|
|
lst = [["count", "values"], ["to filter", ""]]
|
||
|
|
midx = MultiIndex.from_tuples(lst)
|
||
|
|
|
||
|
|
df = DataFrame([[1, "A"]], columns=midx)
|
||
|
|
|
||
|
|
grouped = df.groupby("to filter").groups
|
||
|
|
assert grouped["A"] == [0]
|
||
|
|
|
||
|
|
grouped = df.groupby([("to filter", "")]).groups
|
||
|
|
assert grouped["A"] == [0]
|
||
|
|
|
||
|
|
df = DataFrame([[1, "A"], [2, "B"]], columns=midx)
|
||
|
|
|
||
|
|
expected = df.groupby("to filter").groups
|
||
|
|
result = df.groupby([("to filter", "")]).groups
|
||
|
|
assert result == expected
|
||
|
|
|
||
|
|
df = DataFrame([[1, "A"], [2, "A"]], columns=midx)
|
||
|
|
|
||
|
|
expected = df.groupby("to filter").groups
|
||
|
|
result = df.groupby([("to filter", "")]).groups
|
||
|
|
tm.assert_dict_equal(result, expected)
|
||
|
|
|
||
|
|
def test_groupby_multiindex_tuple(self):
|
||
|
|
# GH 17979
|
||
|
|
df = DataFrame(
|
||
|
|
[[1, 2, 3, 4], [3, 4, 5, 6], [1, 4, 2, 3]],
|
||
|
|
columns=MultiIndex.from_arrays([["a", "b", "b", "c"], [1, 1, 2, 2]]),
|
||
|
|
)
|
||
|
|
expected = df.groupby([("b", 1)]).groups
|
||
|
|
result = df.groupby(("b", 1)).groups
|
||
|
|
tm.assert_dict_equal(expected, result)
|
||
|
|
|
||
|
|
df2 = DataFrame(
|
||
|
|
df.values,
|
||
|
|
columns=MultiIndex.from_arrays(
|
||
|
|
[["a", "b", "b", "c"], ["d", "d", "e", "e"]]
|
||
|
|
),
|
||
|
|
)
|
||
|
|
expected = df2.groupby([("b", "d")]).groups
|
||
|
|
result = df.groupby(("b", 1)).groups
|
||
|
|
tm.assert_dict_equal(expected, result)
|
||
|
|
|
||
|
|
df3 = DataFrame(df.values, columns=[("a", "d"), ("b", "d"), ("b", "e"), "c"])
|
||
|
|
expected = df3.groupby([("b", "d")]).groups
|
||
|
|
result = df.groupby(("b", 1)).groups
|
||
|
|
tm.assert_dict_equal(expected, result)
|
||
|
|
|
||
|
|
def test_groupby_multiindex_partial_indexing_equivalence(self):
|
||
|
|
# GH 17977
|
||
|
|
df = DataFrame(
|
||
|
|
[[1, 2, 3, 4], [3, 4, 5, 6], [1, 4, 2, 3]],
|
||
|
|
columns=MultiIndex.from_arrays([["a", "b", "b", "c"], [1, 1, 2, 2]]),
|
||
|
|
)
|
||
|
|
|
||
|
|
expected_mean = df.groupby([("a", 1)])[[("b", 1), ("b", 2)]].mean()
|
||
|
|
result_mean = df.groupby([("a", 1)])["b"].mean()
|
||
|
|
tm.assert_frame_equal(expected_mean, result_mean)
|
||
|
|
|
||
|
|
expected_sum = df.groupby([("a", 1)])[[("b", 1), ("b", 2)]].sum()
|
||
|
|
result_sum = df.groupby([("a", 1)])["b"].sum()
|
||
|
|
tm.assert_frame_equal(expected_sum, result_sum)
|
||
|
|
|
||
|
|
expected_count = df.groupby([("a", 1)])[[("b", 1), ("b", 2)]].count()
|
||
|
|
result_count = df.groupby([("a", 1)])["b"].count()
|
||
|
|
tm.assert_frame_equal(expected_count, result_count)
|
||
|
|
|
||
|
|
expected_min = df.groupby([("a", 1)])[[("b", 1), ("b", 2)]].min()
|
||
|
|
result_min = df.groupby([("a", 1)])["b"].min()
|
||
|
|
tm.assert_frame_equal(expected_min, result_min)
|
||
|
|
|
||
|
|
expected_max = df.groupby([("a", 1)])[[("b", 1), ("b", 2)]].max()
|
||
|
|
result_max = df.groupby([("a", 1)])["b"].max()
|
||
|
|
tm.assert_frame_equal(expected_max, result_max)
|
||
|
|
|
||
|
|
expected_groups = df.groupby([("a", 1)])[[("b", 1), ("b", 2)]].groups
|
||
|
|
result_groups = df.groupby([("a", 1)])["b"].groups
|
||
|
|
tm.assert_dict_equal(expected_groups, result_groups)
|
||
|
|
|
||
|
|
@pytest.mark.parametrize("sort", [True, False])
|
||
|
|
def test_groupby_level(self, sort, multiindex_dataframe_random_data, df):
|
||
|
|
# GH 17537
|
||
|
|
frame = multiindex_dataframe_random_data
|
||
|
|
deleveled = frame.reset_index()
|
||
|
|
|
||
|
|
result0 = frame.groupby(level=0, sort=sort).sum()
|
||
|
|
result1 = frame.groupby(level=1, sort=sort).sum()
|
||
|
|
|
||
|
|
expected0 = frame.groupby(deleveled["first"].values, sort=sort).sum()
|
||
|
|
expected1 = frame.groupby(deleveled["second"].values, sort=sort).sum()
|
||
|
|
|
||
|
|
expected0.index.name = "first"
|
||
|
|
expected1.index.name = "second"
|
||
|
|
|
||
|
|
assert result0.index.name == "first"
|
||
|
|
assert result1.index.name == "second"
|
||
|
|
|
||
|
|
tm.assert_frame_equal(result0, expected0)
|
||
|
|
tm.assert_frame_equal(result1, expected1)
|
||
|
|
assert result0.index.name == frame.index.names[0]
|
||
|
|
assert result1.index.name == frame.index.names[1]
|
||
|
|
|
||
|
|
# groupby level name
|
||
|
|
result0 = frame.groupby(level="first", sort=sort).sum()
|
||
|
|
result1 = frame.groupby(level="second", sort=sort).sum()
|
||
|
|
tm.assert_frame_equal(result0, expected0)
|
||
|
|
tm.assert_frame_equal(result1, expected1)
|
||
|
|
|
||
|
|
# axis=1
|
||
|
|
msg = "DataFrame.groupby with axis=1 is deprecated"
|
||
|
|
with tm.assert_produces_warning(FutureWarning, match=msg):
|
||
|
|
result0 = frame.T.groupby(level=0, axis=1, sort=sort).sum()
|
||
|
|
result1 = frame.T.groupby(level=1, axis=1, sort=sort).sum()
|
||
|
|
tm.assert_frame_equal(result0, expected0.T)
|
||
|
|
tm.assert_frame_equal(result1, expected1.T)
|
||
|
|
|
||
|
|
# raise exception for non-MultiIndex
|
||
|
|
msg = "level > 0 or level < -1 only valid with MultiIndex"
|
||
|
|
with pytest.raises(ValueError, match=msg):
|
||
|
|
df.groupby(level=1)
|
||
|
|
|
||
|
|
def test_groupby_level_index_names(self, axis):
|
||
|
|
# GH4014 this used to raise ValueError since 'exp'>1 (in py2)
|
||
|
|
df = DataFrame({"exp": ["A"] * 3 + ["B"] * 3, "var1": range(6)}).set_index(
|
||
|
|
"exp"
|
||
|
|
)
|
||
|
|
if axis in (1, "columns"):
|
||
|
|
df = df.T
|
||
|
|
depr_msg = "DataFrame.groupby with axis=1 is deprecated"
|
||
|
|
else:
|
||
|
|
depr_msg = "The 'axis' keyword in DataFrame.groupby is deprecated"
|
||
|
|
with tm.assert_produces_warning(FutureWarning, match=depr_msg):
|
||
|
|
df.groupby(level="exp", axis=axis)
|
||
|
|
msg = f"level name foo is not the name of the {df._get_axis_name(axis)}"
|
||
|
|
with pytest.raises(ValueError, match=msg):
|
||
|
|
with tm.assert_produces_warning(FutureWarning, match=depr_msg):
|
||
|
|
df.groupby(level="foo", axis=axis)
|
||
|
|
|
||
|
|
@pytest.mark.parametrize("sort", [True, False])
|
||
|
|
def test_groupby_level_with_nas(self, sort):
|
||
|
|
# GH 17537
|
||
|
|
index = MultiIndex(
|
||
|
|
levels=[[1, 0], [0, 1, 2, 3]],
|
||
|
|
codes=[[1, 1, 1, 1, 0, 0, 0, 0], [0, 1, 2, 3, 0, 1, 2, 3]],
|
||
|
|
)
|
||
|
|
|
||
|
|
# factorizing doesn't confuse things
|
||
|
|
s = Series(np.arange(8.0), index=index)
|
||
|
|
result = s.groupby(level=0, sort=sort).sum()
|
||
|
|
expected = Series([6.0, 22.0], index=[0, 1])
|
||
|
|
tm.assert_series_equal(result, expected)
|
||
|
|
|
||
|
|
index = MultiIndex(
|
||
|
|
levels=[[1, 0], [0, 1, 2, 3]],
|
||
|
|
codes=[[1, 1, 1, 1, -1, 0, 0, 0], [0, 1, 2, 3, 0, 1, 2, 3]],
|
||
|
|
)
|
||
|
|
|
||
|
|
# factorizing doesn't confuse things
|
||
|
|
s = Series(np.arange(8.0), index=index)
|
||
|
|
result = s.groupby(level=0, sort=sort).sum()
|
||
|
|
expected = Series([6.0, 18.0], index=[0.0, 1.0])
|
||
|
|
tm.assert_series_equal(result, expected)
|
||
|
|
|
||
|
|
def test_groupby_args(self, multiindex_dataframe_random_data):
|
||
|
|
# PR8618 and issue 8015
|
||
|
|
frame = multiindex_dataframe_random_data
|
||
|
|
|
||
|
|
msg = "You have to supply one of 'by' and 'level'"
|
||
|
|
with pytest.raises(TypeError, match=msg):
|
||
|
|
frame.groupby()
|
||
|
|
|
||
|
|
msg = "You have to supply one of 'by' and 'level'"
|
||
|
|
with pytest.raises(TypeError, match=msg):
|
||
|
|
frame.groupby(by=None, level=None)
|
||
|
|
|
||
|
|
@pytest.mark.parametrize(
|
||
|
|
"sort,labels",
|
||
|
|
[
|
||
|
|
[True, [2, 2, 2, 0, 0, 1, 1, 3, 3, 3]],
|
||
|
|
[False, [0, 0, 0, 1, 1, 2, 2, 3, 3, 3]],
|
||
|
|
],
|
||
|
|
)
|
||
|
|
def test_level_preserve_order(self, sort, labels, multiindex_dataframe_random_data):
|
||
|
|
# GH 17537
|
||
|
|
grouped = multiindex_dataframe_random_data.groupby(level=0, sort=sort)
|
||
|
|
exp_labels = np.array(labels, np.intp)
|
||
|
|
tm.assert_almost_equal(grouped._grouper.codes[0], exp_labels)
|
||
|
|
|
||
|
|
def test_grouping_labels(self, multiindex_dataframe_random_data):
|
||
|
|
grouped = multiindex_dataframe_random_data.groupby(
|
||
|
|
multiindex_dataframe_random_data.index.get_level_values(0)
|
||
|
|
)
|
||
|
|
exp_labels = np.array([2, 2, 2, 0, 0, 1, 1, 3, 3, 3], dtype=np.intp)
|
||
|
|
tm.assert_almost_equal(grouped._grouper.codes[0], exp_labels)
|
||
|
|
|
||
|
|
def test_list_grouper_with_nat(self):
|
||
|
|
# GH 14715
|
||
|
|
df = DataFrame({"date": date_range("1/1/2011", periods=365, freq="D")})
|
||
|
|
df.iloc[-1] = pd.NaT
|
||
|
|
grouper = Grouper(key="date", freq="YS")
|
||
|
|
|
||
|
|
# Grouper in a list grouping
|
||
|
|
result = df.groupby([grouper])
|
||
|
|
expected = {Timestamp("2011-01-01"): Index(list(range(364)))}
|
||
|
|
tm.assert_dict_equal(result.groups, expected)
|
||
|
|
|
||
|
|
# Test case without a list
|
||
|
|
result = df.groupby(grouper)
|
||
|
|
expected = {Timestamp("2011-01-01"): 365}
|
||
|
|
tm.assert_dict_equal(result.groups, expected)
|
||
|
|
|
||
|
|
@pytest.mark.parametrize(
|
||
|
|
"func,expected",
|
||
|
|
[
|
||
|
|
(
|
||
|
|
"transform",
|
||
|
|
Series(name=2, dtype=np.float64),
|
||
|
|
),
|
||
|
|
(
|
||
|
|
"agg",
|
||
|
|
Series(
|
||
|
|
name=2, dtype=np.float64, index=Index([], dtype=np.float64, name=1)
|
||
|
|
),
|
||
|
|
),
|
||
|
|
(
|
||
|
|
"apply",
|
||
|
|
Series(
|
||
|
|
name=2, dtype=np.float64, index=Index([], dtype=np.float64, name=1)
|
||
|
|
),
|
||
|
|
),
|
||
|
|
],
|
||
|
|
)
|
||
|
|
def test_evaluate_with_empty_groups(self, func, expected):
|
||
|
|
# 26208
|
||
|
|
# test transform'ing empty groups
|
||
|
|
# (not testing other agg fns, because they return
|
||
|
|
# different index objects.
|
||
|
|
df = DataFrame({1: [], 2: []})
|
||
|
|
g = df.groupby(1, group_keys=False)
|
||
|
|
result = getattr(g[2], func)(lambda x: x)
|
||
|
|
tm.assert_series_equal(result, expected)
|
||
|
|
|
||
|
|
def test_groupby_empty(self):
|
||
|
|
# https://github.com/pandas-dev/pandas/issues/27190
|
||
|
|
s = Series([], name="name", dtype="float64")
|
||
|
|
gr = s.groupby([])
|
||
|
|
|
||
|
|
result = gr.mean()
|
||
|
|
expected = s.set_axis(Index([], dtype=np.intp))
|
||
|
|
tm.assert_series_equal(result, expected)
|
||
|
|
|
||
|
|
# check group properties
|
||
|
|
assert len(gr._grouper.groupings) == 1
|
||
|
|
tm.assert_numpy_array_equal(
|
||
|
|
gr._grouper.group_info[0], np.array([], dtype=np.dtype(np.intp))
|
||
|
|
)
|
||
|
|
|
||
|
|
tm.assert_numpy_array_equal(
|
||
|
|
gr._grouper.group_info[1], np.array([], dtype=np.dtype(np.intp))
|
||
|
|
)
|
||
|
|
|
||
|
|
assert gr._grouper.group_info[2] == 0
|
||
|
|
|
||
|
|
# check name
|
||
|
|
gb = s.groupby(s)
|
||
|
|
msg = "SeriesGroupBy.grouper is deprecated"
|
||
|
|
with tm.assert_produces_warning(FutureWarning, match=msg):
|
||
|
|
grouper = gb.grouper
|
||
|
|
result = grouper.names
|
||
|
|
expected = ["name"]
|
||
|
|
assert result == expected
|
||
|
|
|
||
|
|
def test_groupby_level_index_value_all_na(self):
|
||
|
|
# issue 20519
|
||
|
|
df = DataFrame(
|
||
|
|
[["x", np.nan, 10], [None, np.nan, 20]], columns=["A", "B", "C"]
|
||
|
|
).set_index(["A", "B"])
|
||
|
|
result = df.groupby(level=["A", "B"]).sum()
|
||
|
|
expected = DataFrame(
|
||
|
|
data=[],
|
||
|
|
index=MultiIndex(
|
||
|
|
levels=[Index(["x"], dtype="object"), Index([], dtype="float64")],
|
||
|
|
codes=[[], []],
|
||
|
|
names=["A", "B"],
|
||
|
|
),
|
||
|
|
columns=["C"],
|
||
|
|
dtype="int64",
|
||
|
|
)
|
||
|
|
tm.assert_frame_equal(result, expected)
|
||
|
|
|
||
|
|
def test_groupby_multiindex_level_empty(self):
|
||
|
|
# https://github.com/pandas-dev/pandas/issues/31670
|
||
|
|
df = DataFrame(
|
||
|
|
[[123, "a", 1.0], [123, "b", 2.0]], columns=["id", "category", "value"]
|
||
|
|
)
|
||
|
|
df = df.set_index(["id", "category"])
|
||
|
|
empty = df[df.value < 0]
|
||
|
|
result = empty.groupby("id").sum()
|
||
|
|
expected = DataFrame(
|
||
|
|
dtype="float64",
|
||
|
|
columns=["value"],
|
||
|
|
index=Index([], dtype=np.int64, name="id"),
|
||
|
|
)
|
||
|
|
tm.assert_frame_equal(result, expected)
|
||
|
|
|
||
|
|
|
||
|
|
# get_group
|
||
|
|
# --------------------------------
|
||
|
|
|
||
|
|
|
||
|
|
class TestGetGroup:
|
||
|
|
def test_get_group(self):
|
||
|
|
# GH 5267
|
||
|
|
# be datelike friendly
|
||
|
|
df = DataFrame(
|
||
|
|
{
|
||
|
|
"DATE": pd.to_datetime(
|
||
|
|
[
|
||
|
|
"10-Oct-2013",
|
||
|
|
"10-Oct-2013",
|
||
|
|
"10-Oct-2013",
|
||
|
|
"11-Oct-2013",
|
||
|
|
"11-Oct-2013",
|
||
|
|
"11-Oct-2013",
|
||
|
|
]
|
||
|
|
),
|
||
|
|
"label": ["foo", "foo", "bar", "foo", "foo", "bar"],
|
||
|
|
"VAL": [1, 2, 3, 4, 5, 6],
|
||
|
|
}
|
||
|
|
)
|
||
|
|
|
||
|
|
g = df.groupby("DATE")
|
||
|
|
key = next(iter(g.groups))
|
||
|
|
result1 = g.get_group(key)
|
||
|
|
result2 = g.get_group(Timestamp(key).to_pydatetime())
|
||
|
|
result3 = g.get_group(str(Timestamp(key)))
|
||
|
|
tm.assert_frame_equal(result1, result2)
|
||
|
|
tm.assert_frame_equal(result1, result3)
|
||
|
|
|
||
|
|
g = df.groupby(["DATE", "label"])
|
||
|
|
|
||
|
|
key = next(iter(g.groups))
|
||
|
|
result1 = g.get_group(key)
|
||
|
|
result2 = g.get_group((Timestamp(key[0]).to_pydatetime(), key[1]))
|
||
|
|
result3 = g.get_group((str(Timestamp(key[0])), key[1]))
|
||
|
|
tm.assert_frame_equal(result1, result2)
|
||
|
|
tm.assert_frame_equal(result1, result3)
|
||
|
|
|
||
|
|
# must pass a same-length tuple with multiple keys
|
||
|
|
msg = "must supply a tuple to get_group with multiple grouping keys"
|
||
|
|
with pytest.raises(ValueError, match=msg):
|
||
|
|
g.get_group("foo")
|
||
|
|
with pytest.raises(ValueError, match=msg):
|
||
|
|
g.get_group("foo")
|
||
|
|
msg = "must supply a same-length tuple to get_group with multiple grouping keys"
|
||
|
|
with pytest.raises(ValueError, match=msg):
|
||
|
|
g.get_group(("foo", "bar", "baz"))
|
||
|
|
|
||
|
|
def test_get_group_empty_bins(self, observed):
|
||
|
|
d = DataFrame([3, 1, 7, 6])
|
||
|
|
bins = [0, 5, 10, 15]
|
||
|
|
g = d.groupby(pd.cut(d[0], bins), observed=observed)
|
||
|
|
|
||
|
|
# TODO: should prob allow a str of Interval work as well
|
||
|
|
# IOW '(0, 5]'
|
||
|
|
result = g.get_group(pd.Interval(0, 5))
|
||
|
|
expected = DataFrame([3, 1], index=[0, 1])
|
||
|
|
tm.assert_frame_equal(result, expected)
|
||
|
|
|
||
|
|
msg = r"Interval\(10, 15, closed='right'\)"
|
||
|
|
with pytest.raises(KeyError, match=msg):
|
||
|
|
g.get_group(pd.Interval(10, 15))
|
||
|
|
|
||
|
|
def test_get_group_grouped_by_tuple(self):
|
||
|
|
# GH 8121
|
||
|
|
df = DataFrame([[(1,), (1, 2), (1,), (1, 2)]], index=["ids"]).T
|
||
|
|
gr = df.groupby("ids")
|
||
|
|
expected = DataFrame({"ids": [(1,), (1,)]}, index=[0, 2])
|
||
|
|
result = gr.get_group((1,))
|
||
|
|
tm.assert_frame_equal(result, expected)
|
||
|
|
|
||
|
|
dt = pd.to_datetime(["2010-01-01", "2010-01-02", "2010-01-01", "2010-01-02"])
|
||
|
|
df = DataFrame({"ids": [(x,) for x in dt]})
|
||
|
|
gr = df.groupby("ids")
|
||
|
|
result = gr.get_group(("2010-01-01",))
|
||
|
|
expected = DataFrame({"ids": [(dt[0],), (dt[0],)]}, index=[0, 2])
|
||
|
|
tm.assert_frame_equal(result, expected)
|
||
|
|
|
||
|
|
def test_get_group_grouped_by_tuple_with_lambda(self):
|
||
|
|
# GH 36158
|
||
|
|
df = DataFrame(
|
||
|
|
{
|
||
|
|
"Tuples": (
|
||
|
|
(x, y)
|
||
|
|
for x in [0, 1]
|
||
|
|
for y in np.random.default_rng(2).integers(3, 5, 5)
|
||
|
|
)
|
||
|
|
}
|
||
|
|
)
|
||
|
|
|
||
|
|
gb = df.groupby("Tuples")
|
||
|
|
gb_lambda = df.groupby(lambda x: df.iloc[x, 0])
|
||
|
|
|
||
|
|
expected = gb.get_group(next(iter(gb.groups.keys())))
|
||
|
|
result = gb_lambda.get_group(next(iter(gb_lambda.groups.keys())))
|
||
|
|
|
||
|
|
tm.assert_frame_equal(result, expected)
|
||
|
|
|
||
|
|
def test_groupby_with_empty(self):
|
||
|
|
index = pd.DatetimeIndex(())
|
||
|
|
data = ()
|
||
|
|
series = Series(data, index, dtype=object)
|
||
|
|
grouper = Grouper(freq="D")
|
||
|
|
grouped = series.groupby(grouper)
|
||
|
|
assert next(iter(grouped), None) is None
|
||
|
|
|
||
|
|
def test_groupby_with_single_column(self):
|
||
|
|
df = DataFrame({"a": list("abssbab")})
|
||
|
|
tm.assert_frame_equal(df.groupby("a").get_group("a"), df.iloc[[0, 5]])
|
||
|
|
# GH 13530
|
||
|
|
exp = DataFrame(index=Index(["a", "b", "s"], name="a"), columns=[])
|
||
|
|
tm.assert_frame_equal(df.groupby("a").count(), exp)
|
||
|
|
tm.assert_frame_equal(df.groupby("a").sum(), exp)
|
||
|
|
|
||
|
|
exp = df.iloc[[3, 4, 5]]
|
||
|
|
tm.assert_frame_equal(df.groupby("a").nth(1), exp)
|
||
|
|
|
||
|
|
def test_gb_key_len_equal_axis_len(self):
|
||
|
|
# GH16843
|
||
|
|
# test ensures that index and column keys are recognized correctly
|
||
|
|
# when number of keys equals axis length of groupby
|
||
|
|
df = DataFrame(
|
||
|
|
[["foo", "bar", "B", 1], ["foo", "bar", "B", 2], ["foo", "baz", "C", 3]],
|
||
|
|
columns=["first", "second", "third", "one"],
|
||
|
|
)
|
||
|
|
df = df.set_index(["first", "second"])
|
||
|
|
df = df.groupby(["first", "second", "third"]).size()
|
||
|
|
assert df.loc[("foo", "bar", "B")] == 2
|
||
|
|
assert df.loc[("foo", "baz", "C")] == 1
|
||
|
|
|
||
|
|
|
||
|
|
# groups & iteration
|
||
|
|
# --------------------------------
|
||
|
|
|
||
|
|
|
||
|
|
class TestIteration:
|
||
|
|
def test_groups(self, df):
|
||
|
|
grouped = df.groupby(["A"])
|
||
|
|
groups = grouped.groups
|
||
|
|
assert groups is grouped.groups # caching works
|
||
|
|
|
||
|
|
for k, v in grouped.groups.items():
|
||
|
|
assert (df.loc[v]["A"] == k).all()
|
||
|
|
|
||
|
|
grouped = df.groupby(["A", "B"])
|
||
|
|
groups = grouped.groups
|
||
|
|
assert groups is grouped.groups # caching works
|
||
|
|
|
||
|
|
for k, v in grouped.groups.items():
|
||
|
|
assert (df.loc[v]["A"] == k[0]).all()
|
||
|
|
assert (df.loc[v]["B"] == k[1]).all()
|
||
|
|
|
||
|
|
def test_grouping_is_iterable(self, tsframe):
|
||
|
|
# this code path isn't used anywhere else
|
||
|
|
# not sure it's useful
|
||
|
|
grouped = tsframe.groupby([lambda x: x.weekday(), lambda x: x.year])
|
||
|
|
|
||
|
|
# test it works
|
||
|
|
for g in grouped._grouper.groupings[0]:
|
||
|
|
pass
|
||
|
|
|
||
|
|
def test_multi_iter(self):
|
||
|
|
s = Series(np.arange(6))
|
||
|
|
k1 = np.array(["a", "a", "a", "b", "b", "b"])
|
||
|
|
k2 = np.array(["1", "2", "1", "2", "1", "2"])
|
||
|
|
|
||
|
|
grouped = s.groupby([k1, k2])
|
||
|
|
|
||
|
|
iterated = list(grouped)
|
||
|
|
expected = [
|
||
|
|
("a", "1", s[[0, 2]]),
|
||
|
|
("a", "2", s[[1]]),
|
||
|
|
("b", "1", s[[4]]),
|
||
|
|
("b", "2", s[[3, 5]]),
|
||
|
|
]
|
||
|
|
for i, ((one, two), three) in enumerate(iterated):
|
||
|
|
e1, e2, e3 = expected[i]
|
||
|
|
assert e1 == one
|
||
|
|
assert e2 == two
|
||
|
|
tm.assert_series_equal(three, e3)
|
||
|
|
|
||
|
|
def test_multi_iter_frame(self, three_group):
|
||
|
|
k1 = np.array(["b", "b", "b", "a", "a", "a"])
|
||
|
|
k2 = np.array(["1", "2", "1", "2", "1", "2"])
|
||
|
|
df = DataFrame(
|
||
|
|
{
|
||
|
|
"v1": np.random.default_rng(2).standard_normal(6),
|
||
|
|
"v2": np.random.default_rng(2).standard_normal(6),
|
||
|
|
"k1": k1,
|
||
|
|
"k2": k2,
|
||
|
|
},
|
||
|
|
index=["one", "two", "three", "four", "five", "six"],
|
||
|
|
)
|
||
|
|
|
||
|
|
grouped = df.groupby(["k1", "k2"])
|
||
|
|
|
||
|
|
# things get sorted!
|
||
|
|
iterated = list(grouped)
|
||
|
|
idx = df.index
|
||
|
|
expected = [
|
||
|
|
("a", "1", df.loc[idx[[4]]]),
|
||
|
|
("a", "2", df.loc[idx[[3, 5]]]),
|
||
|
|
("b", "1", df.loc[idx[[0, 2]]]),
|
||
|
|
("b", "2", df.loc[idx[[1]]]),
|
||
|
|
]
|
||
|
|
for i, ((one, two), three) in enumerate(iterated):
|
||
|
|
e1, e2, e3 = expected[i]
|
||
|
|
assert e1 == one
|
||
|
|
assert e2 == two
|
||
|
|
tm.assert_frame_equal(three, e3)
|
||
|
|
|
||
|
|
# don't iterate through groups with no data
|
||
|
|
df["k1"] = np.array(["b", "b", "b", "a", "a", "a"])
|
||
|
|
df["k2"] = np.array(["1", "1", "1", "2", "2", "2"])
|
||
|
|
grouped = df.groupby(["k1", "k2"])
|
||
|
|
# calling `dict` on a DataFrameGroupBy leads to a TypeError,
|
||
|
|
# we need to use a dictionary comprehension here
|
||
|
|
# pylint: disable-next=unnecessary-comprehension
|
||
|
|
groups = {key: gp for key, gp in grouped} # noqa: C416
|
||
|
|
assert len(groups) == 2
|
||
|
|
|
||
|
|
# axis = 1
|
||
|
|
three_levels = three_group.groupby(["A", "B", "C"]).mean()
|
||
|
|
depr_msg = "DataFrame.groupby with axis=1 is deprecated"
|
||
|
|
with tm.assert_produces_warning(FutureWarning, match=depr_msg):
|
||
|
|
grouped = three_levels.T.groupby(axis=1, level=(1, 2))
|
||
|
|
for key, group in grouped:
|
||
|
|
pass
|
||
|
|
|
||
|
|
def test_dictify(self, df):
|
||
|
|
dict(iter(df.groupby("A")))
|
||
|
|
dict(iter(df.groupby(["A", "B"])))
|
||
|
|
dict(iter(df["C"].groupby(df["A"])))
|
||
|
|
dict(iter(df["C"].groupby([df["A"], df["B"]])))
|
||
|
|
dict(iter(df.groupby("A")["C"]))
|
||
|
|
dict(iter(df.groupby(["A", "B"])["C"]))
|
||
|
|
|
||
|
|
def test_groupby_with_small_elem(self):
|
||
|
|
# GH 8542
|
||
|
|
# length=2
|
||
|
|
df = DataFrame(
|
||
|
|
{"event": ["start", "start"], "change": [1234, 5678]},
|
||
|
|
index=pd.DatetimeIndex(["2014-09-10", "2013-10-10"]),
|
||
|
|
)
|
||
|
|
grouped = df.groupby([Grouper(freq="ME"), "event"])
|
||
|
|
assert len(grouped.groups) == 2
|
||
|
|
assert grouped.ngroups == 2
|
||
|
|
assert (Timestamp("2014-09-30"), "start") in grouped.groups
|
||
|
|
assert (Timestamp("2013-10-31"), "start") in grouped.groups
|
||
|
|
|
||
|
|
res = grouped.get_group((Timestamp("2014-09-30"), "start"))
|
||
|
|
tm.assert_frame_equal(res, df.iloc[[0], :])
|
||
|
|
res = grouped.get_group((Timestamp("2013-10-31"), "start"))
|
||
|
|
tm.assert_frame_equal(res, df.iloc[[1], :])
|
||
|
|
|
||
|
|
df = DataFrame(
|
||
|
|
{"event": ["start", "start", "start"], "change": [1234, 5678, 9123]},
|
||
|
|
index=pd.DatetimeIndex(["2014-09-10", "2013-10-10", "2014-09-15"]),
|
||
|
|
)
|
||
|
|
grouped = df.groupby([Grouper(freq="ME"), "event"])
|
||
|
|
assert len(grouped.groups) == 2
|
||
|
|
assert grouped.ngroups == 2
|
||
|
|
assert (Timestamp("2014-09-30"), "start") in grouped.groups
|
||
|
|
assert (Timestamp("2013-10-31"), "start") in grouped.groups
|
||
|
|
|
||
|
|
res = grouped.get_group((Timestamp("2014-09-30"), "start"))
|
||
|
|
tm.assert_frame_equal(res, df.iloc[[0, 2], :])
|
||
|
|
res = grouped.get_group((Timestamp("2013-10-31"), "start"))
|
||
|
|
tm.assert_frame_equal(res, df.iloc[[1], :])
|
||
|
|
|
||
|
|
# length=3
|
||
|
|
df = DataFrame(
|
||
|
|
{"event": ["start", "start", "start"], "change": [1234, 5678, 9123]},
|
||
|
|
index=pd.DatetimeIndex(["2014-09-10", "2013-10-10", "2014-08-05"]),
|
||
|
|
)
|
||
|
|
grouped = df.groupby([Grouper(freq="ME"), "event"])
|
||
|
|
assert len(grouped.groups) == 3
|
||
|
|
assert grouped.ngroups == 3
|
||
|
|
assert (Timestamp("2014-09-30"), "start") in grouped.groups
|
||
|
|
assert (Timestamp("2013-10-31"), "start") in grouped.groups
|
||
|
|
assert (Timestamp("2014-08-31"), "start") in grouped.groups
|
||
|
|
|
||
|
|
res = grouped.get_group((Timestamp("2014-09-30"), "start"))
|
||
|
|
tm.assert_frame_equal(res, df.iloc[[0], :])
|
||
|
|
res = grouped.get_group((Timestamp("2013-10-31"), "start"))
|
||
|
|
tm.assert_frame_equal(res, df.iloc[[1], :])
|
||
|
|
res = grouped.get_group((Timestamp("2014-08-31"), "start"))
|
||
|
|
tm.assert_frame_equal(res, df.iloc[[2], :])
|
||
|
|
|
||
|
|
def test_grouping_string_repr(self):
|
||
|
|
# GH 13394
|
||
|
|
mi = MultiIndex.from_arrays([list("AAB"), list("aba")])
|
||
|
|
df = DataFrame([[1, 2, 3]], columns=mi)
|
||
|
|
gr = df.groupby(df[("A", "a")])
|
||
|
|
|
||
|
|
result = gr._grouper.groupings[0].__repr__()
|
||
|
|
expected = "Grouping(('A', 'a'))"
|
||
|
|
assert result == expected
|
||
|
|
|
||
|
|
|
||
|
|
def test_grouping_by_key_is_in_axis():
|
||
|
|
# GH#50413 - Groupers specified by key are in-axis
|
||
|
|
df = DataFrame({"a": [1, 1, 2], "b": [1, 1, 2], "c": [3, 4, 5]}).set_index("a")
|
||
|
|
gb = df.groupby([Grouper(level="a"), Grouper(key="b")], as_index=False)
|
||
|
|
assert not gb._grouper.groupings[0].in_axis
|
||
|
|
assert gb._grouper.groupings[1].in_axis
|
||
|
|
|
||
|
|
# Currently only in-axis groupings are including in the result when as_index=False;
|
||
|
|
# This is likely to change in the future.
|
||
|
|
msg = "A grouping .* was excluded from the result"
|
||
|
|
with tm.assert_produces_warning(FutureWarning, match=msg):
|
||
|
|
result = gb.sum()
|
||
|
|
expected = DataFrame({"b": [1, 2], "c": [7, 5]})
|
||
|
|
tm.assert_frame_equal(result, expected)
|
||
|
|
|
||
|
|
|
||
|
|
def test_grouper_groups():
|
||
|
|
# GH#51182 check Grouper.groups does not raise AttributeError
|
||
|
|
df = DataFrame({"a": [1, 2, 3], "b": 1})
|
||
|
|
grper = Grouper(key="a")
|
||
|
|
gb = df.groupby(grper)
|
||
|
|
|
||
|
|
msg = "Use GroupBy.groups instead"
|
||
|
|
with tm.assert_produces_warning(FutureWarning, match=msg):
|
||
|
|
res = grper.groups
|
||
|
|
assert res is gb.groups
|
||
|
|
|
||
|
|
msg = "Use GroupBy.grouper instead"
|
||
|
|
with tm.assert_produces_warning(FutureWarning, match=msg):
|
||
|
|
res = grper.grouper
|
||
|
|
assert res is gb._grouper
|
||
|
|
|
||
|
|
msg = "Grouper.obj is deprecated and will be removed"
|
||
|
|
with tm.assert_produces_warning(FutureWarning, match=msg):
|
||
|
|
res = grper.obj
|
||
|
|
assert res is gb.obj
|
||
|
|
|
||
|
|
msg = "Use Resampler.ax instead"
|
||
|
|
with tm.assert_produces_warning(FutureWarning, match=msg):
|
||
|
|
grper.ax
|
||
|
|
|
||
|
|
msg = "Grouper.indexer is deprecated"
|
||
|
|
with tm.assert_produces_warning(FutureWarning, match=msg):
|
||
|
|
grper.indexer
|
||
|
|
|
||
|
|
|
||
|
|
@pytest.mark.parametrize("attr", ["group_index", "result_index", "group_arraylike"])
|
||
|
|
def test_depr_grouping_attrs(attr):
|
||
|
|
# GH#56148
|
||
|
|
df = DataFrame({"a": [1, 1, 2], "b": [3, 4, 5]})
|
||
|
|
gb = df.groupby("a")
|
||
|
|
msg = f"{attr} is deprecated"
|
||
|
|
with tm.assert_produces_warning(FutureWarning, match=msg):
|
||
|
|
getattr(gb._grouper.groupings[0], attr)
|