s464967/Inzynierka
2
1
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
bc721132bc
Inzynierka/Lib/site-packages/pandas/core/groupby/generic.py

2652 lines
86 KiB
Python
Raw Normal View History

first commit
2023-06-02 12:51:02 +02:00
"""
Define the SeriesGroupBy and DataFrameGroupBy
classes that hold the groupby interfaces (and some implementations).
These are user facing as the result of the ``df.groupby(...)`` operations,
which here returns a DataFrameGroupBy object.
"""
from __future__ import annotations
from collections import abc
from functools import partial
from textwrap import dedent
from typing import (
TYPE_CHECKING,
Any,
Callable,
Hashable,
Iterable,
Literal,
Mapping,
NamedTuple,
Sequence,
TypeVar,
Union,
cast,
)
import numpy as np
from pandas._libs import (
Interval,
lib,
reduction as libreduction,
)
from pandas._typing import (
ArrayLike,
Axis,
AxisInt,
CorrelationMethod,
FillnaOptions,
IndexLabel,
Manager,
Manager2D,
SingleManager,
TakeIndexer,
)
from pandas.errors import SpecificationError
from pandas.util._decorators import (
Appender,
Substitution,
doc,
)
from pandas.core.dtypes.common import (
ensure_int64,
is_bool,
is_categorical_dtype,
is_dict_like,
is_integer_dtype,
is_interval_dtype,
is_numeric_dtype,
is_scalar,
)
from pandas.core.dtypes.missing import (
isna,
notna,
)
from pandas.core import algorithms
from pandas.core.apply import (
GroupByApply,
maybe_mangle_lambdas,
reconstruct_func,
validate_func_kwargs,
)
import pandas.core.common as com
from pandas.core.frame import DataFrame
from pandas.core.groupby import base
from pandas.core.groupby.groupby import (
GroupBy,
GroupByPlot,
_agg_template,
_apply_docs,
_transform_template,
)
from pandas.core.indexes.api import (
Index,
MultiIndex,
all_indexes_same,
default_index,
)
from pandas.core.series import Series
from pandas.core.util.numba_ import maybe_use_numba
from pandas.plotting import boxplot_frame_groupby
if TYPE_CHECKING:
from pandas import Categorical
from pandas.core.generic import NDFrame
# TODO(typing) the return value on this callable should be any *scalar*.
AggScalar = Union[str, Callable[..., Any]]
# TODO: validate types on ScalarResult and move to _typing
# Blocked from using by https://github.com/python/mypy/issues/1484
# See note at _mangle_lambda_list
ScalarResult = TypeVar("ScalarResult")
class NamedAgg(NamedTuple):
"""
Helper for column specific aggregation with control over output column names.
Subclass of typing.NamedTuple.
Parameters
----------
column : Hashable
Column label in the DataFrame to apply aggfunc.
aggfunc : function or str
Function to apply to the provided column. If string, the name of a built-in
pandas function.
Examples
--------
>>> df = pd.DataFrame({"key": [1, 1, 2], "a": [-1, 0, 1], 1: [10, 11, 12]})
>>> agg_a = pd.NamedAgg(column="a", aggfunc="min")
>>> agg_1 = pd.NamedAgg(column=1, aggfunc=np.mean)
>>> df.groupby("key").agg(result_a=agg_a, result_1=agg_1)
result_a result_1
key
1 -1 10.5
2 1 12.0
"""
column: Hashable
aggfunc: AggScalar
class SeriesGroupBy(GroupBy[Series]):
def _wrap_agged_manager(self, mgr: Manager) -> Series:
return self.obj._constructor(mgr, name=self.obj.name)
def _get_data_to_aggregate(
self, *, numeric_only: bool = False, name: str | None = None
) -> SingleManager:
ser = self._selected_obj
single = ser._mgr
if numeric_only and not is_numeric_dtype(ser.dtype):
# GH#41291 match Series behavior
kwd_name = "numeric_only"
raise TypeError(
f"Cannot use {kwd_name}=True with "
f"{type(self).__name__}.{name} and non-numeric dtypes."
)
return single
def _iterate_slices(self) -> Iterable[Series]:
yield self._selected_obj
_agg_examples_doc = dedent(
"""
Examples
--------
>>> s = pd.Series([1, 2, 3, 4])
>>> s
0 1
1 2
2 3
3 4
dtype: int64
>>> s.groupby([1, 1, 2, 2]).min()
1 1
2 3
dtype: int64
>>> s.groupby([1, 1, 2, 2]).agg('min')
1 1
2 3
dtype: int64
>>> s.groupby([1, 1, 2, 2]).agg(['min', 'max'])
min max
1 1 2
2 3 4
The output column names can be controlled by passing
the desired column names and aggregations as keyword arguments.
>>> s.groupby([1, 1, 2, 2]).agg(
... minimum='min',
... maximum='max',
... )
minimum maximum
1 1 2
2 3 4
.. versionchanged:: 1.3.0
The resulting dtype will reflect the return value of the aggregating function.
>>> s.groupby([1, 1, 2, 2]).agg(lambda x: x.astype(float).min())
1 1.0
2 3.0
dtype: float64
"""
)
@Appender(
_apply_docs["template"].format(
input="series", examples=_apply_docs["series_examples"]
)
)
def apply(self, func, *args, **kwargs) -> Series:
return super().apply(func, *args, **kwargs)
@doc(_agg_template, examples=_agg_examples_doc, klass="Series")
def aggregate(self, func=None, *args, engine=None, engine_kwargs=None, **kwargs):
if maybe_use_numba(engine):
return self._aggregate_with_numba(
func, *args, engine_kwargs=engine_kwargs, **kwargs
)
relabeling = func is None
columns = None
if relabeling:
columns, func = validate_func_kwargs(kwargs)
kwargs = {}
if isinstance(func, str):
return getattr(self, func)(*args, **kwargs)
elif isinstance(func, abc.Iterable):
# Catch instances of lists / tuples
# but not the class list / tuple itself.
func = maybe_mangle_lambdas(func)
ret = self._aggregate_multiple_funcs(func, *args, **kwargs)
if relabeling:
# columns is not narrowed by mypy from relabeling flag
assert columns is not None # for mypy
ret.columns = columns
if not self.as_index:
ret = ret.reset_index()
return ret
else:
cyfunc = com.get_cython_func(func)
if cyfunc and not args and not kwargs:
return getattr(self, cyfunc)()
if self.ngroups == 0:
# e.g. test_evaluate_with_empty_groups without any groups to
# iterate over, we have no output on which to do dtype
# inference. We default to using the existing dtype.
# xref GH#51445
obj = self._obj_with_exclusions
return self.obj._constructor(
[],
name=self.obj.name,
index=self.grouper.result_index,
dtype=obj.dtype,
)
if self.grouper.nkeys > 1:
return self._python_agg_general(func, *args, **kwargs)
try:
return self._python_agg_general(func, *args, **kwargs)
except KeyError:
# KeyError raised in test_groupby.test_basic is bc the func does
# a dictionary lookup on group.name, but group name is not
# pinned in _python_agg_general, only in _aggregate_named
result = self._aggregate_named(func, *args, **kwargs)
# result is a dict whose keys are the elements of result_index
result = Series(result, index=self.grouper.result_index)
result = self._wrap_aggregated_output(result)
return result
agg = aggregate
def _python_agg_general(self, func, *args, **kwargs):
func = com.is_builtin_func(func)
f = lambda x: func(x, *args, **kwargs)
obj = self._obj_with_exclusions
result = self.grouper.agg_series(obj, f)
res = obj._constructor(result, name=obj.name)
return self._wrap_aggregated_output(res)
def _aggregate_multiple_funcs(self, arg, *args, **kwargs) -> DataFrame:
if isinstance(arg, dict):
if self.as_index:
# GH 15931
raise SpecificationError("nested renamer is not supported")
else:
# GH#50684 - This accidentally worked in 1.x
arg = list(arg.items())
elif any(isinstance(x, (tuple, list)) for x in arg):
arg = [(x, x) if not isinstance(x, (tuple, list)) else x for x in arg]
else:
# list of functions / function names
columns = []
for f in arg:
columns.append(com.get_callable_name(f) or f)
arg = zip(columns, arg)
results: dict[base.OutputKey, DataFrame | Series] = {}
with com.temp_setattr(self, "as_index", True):
# Combine results using the index, need to adjust index after
# if as_index=False (GH#50724)
for idx, (name, func) in enumerate(arg):
key = base.OutputKey(label=name, position=idx)
results[key] = self.aggregate(func, *args, **kwargs)
if any(isinstance(x, DataFrame) for x in results.values()):
from pandas import concat
res_df = concat(
results.values(), axis=1, keys=[key.label for key in results]
)
return res_df
indexed_output = {key.position: val for key, val in results.items()}
output = self.obj._constructor_expanddim(indexed_output, index=None)
output.columns = Index(key.label for key in results)
return output
def _wrap_applied_output(
self,
data: Series,
values: list[Any],
not_indexed_same: bool = False,
is_transform: bool = False,
) -> DataFrame | Series:
"""
Wrap the output of SeriesGroupBy.apply into the expected result.
Parameters
----------
data : Series
Input data for groupby operation.
values : List[Any]
Applied output for each group.
not_indexed_same : bool, default False
Whether the applied outputs are not indexed the same as the group axes.
Returns
-------
DataFrame or Series
"""
if len(values) == 0:
# GH #6265
if is_transform:
# GH#47787 see test_group_on_empty_multiindex
res_index = data.index
else:
res_index = self.grouper.result_index
return self.obj._constructor(
[],
name=self.obj.name,
index=res_index,
dtype=data.dtype,
)
assert values is not None
if isinstance(values[0], dict):
# GH #823 #24880
index = self.grouper.result_index
res_df = self.obj._constructor_expanddim(values, index=index)
res_df = self._reindex_output(res_df)
# if self.observed is False,
# keep all-NaN rows created while re-indexing
res_ser = res_df.stack(dropna=self.observed)
res_ser.name = self.obj.name
return res_ser
elif isinstance(values[0], (Series, DataFrame)):
result = self._concat_objects(
values,
not_indexed_same=not_indexed_same,
is_transform=is_transform,
)
if isinstance(result, Series):
result.name = self.obj.name
if not self.as_index and not_indexed_same:
result = self._insert_inaxis_grouper(result)
result.index = default_index(len(result))
return result
else:
# GH #6265 #24880
result = self.obj._constructor(
data=values, index=self.grouper.result_index, name=self.obj.name
)
if not self.as_index:
result = self._insert_inaxis_grouper(result)
result.index = default_index(len(result))
return self._reindex_output(result)
def _aggregate_named(self, func, *args, **kwargs):
# Note: this is very similar to _aggregate_series_pure_python,
# but that does not pin group.name
result = {}
initialized = False
for name, group in self:
object.__setattr__(group, "name", name)
output = func(group, *args, **kwargs)
output = libreduction.extract_result(output)
if not initialized:
# We only do this validation on the first iteration
libreduction.check_result_array(output, group.dtype)
initialized = True
result[name] = output
return result
__examples_series_doc = dedent(
"""
>>> ser = pd.Series(
... [390.0, 350.0, 30.0, 20.0],
... index=["Falcon", "Falcon", "Parrot", "Parrot"],
... name="Max Speed")
>>> grouped = ser.groupby([1, 1, 2, 2])
>>> grouped.transform(lambda x: (x - x.mean()) / x.std())
Falcon 0.707107
Falcon -0.707107
Parrot 0.707107
Parrot -0.707107
Name: Max Speed, dtype: float64
Broadcast result of the transformation
>>> grouped.transform(lambda x: x.max() - x.min())
Falcon 40.0
Falcon 40.0
Parrot 10.0
Parrot 10.0
Name: Max Speed, dtype: float64
>>> grouped.transform("mean")
Falcon 370.0
Falcon 370.0
Parrot 25.0
Parrot 25.0
Name: Max Speed, dtype: float64
.. versionchanged:: 1.3.0
The resulting dtype will reflect the return value of the passed ``func``,
for example:
>>> grouped.transform(lambda x: x.astype(int).max())
Falcon 390
Falcon 390
Parrot 30
Parrot 30
Name: Max Speed, dtype: int64
"""
)
@Substitution(klass="Series", example=__examples_series_doc)
@Appender(_transform_template)
def transform(self, func, *args, engine=None, engine_kwargs=None, **kwargs):
return self._transform(
func, *args, engine=engine, engine_kwargs=engine_kwargs, **kwargs
)
def _cython_transform(
self, how: str, numeric_only: bool = False, axis: AxisInt = 0, **kwargs
):
assert axis == 0 # handled by caller
obj = self._selected_obj
try:
result = self.grouper._cython_operation(
"transform", obj._values, how, axis, **kwargs
)
except NotImplementedError as err:
# e.g. test_groupby_raises_string
raise TypeError(f"{how} is not supported for {obj.dtype} dtype") from err
return obj._constructor(result, index=self.obj.index, name=obj.name)
def _transform_general(self, func: Callable, *args, **kwargs) -> Series:
"""
Transform with a callable func`.
"""
assert callable(func)
klass = type(self.obj)
results = []
for name, group in self.grouper.get_iterator(
self._selected_obj, axis=self.axis
):
# this setattr is needed for test_transform_lambda_with_datetimetz
object.__setattr__(group, "name", name)
res = func(group, *args, **kwargs)
results.append(klass(res, index=group.index))
# check for empty "results" to avoid concat ValueError
if results:
from pandas.core.reshape.concat import concat
concatenated = concat(results)
result = self._set_result_index_ordered(concatenated)
else:
result = self.obj._constructor(dtype=np.float64)
result.name = self.obj.name
return result
def filter(self, func, dropna: bool = True, *args, **kwargs):
"""
Filter elements from groups that don't satisfy a criterion.
Elements from groups are filtered if they do not satisfy the
boolean criterion specified by func.
Parameters
----------
func : function
Criterion to apply to each group. Should return True or False.
dropna : bool
Drop groups that do not pass the filter. True by default; if False,
groups that evaluate False are filled with NaNs.
Returns
-------
Series
Notes
-----
Functions that mutate the passed object can produce unexpected
behavior or errors and are not supported. See :ref:`gotchas.udf-mutation`
for more details.
Examples
--------
>>> df = pd.DataFrame({'A' : ['foo', 'bar', 'foo', 'bar',
... 'foo', 'bar'],
... 'B' : [1, 2, 3, 4, 5, 6],
... 'C' : [2.0, 5., 8., 1., 2., 9.]})
>>> grouped = df.groupby('A')
>>> df.groupby('A').B.filter(lambda x: x.mean() > 3.)
1 2
3 4
5 6
Name: B, dtype: int64
"""
if isinstance(func, str):
wrapper = lambda x: getattr(x, func)(*args, **kwargs)
else:
wrapper = lambda x: func(x, *args, **kwargs)
# Interpret np.nan as False.
def true_and_notna(x) -> bool:
b = wrapper(x)
return notna(b) and b
try:
indices = [
self._get_index(name) for name, group in self if true_and_notna(group)
]
except (ValueError, TypeError) as err:
raise TypeError("the filter must return a boolean result") from err
filtered = self._apply_filter(indices, dropna)
return filtered
def nunique(self, dropna: bool = True) -> Series | DataFrame:
"""
Return number of unique elements in the group.
Returns
-------
Series
Number of unique values within each group.
"""
ids, _, _ = self.grouper.group_info
val = self.obj._values
codes, _ = algorithms.factorize(val, sort=False)
sorter = np.lexsort((codes, ids))
codes = codes[sorter]
ids = ids[sorter]
# group boundaries are where group ids change
# unique observations are where sorted values change
idx = np.r_[0, 1 + np.nonzero(ids[1:] != ids[:-1])[0]]
inc = np.r_[1, codes[1:] != codes[:-1]]
# 1st item of each group is a new unique observation
mask = codes == -1
if dropna:
inc[idx] = 1
inc[mask] = 0
else:
inc[mask & np.r_[False, mask[:-1]]] = 0
inc[idx] = 1
out = np.add.reduceat(inc, idx).astype("int64", copy=False)
if len(ids):
# NaN/NaT group exists if the head of ids is -1,
# so remove it from res and exclude its index from idx
if ids[0] == -1:
res = out[1:]
idx = idx[np.flatnonzero(idx)]
else:
res = out
else:
res = out[1:]
ri = self.grouper.result_index
# we might have duplications among the bins
if len(res) != len(ri):
res, out = np.zeros(len(ri), dtype=out.dtype), res
if len(ids) > 0:
# GH#21334s
res[ids[idx]] = out
result: Series | DataFrame = self.obj._constructor(
res, index=ri, name=self.obj.name
)
if not self.as_index:
result = self._insert_inaxis_grouper(result)
result.index = default_index(len(result))
return self._reindex_output(result, fill_value=0)
@doc(Series.describe)
def describe(self, **kwargs):
return super().describe(**kwargs)
def value_counts(
self,
normalize: bool = False,
sort: bool = True,
ascending: bool = False,
bins=None,
dropna: bool = True,
) -> Series | DataFrame:
name = "proportion" if normalize else "count"
if bins is None:
result = self._value_counts(
normalize=normalize, sort=sort, ascending=ascending, dropna=dropna
)
result.name = name
return result
from pandas.core.reshape.merge import get_join_indexers
from pandas.core.reshape.tile import cut
ids, _, _ = self.grouper.group_info
val = self.obj._values
index_names = self.grouper.names + [self.obj.name]
if is_categorical_dtype(val.dtype) or (
bins is not None and not np.iterable(bins)
):
# scalar bins cannot be done at top level
# in a backward compatible way
# GH38672 relates to categorical dtype
ser = self.apply(
Series.value_counts,
normalize=normalize,
sort=sort,
ascending=ascending,
bins=bins,
)
ser.name = name
ser.index.names = index_names
return ser
# groupby removes null keys from groupings
mask = ids != -1
ids, val = ids[mask], val[mask]
if bins is None:
lab, lev = algorithms.factorize(val, sort=True)
llab = lambda lab, inc: lab[inc]
else:
# lab is a Categorical with categories an IntervalIndex
cat_ser = cut(Series(val, copy=False), bins, include_lowest=True)
cat_obj = cast("Categorical", cat_ser._values)
lev = cat_obj.categories
lab = lev.take(
cat_obj.codes,
allow_fill=True,
fill_value=lev._na_value,
)
llab = lambda lab, inc: lab[inc]._multiindex.codes[-1]
if is_interval_dtype(lab.dtype):
# TODO: should we do this inside II?
lab_interval = cast(Interval, lab)
sorter = np.lexsort((lab_interval.left, lab_interval.right, ids))
else:
sorter = np.lexsort((lab, ids))
ids, lab = ids[sorter], lab[sorter]
# group boundaries are where group ids change
idchanges = 1 + np.nonzero(ids[1:] != ids[:-1])[0]
idx = np.r_[0, idchanges]
if not len(ids):
idx = idchanges
# new values are where sorted labels change
lchanges = llab(lab, slice(1, None)) != llab(lab, slice(None, -1))
inc = np.r_[True, lchanges]
if not len(val):
inc = lchanges
inc[idx] = True # group boundaries are also new values
out = np.diff(np.nonzero(np.r_[inc, True])[0]) # value counts
# num. of times each group should be repeated
rep = partial(np.repeat, repeats=np.add.reduceat(inc, idx))
# multi-index components
codes = self.grouper.reconstructed_codes
codes = [rep(level_codes) for level_codes in codes] + [llab(lab, inc)]
levels = [ping.group_index for ping in self.grouper.groupings] + [lev]
if dropna:
mask = codes[-1] != -1
if mask.all():
dropna = False
else:
out, codes = out[mask], [level_codes[mask] for level_codes in codes]
if normalize:
out = out.astype("float")
d = np.diff(np.r_[idx, len(ids)])
if dropna:
m = ids[lab == -1]
np.add.at(d, m, -1)
acc = rep(d)[mask]
else:
acc = rep(d)
out /= acc
if sort and bins is None:
cat = ids[inc][mask] if dropna else ids[inc]
sorter = np.lexsort((out if ascending else -out, cat))
out, codes[-1] = out[sorter], codes[-1][sorter]
if bins is not None:
# for compat. with libgroupby.value_counts need to ensure every
# bin is present at every index level, null filled with zeros
diff = np.zeros(len(out), dtype="bool")
for level_codes in codes[:-1]:
diff |= np.r_[True, level_codes[1:] != level_codes[:-1]]
ncat, nbin = diff.sum(), len(levels[-1])
left = [np.repeat(np.arange(ncat), nbin), np.tile(np.arange(nbin), ncat)]
right = [diff.cumsum() - 1, codes[-1]]
_, idx = get_join_indexers(left, right, sort=False, how="left")
out = np.where(idx != -1, out[idx], 0)
if sort:
sorter = np.lexsort((out if ascending else -out, left[0]))
out, left[-1] = out[sorter], left[-1][sorter]
# build the multi-index w/ full levels
def build_codes(lev_codes: np.ndarray) -> np.ndarray:
return np.repeat(lev_codes[diff], nbin)
codes = [build_codes(lev_codes) for lev_codes in codes[:-1]]
codes.append(left[-1])
mi = MultiIndex(
levels=levels, codes=codes, names=index_names, verify_integrity=False
)
if is_integer_dtype(out.dtype):
out = ensure_int64(out)
result = self.obj._constructor(out, index=mi, name=name)
if not self.as_index:
result = result.reset_index()
return result
def fillna(
self,
value: object | ArrayLike | None = None,
method: FillnaOptions | None = None,
axis: Axis | None = None,
inplace: bool = False,
limit: int | None = None,
downcast: dict | None = None,
) -> Series | None:
"""
Fill NA/NaN values using the specified method within groups.
Parameters
----------
value : scalar, dict, Series, or DataFrame
Value to use to fill holes (e.g. 0), alternately a
dict/Series/DataFrame of values specifying which value to use for
each index (for a Series) or column (for a DataFrame). Values not
in the dict/Series/DataFrame will not be filled. This value cannot
be a list. Users wanting to use the ``value`` argument and not ``method``
should prefer :meth:`.Series.fillna` as this
will produce the same result and be more performant.
method : {{'bfill', 'ffill', None}}, default None
Method to use for filling holes. ``'ffill'`` will propagate
the last valid observation forward within a group.
``'bfill'`` will use next valid observation to fill the gap.
axis : {0 or 'index', 1 or 'columns'}
Unused, only for compatibility with :meth:`DataFrameGroupBy.fillna`.
inplace : bool, default False
Broken. Do not set to True.
limit : int, default None
If method is specified, this is the maximum number of consecutive
NaN values to forward/backward fill within a group. In other words,
if there is a gap with more than this number of consecutive NaNs,
it will only be partially filled. If method is not specified, this is the
maximum number of entries along the entire axis where NaNs will be
filled. Must be greater than 0 if not None.
downcast : dict, default is None
A dict of item->dtype of what to downcast if possible,
or the string 'infer' which will try to downcast to an appropriate
equal type (e.g. float64 to int64 if possible).
Returns
-------
Series
Object with missing values filled within groups.
See Also
--------
ffill : Forward fill values within a group.
bfill : Backward fill values within a group.
Examples
--------
>>> ser = pd.Series([np.nan, np.nan, 2, 3, np.nan, np.nan])
>>> ser
0 NaN
1 NaN
2 2.0
3 3.0
4 NaN
5 NaN
dtype: float64
Propagate non-null values forward or backward within each group.
>>> ser.groupby([0, 0, 0, 1, 1, 1]).fillna(method="ffill")
0 NaN
1 NaN
2 2.0
3 3.0
4 3.0
5 3.0
dtype: float64
>>> ser.groupby([0, 0, 0, 1, 1, 1]).fillna(method="bfill")
0 2.0
1 2.0
2 2.0
3 3.0
4 NaN
5 NaN
dtype: float64
Only replace the first NaN element within a group.
>>> ser.groupby([0, 0, 0, 1, 1, 1]).fillna(method="ffill", limit=1)
0 NaN
1 NaN
2 2.0
3 3.0
4 3.0
5 NaN
dtype: float64
"""
result = self._op_via_apply(
"fillna",
value=value,
method=method,
axis=axis,
inplace=inplace,
limit=limit,
downcast=downcast,
)
return result
def take(
self,
indices: TakeIndexer,
axis: Axis = 0,
**kwargs,
) -> Series:
"""
Return the elements in the given *positional* indices in each group.
This means that we are not indexing according to actual values in
the index attribute of the object. We are indexing according to the
actual position of the element in the object.
If a requested index does not exist for some group, this method will raise.
To get similar behavior that ignores indices that don't exist, see
:meth:`.SeriesGroupBy.nth`.
Parameters
----------
indices : array-like
An array of ints indicating which positions to take in each group.
axis : {0 or 'index', 1 or 'columns', None}, default 0
The axis on which to select elements. ``0`` means that we are
selecting rows, ``1`` means that we are selecting columns.
For `SeriesGroupBy` this parameter is unused and defaults to 0.
**kwargs
For compatibility with :meth:`numpy.take`. Has no effect on the
output.
Returns
-------
Series
A Series containing the elements taken from each group.
See Also
--------
Series.take : Take elements from a Series along an axis.
Series.loc : Select a subset of a DataFrame by labels.
Series.iloc : Select a subset of a DataFrame by positions.
numpy.take : Take elements from an array along an axis.
SeriesGroupBy.nth : Similar to take, won't raise if indices don't exist.
Examples
--------
>>> df = pd.DataFrame([('falcon', 'bird', 389.0),
... ('parrot', 'bird', 24.0),
... ('lion', 'mammal', 80.5),
... ('monkey', 'mammal', np.nan),
... ('rabbit', 'mammal', 15.0)],
... columns=['name', 'class', 'max_speed'],
... index=[4, 3, 2, 1, 0])
>>> df
name class max_speed
4 falcon bird 389.0
3 parrot bird 24.0
2 lion mammal 80.5
1 monkey mammal NaN
0 rabbit mammal 15.0
>>> gb = df["name"].groupby([1, 1, 2, 2, 2])
Take elements at positions 0 and 1 along the axis 0 in each group (default).
>>> gb.take([0, 1])
1 4 falcon
3 parrot
2 2 lion
1 monkey
Name: name, dtype: object
We may take elements using negative integers for positive indices,
starting from the end of the object, just like with Python lists.
>>> gb.take([-1, -2])
1 3 parrot
4 falcon
2 0 rabbit
1 monkey
Name: name, dtype: object
"""
result = self._op_via_apply("take", indices=indices, axis=axis, **kwargs)
return result
def skew(
self,
axis: Axis | lib.NoDefault = lib.no_default,
skipna: bool = True,
numeric_only: bool = False,
**kwargs,
) -> Series:
"""
Return unbiased skew within groups.
Normalized by N-1.
Parameters
----------
axis : {0 or 'index', 1 or 'columns', None}, default 0
Axis for the function to be applied on.
This parameter is only for compatibility with DataFrame and is unused.
skipna : bool, default True
Exclude NA/null values when computing the result.
numeric_only : bool, default False
Include only float, int, boolean columns. Not implemented for Series.
**kwargs
Additional keyword arguments to be passed to the function.
Returns
-------
Series
See Also
--------
Series.skew : Return unbiased skew over requested axis.
Examples
--------
>>> ser = pd.Series([390., 350., 357., np.nan, 22., 20., 30.],
... index=['Falcon', 'Falcon', 'Falcon', 'Falcon',
... 'Parrot', 'Parrot', 'Parrot'],
... name="Max Speed")
>>> ser
Falcon 390.0
Falcon 350.0
Falcon 357.0
Falcon NaN
Parrot 22.0
Parrot 20.0
Parrot 30.0
Name: Max Speed, dtype: float64
>>> ser.groupby(level=0).skew()
Falcon 1.525174
Parrot 1.457863
Name: Max Speed, dtype: float64
>>> ser.groupby(level=0).skew(skipna=False)
Falcon NaN
Parrot 1.457863
Name: Max Speed, dtype: float64
"""
result = self._op_via_apply(
"skew",
axis=axis,
skipna=skipna,
numeric_only=numeric_only,
**kwargs,
)
return result
@property
@doc(Series.plot.__doc__)
def plot(self):
result = GroupByPlot(self)
return result
@doc(Series.nlargest.__doc__)
def nlargest(
self, n: int = 5, keep: Literal["first", "last", "all"] = "first"
) -> Series:
f = partial(Series.nlargest, n=n, keep=keep)
data = self._selected_obj
# Don't change behavior if result index happens to be the same, i.e.
# already ordered and n >= all group sizes.
result = self._python_apply_general(f, data, not_indexed_same=True)
return result
@doc(Series.nsmallest.__doc__)
def nsmallest(
self, n: int = 5, keep: Literal["first", "last", "all"] = "first"
) -> Series:
f = partial(Series.nsmallest, n=n, keep=keep)
data = self._selected_obj
# Don't change behavior if result index happens to be the same, i.e.
# already ordered and n >= all group sizes.
result = self._python_apply_general(f, data, not_indexed_same=True)
return result
@doc(Series.idxmin.__doc__)
def idxmin(self, axis: Axis = 0, skipna: bool = True) -> Series:
result = self._op_via_apply("idxmin", axis=axis, skipna=skipna)
return result
@doc(Series.idxmax.__doc__)
def idxmax(self, axis: Axis = 0, skipna: bool = True) -> Series:
result = self._op_via_apply("idxmax", axis=axis, skipna=skipna)
return result
@doc(Series.corr.__doc__)
def corr(
self,
other: Series,
method: CorrelationMethod = "pearson",
min_periods: int | None = None,
) -> Series:
result = self._op_via_apply(
"corr", other=other, method=method, min_periods=min_periods
)
return result
@doc(Series.cov.__doc__)
def cov(
self, other: Series, min_periods: int | None = None, ddof: int | None = 1
) -> Series:
result = self._op_via_apply(
"cov", other=other, min_periods=min_periods, ddof=ddof
)
return result
@property
@doc(Series.is_monotonic_increasing.__doc__)
def is_monotonic_increasing(self) -> Series:
return self.apply(lambda ser: ser.is_monotonic_increasing)
@property
@doc(Series.is_monotonic_decreasing.__doc__)
def is_monotonic_decreasing(self) -> Series:
return self.apply(lambda ser: ser.is_monotonic_decreasing)
@doc(Series.hist.__doc__)
def hist(
self,
by=None,
ax=None,
grid: bool = True,
xlabelsize: int | None = None,
xrot: float | None = None,
ylabelsize: int | None = None,
yrot: float | None = None,
figsize: tuple[int, int] | None = None,
bins: int | Sequence[int] = 10,
backend: str | None = None,
legend: bool = False,
**kwargs,
):
result = self._op_via_apply(
"hist",
by=by,
ax=ax,
grid=grid,
xlabelsize=xlabelsize,
xrot=xrot,
ylabelsize=ylabelsize,
yrot=yrot,
figsize=figsize,
bins=bins,
backend=backend,
legend=legend,
**kwargs,
)
return result
@property
@doc(Series.dtype.__doc__)
def dtype(self) -> Series:
return self.apply(lambda ser: ser.dtype)
@doc(Series.unique.__doc__)
def unique(self) -> Series:
result = self._op_via_apply("unique")
return result
class DataFrameGroupBy(GroupBy[DataFrame]):
_agg_examples_doc = dedent(
"""
Examples
--------
>>> df = pd.DataFrame(
... {
... "A": [1, 1, 2, 2],
... "B": [1, 2, 3, 4],
... "C": [0.362838, 0.227877, 1.267767, -0.562860],
... }
... )
>>> df
A B C
0 1 1 0.362838
1 1 2 0.227877
2 2 3 1.267767
3 2 4 -0.562860
The aggregation is for each column.
>>> df.groupby('A').agg('min')
B C
A
1 1 0.227877
2 3 -0.562860
Multiple aggregations
>>> df.groupby('A').agg(['min', 'max'])
B C
min max min max
A
1 1 2 0.227877 0.362838
2 3 4 -0.562860 1.267767
Select a column for aggregation
>>> df.groupby('A').B.agg(['min', 'max'])
min max
A
1 1 2
2 3 4
User-defined function for aggregation
>>> df.groupby('A').agg(lambda x: sum(x) + 2)
B C
A
1 5 2.590715
2 9 2.704907
Different aggregations per column
>>> df.groupby('A').agg({'B': ['min', 'max'], 'C': 'sum'})
B C
min max sum
A
1 1 2 0.590715
2 3 4 0.704907
To control the output names with different aggregations per column,
pandas supports "named aggregation"
>>> df.groupby("A").agg(
... b_min=pd.NamedAgg(column="B", aggfunc="min"),
... c_sum=pd.NamedAgg(column="C", aggfunc="sum"))
b_min c_sum
A
1 1 0.590715
2 3 0.704907
- The keywords are the *output* column names
- The values are tuples whose first element is the column to select
and the second element is the aggregation to apply to that column.
Pandas provides the ``pandas.NamedAgg`` namedtuple with the fields
``['column', 'aggfunc']`` to make it clearer what the arguments are.
As usual, the aggregation can be a callable or a string alias.
See :ref:`groupby.aggregate.named` for more.
.. versionchanged:: 1.3.0
The resulting dtype will reflect the return value of the aggregating function.
>>> df.groupby("A")[["B"]].agg(lambda x: x.astype(float).min())
B
A
1 1.0
2 3.0
"""
)
@doc(_agg_template, examples=_agg_examples_doc, klass="DataFrame")
def aggregate(self, func=None, *args, engine=None, engine_kwargs=None, **kwargs):
if maybe_use_numba(engine):
return self._aggregate_with_numba(
func, *args, engine_kwargs=engine_kwargs, **kwargs
)
relabeling, func, columns, order = reconstruct_func(func, **kwargs)
func = maybe_mangle_lambdas(func)
op = GroupByApply(self, func, args, kwargs)
result = op.agg()
if not is_dict_like(func) and result is not None:
return result
elif relabeling:
# this should be the only (non-raising) case with relabeling
# used reordered index of columns
result = cast(DataFrame, result)
result = result.iloc[:, order]
result = cast(DataFrame, result)
# error: Incompatible types in assignment (expression has type
# "Optional[List[str]]", variable has type
# "Union[Union[Union[ExtensionArray, ndarray[Any, Any]],
# Index, Series], Sequence[Any]]")
result.columns = columns # type: ignore[assignment]
if result is None:
# grouper specific aggregations
if self.grouper.nkeys > 1:
# test_groupby_as_index_series_scalar gets here with 'not self.as_index'
return self._python_agg_general(func, *args, **kwargs)
elif args or kwargs:
# test_pass_args_kwargs gets here (with and without as_index)
# can't return early
result = self._aggregate_frame(func, *args, **kwargs)
elif self.axis == 1:
# _aggregate_multiple_funcs does not allow self.axis == 1
# Note: axis == 1 precludes 'not self.as_index', see __init__
result = self._aggregate_frame(func)
return result
else:
# try to treat as if we are passing a list
gba = GroupByApply(self, [func], args=(), kwargs={})
try:
result = gba.agg()
except ValueError as err:
if "No objects to concatenate" not in str(err):
raise
# _aggregate_frame can fail with e.g. func=Series.mode,
# where it expects 1D values but would be getting 2D values
# In other tests, using aggregate_frame instead of GroupByApply
# would give correct values but incorrect dtypes
# object vs float64 in test_cython_agg_empty_buckets
# float64 vs int64 in test_category_order_apply
result = self._aggregate_frame(func)
else:
# GH#32040, GH#35246
# e.g. test_groupby_as_index_select_column_sum_empty_df
result = cast(DataFrame, result)
result.columns = self._obj_with_exclusions.columns.copy()
if not self.as_index:
result = self._insert_inaxis_grouper(result)
result.index = default_index(len(result))
return result
agg = aggregate
def _python_agg_general(self, func, *args, **kwargs):
func = com.is_builtin_func(func)
f = lambda x: func(x, *args, **kwargs)
# iterate through "columns" ex exclusions to populate output dict
output: dict[base.OutputKey, ArrayLike] = {}
if self.ngroups == 0:
# e.g. test_evaluate_with_empty_groups different path gets different
# result dtype in empty case.
return self._python_apply_general(f, self._selected_obj, is_agg=True)
for idx, obj in enumerate(self._iterate_slices()):
name = obj.name
result = self.grouper.agg_series(obj, f)
key = base.OutputKey(label=name, position=idx)
output[key] = result
if not output:
# e.g. test_margins_no_values_no_cols
return self._python_apply_general(f, self._selected_obj)
res = self._indexed_output_to_ndframe(output)
return self._wrap_aggregated_output(res)
def _iterate_slices(self) -> Iterable[Series]:
obj = self._selected_obj
if self.axis == 1:
obj = obj.T
if isinstance(obj, Series) and obj.name not in self.exclusions:
# Occurs when doing DataFrameGroupBy(...)["X"]
yield obj
else:
for label, values in obj.items():
if label in self.exclusions:
# Note: if we tried to just iterate over _obj_with_exclusions,
# we would break test_wrap_agg_out by yielding a column
# that is skipped here but not dropped from obj_with_exclusions
continue
yield values
def _aggregate_frame(self, func, *args, **kwargs) -> DataFrame:
if self.grouper.nkeys != 1:
raise AssertionError("Number of keys must be 1")
obj = self._obj_with_exclusions
result: dict[Hashable, NDFrame | np.ndarray] = {}
for name, grp_df in self.grouper.get_iterator(obj, self.axis):
fres = func(grp_df, *args, **kwargs)
result[name] = fres
result_index = self.grouper.result_index
other_ax = obj.axes[1 - self.axis]
out = self.obj._constructor(result, index=other_ax, columns=result_index)
if self.axis == 0:
out = out.T
return out
def _wrap_applied_output(
self,
data: DataFrame,
values: list,
not_indexed_same: bool = False,
is_transform: bool = False,
):
if len(values) == 0:
if is_transform:
# GH#47787 see test_group_on_empty_multiindex
res_index = data.index
else:
res_index = self.grouper.result_index
result = self.obj._constructor(index=res_index, columns=data.columns)
result = result.astype(data.dtypes, copy=False)
return result
# GH12824
# using values[0] here breaks test_groupby_apply_none_first
first_not_none = next(com.not_none(*values), None)
if first_not_none is None:
# GH9684 - All values are None, return an empty frame.
return self.obj._constructor()
elif isinstance(first_not_none, DataFrame):
return self._concat_objects(
values,
not_indexed_same=not_indexed_same,
is_transform=is_transform,
)
key_index = self.grouper.result_index if self.as_index else None
if isinstance(first_not_none, (np.ndarray, Index)):
# GH#1738: values is list of arrays of unequal lengths
# fall through to the outer else clause
# TODO: sure this is right? we used to do this
# after raising AttributeError above
return self.obj._constructor_sliced(
values, index=key_index, name=self._selection
)
elif not isinstance(first_not_none, Series):
# values are not series or array-like but scalars
# self._selection not passed through to Series as the
# result should not take the name of original selection
# of columns
if self.as_index:
return self.obj._constructor_sliced(values, index=key_index)
else:
result = self.obj._constructor(values, columns=[self._selection])
result = self._insert_inaxis_grouper(result)
return result
else:
# values are Series
return self._wrap_applied_output_series(
values,
not_indexed_same,
first_not_none,
key_index,
is_transform,
)
def _wrap_applied_output_series(
self,
values: list[Series],
not_indexed_same: bool,
first_not_none,
key_index: Index | None,
is_transform: bool,
) -> DataFrame | Series:
kwargs = first_not_none._construct_axes_dict()
backup = Series(**kwargs)
values = [x if (x is not None) else backup for x in values]
all_indexed_same = all_indexes_same(x.index for x in values)
if not all_indexed_same:
# GH 8467
return self._concat_objects(
values,
not_indexed_same=True,
is_transform=is_transform,
)
# Combine values
# vstack+constructor is faster than concat and handles MI-columns
stacked_values = np.vstack([np.asarray(v) for v in values])
if self.axis == 0:
index = key_index
columns = first_not_none.index.copy()
if columns.name is None:
# GH6124 - propagate name of Series when it's consistent
names = {v.name for v in values}
if len(names) == 1:
columns.name = list(names)[0]
else:
index = first_not_none.index
columns = key_index
stacked_values = stacked_values.T
if stacked_values.dtype == object:
# We'll have the DataFrame constructor do inference
stacked_values = stacked_values.tolist()
result = self.obj._constructor(stacked_values, index=index, columns=columns)
if not self.as_index:
result = self._insert_inaxis_grouper(result)
return self._reindex_output(result)
def _cython_transform(
self,
how: str,
numeric_only: bool = False,
axis: AxisInt = 0,
**kwargs,
) -> DataFrame:
assert axis == 0 # handled by caller
# With self.axis == 0, we have multi-block tests
# e.g. test_rank_min_int, test_cython_transform_frame
# test_transform_numeric_ret
# With self.axis == 1, _get_data_to_aggregate does a transpose
# so we always have a single block.
mgr: Manager2D = self._get_data_to_aggregate(
numeric_only=numeric_only, name=how
)
def arr_func(bvalues: ArrayLike) -> ArrayLike:
return self.grouper._cython_operation(
"transform", bvalues, how, 1, **kwargs
)
# We could use `mgr.apply` here and not have to set_axis, but
# we would have to do shape gymnastics for ArrayManager compat
res_mgr = mgr.grouped_reduce(arr_func)
res_mgr.set_axis(1, mgr.axes[1])
res_df = self.obj._constructor(res_mgr)
res_df = self._maybe_transpose_result(res_df)
return res_df
def _transform_general(self, func, *args, **kwargs):
from pandas.core.reshape.concat import concat
applied = []
obj = self._obj_with_exclusions
gen = self.grouper.get_iterator(obj, axis=self.axis)
fast_path, slow_path = self._define_paths(func, *args, **kwargs)
# Determine whether to use slow or fast path by evaluating on the first group.
# Need to handle the case of an empty generator and process the result so that
# it does not need to be computed again.
try:
name, group = next(gen)
except StopIteration:
pass
else:
object.__setattr__(group, "name", name)
try:
path, res = self._choose_path(fast_path, slow_path, group)
except ValueError as err:
# e.g. test_transform_with_non_scalar_group
msg = "transform must return a scalar value for each group"
raise ValueError(msg) from err
if group.size > 0:
res = _wrap_transform_general_frame(self.obj, group, res)
applied.append(res)
# Compute and process with the remaining groups
for name, group in gen:
if group.size == 0:
continue
object.__setattr__(group, "name", name)
res = path(group)
res = _wrap_transform_general_frame(self.obj, group, res)
applied.append(res)
concat_index = obj.columns if self.axis == 0 else obj.index
other_axis = 1 if self.axis == 0 else 0 # switches between 0 & 1
concatenated = concat(applied, axis=self.axis, verify_integrity=False)
concatenated = concatenated.reindex(concat_index, axis=other_axis, copy=False)
return self._set_result_index_ordered(concatenated)
__examples_dataframe_doc = dedent(
"""
>>> df = pd.DataFrame({'A' : ['foo', 'bar', 'foo', 'bar',
... 'foo', 'bar'],
... 'B' : ['one', 'one', 'two', 'three',
... 'two', 'two'],
... 'C' : [1, 5, 5, 2, 5, 5],
... 'D' : [2.0, 5., 8., 1., 2., 9.]})
>>> grouped = df.groupby('A')[['C', 'D']]
>>> grouped.transform(lambda x: (x - x.mean()) / x.std())
C D
0 -1.154701 -0.577350
1 0.577350 0.000000
2 0.577350 1.154701
3 -1.154701 -1.000000
4 0.577350 -0.577350
5 0.577350 1.000000
Broadcast result of the transformation
>>> grouped.transform(lambda x: x.max() - x.min())
C D
0 4.0 6.0
1 3.0 8.0
2 4.0 6.0
3 3.0 8.0
4 4.0 6.0
5 3.0 8.0
>>> grouped.transform("mean")
C D
0 3.666667 4.0
1 4.000000 5.0
2 3.666667 4.0
3 4.000000 5.0
4 3.666667 4.0
5 4.000000 5.0
.. versionchanged:: 1.3.0
The resulting dtype will reflect the return value of the passed ``func``,
for example:
>>> grouped.transform(lambda x: x.astype(int).max())
C D
0 5 8
1 5 9
2 5 8
3 5 9
4 5 8
5 5 9
"""
)
@Substitution(klass="DataFrame", example=__examples_dataframe_doc)
@Appender(_transform_template)
def transform(self, func, *args, engine=None, engine_kwargs=None, **kwargs):
return self._transform(
func, *args, engine=engine, engine_kwargs=engine_kwargs, **kwargs
)
def _define_paths(self, func, *args, **kwargs):
if isinstance(func, str):
fast_path = lambda group: getattr(group, func)(*args, **kwargs)
slow_path = lambda group: group.apply(
lambda x: getattr(x, func)(*args, **kwargs), axis=self.axis
)
else:
fast_path = lambda group: func(group, *args, **kwargs)
slow_path = lambda group: group.apply(
lambda x: func(x, *args, **kwargs), axis=self.axis
)
return fast_path, slow_path
def _choose_path(self, fast_path: Callable, slow_path: Callable, group: DataFrame):
path = slow_path
res = slow_path(group)
if self.ngroups == 1:
# no need to evaluate multiple paths when only
# a single group exists
return path, res
# if we make it here, test if we can use the fast path
try:
res_fast = fast_path(group)
except AssertionError:
raise # pragma: no cover
except Exception:
# GH#29631 For user-defined function, we can't predict what may be
# raised; see test_transform.test_transform_fastpath_raises
return path, res
# verify fast path returns either:
# a DataFrame with columns equal to group.columns
# OR a Series with index equal to group.columns
if isinstance(res_fast, DataFrame):
if not res_fast.columns.equals(group.columns):
return path, res
elif isinstance(res_fast, Series):
if not res_fast.index.equals(group.columns):
return path, res
else:
return path, res
if res_fast.equals(res):
path = fast_path
return path, res
def filter(self, func, dropna: bool = True, *args, **kwargs):
"""
Filter elements from groups that don't satisfy a criterion.
Elements from groups are filtered if they do not satisfy the
boolean criterion specified by func.
Parameters
----------
func : function
Criterion to apply to each group. Should return True or False.
dropna : bool
Drop groups that do not pass the filter. True by default; if False,
groups that evaluate False are filled with NaNs.
Returns
-------
DataFrame
Notes
-----
Each subframe is endowed the attribute 'name' in case you need to know
which group you are working on.
Functions that mutate the passed object can produce unexpected
behavior or errors and are not supported. See :ref:`gotchas.udf-mutation`
for more details.
Examples
--------
>>> df = pd.DataFrame({'A' : ['foo', 'bar', 'foo', 'bar',
... 'foo', 'bar'],
... 'B' : [1, 2, 3, 4, 5, 6],
... 'C' : [2.0, 5., 8., 1., 2., 9.]})
>>> grouped = df.groupby('A')
>>> grouped.filter(lambda x: x['B'].mean() > 3.)
A B C
1 bar 2 5.0
3 bar 4 1.0
5 bar 6 9.0
"""
indices = []
obj = self._selected_obj
gen = self.grouper.get_iterator(obj, axis=self.axis)
for name, group in gen:
object.__setattr__(group, "name", name)
res = func(group, *args, **kwargs)
try:
res = res.squeeze()
except AttributeError: # allow e.g., scalars and frames to pass
pass
# interpret the result of the filter
if is_bool(res) or (is_scalar(res) and isna(res)):
if notna(res) and res:
indices.append(self._get_index(name))
else:
# non scalars aren't allowed
raise TypeError(
f"filter function returned a {type(res).__name__}, "
"but expected a scalar bool"
)
return self._apply_filter(indices, dropna)
def __getitem__(self, key) -> DataFrameGroupBy | SeriesGroupBy:
if self.axis == 1:
# GH 37725
raise ValueError("Cannot subset columns when using axis=1")
# per GH 23566
if isinstance(key, tuple) and len(key) > 1:
# if len == 1, then it becomes a SeriesGroupBy and this is actually
# valid syntax, so don't raise
raise ValueError(
"Cannot subset columns with a tuple with more than one element. "
"Use a list instead."
)
return super().__getitem__(key)
def _gotitem(self, key, ndim: int, subset=None):
"""
sub-classes to define
return a sliced object
Parameters
----------
key : string / list of selections
ndim : {1, 2}
requested ndim of result
subset : object, default None
subset to act on
"""
if ndim == 2:
if subset is None:
subset = self.obj
return DataFrameGroupBy(
subset,
self.grouper,
axis=self.axis,
level=self.level,
grouper=self.grouper,
exclusions=self.exclusions,
selection=key,
as_index=self.as_index,
sort=self.sort,
group_keys=self.group_keys,
observed=self.observed,
dropna=self.dropna,
)
elif ndim == 1:
if subset is None:
subset = self.obj[key]
return SeriesGroupBy(
subset,
level=self.level,
grouper=self.grouper,
exclusions=self.exclusions,
selection=key,
as_index=self.as_index,
sort=self.sort,
group_keys=self.group_keys,
observed=self.observed,
dropna=self.dropna,
)
raise AssertionError("invalid ndim for _gotitem")
def _get_data_to_aggregate(
self, *, numeric_only: bool = False, name: str | None = None
) -> Manager2D:
obj = self._obj_with_exclusions
if self.axis == 1:
mgr = obj.T._mgr
else:
mgr = obj._mgr
if numeric_only:
mgr = mgr.get_numeric_data(copy=False)
return mgr
def _indexed_output_to_ndframe(
self, output: Mapping[base.OutputKey, ArrayLike]
) -> DataFrame:
"""
Wrap the dict result of a GroupBy aggregation into a DataFrame.
"""
indexed_output = {key.position: val for key, val in output.items()}
columns = Index([key.label for key in output])
columns._set_names(self._obj_with_exclusions._get_axis(1 - self.axis).names)
result = self.obj._constructor(indexed_output)
result.columns = columns
return result
def _wrap_agged_manager(self, mgr: Manager2D) -> DataFrame:
return self.obj._constructor(mgr)
def _iterate_column_groupbys(self, obj: DataFrame):
for i, colname in enumerate(obj.columns):
yield colname, SeriesGroupBy(
obj.iloc[:, i],
selection=colname,
grouper=self.grouper,
exclusions=self.exclusions,
observed=self.observed,
)
def _apply_to_column_groupbys(self, func, obj: DataFrame) -> DataFrame:
from pandas.core.reshape.concat import concat
columns = obj.columns
results = [
func(col_groupby) for _, col_groupby in self._iterate_column_groupbys(obj)
]
if not len(results):
# concat would raise
return DataFrame([], columns=columns, index=self.grouper.result_index)
else:
return concat(results, keys=columns, axis=1)
def nunique(self, dropna: bool = True) -> DataFrame:
"""
Return DataFrame with counts of unique elements in each position.
Parameters
----------
dropna : bool, default True
Don't include NaN in the counts.
Returns
-------
nunique: DataFrame
Examples
--------
>>> df = pd.DataFrame({'id': ['spam', 'egg', 'egg', 'spam',
... 'ham', 'ham'],
... 'value1': [1, 5, 5, 2, 5, 5],
... 'value2': list('abbaxy')})
>>> df
id value1 value2
0 spam 1 a
1 egg 5 b
2 egg 5 b
3 spam 2 a
4 ham 5 x
5 ham 5 y
>>> df.groupby('id').nunique()
value1 value2
id
egg 1 1
ham 1 2
spam 2 1
Check for rows with the same id but conflicting values:
>>> df.groupby('id').filter(lambda g: (g.nunique() > 1).any())
id value1 value2
0 spam 1 a
3 spam 2 a
4 ham 5 x
5 ham 5 y
"""
if self.axis != 0:
# see test_groupby_crash_on_nunique
return self._python_apply_general(
lambda sgb: sgb.nunique(dropna), self._obj_with_exclusions, is_agg=True
)
obj = self._obj_with_exclusions
results = self._apply_to_column_groupbys(
lambda sgb: sgb.nunique(dropna), obj=obj
)
if not self.as_index:
results.index = default_index(len(results))
results = self._insert_inaxis_grouper(results)
return results
def idxmax(
self,
axis: Axis | None = None,
skipna: bool = True,
numeric_only: bool = False,
) -> DataFrame:
"""
Return index of first occurrence of maximum over requested axis.
NA/null values are excluded.
Parameters
----------
axis : {{0 or 'index', 1 or 'columns'}}, default None
The axis to use. 0 or 'index' for row-wise, 1 or 'columns' for column-wise.
If axis is not provided, grouper's axis is used.
.. versionchanged:: 2.0.0
skipna : bool, default True
Exclude NA/null values. If an entire row/column is NA, the result
will be NA.
numeric_only : bool, default False
Include only `float`, `int` or `boolean` data.
.. versionadded:: 1.5.0
Returns
-------
Series
Indexes of maxima along the specified axis.
Raises
------
ValueError
* If the row/column is empty
See Also
--------
Series.idxmax : Return index of the maximum element.
Notes
-----
This method is the DataFrame version of ``ndarray.argmax``.
Examples
--------
Consider a dataset containing food consumption in Argentina.
>>> df = pd.DataFrame({'consumption': [10.51, 103.11, 55.48],
... 'co2_emissions': [37.2, 19.66, 1712]},
... index=['Pork', 'Wheat Products', 'Beef'])
>>> df
consumption co2_emissions
Pork 10.51 37.20
Wheat Products 103.11 19.66
Beef 55.48 1712.00
By default, it returns the index for the maximum value in each column.
>>> df.idxmax()
consumption Wheat Products
co2_emissions Beef
dtype: object
To return the index for the maximum value in each row, use ``axis="columns"``.
>>> df.idxmax(axis="columns")
Pork co2_emissions
Wheat Products consumption
Beef co2_emissions
dtype: object
"""
if axis is None:
axis = self.axis
def func(df):
return df.idxmax(axis=axis, skipna=skipna, numeric_only=numeric_only)
func.__name__ = "idxmax"
result = self._python_apply_general(
func, self._obj_with_exclusions, not_indexed_same=True
)
return result
def idxmin(
self,
axis: Axis | None = None,
skipna: bool = True,
numeric_only: bool = False,
) -> DataFrame:
"""
Return index of first occurrence of minimum over requested axis.
NA/null values are excluded.
Parameters
----------
axis : {{0 or 'index', 1 or 'columns'}}, default None
The axis to use. 0 or 'index' for row-wise, 1 or 'columns' for column-wise.
If axis is not provided, grouper's axis is used.
.. versionchanged:: 2.0.0
skipna : bool, default True
Exclude NA/null values. If an entire row/column is NA, the result
will be NA.
numeric_only : bool, default False
Include only `float`, `int` or `boolean` data.
.. versionadded:: 1.5.0
Returns
-------
Series
Indexes of minima along the specified axis.
Raises
------
ValueError
* If the row/column is empty
See Also
--------
Series.idxmin : Return index of the minimum element.
Notes
-----
This method is the DataFrame version of ``ndarray.argmin``.
Examples
--------
Consider a dataset containing food consumption in Argentina.
>>> df = pd.DataFrame({'consumption': [10.51, 103.11, 55.48],
... 'co2_emissions': [37.2, 19.66, 1712]},
... index=['Pork', 'Wheat Products', 'Beef'])
>>> df
consumption co2_emissions
Pork 10.51 37.20
Wheat Products 103.11 19.66
Beef 55.48 1712.00
By default, it returns the index for the minimum value in each column.
>>> df.idxmin()
consumption Pork
co2_emissions Wheat Products
dtype: object
To return the index for the minimum value in each row, use ``axis="columns"``.
>>> df.idxmin(axis="columns")
Pork consumption
Wheat Products co2_emissions
Beef consumption
dtype: object
"""
if axis is None:
axis = self.axis
def func(df):
return df.idxmin(axis=axis, skipna=skipna, numeric_only=numeric_only)
func.__name__ = "idxmin"
result = self._python_apply_general(
func, self._obj_with_exclusions, not_indexed_same=True
)
return result
boxplot = boxplot_frame_groupby
def value_counts(
self,
subset: Sequence[Hashable] | None = None,
normalize: bool = False,
sort: bool = True,
ascending: bool = False,
dropna: bool = True,
) -> DataFrame | Series:
"""
Return a Series or DataFrame containing counts of unique rows.
.. versionadded:: 1.4.0
Parameters
----------
subset : list-like, optional
Columns to use when counting unique combinations.
normalize : bool, default False
Return proportions rather than frequencies.
sort : bool, default True
Sort by frequencies.
ascending : bool, default False
Sort in ascending order.
dropna : bool, default True
Don’t include counts of rows that contain NA values.
Returns
-------
Series or DataFrame
Series if the groupby as_index is True, otherwise DataFrame.
See Also
--------
Series.value_counts: Equivalent method on Series.
DataFrame.value_counts: Equivalent method on DataFrame.
SeriesGroupBy.value_counts: Equivalent method on SeriesGroupBy.
Notes
-----
- If the groupby as_index is True then the returned Series will have a
MultiIndex with one level per input column.
- If the groupby as_index is False then the returned DataFrame will have an
additional column with the value_counts. The column is labelled 'count' or
'proportion', depending on the ``normalize`` parameter.
By default, rows that contain any NA values are omitted from
the result.
By default, the result will be in descending order so that the
first element of each group is the most frequently-occurring row.
Examples
--------
>>> df = pd.DataFrame({
... 'gender': ['male', 'male', 'female', 'male', 'female', 'male'],
... 'education': ['low', 'medium', 'high', 'low', 'high', 'low'],
... 'country': ['US', 'FR', 'US', 'FR', 'FR', 'FR']
... })
>>> df
gender education country
0 male low US
1 male medium FR
2 female high US
3 male low FR
4 female high FR
5 male low FR
>>> df.groupby('gender').value_counts()
gender education country
female high FR 1
US 1
male low FR 2
US 1
medium FR 1
Name: count, dtype: int64
>>> df.groupby('gender').value_counts(ascending=True)
gender education country
female high FR 1
US 1
male low US 1
medium FR 1
low FR 2
Name: count, dtype: int64
>>> df.groupby('gender').value_counts(normalize=True)
gender education country
female high FR 0.50
US 0.50
male low FR 0.50
US 0.25
medium FR 0.25
Name: proportion, dtype: float64
>>> df.groupby('gender', as_index=False).value_counts()
gender education country count
0 female high FR 1
1 female high US 1
2 male low FR 2
3 male low US 1
4 male medium FR 1
>>> df.groupby('gender', as_index=False).value_counts(normalize=True)
gender education country proportion
0 female high FR 0.50
1 female high US 0.50
2 male low FR 0.50
3 male low US 0.25
4 male medium FR 0.25
"""
return self._value_counts(subset, normalize, sort, ascending, dropna)
def fillna(
self,
value: Hashable | Mapping | Series | DataFrame = None,
method: FillnaOptions | None = None,
axis: Axis | None = None,
inplace: bool = False,
limit=None,
downcast=None,
) -> DataFrame | None:
"""
Fill NA/NaN values using the specified method within groups.
Parameters
----------
value : scalar, dict, Series, or DataFrame
Value to use to fill holes (e.g. 0), alternately a
dict/Series/DataFrame of values specifying which value to use for
each index (for a Series) or column (for a DataFrame). Values not
in the dict/Series/DataFrame will not be filled. This value cannot
be a list. Users wanting to use the ``value`` argument and not ``method``
should prefer :meth:`.DataFrame.fillna` as this
will produce the same result and be more performant.
method : {{'bfill', 'ffill', None}}, default None
Method to use for filling holes. ``'ffill'`` will propagate
the last valid observation forward within a group.
``'bfill'`` will use next valid observation to fill the gap.
axis : {0 or 'index', 1 or 'columns'}
Axis along which to fill missing values. When the :class:`DataFrameGroupBy`
``axis`` argument is ``0``, using ``axis=1`` here will produce
the same results as :meth:`.DataFrame.fillna`. When the
:class:`DataFrameGroupBy` ``axis`` argument is ``1``, using ``axis=0``
or ``axis=1`` here will produce the same results.
inplace : bool, default False
Broken. Do not set to True.
limit : int, default None
If method is specified, this is the maximum number of consecutive
NaN values to forward/backward fill within a group. In other words,
if there is a gap with more than this number of consecutive NaNs,
it will only be partially filled. If method is not specified, this is the
maximum number of entries along the entire axis where NaNs will be
filled. Must be greater than 0 if not None.
downcast : dict, default is None
A dict of item->dtype of what to downcast if possible,
or the string 'infer' which will try to downcast to an appropriate
equal type (e.g. float64 to int64 if possible).
Returns
-------
DataFrame
Object with missing values filled.
See Also
--------
ffill : Forward fill values within a group.
bfill : Backward fill values within a group.
Examples
--------
>>> df = pd.DataFrame(
... {
... "key": [0, 0, 1, 1, 1],
... "A": [np.nan, 2, np.nan, 3, np.nan],
... "B": [2, 3, np.nan, np.nan, np.nan],
... "C": [np.nan, np.nan, 2, np.nan, np.nan],
... }
... )
>>> df
key A B C
0 0 NaN 2.0 NaN
1 0 2.0 3.0 NaN
2 1 NaN NaN 2.0
3 1 3.0 NaN NaN
4 1 NaN NaN NaN
Propagate non-null values forward or backward within each group along columns.
>>> df.groupby("key").fillna(method="ffill")
A B C
0 NaN 2.0 NaN
1 2.0 3.0 NaN
2 NaN NaN 2.0
3 3.0 NaN 2.0
4 3.0 NaN 2.0
>>> df.groupby("key").fillna(method="bfill")
A B C
0 2.0 2.0 NaN
1 2.0 3.0 NaN
2 3.0 NaN 2.0
3 3.0 NaN NaN
4 NaN NaN NaN
Propagate non-null values forward or backward within each group along rows.
>>> df.groupby([0, 0, 1, 1], axis=1).fillna(method="ffill")
key A B C
0 0.0 0.0 2.0 2.0
1 0.0 2.0 3.0 3.0
2 1.0 1.0 NaN 2.0
3 1.0 3.0 NaN NaN
4 1.0 1.0 NaN NaN
>>> df.groupby([0, 0, 1, 1], axis=1).fillna(method="bfill")
key A B C
0 0.0 NaN 2.0 NaN
1 0.0 2.0 3.0 NaN
2 1.0 NaN 2.0 2.0
3 1.0 3.0 NaN NaN
4 1.0 NaN NaN NaN
Only replace the first NaN element within a group along rows.
>>> df.groupby("key").fillna(method="ffill", limit=1)
A B C
0 NaN 2.0 NaN
1 2.0 3.0 NaN
2 NaN NaN 2.0
3 3.0 NaN 2.0
4 3.0 NaN NaN
"""
result = self._op_via_apply(
"fillna",
value=value,
method=method,
axis=axis,
inplace=inplace,
limit=limit,
downcast=downcast,
)
return result
def take(
self,
indices: TakeIndexer,
axis: Axis | None = 0,
**kwargs,
) -> DataFrame:
"""
Return the elements in the given *positional* indices in each group.
This means that we are not indexing according to actual values in
the index attribute of the object. We are indexing according to the
actual position of the element in the object.
If a requested index does not exist for some group, this method will raise.
To get similar behavior that ignores indices that don't exist, see
:meth:`.DataFrameGroupBy.nth`.
Parameters
----------
indices : array-like
An array of ints indicating which positions to take.
axis : {0 or 'index', 1 or 'columns', None}, default 0
The axis on which to select elements. ``0`` means that we are
selecting rows, ``1`` means that we are selecting columns.
**kwargs
For compatibility with :meth:`numpy.take`. Has no effect on the
output.
Returns
-------
DataFrame
An DataFrame containing the elements taken from each group.
See Also
--------
DataFrame.take : Take elements from a Series along an axis.
DataFrame.loc : Select a subset of a DataFrame by labels.
DataFrame.iloc : Select a subset of a DataFrame by positions.
numpy.take : Take elements from an array along an axis.
Examples
--------
>>> df = pd.DataFrame([('falcon', 'bird', 389.0),
... ('parrot', 'bird', 24.0),
... ('lion', 'mammal', 80.5),
... ('monkey', 'mammal', np.nan),
... ('rabbit', 'mammal', 15.0)],
... columns=['name', 'class', 'max_speed'],
... index=[4, 3, 2, 1, 0])
>>> df
name class max_speed
4 falcon bird 389.0
3 parrot bird 24.0
2 lion mammal 80.5
1 monkey mammal NaN
0 rabbit mammal 15.0
>>> gb = df.groupby([1, 1, 2, 2, 2])
Take elements at positions 0 and 1 along the axis 0 (default).
Note how the indices selected in the result do not correspond to
our input indices 0 and 1. That's because we are selecting the 0th
and 1st rows, not rows whose indices equal 0 and 1.
>>> gb.take([0, 1])
name class max_speed
1 4 falcon bird 389.0
3 parrot bird 24.0
2 2 lion mammal 80.5
1 monkey mammal NaN
The order of the specified indices influences the order in the result.
Here, the order is swapped from the previous example.
>>> gb.take([1, 0])
name class max_speed
1 3 parrot bird 24.0
4 falcon bird 389.0
2 1 monkey mammal NaN
2 lion mammal 80.5
Take elements at indices 1 and 2 along the axis 1 (column selection).
We may take elements using negative integers for positive indices,
starting from the end of the object, just like with Python lists.
>>> gb.take([-1, -2])
name class max_speed
1 3 parrot bird 24.0
4 falcon bird 389.0
2 0 rabbit mammal 15.0
1 monkey mammal NaN
"""
result = self._op_via_apply("take", indices=indices, axis=axis, **kwargs)
return result
def skew(
self,
axis: Axis | None | lib.NoDefault = lib.no_default,
skipna: bool = True,
numeric_only: bool = False,
**kwargs,
) -> DataFrame:
"""
Return unbiased skew within groups.
Normalized by N-1.
Parameters
----------
axis : {0 or 'index', 1 or 'columns', None}, default 0
Axis for the function to be applied on.
Specifying ``axis=None`` will apply the aggregation across both axes.
.. versionadded:: 2.0.0
skipna : bool, default True
Exclude NA/null values when computing the result.
numeric_only : bool, default False
Include only float, int, boolean columns.
**kwargs
Additional keyword arguments to be passed to the function.
Returns
-------
DataFrame
See Also
--------
DataFrame.skew : Return unbiased skew over requested axis.
Examples
--------
>>> arrays = [['falcon', 'parrot', 'cockatoo', 'kiwi',
... 'lion', 'monkey', 'rabbit'],
... ['bird', 'bird', 'bird', 'bird',
... 'mammal', 'mammal', 'mammal']]
>>> index = pd.MultiIndex.from_arrays(arrays, names=('name', 'class'))
>>> df = pd.DataFrame({'max_speed': [389.0, 24.0, 70.0, np.nan,
... 80.5, 21.5, 15.0]},
... index=index)
>>> df
max_speed
name class
falcon bird 389.0
parrot bird 24.0
cockatoo bird 70.0
kiwi bird NaN
lion mammal 80.5
monkey mammal 21.5
rabbit mammal 15.0
>>> gb = df.groupby(["class"])
>>> gb.skew()
max_speed
class
bird 1.628296
mammal 1.669046
>>> gb.skew(skipna=False)
max_speed
class
bird NaN
mammal 1.669046
"""
result = self._op_via_apply(
"skew",
axis=axis,
skipna=skipna,
numeric_only=numeric_only,
**kwargs,
)
return result
@property
@doc(DataFrame.plot.__doc__)
def plot(self) -> GroupByPlot:
result = GroupByPlot(self)
return result
@doc(DataFrame.corr.__doc__)
def corr(
self,
method: str | Callable[[np.ndarray, np.ndarray], float] = "pearson",
min_periods: int = 1,
numeric_only: bool = False,
) -> DataFrame:
result = self._op_via_apply(
"corr", method=method, min_periods=min_periods, numeric_only=numeric_only
)
return result
@doc(DataFrame.cov.__doc__)
def cov(
self,
min_periods: int | None = None,
ddof: int | None = 1,
numeric_only: bool = False,
) -> DataFrame:
result = self._op_via_apply(
"cov", min_periods=min_periods, ddof=ddof, numeric_only=numeric_only
)
return result
@doc(DataFrame.hist.__doc__)
def hist(
self,
column: IndexLabel = None,
by=None,
grid: bool = True,
xlabelsize: int | None = None,
xrot: float | None = None,
ylabelsize: int | None = None,
yrot: float | None = None,
ax=None,
sharex: bool = False,
sharey: bool = False,
figsize: tuple[int, int] | None = None,
layout: tuple[int, int] | None = None,
bins: int | Sequence[int] = 10,
backend: str | None = None,
legend: bool = False,
**kwargs,
):
result = self._op_via_apply(
"hist",
column=column,
by=by,
grid=grid,
xlabelsize=xlabelsize,
xrot=xrot,
ylabelsize=ylabelsize,
yrot=yrot,
ax=ax,
sharex=sharex,
sharey=sharey,
figsize=figsize,
layout=layout,
bins=bins,
backend=backend,
legend=legend,
**kwargs,
)
return result
@property
@doc(DataFrame.dtypes.__doc__)
def dtypes(self) -> Series:
# error: Incompatible return value type (got "DataFrame", expected "Series")
return self.apply(lambda df: df.dtypes) # type: ignore[return-value]
@doc(DataFrame.corrwith.__doc__)
def corrwith(
self,
other: DataFrame | Series,
axis: Axis = 0,
drop: bool = False,
method: CorrelationMethod = "pearson",
numeric_only: bool = False,
) -> DataFrame:
result = self._op_via_apply(
"corrwith",
other=other,
axis=axis,
drop=drop,
method=method,
numeric_only=numeric_only,
)
return result
def _wrap_transform_general_frame(
obj: DataFrame, group: DataFrame, res: DataFrame | Series
) -> DataFrame:
from pandas import concat
if isinstance(res, Series):
# we need to broadcast across the
# other dimension; this will preserve dtypes
# GH14457
if res.index.is_(obj.index):
res_frame = concat([res] * len(group.columns), axis=1)
res_frame.columns = group.columns
res_frame.index = group.index
else:
res_frame = obj._constructor(
np.tile(res.values, (len(group.index), 1)),
columns=group.columns,
index=group.index,
)
assert isinstance(res_frame, DataFrame)
return res_frame
elif isinstance(res, DataFrame) and not res.index.is_(group.index):
return res._align_frame(group)[0]
else:
return res
Reference in New Issue Copy Permalink