from datetime import ( date, timedelta, ) from decimal import Decimal import re from dateutil.tz import tzlocal import numpy as np import pytest from pandas.compat import ( IS64, is_platform_windows, ) from pandas.errors import Pandas4Warning import pandas.util._test_decorators as td import pandas as pd import pandas._testing as tm from pandas.core import ( algorithms, nanops, ) from pandas.tests.extension.decimal import DecimalArray is_windows_np2_or_is32 = not IS64 is_windows_or_is32 = is_platform_windows() or not IS64 def _make_2d_ea_df(col_arrays, col_names): """ Construct a DataFrame with a single 2D ExtensionArray block. Used for dt64tz and period types which support 2D blocks but don't consolidate automatically. """ ea_type = type(col_arrays[0]) ea_2d = ea_type._simple_new( np.stack([arr._ndarray for arr in col_arrays]), dtype=col_arrays[0].dtype, ) return pd.DataFrame(ea_2d.T, columns=col_names) def make_skipna_wrapper(alternative, skipna_alternative=None): """ Create a function for calling on an array. Parameters ---------- alternative : function The function to be called on the array with no NaNs. Only used when 'skipna_alternative' is None. skipna_alternative : function The function to be called on the original array Returns ------- function """ if skipna_alternative: def skipna_wrapper(x): return skipna_alternative(x.values) else: def skipna_wrapper(x): nona = x.dropna() if len(nona) == 0: return np.nan return alternative(nona) return skipna_wrapper def assert_stat_op_calc( opname, alternative, frame, has_skipna=True, check_dtype=True, check_dates=False, rtol=1e-5, atol=1e-8, skipna_alternative=None, ): """ Check that operator opname works as advertised on frame Parameters ---------- opname : str Name of the operator to test on frame alternative : function Function that opname is tested against; i.e. "frame.opname()" should equal "alternative(frame)". frame : DataFrame The object that the tests are executed on has_skipna : bool, default True Whether the method "opname" has the kwarg "skipna" check_dtype : bool, default True Whether the dtypes of the result of "frame.opname()" and "alternative(frame)" should be checked. check_dates : bool, default false Whether opname should be tested on a Datetime Series rtol : float, default 1e-5 Relative tolerance. atol : float, default 1e-8 Absolute tolerance. skipna_alternative : function, default None NaN-safe version of alternative """ f = getattr(frame, opname) if check_dates: df = pd.DataFrame({"b": pd.date_range("1/1/2001", periods=2)}) with tm.assert_produces_warning(None): result = getattr(df, opname)() assert isinstance(result, pd.Series) df["a"] = range(len(df)) with tm.assert_produces_warning(None): result = getattr(df, opname)() assert isinstance(result, pd.Series) assert len(result) if has_skipna: def wrapper(x): return alternative(x.values) skipna_wrapper = make_skipna_wrapper(alternative, skipna_alternative) result0 = f(axis=0, skipna=False) result1 = f(axis=1, skipna=False) tm.assert_series_equal( result0, frame.apply(wrapper), check_dtype=check_dtype, rtol=rtol, atol=atol ) tm.assert_series_equal( result1, frame.apply(wrapper, axis=1), rtol=rtol, atol=atol, ) else: skipna_wrapper = alternative result0 = f(axis=0) result1 = f(axis=1) tm.assert_series_equal( result0, frame.apply(skipna_wrapper), check_dtype=check_dtype, rtol=rtol, atol=atol, ) if opname in ["sum", "prod"]: expected = frame.apply(skipna_wrapper, axis=1) tm.assert_series_equal( result1, expected, check_dtype=False, rtol=rtol, atol=atol ) # check dtypes if check_dtype: lcd_dtype = frame.values.dtype assert lcd_dtype == result0.dtype assert lcd_dtype == result1.dtype # bad axis with pytest.raises(ValueError, match="No axis named 2"): f(axis=2) # all NA case if has_skipna: all_na = frame * np.nan r0 = getattr(all_na, opname)(axis=0) r1 = getattr(all_na, opname)(axis=1) if opname in ["sum", "prod"]: unit = 1 if opname == "prod" else 0 # result for empty sum/prod expected = pd.Series(unit, index=r0.index, dtype=r0.dtype) tm.assert_series_equal(r0, expected) expected = pd.Series(unit, index=r1.index, dtype=r1.dtype) tm.assert_series_equal(r1, expected) @pytest.fixture def bool_frame_with_na(): """ Fixture for DataFrame of booleans with index of unique strings Columns are ['A', 'B', 'C', 'D']; some entries are missing """ df = pd.DataFrame( np.concatenate( [np.ones((15, 4), dtype=bool), np.zeros((15, 4), dtype=bool)], axis=0 ), index=pd.Index([f"foo_{i}" for i in range(30)], dtype=object), columns=pd.Index(list("ABCD"), dtype=object), dtype=object, ) # set some NAs df.iloc[5:10] = np.nan df.iloc[15:20, -2:] = np.nan return df @pytest.fixture def float_frame_with_na(): """ Fixture for DataFrame of floats with index of unique strings Columns are ['A', 'B', 'C', 'D']; some entries are missing """ df = pd.DataFrame( np.random.default_rng(2).standard_normal((30, 4)), index=pd.Index([f"foo_{i}" for i in range(30)], dtype=object), columns=pd.Index(list("ABCD"), dtype=object), ) # set some NAs df.iloc[5:10] = np.nan df.iloc[15:20, -2:] = np.nan return df class TestDataFrameAnalytics: # --------------------------------------------------------------------- # Reductions @pytest.mark.parametrize("axis", [0, 1]) @pytest.mark.parametrize( "opname", [ "count", "sum", "mean", "product", "median", "min", "max", "nunique", "var", "std", "sem", pytest.param("skew", marks=td.skip_if_no("scipy")), pytest.param("kurt", marks=td.skip_if_no("scipy")), ], ) def test_stat_op_api_float_string_frame(self, float_string_frame, axis, opname): if (opname in ("sum", "min", "max") and axis == 0) or opname in ( "count", "nunique", ): getattr(float_string_frame, opname)(axis=axis) else: if opname in ["mean", "median", "var", "std", "sem", "skew", "kurt"]: # GH#66302 every statistic rejects the string column the same way msg = "Could not convert 'bar' to numeric" elif opname == "product": if axis == 1: msg = "can't multiply sequence by non-int of type 'float'" else: msg = "can't multiply sequence by non-int of type 'str'" elif opname == "sum": msg = r"unsupported operand type\(s\) for \+: 'float' and 'str'" elif opname in ["min", "max"]: msg = "'[><]=' not supported between instances of 'float' and 'str'" msg = "|".join([msg, "does not support", "Cannot perform reduction"]) with pytest.raises(TypeError, match=msg): getattr(float_string_frame, opname)(axis=axis) if opname != "nunique": getattr(float_string_frame, opname)(axis=axis, numeric_only=True) @pytest.mark.parametrize("axis", [0, 1]) @pytest.mark.parametrize( "opname", [ "count", "sum", "mean", "product", "median", "min", "max", "var", "std", "sem", pytest.param("skew", marks=td.skip_if_no("scipy")), pytest.param("kurt", marks=td.skip_if_no("scipy")), ], ) def test_stat_op_api_float_frame(self, float_frame, axis, opname): getattr(float_frame, opname)(axis=axis, numeric_only=False) def test_stat_op_calc(self, float_frame_with_na, mixed_float_frame): def count(s): return pd.notna(s).sum() def nunique(s): return len(algorithms.unique1d(s.dropna())) def var(x): return np.var(x, ddof=1) def std(x): return np.std(x, ddof=1) def sem(x): return np.std(x, ddof=1) / np.sqrt(len(x)) assert_stat_op_calc( "nunique", nunique, float_frame_with_na, has_skipna=False, check_dtype=False, check_dates=True, ) # GH#32571: rol needed for flaky CI builds # mixed types (with upcasting happening) assert_stat_op_calc( "sum", np.sum, mixed_float_frame.astype("float32"), check_dtype=False, rtol=1e-3, ) assert_stat_op_calc( "sum", np.sum, float_frame_with_na, skipna_alternative=np.nansum ) assert_stat_op_calc("mean", np.mean, float_frame_with_na, check_dates=True) assert_stat_op_calc( "product", np.prod, float_frame_with_na, skipna_alternative=np.nanprod ) assert_stat_op_calc("var", var, float_frame_with_na) assert_stat_op_calc("std", std, float_frame_with_na) assert_stat_op_calc("sem", sem, float_frame_with_na) assert_stat_op_calc( "count", count, float_frame_with_na, has_skipna=False, check_dtype=False, check_dates=True, ) def test_stat_op_calc_skew_kurtosis(self, float_frame_with_na): sp_stats = pytest.importorskip("scipy.stats") def skewness(x): if len(x) < 3: return np.nan return sp_stats.skew(x, bias=False) def kurt(x): if len(x) < 4: return np.nan return sp_stats.kurtosis(x, bias=False) assert_stat_op_calc("skew", skewness, float_frame_with_na) assert_stat_op_calc("kurt", kurt, float_frame_with_na) def test_median(self, float_frame_with_na, int_frame): def wrapper(x): if pd.isna(x).any(): return np.nan return np.median(x) assert_stat_op_calc("median", wrapper, float_frame_with_na, check_dates=True) assert_stat_op_calc( "median", wrapper, int_frame, check_dtype=False, check_dates=True ) @pytest.mark.parametrize( "method", ["sum", "mean", "prod", "var", "std", "skew", "min", "max"] ) @pytest.mark.parametrize( "df", [ pd.DataFrame( { "a": [ -0.00049987540199591344, -0.0016467257772919831, 0.00067695870775883013, ], "b": [-0, -0, 0.0], "c": [ 0.00031111847529610595, 0.0014902627951905339, -0.00094099200035979691, ], }, index=["foo", "bar", "baz"], dtype="O", ), pd.DataFrame( {0: [np.nan, 2], 1: [np.nan, 3], 2: [np.nan, 4]}, dtype=object ), ], ) def test_stat_operators_attempt_obj_array(self, method, df, axis): # GH#676 assert df.values.dtype == np.object_ result = getattr(df, method)(axis=axis) expected = getattr(df.astype("f8"), method)(axis=axis).astype(object) if axis in [1, "columns"] and method in ["min", "max"]: expected[expected.isna()] = None tm.assert_series_equal(result, expected) @pytest.mark.parametrize("op", ["mean", "std", "var", "skew", "kurt", "sem"]) def test_mixed_ops(self, op): # GH#16116 df = pd.DataFrame( { "int": [1, 2, 3, 4], "float": [1.0, 2.0, 3.0, 4.0], "str": ["a", "b", "c", "d"], } ) msg = "|".join( [ "Could not convert", "could not convert", "can't multiply sequence by non-int", "does not support", "Cannot perform", ] ) with pytest.raises(TypeError, match=msg): getattr(df, op)() with pd.option_context("use_bottleneck", False): with pytest.raises(TypeError, match=msg): getattr(df, op)() def test_reduce_mixed_frame(self): # GH 6806 df = pd.DataFrame( { "bool_data": [True, True, False, False, False], "int_data": [10, 20, 30, 40, 50], "string_data": ["a", "b", "c", "d", "e"], } ) df.reindex(columns=["bool_data", "int_data", "string_data"]) test = df.sum(axis=0) tm.assert_numpy_array_equal( test.values, np.array([2, 150, "abcde"], dtype=object) ) alt = df.T.sum(axis=1) tm.assert_series_equal(test, alt) def test_nunique(self): df = pd.DataFrame({"A": [1, 1, 1], "B": [1, 2, 3], "C": [1, np.nan, 3]}) tm.assert_series_equal(df.nunique(), pd.Series({"A": 1, "B": 3, "C": 2})) tm.assert_series_equal( df.nunique(dropna=False), pd.Series({"A": 1, "B": 3, "C": 3}) ) tm.assert_series_equal(df.nunique(axis=1), pd.Series([1, 2, 2])) tm.assert_series_equal(df.nunique(axis=1, dropna=False), pd.Series([1, 3, 2])) @pytest.mark.parametrize("tz", [None, "UTC"]) def test_mean_mixed_datetime_numeric(self, tz): # https://github.com/pandas-dev/pandas/issues/24752 df = pd.DataFrame({"A": [1, 1], "B": [pd.Timestamp("2000", tz=tz)] * 2}) result = df.mean() expected = pd.Series([1.0, pd.Timestamp("2000", tz=tz)], index=["A", "B"]) tm.assert_series_equal(result, expected) @pytest.mark.parametrize("tz", [None, "UTC"]) def test_mean_includes_datetimes(self, tz): # https://github.com/pandas-dev/pandas/issues/24752 # Behavior in 0.24.0rc1 was buggy. # As of 2.0 with numeric_only=None we do *not* drop datetime columns df = pd.DataFrame({"A": [pd.Timestamp("2000", tz=tz)] * 2}) result = df.mean() expected = pd.Series([pd.Timestamp("2000", tz=tz)], index=["A"]) tm.assert_series_equal(result, expected) def test_mean_mixed_string_decimal(self): # GH 11670 # possible bug when calculating mean of DataFrame? d = [ {"A": 2, "B": None, "C": Decimal("628.00")}, {"A": 1, "B": None, "C": Decimal("383.00")}, {"A": 3, "B": None, "C": Decimal("651.00")}, {"A": 2, "B": None, "C": Decimal("575.00")}, {"A": 4, "B": None, "C": Decimal("1114.00")}, {"A": 1, "B": "TEST", "C": Decimal("241.00")}, {"A": 2, "B": None, "C": Decimal("572.00")}, {"A": 4, "B": None, "C": Decimal("609.00")}, {"A": 3, "B": None, "C": Decimal("820.00")}, {"A": 5, "B": None, "C": Decimal("1223.00")}, ] df = pd.DataFrame(d) with pytest.raises( TypeError, match="|".join( [ "unsupported operand type", "does not support", "Cannot perform", "Could not convert", ] ), ): df.mean() result = df[["A", "C"]].mean() expected = pd.Series([2.7, 681.6], index=["A", "C"], dtype=object) tm.assert_series_equal(result, expected) def test_var_std(self, datetime_frame): result = datetime_frame.std(ddof=4) expected = datetime_frame.apply(lambda x: x.std(ddof=4)) tm.assert_almost_equal(result, expected) result = datetime_frame.var(ddof=4) expected = datetime_frame.apply(lambda x: x.var(ddof=4)) tm.assert_almost_equal(result, expected) arr = np.repeat(np.random.default_rng(2).random((1, 1000)), 1000, 0) result = nanops.nanvar(arr, axis=0) assert not (result < 0).any() with pd.option_context("use_bottleneck", False): result = nanops.nanvar(arr, axis=0) assert not (result < 0).any() @pytest.mark.parametrize("meth", ["sem", "var", "std", "mean", "median", "skew"]) def test_numeric_only_flag(self, meth): # GH 9201 df1 = pd.DataFrame( np.random.default_rng(2).standard_normal((5, 3)), columns=["foo", "bar", "baz"], ) # Cast to object to avoid implicit cast when setting entry to "100" below df1 = df1.astype({"foo": object}) # set one entry to a number in str format df1.loc[0, "foo"] = "100" df2 = pd.DataFrame( np.random.default_rng(2).standard_normal((5, 3)), columns=["foo", "bar", "baz"], ) # Cast to object to avoid implicit cast when setting entry to "a" below df2 = df2.astype({"foo": object}) # set one entry to a non-number str df2.loc[0, "foo"] = "a" result = getattr(df1, meth)(axis=1, numeric_only=True) expected = getattr(df1[["bar", "baz"]], meth)(axis=1) tm.assert_series_equal(expected, result) result = getattr(df2, meth)(axis=1, numeric_only=True) expected = getattr(df2[["bar", "baz"]], meth)(axis=1) tm.assert_series_equal(expected, result) # GH#66302 a numeric string is no more acceptable than any other string msg = "Could not convert '100' to numeric" with pytest.raises(TypeError, match=msg): getattr(df1, meth)(axis=1, numeric_only=False) msg = "Could not convert 'a' to numeric" with pytest.raises(TypeError, match=msg): getattr(df2, meth)(axis=1, numeric_only=False) def test_sem(self, datetime_frame): result = datetime_frame.sem(ddof=4) expected = datetime_frame.apply(lambda x: x.std(ddof=4) / np.sqrt(len(x))) tm.assert_almost_equal(result, expected) arr = np.repeat(np.random.default_rng(2).random((1, 1000)), 1000, 0) result = nanops.nansem(arr, axis=0) assert not (result < 0).any() with pd.option_context("use_bottleneck", False): result = nanops.nansem(arr, axis=0) assert not (result < 0).any() @pytest.mark.parametrize( "dropna, expected", [ ( True, { "A": [12], "B": [10.0], "C": [1.0], "D": ["a"], "E": pd.Categorical(["a"], categories=["a"]), "F": pd.DatetimeIndex(["2000-01-02"], dtype="M8[ns]"), "G": pd.to_timedelta(["1 days"]), }, ), ( False, { "A": [12], "B": [10.0], "C": [np.nan], "D": pd.Series([np.nan], dtype="str"), "E": pd.Categorical([np.nan], categories=["a"]), "F": pd.DatetimeIndex([pd.NaT], dtype="M8[ns]"), "G": pd.to_timedelta([pd.NaT]).as_unit("us"), }, ), ( True, { "H": [8, 9, np.nan, np.nan], "I": [8, 9, np.nan, np.nan], "J": [1, np.nan, np.nan, np.nan], "K": pd.Categorical( ["a", np.nan, np.nan, np.nan], categories=["a"] ), "L": pd.DatetimeIndex( ["2000-01-02", "NaT", "NaT", "NaT"], dtype="M8[ns]" ), "M": pd.to_timedelta(["1 days", "nan", "nan", "nan"]), "N": [0, 1, 2, 3], }, ), ( False, { "H": [8, 9, np.nan, np.nan], "I": [8, 9, np.nan, np.nan], "J": [1, np.nan, np.nan, np.nan], "K": pd.Categorical( [np.nan, "a", np.nan, np.nan], categories=["a"] ), "L": pd.DatetimeIndex( ["NaT", "2000-01-02", "NaT", "NaT"], dtype="M8[ns]" ), "M": pd.to_timedelta(["nan", "1 days", "nan", "nan"]), "N": [0, 1, 2, 3], }, ), ], ) def test_mode_dropna(self, dropna, expected): df = pd.DataFrame( { "A": [12, 12, 19, 11], "B": [10, 10, np.nan, 3], "C": [1, np.nan, np.nan, np.nan], "D": pd.Series([np.nan, np.nan, "a", np.nan], dtype="str"), "E": pd.Categorical([np.nan, np.nan, "a", np.nan]), "F": pd.DatetimeIndex( ["NaT", "2000-01-02", "NaT", "NaT"], dtype="M8[ns]" ), "G": pd.to_timedelta(["1 days", "nan", "nan", "nan"]), "H": [8, 8, 9, 9], "I": [9, 9, 8, 8], "J": [1, 1, np.nan, np.nan], "K": pd.Categorical(["a", np.nan, "a", np.nan]), "L": pd.DatetimeIndex( ["2000-01-02", "2000-01-02", "NaT", "NaT"], dtype="M8[ns]" ), "M": pd.to_timedelta(["1 days", "nan", "1 days", "nan"]), "N": np.arange(4, dtype="int64"), } ) result = df[sorted(expected.keys())].mode(dropna=dropna) expected = pd.DataFrame(expected) tm.assert_frame_equal(result, expected) def test_mode_sort_with_na(self, using_infer_string): df = pd.DataFrame({"A": [np.nan, np.nan, "a", "a"]}) expected = pd.DataFrame({"A": ["a", np.nan]}) result = df.mode(dropna=False) tm.assert_frame_equal(result, expected) def test_mode_empty_df(self): df = pd.DataFrame([], columns=["a", "b"]) expected = df.copy() result = df.mode() tm.assert_frame_equal(result, expected) def test_operators_timedelta64(self): df = pd.DataFrame( { "A": pd.date_range("2012-1-1", periods=3, freq="D", unit="ns"), "B": pd.date_range("2012-1-2", periods=3, freq="D", unit="ns"), "C": pd.Timestamp("20120101") - timedelta(minutes=5, seconds=5), } ) diffs = pd.DataFrame({"A": df["A"] - df["C"], "B": df["A"] - df["B"]}) # min result = diffs.min() assert result.iloc[0] == diffs.loc[0, "A"] assert result.iloc[1] == diffs.loc[0, "B"] result = diffs.min(axis=1) assert (result == diffs.loc[0, "B"]).all() # max result = diffs.max() assert result.iloc[0] == diffs.loc[2, "A"] assert result.iloc[1] == diffs.loc[2, "B"] result = diffs.max(axis=1) assert (result == diffs["A"]).all() # abs result = diffs.abs() result2 = abs(diffs) expected = pd.DataFrame({"A": df["A"] - df["C"], "B": df["B"] - df["A"]}) tm.assert_frame_equal(result, expected) tm.assert_frame_equal(result2, expected) # mixed frame mixed = diffs.copy() mixed["C"] = "foo" mixed["D"] = 1 mixed["E"] = 1.0 mixed["F"] = pd.Timestamp("20130101") # results in an object array result = mixed.min() expected = pd.Series( [ pd.Timedelta(timedelta(seconds=5 * 60 + 5)), pd.Timedelta(timedelta(days=-1)), "foo", 1, 1.0, pd.Timestamp("20130101"), ], index=mixed.columns, ) tm.assert_series_equal(result, expected) # excludes non-numeric result = mixed.min(axis=1, numeric_only=True) expected = pd.Series([1, 1, 1.0]) tm.assert_series_equal(result, expected) # works when only those columns are selected result = mixed[["A", "B"]].min(axis=1) expected = pd.Series([timedelta(days=-1)] * 3, dtype="m8[ns]") tm.assert_series_equal(result, expected) result = mixed[["A", "B"]].min() expected = pd.Series( [timedelta(seconds=5 * 60 + 5), timedelta(days=-1)], index=["A", "B"], dtype="m8[ns]", ) tm.assert_series_equal(result, expected) # GH 3106 df = pd.DataFrame( { "time": pd.date_range("20130102", periods=5, unit="ns"), "time2": pd.date_range("20130105", periods=5, unit="ns"), } ) df["off1"] = df["time2"] - df["time"] assert df["off1"].dtype == "timedelta64[ns]" df["off2"] = df["time"] - df["time2"] df._consolidate_inplace() assert df["off1"].dtype == "timedelta64[ns]" assert df["off2"].dtype == "timedelta64[ns]" def test_std_timedelta64_skipna_false(self): # GH#37392 tdi = pd.timedelta_range("1 Day", periods=10) df = pd.DataFrame({"A": tdi, "B": tdi}, copy=True) df.iloc[-2, -1] = pd.NaT result = df.std(skipna=False) expected = pd.Series( [df["A"].std(), pd.NaT], index=["A", "B"], dtype="timedelta64[us]" ) tm.assert_series_equal(result, expected) result = df.std(axis=1, skipna=False) expected = pd.Series( [pd.Timedelta(0)] * 8 + [pd.NaT, pd.Timedelta(0)], dtype="m8[us]" ) tm.assert_series_equal(result, expected) @pytest.mark.parametrize( "values", [["2022-01-01", "2022-01-02", pd.NaT, "2022-01-03"], 4 * [pd.NaT]] ) def test_std_datetime64_with_nat(self, values, skipna, request, unit): # GH#51335 dti = pd.to_datetime(values).as_unit(unit) df = pd.DataFrame({"a": dti}) result = df.std(skipna=skipna) if not skipna or all(value is pd.NaT for value in values): expected = pd.Series({"a": pd.NaT}, dtype=f"timedelta64[{unit}]") else: expected = pd.Series({"a": "1 days"}, dtype=f"timedelta64[{unit}]") tm.assert_series_equal(result, expected) def test_sum_corner(self): empty_frame = pd.DataFrame() axis0 = empty_frame.sum(axis=0) axis1 = empty_frame.sum(axis=1) assert isinstance(axis0, pd.Series) assert isinstance(axis1, pd.Series) assert len(axis0) == 0 assert len(axis1) == 0 @pytest.mark.parametrize( "index", [ pd.RangeIndex(0), pd.DatetimeIndex([]), pd.Index([], dtype=np.int64), pd.Index([], dtype=np.float64), pd.DatetimeIndex([], freq="ME"), pd.PeriodIndex([], freq="D"), ], ) def test_axis_1_empty(self, all_reductions, index): df = pd.DataFrame(columns=["a"], index=index) result = getattr(df, all_reductions)(axis=1) if all_reductions in ("any", "all"): expected_dtype = "bool" elif all_reductions == "count": expected_dtype = "int64" else: expected_dtype = "object" expected = pd.Series([], index=index, dtype=expected_dtype) tm.assert_series_equal(result, expected) @pytest.mark.parametrize("min_count", [0, 1]) def test_axis_1_sum_na(self, string_dtype_no_object, skipna, min_count): # https://github.com/pandas-dev/pandas/issues/60229 dtype = string_dtype_no_object df = pd.DataFrame({"a": [pd.NA]}, dtype=dtype) result = df.sum(axis=1, skipna=skipna, min_count=min_count) value = "" if skipna and min_count == 0 else pd.NA expected = pd.Series([value], dtype=dtype) tm.assert_series_equal(result, expected) @pytest.mark.parametrize( "min_count, expected", [ (1, [7.0, 3.0, np.nan]), (2, [7.0, np.nan, np.nan]), (3, [np.nan, np.nan, np.nan]), ], ) def test_axis_1_sum_min_count_sparse(self, min_count, expected): # https://github.com/pandas-dev/pandas/issues/55123 df = pd.DataFrame({"A": [2.0, 3.0, np.nan], "B": [5.0, np.nan, np.nan]}).astype( pd.SparseDtype("float64") ) result = df.sum(axis=1, skipna=True, min_count=min_count) expected = pd.Series(expected, dtype="float64") tm.assert_series_equal(result, expected) @pytest.mark.parametrize("method, unit", [("sum", 0), ("prod", 1)]) @pytest.mark.parametrize("numeric_only", [True, False]) def test_sum_prod_nanops(self, method, unit, numeric_only): idx = ["a", "b", "c"] df = pd.DataFrame( {"a": [unit, unit], "b": [unit, np.nan], "c": [np.nan, np.nan]} ) # The default result = getattr(df, method)(numeric_only=numeric_only) expected = pd.Series([unit, unit, unit], index=idx, dtype="float64") tm.assert_series_equal(result, expected) # min_count=1 result = getattr(df, method)(numeric_only=numeric_only, min_count=1) expected = pd.Series([unit, unit, np.nan], index=idx) tm.assert_series_equal(result, expected) # min_count=0 result = getattr(df, method)(numeric_only=numeric_only, min_count=0) expected = pd.Series([unit, unit, unit], index=idx, dtype="float64") tm.assert_series_equal(result, expected) result = getattr(df.iloc[1:], method)(numeric_only=numeric_only, min_count=1) expected = pd.Series([unit, np.nan, np.nan], index=idx) tm.assert_series_equal(result, expected) # min_count > 1 df = pd.DataFrame({"A": [unit] * 10, "B": [unit] * 5 + [np.nan] * 5}) result = getattr(df, method)(numeric_only=numeric_only, min_count=5) expected = pd.Series(result, index=["A", "B"]) tm.assert_series_equal(result, expected) result = getattr(df, method)(numeric_only=numeric_only, min_count=6) expected = pd.Series(result, index=["A", "B"]) tm.assert_series_equal(result, expected) def test_sum_nanops_timedelta(self): # prod isn't defined on timedeltas idx = ["a", "b", "c"] df = pd.DataFrame({"a": [0, 0], "b": [0, np.nan], "c": [np.nan, np.nan]}) df2 = df.apply(pd.to_timedelta) # 0 by default result = df2.sum() expected = pd.Series([0, 0, 0], dtype="m8[ns]", index=idx) tm.assert_series_equal(result, expected) # min_count=0 result = df2.sum(min_count=0) tm.assert_series_equal(result, expected) # min_count=1 result = df2.sum(min_count=1) expected = pd.Series([0, 0, np.nan], dtype="m8[ns]", index=idx) tm.assert_series_equal(result, expected) def test_sum_nanops_min_count(self): # https://github.com/pandas-dev/pandas/issues/39738 df = pd.DataFrame({"x": [1, 2, 3], "y": [4, 5, 6]}) result = df.sum(min_count=10) expected = pd.Series([np.nan, np.nan], index=["x", "y"]) tm.assert_series_equal(result, expected) @pytest.mark.parametrize("float_type", ["float16", "float32", "float64"]) @pytest.mark.parametrize( "kwargs, expected_result", [ ({"axis": 1, "min_count": 2}, [3.2, 5.3, np.nan]), ({"axis": 1, "min_count": 3}, [np.nan, np.nan, np.nan]), ({"axis": 1, "skipna": False}, [3.2, 5.3, np.nan]), ], ) def test_sum_nanops_dtype_min_count(self, float_type, kwargs, expected_result): # GH#46947 df = pd.DataFrame( {"a": [1.0, 2.3, 4.4], "b": [2.2, 3, np.nan]}, dtype=float_type ) result = df.sum(**kwargs) # GH#43929 float16 upcasts to float64 in nansum to avoid overflow expected = pd.Series(expected_result, dtype=float_type) if float_type == "float16": expected = expected.astype("float64") tm.assert_series_equal(result, expected) @pytest.mark.parametrize("float_type", ["float16", "float32", "float64"]) @pytest.mark.parametrize( "kwargs, expected_result", [ ({"axis": 1, "min_count": 2}, [2.0, 4.0, np.nan]), ({"axis": 1, "min_count": 3}, [np.nan, np.nan, np.nan]), ({"axis": 1, "skipna": False}, [2.0, 4.0, np.nan]), ], ) def test_prod_nanops_dtype_min_count(self, float_type, kwargs, expected_result): # GH#46947 df = pd.DataFrame( {"a": [1.0, 2.0, 4.4], "b": [2.0, 2.0, np.nan]}, dtype=float_type ) result = df.prod(**kwargs) expected = pd.Series(expected_result).astype(float_type) tm.assert_series_equal(result, expected) def test_sum_object(self, float_frame): values = float_frame.values.astype(int) frame = pd.DataFrame( values, index=float_frame.index, columns=float_frame.columns ) deltas = frame * timedelta(1) deltas.sum() def test_sum_bool(self, float_frame): # ensure this works, bug report bools = np.isnan(float_frame) bools.sum(axis=1) bools.sum(axis=0) def test_sum_mixed_datetime(self): # GH#30886 df = pd.DataFrame( {"A": pd.date_range("2000", periods=4), "B": [1, 2, 3, 4]} ).reindex([2, 3, 4]) with pytest.raises(TypeError, match="does not support operation 'sum'"): df.sum() def test_mean_corner(self, float_frame): # take mean of boolean column float_frame["bool"] = float_frame["A"] > 0 means = float_frame.mean(axis=0) assert means["bool"] == float_frame["bool"].values.mean() def test_mean_datetimelike(self): # GH#24757 check that datetimelike are excluded by default, handled # correctly with numeric_only=True # As of 2.0, datetimelike are *not* excluded with numeric_only=None df = pd.DataFrame( { "A": np.arange(3), "B": pd.date_range("2016-01-01", periods=3), "C": pd.timedelta_range("1D", periods=3), "D": pd.period_range("2016", periods=3, freq="Y"), } ) result = df.mean(numeric_only=True) expected = pd.Series({"A": 1.0}) tm.assert_series_equal(result, expected) with pytest.raises(TypeError, match="mean is not implemented for PeriodArray"): df.mean() def test_mean_datetimelike_numeric_only_false(self): df = pd.DataFrame( { "A": np.arange(3), "B": pd.date_range("2016-01-01", periods=3), "C": pd.timedelta_range("1D", periods=3), } ) # datetime(tz) and timedelta work result = df.mean(numeric_only=False) expected = pd.Series({"A": 1, "B": df.loc[1, "B"], "C": df.loc[1, "C"]}) tm.assert_series_equal(result, expected) # mean of period is not allowed df["D"] = pd.period_range("2016", periods=3, freq="Y") with pytest.raises(TypeError, match="mean is not implemented for Period"): df.mean(numeric_only=False) def test_mean_extensionarray_numeric_only_true(self): # https://github.com/pandas-dev/pandas/issues/33256 arr = np.random.default_rng(2).integers(1000, size=(10, 5)) df = pd.DataFrame(arr, dtype="Int64") result = df.mean(numeric_only=True) expected = pd.DataFrame(arr).mean().astype("Float64") tm.assert_series_equal(result, expected) @pytest.mark.parametrize( "meth", ["std", "var", "mean", "median", "sem", "skew", "kurt"] ) def test_stats_mixed_type(self, float_string_frame, meth): msg = "Could not convert 'bar' to numeric" with pytest.raises(TypeError, match=msg): getattr(float_string_frame, meth)(axis=1) def test_sum_bools(self): df = pd.DataFrame(index=range(1), columns=range(10)) bools = pd.isna(df) assert bools.sum(axis=1)[0] == 10 @pytest.mark.parametrize( "input_data, expected_data", [ ({"a": ["483", "3"], "b": ["94", "759"]}, ["48394", "3759"]), ( {"a": ["483.948", "3.0"], "b": ["94.2", "759.93"]}, ["483.94894.2", "3.0759.93"], ), ({"a": ["483", "3.0"], "b": ["94.2", "79"]}, ["48394.2", "3.079"]), ], ) def test_sum_string_dtype_coercion(self, input_data, expected_data): # GH#22642 # Check that summing numeric strings results in concatenation # and not conversion to dtype int64 or float64 df = pd.DataFrame(input_data) expected = pd.Series(expected_data) result = df.sum(axis=1) tm.assert_series_equal(result, expected) # ---------------------------------------------------------------------- # Index of max / min @pytest.mark.parametrize("axis", [0, 1]) def test_idxmin(self, float_frame, int_frame, skipna, axis): frame = float_frame frame.iloc[5:10] = np.nan frame.iloc[15:20, -2:] = np.nan for df in [frame, int_frame]: if (not skipna or axis == 1) and df is not int_frame: if skipna: msg = "Encountered all NA values" else: msg = "Encountered an NA value" with pytest.raises(ValueError, match=msg): df.idxmin(axis=axis, skipna=skipna) with pytest.raises(ValueError, match=msg): df.idxmin(axis=axis, skipna=skipna) else: result = df.idxmin(axis=axis, skipna=skipna) expected = df.apply(pd.Series.idxmin, axis=axis, skipna=skipna) expected = expected.astype(df.index.dtype) tm.assert_series_equal(result, expected) @pytest.mark.parametrize("axis", [0, 1]) def test_idxmin_empty(self, index, skipna, axis): # GH53265 if axis == 0: frame = pd.DataFrame(index=index) else: frame = pd.DataFrame(columns=index) result = frame.idxmin(axis=axis, skipna=skipna) expected = pd.Series(dtype=index.dtype) tm.assert_series_equal(result, expected) @pytest.mark.parametrize("numeric_only", [True, False]) def test_idxmin_numeric_only(self, numeric_only): df = pd.DataFrame({"a": [2, 3, 1], "b": [2, 1, 1], "c": list("xyx")}) result = df.idxmin(numeric_only=numeric_only) if numeric_only: expected = pd.Series([2, 1], index=["a", "b"]) else: expected = pd.Series([2, 1, 0], index=["a", "b", "c"]) tm.assert_series_equal(result, expected) def test_idxmin_axis_2(self, float_frame): frame = float_frame msg = "No axis named 2 for object type DataFrame" with pytest.raises(ValueError, match=msg): frame.idxmin(axis=2) @pytest.mark.parametrize("axis", [0, 1]) def test_idxmax(self, float_frame, int_frame, skipna, axis): frame = float_frame frame.iloc[5:10] = np.nan frame.iloc[15:20, -2:] = np.nan for df in [frame, int_frame]: if (skipna is False or axis == 1) and df is frame: if skipna: msg = "Encountered all NA values" else: msg = "Encountered an NA value" with pytest.raises(ValueError, match=msg): df.idxmax(axis=axis, skipna=skipna) return result = df.idxmax(axis=axis, skipna=skipna) expected = df.apply(pd.Series.idxmax, axis=axis, skipna=skipna) expected = expected.astype(df.index.dtype) tm.assert_series_equal(result, expected) @pytest.mark.parametrize("axis", [0, 1]) def test_idxmax_empty(self, index, skipna, axis): # GH53265 if axis == 0: frame = pd.DataFrame(index=index) else: frame = pd.DataFrame(columns=index) result = frame.idxmax(axis=axis, skipna=skipna) expected = pd.Series(dtype=index.dtype) tm.assert_series_equal(result, expected) @pytest.mark.parametrize("numeric_only", [True, False]) def test_idxmax_numeric_only(self, numeric_only): df = pd.DataFrame({"a": [2, 3, 1], "b": [2, 1, 1], "c": list("xyx")}) result = df.idxmax(numeric_only=numeric_only) if numeric_only: expected = pd.Series([1, 0], index=["a", "b"]) else: expected = pd.Series([1, 0, 1], index=["a", "b", "c"]) tm.assert_series_equal(result, expected) def test_idxmax_arrow_types(self): # GH#55368 pytest.importorskip("pyarrow") df = pd.DataFrame({"a": [2, 3, 1], "b": [2, 1, 1]}, dtype="int64[pyarrow]") result = df.idxmax() expected = pd.Series([1, 0], index=["a", "b"]) tm.assert_series_equal(result, expected) result = df.idxmin() expected = pd.Series([2, 1], index=["a", "b"]) tm.assert_series_equal(result, expected) df = pd.DataFrame({"a": ["b", "c", "a"]}, dtype="string[pyarrow]") result = df.idxmax(numeric_only=False) expected = pd.Series([1], index=["a"]) tm.assert_series_equal(result, expected) result = df.idxmin(numeric_only=False) expected = pd.Series([2], index=["a"]) tm.assert_series_equal(result, expected) def test_idxmax_axis_2(self, float_frame): frame = float_frame msg = "No axis named 2 for object type DataFrame" with pytest.raises(ValueError, match=msg): frame.idxmax(axis=2) def test_idxmax_mixed_dtype(self): # don't cast to object, which would raise in nanops dti = pd.date_range("2016-01-01", periods=3) df = pd.DataFrame({1: [0, 2, 1], 2: range(3)[::-1], 3: dti}) result = df.idxmax() expected = pd.Series([1, 0, 2], index=range(1, 4)) tm.assert_series_equal(result, expected) result = df.idxmin() expected = pd.Series([0, 2, 0], index=range(1, 4)) tm.assert_series_equal(result, expected) # with NaTs df.loc[0, 3] = pd.NaT result = df.idxmax() expected = pd.Series([1, 0, 2], index=range(1, 4)) tm.assert_series_equal(result, expected) result = df.idxmin() expected = pd.Series([0, 2, 1], index=range(1, 4)) tm.assert_series_equal(result, expected) # with multi-column dt64 block df[4] = dti[::-1] df._consolidate_inplace() result = df.idxmax() expected = pd.Series([1, 0, 2, 0], index=range(1, 5)) tm.assert_series_equal(result, expected) result = df.idxmin() expected = pd.Series([0, 2, 1, 2], index=range(1, 5)) tm.assert_series_equal(result, expected) @pytest.mark.parametrize( "op, expected_value", [("idxmax", [0, 4]), ("idxmin", [0, 5])], ) def test_idxmax_idxmin_convert_dtypes(self, op, expected_value): # GH 40346 df = pd.DataFrame( { "ID": [100, 100, 100, 200, 200, 200], "value": [0, 0, 0, 1, 2, 0], }, dtype="Int64", ) df = df.groupby("ID") result = getattr(df, op)() expected = pd.DataFrame( {"value": expected_value}, index=pd.Index([100, 200], name="ID", dtype="Int64"), ) tm.assert_frame_equal(result, expected) def test_idxmax_dt64_multicolumn_axis1(self): dti = pd.date_range("2016-01-01", periods=3) df = pd.DataFrame({3: dti, 4: dti[::-1]}, copy=True) df.iloc[0, 0] = pd.NaT df._consolidate_inplace() result = df.idxmax(axis=1) expected = pd.Series([4, 3, 3]) tm.assert_series_equal(result, expected) result = df.idxmin(axis=1) expected = pd.Series([4, 3, 4]) tm.assert_series_equal(result, expected) # ---------------------------------------------------------------------- # Logical reductions @pytest.mark.parametrize("axis", [0, 1]) @pytest.mark.parametrize("bool_only", [False, True]) def test_any_all_mixed_float( self, all_boolean_reductions, axis, bool_only, float_string_frame ): # make sure op works on mixed-type frame mixed = float_string_frame mixed["_bool_"] = np.random.default_rng(2).standard_normal(len(mixed)) > 0.5 getattr(mixed, all_boolean_reductions)(axis=axis, bool_only=bool_only) @pytest.mark.parametrize("axis", [0, 1]) def test_any_all_bool_with_na( self, all_boolean_reductions, axis, bool_frame_with_na ): getattr(bool_frame_with_na, all_boolean_reductions)(axis=axis, bool_only=False) def test_any_all_bool_frame(self, all_boolean_reductions, bool_frame_with_na): # GH#12863: numpy gives back non-boolean data for object type # so fill NaNs to compare with pandas behavior frame = bool_frame_with_na.fillna(True) alternative = getattr(np, all_boolean_reductions) f = getattr(frame, all_boolean_reductions) def skipna_wrapper(x): nona = x.dropna().values return alternative(nona) def wrapper(x): return alternative(x.values) result0 = f(axis=0, skipna=False) result1 = f(axis=1, skipna=False) tm.assert_series_equal(result0, frame.apply(wrapper)) tm.assert_series_equal(result1, frame.apply(wrapper, axis=1)) result0 = f(axis=0) result1 = f(axis=1) tm.assert_series_equal(result0, frame.apply(skipna_wrapper)) tm.assert_series_equal( result1, frame.apply(skipna_wrapper, axis=1), check_dtype=False ) # bad axis with pytest.raises(ValueError, match="No axis named 2"): f(axis=2) # all NA case all_na = frame * np.nan r0 = getattr(all_na, all_boolean_reductions)(axis=0) r1 = getattr(all_na, all_boolean_reductions)(axis=1) if all_boolean_reductions == "any": assert not r0.any() assert not r1.any() else: assert r0.all() assert r1.all() def test_any_all_extra(self, using_python_scalars): df = pd.DataFrame( { "A": [True, False, False], "B": [True, True, False], "C": [True, True, True], }, index=["a", "b", "c"], ) result = df[["A", "B"]].any(axis=1) expected = pd.Series([True, True, False], index=["a", "b", "c"]) tm.assert_series_equal(result, expected) result = df[["A", "B"]].any(axis=1, bool_only=True) tm.assert_series_equal(result, expected) result = df.all(axis=1) expected = pd.Series([True, False, False], index=["a", "b", "c"]) tm.assert_series_equal(result, expected) result = df.all(axis=1, bool_only=True) tm.assert_series_equal(result, expected) # Axis is None result = df.all(axis=None) if not using_python_scalars: result = result.item() assert result is False result = df.any(axis=None) if not using_python_scalars: result = result.item() assert result is True result = df[["C"]].all(axis=None) if not using_python_scalars: result = result.item() assert result is True @pytest.mark.parametrize("axis", [0, 1]) def test_any_all_object_dtype(self, axis, all_boolean_reductions, skipna): # GH#35450 df = pd.DataFrame( data=[ [1, np.nan, np.nan, True], [np.nan, 2, np.nan, True], [np.nan, np.nan, np.nan, True], [np.nan, np.nan, "5", np.nan], ] ) result = getattr(df, all_boolean_reductions)(axis=axis, skipna=skipna) expected = pd.Series([True, True, True, True]) tm.assert_series_equal(result, expected) def test_any_datetime(self): # GH 23070 float_data = [1, np.nan, 3, np.nan] datetime_data = [ pd.Timestamp("1960-02-15"), pd.Timestamp("1960-02-16"), pd.NaT, pd.NaT, ] df = pd.DataFrame({"A": float_data, "B": datetime_data}) msg = "'any' with datetime64 dtypes is not supported" with pytest.raises(TypeError, match=msg): df.any(axis=1) def test_any_all_bool_only(self): # GH 25101 df = pd.DataFrame( {"col1": [1, 2, 3], "col2": [4, 5, 6], "col3": [None, None, None]}, columns=pd.Index(["col1", "col2", "col3"], dtype=object), ) result = df.all(bool_only=True) expected = pd.Series(dtype=np.bool_, index=[]) tm.assert_series_equal(result, expected) df = pd.DataFrame( { "col1": [1, 2, 3], "col2": [4, 5, 6], "col3": [None, None, None], "col4": [False, False, True], } ) result = df.all(bool_only=True) expected = pd.Series({"col4": False}) tm.assert_series_equal(result, expected) @pytest.mark.parametrize( "func, data, expected", [ (np.any, {}, False), (np.all, {}, True), (np.any, {"A": []}, False), (np.all, {"A": []}, True), (np.any, {"A": [False, False]}, False), (np.all, {"A": [False, False]}, False), (np.any, {"A": [True, False]}, True), (np.all, {"A": [True, False]}, False), (np.any, {"A": [True, True]}, True), (np.all, {"A": [True, True]}, True), (np.any, {"A": [False], "B": [False]}, False), (np.all, {"A": [False], "B": [False]}, False), (np.any, {"A": [False, False], "B": [False, True]}, True), (np.all, {"A": [False, False], "B": [False, True]}, False), # other types (np.all, {"A": pd.Series([0.0, 1.0], dtype="float")}, False), (np.any, {"A": pd.Series([0.0, 1.0], dtype="float")}, True), (np.all, {"A": pd.Series([0, 1], dtype=int)}, False), (np.any, {"A": pd.Series([0, 1], dtype=int)}, True), pytest.param(np.all, {"A": pd.Series([0, 1], dtype="M8[ns]")}, False), pytest.param(np.all, {"A": pd.Series([0, 1], dtype="M8[ns, UTC]")}, False), pytest.param(np.any, {"A": pd.Series([0, 1], dtype="M8[ns]")}, True), pytest.param(np.any, {"A": pd.Series([0, 1], dtype="M8[ns, UTC]")}, True), pytest.param(np.all, {"A": pd.Series([1, 2], dtype="M8[ns]")}, True), pytest.param(np.all, {"A": pd.Series([1, 2], dtype="M8[ns, UTC]")}, True), pytest.param(np.any, {"A": pd.Series([1, 2], dtype="M8[ns]")}, True), pytest.param(np.any, {"A": pd.Series([1, 2], dtype="M8[ns, UTC]")}, True), pytest.param(np.all, {"A": pd.Series([0, 1], dtype="m8[ns]")}, False), pytest.param(np.any, {"A": pd.Series([0, 1], dtype="m8[ns]")}, True), pytest.param(np.all, {"A": pd.Series([1, 2], dtype="m8[ns]")}, True), pytest.param(np.any, {"A": pd.Series([1, 2], dtype="m8[ns]")}, True), # np.all on Categorical raises, so the reduction drops the # column, so all is being done on an empty Series, so is True (np.all, {"A": pd.Series([0, 1], dtype="category")}, True), (np.any, {"A": pd.Series([0, 1], dtype="category")}, False), (np.all, {"A": pd.Series([1, 2], dtype="category")}, True), (np.any, {"A": pd.Series([1, 2], dtype="category")}, False), # Mix GH#21484 pytest.param( np.all, { "A": pd.Series([10, 20], dtype="M8[ns]"), "B": pd.Series([10, 20], dtype="m8[ns]"), }, True, ), ], ) def test_any_all_np_func(self, func, data, expected, using_python_scalars): # GH 19976 data = pd.DataFrame(data) if any(isinstance(x, pd.CategoricalDtype) for x in data.dtypes): with pytest.raises( TypeError, match=".* dtype category does not support operation" ): func(data) # method version with pytest.raises( TypeError, match=".* dtype category does not support operation" ): getattr(pd.DataFrame(data), func.__name__)(axis=None) if data.dtypes.apply(lambda x: x.kind == "M").any(): # GH#34479 msg = "'(any|all)' with datetime64 dtypes is not supported" with pytest.raises(TypeError, match=msg): func(data) # method version with pytest.raises(TypeError, match=msg): getattr(pd.DataFrame(data), func.__name__)(axis=None) elif data.dtypes.apply(lambda x: x != "category").any(): result = func(data) if using_python_scalars: assert result is expected else: assert isinstance(result, np.bool_) assert result.item() is expected # method version result = getattr(pd.DataFrame(data), func.__name__)(axis=None) if using_python_scalars: assert result is expected else: assert isinstance(result, np.bool_) assert result.item() is expected def test_any_all_object(self, using_python_scalars): # GH 19976 result = np.all(pd.DataFrame(columns=["a", "b"])) if not using_python_scalars: result = result.item() assert result is True result = np.any(pd.DataFrame(columns=["a", "b"])) if not using_python_scalars: result = result.item() assert result is False def test_any_all_object_bool_only(self): df = pd.DataFrame({"A": ["foo", 2], "B": [True, False]}).astype(object) df._consolidate_inplace() df["C"] = pd.Series([True, True]) # Categorical of bools is _not_ considered booly df["D"] = df["C"].astype("category") # The underlying bug is in DataFrame._get_bool_data, so we check # that while we're here res = df._get_bool_data() expected = df[["C"]] tm.assert_frame_equal(res, expected) res = df.all(bool_only=True, axis=0) expected = pd.Series([True], index=["C"]) tm.assert_series_equal(res, expected) # operating on a subset of columns should not produce a _larger_ Series res = df[["B", "C"]].all(bool_only=True, axis=0) tm.assert_series_equal(res, expected) assert df.all(bool_only=True, axis=None) res = df.any(bool_only=True, axis=0) expected = pd.Series([True], index=["C"]) tm.assert_series_equal(res, expected) # operating on a subset of columns should not produce a _larger_ Series res = df[["C"]].any(bool_only=True, axis=0) tm.assert_series_equal(res, expected) assert df.any(bool_only=True, axis=None) @pytest.mark.parametrize( "dtype", [ "boolean", "Int8", "Int16", "Int32", "Int64", "UInt8", "UInt16", "UInt32", "UInt64", "Float32", "Float64", ], ) @pytest.mark.parametrize( "data, op, expected_value", [ ([False, None], "any", pd.NA), ([True, None], "all", pd.NA), ([None, None], "any", pd.NA), ([None, None], "all", pd.NA), ([True, None], "any", True), ], ) def test_any_all_skipna_false_nullable(self, dtype, data, op, expected_value): # GH#65710 if dtype != "boolean": data = [(0 if v is False else 1 if v is True else None) for v in data] arr = pd.array(data, dtype=dtype) df = pd.DataFrame({"a": arr}) result = getattr(df, op)(skipna=False) expected = pd.Series([expected_value], index=["a"], dtype="boolean") tm.assert_series_equal(result, expected) ser_result = getattr(pd.Series(arr), op)(skipna=False) if expected_value is pd.NA: assert ser_result is pd.NA else: assert ser_result == expected_value def test_any_skipna_false_axis_none_nullable(self): # GH#65710 df = pd.DataFrame({"a": pd.array([False, None], dtype="boolean")}) assert df.any(skipna=False, axis=None) is pd.NA # --------------------------------------------------------------------- # Unsorted def test_series_broadcasting(self): # smoke test for numpy warnings # GH 16378, GH 16306 df = pd.DataFrame([1.0, 1.0, 1.0]) df_nan = pd.DataFrame({"A": [np.nan, 2.0, np.nan]}) s = pd.Series([1, 1, 1]) s_nan = pd.Series([np.nan, np.nan, 1]) with tm.assert_produces_warning(None): df_nan.clip(lower=s, axis=0) for op in ["lt", "le", "gt", "ge", "eq", "ne"]: getattr(df, op)(s_nan, axis=0) class TestDataFrameReductions: def test_min_max_dt64_with_NaT(self): # Both NaT and Timestamp are in DataFrame. df = pd.DataFrame({"foo": [pd.NaT, pd.NaT, pd.Timestamp("2012-05-01")]}) res = df.min() exp = pd.Series([pd.Timestamp("2012-05-01")], index=["foo"]) tm.assert_series_equal(res, exp) res = df.max() exp = pd.Series([pd.Timestamp("2012-05-01")], index=["foo"]) tm.assert_series_equal(res, exp) # GH12941, only NaTs are in DataFrame. df = pd.DataFrame({"foo": [pd.NaT, pd.NaT]}) res = df.min() exp = pd.Series([pd.NaT], index=["foo"]) tm.assert_series_equal(res, exp) res = df.max() exp = pd.Series([pd.NaT], index=["foo"]) tm.assert_series_equal(res, exp) def test_min_max_dt64_with_NaT_precision(self): # GH#60646 Make sure the reduction doesn't cast input timestamps to # float and lose precision. df = pd.DataFrame( {"foo": [pd.NaT, pd.NaT, pd.Timestamp("2012-05-01 09:20:00.123456789")]}, dtype="datetime64[ns]", ) res = df.min(axis=1) exp = df.foo.rename(None) tm.assert_series_equal(res, exp) res = df.max(axis=1) exp = df.foo.rename(None) tm.assert_series_equal(res, exp) def test_min_max_td64_with_NaT_precision(self): # GH#60646 Make sure the reduction doesn't cast input timedeltas to # float and lose precision. df = pd.DataFrame( { "foo": [ pd.NaT, pd.NaT, pd.to_timedelta("10000 days 06:05:01.123456789"), ], }, dtype="timedelta64[ns]", ) res = df.min(axis=1) exp = df.foo.rename(None) tm.assert_series_equal(res, exp) res = df.max(axis=1) exp = df.foo.rename(None) tm.assert_series_equal(res, exp) def test_min_max_dt64_with_NaT_skipna_false(self, request, tz_naive_fixture): # GH#36907 tz = tz_naive_fixture if isinstance(tz, tzlocal) and is_platform_windows(): pytest.skip( "GH#37659 OSError raised within tzlocal bc Windows " "chokes in times before 1970-01-01" ) df = pd.DataFrame( { "a": [ pd.Timestamp("2020-01-01 08:00:00", tz=tz), pd.Timestamp("1920-02-01 09:00:00", tz=tz), ], "b": [pd.Timestamp("2020-02-01 08:00:00", tz=tz), pd.NaT], } ) res = df.min(axis=1, skipna=False) expected = pd.Series([df.loc[0, "a"], pd.NaT]) assert expected.dtype == df["a"].dtype tm.assert_series_equal(res, expected) res = df.max(axis=1, skipna=False) expected = pd.Series([df.loc[0, "b"], pd.NaT]) assert expected.dtype == df["a"].dtype tm.assert_series_equal(res, expected) def test_min_max_dt64_api_consistency_with_NaT(self): # Calling the following sum functions returned an error for dataframes but # returned NaT for series. These tests check that the API is consistent in # min/max calls on empty Series/DataFrames. See GH:33704 for more # information df = pd.DataFrame({"x": pd.to_datetime([])}) expected_dt_series = pd.Series(pd.to_datetime([])) # check axis 0 assert (df.min(axis=0).x is pd.NaT) == (expected_dt_series.min() is pd.NaT) assert (df.max(axis=0).x is pd.NaT) == (expected_dt_series.max() is pd.NaT) # check axis 1 tm.assert_series_equal(df.min(axis=1), expected_dt_series) tm.assert_series_equal(df.max(axis=1), expected_dt_series) def test_min_max_dt64_api_consistency_empty_df(self): # check DataFrame/Series api consistency when calling min/max on an empty # DataFrame/Series. df = pd.DataFrame({"x": []}) expected_float_series = pd.Series([], dtype=float) # check axis 0 assert np.isnan(df.min(axis=0).x) == np.isnan(expected_float_series.min()) assert np.isnan(df.max(axis=0).x) == np.isnan(expected_float_series.max()) # check axis 1 tm.assert_series_equal(df.min(axis=1), expected_float_series) tm.assert_series_equal(df.min(axis=1), expected_float_series) @pytest.mark.parametrize( "initial", ["2018-10-08 13:36:45+00:00", "2018-10-08 13:36:45+03:00"], # Non-UTC timezone ) @pytest.mark.parametrize("method", ["min", "max"]) def test_preserve_timezone(self, initial: str, method): # GH 28552 initial_dt = pd.to_datetime(initial) expected = pd.Series([initial_dt]) df = pd.DataFrame([expected]) result = getattr(df, method)(axis=1) tm.assert_series_equal(result, expected) @pytest.mark.parametrize("method", ["min", "max"]) def test_minmax_tzaware_skipna_axis_1(self, method, skipna): # GH#51242 val = pd.to_datetime("1900-01-01", utc=True) df = pd.DataFrame( {"a": pd.Series([pd.NaT, pd.NaT, val]), "b": pd.Series([pd.NaT, val, val])} ) op = getattr(df, method) result = op(axis=1, skipna=skipna) if skipna: expected = pd.Series([pd.NaT, val, val]) else: expected = pd.Series([pd.NaT, pd.NaT, val]) tm.assert_series_equal(result, expected) @pytest.fixture( params=[ pd.DataFrame( { "i1": np.array([1, 2, 3], dtype="int64"), "i2": np.array([4, 5, 6], dtype="int64"), } ), pd.DataFrame( { "f1": np.array([1.0, np.nan, 3.0], dtype="float64"), "f2": np.array([4.0, 5.0, np.nan], dtype="float64"), } ), pd.DataFrame( { "b1": np.array([True, False, True], dtype="bool"), "b2": np.array([False, True, False], dtype="bool"), } ), pd.DataFrame( { "d1": pd.to_datetime(["2020-01-01", pd.NaT, "2020-01-05"]), "d2": pd.to_datetime(["2020-01-03", "2020-01-02", "2020-01-04"]), } ), _make_2d_ea_df( [ pd.to_datetime(["2020-01-01", pd.NaT, "2020-01-05"]) .as_unit("us") .tz_localize("UTC") ._data, pd.to_datetime(["2020-01-03", "2020-01-02", "2020-01-04"]) .as_unit("us") .tz_localize("UTC") ._data, ], ["dz1", "dz2"], ), pd.DataFrame( { "td1": pd.to_timedelta(["1 day", pd.NaT, "3 days"]), "td2": pd.to_timedelta(["4 days", "5 days", pd.NaT]), } ), pd.DataFrame( { "I1": pd.array([1, 2, pd.NA], dtype="Int64"), "I2": pd.array([4, pd.NA, 6], dtype="Int64"), } ), pd.DataFrame( { "F1": pd.array([1.0, pd.NA, 3.0], dtype="Float64"), "F2": pd.array([4.0, 5.0, pd.NA], dtype="Float64"), } ), pd.DataFrame( { "B1": pd.array([True, pd.NA, True], dtype="boolean"), "B2": pd.array([False, True, pd.NA], dtype="boolean"), } ), _make_2d_ea_df( [ pd.period_range("2020-01-01", periods=3, freq="D")._data, pd.period_range("2020-01-04", periods=3, freq="D")._data, ], ["p1", "p2"], ), ], ids=[ "int64", "float64", "bool", "dt64", "dt64tz", "td64", "Int64", "Float64", "boolean", "period", ], ) def reduction_block_1(self, request): return request.param @pytest.fixture( params=[ pd.DataFrame( { "i3": np.array([7, 8, 9], dtype="int64"), "i4": np.array([10, 11, 12], dtype="int64"), } ), pd.DataFrame( { "f3": np.array([7.0, 8.0, np.nan], dtype="float64"), "f4": np.array([np.nan, 11.0, 12.0], dtype="float64"), } ), pd.DataFrame( { "b3": np.array([True, True, False], dtype="bool"), "b4": np.array([False, False, True], dtype="bool"), } ), pd.DataFrame( { "d3": pd.to_datetime(["2020-02-01", "2020-02-02", pd.NaT]), "d4": pd.to_datetime([pd.NaT, "2020-02-04", "2020-02-05"]), } ), _make_2d_ea_df( [ pd.to_datetime(["2020-02-01", "2020-02-02", pd.NaT]) .as_unit("us") .tz_localize("UTC") ._data, pd.to_datetime([pd.NaT, "2020-02-04", "2020-02-05"]) .as_unit("us") .tz_localize("UTC") ._data, ], ["dz3", "dz4"], ), pd.DataFrame( { "td3": pd.to_timedelta([pd.NaT, "8 days", "9 days"]), "td4": pd.to_timedelta(["10 days", pd.NaT, "12 days"]), } ), pd.DataFrame( { "I3": pd.array([7, pd.NA, 9], dtype="Int64"), "I4": pd.array([pd.NA, 11, 12], dtype="Int64"), } ), pd.DataFrame( { "F3": pd.array([pd.NA, 8.0, 9.0], dtype="Float64"), "F4": pd.array([10.0, pd.NA, 12.0], dtype="Float64"), } ), pd.DataFrame( { "B3": pd.array([pd.NA, True, False], dtype="boolean"), "B4": pd.array([True, pd.NA, True], dtype="boolean"), } ), _make_2d_ea_df( [ pd.period_range("2020-02-01", periods=3, freq="D")._data, pd.period_range("2020-02-04", periods=3, freq="D")._data, ], ["p3", "p4"], ), ], ids=[ "int64", "float64", "bool", "dt64", "dt64tz", "td64", "Int64", "Float64", "boolean", "period", ], ) def reduction_block_2(self, request): return request.param @pytest.mark.parametrize( "method", ["sum", "prod", "min", "max", "mean", "any", "all"] ) @pytest.mark.parametrize( "kwargs", [{}, {"min_count": 2}], ids=["default", "min_count"] ) def test_reduce_axis1_multiblock_exhaustive( self, reduction_block_1, reduction_block_2, method, skipna, kwargs ): # GH#51474 - exhaustive test: axis=1 result must match transpose path, # whether that is a successful result or a raised exception. if "min_count" in kwargs and method not in ("sum", "prod"): pytest.skip("min_count only applies to sum/prod") df = pd.concat([reduction_block_1, reduction_block_2], axis=1) assert len(df._mgr.blocks) > 1 kw = {"skipna": skipna, **kwargs} transpose_err = None try: expected = getattr(df.T, method)(**kw) except (TypeError, ValueError, AttributeError, RuntimeWarning) as err: transpose_err = err axis1_err = None try: result = getattr(df, method)(axis=1, **kw) except (TypeError, ValueError, AttributeError, RuntimeWarning) as err: axis1_err = err if transpose_err is not None or axis1_err is not None: # At least one path raises — acceptable. Mixed-dtype frames # may raise on one path (e.g. axis=1 with datetime64 + any) # but succeed on the other (transpose coerces to object first). return # Compare values, treating NaN and pd.NA as equivalent. # check_dtype=False because the transpose path may coerce to # object (e.g. mixed int64+bool) where axis=1 preserves # the numeric dtype. check_names=False because the transpose # path names the result after the index. result_na = pd.isna(result) expected_na = pd.isna(expected) # axis=1 may correctly propagate NAs that the transpose path # drops (e.g. skipna=False with mixed float+bool where transpose # coerces to a common dtype first), so only require that result # NAs are a superset of expected NAs. assert (result_na | ~expected_na).all(), ( f"axis=1 missing NAs that transpose has:\n" f"result NAs: {result_na.values}\nexpected NAs: {expected_na.values}" ) both_valid = ~result_na & ~expected_na if both_valid.any(): tm.assert_series_equal( result[both_valid].reset_index(drop=True), expected[both_valid].reset_index(drop=True), check_dtype=False, check_names=False, ) @pytest.mark.parametrize( "method", [ "sum", "prod", "min", "max", "mean", "any", "all", "median", "std", "var", "sem", "skew", "kurt", ], ) @pytest.mark.parametrize( "kwargs", [{}, {"min_count": 2}, {"min_count": 5}], ids=["default", "min_count", "min_count_high"], ) def test_reduce_axis1_single_block_exhaustive( self, reduction_block_1, method, skipna, kwargs ): # GH#51474: single-block fastpath must match the transpose path, # result or raised exception alike. if "min_count" in kwargs and method not in ("sum", "prod"): pytest.skip("min_count only applies to sum/prod") df = reduction_block_1 if len(df._mgr.blocks) != 1: # 1D EA columns don't consolidate; covered by the multiblock test pytest.skip("fixture frame is not single-block") kw = {"skipna": skipna, **kwargs} transpose_err = None expected = None try: expected = getattr(df.T, method)(**kw) except (TypeError, ValueError) as err: transpose_err = err if isinstance(df._mgr.blocks[0].values, np.ndarray): # numpy block: identical computation, so match exactly incl. dtype if transpose_err is not None: with pytest.raises( type(transpose_err), match=re.escape(str(transpose_err)) ): getattr(df, method)(axis=1, **kw) else: result = getattr(df, method)(axis=1, **kw) tm.assert_series_equal(result, expected, check_names=False) else: # 2D EA block: paths differ; mirror the multiblock test's lax compare axis1_err = None result = None try: result = getattr(df, method)(axis=1, **kw) except (TypeError, ValueError) as err: axis1_err = err if transpose_err is not None or axis1_err is not None: return result_na = pd.isna(result) expected_na = pd.isna(expected) assert (result_na == expected_na).all() both_valid = ~result_na & ~expected_na if both_valid.any(): tm.assert_series_equal( result[both_valid].reset_index(drop=True), expected[both_valid].reset_index(drop=True), check_dtype=False, check_names=False, ) @pytest.mark.parametrize("method", ["min", "max"]) def test_reduce_axis1_dt64tz_with_nat(self, method): # GH#65500: axis=1 min/max on multi-block dt64tz frames with NaT # must skip NaT under skipna=True, matching the transpose path. ts = pd.Timestamp("2026-01-01 15:13:44", tz="Europe/Budapest") df = pd.DataFrame({"a": [ts, ts], "b": [ts, pd.NaT]}) # Two separate blocks (CoW), same EA dtype. assert len(df._mgr.blocks) == 2 result = getattr(df, method)(axis=1) expected = pd.Series([ts, ts], dtype=df["a"].dtype) tm.assert_series_equal(result, expected) # skipna=False propagates NaT. result = getattr(df, method)(axis=1, skipna=False) expected = pd.Series([ts, pd.NaT], dtype=df["a"].dtype) tm.assert_series_equal(result, expected) @pytest.mark.parametrize("method", ["min", "max"]) def test_reduce_axis1_single_block_dt64tz_with_nat(self, method): # GH#51474, GH#65500: 2D EA block fastpath must skip NaT (skipna=True) # and propagate it (skipna=False) without raising or changing dtype. df = _make_2d_ea_df( [ pd.to_datetime(["2020-01-01", "2020-01-02"]) .as_unit("us") .tz_localize("UTC") ._data, pd.to_datetime(["2020-01-02", pd.NaT]) .as_unit("us") .tz_localize("UTC") ._data, ], ["dz1", "dz2"], ) # the fastpath only applies to a single 2D EA block assert len(df._mgr.blocks) == 1 assert df._mgr.blocks[0].values.ndim == 2 first = pd.Timestamp("2020-01-01", tz="UTC") second = pd.Timestamp("2020-01-02", tz="UTC") # row 0 is [first, second]; row 1 is [second, NaT] winner = first if method == "min" else second result = getattr(df, method)(axis=1) expected = pd.Series([winner, second], dtype=df["dz1"].dtype) tm.assert_series_equal(result, expected) result = getattr(df, method)(axis=1, skipna=False) expected = pd.Series([winner, pd.NaT], dtype=df["dz1"].dtype) tm.assert_series_equal(result, expected) @pytest.mark.parametrize( "values", [ pd.date_range("2020-01-01", periods=1), pd.date_range("2020-01-01", periods=1, tz="US/Pacific"), pd.timedelta_range("1 day", periods=1), pd.period_range("2020-01-01", periods=1, freq="D"), ], ids=["dt64", "dt64tz", "td64", "period"], ) def test_reduce_axis1_median_single_row_single_col(self, values): # GH#68191: the result held a 0-dim array that repr could not render df = pd.DataFrame({"a": values}) result = df.median(axis=1) tm.assert_series_equal(result, df["a"], check_names=False) def test_reduce_axis1_dt64tz_mixed_tz_falls_back(self): # GH#65500: different tz across blocks must not enter the EA fast # path (which would silently produce a wrong-dtype result); fall # through to the slow path so the result has object dtype. ts1 = pd.Timestamp("2026-01-01 15:13:44", tz="Europe/Budapest") ts2 = pd.Timestamp("2026-01-01 15:13:44", tz="Europe/Moscow") # ts2 is the earlier of the two in UTC terms df = pd.DataFrame({"a": [ts1, ts1], "b": [ts2, ts2]}) result = df.max(axis=1) expected = pd.Series([ts1, ts1], dtype=object) tm.assert_series_equal(result, expected) result = df.min(axis=1) expected = pd.Series([ts2, ts2], dtype=object) tm.assert_series_equal(result, expected) @pytest.mark.parametrize("method", ["min", "max"]) def test_reduce_axis1_dt64tz_mixed_tz_with_nat(self, method): # GH#65500: mixed-tz frames reduce via the object path; NaT must # be skipped under skipna=True instead of raising TypeError on # comparison with the NA fill value. ts1 = pd.Timestamp("2026-01-01 15:13:44", tz="Europe/Budapest") ts2 = pd.Timestamp("2026-01-01 15:13:44", tz="Europe/Moscow") # ts2 is the earlier of the two in UTC terms df = pd.DataFrame({"a": [ts1, ts1], "b": [ts2, pd.NaT]}) winner = ts2 if method == "min" else ts1 result = getattr(df, method)(axis=1) expected = pd.Series([winner, ts1], dtype=object) tm.assert_series_equal(result, expected) # GH#4147: the NaT row holds an NA, so skipna=False propagates it. # object-dtype reductions use None as the NA sentinel. result = getattr(df, method)(axis=1, skipna=False) expected = pd.Series([winner, None], dtype=object) tm.assert_series_equal(result, expected) def test_frame_any_with_timedelta(self): # GH#17667 df = pd.DataFrame( { "a": pd.Series([0, 0]), "t": pd.Series([pd.to_timedelta(0, "s"), pd.to_timedelta(1, "ms")]), } ) result = df.any(axis=0) expected = pd.Series(data=[False, True], index=["a", "t"]) tm.assert_series_equal(result, expected) result = df.any(axis=1) expected = pd.Series(data=[False, True]) tm.assert_series_equal(result, expected) def test_reductions_skipna_none_raises( self, request, frame_or_series, all_reductions ): if all_reductions == "count": request.applymarker( pytest.mark.xfail(reason="Count does not accept skipna") ) obj = frame_or_series([1, 2, 3]) msg = 'For argument "skipna" expected type bool, received type NoneType.' with pytest.raises(ValueError, match=msg): getattr(obj, all_reductions)(skipna=None) def test_reduction_timestamp_smallest_unit(self): # GH#52524 df = pd.DataFrame( { "a": pd.Series([pd.Timestamp("2019-12-31")], dtype="datetime64[s]"), "b": pd.Series( [pd.Timestamp("2019-12-31 00:00:00.123")], dtype="datetime64[ms]" ), } ) result = df.max() expected = pd.Series( [pd.Timestamp("2019-12-31"), pd.Timestamp("2019-12-31 00:00:00.123")], dtype="datetime64[ms]", index=["a", "b"], ) tm.assert_series_equal(result, expected) def test_reduction_timedelta_smallest_unit(self): # GH#52524 df = pd.DataFrame( { "a": pd.Series([pd.Timedelta("1 days")], dtype="timedelta64[s]"), "b": pd.Series([pd.Timedelta("1 days")], dtype="timedelta64[ms]"), } ) result = df.max() expected = pd.Series( [pd.Timedelta("1 days"), pd.Timedelta("1 days")], dtype="timedelta64[ms]", index=["a", "b"], ) tm.assert_series_equal(result, expected) class TestNuisanceColumns: def test_any_all_categorical_dtype_nuisance_column(self, all_boolean_reductions): # GH#36076 DataFrame should match Series behavior ser = pd.Series([0, 1], dtype="category", name="A") df = ser.to_frame() # Double-check the Series behavior is to raise with pytest.raises(TypeError, match="does not support operation"): getattr(ser, all_boolean_reductions)() with pytest.raises(TypeError, match="does not support operation"): getattr(np, all_boolean_reductions)(ser) with pytest.raises(TypeError, match="does not support operation"): getattr(df, all_boolean_reductions)(bool_only=False) with pytest.raises(TypeError, match="does not support operation"): getattr(df, all_boolean_reductions)(bool_only=None) with pytest.raises(TypeError, match="does not support operation"): getattr(np, all_boolean_reductions)(df, axis=0) def test_median_categorical_dtype_nuisance_column(self): # GH#21020 DataFrame.median should match Series.median df = pd.DataFrame({"A": pd.Categorical([1, 2, 2, 2, 3])}) ser = df["A"] # Double-check the Series behavior is to raise with pytest.raises(TypeError, match="does not support operation"): ser.median() with pytest.raises(TypeError, match="does not support operation"): df.median(numeric_only=False) with pytest.raises(TypeError, match="does not support operation"): df.median() # same thing, but with an additional non-categorical column df["B"] = df["A"].astype(int) with pytest.raises(TypeError, match="does not support operation"): df.median(numeric_only=False) with pytest.raises(TypeError, match="does not support operation"): df.median() # TODO: np.median(df, axis=0) gives np.array([2.0, 2.0]) instead # of expected.values @pytest.mark.parametrize("method", ["min", "max"]) def test_min_max_categorical_dtype_non_ordered_nuisance_column(self, method): # GH#28949 DataFrame.min should behave like Series.min cat = pd.Categorical(["a", "b", "c", "b"], ordered=False) ser = pd.Series(cat) df = ser.to_frame("A") # Double-check the Series behavior with pytest.raises(TypeError, match="is not ordered for operation"): getattr(ser, method)() with pytest.raises(TypeError, match="is not ordered for operation"): getattr(np, method)(ser) with pytest.raises(TypeError, match="is not ordered for operation"): getattr(df, method)(numeric_only=False) with pytest.raises(TypeError, match="is not ordered for operation"): getattr(df, method)() with pytest.raises(TypeError, match="is not ordered for operation"): getattr(np, method)(df, axis=0) # same thing, but with an additional non-categorical column df["B"] = df["A"].astype(object) with pytest.raises(TypeError, match="is not ordered for operation"): getattr(df, method)() with pytest.raises(TypeError, match="is not ordered for operation"): getattr(np, method)(df, axis=0) class TestEmptyDataFrameReductions: @pytest.mark.parametrize( "opname, dtype, exp_value, exp_dtype", [ ("sum", np.int8, 0, np.int64), ("prod", np.int8, 1, np.int_), ("sum", np.int64, 0, np.int64), ("prod", np.int64, 1, np.int64), ("sum", np.uint8, 0, np.uint64), ("prod", np.uint8, 1, np.uint), ("sum", np.uint64, 0, np.uint64), ("prod", np.uint64, 1, np.uint64), ("sum", np.float32, 0, np.float32), ("prod", np.float32, 1, np.float32), ("sum", np.float64, 0, np.float64), ], ) def test_df_empty_min_count_0(self, opname, dtype, exp_value, exp_dtype): df = pd.DataFrame({0: [], 1: []}, dtype=dtype) result = getattr(df, opname)(min_count=0) expected = pd.Series([exp_value, exp_value], dtype=exp_dtype, index=range(2)) tm.assert_series_equal(result, expected) @pytest.mark.parametrize( "opname, dtype, exp_dtype", [ ("sum", np.int8, np.float64), ("prod", np.int8, np.float64), ("sum", np.int64, np.float64), ("prod", np.int64, np.float64), ("sum", np.uint8, np.float64), ("prod", np.uint8, np.float64), ("sum", np.uint64, np.float64), ("prod", np.uint64, np.float64), ("sum", np.float32, np.float32), ("prod", np.float32, np.float32), ("sum", np.float64, np.float64), ], ) def test_df_empty_min_count_1(self, opname, dtype, exp_dtype): df = pd.DataFrame({0: [], 1: []}, dtype=dtype) result = getattr(df, opname)(min_count=1) expected = pd.Series([np.nan, np.nan], dtype=exp_dtype, index=pd.Index([0, 1])) tm.assert_series_equal(result, expected) @pytest.mark.parametrize( "opname, dtype, exp_value, exp_dtype", [ ("sum", "Int8", 0, ("Int32" if is_windows_np2_or_is32 else "Int64")), ("prod", "Int8", 1, ("Int32" if is_windows_np2_or_is32 else "Int64")), ("sum", "Int64", 0, "Int64"), ("prod", "Int64", 1, "Int64"), ("sum", "UInt8", 0, ("UInt32" if is_windows_np2_or_is32 else "UInt64")), ("prod", "UInt8", 1, ("UInt32" if is_windows_np2_or_is32 else "UInt64")), ("sum", "UInt64", 0, "UInt64"), ("prod", "UInt64", 1, "UInt64"), ("sum", "Float32", 0, "Float32"), ("prod", "Float32", 1, "Float32"), ("sum", "Float64", 0, "Float64"), ], ) def test_df_empty_nullable_min_count_0(self, opname, dtype, exp_value, exp_dtype): df = pd.DataFrame({0: [], 1: []}, dtype=dtype) result = getattr(df, opname)(min_count=0) expected = pd.Series( [exp_value, exp_value], dtype=exp_dtype, index=pd.Index([0, 1]) ) tm.assert_series_equal(result, expected) # TODO: why does min_count=1 impact the resulting Windows dtype # differently than min_count=0? @pytest.mark.parametrize( "opname, dtype, exp_dtype", [ ("sum", "Int8", ("Int32" if is_windows_or_is32 else "Int64")), ("prod", "Int8", ("Int32" if is_windows_or_is32 else "Int64")), ("sum", "Int64", "Int64"), ("prod", "Int64", "Int64"), ("sum", "UInt8", ("UInt32" if is_windows_or_is32 else "UInt64")), ("prod", "UInt8", ("UInt32" if is_windows_or_is32 else "UInt64")), ("sum", "UInt64", "UInt64"), ("prod", "UInt64", "UInt64"), ("sum", "Float32", "Float32"), ("prod", "Float32", "Float32"), ("sum", "Float64", "Float64"), ], ) def test_df_empty_nullable_min_count_1(self, opname, dtype, exp_dtype): df = pd.DataFrame({0: [], 1: []}, dtype=dtype) result = getattr(df, opname)(min_count=1) expected = pd.Series([pd.NA, pd.NA], dtype=exp_dtype, index=pd.Index([0, 1])) tm.assert_series_equal(result, expected) @pytest.mark.parametrize( "data", [ {"a": [0, 1, 2], "b": [pd.NaT, pd.NaT, pd.NaT]}, {"a": [0, 1, 2], "b": [pd.Timestamp("1990-01-01"), pd.NaT, pd.NaT]}, { "a": [0, 1, 2], "b": [ pd.Timestamp("1990-01-01"), pd.Timestamp("1991-01-01"), pd.Timestamp("1992-01-01"), ], }, { "a": [0, 1, 2], "b": [pd.Timedelta("1 days"), pd.Timedelta("2 days"), pd.NaT], }, { "a": [0, 1, 2], "b": [ pd.Timedelta("1 days"), pd.Timedelta("2 days"), pd.Timedelta("3 days"), ], }, ], ) def test_df_cov_pd_nat(self, data): # GH #53115 df = pd.DataFrame(data) with pytest.raises(TypeError, match="not supported for cov"): df.cov() def test_sum_timedelta64_skipna_false(): # GH#17235 arr = np.arange(8).astype(np.int64).view("m8[s]").reshape(4, 2) arr[-1, -1] = "Nat" df = pd.DataFrame(arr) assert (df.dtypes == arr.dtype).all() result = df.sum(skipna=False) expected = pd.Series([pd.Timedelta(seconds=12), pd.NaT], dtype="m8[s]") tm.assert_series_equal(result, expected) result = df.sum(axis=0, skipna=False) tm.assert_series_equal(result, expected) result = df.sum(axis=1, skipna=False) expected = pd.Series( [ pd.Timedelta(seconds=1), pd.Timedelta(seconds=5), pd.Timedelta(seconds=9), pd.NaT, ], dtype="m8[s]", ) tm.assert_series_equal(result, expected) def test_sum_empty_timedelta_column(): # GH#50628 summing an empty timedelta column used to raise # "Cannot cast TimedeltaArray to dtype float64" df = pd.DataFrame({"b": np.array([], dtype="m8[ns]")}) result = df.sum() expected = pd.Series([pd.Timedelta(0)], index=["b"], dtype="m8[ns]") tm.assert_series_equal(result, expected) def test_sum_empty_mixed_with_timedelta_column(): # GH#50628 sum over an empty mixed-dtype frame keeps per-column dtypes # rather than force-casting to float64 (which raised for timedelta) df = pd.DataFrame( { "a": np.array([], dtype="float64"), "b": np.array([], dtype="m8[ns]"), "c": np.array([], dtype="int64"), } ) result = df.sum() expected = pd.Series([0.0, pd.Timedelta(0), 0], index=["a", "b", "c"], dtype=object) tm.assert_series_equal(result, expected) @pytest.mark.parametrize("axis", [0, 1]) @pytest.mark.parametrize("skipna", [True, False]) def test_sum_timedelta64_overflow_raises(axis, skipna): # GH#43178: DataFrame.sum on an overflowing timedelta64 frame should raise # OutOfBoundsTimedelta, consistent with Series.sum, rather than silently # returning a saturated result. half = np.iinfo(np.int64).max // 2 arr = np.array([half, half, half], dtype="m8[ns]") df = pd.DataFrame({"a": arr, "b": arr, "c": arr}) msg = "overflow in timedelta operation" with pytest.raises(pd.errors.OutOfBoundsTimedelta, match=msg): df.sum(axis=axis, skipna=skipna) @pytest.mark.parametrize("axis", [0, 1]) def test_sum_timedelta64_all_nat_skipna_false(axis): # GH#43178: an all-NaT reduction with skipna=False must return NaT, not be # mistaken for an overflow (the NaT sentinels accumulate past int64 bounds) nat = np.array(["NaT", "NaT", "NaT"], dtype="m8[ns]") df = pd.DataFrame({"a": nat, "b": nat, "c": nat}) result = df.sum(axis=axis, skipna=False) assert result.dtype == "m8[ns]" assert result.isna().all() @pytest.mark.parametrize("axis", [0, 1]) def test_sum_timedelta64_exact(axis): # GH#66551: the sum used to accumulate in float64, whose 53-bit mantissa # silently rounded results above 2**53 arr = np.array([2**53 + 1, 0], dtype="m8[ns]") df = pd.DataFrame({"a": arr, "b": arr[::-1]}) result = df.sum(axis=axis) expected = pd.Series( np.array([2**53 + 1] * 2, dtype="m8[ns]"), index=df.axes[1 - axis] ) tm.assert_series_equal(result, expected) @pytest.mark.parametrize("axis", [0, 1]) def test_sum_timedelta64_overflow_only_in_exempt_slice(axis): # GH#66551: the check applies per reduced slice, and a slice that reduces to # NaT under skipna=False is exempt even though its entries overflow df = pd.DataFrame( { "a": [pd.Timedelta(1, "ns"), pd.Timedelta(2, "ns")], "b": [pd.Timedelta.max, pd.NaT], } ) if axis == 1: df = df.T result = df.sum(axis=axis, skipna=False) expected = pd.Series( [pd.Timedelta(3, "ns"), pd.NaT], index=df.axes[1 - axis], dtype="m8[ns]" ) tm.assert_series_equal(result, expected) def test_mixed_frame_with_integer_sum(): # https://github.com/pandas-dev/pandas/issues/34520 df = pd.DataFrame([["a", 1]], columns=list("ab")) df = df.astype({"b": "Int64"}) result = df.sum() expected = pd.Series(["a", 1], index=["a", "b"]) tm.assert_series_equal(result, expected) @pytest.mark.parametrize("numeric_only", [True, False]) @pytest.mark.parametrize("method", ["min", "max"]) def test_minmax_extensionarray(method, numeric_only): # https://github.com/pandas-dev/pandas/issues/32651 int64_info = np.iinfo("int64") ser = pd.Series([int64_info.max, None, int64_info.min], dtype=pd.Int64Dtype()) df = pd.DataFrame({"Int64": ser}) result = getattr(df, method)(numeric_only=numeric_only) expected = pd.Series( [getattr(int64_info, method)], dtype="Int64", index=pd.Index(["Int64"]), ) tm.assert_series_equal(result, expected) @pytest.mark.parametrize("dtype", [object, "str"]) def test_minmax_axis0_string_columns_with_na(dtype): # GH#18588: the string columns were silently dropped as nuisance columns, # and once numeric_only stopped dropping them the +/-inf NA fill raised df = pd.DataFrame( { "col1": [0, 1, 2, 3], "col2": pd.Series(["a", "b", None, "d"], dtype=dtype), "col3": pd.Series(["e", "f", None, "h"], dtype=dtype), } ) index = pd.Index(["col1", "col2", "col3"]) result = df.min(axis=0) tm.assert_series_equal(result, pd.Series([0, "a", "e"], index=index, dtype=object)) result = df.max(axis=0) tm.assert_series_equal(result, pd.Series([3, "d", "h"], index=index, dtype=object)) def test_minmax_axis0_object_date_with_na(): # GH#61204 df = pd.DataFrame( {"dates": [np.nan, np.nan, date(2025, 1, 3), date(2025, 1, 4)]}, dtype=object ) index = pd.Index(["dates"]) result = df.min(axis=0) tm.assert_series_equal( result, pd.Series([date(2025, 1, 3)], index=index, dtype=object) ) result = df.max(axis=0) tm.assert_series_equal( result, pd.Series([date(2025, 1, 4)], index=index, dtype=object) ) @pytest.mark.parametrize("axis", [0, 1]) def test_minmax_object_na_positions_differ_per_slice(axis): # GH#18588: the NA fill is taken per reduction slice, so slices whose NAs # sit in different positions do not borrow each other's values, and an # all-NA slice reduces to NA rather than poisoning its neighbours df = pd.DataFrame( { "a": ["x", None, "b"], "b": [None, "q", "p"], "c": [None, None, None], }, dtype=object, ) if axis == 0: exp_min = pd.Series( ["b", "p", None], index=pd.Index(["a", "b", "c"]), dtype=object ) exp_max = pd.Series( ["x", "q", None], index=pd.Index(["a", "b", "c"]), dtype=object ) else: exp_min = pd.Series(["x", "q", "b"], dtype=object) exp_max = pd.Series(["x", "q", "p"], dtype=object) tm.assert_series_equal(df.min(axis=axis), exp_min) tm.assert_series_equal(df.max(axis=axis), exp_max) @pytest.mark.parametrize("axis", [0, 1]) def test_minmax_object_skipna_false_propagates_per_slice(axis): # GH#4147: only the slices that actually hold an NA reduce to NA df = pd.DataFrame( { "a": ["x", None, "b"], "b": ["r", "q", "p"], }, dtype=object, ) if axis == 0: exp_min = pd.Series([None, "p"], index=pd.Index(["a", "b"]), dtype=object) exp_max = pd.Series([None, "r"], index=pd.Index(["a", "b"]), dtype=object) else: exp_min = pd.Series(["r", None, "b"], dtype=object) exp_max = pd.Series(["x", None, "p"], dtype=object) tm.assert_series_equal(df.min(axis=axis, skipna=False), exp_min) tm.assert_series_equal(df.max(axis=axis, skipna=False), exp_max) @pytest.mark.parametrize("axis", [0, 1]) def test_idxminmax_object_with_na(axis): # GH#4147: idxmin/idxmax raised TypeError on object data holding NAs even # though min/max on the same data works # the NA fill is taken per slice, so the data has to be one where a global # fill (the frame's first non-NA) would give a different answer df = pd.DataFrame( { "a": ["d", None, "a"], "b": ["z", "y", None], "c": ["a", "b", None], }, dtype=object, ) if axis == 0: exp_min = pd.Series([2, 1, 0], index=pd.Index(["a", "b", "c"])) exp_max = pd.Series([0, 0, 1], index=pd.Index(["a", "b", "c"])) else: exp_min = pd.Series(["c", "c", "a"]) exp_max = pd.Series(["b", "b", "a"]) tm.assert_series_equal(df.idxmin(axis=axis), exp_min) tm.assert_series_equal(df.idxmax(axis=axis), exp_max) @pytest.mark.parametrize( "dtype", [object, "str", pytest.param("string[pyarrow]", marks=td.skip_if_no("pyarrow"))], ) @pytest.mark.parametrize("method", ["min", "max"]) def test_string_reduction_skipna_false_axis1(dtype, method): # GH#18588: string columns reduce along axis=1 via the EA _groupby_op # fastpath, whose python fallback used to drop skipna and return a non-NA # answer. object is not an ExtensionDtype, so it transposes onto nanops # instead, where the NA used to raise TypeError. df = pd.DataFrame( { "a": pd.Series(["x", None], dtype=dtype), "b": pd.Series(["y", "z"], dtype=dtype), } ) result = getattr(df, method)(axis=1, skipna=False) expected = pd.Series([("x" if method == "min" else "y"), None], dtype=dtype) tm.assert_series_equal(result, expected) @pytest.mark.parametrize("axis", [0, 1]) def test_sum_prod_object_skipna_false(axis): # GH#4147: only the slices holding an NA are nulled out; the others still # reduce normally. Deliberately non-square, so that nulling along the # wrong axis is distinguishable. df = pd.DataFrame({"a": [1, 2, None], "b": [3, 4, 5]}, dtype=object) if axis == 0: exp_sum = pd.Series([None, 12], index=pd.Index(["a", "b"]), dtype=object) exp_prod = pd.Series([None, 60], index=pd.Index(["a", "b"]), dtype=object) else: exp_sum = pd.Series([4, 6, None], dtype=object) exp_prod = pd.Series([3, 8, None], dtype=object) tm.assert_series_equal(df.sum(axis=axis, skipna=False), exp_sum) tm.assert_series_equal(df.prod(axis=axis, skipna=False), exp_prod) @pytest.mark.parametrize( "method, exp_skipna, exp_no_na", [("min", 1, 1), ("max", 4, 4), ("sum", 8, 10), ("prod", 12, 24)], ) def test_object_reduction_axis_none(method, exp_skipna, exp_no_na): # GH#4147: axis=None reduces over the whole frame, so one NA anywhere # makes the skipna=False result NA df = pd.DataFrame({"a": [1, None], "b": [3, 4]}, dtype=object) assert pd.isna(getattr(df, method)(axis=None, skipna=False)) # ... while skipna=True reduces over the rest assert getattr(df, method)(axis=None) == exp_skipna # without an NA it still reduces normally df = pd.DataFrame({"a": [1, 2], "b": [3, 4]}, dtype=object) assert getattr(df, method)(axis=None, skipna=False) == exp_no_na @pytest.mark.parametrize("axis", [0, 1]) @pytest.mark.parametrize("method", ["sum", "prod"]) def test_sum_prod_object_skipna_false_min_count(axis, method): # GH#4147: a min_count larger than the slice still nulls out the slices # that hold no NA, i.e. skipna=False must not weaken it df = pd.DataFrame({"a": [1, None], "b": [3, 4]}, dtype=object) result = getattr(df, method)(axis=axis, skipna=False, min_count=5) # column/row "b" holds no NA, so only min_count nulls it out index = pd.Index(["a", "b"]) if axis == 0 else pd.Index([0, 1]) expected = pd.Series([None, None], index=index, dtype=object) tm.assert_series_equal(result, expected) @pytest.mark.parametrize("method", ["min", "max"]) def test_minmax_object_incomparable_skipna_false_axis1(method): # GH#4147: every row holds an NA, so every row is nulled out and the # mutually-incomparable values must never be compared df = pd.DataFrame({"a": [1, "p"], "b": ["a", "q"], "c": [None, None]}, dtype=object) result = getattr(df, method)(axis=1, skipna=False) tm.assert_series_equal(result, pd.Series([None, None], dtype=object)) @pytest.mark.parametrize("method", ["sum", "prod"]) def test_sum_prod_ea_skipna_false_axis1(method): # GH#18588: an extension dtype reduces along axis=1 through the same python # fallback, which used to drop skipna. object does not reach it (not an # ExtensionDtype), so it needs an actual EA to cover. df = pd.DataFrame( { "a": DecimalArray([Decimal(2), Decimal("NaN")]), "b": DecimalArray([Decimal(3), Decimal(4)]), } ) result = getattr(df, method)(axis=1, skipna=False) assert result.iloc[0] == (5 if method == "sum" else 6) assert pd.isna(result.iloc[1]) @pytest.mark.parametrize("method", ["min", "max"]) def test_interval_reduction_skipna_false_axis1(method): # GH#18588: IntervalDtype takes the same python fallback as the string dtypes df = pd.DataFrame( { "a": pd.arrays.IntervalArray.from_tuples([(0, 1), (4, 5)]), "b": pd.arrays.IntervalArray.from_tuples([(2, 3), None]), } ) result = getattr(df, method)(axis=1, skipna=False) expected = pd.Series( pd.arrays.IntervalArray.from_tuples( [(0, 1) if method == "min" else (2, 3), None] ) ) tm.assert_series_equal(result, expected) @pytest.mark.parametrize("ts_value", [pd.Timestamp("2000-01-01"), pd.NaT]) def test_frame_mixed_numeric_object_with_timestamp(ts_value): # GH 13912 df = pd.DataFrame({"a": [1], "b": [1.1], "c": ["foo"], "d": [ts_value]}) msg = "|".join(["does not support operation", "Cannot perform"]) with pytest.raises(TypeError, match=msg): df.sum() def test_prod_sum_min_count_mixed_object(): # https://github.com/pandas-dev/pandas/issues/41074 df = pd.DataFrame([1, "a", True]) result = df.prod(axis=0, min_count=1, numeric_only=False) expected = pd.Series(["a"], dtype=object) tm.assert_series_equal(result, expected) msg = re.escape("unsupported operand type(s) for +: 'int' and 'str'") with pytest.raises(TypeError, match=msg): df.sum(axis=0, min_count=1, numeric_only=False) @pytest.mark.parametrize("method", ["min", "max", "mean", "median", "skew", "kurt"]) @pytest.mark.parametrize("numeric_only", [True, False]) @pytest.mark.parametrize("dtype", ["float64", "Float64"]) def test_reduction_axis_none_returns_scalar(method, numeric_only, dtype): # GH#21597 As of 2.0, axis=None reduces over all axes. df = pd.DataFrame(np.random.default_rng(2).standard_normal((4, 4)), dtype=dtype) result = getattr(df, method)(axis=None, numeric_only=numeric_only) np_arr = df.to_numpy(dtype=np.float64) if method in {"skew", "kurt"}: comp_mod = pytest.importorskip("scipy.stats") if method == "kurt": method = "kurtosis" expected = getattr(comp_mod, method)(np_arr, bias=False, axis=None) tm.assert_almost_equal(result, expected) else: expected = getattr(np, method)(np_arr, axis=None) assert result == expected def test_reduction_axis_none_object_dtype_preserves_numpy_scalars(): # GH#64266 left = np.int8(1) df = pd.DataFrame({"a": [left], "b": [np.int8(5)]}, dtype=object) result = df.min(axis=None) assert result is left result = np.minimum.reduce(df, axis=None) assert result is left @pytest.mark.parametrize( "kernel", [ "corr", "corrwith", "cov", "idxmax", "idxmin", "kurt", "max", "mean", "median", "min", "prod", "quantile", "sem", "skew", "std", "sum", "var", ], ) def test_fails_on_non_numeric(kernel): # GH#46852 df = pd.DataFrame({"a": [1, 2, 3], "b": object}) args = (df,) if kernel == "corrwith" else () msg = "|".join( [ "not allowed for this dtype", "argument must be a string or a number", "not supported between instances of", "unsupported operand type", "argument must be a string or a real number", "Could not convert to numeric", ] ) with pytest.raises(TypeError, match=msg): getattr(df, kernel)(*args) @pytest.mark.parametrize( "method", [ "all", "any", "count", "idxmax", "idxmin", "kurt", "kurtosis", "max", "mean", "median", "min", "nunique", "prod", "product", "sem", "skew", "std", "sum", "var", ], ) @pytest.mark.parametrize("min_count", [0, 2]) def test_numeric_ea_axis_1( method, skipna, min_count, any_numeric_ea_dtype, using_nan_is_na ): # GH 54341 df = pd.DataFrame( { "a": pd.Series([0, 1, 2, 3], dtype=any_numeric_ea_dtype), "b": pd.Series([0, 1, pd.NA, 3], dtype=any_numeric_ea_dtype), }, ) expected_df = pd.DataFrame( { "a": [0.0, 1.0, 2.0, 3.0], "b": [0.0, 1.0, np.nan, 3.0], }, ) if method in ("count", "nunique"): expected_dtype = "int64" elif method in ("all", "any"): expected_dtype = "boolean" elif method in ( "kurt", "kurtosis", "mean", "median", "sem", "skew", "std", "var", ) and not any_numeric_ea_dtype.startswith("Float"): expected_dtype = "Float64" else: expected_dtype = any_numeric_ea_dtype kwargs = {} if method not in ("count", "nunique", "quantile"): kwargs["skipna"] = skipna if method in ("prod", "product", "sum"): kwargs["min_count"] = min_count if not skipna and method in ("idxmax", "idxmin"): with pytest.raises(ValueError, match="encountered an NA value"): getattr(df, method)(axis=1, **kwargs) with pytest.raises(ValueError, match="Encountered an NA value"): getattr(expected_df, method)(axis=1, **kwargs) return result = getattr(df, method)(axis=1, **kwargs) expected = getattr(expected_df, method)(axis=1, **kwargs) if method not in ("idxmax", "idxmin"): if using_nan_is_na: expected = expected.astype(expected_dtype) else: mask = np.isnan(expected) expected[mask] = 0 expected = expected.astype(expected_dtype) expected[mask] = pd.NA if not skipna and method == "all": # GH#65710 Kleene "all" on (truthy, NA) row returns NA expected.iloc[2] = pd.NA tm.assert_series_equal(result, expected) @pytest.mark.parametrize("how", ["idxmax", "idxmin"]) @pytest.mark.parametrize("skipna", [True, False]) @pytest.mark.parametrize( "dtype", ["Int64", "Float64", "UInt32", "boolean", "int64[pyarrow]"] ) def test_idxmax_idxmin_axis_1_ea_matches_numpy(how, skipna, dtype): # GH#56903 the EA-backed axis=1 path must agree with the numpy-backed path, # including skipna=False when there are no NA values (previously the cython # kernel returned the all-NA sentinel for every row). if "pyarrow" in dtype: pytest.importorskip("pyarrow") if dtype == "boolean": values = [[True, False, True], [False, False, True], [True, True, False]] np_dtype = bool else: values = [[1, 4, 0], [5, 2, 9], [2, 8, 1], [3, 3, 7]] np_dtype = "float64" df_ea = pd.DataFrame(values, columns=["a", "b", "c"]).astype(dtype) df_np = pd.DataFrame(values, columns=["a", "b", "c"]).astype(np_dtype) result = getattr(df_ea, how)(axis=1, skipna=skipna) expected = getattr(df_np, how)(axis=1, skipna=skipna) tm.assert_series_equal(result, expected) @pytest.mark.parametrize("how", ["idxmax", "idxmin"]) @pytest.mark.parametrize("skipna", [True, False]) @pytest.mark.parametrize("dtype", ["Int64", "Float64", "boolean", "int64[pyarrow]"]) def test_idxmax_idxmin_axis_1_ea_all_na_row_raises(how, skipna, dtype): # GH#56903 a fully-NA row must raise (matching the numpy-backed path) rather # than leak the -1 cython sentinel as a column label. if "pyarrow" in dtype: pytest.importorskip("pyarrow") if dtype == "boolean": values = [[True, False], [pd.NA, pd.NA], [False, True]] else: values = [[1, 4], [pd.NA, pd.NA], [3, 9]] df = pd.DataFrame(values, columns=["a", "b"]).astype(dtype) with pytest.raises(ValueError, match="[Ee]ncountered an?.*NA value"): getattr(df, how)(axis=1, skipna=skipna) @pytest.mark.parametrize("skipna", [True, False]) @pytest.mark.parametrize("dtype", ["boolean", "bool[pyarrow]"]) def test_logical_reductions_axis_1_ea_matches_numpy(dtype, skipna): # GH#56903 any/all axis=1 fastpath for boolean-like EA dtypes if "pyarrow" in dtype: pytest.importorskip("pyarrow") values = [[True, False, True], [False, False, False], [True, True, True]] df_ea = pd.DataFrame(values, columns=["a", "b", "c"]).astype(dtype) df_np = pd.DataFrame(values, columns=["a", "b", "c"]) for how in ["any", "all"]: result = getattr(df_ea, how)(axis=1, skipna=skipna) expected = getattr(df_np, how)(axis=1, skipna=skipna).astype(dtype) tm.assert_series_equal(result, expected) @pytest.mark.parametrize("dtype", ["str", "string"]) @pytest.mark.parametrize("how", ["min", "max"]) def test_str_min_max_axis_1(how, dtype): # GH#56903 no cython groupby op for object-backed str min/max; the # axis=1 EA fastpath must fall back to python aggregation df = pd.DataFrame({"a": ["x", "b"], "b": ["c", "d"]}, dtype=dtype) result = getattr(df, how)(axis=1) values = ["c", "b"] if how == "min" else ["x", "d"] expected = pd.Series(values, dtype=dtype) tm.assert_series_equal(result, expected) def test_third_party_ea_axis_1(): # GH#56903 EAs without a _groupby_op override must fall back to # python aggregation in the axis=1 EA fastpath df = pd.DataFrame( { "a": DecimalArray([Decimal("1"), Decimal("2")]), "b": DecimalArray([Decimal("3"), Decimal("4")]), } ) result = df.sum(axis=1) expected = pd.Series(DecimalArray([Decimal("4"), Decimal("6")])) tm.assert_series_equal(result, expected) def test_mean_nullable_int_axis_1(): # GH##36585 df = pd.DataFrame( {"a": [1, 2, 3, 4], "b": pd.Series([1, 2, 4, None], dtype=pd.Int64Dtype())} ) result = df.mean(axis=1, skipna=True) expected = pd.Series([1.0, 2.0, 3.5, 4.0], dtype="Float64") tm.assert_series_equal(result, expected) result = df.mean(axis=1, skipna=False) expected = pd.Series([1.0, 2.0, 3.5, pd.NA], dtype="Float64") tm.assert_series_equal(result, expected) def test_numeric_only_validates_bool(): # GH#53098 df = pd.DataFrame({"A": [1, 2, 3], "B": [1.0, 2.0, 3.0], "C": ["x", "y", "z"]}) df_num = pd.DataFrame({"A": [1, 2, 3], "B": [4, 5, 6]}) msg = "Passing non-boolean values for 'numeric_only' is deprecated" # _stat_function family: mean, min, max, median, skew, kurt for method in ["mean", "min", "max", "median", "skew", "kurt"]: with tm.assert_produces_warning(Pandas4Warning, match=msg): getattr(df, method)(numeric_only=1) with tm.assert_produces_warning(Pandas4Warning, match=msg): getattr(df, method)(numeric_only="yes") # None is falsy so _reduce includes all columns; use numeric-only df with tm.assert_produces_warning(Pandas4Warning, match=msg): getattr(df_num, method)(numeric_only=None) # _stat_function_ddof family: std, var, sem for method in ["std", "var", "sem"]: with tm.assert_produces_warning(Pandas4Warning, match=msg): getattr(df, method)(numeric_only=1) with tm.assert_produces_warning(Pandas4Warning, match=msg): getattr(df, method)(numeric_only="yes") with tm.assert_produces_warning(Pandas4Warning, match=msg): getattr(df_num, method)(numeric_only=None) # _min_count_stat_function family: sum, prod for method in ["sum", "prod"]: with tm.assert_produces_warning(Pandas4Warning, match=msg): getattr(df, method)(numeric_only=1) with tm.assert_produces_warning(Pandas4Warning, match=msg): getattr(df, method)(numeric_only="yes") with tm.assert_produces_warning(Pandas4Warning, match=msg): getattr(df_num, method)(numeric_only=None) # Valid boolean values must still work (regression guard) df_num.mean(numeric_only=True) df_num.mean(numeric_only=False) df_num.sum(numeric_only=True) df_num.std(numeric_only=True) @pytest.mark.parametrize("na_first", [True, False]) def test_median_skipna_false_keeps_complex(na_first): # GH#68487 the NaN propagated for a column holding an NA was real, so # whenever that column was reduced first the others were cast down to it # and lost their imaginary part cols = {"a": [1 + 2j, np.nan], "b": [1 + 2j, 3 + 4j]} if not na_first: cols = dict(reversed(cols.items())) df = pd.DataFrame(cols) expected = pd.Series( [complex(np.nan), 2 + 3j] if na_first else [2 + 3j, complex(np.nan)], index=list(cols), ) tm.assert_series_equal(df.median(skipna=False), expected) tm.assert_series_equal(df.T.median(axis=1, skipna=False), expected) @pytest.mark.parametrize( "method, expected", [ ("sum", float(2**63) + 0.5), ("prod", float(2**62) * float(2**62) * 0.5), ("mean", (float(2**63) + 0.5) / 3), ], ) def test_reduce_axis1_int_block_does_not_wrap(method, expected): # GH#68641: the int block used to be accumulated in int64, silently # wrapping, while the transpose path accumulates in float64 df = pd.DataFrame({"a": [2**62], "b": [2**62], "c": [0.5]}) assert len(df._mgr.blocks) > 1 result = getattr(df, method)(axis=1) tm.assert_series_equal(result, pd.Series([expected])) tm.assert_series_equal(result, getattr(df.T, method)(), check_names=False) def test_reduce_axis1_int_only_mean_widens_to_float64(): # GH#68641: with no float column the common dtype is int64, so mean has to # widen it the way nanmean does or the sum wraps and flips sign df = pd.DataFrame({"a": [2**62], "b": [2**62], "c": np.array([1], dtype="uint8")}) assert len(df._mgr.blocks) > 1 result = df.mean(axis=1) tm.assert_series_equal(result, pd.Series([(float(2**63) + 1) / 3])) tm.assert_series_equal(result, df.T.mean(), check_names=False) @pytest.mark.parametrize("method", ["sum", "prod", "mean"]) def test_reduce_axis1_uint64_block_does_not_wrap(method): # GH#68641 df = pd.DataFrame( { "a": np.array([2**63], dtype="uint64"), "b": np.array([2**63], dtype="uint64"), "c": [0.5], } ) assert len(df._mgr.blocks) > 1 result = getattr(df, method)(axis=1) tm.assert_series_equal(result, getattr(df.T, method)(), check_names=False) # without the fix the uint64 block wrapped to 0, leaving just the float assert result.iloc[0] > 2**62 def test_reduce_axis1_int_uint_combine_stays_exact(): # GH#68641: combining an int64 block result with a uint64 one promoted # the result to a lossy float64 df = pd.DataFrame( {"a": np.array([2**62 + 1], dtype="int64"), "b": np.array([1], dtype="uint8")} ) assert len(df._mgr.blocks) > 1 result = df.sum(axis=1) tm.assert_series_equal(result, pd.Series([2**62 + 2], dtype="int64")) @pytest.mark.parametrize("method", ["sum", "prod", "mean"]) def test_reduce_axis1_float32_not_upcast(method): # GH#68641: reducing in the common dtype keeps float32 out of float64 df = pd.DataFrame( { "a": np.array([1.5, 2.5], dtype="float32"), "b": np.array([1, 2], dtype="int8"), } ) assert len(df._mgr.blocks) > 1 result = getattr(df, method)(axis=1) assert result.dtype == "float32" tm.assert_series_equal(result, getattr(df.T, method)(), check_names=False) @pytest.mark.parametrize( "method, expected", [ ("sum", float(2**63)), ("prod", float(2**62) * float(2**62) * 0.5), ("mean", float(2**63) / 4), ], ) def test_reduce_axis1_bool_block_keeps_common_dtype(method, expected): # GH#68641: find_common_type would collapse to object here, which is why # the accumulation dtype comes from np.result_type instead df = pd.DataFrame({"a": [2**62], "b": [2**62], "c": [0.5], "d": [True]}) assert len(df._mgr.blocks) == 3 result = getattr(df, method)(axis=1) tm.assert_series_equal(result, pd.Series([expected])) def test_reduce_axis1_complex64_matches_transpose(): # GH#68641 nanmean accumulates in the input dtype and widens only for the # division, so complex64 must not be summed as complex128 here either df = pd.DataFrame( { "a": np.array([32767], dtype="int16"), "b": np.array([2**62], dtype="complex64"), } ) assert len(df._mgr.blocks) > 1 tm.assert_series_equal(df.mean(axis=1), df.T.mean()) @pytest.mark.parametrize("skipna", [True, False]) def test_reduce_axis1_min_count_complex64_matches_transpose(skipna): # GH#68641 _maybe_null_out widens complex64 before writing NaN; the axis=1 # path has to widen the same way, and only where nanops does -- under # skipna=False it builds no mask, so nothing counts as missing df = pd.DataFrame( { "a": np.array([1 + 2j, np.nan], dtype="complex64"), "b": np.array([1.0, 2.0], dtype="float32"), } ) assert len(df._mgr.blocks) > 1 result = df.sum(axis=1, min_count=2, skipna=skipna) assert result.dtype == ("complex128" if skipna else "complex64") tm.assert_series_equal(result, df.T.sum(min_count=2, skipna=skipna)) def test_reduce_axis1_mean_complex_keeps_imaginary_part(): # GH#68641: mean unconditionally cast the combined result to float64, # silently discarding the imaginary part df = pd.DataFrame( {"a": [True, False], "b": np.array([1 + 2j, 3 + 4j], dtype="complex64")} ) assert len(df._mgr.blocks) > 1 result = df.mean(axis=1) tm.assert_series_equal(result, pd.Series([1 + 1j, 1.5 + 2j], dtype="complex128")) def test_reduce_axis1_bool_block_keeps_numpy_int_semantics(): # GH#51474: a bool column no longer routes the reduction through object # dtype, so an integer result wraps like numpy instead of staying exact df = pd.DataFrame({"a": [2**62], "b": [2**62], "c": [True]}) assert len(df._mgr.blocks) > 1 result = df.sum(axis=1) expected = pd.Series([-(2**63) + 1]) assert expected.dtype == "int64" tm.assert_series_equal(result, expected) def test_reduce_axis1_float32_block_does_not_overflow(): # GH#68641: the float32 block was accumulated in float32, overflowing to # inf where the transpose path accumulated in the common float64 df = pd.DataFrame( { "a": np.array([3e38], dtype="float32"), "b": np.array([3e38], dtype="float32"), "c": [1.0], } ) assert len(df._mgr.blocks) > 1 result = df.sum(axis=1) tm.assert_series_equal(result, pd.Series([float(np.float32(3e38)) * 2 + 1.0])) tm.assert_series_equal(result, df.T.sum(), check_names=False)