import re
import sys

import numpy as np
import pytest

from pandas._libs.sparse import IntIndex
from pandas.compat import PYPY

import pandas as pd
import pandas._testing as tm
from pandas.core.arrays.sparse import SparseArray


@pytest.fixture
def arr_data():
    """Fixture returning numpy array with valid and missing entries"""
    return np.array([np.nan, np.nan, 1, 2, 3, np.nan, 4, 5, np.nan, 6])


@pytest.fixture
def arr(arr_data):
    """Fixture returning SparseArray from 'arr_data'"""
    return SparseArray(arr_data)


@pytest.fixture
def zarr():
    """Fixture returning SparseArray with integer entries and 'fill_value=0'"""
    return SparseArray([0, 0, 1, 2, 3, 0, 4, 5, 0, 6], fill_value=0)


class TestSparseArray:
    @pytest.mark.parametrize("fill_value", [0, None, np.nan])
    def test_shift_fill_value(self, fill_value):
        # GH #24128
        sparse = SparseArray(np.array([1, 0, 0, 3, 0]), fill_value=8.0)
        res = sparse.shift(1, fill_value=fill_value)
        if pd.isna(fill_value):
            fill_value = res.dtype.na_value
        exp = SparseArray(np.array([fill_value, 1, 0, 0, 3]), fill_value=8.0)
        tm.assert_sp_array_equal(res, exp)

    def test_set_fill_value(self):
        arr = SparseArray([1.0, np.nan, 2.0], fill_value=np.nan)
        arr.fill_value = 2
        assert arr.fill_value == 2

        arr = SparseArray([1, 0, 2], fill_value=0, dtype=np.int64)
        arr.fill_value = 2
        assert arr.fill_value == 2

        msg = "fill_value must be a valid value for the SparseDtype.subtype"
        with pytest.raises(ValueError, match=msg):
            # GH#53043
            arr.fill_value = 3.1
        assert arr.fill_value == 2

        arr.fill_value = np.nan
        assert np.isnan(arr.fill_value)

        arr = SparseArray([True, False, True], fill_value=False, dtype=np.bool_)
        arr.fill_value = True
        assert arr.fill_value is True

        with pytest.raises(ValueError, match=msg):
            arr.fill_value = 0
        assert arr.fill_value is True

        arr.fill_value = np.nan
        assert np.isnan(arr.fill_value)

    @pytest.mark.parametrize("val", [[1, 2, 3], np.array([1, 2]), (1, 2, 3)])
    def test_set_fill_invalid_non_scalar(self, val):
        arr = SparseArray([True, False, True], fill_value=False, dtype=np.bool_)
        msg = "fill_value must be a scalar"

        with pytest.raises(ValueError, match=msg):
            arr.fill_value = val

    def test_copy(self, arr):
        arr2 = arr.copy()
        assert arr2.sp_values is not arr.sp_values
        assert arr2.sp_index is arr.sp_index

    def test_values_asarray(self, arr_data, arr):
        tm.assert_almost_equal(arr.to_dense(), arr_data)

    @pytest.mark.parametrize(
        "data,shape,dtype",
        [
            ([0, 0, 0, 0, 0], (5,), None),
            ([], (0,), None),
            ([0], (1,), None),
            (["A", "A", np.nan, "B"], (4,), object),
        ],
    )
    def test_shape(self, data, shape, dtype):
        # GH 21126
        out = SparseArray(data, dtype=dtype)
        assert out.shape == shape

    @pytest.mark.parametrize(
        "vals",
        [
            [np.nan, np.nan, np.nan, np.nan, np.nan],
            [1, np.nan, np.nan, 3, np.nan],
            [1, np.nan, 0, 3, 0],
        ],
    )
    @pytest.mark.parametrize("fill_value", [None, 0])
    def test_dense_repr(self, vals, fill_value):
        vals = np.array(vals)
        arr = SparseArray(vals, fill_value=fill_value)

        res = arr.to_dense()
        tm.assert_numpy_array_equal(res, vals)

    @pytest.mark.parametrize("fix", ["arr", "zarr"])
    def test_pickle(self, fix, request, temp_file):
        obj = request.getfixturevalue(fix)
        unpickled = tm.round_trip_pickle(obj, temp_file)
        tm.assert_sp_array_equal(unpickled, obj)

    def test_generator_warnings(self):
        sp_arr = SparseArray([1, 2, 3])
        with tm.assert_produces_warning(None):
            for _ in sp_arr:
                pass

    def test_where_retain_fill_value(self):
        # GH#45691 don't lose fill_value on _where
        arr = SparseArray([np.nan, 1.0], fill_value=0)

        mask = np.array([True, False])

        res = arr._where(~mask, 1)
        exp = SparseArray([1, 1.0], fill_value=0)
        tm.assert_sp_array_equal(res, exp)

        ser = pd.Series(arr)
        res = ser.where(~mask, 1)
        tm.assert_series_equal(res, pd.Series(exp))

    def test_fillna(self):
        s = SparseArray([1, np.nan, np.nan, 3, np.nan])
        res = s.fillna(-1)
        exp = SparseArray([1, -1, -1, 3, -1], fill_value=-1, dtype=np.float64)
        tm.assert_sp_array_equal(res, exp)

        s = SparseArray([1, np.nan, np.nan, 3, np.nan], fill_value=0)
        res = s.fillna(-1)
        exp = SparseArray([1, -1, -1, 3, -1], fill_value=0, dtype=np.float64)
        tm.assert_sp_array_equal(res, exp)

        s = SparseArray([1, np.nan, 0, 3, 0])
        res = s.fillna(-1)
        exp = SparseArray([1, -1, 0, 3, 0], fill_value=-1, dtype=np.float64)
        tm.assert_sp_array_equal(res, exp)

        s = SparseArray([1, np.nan, 0, 3, 0], fill_value=0)
        res = s.fillna(-1)
        exp = SparseArray([1, -1, 0, 3, 0], fill_value=0, dtype=np.float64)
        tm.assert_sp_array_equal(res, exp)

        s = SparseArray([np.nan, np.nan, np.nan, np.nan])
        res = s.fillna(-1)
        exp = SparseArray([-1, -1, -1, -1], fill_value=-1, dtype=np.float64)
        tm.assert_sp_array_equal(res, exp)

        s = SparseArray([np.nan, np.nan, np.nan, np.nan], fill_value=0)
        res = s.fillna(-1)
        exp = SparseArray([-1, -1, -1, -1], fill_value=0, dtype=np.float64)
        tm.assert_sp_array_equal(res, exp)

        # float dtype's fill_value is np.nan, replaced by -1
        s = SparseArray([0.0, 0.0, 0.0, 0.0])
        res = s.fillna(-1)
        exp = SparseArray([0.0, 0.0, 0.0, 0.0], fill_value=-1)
        tm.assert_sp_array_equal(res, exp)

        # int dtype shouldn't have missing. No changes.
        s = SparseArray([0, 0, 0, 0])
        assert s.dtype == pd.SparseDtype(np.int64)
        assert s.fill_value == 0
        res = s.fillna(-1)
        tm.assert_sp_array_equal(res, s)

        s = SparseArray([0, 0, 0, 0], fill_value=0)
        assert s.dtype == pd.SparseDtype(np.int64)
        assert s.fill_value == 0
        res = s.fillna(-1)
        exp = SparseArray([0, 0, 0, 0], fill_value=0)
        tm.assert_sp_array_equal(res, exp)

        # fill_value can be nan if there is no missing hole.
        # only fill_value will be changed
        s = SparseArray([0, 0, 0, 0], fill_value=np.nan)
        assert s.dtype == pd.SparseDtype(np.int64, fill_value=np.nan)
        assert np.isnan(s.fill_value)
        res = s.fillna(-1)
        exp = SparseArray([0, 0, 0, 0], fill_value=-1)
        tm.assert_sp_array_equal(res, exp)

    def test_fillna_overlap(self):
        s = SparseArray([1, np.nan, np.nan, 3, np.nan])
        # filling with existing value doesn't replace existing value with
        # fill_value, i.e. existing 3 remains in sp_values
        res = s.fillna(3)
        exp = np.array([1, 3, 3, 3, 3], dtype=np.float64)
        tm.assert_numpy_array_equal(res.to_dense(), exp)

        s = SparseArray([1, np.nan, np.nan, 3, np.nan], fill_value=0)
        res = s.fillna(3)
        exp = SparseArray([1, 3, 3, 3, 3], fill_value=0, dtype=np.float64)
        tm.assert_sp_array_equal(res, exp)

    def test_nonzero(self):
        # Tests regression #21172.
        sa = SparseArray([float("nan"), float("nan"), 1, 0, 0, 2, 0, 0, 0, 3, 0, 0])
        expected = np.array([2, 5, 9], dtype=np.int32)
        (result,) = sa.nonzero()
        tm.assert_numpy_array_equal(expected, result)

        sa = SparseArray([0, 0, 1, 0, 0, 2, 0, 0, 0, 3, 0, 0])
        (result,) = sa.nonzero()
        tm.assert_numpy_array_equal(expected, result)


class TestSparseArrayAnalytics:
    @pytest.mark.parametrize(
        "data,expected",
        [
            (
                np.array([1, 2, 3, 4, 5], dtype=float),  # non-null data
                SparseArray(np.array([1.0, 3.0, 6.0, 10.0, 15.0])),
            ),
            (
                np.array([1, 2, np.nan, 4, 5], dtype=float),  # null data
                SparseArray(np.array([1.0, 3.0, np.nan, 7.0, 12.0])),
            ),
        ],
    )
    @pytest.mark.parametrize("numpy", [True, False])
    def test_cumsum(self, data, expected, numpy):
        cumsum = np.cumsum if numpy else lambda s: s.cumsum()

        out = cumsum(SparseArray(data))
        tm.assert_sp_array_equal(out, expected)

        out = cumsum(SparseArray(data, fill_value=np.nan))
        tm.assert_sp_array_equal(out, expected)

        out = cumsum(SparseArray(data, fill_value=2))
        tm.assert_sp_array_equal(out, expected)

        if numpy:  # numpy compatibility checks.
            msg = "the 'dtype' parameter is not supported"
            with pytest.raises(ValueError, match=msg):
                np.cumsum(SparseArray(data), dtype=np.int64)

            msg = "the 'out' parameter is not supported"
            with pytest.raises(ValueError, match=msg):
                np.cumsum(SparseArray(data), out=out)
        else:
            axis = 1  # SparseArray currently 1-D, so only axis = 0 is valid.
            msg = re.escape(f"axis(={axis}) out of bounds")
            with pytest.raises(ValueError, match=msg):
                SparseArray(data).cumsum(axis=axis)

    def test_ufunc(self):
        # GH 13853 make sure ufunc is applied to fill_value
        sparse = SparseArray([1, np.nan, 2, np.nan, -2])
        result = SparseArray([1, np.nan, 2, np.nan, 2])
        tm.assert_sp_array_equal(abs(sparse), result)
        tm.assert_sp_array_equal(np.abs(sparse), result)

        sparse = SparseArray([1, -1, 2, -2], fill_value=1)
        result = SparseArray._simple_new(
            np.array([1, 2, 2], dtype=np.int64),
            sparse.sp_index,
            pd.SparseDtype(np.int64, 1),
        )
        tm.assert_sp_array_equal(abs(sparse), result)
        tm.assert_sp_array_equal(np.abs(sparse), result)

        sparse = SparseArray([1, -1, 2, -2], fill_value=-1)
        exp = SparseArray([1, 1, 2, 2], fill_value=1)
        tm.assert_sp_array_equal(abs(sparse), exp)
        tm.assert_sp_array_equal(np.abs(sparse), exp)

        sparse = SparseArray([1, np.nan, 2, np.nan, -2])
        result = SparseArray(np.sin([1, np.nan, 2, np.nan, -2]))
        tm.assert_sp_array_equal(np.sin(sparse), result)

        sparse = SparseArray([1, -1, 2, -2], fill_value=1)
        result = SparseArray(np.sin([1, -1, 2, -2]), fill_value=np.sin(1))
        tm.assert_sp_array_equal(np.sin(sparse), result)

        sparse = SparseArray([1, -1, 0, -2], fill_value=0)
        result = SparseArray(np.sin([1, -1, 0, -2]), fill_value=np.sin(0))
        tm.assert_sp_array_equal(np.sin(sparse), result)

    def test_ufunc_args(self):
        # GH 13853 make sure ufunc is applied to fill_value, including its arg
        sparse = SparseArray([1, np.nan, 2, np.nan, -2])
        result = SparseArray([2, np.nan, 3, np.nan, -1])
        tm.assert_sp_array_equal(np.add(sparse, 1), result)

        sparse = SparseArray([1, -1, 2, -2], fill_value=1)
        result = SparseArray([2, 0, 3, -1], fill_value=2)
        tm.assert_sp_array_equal(np.add(sparse, 1), result)

        sparse = SparseArray([1, -1, 0, -2], fill_value=0)
        result = SparseArray([2, 0, 1, -1], fill_value=1)
        tm.assert_sp_array_equal(np.add(sparse, 1), result)

    @pytest.mark.parametrize("fill_value", [0.0, np.nan])
    def test_modf(self, fill_value):
        # https://github.com/pandas-dev/pandas/issues/26946
        sparse = SparseArray([fill_value] * 10 + [1.1, 2.2], fill_value=fill_value)
        r1, r2 = np.modf(sparse)
        e1, e2 = np.modf(np.asarray(sparse))
        tm.assert_sp_array_equal(r1, SparseArray(e1, fill_value=fill_value))
        tm.assert_sp_array_equal(r2, SparseArray(e2, fill_value=fill_value))

    def test_nbytes_integer(self):
        arr = SparseArray([1, 0, 0, 0, 2], kind="integer")
        result = arr.nbytes
        # (2 * 8) + 2 * 4
        assert result == 24

    def test_nbytes_block(self):
        arr = SparseArray([1, 2, 0, 0, 0], kind="block")
        result = arr.nbytes
        # (2 * 8) + 4 + 4
        # sp_values, blocs, blengths
        assert result == 24

    @pytest.mark.skipif(PYPY, reason="not relevant for PyPy")
    def test_memory_usage_object_subtype(self):
        # GH#68490 deep introspection raised TypeError on an object subtype
        values = np.array(["aaaa"] * 1000 + ["b"], dtype=object)
        arr = SparseArray(values, fill_value="aaaa")
        ser = pd.Series(arr)
        assert arr.memory_usage() == ser.memory_usage(index=False) == arr.nbytes

        # only the stored values are introspected, not the dense expansion
        expected = arr.nbytes + sum(sys.getsizeof(val) for val in arr.sp_values)
        assert arr.memory_usage(deep=True) == expected
        assert ser.memory_usage(index=False, deep=True) == expected
        assert pd.Index(arr).memory_usage(deep=True) == expected
        df = pd.DataFrame({"a": arr})
        assert df.memory_usage(index=False, deep=True).iloc[0] == expected
        assert sys.getsizeof(ser) > expected

    def test_memory_usage_pypy_compat(self, monkeypatch):
        # GH#46176 deep introspection uses sys.getsizeof, which always raises
        # TypeError on PyPy; deep=True should fall back to the shallow result
        arr = SparseArray(np.array(["a", "b", "c"], dtype=object), fill_value="a")

        monkeypatch.setattr("pandas.core.arrays.sparse.array.PYPY", True)
        assert arr.memory_usage(deep=True) == arr.memory_usage()

    def test_memory_usage_matches_nbytes(self):
        # non-object subtypes have nothing to introspect
        arr = SparseArray([1, 0, 0, 0, 2])
        assert arr.memory_usage() == arr.memory_usage(deep=True) == arr.nbytes

    def test_asarray_datetime64(self):
        s = SparseArray(pd.to_datetime(["2012", None, None, "2013"]))
        np.asarray(s)

    def test_density(self):
        arr = SparseArray([0, 1])
        assert arr.density == 0.5

    def test_density_empty(self):
        # GH#68468
        arr = SparseArray(np.array([], dtype="float64"))
        assert np.isnan(arr.density)

    def test_npoints(self):
        arr = SparseArray([0, 1])
        assert arr.npoints == 1


class TestIsna:
    # GH#41023
    @pytest.mark.parametrize("kind", ["integer", "block"])
    def test_isna_non_null_fill_no_nas(self, kind):
        # Common case: non-null fill value, no NAs in sp_values
        arr = SparseArray([0, 1, 0, 2, 0], fill_value=0, kind=kind)
        result = arr.isna()
        assert result.dtype == pd.SparseDtype(bool, False)
        expected_dense = np.array([False, False, False, False, False])
        tm.assert_numpy_array_equal(np.asarray(result), expected_dense)
        assert result.sp_values.size == 0

    @pytest.mark.parametrize("kind", ["integer", "block"])
    def test_isna_non_null_fill_with_nas(self, kind):
        # Non-null fill value, some NAs among sp_values
        arr = SparseArray([0, np.nan, 0, 2, np.nan], fill_value=0, kind=kind)
        result = arr.isna()
        expected_dense = np.array([False, True, False, False, True])
        tm.assert_numpy_array_equal(np.asarray(result), expected_dense)

    @pytest.mark.parametrize("kind", ["integer", "block"])
    def test_isna_non_null_fill_all_sp_values_na(self, kind):
        # Non-null fill value, all sp_values are NA
        arr = SparseArray([0, np.nan, 0, np.nan], fill_value=0, kind=kind)
        result = arr.isna()
        expected_dense = np.array([False, True, False, True])
        tm.assert_numpy_array_equal(np.asarray(result), expected_dense)

    @pytest.mark.parametrize("kind", ["integer", "block"])
    def test_isna_null_fill_value(self, kind):
        # Null fill value
        arr = SparseArray([np.nan, 1, np.nan, 2], kind=kind)
        result = arr.isna()
        expected_dense = np.array([True, False, True, False])
        tm.assert_numpy_array_equal(np.asarray(result), expected_dense)

    def test_isna_empty_sparse(self):
        # All fill values, no sp_values at all
        arr = SparseArray([0, 0, 0], fill_value=0)
        result = arr.isna()
        expected_dense = np.array([False, False, False])
        tm.assert_numpy_array_equal(np.asarray(result), expected_dense)


def test_setting_fill_value_fillna_still_works():
    # This is why letting users update fill_value / dtype is bad
    # astype has the same problem.
    arr = SparseArray([1.0, np.nan, 1.0], fill_value=0.0)
    arr.fill_value = np.nan
    result = arr.isna()
    # Can't do direct comparison, since the sp_index will be different
    # So let's convert to ndarray and check there.
    result = np.asarray(result)

    expected = np.array([False, True, False])
    tm.assert_numpy_array_equal(result, expected)


def test_cumsum_integer_no_recursion():
    # GH 62669: RecursionError in integer SparseArray.cumsum
    arr = SparseArray([1, 2, 3])
    result = arr.cumsum()
    expected = SparseArray([1, 3, 6], fill_value=np.nan)
    tm.assert_sp_array_equal(result, expected)

    # Also test with some zeros interleaved
    arr2 = SparseArray([0, 1, 0, 2])
    result2 = arr2.cumsum()
    expected2 = SparseArray([0, 1, 1, 3], fill_value=np.nan)
    tm.assert_sp_array_equal(result2, expected2)


def test_cumsum_float_fill_value_zero():
    # GH 62669
    arr = pd.arrays.SparseArray([1.0, 0.0, np.nan, 3.0], fill_value=0.0)
    result = arr.cumsum()
    expected = SparseArray([1.0, 1.0, None, 4.0], fill_value=np.nan)
    tm.assert_sp_array_equal(result, expected)


@pytest.mark.parametrize(
    "subtype, values, expected_values, na_value",
    [
        ("float64", [1.0, np.nan, 2.0, 5.0], [1.0, np.nan, 3.0, 8.0], np.nan),
        (
            "complex128",
            [1 + 1j, np.nan, 2 + 0j, 5 + 0j],
            [1 + 1j, np.nan, 3 + 1j, 8 + 1j],
            np.nan,
        ),
        ("m8[ns]", [1, "NaT", 2, 5], [1, "NaT", 3, 8], np.timedelta64("NaT", "ns")),
    ],
)
def test_cumsum_na_stored_in_sp_values(subtype, values, expected_values, na_value):
    # GH#68972 an NA stored in sp_values used to propagate into every later entry
    sp_values = np.array(values, dtype=subtype)
    indices = np.arange(len(sp_values), dtype=np.int32)
    arr = SparseArray.from_indices(
        sp_values, indices=indices, length=len(sp_values), fill_value=na_value
    )

    result = arr.cumsum()
    expected = SparseArray.from_indices(
        np.array(expected_values, dtype=subtype),
        indices=indices,
        length=len(sp_values),
        fill_value=na_value,
    )
    tm.assert_sp_array_equal(result, expected)


def test_cumsum_na_stored_complex_imaginary_nan():
    # GH#68972 the restored value is the dtype's own NA, not the stored object, so
    #  an imaginary-only NaN loses its real part exactly as it does when dense.
    #  tm.assert_* reads any two NAs as equal, so this compares the components
    values = np.array([1 + 1j, complex(1, np.nan), 2 + 0j])
    arr = SparseArray.from_indices(
        values,
        indices=np.arange(3, dtype=np.int32),
        length=3,
        fill_value=np.nan,
    )

    result = pd.Series(arr).cumsum().to_numpy()
    expected = pd.Series(values).cumsum().to_numpy()
    tm.assert_numpy_array_equal(result.real, expected.real)
    tm.assert_numpy_array_equal(result.imag, expected.imag)


def test_cumsum_na_stored_and_gap():
    # GH#68972 a stored NA and a gap NA in one array is the interaction the fix is
    #  about; both have to reach the result
    arr = SparseArray.from_indices(
        np.array([1.0, np.nan, 5.0]),
        indices=np.array([0, 1, 3], dtype=np.int32),
        length=5,
        fill_value=np.nan,
    )

    result = pd.Series(arr).cumsum()
    expected = pd.Series([1.0, np.nan, np.nan, 6.0, np.nan])
    tm.assert_series_equal(result.sparse.to_dense(), expected)


@pytest.mark.parametrize("op_name", ["cumsum", "cumprod", "cummin", "cummax"])
@pytest.mark.parametrize("skipna", [True, False])
@pytest.mark.parametrize("fill_value", [np.nan, 0.0])
def test_accumulate_matches_dense(op_name, skipna, fill_value):
    # GH#68187 these raised NotImplementedError from Series/DataFrame
    values = np.array([1.0, 2.0, np.nan, 3.0])
    ser = pd.Series(SparseArray(values, fill_value=fill_value))

    result = getattr(ser, op_name)(skipna=skipna)
    expected = getattr(pd.Series(values), op_name)(skipna=skipna)

    assert result.dtype == pd.SparseDtype("float64", np.nan)
    tm.assert_series_equal(result.sparse.to_dense(), expected)

    frame_result = getattr(pd.DataFrame({"a": ser}), op_name)(skipna=skipna)
    tm.assert_series_equal(
        frame_result["a"].sparse.to_dense(), expected, check_names=False
    )


def test_accumulate_integer_subtype():
    # GH#68187 the fill value widens to NaN, as it does for cumsum
    arr = SparseArray([1, 0, 2], fill_value=0)
    result = pd.Series(arr).cumprod()
    expected = pd.Series(SparseArray([1, 0, 0], fill_value=np.nan))
    tm.assert_series_equal(result, expected)


@pytest.mark.parametrize("op_name", ["cumsum", "cumprod", "cummin", "cummax"])
@pytest.mark.parametrize("skipna", [True, False])
def test_accumulate_na_gaps_subtype_cannot_hold_na(op_name, skipna):
    # GH#68187 the gaps of a Sparse[int64, nan] are NA, not 0. cumsum is affected
    #  too: _accumulate's fast path for it is gated on skipna.
    values = np.array([1.0, np.nan, 5.0, np.nan])
    arr = SparseArray(values).astype(pd.SparseDtype("int64", np.nan))

    result = getattr(pd.Series(arr), op_name)(skipna=skipna)
    expected = getattr(pd.Series(values), op_name)(skipna=skipna)

    # only that fast path keeps the subtype; the rest promote to hold the gaps
    subtype = "int64" if (op_name == "cumsum" and skipna) else "float64"
    assert result.dtype == pd.SparseDtype(subtype, np.nan)
    # not sparse.to_dense(), which has the same cast-the-gaps bug this fixes
    tm.assert_series_equal(result.astype("float64"), expected)


@pytest.mark.parametrize("op_name", ["cumprod", "cummin", "cummax"])
def test_accumulate_bool_subtype_promotes_to_object(op_name):
    # GH#68187 a Sparse[bool, nan] has no wider numpy bool dtype to hold the gaps.
    #  cumsum is excluded: its fast path returns Sparse[int64, nan] instead.
    values = np.array([True, np.nan, False, np.nan], dtype=object)
    arr = SparseArray([1.0, np.nan, 0.0, np.nan]).astype(pd.SparseDtype("bool", np.nan))

    result = getattr(pd.Series(arr), op_name)()
    expected = getattr(pd.Series(values), op_name)()

    assert result.dtype == pd.SparseDtype(object, np.nan)
    tm.assert_series_equal(result.sparse.to_dense(), expected)


def test_accumulate_datetime64():
    # GH#68187 a datetimelike subtype accumulates through its own array
    dti = pd.to_datetime(["2020-01-02", "NaT", "2020-01-01"])
    ser = pd.Series(SparseArray(dti))

    result = ser.cummax()
    tm.assert_series_equal(result.sparse.to_dense(), pd.Series(dti).cummax())

    msg = "Accumulation cumsum not supported"
    with pytest.raises(TypeError, match=msg):
        ser.cumsum()


@pytest.mark.parametrize(
    "ufunc, data, fill_value",
    [
        (np.add, [1.0, 0.0, 2.0], 0.0),
        # an NA fill value propagates, where Series.cumsum would skip it
        (np.add, [1.0, np.nan, 2.0], np.nan),
        (np.bitwise_or, [1, 4, 0, 0], 4),
        # subtypes np.asarray would widen or objectify; see SparseArray._densify
        (np.bitwise_or, np.array([1, 4, 0, 0], dtype="uint64"), 4),
        (np.maximum, np.array([1000, 2500, 4000], "m8[ns]"), pd.Timedelta("2500ns")),
    ],
)
def test_ufunc_accumulate_includes_fill_value(ufunc, data, fill_value):
    # GH#68566 this accumulated the stored values and the scalar fill_value
    #  separately, and numpy cannot accumulate a scalar
    arr = SparseArray(data, fill_value=fill_value)
    assert arr.sp_index.ngaps

    result = ufunc.accumulate(arr)

    expected = ufunc.accumulate(np.asarray(data, dtype=arr.dtype.subtype))
    tm.assert_numpy_array_equal(result.to_dense(), expected)
    # like cumsum, the fill value is NA regardless of the original one
    assert pd.isna(result.fill_value)


def test_ufunc_accumulate_promotes_na_gaps():
    # GH#68566 to_dense would cast the gaps of an NA-filled int64 subtype to 0,
    #  accumulating to [1, 1, 5, 5]; see SparseArray._densify
    arr = SparseArray([1.0, np.nan, 5.0, np.nan]).astype(
        pd.SparseDtype("int64", np.nan)
    )

    result = np.maximum.accumulate(arr)

    expected = SparseArray([1.0, np.nan, np.nan, np.nan], fill_value=np.nan)
    tm.assert_sp_array_equal(result, expected)


def test_setting_fill_value_updates():
    arr = SparseArray([0.0, np.nan], fill_value=0)
    arr.fill_value = np.nan
    # use private constructor to get the index right
    # otherwise both nans would be un-stored.
    expected = SparseArray._simple_new(
        sparse_array=np.array([np.nan]),
        sparse_index=IntIndex(2, [1]),
        dtype=pd.SparseDtype(float, np.nan),
    )
    tm.assert_sp_array_equal(arr, expected)


@pytest.mark.parametrize(
    "arr,fill_value,loc",
    [
        ([None, 1, 2], None, 0),
        ([0, None, 2], None, 1),
        ([0, 1, None], None, 2),
        ([0, 1, 1, None, None], None, 3),
        ([1, 1, 1, 2], None, -1),
        ([], None, -1),
        ([None, 1, 0, 0, None, 2], None, 0),
        ([None, 1, 0, 0, None, 2], 1, 1),
        ([None, 1, 0, 0, None, 2], 2, 5),
        ([None, 1, 0, 0, None, 2], 3, -1),
        ([None, 0, 0, 1, 2, 1], 0, 1),
        ([None, 0, 0, 1, 2, 1], 1, 3),
    ],
)
def test_first_fill_value_loc(arr, fill_value, loc):
    result = SparseArray(arr, fill_value=fill_value)._first_fill_value_loc()
    assert result == loc


@pytest.mark.parametrize(
    "arr",
    [
        [1, 2, np.nan, np.nan],
        [1, np.nan, 2, np.nan],
        [1, 2, np.nan],
        [np.nan, 1, 0, 0, np.nan, 2],
        [np.nan, 0, 0, 1, 2, 1],
    ],
)
@pytest.mark.parametrize("fill_value", [np.nan, 0, 1])
def test_unique_na_fill(arr, fill_value):
    a = SparseArray(arr, fill_value=fill_value).unique()
    b = pd.Series(arr).unique()
    assert isinstance(a, SparseArray)
    a = np.asarray(a)
    tm.assert_numpy_array_equal(a, b)


def test_unique_all_sparse():
    # https://github.com/pandas-dev/pandas/issues/23168
    arr = SparseArray([0, 0])
    result = arr.unique()
    expected = SparseArray([0])
    tm.assert_sp_array_equal(result, expected)


def test_map():
    arr = SparseArray([0, 1, 2])
    expected = SparseArray([10, 11, 12], fill_value=10)

    # dict
    result = arr.map({0: 10, 1: 11, 2: 12})
    tm.assert_sp_array_equal(result, expected)

    # series
    result = arr.map(pd.Series({0: 10, 1: 11, 2: 12}))
    tm.assert_sp_array_equal(result, expected)

    # function
    result = arr.map(pd.Series({0: 10, 1: 11, 2: 12}))
    expected = SparseArray([10, 11, 12], fill_value=10)
    tm.assert_sp_array_equal(result, expected)


def test_map_missing():
    arr = SparseArray([0, 1, 2])
    expected = SparseArray([10, 11, None], fill_value=10)

    result = arr.map({0: 10, 1: 11})
    tm.assert_sp_array_equal(result, expected)


def test_map_list_valued_mapper():
    # GH#68586 the fill-value collision check must not take the truth of a
    #  mapper result that is not a scalar
    arr = SparseArray(np.array([1, 2, 0, 0]), fill_value=0)

    result = arr.map({0: 9, 1: [1, 2], 2: 5})
    assert list(result) == [[1, 2], 5, 9, 9]


def test_map_object_subtype_with_na():
    # GH#68586 the fill-value collision check took the truth of an object
    #  comparison, which a stored pd.NA makes ambiguous
    values = np.array([1, pd.NA, 2], dtype=object)
    arr = SparseArray(values, fill_value=0)

    result = arr.map(lambda x: x)
    tm.assert_numpy_array_equal(result.to_dense(), values)


@pytest.mark.parametrize("fill_value", [np.nan, 1])
def test_dropna(fill_value):
    # GH-28287
    arr = SparseArray([np.nan, 1], fill_value=fill_value)
    exp = SparseArray([1.0], fill_value=fill_value)
    tm.assert_sp_array_equal(arr.dropna(), exp)

    df = pd.DataFrame({"a": [0, 1], "b": arr})
    expected_df = pd.DataFrame(
        {"a": [1], "b": exp}, index=pd.RangeIndex(start=1, stop=2, step=1)
    )
    tm.assert_equal(df.dropna(), expected_df)


def test_drop_duplicates_fill_value():
    # GH 11726
    df = pd.DataFrame(np.zeros((5, 5))).apply(lambda x: SparseArray(x, fill_value=0))
    result = df.drop_duplicates()
    expected = pd.DataFrame({i: SparseArray([0.0], fill_value=0) for i in range(5)})
    tm.assert_frame_equal(result, expected)


def test_zero_sparse_column():
    # GH 27781
    df1 = pd.DataFrame({"A": SparseArray([0, 0, 0]), "B": [1, 2, 3]})
    df2 = pd.DataFrame({"A": SparseArray([0, 1, 0]), "B": [1, 2, 3]})
    result = df1.loc[df1["B"] != 2]
    expected = df2.loc[df2["B"] != 2]
    tm.assert_frame_equal(result, expected)

    expected = pd.DataFrame({"A": SparseArray([0, 0]), "B": [1, 3]}, index=[0, 2])
    tm.assert_frame_equal(result, expected)


def test_series_repr_datetime64():
    # GH#68073 printed the raw i8 value
    arr = SparseArray(np.array(["2020-01-01", "NaT"], dtype="M8[ns]"))
    expected = "0    2020-01-01 00:00:00\n1                    NaT"
    assert pd.Series(arr).to_string() == expected


@pytest.mark.parametrize("kind", ["M8", "m8"])
@pytest.mark.parametrize("unit", ["s", "ms", "us", "ns"])
def test_array_object_datetimelike(kind, unit):
    # GH#68073 numpy renders datetime64/timedelta64 as ints when casting to
    # object, so we have to box these ourselves
    values = np.array([1, 2, 3], dtype="i8").astype(f"{kind}[{unit}]")
    values[1] = "NaT"
    expected = pd.array(values).astype(object)

    arr = SparseArray(values)
    tm.assert_numpy_array_equal(np.asarray(arr, dtype=object), expected)

    # no gaps, so __array__ takes the sp_values shortcut
    no_gaps = SparseArray(values[::2])
    tm.assert_numpy_array_equal(np.asarray(no_gaps, dtype=object), expected[::2])


@pytest.mark.parametrize("kind", ["M8", "m8"])
@pytest.mark.parametrize("unit", ["s", "ms", "us", "ns"])
@pytest.mark.parametrize("na_fill", [True, False])
def test_to_dense_datetimelike_non_numpy_fill(kind, unit, na_fill):
    # GH#68590 a fill value numpy cannot promote against the subtype made the
    #  dense result object dtype, with the stored values rendered as ints or
    #  datetime.datetime beside the fill; to_dense truncated a ns fill to us
    values = np.array([1, 2, 3], dtype="i8").astype(f"{kind}[{unit}]")
    if na_fill:
        values[1] = "NaT"
        fill_value = np.nan
    else:
        fill_value = (pd.Timestamp if kind == "M8" else pd.Timedelta)(values[1])

    arr = SparseArray(values, fill_value=fill_value)
    assert arr.sp_index.ngaps == 1

    tm.assert_numpy_array_equal(arr.to_dense(), values)
    expected = pd.array(values).astype(object)
    tm.assert_numpy_array_equal(np.asarray(arr, dtype=object), expected)
    tm.assert_numpy_array_equal(arr.astype(object), expected)

    # an explicit numeric dtype casts the gap the way numpy casts the dense
    #  values, so a missing fill value gives iNaT rather than 0
    tm.assert_numpy_array_equal(
        np.asarray(arr, dtype="i8"), np.asarray(values, dtype="i8")
    )


@pytest.mark.parametrize("kind", ["M8", "m8"])
@pytest.mark.parametrize("unit", ["s", "ms", "us"])
@pytest.mark.parametrize("boxed", [True, False])
def test_to_dense_datetimelike_fill_other_unit(kind, unit, boxed):
    # GH#68590 a fill value whose unit differs from the subtype -- the us a
    #  string-parsed Timestamp carries -- densified in the fill's own unit, so
    #  a numeric cast read it as a raw integer
    values = np.array([1, 2, 3], dtype="i8").astype(f"{kind}[{unit}]")
    box = pd.Timestamp if kind == "M8" else pd.Timedelta
    fill_value = box(values[1]).as_unit("ns")

    arr = SparseArray(values, fill_value=fill_value if boxed else fill_value.asm8)
    assert arr.sp_index.ngaps == 1

    tm.assert_numpy_array_equal(arr.to_dense(), values)
    tm.assert_numpy_array_equal(
        np.asarray(arr, dtype="i8"), np.asarray(values, dtype="i8")
    )


@pytest.mark.parametrize("fill", [pd.NaT, np.nan, np.float64("nan"), pd.NA])
@pytest.mark.parametrize("kind", ["M8", "m8"])
@pytest.mark.parametrize("unit", ["s", "ms", "us", "ns"])
def test_datetimelike_na_fill_value_normalized(fill, kind, unit):
    # GH#68449, GH#68558 any NA fill value is stored as the subtype's own NaT,
    #  so it behaves identically to the np.datetime64("NaT") spelling
    values = np.array([1, 2, 3], dtype="i8").astype(f"{kind}[{unit}]")
    values[1] = "NaT"

    arr = SparseArray(values, fill_value=fill)
    expected = SparseArray(values, fill_value=values[1])

    assert arr.dtype == expected.dtype
    assert arr.fill_value.dtype == values.dtype
    tm.assert_sp_array_equal(arr, expected)

    # each of these was wrong for at least one of the fill spellings above,
    #  see GH#68449 and GH#68558
    tm.assert_numpy_array_equal(arr.to_dense(), values)
    tm.assert_numpy_array_equal(np.asarray(arr), values)
    tm.assert_numpy_array_equal(
        np.asarray(arr, dtype=object), pd.array(values).astype(object)
    )
    tm.assert_sp_array_equal(arr - arr, expected - expected)
    tm.assert_sp_array_equal(arr == arr, expected == expected)
    tm.assert_sp_array_equal(arr.unique(), expected.unique())
    all_fill = np.full(2, "NaT", dtype=values.dtype)
    tm.assert_sp_array_equal(
        SparseArray(all_fill, fill_value=fill).take([0, 1], allow_fill=True),
        SparseArray(all_fill, fill_value=values[1]).take([0, 1], allow_fill=True),
    )
    tm.assert_sp_array_equal(
        arr.astype("Sparse[int64]"), expected.astype("Sparse[int64]")
    )


@pytest.mark.parametrize("kind, box", [("M8", pd.Timestamp), ("m8", pd.Timedelta)])
@pytest.mark.parametrize("unit", ["s", "ms", "us", "ns"])
def test_datetimelike_boxed_fill_value_normalized(kind, box, unit):
    # GH#68589 a boxed fill value is stored as the subtype's own scalar, so it
    #  behaves identically to the np.datetime64/np.timedelta64 spelling
    values = np.array([1, 2, 2], dtype="i8").astype(f"{kind}[{unit}]")

    arr = SparseArray(values, fill_value=box(values[0]))
    expected = SparseArray(values, fill_value=values[0])

    assert arr.dtype == expected.dtype
    assert not isinstance(arr.fill_value, box)
    tm.assert_sp_array_equal(arr, expected)

    # the boxed spelling used a truncated fill value here
    tm.assert_numpy_array_equal(arr.to_dense(), values)
    tm.assert_series_equal(arr.value_counts(), expected.value_counts())
    tm.assert_sp_array_equal(arr.unique(), expected.unique())
    tm.assert_sp_array_equal(arr - values[0], expected - values[0])

    # and here it came back as object dtype holding raw ints or datetimes
    tm.assert_numpy_array_equal(np.asarray(arr), values)


def test_tz_aware_fill_value_still_rejected():
    # GH#68589 normalizing before _check_fill_value would turn this into a naive
    #  datetime64 that passes validation, silently dropping the timezone
    msg = "fill_value must be a valid value for the SparseDtype.subtype"
    with pytest.raises(ValueError, match=msg):
        pd.SparseDtype("M8[ns]", fill_value=pd.Timestamp("2020-01-01", tz="UTC"))


@pytest.mark.parametrize(
    "fill_value",
    [
        pd.Timestamp("2016-01-01 00:00:00.000000001"),
        pd.Timestamp("2016-01-01", tz="US/Pacific"),
        pd.Timedelta(1, "ns"),
    ],
)
def test_object_subtype_boxed_fill_value(fill_value):
    # GH#68571 an object subtype holds a boxed scalar as-is; unboxing it would
    #  lose the nanosecond or the tz
    expected = np.array([fill_value, 1, "a"], dtype=object)
    arr = SparseArray(expected, fill_value=fill_value)
    assert arr.sp_index.ngaps == 1

    tm.assert_numpy_array_equal(np.asarray(arr), expected)
    tm.assert_numpy_array_equal(arr.to_dense(), expected)


def test_array_interface(arr_data, arr):
    # https://github.com/pandas-dev/pandas/pull/60046
    result = np.asarray(arr)
    tm.assert_numpy_array_equal(result, arr_data)

    # it always gives a copy by default
    result_copy1 = np.asarray(arr)
    result_copy2 = np.asarray(arr)
    assert not np.may_share_memory(result_copy1, result_copy2)

    # or with explicit copy=True
    result_copy1 = np.array(arr, copy=True)
    result_copy2 = np.array(arr, copy=True)
    assert not np.may_share_memory(result_copy1, result_copy2)

    # for sparse arrays, copy=False is never allowed
    with pytest.raises(ValueError, match="Unable to avoid copy while creating"):
        np.array(arr, copy=False)

    # except when there are actually no sparse filled values
    arr2 = SparseArray(np.array([1, 2, 3]))
    result_nocopy1 = np.array(arr2, copy=False)
    result_nocopy2 = np.array(arr2, copy=False)
    assert np.may_share_memory(result_nocopy1, result_nocopy2)


def test_shift_bool_subtype():
    # GH#68580 np.result_type promoted the NA fill to float64, which the array's
    #  own False fill_value is not valid for
    arr = SparseArray(np.array([True, False]))
    result = arr.shift(1)
    expected = SparseArray(np.array([np.nan, True], dtype=object), fill_value=False)
    tm.assert_sp_array_equal(result, expected)

    # an explicit bool fill needs no promotion
    tm.assert_sp_array_equal(
        arr.shift(1, fill_value=True), SparseArray(np.array([True, True]))
    )


def test_shift_fill_value_promotion():
    # GH#68580 np.result_type is not value-aware, so an int64 subtype both cast a
    #  bool fill to 1 and widened for a float fill it could hold exactly
    arr = SparseArray(np.array([1, 2, 3]))

    result = arr.shift(1, fill_value=True)
    assert result.dtype == pd.SparseDtype(object, 0)
    assert result[0] is True

    result = arr.shift(1, fill_value=2.0)
    tm.assert_sp_array_equal(result, SparseArray(np.array([2, 1, 2])))


@pytest.mark.parametrize("kind", ["M8", "m8"])
@pytest.mark.parametrize("unit", ["s", "ms", "us", "ns"])
def test_shift_datetimelike_subtype(kind, unit):
    # GH#68580 np.result_type cannot promote a datetimelike dtype against the
    #  NA fill value at all, so this raised
    dtype = f"{kind}[{unit}]"
    values = np.array([1, 2, 3], dtype="i8").astype(dtype)
    arr = SparseArray(values)

    result = arr.shift(1)
    expected = SparseArray(np.array(["NaT", values[0], values[1]], dtype=dtype))
    tm.assert_sp_array_equal(result, expected)

    # an explicit fill is a Timestamp/Timedelta, which np.result_type rejects
    #  as a dtype
    box = pd.Timestamp if kind == "M8" else pd.Timedelta
    result = arr.shift(1, fill_value=box(values[2]))
    expected = SparseArray(np.array([values[2], values[0], values[1]], dtype=dtype))
    tm.assert_sp_array_equal(result, expected)


@pytest.mark.parametrize("kind", ["M8", "m8"])
@pytest.mark.parametrize("unit", ["s", "ms", "us", "ns"])
@pytest.mark.parametrize("boxed", [True, False])
@pytest.mark.parametrize("na_fill", [True, False])
def test_fillna_datetimelike_boxed_value(kind, unit, boxed, na_fill):
    # GH#69027 numpy resolved a boxed Timestamp/Timedelta against an M8/m8
    #  array as object, so the stored values came back unboxed
    dtype = f"{kind}[{unit}]"
    values = np.array([1, 2, 3], dtype="i8").astype(dtype)
    values[1] = "NaT"
    box = pd.Timestamp if kind == "M8" else pd.Timedelta
    value = box(values[2])
    # a non-NA fill_value keeps the NaT in sp_values, so the fill is written
    #  there rather than only reaching the result's dtype
    arr = SparseArray(values, fill_value=None if na_fill else values[0])

    result = arr.fillna(value if boxed else value.asm8)

    assert result.sp_values.dtype == values.dtype
    expected = values.copy()
    expected[1] = values[2]
    tm.assert_numpy_array_equal(result.to_dense(), expected)
    expected_fill = values[2] if na_fill else values[0]
    assert result.dtype == pd.SparseDtype(values.dtype, fill_value=expected_fill)


@pytest.mark.parametrize("kind", ["M8", "m8"])
@pytest.mark.parametrize("value", [pd.NaT, None, np.nan, pd.NA])
def test_fillna_datetimelike_na_value(kind, value):
    # GH#69027 NaT, None, and pd.NA went through np.where as object like a
    #  Timestamp; only np.nan raised DTypePromotionError
    dtype = f"{kind}[ns]"
    values = np.array([1, 2, 3], dtype="i8").astype(dtype)
    values[1] = "NaT"
    arr = SparseArray(values, fill_value=values[0])

    result = arr.fillna(value)

    assert result.sp_values.dtype == values.dtype
    tm.assert_numpy_array_equal(result.to_dense(), values)


@pytest.mark.parametrize("kind", ["M8", "m8"])
@pytest.mark.parametrize("unit", ["s", "ms", "us", "ns"])
@pytest.mark.parametrize(
    "spelling", ["boxed", "numpy", "numpy_other_unit", "stdlib", "string"]
)
def test_fillna_datetimelike_value_spellings(kind, unit, spelling):
    # GH#69027 only a numpy scalar in the subtype's own unit or a coarser one
    #  reached np.where in a form it could resolve -- a finer-unit one widened
    #  the stored values
    dtype = f"{kind}[{unit}]"
    values = np.array([1, 2, 3], dtype="i8").astype(dtype)
    values[1] = "NaT"
    arr = SparseArray(values, fill_value=values[0])

    # a whole second, so every spelling below represents it exactly
    other_unit = "s" if unit == "ns" else "ns"
    if kind == "M8":
        fill = pd.Timestamp("2020-01-02").as_unit(unit)
        spellings = {
            "boxed": fill,
            "numpy": fill.asm8,
            "numpy_other_unit": fill.as_unit(other_unit).asm8,
            "stdlib": fill.to_pydatetime(),
            "string": "2020-01-02",
        }
    else:
        fill = pd.Timedelta(1, unit="s").as_unit(unit)
        spellings = {
            "boxed": fill,
            "numpy": fill.asm8,
            "numpy_other_unit": fill.as_unit(other_unit).asm8,
            "stdlib": fill.to_pytimedelta(),
            "string": "1 second",
        }

    result = arr.fillna(spellings[spelling])

    assert result.sp_values.dtype == values.dtype
    expected = values.copy()
    expected[1] = fill.asm8
    tm.assert_numpy_array_equal(result.to_dense(), expected)


@pytest.mark.parametrize("unit, fill_unit", [("s", "ms"), ("ms", "ns"), ("us", "ns")])
def test_fillna_timedelta_lossy_numpy_fill(unit, fill_unit):
    # GH#69027 a fill the subtype cannot hold has to reach numpy's own promotion
    #  untouched: _promote_for_fill raises "Cannot losslessly convert units" for
    #  a timedelta64 it would have to widen
    values = np.array([1, 2, 3], dtype="i8").astype(f"m8[{unit}]")
    values[1] = "NaT"
    arr = SparseArray(values, fill_value=values[0])

    result = arr.fillna(np.timedelta64(2, fill_unit))

    assert result.sp_values.dtype == np.dtype(f"m8[{fill_unit}]")


def test_value_counts_object_subtype_with_na():
    # GH#68586 the fill_value mask took the truth of an object comparison, which
    #  pd.NA makes ambiguous
    values = np.array([1, pd.NA, 0, 1], dtype=object)
    arr = SparseArray(values, fill_value=0)

    result = arr.value_counts(dropna=False)
    expected = pd.Series([1, 2, 1], index=pd.Index([0, 1, pd.NA], dtype=object))
    tm.assert_series_equal(result, expected)


@pytest.mark.parametrize(
    "subtype, item",
    [
        ("int64", 1.5),
        ("int64", 2**63),
        ("int64", pd.Timestamp("2016-01-01")),
        ("bool", 1.5),
        ("M8[ns]", 1.5),
        ("m8[ns]", 1.5),
    ],
)
def test_insert_lossy_raises(subtype, item):
    # GH#69028 a value the subtype cannot hold must raise, not be cast
    arr = SparseArray(np.array([1, 2, 3]).astype(subtype))
    with pytest.raises(TypeError, match="Invalid value"):
        arr.insert(0, item)


def test_insert_lossless_float_into_int_subtype():
    # GH#69028 a float the subtype can hold exactly is not rejected
    arr = SparseArray([1, 2, 3])
    result = arr.insert(0, 1.0)
    tm.assert_extension_array_equal(result, SparseArray([1, 1, 2, 3]))