from datetime import datetime import struct import numpy as np import pytest from pandas._libs import ( algos as libalgos, hashtable as ht, ) from pandas.core.dtypes.common import ( is_bool_dtype, is_complex_dtype, is_float_dtype, is_integer_dtype, is_object_dtype, ) from pandas.core.dtypes.dtypes import ( CategoricalDtype, DatetimeTZDtype, ) import pandas as pd import pandas._testing as tm import pandas.core.algorithms as algos from pandas.core.arrays import ( DatetimeArray, SparseArray, TimedeltaArray, ) import pandas.core.common as com class TestFactorize: def test_factorize_complex(self): # GH#17927 array = np.array([1, 2, 2 + 1j], dtype=complex) labels, uniques = algos.factorize(array) expected_labels = np.array([0, 1, 2], dtype=np.intp) tm.assert_numpy_array_equal(labels, expected_labels) expected_uniques = np.array([(1 + 0j), (2 + 0j), (2 + 1j)], dtype=complex) tm.assert_numpy_array_equal(uniques, expected_uniques) def test_factorize(self, index_or_series_obj_orderable, sort): obj = index_or_series_obj_orderable result_codes, result_uniques = obj.factorize(sort=sort) constructor = pd.Index if isinstance(obj, pd.MultiIndex): constructor = pd.MultiIndex.from_tuples expected_arr = obj.unique() if expected_arr.dtype == np.float16: expected_arr = expected_arr.astype(np.float32) expected_uniques = constructor(expected_arr) if ( isinstance(obj, pd.Index) and expected_uniques.dtype == bool and obj.dtype == object ): expected_uniques = expected_uniques.astype(object) if sort: expected_uniques = expected_uniques.sort_values() # construct an integer ndarray so that # `expected_uniques.take(expected_codes)` is equal to `obj` expected_uniques_list = list(expected_uniques) expected_codes = [expected_uniques_list.index(val) for val in obj] expected_codes = np.asarray(expected_codes, dtype=np.intp) tm.assert_numpy_array_equal(result_codes, expected_codes) tm.assert_index_equal(result_uniques, expected_uniques, exact=True) def test_series_factorize_use_na_sentinel_false(self): # GH#35667 values = np.array([1, 2, 1, np.nan]) ser = pd.Series(values) codes, uniques = ser.factorize(use_na_sentinel=False) expected_codes = np.array([0, 1, 0, 2], dtype=np.intp) expected_uniques = pd.Index([1.0, 2.0, np.nan]) tm.assert_numpy_array_equal(codes, expected_codes) tm.assert_index_equal(uniques, expected_uniques) def test_basic(self): items = np.array(["a", "b", "b", "a", "a", "c", "c", "c"], dtype=object) codes, uniques = algos.factorize(items) tm.assert_numpy_array_equal(uniques, np.array(["a", "b", "c"], dtype=object)) codes, uniques = algos.factorize(items, sort=True) exp = np.array([0, 1, 1, 0, 0, 2, 2, 2], dtype=np.intp) tm.assert_numpy_array_equal(codes, exp) exp = np.array(["a", "b", "c"], dtype=object) tm.assert_numpy_array_equal(uniques, exp) arr = np.arange(5, dtype=np.intp)[::-1] codes, uniques = algos.factorize(arr) exp = np.array([0, 1, 2, 3, 4], dtype=np.intp) tm.assert_numpy_array_equal(codes, exp) exp = np.array([4, 3, 2, 1, 0], dtype=arr.dtype) tm.assert_numpy_array_equal(uniques, exp) codes, uniques = algos.factorize(arr, sort=True) exp = np.array([4, 3, 2, 1, 0], dtype=np.intp) tm.assert_numpy_array_equal(codes, exp) exp = np.array([0, 1, 2, 3, 4], dtype=arr.dtype) tm.assert_numpy_array_equal(uniques, exp) arr = np.arange(5.0)[::-1] codes, uniques = algos.factorize(arr) exp = np.array([0, 1, 2, 3, 4], dtype=np.intp) tm.assert_numpy_array_equal(codes, exp) exp = np.array([4.0, 3.0, 2.0, 1.0, 0.0], dtype=arr.dtype) tm.assert_numpy_array_equal(uniques, exp) codes, uniques = algos.factorize(arr, sort=True) exp = np.array([4, 3, 2, 1, 0], dtype=np.intp) tm.assert_numpy_array_equal(codes, exp) exp = np.array([0.0, 1.0, 2.0, 3.0, 4.0], dtype=arr.dtype) tm.assert_numpy_array_equal(uniques, exp) def test_mixed(self): # doc example reshaping.rst x = pd.Series(["A", "A", np.nan, "B", 3.14, np.inf]) codes, uniques = algos.factorize(x) exp = np.array([0, 0, -1, 1, 2, 3], dtype=np.intp) tm.assert_numpy_array_equal(codes, exp) exp = pd.Index(["A", "B", 3.14, np.inf]) tm.assert_index_equal(uniques, exp) codes, uniques = algos.factorize(x, sort=True) exp = np.array([2, 2, -1, 3, 0, 1], dtype=np.intp) tm.assert_numpy_array_equal(codes, exp) exp = pd.Index([3.14, np.inf, "A", "B"]) tm.assert_index_equal(uniques, exp) def test_factorize_datetime64(self): # M8 v1 = pd.Timestamp("20130101 09:00:00.00004") v2 = pd.Timestamp("20130101") x = pd.Series([v1, v1, v1, v2, v2, v1]) codes, uniques = algos.factorize(x) exp = np.array([0, 0, 0, 1, 1, 0], dtype=np.intp) tm.assert_numpy_array_equal(codes, exp) exp = pd.DatetimeIndex([v1, v2]) tm.assert_index_equal(uniques, exp) codes, uniques = algos.factorize(x, sort=True) exp = np.array([1, 1, 1, 0, 0, 1], dtype=np.intp) tm.assert_numpy_array_equal(codes, exp) exp = pd.DatetimeIndex([v2, v1]) tm.assert_index_equal(uniques, exp) def test_factorize_period(self): # period v1 = pd.Period("201302", freq="M") v2 = pd.Period("201303", freq="M") x = pd.Series([v1, v1, v1, v2, v2, v1]) # periods are not 'sorted' as they are converted back into an index codes, uniques = algos.factorize(x) exp = np.array([0, 0, 0, 1, 1, 0], dtype=np.intp) tm.assert_numpy_array_equal(codes, exp) tm.assert_index_equal(uniques, pd.PeriodIndex([v1, v2])) codes, uniques = algos.factorize(x, sort=True) exp = np.array([0, 0, 0, 1, 1, 0], dtype=np.intp) tm.assert_numpy_array_equal(codes, exp) tm.assert_index_equal(uniques, pd.PeriodIndex([v1, v2])) def test_factorize_timedelta(self): # GH 5986 v1 = pd.to_timedelta("1 day 1 min") v2 = pd.to_timedelta("1 day") x = pd.Series([v1, v2, v1, v1, v2, v2, v1]) codes, uniques = algos.factorize(x) exp = np.array([0, 1, 0, 0, 1, 1, 0], dtype=np.intp) tm.assert_numpy_array_equal(codes, exp) tm.assert_index_equal(uniques, pd.to_timedelta([v1, v2])) codes, uniques = algos.factorize(x, sort=True) exp = np.array([1, 0, 1, 1, 0, 0, 1], dtype=np.intp) tm.assert_numpy_array_equal(codes, exp) tm.assert_index_equal(uniques, pd.to_timedelta([v2, v1])) def test_factorize_nan(self): # nan should map to na_sentinel, not reverse_indexer[na_sentinel] # rizer.factorize should not raise an exception if na_sentinel indexes # outside of reverse_indexer key = np.array([1, 2, 1, np.nan], dtype="O") rizer = ht.ObjectFactorizer(len(key)) for na_sentinel in (-1, 20): ids = rizer.factorize(key, na_sentinel=na_sentinel) expected = np.array([0, 1, 0, na_sentinel], dtype=np.intp) assert len(set(key)) == len(set(expected)) tm.assert_numpy_array_equal(pd.isna(key), expected == na_sentinel) tm.assert_numpy_array_equal(ids, expected) def test_factorizer_with_mask(self): # GH#49549 data = np.array([1, 2, 3, 1, 1, 0], dtype="int64") mask = np.array([False, False, False, False, False, True]) rizer = ht.Int64Factorizer(len(data)) result = rizer.factorize(data, mask=mask) expected = np.array([0, 1, 2, 0, 0, -1], dtype=np.intp) tm.assert_numpy_array_equal(result, expected) expected_uniques = np.array([1, 2, 3], dtype="int64") tm.assert_numpy_array_equal(rizer.uniques.to_array(), expected_uniques) def test_factorizer_object_with_nan(self): # GH#49549 data = np.array([1, 2, 3, 1, np.nan]) rizer = ht.ObjectFactorizer(len(data)) result = rizer.factorize(data.astype(object)) expected = np.array([0, 1, 2, 0, -1], dtype=np.intp) tm.assert_numpy_array_equal(result, expected) expected_uniques = np.array([1, 2, 3], dtype=object) tm.assert_numpy_array_equal(rizer.uniques.to_array(), expected_uniques) @pytest.mark.parametrize( "data, expected_codes, expected_uniques", [ ( [(1, 1), (1, 2), (0, 0), (1, 2), "nonsense"], [0, 1, 2, 1, 3], [(1, 1), (1, 2), (0, 0), "nonsense"], ), ( [(1, 1), (1, 2), (0, 0), (1, 2), (1, 2, 3)], [0, 1, 2, 1, 3], [(1, 1), (1, 2), (0, 0), (1, 2, 3)], ), ([(1, 1), (1, 2), (0, 0), (1, 2)], [0, 1, 2, 1], [(1, 1), (1, 2), (0, 0)]), ], ) def test_factorize_tuple_list(self, data, expected_codes, expected_uniques): # GH9454 data = com.asarray_tuplesafe(data, dtype=object) codes, uniques = pd.factorize(data) tm.assert_numpy_array_equal(codes, np.array(expected_codes, dtype=np.intp)) expected_uniques_array = com.asarray_tuplesafe(expected_uniques, dtype=object) tm.assert_numpy_array_equal(uniques, expected_uniques_array) def test_complex_sorting(self): # gh 12666 - check no segfault x17 = np.array([complex(i) for i in range(17)], dtype=object) msg = "'[<>]' not supported between instances of .*" with pytest.raises(TypeError, match=msg): algos.factorize(x17[::-1], sort=True) def test_numeric_dtype_factorize(self, any_real_numpy_dtype): # GH41132 dtype = any_real_numpy_dtype data = np.array([1, 2, 2, 1], dtype=dtype) expected_codes = np.array([0, 1, 1, 0], dtype=np.intp) expected_uniques = np.array([1, 2], dtype=dtype) codes, uniques = algos.factorize(data) tm.assert_numpy_array_equal(codes, expected_codes) tm.assert_numpy_array_equal(uniques, expected_uniques) def test_float64_factorize(self, writable): data = np.array([1.0, 1e8, 1.0, 1e-8, 1e8, 1.0], dtype=np.float64) data.setflags(write=writable) expected_codes = np.array([0, 1, 0, 2, 1, 0], dtype=np.intp) expected_uniques = np.array([1.0, 1e8, 1e-8], dtype=np.float64) codes, uniques = algos.factorize(data) tm.assert_numpy_array_equal(codes, expected_codes) tm.assert_numpy_array_equal(uniques, expected_uniques) def test_uint64_factorize(self, writable): data = np.array([2**64 - 1, 1, 2**64 - 1], dtype=np.uint64) data.setflags(write=writable) expected_codes = np.array([0, 1, 0], dtype=np.intp) expected_uniques = np.array([2**64 - 1, 1], dtype=np.uint64) codes, uniques = algos.factorize(data) tm.assert_numpy_array_equal(codes, expected_codes) tm.assert_numpy_array_equal(uniques, expected_uniques) def test_int64_factorize(self, writable): data = np.array([2**63 - 1, -(2**63), 2**63 - 1], dtype=np.int64) data.setflags(write=writable) expected_codes = np.array([0, 1, 0], dtype=np.intp) expected_uniques = np.array([2**63 - 1, -(2**63)], dtype=np.int64) codes, uniques = algos.factorize(data) tm.assert_numpy_array_equal(codes, expected_codes) tm.assert_numpy_array_equal(uniques, expected_uniques) def test_string_factorize(self, writable): data = np.array(["a", "c", "a", "b", "c"], dtype=object) data.setflags(write=writable) expected_codes = np.array([0, 1, 0, 2, 1], dtype=np.intp) expected_uniques = np.array(["a", "c", "b"], dtype=object) codes, uniques = algos.factorize(data) tm.assert_numpy_array_equal(codes, expected_codes) tm.assert_numpy_array_equal(uniques, expected_uniques) def test_object_factorize(self, writable): data = np.array(["a", "c", None, np.nan, "a", "b", pd.NaT, "c"], dtype=object) data.setflags(write=writable) expected_codes = np.array([0, 1, -1, -1, 0, 2, -1, 1], dtype=np.intp) expected_uniques = np.array(["a", "c", "b"], dtype=object) codes, uniques = algos.factorize(data) tm.assert_numpy_array_equal(codes, expected_codes) tm.assert_numpy_array_equal(uniques, expected_uniques) def test_datetime64_factorize(self, writable): # GH35650 Verify whether read-only datetime64 array can be factorized data = np.array([np.datetime64("2020-01-01T00:00:00.000")], dtype="M8[ns]") data.setflags(write=writable) expected_codes = np.array([0], dtype=np.intp) expected_uniques = np.array( ["2020-01-01T00:00:00.000000000"], dtype="datetime64[ns]" ) codes, uniques = pd.factorize(data) tm.assert_numpy_array_equal(codes, expected_codes) tm.assert_numpy_array_equal(uniques, expected_uniques) def test_factorize_rangeindex(self, sort): # increasing -> sort doesn't matter ri = pd.RangeIndex.from_range(range(10)) expected = np.arange(10, dtype=np.intp), ri result = algos.factorize(ri, sort=sort) tm.assert_numpy_array_equal(result[0], expected[0]) tm.assert_index_equal(result[1], expected[1], exact=True) result = ri.factorize(sort=sort) tm.assert_numpy_array_equal(result[0], expected[0]) tm.assert_index_equal(result[1], expected[1], exact=True) def test_factorize_rangeindex_decreasing(self, sort): # decreasing -> sort matters ri = pd.RangeIndex.from_range(range(10)) expected = np.arange(10, dtype=np.intp), ri ri2 = ri[::-1] expected = expected[0], ri2 if sort: expected = expected[0][::-1], expected[1][::-1] result = algos.factorize(ri2, sort=sort) tm.assert_numpy_array_equal(result[0], expected[0]) tm.assert_index_equal(result[1], expected[1], exact=True) result = ri2.factorize(sort=sort) tm.assert_numpy_array_equal(result[0], expected[0]) tm.assert_index_equal(result[1], expected[1], exact=True) def test_deprecate_order(self): # gh 19727 - check warning is raised for deprecated keyword, order. # Test not valid once order keyword is removed. data = np.array([2**63, 1, 2**63], dtype=np.uint64) with pytest.raises(TypeError, match="got an unexpected keyword"): algos.factorize(data, order=True) with tm.assert_produces_warning(False): algos.factorize(data) @pytest.mark.parametrize( "data", [ np.array([0, 1, 0], dtype="u8"), np.array([-(2**63), 1, -(2**63)], dtype="i8"), np.array(["__nan__", "foo", "__nan__"], dtype="object"), ], ) def test_parametrized_factorize_na_value_default(self, data): # arrays that include the NA default for that type, but isn't used. codes, uniques = algos.factorize(data) expected_uniques = data[[0, 1]] expected_codes = np.array([0, 1, 0], dtype=np.intp) tm.assert_numpy_array_equal(codes, expected_codes) tm.assert_numpy_array_equal(uniques, expected_uniques) @pytest.mark.parametrize( "data, na_value", [ (np.array([0, 1, 0, 2], dtype="u8"), 0), (np.array([1, 0, 1, 2], dtype="u8"), 1), (np.array([-(2**63), 1, -(2**63), 0], dtype="i8"), -(2**63)), (np.array([1, -(2**63), 1, 0], dtype="i8"), 1), (np.array(["a", "", "a", "b"], dtype=object), "a"), (np.array([(), ("a", 1), (), ("a", 2)], dtype=object), ()), (np.array([("a", 1), (), ("a", 1), ("a", 2)], dtype=object), ("a", 1)), ], ) def test_parametrized_factorize_na_value(self, data, na_value): codes, uniques = algos.factorize_array(data, na_value=na_value) expected_uniques = data[[1, 3]] expected_codes = np.array([-1, 0, -1, 1], dtype=np.intp) tm.assert_numpy_array_equal(codes, expected_codes) tm.assert_numpy_array_equal(uniques, expected_uniques) @pytest.mark.parametrize( "data, uniques", [ ( np.array(["b", "a", None, "b"], dtype=object), np.array(["b", "a"], dtype=object), ), ( pd.array([2, 1, pd.NA, 2], dtype="Int64"), pd.array([2, 1], dtype="Int64"), ), ], ids=["numpy_array", "extension_array"], ) def test_factorize_use_na_sentinel(self, sort, data, uniques): codes, uniques = algos.factorize(data, sort=sort, use_na_sentinel=True) if sort: expected_codes = np.array([1, 0, -1, 1], dtype=np.intp) expected_uniques = algos.safe_sort(uniques) else: expected_codes = np.array([0, 1, -1, 0], dtype=np.intp) expected_uniques = uniques tm.assert_numpy_array_equal(codes, expected_codes) if isinstance(data, np.ndarray): tm.assert_numpy_array_equal(uniques, expected_uniques) else: tm.assert_extension_array_equal(uniques, expected_uniques) @pytest.mark.parametrize( "data, expected_codes, expected_uniques", [ ( ["a", None, "b", "a"], np.array([0, 1, 2, 0], dtype=np.dtype("intp")), np.array(["a", np.nan, "b"], dtype=object), ), ( ["a", np.nan, "b", "a"], np.array([0, 1, 2, 0], dtype=np.dtype("intp")), np.array(["a", np.nan, "b"], dtype=object), ), ], ) def test_object_factorize_use_na_sentinel_false( self, data, expected_codes, expected_uniques ): codes, uniques = algos.factorize( np.array(data, dtype=object), use_na_sentinel=False ) tm.assert_numpy_array_equal(uniques, expected_uniques, strict_nan=True) tm.assert_numpy_array_equal(codes, expected_codes, strict_nan=True) @pytest.mark.parametrize( "data, expected_codes, expected_uniques", [ ( np.array([1, None, 1, 2], dtype=object), np.array([0, 1, 0, 2], dtype=np.dtype("intp")), np.array([1, np.nan, 2], dtype="O"), ), ( np.array([1, np.nan, 1, 2], dtype=np.float64), np.array([0, 1, 0, 2], dtype=np.dtype("intp")), np.array([1, np.nan, 2], dtype=np.float64), ), ], ) def test_int_factorize_use_na_sentinel_false( self, data, expected_codes, expected_uniques ): codes, uniques = algos.factorize(data, use_na_sentinel=False) tm.assert_numpy_array_equal(uniques, expected_uniques, strict_nan=True) tm.assert_numpy_array_equal(codes, expected_codes, strict_nan=True) @pytest.mark.parametrize( "data, expected_codes, expected_uniques", [ ( pd.Index(pd.Categorical(["a", "a", "b"])), np.array([0, 0, 1], dtype=np.intp), pd.CategoricalIndex( ["a", "b"], categories=["a", "b"], dtype="category" ), ), ( pd.Series(pd.Categorical(["a", "a", "b"])), np.array([0, 0, 1], dtype=np.intp), pd.CategoricalIndex( ["a", "b"], categories=["a", "b"], dtype="category" ), ), ( pd.Series(pd.DatetimeIndex(["2017", "2017"], tz="US/Eastern")), np.array([0, 0], dtype=np.intp), pd.DatetimeIndex(["2017"], tz="US/Eastern"), ), ], ) def test_factorize_mixed_values(self, data, expected_codes, expected_uniques): # GH 19721 codes, uniques = algos.factorize(data) tm.assert_numpy_array_equal(codes, expected_codes) tm.assert_index_equal(uniques, expected_uniques) def test_factorize_interval_non_nano(self, unit): # GH#56099 left = pd.DatetimeIndex(["2016-01-01", np.nan, "2015-10-11"]).as_unit(unit) right = pd.DatetimeIndex(["2016-01-02", np.nan, "2015-10-15"]).as_unit(unit) idx = pd.IntervalIndex.from_arrays(left, right) codes, cats = idx.factorize() assert cats.dtype == f"interval[datetime64[{unit}], right]" ts = pd.Timestamp(0).as_unit(unit) idx2 = pd.IntervalIndex.from_arrays(left - ts, right - ts) codes2, cats2 = idx2.factorize() assert cats2.dtype == f"interval[timedelta64[{unit}], right]" idx3 = pd.IntervalIndex.from_arrays( left.tz_localize("US/Pacific"), right.tz_localize("US/Pacific") ) codes3, cats3 = idx3.factorize() assert cats3.dtype == f"interval[datetime64[{unit}, US/Pacific], right]" class TestUnique: def test_ints(self): arr = np.random.default_rng(2).integers(0, 100, size=50) result = algos.unique(arr) assert isinstance(result, np.ndarray) def test_objects(self): arr = np.random.default_rng(2).integers(0, 100, size=50).astype("O") result = algos.unique(arr) assert isinstance(result, np.ndarray) def test_object_refcount_bug(self): lst = np.array(["A", "B", "C", "D", "E"], dtype=object) for i in range(1000): len(algos.unique(lst)) def test_index_returned(self, index): # GH#57043 index = index.repeat(2) result = algos.unique(index) # dict.fromkeys preserves the order unique_values = list(dict.fromkeys(index._values)) if isinstance(index, pd.MultiIndex): expected = pd.MultiIndex.from_tuples(unique_values, names=index.names) else: expected = pd.Index(unique_values, dtype=index.dtype) if isinstance(index.dtype, DatetimeTZDtype): expected = expected.normalize() tm.assert_index_equal(result, expected, exact=True, check_freq=False) def test_factorize_multiindex_empty(self): # GH#57517 mi = pd.MultiIndex.from_product( [pd.Index([], name="a", dtype=object), pd.Index([], name="i", dtype="f4")] ) codes, uniques = mi.factorize() exp_codes = np.array([], dtype=np.intp) tm.assert_numpy_array_equal(codes, exp_codes) tm.assert_index_equal(uniques, mi[:0]) def test_dtype_preservation(self, any_numpy_dtype): # GH 15442 if any_numpy_dtype in (tm.BYTES_DTYPES + tm.STRING_DTYPES): data = [1, 2, 2] uniques = [1, 2] elif is_integer_dtype(any_numpy_dtype): data = [1, 2, 2] uniques = [1, 2] elif is_float_dtype(any_numpy_dtype): data = [1, 2, 2] uniques = [1.0, 2.0] elif is_complex_dtype(any_numpy_dtype): data = [complex(1, 0), complex(2, 0), complex(2, 0)] uniques = [complex(1, 0), complex(2, 0)] elif is_bool_dtype(any_numpy_dtype): data = [True, True, False] uniques = [True, False] elif is_object_dtype(any_numpy_dtype): data = ["A", "B", "B"] uniques = ["A", "B"] else: # datetime64[ns]/M8[ns]/timedelta64[ns]/m8[ns] tested elsewhere data = [1, 2, 2] uniques = [1, 2] result = pd.Series(data, dtype=any_numpy_dtype).unique() expected = np.array(uniques, dtype=any_numpy_dtype) if any_numpy_dtype in tm.STRING_DTYPES: expected = expected.astype(object) if expected.dtype.kind in ["m", "M"]: # We get TimedeltaArray/DatetimeArray assert isinstance(result, (DatetimeArray, TimedeltaArray)) result = np.array(result) tm.assert_numpy_array_equal(result, expected) def test_datetime64_dtype_array_returned(self): # GH 9431 dt_arr = np.array( [ "2015-01-03T00:00:00.000000000", "2015-01-01T00:00:00.000000000", ], dtype="M8[ns]", ) dt_index = pd.to_datetime( [ "2015-01-03T00:00:00.000000000", "2015-01-01T00:00:00.000000000", "2015-01-01T00:00:00.000000000", ] ) result = algos.unique(dt_index) expected = pd.to_datetime(dt_arr) tm.assert_index_equal(result, expected, exact=True) s = pd.Series(dt_index) result = algos.unique(s) tm.assert_numpy_array_equal(result, dt_arr) assert result.dtype == dt_arr.dtype arr = s.values result = algos.unique(arr) tm.assert_numpy_array_equal(result, dt_arr) assert result.dtype == dt_arr.dtype def test_datetime_non_ns(self): a = np.array(["2000", "2000", "2001"], dtype="datetime64[s]") result = pd.unique(a) expected = np.array(["2000", "2001"], dtype="datetime64[s]") tm.assert_numpy_array_equal(result, expected) def test_timedelta_non_ns(self): a = np.array(["2000", "2000", "2001"], dtype="timedelta64[s]") result = pd.unique(a) expected = np.array([2000, 2001], dtype="timedelta64[s]") tm.assert_numpy_array_equal(result, expected) def test_timedelta64_dtype_array_returned(self): # GH 9431 td_arr = np.array([31200, 45678, 10000], dtype="m8[ns]") td_index = pd.to_timedelta([31200, 45678, 31200, 10000, 45678]) result = algos.unique(td_index) expected = pd.to_timedelta(td_arr) tm.assert_index_equal(result, expected) assert result.dtype == expected.dtype s = pd.Series(td_index) result = algos.unique(s) tm.assert_numpy_array_equal(result, td_arr) assert result.dtype == td_arr.dtype arr = s.values result = algos.unique(arr) tm.assert_numpy_array_equal(result, td_arr) assert result.dtype == td_arr.dtype def test_uint64_overflow(self): s = pd.Series([1, 2, 2**63, 2**63], dtype=np.uint64) exp = np.array([1, 2, 2**63], dtype=np.uint64) tm.assert_numpy_array_equal(algos.unique(s), exp) def test_nan_in_object_array(self): duplicated_items = ["a", np.nan, "c", "c"] result = pd.unique(np.array(duplicated_items, dtype=object)) expected = np.array(["a", np.nan, "c"], dtype=object) tm.assert_numpy_array_equal(result, expected) def test_categorical(self): # we are expecting to return in the order # of appearance expected = pd.Categorical(list("bac")) # we are expecting to return in the order # of the categories expected_o = pd.Categorical(list("bac"), categories=list("abc"), ordered=True) # GH 15939 c = pd.Categorical(list("baabc")) result = c.unique() tm.assert_categorical_equal(result, expected) result = algos.unique(c) tm.assert_categorical_equal(result, expected) c = pd.Categorical(list("baabc"), ordered=True) result = c.unique() tm.assert_categorical_equal(result, expected_o) result = algos.unique(c) tm.assert_categorical_equal(result, expected_o) # Series of categorical dtype s = pd.Series(pd.Categorical(list("baabc")), name="foo") result = s.unique() tm.assert_categorical_equal(result, expected) result = pd.unique(s) tm.assert_categorical_equal(result, expected) # CI -> return CI ci = pd.CategoricalIndex(pd.Categorical(list("baabc"), categories=list("abc"))) expected = pd.CategoricalIndex(expected) result = ci.unique() tm.assert_index_equal(result, expected) result = pd.unique(ci) tm.assert_index_equal(result, expected) def test_datetime64tz_aware(self, unit): # GH 15939 dti = pd.Index( [ pd.Timestamp("20160101", tz="US/Eastern"), pd.Timestamp("20160101", tz="US/Eastern"), ] ).as_unit(unit) ser = pd.Series(dti) result = ser.unique() expected = dti[:1]._data tm.assert_extension_array_equal(result, expected) result = dti.unique() expected = dti[:1] tm.assert_index_equal(result, expected) result = pd.unique(ser) expected = dti[:1]._data tm.assert_extension_array_equal(result, expected) result = pd.unique(dti) expected = dti[:1] tm.assert_index_equal(result, expected) def test_order_of_appearance(self): # 9346 # light testing of guarantee of order of appearance # these also are the doc-examples result = pd.unique(pd.Series([2, 1, 3, 3])) tm.assert_numpy_array_equal(result, np.array([2, 1, 3], dtype="int64")) result = pd.unique(pd.Series([2] + [1] * 5)) tm.assert_numpy_array_equal(result, np.array([2, 1], dtype="int64")) data = np.array(["a", "a", "b", "c"], dtype=object) result = pd.unique(data) expected = np.array(["a", "b", "c"], dtype=object) tm.assert_numpy_array_equal(result, expected) result = pd.unique(pd.Series(pd.Categorical(list("aabc")))) expected = pd.Categorical(list("abc")) tm.assert_categorical_equal(result, expected) def test_order_of_appearance_dt64(self, unit): ser = pd.Series( [pd.Timestamp("20160101"), pd.Timestamp("20160101")] ).dt.as_unit(unit) result = pd.unique(ser) expected = np.array(["2016-01-01T00:00:00.000000000"], dtype=f"M8[{unit}]") tm.assert_numpy_array_equal(result, expected) def test_order_of_appearance_dt64tz(self, unit): dti = pd.DatetimeIndex( [ pd.Timestamp("20160101", tz="US/Eastern"), pd.Timestamp("20160101", tz="US/Eastern"), ] ).as_unit(unit) result = pd.unique(dti) expected = pd.DatetimeIndex( ["2016-01-01 00:00:00"], dtype=f"datetime64[{unit}, US/Eastern]", freq=None ) tm.assert_index_equal(result, expected) @pytest.mark.parametrize( "arg ,expected", [ (("1", "1", "2"), np.array(["1", "2"], dtype=object)), (("foo",), np.array(["foo"], dtype=object)), ], ) def test_tuple_with_strings(self, arg, expected): # see GH 17108 arg = com.asarray_tuplesafe(arg, dtype=object) result = pd.unique(arg) tm.assert_numpy_array_equal(result, expected) def test_obj_none_preservation(self): # GH 20866 arr = np.array(["foo", None], dtype=object) result = pd.unique(arr) expected = np.array(["foo", None], dtype=object) tm.assert_numpy_array_equal(result, expected, strict_nan=True) def test_signed_zero(self): # GH 21866 a = np.array([-0.0, 0.0]) result = pd.unique(a) expected = np.array([-0.0]) # 0.0 and -0.0 are equivalent tm.assert_numpy_array_equal(result, expected) def test_different_nans(self): # GH 21866 # create different nans from bit-patterns: NAN1 = struct.unpack("d", struct.pack("=Q", 0x7FF8000000000000))[0] NAN2 = struct.unpack("d", struct.pack("=Q", 0x7FF8000000000001))[0] assert NAN1 != NAN1 assert NAN2 != NAN2 a = np.array([NAN1, NAN2]) # NAN1 and NAN2 are equivalent result = pd.unique(a) expected = np.array([np.nan]) tm.assert_numpy_array_equal(result, expected) @pytest.mark.parametrize("el_type", [np.float64, object]) def test_first_nan_kept(self, el_type): # GH 22295 # create different nans from bit-patterns: bits_for_nan1 = 0xFFF8000000000001 bits_for_nan2 = 0x7FF8000000000001 NAN1 = struct.unpack("d", struct.pack("=Q", bits_for_nan1))[0] NAN2 = struct.unpack("d", struct.pack("=Q", bits_for_nan2))[0] assert NAN1 != NAN1 assert NAN2 != NAN2 a = np.array([NAN1, NAN2], dtype=el_type) result = pd.unique(a) assert result.size == 1 # use bit patterns to identify which nan was kept: result_nan_bits = struct.unpack("=Q", struct.pack("d", result[0]))[0] assert result_nan_bits == bits_for_nan1 def test_do_not_mangle_na_values(self, unique_nulls_fixture, unique_nulls_fixture2): # GH 22295 if unique_nulls_fixture is unique_nulls_fixture2: return # skip it, values not unique a = np.array([unique_nulls_fixture, unique_nulls_fixture2], dtype=object) result = pd.unique(a) assert result.size == 2 assert a[0] is unique_nulls_fixture assert a[1] is unique_nulls_fixture2 def test_unique_masked(self, any_numeric_ea_dtype): # GH#48019 ser = pd.Series([1, pd.NA, 2] * 3, dtype=any_numeric_ea_dtype) result = pd.unique(ser) expected = pd.array([1, pd.NA, 2], dtype=any_numeric_ea_dtype) tm.assert_extension_array_equal(result, expected) def test_unique_NumpyExtensionArray(self): arr_complex = pd.array( [1 + 1j, 2, 3] ) # NumpyEADtype('complex128') => NumpyExtensionArray result = pd.unique(arr_complex) expected = pd.array([1 + 1j, 2 + 0j, 3 + 0j]) tm.assert_extension_array_equal(result, expected) def test_nunique_ints(index_or_series_or_array): # GH#36327 values = index_or_series_or_array(np.random.default_rng(2).integers(0, 20, 30)) result = algos.nunique_ints(values) expected = len(algos.unique(values)) assert result == expected def _isin_target_array(values, dtype): """Targets as an ndarray for a NumPy dtype, masked array for a pandas one.""" if dtype[0].isupper(): return pd.array(values, dtype=dtype) return np.array(values, dtype=dtype) def _isin_result_dtype(comps_dtype): """isin over a masked caller gives "boolean", over a NumPy one gives bool.""" return "boolean" if comps_dtype[0].isupper() else bool class TestIsin: def test_invalid(self): msg = ( r"only list-like objects are allowed to be passed to isin\(\), " r"you passed a `int`" ) with pytest.raises(TypeError, match=msg): algos.isin(1, 1) with pytest.raises(TypeError, match=msg): algos.isin(1, [1]) with pytest.raises(TypeError, match=msg): algos.isin([1], 1) def test_basic(self): result = algos.isin(np.array([1, 2]), [1]) expected = np.array([True, False]) tm.assert_numpy_array_equal(result, expected) result = algos.isin(pd.Series([1, 2]), [1]) expected = np.array([True, False]) tm.assert_numpy_array_equal(result, expected) result = algos.isin(pd.Series([1, 2]), pd.Series([1])) expected = np.array([True, False]) tm.assert_numpy_array_equal(result, expected) result = algos.isin(pd.Series([1, 2]), {1}) expected = np.array([True, False]) tm.assert_numpy_array_equal(result, expected) arg = np.array(["a", "b"], dtype=object) result = algos.isin(arg, ["a"]) expected = np.array([True, False]) tm.assert_numpy_array_equal(result, expected) result = algos.isin(pd.Series(arg), pd.Series(["a"])) expected = np.array([True, False]) tm.assert_numpy_array_equal(result, expected) result = algos.isin(pd.Series(arg), {"a"}) expected = np.array([True, False]) tm.assert_numpy_array_equal(result, expected) result = algos.isin(arg, [1]) expected = np.array([False, False]) tm.assert_numpy_array_equal(result, expected) def test_i8(self): arr = pd.date_range("20130101", periods=3).values result = algos.isin(arr, [arr[0]]) expected = np.array([True, False, False]) tm.assert_numpy_array_equal(result, expected) result = algos.isin(arr, arr[0:2]) expected = np.array([True, True, False]) tm.assert_numpy_array_equal(result, expected) result = algos.isin(arr, set(arr[0:2])) expected = np.array([True, True, False]) tm.assert_numpy_array_equal(result, expected) arr = pd.timedelta_range("1 day", periods=3).values result = algos.isin(arr, [arr[0]]) expected = np.array([True, False, False]) tm.assert_numpy_array_equal(result, expected) result = algos.isin(arr, arr[0:2]) expected = np.array([True, True, False]) tm.assert_numpy_array_equal(result, expected) result = algos.isin(arr, set(arr[0:2])) expected = np.array([True, True, False]) tm.assert_numpy_array_equal(result, expected) @pytest.mark.parametrize("dtype1", ["m8[ns]", "M8[ns]", "M8[ns, UTC]", "period[D]"]) @pytest.mark.parametrize("dtype", ["i8", "f8", "u8"]) def test_isin_datetimelike_values_numeric_comps(self, dtype, dtype1): # Anything but object and we get all-False shortcut dta = pd.date_range("2013-01-01", periods=3)._values arr = pd.Series(dta.view("i8")).array.view(dtype1) comps = arr.view("i8").astype(dtype) result = algos.isin(comps, arr) expected = np.zeros(comps.shape, dtype=bool) tm.assert_numpy_array_equal(result, expected) def test_large(self): s = pd.date_range("20000101", periods=2000000, freq="s").values result = algos.isin(s, s[0:2]) expected = np.zeros(len(s), dtype=bool) expected[0] = True expected[1] = True tm.assert_numpy_array_equal(result, expected) @pytest.mark.parametrize("dtype", ["m8[ns]", "M8[ns]", "M8[ns, UTC]", "period[D]"]) def test_isin_datetimelike_all_nat(self, dtype): # GH#56427 dta = pd.date_range("2013-01-01", periods=3)._values arr = pd.Series(dta.view("i8")).array.view(dtype) arr[0] = pd.NaT result = algos.isin(arr, [pd.NaT]) expected = np.array([True, False, False], dtype=bool) tm.assert_numpy_array_equal(result, expected) @pytest.mark.parametrize("dtype", ["m8[ns]", "M8[ns]", "M8[ns, UTC]"]) def test_isin_datetimelike_strings_returns_false(self, dtype): # GH#53111 dta = pd.date_range("2013-01-01", periods=3)._values arr = pd.Series(dta.view("i8")).array.view(dtype) vals = [str(x) for x in arr] res = algos.isin(arr, vals) assert not res.any() vals2 = np.array(vals, dtype=str) res2 = algos.isin(arr, vals2) assert not res2.any() def test_isin_dt64tz_with_nat(self): # the all-NaT values used to get inferred to tznaive, which was evaluated # as non-matching GH#56427 dti = pd.date_range("2016-01-01", periods=3, tz="UTC") ser = pd.Series(dti) ser[0] = pd.NaT res = algos.isin(ser._values, [pd.NaT]) exp = np.array([True, False, False], dtype=bool) tm.assert_numpy_array_equal(res, exp) def test_categorical_from_codes(self): # GH 16639 vals = np.array([0, 1, 2, 0]) cats = ["a", "b", "c"] Sd = pd.Series(pd.Categorical([1]).from_codes(vals, cats)) St = pd.Series(pd.Categorical([1]).from_codes(np.array([0, 1]), cats)) expected = np.array([True, True, False, True]) result = algos.isin(Sd, St) tm.assert_numpy_array_equal(expected, result) def test_categorical_isin(self): vals = np.array([0, 1, 2, 0]) cats = ["a", "b", "c"] cat = pd.Categorical([1]).from_codes(vals, cats) other = pd.Categorical([1]).from_codes(np.array([0, 1]), cats) expected = np.array([True, True, False, True]) result = algos.isin(cat, other) tm.assert_numpy_array_equal(expected, result) def test_same_nan_is_in(self): # GH 22160 # nan is special, because from " a is b" doesn't follow "a == b" # at least, isin() should follow python's "np.nan in [nan] == True" # casting to -> np.float64 -> another float-object somewhere on # the way could jeopardize this behavior comps = np.array([np.nan], dtype=object) # could be casted to float64 values = [np.nan] expected = np.array([True]) result = algos.isin(comps, values) tm.assert_numpy_array_equal(expected, result) def test_same_nan_is_in_large(self): # https://github.com/pandas-dev/pandas/issues/22205 s = np.tile(1.0, 1_000_001) s[0] = np.nan result = algos.isin(s, np.array([np.nan, 1])) expected = np.ones(len(s), dtype=bool) tm.assert_numpy_array_equal(result, expected) def test_same_nan_is_in_large_series(self): # https://github.com/pandas-dev/pandas/issues/22205 s = np.tile(1.0, 1_000_001) series = pd.Series(s) s[0] = np.nan result = series.isin(np.array([np.nan, 1])) expected = pd.Series(np.ones(len(s), dtype=bool)) tm.assert_series_equal(result, expected) def test_same_object_is_in(self): # GH 22160 # there could be special treatment for nans # the user however could define a custom class # with similar behavior, then we at least should # fall back to usual python's behavior: "a in [a] == True" class LikeNan: def __eq__(self, other) -> bool: return False def __hash__(self): return 0 a, b = LikeNan(), LikeNan() arg = np.array([a], dtype=object) # same object -> True tm.assert_numpy_array_equal(algos.isin(arg, [a]), np.array([True])) # different objects -> False tm.assert_numpy_array_equal(algos.isin(arg, [b]), np.array([False])) def test_different_nans(self): # GH 22160 # all nans are handled as equivalent comps = [float("nan")] values = [float("nan")] assert comps[0] is not values[0] # different nan-objects # as list of python-objects: result = algos.isin(np.array(comps), values) tm.assert_numpy_array_equal(np.array([True]), result) # as object-array: result = algos.isin( np.asarray(comps, dtype=object), np.asarray(values, dtype=object) ) tm.assert_numpy_array_equal(np.array([True]), result) # as float64-array: result = algos.isin( np.asarray(comps, dtype=np.float64), np.asarray(values, dtype=np.float64) ) tm.assert_numpy_array_equal(np.array([True]), result) def test_no_cast(self): # GH 22160 # ensure 42 is not casted to a string comps = np.array(["ss", 42], dtype=object) values = ["42"] expected = np.array([False, False]) result = algos.isin(comps, values) tm.assert_numpy_array_equal(expected, result) @pytest.mark.parametrize("empty", [[], pd.Series(dtype=object), np.array([])]) def test_empty(self, empty): # see gh-16991 vals = pd.Index(["a", "b"]) expected = np.array([False, False]) result = algos.isin(vals, empty) tm.assert_numpy_array_equal(expected, result) def test_different_nan_objects(self): # GH 22119 comps = np.array(["nan", np.nan * 1j, float("nan")], dtype=object) vals = np.array([float("nan")], dtype=object) expected = np.array([False, False, True]) result = algos.isin(comps, vals) tm.assert_numpy_array_equal(expected, result) def test_different_nans_as_float64(self): # GH 21866 # create different nans from bit-patterns, # these nans will land in different buckets in the hash-table # if no special care is taken NAN1 = struct.unpack("d", struct.pack("=Q", 0x7FF8000000000000))[0] NAN2 = struct.unpack("d", struct.pack("=Q", 0x7FF8000000000001))[0] assert NAN1 != NAN1 assert NAN2 != NAN2 # check that NAN1 and NAN2 are equivalent: arr = np.array([NAN1, NAN2], dtype=np.float64) lookup1 = np.array([NAN1], dtype=np.float64) result = algos.isin(arr, lookup1) expected = np.array([True, True]) tm.assert_numpy_array_equal(result, expected) lookup2 = np.array([NAN2], dtype=np.float64) result = algos.isin(arr, lookup2) expected = np.array([True, True]) tm.assert_numpy_array_equal(result, expected) def test_isin_int_df_string_search(self): """Comparing df with int`s (1,2) with a string at isin() ("1") -> should not match values because int 1 is not equal str 1""" df = pd.DataFrame({"values": [1, 2]}) result = df.isin(["1"]) expected_false = pd.DataFrame({"values": [False, False]}) tm.assert_frame_equal(result, expected_false) def test_isin_nan_df_string_search(self): """Comparing df with nan value (np.nan,2) with a string at isin() ("NaN") -> should not match values because np.nan is not equal str NaN""" df = pd.DataFrame({"values": [np.nan, 2]}) result = df.isin(np.array(["NaN"], dtype=object)) expected_false = pd.DataFrame({"values": [False, False]}) tm.assert_frame_equal(result, expected_false) def test_isin_float_df_string_search(self): """Comparing df with floats (1.4245,2.32441) with a string at isin() ("1.4245") -> should not match values because float 1.4245 is not equal str 1.4245""" df = pd.DataFrame({"values": [1.4245, 2.32441]}) result = df.isin(np.array(["1.4245"], dtype=object)) expected_false = pd.DataFrame({"values": [False, False]}) tm.assert_frame_equal(result, expected_false) def test_isin_unsigned_dtype(self): # GH#46485 ser = pd.Series([1378774140726870442], dtype=np.uint64) result = ser.isin([1378774140726870528]) expected = pd.Series(False) tm.assert_series_equal(result, expected) def test_isin_uint64_vs_float_no_precision_loss(self): # GH#46485: a uint64 magnitude > 2**53 must not be down-cast to float64 # when checking membership against float targets. value = 2**63 + 3 ser = pd.Series([value], dtype=np.uint64) result = ser.isin([float(value)]) # float(value) rounds to 2.0**63 expected = pd.Series([False]) tm.assert_series_equal(result, expected) def test_isin_float_vs_int64_no_precision_loss(self): # GH#46485: the symmetric case -- a float caller against int64 targets # whose magnitude exceeds 2**53 must not collide via a float64 cast. ser = pd.Series([2.0**53], dtype="float64") result = ser.isin([2**53 + 1]) expected = pd.Series([False]) tm.assert_series_equal(result, expected) def test_isin_uint64_vs_complex_no_precision_loss(self): # GH#46485: complex128 components are float64, so casting a uint64 # with magnitude > 2**53 to complex128 is just as lossy as float64. ser = pd.Series([2**63 + 3], dtype=np.uint64) result = ser.isin([complex(2**63)]) expected = pd.Series([False]) tm.assert_series_equal(result, expected) def test_isin_complex_vs_int64_no_precision_loss(self): # GH#46485: the symmetric case -- a complex caller against int64 # targets whose magnitude exceeds 2**53. ser = pd.Series([complex(2**53)], dtype="complex128") result = ser.isin([2**53 + 1]) expected = pd.Series([False]) tm.assert_series_equal(result, expected) @pytest.mark.parametrize("dtype", ["float32", "complex128"]) def test_isin_mixed_int_float_targets_no_precision_loss(self, dtype): # GH#46485: a mixed int/float targets list is cast to float64 by # _ensure_arraylike, rounding 2**53 + 1 before the exact-range check # can see it; we must still object-cast and compare exactly. ser = pd.Series([2**53], dtype="int64").astype(dtype) result = ser.isin([2**53 + 1, 1.5]) expected = pd.Series([False]) tm.assert_series_equal(result, expected) def test_isin_float_vs_small_int_keeps_numeric_path(self, monkeypatch): # GH#46485: int targets that fit exactly in float64 (|x| <= 2**53) are # safe to compare numerically, so they must not fall back to the slower # object-dtype path. 2**53 is the inclusive boundary. called = [] monkeypatch.setattr( algos, "construct_1d_object_array_from_listlike", lambda values: called.append(values) or np.asarray(values, dtype=object), ) ser = pd.Series([1.0, 2.0, 2.0**53], dtype="float64") result = ser.isin([2, 2**53]) tm.assert_series_equal(result, pd.Series([False, True, True])) assert called == [] # numeric path, no object cast # ... while an out-of-range target still forces the object path. ser.isin([2**53 + 1]) assert len(called) == 1 @pytest.mark.parametrize( "comps_dtype, values_dtype", [ ("uint64", "int64"), ("uint64", "Int64"), ("UInt64", "int64"), ("UInt64", "Int64"), ("int64", "uint64"), ("int64", "UInt64"), ("Int64", "uint64"), ("Int64", "UInt64"), ], ) def test_isin_uint64_vs_int64_no_precision_loss(self, comps_dtype, values_dtype): # GH#59609: uint64 and int64 have float64 as their common dtype, so # above 2**53 two distinct values can share a float64 and collide. ser = pd.Series([635554097106142143], dtype=comps_dtype) values = _isin_target_array([635554097106142079], values_dtype) result = ser.isin(values) expected = pd.Series([False], dtype=_isin_result_dtype(comps_dtype)) tm.assert_series_equal(result, expected) @pytest.mark.parametrize( "comps_dtype, values_dtype", [("uint64", "int64"), ("int64", "uint64")] ) def test_isin_uint64_vs_int64_still_matches(self, comps_dtype, values_dtype): # GH#59609: the mixed-signedness handling must not lose true matches. ser = pd.Series([635554097106142143], dtype=comps_dtype) values = _isin_target_array([635554097106142143], values_dtype) result = ser.isin(values) tm.assert_series_equal(result, pd.Series([True])) def test_isin_int64_vs_uint64_scalar_targets(self): # GH#61676: the same collision reached through a list of numpy scalars, # which _ensure_arraylike renders as a uint64 array. ser = pd.Series([1378774140726870442], dtype="int64") result = ser.isin([np.uint64(1378774140726870528)]) tm.assert_series_equal(result, pd.Series([False])) def test_isin_uint64_vs_negative_int64_targets(self): # GH#59609: a negative target cannot match any uint64, and must not be # wrapped into a huge uint64 that matches one. ser = pd.Series([2**64 - 1, 1], dtype="uint64") result = ser.isin(np.array([-1, 1], dtype="int64")) tm.assert_series_equal(result, pd.Series([False, True])) def test_isin_int64_vs_out_of_range_uint64_targets(self): # GH#59609: the symmetric case -- a target above int64's max cannot # match any int64, and must not be wrapped into a negative one. ser = pd.Series([-1, 1], dtype="int64") result = ser.isin(np.array([2**64 - 1, 1], dtype="uint64")) tm.assert_series_equal(result, pd.Series([False, True])) @pytest.mark.parametrize("comps_dtype", ["float64", "complex128"]) @pytest.mark.parametrize("values_dtype", ["int64", "uint64"]) def test_isin_float_comps_vs_int64_ndarray_targets(self, comps_dtype, values_dtype): # GH#59609: the GH#46485 protection only covered targets passed as a # list; an ndarray of targets reached the lossy float64 common dtype. ser = pd.Series([2.0**53], dtype=comps_dtype) result = ser.isin(np.array([2**53 + 1], dtype=values_dtype)) tm.assert_series_equal(result, pd.Series([False])) @pytest.mark.parametrize("comps_dtype", ["int64", "uint64"]) def test_isin_int64_comps_vs_float64_ndarray_targets(self, comps_dtype): # GH#59609: the symmetric case -- an integer caller above 2**53 must # not match a float target it merely rounds to. ser = pd.Series([2**53 + 1], dtype=comps_dtype) result = ser.isin(np.array([2.0**53], dtype="float64")) tm.assert_series_equal(result, pd.Series([False])) @pytest.mark.parametrize( "targets", [[635554097106142079, 1.5], [2**63 + 1, 635554097106142079]] ) def test_isin_signed_comps_vs_float64_target_list(self, targets): # GH#59609: a targets list that _ensure_arraylike renders as float64 # -- because a value falls outside int64 range, or because a float is # mixed in -- rounds the targets before isin sees them. The GH#46485 # guard skipped that check whenever comps was a signed integer. ser = pd.Series([635554097106142143], dtype="int64") result = ser.isin(targets) tm.assert_series_equal(result, pd.Series([False])) def test_isin_uint64_vs_int64_keeps_numeric_path(self, monkeypatch): # GH#59609: dropping the unrepresentable targets leaves both sides # uint64, so the fix must not push this onto the object hashtable. seen = [] real_ismember = ht.ismember monkeypatch.setattr( algos.htable, "ismember", lambda comps, values: ( seen.append(comps.dtype) or real_ismember(comps, values) ), ) ser = pd.Series([635554097106142143], dtype="uint64") ser.isin(np.array([635554097106142079], dtype="int64")) assert seen == [np.dtype("uint64")] def test_isin_uint64_vs_int64_agrees_across_size_branch(self, monkeypatch): # GH#59609: comps longer than _MINIMUM_COMP_ARR_LEN go to np.isin # instead of the hashtable, so both branches need the same answer. monkeypatch.setattr(algos, "_MINIMUM_COMP_ARR_LEN", 0) ser = pd.Series([635554097106142143, 635554097106142079], dtype="uint64") result = ser.isin(np.array([635554097106142079], dtype="int64")) tm.assert_series_equal(result, pd.Series([False, True])) class TestValueCounts: def test_value_counts(self): arr = np.random.default_rng(1234).standard_normal(4) factor = pd.cut(arr, 4) # assert isinstance(factor, n) result = algos.value_counts_internal(factor) breaks = [-1.606, -1.018, -0.431, 0.155, 0.741] index = pd.IntervalIndex.from_breaks(breaks).astype( CategoricalDtype(ordered=True) ) expected = pd.Series([1, 0, 2, 1], index=index, name="count") tm.assert_series_equal(result.sort_index(), expected.sort_index()) def test_value_counts_bins(self): s = [1, 2, 3, 4] result = algos.value_counts_internal(s, bins=1) expected = pd.Series( [4], index=pd.IntervalIndex.from_tuples([(0.996, 4.0)]), name="count" ) tm.assert_series_equal(result, expected) result = algos.value_counts_internal(s, bins=2, sort=False) expected = pd.Series( [2, 2], index=pd.IntervalIndex.from_tuples([(0.996, 2.5), (2.5, 4.0)]), name="count", ) tm.assert_series_equal(result, expected) def test_value_counts_dtypes(self): result = algos.value_counts_internal(np.array([1, 1.0])) assert len(result) == 1 result = algos.value_counts_internal(np.array([1, 1.0]), bins=1) assert len(result) == 1 result = algos.value_counts_internal(pd.Series([1, 1.0, "1"])) # object assert len(result) == 2 msg = "bins argument only works with numeric data" with pytest.raises(TypeError, match=msg): algos.value_counts_internal(np.array(["1", 1], dtype=object), bins=1) def test_value_counts_nat(self): td = pd.Series([np.timedelta64(10000, "ns"), pd.NaT], dtype="timedelta64[ns]") dt = pd.to_datetime(["NaT", "2014-01-01"]) for ser in [td, dt]: vc = algos.value_counts_internal(ser) vc_with_na = algos.value_counts_internal(ser, dropna=False) assert len(vc) == 1 assert len(vc_with_na) == 2 exp_dt = pd.Series({pd.Timestamp("2014-01-01 00:00:00"): 1}, name="count") result_dt = algos.value_counts_internal(dt) tm.assert_series_equal(result_dt, exp_dt) exp_td = pd.Series([1], index=[np.timedelta64(10000, "ns")], name="count") result_td = algos.value_counts_internal(td) tm.assert_series_equal(result_td, exp_td) @pytest.mark.parametrize("dtype", [object, "M8[us]"]) def test_value_counts_datetime_outofbounds(self, dtype): # GH 13663 ser = pd.Series( [ datetime(3000, 1, 1), datetime(5000, 1, 1), datetime(5000, 1, 1), datetime(6000, 1, 1), datetime(3000, 1, 1), datetime(3000, 1, 1), ], dtype=dtype, ) res = ser.value_counts() exp_index = pd.Index( [datetime(3000, 1, 1), datetime(5000, 1, 1), datetime(6000, 1, 1)], dtype=dtype, ) exp = pd.Series([3, 2, 1], index=exp_index, name="count") tm.assert_series_equal(res, exp) def test_categorical(self): s = pd.Series(pd.Categorical(list("aaabbc"))) result = s.value_counts() expected = pd.Series( [3, 2, 1], index=pd.CategoricalIndex(["a", "b", "c"]), name="count" ) tm.assert_series_equal(result, expected, check_index_type=True) # preserve order? s = s.cat.as_ordered() result = s.value_counts() expected.index = expected.index.as_ordered() tm.assert_series_equal(result, expected, check_index_type=True) def test_categorical_nans(self): s = pd.Series(pd.Categorical(list("aaaaabbbcc"))) # 4,3,2,1 (nan) s.iloc[1] = np.nan result = s.value_counts() expected = pd.Series( [4, 3, 2], index=pd.CategoricalIndex(["a", "b", "c"], categories=["a", "b", "c"]), name="count", ) tm.assert_series_equal(result, expected, check_index_type=True) result = s.value_counts(dropna=False) expected = pd.Series( [4, 3, 2, 1], index=pd.CategoricalIndex(["a", "b", "c", np.nan]), name="count", ) tm.assert_series_equal(result, expected, check_index_type=True) # out of order s = pd.Series( pd.Categorical(list("aaaaabbbcc"), ordered=True, categories=["b", "a", "c"]) ) s.iloc[1] = np.nan result = s.value_counts() expected = pd.Series( [4, 3, 2], index=pd.CategoricalIndex( ["a", "b", "c"], categories=["b", "a", "c"], ordered=True, ), name="count", ) tm.assert_series_equal(result, expected, check_index_type=True) result = s.value_counts(dropna=False) expected = pd.Series( [4, 3, 2, 1], index=pd.CategoricalIndex( ["a", "b", "c", np.nan], categories=["b", "a", "c"], ordered=True ), name="count", ) tm.assert_series_equal(result, expected, check_index_type=True) def test_categorical_zeroes(self): # keep the `d` category with 0 s = pd.Series( pd.Categorical(list("bbbaac"), categories=list("abcd"), ordered=True) ) result = s.value_counts() expected = pd.Series( [3, 2, 1, 0], index=pd.Categorical( ["b", "a", "c", "d"], categories=list("abcd"), ordered=True ), name="count", ) tm.assert_series_equal(result, expected, check_index_type=True) def test_value_counts_dropna(self): # https://github.com/pandas-dev/pandas/issues/9443#issuecomment-73719328 tm.assert_series_equal( pd.Series([True, True, False]).value_counts(dropna=True), pd.Series([2, 1], index=[True, False], name="count"), ) tm.assert_series_equal( pd.Series([True, True, False]).value_counts(dropna=False), pd.Series([2, 1], index=[True, False], name="count"), ) tm.assert_series_equal( pd.Series([True] * 3 + [False] * 2 + [None] * 5).value_counts(dropna=True), pd.Series( [3, 2], index=pd.Index([True, False], dtype=object), name="count" ), ) tm.assert_series_equal( pd.Series([True] * 5 + [False] * 3 + [None] * 2).value_counts(dropna=False), pd.Series([5, 3, 2], index=[True, False, None], name="count"), ) tm.assert_series_equal( pd.Series([10.3, 5.0, 5.0]).value_counts(dropna=True), pd.Series([2, 1], index=[5.0, 10.3], name="count"), ) tm.assert_series_equal( pd.Series([10.3, 5.0, 5.0]).value_counts(dropna=False), pd.Series([2, 1], index=[5.0, 10.3], name="count"), ) tm.assert_series_equal( pd.Series([10.3, 5.0, 5.0, None]).value_counts(dropna=True), pd.Series([2, 1], index=[5.0, 10.3], name="count"), ) result = pd.Series([10.3, 10.3, 5.0, 5.0, 5.0, None]).value_counts(dropna=False) expected = pd.Series([3, 2, 1], index=[5.0, 10.3, None], name="count") tm.assert_series_equal(result, expected) @pytest.mark.parametrize("dtype", (np.float64, object, "M8[ns]")) def test_value_counts_normalized(self, dtype): # GH12558 s = pd.Series([1] * 2 + [2] * 3 + [np.nan] * 5) s_typed = s.astype(dtype) result = s_typed.value_counts(normalize=True, dropna=False) expected = pd.Series( [0.5, 0.3, 0.2], index=pd.Series([np.nan, 2.0, 1.0], dtype=dtype), name="proportion", ) tm.assert_series_equal(result, expected) result = s_typed.value_counts(normalize=True, dropna=True) expected = pd.Series( [0.6, 0.4], index=pd.Series([2.0, 1.0], dtype=dtype), name="proportion" ) tm.assert_series_equal(result, expected) def test_value_counts_uint64(self): arr = np.array([2**63], dtype=np.uint64) expected = pd.Series([1], index=[2**63], name="count") result = algos.value_counts_internal(arr) tm.assert_series_equal(result, expected) arr = np.array([-1, 2**63], dtype=object) expected = pd.Series([1, 1], index=[-1, 2**63], name="count") result = algos.value_counts_internal(arr) tm.assert_series_equal(result, expected) def test_value_counts_series(self): # GH#54857 values = np.array([3, 1, 2, 3, 4, np.nan]) result = pd.Series(values).value_counts(bins=3) expected = pd.Series( [2, 2, 1], index=pd.IntervalIndex.from_tuples( [(0.996, 2.0), (2.0, 3.0), (3.0, 4.0)], dtype="interval[float64, right]" ), name="count", ) tm.assert_series_equal(result, expected) def test_value_counts_stability(self): # GH 63155 arr = np.random.default_rng(2).integers(0, 32, 64) result = algos.value_counts_internal(arr, sort=True) value_counts = pd.Series(arr).value_counts(sort=False) expected = value_counts.sort_values(ascending=False, kind="stable") tm.assert_series_equal(result, expected) unstable_sorted = value_counts.sort_values(ascending=False, kind="quicksort") with pytest.raises(AssertionError): tm.assert_series_equal(result, unstable_sorted) class TestDuplicated: def test_duplicated_with_nas(self): keys = np.array([0, 1, np.nan, 0, 2, np.nan], dtype=object) result = algos.duplicated(keys) expected = np.array([False, False, False, True, False, True]) tm.assert_numpy_array_equal(result, expected) result = algos.duplicated(keys, keep="first") expected = np.array([False, False, False, True, False, True]) tm.assert_numpy_array_equal(result, expected) result = algos.duplicated(keys, keep="last") expected = np.array([True, False, True, False, False, False]) tm.assert_numpy_array_equal(result, expected) result = algos.duplicated(keys, keep=False) expected = np.array([True, False, True, True, False, True]) tm.assert_numpy_array_equal(result, expected) keys = np.empty(8, dtype=object) for i, t in enumerate( zip([0, 0, np.nan, np.nan] * 2, [0, np.nan, 0, np.nan] * 2, strict=True) ): keys[i] = t result = algos.duplicated(keys) falses = [False] * 4 trues = [True] * 4 expected = np.array(falses + trues) tm.assert_numpy_array_equal(result, expected) result = algos.duplicated(keys, keep="last") expected = np.array(trues + falses) tm.assert_numpy_array_equal(result, expected) result = algos.duplicated(keys, keep=False) expected = np.array(trues + trues) tm.assert_numpy_array_equal(result, expected) @pytest.mark.parametrize( "case", [ np.array([1, 2, 1, 5, 3, 2, 4, 1, 5, 6]), np.array([1.1, 2.2, 1.1, np.nan, 3.3, 2.2, 4.4, 1.1, np.nan, 6.6]), np.array( [ 1 + 1j, 2 + 2j, 1 + 1j, 5 + 5j, 3 + 3j, 2 + 2j, 4 + 4j, 1 + 1j, 5 + 5j, 6 + 6j, ] ), np.array(["a", "b", "a", "e", "c", "b", "d", "a", "e", "f"], dtype=object), np.array([1, 2**63, 1, 3**5, 10, 2**63, 39, 1, 3**5, 7], dtype=np.uint64), ], ) def test_numeric_object_likes(self, case): exp_first = np.array( [False, False, True, False, False, True, False, True, True, False] ) exp_last = np.array( [True, True, True, True, False, False, False, False, False, False] ) exp_false = exp_first | exp_last res_first = algos.duplicated(case, keep="first") tm.assert_numpy_array_equal(res_first, exp_first) res_last = algos.duplicated(case, keep="last") tm.assert_numpy_array_equal(res_last, exp_last) res_false = algos.duplicated(case, keep=False) tm.assert_numpy_array_equal(res_false, exp_false) # index for idx in [pd.Index(case), pd.Index(case, dtype="category")]: res_first = idx.duplicated(keep="first") tm.assert_numpy_array_equal(res_first, exp_first) res_last = idx.duplicated(keep="last") tm.assert_numpy_array_equal(res_last, exp_last) res_false = idx.duplicated(keep=False) tm.assert_numpy_array_equal(res_false, exp_false) # series for s in [pd.Series(case), pd.Series(case, dtype="category")]: res_first = s.duplicated(keep="first") tm.assert_series_equal(res_first, pd.Series(exp_first)) res_last = s.duplicated(keep="last") tm.assert_series_equal(res_last, pd.Series(exp_last)) res_false = s.duplicated(keep=False) tm.assert_series_equal(res_false, pd.Series(exp_false)) def test_datetime_likes(self): dt = [ "2011-01-01", "2011-01-02", "2011-01-01", "NaT", "2011-01-03", "2011-01-02", "2011-01-04", "2011-01-01", "NaT", "2011-01-06", ] td = [ "1 days", "2 days", "1 days", "NaT", "3 days", "2 days", "4 days", "1 days", "NaT", "6 days", ] cases = [ np.array([pd.Timestamp(d) for d in dt]), np.array([pd.Timestamp(d, tz="US/Eastern") for d in dt]), np.array([pd.Period(d, freq="D") for d in dt]), np.array([np.datetime64(d, "ns") for d in dt]), np.array([pd.Timedelta(d) for d in td]), ] exp_first = np.array( [False, False, True, False, False, True, False, True, True, False] ) exp_last = np.array( [True, True, True, True, False, False, False, False, False, False] ) exp_false = exp_first | exp_last for case in cases: res_first = algos.duplicated(case, keep="first") tm.assert_numpy_array_equal(res_first, exp_first) res_last = algos.duplicated(case, keep="last") tm.assert_numpy_array_equal(res_last, exp_last) res_false = algos.duplicated(case, keep=False) tm.assert_numpy_array_equal(res_false, exp_false) # index for idx in [ pd.Index(case), pd.Index(case, dtype="category"), pd.Index(case, dtype=object), ]: res_first = idx.duplicated(keep="first") tm.assert_numpy_array_equal(res_first, exp_first) res_last = idx.duplicated(keep="last") tm.assert_numpy_array_equal(res_last, exp_last) res_false = idx.duplicated(keep=False) tm.assert_numpy_array_equal(res_false, exp_false) # series for s in [ pd.Series(case), pd.Series(case, dtype="category"), pd.Series(case, dtype=object), ]: res_first = s.duplicated(keep="first") tm.assert_series_equal(res_first, pd.Series(exp_first)) res_last = s.duplicated(keep="last") tm.assert_series_equal(res_last, pd.Series(exp_last)) res_false = s.duplicated(keep=False) tm.assert_series_equal(res_false, pd.Series(exp_false)) @pytest.mark.parametrize("case", [pd.Index([1, 2, 3]), pd.RangeIndex(0, 3)]) def test_unique_index(self, case): assert case.is_unique is True tm.assert_numpy_array_equal(case.duplicated(), np.array([False, False, False])) @pytest.mark.parametrize( "arr, uniques", [ ( [(0, 0), (0, 1), (1, 0), (1, 1), (0, 0), (0, 1), (1, 0), (1, 1)], [(0, 0), (0, 1), (1, 0), (1, 1)], ), ( [("b", "c"), ("a", "b"), ("a", "b"), ("b", "c")], [("b", "c"), ("a", "b")], ), ([("a", 1), ("b", 2), ("a", 3), ("a", 1)], [("a", 1), ("b", 2), ("a", 3)]), ], ) def test_unique_tuples(self, arr, uniques): # https://github.com/pandas-dev/pandas/issues/16519 expected = np.empty(len(uniques), dtype=object) expected[:] = uniques msg = ( r"unique requires a Series, Index, ExtensionArray, np.ndarray " r"or NumpyExtensionArray got list" ) with pytest.raises(TypeError, match=msg): # GH#52986 pd.unique(arr) res = pd.unique(com.asarray_tuplesafe(arr, dtype=object)) tm.assert_numpy_array_equal(res, expected) @pytest.mark.parametrize( "array,expected", [ ( [1 + 1j, 0, 1, 1j, 1 + 2j, 1 + 2j], np.array([(1 + 1j), 0j, (1 + 0j), 1j, (1 + 2j)], dtype=complex), ) ], ) def test_unique_complex_numbers(self, array, expected): # GH 17927 msg = ( r"unique requires a Series, Index, ExtensionArray, np.ndarray " r"or NumpyExtensionArray got list" ) with pytest.raises(TypeError, match=msg): # GH#52986 pd.unique(array) res = pd.unique(np.array(array)) tm.assert_numpy_array_equal(res, expected) class TestHashTable: @pytest.mark.parametrize( "htable, data", [ ( ht.PyObjectHashTable, np.array([f"foo_{i}" for i in range(1000)], dtype=object), ), ( ht.StringHashTable, np.array([f"foo_{i}" for i in range(1000)], dtype=object), ), (ht.Float64HashTable, np.arange(1000, dtype=np.float64)), (ht.Int64HashTable, np.arange(1000, dtype=np.int64)), (ht.UInt64HashTable, np.arange(1000, dtype=np.uint64)), ], ) def test_hashtable_unique(self, htable, data, writable): # output of maker has guaranteed unique elements s = pd.Series(data, dtype=data.dtype) if htable == ht.Float64HashTable: # add NaN for float column s.loc[500] = np.nan elif htable == ht.PyObjectHashTable: # use different NaN types for object column s.loc[500:502] = [np.nan, None, pd.NaT] # create duplicated selection s_duplicated = s.sample(frac=3, replace=True).reset_index(drop=True) s_duplicated.values.setflags(write=writable) # drop_duplicates has own cython code (hash_table_func_helper.pxi) # and is tested separately; keeps first occurrence like ht.unique() expected_unique = s_duplicated.drop_duplicates(keep="first").values result_unique = htable().unique(s_duplicated.values) tm.assert_numpy_array_equal(result_unique, expected_unique) # test return_inverse=True # reconstruction can only succeed if the inverse is correct result_unique, result_inverse = htable().unique( s_duplicated.values, return_inverse=True ) tm.assert_numpy_array_equal(result_unique, expected_unique) reconstr = result_unique[result_inverse] tm.assert_numpy_array_equal(reconstr, s_duplicated.values) @pytest.mark.parametrize( "htable, data", [ ( ht.PyObjectHashTable, np.array([f"foo_{i}" for i in range(1000)], dtype=object), ), ( ht.StringHashTable, np.array([f"foo_{i}" for i in range(1000)], dtype=object), ), (ht.Float64HashTable, np.arange(1000, dtype=np.float64)), (ht.Int64HashTable, np.arange(1000, dtype=np.int64)), (ht.UInt64HashTable, np.arange(1000, dtype=np.uint64)), ], ) def test_hashtable_factorize(self, htable, writable, data): # output of maker has guaranteed unique elements s = pd.Series(data, dtype=data.dtype) if htable == ht.Float64HashTable: # add NaN for float column s.loc[500] = np.nan elif htable == ht.PyObjectHashTable: # use different NaN types for object column s.loc[500:502] = [np.nan, None, pd.NaT] # create duplicated selection s_duplicated = s.sample(frac=3, replace=True).reset_index(drop=True) s_duplicated.values.setflags(write=writable) na_mask = s_duplicated.isna().values result_unique, result_inverse = htable().factorize(s_duplicated.values) # drop_duplicates has own cython code (hash_table_func_helper.pxi) # and is tested separately; keeps first occurrence like ht.factorize() # since factorize removes all NaNs, we do the same here expected_unique = s_duplicated.dropna().drop_duplicates().values tm.assert_numpy_array_equal(result_unique, expected_unique) # reconstruction can only succeed if the inverse is correct. Since # factorize removes the NaNs, those have to be excluded here as well result_reconstruct = result_unique[result_inverse[~na_mask]] expected_reconstruct = s_duplicated.dropna().values tm.assert_numpy_array_equal(result_reconstruct, expected_reconstruct) class TestRank: @pytest.mark.parametrize( "arr", [ [np.nan, np.nan, 5.0, 5.0, 5.0, np.nan, 1, 2, 3, np.nan], [4.0, np.nan, 5.0, 5.0, 5.0, np.nan, 1, 2, 4.0, np.nan], ], ) def test_scipy_compat(self, arr): sp_stats = pytest.importorskip("scipy.stats") arr = np.array(arr) mask = ~np.isfinite(arr) result = libalgos.rank_1d(arr) arr[mask] = np.inf exp = sp_stats.rankdata(arr) exp[mask] = np.nan tm.assert_almost_equal(result, exp) def test_basic(self, writable, any_int_numpy_dtype): exp = np.array([1, 2], dtype=np.float64) data = np.array([1, 100], dtype=any_int_numpy_dtype) data.setflags(write=writable) ser = pd.Series(data) result = algos.rank(ser) tm.assert_numpy_array_equal(result, exp) @pytest.mark.parametrize("dtype", [np.float64, np.uint64]) def test_uint64_overflow(self, dtype): exp = np.array([1, 2], dtype=np.float64) s = pd.Series([1, 2**63], dtype=dtype) tm.assert_numpy_array_equal(algos.rank(s), exp) @pytest.mark.parametrize("method", ["average", "min", "max"]) def test_rank_tiny_values(self, method): # GH62036: regression test for ranking with tiny float values exp = np.array([4.0, 1.0, 3.0, np.nan, 2.0], dtype=np.float64) s = pd.Series( [5.4954145e29, -9.791984e-21, 9.3715776e-26, pd.NA, 1.8790257e-28], dtype="Float64", ) s = s.astype(object) result = algos.rank(s, method=method) tm.assert_numpy_array_equal(result, exp) def test_too_many_ndims(self): arr = np.array([[[1, 2, 3], [4, 5, 6], [7, 8, 9]]]) msg = "Array with ndim > 2 is not supported" with pytest.raises(TypeError, match=msg): algos.rank(arr) @pytest.mark.single_cpu def test_pct_max_many_rows(self): # GH 18271 values = np.arange(2**24 + 1) result = algos.rank(values, pct=True).max() assert result == 1 values = np.arange(2**25 + 2).reshape(2**24 + 1, 2) result = algos.rank(values, pct=True).max() assert result == 1 class TestIsMonotonic: @pytest.mark.parametrize( "arr, expected", [ ([1, 2, 3], (True, False, True)), ([3, 2, 1], (False, True, True)), ([1, 2, 2, 3], (True, False, False)), ([3, 2, 2, 1], (False, True, False)), ([5, 5, 5], (True, True, False)), ([1, 3, 2], (False, False, False)), ([7], (True, True, True)), ([], (True, True, True)), ], ) def test_numeric(self, arr, expected, any_real_numpy_dtype): # https://github.com/pandas-dev/pandas/pull/65803 values = np.array(arr, dtype=any_real_numpy_dtype) assert algos.is_monotonic(values) == expected def test_float16(self): # https://github.com/pandas-dev/pandas/pull/65803 # float16 is not supported by libalgos.is_monotonic directly; # _ensure_data converts it to float64 values = np.array([1, 2, 3], dtype="float16") assert algos.is_monotonic(values) == (True, False, True) assert algos.is_monotonic(values[::-1]) == (False, True, True) def test_bool(self): # https://github.com/pandas-dev/pandas/pull/65803 assert algos.is_monotonic(np.array([False, True, True])) == (True, False, False) assert algos.is_monotonic(np.array([True, False])) == (False, True, True) def test_object(self): # https://github.com/pandas-dev/pandas/pull/65803 values = np.array(["a", "b", "c"], dtype=object) assert algos.is_monotonic(values) == (True, False, True) @pytest.mark.parametrize("dtype", ["datetime64[ns]", "timedelta64[ns]"]) def test_datetimelike(self, dtype): # https://github.com/pandas-dev/pandas/pull/65803 values = np.array([1, 2, 3], dtype="int64").view(dtype) assert algos.is_monotonic(values) == (True, False, True) assert algos.is_monotonic(values[::-1]) == (False, True, True) def test_complex_raises(self): # https://github.com/pandas-dev/pandas/pull/65803 # complex is not orderable values = np.array([1 + 0j, 2 + 0j], dtype="complex128") with pytest.raises(TypeError, match="No matching signature found"): algos.is_monotonic(values) class TestMode: def test_no_mode(self): exp = pd.Series([], dtype=np.float64, index=pd.Index([], dtype=int)) result, _ = algos.mode(np.array([])) tm.assert_numpy_array_equal(result, exp.values) def test_mode_single(self, any_real_numpy_dtype): # GH 15714 exp_single = [1] data_single = [1] exp_multi = [1] data_multi = [1, 1] ser = pd.Series(data_single, dtype=any_real_numpy_dtype) exp = pd.Series(exp_single, dtype=any_real_numpy_dtype) result, _ = algos.mode(ser.values) tm.assert_numpy_array_equal(result, exp.values) tm.assert_series_equal(ser.mode(), exp) ser = pd.Series(data_multi, dtype=any_real_numpy_dtype) exp = pd.Series(exp_multi, dtype=any_real_numpy_dtype) result, _ = algos.mode(ser.values) tm.assert_numpy_array_equal(result, exp.values) tm.assert_series_equal(ser.mode(), exp) def test_mode_obj_int(self): exp = pd.Series([1], dtype=int) result, _ = algos.mode(exp.values) tm.assert_numpy_array_equal(result, exp.values) exp = pd.Series(["a", "b", "c"], dtype=object) result, _ = algos.mode(exp.values) tm.assert_numpy_array_equal(result, exp.values) def test_number_mode(self, any_real_numpy_dtype): exp_single = [1] data_single = [1] * 5 + [2] * 3 exp_multi = [1, 3] data_multi = [1] * 5 + [2] * 3 + [3] * 5 ser = pd.Series(data_single, dtype=any_real_numpy_dtype) exp = pd.Series(exp_single, dtype=any_real_numpy_dtype) result, _ = algos.mode(ser.values) tm.assert_numpy_array_equal(result, exp.values) tm.assert_series_equal(ser.mode(), exp) ser = pd.Series(data_multi, dtype=any_real_numpy_dtype) exp = pd.Series(exp_multi, dtype=any_real_numpy_dtype) result, _ = algos.mode(ser.values) tm.assert_numpy_array_equal(result, exp.values) tm.assert_series_equal(ser.mode(), exp) def test_strobj_mode(self): exp = ["b"] data = ["a"] * 2 + ["b"] * 3 ser = pd.Series(data, dtype="c") exp = pd.Series(exp, dtype="c") result, _ = algos.mode(ser.values) tm.assert_numpy_array_equal(result, exp.values) tm.assert_series_equal(ser.mode(), exp) @pytest.mark.parametrize("dt", [str, object]) def test_strobj_multi_char(self, dt, using_infer_string): exp = ["bar"] data = ["foo"] * 2 + ["bar"] * 3 ser = pd.Series(data, dtype=dt) exp = pd.Series(exp, dtype=dt) result, _ = algos.mode(ser.values) if using_infer_string and dt is str: tm.assert_extension_array_equal(result, exp.values) else: tm.assert_numpy_array_equal(result, exp.values) tm.assert_series_equal(ser.mode(), exp) def test_datelike_mode(self): exp = pd.Series(["1900-05-03", "2011-01-03", "2013-01-02"], dtype="M8[ns]") ser = pd.Series(["2011-01-03", "2013-01-02", "1900-05-03"], dtype="M8[ns]") tm.assert_extension_array_equal(algos.mode(ser.values), exp._values) tm.assert_series_equal(ser.mode(), exp) exp = pd.Series(["2011-01-03", "2013-01-02"], dtype="M8[ns]") ser = pd.Series( ["2011-01-03", "2013-01-02", "1900-05-03", "2011-01-03", "2013-01-02"], dtype="M8[ns]", ) tm.assert_extension_array_equal(algos.mode(ser.values), exp._values) tm.assert_series_equal(ser.mode(), exp) def test_timedelta_mode(self): exp = pd.Series(["-1 days", "0 days", "1 days"], dtype="timedelta64[ns]") ser = pd.Series(["1 days", "-1 days", "0 days"], dtype="timedelta64[ns]") tm.assert_extension_array_equal(algos.mode(ser.values), exp._values) tm.assert_series_equal(ser.mode(), exp) exp = pd.Series(["2 min", "1 day"], dtype="timedelta64[ns]") ser = pd.Series( ["1 day", "1 day", "-1 day", "-1 day 2 min", "2 min", "2 min"], dtype="timedelta64[ns]", ) tm.assert_extension_array_equal(algos.mode(ser.values), exp._values) tm.assert_series_equal(ser.mode(), exp) def test_mixed_dtype(self): exp = pd.Series(["foo"], dtype=object) ser = pd.Series([1, "foo", "foo"]) result, _ = algos.mode(ser.values) tm.assert_numpy_array_equal(result, exp.values) tm.assert_series_equal(ser.mode(), exp) def test_uint64_overflow(self): exp = pd.Series([2**63], dtype=np.uint64) ser = pd.Series([1, 2**63, 2**63], dtype=np.uint64) result, _ = algos.mode(ser.values) tm.assert_numpy_array_equal(result, exp.values) tm.assert_series_equal(ser.mode(), exp) exp = pd.Series([1, 2**63], dtype=np.uint64) ser = pd.Series([1, 2**63], dtype=np.uint64) result, _ = algos.mode(ser.values) tm.assert_numpy_array_equal(result, exp.values) tm.assert_series_equal(ser.mode(), exp) def test_categorical(self): c = pd.Categorical([1, 2]) exp = c res = pd.Series(c).mode()._values tm.assert_categorical_equal(res, exp) c = pd.Categorical([1, "a", "a"]) exp = pd.Categorical(["a"], categories=[1, "a"]) res = pd.Series(c).mode()._values tm.assert_categorical_equal(res, exp) c = pd.Categorical([1, 1, 2, 3, 3]) exp = pd.Categorical([1, 3], categories=[1, 2, 3]) res = pd.Series(c).mode()._values tm.assert_categorical_equal(res, exp) def test_index(self): idx = pd.Index([1, 2, 3]) exp = pd.Series([1, 2, 3], dtype=np.int64) result, _ = algos.mode(idx) tm.assert_numpy_array_equal(result, exp.values) idx = pd.Index([1, "a", "a"]) exp = pd.Series(["a"], dtype=object) result, _ = algos.mode(idx) tm.assert_numpy_array_equal(result, exp.values) idx = pd.Index([1, 1, 2, 3, 3]) exp = pd.Series([1, 3], dtype=np.int64) result, _ = algos.mode(idx) tm.assert_numpy_array_equal(result, exp.values) idx = pd.Index( ["1 day", "1 day", "-1 day", "-1 day 2 min", "2 min", "2 min"], dtype="timedelta64[ns]", ) with pytest.raises(AttributeError, match="TimedeltaIndex"): # algos.mode expects Arraylike, does *not* unwrap TimedeltaIndex algos.mode(idx) def test_ser_mode_with_name(self): # GH 46737 ser = pd.Series([1, 1, 3], name="foo") result = ser.mode() expected = pd.Series([1], name="foo") tm.assert_series_equal(result, expected) @pytest.mark.parametrize("subtype", ["float16", "float32", "float64"]) def test_mode_sparse_float(self, subtype): # GH#68421 SparseDtype has no itemsize for _ensure_data to read ser = pd.Series(SparseArray(np.array([1, 2, 2, 3], dtype=subtype))) result = ser.mode() expected = pd.Series([2], dtype=ser.dtype) tm.assert_series_equal(result, expected) def test_mode_sparse_bool(self): # GH#68421 bool goes through _ensure_data's non-ndarray bool branch arr = SparseArray([True, True, True, False]) result = pd.Series(arr).mode() tm.assert_series_equal(result, pd.Series([True], dtype=arr.dtype)) @pytest.mark.parametrize("dropna, expected", [(True, 1.0), (False, np.nan)]) def test_mode_sparse_counts_fill_value(self, dropna, expected): # GH#68421 the fill value is a value like any other, so the implicit # entries count toward the mode arr = SparseArray([np.nan, np.nan, np.nan, 1.0, 1.0]) result = pd.Series(arr).mode(dropna=dropna) tm.assert_series_equal(result, pd.Series([expected], dtype=arr.dtype)) class TestDiff: @pytest.mark.parametrize("dtype", ["M8[ns]", "m8[ns]"]) def test_diff_datetimelike_nat(self, dtype): # NaT - NaT is NaT, not 0 arr = np.arange(12).astype(np.int64).view(dtype).reshape(3, 4) arr[:, 2] = arr.dtype.type("NaT", "ns") result = algos.diff(arr, 1, axis=0) expected = np.ones(arr.shape, dtype="timedelta64[ns]") * 4 expected[:, 2] = np.timedelta64("NaT", "ns") expected[0, :] = np.timedelta64("NaT", "ns") tm.assert_numpy_array_equal(result, expected) result = algos.diff(arr.T, 1, axis=1) tm.assert_numpy_array_equal(result, expected.T) def test_diff_ea_axis(self): dta = pd.date_range("2016-01-01", periods=3, tz="US/Pacific")._data msg = "cannot diff DatetimeArray on axis=1" with pytest.raises(ValueError, match=msg): algos.diff(dta, 1, axis=1) @pytest.mark.parametrize("dtype", ["int8", "int16"]) def test_diff_low_precision_int(self, dtype): arr = np.array([0, 1, 1, 0, 0], dtype=dtype) result = algos.diff(arr, 1) expected = np.array([np.nan, 1, 0, -1, 0], dtype="float32") tm.assert_numpy_array_equal(result, expected) @pytest.mark.parametrize("op", [np.array, pd.array]) def test_union_with_duplicates(op): # GH#36289 lvals = op([3, 1, 3, 4]) rvals = op([2, 3, 1, 1]) expected = op([3, 3, 1, 1, 4, 2]) if isinstance(expected, np.ndarray): result = algos.union_with_duplicates(lvals, rvals) tm.assert_numpy_array_equal(result, expected) else: result = algos.union_with_duplicates(lvals, rvals) tm.assert_extension_array_equal(result, expected) @pytest.mark.parametrize( "codes, expected", [ ([0, 1, 0, 0, 1], [0, 0, 1, 2, 1]), ([0, 1, 2], [0, 0, 0]), ([3, 3, 3], [0, 1, 2]), ([], []), ([5], [0]), ([-1, 0, -1], [0, 0, 1]), ], ) def test_occurrence_rank(codes, expected): # GH#67446 result = algos.occurrence_rank(np.array(codes, dtype=np.intp)) tm.assert_numpy_array_equal(result, np.array(expected, dtype=np.intp))