From 649a898e52f131834ca2fa9c28dd828a1d14dccb Mon Sep 17 00:00:00 2001 From: microslaw Date: Fri, 6 Jun 2025 21:52:42 +0000 Subject: [PATCH 1/3] update docs --- doc/source/whatsnew/v3.0.0.rst | 1 + pandas/core/generic.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/doc/source/whatsnew/v3.0.0.rst b/doc/source/whatsnew/v3.0.0.rst index 9100ce0cdc990..6680925b0cbb6 100644 --- a/doc/source/whatsnew/v3.0.0.rst +++ b/doc/source/whatsnew/v3.0.0.rst @@ -888,6 +888,7 @@ Other - Bug in :meth:`DataFrame.query` where using duplicate column names led to a ``TypeError``. (:issue:`59950`) - Bug in :meth:`DataFrame.query` which raised an exception or produced incorrect results when expressions contained backtick-quoted column names containing the hash character ``#``, backticks, or characters that fall outside the ASCII range (U+0001..U+007F). (:issue:`59285`) (:issue:`49633`) - Bug in :meth:`DataFrame.query` which raised an exception when querying integer column names using backticks. (:issue:`60494`) +- Bug in :meth:`DataFrame.sample` with ``replace=False`` and ``(n * max(weights) / sum(weights)) > 1``, the method would return biased results. Now raises ``ValueError``. (:issue:`61516`) - Bug in :meth:`DataFrame.shift` where passing a ``freq`` on a DataFrame with no columns did not shift the index correctly. (:issue:`60102`) - Bug in :meth:`DataFrame.sort_index` when passing ``axis="columns"`` and ``ignore_index=True`` and ``ascending=False`` not returning a :class:`RangeIndex` columns (:issue:`57293`) - Bug in :meth:`DataFrame.sort_values` where sorting by a column explicitly named ``None`` raised a ``KeyError`` instead of sorting by the column as expected. (:issue:`61512`) diff --git a/pandas/core/generic.py b/pandas/core/generic.py index 8aae4609b1833..ec5e105b24020 100644 --- a/pandas/core/generic.py +++ b/pandas/core/generic.py @@ -5815,6 +5815,8 @@ def sample( If weights do not sum to 1, they will be normalized to sum to 1. Missing values in the weights column will be treated as zero. Infinite values not allowed. + When replace = False will not allow ``(n * max(weights) / sum(weights)) > 1``, + in order to avoid biased results. random_state : int, array-like, BitGenerator, np.random.RandomState, np.random.Generator, optional If int, array-like, or BitGenerator, seed for random number generator. If np.random.RandomState or np.random.Generator, use as given. From e5f9a071ece815cecdcf899aa4e1608ddb805473 Mon Sep 17 00:00:00 2001 From: microslaw Date: Fri, 6 Jun 2025 21:53:01 +0000 Subject: [PATCH 2/3] implement the exception --- pandas/core/sample.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pandas/core/sample.py b/pandas/core/sample.py index 4f12563e3c5e2..89572a371b7ee 100644 --- a/pandas/core/sample.py +++ b/pandas/core/sample.py @@ -150,6 +150,14 @@ def sample( else: raise ValueError("Invalid weights: weights sum to zero") + + is_max_weight_dominating = size * max(weights) > weight_sum + if is_max_weight_dominating and not replace: + raise ValueError( + "Invalid weights: If `replace`=False," + " unit probabilities have to be less than 1" + ) + return random_state.choice(obj_len, size=size, replace=replace, p=weights).astype( np.intp, copy=False ) From 18b287640a98e749947c87fb23a7d9add811e102 Mon Sep 17 00:00:00 2001 From: microslaw Date: Fri, 6 Jun 2025 21:53:54 +0000 Subject: [PATCH 3/3] add test --- pandas/tests/frame/methods/test_sample.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/pandas/tests/frame/methods/test_sample.py b/pandas/tests/frame/methods/test_sample.py index a9d56cbfd2b46..14143e33a4510 100644 --- a/pandas/tests/frame/methods/test_sample.py +++ b/pandas/tests/frame/methods/test_sample.py @@ -137,6 +137,17 @@ def test_sample_inf_weights(self, obj): with pytest.raises(ValueError, match=msg): obj.sample(n=3, weights=weights_with_ninf) + def test_sample_replacement_weight_sum(self, obj): + # GH#61516 + weights_large_total = [1] * 10 + weights_large_total[0] = 100 + msg = ( + "Invalid weights: If `replace`=False," + " unit probabilities have to be less than 1" + ) + with pytest.raises(ValueError, match=msg): + obj.sample(n=3, weights=weights_large_total, replace=False) + def test_sample_zero_weights(self, obj): # All zeros raises errors