From 97c32fda5e429fce11a56e2088fb9ea3584d5dc3 Mon Sep 17 00:00:00 2001
From: Brock Mendel <jbrockmendel@gmail.com>
Date: Sun, 19 Nov 2017 15:47:06 -0800
Subject: [PATCH 01/19] implement tzawareness_compat for datetimeIndex
---
pandas/core/indexes/datetimes.py | 17 +++++++++-
pandas/core/internals.py | 18 ++++++++---
.../tests/indexes/datetimes/test_datetime.py | 32 +++++++++++++++++++
3 files changed, 62 insertions(+), 5 deletions(-)
diff --git a/pandas/core/indexes/datetimes.py b/pandas/core/indexes/datetimes.py
index ba96979435f81..0a957265c556f 100644
--- a/pandas/core/indexes/datetimes.py
+++ b/pandas/core/indexes/datetimes.py
@@ -11,7 +11,7 @@
from pandas.core.dtypes.common import (
_NS_DTYPE, _INT64_DTYPE,
- is_object_dtype, is_datetime64_dtype,
+ is_object_dtype, is_datetime64_dtype, is_datetime64tz_dtype,
is_datetimetz, is_dtype_equal,
is_integer, is_float,
is_integer_dtype,
@@ -104,6 +104,7 @@ def _dt_index_cmp(opname, nat_result=False):
"""
def wrapper(self, other):
+ self._assert_tzawareness_compat(other)
func = getattr(super(DatetimeIndex, self), opname)
if (isinstance(other, datetime) or
isinstance(other, compat.string_types)):
@@ -649,6 +650,20 @@ def _simple_new(cls, values, name=None, freq=None, tz=None,
result._reset_identity()
return result
+ def _assert_tzawareness_compat(self, other):
+ # adapted from _Timestamp._assert_tzawareness_compat
+ other_tz = getattr(other, 'tzinfo', None)
+ if is_datetime64tz_dtype(other):
+ # Get tzinfo from Series dtype
+ other_tz = other.dtype.tz
+ if self.tz is None:
+ if other_tz is not None:
+ raise TypeError('Cannot compare tz-naive and tz-aware '
+ 'datetime-like objects.')
+ elif other_tz is None:
+ raise TypeError('Cannot compare tz-naive and tz-aware '
+ 'datetime-like objects')
+
@property
def tzinfo(self):
"""
diff --git a/pandas/core/internals.py b/pandas/core/internals.py
index 665f9ff8eb7a0..65393f9f0338d 100644
--- a/pandas/core/internals.py
+++ b/pandas/core/internals.py
@@ -3474,10 +3474,20 @@ def replace_list(self, src_list, dest_list, inplace=False, regex=False,
# figure out our mask a-priori to avoid repeated replacements
values = self.as_matrix()
- def comp(s):
- if isna(s):
- return isna(values)
- return _maybe_compare(values, getattr(s, 'asm8', s), operator.eq)
+ if is_datetime64tz_dtype(self):
+ # need to be careful not to compare tznaive to tzaware
+ tz = values.dtype.tz
+ def comp(s):
+ if isna(s):
+ return isna(values)
+ return _maybe_compare(values, s, operator.eq)
+ # TODO: Is just not-converting `s` the right thing to do here?
+ else:
+ def comp(s):
+ if isna(s):
+ return isna(values)
+ return _maybe_compare(values, getattr(s, 'asm8', s),
+ operator.eq)
masks = [comp(s) for i, s in enumerate(src_list)]
diff --git a/pandas/tests/indexes/datetimes/test_datetime.py b/pandas/tests/indexes/datetimes/test_datetime.py
index 36f691903d233..95229264c6f36 100644
--- a/pandas/tests/indexes/datetimes/test_datetime.py
+++ b/pandas/tests/indexes/datetimes/test_datetime.py
@@ -1,3 +1,5 @@
+import operator
+
import pytest
import numpy as np
@@ -223,6 +225,36 @@ def test_append_join_nondatetimeindex(self):
# it works
rng.join(idx, how='outer')
+ def test_comparison_tzawareness_compat(self):
+ # GH#18162
+ dr = pd.date_range('2016-01-01', periods=6)
+ dz = dr.tz_localize('US/Pacific')
+
+ ops = [operator.eq, operator.ne,
+ operator.gt, operator.ge,
+ operator.lt, operator.le]
+
+ for left, right in [(dr, dz), (dz, dr)]:
+ for op in ops:
+ with pytest.raises(TypeError):
+ op(left, right)
+
+ # Check that there isn't a problem aware-aware and naive-naive do not
+ # raise
+ assert (dr == dr).all()
+ assert (dz == dz).all()
+
+ naive_series = pd.Series(dr)
+ aware_series = pd.Series(dz)
+ for op in ops:
+ with pytest.raises(TypeError):
+ op(dz, naive_series)
+ with pytest.raises(TypeError):
+ op(dr, aware_series)
+
+ # TODO: implement _assert_tzawareness_compat for the reverse
+ # comparison with the Series on the left-hand side
+
def test_comparisons_coverage(self):
rng = date_range('1/1/2000', periods=10)
From f6a0e644353f8e2afd6147342d6aa824ce0b61c5 Mon Sep 17 00:00:00 2001
From: Brock Mendel <jbrockmendel@gmail.com>
Date: Sun, 19 Nov 2017 16:20:08 -0800
Subject: [PATCH 02/19] convert before asserting tzawareness_compat
---
pandas/core/indexes/datetimes.py | 9 ++++++---
1 file changed, 6 insertions(+), 3 deletions(-)
diff --git a/pandas/core/indexes/datetimes.py b/pandas/core/indexes/datetimes.py
index b4f76f5244379..322962ff1185f 100644
--- a/pandas/core/indexes/datetimes.py
+++ b/pandas/core/indexes/datetimes.py
@@ -104,10 +104,12 @@ def _dt_index_cmp(opname, nat_result=False):
"""
def wrapper(self, other):
- self._assert_tzawareness_compat(other)
func = getattr(super(DatetimeIndex, self), opname)
- if (isinstance(other, datetime) or
- isinstance(other, compat.string_types)):
+ if isinstance(other, (datetime, compat.string_types)):
+ if isinstance(other, compat.string_types):
+ # We need to convert in order to assert tzawareness
+ other = Timestamp(other)
+ self._assert_tzawareness_compat(other)
other = _to_m8(other, tz=self.tz)
result = func(other)
if isna(other):
@@ -117,6 +119,7 @@ def wrapper(self, other):
other = DatetimeIndex(other)
elif not isinstance(other, (np.ndarray, Index, ABCSeries)):
other = _ensure_datetime64(other)
+ self._assert_tzawareness_compat(other)
result = func(np.asarray(other))
result = _values_from_object(result)
From 28f88965ba19268c740dbf265e2fcecfab690e99 Mon Sep 17 00:00:00 2001
From: Brock Mendel <jbrockmendel@gmail.com>
Date: Fri, 24 Nov 2017 11:14:13 -0800
Subject: [PATCH 03/19] Update test to disallow tzaware vs tznaive comparison
---
pandas/tests/series/test_indexing.py | 14 ++++++++++++++
1 file changed, 14 insertions(+)
diff --git a/pandas/tests/series/test_indexing.py b/pandas/tests/series/test_indexing.py
index d141b378fe214..25966abb697de 100644
--- a/pandas/tests/series/test_indexing.py
+++ b/pandas/tests/series/test_indexing.py
@@ -449,6 +449,13 @@ def test_getitem_setitem_datetimeindex(self):
lb = "1990-01-01 04:00:00"
rb = "1990-01-01 07:00:00"
+ with pytest.raises(TypeError):
+ # tznaive vs tzaware comparison is invalid
+ # see GH#18376, GH#18162
+ ts[(ts.index >= lb) & (ts.index <= rb)]
+
+ lb = "1990-01-01 04:00:00-0500"
+ rb = "1990-01-01 07:00:00-0500"
result = ts[(ts.index >= lb) & (ts.index <= rb)]
expected = ts[4:8]
assert_series_equal(result, expected)
@@ -474,6 +481,13 @@ def test_getitem_setitem_datetimeindex(self):
lb = datetime(1990, 1, 1, 4)
rb = datetime(1990, 1, 1, 7)
+ with pytest.raises(TypeError):
+ # tznaive vs tzaware comparison is invalid
+ # see GH#18376, GH#18162
+ ts[(ts.index >= lb) & (ts.index <= rb)]
+
+ lb = rng.tzinfo.localize(datetime(1990, 1, 1, 4))
+ rb = rng.tzinfo.localize(datetime(1990, 1, 1, 7))
result = ts[(ts.index >= lb) & (ts.index <= rb)]
expected = ts[4:8]
assert_series_equal(result, expected)
From 0f2287c4f9c69742b8f3a41052a42bf82cd3226a Mon Sep 17 00:00:00 2001
From: Brock Mendel <jbrockmendel@gmail.com>
Date: Fri, 24 Nov 2017 13:29:41 -0800
Subject: [PATCH 04/19] flake8 fixup
---
pandas/core/internals.py | 1 -
1 file changed, 1 deletion(-)
diff --git a/pandas/core/internals.py b/pandas/core/internals.py
index f786a9ad9396b..9ea8f72ec9efb 100644
--- a/pandas/core/internals.py
+++ b/pandas/core/internals.py
@@ -3488,7 +3488,6 @@ def replace_list(self, src_list, dest_list, inplace=False, regex=False,
if is_datetime64tz_dtype(self):
# need to be careful not to compare tznaive to tzaware
- tz = values.dtype.tz
def comp(s):
if isna(s):
return isna(values)
From 171238d9140b14c6089703ad8e11fb18f854b8aa Mon Sep 17 00:00:00 2001
From: Brock Mendel <jbrockmendel@gmail.com>
Date: Sat, 25 Nov 2017 09:13:11 -0800
Subject: [PATCH 05/19] Fix typo
---
pandas/tests/test_base.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pandas/tests/test_base.py b/pandas/tests/test_base.py
index 31f4ca146040e..9a608cca96319 100644
--- a/pandas/tests/test_base.py
+++ b/pandas/tests/test_base.py
@@ -114,7 +114,7 @@ def __init__(self, obj):
def setup_method(self, method):
pass
- def test_invalida_delgation(self):
+ def test_invalid_delegation(self):
# these show that in order for the delegation to work
# the _delegate_* methods need to be overriden to not raise a TypeError
From e95d75ec52949767155088c23116759e759a3107 Mon Sep 17 00:00:00 2001
From: Brock Mendel <jbrockmendel@gmail.com>
Date: Sat, 25 Nov 2017 09:13:35 -0800
Subject: [PATCH 06/19] Move part of test per reviewer request
---
pandas/tests/indexes/test_base.py | 22 ++++++++++++++++++++++
1 file changed, 22 insertions(+)
diff --git a/pandas/tests/indexes/test_base.py b/pandas/tests/indexes/test_base.py
index c55f53601848c..e93df7231dc4c 100644
--- a/pandas/tests/indexes/test_base.py
+++ b/pandas/tests/indexes/test_base.py
@@ -2159,6 +2159,28 @@ def test_intersect_str_dates(self):
assert len(res) == 0
+ def test_comparison_tzawareness_compat(self):
+ # GH#18162
+ dr = pd.date_range('2016-01-01', periods=6)
+ dz = dr.tz_localize('US/Pacific')
+
+ ops = [operator.eq, operator.ne,
+ operator.gt, operator.ge,
+ operator.lt, operator.le]
+
+ # Check that there isn't a problem aware-aware and naive-naive do not
+ # raise
+ naive_series = Series(dr)
+ aware_series = Series(dz)
+ for op in ops:
+ with pytest.raises(TypeError):
+ op(dz, naive_series)
+ with pytest.raises(TypeError):
+ op(dr, aware_series)
+
+ # TODO: implement _assert_tzawareness_compat for the reverse
+ # comparison with the Series on the left-hand side
+
class TestIndexUtils(object):
From ba65aafd95917cbd7a1b11718e486b4fee91a09f Mon Sep 17 00:00:00 2001
From: Brock Mendel <jbrockmendel@gmail.com>
Date: Sat, 25 Nov 2017 09:14:48 -0800
Subject: [PATCH 07/19] update test to include scalar comparisons
---
.../tests/indexes/datetimes/test_datetime.py | 33 +++++++++++++------
1 file changed, 23 insertions(+), 10 deletions(-)
diff --git a/pandas/tests/indexes/datetimes/test_datetime.py b/pandas/tests/indexes/datetimes/test_datetime.py
index 5c10f4727596f..80bc7f184adc3 100644
--- a/pandas/tests/indexes/datetimes/test_datetime.py
+++ b/pandas/tests/indexes/datetimes/test_datetime.py
@@ -269,16 +269,29 @@ def test_comparison_tzawareness_compat(self):
assert (dr == dr).all()
assert (dz == dz).all()
- naive_series = pd.Series(dr)
- aware_series = pd.Series(dz)
- for op in ops:
- with pytest.raises(TypeError):
- op(dz, naive_series)
- with pytest.raises(TypeError):
- op(dr, aware_series)
-
- # TODO: implement _assert_tzawareness_compat for the reverse
- # comparison with the Series on the left-hand side
+ # Check comparisons against datetime-like strings
+ str_ts = '2000-03-14 01:59'
+ str_ts_tz = '2000-03-14 01:59-0500'
+
+ assert (dr > str_ts).all()
+ with pytest.raises(TypeError):
+ dr == str_ts_tz
+
+ assert (dz > str_ts_tz).all()
+ with pytest.raises(TypeError):
+ dz != str_ts
+
+ # Check comparisons against scalar Timestamps
+ ts = pd.Timestamp('2000-03-14 01:59')
+ ts_tz = pd.Timestamp('2000-03-14 01:59', tz='Europe/Amsterdam')
+
+ assert (dr > ts).all()
+ with pytest.raises(TypeError):
+ dr == ts_tz
+
+ assert (dz > ts_tz).all()
+ with pytest.raises(TypeError):
+ dz != ts
def test_comparisons_coverage(self):
rng = date_range('1/1/2000', periods=10)
From b9e7c6d7e071ce9b43c521e8a5dd8512f3d5aee6 Mon Sep 17 00:00:00 2001
From: Brock Mendel <jbrockmendel@gmail.com>
Date: Sat, 25 Nov 2017 14:25:16 -0800
Subject: [PATCH 08/19] revert hack, xfail test
---
pandas/core/internals.py | 25 ++++++++++---------------
pandas/tests/indexing/test_coercion.py | 1 +
2 files changed, 11 insertions(+), 15 deletions(-)
diff --git a/pandas/core/internals.py b/pandas/core/internals.py
index a5d179a038420..75f52681c9f3b 100644
--- a/pandas/core/internals.py
+++ b/pandas/core/internals.py
@@ -3486,22 +3486,17 @@ def replace_list(self, src_list, dest_list, inplace=False, regex=False,
# figure out our mask a-priori to avoid repeated replacements
values = self.as_matrix()
- if is_datetime64tz_dtype(self):
- # need to be careful not to compare tznaive to tzaware
- def comp(s):
- if isna(s):
- return isna(values)
- return _maybe_compare(values, s, operator.eq)
- # TODO: Is just not-converting `s` the right thing to do here?
- else:
- def comp(s):
- if isna(s):
- return isna(values)
- return _maybe_compare(values, getattr(s, 'asm8', s),
- operator.eq)
-
- masks = [comp(s) for i, s in enumerate(src_list)]
+ def comp(s):
+ if isna(s):
+ return isna(values)
+ return _maybe_compare(values, getattr(s, 'asm8', s), operator.eq)
+ try:
+ masks = [comp(s) for i, s in enumerate(src_list)]
+ except:
+ print(src_list)
+ print(dest_list)
+ raise
result_blocks = []
src_len = len(src_list) - 1
for blk in self.blocks:
diff --git a/pandas/tests/indexing/test_coercion.py b/pandas/tests/indexing/test_coercion.py
index 752d2deb53304..a126c8c959f60 100644
--- a/pandas/tests/indexing/test_coercion.py
+++ b/pandas/tests/indexing/test_coercion.py
@@ -1311,6 +1311,7 @@ def test_replace_series_datetime64(self):
for to_key in self.rep:
self._assert_replace_conversion(from_key, to_key, how='series')
+ @pytest.mark.xfail
def test_replace_series_datetime64tz(self):
from_key = 'datetime64[ns, US/Eastern]'
for to_key in self.rep:
From eefadf75aeb5e1f6119a7a7e49f320349b65cb86 Mon Sep 17 00:00:00 2001
From: Brock Mendel <jbrockmendel@gmail.com>
Date: Sun, 26 Nov 2017 08:31:35 -0800
Subject: [PATCH 09/19] parametrize tests, comment for xfail
---
.../tests/indexes/datetimes/test_datetime.py | 25 +++++++++----------
pandas/tests/indexes/test_base.py | 18 ++++++-------
pandas/tests/indexing/test_coercion.py | 1 +
pandas/tests/series/test_indexing.py | 4 +--
4 files changed, 23 insertions(+), 25 deletions(-)
diff --git a/pandas/tests/indexes/datetimes/test_datetime.py b/pandas/tests/indexes/datetimes/test_datetime.py
index 80bc7f184adc3..24718bf510a3d 100644
--- a/pandas/tests/indexes/datetimes/test_datetime.py
+++ b/pandas/tests/indexes/datetimes/test_datetime.py
@@ -250,19 +250,18 @@ def test_append_join_nondatetimeindex(self):
# it works
rng.join(idx, how='outer')
- def test_comparison_tzawareness_compat(self):
+ @pytest.mark.parametrize('op', [operator.eq, operator.ne,
+ operator.gt, operator.ge,
+ operator.lt, operator.le])
+ def test_comparison_tzawareness_compat(self, op):
# GH#18162
dr = pd.date_range('2016-01-01', periods=6)
dz = dr.tz_localize('US/Pacific')
- ops = [operator.eq, operator.ne,
- operator.gt, operator.ge,
- operator.lt, operator.le]
-
- for left, right in [(dr, dz), (dz, dr)]:
- for op in ops:
- with pytest.raises(TypeError):
- op(left, right)
+ with pytest.raises(TypeError):
+ op(dr, dz)
+ with pytest.raises(TypeError):
+ op(dz, dr)
# Check that there isn't a problem aware-aware and naive-naive do not
# raise
@@ -275,11 +274,11 @@ def test_comparison_tzawareness_compat(self):
assert (dr > str_ts).all()
with pytest.raises(TypeError):
- dr == str_ts_tz
+ op(dr, str_ts_tz)
assert (dz > str_ts_tz).all()
with pytest.raises(TypeError):
- dz != str_ts
+ op(dz, str_ts)
# Check comparisons against scalar Timestamps
ts = pd.Timestamp('2000-03-14 01:59')
@@ -287,11 +286,11 @@ def test_comparison_tzawareness_compat(self):
assert (dr > ts).all()
with pytest.raises(TypeError):
- dr == ts_tz
+ op(dr, ts_tz)
assert (dz > ts_tz).all()
with pytest.raises(TypeError):
- dz != ts
+ op(dz, ts)
def test_comparisons_coverage(self):
rng = date_range('1/1/2000', periods=10)
diff --git a/pandas/tests/indexes/test_base.py b/pandas/tests/indexes/test_base.py
index b3d3386476c6c..bbd203e335cd8 100644
--- a/pandas/tests/indexes/test_base.py
+++ b/pandas/tests/indexes/test_base.py
@@ -2240,24 +2240,22 @@ def test_intersect_str_dates(self):
assert len(res) == 0
- def test_comparison_tzawareness_compat(self):
+ @pytest.mark.parametrize('op', [operator.eq, operator.ne,
+ operator.gt, operator.ge,
+ operator.lt, operator.le])
+ def test_comparison_tzawareness_compat(self, op):
# GH#18162
dr = pd.date_range('2016-01-01', periods=6)
dz = dr.tz_localize('US/Pacific')
- ops = [operator.eq, operator.ne,
- operator.gt, operator.ge,
- operator.lt, operator.le]
-
# Check that there isn't a problem aware-aware and naive-naive do not
# raise
naive_series = Series(dr)
aware_series = Series(dz)
- for op in ops:
- with pytest.raises(TypeError):
- op(dz, naive_series)
- with pytest.raises(TypeError):
- op(dr, aware_series)
+ with pytest.raises(TypeError):
+ op(dz, naive_series)
+ with pytest.raises(TypeError):
+ op(dr, aware_series)
# TODO: implement _assert_tzawareness_compat for the reverse
# comparison with the Series on the left-hand side
diff --git a/pandas/tests/indexing/test_coercion.py b/pandas/tests/indexing/test_coercion.py
index a126c8c959f60..454a05b4e616e 100644
--- a/pandas/tests/indexing/test_coercion.py
+++ b/pandas/tests/indexing/test_coercion.py
@@ -1311,6 +1311,7 @@ def test_replace_series_datetime64(self):
for to_key in self.rep:
self._assert_replace_conversion(from_key, to_key, how='series')
+ # GH #18376, tzawareness-compat bug in BlockManager.replace_list
@pytest.mark.xfail
def test_replace_series_datetime64tz(self):
from_key = 'datetime64[ns, US/Eastern]'
diff --git a/pandas/tests/series/test_indexing.py b/pandas/tests/series/test_indexing.py
index 18effb26a298f..37c2c1da2e93b 100644
--- a/pandas/tests/series/test_indexing.py
+++ b/pandas/tests/series/test_indexing.py
@@ -486,8 +486,8 @@ def test_getitem_setitem_datetimeindex(self):
# see GH#18376, GH#18162
ts[(ts.index >= lb) & (ts.index <= rb)]
- lb = rng.tzinfo.localize(datetime(1990, 1, 1, 4))
- rb = rng.tzinfo.localize(datetime(1990, 1, 1, 7))
+ lb = pd.Timestamp(datetime(1990, 1, 1, 4)).tz_localize(rng.tzinfo)
+ rb = pd.Timestamp(datetime(1990, 1, 1, 7)).tz_localize(rng.tzinfo)
result = ts[(ts.index >= lb) & (ts.index <= rb)]
expected = ts[4:8]
assert_series_equal(result, expected)
From abe65f80b47346525f44e96149295fb277bd499e Mon Sep 17 00:00:00 2001
From: Brock Mendel <jbrockmendel@gmail.com>
Date: Wed, 6 Dec 2017 09:03:52 -0800
Subject: [PATCH 10/19] remove debugging prints
---
pandas/core/internals.py | 7 +------
1 file changed, 1 insertion(+), 6 deletions(-)
diff --git a/pandas/core/internals.py b/pandas/core/internals.py
index e30f2360ac030..9ee95fbee5985 100644
--- a/pandas/core/internals.py
+++ b/pandas/core/internals.py
@@ -3491,12 +3491,7 @@ def comp(s):
return isna(values)
return _maybe_compare(values, getattr(s, 'asm8', s), operator.eq)
- try:
- masks = [comp(s) for i, s in enumerate(src_list)]
- except:
- print(src_list)
- print(dest_list)
- raise
+ masks = [comp(s) for i, s in enumerate(src_list)]
result_blocks = []
src_len = len(src_list) - 1
for blk in self.blocks:
From 7cba1032b4e687edc6298d0f1a54e31143615e67 Mon Sep 17 00:00:00 2001
From: Brock Mendel <jbrockmendel@gmail.com>
Date: Wed, 6 Dec 2017 09:08:00 -0800
Subject: [PATCH 11/19] rearrange isinstance checks per suggestion
---
pandas/core/indexes/datetimes.py | 10 ++++++----
1 file changed, 6 insertions(+), 4 deletions(-)
diff --git a/pandas/core/indexes/datetimes.py b/pandas/core/indexes/datetimes.py
index 6473404d89ba1..b2eb40b523207 100644
--- a/pandas/core/indexes/datetimes.py
+++ b/pandas/core/indexes/datetimes.py
@@ -105,10 +105,12 @@ def _dt_index_cmp(opname, cls, nat_result=False):
def wrapper(self, other):
func = getattr(super(DatetimeIndex, self), opname)
- if isinstance(other, (datetime, compat.string_types)):
- if isinstance(other, compat.string_types):
- # We need to convert in order to assert tzawareness
- other = Timestamp(other)
+
+ if isinstance(other, compat.string_types):
+ # We need to convert in order to assert tzawareness
+ other = Timestamp(other)
+
+ if isinstance(other, datetime):
self._assert_tzawareness_compat(other)
other = _to_m8(other, tz=self.tz)
result = func(other)
From 2d68f0154cfbb4eafa65ba87ba70361be21a8a2d Mon Sep 17 00:00:00 2001
From: Brock Mendel <jbrockmendel@gmail.com>
Date: Wed, 6 Dec 2017 09:11:37 -0800
Subject: [PATCH 12/19] add whatsnew note
---
doc/source/whatsnew/v0.22.0.txt | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/doc/source/whatsnew/v0.22.0.txt b/doc/source/whatsnew/v0.22.0.txt
index d34c1f3535509..58470a9aa0cc1 100644
--- a/doc/source/whatsnew/v0.22.0.txt
+++ b/doc/source/whatsnew/v0.22.0.txt
@@ -352,4 +352,4 @@ Other
^^^^^
- Improved error message when attempting to use a Python keyword as an identifier in a numexpr query (:issue:`18221`)
--
+- Fixed bug where comparing :class:`DatetimeIndex` failed to raise ``TypeError`` when attempting to compare timezone-aware and timezone-naive datetimelike objects (:issue:`18162`)
From ddf1cea1126f1c5b6267686b6b5ab91dd99214ff Mon Sep 17 00:00:00 2001
From: Brock Mendel <jbrockmendel@gmail.com>
Date: Thu, 7 Dec 2017 07:48:52 -0800
Subject: [PATCH 13/19] fix duplicate line from merge mixup
---
doc/source/whatsnew/v0.22.0.txt | 1 -
1 file changed, 1 deletion(-)
diff --git a/doc/source/whatsnew/v0.22.0.txt b/doc/source/whatsnew/v0.22.0.txt
index 849a739101f1d..56cf5572cc521 100644
--- a/doc/source/whatsnew/v0.22.0.txt
+++ b/doc/source/whatsnew/v0.22.0.txt
@@ -323,7 +323,6 @@ Categorical
Other
^^^^^
-- Improved error message when attempting to use a Python keyword as an identifier in a numexpr query (:issue:`18221`)
- Fixed bug where comparing :class:`DatetimeIndex` failed to raise ``TypeError`` when attempting to compare timezone-aware and timezone-naive datetimelike objects (:issue:`18162`)
- Improved error message when attempting to use a Python keyword as an identifier in a ``numexpr`` backed query (:issue:`18221`)
-
From 3b4a01eba79028b4e42ba405d4b821424298a0c8 Mon Sep 17 00:00:00 2001
From: Brock Mendel <jbrockmendel@gmail.com>
Date: Thu, 7 Dec 2017 07:50:16 -0800
Subject: [PATCH 14/19] put comment into xfail message
---
pandas/tests/indexing/test_coercion.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/pandas/tests/indexing/test_coercion.py b/pandas/tests/indexing/test_coercion.py
index 454a05b4e616e..f042bdbf004be 100644
--- a/pandas/tests/indexing/test_coercion.py
+++ b/pandas/tests/indexing/test_coercion.py
@@ -1311,8 +1311,8 @@ def test_replace_series_datetime64(self):
for to_key in self.rep:
self._assert_replace_conversion(from_key, to_key, how='series')
- # GH #18376, tzawareness-compat bug in BlockManager.replace_list
- @pytest.mark.xfail
+ @pytest.mark.xfail('GH #18376, tzawareness-compat bug '
+ 'in BlockManager.replace_list')
def test_replace_series_datetime64tz(self):
from_key = 'datetime64[ns, US/Eastern]'
for to_key in self.rep:
From aeef3d7ca28e5556d454710d3a77f346bb458c2b Mon Sep 17 00:00:00 2001
From: Brock Mendel <jbrockmendel@gmail.com>
Date: Thu, 7 Dec 2017 09:49:34 -0800
Subject: [PATCH 15/19] Fixup missing keyword
---
pandas/tests/indexing/test_coercion.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/pandas/tests/indexing/test_coercion.py b/pandas/tests/indexing/test_coercion.py
index f042bdbf004be..68d3c86b3eda7 100644
--- a/pandas/tests/indexing/test_coercion.py
+++ b/pandas/tests/indexing/test_coercion.py
@@ -1311,8 +1311,8 @@ def test_replace_series_datetime64(self):
for to_key in self.rep:
self._assert_replace_conversion(from_key, to_key, how='series')
- @pytest.mark.xfail('GH #18376, tzawareness-compat bug '
- 'in BlockManager.replace_list')
+ @pytest.mark.xfail(reason='GH #18376, tzawareness-compat bug '
+ 'in BlockManager.replace_list')
def test_replace_series_datetime64tz(self):
from_key = 'datetime64[ns, US/Eastern]'
for to_key in self.rep:
From f21f0257811cafc15cc7b2887f185bf59072f395 Mon Sep 17 00:00:00 2001
From: Brock Mendel <jbrockmendel@gmail.com>
Date: Sat, 16 Dec 2017 10:09:13 -0800
Subject: [PATCH 16/19] give strings a free pass from tzawareness compat
---
pandas/core/indexes/datetimes.py | 9 ++++-----
pandas/core/internals.py | 1 +
pandas/tests/indexes/datetimes/test_datetime.py | 12 ------------
pandas/tests/series/test_indexing.py | 8 ++++----
4 files changed, 9 insertions(+), 21 deletions(-)
diff --git a/pandas/core/indexes/datetimes.py b/pandas/core/indexes/datetimes.py
index e55e3675b78b0..9085cec4a0853 100644
--- a/pandas/core/indexes/datetimes.py
+++ b/pandas/core/indexes/datetimes.py
@@ -108,12 +108,11 @@ def _dt_index_cmp(opname, cls, nat_result=False):
def wrapper(self, other):
func = getattr(super(DatetimeIndex, self), opname)
- if isinstance(other, compat.string_types):
- # We need to convert in order to assert tzawareness
- other = Timestamp(other)
+ if isinstance(other, (datetime, compat.string_types)):
+ if isinstance(other, datetime):
+ # GH#18435 strings get a pass from tzawareness compat
+ self._assert_tzawareness_compat(other)
- if isinstance(other, datetime):
- self._assert_tzawareness_compat(other)
other = _to_m8(other, tz=self.tz)
result = func(other)
if isna(other):
diff --git a/pandas/core/internals.py b/pandas/core/internals.py
index 059abbc1d131d..3a64a0ef84e3d 100644
--- a/pandas/core/internals.py
+++ b/pandas/core/internals.py
@@ -3482,6 +3482,7 @@ def comp(s):
return _maybe_compare(values, getattr(s, 'asm8', s), operator.eq)
masks = [comp(s) for i, s in enumerate(src_list)]
+
result_blocks = []
src_len = len(src_list) - 1
for blk in self.blocks:
diff --git a/pandas/tests/indexes/datetimes/test_datetime.py b/pandas/tests/indexes/datetimes/test_datetime.py
index 24718bf510a3d..0b56ca6f60294 100644
--- a/pandas/tests/indexes/datetimes/test_datetime.py
+++ b/pandas/tests/indexes/datetimes/test_datetime.py
@@ -268,18 +268,6 @@ def test_comparison_tzawareness_compat(self, op):
assert (dr == dr).all()
assert (dz == dz).all()
- # Check comparisons against datetime-like strings
- str_ts = '2000-03-14 01:59'
- str_ts_tz = '2000-03-14 01:59-0500'
-
- assert (dr > str_ts).all()
- with pytest.raises(TypeError):
- op(dr, str_ts_tz)
-
- assert (dz > str_ts_tz).all()
- with pytest.raises(TypeError):
- op(dz, str_ts)
-
# Check comparisons against scalar Timestamps
ts = pd.Timestamp('2000-03-14 01:59')
ts_tz = pd.Timestamp('2000-03-14 01:59', tz='Europe/Amsterdam')
diff --git a/pandas/tests/series/test_indexing.py b/pandas/tests/series/test_indexing.py
index e90f3634827ed..bcda56df836bb 100644
--- a/pandas/tests/series/test_indexing.py
+++ b/pandas/tests/series/test_indexing.py
@@ -450,10 +450,10 @@ def test_getitem_setitem_datetimeindex(self):
lb = "1990-01-01 04:00:00"
rb = "1990-01-01 07:00:00"
- with pytest.raises(TypeError):
- # tznaive vs tzaware comparison is invalid
- # see GH#18376, GH#18162
- ts[(ts.index >= lb) & (ts.index <= rb)]
+ # GH#18435 strings get a pass from tzawareness compat
+ result = ts[(ts.index >= lb) & (ts.index <= rb)]
+ expected = ts[4:8]
+ assert_series_equal(result, expected)
lb = "1990-01-01 04:00:00-0500"
rb = "1990-01-01 07:00:00-0500"
From a6008e042b45afa1eafb106d0c1b1faa7e89dcc7 Mon Sep 17 00:00:00 2001
From: Brock Mendel <jbrockmendel@gmail.com>
Date: Thu, 4 Jan 2018 11:17:24 -0800
Subject: [PATCH 17/19] add list test cases
---
pandas/core/indexes/datetimes.py | 7 +++++--
pandas/tests/indexes/datetimes/test_datetime.py | 6 ++++++
2 files changed, 11 insertions(+), 2 deletions(-)
diff --git a/pandas/core/indexes/datetimes.py b/pandas/core/indexes/datetimes.py
index 702525a70a561..b7a74b0dbeb32 100644
--- a/pandas/core/indexes/datetimes.py
+++ b/pandas/core/indexes/datetimes.py
@@ -16,7 +16,7 @@
is_timedelta64_dtype,
is_integer, is_float,
is_integer_dtype,
- is_datetime64_ns_dtype,
+ is_datetime64_ns_dtype, is_datetimelike,
is_period_dtype,
is_bool_dtype,
is_string_dtype,
@@ -122,7 +122,10 @@ def wrapper(self, other):
other = DatetimeIndex(other)
elif not isinstance(other, (np.ndarray, Index, ABCSeries)):
other = _ensure_datetime64(other)
- self._assert_tzawareness_compat(other)
+
+ if is_datetimelike(other):
+ self._assert_tzawareness_compat(other)
+
result = func(np.asarray(other))
result = _values_from_object(result)
diff --git a/pandas/tests/indexes/datetimes/test_datetime.py b/pandas/tests/indexes/datetimes/test_datetime.py
index 0b56ca6f60294..41cd654cf22b9 100644
--- a/pandas/tests/indexes/datetimes/test_datetime.py
+++ b/pandas/tests/indexes/datetimes/test_datetime.py
@@ -260,13 +260,19 @@ def test_comparison_tzawareness_compat(self, op):
with pytest.raises(TypeError):
op(dr, dz)
+ with pytest.raises(TypeError):
+ op(dr, list(dz))
with pytest.raises(TypeError):
op(dz, dr)
+ with pytest.raises(TypeError):
+ op(dz, list(dr))
# Check that there isn't a problem aware-aware and naive-naive do not
# raise
assert (dr == dr).all()
+ assert (dr == list(dr)).all()
assert (dz == dz).all()
+ assert (dz == list(dz)).all()
# Check comparisons against scalar Timestamps
ts = pd.Timestamp('2000-03-14 01:59')
From e2611dc3ee384188d56754e07d971dd593c1e7d3 Mon Sep 17 00:00:00 2001
From: Brock Mendel <jbrockmendel@gmail.com>
Date: Thu, 4 Jan 2018 11:48:40 -0800
Subject: [PATCH 18/19] follow precedent to separate out xfailed tests
---
pandas/tests/indexing/test_coercion.py | 35 ++++++++++++++++++++++++--
1 file changed, 33 insertions(+), 2 deletions(-)
diff --git a/pandas/tests/indexing/test_coercion.py b/pandas/tests/indexing/test_coercion.py
index b518a20390067..de756375db8cb 100644
--- a/pandas/tests/indexing/test_coercion.py
+++ b/pandas/tests/indexing/test_coercion.py
@@ -822,8 +822,8 @@ def test_replace_series(self, how, to_key, from_key):
# tested below
return
elif from_key in ['datetime64[ns, US/Eastern]', 'datetime64[ns, UTC]']:
- pytest.xfail(reason='GH #18376, tzawareness-compat bug '
- 'in BlockManager.replace_list')
+ # tested below
+ return
if how == 'dict':
replacer = dict(zip(self.rep[from_key], self.rep[to_key]))
@@ -852,6 +852,37 @@ def test_replace_series(self, how, to_key, from_key):
tm.assert_series_equal(result, exp)
+ # TODO(jbrockmendel) commented out to only have a single xfail printed
+ @pytest.mark.xfail(reason='GH #18376, tzawareness-compat bug '
+ 'in BlockManager.replace_list')
+ # @pytest.mark.parametrize('how', ['dict', 'series'])
+ # @pytest.mark.parametrize('to_key', ['timedelta64[ns]', 'bool', 'object',
+ # 'complex128', 'float64', 'int64'])
+ # @pytest.mark.parametrize('from_key', ['datetime64[ns, UTC]',
+ # 'datetime64[ns, US/Eastern]'])
+ # def test_replace_series_datetime_tz(self, how, to_key, from_key):
+ def test_replace_series_datetime_tz(self):
+ how = 'series'
+ from_key = 'datetime64[ns, US/Eastern]'
+ to_key = 'timedelta64[ns]'
+
+ index = pd.Index([3, 4], name='xxx')
+ obj = pd.Series(self.rep[from_key], index=index, name='yyy')
+ assert obj.dtype == from_key
+
+ if how == 'dict':
+ replacer = dict(zip(self.rep[from_key], self.rep[to_key]))
+ elif how == 'series':
+ replacer = pd.Series(self.rep[to_key], index=self.rep[from_key])
+ else:
+ raise ValueError
+
+ result = obj.replace(replacer)
+ exp = pd.Series(self.rep[to_key], index=index, name='yyy')
+ assert exp.dtype == to_key
+
+ tm.assert_series_equal(result, exp)
+
# TODO(jreback) commented out to only have a single xfail printed
@pytest.mark.xfail(reason="different tz, "
"currently mask_missing raises SystemError")
From 5653c610fd2b8bff5bd4dd4d05d197d871da2df2 Mon Sep 17 00:00:00 2001
From: Brock Mendel <jbrockmendel@gmail.com>
Date: Thu, 4 Jan 2018 17:40:27 -0800
Subject: [PATCH 19/19] move whatsnew to conversion
---
doc/source/whatsnew/v0.23.0.txt | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/doc/source/whatsnew/v0.23.0.txt b/doc/source/whatsnew/v0.23.0.txt
index 569fdf0bfa8bc..d3194075f76bc 100644
--- a/doc/source/whatsnew/v0.23.0.txt
+++ b/doc/source/whatsnew/v0.23.0.txt
@@ -310,6 +310,7 @@ Conversion
- Bug in :class:`Series` floor-division where operating on a scalar ``timedelta`` raises an exception (:issue:`18846`)
- Bug in :class:`FY5253Quarter`, :class:`LastWeekOfMonth` where rollback and rollforward behavior was inconsistent with addition and subtraction behavior (:issue:`18854`)
- Bug in :class:`Index` constructor with ``dtype=CategoricalDtype(...)`` where ``categories`` and ``ordered`` are not maintained (issue:`19032`)
+- Fixed bug where comparing :class:`DatetimeIndex` failed to raise ``TypeError`` when attempting to compare timezone-aware and timezone-naive datetimelike objects (:issue:`18162`)
Indexing
@@ -395,5 +396,4 @@ Categorical
Other
^^^^^
-- Fixed bug where comparing :class:`DatetimeIndex` failed to raise ``TypeError`` when attempting to compare timezone-aware and timezone-naive datetimelike objects (:issue:`18162`)
- Improved error message when attempting to use a Python keyword as an identifier in a ``numexpr`` backed query (:issue:`18221`)