Merge remote-tracking branch 'refs/remotes/pydata/master' into Fix-for-pandas-dev#11317

Author: Dr-Irv

asv_bench/asv.conf.json

@@ -43,6 +43,7 @@
         "numexpr": [],
         "pytables": [],
         "openpyxl": [],
+        "xlsxwriter": [],
         "xlrd": [],
         "xlwt": []
     },

asv_bench/benchmarks/gil.py

@@ -320,3 +320,49 @@ def time_nogil_kth_smallest(self):
         def run(arr):
             algos.kth_smallest(arr, self.k)
         run()
+
+class nogil_datetime_fields(object):
+    goal_time = 0.2
+
+    def setup(self):
+        self.N = 100000000
+        self.dti = pd.date_range('1900-01-01', periods=self.N, freq='D')
+        self.period = self.dti.to_period('D')
+        if (not have_real_test_parallel):
+            raise NotImplementedError
+
+    def time_datetime_field_year(self):
+        @test_parallel(num_threads=2)
+        def run(dti):
+            dti.year
+        run(self.dti)
+
+    def time_datetime_field_day(self):
+        @test_parallel(num_threads=2)
+        def run(dti):
+            dti.day
+        run(self.dti)
+
+    def time_datetime_field_daysinmonth(self):
+        @test_parallel(num_threads=2)
+        def run(dti):
+            dti.days_in_month
+        run(self.dti)
+
+    def time_datetime_field_normalize(self):
+        @test_parallel(num_threads=2)
+        def run(dti):
+            dti.normalize()
+        run(self.dti)
+
+    def time_datetime_to_period(self):
+        @test_parallel(num_threads=2)
+        def run(dti):
+            dti.to_period('S')
+        run(self.dti)
+
+    def time_period_to_datetime(self):
+        @test_parallel(num_threads=2)
+        def run(period):
+            period.to_timestamp()
+        run(self.period)

asv_bench/benchmarks/series_methods.py

@@ -71,3 +71,23 @@ def setup(self):
     def time_series_nsmallest2(self):
         self.s2.nsmallest(3, take_last=True)
         self.s2.nsmallest(3, take_last=False)
+
+
+class series_dropna_int64(object):
+    goal_time = 0.2
+
+    def setup(self):
+        self.s = Series(np.random.randint(1, 10, 1000000))
+
+    def time_series_dropna_int64(self):
+        self.s.dropna()
+
+class series_dropna_datetime(object):
+    goal_time = 0.2
+
+    def setup(self):
+        self.s = Series(pd.date_range('2000-01-01', freq='S', periods=1000000))
+        self.s[np.random.randint(1, 1000000, 100)] = pd.NaT
+
+    def time_series_dropna_datetime(self):
+        self.s.dropna()

ci/install_conda.sh

@@ -73,7 +73,7 @@ bash miniconda.sh -b -p $HOME/miniconda || exit 1
 conda config --set always_yes yes --set changeps1 no || exit 1
 conda update -q conda || exit 1
 conda config --add channels conda-forge || exit 1
-conda config --add channels http://conda.binstar.org/pandas || exit 1
+conda config --add channels http://conda.anaconda.org/pandas || exit 1
 conda config --set ssl_verify false || exit 1
 
 # Useful for debugging any issues with conda

ci/requirements-2.7.pip

@@ -2,3 +2,5 @@ blosc
 httplib2
 google-api-python-client == 1.2
 python-gflags == 2.0
+pathlib
+py

ci/requirements-2.7_SLOW.pip

@@ -2,3 +2,4 @@ python-dateutil
 pytz
 numpy=1.8.1
 cython
+libgfortran

ci/requirements-3.4.build

@@ -299,8 +299,9 @@
 intersphinx_mapping = {
     'statsmodels': ('http://statsmodels.sourceforge.net/devel/', None),
     'matplotlib': ('http://matplotlib.org/', None),
-    'python': ('http://docs.python.org/', None),
-    'numpy': ('http://docs.scipy.org/doc/numpy', None)
+    'python': ('http://docs.python.org/3', None),
+    'numpy': ('http://docs.scipy.org/doc/numpy', None),
+    'py': ('http://pylib.readthedocs.org/en/latest/', None)
 }
 import glob
 autosummary_generate = glob.glob("*.rst")

doc/source/conf.py

@@ -79,9 +79,10 @@ for some advanced strategies
 
 They can take a number of arguments:
 
-  - ``filepath_or_buffer``: Either a string path to a file, URL
+  - ``filepath_or_buffer``: Either a path to a file (a :class:`python:str`,
+    :class:`python:pathlib.Path`, or :class:`py:py._path.local.LocalPath`), URL
     (including http, ftp, and S3 locations), or any object with a ``read``
-    method (such as an open file or ``StringIO``).
+    method (such as an open file or :class:`~python:io.StringIO`).
   - ``sep`` or ``delimiter``: A delimiter / separator to split fields
     on. With ``sep=None``, ``read_csv`` will try to infer the delimiter
     automatically in some cases by "sniffing".

doc/source/io.rst

@@ -28,6 +28,10 @@ Enhancements
 Other Enhancements
 ^^^^^^^^^^^^^^^^^^
 
+- ``pd.read_*`` functions can now also accept :class:`python:pathlib.Path`, or :class:`py:py._path.local.LocalPath`
+  objects for the ``filepath_or_buffer`` argument. (:issue:`11033`)
+- Improve the error message displayed in :func:`pandas.io.gbq.to_gbq` when the DataFrame does not match the schema of the destination table (:issue:`11359`)
+
 .. _whatsnew_0171.api:
 
 API changes
@@ -38,6 +42,7 @@ API changes
 - Regression from 0.16.2 for output formatting of long floats/nan, restored in (:issue:`11302`)
 - Prettyprinting sets (e.g. in DataFrame cells) now uses set literal syntax (``{x, y}``) instead of
   Legacy Python syntax (``set([x, y])``) (:issue:`11215`)
+- Indexing with a null key will raise a ``TypeError``, instead of a ``ValueError`` (:issue:`11356`)
 
 .. _whatsnew_0171.deprecations:
 
@@ -54,6 +59,13 @@ Performance Improvements
 ~~~~~~~~~~~~~~~~~~~~~~~~
 
 - Checking monotonic-ness before sorting on an index (:issue:`11080`)
+- ``Series.dropna`` performance improvement when its dtype can't contain ``NaN`` (:issue:`11159`)
+
+
+- Release the GIL on most datetime field operations (e.g. ``DatetimeIndex.year``, ``Series.dt.year``), normalization, and conversion to and from ``Period``, ``DatetimeIndex.to_period`` and ``PeriodIndex.to_timestamp`` (:issue:`11263`)
+
+
+- Improved performance to ``to_excel`` (:issue:`11352`)
 
 .. _whatsnew_0171.bug_fixes:
 
@@ -65,12 +77,18 @@ Bug Fixes
 
 - Bug in ``HDFStore.select`` when comparing with a numpy scalar in a where clause (:issue:`11283`)
 
+
 - Bug in tz-conversions with an ambiguous time and ``.dt`` accessors (:issue:`11295`)
 - Bug in comparisons of Series vs list-likes (:issue:`11339`)
 
-- Bug in list-like indexing with a mixed-integer Index (:issue:`11320`)
 
+- Bug in ``DataFrame.replace`` with a ``datetime64[ns, tz]`` and a non-compat to_replace (:issue:`11326`, :issue:`11153`)
+
+
+
+- Bug in list-like indexing with a mixed-integer Index (:issue:`11320`)
 
+- Bug in ``pivot_table`` with ``margins=True`` when indexes are of ``Categorical`` dtype (:issue:`10993`)
 - Bug in ``DataFrame.plot`` cannot use hex strings colors (:issue:`10299`)
 
 

doc/source/whatsnew/v0.17.1.txt

@@ -444,14 +444,24 @@ def mask_missing(arr, values_to_mask):
     mask = None
     for x in nonna:
         if mask is None:
-            mask = arr == x
+
+            # numpy elementwise comparison warning
+            if is_numeric_v_string_like(arr, x):
+                mask = False
+            else:
+                mask = arr == x
 
             # if x is a string and arr is not, then we get False and we must
             # expand the mask to size arr.shape
             if np.isscalar(mask):
                 mask = np.zeros(arr.shape, dtype=bool)
         else:
-            mask |= arr == x
+
+            # numpy elementwise comparison warning
+            if is_numeric_v_string_like(arr, x):
+                mask |= False
+            else:
+                mask |= arr == x
 
     if na_mask.any():
         if mask is None:
@@ -2382,6 +2392,9 @@ def _maybe_make_list(obj):
 is_complex = lib.is_complex
 
 
+def is_string_like(obj):
+    return isinstance(obj, (compat.text_type, compat.string_types))
+
 def is_iterator(obj):
     # python 3 generators have __next__ instead of next
     return hasattr(obj, 'next') or hasattr(obj, '__next__')
@@ -2525,6 +2538,27 @@ def is_datetime_or_timedelta_dtype(arr_or_dtype):
     return issubclass(tipo, (np.datetime64, np.timedelta64))
 
 
+def is_numeric_v_string_like(a, b):
+    """
+    numpy doesn't like to compare numeric arrays vs scalar string-likes
+
+    return a boolean result if this is the case for a,b or b,a
+
+    """
+    is_a_array = isinstance(a, np.ndarray)
+    is_b_array = isinstance(b, np.ndarray)
+
+    is_a_numeric_array = is_a_array and is_numeric_dtype(a)
+    is_b_numeric_array = is_b_array and is_numeric_dtype(b)
+
+    is_a_scalar_string_like = not is_a_array and is_string_like(a)
+    is_b_scalar_string_like = not is_b_array and is_string_like(b)
+
+    return (
+        is_a_numeric_array and is_b_scalar_string_like) or (
+        is_b_numeric_array and is_a_scalar_string_like
+        )
+
 def is_datetimelike_v_numeric(a, b):
     # return if we have an i8 convertible and numeric comparision
     if not hasattr(a,'dtype'):

pandas/core/common.py

@@ -1708,9 +1708,9 @@ def _format_value(self, val):
         if lib.checknull(val):
             val = self.na_rep
         elif com.is_float(val):
-            if np.isposinf(val):
+            if lib.isposinf_scalar(val):
                 val = self.inf_rep
-            elif np.isneginf(val):
+            elif lib.isneginf_scalar(val):
                 val = '-%s' % self.inf_rep
             elif self.float_format is not None:
                 val = float(self.float_format % val)

pandas/core/format.py

@@ -2999,8 +2999,6 @@ def replace(self, to_replace=None, value=None, inplace=False, limit=None,
                            '{0!r}').format(type(to_replace).__name__)
                     raise TypeError(msg)  # pragma: no cover
 
-        new_data = new_data.convert(copy=not inplace, numeric=False)
-
         if inplace:
             self._update_inplace(new_data)
         else:

pandas/core/generic.py

@@ -627,6 +627,10 @@ def astype(self, dtype):
         return Index(self.values.astype(dtype), name=self.name,
                      dtype=dtype)
 
+    def _to_safe_for_reshape(self):
+        """ convert to object if we are a categorical """
+        return self
+
     def to_datetime(self, dayfirst=False):
         """
         For an Index containing strings or datetime.datetime objects, attempt
@@ -862,9 +866,10 @@ def to_int():
             return self._invalid_indexer('label', key)
 
         if is_float(key):
-            if not self.is_floating():
-                warnings.warn("scalar indexers for index type {0} should be integers and not floating point".format(
-                    type(self).__name__), FutureWarning, stacklevel=3)
+            if isnull(key):
+                return self._invalid_indexer('label', key)
+            warnings.warn("scalar indexers for index type {0} should be integers and not floating point".format(
+                type(self).__name__), FutureWarning, stacklevel=3)
             return to_int()
 
         return key
@@ -3189,6 +3194,10 @@ def duplicated(self, keep='first'):
         from pandas.hashtable import duplicated_int64
         return duplicated_int64(self.codes.astype('i8'), keep)
 
+    def _to_safe_for_reshape(self):
+        """ convert to object if we are a categorical """
+        return self.astype('object')
+
     def get_loc(self, key, method=None):
         """
         Get integer location for requested label
@@ -3721,9 +3730,23 @@ def astype(self, dtype):
         return Index(self._values, name=self.name, dtype=dtype)
 
     def _convert_scalar_indexer(self, key, kind=None):
+        """
+        convert a scalar indexer
+
+        Parameters
+        ----------
+        key : label of the slice bound
+        kind : optional, type of the indexing operation (loc/ix/iloc/None)
+
+        right now we are converting
+        floats -> ints if the index supports it
+        """
+
         if kind == 'iloc':
-            return super(Float64Index, self)._convert_scalar_indexer(key,
-                                                                     kind=kind)
+            if is_integer(key):
+                return key
+            return super(Float64Index, self)._convert_scalar_indexer(key, kind=kind)
+
         return key
 
     def _convert_slice_indexer(self, key, kind=None):
@@ -4276,7 +4299,7 @@ def _reference_duplicate_name(self, name):
         Returns True if the name refered to in self.names is duplicated.
         """
         # count the times name equals an element in self.names.
-        return np.sum(name == np.asarray(self.names)) > 1
+        return sum(name == n for n in self.names) > 1
 
     def _format_native_types(self, **kwargs):
         return self.values
@@ -4514,6 +4537,10 @@ def format(self, space=2, sparsify=None, adjoin=True, names=False,
         else:
             return result_levels
 
+    def _to_safe_for_reshape(self):
+        """ convert to object if we are a categorical """
+        return self.set_levels([ i._to_safe_for_reshape() for i in self.levels ])
+
     def to_hierarchical(self, n_repeat, n_shuffle=1):
         """
         Return a MultiIndex reshaped to conform to the

pandas/core/index.py

@@ -1285,7 +1285,7 @@ def _has_valid_type(self, key, axis):
 
             def error():
                 if isnull(key):
-                    raise ValueError(
+                    raise TypeError(
                         "cannot use label indexing with a null key")
                 raise KeyError("the label [%s] is not in the [%s]" %
                                (key, self.obj._get_axis_name(axis)))