Merge branch 'main' into gh-130614-test-read

Author: barneygale

.github/CODEOWNERS

@@ -309,3 +309,6 @@ Doc/reference/                @willingc @AA-Turner
 # Colorize
 Lib/_colorize.py              @hugovk
 Lib/test/test__colorize.py    @hugovk
+
+# Fuzzing
+Modules/_xxtestfuzz/          @ammaraskar

.github/workflows/tail-call.yml

@@ -2,12 +2,14 @@ name: Tail calling interpreter
 on:
   pull_request:
     paths:
+      - '.github/workflows/tail-call.yml'
       - 'Python/bytecodes.c'
       - 'Python/ceval.c'
       - 'Python/ceval_macros.h'
       - 'Python/generated_cases.c.h'
   push:
     paths:
+      - '.github/workflows/tail-call.yml'
       - 'Python/bytecodes.c'
       - 'Python/ceval.c'
       - 'Python/ceval_macros.h'
@@ -35,7 +37,7 @@ jobs:
         target:
 # Un-comment as we add support for more platforms for tail-calling interpreters.
 #          - i686-pc-windows-msvc/msvc
-#          - x86_64-pc-windows-msvc/msvc
+          - x86_64-pc-windows-msvc/msvc
 #          - aarch64-pc-windows-msvc/msvc
           - x86_64-apple-darwin/clang
           - aarch64-apple-darwin/clang
@@ -48,9 +50,9 @@ jobs:
 #          - target: i686-pc-windows-msvc/msvc
 #            architecture: Win32
 #            runner: windows-latest
-#          - target: x86_64-pc-windows-msvc/msvc
-#            architecture: x64
-#            runner: windows-latest
+          - target: x86_64-pc-windows-msvc/msvc
+            architecture: x64
+            runner: windows-latest
 #          - target: aarch64-pc-windows-msvc/msvc
 #            architecture: ARM64
 #            runner: windows-latest
@@ -79,23 +81,31 @@ jobs:
 
       - name: Native Windows (debug)
         if: runner.os == 'Windows' && matrix.architecture != 'ARM64'
+        shell: cmd
         run: |
-          choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}.1.0
+          choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}.1.5
+          set PlatformToolset=clangcl
+          set LLVMToolsVersion=${{ matrix.llvm }}.1.5
+          set LLVMInstallDir=C:\Program Files\LLVM
           ./PCbuild/build.bat --tail-call-interp -d -p ${{ matrix.architecture }}
           ./PCbuild/rt.bat -d -p ${{ matrix.architecture }} -q --multiprocess 0 --timeout 4500 --verbose2 --verbose3
 
       # No tests (yet):
       - name: Emulated Windows (release)
         if: runner.os == 'Windows' && matrix.architecture == 'ARM64'
+        shell: cmd
         run: |
-          choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}.1.0
+          choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}.1.5
+          set PlatformToolset=clangcl
+          set LLVMToolsVersion=${{ matrix.llvm }}.1.5
+          set LLVMInstallDir=C:\Program Files\LLVM
           ./PCbuild/build.bat --tail-call-interp -p ${{ matrix.architecture }}
 
         # The `find` line is required as a result of https://github.com/actions/runner-images/issues/9966.
         # This is a bug in the macOS runner image where the pre-installed Python is installed in the same
         # directory as the Homebrew Python, which causes the build to fail for macos-13. This line removes
         # the symlink to the pre-installed Python so that the Homebrew Python is used instead.
-      - name: Native macOS (debug)
+      - name: Native macOS (release)
         if: runner.os == 'macOS'
         run: |
           brew update
@@ -104,16 +114,16 @@ jobs:
           export SDKROOT="$(xcrun --show-sdk-path)"
           export PATH="/opt/homebrew/opt/llvm/bin:$PATH"
           export PATH="/usr/local/opt/llvm/bin:$PATH"
-          CC=clang-19 ./configure --with-tail-call-interp --with-pydebug
+          CC=clang-19 ./configure --with-tail-call-interp
           make all --jobs 4
           ./python.exe -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3
 
-      - name: Native Linux (release)
+      - name: Native Linux (debug)
         if: runner.os == 'Linux' && matrix.target != 'free-threading'
         run: |
           sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }}
           export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH"
-          CC=clang-19 ./configure --with-tail-call-interp
+          CC=clang-19 ./configure --with-tail-call-interp --with-pydebug
           make all --jobs 4
           ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3
 

Doc/c-api/allocation.rst

@@ -35,6 +35,10 @@ Allocating Objects on the Heap
    The size of the memory allocation is determined from the
    :c:member:`~PyTypeObject.tp_basicsize` field of the type object.
 
+   Note that this function is unsuitable if *typeobj* has
+   :c:macro:`Py_TPFLAGS_HAVE_GC` set. For such objects,
+   use :c:func:`PyObject_GC_New` instead.
+
 
 .. c:macro:: PyObject_NewVar(TYPE, typeobj, size)
 
@@ -49,6 +53,10 @@ Allocating Objects on the Heap
    fields into the same allocation decreases the number of allocations,
    improving the memory management efficiency.
 
+   Note that this function is unsuitable if *typeobj* has
+   :c:macro:`Py_TPFLAGS_HAVE_GC` set. For such objects,
+   use :c:func:`PyObject_GC_NewVar` instead.
+
 
 .. c:function:: void PyObject_Del(void *op)
 

Doc/c-api/type.rst

@@ -456,6 +456,9 @@ The following functions and structs are used to create
       class need *in addition* to the superclass.
       Use :c:func:`PyObject_GetTypeData` to get a pointer to subclass-specific
       memory reserved this way.
+      For negative :c:member:`!basicsize`, Python will insert padding when
+      needed to meet :c:member:`~PyTypeObject.tp_basicsize`'s alignment
+      requirements.
 
       .. versionchanged:: 3.12
 

Doc/c-api/typeobj.rst

@@ -537,6 +537,9 @@ PyVarObject Slots
    initialized to zero. For :ref:`dynamically allocated type objects
    <heap-types>`, this field has a special internal meaning.
 
+   This field should be accessed using the :c:func:`Py_SIZE()` and
+   :c:func:`Py_SET_SIZE()` macros.
+
    **Inheritance:**
 
    This field is not inherited by subtypes.
@@ -587,47 +590,86 @@ and :c:data:`PyType_Type` effectively act as defaults.)
 
 
 .. c:member:: Py_ssize_t PyTypeObject.tp_basicsize
-             Py_ssize_t PyTypeObject.tp_itemsize
+              Py_ssize_t PyTypeObject.tp_itemsize
 
    These fields allow calculating the size in bytes of instances of the type.
 
    There are two kinds of types: types with fixed-length instances have a zero
-   :c:member:`~PyTypeObject.tp_itemsize` field, types with variable-length instances have a non-zero
-   :c:member:`~PyTypeObject.tp_itemsize` field.  For a type with fixed-length instances, all
-   instances have the same size, given in :c:member:`~PyTypeObject.tp_basicsize`.
+   :c:member:`!tp_itemsize` field, types with variable-length instances have a non-zero
+   :c:member:`!tp_itemsize` field.  For a type with fixed-length instances, all
+   instances have the same size, given in :c:member:`!tp_basicsize`.
+   (Exceptions to this rule can be made using
+   :c:func:`PyUnstable_Object_GC_NewWithExtraData`.)
 
    For a type with variable-length instances, the instances must have an
-   :c:member:`~PyVarObject.ob_size` field, and the instance size is :c:member:`~PyTypeObject.tp_basicsize` plus N
-   times :c:member:`~PyTypeObject.tp_itemsize`, where N is the "length" of the object.  The value of
-   N is typically stored in the instance's :c:member:`~PyVarObject.ob_size` field.  There are
-   exceptions:  for example, ints use a negative :c:member:`~PyVarObject.ob_size` to indicate a
-   negative number, and N is ``abs(ob_size)`` there.  Also, the presence of an
-   :c:member:`~PyVarObject.ob_size` field in the instance layout doesn't mean that the instance
-   structure is variable-length (for example, the structure for the list type has
-   fixed-length instances, yet those instances have a meaningful :c:member:`~PyVarObject.ob_size`
-   field).
-
-   The basic size includes the fields in the instance declared by the macro
-   :c:macro:`PyObject_HEAD` or :c:macro:`PyObject_VAR_HEAD` (whichever is used to
-   declare the instance struct) and this in turn includes the  :c:member:`~PyObject._ob_prev` and
-   :c:member:`~PyObject._ob_next` fields if they are present.  This means that the only correct
-   way to get an initializer for the :c:member:`~PyTypeObject.tp_basicsize` is to use the
-   ``sizeof`` operator on the struct used to declare the instance layout.
-   The basic size does not include the GC header size.
+   :c:member:`~PyVarObject.ob_size` field, and the instance size is
+   :c:member:`!tp_basicsize` plus N times :c:member:`!tp_itemsize`,
+   where N is the "length" of the object.
+
+   Functions like :c:func:`PyObject_NewVar` will take the value of N as an
+   argument, and store in the instance's :c:member:`~PyVarObject.ob_size` field.
+   Note that the :c:member:`~PyVarObject.ob_size` field may later be used for
+   other purposes. For example, :py:type:`int` instances use the bits of
+   :c:member:`~PyVarObject.ob_size` in an implementation-defined
+   way; the underlying storage and its size should be acessed using
+   :c:func:`PyLong_Export`.
+
+   .. note::
 
-   A note about alignment: if the variable items require a particular alignment,
-   this should be taken care of by the value of :c:member:`~PyTypeObject.tp_basicsize`.  Example:
-   suppose a type implements an array of ``double``. :c:member:`~PyTypeObject.tp_itemsize` is
-   ``sizeof(double)``. It is the programmer's responsibility that
-   :c:member:`~PyTypeObject.tp_basicsize` is a multiple of ``sizeof(double)`` (assuming this is the
-   alignment requirement for ``double``).
+      The :c:member:`~PyVarObject.ob_size` field should be accessed using
+      the :c:func:`Py_SIZE()` and :c:func:`Py_SET_SIZE()` macros.
 
-   For any type with variable-length instances, this field must not be ``NULL``.
+   Also, the presence of an :c:member:`~PyVarObject.ob_size` field in the
+   instance layout doesn't mean that the instance structure is variable-length.
+   For example, the :py:type:`list` type has fixed-length instances, yet those
+   instances have a :c:member:`~PyVarObject.ob_size` field.
+   (As with :py:type:`int`, avoid reading lists' :c:member:`!ob_size` directly.
+   Call :c:func:`PyList_Size` instead.)
+
+   The :c:member:`!tp_basicsize` includes size needed for data of the type's
+   :c:member:`~PyTypeObject.tp_base`, plus any extra data needed
+   by each instance.
+
+   The  correct way to set :c:member:`!tp_basicsize` is to use the
+   ``sizeof`` operator on the struct used to declare the instance layout.
+   This struct must include the struct used to declare the base type.
+   In other words, :c:member:`!tp_basicsize` must be greater than or equal
+   to the base's :c:member:`!tp_basicsize`.
+
+   Since every type is a subtype of :py:type:`object`, this struct must
+   include :c:type:`PyObject` or :c:type:`PyVarObject` (depending on
+   whether :c:member:`~PyVarObject.ob_size` should be included). These are
+   usually defined by the macro :c:macro:`PyObject_HEAD` or
+   :c:macro:`PyObject_VAR_HEAD`, respectively.
+
+   The basic size does not include the GC header size, as that header is not
+   part of :c:macro:`PyObject_HEAD`.
+
+   For cases where struct used to declare the base type is unknown,
+   see :c:member:`PyType_Spec.basicsize` and :c:func:`PyType_FromMetaclass`.
+
+   Notes about alignment:
+
+   - :c:member:`!tp_basicsize` must be a multiple of ``_Alignof(PyObject)``.
+     When using ``sizeof`` on a ``struct`` that includes
+     :c:macro:`PyObject_HEAD`, as recommended, the compiler ensures this.
+     When not using a C ``struct``, or when using compiler
+     extensions like ``__attribute__((packed))``, it is up to you.
+   - If the variable items require a particular alignment,
+     :c:member:`!tp_basicsize` and :c:member:`!tp_itemsize` must each be a
+     multiple of that alignment.
+     For example, if a type's variable part stores a ``double``, it is
+     your responsibility that both fields are a multiple of
+     ``_Alignof(double)``.
 
    **Inheritance:**
 
-   These fields are inherited separately by subtypes.  If the base type has a
-   non-zero :c:member:`~PyTypeObject.tp_itemsize`, it is generally not safe to set
+   These fields are inherited separately by subtypes.
+   (That is, if the field is set to zero, :c:func:`PyType_Ready` will copy
+   the value from the base type, indicating that the instances do not
+   need additional storage.)
+
+   If the base type has a non-zero :c:member:`~PyTypeObject.tp_itemsize`, it is generally not safe to set
    :c:member:`~PyTypeObject.tp_itemsize` to a different non-zero value in a subtype (though this
    depends on the implementation of the base type).
 

Doc/c-api/unicode.rst

@@ -614,6 +614,23 @@ APIs:
    decref'ing the returned objects.
 
 
+.. c:function:: void PyUnicode_Append(PyObject **p_left, PyObject *right)
+
+   Append the string *right* to the end of *p_left*.
+   *p_left* must point to a :term:`strong reference` to a Unicode object;
+   :c:func:`!PyUnicode_Append` releases ("steals") this reference.
+
+   On error, set *\*p_left* to ``NULL`` and set an exception.
+
+   On sucess, set *\*p_left* to a new strong reference to the result.
+
+
+.. c:function:: void PyUnicode_AppendAndDel(PyObject **p_left, PyObject *right)
+
+   The function is similar to :c:func:`PyUnicode_Append`, with the only
+   difference being that it decrements the reference count of *right* by one.
+
+
 .. c:function:: const char* PyUnicode_GetDefaultEncoding(void)
 
    Return the name of the default string encoding, ``"utf-8"``.

Doc/data/refcounts.dat

@@ -2770,6 +2770,14 @@ PyUnicode_FromFormatV:PyObject*::+1:
 PyUnicode_FromFormatV:const char*:format::
 PyUnicode_FromFormatV:va_list:args::
 
+PyUnicode_Append:void:::
+PyUnicode_Append:PyObject**:p_left:0:
+PyUnicode_Append:PyObject*:right::
+
+PyUnicode_AppendAndDel:void:::
+PyUnicode_AppendAndDel:PyObject**:p_left:0:
+PyUnicode_AppendAndDel:PyObject*:right:-1:
+
 PyUnicode_GetDefaultEncoding:const char*:::
 PyUnicode_GetDefaultEncoding::void::
 

Doc/howto/perf_profiling.rst

@@ -162,12 +162,12 @@ the :option:`!-X` option takes precedence over the environment variable.
 
 Example, using the environment variable::
 
-   $ PYTHONPERFSUPPORT=1 perf record -F 9999 -g -o perf.data python script.py
+   $ PYTHONPERFSUPPORT=1 perf record -F 9999 -g -o perf.data python my_script.py
    $ perf report -g -i perf.data
 
 Example, using the :option:`!-X` option::
 
-   $ perf record -F 9999 -g -o perf.data python -X perf script.py
+   $ perf record -F 9999 -g -o perf.data python -X perf my_script.py
    $ perf report -g -i perf.data
 
 Example, using the :mod:`sys` APIs in file :file:`example.py`:
@@ -236,7 +236,7 @@ When using the perf JIT mode, you need an extra step before you can run ``perf
 report``. You need to call the ``perf inject`` command to inject the JIT
 information into the ``perf.data`` file.::
 
-    $ perf record -F 9999 -g --call-graph dwarf -o perf.data python -Xperf_jit my_script.py
+    $ perf record -F 9999 -g -k 1 --call-graph dwarf -o perf.data python -Xperf_jit my_script.py
     $ perf inject -i perf.data --jit --output perf.jit.data
     $ perf report -g -i perf.jit.data
 

Doc/library/typing.rst

@@ -2771,7 +2771,7 @@ types.
 
       .. versionadded:: 3.13
 
-   See :pep:`589` for more examples and detailed rules of using ``TypedDict``.
+   See the `TypedDict <https://typing.python.org/en/latest/spec/typeddict.html#typeddict>`_ section in the typing documentation for more examples and detailed rules.
 
    .. versionadded:: 3.8
 

Doc/whatsnew/3.14.rst

@@ -297,7 +297,7 @@ For further information on how to build Python, see
 
 .. attention::
 
-    This section previously reported a 9-15% geomean speedup. This number has since been
+    This section previously reported a 9-15% geometric mean speedup. This number has since been
     cautiously revised down to 3-5%. While we expect performance results to be better
     than what we report, our estimates are more conservative due to a
     `compiler bug <https://github.com/llvm/llvm-project/issues/106846>`_ found in

Include/internal/pycore_condvar.h

@@ -31,7 +31,9 @@
 #define Py_HAVE_CONDVAR
 
 /* include windows if it hasn't been done before */
-#define WIN32_LEAN_AND_MEAN
+#ifndef WIN32_LEAN_AND_MEAN
+#  define WIN32_LEAN_AND_MEAN
+#endif
 #include <windows.h>              // CRITICAL_SECTION
 
 /* options */

Include/internal/pycore_debug_offsets.h

@@ -23,7 +23,7 @@ extern "C" {
    declaration                                      \
    _GENERATE_DEBUG_SECTION_LINUX(name)
 
-#if defined(MS_WINDOWS)
+#if defined(MS_WINDOWS) && !defined(__clang__)
 #define _GENERATE_DEBUG_SECTION_WINDOWS(name)                       \
    _Pragma(Py_STRINGIFY(section(Py_STRINGIFY(name), read, write))) \
    __declspec(allocate(Py_STRINGIFY(name)))

Include/internal/pycore_semaphore.h

@@ -10,7 +10,9 @@
 #include "pycore_pythread.h"      // _POSIX_SEMAPHORES
 
 #ifdef MS_WINDOWS
-#   define WIN32_LEAN_AND_MEAN
+#   ifndef WIN32_LEAN_AND_MEAN
+#       define WIN32_LEAN_AND_MEAN
+#   endif
 #   include <windows.h>
 #elif defined(HAVE_PTHREAD_H)
 #   include <pthread.h>