Merge branch master into master-slp

Implement the frame evaluation API aspect of PEP 523.
The outcome of this merge does not compile.

Author: Anselm Kruis

Doc/whatsnew/3.6.rst

@@ -97,6 +97,34 @@ Windows improvements:
 New Features
 ============
 
+.. _pep-523:
+
+PEP 523: Adding a frame evaluation API to CPython
+=================================================
+
+While Python provides extensive support to customize how code
+executes, one place it has not done so is in the evaluation of frame
+objects. If you wanted some way to intercept frame evaluation in
+Python there really wasn't any way without directly manipulating
+function pointers for defined functions.
+
+:pep:`523` changes this by providing an API to make frame
+evaluation pluggable at the C level. This will allow for tools such
+as debuggers and JITs to intercept frame evaluation before the
+execution of Python code begins. This enables the use of alternative
+evaluation implementations for Python code, tracking frame
+evaluation, etc.
+
+This API is not part of the limited C API and is marked as private to
+signal that usage of this API is expected to be limited and only
+applicable to very select, low-level use-cases.
+
+.. seealso::
+
+    :pep:`523` - Adding a frame evaluation API to CPython
+       PEP written by Brett Cannon and Dino Viehland.
+
+
 .. _pep-519:
 
 PEP 519: Adding a file system path protocol

Include/ceval.h

@@ -119,6 +119,9 @@ PyAPI_FUNC(const char *) PyEval_GetFuncDesc(PyObject *);
 PyAPI_FUNC(PyObject *) PyEval_GetCallStats(PyObject *);
 PyAPI_FUNC(PyObject *) PyEval_EvalFrame(struct _frame *);
 PyAPI_FUNC(PyObject *) PyEval_EvalFrameEx(struct _frame *f, int exc);
+#ifndef Py_LIMITED_API
+PyAPI_FUNC(PyObject *) _PyEval_EvalFrameDefault(struct _frame *f, int exc);
+#endif
 
 #ifdef STACKLESS
 PyAPI_FUNC(PyObject *) PyEval_EvalFrame_slp(struct _frame *f, int exc, PyObject *retval);

Include/pystate.h

@@ -15,10 +15,13 @@ extern "C" {
 
 struct _ts; /* Forward */
 struct _is; /* Forward */
+struct _frame; /* Forward declaration for PyFrameObject. */
 
 #ifdef Py_LIMITED_API
 typedef struct _is PyInterpreterState;
 #else
+typedef PyObject* (*_PyFrameEvalFunction)(struct _frame *, int);
+
 typedef struct _is {
 
     struct _is *next;
@@ -45,14 +48,14 @@ typedef struct _is {
 
     PyObject *builtins_copy;
     PyObject *import_func;
+    /* Initialized to PyEval_EvalFrameDefault(). */
+    _PyFrameEvalFunction eval_frame;
 } PyInterpreterState;
 #endif
 
 
 /* State unique per thread */
 
-struct _frame; /* Avoid including frameobject.h */
-
 #ifndef Py_LIMITED_API
 /* Py_tracefunc return -1 when raising an exception, or 0 for success. */
 typedef int (*Py_tracefunc)(PyObject *, struct _frame *, int, PyObject *);

Misc/NEWS

@@ -17,7 +17,9 @@ Core and Builtins
   restriction: in beta 2, backslashes will only be disallowed inside
   the braces (where the expressions are). This is a breaking change
   from the 3.6 alpha releases.
-  
+
+- Implement the frame evaluation part of PEP 523.
+
 - Issue #27870: A left shift of zero by a large integer no longer attempts
   to allocate large amounts of memory.
 

Python/ceval.c

@@ -802,6 +802,13 @@ slp_eval_frame_value(PyFrameObject *f, int throwflag, PyObject *retval)
 #else
 PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
 #endif
+{
+    PyThreadState *tstate = PyThreadState_GET();
+    return tstate->interp->eval_frame(f, throwflag);
+}
+
+PyObject *
+_PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag)
 {
 #ifdef DXPAIRS
     int lastopcode = 0;

Python/pystate.c

@@ -109,6 +109,7 @@ PyInterpreterState_New(void)
         interp->fscodec_initialized = 0;
         interp->importlib = NULL;
         interp->import_func = NULL;
+        interp->eval_frame = _PyEval_EvalFrameDefault;
 #ifdef HAVE_DLOPEN
 #if HAVE_DECL_RTLD_NOW
         interp->dlopenflags = RTLD_NOW;