diff --git a/Include/cpython/pyframe.h b/Include/cpython/pyframe.h index 51529763923ec3..e9c289d69be694 100644 --- a/Include/cpython/pyframe.h +++ b/Include/cpython/pyframe.h @@ -22,24 +22,26 @@ PyAPI_FUNC(PyObject*) PyFrame_GetVarString(PyFrameObject *frame, const char *nam /* The following functions are for use by debuggers and other tools * implementing custom frame evaluators with PEP 523. */ +struct _PyInterpreterFrameCore; struct _PyInterpreterFrame; /* Returns the code object of the frame (strong reference). * Does not raise an exception. */ -PyAPI_FUNC(PyObject *) PyUnstable_InterpreterFrame_GetCode(struct _PyInterpreterFrame *frame); +PyAPI_FUNC(PyObject *) PyUnstable_InterpreterFrame_GetCode(struct _PyInterpreterFrameCore *frame); /* Returns a byte offset into the last executed instruction. * Does not raise an exception. */ -PyAPI_FUNC(int) PyUnstable_InterpreterFrame_GetLasti(struct _PyInterpreterFrame *frame); +PyAPI_FUNC(int) PyUnstable_InterpreterFrame_GetLasti(struct _PyInterpreterFrameCore *frame); /* Returns the currently executing line number, or -1 if there is no line number. * Does not raise an exception. */ -PyAPI_FUNC(int) PyUnstable_InterpreterFrame_GetLine(struct _PyInterpreterFrame *frame); +PyAPI_FUNC(int) PyUnstable_InterpreterFrame_GetLine(struct _PyInterpreterFrameCore *frame); #define PyUnstable_EXECUTABLE_KIND_SKIP 0 #define PyUnstable_EXECUTABLE_KIND_PY_FUNCTION 1 #define PyUnstable_EXECUTABLE_KIND_BUILTIN_FUNCTION 3 #define PyUnstable_EXECUTABLE_KIND_METHOD_DESCRIPTOR 4 -#define PyUnstable_EXECUTABLE_KINDS 5 +#define PyUnstable_EXECUTABLE_KIND_JIT 5 +#define PyUnstable_EXECUTABLE_KINDS 6 PyAPI_DATA(const PyTypeObject *) const PyUnstable_ExecutableKinds[PyUnstable_EXECUTABLE_KINDS+1]; diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 22df26bd37a5c5..24b7528792583e 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -133,16 +133,16 @@ struct _ts { int what_event; /* The event currently being monitored, if any. */ /* Pointer to currently executing frame. */ - struct _PyInterpreterFrame *current_frame; + struct _PyInterpreterFrameCore *current_frame; /* Pointer to the base frame (bottommost sentinel frame). Used by profilers to validate complete stack unwinding. Points to the embedded base_frame in _PyThreadStateImpl. - The frame is embedded there rather than here because _PyInterpreterFrame + The frame is embedded there rather than here because _PyInterpreterFrameCore is defined in internal headers that cannot be exposed in the public API. */ - struct _PyInterpreterFrame *base_frame; + struct _PyInterpreterFrameCore *base_frame; - struct _PyInterpreterFrame *last_profiled_frame; + struct _PyInterpreterFrameCore *last_profiled_frame; Py_tracefunc c_profilefunc; Py_tracefunc c_tracefunc; diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index f6bdba3e9916c0..e0e16c274401e6 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -14,7 +14,7 @@ extern "C" { #include "pycore_interp.h" // PyInterpreterState.eval_frame #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_stats.h" // EVAL_CALL_STAT_INC() -#include "pycore_typedefs.h" // _PyInterpreterFrame +#include "pycore_typedefs.h" // _PyInterpreterFrameCore /* Forward declarations */ @@ -268,7 +268,7 @@ PyAPI_FUNC(int) _Py_ReachedRecursionLimitWithMargin( static inline void _Py_LeaveRecursiveCall(void) { } -extern _PyInterpreterFrame* _PyEval_GetFrame(void); +extern _PyInterpreterFrameCore* _PyEval_GetFrame(void); extern PyObject * _PyEval_GetGlobalsFromRunningMain(PyThreadState *); extern int _PyEval_EnsureBuiltins( @@ -310,7 +310,7 @@ PyAPI_FUNC(void) _PyEval_FormatExcCheckArg(PyThreadState *tstate, PyObject *exc, PyAPI_FUNC(void) _PyEval_FormatExcUnbound(PyThreadState *tstate, PyCodeObject *co, int oparg); PyAPI_FUNC(void) _PyEval_FormatKwargsError(PyThreadState *tstate, PyObject *func, PyObject *kwargs); PyAPI_FUNC(PyObject *) _PyEval_ImportFrom(PyThreadState *, PyObject *, PyObject *); -PyAPI_FUNC(PyObject *) _PyEval_ImportName(PyThreadState *, _PyInterpreterFrame *, PyObject *, PyObject *, PyObject *); +PyAPI_FUNC(PyObject *) _PyEval_ImportName(PyThreadState *, _PyInterpreterFrameCore *, PyObject *, PyObject *, PyObject *); PyAPI_FUNC(PyObject *)_PyEval_MatchClass(PyThreadState *tstate, PyObject *subject, PyObject *type, Py_ssize_t nargs, PyObject *kwargs); PyAPI_FUNC(PyObject *)_PyEval_MatchKeys(PyThreadState *tstate, PyObject *map, PyObject *keys); PyAPI_FUNC(void) _PyEval_MonitorRaise(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr); diff --git a/Include/internal/pycore_debug_offsets.h b/Include/internal/pycore_debug_offsets.h index 66f14e69f33f44..a4fe5b2c140b3d 100644 --- a/Include/internal/pycore_debug_offsets.h +++ b/Include/internal/pycore_debug_offsets.h @@ -297,11 +297,11 @@ typedef struct _Py_DebugOffsets { }, \ .interpreter_frame = { \ .size = sizeof(_PyInterpreterFrame), \ - .previous = offsetof(_PyInterpreterFrame, previous), \ - .executable = offsetof(_PyInterpreterFrame, f_executable), \ + .previous = offsetof(_PyInterpreterFrameCore, previous), \ + .executable = offsetof(_PyInterpreterFrameCore, f_executable), \ .instr_ptr = offsetof(_PyInterpreterFrame, instr_ptr), \ .localsplus = offsetof(_PyInterpreterFrame, localsplus), \ - .owner = offsetof(_PyInterpreterFrame, owner), \ + .owner = offsetof(_PyInterpreterFrameCore, owner), \ .stackpointer = offsetof(_PyInterpreterFrame, stackpointer), \ .tlbc_index = _Py_Debug_interpreter_frame_tlbc_index, \ }, \ diff --git a/Include/internal/pycore_genobject.h b/Include/internal/pycore_genobject.h index a3badb59cb771a..93c7806dae1866 100644 --- a/Include/internal/pycore_genobject.h +++ b/Include/internal/pycore_genobject.h @@ -8,7 +8,8 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_interpframe_structs.h" // _PyGenObject +#include "pycore_interpframe.h" // _PyFrame_Core +#include "pycore_interpframe_structs.h" // _PyInterpreterFrame #include // offsetof() @@ -16,7 +17,7 @@ extern "C" { static inline PyGenObject *_PyGen_GetGeneratorFromFrame(_PyInterpreterFrame *frame) { - assert(frame->owner == FRAME_OWNED_BY_GENERATOR); + assert(_PyFrame_Core(frame)->owner & FRAME_OWNED_BY_GENERATOR); size_t offset_in_gen = offsetof(PyGenObject, gi_iframe); return (PyGenObject *)(((char *)frame) - offset_in_gen); } diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index 723657e4cef10d..bdab84ed35ca12 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -224,7 +224,7 @@ struct _gc_runtime_state { int collecting; // The frame that started the current collection. It might be NULL even when // collecting (if no Python frame is running): - _PyInterpreterFrame *frame; + _PyInterpreterFrameCore *frame; /* list of uncollectable objects */ PyObject *garbage; /* a list of callbacks to be invoked when collection is performed */ diff --git a/Include/internal/pycore_interpframe.h b/Include/internal/pycore_interpframe.h index 14e2f245834dca..be3584003cf261 100644 --- a/Include/internal/pycore_interpframe.h +++ b/Include/internal/pycore_interpframe.h @@ -14,28 +14,78 @@ extern "C" { #endif +#define _PyInterpreterFrame_LASTI_FULL(IF) \ + ((int)((IF)->instr_ptr - _PyFrame_GetBytecodeFull((IF)))) + #define _PyInterpreterFrame_LASTI(IF) \ - ((int)((IF)->instr_ptr - _PyFrame_GetBytecode((IF)))) + ((int)(_PyFrame_EnsureFrameFullyInitialized(IF)->instr_ptr - _PyFrame_GetBytecode((IF)))) -static inline PyCodeObject *_PyFrame_GetCode(_PyInterpreterFrame *f) { - assert(!PyStackRef_IsNull(f->f_executable)); - PyObject *executable = PyStackRef_AsPyObjectBorrow(f->f_executable); + +// Safely cast a _PyInterpreterFrame to _PyInterpreterFrameCore +static inline _PyInterpreterFrameCore * +_PyFrame_Core(_PyInterpreterFrame *frame) +{ + return &frame->core; +} + +// Cast a _PyInterpreterFrameCore to _PyInterpreterFrame validating that it +// is a fully initialized frame that doesn't need _PyFrame_InitializeExternalFrame. +static inline _PyInterpreterFrame * +_PyFrame_Full(_PyInterpreterFrameCore *frame) +{ + assert(frame->owner == FRAME_OWNED_BY_THREAD || + frame->owner & FRAME_OWNED_BY_GENERATOR || + frame->owner == FRAME_OWNED_BY_FRAME_OBJECT); + return (_PyInterpreterFrame *)frame; +} + +PyAPI_DATA(PyTypeObject) PyUnstable_ExternalExecutable_Type; + +#define PyUnstable_ExternalExecutable_Check(op) Py_IS_TYPE((op), &PyUnstable_ExternalExecutable_Type) + +// Initialize a potentially external frame and make it safe to access the +// all of the members of the returned _PyInterpreterFrame. The returned +// value will be the same address as the passed in pointer. +PyAPI_FUNC(_PyInterpreterFrame *) _PyFrame_InitializeExternalFrame(_PyInterpreterFrameCore *frame); + +PyAPI_FUNC(PyObject *) PyUnstable_MakeExternalExecutable(_PyFrame_Reifier reifier, PyCodeObject *code, PyObject *state); + +static bool _PyFrame_IsExternalFrame(_PyInterpreterFrameCore *frame) +{ + return frame->owner & FRAME_OWNED_EXTERNALLY; +} + +static inline _PyInterpreterFrame * +_PyFrame_EnsureFrameFullyInitialized(_PyInterpreterFrameCore *frame) +{ + if (_PyFrame_IsExternalFrame(frame)) { + return _PyFrame_InitializeExternalFrame(frame); + } + return _PyFrame_Full(frame); +} + +static inline PyCodeObject *_PyFrame_GetCodeFull(_PyInterpreterFrame *f) { + assert(!PyStackRef_IsNull(_PyFrame_Core(f)->f_executable)); + PyObject *executable = PyStackRef_AsPyObjectBorrow(_PyFrame_Core(f)->f_executable); assert(PyCode_Check(executable)); return (PyCodeObject *)executable; } +static inline PyCodeObject *_PyFrame_GetCode(_PyInterpreterFrameCore *f) { + if (f->owner & FRAME_OWNED_EXTERNALLY) { + PyObject *executable = PyStackRef_AsPyObjectBorrow(f->f_executable); + assert(PyUnstable_ExternalExecutable_Check(executable)); + return ((PyUnstable_PyExternalExecutable *)executable)->ef_code; + } + return _PyFrame_GetCodeFull(_PyFrame_Full(f)); +} + // Similar to _PyFrame_GetCode(), but return NULL if the frame is invalid or // freed. Used by dump_frame() in Python/traceback.c. The function uses // heuristics to detect freed memory, it's not 100% reliable. static inline PyCodeObject* _Py_NO_SANITIZE_THREAD -_PyFrame_SafeGetCode(_PyInterpreterFrame *f) +_PyFrame_SafeGetCode(_PyInterpreterFrameCore *f) { - // globals and builtins may be NULL on a legit frame, but it's unlikely. - // It's more likely that it's a sign of an invalid frame. - if (f->f_globals == NULL || f->f_builtins == NULL) { - return NULL; - } - if (PyStackRef_IsNull(f->f_executable)) { return NULL; } @@ -48,6 +98,18 @@ _PyFrame_SafeGetCode(_PyInterpreterFrame *f) if (_PyObject_IsFreed(executable)) { return NULL; } + if (_PyFrame_IsExternalFrame(f)) { + executable = (PyObject *)((PyUnstable_PyExternalExecutable *)executable)->ef_code; + if (_PyObject_IsFreed(executable)) { + return NULL; + } + } else { + // globals and builtins may be NULL on a legit frame, but it's unlikely. + // It's more likely that it's a sign of an invalid frame. + if (_PyFrame_Full(f)->f_globals == NULL || _PyFrame_Full(f)->f_builtins == NULL) { + return NULL; + } + } if (!PyCode_Check(executable)) { return NULL; } @@ -55,15 +117,31 @@ _PyFrame_SafeGetCode(_PyInterpreterFrame *f) } static inline _Py_CODEUNIT * -_PyFrame_GetBytecode(_PyInterpreterFrame *f) +_PyFrame_GetBytecodeFull(_PyInterpreterFrame *frame) +{ +#ifdef Py_GIL_DISABLED + PyCodeObject *co = _PyFrame_GetCodeFull(frame); + + _PyCodeArray *tlbc = _PyCode_GetTLBCArray(co); + assert(frame->tlbc_index >= 0 && frame->tlbc_index < tlbc->size); + return (_Py_CODEUNIT *)tlbc->entries[frame->tlbc_index]; +#else + return _PyCode_CODE(_PyFrame_GetCodeFull(frame)); +#endif +} + +static inline _Py_CODEUNIT * +_PyFrame_GetBytecode(_PyInterpreterFrameCore *frame) { #ifdef Py_GIL_DISABLED - PyCodeObject *co = _PyFrame_GetCode(f); + PyCodeObject *co = _PyFrame_GetCode(frame); + + _PyInterpreterFrame *f = _PyFrame_EnsureFrameFullyInitialized(frame); _PyCodeArray *tlbc = _PyCode_GetTLBCArray(co); assert(f->tlbc_index >= 0 && f->tlbc_index < tlbc->size); return (_Py_CODEUNIT *)tlbc->entries[f->tlbc_index]; #else - return _PyCode_CODE(_PyFrame_GetCode(f)); + return _PyCode_CODE(_PyFrame_GetCode(frame)); #endif } @@ -71,16 +149,17 @@ _PyFrame_GetBytecode(_PyInterpreterFrame *f) // frame is invalid or freed. Used by dump_frame() in Python/traceback.c. The // function uses heuristics to detect freed memory, it's not 100% reliable. static inline int _Py_NO_SANITIZE_THREAD -_PyFrame_SafeGetLasti(struct _PyInterpreterFrame *f) +_PyFrame_SafeGetLasti(struct _PyInterpreterFrameCore *frame) { // Code based on _PyFrame_GetBytecode() but replace _PyFrame_GetCode() // with _PyFrame_SafeGetCode(). - PyCodeObject *co = _PyFrame_SafeGetCode(f); + PyCodeObject *co = _PyFrame_SafeGetCode(frame); if (co == NULL) { return -1; } _Py_CODEUNIT *bytecode; + _PyInterpreterFrame *f = _PyFrame_EnsureFrameFullyInitialized(frame); #ifdef Py_GIL_DISABLED _PyCodeArray *tlbc = _PyCode_GetTLBCArray(co); assert(f->tlbc_index >= 0 && f->tlbc_index < tlbc->size); @@ -92,14 +171,15 @@ _PyFrame_SafeGetLasti(struct _PyInterpreterFrame *f) return (int)(f->instr_ptr - bytecode) * sizeof(_Py_CODEUNIT); } -static inline PyFunctionObject *_PyFrame_GetFunction(_PyInterpreterFrame *f) { +static inline PyFunctionObject *_PyFrame_GetFunction(_PyInterpreterFrameCore *frame) { + _PyInterpreterFrame *f = _PyFrame_EnsureFrameFullyInitialized(frame); PyObject *func = PyStackRef_AsPyObjectBorrow(f->f_funcobj); assert(PyFunction_Check(func)); return (PyFunctionObject *)func; } static inline _PyStackRef *_PyFrame_Stackbase(_PyInterpreterFrame *f) { - return (f->localsplus + _PyFrame_GetCode(f)->co_nlocalsplus); + return (f->localsplus + _PyFrame_GetCodeFull(f)->co_nlocalsplus); } static inline _PyStackRef _PyFrame_StackPeek(_PyInterpreterFrame *f) { @@ -132,10 +212,10 @@ _PyFrame_NumSlotsForCodeObject(PyCodeObject *code) static inline void _PyFrame_Copy(_PyInterpreterFrame *src, _PyInterpreterFrame *dest) { - dest->f_executable = PyStackRef_MakeHeapSafe(src->f_executable); + _PyFrame_Core(dest)->f_executable = PyStackRef_MakeHeapSafe(_PyFrame_Core(src)->f_executable); // Don't leave a dangling pointer to the old frame when creating generators // and coroutines: - dest->previous = NULL; + _PyFrame_Core(dest)->previous = NULL; dest->f_funcobj = PyStackRef_MakeHeapSafe(src->f_funcobj); dest->f_globals = src->f_globals; dest->f_builtins = src->f_builtins; @@ -181,11 +261,12 @@ _PyFrame_InitializeTLBC(PyThreadState *tstate, _PyInterpreterFrame *frame, static inline void _PyFrame_Initialize( PyThreadState *tstate, _PyInterpreterFrame *frame, _PyStackRef func, - PyObject *locals, PyCodeObject *code, int null_locals_from, _PyInterpreterFrame *previous) + PyObject *locals, PyCodeObject *code, int null_locals_from, _PyInterpreterFrameCore *previous) { - frame->previous = previous; + _PyFrame_Core(frame)->previous = previous; + _PyFrame_Core(frame)->f_executable = PyStackRef_FromPyObjectNew(code); + _PyFrame_Core(frame)->owner = FRAME_OWNED_BY_THREAD; frame->f_funcobj = func; - frame->f_executable = PyStackRef_FromPyObjectNew(code); PyFunctionObject *func_obj = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(func); frame->f_builtins = func_obj->func_builtins; frame->f_globals = func_obj->func_globals; @@ -199,7 +280,6 @@ _PyFrame_Initialize( frame->instr_ptr = _PyCode_CODE(code); #endif frame->return_offset = 0; - frame->owner = FRAME_OWNED_BY_THREAD; frame->visited = 0; #ifdef Py_DEBUG frame->lltrace = 0; @@ -252,18 +332,20 @@ _PyFrame_SetStackPointer(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer) * from other threads for sys._current_frames() and similar APIs. */ static inline bool _Py_NO_SANITIZE_THREAD -_PyFrame_IsIncomplete(_PyInterpreterFrame *frame) +_PyFrame_IsIncomplete(_PyInterpreterFrameCore *frame) { if (frame->owner >= FRAME_OWNED_BY_INTERPRETER) { return true; + } else if (frame->owner & (FRAME_OWNED_BY_GENERATOR|FRAME_OWNED_EXTERNALLY)) { + return false; } - return frame->owner != FRAME_OWNED_BY_GENERATOR && - frame->instr_ptr < _PyFrame_GetBytecode(frame) + - _PyFrame_GetCode(frame)->_co_firsttraceable; + _PyInterpreterFrame *full = (_PyInterpreterFrame *)frame; + return full->instr_ptr < _PyFrame_GetBytecodeFull(full) + + _PyFrame_GetCodeFull(full)->_co_firsttraceable; } -static inline _PyInterpreterFrame * -_PyFrame_GetFirstComplete(_PyInterpreterFrame *frame) +static inline _PyInterpreterFrameCore * +_PyFrame_GetFirstComplete(_PyInterpreterFrameCore *frame) { while (frame && _PyFrame_IsIncomplete(frame)) { frame = frame->previous; @@ -271,12 +353,56 @@ _PyFrame_GetFirstComplete(_PyInterpreterFrame *frame) return frame; } +#if Py_DEBUG + +static inline bool _Py_NO_SANITIZE_THREAD +_PyFrame_IsIncompleteOrUninitialized(_PyInterpreterFrameCore *frame) +{ + if (frame->owner >= FRAME_OWNED_BY_INTERPRETER || _PyFrame_IsExternalFrame(frame)) { + return true; + } + _PyInterpreterFrame *full = _PyFrame_Full(frame); + return !(frame->owner & FRAME_OWNED_BY_GENERATOR) && + full->instr_ptr < _PyFrame_GetBytecode(frame) + + _PyFrame_GetCode(frame)->_co_firsttraceable; +} + static inline _PyInterpreterFrame * +_PyFrame_GetFirstCompleteInitialized(_PyInterpreterFrameCore *frame) +{ + while (frame && _PyFrame_IsIncompleteOrUninitialized(frame)) { + frame = frame->previous; + } + return frame != NULL ? _PyFrame_Full(frame) : NULL; +} + +static inline bool +_PyFrame_StackpointerSaved(void) +{ + PyThreadState *tstate = PyThreadState_GET(); + return _PyFrame_GetFirstCompleteInitialized(tstate->current_frame) == NULL || + _PyFrame_GetFirstCompleteInitialized(tstate->current_frame)->stackpointer != NULL; +} + +#endif + + +static inline _PyInterpreterFrameCore * _PyThreadState_GetFrame(PyThreadState *tstate) { return _PyFrame_GetFirstComplete(tstate->current_frame); } +static inline PyObject * +_PyFrame_GetGlobals(_PyInterpreterFrameCore *frame) { + return _PyFrame_EnsureFrameFullyInitialized(frame)->f_globals; +} + +static inline PyObject * +_PyFrame_GetBuiltins(_PyInterpreterFrameCore *frame) { + return _PyFrame_EnsureFrameFullyInitialized(frame)->f_builtins; +} + /* For use by _PyFrame_GetFrameObject Do not call directly. */ PyAPI_FUNC(PyFrameObject *) @@ -286,10 +412,9 @@ _PyFrame_MakeAndSetFrameObject(_PyInterpreterFrame *frame); * creating it if necessary. * Returns a borrowed reference */ static inline PyFrameObject * -_PyFrame_GetFrameObject(_PyInterpreterFrame *frame) +_PyFrame_GetFrameObjectFull(_PyInterpreterFrame *frame) { - - assert(!_PyFrame_IsIncomplete(frame)); + assert(!_PyFrame_IsIncomplete(_PyFrame_Core(frame))); PyFrameObject *res = frame->frame_obj; if (res != NULL) { return res; @@ -297,6 +422,15 @@ _PyFrame_GetFrameObject(_PyInterpreterFrame *frame) return _PyFrame_MakeAndSetFrameObject(frame); } +/* Gets the PyFrameObject for this frame, lazily + * creating it if necessary. + * Returns a borrowed reference */ +static inline PyFrameObject * +_PyFrame_GetFrameObject(_PyInterpreterFrameCore *f) +{ + return _PyFrame_GetFrameObjectFull(_PyFrame_EnsureFrameFullyInitialized(f)); +} + void _PyFrame_ClearLocals(_PyInterpreterFrame *frame); @@ -309,7 +443,7 @@ _PyFrame_ClearLocals(_PyInterpreterFrame *frame); * take should be set to 1 for heap allocated * frames like the ones in generators and coroutines. */ -void +PyAPI_FUNC(void) _PyFrame_ClearExceptCode(_PyInterpreterFrame * frame); int @@ -319,7 +453,7 @@ bool _PyFrame_HasHiddenLocals(_PyInterpreterFrame *frame); PyObject * -_PyFrame_GetLocals(_PyInterpreterFrame *frame); +_PyFrame_GetLocals(_PyInterpreterFrameCore *frame); static inline bool _PyThreadState_HasStackSpace(PyThreadState *tstate, int size) @@ -336,7 +470,7 @@ _PyThreadState_HasStackSpace(PyThreadState *tstate, int size) extern _PyInterpreterFrame * _PyThreadState_PushFrame(PyThreadState *tstate, size_t size); -PyAPI_FUNC(void) _PyThreadState_PopFrame(PyThreadState *tstate, _PyInterpreterFrame *frame); +PyAPI_FUNC(void) _PyThreadState_PopFrame(PyThreadState *tstate, _PyInterpreterFrameCore *frame); /* Pushes a frame without checking for space. * Must be guarded by _PyThreadState_HasStackSpace() @@ -351,22 +485,23 @@ _PyFrame_PushUnchecked(PyThreadState *tstate, _PyStackRef func, int null_locals_ tstate->datastack_top += code->co_framesize; assert(tstate->datastack_top < tstate->datastack_limit); _PyFrame_Initialize(tstate, new_frame, func, NULL, code, null_locals_from, - previous); + _PyFrame_Core(previous)); return new_frame; } /* Pushes a trampoline frame without checking for space. * Must be guarded by _PyThreadState_HasStackSpace() */ static inline _PyInterpreterFrame * -_PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int stackdepth, _PyInterpreterFrame * previous) +_PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int stackdepth, _PyInterpreterFrameCore * previous) { CALL_STAT_INC(frames_pushed); _PyInterpreterFrame *frame = (_PyInterpreterFrame *)tstate->datastack_top; tstate->datastack_top += code->co_framesize; assert(tstate->datastack_top < tstate->datastack_limit); - frame->previous = previous; + _PyFrame_Core(frame)->previous = previous; + _PyFrame_Core(frame)->f_executable = PyStackRef_FromPyObjectNew(code); + _PyFrame_Core(frame)->owner = FRAME_OWNED_BY_THREAD; frame->f_funcobj = PyStackRef_None; - frame->f_executable = PyStackRef_FromPyObjectNew(code); #ifdef Py_DEBUG frame->f_builtins = NULL; frame->f_globals = NULL; @@ -380,7 +515,6 @@ _PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int #else frame->instr_ptr = _PyCode_CODE(code); #endif - frame->owner = FRAME_OWNED_BY_THREAD; frame->visited = 0; #ifdef Py_DEBUG frame->lltrace = 0; @@ -393,11 +527,11 @@ PyAPI_FUNC(_PyInterpreterFrame *) _PyEvalFramePushAndInit(PyThreadState *tstate, _PyStackRef func, PyObject *locals, _PyStackRef const *args, size_t argcount, PyObject *kwnames, - _PyInterpreterFrame *previous); + _PyInterpreterFrameCore *previous); PyAPI_FUNC(_PyInterpreterFrame *) _PyEvalFramePushAndInit_Ex(PyThreadState *tstate, _PyStackRef func, - PyObject *locals, Py_ssize_t nargs, PyObject *callargs, PyObject *kwargs, _PyInterpreterFrame *previous); + PyObject *locals, Py_ssize_t nargs, PyObject *callargs, PyObject *kwargs, _PyInterpreterFrameCore *previous); #ifdef __cplusplus } diff --git a/Include/internal/pycore_interpframe_structs.h b/Include/internal/pycore_interpframe_structs.h index 38510685f4093c..d18a77fd039201 100644 --- a/Include/internal/pycore_interpframe_structs.h +++ b/Include/internal/pycore_interpframe_structs.h @@ -20,15 +20,30 @@ extern "C" { #endif enum _frameowner { - FRAME_OWNED_BY_THREAD = 0, - FRAME_OWNED_BY_GENERATOR = 1, - FRAME_OWNED_BY_FRAME_OBJECT = 2, - FRAME_OWNED_BY_INTERPRETER = 3, + // The frame is allocated on per-thread memory that will be freed or transferred when + // the frame unwinds. + FRAME_OWNED_BY_THREAD = 0x00, + // The frame is allocated in a generator and may out-live the execution. + FRAME_OWNED_BY_GENERATOR = 0x01, + // A flag which indicates the frame is owned externally. May be combined with + // FRAME_OWNED_BY_THREAD or FRAME_OWNED_BY_GENERATOR. The frame may only have + // _PyInterpreterFrameFields. To access other fields and ensure they are up to + // date _PyFrame_EnsureFrameFullyInitialized must be called first. + FRAME_OWNED_EXTERNALLY = 0x02, + // The frame is owned by the frame object (indicating the frame has unwound). + FRAME_OWNED_BY_FRAME_OBJECT = 0x04, + // The frame is a sentinel frame for entry to the interpreter loop + FRAME_OWNED_BY_INTERPRETER = 0x08, }; -struct _PyInterpreterFrame { +struct _PyInterpreterFrameCore { _PyStackRef f_executable; /* Deferred or strong reference (code object or None) */ - struct _PyInterpreterFrame *previous; + struct _PyInterpreterFrameCore *previous; + char owner; +}; + +struct _PyInterpreterFrame { + _PyInterpreterFrameCore core; _PyStackRef f_funcobj; /* Deferred or strong reference. Only valid if not on C stack */ PyObject *f_globals; /* Borrowed reference. Only valid if not on C stack */ PyObject *f_builtins; /* Borrowed reference. Only valid if not on C stack */ @@ -41,7 +56,6 @@ struct _PyInterpreterFrame { int32_t tlbc_index; #endif uint16_t return_offset; /* Only relevant during a function call */ - char owner; #ifdef Py_DEBUG uint8_t visited:1; uint8_t lltrace:7; @@ -85,6 +99,15 @@ struct _PyAsyncGenObject { _PyGenObject_HEAD(ag) }; +typedef struct _PyInterpreterFrame * (*_PyFrame_Reifier)(struct _PyInterpreterFrameCore *, PyObject *reifier); + +typedef struct { + PyObject_HEAD + PyCodeObject *ef_code; + PyObject *ef_state; + _PyFrame_Reifier ef_reifier; +} PyUnstable_PyExternalExecutable; + #undef _PyGenObject_HEAD diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 2ee518fb82f301..941c4f8922c9ef 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -8,7 +8,7 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_typedefs.h" // _PyInterpreterFrame +#include "pycore_typedefs.h" // _PyInterpreterFrameCore #include "pycore_uop.h" // _PyUOpInstruction #include "pycore_uop_ids.h" #include "pycore_stackref.h" // _PyStackRef diff --git a/Include/internal/pycore_tstate.h b/Include/internal/pycore_tstate.h index 64b90710b8e664..f022b65511c4f6 100644 --- a/Include/internal/pycore_tstate.h +++ b/Include/internal/pycore_tstate.h @@ -33,7 +33,7 @@ typedef struct _PyThreadStateImpl { // Embedded base frame - sentinel at the bottom of the frame stack. // Used by profiling/sampling to detect incomplete stack traces. - _PyInterpreterFrame base_frame; + _PyInterpreterFrameCore base_frame; // The reference count field is used to synchronize deallocation of the // thread state during runtime finalization. diff --git a/Include/internal/pycore_typedefs.h b/Include/internal/pycore_typedefs.h index d330cc4dc2b8fa..93c51542df2695 100644 --- a/Include/internal/pycore_typedefs.h +++ b/Include/internal/pycore_typedefs.h @@ -9,6 +9,7 @@ extern "C" { #endif +typedef struct _PyInterpreterFrameCore _PyInterpreterFrameCore; typedef struct _PyInterpreterFrame _PyInterpreterFrame; typedef struct pyruntimestate _PyRuntimeState; diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 3997acbdf84695..0b95cfc14dcd7e 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -4,6 +4,7 @@ import _thread from collections import deque import contextlib +import dis import importlib.machinery import importlib.util import json @@ -2857,6 +2858,85 @@ def func(): names = ["func", "outer", "outer", "inner", "inner", "outer", "inner"] self.do_test(func, names) + def test_jit_frame(self): + def fakefunc(): + pass + + def f(): + return sys._getframe(1) + + res = _testinternalcapi.call_with_jit_frame(fakefunc, f, ()) + + def test_jit_frame_globals(self): + """jit executable can fill in globals when accessed""" + def fakefunc(): + pass + + fake_globals = {"abc":42} + def callback(): + return {"globals": fake_globals} + + res = _testinternalcapi.call_with_jit_frame(fakefunc, globals, (), callback) + self.assertEqual(res, fake_globals) + + def test_jit_frame_builtins(self): + """jit executable can fill in builtins when accessed""" + def fakefunc(): + pass + + fake_builtins = {"abc":42} + def callback(): + return {"builtins": fake_builtins} + + res = _testinternalcapi.call_with_jit_frame(fakefunc, _testlimitedcapi.eval_getbuiltins, (), callback) + self.assertEqual(res, fake_builtins) + + def test_jit_frame_instr_ptr(self): + """jit executable can fill in the instr ptr each time the frame is queried""" + def fakefunc(): + pass + pass + pass + pass + + offset = 0 + linenos = [] + def test(): + for op in dis.get_instructions(fakefunc): + if op.opname in ("RESUME", "NOP", "RETURN_VALUE"): + nonlocal offset + offset = op.offset//2 + linenos.append(sys._getframe(1).f_lineno) + + def callback(): + return {"instr_ptr": offset} + + _testinternalcapi.call_with_jit_frame(fakefunc, test, (), callback) + base = fakefunc.__code__.co_firstlineno + self.assertEqual(linenos, [base, base + 1, base + 2, base + 3, base + 4]) + + def test_jit_frame_code(self): + """internal C api checks the for a code executor""" + def fakefunc(): + pass + + def callback(): + return _testinternalcapi.iframe_getcode(sys._getframe(1)) + + res = _testinternalcapi.call_with_jit_frame(fakefunc, callback, ()) + self.assertEqual(res, fakefunc.__code__) + + def test_jit_frame_line(self): + """internal C api checks the for a code executor""" + def fakefunc(): + pass + + def callback(): + return _testinternalcapi.iframe_getline(sys._getframe(1)) + + res = _testinternalcapi.call_with_jit_frame(fakefunc, callback, ()) + self.assertEqual(res, fakefunc.__code__.co_firstlineno) + @unittest.skipUnless(support.Py_GIL_DISABLED, 'need Py_GIL_DISABLED') class TestPyThreadId(unittest.TestCase): diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index a5708b298c84a5..170e032fe957bb 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1674,9 +1674,9 @@ def func(): return sys._getframe() x = func() if support.Py_GIL_DISABLED: - INTERPRETER_FRAME = '9PihcP' + INTERPRETER_FRAME = '2Pc7PihP' else: - INTERPRETER_FRAME = '9PhcP' + INTERPRETER_FRAME = '2Pc7PhP' check(x, size('3PiccPPP' + INTERPRETER_FRAME + 'P')) # function def func(): pass diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-12-11-20-23-56.gh-issue-142598.ftYjy7.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-12-11-20-23-56.gh-issue-142598.ftYjy7.rst new file mode 100644 index 00000000000000..61bab81ec2bb48 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-12-11-20-23-56.gh-issue-142598.ftYjy7.rst @@ -0,0 +1 @@ +Adds a new executable type for _PyInterpreterFrame.f_executable PEP 523 JITs to plug in and have their frames visible to external introspection tools. diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 8d63b3e55fc4c1..238a411c789594 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -20,7 +20,7 @@ #include "pycore_dict.h" // PyDictValues #include "pycore_fileutils.h" // _Py_normpath() #include "pycore_flowgraph.h" // _PyCompile_OptimizeCfg() -#include "pycore_frame.h" // _PyInterpreterFrame +#include "pycore_frame.h" // _PyInterpreterFrameCore #include "pycore_function.h" // _PyFunction_GET_BUILTINS #include "pycore_gc.h" // PyGC_Head #include "pycore_hashtable.h" // _Py_hashtable_new() @@ -28,6 +28,7 @@ #include "pycore_initconfig.h" // _Py_GetConfigsAsDict() #include "pycore_instruction_sequence.h" // _PyInstructionSequence_New() #include "pycore_interpframe.h" // _PyFrame_GetFunction() +#include "pycore_interpframe_structs.h" // _PyInterpreterFrameCore #include "pycore_object.h" // _PyObject_IsFreed() #include "pycore_optimizer.h" // _Py_Executor_DependsOn #include "pycore_pathconfig.h" // _PyPathConfig_ClearGlobal() @@ -666,7 +667,7 @@ static PyObject * record_eval(PyThreadState *tstate, struct _PyInterpreterFrame *f, int exc) { if (PyStackRef_FunctionCheck(f->f_funcobj)) { - PyFunctionObject *func = _PyFrame_GetFunction(f); + PyFunctionObject *func = _PyFrame_GetFunction(_PyFrame_Core(f)); PyObject *module = _get_current_module(); assert(module != NULL); module_state *state = get_module_state(module); @@ -698,6 +699,14 @@ extern PyObject* Test_EvalFrame(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag); extern int Test_EvalFrame_Resumes, Test_EvalFrame_Loads; +static PyObject * +set_eval_frame_interp(PyObject *self, PyObject *Py_UNUSED(args)) +{ + _PyInterpreterState_SetEvalFrameFunc(_PyInterpreterState_GET(), Test_EvalFrame); + Py_RETURN_NONE; +} + +// Defined in interpreter.c static PyObject * get_eval_frame_stats(PyObject *self, PyObject *Py_UNUSED(args)) { @@ -723,11 +732,113 @@ get_eval_frame_stats(PyObject *self, PyObject *Py_UNUSED(args)) return res; } +typedef struct { + bool initialized; + _PyInterpreterFrame frame; +} JitFrame; + +_PyInterpreterFrame * +reifier(_PyInterpreterFrameCore *f, PyObject *executable) +{ + JitFrame *jitframe = (JitFrame*)((char *)f - offsetof(JitFrame, frame)); + _PyInterpreterFrame *frame = (_PyInterpreterFrame *)f; + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + if (!jitframe->initialized) { + frame->f_locals = NULL; + frame->f_globals = func->func_globals; // borrowed + frame->f_builtins = func->func_builtins; // borrowed + frame->frame_obj = NULL; + jitframe->initialized = true; + } + PyUnstable_PyExternalExecutable *ext_exec = (PyUnstable_PyExternalExecutable*)executable; + if (ext_exec->ef_state == NULL) { + return frame; + } + + PyObject *res = PyObject_CallNoArgs(ext_exec->ef_state); + if (res == NULL) { + // reifier cannot fail + PyErr_Clear(); + return frame; + } + + // let the test-state function fill in details on the frame + if (PyDict_Check(res)) { + PyObject *globals = PyDict_GetItemString(res, "globals"); + if (globals != NULL) { + frame->f_globals = globals; + } + PyObject *builtins = PyDict_GetItemString(res, "builtins"); + if (builtins != NULL) { + frame->f_builtins = builtins; + } + PyObject *instr_ptr = PyDict_GetItemString(res, "instr_ptr"); + if (instr_ptr != NULL) { + frame->instr_ptr = _PyCode_CODE((PyCodeObject *)func->func_code) + + PyLong_AsLong(instr_ptr); + } + } + Py_DECREF(res); + return frame; +} + static PyObject * -set_eval_frame_interp(PyObject *self, PyObject *Py_UNUSED(args)) +call_with_jit_frame(PyObject *self, PyObject *args) { - _PyInterpreterState_SetEvalFrameFunc(_PyInterpreterState_GET(), Test_EvalFrame); - Py_RETURN_NONE; + PyObject *fakefunc; // used for f_funcobj as-if we were that JITed function + PyObject *call; // the thing to call for testing purposes + PyObject *callargs; // the arguments to provide for the test call + PyObject *state = NULL; // a state object provided to the reifier, for tests we + // callback on it to populate fields. + if (!PyArg_ParseTuple(args, "OOO|O", &fakefunc, &call, &callargs, &state)) { + return NULL; + } + if (!PyTuple_Check(callargs)) { + PyErr_SetString(PyExc_TypeError, "callargs must be a tuple"); + return NULL; + } + + PyThreadState *tstate = PyThreadState_Get(); + PyCodeObject *code = (PyCodeObject *)((PyFunctionObject *)fakefunc)->func_code; + PyObject *executable = PyUnstable_MakeExternalExecutable(reifier, code, state); + if (executable == NULL) { + return NULL; + } + + // Create JIT frame and push onto the _PyInterprerFrame stack. + JitFrame frame; + frame.initialized = false; + // Initialize minimal set of fields + _PyFrame_Core(&frame.frame)->owner = FRAME_OWNED_EXTERNALLY; + _PyFrame_Core(&frame.frame)->previous = tstate->current_frame; + _PyFrame_Core(&frame.frame)->f_executable = PyStackRef_FromPyObjectSteal(executable); + frame.frame.f_funcobj = PyStackRef_FromPyObjectNew(fakefunc); + frame.frame.instr_ptr = _PyCode_CODE(code) + code->_co_firsttraceable; + frame.frame.stackpointer = &frame.frame.localsplus[0]; +#ifdef Py_GIL_DISABLED + frame.frame.tlbc_index = 0; +#endif + tstate->current_frame = _PyFrame_Core(&frame.frame); + + // call the test function + PyObject *res = PyObject_Call(call, callargs, NULL); + + tstate->current_frame = _PyFrame_Core(&frame.frame)->previous; + // the test function may have caused the frame to get reified. + if (frame.initialized && frame.frame.frame_obj != NULL) { + // remove our reifier + PyStackRef_CLOSE(_PyFrame_Core(&frame.frame)->f_executable); + _PyFrame_Core(&frame.frame)->f_executable = PyStackRef_FromPyObjectNew(code); + + // Transfer ownership to the reified frame object + _PyFrame_ClearExceptCode(&frame.frame); + } + else { + // Pop frame from the stack + PyStackRef_CLOSE(frame.frame.f_funcobj); + } + PyStackRef_CLOSE(_PyFrame_Core(&frame.frame)->f_executable); + return res; } /*[clinic input] @@ -970,8 +1081,8 @@ iframe_getcode(PyObject *self, PyObject *frame) PyErr_SetString(PyExc_TypeError, "argument must be a frame"); return NULL; } - struct _PyInterpreterFrame *f = ((PyFrameObject *)frame)->f_frame; - return PyUnstable_InterpreterFrame_GetCode(f); + _PyInterpreterFrame *f = ((PyFrameObject *)frame)->f_frame; + return PyUnstable_InterpreterFrame_GetCode(_PyFrame_Core(f)); } static PyObject * @@ -981,8 +1092,8 @@ iframe_getline(PyObject *self, PyObject *frame) PyErr_SetString(PyExc_TypeError, "argument must be a frame"); return NULL; } - struct _PyInterpreterFrame *f = ((PyFrameObject *)frame)->f_frame; - return PyLong_FromLong(PyUnstable_InterpreterFrame_GetLine(f)); + _PyInterpreterFrame *f = ((PyFrameObject *)frame)->f_frame; + return PyLong_FromLong(PyUnstable_InterpreterFrame_GetLine(_PyFrame_Core(f))); } static PyObject * @@ -992,8 +1103,8 @@ iframe_getlasti(PyObject *self, PyObject *frame) PyErr_SetString(PyExc_TypeError, "argument must be a frame"); return NULL; } - struct _PyInterpreterFrame *f = ((PyFrameObject *)frame)->f_frame; - return PyLong_FromLong(PyUnstable_InterpreterFrame_GetLasti(f)); + _PyInterpreterFrame *f = ((PyFrameObject *)frame)->f_frame; + return PyLong_FromLong(PyUnstable_InterpreterFrame_GetLasti(_PyFrame_Core(f))); } static PyObject * @@ -2587,6 +2698,7 @@ static PyMethodDef module_functions[] = { {"set_eval_frame_default", set_eval_frame_default, METH_NOARGS, NULL}, {"set_eval_frame_interp", set_eval_frame_interp, METH_NOARGS, NULL}, {"set_eval_frame_record", set_eval_frame_record, METH_O, NULL}, + {"call_with_jit_frame", call_with_jit_frame, METH_VARARGS, NULL}, _TESTINTERNALCAPI_COMPILER_CLEANDOC_METHODDEF _TESTINTERNALCAPI_NEW_INSTRUCTION_SEQUENCE_METHODDEF _TESTINTERNALCAPI_COMPILER_CODEGEN_METHODDEF diff --git a/Modules/_testinternalcapi/interpreter.c b/Modules/_testinternalcapi/interpreter.c index 2cd23fa3c58849..e45539f6b3bf76 100644 --- a/Modules/_testinternalcapi/interpreter.c +++ b/Modules/_testinternalcapi/interpreter.c @@ -47,7 +47,7 @@ Test_EvalFrame(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag) #if !_Py_TAIL_CALL_INTERP uint8_t opcode; /* Current opcode */ int oparg; /* Current opcode argument, if any */ - assert(tstate->current_frame == NULL || tstate->current_frame->stackpointer != NULL); + assert(_PyFrame_StackpointerSaved()); #if !USE_COMPUTED_GOTOS uint8_t tracing_mode = 0; uint8_t dispatch_code; @@ -56,7 +56,7 @@ Test_EvalFrame(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag) _PyEntryFrame entry; if (_Py_EnterRecursiveCallTstate(tstate, "")) { - assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + assert(_PyFrame_Core(frame)->owner != FRAME_OWNED_BY_INTERPRETER); _PyEval_FrameClearAndPop(tstate, frame); return NULL; } @@ -76,19 +76,19 @@ Test_EvalFrame(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag) entry.frame.f_globals = (PyObject*)0xaaa3; entry.frame.f_builtins = (PyObject*)0xaaa4; #endif - entry.frame.f_executable = PyStackRef_None; + _PyFrame_Core(&entry.frame)->f_executable = PyStackRef_None; entry.frame.instr_ptr = (_Py_CODEUNIT *)_Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR + 1; entry.frame.stackpointer = entry.stack; - entry.frame.owner = FRAME_OWNED_BY_INTERPRETER; + _PyFrame_Core(&entry.frame)->owner = FRAME_OWNED_BY_INTERPRETER; entry.frame.visited = 0; entry.frame.return_offset = 0; #ifdef Py_DEBUG entry.frame.lltrace = 0; #endif /* Push frame */ - entry.frame.previous = tstate->current_frame; - frame->previous = &entry.frame; - tstate->current_frame = frame; + _PyFrame_Core(&entry.frame)->previous = tstate->current_frame; + _PyFrame_Core(frame)->previous = _PyFrame_Core(&entry.frame); + tstate->current_frame = _PyFrame_Core(frame); entry.frame.localsplus[0] = PyStackRef_NULL; /* support for generator.throw() */ @@ -100,17 +100,17 @@ Test_EvalFrame(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag) /* Load thread-local bytecode */ if (frame->tlbc_index != ((_PyThreadStateImpl *)tstate)->tlbc_index) { _Py_CODEUNIT *bytecode = - _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); + _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(_PyFrame_Core(frame))); if (bytecode == NULL) { goto early_exit; } - ptrdiff_t off = frame->instr_ptr - _PyFrame_GetBytecode(frame); + ptrdiff_t off = frame->instr_ptr - _PyFrame_GetBytecode(_PyFrame_Core(frame)); frame->tlbc_index = ((_PyThreadStateImpl *)tstate)->tlbc_index; frame->instr_ptr = bytecode + off; } #endif /* Because this avoids the RESUME, we need to update instrumentation */ - _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); + _Py_Instrument(_PyFrame_GetCode(_PyFrame_Core(frame)), tstate->interp); next_instr = frame->instr_ptr; monitor_throw(tstate, frame, next_instr); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -140,14 +140,14 @@ Test_EvalFrame(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag) early_exit: assert(_PyErr_Occurred(tstate)); _Py_LeaveRecursiveCallPy(tstate); - assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + assert(_PyFrame_Core(frame)->owner != FRAME_OWNED_BY_INTERPRETER); // GH-99729: We need to unlink the frame *before* clearing it: _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; + frame = (_PyInterpreterFrame *)(tstate->current_frame = _PyFrame_Core(dying)->previous); _PyEval_FrameClearAndPop(tstate, dying); frame->return_offset = 0; - assert(frame->owner == FRAME_OWNED_BY_INTERPRETER); + assert(_PyFrame_Core(frame)->owner == FRAME_OWNED_BY_INTERPRETER); /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; + tstate->current_frame = _PyFrame_Core(frame)->previous; return NULL; } diff --git a/Modules/_testinternalcapi/test_cases.c.h b/Modules/_testinternalcapi/test_cases.c.h index c02d236fc3e8ac..fbeca6931404cc 100644 --- a/Modules/_testinternalcapi/test_cases.c.h +++ b/Modules/_testinternalcapi/test_cases.c.h @@ -746,9 +746,10 @@ stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); - assert(temp->previous == frame || temp->previous->previous == frame); + assert(_PyFrame_Core(temp)->previous == FRAME_CORE || _PyFrame_Core(temp)->previous->previous == FRAME_CORE); CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; + frame = temp; + tstate->current_frame = _PyFrame_Core(temp); tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); @@ -1771,7 +1772,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( tstate, callable, locals, - arguments, total_args, NULL, frame + arguments, total_args, NULL, FRAME_CORE ); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; @@ -1900,14 +1901,14 @@ init = callable; _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( - tstate, (PyCodeObject *)&_Py_InitCleanup, 1, frame); + tstate, (PyCodeObject *)&_Py_InitCleanup, 1, FRAME_CORE); stack_pointer = _PyFrame_GetStackPointer(frame); - assert(_PyFrame_GetBytecode(shim)[0].op.code == EXIT_INIT_CHECK); - assert(_PyFrame_GetBytecode(shim)[1].op.code == RETURN_VALUE); + assert(_PyFrame_GetBytecodeFull(shim)[0].op.code == EXIT_INIT_CHECK); + assert(_PyFrame_GetBytecodeFull(shim)[1].op.code == RETURN_VALUE); shim->localsplus[0] = PyStackRef_DUP(self); _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( - tstate, init, NULL, args-1, oparg+1, NULL, shim); + tstate, init, NULL, args-1, oparg+1, NULL, _PyFrame_Core(shim)); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -1927,9 +1928,10 @@ assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); _PyFrame_SetStackPointer(frame, stack_pointer); - assert(temp->previous == frame || temp->previous->previous == frame); + assert(_PyFrame_Core(temp)->previous == FRAME_CORE || _PyFrame_Core(temp)->previous->previous == FRAME_CORE); CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; + frame = temp; + tstate->current_frame = _PyFrame_Core(temp); tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); @@ -2070,9 +2072,10 @@ stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); - assert(temp->previous == frame || temp->previous->previous == frame); + assert(_PyFrame_Core(temp)->previous == FRAME_CORE || _PyFrame_Core(temp)->previous->previous == FRAME_CORE); CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; + frame = temp; + tstate->current_frame = _PyFrame_Core(temp); tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); @@ -2174,7 +2177,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( tstate, callable, locals, - args, total_args, NULL, frame + args, total_args, NULL, FRAME_CORE ); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; @@ -2198,9 +2201,10 @@ assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); _PyFrame_SetStackPointer(frame, stack_pointer); - assert(temp->previous == frame || temp->previous->previous == frame); + assert(_PyFrame_Core(temp)->previous == FRAME_CORE || _PyFrame_Core(temp)->previous->previous == FRAME_CORE); CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; + frame = temp; + tstate->current_frame = _PyFrame_Core(temp); tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); @@ -2703,7 +2707,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( tstate, func_st, locals, - nargs, callargs, kwargs, frame); + nargs, callargs, kwargs, FRAME_CORE); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -2727,9 +2731,10 @@ assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); _PyFrame_SetStackPointer(frame, stack_pointer); - assert(temp->previous == frame || temp->previous->previous == frame); + assert(_PyFrame_Core(temp)->previous == FRAME_CORE || _PyFrame_Core(temp)->previous->previous == FRAME_CORE); CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; + frame = temp; + tstate->current_frame = _PyFrame_Core(temp); tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); @@ -2867,7 +2872,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( tstate, func_st, locals, - nargs, callargs, kwargs, frame); + nargs, callargs, kwargs, FRAME_CORE); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -3143,7 +3148,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( tstate, callable, locals, - arguments, positional_args, kwnames_o, frame + arguments, positional_args, kwnames_o, FRAME_CORE ); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -3 - oparg; @@ -3274,7 +3279,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( tstate, callable, locals, - arguments, positional_args, kwnames_o, frame + arguments, positional_args, kwnames_o, FRAME_CORE ); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; @@ -3303,9 +3308,10 @@ assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); _PyFrame_SetStackPointer(frame, stack_pointer); - assert(temp->previous == frame || temp->previous->previous == frame); + assert(_PyFrame_Core(temp)->previous == FRAME_CORE || _PyFrame_Core(temp)->previous->previous == FRAME_CORE); CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; + frame = temp; + tstate->current_frame = _PyFrame_Core(temp); tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); @@ -3461,7 +3467,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( tstate, callable, locals, - arguments, positional_args, kwnames_o, frame + arguments, positional_args, kwnames_o, FRAME_CORE ); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; @@ -3490,9 +3496,10 @@ assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); _PyFrame_SetStackPointer(frame, stack_pointer); - assert(temp->previous == frame || temp->previous->previous == frame); + assert(_PyFrame_Core(temp)->previous == FRAME_CORE || _PyFrame_Core(temp)->previous->previous == FRAME_CORE); CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; + frame = temp; + tstate->current_frame = _PyFrame_Core(temp); tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); @@ -4242,9 +4249,10 @@ stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); - assert(temp->previous == frame || temp->previous->previous == frame); + assert(_PyFrame_Core(temp)->previous == FRAME_CORE || _PyFrame_Core(temp)->previous->previous == FRAME_CORE); CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; + frame = temp; + tstate->current_frame = _PyFrame_Core(temp); tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); @@ -4318,7 +4326,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( tstate, callable, locals, - args, total_args, NULL, frame + args, total_args, NULL, FRAME_CORE ); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; @@ -4342,9 +4350,10 @@ assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); _PyFrame_SetStackPointer(frame, stack_pointer); - assert(temp->previous == frame || temp->previous->previous == frame); + assert(_PyFrame_Core(temp)->previous == FRAME_CORE || _PyFrame_Core(temp)->previous->previous == FRAME_CORE); CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; + frame = temp; + tstate->current_frame = _PyFrame_Core(temp); tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); @@ -5282,7 +5291,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(COPY_FREE_VARS); - PyCodeObject *co = _PyFrame_GetCode(frame); + PyCodeObject *co = _PyFrame_GetCodeFull(frame); assert(PyStackRef_FunctionCheck(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); PyObject *closure = func->func_closure; @@ -5332,7 +5341,7 @@ PyObject *oldobj = PyCell_SwapTakeRef((PyCellObject *)cell, NULL); if (oldobj == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCodeFull(frame), oparg); stack_pointer = _PyFrame_GetStackPointer(frame); JUMP_TO_LABEL(error); } @@ -5355,7 +5364,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, UNBOUNDLOCAL_ERROR_MSG, - PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) + PyTuple_GetItem(_PyFrame_GetCodeFull(frame)->co_localsplusnames, oparg) ); stack_pointer = _PyFrame_GetStackPointer(frame); JUMP_TO_LABEL(error); @@ -5640,7 +5649,7 @@ next_instr = this_instr; JUMP_TO_LABEL(stop_tracing); } - PyCodeObject *code = _PyFrame_GetCode(frame); + PyCodeObject *code = _PyFrame_GetCodeFull(frame); _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; assert(executor->vm_data.index == INSTR_OFFSET() - 1); assert(executor->vm_data.code == code); @@ -5870,7 +5879,7 @@ _PyFrame_StackPush(pushed_frame, PyStackRef_None); gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; - pushed_frame->previous = frame; + _PyFrame_Core(pushed_frame)->previous = FRAME_CORE; frame->return_offset = (uint16_t)( 2u + oparg); gen_frame = PyStackRef_Wrap(pushed_frame); } @@ -5880,9 +5889,10 @@ assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); _PyFrame_SetStackPointer(frame, stack_pointer); - assert(temp->previous == frame || temp->previous->previous == frame); + assert(_PyFrame_Core(temp)->previous == FRAME_CORE || _PyFrame_Core(temp)->previous->previous == FRAME_CORE); CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; + frame = temp; + tstate->current_frame = _PyFrame_Core(temp); tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); @@ -6298,7 +6308,7 @@ iterable = stack_pointer[-1]; PyObject *iterable_o = PyStackRef_AsPyObjectBorrow(iterable); if (PyCoro_CheckExact(iterable_o)) { - if (!(_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) { + if (!(_PyFrame_GetCodeFull(frame)->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) { _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_SetString(tstate, PyExc_TypeError, "cannot 'yield from' a coroutine object " @@ -6370,7 +6380,7 @@ level = stack_pointer[-2]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyEval_ImportName(tstate, frame, name, + PyObject *res_o = _PyEval_ImportName(tstate, FRAME_CORE, name, PyStackRef_AsPyObjectBorrow(fromlist), PyStackRef_AsPyObjectBorrow(level)); _PyStackRef tmp = fromlist; @@ -6480,7 +6490,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( tstate, callable, locals, - arguments, total_args, NULL, frame + arguments, total_args, NULL, FRAME_CORE ); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; @@ -6637,7 +6647,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( tstate, func_st, locals, - nargs, callargs, kwargs, frame); + nargs, callargs, kwargs, FRAME_CORE); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -6776,7 +6786,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( tstate, callable, locals, - arguments, positional_args, kwnames_o, frame + arguments, positional_args, kwnames_o, FRAME_CORE ); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -3 - oparg; @@ -7049,8 +7059,8 @@ opcode = INSTRUMENTED_LINE; int original_opcode = 0; if (tstate->tracing) { - PyCodeObject *code = _PyFrame_GetCode(frame); - int index = (int)(this_instr - _PyFrame_GetBytecode(frame)); + PyCodeObject *code = _PyFrame_GetCodeFull(frame); + int index = (int)(this_instr - _PyFrame_GetBytecodeFull(frame)); original_opcode = code->_co_monitoring->lines->data[index*code->_co_monitoring->lines->bytes_per_entry]; next_instr = this_instr; } else { @@ -7364,12 +7374,12 @@ ((_PyThreadStateImpl *)tstate)->tlbc_index) { _PyFrame_SetStackPointer(frame, stack_pointer); _Py_CODEUNIT *bytecode = - _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); + _PyEval_GetExecutableCode(tstate, _PyFrame_GetCodeFull(frame)); stack_pointer = _PyFrame_GetStackPointer(frame); if (bytecode == NULL) { JUMP_TO_LABEL(error); } - ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame); + ptrdiff_t off = this_instr - _PyFrame_GetBytecodeFull(frame); frame->tlbc_index = ((_PyThreadStateImpl *)tstate)->tlbc_index; frame->instr_ptr = bytecode + off; next_instr = frame->instr_ptr; @@ -7387,10 +7397,10 @@ #endif if (check_instrumentation) { uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; - uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); + uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCodeFull(frame)->_co_instrumentation_version); if (code_version != global_version) { _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); + int err = _Py_Instrument(_PyFrame_GetCodeFull(frame), tstate->interp); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { JUMP_TO_LABEL(error); @@ -7456,7 +7466,7 @@ // _RETURN_VALUE { retval = val; - assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + assert(FRAME_CORE->owner != FRAME_OWNED_BY_INTERPRETER); _PyStackRef temp = PyStackRef_MakeHeapSafe(retval); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -7464,7 +7474,7 @@ assert(STACK_LEVEL() == 0); _Py_LeaveRecursiveCallPy(tstate); _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; + frame = (_PyInterpreterFrame *)(tstate->current_frame = _PyFrame_Core(dying)->previous); _PyEval_FrameClearAndPop(tstate, dying); stack_pointer = _PyFrame_GetStackPointer(frame); LOAD_IP(frame->return_offset); @@ -7509,7 +7519,7 @@ // _YIELD_VALUE { retval = val; - assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + assert(FRAME_CORE->owner != FRAME_OWNED_BY_INTERPRETER); frame->instr_ptr++; PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); @@ -7522,8 +7532,8 @@ gen->gi_exc_state.previous_item = NULL; _Py_LeaveRecursiveCallPy(tstate); _PyInterpreterFrame *gen_frame = frame; - frame = tstate->current_frame = frame->previous; - gen_frame->previous = NULL; + frame = (_PyInterpreterFrame *)(tstate->current_frame = FRAME_CORE->previous); + _PyFrame_Core(gen_frame)->previous = NULL; ((_PyThreadStateImpl *)tstate)->generator_return_kind = GENERATOR_YIELD; FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, FRAME_SUSPENDED + oparg); assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); @@ -7556,9 +7566,9 @@ INSTRUCTION_STATS(INTERPRETER_EXIT); _PyStackRef retval; retval = stack_pointer[-1]; - assert(frame->owner == FRAME_OWNED_BY_INTERPRETER); - assert(_PyFrame_IsIncomplete(frame)); - tstate->current_frame = frame->previous; + assert(FRAME_CORE->owner == FRAME_OWNED_BY_INTERPRETER); + assert(_PyFrame_IsIncomplete(FRAME_CORE)); + tstate->current_frame = FRAME_CORE->previous; assert(!_PyErr_Occurred(tstate)); PyObject *result = PyStackRef_AsPyObjectSteal(retval); #if !_Py_TAIL_CALL_INTERP @@ -8636,9 +8646,10 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); - assert(temp->previous == frame || temp->previous->previous == frame); + assert(_PyFrame_Core(temp)->previous == FRAME_CORE || _PyFrame_Core(temp)->previous->previous == FRAME_CORE); CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; + frame = temp; + tstate->current_frame = _PyFrame_Core(temp); tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); @@ -8921,7 +8932,7 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCodeFull(frame), oparg); stack_pointer = _PyFrame_GetStackPointer(frame); JUMP_TO_LABEL(error); } @@ -9017,7 +9028,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, UNBOUNDLOCAL_ERROR_MSG, - PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) + PyTuple_GetItem(_PyFrame_GetCodeFull(frame)->co_localsplusnames, oparg) ); stack_pointer = _PyFrame_GetStackPointer(frame); JUMP_TO_LABEL(error); @@ -9064,8 +9075,8 @@ PyObject *name; PyObject *class_dict = PyStackRef_AsPyObjectBorrow(class_dict_st); assert(class_dict); - assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); - name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); + assert(oparg >= 0 && oparg < _PyFrame_GetCodeFull(frame)->co_nlocalsplus); + name = PyTuple_GET_ITEM(_PyFrame_GetCodeFull(frame)->co_localsplusnames, oparg); int err; _PyFrame_SetStackPointer(frame, stack_pointer); PyObject* value_o = _PyMapping_GetOptionalItem2(class_dict, name, &err); @@ -9078,7 +9089,7 @@ value_o = PyCell_GetRef(cell); if (value_o == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCodeFull(frame), oparg); stack_pointer = _PyFrame_GetStackPointer(frame); JUMP_TO_LABEL(error); } @@ -10273,7 +10284,7 @@ PyObject *exc = PyStackRef_AsPyObjectSteal(exc_st); assert(oparg >= 0 && oparg <= 2); if (oparg) { - frame->instr_ptr = _PyFrame_GetBytecode(frame) + PyStackRef_UntagInt(values[0]); + frame->instr_ptr = _PyFrame_GetBytecodeFull(frame) + PyStackRef_UntagInt(values[0]); } assert(exc && PyExceptionInstance_Check(exc)); stack_pointer += -1; @@ -10315,12 +10326,12 @@ ((_PyThreadStateImpl *)tstate)->tlbc_index) { _PyFrame_SetStackPointer(frame, stack_pointer); _Py_CODEUNIT *bytecode = - _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); + _PyEval_GetExecutableCode(tstate, _PyFrame_GetCodeFull(frame)); stack_pointer = _PyFrame_GetStackPointer(frame); if (bytecode == NULL) { JUMP_TO_LABEL(error); } - ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame); + ptrdiff_t off = this_instr - _PyFrame_GetBytecodeFull(frame); frame->tlbc_index = ((_PyThreadStateImpl *)tstate)->tlbc_index; frame->instr_ptr = bytecode + off; next_instr = frame->instr_ptr; @@ -10338,10 +10349,10 @@ #endif if (check_instrumentation) { uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; - uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); + uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCodeFull(frame)->_co_instrumentation_version); if (code_version != global_version) { _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); + int err = _Py_Instrument(_PyFrame_GetCodeFull(frame), tstate->interp); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { JUMP_TO_LABEL(error); @@ -10394,7 +10405,7 @@ _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; #endif uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); - uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); + uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCodeFull(frame)->_co_instrumentation_version); assert((version & _PY_EVAL_EVENTS_MASK) == 0); if (eval_breaker != version) { UPDATE_MISS_STATS(RESUME); @@ -10436,11 +10447,11 @@ _PyFrame_Copy(frame, gen_frame); assert(frame->frame_obj == NULL); gen->gi_frame_state = FRAME_CREATED; - gen_frame->owner = FRAME_OWNED_BY_GENERATOR; + _PyFrame_Core(gen_frame)->owner = FRAME_OWNED_BY_GENERATOR; _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *prev = frame->previous; - _PyThreadState_PopFrame(tstate, frame); - frame = tstate->current_frame = prev; + _PyInterpreterFrameCore *prev = FRAME_CORE->previous; + _PyThreadState_PopFrame(tstate, FRAME_CORE); + frame = (_PyInterpreterFrame *)(tstate->current_frame = prev); LOAD_IP(frame->return_offset); stack_pointer = _PyFrame_GetStackPointer(frame); res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); @@ -10462,7 +10473,7 @@ _PyStackRef retval; _PyStackRef res; retval = stack_pointer[-1]; - assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + assert(FRAME_CORE->owner != FRAME_OWNED_BY_INTERPRETER); _PyStackRef temp = PyStackRef_MakeHeapSafe(retval); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -10470,7 +10481,7 @@ assert(STACK_LEVEL() == 0); _Py_LeaveRecursiveCallPy(tstate); _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; + frame = (_PyInterpreterFrame *)(tstate->current_frame = _PyFrame_Core(dying)->previous); _PyEval_FrameClearAndPop(tstate, dying); stack_pointer = _PyFrame_GetStackPointer(frame); LOAD_IP(frame->return_offset); @@ -10518,7 +10529,7 @@ v = stack_pointer[-1]; PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); PyObject *retval_o; - assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + assert(FRAME_CORE->owner != FRAME_OWNED_BY_INTERPRETER); if (!IS_PEP523_HOOKED(tstate) && (Py_TYPE(receiver_o) == &PyGen_Type || Py_TYPE(receiver_o) == &PyCoro_Type) && gen_try_set_executing((PyGenObject *)receiver_o)) @@ -10532,8 +10543,8 @@ tstate->exc_info = &gen->gi_exc_state; assert( 2u + oparg <= UINT16_MAX); frame->return_offset = (uint16_t)( 2u + oparg); - assert(gen_frame->previous == NULL); - gen_frame->previous = frame; + assert(_PyFrame_Core(gen_frame)->previous == NULL); + _PyFrame_Core(gen_frame)->previous = FRAME_CORE; DISPATCH_INLINED(gen_frame); } if (PyStackRef_IsNone(v) && PyIter_Check(receiver_o)) { @@ -10632,7 +10643,7 @@ tstate->exc_info = &gen->gi_exc_state; assert( 2u + oparg <= UINT16_MAX); frame->return_offset = (uint16_t)( 2u + oparg); - pushed_frame->previous = frame; + _PyFrame_Core(pushed_frame)->previous = FRAME_CORE; gen_frame = PyStackRef_Wrap(pushed_frame); } // _PUSH_FRAME @@ -10643,9 +10654,10 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); - assert(temp->previous == frame || temp->previous->previous == frame); + assert(_PyFrame_Core(temp)->previous == FRAME_CORE || _PyFrame_Core(temp)->previous->previous == FRAME_CORE); CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; + frame = temp; + tstate->current_frame = _PyFrame_Core(temp); tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); @@ -11881,7 +11893,7 @@ _PyJitTracerState *tracer = _tstate->jit_tracer_state; assert(tracer != NULL); tracer->prev_state.instr = next_instr; - PyObject *prev_code = PyStackRef_AsPyObjectBorrow(frame->f_executable); + PyObject *prev_code = PyStackRef_AsPyObjectBorrow(FRAME_CORE->f_executable); if (tracer->prev_state.instr_code != (PyCodeObject *)prev_code) { _PyFrame_SetStackPointer(frame, stack_pointer); Py_SETREF(tracer->prev_state.instr_code, (PyCodeObject*)Py_NewRef((prev_code))); @@ -11889,7 +11901,7 @@ } tracer->prev_state.instr_frame = frame; tracer->prev_state.instr_oparg = oparg; - tracer->prev_state.instr_stacklevel = PyStackRef_IsNone(frame->f_executable) ? 2 : STACK_LEVEL(); + tracer->prev_state.instr_stacklevel = PyStackRef_IsNone(FRAME_CORE->f_executable) ? 2 : STACK_LEVEL(); if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { (&next_instr[1])->counter = trigger_backoff_counter(); } @@ -12285,7 +12297,7 @@ _PyStackRef retval; _PyStackRef value; retval = stack_pointer[-1]; - assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + assert(FRAME_CORE->owner != FRAME_OWNED_BY_INTERPRETER); frame->instr_ptr++; PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); @@ -12298,8 +12310,8 @@ gen->gi_exc_state.previous_item = NULL; _Py_LeaveRecursiveCallPy(tstate); _PyInterpreterFrame *gen_frame = frame; - frame = tstate->current_frame = frame->previous; - gen_frame->previous = NULL; + frame = (_PyInterpreterFrame *)(tstate->current_frame = FRAME_CORE->previous); + _PyFrame_Core(gen_frame)->previous = NULL; ((_PyThreadStateImpl *)tstate)->generator_return_kind = GENERATOR_YIELD; FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, FRAME_SUSPENDED + oparg); assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); @@ -12333,8 +12345,8 @@ opcode = next_instr->op.code; _PyErr_Format(tstate, PyExc_SystemError, "%U:%d: unknown opcode %d", - _PyFrame_GetCode(frame)->co_filename, - PyUnstable_InterpreterFrame_GetLine(frame), + FRAME_CODE->co_filename, + PyUnstable_InterpreterFrame_GetLine(FRAME_CORE), opcode); JUMP_TO_LABEL(error); @@ -12376,10 +12388,10 @@ JUMP_TO_LABEL(error); _PyFrame_SetStackPointer(frame, stack_pointer); STOP_TRACING(); stack_pointer = _PyFrame_GetStackPointer(frame); - assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); - if (!_PyFrame_IsIncomplete(frame)) { + assert(FRAME_CORE->owner != FRAME_OWNED_BY_INTERPRETER); + if (!_PyFrame_IsIncomplete(FRAME_CORE)) { _PyFrame_SetStackPointer(frame, stack_pointer); - PyFrameObject *f = _PyFrame_GetFrameObject(frame); + PyFrameObject *f = _PyFrame_GetFrameObjectFull(frame); stack_pointer = _PyFrame_GetStackPointer(frame); if (f != NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); @@ -12397,7 +12409,7 @@ JUMP_TO_LABEL(error); STOP_TRACING(); int offset = INSTR_OFFSET()-1; int level, handler, lasti; - int handled = get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti); + int handled = get_exception_handler(_PyFrame_GetCodeFull(frame), offset, &level, &handler, &lasti); if (handled == 0) { assert(_PyErr_Occurred(tstate)); _PyStackRef *stackbase = _PyFrame_Stackbase(frame); @@ -12416,13 +12428,13 @@ JUMP_TO_LABEL(error); PyStackRef_XCLOSE(ref); } if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); + int frame_lasti = _PyInterpreterFrame_LASTI_FULL(frame); _PyStackRef lasti = PyStackRef_TagInt(frame_lasti); _PyFrame_StackPush(frame, lasti); } PyObject *exc = _PyErr_GetRaisedException(tstate); _PyFrame_StackPush(frame, PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; + next_instr = _PyFrame_GetBytecodeFull(frame) + handler; int err = monitor_handled(tstate, frame, next_instr, exc); if (err < 0) { JUMP_TO_LABEL(exception_unwind); @@ -12443,13 +12455,13 @@ JUMP_TO_LABEL(error); { assert(_PyErr_Occurred(tstate)); _Py_LeaveRecursiveCallPy(tstate); - assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + assert(FRAME_CORE->owner != FRAME_OWNED_BY_INTERPRETER); _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; + frame = (_PyInterpreterFrame *)(tstate->current_frame = _PyFrame_Core(dying)->previous); _PyEval_FrameClearAndPop(tstate, dying); frame->return_offset = 0; - if (frame->owner == FRAME_OWNED_BY_INTERPRETER) { - tstate->current_frame = frame->previous; + if (FRAME_CORE->owner == FRAME_OWNED_BY_INTERPRETER) { + tstate->current_frame = FRAME_CORE->previous; #if !_Py_TAIL_CALL_INTERP assert(frame == &entry.frame); #endif diff --git a/Modules/_testinternalcapi/test_targets.h b/Modules/_testinternalcapi/test_targets.h index f57c33feec2ac2..482ec45b351502 100644 --- a/Modules/_testinternalcapi/test_targets.h +++ b/Modules/_testinternalcapi/test_targets.h @@ -764,8 +764,8 @@ static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNKNOWN_OPCODE(TAIL_CALL_PARAMS) int opcode = next_instr->op.code; _PyErr_Format(tstate, PyExc_SystemError, "%U:%d: unknown opcode %d", - _PyFrame_GetCode(frame)->co_filename, - PyUnstable_InterpreterFrame_GetLine(frame), + FRAME_CODE->co_filename, + PyUnstable_InterpreterFrame_GetLine(FRAME_CORE), opcode); JUMP_TO_LABEL(error); } diff --git a/Modules/signalmodule.c b/Modules/signalmodule.c index 4d0e224ff757e7..8901f310d942d6 100644 --- a/Modules/signalmodule.c +++ b/Modules/signalmodule.c @@ -1823,7 +1823,7 @@ _PyErr_CheckSignalsTstate(PyThreadState *tstate) */ _Py_atomic_store_int(&is_tripped, 0); - _PyInterpreterFrame *frame = _PyThreadState_GetFrame(tstate); + _PyInterpreterFrameCore *frame = _PyThreadState_GetFrame(tstate); signal_state_t *state = &signal_global_state; for (int i = 1; i < Py_NSIG; i++) { if (!_Py_atomic_load_int_relaxed(&Handlers[i].tripped)) { diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 1d4c0f6785c4b8..596c2c599dfa31 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -108,7 +108,7 @@ framelocalsproxy_getkeyindex(PyFrameObject *frame, PyObject *key, bool read, PyO // value_ptr should only be given if we are reading the value assert(read || value_ptr == NULL); - PyCodeObject *co = _PyFrame_GetCode(frame->f_frame); + PyCodeObject *co = _PyFrame_GetCode(_PyFrame_Core(frame->f_frame)); // Ensure that the key is hashable. Py_hash_t key_hash = PyObject_Hash(key); @@ -248,7 +248,7 @@ framelocalsproxy_setitem(PyObject *self, PyObject *key, PyObject *value) /* Merge locals into fast locals */ PyFrameObject *frame = PyFrameLocalsProxyObject_CAST(self)->frame; _PyStackRef *fast = _PyFrame_GetLocalsArray(frame->f_frame); - PyCodeObject *co = _PyFrame_GetCode(frame->f_frame); + PyCodeObject *co = _PyFrame_GetCode(_PyFrame_Core(frame->f_frame)); int i = framelocalsproxy_getkeyindex(frame, key, false, NULL); if (i == -2) { @@ -373,7 +373,7 @@ static PyObject * framelocalsproxy_keys(PyObject *self, PyObject *Py_UNUSED(ignored)) { PyFrameObject *frame = PyFrameLocalsProxyObject_CAST(self)->frame; - PyCodeObject *co = _PyFrame_GetCode(frame->f_frame); + PyCodeObject *co = _PyFrame_GetCode(_PyFrame_Core(frame->f_frame)); PyObject *names = PyList_New(0); if (names == NULL) { return NULL; @@ -582,7 +582,7 @@ static PyObject * framelocalsproxy_values(PyObject *self, PyObject *Py_UNUSED(ignored)) { PyFrameObject *frame = PyFrameLocalsProxyObject_CAST(self)->frame; - PyCodeObject *co = _PyFrame_GetCode(frame->f_frame); + PyCodeObject *co = _PyFrame_GetCode(_PyFrame_Core(frame->f_frame)); PyObject *values = PyList_New(0); if (values == NULL) { return NULL; @@ -620,7 +620,7 @@ static PyObject * framelocalsproxy_items(PyObject *self, PyObject *Py_UNUSED(ignored)) { PyFrameObject *frame = PyFrameLocalsProxyObject_CAST(self)->frame; - PyCodeObject *co = _PyFrame_GetCode(frame->f_frame); + PyCodeObject *co = _PyFrame_GetCode(_PyFrame_Core(frame->f_frame)); PyObject *items = PyList_New(0); if (items == NULL) { return NULL; @@ -679,7 +679,7 @@ static Py_ssize_t framelocalsproxy_length(PyObject *self) { PyFrameObject *frame = PyFrameLocalsProxyObject_CAST(self)->frame; - PyCodeObject *co = _PyFrame_GetCode(frame->f_frame); + PyCodeObject *co = _PyFrame_GetCode(_PyFrame_Core(frame->f_frame)); Py_ssize_t size = 0; if (frame->f_extra_locals != NULL) { @@ -978,9 +978,9 @@ static PyObject * frame_locals_get_impl(PyFrameObject *self) /*[clinic end generated code: output=b4ace8bb4cae71f4 input=7bd444d0dc8ddf44]*/ { - assert(!_PyFrame_IsIncomplete(self->f_frame)); + assert(!_PyFrame_IsIncomplete(_PyFrame_Core(self->f_frame))); - PyCodeObject *co = _PyFrame_GetCode(self->f_frame); + PyCodeObject *co = _PyFrame_GetCode(_PyFrame_Core(self->f_frame)); if (!(co->co_flags & CO_OPTIMIZED) && !_PyFrame_HasHiddenLocals(self->f_frame)) { if (self->f_frame->f_locals == NULL) { @@ -1005,7 +1005,7 @@ PyFrame_GetLineNumber(PyFrameObject *f) if (f->f_lineno == -1) { // We should calculate it once. If we can't get the line number, // set f->f_lineno to 0. - f->f_lineno = PyUnstable_InterpreterFrame_GetLine(f->f_frame); + f->f_lineno = PyUnstable_InterpreterFrame_GetLine(_PyFrame_Core(f->f_frame)); if (f->f_lineno < 0) { f->f_lineno = 0; return -1; @@ -1015,7 +1015,7 @@ PyFrame_GetLineNumber(PyFrameObject *f) if (f->f_lineno > 0) { return f->f_lineno; } - return PyUnstable_InterpreterFrame_GetLine(f->f_frame); + return PyUnstable_InterpreterFrame_GetLine(_PyFrame_Core(f->f_frame)); } /*[clinic input] @@ -1049,7 +1049,7 @@ static PyObject * frame_lasti_get_impl(PyFrameObject *self) /*[clinic end generated code: output=03275b4f0327d1a2 input=0225ed49cb1fbeeb]*/ { - int lasti = _PyInterpreterFrame_LASTI(self->f_frame); + int lasti = _PyInterpreterFrame_LASTI(_PyFrame_Core(self->f_frame)); if (lasti < 0) { return PyLong_FromLong(-1); } @@ -1625,8 +1625,8 @@ first_line_not_before(int *lines, int len, int line) static bool frame_is_suspended(PyFrameObject *frame) { - assert(!_PyFrame_IsIncomplete(frame->f_frame)); - if (frame->f_frame->owner == FRAME_OWNED_BY_GENERATOR) { + assert(!_PyFrame_IsIncomplete(_PyFrame_Core(frame->f_frame))); + if (_PyFrame_Core(frame->f_frame)->owner & FRAME_OWNED_BY_GENERATOR) { PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame->f_frame); return FRAME_STATE_SUSPENDED(gen->gi_frame_state); } @@ -1657,7 +1657,7 @@ static int frame_lineno_set_impl(PyFrameObject *self, PyObject *value) /*[clinic end generated code: output=e64c86ff6be64292 input=36ed3c896b27fb91]*/ { - PyCodeObject *code = _PyFrame_GetCode(self->f_frame); + PyCodeObject *code = _PyFrame_GetCode(_PyFrame_Core(self->f_frame)); if (value == NULL) { PyErr_SetString(PyExc_AttributeError, "cannot delete attribute"); return -1; @@ -1769,7 +1769,7 @@ frame_lineno_set_impl(PyFrameObject *self, PyObject *value) int64_t best_stack = OVERFLOWED; int best_addr = -1; - int64_t start_stack = stacks[_PyInterpreterFrame_LASTI(self->f_frame)]; + int64_t start_stack = stacks[_PyInterpreterFrame_LASTI(_PyFrame_Core(self->f_frame))]; int err = -1; const char *msg = "cannot find bytecode for specified line"; for (int i = 0; i < len; i++) { @@ -1847,7 +1847,7 @@ frame_lineno_set_impl(PyFrameObject *self, PyObject *value) } /* Finally set the new lasti and return OK. */ self->f_lineno = 0; - self->f_frame->instr_ptr = _PyFrame_GetBytecode(self->f_frame) + best_addr; + self->f_frame->instr_ptr = _PyFrame_GetBytecode(_PyFrame_Core(self->f_frame)) + best_addr; return 0; } @@ -1906,7 +1906,7 @@ static PyObject * frame_generator_get_impl(PyFrameObject *self) /*[clinic end generated code: output=97aeb2392562e55b input=00a2bd008b239ab0]*/ { - if (self->f_frame->owner == FRAME_OWNED_BY_GENERATOR) { + if (_PyFrame_Core(self->f_frame)->owner & FRAME_OWNED_BY_GENERATOR) { PyObject *gen = (PyObject *)_PyGen_GetGeneratorFromFrame(self->f_frame); return Py_NewRef(gen); } @@ -1942,15 +1942,16 @@ frame_dealloc(PyObject *op) * nesting depth for deallocations, the trashcan may have delayed this * deallocation until after f->f_frame is freed. Avoid dereferencing * f->f_frame unless we know it still points to valid memory. */ - _PyInterpreterFrame *frame = (_PyInterpreterFrame *)f->_f_frame_data; + _PyInterpreterFrameCore *frame = (_PyInterpreterFrameCore *)f->_f_frame_data; /* Kill all local variables including specials, if we own them */ - if (f->f_frame == frame && frame->owner == FRAME_OWNED_BY_FRAME_OBJECT) { + if (_PyFrame_Core(f->f_frame) == frame && frame->owner == FRAME_OWNED_BY_FRAME_OBJECT) { + _PyInterpreterFrame *ff = _PyFrame_Full(frame); PyStackRef_CLEAR(frame->f_executable); - PyStackRef_CLEAR(frame->f_funcobj); - Py_CLEAR(frame->f_locals); - _PyStackRef *locals = _PyFrame_GetLocalsArray(frame); - _PyStackRef *sp = frame->stackpointer; + PyStackRef_CLEAR(ff->f_funcobj); + Py_CLEAR(ff->f_locals); + _PyStackRef *locals = _PyFrame_GetLocalsArray(ff); + _PyStackRef *sp = ff->stackpointer; while (sp > locals) { sp--; PyStackRef_CLEAR(*sp); @@ -1973,7 +1974,7 @@ frame_traverse(PyObject *op, visitproc visit, void *arg) Py_VISIT(f->f_extra_locals); Py_VISIT(f->f_locals_cache); Py_VISIT(f->f_overwritten_fast_locals); - if (f->f_frame->owner != FRAME_OWNED_BY_FRAME_OBJECT) { + if (_PyFrame_Core(f->f_frame)->owner != FRAME_OWNED_BY_FRAME_OBJECT) { return 0; } assert(f->f_frame->frame_obj == NULL); @@ -2013,19 +2014,20 @@ static PyObject * frame_clear_impl(PyFrameObject *self) /*[clinic end generated code: output=864c662f16e9bfcc input=c358f9cff5f9b681]*/ { - if (self->f_frame->owner == FRAME_OWNED_BY_GENERATOR) { + if (_PyFrame_Core(self->f_frame)->owner & FRAME_OWNED_BY_GENERATOR) { PyGenObject *gen = _PyGen_GetGeneratorFromFrame(self->f_frame); if (_PyGen_ClearFrame(gen) < 0) { return NULL; } } - else if (self->f_frame->owner == FRAME_OWNED_BY_THREAD) { + else if (_PyFrame_Core(self->f_frame)->owner == FRAME_OWNED_BY_THREAD || + _PyFrame_IsExternalFrame(_PyFrame_Core(self->f_frame))) { PyErr_SetString(PyExc_RuntimeError, "cannot clear an executing frame"); return NULL; } else { - assert(self->f_frame->owner == FRAME_OWNED_BY_FRAME_OBJECT); + assert(_PyFrame_Core(self->f_frame)->owner == FRAME_OWNED_BY_FRAME_OBJECT); (void)frame_tp_clear((PyObject *)self); } Py_RETURN_NONE; @@ -2044,7 +2046,7 @@ frame___sizeof___impl(PyFrameObject *self) { Py_ssize_t res; res = offsetof(PyFrameObject, _f_frame_data) + offsetof(_PyInterpreterFrame, localsplus); - PyCodeObject *code = _PyFrame_GetCode(self->f_frame); + PyCodeObject *code = _PyFrame_GetCode(_PyFrame_Core(self->f_frame)); res += _PyFrame_NumSlotsForCodeObject(code) * sizeof(PyObject *); return PyLong_FromSsize_t(res); } @@ -2054,7 +2056,7 @@ frame_repr(PyObject *op) { PyFrameObject *f = PyFrameObject_CAST(op); int lineno = PyFrame_GetLineNumber(f); - PyCodeObject *code = _PyFrame_GetCode(f->f_frame); + PyCodeObject *code = _PyFrame_GetCode(_PyFrame_Core(f->f_frame)); return PyUnicode_FromFormat( "", f, code->co_filename, lineno, code->co_name); @@ -2162,10 +2164,10 @@ PyFrame_New(PyThreadState *tstate, PyCodeObject *code, } init_frame(tstate, (_PyInterpreterFrame *)f->_f_frame_data, func, locals); f->f_frame = (_PyInterpreterFrame *)f->_f_frame_data; - f->f_frame->owner = FRAME_OWNED_BY_FRAME_OBJECT; + _PyFrame_Core(f->f_frame)->owner = FRAME_OWNED_BY_FRAME_OBJECT; // This frame needs to be "complete", so pretend that the first RESUME ran: f->f_frame->instr_ptr = _PyCode_CODE(code) + code->_co_firsttraceable + 1; - assert(!_PyFrame_IsIncomplete(f->f_frame)); + assert(!_PyFrame_IsIncomplete(_PyFrame_Core(f->f_frame))); Py_DECREF(func); _PyObject_GC_TRACK(f); return f; @@ -2177,10 +2179,11 @@ frame_init_get_vars(_PyInterpreterFrame *frame) { // COPY_FREE_VARS has no quickened forms, so no need to use _PyOpcode_Deopt // here: - PyCodeObject *co = _PyFrame_GetCode(frame); - int lasti = _PyInterpreterFrame_LASTI(frame); + _PyInterpreterFrameCore *f = _PyFrame_Core(frame); + PyCodeObject *co = _PyFrame_GetCode(f); + int lasti = _PyInterpreterFrame_LASTI(f); if (!(lasti < 0 - && _PyFrame_GetBytecode(frame)->op.code == COPY_FREE_VARS + && _PyFrame_GetBytecode(f)->op.code == COPY_FREE_VARS && PyStackRef_FunctionCheck(frame->f_funcobj))) { /* Free vars are initialized */ @@ -2188,7 +2191,7 @@ frame_init_get_vars(_PyInterpreterFrame *frame) } /* Free vars have not been initialized -- Do that */ - PyFunctionObject *func = _PyFrame_GetFunction(frame); + PyFunctionObject *func = _PyFrame_GetFunction(f); PyObject *closure = func->func_closure; int offset = PyUnstable_Code_GetFirstFree(co); for (int i = 0; i < co->co_nfreevars; ++i) { @@ -2196,7 +2199,7 @@ frame_init_get_vars(_PyInterpreterFrame *frame) frame->localsplus[offset + i] = PyStackRef_FromPyObjectNew(o); } // COPY_FREE_VARS doesn't have inline CACHEs, either: - frame->instr_ptr = _PyFrame_GetBytecode(frame); + frame->instr_ptr = _PyFrame_GetBytecode(f); } @@ -2229,7 +2232,7 @@ frame_get_var(_PyInterpreterFrame *frame, PyCodeObject *co, int i, else if (kind & CO_FAST_CELL) { if (value != NULL) { if (PyCell_Check(value)) { - assert(!_PyFrame_IsIncomplete(frame)); + assert(!_PyFrame_IsIncomplete(_PyFrame_Core(frame))); value = PyCell_GetRef((PyCellObject *)value); } else { @@ -2256,7 +2259,7 @@ _PyFrame_HasHiddenLocals(_PyInterpreterFrame *frame) * This function returns if there are hidden locals introduced by PEP 709, * which are the isolated fast locals for inline comprehensions */ - PyCodeObject* co = _PyFrame_GetCode(frame); + PyCodeObject* co = _PyFrame_GetCode(_PyFrame_Core(frame)); for (int i = 0; i < co->co_nlocalsplus; i++) { _PyLocals_Kind kind = _PyLocals_GetKind(co->co_localspluskinds, i); @@ -2273,28 +2276,28 @@ _PyFrame_HasHiddenLocals(_PyInterpreterFrame *frame) PyObject * -_PyFrame_GetLocals(_PyInterpreterFrame *frame) +_PyFrame_GetLocals(_PyInterpreterFrameCore *frame) { + _PyInterpreterFrame *f = _PyFrame_EnsureFrameFullyInitialized(frame); + // We should try to avoid creating the FrameObject if possible. // So we check if the frame is a module or class level scope PyCodeObject *co = _PyFrame_GetCode(frame); - if (!(co->co_flags & CO_OPTIMIZED) && !_PyFrame_HasHiddenLocals(frame)) { - if (frame->f_locals == NULL) { + if (!(co->co_flags & CO_OPTIMIZED) && !_PyFrame_HasHiddenLocals(f)) { + if (f->f_locals == NULL) { // We found cases when f_locals is NULL for non-optimized code. // We fill the f_locals with an empty dict to avoid crash until // we find the root cause. - frame->f_locals = PyDict_New(); - if (frame->f_locals == NULL) { + f->f_locals = PyDict_New(); + if (f->f_locals == NULL) { return NULL; } } - return Py_NewRef(frame->f_locals); + return Py_NewRef(f->f_locals); } - PyFrameObject* f = _PyFrame_GetFrameObject(frame); - - return _PyFrameLocalsProxy_New(f); + return _PyFrameLocalsProxy_New(_PyFrame_GetFrameObject(frame)); } @@ -2310,7 +2313,7 @@ PyFrame_GetVar(PyFrameObject *frame_obj, PyObject *name) _PyInterpreterFrame *frame = frame_obj->f_frame; frame_init_get_vars(frame); - PyCodeObject *co = _PyFrame_GetCode(frame); + PyCodeObject *co = _PyFrame_GetCode(_PyFrame_Core(frame)); for (int i = 0; i < co->co_nlocalsplus; i++) { PyObject *var_name = PyTuple_GET_ITEM(co->co_localsplusnames, i); if (!_PyUnicode_Equal(var_name, name)) { @@ -2373,7 +2376,7 @@ int _PyFrame_IsEntryFrame(PyFrameObject *frame) { assert(frame != NULL); - _PyInterpreterFrame *f = frame->f_frame; + _PyInterpreterFrameCore *f = _PyFrame_Core(frame->f_frame); assert(!_PyFrame_IsIncomplete(f)); return f->previous && f->previous->owner == FRAME_OWNED_BY_INTERPRETER; } @@ -2384,8 +2387,8 @@ PyFrame_GetCode(PyFrameObject *frame) assert(frame != NULL); PyObject *code; Py_BEGIN_CRITICAL_SECTION(frame); - assert(!_PyFrame_IsIncomplete(frame->f_frame)); - code = Py_NewRef(_PyFrame_GetCode(frame->f_frame)); + assert(!_PyFrame_IsIncomplete(_PyFrame_Core(frame->f_frame))); + code = Py_NewRef(_PyFrame_GetCode(_PyFrame_Core(frame->f_frame))); Py_END_CRITICAL_SECTION(); return (PyCodeObject *)code; } @@ -2395,11 +2398,10 @@ PyFrameObject* PyFrame_GetBack(PyFrameObject *frame) { assert(frame != NULL); - assert(!_PyFrame_IsIncomplete(frame->f_frame)); + assert(!_PyFrame_IsIncomplete(_PyFrame_Core(frame->f_frame))); PyFrameObject *back = frame->f_back; if (back == NULL) { - _PyInterpreterFrame *prev = frame->f_frame->previous; - prev = _PyFrame_GetFirstComplete(prev); + _PyInterpreterFrameCore *prev = _PyFrame_GetFirstComplete(_PyFrame_Core(frame->f_frame)->previous); if (prev) { back = _PyFrame_GetFrameObject(prev); } @@ -2410,21 +2412,21 @@ PyFrame_GetBack(PyFrameObject *frame) PyObject* PyFrame_GetLocals(PyFrameObject *frame) { - assert(!_PyFrame_IsIncomplete(frame->f_frame)); + assert(!_PyFrame_IsIncomplete(_PyFrame_Core(frame->f_frame))); return frame_locals_get((PyObject *)frame, NULL); } PyObject* PyFrame_GetGlobals(PyFrameObject *frame) { - assert(!_PyFrame_IsIncomplete(frame->f_frame)); + assert(!_PyFrame_IsIncomplete(_PyFrame_Core(frame->f_frame))); return frame_globals_get((PyObject *)frame, NULL); } PyObject* PyFrame_GetBuiltins(PyFrameObject *frame) { - assert(!_PyFrame_IsIncomplete(frame->f_frame)); + assert(!_PyFrame_IsIncomplete(_PyFrame_Core(frame->f_frame))); return frame_builtins_get((PyObject *)frame, NULL); } @@ -2433,8 +2435,8 @@ PyFrame_GetLasti(PyFrameObject *frame) { int ret; Py_BEGIN_CRITICAL_SECTION(frame); - assert(!_PyFrame_IsIncomplete(frame->f_frame)); - int lasti = _PyInterpreterFrame_LASTI(frame->f_frame); + assert(!_PyFrame_IsIncomplete(_PyFrame_Core(frame->f_frame))); + int lasti = _PyInterpreterFrame_LASTI(_PyFrame_Core(frame->f_frame)); ret = lasti < 0 ? -1 : lasti * (int)sizeof(_Py_CODEUNIT); Py_END_CRITICAL_SECTION(); return ret; @@ -2443,6 +2445,6 @@ PyFrame_GetLasti(PyFrameObject *frame) PyObject * PyFrame_GetGenerator(PyFrameObject *frame) { - assert(!_PyFrame_IsIncomplete(frame->f_frame)); + assert(!_PyFrame_IsIncomplete(_PyFrame_Core(frame->f_frame))); return frame_generator_get((PyObject *)frame, NULL); } diff --git a/Objects/genobject.c b/Objects/genobject.c index fcdb9017a35f5b..ee02a5c5eb01fc 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -54,7 +54,7 @@ static const char *ASYNC_GEN_IGNORED_EXIT_MSG = /* Returns a borrowed reference */ static inline PyCodeObject * _PyGen_GetCode(PyGenObject *gen) { - return _PyFrame_GetCode(&gen->gi_iframe); + return _PyFrame_GetCode(_PyFrame_Core(&gen->gi_iframe)); } PyCodeObject * @@ -74,7 +74,7 @@ gen_traverse(PyObject *self, visitproc visit, void *arg) if (gen->gi_frame_state != FRAME_CLEARED) { _PyInterpreterFrame *frame = &gen->gi_iframe; assert(frame->frame_obj == NULL || - frame->frame_obj->f_frame->owner == FRAME_OWNED_BY_GENERATOR); + _PyFrame_Core(frame->frame_obj->f_frame)->owner & FRAME_OWNED_BY_GENERATOR); int err = _PyFrame_Traverse(frame, visit, arg); if (err) { return err; @@ -83,7 +83,7 @@ gen_traverse(PyObject *self, visitproc visit, void *arg) else { // We still need to visit the code object when the frame is cleared to // ensure that it's kept alive if the reference is deferred. - _Py_VISIT_STACKREF(gen->gi_iframe.f_executable); + _Py_VISIT_STACKREF(_PyFrame_Core(&gen->gi_iframe)->f_executable); } /* No need to visit cr_origin, because it's just tuples/str/int, so can't participate in a reference cycle. */ @@ -155,7 +155,7 @@ gen_clear_frame(PyGenObject *gen) { assert(FT_ATOMIC_LOAD_INT8_RELAXED(gen->gi_frame_state) == FRAME_CLEARED); _PyInterpreterFrame *frame = &gen->gi_iframe; - frame->previous = NULL; + _PyFrame_Core(frame)->previous = NULL; _PyFrame_ClearExceptCode(frame); _PyErr_ClearExcState(&gen->gi_exc_state); } @@ -217,7 +217,7 @@ gen_dealloc(PyObject *self) gen_clear_frame(gen); } assert(gen->gi_exc_state.exc_value == NULL); - PyStackRef_CLEAR(gen->gi_iframe.f_executable); + PyStackRef_CLEAR(_PyFrame_Core(&gen->gi_iframe)->f_executable); Py_CLEAR(gen->gi_name); Py_CLEAR(gen->gi_qualname); @@ -268,7 +268,7 @@ gen_send_ex2(PyGenObject *gen, PyObject *arg, PyObject **presult, int exc) assert(tstate->exc_info == prev_exc_info); #ifndef Py_GIL_DISABLED assert(gen->gi_exc_state.previous_item == NULL); - assert(frame->previous == NULL); + assert(_PyFrame_Core(frame)->previous == NULL); assert(gen->gi_frame_state != FRAME_EXECUTING); #endif @@ -662,15 +662,15 @@ _gen_throw(PyGenObject *gen, int close_on_genexit, /* Link frame into the stack to enable complete backtraces. */ /* XXX We should probably be updating the current frame somewhere in ceval.c. */ - _PyInterpreterFrame *prev = tstate->current_frame; - frame->previous = prev; - tstate->current_frame = frame; + _PyInterpreterFrameCore *prev = tstate->current_frame; + _PyFrame_Core(frame)->previous = prev; + tstate->current_frame = _PyFrame_Core(frame); /* Close the generator that we are currently iterating with 'yield from' or awaiting on with 'await'. */ ret = _gen_throw((PyGenObject *)yf, close_on_genexit, typ, val, tb); tstate->current_frame = prev; - frame->previous = NULL; + _PyFrame_Core(frame)->previous = NULL; } else { /* `yf` is an iterator or a coroutine-like object. */ @@ -685,12 +685,12 @@ _gen_throw(PyGenObject *gen, int close_on_genexit, goto throw_here; } - _PyInterpreterFrame *prev = tstate->current_frame; - frame->previous = prev; - tstate->current_frame = frame; + _PyInterpreterFrameCore *prev = tstate->current_frame; + _PyFrame_Core(frame)->previous = prev; + tstate->current_frame = _PyFrame_Core(frame); ret = PyObject_CallFunctionObjArgs(meth, typ, val, tb, NULL); tstate->current_frame = prev; - frame->previous = NULL; + _PyFrame_Core(frame)->previous = NULL; Py_DECREF(meth); } Py_DECREF(yf); @@ -912,7 +912,7 @@ _gen_getframe(PyGenObject *gen, const char *const name) Py_RETURN_NONE; } // TODO: still not thread-safe with free threading - return _Py_XNewRef((PyObject *)_PyFrame_GetFrameObject(&gen->gi_iframe)); + return _Py_XNewRef((PyObject *)_PyFrame_GetFrameObject(_PyFrame_Core(&gen->gi_iframe))); } static PyObject * @@ -1053,7 +1053,7 @@ make_gen(PyTypeObject *type, PyFunctionObject *func) gen->gi_weakreflist = NULL; gen->gi_exc_state.exc_value = NULL; gen->gi_exc_state.previous_item = NULL; - gen->gi_iframe.f_executable = PyStackRef_None; + _PyFrame_Core(&gen->gi_iframe)->f_executable = PyStackRef_None; assert(func->func_name != NULL); gen->gi_name = Py_NewRef(func->func_name); assert(func->func_qualname != NULL); @@ -1063,7 +1063,7 @@ make_gen(PyTypeObject *type, PyFunctionObject *func) } static PyObject * -compute_cr_origin(int origin_depth, _PyInterpreterFrame *current_frame); +compute_cr_origin(int origin_depth, _PyInterpreterFrameCore *current_frame); PyObject * _Py_MakeCoro(PyFunctionObject *func) @@ -1098,7 +1098,7 @@ _Py_MakeCoro(PyFunctionObject *func) if (origin_depth == 0) { ((PyCoroObject *)coro)->cr_origin_or_finalizer = NULL; } else { - _PyInterpreterFrame *frame = tstate->current_frame; + _PyInterpreterFrameCore *frame = tstate->current_frame; assert(frame); assert(_PyFrame_IsIncomplete(frame)); frame = _PyFrame_GetFirstComplete(frame->previous); @@ -1116,7 +1116,7 @@ static PyObject * gen_new_with_qualname(PyTypeObject *type, PyFrameObject *f, PyObject *name, PyObject *qualname) { - PyCodeObject *code = _PyFrame_GetCode(f->f_frame); + PyCodeObject *code = _PyFrame_GetCode(_PyFrame_Core(f->f_frame)); int size = code->co_nlocalsplus + code->co_stacksize; PyGenObject *gen = PyObject_GC_NewVar(PyGenObject, type, size); if (gen == NULL) { @@ -1125,13 +1125,13 @@ gen_new_with_qualname(PyTypeObject *type, PyFrameObject *f, } /* Copy the frame */ assert(f->f_frame->frame_obj == NULL); - assert(f->f_frame->owner == FRAME_OWNED_BY_FRAME_OBJECT); + assert(_PyFrame_Core(f->f_frame)->owner == FRAME_OWNED_BY_FRAME_OBJECT); _PyInterpreterFrame *frame = &gen->gi_iframe; _PyFrame_Copy((_PyInterpreterFrame *)f->_f_frame_data, frame); gen->gi_frame_state = FRAME_CREATED; assert(frame->frame_obj == f); f->f_frame = frame; - frame->owner = FRAME_OWNED_BY_GENERATOR; + _PyFrame_Core(frame)->owner = FRAME_OWNED_BY_GENERATOR; assert(PyObject_GC_IsTracked((PyObject *)f)); Py_DECREF(f); gen->gi_weakreflist = NULL; @@ -1466,9 +1466,9 @@ PyTypeObject _PyCoroWrapper_Type = { }; static PyObject * -compute_cr_origin(int origin_depth, _PyInterpreterFrame *current_frame) +compute_cr_origin(int origin_depth, _PyInterpreterFrameCore *current_frame) { - _PyInterpreterFrame *frame = current_frame; + _PyInterpreterFrameCore *frame = current_frame; /* First count how many frames we have */ int frame_count = 0; for (; frame && frame_count < origin_depth; ++frame_count) { diff --git a/Objects/object.c b/Objects/object.c index 649b109d5cb0bc..e199c559148de3 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -2517,6 +2517,7 @@ static PyTypeObject* static_types[] = { &PyTuple_Type, &PyUnicodeIter_Type, &PyUnicode_Type, + &PyUnstable_ExternalExecutable_Type, &PyWrapperDescr_Type, &PyZip_Type, &Py_GenericAliasType, @@ -2753,13 +2754,13 @@ PyUnstable_Object_IsUniqueReferencedTemporary(PyObject *op) return 0; } - _PyInterpreterFrame *frame = _PyEval_GetFrame(); - if (frame == NULL) { + _PyInterpreterFrameCore *frame = _PyEval_GetFrame(); + if (frame == NULL || _PyFrame_IsExternalFrame(frame)) { return 0; } - _PyStackRef *base = _PyFrame_Stackbase(frame); - _PyStackRef *stackpointer = frame->stackpointer; + _PyStackRef *base = _PyFrame_Stackbase(_PyFrame_Full(frame)); + _PyStackRef *stackpointer = _PyFrame_Full(frame)->stackpointer; while (stackpointer > base) { stackpointer--; _PyStackRef ref = *stackpointer; @@ -3190,7 +3191,7 @@ _Py_Dealloc(PyObject *op) #if !defined(Py_GIL_DISABLED) && !defined(Py_STACKREF_DEBUG) /* This assertion doesn't hold for the free-threading build, as * PyStackRef_CLOSE_SPECIALIZED is not implemented */ - assert(tstate->current_frame == NULL || tstate->current_frame->stackpointer != NULL); + assert(_PyFrame_StackpointerSaved()); #endif PyObject *old_exc = tstate != NULL ? tstate->current_exception : NULL; // Keep the old exception type alive to prevent undefined behavior diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 54263fd603ed73..dd5cc9d5fa207e 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -12524,7 +12524,7 @@ super_descr_get(PyObject *self, PyObject *obj, PyObject *type) } static int -super_init_without_args(_PyInterpreterFrame *cframe, PyTypeObject **type_p, +super_init_without_args(_PyInterpreterFrameCore *cframe, PyTypeObject **type_p, PyObject **obj_p) { PyCodeObject *co = _PyFrame_GetCode(cframe); @@ -12535,7 +12535,8 @@ super_init_without_args(_PyInterpreterFrame *cframe, PyTypeObject **type_p, } assert(_PyFrame_GetCode(cframe)->co_nlocalsplus > 0); - PyObject *firstarg = PyStackRef_AsPyObjectBorrow(_PyFrame_GetLocalsArray(cframe)[0]); + _PyInterpreterFrame *frame = _PyFrame_EnsureFrameFullyInitialized(cframe); + PyObject *firstarg = PyStackRef_AsPyObjectBorrow(_PyFrame_GetLocalsArray(frame)[0]); if (firstarg == NULL) { PyErr_SetString(PyExc_RuntimeError, "super(): arg[0] deleted"); return -1; @@ -12568,7 +12569,7 @@ super_init_without_args(_PyInterpreterFrame *cframe, PyTypeObject **type_p, PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i); assert(PyUnicode_Check(name)); if (_PyUnicode_Equal(name, &_Py_ID(__class__))) { - PyObject *cell = PyStackRef_AsPyObjectBorrow(_PyFrame_GetLocalsArray(cframe)[i]); + PyObject *cell = PyStackRef_AsPyObjectBorrow(_PyFrame_GetLocalsArray(frame)[i]); if (cell == NULL || !PyCell_Check(cell)) { PyErr_SetString(PyExc_RuntimeError, "super(): bad __class__ cell"); @@ -12631,7 +12632,7 @@ super_init_impl(PyObject *self, PyTypeObject *type, PyObject *obj) { /* Call super(), without args -- fill in from __class__ and first local variable on the stack. */ PyThreadState *tstate = _PyThreadState_GET(); - _PyInterpreterFrame *frame = _PyThreadState_GetFrame(tstate); + _PyInterpreterFrameCore *frame = _PyThreadState_GetFrame(tstate); if (frame == NULL) { PyErr_SetString(PyExc_RuntimeError, "super(): no current frame"); diff --git a/Objects/typevarobject.c b/Objects/typevarobject.c index 2ec546aff52c0a..0a7044c7794239 100644 --- a/Objects/typevarobject.c +++ b/Objects/typevarobject.c @@ -1,6 +1,6 @@ // TypeVar, TypeVarTuple, ParamSpec, and TypeAlias #include "Python.h" -#include "pycore_interpframe.h" // _PyInterpreterFrame +#include "pycore_interpframe.h" // _PyInterpreterFrameCore #include "pycore_object.h" // _PyObject_GC_TRACK/UNTRACK, PyAnnotateFormat #include "pycore_typevarobject.h" #include "pycore_unicodeobject.h" // _PyUnicode_EqualToASCIIString() @@ -385,10 +385,11 @@ make_union(PyObject *self, PyObject *other) static PyObject * caller(void) { - _PyInterpreterFrame *f = _PyThreadState_GET()->current_frame; - if (f == NULL) { + _PyInterpreterFrameCore *frame = _PyThreadState_GET()->current_frame; + if (frame == NULL) { Py_RETURN_NONE; } + _PyInterpreterFrame *f = _PyFrame_EnsureFrameFullyInitialized(frame); if (f == NULL || PyStackRef_IsNull(f->f_funcobj)) { Py_RETURN_NONE; } diff --git a/Python/_warnings.c b/Python/_warnings.c index d44d414bc93a04..48b918a0a504fa 100644 --- a/Python/_warnings.c +++ b/Python/_warnings.c @@ -1036,7 +1036,7 @@ setup_context(Py_ssize_t stack_level, } else { globals = f->f_frame->f_globals; - *filename = Py_NewRef(_PyFrame_GetCode(f->f_frame)->co_filename); + *filename = Py_NewRef(_PyFrame_GetCode(_PyFrame_Core(f->f_frame))->co_filename); *lineno = PyFrame_GetLineNumber(f); Py_DECREF(f); } diff --git a/Python/bytecodes.c b/Python/bytecodes.c index cef368e9b0721a..7e7fc147aa2adf 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -187,9 +187,9 @@ dummy_func( #endif if (check_instrumentation) { uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; - uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); + uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCodeFull(frame)->_co_instrumentation_version); if (code_version != global_version) { - int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); + int err = _Py_Instrument(_PyFrame_GetCodeFull(frame), tstate->interp); if (err) { ERROR_NO_POP(); } @@ -204,9 +204,9 @@ dummy_func( if (frame->tlbc_index != ((_PyThreadStateImpl *)tstate)->tlbc_index) { _Py_CODEUNIT *bytecode = - _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); + _PyEval_GetExecutableCode(tstate, _PyFrame_GetCodeFull(frame)); ERROR_IF(bytecode == NULL); - ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame); + ptrdiff_t off = this_instr - _PyFrame_GetBytecodeFull(frame); frame->tlbc_index = ((_PyThreadStateImpl *)tstate)->tlbc_index; frame->instr_ptr = bytecode + off; // Make sure this_instr gets reset correctly for any uops that @@ -229,7 +229,7 @@ dummy_func( _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; #endif uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); - uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); + uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCodeFull(frame)->_co_instrumentation_version); assert((version & _PY_EVAL_EVENTS_MASK) == 0); DEOPT_IF(eval_breaker != version); #ifdef Py_GIL_DISABLED @@ -263,7 +263,7 @@ dummy_func( if (PyStackRef_IsNull(value_s)) { _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, UNBOUNDLOCAL_ERROR_MSG, - PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) + PyTuple_GetItem(_PyFrame_GetCodeFull(frame)->co_localsplusnames, oparg) ); ERROR_IF(true); } @@ -1238,10 +1238,10 @@ dummy_func( } tier1 inst(INTERPRETER_EXIT, (retval --)) { - assert(frame->owner == FRAME_OWNED_BY_INTERPRETER); - assert(_PyFrame_IsIncomplete(frame)); + assert(FRAME_CORE->owner == FRAME_OWNED_BY_INTERPRETER); + assert(_PyFrame_IsIncomplete(FRAME_CORE)); /* Restore previous frame and return. */ - tstate->current_frame = frame->previous; + tstate->current_frame = FRAME_CORE->previous; assert(!_PyErr_Occurred(tstate)); PyObject *result = PyStackRef_AsPyObjectSteal(retval); #if !_Py_TAIL_CALL_INTERP @@ -1263,7 +1263,7 @@ dummy_func( // retval is popped from the stack, but res // is pushed to a different frame, the callers' frame. inst(RETURN_VALUE, (retval -- res)) { - assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + assert(FRAME_CORE->owner != FRAME_OWNED_BY_INTERPRETER); _PyStackRef temp = PyStackRef_MakeHeapSafe(retval); DEAD(retval); SAVE_STACK(); @@ -1271,7 +1271,7 @@ dummy_func( _Py_LeaveRecursiveCallPy(tstate); // GH-99729: We need to unlink the frame *before* clearing it: _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; + frame = (_PyInterpreterFrame *)(tstate->current_frame = _PyFrame_Core(dying)->previous); _PyEval_FrameClearAndPop(tstate, dying); RELOAD_STACK(); LOAD_IP(frame->return_offset); @@ -1360,7 +1360,7 @@ dummy_func( op(_SEND, (receiver, v -- receiver, retval)) { PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); PyObject *retval_o; - assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + assert(FRAME_CORE->owner != FRAME_OWNED_BY_INTERPRETER); if (!IS_PEP523_HOOKED(tstate) && (Py_TYPE(receiver_o) == &PyGen_Type || Py_TYPE(receiver_o) == &PyCoro_Type) && gen_try_set_executing((PyGenObject *)receiver_o)) @@ -1374,8 +1374,8 @@ dummy_func( tstate->exc_info = &gen->gi_exc_state; assert(INSTRUCTION_SIZE + oparg <= UINT16_MAX); frame->return_offset = (uint16_t)(INSTRUCTION_SIZE + oparg); - assert(gen_frame->previous == NULL); - gen_frame->previous = frame; + assert(_PyFrame_Core(gen_frame)->previous == NULL); + _PyFrame_Core(gen_frame)->previous = FRAME_CORE; DISPATCH_INLINED(gen_frame); } if (PyStackRef_IsNone(v) && PyIter_Check(receiver_o)) { @@ -1419,7 +1419,7 @@ dummy_func( tstate->exc_info = &gen->gi_exc_state; assert(INSTRUCTION_SIZE + oparg <= UINT16_MAX); frame->return_offset = (uint16_t)(INSTRUCTION_SIZE + oparg); - pushed_frame->previous = frame; + _PyFrame_Core(pushed_frame)->previous = FRAME_CORE; gen_frame = PyStackRef_Wrap(pushed_frame); } @@ -1433,7 +1433,7 @@ dummy_func( // NOTE: It's important that YIELD_VALUE never raises an exception! // The compiler treats any exception raised here as a failed close() // or throw() call. - assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + assert(FRAME_CORE->owner != FRAME_OWNED_BY_INTERPRETER); frame->instr_ptr++; PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); @@ -1445,8 +1445,8 @@ dummy_func( gen->gi_exc_state.previous_item = NULL; _Py_LeaveRecursiveCallPy(tstate); _PyInterpreterFrame *gen_frame = frame; - frame = tstate->current_frame = frame->previous; - gen_frame->previous = NULL; + frame = (_PyInterpreterFrame *)(tstate->current_frame = FRAME_CORE->previous); + _PyFrame_Core(gen_frame)->previous = NULL; ((_PyThreadStateImpl *)tstate)->generator_return_kind = GENERATOR_YIELD; FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, FRAME_SUSPENDED + oparg); /* We don't know which of these is relevant here, so keep them equal */ @@ -1494,7 +1494,7 @@ dummy_func( assert(oparg >= 0 && oparg <= 2); if (oparg) { - frame->instr_ptr = _PyFrame_GetBytecode(frame) + PyStackRef_UntagInt(values[0]); + frame->instr_ptr = _PyFrame_GetBytecodeFull(frame) + PyStackRef_UntagInt(values[0]); } assert(exc && PyExceptionInstance_Check(exc)); _PyErr_SetRaisedException(tstate, exc); @@ -1915,7 +1915,7 @@ dummy_func( if (PyStackRef_IsNull(v)) { _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, UNBOUNDLOCAL_ERROR_MSG, - PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) + PyTuple_GetItem(_PyFrame_GetCodeFull(frame)->co_localsplusnames, oparg) ); ERROR_IF(true); } @@ -1943,7 +1943,7 @@ dummy_func( // Fortunately we don't need its superpower. PyObject *oldobj = PyCell_SwapTakeRef((PyCellObject *)cell, NULL); if (oldobj == NULL) { - _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCodeFull(frame), oparg); ERROR_NO_POP(); } Py_DECREF(oldobj); @@ -1954,8 +1954,8 @@ dummy_func( PyObject *class_dict = PyStackRef_AsPyObjectBorrow(class_dict_st); assert(class_dict); - assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); - name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); + assert(oparg >= 0 && oparg < _PyFrame_GetCodeFull(frame)->co_nlocalsplus); + name = PyTuple_GET_ITEM(_PyFrame_GetCodeFull(frame)->co_localsplusnames, oparg); int err; PyObject* value_o = _PyMapping_GetOptionalItem2(class_dict, name, &err); if (err < 0) { @@ -1965,7 +1965,7 @@ dummy_func( PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); value_o = PyCell_GetRef(cell); if (value_o == NULL) { - _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCodeFull(frame), oparg); ERROR_NO_POP(); } } @@ -1977,7 +1977,7 @@ dummy_func( PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); value = _PyCell_GetStackRef(cell); if (PyStackRef_IsNull(value)) { - _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCodeFull(frame), oparg); ERROR_IF(true); } } @@ -1989,7 +1989,7 @@ dummy_func( inst(COPY_FREE_VARS, (--)) { /* Copy closure variables to free variables */ - PyCodeObject *co = _PyFrame_GetCode(frame); + PyCodeObject *co = _PyFrame_GetCodeFull(frame); assert(PyStackRef_FunctionCheck(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); PyObject *closure = func->func_closure; @@ -2966,7 +2966,7 @@ dummy_func( inst(IMPORT_NAME, (level, fromlist -- res)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - PyObject *res_o = _PyEval_ImportName(tstate, frame, name, + PyObject *res_o = _PyEval_ImportName(tstate, FRAME_CORE, name, PyStackRef_AsPyObjectBorrow(fromlist), PyStackRef_AsPyObjectBorrow(level)); DECREF_INPUTS(); @@ -3069,7 +3069,7 @@ dummy_func( next_instr = this_instr; goto stop_tracing; } - PyCodeObject *code = _PyFrame_GetCode(frame); + PyCodeObject *code = _PyFrame_GetCodeFull(frame); _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; assert(executor->vm_data.index == INSTR_OFFSET() - 1); assert(executor->vm_data.code == code); @@ -3212,7 +3212,7 @@ dummy_func( PyObject *iterable_o = PyStackRef_AsPyObjectBorrow(iterable); if (PyCoro_CheckExact(iterable_o)) { /* `iterable` is a coroutine */ - if (!(_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) { + if (!(_PyFrame_GetCodeFull(frame)->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) { /* and it is used in a 'yield from' expression of a regular generator. */ _PyErr_SetString(tstate, PyExc_TypeError, @@ -3492,7 +3492,7 @@ dummy_func( _PyFrame_StackPush(pushed_frame, PyStackRef_None); gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; - pushed_frame->previous = frame; + _PyFrame_Core(pushed_frame)->previous = FRAME_CORE; // oparg is the return offset from the next instruction. frame->return_offset = (uint16_t)(INSTRUCTION_SIZE + oparg); gen_frame = PyStackRef_Wrap(pushed_frame); @@ -3778,7 +3778,7 @@ dummy_func( PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( tstate, callable, locals, - arguments, total_args, NULL, frame + arguments, total_args, NULL, FRAME_CORE ); DEAD(args); DEAD(self_or_null); @@ -3847,7 +3847,7 @@ dummy_func( PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( tstate, callable, locals, - args, total_args, NULL, frame + args, total_args, NULL, FRAME_CORE ); // The frame has stolen all the arguments from the stack. INPUTS_DEAD(); @@ -4004,9 +4004,10 @@ dummy_func( DEAD(new_frame); SYNC_SP(); _PyFrame_SetStackPointer(frame, stack_pointer); - assert(temp->previous == frame || temp->previous->previous == frame); + assert(_PyFrame_Core(temp)->previous == FRAME_CORE || _PyFrame_Core(temp)->previous->previous == FRAME_CORE); CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; + frame = temp; + tstate->current_frame = _PyFrame_Core(temp); tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); @@ -4156,13 +4157,13 @@ dummy_func( op(_CREATE_INIT_FRAME, (init, self, args[oparg] -- init_frame)) { _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( - tstate, (PyCodeObject *)&_Py_InitCleanup, 1, frame); - assert(_PyFrame_GetBytecode(shim)[0].op.code == EXIT_INIT_CHECK); - assert(_PyFrame_GetBytecode(shim)[1].op.code == RETURN_VALUE); + tstate, (PyCodeObject *)&_Py_InitCleanup, 1, FRAME_CORE); + assert(_PyFrame_GetBytecodeFull(shim)[0].op.code == EXIT_INIT_CHECK); + assert(_PyFrame_GetBytecodeFull(shim)[1].op.code == RETURN_VALUE); /* Push self onto stack of shim */ shim->localsplus[0] = PyStackRef_DUP(self); _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( - tstate, init, NULL, args-1, oparg+1, NULL, shim); + tstate, init, NULL, args-1, oparg+1, NULL, _PyFrame_Core(shim)); DEAD(init); DEAD(self); DEAD(args); @@ -4639,7 +4640,7 @@ dummy_func( PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( tstate, callable, locals, - arguments, positional_args, kwnames_o, frame + arguments, positional_args, kwnames_o, FRAME_CORE ); DEAD(args); DEAD(self_or_null); @@ -4690,7 +4691,7 @@ dummy_func( PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( tstate, callable, locals, - arguments, positional_args, kwnames_o, frame + arguments, positional_args, kwnames_o, FRAME_CORE ); PyStackRef_CLOSE(kwnames); // The frame has stolen all the arguments from the stack, @@ -4888,7 +4889,7 @@ dummy_func( _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( tstate, func_st, locals, - nargs, callargs, kwargs, frame); + nargs, callargs, kwargs, FRAME_CORE); // Need to sync the stack since we exit with DISPATCH_INLINED. INPUTS_DEAD(); SYNC_SP(); @@ -4951,7 +4952,7 @@ dummy_func( _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( tstate, func_st, locals, - nargs, callargs, kwargs, frame); + nargs, callargs, kwargs, FRAME_CORE); INPUTS_DEAD(); SYNC_SP(); if (new_frame == NULL) { @@ -5042,11 +5043,11 @@ dummy_func( _PyFrame_Copy(frame, gen_frame); assert(frame->frame_obj == NULL); gen->gi_frame_state = FRAME_CREATED; - gen_frame->owner = FRAME_OWNED_BY_GENERATOR; + _PyFrame_Core(gen_frame)->owner = FRAME_OWNED_BY_GENERATOR; _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *prev = frame->previous; - _PyThreadState_PopFrame(tstate, frame); - frame = tstate->current_frame = prev; + _PyInterpreterFrameCore *prev = FRAME_CORE->previous; + _PyThreadState_PopFrame(tstate, FRAME_CORE); + frame = (_PyInterpreterFrame *)(tstate->current_frame = prev); LOAD_IP(frame->return_offset); RELOAD_STACK(); res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); @@ -5139,8 +5140,8 @@ dummy_func( inst(INSTRUMENTED_LINE, ( -- )) { int original_opcode = 0; if (tstate->tracing) { - PyCodeObject *code = _PyFrame_GetCode(frame); - int index = (int)(this_instr - _PyFrame_GetBytecode(frame)); + PyCodeObject *code = _PyFrame_GetCodeFull(frame); + int index = (int)(this_instr - _PyFrame_GetBytecodeFull(frame)); original_opcode = code->_co_monitoring->lines->data[index*code->_co_monitoring->lines->bytes_per_entry]; next_instr = this_instr; } else { @@ -5336,8 +5337,8 @@ dummy_func( tier2 op(_EXIT_TRACE, (exit_p/4 --)) { _PyExitData *exit = (_PyExitData *)exit_p; #if defined(Py_DEBUG) && !defined(_Py_JIT) - const _Py_CODEUNIT *target = ((frame->owner == FRAME_OWNED_BY_INTERPRETER) - ? _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR : _PyFrame_GetBytecode(frame)) + const _Py_CODEUNIT *target = ((FRAME_CORE->owner == FRAME_OWNED_BY_INTERPRETER) + ? _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR : _PyFrame_GetBytecodeFull(frame)) + exit->target; OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); if (frame->lltrace >= 3) { @@ -5345,7 +5346,7 @@ dummy_func( _PyUOpPrint(&next_uop[-1]); printf(", exit %tu, temp %d, target %d -> %s, is_control_flow %d]\n", exit - current_executor->exits, exit->temperature.value_and_backoff, - (int)(target - _PyFrame_GetBytecode(frame)), + (int)(target - _PyFrame_GetBytecodeFull(frame)), _PyOpcode_OpName[target->op.code], exit->is_control_flow); } #endif @@ -5363,7 +5364,7 @@ dummy_func( _PyUOpPrint(&next_uop[-1]); printf(", exit %tu, temp %d, target %d -> %s]\n", exit - current_executor->exits, exit->temperature.value_and_backoff, - (int)(target - _PyFrame_GetBytecode(frame)), + (int)(target - _PyFrame_GetBytecodeFull(frame)), _PyOpcode_OpName[target->op.code]); } #endif @@ -5508,21 +5509,21 @@ dummy_func( tier2 op(_DEOPT, (--)) { SYNC_SP(); - GOTO_TIER_ONE((frame->owner == FRAME_OWNED_BY_INTERPRETER) - ? _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR : _PyFrame_GetBytecode(frame) + CURRENT_TARGET()); + GOTO_TIER_ONE((FRAME_CORE->owner == FRAME_OWNED_BY_INTERPRETER) + ? _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR : _PyFrame_GetBytecodeFull(frame) + CURRENT_TARGET()); Py_UNREACHABLE(); } tier2 op(_HANDLE_PENDING_AND_DEOPT, (--)) { SYNC_SP(); int err = _Py_HandlePending(tstate); - GOTO_TIER_ONE(err ? NULL : _PyFrame_GetBytecode(frame) + CURRENT_TARGET()); + GOTO_TIER_ONE(err ? NULL : _PyFrame_GetBytecodeFull(frame) + CURRENT_TARGET()); Py_UNREACHABLE(); } tier2 op(_ERROR_POP_N, (target/2 --)) { assert(oparg == 0); - frame->instr_ptr = _PyFrame_GetBytecode(frame) + target; + frame->instr_ptr = _PyFrame_GetBytecodeFull(frame) + target; SYNC_SP(); GOTO_TIER_ONE(NULL); Py_UNREACHABLE(); @@ -5540,18 +5541,18 @@ dummy_func( #endif uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); HANDLE_PENDING_AND_DEOPT_IF(eval_breaker & _PY_EVAL_EVENTS_MASK); - assert(tstate->tracing || eval_breaker == FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version)); + assert(tstate->tracing || eval_breaker == FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCodeFull(frame)->_co_instrumentation_version)); } tier2 op(_COLD_EXIT, ( -- )) { _PyExitData *exit = tstate->jit_exit; assert(exit != NULL); - assert(frame->owner < FRAME_OWNED_BY_INTERPRETER); - _Py_CODEUNIT *target = _PyFrame_GetBytecode(frame) + exit->target; + assert(FRAME_CORE->owner < FRAME_OWNED_BY_INTERPRETER); + _Py_CODEUNIT *target = _PyFrame_GetBytecodeFull(frame) + exit->target; _Py_BackoffCounter temperature = exit->temperature; _PyExecutorObject *executor; if (target->op.code == ENTER_EXECUTOR) { - PyCodeObject *code = _PyFrame_GetCode(frame); + PyCodeObject *code = _PyFrame_GetCodeFull(frame); executor = code->co_executors->executors[target->op.arg]; Py_INCREF(executor); assert(tstate->jit_exit == exit); @@ -5649,9 +5650,9 @@ dummy_func( RELOAD_STACK(); /* Log traceback info. */ - assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); + assert(FRAME_CORE->owner != FRAME_OWNED_BY_INTERPRETER); + if (!_PyFrame_IsIncomplete(FRAME_CORE)) { + PyFrameObject *f = _PyFrame_GetFrameObjectFull(frame); if (f != NULL) { PyTraceBack_Here(f); } @@ -5667,7 +5668,7 @@ dummy_func( /* We can't use frame->instr_ptr here, as RERAISE may have set it */ int offset = INSTR_OFFSET()-1; int level, handler, lasti; - int handled = get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti); + int handled = get_exception_handler(_PyFrame_GetCodeFull(frame), offset, &level, &handler, &lasti); if (handled == 0) { // No handlers, so exit. assert(_PyErr_Occurred(tstate)); @@ -5688,7 +5689,7 @@ dummy_func( PyStackRef_XCLOSE(ref); } if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); + int frame_lasti = _PyInterpreterFrame_LASTI_FULL(frame); _PyStackRef lasti = PyStackRef_TagInt(frame_lasti); _PyFrame_StackPush(frame, lasti); } @@ -5699,7 +5700,7 @@ dummy_func( Python main loop. */ PyObject *exc = _PyErr_GetRaisedException(tstate); _PyFrame_StackPush(frame, PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; + next_instr = _PyFrame_GetBytecodeFull(frame) + handler; int err = monitor_handled(tstate, frame, next_instr, exc); if (err < 0) { @@ -5721,15 +5722,15 @@ dummy_func( spilled label(exit_unwind) { assert(_PyErr_Occurred(tstate)); _Py_LeaveRecursiveCallPy(tstate); - assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + assert(FRAME_CORE->owner != FRAME_OWNED_BY_INTERPRETER); // GH-99729: We need to unlink the frame *before* clearing it: _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; + frame = (_PyInterpreterFrame *)(tstate->current_frame = _PyFrame_Core(dying)->previous); _PyEval_FrameClearAndPop(tstate, dying); frame->return_offset = 0; - if (frame->owner == FRAME_OWNED_BY_INTERPRETER) { + if (FRAME_CORE->owner == FRAME_OWNED_BY_INTERPRETER) { /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; + tstate->current_frame = FRAME_CORE->previous; #if !_Py_TAIL_CALL_INTERP assert(frame == &entry.frame); #endif @@ -5801,14 +5802,14 @@ dummy_func( _PyJitTracerState *tracer = _tstate->jit_tracer_state; assert(tracer != NULL); tracer->prev_state.instr = next_instr; - PyObject *prev_code = PyStackRef_AsPyObjectBorrow(frame->f_executable); + PyObject *prev_code = PyStackRef_AsPyObjectBorrow(FRAME_CORE->f_executable); if (tracer->prev_state.instr_code != (PyCodeObject *)prev_code) { Py_SETREF(tracer->prev_state.instr_code, (PyCodeObject*)Py_NewRef((prev_code))); } tracer->prev_state.instr_frame = frame; tracer->prev_state.instr_oparg = oparg; - tracer->prev_state.instr_stacklevel = PyStackRef_IsNone(frame->f_executable) ? 2 : STACK_LEVEL(); + tracer->prev_state.instr_stacklevel = PyStackRef_IsNone(FRAME_CORE->f_executable) ? 2 : STACK_LEVEL(); if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { (&next_instr[1])->counter = trigger_backoff_counter(); } diff --git a/Python/ceval.c b/Python/ceval.c index bdf1e9bb742333..8fbaac8ddeff18 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1013,7 +1013,7 @@ _Py_assert_within_stack_bounds( _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, const char *filename, int lineno ) { - if (frame->owner == FRAME_OWNED_BY_INTERPRETER) { + if (_PyFrame_Core(frame)->owner == FRAME_OWNED_BY_INTERPRETER) { return; } int level = (int)(stack_pointer - _PyFrame_Stackbase(frame)); @@ -1022,7 +1022,7 @@ _Py_assert_within_stack_bounds( fflush(stdout); abort(); } - int size = _PyFrame_GetCode(frame)->co_stacksize; + int size = _PyFrame_GetCodeFull(frame)->co_stacksize; if (level > size) { printf("Stack overflow (depth = %d) at %s:%d\n", level, filename, lineno); fflush(stdout); @@ -1166,7 +1166,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #if !_Py_TAIL_CALL_INTERP uint8_t opcode; /* Current opcode */ int oparg; /* Current opcode argument, if any */ - assert(tstate->current_frame == NULL || tstate->current_frame->stackpointer != NULL); + assert(_PyFrame_StackpointerSaved()); #if !USE_COMPUTED_GOTOS uint8_t tracing_mode = 0; uint8_t dispatch_code; @@ -1175,7 +1175,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int _PyEntryFrame entry; if (_Py_EnterRecursiveCallTstate(tstate, "")) { - assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + assert(_PyFrame_Core(frame)->owner != FRAME_OWNED_BY_INTERPRETER); _PyEval_FrameClearAndPop(tstate, frame); return NULL; } @@ -1195,19 +1195,19 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int entry.frame.f_globals = (PyObject*)0xaaa3; entry.frame.f_builtins = (PyObject*)0xaaa4; #endif - entry.frame.f_executable = PyStackRef_None; + _PyFrame_Core(&entry.frame)->f_executable = PyStackRef_None; entry.frame.instr_ptr = (_Py_CODEUNIT *)_Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS + 1; entry.frame.stackpointer = entry.stack; - entry.frame.owner = FRAME_OWNED_BY_INTERPRETER; + _PyFrame_Core(&entry.frame)->owner = FRAME_OWNED_BY_INTERPRETER; entry.frame.visited = 0; entry.frame.return_offset = 0; #ifdef Py_DEBUG entry.frame.lltrace = 0; #endif /* Push frame */ - entry.frame.previous = tstate->current_frame; - frame->previous = &entry.frame; - tstate->current_frame = frame; + _PyFrame_Core(&entry.frame)->previous = tstate->current_frame; + _PyFrame_Core(frame)->previous = _PyFrame_Core(&entry.frame); + tstate->current_frame = _PyFrame_Core(frame); entry.frame.localsplus[0] = PyStackRef_NULL; #ifdef _Py_TIER2 if (tstate->current_executor != NULL) { @@ -1225,17 +1225,17 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int /* Load thread-local bytecode */ if (frame->tlbc_index != ((_PyThreadStateImpl *)tstate)->tlbc_index) { _Py_CODEUNIT *bytecode = - _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); + _PyEval_GetExecutableCode(tstate, _PyFrame_GetCodeFull(frame)); if (bytecode == NULL) { goto early_exit; } - ptrdiff_t off = frame->instr_ptr - _PyFrame_GetBytecode(frame); + ptrdiff_t off = frame->instr_ptr - _PyFrame_GetBytecodeFull(frame); frame->tlbc_index = ((_PyThreadStateImpl *)tstate)->tlbc_index; frame->instr_ptr = bytecode + off; } #endif /* Because this avoids the RESUME, we need to update instrumentation */ - _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); + _Py_Instrument(_PyFrame_GetCodeFull(frame), tstate->interp); next_instr = frame->instr_ptr; monitor_throw(tstate, frame, next_instr); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -1265,15 +1265,15 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int early_exit: assert(_PyErr_Occurred(tstate)); _Py_LeaveRecursiveCallPy(tstate); - assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + assert(_PyFrame_Core(frame)->owner != FRAME_OWNED_BY_INTERPRETER); // GH-99729: We need to unlink the frame *before* clearing it: _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; + frame = (_PyInterpreterFrame *)(tstate->current_frame = _PyFrame_Core(dying)->previous); _PyEval_FrameClearAndPop(tstate, dying); frame->return_offset = 0; - assert(frame->owner == FRAME_OWNED_BY_INTERPRETER); + assert(_PyFrame_Core(frame)->owner == FRAME_OWNED_BY_INTERPRETER); /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; + tstate->current_frame = _PyFrame_Core(frame)->previous; return NULL; } #ifdef _Py_TIER2 @@ -1897,28 +1897,28 @@ initialize_locals(PyThreadState *tstate, PyFunctionObject *func, static void clear_thread_frame(PyThreadState *tstate, _PyInterpreterFrame * frame) { - assert(frame->owner == FRAME_OWNED_BY_THREAD); + assert(_PyFrame_Core(frame)->owner == FRAME_OWNED_BY_THREAD); // Make sure that this is, indeed, the top frame. We can't check this in // _PyThreadState_PopFrame, since f_code is already cleared at that point: - assert((PyObject **)frame + _PyFrame_GetCode(frame)->co_framesize == + assert((PyObject **)frame + _PyFrame_GetCodeFull(frame)->co_framesize == tstate->datastack_top); assert(frame->frame_obj == NULL || frame->frame_obj->f_frame == frame); _PyFrame_ClearExceptCode(frame); - PyStackRef_CLEAR(frame->f_executable); - _PyThreadState_PopFrame(tstate, frame); + PyStackRef_CLEAR(_PyFrame_Core(frame)->f_executable); + _PyThreadState_PopFrame(tstate, _PyFrame_Core(frame)); } static void clear_gen_frame(PyThreadState *tstate, _PyInterpreterFrame * frame) { - assert(frame->owner == FRAME_OWNED_BY_GENERATOR); + assert(_PyFrame_Core(frame)->owner & FRAME_OWNED_BY_GENERATOR); PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, FRAME_CLEARED); assert(tstate->exc_info == &gen->gi_exc_state); tstate->exc_info = gen->gi_exc_state.previous_item; gen->gi_exc_state.previous_item = NULL; assert(frame->frame_obj == NULL || frame->frame_obj->f_frame == frame); - frame->previous = NULL; + _PyFrame_Core(frame)->previous = NULL; _PyFrame_ClearExceptCode(frame); _PyErr_ClearExcState(&gen->gi_exc_state); // gh-143939: There must not be any escaping calls between setting @@ -1927,7 +1927,7 @@ clear_gen_frame(PyThreadState *tstate, _PyInterpreterFrame * frame) } void -_PyEval_FrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame * frame) +_PyEval_FrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame *frame) { // Update last_profiled_frame for remote profiler frame caching. // By this point, tstate->current_frame is already set to the parent frame. @@ -1935,11 +1935,11 @@ _PyEval_FrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame * frame) // This avoids corrupting the cache when transient frames (called and returned // between profiler samples) update last_profiled_frame to addresses the // profiler never saw. - if (tstate->last_profiled_frame != NULL && tstate->last_profiled_frame == frame) { + if (tstate->last_profiled_frame != NULL && tstate->last_profiled_frame == _PyFrame_Core(frame)) { tstate->last_profiled_frame = tstate->current_frame; } - if (frame->owner == FRAME_OWNED_BY_THREAD) { + if (_PyFrame_Core(frame)->owner == FRAME_OWNED_BY_THREAD) { clear_thread_frame(tstate, frame); } else { @@ -1951,7 +1951,7 @@ _PyEval_FrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame * frame) _PyInterpreterFrame * _PyEvalFramePushAndInit(PyThreadState *tstate, _PyStackRef func, PyObject *locals, _PyStackRef const* args, - size_t argcount, PyObject *kwnames, _PyInterpreterFrame *previous) + size_t argcount, PyObject *kwnames, _PyInterpreterFrameCore *previous) { PyFunctionObject *func_obj = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(func); PyCodeObject * code = (PyCodeObject *)func_obj->func_code; @@ -1962,7 +1962,7 @@ _PyEvalFramePushAndInit(PyThreadState *tstate, _PyStackRef func, } _PyFrame_Initialize(tstate, frame, func, locals, code, 0, previous); if (initialize_locals(tstate, func_obj, frame->localsplus, args, argcount, kwnames)) { - assert(frame->owner == FRAME_OWNED_BY_THREAD); + assert(_PyFrame_Core(frame)->owner == FRAME_OWNED_BY_THREAD); clear_thread_frame(tstate, frame); return NULL; } @@ -1989,7 +1989,7 @@ _PyEvalFramePushAndInit(PyThreadState *tstate, _PyStackRef func, */ _PyInterpreterFrame * _PyEvalFramePushAndInit_Ex(PyThreadState *tstate, _PyStackRef func, - PyObject *locals, Py_ssize_t nargs, PyObject *callargs, PyObject *kwargs, _PyInterpreterFrame *previous) + PyObject *locals, Py_ssize_t nargs, PyObject *callargs, PyObject *kwargs, _PyInterpreterFrameCore *previous) { bool has_dict = (kwargs != NULL && PyDict_GET_SIZE(kwargs) > 0); PyObject *kwnames = NULL; @@ -2200,7 +2200,7 @@ _PyEval_ExceptionGroupMatch(_PyInterpreterFrame *frame, PyObject* exc_value, if (wrapped == NULL) { return -1; } - PyFrameObject *f = _PyFrame_GetFrameObject(frame); + PyFrameObject *f = _PyFrame_GetFrameObject(_PyFrame_Core(frame)); if (f != NULL) { PyObject *tb = _PyTraceBack_FromFrame(NULL, f); if (tb == NULL) { @@ -2366,7 +2366,6 @@ _PyEval_UnpackIterableStackRef(PyThreadState *tstate, PyObject *v, } - void _PyEval_MonitorRaise(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr) @@ -2514,7 +2513,7 @@ _PyEval_GetAsyncGenFinalizer(void) return tstate->async_gen_finalizer; } -_PyInterpreterFrame * +_PyInterpreterFrameCore * _PyEval_GetFrame(void) { PyThreadState *tstate = _PyThreadState_GET(); @@ -2524,7 +2523,7 @@ _PyEval_GetFrame(void) PyFrameObject * PyEval_GetFrame(void) { - _PyInterpreterFrame *frame = _PyEval_GetFrame(); + _PyInterpreterFrameCore *frame = _PyEval_GetFrame(); if (frame == NULL) { return NULL; } @@ -2538,9 +2537,9 @@ PyEval_GetFrame(void) PyObject * _PyEval_GetBuiltins(PyThreadState *tstate) { - _PyInterpreterFrame *frame = _PyThreadState_GetFrame(tstate); + _PyInterpreterFrameCore *frame = _PyThreadState_GetFrame(tstate); if (frame != NULL) { - return frame->f_builtins; + return _PyFrame_GetBuiltins(frame); } return tstate->interp->builtins; } @@ -2568,7 +2567,7 @@ PyEval_GetLocals(void) { // We need to return a borrowed reference here, so some tricks are needed PyThreadState *tstate = _PyThreadState_GET(); - _PyInterpreterFrame *current_frame = _PyThreadState_GetFrame(tstate); + _PyInterpreterFrameCore *current_frame = _PyThreadState_GetFrame(tstate); if (current_frame == NULL) { _PyErr_SetString(tstate, PyExc_SystemError, "frame does not exist"); return NULL; @@ -2612,7 +2611,7 @@ PyObject * _PyEval_GetFrameLocals(void) { PyThreadState *tstate = _PyThreadState_GET(); - _PyInterpreterFrame *current_frame = _PyThreadState_GetFrame(tstate); + _PyInterpreterFrameCore *current_frame = _PyThreadState_GetFrame(tstate); if (current_frame == NULL) { _PyErr_SetString(tstate, PyExc_SystemError, "frame does not exist"); return NULL; @@ -2645,11 +2644,11 @@ _PyEval_GetFrameLocals(void) static PyObject * _PyEval_GetGlobals(PyThreadState *tstate) { - _PyInterpreterFrame *current_frame = _PyThreadState_GetFrame(tstate); + _PyInterpreterFrameCore *current_frame = _PyThreadState_GetFrame(tstate); if (current_frame == NULL) { return NULL; } - return current_frame->f_globals; + return _PyFrame_GetGlobals(current_frame); } PyObject * @@ -2775,11 +2774,11 @@ PyEval_GetFrameLocals(void) PyObject* PyEval_GetFrameGlobals(void) { PyThreadState *tstate = _PyThreadState_GET(); - _PyInterpreterFrame *current_frame = _PyThreadState_GetFrame(tstate); + _PyInterpreterFrameCore *current_frame = _PyThreadState_GetFrame(tstate); if (current_frame == NULL) { return NULL; } - return Py_XNewRef(current_frame->f_globals); + return Py_XNewRef(_PyFrame_GetGlobals(current_frame)); } PyObject* PyEval_GetFrameBuiltins(void) @@ -2792,7 +2791,7 @@ int PyEval_MergeCompilerFlags(PyCompilerFlags *cf) { PyThreadState *tstate = _PyThreadState_GET(); - _PyInterpreterFrame *current_frame = tstate->current_frame; + _PyInterpreterFrameCore *current_frame = tstate->current_frame; if (current_frame == tstate->base_frame) { current_frame = NULL; } @@ -2885,11 +2884,11 @@ _PyEval_SliceIndexNotNone(PyObject *v, Py_ssize_t *pi) } PyObject * -_PyEval_ImportName(PyThreadState *tstate, _PyInterpreterFrame *frame, +_PyEval_ImportName(PyThreadState *tstate, _PyInterpreterFrameCore *frame, PyObject *name, PyObject *fromlist, PyObject *level) { PyObject *import_func; - if (PyMapping_GetOptionalItem(frame->f_builtins, &_Py_ID(__import__), &import_func) < 0) { + if (PyMapping_GetOptionalItem(_PyFrame_GetBuiltins(frame), &_Py_ID(__import__), &import_func) < 0) { return NULL; } if (import_func == NULL) { @@ -2897,7 +2896,9 @@ _PyEval_ImportName(PyThreadState *tstate, _PyInterpreterFrame *frame, return NULL; } - PyObject *locals = frame->f_locals; + _PyInterpreterFrame *f = _PyFrame_EnsureFrameFullyInitialized(frame); + + PyObject *locals = f->f_locals; if (locals == NULL) { locals = Py_None; } @@ -2911,13 +2912,13 @@ _PyEval_ImportName(PyThreadState *tstate, _PyInterpreterFrame *frame, } return PyImport_ImportModuleLevelObject( name, - frame->f_globals, + f->f_globals, locals, fromlist, ilevel); } - PyObject* args[5] = {name, frame->f_globals, locals, fromlist, level}; + PyObject* args[5] = {name, f->f_globals, locals, fromlist, level}; PyObject *res = PyObject_Vectorcall(import_func, args, 5, NULL); Py_DECREF(import_func); return res; diff --git a/Python/ceval.h b/Python/ceval.h index b170643e236733..4fe98bf152237f 100644 --- a/Python/ceval.h +++ b/Python/ceval.h @@ -228,9 +228,9 @@ lltrace_instruction(_PyInterpreterFrame *frame, int oparg) { int offset = 0; - if (frame->owner < FRAME_OWNED_BY_INTERPRETER) { + if (_PyFrame_Core(frame)->owner < FRAME_OWNED_BY_INTERPRETER) { dump_stack(frame, stack_pointer); - offset = (int)(next_instr - _PyFrame_GetBytecode(frame)); + offset = (int)(next_instr - _PyFrame_GetBytecode(_PyFrame_Core(frame))); } const char *opname = _PyOpcode_OpName[opcode]; assert(opname != NULL); @@ -247,7 +247,7 @@ static void lltrace_resume_frame(_PyInterpreterFrame *frame) { PyObject *fobj = PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - if (!PyStackRef_CodeCheck(frame->f_executable) || + if (!PyStackRef_CodeCheck(_PyFrame_Core(frame)->f_executable) || fobj == NULL || !PyFunction_Check(fobj) ) { @@ -284,7 +284,7 @@ maybe_lltrace_resume_frame(_PyInterpreterFrame *frame, PyObject *globals) if (globals == NULL) { return 0; } - if (frame->owner >= FRAME_OWNED_BY_INTERPRETER) { + if (_PyFrame_Core(frame)->owner >= FRAME_OWNED_BY_INTERPRETER) { return 0; } int r = PyDict_Contains(globals, &_Py_ID(__lltrace__)); @@ -341,7 +341,7 @@ do_monitor_exc(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, int event) { assert(event < _PY_MONITORING_UNGROUPED_EVENTS); - if (_PyFrame_GetCode(frame)->co_flags & CO_NO_MONITORING_EVENTS) { + if (_PyFrame_GetCode(_PyFrame_Core(frame))->co_flags & CO_NO_MONITORING_EVENTS) { return 0; } PyObject *exc = PyErr_GetRaisedException(); @@ -367,7 +367,7 @@ static inline bool no_tools_for_local_event(PyThreadState *tstate, _PyInterpreterFrame *frame, int event) { assert(event < _PY_MONITORING_LOCAL_EVENTS); - _PyCoMonitoringData *data = _PyFrame_GetCode(frame)->_co_monitoring; + _PyCoMonitoringData *data = _PyFrame_GetCode(_PyFrame_Core(frame))->_co_monitoring; if (data) { return data->active_monitors.tools[event] == 0; } diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index d791ba0e8eca97..b0bead5ad3d830 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -220,14 +220,15 @@ do { \ DISPATCH_GOTO_NON_TRACING(); \ } -#define DISPATCH_INLINED(NEW_FRAME) \ - do { \ - assert(tstate->interp->eval_frame == NULL); \ - _PyFrame_SetStackPointer(frame, stack_pointer); \ - assert((NEW_FRAME)->previous == frame); \ - frame = tstate->current_frame = (NEW_FRAME); \ - CALL_STAT_INC(inlined_py_calls); \ - JUMP_TO_LABEL(start_frame); \ +#define DISPATCH_INLINED(NEW_FRAME) \ + do { \ + assert(tstate->interp->eval_frame == NULL); \ + _PyFrame_SetStackPointer(frame, stack_pointer); \ + assert(_PyFrame_Core((NEW_FRAME))->previous == FRAME_CORE); \ + tstate->current_frame = _PyFrame_Core((NEW_FRAME)); \ + frame = (NEW_FRAME); \ + CALL_STAT_INC(inlined_py_calls); \ + JUMP_TO_LABEL(start_frame); \ } while (0) /* Tuple access macros */ @@ -244,10 +245,16 @@ GETITEM(PyObject *v, Py_ssize_t i) { } #endif +/* Data access macros */ +#define FRAME_CORE (&frame->core) +#define FRAME_CODE (_PyFrame_GetCodeFull(frame)) +#define FRAME_CO_CONSTS (_PyFrame_GetCodeFull(frame)->co_consts) +#define FRAME_CO_NAMES (_PyFrame_GetCodeFull(frame)->co_names) + /* Code access macros */ /* The integer overflow is checked by an assertion below. */ -#define INSTR_OFFSET() ((int)(next_instr - _PyFrame_GetBytecode(frame))) +#define INSTR_OFFSET() ((int)(next_instr - _PyFrame_GetBytecodeFull(frame))) #define NEXTOPARG() do { \ _Py_CODEUNIT word = {.cache = FT_ATOMIC_LOAD_UINT16_RELAXED(*(uint16_t*)next_instr)}; \ opcode = word.op.code; \ @@ -262,23 +269,19 @@ GETITEM(PyObject *v, Py_ssize_t i) { #define SKIP_OVER(x) (next_instr += (x)) #define STACK_LEVEL() ((int)(stack_pointer - _PyFrame_Stackbase(frame))) -#define STACK_SIZE() (_PyFrame_GetCode(frame)->co_stacksize) +#define STACK_SIZE() (_PyFrame_GetCodeFull(frame)->co_stacksize) #define WITHIN_STACK_BOUNDS() \ - (frame->owner == FRAME_OWNED_BY_INTERPRETER || (STACK_LEVEL() >= 0 && STACK_LEVEL() <= STACK_SIZE())) + (FRAME_CORE->owner == FRAME_OWNED_BY_INTERPRETER || (STACK_LEVEL() >= 0 && STACK_LEVEL() <= STACK_SIZE())) #if defined(Py_DEBUG) && !defined(_Py_JIT) // This allows temporary stack "overflows", provided it's all in the cache at any point of time. #define WITHIN_STACK_BOUNDS_IGNORING_CACHE() \ - (frame->owner == FRAME_OWNED_BY_INTERPRETER || (STACK_LEVEL() >= 0 && (STACK_LEVEL()) <= STACK_SIZE())) + (_PyFrame_Core(frame)->owner == FRAME_OWNED_BY_INTERPRETER || (STACK_LEVEL() >= 0 && (STACK_LEVEL()) <= STACK_SIZE())) #else #define WITHIN_STACK_BOUNDS_IGNORING_CACHE WITHIN_STACK_BOUNDS #endif -/* Data access macros */ -#define FRAME_CO_CONSTS (_PyFrame_GetCode(frame)->co_consts) -#define FRAME_CO_NAMES (_PyFrame_GetCode(frame)->co_names) - /* Local variable macros */ #define LOCALS_ARRAY (frame->localsplus) @@ -326,8 +329,6 @@ GETITEM(PyObject *v, Py_ssize_t i) { #define GLOBALS() frame->f_globals #define BUILTINS() frame->f_builtins #define LOCALS() frame->f_locals -#define CONSTS() _PyFrame_GetCode(frame)->co_consts -#define NAMES() _PyFrame_GetCode(frame)->co_names #define DTRACE_FUNCTION_ENTRY() \ if (PyDTrace_FUNCTION_ENTRY_ENABLED()) { \ @@ -422,7 +423,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer) do { \ OPT_STAT_INC(traces_executed); \ next_instr = _Py_jit_entry((EXECUTOR), frame, stack_pointer, tstate); \ - frame = tstate->current_frame; \ + frame = (_PyInterpreterFrame *)(tstate->current_frame); \ stack_pointer = _PyFrame_GetStackPointer(frame); \ int keep_tracing_bit = (uintptr_t)next_instr & 1; \ next_instr = (_Py_CODEUNIT *)(((uintptr_t)next_instr) & (~1)); \ diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 4d4084971c75ad..126ffbe3b40c06 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -109,7 +109,7 @@ _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; #endif uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); - uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); + uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCodeFull(frame)->_co_instrumentation_version); assert((version & _PY_EVAL_EVENTS_MASK) == 0); if (eval_breaker != version) { UOP_STAT_INC(uopcode, miss); @@ -143,7 +143,7 @@ _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; #endif uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); - uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); + uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCodeFull(frame)->_co_instrumentation_version); assert((version & _PY_EVAL_EVENTS_MASK) == 0); if (eval_breaker != version) { UOP_STAT_INC(uopcode, miss); @@ -182,7 +182,7 @@ _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; #endif uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); - uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); + uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCodeFull(frame)->_co_instrumentation_version); assert((version & _PY_EVAL_EVENTS_MASK) == 0); if (eval_breaker != version) { UOP_STAT_INC(uopcode, miss); @@ -226,7 +226,7 @@ _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; #endif uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); - uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); + uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCodeFull(frame)->_co_instrumentation_version); assert((version & _PY_EVAL_EVENTS_MASK) == 0); if (eval_breaker != version) { UOP_STAT_INC(uopcode, miss); @@ -267,7 +267,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, UNBOUNDLOCAL_ERROR_MSG, - PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) + PyTuple_GetItem(_PyFrame_GetCodeFull(frame)->co_localsplusnames, oparg) ); stack_pointer = _PyFrame_GetStackPointer(frame); SET_CURRENT_CACHED_VALUES(0); @@ -294,7 +294,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, UNBOUNDLOCAL_ERROR_MSG, - PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) + PyTuple_GetItem(_PyFrame_GetCodeFull(frame)->co_localsplusnames, oparg) ); stack_pointer = _PyFrame_GetStackPointer(frame); SET_CURRENT_CACHED_VALUES(0); @@ -324,7 +324,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, UNBOUNDLOCAL_ERROR_MSG, - PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) + PyTuple_GetItem(_PyFrame_GetCodeFull(frame)->co_localsplusnames, oparg) ); stack_pointer = _PyFrame_GetStackPointer(frame); SET_CURRENT_CACHED_VALUES(0); @@ -6631,13 +6631,13 @@ _PyStackRef res; _PyStackRef _stack_item_0 = _tos_cache0; retval = _stack_item_0; - assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + assert(FRAME_CORE->owner != FRAME_OWNED_BY_INTERPRETER); _PyStackRef temp = PyStackRef_MakeHeapSafe(retval); _PyFrame_SetStackPointer(frame, stack_pointer); assert(STACK_LEVEL() == 0); _Py_LeaveRecursiveCallPy(tstate); _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; + frame = (_PyInterpreterFrame *)(tstate->current_frame = _PyFrame_Core(dying)->previous); _PyEval_FrameClearAndPop(tstate, dying); stack_pointer = _PyFrame_GetStackPointer(frame); LOAD_IP(frame->return_offset); @@ -6814,7 +6814,7 @@ tstate->exc_info = &gen->gi_exc_state; assert( 2u + oparg <= UINT16_MAX); frame->return_offset = (uint16_t)( 2u + oparg); - pushed_frame->previous = frame; + _PyFrame_Core(pushed_frame)->previous = FRAME_CORE; gen_frame = PyStackRef_Wrap(pushed_frame); _tos_cache1 = gen_frame; _tos_cache0 = receiver; @@ -6832,7 +6832,7 @@ _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); retval = _stack_item_0; - assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + assert(FRAME_CORE->owner != FRAME_OWNED_BY_INTERPRETER); frame->instr_ptr++; PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); @@ -6843,8 +6843,8 @@ gen->gi_exc_state.previous_item = NULL; _Py_LeaveRecursiveCallPy(tstate); _PyInterpreterFrame *gen_frame = frame; - frame = tstate->current_frame = frame->previous; - gen_frame->previous = NULL; + frame = (_PyInterpreterFrame *)(tstate->current_frame = FRAME_CORE->previous); + _PyFrame_Core(gen_frame)->previous = NULL; ((_PyThreadStateImpl *)tstate)->generator_return_kind = GENERATOR_YIELD; FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, FRAME_SUSPENDED + oparg); assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); @@ -7715,7 +7715,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, UNBOUNDLOCAL_ERROR_MSG, - PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) + PyTuple_GetItem(_PyFrame_GetCodeFull(frame)->co_localsplusnames, oparg) ); stack_pointer = _PyFrame_GetStackPointer(frame); SET_CURRENT_CACHED_VALUES(0); @@ -7765,7 +7765,7 @@ PyObject *oldobj = PyCell_SwapTakeRef((PyCellObject *)cell, NULL); if (oldobj == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCodeFull(frame), oparg); stack_pointer = _PyFrame_GetStackPointer(frame); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -7792,8 +7792,8 @@ PyObject *name; PyObject *class_dict = PyStackRef_AsPyObjectBorrow(class_dict_st); assert(class_dict); - assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); - name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); + assert(oparg >= 0 && oparg < _PyFrame_GetCodeFull(frame)->co_nlocalsplus); + name = PyTuple_GET_ITEM(_PyFrame_GetCodeFull(frame)->co_localsplusnames, oparg); int err; stack_pointer[0] = class_dict_st; stack_pointer += 1; @@ -7810,7 +7810,7 @@ value_o = PyCell_GetRef(cell); if (value_o == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCodeFull(frame), oparg); stack_pointer = _PyFrame_GetStackPointer(frame); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -7844,7 +7844,7 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCodeFull(frame), oparg); stack_pointer = _PyFrame_GetStackPointer(frame); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -7885,7 +7885,7 @@ CHECK_CURRENT_CACHED_VALUES(0); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); oparg = CURRENT_OPARG(); - PyCodeObject *co = _PyFrame_GetCode(frame); + PyCodeObject *co = _PyFrame_GetCodeFull(frame); assert(PyStackRef_FunctionCheck(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); PyObject *closure = func->func_closure; @@ -7905,7 +7905,7 @@ assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); - PyCodeObject *co = _PyFrame_GetCode(frame); + PyCodeObject *co = _PyFrame_GetCodeFull(frame); assert(PyStackRef_FunctionCheck(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); PyObject *closure = func->func_closure; @@ -7927,7 +7927,7 @@ _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; oparg = CURRENT_OPARG(); - PyCodeObject *co = _PyFrame_GetCode(frame); + PyCodeObject *co = _PyFrame_GetCodeFull(frame); assert(PyStackRef_FunctionCheck(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); PyObject *closure = func->func_closure; @@ -7951,7 +7951,7 @@ _PyStackRef _stack_item_1 = _tos_cache1; _PyStackRef _stack_item_2 = _tos_cache2; oparg = CURRENT_OPARG(); - PyCodeObject *co = _PyFrame_GetCode(frame); + PyCodeObject *co = _PyFrame_GetCodeFull(frame); assert(PyStackRef_FunctionCheck(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); PyObject *closure = func->func_closure; @@ -10556,7 +10556,7 @@ stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyEval_ImportName(tstate, frame, name, + PyObject *res_o = _PyEval_ImportName(tstate, FRAME_CORE, name, PyStackRef_AsPyObjectBorrow(fromlist), PyStackRef_AsPyObjectBorrow(level)); _PyStackRef tmp = fromlist; @@ -10935,7 +10935,7 @@ iterable = _stack_item_0; PyObject *iterable_o = PyStackRef_AsPyObjectBorrow(iterable); if (PyCoro_CheckExact(iterable_o)) { - if (!(_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) { + if (!(_PyFrame_GetCodeFull(frame)->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) { stack_pointer[0] = iterable; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -11943,7 +11943,7 @@ _PyFrame_StackPush(pushed_frame, PyStackRef_None); gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; - pushed_frame->previous = frame; + _PyFrame_Core(pushed_frame)->previous = FRAME_CORE; frame->return_offset = (uint16_t)( 2u + oparg); gen_frame = PyStackRef_Wrap(pushed_frame); _tos_cache2 = gen_frame; @@ -11982,7 +11982,7 @@ _PyFrame_StackPush(pushed_frame, PyStackRef_None); gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; - pushed_frame->previous = frame; + _PyFrame_Core(pushed_frame)->previous = FRAME_CORE; frame->return_offset = (uint16_t)( 2u + oparg); gen_frame = PyStackRef_Wrap(pushed_frame); _tos_cache2 = gen_frame; @@ -12024,7 +12024,7 @@ _PyFrame_StackPush(pushed_frame, PyStackRef_None); gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; - pushed_frame->previous = frame; + _PyFrame_Core(pushed_frame)->previous = FRAME_CORE; frame->return_offset = (uint16_t)( 2u + oparg); gen_frame = PyStackRef_Wrap(pushed_frame); _tos_cache2 = gen_frame; @@ -12832,7 +12832,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( tstate, callable, locals, - args, total_args, NULL, frame + args, total_args, NULL, FRAME_CORE ); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; @@ -13503,9 +13503,10 @@ assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); _PyFrame_SetStackPointer(frame, stack_pointer); - assert(temp->previous == frame || temp->previous->previous == frame); + assert(_PyFrame_Core(temp)->previous == FRAME_CORE || _PyFrame_Core(temp)->previous->previous == FRAME_CORE); CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; + frame = temp; + tstate->current_frame = _PyFrame_Core(temp); tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); @@ -14303,14 +14304,14 @@ init = stack_pointer[-2 - oparg]; _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( - tstate, (PyCodeObject *)&_Py_InitCleanup, 1, frame); + tstate, (PyCodeObject *)&_Py_InitCleanup, 1, FRAME_CORE); stack_pointer = _PyFrame_GetStackPointer(frame); - assert(_PyFrame_GetBytecode(shim)[0].op.code == EXIT_INIT_CHECK); - assert(_PyFrame_GetBytecode(shim)[1].op.code == RETURN_VALUE); + assert(_PyFrame_GetBytecodeFull(shim)[0].op.code == EXIT_INIT_CHECK); + assert(_PyFrame_GetBytecodeFull(shim)[1].op.code == RETURN_VALUE); shim->localsplus[0] = PyStackRef_DUP(self); _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( - tstate, init, NULL, args-1, oparg+1, NULL, shim); + tstate, init, NULL, args-1, oparg+1, NULL, _PyFrame_Core(shim)); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -15517,7 +15518,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( tstate, callable, locals, - arguments, positional_args, kwnames_o, frame + arguments, positional_args, kwnames_o, FRAME_CORE ); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; @@ -15918,7 +15919,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( tstate, func_st, locals, - nargs, callargs, kwargs, frame); + nargs, callargs, kwargs, FRAME_CORE); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -16248,11 +16249,11 @@ _PyFrame_Copy(frame, gen_frame); assert(frame->frame_obj == NULL); gen->gi_frame_state = FRAME_CREATED; - gen_frame->owner = FRAME_OWNED_BY_GENERATOR; + _PyFrame_Core(gen_frame)->owner = FRAME_OWNED_BY_GENERATOR; _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *prev = frame->previous; - _PyThreadState_PopFrame(tstate, frame); - frame = tstate->current_frame = prev; + _PyInterpreterFrameCore *prev = FRAME_CORE->previous; + _PyThreadState_PopFrame(tstate, FRAME_CORE); + frame = (_PyInterpreterFrame *)(tstate->current_frame = prev); LOAD_IP(frame->return_offset); stack_pointer = _PyFrame_GetStackPointer(frame); res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); @@ -18455,8 +18456,8 @@ PyObject *exit_p = (PyObject *)CURRENT_OPERAND0_64(); _PyExitData *exit = (_PyExitData *)exit_p; #if defined(Py_DEBUG) && !defined(_Py_JIT) - const _Py_CODEUNIT *target = ((frame->owner == FRAME_OWNED_BY_INTERPRETER) - ? _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR : _PyFrame_GetBytecode(frame)) + const _Py_CODEUNIT *target = ((FRAME_CORE->owner == FRAME_OWNED_BY_INTERPRETER) + ? _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR : _PyFrame_GetBytecodeFull(frame)) + exit->target; OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); if (frame->lltrace >= 3) { @@ -18465,7 +18466,7 @@ _PyUOpPrint(&next_uop[-1]); printf(", exit %tu, temp %d, target %d -> %s, is_control_flow %d]\n", exit - current_executor->exits, exit->temperature.value_and_backoff, - (int)(target - _PyFrame_GetBytecode(frame)), + (int)(target - _PyFrame_GetBytecodeFull(frame)), _PyOpcode_OpName[target->op.code], exit->is_control_flow); stack_pointer = _PyFrame_GetStackPointer(frame); } @@ -18482,8 +18483,8 @@ PyObject *exit_p = (PyObject *)CURRENT_OPERAND0_64(); _PyExitData *exit = (_PyExitData *)exit_p; #if defined(Py_DEBUG) && !defined(_Py_JIT) - const _Py_CODEUNIT *target = ((frame->owner == FRAME_OWNED_BY_INTERPRETER) - ? _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR : _PyFrame_GetBytecode(frame)) + const _Py_CODEUNIT *target = ((FRAME_CORE->owner == FRAME_OWNED_BY_INTERPRETER) + ? _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR : _PyFrame_GetBytecodeFull(frame)) + exit->target; OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); if (frame->lltrace >= 3) { @@ -18495,7 +18496,7 @@ _PyUOpPrint(&next_uop[-1]); printf(", exit %tu, temp %d, target %d -> %s, is_control_flow %d]\n", exit - current_executor->exits, exit->temperature.value_and_backoff, - (int)(target - _PyFrame_GetBytecode(frame)), + (int)(target - _PyFrame_GetBytecodeFull(frame)), _PyOpcode_OpName[target->op.code], exit->is_control_flow); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; @@ -18517,8 +18518,8 @@ PyObject *exit_p = (PyObject *)CURRENT_OPERAND0_64(); _PyExitData *exit = (_PyExitData *)exit_p; #if defined(Py_DEBUG) && !defined(_Py_JIT) - const _Py_CODEUNIT *target = ((frame->owner == FRAME_OWNED_BY_INTERPRETER) - ? _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR : _PyFrame_GetBytecode(frame)) + const _Py_CODEUNIT *target = ((FRAME_CORE->owner == FRAME_OWNED_BY_INTERPRETER) + ? _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR : _PyFrame_GetBytecodeFull(frame)) + exit->target; OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); if (frame->lltrace >= 3) { @@ -18531,7 +18532,7 @@ _PyUOpPrint(&next_uop[-1]); printf(", exit %tu, temp %d, target %d -> %s, is_control_flow %d]\n", exit - current_executor->exits, exit->temperature.value_and_backoff, - (int)(target - _PyFrame_GetBytecode(frame)), + (int)(target - _PyFrame_GetBytecodeFull(frame)), _PyOpcode_OpName[target->op.code], exit->is_control_flow); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; @@ -18555,8 +18556,8 @@ PyObject *exit_p = (PyObject *)CURRENT_OPERAND0_64(); _PyExitData *exit = (_PyExitData *)exit_p; #if defined(Py_DEBUG) && !defined(_Py_JIT) - const _Py_CODEUNIT *target = ((frame->owner == FRAME_OWNED_BY_INTERPRETER) - ? _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR : _PyFrame_GetBytecode(frame)) + const _Py_CODEUNIT *target = ((FRAME_CORE->owner == FRAME_OWNED_BY_INTERPRETER) + ? _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR : _PyFrame_GetBytecodeFull(frame)) + exit->target; OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); if (frame->lltrace >= 3) { @@ -18570,7 +18571,7 @@ _PyUOpPrint(&next_uop[-1]); printf(", exit %tu, temp %d, target %d -> %s, is_control_flow %d]\n", exit - current_executor->exits, exit->temperature.value_and_backoff, - (int)(target - _PyFrame_GetBytecode(frame)), + (int)(target - _PyFrame_GetBytecodeFull(frame)), _PyOpcode_OpName[target->op.code], exit->is_control_flow); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -3; @@ -18600,7 +18601,7 @@ _PyUOpPrint(&next_uop[-1]); printf(", exit %tu, temp %d, target %d -> %s]\n", exit - current_executor->exits, exit->temperature.value_and_backoff, - (int)(target - _PyFrame_GetBytecode(frame)), + (int)(target - _PyFrame_GetBytecodeFull(frame)), _PyOpcode_OpName[target->op.code]); stack_pointer = _PyFrame_GetStackPointer(frame); } @@ -18627,7 +18628,7 @@ _PyUOpPrint(&next_uop[-1]); printf(", exit %tu, temp %d, target %d -> %s]\n", exit - current_executor->exits, exit->temperature.value_and_backoff, - (int)(target - _PyFrame_GetBytecode(frame)), + (int)(target - _PyFrame_GetBytecodeFull(frame)), _PyOpcode_OpName[target->op.code]); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; @@ -18660,7 +18661,7 @@ _PyUOpPrint(&next_uop[-1]); printf(", exit %tu, temp %d, target %d -> %s]\n", exit - current_executor->exits, exit->temperature.value_and_backoff, - (int)(target - _PyFrame_GetBytecode(frame)), + (int)(target - _PyFrame_GetBytecodeFull(frame)), _PyOpcode_OpName[target->op.code]); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; @@ -18696,7 +18697,7 @@ _PyUOpPrint(&next_uop[-1]); printf(", exit %tu, temp %d, target %d -> %s]\n", exit - current_executor->exits, exit->temperature.value_and_backoff, - (int)(target - _PyFrame_GetBytecode(frame)), + (int)(target - _PyFrame_GetBytecodeFull(frame)), _PyOpcode_OpName[target->op.code]); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -3; @@ -19687,8 +19688,8 @@ CHECK_CURRENT_CACHED_VALUES(0); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); SET_CURRENT_CACHED_VALUES(0); - GOTO_TIER_ONE((frame->owner == FRAME_OWNED_BY_INTERPRETER) - ? _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR : _PyFrame_GetBytecode(frame) + CURRENT_TARGET()); + GOTO_TIER_ONE((FRAME_CORE->owner == FRAME_OWNED_BY_INTERPRETER) + ? _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR : _PyFrame_GetBytecodeFull(frame) + CURRENT_TARGET()); } case _DEOPT_r10: { @@ -19699,8 +19700,8 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); SET_CURRENT_CACHED_VALUES(0); - GOTO_TIER_ONE((frame->owner == FRAME_OWNED_BY_INTERPRETER) - ? _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR : _PyFrame_GetBytecode(frame) + CURRENT_TARGET()); + GOTO_TIER_ONE((FRAME_CORE->owner == FRAME_OWNED_BY_INTERPRETER) + ? _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR : _PyFrame_GetBytecodeFull(frame) + CURRENT_TARGET()); } case _DEOPT_r20: { @@ -19713,8 +19714,8 @@ stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); SET_CURRENT_CACHED_VALUES(0); - GOTO_TIER_ONE((frame->owner == FRAME_OWNED_BY_INTERPRETER) - ? _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR : _PyFrame_GetBytecode(frame) + CURRENT_TARGET()); + GOTO_TIER_ONE((FRAME_CORE->owner == FRAME_OWNED_BY_INTERPRETER) + ? _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR : _PyFrame_GetBytecodeFull(frame) + CURRENT_TARGET()); } case _DEOPT_r30: { @@ -19729,8 +19730,8 @@ stack_pointer += 3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); SET_CURRENT_CACHED_VALUES(0); - GOTO_TIER_ONE((frame->owner == FRAME_OWNED_BY_INTERPRETER) - ? _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR : _PyFrame_GetBytecode(frame) + CURRENT_TARGET()); + GOTO_TIER_ONE((FRAME_CORE->owner == FRAME_OWNED_BY_INTERPRETER) + ? _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR : _PyFrame_GetBytecodeFull(frame) + CURRENT_TARGET()); } case _HANDLE_PENDING_AND_DEOPT_r00: { @@ -19740,7 +19741,7 @@ int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); SET_CURRENT_CACHED_VALUES(0); - GOTO_TIER_ONE(err ? NULL : _PyFrame_GetBytecode(frame) + CURRENT_TARGET()); + GOTO_TIER_ONE(err ? NULL : _PyFrame_GetBytecodeFull(frame) + CURRENT_TARGET()); } case _HANDLE_PENDING_AND_DEOPT_r10: { @@ -19754,7 +19755,7 @@ int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); SET_CURRENT_CACHED_VALUES(0); - GOTO_TIER_ONE(err ? NULL : _PyFrame_GetBytecode(frame) + CURRENT_TARGET()); + GOTO_TIER_ONE(err ? NULL : _PyFrame_GetBytecodeFull(frame) + CURRENT_TARGET()); } case _HANDLE_PENDING_AND_DEOPT_r20: { @@ -19770,7 +19771,7 @@ int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); SET_CURRENT_CACHED_VALUES(0); - GOTO_TIER_ONE(err ? NULL : _PyFrame_GetBytecode(frame) + CURRENT_TARGET()); + GOTO_TIER_ONE(err ? NULL : _PyFrame_GetBytecodeFull(frame) + CURRENT_TARGET()); } case _HANDLE_PENDING_AND_DEOPT_r30: { @@ -19788,7 +19789,7 @@ int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); SET_CURRENT_CACHED_VALUES(0); - GOTO_TIER_ONE(err ? NULL : _PyFrame_GetBytecode(frame) + CURRENT_TARGET()); + GOTO_TIER_ONE(err ? NULL : _PyFrame_GetBytecodeFull(frame) + CURRENT_TARGET()); } case _ERROR_POP_N_r00: { @@ -19797,7 +19798,7 @@ oparg = CURRENT_OPARG(); uint32_t target = (uint32_t)CURRENT_OPERAND0_32(); assert(oparg == 0); - frame->instr_ptr = _PyFrame_GetBytecode(frame) + target; + frame->instr_ptr = _PyFrame_GetBytecodeFull(frame) + target; SET_CURRENT_CACHED_VALUES(0); GOTO_TIER_ONE(NULL); } @@ -19985,7 +19986,7 @@ SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } - assert(tstate->tracing || eval_breaker == FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version)); + assert(tstate->tracing || eval_breaker == FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCodeFull(frame)->_co_instrumentation_version)); SET_CURRENT_CACHED_VALUES(0); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); break; @@ -20011,7 +20012,7 @@ SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } - assert(tstate->tracing || eval_breaker == FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version)); + assert(tstate->tracing || eval_breaker == FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCodeFull(frame)->_co_instrumentation_version)); _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); @@ -20041,7 +20042,7 @@ SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } - assert(tstate->tracing || eval_breaker == FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version)); + assert(tstate->tracing || eval_breaker == FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCodeFull(frame)->_co_instrumentation_version)); _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); @@ -20075,7 +20076,7 @@ SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } - assert(tstate->tracing || eval_breaker == FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version)); + assert(tstate->tracing || eval_breaker == FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCodeFull(frame)->_co_instrumentation_version)); _tos_cache2 = _stack_item_2; _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; @@ -20089,12 +20090,12 @@ assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); _PyExitData *exit = tstate->jit_exit; assert(exit != NULL); - assert(frame->owner < FRAME_OWNED_BY_INTERPRETER); - _Py_CODEUNIT *target = _PyFrame_GetBytecode(frame) + exit->target; + assert(FRAME_CORE->owner < FRAME_OWNED_BY_INTERPRETER); + _Py_CODEUNIT *target = _PyFrame_GetBytecodeFull(frame) + exit->target; _Py_BackoffCounter temperature = exit->temperature; _PyExecutorObject *executor; if (target->op.code == ENTER_EXECUTOR) { - PyCodeObject *code = _PyFrame_GetCode(frame); + PyCodeObject *code = _PyFrame_GetCodeFull(frame); executor = code->co_executors->executors[target->op.arg]; Py_INCREF(executor); assert(tstate->jit_exit == exit); diff --git a/Python/frame.c b/Python/frame.c index da8f9037e8287a..26b03cb7df9304 100644 --- a/Python/frame.c +++ b/Python/frame.c @@ -13,17 +13,17 @@ _PyFrame_Traverse(_PyInterpreterFrame *frame, visitproc visit, void *arg) Py_VISIT(frame->frame_obj); Py_VISIT(frame->f_locals); _Py_VISIT_STACKREF(frame->f_funcobj); - _Py_VISIT_STACKREF(frame->f_executable); + _Py_VISIT_STACKREF(_PyFrame_Core(frame)->f_executable); return _PyGC_VisitFrameStack(frame, visit, arg); } PyFrameObject * _PyFrame_MakeAndSetFrameObject(_PyInterpreterFrame *frame) { - assert(frame->frame_obj == NULL); PyObject *exc = PyErr_GetRaisedException(); + assert(frame->frame_obj == NULL); - PyFrameObject *f = _PyFrame_New_NoTrack(_PyFrame_GetCode(frame)); + PyFrameObject *f = _PyFrame_New_NoTrack(_PyFrame_GetCode(_PyFrame_Core(frame))); if (f == NULL) { Py_XDECREF(exc); return NULL; @@ -38,7 +38,7 @@ _PyFrame_MakeAndSetFrameObject(_PyInterpreterFrame *frame) // but it won't allocate a traceback until the frame unwinds, so we are safe // here. assert(frame->frame_obj == NULL); - assert(frame->owner != FRAME_OWNED_BY_FRAME_OBJECT); + assert(_PyFrame_Core(frame)->owner != FRAME_OWNED_BY_FRAME_OBJECT); f->f_frame = frame; frame->frame_obj = f; return f; @@ -48,29 +48,29 @@ static void take_ownership(PyFrameObject *f, _PyInterpreterFrame *frame) { Py_BEGIN_CRITICAL_SECTION(f); - assert(frame->owner < FRAME_OWNED_BY_INTERPRETER); - assert(frame->owner != FRAME_OWNED_BY_FRAME_OBJECT); + assert(_PyFrame_Core(frame)->owner < FRAME_OWNED_BY_INTERPRETER); + assert(_PyFrame_Core(frame)->owner != FRAME_OWNED_BY_FRAME_OBJECT); _PyInterpreterFrame *new_frame = (_PyInterpreterFrame *)f->_f_frame_data; _PyFrame_Copy(frame, new_frame); // _PyFrame_Copy takes the reference to the executable, // so we need to restore it. - frame->f_executable = PyStackRef_DUP(new_frame->f_executable); + _PyFrame_Core(frame)->f_executable = PyStackRef_DUP(_PyFrame_Core(new_frame)->f_executable); f->f_frame = new_frame; - new_frame->owner = FRAME_OWNED_BY_FRAME_OBJECT; - if (_PyFrame_IsIncomplete(new_frame)) { + _PyFrame_Core(new_frame)->owner = FRAME_OWNED_BY_FRAME_OBJECT; + if (_PyFrame_IsIncomplete(_PyFrame_Core(new_frame))) { // This may be a newly-created generator or coroutine frame. Since it's // dead anyways, just pretend that the first RESUME ran: - PyCodeObject *code = _PyFrame_GetCode(new_frame); + PyCodeObject *code = _PyFrame_GetCode(_PyFrame_Core(new_frame)); new_frame->instr_ptr = - _PyFrame_GetBytecode(new_frame) + code->_co_firsttraceable + 1; + _PyFrame_GetBytecode(_PyFrame_Core(new_frame)) + code->_co_firsttraceable + 1; } - assert(!_PyFrame_IsIncomplete(new_frame)); + assert(!_PyFrame_IsIncomplete(_PyFrame_Core(new_frame))); assert(f->f_back == NULL); - _PyInterpreterFrame *prev = _PyFrame_GetFirstComplete(frame->previous); + _PyInterpreterFrameCore *prev = _PyFrame_GetFirstComplete(_PyFrame_Core(frame)->previous); if (prev) { assert(prev->owner < FRAME_OWNED_BY_INTERPRETER); PyObject *exc = PyErr_GetRaisedException(); - /* Link PyFrameObjects.f_back and remove link through _PyInterpreterFrame.previous */ + /* Link PyFrameObjects.f_back and remove link through _PyInterpreterFrameCore.previous */ PyFrameObject *back = _PyFrame_GetFrameObject(prev); if (back == NULL) { /* Memory error here. */ @@ -108,11 +108,11 @@ _PyFrame_ClearExceptCode(_PyInterpreterFrame *frame) { /* It is the responsibility of the owning generator/coroutine * to have cleared the enclosing generator, if any. */ - assert(frame->owner != FRAME_OWNED_BY_GENERATOR || + assert(!(_PyFrame_Core(frame)->owner & FRAME_OWNED_BY_GENERATOR) || FT_ATOMIC_LOAD_INT8_RELAXED(_PyGen_GetGeneratorFromFrame(frame)->gi_frame_state) == FRAME_CLEARED); // GH-99729: Clearing this frame can expose the stack (via finalizers). It's // crucial that this frame has been unlinked, and is no longer visible: - assert(_PyThreadState_GET()->current_frame != frame); + assert(_PyThreadState_GET()->current_frame != _PyFrame_Core(frame)); if (frame->frame_obj) { PyFrameObject *f = frame->frame_obj; frame->frame_obj = NULL; @@ -127,16 +127,35 @@ _PyFrame_ClearExceptCode(_PyInterpreterFrame *frame) PyStackRef_CLEAR(frame->f_funcobj); } +// Calls the frame reifier and returns the original function object +_PyInterpreterFrame * +_PyFrame_InitializeExternalFrame(_PyInterpreterFrameCore *frame) +{ + if (_PyFrame_IsExternalFrame(frame)) { + PyObject *executor = PyStackRef_AsPyObjectBorrow(frame->f_executable); + PyUnstable_PyExternalExecutable *jit_exec = (PyUnstable_PyExternalExecutable *)executor; + _PyInterpreterFrame *res = jit_exec->ef_reifier(frame, executor); + assert(_PyFrame_Core(res) == frame); + return res; + } + + return _PyFrame_Full(frame); +} + /* Unstable API functions */ PyObject * -PyUnstable_InterpreterFrame_GetCode(struct _PyInterpreterFrame *frame) +PyUnstable_InterpreterFrame_GetCode(struct _PyInterpreterFrameCore *frame) { + if (_PyFrame_IsExternalFrame(frame)) { + PyObject *executable = PyStackRef_AsPyObjectBorrow(frame->f_executable); + return Py_NewRef(((PyUnstable_PyExternalExecutable *)executable)->ef_code); + } return PyStackRef_AsPyObjectNew(frame->f_executable); } int -PyUnstable_InterpreterFrame_GetLasti(struct _PyInterpreterFrame *frame) +PyUnstable_InterpreterFrame_GetLasti(struct _PyInterpreterFrameCore *frame) { return _PyInterpreterFrame_LASTI(frame) * sizeof(_Py_CODEUNIT); } @@ -144,16 +163,83 @@ PyUnstable_InterpreterFrame_GetLasti(struct _PyInterpreterFrame *frame) // NOTE: We allow racy accesses to the instruction pointer from other threads // for sys._current_frames() and similar APIs. int _Py_NO_SANITIZE_THREAD -PyUnstable_InterpreterFrame_GetLine(_PyInterpreterFrame *frame) +PyUnstable_InterpreterFrame_GetLine(_PyInterpreterFrameCore *frame) { int addr = _PyInterpreterFrame_LASTI(frame) * sizeof(_Py_CODEUNIT); return PyCode_Addr2Line(_PyFrame_GetCode(frame), addr); } +static int +jitexecutable_traverse(PyObject *self, visitproc visit, void *arg) +{ + PyUnstable_PyExternalExecutable *o = (PyUnstable_PyExternalExecutable *)self; + Py_VISIT(o->ef_code); + Py_VISIT(o->ef_state); + return 0; +} + +static int +jitexecutable_clear(PyObject *self) +{ + PyUnstable_PyExternalExecutable *o = (PyUnstable_PyExternalExecutable *)self; + Py_CLEAR(o->ef_code); + Py_CLEAR(o->ef_state); + return 0; +} + +static void +jitexecutable_dealloc(PyObject *self) +{ + PyUnstable_PyExternalExecutable *o = (PyUnstable_PyExternalExecutable *)self; + PyObject_GC_UnTrack(self); + Py_DECREF(o->ef_code); + Py_XDECREF(o->ef_state); + Py_TYPE(self)->tp_free(self); +} + +PyTypeObject PyUnstable_ExternalExecutable_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + .tp_name = "jit_executable", + .tp_basicsize = sizeof(PyUnstable_PyExternalExecutable), + .tp_dealloc = jitexecutable_dealloc, + .tp_traverse = jitexecutable_traverse, + .tp_clear = jitexecutable_clear, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_HAVE_GC, + .tp_alloc = PyType_GenericAlloc, + .tp_free = PyObject_GC_Del, +}; + +PyObject * +PyUnstable_MakeExternalExecutable(_PyFrame_Reifier reifier, PyCodeObject *code, PyObject *state) +{ + if (reifier == NULL) { + PyErr_SetString(PyExc_ValueError, "need reifier"); + return NULL; + } else if (code == NULL) { + PyErr_SetString(PyExc_ValueError, "need code object"); + return NULL; + } + + PyUnstable_PyExternalExecutable *jit_exec = PyObject_GC_New(PyUnstable_PyExternalExecutable, + &PyUnstable_ExternalExecutable_Type); + if (jit_exec == NULL) { + return NULL; + } + + jit_exec->ef_reifier = reifier; + jit_exec->ef_code = (PyCodeObject *)Py_NewRef(code); + jit_exec->ef_state = Py_XNewRef(state); + if (state != NULL && PyObject_GC_IsTracked(state)) { + PyObject_GC_Track((PyObject *)jit_exec); + } + return (PyObject *)jit_exec; +} + const PyTypeObject *const PyUnstable_ExecutableKinds[PyUnstable_EXECUTABLE_KINDS+1] = { [PyUnstable_EXECUTABLE_KIND_SKIP] = &_PyNone_Type, [PyUnstable_EXECUTABLE_KIND_PY_FUNCTION] = &PyCode_Type, [PyUnstable_EXECUTABLE_KIND_BUILTIN_FUNCTION] = &PyMethod_Type, [PyUnstable_EXECUTABLE_KIND_METHOD_DESCRIPTOR] = &PyMethodDescr_Type, + [PyUnstable_EXECUTABLE_KIND_JIT] = &PyUnstable_ExternalExecutable_Type, [PyUnstable_EXECUTABLE_KINDS] = NULL, }; diff --git a/Python/gc.c b/Python/gc.c index 2f373dcb402df3..02ad9483b07487 100644 --- a/Python/gc.c +++ b/Python/gc.c @@ -1540,12 +1540,13 @@ mark_stacks(PyInterpreterState *interp, PyGC_Head *visited, int visited_space, b PyThreadState* ts = PyInterpreterState_ThreadHead(interp); HEAD_UNLOCK(runtime); while (ts) { - _PyInterpreterFrame *frame = ts->current_frame; - while (frame) { - if (frame->owner >= FRAME_OWNED_BY_INTERPRETER) { - frame = frame->previous; + _PyInterpreterFrameCore *f = ts->current_frame; + while (f) { + if (f->owner >= FRAME_OWNED_BY_INTERPRETER || _PyFrame_IsExternalFrame(f)) { + f = f->previous; continue; } + _PyInterpreterFrame *frame = _PyFrame_Full(f); _PyStackRef *locals = frame->localsplus; _PyStackRef *sp = frame->stackpointer; objects_marked += move_to_reachable(frame->f_locals, &reachable, visited_space); @@ -1576,7 +1577,7 @@ mark_stacks(PyInterpreterState *interp, PyGC_Head *visited, int visited_space, b break; } frame->visited = 1; - frame = frame->previous; + f = f->previous; } HEAD_LOCK(runtime); ts = PyThreadState_Next(ts); @@ -2073,7 +2074,7 @@ Py_ssize_t _PyGC_Collect(PyThreadState *tstate, int generation, _PyGC_Reason reason) { GCState *gcstate = &tstate->interp->gc; - assert(tstate->current_frame == NULL || tstate->current_frame->stackpointer != NULL); + assert(_PyFrame_StackpointerSaved()); int expected = 0; if (!_Py_atomic_compare_exchange_int(&gcstate->collecting, &expected, 1)) { diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c index 0ec9c58a792e6d..2d7db800ba2fff 100644 --- a/Python/gc_free_threading.c +++ b/Python/gc_free_threading.c @@ -269,18 +269,23 @@ merge_refcount(PyObject *op, Py_ssize_t extra) } static void -frame_disable_deferred_refcounting(_PyInterpreterFrame *frame) +frame_disable_deferred_refcounting(_PyInterpreterFrameCore *frame) { // Convert locals, variables, and the executable object to strong // references from (possibly) deferred references. - assert(frame->stackpointer != NULL); assert(frame->owner == FRAME_OWNED_BY_FRAME_OBJECT || - frame->owner == FRAME_OWNED_BY_GENERATOR); + frame->owner & FRAME_OWNED_BY_GENERATOR); frame->f_executable = PyStackRef_AsStrongReference(frame->f_executable); + if (_PyFrame_IsExternalFrame(frame)) { + return; + } + + _PyInterpreterFrame *f = _PyFrame_Full(frame); + assert(f->stackpointer != NULL); - if (frame->owner == FRAME_OWNED_BY_GENERATOR) { - PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); + if (frame->owner & FRAME_OWNED_BY_GENERATOR) { + PyGenObject *gen = _PyGen_GetGeneratorFromFrame(f); if (gen->gi_frame_state == FRAME_CLEARED) { // gh-124068: if the generator is cleared, then most fields other // than f_executable are not valid. @@ -288,8 +293,8 @@ frame_disable_deferred_refcounting(_PyInterpreterFrame *frame) } } - frame->f_funcobj = PyStackRef_AsStrongReference(frame->f_funcobj); - for (_PyStackRef *ref = frame->localsplus; ref < frame->stackpointer; ref++) { + f->f_funcobj = PyStackRef_AsStrongReference(f->f_funcobj); + for (_PyStackRef *ref = f->localsplus; ref < f->stackpointer; ref++) { if (!PyStackRef_IsNullOrInt(*ref) && !PyStackRef_RefcountOnObject(*ref)) { *ref = PyStackRef_AsStrongReference(*ref); } @@ -315,10 +320,10 @@ disable_deferred_refcounting(PyObject *op) // use strong references, in case the generator or frame object is // resurrected by a finalizer. if (PyGen_CheckExact(op) || PyCoro_CheckExact(op) || PyAsyncGen_CheckExact(op)) { - frame_disable_deferred_refcounting(&((PyGenObject *)op)->gi_iframe); + frame_disable_deferred_refcounting(_PyFrame_Core(&((PyGenObject *)op)->gi_iframe)); } else if (PyFrame_Check(op)) { - frame_disable_deferred_refcounting(((PyFrameObject *)op)->f_frame); + frame_disable_deferred_refcounting(_PyFrame_Core(((PyFrameObject *)op)->f_frame)); } } @@ -477,12 +482,13 @@ gc_visit_thread_stacks(PyInterpreterState *interp, struct collection_state *stat c_ref = c_ref->next; } - for (_PyInterpreterFrame *f = p->current_frame; f != NULL; f = f->previous) { - if (f->owner >= FRAME_OWNED_BY_INTERPRETER) { + for (_PyInterpreterFrameCore *f = p->current_frame; f != NULL; f = f->previous) { + if (f->owner >= FRAME_OWNED_BY_INTERPRETER || _PyFrame_IsExternalFrame(f)) { continue; } - _PyStackRef *top = f->stackpointer; + _PyInterpreterFrame *frame = _PyFrame_Full(f); + _PyStackRef *top = frame->stackpointer; if (top == NULL) { // GH-129236: The stackpointer may be NULL in cases where // the GC is run during a PyStackRef_CLOSE() call. Skip this @@ -492,7 +498,7 @@ gc_visit_thread_stacks(PyInterpreterState *interp, struct collection_state *stat } gc_visit_stackref(f->f_executable); - while (top != f->localsplus) { + while (top != frame->localsplus) { --top; gc_visit_stackref(*top); } @@ -874,24 +880,25 @@ gc_visit_thread_stacks_mark_alive(PyInterpreterState *interp, gc_mark_args_t *ar { int err = 0; _Py_FOR_EACH_TSTATE_BEGIN(interp, p) { - for (_PyInterpreterFrame *f = p->current_frame; f != NULL; f = f->previous) { - if (f->owner >= FRAME_OWNED_BY_INTERPRETER) { + for (_PyInterpreterFrameCore *f = p->current_frame; f != NULL; f = f->previous) { + if (f->owner >= FRAME_OWNED_BY_INTERPRETER || _PyFrame_IsExternalFrame(f)) { continue; } - if (f->stackpointer == NULL) { + _PyInterpreterFrame *frame = _PyFrame_Full(f); + if (frame->stackpointer == NULL) { // GH-129236: The stackpointer may be NULL in cases where // the GC is run during a PyStackRef_CLOSE() call. Skip this // frame for now. continue; } - _PyStackRef *top = f->stackpointer; + _PyStackRef *top = frame->stackpointer; if (gc_visit_stackref_mark_alive(args, f->f_executable) < 0) { err = -1; goto exit; } - while (top != f->localsplus) { + while (top != frame->localsplus) { --top; if (gc_visit_stackref_mark_alive(args, *top) < 0) { err = -1; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 194fbe4f268cb4..c4f07460715606 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -746,9 +746,10 @@ stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); - assert(temp->previous == frame || temp->previous->previous == frame); + assert(_PyFrame_Core(temp)->previous == FRAME_CORE || _PyFrame_Core(temp)->previous->previous == FRAME_CORE); CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; + frame = temp; + tstate->current_frame = _PyFrame_Core(temp); tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); @@ -1771,7 +1772,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( tstate, callable, locals, - arguments, total_args, NULL, frame + arguments, total_args, NULL, FRAME_CORE ); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; @@ -1900,14 +1901,14 @@ init = callable; _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( - tstate, (PyCodeObject *)&_Py_InitCleanup, 1, frame); + tstate, (PyCodeObject *)&_Py_InitCleanup, 1, FRAME_CORE); stack_pointer = _PyFrame_GetStackPointer(frame); - assert(_PyFrame_GetBytecode(shim)[0].op.code == EXIT_INIT_CHECK); - assert(_PyFrame_GetBytecode(shim)[1].op.code == RETURN_VALUE); + assert(_PyFrame_GetBytecodeFull(shim)[0].op.code == EXIT_INIT_CHECK); + assert(_PyFrame_GetBytecodeFull(shim)[1].op.code == RETURN_VALUE); shim->localsplus[0] = PyStackRef_DUP(self); _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( - tstate, init, NULL, args-1, oparg+1, NULL, shim); + tstate, init, NULL, args-1, oparg+1, NULL, _PyFrame_Core(shim)); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -1927,9 +1928,10 @@ assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); _PyFrame_SetStackPointer(frame, stack_pointer); - assert(temp->previous == frame || temp->previous->previous == frame); + assert(_PyFrame_Core(temp)->previous == FRAME_CORE || _PyFrame_Core(temp)->previous->previous == FRAME_CORE); CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; + frame = temp; + tstate->current_frame = _PyFrame_Core(temp); tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); @@ -2070,9 +2072,10 @@ stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); - assert(temp->previous == frame || temp->previous->previous == frame); + assert(_PyFrame_Core(temp)->previous == FRAME_CORE || _PyFrame_Core(temp)->previous->previous == FRAME_CORE); CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; + frame = temp; + tstate->current_frame = _PyFrame_Core(temp); tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); @@ -2174,7 +2177,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( tstate, callable, locals, - args, total_args, NULL, frame + args, total_args, NULL, FRAME_CORE ); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; @@ -2198,9 +2201,10 @@ assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); _PyFrame_SetStackPointer(frame, stack_pointer); - assert(temp->previous == frame || temp->previous->previous == frame); + assert(_PyFrame_Core(temp)->previous == FRAME_CORE || _PyFrame_Core(temp)->previous->previous == FRAME_CORE); CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; + frame = temp; + tstate->current_frame = _PyFrame_Core(temp); tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); @@ -2703,7 +2707,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( tstate, func_st, locals, - nargs, callargs, kwargs, frame); + nargs, callargs, kwargs, FRAME_CORE); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -2727,9 +2731,10 @@ assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); _PyFrame_SetStackPointer(frame, stack_pointer); - assert(temp->previous == frame || temp->previous->previous == frame); + assert(_PyFrame_Core(temp)->previous == FRAME_CORE || _PyFrame_Core(temp)->previous->previous == FRAME_CORE); CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; + frame = temp; + tstate->current_frame = _PyFrame_Core(temp); tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); @@ -2867,7 +2872,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( tstate, func_st, locals, - nargs, callargs, kwargs, frame); + nargs, callargs, kwargs, FRAME_CORE); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -3143,7 +3148,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( tstate, callable, locals, - arguments, positional_args, kwnames_o, frame + arguments, positional_args, kwnames_o, FRAME_CORE ); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -3 - oparg; @@ -3274,7 +3279,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( tstate, callable, locals, - arguments, positional_args, kwnames_o, frame + arguments, positional_args, kwnames_o, FRAME_CORE ); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; @@ -3303,9 +3308,10 @@ assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); _PyFrame_SetStackPointer(frame, stack_pointer); - assert(temp->previous == frame || temp->previous->previous == frame); + assert(_PyFrame_Core(temp)->previous == FRAME_CORE || _PyFrame_Core(temp)->previous->previous == FRAME_CORE); CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; + frame = temp; + tstate->current_frame = _PyFrame_Core(temp); tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); @@ -3461,7 +3467,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( tstate, callable, locals, - arguments, positional_args, kwnames_o, frame + arguments, positional_args, kwnames_o, FRAME_CORE ); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; @@ -3490,9 +3496,10 @@ assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); _PyFrame_SetStackPointer(frame, stack_pointer); - assert(temp->previous == frame || temp->previous->previous == frame); + assert(_PyFrame_Core(temp)->previous == FRAME_CORE || _PyFrame_Core(temp)->previous->previous == FRAME_CORE); CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; + frame = temp; + tstate->current_frame = _PyFrame_Core(temp); tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); @@ -4242,9 +4249,10 @@ stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); - assert(temp->previous == frame || temp->previous->previous == frame); + assert(_PyFrame_Core(temp)->previous == FRAME_CORE || _PyFrame_Core(temp)->previous->previous == FRAME_CORE); CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; + frame = temp; + tstate->current_frame = _PyFrame_Core(temp); tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); @@ -4318,7 +4326,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( tstate, callable, locals, - args, total_args, NULL, frame + args, total_args, NULL, FRAME_CORE ); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; @@ -4342,9 +4350,10 @@ assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); _PyFrame_SetStackPointer(frame, stack_pointer); - assert(temp->previous == frame || temp->previous->previous == frame); + assert(_PyFrame_Core(temp)->previous == FRAME_CORE || _PyFrame_Core(temp)->previous->previous == FRAME_CORE); CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; + frame = temp; + tstate->current_frame = _PyFrame_Core(temp); tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); @@ -5282,7 +5291,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(COPY_FREE_VARS); - PyCodeObject *co = _PyFrame_GetCode(frame); + PyCodeObject *co = _PyFrame_GetCodeFull(frame); assert(PyStackRef_FunctionCheck(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); PyObject *closure = func->func_closure; @@ -5332,7 +5341,7 @@ PyObject *oldobj = PyCell_SwapTakeRef((PyCellObject *)cell, NULL); if (oldobj == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCodeFull(frame), oparg); stack_pointer = _PyFrame_GetStackPointer(frame); JUMP_TO_LABEL(error); } @@ -5355,7 +5364,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, UNBOUNDLOCAL_ERROR_MSG, - PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) + PyTuple_GetItem(_PyFrame_GetCodeFull(frame)->co_localsplusnames, oparg) ); stack_pointer = _PyFrame_GetStackPointer(frame); JUMP_TO_LABEL(error); @@ -5640,7 +5649,7 @@ next_instr = this_instr; JUMP_TO_LABEL(stop_tracing); } - PyCodeObject *code = _PyFrame_GetCode(frame); + PyCodeObject *code = _PyFrame_GetCodeFull(frame); _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; assert(executor->vm_data.index == INSTR_OFFSET() - 1); assert(executor->vm_data.code == code); @@ -5870,7 +5879,7 @@ _PyFrame_StackPush(pushed_frame, PyStackRef_None); gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; - pushed_frame->previous = frame; + _PyFrame_Core(pushed_frame)->previous = FRAME_CORE; frame->return_offset = (uint16_t)( 2u + oparg); gen_frame = PyStackRef_Wrap(pushed_frame); } @@ -5880,9 +5889,10 @@ assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); _PyFrame_SetStackPointer(frame, stack_pointer); - assert(temp->previous == frame || temp->previous->previous == frame); + assert(_PyFrame_Core(temp)->previous == FRAME_CORE || _PyFrame_Core(temp)->previous->previous == FRAME_CORE); CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; + frame = temp; + tstate->current_frame = _PyFrame_Core(temp); tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); @@ -6298,7 +6308,7 @@ iterable = stack_pointer[-1]; PyObject *iterable_o = PyStackRef_AsPyObjectBorrow(iterable); if (PyCoro_CheckExact(iterable_o)) { - if (!(_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) { + if (!(_PyFrame_GetCodeFull(frame)->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) { _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_SetString(tstate, PyExc_TypeError, "cannot 'yield from' a coroutine object " @@ -6370,7 +6380,7 @@ level = stack_pointer[-2]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyEval_ImportName(tstate, frame, name, + PyObject *res_o = _PyEval_ImportName(tstate, FRAME_CORE, name, PyStackRef_AsPyObjectBorrow(fromlist), PyStackRef_AsPyObjectBorrow(level)); _PyStackRef tmp = fromlist; @@ -6480,7 +6490,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( tstate, callable, locals, - arguments, total_args, NULL, frame + arguments, total_args, NULL, FRAME_CORE ); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; @@ -6637,7 +6647,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( tstate, func_st, locals, - nargs, callargs, kwargs, frame); + nargs, callargs, kwargs, FRAME_CORE); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -6776,7 +6786,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( tstate, callable, locals, - arguments, positional_args, kwnames_o, frame + arguments, positional_args, kwnames_o, FRAME_CORE ); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -3 - oparg; @@ -7049,8 +7059,8 @@ opcode = INSTRUMENTED_LINE; int original_opcode = 0; if (tstate->tracing) { - PyCodeObject *code = _PyFrame_GetCode(frame); - int index = (int)(this_instr - _PyFrame_GetBytecode(frame)); + PyCodeObject *code = _PyFrame_GetCodeFull(frame); + int index = (int)(this_instr - _PyFrame_GetBytecodeFull(frame)); original_opcode = code->_co_monitoring->lines->data[index*code->_co_monitoring->lines->bytes_per_entry]; next_instr = this_instr; } else { @@ -7364,12 +7374,12 @@ ((_PyThreadStateImpl *)tstate)->tlbc_index) { _PyFrame_SetStackPointer(frame, stack_pointer); _Py_CODEUNIT *bytecode = - _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); + _PyEval_GetExecutableCode(tstate, _PyFrame_GetCodeFull(frame)); stack_pointer = _PyFrame_GetStackPointer(frame); if (bytecode == NULL) { JUMP_TO_LABEL(error); } - ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame); + ptrdiff_t off = this_instr - _PyFrame_GetBytecodeFull(frame); frame->tlbc_index = ((_PyThreadStateImpl *)tstate)->tlbc_index; frame->instr_ptr = bytecode + off; next_instr = frame->instr_ptr; @@ -7387,10 +7397,10 @@ #endif if (check_instrumentation) { uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; - uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); + uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCodeFull(frame)->_co_instrumentation_version); if (code_version != global_version) { _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); + int err = _Py_Instrument(_PyFrame_GetCodeFull(frame), tstate->interp); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { JUMP_TO_LABEL(error); @@ -7455,7 +7465,7 @@ // _RETURN_VALUE { retval = val; - assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + assert(FRAME_CORE->owner != FRAME_OWNED_BY_INTERPRETER); _PyStackRef temp = PyStackRef_MakeHeapSafe(retval); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -7463,7 +7473,7 @@ assert(STACK_LEVEL() == 0); _Py_LeaveRecursiveCallPy(tstate); _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; + frame = (_PyInterpreterFrame *)(tstate->current_frame = _PyFrame_Core(dying)->previous); _PyEval_FrameClearAndPop(tstate, dying); stack_pointer = _PyFrame_GetStackPointer(frame); LOAD_IP(frame->return_offset); @@ -7508,7 +7518,7 @@ // _YIELD_VALUE { retval = val; - assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + assert(FRAME_CORE->owner != FRAME_OWNED_BY_INTERPRETER); frame->instr_ptr++; PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); @@ -7521,8 +7531,8 @@ gen->gi_exc_state.previous_item = NULL; _Py_LeaveRecursiveCallPy(tstate); _PyInterpreterFrame *gen_frame = frame; - frame = tstate->current_frame = frame->previous; - gen_frame->previous = NULL; + frame = (_PyInterpreterFrame *)(tstate->current_frame = FRAME_CORE->previous); + _PyFrame_Core(gen_frame)->previous = NULL; ((_PyThreadStateImpl *)tstate)->generator_return_kind = GENERATOR_YIELD; FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, FRAME_SUSPENDED + oparg); assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); @@ -7555,9 +7565,9 @@ INSTRUCTION_STATS(INTERPRETER_EXIT); _PyStackRef retval; retval = stack_pointer[-1]; - assert(frame->owner == FRAME_OWNED_BY_INTERPRETER); - assert(_PyFrame_IsIncomplete(frame)); - tstate->current_frame = frame->previous; + assert(FRAME_CORE->owner == FRAME_OWNED_BY_INTERPRETER); + assert(_PyFrame_IsIncomplete(FRAME_CORE)); + tstate->current_frame = FRAME_CORE->previous; assert(!_PyErr_Occurred(tstate)); PyObject *result = PyStackRef_AsPyObjectSteal(retval); #if !_Py_TAIL_CALL_INTERP @@ -8635,9 +8645,10 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); - assert(temp->previous == frame || temp->previous->previous == frame); + assert(_PyFrame_Core(temp)->previous == FRAME_CORE || _PyFrame_Core(temp)->previous->previous == FRAME_CORE); CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; + frame = temp; + tstate->current_frame = _PyFrame_Core(temp); tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); @@ -8919,7 +8930,7 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCodeFull(frame), oparg); stack_pointer = _PyFrame_GetStackPointer(frame); JUMP_TO_LABEL(error); } @@ -9015,7 +9026,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, UNBOUNDLOCAL_ERROR_MSG, - PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) + PyTuple_GetItem(_PyFrame_GetCodeFull(frame)->co_localsplusnames, oparg) ); stack_pointer = _PyFrame_GetStackPointer(frame); JUMP_TO_LABEL(error); @@ -9062,8 +9073,8 @@ PyObject *name; PyObject *class_dict = PyStackRef_AsPyObjectBorrow(class_dict_st); assert(class_dict); - assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); - name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); + assert(oparg >= 0 && oparg < _PyFrame_GetCodeFull(frame)->co_nlocalsplus); + name = PyTuple_GET_ITEM(_PyFrame_GetCodeFull(frame)->co_localsplusnames, oparg); int err; _PyFrame_SetStackPointer(frame, stack_pointer); PyObject* value_o = _PyMapping_GetOptionalItem2(class_dict, name, &err); @@ -9076,7 +9087,7 @@ value_o = PyCell_GetRef(cell); if (value_o == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCodeFull(frame), oparg); stack_pointer = _PyFrame_GetStackPointer(frame); JUMP_TO_LABEL(error); } @@ -10271,7 +10282,7 @@ PyObject *exc = PyStackRef_AsPyObjectSteal(exc_st); assert(oparg >= 0 && oparg <= 2); if (oparg) { - frame->instr_ptr = _PyFrame_GetBytecode(frame) + PyStackRef_UntagInt(values[0]); + frame->instr_ptr = _PyFrame_GetBytecodeFull(frame) + PyStackRef_UntagInt(values[0]); } assert(exc && PyExceptionInstance_Check(exc)); stack_pointer += -1; @@ -10313,12 +10324,12 @@ ((_PyThreadStateImpl *)tstate)->tlbc_index) { _PyFrame_SetStackPointer(frame, stack_pointer); _Py_CODEUNIT *bytecode = - _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); + _PyEval_GetExecutableCode(tstate, _PyFrame_GetCodeFull(frame)); stack_pointer = _PyFrame_GetStackPointer(frame); if (bytecode == NULL) { JUMP_TO_LABEL(error); } - ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame); + ptrdiff_t off = this_instr - _PyFrame_GetBytecodeFull(frame); frame->tlbc_index = ((_PyThreadStateImpl *)tstate)->tlbc_index; frame->instr_ptr = bytecode + off; next_instr = frame->instr_ptr; @@ -10336,10 +10347,10 @@ #endif if (check_instrumentation) { uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; - uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); + uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCodeFull(frame)->_co_instrumentation_version); if (code_version != global_version) { _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); + int err = _Py_Instrument(_PyFrame_GetCodeFull(frame), tstate->interp); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { JUMP_TO_LABEL(error); @@ -10391,7 +10402,7 @@ _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; #endif uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); - uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); + uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCodeFull(frame)->_co_instrumentation_version); assert((version & _PY_EVAL_EVENTS_MASK) == 0); if (eval_breaker != version) { UPDATE_MISS_STATS(RESUME); @@ -10433,11 +10444,11 @@ _PyFrame_Copy(frame, gen_frame); assert(frame->frame_obj == NULL); gen->gi_frame_state = FRAME_CREATED; - gen_frame->owner = FRAME_OWNED_BY_GENERATOR; + _PyFrame_Core(gen_frame)->owner = FRAME_OWNED_BY_GENERATOR; _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *prev = frame->previous; - _PyThreadState_PopFrame(tstate, frame); - frame = tstate->current_frame = prev; + _PyInterpreterFrameCore *prev = FRAME_CORE->previous; + _PyThreadState_PopFrame(tstate, FRAME_CORE); + frame = (_PyInterpreterFrame *)(tstate->current_frame = prev); LOAD_IP(frame->return_offset); stack_pointer = _PyFrame_GetStackPointer(frame); res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); @@ -10459,7 +10470,7 @@ _PyStackRef retval; _PyStackRef res; retval = stack_pointer[-1]; - assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + assert(FRAME_CORE->owner != FRAME_OWNED_BY_INTERPRETER); _PyStackRef temp = PyStackRef_MakeHeapSafe(retval); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -10467,7 +10478,7 @@ assert(STACK_LEVEL() == 0); _Py_LeaveRecursiveCallPy(tstate); _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; + frame = (_PyInterpreterFrame *)(tstate->current_frame = _PyFrame_Core(dying)->previous); _PyEval_FrameClearAndPop(tstate, dying); stack_pointer = _PyFrame_GetStackPointer(frame); LOAD_IP(frame->return_offset); @@ -10515,7 +10526,7 @@ v = stack_pointer[-1]; PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); PyObject *retval_o; - assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + assert(FRAME_CORE->owner != FRAME_OWNED_BY_INTERPRETER); if (!IS_PEP523_HOOKED(tstate) && (Py_TYPE(receiver_o) == &PyGen_Type || Py_TYPE(receiver_o) == &PyCoro_Type) && gen_try_set_executing((PyGenObject *)receiver_o)) @@ -10529,8 +10540,8 @@ tstate->exc_info = &gen->gi_exc_state; assert( 2u + oparg <= UINT16_MAX); frame->return_offset = (uint16_t)( 2u + oparg); - assert(gen_frame->previous == NULL); - gen_frame->previous = frame; + assert(_PyFrame_Core(gen_frame)->previous == NULL); + _PyFrame_Core(gen_frame)->previous = FRAME_CORE; DISPATCH_INLINED(gen_frame); } if (PyStackRef_IsNone(v) && PyIter_Check(receiver_o)) { @@ -10629,7 +10640,7 @@ tstate->exc_info = &gen->gi_exc_state; assert( 2u + oparg <= UINT16_MAX); frame->return_offset = (uint16_t)( 2u + oparg); - pushed_frame->previous = frame; + _PyFrame_Core(pushed_frame)->previous = FRAME_CORE; gen_frame = PyStackRef_Wrap(pushed_frame); } // _PUSH_FRAME @@ -10640,9 +10651,10 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); - assert(temp->previous == frame || temp->previous->previous == frame); + assert(_PyFrame_Core(temp)->previous == FRAME_CORE || _PyFrame_Core(temp)->previous->previous == FRAME_CORE); CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; + frame = temp; + tstate->current_frame = _PyFrame_Core(temp); tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); @@ -11878,7 +11890,7 @@ _PyJitTracerState *tracer = _tstate->jit_tracer_state; assert(tracer != NULL); tracer->prev_state.instr = next_instr; - PyObject *prev_code = PyStackRef_AsPyObjectBorrow(frame->f_executable); + PyObject *prev_code = PyStackRef_AsPyObjectBorrow(FRAME_CORE->f_executable); if (tracer->prev_state.instr_code != (PyCodeObject *)prev_code) { _PyFrame_SetStackPointer(frame, stack_pointer); Py_SETREF(tracer->prev_state.instr_code, (PyCodeObject*)Py_NewRef((prev_code))); @@ -11886,7 +11898,7 @@ } tracer->prev_state.instr_frame = frame; tracer->prev_state.instr_oparg = oparg; - tracer->prev_state.instr_stacklevel = PyStackRef_IsNone(frame->f_executable) ? 2 : STACK_LEVEL(); + tracer->prev_state.instr_stacklevel = PyStackRef_IsNone(FRAME_CORE->f_executable) ? 2 : STACK_LEVEL(); if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { (&next_instr[1])->counter = trigger_backoff_counter(); } @@ -12282,7 +12294,7 @@ _PyStackRef retval; _PyStackRef value; retval = stack_pointer[-1]; - assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + assert(FRAME_CORE->owner != FRAME_OWNED_BY_INTERPRETER); frame->instr_ptr++; PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); @@ -12295,8 +12307,8 @@ gen->gi_exc_state.previous_item = NULL; _Py_LeaveRecursiveCallPy(tstate); _PyInterpreterFrame *gen_frame = frame; - frame = tstate->current_frame = frame->previous; - gen_frame->previous = NULL; + frame = (_PyInterpreterFrame *)(tstate->current_frame = FRAME_CORE->previous); + _PyFrame_Core(gen_frame)->previous = NULL; ((_PyThreadStateImpl *)tstate)->generator_return_kind = GENERATOR_YIELD; FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, FRAME_SUSPENDED + oparg); assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); @@ -12330,8 +12342,8 @@ opcode = next_instr->op.code; _PyErr_Format(tstate, PyExc_SystemError, "%U:%d: unknown opcode %d", - _PyFrame_GetCode(frame)->co_filename, - PyUnstable_InterpreterFrame_GetLine(frame), + FRAME_CODE->co_filename, + PyUnstable_InterpreterFrame_GetLine(FRAME_CORE), opcode); JUMP_TO_LABEL(error); @@ -12373,10 +12385,10 @@ JUMP_TO_LABEL(error); _PyFrame_SetStackPointer(frame, stack_pointer); STOP_TRACING(); stack_pointer = _PyFrame_GetStackPointer(frame); - assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); - if (!_PyFrame_IsIncomplete(frame)) { + assert(FRAME_CORE->owner != FRAME_OWNED_BY_INTERPRETER); + if (!_PyFrame_IsIncomplete(FRAME_CORE)) { _PyFrame_SetStackPointer(frame, stack_pointer); - PyFrameObject *f = _PyFrame_GetFrameObject(frame); + PyFrameObject *f = _PyFrame_GetFrameObjectFull(frame); stack_pointer = _PyFrame_GetStackPointer(frame); if (f != NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); @@ -12394,7 +12406,7 @@ JUMP_TO_LABEL(error); STOP_TRACING(); int offset = INSTR_OFFSET()-1; int level, handler, lasti; - int handled = get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti); + int handled = get_exception_handler(_PyFrame_GetCodeFull(frame), offset, &level, &handler, &lasti); if (handled == 0) { assert(_PyErr_Occurred(tstate)); _PyStackRef *stackbase = _PyFrame_Stackbase(frame); @@ -12413,13 +12425,13 @@ JUMP_TO_LABEL(error); PyStackRef_XCLOSE(ref); } if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); + int frame_lasti = _PyInterpreterFrame_LASTI_FULL(frame); _PyStackRef lasti = PyStackRef_TagInt(frame_lasti); _PyFrame_StackPush(frame, lasti); } PyObject *exc = _PyErr_GetRaisedException(tstate); _PyFrame_StackPush(frame, PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; + next_instr = _PyFrame_GetBytecodeFull(frame) + handler; int err = monitor_handled(tstate, frame, next_instr, exc); if (err < 0) { JUMP_TO_LABEL(exception_unwind); @@ -12440,13 +12452,13 @@ JUMP_TO_LABEL(error); { assert(_PyErr_Occurred(tstate)); _Py_LeaveRecursiveCallPy(tstate); - assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + assert(FRAME_CORE->owner != FRAME_OWNED_BY_INTERPRETER); _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; + frame = (_PyInterpreterFrame *)(tstate->current_frame = _PyFrame_Core(dying)->previous); _PyEval_FrameClearAndPop(tstate, dying); frame->return_offset = 0; - if (frame->owner == FRAME_OWNED_BY_INTERPRETER) { - tstate->current_frame = frame->previous; + if (FRAME_CORE->owner == FRAME_OWNED_BY_INTERPRETER) { + tstate->current_frame = FRAME_CORE->previous; #if !_Py_TAIL_CALL_INTERP assert(frame == &entry.frame); #endif diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 28bbe1d82a3b88..8f0efa09697faf 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -1141,11 +1141,12 @@ static const char *const event_names [] = { static int call_instrumentation_vector( _Py_CODEUNIT *instr, PyThreadState *tstate, int event, - _PyInterpreterFrame *frame, _Py_CODEUNIT *arg2, Py_ssize_t nargs, PyObject *args[]) + _PyInterpreterFrame *f, _Py_CODEUNIT *arg2, Py_ssize_t nargs, PyObject *args[]) { if (tstate->tracing) { return 0; } + _PyInterpreterFrameCore *frame = _PyFrame_Core(f); assert(!_PyErr_Occurred(tstate)); assert(args[0] == NULL); PyCodeObject *code = _PyFrame_GetCode(frame); @@ -1238,7 +1239,7 @@ _Py_call_instrumentation_jump( assert(event == PY_MONITORING_EVENT_JUMP || event == PY_MONITORING_EVENT_BRANCH_RIGHT || event == PY_MONITORING_EVENT_BRANCH_LEFT); - int to = (int)(dest - _PyFrame_GetBytecode(frame)); + int to = (int)(dest - _PyFrame_GetBytecode(_PyFrame_Core(frame))); PyObject *to_obj = PyLong_FromLong(to * (int)sizeof(_Py_CODEUNIT)); if (to_obj == NULL) { return NULL; @@ -1298,8 +1299,9 @@ _Py_Instrumentation_GetLine(PyCodeObject *code, int index) } Py_NO_INLINE int -_Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, _Py_CODEUNIT *instr, _Py_CODEUNIT *prev) +_Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* f, _Py_CODEUNIT *instr, _Py_CODEUNIT *prev) { + _PyInterpreterFrameCore *frame = _PyFrame_Core(f); PyCodeObject *code = _PyFrame_GetCode(frame); assert(tstate->tracing == 0); assert(debug_check_sanity(tstate->interp, code)); @@ -1401,8 +1403,9 @@ _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, } Py_NO_INLINE int -_Py_call_instrumentation_instruction(PyThreadState *tstate, _PyInterpreterFrame* frame, _Py_CODEUNIT *instr) +_Py_call_instrumentation_instruction(PyThreadState *tstate, _PyInterpreterFrame* f, _Py_CODEUNIT *instr) { + _PyInterpreterFrameCore *frame = _PyFrame_Core(f); PyCodeObject *code = _PyFrame_GetCode(frame); int offset = (int)(instr - _PyFrame_GetBytecode(frame)); _PyCoMonitoringData *instrumentation_data = code->_co_monitoring; @@ -1948,7 +1951,7 @@ instrument_all_executing_code_objects(PyInterpreterState *interp) int err = 0; _Py_FOR_EACH_TSTATE_BEGIN(interp, ts) { - _PyInterpreterFrame *frame = ts->current_frame; + _PyInterpreterFrameCore *frame = ts->current_frame; while (frame) { if (frame->owner < FRAME_OWNED_BY_INTERPRETER) { err = instrument_lock_held(_PyFrame_GetCode(frame), interp); diff --git a/Python/intrinsics.c b/Python/intrinsics.c index 9cfc285c6a5925..d39e9b1a241283 100644 --- a/Python/intrinsics.c +++ b/Python/intrinsics.c @@ -121,7 +121,7 @@ import_all_from(PyThreadState *tstate, PyObject *locals, PyObject *v) static PyObject * import_star(PyThreadState* tstate, PyObject *from) { - _PyInterpreterFrame *frame = tstate->current_frame; + _PyInterpreterFrameCore *frame = tstate->current_frame; PyObject *locals = _PyFrame_GetLocals(frame); if (locals == NULL) { @@ -140,8 +140,8 @@ import_star(PyThreadState* tstate, PyObject *from) static PyObject * stopiteration_error(PyThreadState* tstate, PyObject *exc) { - _PyInterpreterFrame *frame = tstate->current_frame; - assert(frame->owner == FRAME_OWNED_BY_GENERATOR); + _PyInterpreterFrameCore *frame = tstate->current_frame; + assert(frame->owner & FRAME_OWNED_BY_GENERATOR); assert(PyExceptionInstance_Check(exc)); const char *msg = NULL; if (PyErr_GivenExceptionMatches(exc, PyExc_StopIteration)) { diff --git a/Python/legacy_tracing.c b/Python/legacy_tracing.c index 594d5c5ead5021..3a7ed69a2167e7 100644 --- a/Python/legacy_tracing.c +++ b/Python/legacy_tracing.c @@ -153,7 +153,7 @@ _PyEval_SetOpcodeTrace(PyFrameObject *frame, bool enable) { assert(frame != NULL); - PyCodeObject *code = _PyFrame_GetCode(frame->f_frame); + PyCodeObject *code = _PyFrame_GetCode(_PyFrame_Core(frame->f_frame)); #ifdef Py_GIL_DISABLED // First check if a change is necessary outside of the stop-the-world pause @@ -360,7 +360,7 @@ sys_trace_line_func( "Missing frame when calling trace function."); return NULL; } - assert(args[0] == (PyObject *)_PyFrame_GetCode(frame->f_frame)); + assert(args[0] == (PyObject *)_PyFrame_GetCode(_PyFrame_Core(frame->f_frame))); return trace_line(tstate, self, frame, line); } @@ -675,11 +675,11 @@ set_monitoring_trace_events(PyInterpreterState *interp) static int maybe_set_opcode_trace(PyThreadState *tstate) { - _PyInterpreterFrame *iframe = tstate->current_frame; - if (iframe == NULL) { + _PyInterpreterFrameCore *iframe = tstate->current_frame; + if (iframe == NULL || _PyFrame_IsExternalFrame(iframe)) { return 0; } - PyFrameObject *frame = iframe->frame_obj; + PyFrameObject *frame = _PyFrame_Full(iframe)->frame_obj; if (frame == NULL || !frame->f_trace_opcodes) { return 0; } diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index f57c33feec2ac2..482ec45b351502 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -764,8 +764,8 @@ static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNKNOWN_OPCODE(TAIL_CALL_PARAMS) int opcode = next_instr->op.code; _PyErr_Format(tstate, PyExc_SystemError, "%U:%d: unknown opcode %d", - _PyFrame_GetCode(frame)->co_filename, - PyUnstable_InterpreterFrame_GetLine(frame), + FRAME_CODE->co_filename, + PyUnstable_InterpreterFrame_GetLine(FRAME_CORE), opcode); JUMP_TO_LABEL(error); } diff --git a/Python/optimizer.c b/Python/optimizer.c index f25242972efeb1..b05c3bbce02e04 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -927,11 +927,11 @@ _PyJit_translate_single_bytecode_to_trace( Py_FatalError("garbled expansion"); } if (uop == _PUSH_FRAME || uop == _RETURN_VALUE || uop == _RETURN_GENERATOR || uop == _YIELD_VALUE) { - PyCodeObject *new_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + PyCodeObject *new_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(_PyFrame_Core(frame)->f_executable); PyFunctionObject *new_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); operand = 0; - if (frame->owner < FRAME_OWNED_BY_INTERPRETER) { + if (_PyFrame_Core(frame)->owner < FRAME_OWNED_BY_INTERPRETER) { // Don't add nested code objects to the dependency. // It causes endless re-traces. if (new_func != NULL && !Py_IsNone((PyObject*)new_func) && !(new_code->co_flags & CO_NESTED)) { @@ -946,7 +946,7 @@ _PyJit_translate_single_bytecode_to_trace( } } ADD_TO_TRACE(uop, oparg, operand, target); - uop_buffer_last(trace)->operand1 = PyStackRef_IsNone(frame->f_executable) ? 2 : ((int)(frame->stackpointer - _PyFrame_Stackbase(frame))); + uop_buffer_last(trace)->operand1 = PyStackRef_IsNone(_PyFrame_Core(frame)->f_executable) ? 2 : ((int)(frame->stackpointer - _PyFrame_Stackbase(frame))); break; } if (uop == _BINARY_OP_INPLACE_ADD_UNICODE) { @@ -1021,7 +1021,7 @@ _PyJit_TryInitializeTracing( if (func == NULL) { return 0; } - PyCodeObject *code = _PyFrame_GetCode(frame); + PyCodeObject *code = _PyFrame_GetCode(_PyFrame_Core(frame)); #ifdef Py_DEBUG char *python_lltrace = Py_GETENV("PYTHON_LLTRACE"); int lltrace = 0; @@ -1052,7 +1052,7 @@ _PyJit_TryInitializeTracing( tracer->initial_state.chain_depth = chain_depth; tracer->prev_state.instr_frame = frame; tracer->prev_state.dependencies_still_valid = true; - tracer->prev_state.instr_code = (PyCodeObject *)Py_NewRef(_PyFrame_GetCode(frame)); + tracer->prev_state.instr_code = (PyCodeObject *)Py_NewRef(_PyFrame_GetCode(_PyFrame_Core(frame))); tracer->prev_state.instr = curr_instr; tracer->prev_state.instr_frame = frame; tracer->prev_state.instr_oparg = oparg; diff --git a/Python/perf_trampoline.c b/Python/perf_trampoline.c index c0dc1f7a49bdca..136eeb22bd0b3c 100644 --- a/Python/perf_trampoline.c +++ b/Python/perf_trampoline.c @@ -428,7 +428,7 @@ py_trampoline_evaluator(PyThreadState *ts, _PyInterpreterFrame *frame, perf_status == PERF_STATUS_NO_INIT) { goto default_eval; } - PyCodeObject *co = _PyFrame_GetCode(frame); + PyCodeObject *co = _PyFrame_GetCode(_PyFrame_Core(frame)); py_trampoline f = NULL; assert(extra_code_index != -1); int ret = _PyCode_GetExtra((PyObject *)co, extra_code_index, (void **)&f); diff --git a/Python/pystate.c b/Python/pystate.c index 19f1245d60a2f8..60a39e7f8bdfc3 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1539,23 +1539,7 @@ init_threadstate(_PyThreadStateImpl *_tstate, // Initialize the embedded base frame - sentinel at the bottom of the frame stack _tstate->base_frame.previous = NULL; _tstate->base_frame.f_executable = PyStackRef_None; - _tstate->base_frame.f_funcobj = PyStackRef_NULL; - _tstate->base_frame.f_globals = NULL; - _tstate->base_frame.f_builtins = NULL; - _tstate->base_frame.f_locals = NULL; - _tstate->base_frame.frame_obj = NULL; - _tstate->base_frame.instr_ptr = NULL; - _tstate->base_frame.stackpointer = _tstate->base_frame.localsplus; - _tstate->base_frame.return_offset = 0; _tstate->base_frame.owner = FRAME_OWNED_BY_INTERPRETER; - _tstate->base_frame.visited = 0; -#ifdef Py_DEBUG - _tstate->base_frame.lltrace = 0; -#endif -#ifdef Py_GIL_DISABLED - _tstate->base_frame.tlbc_index = 0; -#endif - _tstate->base_frame.localsplus[0] = PyStackRef_NULL; // current_frame starts pointing to the base frame tstate->current_frame = &_tstate->base_frame; @@ -2090,7 +2074,7 @@ PyFrameObject* PyThreadState_GetFrame(PyThreadState *tstate) { assert(tstate != NULL); - _PyInterpreterFrame *f = _PyThreadState_GetFrame(tstate); + _PyInterpreterFrameCore *f = _PyThreadState_GetFrame(tstate); if (f == NULL) { return NULL; } @@ -2683,7 +2667,7 @@ _PyThread_CurrentFrames(void) PyInterpreterState *i; for (i = runtime->interpreters.head; i != NULL; i = i->next) { _Py_FOR_EACH_TSTATE_UNLOCKED(i, t) { - _PyInterpreterFrame *frame = t->current_frame; + _PyInterpreterFrameCore *frame = t->current_frame; frame = _PyFrame_GetFirstComplete(frame); if (frame == NULL) { continue; @@ -3072,7 +3056,7 @@ _PyThreadState_PushFrame(PyThreadState *tstate, size_t size) } void -_PyThreadState_PopFrame(PyThreadState *tstate, _PyInterpreterFrame * frame) +_PyThreadState_PopFrame(PyThreadState *tstate, _PyInterpreterFrameCore * frame) { assert(tstate->datastack_chunk); PyObject **base = (PyObject **)frame; diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 94eb3164ecad58..aacb667d679953 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -2102,7 +2102,7 @@ sys__getframe_impl(PyObject *module, int depth) /*[clinic end generated code: output=d438776c04d59804 input=c1be8a6464b11ee5]*/ { PyThreadState *tstate = _PyThreadState_GET(); - _PyInterpreterFrame *frame = tstate->current_frame; + _PyInterpreterFrameCore *frame = tstate->current_frame; if (frame != NULL) { while (depth > 0) { @@ -2605,14 +2605,18 @@ sys__getframemodulename_impl(PyObject *module, int depth) if (PySys_Audit("sys._getframemodulename", "i", depth) < 0) { return NULL; } - _PyInterpreterFrame *f = _PyThreadState_GET()->current_frame; + _PyInterpreterFrameCore *f = _PyThreadState_GET()->current_frame; while (f && (_PyFrame_IsIncomplete(f) || depth-- > 0)) { f = f->previous; } - if (f == NULL || PyStackRef_IsNull(f->f_funcobj)) { + if (f == NULL) { Py_RETURN_NONE; } - PyObject *func = PyStackRef_AsPyObjectBorrow(f->f_funcobj); + _PyInterpreterFrame *frame =_PyFrame_EnsureFrameFullyInitialized(f); + if (PyStackRef_IsNull(frame->f_funcobj)) { + Py_RETURN_NONE; + } + PyObject *func = PyStackRef_AsPyObjectBorrow(frame->f_funcobj); PyObject *r = PyFunction_GetModule(func); if (!r) { PyErr_Clear(); diff --git a/Python/traceback.c b/Python/traceback.c index 74360a1c73c271..7eb1ed14b12cf4 100644 --- a/Python/traceback.c +++ b/Python/traceback.c @@ -156,7 +156,7 @@ tb_get_lineno(PyObject *op) PyTracebackObject *tb = _PyTracebackObject_CAST(op); _PyInterpreterFrame* frame = tb->tb_frame->f_frame; assert(frame != NULL); - return PyCode_Addr2Line(_PyFrame_GetCode(frame), tb->tb_lasti); + return PyCode_Addr2Line(_PyFrame_GetCode(_PyFrame_Core(frame)), tb->tb_lasti); } static PyObject * @@ -312,7 +312,7 @@ _PyTraceBack_FromFrame(PyObject *tb_next, PyFrameObject *frame) { assert(tb_next == NULL || PyTraceBack_Check(tb_next)); assert(frame != NULL); - int addr = _PyInterpreterFrame_LASTI(frame->f_frame) * sizeof(_Py_CODEUNIT); + int addr = _PyInterpreterFrame_LASTI(_PyFrame_Core(frame->f_frame)) * sizeof(_Py_CODEUNIT); return tb_create_raw((PyTracebackObject *)tb_next, frame, addr, -1); } @@ -1036,7 +1036,7 @@ _Py_DumpWideString(int fd, wchar_t *str) Return 0 on success. Return -1 if the frame is invalid. */ static int _Py_NO_SANITIZE_THREAD -dump_frame(int fd, _PyInterpreterFrame *frame) +dump_frame(int fd, _PyInterpreterFrameCore *frame) { if (frame->owner == FRAME_OWNED_BY_INTERPRETER) { /* Ignore trampoline frames and base frame sentinel */ @@ -1123,7 +1123,7 @@ dump_traceback(int fd, PyThreadState *tstate, int write_header) return; } - _PyInterpreterFrame *frame = tstate->current_frame; + _PyInterpreterFrameCore *frame = tstate->current_frame; if (frame == NULL) { PUTS(fd, " \n"); return; @@ -1146,7 +1146,7 @@ dump_traceback(int fd, PyThreadState *tstate, int write_header) } // Read frame->previous early since memory can be freed during // dump_frame() - _PyInterpreterFrame *previous = frame->previous; + _PyInterpreterFrameCore *previous = frame->previous; if (dump_frame(fd, frame) < 0) { PUTS(fd, " \n"); diff --git a/Python/tracemalloc.c b/Python/tracemalloc.c index cdd96925d1f27a..22bc28f1e04dda 100644 --- a/Python/tracemalloc.c +++ b/Python/tracemalloc.c @@ -3,7 +3,7 @@ #include "pycore_gc.h" // PyGC_Head #include "pycore_hashtable.h" // _Py_hashtable_t #include "pycore_initconfig.h" // _PyStatus_NO_MEMORY() -#include "pycore_interpframe.h" // _PyInterpreterFrame +#include "pycore_interpframe.h" // _PyInterpreterFrameCore #include "pycore_lock.h" // PyMutex_LockFlags() #include "pycore_object.h" // _PyType_PreHeaderSize() #include "pycore_pymem.h" // _Py_tracemalloc_config @@ -219,9 +219,8 @@ hashtable_compare_traceback(const void *key1, const void *key2) static void -tracemalloc_get_frame(_PyInterpreterFrame *pyframe, frame_t *frame) +tracemalloc_get_frame(_PyInterpreterFrameCore *pyframe, frame_t *frame) { - assert(PyStackRef_CodeCheck(pyframe->f_executable)); frame->filename = &_Py_STR(anon_unknown); int lineno = PyUnstable_InterpreterFrame_GetLine(pyframe); @@ -301,7 +300,7 @@ traceback_get_frames(traceback_t *traceback) PyThreadState *tstate = _PyThreadState_GET(); assert(tstate != NULL); - _PyInterpreterFrame *pyframe = _PyThreadState_GetFrame(tstate); + _PyInterpreterFrameCore *pyframe = _PyThreadState_GetFrame(tstate); while (pyframe) { if (traceback->nframe < tracemalloc_config.max_nframe) { tracemalloc_get_frame(pyframe, &traceback->frames[traceback->nframe]); diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index 301784f773d31f..d3dfe4cdf73ce3 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -108,6 +108,7 @@ Python/bltinmodule.c - PyZip_Type - Python/context.c - PyContextToken_Type - Python/context.c - PyContextVar_Type - Python/context.c - PyContext_Type - +Python/frame.c - PyUnstable_ExternalExecutable_Type - Python/instruction_sequence.c - _PyInstructionSequence_Type - Python/instrumentation.c - _PyLegacyBranchEventHandler_Type - Python/instrumentation.c - _PyBranchesIterator - diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 2001fb7c37931a..661712fa68c3de 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -634,8 +634,9 @@ def has_error_without_pop(op: parser.CodeDef) -> bool: "_PyCode_CODE", "_PyDictValues_AddToInsertionOrder", "_PyErr_Occurred", - "_PyFrame_GetBytecode", - "_PyFrame_GetCode", + "_PyFrame_Core", + "_PyFrame_GetBytecodeFull", + "_PyFrame_GetCodeFull", "_PyFrame_IsIncomplete", "_PyFrame_PushUnchecked", "_PyFrame_SetStackPointer", diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index c7ff5de681e6fa..e778c9ecd37970 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -140,8 +140,8 @@ def uses_this(inst: Instruction) -> bool: UNKNOWN_OPCODE_HANDLER ="""\ _PyErr_Format(tstate, PyExc_SystemError, "%U:%d: unknown opcode %d", - _PyFrame_GetCode(frame)->co_filename, - PyUnstable_InterpreterFrame_GetLine(frame), + FRAME_CODE->co_filename, + PyUnstable_InterpreterFrame_GetLine(FRAME_CORE), opcode); JUMP_TO_LABEL(error); """ diff --git a/Tools/check-c-api-docs/ignored_c_api.txt b/Tools/check-c-api-docs/ignored_c_api.txt index e0b2670808c79c..eb33c612a08499 100644 --- a/Tools/check-c-api-docs/ignored_c_api.txt +++ b/Tools/check-c-api-docs/ignored_c_api.txt @@ -78,6 +78,7 @@ PyUnstable_PerfTrampoline_SetPersistAfterFork # cpython/pyframe.h PyUnstable_EXECUTABLE_KINDS PyUnstable_EXECUTABLE_KIND_BUILTIN_FUNCTION +PyUnstable_EXECUTABLE_KIND_JIT PyUnstable_EXECUTABLE_KIND_METHOD_DESCRIPTOR PyUnstable_EXECUTABLE_KIND_PY_FUNCTION PyUnstable_EXECUTABLE_KIND_SKIP diff --git a/Tools/gdb/libpython.py b/Tools/gdb/libpython.py index a85195dcd1016a..c533e03ed1fc70 100755 --- a/Tools/gdb/libpython.py +++ b/Tools/gdb/libpython.py @@ -99,7 +99,7 @@ def interp_frame_has_tlbc_index(): Py_TPFLAGS_TYPE_SUBCLASS = (1 << 31) #From pycore_frame.h -FRAME_OWNED_BY_INTERPRETER = 3 +FRAME_OWNED_BY_INTERPRETER = 8 MAX_OUTPUT_LEN=1024 @@ -1081,7 +1081,7 @@ def get_thread_local_frame(): current_frame = thread_state['current_frame'] if int(current_frame) == 0: return None - return PyFramePtr(current_frame) + return PyFramePtr(current_frame.cast(gdb.lookup_type("_PyInterpreterFrame").pointer())) def is_optimized_out(self): return self._gdbval.is_optimized_out @@ -1116,10 +1116,12 @@ def _f_builtins(self): return self._f_special("f_builtins") def _f_code(self): - return self._f_special("f_executable", PyCodeObjectPtr.from_pyobject_ptr) + exc = self._f_executable() + res = PyCodeObjectPtr.from_pyobject_ptr(exc) + return res def _f_executable(self): - return self._f_special("f_executable") + return self.core()['f_executable'] def _f_nlocalsplus(self): return self._f_special("nlocalsplus", int_from_int) @@ -1135,13 +1137,19 @@ def _f_lasti(self): first_instr = self._f_code().field("co_code_adaptive").cast(codeunit_p) return int(instr_ptr - first_instr) + def core(self): + return self._gdbval['core'] + def is_shim(self): - return self._f_special("owner", int) == FRAME_OWNED_BY_INTERPRETER + return int(self.core()['owner']) == FRAME_OWNED_BY_INTERPRETER def previous(self): - if int(self._gdbval['previous']) == 0: + prev = self.core()['previous'] + if int(prev) == 0: return None - return self._f_special("previous", PyFramePtr) + casted = prev.cast(gdb.lookup_type("_PyInterpreterFrame").pointer()) + res = PyFramePtr(casted) + return res def iter_globals(self): '''