Reduce the size of the debug stencils to less than half.

For AArch64 linux, reduces the total bytes in the code bodies from 489kb to 218kb.
Reduces the size of the stencils files from 394k lines to 167k lines.

Author: markshannon

Include/cpython/pystats.h

@@ -163,6 +163,7 @@ typedef struct _optimization_stats {
     uint64_t jit_code_size;
     uint64_t jit_trampoline_size;
     uint64_t jit_data_size;
+    uint64_t jit_got_size;
     uint64_t jit_padding_size;
     uint64_t jit_freed_memory_size;
     uint64_t trace_total_memory_hist[_Py_UOP_HIST_SIZE];

Include/internal/pycore_interpframe.h

@@ -230,7 +230,9 @@ _PyFrame_GetLocalsArray(_PyInterpreterFrame *frame)
 static inline _PyStackRef*
 _PyFrame_GetStackPointer(_PyInterpreterFrame *frame)
 {
+#ifndef _Py_JIT
     assert(frame->stackpointer != NULL);
+#endif
     _PyStackRef *sp = frame->stackpointer;
 #ifndef NDEBUG
     frame->stackpointer = NULL;
@@ -241,7 +243,10 @@ _PyFrame_GetStackPointer(_PyInterpreterFrame *frame)
 static inline void
 _PyFrame_SetStackPointer(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer)
 {
+/* Avoid bloating the JIT code */
+#ifndef _Py_JIT
     assert(frame->stackpointer == NULL);
+#endif
     frame->stackpointer = stack_pointer;
 }
 

Include/internal/pycore_jit.h

@@ -31,6 +31,8 @@ _Py_CODEUNIT *_PyJIT_Entry(
 int _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction *trace, size_t length);
 void _PyJIT_Free(_PyExecutorObject *executor);
 PyAPI_FUNC(int) _PyJIT_AddressInJitCode(PyInterpreterState *interp, uintptr_t addr);
+PyAPI_FUNC(void) _Py_jit_assert_within_stack_bounds(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, int lineno);
+PyAPI_FUNC(int) _Py_jit_assertion_failure(int line);
 
 #endif  // _Py_JIT
 

Include/internal/pycore_uop.h

@@ -38,12 +38,7 @@ typedef struct _PyUOpInstruction{
 
 // Fitness is the target length of the trace we translate initially. The uop
 // buffer has a small amount of extra space for entry/loop-closing overhead.
-#if defined(Py_DEBUG) && defined(_Py_JIT)
-    // With asserts, the stencils are a lot larger
-#define FITNESS_INITIAL 1000
-#else
 #define FITNESS_INITIAL 2500
-#endif
 
 #define UOP_TRACE_BUFFER_OVERHEAD 10
 #define UOP_MAX_TRACE_LENGTH (FITNESS_INITIAL + UOP_TRACE_BUFFER_OVERHEAD)

Python/ceval.c

@@ -997,6 +997,14 @@ _Py_assert_within_stack_bounds(
         abort();
     }
 }
+#ifdef _Py_JIT
+void
+_Py_jit_assert_within_stack_bounds(
+    _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, int lineno
+) {
+    _Py_assert_within_stack_bounds(frame, stack_pointer, "executor_cases.c.h", lineno);
+}
+#endif
 #endif
 
 int _Py_CheckRecursiveCallPy(

Python/ceval_macros.h

@@ -268,10 +268,10 @@ GETITEM(PyObject *v, Py_ssize_t i) {
 
 #if defined(Py_DEBUG) && !defined(_Py_JIT)
 // This allows temporary stack "overflows", provided it's all in the cache at any point of time.
-#define WITHIN_STACK_BOUNDS_IGNORING_CACHE() \
-   (frame->owner == FRAME_OWNED_BY_INTERPRETER || (STACK_LEVEL() >= 0 && (STACK_LEVEL()) <= STACK_SIZE()))
+#define ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(F, L) \
+   assert(frame->owner == FRAME_OWNED_BY_INTERPRETER || (STACK_LEVEL() >= 0 && (STACK_LEVEL()) <= STACK_SIZE()))
 #else
-#define WITHIN_STACK_BOUNDS_IGNORING_CACHE WITHIN_STACK_BOUNDS
+#define ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE ASSERT_WITHIN_STACK_BOUNDS
 #endif
 
 /* Data access macros */