gh-126868: Add freelist for compact int objects (GH-126865)

python · Dec 13, 2024 · 5fc6bb2 · 5fc6bb2
1 parent 9b4bbf4
commit 5fc6bb2
Show file tree

Hide file tree

Showing 8 changed files with 102 additions and 55 deletions.
diff --git a/Include/internal/pycore_freelist_state.h b/Include/internal/pycore_freelist_state.h
@@ -14,6 +14,7 @@ extern "C" {
 #  define Py_dicts_MAXFREELIST 80
 #  define Py_dictkeys_MAXFREELIST 80
 #  define Py_floats_MAXFREELIST 100
+#  define Py_ints_MAXFREELIST 100
 #  define Py_slices_MAXFREELIST 1
 #  define Py_contexts_MAXFREELIST 255
 #  define Py_async_gens_MAXFREELIST 80
@@ -35,6 +36,7 @@ struct _Py_freelist {
 
 struct _Py_freelists {
     struct _Py_freelist floats;
+    struct _Py_freelist ints;
     struct _Py_freelist tuples[PyTuple_MAXSAVESIZE];
     struct _Py_freelist lists;
     struct _Py_freelist dicts;

diff --git a/Include/internal/pycore_long.h b/Include/internal/pycore_long.h
@@ -55,6 +55,8 @@ extern void _PyLong_FiniTypes(PyInterpreterState *interp);
 
 /* other API */
 
+PyAPI_FUNC(void) _PyLong_ExactDealloc(PyObject *self);
+
 #define _PyLong_SMALL_INTS _Py_SINGLETON(small_ints)
 
 // _PyLong_GetZero() and _PyLong_GetOne() must always be available

diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-11-16-22-37-46.gh-issue-126868.yOoHSY.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-16-22-37-46.gh-issue-126868.yOoHSY.rst
@@ -0,0 +1 @@
+Increase performance of :class:`int` by adding a freelist for compact ints.
diff --git a/Objects/longobject.c b/Objects/longobject.c
@@ -6,6 +6,7 @@
 #include "pycore_bitutils.h"      // _Py_popcount32()
 #include "pycore_initconfig.h"    // _PyStatus_OK()
 #include "pycore_call.h"          // _PyObject_MakeTpCall
+#include "pycore_freelist.h"      // _Py_FREELIST_FREE, _Py_FREELIST_POP
 #include "pycore_long.h"          // _Py_SmallInts
 #include "pycore_object.h"        // _PyObject_Init()
 #include "pycore_runtime.h"       // _PY_NSMALLPOSINTS
@@ -42,7 +43,7 @@ static inline void
 _Py_DECREF_INT(PyLongObject *op)
 {
     assert(PyLong_CheckExact(op));
-    _Py_DECREF_SPECIALIZED((PyObject *)op, (destructor)PyObject_Free);
+    _Py_DECREF_SPECIALIZED((PyObject *)op, _PyLong_ExactDealloc);
 }
 
 static inline int
@@ -220,15 +221,18 @@ _PyLong_FromMedium(sdigit x)
 {
     assert(!IS_SMALL_INT(x));
     assert(is_medium_int(x));
-    /* We could use a freelist here */
-    PyLongObject *v = PyObject_Malloc(sizeof(PyLongObject));
+
+    PyLongObject *v = (PyLongObject *)_Py_FREELIST_POP(PyLongObject, ints);
     if (v == NULL) {
-        PyErr_NoMemory();
-        return NULL;
+        v = PyObject_Malloc(sizeof(PyLongObject));
+        if (v == NULL) {
+            PyErr_NoMemory();
+            return NULL;
+        }
+        _PyObject_Init((PyObject*)v, &PyLong_Type);
     }
     digit abs_x = x < 0 ? -x : x;
     _PyLong_SetSignAndDigitCount(v, x<0?-1:1, 1);
-    _PyObject_Init((PyObject*)v, &PyLong_Type);
     v->long_value.ob_digit[0] = abs_x;
     return (PyObject*)v;
 }
@@ -3611,24 +3615,60 @@ long_richcompare(PyObject *self, PyObject *other, int op)
     Py_RETURN_RICHCOMPARE(result, 0, op);
 }
 
+static inline int
+compact_int_is_small(PyObject *self)
+{
+    PyLongObject *pylong = (PyLongObject *)self;
+    assert(_PyLong_IsCompact(pylong));
+    stwodigits ival = medium_value(pylong);
+    if (IS_SMALL_INT(ival)) {
+        PyLongObject *small_pylong = (PyLongObject *)get_small_int((sdigit)ival);
+        if (pylong == small_pylong) {
+            return 1;
+        }
+    }
+    return 0;
+}
+
+void
+_PyLong_ExactDealloc(PyObject *self)
+{
+    assert(PyLong_CheckExact(self));
+    if (_PyLong_IsCompact((PyLongObject *)self)) {
+        #ifndef Py_GIL_DISABLED
+        if (compact_int_is_small(self)) {
+            // See PEP 683, section Accidental De-Immortalizing for details
+            _Py_SetImmortal(self);
+            return;
+        }
+        #endif
+        _Py_FREELIST_FREE(ints, self, PyObject_Free);
+        return;
+    }
+    PyObject_Free(self);
+}
+
 static void
 long_dealloc(PyObject *self)
 {
-    /* This should never get called, but we also don't want to SEGV if
-     * we accidentally decref small Ints out of existence. Instead,
-     * since small Ints are immortal, re-set the reference count.
-     */
-    PyLongObject *pylong = (PyLongObject*)self;
-    if (pylong && _PyLong_IsCompact(pylong)) {
-        stwodigits ival = medium_value(pylong);
-        if (IS_SMALL_INT(ival)) {
-            PyLongObject *small_pylong = (PyLongObject *)get_small_int((sdigit)ival);
-            if (pylong == small_pylong) {
-                _Py_SetImmortal(self);
-                return;
-            }
+    assert(self);
+    if (_PyLong_IsCompact((PyLongObject *)self)) {
+        if (compact_int_is_small(self)) {
+            /* This should never get called, but we also don't want to SEGV if
+             * we accidentally decref small Ints out of existence. Instead,
+             * since small Ints are immortal, re-set the reference count.
+             *
+             *  See PEP 683, section Accidental De-Immortalizing for details
+             */
+            _Py_SetImmortal(self);
+            return;
+        }
+        if (PyLong_CheckExact(self)) {
+            _Py_FREELIST_FREE(ints, self, PyObject_Free);
+            return;
         }
     }
+
     Py_TYPE(self)->tp_free(self);
 }
 

diff --git a/Objects/object.c b/Objects/object.c
@@ -936,6 +936,7 @@ _PyObject_ClearFreeLists(struct _Py_freelists *freelists, int is_finalization)
         clear_freelist(&freelists->object_stack_chunks, 1, PyMem_RawFree);
     }
     clear_freelist(&freelists->unicode_writers, is_finalization, PyMem_Free);
+    clear_freelist(&freelists->ints, is_finalization, free_object);
 }
 
 /*

diff --git a/Python/bytecodes.c b/Python/bytecodes.c
@@ -26,6 +26,7 @@
 #include "pycore_pyerrors.h"      // _PyErr_GetRaisedException()
 #include "pycore_pystate.h"       // _PyInterpreterState_GET()
 #include "pycore_range.h"         // _PyRangeIterObject
+#include "pycore_long.h"          // _PyLong_ExactDealloc()
 #include "pycore_setobject.h"     // _PySet_NextEntry()
 #include "pycore_sliceobject.h"   // _PyBuildSlice_ConsumeRefs
 #include "pycore_tuple.h"         // _PyTuple_ITEMS()
@@ -514,8 +515,8 @@ dummy_func(
 
             STAT_INC(BINARY_OP, hit);
             PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o);
-            PyStackRef_CLOSE_SPECIALIZED(right, (destructor)PyObject_Free);
-            PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free);
+            PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc);
+            PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc);
             INPUTS_DEAD();
             ERROR_IF(res_o == NULL, error);
             res = PyStackRef_FromPyObjectSteal(res_o);
@@ -527,8 +528,8 @@ dummy_func(
 
             STAT_INC(BINARY_OP, hit);
             PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o);
-            PyStackRef_CLOSE_SPECIALIZED(right, (destructor)PyObject_Free);
-            PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free);
+            PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc);
+            PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc);
             INPUTS_DEAD();
             ERROR_IF(res_o == NULL, error);
             res = PyStackRef_FromPyObjectSteal(res_o);
@@ -540,8 +541,8 @@ dummy_func(
 
             STAT_INC(BINARY_OP, hit);
             PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o);
-            PyStackRef_CLOSE_SPECIALIZED(right, (destructor)PyObject_Free);
-            PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free);
+            PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc);
+            PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc);
             INPUTS_DEAD();
             ERROR_IF(res_o == NULL, error);
             res = PyStackRef_FromPyObjectSteal(res_o);
@@ -801,7 +802,7 @@ dummy_func(
             assert(res_o != NULL);
             Py_INCREF(res_o);
 #endif
-            PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)PyObject_Free);
+            PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc);
             DEAD(sub_st);
             PyStackRef_CLOSE(list_st);
             res = PyStackRef_FromPyObjectSteal(res_o);
@@ -821,7 +822,7 @@ dummy_func(
             DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c);
             STAT_INC(BINARY_SUBSCR, hit);
             PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c];
-            PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)PyObject_Free);
+            PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc);
             DEAD(sub_st);
             PyStackRef_CLOSE(str_st);
             res = PyStackRef_FromPyObjectSteal(res_o);
@@ -842,7 +843,7 @@ dummy_func(
             PyObject *res_o = PyTuple_GET_ITEM(tuple, index);
             assert(res_o != NULL);
             Py_INCREF(res_o);
-            PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)PyObject_Free);
+            PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc);
             DEAD(sub_st);
             PyStackRef_CLOSE(tuple_st);
             res = PyStackRef_FromPyObjectSteal(res_o);
@@ -959,7 +960,7 @@ dummy_func(
             assert(old_value != NULL);
             UNLOCK_OBJECT(list);  // unlock before decrefs!
             Py_DECREF(old_value);
-            PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)PyObject_Free);
+            PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc);
             DEAD(sub_st);
             PyStackRef_CLOSE(list_st);
         }
@@ -2476,9 +2477,9 @@ dummy_func(
             Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right_o);
             // 2 if <, 4 if >, 8 if ==; this matches the low 4 bits of the oparg
             int sign_ish = COMPARISON_BIT(ileft, iright);
-            PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free);
+            PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc);
             DEAD(left);
-            PyStackRef_CLOSE_SPECIALIZED(right, (destructor)PyObject_Free);
+            PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc);
             DEAD(right);
             res =  (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False;
             // It's always a bool, so we don't care about oparg & 16.

diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h
diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		Increase performance of :class:`int` by adding a freelist for compact ints.
-Original file line number
+Diff line change
@@ Expand Up @@
             clear_freelist(&freelists->object_stack_chunks, 1, PyMem_RawFree);
         }
         clear_freelist(&freelists->unicode_writers, is_finalization, PyMem_Free);
+        clear_freelist(&freelists->ints, is_finalization, free_object);
     }
     /*
@@ Expand Down @@