bpo-41073: PyType_GetSlot() can now accept static types. (GH-21931)

PyType_GetSlot() can now accept static types.

Co-Authored-By: Petr Viktorin <encukou@gmail.com>

Automerge-Triggered-By: GH:encukou

Doc/c-api/type.rst

@@ -105,10 +105,12 @@ Type Objects
 
    See :c:member:`PyType_Slot.slot` for possible values of the *slot* argument.
 
-   An exception is raised if *type* is not a heap type.
-
    .. versionadded:: 3.4
 
+   .. versionchanged:: 3.10
+      :c:func:`PyType_GetSlot` can now accept all types.
+      Previously, it was limited to heap types.
+
 .. c:function:: PyObject* PyType_GetModule(PyTypeObject *type)
 
    Return the module object associated with the given type when the type was

Doc/whatsnew/3.10.rst

@@ -436,6 +436,9 @@ New Features
   slot.
   (Contributed by Hai Shi in :issue:`41832`.)
 
+* The :c:func:`PyType_GetSlot` function can accept static types.
+  (Contributed by Hai Shi and Petr Viktorin in :issue:`41073`.)
+
 
 Porting to Python 3.10
 ----------------------

Misc/NEWS.d/next/C API/2020-07-08-21-01-49.bpo-41073.VqQZON.rst

@@ -0,0 +1 @@
+:c:func:`PyType_GetSlot()` can now accept static types.

Modules/_testcapimodule.c

@@ -1018,6 +1018,62 @@ test_buildvalue_N(PyObject *self, PyObject *Py_UNUSED(ignored))
 }
 
 
+static PyObject *
+test_get_statictype_slots(PyObject *self, PyObject *Py_UNUSED(ignored))
+{
+    newfunc tp_new = PyType_GetSlot(&PyLong_Type, Py_tp_new);
+    if (PyLong_Type.tp_new != tp_new) {
+        PyErr_SetString(PyExc_AssertionError, "mismatch: tp_new of long");
+        return NULL;
+    }
+
+    reprfunc tp_repr = PyType_GetSlot(&PyLong_Type, Py_tp_repr);
+    if (PyLong_Type.tp_repr != tp_repr) {
+        PyErr_SetString(PyExc_AssertionError, "mismatch: tp_repr of long");
+        return NULL;
+    }
+
+    ternaryfunc tp_call = PyType_GetSlot(&PyLong_Type, Py_tp_call);
+    if (tp_call != NULL) {
+        PyErr_SetString(PyExc_AssertionError, "mismatch: tp_call of long");
+        return NULL;
+    }
+
+    binaryfunc nb_add = PyType_GetSlot(&PyLong_Type, Py_nb_add);
+    if (PyLong_Type.tp_as_number->nb_add != nb_add) {
+        PyErr_SetString(PyExc_AssertionError, "mismatch: nb_add of long");
+        return NULL;
+    }
+
+    lenfunc mp_length = PyType_GetSlot(&PyLong_Type, Py_mp_length);
+    if (mp_length != NULL) {
+        PyErr_SetString(PyExc_AssertionError, "mismatch: mp_length of long");
+        return NULL;
+    }
+
+    void *over_value = PyType_GetSlot(&PyLong_Type, Py_bf_releasebuffer + 1);
+    if (over_value != NULL) {
+        PyErr_SetString(PyExc_AssertionError, "mismatch: max+1 of long");
+        return NULL;
+    }
+
+    tp_new = PyType_GetSlot(&PyLong_Type, 0);
+    if (tp_new != NULL) {
+        PyErr_SetString(PyExc_AssertionError, "mismatch: slot 0 of long");
+        return NULL;
+    }
+    if (PyErr_ExceptionMatches(PyExc_SystemError)) {
+        // This is the right exception
+        PyErr_Clear();
+    }
+    else {
+        return NULL;
+    }
+
+    Py_RETURN_NONE;
+}
+
+
 static PyObject *
 get_args(PyObject *self, PyObject *args)
 {
@@ -5627,8 +5683,10 @@ static PyMethodDef TestMethods[] = {
     {"PyBuffer_SizeFromFormat",  test_PyBuffer_SizeFromFormat, METH_VARARGS},
     {"test_buildvalue_N",       test_buildvalue_N,               METH_NOARGS},
     {"test_buildvalue_issue38913", test_buildvalue_issue38913,   METH_NOARGS},
-    {"get_args", get_args, METH_VARARGS},
-    {"get_kwargs", (PyCFunction)(void(*)(void))get_kwargs, METH_VARARGS|METH_KEYWORDS},
+    {"get_args",                get_args,                        METH_VARARGS},
+    {"test_get_statictype_slots", test_get_statictype_slots,     METH_NOARGS},
+    {"get_kwargs", (PyCFunction)(void(*)(void))get_kwargs,
+      METH_VARARGS|METH_KEYWORDS},
     {"getargs_tuple",           getargs_tuple,                   METH_VARARGS},
     {"getargs_keywords", (PyCFunction)(void(*)(void))getargs_keywords,
       METH_VARARGS|METH_KEYWORDS},

Objects/typeobject.c

@@ -56,6 +56,11 @@ static struct method_cache_entry method_cache[1 << MCACHE_SIZE_EXP];
 static unsigned int next_version_tag = 0;
 #endif
 
+typedef struct PySlot_Offset {
+    short subslot_offset;
+    short slot_offset;
+} PySlot_Offset;
+
 #define MCACHE_STATS 0
 
 #if MCACHE_STATS
@@ -2870,8 +2875,18 @@ error:
     return NULL;
 }
 
-static const short slotoffsets[] = {
-    -1, /* invalid slot */
+/* An array of type slot offsets corresponding to Py_tp_* constants,
+  * for use in e.g. PyType_Spec and PyType_GetSlot.
+  * Each entry has two offsets: "slot_offset" and "subslot_offset".
+  * If is subslot_offset is -1, slot_offset is an offset within the
+  * PyTypeObject struct.
+  * Otherwise slot_offset is an offset to a pointer to a sub-slots struct
+  * (such as "tp_as_number"), and subslot_offset is the offset within
+  * that struct.
+  * The actual table is generated by a script.
+  */
+static const PySlot_Offset pyslot_offsets[] = {
+    {0, 0},
 #include "typeslots.inc"
 };
 
@@ -2892,6 +2907,7 @@ PyType_FromModuleAndSpec(PyObject *module, PyType_Spec *spec, PyObject *bases)
     const PyType_Slot *slot;
     Py_ssize_t nmembers, weaklistoffset, dictoffset, vectorcalloffset;
     char *res_start;
+    short slot_offset, subslot_offset;
 
     nmembers = weaklistoffset = dictoffset = vectorcalloffset = 0;
     for (slot = spec->slots; slot->slot; slot++) {
@@ -3001,7 +3017,7 @@ PyType_FromModuleAndSpec(PyObject *module, PyType_Spec *spec, PyObject *bases)
 
     for (slot = spec->slots; slot->slot; slot++) {
         if (slot->slot < 0
-            || (size_t)slot->slot >= Py_ARRAY_LENGTH(slotoffsets)) {
+            || (size_t)slot->slot >= Py_ARRAY_LENGTH(pyslot_offsets)) {
             PyErr_SetString(PyExc_RuntimeError, "invalid slot offset");
             goto fail;
         }
@@ -3034,7 +3050,15 @@ PyType_FromModuleAndSpec(PyObject *module, PyType_Spec *spec, PyObject *bases)
         }
         else {
             /* Copy other slots directly */
-            *(void**)(res_start + slotoffsets[slot->slot]) = slot->pfunc;
+            PySlot_Offset slotoffsets = pyslot_offsets[slot->slot];
+            slot_offset = slotoffsets.slot_offset;
+            if (slotoffsets.subslot_offset == -1) {
+                *(void**)((char*)res_start + slot_offset) = slot->pfunc;
+            } else {
+                void *parent_slot = *(void**)((char*)res_start + slot_offset);
+                subslot_offset = slotoffsets.subslot_offset;
+                *(void**)((char*)parent_slot + subslot_offset) = slot->pfunc;
+            }
         }
     }
     if (type->tp_dealloc == NULL) {
@@ -3117,15 +3141,23 @@ PyType_FromSpec(PyType_Spec *spec)
 void *
 PyType_GetSlot(PyTypeObject *type, int slot)
 {
-    if (!_PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE) || slot < 0) {
+    void *parent_slot;
+    int slots_len = Py_ARRAY_LENGTH(pyslot_offsets);
+
+    if (slot <= 0 || slot >= slots_len) {
         PyErr_BadInternalCall();
         return NULL;
     }
-    if ((size_t)slot >= Py_ARRAY_LENGTH(slotoffsets)) {
-        /* Extension module requesting slot from a future version */
+
+    parent_slot = *(void**)((char*)type + pyslot_offsets[slot].slot_offset);
+    if (parent_slot == NULL) {
         return NULL;
     }
-    return  *(void**)(((char*)type) + slotoffsets[slot]);
+    /* Return slot directly if we have no sub slot. */
+    if (pyslot_offsets[slot].subslot_offset == -1) {
+        return parent_slot;
+    }
+    return *(void**)((char*)parent_slot + pyslot_offsets[slot].subslot_offset);
 }
 
 PyObject *

Objects/typeslots.inc

@@ -1,82 +1,82 @@
 /* Generated by typeslots.py */
-offsetof(PyHeapTypeObject, as_buffer.bf_getbuffer),
-offsetof(PyHeapTypeObject, as_buffer.bf_releasebuffer),
-offsetof(PyHeapTypeObject, as_mapping.mp_ass_subscript),
-offsetof(PyHeapTypeObject, as_mapping.mp_length),
-offsetof(PyHeapTypeObject, as_mapping.mp_subscript),
-offsetof(PyHeapTypeObject, as_number.nb_absolute),
-offsetof(PyHeapTypeObject, as_number.nb_add),
-offsetof(PyHeapTypeObject, as_number.nb_and),
-offsetof(PyHeapTypeObject, as_number.nb_bool),
-offsetof(PyHeapTypeObject, as_number.nb_divmod),
-offsetof(PyHeapTypeObject, as_number.nb_float),
-offsetof(PyHeapTypeObject, as_number.nb_floor_divide),
-offsetof(PyHeapTypeObject, as_number.nb_index),
-offsetof(PyHeapTypeObject, as_number.nb_inplace_add),
-offsetof(PyHeapTypeObject, as_number.nb_inplace_and),
-offsetof(PyHeapTypeObject, as_number.nb_inplace_floor_divide),
-offsetof(PyHeapTypeObject, as_number.nb_inplace_lshift),
-offsetof(PyHeapTypeObject, as_number.nb_inplace_multiply),
-offsetof(PyHeapTypeObject, as_number.nb_inplace_or),
-offsetof(PyHeapTypeObject, as_number.nb_inplace_power),
-offsetof(PyHeapTypeObject, as_number.nb_inplace_remainder),
-offsetof(PyHeapTypeObject, as_number.nb_inplace_rshift),
-offsetof(PyHeapTypeObject, as_number.nb_inplace_subtract),
-offsetof(PyHeapTypeObject, as_number.nb_inplace_true_divide),
-offsetof(PyHeapTypeObject, as_number.nb_inplace_xor),
-offsetof(PyHeapTypeObject, as_number.nb_int),
-offsetof(PyHeapTypeObject, as_number.nb_invert),
-offsetof(PyHeapTypeObject, as_number.nb_lshift),
-offsetof(PyHeapTypeObject, as_number.nb_multiply),
-offsetof(PyHeapTypeObject, as_number.nb_negative),
-offsetof(PyHeapTypeObject, as_number.nb_or),
-offsetof(PyHeapTypeObject, as_number.nb_positive),
-offsetof(PyHeapTypeObject, as_number.nb_power),
-offsetof(PyHeapTypeObject, as_number.nb_remainder),
-offsetof(PyHeapTypeObject, as_number.nb_rshift),
-offsetof(PyHeapTypeObject, as_number.nb_subtract),
-offsetof(PyHeapTypeObject, as_number.nb_true_divide),
-offsetof(PyHeapTypeObject, as_number.nb_xor),
-offsetof(PyHeapTypeObject, as_sequence.sq_ass_item),
-offsetof(PyHeapTypeObject, as_sequence.sq_concat),
-offsetof(PyHeapTypeObject, as_sequence.sq_contains),
-offsetof(PyHeapTypeObject, as_sequence.sq_inplace_concat),
-offsetof(PyHeapTypeObject, as_sequence.sq_inplace_repeat),
-offsetof(PyHeapTypeObject, as_sequence.sq_item),
-offsetof(PyHeapTypeObject, as_sequence.sq_length),
-offsetof(PyHeapTypeObject, as_sequence.sq_repeat),
-offsetof(PyHeapTypeObject, ht_type.tp_alloc),
-offsetof(PyHeapTypeObject, ht_type.tp_base),
-offsetof(PyHeapTypeObject, ht_type.tp_bases),
-offsetof(PyHeapTypeObject, ht_type.tp_call),
-offsetof(PyHeapTypeObject, ht_type.tp_clear),
-offsetof(PyHeapTypeObject, ht_type.tp_dealloc),
-offsetof(PyHeapTypeObject, ht_type.tp_del),
-offsetof(PyHeapTypeObject, ht_type.tp_descr_get),
-offsetof(PyHeapTypeObject, ht_type.tp_descr_set),
-offsetof(PyHeapTypeObject, ht_type.tp_doc),
-offsetof(PyHeapTypeObject, ht_type.tp_getattr),
-offsetof(PyHeapTypeObject, ht_type.tp_getattro),
-offsetof(PyHeapTypeObject, ht_type.tp_hash),
-offsetof(PyHeapTypeObject, ht_type.tp_init),
-offsetof(PyHeapTypeObject, ht_type.tp_is_gc),
-offsetof(PyHeapTypeObject, ht_type.tp_iter),
-offsetof(PyHeapTypeObject, ht_type.tp_iternext),
-offsetof(PyHeapTypeObject, ht_type.tp_methods),
-offsetof(PyHeapTypeObject, ht_type.tp_new),
-offsetof(PyHeapTypeObject, ht_type.tp_repr),
-offsetof(PyHeapTypeObject, ht_type.tp_richcompare),
-offsetof(PyHeapTypeObject, ht_type.tp_setattr),
-offsetof(PyHeapTypeObject, ht_type.tp_setattro),
-offsetof(PyHeapTypeObject, ht_type.tp_str),
-offsetof(PyHeapTypeObject, ht_type.tp_traverse),
-offsetof(PyHeapTypeObject, ht_type.tp_members),
-offsetof(PyHeapTypeObject, ht_type.tp_getset),
-offsetof(PyHeapTypeObject, ht_type.tp_free),
-offsetof(PyHeapTypeObject, as_number.nb_matrix_multiply),
-offsetof(PyHeapTypeObject, as_number.nb_inplace_matrix_multiply),
-offsetof(PyHeapTypeObject, as_async.am_await),
-offsetof(PyHeapTypeObject, as_async.am_aiter),
-offsetof(PyHeapTypeObject, as_async.am_anext),
-offsetof(PyHeapTypeObject, ht_type.tp_finalize),
-offsetof(PyHeapTypeObject, as_async.am_send),
+{offsetof(PyBufferProcs, bf_getbuffer), offsetof(PyTypeObject, tp_as_buffer)},
+{offsetof(PyBufferProcs, bf_releasebuffer), offsetof(PyTypeObject, tp_as_buffer)},
+{offsetof(PyMappingMethods, mp_ass_subscript), offsetof(PyTypeObject, tp_as_mapping)},
+{offsetof(PyMappingMethods, mp_length), offsetof(PyTypeObject, tp_as_mapping)},
+{offsetof(PyMappingMethods, mp_subscript), offsetof(PyTypeObject, tp_as_mapping)},
+{offsetof(PyNumberMethods, nb_absolute), offsetof(PyTypeObject, tp_as_number)},
+{offsetof(PyNumberMethods, nb_add), offsetof(PyTypeObject, tp_as_number)},
+{offsetof(PyNumberMethods, nb_and), offsetof(PyTypeObject, tp_as_number)},
+{offsetof(PyNumberMethods, nb_bool), offsetof(PyTypeObject, tp_as_number)},
+{offsetof(PyNumberMethods, nb_divmod), offsetof(PyTypeObject, tp_as_number)},
+{offsetof(PyNumberMethods, nb_float), offsetof(PyTypeObject, tp_as_number)},
+{offsetof(PyNumberMethods, nb_floor_divide), offsetof(PyTypeObject, tp_as_number)},
+{offsetof(PyNumberMethods, nb_index), offsetof(PyTypeObject, tp_as_number)},
+{offsetof(PyNumberMethods, nb_inplace_add), offsetof(PyTypeObject, tp_as_number)},
+{offsetof(PyNumberMethods, nb_inplace_and), offsetof(PyTypeObject, tp_as_number)},
+{offsetof(PyNumberMethods, nb_inplace_floor_divide), offsetof(PyTypeObject, tp_as_number)},
+{offsetof(PyNumberMethods, nb_inplace_lshift), offsetof(PyTypeObject, tp_as_number)},
+{offsetof(PyNumberMethods, nb_inplace_multiply), offsetof(PyTypeObject, tp_as_number)},
+{offsetof(PyNumberMethods, nb_inplace_or), offsetof(PyTypeObject, tp_as_number)},
+{offsetof(PyNumberMethods, nb_inplace_power), offsetof(PyTypeObject, tp_as_number)},
+{offsetof(PyNumberMethods, nb_inplace_remainder), offsetof(PyTypeObject, tp_as_number)},
+{offsetof(PyNumberMethods, nb_inplace_rshift), offsetof(PyTypeObject, tp_as_number)},
+{offsetof(PyNumberMethods, nb_inplace_subtract), offsetof(PyTypeObject, tp_as_number)},
+{offsetof(PyNumberMethods, nb_inplace_true_divide), offsetof(PyTypeObject, tp_as_number)},
+{offsetof(PyNumberMethods, nb_inplace_xor), offsetof(PyTypeObject, tp_as_number)},
+{offsetof(PyNumberMethods, nb_int), offsetof(PyTypeObject, tp_as_number)},
+{offsetof(PyNumberMethods, nb_invert), offsetof(PyTypeObject, tp_as_number)},
+{offsetof(PyNumberMethods, nb_lshift), offsetof(PyTypeObject, tp_as_number)},
+{offsetof(PyNumberMethods, nb_multiply), offsetof(PyTypeObject, tp_as_number)},
+{offsetof(PyNumberMethods, nb_negative), offsetof(PyTypeObject, tp_as_number)},
+{offsetof(PyNumberMethods, nb_or), offsetof(PyTypeObject, tp_as_number)},
+{offsetof(PyNumberMethods, nb_positive), offsetof(PyTypeObject, tp_as_number)},
+{offsetof(PyNumberMethods, nb_power), offsetof(PyTypeObject, tp_as_number)},
+{offsetof(PyNumberMethods, nb_remainder), offsetof(PyTypeObject, tp_as_number)},
+{offsetof(PyNumberMethods, nb_rshift), offsetof(PyTypeObject, tp_as_number)},
+{offsetof(PyNumberMethods, nb_subtract), offsetof(PyTypeObject, tp_as_number)},
+{offsetof(PyNumberMethods, nb_true_divide), offsetof(PyTypeObject, tp_as_number)},
+{offsetof(PyNumberMethods, nb_xor), offsetof(PyTypeObject, tp_as_number)},
+{offsetof(PySequenceMethods, sq_ass_item), offsetof(PyTypeObject, tp_as_sequence)},
+{offsetof(PySequenceMethods, sq_concat), offsetof(PyTypeObject, tp_as_sequence)},
+{offsetof(PySequenceMethods, sq_contains), offsetof(PyTypeObject, tp_as_sequence)},
+{offsetof(PySequenceMethods, sq_inplace_concat), offsetof(PyTypeObject, tp_as_sequence)},
+{offsetof(PySequenceMethods, sq_inplace_repeat), offsetof(PyTypeObject, tp_as_sequence)},
+{offsetof(PySequenceMethods, sq_item), offsetof(PyTypeObject, tp_as_sequence)},
+{offsetof(PySequenceMethods, sq_length), offsetof(PyTypeObject, tp_as_sequence)},
+{offsetof(PySequenceMethods, sq_repeat), offsetof(PyTypeObject, tp_as_sequence)},
+{-1, offsetof(PyTypeObject, tp_alloc)},
+{-1, offsetof(PyTypeObject, tp_base)},
+{-1, offsetof(PyTypeObject, tp_bases)},
+{-1, offsetof(PyTypeObject, tp_call)},
+{-1, offsetof(PyTypeObject, tp_clear)},
+{-1, offsetof(PyTypeObject, tp_dealloc)},
+{-1, offsetof(PyTypeObject, tp_del)},
+{-1, offsetof(PyTypeObject, tp_descr_get)},
+{-1, offsetof(PyTypeObject, tp_descr_set)},
+{-1, offsetof(PyTypeObject, tp_doc)},
+{-1, offsetof(PyTypeObject, tp_getattr)},
+{-1, offsetof(PyTypeObject, tp_getattro)},
+{-1, offsetof(PyTypeObject, tp_hash)},
+{-1, offsetof(PyTypeObject, tp_init)},
+{-1, offsetof(PyTypeObject, tp_is_gc)},
+{-1, offsetof(PyTypeObject, tp_iter)},
+{-1, offsetof(PyTypeObject, tp_iternext)},
+{-1, offsetof(PyTypeObject, tp_methods)},
+{-1, offsetof(PyTypeObject, tp_new)},
+{-1, offsetof(PyTypeObject, tp_repr)},
+{-1, offsetof(PyTypeObject, tp_richcompare)},
+{-1, offsetof(PyTypeObject, tp_setattr)},
+{-1, offsetof(PyTypeObject, tp_setattro)},
+{-1, offsetof(PyTypeObject, tp_str)},
+{-1, offsetof(PyTypeObject, tp_traverse)},
+{-1, offsetof(PyTypeObject, tp_members)},
+{-1, offsetof(PyTypeObject, tp_getset)},
+{-1, offsetof(PyTypeObject, tp_free)},
+{offsetof(PyNumberMethods, nb_matrix_multiply), offsetof(PyTypeObject, tp_as_number)},
+{offsetof(PyNumberMethods, nb_inplace_matrix_multiply), offsetof(PyTypeObject, tp_as_number)},
+{offsetof(PyAsyncMethods, am_await), offsetof(PyTypeObject, tp_as_async)},
+{offsetof(PyAsyncMethods, am_aiter), offsetof(PyTypeObject, tp_as_async)},
+{offsetof(PyAsyncMethods, am_anext), offsetof(PyTypeObject, tp_as_async)},
+{-1, offsetof(PyTypeObject, tp_finalize)},
+{offsetof(PyAsyncMethods, am_send), offsetof(PyTypeObject, tp_as_async)},

Objects/typeslots.py

@@ -3,6 +3,7 @@
 
 import sys, re
 
+
 def generate_typeslots(out=sys.stdout):
     out.write("/* Generated by typeslots.py */\n")
     res = {}
@@ -10,27 +11,34 @@ def generate_typeslots(out=sys.stdout):
         m = re.match("#define Py_([a-z_]+) ([0-9]+)", line)
         if not m:
             continue
+
         member = m.group(1)
         if member.startswith("tp_"):
-            member = "ht_type."+member
+            member = f'{{-1, offsetof(PyTypeObject, {member})}}'
         elif member.startswith("am_"):
-            member = "as_async."+member
+            member = (f'{{offsetof(PyAsyncMethods, {member}),'+
+                      ' offsetof(PyTypeObject, tp_as_async)}')
         elif member.startswith("nb_"):
-            member = "as_number."+member
+            member = (f'{{offsetof(PyNumberMethods, {member}),'+
+                      ' offsetof(PyTypeObject, tp_as_number)}')
         elif member.startswith("mp_"):
-            member = "as_mapping."+member
+            member = (f'{{offsetof(PyMappingMethods, {member}),'+
+                      ' offsetof(PyTypeObject, tp_as_mapping)}')
         elif member.startswith("sq_"):
-            member = "as_sequence."+member
+            member = (f'{{offsetof(PySequenceMethods, {member}),'+
+                      ' offsetof(PyTypeObject, tp_as_sequence)}')
         elif member.startswith("bf_"):
-            member = "as_buffer."+member
+            member = (f'{{offsetof(PyBufferProcs, {member}),'+
+                      ' offsetof(PyTypeObject, tp_as_buffer)}')
         res[int(m.group(2))] = member
 
     M = max(res.keys())+1
     for i in range(1,M):
         if i in res:
-            out.write("offsetof(PyHeapTypeObject, %s),\n" % res[i])
+            out.write("%s,\n" % res[i])
         else:
-            out.write("0,\n")
+            out.write("{0, 0},\n")
+
 
 def main():
     if len(sys.argv) == 2: