Mirrors
/
cpython
mirror of https://github.com/python/cpython
3
0
Fork 0
Code Releases Activity
Browse Source

bpo-34784: Fix PyStructSequence_NewType with heap-allocated StructSequence (GH-9665)

pull/10516/head
Eddie Elizondo 7 years ago
committed by Petr Viktorin
parent
1a6be91e6f
commit
474eedfb3d
5 changed files with 239 additions and 137 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 2
      Misc/NEWS.d/next/Core and Builtins/2018-10-02-09-10-47.bpo-34784.07hdgD.rst
  2. 27
      Modules/_testcapimodule.c
  3. 94
      Modules/posixmodule.c
  4. 225
      Objects/structseq.c
  5. 28
      Objects/typeobject.c

2
Misc/NEWS.d/next/Core and Builtins/2018-10-02-09-10-47.bpo-34784.07hdgD.rst
View File

@ -0,0 +1,2 @@
Fix the implementation of PyStructSequence_NewType in order to create heap
allocated StructSequences.

27
Modules/_testcapimodule.c
View File

@ -3313,6 +3313,31 @@ test_decref_doesnt_leak(PyObject *ob, PyObject *Py_UNUSED(ignored))
Py_RETURN_NONE;
}
static PyObject *
test_structseq_newtype_doesnt_leak(PyObject *Py_UNUSED(self),
PyObject *Py_UNUSED(args))
{
PyStructSequence_Desc descr;
PyStructSequence_Field descr_fields[3];
descr_fields[0] = (PyStructSequence_Field){"foo", "foo value"};
descr_fields[1] = (PyStructSequence_Field){NULL, "some hidden value"};
descr_fields[2] = (PyStructSequence_Field){0, NULL};
descr.name = "_testcapi.test_descr";
descr.doc = "This is used to test for memory leaks in NewType";
descr.fields = descr_fields;
descr.n_in_sequence = 1;
PyTypeObject* structseq_type = PyStructSequence_NewType(&descr);
assert(structseq_type != NULL);
assert(PyType_Check(structseq_type));
assert(PyType_FastSubclass(structseq_type, Py_TPFLAGS_TUPLE_SUBCLASS));
Py_DECREF(structseq_type);
Py_RETURN_NONE;
}
static PyObject *
test_incref_decref_API(PyObject *ob, PyObject *Py_UNUSED(ignored))
{
@ -4721,6 +4746,8 @@ static PyMethodDef TestMethods[] = {
{"test_incref_doesnt_leak", test_incref_doesnt_leak, METH_NOARGS},
{"test_xdecref_doesnt_leak",test_xdecref_doesnt_leak, METH_NOARGS},
{"test_decref_doesnt_leak", test_decref_doesnt_leak, METH_NOARGS},
{"test_structseq_newtype_doesnt_leak",
test_structseq_newtype_doesnt_leak, METH_NOARGS},
{"test_incref_decref_API", test_incref_decref_API, METH_NOARGS},
{"test_long_and_overflow", test_long_and_overflow, METH_NOARGS},
{"test_long_as_double", test_long_as_double, METH_NOARGS},

94
Modules/posixmodule.c
View File

@ -1948,14 +1948,14 @@ static PyStructSequence_Desc waitid_result_desc = {
waitid_result_fields,
5
};
static PyTypeObject WaitidResultType;
static PyTypeObject* WaitidResultType;
#endif
static int initialized;
static PyTypeObject StatResultType;
static PyTypeObject StatVFSResultType;
static PyTypeObject* StatResultType;
static PyTypeObject* StatVFSResultType;
#if defined(HAVE_SCHED_SETPARAM) || defined(HAVE_SCHED_SETSCHEDULER) || defined(POSIX_SPAWN_SETSCHEDULER) || defined(POSIX_SPAWN_SETSCHEDPARAM)
static PyTypeObject SchedParamType;
static PyTypeObject* SchedParamType;
#endif
static newfunc structseq_new;
@ -2029,7 +2029,7 @@ static PyObject*
_pystat_fromstructstat(STRUCT_STAT *st)
{
unsigned long ansec, mnsec, cnsec;
PyObject *v = PyStructSequence_New(&StatResultType);
PyObject *v = PyStructSequence_New(StatResultType);
if (v == NULL)
return NULL;
@ -4407,7 +4407,7 @@ static PyStructSequence_Desc uname_result_desc = {
5
};
static PyTypeObject UnameResultType;
static PyTypeObject* UnameResultType;
#ifdef HAVE_UNAME
@ -4435,7 +4435,7 @@ os_uname_impl(PyObject *module)
if (res < 0)
return posix_error();
value = PyStructSequence_New(&UnameResultType);
value = PyStructSequence_New(UnameResultType);
if (value == NULL)
return NULL;
@ -5941,7 +5941,7 @@ os_sched_getscheduler_impl(PyObject *module, pid_t pid)
#if defined(HAVE_SCHED_SETPARAM) || defined(HAVE_SCHED_SETSCHEDULER) || defined(POSIX_SPAWN_SETSCHEDULER) || defined(POSIX_SPAWN_SETSCHEDPARAM)
/*[clinic input]
class os.sched_param "PyObject *" "&SchedParamType"
class os.sched_param "PyObject *" "SchedParamType"
@classmethod
os.sched_param.__new__
@ -5954,7 +5954,7 @@ Current has only one field: sched_priority");
static PyObject *
os_sched_param_impl(PyTypeObject *type, PyObject *sched_priority)
/*[clinic end generated code: output=48f4067d60f48c13 input=73a4c22f7071fc62]*/
/*[clinic end generated code: output=48f4067d60f48c13 input=ab4de35a9a7811f2]*/
{
PyObject *res;
@ -5986,7 +5986,7 @@ convert_sched_param(PyObject *param, struct sched_param *res)
{
long priority;
if (Py_TYPE(param) != &SchedParamType) {
if (Py_TYPE(param) != SchedParamType) {
PyErr_SetString(PyExc_TypeError, "must have a sched_param object");
return 0;
}
@ -6057,7 +6057,7 @@ os_sched_getparam_impl(PyObject *module, pid_t pid)
if (sched_getparam(pid, &param))
return posix_error();
result = PyStructSequence_New(&SchedParamType);
result = PyStructSequence_New(SchedParamType);
if (!result)
return NULL;
priority = PyLong_FromLong(param.sched_priority);
@ -7422,7 +7422,7 @@ os_waitid_impl(PyObject *module, idtype_t idtype, id_t id, int options)
if (si.si_pid == 0)
Py_RETURN_NONE;
result = PyStructSequence_New(&WaitidResultType);
result = PyStructSequence_New(WaitidResultType);
if (!result)
return NULL;
@ -7857,7 +7857,7 @@ static PyStructSequence_Desc times_result_desc = {
5
};
static PyTypeObject TimesResultType;
static PyTypeObject* TimesResultType;
#ifdef MS_WINDOWS
#define HAVE_TIMES /* mandatory, for the method table */
@ -7870,7 +7870,7 @@ build_times_result(double user, double system,
double children_user, double children_system,
double elapsed)
{
PyObject *value = PyStructSequence_New(&TimesResultType);
PyObject *value = PyStructSequence_New(TimesResultType);
if (value == NULL)
return NULL;
@ -9950,7 +9950,7 @@ os_WSTOPSIG_impl(PyObject *module, int status)
static PyObject*
_pystatvfs_fromstructstatvfs(struct statvfs st) {
PyObject *v = PyStructSequence_New(&StatVFSResultType);
PyObject *v = PyStructSequence_New(StatVFSResultType);
if (v == NULL)
return NULL;
@ -11703,7 +11703,7 @@ os_urandom_impl(PyObject *module, Py_ssize_t size)
/* Terminal size querying */
static PyTypeObject TerminalSizeType;
static PyTypeObject* TerminalSizeType;
PyDoc_STRVAR(TerminalSize_docstring,
"A tuple of (columns, lines) for holding terminal window size");
@ -11795,7 +11795,7 @@ get_terminal_size(PyObject *self, PyObject *args)
}
#endif /* TERMSIZE_USE_CONIO */
termsize = PyStructSequence_New(&TerminalSizeType);
termsize = PyStructSequence_New(TerminalSizeType);
if (termsize == NULL)
return NULL;
PyStructSequence_SET_ITEM(termsize, 0, PyLong_FromLong(columns));
@ -13912,23 +13912,28 @@ INITFUNC(void)
if (!initialized) {
#if defined(HAVE_WAITID) && !defined(__APPLE__)
waitid_result_desc.name = MODNAME ".waitid_result";
if (PyStructSequence_InitType2(&WaitidResultType, &waitid_result_desc) < 0)
WaitidResultType = PyStructSequence_NewType(&waitid_result_desc);
if (WaitidResultType == NULL) {
return NULL;
}
#endif
stat_result_desc.name = "os.stat_result"; /* see issue #19209 */
stat_result_desc.fields[7].name = PyStructSequence_UnnamedField;
stat_result_desc.fields[8].name = PyStructSequence_UnnamedField;
stat_result_desc.fields[9].name = PyStructSequence_UnnamedField;
if (PyStructSequence_InitType2(&StatResultType, &stat_result_desc) < 0)
StatResultType = PyStructSequence_NewType(&stat_result_desc);
if (StatResultType == NULL) {
return NULL;
structseq_new = StatResultType.tp_new;
StatResultType.tp_new = statresult_new;
}
structseq_new = StatResultType->tp_new;
StatResultType->tp_new = statresult_new;
statvfs_result_desc.name = "os.statvfs_result"; /* see issue #19209 */
if (PyStructSequence_InitType2(&StatVFSResultType,
&statvfs_result_desc) < 0)
StatVFSResultType = PyStructSequence_NewType(&statvfs_result_desc);
if (StatVFSResultType == NULL) {
return NULL;
}
#ifdef NEED_TICKS_PER_SECOND
# if defined(HAVE_SYSCONF) && defined(_SC_CLK_TCK)
ticks_per_second = sysconf(_SC_CLK_TCK);
@ -13941,15 +13946,18 @@ INITFUNC(void)
#if defined(HAVE_SCHED_SETPARAM) || defined(HAVE_SCHED_SETSCHEDULER) || defined(POSIX_SPAWN_SETSCHEDULER) || defined(POSIX_SPAWN_SETSCHEDPARAM)
sched_param_desc.name = MODNAME ".sched_param";
if (PyStructSequence_InitType2(&SchedParamType, &sched_param_desc) < 0)
SchedParamType = PyStructSequence_NewType(&sched_param_desc);
if (SchedParamType == NULL) {
return NULL;
SchedParamType.tp_new = os_sched_param;
}
SchedParamType->tp_new = os_sched_param;
#endif
/* initialize TerminalSize_info */
if (PyStructSequence_InitType2(&TerminalSizeType,
&TerminalSize_desc) < 0)
TerminalSizeType = PyStructSequence_NewType(&TerminalSize_desc);
if (TerminalSizeType == NULL) {
return NULL;
}
/* initialize scandir types */
if (PyType_Ready(&ScandirIteratorType) < 0)
@ -13958,29 +13966,33 @@ INITFUNC(void)
return NULL;
}
#if defined(HAVE_WAITID) && !defined(__APPLE__)
Py_INCREF((PyObject*) &WaitidResultType);
PyModule_AddObject(m, "waitid_result", (PyObject*) &WaitidResultType);
Py_INCREF((PyObject*) WaitidResultType);
PyModule_AddObject(m, "waitid_result", (PyObject*) WaitidResultType);
#endif
Py_INCREF((PyObject*) &StatResultType);
PyModule_AddObject(m, "stat_result", (PyObject*) &StatResultType);
Py_INCREF((PyObject*) &StatVFSResultType);
Py_INCREF((PyObject*) StatResultType);
PyModule_AddObject(m, "stat_result", (PyObject*) StatResultType);
Py_INCREF((PyObject*) StatVFSResultType);
PyModule_AddObject(m, "statvfs_result",
(PyObject*) &StatVFSResultType);
(PyObject*) StatVFSResultType);
#if defined(HAVE_SCHED_SETPARAM) || defined(HAVE_SCHED_SETSCHEDULER)
Py_INCREF(&SchedParamType);
PyModule_AddObject(m, "sched_param", (PyObject *)&SchedParamType);
Py_INCREF(SchedParamType);
PyModule_AddObject(m, "sched_param", (PyObject *)SchedParamType);
#endif
times_result_desc.name = MODNAME ".times_result";
if (PyStructSequence_InitType2(&TimesResultType, &times_result_desc) < 0)
TimesResultType = PyStructSequence_NewType(&times_result_desc);
if (TimesResultType == NULL) {
return NULL;
PyModule_AddObject(m, "times_result", (PyObject *)&TimesResultType);
}
PyModule_AddObject(m, "times_result", (PyObject *)TimesResultType);
uname_result_desc.name = MODNAME ".uname_result";
if (PyStructSequence_InitType2(&UnameResultType, &uname_result_desc) < 0)
UnameResultType = PyStructSequence_NewType(&uname_result_desc);
if (UnameResultType == NULL) {
return NULL;
PyModule_AddObject(m, "uname_result", (PyObject *)&UnameResultType);
}
PyModule_AddObject(m, "uname_result", (PyObject *)UnameResultType);
#ifdef __APPLE__
/*
@ -14020,8 +14032,8 @@ INITFUNC(void)
#endif /* __APPLE__ */
Py_INCREF(&TerminalSizeType);
PyModule_AddObject(m, "terminal_size", (PyObject*) &TerminalSizeType);
Py_INCREF(TerminalSizeType);
PyModule_AddObject(m, "terminal_size", (PyObject*)TerminalSizeType);
billion = PyLong_FromLong(1000000000);
if (!billion)

225
Objects/structseq.c
View File

@ -287,115 +287,118 @@ static PyMethodDef structseq_methods[] = {
{NULL, NULL}
};
static PyTypeObject _struct_sequence_template = {
PyVarObject_HEAD_INIT(&PyType_Type, 0)
NULL, /* tp_name */
sizeof(PyStructSequence) - sizeof(PyObject *), /* tp_basicsize */
sizeof(PyObject *), /* tp_itemsize */
(destructor)structseq_dealloc, /* tp_dealloc */
0, /* tp_print */
0, /* tp_getattr */
0, /* tp_setattr */
0, /* tp_reserved */
(reprfunc)structseq_repr, /* tp_repr */
0, /* tp_as_number */
0, /* tp_as_sequence */
0, /* tp_as_mapping */
0, /* tp_hash */
0, /* tp_call */
0, /* tp_str */
0, /* tp_getattro */
0, /* tp_setattro */
0, /* tp_as_buffer */
Py_TPFLAGS_DEFAULT, /* tp_flags */
NULL, /* tp_doc */
0, /* tp_traverse */
0, /* tp_clear */
0, /* tp_richcompare */
0, /* tp_weaklistoffset */
0, /* tp_iter */
0, /* tp_iternext */
structseq_methods, /* tp_methods */
NULL, /* tp_members */
0, /* tp_getset */
0, /* tp_base */
0, /* tp_dict */
0, /* tp_descr_get */
0, /* tp_descr_set */
0, /* tp_dictoffset */
0, /* tp_init */
0, /* tp_alloc */
structseq_new, /* tp_new */
};
static Py_ssize_t
count_members(PyStructSequence_Desc *desc, Py_ssize_t *n_unnamed_members) {
Py_ssize_t i;
*n_unnamed_members = 0;
for (i = 0; desc->fields[i].name != NULL; ++i) {
if (desc->fields[i].name == PyStructSequence_UnnamedField) {
(*n_unnamed_members)++;
}
}
return i;
}
static int
initialize_structseq_dict(PyStructSequence_Desc *desc, PyObject* dict,
Py_ssize_t n_members, Py_ssize_t n_unnamed_members) {
PyObject *v;
#define SET_DICT_FROM_SIZE(key, value) \
do { \
v = PyLong_FromSsize_t(value); \
if (v == NULL) { \
return -1; \
} \
if (PyDict_SetItemString(dict, key, v) < 0) { \
Py_DECREF(v); \
return -1; \
} \
Py_DECREF(v); \
} while (0)
SET_DICT_FROM_SIZE(visible_length_key, desc->n_in_sequence);
SET_DICT_FROM_SIZE(real_length_key, n_members);
SET_DICT_FROM_SIZE(unnamed_fields_key, n_unnamed_members);
return 0;
}
static void
initialize_members(PyStructSequence_Desc *desc, PyMemberDef* members,
Py_ssize_t n_members) {
Py_ssize_t i, k;
for (i = k = 0; i < n_members; ++i) {
if (desc->fields[i].name == PyStructSequence_UnnamedField) {
continue;
}
/* The names and docstrings in these MemberDefs are statically */
/* allocated so it is expected that they'll outlive the MemberDef */
members[k].name = desc->fields[i].name;
members[k].type = T_OBJECT;
members[k].offset = offsetof(PyStructSequence, ob_item)
+ i * sizeof(PyObject*);
members[k].flags = READONLY;
members[k].doc = desc->fields[i].doc;
k++;
}
members[k].name = NULL;
}
int
PyStructSequence_InitType2(PyTypeObject *type, PyStructSequence_Desc *desc)
{
PyObject *dict;
PyMemberDef* members;
Py_ssize_t n_members, n_unnamed_members, i, k;
PyObject *v;
PyMemberDef *members;
Py_ssize_t n_members, n_unnamed_members;
#ifdef Py_TRACE_REFS
/* if the type object was chained, unchain it first
before overwriting its storage */
if (type->ob_base.ob_base._ob_next) {
_Py_ForgetReference((PyObject*)type);
_Py_ForgetReference((PyObject *)type);
}
#endif
n_unnamed_members = 0;
for (i = 0; desc->fields[i].name != NULL; ++i)
if (desc->fields[i].name == PyStructSequence_UnnamedField)
n_unnamed_members++;
n_members = i;
/* PyTypeObject has already been initialized */
if (Py_REFCNT(type) != 0) {
PyErr_BadInternalCall();
return -1;
}
memcpy(type, &_struct_sequence_template, sizeof(PyTypeObject));
type->tp_base = &PyTuple_Type;
type->tp_name = desc->name;
type->tp_basicsize = sizeof(PyStructSequence) - sizeof(PyObject *);
type->tp_itemsize = sizeof(PyObject *);
type->tp_dealloc = (destructor)structseq_dealloc;
type->tp_repr = (reprfunc)structseq_repr;
type->tp_doc = desc->doc;
type->tp_base = &PyTuple_Type;
type->tp_methods = structseq_methods;
type->tp_new = structseq_new;
type->tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC;
members = PyMem_NEW(PyMemberDef, n_members-n_unnamed_members+1);
n_members = count_members(desc, &n_unnamed_members);
members = PyMem_NEW(PyMemberDef, n_members - n_unnamed_members + 1);
if (members == NULL) {
PyErr_NoMemory();
return -1;
}
for (i = k = 0; i < n_members; ++i) {
if (desc->fields[i].name == PyStructSequence_UnnamedField)
continue;
members[k].name = desc->fields[i].name;
members[k].type = T_OBJECT;
members[k].offset = offsetof(PyStructSequence, ob_item)
+ i * sizeof(PyObject*);
members[k].flags = READONLY;
members[k].doc = desc->fields[i].doc;
k++;
}
members[k].name = NULL;
initialize_members(desc, members, n_members);
type->tp_members = members;
if (PyType_Ready(type) < 0)
if (PyType_Ready(type) < 0) {
PyMem_FREE(members);
return -1;
}
Py_INCREF(type);
dict = type->tp_dict;
#define SET_DICT_FROM_SIZE(key, value) \
do { \
v = PyLong_FromSsize_t(value); \
if (v == NULL) \
return -1; \
if (PyDict_SetItemString(dict, key, v) < 0) { \
Py_DECREF(v); \
return -1; \
} \
Py_DECREF(v); \
} while (0)
SET_DICT_FROM_SIZE(visible_length_key, desc->n_in_sequence);
SET_DICT_FROM_SIZE(real_length_key, n_members);
SET_DICT_FROM_SIZE(unnamed_fields_key, n_unnamed_members);
if (initialize_structseq_dict(
desc, type->tp_dict, n_members, n_unnamed_members) < 0) {
PyMem_FREE(members);
Py_DECREF(type);
return -1;
}
return 0;
}
@ -406,19 +409,57 @@ PyStructSequence_InitType(PyTypeObject *type, PyStructSequence_Desc *desc)
(void)PyStructSequence_InitType2(type, desc);
}
PyTypeObject*
PyTypeObject *
PyStructSequence_NewType(PyStructSequence_Desc *desc)
{
PyTypeObject *result;
result = (PyTypeObject*)PyType_GenericAlloc(&PyType_Type, 0);
if (result == NULL)
PyMemberDef *members;
PyObject *bases;
PyTypeObject *type;
PyType_Slot slots[7];
PyType_Spec spec;
Py_ssize_t n_members, n_unnamed_members;
/* Initialize MemberDefs */
n_members = count_members(desc, &n_unnamed_members);
members = PyMem_NEW(PyMemberDef, n_members - n_unnamed_members + 1);
if (members == NULL) {
PyErr_NoMemory();
return NULL;
if (PyStructSequence_InitType2(result, desc) < 0) {
Py_DECREF(result);
}
initialize_members(desc, members, n_members);
/* Initialize Slots */
slots[0] = (PyType_Slot){Py_tp_dealloc, (destructor)structseq_dealloc};
slots[1] = (PyType_Slot){Py_tp_repr, (reprfunc)structseq_repr};
slots[2] = (PyType_Slot){Py_tp_doc, (void *)desc->doc};
slots[3] = (PyType_Slot){Py_tp_methods, structseq_methods};
slots[4] = (PyType_Slot){Py_tp_new, structseq_new};
slots[5] = (PyType_Slot){Py_tp_members, members};
slots[6] = (PyType_Slot){0, 0};
/* Initialize Spec */
/* The name in this PyType_Spec is statically allocated so it is */
/* expected that it'll outlive the PyType_Spec */
spec.name = desc->name;
spec.basicsize = sizeof(PyStructSequence) - sizeof(PyObject *);
spec.itemsize = sizeof(PyObject *);
spec.flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC;
spec.slots = slots;
bases = PyTuple_Pack(1, &PyTuple_Type);
type = (PyTypeObject *)PyType_FromSpecWithBases(&spec, bases);
Py_DECREF(bases);
PyMem_FREE(members);
if (type == NULL) {
return NULL;
}
return result;
if (initialize_structseq_dict(
desc, type->tp_dict, n_members, n_unnamed_members) < 0) {
return NULL;
}
return type;
}
int _PyStructSequence_Init(void)

28
Objects/typeobject.c
View File

@ -2846,15 +2846,28 @@ static const short slotoffsets[] = {
PyObject *
PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases)
{
PyHeapTypeObject *res = (PyHeapTypeObject*)PyType_GenericAlloc(&PyType_Type, 0);
PyTypeObject *type, *base;
PyHeapTypeObject *res;
PyMemberDef *memb;
PyObject *modname;
char *s;
char *res_start = (char*)res;
PyTypeObject *type, *base;
PyType_Slot *slot;
Py_ssize_t nmembers;
char *s, *res_start;
nmembers = 0;
for (slot = spec->slots; slot->slot; slot++) {
if (slot->slot == Py_tp_members) {
for (memb = slot->pfunc; memb->name != NULL; memb++) {
nmembers++;
}
}
}
res = (PyHeapTypeObject*)PyType_GenericAlloc(&PyType_Type, nmembers);
if (res == NULL)
return NULL;
res_start = (char*)res;
if (spec->name == NULL) {
PyErr_SetString(PyExc_SystemError,
@ -2950,6 +2963,13 @@ PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases)
memcpy(tp_doc, old_doc, len);
type->tp_doc = tp_doc;
}
/* Move the slots to the heap type itself */
if (slot->slot == Py_tp_members) {
size_t len = Py_TYPE(type)->tp_itemsize * nmembers;
memcpy(PyHeapType_GET_MEMBERS(res), slot->pfunc, len);
type->tp_members = PyHeapType_GET_MEMBERS(res);
}
}
if (type->tp_dealloc == NULL) {
/* It's a heap type, so needs the heap types' dealloc.

Write Preview
Loading…
Cancel
Save