Add _PyObject_FastCall()

Issue #27128: Add _PyObject_FastCall(), a new calling convention avoiding a
temporary tuple to pass positional parameters in most cases, but create a
temporary tuple if needed (ex: for the tp_call slot).

The API is prepared to support keyword parameters, but the full implementation
will come later (_PyFunction_FastCall() doesn't support keyword parameters
yet).

Add also:

* _PyStack_AsTuple() helper function: convert a "stack" of parameters to
  a tuple.
* _PyCFunction_FastCall(): fast call implementation for C functions
* _PyFunction_FastCall(): fast call implementation for Python functions

Include/abstract.h

@@ -267,10 +267,26 @@ xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx*/
                                           PyObject *args, PyObject *kw);
 
 #ifndef Py_LIMITED_API
+    PyAPI_FUNC(PyObject*) _PyStack_AsTuple(PyObject **stack,
+        Py_ssize_t nargs);
+
+     /* Call the callable object func with the "fast call" calling convention:
+        args is a C array for positional parameters (nargs is the number of
+        positional paramater), kwargs is a dictionary for keyword parameters.
+
+        If nargs is equal to zero, args can be NULL. kwargs can be NULL.
+        nargs must be greater or equal to zero.
+
+        Return the result on success. Raise an exception on return NULL on
+        error. */
+     PyAPI_FUNC(PyObject *) _PyObject_FastCall(PyObject *func,
+                                               PyObject **args, int nargs,
+                                               PyObject *kwargs);
+
      PyAPI_FUNC(PyObject *) _Py_CheckFunctionResult(PyObject *func,
                                                     PyObject *result,
                                                     const char *where);
-#endif
+#endif   /* Py_LIMITED_API */
 
        /*
      Call a callable Python object, callable_object, with

Include/funcobject.h

@@ -58,6 +58,13 @@ PyAPI_FUNC(int) PyFunction_SetClosure(PyObject *, PyObject *);
 PyAPI_FUNC(PyObject *) PyFunction_GetAnnotations(PyObject *);
 PyAPI_FUNC(int) PyFunction_SetAnnotations(PyObject *, PyObject *);
 
+#ifndef Py_LIMITED_API
+PyAPI_FUNC(PyObject *) _PyFunction_FastCall(
+    PyObject *func,
+    PyObject **args, int nargs,
+    PyObject *kwargs);
+#endif
+
 /* Macros for direct access to these values. Type checks are *not*
    done, so use with care. */
 #define PyFunction_GET_CODE(func) \

Include/methodobject.h

@@ -37,6 +37,12 @@ PyAPI_FUNC(int) PyCFunction_GetFlags(PyObject *);
 #endif
 PyAPI_FUNC(PyObject *) PyCFunction_Call(PyObject *, PyObject *, PyObject *);
 
+#ifndef Py_LIMITED_API
+PyAPI_FUNC(PyObject *) _PyCFunction_FastCall(PyObject *func,
+    PyObject **args, int nargs,
+    PyObject *kwargs);
+#endif
+
 struct PyMethodDef {
     const char  *ml_name;   /* The name of the built-in function/method */
     PyCFunction ml_meth;    /* The C function that implements it */

Objects/abstract.c

@@ -2193,6 +2193,82 @@ PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw)
     return _Py_CheckFunctionResult(func, result, NULL);
 }
 
+PyObject*
+_PyStack_AsTuple(PyObject **stack, Py_ssize_t nargs)
+{
+    PyObject *args;
+    Py_ssize_t i;
+
+    args = PyTuple_New(nargs);
+    if (args == NULL) {
+        return NULL;
+    }
+
+    for (i=0; i < nargs; i++) {
+        PyObject *item = stack[i];
+        Py_INCREF(item);
+        PyTuple_SET_ITEM(args, i, item);
+    }
+
+    return args;
+}
+
+PyObject *
+_PyObject_FastCall(PyObject *func, PyObject **args, int nargs, PyObject *kwargs)
+{
+    ternaryfunc call;
+    PyObject *result = NULL;
+
+    /* _PyObject_FastCall() must not be called with an exception set,
+       because it may clear it (directly or indirectly) and so the
+       caller loses its exception */
+    assert(!PyErr_Occurred());
+
+    assert(func != NULL);
+    assert(nargs >= 0);
+    assert(nargs == 0 || args != NULL);
+    /* issue #27128: support for keywords will come later:
+       _PyFunction_FastCall() doesn't support keyword arguments yet */
+    assert(kwargs == NULL);
+
+    if (Py_EnterRecursiveCall(" while calling a Python object")) {
+        return NULL;
+    }
+
+    if (PyFunction_Check(func)) {
+        result = _PyFunction_FastCall(func, args, nargs, kwargs);
+    }
+    else if (PyCFunction_Check(func)) {
+        result = _PyCFunction_FastCall(func, args, nargs, kwargs);
+    }
+    else {
+        PyObject *tuple;
+
+        /* Slow-path: build a temporary tuple */
+        call = func->ob_type->tp_call;
+        if (call == NULL) {
+            PyErr_Format(PyExc_TypeError, "'%.200s' object is not callable",
+                         func->ob_type->tp_name);
+            goto exit;
+        }
+
+        tuple = _PyStack_AsTuple(args, nargs);
+        if (tuple == NULL) {
+            goto exit;
+        }
+
+        result = (*call)(func, tuple, kwargs);
+        Py_DECREF(tuple);
+    }
+
+    result = _Py_CheckFunctionResult(func, result, NULL);
+
+exit:
+    Py_LeaveRecursiveCall();
+
+    return result;
+}
+
 static PyObject*
 call_function_tail(PyObject *callable, PyObject *args)
 {

Objects/methodobject.c

@@ -145,6 +145,99 @@ PyCFunction_Call(PyObject *func, PyObject *args, PyObject *kwds)
     return _Py_CheckFunctionResult(func, res, NULL);
 }
 
+PyObject *
+_PyCFunction_FastCall(PyObject *func_obj, PyObject **args, int nargs,
+                      PyObject *kwargs)
+{
+    PyCFunctionObject* func = (PyCFunctionObject*)func_obj;
+    PyCFunction meth = PyCFunction_GET_FUNCTION(func);
+    PyObject *self = PyCFunction_GET_SELF(func);
+    PyObject *result;
+    int flags;
+
+    /* _PyCFunction_FastCall() must not be called with an exception set,
+       because it may clear it (directly or indirectly) and so the
+       caller loses its exception */
+    assert(!PyErr_Occurred());
+
+    flags = PyCFunction_GET_FLAGS(func) & ~(METH_CLASS | METH_STATIC | METH_COEXIST);
+
+    switch (flags)
+    {
+    case METH_NOARGS:
+        if (kwargs != NULL && PyDict_Size(kwargs) != 0) {
+            PyErr_Format(PyExc_TypeError, "%.200s() takes no keyword arguments",
+                         func->m_ml->ml_name);
+            return NULL;
+        }
+
+        if (nargs != 0) {
+            PyErr_Format(PyExc_TypeError,
+                "%.200s() takes no arguments (%zd given)",
+                func->m_ml->ml_name, nargs);
+            return NULL;
+        }
+
+        result = (*meth) (self, NULL);
+        break;
+
+    case METH_O:
+        if (kwargs != NULL && PyDict_Size(kwargs) != 0) {
+            PyErr_Format(PyExc_TypeError, "%.200s() takes no keyword arguments",
+                         func->m_ml->ml_name);
+            return NULL;
+        }
+
+        if (nargs != 1) {
+            PyErr_Format(PyExc_TypeError,
+                "%.200s() takes exactly one argument (%zd given)",
+                func->m_ml->ml_name, nargs);
+            return NULL;
+        }
+
+        result = (*meth) (self, args[0]);
+        break;
+
+    case METH_VARARGS:
+    case METH_VARARGS | METH_KEYWORDS:
+    {
+        /* Slow-path: create a temporary tuple */
+        PyObject *tuple;
+
+        if (!(flags & METH_KEYWORDS) && kwargs != NULL && PyDict_Size(kwargs) != 0) {
+            PyErr_Format(PyExc_TypeError,
+                         "%.200s() takes no keyword arguments",
+                         func->m_ml->ml_name);
+            return NULL;
+        }
+
+        tuple = _PyStack_AsTuple(args, nargs);
+        if (tuple == NULL) {
+            return NULL;
+        }
+
+        if (flags & METH_KEYWORDS) {
+            result = (*(PyCFunctionWithKeywords)meth) (self, tuple, kwargs);
+        }
+        else {
+            result = (*meth) (self, tuple);
+        }
+        Py_DECREF(tuple);
+        break;
+    }
+
+    default:
+        PyErr_SetString(PyExc_SystemError,
+                        "Bad call flags in PyCFunction_Call. "
+                        "METH_OLDARGS is no longer supported!");
+        return NULL;
+    }
+
+    result = _Py_CheckFunctionResult(func_obj, result, NULL);
+
+    return result;
+}
+
 /* Methods (the standard built-in methods, that is) */
 
 static void

Python/ceval.c

@@ -113,7 +113,7 @@ static PyObject * call_function(PyObject ***, int, uint64*, uint64*);
 #else
 static PyObject * call_function(PyObject ***, int);
 #endif
-static PyObject * fast_function(PyObject *, PyObject ***, int, int, int);
+static PyObject * fast_function(PyObject *, PyObject **, int, int, int);
 static PyObject * do_call(PyObject *, PyObject ***, int, int);
 static PyObject * ext_do_call(PyObject *, PyObject ***, int, int, int);
 static PyObject * update_keyword_args(PyObject *, int, PyObject ***,
@@ -3779,6 +3779,7 @@ too_many_positional(PyCodeObject *co, int given, int defcount, PyObject **fastlo
     Py_DECREF(kwonly_sig);
 }
 
+
 /* This is gonna seem *real weird*, but if you put some other code between
    PyEval_EvalFrame() and PyEval_EvalCodeEx() you will need to adjust
    the test in the if statements in Misc/gdbinit (pystack and pystackv). */
@@ -4068,8 +4069,10 @@ PyEval_EvalCodeEx(PyObject *_co, PyObject *globals, PyObject *locals,
            PyObject **defs, int defcount, PyObject *kwdefs, PyObject *closure)
 {
     return _PyEval_EvalCodeWithName(_co, globals, locals,
-                                    args, argcount, kws, kwcount,
-                                    defs, defcount, kwdefs, closure,
+                                    args, argcount,
+                                    kws, kwcount,
+                                    defs, defcount,
+                                    kwdefs, closure,
                                     NULL, NULL);
 }
 
@@ -4757,10 +4760,12 @@ call_function(PyObject ***pp_stack, int oparg
         } else
             Py_INCREF(func);
         READ_TIMESTAMP(*pintr0);
-        if (PyFunction_Check(func))
-            x = fast_function(func, pp_stack, n, na, nk);
-        else
+        if (PyFunction_Check(func)) {
+            x = fast_function(func, (*pp_stack) - n, n, na, nk);
+        }
+        else {
             x = do_call(func, pp_stack, na, nk);
+        }
         READ_TIMESTAMP(*pintr1);
         Py_DECREF(func);
 
@@ -4790,62 +4795,124 @@ call_function(PyObject ***pp_stack, int oparg
    done before evaluating the frame.
 */
 
+static PyObject*
+_PyFunction_FastCallNoKw(PyObject **args, Py_ssize_t na,
+                         PyCodeObject *co, PyObject *globals)
+{
+    PyFrameObject *f;
+    PyThreadState *tstate = PyThreadState_GET();
+    PyObject **fastlocals;
+    Py_ssize_t i;
+    PyObject *result;
+
+    PCALL(PCALL_FASTER_FUNCTION);
+    assert(globals != NULL);
+    /* XXX Perhaps we should create a specialized
+       PyFrame_New() that doesn't take locals, but does
+       take builtins without sanity checking them.
+       */
+    assert(tstate != NULL);
+    f = PyFrame_New(tstate, co, globals, NULL);
+    if (f == NULL) {
+        return NULL;
+    }
+
+    fastlocals = f->f_localsplus;
+
+    for (i = 0; i < na; i++) {
+        Py_INCREF(*args);
+        fastlocals[i] = *args++;
+    }
+    result = PyEval_EvalFrameEx(f,0);
+
+    ++tstate->recursion_depth;
+    Py_DECREF(f);
+    --tstate->recursion_depth;
+
+    return result;
+}
+
 static PyObject *
-fast_function(PyObject *func, PyObject ***pp_stack, int n, int na, int nk)
+fast_function(PyObject *func, PyObject **stack, int n, int na, int nk)
 {
     PyCodeObject *co = (PyCodeObject *)PyFunction_GET_CODE(func);
     PyObject *globals = PyFunction_GET_GLOBALS(func);
     PyObject *argdefs = PyFunction_GET_DEFAULTS(func);
-    PyObject *kwdefs = PyFunction_GET_KW_DEFAULTS(func);
-    PyObject *name = ((PyFunctionObject *)func) -> func_name;
-    PyObject *qualname = ((PyFunctionObject *)func) -> func_qualname;
-    PyObject **d = NULL;
-    int nd = 0;
+    PyObject *kwdefs, *closure, *name, *qualname;
+    PyObject **d;
+    int nd;
 
     PCALL(PCALL_FUNCTION);
     PCALL(PCALL_FAST_FUNCTION);
-    if (argdefs == NULL && co->co_argcount == n &&
-        co->co_kwonlyargcount == 0 && nk==0 &&
-        co->co_flags == (CO_OPTIMIZED | CO_NEWLOCALS | CO_NOFREE)) {
-        PyFrameObject *f;
-        PyObject *retval = NULL;
-        PyThreadState *tstate = PyThreadState_GET();
-        PyObject **fastlocals, **stack;
-        int i;
 
-        PCALL(PCALL_FASTER_FUNCTION);
-        assert(globals != NULL);
-        /* XXX Perhaps we should create a specialized
-           PyFrame_New() that doesn't take locals, but does
-           take builtins without sanity checking them.
-        */
-        assert(tstate != NULL);
-        f = PyFrame_New(tstate, co, globals, NULL);
-        if (f == NULL)
-            return NULL;
+    if (argdefs == NULL && co->co_argcount == na &&
+        co->co_kwonlyargcount == 0 && nk == 0 &&
+        co->co_flags == (CO_OPTIMIZED | CO_NEWLOCALS | CO_NOFREE))
+    {
+        return _PyFunction_FastCallNoKw(stack, na, co, globals);
+    }
 
-        fastlocals = f->f_localsplus;
-        stack = (*pp_stack) - n;
+    kwdefs = PyFunction_GET_KW_DEFAULTS(func);
+    closure = PyFunction_GET_CLOSURE(func);
+    name = ((PyFunctionObject *)func) -> func_name;
+    qualname = ((PyFunctionObject *)func) -> func_qualname;
 
-        for (i = 0; i < n; i++) {
-            Py_INCREF(*stack);
-            fastlocals[i] = *stack++;
-        }
-        retval = PyEval_EvalFrameEx(f,0);
-        ++tstate->recursion_depth;
-        Py_DECREF(f);
-        --tstate->recursion_depth;
-        return retval;
+    if (argdefs != NULL) {
+        d = &PyTuple_GET_ITEM(argdefs, 0);
+        nd = Py_SIZE(argdefs);
+    }
+    else {
+        d = NULL;
+        nd = 0;
+    }
+    return _PyEval_EvalCodeWithName((PyObject*)co, globals, (PyObject *)NULL,
+                                    stack, na,
+                                    stack + na, nk,
+                                    d, nd, kwdefs,
+                                    closure, name, qualname);
+}
+
+PyObject *
+_PyFunction_FastCall(PyObject *func, PyObject **args, int nargs, PyObject *kwargs)
+{
+    PyCodeObject *co = (PyCodeObject *)PyFunction_GET_CODE(func);
+    PyObject *globals = PyFunction_GET_GLOBALS(func);
+    PyObject *argdefs = PyFunction_GET_DEFAULTS(func);
+    PyObject *kwdefs, *closure, *name, *qualname;
+    PyObject **d;
+    int nd;
+
+    PCALL(PCALL_FUNCTION);
+    PCALL(PCALL_FAST_FUNCTION);
+
+    /* issue #27128: support for keywords will come later */
+    assert(kwargs == NULL);
+
+    if (argdefs == NULL && co->co_argcount == nargs &&
+        co->co_kwonlyargcount == 0 &&
+        co->co_flags == (CO_OPTIMIZED | CO_NEWLOCALS | CO_NOFREE))
+    {
+        return _PyFunction_FastCallNoKw(args, nargs, co, globals);
     }
+
+    kwdefs = PyFunction_GET_KW_DEFAULTS(func);
+    closure = PyFunction_GET_CLOSURE(func);
+    name = ((PyFunctionObject *)func) -> func_name;
+    qualname = ((PyFunctionObject *)func) -> func_qualname;
+
     if (argdefs != NULL) {
         d = &PyTuple_GET_ITEM(argdefs, 0);
         nd = Py_SIZE(argdefs);
     }
-    return _PyEval_EvalCodeWithName((PyObject*)co, globals,
-                                    (PyObject *)NULL, (*pp_stack)-n, na,
-                                    (*pp_stack)-2*nk, nk, d, nd, kwdefs,
-                                    PyFunction_GET_CLOSURE(func),
-                                    name, qualname);
+    else {
+        d = NULL;
+        nd = 0;
+    }
+    return _PyEval_EvalCodeWithName((PyObject*)co, globals, (PyObject *)NULL,
+                                     args, nargs,
+                                    NULL, 0,
+                                    d, nd, kwdefs,
+                                    closure, name, qualname);
 }
 
 static PyObject *