bpo-42725: Render annotations effectless on symbol table with PEP 563 (GH-25583)

Include/internal/pycore_symtable.h

@@ -10,7 +10,7 @@ extern "C" {
 
 struct _mod;   // Type defined in pycore_ast.h
 
-typedef enum _block_type { FunctionBlock, ClassBlock, ModuleBlock }
+typedef enum _block_type { FunctionBlock, ClassBlock, ModuleBlock, AnnotationBlock }
     _Py_block_ty;
 
 struct _symtable_entry;

Lib/test/test_future.py

@@ -171,6 +171,14 @@ class AnnotationsFutureTestCase(unittest.TestCase):
 
         self.assertEqual(actual, expected)
 
+    def _exec_future(self, code):
+        scope = {}
+        exec(
+            "from __future__ import annotations\n"
+            + code, {}, scope
+        )
+        return scope
+
     def test_annotations(self):
         eq = self.assertAnnotationEqual
         eq('...')
@@ -310,10 +318,6 @@ class AnnotationsFutureTestCase(unittest.TestCase):
         eq("f'{x}'")
         eq("f'{x!r}'")
         eq("f'{x!a}'")
-        eq('(yield from outside_of_generator)')
-        eq('(yield)')
-        eq('(yield a + b)')
-        eq('await some.complicated[0].call(with_args=True or 1 is not 1)')
         eq('[x for x in (a if b else c)]')
         eq('[x for x in a if (b if c else d)]')
         eq('f(x for x in a)')
@@ -321,8 +325,6 @@ class AnnotationsFutureTestCase(unittest.TestCase):
         eq('f((x for x in a), 2)')
         eq('(((a)))', 'a')
         eq('(((a, b)))', '(a, b)')
-        eq("(x := 10)")
-        eq("f'{(x := 10):=10}'")
         eq("1 + 2 + 3")
 
     def test_fstring_debug_annotations(self):
@@ -354,6 +356,53 @@ class AnnotationsFutureTestCase(unittest.TestCase):
                 "object.__debug__: int"
             )
 
+    def test_annotations_symbol_table_pass(self):
+        namespace = self._exec_future(dedent("""
+        from __future__ import annotations
+
+        def foo():
+            outer = 1
+            def bar():
+                inner: outer = 1
+            return bar
+        """))
+
+        foo = namespace.pop("foo")
+        self.assertIsNone(foo().__closure__)
+        self.assertEqual(foo.__code__.co_cellvars, ())
+        self.assertEqual(foo().__code__.co_freevars, ())
+
+    def test_annotations_forbidden(self):
+        with self.assertRaises(SyntaxError):
+            self._exec_future("test: (yield)")
+
+        with self.assertRaises(SyntaxError):
+            self._exec_future("test.test: (yield a + b)")
+
+        with self.assertRaises(SyntaxError):
+            self._exec_future("test[something]: (yield from x)")
+
+        with self.assertRaises(SyntaxError):
+            self._exec_future("def func(test: (yield from outside_of_generator)): pass")
+
+        with self.assertRaises(SyntaxError):
+            self._exec_future("def test() -> (await y): pass")
+
+        with self.assertRaises(SyntaxError):
+            self._exec_future("async def test() -> something((a := b)): pass")
+
+        with self.assertRaises(SyntaxError):
+            self._exec_future("test: await some.complicated[0].call(with_args=True or 1 is not 1)")
+
+        with self.assertRaises(SyntaxError):
+            self._exec_future("test: f'{(x := 10):=10}'")
+
+        with self.assertRaises(SyntaxError):
+            self._exec_future(dedent("""\
+            def foo():
+                def bar(arg: (yield)): pass
+            """))
+
 
 if __name__ == "__main__":
     unittest.main()

Misc/NEWS.d/next/Core and Builtins/2021-04-25-05-40-51.bpo-42725.WGloYm.rst

@@ -0,0 +1,2 @@
+Usage of ``await``/``yield``/``yield from`` and named expressions within an
+annotation is now forbidden when PEP 563 is activated.

Modules/symtablemodule.c

@@ -58,7 +58,6 @@ _symtable_symtable_impl(PyObject *module, PyObject *source,
     }
     t = (PyObject *)st->st_top;
     Py_INCREF(t);
-    PyMem_Free((void *)st->st_future);
     _PySymtable_Free(st);
     return t;
 }

Python/symtable.c

@@ -1,6 +1,6 @@
 #include "Python.h"
 #include "pycore_ast.h"           // identifier, stmt_ty
-#include "pycore_compile.h"       // _Py_Mangle()
+#include "pycore_compile.h"       // _Py_Mangle(), _PyFuture_FromAST()
 #include "pycore_parser.h"        // _PyParser_ASTFromString()
 #include "pycore_pystate.h"       // _PyThreadState_GET()
 #include "pycore_symtable.h"      // PySTEntryObject
@@ -45,6 +45,10 @@
 #define NAMED_EXPR_COMP_ITER_EXPR \
 "assignment expression cannot be used in a comprehension iterable expression"
 
+#define ANNOTATION_NOT_ALLOWED \
+"'%s' can not be used within an annotation"
+
+
 static PySTEntryObject *
 ste_new(struct symtable *st, identifier name, _Py_block_ty block,
         void *key, int lineno, int col_offset,
@@ -209,17 +213,19 @@ static int symtable_visit_alias(struct symtable *st, alias_ty);
 static int symtable_visit_comprehension(struct symtable *st, comprehension_ty);
 static int symtable_visit_keyword(struct symtable *st, keyword_ty);
 static int symtable_visit_params(struct symtable *st, asdl_arg_seq *args);
+static int symtable_visit_annotation(struct symtable *st, expr_ty annotation);
 static int symtable_visit_argannotations(struct symtable *st, asdl_arg_seq *args);
 static int symtable_implicit_arg(struct symtable *st, int pos);
-static int symtable_visit_annotations(struct symtable *st, arguments_ty, expr_ty);
+static int symtable_visit_annotations(struct symtable *st, stmt_ty, arguments_ty, expr_ty);
 static int symtable_visit_withitem(struct symtable *st, withitem_ty item);
 static int symtable_visit_match_case(struct symtable *st, match_case_ty m);
 static int symtable_visit_pattern(struct symtable *st, pattern_ty s);
+static int symtable_raise_if_annotation_block(struct symtable *st, const char *, expr_ty);
 
 
 static identifier top = NULL, lambda = NULL, genexpr = NULL,
     listcomp = NULL, setcomp = NULL, dictcomp = NULL,
-    __class__ = NULL;
+    __class__ = NULL, _annotation = NULL;
 
 #define GET_IDENTIFIER(VAR) \
     ((VAR) ? (VAR) : ((VAR) = PyUnicode_InternFromString(# VAR)))
@@ -987,8 +993,17 @@ symtable_enter_block(struct symtable *st, identifier name, _Py_block_ty block,
     /* The entry is owned by the stack. Borrow it for st_cur. */
     Py_DECREF(ste);
     st->st_cur = ste;
+
+    /* Annotation blocks shouldn't have any affect on the symbol table since in
+     * the compilation stage, they will all be transformed to strings. They are
+     * only created if future 'annotations' feature is activated. */
+    if (block == AnnotationBlock) {
+        return 1;
+    }
+
     if (block == ModuleBlock)
         st->st_global = st->st_cur->ste_symbols;
+
     if (prev) {
         if (PyList_Append(prev->ste_children, (PyObject *)ste) < 0) {
             return 0;
@@ -1190,7 +1205,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s)
             VISIT_SEQ(st, expr, s->v.FunctionDef.args->defaults);
         if (s->v.FunctionDef.args->kw_defaults)
             VISIT_SEQ_WITH_NULL(st, expr, s->v.FunctionDef.args->kw_defaults);
-        if (!symtable_visit_annotations(st, s->v.FunctionDef.args,
+        if (!symtable_visit_annotations(st, s, s->v.FunctionDef.args,
                                         s->v.FunctionDef.returns))
             VISIT_QUIT(st, 0);
         if (s->v.FunctionDef.decorator_list)
@@ -1273,7 +1288,10 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s)
         else {
             VISIT(st, expr, s->v.AnnAssign.target);
         }
-        VISIT(st, expr, s->v.AnnAssign.annotation);
+        if (!symtable_visit_annotation(st, s->v.AnnAssign.annotation)) {
+            VISIT_QUIT(st, 0);
+        }
+
         if (s->v.AnnAssign.value) {
             VISIT(st, expr, s->v.AnnAssign.value);
         }
@@ -1422,7 +1440,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s)
         if (s->v.AsyncFunctionDef.args->kw_defaults)
             VISIT_SEQ_WITH_NULL(st, expr,
                                 s->v.AsyncFunctionDef.args->kw_defaults);
-        if (!symtable_visit_annotations(st, s->v.AsyncFunctionDef.args,
+        if (!symtable_visit_annotations(st, s, s->v.AsyncFunctionDef.args,
                                         s->v.AsyncFunctionDef.returns))
             VISIT_QUIT(st, 0);
         if (s->v.AsyncFunctionDef.decorator_list)
@@ -1564,6 +1582,9 @@ symtable_visit_expr(struct symtable *st, expr_ty e)
     }
     switch (e->kind) {
     case NamedExpr_kind:
+        if (!symtable_raise_if_annotation_block(st, "named expression", e)) {
+            VISIT_QUIT(st, 0);
+        }
         if(!symtable_handle_namedexpr(st, e))
             VISIT_QUIT(st, 0);
         break;
@@ -1624,15 +1645,24 @@ symtable_visit_expr(struct symtable *st, expr_ty e)
             VISIT_QUIT(st, 0);
         break;
     case Yield_kind:
+        if (!symtable_raise_if_annotation_block(st, "yield expression", e)) {
+            VISIT_QUIT(st, 0);
+        }
         if (e->v.Yield.value)
             VISIT(st, expr, e->v.Yield.value);
         st->st_cur->ste_generator = 1;
         break;
     case YieldFrom_kind:
+        if (!symtable_raise_if_annotation_block(st, "yield expression", e)) {
+            VISIT_QUIT(st, 0);
+        }
         VISIT(st, expr, e->v.YieldFrom.value);
         st->st_cur->ste_generator = 1;
         break;
     case Await_kind:
+        if (!symtable_raise_if_annotation_block(st, "await expression", e)) {
+            VISIT_QUIT(st, 0);
+        }
         VISIT(st, expr, e->v.Await.value);
         st->st_cur->ste_coroutine = 1;
         break;
@@ -1780,6 +1810,24 @@ symtable_visit_params(struct symtable *st, asdl_arg_seq *args)
     return 1;
 }
 
+static int
+symtable_visit_annotation(struct symtable *st, expr_ty annotation)
+{
+    int future_annotations = st->st_future->ff_features & CO_FUTURE_ANNOTATIONS;
+    if (future_annotations &&
+        !symtable_enter_block(st, GET_IDENTIFIER(_annotation), AnnotationBlock,
+                              (void *)annotation, annotation->lineno,
+                              annotation->col_offset, annotation->end_lineno,
+                              annotation->end_col_offset)) {
+        VISIT_QUIT(st, 0);
+    }
+    VISIT(st, expr, annotation);
+    if (future_annotations && !symtable_exit_block(st)) {
+        VISIT_QUIT(st, 0);
+    }
+    return 1;
+}
+
 static int
 symtable_visit_argannotations(struct symtable *st, asdl_arg_seq *args)
 {
@@ -1798,8 +1846,15 @@ symtable_visit_argannotations(struct symtable *st, asdl_arg_seq *args)
 }
 
 static int
-symtable_visit_annotations(struct symtable *st, arguments_ty a, expr_ty returns)
+symtable_visit_annotations(struct symtable *st, stmt_ty o, arguments_ty a, expr_ty returns)
 {
+    int future_annotations = st->st_future->ff_features & CO_FUTURE_ANNOTATIONS;
+    if (future_annotations &&
+        !symtable_enter_block(st, GET_IDENTIFIER(_annotation), AnnotationBlock,
+                              (void *)o, o->lineno, o->col_offset, o->end_lineno,
+                              o->end_col_offset)) {
+        VISIT_QUIT(st, 0);
+    }
     if (a->posonlyargs && !symtable_visit_argannotations(st, a->posonlyargs))
         return 0;
     if (a->args && !symtable_visit_argannotations(st, a->args))
@@ -1810,8 +1865,12 @@ symtable_visit_annotations(struct symtable *st, arguments_ty a, expr_ty returns)
         VISIT(st, expr, a->kwarg->annotation);
     if (a->kwonlyargs && !symtable_visit_argannotations(st, a->kwonlyargs))
         return 0;
-    if (returns)
-        VISIT(st, expr, returns);
+    if (future_annotations && !symtable_exit_block(st)) {
+        VISIT_QUIT(st, 0);
+    }
+    if (returns && !symtable_visit_annotation(st, returns)) {
+        VISIT_QUIT(st, 0);
+    }
     return 1;
 }
 
@@ -2033,6 +2092,21 @@ symtable_visit_dictcomp(struct symtable *st, expr_ty e)
                                          e->v.DictComp.value);
 }
 
+static int
+symtable_raise_if_annotation_block(struct symtable *st, const char *name, expr_ty e)
+{
+    if (st->st_cur->ste_type != AnnotationBlock) {
+        return 1;
+    }
+
+    PyErr_Format(PyExc_SyntaxError, ANNOTATION_NOT_ALLOWED, name);
+    PyErr_RangedSyntaxLocationObject(st->st_filename,
+                                     e->lineno,
+                                     e->col_offset + 1,
+                                     e->end_lineno,
+                                     e->end_col_offset + 1);
+    return 0;
+}
 
 struct symtable *
 _Py_SymtableStringObjectFlags(const char *str, PyObject *filename,
@@ -2051,7 +2125,14 @@ _Py_SymtableStringObjectFlags(const char *str, PyObject *filename,
         _PyArena_Free(arena);
         return NULL;
     }
-    st = _PySymtable_Build(mod, filename, 0);
+    PyFutureFeatures *future = _PyFuture_FromAST(mod, filename);
+    if (future == NULL) {
+        _PyArena_Free(arena);
+        return NULL;
+    }
+    future->ff_features |= flags->cf_flags;
+    st = _PySymtable_Build(mod, filename, future);
+    PyObject_Free((void *)future);
     _PyArena_Free(arena);
     return st;
 }