bpo-36878: Track extra text added to 'type: ignore' in the AST (GH-13479)

GH-13238 made extra text after a # type: ignore accepted by the parser.
This finishes the job and actually plumbs the extra text through the
parser and makes it available in the AST.

Include/Python-ast.h

@@ -467,6 +467,7 @@ struct _type_ignore {
     union {
         struct {
             int lineno;
+            string tag;
         } TypeIgnore;
 
     } v;
@@ -702,8 +703,8 @@ alias_ty _Py_alias(identifier name, identifier asname, PyArena *arena);
 #define withitem(a0, a1, a2) _Py_withitem(a0, a1, a2)
 withitem_ty _Py_withitem(expr_ty context_expr, expr_ty optional_vars, PyArena
                          *arena);
-#define TypeIgnore(a0, a1) _Py_TypeIgnore(a0, a1)
-type_ignore_ty _Py_TypeIgnore(int lineno, PyArena *arena);
+#define TypeIgnore(a0, a1, a2) _Py_TypeIgnore(a0, a1, a2)
+type_ignore_ty _Py_TypeIgnore(int lineno, string tag, PyArena *arena);
 
 PyObject* PyAST_mod2obj(mod_ty t);
 mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode);

Lib/test/test_type_comments.py

@@ -272,7 +272,16 @@ class TypeCommentTests(unittest.TestCase):
 
     def test_ignores(self):
         for tree in self.parse_all(ignores):
-            self.assertEqual([ti.lineno for ti in tree.type_ignores], [2, 5, 8, 9, 10, 11])
+            self.assertEqual(
+                [(ti.lineno, ti.tag) for ti in tree.type_ignores],
+                [
+                    (2, ''),
+                    (5, ''),
+                    (8, '[excuse]'),
+                    (9, '=excuse'),
+                    (10, ' [excuse]'),
+                    (11, ' whatever'),
+                ])
         tree = self.classic_parse(ignores)
         self.assertEqual(tree.type_ignores, [])
 

Misc/ACKS

@@ -1594,6 +1594,7 @@ Daniel Stutzbach
 Andreas Stührk
 Colin Su
 Pal Subbiah
+Michael J. Sullivan
 Nathan Sullivan
 Mark Summerfield
 Reuben Sumner

Misc/NEWS.d/next/Core and Builtins/2019-05-21-16-21-22.bpo-36878.EFRHZ3.rst

@@ -0,0 +1,3 @@
+Store text appearing after a `# type: ignore` comment in the AST. For
+example a type ignore like `# type: ignore[E1000]` will have the string
+`"[E1000]"` stored in its AST node.

Parser/Python.asdl

@@ -125,6 +125,5 @@ module Python
 
     withitem = (expr context_expr, expr? optional_vars)
 
-    type_ignore = TypeIgnore(int lineno)
+    type_ignore = TypeIgnore(int lineno, string tag)
 }
-

Parser/parsetok.c

@@ -16,13 +16,16 @@ static node *parsetok(struct tok_state *, grammar *, int, perrdetail *, int *);
 static int initerr(perrdetail *err_ret, PyObject * filename);
 
 typedef struct {
-    int *items;
+    struct {
+        int lineno;
+        char *comment;
+    } *items;
     size_t size;
     size_t num_items;
-} growable_int_array;
+} growable_comment_array;
 
 static int
-growable_int_array_init(growable_int_array *arr, size_t initial_size) {
+growable_comment_array_init(growable_comment_array *arr, size_t initial_size) {
     assert(initial_size > 0);
     arr->items = malloc(initial_size * sizeof(*arr->items));
     arr->size = initial_size;
@@ -32,7 +35,7 @@ growable_int_array_init(growable_int_array *arr, size_t initial_size) {
 }
 
 static int
-growable_int_array_add(growable_int_array *arr, int item) {
+growable_comment_array_add(growable_comment_array *arr, int lineno, char *comment) {
     if (arr->num_items >= arr->size) {
         arr->size *= 2;
         arr->items = realloc(arr->items, arr->size * sizeof(*arr->items));
@@ -41,13 +44,17 @@ growable_int_array_add(growable_int_array *arr, int item) {
         }
     }
 
-    arr->items[arr->num_items] = item;
+    arr->items[arr->num_items].lineno = lineno;
+    arr->items[arr->num_items].comment = comment;
     arr->num_items++;
     return 1;
 }
 
 static void
-growable_int_array_deallocate(growable_int_array *arr) {
+growable_comment_array_deallocate(growable_comment_array *arr) {
+    for (unsigned i = 0; i < arr->num_items; i++) {
+        PyObject_FREE(arr->items[i].comment);
+    }
     free(arr->items);
 }
 
@@ -220,9 +227,9 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
     node *n;
     int started = 0;
     int col_offset, end_col_offset;
-    growable_int_array type_ignores;
+    growable_comment_array type_ignores;
 
-    if (!growable_int_array_init(&type_ignores, 10)) {
+    if (!growable_comment_array_init(&type_ignores, 10)) {
         err_ret->error = E_NOMEM;
         PyTokenizer_Free(tok);
         return NULL;
@@ -320,8 +327,7 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
         }
 
         if (type == TYPE_IGNORE) {
-            PyObject_FREE(str);
-            if (!growable_int_array_add(&type_ignores, tok->lineno)) {
+            if (!growable_comment_array_add(&type_ignores, tok->lineno, str)) {
                 err_ret->error = E_NOMEM;
                 break;
             }
@@ -355,9 +361,16 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
             REQ(ch, ENDMARKER);
 
             for (i = 0; i < type_ignores.num_items; i++) {
-                PyNode_AddChild(ch, TYPE_IGNORE, NULL,
-                                type_ignores.items[i], 0,
-                                type_ignores.items[i], 0);
+                int res = PyNode_AddChild(ch, TYPE_IGNORE, type_ignores.items[i].comment,
+                                          type_ignores.items[i].lineno, 0,
+                                          type_ignores.items[i].lineno, 0);
+                if (res != 0) {
+                    err_ret->error = res;
+                    PyNode_Free(n);
+                    n = NULL;
+                    break;
+                }
+                type_ignores.items[i].comment = NULL;
             }
         }
 
@@ -365,7 +378,7 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
            is a single statement by looking at what is left in the
            buffer after parsing.  Trailing whitespace and comments
            are OK.  */
-        if (start == single_input) {
+        if (err_ret->error == E_DONE && start == single_input) {
             char *cur = tok->cur;
             char c = *tok->cur;
 
@@ -392,7 +405,7 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
     else
         n = NULL;
 
-    growable_int_array_deallocate(&type_ignores);
+    growable_comment_array_deallocate(&type_ignores);
 
 #ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
     *flags = ps->p_flags;

Parser/tokenizer.c

@@ -1269,6 +1269,7 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end)
             /* This is a type comment if we matched all of type_comment_prefix. */
             if (!*prefix) {
                 int is_type_ignore = 1;
+                const char *ignore_end = p + 6;
                 tok_backup(tok, c);  /* don't eat the newline or EOF */
 
                 type_start = p;
@@ -1276,10 +1277,13 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end)
                 /* A TYPE_IGNORE is "type: ignore" followed by the end of the token
                  * or anything non-alphanumeric. */
                 is_type_ignore = (
-                    tok->cur >= p + 6 && memcmp(p, "ignore", 6) == 0
-                    && !(tok->cur > p + 6 && isalnum(p[6])));
+                    tok->cur >= ignore_end && memcmp(p, "ignore", 6) == 0
+                    && !(tok->cur > ignore_end && isalnum(p[6])));
 
                 if (is_type_ignore) {
+                    *p_start = (char *) ignore_end;
+                    *p_end = tok->cur;
+
                     /* If this type ignore is the only thing on the line, consume the newline also. */
                     if (blankline) {
                         tok_nextc(tok);

Python/Python-ast.c

@@ -524,8 +524,10 @@ static char *withitem_fields[]={
 static PyTypeObject *type_ignore_type;
 static PyObject* ast2obj_type_ignore(void*);
 static PyTypeObject *TypeIgnore_type;
+_Py_IDENTIFIER(tag);
 static char *TypeIgnore_fields[]={
     "lineno",
+    "tag",
 };
 
 
@@ -1164,7 +1166,7 @@ static int init_types(void)
     if (!type_ignore_type) return 0;
     if (!add_attributes(type_ignore_type, NULL, 0)) return 0;
     TypeIgnore_type = make_type("TypeIgnore", type_ignore_type,
-                                TypeIgnore_fields, 1);
+                                TypeIgnore_fields, 2);
     if (!TypeIgnore_type) return 0;
     initialized = 1;
     return 1;
@@ -2667,14 +2669,20 @@ withitem(expr_ty context_expr, expr_ty optional_vars, PyArena *arena)
 }
 
 type_ignore_ty
-TypeIgnore(int lineno, PyArena *arena)
+TypeIgnore(int lineno, string tag, PyArena *arena)
 {
     type_ignore_ty p;
+    if (!tag) {
+        PyErr_SetString(PyExc_ValueError,
+                        "field tag is required for TypeIgnore");
+        return NULL;
+    }
     p = (type_ignore_ty)PyArena_Malloc(arena, sizeof(*p));
     if (!p)
         return NULL;
     p->kind = TypeIgnore_kind;
     p->v.TypeIgnore.lineno = lineno;
+    p->v.TypeIgnore.tag = tag;
     return p;
 }
 
@@ -4158,6 +4166,11 @@ ast2obj_type_ignore(void* _o)
         if (_PyObject_SetAttrId(result, &PyId_lineno, value) == -1)
             goto failed;
         Py_DECREF(value);
+        value = ast2obj_string(o->v.TypeIgnore.tag);
+        if (!value) goto failed;
+        if (_PyObject_SetAttrId(result, &PyId_tag, value) == -1)
+            goto failed;
+        Py_DECREF(value);
         break;
     }
     return result;
@@ -8738,6 +8751,7 @@ obj2ast_type_ignore(PyObject* obj, type_ignore_ty* out, PyArena* arena)
     }
     if (isinstance) {
         int lineno;
+        string tag;
 
         if (_PyObject_LookupAttrId(obj, &PyId_lineno, &tmp) < 0) {
             return 1;
@@ -8752,7 +8766,20 @@ obj2ast_type_ignore(PyObject* obj, type_ignore_ty* out, PyArena* arena)
             if (res != 0) goto failed;
             Py_CLEAR(tmp);
         }
-        *out = TypeIgnore(lineno, arena);
+        if (_PyObject_LookupAttrId(obj, &PyId_tag, &tmp) < 0) {
+            return 1;
+        }
+        if (tmp == NULL) {
+            PyErr_SetString(PyExc_TypeError, "required field \"tag\" missing from TypeIgnore");
+            return 1;
+        }
+        else {
+            int res;
+            res = obj2ast_string(tmp, &tag, arena);
+            if (res != 0) goto failed;
+            Py_CLEAR(tmp);
+        }
+        *out = TypeIgnore(lineno, tag, arena);
         if (*out == NULL) goto failed;
         return 0;
     }

Python/ast.c

@@ -830,7 +830,10 @@ PyAST_FromNodeObject(const node *n, PyCompilerFlags *flags,
                 goto out;
 
             for (i = 0; i < num; i++) {
-                type_ignore_ty ti = TypeIgnore(LINENO(CHILD(ch, i)), arena);
+                string type_comment = new_type_comment(STR(CHILD(ch, i)), &c);
+                if (!type_comment)
+                    goto out;
+                type_ignore_ty ti = TypeIgnore(LINENO(CHILD(ch, i)), type_comment, arena);
                 if (!ti)
                    goto out;
                asdl_seq_SET(type_ignores, i, ti);