bpo-43950: Add code.co_positions (PEP 657) (GH-26955)

This PR is part of PEP 657 and augments the compiler to emit ending
line numbers as well as starting and ending columns from the AST
into compiled code objects. This allows bytecodes to be correlated
to the exact source code ranges that generated them.

This information is made available through the following public APIs:

* The `co_positions` method on code objects.
* The C API function `PyCode_Addr2Location`.

Co-authored-by: Batuhan Taskaya <isidentical@gmail.com>
Co-authored-by: Ammar Askar <ammar@ammaraskar.com>

Include/cpython/code.h

@@ -75,8 +75,13 @@ struct PyCodeObject {
     PyObject *co_localspluskinds; /* Bytes mapping to local kinds (one byte per variable) */
     PyObject *co_filename;      /* unicode (where it was loaded from) */
     PyObject *co_name;          /* unicode (name, for reference) */
-    PyObject *co_linetable;     /* string (encoding addr<->lineno mapping) See
+    PyObject *co_linetable;     /* bytes (encoding addr<->lineno mapping) See
                                    Objects/lnotab_notes.txt for details. */
+    PyObject *co_endlinetable;  /* bytes object that holds end lineno for
+                                   instructions separated across different
+                                   lines */
+    PyObject *co_columntable;   /* bytes object that holds start/end column
+                                   offset each instruction */
 
     /* These fields are set with computed values on new code objects. */
 
@@ -149,12 +154,14 @@ PyAPI_DATA(PyTypeObject) PyCode_Type;
 PyAPI_FUNC(PyCodeObject *) PyCode_New(
         int, int, int, int, int, PyObject *, PyObject *,
         PyObject *, PyObject *, PyObject *, PyObject *,
-        PyObject *, PyObject *, int, PyObject *, PyObject *);
+        PyObject *, PyObject *, int, PyObject *, PyObject *,
+        PyObject *, PyObject *);
 
 PyAPI_FUNC(PyCodeObject *) PyCode_NewWithPosOnlyArgs(
         int, int, int, int, int, int, PyObject *, PyObject *,
         PyObject *, PyObject *, PyObject *, PyObject *,
-        PyObject *, PyObject *, int, PyObject *, PyObject *);
+        PyObject *, PyObject *, int, PyObject *, PyObject *,
+        PyObject *, PyObject *);
         /* same as struct above */
 
 /* Creates a new empty code object with the specified source location. */
@@ -166,6 +173,15 @@ PyCode_NewEmpty(const char *filename, const char *funcname, int firstlineno);
    use PyFrame_GetLineNumber() instead. */
 PyAPI_FUNC(int) PyCode_Addr2Line(PyCodeObject *, int);
 
+PyAPI_FUNC(int) PyCode_Addr2Location(PyCodeObject *, int, int *, int *, int *, int *);
+
+/* Return the ending source code line number from a bytecode index. */
+PyAPI_FUNC(int) _PyCode_Addr2EndLine(PyCodeObject *, int);
+/* Return the starting source code column offset from a bytecode index. */
+PyAPI_FUNC(int) _PyCode_Addr2Offset(PyCodeObject *, int);
+/* Return the ending source code column offset from a bytecode index. */
+PyAPI_FUNC(int) _PyCode_Addr2EndOffset(PyCodeObject *, int);
+
 /* for internal use only */
 struct _opaque {
     int computed_line;
@@ -203,8 +219,9 @@ PyAPI_FUNC(int) _PyCode_GetExtra(PyObject *code, Py_ssize_t index,
 PyAPI_FUNC(int) _PyCode_SetExtra(PyObject *code, Py_ssize_t index,
                                  void *extra);
 
-/** API for initializing the line number table. */
+/** API for initializing the line number tables. */
 int _PyCode_InitAddressRange(PyCodeObject* co, PyCodeAddressRange *bounds);
+int _PyCode_InitEndAddressRange(PyCodeObject* co, PyCodeAddressRange* bounds);
 
 /** Out of process API for initializing the line number table. */
 void PyLineTable_InitAddressRange(const char *linetable, Py_ssize_t length, int firstlineno, PyCodeAddressRange *range);

Include/internal/pycore_code.h

@@ -218,6 +218,8 @@ struct _PyCodeConstructor {
     PyObject *code;
     int firstlineno;
     PyObject *linetable;
+    PyObject *endlinetable;
+    PyObject *columntable;
 
     /* used by the code */
     PyObject *consts;

Lib/ctypes/test/test_values.py

@@ -80,9 +80,9 @@ class PythonValuesTestCase(unittest.TestCase):
                 continue
             items.append((entry.name.decode("ascii"), entry.size))
 
-        expected = [("__hello__", 133),
-                    ("__phello__", -133),
-                    ("__phello__.spam", 133),
+        expected = [("__hello__", 159),
+                    ("__phello__", -159),
+                    ("__phello__.spam", 159),
                     ]
         self.assertEqual(items, expected, "PyImport_FrozenModules example "
             "in Doc/library/ctypes.rst may be out of date")

Lib/importlib/_bootstrap_external.py

@@ -361,6 +361,7 @@ _code_type = type(_write_atomic.__code__)
 #     Python 3.11a1 3456 (interleave cell args bpo-43693)
 #     Python 3.11a1 3457 (Change localsplus to a bytes object bpo-43693)
 #     Python 3.11a1 3458 (imported objects now don't use LOAD_METHOD/CALL_METHOD)
+#     Python 3.11a1 3459 (PEP 657: add end line numbers and column offsets for instructions)
 
 #
 # MAGIC must change whenever the bytecode emitted by the compiler may no
@@ -370,7 +371,7 @@ _code_type = type(_write_atomic.__code__)
 # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array
 # in PC/launcher.c must also be updated.
 
-MAGIC_NUMBER = (3458).to_bytes(2, 'little') + b'\r\n'
+MAGIC_NUMBER = (3459).to_bytes(2, 'little') + b'\r\n'
 _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little')  # For import.c
 
 _PYCACHE = '__pycache__'

Lib/test/test_code.py

@@ -129,7 +129,9 @@ import inspect
 import sys
 import threading
 import unittest
+import textwrap
 import weakref
+
 try:
     import ctypes
 except ImportError:
@@ -223,6 +225,8 @@ class CodeTest(unittest.TestCase):
                         co.co_name,
                         co.co_firstlineno,
                         co.co_lnotab,
+                        co.co_endlinetable,
+                        co.co_columntable,
                         co.co_exceptiontable,
                         co.co_freevars,
                         co.co_cellvars)
@@ -257,6 +261,8 @@ class CodeTest(unittest.TestCase):
             ("co_filename", "newfilename"),
             ("co_name", "newname"),
             ("co_linetable", code2.co_linetable),
+            ("co_endlinetable", code2.co_endlinetable),
+            ("co_columntable", code2.co_columntable),
         ):
             with self.subTest(attr=attr, value=value):
                 new_code = code.replace(**{attr: value})
@@ -293,6 +299,8 @@ class CodeTest(unittest.TestCase):
                          co.co_name,
                          co.co_firstlineno,
                          co.co_lnotab,
+                         co.co_endlinetable,
+                         co.co_columntable,
                          co.co_exceptiontable,
                          co.co_freevars,
                          co.co_cellvars,
@@ -309,6 +317,81 @@ class CodeTest(unittest.TestCase):
         new_code = code = func.__code__.replace(co_linetable=b'')
         self.assertEqual(list(new_code.co_lines()), [])
 
+    def test_co_positions_artificial_instructions(self):
+        import dis
+
+        namespace = {}
+        exec(textwrap.dedent("""\
+        try:
+            1/0
+        except Exception as e:
+            exc = e
+        """), namespace)
+
+        exc = namespace['exc']
+        traceback = exc.__traceback__
+        code = traceback.tb_frame.f_code
+
+        artificial_instructions = []
+        for instr, positions in zip(
+            dis.get_instructions(code),
+            code.co_positions(),
+            strict=True
+        ):
+            # If any of the positions is None, then all have to
+            # be None as well for the case above. There are still
+            # some places in the compiler, where the artificial instructions
+            # get assigned the first_lineno but they don't have other positions.
+            # There is no easy way of inferring them at that stage, so for now
+            # we don't support it.
+            self.assertTrue(all(positions) or not any(positions))
+
+            if not any(positions):
+                artificial_instructions.append(instr)
+
+        self.assertEqual(
+            [
+                (instruction.opname, instruction.argval)
+                for instruction in artificial_instructions
+            ],
+            [
+                ("PUSH_EXC_INFO", None),
+                ("LOAD_CONST", None), # artificial 'None'
+                ("STORE_NAME", "e"),  # XX: we know the location for this
+                ("DELETE_NAME", "e"),
+                ("RERAISE", 1),
+                ("POP_EXCEPT_AND_RERAISE", None)
+            ]
+        )
+
+    # co_positions behavior when info is missing.
+
+    def test_co_positions_empty_linetable(self):
+        def func():
+            x = 1
+        new_code = func.__code__.replace(co_linetable=b'')
+        for line, end_line, column, end_column in new_code.co_positions():
+            self.assertIsNone(line)
+            self.assertEqual(end_line, new_code.co_firstlineno + 1)
+
+    def test_co_positions_empty_endlinetable(self):
+        def func():
+            x = 1
+        new_code = func.__code__.replace(co_endlinetable=b'')
+        for line, end_line, column, end_column in new_code.co_positions():
+            self.assertEqual(line, new_code.co_firstlineno + 1)
+            self.assertIsNone(end_line)
+
+    def test_co_positions_empty_columntable(self):
+        def func():
+            x = 1
+        new_code = func.__code__.replace(co_columntable=b'')
+        for line, end_line, column, end_column in new_code.co_positions():
+            self.assertEqual(line, new_code.co_firstlineno + 1)
+            self.assertEqual(end_line, new_code.co_firstlineno + 1)
+            self.assertIsNone(column)
+            self.assertIsNone(end_column)
+
 
 def isinterned(s):
     return s is sys.intern(('_' + s + '_')[1:-1])

Lib/test/test_compile.py

@@ -3,6 +3,7 @@ import math
 import os
 import unittest
 import sys
+import ast
 import _ast
 import tempfile
 import types
@@ -985,6 +986,119 @@ if 1:
                 self.assertNotEqual(instr.arg, (line + 1)*INSTR_SIZE)
 
 
+class TestSourcePositions(unittest.TestCase):
+    # Ensure that compiled code snippets have correct line and column numbers
+    # in `co_positions()`.
+
+    def check_positions_against_ast(self, snippet):
+        # Basic check that makes sure each line and column is at least present
+        # in one of the AST nodes of the source code.
+        code = compile(snippet, 'test_compile.py', 'exec')
+        ast_tree = compile(snippet, 'test_compile.py', 'exec', _ast.PyCF_ONLY_AST)
+        self.assertTrue(type(ast_tree) == _ast.Module)
+
+        # Use an AST visitor that notes all the offsets.
+        lines, end_lines, columns, end_columns = set(), set(), set(), set()
+        class SourceOffsetVisitor(ast.NodeVisitor):
+            def generic_visit(self, node):
+                super().generic_visit(node)
+                if not isinstance(node, ast.expr) and not isinstance(node, ast.stmt):
+                    return
+                lines.add(node.lineno)
+                end_lines.add(node.end_lineno)
+                columns.add(node.col_offset + 1)
+                end_columns.add(node.end_col_offset + 1)
+
+        SourceOffsetVisitor().visit(ast_tree)
+
+        # Check against the positions in the code object.
+        for (line, end_line, col, end_col) in code.co_positions():
+            # If the offset is not None (indicating missing data), ensure that
+            # it was part of one of the AST nodes.
+            if line is not None:
+                self.assertIn(line, lines)
+            if end_line is not None:
+                self.assertIn(end_line, end_lines)
+            if col is not None:
+                self.assertIn(col, columns)
+            if end_col is not None:
+                self.assertIn(end_col, end_columns)
+
+        return code, ast_tree
+
+    def assertOpcodeSourcePositionIs(self, code, opcode,
+            line, end_line, column, end_column):
+
+        for instr, position in zip(dis.Bytecode(code), code.co_positions()):
+            if instr.opname == opcode:
+                self.assertEqual(position[0], line)
+                self.assertEqual(position[1], end_line)
+                self.assertEqual(position[2], column)
+                self.assertEqual(position[3], end_column)
+                return
+
+        self.fail(f"Opcode {opcode} not found in code")
+
+    def test_simple_assignment(self):
+        snippet = "x = 1"
+        self.check_positions_against_ast(snippet)
+
+    def test_compiles_to_extended_op_arg(self):
+        # Make sure we still have valid positions when the code compiles to an
+        # EXTENDED_ARG by performing a loop which needs a JUMP_ABSOLUTE after
+        # a bunch of opcodes.
+        snippet = "x = x\n" * 10_000
+        snippet += ("while x != 0:\n"
+                    "  x -= 1\n"
+                    "while x != 0:\n"
+                    "  x +=  1\n"
+                   )
+
+        compiled_code, _ = self.check_positions_against_ast(snippet)
+
+        self.assertOpcodeSourcePositionIs(compiled_code, 'INPLACE_SUBTRACT',
+            line=10_000 + 2, end_line=10_000 + 2,
+            column=3, end_column=9)
+        self.assertOpcodeSourcePositionIs(compiled_code, 'INPLACE_ADD',
+            line=10_000 + 4, end_line=10_000 + 4,
+            column=3, end_column=10)
+
+    def test_multiline_expression(self):
+        snippet = """\
+f(
+    1, 2, 3, 4
+)
+"""
+        compiled_code, _ = self.check_positions_against_ast(snippet)
+        self.assertOpcodeSourcePositionIs(compiled_code, 'CALL_FUNCTION',
+            line=1, end_line=3, column=1, end_column=2)
+
+    def test_very_long_line_end_offset(self):
+        # Make sure we get None for when the column offset is too large to
+        # store in a byte.
+        long_string = "a" * 1000
+        snippet = f"g('{long_string}')"
+
+        compiled_code, _ = self.check_positions_against_ast(snippet)
+        self.assertOpcodeSourcePositionIs(compiled_code, 'CALL_FUNCTION',
+            line=1, end_line=1, column=None, end_column=None)
+
+    def test_complex_single_line_expression(self):
+        snippet = "a - b @ (c * x['key'] + 23)"
+
+        compiled_code, _ = self.check_positions_against_ast(snippet)
+        self.assertOpcodeSourcePositionIs(compiled_code, 'BINARY_SUBSCR',
+            line=1, end_line=1, column=14, end_column=22)
+        self.assertOpcodeSourcePositionIs(compiled_code, 'BINARY_MULTIPLY',
+            line=1, end_line=1, column=10, end_column=22)
+        self.assertOpcodeSourcePositionIs(compiled_code, 'BINARY_ADD',
+            line=1, end_line=1, column=10, end_column=27)
+        self.assertOpcodeSourcePositionIs(compiled_code, 'BINARY_MATRIX_MULTIPLY',
+            line=1, end_line=1, column=5, end_column=28)
+        self.assertOpcodeSourcePositionIs(compiled_code, 'BINARY_SUBTRACT',
+            line=1, end_line=1, column=1, end_column=28)
+
+
 class TestExpressionStackSize(unittest.TestCase):
     # These tests check that the computed stack size for a code object
     # stays within reasonable bounds (see issue #21523 for an example

Objects/clinic/codeobject.c.h

@@ -5,8 +5,8 @@ preserve
 PyDoc_STRVAR(code_new__doc__,
 "code(argcount, posonlyargcount, kwonlyargcount, nlocals, stacksize,\n"
 "     flags, codestring, constants, names, varnames, filename, name,\n"
-"     firstlineno, linetable, exceptiontable, freevars=(), cellvars=(),\n"
-"     /)\n"
+"     firstlineno, linetable, endlinetable, columntable, exceptiontable,\n"
+"     freevars=(), cellvars=(), /)\n"
 "--\n"
 "\n"
 "Create a code object.  Not for the faint of heart.");
@@ -16,7 +16,8 @@ code_new_impl(PyTypeObject *type, int argcount, int posonlyargcount,
               int kwonlyargcount, int nlocals, int stacksize, int flags,
               PyObject *code, PyObject *consts, PyObject *names,
               PyObject *varnames, PyObject *filename, PyObject *name,
-              int firstlineno, PyObject *linetable, PyObject *exceptiontable,
+              int firstlineno, PyObject *linetable, PyObject *endlinetable,
+              PyObject *columntable, PyObject *exceptiontable,
               PyObject *freevars, PyObject *cellvars);
 
 static PyObject *
@@ -37,6 +38,8 @@ code_new(PyTypeObject *type, PyObject *args, PyObject *kwargs)
     PyObject *name;
     int firstlineno;
     PyObject *linetable;
+    PyObject *endlinetable;
+    PyObject *columntable;
     PyObject *exceptiontable;
     PyObject *freevars = NULL;
     PyObject *cellvars = NULL;
@@ -45,7 +48,7 @@ code_new(PyTypeObject *type, PyObject *args, PyObject *kwargs)
         !_PyArg_NoKeywords("code", kwargs)) {
         goto exit;
     }
-    if (!_PyArg_CheckPositional("code", PyTuple_GET_SIZE(args), 15, 17)) {
+    if (!_PyArg_CheckPositional("code", PyTuple_GET_SIZE(args), 17, 19)) {
         goto exit;
     }
     argcount = _PyLong_AsInt(PyTuple_GET_ITEM(args, 0));
@@ -121,25 +124,35 @@ code_new(PyTypeObject *type, PyObject *args, PyObject *kwargs)
         _PyArg_BadArgument("code", "argument 15", "bytes", PyTuple_GET_ITEM(args, 14));
         goto exit;
     }
-    exceptiontable = PyTuple_GET_ITEM(args, 14);
-    if (PyTuple_GET_SIZE(args) < 16) {
+    endlinetable = PyTuple_GET_ITEM(args, 14);
+    if (!PyBytes_Check(PyTuple_GET_ITEM(args, 15))) {
+        _PyArg_BadArgument("code", "argument 16", "bytes", PyTuple_GET_ITEM(args, 15));
+        goto exit;
+    }
+    columntable = PyTuple_GET_ITEM(args, 15);
+    if (!PyBytes_Check(PyTuple_GET_ITEM(args, 16))) {
+        _PyArg_BadArgument("code", "argument 17", "bytes", PyTuple_GET_ITEM(args, 16));
+        goto exit;
+    }
+    exceptiontable = PyTuple_GET_ITEM(args, 16);
+    if (PyTuple_GET_SIZE(args) < 18) {
         goto skip_optional;
     }
-    if (!PyTuple_Check(PyTuple_GET_ITEM(args, 15))) {
-        _PyArg_BadArgument("code", "argument 16", "tuple", PyTuple_GET_ITEM(args, 15));
+    if (!PyTuple_Check(PyTuple_GET_ITEM(args, 17))) {
+        _PyArg_BadArgument("code", "argument 18", "tuple", PyTuple_GET_ITEM(args, 17));
         goto exit;
     }
-    freevars = PyTuple_GET_ITEM(args, 15);
-    if (PyTuple_GET_SIZE(args) < 17) {
+    freevars = PyTuple_GET_ITEM(args, 17);
+    if (PyTuple_GET_SIZE(args) < 19) {
         goto skip_optional;
     }
-    if (!PyTuple_Check(PyTuple_GET_ITEM(args, 16))) {
-        _PyArg_BadArgument("code", "argument 17", "tuple", PyTuple_GET_ITEM(args, 16));
+    if (!PyTuple_Check(PyTuple_GET_ITEM(args, 18))) {
+        _PyArg_BadArgument("code", "argument 19", "tuple", PyTuple_GET_ITEM(args, 18));
         goto exit;
     }
-    cellvars = PyTuple_GET_ITEM(args, 16);
+    cellvars = PyTuple_GET_ITEM(args, 18);
 skip_optional:
-    return_value = code_new_impl(type, argcount, posonlyargcount, kwonlyargcount, nlocals, stacksize, flags, code, consts, names, varnames, filename, name, firstlineno, linetable, exceptiontable, freevars, cellvars);
+    return_value = code_new_impl(type, argcount, posonlyargcount, kwonlyargcount, nlocals, stacksize, flags, code, consts, names, varnames, filename, name, firstlineno, linetable, endlinetable, columntable, exceptiontable, freevars, cellvars);
 
 exit:
     return return_value;
@@ -151,7 +164,8 @@ PyDoc_STRVAR(code_replace__doc__,
 "        co_flags=-1, co_firstlineno=-1, co_code=None, co_consts=None,\n"
 "        co_names=None, co_varnames=None, co_freevars=None,\n"
 "        co_cellvars=None, co_filename=None, co_name=None,\n"
-"        co_linetable=None, co_exceptiontable=None)\n"
+"        co_linetable=None, co_endlinetable=None, co_columntable=None,\n"
+"        co_exceptiontable=None)\n"
 "--\n"
 "\n"
 "Return a copy of the code object with new values for the specified fields.");
@@ -168,15 +182,17 @@ code_replace_impl(PyCodeObject *self, int co_argcount,
                   PyObject *co_varnames, PyObject *co_freevars,
                   PyObject *co_cellvars, PyObject *co_filename,
                   PyObject *co_name, PyBytesObject *co_linetable,
+                  PyBytesObject *co_endlinetable,
+                  PyBytesObject *co_columntable,
                   PyBytesObject *co_exceptiontable);
 
 static PyObject *
 code_replace(PyCodeObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames)
 {
     PyObject *return_value = NULL;
-    static const char * const _keywords[] = {"co_argcount", "co_posonlyargcount", "co_kwonlyargcount", "co_nlocals", "co_stacksize", "co_flags", "co_firstlineno", "co_code", "co_consts", "co_names", "co_varnames", "co_freevars", "co_cellvars", "co_filename", "co_name", "co_linetable", "co_exceptiontable", NULL};
+    static const char * const _keywords[] = {"co_argcount", "co_posonlyargcount", "co_kwonlyargcount", "co_nlocals", "co_stacksize", "co_flags", "co_firstlineno", "co_code", "co_consts", "co_names", "co_varnames", "co_freevars", "co_cellvars", "co_filename", "co_name", "co_linetable", "co_endlinetable", "co_columntable", "co_exceptiontable", NULL};
     static _PyArg_Parser _parser = {NULL, _keywords, "replace", 0};
-    PyObject *argsbuf[17];
+    PyObject *argsbuf[19];
     Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0;
     int co_argcount = self->co_argcount;
     int co_posonlyargcount = self->co_posonlyargcount;
@@ -194,6 +210,8 @@ code_replace(PyCodeObject *self, PyObject *const *args, Py_ssize_t nargs, PyObje
     PyObject *co_filename = self->co_filename;
     PyObject *co_name = self->co_name;
     PyBytesObject *co_linetable = (PyBytesObject *)self->co_linetable;
+    PyBytesObject *co_endlinetable = (PyBytesObject *)self->co_endlinetable;
+    PyBytesObject *co_columntable = (PyBytesObject *)self->co_columntable;
     PyBytesObject *co_exceptiontable = (PyBytesObject *)self->co_exceptiontable;
 
     args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 0, 0, argsbuf);
@@ -362,13 +380,33 @@ code_replace(PyCodeObject *self, PyObject *const *args, Py_ssize_t nargs, PyObje
             goto skip_optional_kwonly;
         }
     }
+    if (args[16]) {
         if (!PyBytes_Check(args[16])) {
-        _PyArg_BadArgument("replace", "argument 'co_exceptiontable'", "bytes", args[16]);
+            _PyArg_BadArgument("replace", "argument 'co_endlinetable'", "bytes", args[16]);
+            goto exit;
+        }
+        co_endlinetable = (PyBytesObject *)args[16];
+        if (!--noptargs) {
+            goto skip_optional_kwonly;
+        }
+    }
+    if (args[17]) {
+        if (!PyBytes_Check(args[17])) {
+            _PyArg_BadArgument("replace", "argument 'co_columntable'", "bytes", args[17]);
+            goto exit;
+        }
+        co_columntable = (PyBytesObject *)args[17];
+        if (!--noptargs) {
+            goto skip_optional_kwonly;
+        }
+    }
+    if (!PyBytes_Check(args[18])) {
+        _PyArg_BadArgument("replace", "argument 'co_exceptiontable'", "bytes", args[18]);
         goto exit;
     }
-    co_exceptiontable = (PyBytesObject *)args[16];
+    co_exceptiontable = (PyBytesObject *)args[18];
 skip_optional_kwonly:
-    return_value = code_replace_impl(self, co_argcount, co_posonlyargcount, co_kwonlyargcount, co_nlocals, co_stacksize, co_flags, co_firstlineno, co_code, co_consts, co_names, co_varnames, co_freevars, co_cellvars, co_filename, co_name, co_linetable, co_exceptiontable);
+    return_value = code_replace_impl(self, co_argcount, co_posonlyargcount, co_kwonlyargcount, co_nlocals, co_stacksize, co_flags, co_firstlineno, co_code, co_consts, co_names, co_varnames, co_freevars, co_cellvars, co_filename, co_name, co_linetable, co_endlinetable, co_columntable, co_exceptiontable);
 
 exit:
     return return_value;
@@ -410,4 +448,4 @@ code__varname_from_oparg(PyCodeObject *self, PyObject *const *args, Py_ssize_t n
 exit:
     return return_value;
 }
-/*[clinic end generated code: output=ba4c5487e0364ce8 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=a75c9ca013d9bf7d input=a9049054013a1b77]*/

Objects/codeobject.c

@@ -244,6 +244,10 @@ _PyCode_Validate(struct _PyCodeConstructor *con)
         con->name == NULL || !PyUnicode_Check(con->name) ||
         con->filename == NULL || !PyUnicode_Check(con->filename) ||
         con->linetable == NULL || !PyBytes_Check(con->linetable) ||
+        con->endlinetable == NULL ||
+        (con->endlinetable != Py_None && !PyBytes_Check(con->endlinetable)) ||
+        con->columntable == NULL ||
+        (con->columntable != Py_None && !PyBytes_Check(con->columntable)) ||
         con->exceptiontable == NULL || !PyBytes_Check(con->exceptiontable)
         ) {
         PyErr_BadInternalCall();
@@ -304,6 +308,10 @@ init_code(PyCodeObject *co, struct _PyCodeConstructor *con)
     co->co_firstlineno = con->firstlineno;
     Py_INCREF(con->linetable);
     co->co_linetable = con->linetable;
+    Py_INCREF(con->endlinetable);
+    co->co_endlinetable = con->endlinetable;
+    Py_INCREF(con->columntable);
+    co->co_columntable = con->columntable;
 
     Py_INCREF(con->consts);
     co->co_consts = con->consts;
@@ -386,7 +394,8 @@ PyCode_NewWithPosOnlyArgs(int argcount, int posonlyargcount, int kwonlyargcount,
                           PyObject *code, PyObject *consts, PyObject *names,
                           PyObject *varnames, PyObject *freevars, PyObject *cellvars,
                           PyObject *filename, PyObject *name, int firstlineno,
-                          PyObject *linetable, PyObject *exceptiontable)
+                          PyObject *linetable, PyObject *endlinetable,
+                          PyObject *columntable, PyObject *exceptiontable)
 {
     PyCodeObject *co = NULL;
     PyObject *localsplusnames = NULL;
@@ -461,6 +470,8 @@ PyCode_NewWithPosOnlyArgs(int argcount, int posonlyargcount, int kwonlyargcount,
         .code = code,
         .firstlineno = firstlineno,
         .linetable = linetable,
+        .endlinetable = endlinetable,
+        .columntable = columntable,
 
         .consts = consts,
         .names = names,
@@ -512,12 +523,14 @@ PyCode_New(int argcount, int kwonlyargcount,
            PyObject *code, PyObject *consts, PyObject *names,
            PyObject *varnames, PyObject *freevars, PyObject *cellvars,
            PyObject *filename, PyObject *name, int firstlineno,
-           PyObject *linetable, PyObject *exceptiontable)
+           PyObject *linetable, PyObject *endlinetable,
+           PyObject *columntable, PyObject *exceptiontable)
 {
     return PyCode_NewWithPosOnlyArgs(argcount, 0, kwonlyargcount, nlocals,
                                      stacksize, flags, code, consts, names,
                                      varnames, freevars, cellvars, filename,
-                                     name, firstlineno, linetable, exceptiontable);
+                                     name, firstlineno, linetable, endlinetable,
+                                     columntable, exceptiontable);
 }
 
 PyCodeObject *
@@ -552,6 +565,8 @@ PyCode_NewEmpty(const char *filename, const char *funcname, int firstlineno)
         .code = emptystring,
         .firstlineno = firstlineno,
         .linetable = emptystring,
+        .endlinetable = emptystring,
+        .columntable = emptystring,
         .consts = nulltuple,
         .names = nulltuple,
         .localsplusnames = nulltuple,
@@ -570,7 +585,7 @@ failed:
 
 
 /******************
- * the line table (co_linetable)
+ * source location tracking (co_lines/co_positions)
  ******************/
 
 /* Use co_linetable to compute the line number from a bytecode index, addrq.  See
@@ -589,6 +604,71 @@ PyCode_Addr2Line(PyCodeObject *co, int addrq)
     return _PyCode_CheckLineNumber(addrq, &bounds);
 }
 
+int
+PyCode_Addr2Location(PyCodeObject *co, int addrq,
+                     int *start_line, int *start_column,
+                     int *end_line, int *end_column)
+{
+    *start_line = PyCode_Addr2Line(co, addrq);
+    if (*start_line == -1) {
+        *start_line = 0;
+    }
+    *start_column = _PyCode_Addr2Offset(co, addrq);
+    *end_line = _PyCode_Addr2EndLine(co, addrq);
+    *end_column = _PyCode_Addr2EndOffset(co, addrq);
+    return 1;
+}
+
+int
+_PyCode_Addr2EndLine(PyCodeObject* co, int addrq)
+{
+    if (addrq < 0) {
+        return co->co_firstlineno;
+    }
+    else if (co->co_endlinetable == Py_None) {
+        return 0;
+    }
+
+    assert(addrq >= 0 && addrq < PyBytes_GET_SIZE(co->co_code));
+    PyCodeAddressRange bounds;
+    _PyCode_InitEndAddressRange(co, &bounds);
+    return _PyCode_CheckLineNumber(addrq, &bounds);
+}
+
+int
+_PyCode_Addr2Offset(PyCodeObject* co, int addrq)
+{
+    if (co->co_columntable == Py_None || addrq < 0) {
+        return 0;
+    }
+    if (addrq % 2 == 1) {
+        --addrq;
+    }
+    if (addrq >= PyBytes_GET_SIZE(co->co_columntable)) {
+        return 0;
+    }
+
+    unsigned char* bytes = (unsigned char*)PyBytes_AS_STRING(co->co_columntable);
+    return bytes[addrq];
+}
+
+int
+_PyCode_Addr2EndOffset(PyCodeObject* co, int addrq)
+{
+    if (co->co_columntable == Py_None || addrq < 0) {
+        return 0;
+    }
+    if (addrq % 2 == 0) {
+        ++addrq;
+    }
+    if (addrq >= PyBytes_GET_SIZE(co->co_columntable)) {
+        return 0;
+    }
+
+    unsigned char* bytes = (unsigned char*)PyBytes_AS_STRING(co->co_columntable);
+    return bytes[addrq];
+}
+
 void
 PyLineTable_InitAddressRange(const char *linetable, Py_ssize_t length, int firstlineno, PyCodeAddressRange *range)
 {
@@ -609,6 +689,15 @@ _PyCode_InitAddressRange(PyCodeObject* co, PyCodeAddressRange *bounds)
     return bounds->ar_line;
 }
 
+int
+_PyCode_InitEndAddressRange(PyCodeObject* co, PyCodeAddressRange* bounds)
+{
+    char* linetable = PyBytes_AS_STRING(co->co_endlinetable);
+    Py_ssize_t length = PyBytes_GET_SIZE(co->co_endlinetable);
+    PyLineTable_InitAddressRange(linetable, length, co->co_firstlineno, bounds);
+    return bounds->ar_line;
+}
+
 /* Update *bounds to describe the first and one-past-the-last instructions in
    the same line as lasti.  Return the number of that line, or -1 if lasti is out of bounds. */
 int
@@ -875,6 +964,106 @@ new_linesiterator(PyCodeObject *code)
     return li;
 }
 
+/* co_positions iterator object. */
+typedef struct {
+    PyObject_HEAD
+    PyCodeObject* pi_code;
+    int pi_offset;
+} positionsiterator;
+
+static void
+positionsiter_dealloc(positionsiterator* pi)
+{
+    Py_DECREF(pi->pi_code);
+    Py_TYPE(pi)->tp_free(pi);
+}
+
+static PyObject*
+_source_offset_converter(int* value) {
+    if (*value <= 0) {
+        Py_RETURN_NONE;
+    }
+    return PyLong_FromLong(*value);
+}
+
+static PyObject*
+positionsiter_next(positionsiterator* pi)
+{
+    if (pi->pi_offset >= PyBytes_GET_SIZE(pi->pi_code->co_code)) {
+        return NULL;
+    }
+
+    int start_line, start_col, end_line, end_col;
+    if (!PyCode_Addr2Location(pi->pi_code, pi->pi_offset, &start_line,
+                              &start_col, &end_line, &end_col)) {
+        return NULL;
+    }
+
+    pi->pi_offset += 2;
+    return Py_BuildValue("(O&O&O&O&)",
+        _source_offset_converter, &start_line,
+        _source_offset_converter, &end_line,
+        _source_offset_converter, &start_col,
+        _source_offset_converter, &end_col);
+}
+
+static PyTypeObject PositionsIterator = {
+    PyVarObject_HEAD_INIT(&PyType_Type, 0)
+    "poisitions_iterator",              /* tp_name */
+    sizeof(positionsiterator),          /* tp_basicsize */
+    0,                                  /* tp_itemsize */
+    /* methods */
+    (destructor)positionsiter_dealloc,  /* tp_dealloc */
+    0,                                  /* tp_vectorcall_offset */
+    0,                                  /* tp_getattr */
+    0,                                  /* tp_setattr */
+    0,                                  /* tp_as_async */
+    0,                                  /* tp_repr */
+    0,                                  /* tp_as_number */
+    0,                                  /* tp_as_sequence */
+    0,                                  /* tp_as_mapping */
+    0,                                  /* tp_hash */
+    0,                                  /* tp_call */
+    0,                                  /* tp_str */
+    0,                                  /* tp_getattro */
+    0,                                  /* tp_setattro */
+    0,                                  /* tp_as_buffer */
+    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,       /* tp_flags */
+    0,                                  /* tp_doc */
+    0,                                  /* tp_traverse */
+    0,                                  /* tp_clear */
+    0,                                  /* tp_richcompare */
+    0,                                  /* tp_weaklistoffset */
+    PyObject_SelfIter,                  /* tp_iter */
+    (iternextfunc)positionsiter_next,   /* tp_iternext */
+    0,                                  /* tp_methods */
+    0,                                  /* tp_members */
+    0,                                  /* tp_getset */
+    0,                                  /* tp_base */
+    0,                                  /* tp_dict */
+    0,                                  /* tp_descr_get */
+    0,                                  /* tp_descr_set */
+    0,                                  /* tp_dictoffset */
+    0,                                  /* tp_init */
+    0,                                  /* tp_alloc */
+    0,                                  /* tp_new */
+    PyObject_Del,                       /* tp_free */
+};
+
+static PyObject*
+code_positionsiterator(PyCodeObject* code, PyObject* Py_UNUSED(args))
+{
+    positionsiterator* pi = (positionsiterator*)PyType_GenericAlloc(&PositionsIterator, 0);
+    if (pi == NULL) {
+        return NULL;
+    }
+    Py_INCREF(code);
+    pi->pi_code = code;
+    pi->pi_offset = 0;
+    return (PyObject*)pi;
+}
+
+
 /******************
  * "extra" frame eval info (see PEP 523)
  ******************/
@@ -1026,6 +1215,8 @@ code.__new__ as code_new
     name: unicode
     firstlineno: int
     linetable: object(subclass_of="&PyBytes_Type")
+    endlinetable: object(subclass_of="&PyBytes_Type")
+    columntable: object(subclass_of="&PyBytes_Type")
     exceptiontable: object(subclass_of="&PyBytes_Type")
     freevars: object(subclass_of="&PyTuple_Type", c_default="NULL") = ()
     cellvars: object(subclass_of="&PyTuple_Type", c_default="NULL") = ()
@@ -1039,9 +1230,10 @@ code_new_impl(PyTypeObject *type, int argcount, int posonlyargcount,
               int kwonlyargcount, int nlocals, int stacksize, int flags,
               PyObject *code, PyObject *consts, PyObject *names,
               PyObject *varnames, PyObject *filename, PyObject *name,
-              int firstlineno, PyObject *linetable, PyObject *exceptiontable,
+              int firstlineno, PyObject *linetable, PyObject *endlinetable,
+              PyObject *columntable, PyObject *exceptiontable,
               PyObject *freevars, PyObject *cellvars)
-/*[clinic end generated code: output=a3899259c3b4cace input=f823c686da4b3a03]*/
+/*[clinic end generated code: output=014e77ed052be1a9 input=b22afe3c31be0b6e]*/
 {
     PyObject *co = NULL;
     PyObject *ournames = NULL;
@@ -1108,6 +1300,7 @@ code_new_impl(PyTypeObject *type, int argcount, int posonlyargcount,
                                                ourvarnames, ourfreevars,
                                                ourcellvars, filename,
                                                name, firstlineno, linetable,
+                                               endlinetable, columntable,
                                                exceptiontable
                                               );
   cleanup:
@@ -1147,6 +1340,8 @@ code_dealloc(PyCodeObject *co)
     Py_XDECREF(co->co_filename);
     Py_XDECREF(co->co_name);
     Py_XDECREF(co->co_linetable);
+    Py_XDECREF(co->co_endlinetable);
+    Py_XDECREF(co->co_columntable);
     Py_XDECREF(co->co_exceptiontable);
     if (co->co_weakreflist != NULL)
         PyObject_ClearWeakRefs((PyObject*)co);
@@ -1284,6 +1479,8 @@ static PyMemberDef code_memberlist[] = {
     {"co_name",         T_OBJECT,       OFF(co_name),            READONLY},
     {"co_firstlineno",  T_INT,          OFF(co_firstlineno),     READONLY},
     {"co_linetable",    T_OBJECT,       OFF(co_linetable),       READONLY},
+    {"co_endlinetable", T_OBJECT,       OFF(co_endlinetable),    READONLY},
+    {"co_columntable",  T_OBJECT,       OFF(co_columntable),     READONLY},
     {"co_exceptiontable",    T_OBJECT,  OFF(co_exceptiontable),  READONLY},
     {NULL}      /* Sentinel */
 };
@@ -1377,6 +1574,8 @@ code.replace
     co_filename: unicode(c_default="self->co_filename") = None
     co_name: unicode(c_default="self->co_name") = None
     co_linetable: PyBytesObject(c_default="(PyBytesObject *)self->co_linetable") = None
+    co_endlinetable: PyBytesObject(c_default="(PyBytesObject *)self->co_endlinetable") = None
+    co_columntable: PyBytesObject(c_default="(PyBytesObject *)self->co_columntable") = None
     co_exceptiontable: PyBytesObject(c_default="(PyBytesObject *)self->co_exceptiontable") = None
 
 Return a copy of the code object with new values for the specified fields.
@@ -1391,8 +1590,10 @@ code_replace_impl(PyCodeObject *self, int co_argcount,
                   PyObject *co_varnames, PyObject *co_freevars,
                   PyObject *co_cellvars, PyObject *co_filename,
                   PyObject *co_name, PyBytesObject *co_linetable,
+                  PyBytesObject *co_endlinetable,
+                  PyBytesObject *co_columntable,
                   PyBytesObject *co_exceptiontable)
-/*[clinic end generated code: output=80957472b7f78ed6 input=38376b1193efbbae]*/
+/*[clinic end generated code: output=1189cc8699162b11 input=29c8d25567d86c0d]*/
 {
 #define CHECK_INT_ARG(ARG) \
         if (ARG < 0) { \
@@ -1448,7 +1649,8 @@ code_replace_impl(PyCodeObject *self, int co_argcount,
         co_argcount, co_posonlyargcount, co_kwonlyargcount, co_nlocals,
         co_stacksize, co_flags, (PyObject*)co_code, co_consts, co_names,
         co_varnames, co_freevars, co_cellvars, co_filename, co_name,
-        co_firstlineno, (PyObject*)co_linetable, (PyObject*)co_exceptiontable);
+        co_firstlineno, (PyObject*)co_linetable, (PyObject*)co_endlinetable,
+        (PyObject*)co_columntable, (PyObject*)co_exceptiontable);
 
 error:
     Py_XDECREF(varnames);
@@ -1484,6 +1686,7 @@ code__varname_from_oparg_impl(PyCodeObject *self, int oparg)
 static struct PyMethodDef code_methods[] = {
     {"__sizeof__", (PyCFunction)code_sizeof, METH_NOARGS},
     {"co_lines", (PyCFunction)code_linesiterator, METH_NOARGS},
+    {"co_positions", (PyCFunction)code_positionsiterator, METH_NOARGS},
     CODE_REPLACE_METHODDEF
     CODE__VARNAME_FROM_OPARG_METHODDEF
     {NULL, NULL}                /* sentinel */

Programs/test_frozenmain.h

@@ -23,5 +23,12 @@ unsigned char M_test_frozenmain[] = {
     250,18,116,101,115,116,95,102,114,111,122,101,110,109,97,105,
     110,46,112,121,218,8,60,109,111,100,117,108,101,62,1,0,
     0,0,115,16,0,0,0,8,3,8,1,8,2,12,1,12,
-    1,8,1,26,7,4,249,114,9,0,0,0,
+    1,8,1,26,7,4,249,115,18,0,0,0,8,3,8,1,
+    8,2,12,1,12,1,2,7,4,1,2,249,30,7,115,86,
+    0,0,0,1,11,1,11,1,11,1,11,1,25,1,25,1,
+    25,1,25,1,6,7,27,1,28,1,28,1,6,7,17,19,
+    22,19,27,1,28,1,28,10,27,10,39,10,41,42,50,10,
+    51,1,7,12,2,1,42,1,42,5,8,5,10,11,41,21,
+    24,11,41,11,41,28,34,35,38,28,39,11,41,11,41,5,
+    42,5,42,5,42,1,42,1,42,114,9,0,0,0,
 };

Python/compile.c

@@ -40,6 +40,7 @@
 #define DEFAULT_BLOCKS 8
 #define DEFAULT_CODE_SIZE 128
 #define DEFAULT_LNOTAB_SIZE 16
+#define DEFAULT_CNOTAB_SIZE 0
 
 #define COMP_GENEXP   0
 #define COMP_LISTCOMP 1
@@ -90,6 +91,9 @@ struct instr {
      /* target block when exception is raised, should not be set by front-end. */
     struct basicblock_ *i_except;
     int i_lineno;
+    int i_end_lineno;
+    int i_col_offset;
+    int i_end_col_offset;
 };
 
 typedef struct excepthandler {
@@ -963,6 +967,19 @@ compiler_next_instr(basicblock *b)
     (c)->u->u_end_lineno = (x)->end_lineno;     \
     (c)->u->u_end_col_offset = (x)->end_col_offset;
 
+// Artificial instructions
+#define UNSET_LOC(c)                            \
+    (c)->u->u_lineno = -1;                      \
+    (c)->u->u_col_offset = -1;                  \
+    (c)->u->u_end_lineno = -1;                  \
+    (c)->u->u_end_col_offset = -1;
+
+#define COPY_INSTR_LOC(old, new)                         \
+    (new).i_lineno = (old).i_lineno;                     \
+    (new).i_col_offset = (old).i_col_offset;             \
+    (new).i_end_lineno = (old).i_end_lineno;             \
+    (new).i_end_col_offset = (old).i_end_col_offset;
+
 /* Return the stack effect of opcode with argument oparg.
 
    Some opcodes have different stack effect when jump to the target and
@@ -1266,7 +1283,8 @@ PyCompile_OpcodeStackEffect(int opcode, int oparg)
 */
 
 static int
-compiler_addop_line(struct compiler *c, int opcode, int line)
+compiler_addop_line(struct compiler *c, int opcode, int line,
+                    int end_line, int col_offset, int end_col_offset)
 {
     basicblock *b;
     struct instr *i;
@@ -1282,19 +1300,23 @@ compiler_addop_line(struct compiler *c, int opcode, int line)
     if (opcode == RETURN_VALUE)
         b->b_return = 1;
     i->i_lineno = line;
+    i->i_end_lineno = end_line;
+    i->i_col_offset = col_offset;
+    i->i_end_col_offset = end_col_offset;
     return 1;
 }
 
 static int
 compiler_addop(struct compiler *c, int opcode)
 {
-    return compiler_addop_line(c, opcode, c->u->u_lineno);
+    return compiler_addop_line(c, opcode, c->u->u_lineno, c->u->u_end_lineno,
+                               c->u->u_col_offset, c->u->u_end_col_offset);
 }
 
 static int
 compiler_addop_noline(struct compiler *c, int opcode)
 {
-    return compiler_addop_line(c, opcode, -1);
+    return compiler_addop_line(c, opcode, -1, 0, 0, 0);
 }
 
 
@@ -1488,7 +1510,9 @@ compiler_addop_name(struct compiler *c, int opcode, PyObject *dict,
 */
 
 static int
-compiler_addop_i_line(struct compiler *c, int opcode, Py_ssize_t oparg, int lineno)
+compiler_addop_i_line(struct compiler *c, int opcode, Py_ssize_t oparg,
+                      int lineno, int end_lineno,
+                      int col_offset, int end_col_offset)
 {
     struct instr *i;
     int off;
@@ -1510,22 +1534,30 @@ compiler_addop_i_line(struct compiler *c, int opcode, Py_ssize_t oparg, int line
     i->i_opcode = opcode;
     i->i_oparg = Py_SAFE_DOWNCAST(oparg, Py_ssize_t, int);
     i->i_lineno = lineno;
+    i->i_end_lineno = end_lineno;
+    i->i_col_offset = col_offset;
+    i->i_end_col_offset = end_col_offset;
     return 1;
 }
 
 static int
 compiler_addop_i(struct compiler *c, int opcode, Py_ssize_t oparg)
 {
-    return compiler_addop_i_line(c, opcode, oparg, c->u->u_lineno);
+    return compiler_addop_i_line(c, opcode, oparg,
+                                 c->u->u_lineno, c->u->u_end_lineno,
+                                 c->u->u_col_offset, c->u->u_end_col_offset);
 }
 
 static int
 compiler_addop_i_noline(struct compiler *c, int opcode, Py_ssize_t oparg)
 {
-    return compiler_addop_i_line(c, opcode, oparg, -1);
+    return compiler_addop_i_line(c, opcode, oparg, -1, 0, 0, 0);
 }
 
-static int add_jump_to_block(basicblock *b, int opcode, int lineno, basicblock *target)
+static int add_jump_to_block(basicblock *b, int opcode,
+                             int lineno, int end_lineno,
+                             int col_offset, int end_col_offset,
+                             basicblock *target)
 {
     assert(HAS_ARG(opcode));
     assert(b != NULL);
@@ -1539,19 +1571,24 @@ static int add_jump_to_block(basicblock *b, int opcode, int lineno, basicblock *
     i->i_opcode = opcode;
     i->i_target = target;
     i->i_lineno = lineno;
+    i->i_end_lineno = end_lineno;
+    i->i_col_offset = col_offset;
+    i->i_end_col_offset = end_col_offset;
     return 1;
 }
 
 static int
 compiler_addop_j(struct compiler *c, int opcode, basicblock *b)
 {
-    return add_jump_to_block(c->u->u_curblock, opcode, c->u->u_lineno, b);
+    return add_jump_to_block(c->u->u_curblock, opcode, c->u->u_lineno,
+                             c->u->u_end_lineno, c->u->u_col_offset,
+                             c->u->u_end_col_offset, b);
 }
 
 static int
 compiler_addop_j_noline(struct compiler *c, int opcode, basicblock *b)
 {
-    return add_jump_to_block(c->u->u_curblock, opcode, -1, b);
+    return add_jump_to_block(c->u->u_curblock, opcode, -1, 0, 0, 0, b);
 }
 
 /* NEXT_BLOCK() creates an implicit jump from the current block
@@ -1834,7 +1871,7 @@ compiler_unwind_fblock(struct compiler *c, struct fblockinfo *info,
             /* The finally block should appear to execute after the
              * statement causing the unwinding, so make the unwinding
              * instruction artificial */
-            c->u->u_lineno = -1;
+            UNSET_LOC(c);
             return 1;
 
         case FINALLY_END:
@@ -1870,7 +1907,7 @@ compiler_unwind_fblock(struct compiler *c, struct fblockinfo *info,
             /* The exit block should appear to execute after the
              * statement causing the unwinding, so make the unwinding
              * instruction artificial */
-            c->u->u_lineno = -1;
+            UNSET_LOC(c);
             return 1;
 
         case HANDLER_CLEANUP:
@@ -2522,7 +2559,7 @@ compiler_class(struct compiler *c, stmt_ty s)
             return 0;
         }
         /* The following code is artificial */
-        c->u->u_lineno = -1;
+        UNSET_LOC(c);
         /* Return __classcell__ if it is referenced, otherwise return None */
         if (c->u->u_ste->ste_needs_class_closure) {
             /* Store __classcell__ into class namespace & return it */
@@ -2913,7 +2950,7 @@ compiler_for(struct compiler *c, stmt_ty s)
     VISIT(c, expr, s->v.For.target);
     VISIT_SEQ(c, stmt, s->v.For.body);
     /* Mark jump as artificial */
-    c->u->u_lineno = -1;
+    UNSET_LOC(c);
     ADDOP_JUMP(c, JUMP_ABSOLUTE, start);
     compiler_use_next_block(c, cleanup);
 
@@ -2966,7 +3003,7 @@ compiler_async_for(struct compiler *c, stmt_ty s)
     /* Except block for __anext__ */
     compiler_use_next_block(c, except);
 
-    c->u->u_lineno = -1;
+    UNSET_LOC(c);
     ADDOP(c, END_ASYNC_FOR);
 
     /* `else` block */
@@ -3153,7 +3190,7 @@ compiler_try_finally(struct compiler *c, stmt_ty s)
     /* `finally` block */
     compiler_use_next_block(c, end);
 
-    c->u->u_lineno = -1;
+    UNSET_LOC(c);
     ADDOP_JUMP(c, SETUP_CLEANUP, cleanup);
     ADDOP(c, PUSH_EXC_INFO);
     if (!compiler_push_fblock(c, FINALLY_END, end, NULL, NULL))
@@ -3221,7 +3258,7 @@ compiler_try_except(struct compiler *c, stmt_ty s)
     n = asdl_seq_LEN(s->v.Try.handlers);
     compiler_use_next_block(c, except);
 
-    c->u->u_lineno = -1;
+    UNSET_LOC(c);
     ADDOP_JUMP(c, SETUP_CLEANUP, cleanup);
     ADDOP(c, PUSH_EXC_INFO);
     /* Runtime will push a block here, so we need to account for that */
@@ -3280,7 +3317,7 @@ compiler_try_except(struct compiler *c, stmt_ty s)
             ADDOP(c, POP_BLOCK);
             ADDOP(c, POP_EXCEPT);
             /* name = None; del name; # Mark as artificial */
-            c->u->u_lineno = -1;
+            UNSET_LOC(c);
             ADDOP_LOAD_CONST(c, Py_None);
             compiler_nameop(c, handler->v.ExceptHandler.name, Store);
             compiler_nameop(c, handler->v.ExceptHandler.name, Del);
@@ -3290,7 +3327,7 @@ compiler_try_except(struct compiler *c, stmt_ty s)
             compiler_use_next_block(c, cleanup_end);
 
             /* name = None; del name; # Mark as artificial */
-            c->u->u_lineno = -1;
+            UNSET_LOC(c);
 
             ADDOP_LOAD_CONST(c, Py_None);
             compiler_nameop(c, handler->v.ExceptHandler.name, Store);
@@ -3313,7 +3350,7 @@ compiler_try_except(struct compiler *c, stmt_ty s)
             VISIT_SEQ(c, stmt, handler->v.ExceptHandler.body);
             compiler_pop_fblock(c, HANDLER_CLEANUP, cleanup_body);
             /* name = None; del name; # Mark as artificial */
-            c->u->u_lineno = -1;
+            UNSET_LOC(c);
             ADDOP(c, POP_BLOCK);
             ADDOP(c, POP_EXCEPT);
             ADDOP_JUMP(c, JUMP_FORWARD, end);
@@ -3321,7 +3358,7 @@ compiler_try_except(struct compiler *c, stmt_ty s)
         compiler_use_next_block(c, except);
     }
     /* Mark as artificial */
-    c->u->u_lineno = -1;
+    UNSET_LOC(c);
     compiler_pop_fblock(c, EXCEPTION_HANDLER, NULL);
     ADDOP_I(c, RERAISE, 0);
     compiler_use_next_block(c, cleanup);
@@ -3542,7 +3579,7 @@ compiler_visit_stmt_expr(struct compiler *c, expr_ty value)
 
     VISIT(c, expr, value);
     /* Mark POP_TOP as artificial */
-    c->u->u_lineno = -1;
+    UNSET_LOC(c);
     ADDOP(c, POP_TOP);
     return 1;
 }
@@ -4896,7 +4933,7 @@ compiler_async_comprehension_generator(struct compiler *c,
     compiler_pop_fblock(c, ASYNC_COMPREHENSION_GENERATOR, start);
 
     compiler_use_next_block(c, except);
-    //c->u->u_lineno = -1;
+    //UNSET_LOC(c);
 
     ADDOP(c, END_ASYNC_FOR);
 
@@ -5076,7 +5113,7 @@ compiler_visit_keyword(struct compiler *c, keyword_ty k)
 
 static int
 compiler_with_except_finish(struct compiler *c, basicblock * cleanup) {
-    c->u->u_lineno = -1;
+    UNSET_LOC(c);
     basicblock *exit;
     exit = compiler_new_block(c);
     if (exit == NULL)
@@ -5271,7 +5308,7 @@ compiler_with(struct compiler *c, stmt_ty s, int pos)
 
 
     /* Mark all following code as artificial */
-    c->u->u_lineno = -1;
+    UNSET_LOC(c);
     ADDOP(c, POP_BLOCK);
     compiler_pop_fblock(c, WITH, block);
 
@@ -6546,12 +6583,18 @@ struct assembler {
     int a_offset;              /* offset into bytecode */
     int a_nblocks;             /* number of reachable blocks */
     PyObject *a_lnotab;    /* bytes containing lnotab */
+    PyObject* a_enotab;    /* bytes containing enotab */
+    PyObject* a_cnotab;    /* bytes containing cnotab */
     int a_lnotab_off;      /* offset into lnotab */
+    int a_enotab_off;      /* offset into enotab */
     PyObject *a_except_table;  /* bytes containing exception table */
     int a_except_table_off;    /* offset into exception table */
     int a_prevlineno;     /* lineno of last emitted line in line table */
+    int a_prev_end_lineno; /* end_lineno of last emitted line in line table */
     int a_lineno;          /* lineno of last emitted instruction */
+    int a_end_lineno;      /* end_lineno of last emitted instruction */
     int a_lineno_start;    /* bytecode start offset of current lineno */
+    int a_end_lineno_start; /* bytecode start offset of current end_lineno */
     basicblock *a_entry;
 };
 
@@ -6649,7 +6692,10 @@ assemble_init(struct assembler *a, int nblocks, int firstlineno)
 {
     memset(a, 0, sizeof(struct assembler));
     a->a_prevlineno = a->a_lineno = firstlineno;
+    a->a_prev_end_lineno = a->a_end_lineno = firstlineno;
     a->a_lnotab = NULL;
+    a->a_enotab = NULL;
+    a->a_cnotab = NULL;
     a->a_except_table = NULL;
     a->a_bytecode = PyBytes_FromStringAndSize(NULL, DEFAULT_CODE_SIZE);
     if (a->a_bytecode == NULL) {
@@ -6659,6 +6705,14 @@ assemble_init(struct assembler *a, int nblocks, int firstlineno)
     if (a->a_lnotab == NULL) {
         goto error;
     }
+    a->a_enotab = PyBytes_FromStringAndSize(NULL, DEFAULT_LNOTAB_SIZE);
+    if (a->a_enotab == NULL) {
+        goto error;
+    }
+    a->a_cnotab = PyBytes_FromStringAndSize(NULL, DEFAULT_CNOTAB_SIZE);
+    if (a->a_cnotab == NULL) {
+        goto error;
+    }
     a->a_except_table = PyBytes_FromStringAndSize(NULL, DEFAULT_LNOTAB_SIZE);
     if (a->a_except_table == NULL) {
         goto error;
@@ -6671,6 +6725,8 @@ assemble_init(struct assembler *a, int nblocks, int firstlineno)
 error:
     Py_XDECREF(a->a_bytecode);
     Py_XDECREF(a->a_lnotab);
+    Py_XDECREF(a->a_enotab);
+    Py_XDECREF(a->a_cnotab);
     Py_XDECREF(a->a_except_table);
     return 0;
 }
@@ -6680,6 +6736,8 @@ assemble_free(struct assembler *a)
 {
     Py_XDECREF(a->a_bytecode);
     Py_XDECREF(a->a_lnotab);
+    Py_XDECREF(a->a_enotab);
+    Py_XDECREF(a->a_cnotab);
     Py_XDECREF(a->a_except_table);
 }
 
@@ -6695,18 +6753,21 @@ blocksize(basicblock *b)
 }
 
 static int
-assemble_emit_linetable_pair(struct assembler *a, int bdelta, int ldelta)
+assemble_emit_table_pair(struct assembler* a, PyObject** table, int* offset,
+                         int left, int right)
 {
-    Py_ssize_t len = PyBytes_GET_SIZE(a->a_lnotab);
-    if (a->a_lnotab_off + 2 >= len) {
-        if (_PyBytes_Resize(&a->a_lnotab, len * 2) < 0)
+    Py_ssize_t len = PyBytes_GET_SIZE(*table);
+    if (*offset + 2 >= len) {
+        if (_PyBytes_Resize(table, len * 2) < 0)
             return 0;
     }
-    unsigned char *lnotab = (unsigned char *) PyBytes_AS_STRING(a->a_lnotab);
-    lnotab += a->a_lnotab_off;
-    a->a_lnotab_off += 2;
-    *lnotab++ = bdelta;
-    *lnotab++ = ldelta;
+    unsigned char* table_entry = (unsigned char*)PyBytes_AS_STRING(*table);
+
+    table_entry += *offset;
+    *offset += 2;
+
+    *table_entry++ = left;
+    *table_entry++ = right;
     return 1;
 }
 
@@ -6961,27 +7022,28 @@ assemble_exception_table(struct assembler *a)
  *  Objects/lnotab_notes.txt for the description of the line number table. */
 
 static int
-assemble_line_range(struct assembler *a)
+assemble_line_range(struct assembler* a, int current, PyObject** table,
+                    int* prev, int* start, int* offset)
 {
     int ldelta, bdelta;
-    bdelta =  (a->a_offset - a->a_lineno_start) * 2;
+    bdelta = (a->a_offset - *start) * 2;
     if (bdelta == 0) {
         return 1;
     }
-    if (a->a_lineno < 0) {
+    if (current < 0) {
         ldelta = -128;
     }
     else {
-        ldelta = a->a_lineno - a->a_prevlineno;
-        a->a_prevlineno = a->a_lineno;
+        ldelta = current - *prev;
+        *prev = current;
         while (ldelta > 127) {
-            if (!assemble_emit_linetable_pair(a, 0, 127)) {
+            if (!assemble_emit_table_pair(a, table, offset, 0, 127)) {
                 return 0;
             }
             ldelta -= 127;
         }
         while (ldelta < -127) {
-            if (!assemble_emit_linetable_pair(a, 0, -127)) {
+            if (!assemble_emit_table_pair(a, table, offset, 0, -127)) {
                 return 0;
             }
             ldelta += 127;
@@ -6989,32 +7051,82 @@ assemble_line_range(struct assembler *a)
     }
     assert(-128 <= ldelta && ldelta < 128);
     while (bdelta > 254) {
-        if (!assemble_emit_linetable_pair(a, 254, ldelta)) {
+        if (!assemble_emit_table_pair(a, table, offset, 254, ldelta)) {
             return 0;
         }
-        ldelta = a->a_lineno < 0 ? -128 : 0;
+        ldelta = current < 0 ? -128 : 0;
         bdelta -= 254;
     }
-    if (!assemble_emit_linetable_pair(a, bdelta, ldelta)) {
+    if (!assemble_emit_table_pair(a, table, offset, bdelta, ldelta)) {
         return 0;
     }
-    a->a_lineno_start = a->a_offset;
+    *start = a->a_offset;
     return 1;
 }
 
 static int
-assemble_lnotab(struct assembler *a, struct instr *i)
+assemble_start_line_range(struct assembler* a) {
+    return assemble_line_range(a, a->a_lineno, &a->a_lnotab,
+        &a->a_prevlineno, &a->a_lineno_start, &a->a_lnotab_off);
+}
+
+static int
+assemble_end_line_range(struct assembler* a) {
+    return assemble_line_range(a, a->a_end_lineno, &a->a_enotab,
+        &a->a_prev_end_lineno, &a->a_end_lineno_start, &a->a_enotab_off);
+}
+
+static int
+assemble_lnotab(struct assembler* a, struct instr* i)
 {
     if (i->i_lineno == a->a_lineno) {
         return 1;
     }
-    if (!assemble_line_range(a)) {
+    if (!assemble_start_line_range(a)) {
         return 0;
     }
     a->a_lineno = i->i_lineno;
     return 1;
 }
 
+static int
+assemble_enotab(struct assembler* a, struct instr* i)
+{
+    if (i->i_end_lineno == a->a_end_lineno) {
+        return 1;
+    }
+    if (!assemble_end_line_range(a)) {
+        return 0;
+    }
+    a->a_end_lineno = i->i_end_lineno;
+    return 1;
+}
+
+static int
+assemble_cnotab(struct assembler* a, struct instr* i, int instr_size)
+{
+    Py_ssize_t len = PyBytes_GET_SIZE(a->a_cnotab);
+    // TODO: Allocate more memory than just what we immediately need
+    //       like a_lnotab does.
+    if (_PyBytes_Resize(&a->a_cnotab, len + (instr_size * 2)) < 0) {
+        return 0;
+    }
+
+    unsigned char* cnotab = (unsigned char*)PyBytes_AS_STRING(a->a_cnotab);
+    cnotab += len;
+
+    for (int j = 0; j < instr_size; j++) {
+        if (i->i_col_offset > 255 || i->i_end_col_offset > 255) {
+            *cnotab++ = 0;
+            *cnotab++ = 0;
+            continue;
+        }
+        *cnotab++ = i->i_col_offset + 1;
+        *cnotab++ = i->i_end_col_offset + 1;
+    }
+    return 1;
+}
+
 
 /* assemble_emit()
    Extend the bytecode with a new instruction.
@@ -7030,8 +7142,15 @@ assemble_emit(struct assembler *a, struct instr *i)
 
     arg = i->i_oparg;
     size = instrsize(arg);
-    if (i->i_lineno && !assemble_lnotab(a, i))
+    if (i->i_lineno && !assemble_lnotab(a, i)) {
+        return 0;
+    }
+    if (!assemble_enotab(a, i)) {
+        return 0;
+    }
+    if (!assemble_cnotab(a, i, size)) {
         return 0;
+    }
     if (a->a_offset + size >= len / (int)sizeof(_Py_CODEUNIT)) {
         if (len > PY_SSIZE_T_MAX / 2)
             return 0;
@@ -7315,6 +7434,8 @@ makecode(struct compiler *c, struct assembler *a, PyObject *constslist,
         .code = a->a_bytecode,
         .firstlineno = c->u->u_firstlineno,
         .linetable = a->a_lnotab,
+        .endlinetable = a->a_enotab,
+        .columntable = a->a_cnotab,
 
         .consts = consts,
         .names = names,
@@ -7478,6 +7599,9 @@ insert_prefix_instructions(struct compiler *c, basicblock *entryblock,
                 // This will get fixed in offset_derefs().
                 .i_oparg = oldindex,
                 .i_lineno = -1,
+                .i_col_offset = -1,
+                .i_end_lineno = -1,
+                .i_end_col_offset = -1,
                 .i_target = NULL,
             };
             if (insert_instruction(entryblock, ncellsused, &make_cell) < 0) {
@@ -7505,6 +7629,9 @@ insert_prefix_instructions(struct compiler *c, basicblock *entryblock,
             .i_opcode = GEN_START,
             .i_oparg = kind,
             .i_lineno = -1,
+            .i_col_offset = -1,
+            .i_end_lineno = -1,
+            .i_end_col_offset = -1,
             .i_target = NULL,
         };
         if (insert_instruction(entryblock, 0, &gen_start) < 0) {
@@ -7602,7 +7729,7 @@ assemble(struct compiler *c, int addNone)
        block ends with a jump or return b_next shouldn't set.
      */
     if (!c->u->u_curblock->b_return) {
-        c->u->u_lineno = -1;
+        UNSET_LOC(c);
         if (addNone)
             ADDOP_LOAD_CONST(c, Py_None);
         ADDOP(c, RETURN_VALUE);
@@ -7707,19 +7834,25 @@ assemble(struct compiler *c, int addNone)
     if (!assemble_exception_table(&a)) {
         goto error;
     }
-    if (!assemble_line_range(&a)) {
+    if (_PyBytes_Resize(&a.a_except_table, a.a_except_table_off) < 0) {
         goto error;
     }
-
+    if (!assemble_start_line_range(&a)) {
+        return 0;
+    }
     if (_PyBytes_Resize(&a.a_lnotab, a.a_lnotab_off) < 0) {
         goto error;
     }
-
-    if (_PyBytes_Resize(&a.a_except_table, a.a_except_table_off) < 0) {
+    if (!merge_const_one(c, &a.a_lnotab)) {
         goto error;
     }
-
-    if (!merge_const_one(c, &a.a_lnotab)) {
+    if (!assemble_end_line_range(&a)) {
+        return 0;
+    }
+    if (_PyBytes_Resize(&a.a_enotab, a.a_enotab_off) < 0) {
+        goto error;
+    }
+    if (!merge_const_one(c, &a.a_enotab)) {
         goto error;
     }
     if (_PyBytes_Resize(&a.a_bytecode, a.a_offset * sizeof(_Py_CODEUNIT)) < 0) {
@@ -7848,7 +7981,11 @@ eliminate_jump_to_jump(basicblock *bb, int opcode) {
         return 0;
     }
     int lineno = target->i_lineno;
-    if (add_jump_to_block(bb, opcode, lineno, target->i_target) == 0) {
+    int end_lineno = target->i_end_lineno;
+    int col_offset = target->i_col_offset;
+    int end_col_offset = target->i_end_col_offset;
+    if (add_jump_to_block(bb, opcode, lineno, end_lineno, col_offset,
+                          end_col_offset, target->i_target) == 0) {
         return -1;
     }
     assert (bb->b_iused >= 2);
@@ -8127,7 +8264,7 @@ clean_basic_block(basicblock *bb) {
             if (src < bb->b_iused - 1) {
                 int next_lineno = bb->b_instr[src+1].i_lineno;
                 if (next_lineno < 0 || next_lineno == lineno) {
-                    bb->b_instr[src+1].i_lineno = lineno;
+                    COPY_INSTR_LOC(bb->b_instr[src], bb->b_instr[src+1]);
                     continue;
                 }
             }
@@ -8262,19 +8399,27 @@ propogate_line_numbers(struct assembler *a) {
         if (b->b_iused == 0) {
             continue;
         }
-        int prev_lineno = -1;
+
+        // Not a real instruction, only to store positions
+        // from previous instructions and propagate them.
+        struct instr prev_instr = {
+            .i_lineno = -1,
+            .i_col_offset = -1,
+            .i_end_lineno = -1,
+            .i_end_col_offset = -1,
+        };
         for (int i = 0; i < b->b_iused; i++) {
             if (b->b_instr[i].i_lineno < 0) {
-                b->b_instr[i].i_lineno = prev_lineno;
+                COPY_INSTR_LOC(prev_instr, b->b_instr[i]);
             }
             else {
-                prev_lineno = b->b_instr[i].i_lineno;
+                COPY_INSTR_LOC(b->b_instr[i], prev_instr);
             }
         }
         if (!b->b_nofallthrough && b->b_next->b_predecessors == 1) {
             assert(b->b_next->b_iused);
             if (b->b_next->b_instr[0].i_lineno < 0) {
-                b->b_next->b_instr[0].i_lineno = prev_lineno;
+                COPY_INSTR_LOC(prev_instr, b->b_next->b_instr[0]);
             }
         }
         if (is_jump(&b->b_instr[b->b_iused-1])) {
@@ -8288,7 +8433,7 @@ propogate_line_numbers(struct assembler *a) {
             basicblock *target = b->b_instr[b->b_iused-1].i_target;
             if (target->b_predecessors == 1) {
                 if (target->b_instr[0].i_lineno < 0) {
-                    target->b_instr[0].i_lineno = prev_lineno;
+                    COPY_INSTR_LOC(prev_instr, target->b_instr[0]);
                 }
             }
         }
@@ -8391,7 +8536,7 @@ ensure_exits_have_lineno(struct compiler *c)
                 if (new_target == NULL) {
                     return -1;
                 }
-                new_target->b_instr[0].i_lineno = b->b_instr[b->b_iused-1].i_lineno;
+                COPY_INSTR_LOC(b->b_instr[b->b_iused-1], new_target->b_instr[0]);
                 b->b_instr[b->b_iused-1].i_target = new_target;
             }
         }
@@ -8412,7 +8557,7 @@ ensure_exits_have_lineno(struct compiler *c)
         if (!b->b_nofallthrough && b->b_next && b->b_iused > 0) {
             if (is_exit_without_lineno(b->b_next)) {
                 assert(b->b_next->b_iused > 0);
-                b->b_next->b_instr[0].i_lineno = b->b_instr[b->b_iused-1].i_lineno;
+                COPY_INSTR_LOC(b->b_instr[b->b_iused-1],  b->b_next->b_instr[0]);
             }
         }
     }

Python/frozen_hello.h

@@ -7,6 +7,7 @@ const unsigned char _Py_M__hello[] = {
     105,110,105,116,105,97,108,105,122,101,100,218,5,112,114,105,
     110,116,169,0,243,0,0,0,0,122,14,60,102,114,111,122,
     101,110,32,104,101,108,108,111,62,218,8,60,109,111,100,117,
-    108,101,62,1,0,0,0,115,4,0,0,0,4,0,12,1,
-    114,2,0,0,0,
+    108,101,62,1,0,0,0,243,4,0,0,0,4,0,12,1,
+    114,4,0,0,0,115,16,0,0,0,15,19,1,12,1,6,
+    7,21,1,22,1,22,1,22,1,22,114,2,0,0,0,
 };

Python/marshal.c

@@ -524,6 +524,8 @@ w_complex_object(PyObject *v, char flag, WFILE *p)
         w_object(co->co_name, p);
         w_long(co->co_firstlineno, p);
         w_object(co->co_linetable, p);
+        w_object(co->co_endlinetable, p);
+        w_object(co->co_columntable, p);
         w_object(co->co_exceptiontable, p);
     }
     else if (PyObject_CheckBuffer(v)) {
@@ -1315,6 +1317,8 @@ r_object(RFILE *p)
             PyObject *name = NULL;
             int firstlineno;
             PyObject *linetable = NULL;
+            PyObject* endlinetable = NULL;
+            PyObject* columntable = NULL;
             PyObject *exceptiontable = NULL;
 
             idx = r_ref_reserve(flag, p);
@@ -1367,6 +1371,12 @@ r_object(RFILE *p)
             linetable = r_object(p);
             if (linetable == NULL)
                 goto code_error;
+            endlinetable = r_object(p);
+            if (endlinetable == NULL)
+                goto code_error;
+            columntable = r_object(p);
+            if (columntable == NULL)
+                goto code_error;
             exceptiontable = r_object(p);
             if (exceptiontable == NULL)
                 goto code_error;
@@ -1379,6 +1389,8 @@ r_object(RFILE *p)
                 .code = code,
                 .firstlineno = firstlineno,
                 .linetable = linetable,
+                .endlinetable = endlinetable,
+                .columntable = columntable,
 
                 .consts = consts,
                 .names = names,
@@ -1415,6 +1427,8 @@ r_object(RFILE *p)
             Py_XDECREF(filename);
             Py_XDECREF(name);
             Py_XDECREF(linetable);
+            Py_XDECREF(endlinetable);
+            Py_XDECREF(columntable);
             Py_XDECREF(exceptiontable);
         }
         retval = v;