Issue #27942: String constants now interned recursively in tuples and frozensets.

9 years ago · 00a0fc1144
5 changed files with 951 additions and 875 deletions
--- a/Lib/test/test_code.py
+++ b/Lib/test/test_code.py
@ -102,6 +102,7 @@ consts: ('None',)

 """

+import sys
 import unittest
 import weakref
 from test.support import run_doctest, run_unittest, cpython_only
@ -134,6 +135,43 @@ class CodeTest(unittest.TestCase):
        self.assertEqual(co.co_name, "funcname")
        self.assertEqual(co.co_firstlineno, 15)

+class CodeConstsTest(unittest.TestCase):
+
+    def find_const(self, consts, value):
+        for v in consts:
+            if v == value:
+                return v
+        self.assertIn(value, consts)  # rises an exception
+        self.fail('Should be never reached')
+
+    def assertIsInterned(self, s):
+        if s is not sys.intern(s):
+            self.fail('String %r is not interned' % (s,))
+
+    @cpython_only
+    def test_interned_string(self):
+        co = compile('res = "str_value"', '?', 'exec')
+        v = self.find_const(co.co_consts, 'str_value')
+        self.assertIsInterned(v)
+
+    @cpython_only
+    def test_interned_string_in_tuple(self):
+        co = compile('res = ("str_value",)', '?', 'exec')
+        v = self.find_const(co.co_consts, ('str_value',))
+        self.assertIsInterned(v[0])
+
+    @cpython_only
+    def test_interned_string_in_frozenset(self):
+        co = compile('res = a in {"str_value"}', '?', 'exec')
+        v = self.find_const(co.co_consts, frozenset(('str_value',)))
+        self.assertIsInterned(tuple(v)[0])
+
+    @cpython_only
+    def test_interned_string_default(self):
+        def f(a='str_value'):
+            return a
+        self.assertIsInterned(f())
+

 class CodeWeakRefTest(unittest.TestCase):

@ -163,7 +201,7 @@ class CodeWeakRefTest(unittest.TestCase):
 def test_main(verbose=None):
    from test import test_code
    run_doctest(test_code, verbose)
-    run_unittest(CodeTest, CodeWeakRefTest)
+    run_unittest(CodeTest, CodeConstsTest, CodeWeakRefTest)


 if __name__ == "__main__":
--- a/Misc/NEWS
+++ b/Misc/NEWS
@ -10,6 +10,8 @@ Release date: TBA
 Core and Builtins
 -----------------

+- Issue #27942: String constants now interned recursively in tuples and frozensets.
+
 - Issue #21578: Fixed misleading error message when ImportError called with
  invalid keyword args.

--- a/Objects/codeobject.c
+++ b/Objects/codeobject.c
@ -46,6 +46,50 @@ intern_strings(PyObject *tuple)
    }
 }

+/* Intern selected string constants */
+static int
+intern_string_constants(PyObject *tuple)
+{
+    int modified = 0;
+    Py_ssize_t i;
+
+    for (i = PyTuple_GET_SIZE(tuple); --i >= 0; ) {
+        PyObject *v = PyTuple_GET_ITEM(tuple, i);
+        if (PyUnicode_CheckExact(v)) {
+            if (all_name_chars(v)) {
+                PyObject *w = v;
+                PyUnicode_InternInPlace(&v);
+                if (w != v) {
+                    PyTuple_SET_ITEM(tuple, i, v);
+                    modified = 1;
+                }
+            }
+        }
+        else if (PyTuple_CheckExact(v)) {
+            intern_string_constants(v);
+        }
+        else if (PyFrozenSet_CheckExact(v)) {
+            PyObject *tmp = PySequence_Tuple(v);
+            if (tmp == NULL) {
+                PyErr_Clear();
+                continue;
+            }
+            if (intern_string_constants(tmp)) {
+                v = PyFrozenSet_New(tmp);
+                if (v == NULL) {
+                    PyErr_Clear();
+                }
+                else {
+                    PyTuple_SET_ITEM(tuple, i, v);
+                    modified = 1;
+                }
+            }
+            Py_DECREF(tmp);
+        }
+    }
+    return modified;
+}
+

 PyCodeObject *
 PyCode_New(int argcount, int kwonlyargcount,
@ -84,13 +128,7 @@ PyCode_New(int argcount, int kwonlyargcount,
    intern_strings(varnames);
    intern_strings(freevars);
    intern_strings(cellvars);
-    /* Intern selected string constants */
-    for (i = PyTuple_GET_SIZE(consts); --i >= 0; ) {
-        PyObject *v = PyTuple_GetItem(consts, i);
-        if (!all_name_chars(v))
-            continue;
-        PyUnicode_InternInPlace(&PyTuple_GET_ITEM(consts, i));
-    }
+    intern_string_constants(consts);
    /* Create mapping between cells and arguments if needed. */
    if (n_cellvars) {
        Py_ssize_t total_args = argcount + kwonlyargcount +
--- a/Python/importlib.h
+++ b/Python/importlib.h
@ -1920,70 +1920,69 @@ const unsigned char _Py_M__importlib[] = {
    115,32,109,117,115,116,32,98,101,32,101,120,112,108,105,99,
    105,116,108,121,32,112,97,115,115,101,100,32,105,110,46,10,
    10,32,32,32,32,114,141,0,0,0,114,34,0,0,0,78,
-    114,62,0,0,0,41,1,122,9,95,119,97,114,110,105,110,
-    103,115,41,16,114,57,0,0,0,114,14,0,0,0,114,13,
-    0,0,0,114,21,0,0,0,218,5,105,116,101,109,115,114,
-    177,0,0,0,114,76,0,0,0,114,150,0,0,0,114,82,
-    0,0,0,114,160,0,0,0,114,132,0,0,0,114,137,0,
-    0,0,114,1,0,0,0,114,200,0,0,0,114,5,0,0,
-    0,114,77,0,0,0,41,12,218,10,115,121,115,95,109,111,
-    100,117,108,101,218,11,95,105,109,112,95,109,111,100,117,108,
-    101,90,11,109,111,100,117,108,101,95,116,121,112,101,114,15,
-    0,0,0,114,89,0,0,0,114,99,0,0,0,114,88,0,
-    0,0,90,11,115,101,108,102,95,109,111,100,117,108,101,90,
-    12,98,117,105,108,116,105,110,95,110,97,109,101,90,14,98,
-    117,105,108,116,105,110,95,109,111,100,117,108,101,90,13,116,
-    104,114,101,97,100,95,109,111,100,117,108,101,90,14,119,101,
-    97,107,114,101,102,95,109,111,100,117,108,101,114,10,0,0,
-    0,114,10,0,0,0,114,11,0,0,0,218,6,95,115,101,
-    116,117,112,61,4,0,0,115,50,0,0,0,0,9,6,1,
-    6,3,12,1,28,1,15,1,15,1,9,1,15,1,9,2,
-    3,1,15,1,17,3,13,1,13,1,15,1,15,2,13,1,
-    20,3,3,1,16,1,13,2,11,1,16,3,12,1,114,204,
-    0,0,0,99,2,0,0,0,0,0,0,0,3,0,0,0,
-    3,0,0,0,67,0,0,0,115,87,0,0,0,116,0,0,
-    124,0,0,124,1,0,131,2,0,1,116,1,0,106,2,0,
-    106,3,0,116,4,0,131,1,0,1,116,1,0,106,2,0,
-    106,3,0,116,5,0,131,1,0,1,100,1,0,100,2,0,
-    108,6,0,125,2,0,124,2,0,97,7,0,124,2,0,106,
-    8,0,116,1,0,106,9,0,116,10,0,25,131,1,0,1,
-    100,2,0,83,41,3,122,50,73,110,115,116,97,108,108,32,
-    105,109,112,111,114,116,108,105,98,32,97,115,32,116,104,101,
-    32,105,109,112,108,101,109,101,110,116,97,116,105,111,110,32,
-    111,102,32,105,109,112,111,114,116,46,114,33,0,0,0,78,
-    41,11,114,204,0,0,0,114,14,0,0,0,114,174,0,0,
-    0,114,113,0,0,0,114,150,0,0,0,114,160,0,0,0,
-    218,26,95,102,114,111,122,101,110,95,105,109,112,111,114,116,
-    108,105,98,95,101,120,116,101,114,110,97,108,114,119,0,0,
-    0,218,8,95,105,110,115,116,97,108,108,114,21,0,0,0,
-    114,1,0,0,0,41,3,114,202,0,0,0,114,203,0,0,
-    0,114,205,0,0,0,114,10,0,0,0,114,10,0,0,0,
-    114,11,0,0,0,114,206,0,0,0,108,4,0,0,115,12,
-    0,0,0,0,2,13,2,16,1,16,3,12,1,6,1,114,
-    206,0,0,0,41,51,114,3,0,0,0,114,119,0,0,0,
-    114,12,0,0,0,114,16,0,0,0,114,17,0,0,0,114,
-    59,0,0,0,114,41,0,0,0,114,48,0,0,0,114,31,
-    0,0,0,114,32,0,0,0,114,53,0,0,0,114,54,0,
-    0,0,114,56,0,0,0,114,63,0,0,0,114,65,0,0,
-    0,114,75,0,0,0,114,81,0,0,0,114,84,0,0,0,
-    114,90,0,0,0,114,101,0,0,0,114,102,0,0,0,114,
-    106,0,0,0,114,85,0,0,0,218,6,111,98,106,101,99,
-    116,90,9,95,80,79,80,85,76,65,84,69,114,132,0,0,
-    0,114,137,0,0,0,114,144,0,0,0,114,97,0,0,0,
-    114,86,0,0,0,114,148,0,0,0,114,149,0,0,0,114,
-    87,0,0,0,114,150,0,0,0,114,160,0,0,0,114,165,
-    0,0,0,114,171,0,0,0,114,173,0,0,0,114,176,0,
-    0,0,114,181,0,0,0,114,191,0,0,0,114,182,0,0,
-    0,114,184,0,0,0,114,185,0,0,0,114,186,0,0,0,
-    114,194,0,0,0,114,196,0,0,0,114,199,0,0,0,114,
-    200,0,0,0,114,204,0,0,0,114,206,0,0,0,114,10,
-    0,0,0,114,10,0,0,0,114,10,0,0,0,114,11,0,
-    0,0,218,8,60,109,111,100,117,108,101,62,8,0,0,0,
-    115,96,0,0,0,6,17,6,2,12,8,12,4,19,20,6,
-    2,6,3,22,4,19,68,19,21,19,19,12,19,12,19,12,
-    11,18,8,12,11,12,12,12,16,12,36,19,27,19,101,24,
-    26,9,3,18,45,18,60,12,18,12,17,12,25,12,29,12,
-    23,12,16,19,73,19,77,19,13,12,9,12,9,15,40,12,
-    17,6,1,10,2,12,27,12,6,18,24,12,32,12,15,24,
-    35,12,7,12,47,
+    114,62,0,0,0,41,1,114,141,0,0,0,41,16,114,57,
+    0,0,0,114,14,0,0,0,114,13,0,0,0,114,21,0,
+    0,0,218,5,105,116,101,109,115,114,177,0,0,0,114,76,
+    0,0,0,114,150,0,0,0,114,82,0,0,0,114,160,0,
+    0,0,114,132,0,0,0,114,137,0,0,0,114,1,0,0,
+    0,114,200,0,0,0,114,5,0,0,0,114,77,0,0,0,
+    41,12,218,10,115,121,115,95,109,111,100,117,108,101,218,11,
+    95,105,109,112,95,109,111,100,117,108,101,90,11,109,111,100,
+    117,108,101,95,116,121,112,101,114,15,0,0,0,114,89,0,
+    0,0,114,99,0,0,0,114,88,0,0,0,90,11,115,101,
+    108,102,95,109,111,100,117,108,101,90,12,98,117,105,108,116,
+    105,110,95,110,97,109,101,90,14,98,117,105,108,116,105,110,
+    95,109,111,100,117,108,101,90,13,116,104,114,101,97,100,95,
+    109,111,100,117,108,101,90,14,119,101,97,107,114,101,102,95,
+    109,111,100,117,108,101,114,10,0,0,0,114,10,0,0,0,
+    114,11,0,0,0,218,6,95,115,101,116,117,112,61,4,0,
+    0,115,50,0,0,0,0,9,6,1,6,3,12,1,28,1,
+    15,1,15,1,9,1,15,1,9,2,3,1,15,1,17,3,
+    13,1,13,1,15,1,15,2,13,1,20,3,3,1,16,1,
+    13,2,11,1,16,3,12,1,114,204,0,0,0,99,2,0,
+    0,0,0,0,0,0,3,0,0,0,3,0,0,0,67,0,
+    0,0,115,87,0,0,0,116,0,0,124,0,0,124,1,0,
+    131,2,0,1,116,1,0,106,2,0,106,3,0,116,4,0,
+    131,1,0,1,116,1,0,106,2,0,106,3,0,116,5,0,
+    131,1,0,1,100,1,0,100,2,0,108,6,0,125,2,0,
+    124,2,0,97,7,0,124,2,0,106,8,0,116,1,0,106,
+    9,0,116,10,0,25,131,1,0,1,100,2,0,83,41,3,
+    122,50,73,110,115,116,97,108,108,32,105,109,112,111,114,116,
+    108,105,98,32,97,115,32,116,104,101,32,105,109,112,108,101,
+    109,101,110,116,97,116,105,111,110,32,111,102,32,105,109,112,
+    111,114,116,46,114,33,0,0,0,78,41,11,114,204,0,0,
+    0,114,14,0,0,0,114,174,0,0,0,114,113,0,0,0,
+    114,150,0,0,0,114,160,0,0,0,218,26,95,102,114,111,
+    122,101,110,95,105,109,112,111,114,116,108,105,98,95,101,120,
+    116,101,114,110,97,108,114,119,0,0,0,218,8,95,105,110,
+    115,116,97,108,108,114,21,0,0,0,114,1,0,0,0,41,
+    3,114,202,0,0,0,114,203,0,0,0,114,205,0,0,0,
+    114,10,0,0,0,114,10,0,0,0,114,11,0,0,0,114,
+    206,0,0,0,108,4,0,0,115,12,0,0,0,0,2,13,
+    2,16,1,16,3,12,1,6,1,114,206,0,0,0,41,51,
+    114,3,0,0,0,114,119,0,0,0,114,12,0,0,0,114,
+    16,0,0,0,114,17,0,0,0,114,59,0,0,0,114,41,
+    0,0,0,114,48,0,0,0,114,31,0,0,0,114,32,0,
+    0,0,114,53,0,0,0,114,54,0,0,0,114,56,0,0,
+    0,114,63,0,0,0,114,65,0,0,0,114,75,0,0,0,
+    114,81,0,0,0,114,84,0,0,0,114,90,0,0,0,114,
+    101,0,0,0,114,102,0,0,0,114,106,0,0,0,114,85,
+    0,0,0,218,6,111,98,106,101,99,116,90,9,95,80,79,
+    80,85,76,65,84,69,114,132,0,0,0,114,137,0,0,0,
+    114,144,0,0,0,114,97,0,0,0,114,86,0,0,0,114,
+    148,0,0,0,114,149,0,0,0,114,87,0,0,0,114,150,
+    0,0,0,114,160,0,0,0,114,165,0,0,0,114,171,0,
+    0,0,114,173,0,0,0,114,176,0,0,0,114,181,0,0,
+    0,114,191,0,0,0,114,182,0,0,0,114,184,0,0,0,
+    114,185,0,0,0,114,186,0,0,0,114,194,0,0,0,114,
+    196,0,0,0,114,199,0,0,0,114,200,0,0,0,114,204,
+    0,0,0,114,206,0,0,0,114,10,0,0,0,114,10,0,
+    0,0,114,10,0,0,0,114,11,0,0,0,218,8,60,109,
+    111,100,117,108,101,62,8,0,0,0,115,96,0,0,0,6,
+    17,6,2,12,8,12,4,19,20,6,2,6,3,22,4,19,
+    68,19,21,19,19,12,19,12,19,12,11,18,8,12,11,12,
+    12,12,16,12,36,19,27,19,101,24,26,9,3,18,45,18,
+    60,12,18,12,17,12,25,12,29,12,23,12,16,19,73,19,
+    77,19,13,12,9,12,9,15,40,12,17,6,1,10,2,12,
+    27,12,6,18,24,12,32,12,15,24,35,12,7,12,47,
 };
--- a/Python/importlib_external.h
+++ b/Python/importlib_external.h