Browse Source
Issue #28038: Remove Tools/parser/com2ann.py and its unit test.
Issue #28038: Remove Tools/parser/com2ann.py and its unit test.
Development is moving to https://github.com/ilevkivskyi/com2annpull/9921/head
Guido van Rossum
10 years ago
2 changed files with 0 additions and 568 deletions
@ -1,260 +0,0 @@ |
|||||
"""Tests for the com2ann.py script in the Tools/parser directory.""" |
|
||||
|
|
||||
import unittest |
|
||||
import test.support |
|
||||
import os |
|
||||
import re |
|
||||
|
|
||||
from test.test_tools import basepath, toolsdir, skip_if_missing |
|
||||
|
|
||||
skip_if_missing() |
|
||||
|
|
||||
parser_path = os.path.join(toolsdir, "parser") |
|
||||
|
|
||||
with test.support.DirsOnSysPath(parser_path): |
|
||||
from com2ann import * |
|
||||
|
|
||||
class BaseTestCase(unittest.TestCase): |
|
||||
|
|
||||
def check(self, code, expected, n=False, e=False): |
|
||||
self.assertEqual(com2ann(code, |
|
||||
drop_None=n, drop_Ellipsis=e, silent=True), |
|
||||
expected) |
|
||||
|
|
||||
class SimpleTestCase(BaseTestCase): |
|
||||
# Tests for basic conversions |
|
||||
|
|
||||
def test_basics(self): |
|
||||
self.check("z = 5", "z = 5") |
|
||||
self.check("z: int = 5", "z: int = 5") |
|
||||
self.check("z = 5 # type: int", "z: int = 5") |
|
||||
self.check("z = 5 # type: int # comment", |
|
||||
"z: int = 5 # comment") |
|
||||
|
|
||||
def test_type_ignore(self): |
|
||||
self.check("foobar = foobaz() #type: ignore", |
|
||||
"foobar = foobaz() #type: ignore") |
|
||||
self.check("a = 42 #type: ignore #comment", |
|
||||
"a = 42 #type: ignore #comment") |
|
||||
|
|
||||
def test_complete_tuple(self): |
|
||||
self.check("t = 1, 2, 3 # type: Tuple[int, ...]", |
|
||||
"t: Tuple[int, ...] = (1, 2, 3)") |
|
||||
self.check("t = 1, # type: Tuple[int]", |
|
||||
"t: Tuple[int] = (1,)") |
|
||||
self.check("t = (1, 2, 3) # type: Tuple[int, ...]", |
|
||||
"t: Tuple[int, ...] = (1, 2, 3)") |
|
||||
|
|
||||
def test_drop_None(self): |
|
||||
self.check("x = None # type: int", |
|
||||
"x: int", True) |
|
||||
self.check("x = None # type: int # another", |
|
||||
"x: int # another", True) |
|
||||
self.check("x = None # type: int # None", |
|
||||
"x: int # None", True) |
|
||||
|
|
||||
def test_drop_Ellipsis(self): |
|
||||
self.check("x = ... # type: int", |
|
||||
"x: int", False, True) |
|
||||
self.check("x = ... # type: int # another", |
|
||||
"x: int # another", False, True) |
|
||||
self.check("x = ... # type: int # ...", |
|
||||
"x: int # ...", False, True) |
|
||||
|
|
||||
def test_newline(self): |
|
||||
self.check("z = 5 # type: int\r\n", "z: int = 5\r\n") |
|
||||
self.check("z = 5 # type: int # comment\x85", |
|
||||
"z: int = 5 # comment\x85") |
|
||||
|
|
||||
def test_wrong(self): |
|
||||
self.check("#type : str", "#type : str") |
|
||||
self.check("x==y #type: bool", "x==y #type: bool") |
|
||||
|
|
||||
def test_pattern(self): |
|
||||
for line in ["#type: int", " # type: str[:] # com"]: |
|
||||
self.assertTrue(re.search(TYPE_COM, line)) |
|
||||
for line in ["", "#", "# comment", "#type", "type int:"]: |
|
||||
self.assertFalse(re.search(TYPE_COM, line)) |
|
||||
|
|
||||
class BigTestCase(BaseTestCase): |
|
||||
# Tests for really crazy formatting, to be sure |
|
||||
# that script works reasonably in extreme situations |
|
||||
|
|
||||
def test_crazy(self): |
|
||||
self.maxDiff = None |
|
||||
self.check(crazy_code, big_result, False, False) |
|
||||
self.check(crazy_code, big_result_ne, True, True) |
|
||||
|
|
||||
crazy_code = """\ |
|
||||
# -*- coding: utf-8 -*- # this should not be spoiled |
|
||||
''' |
|
||||
Docstring here |
|
||||
''' |
|
||||
|
|
||||
import testmod |
|
||||
x = 5 #type : int # this one is OK |
|
||||
ttt \\ |
|
||||
= \\ |
|
||||
1.0, \\ |
|
||||
2.0, \\ |
|
||||
3.0, #type: Tuple[float, float, float] |
|
||||
with foo(x==1) as f: #type: str |
|
||||
print(f) |
|
||||
|
|
||||
for i, j in my_inter(x=1): # type: ignore |
|
||||
i + j # type: int # what about this |
|
||||
|
|
||||
x = y = z = 1 # type: int |
|
||||
x, y, z = [], [], [] # type: (List[int], List[int], List[str]) |
|
||||
class C: |
|
||||
|
|
||||
|
|
||||
l[f(x |
|
||||
=1)] = [ |
|
||||
|
|
||||
1, |
|
||||
2, |
|
||||
] # type: List[int] |
|
||||
|
|
||||
|
|
||||
(C.x[1]) = \\ |
|
||||
42 == 5# type: bool |
|
||||
lst[...] = \\ |
|
||||
((\\ |
|
||||
...)) # type: int # comment .. |
|
||||
|
|
||||
y = ... # type: int # comment ... |
|
||||
z = ... |
|
||||
#type: int |
|
||||
|
|
||||
|
|
||||
#DONE placement of annotation after target rather than before = |
|
||||
|
|
||||
TD.x[1] \\ |
|
||||
= 0 == 5# type: bool |
|
||||
|
|
||||
TD.y[1] =5 == 5# type: bool # one more here |
|
||||
F[G(x == y, |
|
||||
|
|
||||
# hm... |
|
||||
|
|
||||
z)]\\ |
|
||||
= None # type: OMG[int] # comment: None |
|
||||
x = None#type:int #comment : None""" |
|
||||
|
|
||||
big_result = """\ |
|
||||
# -*- coding: utf-8 -*- # this should not be spoiled |
|
||||
''' |
|
||||
Docstring here |
|
||||
''' |
|
||||
|
|
||||
import testmod |
|
||||
x: int = 5 # this one is OK |
|
||||
ttt: Tuple[float, float, float] \\ |
|
||||
= \\ |
|
||||
(1.0, \\ |
|
||||
2.0, \\ |
|
||||
3.0,) |
|
||||
with foo(x==1) as f: #type: str |
|
||||
print(f) |
|
||||
|
|
||||
for i, j in my_inter(x=1): # type: ignore |
|
||||
i + j # type: int # what about this |
|
||||
|
|
||||
x = y = z = 1 # type: int |
|
||||
x, y, z = [], [], [] # type: (List[int], List[int], List[str]) |
|
||||
class C: |
|
||||
|
|
||||
|
|
||||
l[f(x |
|
||||
=1)]: List[int] = [ |
|
||||
|
|
||||
1, |
|
||||
2, |
|
||||
] |
|
||||
|
|
||||
|
|
||||
(C.x[1]): bool = \\ |
|
||||
42 == 5 |
|
||||
lst[...]: int = \\ |
|
||||
((\\ |
|
||||
...)) # comment .. |
|
||||
|
|
||||
y: int = ... # comment ... |
|
||||
z = ... |
|
||||
#type: int |
|
||||
|
|
||||
|
|
||||
#DONE placement of annotation after target rather than before = |
|
||||
|
|
||||
TD.x[1]: bool \\ |
|
||||
= 0 == 5 |
|
||||
|
|
||||
TD.y[1]: bool =5 == 5 # one more here |
|
||||
F[G(x == y, |
|
||||
|
|
||||
# hm... |
|
||||
|
|
||||
z)]: OMG[int]\\ |
|
||||
= None # comment: None |
|
||||
x: int = None #comment : None""" |
|
||||
|
|
||||
big_result_ne = """\ |
|
||||
# -*- coding: utf-8 -*- # this should not be spoiled |
|
||||
''' |
|
||||
Docstring here |
|
||||
''' |
|
||||
|
|
||||
import testmod |
|
||||
x: int = 5 # this one is OK |
|
||||
ttt: Tuple[float, float, float] \\ |
|
||||
= \\ |
|
||||
(1.0, \\ |
|
||||
2.0, \\ |
|
||||
3.0,) |
|
||||
with foo(x==1) as f: #type: str |
|
||||
print(f) |
|
||||
|
|
||||
for i, j in my_inter(x=1): # type: ignore |
|
||||
i + j # type: int # what about this |
|
||||
|
|
||||
x = y = z = 1 # type: int |
|
||||
x, y, z = [], [], [] # type: (List[int], List[int], List[str]) |
|
||||
class C: |
|
||||
|
|
||||
|
|
||||
l[f(x |
|
||||
=1)]: List[int] = [ |
|
||||
|
|
||||
1, |
|
||||
2, |
|
||||
] |
|
||||
|
|
||||
|
|
||||
(C.x[1]): bool = \\ |
|
||||
42 == 5 |
|
||||
lst[...]: int \\ |
|
||||
\\ |
|
||||
# comment .. |
|
||||
|
|
||||
y: int # comment ... |
|
||||
z = ... |
|
||||
#type: int |
|
||||
|
|
||||
|
|
||||
#DONE placement of annotation after target rather than before = |
|
||||
|
|
||||
TD.x[1]: bool \\ |
|
||||
= 0 == 5 |
|
||||
|
|
||||
TD.y[1]: bool =5 == 5 # one more here |
|
||||
F[G(x == y, |
|
||||
|
|
||||
# hm... |
|
||||
|
|
||||
z)]: OMG[int]\\ |
|
||||
# comment: None |
|
||||
x: int #comment : None""" |
|
||||
|
|
||||
if __name__ == '__main__': |
|
||||
unittest.main() |
|
||||
@ -1,308 +0,0 @@ |
|||||
"""Helper module to tranlate 3.5 type comments to 3.6 variable annotations.""" |
|
||||
import re |
|
||||
import os |
|
||||
import ast |
|
||||
import argparse |
|
||||
import tokenize |
|
||||
from collections import defaultdict |
|
||||
from textwrap import dedent |
|
||||
from io import BytesIO |
|
||||
|
|
||||
__all__ = ['com2ann', 'TYPE_COM'] |
|
||||
|
|
||||
TYPE_COM = re.compile(r'\s*#\s*type\s*:.*$', flags=re.DOTALL) |
|
||||
TRAIL_OR_COM = re.compile(r'\s*$|\s*#.*$', flags=re.DOTALL) |
|
||||
|
|
||||
|
|
||||
class _Data: |
|
||||
"""Internal class describing global data on file.""" |
|
||||
def __init__(self, lines, tokens): |
|
||||
self.lines = lines |
|
||||
self.tokens = tokens |
|
||||
ttab = defaultdict(list) # maps line number to token numbers |
|
||||
for i, tok in enumerate(tokens): |
|
||||
ttab[tok.start[0]].append(i) |
|
||||
self.ttab = ttab |
|
||||
self.success = [] # list of lines where type comments where processed |
|
||||
self.fail = [] # list of lines where type comments where rejected |
|
||||
|
|
||||
|
|
||||
def skip_blank(d, lno): |
|
||||
while d.lines[lno].strip() == '': |
|
||||
lno += 1 |
|
||||
return lno |
|
||||
|
|
||||
|
|
||||
def find_start(d, lcom): |
|
||||
"""Find first char of the assignment target.""" |
|
||||
i = d.ttab[lcom + 1][-2] # index of type comment token in tokens list |
|
||||
while ((d.tokens[i].exact_type != tokenize.NEWLINE) and |
|
||||
(d.tokens[i].exact_type != tokenize.ENCODING)): |
|
||||
i -= 1 |
|
||||
lno = d.tokens[i].start[0] |
|
||||
return skip_blank(d, lno) |
|
||||
|
|
||||
|
|
||||
def check_target(stmt): |
|
||||
if len(stmt.body): |
|
||||
assign = stmt.body[0] |
|
||||
else: |
|
||||
return False |
|
||||
if isinstance(assign, ast.Assign) and len(assign.targets) == 1: |
|
||||
targ = assign.targets[0] |
|
||||
else: |
|
||||
return False |
|
||||
if (isinstance(targ, ast.Name) or isinstance(targ, ast.Attribute) |
|
||||
or isinstance(targ, ast.Subscript)): |
|
||||
return True |
|
||||
return False |
|
||||
|
|
||||
|
|
||||
def find_eq(d, lstart): |
|
||||
"""Find equal sign starting from lstart taking care about d[f(x=1)] = 5.""" |
|
||||
col = pars = 0 |
|
||||
lno = lstart |
|
||||
while d.lines[lno][col] != '=' or pars != 0: |
|
||||
ch = d.lines[lno][col] |
|
||||
if ch in '([{': |
|
||||
pars += 1 |
|
||||
elif ch in ')]}': |
|
||||
pars -= 1 |
|
||||
if ch == '#' or col == len(d.lines[lno])-1: |
|
||||
lno = skip_blank(d, lno+1) |
|
||||
col = 0 |
|
||||
else: |
|
||||
col += 1 |
|
||||
return lno, col |
|
||||
|
|
||||
|
|
||||
def find_val(d, poseq): |
|
||||
"""Find position of first char of assignment value starting from poseq.""" |
|
||||
lno, col = poseq |
|
||||
while (d.lines[lno][col].isspace() or d.lines[lno][col] in '=\\'): |
|
||||
if col == len(d.lines[lno])-1: |
|
||||
lno += 1 |
|
||||
col = 0 |
|
||||
else: |
|
||||
col += 1 |
|
||||
return lno, col |
|
||||
|
|
||||
|
|
||||
def find_targ(d, poseq): |
|
||||
"""Find position of last char of target (annotation goes here).""" |
|
||||
lno, col = poseq |
|
||||
while (d.lines[lno][col].isspace() or d.lines[lno][col] in '=\\'): |
|
||||
if col == 0: |
|
||||
lno -= 1 |
|
||||
col = len(d.lines[lno])-1 |
|
||||
else: |
|
||||
col -= 1 |
|
||||
return lno, col+1 |
|
||||
|
|
||||
|
|
||||
def trim(new_lines, string, ltarg, poseq, lcom, ccom): |
|
||||
"""Remove None or Ellipsis from assignment value. |
|
||||
|
|
||||
Also remove parens if one has (None), (...) etc. |
|
||||
string -- 'None' or '...' |
|
||||
ltarg -- line where last char of target is located |
|
||||
poseq -- position of equal sign |
|
||||
lcom, ccom -- position of type comment |
|
||||
""" |
|
||||
nopars = lambda s: s.replace('(', '').replace(')', '') |
|
||||
leq, ceq = poseq |
|
||||
end = ccom if leq == lcom else len(new_lines[leq]) |
|
||||
subline = new_lines[leq][:ceq] |
|
||||
if leq == ltarg: |
|
||||
subline = subline.rstrip() |
|
||||
new_lines[leq] = subline + (new_lines[leq][end:] if leq == lcom |
|
||||
else new_lines[leq][ceq+1:end]) |
|
||||
|
|
||||
for lno in range(leq+1,lcom): |
|
||||
new_lines[lno] = nopars(new_lines[lno]) |
|
||||
|
|
||||
if lcom != leq: |
|
||||
subline = nopars(new_lines[lcom][:ccom]).replace(string, '') |
|
||||
if (not subline.isspace()): |
|
||||
subline = subline.rstrip() |
|
||||
new_lines[lcom] = subline + new_lines[lcom][ccom:] |
|
||||
|
|
||||
|
|
||||
def _com2ann(d, drop_None, drop_Ellipsis): |
|
||||
new_lines = d.lines[:] |
|
||||
for lcom, line in enumerate(d.lines): |
|
||||
match = re.search(TYPE_COM, line) |
|
||||
if match: |
|
||||
# strip " # type : annotation \n" -> "annotation \n" |
|
||||
tp = match.group().lstrip()[1:].lstrip()[4:].lstrip()[1:].lstrip() |
|
||||
submatch = re.search(TRAIL_OR_COM, tp) |
|
||||
subcom = '' |
|
||||
if submatch and submatch.group(): |
|
||||
subcom = submatch.group() |
|
||||
tp = tp[:submatch.start()] |
|
||||
if tp == 'ignore': |
|
||||
continue |
|
||||
ccom = match.start() |
|
||||
if not any(d.tokens[i].exact_type == tokenize.COMMENT |
|
||||
for i in d.ttab[lcom + 1]): |
|
||||
d.fail.append(lcom) |
|
||||
continue # type comment inside string |
|
||||
lstart = find_start(d, lcom) |
|
||||
stmt_str = dedent(''.join(d.lines[lstart:lcom+1])) |
|
||||
try: |
|
||||
stmt = ast.parse(stmt_str) |
|
||||
except SyntaxError: |
|
||||
d.fail.append(lcom) |
|
||||
continue # for or with statements |
|
||||
if not check_target(stmt): |
|
||||
d.fail.append(lcom) |
|
||||
continue |
|
||||
|
|
||||
d.success.append(lcom) |
|
||||
val = stmt.body[0].value |
|
||||
|
|
||||
# writing output now |
|
||||
poseq = find_eq(d, lstart) |
|
||||
lval, cval = find_val(d, poseq) |
|
||||
ltarg, ctarg = find_targ(d, poseq) |
|
||||
|
|
||||
op_par = '' |
|
||||
cl_par = '' |
|
||||
if isinstance(val, ast.Tuple): |
|
||||
if d.lines[lval][cval] != '(': |
|
||||
op_par = '(' |
|
||||
cl_par = ')' |
|
||||
# write the comment first |
|
||||
new_lines[lcom] = d.lines[lcom][:ccom].rstrip() + cl_par + subcom |
|
||||
ccom = len(d.lines[lcom][:ccom].rstrip()) |
|
||||
|
|
||||
string = False |
|
||||
if isinstance(val, ast.Tuple): |
|
||||
# t = 1, 2 -> t = (1, 2); only latter is allowed with annotation |
|
||||
free_place = int(new_lines[lval][cval-2:cval] == ' ') |
|
||||
new_lines[lval] = (new_lines[lval][:cval-free_place] + |
|
||||
op_par + new_lines[lval][cval:]) |
|
||||
elif isinstance(val, ast.Ellipsis) and drop_Ellipsis: |
|
||||
string = '...' |
|
||||
elif (isinstance(val, ast.NameConstant) and |
|
||||
val.value is None and drop_None): |
|
||||
string = 'None' |
|
||||
if string: |
|
||||
trim(new_lines, string, ltarg, poseq, lcom, ccom) |
|
||||
|
|
||||
# finally write an annotation |
|
||||
new_lines[ltarg] = (new_lines[ltarg][:ctarg] + |
|
||||
': ' + tp + new_lines[ltarg][ctarg:]) |
|
||||
return ''.join(new_lines) |
|
||||
|
|
||||
|
|
||||
def com2ann(code, *, drop_None=False, drop_Ellipsis=False, silent=False): |
|
||||
"""Translate type comments to type annotations in code. |
|
||||
|
|
||||
Take code as string and return this string where:: |
|
||||
|
|
||||
variable = value # type: annotation # real comment |
|
||||
|
|
||||
is translated to:: |
|
||||
|
|
||||
variable: annotation = value # real comment |
|
||||
|
|
||||
For unsupported syntax cases, the type comments are |
|
||||
left intact. If drop_None is True or if drop_Ellipsis |
|
||||
is True translate correcpondingly:: |
|
||||
|
|
||||
variable = None # type: annotation |
|
||||
variable = ... # type: annotation |
|
||||
|
|
||||
into:: |
|
||||
|
|
||||
variable: annotation |
|
||||
|
|
||||
The tool tries to preserve code formatting as much as |
|
||||
possible, but an exact translation is not guarateed. |
|
||||
A summary of translated comments id printed by default. |
|
||||
""" |
|
||||
try: |
|
||||
ast.parse(code) # we want to work only with file without syntax errors |
|
||||
except SyntaxError: |
|
||||
return None |
|
||||
lines = code.splitlines(keepends=True) |
|
||||
rl = BytesIO(code.encode('utf-8')).readline |
|
||||
tokens = list(tokenize.tokenize(rl)) |
|
||||
|
|
||||
data = _Data(lines, tokens) |
|
||||
new_code = _com2ann(data, drop_None, drop_Ellipsis) |
|
||||
|
|
||||
if not silent: |
|
||||
if data.success: |
|
||||
print('Comments translated on lines:', |
|
||||
', '.join(str(lno+1) for lno in data.success)) |
|
||||
if data.fail: |
|
||||
print('Comments rejected on lines:', |
|
||||
', '.join(str(lno+1) for lno in data.fail)) |
|
||||
if not data.success and not data.fail: |
|
||||
print('No type comments found') |
|
||||
|
|
||||
return new_code |
|
||||
|
|
||||
|
|
||||
def translate_file(infile, outfile, dnone, dell, silent): |
|
||||
try: |
|
||||
descr = tokenize.open(infile) |
|
||||
except SyntaxError: |
|
||||
print("Cannot open", infile) |
|
||||
return |
|
||||
with descr as f: |
|
||||
code = f.read() |
|
||||
enc = f.encoding |
|
||||
if not silent: |
|
||||
print('File:', infile) |
|
||||
new_code = com2ann(code, drop_None=dnone, |
|
||||
drop_Ellipsis=dell, |
|
||||
silent=silent) |
|
||||
if new_code is None: |
|
||||
print("SyntaxError in", infile) |
|
||||
return |
|
||||
with open(outfile, 'wb') as f: |
|
||||
f.write((new_code).encode(enc)) |
|
||||
|
|
||||
|
|
||||
if __name__ == '__main__': |
|
||||
|
|
||||
parser = argparse.ArgumentParser(description=__doc__) |
|
||||
parser.add_argument("-o", "--outfile", |
|
||||
help="output file, will be overwritten if exists,\n" |
|
||||
"defaults to input file") |
|
||||
parser.add_argument("infile", |
|
||||
help="input file or directory for translation, must\n" |
|
||||
"contain no syntax errors, for directory\n" |
|
||||
"the outfile is ignored and translation is\n" |
|
||||
"made in place") |
|
||||
parser.add_argument("-s", "--silent", |
|
||||
help="Do not print summary for line numbers of\n" |
|
||||
"translated and rejected comments", |
|
||||
action="store_true") |
|
||||
parser.add_argument("-n", "--drop-none", |
|
||||
help="drop any None as assignment value during\n" |
|
||||
"translation if it is annotated by a type coment", |
|
||||
action="store_true") |
|
||||
parser.add_argument("-e", "--drop-ellipsis", |
|
||||
help="drop any Ellipsis (...) as assignment value during\n" |
|
||||
"translation if it is annotated by a type coment", |
|
||||
action="store_true") |
|
||||
args = parser.parse_args() |
|
||||
if args.outfile is None: |
|
||||
args.outfile = args.infile |
|
||||
|
|
||||
if os.path.isfile(args.infile): |
|
||||
translate_file(args.infile, args.outfile, |
|
||||
args.drop_none, args.drop_ellipsis, args.silent) |
|
||||
else: |
|
||||
for root, dirs, files in os.walk(args.infile): |
|
||||
for afile in files: |
|
||||
_, ext = os.path.splitext(afile) |
|
||||
if ext == '.py' or ext == '.pyi': |
|
||||
fname = os.path.join(root, afile) |
|
||||
translate_file(fname, fname, |
|
||||
args.drop_none, args.drop_ellipsis, |
|
||||
args.silent) |
|
||||
Write
Preview
Loading…
Cancel
Save
Reference in new issue