Browse Source
Issue #28038: Remove Tools/parser/com2ann.py and its unit test.
Issue #28038: Remove Tools/parser/com2ann.py and its unit test.
Development is moving to https://github.com/ilevkivskyi/com2annpull/9921/head
Guido van Rossum
10 years ago
2 changed files with 0 additions and 568 deletions
@ -1,260 +0,0 @@ |
|||
"""Tests for the com2ann.py script in the Tools/parser directory.""" |
|||
|
|||
import unittest |
|||
import test.support |
|||
import os |
|||
import re |
|||
|
|||
from test.test_tools import basepath, toolsdir, skip_if_missing |
|||
|
|||
skip_if_missing() |
|||
|
|||
parser_path = os.path.join(toolsdir, "parser") |
|||
|
|||
with test.support.DirsOnSysPath(parser_path): |
|||
from com2ann import * |
|||
|
|||
class BaseTestCase(unittest.TestCase): |
|||
|
|||
def check(self, code, expected, n=False, e=False): |
|||
self.assertEqual(com2ann(code, |
|||
drop_None=n, drop_Ellipsis=e, silent=True), |
|||
expected) |
|||
|
|||
class SimpleTestCase(BaseTestCase): |
|||
# Tests for basic conversions |
|||
|
|||
def test_basics(self): |
|||
self.check("z = 5", "z = 5") |
|||
self.check("z: int = 5", "z: int = 5") |
|||
self.check("z = 5 # type: int", "z: int = 5") |
|||
self.check("z = 5 # type: int # comment", |
|||
"z: int = 5 # comment") |
|||
|
|||
def test_type_ignore(self): |
|||
self.check("foobar = foobaz() #type: ignore", |
|||
"foobar = foobaz() #type: ignore") |
|||
self.check("a = 42 #type: ignore #comment", |
|||
"a = 42 #type: ignore #comment") |
|||
|
|||
def test_complete_tuple(self): |
|||
self.check("t = 1, 2, 3 # type: Tuple[int, ...]", |
|||
"t: Tuple[int, ...] = (1, 2, 3)") |
|||
self.check("t = 1, # type: Tuple[int]", |
|||
"t: Tuple[int] = (1,)") |
|||
self.check("t = (1, 2, 3) # type: Tuple[int, ...]", |
|||
"t: Tuple[int, ...] = (1, 2, 3)") |
|||
|
|||
def test_drop_None(self): |
|||
self.check("x = None # type: int", |
|||
"x: int", True) |
|||
self.check("x = None # type: int # another", |
|||
"x: int # another", True) |
|||
self.check("x = None # type: int # None", |
|||
"x: int # None", True) |
|||
|
|||
def test_drop_Ellipsis(self): |
|||
self.check("x = ... # type: int", |
|||
"x: int", False, True) |
|||
self.check("x = ... # type: int # another", |
|||
"x: int # another", False, True) |
|||
self.check("x = ... # type: int # ...", |
|||
"x: int # ...", False, True) |
|||
|
|||
def test_newline(self): |
|||
self.check("z = 5 # type: int\r\n", "z: int = 5\r\n") |
|||
self.check("z = 5 # type: int # comment\x85", |
|||
"z: int = 5 # comment\x85") |
|||
|
|||
def test_wrong(self): |
|||
self.check("#type : str", "#type : str") |
|||
self.check("x==y #type: bool", "x==y #type: bool") |
|||
|
|||
def test_pattern(self): |
|||
for line in ["#type: int", " # type: str[:] # com"]: |
|||
self.assertTrue(re.search(TYPE_COM, line)) |
|||
for line in ["", "#", "# comment", "#type", "type int:"]: |
|||
self.assertFalse(re.search(TYPE_COM, line)) |
|||
|
|||
class BigTestCase(BaseTestCase): |
|||
# Tests for really crazy formatting, to be sure |
|||
# that script works reasonably in extreme situations |
|||
|
|||
def test_crazy(self): |
|||
self.maxDiff = None |
|||
self.check(crazy_code, big_result, False, False) |
|||
self.check(crazy_code, big_result_ne, True, True) |
|||
|
|||
crazy_code = """\ |
|||
# -*- coding: utf-8 -*- # this should not be spoiled |
|||
''' |
|||
Docstring here |
|||
''' |
|||
|
|||
import testmod |
|||
x = 5 #type : int # this one is OK |
|||
ttt \\ |
|||
= \\ |
|||
1.0, \\ |
|||
2.0, \\ |
|||
3.0, #type: Tuple[float, float, float] |
|||
with foo(x==1) as f: #type: str |
|||
print(f) |
|||
|
|||
for i, j in my_inter(x=1): # type: ignore |
|||
i + j # type: int # what about this |
|||
|
|||
x = y = z = 1 # type: int |
|||
x, y, z = [], [], [] # type: (List[int], List[int], List[str]) |
|||
class C: |
|||
|
|||
|
|||
l[f(x |
|||
=1)] = [ |
|||
|
|||
1, |
|||
2, |
|||
] # type: List[int] |
|||
|
|||
|
|||
(C.x[1]) = \\ |
|||
42 == 5# type: bool |
|||
lst[...] = \\ |
|||
((\\ |
|||
...)) # type: int # comment .. |
|||
|
|||
y = ... # type: int # comment ... |
|||
z = ... |
|||
#type: int |
|||
|
|||
|
|||
#DONE placement of annotation after target rather than before = |
|||
|
|||
TD.x[1] \\ |
|||
= 0 == 5# type: bool |
|||
|
|||
TD.y[1] =5 == 5# type: bool # one more here |
|||
F[G(x == y, |
|||
|
|||
# hm... |
|||
|
|||
z)]\\ |
|||
= None # type: OMG[int] # comment: None |
|||
x = None#type:int #comment : None""" |
|||
|
|||
big_result = """\ |
|||
# -*- coding: utf-8 -*- # this should not be spoiled |
|||
''' |
|||
Docstring here |
|||
''' |
|||
|
|||
import testmod |
|||
x: int = 5 # this one is OK |
|||
ttt: Tuple[float, float, float] \\ |
|||
= \\ |
|||
(1.0, \\ |
|||
2.0, \\ |
|||
3.0,) |
|||
with foo(x==1) as f: #type: str |
|||
print(f) |
|||
|
|||
for i, j in my_inter(x=1): # type: ignore |
|||
i + j # type: int # what about this |
|||
|
|||
x = y = z = 1 # type: int |
|||
x, y, z = [], [], [] # type: (List[int], List[int], List[str]) |
|||
class C: |
|||
|
|||
|
|||
l[f(x |
|||
=1)]: List[int] = [ |
|||
|
|||
1, |
|||
2, |
|||
] |
|||
|
|||
|
|||
(C.x[1]): bool = \\ |
|||
42 == 5 |
|||
lst[...]: int = \\ |
|||
((\\ |
|||
...)) # comment .. |
|||
|
|||
y: int = ... # comment ... |
|||
z = ... |
|||
#type: int |
|||
|
|||
|
|||
#DONE placement of annotation after target rather than before = |
|||
|
|||
TD.x[1]: bool \\ |
|||
= 0 == 5 |
|||
|
|||
TD.y[1]: bool =5 == 5 # one more here |
|||
F[G(x == y, |
|||
|
|||
# hm... |
|||
|
|||
z)]: OMG[int]\\ |
|||
= None # comment: None |
|||
x: int = None #comment : None""" |
|||
|
|||
big_result_ne = """\ |
|||
# -*- coding: utf-8 -*- # this should not be spoiled |
|||
''' |
|||
Docstring here |
|||
''' |
|||
|
|||
import testmod |
|||
x: int = 5 # this one is OK |
|||
ttt: Tuple[float, float, float] \\ |
|||
= \\ |
|||
(1.0, \\ |
|||
2.0, \\ |
|||
3.0,) |
|||
with foo(x==1) as f: #type: str |
|||
print(f) |
|||
|
|||
for i, j in my_inter(x=1): # type: ignore |
|||
i + j # type: int # what about this |
|||
|
|||
x = y = z = 1 # type: int |
|||
x, y, z = [], [], [] # type: (List[int], List[int], List[str]) |
|||
class C: |
|||
|
|||
|
|||
l[f(x |
|||
=1)]: List[int] = [ |
|||
|
|||
1, |
|||
2, |
|||
] |
|||
|
|||
|
|||
(C.x[1]): bool = \\ |
|||
42 == 5 |
|||
lst[...]: int \\ |
|||
\\ |
|||
# comment .. |
|||
|
|||
y: int # comment ... |
|||
z = ... |
|||
#type: int |
|||
|
|||
|
|||
#DONE placement of annotation after target rather than before = |
|||
|
|||
TD.x[1]: bool \\ |
|||
= 0 == 5 |
|||
|
|||
TD.y[1]: bool =5 == 5 # one more here |
|||
F[G(x == y, |
|||
|
|||
# hm... |
|||
|
|||
z)]: OMG[int]\\ |
|||
# comment: None |
|||
x: int #comment : None""" |
|||
|
|||
if __name__ == '__main__': |
|||
unittest.main() |
|||
@ -1,308 +0,0 @@ |
|||
"""Helper module to tranlate 3.5 type comments to 3.6 variable annotations.""" |
|||
import re |
|||
import os |
|||
import ast |
|||
import argparse |
|||
import tokenize |
|||
from collections import defaultdict |
|||
from textwrap import dedent |
|||
from io import BytesIO |
|||
|
|||
__all__ = ['com2ann', 'TYPE_COM'] |
|||
|
|||
TYPE_COM = re.compile(r'\s*#\s*type\s*:.*$', flags=re.DOTALL) |
|||
TRAIL_OR_COM = re.compile(r'\s*$|\s*#.*$', flags=re.DOTALL) |
|||
|
|||
|
|||
class _Data: |
|||
"""Internal class describing global data on file.""" |
|||
def __init__(self, lines, tokens): |
|||
self.lines = lines |
|||
self.tokens = tokens |
|||
ttab = defaultdict(list) # maps line number to token numbers |
|||
for i, tok in enumerate(tokens): |
|||
ttab[tok.start[0]].append(i) |
|||
self.ttab = ttab |
|||
self.success = [] # list of lines where type comments where processed |
|||
self.fail = [] # list of lines where type comments where rejected |
|||
|
|||
|
|||
def skip_blank(d, lno): |
|||
while d.lines[lno].strip() == '': |
|||
lno += 1 |
|||
return lno |
|||
|
|||
|
|||
def find_start(d, lcom): |
|||
"""Find first char of the assignment target.""" |
|||
i = d.ttab[lcom + 1][-2] # index of type comment token in tokens list |
|||
while ((d.tokens[i].exact_type != tokenize.NEWLINE) and |
|||
(d.tokens[i].exact_type != tokenize.ENCODING)): |
|||
i -= 1 |
|||
lno = d.tokens[i].start[0] |
|||
return skip_blank(d, lno) |
|||
|
|||
|
|||
def check_target(stmt): |
|||
if len(stmt.body): |
|||
assign = stmt.body[0] |
|||
else: |
|||
return False |
|||
if isinstance(assign, ast.Assign) and len(assign.targets) == 1: |
|||
targ = assign.targets[0] |
|||
else: |
|||
return False |
|||
if (isinstance(targ, ast.Name) or isinstance(targ, ast.Attribute) |
|||
or isinstance(targ, ast.Subscript)): |
|||
return True |
|||
return False |
|||
|
|||
|
|||
def find_eq(d, lstart): |
|||
"""Find equal sign starting from lstart taking care about d[f(x=1)] = 5.""" |
|||
col = pars = 0 |
|||
lno = lstart |
|||
while d.lines[lno][col] != '=' or pars != 0: |
|||
ch = d.lines[lno][col] |
|||
if ch in '([{': |
|||
pars += 1 |
|||
elif ch in ')]}': |
|||
pars -= 1 |
|||
if ch == '#' or col == len(d.lines[lno])-1: |
|||
lno = skip_blank(d, lno+1) |
|||
col = 0 |
|||
else: |
|||
col += 1 |
|||
return lno, col |
|||
|
|||
|
|||
def find_val(d, poseq): |
|||
"""Find position of first char of assignment value starting from poseq.""" |
|||
lno, col = poseq |
|||
while (d.lines[lno][col].isspace() or d.lines[lno][col] in '=\\'): |
|||
if col == len(d.lines[lno])-1: |
|||
lno += 1 |
|||
col = 0 |
|||
else: |
|||
col += 1 |
|||
return lno, col |
|||
|
|||
|
|||
def find_targ(d, poseq): |
|||
"""Find position of last char of target (annotation goes here).""" |
|||
lno, col = poseq |
|||
while (d.lines[lno][col].isspace() or d.lines[lno][col] in '=\\'): |
|||
if col == 0: |
|||
lno -= 1 |
|||
col = len(d.lines[lno])-1 |
|||
else: |
|||
col -= 1 |
|||
return lno, col+1 |
|||
|
|||
|
|||
def trim(new_lines, string, ltarg, poseq, lcom, ccom): |
|||
"""Remove None or Ellipsis from assignment value. |
|||
|
|||
Also remove parens if one has (None), (...) etc. |
|||
string -- 'None' or '...' |
|||
ltarg -- line where last char of target is located |
|||
poseq -- position of equal sign |
|||
lcom, ccom -- position of type comment |
|||
""" |
|||
nopars = lambda s: s.replace('(', '').replace(')', '') |
|||
leq, ceq = poseq |
|||
end = ccom if leq == lcom else len(new_lines[leq]) |
|||
subline = new_lines[leq][:ceq] |
|||
if leq == ltarg: |
|||
subline = subline.rstrip() |
|||
new_lines[leq] = subline + (new_lines[leq][end:] if leq == lcom |
|||
else new_lines[leq][ceq+1:end]) |
|||
|
|||
for lno in range(leq+1,lcom): |
|||
new_lines[lno] = nopars(new_lines[lno]) |
|||
|
|||
if lcom != leq: |
|||
subline = nopars(new_lines[lcom][:ccom]).replace(string, '') |
|||
if (not subline.isspace()): |
|||
subline = subline.rstrip() |
|||
new_lines[lcom] = subline + new_lines[lcom][ccom:] |
|||
|
|||
|
|||
def _com2ann(d, drop_None, drop_Ellipsis): |
|||
new_lines = d.lines[:] |
|||
for lcom, line in enumerate(d.lines): |
|||
match = re.search(TYPE_COM, line) |
|||
if match: |
|||
# strip " # type : annotation \n" -> "annotation \n" |
|||
tp = match.group().lstrip()[1:].lstrip()[4:].lstrip()[1:].lstrip() |
|||
submatch = re.search(TRAIL_OR_COM, tp) |
|||
subcom = '' |
|||
if submatch and submatch.group(): |
|||
subcom = submatch.group() |
|||
tp = tp[:submatch.start()] |
|||
if tp == 'ignore': |
|||
continue |
|||
ccom = match.start() |
|||
if not any(d.tokens[i].exact_type == tokenize.COMMENT |
|||
for i in d.ttab[lcom + 1]): |
|||
d.fail.append(lcom) |
|||
continue # type comment inside string |
|||
lstart = find_start(d, lcom) |
|||
stmt_str = dedent(''.join(d.lines[lstart:lcom+1])) |
|||
try: |
|||
stmt = ast.parse(stmt_str) |
|||
except SyntaxError: |
|||
d.fail.append(lcom) |
|||
continue # for or with statements |
|||
if not check_target(stmt): |
|||
d.fail.append(lcom) |
|||
continue |
|||
|
|||
d.success.append(lcom) |
|||
val = stmt.body[0].value |
|||
|
|||
# writing output now |
|||
poseq = find_eq(d, lstart) |
|||
lval, cval = find_val(d, poseq) |
|||
ltarg, ctarg = find_targ(d, poseq) |
|||
|
|||
op_par = '' |
|||
cl_par = '' |
|||
if isinstance(val, ast.Tuple): |
|||
if d.lines[lval][cval] != '(': |
|||
op_par = '(' |
|||
cl_par = ')' |
|||
# write the comment first |
|||
new_lines[lcom] = d.lines[lcom][:ccom].rstrip() + cl_par + subcom |
|||
ccom = len(d.lines[lcom][:ccom].rstrip()) |
|||
|
|||
string = False |
|||
if isinstance(val, ast.Tuple): |
|||
# t = 1, 2 -> t = (1, 2); only latter is allowed with annotation |
|||
free_place = int(new_lines[lval][cval-2:cval] == ' ') |
|||
new_lines[lval] = (new_lines[lval][:cval-free_place] + |
|||
op_par + new_lines[lval][cval:]) |
|||
elif isinstance(val, ast.Ellipsis) and drop_Ellipsis: |
|||
string = '...' |
|||
elif (isinstance(val, ast.NameConstant) and |
|||
val.value is None and drop_None): |
|||
string = 'None' |
|||
if string: |
|||
trim(new_lines, string, ltarg, poseq, lcom, ccom) |
|||
|
|||
# finally write an annotation |
|||
new_lines[ltarg] = (new_lines[ltarg][:ctarg] + |
|||
': ' + tp + new_lines[ltarg][ctarg:]) |
|||
return ''.join(new_lines) |
|||
|
|||
|
|||
def com2ann(code, *, drop_None=False, drop_Ellipsis=False, silent=False): |
|||
"""Translate type comments to type annotations in code. |
|||
|
|||
Take code as string and return this string where:: |
|||
|
|||
variable = value # type: annotation # real comment |
|||
|
|||
is translated to:: |
|||
|
|||
variable: annotation = value # real comment |
|||
|
|||
For unsupported syntax cases, the type comments are |
|||
left intact. If drop_None is True or if drop_Ellipsis |
|||
is True translate correcpondingly:: |
|||
|
|||
variable = None # type: annotation |
|||
variable = ... # type: annotation |
|||
|
|||
into:: |
|||
|
|||
variable: annotation |
|||
|
|||
The tool tries to preserve code formatting as much as |
|||
possible, but an exact translation is not guarateed. |
|||
A summary of translated comments id printed by default. |
|||
""" |
|||
try: |
|||
ast.parse(code) # we want to work only with file without syntax errors |
|||
except SyntaxError: |
|||
return None |
|||
lines = code.splitlines(keepends=True) |
|||
rl = BytesIO(code.encode('utf-8')).readline |
|||
tokens = list(tokenize.tokenize(rl)) |
|||
|
|||
data = _Data(lines, tokens) |
|||
new_code = _com2ann(data, drop_None, drop_Ellipsis) |
|||
|
|||
if not silent: |
|||
if data.success: |
|||
print('Comments translated on lines:', |
|||
', '.join(str(lno+1) for lno in data.success)) |
|||
if data.fail: |
|||
print('Comments rejected on lines:', |
|||
', '.join(str(lno+1) for lno in data.fail)) |
|||
if not data.success and not data.fail: |
|||
print('No type comments found') |
|||
|
|||
return new_code |
|||
|
|||
|
|||
def translate_file(infile, outfile, dnone, dell, silent): |
|||
try: |
|||
descr = tokenize.open(infile) |
|||
except SyntaxError: |
|||
print("Cannot open", infile) |
|||
return |
|||
with descr as f: |
|||
code = f.read() |
|||
enc = f.encoding |
|||
if not silent: |
|||
print('File:', infile) |
|||
new_code = com2ann(code, drop_None=dnone, |
|||
drop_Ellipsis=dell, |
|||
silent=silent) |
|||
if new_code is None: |
|||
print("SyntaxError in", infile) |
|||
return |
|||
with open(outfile, 'wb') as f: |
|||
f.write((new_code).encode(enc)) |
|||
|
|||
|
|||
if __name__ == '__main__': |
|||
|
|||
parser = argparse.ArgumentParser(description=__doc__) |
|||
parser.add_argument("-o", "--outfile", |
|||
help="output file, will be overwritten if exists,\n" |
|||
"defaults to input file") |
|||
parser.add_argument("infile", |
|||
help="input file or directory for translation, must\n" |
|||
"contain no syntax errors, for directory\n" |
|||
"the outfile is ignored and translation is\n" |
|||
"made in place") |
|||
parser.add_argument("-s", "--silent", |
|||
help="Do not print summary for line numbers of\n" |
|||
"translated and rejected comments", |
|||
action="store_true") |
|||
parser.add_argument("-n", "--drop-none", |
|||
help="drop any None as assignment value during\n" |
|||
"translation if it is annotated by a type coment", |
|||
action="store_true") |
|||
parser.add_argument("-e", "--drop-ellipsis", |
|||
help="drop any Ellipsis (...) as assignment value during\n" |
|||
"translation if it is annotated by a type coment", |
|||
action="store_true") |
|||
args = parser.parse_args() |
|||
if args.outfile is None: |
|||
args.outfile = args.infile |
|||
|
|||
if os.path.isfile(args.infile): |
|||
translate_file(args.infile, args.outfile, |
|||
args.drop_none, args.drop_ellipsis, args.silent) |
|||
else: |
|||
for root, dirs, files in os.walk(args.infile): |
|||
for afile in files: |
|||
_, ext = os.path.splitext(afile) |
|||
if ext == '.py' or ext == '.pyi': |
|||
fname = os.path.join(root, afile) |
|||
translate_file(fname, fname, |
|||
args.drop_none, args.drop_ellipsis, |
|||
args.silent) |
|||
Write
Preview
Loading…
Cancel
Save
Reference in new issue