From 858d381cb1dd44e0ace36d8020397fa6429acb50 Mon Sep 17 00:00:00 2001 From: Kerry Ivan Kurian Date: Wed, 27 Sep 2017 21:17:25 -0600 Subject: [PATCH] Update to pyflakes 1.6.0 --- contrib/pyflakes/__init__.py | 2 +- contrib/pyflakes/api.py | 44 +- contrib/pyflakes/checker.py | 557 ++++++++++-- contrib/pyflakes/messages.py | 105 ++- contrib/pyflakes/reporter.py | 2 +- contrib/pyflakes/test/harness.py | 31 +- contrib/pyflakes/test/test_api.py | 213 ++++- contrib/pyflakes/test/test_dict.py | 217 +++++ contrib/pyflakes/test/test_doctests.py | 244 ++++- contrib/pyflakes/test/test_imports.py | 289 +++++- contrib/pyflakes/test/test_other.py | 844 +++++++++++++++++- contrib/pyflakes/test/test_undefined_names.py | 240 ++++- 12 files changed, 2647 insertions(+), 141 deletions(-) create mode 100644 contrib/pyflakes/test/test_dict.py diff --git a/contrib/pyflakes/__init__.py b/contrib/pyflakes/__init__.py index 1f356cc..bcd8d54 100644 --- a/contrib/pyflakes/__init__.py +++ b/contrib/pyflakes/__init__.py @@ -1 +1 @@ -__version__ = '1.0.0' +__version__ = '1.6.0' diff --git a/contrib/pyflakes/api.py b/contrib/pyflakes/api.py index 3bc2330..49ee38d 100644 --- a/contrib/pyflakes/api.py +++ b/contrib/pyflakes/api.py @@ -5,6 +5,7 @@ import sys import os +import re import _ast from pyflakes import checker, __version__ @@ -13,6 +14,9 @@ __all__ = ['check', 'checkPath', 'checkRecursive', 'iterSourceCode', 'main'] +PYTHON_SHEBANG_REGEX = re.compile(br'^#!.*\bpython[23w]?\b\s*$') + + def check(codeString, filename, reporter=None): """ Check the Python source given by C{codeString} for flakes. @@ -41,6 +45,18 @@ def check(codeString, filename, reporter=None): (lineno, offset, text) = value.lineno, value.offset, value.text + if checker.PYPY: + if text is None: + lines = codeString.splitlines() + if len(lines) >= lineno: + text = lines[lineno - 1] + if sys.version_info >= (3, ) and isinstance(text, bytes): + try: + text = text.decode('ascii') + except UnicodeDecodeError: + text = None + offset -= 1 + # If there's an encoding problem with the file, the text is None. if text is None: # Avoid using msg, since for the only known case, it contains a @@ -96,6 +112,25 @@ def checkPath(filename, reporter=None): return check(codestr, filename, reporter) +def isPythonFile(filename): + """Return True if filename points to a Python file.""" + if filename.endswith('.py'): + return True + + max_bytes = 128 + + try: + with open(filename, 'rb') as f: + text = f.read(max_bytes) + if not text: + return False + except IOError: + return False + + first_line = text.splitlines()[0] + return PYTHON_SHEBANG_REGEX.match(first_line) + + def iterSourceCode(paths): """ Iterate over all Python source files in C{paths}. @@ -108,8 +143,9 @@ def iterSourceCode(paths): if os.path.isdir(path): for dirpath, dirnames, filenames in os.walk(path): for filename in filenames: - if filename.endswith('.py'): - yield os.path.join(dirpath, filename) + full_path = os.path.join(dirpath, filename) + if isPythonFile(full_path): + yield full_path else: yield path @@ -157,7 +193,7 @@ def handler(sig, f): pass -def main(prog=None): +def main(prog=None, args=None): """Entry point for the script "pyflakes".""" import optparse @@ -166,7 +202,7 @@ def main(prog=None): _exitOnSignal('SIGPIPE', 1) parser = optparse.OptionParser(prog=prog, version=__version__) - (__, args) = parser.parse_args() + (__, args) = parser.parse_args(args=args) reporter = modReporter._makeDefaultReporter() if args: warnings = checkRecursive(args, reporter) diff --git a/contrib/pyflakes/checker.py b/contrib/pyflakes/checker.py index e6e1942..75abdc0 100644 --- a/contrib/pyflakes/checker.py +++ b/contrib/pyflakes/checker.py @@ -4,6 +4,7 @@ Implement the central Checker class. Also, it models the Bindings and Scopes. """ +import __future__ import doctest import os import sys @@ -11,6 +12,13 @@ PY2 = sys.version_info < (3, 0) PY32 = sys.version_info < (3, 3) # Python 2.5 to 3.2 PY33 = sys.version_info < (3, 4) # Python 2.5 to 3.3 +PY34 = sys.version_info < (3, 5) # Python 2.5 to 3.4 +try: + sys.pypy_version_info + PYPY = True +except AttributeError: + PYPY = False + builtin_vars = dir(__import__('__builtin__' if PY2 else 'builtins')) try: @@ -48,6 +56,11 @@ def getAlternatives(n): if isinstance(n, ast.Try): return [n.body + n.orelse] + [[hdl] for hdl in n.handlers] +if PY34: + LOOP_TYPES = (ast.While, ast.For) +else: + LOOP_TYPES = (ast.While, ast.For, ast.AsyncFor) + class _FieldsOrder(dict): """Fix order of AST node fields.""" @@ -68,6 +81,17 @@ def __missing__(self, node_class): return fields +def counter(items): + """ + Simplest required implementation of collections.Counter. Required as 2.6 + does not have Counter in collections. + """ + results = {} + for item in items: + results[item] = results.get(item, 0) + 1 + return results + + def iter_child_nodes(node, omit=None, _fields_order=_FieldsOrder()): """ Yield all direct child nodes of *node*, that is, all fields that @@ -84,6 +108,33 @@ def iter_child_nodes(node, omit=None, _fields_order=_FieldsOrder()): yield item +def convert_to_value(item): + if isinstance(item, ast.Str): + return item.s + elif hasattr(ast, 'Bytes') and isinstance(item, ast.Bytes): + return item.s + elif isinstance(item, ast.Tuple): + return tuple(convert_to_value(i) for i in item.elts) + elif isinstance(item, ast.Num): + return item.n + elif isinstance(item, ast.Name): + result = VariableKey(item=item) + constants_lookup = { + 'True': True, + 'False': False, + 'None': None, + } + return constants_lookup.get( + result.name, + result, + ) + elif (not PY33) and isinstance(item, ast.NameConstant): + # None, True, False are nameconstants in python3, but names in 2 + return item.value + else: + return UnhandledKeyType() + + class Binding(object): """ Represents the binding of a value to a name. @@ -92,8 +143,8 @@ class Binding(object): which names have not. See L{Assignment} for a special type of binding that is checked with stricter rules. - @ivar used: pair of (L{Scope}, line-number) indicating the scope and - line number that this binding was last used + @ivar used: pair of (L{Scope}, node) indicating the scope and + the node that this binding was last used. """ def __init__(self, name, source): @@ -120,6 +171,31 @@ class Definition(Binding): """ +class UnhandledKeyType(object): + """ + A dictionary key of a type that we cannot or do not check for duplicates. + """ + + +class VariableKey(object): + """ + A dictionary key which is a variable. + + @ivar item: The variable AST object. + """ + def __init__(self, item): + self.name = item.id + + def __eq__(self, compare): + return ( + compare.__class__ == self.__class__ + and compare.name == self.name + ) + + def __hash__(self): + return hash(self.name) + + class Importation(Definition): """ A binding created by an import statement. @@ -129,17 +205,137 @@ class Importation(Definition): @type fullName: C{str} """ - def __init__(self, name, source): - self.fullName = name + def __init__(self, name, source, full_name=None): + self.fullName = full_name or name self.redefined = [] - name = name.split('.')[0] super(Importation, self).__init__(name, source) def redefines(self, other): - if isinstance(other, Importation): + if isinstance(other, SubmoduleImportation): + # See note in SubmoduleImportation about RedefinedWhileUnused return self.fullName == other.fullName return isinstance(other, Definition) and self.name == other.name + def _has_alias(self): + """Return whether importation needs an as clause.""" + return not self.fullName.split('.')[-1] == self.name + + @property + def source_statement(self): + """Generate a source statement equivalent to the import.""" + if self._has_alias(): + return 'import %s as %s' % (self.fullName, self.name) + else: + return 'import %s' % self.fullName + + def __str__(self): + """Return import full name with alias.""" + if self._has_alias(): + return self.fullName + ' as ' + self.name + else: + return self.fullName + + +class SubmoduleImportation(Importation): + """ + A binding created by a submodule import statement. + + A submodule import is a special case where the root module is implicitly + imported, without an 'as' clause, and the submodule is also imported. + Python does not restrict which attributes of the root module may be used. + + This class is only used when the submodule import is without an 'as' clause. + + pyflakes handles this case by registering the root module name in the scope, + allowing any attribute of the root module to be accessed. + + RedefinedWhileUnused is suppressed in `redefines` unless the submodule + name is also the same, to avoid false positives. + """ + + def __init__(self, name, source): + # A dot should only appear in the name when it is a submodule import + assert '.' in name and (not source or isinstance(source, ast.Import)) + package_name = name.split('.')[0] + super(SubmoduleImportation, self).__init__(package_name, source) + self.fullName = name + + def redefines(self, other): + if isinstance(other, Importation): + return self.fullName == other.fullName + return super(SubmoduleImportation, self).redefines(other) + + def __str__(self): + return self.fullName + + @property + def source_statement(self): + return 'import ' + self.fullName + + +class ImportationFrom(Importation): + + def __init__(self, name, source, module, real_name=None): + self.module = module + self.real_name = real_name or name + + if module.endswith('.'): + full_name = module + self.real_name + else: + full_name = module + '.' + self.real_name + + super(ImportationFrom, self).__init__(name, source, full_name) + + def __str__(self): + """Return import full name with alias.""" + if self.real_name != self.name: + return self.fullName + ' as ' + self.name + else: + return self.fullName + + @property + def source_statement(self): + if self.real_name != self.name: + return 'from %s import %s as %s' % (self.module, + self.real_name, + self.name) + else: + return 'from %s import %s' % (self.module, self.name) + + +class StarImportation(Importation): + """A binding created by a 'from x import *' statement.""" + + def __init__(self, name, source): + super(StarImportation, self).__init__('*', source) + # Each star importation needs a unique name, and + # may not be the module name otherwise it will be deemed imported + self.name = name + '.*' + self.fullName = name + + @property + def source_statement(self): + return 'from ' + self.fullName + ' import *' + + def __str__(self): + # When the module ends with a ., avoid the ambiguous '..*' + if self.fullName.endswith('.'): + return self.source_statement + else: + return self.name + + +class FutureImportation(ImportationFrom): + """ + A binding created by a from `__future__` import statement. + + `__future__` imports are implicitly used. + """ + + def __init__(self, name, source, scope): + super(FutureImportation, self).__init__(name, source, '__future__') + self.used = (scope, source) + class Argument(Binding): """ @@ -237,7 +433,12 @@ class GeneratorScope(Scope): class ModuleScope(Scope): - pass + """Scope for a module.""" + _futures_allowed = True + + +class DoctestScope(ModuleScope): + """Scope for a doctest.""" # Globally defined names which are not attributes of the builtins module, or @@ -249,7 +450,7 @@ def getNodeName(node): # Returns node.id, or node.name, or None if hasattr(node, 'id'): # One of the many nodes with an id return node.id - if hasattr(node, 'name'): # a ExceptHandler node + if hasattr(node, 'name'): # an ExceptHandler node return node.name @@ -289,7 +490,6 @@ def __init__(self, tree, filename='(none)', builtins=None, self.withDoctest = withDoctest self.scopeStack = [ModuleScope()] self.exceptHandlers = [()] - self.futuresAllowed = True self.root = tree self.handleChildren(tree) self.runDeferred(self._deferredFunctions) @@ -331,6 +531,24 @@ def runDeferred(self, deferred): self.offset = offset handler() + def _in_doctest(self): + return (len(self.scopeStack) >= 2 and + isinstance(self.scopeStack[1], DoctestScope)) + + @property + def futuresAllowed(self): + if not all(isinstance(scope, ModuleScope) + for scope in self.scopeStack): + return False + + return self.scope._futures_allowed + + @futuresAllowed.setter + def futuresAllowed(self, value): + assert value is False + if isinstance(self.scope, ModuleScope): + self.scope._futures_allowed = False + @property def scope(self): return self.scopeStack[-1] @@ -344,17 +562,33 @@ def checkDeadScopes(self): which were imported but unused. """ for scope in self.deadScopes: - if isinstance(scope.get('__all__'), ExportBinding): - all_names = set(scope['__all__'].names) + # imports in classes are public members + if isinstance(scope, ClassScope): + continue + + all_binding = scope.get('__all__') + if all_binding and not isinstance(all_binding, ExportBinding): + all_binding = None + + if all_binding: + all_names = set(all_binding.names) + undefined = all_names.difference(scope) + else: + all_names = undefined = [] + + if undefined: if not scope.importStarred and \ os.path.basename(self.filename) != '__init__.py': # Look for possible mistakes in the export list - undefined = all_names.difference(scope) for name in undefined: self.report(messages.UndefinedExport, scope['__all__'].source, name) - else: - all_names = [] + + # mark all import '*' as used by the undefined in __all__ + if scope.importStarred: + for binding in scope.values(): + if isinstance(binding, StarImportation): + binding.used = all_binding # Look for imported names that aren't used. for value in scope.values(): @@ -362,7 +596,7 @@ def checkDeadScopes(self): used = value.used or value.name in all_names if not used: messg = messages.UnusedImport - self.report(messg, value.source, value.name) + self.report(messg, value.source, str(value)) for node in value.redefined: if isinstance(self.getParent(node), ast.For): messg = messages.ImportShadowedByLoopVar @@ -466,23 +700,17 @@ def handleNodeLoad(self, node): name = getNodeName(node) if not name: return - # try local scope - try: - self.scope[name].used = (self.scope, node) - except KeyError: - pass - else: - return - scopes = [scope for scope in self.scopeStack[:-1] - if isinstance(scope, (FunctionScope, ModuleScope, GeneratorScope))] - if isinstance(self.scope, GeneratorScope) and scopes[-1] != self.scopeStack[-2]: - scopes.append(self.scopeStack[-2]) + in_generators = None + importStarred = None # try enclosing function scopes and global scope - importStarred = self.scope.importStarred - for scope in reversed(scopes): - importStarred = importStarred or scope.importStarred + for scope in self.scopeStack[-1::-1]: + # only generators used in a class scope can access the names + # of the class. this is skipped during the first iteration + if in_generators is False and isinstance(scope, ClassScope): + continue + try: scope[name].used = (self.scope, node) except KeyError: @@ -490,9 +718,30 @@ def handleNodeLoad(self, node): else: return + importStarred = importStarred or scope.importStarred + + if in_generators is not False: + in_generators = isinstance(scope, GeneratorScope) + # look in the built-ins - if importStarred or name in self.builtIns: + if name in self.builtIns: + return + + if importStarred: + from_list = [] + + for scope in self.scopeStack[-1::-1]: + for binding in scope.values(): + if isinstance(binding, StarImportation): + # mark '*' imports as used for each scope + binding.used = (self.scope, node) + from_list.append(binding.fullName) + + # report * usage, with a list of possible sources + from_list = ', '.join(sorted(from_list)) + self.report(messages.ImportStarUsage, node, name, from_list) return + if name == '__path__' and os.path.basename(self.filename) == '__init__.py': # the special name __path__ is valid only in packages return @@ -550,7 +799,7 @@ def on_conditional_branch(): return if on_conditional_branch(): - # We can not predict if this conditional branch is going to + # We cannot predict if this conditional branch is going to # be executed. return @@ -586,8 +835,13 @@ def getDocstring(self, node): node = node.value if not isinstance(node, ast.Str): return (None, None) - # Computed incorrectly if the docstring has backslash - doctest_lineno = node.lineno - node.s.count('\n') - 1 + + if PYPY: + doctest_lineno = node.lineno - 1 + else: + # Computed incorrectly if the docstring has backslash + doctest_lineno = node.lineno - node.s.count('\n') - 1 + return (node.s, doctest_lineno) def handleNode(self, node, parent): @@ -616,7 +870,19 @@ def handleNode(self, node, parent): def handleDoctests(self, node): try: - (docstring, node_lineno) = self.getDocstring(node.body[0]) + if hasattr(node, 'docstring'): + docstring = node.docstring + + # This is just a reasonable guess. In Python 3.7, docstrings no + # longer have line numbers associated with them. This will be + # incorrect if there are empty lines between the beginning + # of the function and the docstring. + node_lineno = node.lineno + if hasattr(node, 'args'): + node_lineno = max([node_lineno] + + [arg.lineno for arg in node.args.args]) + else: + (docstring, node_lineno) = self.getDocstring(node.body[0]) examples = docstring and self._getDoctestExamples(docstring) except (ValueError, IndexError): # e.g. line 6 of the docstring for has inconsistent @@ -624,8 +890,12 @@ def handleDoctests(self, node): return if not examples: return + + # Place doctest in module scope + saved_stack = self.scopeStack + self.scopeStack = [self.scopeStack[0]] node_offset = self.offset or (0, 0) - self.pushScope() + self.pushScope(DoctestScope) underscore_in_builtins = '_' in self.builtIns if not underscore_in_builtins: self.builtIns.add('_') @@ -634,6 +904,8 @@ def handleDoctests(self, node): tree = compile(example.source, "", "exec", ast.PyCF_ONLY_AST) except SyntaxError: e = sys.exc_info()[1] + if PYPY: + e.offset += 1 position = (node_lineno + example.lineno + e.lineno, example.indent + 4 + (e.offset or 0)) self.report(messages.DoctestSyntaxError, node, position) @@ -645,20 +917,21 @@ def handleDoctests(self, node): if not underscore_in_builtins: self.builtIns.remove('_') self.popScope() + self.scopeStack = saved_stack def ignore(self, node): pass # "stmt" type nodes DELETE = PRINT = FOR = ASYNCFOR = WHILE = IF = WITH = WITHITEM = \ - ASYNCWITH = ASYNCWITHITEM = RAISE = TRYFINALLY = ASSERT = EXEC = \ + ASYNCWITH = ASYNCWITHITEM = RAISE = TRYFINALLY = EXEC = \ EXPR = ASSIGN = handleChildren - CONTINUE = BREAK = PASS = ignore + PASS = ignore # "expr" type nodes - BOOLOP = BINOP = UNARYOP = IFEXP = DICT = SET = \ - COMPARE = CALL = REPR = ATTRIBUTE = SUBSCRIPT = LIST = TUPLE = \ + BOOLOP = BINOP = UNARYOP = IFEXP = SET = \ + COMPARE = CALL = REPR = ATTRIBUTE = SUBSCRIPT = \ STARRED = NAMECONSTANT = handleChildren NUM = STR = BYTES = ELLIPSIS = ignore @@ -672,17 +945,58 @@ def ignore(self, node): # same for operators AND = OR = ADD = SUB = MULT = DIV = MOD = POW = LSHIFT = RSHIFT = \ BITOR = BITXOR = BITAND = FLOORDIV = INVERT = NOT = UADD = USUB = \ - EQ = NOTEQ = LT = LTE = GT = GTE = IS = ISNOT = IN = NOTIN = ignore + EQ = NOTEQ = LT = LTE = GT = GTE = IS = ISNOT = IN = NOTIN = \ + MATMULT = ignore # additional node types - COMPREHENSION = KEYWORD = handleChildren + COMPREHENSION = KEYWORD = FORMATTEDVALUE = JOINEDSTR = handleChildren + + def DICT(self, node): + # Complain if there are duplicate keys with different values + # If they have the same value it's not going to cause potentially + # unexpected behaviour so we'll not complain. + keys = [ + convert_to_value(key) for key in node.keys + ] + + key_counts = counter(keys) + duplicate_keys = [ + key for key, count in key_counts.items() + if count > 1 + ] + + for key in duplicate_keys: + key_indices = [i for i, i_key in enumerate(keys) if i_key == key] + + values = counter( + convert_to_value(node.values[index]) + for index in key_indices + ) + if any(count == 1 for value, count in values.items()): + for key_index in key_indices: + key_node = node.keys[key_index] + if isinstance(key, VariableKey): + self.report(messages.MultiValueRepeatedKeyVariable, + key_node, + key.name) + else: + self.report( + messages.MultiValueRepeatedKeyLiteral, + key_node, + key, + ) + self.handleChildren(node) + + def ASSERT(self, node): + if isinstance(node.test, ast.Tuple) and node.test.elts != []: + self.report(messages.AssertTuple, node) + self.handleChildren(node) def GLOBAL(self, node): """ Keep track of globals declarations. """ - # In doctests, the global scope is an anonymous function at index 1. - global_scope_index = 1 if self.withDoctest else 0 + global_scope_index = 1 if self._in_doctest() else 0 global_scope = self.scopeStack[global_scope_index] # Ignore 'global' statement in global scope. @@ -693,10 +1007,12 @@ def GLOBAL(self, node): node_value = Assignment(node_name, node) # Remove UndefinedName messages already reported for this name. + # TODO: if the global is not used in this scope, it does not + # become a globally defined name. See test_unused_global. self.messages = [ m for m in self.messages if not - isinstance(m, messages.UndefinedName) and not - m.message_args[0] == node_name] + isinstance(m, messages.UndefinedName) or + m.message_args[0] != node_name] # Bind name to global scope if it doesn't exist already. global_scope.setdefault(node_name, node_value) @@ -737,8 +1053,33 @@ def NAME(self, node): # arguments, but these aren't dispatched through here raise RuntimeError("Got impossible expression context: %r" % (node.ctx,)) + def CONTINUE(self, node): + # Walk the tree up until we see a loop (OK), a function or class + # definition (not OK), for 'continue', a finally block (not OK), or + # the top module scope (not OK) + n = node + while hasattr(n, 'parent'): + n, n_child = n.parent, n + if isinstance(n, LOOP_TYPES): + # Doesn't apply unless it's in the loop itself + if n_child not in n.orelse: + return + if isinstance(n, (ast.FunctionDef, ast.ClassDef)): + break + # Handle Try/TryFinally difference in Python < and >= 3.3 + if hasattr(n, 'finalbody') and isinstance(node, ast.Continue): + if n_child in n.finalbody: + self.report(messages.ContinueInFinally, node) + return + if isinstance(node, ast.Continue): + self.report(messages.ContinueOutsideLoop, node) + else: # ast.Break + self.report(messages.BreakOutsideLoop, node) + + BREAK = CONTINUE + def RETURN(self, node): - if isinstance(self.scope, ClassScope): + if isinstance(self.scope, (ClassScope, ModuleScope)): self.report(messages.ReturnOutsideFunction, node) return @@ -751,6 +1092,10 @@ def RETURN(self, node): self.handleNode(node.value, node) def YIELD(self, node): + if isinstance(self.scope, (ClassScope, ModuleScope)): + self.report(messages.YieldOutsideFunction, node) + return + self.scope.isGenerator = True self.handleNode(node.value, node) @@ -761,7 +1106,11 @@ def FUNCTIONDEF(self, node): self.handleNode(deco, node) self.LAMBDA(node) self.addBinding(node, FunctionDefinition(node.name, node)) - if self.withDoctest: + # doctest does not process doctest within a doctest, + # or in nested functions. + if (self.withDoctest and + not self._in_doctest() and + not isinstance(self.scope, FunctionScope)): self.deferFunction(lambda: self.handleDoctests(node)) ASYNCFUNCTIONDEF = FUNCTIONDEF @@ -861,7 +1210,11 @@ def CLASSDEF(self, node): for keywordNode in node.keywords: self.handleNode(keywordNode, node) self.pushScope(ClassScope) - if self.withDoctest: + # doctest does not process doctest within a doctest + # classes within classes are processed. + if (self.withDoctest and + not self._in_doctest() and + not isinstance(self.scope, FunctionScope)): self.deferFunction(lambda: self.handleDoctests(node)) for stmt in node.body: self.handleNode(stmt, node) @@ -873,10 +1226,38 @@ def AUGASSIGN(self, node): self.handleNode(node.value, node) self.handleNode(node.target, node) + def TUPLE(self, node): + if not PY2 and isinstance(node.ctx, ast.Store): + # Python 3 advanced tuple unpacking: a, *b, c = d. + # Only one starred expression is allowed, and no more than 1<<8 + # assignments are allowed before a stared expression. There is + # also a limit of 1<<24 expressions after the starred expression, + # which is impossible to test due to memory restrictions, but we + # add it here anyway + has_starred = False + star_loc = -1 + for i, n in enumerate(node.elts): + if isinstance(n, ast.Starred): + if has_starred: + self.report(messages.TwoStarredExpressions, node) + # The SyntaxError doesn't distinguish two from more + # than two. + break + has_starred = True + star_loc = i + if star_loc >= 1 << 8 or len(node.elts) - star_loc - 1 >= 1 << 24: + self.report(messages.TooManyExpressionsInStarredAssignment, node) + self.handleChildren(node) + + LIST = TUPLE + def IMPORT(self, node): for alias in node.names: - name = alias.asname or alias.name - importation = Importation(name, node) + if '.' in alias.name and not alias.asname: + importation = SubmoduleImportation(alias.name, node) + else: + name = alias.asname or alias.name + importation = Importation(name, node, alias.name) self.addBinding(node, importation) def IMPORTFROM(self, node): @@ -887,26 +1268,42 @@ def IMPORTFROM(self, node): else: self.futuresAllowed = False + module = ('.' * node.level) + (node.module or '') + for alias in node.names: - if alias.name == '*': - self.scope.importStarred = True - self.report(messages.ImportStarUsed, node, node.module) - continue name = alias.asname or alias.name - importation = Importation(name, node) if node.module == '__future__': - importation.used = (self.scope, node) + importation = FutureImportation(name, node, self.scope) + if alias.name not in __future__.all_feature_names: + self.report(messages.FutureFeatureNotDefined, + node, alias.name) + elif alias.name == '*': + # Only Python 2, local import * is a SyntaxWarning + if not PY2 and not isinstance(self.scope, ModuleScope): + self.report(messages.ImportStarNotPermitted, + node, module) + continue + + self.scope.importStarred = True + self.report(messages.ImportStarUsed, node, module) + importation = StarImportation(module, node) + else: + importation = ImportationFrom(name, node, + module, alias.name) self.addBinding(node, importation) def TRY(self, node): handler_names = [] # List the exception handlers - for handler in node.handlers: + for i, handler in enumerate(node.handlers): if isinstance(handler.type, ast.Tuple): for exc_type in handler.type.elts: handler_names.append(getNodeName(exc_type)) elif handler.type: handler_names.append(getNodeName(handler.type)) + + if handler.type is None and i < len(node.handlers) - 1: + self.report(messages.DefaultExceptNotLast, handler) # Memorize the except handlers and process the body self.exceptHandlers.append(handler_names) for child in node.body: @@ -918,8 +1315,46 @@ def TRY(self, node): TRYEXCEPT = TRY def EXCEPTHANDLER(self, node): - # 3.x: in addition to handling children, we must handle the name of - # the exception, which is not a Name node, but a simple string. - if isinstance(node.name, str): - self.handleNodeStore(node) + if PY2 or node.name is None: + self.handleChildren(node) + return + + # 3.x: the name of the exception, which is not a Name node, but + # a simple string, creates a local that is only bound within the scope + # of the except: block. + + for scope in self.scopeStack[::-1]: + if node.name in scope: + is_name_previously_defined = True + break + else: + is_name_previously_defined = False + + self.handleNodeStore(node) self.handleChildren(node) + if not is_name_previously_defined: + # See discussion on https://github.com/PyCQA/pyflakes/pull/59 + + # We're removing the local name since it's being unbound + # after leaving the except: block and it's always unbound + # if the except: block is never entered. This will cause an + # "undefined name" error raised if the checked code tries to + # use the name afterwards. + # + # Unless it's been removed already. Then do nothing. + + try: + del self.scope[node.name] + except KeyError: + pass + + def ANNASSIGN(self, node): + if node.value: + # Only bind the *targets* if the assignment has a value. + # Otherwise it's not really ast.Store and shouldn't silence + # UndefinedLocal warnings. + self.handleNode(node.target, node) + self.handleNode(node.annotation, node) + if node.value: + # If the assignment has value, handle the *value* now. + self.handleNode(node.value, node) diff --git a/contrib/pyflakes/messages.py b/contrib/pyflakes/messages.py index 8899b7b..9e9406c 100644 --- a/contrib/pyflakes/messages.py +++ b/contrib/pyflakes/messages.py @@ -49,6 +49,14 @@ def __init__(self, filename, loc, name, orig_loc): self.message_args = (name, orig_loc.lineno) +class ImportStarNotPermitted(Message): + message = "'from %s import *' only allowed at module level" + + def __init__(self, filename, loc, modname): + Message.__init__(self, filename, loc) + self.message_args = (modname,) + + class ImportStarUsed(Message): message = "'from %s import *' used; unable to detect undefined names" @@ -57,6 +65,14 @@ def __init__(self, filename, loc, modname): self.message_args = (modname,) +class ImportStarUsage(Message): + message = "%r may be undefined, or defined from star imports: %s" + + def __init__(self, filename, loc, name, from_list): + Message.__init__(self, filename, loc) + self.message_args = (name, from_list) + + class UndefinedName(Message): message = 'undefined name %r' @@ -100,17 +116,42 @@ def __init__(self, filename, loc, name): self.message_args = (name,) +class MultiValueRepeatedKeyLiteral(Message): + message = 'dictionary key %r repeated with different values' + + def __init__(self, filename, loc, key): + Message.__init__(self, filename, loc) + self.message_args = (key,) + + +class MultiValueRepeatedKeyVariable(Message): + message = 'dictionary key variable %s repeated with different values' + + def __init__(self, filename, loc, key): + Message.__init__(self, filename, loc) + self.message_args = (key,) + + class LateFutureImport(Message): - message = 'future import(s) %r after other statements' + message = 'from __future__ imports must occur at the beginning of the file' def __init__(self, filename, loc, names): Message.__init__(self, filename, loc) - self.message_args = (names,) + self.message_args = () + + +class FutureFeatureNotDefined(Message): + """An undefined __future__ feature name was imported.""" + message = 'future feature %s is not defined' + + def __init__(self, filename, loc, name): + Message.__init__(self, filename, loc) + self.message_args = (name,) class UnusedVariable(Message): """ - Indicates that a variable has been explicity assigned to but not actually + Indicates that a variable has been explicitly assigned to but not actually used. """ message = 'local variable %r is assigned to but never used' @@ -132,3 +173,61 @@ class ReturnOutsideFunction(Message): Indicates a return statement outside of a function/method. """ message = '\'return\' outside function' + + +class YieldOutsideFunction(Message): + """ + Indicates a yield or yield from statement outside of a function/method. + """ + message = '\'yield\' outside function' + + +# For whatever reason, Python gives different error messages for these two. We +# match the Python error message exactly. +class ContinueOutsideLoop(Message): + """ + Indicates a continue statement outside of a while or for loop. + """ + message = '\'continue\' not properly in loop' + + +class BreakOutsideLoop(Message): + """ + Indicates a break statement outside of a while or for loop. + """ + message = '\'break\' outside loop' + + +class ContinueInFinally(Message): + """ + Indicates a continue statement in a finally block in a while or for loop. + """ + message = '\'continue\' not supported inside \'finally\' clause' + + +class DefaultExceptNotLast(Message): + """ + Indicates an except: block as not the last exception handler. + """ + message = 'default \'except:\' must be last' + + +class TwoStarredExpressions(Message): + """ + Two or more starred expressions in an assignment (a, *b, *c = d). + """ + message = 'two starred expressions in assignment' + + +class TooManyExpressionsInStarredAssignment(Message): + """ + Too many expressions in an assignment with star-unpacking + """ + message = 'too many expressions in star-unpacking assignment' + + +class AssertTuple(Message): + """ + Assertion test is a tuple, which are always True. + """ + message = 'assertion is always true, perhaps remove parentheses?' diff --git a/contrib/pyflakes/reporter.py b/contrib/pyflakes/reporter.py index ae645bd..8b56e74 100644 --- a/contrib/pyflakes/reporter.py +++ b/contrib/pyflakes/reporter.py @@ -38,7 +38,7 @@ def unexpectedError(self, filename, msg): def syntaxError(self, filename, msg, lineno, offset, text): """ - There was a syntax errror in C{filename}. + There was a syntax error in C{filename}. @param filename: The path to the file with the syntax error. @ptype filename: C{unicode} diff --git a/contrib/pyflakes/test/harness.py b/contrib/pyflakes/test/harness.py index a781237..009923f 100644 --- a/contrib/pyflakes/test/harness.py +++ b/contrib/pyflakes/test/harness.py @@ -36,8 +36,37 @@ def flakes(self, input, *expectedOutputs, **kw): %s''' % (input, expectedOutputs, '\n'.join([str(o) for o in w.messages]))) return w - if sys.version_info < (2, 7): + if not hasattr(unittest.TestCase, 'assertIs'): def assertIs(self, expr1, expr2, msg=None): if expr1 is not expr2: self.fail(msg or '%r is not %r' % (expr1, expr2)) + + if not hasattr(unittest.TestCase, 'assertIsInstance'): + + def assertIsInstance(self, obj, cls, msg=None): + """Same as self.assertTrue(isinstance(obj, cls)).""" + if not isinstance(obj, cls): + self.fail(msg or '%r is not an instance of %r' % (obj, cls)) + + if not hasattr(unittest.TestCase, 'assertNotIsInstance'): + + def assertNotIsInstance(self, obj, cls, msg=None): + """Same as self.assertFalse(isinstance(obj, cls)).""" + if isinstance(obj, cls): + self.fail(msg or '%r is an instance of %r' % (obj, cls)) + + if not hasattr(unittest.TestCase, 'assertIn'): + + def assertIn(self, member, container, msg=None): + """Just like self.assertTrue(a in b).""" + if member not in container: + self.fail(msg or '%r not found in %r' % (member, container)) + + if not hasattr(unittest.TestCase, 'assertNotIn'): + + def assertNotIn(self, member, container, msg=None): + """Just like self.assertTrue(a not in b).""" + if member in container: + self.fail(msg or + '%r unexpectedly found in %r' % (member, container)) diff --git a/contrib/pyflakes/test/test_api.py b/contrib/pyflakes/test/test_api.py index 34a59bc..3f54ca4 100644 --- a/contrib/pyflakes/test/test_api.py +++ b/contrib/pyflakes/test/test_api.py @@ -8,9 +8,11 @@ import subprocess import tempfile +from pyflakes.checker import PY2 from pyflakes.messages import UnusedImport from pyflakes.reporter import Reporter from pyflakes.api import ( + main, checkPath, checkRecursive, iterSourceCode, @@ -23,6 +25,20 @@ from io import StringIO unichr = chr +try: + sys.pypy_version_info + PYPY = True +except AttributeError: + PYPY = False + +try: + WindowsError + WIN = True +except NameError: + WIN = False + +ERROR_HAS_COL_NUM = ERROR_HAS_LAST_LINE = sys.version_info >= (3, 2) or PYPY + def withStderrTo(stderr, f, *args, **kwargs): """ @@ -44,6 +60,57 @@ def __init__(self, lineno, col_offset=0): self.col_offset = col_offset +class SysStreamCapturing(object): + + """ + Context manager capturing sys.stdin, sys.stdout and sys.stderr. + + The file handles are replaced with a StringIO object. + On environments that support it, the StringIO object uses newlines + set to os.linesep. Otherwise newlines are converted from \\n to + os.linesep during __exit__. + """ + + def _create_StringIO(self, buffer=None): + # Python 3 has a newline argument + try: + return StringIO(buffer, newline=os.linesep) + except TypeError: + self._newline = True + # Python 2 creates an input only stream when buffer is not None + if buffer is None: + return StringIO() + else: + return StringIO(buffer) + + def __init__(self, stdin): + self._newline = False + self._stdin = self._create_StringIO(stdin or '') + + def __enter__(self): + self._orig_stdin = sys.stdin + self._orig_stdout = sys.stdout + self._orig_stderr = sys.stderr + + sys.stdin = self._stdin + sys.stdout = self._stdout_stringio = self._create_StringIO() + sys.stderr = self._stderr_stringio = self._create_StringIO() + + return self + + def __exit__(self, *args): + self.output = self._stdout_stringio.getvalue() + self.error = self._stderr_stringio.getvalue() + + if self._newline and os.linesep != '\n': + self.output = self.output.replace('\n', os.linesep) + self.error = self.error.replace('\n', os.linesep) + + sys.stdin = self._orig_stdin + sys.stdout = self._orig_stdout + sys.stderr = self._orig_stderr + + class LoggingReporter(object): """ Implementation of Reporter that just appends any error to a list. @@ -120,6 +187,36 @@ def test_recurses(self): sorted(iterSourceCode([self.tempdir])), sorted([apath, bpath, cpath])) + def test_shebang(self): + """ + Find Python files that don't end with `.py`, but contain a Python + shebang. + """ + python = os.path.join(self.tempdir, 'a') + with open(python, 'w') as fd: + fd.write('#!/usr/bin/env python\n') + + self.makeEmptyFile('b') + + with open(os.path.join(self.tempdir, 'c'), 'w') as fd: + fd.write('hello\nworld\n') + + python2 = os.path.join(self.tempdir, 'd') + with open(python2, 'w') as fd: + fd.write('#!/usr/bin/env python2\n') + + python3 = os.path.join(self.tempdir, 'e') + with open(python3, 'w') as fd: + fd.write('#!/usr/bin/env python3\n') + + pythonw = os.path.join(self.tempdir, 'f') + with open(pythonw, 'w') as fd: + fd.write('#!/usr/bin/env pythonw\n') + + self.assertEqual( + sorted(iterSourceCode([self.tempdir])), + sorted([python, python2, python3, pythonw])) + def test_multipleDirectories(self): """ L{iterSourceCode} can be given multiple directories. It will recurse @@ -312,18 +409,25 @@ def evaluate(source): evaluate(source) except SyntaxError: e = sys.exc_info()[1] - self.assertTrue(e.text.count('\n') > 1) + if not PYPY: + self.assertTrue(e.text.count('\n') > 1) else: self.fail() sourcePath = self.makeTempFile(source) + + if PYPY: + message = 'EOF while scanning triple-quoted string literal' + else: + message = 'invalid syntax' + self.assertHasErrors( sourcePath, ["""\ -%s:8:11: invalid syntax +%s:8:11: %s '''quux''' ^ -""" % (sourcePath,)]) +""" % (sourcePath, message)]) def test_eofSyntaxError(self): """ @@ -331,13 +435,22 @@ def test_eofSyntaxError(self): syntax error reflects the cause for the syntax error. """ sourcePath = self.makeTempFile("def foo(") - self.assertHasErrors( - sourcePath, - ["""\ + if PYPY: + result = """\ +%s:1:7: parenthesis is never closed +def foo( + ^ +""" % (sourcePath,) + else: + result = """\ %s:1:9: unexpected EOF while parsing def foo( ^ -""" % (sourcePath,)]) +""" % (sourcePath,) + + self.assertHasErrors( + sourcePath, + [result]) def test_eofSyntaxErrorWithTab(self): """ @@ -345,13 +458,16 @@ def test_eofSyntaxErrorWithTab(self): syntax error reflects the cause for the syntax error. """ sourcePath = self.makeTempFile("if True:\n\tfoo =") + column = 5 if PYPY else 7 + last_line = '\t ^' if PYPY else '\t ^' + self.assertHasErrors( sourcePath, ["""\ -%s:2:7: invalid syntax +%s:2:%s: invalid syntax \tfoo = -\t ^ -""" % (sourcePath,)]) +%s +""" % (sourcePath, column, last_line)]) def test_nonDefaultFollowsDefaultSyntaxError(self): """ @@ -364,8 +480,8 @@ def foo(bar=baz, bax): pass """ sourcePath = self.makeTempFile(source) - last_line = ' ^\n' if sys.version_info >= (3, 2) else '' - column = '8:' if sys.version_info >= (3, 2) else '' + last_line = ' ^\n' if ERROR_HAS_LAST_LINE else '' + column = '8:' if ERROR_HAS_COL_NUM else '' self.assertHasErrors( sourcePath, ["""\ @@ -383,8 +499,8 @@ def test_nonKeywordAfterKeywordSyntaxError(self): foo(bar=baz, bax) """ sourcePath = self.makeTempFile(source) - last_line = ' ^\n' if sys.version_info >= (3, 2) else '' - column = '13:' if sys.version_info >= (3, 2) else '' + last_line = ' ^\n' if ERROR_HAS_LAST_LINE else '' + column = '13:' if ERROR_HAS_COL_NUM or PYPY else '' if sys.version_info >= (3, 5): message = 'positional argument follows keyword argument' @@ -407,8 +523,15 @@ def test_invalidEscape(self): sourcePath = self.makeTempFile(r"foo = '\xyz'") if ver < (3,): decoding_error = "%s: problem decoding source\n" % (sourcePath,) + elif PYPY: + # pypy3 only + decoding_error = """\ +%s:1:6: %s: ('unicodeescape', b'\\\\xyz', 0, 2, 'truncated \\\\xXX escape') +foo = '\\xyz' + ^ +""" % (sourcePath, 'UnicodeDecodeError') else: - last_line = ' ^\n' if ver >= (3, 2) else '' + last_line = ' ^\n' if ERROR_HAS_LAST_LINE else '' # Column has been "fixed" since 3.2.4 and 3.3.1 col = 1 if ver >= (3, 3, 1) or ((3, 2, 4) <= ver < (3, 3)) else 2 decoding_error = """\ @@ -425,6 +548,9 @@ def test_permissionDenied(self): If the source file is not readable, this is reported on standard error. """ + if os.getuid() == 0: + self.skipTest('root user can access all files regardless of ' + 'permissions') sourcePath = self.makeTempFile('') os.chmod(sourcePath, 0) count, errors = self.getErrors(sourcePath) @@ -474,8 +600,21 @@ def test_misencodedFileUTF8(self): x = "%s" """ % SNOWMAN).encode('utf-8') sourcePath = self.makeTempFile(source) + + if PYPY and sys.version_info < (3, ): + message = ('\'ascii\' codec can\'t decode byte 0xe2 ' + 'in position 21: ordinal not in range(128)') + result = """\ +%s:0:0: %s +x = "\xe2\x98\x83" + ^\n""" % (sourcePath, message) + + else: + message = 'problem decoding source' + result = "%s: problem decoding source\n" % (sourcePath,) + self.assertHasErrors( - sourcePath, ["%s: problem decoding source\n" % (sourcePath,)]) + sourcePath, [result]) def test_misencodedFileUTF16(self): """ @@ -541,8 +680,8 @@ def runPyflakes(self, paths, stdin=None): """ Launch a subprocess running C{pyflakes}. - @param args: Command-line arguments to pass to pyflakes. - @param kwargs: Options passed on to C{subprocess.Popen}. + @param paths: Command-line arguments to pass to pyflakes. + @param stdin: Text to use as stdin. @return: C{(returncode, stdout, stderr)} of the completed pyflakes process. """ @@ -553,7 +692,7 @@ def runPyflakes(self, paths, stdin=None): if stdin: p = subprocess.Popen(command, env=env, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - (stdout, stderr) = p.communicate(stdin) + (stdout, stderr) = p.communicate(stdin.encode('ascii')) else: p = subprocess.Popen(command, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) @@ -562,6 +701,9 @@ def runPyflakes(self, paths, stdin=None): if sys.version_info >= (3,): stdout = stdout.decode('utf-8') stderr = stderr.decode('utf-8') + # Workaround https://bitbucket.org/pypy/pypy/issues/2350 + if PYPY and PY2 and WIN: + stderr = stderr.replace('\r\r\n', '\r\n') return (stdout, stderr, rv) def test_goodFile(self): @@ -586,7 +728,7 @@ def test_fileWithFlakes(self): expected = UnusedImport(self.tempfilepath, Node(1), 'contraband') self.assertEqual(d, ("%s%s" % (expected, os.linesep), '', 1)) - def test_errors(self): + def test_errors_io(self): """ When pyflakes finds errors with the files it's given, (if they don't exist, say), then the return code is non-zero and the errors are @@ -597,10 +739,39 @@ def test_errors(self): os.linesep) self.assertEqual(d, ('', error_msg, 1)) + def test_errors_syntax(self): + """ + When pyflakes finds errors with the files it's given, (if they don't + exist, say), then the return code is non-zero and the errors are + printed to stderr. + """ + fd = open(self.tempfilepath, 'wb') + fd.write("import".encode('ascii')) + fd.close() + d = self.runPyflakes([self.tempfilepath]) + error_msg = '{0}:1:{2}: invalid syntax{1}import{1} {3}^{1}'.format( + self.tempfilepath, os.linesep, 5 if PYPY else 7, '' if PYPY else ' ') + self.assertEqual(d, ('', error_msg, True)) + def test_readFromStdin(self): """ If no arguments are passed to C{pyflakes} then it reads from stdin. """ - d = self.runPyflakes([], stdin='import contraband'.encode('ascii')) + d = self.runPyflakes([], stdin='import contraband') expected = UnusedImport('', Node(1), 'contraband') self.assertEqual(d, ("%s%s" % (expected, os.linesep), '', 1)) + + +class TestMain(IntegrationTests): + """ + Tests of the pyflakes main function. + """ + + def runPyflakes(self, paths, stdin=None): + try: + with SysStreamCapturing(stdin) as capture: + main(args=paths) + except SystemExit as e: + return (capture.output, capture.error, e.code) + else: + raise RuntimeError('SystemExit not raised') diff --git a/contrib/pyflakes/test/test_dict.py b/contrib/pyflakes/test/test_dict.py new file mode 100644 index 0000000..628ec0c --- /dev/null +++ b/contrib/pyflakes/test/test_dict.py @@ -0,0 +1,217 @@ +""" +Tests for dict duplicate keys Pyflakes behavior. +""" + +from sys import version_info + +from pyflakes import messages as m +from pyflakes.test.harness import TestCase, skipIf + + +class Test(TestCase): + + def test_duplicate_keys(self): + self.flakes( + "{'yes': 1, 'yes': 2}", + m.MultiValueRepeatedKeyLiteral, + m.MultiValueRepeatedKeyLiteral, + ) + + @skipIf(version_info < (3,), + "bytes and strings with same 'value' are not equal in python3") + @skipIf(version_info[0:2] == (3, 2), + "python3.2 does not allow u"" literal string definition") + def test_duplicate_keys_bytes_vs_unicode_py3(self): + self.flakes("{b'a': 1, u'a': 2}") + + @skipIf(version_info < (3,), + "bytes and strings with same 'value' are not equal in python3") + @skipIf(version_info[0:2] == (3, 2), + "python3.2 does not allow u"" literal string definition") + def test_duplicate_values_bytes_vs_unicode_py3(self): + self.flakes( + "{1: b'a', 1: u'a'}", + m.MultiValueRepeatedKeyLiteral, + m.MultiValueRepeatedKeyLiteral, + ) + + @skipIf(version_info >= (3,), + "bytes and strings with same 'value' are equal in python2") + def test_duplicate_keys_bytes_vs_unicode_py2(self): + self.flakes( + "{b'a': 1, u'a': 2}", + m.MultiValueRepeatedKeyLiteral, + m.MultiValueRepeatedKeyLiteral, + ) + + @skipIf(version_info >= (3,), + "bytes and strings with same 'value' are equal in python2") + def test_duplicate_values_bytes_vs_unicode_py2(self): + self.flakes("{1: b'a', 1: u'a'}") + + def test_multiple_duplicate_keys(self): + self.flakes( + "{'yes': 1, 'yes': 2, 'no': 2, 'no': 3}", + m.MultiValueRepeatedKeyLiteral, + m.MultiValueRepeatedKeyLiteral, + m.MultiValueRepeatedKeyLiteral, + m.MultiValueRepeatedKeyLiteral, + ) + + def test_duplicate_keys_in_function(self): + self.flakes( + ''' + def f(thing): + pass + f({'yes': 1, 'yes': 2}) + ''', + m.MultiValueRepeatedKeyLiteral, + m.MultiValueRepeatedKeyLiteral, + ) + + def test_duplicate_keys_in_lambda(self): + self.flakes( + "lambda x: {(0,1): 1, (0,1): 2}", + m.MultiValueRepeatedKeyLiteral, + m.MultiValueRepeatedKeyLiteral, + ) + + def test_duplicate_keys_tuples(self): + self.flakes( + "{(0,1): 1, (0,1): 2}", + m.MultiValueRepeatedKeyLiteral, + m.MultiValueRepeatedKeyLiteral, + ) + + def test_duplicate_keys_tuples_int_and_float(self): + self.flakes( + "{(0,1): 1, (0,1.0): 2}", + m.MultiValueRepeatedKeyLiteral, + m.MultiValueRepeatedKeyLiteral, + ) + + def test_duplicate_keys_ints(self): + self.flakes( + "{1: 1, 1: 2}", + m.MultiValueRepeatedKeyLiteral, + m.MultiValueRepeatedKeyLiteral, + ) + + def test_duplicate_keys_bools(self): + self.flakes( + "{True: 1, True: 2}", + m.MultiValueRepeatedKeyLiteral, + m.MultiValueRepeatedKeyLiteral, + ) + + def test_duplicate_keys_bools_false(self): + # Needed to ensure 2.x correctly coerces these from variables + self.flakes( + "{False: 1, False: 2}", + m.MultiValueRepeatedKeyLiteral, + m.MultiValueRepeatedKeyLiteral, + ) + + def test_duplicate_keys_none(self): + self.flakes( + "{None: 1, None: 2}", + m.MultiValueRepeatedKeyLiteral, + m.MultiValueRepeatedKeyLiteral, + ) + + def test_duplicate_variable_keys(self): + self.flakes( + ''' + a = 1 + {a: 1, a: 2} + ''', + m.MultiValueRepeatedKeyVariable, + m.MultiValueRepeatedKeyVariable, + ) + + def test_duplicate_variable_values(self): + self.flakes( + ''' + a = 1 + b = 2 + {1: a, 1: b} + ''', + m.MultiValueRepeatedKeyLiteral, + m.MultiValueRepeatedKeyLiteral, + ) + + def test_duplicate_variable_values_same_value(self): + # Current behaviour is not to look up variable values. This is to + # confirm that. + self.flakes( + ''' + a = 1 + b = 1 + {1: a, 1: b} + ''', + m.MultiValueRepeatedKeyLiteral, + m.MultiValueRepeatedKeyLiteral, + ) + + def test_duplicate_key_float_and_int(self): + """ + These do look like different values, but when it comes to their use as + keys, they compare as equal and so are actually duplicates. + The literal dict {1: 1, 1.0: 1} actually becomes {1.0: 1}. + """ + self.flakes( + ''' + {1: 1, 1.0: 2} + ''', + m.MultiValueRepeatedKeyLiteral, + m.MultiValueRepeatedKeyLiteral, + ) + + def test_no_duplicate_key_error_same_value(self): + self.flakes(''' + {'yes': 1, 'yes': 1} + ''') + + def test_no_duplicate_key_errors(self): + self.flakes(''' + {'yes': 1, 'no': 2} + ''') + + def test_no_duplicate_keys_tuples_same_first_element(self): + self.flakes("{(0,1): 1, (0,2): 1}") + + def test_no_duplicate_key_errors_func_call(self): + self.flakes(''' + def test(thing): + pass + test({True: 1, None: 2, False: 1}) + ''') + + def test_no_duplicate_key_errors_bool_or_none(self): + self.flakes("{True: 1, None: 2, False: 1}") + + def test_no_duplicate_key_errors_ints(self): + self.flakes(''' + {1: 1, 2: 1} + ''') + + def test_no_duplicate_key_errors_vars(self): + self.flakes(''' + test = 'yes' + rest = 'yes' + {test: 1, rest: 2} + ''') + + def test_no_duplicate_key_errors_tuples(self): + self.flakes(''' + {(0,1): 1, (0,2): 1} + ''') + + def test_no_duplicate_key_errors_instance_attributes(self): + self.flakes(''' + class Test(): + pass + f = Test() + f.a = 1 + {f.a: 1, f.a: 1} + ''') diff --git a/contrib/pyflakes/test/test_doctests.py b/contrib/pyflakes/test/test_doctests.py index 6a4ba00..aed7957 100644 --- a/contrib/pyflakes/test/test_doctests.py +++ b/contrib/pyflakes/test/test_doctests.py @@ -1,11 +1,23 @@ +import sys import textwrap from pyflakes import messages as m +from pyflakes.checker import ( + DoctestScope, + FunctionScope, + ModuleScope, +) from pyflakes.test.test_other import Test as TestOther from pyflakes.test.test_imports import Test as TestImports from pyflakes.test.test_undefined_names import Test as TestUndefinedNames from pyflakes.test.harness import TestCase, skip +try: + sys.pypy_version_info + PYPY = True +except AttributeError: + PYPY = False + class _DoctestMixin(object): @@ -42,6 +54,190 @@ class Test(TestCase): withDoctest = True + def test_scope_class(self): + """Check that a doctest is given a DoctestScope.""" + checker = self.flakes(""" + m = None + + def doctest_stuff(): + ''' + >>> d = doctest_stuff() + ''' + f = m + return f + """) + + scopes = checker.deadScopes + module_scopes = [ + scope for scope in scopes if scope.__class__ is ModuleScope] + doctest_scopes = [ + scope for scope in scopes if scope.__class__ is DoctestScope] + function_scopes = [ + scope for scope in scopes if scope.__class__ is FunctionScope] + + self.assertEqual(len(module_scopes), 1) + self.assertEqual(len(doctest_scopes), 1) + + module_scope = module_scopes[0] + doctest_scope = doctest_scopes[0] + + self.assertIsInstance(doctest_scope, DoctestScope) + self.assertIsInstance(doctest_scope, ModuleScope) + self.assertNotIsInstance(doctest_scope, FunctionScope) + self.assertNotIsInstance(module_scope, DoctestScope) + + self.assertIn('m', module_scope) + self.assertIn('doctest_stuff', module_scope) + + self.assertIn('d', doctest_scope) + + self.assertEqual(len(function_scopes), 1) + self.assertIn('f', function_scopes[0]) + + def test_nested_doctest_ignored(self): + """Check that nested doctests are ignored.""" + checker = self.flakes(""" + m = None + + def doctest_stuff(): + ''' + >>> def function_in_doctest(): + ... \"\"\" + ... >>> ignored_undefined_name + ... \"\"\" + ... df = m + ... return df + ... + >>> function_in_doctest() + ''' + f = m + return f + """) + + scopes = checker.deadScopes + module_scopes = [ + scope for scope in scopes if scope.__class__ is ModuleScope] + doctest_scopes = [ + scope for scope in scopes if scope.__class__ is DoctestScope] + function_scopes = [ + scope for scope in scopes if scope.__class__ is FunctionScope] + + self.assertEqual(len(module_scopes), 1) + self.assertEqual(len(doctest_scopes), 1) + + module_scope = module_scopes[0] + doctest_scope = doctest_scopes[0] + + self.assertIn('m', module_scope) + self.assertIn('doctest_stuff', module_scope) + self.assertIn('function_in_doctest', doctest_scope) + + self.assertEqual(len(function_scopes), 2) + + self.assertIn('f', function_scopes[0]) + self.assertIn('df', function_scopes[1]) + + def test_global_module_scope_pollution(self): + """Check that global in doctest does not pollute module scope.""" + checker = self.flakes(""" + def doctest_stuff(): + ''' + >>> def function_in_doctest(): + ... global m + ... m = 50 + ... df = 10 + ... m = df + ... + >>> function_in_doctest() + ''' + f = 10 + return f + + """) + + scopes = checker.deadScopes + module_scopes = [ + scope for scope in scopes if scope.__class__ is ModuleScope] + doctest_scopes = [ + scope for scope in scopes if scope.__class__ is DoctestScope] + function_scopes = [ + scope for scope in scopes if scope.__class__ is FunctionScope] + + self.assertEqual(len(module_scopes), 1) + self.assertEqual(len(doctest_scopes), 1) + + module_scope = module_scopes[0] + doctest_scope = doctest_scopes[0] + + self.assertIn('doctest_stuff', module_scope) + self.assertIn('function_in_doctest', doctest_scope) + + self.assertEqual(len(function_scopes), 2) + + self.assertIn('f', function_scopes[0]) + self.assertIn('df', function_scopes[1]) + self.assertIn('m', function_scopes[1]) + + self.assertNotIn('m', module_scope) + + def test_global_undefined(self): + self.flakes(""" + global m + + def doctest_stuff(): + ''' + >>> m + ''' + """, m.UndefinedName) + + def test_nested_class(self): + """Doctest within nested class are processed.""" + self.flakes(""" + class C: + class D: + ''' + >>> m + ''' + def doctest_stuff(self): + ''' + >>> m + ''' + return 1 + """, m.UndefinedName, m.UndefinedName) + + def test_ignore_nested_function(self): + """Doctest module does not process doctest in nested functions.""" + # 'syntax error' would cause a SyntaxError if the doctest was processed. + # However doctest does not find doctest in nested functions + # (https://bugs.python.org/issue1650090). If nested functions were + # processed, this use of m should cause UndefinedName, and the + # name inner_function should probably exist in the doctest scope. + self.flakes(""" + def doctest_stuff(): + def inner_function(): + ''' + >>> syntax error + >>> inner_function() + 1 + >>> m + ''' + return 1 + m = inner_function() + return m + """) + + def test_inaccessible_scope_class(self): + """Doctest may not access class scope.""" + self.flakes(""" + class C: + def doctest_stuff(self): + ''' + >>> m + ''' + return 1 + m = 1 + """, m.UndefinedName) + def test_importBeforeDoctest(self): self.flakes(""" import foo @@ -132,12 +328,22 @@ def doctest_stuff(): exc = exceptions[0] self.assertEqual(exc.lineno, 4) self.assertEqual(exc.col, 26) + + # PyPy error column offset is 0, + # for the second and third line of the doctest + # i.e. at the beginning of the line exc = exceptions[1] self.assertEqual(exc.lineno, 5) - self.assertEqual(exc.col, 16) + if PYPY: + self.assertEqual(exc.col, 13) + else: + self.assertEqual(exc.col, 16) exc = exceptions[2] self.assertEqual(exc.lineno, 6) - self.assertEqual(exc.col, 18) + if PYPY: + self.assertEqual(exc.col, 13) + else: + self.assertEqual(exc.col, 18) def test_indentationErrorInDoctest(self): exc = self.flakes(''' @@ -148,7 +354,10 @@ def doctest_stuff(): """ ''', m.DoctestSyntaxError).messages[0] self.assertEqual(exc.lineno, 5) - self.assertEqual(exc.col, 16) + if PYPY: + self.assertEqual(exc.col, 13) + else: + self.assertEqual(exc.col, 16) def test_offsetWithMultiLineArgs(self): (exc1, exc2) = self.flakes( @@ -222,35 +431,12 @@ def func(): class TestOther(_DoctestMixin, TestOther): - pass + """Run TestOther with each test wrapped in a doctest.""" class TestImports(_DoctestMixin, TestImports): - - def test_futureImport(self): - """XXX This test can't work in a doctest""" - - def test_futureImportUsed(self): - """XXX This test can't work in a doctest""" + """Run TestImports with each test wrapped in a doctest.""" class TestUndefinedNames(_DoctestMixin, TestUndefinedNames): - - def test_doubleNestingReportsClosestName(self): - """ - Lines in doctest are a bit different so we can't use the test - from TestUndefinedNames - """ - exc = self.flakes(''' - def a(): - x = 1 - def b(): - x = 2 # line 7 in the file - def c(): - x - x = 3 - return x - return x - return x - ''', m.UndefinedLocal).messages[0] - self.assertEqual(exc.message_args, ('x', 7)) + """Run TestUndefinedNames with each test wrapped in a doctest.""" diff --git a/contrib/pyflakes/test/test_imports.py b/contrib/pyflakes/test/test_imports.py index 6cd0ec5..3e3be88 100644 --- a/contrib/pyflakes/test/test_imports.py +++ b/contrib/pyflakes/test/test_imports.py @@ -2,26 +2,152 @@ from sys import version_info from pyflakes import messages as m +from pyflakes.checker import ( + FutureImportation, + Importation, + ImportationFrom, + StarImportation, + SubmoduleImportation, +) from pyflakes.test.harness import TestCase, skip, skipIf +class TestImportationObject(TestCase): + + def test_import_basic(self): + binding = Importation('a', None, 'a') + assert binding.source_statement == 'import a' + assert str(binding) == 'a' + + def test_import_as(self): + binding = Importation('c', None, 'a') + assert binding.source_statement == 'import a as c' + assert str(binding) == 'a as c' + + def test_import_submodule(self): + binding = SubmoduleImportation('a.b', None) + assert binding.source_statement == 'import a.b' + assert str(binding) == 'a.b' + + def test_import_submodule_as(self): + # A submodule import with an as clause is not a SubmoduleImportation + binding = Importation('c', None, 'a.b') + assert binding.source_statement == 'import a.b as c' + assert str(binding) == 'a.b as c' + + def test_import_submodule_as_source_name(self): + binding = Importation('a', None, 'a.b') + assert binding.source_statement == 'import a.b as a' + assert str(binding) == 'a.b as a' + + def test_importfrom_relative(self): + binding = ImportationFrom('a', None, '.', 'a') + assert binding.source_statement == 'from . import a' + assert str(binding) == '.a' + + def test_importfrom_relative_parent(self): + binding = ImportationFrom('a', None, '..', 'a') + assert binding.source_statement == 'from .. import a' + assert str(binding) == '..a' + + def test_importfrom_relative_with_module(self): + binding = ImportationFrom('b', None, '..a', 'b') + assert binding.source_statement == 'from ..a import b' + assert str(binding) == '..a.b' + + def test_importfrom_relative_with_module_as(self): + binding = ImportationFrom('c', None, '..a', 'b') + assert binding.source_statement == 'from ..a import b as c' + assert str(binding) == '..a.b as c' + + def test_importfrom_member(self): + binding = ImportationFrom('b', None, 'a', 'b') + assert binding.source_statement == 'from a import b' + assert str(binding) == 'a.b' + + def test_importfrom_submodule_member(self): + binding = ImportationFrom('c', None, 'a.b', 'c') + assert binding.source_statement == 'from a.b import c' + assert str(binding) == 'a.b.c' + + def test_importfrom_member_as(self): + binding = ImportationFrom('c', None, 'a', 'b') + assert binding.source_statement == 'from a import b as c' + assert str(binding) == 'a.b as c' + + def test_importfrom_submodule_member_as(self): + binding = ImportationFrom('d', None, 'a.b', 'c') + assert binding.source_statement == 'from a.b import c as d' + assert str(binding) == 'a.b.c as d' + + def test_importfrom_star(self): + binding = StarImportation('a.b', None) + assert binding.source_statement == 'from a.b import *' + assert str(binding) == 'a.b.*' + + def test_importfrom_star_relative(self): + binding = StarImportation('.b', None) + assert binding.source_statement == 'from .b import *' + assert str(binding) == '.b.*' + + def test_importfrom_future(self): + binding = FutureImportation('print_function', None, None) + assert binding.source_statement == 'from __future__ import print_function' + assert str(binding) == '__future__.print_function' + + class Test(TestCase): def test_unusedImport(self): self.flakes('import fu, bar', m.UnusedImport, m.UnusedImport) self.flakes('from baz import fu, bar', m.UnusedImport, m.UnusedImport) + def test_unusedImport_relative(self): + self.flakes('from . import fu', m.UnusedImport) + self.flakes('from . import fu as baz', m.UnusedImport) + self.flakes('from .. import fu', m.UnusedImport) + self.flakes('from ... import fu', m.UnusedImport) + self.flakes('from .. import fu as baz', m.UnusedImport) + self.flakes('from .bar import fu', m.UnusedImport) + self.flakes('from ..bar import fu', m.UnusedImport) + self.flakes('from ...bar import fu', m.UnusedImport) + self.flakes('from ...bar import fu as baz', m.UnusedImport) + + checker = self.flakes('from . import fu', m.UnusedImport) + + error = checker.messages[0] + assert error.message == '%r imported but unused' + assert error.message_args == ('.fu', ) + + checker = self.flakes('from . import fu as baz', m.UnusedImport) + + error = checker.messages[0] + assert error.message == '%r imported but unused' + assert error.message_args == ('.fu as baz', ) + def test_aliasedImport(self): self.flakes('import fu as FU, bar as FU', m.RedefinedWhileUnused, m.UnusedImport) self.flakes('from moo import fu as FU, bar as FU', m.RedefinedWhileUnused, m.UnusedImport) + def test_aliasedImportShadowModule(self): + """Imported aliases can shadow the source of the import.""" + self.flakes('from moo import fu as moo; moo') + self.flakes('import fu as fu; fu') + self.flakes('import fu.bar as fu; fu') + def test_usedImport(self): self.flakes('import fu; print(fu)') self.flakes('from baz import fu; print(fu)') self.flakes('import fu; del fu') + def test_usedImport_relative(self): + self.flakes('from . import fu; assert fu') + self.flakes('from .bar import fu; assert fu') + self.flakes('from .. import fu; assert fu') + self.flakes('from ..bar import fu as baz; assert baz') + def test_redefinedWhileUnused(self): self.flakes('import fu; fu = 3', m.RedefinedWhileUnused) self.flakes('import fu; fu, bar = 3', m.RedefinedWhileUnused) @@ -54,7 +180,7 @@ def test_redefinedIfElse(self): def test_redefinedTry(self): """ - Test that importing a module twice in an try block + Test that importing a module twice in a try block does raise a warning. """ self.flakes(''' @@ -67,7 +193,7 @@ def test_redefinedTry(self): def test_redefinedTryExcept(self): """ - Test that importing a module twice in an try + Test that importing a module twice in a try and except block does not raise a warning. """ self.flakes(''' @@ -236,6 +362,22 @@ class bar: print(fu) ''') + def test_importInClass(self): + """ + Test that import within class is a locally scoped attribute. + """ + self.flakes(''' + class bar: + import fu + ''') + + self.flakes(''' + class bar: + import fu + + fu + ''', m.UndefinedName) + def test_usedInFunction(self): self.flakes(''' import fu @@ -570,7 +712,7 @@ class bar: import fu def fun(self): fu - ''', m.UnusedImport, m.UndefinedName) + ''', m.UndefinedName) def test_nestedFunctionsNestScope(self): self.flakes(''' @@ -590,7 +732,89 @@ def c(self): ''') def test_importStar(self): - self.flakes('from fu import *', m.ImportStarUsed) + """Use of import * at module level is reported.""" + self.flakes('from fu import *', m.ImportStarUsed, m.UnusedImport) + self.flakes(''' + try: + from fu import * + except: + pass + ''', m.ImportStarUsed, m.UnusedImport) + + checker = self.flakes('from fu import *', + m.ImportStarUsed, m.UnusedImport) + + error = checker.messages[0] + assert error.message.startswith("'from %s import *' used; unable ") + assert error.message_args == ('fu', ) + + error = checker.messages[1] + assert error.message == '%r imported but unused' + assert error.message_args == ('fu.*', ) + + def test_importStar_relative(self): + """Use of import * from a relative import is reported.""" + self.flakes('from .fu import *', m.ImportStarUsed, m.UnusedImport) + self.flakes(''' + try: + from .fu import * + except: + pass + ''', m.ImportStarUsed, m.UnusedImport) + + checker = self.flakes('from .fu import *', + m.ImportStarUsed, m.UnusedImport) + + error = checker.messages[0] + assert error.message.startswith("'from %s import *' used; unable ") + assert error.message_args == ('.fu', ) + + error = checker.messages[1] + assert error.message == '%r imported but unused' + assert error.message_args == ('.fu.*', ) + + checker = self.flakes('from .. import *', + m.ImportStarUsed, m.UnusedImport) + + error = checker.messages[0] + assert error.message.startswith("'from %s import *' used; unable ") + assert error.message_args == ('..', ) + + error = checker.messages[1] + assert error.message == '%r imported but unused' + assert error.message_args == ('from .. import *', ) + + @skipIf(version_info < (3,), + 'import * below module level is a warning on Python 2') + def test_localImportStar(self): + """import * is only allowed at module level.""" + self.flakes(''' + def a(): + from fu import * + ''', m.ImportStarNotPermitted) + self.flakes(''' + class a: + from fu import * + ''', m.ImportStarNotPermitted) + + checker = self.flakes(''' + class a: + from .. import * + ''', m.ImportStarNotPermitted) + error = checker.messages[0] + assert error.message == "'from %s import *' only allowed at module level" + assert error.message_args == ('..', ) + + @skipIf(version_info > (3,), + 'import * below module level is an error on Python 3') + def test_importStarNested(self): + """All star imports are marked as used by an undefined variable.""" + self.flakes(''' + from fu import * + def f(): + from bar import * + x + ''', m.ImportStarUsed, m.ImportStarUsed, m.ImportStarUsage) def test_packageImport(self): """ @@ -638,6 +862,35 @@ def test_differentSubmoduleImport(self): fu.bar, fu.baz ''') + def test_used_package_with_submodule_import(self): + """ + Usage of package marks submodule imports as used. + """ + self.flakes(''' + import fu + import fu.bar + fu.x + ''') + + self.flakes(''' + import fu.bar + import fu + fu.x + ''') + + def test_unused_package_with_submodule_import(self): + """ + When a package and its submodule are imported, only report once. + """ + checker = self.flakes(''' + import fu + import fu.bar + ''', m.UnusedImport) + error = checker.messages[0] + assert error.message == '%r imported but unused' + assert error.message_args == ('fu.bar', ) + assert error.lineno == 5 if self.withDoctest else 3 + def test_assignRHSFirst(self): self.flakes('import fu; fu = fu') self.flakes('import fu; fu, bar = fu') @@ -689,7 +942,6 @@ def test_importingForImportError(self): pass ''') - @skip("todo: requires evaluating attribute access") def test_importedInClass(self): """Imports in class scope can be used through self.""" self.flakes(''' @@ -742,6 +994,18 @@ def test_futureImportUsed(self): assert print_function is not division ''') + def test_futureImportUndefined(self): + """Importing undefined names from __future__ fails.""" + self.flakes(''' + from __future__ import print_statement + ''', m.FutureFeatureNotDefined) + + def test_futureImportStar(self): + """Importing '*' from __future__ fails.""" + self.flakes(''' + from __future__ import * + ''', m.FutureFeatureNotDefined) + class TestSpecialAll(TestCase): """ @@ -760,12 +1024,11 @@ def foo(): def test_ignoredInClass(self): """ - An C{__all__} definition does not suppress unused import warnings in a - class scope. + An C{__all__} definition in a class does not suppress unused import warnings. """ self.flakes(''' + import bar class foo: - import bar __all__ = ["bar"] ''', m.UnusedImport) @@ -834,6 +1097,14 @@ def test_importStarExported(self): __all__ = ["foo"] ''', m.ImportStarUsed) + def test_importStarNotExported(self): + """Report unused import when not needed to satisfy __all__.""" + self.flakes(''' + from foolib import * + a = 1 + __all__ = ['a'] + ''', m.ImportStarUsed, m.UnusedImport) + def test_usedInGenExp(self): """ Using a global in a generator expression results in no warnings. @@ -878,7 +1149,7 @@ def f(): class Python26Tests(TestCase): """ - Tests for checking of syntax which is valid in PYthon 2.6 and newer. + Tests for checking of syntax which is valid in Python 2.6 and newer. """ @skipIf(version_info < (2, 6), "Python >= 2.6 only") diff --git a/contrib/pyflakes/test/test_other.py b/contrib/pyflakes/test/test_other.py index 384c4b3..9c8462e 100644 --- a/contrib/pyflakes/test/test_other.py +++ b/contrib/pyflakes/test/test_other.py @@ -353,6 +353,664 @@ class Foo(object): return ''', m.ReturnOutsideFunction) + def test_moduleWithReturn(self): + """ + If a return is used at the module level, a warning is emitted. + """ + self.flakes(''' + return + ''', m.ReturnOutsideFunction) + + def test_classWithYield(self): + """ + If a yield is used inside a class, a warning is emitted. + """ + self.flakes(''' + class Foo(object): + yield + ''', m.YieldOutsideFunction) + + def test_moduleWithYield(self): + """ + If a yield is used at the module level, a warning is emitted. + """ + self.flakes(''' + yield + ''', m.YieldOutsideFunction) + + @skipIf(version_info < (3, 3), "Python >= 3.3 only") + def test_classWithYieldFrom(self): + """ + If a yield from is used inside a class, a warning is emitted. + """ + self.flakes(''' + class Foo(object): + yield from range(10) + ''', m.YieldOutsideFunction) + + @skipIf(version_info < (3, 3), "Python >= 3.3 only") + def test_moduleWithYieldFrom(self): + """ + If a yield from is used at the module level, a warning is emitted. + """ + self.flakes(''' + yield from range(10) + ''', m.YieldOutsideFunction) + + def test_continueOutsideLoop(self): + self.flakes(''' + continue + ''', m.ContinueOutsideLoop) + + self.flakes(''' + def f(): + continue + ''', m.ContinueOutsideLoop) + + self.flakes(''' + while True: + pass + else: + continue + ''', m.ContinueOutsideLoop) + + self.flakes(''' + while True: + pass + else: + if 1: + if 2: + continue + ''', m.ContinueOutsideLoop) + + self.flakes(''' + while True: + def f(): + continue + ''', m.ContinueOutsideLoop) + + self.flakes(''' + while True: + class A: + continue + ''', m.ContinueOutsideLoop) + + def test_continueInsideLoop(self): + self.flakes(''' + while True: + continue + ''') + + self.flakes(''' + for i in range(10): + continue + ''') + + self.flakes(''' + while True: + if 1: + continue + ''') + + self.flakes(''' + for i in range(10): + if 1: + continue + ''') + + self.flakes(''' + while True: + while True: + pass + else: + continue + else: + pass + ''') + + self.flakes(''' + while True: + try: + pass + finally: + while True: + continue + ''') + + def test_continueInFinally(self): + # 'continue' inside 'finally' is a special syntax error + self.flakes(''' + while True: + try: + pass + finally: + continue + ''', m.ContinueInFinally) + + self.flakes(''' + while True: + try: + pass + finally: + if 1: + if 2: + continue + ''', m.ContinueInFinally) + + # Even when not in a loop, this is the error Python gives + self.flakes(''' + try: + pass + finally: + continue + ''', m.ContinueInFinally) + + def test_breakOutsideLoop(self): + self.flakes(''' + break + ''', m.BreakOutsideLoop) + + self.flakes(''' + def f(): + break + ''', m.BreakOutsideLoop) + + self.flakes(''' + while True: + pass + else: + break + ''', m.BreakOutsideLoop) + + self.flakes(''' + while True: + pass + else: + if 1: + if 2: + break + ''', m.BreakOutsideLoop) + + self.flakes(''' + while True: + def f(): + break + ''', m.BreakOutsideLoop) + + self.flakes(''' + while True: + class A: + break + ''', m.BreakOutsideLoop) + + self.flakes(''' + try: + pass + finally: + break + ''', m.BreakOutsideLoop) + + def test_breakInsideLoop(self): + self.flakes(''' + while True: + break + ''') + + self.flakes(''' + for i in range(10): + break + ''') + + self.flakes(''' + while True: + if 1: + break + ''') + + self.flakes(''' + for i in range(10): + if 1: + break + ''') + + self.flakes(''' + while True: + while True: + pass + else: + break + else: + pass + ''') + + self.flakes(''' + while True: + try: + pass + finally: + while True: + break + ''') + + self.flakes(''' + while True: + try: + pass + finally: + break + ''') + + self.flakes(''' + while True: + try: + pass + finally: + if 1: + if 2: + break + ''') + + def test_defaultExceptLast(self): + """ + A default except block should be last. + + YES: + + try: + ... + except Exception: + ... + except: + ... + + NO: + + try: + ... + except: + ... + except Exception: + ... + """ + self.flakes(''' + try: + pass + except ValueError: + pass + ''') + + self.flakes(''' + try: + pass + except ValueError: + pass + except: + pass + ''') + + self.flakes(''' + try: + pass + except: + pass + ''') + + self.flakes(''' + try: + pass + except ValueError: + pass + else: + pass + ''') + + self.flakes(''' + try: + pass + except: + pass + else: + pass + ''') + + self.flakes(''' + try: + pass + except ValueError: + pass + except: + pass + else: + pass + ''') + + def test_defaultExceptNotLast(self): + self.flakes(''' + try: + pass + except: + pass + except ValueError: + pass + ''', m.DefaultExceptNotLast) + + self.flakes(''' + try: + pass + except: + pass + except: + pass + ''', m.DefaultExceptNotLast) + + self.flakes(''' + try: + pass + except: + pass + except ValueError: + pass + except: + pass + ''', m.DefaultExceptNotLast) + + self.flakes(''' + try: + pass + except: + pass + except ValueError: + pass + except: + pass + except ValueError: + pass + ''', m.DefaultExceptNotLast, m.DefaultExceptNotLast) + + self.flakes(''' + try: + pass + except: + pass + except ValueError: + pass + else: + pass + ''', m.DefaultExceptNotLast) + + self.flakes(''' + try: + pass + except: + pass + except: + pass + else: + pass + ''', m.DefaultExceptNotLast) + + self.flakes(''' + try: + pass + except: + pass + except ValueError: + pass + except: + pass + else: + pass + ''', m.DefaultExceptNotLast) + + self.flakes(''' + try: + pass + except: + pass + except ValueError: + pass + except: + pass + except ValueError: + pass + else: + pass + ''', m.DefaultExceptNotLast, m.DefaultExceptNotLast) + + self.flakes(''' + try: + pass + except: + pass + except ValueError: + pass + finally: + pass + ''', m.DefaultExceptNotLast) + + self.flakes(''' + try: + pass + except: + pass + except: + pass + finally: + pass + ''', m.DefaultExceptNotLast) + + self.flakes(''' + try: + pass + except: + pass + except ValueError: + pass + except: + pass + finally: + pass + ''', m.DefaultExceptNotLast) + + self.flakes(''' + try: + pass + except: + pass + except ValueError: + pass + except: + pass + except ValueError: + pass + finally: + pass + ''', m.DefaultExceptNotLast, m.DefaultExceptNotLast) + + self.flakes(''' + try: + pass + except: + pass + except ValueError: + pass + else: + pass + finally: + pass + ''', m.DefaultExceptNotLast) + + self.flakes(''' + try: + pass + except: + pass + except: + pass + else: + pass + finally: + pass + ''', m.DefaultExceptNotLast) + + self.flakes(''' + try: + pass + except: + pass + except ValueError: + pass + except: + pass + else: + pass + finally: + pass + ''', m.DefaultExceptNotLast) + + self.flakes(''' + try: + pass + except: + pass + except ValueError: + pass + except: + pass + except ValueError: + pass + else: + pass + finally: + pass + ''', m.DefaultExceptNotLast, m.DefaultExceptNotLast) + + @skipIf(version_info < (3,), "Python 3 only") + def test_starredAssignmentNoError(self): + """ + Python 3 extended iterable unpacking + """ + self.flakes(''' + a, *b = range(10) + ''') + + self.flakes(''' + *a, b = range(10) + ''') + + self.flakes(''' + a, *b, c = range(10) + ''') + + self.flakes(''' + (a, *b) = range(10) + ''') + + self.flakes(''' + (*a, b) = range(10) + ''') + + self.flakes(''' + (a, *b, c) = range(10) + ''') + + self.flakes(''' + [a, *b] = range(10) + ''') + + self.flakes(''' + [*a, b] = range(10) + ''') + + self.flakes(''' + [a, *b, c] = range(10) + ''') + + # Taken from test_unpack_ex.py in the cPython source + s = ", ".join("a%d" % i for i in range(1 << 8 - 1)) + \ + ", *rest = range(1<<8)" + self.flakes(s) + + s = "(" + ", ".join("a%d" % i for i in range(1 << 8 - 1)) + \ + ", *rest) = range(1<<8)" + self.flakes(s) + + s = "[" + ", ".join("a%d" % i for i in range(1 << 8 - 1)) + \ + ", *rest] = range(1<<8)" + self.flakes(s) + + @skipIf(version_info < (3, ), "Python 3 only") + def test_starredAssignmentErrors(self): + """ + SyntaxErrors (not encoded in the ast) surrounding Python 3 extended + iterable unpacking + """ + # Taken from test_unpack_ex.py in the cPython source + s = ", ".join("a%d" % i for i in range(1 << 8)) + \ + ", *rest = range(1<<8 + 1)" + self.flakes(s, m.TooManyExpressionsInStarredAssignment) + + s = "(" + ", ".join("a%d" % i for i in range(1 << 8)) + \ + ", *rest) = range(1<<8 + 1)" + self.flakes(s, m.TooManyExpressionsInStarredAssignment) + + s = "[" + ", ".join("a%d" % i for i in range(1 << 8)) + \ + ", *rest] = range(1<<8 + 1)" + self.flakes(s, m.TooManyExpressionsInStarredAssignment) + + s = ", ".join("a%d" % i for i in range(1 << 8 + 1)) + \ + ", *rest = range(1<<8 + 2)" + self.flakes(s, m.TooManyExpressionsInStarredAssignment) + + s = "(" + ", ".join("a%d" % i for i in range(1 << 8 + 1)) + \ + ", *rest) = range(1<<8 + 2)" + self.flakes(s, m.TooManyExpressionsInStarredAssignment) + + s = "[" + ", ".join("a%d" % i for i in range(1 << 8 + 1)) + \ + ", *rest] = range(1<<8 + 2)" + self.flakes(s, m.TooManyExpressionsInStarredAssignment) + + # No way we can actually test this! + # s = "*rest, " + ", ".join("a%d" % i for i in range(1<<24)) + \ + # ", *rest = range(1<<24 + 1)" + # self.flakes(s, m.TooManyExpressionsInStarredAssignment) + + self.flakes(''' + a, *b, *c = range(10) + ''', m.TwoStarredExpressions) + + self.flakes(''' + a, *b, c, *d = range(10) + ''', m.TwoStarredExpressions) + + self.flakes(''' + *a, *b, *c = range(10) + ''', m.TwoStarredExpressions) + + self.flakes(''' + (a, *b, *c) = range(10) + ''', m.TwoStarredExpressions) + + self.flakes(''' + (a, *b, c, *d) = range(10) + ''', m.TwoStarredExpressions) + + self.flakes(''' + (*a, *b, *c) = range(10) + ''', m.TwoStarredExpressions) + + self.flakes(''' + [a, *b, *c] = range(10) + ''', m.TwoStarredExpressions) + + self.flakes(''' + [a, *b, c, *d] = range(10) + ''', m.TwoStarredExpressions) + + self.flakes(''' + [*a, *b, *c] = range(10) + ''', m.TwoStarredExpressions) + @skip("todo: Too hard to make this warn but other cases stay silent") def test_doubleAssignment(self): """ @@ -523,7 +1181,7 @@ def a(): return ''', m.UnusedVariable) - @skip("todo: Difficult because it does't apply in the context of a loop") + @skip("todo: Difficult because it doesn't apply in the context of a loop") def test_unusedReassignedVariable(self): """ Shadowing a used variable can still raise an UnusedVariable warning. @@ -831,7 +1489,7 @@ def test_withStatementSingleNameUndefined(self): def test_withStatementTupleNamesUndefined(self): """ - An undefined name warning is emitted if a name first defined by a the + An undefined name warning is emitted if a name first defined by the tuple-unpacking form of the C{with} statement is used before the C{with} statement. """ @@ -975,6 +1633,40 @@ def test_augmentedAssignmentImportedFunctionCall(self): baz += bar() ''') + def test_assert_without_message(self): + """An assert without a message is not an error.""" + self.flakes(''' + a = 1 + assert a + ''') + + def test_assert_with_message(self): + """An assert with a message is not an error.""" + self.flakes(''' + a = 1 + assert a, 'x' + ''') + + def test_assert_tuple(self): + """An assert of a non-empty tuple is always True.""" + self.flakes(''' + assert (False, 'x') + assert (False, ) + ''', m.AssertTuple, m.AssertTuple) + + def test_assert_tuple_empty(self): + """An assert of an empty tuple is always False.""" + self.flakes(''' + assert () + ''') + + def test_assert_static(self): + """An assert of a static value is not an error.""" + self.flakes(''' + assert True + assert 1 + ''') + @skipIf(version_info < (3, 3), 'new in Python 3.3') def test_yieldFromUndefined(self): """ @@ -985,9 +1677,13 @@ def bar(): yield from foo() ''', m.UndefinedName) - def test_returnOnly(self): - """Do not crash on lone "return".""" - self.flakes('return 2') + @skipIf(version_info < (3, 6), 'new in Python 3.6') + def test_f_string(self): + """Test PEP 498 f-strings are treated as a usage.""" + self.flakes(''' + baz = 0 + print(f'\x7b4*baz\N{RIGHT CURLY BRACKET}') + ''') class TestAsyncStatements(TestCase): @@ -1023,6 +1719,63 @@ async def read_data(db): return output ''') + @skipIf(version_info < (3, 5), 'new in Python 3.5') + def test_loopControlInAsyncFor(self): + self.flakes(''' + async def read_data(db): + output = [] + async for row in db.cursor(): + if row[0] == 'skip': + continue + output.append(row) + return output + ''') + + self.flakes(''' + async def read_data(db): + output = [] + async for row in db.cursor(): + if row[0] == 'stop': + break + output.append(row) + return output + ''') + + @skipIf(version_info < (3, 5), 'new in Python 3.5') + def test_loopControlInAsyncForElse(self): + self.flakes(''' + async def read_data(db): + output = [] + async for row in db.cursor(): + output.append(row) + else: + continue + return output + ''', m.ContinueOutsideLoop) + + self.flakes(''' + async def read_data(db): + output = [] + async for row in db.cursor(): + output.append(row) + else: + break + return output + ''', m.BreakOutsideLoop) + + @skipIf(version_info < (3, 5), 'new in Python 3.5') + def test_continueInAsyncForFinally(self): + self.flakes(''' + async def read_data(db): + output = [] + async for row in db.cursor(): + try: + output.append(row) + finally: + continue + return output + ''', m.ContinueInFinally) + @skipIf(version_info < (3, 5), 'new in Python 3.5') def test_asyncWith(self): self.flakes(''' @@ -1040,3 +1793,84 @@ async def commit(session, data): ... await trans.end() ''') + + @skipIf(version_info < (3, 5), 'new in Python 3.5') + def test_matmul(self): + self.flakes(''' + def foo(a, b): + return a @ b + ''') + + @skipIf(version_info < (3, 6), 'new in Python 3.6') + def test_formatstring(self): + self.flakes(''' + hi = 'hi' + mom = 'mom' + f'{hi} {mom}' + ''') + + @skipIf(version_info < (3, 6), 'new in Python 3.6') + def test_variable_annotations(self): + self.flakes(''' + name: str + age: int + ''') + self.flakes(''' + name: str = 'Bob' + age: int = 18 + ''') + self.flakes(''' + class C: + name: str + age: int + ''') + self.flakes(''' + class C: + name: str = 'Bob' + age: int = 18 + ''') + self.flakes(''' + def f(): + name: str + age: int + ''') + self.flakes(''' + def f(): + name: str = 'Bob' + age: int = 18 + foo: not_a_real_type = None + ''', m.UnusedVariable, m.UnusedVariable, m.UnusedVariable, m.UndefinedName) + self.flakes(''' + def f(): + name: str + print(name) + ''', m.UndefinedName) + self.flakes(''' + from typing import Any + def f(): + a: Any + ''') + self.flakes(''' + foo: not_a_real_type + ''', m.UndefinedName) + self.flakes(''' + foo: not_a_real_type = None + ''', m.UndefinedName) + self.flakes(''' + class C: + foo: not_a_real_type + ''', m.UndefinedName) + self.flakes(''' + class C: + foo: not_a_real_type = None + ''', m.UndefinedName) + self.flakes(''' + def f(): + class C: + foo: not_a_real_type + ''', m.UndefinedName) + self.flakes(''' + def f(): + class C: + foo: not_a_real_type = None + ''', m.UndefinedName) diff --git a/contrib/pyflakes/test/test_undefined_names.py b/contrib/pyflakes/test/test_undefined_names.py index faaaf8c..1464d8e 100644 --- a/contrib/pyflakes/test/test_undefined_names.py +++ b/contrib/pyflakes/test/test_undefined_names.py @@ -3,7 +3,7 @@ from sys import version_info from pyflakes import messages as m, checker -from pyflakes.test.harness import TestCase, skipIf +from pyflakes.test.harness import TestCase, skipIf, skip class Test(TestCase): @@ -22,6 +22,184 @@ def test_undefinedInListComp(self): ''', m.UndefinedName) + @skipIf(version_info < (3,), + 'in Python 2 exception names stay bound after the except: block') + def test_undefinedExceptionName(self): + """Exception names can't be used after the except: block.""" + self.flakes(''' + try: + raise ValueError('ve') + except ValueError as exc: + pass + exc + ''', + m.UndefinedName) + + def test_namesDeclaredInExceptBlocks(self): + """Locals declared in except: blocks can be used after the block. + + This shows the example in test_undefinedExceptionName is + different.""" + self.flakes(''' + try: + raise ValueError('ve') + except ValueError as exc: + e = exc + e + ''') + + @skip('error reporting disabled due to false positives below') + def test_undefinedExceptionNameObscuringLocalVariable(self): + """Exception names obscure locals, can't be used after. + + Last line will raise UnboundLocalError on Python 3 after exiting + the except: block. Note next two examples for false positives to + watch out for.""" + self.flakes(''' + exc = 'Original value' + try: + raise ValueError('ve') + except ValueError as exc: + pass + exc + ''', + m.UndefinedName) + + @skipIf(version_info < (3,), + 'in Python 2 exception names stay bound after the except: block') + def test_undefinedExceptionNameObscuringLocalVariable2(self): + """Exception names are unbound after the `except:` block. + + Last line will raise UnboundLocalError on Python 3 but would print out + 've' on Python 2.""" + self.flakes(''' + try: + raise ValueError('ve') + except ValueError as exc: + pass + print(exc) + exc = 'Original value' + ''', + m.UndefinedName) + + def test_undefinedExceptionNameObscuringLocalVariableFalsePositive1(self): + """Exception names obscure locals, can't be used after. Unless. + + Last line will never raise UnboundLocalError because it's only + entered if no exception was raised.""" + self.flakes(''' + exc = 'Original value' + try: + raise ValueError('ve') + except ValueError as exc: + print('exception logged') + raise + exc + ''') + + def test_delExceptionInExcept(self): + """The exception name can be deleted in the except: block.""" + self.flakes(''' + try: + pass + except Exception as exc: + del exc + ''') + + def test_undefinedExceptionNameObscuringLocalVariableFalsePositive2(self): + """Exception names obscure locals, can't be used after. Unless. + + Last line will never raise UnboundLocalError because `error` is + only falsy if the `except:` block has not been entered.""" + self.flakes(''' + exc = 'Original value' + error = None + try: + raise ValueError('ve') + except ValueError as exc: + error = 'exception logged' + if error: + print(error) + else: + exc + ''') + + @skip('error reporting disabled due to false positives below') + def test_undefinedExceptionNameObscuringGlobalVariable(self): + """Exception names obscure globals, can't be used after. + + Last line will raise UnboundLocalError on both Python 2 and + Python 3 because the existence of that exception name creates + a local scope placeholder for it, obscuring any globals, etc.""" + self.flakes(''' + exc = 'Original value' + def func(): + try: + pass # nothing is raised + except ValueError as exc: + pass # block never entered, exc stays unbound + exc + ''', + m.UndefinedLocal) + + @skip('error reporting disabled due to false positives below') + def test_undefinedExceptionNameObscuringGlobalVariable2(self): + """Exception names obscure globals, can't be used after. + + Last line will raise NameError on Python 3 because the name is + locally unbound after the `except:` block, even if it's + nonlocal. We should issue an error in this case because code + only working correctly if an exception isn't raised, is invalid. + Unless it's explicitly silenced, see false positives below.""" + self.flakes(''' + exc = 'Original value' + def func(): + global exc + try: + raise ValueError('ve') + except ValueError as exc: + pass # block never entered, exc stays unbound + exc + ''', + m.UndefinedLocal) + + def test_undefinedExceptionNameObscuringGlobalVariableFalsePositive1(self): + """Exception names obscure globals, can't be used after. Unless. + + Last line will never raise NameError because it's only entered + if no exception was raised.""" + self.flakes(''' + exc = 'Original value' + def func(): + global exc + try: + raise ValueError('ve') + except ValueError as exc: + print('exception logged') + raise + exc + ''') + + def test_undefinedExceptionNameObscuringGlobalVariableFalsePositive2(self): + """Exception names obscure globals, can't be used after. Unless. + + Last line will never raise NameError because `error` is only + falsy if the `except:` block has not been entered.""" + self.flakes(''' + exc = 'Original value' + def func(): + global exc + error = None + try: + raise ValueError('ve') + except ValueError as exc: + error = 'exception logged' + if error: + print(error) + else: + exc + ''') + def test_functionsNeedGlobalScope(self): self.flakes(''' class a: @@ -71,8 +249,10 @@ def test_magicGlobalsPath(self): def test_globalImportStar(self): """Can't find undefined names with import *.""" - self.flakes('from fu import *; bar', m.ImportStarUsed) + self.flakes('from fu import *; bar', + m.ImportStarUsed, m.ImportStarUsage) + @skipIf(version_info >= (3,), 'obsolete syntax') def test_localImportStar(self): """ A local import * still allows undefined names to be found @@ -82,7 +262,7 @@ def test_localImportStar(self): def a(): from fu import * bar - ''', m.ImportStarUsed, m.UndefinedName) + ''', m.ImportStarUsed, m.UndefinedName, m.UnusedImport) @skipIf(version_info >= (3,), 'obsolete syntax') def test_unpackedParameter(self): @@ -125,6 +305,29 @@ def foo(): print(x) ''', m.UndefinedName) + def test_global_reset_name_only(self): + """A global statement does not prevent other names being undefined.""" + # Only different undefined names are reported. + # See following test that fails where the same name is used. + self.flakes(''' + def f1(): + s + + def f2(): + global m + ''', m.UndefinedName) + + @skip("todo") + def test_unused_global(self): + """An unused global statement does not define the name.""" + self.flakes(''' + def f1(): + m + + def f2(): + global m + ''', m.UndefinedName) + def test_del(self): """Del deletes bindings.""" self.flakes('a = 1; del a; a', m.UndefinedName) @@ -286,7 +489,11 @@ def c(): return x return x ''', m.UndefinedLocal).messages[0] - self.assertEqual(exc.message_args, ('x', 5)) + + # _DoctestMixin.flakes adds two lines preceding the code above. + expected_line_num = 7 if self.withDoctest else 5 + + self.assertEqual(exc.message_args, ('x', expected_line_num)) def test_laterRedefinedGlobalFromNestedScope3(self): """ @@ -453,8 +660,20 @@ def test_definedInGenExp(self): Using the loop variable of a generator expression results in no warnings. """ - self.flakes('(a for a in %srange(10) if a)' % - ('x' if version_info < (3,) else '')) + self.flakes('(a for a in [1, 2, 3] if a)') + + self.flakes('(b for b in (a for a in [1, 2, 3] if a) if b)') + + def test_undefinedInGenExpNested(self): + """ + The loop variables of generator expressions nested together are + not defined in the other generator. + """ + self.flakes('(b for b in (a for a in [1, 2, 3] if b) if b)', + m.UndefinedName) + + self.flakes('(b for b in (a for a in [1, 2, 3] if a) if a)', + m.UndefinedName) def test_undefinedWithErrorHandler(self): """ @@ -509,6 +728,15 @@ class A: Y = {x:x for x in T} ''') + def test_definedInClassNested(self): + """Defined name for nested generator expressions in a class.""" + self.flakes(''' + class A: + T = range(10) + + Z = (x for x in (a for a in T)) + ''') + def test_undefinedInLoop(self): """ The loop variable is defined after the expression is computed.