Skip to content

Commit

Permalink
Don't suggest incompatible stub packages (#10610)
Browse files Browse the repository at this point in the history
Keep track of supported Python versions of legacy bundled packages, and
only suggest a stub package if the major Python version is compatible.

Fixes #10602.
  • Loading branch information
JukkaL committed Jun 10, 2021
1 parent 8f51fbf commit 12cf0bf
Show file tree
Hide file tree
Showing 7 changed files with 166 additions and 88 deletions.
10 changes: 6 additions & 4 deletions mypy/build.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@
from mypy.renaming import VariableRenameVisitor
from mypy.config_parser import parse_mypy_comments
from mypy.freetree import free_tree
from mypy.stubinfo import legacy_bundled_packages
from mypy.stubinfo import legacy_bundled_packages, is_legacy_bundled_package
from mypy import errorcodes as codes


Expand Down Expand Up @@ -2449,7 +2449,9 @@ def find_module_and_diagnose(manager: BuildManager,
# otherwise updating mypy can silently result in new false
# negatives.
global_ignore_missing_imports = manager.options.ignore_missing_imports
if ((top_level in legacy_bundled_packages or second_level in legacy_bundled_packages)
py_ver = options.python_version[0]
if ((is_legacy_bundled_package(top_level, py_ver)
or is_legacy_bundled_package(second_level, py_ver))
and global_ignore_missing_imports
and not options.ignore_missing_imports_per_module):
ignore_missing_imports = False
Expand Down Expand Up @@ -2558,10 +2560,10 @@ def module_not_found(manager: BuildManager, line: int, caller_state: State,
top_level = second_level
for note in notes:
if '{stub_dist}' in note:
note = note.format(stub_dist=legacy_bundled_packages[top_level])
note = note.format(stub_dist=legacy_bundled_packages[top_level].name)
errors.report(line, 0, note, severity='note', only_once=True, code=codes.IMPORT)
if reason is ModuleNotFoundReason.APPROVED_STUBS_NOT_INSTALLED:
manager.missing_stub_packages.add(legacy_bundled_packages[top_level])
manager.missing_stub_packages.add(legacy_bundled_packages[top_level].name)
errors.set_import_context(save_import_context)


Expand Down
16 changes: 8 additions & 8 deletions mypy/modulefinder.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@

from mypy.fscache import FileSystemCache
from mypy.options import Options
from mypy.stubinfo import legacy_bundled_packages
from mypy.stubinfo import is_legacy_bundled_package
from mypy import sitepkgs

# Paths to be searched in find_module().
Expand Down Expand Up @@ -136,7 +136,7 @@ def __init__(self,
if options:
custom_typeshed_dir = options.custom_typeshed_dir
self.stdlib_py_versions = load_stdlib_py_versions(custom_typeshed_dir)
self.python2 = options and options.python_version[0] == 2
self.python_major_ver = 3 if options is None else options.python_version[0]

def clear(self) -> None:
self.results.clear()
Expand Down Expand Up @@ -187,7 +187,7 @@ def get_toplevel_possibilities(self, lib_path: Tuple[str, ...], id: str) -> List
name = os.path.splitext(name)[0]
components.setdefault(name, []).append(dir)

if self.python2:
if self.python_major_ver == 2:
components = {id: filter_redundant_py2_dirs(dirs)
for id, dirs in components.items()}

Expand Down Expand Up @@ -230,8 +230,8 @@ def _find_module_non_stub_helper(self, components: List[str],
elif not plausible_match and (self.fscache.isdir(dir_path)
or self.fscache.isfile(dir_path + ".py")):
plausible_match = True
if (components[0] in legacy_bundled_packages
or '.'.join(components[:2]) in legacy_bundled_packages):
if (is_legacy_bundled_package(components[0], self.python_major_ver)
or is_legacy_bundled_package('.'.join(components[:2]), self.python_major_ver)):
return ModuleNotFoundReason.APPROVED_STUBS_NOT_INSTALLED
elif plausible_match:
return ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS
Expand Down Expand Up @@ -280,7 +280,7 @@ def _find_module(self, id: str, use_typeshed: bool) -> ModuleSearchResult:
for pkg_dir in self.search_paths.package_path:
stub_name = components[0] + '-stubs'
stub_dir = os.path.join(pkg_dir, stub_name)
if self.python2:
if self.python_major_ver == 2:
alt_stub_name = components[0] + '-python2-stubs'
alt_stub_dir = os.path.join(pkg_dir, alt_stub_name)
if fscache.isdir(alt_stub_dir):
Expand Down Expand Up @@ -348,7 +348,7 @@ def _find_module(self, id: str, use_typeshed: bool) -> ModuleSearchResult:
for extension in PYTHON_EXTENSIONS:
path = base_path + sepinit + extension
suffix = '-stubs'
if self.python2:
if self.python_major_ver == 2:
if os.path.isdir(base_path + '-python2-stubs'):
suffix = '-python2-stubs'
path_stubs = base_path + suffix + sepinit + extension
Expand Down Expand Up @@ -432,7 +432,7 @@ def _is_compatible_stub_package(self, stub_dir: str) -> bool:
import toml
with open(metadata_fnam, 'r') as f:
metadata = toml.load(f)
if self.python2:
if self.python_major_ver == 2:
return bool(metadata.get('python2', False))
else:
return bool(metadata.get('python3', True))
Expand Down
169 changes: 93 additions & 76 deletions mypy/stubinfo.py
Original file line number Diff line number Diff line change
@@ -1,83 +1,100 @@
from typing import Optional


class StubInfo:
def __init__(self, name: str, py_version: Optional[int] = None) -> None:
self.name = name
# If None, compatible with py2+py3, if 2/3, only compatible with py2/py3
self.py_version = py_version


def is_legacy_bundled_package(prefix: str, py_version: int) -> bool:
if prefix not in legacy_bundled_packages:
return False
package_ver = legacy_bundled_packages[prefix].py_version
return package_ver is None or package_ver == py_version


# Stubs for these third-party packages used to be shipped with mypy.
#
# Map package name to PyPI stub distribution name.
#
# Package name can have one or two components ('a' or 'a.b').
legacy_bundled_packages = {
'aiofiles': 'types-aiofiles',
'atomicwrites': 'types-atomicwrites',
'attr': 'types-attrs',
'backports': 'types-backports',
'backports_abc': 'types-backports_abc',
'bleach': 'types-bleach',
'boto': 'types-boto',
'cachetools': 'types-cachetools',
'certifi': 'types-certifi',
'characteristic': 'types-characteristic',
'chardet': 'types-chardet',
'click': 'types-click',
'click_spinner': 'types-click-spinner',
'concurrent': 'types-futures',
'contextvars': 'types-contextvars',
'croniter': 'types-croniter',
'cryptography': 'types-cryptography',
'dataclasses': 'types-dataclasses',
'dateparser': 'types-dateparser',
'datetimerange': 'types-DateTimeRange',
'dateutil': 'types-python-dateutil',
'decorator': 'types-decorator',
'deprecated': 'types-Deprecated',
'docutils': 'types-docutils',
'emoji': 'types-emoji',
'enum': 'types-enum34',
'fb303': 'types-fb303',
'filelock': 'types-filelock',
'first': 'types-first',
'flask': 'types-Flask',
'freezegun': 'types-freezegun',
'frozendict': 'types-frozendict',
'geoip2': 'types-geoip2',
'gflags': 'types-python-gflags',
'google.protobuf': 'types-protobuf',
'ipaddress': 'types-ipaddress',
'itsdangerous': 'types-itsdangerous',
'jinja2': 'types-Jinja2',
'jwt': 'types-jwt',
'kazoo': 'types-kazoo',
'markdown': 'types-Markdown',
'markupsafe': 'types-MarkupSafe',
'maxminddb': 'types-maxminddb',
'mock': 'types-mock',
'OpenSSL': 'types-openssl-python',
'orjson': 'types-orjson',
'paramiko': 'types-paramiko',
'pathlib2': 'types-pathlib2',
'pkg_resources': 'types-pkg_resources',
'polib': 'types-polib',
'pycurl': 'types-pycurl',
'pymssql': 'types-pymssql',
'pymysql': 'types-PyMySQL',
'pyrfc3339': 'types-pyRFC3339',
'python2': 'types-six',
'pytz': 'types-pytz',
'pyVmomi': 'types-pyvmomi',
'redis': 'types-redis',
'requests': 'types-requests',
'retry': 'types-retry',
'routes': 'types-Routes',
'scribe': 'types-scribe',
'simplejson': 'types-simplejson',
'singledispatch': 'types-singledispatch',
'six': 'types-six',
'slugify': 'types-python-slugify',
'tabulate': 'types-tabulate',
'termcolor': 'types-termcolor',
'toml': 'types-toml',
'tornado': 'types-tornado',
'typed_ast': 'types-typed-ast',
'tzlocal': 'types-tzlocal',
'ujson': 'types-ujson',
'waitress': 'types-waitress',
'werkzeug': 'types-Werkzeug',
'yaml': 'types-PyYAML',
'aiofiles': StubInfo('types-aiofiles', py_version=3),
'atomicwrites': StubInfo('types-atomicwrites'),
'attr': StubInfo('types-attrs'),
'backports': StubInfo('types-backports'),
'backports_abc': StubInfo('types-backports_abc'),
'bleach': StubInfo('types-bleach'),
'boto': StubInfo('types-boto'),
'cachetools': StubInfo('types-cachetools'),
'certifi': StubInfo('types-certifi'),
'characteristic': StubInfo('types-characteristic'),
'chardet': StubInfo('types-chardet'),
'click': StubInfo('types-click'),
'click_spinner': StubInfo('types-click-spinner'),
'concurrent': StubInfo('types-futures', py_version=2),
'contextvars': StubInfo('types-contextvars', py_version=3),
'croniter': StubInfo('types-croniter'),
'cryptography': StubInfo('types-cryptography'),
'dataclasses': StubInfo('types-dataclasses', py_version=3),
'dateparser': StubInfo('types-dateparser'),
'datetimerange': StubInfo('types-DateTimeRange'),
'dateutil': StubInfo('types-python-dateutil'),
'decorator': StubInfo('types-decorator'),
'deprecated': StubInfo('types-Deprecated'),
'docutils': StubInfo('types-docutils', py_version=3),
'emoji': StubInfo('types-emoji'),
'enum': StubInfo('types-enum34', py_version=2),
'fb303': StubInfo('types-fb303', py_version=2),
'filelock': StubInfo('types-filelock', py_version=3),
'first': StubInfo('types-first'),
'flask': StubInfo('types-Flask'),
'freezegun': StubInfo('types-freezegun', py_version=3),
'frozendict': StubInfo('types-frozendict', py_version=3),
'geoip2': StubInfo('types-geoip2'),
'gflags': StubInfo('types-python-gflags'),
'google.protobuf': StubInfo('types-protobuf'),
'ipaddress': StubInfo('types-ipaddress', py_version=2),
'itsdangerous': StubInfo('types-itsdangerous'),
'jinja2': StubInfo('types-Jinja2'),
'jwt': StubInfo('types-jwt'),
'kazoo': StubInfo('types-kazoo', py_version=2),
'markdown': StubInfo('types-Markdown'),
'markupsafe': StubInfo('types-MarkupSafe'),
'maxminddb': StubInfo('types-maxminddb'),
'mock': StubInfo('types-mock'),
'OpenSSL': StubInfo('types-openssl-python', py_version=2),
'orjson': StubInfo('types-orjson', py_version=3),
'paramiko': StubInfo('types-paramiko'),
'pathlib2': StubInfo('types-pathlib2', py_version=2),
'pkg_resources': StubInfo('types-pkg_resources', py_version=3),
'polib': StubInfo('types-polib'),
'pycurl': StubInfo('types-pycurl'),
'pymssql': StubInfo('types-pymssql', py_version=2),
'pymysql': StubInfo('types-PyMySQL'),
'pyrfc3339': StubInfo('types-pyRFC3339', py_version=3),
'python2': StubInfo('types-six'),
'pytz': StubInfo('types-pytz'),
'pyVmomi': StubInfo('types-pyvmomi'),
'redis': StubInfo('types-redis'),
'requests': StubInfo('types-requests'),
'retry': StubInfo('types-retry'),
'routes': StubInfo('types-Routes', py_version=2),
'scribe': StubInfo('types-scribe', py_version=2),
'simplejson': StubInfo('types-simplejson'),
'singledispatch': StubInfo('types-singledispatch'),
'six': StubInfo('types-six'),
'slugify': StubInfo('types-python-slugify'),
'tabulate': StubInfo('types-tabulate'),
'termcolor': StubInfo('types-termcolor'),
'toml': StubInfo('types-toml'),
'tornado': StubInfo('types-tornado', py_version=2),
'typed_ast': StubInfo('types-typed-ast', py_version=3),
'tzlocal': StubInfo('types-tzlocal'),
'ujson': StubInfo('types-ujson'),
'waitress': StubInfo('types-waitress', py_version=3),
'werkzeug': StubInfo('types-Werkzeug'),
'yaml': StubInfo('types-PyYAML'),
}
4 changes: 4 additions & 0 deletions mypy/test/testpythoneval.py
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,10 @@ def test_python_evaluation(testcase: DataDrivenTestCase, cache_dir: str) -> None
interpreter = python3_path
mypy_cmdline.append('--python-version={}'.format('.'.join(map(str, PYTHON3_VERSION))))

m = re.search('# flags: (.*)$', '\n'.join(testcase.input), re.MULTILINE)
if m:
mypy_cmdline.extend(m.group(1).split())

# Write the program to a file.
program = '_' + testcase.name + '.py'
program_path = os.path.join(test_temp_dir, program)
Expand Down
18 changes: 18 additions & 0 deletions mypy/test/teststubinfo.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
import unittest

from mypy.stubinfo import is_legacy_bundled_package


class TestStubInfo(unittest.TestCase):
def test_is_legacy_bundled_packages(self) -> None:
assert not is_legacy_bundled_package('foobar_asdf', 2)
assert not is_legacy_bundled_package('foobar_asdf', 3)

assert is_legacy_bundled_package('click', 2)
assert is_legacy_bundled_package('click', 3)

assert is_legacy_bundled_package('scribe', 2)
assert not is_legacy_bundled_package('scribe', 3)

assert not is_legacy_bundled_package('dataclasses', 2)
assert is_legacy_bundled_package('dataclasses', 3)
14 changes: 14 additions & 0 deletions test-data/unit/python2eval.test
Original file line number Diff line number Diff line change
Expand Up @@ -433,3 +433,17 @@ _testDefaultDictInference.py:5: note: Revealed type is "collections.defaultdict[
from collections import abc
[out]
_testIgnorePython3StdlibStubs_python2.py:1: error: Module "collections" has no attribute "abc"

[case testNoApprovedPython2StubInstalled_python2]
# flags: --ignore-missing-imports
import scribe
from scribe import x
import maxminddb
import foobar_asdf
[out]
_testNoApprovedPython2StubInstalled_python2.py:2: error: Library stubs not installed for "scribe" (or incompatible with Python 2.7)
_testNoApprovedPython2StubInstalled_python2.py:2: note: Hint: "python3 -m pip install types-scribe"
_testNoApprovedPython2StubInstalled_python2.py:2: note: (or run "mypy --install-types" to install all missing stub packages)
_testNoApprovedPython2StubInstalled_python2.py:2: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports
_testNoApprovedPython2StubInstalled_python2.py:4: error: Library stubs not installed for "maxminddb" (or incompatible with Python 2.7)
_testNoApprovedPython2StubInstalled_python2.py:4: note: Hint: "python3 -m pip install types-maxminddb"
23 changes: 23 additions & 0 deletions test-data/unit/pythoneval.test
Original file line number Diff line number Diff line change
Expand Up @@ -1508,3 +1508,26 @@ x = 0
[out]
mypy: "tmp/typing.py" shadows library module "typing"
note: A user-defined top-level module with name "typing" is not supported

[case testIgnoreImportIfNoPython3StubAvailable]
# flags: --ignore-missing-imports
import scribe # No Python 3 stubs available for scribe
from scribe import x
import maxminddb # Python 3 stubs available for maxminddb
import foobar_asdf
[out]
_testIgnoreImportIfNoPython3StubAvailable.py:4: error: Library stubs not installed for "maxminddb" (or incompatible with Python 3.6)
_testIgnoreImportIfNoPython3StubAvailable.py:4: note: Hint: "python3 -m pip install types-maxminddb"
_testIgnoreImportIfNoPython3StubAvailable.py:4: note: (or run "mypy --install-types" to install all missing stub packages)
_testIgnoreImportIfNoPython3StubAvailable.py:4: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports

[case testNoPython3StubAvailable]
import scribe
from scribe import x
import maxminddb
[out]
_testNoPython3StubAvailable.py:1: error: Cannot find implementation or library stub for module named "scribe"
_testNoPython3StubAvailable.py:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports
_testNoPython3StubAvailable.py:3: error: Library stubs not installed for "maxminddb" (or incompatible with Python 3.6)
_testNoPython3StubAvailable.py:3: note: Hint: "python3 -m pip install types-maxminddb"
_testNoPython3StubAvailable.py:3: note: (or run "mypy --install-types" to install all missing stub packages)

0 comments on commit 12cf0bf

Please sign in to comment.