Browse Source
bpo-44771: Apply changes from importlib_resources 5.2.1 (GH-27436)
bpo-44771: Apply changes from importlib_resources 5.2.1 (GH-27436)
* bpo-44771: Apply changes from importlib_resources@3b24bd6307 * Add blurb * Exclude namespacedata01 from eol conversion.pull/27476/head
Jason R. Coombs
4 years ago
committed by
GitHub
GitHub
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 714 additions and 373 deletions
-
1.gitattributes
-
108Lib/importlib/_adapters.py
-
3Lib/importlib/_common.py
-
19Lib/importlib/_itertools.py
-
84Lib/importlib/_legacy.py
-
15Lib/importlib/readers.py
-
186Lib/importlib/resources.py
-
116Lib/importlib/simple.py
-
0Lib/test/test_importlib/resources/__init__.py
-
190Lib/test/test_importlib/resources/util.py
-
102Lib/test/test_importlib/test_compatibilty_files.py
-
42Lib/test/test_importlib/test_contents.py
-
9Lib/test/test_importlib/test_files.py
-
6Lib/test/test_importlib/test_open.py
-
4Lib/test/test_importlib/test_path.py
-
13Lib/test/test_importlib/test_read.py
-
12Lib/test/test_importlib/test_resource.py
-
172Lib/test/test_importlib/util.py
-
5Misc/NEWS.d/next/Library/2021-07-28-22-53-18.bpo-44771.BvLdnU.rst
@ -0,0 +1,19 @@ |
|||
from itertools import filterfalse |
|||
|
|||
|
|||
def unique_everseen(iterable, key=None): |
|||
"List unique elements, preserving order. Remember all elements ever seen." |
|||
# unique_everseen('AAAABBBCCDAABBB') --> A B C D |
|||
# unique_everseen('ABBCcAD', str.lower) --> A B C D |
|||
seen = set() |
|||
seen_add = seen.add |
|||
if key is None: |
|||
for element in filterfalse(seen.__contains__, iterable): |
|||
seen_add(element) |
|||
yield element |
|||
else: |
|||
for element in iterable: |
|||
k = key(element) |
|||
if k not in seen: |
|||
seen_add(k) |
|||
yield element |
|||
@ -0,0 +1,84 @@ |
|||
import os |
|||
import pathlib |
|||
import types |
|||
|
|||
from typing import Union, Iterable, ContextManager, BinaryIO, TextIO |
|||
|
|||
from . import _common |
|||
|
|||
Package = Union[types.ModuleType, str] |
|||
Resource = Union[str, os.PathLike] |
|||
|
|||
|
|||
def open_binary(package: Package, resource: Resource) -> BinaryIO: |
|||
"""Return a file-like object opened for binary reading of the resource.""" |
|||
return (_common.files(package) / _common.normalize_path(resource)).open('rb') |
|||
|
|||
|
|||
def read_binary(package: Package, resource: Resource) -> bytes: |
|||
"""Return the binary contents of the resource.""" |
|||
return (_common.files(package) / _common.normalize_path(resource)).read_bytes() |
|||
|
|||
|
|||
def open_text( |
|||
package: Package, |
|||
resource: Resource, |
|||
encoding: str = 'utf-8', |
|||
errors: str = 'strict', |
|||
) -> TextIO: |
|||
"""Return a file-like object opened for text reading of the resource.""" |
|||
return (_common.files(package) / _common.normalize_path(resource)).open( |
|||
'r', encoding=encoding, errors=errors |
|||
) |
|||
|
|||
|
|||
def read_text( |
|||
package: Package, |
|||
resource: Resource, |
|||
encoding: str = 'utf-8', |
|||
errors: str = 'strict', |
|||
) -> str: |
|||
"""Return the decoded string of the resource. |
|||
|
|||
The decoding-related arguments have the same semantics as those of |
|||
bytes.decode(). |
|||
""" |
|||
with open_text(package, resource, encoding, errors) as fp: |
|||
return fp.read() |
|||
|
|||
|
|||
def contents(package: Package) -> Iterable[str]: |
|||
"""Return an iterable of entries in `package`. |
|||
|
|||
Note that not all entries are resources. Specifically, directories are |
|||
not considered resources. Use `is_resource()` on each entry returned here |
|||
to check if it is a resource or not. |
|||
""" |
|||
return [path.name for path in _common.files(package).iterdir()] |
|||
|
|||
|
|||
def is_resource(package: Package, name: str) -> bool: |
|||
"""True if `name` is a resource inside `package`. |
|||
|
|||
Directories are *not* resources. |
|||
""" |
|||
resource = _common.normalize_path(name) |
|||
return any( |
|||
traversable.name == resource and traversable.is_file() |
|||
for traversable in _common.files(package).iterdir() |
|||
) |
|||
|
|||
|
|||
def path( |
|||
package: Package, |
|||
resource: Resource, |
|||
) -> ContextManager[pathlib.Path]: |
|||
"""A context manager providing a file path object to the resource. |
|||
|
|||
If the resource does not already exist on its own on the file system, |
|||
a temporary file will be created. If the file was created, the file |
|||
will be deleted upon exiting the context manager (no exception is |
|||
raised if the file was deleted prior to the context manager |
|||
exiting). |
|||
""" |
|||
return _common.as_file(_common.files(package) / _common.normalize_path(resource)) |
|||
@ -0,0 +1,116 @@ |
|||
""" |
|||
Interface adapters for low-level readers. |
|||
""" |
|||
|
|||
import abc |
|||
import io |
|||
import itertools |
|||
from typing import BinaryIO, List |
|||
|
|||
from .abc import Traversable, TraversableResources |
|||
|
|||
|
|||
class SimpleReader(abc.ABC): |
|||
""" |
|||
The minimum, low-level interface required from a resource |
|||
provider. |
|||
""" |
|||
|
|||
@abc.abstractproperty |
|||
def package(self): |
|||
# type: () -> str |
|||
""" |
|||
The name of the package for which this reader loads resources. |
|||
""" |
|||
|
|||
@abc.abstractmethod |
|||
def children(self): |
|||
# type: () -> List['SimpleReader'] |
|||
""" |
|||
Obtain an iterable of SimpleReader for available |
|||
child containers (e.g. directories). |
|||
""" |
|||
|
|||
@abc.abstractmethod |
|||
def resources(self): |
|||
# type: () -> List[str] |
|||
""" |
|||
Obtain available named resources for this virtual package. |
|||
""" |
|||
|
|||
@abc.abstractmethod |
|||
def open_binary(self, resource): |
|||
# type: (str) -> BinaryIO |
|||
""" |
|||
Obtain a File-like for a named resource. |
|||
""" |
|||
|
|||
@property |
|||
def name(self): |
|||
return self.package.split('.')[-1] |
|||
|
|||
|
|||
class ResourceHandle(Traversable): |
|||
""" |
|||
Handle to a named resource in a ResourceReader. |
|||
""" |
|||
|
|||
def __init__(self, parent, name): |
|||
# type: (ResourceContainer, str) -> None |
|||
self.parent = parent |
|||
self.name = name # type: ignore |
|||
|
|||
def is_file(self): |
|||
return True |
|||
|
|||
def is_dir(self): |
|||
return False |
|||
|
|||
def open(self, mode='r', *args, **kwargs): |
|||
stream = self.parent.reader.open_binary(self.name) |
|||
if 'b' not in mode: |
|||
stream = io.TextIOWrapper(*args, **kwargs) |
|||
return stream |
|||
|
|||
def joinpath(self, name): |
|||
raise RuntimeError("Cannot traverse into a resource") |
|||
|
|||
|
|||
class ResourceContainer(Traversable): |
|||
""" |
|||
Traversable container for a package's resources via its reader. |
|||
""" |
|||
|
|||
def __init__(self, reader): |
|||
# type: (SimpleReader) -> None |
|||
self.reader = reader |
|||
|
|||
def is_dir(self): |
|||
return True |
|||
|
|||
def is_file(self): |
|||
return False |
|||
|
|||
def iterdir(self): |
|||
files = (ResourceHandle(self, name) for name in self.reader.resources) |
|||
dirs = map(ResourceContainer, self.reader.children()) |
|||
return itertools.chain(files, dirs) |
|||
|
|||
def open(self, *args, **kwargs): |
|||
raise IsADirectoryError() |
|||
|
|||
def joinpath(self, name): |
|||
return next( |
|||
traversable for traversable in self.iterdir() if traversable.name == name |
|||
) |
|||
|
|||
|
|||
class TraversableReader(TraversableResources, SimpleReader): |
|||
""" |
|||
A TraversableResources based on SimpleReader. Resource providers |
|||
may derive from this class to provide the TraversableResources |
|||
interface by supplying the SimpleReader interface. |
|||
""" |
|||
|
|||
def files(self): |
|||
return ResourceContainer(self) |
|||
@ -0,0 +1,190 @@ |
|||
import abc |
|||
import importlib |
|||
import io |
|||
import sys |
|||
import types |
|||
from pathlib import Path, PurePath |
|||
|
|||
from .. import data01 |
|||
from .. import zipdata01 |
|||
from importlib.abc import ResourceReader |
|||
from test.support import import_helper |
|||
|
|||
|
|||
from importlib.machinery import ModuleSpec |
|||
|
|||
|
|||
class Reader(ResourceReader): |
|||
def __init__(self, **kwargs): |
|||
vars(self).update(kwargs) |
|||
|
|||
def get_resource_reader(self, package): |
|||
return self |
|||
|
|||
def open_resource(self, path): |
|||
self._path = path |
|||
if isinstance(self.file, Exception): |
|||
raise self.file |
|||
return self.file |
|||
|
|||
def resource_path(self, path_): |
|||
self._path = path_ |
|||
if isinstance(self.path, Exception): |
|||
raise self.path |
|||
return self.path |
|||
|
|||
def is_resource(self, path_): |
|||
self._path = path_ |
|||
if isinstance(self.path, Exception): |
|||
raise self.path |
|||
|
|||
def part(entry): |
|||
return entry.split('/') |
|||
|
|||
return any( |
|||
len(parts) == 1 and parts[0] == path_ for parts in map(part, self._contents) |
|||
) |
|||
|
|||
def contents(self): |
|||
if isinstance(self.path, Exception): |
|||
raise self.path |
|||
yield from self._contents |
|||
|
|||
|
|||
def create_package_from_loader(loader, is_package=True): |
|||
name = 'testingpackage' |
|||
module = types.ModuleType(name) |
|||
spec = ModuleSpec(name, loader, origin='does-not-exist', is_package=is_package) |
|||
module.__spec__ = spec |
|||
module.__loader__ = loader |
|||
return module |
|||
|
|||
|
|||
def create_package(file=None, path=None, is_package=True, contents=()): |
|||
return create_package_from_loader( |
|||
Reader(file=file, path=path, _contents=contents), |
|||
is_package, |
|||
) |
|||
|
|||
|
|||
class CommonTests(metaclass=abc.ABCMeta): |
|||
""" |
|||
Tests shared by test_open, test_path, and test_read. |
|||
""" |
|||
|
|||
@abc.abstractmethod |
|||
def execute(self, package, path): |
|||
""" |
|||
Call the pertinent legacy API function (e.g. open_text, path) |
|||
on package and path. |
|||
""" |
|||
|
|||
def test_package_name(self): |
|||
# Passing in the package name should succeed. |
|||
self.execute(data01.__name__, 'utf-8.file') |
|||
|
|||
def test_package_object(self): |
|||
# Passing in the package itself should succeed. |
|||
self.execute(data01, 'utf-8.file') |
|||
|
|||
def test_string_path(self): |
|||
# Passing in a string for the path should succeed. |
|||
path = 'utf-8.file' |
|||
self.execute(data01, path) |
|||
|
|||
def test_pathlib_path(self): |
|||
# Passing in a pathlib.PurePath object for the path should succeed. |
|||
path = PurePath('utf-8.file') |
|||
self.execute(data01, path) |
|||
|
|||
def test_absolute_path(self): |
|||
# An absolute path is a ValueError. |
|||
path = Path(__file__) |
|||
full_path = path.parent / 'utf-8.file' |
|||
with self.assertRaises(ValueError): |
|||
self.execute(data01, full_path) |
|||
|
|||
def test_relative_path(self): |
|||
# A reative path is a ValueError. |
|||
with self.assertRaises(ValueError): |
|||
self.execute(data01, '../data01/utf-8.file') |
|||
|
|||
def test_importing_module_as_side_effect(self): |
|||
# The anchor package can already be imported. |
|||
del sys.modules[data01.__name__] |
|||
self.execute(data01.__name__, 'utf-8.file') |
|||
|
|||
def test_non_package_by_name(self): |
|||
# The anchor package cannot be a module. |
|||
with self.assertRaises(TypeError): |
|||
self.execute(__name__, 'utf-8.file') |
|||
|
|||
def test_non_package_by_package(self): |
|||
# The anchor package cannot be a module. |
|||
with self.assertRaises(TypeError): |
|||
module = sys.modules['test.test_importlib.resources.util'] |
|||
self.execute(module, 'utf-8.file') |
|||
|
|||
def test_missing_path(self): |
|||
# Attempting to open or read or request the path for a |
|||
# non-existent path should succeed if open_resource |
|||
# can return a viable data stream. |
|||
bytes_data = io.BytesIO(b'Hello, world!') |
|||
package = create_package(file=bytes_data, path=FileNotFoundError()) |
|||
self.execute(package, 'utf-8.file') |
|||
self.assertEqual(package.__loader__._path, 'utf-8.file') |
|||
|
|||
def test_extant_path(self): |
|||
# Attempting to open or read or request the path when the |
|||
# path does exist should still succeed. Does not assert |
|||
# anything about the result. |
|||
bytes_data = io.BytesIO(b'Hello, world!') |
|||
# any path that exists |
|||
path = __file__ |
|||
package = create_package(file=bytes_data, path=path) |
|||
self.execute(package, 'utf-8.file') |
|||
self.assertEqual(package.__loader__._path, 'utf-8.file') |
|||
|
|||
def test_useless_loader(self): |
|||
package = create_package(file=FileNotFoundError(), path=FileNotFoundError()) |
|||
with self.assertRaises(FileNotFoundError): |
|||
self.execute(package, 'utf-8.file') |
|||
|
|||
|
|||
class ZipSetupBase: |
|||
ZIP_MODULE = None |
|||
|
|||
@classmethod |
|||
def setUpClass(cls): |
|||
data_path = Path(cls.ZIP_MODULE.__file__) |
|||
data_dir = data_path.parent |
|||
cls._zip_path = str(data_dir / 'ziptestdata.zip') |
|||
sys.path.append(cls._zip_path) |
|||
cls.data = importlib.import_module('ziptestdata') |
|||
|
|||
@classmethod |
|||
def tearDownClass(cls): |
|||
try: |
|||
sys.path.remove(cls._zip_path) |
|||
except ValueError: |
|||
pass |
|||
|
|||
try: |
|||
del sys.path_importer_cache[cls._zip_path] |
|||
del sys.modules[cls.data.__name__] |
|||
except KeyError: |
|||
pass |
|||
|
|||
try: |
|||
del cls.data |
|||
del cls._zip_path |
|||
except AttributeError: |
|||
pass |
|||
|
|||
def setUp(self): |
|||
modules = import_helper.modules_setup() |
|||
self.addCleanup(import_helper.modules_cleanup, *modules) |
|||
|
|||
|
|||
class ZipSetup(ZipSetupBase): |
|||
ZIP_MODULE = zipdata01 # type: ignore |
|||
@ -0,0 +1,102 @@ |
|||
import io |
|||
import unittest |
|||
|
|||
from importlib import resources |
|||
|
|||
from importlib._adapters import ( |
|||
CompatibilityFiles, |
|||
wrap_spec, |
|||
) |
|||
|
|||
from .resources import util |
|||
|
|||
|
|||
class CompatibilityFilesTests(unittest.TestCase): |
|||
@property |
|||
def package(self): |
|||
bytes_data = io.BytesIO(b'Hello, world!') |
|||
return util.create_package( |
|||
file=bytes_data, |
|||
path='some_path', |
|||
contents=('a', 'b', 'c'), |
|||
) |
|||
|
|||
@property |
|||
def files(self): |
|||
return resources.files(self.package) |
|||
|
|||
def test_spec_path_iter(self): |
|||
self.assertEqual( |
|||
sorted(path.name for path in self.files.iterdir()), |
|||
['a', 'b', 'c'], |
|||
) |
|||
|
|||
def test_child_path_iter(self): |
|||
self.assertEqual(list((self.files / 'a').iterdir()), []) |
|||
|
|||
def test_orphan_path_iter(self): |
|||
self.assertEqual(list((self.files / 'a' / 'a').iterdir()), []) |
|||
self.assertEqual(list((self.files / 'a' / 'a' / 'a').iterdir()), []) |
|||
|
|||
def test_spec_path_is(self): |
|||
self.assertFalse(self.files.is_file()) |
|||
self.assertFalse(self.files.is_dir()) |
|||
|
|||
def test_child_path_is(self): |
|||
self.assertTrue((self.files / 'a').is_file()) |
|||
self.assertFalse((self.files / 'a').is_dir()) |
|||
|
|||
def test_orphan_path_is(self): |
|||
self.assertFalse((self.files / 'a' / 'a').is_file()) |
|||
self.assertFalse((self.files / 'a' / 'a').is_dir()) |
|||
self.assertFalse((self.files / 'a' / 'a' / 'a').is_file()) |
|||
self.assertFalse((self.files / 'a' / 'a' / 'a').is_dir()) |
|||
|
|||
def test_spec_path_name(self): |
|||
self.assertEqual(self.files.name, 'testingpackage') |
|||
|
|||
def test_child_path_name(self): |
|||
self.assertEqual((self.files / 'a').name, 'a') |
|||
|
|||
def test_orphan_path_name(self): |
|||
self.assertEqual((self.files / 'a' / 'b').name, 'b') |
|||
self.assertEqual((self.files / 'a' / 'b' / 'c').name, 'c') |
|||
|
|||
def test_spec_path_open(self): |
|||
self.assertEqual(self.files.read_bytes(), b'Hello, world!') |
|||
self.assertEqual(self.files.read_text(), 'Hello, world!') |
|||
|
|||
def test_child_path_open(self): |
|||
self.assertEqual((self.files / 'a').read_bytes(), b'Hello, world!') |
|||
self.assertEqual((self.files / 'a').read_text(), 'Hello, world!') |
|||
|
|||
def test_orphan_path_open(self): |
|||
with self.assertRaises(FileNotFoundError): |
|||
(self.files / 'a' / 'b').read_bytes() |
|||
with self.assertRaises(FileNotFoundError): |
|||
(self.files / 'a' / 'b' / 'c').read_bytes() |
|||
|
|||
def test_open_invalid_mode(self): |
|||
with self.assertRaises(ValueError): |
|||
self.files.open('0') |
|||
|
|||
def test_orphan_path_invalid(self): |
|||
with self.assertRaises(ValueError): |
|||
CompatibilityFiles.OrphanPath() |
|||
|
|||
def test_wrap_spec(self): |
|||
spec = wrap_spec(self.package) |
|||
self.assertIsInstance(spec.loader.get_resource_reader(None), CompatibilityFiles) |
|||
|
|||
|
|||
class CompatibilityFilesNoReaderTests(unittest.TestCase): |
|||
@property |
|||
def package(self): |
|||
return util.create_package_from_loader(None) |
|||
|
|||
@property |
|||
def files(self): |
|||
return resources.files(self.package) |
|||
|
|||
def test_spec_path_joinpath(self): |
|||
self.assertIsInstance(self.files / 'a', CompatibilityFiles.OrphanPath) |
|||
@ -0,0 +1,42 @@ |
|||
import unittest |
|||
from importlib import resources |
|||
|
|||
from . import data01 |
|||
from .resources import util |
|||
|
|||
|
|||
class ContentsTests: |
|||
expected = { |
|||
'__init__.py', |
|||
'binary.file', |
|||
'subdirectory', |
|||
'utf-16.file', |
|||
'utf-8.file', |
|||
} |
|||
|
|||
def test_contents(self): |
|||
assert self.expected <= set(resources.contents(self.data)) |
|||
|
|||
|
|||
class ContentsDiskTests(ContentsTests, unittest.TestCase): |
|||
def setUp(self): |
|||
self.data = data01 |
|||
|
|||
|
|||
class ContentsZipTests(ContentsTests, util.ZipSetup, unittest.TestCase): |
|||
pass |
|||
|
|||
|
|||
class ContentsNamespaceTests(ContentsTests, unittest.TestCase): |
|||
expected = { |
|||
# no __init__ because of namespace design |
|||
# no subdirectory as incidental difference in fixture |
|||
'binary.file', |
|||
'utf-16.file', |
|||
'utf-8.file', |
|||
} |
|||
|
|||
def setUp(self): |
|||
from . import namespacedata01 |
|||
|
|||
self.data = namespacedata01 |
|||
@ -0,0 +1,5 @@ |
|||
Added ``importlib.simple`` module implementing adapters from a low-level |
|||
resources reader interface to a ``TraversableResources`` interface. Legacy |
|||
API (``path``, ``contents``, ...) is now supported entirely by the |
|||
``.files()`` API with a compatibility shim supplied for resource loaders |
|||
without that functionality. Feature parity with ``importlib_resources`` 5.2. |
|||
Write
Preview
Loading…
Cancel
Save
Reference in new issue