diff --git a/CHANGES.rst b/CHANGES.rst index 33dba5cfe..415591298 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -27,6 +27,7 @@ The ASDF Standard is at v1.6.0 - Add warning to use of asdftool extract and remove-hdu about deprecation and impending removal [#1411] - Deprecate AsdfFile attributes that use the legacy extension api [#1417] +- Add AsdfDeprecationWarning to asdf.types [#1401] 2.14.3 (2022-12-15) ------------------- diff --git a/asdf/__init__.py b/asdf/__init__.py index 3e1103fae..c6a0064af 100644 --- a/asdf/__init__.py +++ b/asdf/__init__.py @@ -22,6 +22,7 @@ from jsonschema import ValidationError from ._convenience import info +from ._types import CustomType from ._version import version as __version__ from .asdf import AsdfFile from .asdf import open_asdf as open # noqa: A001 @@ -30,4 +31,3 @@ from .stream import Stream from .tags.core import IntegerType from .tags.core.external_reference import ExternalArrayReference -from .types import CustomType diff --git a/asdf/_types.py b/asdf/_types.py new file mode 100644 index 000000000..92dacc9b5 --- /dev/null +++ b/asdf/_types.py @@ -0,0 +1,496 @@ +import importlib +import re +import warnings +from copy import copy + +from . import tagged, util +from .exceptions import AsdfDeprecationWarning +from .versioning import AsdfSpec, AsdfVersion + +__all__ = ["format_tag", "CustomType", "AsdfType", "ExtensionType"] + + +# regex used to parse module name from optional version string +MODULE_RE = re.compile(r"([a-zA-Z]+)(-(\d+\.\d+\.\d+))?") + + +def format_tag(organization, standard, version, tag_name): + """ + Format a YAML tag. + """ + tag = f"tag:{organization}:{standard}/{tag_name}" + + if version is None: + return tag + + if isinstance(version, AsdfSpec): + version = str(version.spec) + + return f"{tag}-{version}" + + +_all_asdftypes = set() + + +def _from_tree_tagged_missing_requirements(cls, tree, ctx): + # A special version of AsdfType.from_tree_tagged for when the + # required dependencies for an AsdfType are missing. + plural, verb = ("s", "are") if len(cls.requires) else ("", "is") + + # This error will be handled by yamlutil.tagged_tree_to_custom_tree, which + # will cause a warning to be issued indicating that the tree failed to be + # converted. + msg = f"{util.human_list(cls.requires)} package{plural} {verb} required to instantiate '{tree._tag}'" + raise TypeError(msg) + + +class ExtensionTypeMeta(type): + """ + Custom class constructor for tag types. + """ + + _import_cache = {} + + @classmethod + def _has_required_modules(cls, requires): + for string in requires: + has_module = True + match = MODULE_RE.match(string) + modname, _, version = match.groups() + if modname in cls._import_cache and not cls._import_cache[modname]: + return False + + try: + module = importlib.import_module(modname) + if version and hasattr(module, "__version__") and module.__version__ < version: + has_module = False + + except ImportError: + has_module = False + + finally: + cls._import_cache[modname] = has_module + if not has_module: + return False # noqa: B012 + + return True + + @classmethod + def _find_in_bases(cls, attrs, bases, name, default=None): + if name in attrs: + return attrs[name] + for base in bases: + if hasattr(base, name): + return getattr(base, name) + return default + + @property + def versioned_siblings(cls): + return getattr(cls, "__versioned_siblings") or [] + + def __new__(cls, name, bases, attrs): + requires = cls._find_in_bases(attrs, bases, "requires", []) + if not cls._has_required_modules(requires): + attrs["from_tree_tagged"] = classmethod(_from_tree_tagged_missing_requirements) + attrs["types"] = [] + attrs["has_required_modules"] = False + else: + attrs["has_required_modules"] = True + types = cls._find_in_bases(attrs, bases, "types", []) + new_types = [] + for typ in types: + if isinstance(typ, str): + typ = util.resolve_name(typ) + new_types.append(typ) + attrs["types"] = new_types + + new_cls = super().__new__(cls, name, bases, attrs) + + if hasattr(new_cls, "version") and not isinstance(new_cls.version, (AsdfVersion, AsdfSpec)): + new_cls.version = AsdfVersion(new_cls.version) + + if hasattr(new_cls, "name"): + if isinstance(new_cls.name, str): + if "yaml_tag" not in attrs: + new_cls.yaml_tag = new_cls.make_yaml_tag(new_cls.name) + elif isinstance(new_cls.name, list): + pass + elif new_cls.name is not None: + msg = "name must be string or list" + raise TypeError(msg) + + if hasattr(new_cls, "supported_versions"): + if not isinstance(new_cls.supported_versions, (list, set)): + new_cls.supported_versions = [new_cls.supported_versions] + supported_versions = set() + for version in new_cls.supported_versions: + if not isinstance(version, (AsdfVersion, AsdfSpec)): + version = AsdfVersion(version) + # This should cause an exception for invalid input + supported_versions.add(version) + # We need to convert back to a list here so that the 'in' operator + # uses actual comparison instead of hash equality + new_cls.supported_versions = list(supported_versions) + siblings = [] + for version in new_cls.supported_versions: + if version != new_cls.version: + new_attrs = copy(attrs) + new_attrs["version"] = version + new_attrs["supported_versions"] = set() + new_attrs["_latest_version"] = new_cls.version + if "__classcell__" in new_attrs: + msg = ( + "Subclasses of ExtensionTypeMeta that define " + "supported_versions cannot used super() to call " + "parent class functions. super() creates a " + "__classcell__ closure that cannot be duplicated " + "during creation of versioned siblings. " + "See https://github.com/asdf-format/asdf/issues/1245" + ) + raise RuntimeError(msg) + siblings.append(ExtensionTypeMeta.__new__(cls, name, bases, new_attrs)) + setattr(new_cls, "__versioned_siblings", siblings) + + return new_cls + + +class AsdfTypeMeta(ExtensionTypeMeta): + """ + Keeps track of `AsdfType` subclasses that are created, and stores them in + `AsdfTypeIndex`. + """ + + def __new__(cls, name, bases, attrs): + new_cls = super().__new__(cls, name, bases, attrs) + # Classes using this metaclass get added to the list of built-in + # extensions + if name != "AsdfType": + _all_asdftypes.add(new_cls) + + return new_cls + + +class ExtensionType: + """ + The base class of all custom types in the tree. + + Besides the attributes defined below, most subclasses will also + override ``to_tree`` and ``from_tree``. + """ + + name = None + organization = "stsci.edu" + standard = "asdf" + version = (1, 0, 0) + supported_versions = set() + types = [] + handle_dynamic_subclasses = False + validators = {} + requires = [] + yaml_tag = None + + @classmethod + def names(cls): + """ + Returns the name(s) represented by this tag type as a list. + + While some tag types represent only a single custom type, others + represent multiple types. In the latter case, the `name` attribute of + the extension is actually a list, not simply a string. This method + normalizes the value of `name` by returning a list in all cases. + + Returns + ------- + `list` of names represented by this tag type + """ + if cls.name is None: + return None + + return cls.name if isinstance(cls.name, list) else [cls.name] + + @classmethod + def make_yaml_tag(cls, name, versioned=True): + """ + Given the name of a type, returns a string representing its YAML tag. + + Parameters + ---------- + name : str + The name of the type. In most cases this will correspond to the + `name` attribute of the tag type. However, it is passed as a + parameter since some tag types represent multiple custom + types. + + versioned : bool + If `True`, the tag will be versioned. Otherwise, a YAML tag without + a version will be returned. + + Returns + ------- + `str` representing the YAML tag + """ + return format_tag(cls.organization, cls.standard, cls.version if versioned else None, name) + + @classmethod + def tag_base(cls): + """ + Returns the base of the YAML tag for types represented by this class. + + This method returns the portion of the tag that represents the standard + and the organization of any type represented by this class. + + Returns + ------- + `str` representing the base of the YAML tag + """ + return cls.make_yaml_tag("", versioned=False) + + @classmethod + def to_tree(cls, node, ctx): + """ + Converts instances of custom types into YAML representations. + + This method should be overridden by custom extension classes in order + to define how custom types are serialized into YAML. The method must + return a single Python object corresponding to one of the basic YAML + types (dict, list, str, or number). However, the types can be nested + and combined in order to represent more complex custom types. + + This method is called as part of the process of writing an `asdf.AsdfFile` + object. Whenever a custom type (or a subclass of that type) that is + listed in the `types` attribute of this class is encountered, this + method will be used to serialize that type. + + The name `to_tree` refers to the act of converting a custom type into + part of a YAML object tree. + + Parameters + ---------- + node : `object` + Instance of a custom type to be serialized. Will be an instance (or + an instance of a subclass) of one of the types listed in the + `types` attribute of this class. + + ctx : `asdf.AsdfFile` + An instance of the `asdf.AsdfFile` object that is being written out. + + Returns + ------- + A basic YAML type (`dict`, `list`, `str`, `int`, `float`, or + `complex`) representing the properties of the custom type to be + serialized. These types can be nested in order to represent more + complex custom types. + """ + return node.__class__.__bases__[0](node) + + @classmethod + def to_tree_tagged(cls, node, ctx): + """ + Converts instances of custom types into tagged objects. + + It is more common for custom tag types to override `to_tree` instead of + this method. This method should be overridden if it is necessary + to modify the YAML tag that will be used to tag this object. + + Parameters + ---------- + node : `object` + Instance of a custom type to be serialized. Will be an instance (or + an instance of a subclass) of one of the types listed in the + `types` attribute of this class. + + ctx : `asdf.AsdfFile` + An instance of the `asdf.AsdfFile` object that is being written out. + + Returns + ------- + An instance of `asdf.tagged.Tagged`. + """ + obj = cls.to_tree(node, ctx) + return tagged.tag_object(cls.yaml_tag, obj, ctx=ctx) + + @classmethod + def from_tree(cls, tree, ctx): + """ + Converts basic types representing YAML trees into custom types. + + This method should be overridden by custom extension classes in order + to define how custom types are deserialized from the YAML + representation back into their original types. Typically the method will + return an instance of the original custom type. It is also permitted + to return a generator, which yields a partially constructed result, then + completes construction once the generator is drained. This is useful + when constructing objects that contain reference cycles. + + This method is called as part of the process of reading an ASDF file in + order to construct an `asdf.AsdfFile` object. Whenever a YAML subtree is + encountered that has a tag that corresponds to the `yaml_tag` property + of this class, this method will be used to deserialize that tree back + into an instance of the original custom type. + + Parameters + ---------- + tree : `object` representing YAML tree + An instance of a basic Python type (possibly nested) that + corresponds to a YAML subtree. + + ctx : `asdf.AsdfFile` + An instance of the `asdf.AsdfFile` object that is being constructed. + + Returns + ------- + An instance of the custom type represented by this extension class, + or a generator that yields that instance. + """ + return cls(tree) + + @classmethod + def from_tree_tagged(cls, tree, ctx): + """ + Converts from tagged tree into custom type. + + It is more common for extension classes to override `from_tree` instead + of this method. This method should only be overridden if it is + necessary to access the `_tag` property of the `~asdf.tagged.Tagged` object + directly. + + Parameters + ---------- + tree : `asdf.tagged.Tagged` object representing YAML tree + + ctx : `asdf.AsdfFile` + An instance of the `asdf.AsdfFile` object that is being constructed. + + Returns + ------- + An instance of the custom type represented by this extension class. + """ + return cls.from_tree(tree.data, ctx) + + @classmethod + def incompatible_version(cls, version): + """ + Indicates if given version is known to be incompatible with this type. + + If this tag class explicitly identifies compatible versions then this + checks whether a given version is compatible or not (see + `supported_versions`). Otherwise, all versions are assumed to be + compatible. + + Child classes can override this method to affect how version + compatibility for this type is determined. + + Parameters + ---------- + version : `str` or `~asdf.versioning.AsdfVersion` + The version to test for compatibility. + """ + if cls.supported_versions and version not in cls.supported_versions: + return True + + return False + + +class AsdfType(ExtensionType, metaclass=AsdfTypeMeta): + """ + Base class for all built-in ASDF types. Types that inherit this class will + be automatically added to the list of built-ins. This should *not* be used + for user-defined extensions. + """ + + +class CustomType(ExtensionType, metaclass=ExtensionTypeMeta): + """ + Base class for all user-defined types. + """ + + # These attributes are duplicated here with docstrings since a bug in + # sphinx prevents the docstrings of class attributes from being inherited + # properly (see https://github.com/sphinx-doc/sphinx/issues/741). The + # docstrings are not included anywhere else in the class hierarchy since + # this class is the only one exposed in the public API. + name = None + """ + `str` or `list`: The name of the type. + """ + + organization = "stsci.edu" + """ + `str`: The organization responsible for the type. + """ + + standard = "asdf" + """ + `str`: The standard the type is defined in. + """ + + version = (1, 0, 0) + """ + `str`, `tuple`, `asdf.versioning.AsdfVersion`, or `asdf.versioning.AsdfSpec`: + The version of the type. + """ + + supported_versions = set() + """ + `set`: Versions that explicitly compatible with this extension class. + + If provided, indicates explicit compatibility with the given set + of versions. Other versions of the same schema that are not included in + this set will not be converted to custom types with this class. """ + + types = [] + """ + `list`: List of types that this extension class can convert to/from YAML. + + Custom Python types that, when found in the tree, will be converted into + basic types for YAML output. Can be either strings referring to the types + or the types themselves.""" + + handle_dynamic_subclasses = False + """ + `bool`: Indicates whether dynamically generated subclasses can be serialized + + Flag indicating whether this type is capable of serializing subclasses + of any of the types listed in ``types`` that are generated dynamically. + """ + + validators = {} + """ + `dict`: Mapping JSON Schema keywords to validation functions for jsonschema. + + Useful if the type defines extra types of validation that can be + performed. + """ + + requires = [] + """ + `list`: Python packages that are required to instantiate the object. + """ + + yaml_tag = None + """ + `str`: The YAML tag to use for the type. + + If not provided, it will be automatically generated from name, + organization, standard and version. + """ + + has_required_modules = True + """ + `bool`: Indicates whether modules specified by `requires` are available. + + NOTE: This value is automatically generated. Do not set it in subclasses as + it will be overwritten. + """ + + def __init_subclass__(cls, **kwargs): + super().__init_subclass__(**kwargs) + + # Create a warning for a direct child of a CustomType class (not in grandchild) + if CustomType in cls.__bases__: + warnings.warn( + f"{cls.__name__} from {cls.__module__} subclasses the deprecated CustomType class. " + "Please see the new extension API " + "https://asdf.readthedocs.io/en/stable/asdf/extending/converters.html", + AsdfDeprecationWarning, + ) diff --git a/asdf/asdftypes.py b/asdf/asdftypes.py index 84387a073..b91927e5f 100644 --- a/asdf/asdftypes.py +++ b/asdf/asdftypes.py @@ -1,7 +1,7 @@ import warnings +from ._types import AsdfType, CustomType, ExtensionTypeMeta, format_tag from .exceptions import AsdfDeprecationWarning -from .types import AsdfType, CustomType, ExtensionTypeMeta, format_tag # This is not exhaustive, but represents the public API from .versioning import join_tag_version, split_tag_version diff --git a/asdf/conftest.py b/asdf/conftest.py index c916ca7f5..2b11722ab 100644 --- a/asdf/conftest.py +++ b/asdf/conftest.py @@ -7,7 +7,7 @@ from asdf.tests.httpserver import HTTPServer, RangeHTTPServer -collect_ignore = ["asdftypes.py", "fits_embed.py", "resolver.py", "type_index.py"] +collect_ignore = ["asdftypes.py", "fits_embed.py", "resolver.py", "type_index.py", "types.py"] @pytest.fixture() diff --git a/asdf/entry_points.py b/asdf/entry_points.py index a04d07ca6..897d073cb 100644 --- a/asdf/entry_points.py +++ b/asdf/entry_points.py @@ -54,15 +54,21 @@ def _handle_error(e): # Catch errors loading entry points and warn instead of raising try: - # Filter out the legacy `CustomType` deprecation warnings from the deprecated astropy.io.misc.asdf - # Testing will turn these into errors with warnings.catch_warnings(): - # Most of the astropy.io.misc.asdf deprecation warnings fall under this category - warnings.filterwarnings( - "ignore", - category=AsdfDeprecationWarning, - message=r".*from astropy.io.misc.asdf.* subclasses the deprecated CustomType .*", - ) + if entry_point.name == "astropy" and entry_point.group == LEGACY_EXTENSIONS_GROUP: + # Filter out the legacy `CustomType` deprecation warnings from the deprecated astropy.io.misc.asdf + # Testing will turn these into errors + # Most of the astropy.io.misc.asdf deprecation warnings fall under this category + warnings.filterwarnings( + "ignore", + category=AsdfDeprecationWarning, + message=r".*from astropy.io.misc.asdf.* subclasses the deprecated CustomType .*", + ) + warnings.filterwarnings( + "ignore", + category=AsdfDeprecationWarning, + message="asdf.types is deprecated", + ) elements = entry_point.load()() except Exception as e: # noqa: BLE001 diff --git a/asdf/extension/_legacy.py b/asdf/extension/_legacy.py index d7b7389c9..cc1db0df9 100644 --- a/asdf/extension/_legacy.py +++ b/asdf/extension/_legacy.py @@ -3,7 +3,7 @@ from functools import lru_cache from asdf import _resolver as resolver -from asdf import types +from asdf import _types as types from asdf._type_index import AsdfTypeIndex from asdf.exceptions import AsdfDeprecationWarning diff --git a/asdf/reference.py b/asdf/reference.py index fa7fdc9fa..4ae7686bd 100644 --- a/asdf/reference.py +++ b/asdf/reference.py @@ -11,8 +11,7 @@ import numpy as np -from . import generic_io, treeutil, util -from .types import AsdfType +from . import _types, generic_io, treeutil, util from .util import patched_urllib_parse __all__ = ["resolve_fragment", "Reference", "find_references", "resolve_references", "make_reference"] @@ -43,7 +42,7 @@ def resolve_fragment(tree, pointer): return tree -class Reference(AsdfType): +class Reference(_types.AsdfType): yaml_tag = "tag:yaml.org,2002:map" def __init__(self, uri, base_uri=None, asdffile=None, target=None): diff --git a/asdf/tags/core/__init__.py b/asdf/tags/core/__init__.py index 9b0f608b2..b654e0701 100644 --- a/asdf/tags/core/__init__.py +++ b/asdf/tags/core/__init__.py @@ -1,4 +1,4 @@ -from asdf.types import AsdfType +from asdf import _types from .complex import ComplexType from .constant import ConstantType @@ -24,7 +24,7 @@ class AsdfObject(dict): pass -class AsdfObjectType(AsdfType): +class AsdfObjectType(_types.AsdfType): name = "core/asdf" version = "1.1.0" supported_versions = {"1.0.0", "1.1.0"} @@ -39,17 +39,17 @@ def to_tree(cls, data, ctx): return dict(data) -class Software(dict, AsdfType): +class Software(dict, _types.AsdfType): name = "core/software" version = "1.0.0" -class HistoryEntry(dict, AsdfType): +class HistoryEntry(dict, _types.AsdfType): name = "core/history_entry" version = "1.0.0" -class ExtensionMetadata(dict, AsdfType): +class ExtensionMetadata(dict, _types.AsdfType): name = "core/extension_metadata" version = "1.0.0" @@ -66,7 +66,7 @@ def software(self): return self.get("software") -class SubclassMetadata(dict, AsdfType): +class SubclassMetadata(dict, _types.AsdfType): """ The tagged object supported by this class is part of an experimental feature that has since been dropped diff --git a/asdf/tags/core/complex.py b/asdf/tags/core/complex.py index 625cc5e20..3a644e7f8 100644 --- a/asdf/tags/core/complex.py +++ b/asdf/tags/core/complex.py @@ -1,10 +1,9 @@ import numpy as np -from asdf import util -from asdf.types import AsdfType +from asdf import _types, util -class ComplexType(AsdfType): +class ComplexType(_types.AsdfType): name = "core/complex" version = "1.0.0" types = [*list(util.iter_subclasses(np.complexfloating)), complex] diff --git a/asdf/tags/core/constant.py b/asdf/tags/core/constant.py index df0dd3f87..a000a2622 100644 --- a/asdf/tags/core/constant.py +++ b/asdf/tags/core/constant.py @@ -1,4 +1,4 @@ -from asdf.types import AsdfType +from asdf import _types class Constant: @@ -10,7 +10,7 @@ def value(self): return self._value -class ConstantType(AsdfType): +class ConstantType(_types.AsdfType): name = "core/constant" version = "1.0.0" types = [Constant] diff --git a/asdf/tags/core/external_reference.py b/asdf/tags/core/external_reference.py index 22b30ff1a..3e7e6000a 100644 --- a/asdf/tags/core/external_reference.py +++ b/asdf/tags/core/external_reference.py @@ -1,7 +1,7 @@ -from asdf.types import AsdfType +from . import _types -class ExternalArrayReference(AsdfType): +class ExternalArrayReference(_types.AsdfType): """ Store a reference to an array in an external File. diff --git a/asdf/tags/core/integer.py b/asdf/tags/core/integer.py index ae3ba6d93..a355d3679 100644 --- a/asdf/tags/core/integer.py +++ b/asdf/tags/core/integer.py @@ -2,10 +2,10 @@ import numpy as np -from asdf.types import AsdfType +from asdf import _types -class IntegerType(AsdfType): +class IntegerType(_types.AsdfType): """ Enables the storage of arbitrarily large integer values diff --git a/asdf/tags/core/ndarray.py b/asdf/tags/core/ndarray.py index 552954c49..3680dab8e 100644 --- a/asdf/tags/core/ndarray.py +++ b/asdf/tags/core/ndarray.py @@ -5,8 +5,7 @@ from jsonschema import ValidationError from numpy import ma -from asdf import util -from asdf.types import AsdfType +from asdf import _types, util _datatype_names = { "int8": "i1", @@ -227,7 +226,7 @@ def ascii_to_unicode(x): return ascii_to_unicode(tolist(array)) -class NDArrayType(AsdfType): +class NDArrayType(_types.AsdfType): name = "core/ndarray" version = "1.0.0" supported_versions = {"1.0.0", "1.1.0"} @@ -393,7 +392,7 @@ def __getattribute__(self, name): msg = f"'{self.__class__.name}' object has no attribute '{name}'" raise AttributeError(msg) - return AsdfType.__getattribute__(self, name) + return _types.AsdfType.__getattribute__(self, name) @classmethod def from_tree(cls, node, ctx): diff --git a/asdf/tags/core/tests/test_history.py b/asdf/tags/core/tests/test_history.py index bb3c38f66..f89dd52cf 100644 --- a/asdf/tags/core/tests/test_history.py +++ b/asdf/tags/core/tests/test_history.py @@ -6,7 +6,8 @@ from jsonschema import ValidationError import asdf -from asdf import types, util +from asdf import _types as types +from asdf import util from asdf.exceptions import AsdfDeprecationWarning, AsdfWarning from asdf.tags.core import HistoryEntry from asdf.tests import helpers diff --git a/asdf/tests/test_deprecated.py b/asdf/tests/test_deprecated.py index ace8617ff..ea7f396c1 100644 --- a/asdf/tests/test_deprecated.py +++ b/asdf/tests/test_deprecated.py @@ -3,10 +3,10 @@ import pytest import asdf +from asdf._types import CustomType from asdf.exceptions import AsdfDeprecationWarning from asdf.tests.helpers import assert_extension_correctness from asdf.tests.objects import CustomExtension -from asdf.types import CustomType def test_custom_type_warning(): @@ -61,3 +61,10 @@ def test_asdfile_run_hook_deprecation(): def test_asdfile_run_modifying_hook_deprecation(): with asdf.AsdfFile() as af, pytest.warns(AsdfDeprecationWarning, match="AsdfFile.run_modifying_hook is deprecated"): af.run_modifying_hook("foo") + + +def test_types_module_deprecation(): + with pytest.warns(AsdfDeprecationWarning, match="^asdf.types is deprecated.*$"): + if "asdf.types" in sys.modules: + del sys.modules["asdf.types"] + import asdf.types # noqa: F401 diff --git a/asdf/tests/test_extension.py b/asdf/tests/test_extension.py index ce488f55e..7af540731 100644 --- a/asdf/tests/test_extension.py +++ b/asdf/tests/test_extension.py @@ -2,6 +2,7 @@ from packaging.specifiers import SpecifierSet from asdf import AsdfFile, config_context +from asdf._types import CustomType from asdf.exceptions import AsdfDeprecationWarning, ValidationError from asdf.extension import ( AsdfExtension, @@ -19,7 +20,6 @@ get_cached_extension_manager, ) from asdf.tests.helpers import assert_extension_correctness -from asdf.types import CustomType def test_builtin_extension(): diff --git a/asdf/tests/test_helpers.py b/asdf/tests/test_helpers.py index 1495a7ae2..de540a88c 100644 --- a/asdf/tests/test_helpers.py +++ b/asdf/tests/test_helpers.py @@ -1,6 +1,6 @@ import pytest -from asdf import types +from asdf import _types as types from asdf.exceptions import AsdfConversionWarning, AsdfDeprecationWarning, AsdfWarning from asdf.tests.helpers import assert_roundtrip_tree diff --git a/asdf/tests/test_schema.py b/asdf/tests/test_schema.py index 5ef5a243f..c6c574cc7 100644 --- a/asdf/tests/test_schema.py +++ b/asdf/tests/test_schema.py @@ -8,7 +8,8 @@ import asdf from asdf import _resolver as resolver -from asdf import config_context, constants, extension, get_config, schema, tagged, types, util, yamlutil +from asdf import _types as types +from asdf import config_context, constants, extension, get_config, schema, tagged, util, yamlutil from asdf.exceptions import AsdfConversionWarning, AsdfDeprecationWarning, AsdfWarning from asdf.tests import helpers from asdf.tests.objects import CustomExtension diff --git a/asdf/tests/test_types.py b/asdf/tests/test_types.py index db825c46d..52ea911fb 100644 --- a/asdf/tests/test_types.py +++ b/asdf/tests/test_types.py @@ -4,7 +4,8 @@ import pytest import asdf -from asdf import extension, types, util, versioning +from asdf import _types as types +from asdf import extension, util, versioning from asdf.exceptions import AsdfConversionWarning, AsdfDeprecationWarning, AsdfWarning from . import helpers diff --git a/asdf/types.py b/asdf/types.py index 92dacc9b5..6ec4d54e0 100644 --- a/asdf/types.py +++ b/asdf/types.py @@ -1,496 +1,26 @@ -import importlib -import re +import sys import warnings -from copy import copy -from . import tagged, util +from . import _types from .exceptions import AsdfDeprecationWarning -from .versioning import AsdfSpec, AsdfVersion -__all__ = ["format_tag", "CustomType", "AsdfType", "ExtensionType"] - - -# regex used to parse module name from optional version string -MODULE_RE = re.compile(r"([a-zA-Z]+)(-(\d+\.\d+\.\d+))?") - - -def format_tag(organization, standard, version, tag_name): - """ - Format a YAML tag. - """ - tag = f"tag:{organization}:{standard}/{tag_name}" - - if version is None: - return tag - - if isinstance(version, AsdfSpec): - version = str(version.spec) - - return f"{tag}-{version}" - - -_all_asdftypes = set() - - -def _from_tree_tagged_missing_requirements(cls, tree, ctx): - # A special version of AsdfType.from_tree_tagged for when the - # required dependencies for an AsdfType are missing. - plural, verb = ("s", "are") if len(cls.requires) else ("", "is") - - # This error will be handled by yamlutil.tagged_tree_to_custom_tree, which - # will cause a warning to be issued indicating that the tree failed to be - # converted. - msg = f"{util.human_list(cls.requires)} package{plural} {verb} required to instantiate '{tree._tag}'" - raise TypeError(msg) - - -class ExtensionTypeMeta(type): - """ - Custom class constructor for tag types. - """ - - _import_cache = {} - - @classmethod - def _has_required_modules(cls, requires): - for string in requires: - has_module = True - match = MODULE_RE.match(string) - modname, _, version = match.groups() - if modname in cls._import_cache and not cls._import_cache[modname]: - return False - - try: - module = importlib.import_module(modname) - if version and hasattr(module, "__version__") and module.__version__ < version: - has_module = False - - except ImportError: - has_module = False - - finally: - cls._import_cache[modname] = has_module - if not has_module: - return False # noqa: B012 - - return True - - @classmethod - def _find_in_bases(cls, attrs, bases, name, default=None): - if name in attrs: - return attrs[name] - for base in bases: - if hasattr(base, name): - return getattr(base, name) - return default - - @property - def versioned_siblings(cls): - return getattr(cls, "__versioned_siblings") or [] - - def __new__(cls, name, bases, attrs): - requires = cls._find_in_bases(attrs, bases, "requires", []) - if not cls._has_required_modules(requires): - attrs["from_tree_tagged"] = classmethod(_from_tree_tagged_missing_requirements) - attrs["types"] = [] - attrs["has_required_modules"] = False - else: - attrs["has_required_modules"] = True - types = cls._find_in_bases(attrs, bases, "types", []) - new_types = [] - for typ in types: - if isinstance(typ, str): - typ = util.resolve_name(typ) - new_types.append(typ) - attrs["types"] = new_types - - new_cls = super().__new__(cls, name, bases, attrs) - - if hasattr(new_cls, "version") and not isinstance(new_cls.version, (AsdfVersion, AsdfSpec)): - new_cls.version = AsdfVersion(new_cls.version) - - if hasattr(new_cls, "name"): - if isinstance(new_cls.name, str): - if "yaml_tag" not in attrs: - new_cls.yaml_tag = new_cls.make_yaml_tag(new_cls.name) - elif isinstance(new_cls.name, list): - pass - elif new_cls.name is not None: - msg = "name must be string or list" - raise TypeError(msg) - - if hasattr(new_cls, "supported_versions"): - if not isinstance(new_cls.supported_versions, (list, set)): - new_cls.supported_versions = [new_cls.supported_versions] - supported_versions = set() - for version in new_cls.supported_versions: - if not isinstance(version, (AsdfVersion, AsdfSpec)): - version = AsdfVersion(version) - # This should cause an exception for invalid input - supported_versions.add(version) - # We need to convert back to a list here so that the 'in' operator - # uses actual comparison instead of hash equality - new_cls.supported_versions = list(supported_versions) - siblings = [] - for version in new_cls.supported_versions: - if version != new_cls.version: - new_attrs = copy(attrs) - new_attrs["version"] = version - new_attrs["supported_versions"] = set() - new_attrs["_latest_version"] = new_cls.version - if "__classcell__" in new_attrs: - msg = ( - "Subclasses of ExtensionTypeMeta that define " - "supported_versions cannot used super() to call " - "parent class functions. super() creates a " - "__classcell__ closure that cannot be duplicated " - "during creation of versioned siblings. " - "See https://github.com/asdf-format/asdf/issues/1245" - ) - raise RuntimeError(msg) - siblings.append(ExtensionTypeMeta.__new__(cls, name, bases, new_attrs)) - setattr(new_cls, "__versioned_siblings", siblings) - - return new_cls - - -class AsdfTypeMeta(ExtensionTypeMeta): - """ - Keeps track of `AsdfType` subclasses that are created, and stores them in - `AsdfTypeIndex`. - """ - - def __new__(cls, name, bases, attrs): - new_cls = super().__new__(cls, name, bases, attrs) - # Classes using this metaclass get added to the list of built-in - # extensions - if name != "AsdfType": - _all_asdftypes.add(new_cls) - - return new_cls - - -class ExtensionType: - """ - The base class of all custom types in the tree. - - Besides the attributes defined below, most subclasses will also - override ``to_tree`` and ``from_tree``. - """ - - name = None - organization = "stsci.edu" - standard = "asdf" - version = (1, 0, 0) - supported_versions = set() - types = [] - handle_dynamic_subclasses = False - validators = {} - requires = [] - yaml_tag = None - - @classmethod - def names(cls): - """ - Returns the name(s) represented by this tag type as a list. - - While some tag types represent only a single custom type, others - represent multiple types. In the latter case, the `name` attribute of - the extension is actually a list, not simply a string. This method - normalizes the value of `name` by returning a list in all cases. - - Returns - ------- - `list` of names represented by this tag type - """ - if cls.name is None: - return None - - return cls.name if isinstance(cls.name, list) else [cls.name] - - @classmethod - def make_yaml_tag(cls, name, versioned=True): - """ - Given the name of a type, returns a string representing its YAML tag. - - Parameters - ---------- - name : str - The name of the type. In most cases this will correspond to the - `name` attribute of the tag type. However, it is passed as a - parameter since some tag types represent multiple custom - types. - - versioned : bool - If `True`, the tag will be versioned. Otherwise, a YAML tag without - a version will be returned. - - Returns - ------- - `str` representing the YAML tag - """ - return format_tag(cls.organization, cls.standard, cls.version if versioned else None, name) - - @classmethod - def tag_base(cls): - """ - Returns the base of the YAML tag for types represented by this class. - - This method returns the portion of the tag that represents the standard - and the organization of any type represented by this class. - - Returns - ------- - `str` representing the base of the YAML tag - """ - return cls.make_yaml_tag("", versioned=False) - - @classmethod - def to_tree(cls, node, ctx): - """ - Converts instances of custom types into YAML representations. - - This method should be overridden by custom extension classes in order - to define how custom types are serialized into YAML. The method must - return a single Python object corresponding to one of the basic YAML - types (dict, list, str, or number). However, the types can be nested - and combined in order to represent more complex custom types. - - This method is called as part of the process of writing an `asdf.AsdfFile` - object. Whenever a custom type (or a subclass of that type) that is - listed in the `types` attribute of this class is encountered, this - method will be used to serialize that type. - - The name `to_tree` refers to the act of converting a custom type into - part of a YAML object tree. - - Parameters - ---------- - node : `object` - Instance of a custom type to be serialized. Will be an instance (or - an instance of a subclass) of one of the types listed in the - `types` attribute of this class. - - ctx : `asdf.AsdfFile` - An instance of the `asdf.AsdfFile` object that is being written out. - - Returns - ------- - A basic YAML type (`dict`, `list`, `str`, `int`, `float`, or - `complex`) representing the properties of the custom type to be - serialized. These types can be nested in order to represent more - complex custom types. - """ - return node.__class__.__bases__[0](node) - - @classmethod - def to_tree_tagged(cls, node, ctx): - """ - Converts instances of custom types into tagged objects. - - It is more common for custom tag types to override `to_tree` instead of - this method. This method should be overridden if it is necessary - to modify the YAML tag that will be used to tag this object. - - Parameters - ---------- - node : `object` - Instance of a custom type to be serialized. Will be an instance (or - an instance of a subclass) of one of the types listed in the - `types` attribute of this class. - - ctx : `asdf.AsdfFile` - An instance of the `asdf.AsdfFile` object that is being written out. - - Returns - ------- - An instance of `asdf.tagged.Tagged`. - """ - obj = cls.to_tree(node, ctx) - return tagged.tag_object(cls.yaml_tag, obj, ctx=ctx) - - @classmethod - def from_tree(cls, tree, ctx): - """ - Converts basic types representing YAML trees into custom types. - - This method should be overridden by custom extension classes in order - to define how custom types are deserialized from the YAML - representation back into their original types. Typically the method will - return an instance of the original custom type. It is also permitted - to return a generator, which yields a partially constructed result, then - completes construction once the generator is drained. This is useful - when constructing objects that contain reference cycles. - - This method is called as part of the process of reading an ASDF file in - order to construct an `asdf.AsdfFile` object. Whenever a YAML subtree is - encountered that has a tag that corresponds to the `yaml_tag` property - of this class, this method will be used to deserialize that tree back - into an instance of the original custom type. - - Parameters - ---------- - tree : `object` representing YAML tree - An instance of a basic Python type (possibly nested) that - corresponds to a YAML subtree. - - ctx : `asdf.AsdfFile` - An instance of the `asdf.AsdfFile` object that is being constructed. - - Returns - ------- - An instance of the custom type represented by this extension class, - or a generator that yields that instance. - """ - return cls(tree) - - @classmethod - def from_tree_tagged(cls, tree, ctx): - """ - Converts from tagged tree into custom type. - - It is more common for extension classes to override `from_tree` instead - of this method. This method should only be overridden if it is - necessary to access the `_tag` property of the `~asdf.tagged.Tagged` object - directly. - - Parameters - ---------- - tree : `asdf.tagged.Tagged` object representing YAML tree - - ctx : `asdf.AsdfFile` - An instance of the `asdf.AsdfFile` object that is being constructed. - - Returns - ------- - An instance of the custom type represented by this extension class. - """ - return cls.from_tree(tree.data, ctx) - - @classmethod - def incompatible_version(cls, version): - """ - Indicates if given version is known to be incompatible with this type. - - If this tag class explicitly identifies compatible versions then this - checks whether a given version is compatible or not (see - `supported_versions`). Otherwise, all versions are assumed to be - compatible. - - Child classes can override this method to affect how version - compatibility for this type is determined. - - Parameters - ---------- - version : `str` or `~asdf.versioning.AsdfVersion` - The version to test for compatibility. - """ - if cls.supported_versions and version not in cls.supported_versions: - return True - - return False - - -class AsdfType(ExtensionType, metaclass=AsdfTypeMeta): - """ - Base class for all built-in ASDF types. Types that inherit this class will - be automatically added to the list of built-ins. This should *not* be used - for user-defined extensions. - """ - - -class CustomType(ExtensionType, metaclass=ExtensionTypeMeta): - """ - Base class for all user-defined types. - """ - - # These attributes are duplicated here with docstrings since a bug in - # sphinx prevents the docstrings of class attributes from being inherited - # properly (see https://github.com/sphinx-doc/sphinx/issues/741). The - # docstrings are not included anywhere else in the class hierarchy since - # this class is the only one exposed in the public API. - name = None - """ - `str` or `list`: The name of the type. - """ - - organization = "stsci.edu" - """ - `str`: The organization responsible for the type. - """ - - standard = "asdf" - """ - `str`: The standard the type is defined in. - """ - - version = (1, 0, 0) - """ - `str`, `tuple`, `asdf.versioning.AsdfVersion`, or `asdf.versioning.AsdfSpec`: - The version of the type. - """ - - supported_versions = set() - """ - `set`: Versions that explicitly compatible with this extension class. - - If provided, indicates explicit compatibility with the given set - of versions. Other versions of the same schema that are not included in - this set will not be converted to custom types with this class. """ - - types = [] - """ - `list`: List of types that this extension class can convert to/from YAML. - - Custom Python types that, when found in the tree, will be converted into - basic types for YAML output. Can be either strings referring to the types - or the types themselves.""" - - handle_dynamic_subclasses = False - """ - `bool`: Indicates whether dynamically generated subclasses can be serialized - - Flag indicating whether this type is capable of serializing subclasses - of any of the types listed in ``types`` that are generated dynamically. - """ - - validators = {} - """ - `dict`: Mapping JSON Schema keywords to validation functions for jsonschema. - - Useful if the type defines extra types of validation that can be - performed. - """ - - requires = [] - """ - `list`: Python packages that are required to instantiate the object. - """ - - yaml_tag = None - """ - `str`: The YAML tag to use for the type. - - If not provided, it will be automatically generated from name, - organization, standard and version. - """ - - has_required_modules = True - """ - `bool`: Indicates whether modules specified by `requires` are available. - - NOTE: This value is automatically generated. Do not set it in subclasses as - it will be overwritten. - """ - - def __init_subclass__(cls, **kwargs): - super().__init_subclass__(**kwargs) - - # Create a warning for a direct child of a CustomType class (not in grandchild) - if CustomType in cls.__bases__: - warnings.warn( - f"{cls.__name__} from {cls.__module__} subclasses the deprecated CustomType class. " - "Please see the new extension API " - "https://asdf.readthedocs.io/en/stable/asdf/extending/converters.html", - AsdfDeprecationWarning, - ) +warnings.warn( + "asdf.types is deprecated " + "Please see the new extension API " + "https://asdf.readthedocs.io/en/stable/asdf/extending/converters.html", + AsdfDeprecationWarning, +) + + +# overwrite the hidden module __file__ so pytest doesn't throw an ImportPathMismatchError +_types.__file__ = __file__ +# overwrite the module name for each defined class to allow doc references to work +for class_ in [ + _types.ExtensionTypeMeta, + _types.AsdfTypeMeta, + _types.ExtensionType, + _types.AsdfType, + _types.CustomType, +]: + class_.__module__ = __name__ +sys.modules[__name__] = _types diff --git a/tox.ini b/tox.ini index 6fc7451d1..1f5f449a4 100644 --- a/tox.ini +++ b/tox.ini @@ -126,8 +126,9 @@ commands_pre = pip install -e astropy[test] pip install -r {env_tmp_dir}/requirements.txt pip freeze -commands = - pytest astropy/astropy/io/misc/asdf --open-files --run-slow --remote-data +commands= + pytest astropy/astropy/io/misc/asdf --open-files --run-slow --remote-data \ + -W "ignore::asdf.exceptions.AsdfDeprecationWarning:asdf.types" [testenv:asdf-astropy] change_dir = {env_tmp_dir} @@ -158,7 +159,8 @@ commands_pre = pip freeze commands = pytest specutils \ - -W "ignore::asdf.exceptions.AsdfDeprecationWarning:asdf.fits_embed" + -W "ignore::asdf.exceptions.AsdfDeprecationWarning:asdf.fits_embed" \ + -W "ignore::asdf.exceptions.AsdfDeprecationWarning:asdf.types" [testenv:gwcs] change_dir = {env_tmp_dir}