Merge pull request #1336 from braingram/feature/deprecate_blocks

CHANGES.rst

@@ -7,6 +7,7 @@ The ASDF Standard is at v1.6.0
 - Add AsdfProvisionalAPIWarning to warn developers of new features that
   may undergo breaking changes but are likely to be included as stable
   features (without this warning) in a future version of ASDF [#1295]
+- Add AsdfDeprecationWarning to AsdfFile.blocks [#1336]
 
 2.14.3 (2022-12-15)
 -------------------

asdf/asdf.py

@@ -658,6 +658,12 @@ def blocks(self):
         """
         Get the block manager associated with the `AsdfFile`.
         """
+        warnings.warn(
+            "The property AsdfFile.blocks has been deprecated and will be removed "
+            "in asdf-3.0. Public use of the block manager is strongly discouraged "
+            "as there is no stable API",
+            AsdfDeprecationWarning,
+        )
         return self._blocks
 
     def set_array_storage(self, arr, array_storage):
@@ -682,8 +688,8 @@ def set_array_storage(self, arr, array_storage):
 
             - ``inline``: Store the data as YAML inline in the tree.
         """
-        block = self.blocks[arr]
-        self.blocks.set_array_storage(block, array_storage)
+        block = self._blocks[arr]
+        self._blocks.set_array_storage(block, array_storage)
 
     def get_array_storage(self, arr):
         """
@@ -693,7 +699,7 @@ def get_array_storage(self, arr):
         ----------
         arr : numpy.ndarray
         """
-        return self.blocks[arr].array_storage
+        return self._blocks[arr].array_storage
 
     def set_array_compression(self, arr, compression, **compression_kwargs):
         """
@@ -721,8 +727,8 @@ def set_array_compression(self, arr, compression, **compression_kwargs):
               If there is no prior file, acts as None.
 
         """
-        self.blocks[arr].output_compression = compression
-        self.blocks[arr].output_compression_kwargs = compression_kwargs
+        self._blocks[arr].output_compression = compression
+        self._blocks[arr].output_compression_kwargs = compression_kwargs
 
     def get_array_compression(self, arr):
         """
@@ -736,11 +742,11 @@ def get_array_compression(self, arr):
         -------
         compression : str or None
         """
-        return self.blocks[arr].output_compression
+        return self._blocks[arr].output_compression
 
     def get_array_compression_kwargs(self, arr):
         """ """
-        return self.blocks[arr].output_compression_kwargs
+        return self._blocks[arr].output_compression_kwargs
 
     @classmethod
     def _parse_header_line(cls, line):
@@ -1070,7 +1076,7 @@ def _write_tree(self, tree, fd, pad_blocks):
         if len(tree):
             serialization_context = self._create_serialization_context()
 
-            compression_extensions = self.blocks.get_output_compression_extensions()
+            compression_extensions = self._blocks.get_output_compression_extensions()
             for ext in compression_extensions:
                 serialization_context._mark_extension_used(ext)
 
@@ -1124,17 +1130,17 @@ def _pre_write(self, fd, all_array_storage, all_array_compression, compression_k
 
     def _serial_write(self, fd, pad_blocks, include_block_index):
         self._write_tree(self._tree, fd, pad_blocks)
-        self.blocks.write_internal_blocks_serial(fd, pad_blocks)
-        self.blocks.write_external_blocks(fd.uri, pad_blocks)
+        self._blocks.write_internal_blocks_serial(fd, pad_blocks)
+        self._blocks.write_external_blocks(fd.uri, pad_blocks)
         if include_block_index:
-            self.blocks.write_block_index(fd, self)
+            self._blocks.write_block_index(fd, self)
 
     def _random_write(self, fd, pad_blocks, include_block_index):
         self._write_tree(self._tree, fd, False)
-        self.blocks.write_internal_blocks_random_access(fd)
-        self.blocks.write_external_blocks(fd.uri, pad_blocks)
+        self._blocks.write_internal_blocks_random_access(fd)
+        self._blocks.write_external_blocks(fd.uri, pad_blocks)
         if include_block_index:
-            self.blocks.write_block_index(fd, self)
+            self._blocks.write_block_index(fd, self)
         fd.truncate()
 
     def _post_write(self, fd):
@@ -1238,7 +1244,7 @@ def update(
                 msg = "Can not update, since associated file is not seekable"
                 raise OSError(msg)
 
-            self.blocks.finish_reading_internal_blocks()
+            self._blocks.finish_reading_internal_blocks()
 
             # flush all pending memmap writes
             if fd.can_memmap():
@@ -1249,7 +1255,7 @@ def update(
             try:
                 fd.seek(0)
 
-                if not self.blocks.has_blocks_with_offset():
+                if not self._blocks.has_blocks_with_offset():
                     # If we don't have any blocks that are being reused, just
                     # write out in a serial fashion.
                     self._serial_write(fd, pad_blocks, include_block_index)
@@ -1263,11 +1269,11 @@ def update(
                 # possible there.
                 tree_serialized = io.BytesIO()
                 self._write_tree(self._tree, tree_serialized, pad_blocks=False)
-                n_internal_blocks = len(self.blocks._internal_blocks)
+                n_internal_blocks = len(self._blocks._internal_blocks)
 
                 serialized_tree_size = tree_serialized.tell() + constants.MAX_BLOCKS_DIGITS * n_internal_blocks
 
-                if not block.calculate_updated_layout(self.blocks, serialized_tree_size, pad_blocks, fd.block_size):
+                if not block.calculate_updated_layout(self._blocks, serialized_tree_size, pad_blocks, fd.block_size):
                     # If we don't have any blocks that are being reused, just
                     # write out in a serial fashion.
                     self._serial_write(fd, pad_blocks, include_block_index)
@@ -1283,7 +1289,7 @@ def update(
                 if fd.can_memmap():
                     fd.close_memmap()
                     # also clean any memmapped blocks
-                    for b in self.blocks._internal_blocks:
+                    for b in self._blocks._internal_blocks:
                         if b._memmapped:
                             b._memmapped = False
                             b._data = None
@@ -1466,10 +1472,10 @@ def resolve_and_inline(self):
         produces something that, when saved, is a 100% valid YAML
         file.
         """
-        self.blocks.finish_reading_internal_blocks()
+        self._blocks.finish_reading_internal_blocks()
         self.resolve_references()
-        for b in list(self.blocks.blocks):
-            self.blocks.set_array_storage(b, "inline")
+        for b in list(self._blocks.blocks):
+            self._blocks.set_array_storage(b, "inline")
 
     def fill_defaults(self):
         """

asdf/block.py

@@ -362,7 +362,7 @@ def write_external_blocks(self, uri, pad_blocks=False):
             asdffile = asdf.AsdfFile()
             block = copy.copy(block)
             block._array_storage = "internal"
-            asdffile.blocks.add(block)
+            asdffile._blocks.add(block)
             block._used = True
             asdffile.write_to(subfd, pad_blocks=pad_blocks)
 
@@ -667,7 +667,7 @@ def get_block(self, source):
 
         if isinstance(source, str):
             asdffile = self._asdffile().open_external(source)
-            block = asdffile.blocks._internal_blocks[0]
+            block = asdffile._blocks._internal_blocks[0]
             self.set_array_storage(block, "external")
 
         # Handle the case of inline data

asdf/commands/tests/test_defragment.py

@@ -23,7 +23,7 @@ def _test_defragment(tmpdir, codec):
     out_path = os.path.join(str(tmpdir), "original.defragment.asdf")
     ff = AsdfFile(tree)
     ff.write_to(path)
-    assert len(ff.blocks) == 2
+    assert len(ff._blocks) == 2
 
     result = main.main_from_args(["defragment", path, "-o", out_path, "-c", codec])
 
@@ -38,7 +38,7 @@ def _test_defragment(tmpdir, codec):
 
     with asdf.open(os.path.join(str(tmpdir), "original.defragment.asdf")) as ff:
         assert_tree_match(ff.tree, tree)
-        assert len(list(ff.blocks.internal_blocks)) == 2
+        assert len(list(ff._blocks.internal_blocks)) == 2
 
 
 def test_defragment_zlib(tmpdir):

asdf/commands/tests/test_exploded.py

@@ -24,7 +24,7 @@ def test_explode_then_implode(tmpdir):
     # in internal blocks rather than letting some of them be automatically put
     # inline.
     ff.write_to(path, all_array_storage="internal")
-    assert len(ff.blocks) == 2
+    assert len(ff._blocks) == 2
 
     result = main.main_from_args(["explode", path])
 
@@ -47,7 +47,7 @@ def test_explode_then_implode(tmpdir):
 
     with asdf.open(str(tmpdir.join("original_exploded_all.asdf"))) as af:
         assert_tree_match(af.tree, tree)
-        assert len(af.blocks) == 2
+        assert len(af._blocks) == 2
 
 
 def test_file_not_found(tmpdir):

asdf/commands/tests/test_to_yaml.py

@@ -21,7 +21,7 @@ def test_to_yaml(tmpdir):
     path = os.path.join(str(tmpdir), "original.asdf")
     ff = AsdfFile(tree)
     ff.write_to(path)
-    assert len(ff.blocks) == 2
+    assert len(ff._blocks) == 2
 
     result = main.main_from_args(["to_yaml", path])
 
@@ -34,4 +34,4 @@ def test_to_yaml(tmpdir):
 
     with asdf.open(os.path.join(str(tmpdir), "original.yaml")) as ff:
         assert_tree_match(ff.tree, tree)
-        assert len(list(ff.blocks.internal_blocks)) == 0
+        assert len(list(ff._blocks.internal_blocks)) == 0

asdf/fits_embed.py

@@ -312,7 +312,7 @@ def _update_asdf_extension(
         use_image_hdu=False,
         **kwargs,
     ):
-        if self.blocks.streamed_block is not None:
+        if self._blocks.streamed_block is not None:
             msg = "Can not save streamed data to ASDF-in-FITS file."
             raise ValueError(msg)
 

asdf/stream.py

@@ -36,15 +36,15 @@ def _make_array(self):
     @classmethod
     def reserve_blocks(cls, data, ctx):
         if isinstance(data, Stream):
-            yield ctx.blocks.get_streamed_block()
+            yield ctx._blocks.get_streamed_block()
 
     @classmethod
     def from_tree(cls, data, ctx):
         return ndarray.NDArrayType.from_tree(data, ctx)
 
     @classmethod
     def to_tree(cls, data, ctx):
-        ctx.blocks.get_streamed_block()
+        ctx._blocks.get_streamed_block()
 
         result = {}
         result["source"] = -1

asdf/tags/core/ndarray.py

@@ -243,7 +243,7 @@ def __init__(self, source, shape, dtype, offset, strides, order, mask, asdffile)
         if isinstance(source, list):
             self._array = inline_data_asarray(source, dtype)
             self._array = self._apply_mask(self._array, self._mask)
-            self._block = asdffile.blocks.add_inline(self._array)
+            self._block = asdffile._blocks.add_inline(self._array)
             if shape is not None and (
                 (shape[0] == "*" and self._array.shape[1:] != tuple(shape[1:])) or (self._array.shape != tuple(shape))
             ):
@@ -255,7 +255,7 @@ def __init__(self, source, shape, dtype, offset, strides, order, mask, asdffile)
         self._offset = offset
         self._strides = strides
         self._order = order
-        if not asdffile.blocks.lazy_load:
+        if not asdffile._blocks.lazy_load:
             self._make_array()
 
     def _make_array(self):
@@ -341,7 +341,7 @@ def get_actual_shape(self, shape, strides, dtype, block_size):
     @property
     def block(self):
         if self._block is None:
-            self._block = self._asdffile.blocks.get_block(self._source)
+            self._block = self._asdffile._blocks.get_block(self._source)
         return self._block
 
     @property
@@ -425,7 +425,7 @@ def reserve_blocks(cls, data, ctx):
         # Find all of the used data buffers so we can add or rearrange
         # them if necessary
         if isinstance(data, np.ndarray):
-            yield ctx.blocks.find_or_create_block_for_array(data, ctx)
+            yield ctx._blocks.find_or_create_block_for_array(data, ctx)
         elif isinstance(data, NDArrayType):
             yield data.block
 
@@ -447,7 +447,7 @@ def to_tree(cls, data, ctx):
 
         shape = data.shape
 
-        block = ctx.blocks.find_or_create_block_for_array(data, ctx)
+        block = ctx._blocks.find_or_create_block_for_array(data, ctx)
 
         if block.array_storage == "fits":
             # Views over arrays stored in FITS files have some idiosyncrasies.
@@ -503,7 +503,7 @@ def to_tree(cls, data, ctx):
             if block.array_storage == "streamed":
                 result["shape"][0] = "*"
 
-            result["source"] = ctx.blocks.get_source(block)
+            result["source"] = ctx._blocks.get_source(block)
             result["datatype"] = dtype
             result["byteorder"] = byteorder
 
@@ -515,7 +515,7 @@ def to_tree(cls, data, ctx):
 
         if isinstance(data, ma.MaskedArray) and np.any(data.mask):
             if block.array_storage == "inline":
-                ctx.blocks.set_array_storage(ctx.blocks[data.mask], "inline")
+                ctx._blocks.set_array_storage(ctx._blocks[data.mask], "inline")
 
             result["mask"] = data.mask
 
@@ -566,7 +566,7 @@ def assert_allclose(cls, old, new):
     def copy_to_new_asdf(cls, node, asdffile):
         if isinstance(node, NDArrayType):
             array = node._make_array()
-            asdffile.blocks.set_array_storage(asdffile.blocks[array], node.block.array_storage)
+            asdffile._blocks.set_array_storage(asdffile._blocks[array], node.block.array_storage)
             return node._make_array()
         return node
 

asdf/tags/core/tests/test_integer.py

@@ -67,7 +67,7 @@ def test_integer_storage_duplication(tmpdir):
 
     with asdf.AsdfFile(tree) as af:
         af.write_to(tmpfile)
-        assert len(af.blocks) == 1
+        assert len(af._blocks) == 1
 
     with asdf.open(tmpfile, _force_raw_types=True) as rf:
         assert rf.tree["integer1"]["words"]["source"] == 0

asdf/tags/core/tests/test_ndarray.py

@@ -64,8 +64,8 @@ def check_asdf(asdf):
 
         assert tree["science_data"].ctypes.data == tree["skipping"].ctypes.data
 
-        assert len(list(asdf.blocks.internal_blocks)) == 1
-        assert next(asdf.blocks.internal_blocks)._size == 80
+        assert len(list(asdf._blocks.internal_blocks)) == 1
+        assert next(asdf._blocks.internal_blocks)._size == 80
 
         if "w" in asdf._mode:
             tree["science_data"][0] = 42
@@ -135,7 +135,7 @@ def test_dont_load_data():
         str(ff.tree["science_data"])
         repr(ff.tree)
 
-        for block in ff.blocks.internal_blocks:
+        for block in ff._blocks.internal_blocks:
             assert block._data is None
 
 
@@ -172,7 +172,7 @@ def test_array_inline_threshold_recursive(tmpdir):
     tree = {"test": aff}
 
     def check_asdf(asdf):
-        assert len(list(asdf.blocks.internal_blocks)) == 0
+        assert len(list(asdf._blocks.internal_blocks)) == 0
 
     with asdf.config_context() as config:
         config.array_inline_threshold = 100
@@ -253,13 +253,13 @@ def test_inline():
     buff = io.BytesIO()
 
     ff = asdf.AsdfFile(tree)
-    ff.blocks.set_array_storage(ff.blocks[tree["science_data"]], "inline")
+    ff._blocks.set_array_storage(ff._blocks[tree["science_data"]], "inline")
     ff.write_to(buff)
 
     buff.seek(0)
     with asdf.open(buff, mode="rw") as ff:
         helpers.assert_tree_match(tree, ff.tree)
-        assert len(list(ff.blocks.internal_blocks)) == 0
+        assert len(list(ff._blocks.internal_blocks)) == 0
         buff = io.BytesIO()
         ff.write_to(buff)
 
@@ -285,7 +285,7 @@ def check_asdf(asdf):
         m = tree["masked_array"]
 
         assert np.all(m.mask[6:])
-        assert len(asdf.blocks) == 2
+        assert len(asdf._blocks) == 2
 
     helpers.assert_roundtrip_tree(tree, tmpdir, asdf_check_func=check_asdf)
 
@@ -415,7 +415,7 @@ def test_inline_masked_array(tmpdir):
     f.write_to(testfile)
 
     with asdf.open(testfile) as f2:
-        assert len(list(f2.blocks.internal_blocks)) == 0
+        assert len(list(f2._blocks.internal_blocks)) == 0
         assert_array_equal(f.tree["test"], f2.tree["test"])
 
     with open(testfile, "rb") as fd: