Merge pull request #1151 from jiangliu/tarfs-merge

Enhance `nydus-image merge` to support tarfs

rafs/src/builder/compact.rs

@@ -3,7 +3,7 @@
 // SPDX-License-Identifier: Apache-2.0
 
 use std::collections::hash_map::Entry;
-use std::collections::HashMap;
+use std::collections::{HashMap, VecDeque};
 use std::io::Write;
 use std::ops::Deref;
 use std::path::PathBuf;
@@ -252,7 +252,7 @@ pub struct BlobCompactor {
     /// new blobs
     new_blob_mgr: BlobManager,
     /// inode list
-    nodes: Vec<Node>,
+    nodes: VecDeque<Node>,
     /// chunk --> list<node_idx, chunk_idx in node>
     c2nodes: HashMap<ChunkKey, Vec<(usize, usize)>>,
     /// original blob index --> list<node_idx, chunk_idx in node>
@@ -265,7 +265,7 @@ impl BlobCompactor {
     pub fn new(
         version: RafsVersion,
         ori_blob_mgr: BlobManager,
-        nodes: Vec<Node>,
+        nodes: VecDeque<Node>,
         backend: Arc<dyn BlobBackend + Send + Sync>,
         digester: digest::Algorithm,
     ) -> Result<Self> {
@@ -596,7 +596,7 @@ impl BlobCompactor {
         let tree = Tree::from_bootstrap(&rs, &mut _dict)?;
         let mut bootstrap = Bootstrap::new()?;
         bootstrap.build(&mut build_ctx, &mut bootstrap_ctx, tree)?;
-        let mut nodes = Vec::new();
+        let mut nodes = VecDeque::new();
         // move out nodes
         std::mem::swap(&mut bootstrap_ctx.nodes, &mut nodes);
         let mut compactor = Self::new(

rafs/src/builder/core/blob.rs

@@ -3,6 +3,7 @@
 // SPDX-License-Identifier: Apache-2.0
 
 use std::borrow::Cow;
+use std::collections::VecDeque;
 use std::io::Write;
 use std::slice;
 
@@ -27,7 +28,7 @@ impl Blob {
     /// Dump blob file and generate chunks
     pub(crate) fn dump(
         ctx: &BuildContext,
-        nodes: &mut [Node],
+        nodes: &mut VecDeque<Node>,
         blob_mgr: &mut BlobManager,
         blob_writer: &mut ArtifactWriter,
     ) -> Result<()> {

rafs/src/builder/core/bootstrap.rs

@@ -3,6 +3,7 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 
+use std::collections::VecDeque;
 use std::ffi::OsString;
 
 use anyhow::{Context, Error, Result};
@@ -15,7 +16,7 @@ use crate::builder::{
     ArtifactStorage, BlobManager, BootstrapContext, BootstrapManager, BuildContext, Tree,
 };
 use crate::metadata::layout::{RafsBlobTable, RAFS_V5_ROOT_INODE};
-use crate::metadata::{RafsSuper, RafsSuperConfig};
+use crate::metadata::{RafsSuper, RafsSuperConfig, RafsSuperFlags};
 
 pub(crate) const STARGZ_DEFAULT_BLOCK_SIZE: u32 = 4 << 20;
 
@@ -52,7 +53,7 @@ impl Bootstrap {
     ) -> Result<()> {
         // used to compute nid(ino) for v6
         let root_offset = bootstrap_ctx.offset;
-        let mut nodes = Vec::with_capacity(0x10000);
+        let mut nodes = VecDeque::with_capacity(0x10000);
 
         // Special handling of the root inode
         assert!(tree.node.is_dir());
@@ -63,7 +64,7 @@ impl Bootstrap {
             tree.node.inode.set_ino(RAFS_V5_ROOT_INODE);
         }
         ctx.prefetch.insert_if_need(&tree.node);
-        nodes.push(tree.node.clone());
+        nodes.push_back(tree.node.clone());
 
         Self::build_rafs(ctx, bootstrap_ctx, &mut tree, &mut nodes)?;
         if ctx.fs_version.is_v6() && !bootstrap_ctx.layered {
@@ -147,7 +148,7 @@ impl Bootstrap {
         ctx: &mut BuildContext,
         bootstrap_ctx: &mut BootstrapContext,
         tree: &mut Tree,
-        nodes: &mut Vec<Node>,
+        nodes: &mut VecDeque<Node>,
     ) -> Result<()> {
         let index = nodes.len() as u32 + 1;
         let parent = &mut nodes[tree.node.index as usize - 1];
@@ -234,14 +235,14 @@ impl Bootstrap {
                 (true, Some(whiteout_type)) => {
                     // Insert removal operations at the head, so they will be handled first when
                     // applying to lower layer.
-                    nodes.insert(0, child.node.clone());
+                    nodes.push_front(child.node.clone());
                     if whiteout_type == WhiteoutType::OverlayFsOpaque {
                         // For the overlayfs opaque, we need to remove the lower node that has the
                         // same name first, then apply upper node to the node tree of lower layer.
                         child
                             .node
                             .remove_xattr(&OsString::from(OVERLAYFS_WHITEOUT_OPAQUE));
-                        nodes.push(child.node.clone());
+                        nodes.push_back(child.node.clone());
                     }
                 }
                 (false, Some(whiteout_type)) => {
@@ -251,9 +252,9 @@ impl Bootstrap {
                             .node
                             .remove_xattr(&OsString::from(OVERLAYFS_WHITEOUT_OPAQUE));
                     }
-                    nodes.push(child.node.clone());
+                    nodes.push_back(child.node.clone());
                 }
-                _ => nodes.push(child.node.clone()),
+                _ => nodes.push_back(child.node.clone()),
             }
 
             ctx.prefetch.insert_if_need(&child.node);
@@ -298,6 +299,7 @@ impl Bootstrap {
             chunk_size: ctx.chunk_size,
             explicit_uidgid: ctx.explicit_uidgid,
             version: ctx.fs_version,
+            is_tarfs_mode: rs.meta.flags.contains(RafsSuperFlags::TARTFS_MODE),
         };
         config.check_compatibility(&rs.meta)?;
 

rafs/src/builder/core/chunk_dict.rs

@@ -264,6 +264,7 @@ mod tests {
             digester: digest::Algorithm::Blake3,
             chunk_size: 0x100000,
             explicit_uidgid: true,
+            is_tarfs_mode: false,
         };
         let dict =
             HashChunkDict::from_commandline_arg(path, Arc::new(ConfigV2::default()), &rafs_config)

rafs/src/builder/core/context.rs

@@ -960,7 +960,7 @@ pub struct BootstrapContext {
     /// Cache node index for hardlinks, HashMap<(layer_index, real_inode, dev), Vec<index>>.
     pub(crate) inode_map: HashMap<(u16, Inode, u64), Vec<u64>>,
     /// Store all nodes in ascendant order, indexed by (node.index - 1).
-    pub nodes: Vec<Node>,
+    pub nodes: VecDeque<Node>,
     /// Current position to write in f_bootstrap
     pub(crate) offset: u64,
     pub(crate) writer: Box<dyn RafsIoWrite>,
@@ -979,7 +979,7 @@ impl BootstrapContext {
         Ok(Self {
             layered,
             inode_map: HashMap::new(),
-            nodes: Vec::new(),
+            nodes: VecDeque::new(),
             offset: EROFS_BLOCK_SIZE_4096,
             writer,
             v6_available_blocks: vec![

rafs/src/builder/core/layout.rs

@@ -3,6 +3,7 @@
 // SPDX-License-Identifier: Apache-2.0
 
 use anyhow::Result;
+use std::collections::VecDeque;
 
 use super::node::Node;
 use crate::builder::{Overlay, Prefetch};
@@ -11,7 +12,10 @@ use crate::builder::{Overlay, Prefetch};
 pub struct BlobLayout {}
 
 impl BlobLayout {
-    pub fn layout_blob_simple(prefetch: &Prefetch, nodes: &[Node]) -> Result<(Vec<usize>, usize)> {
+    pub fn layout_blob_simple(
+        prefetch: &Prefetch,
+        nodes: &VecDeque<Node>,
+    ) -> Result<(Vec<usize>, usize)> {
         let mut inodes = Vec::with_capacity(nodes.len());
 
         // Put all prefetch inodes at the head

rafs/src/builder/core/prefetch.rs

@@ -2,7 +2,7 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 
-use std::collections::BTreeMap;
+use std::collections::{BTreeMap, VecDeque};
 use std::path::PathBuf;
 use std::str::FromStr;
 
@@ -181,7 +181,7 @@ impl Prefetch {
     }
 
     /// Generate filesystem layer prefetch list for RAFS v5.
-    pub fn get_v5_prefetch_table(&mut self, nodes: &[Node]) -> Option<RafsV5PrefetchTable> {
+    pub fn get_v5_prefetch_table(&mut self, nodes: &VecDeque<Node>) -> Option<RafsV5PrefetchTable> {
         if self.policy == PrefetchPolicy::Fs {
             let mut prefetch_table = RafsV5PrefetchTable::new();
             for i in self.patterns.values().filter_map(|v| *v) {
@@ -199,7 +199,7 @@ impl Prefetch {
     /// Generate filesystem layer prefetch list for RAFS v6.
     pub fn get_v6_prefetch_table(
         &mut self,
-        nodes: &[Node],
+        nodes: &VecDeque<Node>,
         meta_addr: u64,
     ) -> Option<RafsV6PrefetchTable> {
         if self.policy == PrefetchPolicy::Fs {

rafs/src/builder/core/v6.rs

@@ -3,7 +3,7 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 
-use std::collections::BTreeMap;
+use std::collections::{BTreeMap, VecDeque};
 use std::ffi::{OsStr, OsString};
 use std::io::SeekFrom;
 use std::mem::size_of;
@@ -555,7 +555,7 @@ impl BuildContext {
 }
 
 impl Bootstrap {
-    pub(crate) fn v6_update_dirents(nodes: &mut Vec<Node>, tree: &Tree, parent_offset: u64) {
+    pub(crate) fn v6_update_dirents(nodes: &mut VecDeque<Node>, tree: &Tree, parent_offset: u64) {
         let node = &mut nodes[tree.node.index as usize - 1];
         let node_offset = node.v6_offset;
         if !node.is_dir() {

src/bin/nydus-image/merge.rs → rafs/src/builder/merge.rs

@@ -2,21 +2,22 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 
-use std::collections::HashMap;
-use std::collections::HashSet;
+use std::collections::hash_map::Entry;
+use std::collections::{HashMap, HashSet, VecDeque};
 use std::convert::TryFrom;
 use std::path::{Path, PathBuf};
 use std::sync::Arc;
 
-use anyhow::{Context, Result};
+use anyhow::{anyhow, bail, ensure, Context, Result};
 use hex::FromHex;
 use nydus_api::ConfigV2;
-use nydus_rafs::builder::{
+use nydus_storage::device::{BlobFeatures, BlobInfo};
+
+use super::{
     ArtifactStorage, BlobContext, BlobManager, Bootstrap, BootstrapContext, BuildContext,
-    BuildOutput, ChunkSource, HashChunkDict, MetadataTreeBuilder, Overlay, Tree, WhiteoutSpec,
+    BuildOutput, ChunkSource, ConversionType, MetadataTreeBuilder, Overlay, Tree, WhiteoutSpec,
 };
-use nydus_rafs::metadata::{RafsInodeExt, RafsSuper, RafsVersion};
-use nydus_storage::device::{BlobFeatures, BlobInfo};
+use crate::metadata::{RafsInodeExt, RafsSuper, RafsVersion};
 
 /// Struct to generate the merged RAFS bootstrap for an image from per layer RAFS bootstraps.
 ///
@@ -49,7 +50,7 @@ impl Merger {
         })
     }
 
-    /// Generate the merged RAFS bootstrap for an image from per layer RAFS bootstraps.
+    /// Overlay multiple RAFS filesystems into a merged RAFS filesystem.
     ///
     /// # Arguments
     /// - sources: contains one or more per layer bootstraps in order of lower to higher.
@@ -78,14 +79,6 @@ impl Merger {
                 sources.len(),
             );
         }
-        if let Some(toc_digests) = blob_toc_digests.as_ref() {
-            ensure!(
-                toc_digests.len() == sources.len(),
-                "number of toc digest entries {} doesn't match number of sources {}",
-                toc_digests.len(),
-                sources.len(),
-            );
-        }
         if let Some(sizes) = blob_sizes.as_ref() {
             ensure!(
                 sizes.len() == sources.len(),
@@ -94,6 +87,14 @@ impl Merger {
                 sources.len(),
             );
         }
+        if let Some(toc_digests) = blob_toc_digests.as_ref() {
+            ensure!(
+                toc_digests.len() == sources.len(),
+                "number of toc digest entries {} doesn't match number of sources {}",
+                toc_digests.len(),
+                sources.len(),
+            );
+        }
         if let Some(sizes) = blob_toc_sizes.as_ref() {
             ensure!(
                 sizes.len() == sources.len(),
@@ -105,10 +106,10 @@ impl Merger {
 
         let mut tree: Option<Tree> = None;
         let mut blob_mgr = BlobManager::new(ctx.digester);
-
-        // Load parent bootstrap
         let mut blob_idx_map = HashMap::new();
         let mut parent_layers = 0;
+
+        // Load parent bootstrap
         if let Some(parent_bootstrap_path) = &parent_bootstrap_path {
             let (rs, _) =
                 RafsSuper::load_from_file(parent_bootstrap_path, config_v2.clone(), false, false)
@@ -123,7 +124,7 @@ impl Merger {
             parent_layers = blobs.len();
         }
 
-        // Get the blobs come from chunk dict bootstrap.
+        // Get the blobs come from chunk dictionary.
         let mut chunk_dict_blobs = HashSet::new();
         let mut config = None;
         if let Some(chunk_dict_path) = &chunk_dict {
@@ -150,6 +151,10 @@ impl Merger {
             ctx.compressor = rs.meta.get_compressor();
             ctx.digester = rs.meta.get_digester();
             ctx.explicit_uidgid = rs.meta.explicit_uidgid();
+            if config.as_ref().unwrap().is_tarfs_mode {
+                ctx.conversion_type = ConversionType::TarToTarfs;
+                ctx.blob_features |= BlobFeatures::TARFS;
+            }
 
             let mut parent_blob_added = false;
             let blobs = &rs.superblock.get_blob_infos();
@@ -166,7 +171,7 @@ impl Merger {
                 } else {
                     chunk_size = Some(blob_ctx.chunk_size);
                 }
-                if chunk_dict_blobs.get(&blob.blob_id()).is_none() {
+                if !chunk_dict_blobs.contains(&blob.blob_id()) {
                     // It is assumed that the `nydus-image create` at each layer and `nydus-image merge` commands
                     // use the same chunk dict bootstrap. So the parent bootstrap includes multiple blobs, but
                     // only at most one new blob, the other blobs should be from the chunk dict image.
@@ -206,14 +211,14 @@ impl Merger {
                     }
                 }
 
-                if !blob_idx_map.contains_key(&blob.blob_id()) {
-                    blob_idx_map.insert(blob.blob_id().clone(), blob_mgr.len());
+                if let Entry::Vacant(e) = blob_idx_map.entry(blob.blob_id()) {
+                    e.insert(blob_mgr.len());
                     blob_mgr.add_blob(blob_ctx);
                 }
             }
 
             if let Some(tree) = &mut tree {
-                let mut nodes = Vec::new();
+                let mut nodes = VecDeque::new();
                 rs.walk_directory::<PathBuf>(
                     rs.superblock.root_ino(),
                     None,
@@ -234,17 +239,21 @@ impl Merger {
                         }
                         // Set node's layer index to distinguish same inode number (from bootstrap)
                         // between different layers.
-                        node.layer_idx = u16::try_from(layer_idx).context(format!(
+                        let idx = u16::try_from(layer_idx).context(format!(
                             "too many layers {}, limited to {}",
                             layer_idx,
                             u16::MAX
-                        ))? + parent_layers as u16;
+                        ))?;
+                        if parent_layers + idx as usize > u16::MAX as usize {
+                            bail!("too many layers {}, limited to {}", layer_idx, u16::MAX);
+                        }
+                        node.layer_idx = idx + parent_layers as u16;
                         node.overlay = Overlay::UpperAddition;
                         match node.whiteout_type(WhiteoutSpec::Oci) {
                             // Insert whiteouts at the head, so they will be handled first when
                             // applying to lower layer.
-                            Some(_) => nodes.insert(0, node),
-                            _ => nodes.push(node),
+                            Some(_) => nodes.push_front(node),
+                            _ => nodes.push_back(node),
                         }
                         Ok(())
                     },
@@ -253,8 +262,16 @@ impl Merger {
                     tree.apply(node, true, WhiteoutSpec::Oci)?;
                 }
             } else {
-                let mut dict = HashChunkDict::new(rs.meta.get_digester());
-                tree = Some(Tree::from_bootstrap(&rs, &mut dict)?);
+                tree = Some(Tree::from_bootstrap(&rs, &mut ())?);
+            }
+        }
+
+        if ctx.conversion_type == ConversionType::TarToTarfs {
+            if parent_layers > 0 {
+                bail!("merging RAFS in TARFS mode conflicts with `--parent-bootstrap`");
+            }
+            if !chunk_dict_blobs.is_empty() {
+                bail!("merging RAFS in TARFS mode conflicts with `--chunk-dict`");
             }
         }