diff --git a/rafs/src/builder/compact.rs b/rafs/src/builder/compact.rs index 1c6040e5c8a..11d30f42c39 100644 --- a/rafs/src/builder/compact.rs +++ b/rafs/src/builder/compact.rs @@ -3,7 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 use std::collections::hash_map::Entry; -use std::collections::HashMap; +use std::collections::{HashMap, VecDeque}; use std::io::Write; use std::ops::Deref; use std::path::PathBuf; @@ -252,7 +252,7 @@ pub struct BlobCompactor { /// new blobs new_blob_mgr: BlobManager, /// inode list - nodes: Vec, + nodes: VecDeque, /// chunk --> list c2nodes: HashMap>, /// original blob index --> list @@ -265,7 +265,7 @@ impl BlobCompactor { pub fn new( version: RafsVersion, ori_blob_mgr: BlobManager, - nodes: Vec, + nodes: VecDeque, backend: Arc, digester: digest::Algorithm, ) -> Result { @@ -596,7 +596,7 @@ impl BlobCompactor { let tree = Tree::from_bootstrap(&rs, &mut _dict)?; let mut bootstrap = Bootstrap::new()?; bootstrap.build(&mut build_ctx, &mut bootstrap_ctx, tree)?; - let mut nodes = Vec::new(); + let mut nodes = VecDeque::new(); // move out nodes std::mem::swap(&mut bootstrap_ctx.nodes, &mut nodes); let mut compactor = Self::new( diff --git a/rafs/src/builder/core/blob.rs b/rafs/src/builder/core/blob.rs index 5623aaeac78..bf08add4d1a 100644 --- a/rafs/src/builder/core/blob.rs +++ b/rafs/src/builder/core/blob.rs @@ -3,6 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 use std::borrow::Cow; +use std::collections::VecDeque; use std::io::Write; use std::slice; @@ -27,7 +28,7 @@ impl Blob { /// Dump blob file and generate chunks pub(crate) fn dump( ctx: &BuildContext, - nodes: &mut [Node], + nodes: &mut VecDeque, blob_mgr: &mut BlobManager, blob_writer: &mut ArtifactWriter, ) -> Result<()> { diff --git a/rafs/src/builder/core/bootstrap.rs b/rafs/src/builder/core/bootstrap.rs index 7e873adcc94..12458046862 100644 --- a/rafs/src/builder/core/bootstrap.rs +++ b/rafs/src/builder/core/bootstrap.rs @@ -3,6 +3,7 @@ // // SPDX-License-Identifier: Apache-2.0 +use std::collections::VecDeque; use std::ffi::OsString; use anyhow::{Context, Error, Result}; @@ -15,7 +16,7 @@ use crate::builder::{ ArtifactStorage, BlobManager, BootstrapContext, BootstrapManager, BuildContext, Tree, }; use crate::metadata::layout::{RafsBlobTable, RAFS_V5_ROOT_INODE}; -use crate::metadata::{RafsSuper, RafsSuperConfig}; +use crate::metadata::{RafsSuper, RafsSuperConfig, RafsSuperFlags}; pub(crate) const STARGZ_DEFAULT_BLOCK_SIZE: u32 = 4 << 20; @@ -52,7 +53,7 @@ impl Bootstrap { ) -> Result<()> { // used to compute nid(ino) for v6 let root_offset = bootstrap_ctx.offset; - let mut nodes = Vec::with_capacity(0x10000); + let mut nodes = VecDeque::with_capacity(0x10000); // Special handling of the root inode assert!(tree.node.is_dir()); @@ -63,7 +64,7 @@ impl Bootstrap { tree.node.inode.set_ino(RAFS_V5_ROOT_INODE); } ctx.prefetch.insert_if_need(&tree.node); - nodes.push(tree.node.clone()); + nodes.push_back(tree.node.clone()); Self::build_rafs(ctx, bootstrap_ctx, &mut tree, &mut nodes)?; if ctx.fs_version.is_v6() && !bootstrap_ctx.layered { @@ -147,7 +148,7 @@ impl Bootstrap { ctx: &mut BuildContext, bootstrap_ctx: &mut BootstrapContext, tree: &mut Tree, - nodes: &mut Vec, + nodes: &mut VecDeque, ) -> Result<()> { let index = nodes.len() as u32 + 1; let parent = &mut nodes[tree.node.index as usize - 1]; @@ -234,14 +235,14 @@ impl Bootstrap { (true, Some(whiteout_type)) => { // Insert removal operations at the head, so they will be handled first when // applying to lower layer. - nodes.insert(0, child.node.clone()); + nodes.push_front(child.node.clone()); if whiteout_type == WhiteoutType::OverlayFsOpaque { // For the overlayfs opaque, we need to remove the lower node that has the // same name first, then apply upper node to the node tree of lower layer. child .node .remove_xattr(&OsString::from(OVERLAYFS_WHITEOUT_OPAQUE)); - nodes.push(child.node.clone()); + nodes.push_back(child.node.clone()); } } (false, Some(whiteout_type)) => { @@ -251,9 +252,9 @@ impl Bootstrap { .node .remove_xattr(&OsString::from(OVERLAYFS_WHITEOUT_OPAQUE)); } - nodes.push(child.node.clone()); + nodes.push_back(child.node.clone()); } - _ => nodes.push(child.node.clone()), + _ => nodes.push_back(child.node.clone()), } ctx.prefetch.insert_if_need(&child.node); @@ -298,6 +299,7 @@ impl Bootstrap { chunk_size: ctx.chunk_size, explicit_uidgid: ctx.explicit_uidgid, version: ctx.fs_version, + is_tarfs_mode: rs.meta.flags.contains(RafsSuperFlags::TARTFS_MODE), }; config.check_compatibility(&rs.meta)?; diff --git a/rafs/src/builder/core/chunk_dict.rs b/rafs/src/builder/core/chunk_dict.rs index 06f9e1a3a17..b83cb9f2d7d 100644 --- a/rafs/src/builder/core/chunk_dict.rs +++ b/rafs/src/builder/core/chunk_dict.rs @@ -264,6 +264,7 @@ mod tests { digester: digest::Algorithm::Blake3, chunk_size: 0x100000, explicit_uidgid: true, + is_tarfs_mode: false, }; let dict = HashChunkDict::from_commandline_arg(path, Arc::new(ConfigV2::default()), &rafs_config) diff --git a/rafs/src/builder/core/context.rs b/rafs/src/builder/core/context.rs index a9c26da1b89..9c427db450e 100644 --- a/rafs/src/builder/core/context.rs +++ b/rafs/src/builder/core/context.rs @@ -960,7 +960,7 @@ pub struct BootstrapContext { /// Cache node index for hardlinks, HashMap<(layer_index, real_inode, dev), Vec>. pub(crate) inode_map: HashMap<(u16, Inode, u64), Vec>, /// Store all nodes in ascendant order, indexed by (node.index - 1). - pub nodes: Vec, + pub nodes: VecDeque, /// Current position to write in f_bootstrap pub(crate) offset: u64, pub(crate) writer: Box, @@ -979,7 +979,7 @@ impl BootstrapContext { Ok(Self { layered, inode_map: HashMap::new(), - nodes: Vec::new(), + nodes: VecDeque::new(), offset: EROFS_BLOCK_SIZE_4096, writer, v6_available_blocks: vec![ diff --git a/rafs/src/builder/core/layout.rs b/rafs/src/builder/core/layout.rs index 1d0346298c4..f3a53258236 100644 --- a/rafs/src/builder/core/layout.rs +++ b/rafs/src/builder/core/layout.rs @@ -3,6 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 use anyhow::Result; +use std::collections::VecDeque; use super::node::Node; use crate::builder::{Overlay, Prefetch}; @@ -11,7 +12,10 @@ use crate::builder::{Overlay, Prefetch}; pub struct BlobLayout {} impl BlobLayout { - pub fn layout_blob_simple(prefetch: &Prefetch, nodes: &[Node]) -> Result<(Vec, usize)> { + pub fn layout_blob_simple( + prefetch: &Prefetch, + nodes: &VecDeque, + ) -> Result<(Vec, usize)> { let mut inodes = Vec::with_capacity(nodes.len()); // Put all prefetch inodes at the head diff --git a/rafs/src/builder/core/prefetch.rs b/rafs/src/builder/core/prefetch.rs index 4391a74e527..d4855f9fc93 100644 --- a/rafs/src/builder/core/prefetch.rs +++ b/rafs/src/builder/core/prefetch.rs @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: Apache-2.0 -use std::collections::BTreeMap; +use std::collections::{BTreeMap, VecDeque}; use std::path::PathBuf; use std::str::FromStr; @@ -181,7 +181,7 @@ impl Prefetch { } /// Generate filesystem layer prefetch list for RAFS v5. - pub fn get_v5_prefetch_table(&mut self, nodes: &[Node]) -> Option { + pub fn get_v5_prefetch_table(&mut self, nodes: &VecDeque) -> Option { if self.policy == PrefetchPolicy::Fs { let mut prefetch_table = RafsV5PrefetchTable::new(); for i in self.patterns.values().filter_map(|v| *v) { @@ -199,7 +199,7 @@ impl Prefetch { /// Generate filesystem layer prefetch list for RAFS v6. pub fn get_v6_prefetch_table( &mut self, - nodes: &[Node], + nodes: &VecDeque, meta_addr: u64, ) -> Option { if self.policy == PrefetchPolicy::Fs { diff --git a/rafs/src/builder/core/v6.rs b/rafs/src/builder/core/v6.rs index d485ec3c79a..793865570cc 100644 --- a/rafs/src/builder/core/v6.rs +++ b/rafs/src/builder/core/v6.rs @@ -3,7 +3,7 @@ // // SPDX-License-Identifier: Apache-2.0 -use std::collections::BTreeMap; +use std::collections::{BTreeMap, VecDeque}; use std::ffi::{OsStr, OsString}; use std::io::SeekFrom; use std::mem::size_of; @@ -555,7 +555,7 @@ impl BuildContext { } impl Bootstrap { - pub(crate) fn v6_update_dirents(nodes: &mut Vec, tree: &Tree, parent_offset: u64) { + pub(crate) fn v6_update_dirents(nodes: &mut VecDeque, tree: &Tree, parent_offset: u64) { let node = &mut nodes[tree.node.index as usize - 1]; let node_offset = node.v6_offset; if !node.is_dir() { diff --git a/src/bin/nydus-image/merge.rs b/rafs/src/builder/merge.rs similarity index 86% rename from src/bin/nydus-image/merge.rs rename to rafs/src/builder/merge.rs index 96def7d27c5..7cc037f8c49 100644 --- a/src/bin/nydus-image/merge.rs +++ b/rafs/src/builder/merge.rs @@ -2,21 +2,22 @@ // // SPDX-License-Identifier: Apache-2.0 -use std::collections::HashMap; -use std::collections::HashSet; +use std::collections::hash_map::Entry; +use std::collections::{HashMap, HashSet, VecDeque}; use std::convert::TryFrom; use std::path::{Path, PathBuf}; use std::sync::Arc; -use anyhow::{Context, Result}; +use anyhow::{anyhow, bail, ensure, Context, Result}; use hex::FromHex; use nydus_api::ConfigV2; -use nydus_rafs::builder::{ +use nydus_storage::device::{BlobFeatures, BlobInfo}; + +use super::{ ArtifactStorage, BlobContext, BlobManager, Bootstrap, BootstrapContext, BuildContext, - BuildOutput, ChunkSource, HashChunkDict, MetadataTreeBuilder, Overlay, Tree, WhiteoutSpec, + BuildOutput, ChunkSource, ConversionType, MetadataTreeBuilder, Overlay, Tree, WhiteoutSpec, }; -use nydus_rafs::metadata::{RafsInodeExt, RafsSuper, RafsVersion}; -use nydus_storage::device::{BlobFeatures, BlobInfo}; +use crate::metadata::{RafsInodeExt, RafsSuper, RafsVersion}; /// Struct to generate the merged RAFS bootstrap for an image from per layer RAFS bootstraps. /// @@ -49,7 +50,7 @@ impl Merger { }) } - /// Generate the merged RAFS bootstrap for an image from per layer RAFS bootstraps. + /// Overlay multiple RAFS filesystems into a merged RAFS filesystem. /// /// # Arguments /// - sources: contains one or more per layer bootstraps in order of lower to higher. @@ -78,14 +79,6 @@ impl Merger { sources.len(), ); } - if let Some(toc_digests) = blob_toc_digests.as_ref() { - ensure!( - toc_digests.len() == sources.len(), - "number of toc digest entries {} doesn't match number of sources {}", - toc_digests.len(), - sources.len(), - ); - } if let Some(sizes) = blob_sizes.as_ref() { ensure!( sizes.len() == sources.len(), @@ -94,6 +87,14 @@ impl Merger { sources.len(), ); } + if let Some(toc_digests) = blob_toc_digests.as_ref() { + ensure!( + toc_digests.len() == sources.len(), + "number of toc digest entries {} doesn't match number of sources {}", + toc_digests.len(), + sources.len(), + ); + } if let Some(sizes) = blob_toc_sizes.as_ref() { ensure!( sizes.len() == sources.len(), @@ -105,10 +106,10 @@ impl Merger { let mut tree: Option = None; let mut blob_mgr = BlobManager::new(ctx.digester); - - // Load parent bootstrap let mut blob_idx_map = HashMap::new(); let mut parent_layers = 0; + + // Load parent bootstrap if let Some(parent_bootstrap_path) = &parent_bootstrap_path { let (rs, _) = RafsSuper::load_from_file(parent_bootstrap_path, config_v2.clone(), false, false) @@ -123,7 +124,7 @@ impl Merger { parent_layers = blobs.len(); } - // Get the blobs come from chunk dict bootstrap. + // Get the blobs come from chunk dictionary. let mut chunk_dict_blobs = HashSet::new(); let mut config = None; if let Some(chunk_dict_path) = &chunk_dict { @@ -150,6 +151,10 @@ impl Merger { ctx.compressor = rs.meta.get_compressor(); ctx.digester = rs.meta.get_digester(); ctx.explicit_uidgid = rs.meta.explicit_uidgid(); + if config.as_ref().unwrap().is_tarfs_mode { + ctx.conversion_type = ConversionType::TarToTarfs; + ctx.blob_features |= BlobFeatures::TARFS; + } let mut parent_blob_added = false; let blobs = &rs.superblock.get_blob_infos(); @@ -166,7 +171,7 @@ impl Merger { } else { chunk_size = Some(blob_ctx.chunk_size); } - if chunk_dict_blobs.get(&blob.blob_id()).is_none() { + if !chunk_dict_blobs.contains(&blob.blob_id()) { // It is assumed that the `nydus-image create` at each layer and `nydus-image merge` commands // use the same chunk dict bootstrap. So the parent bootstrap includes multiple blobs, but // only at most one new blob, the other blobs should be from the chunk dict image. @@ -206,14 +211,14 @@ impl Merger { } } - if !blob_idx_map.contains_key(&blob.blob_id()) { - blob_idx_map.insert(blob.blob_id().clone(), blob_mgr.len()); + if let Entry::Vacant(e) = blob_idx_map.entry(blob.blob_id()) { + e.insert(blob_mgr.len()); blob_mgr.add_blob(blob_ctx); } } if let Some(tree) = &mut tree { - let mut nodes = Vec::new(); + let mut nodes = VecDeque::new(); rs.walk_directory::( rs.superblock.root_ino(), None, @@ -234,17 +239,21 @@ impl Merger { } // Set node's layer index to distinguish same inode number (from bootstrap) // between different layers. - node.layer_idx = u16::try_from(layer_idx).context(format!( + let idx = u16::try_from(layer_idx).context(format!( "too many layers {}, limited to {}", layer_idx, u16::MAX - ))? + parent_layers as u16; + ))?; + if parent_layers + idx as usize > u16::MAX as usize { + bail!("too many layers {}, limited to {}", layer_idx, u16::MAX); + } + node.layer_idx = idx + parent_layers as u16; node.overlay = Overlay::UpperAddition; match node.whiteout_type(WhiteoutSpec::Oci) { // Insert whiteouts at the head, so they will be handled first when // applying to lower layer. - Some(_) => nodes.insert(0, node), - _ => nodes.push(node), + Some(_) => nodes.push_front(node), + _ => nodes.push_back(node), } Ok(()) }, @@ -253,8 +262,16 @@ impl Merger { tree.apply(node, true, WhiteoutSpec::Oci)?; } } else { - let mut dict = HashChunkDict::new(rs.meta.get_digester()); - tree = Some(Tree::from_bootstrap(&rs, &mut dict)?); + tree = Some(Tree::from_bootstrap(&rs, &mut ())?); + } + } + + if ctx.conversion_type == ConversionType::TarToTarfs { + if parent_layers > 0 { + bail!("merging RAFS in TARFS mode conflicts with `--parent-bootstrap`"); + } + if !chunk_dict_blobs.is_empty() { + bail!("merging RAFS in TARFS mode conflicts with `--chunk-dict`"); } } diff --git a/rafs/src/builder/mod.rs b/rafs/src/builder/mod.rs index d6b901187f8..f5327f587b9 100644 --- a/rafs/src/builder/mod.rs +++ b/rafs/src/builder/mod.rs @@ -22,12 +22,14 @@ pub use self::core::overlay::{Overlay, WhiteoutSpec}; pub use self::core::prefetch::{Prefetch, PrefetchPolicy}; pub use self::core::tree::{MetadataTreeBuilder, Tree}; pub use self::directory::DirectoryBuilder; +pub use self::merge::Merger; pub use self::stargz::StargzBuilder; pub use self::tarball::TarballBuilder; -pub mod compact; +mod compact; mod core; mod directory; +mod merge; mod stargz; mod tarball; diff --git a/rafs/src/metadata/chunk.rs b/rafs/src/metadata/chunk.rs index 35e2a644356..88fe42dd34d 100644 --- a/rafs/src/metadata/chunk.rs +++ b/rafs/src/metadata/chunk.rs @@ -15,7 +15,7 @@ use nydus_utils::digest::RafsDigest; use crate::metadata::cached_v5::CachedChunkInfoV5; use crate::metadata::direct_v5::DirectChunkInfoV5; -use crate::metadata::direct_v6::DirectChunkInfoV6; +use crate::metadata::direct_v6::{DirectChunkInfoV6, TarfsChunkInfo}; use crate::metadata::layout::v5::RafsV5ChunkInfo; use crate::metadata::{RafsStore, RafsVersion}; use crate::RafsIoWrite; @@ -331,10 +331,12 @@ impl ChunkWrapper { *self = Self::V6(to_rafs_v5_chunk_info(cki_v6)); } else if let Some(cki_v6) = cki.as_any().downcast_ref::() { *self = Self::V6(to_rafs_v5_chunk_info(cki_v6)); + } else if let Some(cki_v6) = cki.as_any().downcast_ref::() { + *self = Self::V6(to_rafs_v5_chunk_info(cki_v6)); } else if let Some(cki_v5) = cki.as_any().downcast_ref::() { - *self = Self::V6(to_rafs_v5_chunk_info(cki_v5)); + *self = Self::V5(to_rafs_v5_chunk_info(cki_v5)); } else if let Some(cki_v5) = cki.as_any().downcast_ref::() { - *self = Self::V6(to_rafs_v5_chunk_info(cki_v5)); + *self = Self::V5(to_rafs_v5_chunk_info(cki_v5)); } else { panic!("unknown chunk information struct"); } @@ -347,6 +349,8 @@ fn as_blob_v5_chunk_info(cki: &dyn BlobChunkInfo) -> &dyn BlobV5ChunkInfo { cki_v6 } else if let Some(cki_v6) = cki.as_any().downcast_ref::() { cki_v6 + } else if let Some(cki_v6) = cki.as_any().downcast_ref::() { + cki_v6 } else if let Some(cki_v5) = cki.as_any().downcast_ref::() { cki_v5 } else if let Some(cki_v5) = cki.as_any().downcast_ref::() { diff --git a/rafs/src/metadata/direct_v6.rs b/rafs/src/metadata/direct_v6.rs index 72cb2eff836..45ce48e22ca 100644 --- a/rafs/src/metadata/direct_v6.rs +++ b/rafs/src/metadata/direct_v6.rs @@ -1466,3 +1466,21 @@ impl BlobChunkInfo for TarfsChunkInfo { self } } + +impl BlobV5ChunkInfo for TarfsChunkInfo { + fn index(&self) -> u32 { + self.chunk_index + } + + fn file_offset(&self) -> u64 { + 0 + } + + fn flags(&self) -> BlobChunkFlags { + BlobChunkFlags::empty() + } + + fn as_base(&self) -> &dyn BlobChunkInfo { + self + } +} diff --git a/rafs/src/metadata/mod.rs b/rafs/src/metadata/mod.rs index a8ffb02bba1..dd9f0006b61 100644 --- a/rafs/src/metadata/mod.rs +++ b/rafs/src/metadata/mod.rs @@ -369,6 +369,8 @@ pub struct RafsSuperConfig { pub chunk_size: u32, /// Whether `explicit_uidgid` enabled or not. pub explicit_uidgid: bool, + /// RAFS in TARFS mode. + pub is_tarfs_mode: bool, } impl RafsSuperConfig { @@ -405,6 +407,11 @@ impl RafsSuperConfig { ))); } + let is_tarfs_mode = meta.flags.contains(RafsSuperFlags::TARTFS_MODE); + if is_tarfs_mode != self.is_tarfs_mode { + return Err(einval!(format!("Using inconsistent RAFS TARFS mode"))); + } + Ok(()) } } @@ -519,6 +526,7 @@ impl RafsSuperMeta { digester: self.get_digester(), chunk_size: self.chunk_size, explicit_uidgid: self.explicit_uidgid(), + is_tarfs_mode: self.flags.contains(RafsSuperFlags::TARTFS_MODE), } } } diff --git a/src/bin/nydus-image/main.rs b/src/bin/nydus-image/main.rs index 053fad79277..6b4cc4a6ae3 100644 --- a/src/bin/nydus-image/main.rs +++ b/src/bin/nydus-image/main.rs @@ -30,7 +30,7 @@ use nydus_app::setup_logging; use nydus_rafs::builder::{ parse_chunk_dict_arg, ArtifactStorage, BlobCompactor, BlobManager, BootstrapManager, BuildContext, BuildOutput, Builder, ConversionType, DirectoryBuilder, Feature, Features, - HashChunkDict, Prefetch, PrefetchPolicy, StargzBuilder, TarballBuilder, WhiteoutSpec, + HashChunkDict, Merger, Prefetch, PrefetchPolicy, StargzBuilder, TarballBuilder, WhiteoutSpec, }; use nydus_rafs::metadata::{RafsSuper, RafsSuperConfig, RafsVersion}; use nydus_storage::backend::localfs::LocalFs; @@ -43,12 +43,10 @@ use nydus_utils::trace::{EventTracerClass, TimingTracerClass, TraceClass}; use nydus_utils::{compress, digest, event_tracer, register_tracer, root_tracer, timing_tracer}; use serde::{Deserialize, Serialize}; -use crate::merge::Merger; use crate::unpack::{OCIUnpacker, Unpacker}; use crate::validator::Validator; mod inspect; -mod merge; mod stat; mod unpack; mod validator; @@ -885,6 +883,7 @@ impl Command { digester, chunk_size, explicit_uidgid: !repeatable, + is_tarfs_mode: false, }; let rafs_config = Arc::new(build_ctx.configuration.as_ref().clone()); // The separate chunk dict bootstrap doesn't support blob accessible.