Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Implement Tar Global Extended Attributes API changes #70869

Merged
merged 8 commits into from
Jun 22, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions src/libraries/System.Formats.Tar/ref/System.Formats.Tar.cs
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,11 @@ public GnuTarEntry(System.Formats.Tar.TarEntryType entryType, string entryName)
public System.DateTimeOffset AccessTime { get { throw null; } set { } }
public System.DateTimeOffset ChangeTime { get { throw null; } set { } }
}
public sealed partial class PaxGlobalExtendedAttributesTarEntry : System.Formats.Tar.PosixTarEntry
{
public PaxGlobalExtendedAttributesTarEntry(System.Collections.Generic.IEnumerable<System.Collections.Generic.KeyValuePair<string, string>> globalExtendedAttributes) { }
carlossanlop marked this conversation as resolved.
Show resolved Hide resolved
public System.Collections.Generic.IReadOnlyDictionary<string, string> GlobalExtendedAttributes { get { throw null; } }
}
public sealed partial class PaxTarEntry : System.Formats.Tar.PosixTarEntry
{
public PaxTarEntry(System.Formats.Tar.TarEntry other) { }
Expand Down Expand Up @@ -101,14 +106,13 @@ public enum TarFileMode
public sealed partial class TarReader : System.IDisposable
{
public TarReader(System.IO.Stream archiveStream, bool leaveOpen = false) { }
public System.Collections.Generic.IReadOnlyDictionary<string, string>? GlobalExtendedAttributes { get { throw null; } }
public void Dispose() { }
public System.Formats.Tar.TarEntry? GetNextEntry(bool copyData = false) { throw null; }
}
public sealed partial class TarWriter : System.IDisposable
{
public TarWriter(System.IO.Stream archiveStream) { }
public TarWriter(System.IO.Stream archiveStream, System.Collections.Generic.IEnumerable<System.Collections.Generic.KeyValuePair<string, string>>? globalExtendedAttributes = null, bool leaveOpen = false) { }
public TarWriter(System.IO.Stream archiveStream, bool leaveOpen = false) { }
public TarWriter(System.IO.Stream archiveStream, System.Formats.Tar.TarEntryFormat format = System.Formats.Tar.TarEntryFormat.Pax, bool leaveOpen = false) { }
public System.Formats.Tar.TarEntryFormat Format { get { throw null; } }
public void Dispose() { }
Expand Down
9 changes: 3 additions & 6 deletions src/libraries/System.Formats.Tar/src/Resources/Strings.resx
Original file line number Diff line number Diff line change
Expand Up @@ -186,12 +186,12 @@
<data name="SetLengthRequiresSeekingAndWriting" xml:space="preserve">
<value>SetLength requires a stream that supports seeking and writing.</value>
</data>
<data name="TarCannotConvertPaxGlobalExtendedAttributesEntry" xml:space="preserve">
<value>Cannot convert a PaxGlobalExtendedAttributesEntry into another format.</value>
</data>
<data name="TarDuplicateExtendedAttribute" xml:space="preserve">
<value>The entry '{0}' has a duplicate extended attribute.</value>
</data>
<data name="TarEntriesInDifferentFormats" xml:space="preserve">
<value>An entry in '{0}' format was found in an archive where other entries of format '{1}' have been found.</value>
</data>
<data name="TarEntryBlockOrCharacterExpected" xml:space="preserve">
<value>Cannot set the 'DeviceMajor' or 'DeviceMinor' fields on an entry that does not represent a block or character device.</value>
</data>
Expand Down Expand Up @@ -240,9 +240,6 @@
<data name="TarSymbolicLinkTargetNotExists" xml:space="preserve">
<value>Cannot create the symbolic link '{0}' because the specified target '{1}' does not exist.</value>
</data>
<data name="TarTooManyGlobalExtendedAttributesEntries" xml:space="preserve">
<value>The archive has more than one global extended attributes entry.</value>
</data>
<data name="TarUnexpectedMetadataEntry" xml:space="preserve">
<value>A metadata entry of type '{0}' was unexpectedly found after a metadata entry of type '{1}'.</value>
</data>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
<Compile Include="System\Formats\Tar\TarEntryFormat.cs" />
<Compile Include="System\Formats\Tar\UstarTarEntry.cs" />
<Compile Include="System\Formats\Tar\GnuTarEntry.cs" />
<Compile Include="System\Formats\Tar\PaxGlobalExtendedAttributesTarEntry.cs" />
<Compile Include="System\Formats\Tar\PaxTarEntry.cs" />
<Compile Include="System\Formats\Tar\TarEntryType.cs" />
<Compile Include="System\Formats\Tar\TarFile.cs" />
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ internal GnuTarEntry(TarHeader header, TarReader readerOfOrigin)
/// </list>
/// </remarks>
public GnuTarEntry(TarEntryType entryType, string entryName)
: base(entryType, entryName, TarEntryFormat.Gnu)
: base(entryType, entryName, TarEntryFormat.Gnu, isGea: false)
{
_header._aTime = _header._mTime; // mtime was set in base constructor
_header._cTime = _header._mTime;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.

using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.Diagnostics;

namespace System.Formats.Tar
{
/// <summary>
/// Represents a Global Extended Attributes TAR entry from an archive of the PAX format.
/// </summary>
public sealed class PaxGlobalExtendedAttributesTarEntry : PosixTarEntry
{
private ReadOnlyDictionary<string, string>? _readOnlyGlobalExtendedAttributes;

// Constructor used when reading an existing archive.
internal PaxGlobalExtendedAttributesTarEntry(TarHeader header, TarReader readerOfOrigin)
: base(header, readerOfOrigin, TarEntryFormat.Pax)
{
}

/// <summary>
/// Initializes a new <see cref="PaxGlobalExtendedAttributesTarEntry"/> instance with the specified Global Extended Attributes enumeration.
/// </summary>
/// <param name="globalExtendedAttributes">An enumeration of string key-value pairs that represents the metadata to include as Global Extended Attributes.</param>
/// <exception cref="ArgumentNullException"><paramref name="globalExtendedAttributes"/> is <see langword="null"/>.</exception>
public PaxGlobalExtendedAttributesTarEntry(IEnumerable<KeyValuePair<string, string>> globalExtendedAttributes)
: base(TarEntryType.GlobalExtendedAttributes, TarHeader.GlobalHeadFormatPrefix, TarEntryFormat.Pax, isGea: true)
{
ArgumentNullException.ThrowIfNull(globalExtendedAttributes);
_header._extendedAttributes = new Dictionary<string, string>(globalExtendedAttributes);
}

/// <summary>
/// Returns the global extended attributes stored in this entry.
/// </summary>
public IReadOnlyDictionary<string, string> GlobalExtendedAttributes
{
get
{
_header._extendedAttributes ??= new Dictionary<string, string>();
return _readOnlyGlobalExtendedAttributes ??= _header._extendedAttributes.AsReadOnly();
}
}

// Determines if the current instance's entry type supports setting a data stream.
internal override bool IsDataStreamSetterSupported() => false;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ internal PaxTarEntry(TarHeader header, TarReader readerOfOrigin)
/// </list>
/// </remarks>
public PaxTarEntry(TarEntryType entryType, string entryName)
: base(entryType, entryName, TarEntryFormat.Pax)
: base(entryType, entryName, TarEntryFormat.Pax, isGea: false)
{
_header._prefix = string.Empty;
_header._extendedAttributes = new Dictionary<string, string>();
Expand Down Expand Up @@ -87,7 +87,7 @@ public PaxTarEntry(TarEntryType entryType, string entryName)
/// </list>
/// </remarks>
public PaxTarEntry(TarEntryType entryType, string entryName, IEnumerable<KeyValuePair<string, string>> extendedAttributes)
: base(entryType, entryName, TarEntryFormat.Pax)
: base(entryType, entryName, TarEntryFormat.Pax, isGea: false)
{
ArgumentNullException.ThrowIfNull(extendedAttributes);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,8 @@ internal PosixTarEntry(TarHeader header, TarReader readerOfOrigin, TarEntryForma
}

// Constructor called when the user creates a TarEntry instance from scratch.
internal PosixTarEntry(TarEntryType entryType, string entryName, TarEntryFormat format)
: base(entryType, entryName, format)
internal PosixTarEntry(TarEntryType entryType, string entryName, TarEntryFormat format, bool isGea)
: base(entryType, entryName, format, isGea)
{
_header._uName = string.Empty;
_header._gName = string.Empty;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,10 +29,16 @@ internal TarEntry(TarHeader header, TarReader readerOfOrigin, TarEntryFormat for
}

// Constructor called when the user creates a TarEntry instance from scratch.
internal TarEntry(TarEntryType entryType, string entryName, TarEntryFormat format)
internal TarEntry(TarEntryType entryType, string entryName, TarEntryFormat format, bool isGea)
{
ArgumentException.ThrowIfNullOrEmpty(entryName);
TarHelpers.ThrowIfEntryTypeNotSupported(entryType, format);

Debug.Assert(!isGea || entryType is TarEntryType.GlobalExtendedAttributes);

if (!isGea)
{
TarHelpers.ThrowIfEntryTypeNotSupported(entryType, format);
}

_header = default;
_header._format = format;
Expand All @@ -48,6 +54,11 @@ internal TarEntry(TarEntryType entryType, string entryName, TarEntryFormat forma
// Constructor called when converting an entry to the selected format.
internal TarEntry(TarEntry other, TarEntryFormat format)
{
if (other is PaxGlobalExtendedAttributesTarEntry)
{
throw new InvalidOperationException(SR.TarCannotConvertPaxGlobalExtendedAttributesEntry);
}

TarEntryType compatibleEntryType;
if (other.Format is TarEntryFormat.V7 && other.EntryType is TarEntryType.V7RegularFile && format is TarEntryFormat.Ustar or TarEntryFormat.Pax or TarEntryFormat.Gnu)
{
Expand Down Expand Up @@ -208,7 +219,7 @@ public int Uid
/// <exception cref="UnauthorizedAccessException">Operation not permitted due to insufficient permissions.</exception>
public void ExtractToFile(string destinationFileName, bool overwrite)
{
if (EntryType is TarEntryType.SymbolicLink or TarEntryType.HardLink)
if (EntryType is TarEntryType.SymbolicLink or TarEntryType.HardLink or TarEntryType.GlobalExtendedAttributes)
{
throw new InvalidOperationException(string.Format(SR.TarEntryTypeNotSupportedForExtracting, EntryType));
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -255,7 +255,10 @@ private static void ExtractToDirectoryInternal(Stream source, string destination
TarEntry? entry;
while ((entry = reader.GetNextEntry()) != null)
{
entry.ExtractRelativeToDirectory(destinationDirectoryPath, overwriteFiles);
if (entry is not PaxGlobalExtendedAttributesTarEntry)
{
entry.ExtractRelativeToDirectory(destinationDirectoryPath, overwriteFiles);
}
}
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,28 +22,9 @@ internal partial struct TarHeader
// "{dirName}/PaxHeaders.{processId}/{fileName}{trailingSeparator}"
private const string PaxHeadersFormat = "{0}/PaxHeaders.{1}/{2}{3}";

// Global Extended Attribute entries have a special format in the Name field:
// "{tmpFolder}/GlobalHead.{processId}.1"
private const string GlobalHeadFormat = "{0}/GlobalHead.{1}.1";

// Predefined text for the Name field of a GNU long metadata entry. Applies for both LongPath ('L') and LongLink ('K').
private const string GnuLongMetadataName = "././@LongLink";

// Creates a PAX Global Extended Attributes header and writes it into the specified archive stream.
internal static void WriteGlobalExtendedAttributesHeader(Stream archiveStream, Span<byte> buffer, IEnumerable<KeyValuePair<string, string>> globalExtendedAttributes)
{
TarHeader geaHeader = default;
geaHeader._name = GenerateGlobalExtendedAttributeName();
geaHeader._mode = (int)TarHelpers.DefaultMode;
geaHeader._typeFlag = TarEntryType.GlobalExtendedAttributes;
geaHeader._linkName = string.Empty;
geaHeader._magic = string.Empty;
geaHeader._version = string.Empty;
geaHeader._gName = string.Empty;
geaHeader._uName = string.Empty;
geaHeader.WriteAsPaxExtendedAttributes(archiveStream, buffer, globalExtendedAttributes, isGea: true);
}

// Writes the current header as a V7 entry into the archive stream.
internal void WriteAsV7(Stream archiveStream, Span<byte> buffer)
{
Expand Down Expand Up @@ -82,14 +63,27 @@ internal void WriteAsUstar(Stream archiveStream, Span<byte> buffer)
}
}

// Writes the current header as a PAX Global Extended Attributes entry into the archive stream.
internal void WriteAsPaxGlobalExtendedAttributes(Stream archiveStream, Span<byte> buffer, int globalExtendedAttributesEntryNumber)
{
Debug.Assert(_typeFlag is TarEntryType.GlobalExtendedAttributes);

_name = GenerateGlobalExtendedAttributeName(globalExtendedAttributesEntryNumber);
_extendedAttributes ??= new Dictionary<string, string>();
WriteAsPaxExtendedAttributes(archiveStream, buffer, _extendedAttributes, isGea: true);
}

// Writes the current header as a PAX entry into the archive stream.
// Makes sure to add the preceding exteded attributes entry before the actual entry.
// Makes sure to add the preceding extended attributes entry before the actual entry.
internal void WriteAsPax(Stream archiveStream, Span<byte> buffer)
{
Debug.Assert(_typeFlag is not TarEntryType.GlobalExtendedAttributes);

// First, we write the preceding extended attributes header
TarHeader extendedAttributesHeader = default;
// Fill the current header's dict
CollectExtendedAttributesFromStandardFieldsIfNeeded();
// And pass the attributes to the preceding extended attributes header for writing
Debug.Assert(_extendedAttributes != null);
extendedAttributesHeader.WriteAsPaxExtendedAttributes(archiveStream, buffer, _extendedAttributes, isGea: false);

Expand Down Expand Up @@ -611,30 +605,30 @@ private string GenerateExtendedAttributeName()
}

// Gets the special name for the 'name' field in a global extended attribute entry.
// Format: "%d/GlobalHead.%p/%f"
// Format: "%d/GlobalHead.%p/%n"
// - %d: The path of the $TMPDIR variable, if found. Otherwise, the value is '/tmp'.
// - %p: The current process ID.
// - %n: The sequence number of the global extended header record of the archive, starting at 1. In our case, since we only generate one, the value is always 1.
// If the path of $TMPDIR makes the final string too long to fit in the 'name' field,
// then the TMPDIR='/tmp' is used.
private static string GenerateGlobalExtendedAttributeName()
private static string GenerateGlobalExtendedAttributeName(int globalExtendedAttributesEntryNumber)
{
string? tmpDir = Environment.GetEnvironmentVariable("TMPDIR");
if (string.IsNullOrWhiteSpace(tmpDir))
{
tmpDir = "/tmp";
}
else if (Path.EndsInDirectorySeparator(tmpDir))
Debug.Assert(globalExtendedAttributesEntryNumber >= 1);

Copy link
Member

@danmoseley danmoseley Jun 21, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As an aside - the code below reading TMPDIR then falling back to /tmp- why doesn't it just call Path.GetTempPath() that does the same thing?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good suggestion. I'll use Path.GetTempPath.

Here's why I wasn't using that:

The GNU tar manual specifies how the global extended attribute header name field obtains its value:

8.3.7.1 Controlling Extended Header Keywords

$TMPDIR/GlobalHead.%p.%n

[...] $TMPDIR stands for the value of the TMPDIR environment variable. If TMPDIR is not set, tar uses /tmp.

This description looks very similar to the remarks section of Path.GetTempPath:

Remarks

This method checks for the existence of environment variables in the following order and uses the first path found:

Linux

  1. The path specified by the TMPDIR environment variable. If the path is not specified in the TMPDIR environment variable, the default path /tmp/ is used.

Windows

  1. The path specified by the TMP environment variable.
  2. The path specified by the TEMP environment variable.
  3. The path specified by the USERPROFILE environment variable.
  4. The Windows directory.

So a couple of things need to be considered if we are to use Path.GetTempPath:

  • I don't see why any tool would be reading the GEA path, or having reading/writing behavior depend on it. The field isn't very interesting, unless the user cares about the process ID or the sequence number.
  • Keep in mind that none of the tar specs (at least the ones I read) specify what the expected behavior for Windows should be. Everything is Unix focused. If we format that string embedding Windows paths, I hope other tools don't break due to finding unexpected drive names or \ separators in that field. But go back to the previous point: I don't think any tool should make its behavior depend on this field.
  • If the Windows path ends up being larger than than 100 bytes, the sequence number and process ID would get truncated. Which is fine, I guess, because a couple of lines below, I re-generate the string by forcing the usage of /tmp.

string tmpDir = Path.GetTempPath();
if (Path.EndsInDirectorySeparator(tmpDir))
{
tmpDir = Path.TrimEndingDirectorySeparator(tmpDir);
}
int processId = Environment.ProcessId;

string result = string.Format(GlobalHeadFormat, tmpDir, processId);
if (result.Length >= FieldLengths.Name)
string result = string.Format(GlobalHeadFormatPrefix, tmpDir, processId);
string suffix = $".{globalExtendedAttributesEntryNumber}"; // GEA sequence number
if (result.Length + suffix.Length >= FieldLengths.Name)
{
result = string.Format(GlobalHeadFormat, "/tmp", processId);
result = string.Format(GlobalHeadFormatPrefix, "/tmp", processId);
}
result += suffix;

return result;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,11 @@ internal partial struct TarHeader
private const string PaxEaDevMajor = "devmajor";
private const string PaxEaDevMinor = "devminor";

// Global Extended Attribute entries have a special format in the Name field:
// "{tmpFolder}/GlobalHead.{processId}.{GEAEntryNumber}"
// Excludes ".{GEAEntryNumber}" because the number gets added on write.
internal const string GlobalHeadFormatPrefix = "{0}/GlobalHead.{1}";

internal Stream? _dataStream;

// Position in the stream where the data ends in this header.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -252,9 +252,10 @@ TarEntryType.HardLink or
TarEntryType.RegularFile or
TarEntryType.SymbolicLink)
{
// GlobalExtendedAttributes is handled via PaxGlobalExtendedAttributesEntry

// Not supported for writing - internally autogenerated:
// - ExtendedAttributes
// - GlobalExtendedAttributes
return;
}
break;
Expand Down
Loading