Skip to content

[System.Formats.Tar] Add deterministic TarWriter options #132050

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
jetersen wants to merge 1 commit into from
Closed

[System.Formats.Tar] Add deterministic TarWriter options #132050

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions src/libraries/System.Formats.Tar/ref/System.Formats.Tar.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -140,8 +140,14 @@ public void WriteEntry(string fileName, string? entryName) { }
public sealed partial class TarWriterOptions
{
public TarWriterOptions() { }
public bool Deterministic { get { throw null; } set { } }
public System.Formats.Tar.TarEntryFormat Format { get { throw null; } set { } }
public System.Formats.Tar.TarHardLinkMode HardLinkMode { get { throw null; } set { } }
public int? OverrideGid { get { throw null; } set { } }
public string? OverrideGName { get { throw null; } set { } }
public System.DateTimeOffset? OverrideModificationTime { get { throw null; } set { } }
public int? OverrideUid { get { throw null; } set { } }
public string? OverrideUName { get { throw null; } set { } }
Comment on lines 140 to +150
Comment on lines +143 to +150
}
public sealed partial class UstarTarEntry : System.Formats.Tar.PosixTarEntry
{
Expand Down
34 changes: 20 additions & 14 deletions src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHeader.Write.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -127,11 +127,11 @@ private void WriteUstarFieldsToBuffer(Span<byte> buffer)
}

// Writes the current header as a PAX Global Extended Attributes entry into the archive stream.
internal ValueTask WriteAsPaxGlobalExtendedAttributesCoreAsync<TAdapter>(Stream archiveStream, Memory<byte> buffer, int globalExtendedAttributesEntryNumber, CancellationToken cancellationToken)
internal ValueTask WriteAsPaxGlobalExtendedAttributesCoreAsync<TAdapter>(Stream archiveStream, Memory<byte> buffer, int globalExtendedAttributesEntryNumber, bool deterministic, CancellationToken cancellationToken)
where TAdapter : IReadWriteAdapter
{
VerifyGlobalExtendedAttributesDataIsValid(globalExtendedAttributesEntryNumber);
return WriteAsPaxExtendedAttributesCoreAsync<TAdapter>(archiveStream, buffer, ExtendedAttributes, isGea: true, globalExtendedAttributesEntryNumber, cancellationToken);
return WriteAsPaxExtendedAttributesCoreAsync<TAdapter>(archiveStream, buffer, ExtendedAttributes, isGea: true, globalExtendedAttributesEntryNumber, deterministic, cancellationToken);
}

// Verifies the data is valid for writing a Global Extended Attributes entry.
Expand Down Expand Up @@ -171,7 +171,7 @@ internal ValueTask WriteAsUstarCoreAsync<TAdapter>(Stream archiveStream, Memory<

// Writes the current header as a PAX entry into the archive stream.
// Makes sure to add the preceding extended attributes entry before the actual entry.
internal async ValueTask WriteAsPaxCoreAsync<TAdapter>(Stream archiveStream, Memory<byte> buffer, CancellationToken cancellationToken)
internal async ValueTask WriteAsPaxCoreAsync<TAdapter>(Stream archiveStream, Memory<byte> buffer, bool deterministic, CancellationToken cancellationToken)
where TAdapter : IReadWriteAdapter
{
Debug.Assert(archiveStream.CanSeek || _dataStream == null || _dataStream.CanSeek);
Expand All @@ -193,7 +193,7 @@ internal async ValueTask WriteAsPaxCoreAsync<TAdapter>(Stream archiveStream, Mem
CollectExtendedAttributesFromStandardFieldsIfNeeded();

// Write the extended attributes entry into the archive first
await extendedAttributesHeader.WriteAsPaxExtendedAttributesCoreAsync<TAdapter>(archiveStream, buffer, ExtendedAttributes, isGea: false, globalExtendedAttributesEntryNumber: -1, cancellationToken).ConfigureAwait(false);
await extendedAttributesHeader.WriteAsPaxExtendedAttributesCoreAsync<TAdapter>(archiveStream, buffer, ExtendedAttributes, isGea: false, globalExtendedAttributesEntryNumber: -1, deterministic, cancellationToken).ConfigureAwait(false);
buffer.Span.Clear();

// And then write the stored entry into the archive
Expand All @@ -205,7 +205,7 @@ internal async ValueTask WriteAsPaxCoreAsync<TAdapter>(Stream archiveStream, Mem
// Fill the current header's dict
CollectExtendedAttributesFromStandardFieldsIfNeeded();
// And pass the attributes to the preceding extended attributes header for writing
await extendedAttributesHeader.WriteAsPaxExtendedAttributesCoreAsync<TAdapter>(archiveStream, buffer, ExtendedAttributes, isGea: false, globalExtendedAttributesEntryNumber: -1, cancellationToken).ConfigureAwait(false);
await extendedAttributesHeader.WriteAsPaxExtendedAttributesCoreAsync<TAdapter>(archiveStream, buffer, ExtendedAttributes, isGea: false, globalExtendedAttributesEntryNumber: -1, deterministic, cancellationToken).ConfigureAwait(false);
buffer.Span.Clear(); // Reset it to reuse it

// Second, we write this header as a normal one
Expand Down Expand Up @@ -310,23 +310,23 @@ private void WriteGnuFieldsToBuffer(Span<byte> buffer)
}

// Writes the current header as a PAX Extended Attributes entry into the archive stream.
private ValueTask WriteAsPaxExtendedAttributesCoreAsync<TAdapter>(Stream archiveStream, Memory<byte> buffer, Dictionary<string, string> extendedAttributes, bool isGea, int globalExtendedAttributesEntryNumber, CancellationToken cancellationToken)
private ValueTask WriteAsPaxExtendedAttributesCoreAsync<TAdapter>(Stream archiveStream, Memory<byte> buffer, Dictionary<string, string> extendedAttributes, bool isGea, int globalExtendedAttributesEntryNumber, bool deterministic, CancellationToken cancellationToken)
where TAdapter : IReadWriteAdapter
{
WriteAsPaxExtendedAttributesShared(isGea, globalExtendedAttributesEntryNumber, extendedAttributes);
WriteAsPaxExtendedAttributesShared(isGea, globalExtendedAttributesEntryNumber, extendedAttributes, deterministic);
Debug.Assert(_dataStream == null || (extendedAttributes.Count > 0 && _dataStream.CanSeek)); // We generate the extended attributes data stream, should always be seekable
return WriteWithSeekableDataStreamCoreAsync<TAdapter>(TarEntryFormat.Pax, archiveStream, buffer, cancellationToken);
}

// Initializes the name, mode and type flag of a PAX extended attributes entry.
private void WriteAsPaxExtendedAttributesShared(bool isGea, int globalExtendedAttributesEntryNumber, Dictionary<string, string> extendedAttributes)
private void WriteAsPaxExtendedAttributesShared(bool isGea, int globalExtendedAttributesEntryNumber, Dictionary<string, string> extendedAttributes, bool deterministic)
{
Debug.Assert(isGea && globalExtendedAttributesEntryNumber >= 0 || !isGea && globalExtendedAttributesEntryNumber < 0);

_dataStream = GenerateExtendedAttributesDataStream(extendedAttributes);
_name = isGea ?
GenerateGlobalExtendedAttributeName(globalExtendedAttributesEntryNumber) :
GenerateExtendedAttributeName();
GenerateGlobalExtendedAttributeName(globalExtendedAttributesEntryNumber, deterministic) :
GenerateExtendedAttributeName(deterministic);

_mode = TarHelpers.GetDefaultMode(_typeFlag);
_typeFlag = isGea ? TarEntryType.GlobalExtendedAttributes : TarEntryType.ExtendedAttributes;
Expand Down Expand Up @@ -946,17 +946,18 @@ private static int WriteAsUtf8String(ReadOnlySpan<char> text, Span<byte> buffer)
// - %d: The directory name of the file, equivalent to the result of the dirname utility on the translated pathname.
// - %p: The current process ID.
// - %f: The filename of the file, equivalent to the result of the basename utility on the translated pathname.
private string GenerateExtendedAttributeName()
private string GenerateExtendedAttributeName(bool deterministic)
{
ReadOnlySpan<char> dirName = Path.GetDirectoryName(_name.AsSpan());
dirName = dirName.IsEmpty ? "." : dirName;

ReadOnlySpan<char> fileName = Path.GetFileName(_name.AsSpan());
fileName = fileName.IsEmpty ? "." : fileName;

int processId = deterministic ? 0 : Environment.ProcessId;
return _typeFlag is TarEntryType.Directory or TarEntryType.DirectoryList ?
$"{dirName}/PaxHeaders.{Environment.ProcessId}/{fileName}{Path.DirectorySeparatorChar}" :
$"{dirName}/PaxHeaders.{Environment.ProcessId}/{fileName}";
$"{dirName}/PaxHeaders.{processId}/{fileName}{Path.DirectorySeparatorChar}" :
$"{dirName}/PaxHeaders.{processId}/{fileName}";
}

// Gets the special name for the 'name' field in a global extended attribute entry.
Expand All @@ -966,10 +967,15 @@ private string GenerateExtendedAttributeName()
// - %n: The sequence number of the global extended header record of the archive, starting at 1.
// If the path of $TMPDIR makes the final string too long to fit in the 'name' field,
// then the TMPDIR='/tmp' is used.
private static string GenerateGlobalExtendedAttributeName(int globalExtendedAttributesEntryNumber)
private static string GenerateGlobalExtendedAttributeName(int globalExtendedAttributesEntryNumber, bool deterministic)
{
Debug.Assert(globalExtendedAttributesEntryNumber >= 1);

if (deterministic)
{
return $"/tmp/GlobalHead.0.{globalExtendedAttributesEntryNumber}";
}

ReadOnlySpan<char> tmp = Path.TrimEndingDirectorySeparator(Path.GetTempPath());

string result = $"{tmp}/GlobalHead.{Environment.ProcessId}.{globalExtendedAttributesEntryNumber}";
Expand Down
39 changes: 29 additions & 10 deletions src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarWriter.Unix.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ private TarEntry ConstructEntryForWriting(string fullPath, string entryName, Fil
entry._header._devMinor = (int)minor;
}

entry._header._mTime = TarHelpers.GetDateTimeOffsetFromSecondsSinceEpoch(status.MTime);
entry._header._mTime = GetModificationTime(TarHelpers.GetDateTimeOffsetFromSecondsSinceEpoch(status.MTime));
// We do not set atime and ctime by default because many external tools are unable to read GNU entries
// that have these fields set to non-zero values. This is because the GNU format writes atime and ctime in the same
// location where other formats expect the prefix field to be written.
Expand All @@ -95,24 +95,43 @@ private TarEntry ConstructEntryForWriting(string fullPath, string entryName, Fil
entry._header._mode = status.Mode & (int)TarHelpers.ValidUnixFileModes;

// Uid and UName
entry._header._uid = (int)status.Uid;
if (!_userIdentifiers.TryGetValue(status.Uid, out string? uName))
entry._header._uid = _overrideUid ?? (_deterministic ? 0 : (int)status.Uid);
if (_overrideUName is string overrideUName)
{
uName = Interop.Sys.GetUserNameFromPasswd(status.Uid);
_userIdentifiers.Add(status.Uid, uName);
entry._header._uName = overrideUName;
}
else if (_deterministic)
{
entry._header._uName = string.Empty;
}
else
{
if (!_userIdentifiers.TryGetValue(status.Uid, out string? uName))
{
uName = Interop.Sys.GetUserNameFromPasswd(status.Uid);
_userIdentifiers.Add(status.Uid, uName);
}
entry._header._uName = uName;
}
entry._header._uName = uName;

// Gid and GName
entry._header._gid = (int)status.Gid;
if (!_groupIdentifiers.TryGetValue(status.Gid, out string? gName))
entry._header._gid = _overrideGid ?? (_deterministic ? 0 : (int)status.Gid);
if (_overrideGName is string overrideGName)
{
entry._header._gName = overrideGName;
}
else if (_deterministic)
{
entry._header._gName = string.Empty;
}
else
{
if (Interop.Sys.TryGetGroupName(status.Gid, out gName))
if (!_groupIdentifiers.TryGetValue(status.Gid, out string? gName) && Interop.Sys.TryGetGroupName(status.Gid, out gName))
{
_groupIdentifiers.Add(status.Gid, gName);
}
entry._header._gName = gName;
}
entry._header._gName = gName;

if (entry.EntryType == TarEntryType.SymbolicLink)
{
Expand Down
6 changes: 5 additions & 1 deletion src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarWriter.Windows.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -116,14 +116,18 @@ private TarEntry ConstructEntryForWriting(string fullPath, string entryName, Fil
_ => throw new InvalidDataException(SR.Format(SR.TarInvalidFormat, Format)),
};

entry._header._mTime = fileInfo.ftLastWriteTime.ToDateTimeUtc();
entry._header._mTime = GetModificationTime(fileInfo.ftLastWriteTime.ToDateTimeUtc());
// We do not set atime and ctime by default because many external tools are unable to read GNU entries
// that have these fields set to non-zero values. This is because the GNU format writes atime and ctime in the same
// location where other formats expect the prefix field to be written.
// If the user wants to set atime and ctime, they can do so by constructing the entry manually from the file and
// then setting the values.

entry.Mode = DefaultWindowsMode;
entry._header._uid = _overrideUid ?? 0;
entry._header._gid = _overrideGid ?? 0;
entry._header._uName = _overrideUName ?? string.Empty;
entry._header._gName = _overrideGName ?? string.Empty;

if (entry.EntryType == TarEntryType.SymbolicLink)
{
Expand Down
19 changes: 17 additions & 2 deletions src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarWriter.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,12 @@ public sealed partial class TarWriter : IDisposable, IAsyncDisposable
private readonly bool _leaveOpen;
private readonly Stream _archiveStream;
private readonly TarHardLinkMode _hardLinkMode;
private readonly bool _deterministic;
private readonly DateTimeOffset? _overrideModificationTime;
private readonly int? _overrideUid;
private readonly int? _overrideGid;
private readonly string? _overrideUName;
private readonly string? _overrideGName;
private int _nextGlobalExtendedAttributesEntryNumber;

/// <summary>
Expand Down Expand Up @@ -98,6 +104,12 @@ public TarWriter(Stream archiveStream, TarWriterOptions options, bool leaveOpen
_archiveStream = archiveStream;
Format = options.Format;
_hardLinkMode = options.HardLinkMode;
_deterministic = options.Deterministic;
_overrideModificationTime = options.OverrideModificationTime;
_overrideUid = options.OverrideUid;
_overrideGid = options.OverrideGid;
_overrideUName = options.OverrideUName;
_overrideGName = options.OverrideGName;
_leaveOpen = leaveOpen;
_isDisposed = false;
_wroteEntries = false;
Expand Down Expand Up @@ -315,8 +327,8 @@ private async ValueTask WriteEntryCoreAsync<TAdapter>(TarEntry entry, Cancellati
{
TarEntryFormat.V7 => entry._header.WriteAsV7CoreAsync<TAdapter>(_archiveStream, buffer, cancellationToken),
TarEntryFormat.Ustar => entry._header.WriteAsUstarCoreAsync<TAdapter>(_archiveStream, buffer, cancellationToken),
TarEntryFormat.Pax when entry._header._typeFlag is TarEntryType.GlobalExtendedAttributes => entry._header.WriteAsPaxGlobalExtendedAttributesCoreAsync<TAdapter>(_archiveStream, buffer, _nextGlobalExtendedAttributesEntryNumber++, cancellationToken),
TarEntryFormat.Pax => entry._header.WriteAsPaxCoreAsync<TAdapter>(_archiveStream, buffer, cancellationToken),
TarEntryFormat.Pax when entry._header._typeFlag is TarEntryType.GlobalExtendedAttributes => entry._header.WriteAsPaxGlobalExtendedAttributesCoreAsync<TAdapter>(_archiveStream, buffer, _nextGlobalExtendedAttributesEntryNumber++, _deterministic, cancellationToken),
TarEntryFormat.Pax => entry._header.WriteAsPaxCoreAsync<TAdapter>(_archiveStream, buffer, _deterministic, cancellationToken),
TarEntryFormat.Gnu => entry._header.WriteAsGnuCoreAsync<TAdapter>(_archiveStream, buffer, cancellationToken),
_ => throw new InvalidDataException(SR.Format(SR.TarInvalidFormat, Format)),
};
Expand All @@ -330,6 +342,9 @@ private async ValueTask WriteEntryCoreAsync<TAdapter>(TarEntry entry, Cancellati
}
}

private DateTimeOffset GetModificationTime(DateTimeOffset sourceModificationTime) =>
_overrideModificationTime ?? (_deterministic ? DateTimeOffset.UnixEpoch : sourceModificationTime);

// The spec indicates that the end of the archive is indicated
// by two records consisting entirely of zero bytes.
// This method is called from Dispose/DisposeAsync, so we don't want to propagate a cancelled CancellationToken.
Expand Down
Loading
Loading