Skip to content
This repository was archived by the owner on Jan 23, 2023. It is now read-only.

Commit c64171e

Browse files
JeremyKuhneJeremyKuhne
authored
Reduce Unix enumeration allocations (#26942)
* Reduce Unix enumeration allocations This change factors out the FileStatus access into a helper struct and adds overloads for the stat imports that allow passing a span. The next steps: - Handle errors manually - Skip using safehandle - Look for further opportunities around UTF-8/16 conversion * Address feedback Shuffle code around a bit in FileInfo for clarity. Also shift directory determination to FileSystemEntry to avoid allocation on links.
1 parent a6046cb commit c64171e

16 files changed

+550
-372
lines changed

‎src/Common/src/Interop/Unix/System.Native/Interop.Stat.Span.cs‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
// Licensed to the .NET Foundation under one or more agreements.
2+
// The .NET Foundation licenses this file to you under the MIT license.
3+
// See the LICENSE file in the project root for more information.
4+
5+
using System;
6+
using System.Runtime.InteropServices;
7+
using System.Text;
8+
9+
internal static partial class Interop
10+
{
11+
internal static partial class Sys
12+
{
13+
// Unix max paths are typically 1K or 4K UTF-8 bytes, 256 should handle the majority of paths
14+
// without putting too much pressure on the stack.
15+
private const int StackBufferSize = 256;
16+
17+
[DllImport(Libraries.SystemNative, EntryPoint = "SystemNative_Stat2", SetLastError = true)]
18+
internal unsafe static extern int Stat(ref byte path, out FileStatus output);
19+
20+
internal unsafe static int Stat(ReadOnlySpan<char> path, out FileStatus output)
21+
{
22+
byte* buffer = stackalloc byte[StackBufferSize];
23+
var converter = new ValueUtf8Converter(new Span<byte>(buffer, StackBufferSize));
24+
int result = Stat(ref MemoryMarshal.GetReference(converter.ConvertAndTerminateString(path)), out output);
25+
converter.Dispose();
26+
return result;
27+
}
28+
29+
[DllImport(Libraries.SystemNative, EntryPoint = "SystemNative_LStat2", SetLastError = true)]
30+
internal static extern int LStat(ref byte path, out FileStatus output);
31+
32+
internal unsafe static int LStat(ReadOnlySpan<char> path, out FileStatus output)
33+
{
34+
byte* buffer = stackalloc byte[StackBufferSize];
35+
var converter = new ValueUtf8Converter(new Span<byte>(buffer, StackBufferSize));
36+
int result = LStat(ref MemoryMarshal.GetReference(converter.ConvertAndTerminateString(path)), out output);
37+
converter.Dispose();
38+
return result;
39+
}
40+
}
41+
}

‎src/Common/src/System/IO/PathInternal.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -70,8 +70,8 @@ internal static StringBuilder TrimEnd(this StringBuilder builder, params char[]
7070
/// <summary>
7171
/// Returns true if the path ends in a directory separator.
7272
/// </summary>
73-
internal static bool EndsInDirectorySeparator(string path) =>
74-
!string.IsNullOrEmpty(path) && IsDirectorySeparator(path[path.Length - 1]);
73+
internal static bool EndsInDirectorySeparator(ReadOnlySpan<char> path) =>
74+
path.Length > 0 && IsDirectorySeparator(path[path.Length - 1]);
7575

7676
/// <summary>
7777
/// Get the common path length from the start of the string.

‎src/Common/src/System/Text/ValueUtf8Converter.cs‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
// Licensed to the .NET Foundation under one or more agreements.
2+
// The .NET Foundation licenses this file to you under the MIT license.
3+
// See the LICENSE file in the project root for more information.
4+
5+
using System.Buffers;
6+
7+
namespace System.Text
8+
{
9+
/// <summary>
10+
/// Helper to allow utilizing stack buffer for conversion to UTF-8. Will
11+
/// switch to ArrayPool if not given enough memory. As such, make sure to
12+
/// call Clear() to return any potentially rented buffer after conversion.
13+
/// </summary>
14+
internal ref struct ValueUtf8Converter
15+
{
16+
private byte[] _arrayToReturnToPool;
17+
private Span<byte> _bytes;
18+
19+
public ValueUtf8Converter(Span<byte> initialBuffer)
20+
{
21+
_arrayToReturnToPool = null;
22+
_bytes = initialBuffer;
23+
}
24+
25+
public Span<byte> ConvertAndTerminateString(ReadOnlySpan<char> value)
26+
{
27+
int maxSize = Encoding.UTF8.GetMaxByteCount(value.Length) + 1;
28+
if (_bytes.Length < maxSize)
29+
{
30+
Dispose();
31+
_arrayToReturnToPool = ArrayPool<byte>.Shared.Rent(maxSize);
32+
_bytes = new Span<byte>(_arrayToReturnToPool);
33+
}
34+
35+
// Grab the bytes and null terminate
36+
int byteCount = Encoding.UTF8.GetBytes(value, _bytes);
37+
_bytes[byteCount] = 0;
38+
return _bytes.Slice(0, byteCount + 1);
39+
}
40+
41+
public void Dispose()
42+
{
43+
byte[] toReturn = _arrayToReturnToPool;
44+
if (toReturn != null)
45+
{
46+
_arrayToReturnToPool = null;
47+
ArrayPool<byte>.Shared.Return(toReturn);
48+
}
49+
}
50+
}
51+
}

‎src/System.IO.FileSystem/System.IO.FileSystem.sln‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
Microsoft Visual Studio Solution File, Format Version 12.00
2-
# Visual Studio 14
3-
VisualStudioVersion = 14.0.25420.1
2+
# Visual Studio 15
3+
VisualStudioVersion = 15.0.27130.2027
44
MinimumVisualStudioVersion = 10.0.40219.1
55
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "System.IO.FileSystem.Tests", "tests\System.IO.FileSystem.Tests.csproj", "{F0D49126-6A1C-42D5-9428-4374C868BAF8}"
66
ProjectSection(ProjectDependencies) = postProject
@@ -39,8 +39,8 @@ Global
3939
{3C42F714-82AF-4A43-9B9C-744DE31B5C5D}.Debug|Any CPU.Build.0 = netstandard-Debug|Any CPU
4040
{3C42F714-82AF-4A43-9B9C-744DE31B5C5D}.Release|Any CPU.ActiveCfg = netstandard-Release|Any CPU
4141
{3C42F714-82AF-4A43-9B9C-744DE31B5C5D}.Release|Any CPU.Build.0 = netstandard-Release|Any CPU
42-
{1B528B61-14F9-4BFC-A79A-F0BDB3339150}.Debug|Any CPU.ActiveCfg = netcoreapp-Windows_NT-Debug|Any CPU
43-
{1B528B61-14F9-4BFC-A79A-F0BDB3339150}.Debug|Any CPU.Build.0 = netcoreapp-Windows_NT-Debug|Any CPU
42+
{1B528B61-14F9-4BFC-A79A-F0BDB3339150}.Debug|Any CPU.ActiveCfg = netcoreapp-Unix-Debug|Any CPU
43+
{1B528B61-14F9-4BFC-A79A-F0BDB3339150}.Debug|Any CPU.Build.0 = netcoreapp-Unix-Debug|Any CPU
4444
{1B528B61-14F9-4BFC-A79A-F0BDB3339150}.Release|Any CPU.ActiveCfg = netcoreapp-Windows_NT-Release|Any CPU
4545
{1B528B61-14F9-4BFC-A79A-F0BDB3339150}.Release|Any CPU.Build.0 = netcoreapp-Windows_NT-Release|Any CPU
4646
{4B15C12E-B6AB-4B05-8ECA-C2E2AEA67482}.Debug|Any CPU.ActiveCfg = netcoreapp-Debug|Any CPU
@@ -57,4 +57,7 @@ Global
5757
{1B528B61-14F9-4BFC-A79A-F0BDB3339150} = {E107E9C1-E893-4E87-987E-04EF0DCEAEFD}
5858
{4B15C12E-B6AB-4B05-8ECA-C2E2AEA67482} = {2E666815-2EDB-464B-9DF6-380BF4789AD4}
5959
EndGlobalSection
60+
GlobalSection(ExtensibilityGlobals) = postSolution
61+
SolutionGuid = {28498879-453E-42C1-8D74-0F9F80B1B58E}
62+
EndGlobalSection
6063
EndGlobal

‎src/System.IO.FileSystem/src/System.IO.FileSystem.csproj‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -179,7 +179,6 @@
179179
<Compile Include="$(CommonPath)\Interop\Windows\kernel32\Interop.GET_FILEEX_INFO_LEVELS.cs">
180180
<Link>Common\Interop\Windows\Interop.GET_FILEEX_INFO_LEVELS.cs</Link>
181181
</Compile>
182-
<Compile Include="System\IO\FileSystemInfo.Win32.cs" />
183182
<Compile Include="$(CommonPath)\Interop\Windows\kernel32\Interop.SetThreadErrorMode.cs">
184183
<Link>Common\Interop\Windows\Interop.SetThreadErrorMode.cs</Link>
185184
</Compile>
@@ -281,9 +280,12 @@
281280
</ItemGroup>
282281
<!-- Unix -->
283282
<ItemGroup Condition="'$(TargetsUnix)' == 'true'">
283+
<Compile Include="System\IO\FileStatus.Unix.cs" />
284284
<Compile Include="System\IO\Enumeration\FileSystemEntry.Unix.cs" />
285285
<Compile Include="System\IO\Enumeration\FileSystemEnumerator.Unix.cs" />
286286
<Compile Include="System\IO\CharSpanExtensions.Unix.cs" />
287+
<Compile Include="System\IO\FileInfo.Unix.cs" />
288+
<Compile Include="System\IO\DirectoryInfo.Unix.cs" />
287289
<Compile Include="System\IO\FileSystemInfo.Unix.cs" />
288290
<Compile Include="System\IO\PathHelpers.Unix.cs" />
289291
<Compile Include="System\IO\FileSystem.Unix.cs" />
@@ -338,6 +340,9 @@
338340
<Compile Include="$(CommonPath)\Interop\Unix\System.Native\Interop.Stat.cs">
339341
<Link>Common\Interop\Unix\Interop.Stat.cs</Link>
340342
</Compile>
343+
<Compile Include="$(CommonPath)\Interop\Unix\System.Native\Interop.Stat.Span.cs">
344+
<Link>Common\Interop\Unix\Interop.Stat.Span.cs</Link>
345+
</Compile>
341346
<Compile Include="$(CommonPath)\Interop\Unix\System.Native\Interop.ReadDir.cs">
342347
<Link>Common\Interop\Unix\Interop.ReadDir.cs</Link>
343348
</Compile>
@@ -392,6 +397,9 @@
392397
<Compile Include="$(CommonPath)\System\IO\DriveInfoInternal.Unix.cs">
393398
<Link>Common\System\IO\DriveInfoInternal.Unix.cs</Link>
394399
</Compile>
400+
<Compile Include="$(CommonPath)\System\Text\ValueUtf8Converter.cs">
401+
<Link>Common\System\Text\ValueUtf8Converter.cs</Link>
402+
</Compile>
395403
</ItemGroup>
396404
<ItemGroup>
397405
<Reference Include="System.Buffers" />

‎src/System.IO.FileSystem/src/System/IO/DirectoryInfo.Unix.cs‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
// Licensed to the .NET Foundation under one or more agreements.
2+
// The .NET Foundation licenses this file to you under the MIT license.
3+
// See the LICENSE file in the project root for more information.
4+
5+
namespace System.IO
6+
{
7+
partial class DirectoryInfo
8+
{
9+
internal static unsafe DirectoryInfo Create(string fullPath, string fileName, ref FileStatus fileStatus)
10+
{
11+
DirectoryInfo info = new DirectoryInfo(fullPath, fileName: fileName, isNormalized: true);
12+
info.Init(ref fileStatus);
13+
return info;
14+
}
15+
}
16+
}

‎src/System.IO.FileSystem/src/System/IO/Enumeration/FileSystemEntry.Unix.cs‎

Lines changed: 56 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -9,41 +9,60 @@ namespace System.IO.Enumeration
99
/// </summary>
1010
public unsafe ref struct FileSystemEntry
1111
{
12-
// TODO: Unix implementation https://github.com/dotnet/corefx/issues/26715
13-
// Inital implementation is naive and not optimized.
14-
15-
internal static void Initialize(
12+
internal static bool Initialize(
1613
ref FileSystemEntry entry,
1714
Interop.Sys.DirectoryEntry directoryEntry,
18-
bool isDirectory,
1915
ReadOnlySpan<char> directory,
2016
string rootDirectory,
21-
string originalRootDirectory)
17+
string originalRootDirectory,
18+
Span<char> pathBuffer)
2219
{
2320
entry._directoryEntry = directoryEntry;
24-
entry._isDirectory = isDirectory;
2521
entry.Directory = directory;
2622
entry.RootDirectory = rootDirectory;
2723
entry.OriginalRootDirectory = originalRootDirectory;
24+
entry._pathBuffer = pathBuffer;
25+
26+
// Get from the dir entry whether the entry is a file or directory.
27+
// We classify everything as a file unless we know it to be a directory.
28+
// (This includes regular files, FIFOs, etc.)
29+
30+
bool isDirectory = false;
31+
if (directoryEntry.InodeType == Interop.Sys.NodeType.DT_DIR)
32+
{
33+
// We know it's a directory.
34+
isDirectory = true;
35+
}
36+
else if ((directoryEntry.InodeType == Interop.Sys.NodeType.DT_LNK || directoryEntry.InodeType == Interop.Sys.NodeType.DT_UNKNOWN)
37+
&& Interop.Sys.Stat(entry.FullPath, out Interop.Sys.FileStatus targetStatus) >= 0)
38+
{
39+
// It's a symlink or unknown: stat to it to see if we can resolve it to a directory.
40+
isDirectory = (targetStatus.Mode & Interop.Sys.FileTypes.S_IFMT) == Interop.Sys.FileTypes.S_IFDIR;
41+
}
42+
43+
FileStatus.Initialize(ref entry._status, isDirectory);
44+
return isDirectory;
2845
}
2946

3047
internal Interop.Sys.DirectoryEntry _directoryEntry;
31-
private FileSystemInfo _info;
32-
private bool _isDirectory;
48+
private FileStatus _status;
49+
private Span<char> _pathBuffer;
50+
private ReadOnlySpan<char> _fullPath;
3351

34-
private FileSystemInfo Info
52+
private ReadOnlySpan<char> FullPath
3553
{
3654
get
3755
{
38-
if (_info == null)
56+
if (_fullPath.Length == 0)
3957
{
40-
string fullPath = PathHelpers.CombineNoChecks(Directory, _directoryEntry.InodeName);
41-
_info = _isDirectory
42-
? (FileSystemInfo) new DirectoryInfo(fullPath, fullPath, _directoryEntry.InodeName, isNormalized: true)
43-
: new FileInfo(fullPath, fullPath, _directoryEntry.InodeName, isNormalized: true);
44-
_info.Refresh();
58+
ReadOnlySpan<char> directory = Directory;
59+
directory.CopyTo(_pathBuffer);
60+
_pathBuffer[directory.Length] = Path.DirectorySeparatorChar;
61+
ReadOnlySpan<char> fileName = _directoryEntry.InodeName;
62+
fileName.CopyTo(_pathBuffer.Slice(directory.Length + 1));
63+
_fullPath = _pathBuffer.Slice(0, directory.Length + 1 + fileName.Length);
4564
}
46-
return _info;
65+
return _fullPath;
4766
}
4867
}
4968

@@ -63,13 +82,25 @@ private FileSystemInfo Info
6382
public string OriginalRootDirectory { get; private set; }
6483

6584
public ReadOnlySpan<char> FileName => _directoryEntry.InodeName;
66-
public FileAttributes Attributes => Info.Attributes;
67-
public long Length => Info.LengthCore;
68-
public DateTimeOffset CreationTimeUtc => Info.CreationTimeCore;
69-
public DateTimeOffset LastAccessTimeUtc => Info.LastAccessTimeCore;
70-
public DateTimeOffset LastWriteTimeUtc => Info.LastWriteTimeCore;
71-
public bool IsDirectory => _isDirectory;
72-
public FileSystemInfo ToFileSystemInfo() => Info;
85+
public FileAttributes Attributes => _status.GetAttributes(FullPath, FileName);
86+
public long Length => _status.GetLength(FullPath);
87+
public DateTimeOffset CreationTimeUtc => _status.GetCreationTime(FullPath);
88+
public DateTimeOffset LastAccessTimeUtc => _status.GetLastAccessTime(FullPath);
89+
public DateTimeOffset LastWriteTimeUtc => _status.GetLastWriteTime(FullPath);
90+
public bool IsDirectory => _status.InitiallyDirectory;
91+
92+
public FileSystemInfo ToFileSystemInfo()
93+
{
94+
string fullPath = ToFullPath();
95+
if (_status.InitiallyDirectory)
96+
{
97+
return DirectoryInfo.Create(fullPath, _directoryEntry.InodeName, ref _status);
98+
}
99+
else
100+
{
101+
return FileInfo.Create(fullPath, _directoryEntry.InodeName, ref _status);
102+
}
103+
}
73104

74105
/// <summary>
75106
/// Returns the full path for find results, based on the initially provided path.
@@ -81,6 +112,6 @@ public string ToSpecifiedFullPath() =>
81112
/// Returns the full path of the find result.
82113
/// </summary>
83114
public string ToFullPath() =>
84-
PathHelpers.CombineNoChecks(Directory, FileName);
115+
new string(FullPath);
85116
}
86117
}

0 commit comments

Comments
 (0)