Skip to content
This repository was archived by the owner on Jan 23, 2023. It is now read-only.

Commit ef0aca5

Browse files
authored
Make Unix filename conversion lazy (#26978)
* Make Unix filename conversion lazy Also hook error handling. * Address feedback
1 parent 6730463 commit ef0aca5

File tree

3 files changed

+118
-76
lines changed

3 files changed

+118
-76
lines changed

src/Common/src/Interop/Unix/System.Native/Interop.ReadDir.cs

Lines changed: 40 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -3,15 +3,16 @@
33
// See the LICENSE file in the project root for more information.
44

55
using System;
6+
using System.Diagnostics;
67
using System.Runtime.InteropServices;
7-
using System.Threading;
8+
using System.Text;
89
using Microsoft.Win32.SafeHandles;
910

1011
internal static partial class Interop
1112
{
1213
internal static partial class Sys
1314
{
14-
private static readonly int s_readBufferSize = GetReadDirRBufferSize();
15+
internal static int ReadBufferSize { get; } = GetReadDirRBufferSize();
1516

1617
internal enum NodeType : int
1718
{
@@ -27,76 +28,59 @@ internal enum NodeType : int
2728
}
2829

2930
[StructLayout(LayoutKind.Sequential)]
30-
private unsafe struct InternalDirectoryEntry
31+
internal unsafe struct DirectoryEntry
3132
{
32-
internal IntPtr Name;
33-
internal int NameLength;
34-
internal NodeType InodeType;
35-
}
33+
internal byte* Name;
34+
internal int NameLength;
35+
internal NodeType InodeType;
3636

37-
internal struct DirectoryEntry
38-
{
39-
internal NodeType InodeType;
40-
internal string InodeName;
37+
internal ReadOnlySpan<char> GetName(Span<char> buffer)
38+
{
39+
Debug.Assert(buffer.Length >= Encoding.UTF8.GetMaxCharCount(255), "should have enough space for the max file name");
40+
Debug.Assert(Name != null, "should not have a null name");
41+
42+
ReadOnlySpan<byte> nameBytes = NameLength == -1
43+
// In this case the struct was allocated via struct dirent *readdir(DIR *dirp);
44+
? new ReadOnlySpan<byte>(Name, new ReadOnlySpan<byte>(Name, 255).IndexOf<byte>(0))
45+
: new ReadOnlySpan<byte>(Name, NameLength);
46+
47+
Debug.Assert(nameBytes.Length > 0, "we shouldn't have gotten a garbage value from the OS");
48+
if (nameBytes.Length == 0)
49+
return buffer.Slice(0, 0);
50+
51+
int charCount = Encoding.UTF8.GetChars(nameBytes, buffer);
52+
ReadOnlySpan<char> value = buffer.Slice(0, charCount);
53+
Debug.Assert(value.IndexOf('\0') == -1, "should not have embedded nulls");
54+
return value;
55+
}
4156
}
4257

4358
[DllImport(Libraries.SystemNative, EntryPoint = "SystemNative_OpenDir", SetLastError = true)]
44-
internal static extern Microsoft.Win32.SafeHandles.SafeDirectoryHandle OpenDir(string path);
59+
internal static extern SafeDirectoryHandle OpenDir(string path);
4560

4661
[DllImport(Libraries.SystemNative, EntryPoint = "SystemNative_GetReadDirRBufferSize", SetLastError = false)]
4762
internal static extern int GetReadDirRBufferSize();
4863

4964
[DllImport(Libraries.SystemNative, EntryPoint = "SystemNative_ReadDirR", SetLastError = false)]
50-
private static extern unsafe int ReadDirR(IntPtr dir, byte* buffer, int bufferSize, out InternalDirectoryEntry outputEntry);
65+
private static extern unsafe int ReadDirR(IntPtr dir, ref byte buffer, int bufferSize, ref DirectoryEntry outputEntry);
5166

5267
[DllImport(Libraries.SystemNative, EntryPoint = "SystemNative_CloseDir", SetLastError = true)]
5368
internal static extern int CloseDir(IntPtr dir);
5469

55-
// The calling pattern for ReadDir is described in src/Native/System.Native/pal_readdir.cpp
56-
internal static int ReadDir(SafeDirectoryHandle dir, out DirectoryEntry outputEntry)
70+
/// <summary>
71+
/// Get the next directory entry for the given handle. **Note** the actual memory used may be allocated
72+
/// by the OS and will be freed when the handle is closed. As such, the handle lifespan MUST be kept tightly
73+
/// controlled. The DirectoryEntry name cannot be accessed after the handle is closed.
74+
///
75+
/// Call <see cref="ReadBufferSize"/> to see what size buffer to allocate.
76+
/// </summary>
77+
internal static int ReadDir(SafeDirectoryHandle dir, Span<byte> buffer, ref DirectoryEntry entry)
5778
{
58-
bool addedRef = false;
59-
try
60-
{
61-
// We avoid a native string copy into InternalDirectoryEntry.
62-
// - If the platform suppors reading into a buffer, the data is read directly into the buffer. The
63-
// data can be read as long as the buffer is valid.
64-
// - If the platform does not support reading into a buffer, the information returned in
65-
// InternalDirectoryEntry points to native memory owned by the SafeDirectoryHandle. The data is only
66-
// valid until the next call to CloseDir/ReadDir. We extend the reference until we have copied all data
67-
// to ensure it does not become invalid by a CloseDir; and we copy the data so our caller does not
68-
// use the native memory held by the SafeDirectoryHandle.
69-
dir.DangerousAddRef(ref addedRef);
79+
// The calling pattern for ReadDir is described in src/Native/Unix/System.Native/pal_io.cpp|.h
80+
Debug.Assert(buffer.Length >= ReadBufferSize, "should have a big enough buffer for the raw data");
7081

71-
unsafe
72-
{
73-
// s_readBufferSize is zero when the native implementation does not support reading into a buffer.
74-
byte* buffer = stackalloc byte[s_readBufferSize];
75-
InternalDirectoryEntry temp;
76-
int ret = ReadDirR(dir.DangerousGetHandle(), buffer, s_readBufferSize, out temp);
77-
// We copy data into DirectoryEntry to ensure there are no dangling references.
78-
outputEntry = ret == 0 ?
79-
new DirectoryEntry() { InodeName = GetDirectoryEntryName(temp), InodeType = temp.InodeType } :
80-
default(DirectoryEntry);
81-
82-
return ret;
83-
}
84-
}
85-
finally
86-
{
87-
if (addedRef)
88-
{
89-
dir.DangerousRelease();
90-
}
91-
}
92-
}
93-
94-
private static unsafe string GetDirectoryEntryName(InternalDirectoryEntry dirEnt)
95-
{
96-
if (dirEnt.NameLength == -1)
97-
return Marshal.PtrToStringAnsi(dirEnt.Name);
98-
else
99-
return Marshal.PtrToStringAnsi(dirEnt.Name, dirEnt.NameLength);
82+
// ReadBufferSize is zero when the native implementation does not support reading into a buffer.
83+
return ReadDirR(dir.DangerousGetHandle(), ref MemoryMarshal.GetReference(buffer), ReadBufferSize, ref entry);
10084
}
10185
}
10286
}

src/System.IO.FileSystem/src/System/IO/Enumeration/FileSystemEntry.Unix.cs

Lines changed: 29 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,14 @@ namespace System.IO.Enumeration
99
/// </summary>
1010
public unsafe ref struct FileSystemEntry
1111
{
12+
private const int FileNameBufferSize = 256;
13+
internal Interop.Sys.DirectoryEntry _directoryEntry;
14+
private FileStatus _status;
15+
private Span<char> _pathBuffer;
16+
private ReadOnlySpan<char> _fullPath;
17+
private ReadOnlySpan<char> _fileName;
18+
private fixed char _fileNameBuffer[FileNameBufferSize];
19+
1220
internal static bool Initialize(
1321
ref FileSystemEntry entry,
1422
Interop.Sys.DirectoryEntry directoryEntry,
@@ -44,10 +52,6 @@ internal static bool Initialize(
4452
return isDirectory;
4553
}
4654

47-
internal Interop.Sys.DirectoryEntry _directoryEntry;
48-
private FileStatus _status;
49-
private Span<char> _pathBuffer;
50-
private ReadOnlySpan<char> _fullPath;
5155

5256
private ReadOnlySpan<char> FullPath
5357
{
@@ -58,14 +62,32 @@ private ReadOnlySpan<char> FullPath
5862
ReadOnlySpan<char> directory = Directory;
5963
directory.CopyTo(_pathBuffer);
6064
_pathBuffer[directory.Length] = Path.DirectorySeparatorChar;
61-
ReadOnlySpan<char> fileName = _directoryEntry.InodeName;
65+
ReadOnlySpan<char> fileName = FileName;
6266
fileName.CopyTo(_pathBuffer.Slice(directory.Length + 1));
6367
_fullPath = _pathBuffer.Slice(0, directory.Length + 1 + fileName.Length);
6468
}
6569
return _fullPath;
6670
}
6771
}
6872

73+
public ReadOnlySpan<char> FileName
74+
{
75+
get
76+
{
77+
if (_directoryEntry.Name != null)
78+
{
79+
fixed (char* c = _fileNameBuffer)
80+
{
81+
Span<char> buffer = new Span<char>(c, FileNameBufferSize);
82+
_fileName = _directoryEntry.GetName(buffer);
83+
}
84+
_directoryEntry.Name = null;
85+
}
86+
87+
return _fileName;
88+
}
89+
}
90+
6991
/// <summary>
7092
/// The full path of the directory this entry resides in.
7193
/// </summary>
@@ -81,7 +103,6 @@ private ReadOnlySpan<char> FullPath
81103
/// </summary>
82104
public string OriginalRootDirectory { get; private set; }
83105

84-
public ReadOnlySpan<char> FileName => _directoryEntry.InodeName;
85106
public FileAttributes Attributes => _status.GetAttributes(FullPath, FileName);
86107
public long Length => _status.GetLength(FullPath);
87108
public DateTimeOffset CreationTimeUtc => _status.GetCreationTime(FullPath);
@@ -94,11 +115,11 @@ public FileSystemInfo ToFileSystemInfo()
94115
string fullPath = ToFullPath();
95116
if (_status.InitiallyDirectory)
96117
{
97-
return DirectoryInfo.Create(fullPath, _directoryEntry.InodeName, ref _status);
118+
return DirectoryInfo.Create(fullPath, new string(FileName), ref _status);
98119
}
99120
else
100121
{
101-
return FileInfo.Create(fullPath, _directoryEntry.InodeName, ref _status);
122+
return FileInfo.Create(fullPath, new string(FileName), ref _status);
102123
}
103124
}
104125

src/System.IO.FileSystem/src/System/IO/Enumeration/FileSystemEnumerator.Unix.cs

Lines changed: 49 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44

55
using System.Buffers;
66
using System.Collections.Generic;
7+
using System.Diagnostics;
78
using System.Runtime.ConstrainedExecution;
89
using Microsoft.Win32.SafeHandles;
910

@@ -30,6 +31,8 @@ public unsafe abstract partial class FileSystemEnumerator<TResult> : CriticalFin
3031

3132
// Used for creating full paths
3233
private char[] _pathBuffer;
34+
// Used to get the raw entry data
35+
private byte[] _entryBuffer;
3336

3437
/// <summary>
3538
/// Encapsulates a find operation.
@@ -53,6 +56,8 @@ public FileSystemEnumerator(string directory, EnumerationOptions options = null)
5356
try
5457
{
5558
_pathBuffer = ArrayPool<char>.Shared.Rent(StandardBufferSize);
59+
int size = Interop.Sys.ReadBufferSize;
60+
_entryBuffer = size > 0 ? ArrayPool<byte>.Shared.Rent(size) : null;
5661
}
5762
catch
5863
{
@@ -62,15 +67,21 @@ public FileSystemEnumerator(string directory, EnumerationOptions options = null)
6267
}
6368
}
6469

65-
private static SafeDirectoryHandle CreateDirectoryHandle(string path)
70+
private SafeDirectoryHandle CreateDirectoryHandle(string path)
6671
{
6772
// TODO: https://github.com/dotnet/corefx/issues/26715
6873
// - Check access denied option and allow through if specified.
6974
// - Use IntPtr handle directly
7075
SafeDirectoryHandle handle = Interop.Sys.OpenDir(path);
7176
if (handle.IsInvalid)
7277
{
73-
throw Interop.GetExceptionForIoErrno(Interop.Sys.GetLastErrorInfo(), path, isDirectory: true);
78+
Interop.ErrorInfo info = Interop.Sys.GetLastErrorInfo();
79+
if ((_options.IgnoreInaccessible && IsAccessError(info.RawErrno))
80+
|| ContinueOnError(info.RawErrno))
81+
{
82+
return null;
83+
}
84+
throw Interop.GetExceptionForIoErrno(info, path, isDirectory: true);
7485
}
7586
return handle;
7687
}
@@ -107,7 +118,7 @@ public bool MoveNext()
107118
{
108119
// These three we don't have to hit the disk again to evaluate
109120
if (((_options.AttributesToSkip & FileAttributes.Directory) != 0 && isDirectory)
110-
|| ((_options.AttributesToSkip & FileAttributes.Hidden) != 0 && _entry.InodeName[0] == '.')
121+
|| ((_options.AttributesToSkip & FileAttributes.Hidden) != 0 && _entry.Name[0] == '.')
111122
|| ((_options.AttributesToSkip & FileAttributes.ReparsePoint) != 0 && _entry.InodeType == Interop.Sys.NodeType.DT_LNK))
112123
continue;
113124
}
@@ -121,7 +132,7 @@ public bool MoveNext()
121132
if (isDirectory)
122133
{
123134
// Subdirectory found
124-
if (PathHelpers.IsDotOrDotDot(_entry.InodeName))
135+
if (_entry.Name[0] == '.' && (_entry.Name[1] == 0 || (_entry.Name[1] == '.' && _entry.Name[2] == 0)))
125136
{
126137
// "." or "..", don't process unless the option is set
127138
if (!_options.ReturnSpecialDirectories)
@@ -130,7 +141,7 @@ public bool MoveNext()
130141
else if (_options.RecurseSubdirectories && ShouldRecurseIntoEntry(ref entry))
131142
{
132143
// Recursion is on and the directory was accepted, Queue it
133-
string subdirectory = PathHelpers.CombineNoChecks(_currentPath, _entry.InodeName);
144+
string subdirectory = PathHelpers.CombineNoChecks(_currentPath, entry.FileName);
134145
SafeDirectoryHandle subdirectoryHandle = CreateDirectoryHandle(subdirectory);
135146
if (subdirectoryHandle != null)
136147
{
@@ -161,17 +172,36 @@ public bool MoveNext()
161172

162173
private unsafe void FindNextEntry()
163174
{
164-
// Read each entry from the enumerator
165-
if (Interop.Sys.ReadDir(_directoryHandle, out _entry) != 0)
175+
Span<byte> buffer = _entryBuffer == null ? Span<byte>.Empty : new Span<byte>(_entryBuffer);
176+
int result = Interop.Sys.ReadDir(_directoryHandle, buffer, ref _entry);
177+
switch (result)
166178
{
167-
// TODO: https://github.com/dotnet/corefx/issues/26715
168-
// - Refactor ReadDir so we can process errors here
169-
170-
// Directory finished
171-
DirectoryFinished();
179+
case -1:
180+
// End of directory
181+
DirectoryFinished();
182+
break;
183+
case 0:
184+
// Success
185+
break;
186+
default:
187+
// Error
188+
if ((_options.IgnoreInaccessible && IsAccessError(result))
189+
|| ContinueOnError(result))
190+
{
191+
DirectoryFinished();
192+
break;
193+
}
194+
else
195+
{
196+
throw Interop.GetExceptionForIoErrno(new Interop.ErrorInfo(result), _currentPath, isDirectory: true);
197+
}
172198
}
173199
}
174200

201+
private static bool IsAccessError(int error)
202+
=> error == (int)Interop.Error.EACCES || error == (int)Interop.Error.EBADF
203+
|| error == (int)Interop.Error.EPERM;
204+
175205
private void InternalDispose(bool disposing)
176206
{
177207
// It is possible to fail to allocate the lock, but the finalizer will still run
@@ -189,6 +219,13 @@ private void InternalDispose(bool disposing)
189219
_pending.Dequeue().Handle.Dispose();
190220
_pending = null;
191221
}
222+
223+
if (_pathBuffer != null)
224+
ArrayPool<char>.Shared.Return(_pathBuffer);
225+
_pathBuffer = null;
226+
if (_entryBuffer != null)
227+
ArrayPool<byte>.Shared.Return(_entryBuffer);
228+
_entryBuffer = null;
192229
}
193230
}
194231

0 commit comments

Comments
 (0)