FSEventStream: support FSEventStreamCreateFlags.UseExtendedData

When UseExtendedData is specified, the event data type changes
from CFString to CFDictionary; the dictionary contains the path
and inode information for the file and may be extended in the future
with other fields. Previously this was crashing because we assumed
CFString always.

Adds file system tests which cover the existing (non-extended) and
fixed (extended) creation modes, along with using a dispatch queue
instead of run loop.

Fixes xamarin#12007

src/CoreServices/FSEvents.cs

@@ -88,10 +88,11 @@ public struct FSEvent
 		public ulong Id { get; internal set; }
 		public string? Path { get; internal set; }
 		public FSEventStreamEventFlags Flags { get; internal set; }
+		public ulong FileId { get; internal set; }
 
 		public override string ToString ()
 		{
-			return String.Format ("[FSEvent: Id={0}, Path={1}, Flags={2}]", Id, Path, Flags);
+			return String.Format ("[FSEvent: Id={0}, Path={1}, Flags={2}, FileId={3}]", Id, Path, Flags, FileId);
 		}
 
 		public const ulong SinceNowId = UInt64.MaxValue;
@@ -253,6 +254,14 @@ static void FreeGCHandle (IntPtr gchandle)
 			GCHandle.FromIntPtr (gchandle).Free ();
 		}
 
+		static readonly nint CFStringTypeID = CFString.GetTypeID ();
+		static readonly nint CFDictionaryTypeID = CFDictionary.GetTypeID ();
+
+		// These constants are defined in FSEvents.h but do not end up exported in any binaries,
+		// so we cannot use Dlfcn.GetStringConstant against CoreServices. -abock, 2022-03-04
+		static readonly NSString kFSEventStreamEventExtendedDataPathKey = new ("path");
+		static readonly NSString kFSEventStreamEventExtendedFileIDKey = new ("fileID");
+
 #if NET
 		[UnmanagedCallersOnly]
 #endif
@@ -264,12 +273,35 @@ static void FreeGCHandle (IntPtr gchandle)
 			}
 
 			var events = new FSEvent[numEvents];
-			var pathArray = new CFArray (eventPaths, false);
 
 			for (int i = 0; i < events.Length; i++) {
-				events[i].Flags = (FSEventStreamEventFlags)(uint)Marshal.ReadInt32 (eventFlags, i * 4);
-				events[i].Id = (uint)Marshal.ReadInt64 (eventIds, i * 8);
-				events[i].Path = CFString.FromHandle (pathArray.GetValue (i));
+				string? path = null;
+				long fileId = 0;
+
+				var eventDataHandle = CFArray.CFArrayGetValueAtIndex (eventPaths, i);
+				var eventDataType = CFType.GetTypeID (eventDataHandle);
+
+				if (eventDataType == CFStringTypeID) {
+					path = CFString.FromHandle (eventDataHandle);
+				} else if (eventDataType == CFDictionaryTypeID) {
+					path =  CFString.FromHandle (CFDictionary.GetValue (
+						eventDataHandle,
+						kFSEventStreamEventExtendedDataPathKey.Handle));
+
+					var fileIdHandle = CFDictionary.GetValue (
+						eventDataHandle,
+						kFSEventStreamEventExtendedFileIDKey.Handle);
+					if (fileIdHandle != IntPtr.Zero)
+						CFDictionary.CFNumberGetValue (fileIdHandle, 4 /*kCFNumberSInt64Type*/, out fileId);
+				}
+
+				events[i] = new FSEvent
+				{
+					Id = (ulong)Marshal.ReadInt64 (eventIds, i * 8),
+					Path = path,
+					Flags = (FSEventStreamEventFlags)(uint)Marshal.ReadInt32 (eventFlags, i * 4),
+					FileId = (ulong)fileId,
+				};
 			}
 
 			var instance = GCHandle.FromIntPtr (userData).Target as FSEventStream;

tests/monotouch-test/CoreServices/FSEventStreamTest.cs

@@ -5,24 +5,196 @@
 #if __MACOS__
 
 using System;
+using System.Collections.Generic;
 using System.IO;
+using System.Runtime.InteropServices;
+using System.Threading;
+using System.Threading.Tasks;
 
 using CoreFoundation;
 using CoreServices;
 using Foundation;
+using ObjCRuntime;
 
 using NUnit.Framework;
 
 namespace MonoTouchFixtures.CoreServices {
+	using static FSEventStreamCreateFlags;
+	using static FSEventStreamEventFlags;
 
 	[TestFixture]
 	[Preserve (AllMembers = true)]
-	public class FSEventStreamTest {
+	public sealed class FSEventStreamTest {
+		[Test]
+		public void TestFileEvents ()
+			=> RunTest (FileEvents);
 
 		[Test]
-		public void Create ()
-		{
-			using var eventStream = new FSEventStream (new [] { Path.Combine (Environment.GetEnvironmentVariable ("HOME"), "Desktop") }, TimeSpan.FromSeconds (5), FSEventStreamCreateFlags.FileEvents);
+		public void TestExtendedFileEvents ()
+			=> RunTest (FileEvents | UseExtendedData);
+
+		static void RunTest (FSEventStreamCreateFlags createFlags)
+			=> new TestFSMonitor (
+				Xamarin.Cache.CreateTemporaryDirectory (),
+				createFlags,
+				maxFilesToCreate: 256).Run ();
+
+		/// <summary>
+		/// Creates a slew of files on a background thread in some directory
+		/// while simultaneously running an FSEventStream against a private
+		/// dispatch queue for that directory, and blocks/pumps the main thread
+		/// while the following work is settling on the two other threads:
+		///
+		/// (1) create a bunch of files and directories;
+		///
+		/// (2) as the FSEventStream raises events on the dispatch queue,
+		/// reflect the events (e.g. file created vs file deleted) in our state;
+		/// if a file was created, delete it, which will trigger another event
+		/// for the deletion to be recorded.
+		///
+		/// (3) when everything has settled (created + deleted), ensure that
+		/// all created files were seen as created through the FSEventStream and
+		/// then subsequently seen as deleted.
+		/// </summary>
+		sealed class TestFSMonitor : FSEventStream {
+			static readonly TimeSpan s_testTimeout = TimeSpan.FromSeconds (10);
+
+			readonly int _directoriesToCreate;
+			readonly int _filesPerDirectoryToCreate;
+			readonly List<string> _createdDirectories = new ();
+			readonly List<string> _createdThenRemovedFiles = new ();
+			readonly List<string> _createdFiles = new ();
+			readonly List<string> _removedFiles = new ();
+			readonly AutoResetEvent _monitor = new (false);
+			readonly DispatchQueue _dispatchQueue = new (nameof (FSEventStreamTest));
+			readonly List<Exception> _exceptions = new ();
+			readonly string _rootPath;
+			readonly FSEventStreamCreateFlags _createFlags;
+
+			public TestFSMonitor (
+				string rootPath,
+				FSEventStreamCreateFlags createFlags,
+				long maxFilesToCreate)
+				: base (rootPath, TimeSpan.Zero, createFlags)
+			{
+				_rootPath = rootPath;
+				_createFlags = createFlags;
+
+				_directoriesToCreate = (int)Math.Sqrt(maxFilesToCreate);
+				_filesPerDirectoryToCreate = _directoriesToCreate;
+			}
+
+			public void Run ()
+			{
+				SetDispatchQueue (_dispatchQueue);
+				Assert.IsTrue (Start ());
+
+				var isWorking = true;
+
+				Task.Run (CreateFilesAndWaitForFSEventsThread)
+					.ContinueWith (task => {
+						isWorking = false;
+						if (task.Exception is not null)
+							_exceptions.Add (task.Exception);
+					});
+
+				while (isWorking)
+					NSRunLoop.Current.RunUntil (NSDate.Now.AddSeconds (0.1));
+
+				if (_exceptions.Count > 0) {
+					if (_exceptions.Count > 1)
+						throw new AggregateException (_exceptions);
+					else
+						throw _exceptions[0];
+				}
+
+				Assert.IsEmpty (_createdDirectories);
+				Assert.IsEmpty (_createdFiles);
+				Assert.IsNotEmpty (_removedFiles);
+
+				_removedFiles.Sort ();
+				_createdThenRemovedFiles.Sort ();
+				CollectionAssert.AreEqual (_createdThenRemovedFiles, _removedFiles);
+
+				Console.WriteLine(
+					"Observed {0} files created and then removed (flags: {1})",
+					_createdThenRemovedFiles.Count,
+					_createFlags);
+			}
+
+			void CreateFilesAndWaitForFSEventsThread ()
+			{
+				for (var i = 0; i < _directoriesToCreate; i++) {
+					var level1Path = Path.Combine (_rootPath, Guid.NewGuid ().ToString ());
+
+					lock (_monitor) {
+						_createdDirectories.Add (level1Path);
+						Directory.CreateDirectory (level1Path);
+					}
+
+					for (var j = 0; j < _filesPerDirectoryToCreate; j++) {
+						var level2Path = Path.Combine (level1Path, Guid.NewGuid ().ToString ());
+
+						lock (_monitor) {
+							_createdFiles.Add (level2Path);
+							_createdThenRemovedFiles.Add (level2Path);
+							File.Create (level2Path).Dispose ();
+						}
+					}
+
+					FlushSync ();
+				}
+
+				while (true) {
+					if (!_monitor.WaitOne (s_testTimeout))
+						throw new TimeoutException (
+							$"test has timed out at {s_testTimeout.TotalSeconds}s; " +
+							"increase the timeout or reduce the number of files created");
+
+					if (_createdDirectories.Count == 0 &&
+						_createdFiles.Count == 0 &&
+						_removedFiles.Count == _createdThenRemovedFiles.Count)
+						break;
+				}
+			}
+
+			protected override void OnEvents (FSEvent[] events) {
+				try {
+					lock (_monitor) {
+						foreach (var evnt in events)
+							HandleEvent (evnt);
+					}
+				} catch (Exception e) {
+					_exceptions.Add (e);
+				} finally {
+					_monitor.Set ();
+				}
+
+				void HandleEvent (FSEvent evnt)
+				{
+					Assert.IsNotNull (evnt.Path);
+					// Roslyn analyzer doesn't consider the assert above wrt nullability
+					if (evnt.Path is null)
+						return;
+
+					if (_createFlags.HasFlag (UseExtendedData))
+						Assert.Greater (evnt.FileId, 0);
+
+					if (evnt.Flags.HasFlag (ItemCreated)) {
+						if (evnt.Flags.HasFlag (ItemIsFile)) {
+							_createdFiles.Remove (evnt.Path);
+
+							File.Delete (evnt.Path);
+						}
+
+						if (evnt.Flags.HasFlag (ItemIsDir))
+							_createdDirectories.Remove (evnt.Path);
+					}
+
+					if (evnt.Flags.HasFlag (ItemRemoved) && !_removedFiles.Contains (evnt.Path))
+						_removedFiles.Add (evnt.Path);
+				}
+			}
 		}
 	}
 }