readme/notes: safety, etc/fsevents issue reproducer

etc/wip-fsevents-issue/.gitignore

@@ -0,0 +1 @@
+fsevents-issue

etc/wip-fsevents-issue/build-and-test.sh

@@ -0,0 +1,15 @@
+#! /usr/bin/env bash
+(
+  cd "$(dirname "$0")"
+  ./build.sh
+  n=0
+  ok=0
+  while [ $ok -eq 0 ]
+  do
+    ./fsevents-issue
+    ok=$?
+    n=$((n + 1))
+  done
+  echo $n before crash
+)
+

etc/wip-fsevents-issue/build.sh

@@ -0,0 +1,7 @@
+#! /usr/bin/env bash
+(
+  cd "$(dirname "$0")"
+  c++ -framework CoreFoundation -framework CoreServices -std=c++17 main.cpp -o fsevents-issue
+  install_name_tool -add_rpath /usr/local/lib fsevents-issue
+)
+

etc/wip-fsevents-issue/main.cpp

@@ -0,0 +1,147 @@
+#include <array>
+#include <CoreFoundation/CoreFoundation.h>
+#include <CoreServices/CoreServices.h>
+#include <dispatch/dispatch.h>
+#include <filesystem>
+#include <fstream>
+#include <iostream>
+#include <memory>
+#include <set>
+#include <string>
+#include <tuple>
+
+using namespace std;
+
+using Ctx = set<string>;
+
+auto path_from_event_at(void* event_recv_paths, unsigned long i)
+  -> filesystem::path
+{
+  if (event_recv_paths)
+    if (
+      void const* from_arr = CFArrayGetValueAtIndex(
+        static_cast<CFArrayRef>(event_recv_paths),
+        static_cast<CFIndex>(i)))
+      if (
+        void const* from_dict = CFDictionaryGetValue(
+          static_cast<CFDictionaryRef>(from_arr),
+          kFSEventStreamEventExtendedDataPathKey))
+        if (
+          char const* as_cstr = CFStringGetCStringPtr(
+            static_cast<CFStringRef>(from_dict),
+            kCFStringEncodingUTF8))
+          return {as_cstr};
+
+  return {};
+}
+
+auto event_recv(
+  ConstFSEventStreamRef,
+  void* maybe_ctx,
+  unsigned long count,
+  void* paths,
+  unsigned const* flags,
+  FSEventStreamEventId const*) -> void
+{
+  if (maybe_ctx && paths && flags) {
+    auto ctx = static_cast<Ctx*>(maybe_ctx);
+    for (unsigned long i = 0; i < count; i++) {
+      auto ev_path = path_from_event_at(paths, i);
+      auto [at, inserted] = ctx->insert(ev_path);
+      assert(inserted);
+      cout << *at << "\n";
+    }
+  }
+}
+
+auto open_event_stream(
+  filesystem::path const& path,
+  dispatch_queue_t queue,
+  void* ctx) -> FSEventStreamRef
+{
+  auto context = FSEventStreamContext{
+    .version = 0,
+    .info = ctx,
+    .retain = nullptr,
+    .release = nullptr,
+    .copyDescription = nullptr,
+  };
+
+  void const* path_cfstring =
+    CFStringCreateWithCString(nullptr, path.c_str(), kCFStringEncodingUTF8);
+  auto const path_array =
+    CFArrayCreate(nullptr, &path_cfstring, 1, &kCFTypeArrayCallBacks);
+
+  auto fsev_listen_since = kFSEventStreamEventIdSinceNow;
+  unsigned fsev_listen_for = kFSEventStreamCreateFlagFileEvents
+                           | kFSEventStreamCreateFlagUseExtendedData
+                           | kFSEventStreamCreateFlagUseCFTypes;
+  FSEventStreamRef stream = FSEventStreamCreate(
+    nullptr,
+    &event_recv,
+    &context,
+    path_array,
+    fsev_listen_since,
+    0.016,
+    fsev_listen_for);
+
+  if (stream && queue) {
+    FSEventStreamSetDispatchQueue(stream, queue);
+    FSEventStreamStart(stream);
+    return stream;
+  }
+  else
+    return nullptr;
+}
+
+auto close_event_stream(FSEventStreamRef s) -> bool
+{
+  if (s) {
+    FSEventStreamFlushSync(s);
+    FSEventStreamStop(s);
+    auto event_id = FSEventsGetCurrentEventId();
+    auto device = FSEventStreamGetDeviceBeingWatched(s);
+    FSEventsPurgeEventsForDeviceUpToEventId(device, event_id);
+    FSEventStreamInvalidate(s);
+    FSEventStreamRelease(s);
+    s = nullptr;
+    return true;
+  }
+  else
+    return false;
+}
+
+auto main() -> int
+{
+  auto queue = dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0);
+  constexpr auto n_watchers = 5;
+  constexpr auto event_count = 500;
+
+  using Watcher = tuple<unique_ptr<FSEventStreamRef>, unique_ptr<Ctx>>;
+
+  // Some number of watchers
+  auto watchers = array<Watcher, n_watchers>{};
+  for (auto& w : watchers) {
+    auto ctx = make_unique<Ctx>();
+    auto watcher = make_unique<FSEventStreamRef>(
+      open_event_stream("/tmp", queue, ctx.get()));
+    w = {std::move(watcher), std::move(ctx)};
+  }
+
+  // Some events in the background
+  for (auto i = 0; i < event_count; i++) {
+    auto path = "/tmp/hi" + to_string(i);
+    auto _ = ofstream(path);  // touch
+    filesystem::remove(path);
+  }
+
+  // Clean up (we never get here during a crash)
+  cout << "ok" << endl;
+  for (auto& w : watchers) {
+    auto& [watcher, ctx] = w;
+    auto owned = watcher.release();
+    close_event_stream(*owned);
+  }
+
+  return 0;
+}

readme.md

@@ -299,9 +299,43 @@ cd out
 <details>
 <summary>Safety and Darwin</summary>
 
-Unsafe destruction.
+Darwin's adapter, `FSEvents` and `dispatch`, is
+not safe during destruction. It's not clear to me
+what the root problem is, though it appears to be
+a bug in Apple's implementation of one of these APIs
+which allows it to call into the associated callback
+of an already-destroyed `FSEvents` stream. This only
+seems to happen during heavy load -- many `FSEvents`
+streams coming and going and many filesystem events
+being created. Here is an examplem backtrace:
 
-... Details ...
+```
+... Userland symbols here and above in garbage memory ...
+4   FSEvents                      	       0x18cf54ef0 implementation_callback_rpc + 3656
+5   FSEvents                      	       0x18cf54020 _Xcallback_rpc + 220
+6   FSEvents                      	       0x18cf53f18 FSEventsD2F_server + 68
+7   FSEvents                      	       0x18cf57720 receive_and_dispatch_rcv_msg + 316
+8   libdispatch.dylib             	       0x18457f910 _dispatch_client_callout + 20
+9   libdispatch.dylib             	       0x184582dc8 _dispatch_continuation_pop + 600
+10  libdispatch.dylib             	       0x184596be4 _dispatch_source_latch_and_call + 420
+11  libdispatch.dylib             	       0x1845957b4 _dispatch_source_invoke + 832
+12  libdispatch.dylib             	       0x18459261c _dispatch_root_queue_drain_deferred_wlh + 288
+13  libdispatch.dylib             	       0x184591e90 _dispatch_workloop_worker_thread + 404
+14  libsystem_pthread.dylib       	       0x184729114 _pthread_wqthread + 288
+15  libsystem_pthread.dylib       	       0x184727e30 start_wqthread + 8
+```
+
+[This performance test](https://github.com/e-dant/watcher/blob/next/devel/src/wtr/test_watcher/test_performance.cpp#L23)
+seems to trigger the issue most reliably. The test for
+[opening and closing watchers](https://github.com/e-dant/watcher/blob/next/devel/src/wtr/test_watcher/test_openclose.cpp#L7)
+without many filesystem events in the background is a
+less reliable reproducer. Purging old events when closing
+the `FSEvents` stream seems to ameliorate, but not fully
+prevent, the issue. A minimal reproducer is [here](https://github.com/e-dant/watcher/blob/next/etc/wip-fsevents-issue/main.cpp).
+
+```
+
+```
 
 </details>
 
@@ -403,9 +437,16 @@ consider waiting a few seconds.
 <details>
 <summary>Unsupported events</summary>
 
-The owner and attribute events are not supported.
+None of the platform-specific implementations provide
+information on what attributes were changed from.
+This makes supporting those events dependant on storing
+this information ourselves. Storing maps of paths to
+`stat` structures, diffing them on attribute changes,
+is a non-insignificant memory commitment.
+
+The owner and attribute events are unsupported because
+I'm not sure how to support those events efficienty.
 
-... Details ...
 </details>
 
 <details>