Add kqueue implementation for monitoring symlinks

patches/freebsd11_osymlink.patch

@@ -0,0 +1,95 @@
+--- ./lib/libc/sys/open.2.orig	2016-09-28 16:25:57.000000000 -0700
++++ ./lib/libc/sys/open.2	2017-02-18 13:20:40.000000000 -0800
+@@ -115,6 +115,7 @@
+ O_FSYNC		synchronous writes
+ O_SYNC		synchronous writes
+ O_NOFOLLOW	do not follow symlinks
++O_SYMLINK	allow open of symlinks
+ O_NOCTTY	ignored
+ O_TTY_INIT	ignored
+ O_DIRECTORY	error if file is not a directory
+@@ -179,6 +180,14 @@
+ .Fn open
+ will fail.
+ .Pp
++If
++.Dv O_SYMLINK
++is used in the mask and the target file passed to
++.Fn open
++is a symbolic link then the
++.Fn open
++will be for the symbolic link itself, not what it links to.
++.Pp
+ When opening a file, a lock with
+ .Xr flock 2
+ semantics can be obtained by setting
+--- ./sys/kern/vfs_vnops.c.orig	2016-09-28 16:24:40.000000000 -0700
++++ ./sys/kern/vfs_vnops.c	2017-02-18 13:20:40.000000000 -0800
+@@ -266,8 +266,8 @@
+ 		}
+ 	} else {
+ 		ndp->ni_cnd.cn_nameiop = LOOKUP;
+-		ndp->ni_cnd.cn_flags = ISOPEN |
+-		    ((fmode & O_NOFOLLOW) ? NOFOLLOW : FOLLOW) | LOCKLEAF;
++		ndp->ni_cnd.cn_flags = ISOPEN | LOCKLEAF |
++		    ((fmode & (O_NOFOLLOW | O_SYMLINK)) ? NOFOLLOW : FOLLOW);
+ 		if (!(fmode & FWRITE))
+ 			ndp->ni_cnd.cn_flags |= LOCKSHARED;
+ 		if (!(vn_open_flags & VN_OPEN_NOAUDIT))
+@@ -303,7 +303,7 @@
+ 	struct flock lf;
+ 	int error, lock_flags, type;
+
+-	if (vp->v_type == VLNK)
++	if (vp->v_type == VLNK && fmode & O_NOFOLLOW)
+ 		return (EMLINK);
+ 	if (vp->v_type == VSOCK)
+ 		return (EOPNOTSUPP);
+@@ -313,6 +313,8 @@
+ 	if (fmode & (FWRITE | O_TRUNC)) {
+ 		if (vp->v_type == VDIR)
+ 			return (EISDIR);
++		if (vp->v_type == VLNK)
++			return (EMLINK);
+ 		accmode |= VWRITE;
+ 	}
+ 	if (fmode & FREAD)
+@@ -777,6 +779,8 @@
+ 	    uio->uio_td, td));
+ 	KASSERT(flags & FOF_OFFSET, ("No FOF_OFFSET"));
+ 	vp = fp->f_vnode;
++	if (vp->v_type == VLNK)
++		return (EOPNOTSUPP);
+ 	ioflag = 0;
+ 	if (fp->f_flag & FNONBLOCK)
+ 		ioflag |= IO_NDELAY;
+@@ -837,6 +841,8 @@
+ 	    uio->uio_td, td));
+ 	KASSERT(flags & FOF_OFFSET, ("No FOF_OFFSET"));
+ 	vp = fp->f_vnode;
++	if (vp->v_type == VLNK)
++		return (EOPNOTSUPP);
+ 	if (vp->v_type == VREG)
+ 		bwillwrite();
+ 	ioflag = IO_UNIT;
+@@ -1306,6 +1312,10 @@
+ 		error = EISDIR;
+ 		goto out;
+ 	}
++	if (vp->v_type == VLNK) {
++		error = EINVAL;
++		goto out;
++	}
+ #ifdef MAC
+ 	error = mac_vnode_check_write(active_cred, fp->f_cred, vp);
+ 	if (error)
+--- ./sys/sys/fcntl.h.orig	2016-09-28 16:24:41.000000000 -0700
++++ ./sys/sys/fcntl.h	2017-02-18 13:20:40.000000000 -0800
+@@ -131,6 +131,7 @@
+
+ #if __BSD_VISIBLE
+ #define	O_VERIFY	0x00200000	/* open only after verification */
++#define	O_SYMLINK	0x00400000	/* allow open of a symlink */
+ #endif
+
+ /*

source/common/filesystem/filesystem_impl.cc

@@ -21,7 +21,9 @@ bool fileExists(const std::string& path) {
 bool directoryExists(const std::string& path) {
   DIR* dir = opendir(path.c_str());
   bool dirExists = nullptr != dir;
-  closedir(dir);
+  if (dirExists) {
+    closedir(dir);
+  }
 
   return dirExists;
 }

source/common/filesystem/watcher_impl.h

@@ -1,41 +1,5 @@
-#pragma once
-
-#include "envoy/event/dispatcher.h"
-#include "envoy/filesystem/filesystem.h"
-
-#include "common/common/logger.h"
-
-namespace Filesystem {
-
-/**
- * Implementation of Watcher that uses inotify. inotify is an awful API. In order to make this work
- * in a somewhat sane way we always watch the directory that owns the thing being watched, and then
- * filter for events that are relevant to the the thing being watched.
- */
-class WatcherImpl : public Watcher, Logger::Loggable<Logger::Id::file> {
-public:
-  WatcherImpl(Event::Dispatcher& dispatcher);
-  ~WatcherImpl();
-
-  // Filesystem::Watcher
-  void addWatch(const std::string& path, uint32_t events, OnChangedCb cb) override;
-
-private:
-  struct FileWatch {
-    std::string file_;
-    uint32_t events_;
-    OnChangedCb cb_;
-  };
-
-  struct DirectoryWatch {
-    std::list<FileWatch> watches_;
-  };
-
-  void onInotifyEvent();
-
-  int inotify_fd_;
-  Event::FileEventPtr inotify_event_;
-  std::unordered_map<int, DirectoryWatch> callback_map_;
-};
-
-} // Filesystem
+#if defined(LINUX)
+#include "common/filesystem/watcher_impl_linux.h"
+#elif defined(__FreeBSD__)
+#include "common/filesystem/watcher_impl_bsd.h"
+#endif

source/common/filesystem/watcher_impl_bsd.cc

@@ -0,0 +1,117 @@
+#include "watcher_impl_bsd.h"
+
+#include "envoy/common/exception.h"
+#include "envoy/event/dispatcher.h"
+#include "envoy/event/file_event.h"
+
+#include "common/common/assert.h"
+#include "common/common/utility.h"
+
+#include "event2/event.h"
+
+#include <sys/fcntl.h>
+#include <sys/types.h>
+#include <sys/event.h>
+
+
+namespace Filesystem {
+
+WatcherImpl::WatcherImpl(Event::Dispatcher& dispatcher)
+    : queue_(kqueue()),
+      kqueue_event_(dispatcher.createFileEvent(queue_, [this](uint32_t events) -> void {
+        if (events & Event::FileReadyType::Read) {
+          onKqueueEvent();
+        }
+      }, Event::FileTriggerType::Edge)) {}
+
+WatcherImpl::~WatcherImpl() {
+  close(queue_);
+
+  for (FileWatchPtr &file : watches_) {
+    close(file->fd_);
+    file->removeFromList(watches_);
+  }
+}
+
+void WatcherImpl::addWatch(const std::string& path, uint32_t events, Watcher::OnChangedCb cb) {
+  FileWatchPtr watch = addWatch_(path, events, cb);
+  if (watch == nullptr) {
+    throw EnvoyException(fmt::format("invalid watch path {}", path));
+  }
+  watch->moveIntoList(std::move(watch), watches_);
+}
+
+WatcherImpl::FileWatchPtr WatcherImpl::addWatch_(const std::string& path, uint32_t events,
+                                                 Watcher::OnChangedCb cb) {
+  int watch_fd = open(path.c_str(), O_SYMLINK);
+  if (watch_fd == -1) {
+    return nullptr;
+  }
+
+  FileWatchPtr watch(new FileWatch());
+  watch->fd_ = watch_fd;
+  watch->file_ = path;
+  watch->events_ = events;
+  watch->callback_ = cb;
+
+  struct kevent event;
+  EV_SET(&event, watch_fd, EVFILT_VNODE, EV_ADD | EV_CLEAR, NOTE_DELETE | NOTE_RENAME,
+         0, watch.get());
+
+  if (kevent(queue_, &event, 1, NULL, 0, NULL) == -1) {
+    throw EnvoyException(
+        fmt::format("unable to add filesystem watch for file {}: {}", path, strerror(errno)));
+  }
+
+  if (event.flags & EV_ERROR) {
+    throw EnvoyException(
+        fmt::format("unable to add filesystem watch for file {}: {}", path, strerror(event.data)));
+  }
+
+  log_debug("added watch for file: '{}' fd: {}", path, watch_fd);
+  return watch;
+}
+
+void WatcherImpl::onKqueueEvent() {
+  struct kevent event = {};
+  timespec nullts = {0, 0};
+
+  while (true) {
+      int nevents = kevent(queue_, NULL, 0, &event, 1, &nullts);
+      if (nevents < 1 || event.udata == nullptr) {
+        return;
+      }
+
+      FileWatch* file = static_cast<FileWatch*>(event.udata);
+      // kqueue doesn't seem to work well with NOTE_RENAME and O_SYMLINK, so instead if we
+      // get a NOTE_DELETE on the symlink we check if there is another file with the same
+      // name we assume a NOTE_RENAME and re-attach another event to the new file.
+      if (event.fflags & NOTE_DELETE) {
+        close(file->fd_);
+
+        FileWatchPtr newFile = addWatch_(file->file_, file->events_, file->callback_);
+        file->removeFromList(watches_);
+        if (newFile == nullptr) {
+          return;
+        }
+
+        event.fflags |= NOTE_RENAME;
+        file = newFile.get();
+        newFile->moveIntoList(std::move(newFile), watches_);
+      }
+
+      uint32_t events = 0;
+      if (event.fflags & NOTE_RENAME) {
+        events |= Events::MovedTo;
+      }
+
+      if (events & file->events_) {
+        log_debug("matched callback: file: {}", file->file_);
+        file->callback_(events);
+      }
+
+      log_debug("notification: fd: {} flags: {:x} file: {}", file->fd_, event.fflags, file->file_);
+  }
+}
+
+} // Filesystem

source/common/filesystem/watcher_impl_bsd.h

@@ -0,0 +1,41 @@
+#pragma once
+
+#include "envoy/event/dispatcher.h"
+#include "envoy/filesystem/filesystem.h"
+
+#include "common/common/linked_object.h"
+#include "common/common/logger.h"
+
+namespace Filesystem {
+
+/**
+ * Implementation of Watcher that uses kqueue. kqueue API is way more elegant than inotify
+ * and allows multiple files monitoring.
+ */
+class WatcherImpl : public Watcher, Logger::Loggable<Logger::Id::file> {
+public:
+  WatcherImpl(Event::Dispatcher& dispatcher);
+  ~WatcherImpl();
+
+  // Filesystem::Watcher
+  void addWatch(const std::string& path, uint32_t events, OnChangedCb cb) override;
+
+private:
+  struct FileWatch : LinkedObject<FileWatch> {
+    int fd_;
+    uint32_t events_;
+    std::string file_;
+    OnChangedCb callback_;
+  };
+
+  typedef std::unique_ptr<FileWatch> FileWatchPtr;
+
+  void onKqueueEvent();
+  FileWatchPtr addWatch_(const std::string& path, uint32_t events, Watcher::OnChangedCb cb);
+
+  int queue_;
+  std::list<FileWatchPtr> watches_;
+  Event::FileEventPtr kqueue_event_;
+};
+
+} // Filesystem

source/common/filesystem/watcher_impl.cc → source/common/filesystem/watcher_impl_linux.cc

@@ -1,4 +1,4 @@
-#include "watcher_impl.h"
+#include "watcher_impl_linux.h"
 
 #include "envoy/common/exception.h"
 #include "envoy/event/dispatcher.h"

source/common/filesystem/watcher_impl_linux.h

@@ -0,0 +1,41 @@
+#pragma once
+
+#include "envoy/event/dispatcher.h"
+#include "envoy/filesystem/filesystem.h"
+
+#include "common/common/logger.h"
+
+namespace Filesystem {
+
+/**
+ * Implementation of Watcher that uses inotify. inotify is an awful API. In order to make this work
+ * in a somewhat sane way we always watch the directory that owns the thing being watched, and then
+ * filter for events that are relevant to the the thing being watched.
+ */
+class WatcherImpl : public Watcher, Logger::Loggable<Logger::Id::file> {
+public:
+  WatcherImpl(Event::Dispatcher& dispatcher);
+  ~WatcherImpl();
+
+  // Filesystem::Watcher
+  void addWatch(const std::string& path, uint32_t events, OnChangedCb cb) override;
+
+private:
+  struct FileWatch {
+    std::string file_;
+    uint32_t events_;
+    OnChangedCb cb_;
+  };
+
+  struct DirectoryWatch {
+    std::list<FileWatch> watches_;
+  };
+
+  void onInotifyEvent();
+
+  int inotify_fd_;
+  Event::FileEventPtr inotify_event_;
+  std::unordered_map<int, DirectoryWatch> callback_map_;
+};
+
+} // Filesystem

source/common/tracing/http_tracer_impl.cc

@@ -295,4 +295,4 @@ void LightStepRecorder::onSuccess(Http::MessagePtr&& msg) {
   }
 }
 
-} // Tracing
+} // Tracing

source/precompiled/precompiled.h

@@ -18,8 +18,9 @@
 #include <regex>
 #include <signal.h>
 #include <sstream>
+#include <sys/socket.h>
+#include <netinet/in.h>
 #include <string.h>
-#include <sys/signalfd.h>
 #include <unistd.h>
 #include <unordered_set>