Merge pull request #784 from jedi22/directory_monitoring

Adding ability to monitor whole folders

include/osquery/filesystem.h

@@ -26,6 +26,13 @@ namespace osquery {
  * this many wildcards.
  */
 const unsigned int kMaxDirectoryTraversalDepth = 40;
+typedef unsigned int ReturnSetting;
+enum {
+  REC_LIST_FILES = 0x1, // Return only files
+  REC_LIST_FOLDERS = 0x2, // Return only folders
+  REC_EVENT_OPT = 0x4, // Enable optimizations for file event resolutions
+  REC_LIST_ALL = REC_LIST_FILES | REC_LIST_FOLDERS
+};
 
 const std::string kWildcardCharacter = "%";
 const std::string kWildcardCharacterRecursive =
@@ -125,6 +132,30 @@ Status listDirectoriesInDirectory(const boost::filesystem::path& path,
 Status resolveFilePattern(const boost::filesystem::path& fs_path,
                           std::vector<std::string>& results);
 
+/**
+ * @brief Given a wildcard filesystem patten, resolve all possible paths
+ *
+ * @code{.cpp}
+ *   std::vector<std::string> results;
+ *   auto s = resolveFilePattern("/Users/marpaia/Downloads/%", results);
+ *   if (s.ok()) {
+ *     for (const auto& result : results) {
+ *       LOG(INFO) << result;
+ *     }
+ *   }
+ * @endcode
+ *
+ * @param fs_path The filesystem pattern
+ * @param results The vector in which all results will be returned
+ * @param setting Do you want files returned, folders or both?
+ *
+ * @return An instance of osquery::Status which indicates the success or
+ * failure of the operation
+ */
+Status resolveFilePattern(const boost::filesystem::path& fs_path,
+                          std::vector<std::string>& results,
+                          ReturnSetting setting);
+
 /**
  * @brief Get directory portion of a path.
  *

osquery/config/config.cpp

@@ -105,13 +105,15 @@ Status Config::genConfig(OsqueryConfig& conf) {
     }
 
     if (tree.count("additional_monitoring") > 0) {
+      ReturnSetting settings = REC_LIST_FOLDERS | REC_EVENT_OPT;
       for (const pt::ptree::value_type& v :
            tree.get_child("additional_monitoring")) {
         if (v.first == "file_paths") {
           for (const pt::ptree::value_type& file_cat : v.second) {
             for (const pt::ptree::value_type& file : file_cat.second) {
               osquery::resolveFilePattern(file.second.get_value<std::string>(),
-                                          conf.eventFiles[file_cat.first]);
+                                          conf.eventFiles[file_cat.first],
+                                          settings);
             }
           }
         }

osquery/config/config_tests.cpp

@@ -71,8 +71,8 @@ TEST_F(ConfigTests, test_threatfiles_execute) {
   auto files = Config::getInstance().getWatchedFiles();
 
   EXPECT_EQ(files.size(), 2);
-  EXPECT_EQ(files["downloads"].size(), 9);
-  EXPECT_EQ(files["system_binaries"].size(), 5);
+  EXPECT_EQ(files["downloads"].size(), 1);
+  EXPECT_EQ(files["system_binaries"].size(), 2);
 }
 }
 

osquery/core/test_util.cpp

@@ -254,7 +254,7 @@ void createMockFileStructure() {
                                         "/deep11/deep2/deep3/");
   boost::filesystem::create_directories(kFakeDirectory + "/deep1/deep2/");
   writeTextFile(kFakeDirectory + "/root.txt", "root");
-  writeTextFile(kFakeDirectory + "/toor.txt", "toor");
+  writeTextFile(kFakeDirectory + "/door.txt", "toor");
   writeTextFile(kFakeDirectory + "/roto.txt", "roto");
   writeTextFile(kFakeDirectory + "/deep1/level1.txt", "l1");
   writeTextFile(kFakeDirectory + "/deep11/not_bash", "l1");

osquery/events/darwin/fsevents.cpp

@@ -3,7 +3,7 @@
  *  All rights reserved.
  *
  *  This source code is licensed under the BSD-style license found in the
- *  LICENSE file in the root directory of this source tree. An additional grant 
+ *  LICENSE file in the root directory of this source tree. An additional grant
  *  of patent rights can be found in the PATENTS file in the same directory.
  *
  */
@@ -20,6 +20,7 @@ namespace osquery {
 std::map<FSEventStreamEventFlags, std::string> kMaskActions = {
     {kFSEventStreamEventFlagItemChangeOwner, "ATTRIBUTES_MODIFIED"},
     {kFSEventStreamEventFlagItemXattrMod, "ATTRIBUTES_MODIFIED"},
+    {kFSEventStreamEventFlagItemInodeMetaMod, "ATTRIBUTES_MODIFIED"},
     {kFSEventStreamEventFlagItemCreated, "CREATED"},
     {kFSEventStreamEventFlagItemRemoved, "DELETED"},
     {kFSEventStreamEventFlagItemModified, "UPDATED"},
@@ -156,7 +157,6 @@ void FSEventsEventPublisher::Callback(
         break;
       }
     }
-
     ec->path = std::string(((char**)event_paths)[i]);
     EventFactory::fire<FSEventsEventPublisher>(ec);
   }
@@ -165,6 +165,9 @@ void FSEventsEventPublisher::Callback(
 bool FSEventsEventPublisher::shouldFire(
     const FSEventsSubscriptionContextRef& mc,
     const FSEventsEventContextRef& ec) const {
+  // This is stopping us from getting events on links.
+  // If we need this feature later, this line will have to be updated to
+  // understand links.
   ssize_t found = ec->path.find(mc->path);
   if (found != 0) {
     return false;

osquery/events/darwin/fsevents_tests.cpp

@@ -3,7 +3,7 @@
  *  All rights reserved.
  *
  *  This source code is licensed under the BSD-style license found in the
- *  LICENSE file in the root directory of this source tree. An additional grant 
+ *  LICENSE file in the root directory of this source tree. An additional grant
  *  of patent rights can be found in the PATENTS file in the same directory.
  *
  */
@@ -29,9 +29,7 @@ int kMaxEventLatency = 3000;
 
 class FSEventsTests : public testing::Test {
  protected:
-  void TearDown() {
-    boost::filesystem::remove_all(kRealTestPath);
-  }
+  void TearDown() { boost::filesystem::remove_all(kRealTestPath); }
 
   void StartEventLoop() {
     event_pub_ = std::make_shared<FSEventsEventPublisher>();

osquery/events/events.cpp

@@ -3,7 +3,7 @@
  *  All rights reserved.
  *
  *  This source code is licensed under the BSD-style license found in the
- *  LICENSE file in the root directory of this source tree. An additional grant 
+ *  LICENSE file in the root directory of this source tree. An additional grant
  *  of patent rights can be found in the PATENTS file in the same directory.
  *
  */
@@ -401,8 +401,7 @@ Status EventFactory::run(EventPublisherID& type_id) {
   EventPublisherRef publisher;
   try {
     publisher = EventFactory::getInstance().getEventPublisher(type_id);
-  }
-  catch (std::out_of_range& e) {
+  } catch (std::out_of_range& e) {
     return Status(1, "No event type found");
   }
 
@@ -418,7 +417,8 @@ Status EventFactory::run(EventPublisherID& type_id) {
 
   // The runloop status is not reflective of the event type's.
   publisher->tearDown();
-  VLOG(1) << "Event publisher " << publisher->type() << " runloop terminated";
+  VLOG(1) << "Event publisher " << publisher->type()
+          << " runloop terminated for reason: " << status.getMessage();
   return Status(0, "OK");
 }
 
@@ -493,8 +493,7 @@ Status EventFactory::addSubscription(EventPublisherID& type_id,
   EventPublisherRef publisher;
   try {
     publisher = getInstance().getEventPublisher(type_id);
-  }
-  catch (std::out_of_range& e) {
+  } catch (std::out_of_range& e) {
     return Status(1, "No event type found");
   }
 
@@ -508,8 +507,7 @@ size_t EventFactory::numSubscriptions(EventPublisherID& type_id) {
   EventPublisherRef publisher;
   try {
     publisher = EventFactory::getInstance().getEventPublisher(type_id);
-  }
-  catch (std::out_of_range& e) {
+  } catch (std::out_of_range& e) {
     return 0;
   }
   return publisher->numSubscriptions();
@@ -539,8 +537,7 @@ Status EventFactory::deregisterEventPublisher(EventPublisherID& type_id) {
   EventPublisherRef publisher;
   try {
     publisher = ef.getEventPublisher(type_id);
-  }
-  catch (std::out_of_range& e) {
+  } catch (std::out_of_range& e) {
     return Status(1, "No event publisher to deregister");
   }
 

osquery/events/events_tests.cpp

@@ -3,7 +3,7 @@
  *  All rights reserved.
  *
  *  This source code is licensed under the BSD-style license found in the
- *  LICENSE file in the root directory of this source tree. An additional grant 
+ *  LICENSE file in the root directory of this source tree. An additional grant
  *  of patent rights can be found in the PATENTS file in the same directory.
  *
  */

osquery/events/linux/inotify.cpp

@@ -63,6 +63,21 @@ void INotifyEventPublisher::tearDown() {
   inotify_handle_ = -1;
 }
 
+Status INotifyEventPublisher::restartMonitoring(){
+  if (last_restart_ != 0 && getUnixTime() - last_restart_ < 10) {
+    return Status(1, "Overflow");
+  }
+  last_restart_ = getUnixTime();
+  VLOG(1) << "Got an overflow, trying to restart...";
+  for(const auto& desc : descriptors_){
+    removeMonitor(desc, 1);
+  }
+  path_descriptors_.clear();
+  descriptor_paths_.clear();
+  configure();
+  return Status(0, "OK");
+}
+
 Status INotifyEventPublisher::run() {
   // Get a while wrapper for free.
   char buffer[BUFFER_SIZE];
@@ -92,7 +107,10 @@ Status INotifyEventPublisher::run() {
     auto event = reinterpret_cast<struct inotify_event*>(p);
     if (event->mask & IN_Q_OVERFLOW) {
       // The inotify queue was overflown (remove all paths).
-      return Status(1, "Overflow");
+      Status stat = restartMonitoring();
+      if(!stat.ok()){
+        return stat;
+      }
     }
 
     if (event->mask & IN_IGNORED) {
@@ -106,6 +124,9 @@ Status INotifyEventPublisher::run() {
       removeMonitor(event->wd, false);
     } else {
       auto ec = createEventContextFrom(event);
+      if(event->mask & IN_CREATE && isDirectory(ec->path).ok()){
+        addMonitor(ec->path, 1);
+      }
       fire(ec);
     }
     // Continue to iterate
@@ -129,8 +150,6 @@ INotifyEventContextRef INotifyEventPublisher::createEventContextFrom(
     path << "/" << event->name;
   }
   ec->path = path.str();
-
-  // Set the action (may be multiple)
   for (const auto& action : kMaskActions) {
     if (event->mask & action.first) {
       ec->action = action.second;
@@ -179,15 +198,10 @@ bool INotifyEventPublisher::addMonitor(const std::string& path,
   if (recursive && isDirectory(path).ok()) {
     std::vector<std::string> children;
     // Get a list of children of this directory (requesed recursive watches).
-    if (!listFilesInDirectory(path, children).ok()) {
-      return false;
-    }
+    listDirectoriesInDirectory(path, children);
 
     for (const auto& child : children) {
-      // Only watch child directories, a watch on the directory implies files.
-      if (isDirectory(child).ok()) {
-        addMonitor(child, recursive);
-      }
+      addMonitor(child, recursive);
     }
   }
 

osquery/events/linux/inotify.h

@@ -127,12 +127,15 @@ class INotifyEventPublisher
   int getHandle() { return inotify_handle_; }
   /// Get the number of actual INotify active descriptors.
   int numDescriptors() { return descriptors_.size(); }
+  /// If we overflow, try and restart the monitor
+  Status restartMonitoring();
 
   // Consider an event queue if separating buffering from firing/servicing.
   DescriptorVector descriptors_;
   PathDescriptorMap path_descriptors_;
   DescriptorPathMap descriptor_paths_;
   int inotify_handle_;
+  int last_restart_;
 
  public:
   FRIEND_TEST(INotifyTests, test_inotify_optimization);