[#1773] Introduce multi-pack configuration syntax (#2787)

docs/wiki/deployment/configuration.md

@@ -135,6 +135,17 @@ The pack value may also be a string, such as:
 
 If using a string instead of an inline JSON dictionary the configuration plugin will be asked to "generate" that resource. In the case of the default **filesystem** plugin, these strings are considered paths.
 
+The **filesystem** plugin supports another convention for adding a directory of packs:
+```json
+{
+  "packs": {
+    "*": "/path/to/*",
+  }
+}
+```
+
+Here the name `*` asks the plugin to *glob* the value and construct a multi-pack. The name of each pack will correspond to the filename *leaf* without the final extension, e.g. `/path/to/external_pack.conf` will be named `external_pack`.
+
 Queries added to the schedule from packs inherit the pack name as part of the scheduled query name identifier. For example, consider the embedded `active_directory` query above, it is in the `internal_stuff` pack so the scheduled query name becomes: `pack_internal_stuff_active_directory`. The delimiter can be changed using the `--pack_delimiter=_`, see the [CLI Options](../installation/cli-flags.md) for more details.
 
 ### Discovery queries

include/osquery/config.h

@@ -113,10 +113,14 @@ class Config : private boost::noncopyable {
   void hashSource(const std::string& source, const std::string& content);
 
   /// Whether or not the last loaded config was valid.
-  bool isValid() const { return valid_; }
+  bool isValid() const {
+    return valid_;
+  }
 
   /// Get start time of config.
-  size_t getStartTime() const { return start_time_; }
+  size_t getStartTime() const {
+    return start_time_;
+  }
 
   /**
    * @brief Add a pack to the osquery schedule
@@ -322,6 +326,7 @@ class Config : private boost::noncopyable {
   FRIEND_TEST(OptionsConfigParserPluginTests, test_get_option);
   FRIEND_TEST(EventsConfigParserPluginTests, test_get_event);
   FRIEND_TEST(PacksTests, test_discovery_cache);
+  FRIEND_TEST(PacksTests, test_multi_pack);
   FRIEND_TEST(SchedulerTests, test_monitor);
   FRIEND_TEST(SchedulerTests, test_config_results_purge);
   FRIEND_TEST(EventsTests, test_event_subscriber_configure);
@@ -488,7 +493,9 @@ class ConfigParserPlugin : public Plugin {
    *
    * More complex parsers that require dynamic casting are not recommended.
    */
-  const boost::property_tree::ptree& getData() const { return data_; }
+  const boost::property_tree::ptree& getData() const {
+    return data_;
+  }
 
  protected:
   /// Allow the config to request parser state resets.
@@ -501,4 +508,16 @@ class ConfigParserPlugin : public Plugin {
  private:
   friend class Config;
 };
+
+/**
+ * @brief Boost's 1.59 property tree based JSON parser does not accept comments.
+ *
+ * For semi-compatibility with existing configurations we will attempt to strip
+ * hash and C++ style comments. It is OK for the config update to be latent
+ * as it is a single event. But some configuration plugins may update running
+ * configurations.
+ *
+ * @parms json A mutable input/output string that will contain stripped JSON.
+ */
+void stripConfigComments(std::string& json);
 }

include/osquery/filesystem.h

@@ -36,9 +36,9 @@ inline GlobLimits operator|(GlobLimits a, GlobLimits b) {
 }
 
 /// Globbing wildcard character.
-const std::string kSQLGlobWildcard = "%";
+const std::string kSQLGlobWildcard{"%"};
 /// Globbing wildcard recursive character (double wildcard).
-const std::string kSQLGlobRecursive = kSQLGlobWildcard + kSQLGlobWildcard;
+const std::string kSQLGlobRecursive{kSQLGlobWildcard + kSQLGlobWildcard};
 
 /**
  * @brief Read a file from disk.

include/osquery/packs.h

@@ -15,8 +15,8 @@
 #include <string>
 #include <vector>
 
-#include <boost/property_tree/ptree.hpp>
 #include <boost/noncopyable.hpp>
+#include <boost/property_tree/ptree.hpp>
 
 #include <osquery/database.h>
 
@@ -39,6 +39,7 @@ class Pack : private boost::noncopyable {
  public:
   Pack(const std::string& name, const boost::property_tree::ptree& tree)
       : Pack(name, "", tree) {}
+
   Pack(const std::string& name,
        const std::string& source,
        const boost::property_tree::ptree& tree) {
@@ -77,7 +78,9 @@ class Pack : private boost::noncopyable {
   /// Returns the minimum version that the pack is configured to run on
   const std::string& getVersion() const;
 
-  size_t getShard() const { return shard_; }
+  size_t getShard() const {
+    return shard_;
+  }
 
   /// Returns the schedule dictated by the pack
   const std::map<std::string, ScheduledQuery>& getSchedule() const;
@@ -147,7 +150,7 @@ class Pack : private boost::noncopyable {
    *
    * Initialization must include pack content
    */
-  Pack(){};
+  Pack() {}
 
  private:
   FRIEND_TEST(PacksTests, test_check_platform);

osquery/config/config.cpp

@@ -251,14 +251,29 @@ Config::Config()
 void Config::addPack(const std::string& name,
                      const std::string& source,
                      const pt::ptree& tree) {
-  RecursiveLock wlock(config_schedule_mutex_);
-  try {
-    schedule_->add(std::make_shared<Pack>(name, source, tree));
-    if (schedule_->last()->shouldPackExecute()) {
-      applyParsers(source + FLAGS_pack_delimiter + name, tree, true);
+  auto addSinglePack = ([this, &source](const std::string pack_name,
+                                        const pt::ptree& pack_tree) {
+    RecursiveLock wlock(config_schedule_mutex_);
+    try {
+      schedule_->add(std::make_shared<Pack>(pack_name, source, pack_tree));
+      if (schedule_->last()->shouldPackExecute()) {
+        applyParsers(
+            source + FLAGS_pack_delimiter + pack_name, pack_tree, true);
+      }
+    } catch (const std::exception& e) {
+      LOG(WARNING) << "Error adding pack: " << pack_name << ": " << e.what();
     }
-  } catch (const std::exception& e) {
-    LOG(WARNING) << "Error adding pack: " << name << ": " << e.what();
+  });
+
+  if (name == "*") {
+    // This is a multi-pack, expect the config plugin to have generated a
+    // "name": {pack-content} response similar to embedded pack content
+    // within the configuration.
+    for (const auto& pack : tree) {
+      addSinglePack(pack.first, pack.second);
+    }
+  } else {
+    addSinglePack(name, tree);
   }
 }
 
@@ -354,15 +369,7 @@ Status Config::load() {
   return status;
 }
 
-/**
- * @brief Boost's 1.59 property tree based JSON parser does not accept comments.
- *
- * For semi-compatibility with existing configurations we will attempt to strip
- * hash and C++ style comments. It is OK for the config update to be latent
- * as it is a single event. But some configuration plugins may update running
- * configurations.
- */
-inline void stripConfigComments(std::string& json) {
+void stripConfigComments(std::string& json) {
   std::string sink;
 
   boost::replace_all(json, "\\\n", "");

osquery/config/plugins/filesystem.cpp

@@ -11,14 +11,18 @@
 #include <vector>
 
 #include <boost/filesystem/operations.hpp>
+#include <boost/property_tree/ptree.hpp>
 
 #include <osquery/config.h>
 #include <osquery/filesystem.h>
 #include <osquery/flags.h>
 #include <osquery/logger.h>
 
-namespace fs = boost::filesystem;
+#include "osquery/core/json.h"
+
 namespace errc = boost::system::errc;
+namespace fs = boost::filesystem;
+namespace pt = boost::property_tree;
 
 namespace osquery {
 
@@ -63,10 +67,51 @@ Status FilesystemConfigPlugin::genConfig(
 Status FilesystemConfigPlugin::genPack(const std::string& name,
                                        const std::string& value,
                                        std::string& pack) {
+  if (name == "*") {
+    // The config requested a multi-pack.
+    std::vector<std::string> paths;
+    resolveFilePattern(value, paths);
+
+    pt::ptree multi_pack;
+    for (const auto& path : paths) {
+      std::string content;
+      if (!readFile(path, content)) {
+        LOG(WARNING) << "Cannot read multi-pack file: " << path;
+        continue;
+      }
+
+      // Assemble an intermediate property tree for simplified parsing.
+      pt::ptree single_pack;
+      stripConfigComments(content);
+      try {
+        std::stringstream json_stream;
+        json_stream << content;
+        pt::read_json(json_stream, single_pack);
+      } catch (const pt::json_parser::json_parser_error& /* e */) {
+        LOG(WARNING) << "Cannot read multi-pack JSON: " << path;
+        continue;
+      }
+
+      multi_pack.put_child(fs::path(path).stem().string(), single_pack);
+    }
+
+    // We should have a property tree of pack content mimicking embedded
+    // configuration packs, ready to parse as a string.
+    std::ostringstream output;
+    pt::write_json(output, multi_pack, false);
+    pack = output.str();
+    if (pack.empty()) {
+      return Status(1, "Multi-pack content empty");
+    }
+
+    return Status(0);
+  }
+
   boost::system::error_code ec;
   if (!fs::is_regular_file(value, ec) || ec.value() != errc::success) {
     return Status(1, value + " is not a valid path");
   }
+
   return readFile(value, pack);
 }
 }

osquery/config/tests/packs_tests.cpp

@@ -150,6 +150,30 @@ TEST_F(PacksTests, test_discovery_cache) {
   c.reset();
 }
 
+TEST_F(PacksTests, test_multi_pack) {
+  std::string multi_pack_content = "{\"first\": {}, \"second\": {}}";
+  pt::ptree multi_pack;
+
+  {
+    // Convert the content into the expected pack form (ptree).
+    std::stringstream json_stream;
+    json_stream << multi_pack_content;
+    pt::read_json(json_stream, multi_pack);
+  }
+
+  Config c;
+  c.addPack("*", "", multi_pack);
+
+  std::vector<std::string> pack_names;
+  c.packs(([&pack_names](std::shared_ptr<Pack>& p) {
+    pack_names.push_back(p->getName());
+  }));
+
+  std::vector<std::string> expected = {"first", "second"};
+  ASSERT_EQ(expected.size(), pack_names.size());
+  EXPECT_EQ(expected, pack_names);
+}
+
 TEST_F(PacksTests, test_discovery_zero_state) {
   Pack pack("discovery_pack", getPackWithDiscovery());
   auto stats = pack.getStats();