validation: Add config validation mode. (#499) (#863)

This adds a --mode command-line flag. --mode=serve is the default and can be omitted; in that mode Envoy runs as it always has.

With --mode=validate Envoy checks that the configuration file in --config-file is valid, then exits. "Valid" means that it performs the JSON schema check as usual, but also initializes as much of its internal state as possible, exiting with any errors as it would under --mode=serve. When initialization is complete, instead of listening for traffic, Envoy prints an "OK" message to stderr and exits, returning success.

In validation mode, to the extent possible, Envoy avoids interfering with its environment. For example, the hot-restart process is skipped, so validation can safely be run on the same machine as a serving Envoy. Validation mode also doesn't send any upstream traffic or open any listeners.

It does attempt to read any files referenced in the config, like certs and private keys, so validation will fail if those files aren't located at the expected paths. A future "lite validation" mode, that mocks out the filesystem as well as the network, is not yet implemented as of this patch.

docs/operations/cli.rst

@@ -10,6 +10,17 @@ following are the command line options that Envoy supports.
 
   *(required)* The path to the :ref:`JSON configuration file <config>`.
 
+.. option:: --mode <string>
+
+  *(optional)* One of the operating modes for Envoy:
+
+  * ``serve``: *(default)* Validate the JSON configuration and then serve traffic normally.
+
+  * ``validate``: Validate the JSON configuration and then exit, printing either an "OK" message (in
+    which case the exit code is 0) or any errors generated by the configuration file (exit code 1).
+    No network traffic is generated, and the hot restart process is not performed, so no other Envoy
+    process on the machine will be disturbed.
+
 .. option:: --admin-address-path <path string>
 
   *(optional)* The output file path where the admin address and port will be written.

include/envoy/server/options.h

@@ -11,6 +11,28 @@
 namespace Envoy {
 namespace Server {
 
+/**
+ * Whether to run Envoy in serving mode, or in config validation mode at one of two levels (in which
+ * case we'll verify the configuration file is valid, print any errors, and exit without serving.)
+ */
+enum class Mode {
+  /**
+   * Default mode: Regular Envoy serving process. Configs are validated in the normal course of
+   * initialization, but if all is well we proceed to serve traffic.
+   */
+  Serve,
+
+  /**
+   * Validate as much as possible without opening network connections upstream or downstream.
+   */
+  Validate,
+
+  // TODO(rlazarus): Add a third option for "light validation": Mock out access to the filesystem.
+  // Perform no validation of files referenced in the config, such as runtime configs, SSL certs,
+  // etc. Validation will pass even if those files are malformed or don't exist, allowing the config
+  // to be validated in a non-prod environment.
+};
+
 /**
  * General options for the server.
  */
@@ -62,6 +84,12 @@ class Options {
    */
   virtual uint64_t restartEpoch() PURE;
 
+  /**
+   * @return whether to verify the configuration file is valid, print any errors, and exit
+   *         without serving.
+   */
+  virtual Mode mode() const PURE;
+
   /**
     * @return std::chrono::milliseconds the duration in msec between log flushes.
     */

include/envoy/upstream/BUILD

@@ -15,9 +15,12 @@ envoy_cc_library(
         ":load_balancer_interface",
         ":thread_local_cluster_interface",
         ":upstream_interface",
+        "//include/envoy/access_log:access_log_interface",
         "//include/envoy/http:async_client_interface",
         "//include/envoy/http:conn_pool_interface",
         "//include/envoy/json:json_object_interface",
+        "//include/envoy/local_info:local_info_interface",
+        "//include/envoy/runtime:runtime_interface",
     ],
 )
 

include/envoy/upstream/cluster_manager.h

@@ -6,9 +6,12 @@
 #include <string>
 #include <unordered_map>
 
+#include "envoy/access_log/access_log.h"
 #include "envoy/http/async_client.h"
 #include "envoy/http/conn_pool.h"
 #include "envoy/json/json_object.h"
+#include "envoy/local_info/local_info.h"
+#include "envoy/runtime/runtime.h"
 #include "envoy/upstream/load_balancer.h"
 #include "envoy/upstream/thread_local_cluster.h"
 #include "envoy/upstream/upstream.h"
@@ -103,6 +106,8 @@ class ClusterManager {
   virtual void shutdown() PURE;
 };
 
+typedef std::unique_ptr<ClusterManager> ClusterManagerPtr;
+
 /**
  * Global configuration for any SDS clusters.
  */
@@ -139,6 +144,16 @@ class ClusterManagerFactory {
 public:
   virtual ~ClusterManagerFactory() {}
 
+  /**
+   * Allocate a cluster manager from configuration JSON.
+   */
+  virtual ClusterManagerPtr clusterManagerFromJson(const Json::Object& config, Stats::Store& stats,
+                                                   ThreadLocal::Instance& tls,
+                                                   Runtime::Loader& runtime,
+                                                   Runtime::RandomGenerator& random,
+                                                   const LocalInfo::LocalInfo& local_info,
+                                                   AccessLog::AccessLogManager& log_manager) PURE;
+
   /**
    * Allocate an HTTP connection pool.
    */

source/common/common/assert.h

@@ -10,7 +10,7 @@ namespace Envoy {
 #define RELEASE_ASSERT(X)                                                                          \
   {                                                                                                \
     if (!(X)) {                                                                                    \
-      Logger::Registry::getLog(Logger::Id::assert)                                                 \
+      Envoy::Logger::Registry::getLog(Envoy::Logger::Id::assert)                                   \
           .critical("assert failure: {}: {}:{}", #X, __FILE__, __LINE__);                          \
       abort();                                                                                     \
     }                                                                                              \
@@ -26,7 +26,7 @@ namespace Envoy {
  * Indicate a panic situation and exit.
  */
 #define PANIC(X)                                                                                   \
-  Logger::Registry::getLog(Logger::Id::assert)                                                     \
+  Envoy::Logger::Registry::getLog(Envoy::Logger::Id::assert)                                       \
       .critical("panic: {}: {}:{}", X, __FILE__, __LINE__);                                        \
   abort();
 

source/common/http/async_client_impl.cc

@@ -40,6 +40,7 @@ AsyncClient::Request* AsyncClientImpl::send(MessagePtr&& request, AsyncClient::C
                                             const Optional<std::chrono::milliseconds>& timeout) {
   AsyncRequestImpl* async_request =
       new AsyncRequestImpl(std::move(request), *this, callbacks, timeout);
+  async_request->initialize();
   std::unique_ptr<AsyncStreamImpl> new_request{async_request};
 
   // The request may get immediately failed. If so, we will return nullptr.
@@ -155,7 +156,9 @@ AsyncRequestImpl::AsyncRequestImpl(MessagePtr&& request, AsyncClientImpl& parent
                                    AsyncClient::Callbacks& callbacks,
                                    const Optional<std::chrono::milliseconds>& timeout)
     : AsyncStreamImpl(parent, *this, timeout), request_(std::move(request)), callbacks_(callbacks) {
+}
 
+void AsyncRequestImpl::initialize() {
   sendHeaders(request_->headers(), !request_->body());
   if (!remoteClosed() && request_->body()) {
     sendData(*request_->body(), true);

source/common/http/async_client_impl.h

@@ -217,6 +217,7 @@ class AsyncRequestImpl final : public AsyncClient::Request,
   virtual void cancel() override;
 
 private:
+  void initialize();
   void onComplete();
 
   // AsyncClient::StreamCallbacks

source/common/upstream/cluster_manager_impl.cc

@@ -552,6 +552,14 @@ ClusterManagerImpl::ThreadLocalClusterManagerImpl::ClusterEntry::connPool(
   return container.pools_[enumToInt(priority)].get();
 }
 
+ClusterManagerPtr ProdClusterManagerFactory::clusterManagerFromJson(
+    const Json::Object& config, Stats::Store& stats, ThreadLocal::Instance& tls,
+    Runtime::Loader& runtime, Runtime::RandomGenerator& random,
+    const LocalInfo::LocalInfo& local_info, AccessLog::AccessLogManager& log_manager) {
+  return ClusterManagerPtr{
+      new ClusterManagerImpl(config, *this, stats, tls, runtime, random, local_info, log_manager)};
+}
+
 Http::ConnectionPool::InstancePtr
 ProdClusterManagerFactory::allocateConnPool(Event::Dispatcher& dispatcher, HostConstSharedPtr host,
                                             ResourcePriority priority) {

source/common/upstream/cluster_manager_impl.h

@@ -38,6 +38,11 @@ class ProdClusterManagerFactory : public ClusterManagerFactory {
         local_info_(local_info) {}
 
   // Upstream::ClusterManagerFactory
+  ClusterManagerPtr clusterManagerFromJson(const Json::Object& config, Stats::Store& stats,
+                                           ThreadLocal::Instance& tls, Runtime::Loader& runtime,
+                                           Runtime::RandomGenerator& random,
+                                           const LocalInfo::LocalInfo& local_info,
+                                           AccessLog::AccessLogManager& log_manager) override;
   Http::ConnectionPool::InstancePtr allocateConnPool(Event::Dispatcher& dispatcher,
                                                      HostConstSharedPtr host,
                                                      ResourcePriority priority) override;

source/exe/BUILD

@@ -26,6 +26,7 @@ envoy_cc_library(
         "//source/common/event:libevent_lib",
         "//source/common/local_info:local_info_lib",
         "//source/common/network:utility_lib",
+        "//source/common/stats:stats_lib",
         "//source/common/stats:thread_local_store_lib",
         "//source/server:drain_manager_lib",
         "//source/server:options_lib",
@@ -58,6 +59,7 @@ envoy_cc_library(
         "//source/common/common:compiler_requirements_lib",
         ":envoy_common_lib",
         ":hot_restart_lib",
+        "//source/server/config_validation:server_lib",
     ] + select({
         "//bazel:disable_signal_trace": [],
         "//conditions:default": [":sigaction_lib"],

source/exe/main.cc

@@ -5,6 +5,7 @@
 #include "common/event/libevent.h"
 #include "common/local_info/local_info_impl.h"
 #include "common/network/utility.h"
+#include "common/stats/stats_impl.h"
 #include "common/stats/thread_local_store.h"
 
 #include "exe/hot_restart.h"
@@ -13,6 +14,7 @@
 #include "exe/signal_action.h"
 #endif
 
+#include "server/config_validation/server.h"
 #include "server/drain_manager_impl.h"
 #include "server/options_impl.h"
 #include "server/server.h"
@@ -45,10 +47,24 @@ int main(int argc, char** argv) {
   // Enabled by default. Control with "bazel --define=signal_trace=disabled"
   Envoy::SignalAction handle_sigs;
 #endif
-  ares_library_init(ARES_LIB_INIT_ALL);
   Envoy::Event::Libevent::Global::initialize();
   Envoy::OptionsImpl options(argc, argv, Envoy::Server::SharedMemory::version(),
                              spdlog::level::warn);
+  Envoy::Server::ProdComponentFactory component_factory;
+  Envoy::LocalInfo::LocalInfoImpl local_info(
+      Envoy::Network::Utility::getLocalAddress(Envoy::Network::Address::IpVersion::v4),
+      options.serviceZone(), options.serviceClusterName(), options.serviceNodeName());
+
+  switch (options.mode()) {
+  case Envoy::Server::Mode::Serve:
+    break;
+  case Envoy::Server::Mode::Validate:
+    Envoy::Thread::MutexBasicLockable log_lock;
+    Envoy::Logger::Registry::initialize(options.logLevel(), log_lock);
+    return Envoy::Server::validateConfig(options, component_factory, local_info) ? 0 : 1;
+  }
+
+  ares_library_init(ARES_LIB_INIT_ALL);
 
   std::unique_ptr<Envoy::Server::HotRestartImpl> restarter;
   try {
@@ -61,11 +77,7 @@ int main(int argc, char** argv) {
   Envoy::Logger::Registry::initialize(options.logLevel(), restarter->logLock());
   Envoy::DefaultTestHooks default_test_hooks;
   Envoy::Stats::ThreadLocalStoreImpl stats_store(*restarter);
-  Envoy::Server::ProdComponentFactory component_factory;
   // TODO(henna): Add CLI option for local address IP version.
-  Envoy::LocalInfo::LocalInfoImpl local_info(
-      Envoy::Network::Utility::getLocalAddress(Envoy::Network::Address::IpVersion::v4),
-      options.serviceZone(), options.serviceClusterName(), options.serviceNodeName());
   Envoy::Server::InstanceImpl server(options, default_test_hooks, *restarter, stats_store,
                                      restarter->accessLogLock(), component_factory, local_info);
   server.run();

source/server/BUILD

@@ -136,13 +136,13 @@ envoy_cc_library(
         "//source/common/api:api_lib",
         "//source/common/common:utility_lib",
         "//source/common/common:version_lib",
-        "//source/common/json:config_schemas_lib",
         "//source/common/memory:stats_lib",
         "//source/common/network:utility_lib",
         "//source/common/runtime:runtime_lib",
         "//source/common/ssl:context_lib",
         "//source/common/stats:statsd_lib",
         "//source/common/thread_local:thread_local_lib",
+        "//source/common/upstream:cluster_manager_lib",
         "//source/server/http:admin_lib",
     ],
 )

source/server/config_validation/BUILD

@@ -0,0 +1,98 @@
+licenses(["notice"])  # Apache 2
+
+load("//bazel:envoy_build_system.bzl", "envoy_cc_library", "envoy_package")
+
+envoy_package()
+
+envoy_cc_library(
+    name = "api_lib",
+    srcs = ["api.cc"],
+    hdrs = ["api.h"],
+    deps = [
+        ":dispatcher_lib",
+        "//include/envoy/api:api_interface",
+        "//include/envoy/filesystem:filesystem_interface",
+        "//source/common/api:api_lib",
+    ],
+)
+
+envoy_cc_library(
+    name = "async_client_lib",
+    srcs = ["async_client.cc"],
+    hdrs = ["async_client.h"],
+    deps = [
+        "//include/envoy/http:async_client_interface",
+        "//include/envoy/http:message_interface",
+    ],
+)
+
+envoy_cc_library(
+    name = "cluster_manager_lib",
+    srcs = ["cluster_manager.cc"],
+    hdrs = ["cluster_manager.h"],
+    deps = [
+        ":async_client_lib",
+        "//include/envoy/upstream:cluster_manager_interface",
+        "//source/common/upstream:cluster_manager_lib",
+    ],
+)
+
+envoy_cc_library(
+    name = "connection_handler_lib",
+    srcs = ["connection_handler.cc"],
+    hdrs = ["connection_handler.h"],
+    deps = [
+        "//include/envoy/api:api_interface",
+        "//include/envoy/network:connection_handler_interface",
+        "//include/envoy/network:filter_interface",
+        "//include/envoy/network:listen_socket_interface",
+        "//source/common/common:assert_lib",
+    ],
+)
+
+envoy_cc_library(
+    name = "dispatcher_lib",
+    srcs = ["dispatcher.cc"],
+    hdrs = ["dispatcher.h"],
+    deps = [
+        "//include/envoy/event:dispatcher_interface",
+        "//source/common/common:assert_lib",
+        "//source/common/event:dispatcher_lib",
+    ],
+)
+
+envoy_cc_library(
+    name = "dns_lib",
+    srcs = ["dns.cc"],
+    hdrs = ["dns.h"],
+    deps = [
+        "//include/envoy/event:dispatcher_interface",
+        "//include/envoy/network:dns_interface",
+    ],
+)
+
+envoy_cc_library(
+    name = "server_lib",
+    srcs = ["server.cc"],
+    hdrs = ["server.h"],
+    deps = [
+        ":api_lib",
+        ":cluster_manager_lib",
+        ":connection_handler_lib",
+        ":dns_lib",
+        "//include/envoy/common:optional",
+        "//include/envoy/server:drain_manager_interface",
+        "//include/envoy/server:instance_interface",
+        "//include/envoy/ssl:context_manager_interface",
+        "//include/envoy/tracing:http_tracer_interface",
+        "//source/common/access_log:access_log_manager_lib",
+        "//source/common/common:assert_lib",
+        "//source/common/runtime:runtime_lib",
+        "//source/common/ssl:context_lib",
+        "//source/common/stats:stats_lib",
+        "//source/common/thread_local:thread_local_lib",
+        "//source/server:configuration_lib",
+        "//source/server:server_lib",
+        "//source/server/http:admin_lib",
+    ],
+)

source/server/config_validation/api.cc

@@ -0,0 +1,16 @@
+#include "server/config_validation/api.h"
+
+#include "server/config_validation/dispatcher.h"
+
+namespace Envoy {
+namespace Api {
+
+ValidationImpl::ValidationImpl(std::chrono::milliseconds file_flush_interval_msec)
+    : Impl(file_flush_interval_msec) {}
+
+Event::DispatcherPtr ValidationImpl::allocateDispatcher() {
+  return Event::DispatcherPtr{new Event::ValidationDispatcher()};
+}
+
+} // Api
+} // Envoy