Implement multi VA validation (#2802)

Adds basic multi-path validation functionality. A new method `performRemoteValidation` is added to `boulder-va` which is called if it is configured with a list of remote VA gRPC addresses. In this initial implementation the remote VAs are only used to check the validation result of the main VA, if all of the remote validations succeed but the local validation failed, the overall validation will still fail. Remote VAs use the exact same code as the local VA to perform validation. If the local validation succeeds then a configured quorum of the remote VA successes must be met in order to fully complete the validation.

This implementation assumes that metrics are collected from the remote VAs in order to have visibility into their individual validation latencies etc.

Fixes #2621.

Author: Roland Bracewell Shoemaker

cmd/boulder-va/main.go

@@ -3,6 +3,7 @@ package main
 import (
 	"flag"
 	"os"
+	"strings"
 	"time"
 
 	"github.com/jmhodges/clock"
@@ -38,6 +39,9 @@ type config struct {
 		// Feature flag to enable enforcement of CAA SERVFAILs.
 		CAASERVFAILExceptions string
 
+		RemoteVAs                   []cmd.GRPCClientConfig
+		MaxRemoteValidationFailures int
+
 		Features map[string]bool
 	}
 
@@ -119,18 +123,36 @@ func main() {
 		resolver = r
 	}
 
+	tls, err := c.VA.TLS.Load()
+	cmd.FailOnError(err, "TLS config")
+
+	var remotes []va.RemoteVA
+	if len(c.VA.RemoteVAs) > 0 {
+		for _, rva := range c.VA.RemoteVAs {
+			vaConn, err := bgrpc.ClientSetup(&rva, tls, scope)
+			cmd.FailOnError(err, "Unable to create remote VA client")
+			remotes = append(
+				remotes,
+				va.RemoteVA{
+					bgrpc.NewValidationAuthorityGRPCClient(vaConn),
+					strings.Join(rva.ServerAddresses, ","),
+				},
+			)
+		}
+	}
+
 	vai := va.NewValidationAuthorityImpl(
 		pc,
 		sbc,
 		resolver,
+		remotes,
+		c.VA.MaxRemoteValidationFailures,
 		c.VA.UserAgent,
 		c.VA.IssuerDomain,
 		scope,
 		clk,
 		logger)
 
-	tls, err := c.VA.TLS.Load()
-	cmd.FailOnError(err, "TLS config")
 	grpcSrv, l, err := bgrpc.NewServer(c.VA.GRPC, tls, scope)
 	cmd.FailOnError(err, "Unable to setup VA gRPC server")
 	err = bgrpc.RegisterValidationAuthorityGRPCServer(grpcSrv, vai)

test/config-next/va-remote-a.json

@@ -0,0 +1,39 @@
+{
+  "va": {
+    "CAASERVFAILExceptions": "test/caa-servfail-exceptions.txt",
+    "userAgent": "boulder",
+    "debugAddr": ":8011",
+    "portConfig": {
+      "httpPort": 5002,
+      "httpsPort": 5001,
+      "tlsPort": 5001
+    },
+    "dnsTries": 3,
+    "issuerDomain": "happy-hacker-ca.invalid",
+    "tls": {
+      "caCertfile": "test/grpc-creds/minica.pem",
+      "certFile": "test/grpc-creds/va.boulder/cert.pem",
+      "keyFile": "test/grpc-creds/va.boulder/key.pem"
+    },
+    "grpc": {
+      "address": ":9097",
+      "clientNames": [
+        "va.boulder"
+      ]
+    },
+    "features": {
+      "IPv6First": true
+    }
+  },
+
+  "syslog": {
+    "stdoutlevel": 6,
+    "sysloglevel": 4
+  },
+
+  "common": {
+    "dnsResolver": "127.0.0.1:8053",
+    "dnsTimeout": "1s",
+    "dnsAllowLoopbackAddresses": true
+  }
+}

test/config-next/va-remote-b.json

@@ -0,0 +1,39 @@
+{
+  "va": {
+    "CAASERVFAILExceptions": "test/caa-servfail-exceptions.txt",
+    "userAgent": "boulder",
+    "debugAddr": ":8012",
+    "portConfig": {
+      "httpPort": 5002,
+      "httpsPort": 5001,
+      "tlsPort": 5001
+    },
+    "dnsTries": 3,
+    "issuerDomain": "happy-hacker-ca.invalid",
+    "tls": {
+      "caCertfile": "test/grpc-creds/minica.pem",
+      "certFile": "test/grpc-creds/va.boulder/cert.pem",
+      "keyFile": "test/grpc-creds/va.boulder/key.pem"
+    },
+    "grpc": {
+      "address": ":9098",
+      "clientNames": [
+        "va.boulder"
+      ]
+    },
+    "features": {
+      "IPv6First": true
+    }
+  },
+
+  "syslog": {
+    "stdoutlevel": 6,
+    "sysloglevel": 4
+  },
+
+  "common": {
+    "dnsResolver": "127.0.0.1:8053",
+    "dnsTimeout": "1s",
+    "dnsAllowLoopbackAddresses": true
+  }
+}

test/config-next/va.json

@@ -30,7 +30,17 @@
     "features": {
       "GoogleSafeBrowsingV4": true,
       "IPv6First": true
-    }
+    },
+    "remoteVAs": [
+      {
+        "serverAddresses": ["va.boulder:19097"],
+        "timeout": "15s"
+      },
+      {
+        "serverAddresses": ["va.boulder:19098"],
+        "timeout": "15s"
+      }
+    ]
   },
 
   "syslog": {

test/startservers.py

@@ -54,6 +54,8 @@ def start(race_detection):
             [":19094", "ra.boulder:9094"],
             [":19095", "sa.boulder:9095"],
             [":19096", "ca.boulder:9096"],
+            [":19097", "va.boulder:9097"],
+            [":19098", "va.boulder:9098"]
     ]:
         forward(srv[0], srv[1])
     progs = [
@@ -72,6 +74,12 @@ def start(race_detection):
         'dns-test-srv',
         'mail-test-srv --closeFirst 5'
     ]
+    if default_config_dir.startswith("test/config-next"):
+        # Run the two 'remote' VAs
+        progs.extend([
+            'boulder-va --config %s' % os.path.join(default_config_dir, "va-remote-a.json"),
+            'boulder-va --config %s' % os.path.join(default_config_dir, "va-remote-b.json")
+        ])
     if not install(race_detection):
         return False
     for prog in progs:
@@ -93,6 +101,9 @@ def start(race_detection):
             if not check():
                 return False
             ports = range(8000, 8005) + [4000, 4430]
+            if default_config_dir.startswith("test/config-next"):
+                # Add the two 'remote' VA debug ports
+                ports.extend([8011, 8012])
             for debug_port in ports:
                 s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
                 s.connect(('localhost', debug_port))

va/gsb_test.go

@@ -34,6 +34,8 @@ func TestIsSafeDomain(t *testing.T) {
 		&cmd.PortConfig{},
 		sbc,
 		nil,
+		nil,
+		0,
 		"user agent 1.0",
 		"letsencrypt.org",
 		stats,
@@ -83,6 +85,8 @@ func TestAllowNilInIsSafeDomain(t *testing.T) {
 		&cmd.PortConfig{},
 		nil,
 		nil,
+		nil,
+		0,
 		"user agent 1.0",
 		"letsencrypt.org",
 		stats,