[Heartbeat] Add Additional ECS tls.* fields (#17687)

Work in support of elastic/uptime#161

This patch adds additional ECS [TLS](https://www.elastic.co/guide/en/ecs/current/ecs-tls.html) and [x509](elastic/ecs#762) fields. Note that we are blocked on the x509 fields which are not yet merged into ECS.

Sample output of the `tls.*` fields with this patch is below. Note the somewhat strange nesting of data in `issuer` and `subject`. This is per the ECS spec, but a bit awkward. We may want to break this data out into the more specific ECS `x509` type in the future. For UI work we are likely fine to parse this on the client and display the CN section in most cases. I did break out the CN into its own field in `x509.subject/issuer.common_name`. However, if we do want to aggregate on issuer in the future it's good to have the full distinguished name to do that on.

This PR also refactors some `libbeat` code around parsing TLS versions and adds test coverage there as well.

```json
{
	"tls": {
		"certificate_not_valid_after": "2020-07-16T03:15:39Z",
		"certificate_not_valid_before": "2019-08-16T01:40:25Z",
		"server": {
			"hash": {
				"sha1": "b7b4b89ef0d0caf39d223736f0fdbb03c7b426f1",
				"sha256": "12b00d04db0db8caa302bfde043e88f95baceb91e86ac143e93830b4bbec726d"
			},
			"x509": {
				"issuer": {
					"common_name": "GlobalSign CloudSSL CA - SHA256 - G3",
					"distinguished_name": "CN=GlobalSign CloudSSL CA - SHA256 - G3,O=GlobalSign nv-sa,C=BE"
				},
				"not_after": "2020-07-16T03:15:39Z",
				"not_before": "2019-08-16T01:40:25Z",
				"public_key_algorithm": "RSA",
				"public_key_size": 2048,
				"serial_number": "26610543540289562361990401194",
				"signature_algorithm": "SHA256-RSA",
				"subject": {
					"common_name": "r2.shared.global.fastly.net",
					"distinguished_name": "CN=r2.shared.global.fastly.net,O=Fastly\\, Inc.,L=San Francisco,ST=California,C=US"
				}
			}
		}
	}
}
```

## How to test this PR locally

Run against TLS/Non-TLS endpoints

CHANGELOG.next.asciidoc

@@ -279,7 +279,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 *Heartbeat*
 
 - Allow a list of status codes for HTTP checks. {pull}15587[15587]
-
+- Add additional ECS compatible fields for TLS information. {pull}17687[17687]
 
 *Journalbeat*
 

heartbeat/_meta/fields.common.yml

@@ -17,11 +17,19 @@
           type: keyword
           description: >
             The monitors configured name
+          multi_fields:
+            - name: text
+              type: text
+              analyzer: simple
 
         - name: id
           type: keyword
           description: >
             The monitors full job ID as used by heartbeat.
+          multi_fields:
+            - name: text
+              type: text
+              analyzer: simple
 
         - name: duration
           type: group

heartbeat/docs/fields.asciidoc

@@ -215,6 +215,13 @@ type: keyword
 
 --
 
+*`monitor.name.text`*::
++
+--
+type: text
+
+--
+
 *`monitor.id`*::
 +
 --
@@ -225,6 +232,13 @@ type: keyword
 
 --
 
+*`monitor.id.text`*::
++
+--
+type: text
+
+--
+
 [float]
 === duration
 
@@ -7824,7 +7838,10 @@ TLS layer related fields.
 *`tls.certificate_not_valid_before`*::
 +
 --
-Earliest time at which the connection's certificates are valid.
+
+deprecated:[7.8.0]
+
+Deprecated in favor of `tls.server.x509.not_before`. Earliest time at which the connection's certificates are valid.
 
 type: date
 
@@ -7833,7 +7850,10 @@ type: date
 *`tls.certificate_not_valid_after`*::
 +
 --
-Latest time at which the connection's certificates are valid.
+
+deprecated:[7.8.0]
+
+Deprecated in favor of `tls.server.x509.not_after`. Latest time at which the connection's certificates are valid.
 
 type: date
 
@@ -7862,3 +7882,180 @@ type: long
 
 --
 
+[float]
+=== server
+
+Detailed x509 certificate metadata
+
+
+
+*`tls.server.x509.alternative_names`*::
++
+--
+List of subject alternative names (SAN). Name types vary by certificate authority and certificate type but commonly contain IP addresses, DNS names (and wildcards), and email addresses.
+
+type: keyword
+
+example: *.elastic.co
+
+--
+
+
+*`tls.server.x509.issuer.common_name`*::
++
+--
+List of common name (CN) of issuing certificate authority.
+
+type: keyword
+
+example: DigiCert SHA2 High Assurance Server CA
+
+--
+
+*`tls.server.x509.issuer.common_name.text`*::
++
+--
+type: wildcard
+
+--
+
+*`tls.server.x509.issuer.distinguished_name`*::
++
+--
+Distinguished name (DN) of issuing certificate authority.
+
+type: keyword
+
+example: C=US, O=DigiCert Inc, OU=www.digicert.com, CN=DigiCert SHA2 High Assurance Server CA
+
+--
+
+*`tls.server.x509.not_after`*::
++
+--
+Time at which the certificate is no longer considered valid.
+
+type: date
+
+example: 2020-07-16 03:15:39
+
+--
+
+*`tls.server.x509.not_before`*::
++
+--
+Time at which the certificate is first considered valid.
+
+type: date
+
+example: 2019-08-16 01:40:25
+
+--
+
+*`tls.server.x509.public_key_algorithm`*::
++
+--
+Algorithm used to generate the public key.
+
+type: keyword
+
+example: RSA
+
+--
+
+*`tls.server.x509.public_key_curve`*::
++
+--
+The curve used by the elliptic curve public key algorithm. This is algorithm specific.
+
+type: keyword
+
+example: nistp521
+
+--
+
+*`tls.server.x509.public_key_exponent`*::
++
+--
+Exponent used to derive the public key. This is algorithm specific.
+
+type: long
+
+example: 65537
+
+--
+
+*`tls.server.x509.public_key_size`*::
++
+--
+The size of the public key space in bits.
+
+type: long
+
+example: 2048
+
+--
+
+*`tls.server.x509.serial_number`*::
++
+--
+Unique serial number issued by the certificate authority. For consistency, if this value is alphanumeric, it should be formatted without colons and uppercase characters.
+
+type: keyword
+
+example: 55FBB9C7DEBF09809D12CCAA
+
+--
+
+*`tls.server.x509.signature_algorithm`*::
++
+--
+Identifier for certificate signature algorithm. Recommend using names found in Go Lang Crypto library (See https://github.com/golang/go/blob/go1.14/src/crypto/x509/x509.go#L337-L353).
+
+type: keyword
+
+example: SHA256-RSA
+
+--
+
+
+*`tls.server.x509.subject.subject.common_name`*::
++
+--
+List of common names (CN) of subject.
+
+type: keyword
+
+example: r2.shared.global.fastly.net
+
+--
+
+*`tls.server.x509.subject.subject.common_name.text`*::
++
+--
+type: wildcard
+
+--
+
+*`tls.server.x509.subject.subject.distinguished_name`*::
++
+--
+Distinguished name (DN) of the certificate subject entity.
+
+type: keyword
+
+example: C=US, ST=California, L=San Francisco, O=Fastly, Inc., CN=r2.shared.global.fastly.net
+
+--
+
+*`tls.server.x509.version_number`*::
++
+--
+Version of x509 format.
+
+type: keyword
+
+example: 3
+
+--
+

heartbeat/hbtest/hbtestutil.go

@@ -18,17 +18,23 @@
 package hbtest
 
 import (
+	"crypto/tls"
 	"crypto/x509"
 	"fmt"
 	"io"
 	"io/ioutil"
+	"net"
 	"net/http"
 	"net/http/httptest"
 	"net/url"
 	"os"
 	"strconv"
 	"strings"
 	"testing"
+	"time"
+
+	"github.com/elastic/beats/v7/heartbeat/monitors/active/dialchain/tlsmeta"
+	"github.com/elastic/beats/v7/libbeat/common"
 
 	"github.com/elastic/beats/v7/heartbeat/hbtestllext"
 
@@ -107,13 +113,25 @@ func ServerPort(server *httptest.Server) (uint16, error) {
 
 // TLSChecks validates the given x509 cert at the given position.
 func TLSChecks(chainIndex, certIndex int, certificate *x509.Certificate) validator.Validator {
-	return lookslike.MustCompile(map[string]interface{}{
-		"tls": map[string]interface{}{
-			"rtt.handshake.us":             isdef.IsDuration,
-			"certificate_not_valid_before": certificate.NotBefore,
-			"certificate_not_valid_after":  certificate.NotAfter,
-		},
-	})
+	expected := common.MapStr{}
+	// This function is well tested independently, so we just test that things match up here.
+	tlsmeta.AddTLSMetadata(expected, tls.ConnectionState{
+		Version:           tls.VersionTLS13,
+		HandshakeComplete: true,
+		CipherSuite:       tls.TLS_AES_128_GCM_SHA256,
+		ServerName:        certificate.Subject.CommonName,
+		PeerCertificates:  []*x509.Certificate{certificate},
+	}, time.Duration(1))
+
+	expected.Put("tls.rtt.handshake.us", isdef.IsDuration)
+
+	return lookslike.MustCompile(expected)
+}
+
+func TLSCertChecks(certificate *x509.Certificate) validator.Validator {
+	expected := common.MapStr{}
+	tlsmeta.AddCertMetadata(expected, []*x509.Certificate{certificate})
+	return lookslike.MustCompile(expected)
 }
 
 // BaseChecks creates a skima.Validator that represents the "monitor" field present
@@ -196,6 +214,14 @@ func ErrorChecks(msgSubstr string, errType string) validator.Validator {
 	})
 }
 
+func ExpiredCertChecks(cert *x509.Certificate) validator.Validator {
+	msg := x509.CertificateInvalidError{Cert: cert, Reason: x509.Expired}.Error()
+	return lookslike.Compose(
+		ErrorChecks(msg, "io"),
+		TLSCertChecks(cert),
+	)
+}
+
 // RespondingTCPChecks creates a skima.Validator that represents the "tcp" field present
 // in all heartbeat events that use a Tcp connection as part of their DialChain
 func RespondingTCPChecks() validator.Validator {
@@ -215,3 +241,23 @@ func CertToTempFile(t *testing.T, cert *x509.Certificate) *os.File {
 	certFile.WriteString(x509util.CertToPEMString(cert))
 	return certFile
 }
+
+func StartHTTPSServer(t *testing.T, tlsCert tls.Certificate) (host string, port string, cert *x509.Certificate, doClose func() error) {
+	cert, err := x509.ParseCertificate(tlsCert.Certificate[0])
+	require.NoError(t, err)
+
+	// No need to start a real server, since this is invalid, we just
+	l, err := tls.Listen("tcp", "127.0.0.1:0", &tls.Config{
+		Certificates: []tls.Certificate{tlsCert},
+	})
+	require.NoError(t, err)
+
+	srv := &http.Server{Handler: HelloWorldHandler(200)}
+	go func() {
+		srv.Serve(l)
+	}()
+
+	host, port, err = net.SplitHostPort(l.Addr().String())
+	require.NoError(t, err)
+	return host, port, cert, srv.Close
+}