feat(authsign): store additional metadata/fields in certdb (#1126)

This is a major change in that the included DB migrations *must* be run before the new version of `cfssl` is deployed.
This allows for clients (i.e. https://github.com/cloudflare/certmgr) to send some additional optional fields to `/api/v1/cfssl/authsign` to be stored in `certdb`. It also starts saving SANs, common name, and NotBefore from the issued certificates so that they can be queried without having to parse the PEM.

certdb/certdb.go

@@ -1,20 +1,62 @@
 package certdb
 
 import (
+	"encoding/json"
 	"time"
+
+	"github.com/jmoiron/sqlx/types"
 )
 
 // CertificateRecord encodes a certificate and its metadata
 // that will be recorded in a database.
 type CertificateRecord struct {
-	Serial    string    `db:"serial_number"`
-	AKI       string    `db:"authority_key_identifier"`
-	CALabel   string    `db:"ca_label"`
-	Status    string    `db:"status"`
-	Reason    int       `db:"reason"`
-	Expiry    time.Time `db:"expiry"`
-	RevokedAt time.Time `db:"revoked_at"`
-	PEM       string    `db:"pem"`
+	Serial       string         `db:"serial_number"`
+	AKI          string         `db:"authority_key_identifier"`
+	CALabel      string         `db:"ca_label"`
+	Status       string         `db:"status"`
+	Reason       int            `db:"reason"`
+	Expiry       time.Time      `db:"expiry"`
+	RevokedAt    time.Time      `db:"revoked_at"`
+	PEM          string         `db:"pem"`
+	IssuedAt     time.Time      `db:"issued_at"`
+	NotBefore    time.Time      `db:"not_before"`
+	MetadataJSON types.JSONText `db:"metadata"`
+	SANsJSON     types.JSONText `db:"sans"`
+	CommonName   string         `db:"common_name"`
+}
+
+// SetMetadata sets the metadata json
+func (c *CertificateRecord) SetMetadata(meta map[string]interface{}) error {
+	marshaled, err := json.Marshal(meta)
+	if err != nil {
+		return err
+	}
+	c.MetadataJSON = types.JSONText(marshaled)
+	return nil
+}
+
+// GetMetadata returns the json metadata
+func (c *CertificateRecord) GetMetadata() (map[string]interface{}, error) {
+	var meta map[string]interface{}
+	err := c.MetadataJSON.Unmarshal(&meta)
+	return meta, err
+}
+
+// SetSANs sets the list of sans
+func (c *CertificateRecord) SetSANs(meta []string) error {
+	marshaled, err := json.Marshal(meta)
+	if err != nil {
+		return err
+	}
+	c.SANsJSON = types.JSONText(marshaled)
+	return nil
+}
+
+// GetSANs returns the json SANs
+func (c *CertificateRecord) GetSANs() ([]string, error) {
+	var sans []string
+	err := c.SANsJSON.Unmarshal(&sans)
+	return sans, err
 }
 
 // OCSPRecord encodes a OCSP response body and its metadata

certdb/mysql/migrations/002_AddMetadataToCertificates.sql

@@ -0,0 +1,15 @@
+-- +goose Up
+-- SQL in section 'Up' is executed when this migration is applied
+ALTER TABLE certificates
+ADD COLUMN issued_at timestamp DEFAULT '0000-00-00 00:00:00',
+    ADD COLUMN not_before timestamp DEFAULT '0000-00-00 00:00:00',
+    ADD COLUMN metadata JSON,
+    ADD COLUMN sans JSON,
+    ADD COLUMN common_name TEXT;
+-- +goose Down
+-- SQL section 'Down' is executed when this migration is rolled back
+ALTER TABLE certificates DROP COLUMN issued_at,
+    DROP COLUMN not_before,
+    DROP COLUMN metadata,
+    DROP COLUMN sans,
+    DROP COLUMN common_name;

certdb/pg/migrations/002_AddMetadataToCertificates.sql

@@ -0,0 +1,15 @@
+-- +goose Up
+-- SQL in section 'Up' is executed when this migration is applied
+ALTER TABLE certificates
+ADD COLUMN issued_at timestamptz,
+    ADD COLUMN not_before timestamptz,
+    ADD COLUMN metadata jsonb,
+    ADD COLUMN sans jsonb,
+    ADD COLUMN common_name TEXT;
+-- +goose Down
+-- SQL section 'Down' is executed when this migration is rolled back
+ALTER TABLE certificates DROP COLUMN issued_at,
+    DROP COLUMN not_before,
+    DROP COLUMN metadata,
+    DROP COLUMN sans,
+    DROP COLUMN common_name;

certdb/sql/database_accessor.go

@@ -19,8 +19,10 @@ func init() {
 
 const (
 	insertSQL = `
-INSERT INTO certificates (serial_number, authority_key_identifier, ca_label, status, reason, expiry, revoked_at, pem)
-	VALUES (:serial_number, :authority_key_identifier, :ca_label, :status, :reason, :expiry, :revoked_at, :pem);`
+INSERT INTO certificates (serial_number, authority_key_identifier, ca_label, status, reason, expiry, revoked_at, pem,
+	issued_at, not_before, metadata, sans, common_name)
+VALUES (:serial_number, :authority_key_identifier, :ca_label, :status, :reason, :expiry, :revoked_at, :pem,
+	:issued_at, :not_before, :metadata, :sans, :common_name);`
 
 	selectSQL = `
 SELECT %s FROM certificates
@@ -100,14 +102,19 @@ func (d *Accessor) InsertCertificate(cr certdb.CertificateRecord) error {
 	}
 
 	res, err := d.db.NamedExec(insertSQL, &certdb.CertificateRecord{
-		Serial:    cr.Serial,
-		AKI:       cr.AKI,
-		CALabel:   cr.CALabel,
-		Status:    cr.Status,
-		Reason:    cr.Reason,
-		Expiry:    cr.Expiry.UTC(),
-		RevokedAt: cr.RevokedAt.UTC(),
-		PEM:       cr.PEM,
+		Serial:       cr.Serial,
+		AKI:          cr.AKI,
+		CALabel:      cr.CALabel,
+		Status:       cr.Status,
+		Reason:       cr.Reason,
+		Expiry:       cr.Expiry.UTC(),
+		RevokedAt:    cr.RevokedAt.UTC(),
+		PEM:          cr.PEM,
+		IssuedAt:     cr.IssuedAt.UTC(),
+		NotBefore:    cr.NotBefore.UTC(),
+		MetadataJSON: cr.MetadataJSON,
+		SANsJSON:     cr.SANsJSON,
+		CommonName:   cr.CommonName,
 	})
 	if err != nil {
 		return wrapSQLError(err)

certdb/sql/sql_test.go

@@ -9,6 +9,7 @@ import (
 	"github.com/cloudflare/cfssl/certdb/testdb"
 
 	"github.com/jmoiron/sqlx"
+	"github.com/stretchr/testify/require"
 )
 
 const (
@@ -70,7 +71,7 @@ func testInsertCertificateAndGetCertificate(ta TestAccessor, t *testing.T) {
 		Reason: 0,
 		Expiry: expiry,
 	}
-
+	want.SetMetadata(map[string]interface{}{"k": "v"})
 	if err := ta.Accessor.InsertCertificate(want); err != nil {
 		t.Fatal(err)
 	}
@@ -92,6 +93,9 @@ func testInsertCertificateAndGetCertificate(ta TestAccessor, t *testing.T) {
 		want.PEM != got.PEM || !roughlySameTime(got.Expiry, expiry) {
 		t.Errorf("want Certificate %+v, got %+v", want, got)
 	}
+	gotMeta, err := got.GetMetadata()
+	require.NoError(t, err)
+	require.Equal(t, map[string]interface{}{"k": "v"}, gotMeta)
 
 	unexpired, err := ta.Accessor.GetUnexpiredCertificates()
 

certdb/sqlite/migrations/002_AddMetadataToCertificates.sql

@@ -0,0 +1,13 @@
+-- +goose Up
+-- SQL in section 'Up' is executed when this migration is applied
+
+ALTER TABLE certificates ADD COLUMN "issued_at" timestamp;
+ALTER TABLE certificates ADD COLUMN "not_before" timestamp;
+ALTER TABLE certificates ADD COLUMN "metadata" text;
+ALTER TABLE certificates ADD COLUMN "sans" text;
+ALTER TABLE certificates ADD COLUMN "common_name" text;
+
+-- +goose Down
+-- SQL section 'Down' is executed when this migration is rolled back
+
+-- can't drop columns in sqlite

go.mod

@@ -10,23 +10,23 @@ require (
 	github.com/cloudflare/go-metrics v0.0.0-20151117154305-6a9aea36fb41
 	github.com/cloudflare/redoctober v0.0.0-20171127175943-746a508df14c
 	github.com/getsentry/raven-go v0.0.0-20180121060056-563b81fc02b7 // indirect
-	github.com/go-sql-driver/mysql v1.3.0
-	github.com/golang/protobuf v1.3.1 // indirect
+	github.com/go-sql-driver/mysql v1.4.0
 	github.com/google/certificate-transparency-go v1.0.21
 	github.com/jmhodges/clock v0.0.0-20160418191101-880ee4c33548
-	github.com/jmoiron/sqlx v0.0.0-20180124204410-05cef0741ade
+	github.com/jmoiron/sqlx v1.2.0
 	github.com/kisielk/sqlstruct v0.0.0-20150923205031-648daed35d49
 	github.com/kisom/goutils v1.1.0
 	github.com/kylelemons/go-gypsy v0.0.0-20160905020020-08cad365cd28 // indirect
 	github.com/lib/pq v1.3.0
 	github.com/mattn/go-sqlite3 v1.10.0
 	github.com/pkg/errors v0.8.0 // indirect
+	github.com/stretchr/testify v1.3.0
 	github.com/weppos/publicsuffix-go v0.5.0 // indirect
 	github.com/ziutek/mymysql v1.5.4 // indirect
 	github.com/zmap/zcrypto v0.0.0-20191112190257-7f2fe6faf8cf
 	github.com/zmap/zlint/v2 v2.0.0
 	golang.org/x/crypto v0.0.0-20200124225646-8b5121be2f68
 	golang.org/x/lint v0.0.0-20190930215403-16217165b5de
 	golang.org/x/net v0.0.0-20190620200207-3b0461eec859
-	golang.org/x/text v0.3.2 // indirect
+	google.golang.org/appengine v1.6.6 // indirect
 )

go.sum

@@ -21,8 +21,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/getsentry/raven-go v0.0.0-20180121060056-563b81fc02b7 h1:ELaJ1cjF2nEJeIlHXahGme22yG7TK+3jB6IGCq0Cdrc=
 github.com/getsentry/raven-go v0.0.0-20180121060056-563b81fc02b7/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ=
-github.com/go-sql-driver/mysql v1.3.0 h1:pgwjLi/dvffoP9aabwkT3AKpXQM93QARkjFhDDqC1UE=
-github.com/go-sql-driver/mysql v1.3.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
+github.com/go-sql-driver/mysql v1.4.0 h1:7LxgVwFb2hIQtMm87NdgAVfXjnt4OePseqT1tKx+opk=
+github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
 github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/google/certificate-transparency-go v1.0.21 h1:Yf1aXowfZ2nuboBsg7iYGLmwsOARdV86pfH3g95wXmE=
@@ -31,8 +31,8 @@ github.com/jessevdk/go-flags v1.4.0 h1:4IU2WS7AumrZ/40jfhf4QVDMsQwqA7VEHozFRrGAR
 github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
 github.com/jmhodges/clock v0.0.0-20160418191101-880ee4c33548 h1:dYTbLf4m0a5u0KLmPfB6mgxbcV7588bOCx79hxa5Sr4=
 github.com/jmhodges/clock v0.0.0-20160418191101-880ee4c33548/go.mod h1:hGT6jSUVzF6no3QaDSMLGLEHtHSBSefs+MgcDWnmhmo=
-github.com/jmoiron/sqlx v0.0.0-20180124204410-05cef0741ade h1:ryslCsfLTV4Cm/9NXqCJirlbYodWqFiTH454IaSn/fY=
-github.com/jmoiron/sqlx v0.0.0-20180124204410-05cef0741ade/go.mod h1:IiEW3SEiiErVyFdH8NTuWjSifiEQKUoyK3LNqr2kCHU=
+github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA=
+github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
 github.com/kisielk/sqlstruct v0.0.0-20150923205031-648daed35d49 h1:o/c0aWEP/m6n61xlYW2QP4t9424qlJOsxugn5Zds2Rg=
 github.com/kisielk/sqlstruct v0.0.0-20150923205031-648daed35d49/go.mod h1:yyMNCyc/Ib3bDTKd379tNMpB/7/H5TjM2Y9QJ5THLbE=
 github.com/kisom/goutils v1.1.0 h1:z4HEOgAnFq+e1+O4QdVsyDPatJDu5Ei/7w7DRbYjsIA=
@@ -45,8 +45,10 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kylelemons/go-gypsy v0.0.0-20160905020020-08cad365cd28 h1:mkl3tvPHIuPaWsLtmHTybJeoVEW7cbePK73Ir8VtruA=
 github.com/kylelemons/go-gypsy v0.0.0-20160905020020-08cad365cd28/go.mod h1:T/T7jsxVqf9k/zYOqbgNAsANsjxTd1Yq3htjDhQ1H0c=
+github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/lib/pq v1.3.0 h1:/qkRGz8zljWiDcFvgpwUpwIAPu3r07TDvs3Rws+o/pU=
 github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
 github.com/mattn/go-sqlite3 v1.10.0 h1:jbhqpg7tQe4SupckyijYiy0mJJ/pRyHvXf7JdWK860o=
 github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
 github.com/mreiferson/go-httpclient v0.0.0-20160630210159-31f0106b4474/go.mod h1:OQA4XLvDbMgS8P0CevmM4m9Q3Jq4phKUzcocxuGJ5m8=
@@ -89,6 +91,7 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3 h1:0GoQqolDA55aaLxZyTzK/Y2ePZzZTUrRacwib7cNsYQ=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -100,5 +103,7 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd h1:/e+gpKk9r3dJobndpTytxS2gOy6m5uvpg+ISQoEcusQ=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc=
+google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

signer/local/local.go

@@ -21,6 +21,7 @@ import (
 	"net/mail"
 	"net/url"
 	"os"
+	"time"
 
 	"github.com/cloudflare/cfssl/certdb"
 	"github.com/cloudflare/cfssl/config"
@@ -29,7 +30,7 @@ import (
 	"github.com/cloudflare/cfssl/info"
 	"github.com/cloudflare/cfssl/log"
 	"github.com/cloudflare/cfssl/signer"
-	"github.com/google/certificate-transparency-go"
+	ct "github.com/google/certificate-transparency-go"
 	"github.com/google/certificate-transparency-go/client"
 	"github.com/google/certificate-transparency-go/jsonclient"
 
@@ -509,15 +510,24 @@ func (s *Signer) Sign(req signer.SignRequest) (cert []byte, err error) {
 			Serial: certTBS.SerialNumber.String(),
 			// this relies on the specific behavior of x509.CreateCertificate
 			// which sets the AuthorityKeyId from the signer's SubjectKeyId
-			AKI:     hex.EncodeToString(parsedCert.AuthorityKeyId),
-			CALabel: req.Label,
-			Status:  "good",
-			Expiry:  certTBS.NotAfter,
-			PEM:     string(signedCert),
+			AKI:        hex.EncodeToString(parsedCert.AuthorityKeyId),
+			CALabel:    req.Label,
+			Status:     "good",
+			Expiry:     certTBS.NotAfter,
+			PEM:        string(signedCert),
+			IssuedAt:   time.Now(),
+			NotBefore:  certTBS.NotBefore,
+			CommonName: certTBS.Subject.CommonName,
 		}
 
-		err = s.dbAccessor.InsertCertificate(certRecord)
-		if err != nil {
+		if err := certRecord.SetMetadata(req.Metadata); err != nil {
+			return nil, err
+		}
+		if err := certRecord.SetSANs(certTBS.DNSNames); err != nil {
+			return nil, err
+		}
+
+		if err := s.dbAccessor.InsertCertificate(certRecord); err != nil {
 			return nil, err
 		}
 		log.Debug("saved certificate with serial number ", certTBS.SerialNumber)

signer/signer.go

@@ -70,6 +70,9 @@ type SignRequest struct {
 	// be passed to SignFromPrecert with the SCTs in order to create a
 	// valid certificate.
 	ReturnPrecert bool
+
+	// Arbitrary metadata to be stored in certdb.
+	Metadata map[string]interface{} `json:"metadata"`
 }
 
 // appendIf appends to a if s is not an empty string.
@@ -193,8 +196,8 @@ func ParseCertificateRequest(s Signer, p *config.SigningProfile, csrBytes []byte
 		IPAddresses:        csrv.IPAddresses,
 		EmailAddresses:     csrv.EmailAddresses,
 		URIs:               csrv.URIs,
-		Extensions:			csrv.Extensions,
-		ExtraExtensions:	[]pkix.Extension{},
+		Extensions:         csrv.Extensions,
+		ExtraExtensions:    []pkix.Extension{},
 	}
 
 	for _, val := range csrv.Extensions {
@@ -216,7 +219,7 @@ func ParseCertificateRequest(s Signer, p *config.SigningProfile, csrBytes []byte
 			template.MaxPathLenZero = template.MaxPathLen == 0
 		} else {
 			// If the profile has 'copy_extensions' to true then lets add it
-			if (p.CopyExtensions) {
+			if p.CopyExtensions {
 				template.ExtraExtensions = append(template.ExtraExtensions, val)
 			}
 		}