CTFE Extra Data Issuance Chain Deduplication (#1477)

* CTFE Extra Data Issuance Chain Deduplication

* Add CT server with MySQL configuration example

* Update CTFE storage saving changelog

* Fix always if true condition

* Reuse `IsCTFEStorageEnabled`

* Setting default cache size and ttl to -1

* Add code comments for `issuanceChainStorage` and `issuanceChainCache`

* Fix incorrect NewIssuanceChainCache comment

* Refactor `addChainInternal` and remove `buildLogLeafForAddChain`

* Move cache flags to ct_server main

* Remove TODO comment for `[]ct.ASN1Cert` to `[]byte` conversion

* Add FixLogLeaf method comment

* Return err in `GetByHash` and `Add` when storage is nil

* Enable MySQL in ct server test

* Refactor log leaf build logic

* Fix `leaf` var scope bug

* Rename method from public `Add` to private `add`

* Refactor `issuanceChainService.FixLogLeaf` into `rpcGetLeavesByRange` and `rpcGetEntryAndProof`

* Add `rpcGetLeavesByRange` and `rpcGetEntryAndProof` method comment

* Add more information in CHANGELOG.md

* Add table create command in `resetctdb.sh`

* Add `extra_data_issuance_chain_storage_backends` config to integration test

* Fix incorrect config name to `extra_data_issuance_chain_storage_backend`

* Add missing `mysql://` in `ctfe_storage_connection_string` config

* Update the hostname to `db` in `ctfe_storage_connection_string` config

* Update the hostname to `mysql` in `ctfe_storage_connection_string` config

* Update the tested MySQL version to 8.4 in CHANGELOG.md

* Import MySQL schema in integration test flow

* Fix missing `then` in `resetctdb.sh`

* Update integration test to use cttest as the CTFE database

* Reset the CT test database before launching CT personalities in integration test flow

* Add `mariadb-client` to ct_testbase Dockerfile

* Update comment

* Export `MYSQL_HOST` in ct_functions.sh

* Export `MYSQL_ROOT_PASSWORD` in ct_functions.sh

* Export `MYSQL_USER_HOST` in ct_functions.sh

* Add all combination of `extra_data_issuance_chain_storage_backend` config in integration test flow

* Unexport `MYSQL_USER_HOST` for debugging

* Revert: Unexport `MYSQL_USER_HOST` for debugging

* Add debug log to `resectdb.sh`

* Add "SHOW TABLES" in debug log

* Add `USE ${MYSQL_DATABASE}; SHOW TABLES;`

* Move CT test database reset to GCB step 3 (ci-ready)

* Fix missing `$` in cloudbuild.yaml

* Reset CT test database in cloudbuild_master.yaml

* Update CHANGELOG.md

* Remove duplicated comment

* Rename `hash` to `chainHash`

* Unexport `isCTFEStorageEnabled` method

* Add comments for type/struct

* Add code comment about the way we unmarshal leaf.ExtraData

* Rename method to `ExtraDataForChainHash` and remove unused `chain` argument

* Remove unused `chain` argument

* Update `FixLogLeaf` method comment

* Add buildLogLeaf method comment

CHANGELOG.md

@@ -2,6 +2,25 @@
 
 ## HEAD
 
+### CTFE Storage Saving: Extra Data Issuance Chain Deduplication
+
+To reduce CT/Trillian database storage by deduplication of the entire issuance chain (intermediate certificate(s) and root certificate) that is currently stored in the Trillian merkle tree leaf ExtraData field. Storage cost should be reduced by at least 33% for new CT logs with this feature enabled. Currently only MySQL/MariaDB is supported to store the issuance chain in the CTFE database.
+
+Existing logs are not affected by this change. 
+
+Log operators can choose to opt-in this change for new CT logs by adding new CTFE configs in the [LogMultiConfig](trillian/ctfe/configpb/config.proto) and importing the [database schema](trillian/ctfe/storage/mysql/schema.sql). See [example](trillian/examples/deployment/docker/ctfe/ct_server.cfg).
+
+- `ctfe_storage_connection_string`
+- `extra_data_issuance_chain_storage_backend`
+
+An optional LRU cache can be enabled by providing the following flags.
+
+- `cache_type`
+- `cache_size`
+- `cache_ttl`
+
+This change is tested in Cloud Build tests using the `mysql:8.4` Docker image as of the time of writing.
+
 ### Submission proxy: Root compatibility checking
 
 * Adds the ability for a CT client to disable root compatibile checking: https://github.com/google/certificate-transparency-go/pull/1258

cloudbuild.yaml

@@ -85,6 +85,13 @@ steps:
 
       # Wait for trillian logserver to be up
       until nc -z deployment_trillian-log-server_1 8090; do echo .; sleep 5; done
+
+      # Reset the CT test database
+      export CT_GO_PATH="$$(go list -f '{{.Dir}}' github.com/google/certificate-transparency-go)"
+      export MYSQL_HOST="mysql"
+      export MYSQL_ROOT_PASSWORD="zaphod"
+      export MYSQL_USER_HOST="%"
+      yes | bash "$${CT_GO_PATH}/scripts/resetctdb.sh" --verbose
   waitFor: ['prepare']
 
 # Run the presubmit tests

cloudbuild_master.yaml

@@ -85,6 +85,13 @@ steps:
 
       # Wait for trillian logserver to be up
       until nc -z deployment_trillian-log-server_1 8090; do echo .; sleep 5; done
+
+      # Reset the CT test database
+      export CT_GO_PATH="$$(go list -f '{{.Dir}}' github.com/google/certificate-transparency-go)"
+      export MYSQL_HOST="mysql"
+      export MYSQL_ROOT_PASSWORD="zaphod"
+      export MYSQL_USER_HOST="%"
+      yes | bash "$${CT_GO_PATH}/scripts/resetctdb.sh" --verbose
   waitFor: ['prepare']
 
 # Run the presubmit tests

integration/Dockerfile

@@ -7,7 +7,7 @@ WORKDIR /testbase
 ARG GOFLAGS=""
 ENV GOFLAGS=$GOFLAGS
 
-RUN apt-get update && apt-get -y install build-essential curl docker-compose lsof netcat-openbsd unzip wget xxd
+RUN apt-get update && apt-get -y install build-essential curl docker-compose lsof mariadb-client netcat-openbsd unzip wget xxd
 
 RUN cd /usr/bin && curl -L -O https://github.com/jqlang/jq/releases/download/jq-1.7/jq-linux64 && mv jq-linux64 /usr/bin/jq && chmod +x /usr/bin/jq
 RUN curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.55.1

scripts/resetctdb.sh

@@ -41,6 +41,7 @@ collect_vars() {
   # handle flags
   FORCE=false
   VERBOSE=false
+
   while [[ $# -gt 0 ]]; do
     case "$1" in
       --force) FORCE=true ;;
@@ -74,16 +75,21 @@ main() {
 
   if [ -z ${REPLY+x} ] || [[ $REPLY =~ ^[Yy]$ ]]
   then
-      echo "Resetting DB..."
-      mysql "${FLAGS[@]}" -e "DROP DATABASE IF EXISTS ${MYSQL_DATABASE};" || \
-        die "Error: Failed to drop database '${MYSQL_DATABASE}'."
-      mysql "${FLAGS[@]}" -e "CREATE DATABASE ${MYSQL_DATABASE};" || \
-        die "Error: Failed to create database '${MYSQL_DATABASE}'."
-      mysql "${FLAGS[@]}" -e "CREATE USER IF NOT EXISTS ${MYSQL_USER}@'${MYSQL_USER_HOST}' IDENTIFIED BY '${MYSQL_PASSWORD}';" || \
-        die "Error: Failed to create user '${MYSQL_USER}@${MYSQL_USER_HOST}'."
-      mysql "${FLAGS[@]}" -e "GRANT ALL ON ${MYSQL_DATABASE}.* TO ${MYSQL_USER}@'${MYSQL_USER_HOST}'" || \
-        die "Error: Failed to grant '${MYSQL_USER}' user all privileges on '${MYSQL_DATABASE}'."
-      echo "Reset Complete"
+    echo "Resetting DB..."
+    mysql "${FLAGS[@]}" -e "DROP DATABASE IF EXISTS ${MYSQL_DATABASE};" || \
+      die "Error: Failed to drop database '${MYSQL_DATABASE}'."
+    mysql "${FLAGS[@]}" -e "CREATE DATABASE ${MYSQL_DATABASE};" || \
+      die "Error: Failed to create database '${MYSQL_DATABASE}'."
+    mysql "${FLAGS[@]}" -e "CREATE USER IF NOT EXISTS ${MYSQL_USER}@'${MYSQL_USER_HOST}' IDENTIFIED BY '${MYSQL_PASSWORD}';" || \
+      die "Error: Failed to create user '${MYSQL_USER}@${MYSQL_USER_HOST}'."
+    mysql "${FLAGS[@]}" -e "GRANT ALL ON ${MYSQL_DATABASE}.* TO ${MYSQL_USER}@'${MYSQL_USER_HOST}';" || \
+      die "Error: Failed to grant '${MYSQL_USER}' user all privileges on '${MYSQL_DATABASE}'."
+    mysql "${FLAGS[@]}" -e "FLUSH PRIVILEGES;" || \
+      die "Error: Failed to flush privileges."
+    mysql "${FLAGS[@]}" -D ${MYSQL_DATABASE} < ${CT_GO_PATH}/trillian/ctfe/storage/mysql/schema.sql || \
+      die "Error: Failed to import schema in '${MYSQL_DATABASE}' database."
+
+    echo "Reset Complete"
   fi
 }
 

trillian/ctfe/cache/cache.go

@@ -15,7 +15,30 @@
 // Package cache defines the IssuanceChainCache type, which allows different cache implementation with Get and Set operations.
 package cache
 
-import "context"
+import (
+	"context"
+	"errors"
+	"time"
+
+	"github.com/google/certificate-transparency-go/trillian/ctfe/cache/lru"
+	"github.com/google/certificate-transparency-go/trillian/ctfe/cache/noop"
+)
+
+// Type represents the cache type.
+type Type string
+
+// Type constants for the cache type.
+const (
+	Unknown Type = ""
+	NOOP    Type = "noop"
+	LRU     Type = "lru"
+)
+
+// Option represents the cache option, which includes the cache size and time-to-live.
+type Option struct {
+	Size int
+	TTL  time.Duration
+}
 
 // IssuanceChainCache is an interface which allows CTFE binaries to use different cache implementations for issuance chains.
 type IssuanceChainCache interface {
@@ -25,3 +48,21 @@ type IssuanceChainCache interface {
 	// Set inserts the key-value pair of issuance chain.
 	Set(ctx context.Context, key []byte, chain []byte) error
 }
+
+// NewIssuanceChainCache returns noop.IssuanceChainCache for noop type or lru.IssuanceChainCache for lru cache type.
+func NewIssuanceChainCache(_ context.Context, cacheType Type, option Option) (IssuanceChainCache, error) {
+	switch cacheType {
+	case Unknown, NOOP:
+		return &noop.IssuanceChainCache{}, nil
+	case LRU:
+		if option.Size < 0 {
+			return nil, errors.New("invalid cache_size flag")
+		}
+		if option.TTL < 0*time.Second {
+			return nil, errors.New("invalid cache_ttl flag")
+		}
+		return lru.NewIssuanceChainCache(lru.CacheOption{Size: option.Size, TTL: option.TTL}), nil
+	}
+
+	return nil, errors.New("invalid cache_type flag")
+}

trillian/ctfe/cache/noop/noop.go

@@ -0,0 +1,28 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package noop defines the IssuanceChainCache type, which implements IssuanceChainCache interface with Get and Set operations.
+package noop
+
+import "context"
+
+type IssuanceChainCache struct{}
+
+func (c *IssuanceChainCache) Get(_ context.Context, key []byte) ([]byte, error) {
+	return nil, nil
+}
+
+func (c *IssuanceChainCache) Set(_ context.Context, key []byte, chain []byte) error {
+	return nil
+}

trillian/ctfe/ct_server/main.go

@@ -32,6 +32,7 @@ import (
 	"time"
 
 	"github.com/google/certificate-transparency-go/trillian/ctfe"
+	"github.com/google/certificate-transparency-go/trillian/ctfe/cache"
 	"github.com/google/certificate-transparency-go/trillian/ctfe/configpb"
 	"github.com/google/trillian"
 	"github.com/google/trillian/crypto/keys"
@@ -73,6 +74,9 @@ var (
 	quotaIntermediate  = flag.Bool("quota_intermediate", true, "Enable requesting of quota for intermediate certificates in submitted chains")
 	handlerPrefix      = flag.String("handler_prefix", "", "If set e.g. to '/logs' will prefix all handlers that don't define a custom prefix")
 	pkcs11ModulePath   = flag.String("pkcs11_module_path", "", "Path to the PKCS#11 module to use for keys that use the PKCS#11 interface")
+	cacheType          = flag.String("cache_type", "noop", "Supported cache type: noop, lru (Default: noop)")
+	cacheSize          = flag.Int("cache_size", -1, "Size parameter set to 0 makes cache of unlimited size")
+	cacheTTL           = flag.Duration("cache_ttl", -1*time.Second, "Providing 0 TTL turns expiring off")
 )
 
 const unknownRemoteUser = "UNKNOWN_REMOTE"
@@ -218,7 +222,19 @@ func main() {
 	// client.
 	var publicKeys []crypto.PublicKey
 	for _, c := range cfg.LogConfigs.Config {
-		inst, err := setupAndRegister(ctx, clientMap[c.LogBackendName], *rpcDeadline, c, corsMux, *handlerPrefix, *maskInternalErrors)
+		inst, err := setupAndRegister(ctx,
+			clientMap[c.LogBackendName],
+			*rpcDeadline,
+			c,
+			corsMux,
+			*handlerPrefix,
+			*maskInternalErrors,
+			cache.Type(*cacheType),
+			cache.Option{
+				Size: *cacheSize,
+				TTL:  *cacheTTL,
+			},
+		)
 		if err != nil {
 			klog.Exitf("Failed to set up log instance for %+v: %v", cfg, err)
 		}
@@ -330,7 +346,7 @@ func awaitSignal(doneFn func()) {
 	doneFn()
 }
 
-func setupAndRegister(ctx context.Context, client trillian.TrillianLogClient, deadline time.Duration, cfg *configpb.LogConfig, mux *http.ServeMux, globalHandlerPrefix string, maskInternalErrors bool) (*ctfe.Instance, error) {
+func setupAndRegister(ctx context.Context, client trillian.TrillianLogClient, deadline time.Duration, cfg *configpb.LogConfig, mux *http.ServeMux, globalHandlerPrefix string, maskInternalErrors bool, cacheType cache.Type, cacheOption cache.Option) (*ctfe.Instance, error) {
 	vCfg, err := ctfe.ValidateLogConfig(cfg)
 	if err != nil {
 		return nil, err
@@ -343,6 +359,8 @@ func setupAndRegister(ctx context.Context, client trillian.TrillianLogClient, de
 		MetricFactory:      prometheus.MetricFactory{},
 		RequestLog:         new(ctfe.DefaultRequestLog),
 		MaskInternalErrors: maskInternalErrors,
+		CacheType:          cacheType,
+		CacheOption:        cacheOption,
 	}
 	if *quotaRemote {
 		klog.Info("Enabling quota for requesting IP")

trillian/ctfe/handlers.go

@@ -278,6 +278,8 @@ type logInfo struct {
 	signer crypto.Signer
 	// sthGetter provides STHs for the log
 	sthGetter STHGetter
+	// issuanceChainService provides the issuance chain add and get operations
+	issuanceChainService *issuanceChainService
 }
 
 // newLogInfo creates a new instance of logInfo.
@@ -286,6 +288,7 @@ func newLogInfo(
 	validationOpts CertValidationOpts,
 	signer crypto.Signer,
 	timeSource util.TimeSource,
+	issuanceChainService *issuanceChainService,
 ) *logInfo {
 	vCfg := instanceOpts.Validated
 	cfg := vCfg.Config
@@ -330,6 +333,8 @@ func newLogInfo(
 	maxMergeDelay.Set(float64(cfg.MaxMergeDelaySec), label)
 	expMergeDelay.Set(float64(cfg.ExpectedMergeDelaySec), label)
 
+	li.issuanceChainService = issuanceChainService
+
 	return li
 }
 
@@ -465,15 +470,15 @@ func addChainInternal(ctx context.Context, li *logInfo, w http.ResponseWriter, r
 	if err != nil {
 		return http.StatusBadRequest, fmt.Errorf("failed to build MerkleTreeLeaf: %s", err)
 	}
-	leaf, err := buildLogLeafForAddChain(li, *merkleLeaf, chain, isPrecert)
+	leaf, err := li.issuanceChainService.BuildLogLeaf(ctx, chain, li.LogPrefix, merkleLeaf, isPrecert)
 	if err != nil {
-		return http.StatusInternalServerError, fmt.Errorf("failed to build LogLeaf: %s", err)
+		return http.StatusInternalServerError, err
 	}
 
 	// Send the Merkle tree leaf on to the Log server.
 	req := trillian.QueueLeafRequest{
 		LogId:    li.logID,
-		Leaf:     &leaf,
+		Leaf:     leaf,
 		ChargeTo: li.chargeUser(r),
 	}
 	if li.instanceOpts.CertificateQuotaUser != nil {
@@ -745,10 +750,11 @@ func getEntries(ctx context.Context, li *logInfo, w http.ResponseWriter, r *http
 		Count:      count,
 		ChargeTo:   li.chargeUser(r),
 	}
-	rsp, err := li.rpcClient.GetLeavesByRange(ctx, &req)
+	rsp, httpStatus, err := rpcGetLeavesByRange(ctx, li, &req)
 	if err != nil {
-		return li.toHTTPStatus(err), fmt.Errorf("backend GetLeavesByRange request failed: %s", err)
+		return httpStatus, err
 	}
+
 	var currentRoot types.LogRootV1
 	if err := currentRoot.UnmarshalBinary(rsp.GetSignedLogRoot().GetLogRoot()); err != nil {
 		return http.StatusInternalServerError, fmt.Errorf("failed to unmarshal root: %v", rsp.GetSignedLogRoot().GetLogRoot())
@@ -798,6 +804,21 @@ func getEntries(ctx context.Context, li *logInfo, w http.ResponseWriter, r *http
 	return http.StatusOK, nil
 }
 
+// rpcGetLeavesByRange calls Trillian GetLeavesByRange RPC and fixes issuance chain in each log leaf if necessary.
+func rpcGetLeavesByRange(ctx context.Context, li *logInfo, req *trillian.GetLeavesByRangeRequest) (*trillian.GetLeavesByRangeResponse, int, error) {
+	rsp, err := li.rpcClient.GetLeavesByRange(ctx, req)
+	if err != nil {
+		return nil, li.toHTTPStatus(err), fmt.Errorf("backend GetLeavesByRange request failed: %s", err)
+	}
+	for _, leaf := range rsp.Leaves {
+		if err := li.issuanceChainService.FixLogLeaf(ctx, leaf); err != nil {
+			return nil, http.StatusInternalServerError, fmt.Errorf("failed to fix log leaf: %v", rsp)
+		}
+	}
+
+	return rsp, http.StatusOK, nil
+}
+
 func getRoots(_ context.Context, li *logInfo, w http.ResponseWriter, _ *http.Request) (int, error) {
 	// Pull out the raw certificates from the parsed versions
 	rawCerts := make([][]byte, 0, len(li.validationOpts.trustedRoots.RawCertificates()))
@@ -834,9 +855,9 @@ func getEntryAndProof(ctx context.Context, li *logInfo, w http.ResponseWriter, r
 		TreeSize:  treeSize,
 		ChargeTo:  li.chargeUser(r),
 	}
-	rsp, err := li.rpcClient.GetEntryAndProof(ctx, &req)
+	rsp, httpStatus, err := rpcGetEntryAndProof(ctx, li, &req)
 	if err != nil {
-		return li.toHTTPStatus(err), fmt.Errorf("backend GetEntryAndProof request failed: %s", err)
+		return httpStatus, err
 	}
 
 	var currentRoot types.LogRootV1
@@ -881,6 +902,19 @@ func getEntryAndProof(ctx context.Context, li *logInfo, w http.ResponseWriter, r
 	return http.StatusOK, nil
 }
 
+// rpcGetEntryAndProof calls Trillian GetEntryAndProof RPC and fixes issuance chain in the log leaf if necessary.
+func rpcGetEntryAndProof(ctx context.Context, li *logInfo, req *trillian.GetEntryAndProofRequest) (*trillian.GetEntryAndProofResponse, int, error) {
+	rsp, err := li.rpcClient.GetEntryAndProof(ctx, req)
+	if err != nil {
+		return nil, li.toHTTPStatus(err), fmt.Errorf("backend GetEntryAndProof request failed: %s", err)
+	}
+	if err := li.issuanceChainService.FixLogLeaf(ctx, rsp.Leaf); err != nil {
+		return nil, http.StatusInternalServerError, fmt.Errorf("failed to fix log leaf: %v", rsp)
+	}
+
+	return rsp, http.StatusOK, nil
+}
+
 // getRPCDeadlineTime calculates the future time an RPC should expire based on our config
 func getRPCDeadlineTime(li *logInfo) time.Time {
 	return li.TimeSource.Now().Add(li.instanceOpts.Deadline)
@@ -923,15 +957,6 @@ func extractRawCerts(chain []*x509.Certificate) []ct.ASN1Cert {
 	return raw
 }
 
-// buildLogLeafForAddChain does the hashing to build a LogLeaf that will be
-// sent to the backend by add-chain and add-pre-chain endpoints.
-func buildLogLeafForAddChain(li *logInfo,
-	merkleLeaf ct.MerkleTreeLeaf, chain []*x509.Certificate, isPrecert bool,
-) (trillian.LogLeaf, error) {
-	raw := extractRawCerts(chain)
-	return util.BuildLogLeaf(li.LogPrefix, merkleLeaf, 0, raw[0], raw[1:], isPrecert)
-}
-
 // marshalAndWriteAddChainResponse is used by add-chain and add-pre-chain to create and write
 // the JSON response to the client
 func marshalAndWriteAddChainResponse(sct *ct.SignedCertificateTimestamp, signer crypto.Signer, w http.ResponseWriter) error {

trillian/ctfe/handlers_test.go

@@ -165,7 +165,7 @@ func setupTest(t *testing.T, pemRoots []string, signer crypto.Signer) handlerTes
 	cfg := &configpb.LogConfig{LogId: 0x42, Prefix: "test", IsMirror: false}
 	vCfg := &ValidatedLogConfig{Config: cfg}
 	iOpts := InstanceOptions{Validated: vCfg, Client: info.client, Deadline: time.Millisecond * 500, MetricFactory: monitoring.InertMetricFactory{}, RequestLog: new(DefaultRequestLog)}
-	info.li = newLogInfo(iOpts, vOpts, signer, fakeTimeSource)
+	info.li = newLogInfo(iOpts, vOpts, signer, fakeTimeSource, newIssuanceChainService(nil, nil))
 
 	for _, pemRoot := range pemRoots {
 		if !info.roots.AppendCertsFromPEM([]byte(pemRoot)) {