CTFE Extra Data Issuance Chain Deduplication

CHANGELOG.md

@@ -2,6 +2,21 @@
 
 ## HEAD
 
+### CTFE Storage Saving: Extra Data Issuance Chain Deduplication
+
+To reduce CT/Trillian database storage by deduplication of the entire issuance chain (intermediate certificate(s) and root certificate) that is currently stored in the Trillian merkle tree leaf ExtraData field. Storage cost should be reduced by at least 33% for new CT logs with this feature enabled.
+
+Existing logs are not affected by this change. Log operators can choose to opt-in this change for new CT logs by adding new CTFE configs in the [LogMultiConfig](trillian/ctfe/configpb/config.proto). See [example](trillian/examples/deployment/docker/ctfe/ct_server_mysql.cfg).
+
+- `ctfe_storage_connection_string`
+- `extra_data_issuance_chain_storage_backend`
+
+An optional LRU cache can be enabled by providing the following flags.
+
+- `cache_type`
+- `cache_size`
+- `cache_ttl`
+
 ### Submission proxy: Root compatibility checking
 
 * Adds the ability for a CT client to disable root compatibile checking: https://github.com/google/certificate-transparency-go/pull/1258

trillian/ctfe/cache/cache.go

@@ -15,7 +15,21 @@
 // Package cache defines the IssuanceChainCache type, which allows different cache implementation with Get and Set operations.
 package cache
 
-import "context"
+import (
+	"context"
+	"errors"
+	"flag"
+	"time"
+
+	"github.com/google/certificate-transparency-go/trillian/ctfe/cache/lru"
+	"github.com/google/certificate-transparency-go/trillian/ctfe/cache/noop"
+)
+
+var (
+	cacheType = flag.String("cache_type", "noop", "Supported cache type: noop, lru (Default: noop)")
+	size      = flag.Int("cache_size", -1, "Size parameter set to 0 makes cache of unlimited size")
+	ttl       = flag.Duration("cache_ttl", -1*time.Second, "Providing 0 TTL turns expiring off")
+)
 
 // IssuanceChainCache is an interface which allows CTFE binaries to use different cache implementations for issuance chains.
 type IssuanceChainCache interface {
@@ -25,3 +39,21 @@ type IssuanceChainCache interface {
 	// Set inserts the key-value pair of issuance chain.
 	Set(ctx context.Context, key []byte, chain []byte) error
 }
+
+// NewIssuanceChainCache returns noop.IssuanceChainCache for noop type or lru.IssuanceChainCache for lru cache type.
+func NewIssuanceChainCache(_ context.Context) (IssuanceChainCache, error) {
+	switch *cacheType {
+	case "noop":
+		return &noop.IssuanceChainCache{}, nil
+	case "lru":
+		if *size < 0 {
+			return nil, errors.New("invalid cache_size flag")
+		}
+		if *ttl < 0*time.Second {
+			return nil, errors.New("invalid cache_ttl flag")
+		}
+		return lru.NewIssuanceChainCache(lru.CacheOption{Size: *size, TTL: *ttl}), nil
+	}
+
+	return nil, errors.New("invalid cache_type flag")
+}

trillian/ctfe/cache/noop/noop.go

@@ -0,0 +1,28 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package noop defines the IssuanceChainCache type, which implements IssuanceChainCache interface with Get and Set operations.
+package noop
+
+import "context"
+
+type IssuanceChainCache struct{}
+
+func (c *IssuanceChainCache) Get(_ context.Context, key []byte) ([]byte, error) {
+	return nil, nil
+}
+
+func (c *IssuanceChainCache) Set(_ context.Context, key []byte, chain []byte) error {
+	return nil
+}

trillian/ctfe/handlers.go

@@ -278,6 +278,8 @@ type logInfo struct {
 	signer crypto.Signer
 	// sthGetter provides STHs for the log
 	sthGetter STHGetter
+	// issuanceChainService provides the issuance chain add and get operations
+	issuanceChainService *issuanceChainService
 }
 
 // newLogInfo creates a new instance of logInfo.
@@ -286,6 +288,7 @@ func newLogInfo(
 	validationOpts CertValidationOpts,
 	signer crypto.Signer,
 	timeSource util.TimeSource,
+	issuanceChainService *issuanceChainService,
 ) *logInfo {
 	vCfg := instanceOpts.Validated
 	cfg := vCfg.Config
@@ -330,6 +333,8 @@ func newLogInfo(
 	maxMergeDelay.Set(float64(cfg.MaxMergeDelaySec), label)
 	expMergeDelay.Set(float64(cfg.ExpectedMergeDelaySec), label)
 
+	li.issuanceChainService = issuanceChainService
+
 	return li
 }
 
@@ -461,13 +466,35 @@ func addChainInternal(ctx context.Context, li *logInfo, w http.ResponseWriter, r
 	timeMillis := uint64(li.TimeSource.Now().UnixNano() / millisPerNano)
 
 	// Build the MerkleTreeLeaf that gets sent to the backend, and make a trillian.LogLeaf for it.
+	var leaf trillian.LogLeaf
 	merkleLeaf, err := ct.MerkleTreeLeafFromChain(chain, etype, timeMillis)
 	if err != nil {
 		return http.StatusBadRequest, fmt.Errorf("failed to build MerkleTreeLeaf: %s", err)
 	}
-	leaf, err := buildLogLeafForAddChain(li, *merkleLeaf, chain, isPrecert)
-	if err != nil {
-		return http.StatusInternalServerError, fmt.Errorf("failed to build LogLeaf: %s", err)
+	raw := extractRawCerts(chain)
+
+	// If CTFE storage is enabled for issuance chain, add the chain to storage
+	// and cache, and then build log leaf. If Trillian gRPC is enabled for
+	// issuance chain, build the log leaf.
+	if li.issuanceChainService.IsCTFEStorageEnabled() {
+		// TODO: Check how to convert []ct.ASN1Cert to []byte correctly.
+		issuanceChain, err := asn1.Marshal(raw[1:])
+		if err != nil {
+			return http.StatusInternalServerError, fmt.Errorf("failed to marshal issuance chain: %s", err)
+		}
+		hash, err := li.issuanceChainService.Add(ctx, issuanceChain)
+		if err != nil {
+			return http.StatusInternalServerError, fmt.Errorf("failed to add issuance chain into CTFE storage: %s", err)
+		}
+		leaf, err = util.BuildLogLeafWithHash(li.LogPrefix, *merkleLeaf, 0, raw[0], nil, hash, isPrecert)
+		if err != nil {
+			return http.StatusInternalServerError, fmt.Errorf("failed to build LogLeaf: %s", err)
+		}
+	} else {
+		leaf, err = util.BuildLogLeaf(li.LogPrefix, *merkleLeaf, 0, raw[0], raw[1:], isPrecert)
+		if err != nil {
+			return http.StatusInternalServerError, fmt.Errorf("failed to build LogLeaf: %s", err)
+		}
 	}
 
 	// Send the Merkle tree leaf on to the Log server.
@@ -770,6 +797,10 @@ func getEntries(ctx context.Context, li *logInfo, w http.ResponseWriter, r *http
 		if leaf.LeafIndex != start+int64(i) {
 			return http.StatusInternalServerError, fmt.Errorf("backend returned unexpected leaf index: rsp.Leaves[%d].LeafIndex=%d for range [%d,%d]", i, leaf.LeafIndex, start, end)
 		}
+
+		if err := li.issuanceChainService.FixLogLeaf(ctx, leaf); err != nil {
+			return http.StatusInternalServerError, fmt.Errorf("failed to fix log leaf: %v", rsp)
+		}
 	}
 	leaves = rsp.Leaves
 
@@ -857,6 +888,10 @@ func getEntryAndProof(ctx context.Context, li *logInfo, w http.ResponseWriter, r
 		return http.StatusInternalServerError, fmt.Errorf("got RPC bad response (missing proof), possible extra info: %v", rsp)
 	}
 
+	if err := li.issuanceChainService.FixLogLeaf(ctx, rsp.Leaf); err != nil {
+		return http.StatusInternalServerError, fmt.Errorf("failed to fix log leaf: %v", rsp)
+	}
+
 	// Build and marshal the response to the client
 	jsonRsp := ct.GetEntryAndProofResponse{
 		LeafInput: rsp.Leaf.LeafValue,
@@ -923,15 +958,6 @@ func extractRawCerts(chain []*x509.Certificate) []ct.ASN1Cert {
 	return raw
 }
 
-// buildLogLeafForAddChain does the hashing to build a LogLeaf that will be
-// sent to the backend by add-chain and add-pre-chain endpoints.
-func buildLogLeafForAddChain(li *logInfo,
-	merkleLeaf ct.MerkleTreeLeaf, chain []*x509.Certificate, isPrecert bool,
-) (trillian.LogLeaf, error) {
-	raw := extractRawCerts(chain)
-	return util.BuildLogLeaf(li.LogPrefix, merkleLeaf, 0, raw[0], raw[1:], isPrecert)
-}
-
 // marshalAndWriteAddChainResponse is used by add-chain and add-pre-chain to create and write
 // the JSON response to the client
 func marshalAndWriteAddChainResponse(sct *ct.SignedCertificateTimestamp, signer crypto.Signer, w http.ResponseWriter) error {

trillian/ctfe/handlers_test.go

@@ -165,7 +165,7 @@ func setupTest(t *testing.T, pemRoots []string, signer crypto.Signer) handlerTes
 	cfg := &configpb.LogConfig{LogId: 0x42, Prefix: "test", IsMirror: false}
 	vCfg := &ValidatedLogConfig{Config: cfg}
 	iOpts := InstanceOptions{Validated: vCfg, Client: info.client, Deadline: time.Millisecond * 500, MetricFactory: monitoring.InertMetricFactory{}, RequestLog: new(DefaultRequestLog)}
-	info.li = newLogInfo(iOpts, vOpts, signer, fakeTimeSource)
+	info.li = newLogInfo(iOpts, vOpts, signer, fakeTimeSource, newIssuanceChainService(nil, nil))
 
 	for _, pemRoot := range pemRoots {
 		if !info.roots.AppendCertsFromPEM([]byte(pemRoot)) {

trillian/ctfe/instance.go

@@ -29,6 +29,8 @@ import (
 
 	"github.com/google/certificate-transparency-go/asn1"
 	"github.com/google/certificate-transparency-go/schedule"
+	"github.com/google/certificate-transparency-go/trillian/ctfe/cache"
+	"github.com/google/certificate-transparency-go/trillian/ctfe/storage"
 	"github.com/google/certificate-transparency-go/trillian/util"
 	"github.com/google/certificate-transparency-go/x509"
 	"github.com/google/certificate-transparency-go/x509util"
@@ -174,7 +176,21 @@ func setUpLogInfo(ctx context.Context, opts InstanceOptions) (*logInfo, error) {
 		return nil, fmt.Errorf("failed to parse RejectExtensions: %v", err)
 	}
 
-	logInfo := newLogInfo(opts, validationOpts, signer, new(util.SystemTimeSource))
+	// Initialise IssuanceChainService with IssuanceChainStorage and IssuanceChainCache.
+	// issuanceChainStorage is nil for Trillian gRPC or mysql.IssuanceChainStorage when MySQL is the prefix in database connection string.
+	issuanceChainStorage, err := storage.NewIssuanceChainStorage(ctx, vCfg.ExtraDataIssuanceChainStorageBackend, vCfg.CTFEStorageConnectionString)
+	if err != nil {
+		return nil, err
+	}
+	// issuanceChainCache is nil if the cache related flags are not defined.
+	issuanceChainCache, err := cache.NewIssuanceChainCache(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	issuanceChainService := newIssuanceChainService(issuanceChainStorage, issuanceChainCache)
+
+	logInfo := newLogInfo(opts, validationOpts, signer, new(util.SystemTimeSource), issuanceChainService)
 	return logInfo, nil
 }
 

trillian/ctfe/services.go

@@ -17,10 +17,16 @@ package ctfe
 import (
 	"context"
 	"crypto/sha256"
+	"fmt"
 
+	"github.com/google/certificate-transparency-go/asn1"
+	"github.com/google/certificate-transparency-go/tls"
 	"github.com/google/certificate-transparency-go/trillian/ctfe/cache"
 	"github.com/google/certificate-transparency-go/trillian/ctfe/storage"
+	"github.com/google/trillian"
 	"k8s.io/klog/v2"
+
+	ct "github.com/google/certificate-transparency-go"
 )
 
 type issuanceChainService struct {
@@ -37,6 +43,10 @@ func newIssuanceChainService(s storage.IssuanceChainStorage, c cache.IssuanceCha
 	return service
 }
 
+func (s *issuanceChainService) IsCTFEStorageEnabled() bool {
+	return s.storage != nil
+}
+
 // GetByHash returns the issuance chain with hash as the input.
 func (s *issuanceChainService) GetByHash(ctx context.Context, hash []byte) ([]byte, error) {
 	// Return if found in cache.
@@ -82,6 +92,82 @@ func (s *issuanceChainService) Add(ctx context.Context, chain []byte) ([]byte, e
 	return hash, nil
 }
 
+func (s *issuanceChainService) FixLogLeaf(ctx context.Context, leaf *trillian.LogLeaf) error {
+	// Skip if CTFE storage backend is not enabled.
+	if !s.IsCTFEStorageEnabled() {
+		return nil
+	}
+
+	var precertChainHash ct.PrecertChainEntryHash
+	if rest, err := tls.Unmarshal(leaf.ExtraData, &precertChainHash); err == nil && len(rest) == 0 {
+		var chain []ct.ASN1Cert
+		if len(precertChainHash.IssuanceChainHash) > 0 {
+			chainBytes, err := s.GetByHash(ctx, precertChainHash.IssuanceChainHash)
+			if err != nil {
+				return err
+			}
+
+			if rest, err := asn1.Unmarshal(chainBytes, &chain); err != nil {
+				return err
+			} else if len(rest) > 0 {
+				return fmt.Errorf("IssuanceChain: trailing data %d bytes", len(rest))
+			}
+		}
+
+		precertChain := ct.PrecertChainEntry{
+			PreCertificate:   precertChainHash.PreCertificate,
+			CertificateChain: chain,
+		}
+		extraData, err := tls.Marshal(precertChain)
+		if err != nil {
+			return err
+		}
+
+		leaf.ExtraData = extraData
+		return nil
+	}
+
+	var certChainHash ct.CertificateChainHash
+	if rest, err := tls.Unmarshal(leaf.ExtraData, &certChainHash); err == nil && len(rest) == 0 {
+		var entries []ct.ASN1Cert
+		if len(certChainHash.IssuanceChainHash) > 0 {
+			chainBytes, err := s.GetByHash(ctx, certChainHash.IssuanceChainHash)
+			if err != nil {
+				return err
+			}
+
+			if rest, err := asn1.Unmarshal(chainBytes, &entries); err != nil {
+				return err
+			} else if len(rest) > 0 {
+				return fmt.Errorf("IssuanceChain: trailing data %d bytes", len(rest))
+			}
+		}
+
+		certChain := ct.CertificateChain{
+			Entries: entries,
+		}
+		extraData, err := tls.Marshal(certChain)
+		if err != nil {
+			return err
+		}
+
+		leaf.ExtraData = extraData
+		return nil
+	}
+
+	// Skip if the types are ct.PrecertChainEntry or ct.CertificateChain as there is no hash.
+	var precertChain ct.PrecertChainEntry
+	if rest, err := tls.Unmarshal(leaf.ExtraData, &precertChain); err == nil && len(rest) == 0 {
+		return nil
+	}
+	var certChain ct.CertificateChain
+	if rest, err := tls.Unmarshal(leaf.ExtraData, &certChain); err == nil && len(rest) == 0 {
+		return nil
+	}
+
+	return fmt.Errorf("unknown extra data type in log leaf: %s", string(leaf.MerkleLeafHash))
+}
+
 // issuanceChainHash returns the SHA-256 hash of the chain.
 func issuanceChainHash(chain []byte) []byte {
 	checksum := sha256.Sum256(chain)

trillian/ctfe/storage/storage.go

@@ -17,6 +17,11 @@ package storage
 
 import (
 	"context"
+	"errors"
+	"strings"
+
+	"github.com/google/certificate-transparency-go/trillian/ctfe/configpb"
+	"github.com/google/certificate-transparency-go/trillian/ctfe/storage/mysql"
 )
 
 // IssuanceChainStorage is an interface which allows CTFE binaries to use different storage implementations for issuance chains.
@@ -27,3 +32,19 @@ type IssuanceChainStorage interface {
 	// Add inserts the key-value pair of issuance chain.
 	Add(ctx context.Context, key []byte, chain []byte) error
 }
+
+// NewIssuanceChainStorage returns nil for Trillian gRPC or mysql.IssuanceChainStorage when MySQL is the prefix in database connection string.
+func NewIssuanceChainStorage(ctx context.Context, backend configpb.LogConfig_IssuanceChainStorageBackend, dbConn string) (IssuanceChainStorage, error) {
+	switch backend {
+	case configpb.LogConfig_ISSUANCE_CHAIN_STORAGE_BACKEND_TRILLIAN_GRPC:
+		return nil, nil
+	case configpb.LogConfig_ISSUANCE_CHAIN_STORAGE_BACKEND_CTFE:
+		if strings.HasPrefix(dbConn, "mysql") {
+			return mysql.NewIssuanceChainStorage(ctx, dbConn), nil
+		} else {
+			return nil, errors.New("failed to initialise IssuanceChainService due to unsupported driver in CTFE storage connection string")
+		}
+	}
+
+	return nil, errors.New("unsupported issuance chain storage backend")
+}