From 02eebc9f40066447b12fee727a6c03815894773b Mon Sep 17 00:00:00 2001 From: Roger Ng Date: Sun, 12 May 2024 23:54:09 +0000 Subject: [PATCH 01/55] CTFE Extra Data Issuance Chain Deduplication --- CHANGELOG.md | 15 ++++ trillian/ctfe/cache/cache.go | 34 +++++++- trillian/ctfe/cache/noop/noop.go | 28 ++++++ trillian/ctfe/handlers.go | 48 ++++++++++- trillian/ctfe/handlers_test.go | 2 +- trillian/ctfe/instance.go | 16 +++- trillian/ctfe/services.go | 86 +++++++++++++++++++ trillian/ctfe/storage/storage.go | 21 +++++ .../docker/ctfe/ct_server_mysql.cfg | 17 ++++ trillian/util/log_leaf.go | 77 ++++++++++++----- 10 files changed, 318 insertions(+), 26 deletions(-) create mode 100644 trillian/ctfe/cache/noop/noop.go create mode 100644 trillian/examples/deployment/docker/ctfe/ct_server_mysql.cfg diff --git a/CHANGELOG.md b/CHANGELOG.md index 5737e823e9..a4d9deb713 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,21 @@ ## HEAD +### CTFE Storage Saving: Extra Data Issuance Chain Deduplication + +To reduce CT/Trillian database storage by deduplication of the entire issuance chain (intermediate certificate(s) and root certificate) that is currently stored in the Trillian merkle tree leaf ExtraData field. Storage cost should be reduced by at least 33% for newly deduplicated entries. + +No operation is required for existing logs when the change is rolled out. Log operators can choose to opt-in this change for new CT logs by adding new CTFE configs in the [LogMultiConfig](https://github.com/google/certificate-transparency-go/blob/master/trillian/ctfe/configpb/config.proto). + +- `ctfe_storage_connection_string` +- `extra_data_issuance_chain_storage_backend` + +An optional LRU cache can be enabled by providing the following flags. + +- `cache_type` +- `cache_size` +- `cache_ttl` + ### Submission proxy: Root compatibility checking * Adds the ability for a CT client to disable root compatibile checking: https://github.com/google/certificate-transparency-go/pull/1258 diff --git a/trillian/ctfe/cache/cache.go b/trillian/ctfe/cache/cache.go index 3ef06cb970..a7918d45b2 100644 --- a/trillian/ctfe/cache/cache.go +++ b/trillian/ctfe/cache/cache.go @@ -15,7 +15,21 @@ // Package cache defines the IssuanceChainCache type, which allows different cache implementation with Get and Set operations. package cache -import "context" +import ( + "context" + "errors" + "flag" + "time" + + "github.com/google/certificate-transparency-go/trillian/ctfe/cache/lru" + "github.com/google/certificate-transparency-go/trillian/ctfe/cache/noop" +) + +var ( + cacheType = flag.String("cache_type", "noop", "Supported cache type: noop, lru (Default: noop)") + size = flag.Int("cache_size", 0, "Size parameter set to 0 makes cache of unlimited size") + ttl = flag.Duration("cache_ttl", 0*time.Second, "Providing 0 TTL turns expiring off") +) // IssuanceChainCache is an interface which allows CTFE binaries to use different cache implementations for issuance chains. type IssuanceChainCache interface { @@ -25,3 +39,21 @@ type IssuanceChainCache interface { // Set inserts the key-value pair of issuance chain. Set(ctx context.Context, key []byte, chain []byte) error } + +// NewIssuanceChainCache returns nil for noop type or lru.IssuanceChainCache for lru cache type. +func NewIssuanceChainCache(_ context.Context) (IssuanceChainCache, error) { + switch *cacheType { + case "noop": + return &noop.IssuanceChainCache{}, nil + case "lru": + if *size < 0 { + return nil, errors.New("invalid cache_size flag") + } + if *ttl < 0*time.Second { + return nil, errors.New("invalid cache_ttl flag") + } + return lru.NewIssuanceChainCache(lru.CacheOption{Size: *size, TTL: *ttl}), nil + } + + return nil, errors.New("invalid cache_type flag") +} diff --git a/trillian/ctfe/cache/noop/noop.go b/trillian/ctfe/cache/noop/noop.go new file mode 100644 index 0000000000..ed8f4ed9bf --- /dev/null +++ b/trillian/ctfe/cache/noop/noop.go @@ -0,0 +1,28 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package noop defines the IssuanceChainCache type, which implements IssuanceChainCache interface with Get and Set operations. +package noop + +import "context" + +type IssuanceChainCache struct{} + +func (c *IssuanceChainCache) Get(_ context.Context, key []byte) ([]byte, error) { + return nil, nil +} + +func (c *IssuanceChainCache) Set(_ context.Context, key []byte, chain []byte) error { + return nil +} diff --git a/trillian/ctfe/handlers.go b/trillian/ctfe/handlers.go index 303fcae375..9d89336104 100644 --- a/trillian/ctfe/handlers.go +++ b/trillian/ctfe/handlers.go @@ -278,6 +278,8 @@ type logInfo struct { signer crypto.Signer // sthGetter provides STHs for the log sthGetter STHGetter + // issuanceChainService provides the issuance chain add and get operations + issuanceChainService *issuanceChainService } // newLogInfo creates a new instance of logInfo. @@ -286,6 +288,7 @@ func newLogInfo( validationOpts CertValidationOpts, signer crypto.Signer, timeSource util.TimeSource, + issuanceChainService *issuanceChainService, ) *logInfo { vCfg := instanceOpts.Validated cfg := vCfg.Config @@ -330,6 +333,8 @@ func newLogInfo( maxMergeDelay.Set(float64(cfg.MaxMergeDelaySec), label) expMergeDelay.Set(float64(cfg.ExpectedMergeDelaySec), label) + li.issuanceChainService = issuanceChainService + return li } @@ -460,6 +465,19 @@ func addChainInternal(ctx context.Context, li *logInfo, w http.ResponseWriter, r // epoch, and use this throughout. timeMillis := uint64(li.TimeSource.Now().UnixNano() / millisPerNano) + // If CTFE storage is enabled for issuance chain, add the chain to storage and cache, get the chain hash for Trillian gRPC. + if li.issuanceChainService.IsCTFEStorageEnabled() { + // TODO: Check how to convert []ct.ASN1Cert to []byte correctly. + issuanceChain, err := asn1.Marshal(extractRawCerts(chain)[1:]) + if err != nil { + return http.StatusInternalServerError, fmt.Errorf("failed to marshal issuance chain: %s", err) + } + _, err = li.issuanceChainService.Add(ctx, issuanceChain) + if err != nil { + return http.StatusInternalServerError, fmt.Errorf("failed to add issuance chain into CTFE storage: %s", err) + } + } + // Build the MerkleTreeLeaf that gets sent to the backend, and make a trillian.LogLeaf for it. merkleLeaf, err := ct.MerkleTreeLeafFromChain(chain, etype, timeMillis) if err != nil { @@ -770,6 +788,10 @@ func getEntries(ctx context.Context, li *logInfo, w http.ResponseWriter, r *http if leaf.LeafIndex != start+int64(i) { return http.StatusInternalServerError, fmt.Errorf("backend returned unexpected leaf index: rsp.Leaves[%d].LeafIndex=%d for range [%d,%d]", i, leaf.LeafIndex, start, end) } + + if err := li.issuanceChainService.FixLogLeaf(ctx, leaf); err != nil { + return http.StatusInternalServerError, fmt.Errorf("failed to fix log leaf: %v", rsp) + } } leaves = rsp.Leaves @@ -857,6 +879,10 @@ func getEntryAndProof(ctx context.Context, li *logInfo, w http.ResponseWriter, r return http.StatusInternalServerError, fmt.Errorf("got RPC bad response (missing proof), possible extra info: %v", rsp) } + if err := li.issuanceChainService.FixLogLeaf(ctx, rsp.Leaf); err != nil { + return http.StatusInternalServerError, fmt.Errorf("failed to fix log leaf: %v", rsp) + } + // Build and marshal the response to the client jsonRsp := ct.GetEntryAndProofResponse{ LeafInput: rsp.Leaf.LeafValue, @@ -929,7 +955,27 @@ func buildLogLeafForAddChain(li *logInfo, merkleLeaf ct.MerkleTreeLeaf, chain []*x509.Certificate, isPrecert bool, ) (trillian.LogLeaf, error) { raw := extractRawCerts(chain) - return util.BuildLogLeaf(li.LogPrefix, merkleLeaf, 0, raw[0], raw[1:], isPrecert) + issuanceChain := raw[1:] + + // Trillian gRPC storage backend is enabled and CTFE storage backend is disabled. + if li.issuanceChainService.storage == nil { + return util.BuildLogLeaf(li.LogPrefix, merkleLeaf, 0, raw[0], issuanceChain, isPrecert) + } + + // CTFE storage backend is enabled. + // TODO: Check how to convert []ct.ASN1Cert to []byte correctly. + chainBytes, err := asn1.Marshal(issuanceChain) + if err != nil { + return trillian.LogLeaf{}, err + } + hash := issuanceChainHash(chainBytes) + + // Set issuance chain to nil if Trillian gRPC storage backend is not enabled. + if li.issuanceChainService.storage != nil { + issuanceChain = nil + } + + return util.BuildLogLeafWithHash(li.LogPrefix, merkleLeaf, 0, raw[0], issuanceChain, []byte(hash), isPrecert) } // marshalAndWriteAddChainResponse is used by add-chain and add-pre-chain to create and write diff --git a/trillian/ctfe/handlers_test.go b/trillian/ctfe/handlers_test.go index bc786409a0..ada5220cbd 100644 --- a/trillian/ctfe/handlers_test.go +++ b/trillian/ctfe/handlers_test.go @@ -165,7 +165,7 @@ func setupTest(t *testing.T, pemRoots []string, signer crypto.Signer) handlerTes cfg := &configpb.LogConfig{LogId: 0x42, Prefix: "test", IsMirror: false} vCfg := &ValidatedLogConfig{Config: cfg} iOpts := InstanceOptions{Validated: vCfg, Client: info.client, Deadline: time.Millisecond * 500, MetricFactory: monitoring.InertMetricFactory{}, RequestLog: new(DefaultRequestLog)} - info.li = newLogInfo(iOpts, vOpts, signer, fakeTimeSource) + info.li = newLogInfo(iOpts, vOpts, signer, fakeTimeSource, newIssuanceChainService(nil, nil)) for _, pemRoot := range pemRoots { if !info.roots.AppendCertsFromPEM([]byte(pemRoot)) { diff --git a/trillian/ctfe/instance.go b/trillian/ctfe/instance.go index 0056fd0756..f6bb4816fe 100644 --- a/trillian/ctfe/instance.go +++ b/trillian/ctfe/instance.go @@ -29,6 +29,8 @@ import ( "github.com/google/certificate-transparency-go/asn1" "github.com/google/certificate-transparency-go/schedule" + "github.com/google/certificate-transparency-go/trillian/ctfe/cache" + "github.com/google/certificate-transparency-go/trillian/ctfe/storage" "github.com/google/certificate-transparency-go/trillian/util" "github.com/google/certificate-transparency-go/x509" "github.com/google/certificate-transparency-go/x509util" @@ -174,7 +176,19 @@ func setUpLogInfo(ctx context.Context, opts InstanceOptions) (*logInfo, error) { return nil, fmt.Errorf("failed to parse RejectExtensions: %v", err) } - logInfo := newLogInfo(opts, validationOpts, signer, new(util.SystemTimeSource)) + // Initialise IssuanceChainService with IssuanceChainStorage and IssuanceChainCache + issuanceChainStorage, err := storage.NewIssuanceChainStorage(ctx, vCfg.ExtraDataIssuanceChainStorageBackend, vCfg.CTFEStorageConnectionString) + if err != nil { + return nil, err + } + issuanceChainCache, err := cache.NewIssuanceChainCache(ctx) + if err != nil { + return nil, err + } + + issuanceChainService := newIssuanceChainService(issuanceChainStorage, issuanceChainCache) + + logInfo := newLogInfo(opts, validationOpts, signer, new(util.SystemTimeSource), issuanceChainService) return logInfo, nil } diff --git a/trillian/ctfe/services.go b/trillian/ctfe/services.go index 580cf8763e..9a0b056c00 100644 --- a/trillian/ctfe/services.go +++ b/trillian/ctfe/services.go @@ -17,10 +17,16 @@ package ctfe import ( "context" "crypto/sha256" + "fmt" + "github.com/google/certificate-transparency-go/asn1" + "github.com/google/certificate-transparency-go/tls" "github.com/google/certificate-transparency-go/trillian/ctfe/cache" "github.com/google/certificate-transparency-go/trillian/ctfe/storage" + "github.com/google/trillian" "k8s.io/klog/v2" + + ct "github.com/google/certificate-transparency-go" ) type issuanceChainService struct { @@ -37,6 +43,10 @@ func newIssuanceChainService(s storage.IssuanceChainStorage, c cache.IssuanceCha return service } +func (s *issuanceChainService) IsCTFEStorageEnabled() bool { + return s.storage != nil +} + // GetByHash returns the issuance chain with hash as the input. func (s *issuanceChainService) GetByHash(ctx context.Context, hash []byte) ([]byte, error) { // Return if found in cache. @@ -82,6 +92,82 @@ func (s *issuanceChainService) Add(ctx context.Context, chain []byte) ([]byte, e return hash, nil } +func (s *issuanceChainService) FixLogLeaf(ctx context.Context, leaf *trillian.LogLeaf) error { + // Skip if CTFE storage backend is not enabled. + if s.storage == nil { + return nil + } + + var precertChainHash ct.PrecertChainEntryHash + if rest, err := tls.Unmarshal(leaf.ExtraData, &precertChainHash); err == nil && len(rest) == 0 { + var chain []ct.ASN1Cert + if len(precertChainHash.IssuanceChainHash) > 0 { + chainBytes, err := s.GetByHash(ctx, precertChainHash.IssuanceChainHash) + if err != nil { + return err + } + + if rest, err := asn1.Unmarshal(chainBytes, &chain); err != nil { + return err + } else if len(rest) > 0 { + return fmt.Errorf("IssuanceChain: trailing data %d bytes", len(rest)) + } + } + + precertChain := ct.PrecertChainEntry{ + PreCertificate: precertChainHash.PreCertificate, + CertificateChain: chain, + } + extraData, err := tls.Marshal(precertChain) + if err != nil { + return err + } + + leaf.ExtraData = extraData + return nil + } + + var certChainHash ct.CertificateChainHash + if rest, err := tls.Unmarshal(leaf.ExtraData, &certChainHash); err == nil && len(rest) == 0 { + var entries []ct.ASN1Cert + if len(certChainHash.IssuanceChainHash) > 0 { + chainBytes, err := s.GetByHash(ctx, certChainHash.IssuanceChainHash) + if err != nil { + return err + } + + if rest, err := asn1.Unmarshal(chainBytes, &entries); err != nil { + return err + } else if len(rest) > 0 { + return fmt.Errorf("IssuanceChain: trailing data %d bytes", len(rest)) + } + } + + certChain := ct.CertificateChain{ + Entries: entries, + } + extraData, err := tls.Marshal(certChain) + if err != nil { + return err + } + + leaf.ExtraData = extraData + return nil + } + + // Skip if the types are ct.PrecertChainEntry or ct.CertificateChain as there is no hash. + var precertChain ct.PrecertChainEntry + if rest, err := tls.Unmarshal(leaf.ExtraData, &precertChain); err == nil && len(rest) == 0 { + return nil + } + var certChain ct.CertificateChain + if rest, err := tls.Unmarshal(leaf.ExtraData, &certChain); err == nil && len(rest) == 0 { + return nil + } + + return fmt.Errorf("unknown extra data type in log leaf: %s", string(leaf.MerkleLeafHash)) +} + // issuanceChainHash returns the SHA-256 hash of the chain. func issuanceChainHash(chain []byte) []byte { checksum := sha256.Sum256(chain) diff --git a/trillian/ctfe/storage/storage.go b/trillian/ctfe/storage/storage.go index 5fccb72419..d9334cbc6b 100644 --- a/trillian/ctfe/storage/storage.go +++ b/trillian/ctfe/storage/storage.go @@ -17,6 +17,11 @@ package storage import ( "context" + "errors" + "strings" + + "github.com/google/certificate-transparency-go/trillian/ctfe/configpb" + "github.com/google/certificate-transparency-go/trillian/ctfe/storage/mysql" ) // IssuanceChainStorage is an interface which allows CTFE binaries to use different storage implementations for issuance chains. @@ -27,3 +32,19 @@ type IssuanceChainStorage interface { // Add inserts the key-value pair of issuance chain. Add(ctx context.Context, key []byte, chain []byte) error } + +// NewIssuanceChainStorage returns nil for Trillian gRPC or mysql.IssuanceChainStorage when MySQL is the prefix in database connection string. +func NewIssuanceChainStorage(ctx context.Context, backend configpb.LogConfig_IssuanceChainStorageBackend, dbConn string) (IssuanceChainStorage, error) { + switch backend { + case configpb.LogConfig_ISSUANCE_CHAIN_STORAGE_BACKEND_TRILLIAN_GRPC: + return nil, nil + case configpb.LogConfig_ISSUANCE_CHAIN_STORAGE_BACKEND_CTFE: + if strings.HasPrefix(dbConn, "mysql") { + return mysql.NewIssuanceChainStorage(ctx, dbConn), nil + } else { + return nil, errors.New("failed to initialise IssuanceChainService due to unsupported driver in CTFE storage connection string") + } + } + + return nil, errors.New("unsupported issuance chain storage backend") +} diff --git a/trillian/examples/deployment/docker/ctfe/ct_server_mysql.cfg b/trillian/examples/deployment/docker/ctfe/ct_server_mysql.cfg new file mode 100644 index 0000000000..f380c5ffc7 --- /dev/null +++ b/trillian/examples/deployment/docker/ctfe/ct_server_mysql.cfg @@ -0,0 +1,17 @@ +config { + log_id: @TREE_ID@ + prefix: "testlog" + roots_pem_file: "/ctfe-config/fake-ca.cert" + public_key: { + der: "\x30\x59\x30\x13\x06\x07\x2a\x86\x48\xce\x3d\x02\x01\x06\x08\x2a\x86\x48\xce\x3d\x03\x01\x07\x03\x42\x00\x04\x44\x6d\x69\x2c\x00\xec\xf3\xc7\xbb\x87\x7e\x57\xea\x04\xc3\x4b\x49\x01\xc4\x9a\x19\xf2\x49\x9b\x4c\x44\x1c\xac\xe0\xff\x27\x11\xce\x94\xa8\x85\xd9\xed\x42\x22\x5c\x54\xf6\x33\x73\xa3\x3d\x8b\xe8\x53\x48\xf5\x57\x50\x61\x96\x30\x5b\xc4\x9b\xa3\x04\xc3\x4b" + } + private_key: { + [type.googleapis.com/keyspb.PrivateKey] { + der: "\x30\x81\x87\x02\x01\x00\x30\x13\x06\x07\x2a\x86\x48\xce\x3d\x02\x01\x06\x08\x2a\x86\x48\xce\x3d\x03\x01\x07\x04\x6d\x30\x6b\x02\x01\x01\x04\x20\xd8\x8a\x49\xa2\x15\x3c\xbe\xb5\xb7\x6c\x63\xdc\xfd\xc0\x36\x64\x24\x88\xc3\x57\x9d\xfa\xd4\xa8\x70\x78\x32\x72\x29\x1a\xb1\x6f\xa1\x44\x03\x42\x00\x04\x44\x6d\x69\x2c\x00\xec\xf3\xc7\xbb\x87\x7e\x57\xea\x04\xc3\x4b\x49\x01\xc4\x9a\x19\xf2\x49\x9b\x4c\x44\x1c\xac\xe0\xff\x27\x11\xce\x94\xa8\x85\xd9\xed\x42\x22\x5c\x54\xf6\x33\x73\xa3\x3d\x8b\xe8\x53\x48\xf5\x57\x50\x61\x96\x30\x5b\xc4\x9b\xa3\x04\xc3\x4b" + } + } + max_merge_delay_sec: 86400 + expected_merge_delay_sec: 120 + ctfe_storage_connection_string: "test:zaphod@tcp(localhost:3306)/test" + extra_data_issuance_chain_storage_backends: ISSUANCE_CHAIN_STORAGE_BACKEND_CTFE +} diff --git a/trillian/util/log_leaf.go b/trillian/util/log_leaf.go index 71d3d89b69..8d24fc17a7 100644 --- a/trillian/util/log_leaf.go +++ b/trillian/util/log_leaf.go @@ -29,28 +29,7 @@ func BuildLogLeaf(logPrefix string, merkleLeaf ct.MerkleTreeLeaf, leafIndex int64, cert ct.ASN1Cert, chain []ct.ASN1Cert, isPrecert bool, ) (trillian.LogLeaf, error) { - leafData, err := tls.Marshal(merkleLeaf) - if err != nil { - klog.Warningf("%s: Failed to serialize Merkle leaf: %v", logPrefix, err) - return trillian.LogLeaf{}, err - } - - extraData, err := ExtraDataForChain(cert, chain, isPrecert) - if err != nil { - klog.Warningf("%s: Failed to serialize chain for ExtraData: %v", logPrefix, err) - return trillian.LogLeaf{}, err - } - - // leafIDHash allows Trillian to detect duplicate entries, so this should be - // a hash over the cert data. - leafIDHash := sha256.Sum256(cert.Data) - - return trillian.LogLeaf{ - LeafValue: leafData, - ExtraData: extraData, - LeafIndex: leafIndex, - LeafIdentityHash: leafIDHash[:], - }, nil + return buildLogLeaf(logPrefix, merkleLeaf, leafIndex, cert, chain, nil, isPrecert) } // ExtraDataForChain creates the extra data associated with a log entry as @@ -71,3 +50,57 @@ func ExtraDataForChain(cert ct.ASN1Cert, chain []ct.ASN1Cert, isPrecert bool) ([ } return tls.Marshal(extra) } + +func BuildLogLeafWithHash(logPrefix string, merkleLeaf ct.MerkleTreeLeaf, leafIndex int64, cert ct.ASN1Cert, chain []ct.ASN1Cert, hash []byte, isPrecert bool) (trillian.LogLeaf, error) { + return buildLogLeaf(logPrefix, merkleLeaf, leafIndex, cert, chain, hash, isPrecert) +} + +// ExtraDataForChainWithHash creates the extra data associated with a log entry as +// described in RFC6962 section 4.6. +func ExtraDataForChainWithHash(cert ct.ASN1Cert, chain []ct.ASN1Cert, hash []byte, isPrecert bool) ([]byte, error) { + var extra interface{} + + if isPrecert { + // For a pre-cert, the extra data is a TLS-encoded PrecertChainEntry. + extra = ct.PrecertChainEntryHash{ + PreCertificate: cert, + IssuanceChainHash: hash, + } + } else { + // For a certificate, the extra data is a TLS-encoded: + // ASN.1Cert certificate_chain<0..2^24-1>; + // containing the chain after the leaf. + extra = ct.CertificateChainHash{ + IssuanceChainHash: hash, + } + } + return tls.Marshal(extra) +} + +func buildLogLeaf(logPrefix string, merkleLeaf ct.MerkleTreeLeaf, leafIndex int64, cert ct.ASN1Cert, chain []ct.ASN1Cert, hash []byte, isPrecert bool) (trillian.LogLeaf, error) { + leafData, err := tls.Marshal(merkleLeaf) + if err != nil { + klog.Warningf("%s: Failed to serialize Merkle leaf: %v", logPrefix, err) + return trillian.LogLeaf{}, err + } + + var extraData []byte + if hash == nil { + extraData, err = ExtraDataForChain(cert, chain, isPrecert) + } else { + extraData, err = ExtraDataForChainWithHash(cert, chain, hash, isPrecert) + } + if err != nil { + klog.Warningf("%s: Failed to serialize chain for ExtraData: %v", logPrefix, err) + return trillian.LogLeaf{}, err + } + // leafIDHash allows Trillian to detect duplicate entries, so this should be + // a hash over the cert data. + leafIDHash := sha256.Sum256(cert.Data) + return trillian.LogLeaf{ + LeafValue: leafData, + ExtraData: extraData, + LeafIndex: leafIndex, + LeafIdentityHash: leafIDHash[:], + }, nil +} From 6814ecb8c14cbc0b3580839fdb8d2b2ba0a6a21a Mon Sep 17 00:00:00 2001 From: Roger Ng Date: Mon, 13 May 2024 00:02:34 +0000 Subject: [PATCH 02/55] Add CT server with MySQL configuration example --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a4d9deb713..62dfc8f7bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ To reduce CT/Trillian database storage by deduplication of the entire issuance chain (intermediate certificate(s) and root certificate) that is currently stored in the Trillian merkle tree leaf ExtraData field. Storage cost should be reduced by at least 33% for newly deduplicated entries. -No operation is required for existing logs when the change is rolled out. Log operators can choose to opt-in this change for new CT logs by adding new CTFE configs in the [LogMultiConfig](https://github.com/google/certificate-transparency-go/blob/master/trillian/ctfe/configpb/config.proto). +No operation is required for existing logs when the change is rolled out. Log operators can choose to opt-in this change for new CT logs by adding new CTFE configs in the [LogMultiConfig](trillian/ctfe/configpb/config.proto). See [example](trillian/examples/deployment/docker/ctfe/ct_server_mysql.cfg). - `ctfe_storage_connection_string` - `extra_data_issuance_chain_storage_backend` From 29bdc2aa8f72054eecdb5d3855b282cf942ad902 Mon Sep 17 00:00:00 2001 From: Roger Ng Date: Mon, 13 May 2024 12:19:47 +0000 Subject: [PATCH 03/55] Update CTFE storage saving changelog --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 62dfc8f7bf..f23f8bd238 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,9 +4,9 @@ ### CTFE Storage Saving: Extra Data Issuance Chain Deduplication -To reduce CT/Trillian database storage by deduplication of the entire issuance chain (intermediate certificate(s) and root certificate) that is currently stored in the Trillian merkle tree leaf ExtraData field. Storage cost should be reduced by at least 33% for newly deduplicated entries. +To reduce CT/Trillian database storage by deduplication of the entire issuance chain (intermediate certificate(s) and root certificate) that is currently stored in the Trillian merkle tree leaf ExtraData field. Storage cost should be reduced by at least 33% for new CT logs with this feature enabled. -No operation is required for existing logs when the change is rolled out. Log operators can choose to opt-in this change for new CT logs by adding new CTFE configs in the [LogMultiConfig](trillian/ctfe/configpb/config.proto). See [example](trillian/examples/deployment/docker/ctfe/ct_server_mysql.cfg). +Existing logs are not affected by this change. Log operators can choose to opt-in this change for new CT logs by adding new CTFE configs in the [LogMultiConfig](trillian/ctfe/configpb/config.proto). See [example](trillian/examples/deployment/docker/ctfe/ct_server_mysql.cfg). - `ctfe_storage_connection_string` - `extra_data_issuance_chain_storage_backend` From 694812432a26f66d02dfca685ffabc3730a56c36 Mon Sep 17 00:00:00 2001 From: Roger Ng Date: Mon, 13 May 2024 13:11:19 +0000 Subject: [PATCH 04/55] Fix always if true condition --- trillian/ctfe/handlers.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/trillian/ctfe/handlers.go b/trillian/ctfe/handlers.go index 9d89336104..b2e7a0d01e 100644 --- a/trillian/ctfe/handlers.go +++ b/trillian/ctfe/handlers.go @@ -958,7 +958,7 @@ func buildLogLeafForAddChain(li *logInfo, issuanceChain := raw[1:] // Trillian gRPC storage backend is enabled and CTFE storage backend is disabled. - if li.issuanceChainService.storage == nil { + if !li.issuanceChainService.IsCTFEStorageEnabled() { return util.BuildLogLeaf(li.LogPrefix, merkleLeaf, 0, raw[0], issuanceChain, isPrecert) } @@ -970,10 +970,8 @@ func buildLogLeafForAddChain(li *logInfo, } hash := issuanceChainHash(chainBytes) - // Set issuance chain to nil if Trillian gRPC storage backend is not enabled. - if li.issuanceChainService.storage != nil { - issuanceChain = nil - } + // Set issuance chain to nil to save the Trillian storage. + issuanceChain = nil return util.BuildLogLeafWithHash(li.LogPrefix, merkleLeaf, 0, raw[0], issuanceChain, []byte(hash), isPrecert) } From 041f07d859df49086312aa67d9f1a06cb52ab15c Mon Sep 17 00:00:00 2001 From: Roger Ng Date: Mon, 13 May 2024 13:25:50 +0000 Subject: [PATCH 05/55] Reuse `IsCTFEStorageEnabled` --- trillian/ctfe/services.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trillian/ctfe/services.go b/trillian/ctfe/services.go index 9a0b056c00..4547a9a7ee 100644 --- a/trillian/ctfe/services.go +++ b/trillian/ctfe/services.go @@ -94,7 +94,7 @@ func (s *issuanceChainService) Add(ctx context.Context, chain []byte) ([]byte, e func (s *issuanceChainService) FixLogLeaf(ctx context.Context, leaf *trillian.LogLeaf) error { // Skip if CTFE storage backend is not enabled. - if s.storage == nil { + if !s.IsCTFEStorageEnabled() { return nil } From f86cf76eee301b63d29e054e02d138d5ee4e27c2 Mon Sep 17 00:00:00 2001 From: Roger Ng Date: Mon, 13 May 2024 15:45:25 +0000 Subject: [PATCH 06/55] Setting default cache size and ttl to -1 --- trillian/ctfe/cache/cache.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/trillian/ctfe/cache/cache.go b/trillian/ctfe/cache/cache.go index a7918d45b2..a4b8402e8b 100644 --- a/trillian/ctfe/cache/cache.go +++ b/trillian/ctfe/cache/cache.go @@ -27,8 +27,8 @@ import ( var ( cacheType = flag.String("cache_type", "noop", "Supported cache type: noop, lru (Default: noop)") - size = flag.Int("cache_size", 0, "Size parameter set to 0 makes cache of unlimited size") - ttl = flag.Duration("cache_ttl", 0*time.Second, "Providing 0 TTL turns expiring off") + size = flag.Int("cache_size", -1, "Size parameter set to 0 makes cache of unlimited size") + ttl = flag.Duration("cache_ttl", -1*time.Second, "Providing 0 TTL turns expiring off") ) // IssuanceChainCache is an interface which allows CTFE binaries to use different cache implementations for issuance chains. From d632250a6a0cb8023276a930496c707708ed422b Mon Sep 17 00:00:00 2001 From: Roger Ng Date: Tue, 14 May 2024 09:54:57 +0000 Subject: [PATCH 07/55] Add code comments for `issuanceChainStorage` and `issuanceChainCache` --- trillian/ctfe/instance.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/trillian/ctfe/instance.go b/trillian/ctfe/instance.go index f6bb4816fe..c3048c580b 100644 --- a/trillian/ctfe/instance.go +++ b/trillian/ctfe/instance.go @@ -176,11 +176,13 @@ func setUpLogInfo(ctx context.Context, opts InstanceOptions) (*logInfo, error) { return nil, fmt.Errorf("failed to parse RejectExtensions: %v", err) } - // Initialise IssuanceChainService with IssuanceChainStorage and IssuanceChainCache + // Initialise IssuanceChainService with IssuanceChainStorage and IssuanceChainCache. + // issuanceChainStorage is nil for Trillian gRPC or mysql.IssuanceChainStorage when MySQL is the prefix in database connection string. issuanceChainStorage, err := storage.NewIssuanceChainStorage(ctx, vCfg.ExtraDataIssuanceChainStorageBackend, vCfg.CTFEStorageConnectionString) if err != nil { return nil, err } + // issuanceChainCache is nil if the cache related flags are not defined. issuanceChainCache, err := cache.NewIssuanceChainCache(ctx) if err != nil { return nil, err From 6f5c6ded685102f4b127e6022aa3115e5b68b6d1 Mon Sep 17 00:00:00 2001 From: Roger Ng Date: Tue, 14 May 2024 20:04:35 +0000 Subject: [PATCH 08/55] Fix incorrect NewIssuanceChainCache comment --- trillian/ctfe/cache/cache.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trillian/ctfe/cache/cache.go b/trillian/ctfe/cache/cache.go index a4b8402e8b..8b9feeec80 100644 --- a/trillian/ctfe/cache/cache.go +++ b/trillian/ctfe/cache/cache.go @@ -40,7 +40,7 @@ type IssuanceChainCache interface { Set(ctx context.Context, key []byte, chain []byte) error } -// NewIssuanceChainCache returns nil for noop type or lru.IssuanceChainCache for lru cache type. +// NewIssuanceChainCache returns noop.IssuanceChainCache for noop type or lru.IssuanceChainCache for lru cache type. func NewIssuanceChainCache(_ context.Context) (IssuanceChainCache, error) { switch *cacheType { case "noop": From 0350d94c7098f2aa0068920038fc0d0a88aa691a Mon Sep 17 00:00:00 2001 From: Roger Ng Date: Tue, 14 May 2024 21:48:43 +0000 Subject: [PATCH 09/55] Refactor `addChainInternal` and remove `buildLogLeafForAddChain` --- trillian/ctfe/handlers.go | 62 ++++++++++++++------------------------- 1 file changed, 22 insertions(+), 40 deletions(-) diff --git a/trillian/ctfe/handlers.go b/trillian/ctfe/handlers.go index b2e7a0d01e..5321562cfd 100644 --- a/trillian/ctfe/handlers.go +++ b/trillian/ctfe/handlers.go @@ -465,27 +465,36 @@ func addChainInternal(ctx context.Context, li *logInfo, w http.ResponseWriter, r // epoch, and use this throughout. timeMillis := uint64(li.TimeSource.Now().UnixNano() / millisPerNano) - // If CTFE storage is enabled for issuance chain, add the chain to storage and cache, get the chain hash for Trillian gRPC. + // Build the MerkleTreeLeaf that gets sent to the backend, and make a trillian.LogLeaf for it. + var leaf trillian.LogLeaf + merkleLeaf, err := ct.MerkleTreeLeafFromChain(chain, etype, timeMillis) + if err != nil { + return http.StatusBadRequest, fmt.Errorf("failed to build MerkleTreeLeaf: %s", err) + } + raw := extractRawCerts(chain) + + // If CTFE storage is enabled for issuance chain, add the chain to storage + // and cache, and then build log leaf. If Trillian gRPC is enabled for + // issuance chain, build the log leaf. if li.issuanceChainService.IsCTFEStorageEnabled() { // TODO: Check how to convert []ct.ASN1Cert to []byte correctly. - issuanceChain, err := asn1.Marshal(extractRawCerts(chain)[1:]) + issuanceChain, err := asn1.Marshal(raw[1:]) if err != nil { return http.StatusInternalServerError, fmt.Errorf("failed to marshal issuance chain: %s", err) } - _, err = li.issuanceChainService.Add(ctx, issuanceChain) + hash, err := li.issuanceChainService.Add(ctx, issuanceChain) if err != nil { return http.StatusInternalServerError, fmt.Errorf("failed to add issuance chain into CTFE storage: %s", err) } - } - - // Build the MerkleTreeLeaf that gets sent to the backend, and make a trillian.LogLeaf for it. - merkleLeaf, err := ct.MerkleTreeLeafFromChain(chain, etype, timeMillis) - if err != nil { - return http.StatusBadRequest, fmt.Errorf("failed to build MerkleTreeLeaf: %s", err) - } - leaf, err := buildLogLeafForAddChain(li, *merkleLeaf, chain, isPrecert) - if err != nil { - return http.StatusInternalServerError, fmt.Errorf("failed to build LogLeaf: %s", err) + leaf, err = util.BuildLogLeafWithHash(li.LogPrefix, *merkleLeaf, 0, raw[0], nil, hash, isPrecert) + if err != nil { + return http.StatusInternalServerError, fmt.Errorf("failed to build LogLeaf: %s", err) + } + } else { + leaf, err = util.BuildLogLeaf(li.LogPrefix, *merkleLeaf, 0, raw[0], raw[1:], isPrecert) + if err != nil { + return http.StatusInternalServerError, fmt.Errorf("failed to build LogLeaf: %s", err) + } } // Send the Merkle tree leaf on to the Log server. @@ -949,33 +958,6 @@ func extractRawCerts(chain []*x509.Certificate) []ct.ASN1Cert { return raw } -// buildLogLeafForAddChain does the hashing to build a LogLeaf that will be -// sent to the backend by add-chain and add-pre-chain endpoints. -func buildLogLeafForAddChain(li *logInfo, - merkleLeaf ct.MerkleTreeLeaf, chain []*x509.Certificate, isPrecert bool, -) (trillian.LogLeaf, error) { - raw := extractRawCerts(chain) - issuanceChain := raw[1:] - - // Trillian gRPC storage backend is enabled and CTFE storage backend is disabled. - if !li.issuanceChainService.IsCTFEStorageEnabled() { - return util.BuildLogLeaf(li.LogPrefix, merkleLeaf, 0, raw[0], issuanceChain, isPrecert) - } - - // CTFE storage backend is enabled. - // TODO: Check how to convert []ct.ASN1Cert to []byte correctly. - chainBytes, err := asn1.Marshal(issuanceChain) - if err != nil { - return trillian.LogLeaf{}, err - } - hash := issuanceChainHash(chainBytes) - - // Set issuance chain to nil to save the Trillian storage. - issuanceChain = nil - - return util.BuildLogLeafWithHash(li.LogPrefix, merkleLeaf, 0, raw[0], issuanceChain, []byte(hash), isPrecert) -} - // marshalAndWriteAddChainResponse is used by add-chain and add-pre-chain to create and write // the JSON response to the client func marshalAndWriteAddChainResponse(sct *ct.SignedCertificateTimestamp, signer crypto.Signer, w http.ResponseWriter) error { From 9b43c56be59ccde0719c95d545283de062d74aab Mon Sep 17 00:00:00 2001 From: Roger Ng Date: Wed, 15 May 2024 17:26:38 +0000 Subject: [PATCH 10/55] Move cache flags to ct_server main --- trillian/ctfe/cache/cache.go | 30 ++++++++++++++++++------------ trillian/ctfe/ct_server/main.go | 22 ++++++++++++++++++++-- trillian/ctfe/instance.go | 6 +++++- trillian/ctfe/instance_test.go | 3 ++- 4 files changed, 45 insertions(+), 16 deletions(-) diff --git a/trillian/ctfe/cache/cache.go b/trillian/ctfe/cache/cache.go index 8b9feeec80..0978e2ee45 100644 --- a/trillian/ctfe/cache/cache.go +++ b/trillian/ctfe/cache/cache.go @@ -18,19 +18,25 @@ package cache import ( "context" "errors" - "flag" "time" "github.com/google/certificate-transparency-go/trillian/ctfe/cache/lru" "github.com/google/certificate-transparency-go/trillian/ctfe/cache/noop" ) -var ( - cacheType = flag.String("cache_type", "noop", "Supported cache type: noop, lru (Default: noop)") - size = flag.Int("cache_size", -1, "Size parameter set to 0 makes cache of unlimited size") - ttl = flag.Duration("cache_ttl", -1*time.Second, "Providing 0 TTL turns expiring off") +type Type string + +const ( + Unknown Type = "" + NOOP Type = "noop" + LRU Type = "lru" ) +type Option struct { + Size int + TTL time.Duration +} + // IssuanceChainCache is an interface which allows CTFE binaries to use different cache implementations for issuance chains. type IssuanceChainCache interface { // Get returns the issuance chain associated with the provided hash. @@ -41,18 +47,18 @@ type IssuanceChainCache interface { } // NewIssuanceChainCache returns noop.IssuanceChainCache for noop type or lru.IssuanceChainCache for lru cache type. -func NewIssuanceChainCache(_ context.Context) (IssuanceChainCache, error) { - switch *cacheType { - case "noop": +func NewIssuanceChainCache(_ context.Context, cacheType Type, option Option) (IssuanceChainCache, error) { + switch cacheType { + case Unknown, NOOP: return &noop.IssuanceChainCache{}, nil - case "lru": - if *size < 0 { + case LRU: + if option.Size < 0 { return nil, errors.New("invalid cache_size flag") } - if *ttl < 0*time.Second { + if option.TTL < 0*time.Second { return nil, errors.New("invalid cache_ttl flag") } - return lru.NewIssuanceChainCache(lru.CacheOption{Size: *size, TTL: *ttl}), nil + return lru.NewIssuanceChainCache(lru.CacheOption{Size: option.Size, TTL: option.TTL}), nil } return nil, errors.New("invalid cache_type flag") diff --git a/trillian/ctfe/ct_server/main.go b/trillian/ctfe/ct_server/main.go index 0aa27a21df..4e20f58b43 100644 --- a/trillian/ctfe/ct_server/main.go +++ b/trillian/ctfe/ct_server/main.go @@ -32,6 +32,7 @@ import ( "time" "github.com/google/certificate-transparency-go/trillian/ctfe" + "github.com/google/certificate-transparency-go/trillian/ctfe/cache" "github.com/google/certificate-transparency-go/trillian/ctfe/configpb" "github.com/google/trillian" "github.com/google/trillian/crypto/keys" @@ -73,6 +74,9 @@ var ( quotaIntermediate = flag.Bool("quota_intermediate", true, "Enable requesting of quota for intermediate certificates in submitted chains") handlerPrefix = flag.String("handler_prefix", "", "If set e.g. to '/logs' will prefix all handlers that don't define a custom prefix") pkcs11ModulePath = flag.String("pkcs11_module_path", "", "Path to the PKCS#11 module to use for keys that use the PKCS#11 interface") + cacheType = flag.String("cache_type", "noop", "Supported cache type: noop, lru (Default: noop)") + cacheSize = flag.Int("cache_size", -1, "Size parameter set to 0 makes cache of unlimited size") + cacheTTL = flag.Duration("cache_ttl", -1*time.Second, "Providing 0 TTL turns expiring off") ) const unknownRemoteUser = "UNKNOWN_REMOTE" @@ -218,7 +222,19 @@ func main() { // client. var publicKeys []crypto.PublicKey for _, c := range cfg.LogConfigs.Config { - inst, err := setupAndRegister(ctx, clientMap[c.LogBackendName], *rpcDeadline, c, corsMux, *handlerPrefix, *maskInternalErrors) + inst, err := setupAndRegister(ctx, + clientMap[c.LogBackendName], + *rpcDeadline, + c, + corsMux, + *handlerPrefix, + *maskInternalErrors, + cache.Type(*cacheType), + cache.Option{ + Size: *cacheSize, + TTL: *cacheTTL, + }, + ) if err != nil { klog.Exitf("Failed to set up log instance for %+v: %v", cfg, err) } @@ -330,7 +346,7 @@ func awaitSignal(doneFn func()) { doneFn() } -func setupAndRegister(ctx context.Context, client trillian.TrillianLogClient, deadline time.Duration, cfg *configpb.LogConfig, mux *http.ServeMux, globalHandlerPrefix string, maskInternalErrors bool) (*ctfe.Instance, error) { +func setupAndRegister(ctx context.Context, client trillian.TrillianLogClient, deadline time.Duration, cfg *configpb.LogConfig, mux *http.ServeMux, globalHandlerPrefix string, maskInternalErrors bool, cacheType cache.Type, cacheOption cache.Option) (*ctfe.Instance, error) { vCfg, err := ctfe.ValidateLogConfig(cfg) if err != nil { return nil, err @@ -343,6 +359,8 @@ func setupAndRegister(ctx context.Context, client trillian.TrillianLogClient, de MetricFactory: prometheus.MetricFactory{}, RequestLog: new(ctfe.DefaultRequestLog), MaskInternalErrors: maskInternalErrors, + CacheType: cacheType, + CacheOption: cacheOption, } if *quotaRemote { klog.Info("Enabling quota for requesting IP") diff --git a/trillian/ctfe/instance.go b/trillian/ctfe/instance.go index c3048c580b..206b3cbd13 100644 --- a/trillian/ctfe/instance.go +++ b/trillian/ctfe/instance.go @@ -74,6 +74,10 @@ type InstanceOptions struct { // MaskInternalErrors indicates if internal server errors should be masked // or returned to the user containing the full error message. MaskInternalErrors bool + // CacheType is the CTFE cache type. + CacheType cache.Type + // CacheOption includes the cache size and time-to-live (TTL). + CacheOption cache.Option } // Instance is a set up log/mirror instance. It must be created with the @@ -183,7 +187,7 @@ func setUpLogInfo(ctx context.Context, opts InstanceOptions) (*logInfo, error) { return nil, err } // issuanceChainCache is nil if the cache related flags are not defined. - issuanceChainCache, err := cache.NewIssuanceChainCache(ctx) + issuanceChainCache, err := cache.NewIssuanceChainCache(ctx, opts.CacheType, opts.CacheOption) if err != nil { return nil, err } diff --git a/trillian/ctfe/instance_test.go b/trillian/ctfe/instance_test.go index 10cf9e27b5..c191a5157a 100644 --- a/trillian/ctfe/instance_test.go +++ b/trillian/ctfe/instance_test.go @@ -24,6 +24,7 @@ import ( "time" ct "github.com/google/certificate-transparency-go" + "github.com/google/certificate-transparency-go/trillian/ctfe/cache" "github.com/google/certificate-transparency-go/trillian/ctfe/configpb" "github.com/google/trillian/crypto/keys" "github.com/google/trillian/crypto/keys/pem" @@ -257,7 +258,7 @@ func TestSetUpInstanceSetsValidationOpts(t *testing.T) { if err != nil { t.Fatalf("ValidateLogConfig(): %v", err) } - opts := InstanceOptions{Validated: vCfg, Deadline: time.Second, MetricFactory: monitoring.InertMetricFactory{}} + opts := InstanceOptions{Validated: vCfg, Deadline: time.Second, MetricFactory: monitoring.InertMetricFactory{}, CacheType: cache.NOOP, CacheOption: cache.Option{}} inst, err := SetUpInstance(ctx, opts) if err != nil { From 2e05680841643b9204b9f2ecc8cd3ffb6dc72dc5 Mon Sep 17 00:00:00 2001 From: Roger Ng Date: Wed, 15 May 2024 17:37:16 +0000 Subject: [PATCH 11/55] Remove TODO comment for `[]ct.ASN1Cert` to `[]byte` conversion --- trillian/ctfe/handlers.go | 1 - 1 file changed, 1 deletion(-) diff --git a/trillian/ctfe/handlers.go b/trillian/ctfe/handlers.go index 5321562cfd..b7efaa0195 100644 --- a/trillian/ctfe/handlers.go +++ b/trillian/ctfe/handlers.go @@ -477,7 +477,6 @@ func addChainInternal(ctx context.Context, li *logInfo, w http.ResponseWriter, r // and cache, and then build log leaf. If Trillian gRPC is enabled for // issuance chain, build the log leaf. if li.issuanceChainService.IsCTFEStorageEnabled() { - // TODO: Check how to convert []ct.ASN1Cert to []byte correctly. issuanceChain, err := asn1.Marshal(raw[1:]) if err != nil { return http.StatusInternalServerError, fmt.Errorf("failed to marshal issuance chain: %s", err) From 2aa94b1203fd715028d4d2901778ad409a8fe2b1 Mon Sep 17 00:00:00 2001 From: Roger Ng Date: Wed, 15 May 2024 23:03:53 +0000 Subject: [PATCH 12/55] Add FixLogLeaf method comment --- trillian/ctfe/services.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/trillian/ctfe/services.go b/trillian/ctfe/services.go index 4547a9a7ee..1b887577d8 100644 --- a/trillian/ctfe/services.go +++ b/trillian/ctfe/services.go @@ -92,6 +92,9 @@ func (s *issuanceChainService) Add(ctx context.Context, chain []byte) ([]byte, e return hash, nil } +// FixLogLeaf recreates the LogLeaf.ExtraData if CTFE storage backend is +// enabled and the type of LogLeaf.ExtraData contains any hash (e.g. +// PrecertChainEntryHash, CertificateChainHash). func (s *issuanceChainService) FixLogLeaf(ctx context.Context, leaf *trillian.LogLeaf) error { // Skip if CTFE storage backend is not enabled. if !s.IsCTFEStorageEnabled() { From e4a8b776b3be7052e585dc0160210b5304de5fa6 Mon Sep 17 00:00:00 2001 From: Roger Ng Date: Wed, 15 May 2024 23:07:09 +0000 Subject: [PATCH 13/55] Return err in `GetByHash` and `Add` when storage is nil --- trillian/ctfe/services.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/trillian/ctfe/services.go b/trillian/ctfe/services.go index 1b887577d8..e81359e343 100644 --- a/trillian/ctfe/services.go +++ b/trillian/ctfe/services.go @@ -17,6 +17,7 @@ package ctfe import ( "context" "crypto/sha256" + "errors" "fmt" "github.com/google/certificate-transparency-go/asn1" @@ -49,6 +50,11 @@ func (s *issuanceChainService) IsCTFEStorageEnabled() bool { // GetByHash returns the issuance chain with hash as the input. func (s *issuanceChainService) GetByHash(ctx context.Context, hash []byte) ([]byte, error) { + // Return err if CTFE storage backend is not enabled. + if !s.IsCTFEStorageEnabled() { + return nil, errors.New("failed to GetByHash when storage is nil") + } + // Return if found in cache. chain, err := s.cache.Get(ctx, hash) if chain != nil || err != nil { @@ -75,6 +81,11 @@ func (s *issuanceChainService) GetByHash(ctx context.Context, hash []byte) ([]by // Add adds the issuance chain into the storage and cache and returns the hash // of the chain. func (s *issuanceChainService) Add(ctx context.Context, chain []byte) ([]byte, error) { + // Return err if CTFE storage backend is not enabled. + if !s.IsCTFEStorageEnabled() { + return nil, errors.New("failed to Add when storage is nil") + } + hash := issuanceChainHash(chain) if err := s.storage.Add(ctx, hash, chain); err != nil { From f05e1a8045edb9ff088ddb6085a530e4da6dc701 Mon Sep 17 00:00:00 2001 From: Roger Ng Date: Thu, 16 May 2024 10:09:31 +0000 Subject: [PATCH 14/55] Enable MySQL in ct server test --- .../examples/deployment/docker/ctfe/README.md | 1 + .../deployment/docker/ctfe/ct_server.cfg | 2 ++ .../deployment/docker/ctfe/ct_server_mysql.cfg | 17 ----------------- .../deployment/docker/ctfe/docker-compose.yaml | 1 + 4 files changed, 4 insertions(+), 17 deletions(-) delete mode 100644 trillian/examples/deployment/docker/ctfe/ct_server_mysql.cfg diff --git a/trillian/examples/deployment/docker/ctfe/README.md b/trillian/examples/deployment/docker/ctfe/README.md index ab5d2bff34..c3953433ce 100644 --- a/trillian/examples/deployment/docker/ctfe/README.md +++ b/trillian/examples/deployment/docker/ctfe/README.md @@ -39,6 +39,7 @@ First bring up the trillian instance and the database: # Terminal 1 cd ${GIT_HOME}/certificate-transparency-go/trillian/examples/deployment/docker/ctfe/ docker compose up +docker exec -i ctfe-db mariadb -pzaphod -Dtest < ${GIT_HOME}/certificate-transparency-go/trillian/ctfe/storage/mysql/schema.sql ``` This brings up everything except the CTFE. Now to provision the logs. diff --git a/trillian/examples/deployment/docker/ctfe/ct_server.cfg b/trillian/examples/deployment/docker/ctfe/ct_server.cfg index d16998f261..f380c5ffc7 100644 --- a/trillian/examples/deployment/docker/ctfe/ct_server.cfg +++ b/trillian/examples/deployment/docker/ctfe/ct_server.cfg @@ -12,4 +12,6 @@ config { } max_merge_delay_sec: 86400 expected_merge_delay_sec: 120 + ctfe_storage_connection_string: "test:zaphod@tcp(localhost:3306)/test" + extra_data_issuance_chain_storage_backends: ISSUANCE_CHAIN_STORAGE_BACKEND_CTFE } diff --git a/trillian/examples/deployment/docker/ctfe/ct_server_mysql.cfg b/trillian/examples/deployment/docker/ctfe/ct_server_mysql.cfg deleted file mode 100644 index f380c5ffc7..0000000000 --- a/trillian/examples/deployment/docker/ctfe/ct_server_mysql.cfg +++ /dev/null @@ -1,17 +0,0 @@ -config { - log_id: @TREE_ID@ - prefix: "testlog" - roots_pem_file: "/ctfe-config/fake-ca.cert" - public_key: { - der: "\x30\x59\x30\x13\x06\x07\x2a\x86\x48\xce\x3d\x02\x01\x06\x08\x2a\x86\x48\xce\x3d\x03\x01\x07\x03\x42\x00\x04\x44\x6d\x69\x2c\x00\xec\xf3\xc7\xbb\x87\x7e\x57\xea\x04\xc3\x4b\x49\x01\xc4\x9a\x19\xf2\x49\x9b\x4c\x44\x1c\xac\xe0\xff\x27\x11\xce\x94\xa8\x85\xd9\xed\x42\x22\x5c\x54\xf6\x33\x73\xa3\x3d\x8b\xe8\x53\x48\xf5\x57\x50\x61\x96\x30\x5b\xc4\x9b\xa3\x04\xc3\x4b" - } - private_key: { - [type.googleapis.com/keyspb.PrivateKey] { - der: "\x30\x81\x87\x02\x01\x00\x30\x13\x06\x07\x2a\x86\x48\xce\x3d\x02\x01\x06\x08\x2a\x86\x48\xce\x3d\x03\x01\x07\x04\x6d\x30\x6b\x02\x01\x01\x04\x20\xd8\x8a\x49\xa2\x15\x3c\xbe\xb5\xb7\x6c\x63\xdc\xfd\xc0\x36\x64\x24\x88\xc3\x57\x9d\xfa\xd4\xa8\x70\x78\x32\x72\x29\x1a\xb1\x6f\xa1\x44\x03\x42\x00\x04\x44\x6d\x69\x2c\x00\xec\xf3\xc7\xbb\x87\x7e\x57\xea\x04\xc3\x4b\x49\x01\xc4\x9a\x19\xf2\x49\x9b\x4c\x44\x1c\xac\xe0\xff\x27\x11\xce\x94\xa8\x85\xd9\xed\x42\x22\x5c\x54\xf6\x33\x73\xa3\x3d\x8b\xe8\x53\x48\xf5\x57\x50\x61\x96\x30\x5b\xc4\x9b\xa3\x04\xc3\x4b" - } - } - max_merge_delay_sec: 86400 - expected_merge_delay_sec: 120 - ctfe_storage_connection_string: "test:zaphod@tcp(localhost:3306)/test" - extra_data_issuance_chain_storage_backends: ISSUANCE_CHAIN_STORAGE_BACKEND_CTFE -} diff --git a/trillian/examples/deployment/docker/ctfe/docker-compose.yaml b/trillian/examples/deployment/docker/ctfe/docker-compose.yaml index e2db6c2d0c..2e06abb89d 100644 --- a/trillian/examples/deployment/docker/ctfe/docker-compose.yaml +++ b/trillian/examples/deployment/docker/ctfe/docker-compose.yaml @@ -66,6 +66,7 @@ services: volumes: - ctfe_config:/ctfe-config:ro depends_on: + - db - trillian-log-server volumes: From 573c9cb5a6ce743d69252599afeaa4e5924c6af9 Mon Sep 17 00:00:00 2001 From: Roger Ng Date: Thu, 16 May 2024 14:48:00 +0000 Subject: [PATCH 15/55] Refactor log leaf build logic --- trillian/ctfe/handlers.go | 29 ++++------------------------- trillian/ctfe/services.go | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 25 deletions(-) diff --git a/trillian/ctfe/handlers.go b/trillian/ctfe/handlers.go index b7efaa0195..69eba6cd4b 100644 --- a/trillian/ctfe/handlers.go +++ b/trillian/ctfe/handlers.go @@ -466,40 +466,19 @@ func addChainInternal(ctx context.Context, li *logInfo, w http.ResponseWriter, r timeMillis := uint64(li.TimeSource.Now().UnixNano() / millisPerNano) // Build the MerkleTreeLeaf that gets sent to the backend, and make a trillian.LogLeaf for it. - var leaf trillian.LogLeaf merkleLeaf, err := ct.MerkleTreeLeafFromChain(chain, etype, timeMillis) if err != nil { return http.StatusBadRequest, fmt.Errorf("failed to build MerkleTreeLeaf: %s", err) } - raw := extractRawCerts(chain) - - // If CTFE storage is enabled for issuance chain, add the chain to storage - // and cache, and then build log leaf. If Trillian gRPC is enabled for - // issuance chain, build the log leaf. - if li.issuanceChainService.IsCTFEStorageEnabled() { - issuanceChain, err := asn1.Marshal(raw[1:]) - if err != nil { - return http.StatusInternalServerError, fmt.Errorf("failed to marshal issuance chain: %s", err) - } - hash, err := li.issuanceChainService.Add(ctx, issuanceChain) - if err != nil { - return http.StatusInternalServerError, fmt.Errorf("failed to add issuance chain into CTFE storage: %s", err) - } - leaf, err = util.BuildLogLeafWithHash(li.LogPrefix, *merkleLeaf, 0, raw[0], nil, hash, isPrecert) - if err != nil { - return http.StatusInternalServerError, fmt.Errorf("failed to build LogLeaf: %s", err) - } - } else { - leaf, err = util.BuildLogLeaf(li.LogPrefix, *merkleLeaf, 0, raw[0], raw[1:], isPrecert) - if err != nil { - return http.StatusInternalServerError, fmt.Errorf("failed to build LogLeaf: %s", err) - } + leaf, err := li.issuanceChainService.BuildLogLeaf(ctx, chain, li.LogPrefix, merkleLeaf, isPrecert) + if err != nil { + return http.StatusInternalServerError, err } // Send the Merkle tree leaf on to the Log server. req := trillian.QueueLeafRequest{ LogId: li.logID, - Leaf: &leaf, + Leaf: leaf, ChargeTo: li.chargeUser(r), } if li.instanceOpts.CertificateQuotaUser != nil { diff --git a/trillian/ctfe/services.go b/trillian/ctfe/services.go index e81359e343..b72dde2abe 100644 --- a/trillian/ctfe/services.go +++ b/trillian/ctfe/services.go @@ -24,6 +24,8 @@ import ( "github.com/google/certificate-transparency-go/tls" "github.com/google/certificate-transparency-go/trillian/ctfe/cache" "github.com/google/certificate-transparency-go/trillian/ctfe/storage" + "github.com/google/certificate-transparency-go/trillian/util" + "github.com/google/certificate-transparency-go/x509" "github.com/google/trillian" "k8s.io/klog/v2" @@ -103,6 +105,37 @@ func (s *issuanceChainService) Add(ctx context.Context, chain []byte) ([]byte, e return hash, nil } +// BuildLogLeaf builds the MerkleTreeLeaf that gets sent to the backend, and make a trillian.LogLeaf for it. +func (s *issuanceChainService) BuildLogLeaf(ctx context.Context, chain []*x509.Certificate, logPrefix string, merkleLeaf *ct.MerkleTreeLeaf, isPrecert bool) (*trillian.LogLeaf, error) { + leaf := trillian.LogLeaf{} + raw := extractRawCerts(chain) + + // If CTFE storage is enabled for issuance chain, add the chain to storage + // and cache, and then build log leaf. If Trillian gRPC is enabled for + // issuance chain, build the log leaf. + if s.IsCTFEStorageEnabled() { + issuanceChain, err := asn1.Marshal(raw[1:]) + if err != nil { + return &leaf, fmt.Errorf("failed to marshal issuance chain: %s", err) + } + hash, err := s.Add(ctx, issuanceChain) + if err != nil { + return &leaf, fmt.Errorf("failed to add issuance chain into CTFE storage: %s", err) + } + leaf, err := util.BuildLogLeafWithHash(logPrefix, *merkleLeaf, 0, raw[0], nil, hash, isPrecert) + if err != nil { + return &leaf, fmt.Errorf("failed to build LogLeaf: %s", err) + } + } else { + leaf, err := util.BuildLogLeaf(logPrefix, *merkleLeaf, 0, raw[0], raw[1:], isPrecert) + if err != nil { + return &leaf, fmt.Errorf("failed to build LogLeaf: %s", err) + } + } + + return &leaf, nil +} + // FixLogLeaf recreates the LogLeaf.ExtraData if CTFE storage backend is // enabled and the type of LogLeaf.ExtraData contains any hash (e.g. // PrecertChainEntryHash, CertificateChainHash). From d11912ba0f2f5022b59baf39613d8a3affdb406a Mon Sep 17 00:00:00 2001 From: Roger Ng Date: Thu, 16 May 2024 18:18:03 +0000 Subject: [PATCH 16/55] Fix `leaf` var scope bug --- trillian/ctfe/services.go | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/trillian/ctfe/services.go b/trillian/ctfe/services.go index b72dde2abe..12850018f6 100644 --- a/trillian/ctfe/services.go +++ b/trillian/ctfe/services.go @@ -107,7 +107,6 @@ func (s *issuanceChainService) Add(ctx context.Context, chain []byte) ([]byte, e // BuildLogLeaf builds the MerkleTreeLeaf that gets sent to the backend, and make a trillian.LogLeaf for it. func (s *issuanceChainService) BuildLogLeaf(ctx context.Context, chain []*x509.Certificate, logPrefix string, merkleLeaf *ct.MerkleTreeLeaf, isPrecert bool) (*trillian.LogLeaf, error) { - leaf := trillian.LogLeaf{} raw := extractRawCerts(chain) // If CTFE storage is enabled for issuance chain, add the chain to storage @@ -116,24 +115,24 @@ func (s *issuanceChainService) BuildLogLeaf(ctx context.Context, chain []*x509.C if s.IsCTFEStorageEnabled() { issuanceChain, err := asn1.Marshal(raw[1:]) if err != nil { - return &leaf, fmt.Errorf("failed to marshal issuance chain: %s", err) + return &trillian.LogLeaf{}, fmt.Errorf("failed to marshal issuance chain: %s", err) } hash, err := s.Add(ctx, issuanceChain) if err != nil { - return &leaf, fmt.Errorf("failed to add issuance chain into CTFE storage: %s", err) + return &trillian.LogLeaf{}, fmt.Errorf("failed to add issuance chain into CTFE storage: %s", err) } leaf, err := util.BuildLogLeafWithHash(logPrefix, *merkleLeaf, 0, raw[0], nil, hash, isPrecert) if err != nil { - return &leaf, fmt.Errorf("failed to build LogLeaf: %s", err) + return &trillian.LogLeaf{}, fmt.Errorf("failed to build LogLeaf: %s", err) } + return &leaf, nil } else { leaf, err := util.BuildLogLeaf(logPrefix, *merkleLeaf, 0, raw[0], raw[1:], isPrecert) if err != nil { - return &leaf, fmt.Errorf("failed to build LogLeaf: %s", err) + return &trillian.LogLeaf{}, fmt.Errorf("failed to build LogLeaf: %s", err) } + return &leaf, nil } - - return &leaf, nil } // FixLogLeaf recreates the LogLeaf.ExtraData if CTFE storage backend is From 3168c3be84e20eadc9bcd3963e5430eaa3ace6b8 Mon Sep 17 00:00:00 2001 From: Roger Ng Date: Thu, 16 May 2024 18:21:07 +0000 Subject: [PATCH 17/55] Rename method from public `Add` to private `add` --- trillian/ctfe/services.go | 6 +++--- trillian/ctfe/services_test.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/trillian/ctfe/services.go b/trillian/ctfe/services.go index 12850018f6..d53efaff93 100644 --- a/trillian/ctfe/services.go +++ b/trillian/ctfe/services.go @@ -80,9 +80,9 @@ func (s *issuanceChainService) GetByHash(ctx context.Context, hash []byte) ([]by return chain, nil } -// Add adds the issuance chain into the storage and cache and returns the hash +// add adds the issuance chain into the storage and cache and returns the hash // of the chain. -func (s *issuanceChainService) Add(ctx context.Context, chain []byte) ([]byte, error) { +func (s *issuanceChainService) add(ctx context.Context, chain []byte) ([]byte, error) { // Return err if CTFE storage backend is not enabled. if !s.IsCTFEStorageEnabled() { return nil, errors.New("failed to Add when storage is nil") @@ -117,7 +117,7 @@ func (s *issuanceChainService) BuildLogLeaf(ctx context.Context, chain []*x509.C if err != nil { return &trillian.LogLeaf{}, fmt.Errorf("failed to marshal issuance chain: %s", err) } - hash, err := s.Add(ctx, issuanceChain) + hash, err := s.add(ctx, issuanceChain) if err != nil { return &trillian.LogLeaf{}, fmt.Errorf("failed to add issuance chain into CTFE storage: %s", err) } diff --git a/trillian/ctfe/services_test.go b/trillian/ctfe/services_test.go index 3a9f7c12b4..4082fc4bff 100644 --- a/trillian/ctfe/services_test.go +++ b/trillian/ctfe/services_test.go @@ -39,7 +39,7 @@ func TestIssuanceChainServiceAddAndGet(t *testing.T) { issuanceChainService := newIssuanceChainService(storage, cache) for _, test := range tests { - hash, err := issuanceChainService.Add(ctx, test.chain) + hash, err := issuanceChainService.add(ctx, test.chain) if err != nil { t.Errorf("IssuanceChainService.Add(): %v", err) } From 1c3bf0bb7f4cf88d65c384c29629d7d5b206cc3b Mon Sep 17 00:00:00 2001 From: Roger Ng Date: Thu, 16 May 2024 18:42:35 +0000 Subject: [PATCH 18/55] Refactor `issuanceChainService.FixLogLeaf` into `rpcGetLeavesByRange` and `rpcGetEntryAndProof` --- trillian/ctfe/handlers.go | 43 ++++++++++++++++++++++++++++----------- 1 file changed, 31 insertions(+), 12 deletions(-) diff --git a/trillian/ctfe/handlers.go b/trillian/ctfe/handlers.go index 69eba6cd4b..cb4b263130 100644 --- a/trillian/ctfe/handlers.go +++ b/trillian/ctfe/handlers.go @@ -750,10 +750,11 @@ func getEntries(ctx context.Context, li *logInfo, w http.ResponseWriter, r *http Count: count, ChargeTo: li.chargeUser(r), } - rsp, err := li.rpcClient.GetLeavesByRange(ctx, &req) + rsp, httpStatus, err := rpcGetLeavesByRange(ctx, li, &req) if err != nil { - return li.toHTTPStatus(err), fmt.Errorf("backend GetLeavesByRange request failed: %s", err) + return httpStatus, err } + var currentRoot types.LogRootV1 if err := currentRoot.UnmarshalBinary(rsp.GetSignedLogRoot().GetLogRoot()); err != nil { return http.StatusInternalServerError, fmt.Errorf("failed to unmarshal root: %v", rsp.GetSignedLogRoot().GetLogRoot()) @@ -775,10 +776,6 @@ func getEntries(ctx context.Context, li *logInfo, w http.ResponseWriter, r *http if leaf.LeafIndex != start+int64(i) { return http.StatusInternalServerError, fmt.Errorf("backend returned unexpected leaf index: rsp.Leaves[%d].LeafIndex=%d for range [%d,%d]", i, leaf.LeafIndex, start, end) } - - if err := li.issuanceChainService.FixLogLeaf(ctx, leaf); err != nil { - return http.StatusInternalServerError, fmt.Errorf("failed to fix log leaf: %v", rsp) - } } leaves = rsp.Leaves @@ -807,6 +804,20 @@ func getEntries(ctx context.Context, li *logInfo, w http.ResponseWriter, r *http return http.StatusOK, nil } +func rpcGetLeavesByRange(ctx context.Context, li *logInfo, req *trillian.GetLeavesByRangeRequest) (*trillian.GetLeavesByRangeResponse, int, error) { + rsp, err := li.rpcClient.GetLeavesByRange(ctx, req) + if err != nil { + return nil, li.toHTTPStatus(err), fmt.Errorf("backend GetLeavesByRange request failed: %s", err) + } + for _, leaf := range rsp.Leaves { + if err := li.issuanceChainService.FixLogLeaf(ctx, leaf); err != nil { + return nil, http.StatusInternalServerError, fmt.Errorf("failed to fix log leaf: %v", rsp) + } + } + + return rsp, http.StatusOK, nil +} + func getRoots(_ context.Context, li *logInfo, w http.ResponseWriter, _ *http.Request) (int, error) { // Pull out the raw certificates from the parsed versions rawCerts := make([][]byte, 0, len(li.validationOpts.trustedRoots.RawCertificates())) @@ -843,9 +854,9 @@ func getEntryAndProof(ctx context.Context, li *logInfo, w http.ResponseWriter, r TreeSize: treeSize, ChargeTo: li.chargeUser(r), } - rsp, err := li.rpcClient.GetEntryAndProof(ctx, &req) + rsp, httpStatus, err := rpcGetEntryAndProof(ctx, li, &req) if err != nil { - return li.toHTTPStatus(err), fmt.Errorf("backend GetEntryAndProof request failed: %s", err) + return httpStatus, err } var currentRoot types.LogRootV1 @@ -866,10 +877,6 @@ func getEntryAndProof(ctx context.Context, li *logInfo, w http.ResponseWriter, r return http.StatusInternalServerError, fmt.Errorf("got RPC bad response (missing proof), possible extra info: %v", rsp) } - if err := li.issuanceChainService.FixLogLeaf(ctx, rsp.Leaf); err != nil { - return http.StatusInternalServerError, fmt.Errorf("failed to fix log leaf: %v", rsp) - } - // Build and marshal the response to the client jsonRsp := ct.GetEntryAndProofResponse{ LeafInput: rsp.Leaf.LeafValue, @@ -894,6 +901,18 @@ func getEntryAndProof(ctx context.Context, li *logInfo, w http.ResponseWriter, r return http.StatusOK, nil } +func rpcGetEntryAndProof(ctx context.Context, li *logInfo, req *trillian.GetEntryAndProofRequest) (*trillian.GetEntryAndProofResponse, int, error) { + rsp, err := li.rpcClient.GetEntryAndProof(ctx, req) + if err != nil { + return nil, li.toHTTPStatus(err), fmt.Errorf("backend GetEntryAndProof request failed: %s", err) + } + if err := li.issuanceChainService.FixLogLeaf(ctx, rsp.Leaf); err != nil { + return nil, http.StatusInternalServerError, fmt.Errorf("failed to fix log leaf: %v", rsp) + } + + return rsp, http.StatusOK, nil +} + // getRPCDeadlineTime calculates the future time an RPC should expire based on our config func getRPCDeadlineTime(li *logInfo) time.Time { return li.TimeSource.Now().Add(li.instanceOpts.Deadline) From b51e8dbf0c2633fc953e0934d76ec82799205b09 Mon Sep 17 00:00:00 2001 From: Roger Ng Date: Thu, 16 May 2024 18:47:18 +0000 Subject: [PATCH 19/55] Add `rpcGetLeavesByRange` and `rpcGetEntryAndProof` method comment --- trillian/ctfe/handlers.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/trillian/ctfe/handlers.go b/trillian/ctfe/handlers.go index cb4b263130..2ade820d3d 100644 --- a/trillian/ctfe/handlers.go +++ b/trillian/ctfe/handlers.go @@ -804,6 +804,7 @@ func getEntries(ctx context.Context, li *logInfo, w http.ResponseWriter, r *http return http.StatusOK, nil } +// rpcGetLeavesByRange calls Trillian GetLeavesByRange RPC and fixes issuance chain in each log leaf if necessary. func rpcGetLeavesByRange(ctx context.Context, li *logInfo, req *trillian.GetLeavesByRangeRequest) (*trillian.GetLeavesByRangeResponse, int, error) { rsp, err := li.rpcClient.GetLeavesByRange(ctx, req) if err != nil { @@ -901,6 +902,7 @@ func getEntryAndProof(ctx context.Context, li *logInfo, w http.ResponseWriter, r return http.StatusOK, nil } +// rpcGetEntryAndProof calls Trillian GetEntryAndProof RPC and fixes issuance chain in the log leaf if necessary. func rpcGetEntryAndProof(ctx context.Context, li *logInfo, req *trillian.GetEntryAndProofRequest) (*trillian.GetEntryAndProofResponse, int, error) { rsp, err := li.rpcClient.GetEntryAndProof(ctx, req) if err != nil { From 7e0f5b0268fad1b5012e8c6fdd7cb0f7c867b174 Mon Sep 17 00:00:00 2001 From: Roger Ng Date: Thu, 16 May 2024 19:05:29 +0000 Subject: [PATCH 20/55] Add more information in CHANGELOG.md --- CHANGELOG.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f23f8bd238..72d68c13a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,9 +4,11 @@ ### CTFE Storage Saving: Extra Data Issuance Chain Deduplication -To reduce CT/Trillian database storage by deduplication of the entire issuance chain (intermediate certificate(s) and root certificate) that is currently stored in the Trillian merkle tree leaf ExtraData field. Storage cost should be reduced by at least 33% for new CT logs with this feature enabled. +To reduce CT/Trillian database storage by deduplication of the entire issuance chain (intermediate certificate(s) and root certificate) that is currently stored in the Trillian merkle tree leaf ExtraData field. Storage cost should be reduced by at least 33% for new CT logs with this feature enabled. Currently only MySQL/MariaDB is supported to store the issuance chain in the CTFE database. -Existing logs are not affected by this change. Log operators can choose to opt-in this change for new CT logs by adding new CTFE configs in the [LogMultiConfig](trillian/ctfe/configpb/config.proto). See [example](trillian/examples/deployment/docker/ctfe/ct_server_mysql.cfg). +Existing logs are not affected by this change. + +Log operators can choose to opt-in this change for new CT logs by adding new CTFE configs in the [LogMultiConfig](trillian/ctfe/configpb/config.proto). See [example](trillian/examples/deployment/docker/ctfe/ct_server.cfg). - `ctfe_storage_connection_string` - `extra_data_issuance_chain_storage_backend` @@ -17,6 +19,8 @@ An optional LRU cache can be enabled by providing the following flags. - `cache_size` - `cache_ttl` +This change is tested in Cloud Build tests using the `mysql:8.2` Docker image. + ### Submission proxy: Root compatibility checking * Adds the ability for a CT client to disable root compatibile checking: https://github.com/google/certificate-transparency-go/pull/1258 From 116090cd818d53f04dfa9015504fe705115f5602 Mon Sep 17 00:00:00 2001 From: Roger Ng Date: Fri, 17 May 2024 10:21:59 +0000 Subject: [PATCH 21/55] Add table create command in `resetctdb.sh` --- scripts/resetctdb.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/resetctdb.sh b/scripts/resetctdb.sh index 438bbd8695..dfff56b888 100755 --- a/scripts/resetctdb.sh +++ b/scripts/resetctdb.sh @@ -83,6 +83,8 @@ main() { die "Error: Failed to create user '${MYSQL_USER}@${MYSQL_USER_HOST}'." mysql "${FLAGS[@]}" -e "GRANT ALL ON ${MYSQL_DATABASE}.* TO ${MYSQL_USER}@'${MYSQL_USER_HOST}'" || \ die "Error: Failed to grant '${MYSQL_USER}' user all privileges on '${MYSQL_DATABASE}'." + mysql "${FLAGS[@]}" -D ${MYSQL_DATABASE} < ${CT_GO_PATH}/trillian/ctfe/storage/mysql/schema.sql || \ + die "Error: Failed to create tables in '${MYSQL_DATABASE}' database." echo "Reset Complete" fi } From c49d07fec42d95b71eede807bd2bca497ccb7579 Mon Sep 17 00:00:00 2001 From: Roger Ng Date: Fri, 17 May 2024 10:32:57 +0000 Subject: [PATCH 22/55] Add `extra_data_issuance_chain_storage_backends` config to integration test --- trillian/integration/ct_integration_test.cfg | 4 ++++ trillian/integration/ct_lifecycle_test.cfg | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/trillian/integration/ct_integration_test.cfg b/trillian/integration/ct_integration_test.cfg index dcdbd206a9..7e1ed6e395 100644 --- a/trillian/integration/ct_integration_test.cfg +++ b/trillian/integration/ct_integration_test.cfg @@ -13,6 +13,7 @@ config { } max_merge_delay_sec: 86400 expected_merge_delay_sec: 120 # Note: 120s is unrealistically fast for a real log, but OK for testing. + extra_data_issuance_chain_storage_backends: ISSUANCE_CHAIN_STORAGE_BACKEND_TRILLIAN_GRPC } config { log_id: @TREE_ID@ @@ -28,6 +29,7 @@ config { } max_merge_delay_sec: 86400 expected_merge_delay_sec: 120 + extra_data_issuance_chain_storage_backends: ISSUANCE_CHAIN_STORAGE_BACKEND_TRILLIAN_GRPC } config { log_id: @TREE_ID@ @@ -43,4 +45,6 @@ config { } max_merge_delay_sec: 86400 expected_merge_delay_sec: 120 + ctfe_storage_connection_string: "test:zaphod@tcp(localhost:3306)/test" + extra_data_issuance_chain_storage_backends: ISSUANCE_CHAIN_STORAGE_BACKEND_CTFE } diff --git a/trillian/integration/ct_lifecycle_test.cfg b/trillian/integration/ct_lifecycle_test.cfg index c0d704c7d1..8d0d7e2a60 100644 --- a/trillian/integration/ct_lifecycle_test.cfg +++ b/trillian/integration/ct_lifecycle_test.cfg @@ -12,6 +12,7 @@ config { } max_merge_delay_sec: 86400 expected_merge_delay_sec: 120 # Note: 120s is unrealistically fast for a real log, but OK for testing. + extra_data_issuance_chain_storage_backends: ISSUANCE_CHAIN_STORAGE_BACKEND_TRILLIAN_GRPC } config { log_id: @TREE_ID@ @@ -27,6 +28,7 @@ config { } max_merge_delay_sec: 86400 expected_merge_delay_sec: 120 + extra_data_issuance_chain_storage_backends: ISSUANCE_CHAIN_STORAGE_BACKEND_TRILLIAN_GRPC } config { log_id: @TREE_ID@ @@ -42,4 +44,6 @@ config { } max_merge_delay_sec: 86400 expected_merge_delay_sec: 120 + ctfe_storage_connection_string: "test:zaphod@tcp(localhost:3306)/test" + extra_data_issuance_chain_storage_backends: ISSUANCE_CHAIN_STORAGE_BACKEND_CTFE } From fabc88da75a75f7c85b353501a39f9dfee7b957b Mon Sep 17 00:00:00 2001 From: Roger Ng Date: Fri, 17 May 2024 10:49:47 +0000 Subject: [PATCH 23/55] Fix incorrect config name to `extra_data_issuance_chain_storage_backend` --- trillian/examples/deployment/docker/ctfe/ct_server.cfg | 2 +- trillian/integration/ct_integration_test.cfg | 6 +++--- trillian/integration/ct_lifecycle_test.cfg | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/trillian/examples/deployment/docker/ctfe/ct_server.cfg b/trillian/examples/deployment/docker/ctfe/ct_server.cfg index f380c5ffc7..ea965c91cb 100644 --- a/trillian/examples/deployment/docker/ctfe/ct_server.cfg +++ b/trillian/examples/deployment/docker/ctfe/ct_server.cfg @@ -13,5 +13,5 @@ config { max_merge_delay_sec: 86400 expected_merge_delay_sec: 120 ctfe_storage_connection_string: "test:zaphod@tcp(localhost:3306)/test" - extra_data_issuance_chain_storage_backends: ISSUANCE_CHAIN_STORAGE_BACKEND_CTFE + extra_data_issuance_chain_storage_backend: ISSUANCE_CHAIN_STORAGE_BACKEND_CTFE } diff --git a/trillian/integration/ct_integration_test.cfg b/trillian/integration/ct_integration_test.cfg index 7e1ed6e395..c98f9b5349 100644 --- a/trillian/integration/ct_integration_test.cfg +++ b/trillian/integration/ct_integration_test.cfg @@ -13,7 +13,7 @@ config { } max_merge_delay_sec: 86400 expected_merge_delay_sec: 120 # Note: 120s is unrealistically fast for a real log, but OK for testing. - extra_data_issuance_chain_storage_backends: ISSUANCE_CHAIN_STORAGE_BACKEND_TRILLIAN_GRPC + extra_data_issuance_chain_storage_backend: ISSUANCE_CHAIN_STORAGE_BACKEND_TRILLIAN_GRPC } config { log_id: @TREE_ID@ @@ -29,7 +29,7 @@ config { } max_merge_delay_sec: 86400 expected_merge_delay_sec: 120 - extra_data_issuance_chain_storage_backends: ISSUANCE_CHAIN_STORAGE_BACKEND_TRILLIAN_GRPC + extra_data_issuance_chain_storage_backend: ISSUANCE_CHAIN_STORAGE_BACKEND_TRILLIAN_GRPC } config { log_id: @TREE_ID@ @@ -46,5 +46,5 @@ config { max_merge_delay_sec: 86400 expected_merge_delay_sec: 120 ctfe_storage_connection_string: "test:zaphod@tcp(localhost:3306)/test" - extra_data_issuance_chain_storage_backends: ISSUANCE_CHAIN_STORAGE_BACKEND_CTFE + extra_data_issuance_chain_storage_backend: ISSUANCE_CHAIN_STORAGE_BACKEND_CTFE } diff --git a/trillian/integration/ct_lifecycle_test.cfg b/trillian/integration/ct_lifecycle_test.cfg index 8d0d7e2a60..f4c3cec42b 100644 --- a/trillian/integration/ct_lifecycle_test.cfg +++ b/trillian/integration/ct_lifecycle_test.cfg @@ -12,7 +12,7 @@ config { } max_merge_delay_sec: 86400 expected_merge_delay_sec: 120 # Note: 120s is unrealistically fast for a real log, but OK for testing. - extra_data_issuance_chain_storage_backends: ISSUANCE_CHAIN_STORAGE_BACKEND_TRILLIAN_GRPC + extra_data_issuance_chain_storage_backend: ISSUANCE_CHAIN_STORAGE_BACKEND_TRILLIAN_GRPC } config { log_id: @TREE_ID@ @@ -28,7 +28,7 @@ config { } max_merge_delay_sec: 86400 expected_merge_delay_sec: 120 - extra_data_issuance_chain_storage_backends: ISSUANCE_CHAIN_STORAGE_BACKEND_TRILLIAN_GRPC + extra_data_issuance_chain_storage_backend: ISSUANCE_CHAIN_STORAGE_BACKEND_TRILLIAN_GRPC } config { log_id: @TREE_ID@ @@ -45,5 +45,5 @@ config { max_merge_delay_sec: 86400 expected_merge_delay_sec: 120 ctfe_storage_connection_string: "test:zaphod@tcp(localhost:3306)/test" - extra_data_issuance_chain_storage_backends: ISSUANCE_CHAIN_STORAGE_BACKEND_CTFE + extra_data_issuance_chain_storage_backend: ISSUANCE_CHAIN_STORAGE_BACKEND_CTFE } From e17250bc5b46b4decaf86a6ccb3f06c568ad50b6 Mon Sep 17 00:00:00 2001 From: Roger Ng Date: Fri, 17 May 2024 11:11:51 +0000 Subject: [PATCH 24/55] Add missing `mysql://` in `ctfe_storage_connection_string` config --- trillian/examples/deployment/docker/ctfe/ct_server.cfg | 2 +- trillian/integration/ct_integration_test.cfg | 2 +- trillian/integration/ct_lifecycle_test.cfg | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/trillian/examples/deployment/docker/ctfe/ct_server.cfg b/trillian/examples/deployment/docker/ctfe/ct_server.cfg index ea965c91cb..213b6c565d 100644 --- a/trillian/examples/deployment/docker/ctfe/ct_server.cfg +++ b/trillian/examples/deployment/docker/ctfe/ct_server.cfg @@ -12,6 +12,6 @@ config { } max_merge_delay_sec: 86400 expected_merge_delay_sec: 120 - ctfe_storage_connection_string: "test:zaphod@tcp(localhost:3306)/test" + ctfe_storage_connection_string: "mysql://test:zaphod@tcp(localhost:3306)/test" extra_data_issuance_chain_storage_backend: ISSUANCE_CHAIN_STORAGE_BACKEND_CTFE } diff --git a/trillian/integration/ct_integration_test.cfg b/trillian/integration/ct_integration_test.cfg index c98f9b5349..9999571cf9 100644 --- a/trillian/integration/ct_integration_test.cfg +++ b/trillian/integration/ct_integration_test.cfg @@ -45,6 +45,6 @@ config { } max_merge_delay_sec: 86400 expected_merge_delay_sec: 120 - ctfe_storage_connection_string: "test:zaphod@tcp(localhost:3306)/test" + ctfe_storage_connection_string: "mysql://test:zaphod@tcp(localhost:3306)/test" extra_data_issuance_chain_storage_backend: ISSUANCE_CHAIN_STORAGE_BACKEND_CTFE } diff --git a/trillian/integration/ct_lifecycle_test.cfg b/trillian/integration/ct_lifecycle_test.cfg index f4c3cec42b..ec8cf58a77 100644 --- a/trillian/integration/ct_lifecycle_test.cfg +++ b/trillian/integration/ct_lifecycle_test.cfg @@ -44,6 +44,6 @@ config { } max_merge_delay_sec: 86400 expected_merge_delay_sec: 120 - ctfe_storage_connection_string: "test:zaphod@tcp(localhost:3306)/test" + ctfe_storage_connection_string: "mysql://test:zaphod@tcp(localhost:3306)/test" extra_data_issuance_chain_storage_backend: ISSUANCE_CHAIN_STORAGE_BACKEND_CTFE } From abec73a85eac12a11016f3b2387e7cdf878043b8 Mon Sep 17 00:00:00 2001 From: Roger Ng Date: Fri, 17 May 2024 14:05:21 +0000 Subject: [PATCH 25/55] Update the hostname to `db` in `ctfe_storage_connection_string` config --- trillian/examples/deployment/docker/ctfe/ct_server.cfg | 2 +- trillian/integration/ct_integration_test.cfg | 2 +- trillian/integration/ct_lifecycle_test.cfg | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/trillian/examples/deployment/docker/ctfe/ct_server.cfg b/trillian/examples/deployment/docker/ctfe/ct_server.cfg index 213b6c565d..172fcb9c26 100644 --- a/trillian/examples/deployment/docker/ctfe/ct_server.cfg +++ b/trillian/examples/deployment/docker/ctfe/ct_server.cfg @@ -12,6 +12,6 @@ config { } max_merge_delay_sec: 86400 expected_merge_delay_sec: 120 - ctfe_storage_connection_string: "mysql://test:zaphod@tcp(localhost:3306)/test" + ctfe_storage_connection_string: "mysql://test:zaphod@tcp(db:3306)/test" extra_data_issuance_chain_storage_backend: ISSUANCE_CHAIN_STORAGE_BACKEND_CTFE } diff --git a/trillian/integration/ct_integration_test.cfg b/trillian/integration/ct_integration_test.cfg index 9999571cf9..a05d69d802 100644 --- a/trillian/integration/ct_integration_test.cfg +++ b/trillian/integration/ct_integration_test.cfg @@ -45,6 +45,6 @@ config { } max_merge_delay_sec: 86400 expected_merge_delay_sec: 120 - ctfe_storage_connection_string: "mysql://test:zaphod@tcp(localhost:3306)/test" + ctfe_storage_connection_string: "mysql://test:zaphod@tcp(db:3306)/test" extra_data_issuance_chain_storage_backend: ISSUANCE_CHAIN_STORAGE_BACKEND_CTFE } diff --git a/trillian/integration/ct_lifecycle_test.cfg b/trillian/integration/ct_lifecycle_test.cfg index ec8cf58a77..226f8d0af5 100644 --- a/trillian/integration/ct_lifecycle_test.cfg +++ b/trillian/integration/ct_lifecycle_test.cfg @@ -44,6 +44,6 @@ config { } max_merge_delay_sec: 86400 expected_merge_delay_sec: 120 - ctfe_storage_connection_string: "mysql://test:zaphod@tcp(localhost:3306)/test" + ctfe_storage_connection_string: "mysql://test:zaphod@tcp(db:3306)/test" extra_data_issuance_chain_storage_backend: ISSUANCE_CHAIN_STORAGE_BACKEND_CTFE } From c1c49c512935c4e025af4cb8c6db5abdd798616e Mon Sep 17 00:00:00 2001 From: Roger Ng Date: Fri, 17 May 2024 14:35:48 +0000 Subject: [PATCH 26/55] Update the hostname to `mysql` in `ctfe_storage_connection_string` config --- trillian/examples/deployment/docker/ctfe/ct_server.cfg | 2 +- trillian/integration/ct_integration_test.cfg | 2 +- trillian/integration/ct_lifecycle_test.cfg | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/trillian/examples/deployment/docker/ctfe/ct_server.cfg b/trillian/examples/deployment/docker/ctfe/ct_server.cfg index 172fcb9c26..213b6c565d 100644 --- a/trillian/examples/deployment/docker/ctfe/ct_server.cfg +++ b/trillian/examples/deployment/docker/ctfe/ct_server.cfg @@ -12,6 +12,6 @@ config { } max_merge_delay_sec: 86400 expected_merge_delay_sec: 120 - ctfe_storage_connection_string: "mysql://test:zaphod@tcp(db:3306)/test" + ctfe_storage_connection_string: "mysql://test:zaphod@tcp(localhost:3306)/test" extra_data_issuance_chain_storage_backend: ISSUANCE_CHAIN_STORAGE_BACKEND_CTFE } diff --git a/trillian/integration/ct_integration_test.cfg b/trillian/integration/ct_integration_test.cfg index a05d69d802..c569b9cc80 100644 --- a/trillian/integration/ct_integration_test.cfg +++ b/trillian/integration/ct_integration_test.cfg @@ -45,6 +45,6 @@ config { } max_merge_delay_sec: 86400 expected_merge_delay_sec: 120 - ctfe_storage_connection_string: "mysql://test:zaphod@tcp(db:3306)/test" + ctfe_storage_connection_string: "mysql://test:zaphod@tcp(mysql:3306)/test" extra_data_issuance_chain_storage_backend: ISSUANCE_CHAIN_STORAGE_BACKEND_CTFE } diff --git a/trillian/integration/ct_lifecycle_test.cfg b/trillian/integration/ct_lifecycle_test.cfg index 226f8d0af5..9ac397d3b4 100644 --- a/trillian/integration/ct_lifecycle_test.cfg +++ b/trillian/integration/ct_lifecycle_test.cfg @@ -44,6 +44,6 @@ config { } max_merge_delay_sec: 86400 expected_merge_delay_sec: 120 - ctfe_storage_connection_string: "mysql://test:zaphod@tcp(db:3306)/test" + ctfe_storage_connection_string: "mysql://test:zaphod@tcp(mysql:3306)/test" extra_data_issuance_chain_storage_backend: ISSUANCE_CHAIN_STORAGE_BACKEND_CTFE } From c02e86585d14d8424bff0dd28f53f82f3284f298 Mon Sep 17 00:00:00 2001 From: Roger Ng Date: Fri, 17 May 2024 16:41:30 +0000 Subject: [PATCH 27/55] Update the tested MySQL version to 8.4 in CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 72d68c13a3..f4f24b6394 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,7 +19,7 @@ An optional LRU cache can be enabled by providing the following flags. - `cache_size` - `cache_ttl` -This change is tested in Cloud Build tests using the `mysql:8.2` Docker image. +This change is tested in Cloud Build tests using the `mysql:8.4` Docker image as of the time of writing. ### Submission proxy: Root compatibility checking From 670d024d01f221880e941b17bfd05eaa825248c8 Mon Sep 17 00:00:00 2001 From: Roger Ng Date: Fri, 17 May 2024 16:57:22 +0000 Subject: [PATCH 28/55] Import MySQL schema in integration test flow --- scripts/resetctdb.sh | 38 +++++++++++++++++++--------- trillian/integration/ct_functions.sh | 3 +++ 2 files changed, 29 insertions(+), 12 deletions(-) diff --git a/scripts/resetctdb.sh b/scripts/resetctdb.sh index dfff56b888..deb432d910 100755 --- a/scripts/resetctdb.sh +++ b/scripts/resetctdb.sh @@ -41,10 +41,13 @@ collect_vars() { # handle flags FORCE=false VERBOSE=false + ONLY_IMPORT_SCHEMA=false + while [[ $# -gt 0 ]]; do case "$1" in --force) FORCE=true ;; --verbose) VERBOSE=true ;; + --only_import_schema) ONLY_IMPORT_SCHEMA=true ;; --help) usage; exit ;; *) FLAGS+=("$1") esac @@ -74,19 +77,30 @@ main() { if [ -z ${REPLY+x} ] || [[ $REPLY =~ ^[Yy]$ ]] then - echo "Resetting DB..." - mysql "${FLAGS[@]}" -e "DROP DATABASE IF EXISTS ${MYSQL_DATABASE};" || \ - die "Error: Failed to drop database '${MYSQL_DATABASE}'." - mysql "${FLAGS[@]}" -e "CREATE DATABASE ${MYSQL_DATABASE};" || \ - die "Error: Failed to create database '${MYSQL_DATABASE}'." - mysql "${FLAGS[@]}" -e "CREATE USER IF NOT EXISTS ${MYSQL_USER}@'${MYSQL_USER_HOST}' IDENTIFIED BY '${MYSQL_PASSWORD}';" || \ - die "Error: Failed to create user '${MYSQL_USER}@${MYSQL_USER_HOST}'." - mysql "${FLAGS[@]}" -e "GRANT ALL ON ${MYSQL_DATABASE}.* TO ${MYSQL_USER}@'${MYSQL_USER_HOST}'" || \ - die "Error: Failed to grant '${MYSQL_USER}' user all privileges on '${MYSQL_DATABASE}'." - mysql "${FLAGS[@]}" -D ${MYSQL_DATABASE} < ${CT_GO_PATH}/trillian/ctfe/storage/mysql/schema.sql || \ - die "Error: Failed to create tables in '${MYSQL_DATABASE}' database." - echo "Reset Complete" + if [[ ${ONLY_IMPORT_SCHEMA} = 'true' ]] + echo "Importing schema only..." + import_schema + echo "Imported schema" + exit 0 + fi + + echo "Resetting DB..." + mysql "${FLAGS[@]}" -e "DROP DATABASE IF EXISTS ${MYSQL_DATABASE};" || \ + die "Error: Failed to drop database '${MYSQL_DATABASE}'." + mysql "${FLAGS[@]}" -e "CREATE DATABASE ${MYSQL_DATABASE};" || \ + die "Error: Failed to create database '${MYSQL_DATABASE}'." + mysql "${FLAGS[@]}" -e "CREATE USER IF NOT EXISTS ${MYSQL_USER}@'${MYSQL_USER_HOST}' IDENTIFIED BY '${MYSQL_PASSWORD}';" || \ + die "Error: Failed to create user '${MYSQL_USER}@${MYSQL_USER_HOST}'." + mysql "${FLAGS[@]}" -e "GRANT ALL ON ${MYSQL_DATABASE}.* TO ${MYSQL_USER}@'${MYSQL_USER_HOST}'" || \ + die "Error: Failed to grant '${MYSQL_USER}' user all privileges on '${MYSQL_DATABASE}'." + import_schema + echo "Reset Complete" fi } +import_schema() { + mysql "${FLAGS[@]}" -D ${MYSQL_DATABASE} < ${CT_GO_PATH}/trillian/ctfe/storage/mysql/schema.sql || \ + die "Error: Failed to import schema in '${MYSQL_DATABASE}' database." +} + main "$@" diff --git a/trillian/integration/ct_functions.sh b/trillian/integration/ct_functions.sh index 1e9265c792..4a6c0eb67a 100644 --- a/trillian/integration/ct_functions.sh +++ b/trillian/integration/ct_functions.sh @@ -68,6 +68,9 @@ ct_prep_test() { PROMETHEUS_PID=$! fi fi + + echo "Importing schema via resetctdb.sh" + ${CT_GO_PATH}/scripts/resetctdb.sh --only_import_schema } # ct_provision generates a CT configuration file and provisions the trees for it. From 6295884eee823b96b5fd3c86d231bf869b819c58 Mon Sep 17 00:00:00 2001 From: Roger Ng Date: Fri, 17 May 2024 17:02:54 +0000 Subject: [PATCH 29/55] Fix missing `then` in `resetctdb.sh` --- scripts/resetctdb.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/resetctdb.sh b/scripts/resetctdb.sh index deb432d910..c7a234054b 100755 --- a/scripts/resetctdb.sh +++ b/scripts/resetctdb.sh @@ -78,6 +78,7 @@ main() { if [ -z ${REPLY+x} ] || [[ $REPLY =~ ^[Yy]$ ]] then if [[ ${ONLY_IMPORT_SCHEMA} = 'true' ]] + then echo "Importing schema only..." import_schema echo "Imported schema" From 7e9fceb656076bbbac100a809d9f7333dac90056 Mon Sep 17 00:00:00 2001 From: Roger Ng Date: Fri, 17 May 2024 19:10:45 +0000 Subject: [PATCH 30/55] Update integration test to use cttest as the CTFE database --- scripts/resetctdb.sh | 18 ++---------------- trillian/integration/ct_functions.sh | 4 ++-- trillian/integration/ct_integration_test.cfg | 2 +- trillian/integration/ct_lifecycle_test.cfg | 2 +- 4 files changed, 6 insertions(+), 20 deletions(-) diff --git a/scripts/resetctdb.sh b/scripts/resetctdb.sh index c7a234054b..1b1657816e 100755 --- a/scripts/resetctdb.sh +++ b/scripts/resetctdb.sh @@ -41,13 +41,11 @@ collect_vars() { # handle flags FORCE=false VERBOSE=false - ONLY_IMPORT_SCHEMA=false while [[ $# -gt 0 ]]; do case "$1" in --force) FORCE=true ;; --verbose) VERBOSE=true ;; - --only_import_schema) ONLY_IMPORT_SCHEMA=true ;; --help) usage; exit ;; *) FLAGS+=("$1") esac @@ -77,14 +75,6 @@ main() { if [ -z ${REPLY+x} ] || [[ $REPLY =~ ^[Yy]$ ]] then - if [[ ${ONLY_IMPORT_SCHEMA} = 'true' ]] - then - echo "Importing schema only..." - import_schema - echo "Imported schema" - exit 0 - fi - echo "Resetting DB..." mysql "${FLAGS[@]}" -e "DROP DATABASE IF EXISTS ${MYSQL_DATABASE};" || \ die "Error: Failed to drop database '${MYSQL_DATABASE}'." @@ -94,14 +84,10 @@ main() { die "Error: Failed to create user '${MYSQL_USER}@${MYSQL_USER_HOST}'." mysql "${FLAGS[@]}" -e "GRANT ALL ON ${MYSQL_DATABASE}.* TO ${MYSQL_USER}@'${MYSQL_USER_HOST}'" || \ die "Error: Failed to grant '${MYSQL_USER}' user all privileges on '${MYSQL_DATABASE}'." - import_schema + mysql "${FLAGS[@]}" -D ${MYSQL_DATABASE} < ${CT_GO_PATH}/trillian/ctfe/storage/mysql/schema.sql || \ + die "Error: Failed to import schema in '${MYSQL_DATABASE}' database." echo "Reset Complete" fi } -import_schema() { - mysql "${FLAGS[@]}" -D ${MYSQL_DATABASE} < ${CT_GO_PATH}/trillian/ctfe/storage/mysql/schema.sql || \ - die "Error: Failed to import schema in '${MYSQL_DATABASE}' database." -} - main "$@" diff --git a/trillian/integration/ct_functions.sh b/trillian/integration/ct_functions.sh index 4a6c0eb67a..90c9f2b0f5 100644 --- a/trillian/integration/ct_functions.sh +++ b/trillian/integration/ct_functions.sh @@ -69,8 +69,8 @@ ct_prep_test() { fi fi - echo "Importing schema via resetctdb.sh" - ${CT_GO_PATH}/scripts/resetctdb.sh --only_import_schema + # Wipe the CT test database + yes | bash "${CT_GO_PATH}/scripts/resetctdb.sh" } # ct_provision generates a CT configuration file and provisions the trees for it. diff --git a/trillian/integration/ct_integration_test.cfg b/trillian/integration/ct_integration_test.cfg index c569b9cc80..bd794553a1 100644 --- a/trillian/integration/ct_integration_test.cfg +++ b/trillian/integration/ct_integration_test.cfg @@ -45,6 +45,6 @@ config { } max_merge_delay_sec: 86400 expected_merge_delay_sec: 120 - ctfe_storage_connection_string: "mysql://test:zaphod@tcp(mysql:3306)/test" + ctfe_storage_connection_string: "mysql://cttest:beeblebrox@tcp(mysql:3306)/cttest" extra_data_issuance_chain_storage_backend: ISSUANCE_CHAIN_STORAGE_BACKEND_CTFE } diff --git a/trillian/integration/ct_lifecycle_test.cfg b/trillian/integration/ct_lifecycle_test.cfg index 9ac397d3b4..f1406fe3c9 100644 --- a/trillian/integration/ct_lifecycle_test.cfg +++ b/trillian/integration/ct_lifecycle_test.cfg @@ -44,6 +44,6 @@ config { } max_merge_delay_sec: 86400 expected_merge_delay_sec: 120 - ctfe_storage_connection_string: "mysql://test:zaphod@tcp(mysql:3306)/test" + ctfe_storage_connection_string: "mysql://cttest:beeblebrox@tcp(mysql:3306)/cttest" extra_data_issuance_chain_storage_backend: ISSUANCE_CHAIN_STORAGE_BACKEND_CTFE } From e6e870031bdce0d9e6ce9cff8a9e7948abac3994 Mon Sep 17 00:00:00 2001 From: Roger Ng Date: Fri, 17 May 2024 19:20:10 +0000 Subject: [PATCH 31/55] Reset the CT test database before launching CT personalities in integration test flow --- trillian/integration/ct_functions.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/trillian/integration/ct_functions.sh b/trillian/integration/ct_functions.sh index 90c9f2b0f5..9a61096ffe 100644 --- a/trillian/integration/ct_functions.sh +++ b/trillian/integration/ct_functions.sh @@ -33,6 +33,9 @@ ct_prep_test() { echo "Provisioning logs for CT" ct_provision "${RPC_SERVER_1}" + # Wipe the CT test database + yes | bash "${CT_GO_PATH}/scripts/resetctdb.sh" --verbose + echo "Launching CT personalities" for ((i=0; i < http_server_count; i++)); do local port=$(pick_unused_port) @@ -68,9 +71,6 @@ ct_prep_test() { PROMETHEUS_PID=$! fi fi - - # Wipe the CT test database - yes | bash "${CT_GO_PATH}/scripts/resetctdb.sh" } # ct_provision generates a CT configuration file and provisions the trees for it. From 3e68a9144ff8f430d726702a539535ea967943bd Mon Sep 17 00:00:00 2001 From: Roger Ng Date: Fri, 17 May 2024 19:32:15 +0000 Subject: [PATCH 32/55] Add `mariadb-client` to ct_testbase Dockerfile --- integration/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration/Dockerfile b/integration/Dockerfile index 24f26455d0..42c4152d46 100644 --- a/integration/Dockerfile +++ b/integration/Dockerfile @@ -7,7 +7,7 @@ WORKDIR /testbase ARG GOFLAGS="" ENV GOFLAGS=$GOFLAGS -RUN apt-get update && apt-get -y install build-essential curl docker-compose lsof netcat-openbsd unzip wget xxd +RUN apt-get update && apt-get -y install build-essential curl docker-compose lsof mariadb-client netcat-openbsd unzip wget xxd RUN cd /usr/bin && curl -L -O https://github.com/jqlang/jq/releases/download/jq-1.7/jq-linux64 && mv jq-linux64 /usr/bin/jq && chmod +x /usr/bin/jq RUN curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.55.1 From 4c93e3f5ce81d10e44c09a1e83c33526ee340415 Mon Sep 17 00:00:00 2001 From: Roger Ng Date: Fri, 17 May 2024 19:32:31 +0000 Subject: [PATCH 33/55] Update comment --- trillian/integration/ct_functions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trillian/integration/ct_functions.sh b/trillian/integration/ct_functions.sh index 9a61096ffe..6eafcba6d7 100644 --- a/trillian/integration/ct_functions.sh +++ b/trillian/integration/ct_functions.sh @@ -33,7 +33,7 @@ ct_prep_test() { echo "Provisioning logs for CT" ct_provision "${RPC_SERVER_1}" - # Wipe the CT test database + # Reset the CT test database yes | bash "${CT_GO_PATH}/scripts/resetctdb.sh" --verbose echo "Launching CT personalities" From d08c7f4df43181aad29efc64c504893469e71ce2 Mon Sep 17 00:00:00 2001 From: Roger Ng Date: Fri, 17 May 2024 19:51:46 +0000 Subject: [PATCH 34/55] Export `MYSQL_HOST` in ct_functions.sh --- trillian/integration/ct_functions.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/trillian/integration/ct_functions.sh b/trillian/integration/ct_functions.sh index 6eafcba6d7..a66e7e11aa 100644 --- a/trillian/integration/ct_functions.sh +++ b/trillian/integration/ct_functions.sh @@ -34,6 +34,7 @@ ct_prep_test() { ct_provision "${RPC_SERVER_1}" # Reset the CT test database + export MYSQL_HOST="mysql" yes | bash "${CT_GO_PATH}/scripts/resetctdb.sh" --verbose echo "Launching CT personalities" From 1723da3f58f445ac0723b2ee70b37f500e5b5d6e Mon Sep 17 00:00:00 2001 From: Roger Ng Date: Fri, 17 May 2024 20:00:11 +0000 Subject: [PATCH 35/55] Export `MYSQL_ROOT_PASSWORD` in ct_functions.sh --- trillian/integration/ct_functions.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/trillian/integration/ct_functions.sh b/trillian/integration/ct_functions.sh index a66e7e11aa..241da432df 100644 --- a/trillian/integration/ct_functions.sh +++ b/trillian/integration/ct_functions.sh @@ -35,6 +35,7 @@ ct_prep_test() { # Reset the CT test database export MYSQL_HOST="mysql" + export MYSQL_ROOT_PASSWORD="bananas" yes | bash "${CT_GO_PATH}/scripts/resetctdb.sh" --verbose echo "Launching CT personalities" From 783d6979f19852f7d7123d35508d635e29847035 Mon Sep 17 00:00:00 2001 From: Roger Ng Date: Fri, 17 May 2024 20:35:40 +0000 Subject: [PATCH 36/55] Export `MYSQL_USER_HOST` in ct_functions.sh --- trillian/integration/ct_functions.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/trillian/integration/ct_functions.sh b/trillian/integration/ct_functions.sh index 241da432df..cef5f31e33 100644 --- a/trillian/integration/ct_functions.sh +++ b/trillian/integration/ct_functions.sh @@ -35,7 +35,8 @@ ct_prep_test() { # Reset the CT test database export MYSQL_HOST="mysql" - export MYSQL_ROOT_PASSWORD="bananas" + export MYSQL_ROOT_PASSWORD="zaphod" + export MYSQL_USER_HOST="%" yes | bash "${CT_GO_PATH}/scripts/resetctdb.sh" --verbose echo "Launching CT personalities" From 77093a07b70b0df6e7e1821702b9d6ff212a7c7a Mon Sep 17 00:00:00 2001 From: Roger Ng Date: Fri, 17 May 2024 20:55:00 +0000 Subject: [PATCH 37/55] Add all combination of `extra_data_issuance_chain_storage_backend` config in integration test flow --- trillian/integration/ct_integration_test.cfg | 1 - trillian/integration/ct_lifecycle_test.cfg | 1 - 2 files changed, 2 deletions(-) diff --git a/trillian/integration/ct_integration_test.cfg b/trillian/integration/ct_integration_test.cfg index bd794553a1..f94d9474fb 100644 --- a/trillian/integration/ct_integration_test.cfg +++ b/trillian/integration/ct_integration_test.cfg @@ -13,7 +13,6 @@ config { } max_merge_delay_sec: 86400 expected_merge_delay_sec: 120 # Note: 120s is unrealistically fast for a real log, but OK for testing. - extra_data_issuance_chain_storage_backend: ISSUANCE_CHAIN_STORAGE_BACKEND_TRILLIAN_GRPC } config { log_id: @TREE_ID@ diff --git a/trillian/integration/ct_lifecycle_test.cfg b/trillian/integration/ct_lifecycle_test.cfg index f1406fe3c9..e845b2c489 100644 --- a/trillian/integration/ct_lifecycle_test.cfg +++ b/trillian/integration/ct_lifecycle_test.cfg @@ -12,7 +12,6 @@ config { } max_merge_delay_sec: 86400 expected_merge_delay_sec: 120 # Note: 120s is unrealistically fast for a real log, but OK for testing. - extra_data_issuance_chain_storage_backend: ISSUANCE_CHAIN_STORAGE_BACKEND_TRILLIAN_GRPC } config { log_id: @TREE_ID@ From b8b65ac8c195277366f63267bbc445ccb110fa96 Mon Sep 17 00:00:00 2001 From: Roger Ng Date: Sat, 18 May 2024 20:12:05 +0000 Subject: [PATCH 38/55] Unexport `MYSQL_USER_HOST` for debugging --- trillian/integration/ct_functions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trillian/integration/ct_functions.sh b/trillian/integration/ct_functions.sh index cef5f31e33..377beb04cc 100644 --- a/trillian/integration/ct_functions.sh +++ b/trillian/integration/ct_functions.sh @@ -36,7 +36,7 @@ ct_prep_test() { # Reset the CT test database export MYSQL_HOST="mysql" export MYSQL_ROOT_PASSWORD="zaphod" - export MYSQL_USER_HOST="%" + # export MYSQL_USER_HOST="%" yes | bash "${CT_GO_PATH}/scripts/resetctdb.sh" --verbose echo "Launching CT personalities" From 75ea5f5b92160100a8b811e29ea8917e66d66f8f Mon Sep 17 00:00:00 2001 From: Roger Ng Date: Sat, 18 May 2024 20:17:22 +0000 Subject: [PATCH 39/55] Revert: Unexport `MYSQL_USER_HOST` for debugging --- trillian/integration/ct_functions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trillian/integration/ct_functions.sh b/trillian/integration/ct_functions.sh index 377beb04cc..cef5f31e33 100644 --- a/trillian/integration/ct_functions.sh +++ b/trillian/integration/ct_functions.sh @@ -36,7 +36,7 @@ ct_prep_test() { # Reset the CT test database export MYSQL_HOST="mysql" export MYSQL_ROOT_PASSWORD="zaphod" - # export MYSQL_USER_HOST="%" + export MYSQL_USER_HOST="%" yes | bash "${CT_GO_PATH}/scripts/resetctdb.sh" --verbose echo "Launching CT personalities" From f018e5da352e2c0b29fe941778827d447f39d93b Mon Sep 17 00:00:00 2001 From: Roger Ng Date: Sat, 18 May 2024 20:26:24 +0000 Subject: [PATCH 40/55] Add debug log to `resectdb.sh` --- scripts/resetctdb.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/scripts/resetctdb.sh b/scripts/resetctdb.sh index 1b1657816e..6a93670d69 100755 --- a/scripts/resetctdb.sh +++ b/scripts/resetctdb.sh @@ -82,10 +82,17 @@ main() { die "Error: Failed to create database '${MYSQL_DATABASE}'." mysql "${FLAGS[@]}" -e "CREATE USER IF NOT EXISTS ${MYSQL_USER}@'${MYSQL_USER_HOST}' IDENTIFIED BY '${MYSQL_PASSWORD}';" || \ die "Error: Failed to create user '${MYSQL_USER}@${MYSQL_USER_HOST}'." - mysql "${FLAGS[@]}" -e "GRANT ALL ON ${MYSQL_DATABASE}.* TO ${MYSQL_USER}@'${MYSQL_USER_HOST}'" || \ + mysql "${FLAGS[@]}" -e "GRANT ALL ON ${MYSQL_DATABASE}.* TO ${MYSQL_USER}@'${MYSQL_USER_HOST}';" || \ die "Error: Failed to grant '${MYSQL_USER}' user all privileges on '${MYSQL_DATABASE}'." + mysql "${FLAGS[@]}" -e "FLUSH PRIVILEGES;" || \ + die "Error: Failed to flush privileges." mysql "${FLAGS[@]}" -D ${MYSQL_DATABASE} < ${CT_GO_PATH}/trillian/ctfe/storage/mysql/schema.sql || \ die "Error: Failed to import schema in '${MYSQL_DATABASE}' database." + + # Debug log + mysql "${FLAGS[@]}" -e "SHOW DATABASES;" + mysql "${FLAGS[@]}" -e "SHOW GRANTS FOR ${MYSQL_USER}@'${MYSQL_USER_HOST}';" + echo "Reset Complete" fi } From fbf800f274140163511ea9799e50118b31a1c42b Mon Sep 17 00:00:00 2001 From: Roger Ng Date: Sat, 18 May 2024 20:54:05 +0000 Subject: [PATCH 41/55] Add "SHOW TABLES" in debug log --- scripts/resetctdb.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/resetctdb.sh b/scripts/resetctdb.sh index 6a93670d69..73b87316bb 100755 --- a/scripts/resetctdb.sh +++ b/scripts/resetctdb.sh @@ -92,6 +92,7 @@ main() { # Debug log mysql "${FLAGS[@]}" -e "SHOW DATABASES;" mysql "${FLAGS[@]}" -e "SHOW GRANTS FOR ${MYSQL_USER}@'${MYSQL_USER_HOST}';" + mysql "${FLAGS[@]}" -D ${MYSQL_DATABASE} -e "SHOW TABLES;" echo "Reset Complete" fi From 0f4007e0c6787e6c22f47e05089de2a81491e3a6 Mon Sep 17 00:00:00 2001 From: Roger Ng Date: Sat, 18 May 2024 21:29:15 +0000 Subject: [PATCH 42/55] Add `USE ${MYSQL_DATABASE}; SHOW TABLES;` --- scripts/resetctdb.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/resetctdb.sh b/scripts/resetctdb.sh index 73b87316bb..dd95418d88 100755 --- a/scripts/resetctdb.sh +++ b/scripts/resetctdb.sh @@ -92,6 +92,7 @@ main() { # Debug log mysql "${FLAGS[@]}" -e "SHOW DATABASES;" mysql "${FLAGS[@]}" -e "SHOW GRANTS FOR ${MYSQL_USER}@'${MYSQL_USER_HOST}';" + mysql "${FLAGS[@]}" -e "USE ${MYSQL_DATABASE}; SHOW TABLES;" mysql "${FLAGS[@]}" -D ${MYSQL_DATABASE} -e "SHOW TABLES;" echo "Reset Complete" From 9bf4ce4e9432c1ae0e3812f5ef6e934f8f66ab71 Mon Sep 17 00:00:00 2001 From: Roger Ng Date: Sat, 18 May 2024 21:44:37 +0000 Subject: [PATCH 43/55] Move CT test database reset to GCB step 3 (ci-ready) --- cloudbuild.yaml | 7 +++++++ scripts/resetctdb.sh | 6 ------ trillian/integration/ct_functions.sh | 6 ------ 3 files changed, 7 insertions(+), 12 deletions(-) diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 6c856c11c1..9e9c30282a 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -85,6 +85,13 @@ steps: # Wait for trillian logserver to be up until nc -z deployment_trillian-log-server_1 8090; do echo .; sleep 5; done + + # Reset the CT test database + export CT_GO_PATH="$$(go list -f '{{.Dir}}' github.com/google/certificate-transparency-go)" + export MYSQL_HOST="mysql" + export MYSQL_ROOT_PASSWORD="zaphod" + export MYSQL_USER_HOST="%" + yes | bash "${CT_GO_PATH}/scripts/resetctdb.sh" --verbose waitFor: ['prepare'] # Run the presubmit tests diff --git a/scripts/resetctdb.sh b/scripts/resetctdb.sh index dd95418d88..488f2b7630 100755 --- a/scripts/resetctdb.sh +++ b/scripts/resetctdb.sh @@ -88,12 +88,6 @@ main() { die "Error: Failed to flush privileges." mysql "${FLAGS[@]}" -D ${MYSQL_DATABASE} < ${CT_GO_PATH}/trillian/ctfe/storage/mysql/schema.sql || \ die "Error: Failed to import schema in '${MYSQL_DATABASE}' database." - - # Debug log - mysql "${FLAGS[@]}" -e "SHOW DATABASES;" - mysql "${FLAGS[@]}" -e "SHOW GRANTS FOR ${MYSQL_USER}@'${MYSQL_USER_HOST}';" - mysql "${FLAGS[@]}" -e "USE ${MYSQL_DATABASE}; SHOW TABLES;" - mysql "${FLAGS[@]}" -D ${MYSQL_DATABASE} -e "SHOW TABLES;" echo "Reset Complete" fi diff --git a/trillian/integration/ct_functions.sh b/trillian/integration/ct_functions.sh index cef5f31e33..1e9265c792 100644 --- a/trillian/integration/ct_functions.sh +++ b/trillian/integration/ct_functions.sh @@ -33,12 +33,6 @@ ct_prep_test() { echo "Provisioning logs for CT" ct_provision "${RPC_SERVER_1}" - # Reset the CT test database - export MYSQL_HOST="mysql" - export MYSQL_ROOT_PASSWORD="zaphod" - export MYSQL_USER_HOST="%" - yes | bash "${CT_GO_PATH}/scripts/resetctdb.sh" --verbose - echo "Launching CT personalities" for ((i=0; i < http_server_count; i++)); do local port=$(pick_unused_port) From 6b187d3745a3263990622839588eb2f2731c67ad Mon Sep 17 00:00:00 2001 From: Roger Ng Date: Sat, 18 May 2024 21:46:58 +0000 Subject: [PATCH 44/55] Fix missing `$` in cloudbuild.yaml --- cloudbuild.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 9e9c30282a..50ac5ef7b1 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -91,7 +91,7 @@ steps: export MYSQL_HOST="mysql" export MYSQL_ROOT_PASSWORD="zaphod" export MYSQL_USER_HOST="%" - yes | bash "${CT_GO_PATH}/scripts/resetctdb.sh" --verbose + yes | bash "$${CT_GO_PATH}/scripts/resetctdb.sh" --verbose waitFor: ['prepare'] # Run the presubmit tests From fe2c8a0cd0fc758642c3dcf702e4cc3ca4a85890 Mon Sep 17 00:00:00 2001 From: Roger Ng Date: Sat, 18 May 2024 22:01:59 +0000 Subject: [PATCH 45/55] Reset CT test database in cloudbuild_master.yaml --- cloudbuild_master.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cloudbuild_master.yaml b/cloudbuild_master.yaml index 0e47e326f6..566edfe0f9 100644 --- a/cloudbuild_master.yaml +++ b/cloudbuild_master.yaml @@ -85,6 +85,13 @@ steps: # Wait for trillian logserver to be up until nc -z deployment_trillian-log-server_1 8090; do echo .; sleep 5; done + + # Reset the CT test database + export CT_GO_PATH="$$(go list -f '{{.Dir}}' github.com/google/certificate-transparency-go)" + export MYSQL_HOST="mysql" + export MYSQL_ROOT_PASSWORD="zaphod" + export MYSQL_USER_HOST="%" + yes | bash "$${CT_GO_PATH}/scripts/resetctdb.sh" --verbose waitFor: ['prepare'] # Run the presubmit tests From a55b3be51f4dfc3ea491b05ba4a27173a8ca9192 Mon Sep 17 00:00:00 2001 From: Roger Ng Date: Sat, 18 May 2024 22:10:55 +0000 Subject: [PATCH 46/55] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f4f24b6394..283d33f0f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ To reduce CT/Trillian database storage by deduplication of the entire issuance c Existing logs are not affected by this change. -Log operators can choose to opt-in this change for new CT logs by adding new CTFE configs in the [LogMultiConfig](trillian/ctfe/configpb/config.proto). See [example](trillian/examples/deployment/docker/ctfe/ct_server.cfg). +Log operators can choose to opt-in this change for new CT logs by adding new CTFE configs in the [LogMultiConfig](trillian/ctfe/configpb/config.proto) and importing the [database schema](trillian/ctfe/storage/mysql/schema.sql). See [example](trillian/examples/deployment/docker/ctfe/ct_server.cfg). - `ctfe_storage_connection_string` - `extra_data_issuance_chain_storage_backend` From 60b08661fd40696c1c4db78c92e188e1ca29bafd Mon Sep 17 00:00:00 2001 From: Roger Ng Date: Mon, 20 May 2024 15:03:27 +0000 Subject: [PATCH 47/55] Remove duplicated comment --- trillian/ctfe/instance.go | 1 - 1 file changed, 1 deletion(-) diff --git a/trillian/ctfe/instance.go b/trillian/ctfe/instance.go index 206b3cbd13..6fe480ce03 100644 --- a/trillian/ctfe/instance.go +++ b/trillian/ctfe/instance.go @@ -181,7 +181,6 @@ func setUpLogInfo(ctx context.Context, opts InstanceOptions) (*logInfo, error) { } // Initialise IssuanceChainService with IssuanceChainStorage and IssuanceChainCache. - // issuanceChainStorage is nil for Trillian gRPC or mysql.IssuanceChainStorage when MySQL is the prefix in database connection string. issuanceChainStorage, err := storage.NewIssuanceChainStorage(ctx, vCfg.ExtraDataIssuanceChainStorageBackend, vCfg.CTFEStorageConnectionString) if err != nil { return nil, err From 974e925a1860c0e916860c8beabe96c462bbc06d Mon Sep 17 00:00:00 2001 From: Roger Ng Date: Mon, 20 May 2024 15:05:20 +0000 Subject: [PATCH 48/55] Rename `hash` to `chainHash` --- trillian/util/log_leaf.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/trillian/util/log_leaf.go b/trillian/util/log_leaf.go index 8d24fc17a7..c498016242 100644 --- a/trillian/util/log_leaf.go +++ b/trillian/util/log_leaf.go @@ -51,33 +51,33 @@ func ExtraDataForChain(cert ct.ASN1Cert, chain []ct.ASN1Cert, isPrecert bool) ([ return tls.Marshal(extra) } -func BuildLogLeafWithHash(logPrefix string, merkleLeaf ct.MerkleTreeLeaf, leafIndex int64, cert ct.ASN1Cert, chain []ct.ASN1Cert, hash []byte, isPrecert bool) (trillian.LogLeaf, error) { - return buildLogLeaf(logPrefix, merkleLeaf, leafIndex, cert, chain, hash, isPrecert) +func BuildLogLeafWithHash(logPrefix string, merkleLeaf ct.MerkleTreeLeaf, leafIndex int64, cert ct.ASN1Cert, chain []ct.ASN1Cert, chainHash []byte, isPrecert bool) (trillian.LogLeaf, error) { + return buildLogLeaf(logPrefix, merkleLeaf, leafIndex, cert, chain, chainHash, isPrecert) } // ExtraDataForChainWithHash creates the extra data associated with a log entry as // described in RFC6962 section 4.6. -func ExtraDataForChainWithHash(cert ct.ASN1Cert, chain []ct.ASN1Cert, hash []byte, isPrecert bool) ([]byte, error) { +func ExtraDataForChainWithHash(cert ct.ASN1Cert, chain []ct.ASN1Cert, chainHash []byte, isPrecert bool) ([]byte, error) { var extra interface{} if isPrecert { // For a pre-cert, the extra data is a TLS-encoded PrecertChainEntry. extra = ct.PrecertChainEntryHash{ PreCertificate: cert, - IssuanceChainHash: hash, + IssuanceChainHash: chainHash, } } else { // For a certificate, the extra data is a TLS-encoded: // ASN.1Cert certificate_chain<0..2^24-1>; // containing the chain after the leaf. extra = ct.CertificateChainHash{ - IssuanceChainHash: hash, + IssuanceChainHash: chainHash, } } return tls.Marshal(extra) } -func buildLogLeaf(logPrefix string, merkleLeaf ct.MerkleTreeLeaf, leafIndex int64, cert ct.ASN1Cert, chain []ct.ASN1Cert, hash []byte, isPrecert bool) (trillian.LogLeaf, error) { +func buildLogLeaf(logPrefix string, merkleLeaf ct.MerkleTreeLeaf, leafIndex int64, cert ct.ASN1Cert, chain []ct.ASN1Cert, chainHash []byte, isPrecert bool) (trillian.LogLeaf, error) { leafData, err := tls.Marshal(merkleLeaf) if err != nil { klog.Warningf("%s: Failed to serialize Merkle leaf: %v", logPrefix, err) @@ -85,10 +85,10 @@ func buildLogLeaf(logPrefix string, merkleLeaf ct.MerkleTreeLeaf, leafIndex int6 } var extraData []byte - if hash == nil { + if chainHash == nil { extraData, err = ExtraDataForChain(cert, chain, isPrecert) } else { - extraData, err = ExtraDataForChainWithHash(cert, chain, hash, isPrecert) + extraData, err = ExtraDataForChainWithHash(cert, chain, chainHash, isPrecert) } if err != nil { klog.Warningf("%s: Failed to serialize chain for ExtraData: %v", logPrefix, err) From 7bf9a5209cfe8db180a40fe3b7195eb7babb45a1 Mon Sep 17 00:00:00 2001 From: Roger Ng Date: Mon, 20 May 2024 16:40:04 +0000 Subject: [PATCH 49/55] Unexport `isCTFEStorageEnabled` method --- trillian/ctfe/services.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/trillian/ctfe/services.go b/trillian/ctfe/services.go index d53efaff93..0440ccb3e8 100644 --- a/trillian/ctfe/services.go +++ b/trillian/ctfe/services.go @@ -46,14 +46,14 @@ func newIssuanceChainService(s storage.IssuanceChainStorage, c cache.IssuanceCha return service } -func (s *issuanceChainService) IsCTFEStorageEnabled() bool { +func (s *issuanceChainService) isCTFEStorageEnabled() bool { return s.storage != nil } // GetByHash returns the issuance chain with hash as the input. func (s *issuanceChainService) GetByHash(ctx context.Context, hash []byte) ([]byte, error) { // Return err if CTFE storage backend is not enabled. - if !s.IsCTFEStorageEnabled() { + if !s.isCTFEStorageEnabled() { return nil, errors.New("failed to GetByHash when storage is nil") } @@ -84,7 +84,7 @@ func (s *issuanceChainService) GetByHash(ctx context.Context, hash []byte) ([]by // of the chain. func (s *issuanceChainService) add(ctx context.Context, chain []byte) ([]byte, error) { // Return err if CTFE storage backend is not enabled. - if !s.IsCTFEStorageEnabled() { + if !s.isCTFEStorageEnabled() { return nil, errors.New("failed to Add when storage is nil") } @@ -112,7 +112,7 @@ func (s *issuanceChainService) BuildLogLeaf(ctx context.Context, chain []*x509.C // If CTFE storage is enabled for issuance chain, add the chain to storage // and cache, and then build log leaf. If Trillian gRPC is enabled for // issuance chain, build the log leaf. - if s.IsCTFEStorageEnabled() { + if s.isCTFEStorageEnabled() { issuanceChain, err := asn1.Marshal(raw[1:]) if err != nil { return &trillian.LogLeaf{}, fmt.Errorf("failed to marshal issuance chain: %s", err) @@ -140,7 +140,7 @@ func (s *issuanceChainService) BuildLogLeaf(ctx context.Context, chain []*x509.C // PrecertChainEntryHash, CertificateChainHash). func (s *issuanceChainService) FixLogLeaf(ctx context.Context, leaf *trillian.LogLeaf) error { // Skip if CTFE storage backend is not enabled. - if !s.IsCTFEStorageEnabled() { + if !s.isCTFEStorageEnabled() { return nil } From f6e7f4a8ff9e5cc93f8b8e753155be8317a279fa Mon Sep 17 00:00:00 2001 From: Roger Ng Date: Mon, 20 May 2024 16:43:19 +0000 Subject: [PATCH 50/55] Add comments for type/struct --- trillian/ctfe/cache/cache.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/trillian/ctfe/cache/cache.go b/trillian/ctfe/cache/cache.go index 0978e2ee45..edd6b8059a 100644 --- a/trillian/ctfe/cache/cache.go +++ b/trillian/ctfe/cache/cache.go @@ -24,14 +24,17 @@ import ( "github.com/google/certificate-transparency-go/trillian/ctfe/cache/noop" ) +// Type represents the cache type. type Type string +// Type constants for the cache type. const ( Unknown Type = "" NOOP Type = "noop" LRU Type = "lru" ) +// Option represents the cache option, which includes the cache size and time-to-live. type Option struct { Size int TTL time.Duration From 1a6afe94611eedbeb33fce8b6fb666d7378421b6 Mon Sep 17 00:00:00 2001 From: Roger Ng Date: Mon, 20 May 2024 17:27:59 +0000 Subject: [PATCH 51/55] Add code comment about the way we unmarshal leaf.ExtraData --- trillian/ctfe/services.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/trillian/ctfe/services.go b/trillian/ctfe/services.go index 0440ccb3e8..6c95b701ed 100644 --- a/trillian/ctfe/services.go +++ b/trillian/ctfe/services.go @@ -144,6 +144,8 @@ func (s *issuanceChainService) FixLogLeaf(ctx context.Context, leaf *trillian.Lo return nil } + // As the struct stored in leaf.ExtraData is unknown, the only way is to try to unmarshal with each possible struct. + // Try to unmarshal with ct.PrecertChainEntryHash struct. var precertChainHash ct.PrecertChainEntryHash if rest, err := tls.Unmarshal(leaf.ExtraData, &precertChainHash); err == nil && len(rest) == 0 { var chain []ct.ASN1Cert @@ -173,6 +175,7 @@ func (s *issuanceChainService) FixLogLeaf(ctx context.Context, leaf *trillian.Lo return nil } + // Try to unmarshal with ct.CertificateChainHash struct. var certChainHash ct.CertificateChainHash if rest, err := tls.Unmarshal(leaf.ExtraData, &certChainHash); err == nil && len(rest) == 0 { var entries []ct.ASN1Cert From 7265f9959b383ef36eef5f1f50b0b9de179b6e06 Mon Sep 17 00:00:00 2001 From: Roger Ng Date: Tue, 21 May 2024 19:44:58 +0000 Subject: [PATCH 52/55] Rename method to `ExtraDataForChainHash` and remove unused `chain` argument --- trillian/util/log_leaf.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/trillian/util/log_leaf.go b/trillian/util/log_leaf.go index c498016242..08e5fa99a9 100644 --- a/trillian/util/log_leaf.go +++ b/trillian/util/log_leaf.go @@ -55,9 +55,9 @@ func BuildLogLeafWithHash(logPrefix string, merkleLeaf ct.MerkleTreeLeaf, leafIn return buildLogLeaf(logPrefix, merkleLeaf, leafIndex, cert, chain, chainHash, isPrecert) } -// ExtraDataForChainWithHash creates the extra data associated with a log entry as -// described in RFC6962 section 4.6. -func ExtraDataForChainWithHash(cert ct.ASN1Cert, chain []ct.ASN1Cert, chainHash []byte, isPrecert bool) ([]byte, error) { +// ExtraDataForChainHash creates the extra data associated with a log entry as +// described in RFC6962 section 4.6 except the chain being replaced with its hash. +func ExtraDataForChainHash(cert ct.ASN1Cert, chainHash []byte, isPrecert bool) ([]byte, error) { var extra interface{} if isPrecert { @@ -88,7 +88,7 @@ func buildLogLeaf(logPrefix string, merkleLeaf ct.MerkleTreeLeaf, leafIndex int6 if chainHash == nil { extraData, err = ExtraDataForChain(cert, chain, isPrecert) } else { - extraData, err = ExtraDataForChainWithHash(cert, chain, chainHash, isPrecert) + extraData, err = ExtraDataForChainHash(cert, chainHash, isPrecert) } if err != nil { klog.Warningf("%s: Failed to serialize chain for ExtraData: %v", logPrefix, err) From 8a9053460d552ee839bc12bdcca86cc7ea4b3a7b Mon Sep 17 00:00:00 2001 From: Roger Ng Date: Tue, 21 May 2024 19:48:55 +0000 Subject: [PATCH 53/55] Remove unused `chain` argument --- trillian/ctfe/services.go | 2 +- trillian/util/log_leaf.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/trillian/ctfe/services.go b/trillian/ctfe/services.go index 6c95b701ed..d9611c0ef5 100644 --- a/trillian/ctfe/services.go +++ b/trillian/ctfe/services.go @@ -121,7 +121,7 @@ func (s *issuanceChainService) BuildLogLeaf(ctx context.Context, chain []*x509.C if err != nil { return &trillian.LogLeaf{}, fmt.Errorf("failed to add issuance chain into CTFE storage: %s", err) } - leaf, err := util.BuildLogLeafWithHash(logPrefix, *merkleLeaf, 0, raw[0], nil, hash, isPrecert) + leaf, err := util.BuildLogLeafWithChainHash(logPrefix, *merkleLeaf, 0, raw[0], hash, isPrecert) if err != nil { return &trillian.LogLeaf{}, fmt.Errorf("failed to build LogLeaf: %s", err) } diff --git a/trillian/util/log_leaf.go b/trillian/util/log_leaf.go index 08e5fa99a9..cc3761fca7 100644 --- a/trillian/util/log_leaf.go +++ b/trillian/util/log_leaf.go @@ -51,8 +51,8 @@ func ExtraDataForChain(cert ct.ASN1Cert, chain []ct.ASN1Cert, isPrecert bool) ([ return tls.Marshal(extra) } -func BuildLogLeafWithHash(logPrefix string, merkleLeaf ct.MerkleTreeLeaf, leafIndex int64, cert ct.ASN1Cert, chain []ct.ASN1Cert, chainHash []byte, isPrecert bool) (trillian.LogLeaf, error) { - return buildLogLeaf(logPrefix, merkleLeaf, leafIndex, cert, chain, chainHash, isPrecert) +func BuildLogLeafWithChainHash(logPrefix string, merkleLeaf ct.MerkleTreeLeaf, leafIndex int64, cert ct.ASN1Cert, chainHash []byte, isPrecert bool) (trillian.LogLeaf, error) { + return buildLogLeaf(logPrefix, merkleLeaf, leafIndex, cert, nil, chainHash, isPrecert) } // ExtraDataForChainHash creates the extra data associated with a log entry as From e65d8f56c61b702ea4e4d3cf1f6ee3ff7978745a Mon Sep 17 00:00:00 2001 From: Roger Ng Date: Tue, 21 May 2024 20:20:48 +0000 Subject: [PATCH 54/55] Update `FixLogLeaf` method comment --- trillian/ctfe/services.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/trillian/ctfe/services.go b/trillian/ctfe/services.go index d9611c0ef5..46c3e1cba4 100644 --- a/trillian/ctfe/services.go +++ b/trillian/ctfe/services.go @@ -135,9 +135,9 @@ func (s *issuanceChainService) BuildLogLeaf(ctx context.Context, chain []*x509.C } } -// FixLogLeaf recreates the LogLeaf.ExtraData if CTFE storage backend is -// enabled and the type of LogLeaf.ExtraData contains any hash (e.g. -// PrecertChainEntryHash, CertificateChainHash). +// FixLogLeaf recreates and populates the LogLeaf.ExtraData if CTFE storage +// backend is enabled and the type of LogLeaf.ExtraData contains any hash +// (e.g. PrecertChainEntryHash, CertificateChainHash). func (s *issuanceChainService) FixLogLeaf(ctx context.Context, leaf *trillian.LogLeaf) error { // Skip if CTFE storage backend is not enabled. if !s.isCTFEStorageEnabled() { From 1dd7f1049d405fa688bc153a829b41173bf89320 Mon Sep 17 00:00:00 2001 From: Roger Ng Date: Tue, 21 May 2024 20:32:52 +0000 Subject: [PATCH 55/55] Add buildLogLeaf method comment --- trillian/util/log_leaf.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/trillian/util/log_leaf.go b/trillian/util/log_leaf.go index cc3761fca7..6673195016 100644 --- a/trillian/util/log_leaf.go +++ b/trillian/util/log_leaf.go @@ -77,6 +77,9 @@ func ExtraDataForChainHash(cert ct.ASN1Cert, chainHash []byte, isPrecert bool) ( return tls.Marshal(extra) } +// buildLogLeaf builds the trillian.LogLeaf. The chainHash argument controls +// whether ExtraDataForChain or ExtraDataForChainHash method will be called. +// If chainHash is not nil, but neither is chain, then chain will be ignored. func buildLogLeaf(logPrefix string, merkleLeaf ct.MerkleTreeLeaf, leafIndex int64, cert ct.ASN1Cert, chain []ct.ASN1Cert, chainHash []byte, isPrecert bool) (trillian.LogLeaf, error) { leafData, err := tls.Marshal(merkleLeaf) if err != nil {