From db4382f609116078d35a88cb94b5f1fbcb8ecbe1 Mon Sep 17 00:00:00 2001 From: Filippo Valsorda Date: Mon, 8 Apr 2024 13:58:35 +0200 Subject: [PATCH] sunlight: expose LogEntry --- internal/ctlog/cache.go | 21 ++-- internal/ctlog/ctlog.go | 203 +++++++-------------------------- internal/ctlog/ctlog_test.go | 6 +- internal/ctlog/export_test.go | 2 +- internal/ctlog/http.go | 8 +- internal/ctlog/testlog_test.go | 42 ++++--- tile.go | 156 +++++++++++++++++++++++++ 7 files changed, 243 insertions(+), 195 deletions(-) create mode 100644 tile.go diff --git a/internal/ctlog/cache.go b/internal/ctlog/cache.go index efbb9b4..be009b7 100644 --- a/internal/ctlog/cache.go +++ b/internal/ctlog/cache.go @@ -3,6 +3,7 @@ package ctlog import ( "crawshaw.io/sqlite" "crawshaw.io/sqlite/sqlitex" + "filippo.io/sunlight" "github.com/prometheus/client_golang/prometheus" ) @@ -40,17 +41,13 @@ func (l *Log) CloseCache() error { return l.cacheWrite.Close() } -func (l *Log) cacheGet(leaf *LogEntry) (*SequencedLogEntry, error) { +func (l *Log) cacheGet(leaf *PendingLogEntry) (*sunlight.LogEntry, error) { defer prometheus.NewTimer(l.m.CacheGetDuration).ObserveDuration() h := leaf.cacheHash() - var se *SequencedLogEntry + var se *sunlight.LogEntry err := sqlitex.Exec(l.cacheRead, "SELECT timestamp, leaf_index FROM cache WHERE key = ?", func(stmt *sqlite.Stmt) error { - se = &SequencedLogEntry{ - LogEntry: *leaf, - LeafIndex: stmt.GetInt64("leaf_index"), - Timestamp: stmt.GetInt64("timestamp"), - } + se = leaf.asLogEntry(stmt.GetInt64("leaf_index"), stmt.GetInt64("timestamp")) return nil }, h[:]) if err != nil { @@ -59,11 +56,17 @@ func (l *Log) cacheGet(leaf *LogEntry) (*SequencedLogEntry, error) { return se, nil } -func (l *Log) cachePut(entries []*SequencedLogEntry) (err error) { +func (l *Log) cachePut(entries []*sunlight.LogEntry) (err error) { defer prometheus.NewTimer(l.m.CachePutDuration).ObserveDuration() defer sqlitex.Save(l.cacheWrite)(&err) for _, se := range entries { - h := se.cacheHash() + h := (&PendingLogEntry{ + Certificate: se.Certificate, + IsPrecert: se.IsPrecert, + IssuerKeyHash: se.IssuerKeyHash, + PreCertificate: se.PreCertificate, + PrecertSigningCert: se.PrecertSigningCert, + }).cacheHash() err := sqlitex.Exec(l.cacheWrite, "INSERT INTO cache (key, timestamp, leaf_index) VALUES (?, ?, ?)", nil, h[:], se.Timestamp, se.LeafIndex) if err != nil { diff --git a/internal/ctlog/ctlog.go b/internal/ctlog/ctlog.go index 19f9d8c..5e77cc0 100644 --- a/internal/ctlog/ctlog.go +++ b/internal/ctlog/ctlog.go @@ -12,7 +12,6 @@ import ( "fmt" "log/slog" "maps" - "math" "math/rand" "sync" "time" @@ -259,9 +258,9 @@ func LoadLog(ctx context.Context, config *Config) (*Log, error) { // Verify the data tile against the level 0 tile. b := edgeTiles[-1].B - start := tileWidth * dataTile.N + start := sunlight.TileWidth * dataTile.N for i := start; i < start+int64(dataTile.W); i++ { - e, rest, err := ReadTileLeaf(b) + e, rest, err := sunlight.ReadTileLeaf(b) if err != nil { return nil, fmt.Errorf("invalid data tile %v: %w", dataTile.Tile, err) } @@ -374,16 +373,13 @@ type LockedCheckpoint interface { Bytes() []byte } -const TileHeight = 8 -const tileWidth = 1 << TileHeight - type tileReader struct { fetch func(key string) ([]byte, error) saveTiles func(tiles []tlog.Tile, data [][]byte) } func (r *tileReader) Height() int { - return TileHeight + return sunlight.TileHeight } func (r *tileReader) ReadTiles(tiles []tlog.Tile) (data [][]byte, err error) { @@ -399,27 +395,31 @@ func (r *tileReader) ReadTiles(tiles []tlog.Tile) (data [][]byte, err error) { func (r *tileReader) SaveTiles(tiles []tlog.Tile, data [][]byte) { r.saveTiles(tiles, data) } -type LogEntry struct { - // Certificate is either the x509_entry or the tbs_certificate for precerts. - Certificate []byte - +// PendingLogEntry is a subset of sunlight.LogEntry that was not yet sequenced, +// so doesn't have an index or timestamp. +type PendingLogEntry struct { + Certificate []byte IsPrecert bool IssuerKeyHash [32]byte PreCertificate []byte PrecertSigningCert []byte } -// signedEntry returns the entry_type and signed_entry fields of -// a RFC 6962 TimestampedEntry. -func (e *LogEntry) signedEntry() []byte { - // struct { - // LogEntryType entry_type; - // select(entry_type) { - // case x509_entry: ASN.1Cert; - // case precert_entry: PreCert; - // } signed_entry; - // } SignedEntry; +func (e *PendingLogEntry) asLogEntry(idx, timestamp int64) *sunlight.LogEntry { + return &sunlight.LogEntry{ + Certificate: e.Certificate, + IsPrecert: e.IsPrecert, + IssuerKeyHash: e.IssuerKeyHash, + PreCertificate: e.PreCertificate, + PrecertSigningCert: e.PrecertSigningCert, + LeafIndex: idx, + Timestamp: timestamp, + } +} +type cacheHash [16]byte // birthday bound of 2⁴⁸ entries with collision chance 2⁻³² + +func (e *PendingLogEntry) cacheHash() cacheHash { b := &cryptobyte.Builder{} if !e.IsPrecert { b.AddUint16(0 /* entry_type = x509_entry */) @@ -433,76 +433,12 @@ func (e *LogEntry) signedEntry() []byte { b.AddBytes(e.Certificate) }) } - return b.BytesOrPanic() -} - -type cacheHash [16]byte // birthday bound of 2⁴⁸ entries with collision chance 2⁻³² - -func (e *LogEntry) cacheHash() cacheHash { - h := sha256.Sum256(e.signedEntry()) + h := sha256.Sum256(b.BytesOrPanic()) return cacheHash(h[:16]) } -type SequencedLogEntry struct { - LogEntry - LeafIndex int64 - Timestamp int64 -} - -// MerkleTreeLeaf returns a RFC 6962 MerkleTreeLeaf. -func (e *SequencedLogEntry) MerkleTreeLeaf() []byte { - b := &cryptobyte.Builder{} - b.AddUint8(0 /* version = v1 */) - b.AddUint8(0 /* leaf_type = timestamped_entry */) - b.AddUint64(uint64(e.Timestamp)) - b.AddBytes(e.signedEntry()) - addExtensions(b, e.LeafIndex) - return b.BytesOrPanic() -} - -func addExtensions(b *cryptobyte.Builder, leafIndex int64) { - b.AddUint16LengthPrefixed(func(b *cryptobyte.Builder) { - ext, err := sunlight.MarshalExtensions(sunlight.Extensions{LeafIndex: leafIndex}) - if err != nil { - b.SetError(err) - return - } - b.AddBytes(ext) - }) -} - -// TileLeaf returns the custom structure that's encoded in data tiles. -func (e *SequencedLogEntry) TileLeaf() []byte { - // struct { - // TimestampedEntry timestamped_entry; - // select(entry_type) { - // case x509_entry: Empty; - // case precert_entry: PreCertExtraData; - // } extra_data; - // } TileLeaf; - // - // struct { - // ASN.1Cert pre_certificate; - // opaque PrecertificateSigningCertificate<0..2^24-1>; - // } PreCertExtraData; - - b := &cryptobyte.Builder{} - b.AddUint64(uint64(e.Timestamp)) - b.AddBytes(e.signedEntry()) - addExtensions(b, e.LeafIndex) - if e.IsPrecert { - b.AddUint24LengthPrefixed(func(b *cryptobyte.Builder) { - b.AddBytes(e.PreCertificate) - }) - b.AddUint24LengthPrefixed(func(b *cryptobyte.Builder) { - b.AddBytes(e.PrecertSigningCert) - }) - } - return b.BytesOrPanic() -} - type pool struct { - pendingLeaves []*LogEntry + pendingLeaves []*PendingLogEntry byHash map[cacheHash]waitEntryFunc // done is closed when the pool has been sequenced and @@ -519,7 +455,7 @@ type pool struct { timestamp int64 } -type waitEntryFunc func(ctx context.Context) (*SequencedLogEntry, error) +type waitEntryFunc func(ctx context.Context) (*sunlight.LogEntry, error) func newPool() *pool { return &pool{ @@ -534,7 +470,7 @@ var errPoolFull = fmtErrorf("rate limited") // deduplication cache. It returns a function that will wait until the pool is // sequenced and return the sequenced leaf, as well as the source of the // sequenced leaf (pool or cache if deduplicated, sequencer otherwise). -func (l *Log) addLeafToPool(leaf *LogEntry) (f waitEntryFunc, source string) { +func (l *Log) addLeafToPool(leaf *PendingLogEntry) (f waitEntryFunc, source string) { l.poolMu.Lock() defer l.poolMu.Unlock() p := l.currentPool @@ -546,22 +482,22 @@ func (l *Log) addLeafToPool(leaf *LogEntry) (f waitEntryFunc, source string) { return f, "pool" } if leaf, err := l.cacheGet(leaf); err != nil { - return func(ctx context.Context) (*SequencedLogEntry, error) { + return func(ctx context.Context) (*sunlight.LogEntry, error) { return nil, fmtErrorf("deduplication cache get failed: %w", err) }, "cache" } else if leaf != nil { - return func(ctx context.Context) (*SequencedLogEntry, error) { + return func(ctx context.Context) (*sunlight.LogEntry, error) { return leaf, nil }, "cache" } n := len(p.pendingLeaves) if l.c.PoolSize > 0 && n >= l.c.PoolSize { - return func(ctx context.Context) (*SequencedLogEntry, error) { + return func(ctx context.Context) (*sunlight.LogEntry, error) { return nil, errPoolFull }, "ratelimit" } p.pendingLeaves = append(p.pendingLeaves, leaf) - f = func(ctx context.Context) (*SequencedLogEntry, error) { + f = func(ctx context.Context) (*sunlight.LogEntry, error) { select { case <-ctx.Done(): return nil, fmtErrorf("context canceled while waiting for sequencing: %w", ctx.Err()) @@ -572,11 +508,8 @@ func (l *Log) addLeafToPool(leaf *LogEntry) (f waitEntryFunc, source string) { if p.timestamp == 0 { panic("internal error: pool is ready but result is missing") } - return &SequencedLogEntry{ - LogEntry: *leaf, - LeafIndex: p.firstLeafIndex + int64(n), - Timestamp: p.timestamp, - }, nil + idx := p.firstLeafIndex + int64(n) + return leaf.asLogEntry(idx, p.timestamp), nil } } p.byHash[h] = f @@ -663,19 +596,19 @@ func (l *Log) sequencePool(ctx context.Context, p *pool) (err error) { edgeTiles := maps.Clone(l.edgeTiles) var dataTile []byte // Load the current partial data tile, if any. - if t, ok := edgeTiles[-1]; ok && t.W < tileWidth { + if t, ok := edgeTiles[-1]; ok && t.W < sunlight.TileWidth { dataTile = bytes.Clone(t.B) } newHashes := make(map[int64]tlog.Hash) hashReader := l.hashReader(newHashes) n := l.tree.N - var sequencedLeaves []*SequencedLogEntry + var sequencedLeaves []*sunlight.LogEntry for _, leaf := range p.pendingLeaves { - leaf := &SequencedLogEntry{LogEntry: *leaf, Timestamp: timestamp, LeafIndex: n} + leaf := leaf.asLogEntry(n, timestamp) sequencedLeaves = append(sequencedLeaves, leaf) - leafData := leaf.TileLeaf() - dataTile = append(dataTile, leafData...) - l.m.SeqLeafSize.Observe(float64(len(leafData))) + oldTileSize := len(dataTile) + dataTile = sunlight.AppendTileLeaf(dataTile, leaf) + l.m.SeqLeafSize.Observe(float64(len(dataTile) - oldTileSize)) // Compute the new tree hashes and add them to the hashReader overlay // (we will use them later to insert more leaves and finally to produce @@ -692,8 +625,8 @@ func (l *Log) sequencePool(ctx context.Context, p *pool) (err error) { n++ // If the data tile is full, upload it. - if n%tileWidth == 0 { - tile := tlog.TileForIndex(TileHeight, tlog.StoredHashIndex(0, n-1)) + if n%sunlight.TileWidth == 0 { + tile := tlog.TileForIndex(sunlight.TileHeight, tlog.StoredHashIndex(0, n-1)) tile.L = -1 edgeTiles[-1] = tileWithBytes{tile, dataTile} l.c.Log.DebugContext(ctx, "uploading full data tile", @@ -707,8 +640,8 @@ func (l *Log) sequencePool(ctx context.Context, p *pool) (err error) { } // Upload leftover partial data tile, if any. - if n != l.tree.N && n%tileWidth != 0 { - tile := tlog.TileForIndex(TileHeight, tlog.StoredHashIndex(0, n-1)) + if n != l.tree.N && n%sunlight.TileWidth != 0 { + tile := tlog.TileForIndex(sunlight.TileHeight, tlog.StoredHashIndex(0, n-1)) tile.L = -1 edgeTiles[-1] = tileWithBytes{tile, dataTile} l.c.Log.DebugContext(ctx, "uploading partial data tile", @@ -719,7 +652,7 @@ func (l *Log) sequencePool(ctx context.Context, p *pool) (err error) { } // Produce and upload new tree tiles. - tiles := tlog.NewTiles(TileHeight, l.tree.N, n) + tiles := tlog.NewTiles(sunlight.TileHeight, l.tree.N, n) for _, tile := range tiles { tile := tile // tile is captured by the g.Go function. data, err := tlog.ReadTileData(tile, hashReader) @@ -890,7 +823,7 @@ func (l *Log) hashReader(overlay map[int64]tlog.Hash) tlog.HashReaderFunc { list = append(list, h) continue } - t := l.edgeTiles[tlog.TileForIndex(TileHeight, id).L] + t := l.edgeTiles[tlog.TileForIndex(sunlight.TileHeight, id).L] h, err := tlog.HashFromTile(t.Tile, t.B, id) if err != nil { return nil, fmt.Errorf("index %d not in overlay and %w", id, err) @@ -900,53 +833,3 @@ func (l *Log) hashReader(overlay map[int64]tlog.Hash) tlog.HashReaderFunc { return list, nil } } - -func ReadTileLeaf(tile []byte) (e *SequencedLogEntry, rest []byte, err error) { - e = &SequencedLogEntry{} - s := cryptobyte.String(tile) - var timestamp uint64 - var entryType uint16 - var extensions cryptobyte.String - if !s.ReadUint64(×tamp) || !s.ReadUint16(&entryType) || timestamp > math.MaxInt64 { - return nil, s, fmtErrorf("invalid data tile") - } - e.Timestamp = int64(timestamp) - switch entryType { - case 0: // x509_entry - if !s.ReadUint24LengthPrefixed((*cryptobyte.String)(&e.Certificate)) || - !s.ReadUint16LengthPrefixed(&extensions) { - return nil, s, fmtErrorf("invalid data tile x509_entry") - } - case 1: // precert_entry - e.IsPrecert = true - if !s.CopyBytes(e.IssuerKeyHash[:]) || - !s.ReadUint24LengthPrefixed((*cryptobyte.String)(&e.Certificate)) || - !s.ReadUint16LengthPrefixed(&extensions) || - !s.ReadUint24LengthPrefixed((*cryptobyte.String)(&e.PreCertificate)) || - !s.ReadUint24LengthPrefixed((*cryptobyte.String)(&e.PrecertSigningCert)) { - return nil, s, fmtErrorf("invalid data tile precert_entry") - } - default: - return nil, s, fmtErrorf("invalid data tile: unknown type %d", entryType) - } - var extensionType uint8 - var extensionData cryptobyte.String - if !extensions.ReadUint8(&extensionType) || extensionType != 0 || - !extensions.ReadUint16LengthPrefixed(&extensionData) || - !readUint40(&extensionData, &e.LeafIndex) || !extensionData.Empty() || - !extensions.Empty() { - return nil, s, fmtErrorf("invalid data tile extensions") - } - return e, s, nil -} - -// readUint40 decodes a big-endian, 40-bit value into out and advances over it. -// It reports whether the read was successful. -func readUint40(s *cryptobyte.String, out *int64) bool { - var v []byte - if !s.ReadBytes(&v, 5) { - return false - } - *out = int64(v[0])<<32 | int64(v[1])<<24 | int64(v[2])<<16 | int64(v[3])<<8 | int64(v[4]) - return true -} diff --git a/internal/ctlog/ctlog_test.go b/internal/ctlog/ctlog_test.go index 4f88e9f..ecc0504 100644 --- a/internal/ctlog/ctlog_test.go +++ b/internal/ctlog/ctlog_test.go @@ -161,7 +161,7 @@ func TestDuplicates(t *testing.T) { }) } -func testDuplicates(t *testing.T, addWithSeed func(*testing.T, *TestLog, int64) func(context.Context) (*ctlog.SequencedLogEntry, error)) { +func testDuplicates(t *testing.T, addWithSeed func(*testing.T, *TestLog, int64) func(context.Context) (*sunlight.LogEntry, error)) { tl := NewEmptyTestLog(t) addWithSeed(t, tl, mathrand.Int63()) // 0 addWithSeed(t, tl, mathrand.Int63()) // 1 @@ -241,7 +241,7 @@ func TestReloadLog(t *testing.T) { }) } -func testReloadLog(t *testing.T, add func(*testing.T, *TestLog) func(context.Context) (*ctlog.SequencedLogEntry, error)) { +func testReloadLog(t *testing.T, add func(*testing.T, *TestLog) func(context.Context) (*sunlight.LogEntry, error)) { // TODO: test reloading after uploading tiles but before uploading STH. tl := NewEmptyTestLog(t) n := int64(tileWidth + 2) @@ -393,7 +393,7 @@ func BenchmarkSequencer(b *testing.B) { if i%poolSize == 0 && i != 0 { fatalIfErr(b, tl.Log.Sequence()) } - tl.Log.AddLeafToPool(&ctlog.LogEntry{Certificate: bytes.Repeat([]byte("A"), 2350)}) + tl.Log.AddLeafToPool(&ctlog.PendingLogEntry{Certificate: bytes.Repeat([]byte("A"), 2350)}) } } diff --git a/internal/ctlog/export_test.go b/internal/ctlog/export_test.go index 5e5d709..d0f8a7f 100644 --- a/internal/ctlog/export_test.go +++ b/internal/ctlog/export_test.go @@ -2,7 +2,7 @@ package ctlog import "context" -func (l *Log) AddLeafToPool(e *LogEntry) (waitEntryFunc, string) { +func (l *Log) AddLeafToPool(e *PendingLogEntry) (waitEntryFunc, string) { return l.addLeafToPool(e) } diff --git a/internal/ctlog/http.go b/internal/ctlog/http.go index 212a05b..61a6e9d 100644 --- a/internal/ctlog/http.go +++ b/internal/ctlog/http.go @@ -59,7 +59,7 @@ func ReusedConnContext(ctx context.Context, c net.Conn) context.Context { } func (l *Log) addChain(rw http.ResponseWriter, r *http.Request) { - rsp, code, err := l.addChainOrPreChain(r.Context(), r.Body, func(le *LogEntry) error { + rsp, code, err := l.addChainOrPreChain(r.Context(), r.Body, func(le *PendingLogEntry) error { if le.IsPrecert { return fmtErrorf("pre-certificate submitted to add-chain") } @@ -85,7 +85,7 @@ func (l *Log) addChain(rw http.ResponseWriter, r *http.Request) { } func (l *Log) addPreChain(rw http.ResponseWriter, r *http.Request) { - rsp, code, err := l.addChainOrPreChain(r.Context(), r.Body, func(le *LogEntry) error { + rsp, code, err := l.addChainOrPreChain(r.Context(), r.Body, func(le *PendingLogEntry) error { if !le.IsPrecert { return fmtErrorf("final certificate submitted to add-pre-chain") } @@ -110,7 +110,7 @@ func (l *Log) addPreChain(rw http.ResponseWriter, r *http.Request) { } } -func (l *Log) addChainOrPreChain(ctx context.Context, reqBody io.ReadCloser, checkType func(*LogEntry) error) (response []byte, code int, err error) { +func (l *Log) addChainOrPreChain(ctx context.Context, reqBody io.ReadCloser, checkType func(*PendingLogEntry) error) (response []byte, code int, err error) { labels := prometheus.Labels{"error": "", "issuer": "", "root": "", "reused": "", "precert": "", "preissuer": "", "chain_len": "", "source": ""} defer func() { @@ -145,7 +145,7 @@ func (l *Log) addChainOrPreChain(ctx context.Context, reqBody io.ReadCloser, che labels["root"] = x509util.NameToString(chain[len(chain)-1].Subject) labels["issuer"] = x509util.NameToString(chain[0].Issuer) - e := &LogEntry{Certificate: chain[0].Raw} + e := &PendingLogEntry{Certificate: chain[0].Raw} issuers := chain[1:] if isPrecert, err := ctfe.IsPrecertificate(chain[0]); err != nil { l.c.Log.WarnContext(ctx, "invalid precertificate", "err", err, "body", body) diff --git a/internal/ctlog/testlog_test.go b/internal/ctlog/testlog_test.go index f01ae6f..a87ba52 100644 --- a/internal/ctlog/testlog_test.go +++ b/internal/ctlog/testlog_test.go @@ -150,17 +150,17 @@ func (tl *TestLog) CheckLog() (sthTimestamp int64) { leafHashes, err := tlog.TileHashReader(tree, (*tileReader)(tl)).ReadHashes(indexes) fatalIfErr(t, err) - lastTile := tlog.TileForIndex(ctlog.TileHeight, tlog.StoredHashIndex(0, c.N-1)) + lastTile := tlog.TileForIndex(sunlight.TileHeight, tlog.StoredHashIndex(0, c.N-1)) lastTile.L = -1 for n := int64(0); n <= lastTile.N; n++ { - tile := tlog.Tile{H: ctlog.TileHeight, L: -1, N: n, W: tileWidth} + tile := tlog.Tile{H: sunlight.TileHeight, L: -1, N: n, W: tileWidth} if n == lastTile.N { tile = lastTile } b, err := tl.Config.Backend.Fetch(context.Background(), tile.Path()) fatalIfErr(t, err) for i := 0; i < tile.W; i++ { - e, rest, err := ctlog.ReadTileLeaf(b) + e, rest, err := sunlight.ReadTileLeaf(b) if err != nil { t.Fatalf("invalid data tile %v", tile) } @@ -226,11 +226,11 @@ func (tl *TestLog) StartSequencer() { }() } -func waitFuncWrapper(t testing.TB, le *ctlog.LogEntry, expectSuccess bool, - f func(ctx context.Context) (*ctlog.SequencedLogEntry, error), -) func(ctx context.Context) (*ctlog.SequencedLogEntry, error) { +func waitFuncWrapper(t testing.TB, le *ctlog.PendingLogEntry, expectSuccess bool, + f func(ctx context.Context) (*sunlight.LogEntry, error), +) func(ctx context.Context) (*sunlight.LogEntry, error) { var called bool - fw := func(ctx context.Context) (*ctlog.SequencedLogEntry, error) { + fw := func(ctx context.Context) (*sunlight.LogEntry, error) { called = true se, err := f(ctx) if !expectSuccess { @@ -239,7 +239,13 @@ func waitFuncWrapper(t testing.TB, le *ctlog.LogEntry, expectSuccess bool, } } else if err != nil { t.Error(err) - } else if !reflect.DeepEqual(*le, se.LogEntry) { + } else if !reflect.DeepEqual(le, &ctlog.PendingLogEntry{ + Certificate: se.Certificate, + IsPrecert: se.IsPrecert, + IssuerKeyHash: se.IssuerKeyHash, + PreCertificate: se.PreCertificate, + PrecertSigningCert: se.PrecertSigningCert, + }) { t.Error("LogEntry is different") } return se, err @@ -252,13 +258,13 @@ func waitFuncWrapper(t testing.TB, le *ctlog.LogEntry, expectSuccess bool, return fw } -func addCertificate(t *testing.T, tl *TestLog) func(ctx context.Context) (*ctlog.SequencedLogEntry, error) { +func addCertificate(t *testing.T, tl *TestLog) func(ctx context.Context) (*sunlight.LogEntry, error) { return addCertificateWithSeed(t, tl, mathrand.Int63()) // 2⁻³² chance of collision after 2¹⁶ entries } -func addCertificateWithSeed(t *testing.T, tl *TestLog, seed int64) func(ctx context.Context) (*ctlog.SequencedLogEntry, error) { +func addCertificateWithSeed(t *testing.T, tl *TestLog, seed int64) func(ctx context.Context) (*sunlight.LogEntry, error) { r := mathrand.New(mathrand.NewSource(seed)) - e := &ctlog.LogEntry{} + e := &ctlog.PendingLogEntry{} e.Certificate = make([]byte, r.Intn(4)+8) r.Read(e.Certificate) f, _ := tl.Log.AddLeafToPool(e) @@ -266,27 +272,27 @@ func addCertificateWithSeed(t *testing.T, tl *TestLog, seed int64) func(ctx cont } func addCertificateFast(t *testing.T, tl *TestLog) { - e := &ctlog.LogEntry{} + e := &ctlog.PendingLogEntry{} e.Certificate = make([]byte, mathrand.Intn(3)+1) rand.Read(e.Certificate) tl.Log.AddLeafToPool(e) } func addCertificateExpectFailure(t *testing.T, tl *TestLog) { - e := &ctlog.LogEntry{} + e := &ctlog.PendingLogEntry{} e.Certificate = make([]byte, mathrand.Intn(4)+8) rand.Read(e.Certificate) f, _ := tl.Log.AddLeafToPool(e) waitFuncWrapper(t, e, false, f) } -func addPreCertificate(t *testing.T, tl *TestLog) func(ctx context.Context) (*ctlog.SequencedLogEntry, error) { +func addPreCertificate(t *testing.T, tl *TestLog) func(ctx context.Context) (*sunlight.LogEntry, error) { return addPreCertificateWithSeed(t, tl, mathrand.Int63()) } -func addPreCertificateWithSeed(t *testing.T, tl *TestLog, seed int64) func(ctx context.Context) (*ctlog.SequencedLogEntry, error) { +func addPreCertificateWithSeed(t *testing.T, tl *TestLog, seed int64) func(ctx context.Context) (*sunlight.LogEntry, error) { r := mathrand.New(mathrand.NewSource(seed)) - e := &ctlog.LogEntry{IsPrecert: true} + e := &ctlog.PendingLogEntry{IsPrecert: true} e.Certificate = make([]byte, r.Intn(4)+8) r.Read(e.Certificate) e.PreCertificate = make([]byte, r.Intn(4)+1) @@ -300,12 +306,12 @@ func addPreCertificateWithSeed(t *testing.T, tl *TestLog, seed int64) func(ctx c return waitFuncWrapper(t, e, true, f) } -const tileWidth = 1 << ctlog.TileHeight +const tileWidth = 1 << sunlight.TileHeight type tileReader TestLog func (r *tileReader) Height() int { - return ctlog.TileHeight + return sunlight.TileHeight } func (r *tileReader) ReadTiles(tiles []tlog.Tile) (data [][]byte, err error) { diff --git a/tile.go b/tile.go new file mode 100644 index 0000000..028dacc --- /dev/null +++ b/tile.go @@ -0,0 +1,156 @@ +package sunlight + +import ( + "fmt" + "math" + + "golang.org/x/crypto/cryptobyte" +) + +const TileHeight = 8 +const TileWidth = 1 << TileHeight + +type LogEntry struct { + // Certificate is either the TimestampedEntry.signed_entry, or the + // PreCert.tbs_certificate for Precertificates. + // It must be at most 2^24-1 bytes long. + Certificate []byte + + // IsPrecert is true if LogEntryType is precert_entry. Otherwise, the + // following three fields are zero and ignored. + IsPrecert bool + + // IssuerKeyHash is the PreCert.issuer_key_hash. + IssuerKeyHash [32]byte + + // PreCertificate is the PrecertChainEntry.pre_certificate. + // It must be at most 2^24-1 bytes long. + PreCertificate []byte + + // PrecertSigningCert is the Precertificate Signing Certificate, if any, as + // represented in the first entry of PrecertChainEntry.precertificate_chain. + // It may be nil, and must be at most 2^24-1 bytes long. + PrecertSigningCert []byte + + // LeafIndex is the zero-based index of the leaf in the log. + // It must be between 0 and 2^40-1. + LeafIndex int64 + + // Timestamp is the TimestampedEntry.timestamp. + Timestamp int64 +} + +// MerkleTreeLeaf returns a RFC 6962 MerkleTreeLeaf. +func (e *LogEntry) MerkleTreeLeaf() []byte { + b := &cryptobyte.Builder{} + b.AddUint8(0 /* version = v1 */) + b.AddUint8(0 /* leaf_type = timestamped_entry */) + b.AddUint64(uint64(e.Timestamp)) + if !e.IsPrecert { + b.AddUint16(0 /* entry_type = x509_entry */) + b.AddUint24LengthPrefixed(func(b *cryptobyte.Builder) { + b.AddBytes(e.Certificate) + }) + } else { + b.AddUint16(1 /* entry_type = precert_entry */) + b.AddBytes(e.IssuerKeyHash[:]) + b.AddUint24LengthPrefixed(func(b *cryptobyte.Builder) { + b.AddBytes(e.Certificate) + }) + } + addExtensions(b, e.LeafIndex) + return b.BytesOrPanic() +} + +// struct { +// TimestampedEntry timestamped_entry; +// select(entry_type) { +// case x509_entry: Empty; +// case precert_entry: PreCertExtraData; +// } extra_data; +// } TileLeaf; +// +// struct { +// ASN.1Cert pre_certificate; +// opaque PrecertificateSigningCertificate<0..2^24-1>; +// } PreCertExtraData; + +// ReadTileLeaf reads a LogEntry from a data tile, and returns the remaining +// data in the tile. +func ReadTileLeaf(tile []byte) (e *LogEntry, rest []byte, err error) { + e = &LogEntry{} + s := cryptobyte.String(tile) + var timestamp uint64 + var entryType uint16 + var extensions cryptobyte.String + if !s.ReadUint64(×tamp) || !s.ReadUint16(&entryType) || timestamp > math.MaxInt64 { + return nil, s, fmt.Errorf("invalid data tile") + } + e.Timestamp = int64(timestamp) + switch entryType { + case 0: // x509_entry + if !s.ReadUint24LengthPrefixed((*cryptobyte.String)(&e.Certificate)) || + !s.ReadUint16LengthPrefixed(&extensions) { + return nil, s, fmt.Errorf("invalid data tile x509_entry") + } + case 1: // precert_entry + e.IsPrecert = true + if !s.CopyBytes(e.IssuerKeyHash[:]) || + !s.ReadUint24LengthPrefixed((*cryptobyte.String)(&e.Certificate)) || + !s.ReadUint16LengthPrefixed(&extensions) || + !s.ReadUint24LengthPrefixed((*cryptobyte.String)(&e.PreCertificate)) || + !s.ReadUint24LengthPrefixed((*cryptobyte.String)(&e.PrecertSigningCert)) { + return nil, s, fmt.Errorf("invalid data tile precert_entry") + } + default: + return nil, s, fmt.Errorf("invalid data tile: unknown type %d", entryType) + } + var extensionType uint8 + var extensionData cryptobyte.String + if !extensions.ReadUint8(&extensionType) || extensionType != 0 || + !extensions.ReadUint16LengthPrefixed(&extensionData) || + !readUint40(&extensionData, &e.LeafIndex) || !extensionData.Empty() || + !extensions.Empty() { + return nil, s, fmt.Errorf("invalid data tile extensions") + } + return e, s, nil +} + +// AppendTileLeaf appends a LogEntry to a data tile. +func AppendTileLeaf(t []byte, e *LogEntry) []byte { + b := cryptobyte.NewBuilder(t) + b.AddUint64(uint64(e.Timestamp)) + if !e.IsPrecert { + b.AddUint16(0 /* entry_type = x509_entry */) + b.AddUint24LengthPrefixed(func(b *cryptobyte.Builder) { + b.AddBytes(e.Certificate) + }) + } else { + b.AddUint16(1 /* entry_type = precert_entry */) + b.AddBytes(e.IssuerKeyHash[:]) + b.AddUint24LengthPrefixed(func(b *cryptobyte.Builder) { + b.AddBytes(e.Certificate) + }) + } + addExtensions(b, e.LeafIndex) + if e.IsPrecert { + b.AddUint24LengthPrefixed(func(b *cryptobyte.Builder) { + b.AddBytes(e.PreCertificate) + }) + b.AddUint24LengthPrefixed(func(b *cryptobyte.Builder) { + b.AddBytes(e.PrecertSigningCert) + }) + } + return b.BytesOrPanic() +} + +func addExtensions(b *cryptobyte.Builder, leafIndex int64) { + b.AddUint16LengthPrefixed(func(b *cryptobyte.Builder) { + ext, err := MarshalExtensions(Extensions{LeafIndex: leafIndex}) + if err != nil { + b.SetError(err) + return + } + b.AddBytes(ext) + }) +}