Include checkpoint (STH) in entry upload and retrieve responses (#1015)

* Include checkpoint (STH) in entry upload and retrieve responses

This associates a root hash in an inclusion proof with a signed
commitment from the log. Previously, without this included, there was no
connection between an inclusion proof and the log. An inclusion proof
and checkpoint can be an alternative proof of inclusion instead of a
SET.

Ref #988

Signed-off-by: Hayden Blauzvern <hblauzvern@google.com>

* Refactor with common implementation for creating signed checkpoint

Signed-off-by: Hayden Blauzvern <hblauzvern@google.com>

* Update client to verify checkpoint signature

Signed-off-by: Hayden Blauzvern <hblauzvern@google.com>

* Address linter

Signed-off-by: Hayden Blauzvern <hblauzvern@google.com>

* Add checkpoint verification to VerifyLogEntry

Signed-off-by: Hayden Blauzvern <hblauzvern@google.com>

Signed-off-by: Hayden Blauzvern <hblauzvern@google.com>

cmd/rekor-cli/app/get.go

@@ -118,6 +118,13 @@ var getCmd = &cobra.Command{
 					return nil, fmt.Errorf("unable to verify entry was added to log: %w", err)
 				}
 
+				// verify checkpoint
+				if entry.Verification.InclusionProof.Checkpoint != nil {
+					if err := verify.VerifyCheckpointSignature(&e, verifier); err != nil {
+						return nil, err
+					}
+				}
+
 				return parseEntry(ix, entry)
 			}
 		}

cmd/rekor-cli/app/upload.go

@@ -141,6 +141,17 @@ var uploadCmd = &cobra.Command{
 		if err := verify.VerifySignedEntryTimestamp(ctx, &logEntry, verifier); err != nil {
 			return nil, fmt.Errorf("unable to verify entry was added to log: %w", err)
 		}
+		// TODO: Remove conditional once inclusion proof/checkpoint is always returned by server.
+		if logEntry.Verification.InclusionProof != nil {
+			// verify inclusion proof
+			if err := verify.VerifyInclusion(ctx, &logEntry); err != nil {
+				return nil, fmt.Errorf("error verifying inclusion proof: %w", err)
+			}
+			// verify checkpoint
+			if err := verify.VerifyCheckpointSignature(&logEntry, verifier); err != nil {
+				return nil, err
+			}
+		}
 
 		return &uploadCmdOutput{
 			Location: string(resp.Location),

cmd/rekor-cli/app/verify.go

@@ -37,18 +37,24 @@ import (
 )
 
 type verifyCmdOutput struct {
-	RootHash  string
-	EntryUUID string
-	Index     int64
-	Size      int64
-	Hashes    []string
+	RootHash   string
+	EntryUUID  string
+	Index      int64
+	Size       int64
+	Hashes     []string
+	Checkpoint string
 }
 
 func (v *verifyCmdOutput) String() string {
 	s := fmt.Sprintf("Current Root Hash: %v\n", v.RootHash)
 	s += fmt.Sprintf("Entry Hash: %v\n", v.EntryUUID)
 	s += fmt.Sprintf("Entry Index: %v\n", v.Index)
-	s += fmt.Sprintf("Current Tree Size: %v\n\n", v.Size)
+	s += fmt.Sprintf("Current Tree Size: %v\n", v.Size)
+	if len(v.Checkpoint) > 0 {
+		s += fmt.Sprintf("Checkpoint:\n%v\n\n", v.Checkpoint)
+	} else {
+		s += "\n"
+	}
 
 	s += "Inclusion Proof:\n"
 	hasher := rfc6962.DefaultHasher
@@ -147,6 +153,9 @@ var verifyCmd = &cobra.Command{
 				Size:      *v.Verification.InclusionProof.TreeSize,
 				Hashes:    v.Verification.InclusionProof.Hashes,
 			}
+			if v.Verification.InclusionProof.Checkpoint != nil {
+				o.Checkpoint = *v.Verification.InclusionProof.Checkpoint
+			}
 			entry = v
 		}
 
@@ -167,6 +176,7 @@ var verifyCmd = &cobra.Command{
 			return nil, err
 		}
 
+		// verify inclusion proof, checkpoint, and SET
 		if err := verify.VerifyLogEntry(ctx, &entry, verifier); err != nil {
 			return nil, fmt.Errorf("validating entry: %w", err)
 		}

openapi.yaml

@@ -631,11 +631,16 @@ definitions:
           type: string
           description: SHA256 hash value expressed in hexadecimal format
           pattern: '^[0-9a-fA-F]{64}$'
+      checkpoint:
+        type: string
+        format: signedCheckpoint
+        description: The checkpoint (signed tree head) that the inclusion proof is based on
     required:
       - logIndex
       - rootHash
       - treeSize
       - hashes
+      - checkpoint
 
   Error:
     type: object

pkg/api/entries.go

@@ -42,6 +42,7 @@ import (
 	"github.com/sigstore/rekor/pkg/log"
 	"github.com/sigstore/rekor/pkg/sharding"
 	"github.com/sigstore/rekor/pkg/types"
+	"github.com/sigstore/rekor/pkg/util"
 	"github.com/sigstore/sigstore/pkg/signature"
 	"github.com/sigstore/sigstore/pkg/signature/options"
 )
@@ -92,11 +93,17 @@ func logEntryFromLeaf(ctx context.Context, signer signature.Signer, tc TrillianC
 		return nil, fmt.Errorf("signing entry error: %w", err)
 	}
 
+	scBytes, err := util.CreateAndSignCheckpoint(ctx, viper.GetString("rekor_server.hostname"), tc.logID, root, api.signer)
+	if err != nil {
+		return nil, err
+	}
+
 	inclusionProof := models.InclusionProof{
-		TreeSize: swag.Int64(int64(root.TreeSize)),
-		RootHash: swag.String(hex.EncodeToString(root.RootHash)),
-		LogIndex: swag.Int64(proof.GetLeafIndex()),
-		Hashes:   hashes,
+		TreeSize:   swag.Int64(int64(root.TreeSize)),
+		RootHash:   swag.String(hex.EncodeToString(root.RootHash)),
+		LogIndex:   swag.Int64(proof.GetLeafIndex()),
+		Hashes:     hashes,
+		Checkpoint: stringPointer(string(scBytes)),
 	}
 
 	uuid := hex.EncodeToString(leaf.MerkleLeafHash)
@@ -261,7 +268,30 @@ func createLogEntry(params entries.CreateLogEntryParams) (models.LogEntry, middl
 		return nil, handleRekorAPIError(params, http.StatusInternalServerError, fmt.Errorf("signing entry error: %v", err), signingError)
 	}
 
+	root := &ttypes.LogRootV1{}
+	if err := root.UnmarshalBinary(resp.getLeafAndProofResult.SignedLogRoot.LogRoot); err != nil {
+		return nil, handleRekorAPIError(params, http.StatusInternalServerError, fmt.Errorf("error unmarshalling log root: %v", err), sthGenerateError)
+	}
+	hashes := []string{}
+	for _, hash := range resp.getLeafAndProofResult.Proof.Hashes {
+		hashes = append(hashes, hex.EncodeToString(hash))
+	}
+
+	scBytes, err := util.CreateAndSignCheckpoint(ctx, viper.GetString("rekor_server.hostname"), tc.logID, root, api.signer)
+	if err != nil {
+		return nil, handleRekorAPIError(params, http.StatusInternalServerError, err, sthGenerateError)
+	}
+
+	inclusionProof := models.InclusionProof{
+		TreeSize:   swag.Int64(int64(root.TreeSize)),
+		RootHash:   swag.String(hex.EncodeToString(root.RootHash)),
+		LogIndex:   swag.Int64(queuedLeaf.LeafIndex),
+		Hashes:     hashes,
+		Checkpoint: stringPointer(string(scBytes)),
+	}
+
 	logEntryAnon.Verification = &models.LogEntryAnonVerification{
+		InclusionProof:       &inclusionProof,
 		SignedEntryTimestamp: strfmt.Base64(signature),
 	}
 

pkg/api/tlog.go

@@ -21,7 +21,6 @@ import (
 	"fmt"
 	"net/http"
 	"strconv"
-	"time"
 
 	"github.com/go-openapi/runtime/middleware"
 	"github.com/go-openapi/swag"
@@ -33,7 +32,6 @@ import (
 	"github.com/sigstore/rekor/pkg/generated/restapi/operations/tlog"
 	"github.com/sigstore/rekor/pkg/log"
 	"github.com/sigstore/rekor/pkg/util"
-	"github.com/sigstore/sigstore/pkg/signature/options"
 )
 
 // GetLogInfoHandler returns the current size of the tree and the STH
@@ -68,32 +66,16 @@ func GetLogInfoHandler(params tlog.GetLogInfoParams) middleware.Responder {
 	hashString := hex.EncodeToString(root.RootHash)
 	treeSize := int64(root.TreeSize)
 
-	sth, err := util.CreateSignedCheckpoint(util.Checkpoint{
-		Origin: fmt.Sprintf("%s - %d", viper.GetString("rekor_server.hostname"), tc.ranges.ActiveTreeID()),
-		Size:   root.TreeSize,
-		Hash:   root.RootHash,
-	})
+	scBytes, err := util.CreateAndSignCheckpoint(params.HTTPRequest.Context(),
+		viper.GetString("rekor_server.hostname"), tc.ranges.ActiveTreeID(), root, api.signer)
 	if err != nil {
-		return handleRekorAPIError(params, http.StatusInternalServerError, fmt.Errorf("marshalling error: %w", err), sthGenerateError)
+		return handleRekorAPIError(params, http.StatusInternalServerError, err, sthGenerateError)
 	}
-	sth.SetTimestamp(uint64(time.Now().UnixNano()))
-
-	// sign the log root ourselves to get the log root signature
-	_, err = sth.Sign(viper.GetString("rekor_server.hostname"), api.signer, options.WithContext(params.HTTPRequest.Context()))
-	if err != nil {
-		return handleRekorAPIError(params, http.StatusInternalServerError, fmt.Errorf("signing error: %w", err), signingError)
-	}
-
-	scBytes, err := sth.SignedNote.MarshalText()
-	if err != nil {
-		return handleRekorAPIError(params, http.StatusInternalServerError, fmt.Errorf("marshalling error: %w", err), sthGenerateError)
-	}
-	scString := string(scBytes)
 
 	logInfo := models.LogInfo{
 		RootHash:       &hashString,
 		TreeSize:       &treeSize,
-		SignedTreeHead: &scString,
+		SignedTreeHead: stringPointer(string(scBytes)),
 		TreeID:         stringPointer(fmt.Sprintf("%d", tc.logID)),
 		InactiveShards: inactiveShards,
 	}
@@ -169,25 +151,11 @@ func inactiveShardLogInfo(ctx context.Context, tid int64) (*models.InactiveShard
 	hashString := hex.EncodeToString(root.RootHash)
 	treeSize := int64(root.TreeSize)
 
-	sth, err := util.CreateSignedCheckpoint(util.Checkpoint{
-		Origin: fmt.Sprintf("%s - %d", viper.GetString("rekor_server.hostname"), tid),
-		Size:   root.TreeSize,
-		Hash:   root.RootHash,
-	})
+	scBytes, err := util.CreateAndSignCheckpoint(ctx, viper.GetString("rekor_server.hostname"), tid, root, api.signer)
 	if err != nil {
 		return nil, err
 	}
-	sth.SetTimestamp(uint64(time.Now().UnixNano()))
 
-	// sign the log root ourselves to get the log root signature
-	if _, err := sth.Sign(viper.GetString("rekor_server.hostname"), api.signer, options.WithContext(ctx)); err != nil {
-		return nil, err
-	}
-
-	scBytes, err := sth.SignedNote.MarshalText()
-	if err != nil {
-		return nil, err
-	}
 	m := models.InactiveShardLogInfo{
 		RootHash:       &hashString,
 		TreeSize:       &treeSize,

pkg/api/trillian_client.go

@@ -185,6 +185,8 @@ func (t *TrillianClient) addLeaf(byteValue []byte) *Response {
 		status:       status.Code(err),
 		err:          err,
 		getAddResult: resp,
+		// include getLeafAndProofResult for inclusion proof
+		getLeafAndProofResult: leafResp.getLeafAndProofResult,
 	}
 }
 

pkg/util/checkpoint.go

@@ -17,11 +17,17 @@ package util
 
 import (
 	"bytes"
+	"context"
 	"encoding/base64"
 	"errors"
 	"fmt"
 	"strconv"
 	"strings"
+	"time"
+
+	"github.com/google/trillian/types"
+	"github.com/sigstore/sigstore/pkg/signature"
+	"github.com/sigstore/sigstore/pkg/signature/options"
 )
 
 // heavily borrowed from https://github.com/google/trillian-examples/blob/master/formats/log/checkpoint.go
@@ -160,3 +166,24 @@ func (r *SignedCheckpoint) GetTimestamp() uint64 {
 	}
 	return ts
 }
+
+// CreateAndSignCheckpoint creates a signed checkpoint as a commitment to the current root hash
+func CreateAndSignCheckpoint(ctx context.Context, hostname string, treeID int64, root *types.LogRootV1, signer signature.Signer) ([]byte, error) {
+	sth, err := CreateSignedCheckpoint(Checkpoint{
+		Origin: fmt.Sprintf("%s - %d", hostname, treeID),
+		Size:   root.TreeSize,
+		Hash:   root.RootHash,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("error creating checkpoint: %v", err)
+	}
+	sth.SetTimestamp(uint64(time.Now().UnixNano()))
+	if _, err := sth.Sign(hostname, signer, options.WithContext(ctx)); err != nil {
+		return nil, fmt.Errorf("error signing checkpoint: %v", err)
+	}
+	scBytes, err := sth.SignedNote.MarshalText()
+	if err != nil {
+		return nil, fmt.Errorf("error marshalling checkpoint: %v", err)
+	}
+	return scBytes, nil
+}