Merge branch 'master' into mutator

* master:
  Bump the golangci-lint version to v1.13.1 (google#1171)
  Remove deleted status from map and log trees (google#1172)
  tinkio.ProtoKeysetFile (google#1173)
  Update API links (google#1170)
  Store trillian map,log trees in DB (google#1167)

.travis.yml

@@ -34,7 +34,7 @@ before_install:
 
 install:
   - gcloud -q components install kubectl
-  - curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s -- -b $(go env GOPATH)/bin v1.12.5
+  - curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s -- -b $(go env GOPATH)/bin v1.13.1
   - go get -t ./...
 
 script:

README.md

@@ -20,7 +20,7 @@ account has been active and stable before trusting it.
 
 * [Overview](docs/overview.md)
 * [Design document](docs/design.md)
-* [API](docs/http_apis.md)
+* [API](docs/api.md)
 
 Key Transparency is inspired by [CONIKS](https://eprint.iacr.org/2014/1004.pdf)
 and [Certificate Transparency](https://www.certificate-transparency.org/).

cmd/keytransparency-client/cmd/get.go

@@ -28,7 +28,7 @@ var getCmd = &cobra.Command{
 	Short: "Retrieve and verify the current keyset",
 	Long: `Retrieve the user profile from the key server and verify that the
 results are consistent.`,
-	RunE: func(cmd *cobra.Command, args []string) error {
+	RunE: func(_ *cobra.Command, args []string) error {
 		if len(args) < 1 {
 			return fmt.Errorf("user email needs to be provided")
 		}

cmd/keytransparency-client/cmd/hammer.go

@@ -21,12 +21,14 @@ import (
 	"strings"
 	"time"
 
-	"github.com/google/keytransparency/core/client/hammer"
-	"github.com/google/keytransparency/impl/authentication"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 	"github.com/spf13/viper"
 	"google.golang.org/grpc"
+
+	"github.com/google/keytransparency/core/client/hammer"
+	"github.com/google/keytransparency/core/crypto/tinkio"
+	"github.com/google/keytransparency/impl/authentication"
 )
 
 var (
@@ -60,14 +62,20 @@ var hammerCmd = &cobra.Command{
 	Short: "Loadtest the server",
 	Long:  `Sends update requests for user_1 through user_n using a select number of workers in parallel.`,
 
-	PreRun: func(cmd *cobra.Command, args []string) {
-		handle, err := readKeysetFile(keysetFile, masterPassword)
+	PreRun: func(_ *cobra.Command, _ []string) {
+		masterKey, err := tinkio.MasterPBKDF(masterPassword)
+		if err != nil {
+			log.Fatal(err)
+		}
+		handle, err := tinkio.KeysetHandleFromEncryptedReader(
+			&tinkio.ProtoKeysetFile{File: keysetFile},
+			masterKey)
 		if err != nil {
 			log.Fatal(err)
 		}
 		keyset = handle
 	},
-	RunE: func(cmd *cobra.Command, args []string) error {
+	RunE: func(_ *cobra.Command, _ []string) error {
 		ktURL := viper.GetString("kt-url")
 		directoryID := viper.GetString("directory")
 		timeout := viper.GetDuration("timeout")

cmd/keytransparency-client/cmd/history.go

@@ -37,7 +37,7 @@ var histCmd = &cobra.Command{
 	Short: "Retrieve and verify all keys used for this account",
 	Long: `Retrieve all user profiles for this account from the key server
 and verify that the results are consistent.`,
-	RunE: func(cmd *cobra.Command, args []string) error {
+	RunE: func(_ *cobra.Command, args []string) error {
 		if len(args) < 1 {
 			return fmt.Errorf("user email needs to be provided")
 		}

cmd/keytransparency-client/cmd/keys.go

@@ -15,37 +15,25 @@
 package cmd
 
 import (
-	"crypto/sha256"
-	"encoding/hex"
 	"fmt"
-	"io/ioutil"
 	"log"
 	"os"
 	"text/tabwriter"
 
-	"github.com/golang/protobuf/proto"
-	"github.com/google/tink/go/insecure"
 	"github.com/google/tink/go/signature"
-	"github.com/google/tink/go/subtle/aead"
 	"github.com/google/tink/go/tink"
 	"github.com/spf13/cobra"
-	"golang.org/x/crypto/pbkdf2"
+
+	"github.com/google/keytransparency/core/crypto/tinkio"
 
 	tinkpb "github.com/google/tink/proto/tink_go_proto"
 )
 
-const (
-	keysetFile          = ".keyset"
-	masterKeyLen        = 32
-	masterKeyIterations = 4096
-)
+const keysetFile = ".keyset"
 
 var (
-	// openssl rand -hex 32
-	salt, _           = hex.DecodeString("00afc05d5b131a1dfd140a146b87f2f07826a8d4576cb4feef43f80f0c9b1c2f")
-	masterKeyHashFunc = sha256.New
-	keyType           string
-	masterPassword    string
+	keyType        string
+	masterPassword string
 )
 
 var keyset *tink.KeysetHandle
@@ -66,7 +54,7 @@ var createCmd = &cobra.Command{
 
 ./keytransparency-client authorized-keys create-keyset
 `,
-	RunE: func(cmd *cobra.Command, args []string) error {
+	RunE: func(_ *cobra.Command, _ []string) error {
 		template, err := keyTemplate(keyType)
 		if err != nil {
 			return err
@@ -76,8 +64,14 @@ var createCmd = &cobra.Command{
 		if err != nil {
 			return err
 		}
+		masterKey, err := tinkio.MasterPBKDF(masterPassword)
+		if err != nil {
+			return err
+		}
 
-		return writeKeysetFile(keyset, keysetFile, masterPassword)
+		return tinkio.WriteKeyset(keyset,
+			&tinkio.ProtoKeysetFile{File: keysetFile},
+			masterKey)
 	},
 }
 
@@ -104,14 +98,20 @@ var listCmd = &cobra.Command{
 
 The actual keys are not listed, only their corresponding metadata.
 `,
-	PreRun: func(cmd *cobra.Command, args []string) {
-		handle, err := readKeysetFile(keysetFile, masterPassword)
+	PreRun: func(_ *cobra.Command, _ []string) {
+		masterKey, err := tinkio.MasterPBKDF(masterPassword)
+		if err != nil {
+			log.Fatal(err)
+		}
+		handle, err := tinkio.KeysetHandleFromEncryptedReader(
+			&tinkio.ProtoKeysetFile{File: keysetFile},
+			masterKey)
 		if err != nil {
 			log.Fatal(err)
 		}
 		keyset = handle
 	},
-	RunE: func(cmd *cobra.Command, args []string) error {
+	RunE: func(_ *cobra.Command, _ []string) error {
 		keysetInfo, err := tink.GetKeysetInfo(keyset.Keyset())
 		if err != nil {
 			return err
@@ -136,88 +136,6 @@ The actual keys are not listed, only their corresponding metadata.
 	},
 }
 
-// masterPBKDF converts the master password into the master key.
-func masterPBKDF(masterPassword string) (tink.AEAD, error) {
-	if masterPassword == "" {
-		return nil, fmt.Errorf("please provide a master password")
-	}
-	dk := pbkdf2.Key([]byte(masterPassword), salt,
-		masterKeyIterations, masterKeyLen, masterKeyHashFunc)
-	return aead.NewAESGCM(dk)
-}
-
-func encryptKeyset(keyset *tinkpb.Keyset, masterKey tink.AEAD) (*tinkpb.EncryptedKeyset, error) {
-	serializedKeyset, err := proto.Marshal(keyset)
-	if err != nil {
-		return nil, fmt.Errorf("invalid keyset")
-	}
-	encrypted, err := masterKey.Encrypt(serializedKeyset, []byte{})
-	if err != nil {
-		return nil, fmt.Errorf("encrypted failed: %s", err)
-	}
-	// get keyset info
-	info, err := tink.GetKeysetInfo(keyset)
-	if err != nil {
-		return nil, fmt.Errorf("cannot get keyset info: %s", err)
-	}
-	encryptedKeyset := &tinkpb.EncryptedKeyset{
-		EncryptedKeyset: encrypted,
-		KeysetInfo:      info,
-	}
-	return encryptedKeyset, nil
-}
-
-func readKeysetFile(file, password string) (*tink.KeysetHandle, error) {
-	masterKey, err := masterPBKDF(password)
-	if err != nil {
-		return nil, err
-	}
-	data, err := ioutil.ReadFile(file)
-	if err != nil {
-		return nil, fmt.Errorf("reading keystore file %q failed: %v", file, err)
-	}
-
-	encryptedKeyset := new(tinkpb.EncryptedKeyset)
-	if err := proto.Unmarshal(data, encryptedKeyset); err != nil {
-		return nil, fmt.Errorf("could not parse encrypted keyset: %v", err)
-	}
-
-	keyset, err := decryptKeyset(encryptedKeyset, masterKey)
-	if err != nil {
-		return nil, err
-	}
-
-	return insecure.KeysetHandle(keyset)
-}
-
-func decryptKeyset(encryptedKeyset *tinkpb.EncryptedKeyset, masterKey tink.AEAD) (*tinkpb.Keyset, error) {
-	decrypted, err := masterKey.Decrypt(encryptedKeyset.EncryptedKeyset, []byte{})
-	if err != nil {
-		return nil, fmt.Errorf("decryption failed: %s", err)
-	}
-	keyset := new(tinkpb.Keyset)
-	if err := proto.Unmarshal(decrypted, keyset); err != nil {
-		return nil, fmt.Errorf("invalid encrypted keyset")
-	}
-	return keyset, nil
-}
-
-func writeKeysetFile(keyset *tink.KeysetHandle, file, password string) error {
-	masterKey, err := masterPBKDF(password)
-	if err != nil {
-		return err
-	}
-	encryptedKeyset, err := encryptKeyset(keyset.Keyset(), masterKey)
-	if err != nil {
-		return err
-	}
-	serialized, err := proto.Marshal(encryptedKeyset)
-	if err != nil {
-		return err
-	}
-	return ioutil.WriteFile(file, serialized, 0600)
-}
-
 func init() {
 	RootCmd.AddCommand(keysCmd)
 	keysCmd.AddCommand(listCmd)

cmd/keytransparency-client/cmd/post.go

@@ -25,6 +25,7 @@ import (
 	"google.golang.org/grpc"
 
 	tpb "github.com/google/keytransparency/core/api/type/type_go_proto"
+	"github.com/google/keytransparency/core/crypto/tinkio"
 	"github.com/google/tink/go/signature"
 	"github.com/google/tink/go/tink"
 )
@@ -45,14 +46,20 @@ and verifies that both the previous and current key-sets are accurate. eg:
 User email MUST match the OAuth account used to authorize the update.
 `,
 
-	PreRun: func(cmd *cobra.Command, args []string) {
-		handle, err := readKeysetFile(keysetFile, masterPassword)
+	PreRun: func(_ *cobra.Command, _ []string) {
+		masterKey, err := tinkio.MasterPBKDF(masterPassword)
+		if err != nil {
+			log.Fatal(err)
+		}
+		handle, err := tinkio.KeysetHandleFromEncryptedReader(
+			&tinkio.ProtoKeysetFile{File: keysetFile},
+			masterKey)
 		if err != nil {
 			log.Fatal(err)
 		}
 		keyset = handle
 	},
-	RunE: func(cmd *cobra.Command, args []string) error {
+	RunE: func(_ *cobra.Command, args []string) error {
 		// Validate input.
 		if len(args) < 1 {
 			return fmt.Errorf("user email needs to be provided")

cmd/keytransparency-client/cmd/root.go

@@ -53,7 +53,7 @@ var RootCmd = &cobra.Command{
 	Long: `The key transparency client retrieves and sets keys in the
 key transparency server.  The client verifies all cryptographic proofs the
 server provides to ensure that account data is accurate.`,
-	PersistentPreRun: func(cmd *cobra.Command, args []string) {
+	PersistentPreRun: func(_ *cobra.Command, _ []string) {
 		if verbose {
 			client.Vlog = log.New(os.Stdout, "", log.LstdFlags)
 		}

core/adminserver/admin_server.go

@@ -138,18 +138,10 @@ func (s *Server) ListDirectories(ctx context.Context, in *pb.ListDirectoriesRequ
 // fetchDirectory converts an adminstorage.Directory object into a pb.Directory object
 // by fetching the relevant info from Trillian.
 func (s *Server) fetchDirectory(ctx context.Context, d *directory.Directory) (*pb.Directory, error) {
-	logTree, err := s.logAdmin.GetTree(ctx, &tpb.GetTreeRequest{TreeId: d.LogID})
-	if err != nil {
-		return nil, err
-	}
-	mapTree, err := s.mapAdmin.GetTree(ctx, &tpb.GetTreeRequest{TreeId: d.MapID})
-	if err != nil {
-		return nil, err
-	}
 	return &pb.Directory{
 		DirectoryId: d.DirectoryID,
-		Log:         trimTree(logTree),
-		Map:         trimTree(mapTree),
+		Log:         d.Log,
+		Map:         d.Map,
 		Vrf:         d.VRF,
 		MinInterval: ptypes.DurationProto(d.MinInterval),
 		MaxInterval: ptypes.DurationProto(d.MaxInterval),
@@ -165,7 +157,6 @@ func trimTree(t *tpb.Tree) *tpb.Tree {
 		HashStrategy:       t.HashStrategy,
 		HashAlgorithm:      t.HashAlgorithm,
 		SignatureAlgorithm: t.SignatureAlgorithm,
-		Deleted:            t.Deleted,
 	}
 }
 
@@ -266,11 +257,14 @@ func (s *Server) CreateDirectory(ctx context.Context, in *pb.CreateDirectoryRequ
 			err, logTree.TreeId, delLogErr, mapTree.TreeId, delMapErr)
 	}
 
+	trimmedMap := trimTree(mapTree)
+	trimmedLog := trimTree(logTree)
+
 	// Create directory - {log, map} binding.
 	dir := &directory.Directory{
 		DirectoryID: in.GetDirectoryId(),
-		MapID:       mapTree.TreeId,
-		LogID:       logTree.TreeId,
+		Map:         trimmedMap,
+		Log:         trimmedLog,
 		VRF:         vrfPublicPB,
 		VRFPriv:     wrapped,
 		MinInterval: minInterval,
@@ -289,8 +283,8 @@ func (s *Server) CreateDirectory(ctx context.Context, in *pb.CreateDirectoryRequ
 
 	d := &pb.Directory{
 		DirectoryId: in.GetDirectoryId(),
-		Log:         trimTree(logTree),
-		Map:         trimTree(mapTree),
+		Log:         trimmedLog,
+		Map:         trimmedMap,
 		Vrf:         vrfPublicPB,
 		MinInterval: in.MinInterval,
 		MaxInterval: in.MaxInterval,

core/adminserver/admin_server_test.go

@@ -245,16 +245,6 @@ func TestDelete(t *testing.T) {
 		if _, err := svr.DeleteDirectory(ctx, &pb.DeleteDirectoryRequest{DirectoryId: tc.directoryID}); err != nil {
 			t.Fatalf("DeleteDirectory(): %v", err)
 		}
-		directory, err := svr.GetDirectory(ctx, &pb.GetDirectoryRequest{DirectoryId: tc.directoryID, ShowDeleted: true})
-		if err != nil {
-			t.Fatalf("GetDirectory(): %v", err)
-		}
-		if got, want := directory.Log.Deleted, true; got != want {
-			t.Errorf("Log.Deleted: %v, want %v", got, want)
-		}
-		if got, want := directory.Map.Deleted, true; got != want {
-			t.Errorf("Map.Deleted: %v, want %v", got, want)
-		}
 		if _, err := svr.GarbageCollect(ctx, &pb.GarbageCollectRequest{
 			Before: ptypes.TimestampNow(),
 		}); err != nil {