add new server API for listing keys

This commit adds a new server API:
```
/v1/key/list/<pattern>
```

This API returns a nd-json stream of
`KeyDescription` objects - each one
describing one key at the KES server.

The key listing API is designed to have
a low memory footprint (since there may
be thousands of keys) even when many
clients try to list keys in parallel.

Therefore, the listing implementation
(at the KES server as well as at the client)
uses iterators to list keys lazy. As soon
as a client disconnects / times out /...
the listing is aborted to not waste server
resources.

This commit also adds a CLI command:
```
kes key list [options] [<pattern>]
```

client.go

@@ -6,6 +6,7 @@ package kes
 
 import (
 	"bytes"
+	"context"
 	"crypto/tls"
 	"encoding"
 	"encoding/base64"
@@ -188,6 +189,83 @@ func (d *DEK) UnmarshalBinary(data []byte) error {
 	return nil
 }
 
+// KeyIterator iterates over list of KeyDescription objects.
+//   for iterator.Next() {
+//       _ = iterator.Value() // Use the KeyDescription
+//   }
+//   if err := iterator.Err(); err != nil {
+//   }
+//
+// Once done with iterating over the list of KeyDescription
+// objects, an iterator should be closed using the Close
+// method.
+//
+// In general, a KeyIterator does not provide any guarantees
+// about ordering or the when its underlying source is modified
+// concurrently.
+// Particularly, if a key is created or deleted at the KES a
+// KeyIterator may or may not be affected by this change.
+type KeyIterator struct {
+	response *http.Response
+	decoder  *json.Decoder
+
+	last   KeyDescription
+	err    error
+	closed bool
+}
+
+// KeyDescription describes a cryptographic key at a KES server.
+type KeyDescription struct {
+	// Name is the name of the cryptographic key.
+	Name string `json:"name"`
+}
+
+// Next returns true if there is another KeyDescription.
+// This KeyDescription can be retrieved via the Value method.
+//
+// It returns false once there is no more KeyDescription
+// or if the KeyIterator encountered an error. The error,
+// if any, can be retrieved via the Err method.
+func (i *KeyIterator) Next() bool {
+	if i.closed || i.err != nil {
+		return false
+	}
+	if err := i.decoder.Decode(&i.last); err != nil {
+		if err == io.EOF {
+			i.err = i.Close()
+		} else {
+			i.err = err
+		}
+		return false
+	}
+	return true
+}
+
+// Value returns the current KeyDescription. It returns
+// the same KeyDescription until Next is called again.
+//
+// If KeyIterator has been closed or if Next has not been
+// called once resp. once Next returns false then the
+// behavior of Value is undefined.
+func (i *KeyIterator) Value() KeyDescription { return i.last }
+
+// Err returns the first error encountered by the KeyIterator,
+// if any.
+func (i *KeyIterator) Err() error { return i.err }
+
+// Close closes the underlying connection to the KES server
+// and returns any encountered error, if any.
+func (i *KeyIterator) Close() error {
+	i.closed = true
+	if err := i.response.Body.Close(); err != nil {
+		return err
+	}
+	if err := parseErrorTrailer(i.response.Trailer); err != nil {
+		return err
+	}
+	return nil
+}
+
 // Version tries to fetch the version information from the
 // KES server.
 func (c *Client) Version() (string, error) {
@@ -417,6 +495,33 @@ func (c *Client) Decrypt(name string, ciphertext, context []byte) ([]byte, error
 	return response.Plaintext, nil
 }
 
+// ListKeys returns a new KeyIterator that iterates over all keys
+// matching the given glob pattern.
+//
+// The KeyIterator will stop once the given context.Done() completes,
+// an error occurs while iterating or once there are no more
+// KeyDescription objects - whatever happens first.
+//
+// If the pattern is empty it defaults to "*".
+func (c *Client) ListKeys(ctx context.Context, pattern string) (*KeyIterator, error) {
+	if pattern == "" { // The empty pattern never matches anything
+		pattern = "*" // => default to: list all keys
+	}
+
+	client := retry(c.HTTPClient)
+	resp, err := client.Get(endpoint(c.Endpoint, "/v1/key/list", url.PathEscape(pattern)))
+	if err != nil {
+		return nil, err
+	}
+	if resp.StatusCode != http.StatusOK {
+		return nil, parseErrorResponse(resp)
+	}
+	return &KeyIterator{
+		response: resp,
+		decoder:  json.NewDecoder(resp.Body),
+	}, nil
+}
+
 // SetPolicy adds the given policy to the set of policies.
 // There can be just one policy with one particular name at
 // one point in time.

cmd/kes/key.go

@@ -5,7 +5,9 @@
 package main
 
 import (
+	"context"
 	"encoding/base64"
+	"encoding/json"
 	"flag"
 	"fmt"
 	stdlog "log"
@@ -18,6 +20,7 @@ const keyCmdUsage = `Usage:
 Commands:
     create                  Create a new secret key at a KES server.
     delete                  Delete a secret key from a KES server.
+	list                    List secret key names at a KES server.
     derive                  Derive a new key from a secret key.
     decrypt                 Decrypt a ciphertext with secret key.
 
@@ -38,6 +41,8 @@ func key(args []string) {
 	switch args = cli.Args(); args[0] {
 	case "create":
 		createKey(args)
+	case "list":
+		listKeys(args)
 	case "delete":
 		deleteKey(args)
 	case "derive":
@@ -232,6 +237,67 @@ func deriveKey(args []string) {
 	}
 }
 
+const listKeyCmdUsage = `Usage:
+    kes key list [options] [<pattern>]
+
+Options:
+    --json                 Print key names as JSON
+    -k, --insecure         Skip X.509 certificate validation during TLS handshake
+    -h, --help             Show list of command-line options
+
+Lists the description for all keys that match the optional <pattern>. If no
+pattern is provided the default pattern ('*') is used - which matches any
+key name, and therefore, lists all keys.
+
+Examples:
+    $ kes key list my-key*
+`
+
+func listKeys(args []string) {
+	cli := flag.NewFlagSet(args[0], flag.ExitOnError)
+	cli.Usage = func() { fmt.Fprint(os.Stderr, listKeyCmdUsage) }
+
+	var (
+		insecureSkipVerify bool
+		jsonFlag           bool
+	)
+	cli.BoolVar(&jsonFlag, "json", false, "Print key names as JSON")
+	cli.BoolVar(&insecureSkipVerify, "k", false, "Skip X.509 certificate validation during TLS handshake")
+	cli.BoolVar(&insecureSkipVerify, "insecure", false, "Skip X.509 certificate validation during TLS handshake")
+	cli.Parse(args[1:])
+
+	if cli.NArg() > 1 {
+		stdlog.Fatal("Error: too many arguments")
+	}
+
+	var pattern = "*"
+	if cli.NArg() == 1 {
+		pattern = cli.Arg(0)
+	}
+	iterator, err := newClient(insecureSkipVerify).ListKeys(context.Background(), pattern)
+	if err != nil {
+		stdlog.Fatalf("Error: failed to list keys matching %q: %v", pattern, err)
+	}
+
+	if !isTerm(os.Stdout) || jsonFlag {
+		encoder := json.NewEncoder(os.Stdout)
+		for iterator.Next() {
+			encoder.Encode(iterator.Value())
+		}
+	} else {
+		for iterator.Next() {
+			fmt.Println(iterator.Value().Name)
+		}
+	}
+	if err = iterator.Err(); err != nil {
+		iterator.Close()
+		stdlog.Fatalf("Error: %v", err)
+	}
+	if err = iterator.Close(); err != nil {
+		stdlog.Fatalf("Error: %v", err)
+	}
+}
+
 const deleteCmdUsage = `Usage:
     kes key delete [options] <name>
 

cmd/kes/log.go

@@ -164,7 +164,7 @@ func traceAuditLogWithUI(stream *kes.AuditStream) {
 				table.Draw()
 			case event.ID == "<C-c>" || event.ID == "<Escape>":
 				if err := stream.Close(); err != nil {
-					fmt.Fprintln(os.Stderr, fmt.Sprintf("Error: audit log stream closed with: %v", err))
+					fmt.Fprintf(os.Stderr, "Error: audit log stream closed with: %v\n", err)
 				}
 				return
 			}
@@ -241,7 +241,7 @@ func traceErrorLogWithUI(stream *kes.ErrorStream) {
 				table.Draw()
 			case event.ID == "<C-c>" || event.ID == "<Escape>":
 				if err := stream.Close(); err != nil {
-					fmt.Fprintln(os.Stderr, fmt.Sprintf("Error: error log stream closed with: %v", err))
+					fmt.Fprintf(os.Stderr, "Error: error log stream closed with: %v\n", err)
 				}
 				return
 			}