Consolidate identity management to identity cli commands (#1083)

* Consolidate identity management:

Move identity cretaion/signing out of storagenode setup command.

* fixes

* linters

* Consolidate identity management:

Move identity cretaion/signing out of storagenode setup command.

* fixes

* sava backups before saving signed certs

* add "-prebuilt-test-cmds" test flag

* linters

* prepare cli tests for travis

* linter fixes

* more fixes

* linter gods

* sp/sdk/sim

* remove ca.difficulty

* remove unused difficulty

* return setup to its rightful place

* wip travis

* Revert "wip travis"

This reverts commit 5683484.

* typo in travis.yaml

* remove tests

* remove more

* make it only create one identity at a time for consistency

* add config-dir for consitency

* add identity creation to storj-sim

* add flags

* simplify

* fix nolint and compile

* prevent overwrite and pass difficulty, concurrency, and parent creds

* goimports

.travis.yml

@@ -64,14 +64,13 @@ matrix:
         - golangci-lint run
         # - gospace istidy
 
-    ### service integration tests ###
+    ### integration tests ###
     - env: MODE=integration
       services:
         - redis
       install:
         - source scripts/install-awscli.sh
         - make install-sim
-        - go install -race storj.io/storj/cmd/certificates
       script:
         - set -o pipefail && make test-sim |& go run scripts/fail-on-race.go
         - set -o pipefail && make test-certificate-signing |& go run scripts/fail-on-race.go

Makefile

@@ -70,7 +70,7 @@ proto: ## Rebuild protobuf files
 .PHONY: install-sim
 install-sim: ## install storj-sim
 	@echo "Running ${@}"
-	@go install -race -v storj.io/storj/cmd/storj-sim storj.io/storj/cmd/bootstrap storj.io/storj/cmd/satellite storj.io/storj/cmd/storagenode storj.io/storj/cmd/uplink storj.io/storj/cmd/gateway
+	@go install -race -v storj.io/storj/cmd/storj-sim storj.io/storj/cmd/bootstrap storj.io/storj/cmd/satellite storj.io/storj/cmd/storagenode storj.io/storj/cmd/uplink storj.io/storj/cmd/gateway storj.io/storj/cmd/identity storj.io/storj/cmd/certificates
 
 ##@ Test
 

cmd/bootstrap/main.go

@@ -4,6 +4,7 @@
 package main
 
 import (
+	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -101,9 +102,9 @@ func cmdSetup(cmd *cobra.Command, args []string) (err error) {
 		cfg.Identity.CertPath = filepath.Join(setupDir, "identity.cert")
 		cfg.Identity.KeyPath = filepath.Join(setupDir, "identity.key")
 	}
-	err = identity.SetupIdentity(process.Ctx(cmd), cfg.CA, cfg.Identity)
-	if err != nil {
-		return err
+
+	if cfg.Identity.Status() != identity.CertKey {
+		return errors.New("identity is missing")
 	}
 
 	overrides := map[string]interface{}{

cmd/certificates/setup.go

@@ -4,6 +4,7 @@
 package main
 
 import (
+	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -69,14 +70,15 @@ func cmdSetup(cmd *cobra.Command, args []string) error {
 	if _, err := setupCfg.Signer.NewAuthDB(); err != nil {
 		return err
 	}
-	setupCfg.CA.CertPath = filepath.Join(setupDir, "ca.cert")
-	setupCfg.CA.KeyPath = filepath.Join(setupDir, "ca.key")
-	setupCfg.Identity.CertPath = filepath.Join(setupDir, "identity.cert")
-	setupCfg.Identity.KeyPath = filepath.Join(setupDir, "identity.key")
 
-	err = identity.SetupIdentity(process.Ctx(cmd), setupCfg.CA, setupCfg.Identity)
-	if err != nil {
-		return err
+	if setupDir != defaultConfDir {
+		setupCfg.CA.CertPath = filepath.Join(setupDir, "ca.cert")
+		setupCfg.CA.KeyPath = filepath.Join(setupDir, "ca.key")
+		setupCfg.Identity.CertPath = filepath.Join(setupDir, "identity.cert")
+		setupCfg.Identity.KeyPath = filepath.Join(setupDir, "identity.key")
+	}
+	if setupCfg.Identity.Status() != identity.CertKey {
+		return errors.New("identity is missing")
 	}
 
 	o := map[string]interface{}{

cmd/identity/certificate_authority.go

@@ -16,31 +16,36 @@ import (
 
 var (
 	caCmd = &cobra.Command{
-		Use:   "ca",
-		Short: "Manage certificate authorities",
+		Use:         "ca",
+		Short:       "Manage certificate authorities",
+		Annotations: map[string]string{"type": "setup"},
 	}
 
 	newCACmd = &cobra.Command{
-		Use:   "new",
-		Short: "Create a new certificate authority",
-		RunE:  cmdNewCA,
+		Use:         "new",
+		Short:       "Create a new certificate authority",
+		RunE:        cmdNewCA,
+		Annotations: map[string]string{"type": "setup"},
 	}
 
 	getIDCmd = &cobra.Command{
-		Use:   "id",
-		Short: "Get the id of a CA",
-		RunE:  cmdGetID,
+		Use:         "id",
+		Short:       "Get the id of a CA",
+		RunE:        cmdGetID,
+		Annotations: map[string]string{"type": "setup"},
 	}
 
 	caExtCmd = &cobra.Command{
-		Use:   "extensions",
-		Short: "Prints the extensions attached to the identity CA certificate",
-		RunE:  cmdCAExtensions,
+		Use:         "extensions",
+		Short:       "Prints the extensions attached to the identity CA certificate",
+		RunE:        cmdCAExtensions,
+		Annotations: map[string]string{"type": "setup"},
 	}
 	revokeCACmd = &cobra.Command{
-		Use:   "revoke",
-		Short: "Revoke the identity's CA certificate (creates backup)",
-		RunE:  cmdRevokeCA,
+		Use:         "revoke",
+		Short:       "Revoke the identity's CA certificate (creates backup)",
+		RunE:        cmdRevokeCA,
+		Annotations: map[string]string{"type": "setup"},
 	}
 
 	newCACfg struct {
@@ -63,13 +68,15 @@ var (
 
 func init() {
 	rootCmd.AddCommand(caCmd)
+
 	caCmd.AddCommand(newCACmd)
-	cfgstruct.Bind(newCACmd.Flags(), &newCACfg, cfgstruct.ConfDir(defaultConfDir))
 	caCmd.AddCommand(getIDCmd)
-	cfgstruct.Bind(getIDCmd.Flags(), &getIDCfg, cfgstruct.ConfDir(defaultConfDir))
 	caCmd.AddCommand(caExtCmd)
-	cfgstruct.Bind(caExtCmd.Flags(), &caExtCfg, cfgstruct.ConfDir(defaultConfDir))
 	caCmd.AddCommand(revokeCACmd)
+
+	cfgstruct.Bind(newCACmd.Flags(), &newCACfg, cfgstruct.ConfDir(defaultConfDir))
+	cfgstruct.Bind(getIDCmd.Flags(), &getIDCfg, cfgstruct.ConfDir(defaultConfDir))
+	cfgstruct.Bind(caExtCmd.Flags(), &caExtCfg, cfgstruct.ConfDir(defaultConfDir))
 	cfgstruct.Bind(revokeCACmd.Flags(), &revokeCACfg, cfgstruct.ConfDir(defaultConfDir))
 }
 

cmd/identity/identity.go

@@ -13,26 +13,30 @@ import (
 
 var (
 	idCmd = &cobra.Command{
-		Use:   "id",
-		Short: "Manage identities",
+		Use:         "id",
+		Short:       "Manage identities",
+		Annotations: map[string]string{"type": "setup"},
 	}
 
 	newIDCmd = &cobra.Command{
-		Use:   "new",
-		Short: "Creates a new identity from an existing certificate authority",
-		RunE:  cmdNewID,
+		Use:         "new",
+		Short:       "Creates a new identity from an existing certificate authority",
+		RunE:        cmdNewID,
+		Annotations: map[string]string{"type": "setup"},
 	}
 
 	leafExtCmd = &cobra.Command{
-		Use:   "extensions",
-		Short: "Prints the extensions attached to the identity leaf certificate",
-		RunE:  cmdLeafExtensions,
+		Use:         "extensions",
+		Short:       "Prints the extensions attached to the identity leaf certificate",
+		RunE:        cmdLeafExtensions,
+		Annotations: map[string]string{"type": "setup"},
 	}
 
 	revokeLeafCmd = &cobra.Command{
-		Use:   "revoke",
-		Short: "Revoke the identity's leaf certificate (creates backup)",
-		RunE:  cmdRevokeLeaf,
+		Use:         "revoke",
+		Short:       "Revoke the identity's leaf certificate (creates backup)",
+		RunE:        cmdRevokeLeaf,
+		Annotations: map[string]string{"type": "setup"},
 	}
 
 	newIDCfg struct {

cmd/identity/keys.go

@@ -26,9 +26,10 @@ var (
 		Short: "Manage keys",
 	}
 	keyGenerateCmd = &cobra.Command{
-		Use:   "generate",
-		Short: "generate lots of keys",
-		RunE:  cmdKeyGenerate,
+		Use:         "generate",
+		Short:       "generate lots of keys",
+		RunE:        cmdKeyGenerate,
+		Annotations: map[string]string{"type": "setup"},
 	}
 
 	keyCfg struct {

cmd/identity/main.go

@@ -7,10 +7,16 @@ import (
 	"crypto/x509/pkix"
 	"encoding/json"
 	"fmt"
+	"path/filepath"
 
 	"github.com/spf13/cobra"
+	"github.com/zeebo/errs"
+	"go.uber.org/zap"
 
 	"storj.io/storj/internal/fpath"
+	"storj.io/storj/pkg/certificates"
+	"storj.io/storj/pkg/cfgstruct"
+	"storj.io/storj/pkg/identity"
 	"storj.io/storj/pkg/peertls"
 	"storj.io/storj/pkg/process"
 )
@@ -21,13 +27,166 @@ var (
 		Short: "Identity management",
 	}
 
+	newServiceCmd = &cobra.Command{
+		Use:         "new <service>",
+		Short:       "Create a new full identity for a service",
+		Args:        cobra.ExactArgs(1),
+		RunE:        cmdNewService,
+		Annotations: map[string]string{"type": "setup"},
+	}
+
+	csrCmd = &cobra.Command{
+		Use:         "csr <service>",
+		Short:       "Send a certificate signing request for a service's CA certificate",
+		Args:        cobra.ExactArgs(1),
+		RunE:        cmdCSR,
+		Annotations: map[string]string{"type": "setup"},
+	}
+
+	//nolint
+	config struct {
+		Difficulty     uint64 `default:"15" help:"minimum difficulty for identity generation"`
+		Concurrency    uint   `default:"4" help:"number of concurrent workers for certificate authority generation"`
+		ParentCertPath string `help:"path to the parent authority's certificate chain"`
+		ParentKeyPath  string `help:"path to the parent authority's private key"`
+		Signer         certificates.CertClientConfig
+	}
+
+	confDir        string
 	defaultConfDir = fpath.ApplicationDir("storj", "identity")
 )
 
+func init() {
+	dirParam := cfgstruct.FindConfigDirParam()
+	if dirParam != "" {
+		defaultConfDir = dirParam
+	}
+
+	rootCmd.PersistentFlags().StringVar(&confDir, "config-dir", defaultConfDir, "main directory for storagenode configuration")
+	err := rootCmd.PersistentFlags().SetAnnotation("config-dir", "setup", []string{"true"})
+	if err != nil {
+		zap.S().Error("Failed to set 'setup' annotation for 'config-dir'")
+	}
+
+	rootCmd.AddCommand(newServiceCmd)
+	rootCmd.AddCommand(csrCmd)
+
+	cfgstruct.Bind(newServiceCmd.Flags(), &config, cfgstruct.ConfDir(defaultConfDir))
+	cfgstruct.Bind(csrCmd.Flags(), &config, cfgstruct.ConfDir(defaultConfDir))
+}
+
 func main() {
 	process.Exec(rootCmd)
 }
 
+func serviceDirectory(serviceName string) string {
+	return filepath.Join(confDir, serviceName)
+}
+
+func cmdNewService(cmd *cobra.Command, args []string) error {
+	serviceDir := serviceDirectory(args[0])
+
+	caCertPath := filepath.Join(serviceDir, "ca.cert")
+	caKeyPath := filepath.Join(serviceDir, "ca.key")
+	identCertPath := filepath.Join(serviceDir, "identity.cert")
+	identKeyPath := filepath.Join(serviceDir, "identity.key")
+
+	caConfig := identity.CASetupConfig{
+		CertPath:       caCertPath,
+		KeyPath:        caKeyPath,
+		Difficulty:     config.Difficulty,
+		Concurrency:    config.Concurrency,
+		ParentCertPath: config.ParentCertPath,
+		ParentKeyPath:  config.ParentKeyPath,
+	}
+
+	if caConfig.Status() != identity.NoCertNoKey {
+		return errs.New("CA certificate and/or key already exits, NOT overwriting!")
+	}
+
+	ca, caerr := caConfig.Create(process.Ctx(cmd))
+
+	identConfig := identity.SetupConfig{
+		CertPath: identCertPath,
+		KeyPath:  identKeyPath,
+	}
+
+	if identConfig.Status() != identity.NoCertNoKey {
+		return errs.New("Identity certificate and/or key already exits, NOT overwriting!")
+	}
+
+	_, iderr := identConfig.Create(ca)
+
+	return errs.Combine(caerr, iderr)
+}
+
+func cmdCSR(cmd *cobra.Command, args []string) error {
+	ctx := process.Ctx(cmd)
+
+	serviceDir := serviceDirectory(args[0])
+
+	caCertPath := filepath.Join(serviceDir, "ca.cert")
+	caKeyPath := filepath.Join(serviceDir, "ca.key")
+	caConfig := identity.FullCAConfig{
+		CertPath: caCertPath,
+		KeyPath:  caKeyPath,
+	}
+	identCertPath := filepath.Join(serviceDir, "identity.cert")
+	identKeyPath := filepath.Join(serviceDir, "identity.key")
+	identConfig := identity.Config{
+		CertPath: identCertPath,
+		KeyPath:  identKeyPath,
+	}
+
+	ca, err := caConfig.Load()
+	if err != nil {
+		return err
+	}
+	ident, err := identConfig.Load()
+	if err != nil {
+		return err
+	}
+
+	signedChainBytes, err := config.Signer.Sign(ctx, ident)
+	if err != nil {
+		return errs.New("error occurred while signing certificate: %s\n(identity files were still generated and saved, if you try again existing files will be loaded)", err)
+	}
+
+	signedChain, err := identity.ParseCertChain(signedChainBytes)
+	if err != nil {
+		return nil
+	}
+
+	err = caConfig.SaveBackup(ca)
+	if err != nil {
+		return err
+	}
+
+	ca.Cert = signedChain[0]
+	ca.RestChain = signedChain[1:]
+	err = identity.FullCAConfig{
+		CertPath: caConfig.CertPath,
+	}.Save(ca)
+	if err != nil {
+		return err
+	}
+
+	err = identConfig.SaveBackup(ident)
+	if err != nil {
+		return err
+	}
+
+	ident.RestChain = signedChain[1:]
+	ident.CA = ca.Cert
+	err = identity.Config{
+		CertPath: identConfig.CertPath,
+	}.Save(ident)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
 func printExtensions(cert []byte, exts []pkix.Extension) error {
 	hash, err := peertls.SHA256Hash(cert)
 	if err != nil {