Merge pull request #111 from kaleido-io/fix-244

Add retry for docker pull to deal with occasional errors from ghcr

cmd/pull.go

@@ -0,0 +1,74 @@
+// Copyright © 2021 Kaleido, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+import (
+	"errors"
+	"time"
+
+	"github.com/briandowns/spinner"
+	"github.com/hyperledger/firefly-cli/internal/log"
+	"github.com/hyperledger/firefly-cli/internal/stacks"
+	"github.com/spf13/cobra"
+)
+
+var pullOptions stacks.PullOptions
+
+var pullCmd = &cobra.Command{
+	Use:   "pull <stack_name>",
+	Short: "Pull a stack",
+	Long: `Pull a stack
+
+Pull the images for a stack .
+`,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		var spin *spinner.Spinner
+		if fancyFeatures && !verbose {
+			spin = spinner.New(spinner.CharSets[11], 100*time.Millisecond)
+			spin.FinalMSG = "done"
+			logger = &log.SpinnerLogger{
+				Spinner: spin,
+			}
+		}
+
+		stackManager := stacks.NewStackManager(logger)
+		if len(args) == 0 {
+			return errors.New("no stack specified")
+		}
+		stackName := args[0]
+
+		if err := stackManager.LoadStack(stackName); err != nil {
+			return err
+		}
+		if spin != nil {
+			spin.Start()
+		}
+		if err := stackManager.PullStack(verbose, &pullOptions); err != nil {
+			return err
+		}
+		if spin != nil {
+			spin.Stop()
+		}
+		return nil
+	},
+}
+
+func init() {
+	pullCmd.Flags().IntVarP(&pullOptions.Retries, "retries", "r", 0, "Retry attempts to perform on image pull failure")
+
+	rootCmd.AddCommand(pullCmd)
+}

cmd/start.go

@@ -65,7 +65,7 @@ This command will start a stack and run it in the background.
 		if spin != nil {
 			spin.Start()
 		}
-		if err := stackManager.StartStack(fancyFeatures, verbose, &startOptions); err != nil {
+		if err := stackManager.StartStack(verbose, &startOptions); err != nil {
 			return err
 		}
 		if spin != nil {
@@ -81,7 +81,6 @@ This command will start a stack and run it in the background.
 }
 
 func init() {
-	startCmd.Flags().BoolVarP(&startOptions.NoPull, "no-pull", "n", false, "Do not pull latest images when starting")
 	startCmd.Flags().BoolVarP(&startOptions.NoRollback, "no-rollback", "b", false, "Do not automatically rollback changes if first time setup fails")
 
 	rootCmd.AddCommand(startCmd)

internal/core/manifest.go

@@ -67,6 +67,7 @@ func ReadManifestFile(p string) (*types.VersionManifest, error) {
 	if manifest.FireFly == nil {
 		manifest.FireFly = &types.ManifestEntry{
 			Image: "hyperledger/firefly",
+			Local: true,
 		}
 	}
 	return manifest, err

internal/docker/docker.go

@@ -42,6 +42,21 @@ func RemoveVolume(volumeName string, verbose bool) error {
 	return RunDockerCommand(".", verbose, verbose, "volume", "remove", volumeName)
 }
 
+func RunDockerCommandRetry(workingDir string, showCommand bool, pipeStdout bool, retries int, command ...string) error {
+	attempt := 0
+	for {
+		err := RunDockerCommand(workingDir, showCommand, pipeStdout, command...)
+		if err != nil && attempt < retries {
+			attempt++
+			continue
+		} else if err != nil {
+			return err
+		}
+		break
+	}
+	return nil
+}
+
 func RunDockerCommand(workingDir string, showCommand bool, pipeStdout bool, command ...string) error {
 	dockerCmd := exec.Command("docker", command...)
 	dockerCmd.Dir = workingDir

internal/stacks/stack_manager.go

@@ -57,8 +57,11 @@ type StackManager struct {
 	tokensProvider     tokens.ITokensProvider
 }
 
+type PullOptions struct {
+	Retries int
+}
+
 type StartOptions struct {
-	NoPull     bool
 	NoRollback bool
 }
 
@@ -354,7 +357,7 @@ func createMember(id string, index int, options *InitOptions, external bool) *ty
 	}
 }
 
-func (s *StackManager) StartStack(fancyFeatures bool, verbose bool, options *StartOptions) error {
+func (s *StackManager) StartStack(verbose bool, options *StartOptions) error {
 	fmt.Printf("starting FireFly stack '%s'... ", s.Stack.Name)
 	// Check to make sure all of our ports are available
 	if err := s.checkPortsAvailable(); err != nil {
@@ -391,6 +394,24 @@ func (s *StackManager) StartStack(fancyFeatures bool, verbose bool, options *Sta
 	}
 }
 
+func (s *StackManager) PullStack(verbose bool, options *PullOptions) error {
+	workingDir := filepath.Join(constants.StacksDir, s.Stack.Name)
+	for _, entry := range s.Stack.VersionManifest.Entries() {
+		if entry.Local {
+			continue
+		}
+		fullImage := fmt.Sprintf("%s@sha256:%s", entry.Image, entry.SHA)
+		if entry.SHA == "" {
+			fullImage = fmt.Sprintf("%s:%s", entry.Image, entry.Tag)
+		}
+		s.Log.Info(fmt.Sprintf("pulling '%s", fullImage))
+		if err := docker.RunDockerCommandRetry(workingDir, verbose, verbose, options.Retries, "pull", fullImage); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func (s *StackManager) removeVolumes(verbose bool) {
 	var volumes []string
 	for _, service := range s.blockchainProvider.GetDockerServiceDefinitions() {
@@ -537,13 +558,6 @@ func (s *StackManager) runFirstTimeSetup(verbose bool, options *StartOptions) er
 		}
 	}
 
-	if !options.NoPull {
-		s.Log.Info("pulling latest versions")
-		if err := docker.RunDockerComposeCommand(workingDir, verbose, verbose, "pull"); err != nil {
-			return err
-		}
-	}
-
 	if err := s.runStartupSequence(workingDir, verbose, true); err != nil {
 		return err
 	}

pkg/types/manifest.go

@@ -30,8 +30,22 @@ type VersionManifest struct {
 	Tokens       *ManifestEntry `json:"tokens-erc1155"`
 }
 
+func (m *VersionManifest) Entries() []*ManifestEntry {
+	if m == nil {
+		return []*ManifestEntry{}
+	}
+	return []*ManifestEntry{
+		m.FireFly,
+		m.Ethconnect,
+		m.Fabconnect,
+		m.DataExchange,
+		m.Tokens,
+	}
+}
+
 type ManifestEntry struct {
 	Image string `json:"image,omitempty"`
+	Local bool   `json:"local,omitempty"`
 	Tag   string `json:"tag,omitempty"`
 	SHA   string `json:"sha,omitempty"`
 }