Option to increase the timeouts (#885)

* apply custom timeout to different polling events
* add timeout override

Signed-off-by: Ramiro Berrelleza <rberrelleza@gmail.com>

cmd/down.go

@@ -14,6 +14,8 @@
 package cmd
 
 import (
+	"context"
+
 	"github.com/okteto/okteto/cmd/utils"
 	"github.com/okteto/okteto/pkg/analytics"
 	"github.com/okteto/okteto/pkg/cmd/down"
@@ -38,7 +40,7 @@ func Down() *cobra.Command {
 		Short: "Deactivates your development environment",
 		RunE: func(cmd *cobra.Command, args []string) error {
 			log.Info("starting down command")
-
+			ctx := context.Background()
 			dev, err := utils.LoadDev(devPath)
 			if err != nil {
 				return err
@@ -57,7 +59,7 @@ func Down() *cobra.Command {
 			log.Information("Run 'okteto push' to deploy your code changes to the cluster")
 
 			if rm {
-				if err := removeVolume(dev); err != nil {
+				if err := removeVolume(ctx, dev); err != nil {
 					analytics.TrackDownVolumes(false)
 					return err
 				}
@@ -113,7 +115,7 @@ func runDown(dev *model.Dev) error {
 	return nil
 }
 
-func removeVolume(dev *model.Dev) error {
+func removeVolume(ctx context.Context, dev *model.Dev) error {
 	spinner := utils.NewSpinner("Removing persistent volume...")
 	spinner.Start()
 	defer spinner.Stop()
@@ -126,5 +128,5 @@ func removeVolume(dev *model.Dev) error {
 		dev.Namespace = namespace
 	}
 
-	return volumes.Destroy(dev, client)
+	return volumes.Destroy(ctx, dev, client)
 }

cmd/up.go

@@ -537,6 +537,7 @@ func (up *UpContext) devMode(d *appsv1.Deployment, create bool) error {
 				return err
 			}
 		}
+
 		if trList[name].Deployment.Annotations[okLabels.DeploymentAnnotation] == "" {
 			continue
 		}

main.go

@@ -51,7 +51,7 @@ func init() {
 }
 
 func main() {
-	log.Init(logrus.WarnLevel)
+	log.Init(logrus.WarnLevel, config.GetOktetoHome(), config.VersionString)
 	log.Info("start")
 	var logLevel string
 

pkg/config/config.go

@@ -14,11 +14,14 @@
 package config
 
 import (
-	"log"
 	"os"
 	"path/filepath"
 	"runtime"
 	"strings"
+	"sync"
+	"time"
+
+	"github.com/okteto/okteto/pkg/log"
 )
 
 const (
@@ -28,6 +31,9 @@ const (
 // VersionString the version of the cli
 var VersionString string
 
+var timeout time.Duration
+var tOnce sync.Once
+
 // Config holds all the configuration values.
 type Config struct {
 	// HomePath is the path of the base folder for all the Okteto files
@@ -120,3 +126,25 @@ func splitKubeConfigEnv(value string) string {
 	}
 	return strings.Split(value, ":")[0]
 }
+
+// GetTimeout returns the per-action timeout
+func GetTimeout() time.Duration {
+	tOnce.Do(func() {
+		timeout = (30 * time.Second)
+		t, ok := os.LookupEnv("OKTETO_TIMEOUT")
+		if !ok {
+			return
+		}
+
+		parsed, err := time.ParseDuration(t)
+		if err != nil {
+			log.Infof("'%s' is not a valid duration, ignoring", t)
+			return
+		}
+
+		log.Infof("OKTETO_TIMEOUT applied: '%s'", parsed.String())
+		timeout = parsed
+	})
+
+	return timeout
+}

pkg/k8s/deployments/crud.go

@@ -20,6 +20,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/okteto/okteto/pkg/config"
 	"github.com/okteto/okteto/pkg/errors"
 	okLabels "github.com/okteto/okteto/pkg/k8s/labels"
 	"github.com/okteto/okteto/pkg/log"
@@ -148,30 +149,34 @@ func Deploy(d *appsv1.Deployment, forceCreate bool, client *kubernetes.Clientset
 
 //UpdateOktetoRevision updates the okteto version annotation
 func UpdateOktetoRevision(ctx context.Context, d *appsv1.Deployment, client *kubernetes.Clientset) error {
-	tries := 0
 	ticker := time.NewTicker(200 * time.Millisecond)
-	for tries < maxRetriesUpdateRevision {
+	timeout := time.Now().Add(2 * config.GetTimeout()) // 60 seconds
+
+	for i := 0; ; i++ {
 		updated, err := client.AppsV1().Deployments(d.Namespace).Get(d.Name, metav1.GetOptions{})
 		if err != nil {
 			log.Debugf("error while retrieving deployment %s/%s: %s", d.Namespace, d.Name, err)
 			return err
 		}
+
 		revision := updated.Annotations[revisionAnnotation]
 		if revision != "" {
 			d.Annotations[okLabels.RevisionAnnotation] = revision
 			return update(d, client)
 		}
 
+		if time.Now().After(timeout) {
+			return fmt.Errorf("kubernetes is taking too long to update the '%s' annotation of the deployment '%s'. Please check for errors and try again", revisionAnnotation, d.Name)
+		}
+
 		select {
 		case <-ticker.C:
-			tries++
 			continue
 		case <-ctx.Done():
 			log.Debug("cancelling call to update okteto revision")
 			return ctx.Err()
 		}
 	}
-	return fmt.Errorf("kubernetes is taking too long to update the '%s' annotation of the deployment '%s'. Please check for errors and try again", revisionAnnotation, d.Name)
 }
 
 //TranslateDevMode translates the deployment manifests to put them in dev mode

pkg/k8s/pods/pod.go

@@ -22,6 +22,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/okteto/okteto/pkg/config"
 	"github.com/okteto/okteto/pkg/errors"
 	"github.com/okteto/okteto/pkg/k8s/deployments"
 	okLabels "github.com/okteto/okteto/pkg/k8s/labels"
@@ -86,26 +87,32 @@ func ListBySelector(namespace string, selector map[string]string, c kubernetes.I
 
 // GetDevPodInLoop returns the dev pod for a deployment and loops until it success
 func GetDevPodInLoop(ctx context.Context, dev *model.Dev, c *kubernetes.Clientset, waitUntilDeployed bool) (*apiv1.Pod, error) {
-	tries := 0
 	ticker := time.NewTicker(200 * time.Millisecond)
-	for tries < maxRetriesPodRunning {
+	to := 2 * config.GetTimeout() // 60 seconds
+	timeout := time.Now().Add(to)
+
+	for i := 0; ; i++ {
 		pod, err := GetDevPod(ctx, dev, c, waitUntilDeployed)
 		if err != nil {
 			return nil, err
 		}
 		if pod != nil {
 			return pod, nil
 		}
+
+		if time.Now().After(timeout) {
+			return nil, fmt.Errorf("kubernetes is taking too long to create the pod of your development environment. Please check for errors and try again")
+		}
+
 		select {
 		case <-ticker.C:
-			tries++
 			continue
 		case <-ctx.Done():
 			log.Debug("cancelling call to get dev pod")
 			return nil, ctx.Err()
 		}
 	}
-	return nil, fmt.Errorf("kubernetes is taking too long to create the pod of your development environment. Please check for errors and try again")
+
 }
 
 // GetDevPod returns the dev pod for a deployment
@@ -181,9 +188,10 @@ func MonitorDevPod(ctx context.Context, dev *model.Dev, pod *apiv1.Pod, c *kuber
 		case event := <-watchPodEvents.ResultChan():
 			e, ok := event.Object.(*v1.Event)
 			if !ok {
-				log.Errorf("type error getting event: %s", event)
+				log.Infof("unknown event type: %s", event)
 				continue
 			}
+
 			log.Infof("pod %s event: %s", pod.Name, e.Message)
 			switch e.Reason {
 			case "Failed", "FailedScheduling", "FailedCreatePodSandBox", "ErrImageNeverPull", "InspectFailed", "FailedCreatePodContainer":

pkg/k8s/volumes/crud.go

@@ -19,6 +19,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/okteto/okteto/pkg/config"
 	"github.com/okteto/okteto/pkg/log"
 	"github.com/okteto/okteto/pkg/model"
 
@@ -29,10 +30,6 @@ import (
 	"k8s.io/client-go/kubernetes"
 )
 
-const (
-	maxRetries = 90
-)
-
 //Create deploys the volume claim for a given dev environment
 func Create(ctx context.Context, dev *model.Dev, c *kubernetes.Clientset) error {
 	vClient := c.CoreV1().PersistentVolumeClaims(dev.Namespace)
@@ -85,12 +82,15 @@ func checkPVCValues(pvc *apiv1.PersistentVolumeClaim, dev *model.Dev) error {
 }
 
 //Destroy destroys the volume claim for a given dev environment
-func Destroy(dev *model.Dev, c *kubernetes.Clientset) error {
+func Destroy(ctx context.Context, dev *model.Dev, c *kubernetes.Clientset) error {
 	vClient := c.CoreV1().PersistentVolumeClaims(dev.Namespace)
 	log.Infof("destroying volume claim '%s'...", dev.GetVolumeName())
 
 	ticker := time.NewTicker(1 * time.Second)
-	for i := 0; i < maxRetries; i++ {
+	to := 3 * config.GetTimeout() // 90 seconds
+	timeout := time.Now().Add(to)
+
+	for i := 0; ; i++ {
 		err := vClient.Delete(dev.GetVolumeName(), &metav1.DeleteOptions{})
 		if err != nil {
 			if strings.Contains(err.Error(), "not found") {
@@ -101,15 +101,27 @@ func Destroy(dev *model.Dev, c *kubernetes.Clientset) error {
 			return fmt.Errorf("error getting kubernetes volume claim: %s", err)
 		}
 
-		<-ticker.C
+		if time.Now().After(timeout) {
+			if err := checkIfAttached(dev, c); err != nil {
+				return err
+			}
+
+			return fmt.Errorf("volume claim '%s' wasn't destroyed after %s", dev.GetVolumeName(), to.String())
+		}
+
 		if i%10 == 5 {
 			log.Infof("waiting for volume claim '%s' to be destroyed...", dev.GetVolumeName())
 		}
+
+		select {
+		case <-ticker.C:
+			continue
+		case <-ctx.Done():
+			log.Debug("cancelling call to update okteto revision")
+			return ctx.Err()
+		}
 	}
-	if err := checkIfAttached(dev, c); err != nil {
-		return err
-	}
-	return fmt.Errorf("volume claim '%s' wasn't destroyed after 120s", dev.GetVolumeName())
+
 }
 
 func checkIfAttached(dev *model.Dev, c *kubernetes.Clientset) error {

pkg/log/log.go

@@ -22,7 +22,6 @@ import (
 
 	"github.com/fatih/color"
 	"github.com/google/uuid"
-	"github.com/okteto/okteto/pkg/config"
 	"github.com/sirupsen/logrus"
 	lumberjack "gopkg.in/natefinch/lumberjack.v2"
 )
@@ -59,7 +58,7 @@ func init() {
 }
 
 // Init configures the logger for the package to use.
-func Init(level logrus.Level) {
+func Init(level logrus.Level, dir, version string) {
 	log.out.SetOutput(os.Stdout)
 	log.out.SetLevel(level)
 
@@ -69,13 +68,13 @@ func Init(level logrus.Level) {
 		FullTimestamp: true,
 	})
 
-	logPath := filepath.Join(config.GetOktetoHome(), "okteto.log")
+	logPath := filepath.Join(dir, "okteto.log")
 	rolling := getRollingLog(logPath)
 	fileLogger.SetOutput(rolling)
 	fileLogger.SetLevel(logrus.DebugLevel)
 
 	actionID := uuid.New().String()
-	log.file = fileLogger.WithFields(logrus.Fields{"action": actionID, "version": config.VersionString})
+	log.file = fileLogger.WithFields(logrus.Fields{"action": actionID, "version": version})
 }
 
 func getRollingLog(path string) io.Writer {
@@ -144,6 +143,15 @@ func Errorf(format string, args ...interface{}) {
 	}
 }
 
+// Fatalf writes a error-level log with a format
+func Fatalf(format string, args ...interface{}) {
+	if log.file != nil {
+		log.file.Errorf(format, args...)
+	}
+
+	log.out.Fatalf(format, args...)
+}
+
 // Yellow writes a line in yellow
 func Yellow(format string, args ...interface{}) {
 	log.out.Infof(format, args...)