Add connection retry when starting Web and Runner

go.mod

@@ -3,6 +3,7 @@ module github.com/trento-project/trento
 go 1.16
 
 require (
+	github.com/avast/retry-go/v4 v4.0.2
 	github.com/gin-contrib/sessions v0.0.4
 	github.com/gin-gonic/gin v1.7.7
 	github.com/gomarkdown/markdown v0.0.0-20210514010506-3b9f47219fe7

go.sum

@@ -82,6 +82,8 @@ github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A=
+github.com/avast/retry-go/v4 v4.0.2 h1:tNRqagpXY84w94xeiVgzQ3G9TDI1WemRUxLPoJ5OTXc=
+github.com/avast/retry-go/v4 v4.0.2/go.mod h1:HqmLvS2VLdStPCGDFjSuZ9pzlTqVRldCI4w2dO4m1Ms=
 github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU=
 github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g=
@@ -578,6 +580,7 @@ github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCko
 github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac=
 github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc=
 github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
+github.com/pierrre/gotestcover v0.0.0-20160517101806-924dca7d15f0/go.mod h1:4xpMLz7RBWyB+ElzHu8Llua96TRCB3YwX+l5EP1wmHk=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
@@ -610,6 +613,7 @@ github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsT
 github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
 github.com/quasoft/memstore v0.0.0-20191010062613-2bce066d2b0b/go.mod h1:wTPjTepVu7uJBYgZ0SdWHQlIas582j6cn2jgk4DDdlg=
 github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
+github.com/robertkrimen/godocdown v0.0.0-20130622164427-0bfa04905481/go.mod h1:C9WhFzY47SzYBIvzFqSvHIR6ROgDo4TtdTuRaOMjF/s=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
@@ -971,6 +975,7 @@ golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211107104306-e0b2ad06fe42/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

runner/runner.go

@@ -10,10 +10,11 @@ import (
 	"sync"
 	"time"
 
+	retryGo "github.com/avast/retry-go/v4"
 	log "github.com/sirupsen/logrus"
-	"github.com/trento-project/trento/internal"
 
 	"github.com/trento-project/trento/api"
+	"github.com/trento-project/trento/internal"
 )
 
 //go:embed ansible
@@ -59,9 +60,26 @@ func (c *Runner) Start() error {
 		return err
 	}
 
-	trentoApi := api.NewTrentoApiService(c.config.ApiHost, c.config.ApiPort)
-	if !trentoApi.IsWebServerUp() {
-		return fmt.Errorf("Trento server api not available")
+	var trentoApi api.TrentoApiService
+	err := retryGo.Do(
+		func() error {
+			trentoApi = api.NewTrentoApiService(c.config.ApiHost, c.config.ApiPort)
+			if !trentoApi.IsWebServerUp() {
+				return fmt.Errorf("Trento server api not available")
+			}
+			return nil
+		},
+		retryGo.OnRetry(func(n uint, err error) {
+			log.Error(err)
+		}),
+		retryGo.Delay(2*time.Second),
+		retryGo.MaxJitter(3*time.Second),
+		retryGo.Attempts(8),
+		retryGo.LastErrorOnly(true),
+		retryGo.Context(c.ctx),
+	)
+	if err != nil {
+		return err
 	}
 
 	c.trentoApi = trentoApi

web/app.go

@@ -19,14 +19,15 @@ import (
 	"golang.org/x/sync/errgroup"
 	"gorm.io/gorm"
 
-	"github.com/trento-project/trento/internal/db"
+	trentoDB "github.com/trento-project/trento/internal/db"
 	"github.com/trento-project/trento/version"
 	"github.com/trento-project/trento/web/datapipeline"
 	"github.com/trento-project/trento/web/entities"
 	"github.com/trento-project/trento/web/models"
 	"github.com/trento-project/trento/web/services"
 	"github.com/trento-project/trento/web/telemetry"
 
+	retryGo "github.com/avast/retry-go/v4"
 	swaggerFiles "github.com/swaggo/files"
 	ginSwagger "github.com/swaggo/gin-swagger"
 	_ "github.com/trento-project/trento/docs/api" // docs is generated by Swag CLI, you have to import it.
@@ -59,7 +60,7 @@ type Config struct {
 	Cert          string
 	Key           string
 	CA            string
-	DBConfig      *db.Config
+	DBConfig      *trentoDB.Config
 }
 type Dependencies struct {
 	webEngine               *gin.Engine
@@ -87,7 +88,21 @@ func DefaultDependencies(config *Config) Dependencies {
 
 	gin.SetMode(mode)
 
-	db, err := db.InitDB(config.DBConfig)
+	var db *gorm.DB
+	err := retryGo.Do(
+		func() error {
+			var err error
+			db, err = trentoDB.InitDB(config.DBConfig)
+			if err != nil {
+				return err
+			}
+			return nil
+		},
+		retryGo.Delay(1*time.Second),
+		retryGo.MaxJitter(2*time.Second),
+		retryGo.Attempts(8),
+		retryGo.LastErrorOnly(true),
+	)
 	if err != nil {
 		log.Fatalf("failed to connect database: %s", err)
 	}