Configurable default function readiness timeout (#2205)

hack/k8s/helm/nuclio/values.yaml

@@ -296,6 +296,7 @@ platform: {}
 #       "python:3.6": "myregistry"
 #     onbuildImageRegistries:
 #       "golang": "myregistry"
+#   functionReadinessTimeout: 120s
 
 # global is a stanza that is used if this is used as a subchart. Ignore otherwise
 global:

pkg/dashboard/resource/frontendspec.go

@@ -108,7 +108,7 @@ func (fesr *frontendSpecResource) getDefaultFunctionConfig() map[string]interfac
 	defaultFunctionSpec := functionconfig.Spec{
 		MinReplicas:             &one,
 		MaxReplicas:             &one,
-		ReadinessTimeoutSeconds: abstract.DefaultReadinessTimeoutSeconds,
+		ReadinessTimeoutSeconds: fesr.resolveFunctionReadinessTimeoutSeconds(),
 		TargetCPU:               abstract.DefaultTargetCPU,
 		Triggers: map[string]functionconfig.Trigger{
 
@@ -153,6 +153,14 @@ func (fesr *frontendSpecResource) resolveDefaultServiceType() v1.ServiceType {
 	return defaultServiceType
 }
 
+func (fesr *frontendSpecResource) resolveFunctionReadinessTimeoutSeconds() int {
+	readinessTimeoutSeconds := platformconfig.DefaultFunctionReadinessTimeoutSeconds
+	if dashboardServer, ok := fesr.resource.GetServer().(*dashboard.Server); ok {
+		return int(dashboardServer.GetPlatformConfiguration().GetDefaultFunctionReadinessTimeout().Seconds())
+	}
+	return readinessTimeoutSeconds
+}
+
 // register the resource
 var frontendSpecResourceInstance = &frontendSpecResource{
 	resource: newResource("api/frontend_spec", []restful.ResourceMethod{}),

pkg/nuctl/command/deploy.go

@@ -340,9 +340,12 @@ func (d *deployCommandeer) populateDeploymentDefaults() {
 	if d.functionConfig.Spec.RunRegistry == "" {
 		d.functionConfig.Spec.RunRegistry = os.Getenv("NUCTL_RUN_REGISTRY")
 	}
+
 	if d.functionConfig.Spec.ReadinessTimeoutSeconds == 0 {
-		d.functionConfig.Spec.ReadinessTimeoutSeconds = abstract.DefaultReadinessTimeoutSeconds
+		d.functionConfig.Spec.ReadinessTimeoutSeconds = int(
+			d.rootCommandeer.platformConfiguration.GetDefaultFunctionReadinessTimeout().Seconds())
 	}
+
 	if d.functionConfig.Spec.DataBindings == nil {
 		d.functionConfig.Spec.DataBindings = map[string]functionconfig.DataBinding{}
 	}

pkg/platform/abstract/platform.go

@@ -50,9 +50,8 @@ import (
 //
 
 const (
-	FunctionContainerHTTPPort      = 8080
-	DefaultReadinessTimeoutSeconds = 60
-	DefaultTargetCPU               = 75
+	FunctionContainerHTTPPort = 8080
+	DefaultTargetCPU          = 75
 )
 
 type Platform struct {

pkg/platform/kube/controller/nucliofunction.go

@@ -24,7 +24,6 @@ import (
 
 	"github.com/nuclio/nuclio/pkg/common"
 	"github.com/nuclio/nuclio/pkg/functionconfig"
-	"github.com/nuclio/nuclio/pkg/platform/abstract"
 	nuclioio "github.com/nuclio/nuclio/pkg/platform/kube/apis/nuclio.io/v1beta1"
 	"github.com/nuclio/nuclio/pkg/platform/kube/client"
 	"github.com/nuclio/nuclio/pkg/platform/kube/functionres"
@@ -150,8 +149,18 @@ func (fo *functionOperator) CreateOrUpdate(ctx context.Context, object runtime.O
 		})
 	}
 
+	// wait for up to the default readiness timeout or whatever was set in the spec
+	readinessTimeout := function.Spec.ReadinessTimeoutSeconds
+	if readinessTimeout == 0 {
+		readinessTimeout = int(fo.
+			controller.
+			GetPlatformConfiguration().
+			GetDefaultFunctionReadinessTimeout().Seconds())
+	}
+
 	fo.logger.DebugWith("Ensuring function resources",
 		"functionNamespace", function.Namespace,
+		"readinessTimeout", readinessTimeout,
 		"functionName", function.Name)
 
 	// ensure function resources (deployment, ingress, configmap, etc ...)
@@ -162,20 +171,19 @@ func (fo *functionOperator) CreateOrUpdate(ctx context.Context, object runtime.O
 			errors.Wrap(err, "Failed to create/update function"))
 	}
 
-	// wait for up to the default readiness timeout or whatever was set in the spec
-	readinessTimeout := function.Spec.ReadinessTimeoutSeconds
-	if readinessTimeout == 0 {
-		readinessTimeout = abstract.DefaultReadinessTimeoutSeconds
-	}
-
-	waitContext, cancel := context.WithDeadline(ctx, time.Now().Add(time.Duration(readinessTimeout)*time.Second))
-	defer cancel()
-
-	// wait until the function resources are ready
-	if err = fo.functionresClient.WaitAvailable(waitContext, function.Namespace, function.Name); err != nil {
-		return fo.setFunctionError(function,
-			functionconfig.FunctionStateUnhealthy,
-			errors.Wrap(err, "Failed to wait for function resources to be available"))
+	// readinessTimeout would be zero when
+	// - not defined on function spec
+	// - defined 0 on platform-config
+	if readinessTimeout != 0 {
+		waitContext, cancel := context.WithDeadline(ctx, time.Now().Add(time.Duration(readinessTimeout)*time.Second))
+		defer cancel()
+
+		// wait until the function resources are ready
+		if err = fo.functionresClient.WaitAvailable(waitContext, function.Namespace, function.Name); err != nil {
+			return fo.setFunctionError(function,
+				functionconfig.FunctionStateUnhealthy,
+				errors.Wrap(err, "Failed to wait for function resources to be available"))
+		}
 	}
 
 	waitingStates := []functionconfig.FunctionState{

pkg/platform/local/platform.go

@@ -1033,11 +1033,9 @@ func (p *Platform) waitForContainer(containerID string, timeout int) error {
 	p.Logger.InfoWith("Waiting for function to be ready",
 		"timeout", timeout)
 
-	var readinessTimeout time.Duration
-	if timeout != 0 {
-		readinessTimeout = time.Duration(timeout) * time.Second
-	} else {
-		readinessTimeout = abstract.DefaultReadinessTimeoutSeconds * time.Second
+	readinessTimeout := time.Duration(timeout) * time.Second
+	if timeout == 0 {
+		readinessTimeout = p.Config.GetDefaultFunctionReadinessTimeout()
 	}
 
 	if err := p.dockerClient.AwaitContainerHealth(containerID, &readinessTimeout); err != nil {

pkg/platformconfig/platformconfig.go

@@ -38,6 +38,7 @@ type Config struct {
 	AutoScale                AutoScale                    `json:"autoScale,omitempty"`
 	CronTriggerCreationMode  CronTriggerCreationMode      `json:"cronTriggerCreationMode,omitempty"`
 	FunctionAugmentedConfigs []LabelSelectorAndConfig     `json:"functionAugmentedConfigs,omitempty"`
+	FunctionReadinessTimeout *string                      `json:"functionReadinessTimeout,omitempty"`
 	IngressConfig            IngressConfig                `json:"ingressConfig,omitempty"`
 	Kube                     PlatformKubeConfig           `json:"kube,omitempty"`
 	Local                    PlatformLocalConfig          `json:"local,omitempty"`
@@ -46,6 +47,9 @@ type Config struct {
 	ProjectsLeader           *ProjectsLeader              `json:"projectsLeader,omitempty"`
 
 	ContainerBuilderConfiguration *containerimagebuilderpusher.ContainerBuilderConfiguration `json:"containerBuilderConfiguration,omitempty"`
+
+	// stores the encoded FunctionReadinessTimeout as time.Duration
+	functionReadinessTimeout *time.Duration
 }
 
 func NewPlatformConfig(configurationPath string) (*Config, error) {
@@ -81,6 +85,17 @@ func NewPlatformConfig(configurationPath string) (*Config, error) {
 		config.Kube.DefaultServiceType = DefaultServiceType
 	}
 
+	if config.FunctionReadinessTimeout == nil {
+		encodedReadinessTimeoutDuration := (DefaultFunctionReadinessTimeoutSeconds * time.Second).String()
+		config.FunctionReadinessTimeout = &encodedReadinessTimeoutDuration
+	}
+
+	functionReadinessTimeout, err := time.ParseDuration(*config.FunctionReadinessTimeout)
+	if err != nil {
+		return nil, errors.Wrap(err, "Failed to parse function readiness timeout")
+	}
+	config.functionReadinessTimeout = &functionReadinessTimeout
+
 	return config, nil
 }
 
@@ -117,6 +132,17 @@ func (config *Config) GetFunctionLoggerSinks(functionConfig *functionconfig.Conf
 	return config.getLoggerSinksWithLevel(loggerSinkBindings)
 }
 
+func (config *Config) GetDefaultFunctionReadinessTimeout() time.Duration {
+
+	// provided by the platform-config
+	if config.functionReadinessTimeout != nil {
+		return *config.functionReadinessTimeout
+	}
+
+	// no configuration were explicitly given, return default
+	return DefaultFunctionReadinessTimeoutSeconds * time.Second
+}
+
 func (config *Config) GetSystemMetricSinks() (map[string]MetricSink, error) {
 	return config.getMetricSinks(config.Metrics.System)
 }

pkg/platformconfig/platformconfig_test.go

@@ -24,8 +24,9 @@ import (
 	"testing"
 
 	"github.com/nuclio/nuclio/pkg/functionconfig"
-	"github.com/nuclio/nuclio/test/compare"
 
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
 	"github.com/nuclio/logger"
 	"github.com/nuclio/zap"
 	"github.com/stretchr/testify/suite"
@@ -46,6 +47,7 @@ func (suite *PlatformConfigTestSuite) SetupTest() {
 
 func (suite *PlatformConfigTestSuite) TestReadConfiguration() {
 	configurationContents := `
+functionReadinessTimeout: 10s
 webAdmin:
   enabled: true
   listenAddress: :8081
@@ -89,6 +91,9 @@ metrics:
 	expectedConfiguration.WebAdmin.Enabled = &trueValue
 	expectedConfiguration.WebAdmin.ListenAddress = ":8081"
 
+	tenSecondsStr := "10s"
+	expectedConfiguration.FunctionReadinessTimeout = &tenSecondsStr
+
 	// logger
 	expectedConfiguration.Logger.System = append(expectedConfiguration.Logger.System, LoggerSinkBinding{
 		Level: "debug",
@@ -144,7 +149,7 @@ metrics:
 	err := suite.reader.Read(bytes.NewBufferString(configurationContents), "yaml", &readConfiguration)
 	suite.Require().NoError(err)
 
-	suite.Require().True(compare.NoOrder(expectedConfiguration, readConfiguration))
+	suite.Require().Empty(cmp.Diff(&expectedConfiguration, &readConfiguration, cmpopts.IgnoreUnexported(Config{})))
 }
 
 func (suite *PlatformConfigTestSuite) TestGetSystemLoggerSinks() {
@@ -191,7 +196,7 @@ logger:
 		},
 	}
 
-	suite.Require().True(compare.NoOrder(expectedSystemLoggerSinks, systemLoggerSinks))
+	suite.Require().Empty(cmp.Diff(&expectedSystemLoggerSinks, &systemLoggerSinks, cmpopts.IgnoreUnexported(Config{})))
 }
 
 func (suite *PlatformConfigTestSuite) TestGetSystemLoggerSinksInvalidSink() {
@@ -258,7 +263,7 @@ logger:
 		},
 	}
 
-	suite.Require().True(compare.NoOrder(expectedFunctionLoggerSinks, functionLoggerSinks))
+	suite.Require().Empty(cmp.Diff(&expectedFunctionLoggerSinks, &functionLoggerSinks, cmpopts.IgnoreUnexported(Config{})))
 }
 
 func (suite *PlatformConfigTestSuite) TestGetFunctionLoggerSinksWithFunctionConfig() {
@@ -315,7 +320,7 @@ logger:
 		},
 	}
 
-	suite.Require().True(compare.NoOrder(expectedFunctionLoggerSinks, functionLoggerSinks))
+	suite.Require().Empty(cmp.Diff(&expectedFunctionLoggerSinks, &functionLoggerSinks, cmpopts.IgnoreUnexported(Config{})))
 }
 
 func (suite *PlatformConfigTestSuite) TestGetFunctionLoggerSinksInvalidSink() {
@@ -381,7 +386,7 @@ metrics:
 		},
 	}
 
-	suite.Require().True(compare.NoOrder(expectedSystemMetricSinks, systemMetricSinks))
+	suite.Require().Empty(cmp.Diff(&expectedSystemMetricSinks, &systemMetricSinks, cmpopts.IgnoreUnexported(Config{})))
 }
 
 func (suite *PlatformConfigTestSuite) TestGetSystemMetricSinksInvalidSink() {
@@ -443,7 +448,7 @@ metrics:
 		},
 	}
 
-	suite.Require().True(compare.NoOrder(expectedFunctionMetricSinks, functionMetricSinks))
+	suite.Require().Empty(cmp.Diff(&expectedFunctionMetricSinks, &functionMetricSinks, cmpopts.IgnoreUnexported(Config{})))
 }
 
 func (suite *PlatformConfigTestSuite) TestFunctionAugmentedConfigs() {
@@ -499,8 +504,7 @@ functionAugmentedConfigs:
 		},
 	}
 
-	suite.Require().True(compare.NoOrder(expectedFunctionAugmentedConfigs,
-		readConfiguration.FunctionAugmentedConfigs))
+	suite.Require().Empty(cmp.Diff(&expectedFunctionAugmentedConfigs, &readConfiguration.FunctionAugmentedConfigs, cmpopts.IgnoreUnexported(Config{})))
 }
 
 func TestRegistryTestSuite(t *testing.T) {

pkg/platformconfig/types.go

@@ -26,6 +26,8 @@ import (
 	machinarymetav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
+const DefaultFunctionReadinessTimeoutSeconds = 60
+
 type LoggerSink struct {
 	Kind       string                 `json:"kind,omitempty"`
 	URL        string                 `json:"url,omitempty"`