Build - make backend-plugin readiness a condition of JetBrains IDEs r…

…eadiness (#19710)

* Make backend-plugin readiness a condition of IDE readiness

* Add compatibility to `2022.3.3`

* fixup

* Observe ide readiness as Inf after 11 seconds

* shutdown after 10 minutes

* fix hot-deploy.sh

* Add shutdown reason

* address feedback

* Add Feature Flags

* fixup

* fix

* Add one line log

* address feedback

components/ide/jetbrains/backend-plugin/src/main/kotlin/io/gitpod/jetbrains/remote/GitpodCLIService.kt

@@ -50,6 +50,9 @@ class GitpodCLIService : RestService() {
         /**
          * prod: curl http://localhost:63342/api/gitpod/cli?op=metrics
          * dev:  curl http://localhost:63343/api/gitpod/cli?op=metrics
+         *
+         * We will use this endpoint in JetBrains launcher to check if backend-plugin is ready.
+         * Please make sure this operation:metrics to respond 200
          */
         if (operation == "metrics") {
             val out = BufferExposingByteArrayOutputStream()

components/ide/jetbrains/launcher/go.mod

@@ -10,8 +10,10 @@ require (
 
 require (
 	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/blang/semver v3.5.1+incompatible // indirect
 	github.com/cenkalti/backoff/v4 v4.1.3 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/configcat/go-sdk/v7 v7.6.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/gitpod-io/gitpod/components/scrubber v0.0.0-00010101000000-000000000000 // indirect
 	github.com/golang/mock v1.6.0 // indirect

components/ide/jetbrains/launcher/main.go

@@ -35,6 +35,7 @@ import (
 	"google.golang.org/grpc/credentials/insecure"
 	yaml "gopkg.in/yaml.v2"
 
+	"github.com/gitpod-io/gitpod/common-go/experiments"
 	"github.com/gitpod-io/gitpod/common-go/log"
 	"github.com/gitpod-io/gitpod/common-go/util"
 	gitpod "github.com/gitpod-io/gitpod/gitpod-protocol"
@@ -64,6 +65,8 @@ type LaunchContext struct {
 	info           *ProductInfo
 	backendVersion *version.Version
 	wsInfo         *supervisor.WorkspaceInfoResponse
+	exps           experiments.Client
+	gitpodHostname string
 
 	vmOptionsFile          string
 	platformPropertiesFile string
@@ -91,7 +94,8 @@ func main() {
 		return
 	}
 
-	log.Init(ServiceName, Version, true, false)
+	debugEnabled := os.Getenv("SUPERVISOR_DEBUG_ENABLE") == "true"
+	log.Init(ServiceName, Version, true, debugEnabled)
 	log.Info(ServiceName + ": " + Version)
 	startTime := time.Now()
 
@@ -149,6 +153,16 @@ func main() {
 		return
 	}
 
+	var exps experiments.Client
+	var gitpodHostname string
+	if gitpodUrl, err := url.Parse(wsInfo.GitpodHost); err == nil {
+		gitpodHost := gitpodUrl.Hostname()
+		gitpodHostname = gitpodHost
+		exps = experiments.NewClient(experiments.WithGitpodProxy(gitpodHost))
+	} else {
+		log.WithField("gitpodHost", wsInfo.GitpodHost).WithError(err).Error("failed to parse url")
+	}
+
 	launchCtx := &LaunchContext{
 		startTime: startTime,
 
@@ -163,6 +177,8 @@ func main() {
 		info:           info,
 		backendVersion: backendVersion,
 		wsInfo:         wsInfo,
+		exps:           exps,
+		gitpodHostname: gitpodHostname,
 	}
 
 	if launchCtx.warmup {
@@ -272,6 +288,10 @@ func serve(launchCtx *LaunchContext) {
 		if backendPort == "" {
 			backendPort = defaultBackendPort
 		}
+		if err := isBackendPluginReady(r.Context(), backendPort, launchCtx.exps, launchCtx.gitpodHostname); err != nil {
+			http.Error(w, err.Error(), http.StatusServiceUnavailable)
+			return
+		}
 		gatewayLink, err := resolveGatewayLink(backendPort, launchCtx.wsInfo)
 		if err != nil {
 			log.WithError(err).Error("cannot resolve gateway link")
@@ -293,6 +313,37 @@ func serve(launchCtx *LaunchContext) {
 	}
 }
 
+// isBackendPluginReady checks if the backend plugin is ready via backend plugin CLI GitpodCLIService.kt
+func isBackendPluginReady(ctx context.Context, backendPort string, exps experiments.Client, gitpodHostname string) error {
+	if exps == nil {
+		log.Error("no experiments.Client setup")
+		return nil
+	}
+	if !exps.GetBoolValue(ctx, "jb_wait_backend_plugin_readiness", false, experiments.Attributes{GitpodHost: gitpodHostname}) {
+		log.Debug("will not wait plugin ready")
+		return nil
+	}
+	log.WithField("backendPort", backendPort).Debug("wait backend plugin to be ready")
+	// Use op=metrics so that we don't need to rebuild old backend-plugin
+	url, err := url.Parse("http://localhost:" + backendPort + "/api/gitpod/cli?op=metrics")
+	if err != nil {
+		return err
+	}
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url.String(), nil)
+	if err != nil {
+		return err
+	}
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("backend plugin is not ready: %d", resp.StatusCode)
+	}
+	return nil
+}
+
 func restart(r *http.Request) {
 	backendPort := r.URL.Query().Get("backendPort")
 	if backendPort == "" {
@@ -1068,6 +1119,9 @@ func getProductConfig(config *gitpod.GitpodConfig, alias string) *gitpod.Jetbrai
 
 func linkRemotePlugin(launchCtx *LaunchContext) error {
 	remotePluginsFolder := launchCtx.configDir + "/plugins"
+	if launchCtx.info.Version == "2022.3.3" {
+		remotePluginsFolder = launchCtx.backendDir + "/plugins"
+	}
 	remotePluginDir := remotePluginsFolder + "/gitpod-remote"
 	_, err := os.Stat(remotePluginDir)
 	if err == nil || !errors.Is(err, os.ErrNotExist) {

components/supervisor/cmd/init.go

@@ -20,6 +20,7 @@ import (
 
 	"github.com/gitpod-io/gitpod/common-go/log"
 	"github.com/gitpod-io/gitpod/common-go/process"
+	"github.com/gitpod-io/gitpod/supervisor/pkg/shared"
 	"github.com/gitpod-io/gitpod/supervisor/pkg/supervisor"
 	"github.com/prometheus/procfs"
 	reaper "github.com/ramr/go-reaper"
@@ -83,7 +84,11 @@ var initCmd = &cobra.Command{
 			if err != nil && !(strings.Contains(err.Error(), "signal: ") || strings.Contains(err.Error(), "no child processes")) {
 				if eerr, ok := err.(*exec.ExitError); ok && eerr.ExitCode() != 0 {
 					logs := extractFailureFromRun()
-					log.WithError(fmt.Errorf(logs)).Fatal("supervisor run error with unexpected exit code")
+					if shared.IsExpectedShutdown(eerr.ExitCode()) {
+						log.Fatal(logs)
+					} else {
+						log.WithError(fmt.Errorf(logs)).Fatal("supervisor run error with unexpected exit code")
+					}
 				}
 				log.WithError(err).Error("supervisor run error")
 				return

components/supervisor/hot-deploy.sh

@@ -19,7 +19,7 @@ echo "Image Version: $version"
 bldfn="/tmp/build-$version.tar.gz"
 
 docker ps &> /dev/null || (echo "You need a working Docker daemon. Maybe set DOCKER_HOST?"; exit 1)
-leeway build -Dversion="$version" -DimageRepoBase=eu.gcr.io/gitpod-core-dev/build .:docker --save "$bldfn"
+leeway build -Dversion="$version" -DimageRepoBase=eu.gcr.io/gitpod-dev-artifact/build .:docker --save "$bldfn"
 dev_image="$(tar xfO "$bldfn" ./imgnames.txt | head -n1)"
 echo "Dev Image: $dev_image"
 

components/supervisor/pkg/metrics/metrics.go

@@ -7,6 +7,7 @@ package metrics
 import (
 	"fmt"
 
+	"github.com/gitpod-io/gitpod/supervisor/pkg/shared"
 	"github.com/prometheus/client_golang/prometheus"
 )
 
@@ -22,7 +23,7 @@ func NewMetrics() *SupervisorMetrics {
 		IDEReadyDurationTotal: prometheus.NewHistogramVec(prometheus.HistogramOpts{
 			Name:    "supervisor_ide_ready_duration_total",
 			Help:    "the IDE startup time",
-			Buckets: []float64{0.1, 0.5, 1, 1.5, 2, 2.5, 5, 10},
+			Buckets: []float64{0.1, 0.5, 1, 1.5, 2, 2.5, 5, shared.IDEReadyDurationTotalMaxBucketSecond},
 		}, []string{"kind"}),
 		InitializerHistogram: prometheus.NewHistogramVec(prometheus.HistogramOpts{
 			Name:    "supervisor_initializer_bytes_second",

components/supervisor/pkg/shared/constant.go

@@ -0,0 +1,14 @@
+// Copyright (c) 2024 Gitpod GmbH. All rights reserved.
+// Licensed under the GNU Affero General Public License (AGPL).
+// See License.AGPL.txt in the project root for license information.
+
+package shared
+
+const (
+	IDEReadyDurationTotalMaxBucketSecond = 10
+	ExitCodeReasonIDEReadinessTimedOut   = 2
+)
+
+func IsExpectedShutdown(exitCode int) bool {
+	return exitCode == ExitCodeReasonIDEReadinessTimedOut
+}

components/supervisor/pkg/supervisor/services.go

@@ -53,9 +53,14 @@ type DesktopIDEStatus struct {
 }
 
 type ideReadyState struct {
-	ready bool
-	info  *DesktopIDEStatus
-	cond  *sync.Cond
+	ready     bool
+	ideConfig *IDEConfig
+	info      *DesktopIDEStatus
+	cond      *sync.Cond
+}
+
+func newIDEReadyState(ideConfig *IDEConfig) *ideReadyState {
+	return &ideReadyState{cond: sync.NewCond(&sync.Mutex{}), ideConfig: ideConfig}
 }
 
 // Wait returns a channel that emits when IDE is ready.