Cherry-pick #4451 #4457 #4462 #4463 #4464 #4465 #4466 #4469 #4482 (#4485

)

* Update web deps (#4451)

Signed-off-by: khanhtc1202 <khanhtc1202@gmail.com>
Signed-off-by: Kenta Kozuka <kenta.kozuka@gmail.com>

* Update quickstart to support arm hardware(ex: M1) (#4457)

* Add: arch selection for mysql image
Update chart values and templates for mysql deployment

Signed-off-by: gitbluf <gitbluf@tuta.io>

* Update: MySql image selector function to be more descriptive

Signed-off-by: gitbluf <gitbluf@tuta.io>

* Remove: mysql image set on piped installation

Signed-off-by: gitbluf <gitbluf@tuta.io>

* Update method name

Signed-off-by: gitbluf <gitbluf@tuta.io>

---------

Signed-off-by: gitbluf <gitbluf@tuta.io>
Signed-off-by: Kenta Kozuka <kenta.kozuka@gmail.com>

* Support AWS LoadBalancer with multi listeners (#4462)

* Support AWS LoadBalancer with multi listeners

Signed-off-by: khanhtc1202 <khanhtc1202@gmail.com>

* Fix go array index

Signed-off-by: khanhtc1202 <khanhtc1202@gmail.com>

* Rename symbol

Signed-off-by: khanhtc1202 <khanhtc1202@gmail.com>

---------

Signed-off-by: khanhtc1202 <khanhtc1202@gmail.com>
Signed-off-by: Kenta Kozuka <kenta.kozuka@gmail.com>

* Specify Version For Templating Tools (#4463)

* specify version for templating tools

Signed-off-by: Viet Anh Pham Nhu <nhuvietanh271098@gmail.com>

* Handle Error Both Install Helm And Kustomize

Signed-off-by: Viet Anh Pham Nhu <nhuvietanh271098@gmail.com>

---------

Signed-off-by: Viet Anh Pham Nhu <nhuvietanh271098@gmail.com>
Signed-off-by: Kenta Kozuka <kenta.kozuka@gmail.com>

* Output usage only when flag parsing fails #4381 (#4464)

Signed-off-by: Yoshiki Fujikane <ffjlabo@gmail.com>
Signed-off-by: Kenta Kozuka <kenta.kozuka@gmail.com>

* Fix ECS rollback stage does not remove canary created tasks (#4465)

* Fix cannot remove canary created task

Signed-off-by: ductnn <trannhuduc531998@gmail.com>

* Fix reuse clean function

Signed-off-by: ductnn <trannhuduc531998@gmail.com>

* Update logs message for ECS routing stage rollback

Signed-off-by: ductnn <trannhuduc531998@gmail.com>

---------

Signed-off-by: ductnn <trannhuduc531998@gmail.com>
Signed-off-by: Kenta Kozuka <kenta.kozuka@gmail.com>

* Update logs message for ECS routing stage executor (#4466)

Signed-off-by: khanhtc1202 <khanhtc1202@gmail.com>
Signed-off-by: Kenta Kozuka <kenta.kozuka@gmail.com>

* fix: update deprecated apiVersion since gke removed (#4469)

GoogleCloudPlatform/gke-managed-certs#58

Signed-off-by: hungran <26101787+hungran@users.noreply.github.com>
Signed-off-by: Kenta Kozuka <kenta.kozuka@gmail.com>

* Skip update AppState when no log updates (#4482)

Signed-off-by: Kenta Kozuka <kenta.kozuka@gmail.com>

---------

Signed-off-by: khanhtc1202 <khanhtc1202@gmail.com>
Signed-off-by: Kenta Kozuka <kenta.kozuka@gmail.com>
Signed-off-by: gitbluf <gitbluf@tuta.io>
Signed-off-by: Viet Anh Pham Nhu <nhuvietanh271098@gmail.com>
Signed-off-by: Yoshiki Fujikane <ffjlabo@gmail.com>
Signed-off-by: ductnn <trannhuduc531998@gmail.com>
Signed-off-by: hungran <26101787+hungran@users.noreply.github.com>
Co-authored-by: Khanh Tran <32532742+khanhtc1202@users.noreply.github.com>
Co-authored-by: Marko Petrovic <22802784+gitbluf@users.noreply.github.com>
Co-authored-by: Viet Anh Pham Nhu <40441000+anhpnv@users.noreply.github.com>
Co-authored-by: Yoshiki Fujikane <40124947+ffjlabo@users.noreply.github.com>
Co-authored-by: Duc Tran <trannhuduc531998@gmail.com>
Co-authored-by: Henry Vu <26101787+hungran@users.noreply.github.com>
Co-authored-by: Kurochan <kuro@kurochan.org>

manifests/pipecd/templates/deployment.yaml

@@ -354,7 +354,7 @@ spec:
       {{- end }}
       containers:
         - name: mysql
-          image: mysql:{{ .Values.mysql.imageTag }}
+          image: {{ .Values.mysql.image }}:{{ .Values.mysql.imageTag }}
           imagePullPolicy: IfNotPresent
           env:
             - name: MYSQL_ROOT_PASSWORD

manifests/pipecd/templates/managed-certificate.yaml

@@ -1,5 +1,5 @@
 {{- if .Values.managedCertificate.enabled -}}
-apiVersion: networking.gke.io/v1beta2
+apiVersion: networking.gke.io/v1
 kind: ManagedCertificate
 metadata:
   name: {{ include "pipecd.fullname" . }}
@@ -12,7 +12,7 @@ spec:
 
 {{ if .Values.monitoring.managedCertificate.enabled -}}
 ---
-apiVersion: networking.gke.io/v1beta2
+apiVersion: networking.gke.io/v1
 kind: ManagedCertificate
 metadata:
   name: {{ include "pipecd.fullname" . }}-grafana

manifests/pipecd/values.yaml

@@ -69,7 +69,8 @@ cloudSQLProxy:
   resources: {}
 
 mysql:
-  imageTag: "8.0.23"
+  image: mysql
+  imageTag: 8.0.33
   resources: {}
   port: 3306
 

manifests/site/templates/managed-certificate.yaml

@@ -1,5 +1,5 @@
 {{- if .Values.managedCertificate.enabled -}}
-apiVersion: networking.gke.io/v1beta2
+apiVersion: networking.gke.io/v1
 kind: ManagedCertificate
 metadata:
   name: {{ include "site.fullname" . }}

pkg/app/pipectl/cmd/quickstart/quickstart.go

@@ -149,6 +149,8 @@ func (c *command) installControlPlane(ctx context.Context, helm string, input cl
 		"--create-namespace",
 		"--values",
 		fmt.Sprintf(helmQuickstartValueRemotePath, c.version),
+		"--set",
+		fmt.Sprintf("mysql.image=%s", selectMySQLImage()),
 	}
 
 	var stderr, stdout bytes.Buffer
@@ -312,6 +314,19 @@ func openbrowser(url string) error {
 	return err
 }
 
+func selectMySQLImage() string {
+	var mysqlImage string
+	switch runtime.GOARCH {
+	case "amd64":
+		mysqlImage = "mysql"
+	case "arm64":
+		mysqlImage = "arm64v8/mysql"
+	default:
+		mysqlImage = "mysql"
+	}
+	return mysqlImage
+}
+
 func (c *command) uninstallAll(ctx context.Context, helm string, input cli.Input) error {
 	input.Logger.Info("Uninstalling PipeCD components...")
 
@@ -380,9 +395,9 @@ func (c *command) getKubectl() (string, error) {
 }
 
 // getHelm finds and returns helm executable binary in the following priority:
-//   1. pre-installed in command specified toolsDir (default is $HOME/.pipectl/tools)
-//   2. $PATH
-//   3. install new helm to command specified toolsDir
+//  1. pre-installed in command specified toolsDir (default is $HOME/.pipectl/tools)
+//  2. $PATH
+//  3. install new helm to command specified toolsDir
 func (c *command) getHelm(ctx context.Context) (string, error) {
 	binName := "helm"
 

pkg/app/piped/executor/ecs/ecs.go

@@ -409,14 +409,14 @@ func routing(ctx context.Context, in *executor.Input, platformProviderName strin
 		in.Logger.Error("Failed to store traffic routing config to metadata store", zap.Error(err))
 	}
 
-	currListenerArn, err := client.GetListener(ctx, primaryTargetGroup)
+	currListenerArns, err := client.GetListenerArns(ctx, primaryTargetGroup)
 	if err != nil {
-		in.LogPersister.Errorf("Failed to get current active listener: %v", err)
+		in.LogPersister.Errorf("Failed to get current active listeners: %v", err)
 		return false
 	}
 
-	if err := client.ModifyListener(ctx, currListenerArn, routingTrafficCfg); err != nil {
-		in.LogPersister.Errorf("Failed to routing traffic to CANARY variant: %v", err)
+	if err := client.ModifyListeners(ctx, currListenerArns, routingTrafficCfg); err != nil {
+		in.LogPersister.Errorf("Failed to routing traffic to PRIMARY/CANARY variants: %v", err)
 		return false
 	}
 

pkg/app/piped/executor/ecs/rollback.go

@@ -82,19 +82,19 @@ func (e *rollbackExecutor) ensureRollback(ctx context.Context) model.StageStatus
 		return model.StageStatus_STAGE_FAILURE
 	}
 
-	primary, _, ok := loadTargetGroups(&e.Input, appCfg, runningDS)
+	primary, canary, ok := loadTargetGroups(&e.Input, appCfg, runningDS)
 	if !ok {
 		return model.StageStatus_STAGE_FAILURE
 	}
 
-	if !rollback(ctx, &e.Input, platformProviderName, platformProviderCfg, taskDefinition, serviceDefinition, primary) {
+	if !rollback(ctx, &e.Input, platformProviderName, platformProviderCfg, taskDefinition, serviceDefinition, primary, canary) {
 		return model.StageStatus_STAGE_FAILURE
 	}
 
 	return model.StageStatus_STAGE_SUCCESS
 }
 
-func rollback(ctx context.Context, in *executor.Input, platformProviderName string, platformProviderCfg *config.PlatformProviderECSConfig, taskDefinition types.TaskDefinition, serviceDefinition types.Service, targetGroup *types.LoadBalancer) bool {
+func rollback(ctx context.Context, in *executor.Input, platformProviderName string, platformProviderCfg *config.PlatformProviderECSConfig, taskDefinition types.TaskDefinition, serviceDefinition types.Service, primaryTargetGroup *types.LoadBalancer, canaryTargetGroup *types.LoadBalancer) bool {
 	in.LogPersister.Infof("Start rollback the ECS service and task family: %s and %s to original stage", *serviceDefinition.ServiceName, *taskDefinition.Family)
 	client, err := provider.DefaultRegistry().Client(platformProviderName, platformProviderCfg, in.Logger)
 	if err != nil {
@@ -127,7 +127,7 @@ func rollback(ctx context.Context, in *executor.Input, platformProviderName stri
 	}
 
 	// On rolling back, the scale of desired tasks will be set to 100 (same as the original state).
-	taskSet, err := client.CreateTaskSet(ctx, *service, *td, targetGroup, 100)
+	taskSet, err := client.CreateTaskSet(ctx, *service, *td, primaryTargetGroup, 100)
 	if err != nil {
 		in.LogPersister.Errorf("Failed to create ECS task set %s: %v", *serviceDefinition.ServiceName, err)
 		return false
@@ -147,6 +147,35 @@ func rollback(ctx context.Context, in *executor.Input, platformProviderName stri
 		}
 	}
 
+	// Reset routing
+	routingTrafficCfg := provider.RoutingTrafficConfig{
+		{
+			TargetGroupArn: *primaryTargetGroup.TargetGroupArn,
+			Weight:         100,
+		},
+		{
+			TargetGroupArn: *canaryTargetGroup.TargetGroupArn,
+			Weight:         0,
+		},
+	}
+
+	currListenerArns, err := client.GetListenerArns(ctx, *primaryTargetGroup)
+	if err != nil {
+		in.LogPersister.Errorf("Failed to get current active listeners: %v", err)
+		return false
+	}
+
+	if err := client.ModifyListeners(ctx, currListenerArns, routingTrafficCfg); err != nil {
+		in.LogPersister.Errorf("Failed to routing traffic to PRIMARY variant: %v", err)
+		return false
+	}
+
+	// Delete Canary taskSet
+	if !clean(ctx, in, platformProviderName, platformProviderCfg) {
+		in.LogPersister.Error("Failed to delete CANARY TaskSet")
+		return false
+	}
+
 	in.LogPersister.Infof("Rolled back the ECS service %s and task definition %s configuration to original stage", *serviceDefinition.ServiceName, *taskDefinition.Family)
 	return true
 }

pkg/app/piped/platformprovider/ecs/client.go

@@ -277,29 +277,29 @@ func (c *client) ServiceExists(ctx context.Context, clusterName string, serviceN
 	return false, nil
 }
 
-func (c *client) GetListener(ctx context.Context, targetGroup types.LoadBalancer) (string, error) {
+func (c *client) GetListenerArns(ctx context.Context, targetGroup types.LoadBalancer) ([]string, error) {
 	loadBalancerArn, err := c.getLoadBalancerArn(ctx, *targetGroup.TargetGroupArn)
 	if err != nil {
-		return "", err
+		return nil, err
 	}
 
 	input := &elasticloadbalancingv2.DescribeListenersInput{
 		LoadBalancerArn: aws.String(loadBalancerArn),
 	}
 	output, err := c.elbClient.DescribeListeners(ctx, input)
 	if err != nil {
-		return "", err
+		return nil, err
 	}
 	if len(output.Listeners) == 0 {
-		return "", platformprovider.ErrNotFound
+		return nil, platformprovider.ErrNotFound
 	}
-	// Note: Suppose the load balancer only have one listener.
-	// TODO: Support multi listeners pattern.
-	if len(output.Listeners) > 1 {
-		return "", fmt.Errorf("invalid listener configuration pointed to %s target group", *targetGroup.TargetGroupArn)
+
+	arns := make([]string, len(output.Listeners))
+	for i := range output.Listeners {
+		arns[i] = *output.Listeners[i].ListenerArn
 	}
 
-	return *output.Listeners[0].ListenerArn, nil
+	return arns, nil
 }
 
 func (c *client) getLoadBalancerArn(ctx context.Context, targetGroupArn string) (string, error) {
@@ -317,32 +317,42 @@ func (c *client) getLoadBalancerArn(ctx context.Context, targetGroupArn string)
 	return output.TargetGroups[0].LoadBalancerArns[0], nil
 }
 
-func (c *client) ModifyListener(ctx context.Context, listenerArn string, routingTrafficCfg RoutingTrafficConfig) error {
+func (c *client) ModifyListeners(ctx context.Context, listenerArns []string, routingTrafficCfg RoutingTrafficConfig) error {
 	if len(routingTrafficCfg) != 2 {
 		return fmt.Errorf("invalid listener configuration: requires 2 target groups")
 	}
-	input := &elasticloadbalancingv2.ModifyListenerInput{
-		ListenerArn: aws.String(listenerArn),
-		DefaultActions: []elbtypes.Action{
-			{
-				Type: elbtypes.ActionTypeEnumForward,
-				ForwardConfig: &elbtypes.ForwardActionConfig{
-					TargetGroups: []elbtypes.TargetGroupTuple{
-						{
-							TargetGroupArn: aws.String(routingTrafficCfg[0].TargetGroupArn),
-							Weight:         aws.Int32(int32(routingTrafficCfg[0].Weight)),
-						},
-						{
-							TargetGroupArn: aws.String(routingTrafficCfg[1].TargetGroupArn),
-							Weight:         aws.Int32(int32(routingTrafficCfg[1].Weight)),
+
+	modifyListener := func(ctx context.Context, listenerArn string) error {
+		input := &elasticloadbalancingv2.ModifyListenerInput{
+			ListenerArn: aws.String(listenerArn),
+			DefaultActions: []elbtypes.Action{
+				{
+					Type: elbtypes.ActionTypeEnumForward,
+					ForwardConfig: &elbtypes.ForwardActionConfig{
+						TargetGroups: []elbtypes.TargetGroupTuple{
+							{
+								TargetGroupArn: aws.String(routingTrafficCfg[0].TargetGroupArn),
+								Weight:         aws.Int32(int32(routingTrafficCfg[0].Weight)),
+							},
+							{
+								TargetGroupArn: aws.String(routingTrafficCfg[1].TargetGroupArn),
+								Weight:         aws.Int32(int32(routingTrafficCfg[1].Weight)),
+							},
 						},
 					},
 				},
 			},
-		},
+		}
+		_, err := c.elbClient.ModifyListener(ctx, input)
+		return err
+	}
+
+	for _, listener := range listenerArns {
+		if err := modifyListener(ctx, listener); err != nil {
+			return err
+		}
 	}
-	_, err := c.elbClient.ModifyListener(ctx, input)
-	return err
+	return nil
 }
 
 func (c *client) TagResource(ctx context.Context, resourceArn string, tags []types.Tag) error {

pkg/app/piped/platformprovider/ecs/ecs.go

@@ -55,8 +55,8 @@ type ECS interface {
 }
 
 type ELB interface {
-	GetListener(ctx context.Context, targetGroup types.LoadBalancer) (string, error)
-	ModifyListener(ctx context.Context, listenerArn string, routingTrafficCfg RoutingTrafficConfig) error
+	GetListenerArns(ctx context.Context, targetGroup types.LoadBalancer) ([]string, error)
+	ModifyListeners(ctx context.Context, listenerArns []string, routingTrafficCfg RoutingTrafficConfig) error
 }
 
 // Registry holds a pool of aws client wrappers.

pkg/app/piped/platformprovider/kubernetes/loader.go

@@ -16,6 +16,7 @@ package kubernetes
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -90,13 +91,12 @@ func (l *loader) LoadManifests(ctx context.Context) (manifests []Manifest, err e
 		sortManifests(manifests)
 	}()
 	l.initOnce.Do(func() {
+		var initErrorHelm, initErrorKustomize error
 		l.templatingMethod = determineTemplatingMethod(l.input, l.appDir)
-		switch l.templatingMethod {
-		case TemplatingMethodHelm:
-			l.helm, l.initErr = l.findHelm(ctx, l.input.HelmVersion)
-
-		case TemplatingMethodKustomize:
-			l.kustomize, l.initErr = l.findKustomize(ctx, l.input.KustomizeVersion)
+		if l.templatingMethod != TemplatingMethodNone {
+			l.helm, initErrorHelm = l.findHelm(ctx, l.input.HelmVersion)
+			l.kustomize, initErrorKustomize = l.findKustomize(ctx, l.input.KustomizeVersion)
+			l.initErr = errors.Join(initErrorHelm, initErrorKustomize)
 		}
 	})
 	if l.initErr != nil {

pkg/cli/app.go

@@ -15,6 +15,7 @@
 package cli
 
 import (
+	"errors"
 	"fmt"
 
 	"github.com/spf13/cobra"
@@ -29,15 +30,23 @@ type App struct {
 	telemetryFlags TelemetryFlags
 }
 
+var FlagParseErr = errors.New("FlagParseErr")
+
 func NewApp(name, desc string) *App {
 	a := &App{
 		rootCmd: &cobra.Command{
 			Use:           name,
 			Short:         desc,
 			SilenceErrors: true,
+			SilenceUsage:  true,
 		},
 		telemetryFlags: defaultTelemetryFlags,
 	}
+	a.rootCmd.SetFlagErrorFunc(func(cmd *cobra.Command, err error) error {
+		cmd.Println(err)
+		cmd.Println(cmd.UsageString())
+		return FlagParseErr
+	})
 	versionCmd := &cobra.Command{
 		Use:   "version",
 		Short: "Print the information of current binary.",

web/custom-jsdom.js

@@ -1,6 +1,6 @@
 const Environment = require("jest-environment-jsdom");
 
-module.exports = class CustomTestEnvironment extends Environment {
+module.exports = class CustomTestEnvironment extends Environment.default {
   async setup() {
     await super.setup();
     if (typeof this.global.TextEncoder === "undefined") {

web/file-transformer.js

@@ -2,6 +2,8 @@ const path = require("path");
 
 module.exports = {
   process(src, filename, config, options) {
-    return "module.exports = " + JSON.stringify(path.basename(filename)) + ";";
+    return {
+      code: `module.exports = ${JSON.stringify(path.basename(filename))};`,
+    };
   },
 };

web/jest.config.js

@@ -1,7 +1,12 @@
 module.exports = {
   roots: ["<rootDir>/src"],
   transform: {
-    "^.+\\.tsx?$": "ts-jest",
+    "^.+\\.tsx?$": [
+      "ts-jest",
+      {
+        isolatedModules: true,
+      },
+    ],
     "\\.(jpg|jpeg|png|gif|eot|otf|webp|svg|ttf|woff|woff2|mp4|webm|wav|mp3|m4a|aac|oga|ico)$":
       "<rootDir>/file-transformer.js",
   },
@@ -23,9 +28,4 @@ module.exports = {
   setupFilesAfterEnv: ["./jest.after-env.ts"],
   coverageReporters: ["lcovonly", "text-summary"],
   maxWorkers: 1,
-  globals: {
-    "ts-jest": {
-      isolatedModules: true,
-    },
-  },
 };