Merge branch 'dev' into feat/cstore/serde_optimizations

.buildkite/hooks/pre-exit

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+if [[ -f vtools/.buildkite/hooks/pre-exit ]]; then
+  echo "~~~ Executing vtools pre-exit hook"
+  vtools/.buildkite/hooks/pre-exit
+else
+  echo "No pre-exit hook found in vtools/.buildkite/hooks/"
+fi

.github/workflows/backport-command.yml

@@ -192,6 +192,7 @@ jobs:
           ORIG_REVIEWERS: ${{ steps.reviewers.outputs.reviewers }}
           HEAD_BRANCH: ${{ steps.pr_details.outputs.head_branch }}
           GIT_USER: ${{ steps.user.outputs.username }}
+          BACKPORT_COMMITS: ${{ steps.backport_commits.outputs.backport_commits }}
         id: create_issue_on_backport_error
         run: $SCRIPT_DIR/create_issue.sh
         shell: bash

.github/workflows/scripts/backport-command/create_issue.sh

@@ -20,17 +20,26 @@ orig_assignees=$ORIG_ASSIGNEES
 if [[ -n $CREATE_ISSUE_ON_ERROR ]]; then
   additional_body="Note that this issue was created as a placeholder, since the original PR's commit(s) could not be automatically cherry-picked."
 
+  local_user=$(gh api user --jq .login)
+  local_branch="$local_user/backport-$PR_NUMBER-$BACKPORT_BRANCH-$((RANDOM % 1000))"
+
   additional_body="$additional_body
-  Command I attempted to execute:
-  gh pr create
-    --title \"[$BACKPORT_BRANCH] $ORIG_TITLE\"
-    --base \"$BACKPORT_BRANCH\"
-    --label \"kind/backport\"
-    --head \"$GIT_USER:$HEAD_BRANCH\"
-    --draft
-    --repo \"$TARGET_ORG/$TARGET_REPO\"
-    --reviewer \"$ORIG_REVIEWERS\"
-    --milestone \"$TARGET_MILESTONE\"
+  Here are the commands to execute:
+  \`\`\`
+  git checkout $BACKPORT_BRANCH
+  git checkout -b $local_branch
+  git cherry-pick -x $BACKPORT_COMMITS
+
+  git push origin $local_branch
+  gh pr create \\
+    --title \"[$BACKPORT_BRANCH] $ORIG_TITLE\" \\
+    --base \"$BACKPORT_BRANCH\" \\
+    --label \"kind/backport\" \\
+    --head \"$local_branch\" \\
+    --draft \\
+    --repo \"$TARGET_ORG/$TARGET_REPO\" \\
+    --reviewer \"$ORIG_REVIEWERS\" \\
+    --milestone \"$TARGET_MILESTONE\" \\
     --body \"Backport of PR $ORIG_ISSUE_URL \""
 
   orig_assignees=$(gh issue view $PR_NUMBER --json author --jq .author.login)

.github/workflows/scripts/backport-command/pr_details.sh

@@ -51,8 +51,9 @@ head_branch=$(echo "backport-$backport_issues_numbers$BACKPORT_BRANCH-$suffix" |
 git checkout -b "$head_branch" "remotes/upstream/$BACKPORT_BRANCH"
 
 if ! git cherry-pick -x $BACKPORT_COMMITS; then
-  msg="Failed to run cherry-pick command. I executed the below command:\n
+  msg="Failed to run cherry-pick command. I executed the commands below:\n
 \`\`\`\r
+git checkout -b "$head_branch" "remotes/upstream/$BACKPORT_BRANCH"
 git cherry-pick -x $BACKPORT_COMMITS
 \`\`\`"
 

src/go/k8s/controllers/redpanda/cluster_controller.go

@@ -655,7 +655,15 @@ func (r *ClusterReconciler) reportStatus(
 	nodeList.Internal = observedNodesInternal
 	nodeList.SchemaRegistry.Internal = fmt.Sprintf("%s:%d", clusterFQDN, schemaRegistryPort)
 
-	if statusShouldBeUpdated(&redpandaCluster.Status, nodeList, sts) {
+	//nolint:nestif // the code won't get clearer if it's splitted out in my opinion
+	version, versionErr := sts.CurrentVersion(ctx)
+	if versionErr != nil {
+		// this is non-fatal error, it will return error even if e.g.
+		// the rollout is not finished because then the currentversion
+		// of the cluster cannot be determined
+		r.Log.Info(fmt.Sprintf("cannot get CurrentVersion of statefulset, %s", err))
+	}
+	if statusShouldBeUpdated(&redpandaCluster.Status, nodeList, sts, version, versionErr) {
 		err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
 			var cluster redpandav1alpha1.Cluster
 			err := r.Get(ctx, types.NamespacedName{
@@ -669,7 +677,9 @@ func (r *ClusterReconciler) reportStatus(
 			cluster.Status.Nodes = *nodeList
 			cluster.Status.ReadyReplicas = sts.LastObservedState.Status.ReadyReplicas
 			cluster.Status.Replicas = sts.LastObservedState.Status.Replicas
-			cluster.Status.Version = sts.Version()
+			if versionErr == nil {
+				cluster.Status.Version = version
+			}
 
 			err = r.Status().Update(ctx, &cluster)
 			if err == nil {
@@ -689,6 +699,8 @@ func statusShouldBeUpdated(
 	status *redpandav1alpha1.ClusterStatus,
 	nodeList *redpandav1alpha1.NodesList,
 	sts *resources.StatefulSetResource,
+	newVersion string,
+	versionErr error,
 ) bool {
 	return nodeList != nil &&
 		(!reflect.DeepEqual(nodeList.Internal, status.Nodes.Internal) ||
@@ -699,7 +711,7 @@ func statusShouldBeUpdated(
 			!reflect.DeepEqual(nodeList.ExternalBootstrap, status.Nodes.ExternalBootstrap)) ||
 		status.Replicas != sts.LastObservedState.Status.Replicas ||
 		status.ReadyReplicas != sts.LastObservedState.Status.ReadyReplicas ||
-		status.Version != sts.Version()
+		(versionErr == nil && status.Version != newVersion)
 }
 
 func (r *ClusterReconciler) podList(ctx context.Context, redpandaCluster *redpandav1alpha1.Cluster) (corev1.PodList, error) {

src/go/k8s/controllers/redpanda/cluster_controller_test.go

@@ -27,6 +27,7 @@ import (
 	"k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/utils/pointer"
 	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 
 	"github.com/redpanda-data/redpanda/src/go/k8s/apis/redpanda/v1alpha1"
@@ -852,7 +853,12 @@ var _ = Describe("RedPandaCluster controller", func() {
 		})
 		It("Should not throw error; redpanda version allowed", func() {
 			key, redpandaCluster := getVersionedRedpanda("restricted-redpanda-positive", allowedVersion)
-			fc := fake.NewClientBuilder().WithObjects(redpandaCluster).Build()
+			pods := readyPodsForCluster(redpandaCluster)
+			objects := []client.Object{redpandaCluster}
+			for i := range pods {
+				objects = append(objects, pods[i])
+			}
+			fc := fake.NewClientBuilder().WithObjects(objects...).Build()
 			r := &redpanda.ClusterReconciler{
 				Client:                    fc,
 				Log:                       ctrl.Log,
@@ -898,6 +904,33 @@ var _ = Describe("RedPandaCluster controller", func() {
 		Entry("Random image pull policy", "asdvasd", Not(Succeed())))
 })
 
+func readyPodsForCluster(cluster *v1alpha1.Cluster) []*corev1.Pod {
+	var result []*corev1.Pod
+	for i := 0; i < int(*cluster.Spec.Replicas); i++ {
+		pod := &corev1.Pod{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      fmt.Sprintf("pod-%d", i),
+				Namespace: cluster.Namespace,
+				Labels: map[string]string{
+					"app.kubernetes.io/component": "redpanda",
+					"app.kubernetes.io/instance":  cluster.Name,
+					"app.kubernetes.io/name":      "redpanda",
+				},
+			},
+			Spec: corev1.PodSpec{
+				NodeName: "test-node",
+				Containers: []corev1.Container{{
+					Name:  "redpanda",
+					Image: fmt.Sprintf("redpanda:%s", cluster.Spec.Version),
+				}},
+			},
+			Status: corev1.PodStatus{Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}}},
+		}
+		result = append(result, pod)
+	}
+	return result
+}
+
 func getVersionedRedpanda(
 	name string, version string,
 ) (key types.NamespacedName, cluster *v1alpha1.Cluster) {

src/go/k8s/pkg/resources/statefulset.go

@@ -37,6 +37,7 @@ import (
 	"github.com/redpanda-data/redpanda/src/go/k8s/pkg/labels"
 	"github.com/redpanda-data/redpanda/src/go/k8s/pkg/resources/featuregates"
 	resourcetypes "github.com/redpanda-data/redpanda/src/go/k8s/pkg/resources/types"
+	"github.com/redpanda-data/redpanda/src/go/k8s/pkg/utils"
 )
 
 var _ Resource = &StatefulSetResource{}
@@ -864,26 +865,61 @@ func (r *StatefulSetResource) fullConfiguratorImage() string {
 	return fmt.Sprintf("%s:%s", r.configuratorSettings.ConfiguratorBaseImage, r.configuratorSettings.ConfiguratorTag)
 }
 
-// Version returns the cluster version specified in the image tag.
+// Version returns the cluster version specified in the image tag of the
+// statefulset spec. Depending on the rollout status it might not be the same as
+// version of the pods.
 func (r *StatefulSetResource) Version() string {
 	lastObservedSts := r.LastObservedState
 	if lastObservedSts != nil {
-		cc := lastObservedSts.Spec.Template.Spec.Containers
-		for i := range cc {
-			c := cc[i]
-			if c.Name != redpandaContainerName {
-				continue
-			}
-			// Will always have tag even for latest because of pandaCluster.FullImageName().
-			if s := strings.Split(c.Image, ":"); len(s) > 1 {
-				version := s[len(s)-1]
-				// Image uses registry with port and no tag (e.g. localhost:5000/redpanda)
-				if strings.Contains(version, "/") {
-					version = ""
-				}
-				return version
+		return redpandaContainerVersion(lastObservedSts.Spec.Template.Spec.Containers)
+	}
+	return ""
+}
+
+func redpandaContainerVersion(containers []corev1.Container) string {
+	for i := range containers {
+		c := containers[i]
+		if c.Name != redpandaContainerName {
+			continue
+		}
+		// Will always have tag even for latest because of pandaCluster.FullImageName().
+		if s := strings.Split(c.Image, ":"); len(s) > 1 {
+			version := s[len(s)-1]
+			// Image uses registry with port and no tag (e.g. localhost:5000/redpanda)
+			if strings.Contains(version, "/") {
+				version = ""
 			}
+			return version
 		}
 	}
 	return ""
 }
+
+// CurrentVersion is the version that's rolled out to all nodes (pods) of the cluster
+func (r *StatefulSetResource) CurrentVersion(ctx context.Context) (string, error) {
+	stsVersion := r.Version()
+	if stsVersion == "" {
+		return "", nil
+	}
+	replicas := *r.LastObservedState.Spec.Replicas
+	pods, err := r.getPodList(ctx)
+	if err != nil {
+		return "", err
+	}
+	if int32(len(pods.Items)) != replicas {
+		//nolint:goerr113 // not going to use wrapped static error here this time
+		return stsVersion, fmt.Errorf("rollout incomplete: pods count %d does not match expected replicas %d", len(pods.Items), replicas)
+	}
+	for i := range pods.Items {
+		if !utils.IsPodReady(&pods.Items[i]) {
+			//nolint:goerr113 // no need for static error
+			return stsVersion, fmt.Errorf("rollout incomplete: at least one pod (%s) is not READY", pods.Items[i].Name)
+		}
+		podVersion := redpandaContainerVersion(pods.Items[i].Spec.Containers)
+		if podVersion != stsVersion {
+			//nolint:goerr113 // no need for static error
+			return stsVersion, fmt.Errorf("rollout incomplete: at least one pod has version %s not %s", podVersion, stsVersion)
+		}
+	}
+	return stsVersion, nil
+}