Explicitly annotate TaskRuns with release version.

Previously we were expecting TaskRuns to be decorated with an annotation specifying
the release version of the tekton controller, however, the place that was populating
these annotations was extremely subtle and fragile.  The annotation was set as part
of synthesizing the `Pod` to create where the Pod's annotations were based on the
`TaskRun`'s.

This contains two main changes:
1. Explicitly copy the `TaskRun`'s annotations when populating the pod's annotations
  to eliminate this implicit and fragile back-propagation of annotations.

2. Add an explicit decoration of the annotation, which is less fragile and executed
  regardless of whether the Pod is being created during this reconciliation pass.

Fixes: #4421

pkg/pod/pod.go

@@ -286,7 +286,7 @@ func (b *Builder) Build(ctx context.Context, taskRun *v1beta1.TaskRun, taskSpec
 		priorityClassName = *podTemplate.PriorityClassName
 	}
 
-	podAnnotations := taskRun.Annotations
+	podAnnotations := kmeta.CopyMap(taskRun.Annotations)
 	version, err := changeset.Get()
 	if err != nil {
 		return nil, err

pkg/reconciler/taskrun/taskrun.go

@@ -56,6 +56,7 @@ import (
 	"k8s.io/client-go/kubernetes"
 	corev1Listers "k8s.io/client-go/listers/core/v1"
 	"knative.dev/pkg/apis"
+	"knative.dev/pkg/changeset"
 	"knative.dev/pkg/controller"
 	"knative.dev/pkg/kmeta"
 	"knative.dev/pkg/logging"
@@ -506,6 +507,16 @@ func (c *Reconciler) updateTaskRunWithDefaultWorkspaces(ctx context.Context, tr
 }
 
 func (c *Reconciler) updateLabelsAndAnnotations(ctx context.Context, tr *v1beta1.TaskRun) (*v1beta1.TaskRun, error) {
+	// Ensure the TaskRun is properly decorated with the version of the Tekton controller processing it.
+	version, err := changeset.Get()
+	if err != nil {
+		return nil, err
+	}
+	if tr.Annotations == nil {
+		tr.Annotations = make(map[string]string, 1)
+	}
+	tr.Annotations[podconvert.ReleaseAnnotation] = version
+
 	newTr, err := c.taskRunLister.TaskRuns(tr.Namespace).Get(tr.Name)
 	if err != nil {
 		return nil, fmt.Errorf("error getting TaskRun %s when updating labels/annotations: %w", tr.Name, err)