chore: dedup destination metadata in router based on job id (#2982)

router/router.go

@@ -16,6 +16,7 @@ import (
 	"time"
 
 	jsoniter "github.com/json-iterator/go"
+	"github.com/samber/lo"
 	"github.com/tidwall/gjson"
 	"golang.org/x/sync/errgroup"
 
@@ -781,7 +782,11 @@ func (worker *workerT) processDestinationJobs() {
 		// elements in routerJobResponses have pointer to the right job.
 		_destinationJob := destinationJob
 
-		for _, destinationJobMetadata := range _destinationJob.JobMetadataArray {
+		// TODO: remove this once we enforce the necessary validations in the transformer's response
+		dedupedJobMetadata := lo.UniqBy(_destinationJob.JobMetadataArray, func(jobMetadata types.JobMetadataT) int64 {
+			return jobMetadata.JobID
+		})
+		for _, destinationJobMetadata := range dedupedJobMetadata {
 			_destinationJobMetadata := destinationJobMetadata
 			// assigning the destinationJobMetadata to a local variable (_destinationJobMetadata), so that
 			// elements in routerJobResponses have pointer to the right destinationJobMetadata.