feat: server-side manifest generation for local diff (#8145)

assets/swagger.json

@@ -324,6 +324,40 @@
         }
       }
     },
+    "/api/v1/applications/manifestsWithFiles": {
+      "post": {
+        "tags": [
+          "ApplicationService"
+        ],
+        "summary": "GetManifestsWithFiles returns application manifests using provided files to generate them",
+        "operationId": "ApplicationService_GetManifestsWithFiles",
+        "parameters": [
+          {
+            "description": " (streaming inputs)",
+            "name": "body",
+            "in": "body",
+            "required": true,
+            "schema": {
+              "$ref": "#/definitions/applicationApplicationManifestQueryWithFilesWrapper"
+            }
+          }
+        ],
+        "responses": {
+          "200": {
+            "description": "A successful response.",
+            "schema": {
+              "$ref": "#/definitions/repositoryManifestResponse"
+            }
+          },
+          "default": {
+            "description": "An unexpected error response.",
+            "schema": {
+              "$ref": "#/definitions/runtimeError"
+            }
+          }
+        }
+      }
+    },
     "/api/v1/applications/{application.metadata.name}": {
       "put": {
         "tags": [
@@ -3386,6 +3420,28 @@
     "accountUpdatePasswordResponse": {
       "type": "object"
     },
+    "applicationApplicationManifestQueryWithFiles": {
+      "type": "object",
+      "properties": {
+        "checksum": {
+          "type": "string"
+        },
+        "name": {
+          "type": "string"
+        }
+      }
+    },
+    "applicationApplicationManifestQueryWithFilesWrapper": {
+      "type": "object",
+      "properties": {
+        "chunk": {
+          "$ref": "#/definitions/applicationFileChunk"
+        },
+        "query": {
+          "$ref": "#/definitions/applicationApplicationManifestQueryWithFiles"
+        }
+      }
+    },
     "applicationApplicationPatchRequest": {
       "type": "object",
       "title": "ApplicationPatchRequest is a request to patch an application",
@@ -3512,6 +3568,15 @@
         }
       }
     },
+    "applicationFileChunk": {
+      "type": "object",
+      "properties": {
+        "chunk": {
+          "type": "string",
+          "format": "byte"
+        }
+      }
+    },
     "applicationLogEntry": {
       "type": "object",
       "properties": {

cmd/argocd-repo-server/commands/argocd_repo_server.go

@@ -82,6 +82,8 @@ func NewCommand() *cobra.Command {
 		maxCombinedDirectoryManifestsSize string
 		cmpTarExcludedGlobs               []string
 		allowOutOfBoundsSymlinks          bool
+		streamedManifestMaxTarSize        int64
+		streamedManifestMaxExtractedSize  int64
 	)
 	var command = cobra.Command{
 		Use:               cliName,
@@ -126,6 +128,8 @@ func NewCommand() *cobra.Command {
 				MaxCombinedDirectoryManifestsSize:            maxCombinedDirectoryManifestsQuantity,
 				CMPTarExcludedGlobs:                          cmpTarExcludedGlobs,
 				AllowOutOfBoundsSymlinks:                     allowOutOfBoundsSymlinks,
+				StreamedManifestMaxExtractedSize:             streamedManifestMaxExtractedSize,
+				StreamedManifestMaxTarSize:                   streamedManifestMaxTarSize,
 			}, askPassServer)
 			errors.CheckError(err)
 
@@ -204,7 +208,8 @@ func NewCommand() *cobra.Command {
 	command.Flags().StringVar(&maxCombinedDirectoryManifestsSize, "max-combined-directory-manifests-size", env.StringFromEnv("ARGOCD_REPO_SERVER_MAX_COMBINED_DIRECTORY_MANIFESTS_SIZE", "10M"), "Max combined size of manifest files in a directory-type Application")
 	command.Flags().StringArrayVar(&cmpTarExcludedGlobs, "plugin-tar-exclude", env.StringsFromEnv("ARGOCD_REPO_SERVER_PLUGIN_TAR_EXCLUSIONS", []string{}, ";"), "Globs to filter when sending tarballs to plugins.")
 	command.Flags().BoolVar(&allowOutOfBoundsSymlinks, "allow-oob-symlinks", env.ParseBoolFromEnv("ARGOCD_REPO_SERVER_ALLOW_OUT_OF_BOUNDS_SYMLINKS", false), "Allow out-of-bounds symlinks in repositories (not recommended)")
-
+	command.Flags().Int64Var(&streamedManifestMaxTarSize, "streamed-manifest-max-tar-size", env.ParseInt64FromEnv("ARGOCD_REPO_SERVER_STREAMED_MANIFEST_MAX_TAR_SIZE", 1024*1024*100, 0, math.MaxInt64), "Maximum size of streamed manifest archives")
+	command.Flags().Int64Var(&streamedManifestMaxExtractedSize, "streamed-manifest-max-extracted-size", env.ParseInt64FromEnv("ARGOCD_REPO_SERVER_STREAMED_MANIFEST_MAX_EXTRACTED_SIZE", 1024*1024*1000, 0, math.MaxInt64), "Maximum size of streamed manifest archives when extracted")
 	tlsConfigCustomizerSrc = tls.AddTLSFlagsToCmd(&command)
 	cacheSrc = reposervercache.AddCacheFlagsToCmd(&command, func(client *redis.Client) {
 		redisClient = client

cmd/argocd/commands/app.go

@@ -20,6 +20,7 @@ import (
 	"github.com/argoproj/gitops-engine/pkg/sync/ignore"
 	"github.com/argoproj/gitops-engine/pkg/utils/kube"
 	"github.com/ghodss/yaml"
+	grpc_retry "github.com/grpc-ecosystem/go-grpc-middleware/retry"
 	"github.com/mattn/go-isatty"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
@@ -49,6 +50,7 @@ import (
 	"github.com/argoproj/argo-cd/v2/util/errors"
 	"github.com/argoproj/argo-cd/v2/util/git"
 	argoio "github.com/argoproj/argo-cd/v2/util/io"
+	"github.com/argoproj/argo-cd/v2/util/manifeststream"
 	"github.com/argoproj/argo-cd/v2/util/templates"
 	"github.com/argoproj/argo-cd/v2/util/text/label"
 )
@@ -819,12 +821,14 @@ type objKeyLiveTarget struct {
 // NewApplicationDiffCommand returns a new instance of an `argocd app diff` command
 func NewApplicationDiffCommand(clientOpts *argocdclient.ClientOptions) *cobra.Command {
 	var (
-		refresh       bool
-		hardRefresh   bool
-		exitCode      bool
-		local         string
-		revision      string
-		localRepoRoot string
+		refresh            bool
+		hardRefresh        bool
+		exitCode           bool
+		local              string
+		revision           string
+		localRepoRoot      string
+		serverSideGenerate bool
+		localIncludes      []string
 	)
 	shortDesc := "Perform a diff against the target and live state."
 	var command = &cobra.Command{
@@ -860,15 +864,27 @@ func NewApplicationDiffCommand(clientOpts *argocdclient.ClientOptions) *cobra.Co
 				errors.CheckError(err)
 				diffOption.res = res
 				diffOption.revision = revision
-			}
-			if local != "" {
-				conn, clusterIf := clientset.NewClusterClientOrDie()
-				defer argoio.Close(conn)
-				cluster, err := clusterIf.Get(ctx, &clusterpkg.ClusterQuery{Name: app.Spec.Destination.Name, Server: app.Spec.Destination.Server})
-				errors.CheckError(err)
-				diffOption.local = local
-				diffOption.localRepoRoot = localRepoRoot
-				diffOption.cluster = cluster
+			} else if local != "" {
+				if serverSideGenerate {
+					client, err := appIf.GetManifestsWithFiles(ctx, grpc_retry.Disable())
+					errors.CheckError(err)
+
+					err = manifeststream.SendApplicationManifestQueryWithFiles(ctx, client, appName, local, localIncludes)
+					errors.CheckError(err)
+
+					res, err := client.CloseAndRecv()
+					errors.CheckError(err)
+
+					diffOption.serversideRes = res
+				} else {
+					conn, clusterIf := clientset.NewClusterClientOrDie()
+					defer argoio.Close(conn)
+					cluster, err := clusterIf.Get(ctx, &clusterpkg.ClusterQuery{Name: app.Spec.Destination.Name, Server: app.Spec.Destination.Server})
+					errors.CheckError(err)
+					diffOption.local = local
+					diffOption.localRepoRoot = localRepoRoot
+					diffOption.cluster = cluster
+				}
 			}
 			foundDiffs := findandPrintDiff(ctx, app, resources, argoSettings, appName, diffOption)
 			if foundDiffs && exitCode {
@@ -882,6 +898,8 @@ func NewApplicationDiffCommand(clientOpts *argocdclient.ClientOptions) *cobra.Co
 	command.Flags().StringVar(&local, "local", "", "Compare live app to a local manifests")
 	command.Flags().StringVar(&revision, "revision", "", "Compare live app to a particular revision")
 	command.Flags().StringVar(&localRepoRoot, "local-repo-root", "/", "Path to the repository root. Used together with --local allows setting the repository root")
+	command.Flags().BoolVar(&serverSideGenerate, "server-side-generate", false, "Used with --local, this will send your manifests to the server for diffing")
+	command.Flags().StringArrayVar(&localIncludes, "local-include", []string{"*.yaml", "*.yml", "*.json"}, "Used with --server-side-generate, specify patterns of filenames to send")
 	return command
 }
 
@@ -892,6 +910,7 @@ type DifferenceOption struct {
 	revision      string
 	cluster       *argoappv1.Cluster
 	res           *repoapiclient.ManifestResponse
+	serversideRes *repoapiclient.ManifestResponse
 }
 
 // findandPrintDiff ... Prints difference between application current state and state stored in git or locally, returns boolean as true if difference is found else returns false
@@ -912,6 +931,15 @@ func findandPrintDiff(ctx context.Context, app *argoappv1.Application, resources
 		}
 		groupedObjs := groupObjsByKey(unstructureds, liveObjs, app.Spec.Destination.Namespace)
 		items = groupObjsForDiff(resources, groupedObjs, items, argoSettings, appName)
+	} else if diffOptions.serversideRes != nil {
+		var unstructureds []*unstructured.Unstructured
+		for _, mfst := range diffOptions.serversideRes.Manifests {
+			obj, err := argoappv1.UnmarshalToUnstructured(mfst)
+			errors.CheckError(err)
+			unstructureds = append(unstructureds, obj)
+		}
+		groupedObjs := groupObjsByKey(unstructureds, liveObjs, app.Spec.Destination.Namespace)
+		items = groupObjsForDiff(resources, groupedObjs, items, argoSettings, appName)
 	} else {
 		for i := range resources.Items {
 			res := resources.Items[i]

docs/operator-manual/argocd-cmd-params-cm.yaml

@@ -121,6 +121,10 @@ data:
   # Changing this to "true" will not allow _all_ out-of-bounds symlinks. Those will still be blocked for things like values 
   # files in Helm charts. But symlinks which are not explicitly blocked by other checks will be allowed.
   reposerver.allow.oob.symlinks: "false"
+  # Maximum size of tarball when streaming manifests to the repo server for generation
+  reposerver.streamed.manifest.max.tar.size: "10485760"
+  # Maximum size of extracted manifests when streaming manifests to the repo server for generation
+  reposerver.streamed.manifest.max.extracted.size: "104857600"
 
   # Disable TLS on the HTTP endpoint
   dexserver.disable.tls: "false"

docs/operator-manual/server-commands/argocd-repo-server.md

@@ -36,6 +36,8 @@ argocd-repo-server [flags]
       --revision-cache-expiration duration             Cache expiration for cached revision (default 3m0s)
       --sentinel stringArray                           Redis sentinel hostname and port (e.g. argocd-redis-ha-announce-0:6379). 
       --sentinelmaster string                          Redis sentinel master group name. (default "master")
+      --streamed-manifest-max-extracted-size int       Maximum size of streamed manifest archives when extracted (default 1048576000)
+      --streamed-manifest-max-tar-size int             Maximum size of streamed manifest archives (default 104857600)
       --tlsciphers string                              The list of acceptable ciphers to be used when establishing TLS connections. Use 'list' to list available ciphers. (default "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384:TLS_RSA_WITH_AES_256_GCM_SHA384")
       --tlsmaxversion string                           The maximum SSL/TLS version that is acceptable (one of: 1.0|1.1|1.2|1.3) (default "1.3")
       --tlsminversion string                           The minimum SSL/TLS version that is acceptable (one of: 1.0|1.1|1.2|1.3) (default "1.2")

docs/user-guide/commands/argocd_app_diff.md

@@ -15,13 +15,15 @@ argocd app diff APPNAME [flags]
 ### Options
 
 ```
-      --exit-code                Return non-zero exit code when there is a diff (default true)
-      --hard-refresh             Refresh application data as well as target manifests cache
-  -h, --help                     help for diff
-      --local string             Compare live app to a local manifests
-      --local-repo-root string   Path to the repository root. Used together with --local allows setting the repository root (default "/")
-      --refresh                  Refresh application data when retrieving
-      --revision string          Compare live app to a particular revision
+      --exit-code                   Return non-zero exit code when there is a diff (default true)
+      --hard-refresh                Refresh application data as well as target manifests cache
+  -h, --help                        help for diff
+      --local string                Compare live app to a local manifests
+      --local-include stringArray   Used with --server-side-generate, specify patterns of filenames to send (default [*.yaml,*.yml,*.json])
+      --local-repo-root string      Path to the repository root. Used together with --local allows setting the repository root (default "/")
+      --refresh                     Refresh application data when retrieving
+      --revision string             Compare live app to a particular revision
+      --server-side-generate        Used with --local, this will send your manifests to the server for diffing
 ```
 
 ### Options inherited from parent commands

manifests/base/repo-server/argocd-repo-server-deployment.yaml

@@ -119,6 +119,18 @@ spec:
                 key: reposerver.allow.oob.symlinks
                 name: argocd-cmd-params-cm
                 optional: true
+          - name: ARGOCD_REPO_SERVER_STREAMED_MANIFEST_MAX_TAR_SIZE
+            valueFrom:
+              configMapKeyRef:
+                key: reposerver.streamed.manifest.max.tar.size
+                name: argocd-cmd-params-cm
+                optional: true
+          - name: ARGOCD_REPO_SERVER_STREAMED_MANIFEST_MAX_EXTRACTED_SIZE
+            valueFrom:
+              configMapKeyRef:
+                key: reposerver.streamed.manifest.max.extracted.size
+                name: argocd-cmd-params-cm
+                optional: true
           - name: HELM_CACHE_HOME
             value: /helm-working-dir
           - name: HELM_CONFIG_HOME

manifests/core-install.yaml

@@ -9769,6 +9769,18 @@ spec:
               key: reposerver.allow.oob.symlinks
               name: argocd-cmd-params-cm
               optional: true
+        - name: ARGOCD_REPO_SERVER_STREAMED_MANIFEST_MAX_TAR_SIZE
+          valueFrom:
+            configMapKeyRef:
+              key: reposerver.streamed.manifest.max.tar.size
+              name: argocd-cmd-params-cm
+              optional: true
+        - name: ARGOCD_REPO_SERVER_STREAMED_MANIFEST_MAX_EXTRACTED_SIZE
+          valueFrom:
+            configMapKeyRef:
+              key: reposerver.streamed.manifest.max.extracted.size
+              name: argocd-cmd-params-cm
+              optional: true
         - name: HELM_CACHE_HOME
           value: /helm-working-dir
         - name: HELM_CONFIG_HOME

manifests/ha/install.yaml

@@ -10896,6 +10896,18 @@ spec:
               key: reposerver.allow.oob.symlinks
               name: argocd-cmd-params-cm
               optional: true
+        - name: ARGOCD_REPO_SERVER_STREAMED_MANIFEST_MAX_TAR_SIZE
+          valueFrom:
+            configMapKeyRef:
+              key: reposerver.streamed.manifest.max.tar.size
+              name: argocd-cmd-params-cm
+              optional: true
+        - name: ARGOCD_REPO_SERVER_STREAMED_MANIFEST_MAX_EXTRACTED_SIZE
+          valueFrom:
+            configMapKeyRef:
+              key: reposerver.streamed.manifest.max.extracted.size
+              name: argocd-cmd-params-cm
+              optional: true
         - name: HELM_CACHE_HOME
           value: /helm-working-dir
         - name: HELM_CONFIG_HOME

manifests/ha/namespace-install.yaml

@@ -1670,6 +1670,18 @@ spec:
               key: reposerver.allow.oob.symlinks
               name: argocd-cmd-params-cm
               optional: true
+        - name: ARGOCD_REPO_SERVER_STREAMED_MANIFEST_MAX_TAR_SIZE
+          valueFrom:
+            configMapKeyRef:
+              key: reposerver.streamed.manifest.max.tar.size
+              name: argocd-cmd-params-cm
+              optional: true
+        - name: ARGOCD_REPO_SERVER_STREAMED_MANIFEST_MAX_EXTRACTED_SIZE
+          valueFrom:
+            configMapKeyRef:
+              key: reposerver.streamed.manifest.max.extracted.size
+              name: argocd-cmd-params-cm
+              optional: true
         - name: HELM_CACHE_HOME
           value: /helm-working-dir
         - name: HELM_CONFIG_HOME

manifests/install.yaml

@@ -10238,6 +10238,18 @@ spec:
               key: reposerver.allow.oob.symlinks
               name: argocd-cmd-params-cm
               optional: true
+        - name: ARGOCD_REPO_SERVER_STREAMED_MANIFEST_MAX_TAR_SIZE
+          valueFrom:
+            configMapKeyRef:
+              key: reposerver.streamed.manifest.max.tar.size
+              name: argocd-cmd-params-cm
+              optional: true
+        - name: ARGOCD_REPO_SERVER_STREAMED_MANIFEST_MAX_EXTRACTED_SIZE
+          valueFrom:
+            configMapKeyRef:
+              key: reposerver.streamed.manifest.max.extracted.size
+              name: argocd-cmd-params-cm
+              optional: true
         - name: HELM_CACHE_HOME
           value: /helm-working-dir
         - name: HELM_CONFIG_HOME

manifests/namespace-install.yaml

@@ -1012,6 +1012,18 @@ spec:
               key: reposerver.allow.oob.symlinks
               name: argocd-cmd-params-cm
               optional: true
+        - name: ARGOCD_REPO_SERVER_STREAMED_MANIFEST_MAX_TAR_SIZE
+          valueFrom:
+            configMapKeyRef:
+              key: reposerver.streamed.manifest.max.tar.size
+              name: argocd-cmd-params-cm
+              optional: true
+        - name: ARGOCD_REPO_SERVER_STREAMED_MANIFEST_MAX_EXTRACTED_SIZE
+          valueFrom:
+            configMapKeyRef:
+              key: reposerver.streamed.manifest.max.extracted.size
+              name: argocd-cmd-params-cm
+              optional: true
         - name: HELM_CACHE_HOME
           value: /helm-working-dir
         - name: HELM_CONFIG_HOME