Merge remote-tracking branch 'origin/master' into jelena-redactors

design/redactors-api.md

@@ -0,0 +1,113 @@
+# Individual Redactors API
+
+Add a set of APIs to allow getting/setting redactors by name, not just as a group.
+
+## Goals
+
+- allow listing redactors by name
+- allow editing an individual redactor
+- allow creating/removing an individual redactor
+- allow storing metadata, including enabling/disabling redactors
+
+## Non Goals
+
+- preservation of ordering/comments in the redaction spec
+
+## Background
+
+Custom redactors are being moved to a more informative UI, planned to look something like this:
+![](./images/redactors-api-spec-list.png)
+API routes for individual redactors would greatly simplify implementation and move logic from frontend to backend.
+
+
+## High-Level Design
+
+Two new API routes will be added - `/api/v1/redacts` to list redactor information and GET/POST/DELETE `/api/v1/redact/spec/{slug}` to get/set/delete them by slug.
+'slug' is the name, in lowercase, with ` ` changed to `-` and symbols removed.
+
+The second set of APIs will not operate on full troubleshoot/Redactor documents - instead, it will operate on the [redact spec itself](https://github.com/replicatedhq/troubleshoot/blob/8f594e876470fa72c6fb0389e8f89fabcde9e017/pkg/apis/troubleshoot/v1beta1/redact_shared.go#L8-L16).
+This will result in yaml like the following, which will then be combined when using the existing `/api/v1/redact/get` route:
+```yaml
+name: replace literal string
+values:
+- redact-me-first
+files:
+- literals/a-redact-file
+```
+
+`/api/v1/redact/get` will retain the previous behavior. `/api/v1/redact/set` will still add/update redact specs, but it will do so by splitting it into individual redactors and updating those by name.
+If no name is provided, one will be autogenerated.
+
+## Detailed Design
+
+### Method Details
+
+The `get` method of `/api/v1/redact/spec/{slug}` will return the following struct:
+```go
+type GetRedactorResponse struct {
+	Redactor string       `json:"redactor"`
+	Metadata RedactorList `json:"redactorMetadata"`
+
+	Success bool   `json:"success"`
+	Error   string `json:"error,omitempty"`
+}
+```
+
+`post` to `/api/v1/redact/spec/{slug}` should be of the following form:
+```go
+type PostRedactorMetadata struct {
+	Name        string `json:"name"`
+	Enabled     bool   `json:"enabled"`
+	Description string `json:"description"`
+	New         bool   `json:"new"`
+	Redactor    string `json:"redactor"`
+}
+```
+
+If 'name' changes in the yaml or request body, the name will be changed.
+If 'name' is removed, it will be readded.
+
+If 'new' is set to true, the destination slug is ingored in favor of the name in the request or spec.
+
+It will return `GetRedactorResponse`.
+
+`delete` to `/api/v1/redact/spec/{slug}` will return 200 on success.
+
+Getting `/api/v1/redacts` wil return this:
+
+```go
+type ListRedactorsResponse struct {
+	Redactors []RedactorList `json:"redactors"`
+
+	Success bool   `json:"success"`
+	Error   string `json:"error,omitempty"`
+}
+
+type RedactorList struct {
+	Name        string    `json:"name"`
+	Slug        string    `json:"slug"`
+	Created     time.Time `json:"createdAt"`
+	Updated     time.Time `json:"updatedAt"`
+	Enabled     bool      `json:"enabled"`
+	Description string    `json:"description"`
+}
+```
+
+### Backend Changes
+
+Storing metadata about individual redactors requires additional changes.
+Currently, all redactors are stored as a yaml document at the `kotsadm-redact` key within the `kotsadm-redact` configmap.
+Now, redactors will be stored individually at the `<name>` key within the `kotsadm-redact` configmap.
+Further, instead of storing raw yaml there, an object containing metadata and the redactor will be stored.
+
+```go
+type RedactorMetadata struct {
+	Metadata RedactorList `json:"metadata"`
+
+	Redact v1beta1.Redact `json:"redact"`
+}
+```
+
+## Security Considerations
+
+There are no security considerations for this change.

kotsadm/api/src/kots_app/gitops.ts

@@ -28,6 +28,7 @@ export async function sendInitialGitCommitsForAppDownstream(stores: Stores, appI
   }
 
   const pendingVersions = await stores.kotsAppStore.listPendingVersions(appId, clusterId);
+  pendingVersions.sort((p1, p2) => p1.sequence > p2.sequence ? 1 : p1.sequence < p2.sequence ? -1 : 0); // sort ascending
   for (const pendingVersion of pendingVersions) {
     const commitMessage = `Updating ${app.name} to version ${pendingVersion.sequence}`;
     await createGitCommitForVersion(stores, appId, clusterId, pendingVersion.parentSequence!, commitMessage);

kotsadm/api/src/kots_app/graphql/kots_app_types.ts

@@ -73,6 +73,13 @@ const KotsGitOpsInput = `
   }
 `;
 
+const InstallationYamlError = `
+  type InstallationYamlError {
+    path: String!
+    error: String
+  }
+`
+
 const KotsVersion = `
   type KotsVersion {
     title: String!
@@ -88,16 +95,10 @@ const KotsVersion = `
     preflightResultCreatedAt: String
     commitUrl: String
     gitDeployable: Boolean
+    yamlErrors: [InstallationYamlError]
   }
 `;
 
-const InstallationYamlError = `
-  type InstallationYamlError {
-    path: String!
-    error: String
-  }
-`
-
 // midstream
 const KotsAppVersion = `
   type KotsAppVersion {

kotsadm/api/src/kots_app/kots_app.ts

@@ -608,6 +608,7 @@ export interface KotsVersion {
   diffSummary?: string;
   commitUrl?: string;
   gitDeployable?: boolean;
+  yamlErrors?: InstallationYAMLError[];
 }
 
 export interface AppRegistryDetails {

kotsadm/api/src/kots_app/kots_app_store.ts

@@ -707,9 +707,14 @@ export class KotsAppStore {
          adv.preflight_result_created_at,
          adv.git_commit_url,
          adv.git_deployable,
-         ado.is_error AS has_error
+         ado.is_error AS has_error,
+         av.kots_installation_spec
         FROM
           app_downstream_version AS adv
+        LEFT JOIN
+          app_version AS av
+        ON
+          adv.app_id = av.app_id AND adv.sequence = av.sequence
         LEFT JOIN
           app_downstream_output AS ado
         ON
@@ -748,6 +753,14 @@ export class KotsAppStore {
         commitUrl: row.git_commit_url || "",
         gitDeployable: row.git_deployable
       };
+      if (row.kots_installation_spec) {
+        try {
+          const installationSpec = yaml.safeLoad(row.kots_installation_spec).spec as InstallationSpec;
+          versionItem.yamlErrors = installationSpec.yamlErrors;
+        } catch (err) {
+          console.log(`Failed to unmarshal installation spec yaml for sequence ${versionItem.sequence}`, err);
+        }
+      }
       versionItems.push(versionItem);
     }
 
@@ -772,11 +785,13 @@ export class KotsAppStore {
       sequence = -1;
     }
 
-    q = `select created_at, version_label, status, sequence, parent_sequence,
-applied_at, source, diff_summary, preflight_result, preflight_result_created_at, git_commit_url, git_deployable
-from app_downstream_version
-where app_id = $1 and cluster_id = $3 and sequence > $2
-order by sequence desc`;
+    q = `select adv.created_at, adv.version_label, adv.status, adv.sequence, adv.parent_sequence,
+adv.applied_at, adv.source, adv.diff_summary, adv.preflight_result, adv.preflight_result_created_at, adv.git_commit_url, adv.git_deployable,
+av.kots_installation_spec
+from app_downstream_version as adv
+left join app_version as av on adv.app_id = av.app_id and adv.sequence = av.sequence
+where adv.app_id = $1 and adv.cluster_id = $3 and adv.sequence > $2
+order by adv.sequence desc`;
 
     v = [
       appId,
@@ -805,6 +820,14 @@ order by sequence desc`;
         commitUrl: row.git_commit_url || "",
         gitDeployable: row.git_deployable
       };
+      if (row.kots_installation_spec) {
+        try {
+          const installationSpec = yaml.safeLoad(row.kots_installation_spec).spec as InstallationSpec;
+          versionItem.yamlErrors = installationSpec.yamlErrors;
+        } catch (err) {
+          console.log(`Failed to unmarshal installation spec yaml for sequence ${versionItem.sequence}`, err);
+        }
+      }
       versionItems.push(versionItem);
     }
 
@@ -928,8 +951,10 @@ order by sequence desc`;
 
     q = `select adv.created_at, adv.version_label, adv.status, adv.sequence,
 adv.parent_sequence, adv.applied_at, adv.source, adv.diff_summary, adv.preflight_result,
-adv.preflight_result_created_at, adv.git_commit_url, adv.git_deployable, ado.is_error AS has_error
+adv.preflight_result_created_at, adv.git_commit_url, adv.git_deployable, ado.is_error AS has_error,
+av.kots_installation_spec
 from app_downstream_version as adv
+left join app_version as av on adv.app_id = av.app_id and adv.sequence = av.sequence
 left join app_downstream_output as ado
 on adv.app_id = ado.app_id and adv.cluster_id = ado.cluster_id and adv.sequence = ado.downstream_sequence
 where adv.app_id = $1 and adv.cluster_id = $3 and adv.sequence = $2
@@ -965,6 +990,14 @@ order by adv.sequence desc`;
       commitUrl: row.git_commit_url || "",
       gitDeployable: row.git_deployable
     };
+    if (row.kots_installation_spec) {
+      try {
+        const installationSpec = yaml.safeLoad(row.kots_installation_spec).spec as InstallationSpec;
+        versionItem.yamlErrors = installationSpec.yamlErrors;
+      } catch (err) {
+        console.log(`Failed to unmarshal installation spec yaml for sequence ${versionItem.sequence}`, err);
+      }
+    }
 
     return versionItem;
   }
@@ -1020,7 +1053,7 @@ where app_id = $1 and sequence = $2`;
         const installationSpec = yaml.safeLoad(row.kots_installation_spec).spec as InstallationSpec;
         versionItem.yamlErrors = installationSpec.yamlErrors;
       } catch (err) {
-        console.log("Failed to unmarshal installation spec yaml", err);
+        console.log(`Failed to unmarshal installation spec yaml for sequence ${versionItem.sequence}`, err);
       }
     }
 

kotsadm/kustomize/overlays/dev/minio/statefulset.yaml

@@ -6,6 +6,7 @@ metadata:
     release: minio
   name: minio
 spec:
+  serviceName: minio
   selector:
     matchLabels:
       app: minio

kotsadm/operator/kustomize/base/kustomization.yaml

@@ -1,4 +1,4 @@
 
 resources:
 - ./deployment.yaml
-# - ./rbac.yaml
+- ./rbac.yaml

kotsadm/operator/kustomize/base/rbac.yaml

@@ -20,3 +20,4 @@ roleRef:
 subjects:
 - kind: ServiceAccount
   name: default
+  namespace: default

kotsadm/pkg/gitops/gitops.go

@@ -262,16 +262,24 @@ func CreateGitOpsCommit(gitOpsConfig *GitOpsConfig, appSlug string, appName stri
 		}
 	}
 
-	// if the file has not changed, end now
-	currentRevision, err := ioutil.ReadFile(filepath.Join(workDir, gitOpsConfig.Path, fmt.Sprintf("%s.yaml", appSlug)))
-	if err != nil {
-		return "", errors.Wrap(err, "failed to read current file")
-	}
-	if string(currentRevision) == string(out) {
-		return "", nil
+	filePath := filepath.Join(workDir, gitOpsConfig.Path, fmt.Sprintf("%s.yaml", appSlug))
+	_, err = os.Stat(filePath)
+	if err == nil { // if the file has not changed, end now
+		currentRevision, err := ioutil.ReadFile(filePath)
+		if err != nil {
+			return "", errors.Wrap(err, "failed to read current file")
+		}
+		if string(currentRevision) == string(out) {
+			return "", nil
+		}
+	} else if os.IsNotExist(err) { // create subdirectory if not exist
+		err := os.MkdirAll(filepath.Join(workDir, gitOpsConfig.Path), 0644)
+		if err != nil {
+			return "", errors.Wrap(err, "failed to mkdir for file")
+		}
 	}
 
-	err = ioutil.WriteFile(filepath.Join(workDir, gitOpsConfig.Path, fmt.Sprintf("%s.yaml", appSlug)), out, 0644)
+	err = ioutil.WriteFile(filePath, out, 0644)
 	if err != nil {
 		return "", errors.Wrap(err, "failed to write updated app yaml")
 	}