fix(storage): Improve storage layer for projects (#507)

* fix(storage): Improve storage layer for projects

Previously, storage did not need to know much about projects. This adds
better support for projects to the storage layer, and adds a
CreateProject function.

* fix(*): fix style errors

brig/cmd/brig/commands/rerun.go

@@ -2,6 +2,7 @@ package commands
 
 import (
 	"errors"
+
 	"github.com/spf13/cobra"
 )
 

brigade-cr-gateway/cmd/brigade-cr-gateway/server_test.go

@@ -12,7 +12,7 @@ import (
 
 func TestNewRouter(t *testing.T) {
 	s := mock.New()
-	s.Project.Name = "pequod/stubbs"
+	s.ProjectList[0].Name = "pequod/stubbs"
 	r := newRouter(s)
 
 	if r == nil {

brigade-worker/yarn.lock

@@ -634,12 +634,6 @@ is-buffer@^1.1.5:
   version "1.1.6"
   resolved "https://registry.yarnpkg.com/is-buffer/-/is-buffer-1.1.6.tgz#efaa2ea9daa0d7ab2ea13a97b2b8ad51fefbe8be"
 
-is-thirteen@2.0.0:
-  version "2.0.0"
-  resolved "https://registry.yarnpkg.com/is-thirteen/-/is-thirteen-2.0.0.tgz#a2dbd0f5ab7e10a4d0186e9a0b24263224d3f9b1"
-  dependencies:
-    noop3 "^13.7.2"
-
 is-typedarray@^1.0.0, is-typedarray@~1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/is-typedarray/-/is-typedarray-1.0.0.tgz#e479c80858df0c1b11ddda6940f96011fcda4a9a"
@@ -881,10 +875,6 @@ nomnom@1.5.2:
     colors "0.5.x"
     underscore "1.1.x"
 
-noop3@^13.7.2:
-  version "13.8.1"
-  resolved "https://registry.yarnpkg.com/noop3/-/noop3-13.8.1.tgz#0ae6414b6d7de3b6d85055cd2a920c1a0d61ad6e"
-
 nth-check@~1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/nth-check/-/nth-check-1.0.1.tgz#9929acdf628fc2c41098deab82ac580cf149aae4"

pkg/brigade/project.go

@@ -32,6 +32,22 @@ type Project struct {
 	Secrets SecretsMap `json:"secrets"`
 	// Worker holds a set of project-specific worker settings which takes precedence over brigade-wide settings
 	Worker WorkerConfig `json:"worker"`
+
+	// InitGitSubmodules initializes Git submodules in VCS if true.
+	InitGitSubmodules bool `json:"initGitSubmodules"`
+
+	// AllowPrivilegedJobs allows jobs to use privileged mode.
+	AllowPrivilegedJobs bool `json:"allowPrivilegedJobs"`
+
+	// AllowHostMounts lets the worker use host mounted volumes
+	AllowHostMounts bool `json:"allowHostMounts"`
+
+	// ImagePullSecrets is a comma-separated list of image pull secrets
+	ImagePullSecrets string `json:"imagePullSecrets"`
+
+	// WorkerCommand is a string command that can be issued to the worker image.
+	// This is an alternative to the 'yarn start' command usually issued.
+	WorkerCommand string `json:"workerCommand"`
 }
 
 // SecretsMap is a map[string]string for storing secrets.
@@ -97,4 +113,8 @@ type Kubernetes struct {
 	VCSSidecar string `json:"vcsSidecar"`
 	// BuildStorageSize is the size of the build shared storage used by the jobs
 	BuildStorageSize string `json:"buildStorageSize"`
+	// BuildStorageCache is the storage class used for build storage.
+	BuildStorageClass string `json:"buildStorageClass"`
+	// CacheStorageClass is the storage class used for caching jobs.
+	CacheStorageClass string `json:"cacheStorageClass"`
 }

pkg/storage/kube/project.go

@@ -2,6 +2,8 @@ package kube
 
 import (
 	"encoding/json"
+	"errors"
+	"fmt"
 	"strings"
 
 	"k8s.io/api/core/v1"
@@ -10,6 +12,8 @@ import (
 	"github.com/Azure/brigade/pkg/brigade"
 )
 
+const secretTypeProject = "brigade.sh/project"
+
 // GetProjects retrieves all projects from storage.
 func (s *store) GetProjects() ([]*brigade.Project, error) {
 	lo := meta.ListOptions{LabelSelector: "app=brigade,component=project"}
@@ -33,6 +37,83 @@ func (s *store) GetProject(id string) (*brigade.Project, error) {
 	return s.loadProjectConfig(brigade.ProjectID(id))
 }
 
+// CreateProject stores a given project.
+//
+// Project Name is a required field. If not present, Project ID will be calculated
+// from project name. This is preferred.
+//
+// Note that project secrets are not redacted.
+func (s *store) CreateProject(project *brigade.Project) error {
+	if project.Name == "" {
+		return errors.New("project name is required")
+	}
+
+	if project.ID == "" {
+		project.ID = brigade.ProjectID(project.Name)
+	}
+
+	// The marshal on SecretsMap redacts secrts, so we cast and marshal as a raw
+	// map[string]string
+	var secrets map[string]string = project.Secrets
+	secretsJSON, err := json.Marshal(secrets)
+	if err != nil {
+		return err
+	}
+
+	bfmt := func(b bool) string { return fmt.Sprintf("%t", b) }
+
+	secret := v1.Secret{
+		ObjectMeta: meta.ObjectMeta{
+			Name: project.ID,
+			Labels: map[string]string{
+				"app":       "brigade",
+				"heritage":  "brigade",
+				"component": "project",
+			},
+			Annotations: map[string]string{
+				"projectName": project.Name,
+			},
+		},
+		Type: secretTypeProject,
+		StringData: map[string]string{
+			"sharedSecret":     project.SharedSecret,
+			"github.token":     project.Github.Token,
+			"github.baseURL":   project.Github.BaseURL,
+			"github.uploadURL": project.Github.UploadURL,
+
+			"vcsSidecar":        project.Kubernetes.VCSSidecar,
+			"namespace":         project.Kubernetes.Namespace,
+			"buildStorageSize":  project.Kubernetes.BuildStorageSize,
+			"defaultScript":     project.DefaultScript,
+			"defaultScriptName": project.DefaultScriptName,
+
+			"repository": project.Repo.Name,
+			"sshKey":     project.Repo.SSHKey,
+			"cloneURL":   project.Repo.CloneURL,
+
+			"secrets": string(secretsJSON),
+
+			"worker.registry":   project.Worker.Registry,
+			"worker.name":       project.Worker.Name,
+			"worker.tag":        project.Worker.Tag,
+			"worker.pullPolicy": project.Worker.PullPolicy,
+
+			// These exist in the chart, but not in the brigade.Project
+			"initGitSubmodules":   bfmt(project.InitGitSubmodules),
+			"imagePullSecrets":    project.ImagePullSecrets,
+			"allowPrivilegedJobs": bfmt(project.AllowPrivilegedJobs),
+			"allowHostMounts":     bfmt(project.AllowHostMounts),
+			"workerCommand":       project.WorkerCommand,
+
+			"kubernetes.cacheStorageClass": project.Kubernetes.CacheStorageClass,
+			"kubernetes.buildStorageClass": project.Kubernetes.BuildStorageClass,
+		},
+	}
+
+	_, err = s.client.CoreV1().Secrets(s.namespace).Create(&secret)
+	return err
+}
+
 // loadProjectConfig loads a project config from inside of Kubernetes.
 //
 // The namespace is the namespace where the secret is stored.
@@ -58,9 +139,12 @@ func NewProjectFromSecret(secret *v1.Secret, namespace string) (*brigade.Project
 	proj.Github.BaseURL = sv.String("github.baseURL")
 	proj.Github.UploadURL = sv.String("github.uploadURL")
 
-	proj.Kubernetes.Namespace = def(sv.String("namespace"), namespace)
 	proj.Kubernetes.VCSSidecar = sv.String("vcsSidecar")
+	proj.Kubernetes.Namespace = def(sv.String("namespace"), namespace)
 	proj.Kubernetes.BuildStorageSize = def(sv.String("buildStorageSize"), "50Mi")
+	proj.Kubernetes.BuildStorageClass = sv.String("kubernetes.buildStorageClass")
+	proj.Kubernetes.CacheStorageClass = sv.String("kubernetes.cacheStorageClass")
+
 	proj.DefaultScript = sv.String("defaultScript")
 	proj.DefaultScriptName = sv.String("defaultScriptName")
 
@@ -85,6 +169,13 @@ func NewProjectFromSecret(secret *v1.Secret, namespace string) (*brigade.Project
 		Tag:        sv.String("worker.tag"),
 		PullPolicy: sv.String("worker.pullPolicy"),
 	}
+
+	// git submodules and host mounts are false by default. Priv jobs are true by default.
+	proj.InitGitSubmodules = strings.ToLower(def(sv.String("initGitSubmodules"), "false")) == "true"
+	proj.AllowPrivilegedJobs = strings.ToLower(def(sv.String("allowPrivilegedJobs"), "true")) == "true"
+	proj.AllowHostMounts = strings.ToLower(def(sv.String("allowHostMounts"), "false")) == "true"
+
+	proj.WorkerCommand = sv.String("workerCommand")
 	return proj, nil
 }