Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Integrate apply command #175

Merged
merged 9 commits into from
Jun 24, 2020
Merged
Show file tree
Hide file tree
Changes from 8 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
11 changes: 9 additions & 2 deletions cmd/apply.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
package cmd

import (
"log"
"os"

"github.com/commitdev/zero/internal/apply"
"github.com/commitdev/zero/internal/config/projectconfig"
"github.com/commitdev/zero/internal/constants"
Expand All @@ -21,7 +24,11 @@ var applyCmd = &cobra.Command{
Use: "apply",
Short: "Execute modules to create projects, infrastructure, etc.",
Run: func(cmd *cobra.Command, args []string) {
// @TODO rootdir?
apply.Apply(projectconfig.RootDir, applyConfigPath, applyEnvironments)
rootDir, err := os.Getwd()
if err != nil {
log.Println(err)
rootDir = projectconfig.RootDir
}
apply.Apply(rootDir, applyConfigPath, applyEnvironments)
},
}
11 changes: 0 additions & 11 deletions cmd/apply_test.go

This file was deleted.

30 changes: 0 additions & 30 deletions cmd/init_test.go

This file was deleted.

3 changes: 2 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
module github.com/commitdev/zero

go 1.12
go 1.13

require (
github.com/aws/aws-sdk-go v1.25.33
Expand All @@ -24,6 +24,7 @@ require (
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect
github.com/spf13/cobra v0.0.6
github.com/stretchr/testify v1.4.0
github.com/termie/go-shutil v0.0.0-20140729215957-bcacb06fecae
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b // indirect
golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980 // indirect
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect
Expand Down
4 changes: 4 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
github.com/docker/docker v1.13.1 h1:IkZjBSIc8hBjLpqeAbeE5mca5mNgeatLHBy3GO78BWo=
github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
Expand Down Expand Up @@ -142,6 +143,7 @@ github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrk
github.com/mitchellh/go-testing-interface v1.0.0 h1:fzU/JVNcaqHQEcVFAKeR41fkiLdIPrefOvVG1VZ96U0=
github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/moby/moby v1.13.1 h1:mC5WwQwCXt/dYxZ1cIrRsnJAWw7VdtcTZUIGr4tXzOM=
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
Expand Down Expand Up @@ -179,6 +181,8 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/termie/go-shutil v0.0.0-20140729215957-bcacb06fecae h1:vgGSvdW5Lqg+I1aZOlG32uyE6xHpLdKhZzcTEktz5wM=
github.com/termie/go-shutil v0.0.0-20140729215957-bcacb06fecae/go.mod h1:quDq6Se6jlGwiIKia/itDZxqC5rj6/8OdFyMMAwTxCs=
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
github.com/ulikunitz/xz v0.5.5 h1:pFrO0lVpTBXLpYw+pnLj6TbvHuyjXMfjGeCwSqCVwok=
Expand Down
80 changes: 44 additions & 36 deletions internal/apply/apply.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,54 +2,77 @@ package apply

import (
"fmt"
"path/filepath"

"log"
"os/exec"
"path"
"strings"

"github.com/commitdev/zero/internal/module"
"github.com/commitdev/zero/internal/util"
"github.com/commitdev/zero/pkg/util/flog"

"github.com/commitdev/zero/internal/config/globalconfig"
"github.com/commitdev/zero/internal/config/projectconfig"
"github.com/commitdev/zero/pkg/util/exit"
"github.com/commitdev/zero/pkg/util/flog"
"github.com/manifoldco/promptui"
)

// Apply will bootstrap the runtime environment for the project
func Apply(dir string, applyConfigPath string, applyEnvironments []string) []string {
context := loadContext(dir, applyConfigPath, applyEnvironments)
func Apply(rootDir string, configPath string, environments []string) {
if strings.Trim(configPath, " ") == "" {
exit.Fatal("config path cannot be empty!")
}
Comment on lines +23 to +25
Copy link
Contributor

@dtoki dtoki Jun 24, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Would it be better to return the error in this case and let the caller throw the error for consistency like initin #130?

Copy link
Contributor

@dtoki dtoki Jun 24, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nvm, I just realized your comment was specific to if I wanted to test the error case so I guess that doesn't apply here because it's not a standard we're enforcing.

configFilePath := path.Join(rootDir, configPath)
projectConfig := projectconfig.LoadConfig(configFilePath)

if len(environments) == 0 {
fmt.Println(`Choose the environments to apply. This will create infrastructure, CI pipelines, etc.
At this point, real things will be generated that may cost money!
Only a single environment may be suitable for an initial test, but for a real system we suggest setting up both staging and production environments.`)
environments = promptEnvironments()
}

flog.Infof(":tada: Bootstrapping project %s. Please use the zero.[hcl, yaml] file to modify the project as needed. %s.", context.Name)
flog.Infof(":tada: Bootstrapping project %s. Please use the zero-project.yml file to modify the project as needed.", projectConfig.Name)

flog.Infof("Cloud provider: %s", "AWS") // will this come from the config?

flog.Infof("Runtime platform: %s", "Kubernetes")

flog.Infof("Infrastructure executor: %s", "Terraform")

// other details...
applyAll(rootDir, *projectConfig, environments)

return makeAll(dir, context, applyEnvironments)
// TODO Summary
flog.Infof(":check_mark_button: Done - Summary goes here.")
}

// loadContext will load the context/configuration to be used by the apply command
func loadContext(dir string, applyConfigPath string, applyEnvironments []string) *projectconfig.ZeroProjectConfig {
if len(applyEnvironments) == 0 {
fmt.Println(`Choose the environments to apply. This will create infrastructure, CI pipelines, etc.
At this point, real things will be generated that may cost money!
Only a single environment may be suitable for an initial test, but for a real system we suggest setting up both staging and production environments.`)
applyEnvironments = promptEnvironments()
}
func applyAll(dir string, projectConfig projectconfig.ZeroProjectConfig, applyEnvironments []string) {
environmentArg := fmt.Sprintf("ENVIRONMENT=%s", strings.Join(applyEnvironments, ","))

validateEnvironments(applyEnvironments)
// Go through each of the modules and run `make`
for _, mod := range projectConfig.Modules {
// Add env vars for the makefile
envList := []string{
environmentArg,
fmt.Sprintf("PROJECT_DIR=%s", path.Join(dir, mod.Files.Directory)),
fmt.Sprintf("REPOSITORY=%s", mod.Files.Repository),
}

if applyConfigPath == "" {
exit.Fatal("config path cannot be empty!")
modulePath := module.GetSourceDir(mod.Files.Source)
// Passed in `dir` will only be used to find the project path, not the module path,
// unless the module path is relative
if module.IsLocal(mod.Files.Source) && !filepath.IsAbs(modulePath) {
modulePath = filepath.Join(dir, modulePath)
}

// Get project credentials for the makefile
credentials := globalconfig.GetProjectCredentials(projectConfig.Name)

envList = util.AppendProjectEnvToCmdEnv(mod.Parameters, envList)
envList = util.AppendProjectEnvToCmdEnv(credentials.AsEnvVars(), envList)
util.ExecuteCommand(exec.Command("make"), modulePath, envList)
}
configPath := path.Join(dir, applyConfigPath)
projectConfig := projectconfig.LoadConfig(configPath)
return projectConfig
}

// promptEnvironments Prompts the user for the environments to apply against and returns a slice of strings representing the environments
Expand Down Expand Up @@ -85,18 +108,3 @@ func validateEnvironments(applyEnvironments []string) {
}
}
}

func makeAll(dir string, projectContext *projectconfig.ZeroProjectConfig, applyEnvironments []string) []string {
environmentArg := fmt.Sprintf("ENVIRONMENT=%s", strings.Join(applyEnvironments, ","))
envList := []string{environmentArg}
outputs := []string{}

for _, mod := range projectContext.Modules {
modulePath := path.Join(dir, mod.Files.Directory)
envList = util.AppendProjectEnvToCmdEnv(mod.Parameters, envList)

output := util.ExecuteCommandOutput(exec.Command("make"), modulePath, envList)
outputs = append(outputs, output)
}
return outputs
}
32 changes: 23 additions & 9 deletions internal/apply/apply_test.go
Original file line number Diff line number Diff line change
@@ -1,27 +1,41 @@
package apply_test

import (
"io/ioutil"
"os"
"path/filepath"
"testing"

"github.com/commitdev/zero/internal/apply"
"github.com/commitdev/zero/internal/constants"
"github.com/stretchr/testify/assert"
"github.com/termie/go-shutil"
)

func TestApply(t *testing.T) {
// @TODO is there a way to do this without relative paths?
dir := "../../tests/test_data/sample_project/"
dir := "../../tests/test_data/apply/"
applyConfigPath := constants.ZeroProjectYml
applyEnvironments := []string{"staging", "production"}

want := []string{
"make module zero-aws-eks-stack\n",
"make module zero-deployable-backend\n",
"make module zero-deployable-react-frontend\n",
}
tmpDir := filepath.Join(os.TempDir(), "apply")

err := os.RemoveAll(tmpDir)
assert.NoError(t, err)

err = shutil.CopyTree(dir, tmpDir, nil)
assert.NoError(t, err)

t.Run("Should run apply and execute make on each folder module", func(t *testing.T) {
got := apply.Apply(dir, applyConfigPath, applyEnvironments)
assert.ElementsMatch(t, want, got)
apply.Apply(tmpDir, applyConfigPath, applyEnvironments)
assert.FileExists(t, filepath.Join(tmpDir, "project1/project.out"))
assert.FileExists(t, filepath.Join(tmpDir, "project2/project.out"))

content, err := ioutil.ReadFile(filepath.Join(tmpDir, "project1/project.out"))
assert.NoError(t, err)
assert.Equal(t, "foo: bar\nrepo: github.com/commitdev/project1\n", string(content))

content, err = ioutil.ReadFile(filepath.Join(tmpDir, "project2/project.out"))
assert.NoError(t, err)
assert.Equal(t, "baz: qux\n", string(content))
})
}
38 changes: 34 additions & 4 deletions internal/config/globalconfig/global_config.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"os"
"os/user"
"path"
"reflect"

"github.com/commitdev/zero/internal/constants"
"github.com/commitdev/zero/pkg/util/exit"
Expand All @@ -25,14 +26,14 @@ type ProjectCredential struct {
}

type AWSResourceConfig struct {
AccessKeyId string `yaml:"accessKeyId,omitempty"`
SecretAccessKey string `yaml:"secretAccessKey,omitempty"`
AccessKeyID string `yaml:"accessKeyId,omitempty" env:"AWS_ACCESS_KEY_ID"`
SecretAccessKey string `yaml:"secretAccessKey,omitempty" env:"AWS_SECRET_ACCESS_KEY"`
}
type GithubResourceConfig struct {
AccessToken string `yaml:"accessToken,omitempty"`
AccessToken string `yaml:"accessToken,omitempty" env:"GITHUB_ACCESS_TOKEN"`
}
type CircleCiResourceConfig struct {
ApiKey string `yaml:"apiKey,omitempty"`
ApiKey string `yaml:"apiKey,omitempty" env:"CIRCLECI_API_KEY"`
}

func (p ProjectCredentials) Unmarshal(data []byte) error {
Expand All @@ -50,6 +51,35 @@ func (p ProjectCredentials) Unmarshal(data []byte) error {
return nil
}

// AsEnvVars marshals ProjectCredential as a map of key/value strings suitable for environment variables
func (p ProjectCredential) AsEnvVars() map[string]string {
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@davidcheung This should help with #173

t := reflect.ValueOf(p)

list := make(map[string]string)
list = gatherFieldTags(t, list)

return list
}

func gatherFieldTags(t reflect.Value, list map[string]string) map[string]string {
reflectType := t.Type()

for i := 0; i < t.NumField(); i++ {
fieldValue := t.Field(i)
fieldType := reflectType.Field(i)

if fieldType.Type.Kind() == reflect.Struct {
list = gatherFieldTags(fieldValue, list)
continue
}

if env := fieldType.Tag.Get("env"); env != "" {
list[env] = fieldValue.String()
}
}
return list
}

func LoadUserCredentials() ProjectCredentials {
data := readOrCreateUserCredentialsFile()

Expand Down
29 changes: 24 additions & 5 deletions internal/config/globalconfig/global_config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ func TestGetUserCredentials(t *testing.T) {
project := globalconfig.GetProjectCredentials(projectName)

// Reading from fixtures: tests/test_data/configs/credentials.yml
assert.Equal(t, "AKIAABCD", project.AWSResourceConfig.AccessKeyId)
assert.Equal(t, "AKIAABCD", project.AWSResourceConfig.AccessKeyID)
assert.Equal(t, "ZXCV", project.AWSResourceConfig.SecretAccessKey)
assert.Equal(t, "0987", project.GithubResourceConfig.AccessToken)
assert.Equal(t, "SOME_API_KEY", project.CircleCiResourceConfig.ApiKey)
Expand All @@ -87,17 +87,36 @@ func TestEditUserCredentials(t *testing.T) {
t.Run("Should create new project if not exist", func(t *testing.T) {
projectName := "test-project3"
project := globalconfig.GetProjectCredentials(projectName)
project.AWSResourceConfig.AccessKeyId = "TEST_KEY_ID_1"
project.AWSResourceConfig.AccessKeyID = "TEST_KEY_ID_1"
globalconfig.Save(project)
newKeyID := globalconfig.GetProjectCredentials(projectName).AWSResourceConfig.AccessKeyId
newKeyID := globalconfig.GetProjectCredentials(projectName).AWSResourceConfig.AccessKeyID
assert.Equal(t, "TEST_KEY_ID_1", newKeyID)
})
t.Run("Should edit old project if already exist", func(t *testing.T) {
projectName := "my-project"
project := globalconfig.GetProjectCredentials(projectName)
project.AWSResourceConfig.AccessKeyId = "EDITED_ACCESS_KEY_ID"
project.AWSResourceConfig.AccessKeyID = "EDITED_ACCESS_KEY_ID"
globalconfig.Save(project)
newKeyID := globalconfig.GetProjectCredentials(projectName).AWSResourceConfig.AccessKeyId
newKeyID := globalconfig.GetProjectCredentials(projectName).AWSResourceConfig.AccessKeyID
assert.Equal(t, "EDITED_ACCESS_KEY_ID", newKeyID)
})
}

func TestMarshalProjectCredentialAsEnvVars(t *testing.T) {
t.Run("Should be able to marshal a ProjectCredential into env vars", func(t *testing.T) {
pc := globalconfig.ProjectCredential{
AWSResourceConfig: globalconfig.AWSResourceConfig{
AccessKeyID: "AKID",
SecretAccessKey: "SAK",
},
CircleCiResourceConfig: globalconfig.CircleCiResourceConfig{
ApiKey: "APIKEY",
},
}

envVars := pc.AsEnvVars()
assert.Equal(t, "AKID", envVars["AWS_ACCESS_KEY_ID"])
assert.Equal(t, "SAK", envVars["AWS_SECRET_ACCESS_KEY"])
assert.Equal(t, "APIKEY", envVars["CIRCLECI_API_KEY"])
})
}