From cec81be7358fff425e8d5c37c8794dbda01398cc Mon Sep 17 00:00:00 2001 From: aknysh Date: Fri, 16 Feb 2024 14:58:50 -0500 Subject: [PATCH 01/35] updates --- examples/quick-start/Dockerfile | 2 +- .../rootfs/usr/local/etc/atmos/atmos.yaml | 18 ------------------ examples/tests/atmos.yaml | 18 ------------------ .../rootfs/usr/local/etc/atmos/atmos.yaml | 18 ------------------ pkg/atlantis/atmos.yaml | 18 ------------------ pkg/aws/atmos.yaml | 18 ------------------ pkg/component/atmos.yaml | 18 ------------------ pkg/describe/atmos.yaml | 18 ------------------ pkg/generate/atmos.yaml | 18 ------------------ pkg/spacelift/atmos.yaml | 18 ------------------ pkg/validate/atmos.yaml | 18 ------------------ pkg/vender/atmos.yaml | 18 ------------------ pkg/workflow/atmos.yaml | 18 ------------------ 13 files changed, 1 insertion(+), 217 deletions(-) diff --git a/examples/quick-start/Dockerfile b/examples/quick-start/Dockerfile index 0e2d27b17..7a82bae17 100644 --- a/examples/quick-start/Dockerfile +++ b/examples/quick-start/Dockerfile @@ -6,7 +6,7 @@ ARG GEODESIC_OS=debian # https://atmos.tools/ # https://github.com/cloudposse/atmos # https://github.com/cloudposse/atmos/releases -ARG ATMOS_VERSION=1.63.0 +ARG ATMOS_VERSION=1.64.0 # Terraform: https://github.com/hashicorp/terraform/releases ARG TF_VERSION=1.7.3 diff --git a/examples/quick-start/rootfs/usr/local/etc/atmos/atmos.yaml b/examples/quick-start/rootfs/usr/local/etc/atmos/atmos.yaml index 68ec77e90..c6a396642 100644 --- a/examples/quick-start/rootfs/usr/local/etc/atmos/atmos.yaml +++ b/examples/quick-start/rootfs/usr/local/etc/atmos/atmos.yaml @@ -116,24 +116,6 @@ commands: steps: - atmos terraform plan $ATMOS_COMPONENT -s $ATMOS_STACK - atmos terraform apply $ATMOS_COMPONENT -s $ATMOS_STACK - - name: play - description: This command plays games - steps: - - echo Playing... - # subcommands - commands: - - name: hello - description: This command says Hello world - steps: - - echo Hello world - - name: ping - description: This command plays ping-pong - # If 'verbose' is set to 'true', atmos will output some info messages to the console before executing the command's steps - # If 'verbose' is not defined, it implicitly defaults to 'false' - verbose: true - steps: - - echo Playing ping-pong... - - echo pong - name: show description: Execute 'show' commands # subcommands diff --git a/examples/tests/atmos.yaml b/examples/tests/atmos.yaml index 588453048..dc2a754b5 100644 --- a/examples/tests/atmos.yaml +++ b/examples/tests/atmos.yaml @@ -118,24 +118,6 @@ commands: steps: - atmos terraform plan $ATMOS_COMPONENT -s $ATMOS_STACK - atmos terraform apply $ATMOS_COMPONENT -s $ATMOS_STACK - - name: play - description: This command plays games - steps: - - echo Playing... - # subcommands - commands: - - name: hello - description: This command says Hello world - steps: - - echo Hello world - - name: ping - description: This command plays ping-pong - # If 'verbose' is set to 'true', atmos will output some info messages to the console before executing the command's steps - # If 'verbose' is not defined, it implicitly defaults to 'false' - verbose: true - steps: - - echo Playing ping-pong... - - echo pong - name: show description: Execute 'show' commands # subcommands diff --git a/examples/tests/rootfs/usr/local/etc/atmos/atmos.yaml b/examples/tests/rootfs/usr/local/etc/atmos/atmos.yaml index b4de87b70..31c8b77af 100644 --- a/examples/tests/rootfs/usr/local/etc/atmos/atmos.yaml +++ b/examples/tests/rootfs/usr/local/etc/atmos/atmos.yaml @@ -116,24 +116,6 @@ commands: steps: - atmos terraform plan $ATMOS_COMPONENT -s $ATMOS_STACK - atmos terraform apply $ATMOS_COMPONENT -s $ATMOS_STACK - - name: play - description: This command plays games - steps: - - echo Playing... - # subcommands - commands: - - name: hello - description: This command says Hello world - steps: - - echo Hello world - - name: ping - description: This command plays ping-pong - # If 'verbose' is set to 'true', atmos will output some info messages to the console before executing the command's steps - # If 'verbose' is not defined, it implicitly defaults to 'false' - verbose: true - steps: - - echo Playing ping-pong... - - echo pong - name: show description: Execute 'show' commands # subcommands diff --git a/pkg/atlantis/atmos.yaml b/pkg/atlantis/atmos.yaml index cd56e2835..6c8c44166 100644 --- a/pkg/atlantis/atmos.yaml +++ b/pkg/atlantis/atmos.yaml @@ -116,24 +116,6 @@ commands: steps: - atmos terraform plan $ATMOS_COMPONENT -s $ATMOS_STACK - atmos terraform apply $ATMOS_COMPONENT -s $ATMOS_STACK - - name: play - description: This command plays games - steps: - - echo Playing... - # subcommands - commands: - - name: hello - description: This command says Hello world - steps: - - echo Hello world - - name: ping - description: This command plays ping-pong - # If 'verbose' is set to 'true', atmos will output some info messages to the console before executing the command's steps - # If 'verbose' is not defined, it implicitly defaults to 'false' - verbose: true - steps: - - echo Playing ping-pong... - - echo pong - name: show description: Execute 'show' commands # subcommands diff --git a/pkg/aws/atmos.yaml b/pkg/aws/atmos.yaml index 6200f88ae..667c4445e 100644 --- a/pkg/aws/atmos.yaml +++ b/pkg/aws/atmos.yaml @@ -116,24 +116,6 @@ commands: steps: - atmos terraform plan $ATMOS_COMPONENT -s $ATMOS_STACK - atmos terraform apply $ATMOS_COMPONENT -s $ATMOS_STACK - - name: play - description: This command plays games - steps: - - echo Playing... - # subcommands - commands: - - name: hello - description: This command says Hello world - steps: - - echo Hello world - - name: ping - description: This command plays ping-pong - # If 'verbose' is set to 'true', atmos will output some info messages to the console before executing the command's steps - # If 'verbose' is not defined, it implicitly defaults to 'false' - verbose: true - steps: - - echo Playing ping-pong... - - echo pong - name: show description: Execute 'show' commands # subcommands diff --git a/pkg/component/atmos.yaml b/pkg/component/atmos.yaml index cd56e2835..6c8c44166 100644 --- a/pkg/component/atmos.yaml +++ b/pkg/component/atmos.yaml @@ -116,24 +116,6 @@ commands: steps: - atmos terraform plan $ATMOS_COMPONENT -s $ATMOS_STACK - atmos terraform apply $ATMOS_COMPONENT -s $ATMOS_STACK - - name: play - description: This command plays games - steps: - - echo Playing... - # subcommands - commands: - - name: hello - description: This command says Hello world - steps: - - echo Hello world - - name: ping - description: This command plays ping-pong - # If 'verbose' is set to 'true', atmos will output some info messages to the console before executing the command's steps - # If 'verbose' is not defined, it implicitly defaults to 'false' - verbose: true - steps: - - echo Playing ping-pong... - - echo pong - name: show description: Execute 'show' commands # subcommands diff --git a/pkg/describe/atmos.yaml b/pkg/describe/atmos.yaml index cd56e2835..6c8c44166 100644 --- a/pkg/describe/atmos.yaml +++ b/pkg/describe/atmos.yaml @@ -116,24 +116,6 @@ commands: steps: - atmos terraform plan $ATMOS_COMPONENT -s $ATMOS_STACK - atmos terraform apply $ATMOS_COMPONENT -s $ATMOS_STACK - - name: play - description: This command plays games - steps: - - echo Playing... - # subcommands - commands: - - name: hello - description: This command says Hello world - steps: - - echo Hello world - - name: ping - description: This command plays ping-pong - # If 'verbose' is set to 'true', atmos will output some info messages to the console before executing the command's steps - # If 'verbose' is not defined, it implicitly defaults to 'false' - verbose: true - steps: - - echo Playing ping-pong... - - echo pong - name: show description: Execute 'show' commands # subcommands diff --git a/pkg/generate/atmos.yaml b/pkg/generate/atmos.yaml index cd56e2835..6c8c44166 100644 --- a/pkg/generate/atmos.yaml +++ b/pkg/generate/atmos.yaml @@ -116,24 +116,6 @@ commands: steps: - atmos terraform plan $ATMOS_COMPONENT -s $ATMOS_STACK - atmos terraform apply $ATMOS_COMPONENT -s $ATMOS_STACK - - name: play - description: This command plays games - steps: - - echo Playing... - # subcommands - commands: - - name: hello - description: This command says Hello world - steps: - - echo Hello world - - name: ping - description: This command plays ping-pong - # If 'verbose' is set to 'true', atmos will output some info messages to the console before executing the command's steps - # If 'verbose' is not defined, it implicitly defaults to 'false' - verbose: true - steps: - - echo Playing ping-pong... - - echo pong - name: show description: Execute 'show' commands # subcommands diff --git a/pkg/spacelift/atmos.yaml b/pkg/spacelift/atmos.yaml index cd56e2835..6c8c44166 100644 --- a/pkg/spacelift/atmos.yaml +++ b/pkg/spacelift/atmos.yaml @@ -116,24 +116,6 @@ commands: steps: - atmos terraform plan $ATMOS_COMPONENT -s $ATMOS_STACK - atmos terraform apply $ATMOS_COMPONENT -s $ATMOS_STACK - - name: play - description: This command plays games - steps: - - echo Playing... - # subcommands - commands: - - name: hello - description: This command says Hello world - steps: - - echo Hello world - - name: ping - description: This command plays ping-pong - # If 'verbose' is set to 'true', atmos will output some info messages to the console before executing the command's steps - # If 'verbose' is not defined, it implicitly defaults to 'false' - verbose: true - steps: - - echo Playing ping-pong... - - echo pong - name: show description: Execute 'show' commands # subcommands diff --git a/pkg/validate/atmos.yaml b/pkg/validate/atmos.yaml index cd56e2835..6c8c44166 100644 --- a/pkg/validate/atmos.yaml +++ b/pkg/validate/atmos.yaml @@ -116,24 +116,6 @@ commands: steps: - atmos terraform plan $ATMOS_COMPONENT -s $ATMOS_STACK - atmos terraform apply $ATMOS_COMPONENT -s $ATMOS_STACK - - name: play - description: This command plays games - steps: - - echo Playing... - # subcommands - commands: - - name: hello - description: This command says Hello world - steps: - - echo Hello world - - name: ping - description: This command plays ping-pong - # If 'verbose' is set to 'true', atmos will output some info messages to the console before executing the command's steps - # If 'verbose' is not defined, it implicitly defaults to 'false' - verbose: true - steps: - - echo Playing ping-pong... - - echo pong - name: show description: Execute 'show' commands # subcommands diff --git a/pkg/vender/atmos.yaml b/pkg/vender/atmos.yaml index cd56e2835..6c8c44166 100644 --- a/pkg/vender/atmos.yaml +++ b/pkg/vender/atmos.yaml @@ -116,24 +116,6 @@ commands: steps: - atmos terraform plan $ATMOS_COMPONENT -s $ATMOS_STACK - atmos terraform apply $ATMOS_COMPONENT -s $ATMOS_STACK - - name: play - description: This command plays games - steps: - - echo Playing... - # subcommands - commands: - - name: hello - description: This command says Hello world - steps: - - echo Hello world - - name: ping - description: This command plays ping-pong - # If 'verbose' is set to 'true', atmos will output some info messages to the console before executing the command's steps - # If 'verbose' is not defined, it implicitly defaults to 'false' - verbose: true - steps: - - echo Playing ping-pong... - - echo pong - name: show description: Execute 'show' commands # subcommands diff --git a/pkg/workflow/atmos.yaml b/pkg/workflow/atmos.yaml index cd56e2835..6c8c44166 100644 --- a/pkg/workflow/atmos.yaml +++ b/pkg/workflow/atmos.yaml @@ -116,24 +116,6 @@ commands: steps: - atmos terraform plan $ATMOS_COMPONENT -s $ATMOS_STACK - atmos terraform apply $ATMOS_COMPONENT -s $ATMOS_STACK - - name: play - description: This command plays games - steps: - - echo Playing... - # subcommands - commands: - - name: hello - description: This command says Hello world - steps: - - echo Hello world - - name: ping - description: This command plays ping-pong - # If 'verbose' is set to 'true', atmos will output some info messages to the console before executing the command's steps - # If 'verbose' is not defined, it implicitly defaults to 'false' - verbose: true - steps: - - echo Playing ping-pong... - - echo pong - name: show description: Execute 'show' commands # subcommands From f293ef5677c8a702004a6bcf1ff3d3f89180422f Mon Sep 17 00:00:00 2001 From: aknysh Date: Sun, 21 Apr 2024 10:02:14 -0400 Subject: [PATCH 02/35] updates --- examples/quick-start/Dockerfile | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- website/docs/integrations/atlantis.mdx | 2 +- .../integrations/github-actions/setup-atmos.md | 2 +- website/package-lock.json | 16 ++++++++-------- website/package.json | 4 ++-- 7 files changed, 17 insertions(+), 17 deletions(-) diff --git a/examples/quick-start/Dockerfile b/examples/quick-start/Dockerfile index 6891f4e63..cabb8e746 100644 --- a/examples/quick-start/Dockerfile +++ b/examples/quick-start/Dockerfile @@ -6,10 +6,10 @@ ARG GEODESIC_OS=debian # https://atmos.tools/ # https://github.com/cloudposse/atmos # https://github.com/cloudposse/atmos/releases -ARG ATMOS_VERSION=1.70.0 +ARG ATMOS_VERSION=1.71.0 # Terraform: https://github.com/hashicorp/terraform/releases -ARG TF_VERSION=1.8.0 +ARG TF_VERSION=1.8.1 FROM cloudposse/geodesic:${GEODESIC_VERSION}-${GEODESIC_OS} diff --git a/go.mod b/go.mod index 6c932a3d3..1718271a8 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/google/go-github/v59 v59.0.0 github.com/google/uuid v1.6.0 github.com/hairyhenderson/gomplate/v3 v3.11.7 - github.com/hashicorp/go-getter v1.7.3 + github.com/hashicorp/go-getter v1.7.4 github.com/hashicorp/hcl v1.0.0 github.com/hashicorp/hcl/v2 v2.20.1 github.com/hashicorp/terraform-config-inspect v0.0.0-20231204233900-a34142ec2a72 diff --git a/go.sum b/go.sum index 507703aa5..d9125ad92 100644 --- a/go.sum +++ b/go.sum @@ -726,8 +726,8 @@ github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtng github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= -github.com/hashicorp/go-getter v1.7.3 h1:bN2+Fw9XPFvOCjB0UOevFIMICZ7G2XSQHzfvLUyOM5E= -github.com/hashicorp/go-getter v1.7.3/go.mod h1:W7TalhMmbPmsSMdNjD0ZskARur/9GJ17cfHTRtXV744= +github.com/hashicorp/go-getter v1.7.4 h1:3yQjWuxICvSpYwqSayAdKRFcvBl1y/vogCxczWSmix0= +github.com/hashicorp/go-getter v1.7.4/go.mod h1:W7TalhMmbPmsSMdNjD0ZskARur/9GJ17cfHTRtXV744= github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= github.com/hashicorp/go-hclog v0.14.1/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-hclog v0.16.2/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= diff --git a/website/docs/integrations/atlantis.mdx b/website/docs/integrations/atlantis.mdx index 33e43372b..0869aa893 100644 --- a/website/docs/integrations/atlantis.mdx +++ b/website/docs/integrations/atlantis.mdx @@ -686,7 +686,7 @@ on: branches: [ main ] env: - ATMOS_VERSION: 1.70.0 + ATMOS_VERSION: 1.71.0 ATMOS_CLI_CONFIG_PATH: ./ jobs: diff --git a/website/docs/integrations/github-actions/setup-atmos.md b/website/docs/integrations/github-actions/setup-atmos.md index 1adc73ce6..d8a31dce6 100644 --- a/website/docs/integrations/github-actions/setup-atmos.md +++ b/website/docs/integrations/github-actions/setup-atmos.md @@ -27,5 +27,5 @@ jobs: uses: cloudposse/github-action-setup-atmos with: # Make sure to pin to the latest version of atmos - atmos_version: 1.70.0 + atmos_version: 1.71.0 ``` diff --git a/website/package-lock.json b/website/package-lock.json index fa41cca6f..ddf3e1efe 100644 --- a/website/package-lock.json +++ b/website/package-lock.json @@ -19,12 +19,12 @@ "custom-loaders": "file:plugins/custom-loaders", "docusaurus-plugin-image-zoom": "^2.0.0", "html-loader": "^5.0.0", - "marked": "^12.0.1", + "marked": "^12.0.2", "prism-react-renderer": "^2.3.1", "react": "^18.2.0", "react-dom": "^18.2.0", "react-image-gallery": "^1.3.0", - "react-player": "^2.15.1", + "react-player": "^2.16.0", "react-table": "^7.8.0", "remark-html": "^16.0.1", "remark-parse": "^11.0.0", @@ -9234,9 +9234,9 @@ } }, "node_modules/marked": { - "version": "12.0.1", - "resolved": "https://registry.npmjs.org/marked/-/marked-12.0.1.tgz", - "integrity": "sha512-Y1/V2yafOcOdWQCX0XpAKXzDakPOpn6U0YLxTJs3cww6VxOzZV1BTOOYWLvH3gX38cq+iLwljHHTnMtlDfg01Q==", + "version": "12.0.2", + "resolved": "https://registry.npmjs.org/marked/-/marked-12.0.2.tgz", + "integrity": "sha512-qXUm7e/YKFoqFPYPa3Ukg9xlI5cyAtGmyEIzMfW//m6kXwCy2Ps9DYf5ioijFKQ8qyuscrHoY04iJGctu2Kg0Q==", "bin": { "marked": "bin/marked.js" }, @@ -14544,9 +14544,9 @@ } }, "node_modules/react-player": { - "version": "2.15.1", - "resolved": "https://registry.npmjs.org/react-player/-/react-player-2.15.1.tgz", - "integrity": "sha512-ni1XFuYZuhIKKdeFII+KRLmIPcvCYlyXvtSMhNOgssdfnSovmakBtBTW2bxowPvmpKy5BTR4jC4CF79ucgHT+g==", + "version": "2.16.0", + "resolved": "https://registry.npmjs.org/react-player/-/react-player-2.16.0.tgz", + "integrity": "sha512-mAIPHfioD7yxO0GNYVFD1303QFtI3lyyQZLY229UEAp/a10cSW+hPcakg0Keq8uWJxT2OiT/4Gt+Lc9bD6bJmQ==", "dependencies": { "deepmerge": "^4.0.0", "load-script": "^1.0.0", diff --git a/website/package.json b/website/package.json index ac92a467a..a83400053 100644 --- a/website/package.json +++ b/website/package.json @@ -25,12 +25,12 @@ "custom-loaders": "file:plugins/custom-loaders", "docusaurus-plugin-image-zoom": "^2.0.0", "html-loader": "^5.0.0", - "marked": "^12.0.1", + "marked": "^12.0.2", "prism-react-renderer": "^2.3.1", "react": "^18.2.0", "react-dom": "^18.2.0", "react-image-gallery": "^1.3.0", - "react-player": "^2.15.1", + "react-player": "^2.16.0", "react-table": "^7.8.0", "remark-html": "^16.0.1", "remark-parse": "^11.0.0", From abd0a222a0328c155aadfa47247c64196e70cdc3 Mon Sep 17 00:00:00 2001 From: aknysh Date: Tue, 23 Apr 2024 17:55:50 -0400 Subject: [PATCH 03/35] updates --- cmd/describe_affected.go | 2 +- internal/exec/atlantis_generate_repo_config.go | 10 +++++----- pkg/utils/glob_utils.go | 1 - 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/cmd/describe_affected.go b/cmd/describe_affected.go index 7f83ecf7f..0f0523368 100644 --- a/cmd/describe_affected.go +++ b/cmd/describe_affected.go @@ -30,7 +30,7 @@ func init() { describeAffectedCmd.PersistentFlags().String("repo-path", "", "Filesystem path to the already cloned target repository with which to compare the current branch: atmos describe affected --repo-path ") describeAffectedCmd.PersistentFlags().String("ref", "", "Git reference with which to compare the current branch: atmos describe affected --ref refs/heads/main. Refer to https://git-scm.com/book/en/v2/Git-Internals-Git-References for more details") describeAffectedCmd.PersistentFlags().String("sha", "", "Git commit SHA with which to compare the current branch: atmos describe affected --sha 3a5eafeab90426bd82bf5899896b28cc0bab3073") - describeAffectedCmd.PersistentFlags().String("file", "", "Write the result to the file: atmos describe affected --ref refs/tags/v1.16.0 --file affected.json") + describeAffectedCmd.PersistentFlags().String("file", "", "Write the result to the file: atmos describe affected --ref refs/tags/v1.71.0 --file affected.json") describeAffectedCmd.PersistentFlags().String("format", "json", "The output format: atmos describe affected --format=json|yaml ('json' is default)") describeAffectedCmd.PersistentFlags().Bool("verbose", false, "Print more detailed output when cloning and checking out the Git repository: atmos describe affected --verbose=true") describeAffectedCmd.PersistentFlags().String("ssh-key", "", "Path to PEM-encoded private key to clone private repos using SSH: atmos describe affected --ssh-key ") diff --git a/internal/exec/atlantis_generate_repo_config.go b/internal/exec/atlantis_generate_repo_config.go index 5e4ed771e..5669526e9 100644 --- a/internal/exec/atlantis_generate_repo_config.go +++ b/internal/exec/atlantis_generate_repo_config.go @@ -319,6 +319,11 @@ func ExecuteAtlantisGenerateRepoConfig( return err } + // Base component is required to calculate terraform workspace for derived components + if terraformComponent != componentName { + context.BaseComponent = terraformComponent + } + configAndStacksInfo := schema.ConfigAndStacksInfo{ ComponentFromArg: componentName, Stack: stackConfigFileName, @@ -334,11 +339,6 @@ func ExecuteAtlantisGenerateRepoConfig( } // Calculate terraform workspace - // Base component is required to calculate terraform workspace for derived components - if terraformComponent != componentName { - context.BaseComponent = terraformComponent - } - workspace, err := BuildTerraformWorkspace(cliConfig, configAndStacksInfo) if err != nil { return err diff --git a/pkg/utils/glob_utils.go b/pkg/utils/glob_utils.go index 3b3ad925b..902a6a0c3 100644 --- a/pkg/utils/glob_utils.go +++ b/pkg/utils/glob_utils.go @@ -52,7 +52,6 @@ func GetGlobMatches(pattern string) ([]string, error) { // assumes that both `pattern` and `name` are using the system's path // separator. If you can't be sure of that, use filepath.ToSlash() on both // `pattern` and `name`, and then use the Match() function instead. -// func PathMatch(pattern, name string) (bool, error) { return doublestar.PathMatch(pattern, name) } From 0b7281bb2cb15ccda2de9f7faf211966261bbb7c Mon Sep 17 00:00:00 2001 From: aknysh Date: Wed, 24 Apr 2024 18:06:39 -0400 Subject: [PATCH 04/35] updates --- cmd/describe_affected.go | 2 ++ .../commands/describe/describe-affected.mdx | 30 ++++++++++--------- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/cmd/describe_affected.go b/cmd/describe_affected.go index 0f0523368..ce0709921 100644 --- a/cmd/describe_affected.go +++ b/cmd/describe_affected.go @@ -36,6 +36,8 @@ func init() { describeAffectedCmd.PersistentFlags().String("ssh-key", "", "Path to PEM-encoded private key to clone private repos using SSH: atmos describe affected --ssh-key ") describeAffectedCmd.PersistentFlags().String("ssh-key-password", "", "Encryption password for the PEM-encoded private key if the key contains a password-encrypted PEM block: atmos describe affected --ssh-key --ssh-key-password ") describeAffectedCmd.PersistentFlags().Bool("include-spacelift-admin-stacks", false, "Include the Spacelift admin stack of any stack that is affected by config changes: atmos describe affected --include-spacelift-admin-stacks=true") + describeAffectedCmd.PersistentFlags().Bool("clone-target-ref", false, "Clone the target reference with which to compare the current branch: atmos describe affected --clone-target-ref=true. "+ + "If set to 'false' (default), the target reference will be checked out instead (this means that the target reference must be already cloned by Git in the '.git' directory)") describeCmd.AddCommand(describeAffectedCmd) } diff --git a/website/docs/cli/commands/describe/describe-affected.mdx b/website/docs/cli/commands/describe/describe-affected.mdx index f96c94a57..990e28e4f 100644 --- a/website/docs/cli/commands/describe/describe-affected.mdx +++ b/website/docs/cli/commands/describe/describe-affected.mdx @@ -93,6 +93,7 @@ atmos describe affected --ssh-key atmos describe affected --ssh-key --ssh-key-password atmos describe affected --repo-path atmos describe affected --include-spacelift-admin-stacks=true +atmos describe affected --clone-target-ref=true ``` @@ -181,17 +182,18 @@ Affected components and stacks: ## Flags -| Flag | Description | Required | -|:-----------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------|:---------| -| `--ref` | [Git Reference](https://git-scm.com/book/en/v2/Git-Internals-Git-References) with which to compare the current working branch | no | -| `--sha` | Git commit SHA with which to compare the current working branch | no | -| `--file` | If specified, write the result to the file | no | -| `--format` | Specify the output format: `json` or `yaml` (`json` is default) | no | -| `--ssh-key` | Path to PEM-encoded private key to clone private repos using SSH | no | -| `--ssh-key-password` | Encryption password for the PEM-encoded private key if the key contains
a password-encrypted PEM block | no | -| `--repo-path` | Path to the already cloned target repository with which to compare the current branch.
Conflicts with `--ref`, `--sha`, `--ssh-key` and `--ssh-key-password` | no | -| `--verbose` | Print more detailed output when cloning and checking out the target
Git repository and processing the result | no | -| `--include-spacelift-admin-stacks` | Include the Spacelift admin stack of any stack
that is affected by config changes | no | +| Flag | Description | Required | +|:-----------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:---------| +| `--ref` | [Git Reference](https://git-scm.com/book/en/v2/Git-Internals-Git-References) with which to compare the current working branch | no | +| `--sha` | Git commit SHA with which to compare the current working branch | no | +| `--file` | If specified, write the result to the file | no | +| `--format` | Specify the output format: `json` or `yaml` (`json` is default) | no | +| `--ssh-key` | Path to PEM-encoded private key to clone private repos using SSH | no | +| `--ssh-key-password` | Encryption password for the PEM-encoded private key if the key contains
a password-encrypted PEM block | no | +| `--repo-path` | Path to the already cloned target repository with which to compare the current branch.
Conflicts with `--ref`, `--sha`, `--ssh-key` and `--ssh-key-password` | no | +| `--verbose` | Print more detailed output when cloning and checking out the target
Git repository and processing the result | no | +| `--include-spacelift-admin-stacks` | Include the Spacelift admin stack of any stack
that is affected by config changes | no | +| `--clone-target-ref` | Clone the target reference with which to compare the current branch.
`atmos describe affected --clone-target-ref=true`

If set to `false` (default), the target reference will be checked out instead
(this means that the target reference must be already cloned by Git
in the `.git` directory) | no | ## Output @@ -328,11 +330,11 @@ where: ```yaml title="stacks/orgs/cp/tenant1/_defaults.yaml" settings: spacelift: - # All Spacelift child stacks for the `tenant1` tenant are managed by the + # All Spacelift child stacks for the `tenant1` tenant are managed by the # `tenant1-ue2-prod-infrastructure-tenant1` Spacelift admin stack. - # The `admin_stack_selector` attribute is used to find the affected Spacelift + # The `admin_stack_selector` attribute is used to find the affected Spacelift # admin stack for each affected Atmos stack - # when executing the command + # when executing the command # `atmos describe affected --include-spacelift-admin-stacks=true` admin_stack_selector: component: infrastructure-tenant1 From 228fc526800ece5dd1ab1c52f2f5f94d505b8204 Mon Sep 17 00:00:00 2001 From: aknysh Date: Wed, 24 Apr 2024 22:01:14 -0400 Subject: [PATCH 05/35] updates --- cmd/atlantis_generate_repo_config.go | 5 +++ .../exec/atlantis_generate_repo_config.go | 13 +++++-- internal/exec/describe_affected.go | 11 ++++-- internal/exec/describe_affected_utils.go | 13 +++---- .../atlantis_generate_repo_config_test.go | 1 + pkg/describe/describe_affected_test.go | 2 +- .../atlantis-generate-repo-config.mdx | 34 +++++++++++-------- 7 files changed, 51 insertions(+), 28 deletions(-) diff --git a/cmd/atlantis_generate_repo_config.go b/cmd/atlantis_generate_repo_config.go index 9296b4726..1a1cab44c 100644 --- a/cmd/atlantis_generate_repo_config.go +++ b/cmd/atlantis_generate_repo_config.go @@ -57,5 +57,10 @@ func init() { atlantisGenerateRepoConfigCmd.PersistentFlags().String("ssh-key", "", "Path to PEM-encoded private key to clone private repos using SSH: atmos atlantis generate repo-config --affected-only=true --ssh-key ") atlantisGenerateRepoConfigCmd.PersistentFlags().String("ssh-key-password", "", "Encryption password for the PEM-encoded private key if the key contains a password-encrypted PEM block: atmos atlantis generate repo-config --affected-only=true --ssh-key --ssh-key-password ") + atlantisGenerateCmd.PersistentFlags().Bool("clone-target-ref", false, "Clone the target reference with which to compare the current branch: "+ + "atmos atlantis generate repo-config --affected-only=true --clone-target-ref=true\n"+ + "The flag is only used when '--affected-only=true'\n"+ + "If set to 'false' (default), the target reference will be checked out instead (this means that the target reference must be already cloned by Git in the '.git' directory)") + atlantisGenerateCmd.AddCommand(atlantisGenerateRepoConfigCmd) } diff --git a/internal/exec/atlantis_generate_repo_config.go b/internal/exec/atlantis_generate_repo_config.go index 5669526e9..6cbec4d93 100644 --- a/internal/exec/atlantis_generate_repo_config.go +++ b/internal/exec/atlantis_generate_repo_config.go @@ -102,6 +102,11 @@ func ExecuteAtlantisGenerateRepoConfigCmd(cmd *cobra.Command, args []string) err // If the flag `--affected-only=true` is passed, find the affected components and stacks if affectedOnly { + cloneTargetRef, err := flags.GetBool("clone-target-ref") + if err != nil { + return err + } + return ExecuteAtlantisGenerateRepoConfigAffectedOnly( cliConfig, outputPath, @@ -113,6 +118,7 @@ func ExecuteAtlantisGenerateRepoConfigCmd(cmd *cobra.Command, args []string) err sshKeyPath, sshKeyPassword, verbose, + cloneTargetRef, ) } @@ -138,6 +144,7 @@ func ExecuteAtlantisGenerateRepoConfigAffectedOnly( sshKeyPath string, sshKeyPassword string, verbose bool, + cloneTargetRef bool, ) error { if repoPath != "" && (ref != "" || sha != "" || sshKeyPath != "" || sshKeyPassword != "") { return errors.New("if the '--repo-path' flag is specified, the '--ref', '--sha', '--ssh-key' and '--ssh-key-password' flags can't be used") @@ -146,10 +153,10 @@ func ExecuteAtlantisGenerateRepoConfigAffectedOnly( var affected []schema.Affected var err error - if repoPath == "" { - affected, err = ExecuteDescribeAffectedWithTargetRepoClone(cliConfig, ref, sha, sshKeyPath, sshKeyPassword, verbose, false) - } else { + if repoPath != "" { affected, err = ExecuteDescribeAffectedWithTargetRepoPath(cliConfig, repoPath, verbose, false) + } else { + affected, err = ExecuteDescribeAffectedWithTargetRepoCloneOrCheckout(cliConfig, ref, sha, sshKeyPath, sshKeyPassword, verbose, cloneTargetRef, false) } if err != nil { diff --git a/internal/exec/describe_affected.go b/internal/exec/describe_affected.go index 1f92a9e90..d6510ca94 100644 --- a/internal/exec/describe_affected.go +++ b/internal/exec/describe_affected.go @@ -79,15 +79,20 @@ func ExecuteDescribeAffectedCmd(cmd *cobra.Command, args []string) error { return err } + cloneTargetRef, err := flags.GetBool("clone-target-ref") + if err != nil { + return err + } + if repoPath != "" && (ref != "" || sha != "" || sshKeyPath != "" || sshKeyPassword != "") { return errors.New("if the '--repo-path' flag is specified, the '--ref', '--sha', '--ssh-key' and '--ssh-key-password' flags can't be used") } var affected []schema.Affected - if repoPath == "" { - affected, err = ExecuteDescribeAffectedWithTargetRepoClone(cliConfig, ref, sha, sshKeyPath, sshKeyPassword, verbose, includeSpaceliftAdminStacks) - } else { + if repoPath != "" { affected, err = ExecuteDescribeAffectedWithTargetRepoPath(cliConfig, repoPath, verbose, includeSpaceliftAdminStacks) + } else { + affected, err = ExecuteDescribeAffectedWithTargetRepoCloneOrCheckout(cliConfig, ref, sha, sshKeyPath, sshKeyPassword, verbose, cloneTargetRef, includeSpaceliftAdminStacks) } if err != nil { diff --git a/internal/exec/describe_affected_utils.go b/internal/exec/describe_affected_utils.go index 046729989..d0ce88c9f 100644 --- a/internal/exec/describe_affected_utils.go +++ b/internal/exec/describe_affected_utils.go @@ -27,15 +27,16 @@ var ( remoteRepoIsNotGitRepoError = errors.New("the target remote repo is not a Git repository. Check that it was initialized and has '.git' folder") ) -// ExecuteDescribeAffectedWithTargetRepoClone clones the remote repo using `ref` or `sha`, and processes stack configs -// and returns a list of the affected Atmos components and stacks given two Git commits -func ExecuteDescribeAffectedWithTargetRepoClone( +// ExecuteDescribeAffectedWithTargetRepoCloneOrCheckout clones or checks out the remote repo using `ref` or `sha`, +// processes stack configs, and returns a list of the affected Atmos components and stacks given two Git commits +func ExecuteDescribeAffectedWithTargetRepoCloneOrCheckout( cliConfig schema.CliConfiguration, ref string, sha string, sshKeyPath string, sshKeyPassword string, verbose bool, + cloneTargetRef bool, includeSpaceliftAdminStacks bool, ) ([]schema.Affected, error) { @@ -192,7 +193,7 @@ func ExecuteDescribeAffectedWithTargetRepoClone( // and returns a list of the affected Atmos components and stacks given two Git commits func ExecuteDescribeAffectedWithTargetRepoPath( cliConfig schema.CliConfiguration, - repoPath string, + targetRepoPath string, verbose bool, includeSpaceliftAdminStacks bool, ) ([]schema.Affected, error) { @@ -218,7 +219,7 @@ func ExecuteDescribeAffectedWithTargetRepoPath( localRepoPath := localRepoWorktree.Filesystem.Root() - remoteRepo, err := git.PlainOpenWithOptions(repoPath, &git.PlainOpenOptions{ + remoteRepo, err := git.PlainOpenWithOptions(targetRepoPath, &git.PlainOpenOptions{ DetectDotGit: false, EnableDotGitCommonDir: false, }) @@ -232,7 +233,7 @@ func ExecuteDescribeAffectedWithTargetRepoPath( return nil, errors.Wrapf(err, "%v", remoteRepoIsNotGitRepoError) } - affected, err := executeDescribeAffected(cliConfig, localRepoPath, repoPath, localRepo, remoteRepo, verbose, includeSpaceliftAdminStacks) + affected, err := executeDescribeAffected(cliConfig, localRepoPath, targetRepoPath, localRepo, remoteRepo, verbose, includeSpaceliftAdminStacks) if err != nil { return nil, err } diff --git a/pkg/atlantis/atlantis_generate_repo_config_test.go b/pkg/atlantis/atlantis_generate_repo_config_test.go index ff99bcf50..ecafe72dd 100644 --- a/pkg/atlantis/atlantis_generate_repo_config_test.go +++ b/pkg/atlantis/atlantis_generate_repo_config_test.go @@ -87,6 +87,7 @@ func TestExecuteAtlantisGenerateRepoConfigAffectedOnly(t *testing.T) { "", "", "", + false, true, ) diff --git a/pkg/describe/describe_affected_test.go b/pkg/describe/describe_affected_test.go index 2913e9c12..4160daaa7 100644 --- a/pkg/describe/describe_affected_test.go +++ b/pkg/describe/describe_affected_test.go @@ -28,7 +28,7 @@ func TestDescribeAffectedWithTargetRepoClone(t *testing.T) { ref := "refs/heads/master" sha := "" - affected, err := e.ExecuteDescribeAffectedWithTargetRepoClone(cliConfig, ref, sha, "", "", true, true) + affected, err := e.ExecuteDescribeAffectedWithTargetRepoCloneOrCheckout(cliConfig, ref, sha, "", "", true, true, true) assert.Nil(t, err) affectedYaml, err := yaml.Marshal(affected) diff --git a/website/docs/cli/commands/atlantis/atlantis-generate-repo-config.mdx b/website/docs/cli/commands/atlantis/atlantis-generate-repo-config.mdx index 95ce595f5..d893e29a2 100644 --- a/website/docs/cli/commands/atlantis/atlantis-generate-repo-config.mdx +++ b/website/docs/cli/commands/atlantis/atlantis-generate-repo-config.mdx @@ -5,6 +5,7 @@ sidebar_class_name: command id: generate-repo-config description: Use this command to generate a repository configuration for Atlantis. --- + import Screengrab from '@site/src/components/Screengrab' :::info Purpose @@ -13,7 +14,7 @@ Use this command to generate a repository configuration for Atlantis.
- +

@@ -64,24 +65,27 @@ atmos atlantis generate repo-config --affected-only=true --ref refs/tags/v1.2.0 atmos atlantis generate repo-config --affected-only=true --ssh-key atmos atlantis generate repo-config --affected-only=true --ssh-key --ssh-key-password + +atmos atlantis generate repo-config --affected-only=true --clone-target-ref=true ``` ## Flags -| Flag | Description | Required | -|:---------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------|:---------| -| `--config-template` | Atlantis config template name | no | -| `--project-template` | Atlantis project template name | no | -| `--output-path` | Output path to write `atlantis.yaml` file | no | -| `--stacks` | Generate Atlantis projects for the specified stacks only (comma-separated values) | no | -| `--components` | Generate Atlantis projects for the specified components only (comma-separated values) | no | -| `--affected-only` | Generate Atlantis projects only for the Atmos components changed
between two Git commits | no | -| `--ref` | [Git Reference](https://git-scm.com/book/en/v2/Git-Internals-Git-References) with which to compare the current working branch | no | -| `--sha` | Git commit SHA with which to compare the current working branch | no | -| `--ssh-key` | Path to PEM-encoded private key to clone private repos using SSH | no | -| `--ssh-key-password` | Encryption password for the PEM-encoded private key if the key contains
a password-encrypted PEM block | no | -| `--repo-path` | Path to the already cloned target repository with which to compare the current branch.
Conflicts with `--ref`, `--sha`, `--ssh-key` and `--ssh-key-password` | no | -| `--verbose` | Print more detailed output when cloning and checking out the target
Git repository and processing the result | no | +| Flag | Description | Required | +|:---------------------|:----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:---------| +| `--config-template` | Atlantis config template name | no | +| `--project-template` | Atlantis project template name | no | +| `--output-path` | Output path to write `atlantis.yaml` file | no | +| `--stacks` | Generate Atlantis projects for the specified stacks only (comma-separated values) | no | +| `--components` | Generate Atlantis projects for the specified components only (comma-separated values) | no | +| `--affected-only` | Generate Atlantis projects only for the Atmos components changed
between two Git commits | no | +| `--ref` | [Git Reference](https://git-scm.com/book/en/v2/Git-Internals-Git-References) with which to compare the current working branch | no | +| `--sha` | Git commit SHA with which to compare the current working branch | no | +| `--ssh-key` | Path to PEM-encoded private key to clone private repos using SSH | no | +| `--ssh-key-password` | Encryption password for the PEM-encoded private key if the key contains
a password-encrypted PEM block | no | +| `--repo-path` | Path to the already cloned target repository with which to compare the current branch.
Conflicts with `--ref`, `--sha`, `--ssh-key` and `--ssh-key-password` | no | +| `--verbose` | Print more detailed output when cloning and checking out the target
Git repository and processing the result | no | +| `--clone-target-ref` | Clone the target reference with which to compare the current branch.
`atmos atlantis generate repo-config --affected-only=true --clone-target-ref=true`

The flag is only used when `--affected-only=true`.

If set to `false` (default), the target reference will be checked out instead
(this means that the target reference must be already cloned by Git in the `.git` directory) | no |
From f81a289cbf7fc70b793df4b7f664024c4fba5ab0 Mon Sep 17 00:00:00 2001 From: aknysh Date: Wed, 24 Apr 2024 23:16:55 -0400 Subject: [PATCH 06/35] updates --- .../exec/atlantis_generate_repo_config.go | 4 +- internal/exec/describe_affected.go | 5 +- internal/exec/describe_affected_utils.go | 152 +++++++++++++++++- pkg/describe/describe_affected_test.go | 2 +- 4 files changed, 152 insertions(+), 11 deletions(-) diff --git a/internal/exec/atlantis_generate_repo_config.go b/internal/exec/atlantis_generate_repo_config.go index 6cbec4d93..90424f7b7 100644 --- a/internal/exec/atlantis_generate_repo_config.go +++ b/internal/exec/atlantis_generate_repo_config.go @@ -155,8 +155,10 @@ func ExecuteAtlantisGenerateRepoConfigAffectedOnly( if repoPath != "" { affected, err = ExecuteDescribeAffectedWithTargetRepoPath(cliConfig, repoPath, verbose, false) + } else if cloneTargetRef { + affected, err = ExecuteDescribeAffectedWithTargetRefClone(cliConfig, ref, sha, sshKeyPath, sshKeyPassword, verbose, false) } else { - affected, err = ExecuteDescribeAffectedWithTargetRepoCloneOrCheckout(cliConfig, ref, sha, sshKeyPath, sshKeyPassword, verbose, cloneTargetRef, false) + affected, err = ExecuteDescribeAffectedWithTargetRefCheckout(cliConfig, ref, sha, verbose, false) } if err != nil { diff --git a/internal/exec/describe_affected.go b/internal/exec/describe_affected.go index d6510ca94..2ef7e5a8f 100644 --- a/internal/exec/describe_affected.go +++ b/internal/exec/describe_affected.go @@ -89,10 +89,13 @@ func ExecuteDescribeAffectedCmd(cmd *cobra.Command, args []string) error { } var affected []schema.Affected + if repoPath != "" { affected, err = ExecuteDescribeAffectedWithTargetRepoPath(cliConfig, repoPath, verbose, includeSpaceliftAdminStacks) + } else if cloneTargetRef { + affected, err = ExecuteDescribeAffectedWithTargetRefClone(cliConfig, ref, sha, sshKeyPath, sshKeyPassword, verbose, includeSpaceliftAdminStacks) } else { - affected, err = ExecuteDescribeAffectedWithTargetRepoCloneOrCheckout(cliConfig, ref, sha, sshKeyPath, sshKeyPassword, verbose, cloneTargetRef, includeSpaceliftAdminStacks) + affected, err = ExecuteDescribeAffectedWithTargetRefCheckout(cliConfig, ref, sha, verbose, includeSpaceliftAdminStacks) } if err != nil { diff --git a/internal/exec/describe_affected_utils.go b/internal/exec/describe_affected_utils.go index d0ce88c9f..f72b444f9 100644 --- a/internal/exec/describe_affected_utils.go +++ b/internal/exec/describe_affected_utils.go @@ -27,16 +27,15 @@ var ( remoteRepoIsNotGitRepoError = errors.New("the target remote repo is not a Git repository. Check that it was initialized and has '.git' folder") ) -// ExecuteDescribeAffectedWithTargetRepoCloneOrCheckout clones or checks out the remote repo using `ref` or `sha`, +// ExecuteDescribeAffectedWithTargetRefClone clones the remote reference, // processes stack configs, and returns a list of the affected Atmos components and stacks given two Git commits -func ExecuteDescribeAffectedWithTargetRepoCloneOrCheckout( +func ExecuteDescribeAffectedWithTargetRefClone( cliConfig schema.CliConfiguration, ref string, sha string, sshKeyPath string, sshKeyPassword string, verbose bool, - cloneTargetRef bool, includeSpaceliftAdminStacks bool, ) ([]schema.Affected, error) { @@ -152,9 +151,146 @@ func ExecuteDescribeAffectedWithTargetRepoCloneOrCheckout( } if ref != "" { - u.LogTrace(cliConfig, fmt.Sprintf("\nChecked out Git ref '%s'\n", ref)) + u.LogTrace(cliConfig, fmt.Sprintf("\nCloned Git ref '%s'\n", ref)) } else { - u.LogTrace(cliConfig, fmt.Sprintf("\nChecked out Git ref '%s'\n", remoteRepoHead.Name())) + u.LogTrace(cliConfig, fmt.Sprintf("\nCloned Git ref '%s'\n", remoteRepoHead.Name())) + } + + // Check if a commit SHA was provided and checkout the repo at that commit SHA + if sha != "" { + u.LogTrace(cliConfig, fmt.Sprintf("\nChecking out commit SHA '%s' ...\n", sha)) + + w, err := remoteRepo.Worktree() + if err != nil { + return nil, err + } + + checkoutOptions := git.CheckoutOptions{ + Hash: plumbing.NewHash(sha), + Create: false, + Force: true, + Keep: false, + } + + err = w.Checkout(&checkoutOptions) + if err != nil { + return nil, err + } + + u.LogTrace(cliConfig, fmt.Sprintf("\nChecked out commit SHA '%s'\n", sha)) + } + + affected, err := executeDescribeAffected(cliConfig, localRepoPath, tempDir, localRepo, remoteRepo, verbose, includeSpaceliftAdminStacks) + if err != nil { + return nil, err + } + + return affected, nil +} + +// ExecuteDescribeAffectedWithTargetRefCheckout checks out the target reference, +// processes stack configs, and returns a list of the affected Atmos components and stacks given two Git commits +func ExecuteDescribeAffectedWithTargetRefCheckout( + cliConfig schema.CliConfiguration, + ref string, + sha string, + verbose bool, + includeSpaceliftAdminStacks bool, +) ([]schema.Affected, error) { + + localRepo, err := git.PlainOpenWithOptions(".", &git.PlainOpenOptions{ + DetectDotGit: true, + EnableDotGitCommonDir: false, + }) + if err != nil { + return nil, errors.Wrapf(err, "%v", localRepoIsNotGitRepoError) + } + + // Get the Git config of the local repo + localRepoConfig, err := localRepo.Config() + if err != nil { + return nil, errors.Wrapf(err, "%v", localRepoIsNotGitRepoError) + } + + localRepoWorktree, err := localRepo.Worktree() + if err != nil { + return nil, errors.Wrapf(err, "%v", localRepoIsNotGitRepoError) + } + + localRepoPath := localRepoWorktree.Filesystem.Root() + + // Get the remotes of the local repo + keys := []string{} + for k := range localRepoConfig.Remotes { + keys = append(keys, k) + } + + if len(keys) == 0 { + return nil, localRepoIsNotGitRepoError + } + + // Get the origin URL of the current remoteRepo + remoteUrls := localRepoConfig.Remotes[keys[0]].URLs + if len(remoteUrls) == 0 { + return nil, localRepoIsNotGitRepoError + } + + repoUrl := remoteUrls[0] + if repoUrl == "" { + return nil, localRepoIsNotGitRepoError + } + + // Clone the remote repo + // https://git-scm.com/book/en/v2/Git-Internals-Git-References + // https://git-scm.com/docs/git-show-ref + // https://github.com/go-git/go-git/tree/master/_examples + // https://stackoverflow.com/questions/56810719/how-to-checkout-a-specific-sha-in-a-git-repo-using-golang + // https://golang.hotexamples.com/examples/gopkg.in.src-d.go-git.v4.plumbing/-/ReferenceName/golang-referencename-function-examples.html + // https://stackoverflow.com/questions/58905690/how-to-identify-which-files-have-changed-between-git-commits + // https://github.com/src-d/go-git/issues/604 + + // Create a temp dir to clone the remote repo to + tempDir, err := os.MkdirTemp("", strconv.FormatInt(time.Now().Unix(), 10)) + if err != nil { + return nil, err + } + + defer removeTempDir(cliConfig, tempDir) + + u.LogTrace(cliConfig, fmt.Sprintf("\nCloning repo '%s' into the temp dir '%s'", repoUrl, tempDir)) + + cloneOptions := git.CloneOptions{ + URL: repoUrl, + NoCheckout: false, + SingleBranch: false, + } + + // If `ref` flag is not provided, it will clone the HEAD of the default branch + if ref != "" { + cloneOptions.ReferenceName = plumbing.ReferenceName(ref) + u.LogTrace(cliConfig, fmt.Sprintf("\nChecking out Git ref '%s' ...\n", ref)) + } else { + u.LogTrace(cliConfig, "\nChecking out the HEAD of the default branch ...\n") + } + + if verbose { + cloneOptions.Progress = os.Stdout + } + + remoteRepo, err := git.PlainClone(tempDir, false, &cloneOptions) + if err != nil { + return nil, err + } + + remoteRepoHead, err := remoteRepo.Head() + if err != nil { + return nil, err + } + + if ref != "" { + u.LogTrace(cliConfig, fmt.Sprintf("\nCloned Git ref '%s'\n", ref)) + } else { + u.LogTrace(cliConfig, fmt.Sprintf("\nCloned Git ref '%s'\n", remoteRepoHead.Name())) } // Check if a commit SHA was provided and checkout the repo at that commit SHA @@ -193,7 +329,7 @@ func ExecuteDescribeAffectedWithTargetRepoCloneOrCheckout( // and returns a list of the affected Atmos components and stacks given two Git commits func ExecuteDescribeAffectedWithTargetRepoPath( cliConfig schema.CliConfiguration, - targetRepoPath string, + targetRefPath string, verbose bool, includeSpaceliftAdminStacks bool, ) ([]schema.Affected, error) { @@ -219,7 +355,7 @@ func ExecuteDescribeAffectedWithTargetRepoPath( localRepoPath := localRepoWorktree.Filesystem.Root() - remoteRepo, err := git.PlainOpenWithOptions(targetRepoPath, &git.PlainOpenOptions{ + remoteRepo, err := git.PlainOpenWithOptions(targetRefPath, &git.PlainOpenOptions{ DetectDotGit: false, EnableDotGitCommonDir: false, }) @@ -233,7 +369,7 @@ func ExecuteDescribeAffectedWithTargetRepoPath( return nil, errors.Wrapf(err, "%v", remoteRepoIsNotGitRepoError) } - affected, err := executeDescribeAffected(cliConfig, localRepoPath, targetRepoPath, localRepo, remoteRepo, verbose, includeSpaceliftAdminStacks) + affected, err := executeDescribeAffected(cliConfig, localRepoPath, targetRefPath, localRepo, remoteRepo, verbose, includeSpaceliftAdminStacks) if err != nil { return nil, err } diff --git a/pkg/describe/describe_affected_test.go b/pkg/describe/describe_affected_test.go index 4160daaa7..0994494a2 100644 --- a/pkg/describe/describe_affected_test.go +++ b/pkg/describe/describe_affected_test.go @@ -28,7 +28,7 @@ func TestDescribeAffectedWithTargetRepoClone(t *testing.T) { ref := "refs/heads/master" sha := "" - affected, err := e.ExecuteDescribeAffectedWithTargetRepoCloneOrCheckout(cliConfig, ref, sha, "", "", true, true, true) + affected, err := e.ExecuteDescribeAffectedWithTargetRefClone(cliConfig, ref, sha, "", "", true, true) assert.Nil(t, err) affectedYaml, err := yaml.Marshal(affected) From 1a7db6529ec2734bee2b02a16541eeb68e97c5ee Mon Sep 17 00:00:00 2001 From: aknysh Date: Thu, 25 Apr 2024 00:16:54 -0400 Subject: [PATCH 07/35] updates --- internal/exec/describe_affected_utils.go | 120 +++++++---------------- pkg/describe/describe_affected_test.go | 27 ++++- 2 files changed, 63 insertions(+), 84 deletions(-) diff --git a/internal/exec/describe_affected_utils.go b/internal/exec/describe_affected_utils.go index f72b444f9..e286f7707 100644 --- a/internal/exec/describe_affected_utils.go +++ b/internal/exec/describe_affected_utils.go @@ -2,6 +2,7 @@ package exec import ( "fmt" + cp "github.com/otiai10/copy" "os" "path" "path/filepath" @@ -39,7 +40,9 @@ func ExecuteDescribeAffectedWithTargetRefClone( includeSpaceliftAdminStacks bool, ) ([]schema.Affected, error) { - localRepo, err := git.PlainOpenWithOptions(".", &git.PlainOpenOptions{ + localPath := "." + + localRepo, err := git.PlainOpenWithOptions(localPath, &git.PlainOpenOptions{ DetectDotGit: true, EnableDotGitCommonDir: false, }) @@ -198,7 +201,9 @@ func ExecuteDescribeAffectedWithTargetRefCheckout( includeSpaceliftAdminStacks bool, ) ([]schema.Affected, error) { - localRepo, err := git.PlainOpenWithOptions(".", &git.PlainOpenOptions{ + localPath := "." + + localRepo, err := git.PlainOpenWithOptions(localPath, &git.PlainOpenOptions{ DetectDotGit: true, EnableDotGitCommonDir: false, }) @@ -206,8 +211,8 @@ func ExecuteDescribeAffectedWithTargetRefCheckout( return nil, errors.Wrapf(err, "%v", localRepoIsNotGitRepoError) } - // Get the Git config of the local repo - localRepoConfig, err := localRepo.Config() + // Check the Git config of the local repo + _, err = localRepo.Config() if err != nil { return nil, errors.Wrapf(err, "%v", localRepoIsNotGitRepoError) } @@ -219,37 +224,7 @@ func ExecuteDescribeAffectedWithTargetRefCheckout( localRepoPath := localRepoWorktree.Filesystem.Root() - // Get the remotes of the local repo - keys := []string{} - for k := range localRepoConfig.Remotes { - keys = append(keys, k) - } - - if len(keys) == 0 { - return nil, localRepoIsNotGitRepoError - } - - // Get the origin URL of the current remoteRepo - remoteUrls := localRepoConfig.Remotes[keys[0]].URLs - if len(remoteUrls) == 0 { - return nil, localRepoIsNotGitRepoError - } - - repoUrl := remoteUrls[0] - if repoUrl == "" { - return nil, localRepoIsNotGitRepoError - } - - // Clone the remote repo - // https://git-scm.com/book/en/v2/Git-Internals-Git-References - // https://git-scm.com/docs/git-show-ref - // https://github.com/go-git/go-git/tree/master/_examples - // https://stackoverflow.com/questions/56810719/how-to-checkout-a-specific-sha-in-a-git-repo-using-golang - // https://golang.hotexamples.com/examples/gopkg.in.src-d.go-git.v4.plumbing/-/ReferenceName/golang-referencename-function-examples.html - // https://stackoverflow.com/questions/58905690/how-to-identify-which-files-have-changed-between-git-commits - // https://github.com/src-d/go-git/issues/604 - - // Create a temp dir to clone the remote repo to + // Create a temp dir for the target ref tempDir, err := os.MkdirTemp("", strconv.FormatInt(time.Now().Unix(), 10)) if err != nil { return nil, err @@ -257,64 +232,41 @@ func ExecuteDescribeAffectedWithTargetRefCheckout( defer removeTempDir(cliConfig, tempDir) - u.LogTrace(cliConfig, fmt.Sprintf("\nCloning repo '%s' into the temp dir '%s'", repoUrl, tempDir)) + // Copy the local repo into the temp directory + copyOptions := cp.Options{ + // Preserve the atime and the mtime of the entries + // On linux we can preserve only up to 1 millisecond accuracy + PreserveTimes: false, - cloneOptions := git.CloneOptions{ - URL: repoUrl, - NoCheckout: false, - SingleBranch: false, - } + // Preserve the uid and the gid of all entries + PreserveOwner: false, - // If `ref` flag is not provided, it will clone the HEAD of the default branch - if ref != "" { - cloneOptions.ReferenceName = plumbing.ReferenceName(ref) - u.LogTrace(cliConfig, fmt.Sprintf("\nChecking out Git ref '%s' ...\n", ref)) - } else { - u.LogTrace(cliConfig, "\nChecking out the HEAD of the default branch ...\n") - } + // OnSymlink specifies what to do on symlink + // Override the destination file if it already exists + OnSymlink: func(src string) cp.SymlinkAction { + return cp.Deep + }, - if verbose { - cloneOptions.Progress = os.Stdout + // Specials includes special files to be copied. default false. + Specials: true, } - remoteRepo, err := git.PlainClone(tempDir, false, &cloneOptions) - if err != nil { + if err = cp.Copy(localRepoPath, tempDir, copyOptions); err != nil { return nil, err } - remoteRepoHead, err := remoteRepo.Head() + remoteRepo, err := git.PlainOpenWithOptions(tempDir, &git.PlainOpenOptions{ + DetectDotGit: false, + EnableDotGitCommonDir: false, + }) if err != nil { - return nil, err - } - - if ref != "" { - u.LogTrace(cliConfig, fmt.Sprintf("\nCloned Git ref '%s'\n", ref)) - } else { - u.LogTrace(cliConfig, fmt.Sprintf("\nCloned Git ref '%s'\n", remoteRepoHead.Name())) + return nil, errors.Wrapf(err, "%v", remoteRepoIsNotGitRepoError) } - // Check if a commit SHA was provided and checkout the repo at that commit SHA - if sha != "" { - u.LogTrace(cliConfig, fmt.Sprintf("\nChecking out commit SHA '%s' ...\n", sha)) - - w, err := remoteRepo.Worktree() - if err != nil { - return nil, err - } - - checkoutOptions := git.CheckoutOptions{ - Hash: plumbing.NewHash(sha), - Create: false, - Force: true, - Keep: false, - } - - err = w.Checkout(&checkoutOptions) - if err != nil { - return nil, err - } - - u.LogTrace(cliConfig, fmt.Sprintf("\nChecked out commit SHA '%s'\n", sha)) + // Check the Git config of the target ref + _, err = remoteRepo.Config() + if err != nil { + return nil, errors.Wrapf(err, "%v", remoteRepoIsNotGitRepoError) } affected, err := executeDescribeAffected(cliConfig, localRepoPath, tempDir, localRepo, remoteRepo, verbose, includeSpaceliftAdminStacks) @@ -334,7 +286,9 @@ func ExecuteDescribeAffectedWithTargetRepoPath( includeSpaceliftAdminStacks bool, ) ([]schema.Affected, error) { - localRepo, err := git.PlainOpenWithOptions(".", &git.PlainOpenOptions{ + localPath := "." + + localRepo, err := git.PlainOpenWithOptions(localPath, &git.PlainOpenOptions{ DetectDotGit: true, EnableDotGitCommonDir: false, }) diff --git a/pkg/describe/describe_affected_test.go b/pkg/describe/describe_affected_test.go index 0994494a2..8812e50a5 100644 --- a/pkg/describe/describe_affected_test.go +++ b/pkg/describe/describe_affected_test.go @@ -12,7 +12,7 @@ import ( "github.com/cloudposse/atmos/pkg/schema" ) -func TestDescribeAffectedWithTargetRepoClone(t *testing.T) { +func TestDescribeAffectedWithTargetRefClone(t *testing.T) { configAndStacksInfo := schema.ConfigAndStacksInfo{} cliConfig, err := cfg.InitCliConfig(configAndStacksInfo, true) @@ -37,6 +37,31 @@ func TestDescribeAffectedWithTargetRepoClone(t *testing.T) { t.Log(fmt.Sprintf("\nAffected components and stacks:\n%v", string(affectedYaml))) } +func TestDescribeAffectedWithTargetRefCheckout(t *testing.T) { + configAndStacksInfo := schema.ConfigAndStacksInfo{} + + cliConfig, err := cfg.InitCliConfig(configAndStacksInfo, true) + assert.Nil(t, err) + + // We are using `atmos.yaml` from this dir. This `atmos.yaml` has set base_path: "../../examples/tests", + // which will be wrong for the remote repo which is cloned into a temp dir. + // Set the correct base path for the cloned remote repo + cliConfig.BasePath = "./examples/tests" + + // Git reference and commit SHA + // Refer to https://git-scm.com/book/en/v2/Git-Internals-Git-References for more details + ref := "refs/heads/master" + sha := "" + + affected, err := e.ExecuteDescribeAffectedWithTargetRefCheckout(cliConfig, ref, sha, true, true) + assert.Nil(t, err) + + affectedYaml, err := yaml.Marshal(affected) + assert.Nil(t, err) + + t.Log(fmt.Sprintf("\nAffected components and stacks:\n%v", string(affectedYaml))) +} + func TestDescribeAffectedWithTargetRepoPath(t *testing.T) { configAndStacksInfo := schema.ConfigAndStacksInfo{} From d6f21217f5faccc860ec8a86da9984579fdba193 Mon Sep 17 00:00:00 2001 From: aknysh Date: Thu, 25 Apr 2024 00:48:28 -0400 Subject: [PATCH 08/35] updates --- internal/exec/describe_affected_utils.go | 69 +++++++++++++++++++----- 1 file changed, 56 insertions(+), 13 deletions(-) diff --git a/internal/exec/describe_affected_utils.go b/internal/exec/describe_affected_utils.go index e286f7707..f6a425322 100644 --- a/internal/exec/describe_affected_utils.go +++ b/internal/exec/describe_affected_utils.go @@ -40,6 +40,10 @@ func ExecuteDescribeAffectedWithTargetRefClone( includeSpaceliftAdminStacks bool, ) ([]schema.Affected, error) { + if verbose { + cliConfig.Logs.Level = u.LogLevelTrace + } + localPath := "." localRepo, err := git.PlainOpenWithOptions(localPath, &git.PlainOpenOptions{ @@ -201,6 +205,10 @@ func ExecuteDescribeAffectedWithTargetRefCheckout( includeSpaceliftAdminStacks bool, ) ([]schema.Affected, error) { + if verbose { + cliConfig.Logs.Level = u.LogLevelTrace + } + localPath := "." localRepo, err := git.PlainOpenWithOptions(localPath, &git.PlainOpenOptions{ @@ -233,28 +241,19 @@ func ExecuteDescribeAffectedWithTargetRefCheckout( defer removeTempDir(cliConfig, tempDir) // Copy the local repo into the temp directory + u.LogTrace(cliConfig, fmt.Sprintf("\nCopying the local repo into temp directory '%s' ...", tempDir)) + copyOptions := cp.Options{ - // Preserve the atime and the mtime of the entries - // On linux we can preserve only up to 1 millisecond accuracy PreserveTimes: false, - - // Preserve the uid and the gid of all entries PreserveOwner: false, - - // OnSymlink specifies what to do on symlink - // Override the destination file if it already exists - OnSymlink: func(src string) cp.SymlinkAction { - return cp.Deep - }, - - // Specials includes special files to be copied. default false. - Specials: true, } if err = cp.Copy(localRepoPath, tempDir, copyOptions); err != nil { return nil, err } + u.LogTrace(cliConfig, fmt.Sprintf("Copied the local repo into temp directory '%s' ...", tempDir)) + remoteRepo, err := git.PlainOpenWithOptions(tempDir, &git.PlainOpenOptions{ DetectDotGit: false, EnableDotGitCommonDir: false, @@ -269,6 +268,50 @@ func ExecuteDescribeAffectedWithTargetRefCheckout( return nil, errors.Wrapf(err, "%v", remoteRepoIsNotGitRepoError) } + if ref != "" { + u.LogTrace(cliConfig, fmt.Sprintf("\nChecking out Git ref '%s' ...", ref)) + + w, err := remoteRepo.Worktree() + if err != nil { + return nil, err + } + + checkoutOptions := git.CheckoutOptions{ + Branch: plumbing.ReferenceName(ref), + Create: false, + Force: true, + Keep: false, + } + + err = w.Checkout(&checkoutOptions) + if err != nil { + return nil, err + } + + u.LogTrace(cliConfig, fmt.Sprintf("Checked out Git ref '%s'\n", ref)) + } else if sha != "" { + u.LogTrace(cliConfig, fmt.Sprintf("\nChecking out commit SHA '%s' ...\n", sha)) + + w, err := remoteRepo.Worktree() + if err != nil { + return nil, err + } + + checkoutOptions := git.CheckoutOptions{ + Hash: plumbing.NewHash(sha), + Create: false, + Force: true, + Keep: false, + } + + err = w.Checkout(&checkoutOptions) + if err != nil { + return nil, err + } + + u.LogTrace(cliConfig, fmt.Sprintf("Checked out commit SHA '%s'\n", sha)) + } + affected, err := executeDescribeAffected(cliConfig, localRepoPath, tempDir, localRepo, remoteRepo, verbose, includeSpaceliftAdminStacks) if err != nil { return nil, err From 185aeeeeba054265beef68fa2d5d73b932874609 Mon Sep 17 00:00:00 2001 From: aknysh Date: Thu, 25 Apr 2024 01:13:11 -0400 Subject: [PATCH 09/35] updates --- internal/exec/describe_affected_utils.go | 31 +++++++++++++++--------- pkg/describe/describe_affected_test.go | 2 +- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/internal/exec/describe_affected_utils.go b/internal/exec/describe_affected_utils.go index f6a425322..7c506ee08 100644 --- a/internal/exec/describe_affected_utils.go +++ b/internal/exec/describe_affected_utils.go @@ -116,9 +116,9 @@ func ExecuteDescribeAffectedWithTargetRefClone( // If `ref` flag is not provided, it will clone the HEAD of the default branch if ref != "" { cloneOptions.ReferenceName = plumbing.ReferenceName(ref) - u.LogTrace(cliConfig, fmt.Sprintf("\nChecking out Git ref '%s' ...\n", ref)) + u.LogTrace(cliConfig, fmt.Sprintf("\nCloning Git ref '%s' ...\n", ref)) } else { - u.LogTrace(cliConfig, "\nChecking out the HEAD of the default branch ...\n") + u.LogTrace(cliConfig, "\nCloned the HEAD of the default branch ...\n") } if verbose { @@ -252,7 +252,7 @@ func ExecuteDescribeAffectedWithTargetRefCheckout( return nil, err } - u.LogTrace(cliConfig, fmt.Sprintf("Copied the local repo into temp directory '%s' ...", tempDir)) + u.LogTrace(cliConfig, fmt.Sprintf("Copied the local repo into temp directory '%s' ...\n", tempDir)) remoteRepo, err := git.PlainOpenWithOptions(tempDir, &git.PlainOpenOptions{ DetectDotGit: false, @@ -268,8 +268,17 @@ func ExecuteDescribeAffectedWithTargetRefCheckout( return nil, errors.Wrapf(err, "%v", remoteRepoIsNotGitRepoError) } - if ref != "" { - u.LogTrace(cliConfig, fmt.Sprintf("\nChecking out Git ref '%s' ...", ref)) + remoteRepoHead, err := remoteRepo.Head() + if err != nil { + return nil, err + } + + if ref == "" { + ref = remoteRepoHead.Name().String() + } + + if sha != "" { + u.LogTrace(cliConfig, fmt.Sprintf("\nChecking out commit SHA '%s' ...\n", sha)) w, err := remoteRepo.Worktree() if err != nil { @@ -277,7 +286,7 @@ func ExecuteDescribeAffectedWithTargetRefCheckout( } checkoutOptions := git.CheckoutOptions{ - Branch: plumbing.ReferenceName(ref), + Hash: plumbing.NewHash(sha), Create: false, Force: true, Keep: false, @@ -288,9 +297,9 @@ func ExecuteDescribeAffectedWithTargetRefCheckout( return nil, err } - u.LogTrace(cliConfig, fmt.Sprintf("Checked out Git ref '%s'\n", ref)) - } else if sha != "" { - u.LogTrace(cliConfig, fmt.Sprintf("\nChecking out commit SHA '%s' ...\n", sha)) + u.LogTrace(cliConfig, fmt.Sprintf("Checked out commit SHA '%s'\n", sha)) + } else { + u.LogTrace(cliConfig, fmt.Sprintf("\nChecking out Git ref '%s' ...", ref)) w, err := remoteRepo.Worktree() if err != nil { @@ -298,7 +307,7 @@ func ExecuteDescribeAffectedWithTargetRefCheckout( } checkoutOptions := git.CheckoutOptions{ - Hash: plumbing.NewHash(sha), + Branch: plumbing.ReferenceName(ref), Create: false, Force: true, Keep: false, @@ -309,7 +318,7 @@ func ExecuteDescribeAffectedWithTargetRefCheckout( return nil, err } - u.LogTrace(cliConfig, fmt.Sprintf("Checked out commit SHA '%s'\n", sha)) + u.LogTrace(cliConfig, fmt.Sprintf("Checked out Git ref '%s'\n", ref)) } affected, err := executeDescribeAffected(cliConfig, localRepoPath, tempDir, localRepo, remoteRepo, verbose, includeSpaceliftAdminStacks) diff --git a/pkg/describe/describe_affected_test.go b/pkg/describe/describe_affected_test.go index 8812e50a5..6c947cea5 100644 --- a/pkg/describe/describe_affected_test.go +++ b/pkg/describe/describe_affected_test.go @@ -50,7 +50,7 @@ func TestDescribeAffectedWithTargetRefCheckout(t *testing.T) { // Git reference and commit SHA // Refer to https://git-scm.com/book/en/v2/Git-Internals-Git-References for more details - ref := "refs/heads/master" + ref := "" sha := "" affected, err := e.ExecuteDescribeAffectedWithTargetRefCheckout(cliConfig, ref, sha, true, true) From 882e98ff3074166e87020c3b8e39f4078fea63bb Mon Sep 17 00:00:00 2001 From: aknysh Date: Thu, 25 Apr 2024 01:26:10 -0400 Subject: [PATCH 10/35] updates --- internal/exec/describe_affected_utils.go | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/internal/exec/describe_affected_utils.go b/internal/exec/describe_affected_utils.go index 7c506ee08..feb2cd143 100644 --- a/internal/exec/describe_affected_utils.go +++ b/internal/exec/describe_affected_utils.go @@ -268,13 +268,8 @@ func ExecuteDescribeAffectedWithTargetRefCheckout( return nil, errors.Wrapf(err, "%v", remoteRepoIsNotGitRepoError) } - remoteRepoHead, err := remoteRepo.Head() - if err != nil { - return nil, err - } - if ref == "" { - ref = remoteRepoHead.Name().String() + ref = "refs/remotes/origin/HEAD" } if sha != "" { @@ -500,7 +495,7 @@ func executeDescribeAffected( changedFiles = append(changedFiles, fileStat.Name) } - u.LogTrace(cliConfig, "") + u.LogTrace(cliConfig, "\nAffected components and stacks:\n") affected, err := findAffected(currentStacks, remoteStacks, cliConfig, changedFiles, includeSpaceliftAdminStacks) if err != nil { From eaf64aa68b8566ed4332f077b0fc2d3aa3cddd78 Mon Sep 17 00:00:00 2001 From: aknysh Date: Thu, 25 Apr 2024 10:49:22 -0400 Subject: [PATCH 11/35] updates --- internal/exec/describe_affected_utils.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/internal/exec/describe_affected_utils.go b/internal/exec/describe_affected_utils.go index feb2cd143..410cd65de 100644 --- a/internal/exec/describe_affected_utils.go +++ b/internal/exec/describe_affected_utils.go @@ -241,7 +241,7 @@ func ExecuteDescribeAffectedWithTargetRefCheckout( defer removeTempDir(cliConfig, tempDir) // Copy the local repo into the temp directory - u.LogTrace(cliConfig, fmt.Sprintf("\nCopying the local repo into temp directory '%s' ...", tempDir)) + u.LogTrace(cliConfig, fmt.Sprintf("\nCopying the local repo into the temp directory '%s' ...", tempDir)) copyOptions := cp.Options{ PreserveTimes: false, @@ -252,7 +252,7 @@ func ExecuteDescribeAffectedWithTargetRefCheckout( return nil, err } - u.LogTrace(cliConfig, fmt.Sprintf("Copied the local repo into temp directory '%s' ...\n", tempDir)) + u.LogTrace(cliConfig, fmt.Sprintf("Copied the local repo into the temp directory '%s' ...\n", tempDir)) remoteRepo, err := git.PlainOpenWithOptions(tempDir, &git.PlainOpenOptions{ DetectDotGit: false, @@ -268,10 +268,6 @@ func ExecuteDescribeAffectedWithTargetRefCheckout( return nil, errors.Wrapf(err, "%v", remoteRepoIsNotGitRepoError) } - if ref == "" { - ref = "refs/remotes/origin/HEAD" - } - if sha != "" { u.LogTrace(cliConfig, fmt.Sprintf("\nChecking out commit SHA '%s' ...\n", sha)) @@ -294,6 +290,11 @@ func ExecuteDescribeAffectedWithTargetRefCheckout( u.LogTrace(cliConfig, fmt.Sprintf("Checked out commit SHA '%s'\n", sha)) } else { + // If `ref` is not provided, use the HEAD of the origin + if ref == "" { + ref = "refs/remotes/origin/HEAD" + } + u.LogTrace(cliConfig, fmt.Sprintf("\nChecking out Git ref '%s' ...", ref)) w, err := remoteRepo.Worktree() From 06ffa7f3854b27608cfa5f3053188d4da0e4ede0 Mon Sep 17 00:00:00 2001 From: aknysh Date: Thu, 25 Apr 2024 11:29:46 -0400 Subject: [PATCH 12/35] updates --- internal/exec/describe_affected_utils.go | 10 ++++++---- internal/exec/describe_stacks.go | 16 ++++++++-------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/internal/exec/describe_affected_utils.go b/internal/exec/describe_affected_utils.go index 410cd65de..e041d5efe 100644 --- a/internal/exec/describe_affected_utils.go +++ b/internal/exec/describe_affected_utils.go @@ -252,7 +252,7 @@ func ExecuteDescribeAffectedWithTargetRefCheckout( return nil, err } - u.LogTrace(cliConfig, fmt.Sprintf("Copied the local repo into the temp directory '%s' ...\n", tempDir)) + u.LogTrace(cliConfig, fmt.Sprintf("Copied the local repo into the temp directory '%s'\n", tempDir)) remoteRepo, err := git.PlainOpenWithOptions(tempDir, &git.PlainOpenOptions{ DetectDotGit: false, @@ -290,7 +290,7 @@ func ExecuteDescribeAffectedWithTargetRefCheckout( u.LogTrace(cliConfig, fmt.Sprintf("Checked out commit SHA '%s'\n", sha)) } else { - // If `ref` is not provided, use the HEAD of the origin + // If `ref` is not provided, use the HEAD of the remote origin if ref == "" { ref = "refs/remotes/origin/HEAD" } @@ -496,13 +496,15 @@ func executeDescribeAffected( changedFiles = append(changedFiles, fileStat.Name) } - u.LogTrace(cliConfig, "\nAffected components and stacks:\n") - affected, err := findAffected(currentStacks, remoteStacks, cliConfig, changedFiles, includeSpaceliftAdminStacks) if err != nil { return nil, err } + if len(affected) > 0 { + u.LogTrace(cliConfig, "\nAffected components and stacks:\n") + } + return affected, nil } diff --git a/internal/exec/describe_stacks.go b/internal/exec/describe_stacks.go index b32ab2c05..32fd30e3a 100644 --- a/internal/exec/describe_stacks.go +++ b/internal/exec/describe_stacks.go @@ -200,10 +200,6 @@ func ExecuteDescribeStacks( }, } - configAndStacksInfo.ComponentSection["atmos_component"] = componentName - configAndStacksInfo.ComponentSection["atmos_stack"] = stackName - configAndStacksInfo.ComponentSection["atmos_stack_file"] = stackFileName - if comp, ok := configAndStacksInfo.ComponentSection["component"].(string); !ok || comp == "" { configAndStacksInfo.ComponentSection["component"] = componentName } @@ -235,6 +231,10 @@ func ExecuteDescribeStacks( finalStacksMap[stackName] = make(map[string]any) } + configAndStacksInfo.ComponentSection["atmos_component"] = componentName + configAndStacksInfo.ComponentSection["atmos_stack"] = stackName + configAndStacksInfo.ComponentSection["atmos_stack_file"] = stackFileName + if len(components) == 0 || u.SliceContainsString(components, componentName) || u.SliceContainsString(derivedComponents, componentName) { if !u.MapKeyExists(finalStacksMap[stackName].(map[string]any), "components") { finalStacksMap[stackName].(map[string]any)["components"] = make(map[string]any) @@ -374,10 +374,6 @@ func ExecuteDescribeStacks( }, } - configAndStacksInfo.ComponentSection["atmos_component"] = componentName - configAndStacksInfo.ComponentSection["atmos_stack"] = stackName - configAndStacksInfo.ComponentSection["atmos_stack_file"] = stackFileName - if comp, ok := configAndStacksInfo.ComponentSection["component"].(string); !ok || comp == "" { configAndStacksInfo.ComponentSection["component"] = componentName } @@ -409,6 +405,10 @@ func ExecuteDescribeStacks( finalStacksMap[stackName] = make(map[string]any) } + configAndStacksInfo.ComponentSection["atmos_component"] = componentName + configAndStacksInfo.ComponentSection["atmos_stack"] = stackName + configAndStacksInfo.ComponentSection["atmos_stack_file"] = stackFileName + if len(components) == 0 || u.SliceContainsString(components, componentName) || u.SliceContainsString(derivedComponents, componentName) { if !u.MapKeyExists(finalStacksMap[stackName].(map[string]any), "components") { finalStacksMap[stackName].(map[string]any)["components"] = make(map[string]any) From 7125c8825b726be8e468768a2987cbf18d9ad726 Mon Sep 17 00:00:00 2001 From: aknysh Date: Thu, 25 Apr 2024 12:13:14 -0400 Subject: [PATCH 13/35] updates --- internal/exec/describe_affected_utils.go | 47 +++++++++++++++--------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/internal/exec/describe_affected_utils.go b/internal/exec/describe_affected_utils.go index e041d5efe..c4f8a89f7 100644 --- a/internal/exec/describe_affected_utils.go +++ b/internal/exec/describe_affected_utils.go @@ -1,8 +1,8 @@ package exec import ( + "errors" "fmt" - cp "github.com/otiai10/copy" "os" "path" "path/filepath" @@ -16,7 +16,7 @@ import ( "github.com/go-git/go-git/v5/plumbing/transport/ssh" "github.com/hashicorp/terraform-config-inspect/tfconfig" "github.com/mitchellh/mapstructure" - "github.com/pkg/errors" + cp "github.com/otiai10/copy" cfg "github.com/cloudposse/atmos/pkg/config" "github.com/cloudposse/atmos/pkg/schema" @@ -51,18 +51,18 @@ func ExecuteDescribeAffectedWithTargetRefClone( EnableDotGitCommonDir: false, }) if err != nil { - return nil, errors.Wrapf(err, "%v", localRepoIsNotGitRepoError) + return nil, errors.Join(err, localRepoIsNotGitRepoError) } // Get the Git config of the local repo localRepoConfig, err := localRepo.Config() if err != nil { - return nil, errors.Wrapf(err, "%v", localRepoIsNotGitRepoError) + return nil, errors.Join(err, localRepoIsNotGitRepoError) } localRepoWorktree, err := localRepo.Worktree() if err != nil { - return nil, errors.Wrapf(err, "%v", localRepoIsNotGitRepoError) + return nil, errors.Join(err, localRepoIsNotGitRepoError) } localRepoPath := localRepoWorktree.Filesystem.Root() @@ -216,18 +216,18 @@ func ExecuteDescribeAffectedWithTargetRefCheckout( EnableDotGitCommonDir: false, }) if err != nil { - return nil, errors.Wrapf(err, "%v", localRepoIsNotGitRepoError) + return nil, errors.Join(err, localRepoIsNotGitRepoError) } // Check the Git config of the local repo _, err = localRepo.Config() if err != nil { - return nil, errors.Wrapf(err, "%v", localRepoIsNotGitRepoError) + return nil, errors.Join(err, localRepoIsNotGitRepoError) } localRepoWorktree, err := localRepo.Worktree() if err != nil { - return nil, errors.Wrapf(err, "%v", localRepoIsNotGitRepoError) + return nil, errors.Join(err, localRepoIsNotGitRepoError) } localRepoPath := localRepoWorktree.Filesystem.Root() @@ -246,6 +246,13 @@ func ExecuteDescribeAffectedWithTargetRefCheckout( copyOptions := cp.Options{ PreserveTimes: false, PreserveOwner: false, + // Skip specifies which files should be skipped + Skip: func(srcInfo os.FileInfo, src, dest string) (bool, error) { + if strings.Contains(src, "node_modules") { + return true, nil + } + return false, nil + }, } if err = cp.Copy(localRepoPath, tempDir, copyOptions); err != nil { @@ -259,13 +266,13 @@ func ExecuteDescribeAffectedWithTargetRefCheckout( EnableDotGitCommonDir: false, }) if err != nil { - return nil, errors.Wrapf(err, "%v", remoteRepoIsNotGitRepoError) + return nil, errors.Join(err, remoteRepoIsNotGitRepoError) } // Check the Git config of the target ref _, err = remoteRepo.Config() if err != nil { - return nil, errors.Wrapf(err, "%v", remoteRepoIsNotGitRepoError) + return nil, errors.Join(err, remoteRepoIsNotGitRepoError) } if sha != "" { @@ -311,6 +318,12 @@ func ExecuteDescribeAffectedWithTargetRefCheckout( err = w.Checkout(&checkoutOptions) if err != nil { + if strings.Contains(err.Error(), "reference not found") { + errorMessage := fmt.Sprintf("the Git ref '%s' does not exist on the local filesystem"+ + "\nmake sure it's correct and was cloned by Git from the remote, or use the '--clone-target-ref=true' flag to clone it"+ + "\nrefer to https://atmos.tools/cli/commands/describe/affected for more details", ref) + err = errors.New(errorMessage) + } return nil, err } @@ -341,18 +354,18 @@ func ExecuteDescribeAffectedWithTargetRepoPath( EnableDotGitCommonDir: false, }) if err != nil { - return nil, errors.Wrapf(err, "%v", localRepoIsNotGitRepoError) + return nil, errors.Join(err, localRepoIsNotGitRepoError) } // Check the Git config of the local repo _, err = localRepo.Config() if err != nil { - return nil, errors.Wrapf(err, "%v", localRepoIsNotGitRepoError) + return nil, errors.Join(err, localRepoIsNotGitRepoError) } localRepoWorktree, err := localRepo.Worktree() if err != nil { - return nil, errors.Wrapf(err, "%v", localRepoIsNotGitRepoError) + return nil, errors.Join(err, localRepoIsNotGitRepoError) } localRepoPath := localRepoWorktree.Filesystem.Root() @@ -362,13 +375,13 @@ func ExecuteDescribeAffectedWithTargetRepoPath( EnableDotGitCommonDir: false, }) if err != nil { - return nil, errors.Wrapf(err, "%v", remoteRepoIsNotGitRepoError) + return nil, errors.Join(err, remoteRepoIsNotGitRepoError) } // Check the Git config of the remote target repo _, err = remoteRepo.Config() if err != nil { - return nil, errors.Wrapf(err, "%v", remoteRepoIsNotGitRepoError) + return nil, errors.Join(err, remoteRepoIsNotGitRepoError) } affected, err := executeDescribeAffected(cliConfig, localRepoPath, targetRefPath, localRepo, remoteRepo, verbose, includeSpaceliftAdminStacks) @@ -501,9 +514,7 @@ func executeDescribeAffected( return nil, err } - if len(affected) > 0 { - u.LogTrace(cliConfig, "\nAffected components and stacks:\n") - } + u.LogTrace(cliConfig, "\nAffected components and stacks:\n") return affected, nil } From 94c8db7428af50cf0dd3ef7486e875ee2090ea8c Mon Sep 17 00:00:00 2001 From: aknysh Date: Thu, 25 Apr 2024 14:53:18 -0400 Subject: [PATCH 14/35] updates --- cmd/atlantis_generate_repo_config.go | 3 +- cmd/describe_affected.go | 5 +- .../atlantis-generate-repo-config.mdx | 30 +++---- .../commands/describe/describe-affected.mdx | 80 +++++++++++-------- 4 files changed, 67 insertions(+), 51 deletions(-) diff --git a/cmd/atlantis_generate_repo_config.go b/cmd/atlantis_generate_repo_config.go index 1a1cab44c..7ce15081e 100644 --- a/cmd/atlantis_generate_repo_config.go +++ b/cmd/atlantis_generate_repo_config.go @@ -60,7 +60,8 @@ func init() { atlantisGenerateCmd.PersistentFlags().Bool("clone-target-ref", false, "Clone the target reference with which to compare the current branch: "+ "atmos atlantis generate repo-config --affected-only=true --clone-target-ref=true\n"+ "The flag is only used when '--affected-only=true'\n"+ - "If set to 'false' (default), the target reference will be checked out instead (this means that the target reference must be already cloned by Git in the '.git' directory)") + "If set to 'false' (default), the target reference will be checked out instead\n"+ + "This requires that the target reference is already cloned by Git, and the information about it exists in the '.git' directory") atlantisGenerateCmd.AddCommand(atlantisGenerateRepoConfigCmd) } diff --git a/cmd/describe_affected.go b/cmd/describe_affected.go index ce0709921..5fc4339d6 100644 --- a/cmd/describe_affected.go +++ b/cmd/describe_affected.go @@ -36,8 +36,9 @@ func init() { describeAffectedCmd.PersistentFlags().String("ssh-key", "", "Path to PEM-encoded private key to clone private repos using SSH: atmos describe affected --ssh-key ") describeAffectedCmd.PersistentFlags().String("ssh-key-password", "", "Encryption password for the PEM-encoded private key if the key contains a password-encrypted PEM block: atmos describe affected --ssh-key --ssh-key-password ") describeAffectedCmd.PersistentFlags().Bool("include-spacelift-admin-stacks", false, "Include the Spacelift admin stack of any stack that is affected by config changes: atmos describe affected --include-spacelift-admin-stacks=true") - describeAffectedCmd.PersistentFlags().Bool("clone-target-ref", false, "Clone the target reference with which to compare the current branch: atmos describe affected --clone-target-ref=true. "+ - "If set to 'false' (default), the target reference will be checked out instead (this means that the target reference must be already cloned by Git in the '.git' directory)") + describeAffectedCmd.PersistentFlags().Bool("clone-target-ref", false, "Clone the target reference with which to compare the current branch: atmos describe affected --clone-target-ref=true\n"+ + "If set to 'false' (default), the target reference will be checked out instead\n"+ + "This requires that the target reference is already cloned by Git, and the information about it exists in the '.git' directory") describeCmd.AddCommand(describeAffectedCmd) } diff --git a/website/docs/cli/commands/atlantis/atlantis-generate-repo-config.mdx b/website/docs/cli/commands/atlantis/atlantis-generate-repo-config.mdx index d893e29a2..f099cc17c 100644 --- a/website/docs/cli/commands/atlantis/atlantis-generate-repo-config.mdx +++ b/website/docs/cli/commands/atlantis/atlantis-generate-repo-config.mdx @@ -71,21 +71,21 @@ atmos atlantis generate repo-config --affected-only=true --clone-target-ref=true ## Flags -| Flag | Description | Required | -|:---------------------|:----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:---------| -| `--config-template` | Atlantis config template name | no | -| `--project-template` | Atlantis project template name | no | -| `--output-path` | Output path to write `atlantis.yaml` file | no | -| `--stacks` | Generate Atlantis projects for the specified stacks only (comma-separated values) | no | -| `--components` | Generate Atlantis projects for the specified components only (comma-separated values) | no | -| `--affected-only` | Generate Atlantis projects only for the Atmos components changed
between two Git commits | no | -| `--ref` | [Git Reference](https://git-scm.com/book/en/v2/Git-Internals-Git-References) with which to compare the current working branch | no | -| `--sha` | Git commit SHA with which to compare the current working branch | no | -| `--ssh-key` | Path to PEM-encoded private key to clone private repos using SSH | no | -| `--ssh-key-password` | Encryption password for the PEM-encoded private key if the key contains
a password-encrypted PEM block | no | -| `--repo-path` | Path to the already cloned target repository with which to compare the current branch.
Conflicts with `--ref`, `--sha`, `--ssh-key` and `--ssh-key-password` | no | -| `--verbose` | Print more detailed output when cloning and checking out the target
Git repository and processing the result | no | -| `--clone-target-ref` | Clone the target reference with which to compare the current branch.
`atmos atlantis generate repo-config --affected-only=true --clone-target-ref=true`

The flag is only used when `--affected-only=true`.

If set to `false` (default), the target reference will be checked out instead
(this means that the target reference must be already cloned by Git in the `.git` directory) | no | +| Flag | Description | Required | +|:---------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:---------| +| `--config-template` | Atlantis config template name | no | +| `--project-template` | Atlantis project template name | no | +| `--output-path` | Output path to write `atlantis.yaml` file | no | +| `--stacks` | Generate Atlantis projects for the specified stacks only (comma-separated values) | no | +| `--components` | Generate Atlantis projects for the specified components only (comma-separated values) | no | +| `--affected-only` | Generate Atlantis projects only for the Atmos components changed
between two Git commits | no | +| `--ref` | [Git Reference](https://git-scm.com/book/en/v2/Git-Internals-Git-References) with which to compare the current working branch | no | +| `--sha` | Git commit SHA with which to compare the current working branch | no | +| `--ssh-key` | Path to PEM-encoded private key to clone private repos using SSH | no | +| `--ssh-key-password` | Encryption password for the PEM-encoded private key if the key contains
a password-encrypted PEM block | no | +| `--repo-path` | Path to the already cloned target repository with which to compare the current branch.
Conflicts with `--ref`, `--sha`, `--ssh-key` and `--ssh-key-password` | no | +| `--verbose` | Print more detailed output when cloning and checking out the target
Git repository and processing the result | no | +| `--clone-target-ref` | Clone the target reference with which to compare the current branch.
`atmos atlantis generate repo-config --affected-only=true --clone-target-ref=true`

The flag is only used when `--affected-only=true`.

If set to `false` (default), the target reference will be checked out instead
This requires that the target reference is already cloned by Git
, and the information about it exists in the `.git` directory | no |
diff --git a/website/docs/cli/commands/describe/describe-affected.mdx b/website/docs/cli/commands/describe/describe-affected.mdx index 990e28e4f..b61e79035 100644 --- a/website/docs/cli/commands/describe/describe-affected.mdx +++ b/website/docs/cli/commands/describe/describe-affected.mdx @@ -20,46 +20,60 @@ Use this command to show a list of the affected Atmos components and stacks give The command uses two different Git commits to produce a list of affected Atmos components and stacks. -For the first commit, the command assumes that the current repo root is a Git checkout. An error will be thrown if the current repo is not a Git -repository (the `.git/` folder does not exist or is configured incorrectly). +For the first commit, the command assumes that the current repo root is a Git checkout. An error will be thrown if the +current repo is not a Git repository (the `.git/` folder does not exist or is configured incorrectly). The second commit can be specified on the command line by using the `--ref` ([Git References](https://git-scm.com/book/en/v2/Git-Internals-Git-References)) or `--sha` (commit SHA) flags. - -Either `--ref` or `--sha` should be used. If both flags are provided at the same time, the command will first clone the remote branch pointed to by -the `--ref` flag and then checkout the Git commit pointed to by the `--sha` flag (`--sha` flag overrides `--ref` flag). +The `--sha` takes precedence over the `--ref` flag. :::tip -If the flags are not provided, the `ref` will be set automatically to the reference to the default branch (e.g. `main`) and the commit SHA -will point to the `HEAD` of the branch. +If the flags are not provided, the `ref` will be set automatically to the reference to the default branch +(`refs/remotes/origin/HEAD` Git ref, usually the `main` branch). ::: -If you specify the `--repo-path` flag with the path to the already cloned repository, the command will not clone the target -repository, but instead will use the already cloned one to compare the current branch with. In this case, the `--ref`, `--sha`, `--ssh-key` -and `--ssh-key-password` flags are not used, and an error will be thrown if the `--repo-path` flag and any of the `--ref`, `--sha`, `--ssh-key` -or `--ssh-key-password` flags are provided at the same time. - ## How does it work? The command performs the following: -- Cloning the target branch (`--ref`) or checking out the commit (`--sha`) of the remote target branch, or using the already cloned target repository - specified by the `--repo-path` flag +- If the `--repo-path` flag is passed, the command uses it as the path to the already cloned target repo with which to + compare the current working branch. I this case, the command will not clone and checkout the + target reference, but instead will use the already cloned one to compare the current branch with. In this case, the + `--ref`, `--sha`, `--ssh-key` and `--ssh-key-password` flags are not used, and an error will be thrown if the `--repo-path` + flag and any of the `--ref`, `--sha`, `--ssh-key` or `--ssh-key-password` flags are provided at the same time + +- Otherwise, if the `--clone-target-ref=true` flag is specified, the command clones (into a temp directory) the remote + target with which to compare the current working branch. If the `--ref` flag or the commit SHA flag `--sha` are provided, + the command uses them to clone and checkout the remote target. Otherwise, the `HEAD` of the remote origin is + used (`refs/remotes/origin/HEAD` Git ref, usually the `main` branch) + +- Otherwise, (if the `--repo-path` and `--clone-target-ref=true` flags are not passed), the command does not clone anything + from the remote origin, but instead just copies the current repo into a temp directory and checks out the target + reference with which to compare the current working branch. If the `--ref` flag or the commit SHA flag `--sha` are + provided, the command uses them to check out. Otherwise, the `HEAD` of the remote origin is used + (`refs/remotes/origin/HEAD` Git ref, usually the `main` branch). This requires that the target reference is already + cloned by Git, and the information about it exists in the `.git` directory. This is the recommended way to execute the + `atmos describe affected` command since it allows [working with private repositories](#working-with-private-repositories) + without providing the SSH credentials (`--ssh-key` and `--ssh-key-password) since in this case Atmos does not + access the remote origin and instead just checks out the target reference -- Deep merging all stack configurations for both the current working branch and the remote target branch +- The command deep-merges all stack configurations from both sources: the current working branch and the target reference -- Looking for changes in the component directories +- The command searches for changes in the component directories -- Comparing each section of the stack configuration looking for differences +- The command compares each stack manifest section of the stack configurations from both sources looking for differences -- Outputting a JSON or YAML document consisting of a list of the affected components and stacks and what caused it to be affected +- And finally, the command outputs a JSON or YAML document consisting of a list of the affected components and stacks + and what caused it to be affected -Since Atmos first checks the component folders for changes, if it finds any affected files, it will mark all related components and stacks as -affected. Atmos will then skip evaluating those stacks for differences since we already know that they are affected. +Since Atmos first checks the component folders for changes, if it finds any affected files, it will mark all related +components and stacks as affected. Atmos will then skip evaluating the stacks for differences since it already +knows that they are affected.
:::tip Use our GitHub Action -Our [affected stacks](/integrations/github-actions/affected-stacks) GitHub Action provides a ready-to-go way to run `describe affected` and produce a GitHub matrix. +Our [affected stacks](/integrations/github-actions/affected-stacks) GitHub Action provides a ready-to-go way to run +`describe affected` and produce a GitHub matrix. ::: # @@ -182,18 +196,18 @@ Affected components and stacks: ## Flags -| Flag | Description | Required | -|:-----------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:---------| -| `--ref` | [Git Reference](https://git-scm.com/book/en/v2/Git-Internals-Git-References) with which to compare the current working branch | no | -| `--sha` | Git commit SHA with which to compare the current working branch | no | -| `--file` | If specified, write the result to the file | no | -| `--format` | Specify the output format: `json` or `yaml` (`json` is default) | no | -| `--ssh-key` | Path to PEM-encoded private key to clone private repos using SSH | no | -| `--ssh-key-password` | Encryption password for the PEM-encoded private key if the key contains
a password-encrypted PEM block | no | -| `--repo-path` | Path to the already cloned target repository with which to compare the current branch.
Conflicts with `--ref`, `--sha`, `--ssh-key` and `--ssh-key-password` | no | -| `--verbose` | Print more detailed output when cloning and checking out the target
Git repository and processing the result | no | -| `--include-spacelift-admin-stacks` | Include the Spacelift admin stack of any stack
that is affected by config changes | no | -| `--clone-target-ref` | Clone the target reference with which to compare the current branch.
`atmos describe affected --clone-target-ref=true`

If set to `false` (default), the target reference will be checked out instead
(this means that the target reference must be already cloned by Git
in the `.git` directory) | no | +| Flag | Description | Required | +|:-----------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:---------| +| `--ref` | [Git Reference](https://git-scm.com/book/en/v2/Git-Internals-Git-References) with which to compare the current working branch | no | +| `--sha` | Git commit SHA with which to compare the current working branch | no | +| `--file` | If specified, write the result to the file | no | +| `--format` | Specify the output format: `json` or `yaml` (`json` is default) | no | +| `--ssh-key` | Path to PEM-encoded private key to clone private repos using SSH | no | +| `--ssh-key-password` | Encryption password for the PEM-encoded private key if the key contains
a password-encrypted PEM block | no | +| `--repo-path` | Path to the already cloned target repository with which to compare the current branch.
Conflicts with `--ref`, `--sha`, `--ssh-key` and `--ssh-key-password` | no | +| `--verbose` | Print more detailed output when cloning and checking out the target
Git repository and processing the result | no | +| `--include-spacelift-admin-stacks` | Include the Spacelift admin stack of any stack
that is affected by config changes | no | +| `--clone-target-ref` | Clone the target reference with which to compare the current branch.
`atmos describe affected --clone-target-ref=true`

If set to `false` (default), the target reference will be checked out instead
This requires that the target reference is already cloned by Git
, and the information about it exists in the `.git` directory | no | ## Output From 5c369bd55dfb911501aa13ca6577330608c45e30 Mon Sep 17 00:00:00 2001 From: aknysh Date: Thu, 25 Apr 2024 14:54:42 -0400 Subject: [PATCH 15/35] updates --- website/docs/cli/commands/describe/describe-affected.mdx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/website/docs/cli/commands/describe/describe-affected.mdx b/website/docs/cli/commands/describe/describe-affected.mdx index b61e79035..2b33efc19 100644 --- a/website/docs/cli/commands/describe/describe-affected.mdx +++ b/website/docs/cli/commands/describe/describe-affected.mdx @@ -54,8 +54,8 @@ The command performs the following: (`refs/remotes/origin/HEAD` Git ref, usually the `main` branch). This requires that the target reference is already cloned by Git, and the information about it exists in the `.git` directory. This is the recommended way to execute the `atmos describe affected` command since it allows [working with private repositories](#working-with-private-repositories) - without providing the SSH credentials (`--ssh-key` and `--ssh-key-password) since in this case Atmos does not - access the remote origin and instead just checks out the target reference + without providing the SSH credentials (`--ssh-key` and `--ssh-key-password`) since in this case Atmos does not + access the remote origin and instead just checks out the target reference (which is already on the local file system) - The command deep-merges all stack configurations from both sources: the current working branch and the target reference From 24fd13fa3b3505673e6cf89a5f0586dfe5e70c4d Mon Sep 17 00:00:00 2001 From: aknysh Date: Thu, 25 Apr 2024 15:01:56 -0400 Subject: [PATCH 16/35] updates --- .../atlantis-generate-repo-config.mdx | 30 +++++++++---------- .../commands/describe/describe-affected.mdx | 24 +++++++-------- 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/website/docs/cli/commands/atlantis/atlantis-generate-repo-config.mdx b/website/docs/cli/commands/atlantis/atlantis-generate-repo-config.mdx index f099cc17c..c3727c431 100644 --- a/website/docs/cli/commands/atlantis/atlantis-generate-repo-config.mdx +++ b/website/docs/cli/commands/atlantis/atlantis-generate-repo-config.mdx @@ -71,21 +71,21 @@ atmos atlantis generate repo-config --affected-only=true --clone-target-ref=true ## Flags -| Flag | Description | Required | -|:---------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:---------| -| `--config-template` | Atlantis config template name | no | -| `--project-template` | Atlantis project template name | no | -| `--output-path` | Output path to write `atlantis.yaml` file | no | -| `--stacks` | Generate Atlantis projects for the specified stacks only (comma-separated values) | no | -| `--components` | Generate Atlantis projects for the specified components only (comma-separated values) | no | -| `--affected-only` | Generate Atlantis projects only for the Atmos components changed
between two Git commits | no | -| `--ref` | [Git Reference](https://git-scm.com/book/en/v2/Git-Internals-Git-References) with which to compare the current working branch | no | -| `--sha` | Git commit SHA with which to compare the current working branch | no | -| `--ssh-key` | Path to PEM-encoded private key to clone private repos using SSH | no | -| `--ssh-key-password` | Encryption password for the PEM-encoded private key if the key contains
a password-encrypted PEM block | no | -| `--repo-path` | Path to the already cloned target repository with which to compare the current branch.
Conflicts with `--ref`, `--sha`, `--ssh-key` and `--ssh-key-password` | no | -| `--verbose` | Print more detailed output when cloning and checking out the target
Git repository and processing the result | no | -| `--clone-target-ref` | Clone the target reference with which to compare the current branch.
`atmos atlantis generate repo-config --affected-only=true --clone-target-ref=true`

The flag is only used when `--affected-only=true`.

If set to `false` (default), the target reference will be checked out instead
This requires that the target reference is already cloned by Git
, and the information about it exists in the `.git` directory | no | +| Flag | Description | Required | +|:---------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:---------| +| `--config-template` | Atlantis config template name | no | +| `--project-template` | Atlantis project template name | no | +| `--output-path` | Output path to write `atlantis.yaml` file | no | +| `--stacks` | Generate Atlantis projects for the specified stacks only (comma-separated values) | no | +| `--components` | Generate Atlantis projects for the specified components only (comma-separated values) | no | +| `--affected-only` | Generate Atlantis projects only for the Atmos components changed
between two Git commits | no | +| `--ref` | [Git Reference](https://git-scm.com/book/en/v2/Git-Internals-Git-References) with which to compare the current working branch | no | +| `--sha` | Git commit SHA with which to compare the current working branch | no | +| `--ssh-key` | Path to PEM-encoded private key to clone private repos using SSH | no | +| `--ssh-key-password` | Encryption password for the PEM-encoded private key if the key contains
a password-encrypted PEM block | no | +| `--repo-path` | Path to the already cloned target repository with which to compare the current branch.
Conflicts with `--ref`, `--sha`, `--ssh-key` and `--ssh-key-password` | no | +| `--verbose` | Print more detailed output when cloning and checking out the target
Git repository and processing the result | no | +| `--clone-target-ref` | Clone the target reference with which to compare the current branch.
`atmos atlantis generate repo-config --affected-only=true --clone-target-ref=true`

The flag is only used when `--affected-only=true`.

If set to `false` (default), the target reference will be checked out instead
This requires that the target reference is already cloned by Git,
and the information about it exists in the `.git` directory | no |
diff --git a/website/docs/cli/commands/describe/describe-affected.mdx b/website/docs/cli/commands/describe/describe-affected.mdx index 2b33efc19..fbccc28a6 100644 --- a/website/docs/cli/commands/describe/describe-affected.mdx +++ b/website/docs/cli/commands/describe/describe-affected.mdx @@ -196,18 +196,18 @@ Affected components and stacks: ## Flags -| Flag | Description | Required | -|:-----------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:---------| -| `--ref` | [Git Reference](https://git-scm.com/book/en/v2/Git-Internals-Git-References) with which to compare the current working branch | no | -| `--sha` | Git commit SHA with which to compare the current working branch | no | -| `--file` | If specified, write the result to the file | no | -| `--format` | Specify the output format: `json` or `yaml` (`json` is default) | no | -| `--ssh-key` | Path to PEM-encoded private key to clone private repos using SSH | no | -| `--ssh-key-password` | Encryption password for the PEM-encoded private key if the key contains
a password-encrypted PEM block | no | -| `--repo-path` | Path to the already cloned target repository with which to compare the current branch.
Conflicts with `--ref`, `--sha`, `--ssh-key` and `--ssh-key-password` | no | -| `--verbose` | Print more detailed output when cloning and checking out the target
Git repository and processing the result | no | -| `--include-spacelift-admin-stacks` | Include the Spacelift admin stack of any stack
that is affected by config changes | no | -| `--clone-target-ref` | Clone the target reference with which to compare the current branch.
`atmos describe affected --clone-target-ref=true`

If set to `false` (default), the target reference will be checked out instead
This requires that the target reference is already cloned by Git
, and the information about it exists in the `.git` directory | no | +| Flag | Description | Required | +|:-----------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:---------| +| `--ref` | [Git Reference](https://git-scm.com/book/en/v2/Git-Internals-Git-References) with which to compare the current working branch | no | +| `--sha` | Git commit SHA with which to compare the current working branch | no | +| `--file` | If specified, write the result to the file | no | +| `--format` | Specify the output format: `json` or `yaml` (`json` is default) | no | +| `--ssh-key` | Path to PEM-encoded private key to clone private repos using SSH | no | +| `--ssh-key-password` | Encryption password for the PEM-encoded private key if the key contains
a password-encrypted PEM block | no | +| `--repo-path` | Path to the already cloned target repository with which to compare the current branch.
Conflicts with `--ref`, `--sha`, `--ssh-key` and `--ssh-key-password` | no | +| `--verbose` | Print more detailed output when cloning and checking out the target
Git repository and processing the result | no | +| `--include-spacelift-admin-stacks` | Include the Spacelift admin stack of any stack
that is affected by config changes | no | +| `--clone-target-ref` | Clone the target reference with which to compare the current branch.
`atmos describe affected --clone-target-ref=true`

If set to `false` (default), the target reference will be checked out instead
This requires that the target reference is already cloned by Git,
and the information about it exists in the `.git` directory | no | ## Output From 0200ddc2df488922294afddbd852d8e27e41d8ae Mon Sep 17 00:00:00 2001 From: aknysh Date: Thu, 25 Apr 2024 15:09:03 -0400 Subject: [PATCH 17/35] updates --- .../atlantis-generate-repo-config.mdx | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/website/docs/cli/commands/atlantis/atlantis-generate-repo-config.mdx b/website/docs/cli/commands/atlantis/atlantis-generate-repo-config.mdx index c3727c431..c1dc5b785 100644 --- a/website/docs/cli/commands/atlantis/atlantis-generate-repo-config.mdx +++ b/website/docs/cli/commands/atlantis/atlantis-generate-repo-config.mdx @@ -71,21 +71,21 @@ atmos atlantis generate repo-config --affected-only=true --clone-target-ref=true ## Flags -| Flag | Description | Required | -|:---------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:---------| -| `--config-template` | Atlantis config template name | no | -| `--project-template` | Atlantis project template name | no | -| `--output-path` | Output path to write `atlantis.yaml` file | no | -| `--stacks` | Generate Atlantis projects for the specified stacks only (comma-separated values) | no | -| `--components` | Generate Atlantis projects for the specified components only (comma-separated values) | no | -| `--affected-only` | Generate Atlantis projects only for the Atmos components changed
between two Git commits | no | -| `--ref` | [Git Reference](https://git-scm.com/book/en/v2/Git-Internals-Git-References) with which to compare the current working branch | no | -| `--sha` | Git commit SHA with which to compare the current working branch | no | -| `--ssh-key` | Path to PEM-encoded private key to clone private repos using SSH | no | -| `--ssh-key-password` | Encryption password for the PEM-encoded private key if the key contains
a password-encrypted PEM block | no | -| `--repo-path` | Path to the already cloned target repository with which to compare the current branch.
Conflicts with `--ref`, `--sha`, `--ssh-key` and `--ssh-key-password` | no | -| `--verbose` | Print more detailed output when cloning and checking out the target
Git repository and processing the result | no | -| `--clone-target-ref` | Clone the target reference with which to compare the current branch.
`atmos atlantis generate repo-config --affected-only=true --clone-target-ref=true`

The flag is only used when `--affected-only=true`.

If set to `false` (default), the target reference will be checked out instead
This requires that the target reference is already cloned by Git,
and the information about it exists in the `.git` directory | no | +| Flag | Description | Required | +|:---------------------|:----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:---------| +| `--config-template` | Atlantis config template name | no | +| `--project-template` | Atlantis project template name | no | +| `--output-path` | Output path to write `atlantis.yaml` file | no | +| `--stacks` | Generate Atlantis projects for the specified stacks only (comma-separated values) | no | +| `--components` | Generate Atlantis projects for the specified components only (comma-separated values) | no | +| `--affected-only` | Generate Atlantis projects only for the Atmos components changed
between two Git commits | no | +| `--ref` | [Git Reference](https://git-scm.com/book/en/v2/Git-Internals-Git-References) with which to compare the current working branch | no | +| `--sha` | Git commit SHA with which to compare the current working branch | no | +| `--ssh-key` | Path to PEM-encoded private key to clone private repos using SSH | no | +| `--ssh-key-password` | Encryption password for the PEM-encoded private key if the key contains
a password-encrypted PEM block | no | +| `--repo-path` | Path to the already cloned target repository with which to compare the current branch.
Conflicts with `--ref`, `--sha`, `--ssh-key` and `--ssh-key-password` | no | +| `--verbose` | Print more detailed output when cloning and checking out the target
Git repository and processing the result | no | +| `--clone-target-ref` | Clone the target reference with which to compare the current branch.
`atmos atlantis generate repo-config --affected-only=true --clone-target-ref=true`

The flag is only used when `--affected-only=true`

If set to `false` (default), the target reference will be checked out instead
This requires that the target reference is already cloned by Git,
and the information about it exists in the `.git` directory | no |
From 18b656412f2c74c056dbd74ea0878da337d8eb40 Mon Sep 17 00:00:00 2001 From: aknysh Date: Thu, 25 Apr 2024 15:30:32 -0400 Subject: [PATCH 18/35] updates --- website/docs/cli/commands/describe/describe-affected.mdx | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/website/docs/cli/commands/describe/describe-affected.mdx b/website/docs/cli/commands/describe/describe-affected.mdx index fbccc28a6..693ad9c16 100644 --- a/website/docs/cli/commands/describe/describe-affected.mdx +++ b/website/docs/cli/commands/describe/describe-affected.mdx @@ -49,10 +49,13 @@ The command performs the following: - Otherwise, (if the `--repo-path` and `--clone-target-ref=true` flags are not passed), the command does not clone anything from the remote origin, but instead just copies the current repo into a temp directory and checks out the target - reference with which to compare the current working branch. If the `--ref` flag or the commit SHA flag `--sha` are + reference with which to compare the current working branch. + If the `--ref` flag or the commit SHA flag `--sha` are provided, the command uses them to check out. Otherwise, the `HEAD` of the remote origin is used - (`refs/remotes/origin/HEAD` Git ref, usually the `main` branch). This requires that the target reference is already - cloned by Git, and the information about it exists in the `.git` directory. This is the recommended way to execute the + (`refs/remotes/origin/HEAD` Git ref, usually the `main` branch). + This requires that the target reference is already cloned by Git, and the information about it exists in + the `.git` directory (Git deep clone was executed instead of a shallow clone). + This is the recommended way to execute the `atmos describe affected` command since it allows [working with private repositories](#working-with-private-repositories) without providing the SSH credentials (`--ssh-key` and `--ssh-key-password`) since in this case Atmos does not access the remote origin and instead just checks out the target reference (which is already on the local file system) From d289664a34d87cfd4e3359a1ea7c3b6f1d32bb92 Mon Sep 17 00:00:00 2001 From: aknysh Date: Thu, 25 Apr 2024 15:33:57 -0400 Subject: [PATCH 19/35] updates --- .../docs/cli/commands/describe/describe-affected.mdx | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/website/docs/cli/commands/describe/describe-affected.mdx b/website/docs/cli/commands/describe/describe-affected.mdx index 693ad9c16..9d7c057e6 100644 --- a/website/docs/cli/commands/describe/describe-affected.mdx +++ b/website/docs/cli/commands/describe/describe-affected.mdx @@ -54,11 +54,12 @@ The command performs the following: provided, the command uses them to check out. Otherwise, the `HEAD` of the remote origin is used (`refs/remotes/origin/HEAD` Git ref, usually the `main` branch). This requires that the target reference is already cloned by Git, and the information about it exists in - the `.git` directory (Git deep clone was executed instead of a shallow clone). - This is the recommended way to execute the - `atmos describe affected` command since it allows [working with private repositories](#working-with-private-repositories) - without providing the SSH credentials (`--ssh-key` and `--ssh-key-password`) since in this case Atmos does not - access the remote origin and instead just checks out the target reference (which is already on the local file system) + the `.git` directory (in case of using a non-default branch as the target, Git deep clone needs to be executed instead + of a shallow clone). + This is the recommended way to execute the `atmos describe affected` command since it allows + [working with private repositories](#working-with-private-repositories) without providing the SSH credentials + (`--ssh-key` and `--ssh-key-password` flags), since in this case Atmos does not access the remote origin and instead + just checks out the target reference (which is already on the local file system) - The command deep-merges all stack configurations from both sources: the current working branch and the target reference From f0ad83d29a1e8586aa1428a354b1b34d52cd8e11 Mon Sep 17 00:00:00 2001 From: aknysh Date: Thu, 25 Apr 2024 15:52:11 -0400 Subject: [PATCH 20/35] updates --- pkg/describe/describe_affected_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/describe/describe_affected_test.go b/pkg/describe/describe_affected_test.go index 6c947cea5..8812e50a5 100644 --- a/pkg/describe/describe_affected_test.go +++ b/pkg/describe/describe_affected_test.go @@ -50,7 +50,7 @@ func TestDescribeAffectedWithTargetRefCheckout(t *testing.T) { // Git reference and commit SHA // Refer to https://git-scm.com/book/en/v2/Git-Internals-Git-References for more details - ref := "" + ref := "refs/heads/master" sha := "" affected, err := e.ExecuteDescribeAffectedWithTargetRefCheckout(cliConfig, ref, sha, true, true) From f4c1142ccee1702bcaec7564e96e2f91cd09fdd2 Mon Sep 17 00:00:00 2001 From: aknysh Date: Thu, 25 Apr 2024 16:01:47 -0400 Subject: [PATCH 21/35] updates --- pkg/describe/describe_affected_test.go | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/pkg/describe/describe_affected_test.go b/pkg/describe/describe_affected_test.go index 8812e50a5..988201c6a 100644 --- a/pkg/describe/describe_affected_test.go +++ b/pkg/describe/describe_affected_test.go @@ -37,31 +37,6 @@ func TestDescribeAffectedWithTargetRefClone(t *testing.T) { t.Log(fmt.Sprintf("\nAffected components and stacks:\n%v", string(affectedYaml))) } -func TestDescribeAffectedWithTargetRefCheckout(t *testing.T) { - configAndStacksInfo := schema.ConfigAndStacksInfo{} - - cliConfig, err := cfg.InitCliConfig(configAndStacksInfo, true) - assert.Nil(t, err) - - // We are using `atmos.yaml` from this dir. This `atmos.yaml` has set base_path: "../../examples/tests", - // which will be wrong for the remote repo which is cloned into a temp dir. - // Set the correct base path for the cloned remote repo - cliConfig.BasePath = "./examples/tests" - - // Git reference and commit SHA - // Refer to https://git-scm.com/book/en/v2/Git-Internals-Git-References for more details - ref := "refs/heads/master" - sha := "" - - affected, err := e.ExecuteDescribeAffectedWithTargetRefCheckout(cliConfig, ref, sha, true, true) - assert.Nil(t, err) - - affectedYaml, err := yaml.Marshal(affected) - assert.Nil(t, err) - - t.Log(fmt.Sprintf("\nAffected components and stacks:\n%v", string(affectedYaml))) -} - func TestDescribeAffectedWithTargetRepoPath(t *testing.T) { configAndStacksInfo := schema.ConfigAndStacksInfo{} From 16b1ea2c0a59e6342a71656f3ff4f0cbd25a251d Mon Sep 17 00:00:00 2001 From: aknysh Date: Fri, 26 Apr 2024 11:25:39 -0400 Subject: [PATCH 22/35] updates --- .../stacks/schemas/atmos/atmos-manifest/1.0/atmos-manifest.json | 2 ++ .../static/schemas/atmos/atmos-manifest/1.0/atmos-manifest.json | 2 ++ 2 files changed, 4 insertions(+) diff --git a/examples/quick-start/stacks/schemas/atmos/atmos-manifest/1.0/atmos-manifest.json b/examples/quick-start/stacks/schemas/atmos/atmos-manifest/1.0/atmos-manifest.json index 445e9c7f2..c337407e0 100644 --- a/examples/quick-start/stacks/schemas/atmos/atmos-manifest/1.0/atmos-manifest.json +++ b/examples/quick-start/stacks/schemas/atmos/atmos-manifest/1.0/atmos-manifest.json @@ -437,6 +437,7 @@ "backend_type": { "type": "string", "enum": [ + "local", "s3", "remote", "vault", @@ -455,6 +456,7 @@ "remote_state_backend_type": { "type": "string", "enum": [ + "local", "s3", "remote", "vault", diff --git a/website/static/schemas/atmos/atmos-manifest/1.0/atmos-manifest.json b/website/static/schemas/atmos/atmos-manifest/1.0/atmos-manifest.json index 445e9c7f2..c337407e0 100644 --- a/website/static/schemas/atmos/atmos-manifest/1.0/atmos-manifest.json +++ b/website/static/schemas/atmos/atmos-manifest/1.0/atmos-manifest.json @@ -437,6 +437,7 @@ "backend_type": { "type": "string", "enum": [ + "local", "s3", "remote", "vault", @@ -455,6 +456,7 @@ "remote_state_backend_type": { "type": "string", "enum": [ + "local", "s3", "remote", "vault", From a12d879aa8a52a07bc8ae9cea898544f4e55d90a Mon Sep 17 00:00:00 2001 From: aknysh Date: Fri, 26 Apr 2024 20:05:22 -0400 Subject: [PATCH 23/35] updates --- internal/exec/utils.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/exec/utils.go b/internal/exec/utils.go index e419db7ce..57eace027 100644 --- a/internal/exec/utils.go +++ b/internal/exec/utils.go @@ -311,7 +311,6 @@ func ProcessStacks( return configAndStacksInfo, err } - configAndStacksInfo.ComponentEnvList = u.ConvertEnvVars(configAndStacksInfo.ComponentEnvSection) configAndStacksInfo.StackFile = configAndStacksInfo.Stack // Process context @@ -345,8 +344,6 @@ func ProcessStacks( continue } - configAndStacksInfo.ComponentEnvList = u.ConvertEnvVars(configAndStacksInfo.ComponentEnvSection) - if cliConfig.Stacks.NameTemplate != "" { tmpl, err2 := u.ProcessTmpl("name-template", cliConfig.Stacks.NameTemplate, configAndStacksInfo.ComponentSection, false) if err2 != nil { @@ -602,6 +599,9 @@ func ProcessStacks( configAndStacksInfo.ComponentBackendType = i } + // Process the ENV variables from the `env` section + configAndStacksInfo.ComponentEnvList = u.ConvertEnvVars(configAndStacksInfo.ComponentEnvSection) + return configAndStacksInfo, nil } From 6f5cd7095e2881d6ba960d83195f875ab9198cc4 Mon Sep 17 00:00:00 2001 From: aknysh Date: Sun, 28 Apr 2024 16:14:12 -0400 Subject: [PATCH 24/35] updates --- website/docs/core-concepts/stacks/templating.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/website/docs/core-concepts/stacks/templating.md b/website/docs/core-concepts/stacks/templating.md index 491d4f7e2..0dcc60086 100644 --- a/website/docs/core-concepts/stacks/templating.md +++ b/website/docs/core-concepts/stacks/templating.md @@ -126,9 +126,13 @@ For example, the `env` function has the same name in [Sprig](https://masterminds If you use the `env` function from one templating engine and enable both [Sprig](https://masterminds.github.io/sprig/) and [Gomplate](https://docs.gomplate.ca/functions/), it will be invalid in the other templating engine, and an error will be thrown. -For this reason, you can use the `templates.settings.sprig.enabled` and `templates.settings,gomplate.enabled` settings to selectively -enable/disable the [Sprig](https://masterminds.github.io/sprig/) and [Gomplate](https://docs.gomplate.ca/functions/) -functions. +To be able to use the `env` function from both templating engines, you can do one of the following: + +- Use the `env` function from one templating engine, and disable the other templating engine by using the + `templates.settings.sprig.enabled` and `templates.settings,gomplate.enabled` settings + +- Enable both engines and use the Gomplate's `env` function via its + [`getenv`](https://docs.gomplate.ca/functions/env/#examples) alias ::: From c07fbbf3b7b8221547093adb1b4196221e8e11b2 Mon Sep 17 00:00:00 2001 From: aknysh Date: Sun, 28 Apr 2024 22:31:09 -0400 Subject: [PATCH 25/35] updates --- .../components/remote-state-backend.md | 339 ++++++++++++++++++ .../core-concepts/components/remote-state.md | 14 +- 2 files changed, 346 insertions(+), 7 deletions(-) create mode 100644 website/docs/core-concepts/components/remote-state-backend.md diff --git a/website/docs/core-concepts/components/remote-state-backend.md b/website/docs/core-concepts/components/remote-state-backend.md new file mode 100644 index 000000000..6fd7f4ab3 --- /dev/null +++ b/website/docs/core-concepts/components/remote-state-backend.md @@ -0,0 +1,339 @@ +--- +title: Remote State Backend +sidebar_position: 16 +sidebar_label: Remote State Backend +id: remote-state-backend +--- + +The Terraform Component Remote State is used when we need to get the outputs of a [Terraform component](/core-concepts/components), +provisioned in the same or a different [Atmos stack](/core-concepts/stacks), and use the outputs as inputs to another Atmos component. + +:::info + +In Atmos, Terraform Remote State is implemented by using these modules: + +- [terraform-provider-utils](https://github.com/cloudposse/terraform-provider-utils) - The Cloud Posse Terraform Provider for various utilities, + including stack configuration management + +- [remote-state](https://github.com/cloudposse/terraform-yaml-stack-config/tree/main/modules/remote-state) - Terraform module that loads and processes + stack configurations from YAML sources and returns remote state outputs for Terraform components + +::: + +
+ +[terraform-provider-utils](https://github.com/cloudposse/terraform-provider-utils) is implemented in [Go](https://go.dev/) and uses Atmos `Go` +modules to work with [Atmos CLI config](/cli/configuration) and [Atmos stacks](/core-concepts/stacks). The provider processes stack +configurations to get the final config for an Atmos component in an Atmos stack. The final component config is then used by +the [remote-state](https://github.com/cloudposse/terraform-yaml-stack-config/tree/main/modules/remote-state) Terraform module to return the remote +state for the component in the stack. + +
+ +:::info Disambiguation + +- **Terraform Component** is a [Terraform Root Module](https://developer.hashicorp.com/terraform/language/modules#the-root-module) and stored typically in `components/terraform/$name` + that consists of the resources defined in the `.tf` files in a working directory + (e.g. [components/terraform/infra/vpc](https://github.com/cloudposse/atmos/tree/master/examples/quick-start/components/terraform/vpc)) + +- **Atmos Component** provides configuration (variables and other settings) for a component and is defined in one or more Atmos stack manifests + (a.k.a. stack conffig files) + +::: + +
+ +## Example + +Here is an example. + +Suppose that we need to provision two Terraform components: + +- [vpc-flow-logs-bucket](https://github.com/cloudposse/atmos/tree/master/examples/quick-start/components/terraform/vpc-flow-logs-bucket) +- [vpc](https://github.com/cloudposse/atmos/tree/master/examples/quick-start/components/terraform/vpc) + +The `vpc` Terraform component needs the outputs from the `vpc-flow-logs-bucket` Terraform component to +configure [VPC Flow Logs](https://docs.aws.amazon.com/vpc/latest/userguide/flow-logs.html) and store them in the S3 bucket. + +We will provision the two Terraform components in the `ue2-dev` Atmos stack (in the `dev` AWS account by setting `stage = "dev"` and in +the `us-east-2` region by setting `environment = "ue2"`). + +
+ +### Configure and Provision the `vpc-flow-logs-bucket` Component + +In the `stacks/catalog/vpc-flow-logs-bucket.yaml` file, add the following default configuration for the `vpc-flow-logs-bucket/defaults` Atmos +component: + +```yaml title="stacks/catalog/vpc-flow-logs-bucket.yaml" +components: + terraform: + vpc-flow-logs-bucket/defaults: + metadata: + # `metadata.type: abstract` makes the component `abstract`, + # explicitly prohibiting the component from being deployed. + # `atmos terraform apply` will fail with an error. + # If `metadata.type` attribute is not specified, it defaults to `real`. + # `real` components can be provisioned by `atmos` and CI/CD like Spacelift and Atlantis. + type: abstract + # Default variables, which will be inherited and can be overridden in the derived components + vars: + force_destroy: false + lifecycle_rule_enabled: false + traffic_type: "ALL" +``` + +
+ +In the `stacks/ue2-dev.yaml` stack config file, add the following config for the `vpc-flow-logs-bucket-1` Atmos component in the `ue2-dev` Atmos +stack: + +```yaml title="stacks/ue2-dev.yaml" +# Import the base Atmos component configuration from the `catalog`. +# `import` supports POSIX-style Globs for file names/paths (double-star `**` is supported). +# File extensions are optional (if not specified, `.yaml` is used by default). +import: + - catalog/vpc-flow-logs-bucket + +components: + terraform: + vpc-flow-logs-bucket-1: + metadata: + # Point to the Terraform component in `components/terraform` folder + component: infra/vpc-flow-logs-bucket + inherits: + # Inherit all settings and variables from the + # `vpc-flow-logs-bucket/defaults` base Atmos component + - vpc-flow-logs-bucket/defaults + vars: + # Define variables that are specific for this component + # and are not set in the base component + name: vpc-flow-logs-bucket-1 + # Override the default variables from the base component + traffic_type: "REJECT" +``` + +
+ +Having the stacks configured as shown above, we can now provision the `vpc-flow-logs-bucket-1` Atmos component into the `ue2-dev` stack by executing +the following Atmos commands: + +```shell +atmos terraform plan vpc-flow-logs-bucket-1 -s ue2-dev +atmos terraform apply vpc-flow-logs-bucket-1 -s ue2-dev +``` + +
+ +### Configure and Provision the `vpc` Component + +Having the `vpc-flow-logs-bucket` Terraform component provisioned into the `ue2-dev` stack, we can now configure the `vpc` Terraform component +to obtain the outputs from the remote state of the `vpc-flow-logs-bucket-1` Atmos component. + +In the `components/terraform/infra/vpc/remote-state.tf` file, configure the +[remote-state](https://github.com/cloudposse/terraform-yaml-stack-config/tree/main/modules/remote-state) Terraform module to obtain the remote state +for the `vpc-flow-logs-bucket-1` Atmos component: + +```hcl title="components/terraform/infra/vpc/remote-state.tf" +module "vpc_flow_logs_bucket" { + count = local.vpc_flow_logs_enabled ? 1 : 0 + + source = "cloudposse/stack-config/yaml//modules/remote-state" + version = "1.5.0" + + # Specify the Atmos component name (defined in YAML stack config files) + # for which to get the remote state outputs + component = var.vpc_flow_logs_bucket_component_name + + # Override the context variables to point to a different Atmos stack if the + # `vpc-flow-logs-bucket-1` Atmos component is provisioned in another AWS account, OU or region + stage = try(coalesce(var.vpc_flow_logs_bucket_stage_name, module.this.stage), null) + tenant = try(coalesce(var.vpc_flow_logs_bucket_tenant_name, module.this.tenant), null) + environment = try(coalesce(var.vpc_flow_logs_bucket_environment_name, module.this.environment), null) + + # `context` input is a way to provide the information about the stack (using the context + # variables `namespace`, `tenant`, `environment`, `stage` defined in the stack config) + context = module.this.context +} +``` + +In the `components/terraform/infra/vpc/vpc-flow-logs.tf` file, configure the `aws_flow_log` resource for the `vpc` Terraform component to use the +remote state output `vpc_flow_logs_bucket_arn` from the `vpc-flow-logs-bucket-1` Atmos component: + +```hcl title="components/terraform/infra/vpc/vpc-flow-logs.tf" +locals { + enabled = module.this.enabled + vpc_flow_logs_enabled = local.enabled && var.vpc_flow_logs_enabled +} + +resource "aws_flow_log" "default" { + count = local.vpc_flow_logs_enabled ? 1 : 0 + + # Use the remote state output `vpc_flow_logs_bucket_arn` of the `vpc_flow_logs_bucket` component + log_destination = module.vpc_flow_logs_bucket[0].outputs.vpc_flow_logs_bucket_arn + + log_destination_type = var.vpc_flow_logs_log_destination_type + traffic_type = var.vpc_flow_logs_traffic_type + vpc_id = module.vpc.vpc_id + + tags = module.this.tags +} +``` + +In the `stacks/catalog/vpc.yaml` file, add the following default config for the `vpc/defaults` Atmos component: + +```yaml title="stacks/catalog/vpc.yaml" +components: + terraform: + vpc/defaults: + metadata: + # `metadata.type: abstract` makes the component `abstract`, + # explicitly prohibiting the component from being deployed. + # `atmos terraform apply` will fail with an error. + # If `metadata.type` attribute is not specified, it defaults to `real`. + # `real` components can be provisioned by `atmos` and CI/CD like Spacelift and Atlantis. + type: abstract + # Default variables, which will be inherited and can be overridden in the derived components + vars: + public_subnets_enabled: false + nat_gateway_enabled: false + nat_instance_enabled: false + max_subnet_count: 3 + vpc_flow_logs_enabled: false + vpc_flow_logs_log_destination_type: s3 + vpc_flow_logs_traffic_type: "ALL" +``` + +
+ +In the `stacks/ue2-dev.yaml` stack config file, add the following config for the `vpc/1` Atmos component in the `ue2-dev` stack: + +```yaml title="stacks/ue2-dev.yaml" +# Import the base component configuration from the `catalog`. +# `import` supports POSIX-style Globs for file names/paths (double-star `**` is supported). +# File extensions are optional (if not specified, `.yaml` is used by default). +import: + - catalog/vpc + +components: + terraform: + vpc/1: + metadata: + # Point to the Terraform component in `components/terraform` folder + component: infra/vpc + inherits: + # Inherit all settings and variables from the `vpc/defaults` base Atmos component + - vpc/defaults + vars: + # Define variables that are specific for this component + # and are not set in the base component + name: vpc-1 + ipv4_primary_cidr_block: 10.8.0.0/18 + # Override the default variables from the base component + vpc_flow_logs_enabled: true + vpc_flow_logs_traffic_type: "REJECT" + + # Specify the name of the Atmos component that provides configuration + # for the `infra/vpc-flow-logs-bucket` Terraform component + vpc_flow_logs_bucket_component_name: vpc-flow-logs-bucket-1 + + # Override the context variables to point to a different Atmos stack if the + # `vpc-flow-logs-bucket-1` Atmos component is provisioned in another AWS account, OU or region. + + # If the bucket is provisioned in a different AWS account, + # set `vpc_flow_logs_bucket_stage_name` + # vpc_flow_logs_bucket_stage_name: prod + + # If the bucket is provisioned in a different AWS OU, + # set `vpc_flow_logs_bucket_tenant_name` + # vpc_flow_logs_bucket_tenant_name: core + + # If the bucket is provisioned in a different AWS region, + # set `vpc_flow_logs_bucket_environment_name` + # vpc_flow_logs_bucket_environment_name: uw2 +``` + +
+ +Having the stacks configured as shown above, we can now provision the `vpc/1` Atmos component into the `ue2-dev` stack by +executing the following Atmos commands: + +```shell +atmos terraform plan vpc/1 -s ue2-dev +atmos terraform apply vpc/1 -s ue2-dev +``` + +## Caveats + +Both the `atmos` [CLI](/cli) and [terraform-provider-utils](https://github.com/cloudposse/terraform-provider-utils) Terraform provider use the same `Go` code, +which try to locate the [CLI config](/cli/configuration) `atmos.yaml` file before parsing and processing [Atmos stacks](/core-concepts/stacks). + +This means that `atmos.yaml` file must be at a location in the file system where all the processes can find it. + +While placing `atmos.yaml` at the root of the repository will work for Atmos, it will not work for +the [terraform-provider-utils](https://github.com/cloudposse/terraform-provider-utils) Terraform provider because the provider gets executed from the +component's directory (e.g. `components/terraform/infra/vpc`), and we don't want to replicate `atmos.yaml` into every component's folder. + +:::info + +`atmos.yaml` is loaded from the following locations (from lowest to highest priority): + +- System dir (`/usr/local/etc/atmos/atmos.yaml` on Linux, `%LOCALAPPDATA%/atmos/atmos.yaml` on Windows) +- Home dir (`~/.atmos/atmos.yaml`) +- Current directory +- ENV variables `ATMOS_CLI_CONFIG_PATH` and `ATMOS_BASE_PATH` + +::: + +:::note + +Initial Atmos configuration can be controlled by these ENV vars: + +- `ATMOS_CLI_CONFIG_PATH` - where to find `atmos.yaml`. Absolute path to a folder where the `atmos.yaml` CLI config file is located +- `ATMOS_BASE_PATH` - absolute path to the folder containing the `components` and `stacks` folders + +::: + +
+ +For this to work for both the `atmos` CLI and the Terraform provider, we recommend doing one of the following: + +- Put `atmos.yaml` at `/usr/local/etc/atmos/atmos.yaml` on local host and set the ENV var `ATMOS_BASE_PATH` to point to the absolute path of the root + of the repo + +- Put `atmos.yaml` into the home directory (`~/.atmos/atmos.yaml`) and set the ENV var `ATMOS_BASE_PATH` pointing to the absolute path of the root of + the repo + +- Put `atmos.yaml` at a location in the file system and then set the ENV var `ATMOS_CLI_CONFIG_PATH` to point to that location. The ENV var must + point to a folder without the `atmos.yaml` file name. For example, if `atmos.yaml` is at `/atmos/config/atmos.yaml`, + set `ATMOS_CLI_CONFIG_PATH=/atmos/config`. Then set the ENV var `ATMOS_BASE_PATH` pointing to the absolute path of the root of the repo + +- When working in a Docker container, place `atmos.yaml` in the `rootfs` directory + at [/rootfs/usr/local/etc/atmos/atmos.yaml](https://github.com/cloudposse/atmos/blob/master/examples/quick-start/rootfs/usr/local/etc/atmos/atmos.yaml) + and then copy it into the container's file system in the [Dockerfile](https://github.com/cloudposse/atmos/blob/master/examples/quick-start/Dockerfile) + by executing the `COPY rootfs/ /` Docker command. Then in the Dockerfile, set the ENV var `ATMOS_BASE_PATH` pointing to the absolute path of the + root of the repo. Note that the [Atmos example](https://github.com/cloudposse/atmos/blob/master/examples/quick-start) + uses [Geodesic](https://github.com/cloudposse/geodesic) as the base Docker image. [Geodesic](https://github.com/cloudposse/geodesic) sets the ENV + var `ATMOS_BASE_PATH` automatically to the absolute path of the root of the repo on local host + +## Summary + +- Remote State for an Atmos component in an Atmos stack is obtained by using + the [remote-state](https://github.com/cloudposse/terraform-yaml-stack-config/tree/main/modules/remote-state) Terraform module + +- The module calls the [terraform-provider-utils](https://github.com/cloudposse/terraform-provider-utils) Terraform provider which processes the stack + configs and returns the configuration for the Atmos component in the stack. + The [terraform-provider-utils](https://github.com/cloudposse/terraform-provider-utils) Terraform provider utilizes Atmos `Go` modules to parse and + process stack configurations + +- The [remote-state](https://github.com/cloudposse/terraform-yaml-stack-config/tree/main/modules/remote-state) module accepts the `component` input as + the Atmos component name for which to get the remote state outputs + +- The module accepts the `context` input as a way to provide the information about the stack (using the context + variables `namespace`, `tenant`, `environment`, `stage` defined in the stack manifests) + +- If the Atmos component (for which we want to get the remote state outputs) is provisioned in a different Atmos stack (in a different AWS OU, or + different AWS account, or different AWS region), we can override the context variables `tenant`, `stage` and `environment` to point the module to + the correct stack. For example, if the component is provisioned in a different AWS region (let's say `us-west-2`), we can set `environment = "uw2"`, + and the [remote-state](https://github.com/cloudposse/terraform-yaml-stack-config/tree/main/modules/remote-state) module will get the remote state + outputs for the Atmos component provisioned in that region diff --git a/website/docs/core-concepts/components/remote-state.md b/website/docs/core-concepts/components/remote-state.md index 89e30cb46..b8e57fb34 100644 --- a/website/docs/core-concepts/components/remote-state.md +++ b/website/docs/core-concepts/components/remote-state.md @@ -1,11 +1,11 @@ --- title: Terraform Component Remote State -sidebar_position: 8 +sidebar_position: 14 sidebar_label: Remote State id: remote-state --- -The Terraform Component Remote State is used when we need to get the outputs of an [Terraform component](/core-concepts/components), +The Terraform Component Remote State is used when we need to get the outputs of a [Terraform component](/core-concepts/components), provisioned in the same or a different [Atmos stack](/core-concepts/stacks), and use the outputs as inputs to another Atmos component. :::info @@ -34,10 +34,10 @@ state for the component in the stack. - **Terraform Component** is a [Terraform Root Module](https://developer.hashicorp.com/terraform/language/modules#the-root-module) and stored typically in `components/terraform/$name` that consists of the resources defined in the `.tf` files in a working directory - (e.g. [components/terraform/infra/vpc](https://github.com/cloudposse/atmos/tree/master/examples/tests/components/terraform/infra/vpc)) + (e.g. [components/terraform/infra/vpc](https://github.com/cloudposse/atmos/tree/master/examples/quick-start/components/terraform/vpc)) -- **Atmos Component** provides configuration (variables and other settings) for a component and is defined in one or more YAML stack config - files (which are called [Atmos stacks](/core-concepts/stacks)) +- **Atmos Component** provides configuration (variables and other settings) for a component and is defined in one or more Atmos stack manifests + (a.k.a. stack conffig files) ::: @@ -268,7 +268,7 @@ atmos terraform apply vpc/1 -s ue2-dev Both the `atmos` [CLI](/cli) and [terraform-provider-utils](https://github.com/cloudposse/terraform-provider-utils) Terraform provider use the same `Go` code, which try to locate the [CLI config](/cli/configuration) `atmos.yaml` file before parsing and processing [Atmos stacks](/core-concepts/stacks). -This means that `atmos.yaml` file must be at a location in the file system where all processes can find it. +This means that `atmos.yaml` file must be at a location in the file system where all the processes can find it. While placing `atmos.yaml` at the root of the repository will work for Atmos, it will not work for the [terraform-provider-utils](https://github.com/cloudposse/terraform-provider-utils) Terraform provider because the provider gets executed from the @@ -330,7 +330,7 @@ For this to work for both the `atmos` CLI and the Terraform provider, we recomme the Atmos component name for which to get the remote state outputs - The module accepts the `context` input as a way to provide the information about the stack (using the context - variables `namespace`, `tenant`, `environment`, `stage` defined in the stack config) + variables `namespace`, `tenant`, `environment`, `stage` defined in the stack manifests) - If the Atmos component (for which we want to get the remote state outputs) is provisioned in a different Atmos stack (in a different AWS OU, or different AWS account, or different AWS region), we can override the context variables `tenant`, `stage` and `environment` to point the module to From 9902c1e70ab2f222a320231c9ff1510f84a4b391 Mon Sep 17 00:00:00 2001 From: aknysh Date: Sun, 28 Apr 2024 23:40:09 -0400 Subject: [PATCH 26/35] updates --- .../components/remote-state-backend.md | 344 +----------------- 1 file changed, 20 insertions(+), 324 deletions(-) diff --git a/website/docs/core-concepts/components/remote-state-backend.md b/website/docs/core-concepts/components/remote-state-backend.md index 6fd7f4ab3..fc6fec8c4 100644 --- a/website/docs/core-concepts/components/remote-state-backend.md +++ b/website/docs/core-concepts/components/remote-state-backend.md @@ -5,335 +5,31 @@ sidebar_label: Remote State Backend id: remote-state-backend --- -The Terraform Component Remote State is used when we need to get the outputs of a [Terraform component](/core-concepts/components), -provisioned in the same or a different [Atmos stack](/core-concepts/stacks), and use the outputs as inputs to another Atmos component. +Atmos supports configuring [Terraform Backends](/core-concepts/components/terraform-backends) to define where +Terraform stores its [state](https://developer.hashicorp.com/terraform/language/state) data files, +and [Remote State](/core-concepts/components/remote-state) to get the outputs +of a [Terraform component](/core-concepts/components), provisioned in the same or a +different [Atmos stack](/core-concepts/stacks), and use +the outputs as inputs to another Atmos component -:::info +Atmos also supports Remote State Backends (in the `remote_state_backend` section), which can be used to configure the +following: -In Atmos, Terraform Remote State is implemented by using these modules: +- Override [Terraform Backend](/core-concepts/components/terraform-backends) configuration when accessing the + remote state of a component (e.g. override the IAM role to assume, which in this case can be a read-only role) -- [terraform-provider-utils](https://github.com/cloudposse/terraform-provider-utils) - The Cloud Posse Terraform Provider for various utilities, - including stack configuration management +- Configure a remote state of type `static` which can be used to provide configurations for Atmos components for + [Brownfield development](https://en.wikipedia.org/wiki/Brownfield_(software_development)) -- [remote-state](https://github.com/cloudposse/terraform-yaml-stack-config/tree/main/modules/remote-state) - Terraform module that loads and processes - stack configurations from YAML sources and returns remote state outputs for Terraform components +## Override Terraform Backend Configuration -::: +## `static` Remote State for Brownfield development -
+[Brownfield development](https://en.wikipedia.org/wiki/Brownfield_(software_development)) is a term commonly used in the +information technology industry to describe problem spaces needing the development and deployment of new software +systems in the immediate presence of existing (legacy) software applications/systems. This implies that any new software +architecture must take into account and coexist with the existing software. -[terraform-provider-utils](https://github.com/cloudposse/terraform-provider-utils) is implemented in [Go](https://go.dev/) and uses Atmos `Go` -modules to work with [Atmos CLI config](/cli/configuration) and [Atmos stacks](/core-concepts/stacks). The provider processes stack -configurations to get the final config for an Atmos component in an Atmos stack. The final component config is then used by -the [remote-state](https://github.com/cloudposse/terraform-yaml-stack-config/tree/main/modules/remote-state) Terraform module to return the remote -state for the component in the stack. +Similarly, in Atmos, brownfield development describes the process of configuring Atmos components and stacks for the +existing (already provisioned) resources. -
- -:::info Disambiguation - -- **Terraform Component** is a [Terraform Root Module](https://developer.hashicorp.com/terraform/language/modules#the-root-module) and stored typically in `components/terraform/$name` - that consists of the resources defined in the `.tf` files in a working directory - (e.g. [components/terraform/infra/vpc](https://github.com/cloudposse/atmos/tree/master/examples/quick-start/components/terraform/vpc)) - -- **Atmos Component** provides configuration (variables and other settings) for a component and is defined in one or more Atmos stack manifests - (a.k.a. stack conffig files) - -::: - -
- -## Example - -Here is an example. - -Suppose that we need to provision two Terraform components: - -- [vpc-flow-logs-bucket](https://github.com/cloudposse/atmos/tree/master/examples/quick-start/components/terraform/vpc-flow-logs-bucket) -- [vpc](https://github.com/cloudposse/atmos/tree/master/examples/quick-start/components/terraform/vpc) - -The `vpc` Terraform component needs the outputs from the `vpc-flow-logs-bucket` Terraform component to -configure [VPC Flow Logs](https://docs.aws.amazon.com/vpc/latest/userguide/flow-logs.html) and store them in the S3 bucket. - -We will provision the two Terraform components in the `ue2-dev` Atmos stack (in the `dev` AWS account by setting `stage = "dev"` and in -the `us-east-2` region by setting `environment = "ue2"`). - -
- -### Configure and Provision the `vpc-flow-logs-bucket` Component - -In the `stacks/catalog/vpc-flow-logs-bucket.yaml` file, add the following default configuration for the `vpc-flow-logs-bucket/defaults` Atmos -component: - -```yaml title="stacks/catalog/vpc-flow-logs-bucket.yaml" -components: - terraform: - vpc-flow-logs-bucket/defaults: - metadata: - # `metadata.type: abstract` makes the component `abstract`, - # explicitly prohibiting the component from being deployed. - # `atmos terraform apply` will fail with an error. - # If `metadata.type` attribute is not specified, it defaults to `real`. - # `real` components can be provisioned by `atmos` and CI/CD like Spacelift and Atlantis. - type: abstract - # Default variables, which will be inherited and can be overridden in the derived components - vars: - force_destroy: false - lifecycle_rule_enabled: false - traffic_type: "ALL" -``` - -
- -In the `stacks/ue2-dev.yaml` stack config file, add the following config for the `vpc-flow-logs-bucket-1` Atmos component in the `ue2-dev` Atmos -stack: - -```yaml title="stacks/ue2-dev.yaml" -# Import the base Atmos component configuration from the `catalog`. -# `import` supports POSIX-style Globs for file names/paths (double-star `**` is supported). -# File extensions are optional (if not specified, `.yaml` is used by default). -import: - - catalog/vpc-flow-logs-bucket - -components: - terraform: - vpc-flow-logs-bucket-1: - metadata: - # Point to the Terraform component in `components/terraform` folder - component: infra/vpc-flow-logs-bucket - inherits: - # Inherit all settings and variables from the - # `vpc-flow-logs-bucket/defaults` base Atmos component - - vpc-flow-logs-bucket/defaults - vars: - # Define variables that are specific for this component - # and are not set in the base component - name: vpc-flow-logs-bucket-1 - # Override the default variables from the base component - traffic_type: "REJECT" -``` - -
- -Having the stacks configured as shown above, we can now provision the `vpc-flow-logs-bucket-1` Atmos component into the `ue2-dev` stack by executing -the following Atmos commands: - -```shell -atmos terraform plan vpc-flow-logs-bucket-1 -s ue2-dev -atmos terraform apply vpc-flow-logs-bucket-1 -s ue2-dev -``` - -
- -### Configure and Provision the `vpc` Component - -Having the `vpc-flow-logs-bucket` Terraform component provisioned into the `ue2-dev` stack, we can now configure the `vpc` Terraform component -to obtain the outputs from the remote state of the `vpc-flow-logs-bucket-1` Atmos component. - -In the `components/terraform/infra/vpc/remote-state.tf` file, configure the -[remote-state](https://github.com/cloudposse/terraform-yaml-stack-config/tree/main/modules/remote-state) Terraform module to obtain the remote state -for the `vpc-flow-logs-bucket-1` Atmos component: - -```hcl title="components/terraform/infra/vpc/remote-state.tf" -module "vpc_flow_logs_bucket" { - count = local.vpc_flow_logs_enabled ? 1 : 0 - - source = "cloudposse/stack-config/yaml//modules/remote-state" - version = "1.5.0" - - # Specify the Atmos component name (defined in YAML stack config files) - # for which to get the remote state outputs - component = var.vpc_flow_logs_bucket_component_name - - # Override the context variables to point to a different Atmos stack if the - # `vpc-flow-logs-bucket-1` Atmos component is provisioned in another AWS account, OU or region - stage = try(coalesce(var.vpc_flow_logs_bucket_stage_name, module.this.stage), null) - tenant = try(coalesce(var.vpc_flow_logs_bucket_tenant_name, module.this.tenant), null) - environment = try(coalesce(var.vpc_flow_logs_bucket_environment_name, module.this.environment), null) - - # `context` input is a way to provide the information about the stack (using the context - # variables `namespace`, `tenant`, `environment`, `stage` defined in the stack config) - context = module.this.context -} -``` - -In the `components/terraform/infra/vpc/vpc-flow-logs.tf` file, configure the `aws_flow_log` resource for the `vpc` Terraform component to use the -remote state output `vpc_flow_logs_bucket_arn` from the `vpc-flow-logs-bucket-1` Atmos component: - -```hcl title="components/terraform/infra/vpc/vpc-flow-logs.tf" -locals { - enabled = module.this.enabled - vpc_flow_logs_enabled = local.enabled && var.vpc_flow_logs_enabled -} - -resource "aws_flow_log" "default" { - count = local.vpc_flow_logs_enabled ? 1 : 0 - - # Use the remote state output `vpc_flow_logs_bucket_arn` of the `vpc_flow_logs_bucket` component - log_destination = module.vpc_flow_logs_bucket[0].outputs.vpc_flow_logs_bucket_arn - - log_destination_type = var.vpc_flow_logs_log_destination_type - traffic_type = var.vpc_flow_logs_traffic_type - vpc_id = module.vpc.vpc_id - - tags = module.this.tags -} -``` - -In the `stacks/catalog/vpc.yaml` file, add the following default config for the `vpc/defaults` Atmos component: - -```yaml title="stacks/catalog/vpc.yaml" -components: - terraform: - vpc/defaults: - metadata: - # `metadata.type: abstract` makes the component `abstract`, - # explicitly prohibiting the component from being deployed. - # `atmos terraform apply` will fail with an error. - # If `metadata.type` attribute is not specified, it defaults to `real`. - # `real` components can be provisioned by `atmos` and CI/CD like Spacelift and Atlantis. - type: abstract - # Default variables, which will be inherited and can be overridden in the derived components - vars: - public_subnets_enabled: false - nat_gateway_enabled: false - nat_instance_enabled: false - max_subnet_count: 3 - vpc_flow_logs_enabled: false - vpc_flow_logs_log_destination_type: s3 - vpc_flow_logs_traffic_type: "ALL" -``` - -
- -In the `stacks/ue2-dev.yaml` stack config file, add the following config for the `vpc/1` Atmos component in the `ue2-dev` stack: - -```yaml title="stacks/ue2-dev.yaml" -# Import the base component configuration from the `catalog`. -# `import` supports POSIX-style Globs for file names/paths (double-star `**` is supported). -# File extensions are optional (if not specified, `.yaml` is used by default). -import: - - catalog/vpc - -components: - terraform: - vpc/1: - metadata: - # Point to the Terraform component in `components/terraform` folder - component: infra/vpc - inherits: - # Inherit all settings and variables from the `vpc/defaults` base Atmos component - - vpc/defaults - vars: - # Define variables that are specific for this component - # and are not set in the base component - name: vpc-1 - ipv4_primary_cidr_block: 10.8.0.0/18 - # Override the default variables from the base component - vpc_flow_logs_enabled: true - vpc_flow_logs_traffic_type: "REJECT" - - # Specify the name of the Atmos component that provides configuration - # for the `infra/vpc-flow-logs-bucket` Terraform component - vpc_flow_logs_bucket_component_name: vpc-flow-logs-bucket-1 - - # Override the context variables to point to a different Atmos stack if the - # `vpc-flow-logs-bucket-1` Atmos component is provisioned in another AWS account, OU or region. - - # If the bucket is provisioned in a different AWS account, - # set `vpc_flow_logs_bucket_stage_name` - # vpc_flow_logs_bucket_stage_name: prod - - # If the bucket is provisioned in a different AWS OU, - # set `vpc_flow_logs_bucket_tenant_name` - # vpc_flow_logs_bucket_tenant_name: core - - # If the bucket is provisioned in a different AWS region, - # set `vpc_flow_logs_bucket_environment_name` - # vpc_flow_logs_bucket_environment_name: uw2 -``` - -
- -Having the stacks configured as shown above, we can now provision the `vpc/1` Atmos component into the `ue2-dev` stack by -executing the following Atmos commands: - -```shell -atmos terraform plan vpc/1 -s ue2-dev -atmos terraform apply vpc/1 -s ue2-dev -``` - -## Caveats - -Both the `atmos` [CLI](/cli) and [terraform-provider-utils](https://github.com/cloudposse/terraform-provider-utils) Terraform provider use the same `Go` code, -which try to locate the [CLI config](/cli/configuration) `atmos.yaml` file before parsing and processing [Atmos stacks](/core-concepts/stacks). - -This means that `atmos.yaml` file must be at a location in the file system where all the processes can find it. - -While placing `atmos.yaml` at the root of the repository will work for Atmos, it will not work for -the [terraform-provider-utils](https://github.com/cloudposse/terraform-provider-utils) Terraform provider because the provider gets executed from the -component's directory (e.g. `components/terraform/infra/vpc`), and we don't want to replicate `atmos.yaml` into every component's folder. - -:::info - -`atmos.yaml` is loaded from the following locations (from lowest to highest priority): - -- System dir (`/usr/local/etc/atmos/atmos.yaml` on Linux, `%LOCALAPPDATA%/atmos/atmos.yaml` on Windows) -- Home dir (`~/.atmos/atmos.yaml`) -- Current directory -- ENV variables `ATMOS_CLI_CONFIG_PATH` and `ATMOS_BASE_PATH` - -::: - -:::note - -Initial Atmos configuration can be controlled by these ENV vars: - -- `ATMOS_CLI_CONFIG_PATH` - where to find `atmos.yaml`. Absolute path to a folder where the `atmos.yaml` CLI config file is located -- `ATMOS_BASE_PATH` - absolute path to the folder containing the `components` and `stacks` folders - -::: - -
- -For this to work for both the `atmos` CLI and the Terraform provider, we recommend doing one of the following: - -- Put `atmos.yaml` at `/usr/local/etc/atmos/atmos.yaml` on local host and set the ENV var `ATMOS_BASE_PATH` to point to the absolute path of the root - of the repo - -- Put `atmos.yaml` into the home directory (`~/.atmos/atmos.yaml`) and set the ENV var `ATMOS_BASE_PATH` pointing to the absolute path of the root of - the repo - -- Put `atmos.yaml` at a location in the file system and then set the ENV var `ATMOS_CLI_CONFIG_PATH` to point to that location. The ENV var must - point to a folder without the `atmos.yaml` file name. For example, if `atmos.yaml` is at `/atmos/config/atmos.yaml`, - set `ATMOS_CLI_CONFIG_PATH=/atmos/config`. Then set the ENV var `ATMOS_BASE_PATH` pointing to the absolute path of the root of the repo - -- When working in a Docker container, place `atmos.yaml` in the `rootfs` directory - at [/rootfs/usr/local/etc/atmos/atmos.yaml](https://github.com/cloudposse/atmos/blob/master/examples/quick-start/rootfs/usr/local/etc/atmos/atmos.yaml) - and then copy it into the container's file system in the [Dockerfile](https://github.com/cloudposse/atmos/blob/master/examples/quick-start/Dockerfile) - by executing the `COPY rootfs/ /` Docker command. Then in the Dockerfile, set the ENV var `ATMOS_BASE_PATH` pointing to the absolute path of the - root of the repo. Note that the [Atmos example](https://github.com/cloudposse/atmos/blob/master/examples/quick-start) - uses [Geodesic](https://github.com/cloudposse/geodesic) as the base Docker image. [Geodesic](https://github.com/cloudposse/geodesic) sets the ENV - var `ATMOS_BASE_PATH` automatically to the absolute path of the root of the repo on local host - -## Summary - -- Remote State for an Atmos component in an Atmos stack is obtained by using - the [remote-state](https://github.com/cloudposse/terraform-yaml-stack-config/tree/main/modules/remote-state) Terraform module - -- The module calls the [terraform-provider-utils](https://github.com/cloudposse/terraform-provider-utils) Terraform provider which processes the stack - configs and returns the configuration for the Atmos component in the stack. - The [terraform-provider-utils](https://github.com/cloudposse/terraform-provider-utils) Terraform provider utilizes Atmos `Go` modules to parse and - process stack configurations - -- The [remote-state](https://github.com/cloudposse/terraform-yaml-stack-config/tree/main/modules/remote-state) module accepts the `component` input as - the Atmos component name for which to get the remote state outputs - -- The module accepts the `context` input as a way to provide the information about the stack (using the context - variables `namespace`, `tenant`, `environment`, `stage` defined in the stack manifests) - -- If the Atmos component (for which we want to get the remote state outputs) is provisioned in a different Atmos stack (in a different AWS OU, or - different AWS account, or different AWS region), we can override the context variables `tenant`, `stage` and `environment` to point the module to - the correct stack. For example, if the component is provisioned in a different AWS region (let's say `us-west-2`), we can set `environment = "uw2"`, - and the [remote-state](https://github.com/cloudposse/terraform-yaml-stack-config/tree/main/modules/remote-state) module will get the remote state - outputs for the Atmos component provisioned in that region From f55db7a7d891dcac8e8ff08ef1125853b703e61f Mon Sep 17 00:00:00 2001 From: aknysh Date: Sun, 28 Apr 2024 23:52:48 -0400 Subject: [PATCH 27/35] updates --- .../components/remote-state-backend.md | 53 +++++++++++++++++-- 1 file changed, 48 insertions(+), 5 deletions(-) diff --git a/website/docs/core-concepts/components/remote-state-backend.md b/website/docs/core-concepts/components/remote-state-backend.md index fc6fec8c4..ea2e03403 100644 --- a/website/docs/core-concepts/components/remote-state-backend.md +++ b/website/docs/core-concepts/components/remote-state-backend.md @@ -6,7 +6,7 @@ id: remote-state-backend --- Atmos supports configuring [Terraform Backends](/core-concepts/components/terraform-backends) to define where -Terraform stores its [state](https://developer.hashicorp.com/terraform/language/state) data files, +Terraform stores its [state](https://developer.hashicorp.com/terraform/language/state), and [Remote State](/core-concepts/components/remote-state) to get the outputs of a [Terraform component](/core-concepts/components), provisioned in the same or a different [Atmos stack](/core-concepts/stacks), and use @@ -23,13 +23,56 @@ following: ## Override Terraform Backend Configuration -## `static` Remote State for Brownfield development +## Brownfield Development [Brownfield development](https://en.wikipedia.org/wiki/Brownfield_(software_development)) is a term commonly used in the information technology industry to describe problem spaces needing the development and deployment of new software systems in the immediate presence of existing (legacy) software applications/systems. This implies that any new software -architecture must take into account and coexist with the existing software. +architecture must take into account and coexist with the existing software. The term "brownfield" itself is borrowed +from urban planning, where it describes the process of developing on previously used land that may require cleanup or +modification. -Similarly, in Atmos, brownfield development describes the process of configuring Atmos components and stacks for the -existing (already provisioned) resources. +In the context of DevOps, brownfield development involves integrating new tools, practices, and technologies into +established systems. This can be challenging due to several factors: +- **Legacy Systems**: These are older technologies or systems that are still in use. They might not support modern + practices or tools easily, and modifying them can be risky and time-consuming. + +- **Complex Integrations**: Existing systems often have a complex set of integrations and dependencies which need to be + understood and managed when new elements are introduced. + +- **Cultural Shifts**: DevOps emphasizes a culture of collaboration between development and operations teams. In + brownfield projects, shifting the organizational culture can be a significant hurdle as existing processes and + mindsets may be deeply ingrained. + +- **Technical Debt**: Over time, systems accumulate technical debt, which includes outdated code, lack of proper + documentation, and suboptimal previous decisions that were made for expediency. Addressing technical debt is crucial + when introducing DevOps practices to ensure that the system remains maintainable and scalable. + +- **Compliance and Security**: Updating old systems often requires careful consideration of security and compliance + issues, especially if the system handles sensitive data or must meet specific regulatory standards. + +Strategies for Successful Brownfield DevOps Development: + +- **Incremental Changes**: Instead of large-scale overhauls, implementing small, manageable changes can reduce risk and + help the team learn and adapt progressively. + +- **Automation**: Automating as many processes as possible, such as testing, deployments, and monitoring, can help + integrate DevOps practices without disrupting existing operations. + +- **Documentation and Training**: Proper documentation of new processes and extensive training for team members can + facilitate a smoother transition and adoption of DevOps methodologies. + +- **Tool Compatibility**: Choosing tools and technologies that can integrate well with the existing stack is essential + to avoid disruptions and compatibility issues. + +- **Continuous Feedback**: Encouraging continuous feedback from all stakeholders involved can help identify pain points + and areas for improvement early in the development process. + +## `static` Remote State for Brownfield Development + +In Atmos, brownfield development describes the process of configuring Atmos components and stacks for the +existing (already provisioned) resources, and working on and updating existing infrastructure rather than creating new +ones from scratch (which is known as "greenfield development"). The process respects the existing systems' constraints +while progressively introducing improvements and modern practices. This can ultimately lead to more robust, flexible, +and efficient systems. From 711596232acdef70249747e4c93fa4b71bd4abd3 Mon Sep 17 00:00:00 2001 From: aknysh Date: Mon, 29 Apr 2024 13:46:15 -0400 Subject: [PATCH 28/35] updates --- .../components/remote-state-backend.md | 71 ++++++++++++++++++- 1 file changed, 68 insertions(+), 3 deletions(-) diff --git a/website/docs/core-concepts/components/remote-state-backend.md b/website/docs/core-concepts/components/remote-state-backend.md index ea2e03403..77a69c9db 100644 --- a/website/docs/core-concepts/components/remote-state-backend.md +++ b/website/docs/core-concepts/components/remote-state-backend.md @@ -15,13 +15,78 @@ the outputs as inputs to another Atmos component Atmos also supports Remote State Backends (in the `remote_state_backend` section), which can be used to configure the following: -- Override [Terraform Backend](/core-concepts/components/terraform-backends) configuration when accessing the +- Override [Terraform Backend](/core-concepts/components/terraform-backends) configuration to access the remote state of a component (e.g. override the IAM role to assume, which in this case can be a read-only role) -- Configure a remote state of type `static` which can be used to provide configurations for Atmos components for +- Configure a remote state of type `static` which can be used to provide configurations for [Brownfield development](https://en.wikipedia.org/wiki/Brownfield_(software_development)) -## Override Terraform Backend Configuration +## Override Terraform Backend Configuration to Access Remote State + +Atmos supports the `remote_state_backend` section which can be used to provide configuration to access the remote state +of components. + +To access the remote state of components, you can override +any [Terraform Backend](/core-concepts/components/terraform-backends) +configuration in the `backend` section using the `remote_state_backend` section. The `remote_state_backend` section +is a first-class section, and it can be defined globally at any scope (organization, tenant, account, region), or per +component, and then deep-merged using [Atmos Component Inheritance](/core-concepts/components/inheritance). + +For example, let's suppose we have the following S3 backend configuration for the entire organization +(refer to [AWS S3 Backend](/core-concepts/components/terraform-backends#aws-s3-backend) for more details): + +```yaml title="stacks/orgs/acme/_defaults.yaml" +terraform: + backend_type: s3 + backend: + s3: + acl: "bucket-owner-full-control" + encrypt: true + bucket: "your-s3-bucket-name" + dynamodb_table: "your-dynamodb-table-name" + key: "terraform.tfstate" + region: "your-aws-region" + role_arn: "arn:aws:iam::xxxxxxxx:role/terraform-backend-read-write" +``` + +
+ +Let's say we also have a read-only IAM role, and we want to use it to access the remote state instead of the read-write +role, because accessing remote state is a read-only operation, and we don't want to give the role more permissions than +it requires - this is the principle of least privileges. + +We can add the `remote_state_backend` and `remote_state_backend_type` to override the required attributes from the +`backend` section: + +```yaml title="stacks/orgs/acme/_defaults.yaml" +terraform: + backend_type: s3 + backend: + s3: + acl: "bucket-owner-full-control" + encrypt: true + bucket: "your-s3-bucket-name" + dynamodb_table: "your-dynamodb-table-name" + key: "terraform.tfstate" + region: "your-aws-region" + role_arn: "arn:aws:iam::xxxxxxxx:role/terraform-backend-read-write" + + remote_state_backend_type: s3 + remote_state_backend: + s3: + role_arn: "arn:aws:iam::xxxxxxxx:role/terraform-backend-read-only" + # Override the other attributes as needed +``` + +
+ +In the example above, we've overridden the `role_arn` attribute for the `s3` backend to use the read-only role when +accessing the remote state of all components. All other attributes will be taken from the `backend` section (Atmos +deep-merges the `remote_state_backend` section with the `backend` section). + +When working with Terraform backends and writing/updating the state, the `terraform-backend-read-write` role will be +used. +But when reading the remote state of components, the `terraform-backend-read-only` role will be used. ## Brownfield Development From d9bc1b9c280939791c37cbfbe47113bc1671db37 Mon Sep 17 00:00:00 2001 From: aknysh Date: Mon, 29 Apr 2024 13:49:34 -0400 Subject: [PATCH 29/35] updates --- .../docs/core-concepts/components/remote-state-backend.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/website/docs/core-concepts/components/remote-state-backend.md b/website/docs/core-concepts/components/remote-state-backend.md index 77a69c9db..923ade322 100644 --- a/website/docs/core-concepts/components/remote-state-backend.md +++ b/website/docs/core-concepts/components/remote-state-backend.md @@ -53,7 +53,7 @@ terraform: Let's say we also have a read-only IAM role, and we want to use it to access the remote state instead of the read-write role, because accessing remote state is a read-only operation, and we don't want to give the role more permissions than -it requires - this is the principle of least privileges. +it requires - this is the [principle of least privilege](https://en.wikipedia.org/wiki/Principle_of_least_privilege). We can add the `remote_state_backend` and `remote_state_backend_type` to override the required attributes from the `backend` section: @@ -75,7 +75,7 @@ terraform: remote_state_backend: s3: role_arn: "arn:aws:iam::xxxxxxxx:role/terraform-backend-read-only" - # Override the other attributes as needed + # Override the other attributes from the `backend.s3` section as needed ```
@@ -85,8 +85,7 @@ accessing the remote state of all components. All other attributes will be taken deep-merges the `remote_state_backend` section with the `backend` section). When working with Terraform backends and writing/updating the state, the `terraform-backend-read-write` role will be -used. -But when reading the remote state of components, the `terraform-backend-read-only` role will be used. +used. But when reading the remote state of components, the `terraform-backend-read-only` role will be used. ## Brownfield Development From 7b4cb8d26e2c9b4d8fb492853844908206755b39 Mon Sep 17 00:00:00 2001 From: aknysh Date: Mon, 29 Apr 2024 14:05:56 -0400 Subject: [PATCH 30/35] updates --- .../spacelift-and-backend-override-1.yaml | 2 +- examples/tests/stacks/orgs/cp/_defaults.yaml | 2 +- .../components/remote-state-backend.md | 14 +++++++++++--- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/examples/tests/stacks/catalog/terraform/spacelift-and-backend-override-1.yaml b/examples/tests/stacks/catalog/terraform/spacelift-and-backend-override-1.yaml index f2a9fe15e..34b56a350 100644 --- a/examples/tests/stacks/catalog/terraform/spacelift-and-backend-override-1.yaml +++ b/examples/tests/stacks/catalog/terraform/spacelift-and-backend-override-1.yaml @@ -7,7 +7,7 @@ settings: terraform: vars: {} - backend_type: s3 # s3, remote, vault, static, azurerm, gcs, cloud + backend_type: s3 # s3, remote, vault, azurerm, gcs, cloud backend: s3: encrypt: true diff --git a/examples/tests/stacks/orgs/cp/_defaults.yaml b/examples/tests/stacks/orgs/cp/_defaults.yaml index fbee39c1f..a46aee6c4 100644 --- a/examples/tests/stacks/orgs/cp/_defaults.yaml +++ b/examples/tests/stacks/orgs/cp/_defaults.yaml @@ -12,7 +12,7 @@ terraform: region: "{{ .vars.region }}" terraform_workspace: "{{ .workspace }}" - backend_type: s3 # s3, remote, vault, static, azurerm, gcs, cloud + backend_type: s3 # s3, remote, vault, azurerm, gcs, cloud # https://developer.hashicorp.com/terraform/language/settings/backends/configuration backend: diff --git a/website/docs/core-concepts/components/remote-state-backend.md b/website/docs/core-concepts/components/remote-state-backend.md index 923ade322..b0b85f148 100644 --- a/website/docs/core-concepts/components/remote-state-backend.md +++ b/website/docs/core-concepts/components/remote-state-backend.md @@ -60,7 +60,7 @@ We can add the `remote_state_backend` and `remote_state_backend_type` to overrid ```yaml title="stacks/orgs/acme/_defaults.yaml" terraform: - backend_type: s3 + backend_type: s3 # s3, remote, vault, azurerm, gcs, cloud backend: s3: acl: "bucket-owner-full-control" @@ -71,7 +71,7 @@ terraform: region: "your-aws-region" role_arn: "arn:aws:iam::xxxxxxxx:role/terraform-backend-read-write" - remote_state_backend_type: s3 + remote_state_backend_type: s3 # s3, remote, vault, azurerm, gcs, cloud, static remote_state_backend: s3: role_arn: "arn:aws:iam::xxxxxxxx:role/terraform-backend-read-only" @@ -133,10 +133,18 @@ Strategies for Successful Brownfield DevOps Development: - **Continuous Feedback**: Encouraging continuous feedback from all stakeholders involved can help identify pain points and areas for improvement early in the development process. -## `static` Remote State for Brownfield Development +## Brownfield Development in Atmos In Atmos, brownfield development describes the process of configuring Atmos components and stacks for the existing (already provisioned) resources, and working on and updating existing infrastructure rather than creating new ones from scratch (which is known as "greenfield development"). The process respects the existing systems' constraints while progressively introducing improvements and modern practices. This can ultimately lead to more robust, flexible, and efficient systems. + +Atmos supports brownfield configuration by using the `static` type of remote state. + +## `static` Remote State for Brownfield Development + +### Example 1 + +### Example 2 From 014e976c565588ca930247c6b5c4cec18436c8b9 Mon Sep 17 00:00:00 2001 From: aknysh Date: Mon, 29 Apr 2024 15:53:34 -0400 Subject: [PATCH 31/35] updates --- .../components/remote-state-backend.md | 181 +++++++++++++++++- website/docs/integrations/terraform.md | 2 +- 2 files changed, 179 insertions(+), 4 deletions(-) diff --git a/website/docs/core-concepts/components/remote-state-backend.md b/website/docs/core-concepts/components/remote-state-backend.md index b0b85f148..bf005aab9 100644 --- a/website/docs/core-concepts/components/remote-state-backend.md +++ b/website/docs/core-concepts/components/remote-state-backend.md @@ -141,10 +141,185 @@ ones from scratch (which is known as "greenfield development"). The process resp while progressively introducing improvements and modern practices. This can ultimately lead to more robust, flexible, and efficient systems. -Atmos supports brownfield configuration by using the `static` type of remote state. +Atmos supports brownfield configuration by using the remote state of type `static`. ## `static` Remote State for Brownfield Development -### Example 1 +Suppose that we need to provision +the [`vpc`](https://github.com/cloudposse/atmos/tree/master/examples/quick-start/components/terraform/vpc) +Terraform component and, instead of provisioning an S3 bucket for VPC Flow Logs, we want to use an existing bucket. + +The `vpc` Terraform component needs the outputs from the `vpc-flow-logs-bucket` Terraform component to +configure [VPC Flow Logs](https://docs.aws.amazon.com/vpc/latest/userguide/flow-logs.html). + +Let's redesign the example with the `vpc` and `vpc-flow-logs-bucket` components described in +[Terraform Component Remote State](/core-concepts/components/remote-state) and configure the `static` remote state for +the `vpc-flow-logs-bucket` component to use an existing S3 bucket. + +### Configure the `vpc-flow-logs-bucket` Component + +In the `stacks/catalog/vpc-flow-logs-bucket.yaml` file, add the following configuration for +the `vpc-flow-logs-bucket/defaults` Atmos component: + +```yaml title="stacks/catalog/vpc-flow-logs-bucket.yaml" +components: + terraform: + vpc-flow-logs-bucket/defaults: + metadata: + type: abstract + # Use `static` remote state to configure the attributes for an existing + # S3 bucket for VPC Flow Logs + remote_state_backend_type: static + remote_state_backend: + static: + # ARN of the existing S3 bucket + # `vpc_flow_logs_bucket_arn` is used as an input for the `vpc` component + vpc_flow_logs_bucket_arn: "arn:aws:s3::/my-vpc-flow-logs-bucket" +``` + +
+ +In the `stacks/ue2-dev.yaml` stack config file, add the following config for the `vpc-flow-logs-bucket-1` Atmos +component in the `ue2-dev` Atmos stack: + +```yaml title="stacks/ue2-dev.yaml" +# Import the base Atmos component configuration from the `catalog`. +# `import` supports POSIX-style Globs for file names/paths (double-star `**` is supported). +# File extensions are optional (if not specified, `.yaml` is used by default). +import: + - catalog/vpc-flow-logs-bucket + +components: + terraform: + vpc-flow-logs-bucket-1: + metadata: + # Point to the Terraform component in `components/terraform` folder + component: infra/vpc-flow-logs-bucket + inherits: + # Inherit all settings and variables from the + # `vpc-flow-logs-bucket/defaults` base Atmos component + - vpc-flow-logs-bucket/defaults +``` + +
+ +### Configure and Provision the `vpc` Component + +In the `components/terraform/infra/vpc/remote-state.tf` file, configure the +[remote-state](https://github.com/cloudposse/terraform-yaml-stack-config/tree/main/modules/remote-state) Terraform +module to obtain the remote state for the `vpc-flow-logs-bucket-1` Atmos component: + +```hcl title="components/terraform/infra/vpc/remote-state.tf" +module "vpc_flow_logs_bucket" { + count = local.vpc_flow_logs_enabled ? 1 : 0 + + source = "cloudposse/stack-config/yaml//modules/remote-state" + version = "1.5.0" + + # Specify the Atmos component name (defined in YAML stack config files) + # for which to get the remote state outputs + component = var.vpc_flow_logs_bucket_component_name + + # `context` input is a way to provide the information about the stack (using the context + # variables `namespace`, `tenant`, `environment`, `stage` defined in the stack config) + context = module.this.context +} +``` + +In the `components/terraform/infra/vpc/vpc-flow-logs.tf` file, configure the `aws_flow_log` resource for the `vpc` +Terraform component to use the remote state output `vpc_flow_logs_bucket_arn` from the `vpc-flow-logs-bucket-1` Atmos +component: + +```hcl title="components/terraform/infra/vpc/vpc-flow-logs.tf" +locals { + enabled = module.this.enabled + vpc_flow_logs_enabled = local.enabled && var.vpc_flow_logs_enabled +} + +resource "aws_flow_log" "default" { + count = local.vpc_flow_logs_enabled ? 1 : 0 + + # Use the remote state output `vpc_flow_logs_bucket_arn` of the `vpc_flow_logs_bucket` component + log_destination = module.vpc_flow_logs_bucket[0].outputs.vpc_flow_logs_bucket_arn + + log_destination_type = var.vpc_flow_logs_log_destination_type + traffic_type = var.vpc_flow_logs_traffic_type + vpc_id = module.vpc.vpc_id + + tags = module.this.tags +} +``` + +In the `stacks/catalog/vpc.yaml` file, add the following default config for the `vpc/defaults` Atmos component: + +```yaml title="stacks/catalog/vpc.yaml" +components: + terraform: + vpc/defaults: + metadata: + # `metadata.type: abstract` makes the component `abstract`, + # explicitly prohibiting the component from being deployed. + # `atmos terraform apply` will fail with an error. + # If `metadata.type` attribute is not specified, it defaults to `real`. + # `real` components can be provisioned by `atmos` and CI/CD like Spacelift and Atlantis. + type: abstract + # Default variables, which will be inherited and can be overridden in the derived components + vars: + public_subnets_enabled: false + nat_gateway_enabled: false + nat_instance_enabled: false + max_subnet_count: 3 + vpc_flow_logs_enabled: false + vpc_flow_logs_log_destination_type: s3 + vpc_flow_logs_traffic_type: "ALL" +``` + +
+ +In the `stacks/ue2-dev.yaml` stack config file, add the following config for the `vpc/1` Atmos component in +the `ue2-dev` stack: + +```yaml title="stacks/ue2-dev.yaml" +# Import the base component configuration from the `catalog`. +# `import` supports POSIX-style Globs for file names/paths (double-star `**` is supported). +# File extensions are optional (if not specified, `.yaml` is used by default). +import: + - catalog/vpc + +components: + terraform: + vpc/1: + metadata: + # Point to the Terraform component in `components/terraform` folder + component: infra/vpc + inherits: + # Inherit all settings and variables from the `vpc/defaults` base Atmos component + - vpc/defaults + vars: + # Define variables that are specific for this component + # and are not set in the base component + name: vpc-1 + ipv4_primary_cidr_block: 10.8.0.0/18 + # Override the default variables from the base component + vpc_flow_logs_enabled: true + vpc_flow_logs_traffic_type: "REJECT" + + # Specify the name of the Atmos component that provides configuration + # for the `infra/vpc-flow-logs-bucket` Terraform component + vpc_flow_logs_bucket_component_name: vpc-flow-logs-bucket-1 +``` + +
+ +Having the stacks configured as shown above, we can now provision the `vpc/1` Atmos component in the `ue2-dev` stack +by executing the following Atmos commands: + +```shell +atmos terraform plan vpc/1 -s ue2-dev +atmos terraform apply vpc/1 -s ue2-dev +``` -### Example 2 +When the commands are executed, the `vpc_flow_logs_bucket` remote-state module detects that the `vpc-flow-logs-bucket-1` +component has the `static` remote state configured, and instead of reading its remote state from the S3 state +bucket, it just returns the static values from the `remote_state_backend.static` section. +The `vpc_flow_logs_bucket_arn` is then used as an input for the `vpc` component. diff --git a/website/docs/integrations/terraform.md b/website/docs/integrations/terraform.md index 6f528a23d..7cfc60b67 100644 --- a/website/docs/integrations/terraform.md +++ b/website/docs/integrations/terraform.md @@ -103,7 +103,7 @@ atmos terraform plan eks -s ue2-dev atmos terraform apply eks -s ue2-dev ``` -`terraform deploy` command executes `terraform apply -auto-approve` to provision components into stacks without user interaction: +`terraform deploy` command executes `terraform apply -auto-approve` to provision components in stacks without user interaction: ```console atmos terraform deploy eks -s ue2-dev From 3fd55cedb1a0c047bab1ecae88755027f8335934 Mon Sep 17 00:00:00 2001 From: aknysh Date: Mon, 29 Apr 2024 17:45:47 -0400 Subject: [PATCH 32/35] updates --- .../components/remote-state-backend.md | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/website/docs/core-concepts/components/remote-state-backend.md b/website/docs/core-concepts/components/remote-state-backend.md index bf005aab9..ab7147c39 100644 --- a/website/docs/core-concepts/components/remote-state-backend.md +++ b/website/docs/core-concepts/components/remote-state-backend.md @@ -116,23 +116,6 @@ established systems. This can be challenging due to several factors: - **Compliance and Security**: Updating old systems often requires careful consideration of security and compliance issues, especially if the system handles sensitive data or must meet specific regulatory standards. -Strategies for Successful Brownfield DevOps Development: - -- **Incremental Changes**: Instead of large-scale overhauls, implementing small, manageable changes can reduce risk and - help the team learn and adapt progressively. - -- **Automation**: Automating as many processes as possible, such as testing, deployments, and monitoring, can help - integrate DevOps practices without disrupting existing operations. - -- **Documentation and Training**: Proper documentation of new processes and extensive training for team members can - facilitate a smoother transition and adoption of DevOps methodologies. - -- **Tool Compatibility**: Choosing tools and technologies that can integrate well with the existing stack is essential - to avoid disruptions and compatibility issues. - -- **Continuous Feedback**: Encouraging continuous feedback from all stakeholders involved can help identify pain points - and areas for improvement early in the development process. - ## Brownfield Development in Atmos In Atmos, brownfield development describes the process of configuring Atmos components and stacks for the From fa0301cba998c09125bf65aa3959a0fc3bcab399 Mon Sep 17 00:00:00 2001 From: aknysh Date: Mon, 29 Apr 2024 17:59:13 -0400 Subject: [PATCH 33/35] updates --- .../components/remote-state-backend.md | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/website/docs/core-concepts/components/remote-state-backend.md b/website/docs/core-concepts/components/remote-state-backend.md index ab7147c39..e4e597e01 100644 --- a/website/docs/core-concepts/components/remote-state-backend.md +++ b/website/docs/core-concepts/components/remote-state-backend.md @@ -118,9 +118,22 @@ established systems. This can be challenging due to several factors: ## Brownfield Development in Atmos +Atmos is easier for new organizations or "greenfield" environments because you need to architect Terraform according to +our best practices to get all the benefits of Atmos. For example, when using +our [Terraform components](https://github.com/cloudposse/terraform-aws-components), we frequently use +[Terraform Remote State](/core-concepts/components/remote-state) to retrieve the outputs from other components. +This works well when you use our components but less so when you operate in a "brownfield" environment, for example, +with an existing VPC, S3 bucket, or IAM role. + +But what happens if those things weren't provisioned by Atmos or predates your infrastructure? For this reason, we +support something we refer to as the `static` remote state backend. Using the static remote state backend, you can +populate a virtual state backend with the outputs as though it had been provisioned with Terraform. You can use this +technique anytime you want to use the remote state functionality in Atmos, but when the remote state was provisioned +elsewhere. + In Atmos, brownfield development describes the process of configuring Atmos components and stacks for the -existing (already provisioned) resources, and working on and updating existing infrastructure rather than creating new -ones from scratch (which is known as "greenfield development"). The process respects the existing systems' constraints +existing (already provisioned) resources, and working on and updating existing infrastructure rather than creating a new +one from scratch (which is known as "greenfield" development). The process respects the existing systems' constraints while progressively introducing improvements and modern practices. This can ultimately lead to more robust, flexible, and efficient systems. From ff5f70cd793c62940991064614b2592e596afc0b Mon Sep 17 00:00:00 2001 From: Andriy Knysh Date: Tue, 30 Apr 2024 10:58:05 -0400 Subject: [PATCH 34/35] Apply suggestions from code review address comments Co-authored-by: Erik Osterman (CEO @ Cloud Posse) --- .../components/remote-state-backend.md | 27 +++++-------------- 1 file changed, 7 insertions(+), 20 deletions(-) diff --git a/website/docs/core-concepts/components/remote-state-backend.md b/website/docs/core-concepts/components/remote-state-backend.md index e4e597e01..ba4c13d52 100644 --- a/website/docs/core-concepts/components/remote-state-backend.md +++ b/website/docs/core-concepts/components/remote-state-backend.md @@ -87,34 +87,21 @@ deep-merges the `remote_state_backend` section with the `backend` section). When working with Terraform backends and writing/updating the state, the `terraform-backend-read-write` role will be used. But when reading the remote state of components, the `terraform-backend-read-only` role will be used. -## Brownfield Development +## Brownfield Considerations -[Brownfield development](https://en.wikipedia.org/wiki/Brownfield_(software_development)) is a term commonly used in the -information technology industry to describe problem spaces needing the development and deployment of new software -systems in the immediate presence of existing (legacy) software applications/systems. This implies that any new software -architecture must take into account and coexist with the existing software. The term "brownfield" itself is borrowed -from urban planning, where it describes the process of developing on previously used land that may require cleanup or -modification. +The term "brownfield" comes from urban planning and refers to the redevelopment of land that was previously used and may need cleaning or modification. As it relates to infrastructure, [Brownfield development](https://en.wikipedia.org/wiki/Brownfield_(software_development)) describes development and deployment of new software +systems in the presence of existing (legacy) software applications/systems. Anytime this happens, new software architectures must take into account and coexist with the existing software. -In the context of DevOps, brownfield development involves integrating new tools, practices, and technologies into +In the context of Atmos, brownfield development involves adopting some new techniques, practices, and integrating them into established systems. This can be challenging due to several factors: -- **Legacy Systems**: These are older technologies or systems that are still in use. They might not support modern - practices or tools easily, and modifying them can be risky and time-consuming. +- **Legacy Systems**: These are older systems that are still in use and may be harder to change. They might not even be managed with Terraform, or if they are, they might not provide outputs that are easily accessible in a conventional manner. -- **Complex Integrations**: Existing systems often have a complex set of integrations and dependencies which need to be - understood and managed when new elements are introduced. +- **Complex Integrations**: Existing systems might leverage entirely different toolchains, such as CloudFormation, CDK, or Pulumi. Managing integrations and dependencies across toolchains is complicated. -- **Cultural Shifts**: DevOps emphasizes a culture of collaboration between development and operations teams. In - brownfield projects, shifting the organizational culture can be a significant hurdle as existing processes and - mindsets may be deeply ingrained. -- **Technical Debt**: Over time, systems accumulate technical debt, which includes outdated code, lack of proper - documentation, and suboptimal previous decisions that were made for expediency. Addressing technical debt is crucial - when introducing DevOps practices to ensure that the system remains maintainable and scalable. +- **Technical Debt**: Over time, systems naturally accumulate technical debt, which includes outdated code, inconsistent conventions or lack of naming conventions, and suboptimal decisions that were previously made for expediency. Addressing this technical debt is crucial when adopting Atmos, as it introduces practices that ensure your system remains maintainable and scalable. -- **Compliance and Security**: Updating old systems often requires careful consideration of security and compliance - issues, especially if the system handles sensitive data or must meet specific regulatory standards. ## Brownfield Development in Atmos From 2473cbdbf629e7ffb5deb39405de793d3a20f5cf Mon Sep 17 00:00:00 2001 From: aknysh Date: Tue, 30 Apr 2024 11:59:02 -0400 Subject: [PATCH 35/35] Add Google Tag Manager --- website/docusaurus.config.js | 8 +++++++- website/package-lock.json | 27 ++++++++++++++------------- website/package.json | 5 +++-- 3 files changed, 24 insertions(+), 16 deletions(-) diff --git a/website/docusaurus.config.js b/website/docusaurus.config.js index 83f55759e..7b3ced105 100644 --- a/website/docusaurus.config.js +++ b/website/docusaurus.config.js @@ -57,7 +57,13 @@ const config = { }], [ 'custom-loaders', {} - ] + ], + [ + '@docusaurus/plugin-google-tag-manager', + { + containerId: 'GTM-KQ62MGX9', + }, + ], ], presets: [ diff --git a/website/package-lock.json b/website/package-lock.json index ddf3e1efe..f7d11419f 100644 --- a/website/package-lock.json +++ b/website/package-lock.json @@ -11,6 +11,7 @@ "dependencies": { "@docusaurus/core": "^3.2.1", "@docusaurus/plugin-client-redirects": "^3.2.1", + "@docusaurus/plugin-google-gtag": "^3.2.1", "@docusaurus/preset-classic": "^3.2.1", "@docusaurus/theme-mermaid": "^3.2.1", "@grnet/docusaurus-terminology": "^2.0.0-rc.1", @@ -21,8 +22,8 @@ "html-loader": "^5.0.0", "marked": "^12.0.2", "prism-react-renderer": "^2.3.1", - "react": "^18.2.0", - "react-dom": "^18.2.0", + "react": "^18.3.1", + "react-dom": "^18.3.1", "react-image-gallery": "^1.3.0", "react-player": "^2.16.0", "react-table": "^7.8.0", @@ -14344,9 +14345,9 @@ } }, "node_modules/react": { - "version": "18.2.0", - "resolved": "https://registry.npmjs.org/react/-/react-18.2.0.tgz", - "integrity": "sha512-/3IjMdb2L9QbBdWiW5e3P2/npwMBaU9mHCSCUzNln0ZCYbcfTsGbTJrU/kGemdH2IWmB2ioZ+zkxtmq6g09fGQ==", + "version": "18.3.1", + "resolved": "https://registry.npmjs.org/react/-/react-18.3.1.tgz", + "integrity": "sha512-wS+hAgJShR0KhEvPJArfuPVN1+Hz1t0Y6n5jLrGQbkb4urgPE/0Rve+1kMB1v/oWgHgm4WIcV+i7F2pTVj+2iQ==", "dependencies": { "loose-envify": "^1.1.0" }, @@ -14454,15 +14455,15 @@ } }, "node_modules/react-dom": { - "version": "18.2.0", - "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-18.2.0.tgz", - "integrity": "sha512-6IMTriUmvsjHUjNtEDudZfuDQUoWXVxKHhlEGSk81n4YFS+r/Kl99wXiwlVXtPBtJenozv2P+hxDsw9eA7Xo6g==", + "version": "18.3.1", + "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-18.3.1.tgz", + "integrity": "sha512-5m4nQKp+rZRb09LNH59GM4BxTh9251/ylbKIbpe7TpGxfJ+9kv6BLkLBXIjjspbgbnIBNqlI23tRnTWT0snUIw==", "dependencies": { "loose-envify": "^1.1.0", - "scheduler": "^0.23.0" + "scheduler": "^0.23.2" }, "peerDependencies": { - "react": "^18.2.0" + "react": "^18.3.1" } }, "node_modules/react-error-overlay": { @@ -15562,9 +15563,9 @@ "integrity": "sha512-0s+oAmw9zLl1V1cS9BtZN7JAd0cW5e0QH4W3LWEK6a4LaLEA2OTpGYWDY+6XasBLtz6wkm3u1xRw95mRuJ59WA==" }, "node_modules/scheduler": { - "version": "0.23.0", - "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.23.0.tgz", - "integrity": "sha512-CtuThmgHNg7zIZWAXi3AsyIzA3n4xx7aNyjwC2VJldO2LMVDhFK+63xGqq6CsJH4rTAt6/M+N4GhZiDYPx9eUw==", + "version": "0.23.2", + "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.23.2.tgz", + "integrity": "sha512-UOShsPwz7NrMUqhR6t0hWjFduvOzbtv7toDH1/hIrfRNIDBnnBWd0CwJTGvTpngVlmwGCdP9/Zl/tVrDqcuYzQ==", "dependencies": { "loose-envify": "^1.1.0" } diff --git a/website/package.json b/website/package.json index a83400053..e5adf3df1 100644 --- a/website/package.json +++ b/website/package.json @@ -19,6 +19,7 @@ "@docusaurus/plugin-client-redirects": "^3.2.1", "@docusaurus/preset-classic": "^3.2.1", "@docusaurus/theme-mermaid": "^3.2.1", + "@docusaurus/plugin-google-tag-manager": "^3.2.1", "@grnet/docusaurus-terminology": "^2.0.0-rc.1", "@mdx-js/react": "^3.0.1", "clsx": "^2.1.0", @@ -27,8 +28,8 @@ "html-loader": "^5.0.0", "marked": "^12.0.2", "prism-react-renderer": "^2.3.1", - "react": "^18.2.0", - "react-dom": "^18.2.0", + "react": "^18.3.1", + "react-dom": "^18.3.1", "react-image-gallery": "^1.3.0", "react-player": "^2.16.0", "react-table": "^7.8.0",