Skip to content

Commit

Permalink
fix parallel projects creation (#141)
Browse files Browse the repository at this point in the history
  • Loading branch information
@fbad
fbad committed May 20, 2021
1 parent bb10365 commit 3c78a99
Show file tree
Hide file tree
Showing 5 changed files with 184 additions and 237 deletions.
1 change: 1 addition & 0 deletions README.md
View file
Expand Up @@ -129,6 +129,7 @@ One way to customize the behavior of this module is through CLI flag values pass
| `--terraform-version` | Default terraform version to specify for all modules. Can be overriden by locals | "" |
| `--ignore-dependency-blocks` | When true, dependencies found in `dependency` and `dependencies` blocks will be ignored | false |
| `--filter` | Path or glob expression to the directory you want scope down the config for. Default is all files in root | "" |
| `--num-executors` | Number of executors used for parallel generation of projects. Default is 15 | 15 |

## All Locals

Expand Down
285 changes: 164 additions & 121 deletions cmd/generate.go
View file
Expand Up @@ -12,6 +12,8 @@ import (
"github.com/spf13/cobra"

"golang.org/x/sync/errgroup"
"golang.org/x/sync/semaphore"
"golang.org/x/sync/singleflight"

"context"
"io/ioutil"
Expand Down Expand Up @@ -46,159 +48,190 @@ func makePathAbsolute(path string, parentPath string) string {
return filepath.Join(parentDir, path)
}

var requestGroup singleflight.Group

// Set up a cache for the getDependencies function
type getDependenciesOutput struct {
dependencies []string
err error
}

var getDependenciesCache = make(map[string]getDependenciesOutput)
type GetDependenciesCache struct {
mtx sync.RWMutex
data map[string]getDependenciesOutput
}

func newGetDependenciesCache() *GetDependenciesCache {
return &GetDependenciesCache{data: map[string]getDependenciesOutput{}}
}

func (m *GetDependenciesCache) set(k string, v getDependenciesOutput) {
m.mtx.Lock()
defer m.mtx.Unlock()
m.data[k] = v
}

func (m *GetDependenciesCache) get(k string) (getDependenciesOutput, bool) {
m.mtx.RLock()
defer m.mtx.RUnlock()
v, ok := m.data[k]
return v, ok
}

var getDependenciesCache = newGetDependenciesCache()

// Parses the terragrunt config at `path` to find all modules it depends on
func getDependencies(path string, terragruntOptions *options.TerragruntOptions) ([]string, error) {
// Check if this path has already been computed
cachedResult, ok := getDependenciesCache[path]
if ok {
return cachedResult.dependencies, cachedResult.err
}

// if theres no terraform source and we're ignoring parent terragrunt configs
// return nils to indicate we should skip this project
isParent, err := isParentModule(path, terragruntOptions)
if err != nil {
getDependenciesCache[path] = getDependenciesOutput{nil, err}
return nil, err
}
if ignoreParentTerragrunt && isParent {
getDependenciesCache[path] = getDependenciesOutput{nil, nil}
return nil, nil
}
res, err, _ := requestGroup.Do(path, func() (interface{}, error) {
// Check if this path has already been computed
cachedResult, ok := getDependenciesCache.get(path)
if ok {
return cachedResult.dependencies, cachedResult.err
}
// if theres no terraform source and we're ignoring parent terragrunt configs
// return nils to indicate we should skip this project
isParent, err := isParentModule(path, terragruntOptions)
if err != nil {
getDependenciesCache.set(path, getDependenciesOutput{nil, err})
return nil, err
}
if ignoreParentTerragrunt && isParent {
getDependenciesCache.set(path, getDependenciesOutput{nil, nil})
return nil, nil
}

// Parse the HCL file
decodeTypes := []config.PartialDecodeSectionType{
config.DependencyBlock,
config.DependenciesBlock,
config.TerraformBlock,
}
parsedConfig, err := config.PartialParseConfigFile(path, terragruntOptions, nil, decodeTypes)
if err != nil {
getDependenciesCache[path] = getDependenciesOutput{nil, err}
return nil, err
}
// Parse the HCL file
decodeTypes := []config.PartialDecodeSectionType{
config.DependencyBlock,
config.DependenciesBlock,
config.TerraformBlock,
}
parsedConfig, err := config.PartialParseConfigFile(path, terragruntOptions, nil, decodeTypes)
if err != nil {
getDependenciesCache.set(path, getDependenciesOutput{nil, err})
return nil, err
}

// Parse out locals
locals, err := parseLocals(path, terragruntOptions, nil)
if err != nil {
getDependenciesCache[path] = getDependenciesOutput{nil, err}
return nil, err
}
// Parse out locals
locals, err := parseLocals(path, terragruntOptions, nil)
if err != nil {
getDependenciesCache.set(path, getDependenciesOutput{nil, err})
return nil, err
}

// Get deps from locals
dependencies := []string{}
if locals.ExtraAtlantisDependencies != nil {
dependencies = locals.ExtraAtlantisDependencies
}
// Get deps from locals
dependencies := []string{}
if locals.ExtraAtlantisDependencies != nil {
dependencies = locals.ExtraAtlantisDependencies
}

// Get deps from `dependencies` and `dependency` blocks
if parsedConfig.Dependencies != nil && !ignoreDependencyBlocks {
for _, parsedPaths := range parsedConfig.Dependencies.Paths {
dependencies = append(dependencies, filepath.Join(parsedPaths, "terragrunt.hcl"))
// Get deps from `dependencies` and `dependency` blocks
if parsedConfig.Dependencies != nil && !ignoreDependencyBlocks {
for _, parsedPaths := range parsedConfig.Dependencies.Paths {
dependencies = append(dependencies, filepath.Join(parsedPaths, "terragrunt.hcl"))
}
}
}

// Get deps from the `Source` field of the `Terraform` block
if parsedConfig.Terraform != nil && parsedConfig.Terraform.Source != nil {
source := parsedConfig.Terraform.Source
// TODO: Make more robust. Check for bitbucket, etc.
if !strings.Contains(*source, "git::") && !strings.Contains(*source, "github.com") {
dependencies = append(dependencies, filepath.Join(*source, "*.tf*"))
// Get deps from the `Source` field of the `Terraform` block
if parsedConfig.Terraform != nil && parsedConfig.Terraform.Source != nil {
source := parsedConfig.Terraform.Source
// TODO: Make more robust. Check for bitbucket, etc.
if !strings.Contains(*source, "git::") && !strings.Contains(*source, "github.com") {
dependencies = append(dependencies, filepath.Join(*source, "*.tf*"))
}
}
}

// Get deps from `extra_arguments` fields of the `Terraform` block
if parsedConfig.Terraform != nil && parsedConfig.Terraform.ExtraArgs != nil {
extraArgs := parsedConfig.Terraform.ExtraArgs
for _, arg := range extraArgs {
if arg.RequiredVarFiles != nil {
for _, file := range *arg.RequiredVarFiles {
dependencies = append(dependencies, file)
// Get deps from `extra_arguments` fields of the `Terraform` block
if parsedConfig.Terraform != nil && parsedConfig.Terraform.ExtraArgs != nil {
extraArgs := parsedConfig.Terraform.ExtraArgs
for _, arg := range extraArgs {
if arg.RequiredVarFiles != nil {
for _, file := range *arg.RequiredVarFiles {
dependencies = append(dependencies, file)
}
}
}
if arg.OptionalVarFiles != nil {
for _, file := range *arg.OptionalVarFiles {
dependencies = append(dependencies, file)
if arg.OptionalVarFiles != nil {
for _, file := range *arg.OptionalVarFiles {
dependencies = append(dependencies, file)
}
}
}
if arg.Arguments != nil {
for _, cliFlag := range *arg.Arguments {
if strings.HasPrefix(cliFlag, "-var-file=") {
dependencies = append(dependencies, strings.TrimPrefix(cliFlag, "-var-file="))
if arg.Arguments != nil {
for _, cliFlag := range *arg.Arguments {
if strings.HasPrefix(cliFlag, "-var-file=") {
dependencies = append(dependencies, strings.TrimPrefix(cliFlag, "-var-file="))
}
}
}
}
}
}

// Filter out and dependencies that are the empty string
nonEmptyDeps := []string{}
for _, dep := range dependencies {
if dep != "" {
childDepAbsPath := dep
if !filepath.IsAbs(childDepAbsPath) {
childDepAbsPath = makePathAbsolute(dep, path)
// Filter out and dependencies that are the empty string
nonEmptyDeps := []string{}
for _, dep := range dependencies {
if dep != "" {
childDepAbsPath := dep
if !filepath.IsAbs(childDepAbsPath) {
childDepAbsPath = makePathAbsolute(dep, path)
}
childDepAbsPath = filepath.ToSlash(childDepAbsPath)
nonEmptyDeps = append(nonEmptyDeps, childDepAbsPath)
}
childDepAbsPath = filepath.ToSlash(childDepAbsPath)
nonEmptyDeps = append(nonEmptyDeps, childDepAbsPath)
}
}

// Recurse to find dependencies of all dependencies
cascadedDeps := []string{}
for _, dep := range nonEmptyDeps {
cascadedDeps = append(cascadedDeps, dep)
// Recurse to find dependencies of all dependencies
cascadedDeps := []string{}
for _, dep := range nonEmptyDeps {
cascadedDeps = append(cascadedDeps, dep)

// The "cascading" feature is protected by a flag
if !cascadeDependencies {
continue
}
// The "cascading" feature is protected by a flag
if !cascadeDependencies {
continue
}

depPath := dep
terrOpts, err := options.NewTerragruntOptions(depPath)
childDeps, err := getDependencies(depPath, terrOpts)
if err != nil {
continue
}
depPath := dep
terrOpts, err := options.NewTerragruntOptions(depPath)
childDeps, err := getDependencies(depPath, terrOpts)
if err != nil {
continue
}

for _, childDep := range childDeps {
// If `childDep` is a relative path, it will be relative to `childDep`, as it is from the nested
// `getDependencies` call on the top level module's dependencies. So here we update any relative
// path to be from the top level module instead.
childDepAbsPath := childDep
if !filepath.IsAbs(childDep) {
childDepAbsPath, err = filepath.Abs(filepath.Join(depPath, "..", childDep))
if err != nil {
getDependenciesCache[path] = getDependenciesOutput{nil, err}
return nil, err
for _, childDep := range childDeps {
// If `childDep` is a relative path, it will be relative to `childDep`, as it is from the nested
// `getDependencies` call on the top level module's dependencies. So here we update any relative
// path to be from the top level module instead.
childDepAbsPath := childDep
if !filepath.IsAbs(childDep) {
childDepAbsPath, err = filepath.Abs(filepath.Join(depPath, "..", childDep))
if err != nil {
getDependenciesCache.set(path, getDependenciesOutput{nil, err})
return nil, err
}
}
}
childDepAbsPath = filepath.ToSlash(childDepAbsPath)

// Ensure we are not adding a duplicate dependency
alreadyExists := false
for _, dep := range cascadedDeps {
if dep == childDepAbsPath {
alreadyExists = true
break
childDepAbsPath = filepath.ToSlash(childDepAbsPath)

// Ensure we are not adding a duplicate dependency
alreadyExists := false
for _, dep := range cascadedDeps {
if dep == childDepAbsPath {
alreadyExists = true
break
}
}
if !alreadyExists {
cascadedDeps = append(cascadedDeps, childDepAbsPath)
}
}
if !alreadyExists {
cascadedDeps = append(cascadedDeps, childDepAbsPath)
}
}

getDependenciesCache.set(path, getDependenciesOutput{cascadedDeps, err})
return cascadedDeps, nil
})
if res != nil {
return res.([]string), err
} else {
return nil, err
}

getDependenciesCache[path] = getDependenciesOutput{cascadedDeps, nil}
return cascadedDeps, nil
}

// Creates an AtlantisProject for a directory
Expand Down Expand Up @@ -389,13 +422,21 @@ func main(cmd *cobra.Command, args []string) error {
}

lock := sync.Mutex{}
errGroup, _ := errgroup.WithContext(context.Background())
ctx := context.Background()
errGroup, _ := errgroup.WithContext(ctx)
sem := semaphore.NewWeighted(numExecutors)

// Concurrently looking all dependencies
for _, terragruntPath := range terragruntFiles {
terragruntPath := terragruntPath // https://golang.org/doc/faq#closures_and_goroutines

err := sem.Acquire(ctx, 1)
if err != nil {
return err
}

errGroup.Go(func() error {
defer sem.Release(1)
project, err := createProject(terragruntPath)
if err != nil {
return err
Expand All @@ -414,10 +455,10 @@ func main(cmd *cobra.Command, args []string) error {

return nil
})
}

if err := errGroup.Wait(); err != nil {
return err
}
if err := errGroup.Wait(); err != nil {
return err
}

// Convert config to YAML string
Expand Down Expand Up @@ -458,6 +499,7 @@ var outputPath string
var preserveWorkflows bool
var cascadeDependencies bool
var defaultApplyRequirements []string
var numExecutors int64

// generateCmd represents the generate command
var generateCmd = &cobra.Command{
Expand Down Expand Up @@ -490,6 +532,7 @@ func init() {
generateCmd.PersistentFlags().StringVar(&filterPath, "filter", "", "Path or glob expression to the directory you want scope down the config for. Default is all files in root")
generateCmd.PersistentFlags().StringVar(&gitRoot, "root", pwd, "Path to the root directory of the git repo you want to build config for. Default is current dir")
generateCmd.PersistentFlags().StringVar(&defaultTerraformVersion, "terraform-version", "", "Default terraform version to specify for all modules. Can be overriden by locals")
generateCmd.PersistentFlags().Int64Var(&numExecutors, "num-executors", 15, "Number of executors used for parallel generation of projects. Default is 15")
}

// Runs a set of arguments, returning the output
Expand Down

0 comments on commit 3c78a99

Please sign in to comment.