feat: Generate dependencies concurrently (#283)

* feat: Generate dependencies concurrently

* docs: Documenting `--parallelism`

* fix: Adding some testing

* fix: Sorting with best practices for Golang funcs

* fix: Removing silly tests

* fix: Removing unused imports

* fix: Use no limit when `Parallelism()` isn't set

* fix: Setting `ctx` parameter for `doProcess`

* fix: Resolving conflicts

* fix: Preventing non-deterministic ordering of dependencies, despite concurrent generation

* fix: Propagating manifest management to child templates

* fix: Addressing lint findings

* feat: Adding external manifest management controls

* fix: Use byte slice / file instead of parsed manifest for validate

* fix: Making it more straightforward to handle future versions of the schema

* fix: Addressing lint findings

* fix: Adding tests to the `v1` package

Author: yhakbar

cli/boilerplate_cli.go

@@ -5,10 +5,11 @@ import (
 	"context"
 	"fmt"
 	"path/filepath"
+	"runtime"
 
 	"github.com/urfave/cli/v2"
 
-	"github.com/gruntwork-io/boilerplate/internal/manifest"
+	"github.com/gruntwork-io/boilerplate/manifest"
 	"github.com/gruntwork-io/boilerplate/options"
 	"github.com/gruntwork-io/boilerplate/templates"
 	"github.com/gruntwork-io/boilerplate/variables"
@@ -105,6 +106,11 @@ func CreateBoilerplateCli() *cli.App {
 			Name:  options.OptManifestFile,
 			Usage: "Write the manifest to `FILE` instead of the default location. Implies --manifest. Format is auto-detected from extension (.yaml/.yml for YAML, otherwise JSON).",
 		},
+		&cli.IntFlag{
+			Name:  options.OptParallelism,
+			Value: runtime.NumCPU(),
+			Usage: "Maximum number of parallel operations Boilerplate will perform at once (default: number of CPUs).",
+		},
 	}
 
 	// We pass JSON/YAML content to various CLI flags, such as --var, and this JSON/YAML content may contain commas or

cli/parse_options.go

@@ -41,20 +41,21 @@ func ParseCLIContext(cliContext *cli.Context) (*options.BoilerplateOptions, erro
 	}
 
 	opts := &options.BoilerplateOptions{
+		Vars:                    vars,
+		ShellCommandAnswers:     make(map[string]bool),
 		TemplateURL:             templateURL,
 		TemplateFolder:          templateFolder,
 		OutputFolder:            cliContext.String(options.OptOutputFolder),
-		NonInteractive:          cliContext.Bool(options.OptNonInteractive),
 		OnMissingKey:            missingKeyAction,
 		OnMissingConfig:         missingConfigAction,
-		Vars:                    vars,
+		NonInteractive:          cliContext.Bool(options.OptNonInteractive),
 		NoHooks:                 cliContext.Bool(options.OptNoHooks),
 		NoShell:                 cliContext.Bool(options.OptNoShell),
 		DisableDependencyPrompt: cliContext.Bool(options.OptDisableDependencyPrompt),
 		ExecuteAllShellCommands: false,
-		ShellCommandAnswers:     make(map[string]bool),
 		Manifest:                cliContext.Bool(options.OptManifest) || cliContext.String(options.OptManifestFile) != "",
 		ManifestFile:            cliContext.String(options.OptManifestFile),
+		Parallelism:             cliContext.Int(options.OptParallelism),
 	}
 
 	if err := validateOptions(opts); err != nil {

docs/src/content/docs/cli/flags.mdx

@@ -56,6 +56,7 @@ boilerplate [options]
 | `--disable-dependency-prompt` | `false` | Skip confirmation prompts for dependencies (keeps variable prompts) |
 | `--no-hooks` | `false` | Don't execute any hooks |
 | `--no-shell` | `false` | Don't execute shell helpers (returns `"replace-me"` instead) |
+| `--parallelism` | Number of CPUs | Maximum number of concurrent parallel operations Boilerplate will perform. Use `--parallelism=1` to disable concurrency |
 
 ## Manifest Flags
 

docs/src/content/docs/configuration/dependencies.mdx

@@ -169,6 +169,12 @@ dependencies:
 
 Inside the child template, access the current item with `{{ "{{" }} .__each__ {{ "}}" }}`.
 
+`for_each` iterations are processed **concurrently** by default, using up to one goroutine per CPU. You can control this with the [`--parallelism`](/docs/cli/flags/) flag. If your dependency has side effects that require sequential execution (e.g., writing to the same file, depending on ordering), disable concurrency with:
+
+```bash
+boilerplate --parallelism=1 ...
+```
+
 ### `for_each_reference`
 
 Reference a list variable instead of a static list:

go.mod

@@ -18,6 +18,7 @@ require (
 	github.com/stuart-warren/yamlfmt v0.1.2
 	github.com/urfave/cli/v2 v2.27.7
 	github.com/xeipuuv/gojsonschema v1.2.0
+	golang.org/x/sync v0.19.0
 	golang.org/x/text v0.33.0
 	gopkg.in/yaml.v3 v3.0.1
 )
@@ -119,7 +120,6 @@ require (
 	golang.org/x/crypto v0.47.0 // indirect
 	golang.org/x/net v0.49.0 // indirect
 	golang.org/x/oauth2 v0.34.0 // indirect
-	golang.org/x/sync v0.19.0 // indirect
 	golang.org/x/sys v0.40.0 // indirect
 	golang.org/x/term v0.39.0 // indirect
 	golang.org/x/time v0.14.0 // indirect

integration-tests/examples_test.go

@@ -13,7 +13,7 @@ import (
 
 	"github.com/gruntwork-io/boilerplate/cli"
 	"github.com/gruntwork-io/boilerplate/internal/fileutil"
-	"github.com/gruntwork-io/boilerplate/internal/manifest"
+	"github.com/gruntwork-io/boilerplate/manifest"
 	"github.com/gruntwork-io/boilerplate/options"
 )