Skip to content

Commit

Permalink
feat(template): archive rendering.
Browse files Browse the repository at this point in the history
  • Loading branch information
@Zenithar
Zenithar committed Mar 13, 2022
1 parent 58026c5 commit 4574eed
Show file tree
Hide file tree
Showing 15 changed files with 643 additions and 162 deletions.
113 changes: 113 additions & 0 deletions cmd/harp/internal/cmd/render.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
// Licensed to Elasticsearch B.V. under one or more contributor
// license agreements. See the NOTICE file distributed with
// this work for additional information regarding copyright
// ownership. Elasticsearch B.V. licenses this file to you under
// the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

package cmd

import (
"github.com/spf13/cobra"
"go.uber.org/zap"

"github.com/elastic/harp/pkg/sdk/cmdutil"
"github.com/elastic/harp/pkg/sdk/log"
"github.com/elastic/harp/pkg/tasks/template"
)

type renderParams struct {
InputPath string
OutputPath string
ValueFiles []string
SecretLoaders []string
Values []string
StringValues []string
FileValues []string
LeftDelims string
RightDelims string
AltDelims bool
RootPath string
DryRun bool
}

// -----------------------------------------------------------------------------

var renderCmd = func() *cobra.Command {
params := &renderParams{}

longDesc := cmdutil.LongDesc(`
Generate a config filesytem from a template hierarchy or archive.
`)
examples := cmdutil.Examples(`
# Generate a configuration filesystem from a folder hierarchy
harp render --in templates/database --out postgres
# Generate a configuration filesystem from an archive
harp render --in templates.tar.gz --out configMap
# Test template generation
harp render --in templates.tar.gz --dry-run
`)

cmd := &cobra.Command{
Use: "render",
Aliases: []string{"r"},
Short: "Render a template filesystem",
Long: longDesc,
Example: examples,
Run: func(cmd *cobra.Command, args []string) {
// Initialize logger and context
ctx, cancel := cmdutil.Context(cmd.Context(), "template-render", conf.Debug.Enable, conf.Instrumentation.Logs.Level)
defer cancel()

// Prepare task
t := &template.FileSystemTask{
InputPath: params.InputPath,
OutputPath: params.OutputPath,
ValueFiles: params.ValueFiles,
Values: params.Values,
StringValues: params.StringValues,
FileValues: params.FileValues,
FileLoaderRootPath: params.RootPath,
SecretLoaders: params.SecretLoaders,
LeftDelims: params.LeftDelims,
RightDelims: params.RightDelims,
AltDelims: params.AltDelims,
DryRun: params.DryRun,
}

// Run the task
if err := t.Run(ctx); err != nil {
log.For(ctx).Fatal("unable to execute task", zap.Error(err))
}
},
}

// Parameters
cmd.Flags().StringVar(&params.InputPath, "in", "", "Template input path (directory or archive)")
log.CheckErr("unable to mark 'in' flag as required.", cmd.MarkFlagRequired("in"))
cmd.Flags().StringVar(&params.OutputPath, "out", "", "Output path")
cmd.Flags().StringVar(&params.RootPath, "root", "", "Defines file loader root base path")
cmd.Flags().StringArrayVarP(&params.SecretLoaders, "secrets-from", "s", []string{"vault"}, "Specifies secret containers to load ('vault' for Vault loader or '-' for stdin or filename)")
cmd.Flags().StringArrayVarP(&params.ValueFiles, "values", "f", []string{}, "Specifies value files to load")
cmd.Flags().StringArrayVar(&params.Values, "set", []string{}, "Specifies value (k=v)")
cmd.Flags().StringArrayVar(&params.StringValues, "set-string", []string{}, "Specifies value (k=string)")
cmd.Flags().StringArrayVar(&params.FileValues, "set-file", []string{}, "Specifies value (k=filepath)")
cmd.Flags().StringVar(&params.LeftDelims, "left-delimiter", "{{", "Template left delimiter (default to '{{')")
cmd.Flags().StringVar(&params.RightDelims, "right-delimiter", "}}", "Template right delimiter (default to '}}')")
cmd.Flags().BoolVar(&params.AltDelims, "alt-delims", false, "Define '[[' and ']]' as template delimiters.")
cmd.Flags().BoolVar(&params.DryRun, "dry-run", false, "Generate in-memory only.")

return cmd
}
1 change: 1 addition & 0 deletions cmd/harp/internal/cmd/root.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@ var mainCmd = func() *cobra.Command {
cmd.AddCommand(csoCmd())

cmd.AddCommand(templateCmd())
cmd.AddCommand(renderCmd())
cmd.AddCommand(valuesCmd())

cmd.AddCommand(fromCmd())
Expand Down
142 changes: 24 additions & 118 deletions cmd/harp/internal/cmd/template.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,22 +18,12 @@
package cmd

import (
"context"
"fmt"
"io"
"os"
"path/filepath"

"github.com/hashicorp/vault/api"
"github.com/spf13/cobra"
"go.uber.org/zap"

"github.com/elastic/harp/pkg/bundle"
"github.com/elastic/harp/pkg/sdk/cmdutil"
"github.com/elastic/harp/pkg/sdk/log"
tplcmdutil "github.com/elastic/harp/pkg/template/cmdutil"
"github.com/elastic/harp/pkg/template/engine"
"github.com/elastic/harp/pkg/vault/kv"
"github.com/elastic/harp/pkg/tasks/template"
)

type templateParams struct {
Expand All @@ -60,7 +50,29 @@ var templateCmd = func() *cobra.Command {
Aliases: []string{"t", "tpl"},
Short: "Read a template and execute it",
Run: func(cmd *cobra.Command, args []string) {
runTemplate(cmd.Context(), params)
// Initialize logger and context
ctx, cancel := cmdutil.Context(cmd.Context(), "template-render", conf.Debug.Enable, conf.Instrumentation.Logs.Level)
defer cancel()

// Prepare task
t := &template.RenderTask{
InputReader: cmdutil.FileReader(params.InputPath),
OutputWriter: cmdutil.FileWriter(params.OutputPath),
ValueFiles: params.ValueFiles,
Values: params.Values,
StringValues: params.StringValues,
FileValues: params.FileValues,
RootPath: params.RootPath,
SecretLoaders: params.SecretLoaders,
LeftDelims: params.LeftDelims,
RightDelims: params.RightDelims,
AltDelims: params.AltDelims,
}

// Run the task
if err := t.Run(ctx); err != nil {
log.For(ctx).Fatal("unable to execute task", zap.Error(err))
}
},
}

Expand All @@ -79,109 +91,3 @@ var templateCmd = func() *cobra.Command {

return cmd
}

//nolint:funlen // to split
func runTemplate(ctx context.Context, params *templateParams) {
ctx, cancel := cmdutil.Context(ctx, "harp-template", conf.Debug.Enable, conf.Instrumentation.Logs.Level)
defer cancel()

var (
reader io.Reader
err error
)

// Create input reader
reader, err = cmdutil.Reader(params.InputPath)
if err != nil {
log.For(ctx).Fatal("unable to open input template", zap.Error(err), zap.String("path", params.InputPath))
}

// Load values
valueOpts := tplcmdutil.ValueOptions{
ValueFiles: params.ValueFiles,
Values: params.Values,
StringValues: params.StringValues,
FileValues: params.FileValues,
}
values, err := valueOpts.MergeValues()
if err != nil {
log.For(ctx).Fatal("unable to process values", zap.Error(err))
}

// Load files
var files engine.Files
if params.RootPath != "" {
absRootPath, errAbs := filepath.Abs(params.RootPath)
if errAbs != nil {
log.For(ctx).Fatal("unable to get absolute template root path", zap.Error(errAbs))
}

files, errAbs = tplcmdutil.Files(os.DirFS(absRootPath), ".")
if errAbs != nil {
log.For(ctx).Fatal("unable to process files", zap.Error(errAbs))
}
}

// Drain reader
body, err := io.ReadAll(reader)
if err != nil {
log.For(ctx).Fatal("unable to drain input template reader", zap.Error(err), zap.String("path", params.InputPath))
}

// If alternative delimiters is used
if params.AltDelims {
params.LeftDelims = "[["
params.RightDelims = "]]"
}

// Process secret readers
secretReaders := []engine.SecretReaderFunc{}
for _, sr := range params.SecretLoaders {
if sr == "vault" {
// Initialize Vault connection
vaultClient, errVault := api.NewClient(api.DefaultConfig())
if errVault != nil {
log.For(ctx).Fatal("unable to initialize vault secret loader", zap.Error(errVault), zap.String("container-path", sr))
}

secretReaders = append(secretReaders, kv.SecretGetter(vaultClient))
continue
}

// Read container
containerReader, errLoader := cmdutil.Reader(sr)
if errLoader != nil {
log.For(ctx).Fatal("unable to read secret container", zap.Error(errLoader), zap.String("container-path", sr))
}

// Load container
b, errBundle := bundle.FromContainerReader(containerReader)
if errBundle != nil {
log.For(ctx).Fatal("unable to decode secret container", zap.Error(errBundle), zap.String("container-path", sr))
}

// Append secret loader
secretReaders = append(secretReaders, bundle.SecretReader(b))
}

// Compile and execute template
out, err := engine.RenderContext(engine.NewContext(
engine.WithName(params.InputPath),
engine.WithDelims(params.LeftDelims, params.RightDelims),
engine.WithValues(values),
engine.WithFiles(files),
engine.WithSecretReaders(secretReaders...),
), string(body))
if err != nil {
log.For(ctx).Fatal("unable to produce output content", zap.Error(err), zap.String("path", params.InputPath))
}

// Create output writer
writer, err := cmdutil.Writer(params.OutputPath)
if err != nil {
log.For(ctx).Fatal("unable to create output writer", zap.Error(err), zap.String("path", params.OutputPath))
}

// Write rendered content
fmt.Fprintf(writer, "%s", out)
}
59 changes: 25 additions & 34 deletions cmd/harp/internal/cmd/values.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,26 +18,26 @@
package cmd

import (
"encoding/json"

"github.com/spf13/cobra"
"go.uber.org/zap"

"github.com/elastic/harp/pkg/sdk/cmdutil"
"github.com/elastic/harp/pkg/sdk/log"
tplcmdutil "github.com/elastic/harp/pkg/template/cmdutil"
"github.com/elastic/harp/pkg/tasks/template"
)

type valuesParams struct {
OutputPath string
ValueFiles []string
Values []string
StringValues []string
FileValues []string
}

// -----------------------------------------------------------------------------

var valuesCmd = func() *cobra.Command {
var (
outputPath string
valueFiles []string
values []string
stringValues []string
fileValues []string
)
params := &valuesParams{}

cmd := &cobra.Command{
Use: "values",
Expand All @@ -48,37 +48,28 @@ var valuesCmd = func() *cobra.Command {
ctx, cancel := cmdutil.Context(cmd.Context(), "harp-values", conf.Debug.Enable, conf.Instrumentation.Logs.Level)
defer cancel()

// Load values
valueOpts := tplcmdutil.ValueOptions{
ValueFiles: valueFiles,
Values: values,
StringValues: stringValues,
FileValues: fileValues,
}
values, err := valueOpts.MergeValues()
if err != nil {
log.For(ctx).Fatal("unable to process values", zap.Error(err))
}

// Create output writer
writer, err := cmdutil.Writer(outputPath)
if err != nil {
log.For(ctx).Fatal("unable to create output writer", zap.Error(err), zap.String("path", outputPath))
// Prepare task
t := &template.ValueTask{
OutputWriter: cmdutil.FileWriter(params.OutputPath),
ValueFiles: params.ValueFiles,
Values: params.Values,
StringValues: params.StringValues,
FileValues: params.FileValues,
}

// Write rendered content
if err := json.NewEncoder(writer).Encode(values); err != nil {
log.For(ctx).Fatal("unable to dump values as JSON", zap.Error(err), zap.String("path", outputPath))
// Run the task
if err := t.Run(ctx); err != nil {
log.For(ctx).Fatal("unable to execute task", zap.Error(err))
}
},
}

// Parameters
cmd.Flags().StringVar(&outputPath, "out", "", "Output file ('-' for stdout or a filename)")
cmd.Flags().StringArrayVarP(&valueFiles, "values", "f", []string{}, "Specifies value files to load. Use <path>:<type>[:<prefix>] to override type detection (json,yaml,xml,hocon,toml)")
cmd.Flags().StringArrayVar(&values, "set", []string{}, "Specifies value (k=v)")
cmd.Flags().StringArrayVar(&stringValues, "set-string", []string{}, "Specifies value (k=string)")
cmd.Flags().StringArrayVar(&fileValues, "set-file", []string{}, "Specifies value (k=filepath)")
cmd.Flags().StringVar(&params.OutputPath, "out", "", "Output file ('-' for stdout or a filename)")
cmd.Flags().StringArrayVarP(&params.ValueFiles, "values", "f", []string{}, "Specifies value files to load. Use <path>:<type>[:<prefix>] to override type detection (json,yaml,xml,hocon,toml)")
cmd.Flags().StringArrayVar(&params.Values, "set", []string{}, "Specifies value (k=v)")
cmd.Flags().StringArrayVar(&params.StringValues, "set-string", []string{}, "Specifies value (k=string)")
cmd.Flags().StringArrayVar(&params.FileValues, "set-file", []string{}, "Specifies value (k=filepath)")

return cmd
}
1 change: 1 addition & 0 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@ require (
github.com/ory/dockertest/v3 v3.8.1
github.com/pelletier/go-toml v1.9.4
github.com/pkg/errors v0.9.1
github.com/psanford/memfs v0.0.0-20210214183328-a001468d78ef
github.com/sethvargo/go-diceware v0.2.1
github.com/sethvargo/go-password v0.2.0
github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966
Expand Down
2 changes: 2 additions & 0 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -902,6 +902,8 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1
github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU=
github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
github.com/psanford/memfs v0.0.0-20210214183328-a001468d78ef h1:NKxTG6GVGbfMXc2mIk+KphcH6hagbVXhcFkbTgYleTI=
github.com/psanford/memfs v0.0.0-20210214183328-a001468d78ef/go.mod h1:tcaRap0jS3eifrEEllL6ZMd9dg8IlDpi2S1oARrQ+NI=
github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 h1:MkV+77GLUNo5oJ0jf870itWm3D0Sjh7+Za9gazKc5LQ=
github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
Expand Down
Loading

0 comments on commit 4574eed

Please sign in to comment.