feat: support layered local variable definitions (#547)

* feat: support layered local variable definitions

`with` can be a list of layers, each layer is merged in order and later layer can
use values from previous layer.

* docs: describe the layers in local context with

docs/workflow.md

@@ -296,7 +296,23 @@ Every workflow receives contextual data from a few sources:
 
 Since the data are received in that particular order listed above, the later source can override data from previous sources. Child workflow context data is independent from parent workflow, anything defined in `with` or inherited will only be in effect during the life cycle of current workflow, except the exported data. Once a field is exported, it will be available to all outer workflows. You can override this by specifying the list of fields that you don't want to export.
 
-Pay attention to the example `retry_func_count_with_exp_backoff` in the previous section. In order to not contaminate parent context with temporary fields, we use `no_export` to block the exporting of certain fields.
+Pay attention to the example `retry_func_count_with_exp_backoff` in the previous section. In order to not contaminate parent context with temporary fields, we use `no_export` to block the exporting of certain fields. For example
+
+The `with` field in the workflow can be a map or a list of maps. If it is a map, each key defines a variable. If it is a list of maps, each map is a layer. The layers are processed in the order they appear. The variables defined in previous layer can be used to define values in later layers.
+```yaml
+---
+call_workflow: something
+with:
+  - var1: initial value
+    var2: val2
+    foo:
+      - bar
+  - var3: '{{ .ctx.var2 }}, {{ .ctx.var1 }}'
+    foo+:
+      - another bar
+```
+
+The final value for `var3` will be `initial value, val2`, and the final value of list `foo` will contain both `bar` and `another bar`.
 
 ### Interpolation
 We can use interpolation in workflows to make the workflow flexible and versatile. You can use interpolation in most of the fields of a workflow. Besides contextual data, other data available for interpolation includes:
@@ -368,6 +384,8 @@ Usage:
  * `var+`: if the `var` is a list or string, the new value will be appended to the existing values
  * `var*`: forcefully override the value
 
+Note that, the merging modifier works in layers too. See previous example for details.
+
 ## Essential Workflows
 
 We have made a few helper workflows available in the `honeydipper-config-essentials` repo. Hopefully, they will make it easier for you to write your own workflows.

internal/config/schema.go

@@ -1,4 +1,4 @@
-// Copyright 2022 PayPal Inc.
+// Copyright 2023 PayPal Inc.
 
 // This Source Code Form is subject to the terms of the MIT License.
 // If a copy of the MIT License was not distributed with this file,
@@ -62,7 +62,7 @@ type Workflow struct {
 	Meta        interface{}
 	Context     string
 	Contexts    interface{}
-	Local       map[string]interface{} `json:"with" mapstructure:"with"`
+	Local       interface{} `json:"with" mapstructure:"with"`
 
 	Match       interface{} `json:"if_match" mapstructure:"if_match"`
 	UnlessMatch interface{} `json:"unless_match" mapstructure:"unelss_match"`

internal/workflow/session.go

@@ -1,4 +1,4 @@
-// Copyright 2022 PayPal Inc.
+// Copyright 2023 PayPal Inc.
 
 // This Source Code Form is subject to the terms of the MIT License.
 // If a copy of the MIT License was not distributed with this file,
@@ -270,10 +270,16 @@ func (w *Session) injectEventCTX(ctx map[string]interface{}) {
 // injectLocalCTX injects the workflow local context data.
 func (w *Session) injectLocalCTX(msg *dipper.Message) {
 	if w.workflow.Local != nil && w.workflow.CallDriver == "" {
+		layers, ok := w.workflow.Local.([]interface{})
+		if !ok {
+			layers = []interface{}{w.workflow.Local}
+		}
 		envData := w.buildEnvData(msg)
-		locals := dipper.Interpolate(w.workflow.Local, envData)
-
-		w.ctx = dipper.MergeMap(w.ctx, locals)
+		for _, l := range layers {
+			locals := dipper.Interpolate(l, envData)
+			envData["ctx"] = dipper.MergeMap(envData["ctx"].(map[string]interface{}), locals)
+		}
+		w.ctx = envData["ctx"].(map[string]interface{})
 	}
 }
 

internal/workflow/session_layered_test.go

@@ -0,0 +1,52 @@
+// Copyright 2023 PayPal Inc.
+
+// This Source Code Form is subject to the terms of the MIT License.
+// If a copy of the MIT License was not distributed with this file,
+// you can obtain one at https://mit-license.org/.
+
+//go:build !integration
+// +build !integration
+
+package workflow
+
+import (
+	"testing"
+
+	"github.com/ghodss/yaml"
+	"github.com/golang/mock/gomock"
+	"github.com/honeydipper/honeydipper/internal/config"
+	"github.com/honeydipper/honeydipper/internal/workflow/mock_workflow"
+	"github.com/honeydipper/honeydipper/pkg/dipper"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestSessionLayered(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	mockHelper := mock_workflow.NewMockSessionStoreHelper(ctrl)
+	s := NewSessionStore(mockHelper)
+
+	// tear down
+	defer delete(dipper.IDMapMetadata, &s.sessions)
+
+	testDataSet := &config.DataSet{}
+	testConfig := &config.Config{DataSet: testDataSet}
+	mockHelper.EXPECT().GetConfig().AnyTimes().Return(testConfig)
+
+	wf1 := &config.Workflow{}
+	err := yaml.Unmarshal([]byte(`
+with:
+  - foo: bar
+  - foo+: " and "
+    var1: $ctx.foo
+  - foo+: bar2
+  - var2: $ctx.foo
+    `), wf1)
+	assert.Nil(t, err, "test config")
+	w := s.newSession("", "uuid1", wf1).(*Session)
+	w.prepare(&dipper.Message{}, nil, map[string]interface{}{})
+	assert.Equal(t, "bar and bar2", w.ctx["foo"], "inheriting from previous layer")
+	assert.Equal(t, "bar", w.ctx["var1"], "using value from previous layer")
+	assert.Equal(t, "bar and bar2", w.ctx["var2"], "using value from previous multiple layers")
+}