Merge d3b00da into c4c2ac0

README.md

@@ -133,6 +133,7 @@ All are optional and have a default value:
 - `init-path`: the directory from where initialization plugins (see "Developing plugins") are loaded in *.so form (default: `./init`)
 - `plugins-path`: the directory from where action plugins (see "Developing plugins") are loaded in *.so form (default: `./plugins`)
 - `templates-path`: the directory where yaml-formatted task templates are loaded from (default: `./templates`)
+- `functions-path`: the directory where yaml-formatted functions templates are loaded from (default: `./functions`)
 - `region`: an arbitrary identifier, to aggregate a running group of µTask instances (commonly containers), and differentiate them from another group, in a separate region (default: `default`)
 - `http-port`: the port on which the HTTP API listents (default: `8081`)
 - `debug`: a boolean flag to activate verbose logs (default: `false`)
@@ -175,6 +176,9 @@ A user can be allowed to resolve a task in three ways:
 - `.step.[STEP_NAME].state`: current state of the given step
 - `.config.[CONFIG_ITEM].bar`: field `bar` from a config item (configstore, see above)
 - `.iterator.foo`: field `foo` from the iterator in a loop (see `foreach` steps below)
+- `.pre_hook.output.foo`: field `foo` from the output of the step's preHook (see `preHooks` below)
+- `.pre_hook.metadata.HTTPStatus`: field `HTTPStatus` from the metadata of the step's preHook (see `preHooks` below)
+- `.function_args.[ARG_NAME]`: argument that needs to be given in the conifguration section to the function (see `functions` below)
 
 The following templating functions are available:
 
@@ -321,6 +325,8 @@ Note that the operators `IN` and `NOTIN` expect a list of acceptable values in t
 
 - `name`: a unique identifier
 - `description`: a human readable sentence to convey the step's intent
+- `action`: the actual task the step executes
+- `pre_hook`: an action that can be executed before the actual action of the step
 - `dependencies`: a list of step names on which this step waits before running
 - `custom_states`: a list of personnalised allowed state for this step (can be assigned to the state's step using `conditions`)
 - `retry_pattern`: (`seconds`, `minutes`, `hours`) define on what temporal order of magnitude the re-runs of this step should be spread (default = `seconds`)
@@ -434,6 +440,67 @@ Browse [builtin actions](./pkg/plugins/builtin)
 | **`ping`**    | Send a ping to an hostname *Warn: This plugin will keep running until the count is done*                                                                                                                                                          | [Access plugin doc](./pkg/plugins/builtin/ping/README.md)    |
 | **`script`**  | Execute a script under `scripts` folder                                                                                                                                                                                                           | [Access plugin doc](./pkg/plugins/builtin/script/README.md)  |
 
+#### PreHooks <a name="preHooks"></a>
+
+The `pre_hook` field of a step can be set to define an action that is executed before the step's action. This fields supports all the sames fields as the action. It aims to fetch data for the execution of the action that can change over time and needs to be fetched at every retry, such as OTPs. All the result values of the preHook are available under the templating variable `.pre_hook`
+
+```yaml
+doSomeAuthPost:
+  pre_hook:
+    type: http
+    method: "GET"
+    url: "https://myAwesomeApi/otp"
+  action:
+    type: http
+    method: "POST"
+    url: "https://myAwesomeApi/doSomePost"
+    headers:
+      X-Otp: "{{ .pre_hook.output }}"
+```
+
+#### Functions <a name="functions"></a>
+
+Functions are abstraction of the actions to define a behavior that can be re-used in templates. They act like a plugin but are fully declared in dedicated directory `functions`. They can have arguments that need to be given in the `configuration` section of the action and can be used in the declaration of the function by accessing the templating variables under `.function_args`.
+
+```yaml
+name: ovh::request
+description: Execute a call to the ovh API
+pre_hook:
+  type: http
+  configuration:
+    method: "GET"
+    url: https://api.ovh.com/1.0/auth/time
+action:
+  type: http
+  configuration:
+    headers:
+    - name: X-Ovh-Signature
+      value: '{{ printf "%s+%s+%s+%s%s+%s+%v" .config.apiovh.applicationSecret .config.apiovh.consumerKey .function_args.method .config.apiovh.basePath .function_args.path .function_args.body .pre_hook.output | sha1sum | printf "$1$%s"}}'
+    - name: X-Ovh-Timestamp
+      value: "{{ .pre_hook.output }}"
+    - name: X-Ovh-Consumer
+      value: "{{ .config.apiovh.consumerKey }}"
+    - name: X-Ovh-Application
+      value: "{{ .config.apiovh.applicationKey }}"
+    method: "{{ .function_args.method }}"
+    url: "{{.config.apiovh.basePath}}{{ .function_args.path }}"
+    body: "{{ .function_args.body }}"
+```
+
+This function can be used in a template like this:
+
+```yaml
+steps:
+  getService:
+    description: Get Service
+    action:
+      type: ovh::request
+      configuration:
+        path: "{{.input.path}}"
+        method: GET
+        body: ""
+```
+
 #### Dependencies <a name="dependencies"></a>
 
 Dependencies can be declared on a step, to indicate what requirements should be met before the step can actually run. A step can have multiple dependencies, which will all have to be met before the step can start running.

api/handler/common.go

@@ -45,6 +45,13 @@ func buildTemplateNextLink(pageSize uint64, last string) string {
 	return buildLink("next", "/template", values.Encode())
 }
 
+func buildFunctionNextLink(pageSize uint64, last string) string {
+	values := &url.Values{}
+	values.Add("page_size", strconv.FormatUint(pageSize, 10))
+	values.Add("last", last)
+	return buildLink("next", "/function", values.Encode())
+}
+
 func buildTaskNextLink(typ string, state, batch *string, pageSize uint64, last string) string {
 	values := &url.Values{}
 	values.Add("type", typ)

api/handler/function.go

@@ -0,0 +1,66 @@
+package handler
+
+import (
+	"fmt"
+
+	"github.com/gin-gonic/gin"
+	"github.com/juju/errors"
+	"github.com/ovh/utask/engine/functions"
+)
+
+type listFunctionsIn struct {
+	PageSize uint64  `query:"page_size"`
+	Last     *string `query:"last"`
+}
+
+// ListFunctions returns a list of available functions in simplified format (steps not included)
+func ListFunctions(c *gin.Context, in *listFunctionsIn) ([]*functions.Function, error) {
+	var ret = []*functions.Function{}
+
+	in.PageSize = normalizePageSize(in.PageSize)
+
+	list := functions.List()
+
+	// Get the offset on which get the functions
+	from := 0
+	if in.Last != nil && *in.Last != "" {
+		var value string
+		for from, value = range list {
+			if value == *in.Last {
+				from++
+				break
+			}
+		}
+	}
+
+	for i := from; i < len(list) && i < from+int(in.PageSize); i++ {
+		f, _ := functions.Get(list[i])
+		ret = append(ret, f)
+	}
+
+	if uint64(len(ret)) == in.PageSize {
+		last := ret[len(ret)-1].Name
+		c.Header(
+			linkHeader,
+			buildFunctionNextLink(in.PageSize, last),
+		)
+	}
+
+	c.Header(pageSizeHeader, fmt.Sprintf("%v", in.PageSize))
+
+	return ret, nil
+}
+
+type getFunctionIn struct {
+	Name string `path:"name, required"`
+}
+
+// GetFunction returns the full representation of a function, steps included
+func GetFunction(c *gin.Context, in *getFunctionIn) (*functions.Function, error) {
+	function, exists := functions.Get(in.Name)
+	if !exists {
+		return nil, errors.NewNotFound(nil, "function not found")
+	}
+	return function, nil
+
+}

api/server.go

@@ -201,6 +201,21 @@ func (s *Server) build(ctx context.Context) {
 					tonic.Handler(handler.GetTemplate, 200))
 			}
 
+			functionRoutes := authRoutes.Group("/", "05 - function", "Manage uTask task functions")
+			{
+				// public function listing
+				functionRoutes.GET("/function",
+					[]fizz.OperationOption{
+						fizz.Summary("List task functions"),
+					},
+					tonic.Handler(handler.ListFunctions, 200))
+				functionRoutes.GET("/function/:name",
+					[]fizz.OperationOption{
+						fizz.Summary("Get task function details"),
+					},
+					tonic.Handler(handler.GetFunction, 200))
+			}
+
 			// task
 			taskRoutes := authRoutes.Group("/", "01 - task", "Manage uTask tasks")
 			{

engine/engine_test.go

@@ -226,6 +226,17 @@ func TestFunctionCustomState(t *testing.T) {
 	assert.Equal(t, []string{"STATE_HELLO"}, customStates)
 }
 
+func TestFunctionPreHook(t *testing.T) {
+	input := map[string]interface{}{}
+	res, err := runTask("functionPreHook.yaml", input, nil)
+
+	assert.Equal(t, nil, err)
+	assert.Equal(t, res.Steps["stepOne"].Output, map[string]interface{}{
+		"value":    "Hello 42 !",
+		"coalesce": "Coalesce 42!",
+	})
+}
+
 func TestClientError(t *testing.T) {
 	res, err := runTask("clientError.yaml", map[string]interface{}{}, nil)
 
@@ -302,6 +313,7 @@ func TestLintingAndValidation(t *testing.T) {
 		"allowedStateImpact.yaml":     {false, true},
 		"functionEchoHelloWorld.yaml": {false, true},
 		"functionCustomState.yaml":    {false, true},
+		"functionPreHook.yaml":        {false, true},
 	}
 
 	for template, testCase := range expectedResult {

engine/functions/functions.go

@@ -8,6 +8,7 @@ import (
 	"path"
 	"reflect"
 	"regexp"
+	"sort"
 	"strings"
 
 	"github.com/ghodss/yaml"
@@ -31,6 +32,7 @@ var (
 type Function struct {
 	Name         string                 `json:"name"`
 	Action       executor.Executor      `json:"action"`
+	PreHook      *executor.Executor     `json:"pre_hook,omitempty"`
 	Conditions   []*condition.Condition `json:"conditions,omitempty"`
 	CustomStates []string               `json:"custom_states,omitempty"`
 
@@ -197,6 +199,7 @@ func List() []string {
 	for k := range functionsImported {
 		result = append(result, k)
 	}
+	sort.Strings(result)
 	return result
 }
 

engine/functions_tests/echo-prehook.yaml

@@ -0,0 +1,12 @@
+name: echo::preHook
+pre_hook:
+  type: echo
+  configuration:
+    output:
+      value: '42'
+action:
+  type: echo
+  configuration:
+    output:
+      value: 'Hello {{ .pre_hook.output.value }} !'
+      coalesce: 'Coalesce {{ coalesce .pre_hook.output.notExist .pre_hook.output.value }}!'

engine/step/apply.go

@@ -34,6 +34,14 @@ func rawResolveObject(val *values.Values, objson json.RawMessage, item interface
 	return obj, nil
 }
 
+func rawResolve(val *values.Values, obj interface{}, item interface{}, stepName string) (interface{}, error) {
+	v := reflect.ValueOf(obj)
+	if err := apply(val, v, item, stepName); err != nil {
+		return nil, err
+	}
+	return obj, nil
+}
+
 func apply(val *values.Values, v reflect.Value, item interface{}, stepName string) error {
 	for v.Kind() == reflect.Ptr || v.Kind() == reflect.Interface {
 		v = v.Elem()