diff --git a/api/http/workflow.go b/api/http/workflow.go index 2c93287e..e954b2e0 100644 --- a/api/http/workflow.go +++ b/api/http/workflow.go @@ -1,4 +1,4 @@ -// Copyright 2021 Northern.tech AS +// Copyright 2022 Northern.tech AS // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -28,10 +28,10 @@ import ( "github.com/mendersoftware/go-lib-micro/log" - "github.com/mendersoftware/workflows/app/worker" "github.com/mendersoftware/workflows/client/nats" "github.com/mendersoftware/workflows/model" "github.com/mendersoftware/workflows/store" + "github.com/mendersoftware/workflows/utils" ) const ( @@ -136,7 +136,7 @@ func (h WorkflowController) startWorkflowGetJob( if ok { values := make([]string, 0, 10) for _, value := range valueSlice { - valueString, err := worker.ConvertAnythingToString(value) + valueString, err := utils.ConvertAnythingToString(value) if err == nil { values = append(values, valueString) } @@ -147,7 +147,7 @@ func (h WorkflowController) startWorkflowGetJob( Raw: value, }) } else { - valueString, err := worker.ConvertAnythingToString(value) + valueString, err := utils.ConvertAnythingToString(value) if err == nil { jobInputParameters = append(jobInputParameters, model.InputParameter{ Name: key, diff --git a/app/processor/job.go b/app/processor/job.go new file mode 100644 index 00000000..828c5de0 --- /dev/null +++ b/app/processor/job.go @@ -0,0 +1,85 @@ +// Copyright 2022 Northern.tech AS +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package processor + +import ( + "strings" + + "go.mongodb.org/mongo-driver/bson/primitive" + + "github.com/mendersoftware/workflows/model" +) + +type JobProcessor struct { + job *model.Job +} + +type JsonOptions struct { +} + +func NewJobProcessor(job *model.Job) *JobProcessor { + return &JobProcessor{ + job: job, + } +} + +func (j JobProcessor) ProcessJSON( + data interface{}, + ps *JobStringProcessor, + options ...*JsonOptions, +) interface{} { + switch value := data.(type) { + case []interface{}: + result := make([]interface{}, len(value)) + for i, item := range value { + result[i] = j.ProcessJSON(item, ps) + } + return result + case map[string]interface{}: + result := make(map[string]interface{}) + for key, item := range value { + result[key] = j.ProcessJSON(item, ps) + } + return result + case string: + if len(value) > 3 && value[0:2] == "${" && value[len(value)-1:] == "}" { + key := value[2 : len(value)-1] + if strings.HasPrefix(key, workflowInputVariable) && + len(key) > len(workflowInputVariable) { + key = key[len(workflowInputVariable):] + for _, param := range j.job.InputParameters { + if param.Name == key && param.Raw != nil { + return j.ProcessJSON(param.Raw, ps) + } + } + return nil + } + } + return ps.ProcessJobString(value) + case primitive.D: + result := make(map[string]interface{}) + for key, item := range value.Map() { + result[key] = j.ProcessJSON(item, ps) + } + return result + case []primitive.D: + result := make([]interface{}, len(value)) + for i, item := range value { + result[i] = j.ProcessJSON(item, ps) + } + return result + } + return data +} diff --git a/app/processor/string.go b/app/processor/string.go new file mode 100644 index 00000000..e4579ade --- /dev/null +++ b/app/processor/string.go @@ -0,0 +1,181 @@ +// Copyright 2022 Northern.tech AS +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package processor + +import ( + "bytes" + "net/url" + "os" + "regexp" + "strings" + "text/template" + + "github.com/thedevsaddam/gojsonq" + + "github.com/mendersoftware/workflows/model" + "github.com/mendersoftware/workflows/utils" +) + +const ( + workflowEnvVariable = "env." + workflowInputVariable = "workflow.input." + regexVariable = `\$\{(?P(?:(?:[a-zA-Z]+)=(?:[a-zA-Z0-9]+);)*)` + + `(?P[^;\}\|]+)(?:\|(?P[^\}]+))?}` + regexOutputVariable = `(.*)\.json\.(.*)` + encodingFlag = "encoding" + urlEncodingFlag = "url" +) + +var ( + reExpression = regexp.MustCompile(regexVariable) + reExpressionOutput = regexp.MustCompile(regexOutputVariable) + + reMatchIndexOptions = reExpression.SubexpIndex("options") + reMatchIndexName = reExpression.SubexpIndex("name") + reMatchIndexDefault = reExpression.SubexpIndex("default") +) + +type Encoding int64 + +const ( + EncodingPlain Encoding = iota + EncodingURL +) + +type JobStringProcessor struct { + workflow *model.Workflow + job *model.Job +} + +type Options struct { + Encoding Encoding +} + +func NewJobStringProcessor( + workflow *model.Workflow, + job *model.Job, +) *JobStringProcessor { + return &JobStringProcessor{ + workflow: workflow, + job: job, + } +} + +func processOptionString(expression string) (opts Options) { + const ( + flagTokenCount = 2 + lValueIndex = 0 + rValueIndex = 1 + ) + for _, flagToken := range strings.Split(expression, ";") { + flagValueTokens := strings.Split(flagToken, "=") + if len(flagValueTokens) < flagTokenCount { + continue + } + if flagValueTokens[lValueIndex] == encodingFlag { + switch flagValueTokens[rValueIndex] { + case urlEncodingFlag: + opts.Encoding = EncodingURL + } + } + } + return +} + +func (j *JobStringProcessor) ProcessJobString(data string) string { + matches := reExpression.FindAllStringSubmatch(data, -1) + + // search for ${...} expressions in the data string +SubMatchLoop: + for _, submatch := range matches { + // content of the ${...} expression, without the brackets + varName := submatch[reMatchIndexName] + value := submatch[reMatchIndexDefault] + options := processOptionString(submatch[reMatchIndexOptions]) + // now it is possible to override the encoding with flags: ${encoding=plain;identifier} + // if encoding is supplied via flags, it takes precedence, we return the match + // without the flags, otherwise fail back to original match and encoding + if strings.HasPrefix(varName, workflowInputVariable) && + len(varName) > len(workflowInputVariable) { + // Replace ${workflow.input.KEY} with the KEY input variable + paramName := varName[len(workflowInputVariable):] + for _, param := range j.job.InputParameters { + if param.Name == paramName { + value = param.Value + break + } + } + } else if strings.HasPrefix(varName, workflowEnvVariable) && + len(varName) > len(workflowEnvVariable) { + // Replace ${env.KEY} with the KEY environment variable + envName := varName[len(workflowEnvVariable):] + if envValue := os.Getenv(envName); envValue != "" { + value = envValue + } + } else if output := reExpressionOutput.FindStringSubmatch(varName); len(output) > 0 { + // Replace ${TASK_NAME.json.JSONPATH} with the value of the JSONPATH expression from the + // JSON output of the previous task with name TASK_NAME. If the output is not a valid + // JSON or the JSONPATH does not resolve to a value, replace with empty string + for _, result := range j.job.Results { + if result.Name == output[1] { + varKey := output[2] + var output string + if result.Type == model.TaskTypeHTTP { + output = result.HTTPResponse.Body + } else if result.Type == model.TaskTypeCLI { + output = result.CLI.Output + } else { + continue + } + varValue := gojsonq.New().FromString(output).Find(varKey) + if varValue == nil { + varValue = "" + } + varValueString, err := utils.ConvertAnythingToString(varValue) + if err == nil { + if varValueString != "" { + value = varValueString + } + } else { + continue SubMatchLoop + } + break + } + } + } + if options.Encoding == EncodingURL { + value = url.QueryEscape(value) + } + data = strings.ReplaceAll(data, submatch[0], value) + } + + return data +} + +// MaybeExecuteGoTemplate tries to parse and execute data as a go template +// if it fails to do so, data is returned. +func (j *JobStringProcessor) MaybeExecuteGoTemplate(data string) string { + input := j.job.InputParameters.Map() + tmpl, err := template.New("go-template").Parse(data) + if err != nil { + return data + } + buf := &bytes.Buffer{} + err = tmpl.Execute(buf, input) + if err != nil { + return data + } + return buf.String() +} diff --git a/app/processor/string_test.go b/app/processor/string_test.go new file mode 100644 index 00000000..113bfb6e --- /dev/null +++ b/app/processor/string_test.go @@ -0,0 +1,74 @@ +// Copyright 2022 Northern.tech AS +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package processor + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestProcessOptionString(t *testing.T) { + var data string + var setEncoding string + var expectedEncoding Encoding + var o Options + + setEncoding = urlEncodingFlag + expectedEncoding = EncodingURL + data = "encoding=" + setEncoding + ";" + o = processOptionString(data) + assert.Equal(t, Options{Encoding: expectedEncoding}, o) + + setEncoding = urlEncodingFlag + expectedEncoding = EncodingURL + data = "rightFlag=right;encoding=" + setEncoding + ";leftFlag=left;" + o = processOptionString(data) + assert.Equal(t, Options{Encoding: expectedEncoding}, o) + + setEncoding = "plain" + expectedEncoding = EncodingPlain + data = "encoding=" + setEncoding + ";" + o = processOptionString(data) + assert.Equal(t, Options{Encoding: expectedEncoding}, o) + + setEncoding = "plain" + expectedEncoding = EncodingPlain + data = "rightFlag=right;encoding=" + setEncoding + ";leftFlag=left;" + o = processOptionString(data) + assert.Equal(t, Options{Encoding: expectedEncoding}, o) +} + +func TestVariableParse(t *testing.T) { + var data string + var setEncoding string + var expectedVariableIdentifier string + var expectedEncoding Encoding + + expectedEncoding = EncodingURL + setEncoding = urlEncodingFlag + expectedVariableIdentifier = "newVariable0" + data = "${encoding=" + setEncoding + ";" + workflowInputVariable + expectedVariableIdentifier + "}" + matches := reExpression.FindAllStringSubmatch(data, -1) + + for _, submatch := range matches { + varName := submatch[reMatchIndexName] + value := submatch[reMatchIndexDefault] + options := processOptionString(submatch[reMatchIndexOptions]) + assert.Equal(t, varName, workflowInputVariable+expectedVariableIdentifier) + assert.Equal(t, value, "") + assert.Equal(t, Options{Encoding: expectedEncoding}, options) + } +} diff --git a/app/worker/cli.go b/app/worker/cli.go index fe7f4c32..b4de367f 100644 --- a/app/worker/cli.go +++ b/app/worker/cli.go @@ -1,4 +1,4 @@ -// Copyright 2020 Northern.tech AS +// Copyright 2022 Northern.tech AS // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -20,6 +20,7 @@ import ( "os/exec" "time" + "github.com/mendersoftware/workflows/app/processor" "github.com/mendersoftware/workflows/model" ) @@ -28,18 +29,21 @@ const ( MaxExecutionTime int = 3600 * 4 ) -func processCLITask(cliTask *model.CLITask, job *model.Job, - workflow *model.Workflow) (*model.TaskResult, error) { +func processCLITask( + cliTask *model.CLITask, + ps *processor.JobStringProcessor, + jp *processor.JobProcessor, +) (*model.TaskResult, error) { commands := make([]string, 0, 10) for _, command := range cliTask.Command { - command := processJobString(command, workflow, job) + command := ps.ProcessJobString(command) commands = append(commands, command) } ctx := context.Background() timeout := cliTask.ExecutionTimeOut if timeout <= 0 { - timeout = 3600 * 4 + timeout = MaxExecutionTime } ctxWithOptionalTimeOut, cancel := context.WithTimeout(ctx, time.Duration(timeout)*time.Second) defer cancel() diff --git a/app/worker/http.go b/app/worker/http.go index 15ddeff7..87793cc3 100644 --- a/app/worker/http.go +++ b/app/worker/http.go @@ -1,4 +1,4 @@ -// Copyright 2021 Northern.tech AS +// Copyright 2022 Northern.tech AS // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -25,6 +25,7 @@ import ( "github.com/mendersoftware/go-lib-micro/log" + "github.com/mendersoftware/workflows/app/processor" "github.com/mendersoftware/workflows/model" ) @@ -39,34 +40,39 @@ var makeHTTPRequest = func(req *http.Request, timeout time.Duration) (*http.Resp return res, nil } -func processHTTPTask(httpTask *model.HTTPTask, job *model.Job, - workflow *model.Workflow, l *log.Logger) (*model.TaskResult, error) { - uri := processJobString(httpTask.URI, workflow, job) +func processHTTPTask( + httpTask *model.HTTPTask, + ps *processor.JobStringProcessor, + jp *processor.JobProcessor, + l *log.Logger, +) (*model.TaskResult, error) { + uri := ps.ProcessJobString(httpTask.URI) + l.Infof("processHTTPTask: starting with: method=%s uri=%s", + httpTask.Method, + uri, + ) var payloadString string if len(httpTask.FormData) > 0 { form := url.Values{} for key, value := range httpTask.FormData { - key = processJobString(key, workflow, job) - key = maybeExecuteGoTemplate(key, job.InputParameters.Map()) - value = processJobString(value, workflow, job) - value = maybeExecuteGoTemplate(value, job.InputParameters.Map()) + key = ps.ProcessJobString(key) + key = ps.MaybeExecuteGoTemplate(key) + value = ps.ProcessJobString(value) + value = ps.MaybeExecuteGoTemplate(value) form.Add(key, value) } payloadString = form.Encode() } else if httpTask.JSON != nil { - payloadJSON := processJobJSON(httpTask.JSON, workflow, job) + payloadJSON := jp.ProcessJSON(httpTask.JSON, ps) payloadBytes, err := json.Marshal(payloadJSON) if err != nil { return nil, err } payloadString = string(payloadBytes) } else { - payloadString = processJobString(httpTask.Body, workflow, job) - payloadString = maybeExecuteGoTemplate( - payloadString, - job.InputParameters.Map(), - ) + payloadString = ps.ProcessJobString(httpTask.Body) + payloadString = ps.MaybeExecuteGoTemplate(payloadString) } payload := strings.NewReader(payloadString) @@ -81,7 +87,7 @@ func processHTTPTask(httpTask *model.HTTPTask, job *model.Job, var headersToBeSent []string for name, value := range httpTask.Headers { - headerValue := processJobString(value, workflow, job) + headerValue := ps.ProcessJobString(value) req.Header.Add(name, headerValue) headersToBeSent = append(headersToBeSent, fmt.Sprintf("%s: %s", name, headerValue)) diff --git a/app/worker/nats.go b/app/worker/nats.go index 603f7272..55ac9177 100644 --- a/app/worker/nats.go +++ b/app/worker/nats.go @@ -1,4 +1,4 @@ -// Copyright 2021 Northern.tech AS +// Copyright 2022 Northern.tech AS // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -17,18 +17,23 @@ package worker import ( "encoding/json" + "github.com/mendersoftware/workflows/app/processor" "github.com/mendersoftware/workflows/client/nats" "github.com/mendersoftware/workflows/model" ) -func processNATSTask(natsTask *model.NATSTask, job *model.Job, - workflow *model.Workflow, nats nats.Client) (*model.TaskResult, error) { +func processNATSTask( + natsTask *model.NATSTask, + ps *processor.JobStringProcessor, + jp *processor.JobProcessor, + nats nats.Client, +) (*model.TaskResult, error) { var result *model.TaskResult = &model.TaskResult{ NATS: &model.TaskResultNATS{}, } - dataJSON := processJobJSON(natsTask.Data, workflow, job) + dataJSON := jp.ProcessJSON(natsTask.Data, ps) dataJSONBytes, err := json.Marshal(dataJSON) if err == nil { subject := nats.StreamName() + "." + natsTask.Subject diff --git a/app/worker/process.go b/app/worker/process.go index 42625457..af7fd45d 100644 --- a/app/worker/process.go +++ b/app/worker/process.go @@ -1,4 +1,4 @@ -// Copyright 2021 Northern.tech AS +// Copyright 2022 Northern.tech AS // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -24,6 +24,7 @@ import ( "github.com/mendersoftware/go-lib-micro/log" + "github.com/mendersoftware/workflows/app/processor" "github.com/mendersoftware/workflows/client/nats" "github.com/mendersoftware/workflows/model" "github.com/mendersoftware/workflows/store" @@ -118,10 +119,12 @@ func processTask(task model.Task, job *model.Job, var result *model.TaskResult var err error + ps := processor.NewJobStringProcessor(workflow, job) + jp := processor.NewJobProcessor(job) if len(task.Requires) > 0 { for _, require := range task.Requires { - require = processJobString(require, workflow, job) - require = maybeExecuteGoTemplate(require, job.InputParameters.Map()) + require = ps.ProcessJobString(require) + require = ps.MaybeExecuteGoTemplate(require) if require == "" { result := &model.TaskResult{ Name: task.Name, @@ -142,9 +145,7 @@ func processTask(task model.Task, job *model.Job, "Error: Task definition incompatible " + "with specified type (http)") } - l.Infof("processTask: calling http task: %s %s", httpTask.Method, - processJobString(httpTask.URI, workflow, job)) - result, err = processHTTPTask(httpTask, job, workflow, l) + result, err = processHTTPTask(httpTask, ps, jp, l) case model.TaskTypeCLI: var cliTask *model.CLITask = task.CLI if cliTask == nil { @@ -152,7 +153,7 @@ func processTask(task model.Task, job *model.Job, "Error: Task definition incompatible " + "with specified type (cli)") } - result, err = processCLITask(cliTask, job, workflow) + result, err = processCLITask(cliTask, ps, jp) case model.TaskTypeNATS: var natsTask *model.NATSTask = task.NATS if natsTask == nil { @@ -160,7 +161,7 @@ func processTask(task model.Task, job *model.Job, "Error: Task definition incompatible " + "with specified type (nats)") } - result, err = processNATSTask(natsTask, job, workflow, nats) + result, err = processNATSTask(natsTask, ps, jp, nats) case model.TaskTypeSMTP: var smtpTask *model.SMTPTask = task.SMTP if smtpTask == nil { @@ -169,11 +170,11 @@ func processTask(task model.Task, job *model.Job, "with specified type (smtp)") } l.Infof("processTask: calling smtp task: From: %s To: %s Subject: %s", - processJobString(smtpTask.From, workflow, job), - processJobString(strings.Join(smtpTask.To, ","), workflow, job), - processJobString(smtpTask.Subject, workflow, job), + ps.ProcessJobString(smtpTask.From), + ps.ProcessJobString(strings.Join(smtpTask.To, ",")), + ps.ProcessJobString(smtpTask.Subject), ) - result, err = processSMTPTask(smtpTask, job, workflow, l) + result, err = processSMTPTask(smtpTask, ps, jp, l) default: result = nil err = fmt.Errorf("Unrecognized task type: %s", task.Type) diff --git a/app/worker/smtp.go b/app/worker/smtp.go index 63f0bba7..22d9f67d 100644 --- a/app/worker/smtp.go +++ b/app/worker/smtp.go @@ -1,4 +1,4 @@ -// Copyright 2021 Northern.tech AS +// Copyright 2022 Northern.tech AS // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -26,14 +26,19 @@ import ( "github.com/mendersoftware/go-lib-micro/config" "github.com/mendersoftware/go-lib-micro/log" + "github.com/mendersoftware/workflows/app/processor" dconfig "github.com/mendersoftware/workflows/config" "github.com/mendersoftware/workflows/model" ) var smtpClient SMTPClientInterface = new(SMTPClient) -func processSMTPTask(smtpTask *model.SMTPTask, job *model.Job, - workflow *model.Workflow, l *log.Logger) (*model.TaskResult, error) { +func processSMTPTask( + smtpTask *model.SMTPTask, + ps *processor.JobStringProcessor, + jp *processor.JobProcessor, + l *log.Logger, +) (*model.TaskResult, error) { var result *model.TaskResult = &model.TaskResult{ SMTP: &model.TaskResultSMTP{}, } @@ -43,31 +48,31 @@ func processSMTPTask(smtpTask *model.SMTPTask, job *model.Job, to := make([]string, 0, 10) for _, address := range smtpTask.To { - addresses := strings.Split(processJobString(address, workflow, job), ",") + addresses := strings.Split(ps.ProcessJobString(address), ",") recipients = append(recipients, addresses...) to = append(to, addresses...) } cc := make([]string, 0, 10) for _, address := range smtpTask.Cc { - addresses := strings.Split(processJobString(address, workflow, job), ",") + addresses := strings.Split(ps.ProcessJobString(address), ",") recipients = append(recipients, addresses...) cc = append(cc, addresses...) } bcc := make([]string, 0, 10) for _, address := range smtpTask.Bcc { - addresses := strings.Split(processJobString(address, workflow, job), ",") + addresses := strings.Split(ps.ProcessJobString(address), ",") recipients = append(recipients, addresses...) bcc = append(bcc, addresses...) } - from := processJobString(smtpTask.From, workflow, job) - subject := processJobString(smtpTask.Subject, workflow, job) + from := ps.ProcessJobString(smtpTask.From) + subject := ps.ProcessJobString(smtpTask.Subject) var err error var body, HTML string - if body, err = processJobStringOrFile(smtpTask.Body, workflow, job); err != nil { + if body, err = processJobStringOrFile(smtpTask.Body, ps); err != nil { result.Success = false result.SMTP.Error = err.Error() l.Infof("processSMTPTask error reading file: '%s'", err.Error()) @@ -75,7 +80,7 @@ func processSMTPTask(smtpTask *model.SMTPTask, job *model.Job, } l.Debugf("processSMTPTask body text: '\n%s\n'", body) - if HTML, err = processJobStringOrFile(smtpTask.HTML, workflow, job); err != nil { + if HTML, err = processJobStringOrFile(smtpTask.HTML, ps); err != nil { result.Success = false result.SMTP.Error = err.Error() l.Infof("processSMTPTask error reading file: '%s'", err.Error()) diff --git a/app/worker/utils.go b/app/worker/utils.go index fd1243fe..47aba576 100644 --- a/app/worker/utils.go +++ b/app/worker/utils.go @@ -1,4 +1,4 @@ -// Copyright 2021 Northern.tech AS +// Copyright 2022 Northern.tech AS // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -15,180 +15,21 @@ package worker import ( - "bytes" - "encoding/json" "io/ioutil" - "os" - "regexp" "strings" - "text/template" - "github.com/thedevsaddam/gojsonq" - "go.mongodb.org/mongo-driver/bson/primitive" - - "github.com/mendersoftware/workflows/model" -) - -const ( - workflowEnvVariable = "env." - workflowInputVariable = "workflow.input." - regexVariable = `\$\{([^\}\|]+)(?:\|([^\}]+))?}` - regexOutputVariable = `(.*)\.json\.(.*)` + "github.com/mendersoftware/workflows/app/processor" ) -var reExpression = regexp.MustCompile(regexVariable) -var reExpressionOutput = regexp.MustCompile(regexOutputVariable) - -func processJobString(data string, workflow *model.Workflow, job *model.Job) string { - matches := reExpression.FindAllStringSubmatch(data, -1) - - // search for ${...} expressions in the data string - for _, submatch := range matches { - // content of the ${...} expression, without the brackets - match := submatch[1] - defaultValue := submatch[2] - if strings.HasPrefix(match, workflowInputVariable) && - len(match) > len(workflowInputVariable) { - // Replace ${workflow.input.KEY} with the KEY input variable - paramName := match[len(workflowInputVariable):] - found := false - for _, param := range job.InputParameters { - if param.Name == paramName { - value := param.Value - data = strings.ReplaceAll(data, submatch[0], value) - found = true - break - } - } - if !found { - data = strings.ReplaceAll(data, submatch[0], defaultValue) - } - } else if strings.HasPrefix(match, workflowEnvVariable) && - len(match) > len(workflowEnvVariable) { - // Replace ${env.KEY} with the KEY environment variable - envName := match[len(workflowEnvVariable):] - envValue := os.Getenv(envName) - if envValue == "" { - envValue = defaultValue - } - data = strings.ReplaceAll(data, submatch[0], envValue) - } else if output := reExpressionOutput.FindStringSubmatch(match); len(output) > 0 { - // Replace ${TASK_NAME.json.JSONPATH} with the value of the JSONPATH expression from the - // JSON output of the previous task with name TASK_NAME. If the output is not a valid - // JSON or the JSONPATH does not resolve to a value, replace with empty string - for _, result := range job.Results { - if result.Name == output[1] { - varKey := output[2] - var output string - if result.Type == model.TaskTypeHTTP { - output = result.HTTPResponse.Body - } else if result.Type == model.TaskTypeCLI { - output = result.CLI.Output - } else { - continue - } - varValue := gojsonq.New().FromString(output).Find(varKey) - if varValue == nil { - varValue = "" - } - varValueString, err := ConvertAnythingToString(varValue) - if err == nil { - if varValueString == "" { - varValueString = defaultValue - } - data = strings.ReplaceAll(data, submatch[0], varValueString) - } - break - } - } - } - } - - return data -} - -// maybeExecuteGoTemplate tries to parse and execute data as a go template -// if it fails to do so, data is returned. -func maybeExecuteGoTemplate(data string, input map[string]interface{}) string { - tmpl, err := template.New("go-template").Parse(data) - if err != nil { - return data - } - buf := &bytes.Buffer{} - err = tmpl.Execute(buf, input) - if err != nil { - return data - } - return buf.String() -} - -func processJobStringOrFile(data string, workflow *model.Workflow, job *model.Job) (string, error) { - data = processJobString(data, workflow, job) +func processJobStringOrFile(data string, ps *processor.JobStringProcessor) (string, error) { + data = ps.ProcessJobString(data) if strings.HasPrefix(data, "@") { filePath := data[1:] buffer, err := ioutil.ReadFile(filePath) if err != nil { return "", err } - data = processJobString(string(buffer), workflow, job) + data = ps.ProcessJobString(string(buffer)) } return data, nil } - -func processJobJSON(data interface{}, workflow *model.Workflow, job *model.Job) interface{} { - switch value := data.(type) { - case []interface{}: - result := make([]interface{}, len(value)) - for i, item := range value { - result[i] = processJobJSON(item, workflow, job) - } - return result - case map[string]interface{}: - result := make(map[string]interface{}) - for key, item := range value { - result[key] = processJobJSON(item, workflow, job) - } - return result - case string: - if len(value) > 3 && value[0:2] == "${" && value[len(value)-1:] == "}" { - key := value[2 : len(value)-1] - if strings.HasPrefix(key, workflowInputVariable) && - len(key) > len(workflowInputVariable) { - key = key[len(workflowInputVariable):] - for _, param := range job.InputParameters { - if param.Name == key && param.Raw != nil { - return processJobJSON(param.Raw, workflow, job) - } - } - return nil - } - } - return processJobString(value, workflow, job) - case primitive.D: - result := make(map[string]interface{}) - for key, item := range value.Map() { - result[key] = processJobJSON(item, workflow, job) - } - return result - case []primitive.D: - result := make([]interface{}, len(value)) - for i, item := range value { - result[i] = processJobJSON(item, workflow, job) - } - return result - } - return data -} - -// ConvertAnythingToString returns the string representation of anything -func ConvertAnythingToString(value interface{}) (string, error) { - valueString, ok := value.(string) - if !ok { - valueBytes, err := json.Marshal(value) - if err != nil { - return "", err - } - valueString = string(valueBytes) - } - return valueString, nil -} diff --git a/app/worker/utils_test.go b/app/worker/utils_test.go index 83c14274..f03e1db2 100644 --- a/app/worker/utils_test.go +++ b/app/worker/utils_test.go @@ -1,4 +1,4 @@ -// Copyright 2021 Northern.tech AS +// Copyright 2022 Northern.tech AS // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -24,6 +24,7 @@ import ( "github.com/stretchr/testify/assert" "go.mongodb.org/mongo-driver/bson" + "github.com/mendersoftware/workflows/app/processor" "github.com/mendersoftware/workflows/model" ) @@ -38,16 +39,23 @@ func TestProcessJobString(t *testing.T) { InputParameters: []model.InputParameter{ { Name: "key", - Value: "test", + Value: "test+test", }, }, } - res := processJobString("_${workflow.input.key}_", workflow, job) - assert.Equal(t, "_test_", res) + ps := processor.NewJobStringProcessor(workflow, job) + res := ps.ProcessJobString("_${workflow.input.key}_") + assert.Equal(t, "_test+test_", res) - res = processJobString("_${workflow.input.another_key|default}_", workflow, job) + res = ps.ProcessJobString("_${workflow.input.another_key|default}_") assert.Equal(t, "_default_", res) + + res = ps.ProcessJobString("_${encoding=url;workflow.input.key}_") + assert.Equal(t, "_test%2Btest_", res) + + res = ps.ProcessJobString("_${encoding=plain;workflow.input.key}_") + assert.Equal(t, "_test+test_", res) } func TestProcessJobStringEnvVariable(t *testing.T) { @@ -56,12 +64,13 @@ func TestProcessJobStringEnvVariable(t *testing.T) { } job := &model.Job{} - res := processJobString("_${env.PWD}_", workflow, job) + ps := processor.NewJobStringProcessor(workflow, job) + res := ps.ProcessJobString("_${env.PWD}_") pwd := os.Getenv("PWD") expected := fmt.Sprintf("_%s_", pwd) assert.Equal(t, expected, res) - res = processJobString("_${env.ENV_VARIABLE_WHICH_DOES_NOT_EXIST|default}_", workflow, job) + res = ps.ProcessJobString("_${env.ENV_VARIABLE_WHICH_DOES_NOT_EXIST|default}_") expected = "_default_" assert.Equal(t, expected, res) } @@ -160,7 +169,8 @@ func TestProcessJobStringJSONOutputFromPreviousResult(t *testing.T) { Results: []model.TaskResult{test.taskResult}, } - res := processJobString(test.expression, workflow, job) + ps := processor.NewJobStringProcessor(workflow, job) + res := ps.ProcessJobString(test.expression) assert.Equal(t, test.expectedValue, res) } } @@ -281,7 +291,10 @@ func TestProcessJobJSON(t *testing.T) { }, } - res := processJobJSON(test.json, workflow, job) + ps := processor.NewJobStringProcessor(workflow, job) + jp := processor.NewJobProcessor(job) + + res := jp.ProcessJSON(test.json, ps) assert.Equal(t, test.result, res) jsonString, err := json.Marshal(res) diff --git a/utils/utils.go b/utils/utils.go new file mode 100644 index 00000000..171d0dee --- /dev/null +++ b/utils/utils.go @@ -0,0 +1,30 @@ +// Copyright 2022 Northern.tech AS +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +import "encoding/json" + +// ConvertAnythingToString returns the string representation of anything +func ConvertAnythingToString(value interface{}) (string, error) { + valueString, ok := value.(string) + if !ok { + valueBytes, err := json.Marshal(value) + if err != nil { + return "", err + } + valueString = string(valueBytes) + } + return valueString, nil +} diff --git a/worker/decommission_device.json b/worker/decommission_device.json index 75170772..68ec0af0 100644 --- a/worker/decommission_device.json +++ b/worker/decommission_device.json @@ -8,7 +8,7 @@ "type": "http", "retries": 3, "http": { - "uri": "http://${env.INVENTORY_ADDR|mender-inventory:8080}/api/internal/v1/inventory/tenants/${workflow.input.tenant_id}/devices/${workflow.input.device_id}", + "uri": "http://${env.INVENTORY_ADDR|mender-inventory:8080}/api/internal/v1/inventory/tenants/${encoding=url;workflow.input.tenant_id}/devices/${encoding=url;workflow.input.device_id}", "method": "DELETE", "headers": { "X-MEN-RequestID": "${workflow.input.request_id}" @@ -22,7 +22,7 @@ "type": "http", "retries": 3, "http": { - "uri": "http://${env.DEPLOYMENTS_ADDR|mender-deployments:8080}/api/internal/v1/deployments/tenants/${workflow.input.tenant_id}/deployments/devices/${workflow.input.device_id}", + "uri": "http://${env.DEPLOYMENTS_ADDR|mender-deployments:8080}/api/internal/v1/deployments/tenants/${encoding=url;workflow.input.tenant_id}/deployments/devices/${encoding=url;workflow.input.device_id}", "method": "DELETE", "headers": { "X-MEN-RequestID": "${workflow.input.request_id}" @@ -39,7 +39,7 @@ "${env.HAVE_DEVICECONNECT}" ], "http": { - "uri": "http://${env.DEVICECONNECT_ADDR|mender-deviceconnect:8080}/api/internal/v1/deviceconnect/tenants/${workflow.input.tenant_id}/devices/${workflow.input.device_id}", + "uri": "http://${env.DEVICECONNECT_ADDR|mender-deviceconnect:8080}/api/internal/v1/deviceconnect/tenants/${encoding=url;workflow.input.tenant_id}/devices/${encoding=url;workflow.input.device_id}", "method": "DELETE", "headers": { "X-MEN-RequestID": "${workflow.input.request_id}" @@ -57,7 +57,7 @@ "${env.HAVE_DEVICECONFIG}" ], "http": { - "uri": "http://${env.DEVICECONFIG_ADDR|mender-deviceconfig:8080}/api/internal/v1/deviceconfig/tenants/${workflow.input.tenant_id}/devices/${workflow.input.device_id}", + "uri": "http://${env.DEVICECONFIG_ADDR|mender-deviceconfig:8080}/api/internal/v1/deviceconfig/tenants/${encoding=url;workflow.input.tenant_id}/devices/${encoding=url;workflow.input.device_id}", "method": "DELETE", "headers": { "X-MEN-RequestID": "${workflow.input.request_id}" @@ -72,7 +72,7 @@ "type": "http", "retries": 3, "http": { - "uri": "http://${env.IOT_MANAGER_ADDR|mender-iot-manager:8080}/api/internal/v1/iot-manager/tenants/${workflow.input.tenant_id}/devices/${workflow.input.device_id}", + "uri": "http://${env.IOT_MANAGER_ADDR|mender-iot-manager:8080}/api/internal/v1/iot-manager/tenants/${encoding=url;workflow.input.tenant_id}/devices/${encoding=url;workflow.input.device_id}", "method": "DELETE", "headers": { "X-MEN-RequestID": "${workflow.input.request_id}" diff --git a/worker/deploy_device_configuration.json b/worker/deploy_device_configuration.json index c20ec008..69bb66e1 100644 --- a/worker/deploy_device_configuration.json +++ b/worker/deploy_device_configuration.json @@ -8,7 +8,7 @@ "type": "http", "retries": 3, "http": { - "uri": "http://${env.DEPLOYMENTS_ADDR|mender-deployments:8080}/api/internal/v1/deployments/tenants/${workflow.input.tenant_id}/configuration/deployments/${workflow.input.deployment_id}/devices/${workflow.input.device_id}", + "uri": "http://${env.DEPLOYMENTS_ADDR|mender-deployments:8080}/api/internal/v1/deployments/tenants/${encoding=url;workflow.input.tenant_id}/configuration/deployments/${encoding=url;workflow.input.deployment_id}/devices/${encoding=url;workflow.input.device_id}", "method": "POST", "contentType": "application/json", "json": { @@ -28,7 +28,7 @@ "type": "http", "retries": 3, "http": { - "uri": "http://${env.DEVICECONNECT_ADDR|mender-deviceconnect:8080}/api/internal/v1/deviceconnect/tenants/${workflow.input.tenant_id}/devices/${workflow.input.device_id}/check-update", + "uri": "http://${env.DEVICECONNECT_ADDR|mender-deviceconnect:8080}/api/internal/v1/deviceconnect/tenants/${encoding=url;workflow.input.tenant_id}/devices/${encoding=url;workflow.input.device_id}/check-update", "method": "POST", "headers": { "X-MEN-RequestID": "${workflow.input.request_id}" diff --git a/worker/provision_device.json b/worker/provision_device.json index d42f2241..a411d138 100644 --- a/worker/provision_device.json +++ b/worker/provision_device.json @@ -7,7 +7,7 @@ "type": "http", "retries": 3, "http": { - "uri": "http://${env.INVENTORY_ADDR|mender-inventory:8080}/api/internal/v1/inventory/tenants/${workflow.input.tenant_id}/devices", + "uri": "http://${env.INVENTORY_ADDR|mender-inventory:8080}/api/internal/v1/inventory/tenants/${encoding=url;workflow.input.tenant_id}/devices", "method": "POST", "contentType": "application/json", "json": { @@ -33,7 +33,7 @@ "${env.HAVE_DEVICECONNECT}" ], "http": { - "uri": "http://${env.DEVICECONNECT_ADDR|mender-deviceconnect:8080}/api/internal/v1/deviceconnect/tenants/${workflow.input.tenant_id}/devices", + "uri": "http://${env.DEVICECONNECT_ADDR|mender-deviceconnect:8080}/api/internal/v1/deviceconnect/tenants/${encoding=url;workflow.input.tenant_id}/devices", "method": "POST", "contentType": "application/json", "json": {"device_id": "${workflow.input.device_id}"}, @@ -52,7 +52,7 @@ "${env.HAVE_DEVICECONFIG}" ], "http": { - "uri": "http://${env.DEVICECONFIG_ADDR|mender-deviceconfig:8080}/api/internal/v1/deviceconfig/tenants/${workflow.input.tenant_id}/devices", + "uri": "http://${env.DEVICECONFIG_ADDR|mender-deviceconfig:8080}/api/internal/v1/deviceconfig/tenants/${encoding=url;workflow.input.tenant_id}/devices", "method": "POST", "contentType": "application/json", "json": {"device_id": "${workflow.input.device_id}"}, @@ -69,7 +69,7 @@ "type": "http", "retries": 3, "http": { - "uri": "http://${env.IOT_MANAGER_ADDR|mender-iot-manager:8080}/api/internal/v1/iot-manager/tenants/${workflow.input.tenant_id}/devices", + "uri": "http://${env.IOT_MANAGER_ADDR|mender-iot-manager:8080}/api/internal/v1/iot-manager/tenants/${encoding=url;workflow.input.tenant_id}/devices", "method": "POST", "contentType": "application/json", "json": "${workflow.input.device}", diff --git a/worker/provision_external_device.json b/worker/provision_external_device.json index df8db71c..fe6e3ca3 100644 --- a/worker/provision_external_device.json +++ b/worker/provision_external_device.json @@ -12,7 +12,7 @@ "requires": [ "${env.HAVE_DEVICECONFIG}" ], - "uri": "http://${env.DEVICECONFIG_ADDR|mender-deviceconfig:8080}/api/internal/v1/deviceconfig/tenants/${workflow.input.tenant_id}/configurations/device/${workflow.input.device_id}", + "uri": "http://${env.DEVICECONFIG_ADDR|mender-deviceconfig:8080}/api/internal/v1/deviceconfig/tenants/${encoding=url;workflow.input.tenant_id}/configurations/device/${encoding=url;workflow.input.device_id}", "method": "PATCH", "headers": { "X-MEN-Requestid": "${workflow.input.request_id}", @@ -36,7 +36,7 @@ "requires": [ "${env.HAVE_DEVICECONFIG}" ], - "uri": "http://${env.DEVICECONFIG_ADDR|mender-deviceconfig:8080}/api/internal/v1/deviceconfig/tenants/${workflow.input.tenant_id}/configurations/device/${workflow.input.device_id}/deploy", + "uri": "http://${env.DEVICECONFIG_ADDR|mender-deviceconfig:8080}/api/internal/v1/deviceconfig/tenants/${encoding=url;workflow.input.tenant_id}/configurations/device/${encoding=url;workflow.input.device_id}/deploy", "method": "POST", "contentType": "application/json", "headers": { diff --git a/worker/reindex_inventory.json b/worker/reindex_inventory.json index 775e030a..8375c8b1 100644 --- a/worker/reindex_inventory.json +++ b/worker/reindex_inventory.json @@ -9,7 +9,7 @@ "type": "http", "retries": 3, "http": { - "uri": "http://${env.INVENTORY_ADDR|mender-inventory:8080}/api/internal/v1/inventory/tenants/${workflow.input.tenant_id}/devices/${workflow.input.device_id}/reindex?service=${workflow.input.service}", + "uri": "http://${env.INVENTORY_ADDR|mender-inventory:8080}/api/internal/v1/inventory/tenants/${encoding=url;workflow.input.tenant_id}/devices/${encoding=url;workflow.input.device_id}/reindex?service=${encoding=url;workflow.input.service}", "method": "POST", "contentType": "application/json", "headers": { diff --git a/worker/update_device_inventory.json b/worker/update_device_inventory.json index e818caf1..976c21b3 100644 --- a/worker/update_device_inventory.json +++ b/worker/update_device_inventory.json @@ -8,7 +8,7 @@ "type": "http", "retries": 3, "http": { - "uri": "http://${env.INVENTORY_ADDR|mender-inventory:8080}/api/internal/v1/inventory/tenants/${workflow.input.tenant_id}/device/${workflow.input.device_id}/attribute/scope/${workflow.input.scope}", + "uri": "http://${env.INVENTORY_ADDR|mender-inventory:8080}/api/internal/v1/inventory/tenants/${encoding=url;workflow.input.tenant_id}/device/${encoding=url;workflow.input.device_id}/attribute/scope/${encoding=url;workflow.input.scope}", "method": "PATCH", "contentType": "application/json", "body": "${workflow.input.attributes}", diff --git a/worker/update_device_status.json b/worker/update_device_status.json index fdc4b809..1b9e0865 100644 --- a/worker/update_device_status.json +++ b/worker/update_device_status.json @@ -8,7 +8,7 @@ "type": "http", "retries": 3, "http": { - "uri": "http://${env.INVENTORY_ADDR|mender-inventory:8080}/api/internal/v1/inventory/tenants/${workflow.input.tenant_id}/devices/status/${workflow.input.device_status}", + "uri": "http://${env.INVENTORY_ADDR|mender-inventory:8080}/api/internal/v1/inventory/tenants/${encoding=url;workflow.input.tenant_id}/devices/status/${encoding=url;workflow.input.device_status}", "method": "POST", "contentType": "application/json", "json": "${workflow.input.devices}", @@ -24,7 +24,7 @@ "type": "http", "retries": 3, "http": { - "uri": "http://${env.IOT_MANAGER_ADDR|mender-iot-manager:8080}/api/internal/v1/iot-manager/tenants/${workflow.input.tenant_id}/bulk/devices/status/${workflow.input.device_status}", + "uri": "http://${env.IOT_MANAGER_ADDR|mender-iot-manager:8080}/api/internal/v1/iot-manager/tenants/${encoding=url;workflow.input.tenant_id}/bulk/devices/status/${encoding=url;workflow.input.device_status}", "method": "PUT", "contentType": "application/json", "json": "${workflow.input.devices}",