fix(api): workflow as code improvements (#3423)

* Since a hook is mandatory for workflow as code, do not export it
* Do not export default VCS payload

closes #3239

cli/cdsctl/workflow_export.go

@@ -2,9 +2,11 @@ package main
 
 import (
 	"fmt"
+	"net/http"
 	"reflect"
 
 	"github.com/ovh/cds/cli"
+	"github.com/ovh/cds/sdk/cdsclient"
 )
 
 var workflowExportCmd = cli.Command{
@@ -31,7 +33,24 @@ var workflowExportCmd = cli.Command{
 }
 
 func workflowExportRun(c cli.Values) error {
-	btes, err := client.WorkflowExport(c.GetString(_ProjectKey), c.GetString(_WorkflowName), c.GetBool("with-permissions"), c.GetString("format"))
+	mods := []cdsclient.RequestModifier{
+		func(r *http.Request) {
+			q := r.URL.Query()
+			q.Set("format", c.GetString("format"))
+			r.URL.RawQuery = q.Encode()
+		},
+	}
+	if c.GetBool("with-permissions") {
+		mods = append(mods,
+			func(r *http.Request) {
+				q := r.URL.Query()
+				q.Set("withPermissions", "true")
+				r.URL.RawQuery = q.Encode()
+			},
+		)
+	}
+
+	btes, err := client.WorkflowExport(c.GetString(_ProjectKey), c.GetString(_WorkflowName), mods...)
 	if err != nil {
 		return err
 	}

cli/cdsctl/workflow_pull.go

@@ -4,12 +4,14 @@ import (
 	"archive/tar"
 	"fmt"
 	"io"
+	"net/http"
 	"os"
 	"path/filepath"
 	"reflect"
 	"strings"
 
 	"github.com/ovh/cds/cli"
+	"github.com/ovh/cds/sdk/cdsclient"
 )
 
 var workflowPullCmd = cli.Command{
@@ -57,7 +59,18 @@ func workflowPullRun(c cli.Values) error {
 		return fmt.Errorf("Unable to create directory %s: %v", c.GetString("output-dir"), err)
 	}
 
-	tr, err := client.WorkflowPull(c.GetString(_ProjectKey), c.GetString(_WorkflowName), c.GetBool("with-permissions"))
+	mods := []cdsclient.RequestModifier{}
+	if c.GetBool("with-permissions") {
+		mods = append(mods,
+			func(r *http.Request) {
+				q := r.URL.Query()
+				q.Set("withPermissions", "true")
+				r.URL.RawQuery = q.Encode()
+			},
+		)
+	}
+
+	tr, err := client.WorkflowPull(c.GetString(_ProjectKey), c.GetString(_WorkflowName), mods...)
 	if err != nil {
 		return err
 	}

engine/api/workflow/audit.go

@@ -63,7 +63,7 @@ func (a addWorkflowAudit) Compute(db gorp.SqlExecutor, e sdk.Event) error {
 	}
 
 	buffer := bytes.NewBufferString("")
-	_, errE := exportWorkflow(wEvent.Workflow, exportentities.FormatYAML, false, buffer)
+	_, errE := exportWorkflow(wEvent.Workflow, exportentities.FormatYAML, buffer)
 	if errE != nil {
 		return sdk.WrapError(errE, "addWorkflowAudit.Compute> Unable to export workflow")
 	}
@@ -88,12 +88,12 @@ func (u updateWorkflowAudit) Compute(db gorp.SqlExecutor, e sdk.Event) error {
 	}
 
 	oldWorkflowBuffer := bytes.NewBufferString("")
-	_, errE := exportWorkflow(wEvent.OldWorkflow, exportentities.FormatYAML, false, oldWorkflowBuffer)
+	_, errE := exportWorkflow(wEvent.OldWorkflow, exportentities.FormatYAML, oldWorkflowBuffer)
 	if errE != nil {
 		return sdk.WrapError(errE, "updateWorkflowAudit.Compute> Unable to export workflow")
 	}
 	newWorkflowBuffer := bytes.NewBufferString("")
-	_, errN := exportWorkflow(wEvent.NewWorkflow, exportentities.FormatYAML, false, newWorkflowBuffer)
+	_, errN := exportWorkflow(wEvent.NewWorkflow, exportentities.FormatYAML, newWorkflowBuffer)
 	if errN != nil {
 		return sdk.WrapError(errN, "updateWorkflowAudit.Compute> Unable to export workflow")
 	}
@@ -119,7 +119,7 @@ func (d deleteWorkflowAudit) Compute(db gorp.SqlExecutor, e sdk.Event) error {
 	}
 
 	oldWorkflowBuffer := bytes.NewBufferString("")
-	_, errE := exportWorkflow(wEvent.Workflow, exportentities.FormatYAML, false, oldWorkflowBuffer)
+	_, errE := exportWorkflow(wEvent.Workflow, exportentities.FormatYAML, oldWorkflowBuffer)
 	if errE != nil {
 		return sdk.WrapError(errE, "deleteWorkflowAudit.Compute> Unable to export workflow")
 	}

engine/api/workflow/dao.go

@@ -1040,6 +1040,22 @@ func Push(ctx context.Context, db *gorp.DbMap, store cache.Store, proj *sdk.Proj
 		dryRun = opts.DryRun
 	}
 
+	// In workflow as code context, if we only have the repowebhook, we skip it
+	//  because it will be automatically recreated later with the proper configuration
+	if opts != nil && opts.FromRepository != "" {
+		if len(wrkflw.Workflow) == 0 {
+			if len(wrkflw.PipelineHooks) == 1 && wrkflw.PipelineHooks[0].Model == sdk.RepositoryWebHookModelName {
+				wrkflw.PipelineHooks = nil
+			}
+		} else {
+			for node, hooks := range wrkflw.Hooks {
+				if len(hooks) == 1 && hooks[0].Model == sdk.RepositoryWebHookModelName {
+					wrkflw.Hooks[node] = nil
+				}
+			}
+		}
+	}
+
 	wf, msgList, err := ParseAndImport(ctx, tx, store, proj, &wrkflw, u, ImportOptions{DryRun: dryRun, Force: true})
 	if err != nil {
 		log.Error("Push> Unable to import workflow: %v", err)

engine/api/workflow/dao_node_run_vulnerability.go

@@ -205,6 +205,7 @@ func loadLatestRunVulnerabilityReport(db gorp.SqlExecutor, nr *sdk.WorkflowNodeR
 	return dbReport.Report.Summary, nil
 }
 
+// InsertVulnerabilityReport inserts vulnerability report
 func InsertVulnerabilityReport(db gorp.SqlExecutor, report sdk.WorkflowNodeRunVulnerabilityReport) error {
 	dbReport := dbNodeRunVulenrabilitiesReport(report)
 	if err := db.Insert(&dbReport); err != nil {

engine/api/workflow/dao_test.go

@@ -80,7 +80,7 @@ func TestInsertSimpleWorkflowAndExport(t *testing.T) {
 	test.NoError(t, err)
 	assert.Equal(t, 1, len(ws))
 
-	exp, err := exportentities.NewWorkflow(*w1, false)
+	exp, err := exportentities.NewWorkflow(*w1)
 	test.NoError(t, err)
 	btes, err := exportentities.Marshal(exp, exportentities.FormatYAML)
 	test.NoError(t, err)
@@ -311,7 +311,7 @@ func TestInsertComplexeWorkflowAndExport(t *testing.T) {
 
 	assertEqualNode(t, w.Root, w1.Root)
 
-	exp, err := exportentities.NewWorkflow(w, false)
+	exp, err := exportentities.NewWorkflow(w)
 	test.NoError(t, err)
 	btes, err := exportentities.Marshal(exp, exportentities.FormatYAML)
 	test.NoError(t, err)
@@ -789,7 +789,7 @@ func TestInsertComplexeWorkflowWithJoinsAndExport(t *testing.T) {
 	}, w1.Joins[0].SourceNodeIDs)
 	assert.Equal(t, pip5.Name, w.Joins[0].Triggers[0].WorkflowDestNode.PipelineName)
 
-	exp, err := exportentities.NewWorkflow(*w1, false)
+	exp, err := exportentities.NewWorkflow(*w1)
 	test.NoError(t, err)
 	btes, err := exportentities.Marshal(exp, exportentities.FormatYAML)
 	test.NoError(t, err)
@@ -1250,7 +1250,7 @@ func TestInsertSimpleWorkflowWithHookAndExport(t *testing.T) {
 	assert.Len(t, w.Root.Hooks, 1)
 	t.Log(w.Root.Hooks)
 
-	exp, err := exportentities.NewWorkflow(*w1, false)
+	exp, err := exportentities.NewWorkflow(*w1)
 	test.NoError(t, err)
 	btes, err := exportentities.Marshal(exp, exportentities.FormatYAML)
 	test.NoError(t, err)

engine/api/workflow/hook.go

@@ -295,8 +295,17 @@ func mergeAndDiffHook(oldHooks map[string]sdk.WorkflowNodeHook, newHooks map[str
 
 // DefaultPayload returns the default payload for the workflow root
 func DefaultPayload(ctx context.Context, db gorp.SqlExecutor, store cache.Store, p *sdk.Project, u *sdk.User, wf *sdk.Workflow) (interface{}, error) {
+	if wf.Root.Context == nil {
+		return nil, nil
+	}
+
+	if wf.Root.Context.Application == nil {
+		return wf.Root.Context.DefaultPayload, nil
+	}
+
 	var defaultPayload interface{}
-	if wf.Root.Context == nil || wf.Root.Context.Application == nil || wf.Root.Context.Application.ID == 0 {
+	// Load application if not available
+	if wf.Root.Context != nil && wf.Root.Context.Application == nil && wf.Root.Context.ApplicationID != 0 {
 		app, errLa := application.LoadByID(db, store, wf.Root.Context.ApplicationID, u)
 		if errLa != nil {
 			return wf.Root.Context.DefaultPayload, sdk.WrapError(errLa, "DefaultPayload> unable to load application by id %d", wf.Root.Context.ApplicationID)
@@ -328,10 +337,14 @@ func DefaultPayload(ctx context.Context, db gorp.SqlExecutor, store cache.Store,
 
 		defaultPayload = wf.Root.Context.DefaultPayload
 		if !wf.Root.Context.HasDefaultPayload() {
-			defaultPayload = sdk.WorkflowNodeContextDefaultPayloadVCS{
+			structuredDefaultPayload := sdk.WorkflowNodeContextDefaultPayloadVCS{
 				GitBranch:     defaultBranch,
 				GitRepository: wf.Root.Context.Application.RepositoryFullname,
 			}
+			defaultPayloadBtes, _ := json.Marshal(structuredDefaultPayload)
+			if err := json.Unmarshal(defaultPayloadBtes, &defaultPayload); err != nil {
+				return nil, err
+			}
 		} else if defaultPayloadMap, err := wf.Root.Context.DefaultPayloadToMap(); err == nil && defaultPayloadMap["git.branch"] == "" {
 			defaultPayloadMap["git.branch"] = defaultBranch
 			defaultPayloadMap["git.repository"] = wf.Root.Context.Application.RepositoryFullname

engine/api/workflow/workflow_exporter.go

@@ -6,6 +6,7 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"reflect"
 
 	"github.com/go-gorp/gorp"
 
@@ -19,7 +20,7 @@ import (
 )
 
 // Export a workflow
-func Export(ctx context.Context, db gorp.SqlExecutor, cache cache.Store, proj *sdk.Project, name string, f exportentities.Format, withPermissions bool, u *sdk.User, w io.Writer) (int, error) {
+func Export(ctx context.Context, db gorp.SqlExecutor, cache cache.Store, proj *sdk.Project, name string, f exportentities.Format, u *sdk.User, w io.Writer, opts ...exportentities.WorkflowOptions) (int, error) {
 	ctx, end := observability.Span(ctx, "workflow.Export")
 	defer end()
 
@@ -28,11 +29,16 @@ func Export(ctx context.Context, db gorp.SqlExecutor, cache cache.Store, proj *s
 		return 0, sdk.WrapError(errload, "workflow.Export> Cannot load workflow %s", name)
 	}
 
-	return exportWorkflow(*wf, f, withPermissions, w)
+	// If repo is from as-code do not export WorkflowSkipIfOnlyOneRepoWebhook
+	if wf.FromRepository != "" {
+		opts = append(opts, exportentities.WorkflowSkipIfOnlyOneRepoWebhook)
+	}
+
+	return exportWorkflow(*wf, f, w, opts...)
 }
 
-func exportWorkflow(wf sdk.Workflow, f exportentities.Format, withPermissions bool, w io.Writer) (int, error) {
-	e, err := exportentities.NewWorkflow(wf, withPermissions)
+func exportWorkflow(wf sdk.Workflow, f exportentities.Format, w io.Writer, opts ...exportentities.WorkflowOptions) (int, error) {
+	e, err := exportentities.NewWorkflow(wf, opts...)
 	if err != nil {
 		return 0, err
 	}
@@ -52,7 +58,7 @@ func exportWorkflow(wf sdk.Workflow, f exportentities.Format, withPermissions bo
 }
 
 // Pull a workflow with all it dependencies; it writes a tar buffer in the writer
-func Pull(ctx context.Context, db gorp.SqlExecutor, cache cache.Store, proj *sdk.Project, name string, f exportentities.Format, withPermissions bool, encryptFunc sdk.EncryptFunc, u *sdk.User, w io.Writer) error {
+func Pull(ctx context.Context, db gorp.SqlExecutor, cache cache.Store, proj *sdk.Project, name string, f exportentities.Format, encryptFunc sdk.EncryptFunc, u *sdk.User, w io.Writer, opts ...exportentities.WorkflowOptions) error {
 	ctx, end := observability.Span(ctx, "workflow.Pull")
 	defer end()
 
@@ -99,7 +105,7 @@ func Pull(ctx context.Context, db gorp.SqlExecutor, cache cache.Store, proj *sdk
 	tw := tar.NewWriter(w)
 
 	buffw := new(bytes.Buffer)
-	size, errw := exportWorkflow(*wf, f, withPermissions, buffw)
+	size, errw := exportWorkflow(*wf, f, buffw, opts...)
 	if errw != nil {
 		tw.Close()
 		return sdk.WrapError(errw, "workflow.Pull> Unable to export workflow")
@@ -119,6 +125,13 @@ func Pull(ctx context.Context, db gorp.SqlExecutor, cache cache.Store, proj *sdk
 		return sdk.WrapError(err, "workflow.Pull> Unable to copy workflow buffer")
 	}
 
+	var withPermissions bool
+	for _, f := range opts {
+		if reflect.ValueOf(f).Pointer() == reflect.ValueOf(exportentities.WorkflowWithPermissions).Pointer() {
+			withPermissions = true
+		}
+	}
+
 	for _, a := range apps {
 		buff := new(bytes.Buffer)
 		size, err := application.ExportApplication(db, a, f, withPermissions, encryptFunc, buff)

engine/api/workflow/workflow_exporter_test.go

@@ -123,7 +123,7 @@ func TestPull(t *testing.T) {
 	test.Equal(t, w.PurgeTags, w1.PurgeTags)
 
 	buff := new(bytes.Buffer)
-	test.NoError(t, workflow.Pull(context.TODO(), db, cache, proj, w1.Name, exportentities.FormatYAML, false, project.EncryptWithBuiltinKey, u, buff))
+	test.NoError(t, workflow.Pull(context.TODO(), db, cache, proj, w1.Name, exportentities.FormatYAML, project.EncryptWithBuiltinKey, u, buff))
 
 	// Open the tar archive for reading.
 	r := bytes.NewReader(buff.Bytes())

engine/api/workflow/workflow_importer.go

@@ -117,6 +117,15 @@ func Import(ctx context.Context, db gorp.SqlExecutor, store cache.Store, proj *s
 		return sdk.WrapError(errE, "Import> Cannot check if workflow exists")
 	}
 
+	//Manage default payload
+	var err error
+	if w.Root.Context == nil {
+		w.Root.Context = &sdk.WorkflowNodeContext{}
+	}
+	if w.Root.Context.DefaultPayload, err = DefaultPayload(ctx, db, store, proj, u, w); err != nil {
+		log.Warning("workflow.Import> Cannot set default payload : %v", err)
+	}
+
 	if !doUpdate {
 		if err := Insert(db, store, w, proj, u); err != nil {
 			return sdk.WrapError(err, "Import> Unable to insert workflow")

engine/api/workflow_export.go

@@ -27,6 +27,11 @@ func (api *API) getWorkflowExportHandler() service.Handler {
 		}
 		withPermissions := FormBool(r, "withPermissions")
 
+		opts := []exportentities.WorkflowOptions{}
+		if withPermissions {
+			opts = append(opts, exportentities.WorkflowWithPermissions)
+		}
+
 		f, err := exportentities.GetFormat(format)
 		if err != nil {
 			return sdk.WrapError(err, "getWorkflowExportHandler> Format invalid")
@@ -36,7 +41,7 @@ func (api *API) getWorkflowExportHandler() service.Handler {
 		if err != nil {
 			return sdk.WrapError(err, "getWorkflowExportHandler> unable to load projet")
 		}
-		if _, err := workflow.Export(ctx, api.mustDB(), api.Cache, proj, name, f, withPermissions, getUser(ctx), w); err != nil {
+		if _, err := workflow.Export(ctx, api.mustDB(), api.Cache, proj, name, f, getUser(ctx), w, opts...); err != nil {
 			return sdk.WrapError(err, "getWorkflowExportHandler>")
 		}
 
@@ -53,13 +58,18 @@ func (api *API) getWorkflowPullHandler() service.Handler {
 		name := vars["permWorkflowName"]
 		withPermissions := FormBool(r, "withPermissions")
 
+		opts := []exportentities.WorkflowOptions{}
+		if withPermissions {
+			opts = append(opts, exportentities.WorkflowWithPermissions)
+		}
+
 		proj, err := project.Load(api.mustDB(), api.Cache, key, getUser(ctx), project.LoadOptions.WithPlatforms)
 		if err != nil {
 			return sdk.WrapError(err, "getWorkflowPullHandler> unable to load projet")
 		}
 
 		buf := new(bytes.Buffer)
-		if err := workflow.Pull(ctx, api.mustDB(), api.Cache, proj, name, exportentities.FormatYAML, withPermissions, project.EncryptWithBuiltinKey, getUser(ctx), buf); err != nil {
+		if err := workflow.Pull(ctx, api.mustDB(), api.Cache, proj, name, exportentities.FormatYAML, project.EncryptWithBuiltinKey, getUser(ctx), buf, opts...); err != nil {
 			return sdk.WrapError(err, "getWorkflowPullHandler")
 		}
 

engine/api/workflow_test.go

@@ -441,11 +441,11 @@ func Test_postWorkflowRollbackHandler(t *testing.T) {
 
 	assert.NotEmpty(t, payload["git.branch"], "git.branch should not be empty")
 
-	eWf, err := exportentities.NewWorkflow(*wf, false)
+	eWf, err := exportentities.NewWorkflow(*wf)
 	test.NoError(t, err)
 	wfBts, err := yaml.Marshal(eWf)
 	test.NoError(t, err)
-	eWfUpdate, err := exportentities.NewWorkflow(*workflow1, false)
+	eWfUpdate, err := exportentities.NewWorkflow(*workflow1)
 	test.NoError(t, err)
 	wfUpdatedBts, err := yaml.Marshal(eWfUpdate)
 	test.NoError(t, err)

sdk/cdsclient/client_export.go

@@ -58,12 +58,9 @@ func (c *client) EnvironmentExport(projectKey, name string, exportWithPermission
 	return body, nil
 }
 
-func (c *client) WorkflowExport(projectKey, name string, exportWithPermissions bool, exportFormat string) ([]byte, error) {
-	path := fmt.Sprintf("/project/%s/export/workflows/%s?format=%s", projectKey, name, exportFormat)
-	if exportWithPermissions {
-		path += "&withPermissions=true"
-	}
-	bodyReader, _, _, err := c.Stream(context.Background(), "GET", path, nil, true)
+func (c *client) WorkflowExport(projectKey, name string, mods ...RequestModifier) ([]byte, error) {
+	path := fmt.Sprintf("/project/%s/export/workflows/%s", projectKey, name)
+	bodyReader, _, _, err := c.Stream(context.Background(), "GET", path, nil, true, mods...)
 	if err != nil {
 		return nil, err
 	}
@@ -76,12 +73,9 @@ func (c *client) WorkflowExport(projectKey, name string, exportWithPermissions b
 	return body, nil
 }
 
-func (c *client) WorkflowPull(projectKey, name string, exportWithPermissions bool) (*tar.Reader, error) {
+func (c *client) WorkflowPull(projectKey, name string, mods ...RequestModifier) (*tar.Reader, error) {
 	path := fmt.Sprintf("/project/%s/pull/workflows/%s", projectKey, name)
-	if exportWithPermissions {
-		path += "?withPermissions=true"
-	}
-	body, _, _, err := c.Request(context.Background(), "GET", path, nil)
+	body, _, _, err := c.Request(context.Background(), "GET", path, nil, mods...)
 	if err != nil {
 		return nil, err
 	}

sdk/cdsclient/client_import.go

@@ -197,7 +197,7 @@ func (c *client) WorkflowPush(projectKey string, tarContent io.Reader, mods ...R
 	if wName == "" {
 		return messages, nil, nil
 	}
-	tarReader, err := c.WorkflowPull(projectKey, wName, false)
+	tarReader, err := c.WorkflowPull(projectKey, wName, mods...)
 	if err != nil {
 		return nil, nil, err
 	}