From d72726330b37a303b4361b7b051b170a433a42c5 Mon Sep 17 00:00:00 2001 From: Miguel Martinez Trivino Date: Thu, 15 Jun 2023 16:49:40 +0200 Subject: [PATCH 1/6] feat: add discord Signed-off-by: Miguel Martinez Trivino --- app/cli/cmd/attached_integration_list.go | 33 ++- .../extensions/core/discord/v1/README.md | 20 ++ .../extensions/core/discord/v1/discord.go | 260 ++++++++++++++++++ app/controlplane/extensions/extensions.go | 2 + 4 files changed, 304 insertions(+), 11 deletions(-) create mode 100644 app/controlplane/extensions/core/discord/v1/README.md create mode 100644 app/controlplane/extensions/core/discord/v1/discord.go diff --git a/app/cli/cmd/attached_integration_list.go b/app/cli/cmd/attached_integration_list.go index 415051de7..32389faa0 100644 --- a/app/cli/cmd/attached_integration_list.go +++ b/app/cli/cmd/attached_integration_list.go @@ -53,22 +53,33 @@ func attachedIntegrationListTableOutput(attachments []*action.AttachedIntegratio fmt.Println("Integrations attached to workflows") t := newTableWriter() t.AppendHeader(table.Row{"ID", "Kind", "Config", "Attached At", "Workflow"}) - for _, i := range attachments { - wf := i.Workflow - integration := i.Integration + for _, attachment := range attachments { + wf := attachment.Workflow + integration := attachment.Integration + + // Merge attachment and integration configs + + // If there are not attachment configuration then create an empty map + if attachment.Config == nil { + attachment.Config = make(map[string]any) + } + + if integration.Config == nil { + integration.Config = make(map[string]any) + } var options []string - if i.Config != nil { - maps.Copy(i.Config, integration.Config) - for k, v := range i.Config { - if v == "" { - continue - } - options = append(options, fmt.Sprintf("%s: %v", k, v)) + maps.Copy(attachment.Config, integration.Config) + + // Show it as key-value pairs + for k, v := range attachment.Config { + if v == "" { + continue } + options = append(options, fmt.Sprintf("%s: %v", k, v)) } - t.AppendRow(table.Row{i.ID, integration.Kind, strings.Join(options, "\n"), i.CreatedAt.Format(time.RFC822), wf.NamespacedName()}) + t.AppendRow(table.Row{attachment.ID, integration.Kind, strings.Join(options, "\n"), attachment.CreatedAt.Format(time.RFC822), wf.NamespacedName()}) t.AppendSeparator() } diff --git a/app/controlplane/extensions/core/discord/v1/README.md b/app/controlplane/extensions/core/discord/v1/README.md new file mode 100644 index 000000000..d702fd1ef --- /dev/null +++ b/app/controlplane/extensions/core/discord/v1/README.md @@ -0,0 +1,20 @@ +# Discord extension + +You can use this template as a placeholder to create your own fan-out extension. +## How to use it + +These are the required steps + +### Setup: + +- Copy and rename the folder to your extension name +- Replace all the occurrences of `template` with your extension name + +### Implementation + +- Define the API request payloads for both Registration and Attachment +- Implement the [FanOutExtension interface](https://github.com/chainloop-dev/chainloop/blob/main/app/controlplane/extensions/sdk/v1/fanout.go#L55). The template comes prefilled with some commented out code. + +### Enable extension for + +- Add it to the list of available extensions [here](`../../../../../extensions.go`). This will make this extension available the next time the control plane starts. \ No newline at end of file diff --git a/app/controlplane/extensions/core/discord/v1/discord.go b/app/controlplane/extensions/core/discord/v1/discord.go new file mode 100644 index 000000000..d5462aab9 --- /dev/null +++ b/app/controlplane/extensions/core/discord/v1/discord.go @@ -0,0 +1,260 @@ +// +// Copyright 2023 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package discord + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "mime/multipart" + "net/http" + + "github.com/chainloop-dev/chainloop/app/controlplane/extensions/sdk/v1" + "github.com/go-kratos/kratos/v2/log" +) + +type Integration struct { + *sdk.FanOutIntegration +} + +// 1 - API schema definitions +type registrationRequest struct { + WebhookURL string `json:"webhook" jsonschema:"format=uri,description=URL of the discord webhook"` + Username string `json:"username,omitempty" jsonschema:"minLenght=1,description=Override the default username of the webhook "` +} + +type attachmentRequest struct{} + +// 2 - Configuration state +type registrationState struct { + // Information from the webhook + WebhookName string `json:"name"` + WebhookOwner string `json:"owner"` + + // Username to be used while posting the message + Username string `json:"username"` +} + +func New(l log.Logger) (sdk.FanOut, error) { + base, err := sdk.NewFanOut( + &sdk.NewParams{ + ID: "discord-webhook", + Version: "0.1", + Description: "Send attestations to Discord", + Logger: l, + InputSchema: &sdk.InputSchema{ + Registration: registrationRequest{}, + Attachment: attachmentRequest{}, + }, + }, + sdk.WithEnvelope(), + ) + + if err != nil { + return nil, err + } + + return &Integration{base}, nil +} + +type webhookResponse struct { + Name string `json:"name"` + User struct { + Username string `json:"username"` + } `json:"user"` +} + +// Register is executed when a operator wants to register a specific instance of this integration with their Chainloop organization +func (i *Integration) Register(_ context.Context, req *sdk.RegistrationRequest) (*sdk.RegistrationResponse, error) { + i.Logger.Info("registration requested") + + var request *registrationRequest + if err := sdk.FromConfig(req.Payload, &request); err != nil { + return nil, fmt.Errorf("invalid registration request: %w", err) + } + + // Test the webhook URL and extract some information from it to use it as reference for the user + resp, err := http.Get(request.WebhookURL) + if err != nil { + return nil, fmt.Errorf("invalid webhook URL: %w", err) + } + defer resp.Body.Close() + + var webHookInfo webhookResponse + if err := json.NewDecoder(resp.Body).Decode(&webHookInfo); err != nil { + return nil, fmt.Errorf("invalid webhook URL: %w", err) + } + + // Configuration State + config, err := sdk.ToConfig(®istrationState{ + WebhookName: webHookInfo.Name, + WebhookOwner: webHookInfo.User.Username, + Username: request.Username, + }) + if err != nil { + return nil, fmt.Errorf("marshalling configuration: %w", err) + } + + return &sdk.RegistrationResponse{ + Configuration: config, + // We treat the webhook URL as a sensitive field so we store it in the credentials storage + Credentials: &sdk.Credentials{Password: request.WebhookURL}, + }, nil +} + +// Attachment is executed when to attach a registered instance of this integration to a specific workflow +func (i *Integration) Attach(_ context.Context, _ *sdk.AttachmentRequest) (*sdk.AttachmentResponse, error) { + i.Logger.Info("attachment requested") + return &sdk.AttachmentResponse{}, nil +} + +// Execute will be instantiate when either an attestation or a material has been received +// It's up to the extension builder to differentiate between inputs +func (i *Integration) Execute(_ context.Context, req *sdk.ExecutionRequest) error { + i.Logger.Info("execution requested") + + if err := validateExecuteRequest(req); err != nil { + return fmt.Errorf("running validation: %w", err) + } + + var config *registrationState + if err := sdk.FromConfig(req.RegistrationInfo.Configuration, &config); err != nil { + return fmt.Errorf("invalid registration config: %w", err) + } + + attestationJSON, err := json.MarshalIndent(req.Input.Attestation.Statement, "", " ") + if err != nil { + return fmt.Errorf("error marshaling JSON: %w", err) + } + + webhookURL := req.RegistrationInfo.Credentials.Password + if err := executeWebhook(webhookURL, config.Username, attestationJSON); err != nil { + return fmt.Errorf("error executing webhook: %w", err) + } + + i.Logger.Info("execution finished") + return nil +} + +// Send attestation to Discord + +// https://discord.com/developers/docs/reference#uploading-files +// --boundary +// Content-Disposition: form-data; name="payload_json" +// Content-Type: application/json +// +// { +// "content": "New attestation!", +// "attachments": [{ +// "id": 0, +// "filename": "attestation.json" +// }] +// } +// +// --boundary +// Content-Disposition: form-data; name="files[0]"; filename="statement.json" +// --boundary +func executeWebhook(webhookURL, username string, jsonStatement []byte) error { + var b bytes.Buffer + multipartWriter := multipart.NewWriter(&b) + + // webhook POST payload JSON + payload := payloadJSON{ + Content: "New attestation!", + Username: username, + Attachments: []payloadAttachment{ + { + ID: 0, + Filename: "attestation.json", + }, + }, + } + + payloadJSON, err := json.Marshal(payload) + if err != nil { + return fmt.Errorf("marshalling payload: %w", err) + } + + payloadWriter, err := multipartWriter.CreateFormField("payload_json") + if err != nil { + return fmt.Errorf("creating payload form field: %w", err) + } + + if _, err := payloadWriter.Write(payloadJSON); err != nil { + return fmt.Errorf("writing payload form field: %w", err) + } + + // attach attestation JSON + attachmentWriter, err := multipartWriter.CreateFormFile("files[0]", "statement.json") + if err != nil { + return fmt.Errorf("creating attachment form field: %w", err) + } + + if _, err := attachmentWriter.Write(jsonStatement); err != nil { + return fmt.Errorf("writing attachment form field: %w", err) + } + + // Needed to dump the content of the multipartWriter to the buffer + multipartWriter.Close() + + // #nosec G107 - we are using a constant API URL that is not user input at this stage + r, err := http.Post(webhookURL, multipartWriter.FormDataContentType(), &b) + if err != nil { + return fmt.Errorf("creating request: %w", err) + } + defer r.Body.Close() + + if r.StatusCode != http.StatusOK { + b, _ := ioutil.ReadAll(r.Body) + return fmt.Errorf("non-OK HTTP status while calling the webhook: %d, body: %s", r.StatusCode, string(b)) + } + + return nil +} + +type payloadJSON struct { + Content string `json:"content"` + Username string `json:"username,omitempty"` + Attachments []payloadAttachment `json:"attachments"` +} + +type payloadAttachment struct { + ID int `json:"id"` + Filename string `json:"filename"` +} + +func validateExecuteRequest(req *sdk.ExecutionRequest) error { + if req == nil || req.Input == nil { + return errors.New("execution input not received") + } + + if req.Input.Attestation == nil { + return errors.New("execution input invalid, envelope is nil") + } + + if req.RegistrationInfo == nil || req.RegistrationInfo.Configuration == nil { + return errors.New("missing registration configuration") + } + + if req.RegistrationInfo.Credentials == nil { + return errors.New("missing credentials") + } + + return nil +} diff --git a/app/controlplane/extensions/extensions.go b/app/controlplane/extensions/extensions.go index 5a67f7a9c..fc1f07764 100644 --- a/app/controlplane/extensions/extensions.go +++ b/app/controlplane/extensions/extensions.go @@ -19,6 +19,7 @@ import ( "fmt" "github.com/chainloop-dev/chainloop/app/controlplane/extensions/core/dependencytrack/v1" + "github.com/chainloop-dev/chainloop/app/controlplane/extensions/core/discord/v1" "github.com/chainloop-dev/chainloop/app/controlplane/extensions/core/ociregistry/v1" "github.com/chainloop-dev/chainloop/app/controlplane/extensions/core/smtp/v1" "github.com/chainloop-dev/chainloop/app/controlplane/extensions/sdk/v1" @@ -36,6 +37,7 @@ func Load(l log.Logger) (sdk.AvailableExtensions, error) { dependencytrack.New, smtp.New, ociregistry.New, + discord.New, } return doLoad(toEnable, l) From e67c7711adc92a650b6994e7363483a5555876dc Mon Sep 17 00:00:00 2001 From: Miguel Martinez Trivino Date: Thu, 15 Jun 2023 18:29:18 +0200 Subject: [PATCH 2/6] feat: discord server Signed-off-by: Miguel Martinez Trivino --- app/cli/cmd/attached_integration_list.go | 1 - app/controlplane/cmd/wire_gen.go | 2 +- .../extensions/core/discord/v1/discord.go | 43 +++++- .../core/discord/v1/discord_test.go | 130 ++++++++++++++++++ app/controlplane/extensions/sdk/v1/fanout.go | 4 +- .../internal/dispatcher/dispatcher.go | 27 +++- .../internal/dispatcher/dispatcher_test.go | 2 +- .../renderer/chainloop/chainloop.go | 5 + 8 files changed, 199 insertions(+), 15 deletions(-) create mode 100644 app/controlplane/extensions/core/discord/v1/discord_test.go diff --git a/app/cli/cmd/attached_integration_list.go b/app/cli/cmd/attached_integration_list.go index 32389faa0..4be4604af 100644 --- a/app/cli/cmd/attached_integration_list.go +++ b/app/cli/cmd/attached_integration_list.go @@ -58,7 +58,6 @@ func attachedIntegrationListTableOutput(attachments []*action.AttachedIntegratio integration := attachment.Integration // Merge attachment and integration configs - // If there are not attachment configuration then create an empty map if attachment.Config == nil { attachment.Config = make(map[string]any) diff --git a/app/controlplane/cmd/wire_gen.go b/app/controlplane/cmd/wire_gen.go index 1b905f050..3641d39c8 100644 --- a/app/controlplane/cmd/wire_gen.go +++ b/app/controlplane/cmd/wire_gen.go @@ -97,7 +97,7 @@ func wireApp(bootstrap *conf.Bootstrap, readerWriter credentials.ReaderWriter, l cleanup() return nil, nil, err } - fanOutDispatcher := dispatcher.New(integrationUseCase, readerWriter, casClientUseCase, availableExtensions, logger) + fanOutDispatcher := dispatcher.New(integrationUseCase, workflowUseCase, readerWriter, casClientUseCase, availableExtensions, logger) newAttestationServiceOpts := &service.NewAttestationServiceOpts{ WorkflowRunUC: workflowRunUseCase, WorkflowUC: workflowUseCase, diff --git a/app/controlplane/extensions/core/discord/v1/discord.go b/app/controlplane/extensions/core/discord/v1/discord.go index d5462aab9..56f89256f 100644 --- a/app/controlplane/extensions/core/discord/v1/discord.go +++ b/app/controlplane/extensions/core/discord/v1/discord.go @@ -24,6 +24,8 @@ import ( "io/ioutil" "mime/multipart" "net/http" + "strings" + "text/template" "github.com/chainloop-dev/chainloop/app/controlplane/extensions/sdk/v1" "github.com/go-kratos/kratos/v2/log" @@ -36,7 +38,7 @@ type Integration struct { // 1 - API schema definitions type registrationRequest struct { WebhookURL string `json:"webhook" jsonschema:"format=uri,description=URL of the discord webhook"` - Username string `json:"username,omitempty" jsonschema:"minLenght=1,description=Override the default username of the webhook "` + Username string `json:"username,omitempty" jsonschema:"minLength=1,description=Override the default username of the webhook "` } type attachmentRequest struct{} @@ -143,8 +145,16 @@ func (i *Integration) Execute(_ context.Context, req *sdk.ExecutionRequest) erro return fmt.Errorf("error marshaling JSON: %w", err) } + metadata := req.ChainloopMetadata + tplData := &templateContent{ + WorkflowID: metadata.WorkflowID, + WorkflowName: metadata.WorkflowName, + WorkflowProject: metadata.WorkflowProject, + RunnerLink: req.Input.Attestation.Predicate.GetRunLink(), + } + webhookURL := req.RegistrationInfo.Credentials.Password - if err := executeWebhook(webhookURL, config.Username, attestationJSON); err != nil { + if err := executeWebhook(webhookURL, config.Username, attestationJSON, renderContent(tplData)); err != nil { return fmt.Errorf("error executing webhook: %w", err) } @@ -170,14 +180,14 @@ func (i *Integration) Execute(_ context.Context, req *sdk.ExecutionRequest) erro // --boundary // Content-Disposition: form-data; name="files[0]"; filename="statement.json" // --boundary -func executeWebhook(webhookURL, username string, jsonStatement []byte) error { +func executeWebhook(webhookURL, usernameOverride string, jsonStatement []byte, msgContent string) error { var b bytes.Buffer multipartWriter := multipart.NewWriter(&b) // webhook POST payload JSON payload := payloadJSON{ - Content: "New attestation!", - Username: username, + Content: msgContent, + Username: usernameOverride, Attachments: []payloadAttachment{ { ID: 0, @@ -258,3 +268,26 @@ func validateExecuteRequest(req *sdk.ExecutionRequest) error { return nil } + +type templateContent struct { + WorkflowID, WorkflowName, WorkflowProject, RunnerLink string +} + +func renderContent(metadata *templateContent) string { + t := template.Must(template.New("content").Parse(msgTemplate)) + + var b bytes.Buffer + if err := t.Execute(&b, metadata); err != nil { + return "" + } + + return strings.Trim(b.String(), "\n") +} + +const msgTemplate = ` +New attestation received! +- workflow: {{.WorkflowProject}}/{{.WorkflowName}} +{{- if .RunnerLink }} +- link to run: {{.RunnerLink}} +{{end}} +` diff --git a/app/controlplane/extensions/core/discord/v1/discord_test.go b/app/controlplane/extensions/core/discord/v1/discord_test.go new file mode 100644 index 000000000..02d0d311a --- /dev/null +++ b/app/controlplane/extensions/core/discord/v1/discord_test.go @@ -0,0 +1,130 @@ +// +// Copyright 2023 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package discord + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestValidateRegistrationInput(t *testing.T) { + testCases := []struct { + name string + input map[string]interface{} + errMsg string + }{ + { + name: "not ok, missing required property", + input: map[string]interface{}{}, + errMsg: "missing properties: 'webhook'", + }, + { + name: "not ok, random properties", + input: map[string]interface{}{"foo": "bar"}, + errMsg: "additionalProperties 'foo' not allowed", + }, + { + name: "ok, all properties", + input: map[string]interface{}{"webhook": "http://repo.io", "username": "u"}, + }, + { + name: "ok, only required properties", + input: map[string]interface{}{"webhook": "http://repo.io"}, + }, + { + name: "not ok, empty username", + input: map[string]interface{}{"webhook": "http://repo.io", "username": ""}, + errMsg: "length must be >= 1, but got 0", + }, + { + name: "ok, webhook with path", + input: map[string]interface{}{"webhook": "http://repo/foo/bar"}, + }, + { + name: "not ok, invalid webhook, missing protocol", + input: map[string]interface{}{"webhook": "repo.io"}, + errMsg: "is not valid 'uri'", + }, + { + name: "not ok, empty webhook", + input: map[string]interface{}{"webhook": ""}, + errMsg: "is not valid 'uri'", + }, + } + + integration, err := New(nil) + require.NoError(t, err) + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + payload, err := json.Marshal(tc.input) + require.NoError(t, err) + + err = integration.ValidateRegistrationRequest(payload) + if tc.errMsg != "" { + assert.ErrorContains(t, err, tc.errMsg) + } else { + assert.NoError(t, err) + } + }) + } +} + +func TestRenderContent(t *testing.T) { + testCases := []struct { + name string + input *templateContent + expected string + }{ + { + name: "all fields", + input: &templateContent{ + WorkflowID: "deadbeef", + WorkflowName: "test", + WorkflowProject: "project", + RunnerLink: "http://runner.io", + }, + expected: `New attestation received! +- workflow: project/test +- link to run: http://runner.io`, + }, + { + name: "no runner link", + input: &templateContent{ + WorkflowID: "deadbeef", + WorkflowName: "test", + WorkflowProject: "project", + }, + expected: `New attestation received! +- workflow: project/test`, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + actual := renderContent(tc.input) + assert.Equal(t, tc.expected, actual) + }) + } +} + +func TestNewIntegration(t *testing.T) { + _, err := New(nil) + assert.NoError(t, err) +} diff --git a/app/controlplane/extensions/sdk/v1/fanout.go b/app/controlplane/extensions/sdk/v1/fanout.go index e959a77f3..e9bb28a0e 100644 --- a/app/controlplane/extensions/sdk/v1/fanout.go +++ b/app/controlplane/extensions/sdk/v1/fanout.go @@ -109,7 +109,9 @@ type AttachmentResponse struct { } type ChainloopMetadata struct { - WorkflowID string + WorkflowID string + WorkflowName string + WorkflowProject string } // ExecutionRequest is the request to execute the integration diff --git a/app/controlplane/internal/dispatcher/dispatcher.go b/app/controlplane/internal/dispatcher/dispatcher.go index afadea8e2..d3d7e9098 100644 --- a/app/controlplane/internal/dispatcher/dispatcher.go +++ b/app/controlplane/internal/dispatcher/dispatcher.go @@ -36,6 +36,7 @@ import ( type FanOutDispatcher struct { integrationUC *biz.IntegrationUseCase + wfUC *biz.WorkflowUseCase credentialsProvider credentials.ReaderWriter casClient biz.CASClient log *log.Helper @@ -43,8 +44,8 @@ type FanOutDispatcher struct { loaded sdk.AvailableExtensions } -func New(integrationUC *biz.IntegrationUseCase, creds credentials.ReaderWriter, c biz.CASClient, registered sdk.AvailableExtensions, l log.Logger) *FanOutDispatcher { - return &FanOutDispatcher{integrationUC, creds, c, servicelogger.ScopedHelper(l, "fanout-dispatcher"), l, registered} +func New(integrationUC *biz.IntegrationUseCase, wfUC *biz.WorkflowUseCase, creds credentials.ReaderWriter, c biz.CASClient, registered sdk.AvailableExtensions, l log.Logger) *FanOutDispatcher { + return &FanOutDispatcher{integrationUC, wfUC, creds, c, servicelogger.ScopedHelper(l, "fanout-dispatcher"), l, registered} } type integrationInfo struct { @@ -154,6 +155,20 @@ func (d *FanOutDispatcher) calculateDispatchQueue(ctx context.Context, orgID, wo // Run attestation and materials to the attached integrations func (d *FanOutDispatcher) Run(ctx context.Context, envelope *dsse.Envelope, orgID, workflowID, downloadSecretName string) error { + // Calculate metadata + wf, err := d.wfUC.FindByID(ctx, workflowID) + if err != nil { + return fmt.Errorf("finding workflow: %w", err) + } else if wf == nil { + return fmt.Errorf("workflow not found") + } + + workflowMetadata := &sdk.ChainloopMetadata{ + WorkflowID: workflowID, + WorkflowName: wf.Name, + WorkflowProject: wf.Project, + } + queue, err := d.calculateDispatchQueue(ctx, orgID, workflowID) if err != nil { return fmt.Errorf("calculating dispatch queue: %w", err) @@ -179,7 +194,7 @@ func (d *FanOutDispatcher) Run(ctx context.Context, envelope *dsse.Envelope, org // Send the envelope to the integrations that are subscribed to it for _, integration := range queue.attestations { - req := generateRequest(integration) + req := generateRequest(integration, workflowMetadata) req.Input = &sdk.ExecuteInput{ Attestation: attestationInput, } @@ -231,7 +246,7 @@ func (d *FanOutDispatcher) Run(ctx context.Context, envelope *dsse.Envelope, org // Execute the integration backends for _, b := range backends { - req := generateRequest(b) + req := generateRequest(b, workflowMetadata) req.Input = &sdk.ExecuteInput{ // They receive both the attestation information and the specific material information Material: materialInput, @@ -280,9 +295,9 @@ func dispatch(ctx context.Context, backend sdk.FanOut, opts *sdk.ExecutionReques ) } -func generateRequest(in *integrationInfo) *sdk.ExecutionRequest { +func generateRequest(in *integrationInfo, metadata *sdk.ChainloopMetadata) *sdk.ExecutionRequest { return &sdk.ExecutionRequest{ - ChainloopMetadata: &sdk.ChainloopMetadata{WorkflowID: in.workflowID}, + ChainloopMetadata: metadata, RegistrationInfo: &sdk.RegistrationResponse{ Credentials: in.credentials, Configuration: in.registrationConfig, diff --git a/app/controlplane/internal/dispatcher/dispatcher_test.go b/app/controlplane/internal/dispatcher/dispatcher_test.go index f81d60cab..76cb34a28 100644 --- a/app/controlplane/internal/dispatcher/dispatcher_test.go +++ b/app/controlplane/internal/dispatcher/dispatcher_test.go @@ -227,7 +227,7 @@ func (s *dispatcherTestSuite) SetupTest() { // Register the integrations in the dispatcher registeredIntegrations := sdk.AvailableExtensions{s.cdxIntegrationBackend, s.anyIntegrationBackend, s.ociIntegrationBackend} - s.dispatcher = New(s.Integration, nil, mocks.NewCASClient(s.T()), registeredIntegrations, s.L) + s.dispatcher = New(s.Integration, nil, nil, mocks.NewCASClient(s.T()), registeredIntegrations, s.L) } type mockedIntegration struct { diff --git a/internal/attestation/renderer/chainloop/chainloop.go b/internal/attestation/renderer/chainloop/chainloop.go index 875524844..db05ce8df 100644 --- a/internal/attestation/renderer/chainloop/chainloop.go +++ b/internal/attestation/renderer/chainloop/chainloop.go @@ -37,6 +37,7 @@ const builderIDFmt = "chainloop.dev/cli/%s@%s" type NormalizablePredicate interface { GetEnvVars() map[string]string GetMaterials() []*NormalizedMaterial + GetRunLink() string } type NormalizedMaterial struct { @@ -181,3 +182,7 @@ func extractPredicate(statement *in_toto.Statement, v any) error { func (p *ProvenancePredicateCommon) GetEnvVars() map[string]string { return p.Env } + +func (p *ProvenancePredicateCommon) GetRunLink() string { + return p.RunnerURL +} From 75c1439f0cb1df89ef23aa532bc60eeddd6d7207 Mon Sep 17 00:00:00 2001 From: Miguel Martinez Trivino Date: Thu, 15 Jun 2023 18:46:34 +0200 Subject: [PATCH 3/6] feat: discord server Signed-off-by: Miguel Martinez Trivino --- .../extensions/core/discord/v1/README.md | 26 ++++++++++--------- .../extensions/core/discord/v1/discord.go | 4 +-- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/app/controlplane/extensions/core/discord/v1/README.md b/app/controlplane/extensions/core/discord/v1/README.md index d702fd1ef..38f0e46b5 100644 --- a/app/controlplane/extensions/core/discord/v1/README.md +++ b/app/controlplane/extensions/core/discord/v1/README.md @@ -1,20 +1,22 @@ -# Discord extension +# Discord Webhook Extension -You can use this template as a placeholder to create your own fan-out extension. +Send attestations to Discord using webhooks. ## How to use it -These are the required steps +1. To get started, you need to register the extension in your Chainloop organization. -### Setup: +```console +$ chainloop integration registered add discord-webhook --opt webhook=[webhookURL] +``` -- Copy and rename the folder to your extension name -- Replace all the occurrences of `template` with your extension name +optionally you can specify a custom username -### Implementation +```console +$ chainloop integration registered add discord-webhook --opt webhook=[webhookURL] --opt username=[username] +``` -- Define the API request payloads for both Registration and Attachment -- Implement the [FanOutExtension interface](https://github.com/chainloop-dev/chainloop/blob/main/app/controlplane/extensions/sdk/v1/fanout.go#L55). The template comes prefilled with some commented out code. +2. Attach the integration to your workflow. -### Enable extension for - -- Add it to the list of available extensions [here](`../../../../../extensions.go`). This will make this extension available the next time the control plane starts. \ No newline at end of file +```console +chainloop integration attached add --workflow $WID --integration $IID +``` \ No newline at end of file diff --git a/app/controlplane/extensions/core/discord/v1/discord.go b/app/controlplane/extensions/core/discord/v1/discord.go index 56f89256f..f6c29e618 100644 --- a/app/controlplane/extensions/core/discord/v1/discord.go +++ b/app/controlplane/extensions/core/discord/v1/discord.go @@ -21,7 +21,7 @@ import ( "encoding/json" "errors" "fmt" - "io/ioutil" + "io" "mime/multipart" "net/http" "strings" @@ -231,7 +231,7 @@ func executeWebhook(webhookURL, usernameOverride string, jsonStatement []byte, m defer r.Body.Close() if r.StatusCode != http.StatusOK { - b, _ := ioutil.ReadAll(r.Body) + b, _ := io.ReadAll(r.Body) return fmt.Errorf("non-OK HTTP status while calling the webhook: %d, body: %s", r.StatusCode, string(b)) } From 0be94a2c3360959c47ec2b6395d19af4c5a76fe0 Mon Sep 17 00:00:00 2001 From: Miguel Martinez Trivino Date: Thu, 15 Jun 2023 19:00:07 +0200 Subject: [PATCH 4/6] feat: add workflowRunID Signed-off-by: Miguel Martinez Trivino --- .../extensions/core/discord/v1/discord.go | 8 +++-- .../core/discord/v1/discord_test.go | 12 ++++--- app/controlplane/extensions/sdk/v1/fanout.go | 2 ++ .../internal/dispatcher/dispatcher.go | 31 ++++++++++++------- .../internal/service/attestation.go | 4 ++- 5 files changed, 37 insertions(+), 20 deletions(-) diff --git a/app/controlplane/extensions/core/discord/v1/discord.go b/app/controlplane/extensions/core/discord/v1/discord.go index f6c29e618..ee9490303 100644 --- a/app/controlplane/extensions/core/discord/v1/discord.go +++ b/app/controlplane/extensions/core/discord/v1/discord.go @@ -149,6 +149,7 @@ func (i *Integration) Execute(_ context.Context, req *sdk.ExecutionRequest) erro tplData := &templateContent{ WorkflowID: metadata.WorkflowID, WorkflowName: metadata.WorkflowName, + WorkflowRunID: metadata.WorkflowRunID, WorkflowProject: metadata.WorkflowProject, RunnerLink: req.Input.Attestation.Predicate.GetRunLink(), } @@ -270,7 +271,7 @@ func validateExecuteRequest(req *sdk.ExecutionRequest) error { } type templateContent struct { - WorkflowID, WorkflowName, WorkflowProject, RunnerLink string + WorkflowID, WorkflowName, WorkflowProject, WorkflowRunID, RunnerLink string } func renderContent(metadata *templateContent) string { @@ -286,8 +287,9 @@ func renderContent(metadata *templateContent) string { const msgTemplate = ` New attestation received! -- workflow: {{.WorkflowProject}}/{{.WorkflowName}} +- Workflow: {{.WorkflowProject}}/{{.WorkflowName}} +- Workflow Run: {{.WorkflowRunID}} {{- if .RunnerLink }} -- link to run: {{.RunnerLink}} +- Link to runner: {{.RunnerLink}} {{end}} ` diff --git a/app/controlplane/extensions/core/discord/v1/discord_test.go b/app/controlplane/extensions/core/discord/v1/discord_test.go index 02d0d311a..eb514ba12 100644 --- a/app/controlplane/extensions/core/discord/v1/discord_test.go +++ b/app/controlplane/extensions/core/discord/v1/discord_test.go @@ -95,24 +95,26 @@ func TestRenderContent(t *testing.T) { { name: "all fields", input: &templateContent{ - WorkflowID: "deadbeef", + WorkflowRunID: "deadbeef", WorkflowName: "test", WorkflowProject: "project", RunnerLink: "http://runner.io", }, expected: `New attestation received! -- workflow: project/test -- link to run: http://runner.io`, +- Workflow: project/test +- Workflow Run: deadbeef +- Link to runner: http://runner.io`, }, { name: "no runner link", input: &templateContent{ - WorkflowID: "deadbeef", + WorkflowRunID: "deadbeef", WorkflowName: "test", WorkflowProject: "project", }, expected: `New attestation received! -- workflow: project/test`, +- Workflow: project/test +- Workflow Run: deadbeef`, }, } diff --git a/app/controlplane/extensions/sdk/v1/fanout.go b/app/controlplane/extensions/sdk/v1/fanout.go index e9bb28a0e..b91fb1061 100644 --- a/app/controlplane/extensions/sdk/v1/fanout.go +++ b/app/controlplane/extensions/sdk/v1/fanout.go @@ -112,6 +112,8 @@ type ChainloopMetadata struct { WorkflowID string WorkflowName string WorkflowProject string + + WorkflowRunID string } // ExecutionRequest is the request to execute the integration diff --git a/app/controlplane/internal/dispatcher/dispatcher.go b/app/controlplane/internal/dispatcher/dispatcher.go index d3d7e9098..375f8bee1 100644 --- a/app/controlplane/internal/dispatcher/dispatcher.go +++ b/app/controlplane/internal/dispatcher/dispatcher.go @@ -153,10 +153,18 @@ func (d *FanOutDispatcher) calculateDispatchQueue(ctx context.Context, orgID, wo return &dispatchQueue{materials: materialDispatch, attestations: attestationDispatch}, nil } +type DispatcherOpts struct { + Envelope *dsse.Envelope + OrgID string + WorkflowID string + WorkflowRunID string + DownloadSecretName string +} + // Run attestation and materials to the attached integrations -func (d *FanOutDispatcher) Run(ctx context.Context, envelope *dsse.Envelope, orgID, workflowID, downloadSecretName string) error { +func (d *FanOutDispatcher) Run(ctx context.Context, opts *DispatcherOpts) error { // Calculate metadata - wf, err := d.wfUC.FindByID(ctx, workflowID) + wf, err := d.wfUC.FindByID(ctx, opts.WorkflowID) if err != nil { return fmt.Errorf("finding workflow: %w", err) } else if wf == nil { @@ -164,30 +172,31 @@ func (d *FanOutDispatcher) Run(ctx context.Context, envelope *dsse.Envelope, org } workflowMetadata := &sdk.ChainloopMetadata{ - WorkflowID: workflowID, + WorkflowID: opts.WorkflowID, + WorkflowRunID: opts.WorkflowRunID, WorkflowName: wf.Name, WorkflowProject: wf.Project, } - queue, err := d.calculateDispatchQueue(ctx, orgID, workflowID) + queue, err := d.calculateDispatchQueue(ctx, opts.OrgID, opts.WorkflowID) if err != nil { return fmt.Errorf("calculating dispatch queue: %w", err) } // get the in_toto statement from the envelope if present - statement, err := chainloop.ExtractStatement(envelope) + statement, err := chainloop.ExtractStatement(opts.Envelope) if err != nil { return fmt.Errorf("extracting statement: %w", err) } // Iterate over the materials in the attestation and dispatch them to the integrations that are subscribed to them - predicate, err := chainloop.ExtractPredicate(envelope) + predicate, err := chainloop.ExtractPredicate(opts.Envelope) if err != nil { return fmt.Errorf("extracting predicate: %w", err) } var attestationInput = &sdk.ExecuteAttestation{ - Envelope: envelope, + Envelope: opts.Envelope, Statement: statement, Predicate: predicate, } @@ -222,16 +231,16 @@ func (d *FanOutDispatcher) Run(ctx context.Context, envelope *dsse.Envelope, org continue } - d.log.Infow("msg", fmt.Sprintf("%d integrations found for this material type", len(backends)), "workflowID", workflowID, "materialType", material.Type, "name", material.Name) + d.log.Infow("msg", fmt.Sprintf("%d integrations found for this material type", len(backends)), "workflowID", opts.WorkflowID, "materialType", material.Type, "name", material.Name) // Retrieve material content content := []byte(material.Value) // It's a downloadable so we retrieve and override the content variable if material.Hash != nil && material.UploadedToCAS { digest := material.Hash.String() - d.log.Infow("msg", "downloading material", "workflowID", workflowID, "materialType", material.Type, "name", material.Name) + d.log.Infow("msg", "downloading material", "workflowID", opts.WorkflowID, "materialType", material.Type, "name", material.Name) buf := bytes.NewBuffer(nil) - if err := d.casClient.Download(ctx, downloadSecretName, buf, digest); err != nil { + if err := d.casClient.Download(ctx, opts.DownloadSecretName, buf, digest); err != nil { return fmt.Errorf("downloading from CAS: %w", err) } @@ -257,7 +266,7 @@ func (d *FanOutDispatcher) Run(ctx context.Context, envelope *dsse.Envelope, org _ = dispatch(ctx, b.backend, req, d.log) }() - d.log.Infow("msg", "integration executed!", "workflowID", workflowID, "materialType", material.Type, "integration", b.backend.Describe().ID) + d.log.Infow("msg", "integration executed!", "workflowID", opts.WorkflowID, "materialType", material.Type, "integration", b.backend.Describe().ID) } } diff --git a/app/controlplane/internal/service/attestation.go b/app/controlplane/internal/service/attestation.go index 05b3246ea..b3b554d90 100644 --- a/app/controlplane/internal/service/attestation.go +++ b/app/controlplane/internal/service/attestation.go @@ -206,7 +206,9 @@ func (s *AttestationService) Store(ctx context.Context, req *cpAPI.AttestationSe }() go func() { - if err := s.integrationDispatcher.Run(context.TODO(), envelope, robotAccount.OrgID, robotAccount.WorkflowID, repo.SecretName); err != nil { + if err := s.integrationDispatcher.Run(context.TODO(), &dispatcher.DispatcherOpts{ + Envelope: envelope, OrgID: robotAccount.OrgID, WorkflowID: robotAccount.WorkflowID, DownloadSecretName: repo.SecretName, WorkflowRunID: req.WorkflowRunId, + }); err != nil { _ = sl.LogAndMaskErr(err, s.log) } }() From cc28b49957148fcf2c1353e91f710601444c383e Mon Sep 17 00:00:00 2001 From: Miguel Martinez Trivino Date: Thu, 15 Jun 2023 19:05:07 +0200 Subject: [PATCH 5/6] feat: add workflowRunID Signed-off-by: Miguel Martinez Trivino --- app/controlplane/internal/dispatcher/dispatcher.go | 4 ++-- app/controlplane/internal/service/attestation.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/controlplane/internal/dispatcher/dispatcher.go b/app/controlplane/internal/dispatcher/dispatcher.go index 375f8bee1..7b7811ab5 100644 --- a/app/controlplane/internal/dispatcher/dispatcher.go +++ b/app/controlplane/internal/dispatcher/dispatcher.go @@ -153,7 +153,7 @@ func (d *FanOutDispatcher) calculateDispatchQueue(ctx context.Context, orgID, wo return &dispatchQueue{materials: materialDispatch, attestations: attestationDispatch}, nil } -type DispatcherOpts struct { +type RunOpts struct { Envelope *dsse.Envelope OrgID string WorkflowID string @@ -162,7 +162,7 @@ type DispatcherOpts struct { } // Run attestation and materials to the attached integrations -func (d *FanOutDispatcher) Run(ctx context.Context, opts *DispatcherOpts) error { +func (d *FanOutDispatcher) Run(ctx context.Context, opts *RunOpts) error { // Calculate metadata wf, err := d.wfUC.FindByID(ctx, opts.WorkflowID) if err != nil { diff --git a/app/controlplane/internal/service/attestation.go b/app/controlplane/internal/service/attestation.go index b3b554d90..8ed450363 100644 --- a/app/controlplane/internal/service/attestation.go +++ b/app/controlplane/internal/service/attestation.go @@ -206,7 +206,7 @@ func (s *AttestationService) Store(ctx context.Context, req *cpAPI.AttestationSe }() go func() { - if err := s.integrationDispatcher.Run(context.TODO(), &dispatcher.DispatcherOpts{ + if err := s.integrationDispatcher.Run(context.TODO(), &dispatcher.RunOpts{ Envelope: envelope, OrgID: robotAccount.OrgID, WorkflowID: robotAccount.WorkflowID, DownloadSecretName: repo.SecretName, WorkflowRunID: req.WorkflowRunId, }); err != nil { _ = sl.LogAndMaskErr(err, s.log) From e2b377c0232f74089c26aaf0612787015cc690a1 Mon Sep 17 00:00:00 2001 From: Miguel Martinez Trivino Date: Fri, 16 Jun 2023 08:15:31 +0200 Subject: [PATCH 6/6] fix: merge options Signed-off-by: Miguel Martinez Trivino --- app/cli/cmd/attached_integration_list.go | 8 ++++---- app/cli/cmd/available_integration_list.go | 2 -- app/controlplane/extensions/core/discord/v1/discord.go | 2 +- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/app/cli/cmd/attached_integration_list.go b/app/cli/cmd/attached_integration_list.go index 4be4604af..b10de5cde 100644 --- a/app/cli/cmd/attached_integration_list.go +++ b/app/cli/cmd/attached_integration_list.go @@ -57,8 +57,8 @@ func attachedIntegrationListTableOutput(attachments []*action.AttachedIntegratio wf := attachment.Workflow integration := attachment.Integration - // Merge attachment and integration configs - // If there are not attachment configuration then create an empty map + // Merge attachment and integration configs to show them in the same table + // If the same key exists in both configs, the value in attachment config will be used if attachment.Config == nil { attachment.Config = make(map[string]any) } @@ -68,10 +68,10 @@ func attachedIntegrationListTableOutput(attachments []*action.AttachedIntegratio } var options []string - maps.Copy(attachment.Config, integration.Config) + maps.Copy(integration.Config, attachment.Config) // Show it as key-value pairs - for k, v := range attachment.Config { + for k, v := range integration.Config { if v == "" { continue } diff --git a/app/cli/cmd/available_integration_list.go b/app/cli/cmd/available_integration_list.go index 1053d9356..8096729df 100644 --- a/app/cli/cmd/available_integration_list.go +++ b/app/cli/cmd/available_integration_list.go @@ -47,8 +47,6 @@ func availableIntegrationListTableOutput(items []*action.AvailableIntegrationIte return nil } - fmt.Println("Available integrations ready to be used during registration") - t := newTableWriter() t.AppendHeader(table.Row{"ID", "Version", "Description"}) for _, i := range items { diff --git a/app/controlplane/extensions/core/discord/v1/discord.go b/app/controlplane/extensions/core/discord/v1/discord.go index ee9490303..b21ce77f2 100644 --- a/app/controlplane/extensions/core/discord/v1/discord.go +++ b/app/controlplane/extensions/core/discord/v1/discord.go @@ -50,7 +50,7 @@ type registrationState struct { WebhookOwner string `json:"owner"` // Username to be used while posting the message - Username string `json:"username"` + Username string `json:"username,omitempty"` } func New(l log.Logger) (sdk.FanOut, error) {