From 9e698b250f038677648560ed2344ef39d5ec19fe Mon Sep 17 00:00:00 2001 From: Miguel Martinez Trivino Date: Tue, 20 Jun 2023 20:00:48 +0200 Subject: [PATCH 01/12] feat: test generator Signed-off-by: Miguel Martinez Trivino --- .../v1/README.md | 0 .../v1/client/sbom.go | 0 .../v1/client/sbom_test.go | 0 .../v1/extension.go | 2 +- .../v1/extension_test.go | 0 .../{discord => discord-webhook}/v1/README.md | 6 +- .../v1/discord.go | 2 +- .../v1/discord_test.go | 0 .../v1/README.md | 0 .../v1/ociregistry.go | 0 .../v1/ociregistry_test.go | 0 .../extensions/core/template/v1/README.md | 5 +- app/controlplane/extensions/extensions.go | 6 +- .../extensions/sdk/readme-generator/main.go | 77 +++++++++++++++++++ 14 files changed, 91 insertions(+), 7 deletions(-) rename app/controlplane/extensions/core/{dependencytrack => dependency-track}/v1/README.md (100%) rename app/controlplane/extensions/core/{dependencytrack => dependency-track}/v1/client/sbom.go (100%) rename app/controlplane/extensions/core/{dependencytrack => dependency-track}/v1/client/sbom_test.go (100%) rename app/controlplane/extensions/core/{dependencytrack => dependency-track}/v1/extension.go (99%) rename app/controlplane/extensions/core/{dependencytrack => dependency-track}/v1/extension_test.go (100%) rename app/controlplane/extensions/core/{discord => discord-webhook}/v1/README.md (90%) rename app/controlplane/extensions/core/{discord => discord-webhook}/v1/discord.go (99%) rename app/controlplane/extensions/core/{discord => discord-webhook}/v1/discord_test.go (100%) rename app/controlplane/extensions/core/{ociregistry => oci-registry}/v1/README.md (100%) rename app/controlplane/extensions/core/{ociregistry => oci-registry}/v1/ociregistry.go (100%) rename app/controlplane/extensions/core/{ociregistry => oci-registry}/v1/ociregistry_test.go (100%) create mode 100644 app/controlplane/extensions/sdk/readme-generator/main.go diff --git a/app/controlplane/extensions/core/dependencytrack/v1/README.md b/app/controlplane/extensions/core/dependency-track/v1/README.md similarity index 100% rename from app/controlplane/extensions/core/dependencytrack/v1/README.md rename to app/controlplane/extensions/core/dependency-track/v1/README.md diff --git a/app/controlplane/extensions/core/dependencytrack/v1/client/sbom.go b/app/controlplane/extensions/core/dependency-track/v1/client/sbom.go similarity index 100% rename from app/controlplane/extensions/core/dependencytrack/v1/client/sbom.go rename to app/controlplane/extensions/core/dependency-track/v1/client/sbom.go diff --git a/app/controlplane/extensions/core/dependencytrack/v1/client/sbom_test.go b/app/controlplane/extensions/core/dependency-track/v1/client/sbom_test.go similarity index 100% rename from app/controlplane/extensions/core/dependencytrack/v1/client/sbom_test.go rename to app/controlplane/extensions/core/dependency-track/v1/client/sbom_test.go diff --git a/app/controlplane/extensions/core/dependencytrack/v1/extension.go b/app/controlplane/extensions/core/dependency-track/v1/extension.go similarity index 99% rename from app/controlplane/extensions/core/dependencytrack/v1/extension.go rename to app/controlplane/extensions/core/dependency-track/v1/extension.go index 8b130bab3..20eaf54bc 100644 --- a/app/controlplane/extensions/core/dependencytrack/v1/extension.go +++ b/app/controlplane/extensions/core/dependency-track/v1/extension.go @@ -62,7 +62,7 @@ const description = "Send CycloneDX SBOMs to your Dependency-Track instance" func New(l log.Logger) (sdk.FanOut, error) { base, err := sdk.NewFanOut( &sdk.NewParams{ - ID: "dependencytrack", + ID: "dependency-track", Version: "0.2", Description: description, Logger: l, diff --git a/app/controlplane/extensions/core/dependencytrack/v1/extension_test.go b/app/controlplane/extensions/core/dependency-track/v1/extension_test.go similarity index 100% rename from app/controlplane/extensions/core/dependencytrack/v1/extension_test.go rename to app/controlplane/extensions/core/dependency-track/v1/extension_test.go diff --git a/app/controlplane/extensions/core/discord/v1/README.md b/app/controlplane/extensions/core/discord-webhook/v1/README.md similarity index 90% rename from app/controlplane/extensions/core/discord/v1/README.md rename to app/controlplane/extensions/core/discord-webhook/v1/README.md index 38f0e46b5..d641e49fc 100644 --- a/app/controlplane/extensions/core/discord/v1/README.md +++ b/app/controlplane/extensions/core/discord-webhook/v1/README.md @@ -19,4 +19,8 @@ $ chainloop integration registered add discord-webhook --opt webhook=[webhookURL ```console chainloop integration attached add --workflow $WID --integration $IID -``` \ No newline at end of file +``` + +## Registration Input Schema + +## Attachment Input Schema \ No newline at end of file diff --git a/app/controlplane/extensions/core/discord/v1/discord.go b/app/controlplane/extensions/core/discord-webhook/v1/discord.go similarity index 99% rename from app/controlplane/extensions/core/discord/v1/discord.go rename to app/controlplane/extensions/core/discord-webhook/v1/discord.go index b21ce77f2..378931ddc 100644 --- a/app/controlplane/extensions/core/discord/v1/discord.go +++ b/app/controlplane/extensions/core/discord-webhook/v1/discord.go @@ -38,7 +38,7 @@ type Integration struct { // 1 - API schema definitions type registrationRequest struct { WebhookURL string `json:"webhook" jsonschema:"format=uri,description=URL of the discord webhook"` - Username string `json:"username,omitempty" jsonschema:"minLength=1,description=Override the default username of the webhook "` + Username string `json:"username,omitempty" jsonschema:"minLength=1,description=Override the default username of the webhook"` } type attachmentRequest struct{} diff --git a/app/controlplane/extensions/core/discord/v1/discord_test.go b/app/controlplane/extensions/core/discord-webhook/v1/discord_test.go similarity index 100% rename from app/controlplane/extensions/core/discord/v1/discord_test.go rename to app/controlplane/extensions/core/discord-webhook/v1/discord_test.go diff --git a/app/controlplane/extensions/core/ociregistry/v1/README.md b/app/controlplane/extensions/core/oci-registry/v1/README.md similarity index 100% rename from app/controlplane/extensions/core/ociregistry/v1/README.md rename to app/controlplane/extensions/core/oci-registry/v1/README.md diff --git a/app/controlplane/extensions/core/ociregistry/v1/ociregistry.go b/app/controlplane/extensions/core/oci-registry/v1/ociregistry.go similarity index 100% rename from app/controlplane/extensions/core/ociregistry/v1/ociregistry.go rename to app/controlplane/extensions/core/oci-registry/v1/ociregistry.go diff --git a/app/controlplane/extensions/core/ociregistry/v1/ociregistry_test.go b/app/controlplane/extensions/core/oci-registry/v1/ociregistry_test.go similarity index 100% rename from app/controlplane/extensions/core/ociregistry/v1/ociregistry_test.go rename to app/controlplane/extensions/core/oci-registry/v1/ociregistry_test.go diff --git a/app/controlplane/extensions/core/template/v1/README.md b/app/controlplane/extensions/core/template/v1/README.md index 44e301a8a..7772d129e 100644 --- a/app/controlplane/extensions/core/template/v1/README.md +++ b/app/controlplane/extensions/core/template/v1/README.md @@ -14,4 +14,7 @@ These are the required steps ### Implementation - Define the API request payloads for both Registration and Attachment -- Implement the [FanOutExtension interface](https://github.com/chainloop-dev/chainloop/blob/main/app/controlplane/extensions/sdk/v1/fanout.go#L55). The template comes prefilled with some commented out code. \ No newline at end of file +- Implement the [FanOutExtension interface](https://github.com/chainloop-dev/chainloop/blob/main/app/controlplane/extensions/sdk/v1/fanout.go#L55). The template comes prefilled with some commented out code. + +## Registration Input Schema +## Attachment Input Schema diff --git a/app/controlplane/extensions/extensions.go b/app/controlplane/extensions/extensions.go index fc1f07764..0518ff007 100644 --- a/app/controlplane/extensions/extensions.go +++ b/app/controlplane/extensions/extensions.go @@ -18,9 +18,9 @@ package extensions import ( "fmt" - "github.com/chainloop-dev/chainloop/app/controlplane/extensions/core/dependencytrack/v1" - "github.com/chainloop-dev/chainloop/app/controlplane/extensions/core/discord/v1" - "github.com/chainloop-dev/chainloop/app/controlplane/extensions/core/ociregistry/v1" + dependencytrack "github.com/chainloop-dev/chainloop/app/controlplane/extensions/core/dependency-track/v1" + "github.com/chainloop-dev/chainloop/app/controlplane/extensions/core/discord-webhook/v1" + ociregistry "github.com/chainloop-dev/chainloop/app/controlplane/extensions/core/oci-registry/v1" "github.com/chainloop-dev/chainloop/app/controlplane/extensions/core/smtp/v1" "github.com/chainloop-dev/chainloop/app/controlplane/extensions/sdk/v1" "github.com/chainloop-dev/chainloop/internal/servicelogger" diff --git a/app/controlplane/extensions/sdk/readme-generator/main.go b/app/controlplane/extensions/sdk/readme-generator/main.go new file mode 100644 index 000000000..75629337d --- /dev/null +++ b/app/controlplane/extensions/sdk/readme-generator/main.go @@ -0,0 +1,77 @@ +// +// Copyright 2023 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "encoding/json" + "io" + "os" + "path/filepath" + "regexp" + + extensionsSDK "github.com/chainloop-dev/chainloop/app/controlplane/extensions" + "github.com/go-kratos/kratos/v2/log" +) + +const registrationInputHeader = "## Registration Input Schema" + +var registrationInputRe = regexp.MustCompile(registrationInputHeader) + +func main() { + l := log.NewStdLogger(os.Stdout) + + extensions, err := extensionsSDK.Load(l) + if err != nil { + panic(err) + } + + for _, e := range extensions { + + // Find README file + + file, err := os.OpenFile(filepath.Join("../../core", e.Describe().ID, "v1", "README.md"), os.O_RDWR, 0644) + if err != nil { + _ = l.Log(log.LevelWarn, "msg", "failed to open README.md file", "err", err) + continue + } + + fileContent, err := io.ReadAll(file) + if err != nil { + _ = l.Log(log.LevelWarn, "msg", "failed to read README.md file", "err", err) + continue + } + + // Replace registration input schema + var prettyRegistrationJSON bytes.Buffer + err = json.Indent(&prettyRegistrationJSON, e.Describe().RegistrationJSONSchema, "", " ") + if err != nil { + _ = l.Log(log.LevelWarn, "msg", "failed to indent JSON", "err", err) + } + + newContent := registrationInputRe.ReplaceAllLiteral(fileContent, []byte(registrationInputHeader+"\n```json\n"+prettyRegistrationJSON.String()+"\n```")) + + // Replace attachment schema + + var prettyAttachmentJSON bytes.Buffer + err = json.Indent(&prettyAttachmentJSON, e.Describe().AttachmentJSONSchema, "", " ") + if err != nil { + panic(err) + } + + // fmt.Println(prettyAttachmentJSON.String()) + } +} From 7651065fa36bf13b45db7f7f6635001544f82f89 Mon Sep 17 00:00:00 2001 From: Miguel Martinez Trivino Date: Tue, 20 Jun 2023 20:44:52 +0200 Subject: [PATCH 02/12] update readme Signed-off-by: Miguel Martinez Trivino --- .../core/dependency-track/v1/extension.go | 2 +- .../core/discord-webhook/v1/README.md | 50 ++++++++++++++++++- .../extensions/sdk/readme-generator/main.go | 31 +++++++++--- 3 files changed, 75 insertions(+), 8 deletions(-) diff --git a/app/controlplane/extensions/core/dependency-track/v1/extension.go b/app/controlplane/extensions/core/dependency-track/v1/extension.go index 20eaf54bc..fd97e2fab 100644 --- a/app/controlplane/extensions/core/dependency-track/v1/extension.go +++ b/app/controlplane/extensions/core/dependency-track/v1/extension.go @@ -22,7 +22,7 @@ import ( "fmt" schemaapi "github.com/chainloop-dev/chainloop/app/controlplane/api/workflowcontract/v1" - "github.com/chainloop-dev/chainloop/app/controlplane/extensions/core/dependencytrack/v1/client" + "github.com/chainloop-dev/chainloop/app/controlplane/extensions/core/dependency-track/v1/client" "github.com/chainloop-dev/chainloop/app/controlplane/extensions/sdk/v1" "github.com/go-kratos/kratos/v2/log" ) diff --git a/app/controlplane/extensions/core/discord-webhook/v1/README.md b/app/controlplane/extensions/core/discord-webhook/v1/README.md index d641e49fc..03e06ba78 100644 --- a/app/controlplane/extensions/core/discord-webhook/v1/README.md +++ b/app/controlplane/extensions/core/discord-webhook/v1/README.md @@ -23,4 +23,52 @@ chainloop integration attached add --workflow $WID --integration $IID ## Registration Input Schema -## Attachment Input Schema \ No newline at end of file +```json +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/chainloop-dev/chainloop/app/controlplane/extensions/core/discord-webhook/v1/registration-request", + "properties": { + "webhook": { + "type": "string", + "format": "uri", + "description": "URL of the discord webhook" + }, + "username": { + "type": "string", + "minLength": 1, + "description": "Override the default username of the webhook" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "webhook" + ] +} +``` + +## Attachment Input Schema + +```json +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/chainloop-dev/chainloop/app/controlplane/extensions/core/discord-webhook/v1/registration-request", + "properties": { + "webhook": { + "type": "string", + "format": "uri", + "description": "URL of the discord webhook" + }, + "username": { + "type": "string", + "minLength": 1, + "description": "Override the default username of the webhook" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "webhook" + ] +} +``` \ No newline at end of file diff --git a/app/controlplane/extensions/sdk/readme-generator/main.go b/app/controlplane/extensions/sdk/readme-generator/main.go index 75629337d..f9fdacbb7 100644 --- a/app/controlplane/extensions/sdk/readme-generator/main.go +++ b/app/controlplane/extensions/sdk/readme-generator/main.go @@ -18,6 +18,7 @@ package main import ( "bytes" "encoding/json" + "flag" "io" "os" "path/filepath" @@ -28,8 +29,12 @@ import ( ) const registrationInputHeader = "## Registration Input Schema" +const AttachmentInputHeader = "## Attachment Input Schema" var registrationInputRe = regexp.MustCompile(registrationInputHeader) +var attachmentInputRe = regexp.MustCompile(AttachmentInputHeader) + +var extensionsDir string func main() { l := log.NewStdLogger(os.Stdout) @@ -40,10 +45,8 @@ func main() { } for _, e := range extensions { - // Find README file - - file, err := os.OpenFile(filepath.Join("../../core", e.Describe().ID, "v1", "README.md"), os.O_RDWR, 0644) + file, err := os.OpenFile(filepath.Join(extensionsDir, e.Describe().ID, "v1", "README.md"), os.O_RDWR, 0644) if err != nil { _ = l.Log(log.LevelWarn, "msg", "failed to open README.md file", "err", err) continue @@ -62,8 +65,7 @@ func main() { _ = l.Log(log.LevelWarn, "msg", "failed to indent JSON", "err", err) } - newContent := registrationInputRe.ReplaceAllLiteral(fileContent, []byte(registrationInputHeader+"\n```json\n"+prettyRegistrationJSON.String()+"\n```")) - + fileContent = registrationInputRe.ReplaceAllLiteral(fileContent, []byte(registrationInputHeader+"\n\n```json\n"+prettyRegistrationJSON.String()+"\n```")) // Replace attachment schema var prettyAttachmentJSON bytes.Buffer @@ -72,6 +74,23 @@ func main() { panic(err) } - // fmt.Println(prettyAttachmentJSON.String()) + fileContent = attachmentInputRe.ReplaceAllLiteral(fileContent, []byte(AttachmentInputHeader+"\n\n```json\n"+prettyRegistrationJSON.String()+"\n```")) + // Write the new content in the file + _, err = file.Seek(0, 0) + if err != nil { + _ = l.Log(log.LevelWarn, "msg", "failed to seek README.md file", "err", err) + continue + } + + _, err = file.Write(fileContent) + if err != nil { + _ = l.Log(log.LevelWarn, "msg", "failed to write README.md file", "err", err) + continue + } } } + +func init() { + flag.StringVar(&extensionsDir, "dir", "", "base directory for extensions i.e ./core") + flag.Parse() +} From 56ffa521f2dfdbc4bb208021cd7e566e6adfa038 Mon Sep 17 00:00:00 2001 From: Miguel Martinez Trivino Date: Tue, 20 Jun 2023 20:45:28 +0200 Subject: [PATCH 03/12] update readme Signed-off-by: Miguel Martinez Trivino --- app/controlplane/extensions/sdk/readme-generator/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controlplane/extensions/sdk/readme-generator/main.go b/app/controlplane/extensions/sdk/readme-generator/main.go index f9fdacbb7..c213a827d 100644 --- a/app/controlplane/extensions/sdk/readme-generator/main.go +++ b/app/controlplane/extensions/sdk/readme-generator/main.go @@ -74,7 +74,7 @@ func main() { panic(err) } - fileContent = attachmentInputRe.ReplaceAllLiteral(fileContent, []byte(AttachmentInputHeader+"\n\n```json\n"+prettyRegistrationJSON.String()+"\n```")) + fileContent = attachmentInputRe.ReplaceAllLiteral(fileContent, []byte(AttachmentInputHeader+"\n\n```json\n"+prettyAttachmentJSON.String()+"\n```")) // Write the new content in the file _, err = file.Seek(0, 0) if err != nil { From 56b21a5917b2d351d40d460887df8abc94a6d32b Mon Sep 17 00:00:00 2001 From: Miguel Martinez Trivino Date: Tue, 20 Jun 2023 21:49:10 +0200 Subject: [PATCH 04/12] readme Signed-off-by: Miguel Martinez Trivino --- .../extensions/sdk/readme-generator/main.go | 31 ++++++++++++++----- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/app/controlplane/extensions/sdk/readme-generator/main.go b/app/controlplane/extensions/sdk/readme-generator/main.go index c213a827d..732b16503 100644 --- a/app/controlplane/extensions/sdk/readme-generator/main.go +++ b/app/controlplane/extensions/sdk/readme-generator/main.go @@ -19,6 +19,7 @@ import ( "bytes" "encoding/json" "flag" + "fmt" "io" "os" "path/filepath" @@ -29,10 +30,10 @@ import ( ) const registrationInputHeader = "## Registration Input Schema" -const AttachmentInputHeader = "## Attachment Input Schema" +const attachmentInputHeader = "## Attachment Input Schema" -var registrationInputRe = regexp.MustCompile(registrationInputHeader) -var attachmentInputRe = regexp.MustCompile(AttachmentInputHeader) +var registrationInputRe = regexp.MustCompile(fmt.Sprintf("%s\n+```json\n+(.|\\s)*```", registrationInputHeader)) +var attachmentInputRe = regexp.MustCompile(fmt.Sprintf("%s\n+```json\n+(.|\\s)*```", attachmentInputHeader)) var extensionsDir string @@ -45,7 +46,7 @@ func main() { } for _, e := range extensions { - // Find README file + // Find README file and extract its content file, err := os.OpenFile(filepath.Join(extensionsDir, e.Describe().ID, "v1", "README.md"), os.O_RDWR, 0644) if err != nil { _ = l.Log(log.LevelWarn, "msg", "failed to open README.md file", "err", err) @@ -65,16 +66,32 @@ func main() { _ = l.Log(log.LevelWarn, "msg", "failed to indent JSON", "err", err) } - fileContent = registrationInputRe.ReplaceAllLiteral(fileContent, []byte(registrationInputHeader+"\n\n```json\n"+prettyRegistrationJSON.String()+"\n```")) - // Replace attachment schema + registrationInputSection := registrationInputHeader + "\n\n```json\n" + prettyRegistrationJSON.String() + "\n```" + // If the section already exists, replace it + if registrationInputRe.Match(fileContent) { + // fileContent = registrationInputRe.ReplaceAllLiteral(fileContent, []byte(registrationInputSection)) + fileContent = registrationInputRe.ReplaceAllLiteral(fileContent, []byte(registrationInputSection)) + } else { + // Otherwise, append it + fileContent = append(fileContent, []byte("\n\n"+registrationInputSection)...) + } + // Replace attachment schema var prettyAttachmentJSON bytes.Buffer err = json.Indent(&prettyAttachmentJSON, e.Describe().AttachmentJSONSchema, "", " ") if err != nil { panic(err) } - fileContent = attachmentInputRe.ReplaceAllLiteral(fileContent, []byte(AttachmentInputHeader+"\n\n```json\n"+prettyAttachmentJSON.String()+"\n```")) + attachmentInputSection := attachmentInputHeader + "\n\n```json\n" + prettyAttachmentJSON.String() + "\n```" + // If the section already exists, replace it + if attachmentInputRe.Match(fileContent) { + fileContent = attachmentInputRe.ReplaceAllLiteral(fileContent, []byte(attachmentInputSection)) + } else { + // Otherwise, append it + fileContent = append(fileContent, []byte("\n\n"+attachmentInputSection)...) + } + // Write the new content in the file _, err = file.Seek(0, 0) if err != nil { From 8cc4f524e25beae13ecac5e3d56390f153bcae96 Mon Sep 17 00:00:00 2001 From: Miguel Martinez Trivino Date: Tue, 20 Jun 2023 21:49:38 +0200 Subject: [PATCH 05/12] readme Signed-off-by: Miguel Martinez Trivino --- .../core/discord-webhook/v1/README.md | 52 ------------------- 1 file changed, 52 deletions(-) diff --git a/app/controlplane/extensions/core/discord-webhook/v1/README.md b/app/controlplane/extensions/core/discord-webhook/v1/README.md index 03e06ba78..38f0e46b5 100644 --- a/app/controlplane/extensions/core/discord-webhook/v1/README.md +++ b/app/controlplane/extensions/core/discord-webhook/v1/README.md @@ -19,56 +19,4 @@ $ chainloop integration registered add discord-webhook --opt webhook=[webhookURL ```console chainloop integration attached add --workflow $WID --integration $IID -``` - -## Registration Input Schema - -```json -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://github.com/chainloop-dev/chainloop/app/controlplane/extensions/core/discord-webhook/v1/registration-request", - "properties": { - "webhook": { - "type": "string", - "format": "uri", - "description": "URL of the discord webhook" - }, - "username": { - "type": "string", - "minLength": 1, - "description": "Override the default username of the webhook" - } - }, - "additionalProperties": false, - "type": "object", - "required": [ - "webhook" - ] -} -``` - -## Attachment Input Schema - -```json -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://github.com/chainloop-dev/chainloop/app/controlplane/extensions/core/discord-webhook/v1/registration-request", - "properties": { - "webhook": { - "type": "string", - "format": "uri", - "description": "URL of the discord webhook" - }, - "username": { - "type": "string", - "minLength": 1, - "description": "Override the default username of the webhook" - } - }, - "additionalProperties": false, - "type": "object", - "required": [ - "webhook" - ] -} ``` \ No newline at end of file From e9df8d260afb0312076446d610f3769c33ef6e0f Mon Sep 17 00:00:00 2001 From: Miguel Martinez Trivino Date: Tue, 20 Jun 2023 21:50:20 +0200 Subject: [PATCH 06/12] update readmes Signed-off-by: Miguel Martinez Trivino --- .../core/dependency-track/v1/README.md | 69 ++++++++++++++++++- .../core/discord-webhook/v1/README.md | 38 ++++++++++ .../extensions/core/oci-registry/v1/README.md | 52 ++++++++++++++ .../extensions/core/smtp/v1/README.md | 68 +++++++++++++++++- 4 files changed, 225 insertions(+), 2 deletions(-) diff --git a/app/controlplane/extensions/core/dependency-track/v1/README.md b/app/controlplane/extensions/core/dependency-track/v1/README.md index b5a0867a9..0f2cc2d87 100644 --- a/app/controlplane/extensions/core/dependency-track/v1/README.md +++ b/app/controlplane/extensions/core/dependency-track/v1/README.md @@ -2,4 +2,71 @@ This extension implements sending cycloneDX Software Bill of Materials (SBOM) to Dependency-Track. -See https://docs.chainloop.dev/guides/dependency-track/ \ No newline at end of file +See https://docs.chainloop.dev/guides/dependency-track/ + +## Registration Input Schema + +```json +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/chainloop-dev/chainloop/app/controlplane/extensions/core/dependency-track/v1/registration-request", + "properties": { + "instanceURI": { + "type": "string", + "format": "uri", + "description": "The URL of the Dependency-Track instance" + }, + "apiKey": { + "type": "string", + "description": "The API key to use for authentication" + }, + "allowAutoCreate": { + "type": "boolean", + "description": "Support of creating projects on demand" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "instanceURI", + "apiKey" + ] +} +``` + +## Attachment Input Schema + +```json +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/chainloop-dev/chainloop/app/controlplane/extensions/core/dependency-track/v1/attachment-request", + "oneOf": [ + { + "required": [ + "projectID" + ], + "title": "projectID" + }, + { + "required": [ + "projectName" + ], + "title": "projectName" + } + ], + "properties": { + "projectID": { + "type": "string", + "minLength": 1, + "description": "The ID of the existing project to send the SBOMs to" + }, + "projectName": { + "type": "string", + "minLength": 1, + "description": "The name of the project to create and send the SBOMs to" + } + }, + "additionalProperties": false, + "type": "object" +} +``` \ No newline at end of file diff --git a/app/controlplane/extensions/core/discord-webhook/v1/README.md b/app/controlplane/extensions/core/discord-webhook/v1/README.md index 38f0e46b5..6799501bd 100644 --- a/app/controlplane/extensions/core/discord-webhook/v1/README.md +++ b/app/controlplane/extensions/core/discord-webhook/v1/README.md @@ -19,4 +19,42 @@ $ chainloop integration registered add discord-webhook --opt webhook=[webhookURL ```console chainloop integration attached add --workflow $WID --integration $IID +``` + +## Registration Input Schema + +```json +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/chainloop-dev/chainloop/app/controlplane/extensions/core/discord-webhook/v1/registration-request", + "properties": { + "webhook": { + "type": "string", + "format": "uri", + "description": "URL of the discord webhook" + }, + "username": { + "type": "string", + "minLength": 1, + "description": "Override the default username of the webhook" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "webhook" + ] +} +``` + +## Attachment Input Schema + +```json +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/chainloop-dev/chainloop/app/controlplane/extensions/core/discord-webhook/v1/attachment-request", + "properties": {}, + "additionalProperties": false, + "type": "object" +} ``` \ No newline at end of file diff --git a/app/controlplane/extensions/core/oci-registry/v1/README.md b/app/controlplane/extensions/core/oci-registry/v1/README.md index 1a1012016..6738b2029 100644 --- a/app/controlplane/extensions/core/oci-registry/v1/README.md +++ b/app/controlplane/extensions/core/oci-registry/v1/README.md @@ -58,3 +58,55 @@ $ chainloop integration registered add oci-registry \ ### AWS Container Registry Not supported at the moment + + +## Registration Input Schema + +```json +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/chainloop-dev/chainloop/app/controlplane/extensions/core/oci-registry/v1/registration-request", + "properties": { + "repository": { + "type": "string", + "minLength": 1, + "description": "OCI repository uri and path" + }, + "username": { + "type": "string", + "minLength": 1, + "description": "OCI repository username" + }, + "password": { + "type": "string", + "minLength": 1, + "description": "OCI repository password" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "repository", + "username", + "password" + ] +} +``` + +## Attachment Input Schema + +```json +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/chainloop-dev/chainloop/app/controlplane/extensions/core/oci-registry/v1/attachment-request", + "properties": { + "prefix": { + "type": "string", + "minLength": 1, + "description": "OCI images name prefix (default chainloop)" + } + }, + "additionalProperties": false, + "type": "object" +} +``` \ No newline at end of file diff --git a/app/controlplane/extensions/core/smtp/v1/README.md b/app/controlplane/extensions/core/smtp/v1/README.md index 76c8070d4..f32885388 100644 --- a/app/controlplane/extensions/core/smtp/v1/README.md +++ b/app/controlplane/extensions/core/smtp/v1/README.md @@ -23,4 +23,70 @@ cc is optional: chainloop workflow integration attach --workflow $WID --integration $IID ``` -Starting now, every time a workflow run occurs, an email notification will be sent containing the details of the run and attestation. \ No newline at end of file +Starting now, every time a workflow run occurs, an email notification will be sent containing the details of the run and attestation. + +## Registration Input Schema + +```json +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/chainloop-dev/chainloop/app/controlplane/extensions/core/smtp/v1/registration-request", + "properties": { + "to": { + "type": "string", + "format": "email", + "description": "The email address to send the email to." + }, + "from": { + "type": "string", + "format": "email", + "description": "The email address of the sender." + }, + "user": { + "type": "string", + "minLength": 1, + "description": "The username to use for the SMTP authentication." + }, + "password": { + "type": "string", + "description": "The password to use for the SMTP authentication." + }, + "host": { + "type": "string", + "description": "The host to use for the SMTP authentication." + }, + "port": { + "type": "string", + "description": "The port to use for the SMTP authentication" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "to", + "from", + "user", + "password", + "host", + "port" + ] +} +``` + +## Attachment Input Schema + +```json +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/chainloop-dev/chainloop/app/controlplane/extensions/core/smtp/v1/attachment-request", + "properties": { + "cc": { + "type": "string", + "format": "email", + "description": "The email address of the carbon copy recipient." + } + }, + "additionalProperties": false, + "type": "object" +} +``` \ No newline at end of file From 2d6684cddb3f32345ea89d79b6607f1d214b859a Mon Sep 17 00:00:00 2001 From: Miguel Martinez Trivino Date: Wed, 21 Jun 2023 00:02:21 +0200 Subject: [PATCH 07/12] refactor: generator Signed-off-by: Miguel Martinez Trivino --- .../extensions/sdk/readme-generator/main.go | 81 ++++++++++--------- 1 file changed, 41 insertions(+), 40 deletions(-) diff --git a/app/controlplane/extensions/sdk/readme-generator/main.go b/app/controlplane/extensions/sdk/readme-generator/main.go index 732b16503..04ead0917 100644 --- a/app/controlplane/extensions/sdk/readme-generator/main.go +++ b/app/controlplane/extensions/sdk/readme-generator/main.go @@ -32,78 +32,79 @@ import ( const registrationInputHeader = "## Registration Input Schema" const attachmentInputHeader = "## Attachment Input Schema" -var registrationInputRe = regexp.MustCompile(fmt.Sprintf("%s\n+```json\n+(.|\\s)*```", registrationInputHeader)) -var attachmentInputRe = regexp.MustCompile(fmt.Sprintf("%s\n+```json\n+(.|\\s)*```", attachmentInputHeader)) - +// base path to the extensions directory var extensionsDir string -func main() { +func addSchemaToSection(src []byte, sectionHeader string, schema []byte) ([]byte, error) { + var prettyJSON bytes.Buffer + err := json.Indent(&prettyJSON, schema, "", " ") + if err != nil { + return nil, err + } + + inputSection := sectionHeader + "\n\n```json\n" + prettyJSON.String() + "\n```" + r := regexp.MustCompile(fmt.Sprintf("%s\n+```json\n+(.|\\s)*```", sectionHeader)) + // If the section already exists, replace it + if r.Match(src) { + return r.ReplaceAllLiteral(src, []byte(inputSection)), nil + } + + // Append it + return append(src, []byte("\n\n"+inputSection)...), nil +} + +func mainE() error { l := log.NewStdLogger(os.Stdout) extensions, err := extensionsSDK.Load(l) if err != nil { - panic(err) + return fmt.Errorf("failed to load extensions: %w", err) } for _, e := range extensions { // Find README file and extract its content file, err := os.OpenFile(filepath.Join(extensionsDir, e.Describe().ID, "v1", "README.md"), os.O_RDWR, 0644) if err != nil { - _ = l.Log(log.LevelWarn, "msg", "failed to open README.md file", "err", err) - continue + return fmt.Errorf("failed to open README.md file: %w", err) } fileContent, err := io.ReadAll(file) if err != nil { - _ = l.Log(log.LevelWarn, "msg", "failed to read README.md file", "err", err) - continue + return fmt.Errorf("failed to read README.md file: %w", err) } - // Replace registration input schema - var prettyRegistrationJSON bytes.Buffer - err = json.Indent(&prettyRegistrationJSON, e.Describe().RegistrationJSONSchema, "", " ") + // Replace/Add registration input schema + fileContent, err = addSchemaToSection(fileContent, registrationInputHeader, e.Describe().RegistrationJSONSchema) if err != nil { - _ = l.Log(log.LevelWarn, "msg", "failed to indent JSON", "err", err) + return fmt.Errorf("failed to add registration schema to README.md file: %w", err) } - registrationInputSection := registrationInputHeader + "\n\n```json\n" + prettyRegistrationJSON.String() + "\n```" - // If the section already exists, replace it - if registrationInputRe.Match(fileContent) { - // fileContent = registrationInputRe.ReplaceAllLiteral(fileContent, []byte(registrationInputSection)) - fileContent = registrationInputRe.ReplaceAllLiteral(fileContent, []byte(registrationInputSection)) - } else { - // Otherwise, append it - fileContent = append(fileContent, []byte("\n\n"+registrationInputSection)...) - } - - // Replace attachment schema - var prettyAttachmentJSON bytes.Buffer - err = json.Indent(&prettyAttachmentJSON, e.Describe().AttachmentJSONSchema, "", " ") + // Replace/Add attachment input schema + fileContent, err = addSchemaToSection(fileContent, attachmentInputHeader, e.Describe().AttachmentJSONSchema) if err != nil { - panic(err) - } - - attachmentInputSection := attachmentInputHeader + "\n\n```json\n" + prettyAttachmentJSON.String() + "\n```" - // If the section already exists, replace it - if attachmentInputRe.Match(fileContent) { - fileContent = attachmentInputRe.ReplaceAllLiteral(fileContent, []byte(attachmentInputSection)) - } else { - // Otherwise, append it - fileContent = append(fileContent, []byte("\n\n"+attachmentInputSection)...) + return fmt.Errorf("failed to add attachment schema to README.md file: %w", err) } // Write the new content in the file _, err = file.Seek(0, 0) if err != nil { - _ = l.Log(log.LevelWarn, "msg", "failed to seek README.md file", "err", err) - continue + return fmt.Errorf("failed to seek README.md file: %w", err) } _, err = file.Write(fileContent) if err != nil { - _ = l.Log(log.LevelWarn, "msg", "failed to write README.md file", "err", err) - continue + fmt.Errorf("failed to write README.md file: %w", err) } + + _ = l.Log(log.LevelInfo, "msg", "README.md file updated", "extension", e.Describe().ID) + } + + return nil +} + +func main() { + if err := mainE(); err != nil { + panic(err) } } From 7fd31b573eaa1bc48415f5dbdaee1c1dd55bcc4c Mon Sep 17 00:00:00 2001 From: Miguel Martinez Trivino Date: Wed, 21 Jun 2023 00:08:36 +0200 Subject: [PATCH 08/12] refactor: generator Signed-off-by: Miguel Martinez Trivino --- .../extensions/core/smtp/v1/README.md | 18 ------------------ .../extensions/sdk/readme-generator/main.go | 2 ++ 2 files changed, 2 insertions(+), 18 deletions(-) diff --git a/app/controlplane/extensions/core/smtp/v1/README.md b/app/controlplane/extensions/core/smtp/v1/README.md index f32885388..f23a43bc8 100644 --- a/app/controlplane/extensions/core/smtp/v1/README.md +++ b/app/controlplane/extensions/core/smtp/v1/README.md @@ -72,21 +72,3 @@ Starting now, every time a workflow run occurs, an email notification will be se ] } ``` - -## Attachment Input Schema - -```json -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://github.com/chainloop-dev/chainloop/app/controlplane/extensions/core/smtp/v1/attachment-request", - "properties": { - "cc": { - "type": "string", - "format": "email", - "description": "The email address of the carbon copy recipient." - } - }, - "additionalProperties": false, - "type": "object" -} -``` \ No newline at end of file diff --git a/app/controlplane/extensions/sdk/readme-generator/main.go b/app/controlplane/extensions/sdk/readme-generator/main.go index 04ead0917..5d0aee7e9 100644 --- a/app/controlplane/extensions/sdk/readme-generator/main.go +++ b/app/controlplane/extensions/sdk/readme-generator/main.go @@ -13,6 +13,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:generate go run main.go --dir ../../core + package main import ( From c70e0f6c612e64a7719f562e38cff1b570ff06f5 Mon Sep 17 00:00:00 2001 From: Miguel Martinez Trivino Date: Wed, 21 Jun 2023 00:12:02 +0200 Subject: [PATCH 09/12] refactor: generator Signed-off-by: Miguel Martinez Trivino --- .../extensions/core/smtp/v1/README.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/app/controlplane/extensions/core/smtp/v1/README.md b/app/controlplane/extensions/core/smtp/v1/README.md index f23a43bc8..951911b82 100644 --- a/app/controlplane/extensions/core/smtp/v1/README.md +++ b/app/controlplane/extensions/core/smtp/v1/README.md @@ -72,3 +72,22 @@ Starting now, every time a workflow run occurs, an email notification will be se ] } ``` + + +## Attachment Input Schema + +```json +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/chainloop-dev/chainloop/app/controlplane/extensions/core/smtp/v1/attachment-request", + "properties": { + "cc": { + "type": "string", + "format": "email", + "description": "The email address of the carbon copy recipient." + } + }, + "additionalProperties": false, + "type": "object" +} +``` \ No newline at end of file From 57fe907042328a87098caf93aa11067a0a4a80a3 Mon Sep 17 00:00:00 2001 From: Miguel Martinez Trivino Date: Wed, 21 Jun 2023 09:05:31 +0200 Subject: [PATCH 10/12] refactor: generator Signed-off-by: Miguel Martinez Trivino --- .../extensions/core/smtp/v1/README.md | 3 +- .../extensions/sdk/readme-generator/main.go | 39 ++++++++++--------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/app/controlplane/extensions/core/smtp/v1/README.md b/app/controlplane/extensions/core/smtp/v1/README.md index 951911b82..320970f74 100644 --- a/app/controlplane/extensions/core/smtp/v1/README.md +++ b/app/controlplane/extensions/core/smtp/v1/README.md @@ -73,7 +73,6 @@ Starting now, every time a workflow run occurs, an email notification will be se } ``` - ## Attachment Input Schema ```json @@ -90,4 +89,4 @@ Starting now, every time a workflow run occurs, an email notification will be se "additionalProperties": false, "type": "object" } -``` \ No newline at end of file +```` \ No newline at end of file diff --git a/app/controlplane/extensions/sdk/readme-generator/main.go b/app/controlplane/extensions/sdk/readme-generator/main.go index 5d0aee7e9..1282f1a8c 100644 --- a/app/controlplane/extensions/sdk/readme-generator/main.go +++ b/app/controlplane/extensions/sdk/readme-generator/main.go @@ -37,24 +37,7 @@ const attachmentInputHeader = "## Attachment Input Schema" // base path to the extensions directory var extensionsDir string -func addSchemaToSection(src []byte, sectionHeader string, schema []byte) ([]byte, error) { - var prettyJSON bytes.Buffer - err := json.Indent(&prettyJSON, schema, "", " ") - if err != nil { - return nil, err - } - - inputSection := sectionHeader + "\n\n```json\n" + prettyJSON.String() + "\n```" - r := regexp.MustCompile(fmt.Sprintf("%s\n+```json\n+(.|\\s)*```", sectionHeader)) - // If the section already exists, replace it - if r.Match(src) { - return r.ReplaceAllLiteral(src, []byte(inputSection)), nil - } - - // Append it - return append(src, []byte("\n\n"+inputSection)...), nil -} - +// Enhance README.md files for the registrations with the registration and attachment input schemas func mainE() error { l := log.NewStdLogger(os.Stdout) @@ -95,7 +78,7 @@ func mainE() error { _, err = file.Write(fileContent) if err != nil { - fmt.Errorf("failed to write README.md file: %w", err) + return fmt.Errorf("failed to write README.md file: %w", err) } _ = l.Log(log.LevelInfo, "msg", "README.md file updated", "extension", e.Describe().ID) @@ -114,3 +97,21 @@ func init() { flag.StringVar(&extensionsDir, "dir", "", "base directory for extensions i.e ./core") flag.Parse() } + +func addSchemaToSection(src []byte, sectionHeader string, schema []byte) ([]byte, error) { + var prettyJSON bytes.Buffer + err := json.Indent(&prettyJSON, schema, "", " ") + if err != nil { + return nil, err + } + + inputSection := sectionHeader + "\n\n```json\n" + prettyJSON.String() + "\n```" + r := regexp.MustCompile(fmt.Sprintf("%s\n+```json\n+(.|\\s)*```", sectionHeader)) + // If the section already exists, replace it + if r.Match(src) { + return r.ReplaceAllLiteral(src, []byte(inputSection)), nil + } + + // Append it + return append(src, []byte("\n\n"+inputSection)...), nil +} From eea29ab58de2654728d7da902161bad2d9bd3a8c Mon Sep 17 00:00:00 2001 From: Miguel Martinez Trivino Date: Wed, 21 Jun 2023 10:07:55 +0200 Subject: [PATCH 11/12] feat: show inputs table Signed-off-by: Miguel Martinez Trivino --- app/cli/cmd/available_integration_describe.go | 3 +- app/cli/cmd/registered_integration_add.go | 3 +- .../action/available_integration_list.go | 113 ++---------------- .../action/available_integration_list_test.go | 93 -------------- .../core/dependency-track/v1/README.md | 12 ++ .../core/discord-webhook/v1/README.md | 7 +- .../extensions/core/oci-registry/v1/README.md | 11 ++ .../extensions/core/smtp/v1/README.md | 15 ++- .../extensions/sdk/readme-generator/main.go | 60 +++++++++- app/controlplane/extensions/sdk/v1/fanout.go | 105 ++++++++++++++-- .../extensions/sdk/v1/fanout_test.go | 70 +++++++++++ .../sdk/v1}/testdata/schemas/basic.json | 0 .../v1}/testdata/schemas/oneof_required.json | 0 13 files changed, 278 insertions(+), 214 deletions(-) delete mode 100644 app/cli/internal/action/available_integration_list_test.go rename app/{cli/internal/action => controlplane/extensions/sdk/v1}/testdata/schemas/basic.json (100%) rename app/{cli/internal/action => controlplane/extensions/sdk/v1}/testdata/schemas/oneof_required.json (100%) diff --git a/app/cli/cmd/available_integration_describe.go b/app/cli/cmd/available_integration_describe.go index 0eb7552ae..64144980d 100644 --- a/app/cli/cmd/available_integration_describe.go +++ b/app/cli/cmd/available_integration_describe.go @@ -22,6 +22,7 @@ import ( "sort" "github.com/chainloop-dev/chainloop/app/cli/internal/action" + "github.com/chainloop-dev/chainloop/app/controlplane/extensions/sdk/v1" "github.com/jedib0t/go-pretty/v6/table" "github.com/jedib0t/go-pretty/v6/text" "github.com/spf13/cobra" @@ -92,7 +93,7 @@ func availableIntegrationDescribeTableOutput(items []*action.AvailableIntegratio } // render de-normalized schema format -func renderSchemaTable(tableTitle string, properties action.SchemaPropertiesMap) error { +func renderSchemaTable(tableTitle string, properties sdk.SchemaPropertiesMap) error { if len(properties) == 0 { return nil } diff --git a/app/cli/cmd/registered_integration_add.go b/app/cli/cmd/registered_integration_add.go index 3dbac92a7..0bc741e97 100644 --- a/app/cli/cmd/registered_integration_add.go +++ b/app/cli/cmd/registered_integration_add.go @@ -22,6 +22,7 @@ import ( "strings" "github.com/chainloop-dev/chainloop/app/cli/internal/action" + "github.com/chainloop-dev/chainloop/app/controlplane/extensions/sdk/v1" "github.com/santhosh-tekuri/jsonschema/v5" "github.com/spf13/cobra" ) @@ -98,7 +99,7 @@ func parseAndValidateOpts(opts []string, schema *action.JSONSchema) (map[string] // parseKeyValOpts performs two steps // 1 - Split the options into key/value pairs // 2 - Cast the values to the expected type defined in the schema -func parseKeyValOpts(opts []string, propertiesMap action.SchemaPropertiesMap) (map[string]any, error) { +func parseKeyValOpts(opts []string, propertiesMap sdk.SchemaPropertiesMap) (map[string]any, error) { // 1 - Split the options into key/value pairs var options = make(map[string]any) for _, opt := range opts { diff --git a/app/cli/internal/action/available_integration_list.go b/app/cli/internal/action/available_integration_list.go index 0f0967efa..fc38e87b8 100644 --- a/app/cli/internal/action/available_integration_list.go +++ b/app/cli/internal/action/available_integration_list.go @@ -16,13 +16,12 @@ package action import ( - "bytes" "context" "errors" "fmt" - "sort" pb "github.com/chainloop-dev/chainloop/app/controlplane/api/controlplane/v1" + "github.com/chainloop-dev/chainloop/app/controlplane/extensions/sdk/v1" "github.com/santhosh-tekuri/jsonschema/v5" ) @@ -45,22 +44,8 @@ type JSONSchema struct { Raw string `json:"schema"` // Parsed schema so it can be used for validation or other purposes // It's not shown in the json output - Parsed *jsonschema.Schema `json:"-"` - Properties SchemaPropertiesMap `json:"-"` -} - -type SchemaPropertiesMap map[string]*SchemaProperty -type SchemaProperty struct { - // Name of the property - Name string - // optional description - Description string - // Type of the property (string, boolean, number) - Type string - // If the property is required - Required bool - // Optional format (email, host) - Format string + Parsed *jsonschema.Schema `json:"-"` + Properties sdk.SchemaPropertiesMap `json:"-"` } func NewAvailableIntegrationList(cfg *ActionsOpts) *AvailableIntegrationList { @@ -107,24 +92,24 @@ func pbAvailableIntegrationItemToAction(in *pb.IntegrationAvailableItem) (*Avail // Parse the schemas so they can be used for validation or other purposes var err error - i.Registration.Parsed, err = compileJSONSchema(foType.GetRegistrationSchema()) + i.Registration.Parsed, err = sdk.CompileJSONSchema(foType.GetRegistrationSchema()) if err != nil { return nil, fmt.Errorf("failed to compile registration schema: %w", err) } - i.Attachment.Parsed, err = compileJSONSchema(foType.GetAttachmentSchema()) + i.Attachment.Parsed, err = sdk.CompileJSONSchema(foType.GetAttachmentSchema()) if err != nil { return nil, fmt.Errorf("failed to compile registration schema: %w", err) } // Calculate the properties map - i.Registration.Properties = make(SchemaPropertiesMap) - if err := calculatePropertiesMap(i.Registration.Parsed, &i.Registration.Properties); err != nil { + i.Registration.Properties = make(sdk.SchemaPropertiesMap) + if err := sdk.CalculatePropertiesMap(i.Registration.Parsed, &i.Registration.Properties); err != nil { return nil, fmt.Errorf("failed to calculate registration properties: %w", err) } - i.Attachment.Properties = make(SchemaPropertiesMap) - if err := calculatePropertiesMap(i.Attachment.Parsed, &i.Attachment.Properties); err != nil { + i.Attachment.Properties = make(sdk.SchemaPropertiesMap) + if err := sdk.CalculatePropertiesMap(i.Attachment.Parsed, &i.Attachment.Properties); err != nil { return nil, fmt.Errorf("failed to calculate attachment properties: %w", err) } @@ -133,83 +118,3 @@ func pbAvailableIntegrationItemToAction(in *pb.IntegrationAvailableItem) (*Avail return i, nil } - -func compileJSONSchema(in []byte) (*jsonschema.Schema, error) { - // Parse the schemas - compiler := jsonschema.NewCompiler() - // Enable format validation - compiler.AssertFormat = true - // Show description - compiler.ExtractAnnotations = true - - if err := compiler.AddResource("schema.json", bytes.NewReader(in)); err != nil { - return nil, fmt.Errorf("failed to compile schema: %w", err) - } - - return compiler.Compile("schema.json") -} - -// calculate a map with all the properties of a schema -func calculatePropertiesMap(s *jsonschema.Schema, m *SchemaPropertiesMap) error { - if m == nil { - return nil - } - - // Schema with reference - if s.Ref != nil { - return calculatePropertiesMap(s.Ref, m) - } - - // Appended schemas - if s.AllOf != nil { - for _, s := range s.AllOf { - if err := calculatePropertiesMap(s, m); err != nil { - return err - } - } - } - - if s.Properties != nil { - requiredMap := make(map[string]bool) - for _, r := range s.Required { - requiredMap[r] = true - } - - for k, v := range s.Properties { - if err := calculatePropertiesMap(v, m); err != nil { - return err - } - - var required = requiredMap[k] - (*m)[k] = &SchemaProperty{ - Name: k, - Type: v.Types[0], - Required: required, - Description: v.Description, - Format: v.Format, - } - } - } - - // We return the map sorted - // This is not strictly necessary but it makes the output more readable - // and it's easier to test - - // Sort the keys - keys := make([]string, 0, len(*m)) - for k := range *m { - keys = append(keys, k) - } - - sort.Strings(keys) - - // Create a new map with the sorted keys - newMap := make(SchemaPropertiesMap) - for _, k := range keys { - newMap[k] = (*m)[k] - } - - *m = newMap - - return nil -} diff --git a/app/cli/internal/action/available_integration_list_test.go b/app/cli/internal/action/available_integration_list_test.go deleted file mode 100644 index 8d18974ae..000000000 --- a/app/cli/internal/action/available_integration_list_test.go +++ /dev/null @@ -1,93 +0,0 @@ -// -// Copyright 2023 The Chainloop Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package action - -import ( - "fmt" - "os" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestCalculatePropertiesMap(t *testing.T) { - testCases := []struct { - schemaPath string - want SchemaPropertiesMap - }{ - { - "basic.json", - SchemaPropertiesMap{ - "allowAutoCreate": { - Name: "allowAutoCreate", - Description: "Support of creating projects on demand", - Type: "boolean", - Required: false, - }, - "apiKey": { - Name: "apiKey", - Description: "The API key to use for authentication", - Type: "string", - Required: true, - }, - "instanceURI": { - Name: "instanceURI", - Description: "The URL of the Dependency-Track instance", - Type: "string", - Required: true, - Format: "uri", - }, - "port": { - Name: "port", - Type: "number", - }, - }, - }, - { - // NOTE: oneof work in the validation but are not shown in the map - // This testCase is here to document this limitation - "oneof_required.json", - SchemaPropertiesMap{ - "projectID": { - Name: "projectID", - Description: "The ID of the existing project to send the SBOMs to", - Type: "string", - }, - "projectName": { - Name: "projectName", - Description: "The name of the project to create and send the SBOMs to", - Type: "string", - }, - }, - }, - } - - for _, tc := range testCases { - t.Run(tc.schemaPath, func(t *testing.T) { - schemaRaw, err := os.ReadFile(fmt.Sprintf("testdata/schemas/%s", tc.schemaPath)) - require.NoError(t, err) - schema, err := compileJSONSchema(schemaRaw) - require.NoError(t, err) - - var got = make(SchemaPropertiesMap) - err = calculatePropertiesMap(schema, &got) - assert.NoError(t, err) - - assert.Equal(t, tc.want, got) - }) - } -} diff --git a/app/controlplane/extensions/core/dependency-track/v1/README.md b/app/controlplane/extensions/core/dependency-track/v1/README.md index 0f2cc2d87..eb969664c 100644 --- a/app/controlplane/extensions/core/dependency-track/v1/README.md +++ b/app/controlplane/extensions/core/dependency-track/v1/README.md @@ -4,8 +4,15 @@ This extension implements sending cycloneDX Software Bill of Materials (SBOM) to See https://docs.chainloop.dev/guides/dependency-track/ + ## Registration Input Schema +|Field|Type|Required|Description| +|---|---|---|---| +|allowAutoCreate|boolean|no|Support of creating projects on demand| +|apiKey|string|yes|The API key to use for authentication| +|instanceURI|string (uri)|yes|The URL of the Dependency-Track instance| + ```json { "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -36,6 +43,11 @@ See https://docs.chainloop.dev/guides/dependency-track/ ## Attachment Input Schema +|Field|Type|Required|Description| +|---|---|---|---| +|projectID|string|no|The ID of the existing project to send the SBOMs to| +|projectName|string|no|The name of the project to create and send the SBOMs to| + ```json { "$schema": "https://json-schema.org/draft/2020-12/schema", diff --git a/app/controlplane/extensions/core/discord-webhook/v1/README.md b/app/controlplane/extensions/core/discord-webhook/v1/README.md index 6799501bd..e18e1f4c1 100644 --- a/app/controlplane/extensions/core/discord-webhook/v1/README.md +++ b/app/controlplane/extensions/core/discord-webhook/v1/README.md @@ -23,6 +23,11 @@ chainloop integration attached add --workflow $WID --integration $IID ## Registration Input Schema +|Field|Type|Required|Description| +|---|---|---|---| +|username|string|no|Override the default username of the webhook| +|webhook|string (uri)|yes|URL of the discord webhook| + ```json { "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -57,4 +62,4 @@ chainloop integration attached add --workflow $WID --integration $IID "additionalProperties": false, "type": "object" } -``` \ No newline at end of file +```` \ No newline at end of file diff --git a/app/controlplane/extensions/core/oci-registry/v1/README.md b/app/controlplane/extensions/core/oci-registry/v1/README.md index 6738b2029..092fd2848 100644 --- a/app/controlplane/extensions/core/oci-registry/v1/README.md +++ b/app/controlplane/extensions/core/oci-registry/v1/README.md @@ -60,8 +60,15 @@ $ chainloop integration registered add oci-registry \ Not supported at the moment + ## Registration Input Schema +|Field|Type|Required|Description| +|---|---|---|---| +|password|string|yes|OCI repository password| +|repository|string|yes|OCI repository uri and path| +|username|string|yes|OCI repository username| + ```json { "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -95,6 +102,10 @@ Not supported at the moment ## Attachment Input Schema +|Field|Type|Required|Description| +|---|---|---|---| +|prefix|string|no|OCI images name prefix (default chainloop)| + ```json { "$schema": "https://json-schema.org/draft/2020-12/schema", diff --git a/app/controlplane/extensions/core/smtp/v1/README.md b/app/controlplane/extensions/core/smtp/v1/README.md index 320970f74..43b1d12e7 100644 --- a/app/controlplane/extensions/core/smtp/v1/README.md +++ b/app/controlplane/extensions/core/smtp/v1/README.md @@ -27,6 +27,15 @@ Starting now, every time a workflow run occurs, an email notification will be se ## Registration Input Schema +|Field|Type|Required|Description| +|---|---|---|---| +|from|string (email)|yes|The email address of the sender.| +|host|string|yes|The host to use for the SMTP authentication.| +|password|string|yes|The password to use for the SMTP authentication.| +|port|string|yes|The port to use for the SMTP authentication| +|to|string (email)|yes|The email address to send the email to.| +|user|string|yes|The username to use for the SMTP authentication.| + ```json { "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -75,6 +84,10 @@ Starting now, every time a workflow run occurs, an email notification will be se ## Attachment Input Schema +|Field|Type|Required|Description| +|---|---|---|---| +|cc|string (email)|no|The email address of the carbon copy recipient.| + ```json { "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -89,4 +102,4 @@ Starting now, every time a workflow run occurs, an email notification will be se "additionalProperties": false, "type": "object" } -```` \ No newline at end of file +``` \ No newline at end of file diff --git a/app/controlplane/extensions/sdk/readme-generator/main.go b/app/controlplane/extensions/sdk/readme-generator/main.go index 1282f1a8c..77f2a2c27 100644 --- a/app/controlplane/extensions/sdk/readme-generator/main.go +++ b/app/controlplane/extensions/sdk/readme-generator/main.go @@ -26,8 +26,10 @@ import ( "os" "path/filepath" "regexp" + "sort" extensionsSDK "github.com/chainloop-dev/chainloop/app/controlplane/extensions" + "github.com/chainloop-dev/chainloop/app/controlplane/extensions/sdk/v1" "github.com/go-kratos/kratos/v2/log" ) @@ -99,14 +101,19 @@ func init() { } func addSchemaToSection(src []byte, sectionHeader string, schema []byte) ([]byte, error) { - var prettyJSON bytes.Buffer - err := json.Indent(&prettyJSON, schema, "", " ") + var jsonSchema bytes.Buffer + err := json.Indent(&jsonSchema, schema, "", " ") if err != nil { return nil, err } - inputSection := sectionHeader + "\n\n```json\n" + prettyJSON.String() + "\n```" - r := regexp.MustCompile(fmt.Sprintf("%s\n+```json\n+(.|\\s)*```", sectionHeader)) + propertiesTable, err := renderSchemaTable(schema) + if err != nil { + return nil, err + } + + inputSection := sectionHeader + "\n\n" + propertiesTable + "```json\n" + jsonSchema.String() + "\n```" + r := regexp.MustCompile(fmt.Sprintf("%s\n+(.|\\s)*```", sectionHeader)) // If the section already exists, replace it if r.Match(src) { return r.ReplaceAllLiteral(src, []byte(inputSection)), nil @@ -115,3 +122,48 @@ func addSchemaToSection(src []byte, sectionHeader string, schema []byte) ([]byte // Append it return append(src, []byte("\n\n"+inputSection)...), nil } + +func renderSchemaTable(schemaRaw []byte) (string, error) { + schema, err := sdk.CompileJSONSchema(schemaRaw) + if err != nil { + return "", fmt.Errorf("failed to compile schema: %w", err) + } + + properties := make(sdk.SchemaPropertiesMap) + err = sdk.CalculatePropertiesMap(schema, &properties) + if err != nil { + return "", fmt.Errorf("failed to calculate properties map: %w", err) + } + + if len(properties) == 0 { + return "", nil + } + + table := "|Field|Type|Required|Description|\n|---|---|---|---|\n" + + // Sort map + keys := make([]string, 0, len(properties)) + for k := range properties { + keys = append(keys, k) + } + + sort.Strings(keys) + + for _, k := range keys { + v := properties[k] + + propertyType := v.Type + if v.Format != "" { + propertyType = fmt.Sprintf("%s (%s)", propertyType, v.Format) + } + + required := "no" + if v.Required { + required = "yes" + } + + table += fmt.Sprintf("|%s|%s|%s|%s|\n", v.Name, propertyType, required, v.Description) + } + + return table + "\n", nil +} diff --git a/app/controlplane/extensions/sdk/v1/fanout.go b/app/controlplane/extensions/sdk/v1/fanout.go index b91fb1061..3e6ff7154 100644 --- a/app/controlplane/extensions/sdk/v1/fanout.go +++ b/app/controlplane/extensions/sdk/v1/fanout.go @@ -22,6 +22,7 @@ import ( "errors" "fmt" "io" + "sort" schemaapi "github.com/chainloop-dev/chainloop/app/controlplane/api/workflowcontract/v1" "github.com/chainloop-dev/chainloop/internal/attestation/renderer/chainloop" @@ -301,15 +302,7 @@ func (i *FanOutIntegration) ValidateAttachmentRequest(jsonPayload []byte) error } func validatePayloadAgainstJSONSchema(jsonPayload []byte, jsonSchema []byte) error { - compiler := schema_validator.NewCompiler() - // Enable format validation - compiler.AssertFormat = true - - if err := compiler.AddResource("schema.json", bytes.NewReader(jsonSchema)); err != nil { - return fmt.Errorf("failed to compile schema: %w", err) - } - - schema, err := compiler.Compile("schema.json") + schema, err := CompileJSONSchema(jsonSchema) if err != nil { return fmt.Errorf("failed to compile schema: %w", err) } @@ -420,3 +413,97 @@ func generateJSONSchema(schema any) ([]byte, error) { return json.Marshal(s) } + +type SchemaPropertiesMap map[string]*SchemaProperty +type SchemaProperty struct { + // Name of the property + Name string + // optional description + Description string + // Type of the property (string, boolean, number) + Type string + // If the property is required + Required bool + // Optional format (email, host) + Format string +} + +func CompileJSONSchema(in []byte) (*schema_validator.Schema, error) { + // Parse the schemas + compiler := schema_validator.NewCompiler() + // Enable format validation + compiler.AssertFormat = true + // Show description + compiler.ExtractAnnotations = true + + if err := compiler.AddResource("schema.json", bytes.NewReader(in)); err != nil { + return nil, fmt.Errorf("failed to compile schema: %w", err) + } + + return compiler.Compile("schema.json") +} + +// Denormalize the properties of a json schema +func CalculatePropertiesMap(s *schema_validator.Schema, m *SchemaPropertiesMap) error { + if m == nil { + return nil + } + + // Schema with reference + if s.Ref != nil { + return CalculatePropertiesMap(s.Ref, m) + } + + // Appended schemas + if s.AllOf != nil { + for _, s := range s.AllOf { + if err := CalculatePropertiesMap(s, m); err != nil { + return err + } + } + } + + if s.Properties != nil { + requiredMap := make(map[string]bool) + for _, r := range s.Required { + requiredMap[r] = true + } + + for k, v := range s.Properties { + if err := CalculatePropertiesMap(v, m); err != nil { + return err + } + + var required = requiredMap[k] + (*m)[k] = &SchemaProperty{ + Name: k, + Type: v.Types[0], + Required: required, + Description: v.Description, + Format: v.Format, + } + } + } + + // We return the map sorted + // This is not strictly necessary but it makes the output more readable + // and it's easier to test + + // Sort the keys + keys := make([]string, 0, len(*m)) + for k := range *m { + keys = append(keys, k) + } + + sort.Strings(keys) + + // Create a new map with the sorted keys + newMap := make(SchemaPropertiesMap) + for _, k := range keys { + newMap[k] = (*m)[k] + } + + *m = newMap + + return nil +} diff --git a/app/controlplane/extensions/sdk/v1/fanout_test.go b/app/controlplane/extensions/sdk/v1/fanout_test.go index 5e930fe1c..a50587292 100644 --- a/app/controlplane/extensions/sdk/v1/fanout_test.go +++ b/app/controlplane/extensions/sdk/v1/fanout_test.go @@ -17,6 +17,8 @@ package sdk_test import ( "encoding/json" + "fmt" + "os" "testing" schemaapi "github.com/chainloop-dev/chainloop/app/controlplane/api/workflowcontract/v1" @@ -337,3 +339,71 @@ func TestValidateAttachmentRequest(t *testing.T) { }) } } + +func TestCalculatePropertiesMap(t *testing.T) { + testCases := []struct { + schemaPath string + want sdk.SchemaPropertiesMap + }{ + { + "basic.json", + sdk.SchemaPropertiesMap{ + "allowAutoCreate": { + Name: "allowAutoCreate", + Description: "Support of creating projects on demand", + Type: "boolean", + Required: false, + }, + "apiKey": { + Name: "apiKey", + Description: "The API key to use for authentication", + Type: "string", + Required: true, + }, + "instanceURI": { + Name: "instanceURI", + Description: "The URL of the Dependency-Track instance", + Type: "string", + Required: true, + Format: "uri", + }, + "port": { + Name: "port", + Type: "number", + }, + }, + }, + { + // NOTE: oneof work in the validation but are not shown in the map + // This testCase is here to document this limitation + "oneof_required.json", + sdk.SchemaPropertiesMap{ + "projectID": { + Name: "projectID", + Description: "The ID of the existing project to send the SBOMs to", + Type: "string", + }, + "projectName": { + Name: "projectName", + Description: "The name of the project to create and send the SBOMs to", + Type: "string", + }, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.schemaPath, func(t *testing.T) { + schemaRaw, err := os.ReadFile(fmt.Sprintf("testdata/schemas/%s", tc.schemaPath)) + require.NoError(t, err) + schema, err := sdk.CompileJSONSchema(schemaRaw) + require.NoError(t, err) + + var got = make(sdk.SchemaPropertiesMap) + err = sdk.CalculatePropertiesMap(schema, &got) + assert.NoError(t, err) + + assert.Equal(t, tc.want, got) + }) + } +} diff --git a/app/cli/internal/action/testdata/schemas/basic.json b/app/controlplane/extensions/sdk/v1/testdata/schemas/basic.json similarity index 100% rename from app/cli/internal/action/testdata/schemas/basic.json rename to app/controlplane/extensions/sdk/v1/testdata/schemas/basic.json diff --git a/app/cli/internal/action/testdata/schemas/oneof_required.json b/app/controlplane/extensions/sdk/v1/testdata/schemas/oneof_required.json similarity index 100% rename from app/cli/internal/action/testdata/schemas/oneof_required.json rename to app/controlplane/extensions/sdk/v1/testdata/schemas/oneof_required.json From 2db1259b8ae0aa122478b9fb4953677a8e20f6aa Mon Sep 17 00:00:00 2001 From: Miguel Martinez Trivino Date: Wed, 21 Jun 2023 10:19:37 +0200 Subject: [PATCH 12/12] update readmes Signed-off-by: Miguel Martinez Trivino --- app/controlplane/extensions/core/discord-webhook/v1/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controlplane/extensions/core/discord-webhook/v1/README.md b/app/controlplane/extensions/core/discord-webhook/v1/README.md index e18e1f4c1..63160e483 100644 --- a/app/controlplane/extensions/core/discord-webhook/v1/README.md +++ b/app/controlplane/extensions/core/discord-webhook/v1/README.md @@ -62,4 +62,4 @@ chainloop integration attached add --workflow $WID --integration $IID "additionalProperties": false, "type": "object" } -```` \ No newline at end of file +``` \ No newline at end of file