docs(integrations): list and initial user guide (#197)

Signed-off-by: Miguel Martinez Trivino <miguel@chainloop.dev>

Author: migmartri

README.md

@@ -108,7 +108,7 @@ Operators can set up third-party integrations such as [Dependency-Track](https:/
 
 Ops can mix and match with different integrations while **not requiring developers to make any changes on their side**!
 
-You can see the list of available integrations by running `chainloop integration available list` or by browsing [their source code](./app/controlplane/plugins/).
+To learn more and to find the list of available integrations, check our [integrations page](./docs/integrations.md)
 
 ### Role-tailored experience
 

app/controlplane/plugins/core/dependency-track/v1/extension.go

@@ -63,7 +63,7 @@ func New(l log.Logger) (sdk.FanOut, error) {
 	base, err := sdk.NewFanOut(
 		&sdk.NewParams{
 			ID:          "dependency-track",
-			Version:     "0.2",
+			Version:     "1.2",
 			Description: description,
 			Logger:      l,
 			InputSchema: &sdk.InputSchema{

app/controlplane/plugins/core/discord-webhook/v1/discord.go

@@ -57,7 +57,7 @@ func New(l log.Logger) (sdk.FanOut, error) {
 	base, err := sdk.NewFanOut(
 		&sdk.NewParams{
 			ID:          "discord-webhook",
-			Version:     "0.1",
+			Version:     "1.1",
 			Description: "Send attestations to Discord",
 			Logger:      l,
 			InputSchema: &sdk.InputSchema{

app/controlplane/plugins/core/oci-registry/v1/ociregistry.go

@@ -59,7 +59,7 @@ func New(l log.Logger) (sdk.FanOut, error) {
 	base, err := sdk.NewFanOut(
 		&sdk.NewParams{
 			ID:          "oci-registry",
-			Version:     "0.1",
+			Version:     "1.0",
 			Description: "Send attestations to a compatible OCI registry",
 			Logger:      l,
 			InputSchema: &sdk.InputSchema{

app/controlplane/plugins/core/smtp/v1/extension.go

@@ -62,7 +62,7 @@ func New(l log.Logger) (sdk.FanOut, error) {
 	base, err := sdk.NewFanOut(
 		&sdk.NewParams{
 			ID:          "smtp",
-			Version:     "0.1",
+			Version:     "1.0",
 			Description: description,
 			Logger:      l,
 			InputSchema: &sdk.InputSchema{

app/controlplane/plugins/sdk/readme-generator/main.go

@@ -13,7 +13,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-//go:generate go run main.go --dir ../../core
+//go:generate go run main.go --dir ../../core --integrations-index-path ../../../../../docs/integrations.md
 
 package main
 
@@ -27,17 +27,16 @@ import (
 	"path/filepath"
 	"regexp"
 	"sort"
+	"strings"
 
 	"github.com/chainloop-dev/chainloop/app/controlplane/plugins"
 	"github.com/chainloop-dev/chainloop/app/controlplane/plugins/sdk/v1"
 	"github.com/go-kratos/kratos/v2/log"
 )
 
-const registrationInputHeader = "## Registration Input Schema"
-const attachmentInputHeader = "## Attachment Input Schema"
-
 // base path to the plugins directory
 var pluginsDir string
+var integrationsIndexPath string
 
 // Enhance README.md files for the registrations with the registration and attachment input schemas
 func mainE() error {
@@ -48,47 +47,113 @@ func mainE() error {
 		return fmt.Errorf("failed to load plugins: %w", err)
 	}
 
+	// Update the list of available plugins
+	// Update each readme file
 	for _, e := range plugins {
-		// Find README file and extract its content
-		file, err := os.OpenFile(filepath.Join(pluginsDir, e.Describe().ID, "v1", "README.md"), os.O_RDWR, 0644)
-		if err != nil {
-			return fmt.Errorf("failed to open README.md file: %w", err)
+		if err := updatePluginReadme(e); err != nil {
+			return fmt.Errorf("failed to update README.md file: %w", err)
 		}
+	}
 
-		fileContent, err := io.ReadAll(file)
-		if err != nil {
-			return fmt.Errorf("failed to read README.md file: %w", err)
-		}
+	// Update integrations index file
+	if err := updateIntegrationsIndex(plugins); err != nil {
+		return fmt.Errorf("failed to update integrations index: %w", err)
+	}
 
-		// Replace/Add registration input schema
-		fileContent, err = addSchemaToSection(fileContent, registrationInputHeader, e.Describe().RegistrationJSONSchema)
-		if err != nil {
-			return fmt.Errorf("failed to add registration schema to README.md file: %w", err)
-		}
+	return nil
+}
 
-		// Replace/Add attachment input schema
-		fileContent, err = addSchemaToSection(fileContent, attachmentInputHeader, e.Describe().AttachmentJSONSchema)
-		if err != nil {
-			return fmt.Errorf("failed to add attachment schema to README.md file: %w", err)
-		}
+func updateIntegrationsIndex(plugins sdk.AvailablePlugins) error {
+	const indexHeader = "## Available Integrations"
 
-		// Write the new content in the file
-		_, err = file.Seek(0, 0)
-		if err != nil {
-			return fmt.Errorf("failed to seek README.md file: %w", err)
-		}
+	// Find README integrationsIndex and extract its content
+	indexFile, err := os.OpenFile(integrationsIndexPath, os.O_RDWR, 0644)
+	if err != nil {
+		return fmt.Errorf("failed to open index file %q: %w", integrationsIndexPath, err)
+	}
+	defer indexFile.Close()
 
-		_, err = file.Write(fileContent)
-		if err != nil {
-			return fmt.Errorf("failed to write README.md file: %w", err)
+	fileContent, err := io.ReadAll(indexFile)
+	if err != nil {
+		return fmt.Errorf("failed to read file %q: %w", integrationsIndexPath, err)
+	}
+
+	indexTable := "| ID | Version | Description | Material Requirement |\n| --- | --- | --- | --- |\n"
+	for _, p := range plugins {
+		info := p.Describe()
+		// Load the materials
+		var subscribedMaterials = make([]string, 0)
+		for _, m := range info.SubscribedInputs.Materials {
+			subscribedMaterials = append(subscribedMaterials, m.Type.String())
 		}
 
-		_ = l.Log(log.LevelInfo, "msg", "README.md file updated", "plugin", e.Describe().ID)
+		// We need to full URL path because we render this file in the website
+		const repoBase = "https://github.com/chainloop-dev/chainloop/blob/main/app/controlplane/plugins/core"
+		pathToPlugin := filepath.Join(repoBase, p.Describe().ID, "v1", "README.md")
+
+		indexTable += fmt.Sprintf("| [%s](%s) | %s | %s | %s |\n", info.ID, pathToPlugin, info.Version, info.Description, strings.Join(subscribedMaterials, ", "))
+	}
+
+	// Replace the table
+	section := indexHeader + "\n\n" + indexTable + "\n"
+	// Find the content that starts with the indexHeader and contains a markdown table
+	// letters, |, _, -, \n, separators, ... are allowed between the indexHeader and the table
+	r := regexp.MustCompile(fmt.Sprintf("%s\n*[\\w|\\||\\-|\\s|\\.|_|\\[|\\]|\\(|\\)|\\/|:]*", indexHeader))
+
+	fileContent = r.ReplaceAllLiteral(fileContent, []byte(section))
+
+	return truncateAndWriteFile(indexFile, fileContent)
+}
+
+func truncateAndWriteFile(f *os.File, content []byte) error {
+	// Write the new content in the file
+	if err := f.Truncate(0); err != nil {
+		return fmt.Errorf("failed to truncate file: %w", err)
+	}
+
+	if _, err := f.Seek(0, 0); err != nil {
+		return fmt.Errorf("failed to seek file: %w", err)
+	}
+
+	_, err := f.Write(content)
+	if err != nil {
+		return fmt.Errorf("failed to write file %q: %w", integrationsIndexPath, err)
 	}
 
 	return nil
 }
 
+func updatePluginReadme(p sdk.FanOut) error {
+	const registrationInputHeader = "## Registration Input Schema"
+	const attachmentInputHeader = "## Attachment Input Schema"
+
+	// Find README file and extract its content
+	file, err := os.OpenFile(filepath.Join(pluginsDir, p.Describe().ID, "v1", "README.md"), os.O_RDWR, 0644)
+	if err != nil {
+		return fmt.Errorf("failed to open README.md file: %w", err)
+	}
+	defer file.Close()
+
+	fileContent, err := io.ReadAll(file)
+	if err != nil {
+		return fmt.Errorf("failed to read README.md file: %w", err)
+	}
+
+	// Replace/Add registration input schema
+	fileContent, err = addSchemaToSection(fileContent, registrationInputHeader, p.Describe().RegistrationJSONSchema)
+	if err != nil {
+		return fmt.Errorf("failed to add registration schema to README.md file: %w", err)
+	}
+
+	// Replace/Add attachment input schema
+	fileContent, err = addSchemaToSection(fileContent, attachmentInputHeader, p.Describe().AttachmentJSONSchema)
+	if err != nil {
+		return fmt.Errorf("failed to add attachment schema to README.md file: %w", err)
+	}
+
+	return truncateAndWriteFile(file, fileContent)
+}
+
 func main() {
 	if err := mainE(); err != nil {
 		panic(err)
@@ -97,6 +162,7 @@ func main() {
 
 func init() {
 	flag.StringVar(&pluginsDir, "dir", "", "base directory for plugins i.e ./core")
+	flag.StringVar(&integrationsIndexPath, "integrations-index-path", "", "integrations list markdown file i.e docs/integrations.md")
 	flag.Parse()
 }
 

docs/integrations.md

@@ -0,0 +1,122 @@
+# Chainloop Integrations
+
+Operators can set up third-party integrations that extend Chainloop functionality by operating on your attestation metadata. This logic could go from sending a Slack message, uploading the attestation to a storage backend or sending a Software Bill Of Materials (SBOMs) to a third-party for analysis.
+
+Below you can find the list of currently available integrations. If you can't find the integration you are looking for, feel free [to reach out](https://github.com/chainloop-dev/chainloop/issues) or [contribute your own](https://github.com/chainloop-dev/chainloop/blob/main/app/controlplane/plugins/README.md)!
+
+![FanOut Plugin](./img/fanout.png)
+
+## Available Integrations
+
+| ID | Version | Description | Material Requirement |
+| --- | --- | --- | --- |
+| [dependency-track](https:/github.com/chainloop-dev/chainloop/blob/main/app/controlplane/plugins/core/dependency-track/v1/README.md) | 1.2 | Send CycloneDX SBOMs to your Dependency-Track instance | SBOM_CYCLONEDX_JSON |
+| [smtp](https:/github.com/chainloop-dev/chainloop/blob/main/app/controlplane/plugins/core/smtp/v1/README.md) | 1.0 | Send emails with information about a received attestation |  |
+| [oci-registry](https:/github.com/chainloop-dev/chainloop/blob/main/app/controlplane/plugins/core/oci-registry/v1/README.md) | 1.0 | Send attestations to a compatible OCI registry |  |
+| [discord-webhook](https:/github.com/chainloop-dev/chainloop/blob/main/app/controlplane/plugins/core/discord-webhook/v1/README.md) | 1.1 | Send attestations to Discord |  |
+
+## How to use integrations
+
+First you need to make sure that the integration that you are looking for is available in your Chainloop instance, to do so you will
+
+```console
+$ chainloop integration available list
+┌──────────────────┬─────────┬──────────────────────┬───────────────────────────────────────────────────────────┐
+│ ID               │ VERSION │ MATERIAL REQUIREMENT │ DESCRIPTION                                               │
+├──────────────────┼─────────┼──────────────────────┼───────────────────────────────────────────────────────────┤
+│ dependency-track │ 1.2     │ SBOM_CYCLONEDX_JSON  │ Send CycloneDX SBOMs to your Dependency-Track instance    │
+├──────────────────┼─────────┼──────────────────────┼───────────────────────────────────────────────────────────┤
+│ smtp             │ 1.0     │                      │ Send emails with information about a received attestation │
+├──────────────────┼─────────┼──────────────────────┼───────────────────────────────────────────────────────────┤
+│ oci-registry     │ 1.0     │                      │ Send attestations to a compatible OCI registry            │
+├──────────────────┼─────────┼──────────────────────┼───────────────────────────────────────────────────────────┤
+│ discord-webhook  │ 1.1     │                      │ Send attestations to Discord                              │
+└──────────────────┴─────────┴──────────────────────┴───────────────────────────────────────────────────────────┘
+```
+
+Once you find the integration you are looking for, i.e `oci-registry`, it's time to configure them.
+
+Configuring an extension has two steps: 1) register the extension in your organization and 2)attach the extension to your workflows.
+
+### Registering an extension
+
+Registration is when a specific instance of the integration is configured on a Chainloop organization. A registered instance is then available to be attached to any workflow.
+
+In our case, we want to register an instance of the `oci-registry` integration. To do so, we need to first figure out what configuration parameters are required by the integration. We can do so by running:
+
+```console
+chainloop integration available describe --id oci-registry   
+┌──────────────┬─────────┬──────────────────────┬────────────────────────────────────────────────┐
+│ ID           │ VERSION │ MATERIAL REQUIREMENT │ DESCRIPTION                                    │
+├──────────────┼─────────┼──────────────────────┼────────────────────────────────────────────────┤
+│ oci-registry │ 1.1     │                      │ Send attestations to a compatible OCI registry │
+└──────────────┴─────────┴──────────────────────┴────────────────────────────────────────────────┘
+┌──────────────────────────────────────────────────────────────┐
+│ Registration inputs                                          │
+├────────────┬────────┬──────────┬─────────────────────────────┤
+│ FIELD      │ TYPE   │ REQUIRED │ DESCRIPTION                 │
+├────────────┼────────┼──────────┼─────────────────────────────┤
+│ password   │ string │ yes      │ OCI repository password     │
+│ repository │ string │ yes      │ OCI repository uri and path │
+│ username   │ string │ yes      │ OCI repository username     │
+└────────────┴────────┴──────────┴─────────────────────────────┘
+┌─────────────────────────────────────────────────────────────────────────┐
+│ Attachment inputs                                                       │
+├────────┬────────┬──────────┬────────────────────────────────────────────┤
+│ FIELD  │ TYPE   │ REQUIRED │ DESCRIPTION                                │
+├────────┼────────┼──────────┼────────────────────────────────────────────┤
+│ prefix │ string │ no       │ OCI images name prefix (default chainloop) │
+└────────┴────────┴──────────┴────────────────────────────────────────────┘
+```
+
+In the console output we can see a registration section that indicates that 3 parameters are required, let's go ahead and register it by using our Google Artifact Registry Credentials, for example.
+
+```console
+$ chainloop integration registered add oci-registry \
+    # i.e us-east1-docker.pkg.dev/my-project/chainloop-cas-devel
+    --opt repository=[region]-docker.pkg.dev/[my-project]/[my-repository] \
+    --opt username=_json_key \
+    --opt "password=$(cat service-account.json)"
+```
+
+> Note: You can find more examples on how to configure this specific integration [here](https://github.com/chainloop-dev/chainloop/tree/main/app/controlplane/plugins/core/oci-registry/v1)
+
+### Attaching an extension
+
+Once the integration is registered, we can attach it to any workflow. In practice this means that attestations and material information received by this workflow will be sent to the registered integration. 
+
+Attachment has at least two required parameters, the workflowID and the registered integration ID. Additionally each integration might have additional configuration parameters that could allow you to customize its behavior. In our case, on the table above, you can see that the `oci-registry` integration has an optional attachment parameter called `prefix` that allows you to customize the name of the image that will be pushed to the registry. 
+
+```console 
+$ chainloop integration attached add --workflow $WID --integration $IID --opt prefix=custom-prefix
+```
+
+Congratulations, you are done now! Any attestation or material information received by the workflow will be sent to the registered integration.
+
+## FAQ
+
+### How do I know if an integration is available?
+
+You can use the `chainloop integration available list` command to list all the available integrations.
+
+### How do I know what configuration parameters are required by an integration?
+
+You can use the `chainloop integration available describe` command to list all the required configuration parameters.
+
+### How do I know what registered integrations I have in my organization?
+
+You can use the `chainloop integration registered list` command to list all the registered integrations.
+
+You can also delete a registered integration by using the `chainloop integration registered delete` command.
+
+### How do I know what attachments I have in my organization?
+
+You can use the `chainloop integration attached list` command to list all the attachments, and detach them by using the `chainloop integration attached delete` command.
+
+### What If I can't find the integration I am looking for?
+
+If you can't find the integration you are looking for, feel free [to report it](https://github.com/chainloop-dev/chainloop/issues) or [contribute your own](https://github.com/chainloop-dev/chainloop/blob/main/app/controlplane/plugins/README.md)!
+
+### I am stuck, what do I do?
+
+If you have any questions or run into any issues, don't hesitate to reach out via our [Discord Server](https://discord.gg/f7atkaZact) or open an [Issue](https://github.com/chainloop-dev/chainloop/issues/new). We'll be happy to help.