CLIPlugin Discovery: Use builder publish to publish artifacts and support Local Discovery and Distribution

Makefile

@@ -84,9 +84,11 @@ BUILD_TAGS ?=
 ARTIFACTS_DIR ?= ./artifacts
 
 XDG_CACHE_HOME := ${HOME}/.cache
+XDG_CONFIG_HOME :=${HOME}/.config
 
 export XDG_DATA_HOME
 export XDG_CACHE_HOME
+export XDG_CONFIG_HOME
 export OCI_REGISTRY
 
 ## --------------------------------------
@@ -249,6 +251,29 @@ build-cli-local: configure-buildtags-embedproviders build-cli-${GOHOSTOS}-${GOHO
 .PHONY: build-install-cli-local
 build-install-cli-local: clean-catalog-cache clean-cli-plugins build-cli-local install-cli-plugins install-cli ## Local build and install the CLI plugins
 
+## --------------------------------------
+## Build and publish CLIPlugin Discovery resource files and binaries
+## --------------------------------------
+
+STANDALONE_PLUGINS := login management-cluster package pinniped-auth
+CONTEXT_PLUGINS := cluster kubernetes-release secret
+
+.PHONY: publish-plugins
+publish-plugins:
+	$(GO) run ./cmd/cli/plugin-admin/builder/main.go publish --type local --plugins "$(STANDALONE_PLUGINS)" --version $(BUILD_VERSION) --os-arch "${ENVS}" --local-output-discovery-dir "$(XDG_CONFIG_HOME)/tanzu-plugins/discovery/standalone" --local-output-distribution-dir "$(XDG_CONFIG_HOME)/tanzu-plugins/distribution" --input-artifact-dir $(ARTIFACTS_DIR)
+	$(GO) run ./cmd/cli/plugin-admin/builder/main.go publish --type local --plugins "$(CONTEXT_PLUGINS)" --version $(BUILD_VERSION) --os-arch "${ENVS}" --local-output-discovery-dir "$(XDG_CONFIG_HOME)/tanzu-plugins/discovery/context" --local-output-distribution-dir "$(XDG_CONFIG_HOME)/tanzu-plugins/distribution" --input-artifact-dir $(ARTIFACTS_DIR)
+
+.PHONY: publish-plugins-local
+publish-plugins-local:
+	$(GO) run ./cmd/cli/plugin-admin/builder/main.go publish --type local --plugins "$(STANDALONE_PLUGINS)" --version $(BUILD_VERSION) --os-arch "${GOHOSTOS}-${GOHOSTARCH}" --local-output-discovery-dir "$(XDG_CONFIG_HOME)/tanzu-plugins/discovery/standalone" --local-output-distribution-dir "$(XDG_CONFIG_HOME)/tanzu-plugins/distribution" --input-artifact-dir $(ARTIFACTS_DIR)
+	$(GO) run ./cmd/cli/plugin-admin/builder/main.go publish --type local --plugins "$(CONTEXT_PLUGINS)" --version $(BUILD_VERSION) --os-arch "${GOHOSTOS}-${GOHOSTARCH}" --local-output-discovery-dir "$(XDG_CONFIG_HOME)/tanzu-plugins/discovery/context" --local-output-distribution-dir "$(XDG_CONFIG_HOME)/tanzu-plugins/distribution" --input-artifact-dir $(ARTIFACTS_DIR)
+
+.PHONY: build-publish-plugins
+build-publish-plugins: clean-catalog-cache clean-cli-plugins build-cli install-cli publish-plugins
+
+.PHONY: build-publish-plugins-local
+build-publish-plugins-local: clean-catalog-cache clean-cli-plugins build-cli-local install-cli publish-plugins-local
+
 ## --------------------------------------
 ## manage cli mocks
 ## --------------------------------------

apis/cli/v1alpha1/groupversion_info.go

@@ -21,6 +21,9 @@ var (
 	// AddToScheme adds the types in this group-version to the given scheme.
 	AddToScheme = SchemeBuilder.AddToScheme
 
-	// GroupVersionKind has information about group, version and kind of this object.
-	GroupVersionKind = GroupVersion.WithKind("Catalog")
+	// GroupVersionKindCatalog has information about group, version and kind of Catalog object.
+	GroupVersionKindCatalog = GroupVersion.WithKind("Catalog")
+
+	// GroupVersionKindCLIPlugin has information about group, version and kind of CLIPlugin object.
+	GroupVersionKindCLIPlugin = GroupVersion.WithKind("CLIPlugin")
 )

cmd/cli/plugin-admin/builder/main.go

@@ -28,6 +28,7 @@ func main() {
 	p.AddCommands(
 		command.CLICmd,
 		command.NewInitCmd(),
+		command.PublishCmd,
 	)
 	if err := p.Execute(); err != nil {
 		log.Fatal(err)

docs/design/context-aware-plugin-discovery-design.md

@@ -0,0 +1,44 @@
+# Context-aware API-driven Plugin Discovery
+
+## Abstract
+
+The Tanzu CLI is an amalgamation of all the Tanzu infrastructure elements under one unified core CLI experience. The core CLI supports a plugin model where the developers of different Tanzu services (bundled or SaaS) can distribute plugins that target functionalities of the services they own. When users switch between different services via the CLI context, we want to surface only the relevant plugins for the given context for a crisp user experience.
+
+## Key Concepts
+
+- CLI - The Tanzu command line interface, built to be pluggable.
+- Service - Any tanzu service, user-managed or SaaS. E.g., TKG, TCE, TMC, etc
+- Server - An instance of service. E.g., A single TKG management-cluster, a specific TMC endpoint, etc.
+- Context - an isolated scope of relevant client-side configurations for a combination of user identity and server identity. There can be multiple contexts for the same combination of {user, server}. This is currently referred to as `Server` in the Tanzu CLI, which can also mean an instance of a service. Hence, we shall use `Context` to avoid confusion.
+- Plugin - A scoped piece of functionality that can be used to extend the CLI. Usually refers to a single binary that is invoked by the root Tanzu CLI.
+- Scope - the context association level of a plugin
+- Stand-alone - independent of the CLI context
+- Context-scoped - scoped to one or more contexts
+- Discovery - the interface to fetch the list of available plugins and their supported versions
+- Distribution - the interface to deliver a plugin for user download
+- Scheme - the specific mechanism to discover or download a plugin
+- Discovery Scheme - e.g., REST API, CLIPlugin kubernetes API, manifest YAML
+- Distribution Scheme - e.g., OCI image, Google Cloud Storage, S3
+- Discovery Source - the source server of a plugin metadata for discovery, e.g., a REST API server, a management cluster, a local manifest file, OCI compliant image containing manifest file
+- Distribution Repository - the repository of plugin binary for distribution, e.g., an OCI compliant image registry, Google Cloud Storage, an S3 compatible object storage server
+- Plugin Descriptor - the metadata about a single plugin version that is installed locally and is used by the core to construct a sub-command under `tanzu`
+
+## Background
+
+## Goals
+
+## Non Goals
+
+## High-Level Design
+
+## Detailed Design
+
+## Alternatives Considered
+
+## Security Considerations
+
+## Compatibility
+
+## Implementation
+
+## Open Issues

docs/dev/build.md

@@ -81,3 +81,11 @@ safely. With these powerful APIs the teams can modify the system behavior withou
 experimentation over the lifecyle of features, these can be incredibly useful for agile management style environments.
 
 For more detailed information on these APIs check out this [doc](../api-machinery/features-and-featuregates.md)
+
+### Context-aware API-driven Plugin Discovery
+
+The Tanzu CLI is an amalgamation of all the Tanzu infrastructure elements under one unified core CLI experience. The core CLI supports a plugin model where the developers of different Tanzu services (bundled or SaaS) can distribute plugins that target functionalities of the services they own. When users switch between different services via the CLI context, we want to surface only the relevant plugins for the given context for a crisp user experience.
+
+This feature is gated by `features.global.context-aware-discovery` CLI feature flag and can be turned on/off as described [here](../cli/config-features.md). When this feature is enabled, CLI will not automatically use already installed plugins with context-aware discovery and users will need to install plugins again with `tanzu plugin install` command.
+
+For more detailed information on these design check out this [doc](../design/context-aware-plugin-discovery.md)

go.mod

@@ -78,10 +78,12 @@ require (
 	github.com/rs/xid v1.2.1
 	github.com/satori/go.uuid v1.2.0
 	github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966
+	github.com/spf13/afero v1.2.2
 	github.com/spf13/cobra v1.1.3
 	github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper v1.7.1
 	github.com/stretchr/testify v1.7.1-0.20210427113832-6241f9ab9942
+	github.com/tj/assert v0.0.0-20171129193455-018094318fb0
 	github.com/vmware-labs/yaml-jsonpath v0.3.2
 	github.com/vmware-tanzu/carvel-kapp-controller v0.25.0
 	github.com/vmware-tanzu/carvel-secretgen-controller v0.5.0

go.sum

@@ -1161,6 +1161,7 @@ github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
 github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
 github.com/timakin/bodyclose v0.0.0-20190930140734-f7f2e9bca95e/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk=
 github.com/timakin/bodyclose v0.0.0-20200424151742-cb6215831a94/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk=
+github.com/tj/assert v0.0.0-20171129193455-018094318fb0 h1:Rw8kxzWo1mr6FSaYXjQELRe88y2KdfynXdnK72rdjtA=
 github.com/tj/assert v0.0.0-20171129193455-018094318fb0/go.mod h1:mZ9/Rh9oLWpLLDRpvE+3b7gP/C2YyLFYxNmcLnPTMe0=
 github.com/tj/go-elastic v0.0.0-20171221160941-36157cbbebc2/go.mod h1:WjeM0Oo1eNAjXGDx2yma7uG2XoyRZTq1uv3M/o7imD0=
 github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b/go.mod h1:/yhzCV0xPfx6jb1bBgRFjl5lytqVqZXEaeqWP8lTEao=

pkg/v1/builder/command/publish.go

@@ -0,0 +1,80 @@
+// Copyright 2021 VMware, Inc. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package command
+
+import (
+	"path/filepath"
+	"strings"
+
+	"github.com/pkg/errors"
+	"github.com/spf13/cobra"
+
+	"github.com/vmware-tanzu/tanzu-framework/pkg/v1/builder/command/publish"
+	"github.com/vmware-tanzu/tanzu-framework/pkg/v1/cli/common"
+)
+
+var (
+	distroType, pluginsString, oa, inputArtifactDir     string
+	localOutputDiscoveryDir, localOutputDistributionDir string
+	ociDiscoveryImage, ociDistributionImageRepository   string
+	recommendedVersion                                  string
+)
+
+// PublishCmd publishes plugin resources
+var PublishCmd = &cobra.Command{
+	Use:   "publish",
+	Short: "publish operations",
+	RunE:  publishPlugins,
+}
+
+func init() {
+	PublishCmd.Flags().StringVar(&distroType, "type", "", "type of discovery and distribution for publishing plugins. Supported: local")
+	PublishCmd.Flags().StringVar(&pluginsString, "plugins", "", "list of plugin names. Example: 'login management-cluster cluster'")
+	PublishCmd.Flags().StringVar(&inputArtifactDir, "input-artifact-dir", "", "artifact directory which is a output of 'tanzu builder cli compile' command")
+
+	PublishCmd.Flags().StringVar(&oa, "os-arch", common.DefaultOSArch, "list of os-arch")
+	PublishCmd.Flags().StringVar(&recommendedVersion, "version", "", "recommended version of the plugins")
+
+	PublishCmd.Flags().StringVar(&localOutputDiscoveryDir, "local-output-discovery-dir", "", "local output directory where CLIPlugin resource yamls for discovery will be placed. Applicable to 'local' type")
+	PublishCmd.Flags().StringVar(&localOutputDistributionDir, "local-output-distribution-dir", "", "local output directory where plugin binaries will be placed. Applicable to 'local' type")
+
+	PublishCmd.Flags().StringVar(&ociDiscoveryImage, "oci-discovery-image", "", "image path to publish oci image with CLIPlugin resource yamls. Applicable to 'oci' type")
+	PublishCmd.Flags().StringVar(&ociDistributionImageRepository, "oci-distribution-image-repository", "", "image path prefix to publish oci image for plugin binaries. Applicable to 'oci' type")
+
+	_ = PublishCmd.MarkFlagRequired("type")
+	_ = PublishCmd.MarkFlagRequired("version")
+	_ = PublishCmd.MarkFlagRequired("plugins")
+	_ = PublishCmd.MarkFlagRequired("input-artifact-dir")
+}
+
+func publishPlugins(cmd *cobra.Command, args []string) error {
+	plugins := strings.Split(pluginsString, " ")
+	osArch := strings.Split(oa, " ")
+
+	if localOutputDiscoveryDir == "" {
+		localOutputDiscoveryDir = filepath.Join(common.DefaultLocalPluginDistroDir, "discovery", "oci")
+	}
+
+	var publisherInterface publish.Publisher
+
+	switch strings.ToLower(distroType) {
+	case "local":
+		publisherInterface = publish.NewLocalPublisher(localOutputDistributionDir)
+	case "oci":
+		publisherInterface = publish.NewOCIPublisher(ociDiscoveryImage, ociDistributionImageRepository, localOutputDiscoveryDir)
+	default:
+		return errors.Errorf("publish plugins with type %s is not yet supported", distroType)
+	}
+
+	publishMetadata := publish.Metadata{
+		Plugins:            plugins,
+		OSArch:             osArch,
+		RecommendedVersion: recommendedVersion,
+		InputArtifactDir:   inputArtifactDir,
+		LocalDiscoveryPath: localOutputDiscoveryDir,
+		PublisherInterface: publisherInterface,
+	}
+
+	return publish.PublishPlugins(&publishMetadata)
+}