feat(bundle): introduce bundle archive format, implement copy command to convert archive to registry and vice versa

* `werf bundle copy` command:
    * `--repo` param has been deprecated in favor of new `--from=ADDR` param
    * `--tag` param has been deprecated in favor of new `--from=ADDR` param
    * `--to-tag` param has been deprecated in favor of `--to=ADDR` param new format
    * New `--from=ADDR` and `--to=ADDR` params accept addr in different formats:
        * `archive:PATH_TO_ARCHIVE.tar.gz` — bundle archive
        * `docker://registry.example.com/group/project` — remote bundle in container registry
        * without explicitly specified schema address parsed also as a remote bundle in container registry
* Bundle archive has following structure:
    * `chart.tar.gz` — regular helm chart archive
    * `images/TAG.tar.gz` — dir contains all werf images referenced from chart values `.Values.werf.image.NAME`, named by tags

Refs werf/3p-helm#216

Signed-off-by: Timofey Kirillov <timofey.kirillov@flant.com>

Author: distorhead

cmd/werf/bundle/copy/copy.go

@@ -8,17 +8,23 @@ import (
 	"github.com/spf13/cobra"
 	helm_v3 "helm.sh/helm/v3/cmd/helm"
 
+	"github.com/werf/logboek"
 	"github.com/werf/werf/cmd/werf/common"
 	"github.com/werf/werf/pkg/deploy/bundles"
+	"github.com/werf/werf/pkg/docker_registry"
 	"github.com/werf/werf/pkg/werf"
 	"github.com/werf/werf/pkg/werf/global_warnings"
 )
 
 var cmdData struct {
-	Repo  *common.RepoData
+	// TODO(2.0): Legacy {
+	Repo  string
 	Tag   string
-	To    *common.RepoData
 	ToTag string
+	// TODO(2.0): } Legacy
+
+	From string
+	To   string
 }
 
 var commonCmdData common.CmdData
@@ -57,22 +63,16 @@ func NewCmd(ctx context.Context) *cobra.Command {
 	common.SetupInsecureHelmDependencies(&commonCmdData, cmd)
 	common.SetupSkipTlsVerifyRegistry(&commonCmdData, cmd)
 
-	cmdData.Repo = common.NewRepoData("repo", common.RepoDataOptions{OnlyAddress: true})
-	cmdData.Repo.SetupCmd(cmd)
-
-	cmdData.To = common.NewRepoData("to", common.RepoDataOptions{OnlyAddress: true})
-	cmdData.To.SetupCmd(cmd)
-
 	common.SetupLogOptions(&commonCmdData, cmd)
 	common.SetupLogProjectDir(&commonCmdData, cmd)
 	common.SetupPlatform(&commonCmdData, cmd)
 
-	defaultTag := os.Getenv("WERF_TAG")
-	if defaultTag == "" {
-		defaultTag = "latest"
-	}
-	cmd.Flags().StringVarP(&cmdData.Tag, "tag", "", defaultTag, "Provide from tag version of the bundle to copy ($WERF_TAG or latest by default)")
-	cmd.Flags().StringVarP(&cmdData.ToTag, "to-tag", "", "", "Provide to tag version of the bundle to copy ($WERF_TO_TAG or same as --tag by default)")
+	cmd.Flags().StringVarP(&cmdData.Repo, "repo", "", os.Getenv("WERF_REPO"), "Deprecated param, use --from=ADDR instead. Source address of bundle which should be copied.")
+	cmd.Flags().StringVarP(&cmdData.Tag, "tag", "", os.Getenv("WERF_TAG"), "Deprecated param, use --from=REPO:TAG instead. Provide from tag version of the bundle to copy ($WERF_TAG or latest by default).")
+	cmd.Flags().StringVarP(&cmdData.ToTag, "to-tag", "", os.Getenv("WERF_TO_TAG"), "Deprecated param, use --to=REPO:TAG instead. Provide to tag version of the bundle to copy ($WERF_TO_TAG or same as --tag by default).")
+
+	cmd.Flags().StringVarP(&cmdData.From, "from", "", os.Getenv("WERF_FROM"), "Source address of the bundle to copy, specify bundle archive using schema `archive:PATH_TO_ARCHIVE.tar.gz`, specify remote bundle with schema `[docker://]REPO:TAG` or without schema.")
+	cmd.Flags().StringVarP(&cmdData.To, "to", "", os.Getenv("WERF_TO"), "Destination address of the bundle to copy, specify bundle archive using schema `archive:PATH_TO_ARCHIVE.tar.gz`, specify remote bundle with schema `[docker://]REPO:TAG` or without schema.")
 
 	return cmd
 }
@@ -93,26 +93,68 @@ func runCopy(ctx context.Context) error {
 		return err
 	}
 
-	if *cmdData.Repo.Address == "" {
-		return fmt.Errorf("--repo=ADDRESS param required")
+	fromAddrRaw := cmdData.From
+	if fromAddrRaw != "" && cmdData.Repo != "" {
+		return fmt.Errorf("unable to use --repo=ADDRESS and --from=ADDRESS params at the same time: please specify only --from=ADDRESS param, --repo param has been deprecated")
+	} else if cmdData.Repo != "" {
+		logboek.Context(ctx).Warn().LogF("Please use --from=ADDRESS param instead of deprecated --repo=ADDRESS param\n")
+		fromAddrRaw = cmdData.Repo
+	}
+	if fromAddrRaw == "" {
+		return fmt.Errorf("--from=ADDRESS param required")
 	}
-	if *cmdData.To.Address == "" {
+
+	toAddrRaw := cmdData.To
+	if toAddrRaw == "" {
 		return fmt.Errorf("--to=ADDRESS param required")
 	}
 
-	fromRegistry, err := cmdData.Repo.CreateDockerRegistry(ctx, *commonCmdData.InsecureRegistry, *commonCmdData.SkipTlsVerifyRegistry)
+	fromAddr, err := bundles.ParseAddr(fromAddrRaw)
 	if err != nil {
-		return fmt.Errorf("error creating container registry accessor for repo %s: %w", *cmdData.Repo.Address, err)
+		return fmt.Errorf("invalid from addr %q: %w", fromAddrRaw, err)
 	}
 
-	fromTag := cmdData.Tag
-	toTag := cmdData.ToTag
-	if toTag == "" {
-		toTag = fromTag
+	toAddr, err := bundles.ParseAddr(toAddrRaw)
+	if err != nil {
+		return fmt.Errorf("invalid to addr %q: %w", toAddrRaw, err)
 	}
 
-	fromRef := fmt.Sprintf("%s:%s", *cmdData.Repo.Address, fromTag)
-	toRef := fmt.Sprintf("%s:%s", *cmdData.To.Address, toTag)
+	var fromRegistry, toRegistry docker_registry.Interface
 
-	return bundles.Copy(ctx, fromRef, toRef, bundlesRegistryClient, fromRegistry)
+	if fromAddr.RegistryAddress != nil {
+		// TODO(2.0): remove legacy compatibility param
+		if cmdData.Tag != "" {
+			logboek.Context(ctx).Warn().LogF("Please use --from=REPO:TAG tag specification instead of deprecated --tag=TAG param\n")
+			fromAddr.RegistryAddress.Tag = cmdData.Tag
+		}
+
+		fromRegistry, err = common.CreateDockerRegistry(fromAddr.RegistryAddress.Repo, *commonCmdData.InsecureRegistry, *commonCmdData.SkipTlsVerifyRegistry)
+		if err != nil {
+			return err
+		}
+	}
+
+	if toAddr.RegistryAddress != nil {
+		// TODO(2.0): remove legacy compatibility param
+		if cmdData.ToTag != "" {
+			logboek.Context(ctx).Warn().LogF("Please use --to=REPO:TAG tag specification instead of deprecated --to-tag=TAG param\n")
+			toAddr.RegistryAddress.Tag = cmdData.ToTag
+		}
+
+		toRegistry, err = common.CreateDockerRegistry(toAddr.RegistryAddress.Repo, *commonCmdData.InsecureRegistry, *commonCmdData.SkipTlsVerifyRegistry)
+		if err != nil {
+			return err
+		}
+	}
+
+	return logboek.Context(ctx).LogProcess("Copy bundle").DoError(func() error {
+		logboek.Context(ctx).LogFDetails("From: %s\n", fromAddr.String())
+		logboek.Context(ctx).LogFDetails("To: %s\n", toAddr.String())
+
+		return bundles.Copy(ctx, fromAddr, toAddr, bundles.CopyOptions{
+			BundlesRegistryClient: bundlesRegistryClient,
+			FromRegistry:          fromRegistry,
+			ToRegistry:            toRegistry,
+		})
+	})
 }

cmd/werf/common/repo_data.go

@@ -14,6 +14,20 @@ import (
 	"github.com/werf/werf/pkg/storage"
 )
 
+func CreateDockerRegistry(addr string, insecureRegistry, skipTlsVerifyRegistry bool) (docker_registry.Interface, error) {
+	regOpts := docker_registry.DockerRegistryOptions{
+		InsecureRegistry:      insecureRegistry,
+		SkipTlsVerifyRegistry: skipTlsVerifyRegistry,
+	}
+
+	dockerRegistry, err := docker_registry.NewDockerRegistry(addr, "", regOpts)
+	if err != nil {
+		return nil, fmt.Errorf("error creating container registry accessor for repo %q: %w", addr, err)
+	}
+
+	return dockerRegistry, nil
+}
+
 func (repoData *RepoData) CreateDockerRegistry(ctx context.Context, insecureRegistry, skipTlsVerifyRegistry bool) (docker_registry.Interface, error) {
 	addr, err := repoData.GetAddress()
 	if err != nil {

go.mod

@@ -314,6 +314,6 @@ replace k8s.io/helm => github.com/werf/helm v0.0.0-20210202111118-81e74d46da0f
 
 replace github.com/deislabs/oras => github.com/werf/third-party-oras v0.9.1-0.20210927171747-6d045506f4c8
 
-replace helm.sh/helm/v3 => github.com/werf/3p-helm/v3 v3.0.0-20220823144404-27eb1367786c
+replace helm.sh/helm/v3 => github.com/werf/3p-helm/v3 v3.0.0-20220902145201-6265178e3c32
 
 replace github.com/go-git/go-git/v5 => github.com/ZauberNerd/go-git/v5 v5.4.3-0.20220315170230-29ec1bc1e5db

go.sum

@@ -2050,8 +2050,8 @@ github.com/vmware/govmomi v0.20.3/go.mod h1:URlwyTFZX72RmxtxuaFL2Uj3fD1JTvZdx59b
 github.com/weppos/publicsuffix-go v0.4.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k=
 github.com/weppos/publicsuffix-go v0.5.0 h1:rutRtjBJViU/YjcI5d80t4JAVvDltS6bciJg2K1HrLU=
 github.com/weppos/publicsuffix-go v0.5.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k=
-github.com/werf/3p-helm/v3 v3.0.0-20220823144404-27eb1367786c h1:/cMYGHrCXCohS+2Thjj+7fiVVG1a/LmGC4BxZOK1sYY=
-github.com/werf/3p-helm/v3 v3.0.0-20220823144404-27eb1367786c/go.mod h1:NxtE2KObf2PrzDl6SIamPFPKyAqWi10iWuvKlQn/Yao=
+github.com/werf/3p-helm/v3 v3.0.0-20220902145201-6265178e3c32 h1:Z3XL0banUFF7IkIzY82onwWA2qiTvuQ0pBMERA/scis=
+github.com/werf/3p-helm/v3 v3.0.0-20220902145201-6265178e3c32/go.mod h1:NxtE2KObf2PrzDl6SIamPFPKyAqWi10iWuvKlQn/Yao=
 github.com/werf/copy-recurse v0.2.4 h1:kEyGUKhgS8WdEOjInNQKgk4lqPWzP2AgR27F3dcGsVc=
 github.com/werf/copy-recurse v0.2.4/go.mod h1:KVHSQ90p19xflWW0B7BJhLBwmSbEtuxIaBnjlUYRPhk=
 github.com/werf/helm v0.0.0-20210202111118-81e74d46da0f h1:81YscYTF9mmTf0ULOsCmm42YWQp+qWDzWi1HjWniZrg=

pkg/deploy/bundles/addr.go

@@ -0,0 +1,75 @@
+package bundles
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/werf/werf/pkg/deploy/bundles/registry"
+)
+
+const (
+	ArchiveSchema  = "archive:"
+	RegistrySchema = "docker://"
+)
+
+type Addr struct {
+	*ArchiveAddress
+	*RegistryAddress
+}
+
+func (addr *Addr) String() string {
+	if addr.RegistryAddress != nil {
+		return addr.RegistryAddress.FullName()
+	}
+	if addr.ArchiveAddress != nil {
+		return addr.ArchiveAddress.Path
+	}
+	return ""
+}
+
+type ArchiveAddress struct {
+	Path string
+}
+
+type RegistryAddress struct {
+	*registry.Reference
+}
+
+func ParseAddr(addr string) (*Addr, error) {
+	switch {
+	case strings.HasPrefix(addr, ArchiveSchema):
+		return &Addr{ArchiveAddress: parseArchiveAddress(addr)}, nil
+	case strings.HasPrefix(addr, RegistrySchema):
+		if regAddr, err := parseRegistryAddress(addr); err != nil {
+			return nil, fmt.Errorf("unable to parse registry address %q: %w", addr, err)
+		} else {
+			return &Addr{RegistryAddress: regAddr}, nil
+		}
+	default:
+		if regAddr, err := parseRegistryAddress(addr); err != nil {
+			return nil, fmt.Errorf("unable to parse registry address %q: %w", addr, err)
+		} else {
+			return &Addr{RegistryAddress: regAddr}, nil
+		}
+	}
+}
+
+func parseRegistryAddress(addr string) (*RegistryAddress, error) {
+	cleanAddr := strings.TrimPrefix(addr, RegistrySchema)
+
+	ref, err := registry.ParseReference(cleanAddr)
+	if err != nil {
+		return nil, err
+	}
+
+	if ref.Tag == "" {
+		ref.Tag = "latest"
+	}
+
+	return &RegistryAddress{Reference: ref}, nil
+}
+
+func parseArchiveAddress(addr string) *ArchiveAddress {
+	path := strings.TrimPrefix(addr, ArchiveSchema)
+	return &ArchiveAddress{Path: path}
+}

pkg/deploy/bundles/addr_test.go

@@ -0,0 +1,44 @@
+package bundles
+
+import (
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Bundle addr", func() {
+	It("should parse different bundle address schemas", func() {
+		{
+			addr, err := ParseAddr("registry.werf.io/group/image:mytag")
+			Expect(err).NotTo(HaveOccurred())
+			Expect(addr.RegistryAddress).NotTo(BeNil())
+			Expect(addr.ArchiveAddress).To(BeNil())
+			Expect(addr.Repo).To(Equal("registry.werf.io/group/image"))
+			Expect(addr.Tag).To(Equal("mytag"))
+		}
+
+		{
+			addr, err := ParseAddr("registry.werf.io/group/image")
+			Expect(err).NotTo(HaveOccurred())
+			Expect(addr.RegistryAddress).NotTo(BeNil())
+			Expect(addr.ArchiveAddress).To(BeNil())
+			Expect(addr.Repo).To(Equal("registry.werf.io/group/image"))
+			Expect(addr.Tag).To(Equal("latest"))
+		}
+
+		{
+			addr, err := ParseAddr("archive:path/to/file.tar.gz")
+			Expect(err).NotTo(HaveOccurred())
+			Expect(addr.RegistryAddress).To(BeNil())
+			Expect(addr.ArchiveAddress).NotTo(BeNil())
+			Expect(addr.Path).To(Equal("path/to/file.tar.gz"))
+		}
+
+		{
+			addr, err := ParseAddr("archive:/absolute/path/to/file.tar.gz")
+			Expect(err).NotTo(HaveOccurred())
+			Expect(addr.RegistryAddress).To(BeNil())
+			Expect(addr.ArchiveAddress).NotTo(BeNil())
+			Expect(addr.Path).To(Equal("/absolute/path/to/file.tar.gz"))
+		}
+	})
+})