Prune operator version

.gitignore

@@ -472,3 +472,5 @@ test/e2e/index_tmp*
 
 # don't check in the certs directory
 certs/*
+
+index_tmp_*

cmd/opm/index/cmd.go

@@ -31,6 +31,7 @@ func AddCommand(parent *cobra.Command) {
 	addIndexAddCmd(cmd)
 	cmd.AddCommand(newIndexExportCmd())
 	cmd.AddCommand(newIndexPruneCmd())
+	cmd.AddCommand(newIndexPruneVersionCmd())
 	cmd.AddCommand(newIndexDeprecateTruncateCmd())
 	cmd.AddCommand(newIndexPruneStrandedCmd())
 }

cmd/opm/index/pruneversion.go

@@ -0,0 +1,127 @@
+package index
+
+import (
+	"fmt"
+
+	"github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+
+	"github.com/operator-framework/operator-registry/pkg/containertools"
+	"github.com/operator-framework/operator-registry/pkg/lib/indexer"
+)
+
+func newIndexPruneVersionCmd() *cobra.Command {
+	indexCmd := &cobra.Command{
+		Hidden: true,
+		Use:    "prune-version",
+		Short:  "prune an index of all but specified package versions",
+		Long:   `prune an index of all but specified package versions`,
+
+		PreRunE: func(cmd *cobra.Command, args []string) error {
+			if debug, _ := cmd.Flags().GetBool("debug"); debug {
+				logrus.SetLevel(logrus.DebugLevel)
+			}
+			return nil
+		},
+
+		RunE: runIndexPruneVersionCmdFunc,
+	}
+
+	indexCmd.Flags().Bool("debug", false, "enable debug logging")
+	indexCmd.Flags().Bool("generate", false, "if enabled, just creates the dockerfile and saves it to local disk")
+	indexCmd.Flags().StringP("out-dockerfile", "d", "", "if generating the dockerfile, this flag is used to (optionally) specify a dockerfile name")
+	indexCmd.Flags().StringP("from-index", "f", "", "index to prune")
+	if err := indexCmd.MarkFlagRequired("from-index"); err != nil {
+		logrus.Panic("Failed to set required `from-index` flag for `index prune`")
+	}
+	indexCmd.Flags().StringSliceP("package-versions", "p", nil, "comma separated list of package and versions to keep")
+	if err := indexCmd.MarkFlagRequired("package-versions"); err != nil {
+		logrus.Panic("Failed to set required `package-versions` flag for `index pruneversion`")
+	}
+	indexCmd.Flags().StringP("binary-image", "i", "", "container image for on-image `opm` command")
+	indexCmd.Flags().StringP("container-tool", "c", "podman", "tool to interact with container images (save, build, etc.). One of: [docker, podman]")
+	indexCmd.Flags().StringP("tag", "t", "", "custom tag for container image being built")
+	indexCmd.Flags().Bool("permissive", false, "allow registry load errors")
+
+	if err := indexCmd.Flags().MarkHidden("debug"); err != nil {
+		logrus.Panic(err.Error())
+	}
+
+	return indexCmd
+
+}
+
+func runIndexPruneVersionCmdFunc(cmd *cobra.Command, args []string) error {
+	generate, err := cmd.Flags().GetBool("generate")
+	if err != nil {
+		return err
+	}
+
+	outDockerfile, err := cmd.Flags().GetString("out-dockerfile")
+	if err != nil {
+		return err
+	}
+
+	fromIndex, err := cmd.Flags().GetString("from-index")
+	if err != nil {
+		return err
+	}
+
+	packageVersions, err := cmd.Flags().GetStringSlice("package-versions")
+	if err != nil {
+		return err
+	}
+
+	binaryImage, err := cmd.Flags().GetString("binary-image")
+	if err != nil {
+		return err
+	}
+
+	containerTool, err := cmd.Flags().GetString("container-tool")
+	if err != nil {
+		return err
+	}
+
+	if containerTool == "none" {
+		return fmt.Errorf("none is not a valid container-tool for index prune")
+	}
+
+	tag, err := cmd.Flags().GetString("tag")
+	if err != nil {
+		return err
+	}
+
+	permissive, err := cmd.Flags().GetBool("permissive")
+	if err != nil {
+		return err
+	}
+
+	skipTLS, err := cmd.Flags().GetBool("skip-tls")
+	if err != nil {
+		return err
+	}
+
+	logger := logrus.WithFields(logrus.Fields{"package-versions": packageVersions})
+
+	logger.Info("pruning the index")
+
+	indexPruner := indexer.NewIndexPruner(containertools.NewContainerTool(containerTool, containertools.PodmanTool), logger)
+
+	request := indexer.PruneVersionFromIndexRequest{
+		Generate:          generate,
+		FromIndex:         fromIndex,
+		BinarySourceImage: binaryImage,
+		OutDockerfile:     outDockerfile,
+		PackageVersions:   packageVersions,
+		Tag:               tag,
+		Permissive:        permissive,
+		SkipTLS:           skipTLS,
+	}
+
+	err = indexPruner.PruneVersionFromIndex(request)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

pkg/lib/indexer/indexer.go

@@ -293,6 +293,64 @@ func (i ImageIndexer) PruneFromIndex(request PruneFromIndexRequest) error {
 	return nil
 }
 
+// PruneFromIndexRequest defines the parameters to send to the PruneFromIndex API
+type PruneVersionFromIndexRequest struct {
+	Generate          bool
+	Permissive        bool
+	BinarySourceImage string
+	FromIndex         string
+	OutDockerfile     string
+	Tag               string
+	PackageVersions   []string
+	CaFile            string
+	SkipTLS           bool
+}
+
+func (i ImageIndexer) PruneVersionFromIndex(request PruneVersionFromIndexRequest) error {
+	buildDir, outDockerfile, cleanup, err := buildContext(request.Generate, request.OutDockerfile)
+	defer cleanup()
+	if err != nil {
+		return err
+	}
+
+	databasePath, err := i.ExtractDatabase(buildDir, request.FromIndex, request.CaFile, request.SkipTLS)
+	if err != nil {
+		return err
+	}
+
+	// Run opm registry prune on the database
+	pruneFromRegistryReq := registry.PruneVersionFromRegistryRequest{
+		PackageVersions: request.PackageVersions,
+		InputDatabase:   databasePath,
+		Permissive:      request.Permissive,
+	}
+
+	// Prune the bundles from the registry
+	err = i.RegistryPruner.PruneVersionFromRegistry(pruneFromRegistryReq)
+	if err != nil {
+		return err
+	}
+
+	// generate the dockerfile
+	dockerfile := i.DockerfileGenerator.GenerateIndexDockerfile(request.BinarySourceImage, databasePath)
+	err = write(dockerfile, outDockerfile, i.Logger)
+	if err != nil {
+		return err
+	}
+
+	if request.Generate {
+		return nil
+	}
+
+	// build the dockerfile
+	err = build(outDockerfile, request.Tag, i.CommandRunner, i.Logger)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 // ExtractDatabase sets a temp directory for unpacking an image
 func (i ImageIndexer) ExtractDatabase(buildDir, fromIndex, caFile string, skipTLS bool) (string, error) {
 	tmpDir, err := ioutil.TempDir("./", tmpDirPrefix)

pkg/lib/indexer/interfaces.go

@@ -84,6 +84,7 @@ func NewIndexStrandedPruner(containerTool containertools.ContainerTool, logger *
 // IndexPruner prunes operators out of an index
 type IndexPruner interface {
 	PruneFromIndex(PruneFromIndexRequest) error
+	PruneVersionFromIndex(PruneVersionFromIndexRequest) error
 }
 
 func NewIndexPruner(containerTool containertools.ContainerTool, logger *logrus.Entry) IndexPruner {

pkg/lib/registry/interfaces.go

@@ -39,6 +39,7 @@ func NewRegistryStrandedPruner(logger *logrus.Entry) RegistryStrandedPruner {
 
 type RegistryPruner interface {
 	PruneFromRegistry(PruneFromRegistryRequest) error
+	PruneVersionFromRegistry(PruneVersionFromRegistryRequest) error
 }
 
 func NewRegistryPruner(logger *logrus.Entry) RegistryPruner {

pkg/lib/registry/registry.go

@@ -5,8 +5,11 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
+	"sort"
+	"strings"
 
 	"github.com/sirupsen/logrus"
+	"golang.org/x/mod/semver"
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 
 	"github.com/operator-framework/operator-registry/pkg/containertools"
@@ -321,12 +324,154 @@ func (r RegistryUpdater) PruneFromRegistry(request PruneFromRegistryRequest) err
 	return nil
 }
 
+type PruneVersionFromRegistryRequest struct {
+	Permissive      bool
+	InputDatabase   string
+	PackageVersions []string
+}
+
+func (r RegistryUpdater) PruneVersionFromRegistry(request PruneVersionFromRegistryRequest) error {
+	// First we'll prune the packages
+	// Create a map of the operator and versions we want to keep
+	operatorVerMap := make(map[string][]string)
+	for _, pkgVersion := range request.PackageVersions {
+		split := strings.Split(pkgVersion, ":")
+		operatorVerMap[split[0]] = append(operatorVerMap[split[0]], split[1])
+	}
+
+	// now we sort those lists of versions for later (might only contain one version each)
+	for _, versionList := range operatorVerMap {
+		sort.Slice(versionList, func(i, j int) bool {
+			return semver.Compare(versionList[i], versionList[j]) < 0
+		})
+	}
+	packageList := make([]string, 0, len(operatorVerMap))
+	for operatorName := range operatorVerMap {
+		packageList = append(packageList, operatorName)
+	}
+
+	logrus.Info(fmt.Sprintf("Keeping %s", packageList))
+
+	prunePackageReq := PruneFromRegistryRequest{
+		Permissive:    request.Permissive,
+		InputDatabase: request.InputDatabase,
+		Packages:      packageList,
+	}
+	r.PruneFromRegistry(prunePackageReq)
+
+	// Now we go delete the versions we don't want
+	db, err := sqlite.Open(request.InputDatabase)
+	if err != nil {
+		return err
+	}
+	defer db.Close()
+
+	dbLoader, err := sqlite.NewSQLLiteLoader(db)
+	if err != nil {
+		return err
+	}
+	if err := dbLoader.Migrate(context.TODO()); err != nil {
+		return err
+	}
+
+	// get all the packages
+	lister := sqlite.NewSQLLiteQuerierFromDb(db)
+	if err != nil {
+		return err
+	}
+
+	// prune packages from registry
+	for operatorName, versionList := range operatorVerMap {
+		operatorBundleVersions := make(map[string]bool)
+		for _, version := range versionList {
+			operatorBundleVersions[version] = true
+		}
+		// bundlesForPackage, err := lister.GetBundlesForPackage(context.TODO(), operatorName)
+		channelEntriesForPackage, err := lister.GetChannelEntriesFromPackage(context.TODO(), operatorName)
+		if err != nil {
+			return err
+		}
+
+		for _, channelEntryForPackage := range channelEntriesForPackage {
+			// Find the newest of the package version for this channel (otherwise we lose everything if we delete)
+			// the head bundle
+			channel := channelEntryForPackage.ChannelName
+			bundleToSave := findNewestVersionToSave(channelEntriesForPackage, operatorVerMap[channelEntryForPackage.PackageName], channel)
+			if err != nil {
+				return err
+			}
+
+			// Check our map to see if the bundle we found is in the list of bundles we want to keep
+			if _, found := operatorBundleVersions[channelEntryForPackage.Version]; !found {
+				// if not, then we delete that bundle
+				remover := sqlite.NewSQLRemoverForOperatorCsvNames(dbLoader, channelEntryForPackage.BundleName, bundleToSave)
+				if err := remover.Remove(); err != nil {
+					err = fmt.Errorf("error deleting bundles by operator csv name from database: %s", err)
+					if !request.Permissive {
+						logrus.WithError(err).Fatal("permissive mode disabled")
+						return err
+					}
+					logrus.WithError(err).Warn("permissive mode enabled")
+				}
+			}
+		}
+	}
+
+	return nil
+}
+
 type DeprecateFromRegistryRequest struct {
 	Permissive    bool
 	InputDatabase string
 	Bundles       []string
 }
 
+func findNewestVersionToSave(channelEntries []registry.ChannelEntryAnnotated, operatorVerList []string, channelName string) *string {
+	// filter the channel entries for the specific channel name
+	filteredChannelEntries := []registry.ChannelEntryAnnotated{}
+	for _, channelEntryForPackage := range channelEntries {
+		if channelEntryForPackage.ChannelName == channelName {
+			filteredChannelEntries = append(filteredChannelEntries, channelEntryForPackage)
+		}
+	}
+
+	sort.Slice(filteredChannelEntries, func(i, j int) bool {
+		return semver.Compare(filteredChannelEntries[i].Version, filteredChannelEntries[j].Version) < 0
+	})
+
+	// Find all the versions that the user requested that are also in this channel
+	filteredOperatorVerList := []string{}
+	// this probably could be improved
+	for _, operatorVer := range operatorVerList {
+		for _, channelEntry := range filteredChannelEntries {
+			if semver.Compare(operatorVer, channelEntry.Version) == 0 {
+				filteredOperatorVerList = append(filteredOperatorVerList, operatorVer)
+			}
+		}
+	}
+
+	// if the list is empty, then we didn't find any that matched this channel
+	if len(filteredOperatorVerList) == 0 {
+		return nil
+	}
+
+	// now sort it to get the highest version we want to save for this channel
+	sort.Slice(filteredOperatorVerList, func(i, j int) bool {
+		return semver.Compare(filteredOperatorVerList[i], filteredOperatorVerList[j]) < 0
+	})
+
+	highestVersion := filteredOperatorVerList[len(filteredOperatorVerList)-1]
+
+	for _, i := range filteredChannelEntries {
+		if i.Version == highestVersion {
+			return &i.BundleName
+		}
+	}
+
+	// If we get here, there's no version we could find in this channel that we'd want to save
+	return nil
+}
+
 func (r RegistryUpdater) DeprecateFromRegistry(request DeprecateFromRegistryRequest) error {
 	db, err := sqlite.Open(request.InputDatabase)
 	if err != nil {

pkg/registry/interface.go

@@ -12,6 +12,7 @@ type Load interface {
 	AddPackageChannels(manifest PackageManifest) error
 	AddBundlePackageChannels(manifest PackageManifest, bundle *Bundle) error
 	RemovePackage(packageName string) error
+	RemoveBundle(csvToRemove string, csvToSave *string) error
 	RemoveStrandedBundles() error
 	DeprecateBundle(path string) error
 	ClearNonHeadBundles() error