diff --git a/.gitignore b/.gitignore index 0dc911b82..e16f84274 100644 --- a/.gitignore +++ b/.gitignore @@ -472,3 +472,5 @@ test/e2e/index_tmp* # don't check in the certs directory certs/* + +index_tmp_* diff --git a/cmd/opm/index/cmd.go b/cmd/opm/index/cmd.go index c555a7253..e921a0d88 100644 --- a/cmd/opm/index/cmd.go +++ b/cmd/opm/index/cmd.go @@ -31,6 +31,7 @@ func AddCommand(parent *cobra.Command) { addIndexAddCmd(cmd) cmd.AddCommand(newIndexExportCmd()) cmd.AddCommand(newIndexPruneCmd()) + cmd.AddCommand(newIndexPruneVersionCmd()) cmd.AddCommand(newIndexDeprecateTruncateCmd()) cmd.AddCommand(newIndexPruneStrandedCmd()) } diff --git a/cmd/opm/index/pruneversion.go b/cmd/opm/index/pruneversion.go new file mode 100644 index 000000000..31eb80383 --- /dev/null +++ b/cmd/opm/index/pruneversion.go @@ -0,0 +1,127 @@ +package index + +import ( + "fmt" + + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + + "github.com/operator-framework/operator-registry/pkg/containertools" + "github.com/operator-framework/operator-registry/pkg/lib/indexer" +) + +func newIndexPruneVersionCmd() *cobra.Command { + indexCmd := &cobra.Command{ + Hidden: true, + Use: "prune-version", + Short: "prune an index of all but specified package versions", + Long: `prune an index of all but specified package versions`, + + PreRunE: func(cmd *cobra.Command, args []string) error { + if debug, _ := cmd.Flags().GetBool("debug"); debug { + logrus.SetLevel(logrus.DebugLevel) + } + return nil + }, + + RunE: runIndexPruneVersionCmdFunc, + } + + indexCmd.Flags().Bool("debug", false, "enable debug logging") + indexCmd.Flags().Bool("generate", false, "if enabled, just creates the dockerfile and saves it to local disk") + indexCmd.Flags().StringP("out-dockerfile", "d", "", "if generating the dockerfile, this flag is used to (optionally) specify a dockerfile name") + indexCmd.Flags().StringP("from-index", "f", "", "index to prune") + if err := indexCmd.MarkFlagRequired("from-index"); err != nil { + logrus.Panic("Failed to set required `from-index` flag for `index prune`") + } + indexCmd.Flags().StringSliceP("package-versions", "p", nil, "comma separated list of package and versions to keep") + if err := indexCmd.MarkFlagRequired("package-versions"); err != nil { + logrus.Panic("Failed to set required `package-versions` flag for `index pruneversion`") + } + indexCmd.Flags().StringP("binary-image", "i", "", "container image for on-image `opm` command") + indexCmd.Flags().StringP("container-tool", "c", "podman", "tool to interact with container images (save, build, etc.). One of: [docker, podman]") + indexCmd.Flags().StringP("tag", "t", "", "custom tag for container image being built") + indexCmd.Flags().Bool("permissive", false, "allow registry load errors") + + if err := indexCmd.Flags().MarkHidden("debug"); err != nil { + logrus.Panic(err.Error()) + } + + return indexCmd + +} + +func runIndexPruneVersionCmdFunc(cmd *cobra.Command, args []string) error { + generate, err := cmd.Flags().GetBool("generate") + if err != nil { + return err + } + + outDockerfile, err := cmd.Flags().GetString("out-dockerfile") + if err != nil { + return err + } + + fromIndex, err := cmd.Flags().GetString("from-index") + if err != nil { + return err + } + + packageVersions, err := cmd.Flags().GetStringSlice("package-versions") + if err != nil { + return err + } + + binaryImage, err := cmd.Flags().GetString("binary-image") + if err != nil { + return err + } + + containerTool, err := cmd.Flags().GetString("container-tool") + if err != nil { + return err + } + + if containerTool == "none" { + return fmt.Errorf("none is not a valid container-tool for index prune") + } + + tag, err := cmd.Flags().GetString("tag") + if err != nil { + return err + } + + permissive, err := cmd.Flags().GetBool("permissive") + if err != nil { + return err + } + + skipTLS, err := cmd.Flags().GetBool("skip-tls") + if err != nil { + return err + } + + logger := logrus.WithFields(logrus.Fields{"package-versions": packageVersions}) + + logger.Info("pruning the index") + + indexPruner := indexer.NewIndexPruner(containertools.NewContainerTool(containerTool, containertools.PodmanTool), logger) + + request := indexer.PruneVersionFromIndexRequest{ + Generate: generate, + FromIndex: fromIndex, + BinarySourceImage: binaryImage, + OutDockerfile: outDockerfile, + PackageVersions: packageVersions, + Tag: tag, + Permissive: permissive, + SkipTLS: skipTLS, + } + + err = indexPruner.PruneVersionFromIndex(request) + if err != nil { + return err + } + + return nil +} diff --git a/pkg/lib/indexer/indexer.go b/pkg/lib/indexer/indexer.go index 1f8cf22f1..4a863c4d5 100644 --- a/pkg/lib/indexer/indexer.go +++ b/pkg/lib/indexer/indexer.go @@ -293,6 +293,64 @@ func (i ImageIndexer) PruneFromIndex(request PruneFromIndexRequest) error { return nil } +// PruneFromIndexRequest defines the parameters to send to the PruneFromIndex API +type PruneVersionFromIndexRequest struct { + Generate bool + Permissive bool + BinarySourceImage string + FromIndex string + OutDockerfile string + Tag string + PackageVersions []string + CaFile string + SkipTLS bool +} + +func (i ImageIndexer) PruneVersionFromIndex(request PruneVersionFromIndexRequest) error { + buildDir, outDockerfile, cleanup, err := buildContext(request.Generate, request.OutDockerfile) + defer cleanup() + if err != nil { + return err + } + + databasePath, err := i.ExtractDatabase(buildDir, request.FromIndex, request.CaFile, request.SkipTLS) + if err != nil { + return err + } + + // Run opm registry prune on the database + pruneFromRegistryReq := registry.PruneVersionFromRegistryRequest{ + PackageVersions: request.PackageVersions, + InputDatabase: databasePath, + Permissive: request.Permissive, + } + + // Prune the bundles from the registry + err = i.RegistryPruner.PruneVersionFromRegistry(pruneFromRegistryReq) + if err != nil { + return err + } + + // generate the dockerfile + dockerfile := i.DockerfileGenerator.GenerateIndexDockerfile(request.BinarySourceImage, databasePath) + err = write(dockerfile, outDockerfile, i.Logger) + if err != nil { + return err + } + + if request.Generate { + return nil + } + + // build the dockerfile + err = build(outDockerfile, request.Tag, i.CommandRunner, i.Logger) + if err != nil { + return err + } + + return nil +} + // ExtractDatabase sets a temp directory for unpacking an image func (i ImageIndexer) ExtractDatabase(buildDir, fromIndex, caFile string, skipTLS bool) (string, error) { tmpDir, err := ioutil.TempDir("./", tmpDirPrefix) diff --git a/pkg/lib/indexer/interfaces.go b/pkg/lib/indexer/interfaces.go index df9e6e47a..ba942847a 100644 --- a/pkg/lib/indexer/interfaces.go +++ b/pkg/lib/indexer/interfaces.go @@ -84,6 +84,7 @@ func NewIndexStrandedPruner(containerTool containertools.ContainerTool, logger * // IndexPruner prunes operators out of an index type IndexPruner interface { PruneFromIndex(PruneFromIndexRequest) error + PruneVersionFromIndex(PruneVersionFromIndexRequest) error } func NewIndexPruner(containerTool containertools.ContainerTool, logger *logrus.Entry) IndexPruner { diff --git a/pkg/lib/registry/interfaces.go b/pkg/lib/registry/interfaces.go index f392d16e3..b33ccc94f 100644 --- a/pkg/lib/registry/interfaces.go +++ b/pkg/lib/registry/interfaces.go @@ -39,6 +39,7 @@ func NewRegistryStrandedPruner(logger *logrus.Entry) RegistryStrandedPruner { type RegistryPruner interface { PruneFromRegistry(PruneFromRegistryRequest) error + PruneVersionFromRegistry(PruneVersionFromRegistryRequest) error } func NewRegistryPruner(logger *logrus.Entry) RegistryPruner { diff --git a/pkg/lib/registry/registry.go b/pkg/lib/registry/registry.go index 4232e07b0..f0b601ad0 100644 --- a/pkg/lib/registry/registry.go +++ b/pkg/lib/registry/registry.go @@ -5,8 +5,11 @@ import ( "fmt" "io/ioutil" "os" + "sort" + "strings" "github.com/sirupsen/logrus" + "golang.org/x/mod/semver" utilerrors "k8s.io/apimachinery/pkg/util/errors" "github.com/operator-framework/operator-registry/pkg/containertools" @@ -321,12 +324,154 @@ func (r RegistryUpdater) PruneFromRegistry(request PruneFromRegistryRequest) err return nil } +type PruneVersionFromRegistryRequest struct { + Permissive bool + InputDatabase string + PackageVersions []string +} + +func (r RegistryUpdater) PruneVersionFromRegistry(request PruneVersionFromRegistryRequest) error { + // First we'll prune the packages + // Create a map of the operator and versions we want to keep + operatorVerMap := make(map[string][]string) + for _, pkgVersion := range request.PackageVersions { + split := strings.Split(pkgVersion, ":") + operatorVerMap[split[0]] = append(operatorVerMap[split[0]], split[1]) + } + + // now we sort those lists of versions for later (might only contain one version each) + for _, versionList := range operatorVerMap { + sort.Slice(versionList, func(i, j int) bool { + return semver.Compare(versionList[i], versionList[j]) < 0 + }) + } + packageList := make([]string, 0, len(operatorVerMap)) + for operatorName := range operatorVerMap { + packageList = append(packageList, operatorName) + } + + logrus.Info(fmt.Sprintf("Keeping %s", packageList)) + + prunePackageReq := PruneFromRegistryRequest{ + Permissive: request.Permissive, + InputDatabase: request.InputDatabase, + Packages: packageList, + } + r.PruneFromRegistry(prunePackageReq) + + // Now we go delete the versions we don't want + db, err := sqlite.Open(request.InputDatabase) + if err != nil { + return err + } + defer db.Close() + + dbLoader, err := sqlite.NewSQLLiteLoader(db) + if err != nil { + return err + } + if err := dbLoader.Migrate(context.TODO()); err != nil { + return err + } + + // get all the packages + lister := sqlite.NewSQLLiteQuerierFromDb(db) + if err != nil { + return err + } + + // prune packages from registry + for operatorName, versionList := range operatorVerMap { + operatorBundleVersions := make(map[string]bool) + for _, version := range versionList { + operatorBundleVersions[version] = true + } + // bundlesForPackage, err := lister.GetBundlesForPackage(context.TODO(), operatorName) + channelEntriesForPackage, err := lister.GetChannelEntriesFromPackage(context.TODO(), operatorName) + if err != nil { + return err + } + + for _, channelEntryForPackage := range channelEntriesForPackage { + // Find the newest of the package version for this channel (otherwise we lose everything if we delete) + // the head bundle + channel := channelEntryForPackage.ChannelName + bundleToSave := findNewestVersionToSave(channelEntriesForPackage, operatorVerMap[channelEntryForPackage.PackageName], channel) + if err != nil { + return err + } + + // Check our map to see if the bundle we found is in the list of bundles we want to keep + if _, found := operatorBundleVersions[channelEntryForPackage.Version]; !found { + // if not, then we delete that bundle + remover := sqlite.NewSQLRemoverForOperatorCsvNames(dbLoader, channelEntryForPackage.BundleName, bundleToSave) + if err := remover.Remove(); err != nil { + err = fmt.Errorf("error deleting bundles by operator csv name from database: %s", err) + if !request.Permissive { + logrus.WithError(err).Fatal("permissive mode disabled") + return err + } + logrus.WithError(err).Warn("permissive mode enabled") + } + } + } + } + + return nil +} + type DeprecateFromRegistryRequest struct { Permissive bool InputDatabase string Bundles []string } +func findNewestVersionToSave(channelEntries []registry.ChannelEntryAnnotated, operatorVerList []string, channelName string) *string { + // filter the channel entries for the specific channel name + filteredChannelEntries := []registry.ChannelEntryAnnotated{} + for _, channelEntryForPackage := range channelEntries { + if channelEntryForPackage.ChannelName == channelName { + filteredChannelEntries = append(filteredChannelEntries, channelEntryForPackage) + } + } + + sort.Slice(filteredChannelEntries, func(i, j int) bool { + return semver.Compare(filteredChannelEntries[i].Version, filteredChannelEntries[j].Version) < 0 + }) + + // Find all the versions that the user requested that are also in this channel + filteredOperatorVerList := []string{} + // this probably could be improved + for _, operatorVer := range operatorVerList { + for _, channelEntry := range filteredChannelEntries { + if semver.Compare(operatorVer, channelEntry.Version) == 0 { + filteredOperatorVerList = append(filteredOperatorVerList, operatorVer) + } + } + } + + // if the list is empty, then we didn't find any that matched this channel + if len(filteredOperatorVerList) == 0 { + return nil + } + + // now sort it to get the highest version we want to save for this channel + sort.Slice(filteredOperatorVerList, func(i, j int) bool { + return semver.Compare(filteredOperatorVerList[i], filteredOperatorVerList[j]) < 0 + }) + + highestVersion := filteredOperatorVerList[len(filteredOperatorVerList)-1] + + for _, i := range filteredChannelEntries { + if i.Version == highestVersion { + return &i.BundleName + } + } + + // If we get here, there's no version we could find in this channel that we'd want to save + return nil +} + func (r RegistryUpdater) DeprecateFromRegistry(request DeprecateFromRegistryRequest) error { db, err := sqlite.Open(request.InputDatabase) if err != nil { diff --git a/pkg/registry/interface.go b/pkg/registry/interface.go index fd7763611..07731b1ae 100644 --- a/pkg/registry/interface.go +++ b/pkg/registry/interface.go @@ -12,6 +12,7 @@ type Load interface { AddPackageChannels(manifest PackageManifest) error AddBundlePackageChannels(manifest PackageManifest, bundle *Bundle) error RemovePackage(packageName string) error + RemoveBundle(csvToRemove string, csvToSave *string) error RemoveStrandedBundles() error DeprecateBundle(path string) error ClearNonHeadBundles() error diff --git a/pkg/sqlite/load.go b/pkg/sqlite/load.go index cfb3b7015..319046d33 100644 --- a/pkg/sqlite/load.go +++ b/pkg/sqlite/load.go @@ -972,6 +972,48 @@ func (s *sqlLoader) RemovePackage(packageName string) error { return s.RemoveStrandedBundles() } +func (s *sqlLoader) RemoveBundle(csvToRemove string, csvToSave *string) error { + if err := func() error { + tx, err := s.db.Begin() + if err != nil { + return err + } + defer func() { + tx.Rollback() + }() + + if csvToSave != nil { + s.replaceChannelHead(tx, *csvToSave, csvToRemove) + } + + if err := s.rmChannelEntry(tx, csvToRemove); err != nil { + return err + } + if err := s.rmBundle(tx, csvToRemove); err != nil { + return err + } + + return tx.Commit() + }(); err != nil { + return err + } + + return s.RemoveStrandedBundles() +} + +func (s *sqlLoader) replaceChannelHead(tx *sql.Tx, newCsvName string, oldCsvName string) error { + replaceChannelHead, err := tx.Prepare("UPDATE channel set head_operatorbundle_name=? WHERE head_operatorbundle_name=?") + if err != nil { + return err + } + defer replaceChannelHead.Close() + if _, err := replaceChannelHead.Exec(newCsvName, oldCsvName); err != nil { + return err + } + + return nil +} + func (s *sqlLoader) rmBundle(tx *sql.Tx, csvName string) error { deleteBundle, err := tx.Prepare("DELETE FROM operatorbundle WHERE operatorbundle.name=?") if err != nil { diff --git a/pkg/sqlite/load_test.go b/pkg/sqlite/load_test.go index 9cde831ae..b714fb88f 100644 --- a/pkg/sqlite/load_test.go +++ b/pkg/sqlite/load_test.go @@ -2,6 +2,7 @@ package sqlite import ( "context" + "database/sql" "encoding/json" "fmt" "strings" @@ -577,3 +578,116 @@ func TestAddBundlePropertiesFromAnnotations(t *testing.T) { }) } } + +func TestReplaceChannelHead(t *testing.T) { + type fields struct { + bundles []*registry.Bundle + pkgs []registry.PackageManifest + } + type args struct { + bundle string + } + type expected struct { + err error + alphaChannel string + stableChannel string + } + tests := []struct { + description string + fields fields + args args + expected expected + }{ + { + description: "ContainsDefaultChannel", + fields: fields{ + bundles: []*registry.Bundle{ + newBundle(t, "csv-a", "pkg-0", []string{"alpha"}, newUnstructuredCSV(t, "csv-a", "csv-b")), + newBundle(t, "csv-b", "pkg-0", []string{"alpha", "stable"}, newUnstructuredCSV(t, "csv-b", "csv-c")), + newBundle(t, "csv-c", "pkg-0", []string{"alpha", "stable"}, newUnstructuredCSV(t, "csv-c", "")), + }, + pkgs: []registry.PackageManifest{ + { + PackageName: "pkg-0", + Channels: []registry.PackageChannel{ + { + Name: "alpha", + CurrentCSVName: "csv-a", + }, + { + Name: "stable", + CurrentCSVName: "csv-b", + }, + }, + DefaultChannelName: "stable", + }, + }, + }, + args: args{ + bundle: "csv-a", + }, + expected: expected{ + err: nil, + alphaChannel: "csv-b", + stableChannel: "csv-b", + }, + }, + } + for _, tt := range tests { + t.Run(tt.description, func(t *testing.T) { + db, cleanup := CreateTestDb(t) + defer cleanup() + store, err := NewSQLLiteLoader(db) + require.NoError(t, err) + err = store.Migrate(context.TODO()) + require.NoError(t, err) + + for _, bundle := range tt.fields.bundles { + // Throw away any errors loading bundles (not testing this) + store.AddOperatorBundle(bundle) + } + + for _, pkg := range tt.fields.pkgs { + // Throw away any errors loading packages (not testing this) + store.AddPackageChannels(pkg) + } + tx, err := db.Begin() + require.NoError(t, err) + loader := store.(*sqlLoader) + err = loader.replaceChannelHead(tx, "csv-b", "csv-a") + require.NoError(t, err) + var ( + alphaChannel sql.NullString + stableChannel sql.NullString + ) + newChannelHeadRows, err := tx.Query("SELECT head_operatorbundle_name FROM channel WHERE name = ?", "alpha") + require.NoError(t, err) + if newChannelHeadRows.Next() { + if err := newChannelHeadRows.Scan(&alphaChannel); err != nil { + if nerr := newChannelHeadRows.Close(); nerr != nil { + require.NoError(t, nerr) + } + require.NoError(t, err) + } + } + newChannelHeadRows, err = tx.Query("SELECT head_operatorbundle_name FROM channel WHERE name = ?", "stable") + require.NoError(t, err) + if newChannelHeadRows.Next() { + if err := newChannelHeadRows.Scan(&stableChannel); err != nil { + if nerr := newChannelHeadRows.Close(); nerr != nil { + require.NoError(t, nerr) + } + require.NoError(t, err) + } + } + require.NoError(t, err) + require.Equal(t, tt.expected.err, err) + t.Logf("tt.expected.alphaChannel %#v", tt.expected.alphaChannel) + t.Logf("tt.expected.stableChannel %#v", tt.expected.stableChannel) + t.Logf("actual alphaChannel %#v", alphaChannel.String) + t.Logf("actual stableChannel %#v", stableChannel.String) + require.Equal(t, tt.expected.alphaChannel, alphaChannel.String) + require.Equal(t, tt.expected.stableChannel, stableChannel.String) + }) + } +} diff --git a/pkg/sqlite/remove.go b/pkg/sqlite/remove.go index 1d2a2cb4d..1cf5472da 100644 --- a/pkg/sqlite/remove.go +++ b/pkg/sqlite/remove.go @@ -14,13 +14,24 @@ type SQLRemover interface { Remove() error } +type SQLOperatorCsvNamesRemover interface { + Remove() error +} + // PackageRemover removes a package from the database type PackageRemover struct { store registry.Load packages string } +type OperatorPackageVersionRemover struct { + store registry.Load + CsvToRemove string + CsvToSave *string +} + var _ SQLRemover = &PackageRemover{} +var _ SQLOperatorCsvNamesRemover = &OperatorPackageVersionRemover{} func NewSQLRemoverForPackages(store registry.Load, packages string) *PackageRemover { return &PackageRemover{ @@ -29,6 +40,14 @@ func NewSQLRemoverForPackages(store registry.Load, packages string) *PackageRemo } } +func NewSQLRemoverForOperatorCsvNames(store registry.Load, csvToRemove string, csvToSave *string) *OperatorPackageVersionRemover { + return &OperatorPackageVersionRemover{ + store: store, + CsvToRemove: csvToRemove, + CsvToSave: csvToSave, + } +} + func (d *PackageRemover) Remove() error { log := logrus.WithField("pkg", d.packages) @@ -47,6 +66,26 @@ func (d *PackageRemover) Remove() error { return utilerrors.NewAggregate(errs) } +func (d *OperatorPackageVersionRemover) Remove() error { + fields := logrus.Fields{ + "csv": d.CsvToRemove, + } + log := logrus.WithFields(fields) + + log.Infof("deleting package version %s", d.CsvToRemove) + if d.CsvToSave != nil { + log.Infof("replacing with %s as head in channel", *d.CsvToSave) + } + + var errs []error + + if err := d.store.RemoveBundle(d.CsvToRemove, d.CsvToSave); err != nil { + errs = append(errs, fmt.Errorf("error removing operator bundle %s: %s", d.CsvToRemove, err)) + } + + return utilerrors.NewAggregate(errs) +} + // sanitizePackageList sanitizes the set of package(s) specified. It removes // duplicates and ignores empty string. func sanitizePackageList(in []string) []string {