opm serve: pre-compute and store api.Bundles to disk; requests read from disk

Makefile

@@ -57,7 +57,7 @@ static: build
 
 .PHONY: unit
 unit:
-	$(GO) test -coverprofile=coverage.out $(SPECIFIC_UNIT_TEST) $(TAGS) $(TEST_RACE) -count=1 -v ./pkg/... ./alpha/...
+	$(GO) test -coverprofile=coverage.out $(SPECIFIC_UNIT_TEST) $(TAGS) $(TEST_RACE) -count=1 ./pkg/... ./alpha/...
 
 .PHONY: sanity-check
 sanity-check:

cmd/opm/serve/serve.go

@@ -87,7 +87,11 @@ func (s *serve) run(ctx context.Context) error {
 	if err != nil {
 		return fmt.Errorf("could not build index model from declarative config: %v", err)
 	}
-	store := registry.NewQuerier(m)
+	store, err := registry.NewQuerier(m)
+	defer store.Close()
+	if err != nil {
+		return err
+	}
 
 	lis, err := net.Listen("tcp", ":"+s.port)
 	if err != nil {

pkg/lib/registry/registry_test.go

@@ -32,7 +32,8 @@ func fakeBundlePathFromName(name string) string {
 	return fmt.Sprintf("%s-path", name)
 }
 
-func newQuerier(bundles []*model.Bundle) *registry.Querier {
+func newQuerier(t *testing.T, bundles []*model.Bundle) *registry.Querier {
+	t.Helper()
 	pkgs := map[string]*model.Package{}
 	channels := map[string]map[string]*model.Channel{}
 
@@ -85,7 +86,9 @@ func newQuerier(bundles []*model.Bundle) *registry.Querier {
 			})
 		}
 	}
-	return registry.NewQuerier(pkgs)
+	reg, err := registry.NewQuerier(pkgs)
+	require.NoError(t, err)
+	return reg
 }
 
 func TestCheckForBundlePaths(t *testing.T) {
@@ -103,7 +106,7 @@ func TestCheckForBundlePaths(t *testing.T) {
 	}{
 		{
 			description: "BundleListPresent",
-			querier: newQuerier([]*model.Bundle{
+			querier: newQuerier(t, []*model.Bundle{
 				{
 					Package: &model.Package{Name: "pkg-0"},
 					Channel: &model.Channel{Name: "stable"},
@@ -126,7 +129,7 @@ func TestCheckForBundlePaths(t *testing.T) {
 		},
 		{
 			description: "BundleListPartiallyMissing",
-			querier: newQuerier([]*model.Bundle{
+			querier: newQuerier(t, []*model.Bundle{
 				{
 					Package: &model.Package{Name: "pkg-0"},
 					Channel: &model.Channel{Name: "stable"},
@@ -150,7 +153,7 @@ func TestCheckForBundlePaths(t *testing.T) {
 		},
 		{
 			description: "EmptyRegistry",
-			querier:     newQuerier(nil),
+			querier:     newQuerier(t, nil),
 			checkPaths: []string{
 				fakeBundlePathFromName("missing"),
 			},
@@ -161,7 +164,7 @@ func TestCheckForBundlePaths(t *testing.T) {
 		},
 		{
 			description: "EmptyDeprecateList",
-			querier: newQuerier([]*model.Bundle{
+			querier: newQuerier(t, []*model.Bundle{
 				{
 					Package: &model.Package{Name: "pkg-0"},
 					Channel: &model.Channel{Name: "stable"},
@@ -189,6 +192,9 @@ func TestCheckForBundlePaths(t *testing.T) {
 	for _, tt := range tests {
 		t.Run(tt.description, func(t *testing.T) {
 			found, missing, err := checkForBundlePaths(tt.querier, tt.checkPaths)
+			if qc, ok := tt.querier.(*registry.Querier); ok {
+				defer qc.Close()
+			}
 			if tt.expected.err != nil {
 				require.EqualError(t, err, tt.expected.err.Error())
 				return

pkg/registry/parse.go

@@ -6,10 +6,9 @@ import (
 	"io/fs"
 	"strings"
 
+	operatorsv1alpha1 "github.com/operator-framework/api/pkg/operators/v1alpha1"
 	"github.com/sirupsen/logrus"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
-
-	operatorsv1alpha1 "github.com/operator-framework/api/pkg/operators/v1alpha1"
 )
 
 type bundleParser struct {

pkg/registry/query.go

@@ -2,7 +2,10 @@ package registry
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
+	"os"
+	"path/filepath"
 	"sort"
 
 	"github.com/operator-framework/operator-registry/alpha/model"
@@ -11,6 +14,19 @@ import (
 
 type Querier struct {
 	pkgs model.Model
+
+	tmpDir     string
+	apiBundles map[apiBundleKey]string
+}
+
+func (q Querier) Close() error {
+	return os.RemoveAll(q.tmpDir)
+}
+
+type apiBundleKey struct {
+	pkgName string
+	chName  string
+	name    string
 }
 
 type SliceBundleSender []*api.Bundle
@@ -23,10 +39,60 @@ func (s *SliceBundleSender) Send(b *api.Bundle) error {
 
 var _ GRPCQuery = &Querier{}
 
-func NewQuerier(packages model.Model) *Querier {
-	return &Querier{
-		pkgs: packages,
+func NewQuerier(packages model.Model) (*Querier, error) {
+	q := &Querier{}
+
+	tmpDir, err := os.MkdirTemp("", "opm-registry-querier-")
+	if err != nil {
+		return nil, err
 	}
+	q.tmpDir = tmpDir
+
+	q.apiBundles = map[apiBundleKey]string{}
+	for _, pkg := range packages {
+		for _, ch := range pkg.Channels {
+			for _, b := range ch.Bundles {
+				apiBundle, err := api.ConvertModelBundleToAPIBundle(*b)
+				if err != nil {
+					return q, err
+				}
+				jsonBundle, err := json.Marshal(apiBundle)
+				if err != nil {
+					return q, err
+				}
+				filename := filepath.Join(tmpDir, fmt.Sprintf("%s_%s_%s.json", pkg.Name, ch.Name, b.Name))
+				if err := os.WriteFile(filename, jsonBundle, 0666); err != nil {
+					return q, err
+				}
+				q.apiBundles[apiBundleKey{pkg.Name, ch.Name, b.Name}] = filename
+				packages[pkg.Name].Channels[ch.Name].Bundles[b.Name] = &model.Bundle{
+					Package:  pkg,
+					Channel:  ch,
+					Name:     b.Name,
+					Replaces: b.Replaces,
+					Skips:    b.Skips,
+				}
+			}
+		}
+	}
+	q.pkgs = packages
+	return q, nil
+}
+
+func (q Querier) loadAPIBundle(k apiBundleKey) (*api.Bundle, error) {
+	filename, ok := q.apiBundles[k]
+	if !ok {
+		return nil, fmt.Errorf("package %q, channel %q, bundle %q not found", k.pkgName, k.chName, k.name)
+	}
+	d, err := os.ReadFile(filename)
+	if err != nil {
+		return nil, err
+	}
+	var b api.Bundle
+	if err := json.Unmarshal(d, &b); err != nil {
+		return nil, err
+	}
+	return &b, nil
 }
 
 func (q Querier) ListPackages(_ context.Context) ([]string, error) {
@@ -52,7 +118,7 @@ func (q Querier) SendBundles(_ context.Context, s BundleSender) error {
 	for _, pkg := range q.pkgs {
 		for _, ch := range pkg.Channels {
 			for _, b := range ch.Bundles {
-				apiBundle, err := api.ConvertModelBundleToAPIBundle(*b)
+				apiBundle, err := q.loadAPIBundle(apiBundleKey{pkg.Name, ch.Name, b.Name})
 				if err != nil {
 					return fmt.Errorf("convert bundle %q: %v", b.Name, err)
 				}
@@ -110,7 +176,7 @@ func (q Querier) GetBundle(_ context.Context, pkgName, channelName, csvName stri
 	if !ok {
 		return nil, fmt.Errorf("package %q, channel %q, bundle %q not found", pkgName, channelName, csvName)
 	}
-	apiBundle, err := api.ConvertModelBundleToAPIBundle(*b)
+	apiBundle, err := q.loadAPIBundle(apiBundleKey{pkg.Name, ch.Name, b.Name})
 	if err != nil {
 		return nil, fmt.Errorf("convert bundle %q: %v", b.Name, err)
 	}
@@ -134,7 +200,7 @@ func (q Querier) GetBundleForChannel(_ context.Context, pkgName string, channelN
 	if err != nil {
 		return nil, fmt.Errorf("package %q, channel %q has invalid head: %v", pkgName, channelName, err)
 	}
-	apiBundle, err := api.ConvertModelBundleToAPIBundle(*head)
+	apiBundle, err := q.loadAPIBundle(apiBundleKey{pkg.Name, ch.Name, head.Name})
 	if err != nil {
 		return nil, fmt.Errorf("convert bundle %q: %v", head.Name, err)
 	}
@@ -177,7 +243,7 @@ func (q Querier) GetBundleThatReplaces(_ context.Context, name, pkgName, channel
 	//       implementation to be non-deterministic as well.
 	for _, b := range ch.Bundles {
 		if bundleReplaces(*b, name) {
-			apiBundle, err := api.ConvertModelBundleToAPIBundle(*b)
+			apiBundle, err := q.loadAPIBundle(apiBundleKey{pkg.Name, ch.Name, b.Name})
 			if err != nil {
 				return nil, fmt.Errorf("convert bundle %q: %v", b.Name, err)
 			}
@@ -197,7 +263,7 @@ func (q Querier) GetChannelEntriesThatProvide(_ context.Context, group, version,
 	for _, pkg := range q.pkgs {
 		for _, ch := range pkg.Channels {
 			for _, b := range ch.Bundles {
-				provides, err := doesModelBundleProvide(*b, group, version, kind)
+				provides, err := q.doesModelBundleProvide(*b, group, version, kind)
 				if err != nil {
 					return nil, err
 				}
@@ -236,7 +302,7 @@ func (q Querier) GetLatestChannelEntriesThatProvide(_ context.Context, group, ve
 				return nil, fmt.Errorf("package %q, channel %q has invalid head: %v", pkg.Name, ch.Name, err)
 			}
 
-			provides, err := doesModelBundleProvide(*b, group, version, kind)
+			provides, err := q.doesModelBundleProvide(*b, group, version, kind)
 			if err != nil {
 				return nil, err
 			}
@@ -278,8 +344,8 @@ func (q Querier) GetBundleThatProvides(ctx context.Context, group, version, kind
 	return nil, fmt.Errorf("no entry found that provides group:%q version:%q kind:%q", group, version, kind)
 }
 
-func doesModelBundleProvide(b model.Bundle, group, version, kind string) (bool, error) {
-	apiBundle, err := api.ConvertModelBundleToAPIBundle(b)
+func (q Querier) doesModelBundleProvide(b model.Bundle, group, version, kind string) (bool, error) {
+	apiBundle, err := q.loadAPIBundle(apiBundleKey{b.Package.Name, b.Channel.Name, b.Name})
 	if err != nil {
 		return false, fmt.Errorf("convert bundle %q: %v", b.Name, err)
 	}

pkg/registry/query_test.go

@@ -10,9 +10,9 @@ import (
 	"github.com/operator-framework/operator-registry/alpha/declcfg"
 )
 
-var testModelQuerier = genTestModelQuerier()
-
 func TestQuerier_GetBundle(t *testing.T) {
+	testModelQuerier := genTestModelQuerier(t)
+	defer testModelQuerier.Close()
 	b, err := testModelQuerier.GetBundle(context.TODO(), "etcd", "singlenamespace-alpha", "etcdoperator.v0.9.4")
 	require.NoError(t, err)
 	require.Equal(t, b.PackageName, "etcd")
@@ -21,6 +21,8 @@ func TestQuerier_GetBundle(t *testing.T) {
 }
 
 func TestQuerier_GetBundleForChannel(t *testing.T) {
+	testModelQuerier := genTestModelQuerier(t)
+	defer testModelQuerier.Close()
 	b, err := testModelQuerier.GetBundleForChannel(context.TODO(), "etcd", "singlenamespace-alpha")
 	require.NoError(t, err)
 	require.NotNil(t, b)
@@ -30,6 +32,8 @@ func TestQuerier_GetBundleForChannel(t *testing.T) {
 }
 
 func TestQuerier_GetBundleThatProvides(t *testing.T) {
+	testModelQuerier := genTestModelQuerier(t)
+	defer testModelQuerier.Close()
 	b, err := testModelQuerier.GetBundleThatProvides(context.TODO(), "etcd.database.coreos.com", "v1beta2", "EtcdBackup")
 	require.NoError(t, err)
 	require.NotNil(t, b)
@@ -39,6 +43,8 @@ func TestQuerier_GetBundleThatProvides(t *testing.T) {
 }
 
 func TestQuerier_GetBundleThatReplaces(t *testing.T) {
+	testModelQuerier := genTestModelQuerier(t)
+	defer testModelQuerier.Close()
 	b, err := testModelQuerier.GetBundleThatReplaces(context.TODO(), "etcdoperator.v0.9.0", "etcd", "singlenamespace-alpha")
 	require.NoError(t, err)
 	require.NotNil(t, b)
@@ -48,6 +54,8 @@ func TestQuerier_GetBundleThatReplaces(t *testing.T) {
 }
 
 func TestQuerier_GetChannelEntriesThatProvide(t *testing.T) {
+	testModelQuerier := genTestModelQuerier(t)
+	defer testModelQuerier.Close()
 	entries, err := testModelQuerier.GetChannelEntriesThatProvide(context.TODO(), "etcd.database.coreos.com", "v1beta2", "EtcdBackup")
 	require.NoError(t, err)
 	require.NotNil(t, entries)
@@ -92,6 +100,8 @@ func TestQuerier_GetChannelEntriesThatProvide(t *testing.T) {
 }
 
 func TestQuerier_GetChannelEntriesThatReplace(t *testing.T) {
+	testModelQuerier := genTestModelQuerier(t)
+	defer testModelQuerier.Close()
 	entries, err := testModelQuerier.GetChannelEntriesThatReplace(context.TODO(), "etcdoperator.v0.9.0")
 	require.NoError(t, err)
 	require.NotNil(t, entries)
@@ -112,6 +122,8 @@ func TestQuerier_GetChannelEntriesThatReplace(t *testing.T) {
 }
 
 func TestQuerier_GetLatestChannelEntriesThatProvide(t *testing.T) {
+	testModelQuerier := genTestModelQuerier(t)
+	defer testModelQuerier.Close()
 	entries, err := testModelQuerier.GetLatestChannelEntriesThatProvide(context.TODO(), "etcd.database.coreos.com", "v1beta2", "EtcdBackup")
 	require.NoError(t, err)
 	require.NotNil(t, entries)
@@ -132,6 +144,8 @@ func TestQuerier_GetLatestChannelEntriesThatProvide(t *testing.T) {
 }
 
 func TestQuerier_GetPackage(t *testing.T) {
+	testModelQuerier := genTestModelQuerier(t)
+	defer testModelQuerier.Close()
 	p, err := testModelQuerier.GetPackage(context.TODO(), "etcd")
 	require.NoError(t, err)
 	require.NotNil(t, p)
@@ -161,6 +175,8 @@ func TestQuerier_GetPackage(t *testing.T) {
 }
 
 func TestQuerier_ListBundles(t *testing.T) {
+	testModelQuerier := genTestModelQuerier(t)
+	defer testModelQuerier.Close()
 	bundles, err := testModelQuerier.ListBundles(context.TODO())
 	require.NoError(t, err)
 	require.NotNil(t, bundles)
@@ -172,22 +188,27 @@ func TestQuerier_ListBundles(t *testing.T) {
 }
 
 func TestQuerier_ListPackages(t *testing.T) {
+	testModelQuerier := genTestModelQuerier(t)
+	defer testModelQuerier.Close()
 	packages, err := testModelQuerier.ListPackages(context.TODO())
 	require.NoError(t, err)
 	require.NotNil(t, packages)
 	require.Equal(t, 2, len(packages))
 }
 
-func genTestModelQuerier() *Querier {
+func genTestModelQuerier(t *testing.T) *Querier {
+	t.Helper()
+
 	cfg, err := declcfg.LoadFS(validFS)
-	if err != nil {
-		panic(err)
-	}
+	require.NoError(t, err)
+
 	m, err := declcfg.ConvertToModel(*cfg)
-	if err != nil {
-		panic(err)
-	}
-	return NewQuerier(m)
+	require.NoError(t, err)
+
+	reg, err := NewQuerier(m)
+	require.NoError(t, err)
+
+	return reg
 }
 
 var validFS = fstest.MapFS{