diff --git a/.golangci.toml b/.golangci.toml index 648b0dad93cb..33432f52d2e2 100644 --- a/.golangci.toml +++ b/.golangci.toml @@ -78,6 +78,8 @@ allow = [ "github.com/grafana/grafana/pkg/services/apiserver/utils", "github.com/grafana/grafana/pkg/services/featuremgmt", "github.com/grafana/grafana/pkg/infra/kvstore", + "github.com/grafana/grafana/pkg/services/apiserver/options", + "github.com/grafana/grafana/pkg/apis/playlist/v0alpha1", ] deny = [ { pkg = "github.com/grafana/grafana/pkg", desc = "apiserver is not allowed to import grafana core" } diff --git a/pkg/apis/playlist/v0alpha1/register.go b/pkg/apis/playlist/v0alpha1/register.go index c1886b2d572b..0b10256f7590 100644 --- a/pkg/apis/playlist/v0alpha1/register.go +++ b/pkg/apis/playlist/v0alpha1/register.go @@ -8,13 +8,15 @@ import ( ) const ( - GROUP = "playlist.grafana.app" - VERSION = "v0alpha1" - APIVERSION = GROUP + "/" + VERSION + GROUP = "playlist.grafana.app" + VERSION = "v0alpha1" + APIVERSION = GROUP + "/" + VERSION + RESOURCE = "playlists" + GROUPRESOURCE = GROUP + "/" + RESOURCE ) var PlaylistResourceInfo = common.NewResourceInfo(GROUP, VERSION, - "playlists", "playlist", "Playlist", + RESOURCE, "playlist", "Playlist", func() runtime.Object { return &Playlist{} }, func() runtime.Object { return &PlaylistList{} }, ) diff --git a/pkg/apiserver/builder/common.go b/pkg/apiserver/builder/common.go index 0f380c1eafe4..d4a0d661c0e1 100644 --- a/pkg/apiserver/builder/common.go +++ b/pkg/apiserver/builder/common.go @@ -3,6 +3,7 @@ package builder import ( "net/http" + grafanarest "github.com/grafana/grafana/pkg/apiserver/rest" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/runtime/serializer" @@ -27,7 +28,7 @@ type APIGroupBuilder interface { scheme *runtime.Scheme, codecs serializer.CodecFactory, optsGetter generic.RESTOptionsGetter, - dualWrite bool, + desiredMode grafanarest.DualWriterMode, ) (*genericapiserver.APIGroupInfo, error) // Get OpenAPI definitions @@ -40,6 +41,11 @@ type APIGroupBuilder interface { // Standard namespace checking will happen before this is called, specifically // the namespace must matches an org|stack that the user belongs to GetAuthorizer() authorizer.Authorizer + + // Get the desired dual writing mode. These are modes 1, 2, 3 and 4 if + // the feature flag `unifiedStorage` is enabled and mode 0 if it is not enabled. + // #TODO add type for map[string]grafanarest.DualWriterMode? + GetDesiredDualWriterMode(dualWrite bool, toMode map[string]grafanarest.DualWriterMode) grafanarest.DualWriterMode } // Builders that implement OpenAPIPostProcessor are given a chance to modify the schema directly diff --git a/pkg/apiserver/builder/helper.go b/pkg/apiserver/builder/helper.go index 12cf81896a2e..496983a72925 100644 --- a/pkg/apiserver/builder/helper.go +++ b/pkg/apiserver/builder/helper.go @@ -24,6 +24,7 @@ import ( "k8s.io/kube-openapi/pkg/common" "github.com/grafana/grafana/pkg/apiserver/endpoints/filters" + "github.com/grafana/grafana/pkg/services/apiserver/options" ) // TODO: this is a temporary hack to make rest.Connecter work with resource level routes @@ -126,10 +127,16 @@ func InstallAPIs( server *genericapiserver.GenericAPIServer, optsGetter generic.RESTOptionsGetter, builders []APIGroupBuilder, - dualWrite bool, + storageOpts *options.StorageOptions, ) error { + // dual writing is only enabled when the storage type is not legacy. + // this is needed to support setting a default RESTOptionsGetter for new APIs that don't + // support the legacy storage type. + dualWriteEnabled := storageOpts.StorageType != options.StorageTypeLegacy + for _, b := range builders { - g, err := b.GetAPIGroupInfo(scheme, codecs, optsGetter, dualWrite) + mode := b.GetDesiredDualWriterMode(dualWriteEnabled, storageOpts.DualWriterDesiredModes) + g, err := b.GetAPIGroupInfo(scheme, codecs, optsGetter, mode) if err != nil { return err } diff --git a/pkg/apiserver/rest/dualwriter.go b/pkg/apiserver/rest/dualwriter.go index a9777739e0ae..9031229b999c 100644 --- a/pkg/apiserver/rest/dualwriter.go +++ b/pkg/apiserver/rest/dualwriter.go @@ -6,7 +6,6 @@ import ( "fmt" "github.com/grafana/grafana/pkg/infra/kvstore" - "github.com/grafana/grafana/pkg/services/featuremgmt" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apiserver/pkg/registry/rest" @@ -82,15 +81,25 @@ type DualWriter interface { type DualWriterMode int const ( - Mode1 DualWriterMode = iota + 1 + // Mode0 represents writing to and reading from solely LegacyStorage. This mode is enabled when the + // `unifiedStorage` feature flag is not set. All reads and writes are made to LegacyStorage. None are made to Storage. + Mode0 DualWriterMode = iota + // Mode1 represents writing to and reading from LegacyStorage for all primary functionality while additionally + // reading and writing to Storage on a best effort basis for the sake of collecting metrics. + Mode1 + // Mode2 is the dual writing mode that represents writing to LegacyStorage and Storage and reading from LegacyStorage. Mode2 + // Mode3 represents writing to LegacyStorage and Storage and reading from Storage. Mode3 + // Mode4 represents writing and reading from Storage. Mode4 ) // NewDualWriter returns a new DualWriter. func NewDualWriter(mode DualWriterMode, legacy LegacyStorage, storage Storage) DualWriter { switch mode { + // It is not possible to initialize a mode 0 dual writer. Mode 0 represents + // writing to legacy storage without `unifiedStorage` enabled. case Mode1: // read and write only from legacy storage return newDualWriterMode1(legacy, storage) @@ -129,12 +138,14 @@ func (u *updateWrapper) UpdatedObject(ctx context.Context, oldObj runtime.Object func SetDualWritingMode( ctx context.Context, kvs *kvstore.NamespacedKVStore, - features featuremgmt.FeatureToggles, - entity string, legacy LegacyStorage, storage Storage, + entity string, + desiredMode DualWriterMode, ) (DualWriter, error) { toMode := map[string]DualWriterMode{ + // It is not possible to initialize a mode 0 dual writer. Mode 0 represents + // writing to legacy storage without `unifiedStorage` enabled. "1": Mode1, "2": Mode2, "3": Mode3, @@ -166,7 +177,7 @@ func SetDualWritingMode( } // Desired mode is 2 and current mode is 1 - if features.IsEnabledGlobally(featuremgmt.FlagDualWritePlaylistsMode2) && (currentMode == Mode1) { + if (desiredMode == Mode2) && (currentMode == Mode1) { // This is where we go through the different gates to allow the instance to migrate from mode 1 to mode 2. // There are none between mode 1 and mode 2 currentMode = Mode2 @@ -176,17 +187,16 @@ func SetDualWritingMode( return nil, errDualWriterSetCurrentMode } } - // #TODO enable this check when we have a flag/config for setting mode 1 as the desired mode - // if features.IsEnabledGlobally(featuremgmt.FlagDualWritePlaylistsMode1) && (currentMode == Mode2) { - // // This is where we go through the different gates to allow the instance to migrate from mode 2 to mode 1. - // // There are none between mode 1 and mode 2 - // currentMode = Mode1 - - // err := kvs.Set(ctx, entity, fmt.Sprint(currentMode)) - // if err != nil { - // return nil, errDualWriterSetCurrentMode - // } - // } + if (desiredMode == Mode1) && (currentMode == Mode2) { + // This is where we go through the different gates to allow the instance to migrate from mode 2 to mode 1. + // There are none between mode 1 and mode 2 + currentMode = Mode1 + + err := kvs.Set(ctx, entity, fmt.Sprint(currentMode)) + if err != nil { + return nil, errDualWriterSetCurrentMode + } + } // #TODO add support for other combinations of desired and current modes diff --git a/pkg/apiserver/rest/dualwriter_test.go b/pkg/apiserver/rest/dualwriter_test.go index 14fc14218243..10cb16644d55 100644 --- a/pkg/apiserver/rest/dualwriter_test.go +++ b/pkg/apiserver/rest/dualwriter_test.go @@ -5,8 +5,8 @@ import ( "fmt" "testing" + playlist "github.com/grafana/grafana/pkg/apis/playlist/v0alpha1" "github.com/grafana/grafana/pkg/infra/kvstore" - "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" ) @@ -14,24 +14,24 @@ import ( func TestSetDualWritingMode(t *testing.T) { type testCase struct { name string - features []any stackID string + desiredMode DualWriterMode expectedMode DualWriterMode } tests := // #TODO add test cases for kv store failures. Requires adding support in kvstore test_utils.go []testCase{ { - name: "should return a mode 1 dual writer when no desired mode is set", - features: []any{}, + name: "should return a mode 2 dual writer when mode 2 is set as the desired mode", stackID: "stack-1", - expectedMode: Mode1, + desiredMode: Mode2, + expectedMode: Mode2, }, { - name: "should return a mode 2 dual writer when mode 2 is set as the desired mode", - features: []any{featuremgmt.FlagDualWritePlaylistsMode2}, + name: "should return a mode 1 dual writer when mode 1 is set as the desired mode", stackID: "stack-1", - expectedMode: Mode2, + desiredMode: Mode1, + expectedMode: Mode1, }, } @@ -43,17 +43,14 @@ func TestSetDualWritingMode(t *testing.T) { ls := legacyStoreMock{m, l} us := storageMock{m, s} - f := featuremgmt.WithFeatures(tt.features...) kvStore := kvstore.WithNamespace(kvstore.NewFakeKVStore(), 0, "storage.dualwriting."+tt.stackID) - key := "playlist" - - dw, err := SetDualWritingMode(context.Background(), kvStore, f, key, ls, us) + dw, err := SetDualWritingMode(context.Background(), kvStore, ls, us, playlist.GROUPRESOURCE, tt.desiredMode) assert.NoError(t, err) assert.Equal(t, tt.expectedMode, dw.Mode()) // check kv store - val, ok, err := kvStore.Get(context.Background(), key) + val, ok, err := kvStore.Get(context.Background(), playlist.GROUPRESOURCE) assert.True(t, ok) assert.NoError(t, err) assert.Equal(t, val, fmt.Sprint(tt.expectedMode)) diff --git a/pkg/cmd/grafana/apiserver/server.go b/pkg/cmd/grafana/apiserver/server.go index 1ead6da20e54..993aeac28e5b 100644 --- a/pkg/cmd/grafana/apiserver/server.go +++ b/pkg/cmd/grafana/apiserver/server.go @@ -162,7 +162,8 @@ func (o *APIServerOptions) RunAPIServer(config *genericapiserver.RecommendedConf } // Install the API Group+version - err = builder.InstallAPIs(grafanaAPIServer.Scheme, grafanaAPIServer.Codecs, server, config.RESTOptionsGetter, o.builders, true) + // #TODO figure out how to configure storage type in o.Options.StorageOptions + err = builder.InstallAPIs(grafanaAPIServer.Scheme, grafanaAPIServer.Codecs, server, config.RESTOptionsGetter, o.builders, o.Options.StorageOptions) if err != nil { return err } diff --git a/pkg/registry/apis/dashboard/register.go b/pkg/registry/apis/dashboard/register.go index 4bb2bf6ff9c3..731939ce1bfb 100644 --- a/pkg/registry/apis/dashboard/register.go +++ b/pkg/registry/apis/dashboard/register.go @@ -73,6 +73,11 @@ func (b *DashboardsAPIBuilder) GetGroupVersion() schema.GroupVersion { return v0alpha1.DashboardResourceInfo.GroupVersion() } +func (b *DashboardsAPIBuilder) GetDesiredDualWriterMode(dualWrite bool, modeMap map[string]grafanarest.DualWriterMode) grafanarest.DualWriterMode { + // Add required configuration support in order to enable other modes. For an example, see pkg/registry/apis/playlist/register.go + return grafanarest.Mode0 +} + func addKnownTypes(scheme *runtime.Scheme, gv schema.GroupVersion) { scheme.AddKnownTypes(gv, &v0alpha1.Dashboard{}, @@ -109,7 +114,7 @@ func (b *DashboardsAPIBuilder) GetAPIGroupInfo( scheme *runtime.Scheme, codecs serializer.CodecFactory, // pointer? optsGetter generic.RESTOptionsGetter, - dualWrite bool, + desiredMode grafanarest.DualWriterMode, ) (*genericapiserver.APIGroupInfo, error) { apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(v0alpha1.GROUP, scheme, metav1.ParameterCodec, codecs) @@ -135,7 +140,7 @@ func (b *DashboardsAPIBuilder) GetAPIGroupInfo( } // Dual writes if a RESTOptionsGetter is provided - if dualWrite && optsGetter != nil { + if desiredMode != grafanarest.Mode0 && optsGetter != nil { options := &generic.StoreOptions{RESTOptions: optsGetter, AttrFunc: grafanaregistry.GetAttrs} if err := store.CompleteWithOptions(options); err != nil { return nil, err diff --git a/pkg/registry/apis/dashboardsnapshot/register.go b/pkg/registry/apis/dashboardsnapshot/register.go index 4f74c4f5fba8..96f946245de1 100644 --- a/pkg/registry/apis/dashboardsnapshot/register.go +++ b/pkg/registry/apis/dashboardsnapshot/register.go @@ -22,6 +22,7 @@ import ( dashboardsnapshot "github.com/grafana/grafana/pkg/apis/dashboardsnapshot/v0alpha1" "github.com/grafana/grafana/pkg/apiserver/builder" + grafanarest "github.com/grafana/grafana/pkg/apiserver/rest" "github.com/grafana/grafana/pkg/infra/appcontext" "github.com/grafana/grafana/pkg/infra/db" "github.com/grafana/grafana/pkg/infra/log" @@ -86,6 +87,11 @@ func (b *SnapshotsAPIBuilder) GetGroupVersion() schema.GroupVersion { return resourceInfo.GroupVersion() } +func (b *SnapshotsAPIBuilder) GetDesiredDualWriterMode(dualWrite bool, modeMap map[string]grafanarest.DualWriterMode) grafanarest.DualWriterMode { + // Add required configuration support in order to enable other modes. For an example, see pkg/registry/apis/playlist/register.go + return grafanarest.Mode0 +} + func addKnownTypes(scheme *runtime.Scheme, gv schema.GroupVersion) { scheme.AddKnownTypes(gv, &dashboardsnapshot.DashboardSnapshot{}, @@ -122,7 +128,7 @@ func (b *SnapshotsAPIBuilder) GetAPIGroupInfo( scheme *runtime.Scheme, codecs serializer.CodecFactory, // pointer? optsGetter generic.RESTOptionsGetter, - dualWrite bool, + _ grafanarest.DualWriterMode, ) (*genericapiserver.APIGroupInfo, error) { apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(dashboardsnapshot.GROUP, scheme, metav1.ParameterCodec, codecs) storage := map[string]rest.Storage{} diff --git a/pkg/registry/apis/datasource/register.go b/pkg/registry/apis/datasource/register.go index c880f2b0e14e..e6d2d838bd83 100644 --- a/pkg/registry/apis/datasource/register.go +++ b/pkg/registry/apis/datasource/register.go @@ -23,6 +23,7 @@ import ( datasource "github.com/grafana/grafana/pkg/apis/datasource/v0alpha1" query "github.com/grafana/grafana/pkg/apis/query/v0alpha1" "github.com/grafana/grafana/pkg/apiserver/builder" + grafanarest "github.com/grafana/grafana/pkg/apiserver/rest" "github.com/grafana/grafana/pkg/plugins" "github.com/grafana/grafana/pkg/promlib/models" "github.com/grafana/grafana/pkg/registry/apis/query/queryschema" @@ -151,6 +152,11 @@ func (b *DataSourceAPIBuilder) GetGroupVersion() schema.GroupVersion { return b.connectionResourceInfo.GroupVersion() } +func (b *DataSourceAPIBuilder) GetDesiredDualWriterMode(dualWrite bool, modeMap map[string]grafanarest.DualWriterMode) grafanarest.DualWriterMode { + // Add required configuration support in order to enable other modes. For an example, see pkg/registry/apis/playlist/register.go + return grafanarest.Mode0 +} + func addKnownTypes(scheme *runtime.Scheme, gv schema.GroupVersion) { scheme.AddKnownTypes(gv, &datasource.DataSourceConnection{}, @@ -198,7 +204,7 @@ func (b *DataSourceAPIBuilder) GetAPIGroupInfo( scheme *runtime.Scheme, codecs serializer.CodecFactory, // pointer? _ generic.RESTOptionsGetter, - _ bool, + _ grafanarest.DualWriterMode, ) (*genericapiserver.APIGroupInfo, error) { storage := map[string]rest.Storage{} diff --git a/pkg/registry/apis/example/register.go b/pkg/registry/apis/example/register.go index c18d5d4c4e09..9b7da8ff1ef1 100644 --- a/pkg/registry/apis/example/register.go +++ b/pkg/registry/apis/example/register.go @@ -22,6 +22,7 @@ import ( example "github.com/grafana/grafana/pkg/apis/example/v0alpha1" "github.com/grafana/grafana/pkg/apiserver/builder" + grafanarest "github.com/grafana/grafana/pkg/apiserver/rest" "github.com/grafana/grafana/pkg/infra/appcontext" "github.com/grafana/grafana/pkg/services/featuremgmt" ) @@ -53,6 +54,11 @@ func (b *TestingAPIBuilder) GetGroupVersion() schema.GroupVersion { return b.gv } +func (b *TestingAPIBuilder) GetDesiredDualWriterMode(dualWrite bool, modeMap map[string]grafanarest.DualWriterMode) grafanarest.DualWriterMode { + // Add required configuration support in order to enable other modes. For an example, see pkg/registry/apis/playlist/register.go + return grafanarest.Mode0 +} + func addKnownTypes(scheme *runtime.Scheme, gv schema.GroupVersion) { scheme.AddKnownTypes(gv, &example.RuntimeInfo{}, @@ -85,7 +91,7 @@ func (b *TestingAPIBuilder) GetAPIGroupInfo( scheme *runtime.Scheme, codecs serializer.CodecFactory, // pointer? _ generic.RESTOptionsGetter, - _ bool, + _ grafanarest.DualWriterMode, ) (*genericapiserver.APIGroupInfo, error) { b.codecs = codecs apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(b.gv.Group, scheme, metav1.ParameterCodec, codecs) diff --git a/pkg/registry/apis/featuretoggle/register.go b/pkg/registry/apis/featuretoggle/register.go index e93b396d55a4..72d135d451cb 100644 --- a/pkg/registry/apis/featuretoggle/register.go +++ b/pkg/registry/apis/featuretoggle/register.go @@ -15,6 +15,7 @@ import ( "github.com/grafana/grafana/pkg/apis/featuretoggle/v0alpha1" "github.com/grafana/grafana/pkg/apiserver/builder" + grafanarest "github.com/grafana/grafana/pkg/apiserver/rest" "github.com/grafana/grafana/pkg/services/accesscontrol" "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/setting" @@ -49,6 +50,11 @@ func (b *FeatureFlagAPIBuilder) GetGroupVersion() schema.GroupVersion { return gv } +func (b *FeatureFlagAPIBuilder) GetDesiredDualWriterMode(dualWrite bool, modeMap map[string]grafanarest.DualWriterMode) grafanarest.DualWriterMode { + // Add required configuration support in order to enable other modes. For an example, see pkg/registry/apis/playlist/register.go + return grafanarest.Mode0 +} + func addKnownTypes(scheme *runtime.Scheme, gv schema.GroupVersion) { scheme.AddKnownTypes(gv, &v0alpha1.Feature{}, @@ -82,7 +88,7 @@ func (b *FeatureFlagAPIBuilder) GetAPIGroupInfo( scheme *runtime.Scheme, codecs serializer.CodecFactory, // pointer? _ generic.RESTOptionsGetter, - _ bool, + _ grafanarest.DualWriterMode, ) (*genericapiserver.APIGroupInfo, error) { apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(v0alpha1.GROUP, scheme, metav1.ParameterCodec, codecs) diff --git a/pkg/registry/apis/folders/register.go b/pkg/registry/apis/folders/register.go index b10d5bd4edc0..685d1ab737ae 100644 --- a/pkg/registry/apis/folders/register.go +++ b/pkg/registry/apis/folders/register.go @@ -66,6 +66,11 @@ func (b *FolderAPIBuilder) GetGroupVersion() schema.GroupVersion { return b.gv } +func (b *FolderAPIBuilder) GetDesiredDualWriterMode(dualWrite bool, modeMap map[string]grafanarest.DualWriterMode) grafanarest.DualWriterMode { + // Add required configuration support in order to enable other modes. For an example, see pkg/registry/apis/playlist/register.go + return grafanarest.Mode0 +} + func addKnownTypes(scheme *runtime.Scheme, gv schema.GroupVersion) { scheme.AddKnownTypes(gv, &v0alpha1.Folder{}, @@ -99,7 +104,7 @@ func (b *FolderAPIBuilder) GetAPIGroupInfo( scheme *runtime.Scheme, codecs serializer.CodecFactory, // pointer? optsGetter generic.RESTOptionsGetter, - dualWrite bool, + desiredMode grafanarest.DualWriterMode, ) (*genericapiserver.APIGroupInfo, error) { apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(v0alpha1.GROUP, scheme, metav1.ParameterCodec, codecs) @@ -134,7 +139,7 @@ func (b *FolderAPIBuilder) GetAPIGroupInfo( storage[resourceInfo.StoragePath("access")] = &subAccessREST{b.folderSvc} // enable dual writes if a RESTOptionsGetter is provided - if dualWrite && optsGetter != nil { + if optsGetter != nil && desiredMode != grafanarest.Mode0 { store, err := newStorage(scheme, optsGetter, legacyStore) if err != nil { return nil, err diff --git a/pkg/registry/apis/peakq/register.go b/pkg/registry/apis/peakq/register.go index 076b3b427a0f..718c9746edf1 100644 --- a/pkg/registry/apis/peakq/register.go +++ b/pkg/registry/apis/peakq/register.go @@ -15,6 +15,7 @@ import ( peakq "github.com/grafana/grafana/pkg/apis/peakq/v0alpha1" "github.com/grafana/grafana/pkg/apiserver/builder" + grafanarest "github.com/grafana/grafana/pkg/apiserver/rest" "github.com/grafana/grafana/pkg/services/featuremgmt" ) @@ -44,6 +45,11 @@ func (b *PeakQAPIBuilder) GetGroupVersion() schema.GroupVersion { return peakq.SchemeGroupVersion } +func (b *PeakQAPIBuilder) GetDesiredDualWriterMode(dualWrite bool, modeMap map[string]grafanarest.DualWriterMode) grafanarest.DualWriterMode { + // Add required configuration support in order to enable other modes. For an example, see pkg/registry/apis/playlist/register.go + return grafanarest.Mode0 +} + func (b *PeakQAPIBuilder) InstallSchema(scheme *runtime.Scheme) error { gv := peakq.SchemeGroupVersion err := peakq.AddToScheme(scheme) @@ -66,7 +72,7 @@ func (b *PeakQAPIBuilder) GetAPIGroupInfo( scheme *runtime.Scheme, codecs serializer.CodecFactory, optsGetter generic.RESTOptionsGetter, - _ bool, // dual write (not relevant) + _ grafanarest.DualWriterMode, // dual write desired mode (not relevant) ) (*genericapiserver.APIGroupInfo, error) { apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(peakq.GROUP, scheme, metav1.ParameterCodec, codecs) diff --git a/pkg/registry/apis/playlist/register.go b/pkg/registry/apis/playlist/register.go index 98e52a22e7f4..fa4ea3ab2e4f 100644 --- a/pkg/registry/apis/playlist/register.go +++ b/pkg/registry/apis/playlist/register.go @@ -21,7 +21,6 @@ import ( "github.com/grafana/grafana/pkg/infra/kvstore" "github.com/grafana/grafana/pkg/services/apiserver/endpoints/request" "github.com/grafana/grafana/pkg/services/apiserver/utils" - "github.com/grafana/grafana/pkg/services/featuremgmt" playlistsvc "github.com/grafana/grafana/pkg/services/playlist" "github.com/grafana/grafana/pkg/setting" ) @@ -33,21 +32,18 @@ type PlaylistAPIBuilder struct { service playlistsvc.Service namespacer request.NamespaceMapper gv schema.GroupVersion - features featuremgmt.FeatureToggles kvStore *kvstore.NamespacedKVStore } func RegisterAPIService(p playlistsvc.Service, apiregistration builder.APIRegistrar, cfg *setting.Cfg, - features featuremgmt.FeatureToggles, kvStore kvstore.KVStore, ) *PlaylistAPIBuilder { builder := &PlaylistAPIBuilder{ service: p, namespacer: request.GetNamespaceMapper(cfg), gv: playlist.PlaylistResourceInfo.GroupVersion(), - features: features, kvStore: kvstore.WithNamespace(kvStore, 0, "storage.dualwriting"), } apiregistration.RegisterAPI(builder) @@ -58,6 +54,15 @@ func (b *PlaylistAPIBuilder) GetGroupVersion() schema.GroupVersion { return b.gv } +func (b *PlaylistAPIBuilder) GetDesiredDualWriterMode(dualWrite bool, modeMap map[string]grafanarest.DualWriterMode) grafanarest.DualWriterMode { + m, ok := modeMap[playlist.GROUPRESOURCE] + if !dualWrite || !ok { + return grafanarest.Mode0 + } + + return m +} + func addKnownTypes(scheme *runtime.Scheme, gv schema.GroupVersion) { scheme.AddKnownTypes(gv, &playlist.Playlist{}, @@ -88,7 +93,7 @@ func (b *PlaylistAPIBuilder) GetAPIGroupInfo( scheme *runtime.Scheme, codecs serializer.CodecFactory, // pointer? optsGetter generic.RESTOptionsGetter, - dualWrite bool, + desiredMode grafanarest.DualWriterMode, ) (*genericapiserver.APIGroupInfo, error) { apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(playlist.GROUP, scheme, metav1.ParameterCodec, codecs) storage := map[string]rest.Storage{} @@ -122,13 +127,13 @@ func (b *PlaylistAPIBuilder) GetAPIGroupInfo( storage[resource.StoragePath()] = legacyStore // enable dual writes if a RESTOptionsGetter is provided - if optsGetter != nil && dualWrite { + if optsGetter != nil && desiredMode != grafanarest.Mode0 { store, err := newStorage(scheme, optsGetter, legacyStore) if err != nil { return nil, err } - dualWriter, err := grafanarest.SetDualWritingMode(context.Background(), b.kvStore, b.features, "playlist", legacyStore, store) + dualWriter, err := grafanarest.SetDualWritingMode(context.Background(), b.kvStore, legacyStore, store, playlist.GROUPRESOURCE, desiredMode) if err != nil { return nil, err } diff --git a/pkg/registry/apis/query/register.go b/pkg/registry/apis/query/register.go index df6d5fa42030..897f045c102e 100644 --- a/pkg/registry/apis/query/register.go +++ b/pkg/registry/apis/query/register.go @@ -17,6 +17,7 @@ import ( query "github.com/grafana/grafana/pkg/apis/query/v0alpha1" "github.com/grafana/grafana/pkg/apiserver/builder" + grafanarest "github.com/grafana/grafana/pkg/apiserver/rest" "github.com/grafana/grafana/pkg/expr" "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/infra/tracing" @@ -123,6 +124,11 @@ func (b *QueryAPIBuilder) GetGroupVersion() schema.GroupVersion { return query.SchemeGroupVersion } +func (b *QueryAPIBuilder) GetDesiredDualWriterMode(dualWrite bool, modeMap map[string]grafanarest.DualWriterMode) grafanarest.DualWriterMode { + // Add required configuration support in order to enable other modes. For an example, see pkg/registry/apis/playlist/register.go + return grafanarest.Mode0 +} + func addKnownTypes(scheme *runtime.Scheme, gv schema.GroupVersion) { scheme.AddKnownTypes(gv, &query.DataSourceApiServer{}, @@ -144,7 +150,7 @@ func (b *QueryAPIBuilder) GetAPIGroupInfo( scheme *runtime.Scheme, codecs serializer.CodecFactory, // pointer? optsGetter generic.RESTOptionsGetter, - _ bool, + _ grafanarest.DualWriterMode, ) (*genericapiserver.APIGroupInfo, error) { gv := query.SchemeGroupVersion apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(gv.Group, scheme, metav1.ParameterCodec, codecs) diff --git a/pkg/registry/apis/scope/register.go b/pkg/registry/apis/scope/register.go index f5bbbdf20e97..243dd978967e 100644 --- a/pkg/registry/apis/scope/register.go +++ b/pkg/registry/apis/scope/register.go @@ -17,6 +17,7 @@ import ( scope "github.com/grafana/grafana/pkg/apis/scope/v0alpha1" "github.com/grafana/grafana/pkg/apiserver/builder" + grafanarest "github.com/grafana/grafana/pkg/apiserver/rest" "github.com/grafana/grafana/pkg/services/featuremgmt" ) @@ -46,6 +47,11 @@ func (b *ScopeAPIBuilder) GetGroupVersion() schema.GroupVersion { return scope.SchemeGroupVersion } +func (b *ScopeAPIBuilder) GetDesiredDualWriterMode(dualWrite bool, modeMap map[string]grafanarest.DualWriterMode) grafanarest.DualWriterMode { + // Add required configuration support in order to enable other modes. For an example, see pkg/registry/apis/playlist/register.go + return grafanarest.Mode0 +} + func (b *ScopeAPIBuilder) InstallSchema(scheme *runtime.Scheme) error { err := scope.AddToScheme(scheme) if err != nil { @@ -114,7 +120,7 @@ func (b *ScopeAPIBuilder) GetAPIGroupInfo( scheme *runtime.Scheme, codecs serializer.CodecFactory, optsGetter generic.RESTOptionsGetter, - _ bool, // dual write (not relevant) + _ grafanarest.DualWriterMode, // dual write desired mode (not relevant) ) (*genericapiserver.APIGroupInfo, error) { apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(scope.GROUP, scheme, metav1.ParameterCodec, codecs) diff --git a/pkg/registry/apis/service/register.go b/pkg/registry/apis/service/register.go index 17dbc3b07aa3..69b01a2707dc 100644 --- a/pkg/registry/apis/service/register.go +++ b/pkg/registry/apis/service/register.go @@ -13,6 +13,7 @@ import ( service "github.com/grafana/grafana/pkg/apis/service/v0alpha1" "github.com/grafana/grafana/pkg/apiserver/builder" + grafanarest "github.com/grafana/grafana/pkg/apiserver/rest" "github.com/grafana/grafana/pkg/services/featuremgmt" ) @@ -43,6 +44,11 @@ func (b *ServiceAPIBuilder) GetGroupVersion() schema.GroupVersion { return service.SchemeGroupVersion } +func (b *ServiceAPIBuilder) GetDesiredDualWriterMode(dualWrite bool, modeMap map[string]grafanarest.DualWriterMode) grafanarest.DualWriterMode { + // Add required configuration support in order to enable other modes. For an example, see pkg/registry/apis/playlist/register.go + return grafanarest.Mode0 +} + func addKnownTypes(scheme *runtime.Scheme, gv schema.GroupVersion) { scheme.AddKnownTypes(gv, &service.ExternalName{}, @@ -72,7 +78,7 @@ func (b *ServiceAPIBuilder) GetAPIGroupInfo( scheme *runtime.Scheme, codecs serializer.CodecFactory, optsGetter generic.RESTOptionsGetter, - _ bool, + _ grafanarest.DualWriterMode, ) (*genericapiserver.APIGroupInfo, error) { apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(service.GROUP, scheme, metav1.ParameterCodec, codecs) diff --git a/pkg/services/apiserver/aggregator/aggregator.go b/pkg/services/apiserver/aggregator/aggregator.go index 75028060d4c6..7107b2d77f51 100644 --- a/pkg/services/apiserver/aggregator/aggregator.go +++ b/pkg/services/apiserver/aggregator/aggregator.go @@ -47,6 +47,7 @@ import ( "k8s.io/kube-aggregator/pkg/controllers/autoregister" "github.com/grafana/grafana/pkg/apiserver/builder" + grafanarest "github.com/grafana/grafana/pkg/apiserver/rest" servicev0alpha1applyconfiguration "github.com/grafana/grafana/pkg/generated/applyconfiguration/service/v0alpha1" serviceclientset "github.com/grafana/grafana/pkg/generated/clientset/versioned" informersv0alpha1 "github.com/grafana/grafana/pkg/generated/informers/externalversions" @@ -284,7 +285,7 @@ func CreateAggregatorServer(config *Config, delegateAPIServer genericapiserver.D }) for _, b := range config.Builders { - serviceAPIGroupInfo, err := b.GetAPIGroupInfo(aggregatorscheme.Scheme, aggregatorscheme.Codecs, aggregatorConfig.GenericConfig.RESTOptionsGetter, false) + serviceAPIGroupInfo, err := b.GetAPIGroupInfo(aggregatorscheme.Scheme, aggregatorscheme.Codecs, aggregatorConfig.GenericConfig.RESTOptionsGetter, grafanarest.Mode0) if err != nil { return nil, err } diff --git a/pkg/services/apiserver/config.go b/pkg/services/apiserver/config.go index 39346bb26c84..ab760dadd4b0 100644 --- a/pkg/services/apiserver/config.go +++ b/pkg/services/apiserver/config.go @@ -6,6 +6,8 @@ import ( "path/filepath" "strconv" + playlist "github.com/grafana/grafana/pkg/apis/playlist/v0alpha1" + grafanarest "github.com/grafana/grafana/pkg/apiserver/rest" "github.com/grafana/grafana/pkg/services/apiserver/options" "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/setting" @@ -33,6 +35,7 @@ func applyGrafanaConfig(cfg *setting.Cfg, features featuremgmt.FeatureToggles, o host := net.JoinHostPort(cfg.HTTPAddr, strconv.Itoa(port)) apiserverCfg := cfg.SectionWithEnvOverrides("grafana-apiserver") + unifiedStorageModeCfg := cfg.SectionWithEnvOverrides("unified_storage_mode") o.RecommendedOptions.Etcd.StorageConfig.Transport.ServerList = apiserverCfg.Key("etcd_servers").Strings(",") @@ -53,6 +56,9 @@ func applyGrafanaConfig(cfg *setting.Cfg, features featuremgmt.FeatureToggles, o o.StorageOptions.StorageType = options.StorageType(apiserverCfg.Key("storage_type").MustString(string(options.StorageTypeLegacy))) o.StorageOptions.DataPath = apiserverCfg.Key("storage_path").MustString(filepath.Join(cfg.DataPath, "grafana-apiserver")) o.StorageOptions.Address = apiserverCfg.Key("address").MustString(o.StorageOptions.Address) + o.StorageOptions.DualWriterDesiredModes = map[string]grafanarest.DualWriterMode{ + playlist.GROUPRESOURCE: grafanarest.DualWriterMode(unifiedStorageModeCfg.Key(playlist.GROUPRESOURCE).MustInt(0)), + } o.ExtraOptions.DevMode = features.IsEnabledGlobally(featuremgmt.FlagGrafanaAPIServerEnsureKubectlAccess) o.ExtraOptions.ExternalAddress = host o.ExtraOptions.APIURL = apiURL diff --git a/pkg/services/apiserver/options/storage.go b/pkg/services/apiserver/options/storage.go index dc99d98e9b40..110215b52a55 100644 --- a/pkg/services/apiserver/options/storage.go +++ b/pkg/services/apiserver/options/storage.go @@ -4,6 +4,7 @@ import ( "fmt" "net" + grafanarest "github.com/grafana/grafana/pkg/apiserver/rest" "github.com/spf13/pflag" genericapiserver "k8s.io/apiserver/pkg/server" "k8s.io/apiserver/pkg/server/options" @@ -20,9 +21,10 @@ const ( ) type StorageOptions struct { - StorageType StorageType - DataPath string - Address string + StorageType StorageType + DataPath string + Address string + DualWriterDesiredModes map[string]grafanarest.DualWriterMode } func NewStorageOptions() *StorageOptions { diff --git a/pkg/services/apiserver/service.go b/pkg/services/apiserver/service.go index 89d89390e8e2..684b98340bee 100644 --- a/pkg/services/apiserver/service.go +++ b/pkg/services/apiserver/service.go @@ -308,13 +308,8 @@ func (s *service) start(ctx context.Context) error { return err } - // dual writing is only enabled when the storage type is not legacy. - // this is needed to support setting a default RESTOptionsGetter for new APIs that don't - // support the legacy storage type. - dualWriteEnabled := o.StorageOptions.StorageType != grafanaapiserveroptions.StorageTypeLegacy - // Install the API group+version - err = builder.InstallAPIs(Scheme, Codecs, server, serverConfig.RESTOptionsGetter, builders, dualWriteEnabled) + err = builder.InstallAPIs(Scheme, Codecs, server, serverConfig.RESTOptionsGetter, builders, o.StorageOptions) if err != nil { return err } diff --git a/pkg/services/apiserver/standalone/options/options.go b/pkg/services/apiserver/standalone/options/options.go index 35feedf4844a..76085fc15586 100644 --- a/pkg/services/apiserver/standalone/options/options.go +++ b/pkg/services/apiserver/standalone/options/options.go @@ -16,6 +16,7 @@ type Options struct { TracingOptions *TracingOptions MetricsOptions *MetricsOptions ServerRunOptions *genericoptions.ServerRunOptions + StorageOptions *options.StorageOptions } func New(logger log.Logger, codec runtime.Codec) *Options { @@ -26,6 +27,7 @@ func New(logger log.Logger, codec runtime.Codec) *Options { TracingOptions: NewTracingOptions(logger), MetricsOptions: NewMetrcicsOptions(logger), ServerRunOptions: genericoptions.NewServerRunOptions(), + StorageOptions: options.NewStorageOptions(), } } diff --git a/pkg/tests/apis/playlist/playlist_test.go b/pkg/tests/apis/playlist/playlist_test.go index 50b7db5424e4..ca6b4e2fdf63 100644 --- a/pkg/tests/apis/playlist/playlist_test.go +++ b/pkg/tests/apis/playlist/playlist_test.go @@ -14,6 +14,8 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" + playlistv0alpha1 "github.com/grafana/grafana/pkg/apis/playlist/v0alpha1" + grafanarest "github.com/grafana/grafana/pkg/apiserver/rest" "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/services/playlist" "github.com/grafana/grafana/pkg/tests/apis" @@ -85,20 +87,51 @@ func TestIntegrationPlaylist(t *testing.T) { })) }) - // #TODO add equivalent tests for the other modes - t.Run("with dual write (file)", func(t *testing.T) { + t.Run("with dual write (file, mode 0)", func(t *testing.T) { doPlaylistTests(t, apis.NewK8sTestHelper(t, testinfra.GrafanaOpts{ AppModeProduction: true, DisableAnonymous: true, APIServerStorageType: "file", // write the files to disk EnableFeatureToggles: []string{ featuremgmt.FlagKubernetesPlaylists, // Required so that legacy calls are also written - featuremgmt.FlagDualWritePlaylistsMode2, + }, + DualWriterDesiredModes: map[string]grafanarest.DualWriterMode{ + playlistv0alpha1.GROUPRESOURCE: grafanarest.Mode0, }, })) }) - t.Run("with dual write (unified storage)", func(t *testing.T) { + // #TODO Enable this test once we have fixed dual writing mode 1 behavior + // Do_CRUD_via_k8s_(and_check_that_legacy_api_still_works) breaks + // t.Run("with dual write (file, mode 1)", func(t *testing.T) { + // doPlaylistTests(t, apis.NewK8sTestHelper(t, testinfra.GrafanaOpts{ + // AppModeProduction: true, + // DisableAnonymous: true, + // APIServerStorageType: "file", // write the files to disk + // EnableFeatureToggles: []string{ + // featuremgmt.FlagKubernetesPlaylists, // Required so that legacy calls are also written + // }, + // DualWriterDesiredModes: map[string]grafanarest.DualWriterMode{ + // playlistv0alpha1.GROUPRESOURCE: grafanarest.Mode1, + // }, + // })) + // }) + + t.Run("with dual write (file, mode 2)", func(t *testing.T) { + doPlaylistTests(t, apis.NewK8sTestHelper(t, testinfra.GrafanaOpts{ + AppModeProduction: true, + DisableAnonymous: true, + APIServerStorageType: "file", // write the files to disk + EnableFeatureToggles: []string{ + featuremgmt.FlagKubernetesPlaylists, // Required so that legacy calls are also written + }, + DualWriterDesiredModes: map[string]grafanarest.DualWriterMode{ + playlistv0alpha1.GROUPRESOURCE: grafanarest.Mode2, + }, + })) + }) + + t.Run("with dual write (unified storage, mode 0)", func(t *testing.T) { doPlaylistTests(t, apis.NewK8sTestHelper(t, testinfra.GrafanaOpts{ AppModeProduction: false, // required for unified storage DisableAnonymous: true, @@ -106,12 +139,73 @@ func TestIntegrationPlaylist(t *testing.T) { EnableFeatureToggles: []string{ featuremgmt.FlagUnifiedStorage, featuremgmt.FlagKubernetesPlaylists, // Required so that legacy calls are also written - featuremgmt.FlagDualWritePlaylistsMode2, + }, + DualWriterDesiredModes: map[string]grafanarest.DualWriterMode{ + playlistv0alpha1.GROUPRESOURCE: grafanarest.Mode0, }, })) }) - t.Run("with dual write (etcd)", func(t *testing.T) { + // #TODO Enable this test once we have fixed dual writing mode 1 behavior + // Do_CRUD_via_k8s_(and_check_that_legacy_api_still_works) breaks + // t.Run("with dual write (unified storage, mode 1)", func(t *testing.T) { + // doPlaylistTests(t, apis.NewK8sTestHelper(t, testinfra.GrafanaOpts{ + // AppModeProduction: false, // required for unified storage + // DisableAnonymous: true, + // APIServerStorageType: "unified", // use the entity api tables + // EnableFeatureToggles: []string{ + // featuremgmt.FlagUnifiedStorage, + // featuremgmt.FlagKubernetesPlaylists, // Required so that legacy calls are also written + // }, + // DualWriterDesiredModes: map[string]grafanarest.DualWriterMode{ + // playlistv0alpha1.GROUPRESOURCE: grafanarest.Mode1, + // }, + // })) + // }) + + t.Run("with dual write (unified storage, mode 2)", func(t *testing.T) { + doPlaylistTests(t, apis.NewK8sTestHelper(t, testinfra.GrafanaOpts{ + AppModeProduction: false, // required for unified storage + DisableAnonymous: true, + APIServerStorageType: "unified", // use the entity api tables + EnableFeatureToggles: []string{ + featuremgmt.FlagUnifiedStorage, + featuremgmt.FlagKubernetesPlaylists, // Required so that legacy calls are also written + }, + DualWriterDesiredModes: map[string]grafanarest.DualWriterMode{ + playlistv0alpha1.GROUPRESOURCE: grafanarest.Mode2, + }, + })) + }) + + t.Run("with dual write (etcd, mode 0)", func(t *testing.T) { + // NOTE: running local etcd, that will be wiped clean! + t.Skip("local etcd testing") + + helper := apis.NewK8sTestHelper(t, testinfra.GrafanaOpts{ + AppModeProduction: true, + DisableAnonymous: true, + APIServerStorageType: "etcd", // requires etcd running on localhost:2379 + EnableFeatureToggles: []string{ + featuremgmt.FlagKubernetesPlaylists, // Required so that legacy calls are also written + }, + DualWriterDesiredModes: map[string]grafanarest.DualWriterMode{ + playlistv0alpha1.GROUPRESOURCE: grafanarest.Mode0, + }, + }) + + // Clear the collection before starting (etcd) + client := helper.GetResourceClient(apis.ResourceClientArgs{ + User: helper.Org1.Admin, + GVR: gvr, + }) + err := client.Resource.DeleteCollection(context.Background(), metav1.DeleteOptions{}, metav1.ListOptions{}) + require.NoError(t, err) + + doPlaylistTests(t, helper) + }) + + t.Run("with dual write (etcd, mode 1)", func(t *testing.T) { // NOTE: running local etcd, that will be wiped clean! t.Skip("local etcd testing") @@ -121,7 +215,36 @@ func TestIntegrationPlaylist(t *testing.T) { APIServerStorageType: "etcd", // requires etcd running on localhost:2379 EnableFeatureToggles: []string{ featuremgmt.FlagKubernetesPlaylists, // Required so that legacy calls are also written - featuremgmt.FlagDualWritePlaylistsMode2, + }, + DualWriterDesiredModes: map[string]grafanarest.DualWriterMode{ + playlistv0alpha1.GROUPRESOURCE: grafanarest.Mode1, + }, + }) + + // Clear the collection before starting (etcd) + client := helper.GetResourceClient(apis.ResourceClientArgs{ + User: helper.Org1.Admin, + GVR: gvr, + }) + err := client.Resource.DeleteCollection(context.Background(), metav1.DeleteOptions{}, metav1.ListOptions{}) + require.NoError(t, err) + + doPlaylistTests(t, helper) + }) + + t.Run("with dual write (etcd, mode 2)", func(t *testing.T) { + // NOTE: running local etcd, that will be wiped clean! + t.Skip("local etcd testing") + + helper := apis.NewK8sTestHelper(t, testinfra.GrafanaOpts{ + AppModeProduction: true, + DisableAnonymous: true, + APIServerStorageType: "etcd", // requires etcd running on localhost:2379 + EnableFeatureToggles: []string{ + featuremgmt.FlagKubernetesPlaylists, // Required so that legacy calls are also written + }, + DualWriterDesiredModes: map[string]grafanarest.DualWriterMode{ + playlistv0alpha1.GROUPRESOURCE: grafanarest.Mode2, }, }) diff --git a/pkg/tests/testinfra/testinfra.go b/pkg/tests/testinfra/testinfra.go index c5c9b14452ce..277fd4e5b9be 100644 --- a/pkg/tests/testinfra/testinfra.go +++ b/pkg/tests/testinfra/testinfra.go @@ -17,6 +17,7 @@ import ( "gopkg.in/ini.v1" "github.com/grafana/grafana/pkg/api" + grafanarest "github.com/grafana/grafana/pkg/apiserver/rest" "github.com/grafana/grafana/pkg/extensions" "github.com/grafana/grafana/pkg/infra/db" "github.com/grafana/grafana/pkg/infra/fs" @@ -384,6 +385,15 @@ func CreateGrafDir(t *testing.T, opts ...GrafanaOpts) (string, string) { _, err = grafanaComSection.NewKey("api_url", o.GrafanaComAPIURL) require.NoError(t, err) } + + if o.DualWriterDesiredModes != nil { + unifiedStorageMode, err := getOrCreateSection("unified_storage_mode") + require.NoError(t, err) + for k, v := range o.DualWriterDesiredModes { + _, err = unifiedStorageMode.NewKey(k, fmt.Sprint(v)) + require.NoError(t, err) + } + } } logSection, err := getOrCreateSection("database") require.NoError(t, err) @@ -431,6 +441,7 @@ type GrafanaOpts struct { QueryRetries int64 APIServerStorageType string GrafanaComAPIURL string + DualWriterDesiredModes map[string]grafanarest.DualWriterMode } func CreateUser(t *testing.T, store db.DB, cfg *setting.Cfg, cmd user.CreateUserCommand) *user.User {