From eef5eb6db43a0ebbbb2267e9fc18c524cb35db70 Mon Sep 17 00:00:00 2001 From: Artsiom Koltun Date: Fri, 28 Apr 2023 10:18:23 +0200 Subject: [PATCH 1/3] Initial implementation Signed-off-by: Artsiom Koltun --- pkg/volume/descriptor.go | 37 ++++++++++ pkg/volume/registry.go | 49 +++++++++++++ pkg/volume/stack.go | 44 ++++++++++++ pkg/volume/volume.go | 150 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 280 insertions(+) create mode 100644 pkg/volume/descriptor.go create mode 100644 pkg/volume/registry.go create mode 100644 pkg/volume/stack.go create mode 100644 pkg/volume/volume.go diff --git a/pkg/volume/descriptor.go b/pkg/volume/descriptor.go new file mode 100644 index 00000000..584e1e33 --- /dev/null +++ b/pkg/volume/descriptor.go @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) 2023 Intel Corporation + +// Package volume contains volume layer abstractions +package volume + +import pb "github.com/opiproject/opi-api/storage/v1alpha1/gen/go" + +type Descriptor struct { + value any +} + +// return copies?? + +func (d Descriptor) ToNullDebug() *pb.NullDebug { + val, ok := d.value.(*pb.NullDebug) + if ok { + return val + } + return nil +} + +func (d Descriptor) ToQosVolume() *pb.QosVolume { + val, ok := d.value.(*pb.QosVolume) + if ok { + return val + } + return nil +} + +func (d Descriptor) ToEncryptedVolume() *pb.EncryptedVolume { + val, ok := d.value.(*pb.EncryptedVolume) + if ok { + return val + } + return nil +} diff --git a/pkg/volume/registry.go b/pkg/volume/registry.go new file mode 100644 index 00000000..94161f2d --- /dev/null +++ b/pkg/volume/registry.go @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) 2023 Intel Corporation + +// Package volume contains volume layer abstractions +package volume + +import "fmt" + +type Registry struct { + volumes map[string]*Volume +} + +func NewRegistry() *Registry { + return &Registry{ + volumes: make(map[string]*Volume), + } +} + +func (r *Registry) Find(ID string) *Volume { + vol, ok := r.volumes[ID] + if !ok { + return nil + } + return vol +} + +func (r *Registry) Add(ID string, volume *Volume) error { + _, ok := r.volumes[ID] + if ok { + return fmt.Errorf("Volume %v already exists", ID) + } + volume.addToStack() + r.volumes[ID] = volume + return nil +} + +func (r *Registry) Delete(ID string) error { + vol, ok := r.volumes[ID] + if !ok { + return fmt.Errorf("Volume %v not found", ID) + } + if err := vol.canBeDeleted(); err != nil { + return err + } + + vol.removeFromStack() + delete(r.volumes, ID) + return nil +} diff --git a/pkg/volume/stack.go b/pkg/volume/stack.go new file mode 100644 index 00000000..9a6ae88c --- /dev/null +++ b/pkg/volume/stack.go @@ -0,0 +1,44 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) 2023 Intel Corporation + +// Package volume contains volume layer abstractions +package volume + +type volumeStack struct { + volumes []*Volume +} + +func newVolumeStack() *volumeStack { + return &volumeStack{} +} + +func (s *volumeStack) top() *Volume { + i := len(s.volumes) - 1 + if i < 0 { + return nil + } + val := s.volumes[i] + return val +} + +func (s *volumeStack) push(vol *Volume) { + s.volumes = append(s.volumes, vol) +} + +func (s *volumeStack) pop() *Volume { + vol := s.top() + if vol == nil { + return nil + } + s.volumes = s.volumes[:len(s.volumes)-1] + return vol +} + +func (s *volumeStack) hasType(kind volumeType) bool { + for _, v := range s.volumes { + if v.kind == kind { + return true + } + } + return false +} diff --git a/pkg/volume/volume.go b/pkg/volume/volume.go new file mode 100644 index 00000000..bd24c14f --- /dev/null +++ b/pkg/volume/volume.go @@ -0,0 +1,150 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) 2023 Intel Corporation + +// Package volume contains volume layer abstractions +package volume + +import ( + "fmt" + + pb "github.com/opiproject/opi-api/storage/v1alpha1/gen/go" +) + +type volumeType int + +const ( + // Backend volumes + aioVolumeType volumeType = iota + nullVolumeType + remoteVolumeType + // Middleend volumes + encryptedVolumeType + qosVolumeType +) + +type Volume struct { + bdevName string + kind volumeType + stack *volumeStack + descriptor Descriptor +} + +func NewAioVolume(descr *pb.AioController) *Volume { + return newBackendVolume(descr.Handle.Value, aioVolumeType, descr) +} + +func NewNullVolume(descr *pb.NullDebug) *Volume { + return newBackendVolume(descr.Handle.Value, nullVolumeType, descr) +} + +func newBackendVolume(bdevName string, kind volumeType, descr any) *Volume { + // check bdevName is not empty + // check kind + // nil check + vol := Volume{ + bdevName: bdevName, + kind: kind, + stack: newVolumeStack(), + descriptor: Descriptor{value: descr}, + } + return &vol +} + +func (v *Volume) CreateEncryptedVolume(descr *pb.EncryptedVolume) (*Volume, error) { + vol := newMiddleendVolume(descr.EncryptedVolumeId.Value, encryptedVolumeType, descr, v.stack) + if err := v.canCreate(vol); err != nil { + return nil, err + } + return vol, nil +} + +func (v *Volume) CreateQosVolume(descr *pb.QosVolume) (*Volume, error) { + // QoS in SPDK does not create its own bdev, only applies limits to existing bdev. + // Copy parent bdev name + parent := v.stack.top() + if parent == nil { + return nil, fmt.Errorf("no underlying volume found to create volume") + } + vol := newMiddleendVolume(parent.bdevName, qosVolumeType, descr, v.stack) + if err := v.canCreate(vol); err != nil { + return nil, err + } + return vol, nil +} + +func newMiddleendVolume(bdevName string, kind volumeType, descr any, stack *volumeStack) *Volume { + // check kind + // nil check + vol := Volume{ + bdevName: bdevName, + kind: kind, + stack: stack, + descriptor: Descriptor{value: descr}, + } + return &vol +} + +func (v *Volume) BdevName() string { + return v.bdevName +} + +func (v *Volume) Descriptor() Descriptor { + return v.descriptor +} + +func (v *Volume) canBeDeleted() error { + topVol := v.stack.top() + if topVol == nil { + // should be unreachable if object is used through public API... panic? + return fmt.Errorf("no volume to delete") + } + if !v.equal(topVol) { + return fmt.Errorf("only volume on top can be deleted") + } + return nil +} + +func (v *Volume) canCreate(vol *Volume) error { + topVol := v.stack.top() + if topVol == nil { + return fmt.Errorf("no underlying volume found to create volume") + } + if !v.equal(topVol) { + return fmt.Errorf("volume is not on top of stack") + } + if vol.isBackend() { + // should be unreachable if object is created by public API... panic? + return fmt.Errorf("cannot add backend volume") + } + if v.stack.hasType(vol.kind) { + return fmt.Errorf("volume of that type already exists") + } + + return nil +} + +func (v *Volume) addToStack() { + v.stack.push(v) +} + +func (v *Volume) removeFromStack() { + v.stack.pop() +} + +func (v *Volume) isBackend() bool { + backendVolumeTypes := map[volumeType]struct{}{ + aioVolumeType: {}, + nullVolumeType: {}, + remoteVolumeType: {}, + } + _, ok := backendVolumeTypes[v.kind] + return ok +} + +func (v *Volume) isMiddleend() bool { + return !v.isBackend() +} + +func (v *Volume) equal(other *Volume) bool { + return *v == *other +} From 434e07f7f2ae4447c44e3624368e98c00f1d1d9c Mon Sep 17 00:00:00 2001 From: Artsiom Koltun Date: Fri, 28 Apr 2023 10:18:37 +0200 Subject: [PATCH 2/3] Initial implementation tests Signed-off-by: Artsiom Koltun --- pkg/volume/descriptor_test.go | 54 ++++++++ pkg/volume/registry_test.go | 234 ++++++++++++++++++++++++++++++++++ 2 files changed, 288 insertions(+) create mode 100644 pkg/volume/descriptor_test.go create mode 100644 pkg/volume/registry_test.go diff --git a/pkg/volume/descriptor_test.go b/pkg/volume/descriptor_test.go new file mode 100644 index 00000000..531a60f5 --- /dev/null +++ b/pkg/volume/descriptor_test.go @@ -0,0 +1,54 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) 2023 Intel Corporation + +// Package volume contains volume layer abstractions +package volume + +import ( + "testing" + + pc "github.com/opiproject/opi-api/common/v1/gen/go" + pb "github.com/opiproject/opi-api/storage/v1alpha1/gen/go" +) + +func TestNullVolumeReturnsItsOwnDescriptor(t *testing.T) { + value := &pb.NullDebug{Handle: &pc.ObjectKey{Value: "Handle42"}, BlockSize: 4096} + nullVol0 := NewNullVolume(value) + val := nullVol0.Descriptor().ToNullDebug() + if val != value { + t.Errorf("Expect %v equal to %v", val, value) + } +} + +func TestQosVolumeReturnsItsOwnDescriptor(t *testing.T) { + registry := NewRegistry() + nullVol0 := NewNullVolume(&pb.NullDebug{Handle: &pc.ObjectKey{Value: "Handle42"}, BlockSize: 4096}) + registry.Add("nullId0", nullVol0) + value := &pb.QosVolume{QosVolumeId: &pc.ObjectKey{Value: "Qos42"}} + qosVol0, _ := nullVol0.CreateQosVolume(value) + val := qosVol0.Descriptor().ToQosVolume() + if val != value { + t.Errorf("Expect %v equal to %v", val, value) + } +} + +func TestEncryptedVolumeReturnsItsOwnDescriptor(t *testing.T) { + registry := NewRegistry() + nullVol0 := NewNullVolume(&pb.NullDebug{Handle: &pc.ObjectKey{Value: "Handle42"}, BlockSize: 4096}) + registry.Add("nullId0", nullVol0) + value := &pb.EncryptedVolume{EncryptedVolumeId: &pc.ObjectKey{Value: "Encr42"}} + encrVol0, _ := nullVol0.CreateEncryptedVolume(value) + val := encrVol0.Descriptor().ToEncryptedVolume() + if val != value { + t.Errorf("Expect %v equal to %v", val, value) + } +} + +func TestVolumeDescriptorConvertToInvalidType(t *testing.T) { + value := &pb.NullDebug{Handle: &pc.ObjectKey{Value: "Handle42"}, BlockSize: 4096} + nullVol0 := NewNullVolume(value) + val := nullVol0.Descriptor().ToQosVolume() + if val != nil { + t.Errorf("Expect invalid type conversion to Qos Volume") + } +} diff --git a/pkg/volume/registry_test.go b/pkg/volume/registry_test.go new file mode 100644 index 00000000..e95e2f8e --- /dev/null +++ b/pkg/volume/registry_test.go @@ -0,0 +1,234 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) 2023 Intel Corporation + +// Package volume contains volume layer abstractions +package volume + +import ( + "reflect" + "testing" + + pc "github.com/opiproject/opi-api/common/v1/gen/go" + pb "github.com/opiproject/opi-api/storage/v1alpha1/gen/go" +) + +func TestAddDeviceToRegistry(t *testing.T) { + registry := NewRegistry() + nullVol0 := NewNullVolume(&pb.NullDebug{Handle: &pc.ObjectKey{Value: "Null0"}}) + + err := registry.Add("nullId0", nullVol0) + + if err != nil { + t.Error("Expected nil error, received", err) + } +} + +func TestAddNewDeviceWithExistingID(t *testing.T) { + registry := NewRegistry() + nullVol0 := NewNullVolume(&pb.NullDebug{Handle: &pc.ObjectKey{Value: "Null0"}}) + nullVol1 := NewNullVolume(&pb.NullDebug{Handle: &pc.ObjectKey{Value: "Null1"}}) + id := "nullId0" + registry.Add(id, nullVol0) + + err := registry.Add(id, nullVol1) + + if err == nil { + t.Error("Expected error, received nil") + } +} + +func TestCreateMiddleendVolumeOnlyOnVolumeInRegistry(t *testing.T) { + registry := NewRegistry() + id := "nullId0" + nullVol0 := NewNullVolume(&pb.NullDebug{Handle: &pc.ObjectKey{Value: "Null0"}}) + registry.Add(id, nullVol0) + + vol, err := nullVol0.CreateQosVolume(&pb.QosVolume{}) + + if err != nil { + t.Error("Expected nil error, received", err) + } + if vol == nil { + t.Error("Expected not nil volume") + } +} + +func TestFailedToCreateMiddleendVolumeOnVolumeNotAddedToRegistry(t *testing.T) { + registry := NewRegistry() + id := "nullId0" + nullVol0 := NewNullVolume(&pb.NullDebug{Handle: &pc.ObjectKey{Value: "Null0"}}) + registry.Add(id, nullVol0) + encVol0, _ := nullVol0.CreateEncryptedVolume(&pb.EncryptedVolume{EncryptedVolumeId: &pc.ObjectKey{Value: "EncVol0"}}) + + vol, err := encVol0.CreateQosVolume(&pb.QosVolume{}) + + if err == nil { + t.Error("Expected error, received nil") + } + if vol != nil { + t.Error("Expected nil volume") + } +} + +func TestFailedToAddVolumeToVolumeWithAnotherVolume(t *testing.T) { + registry := NewRegistry() + nullVol0 := NewNullVolume(&pb.NullDebug{Handle: &pc.ObjectKey{Value: "Null0"}}) + registry.Add("nullId0", nullVol0) + encVol0, _ := nullVol0.CreateEncryptedVolume(&pb.EncryptedVolume{EncryptedVolumeId: &pc.ObjectKey{Value: "EncVol0"}}) + registry.Add("encrId0", encVol0) + + vol, err := nullVol0.CreateQosVolume(&pb.QosVolume{}) + + if err == nil { + t.Error("Expected error, received nil") + } + if vol != nil { + t.Error("Expected nil volume") + } +} + +func TestFailedToAddVolumeOfAlreadyAddedType(t *testing.T) { + registry := NewRegistry() + nullVol0 := NewNullVolume(&pb.NullDebug{Handle: &pc.ObjectKey{Value: "Null0"}}) + registry.Add("nullId0", nullVol0) + encVol0, _ := nullVol0.CreateEncryptedVolume(&pb.EncryptedVolume{EncryptedVolumeId: &pc.ObjectKey{Value: "EncVol0"}}) + registry.Add("encrId0", encVol0) + + vol, err := encVol0.CreateEncryptedVolume(&pb.EncryptedVolume{EncryptedVolumeId: &pc.ObjectKey{Value: "EncVol1"}}) + + if err == nil { + t.Error("Expected error, received nil") + } + if vol != nil { + t.Error("Expected nil volume") + } +} + +func TestCreateQosVolumeOnBackendVolumeNotAddedToRegistry(t *testing.T) { + nullVol0 := NewNullVolume(&pb.NullDebug{Handle: &pc.ObjectKey{Value: "Null0"}}) + + vol, err := nullVol0.CreateQosVolume(&pb.QosVolume{}) + + if err == nil { + t.Error("Expected error, received nil") + } + if vol != nil { + t.Error("Expected nil volume") + } +} + +func TestCreateEncryptedVolumeOnBackendVolumeNotAddedToRegistry(t *testing.T) { + nullVol0 := NewNullVolume(&pb.NullDebug{Handle: &pc.ObjectKey{Value: "Null0"}}) + + vol, err := nullVol0.CreateEncryptedVolume(&pb.EncryptedVolume{EncryptedVolumeId: &pc.ObjectKey{Value: "EncVol0"}}) + + if err == nil { + t.Error("Expected error, received nil") + } + if vol != nil { + t.Error("Expected nil volume") + } +} + +func TestBackendVolumeReturnsItsOwnBdevName(t *testing.T) { + nullVol0 := NewNullVolume(&pb.NullDebug{Handle: &pc.ObjectKey{Value: "Null0"}}) + + if nullVol0.BdevName() != "Null0" { + t.Error("Expected Null0 as bdev name, received", nullVol0.BdevName()) + } +} + +func TestQosVolumeReturnsBdevNameOfUnderlyingVolume(t *testing.T) { + registry := NewRegistry() + nullVol0 := NewNullVolume(&pb.NullDebug{Handle: &pc.ObjectKey{Value: "Null0"}}) + registry.Add("nullId0", nullVol0) + encVol0, _ := nullVol0.CreateEncryptedVolume(&pb.EncryptedVolume{EncryptedVolumeId: &pc.ObjectKey{Value: "EncVol0"}}) + registry.Add("encrId0", encVol0) + + qosVol0, _ := encVol0.CreateQosVolume(&pb.QosVolume{}) + + if qosVol0.BdevName() != "EncVol0" { + t.Error("Expected EncVol0 as bdev name, received", qosVol0.BdevName()) + } +} + +func TestEncryptedVolumeReturnsItsOwnBdevName(t *testing.T) { + registry := NewRegistry() + nullVol0 := NewNullVolume(&pb.NullDebug{Handle: &pc.ObjectKey{Value: "Null0"}}) + registry.Add("nullId0", nullVol0) + qosVol0, _ := nullVol0.CreateQosVolume(&pb.QosVolume{}) + registry.Add("qosId0", qosVol0) + + encVol0, _ := qosVol0.CreateEncryptedVolume(&pb.EncryptedVolume{EncryptedVolumeId: &pc.ObjectKey{Value: "EncVol0"}}) + + if encVol0.BdevName() != "EncVol0" { + t.Error("Expected EncVol0 as bdev name, received", encVol0.BdevName()) + } +} + +func TestCannotDeleteNotAddedDevice(t *testing.T) { + registry := NewRegistry() + id := "nullId0" + + err := registry.Delete(id) + + if err == nil { + t.Error("Expected error, received nil") + } +} + +func TestDeleteAddedDevice(t *testing.T) { + registry := NewRegistry() + id := "nullId0" + nullVol0 := NewNullVolume(&pb.NullDebug{Handle: &pc.ObjectKey{Value: "Null0"}}) + registry.Add(id, nullVol0) + + err := registry.Delete(id) + + if err != nil { + t.Error("Expected nil error, received", err) + } +} + +func TestDeleteVolumeNotFromTopOfStack(t *testing.T) { + registry := NewRegistry() + nullVol0 := NewNullVolume(&pb.NullDebug{Handle: &pc.ObjectKey{Value: "Null0"}}) + registry.Add("nullId0", nullVol0) + encVol0, _ := nullVol0.CreateEncryptedVolume(&pb.EncryptedVolume{EncryptedVolumeId: &pc.ObjectKey{Value: "EncVol0"}}) + registry.Add("encrId0", encVol0) + qosVol0, _ := encVol0.CreateQosVolume(&pb.QosVolume{}) + registry.Add("qosVol0", qosVol0) + + err := registry.Delete("encrId0") + + if err == nil { + t.Error("Expected error, received nil") + } +} + +func TestAddedDeviceCanBeFound(t *testing.T) { + registry := NewRegistry() + nullVol0 := NewNullVolume(&pb.NullDebug{Handle: &pc.ObjectKey{Value: "Null0"}}) + id := "nullId0" + registry.Add(id, nullVol0) + + foundVol := registry.Find(id) + + if foundVol == nil { + t.Error("Expected to find volume") + } + if !reflect.DeepEqual(nullVol0, foundVol) { + t.Error("Expected", nullVol0, "found", foundVol) + } +} + +func TestNotAddedDeviceCannotBeFound(t *testing.T) { + registry := NewRegistry() + id := "nullId0" + + foundVol := registry.Find(id) + + if foundVol != nil { + t.Error("Expected volume not found") + } +} From fb86cd87fe36a040e8772054c230880f5aeb3541 Mon Sep 17 00:00:00 2001 From: Artsiom Koltun Date: Fri, 28 Apr 2023 10:18:49 +0200 Subject: [PATCH 3/3] Exemplary integration Signed-off-by: Artsiom Koltun --- cmd/main.go | 6 +++-- pkg/backend/backend.go | 5 +++- pkg/backend/null.go | 32 +++++++++++++++++-------- pkg/middleend/encryption.go | 47 ++++++++++++++++++++++++++++++++----- pkg/middleend/middleend.go | 5 +++- pkg/middleend/qos.go | 39 +++++++++++++++++++++--------- 6 files changed, 103 insertions(+), 31 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 6ba6ac68..40dfc450 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -15,6 +15,7 @@ import ( "github.com/opiproject/opi-spdk-bridge/pkg/frontend" "github.com/opiproject/opi-spdk-bridge/pkg/kvm" "github.com/opiproject/opi-spdk-bridge/pkg/middleend" + "github.com/opiproject/opi-spdk-bridge/pkg/volume" pb "github.com/opiproject/opi-api/storage/v1alpha1/gen/go" "google.golang.org/grpc" @@ -47,9 +48,10 @@ func main() { } s := grpc.NewServer() + registry := volume.NewRegistry() jsonRPC := spdk.NewSpdkJSONRPC(spdkAddress) - backendServer := backend.NewServer(jsonRPC) - middleendServer := middleend.NewServer(jsonRPC) + backendServer := backend.NewServer(jsonRPC, registry) + middleendServer := middleend.NewServer(jsonRPC, registry) if useKvm { log.Println("Creating KVM server.") diff --git a/pkg/backend/backend.go b/pkg/backend/backend.go index 9072439b..54074873 100644 --- a/pkg/backend/backend.go +++ b/pkg/backend/backend.go @@ -8,6 +8,7 @@ package backend import ( "github.com/opiproject/gospdk/spdk" pb "github.com/opiproject/opi-api/storage/v1alpha1/gen/go" + "github.com/opiproject/opi-spdk-bridge/pkg/volume" ) // TODO: can we combine all of volume types into a single list? @@ -29,11 +30,12 @@ type Server struct { rpc spdk.JSONRPC Volumes VolumeParameters Pagination map[string]int + registry *volume.Registry } // NewServer creates initialized instance of BackEnd server communicating // with provided jsonRPC -func NewServer(jsonRPC spdk.JSONRPC) *Server { +func NewServer(jsonRPC spdk.JSONRPC, registry *volume.Registry) *Server { return &Server{ rpc: jsonRPC, Volumes: VolumeParameters{ @@ -42,5 +44,6 @@ func NewServer(jsonRPC spdk.JSONRPC) *Server { NvmeVolumes: make(map[string]*pb.NVMfRemoteController), }, Pagination: make(map[string]int), + registry: registry, } } diff --git a/pkg/backend/null.go b/pkg/backend/null.go index ae7f406e..cae0c48f 100644 --- a/pkg/backend/null.go +++ b/pkg/backend/null.go @@ -14,6 +14,7 @@ import ( pc "github.com/opiproject/opi-api/common/v1/gen/go" pb "github.com/opiproject/opi-api/storage/v1alpha1/gen/go" "github.com/opiproject/opi-spdk-bridge/pkg/server" + "github.com/opiproject/opi-spdk-bridge/pkg/volume" "github.com/google/uuid" "github.com/ulule/deepcopier" @@ -26,14 +27,20 @@ import ( func (s *Server) CreateNullDebug(_ context.Context, in *pb.CreateNullDebugRequest) (*pb.NullDebug, error) { log.Printf("CreateNullDebug: Received from client: %v", in) // idempotent API when called with same key, should return same object - volume, ok := s.Volumes.NullVolumes[in.NullDebug.Handle.Value] - if ok { - log.Printf("Already existing NullDebug with id %v", in.NullDebug.Handle.Value) - return volume, nil + // vol, ok := s.Volumes.NullVolumes[in.NullDebug.Handle.Value] + vol := s.registry.Find(in.NullDebug.Handle.Value) + if vol != nil { + if null := vol.Descriptor().ToNullDebug(); null != nil { + log.Printf("Already existing NullDebug with id %v", in.NullDebug.Handle.Value) + return null, nil + } else { + // panic? + } } + vol = volume.NewNullVolume(in.NullDebug) // not found, so create a new one params := spdk.BdevNullCreateParams{ - Name: in.NullDebug.Handle.Value, + Name: vol.BdevName(), BlockSize: 512, NumBlocks: 64, } @@ -55,6 +62,7 @@ func (s *Server) CreateNullDebug(_ context.Context, in *pb.CreateNullDebugReques log.Printf("error: %v", err) return nil, err } + s.registry.Add(in.NullDebug.Handle.Value, vol) s.Volumes.NullVolumes[in.NullDebug.Handle.Value] = response return response, nil } @@ -62,8 +70,8 @@ func (s *Server) CreateNullDebug(_ context.Context, in *pb.CreateNullDebugReques // DeleteNullDebug deletes a Null Debug instance func (s *Server) DeleteNullDebug(_ context.Context, in *pb.DeleteNullDebugRequest) (*emptypb.Empty, error) { log.Printf("DeleteNullDebug: Received from client: %v", in) - volume, ok := s.Volumes.NullVolumes[in.Name] - if !ok { + vol := s.registry.Find(in.Name) + if vol == nil { if in.AllowMissing { return &emptypb.Empty{}, nil } @@ -71,22 +79,26 @@ func (s *Server) DeleteNullDebug(_ context.Context, in *pb.DeleteNullDebugReques log.Printf("error: %v", err) return nil, err } + + err := s.registry.Delete(in.Name) + if err != nil { + return nil, status.Error(codes.Unknown, err.Error()) + } params := spdk.BdevNullDeleteParams{ Name: in.Name, } var result spdk.BdevNullDeleteResult - err := s.rpc.Call("bdev_null_delete", ¶ms, &result) + err = s.rpc.Call("bdev_null_delete", ¶ms, &result) if err != nil { log.Printf("error: %v", err) return nil, err } log.Printf("Received from SPDK: %v", result) if !result { - msg := fmt.Sprintf("Could not delete Null Dev: %s", volume.Handle.Value) + msg := fmt.Sprintf("Could not delete Null Dev: %s", in.Name) log.Print(msg) return nil, status.Errorf(codes.InvalidArgument, msg) } - delete(s.Volumes.NullVolumes, volume.Handle.Value) return &emptypb.Empty{}, nil } diff --git a/pkg/middleend/encryption.go b/pkg/middleend/encryption.go index 59ba2bbd..a3f23bbb 100644 --- a/pkg/middleend/encryption.go +++ b/pkg/middleend/encryption.go @@ -30,6 +30,24 @@ func (s *Server) CreateEncryptedVolume(_ context.Context, in *pb.CreateEncrypted return nil, status.Error(codes.InvalidArgument, err.Error()) } + encrVol := s.registry.Find(in.EncryptedVolume.EncryptedVolumeId.Value) + if encrVol != nil { + log.Printf("Already existing encrypted volume with id %v", in.EncryptedVolume.EncryptedVolumeId.Value) + if encr := encrVol.Descriptor().ToEncryptedVolume(); encr != nil { + return encr, nil + } else { + // panic? + } + } + vol := s.registry.Find(in.EncryptedVolume.VolumeId.Value) + if vol == nil { + return nil, status.Errorf(codes.NotFound, "Underlying volume %v not found", in.EncryptedVolume.VolumeId.Value) + } + encrVol, err := vol.CreateEncryptedVolume(in.EncryptedVolume) + if err != nil { + return nil, status.Errorf(codes.InvalidArgument, err.Error()) + } + // first create a key params1 := s.getAccelCryptoKeyCreateParams(in.EncryptedVolume) var result1 spdk.AccelCryptoKeyCreateResult @@ -46,12 +64,12 @@ func (s *Server) CreateEncryptedVolume(_ context.Context, in *pb.CreateEncrypted } // create bdev now params := spdk.BdevCryptoCreateParams{ - Name: in.EncryptedVolume.EncryptedVolumeId.Value, - BaseBdevName: in.EncryptedVolume.VolumeId.Value, + Name: encrVol.BdevName(), + BaseBdevName: vol.BdevName(), KeyName: in.EncryptedVolume.EncryptedVolumeId.Value, } var result spdk.BdevCryptoCreateResult - err := s.rpc.Call("bdev_crypto_create", ¶ms, &result) + err = s.rpc.Call("bdev_crypto_create", ¶ms, &result) if err != nil { log.Printf("error: %v", err) return nil, err @@ -68,17 +86,34 @@ func (s *Server) CreateEncryptedVolume(_ context.Context, in *pb.CreateEncrypted log.Printf("error: %v", err) return nil, err } + + s.registry.Add(in.EncryptedVolume.EncryptedVolumeId.Value, encrVol) return response, nil } // DeleteEncryptedVolume deletes an encrypted volume func (s *Server) DeleteEncryptedVolume(_ context.Context, in *pb.DeleteEncryptedVolumeRequest) (*emptypb.Empty, error) { log.Printf("DeleteEncryptedVolume: Received from client: %v", in) + + vol := s.registry.Find(in.Name) + if vol == nil { + if in.AllowMissing { + return &emptypb.Empty{}, nil + } + err := status.Errorf(codes.NotFound, "unable to find key %s", in.Name) + log.Printf("error: %v", err) + return nil, err + } + + err := s.registry.Delete(in.Name) + if err != nil { + return nil, status.Error(codes.Unknown, err.Error()) + } bdevCryptoDeleteParams := spdk.BdevCryptoDeleteParams{ - Name: in.Name, + Name: vol.BdevName(), } var bdevCryptoDeleteResult spdk.BdevCryptoDeleteResult - err := s.rpc.Call("bdev_crypto_delete", &bdevCryptoDeleteParams, &bdevCryptoDeleteResult) + err = s.rpc.Call("bdev_crypto_delete", &bdevCryptoDeleteParams, &bdevCryptoDeleteResult) if err != nil { log.Printf("error: %v", err) return nil, err @@ -91,7 +126,7 @@ func (s *Server) DeleteEncryptedVolume(_ context.Context, in *pb.DeleteEncrypted } keyDestroyParams := spdk.AccelCryptoKeyDestroyParams{ - KeyName: in.Name, + KeyName: vol.BdevName(), } var keyDestroyResult spdk.AccelCryptoKeyDestroyResult err = s.rpc.Call("accel_crypto_key_destroy", &keyDestroyParams, &keyDestroyResult) diff --git a/pkg/middleend/middleend.go b/pkg/middleend/middleend.go index 04e56325..697cce3a 100644 --- a/pkg/middleend/middleend.go +++ b/pkg/middleend/middleend.go @@ -8,6 +8,7 @@ package middleend import ( "github.com/opiproject/gospdk/spdk" pb "github.com/opiproject/opi-api/storage/v1alpha1/gen/go" + "github.com/opiproject/opi-spdk-bridge/pkg/volume" ) // VolumeParameters contains MiddleEnd volume related structures @@ -23,16 +24,18 @@ type Server struct { rpc spdk.JSONRPC volumes VolumeParameters Pagination map[string]int + registry *volume.Registry } // NewServer creates initialized instance of MiddleEnd server communicating // with provided jsonRPC -func NewServer(jsonRPC spdk.JSONRPC) *Server { +func NewServer(jsonRPC spdk.JSONRPC, registry *volume.Registry) *Server { return &Server{ rpc: jsonRPC, volumes: VolumeParameters{ qosVolumes: make(map[string]*pb.QosVolume), }, Pagination: make(map[string]int), + registry: registry, } } diff --git a/pkg/middleend/qos.go b/pkg/middleend/qos.go index af8df485..a8d92bbf 100644 --- a/pkg/middleend/qos.go +++ b/pkg/middleend/qos.go @@ -13,7 +13,6 @@ import ( pb "github.com/opiproject/opi-api/storage/v1alpha1/gen/go" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/emptypb" ) @@ -24,20 +23,33 @@ func (s *Server) CreateQosVolume(_ context.Context, in *pb.CreateQosVolumeReques log.Println("error:", err) return nil, status.Error(codes.InvalidArgument, err.Error()) } - if volume, ok := s.volumes.qosVolumes[in.QosVolume.QosVolumeId.Value]; ok { + qosVol := s.registry.Find(in.QosVolume.QosVolumeId.Value) + if qosVol != nil { log.Printf("Already existing QoS volume with id %v", in.QosVolume.QosVolumeId.Value) - return volume, nil + if qos := qosVol.Descriptor().ToQosVolume(); qos != nil { + return qos, nil + } else { + // panic? + } + } + vol := s.registry.Find(in.QosVolume.VolumeId.Value) + if vol == nil { + return nil, status.Errorf(codes.NotFound, "Underlying volume %v not found", in.QosVolume.VolumeId.Value) + } + qosVol, err := vol.CreateQosVolume(in.QosVolume) + if err != nil { + return nil, status.Errorf(codes.InvalidArgument, err.Error()) } params := spdk.BdevQoSParams{ - Name: in.QosVolume.VolumeId.Value, + Name: qosVol.BdevName(), RwIosPerSec: int(in.QosVolume.LimitMax.RwIopsKiops * 1000), RwMbytesPerSec: int(in.QosVolume.LimitMax.RwBandwidthMbs), RMbytesPerSec: int(in.QosVolume.LimitMax.RdBandwidthMbs), WMbytesPerSec: int(in.QosVolume.LimitMax.RdBandwidthMbs), } var result spdk.BdevQoSResult - err := s.rpc.Call("bdev_set_qos_limit", ¶ms, &result) + err = s.rpc.Call("bdev_set_qos_limit", ¶ms, &result) if err != nil { log.Printf("error: %v", err) return nil, spdk.ErrFailedSpdkCall @@ -49,15 +61,16 @@ func (s *Server) CreateQosVolume(_ context.Context, in *pb.CreateQosVolumeReques return nil, spdk.ErrUnexpectedSpdkCallResult } - s.volumes.qosVolumes[in.QosVolume.QosVolumeId.Value] = proto.Clone(in.QosVolume).(*pb.QosVolume) + s.registry.Add(in.QosVolume.QosVolumeId.Value, qosVol) + // s.volumes.qosVolumes[in.QosVolume.QosVolumeId.Value] = proto.Clone(in.QosVolume).(*pb.QosVolume) return in.QosVolume, nil } // DeleteQosVolume creates a QoS volume func (s *Server) DeleteQosVolume(_ context.Context, in *pb.DeleteQosVolumeRequest) (*emptypb.Empty, error) { log.Printf("CreateQosVolume: Received from client: %v", in) - qosVolume, ok := s.volumes.qosVolumes[in.Name] - if !ok { + vol := s.registry.Find(in.Name) + if vol == nil { if in.AllowMissing { return &emptypb.Empty{}, nil } @@ -65,15 +78,19 @@ func (s *Server) DeleteQosVolume(_ context.Context, in *pb.DeleteQosVolumeReques log.Printf("error: %v", err) return nil, err } + err := s.registry.Delete(in.Name) + if err != nil { + return nil, status.Error(codes.Unknown, err.Error()) + } params := spdk.BdevQoSParams{ - Name: qosVolume.VolumeId.Value, + Name: vol.BdevName(), RwIosPerSec: 0, RwMbytesPerSec: 0, RMbytesPerSec: 0, WMbytesPerSec: 0, } var result spdk.BdevQoSResult - err := s.rpc.Call("bdev_set_qos_limit", ¶ms, &result) + err = s.rpc.Call("bdev_set_qos_limit", ¶ms, &result) if err != nil { log.Printf("error: %v", err) return nil, spdk.ErrFailedSpdkCall @@ -85,7 +102,7 @@ func (s *Server) DeleteQosVolume(_ context.Context, in *pb.DeleteQosVolumeReques return nil, spdk.ErrUnexpectedSpdkCallResult } - delete(s.volumes.qosVolumes, in.Name) + // delete(s.volumes.qosVolumes, in.Name) return &emptypb.Empty{}, nil }