Add wrappers for EOS and EOS Home storage drivers (cs3org#1624)

changelog/unreleased/eos-wrappers.md

@@ -0,0 +1,9 @@
+Enhancement: Add wrappers for EOS and EOS Home storage drivers
+
+For CERNBox, we need the mount ID to be configured according to the owner of a
+resource. Setting this in the storageprovider means having different instances
+of this service to cater to different users, which does not scale. This driver
+forms a wrapper around the EOS driver and sets the mount ID according to a
+configurable mapping based on the owner of the resource.
+
+https://github.com/cs3org/reva/pull/1624

internal/grpc/services/storageprovider/storageprovider.go

@@ -1142,7 +1142,10 @@ func (s *service) trimMountPrefix(fn string) (string, error) {
 }
 
 func (s *service) wrap(ctx context.Context, ri *provider.ResourceInfo) error {
-	ri.Id.StorageId = s.mountID
+	if ri.Id.StorageId == "" {
+		// For wrapper drivers, the storage ID might already be set. In that case, skip setting it
+		ri.Id.StorageId = s.mountID
+	}
 	ri.Path = path.Join(s.mountPath, ri.Path)
 	return nil
 }

pkg/cbox/loader/loader.go

@@ -23,5 +23,7 @@ import (
 	_ "github.com/cs3org/reva/pkg/cbox/group/rest"
 	_ "github.com/cs3org/reva/pkg/cbox/publicshare/sql"
 	_ "github.com/cs3org/reva/pkg/cbox/share/sql"
+	_ "github.com/cs3org/reva/pkg/cbox/storage/eoshomewrapper"
+	_ "github.com/cs3org/reva/pkg/cbox/storage/eoswrapper"
 	_ "github.com/cs3org/reva/pkg/cbox/user/rest"
 )

pkg/cbox/publicshare/sql/sql.go

@@ -437,7 +437,7 @@ func (m *manager) cleanupExpiredShares() error {
 		return nil
 	}
 
-	query := "delete from oc_share where expiration IS NOT NULL AND expiration < ?"
+	query := "update oc_share set orphan = 1 where expiration IS NOT NULL AND expiration < ?"
 	params := []interface{}{time.Now().Format("2006-01-02 03:04:05")}
 
 	stmt, err := m.db.Prepare(query)

pkg/cbox/storage/eoshomewrapper/eoshomewrapper.go

@@ -0,0 +1,123 @@
+// Copyright 2018-2021 CERN
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// In applying this license, CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+package eoshome
+
+import (
+	"bytes"
+	"context"
+	"text/template"
+
+	"github.com/Masterminds/sprig"
+	provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
+	"github.com/cs3org/reva/pkg/storage"
+	"github.com/cs3org/reva/pkg/storage/fs/registry"
+	"github.com/cs3org/reva/pkg/storage/utils/eosfs"
+	"github.com/mitchellh/mapstructure"
+	"github.com/pkg/errors"
+)
+
+func init() {
+	registry.Register("eoshomewrapper", New)
+}
+
+type wrapper struct {
+	storage.FS
+	mountIDTemplate *template.Template
+}
+
+func parseConfig(m map[string]interface{}) (*eosfs.Config, string, error) {
+	c := &eosfs.Config{}
+	if err := mapstructure.Decode(m, c); err != nil {
+		err = errors.Wrap(err, "error decoding conf")
+		return nil, "", err
+	}
+
+	// default to version invariance if not configured
+	if _, ok := m["version_invariant"]; !ok {
+		c.VersionInvariant = true
+	}
+
+	t, ok := m["mount_id_template"].(string)
+	if !ok || t == "" {
+		t = "eoshome-{{ trimAll \"/\" .Path | substr 0 1 }}"
+	}
+
+	return c, t, nil
+}
+
+// New returns an implementation of the storage.FS interface that forms a wrapper
+// around separate connections to EOS.
+func New(m map[string]interface{}) (storage.FS, error) {
+	c, t, err := parseConfig(m)
+	if err != nil {
+		return nil, err
+	}
+	c.EnableHome = true
+
+	eos, err := eosfs.NewEOSFS(c)
+	if err != nil {
+		return nil, err
+	}
+
+	mountIDTemplate, err := template.New("mountID").Funcs(sprig.TxtFuncMap()).Parse(t)
+	if err != nil {
+		return nil, err
+	}
+
+	return &wrapper{FS: eos, mountIDTemplate: mountIDTemplate}, nil
+}
+
+// We need to override the two methods, GetMD and ListFolder to fill the
+// StorageId in the ResourceInfo objects.
+
+func (w *wrapper) GetMD(ctx context.Context, ref *provider.Reference, mdKeys []string) (*provider.ResourceInfo, error) {
+	res, err := w.FS.GetMD(ctx, ref, mdKeys)
+	if err != nil {
+		return nil, err
+	}
+
+	// We need to extract the mount ID based on the mapping template.
+	//
+	// Take the first letter of the resource path after the namespace has been removed.
+	// If it's empty, leave it empty to be filled by storageprovider.
+	res.Id.StorageId = w.getMountID(ctx, res)
+	return res, nil
+}
+
+func (w *wrapper) ListFolder(ctx context.Context, ref *provider.Reference, mdKeys []string) ([]*provider.ResourceInfo, error) {
+	res, err := w.FS.ListFolder(ctx, ref, mdKeys)
+	if err != nil {
+		return nil, err
+	}
+	for _, r := range res {
+		r.Id.StorageId = w.getMountID(ctx, r)
+	}
+	return res, nil
+}
+
+func (w *wrapper) getMountID(ctx context.Context, r *provider.ResourceInfo) string {
+	if r == nil {
+		return ""
+	}
+	b := bytes.Buffer{}
+	if err := w.mountIDTemplate.Execute(&b, r); err != nil {
+		return ""
+	}
+	return b.String()
+}

pkg/cbox/storage/eoswrapper/eoswrapper.go

@@ -0,0 +1,123 @@
+// Copyright 2018-2021 CERN
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// In applying this license, CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+package eoshome
+
+import (
+	"bytes"
+	"context"
+	"text/template"
+
+	"github.com/Masterminds/sprig"
+	provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
+	"github.com/cs3org/reva/pkg/storage"
+	"github.com/cs3org/reva/pkg/storage/fs/registry"
+	"github.com/cs3org/reva/pkg/storage/utils/eosfs"
+	"github.com/mitchellh/mapstructure"
+	"github.com/pkg/errors"
+)
+
+func init() {
+	registry.Register("eoswrapper", New)
+}
+
+type wrapper struct {
+	storage.FS
+	mountIDTemplate *template.Template
+}
+
+func parseConfig(m map[string]interface{}) (*eosfs.Config, string, error) {
+	c := &eosfs.Config{}
+	if err := mapstructure.Decode(m, c); err != nil {
+		err = errors.Wrap(err, "error decoding conf")
+		return nil, "", err
+	}
+
+	// default to version invariance if not configured
+	if _, ok := m["version_invariant"]; !ok {
+		c.VersionInvariant = true
+	}
+
+	t, ok := m["mount_id_template"].(string)
+	if !ok || t == "" {
+		t = "eoshome-{{ trimAll \"/\" .Path | substr 0 1 }}"
+	}
+
+	return c, t, nil
+}
+
+// New returns an implementation of the storage.FS interface that forms a wrapper
+// around separate connections to EOS.
+func New(m map[string]interface{}) (storage.FS, error) {
+	c, t, err := parseConfig(m)
+	if err != nil {
+		return nil, err
+	}
+
+	eos, err := eosfs.NewEOSFS(c)
+	if err != nil {
+		return nil, err
+	}
+
+	mountIDTemplate, err := template.New("mountID").Funcs(sprig.TxtFuncMap()).Parse(t)
+	if err != nil {
+		return nil, err
+	}
+
+	return &wrapper{FS: eos, mountIDTemplate: mountIDTemplate}, nil
+}
+
+// We need to override the two methods, GetMD and ListFolder to fill the
+// StorageId in the ResourceInfo objects.
+
+func (w *wrapper) GetMD(ctx context.Context, ref *provider.Reference, mdKeys []string) (*provider.ResourceInfo, error) {
+	res, err := w.FS.GetMD(ctx, ref, mdKeys)
+	if err != nil {
+		return nil, err
+	}
+
+	// We need to extract the mount ID based on the mapping template.
+	//
+	// Take the first letter of the resource path after the namespace has been removed.
+	// If it's empty, leave it empty to be filled by storageprovider.
+	res.Id.StorageId = w.getMountID(ctx, res)
+	return res, nil
+
+}
+
+func (w *wrapper) ListFolder(ctx context.Context, ref *provider.Reference, mdKeys []string) ([]*provider.ResourceInfo, error) {
+	res, err := w.FS.ListFolder(ctx, ref, mdKeys)
+	if err != nil {
+		return nil, err
+	}
+	for _, r := range res {
+		r.Id.StorageId = w.getMountID(ctx, r)
+	}
+	return res, nil
+}
+
+func (w *wrapper) getMountID(ctx context.Context, r *provider.ResourceInfo) string {
+	if r == nil {
+		return ""
+	}
+	b := bytes.Buffer{}
+	if err := w.mountIDTemplate.Execute(&b, r); err != nil {
+		return ""
+	}
+	return b.String()
+}