From 9d07199f97f0cf70b5c4c8acec91fb91e872b749 Mon Sep 17 00:00:00 2001 From: "v.scharf" Date: Tue, 19 May 2026 11:19:25 +0200 Subject: [PATCH] [full-ci] chore: bump reva to v2.45.0 --- go.mod | 2 +- go.sum | 4 +- .../pkg/storage/fs/posix/idcache/idcache.go | 51 ++++--- .../v2/pkg/storage/fs/posix/lookup/lookup.go | 50 ++++--- .../reva/v2/pkg/storage/fs/posix/posix.go | 4 +- .../pkg/storage/fs/posix/tree/assimilation.go | 29 ++-- .../pkg/storage/fs/posix/tree/cache_test.py | 45 ++++++ .../v2/pkg/storage/fs/posix/tree/fix_test.py | 10 ++ .../fs/posix/tree/generate_cache_test.py | 128 ++++++++++++++++++ .../v2/pkg/storage/fs/posix/tree/getxattr.py | 23 ++++ .../reva/v2/pkg/storage/fs/posix/tree/tree.go | 39 ++++-- .../storage/pkg/decomposedfs/decomposedfs.go | 2 +- .../pkg/storage/pkg/decomposedfs/node/node.go | 2 +- vendor/modules.txt | 2 +- 14 files changed, 323 insertions(+), 68 deletions(-) create mode 100644 vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/tree/cache_test.py create mode 100644 vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/tree/fix_test.py create mode 100644 vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/tree/generate_cache_test.py create mode 100644 vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/tree/getxattr.py diff --git a/go.mod b/go.mod index 7164d5b0d3..9f06c39470 100644 --- a/go.mod +++ b/go.mod @@ -64,7 +64,7 @@ require ( github.com/open-policy-agent/opa v1.15.2 github.com/opencloud-eu/icap-client v0.0.0-20250930132611-28a2afe62d89 github.com/opencloud-eu/libre-graph-api-go v1.0.8-0.20260310090739-853d972b282d - github.com/opencloud-eu/reva/v2 v2.44.1-0.20260513122109-583a11875721 + github.com/opencloud-eu/reva/v2 v2.45.0 github.com/opensearch-project/opensearch-go/v4 v4.6.0 github.com/orcaman/concurrent-map v1.0.0 github.com/pkg/errors v0.9.1 diff --git a/go.sum b/go.sum index 056936c8af..1b3c506a8e 100644 --- a/go.sum +++ b/go.sum @@ -952,8 +952,8 @@ github.com/opencloud-eu/inotifywaitgo v0.0.0-20251111171128-a390bae3c5e9 h1:dIft github.com/opencloud-eu/inotifywaitgo v0.0.0-20251111171128-a390bae3c5e9/go.mod h1:JWyDC6H+5oZRdUJUgKuaye+8Ph5hEs6HVzVoPKzWSGI= github.com/opencloud-eu/libre-graph-api-go v1.0.8-0.20260310090739-853d972b282d h1:JcqGDiyrcaQwVyV861TUyQgO7uEmsjkhfm7aQd84dOw= github.com/opencloud-eu/libre-graph-api-go v1.0.8-0.20260310090739-853d972b282d/go.mod h1:pzatilMEHZFT3qV7C/X3MqOa3NlRQuYhlRhZTL+hN6Q= -github.com/opencloud-eu/reva/v2 v2.44.1-0.20260513122109-583a11875721 h1:PH9Ia0HwdvpfaThYCid2atc5y+uwn4EJxvJU6L9wU6M= -github.com/opencloud-eu/reva/v2 v2.44.1-0.20260513122109-583a11875721/go.mod h1:tUL2X47YxLHrnBDArHrIP73UJliMI0PaY/3tPs31dTM= +github.com/opencloud-eu/reva/v2 v2.45.0 h1:62XctwzrGKjmWng9QI+tACEDQb6R9fG7oHlMPfQkjNs= +github.com/opencloud-eu/reva/v2 v2.45.0/go.mod h1:tUL2X47YxLHrnBDArHrIP73UJliMI0PaY/3tPs31dTM= github.com/opencloud-eu/secure v0.0.0-20260312082735-b6f5cb2244e4 h1:l2oB/RctH+t8r7QBj5p8thfEHCM/jF35aAY3WQ3hADI= github.com/opencloud-eu/secure v0.0.0-20260312082735-b6f5cb2244e4/go.mod h1:BmF5hyM6tXczk3MpQkFf1hpKSRqCyhqcbiQtiAF7+40= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/idcache/idcache.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/idcache/idcache.go index 0228dd5d7d..8c3a415461 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/idcache/idcache.go +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/idcache/idcache.go @@ -24,9 +24,9 @@ import ( "path/filepath" "strings" - "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" "github.com/opencloud-eu/reva/v2/pkg/appctx" + "github.com/opencloud-eu/reva/v2/pkg/errtypes" ) type IDCache struct { @@ -59,17 +59,20 @@ func (c *IDCache) Delete(ctx context.Context, spaceID, nodeID string) error { func (c *IDCache) DeleteByPath(ctx context.Context, path string) error { baseKey := reverseCacheKey(path) - spaceID, nodeID, ok := c.GetByPath(ctx, path) - if !ok { - appctx.GetLogger(ctx).Error().Str("record", path).Msg("could not get spaceID and nodeID from cache") + spaceID, nodeID, err := c.GetByPath(ctx, path) + if err != nil { + if _, ok := err.(errtypes.NotFound); !ok { + return err + } + appctx.GetLogger(ctx).Error().Err(err).Str("record", path).Msg("could not get spaceID and nodeID from cache") } else { err := c.kv.Purge(ctx, baseKey) - if err != nil && err != nats.ErrKeyNotFound { + if err != nil && err != jetstream.ErrKeyNotFound { appctx.GetLogger(ctx).Error().Err(err).Str("record", baseKey).Str("spaceID", spaceID).Str("nodeID", nodeID).Msg("could not purge from cache") } err = c.kv.Purge(ctx, cacheKey(spaceID, nodeID)) - if err != nil && err != nats.ErrKeyNotFound { + if err != nil && err != jetstream.ErrKeyNotFound { appctx.GetLogger(ctx).Error().Err(err).Str("record", cacheKey(spaceID, nodeID)).Str("spaceID", spaceID).Str("nodeID", nodeID).Msg("could not purge from cache") } } @@ -85,19 +88,19 @@ func (c *IDCache) DeleteByPath(ctx context.Context, path string) error { break } key := update.Key() - spaceID, nodeID, ok := c.getByReverseCacheKey(ctx, key) - if !ok { - appctx.GetLogger(ctx).Error().Str("record", key).Msg("could not get spaceID and nodeID from cache") + spaceID, nodeID, err := c.getByReverseCacheKey(ctx, key) + if err != nil { + appctx.GetLogger(ctx).Error().Err(err).Str("record", key).Msg("could not get spaceID and nodeID from cache") continue } - err := c.kv.Purge(ctx, key) - if err != nil && err != nats.ErrKeyNotFound { + err = c.kv.Purge(ctx, key) + if err != nil && err != jetstream.ErrKeyNotFound { appctx.GetLogger(ctx).Error().Err(err).Str("record", key).Str("spaceID", spaceID).Str("nodeID", nodeID).Msg("could not purge from cache") } err = c.kv.Purge(ctx, cacheKey(spaceID, nodeID)) - if err != nil && err != nats.ErrKeyNotFound { + if err != nil && err != jetstream.ErrKeyNotFound { appctx.GetLogger(ctx).Error().Err(err).Str("record", cacheKey(spaceID, nodeID)).Str("spaceID", spaceID).Str("nodeID", nodeID).Msg("could not purge from cache") } } @@ -121,32 +124,38 @@ func (c *IDCache) Set(ctx context.Context, spaceID, nodeID, val string) error { } // Get returns the value for a given key -func (c *IDCache) Get(ctx context.Context, spaceID, nodeID string) (string, bool) { +func (c *IDCache) Get(ctx context.Context, spaceID, nodeID string) (string, error) { record, err := c.kv.Get(ctx, cacheKey(spaceID, nodeID)) if err != nil { - return "", false + if err == jetstream.ErrKeyNotFound { + return "", errtypes.NotFound("record not found in cache") + } + return "", err } - return string(record.Value()), true + return string(record.Value()), nil } -func (c *IDCache) getByReverseCacheKey(ctx context.Context, reverseKey string) (string, string, bool) { +func (c *IDCache) getByReverseCacheKey(ctx context.Context, reverseKey string) (string, string, error) { record, err := c.kv.Get(ctx, reverseKey) if err != nil { - return "", "", false + if err == jetstream.ErrKeyNotFound { + return "", "", errtypes.NotFound("record not found in cache") + } + return "", "", err } decoded, err := base32.StdEncoding.DecodeString(string(record.Value())) if err != nil { - return "", "", false + return "", "", err } parts := strings.SplitN(string(decoded), "!", 2) if len(parts) != 2 { - return "", "", false + return "", "", errtypes.InternalError("invalid cache record") } - return parts[0], parts[1], true + return parts[0], parts[1], nil } // GetByPath returns the key for a given value -func (c *IDCache) GetByPath(ctx context.Context, path string) (string, string, bool) { +func (c *IDCache) GetByPath(ctx context.Context, path string) (string, string, error) { return c.getByReverseCacheKey(ctx, reverseCacheKey(path)) } diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/lookup/lookup.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/lookup/lookup.go index be78a47aa9..3409749597 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/lookup/lookup.go +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/lookup/lookup.go @@ -41,6 +41,7 @@ import ( "github.com/opencloud-eu/reva/v2/pkg/storage/utils/templates" "github.com/pkg/errors" "github.com/rogpeppe/go-internal/lockedfile" + "github.com/rs/zerolog" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/trace" ) @@ -58,8 +59,8 @@ func init() { // IDCache is a cache for node ids type IDCache interface { - Get(ctx context.Context, spaceID, nodeID string) (string, bool) - GetByPath(ctx context.Context, path string) (string, string, bool) + Get(ctx context.Context, spaceID, nodeID string) (string, error) + GetByPath(ctx context.Context, path string) (string, string, error) Set(ctx context.Context, spaceID, nodeID, val string) error @@ -79,10 +80,12 @@ type Lookup struct { metadataBackend metadata.Backend userMapper usermapper.Mapper tm node.TimeManager + log *zerolog.Logger } // New returns a new Lookup instance -func New(b metadata.Backend, um usermapper.Mapper, o *options.Options, tm node.TimeManager, cache, historyCache *idcache.IDCache) (*Lookup, error) { +func New(b metadata.Backend, um usermapper.Mapper, o *options.Options, tm node.TimeManager, cache, historyCache *idcache.IDCache, log *zerolog.Logger) (*Lookup, error) { + spaceRootCache, _ := lru.New[string, string](1000) lu := &Lookup{ @@ -93,6 +96,7 @@ func New(b metadata.Backend, um usermapper.Mapper, o *options.Options, tm node.T spaceRootCache: spaceRootCache, userMapper: um, tm: tm, + log: log, } return lu, nil @@ -107,7 +111,7 @@ func (lu *Lookup) CacheID(ctx context.Context, spaceID, nodeID, val string) erro } // GetCachedID returns the cached path for the given space and node id -func (lu *Lookup) GetCachedID(ctx context.Context, spaceID, nodeID string) (string, bool) { +func (lu *Lookup) GetCachedID(ctx context.Context, spaceID, nodeID string) (string, error) { if spaceID == nodeID { return lu.getSpaceRootPathWithStatus(ctx, spaceID) } @@ -116,18 +120,26 @@ func (lu *Lookup) GetCachedID(ctx context.Context, spaceID, nodeID string) (stri func (lu *Lookup) IDsForPath(ctx context.Context, path string) (string, string, error) { // IDsForPath returns the space and opaque id for the given path - spaceID, nodeID, ok := lu.IDCache.GetByPath(ctx, path) - if !ok { - return "", "", errtypes.NotFound("path not found in cache:" + path) + spaceID, nodeID, err := lu.IDCache.GetByPath(ctx, path) + if err != nil { + if _, ok := err.(errtypes.NotFound); !ok { + lu.log.Error().Err(err).Str("path", path).Msg("error looking up path in cache") + } + // fallback to disk + sID, nID, _, _, mErr := lu.metadataBackend.IdentifyPath(ctx, path) + if mErr == nil && nID != "" { + return sID, nID, nil + } + return "", "", err } return spaceID, nodeID, nil } // NodeFromPath returns the node for the given path func (lu *Lookup) NodeIDFromParentAndName(ctx context.Context, parent *node.Node, name string) (string, error) { - parentPath, ok := lu.GetCachedID(ctx, parent.SpaceID, parent.ID) - if !ok { - return "", errtypes.NotFound(parent.ID) + parentPath, err := lu.GetCachedID(ctx, parent.SpaceID, parent.ID) + if err != nil { + return "", err } childPath := filepath.Join(parentPath, name) @@ -290,15 +302,15 @@ func (lu *Lookup) InternalRoot() string { return lu.Options.Root } -func (lu *Lookup) getSpaceRootPathWithStatus(ctx context.Context, spaceID string) (string, bool) { +func (lu *Lookup) getSpaceRootPathWithStatus(ctx context.Context, spaceID string) (string, error) { if val, ok := lu.spaceRootCache.Get(spaceID); ok { - return val, true + return val, nil } - val, ok := lu.IDCache.Get(ctx, spaceID, spaceID) - if ok { + val, err := lu.IDCache.Get(ctx, spaceID, spaceID) + if err == nil { lu.spaceRootCache.Add(spaceID, val) } - return val, ok + return val, err } func (lu *Lookup) getSpaceRootPath(ctx context.Context, spaceID string) string { @@ -338,9 +350,11 @@ func (lu *Lookup) LockfilePaths(n *node.Node) []string { } paths := []string{filepath.Join(spaceRoot, MetadataDir, Pathify(n.ID, 4, 2)+".lock")} - nodepath := n.InternalPath() - if len(nodepath) > 0 { - paths = append(paths, nodepath+".lock") + if lu.Options.WatchFS { + nodepath := n.InternalPath() + if len(nodepath) > 0 { + paths = append(paths, nodepath+".lock") + } } return paths diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/posix.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/posix.go index 9c60f3a68e..0b2f616377 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/posix.go +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/posix.go @@ -122,7 +122,7 @@ func New(o *options.Options, stream events.Stream, cache, historyCache *idcache. var lu *lookup.Lookup switch o.MetadataBackend { case "xattrs": - lu, err = lookup.New(metadata.NewXattrsBackend(o.FileMetadataCache), um, o, &timemanager.Manager{}, cache, historyCache) + lu, err = lookup.New(metadata.NewXattrsBackend(o.FileMetadataCache), um, o, &timemanager.Manager{}, cache, historyCache, log) if err != nil { return nil, err } @@ -136,7 +136,7 @@ func New(o *options.Options, stream events.Stream, cache, historyCache *idcache. return filepath.Join(spaceRoot, lookup.MetadataDir) }, - o.FileMetadataCache), um, o, &timemanager.Manager{}, cache, historyCache) + o.FileMetadataCache), um, o, &timemanager.Manager{}, cache, historyCache, log) if err != nil { return nil, err } diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/tree/assimilation.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/tree/assimilation.go index c1ad1c07ae..484608b79d 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/tree/assimilation.go +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/tree/assimilation.go @@ -39,6 +39,7 @@ import ( provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/opencloud-eu/reva/v2/pkg/errtypes" "github.com/opencloud-eu/reva/v2/pkg/events" "github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/watcher" "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata" @@ -453,9 +454,9 @@ func (t *Tree) assimilate(item scanItem) error { _ = unlock() }() - previousPath, ok := t.lookup.GetCachedID(context.Background(), spaceID, id) - if previousPath == "" || !ok { - previousPath, ok = t.lookup.IDHistoryCache.Get(context.Background(), spaceID, id) + previousPath, err := t.lookup.GetCachedID(context.Background(), spaceID, id) + if previousPath == "" || err != nil { + previousPath, err = t.lookup.IDHistoryCache.Get(context.Background(), spaceID, id) } // compare metadata mtime with actual mtime. if it matches AND the path hasn't changed (move operation) @@ -465,7 +466,7 @@ func (t *Tree) assimilate(item scanItem) error { } // was it moved or copied/restored with a clashing id? - if ok && len(parentID) > 0 && previousPath != item.Path { + if err == nil && len(parentID) > 0 && previousPath != item.Path { _, err := os.Stat(previousPath) if err == nil { // this id clashes with an existing item -> clear metadata and re-assimilate @@ -942,14 +943,22 @@ func (t *Tree) WarmupIDCache(root string, assimilate, onlyDirty bool) error { if id != "" { // Check if the item on the previous path still exists. In this case it might have been a copy with extended attributes -> set new ID isCopy := false - previousPath, ok := t.lookup.GetCachedID(context.Background(), spaceID, id) - if ok && previousPath != path { - _, err := os.Stat(previousPath) - if err == nil { - // previous path (using the same id) still exists -> this is a copy - isCopy = true + previousPath, err := t.lookup.GetCachedID(context.Background(), spaceID, id) + switch err.(type) { + case errtypes.NotFound: + // previous path not found -> not a copy + case nil: + if previousPath != path { + _, err := os.Stat(previousPath) + if err == nil { + // previous path (using the same id) still exists -> this is a copy + isCopy = true + } } + default: + return errors.Wrap(err, "failed to get previous path from cache") } + if isCopy { // copy detected -> re-assimilate _ = t.assimilate(scanItem{Path: path}) diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/tree/cache_test.py b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/tree/cache_test.py new file mode 100644 index 0000000000..d354bdac91 --- /dev/null +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/tree/cache_test.py @@ -0,0 +1,45 @@ +import re +with open("/home/andre/src/opencloud/reva/pkg/storage/fs/posix/tree/assimilation_test.go", "r") as f: + text = f.read() + +new_content = """ +Describe("WarmupIDCache", func() { +var ( +tree *Tree +tmpDir string +logger zerolog.Logger +) + +BeforeEach(func() { +var err error +tmpDir, err = os.MkdirTemp("", "warmupidcache-*") +Expect(err).ToNot(HaveOccurred()) + +logger = zerolog.Nop() + +tree = &Tree{ +log: &logger, +options: &options.Options{ +Options: decomposedoptions.Options{ +Root: tmpDir, +}, +}, +} +}) + +AfterEach(func() { +os.RemoveAll(tmpDir) +}) + +It("returns nil for an empty directory", func() { +err := tree.WarmupIDCache(tmpDir, false, false) +Expect(err).ToNot(HaveOccurred()) +}) +}) +""" + +text = re.sub(r'Describe\("WarmupIDCache", func\(\).*$', new_content, text, flags=re.DOTALL) +text += "\n})\n" +with open("/home/andre/src/opencloud/reva/pkg/storage/fs/posix/tree/assimilation_test.go", "w") as f: + f.write(text) + diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/tree/fix_test.py b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/tree/fix_test.py new file mode 100644 index 0000000000..8afcb79dc1 --- /dev/null +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/tree/fix_test.py @@ -0,0 +1,10 @@ +import re + +with open("assimilation_test.go", "r") as f: + text = f.read() + +# remove CreateTestStorageSpace which throws error +text = text.replace('_, err = env.CreateTestStorageSpace("personal", nil)\n\t\tExpect(err).ToNot(HaveOccurred())', '') + +with open("assimilation_test.go", "w") as f: + f.write(text) diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/tree/generate_cache_test.py b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/tree/generate_cache_test.py new file mode 100644 index 0000000000..d720250e2e --- /dev/null +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/tree/generate_cache_test.py @@ -0,0 +1,128 @@ +import sys + +content = """ +Describe("WarmupIDCache", func() { +var ( +tree *Tree +tmpDir string +logger zerolog.Logger +ctx context.Context +) + +BeforeEach(func() { +ctx = context.Background() +var err error +tmpDir, err = os.MkdirTemp("", "warmupidcache-*") +Expect(err).ToNot(HaveOccurred()) + +logger = zerolog.Nop() + +o := &options.Options{ +Options: decomposedoptions.Options{ +Root: tmpDir, +}, +} + +// We need a backend and caches +c, _ := idcache.NewMemoryIDCache() +historyCache, _ := idcache.NewMemoryIDCache() +um := &usermapper.NullMapper{} + +backend := metadata.NewMessagePackBackend(o.FileMetadataCache) +lu, err := lookup.New(backend, um, o, &timemanager.Manager{}, c, historyCache) +Expect(err).ToNot(HaveOccurred()) + +tree = &Tree{ +log: &logger, +options: o, +lookup: lu, +} +}) + +AfterEach(func() { +os.RemoveAll(tmpDir) +}) + +It("returns nil for an empty directory", func() { +err := tree.WarmupIDCache(tmpDir, false, false) +Expect(err).ToNot(HaveOccurred()) +}) + + It("picks up new files and directories", func() { + subDir := filepath.Join(tmpDir, "sub") + err := os.Mkdir(subDir, 0755) + Expect(err).ToNot(HaveOccurred()) + + filePath := filepath.Join(subDir, "test.txt") + err = os.WriteFile(filePath, []byte("hello world"), 0644) + Expect(err).ToNot(HaveOccurred()) + + // Should not crash, tests basic traverse + err = tree.WarmupIDCache(tmpDir, false, false) + Expect(err).ToNot(HaveOccurred()) + }) + + It("verifies tree sizes and recursion", func() { + // Setup a small directory structure + subDir := filepath.Join(tmpDir, "sub2") + err := os.Mkdir(subDir, 0755) + Expect(err).ToNot(HaveOccurred()) + + nestedDir := filepath.Join(subDir, "nested") + err = os.Mkdir(nestedDir, 0755) + Expect(err).ToNot(HaveOccurred()) + + filePath := filepath.Join(nestedDir, "test.txt") + err = os.WriteFile(filePath, []byte("hello world"), 0644) // 11 bytes + Expect(err).ToNot(HaveOccurred()) + + // Run assimilation + err = tree.WarmupIDCache(tmpDir, true, false) + Expect(err).ToNot(HaveOccurred()) + + // If assimilation runs, the files will have xattrs and tree sizes evaluated + // wait, but without mocked idResolver for the Tree, it might fail? Let's check! + }) +}) +""" + +with open("assimilation_test.go", "r") as f: + text = f.read() + +import re + +# Add imports +imports = """ +import ( +"context" +"os" +"path/filepath" + +"github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/lookup" +"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata" +"github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/idcache" +"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/usermapper" +"github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/timemanager" + +. "github.com/onsi/ginkgo/v2" +. "github.com/onsi/gomega" +"github.com/rs/zerolog" + +"github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/options" +decomposedoptions "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/options" +) +""" + +# Replace imports +start_import = text.find('import (') +end_import = text.find(')', start_import) + 1 +text = text[:start_import] + imports.strip() + text[end_import:] + +start_idx = text.find('Describe("WarmupIDCache", func() {') +end_idx = text.find('})\n\n})', start_idx) + 2 + +if start_idx != -1 and end_idx != -1: + new_text = text[:start_idx] + content.strip() + text[end_idx:] + with open("assimilation_test.go", "w") as f: + f.write(new_text) + diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/tree/getxattr.py b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/tree/getxattr.py new file mode 100644 index 0000000000..612cfa79f3 --- /dev/null +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/tree/getxattr.py @@ -0,0 +1,23 @@ +with open("assimilation_test.go", "r") as f: + text = f.read() + +import re + +# We will inject the xattr check. +insert = """// verify that tree sizes are updated +// Since we used assimilate=true, the treesize xattr on sub2 and nested should be 11. + +b, err := env.Lookup.MetadataBackend().Get(env.Ctx, subDir, "user.oc.treesize") +Expect(err).ToNot(HaveOccurred()) +Expect(string(b)).To(Equal("11")) + +b, err = env.Lookup.MetadataBackend().Get(env.Ctx, nestedDir, "user.oc.treesize") +Expect(err).ToNot(HaveOccurred()) +Expect(string(b)).To(Equal("11"))""" + +# inject right after env.Tree.WarmupIDCache call +text = text.replace("err = env.Tree.WarmupIDCache(tmpDir+\"/users/admin\", true, false)\n\t\tExpect(err).ToNot(HaveOccurred())", "err = env.Tree.WarmupIDCache(tmpDir+\"/users/admin\", true, false)\n\t\tExpect(err).ToNot(HaveOccurred())\n\n" + insert) + +with open("assimilation_test.go", "w") as f: + f.write(text) + diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/tree/tree.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/tree/tree.go index ff510bbc26..045362b7b3 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/tree/tree.go +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/tree/tree.go @@ -73,6 +73,10 @@ type Watcher interface { Watch(path string) } +type IDResolver interface { + IDsForPath(ctx context.Context, path string) (spaceID string, nodeID string, err error) +} + type scanItem struct { Path string Recurse bool @@ -80,12 +84,15 @@ type scanItem struct { // Tree manages a hierarchical tree type Tree struct { - lookup *lookup.Lookup blobstore node.Blobstore trashbin *trashbin.Trashbin propagator propagator.Propagator permissions permissions.Permissions + lookup *lookup.Lookup + idResolver IDResolver // points at the lookup but can be overridden for testing + assimilateFunc func(item scanItem) error // function to call to assimilate a node, can be overridden for testing + options *options.Options personalSpacesRoot string projectSpacesRoot string @@ -108,9 +115,10 @@ func New(lu node.PathLookup, bs node.Blobstore, um usermapper.Mapper, trashbin * scanQueue := make(chan scanItem) t := &Tree{ - lookup: lu.(*lookup.Lookup), - blobstore: bs, - userMapper: um, + lookup: lu.(*lookup.Lookup), + blobstore: bs, + userMapper: um, + // idResolver and assimilateFunc are wired below once t exists. trashbin: trashbin, permissions: permissions, options: o, @@ -125,6 +133,8 @@ func New(lu node.PathLookup, bs node.Blobstore, um usermapper.Mapper, trashbin * personalSpacesRoot: filepath.Clean(filepath.Join(o.Root, templates.Base(o.PersonalSpacePathTemplate))), projectSpacesRoot: filepath.Clean(filepath.Join(o.Root, templates.Base(o.GeneralSpacePathTemplate))), } + t.idResolver = t.lookup + t.assimilateFunc = t.assimilate if err := t.checkStorage(); err != nil { return nil, errors.Wrap(err, "tree: unfit storage '"+o.Root+"'") } @@ -518,16 +528,23 @@ func (t *Tree) ListFolder(ctx context.Context, n *node.Node) ([]*node.Node, erro for name := range work { path := filepath.Join(dir, name) - _, nodeID, err := t.lookup.IDsForPath(ctx, path) + _, nodeID, err := t.idResolver.IDsForPath(ctx, path) if err != nil { - // we don't know about this node yet for some reason, assimilate it on the fly + if _, ok := err.(errtypes.IsNotFound); !ok { + // a NotFound error just means we don't know about this + // node yet. Any other error (e.g. an unavailable id + // cache backend) is a real failure that must not be + // silently turned into an assimilation attempt. + return errors.Wrap(err, "tree: error resolving ids for "+path) + } + // we don't know about this node yet, assimilate it on the fly t.log.Info().Err(err).Str("path", path).Msg("encountered unknown entity while listing the directory. Assimilate.") - err = t.assimilate(scanItem{Path: path}) + err = t.assimilateFunc(scanItem{Path: path}) if err != nil { t.log.Error().Err(err).Str("path", path).Msg("failed to assimilate node") continue } - _, nodeID, err = t.lookup.IDsForPath(ctx, path) + _, nodeID, err = t.idResolver.IDsForPath(ctx, path) if err != nil || nodeID == "" { t.log.Error().Err(err).Str("path", path).Msg("still could not resolve node after assimilation") continue @@ -711,9 +728,9 @@ func (t *Tree) createDirNode(ctx context.Context, n *node.Node) (err error) { idcache := t.lookup.IDCache // create a directory node - parentPath, ok := idcache.Get(ctx, n.SpaceID, n.ParentID) - if !ok { - return errtypes.NotFound(n.ParentID) + parentPath, err := idcache.Get(ctx, n.SpaceID, n.ParentID) + if err != nil { + return err } path := filepath.Join(parentPath, n.Name) diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/decomposedfs.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/decomposedfs.go index 82260a6093..afcb06eeab 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/decomposedfs.go +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/decomposedfs.go @@ -1379,7 +1379,7 @@ func (fs *Decomposedfs) RestoreRecycleItem(ctx context.Context, space *provider. // Warmup posix IDCache if restored path is a directory if cachingTree, ok := fs.tp.(IDCachingTree); ok { - _ = cachingTree.WarmupIDCache(restoredNode.InternalPath(), false, false) + _ = cachingTree.WarmupIDCache(restoredNode.InternalPath(), true, false) } } else { sizeDiff = restoredNode.Blobsize diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/node/node.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/node/node.go index 810cc2362e..b5192ca4c7 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/node/node.go +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/node/node.go @@ -173,7 +173,7 @@ type PathLookup interface { type IDCacher interface { CacheID(ctx context.Context, spaceID, nodeID, val string) error - GetCachedID(ctx context.Context, spaceID, nodeID string) (string, bool) + GetCachedID(ctx context.Context, spaceID, nodeID string) (string, error) } type BaseNode struct { diff --git a/vendor/modules.txt b/vendor/modules.txt index 6613efc856..125f4f8070 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1362,7 +1362,7 @@ github.com/opencloud-eu/icap-client # github.com/opencloud-eu/libre-graph-api-go v1.0.8-0.20260310090739-853d972b282d ## explicit; go 1.18 github.com/opencloud-eu/libre-graph-api-go -# github.com/opencloud-eu/reva/v2 v2.44.1-0.20260513122109-583a11875721 +# github.com/opencloud-eu/reva/v2 v2.45.0 ## explicit; go 1.25.0 github.com/opencloud-eu/reva/v2/cmd/revad/internal/grace github.com/opencloud-eu/reva/v2/cmd/revad/runtime