From 518ae4aa9050dd163246ed834058e3697eb23509 Mon Sep 17 00:00:00 2001 From: luke-lombardi <33990301+luke-lombardi@users.noreply.github.com> Date: Thu, 12 Sep 2024 20:10:56 -0400 Subject: [PATCH 01/18] debug print --- pkg/blobfs_node.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/blobfs_node.go b/pkg/blobfs_node.go index 95ce59e..1582486 100644 --- a/pkg/blobfs_node.go +++ b/pkg/blobfs_node.go @@ -3,6 +3,7 @@ package blobcache import ( "context" "fmt" + "log" "path" "syscall" @@ -84,9 +85,12 @@ func (n *FSNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) (* // Construct the full of this file path from root fullPath := path.Join(n.bfsNode.Path, name) + log.Println("LOOKING UP USING FULL PATH: ", fullPath) + id := GenerateFsID(fullPath) metadata, err := n.filesystem.Metadata.GetFsNode(ctx, id) if err != nil { + // n.filesystem.Client.StoreContentFromSource(fullPath, ) return nil, syscall.ENOENT } From 7cfe500d89445eb5f317479c2a6d7038bf7f0740 Mon Sep 17 00:00:00 2001 From: luke-lombardi <33990301+luke-lombardi@users.noreply.github.com> Date: Thu, 12 Sep 2024 20:40:59 -0400 Subject: [PATCH 02/18] try ioctl --- pkg/blobfs_node.go | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/pkg/blobfs_node.go b/pkg/blobfs_node.go index 1582486..8dd6af2 100644 --- a/pkg/blobfs_node.go +++ b/pkg/blobfs_node.go @@ -37,6 +37,30 @@ func (n *FSNode) OnAdd(ctx context.Context) { n.log("OnAdd called") } +func (n *FSNode) Ioctl(ctx context.Context, fd uint32, cmd uint32, arg []byte) (uintptr, syscall.Errno) { + log.Println("called ioctl") + + // Define a custom command for direct lookup (use a unique command number) + const DIRECT_LOOKUP_CMD = 0xABCDEFFF + + if cmd == DIRECT_LOOKUP_CMD { + // Assume 'arg' contains the path in a byte slice + fullPath := string(arg) + + log.Println("hey there: ", fullPath) + // // Perform the direct lookup + // inode, err := n.filesystem.DirectLookup(ctx, fullPath) + // if err != fs.OK { + // return 0, err + // } + + // Optionally return the inode number or other information + // return uintptr(inode.StableAttr().Ino), fs.OK + } + + return 0, syscall.ENOSYS +} + func (n *FSNode) Getattr(ctx context.Context, fh fs.FileHandle, out *fuse.AttrOut) syscall.Errno { n.log("Getattr called") From f8233f33ce0a19c94ead63e99eb1781c5fb087d6 Mon Sep 17 00:00:00 2001 From: luke-lombardi <33990301+luke-lombardi@users.noreply.github.com> Date: Thu, 12 Sep 2024 20:47:40 -0400 Subject: [PATCH 03/18] wip --- pkg/blobfs_node.go | 25 +------------------------ 1 file changed, 1 insertion(+), 24 deletions(-) diff --git a/pkg/blobfs_node.go b/pkg/blobfs_node.go index 8dd6af2..2cfd808 100644 --- a/pkg/blobfs_node.go +++ b/pkg/blobfs_node.go @@ -37,30 +37,6 @@ func (n *FSNode) OnAdd(ctx context.Context) { n.log("OnAdd called") } -func (n *FSNode) Ioctl(ctx context.Context, fd uint32, cmd uint32, arg []byte) (uintptr, syscall.Errno) { - log.Println("called ioctl") - - // Define a custom command for direct lookup (use a unique command number) - const DIRECT_LOOKUP_CMD = 0xABCDEFFF - - if cmd == DIRECT_LOOKUP_CMD { - // Assume 'arg' contains the path in a byte slice - fullPath := string(arg) - - log.Println("hey there: ", fullPath) - // // Perform the direct lookup - // inode, err := n.filesystem.DirectLookup(ctx, fullPath) - // if err != fs.OK { - // return 0, err - // } - - // Optionally return the inode number or other information - // return uintptr(inode.StableAttr().Ino), fs.OK - } - - return 0, syscall.ENOSYS -} - func (n *FSNode) Getattr(ctx context.Context, fh fs.FileHandle, out *fuse.AttrOut) syscall.Errno { n.log("Getattr called") @@ -204,6 +180,7 @@ func (n *FSNode) Readdir(ctx context.Context) (fs.DirStream, syscall.Errno) { func (n *FSNode) Create(ctx context.Context, name string, flags uint32, mode uint32, out *fuse.EntryOut) (inode *fs.Inode, fh fs.FileHandle, fuseFlags uint32, errno syscall.Errno) { n.log("Create called with name: %s, flags: %v, mode: %v", name, flags, mode) + log.Println("TOUCH ME!!!!") return nil, nil, 0, syscall.EROFS } From 3463b99de05d4f9588b306c5a96df98a444da673 Mon Sep 17 00:00:00 2001 From: luke-lombardi <33990301+luke-lombardi@users.noreply.github.com> Date: Thu, 12 Sep 2024 22:32:10 -0400 Subject: [PATCH 04/18] clean up --- pkg/blobfs_node.go | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/pkg/blobfs_node.go b/pkg/blobfs_node.go index 2cfd808..e952822 100644 --- a/pkg/blobfs_node.go +++ b/pkg/blobfs_node.go @@ -5,6 +5,7 @@ import ( "fmt" "log" "path" + "strings" "syscall" "github.com/hanwen/go-fuse/v2/fs" @@ -86,11 +87,22 @@ func (n *FSNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) (* fullPath := path.Join(n.bfsNode.Path, name) log.Println("LOOKING UP USING FULL PATH: ", fullPath) + if strings.Contains(fullPath, "%") { + sourcePath := strings.ReplaceAll(fullPath, "%", "/") + + log.Println("storing content from source with path: ", sourcePath) + hash, err := n.filesystem.Client.StoreContentFromSource(sourcePath, 0) + if err != nil { + return nil, syscall.ENOENT + } + + log.Println("hash: ", hash) + return nil, fs.OK + } id := GenerateFsID(fullPath) metadata, err := n.filesystem.Metadata.GetFsNode(ctx, id) if err != nil { - // n.filesystem.Client.StoreContentFromSource(fullPath, ) return nil, syscall.ENOENT } @@ -122,6 +134,11 @@ func (n *FSNode) Opendir(ctx context.Context) syscall.Errno { func (n *FSNode) Open(ctx context.Context, flags uint32) (fh fs.FileHandle, fuseFlags uint32, errno syscall.Errno) { n.log("Open called with flags: %v", flags) + + if !n.filesystem.Client.HostsAvailable() { + return nil, 0, syscall.EIO + } + return nil, 0, fs.OK } @@ -180,7 +197,6 @@ func (n *FSNode) Readdir(ctx context.Context) (fs.DirStream, syscall.Errno) { func (n *FSNode) Create(ctx context.Context, name string, flags uint32, mode uint32, out *fuse.EntryOut) (inode *fs.Inode, fh fs.FileHandle, fuseFlags uint32, errno syscall.Errno) { n.log("Create called with name: %s, flags: %v, mode: %v", name, flags, mode) - log.Println("TOUCH ME!!!!") return nil, nil, 0, syscall.EROFS } From 23f7606d2008c039afc3fd961e7f0fe3659ce273 Mon Sep 17 00:00:00 2001 From: luke-lombardi <33990301+luke-lombardi@users.noreply.github.com> Date: Fri, 13 Sep 2024 11:27:51 -0400 Subject: [PATCH 05/18] wip --- pkg/blobfs_node.go | 35 +++++++++++++++++++++++++++++++---- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/pkg/blobfs_node.go b/pkg/blobfs_node.go index e952822..48f0b96 100644 --- a/pkg/blobfs_node.go +++ b/pkg/blobfs_node.go @@ -88,16 +88,43 @@ func (n *FSNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) (* log.Println("LOOKING UP USING FULL PATH: ", fullPath) if strings.Contains(fullPath, "%") { + if !n.filesystem.Client.HostsAvailable() { + return nil, syscall.ENOENT + } + sourcePath := strings.ReplaceAll(fullPath, "%", "/") - log.Println("storing content from source with path: ", sourcePath) - hash, err := n.filesystem.Client.StoreContentFromSource(sourcePath, 0) + log.Println("Storing content from source with path: ", sourcePath) + + _, err := n.filesystem.Client.StoreContentFromSource(sourcePath, 0) + if err != nil { + return nil, syscall.ENOENT + } + + metadata, err := n.filesystem.Metadata.GetFsNode(ctx, GenerateFsID(fullPath)) if err != nil { return nil, syscall.ENOENT } - log.Println("hash: ", hash) - return nil, fs.OK + // Fill out the child node's attributes + attr := metaToAttr(metadata) + out.Attr = attr + + // Create a new Inode on lookup + node := n.NewInode(ctx, + &FSNode{filesystem: n.filesystem, bfsNode: &BlobFsNode{ + Path: metadata.Path, + ID: metadata.ID, + PID: metadata.PID, + Name: metadata.Name, + Hash: metadata.Hash, + Attr: attr, + Target: "", + }, attr: attr}, + fs.StableAttr{Mode: metadata.Mode, Ino: metadata.Ino}, + ) + + return node, fs.OK } id := GenerateFsID(fullPath) From ec80f2b412b69a7d5508a63cc2022689a758f776 Mon Sep 17 00:00:00 2001 From: luke-lombardi <33990301+luke-lombardi@users.noreply.github.com> Date: Fri, 13 Sep 2024 14:20:18 -0400 Subject: [PATCH 06/18] wip --- pkg/metadata.go | 34 ++++++++++++++++++++++++++++------ 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/pkg/metadata.go b/pkg/metadata.go index e07f8ca..a034238 100644 --- a/pkg/metadata.go +++ b/pkg/metadata.go @@ -3,6 +3,7 @@ package blobcache import ( "context" "fmt" + "os" "path/filepath" "strings" "time" @@ -151,10 +152,10 @@ func (m *BlobCacheMetadata) StoreContentInBlobFs(ctx context.Context, path strin return err } + // Initialize default metadata now := time.Now() nowSec := uint64(now.Unix()) nowNsec := uint32(now.Nanosecond()) - metadata := &BlobFsMetadata{ PID: previousParentId, ID: currentNodeId, @@ -170,18 +171,39 @@ func (m *BlobCacheMetadata) StoreContentInBlobFs(ctx context.Context, path strin Ctimensec: nowNsec, } - // Since this is the last file, store as a file, not a dir - if path == currentPath { - metadata.Mode = fuse.S_IFREG | 0755 - metadata.Hash = hash - metadata.Size = size + // If currentPath matches the input path, use the actual file info + if currentPath == path { + fileInfo, err := os.Stat(currentPath) + if err != nil { + return err + } + + // Update metadata fields with actual file info values + modTime := fileInfo.ModTime() + metadata.Mode = uint32(fileInfo.Mode()) + metadata.Atime = uint64(modTime.Unix()) + metadata.Mtime = uint64(modTime.Unix()) + metadata.Ctime = uint64(modTime.Unix()) + metadata.Atimensec = uint32(modTime.Nanosecond()) + metadata.Mtimensec = uint32(modTime.Nanosecond()) + metadata.Ctimensec = uint32(modTime.Nanosecond()) + + if fileInfo.IsDir() { + metadata.Hash = GenerateFsID(currentPath) // Hash for directory + metadata.Size = 0 + } else { + metadata.Hash = hash + metadata.Size = size + } } + // Set metadata err = m.SetFsNode(ctx, currentNodeId, metadata) if err != nil { return err } + // Add the current node as a child of the previous node err = m.AddFsNodeChild(ctx, previousParentId, currentNodeId) if err != nil { return err From 7bca962116d8257e3f842b935f0e25870190a306 Mon Sep 17 00:00:00 2001 From: luke-lombardi <33990301+luke-lombardi@users.noreply.github.com> Date: Fri, 13 Sep 2024 15:17:38 -0400 Subject: [PATCH 07/18] wip --- go.mod | 1 + go.sum | 2 ++ pkg/metadata.go | 16 ++++++++++++---- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 91ce2be..bf272bf 100644 --- a/go.mod +++ b/go.mod @@ -47,6 +47,7 @@ require ( github.com/dblohm7/wingoes v0.0.0-20240119213807-a09d6be7affa // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/digitalocean/go-smbios v0.0.0-20180907143718-390a4f403a8e // indirect + github.com/djherbis/atime v1.1.0 // indirect github.com/dustin/go-humanize v1.0.0 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/fxamacker/cbor/v2 v2.6.0 // indirect diff --git a/go.sum b/go.sum index dcdafd6..6a18fbc 100644 --- a/go.sum +++ b/go.sum @@ -73,6 +73,8 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/r github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/digitalocean/go-smbios v0.0.0-20180907143718-390a4f403a8e h1:vUmf0yezR0y7jJ5pceLHthLaYf4bA5T14B6q39S4q2Q= github.com/digitalocean/go-smbios v0.0.0-20180907143718-390a4f403a8e/go.mod h1:YTIHhz/QFSYnu/EhlF2SpU2Uk+32abacUYA5ZPljz1A= +github.com/djherbis/atime v1.1.0 h1:rgwVbP/5by8BvvjBNrbh64Qz33idKT3pSnMSJsxhi0g= +github.com/djherbis/atime v1.1.0/go.mod h1:28OF6Y8s3NQWwacXc5eZTsEsiMzp7LF8MbXE+XJPdBE= github.com/djherbis/times v1.6.0 h1:w2ctJ92J8fBvWPxugmXIv7Nz7Q3iDMKNx9v5ocVH20c= github.com/djherbis/times v1.6.0/go.mod h1:gOHeRAz2h+VJNZ5Gmc/o7iD9k4wW7NMVqieYCY99oc0= github.com/dsnet/try v0.0.3 h1:ptR59SsrcFUYbT/FhAbKTV6iLkeD6O18qfIWRml2fqI= diff --git a/pkg/metadata.go b/pkg/metadata.go index a034238..a3d3b1a 100644 --- a/pkg/metadata.go +++ b/pkg/metadata.go @@ -3,12 +3,14 @@ package blobcache import ( "context" "fmt" + "log" "os" "path/filepath" "strings" "time" mapset "github.com/deckarep/golang-set/v2" + "github.com/djherbis/atime" "github.com/hanwen/go-fuse/v2/fuse" redis "github.com/redis/go-redis/v9" ) @@ -180,16 +182,20 @@ func (m *BlobCacheMetadata) StoreContentInBlobFs(ctx context.Context, path strin // Update metadata fields with actual file info values modTime := fileInfo.ModTime() + accessTime := atime.Get(fileInfo) metadata.Mode = uint32(fileInfo.Mode()) - metadata.Atime = uint64(modTime.Unix()) + metadata.Atime = uint64(accessTime.Unix()) + metadata.Atimensec = uint32(accessTime.Nanosecond()) metadata.Mtime = uint64(modTime.Unix()) - metadata.Ctime = uint64(modTime.Unix()) - metadata.Atimensec = uint32(modTime.Nanosecond()) metadata.Mtimensec = uint32(modTime.Nanosecond()) + + // Since we cannot get Ctime in a platform-independent way, set it to ModTime + metadata.Ctime = uint64(modTime.Unix()) metadata.Ctimensec = uint32(modTime.Nanosecond()) + metadata.Size = uint64(fileInfo.Size()) if fileInfo.IsDir() { - metadata.Hash = GenerateFsID(currentPath) // Hash for directory + metadata.Hash = GenerateFsID(currentPath) metadata.Size = 0 } else { metadata.Hash = hash @@ -238,6 +244,8 @@ func (m *BlobCacheMetadata) GetFsNode(ctx context.Context, id string) (*BlobFsMe func (m *BlobCacheMetadata) SetFsNode(ctx context.Context, id string, metadata *BlobFsMetadata) error { key := MetadataKeys.MetadataFsNode(id) + log.Printf("setting fs mode: %+v\n", metadata) + err := m.rdb.HSet(ctx, key, ToSlice(metadata)).Err() if err != nil { return fmt.Errorf("failed to set blobfs node metadata <%v>: %w", key, err) From 4a04168eeaca7e507aeca38add0ea87bc5c1e0a4 Mon Sep 17 00:00:00 2001 From: luke-lombardi <33990301+luke-lombardi@users.noreply.github.com> Date: Fri, 13 Sep 2024 16:14:24 -0400 Subject: [PATCH 08/18] wip --- pkg/blobfs.go | 2 +- pkg/blobfs_node.go | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/blobfs.go b/pkg/blobfs.go index 215221f..6d4d218 100644 --- a/pkg/blobfs.go +++ b/pkg/blobfs.go @@ -117,7 +117,7 @@ func Mount(ctx context.Context, opts BlobFsSystemOpts) (func() error, <-chan err server, err := fuse.NewServer(fs.NewNodeFS(root, fsOptions), mountPoint, &fuse.MountOptions{ MaxBackground: maxBackgroundTasks, DisableXAttrs: true, - EnableSymlinkCaching: true, + EnableSymlinkCaching: false, SyncRead: false, RememberInodes: true, MaxReadAhead: maxReadAheadKB * 1024, diff --git a/pkg/blobfs_node.go b/pkg/blobfs_node.go index 48f0b96..1cc4a9c 100644 --- a/pkg/blobfs_node.go +++ b/pkg/blobfs_node.go @@ -151,6 +151,8 @@ func (n *FSNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) (* fs.StableAttr{Mode: metadata.Mode, Ino: metadata.Ino}, ) + log.Printf("node metadata attr: %+v", attr) + return node, fs.OK } From 73a200dfa2df1703c5bbe4bbef90a557f8ed3744 Mon Sep 17 00:00:00 2001 From: luke-lombardi <33990301+luke-lombardi@users.noreply.github.com> Date: Fri, 13 Sep 2024 16:23:28 -0400 Subject: [PATCH 09/18] print attr --- pkg/blobfs_node.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/blobfs_node.go b/pkg/blobfs_node.go index 1cc4a9c..4b5c150 100644 --- a/pkg/blobfs_node.go +++ b/pkg/blobfs_node.go @@ -135,6 +135,7 @@ func (n *FSNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) (* // Fill out the child node's attributes attr := metaToAttr(metadata) + log.Printf("attr: %+v\n", attr) out.Attr = attr // Create a new Inode on lookup From 981564f27c83a7446986111ca933b5877e97e7c4 Mon Sep 17 00:00:00 2001 From: luke-lombardi <33990301+luke-lombardi@users.noreply.github.com> Date: Fri, 13 Sep 2024 16:28:03 -0400 Subject: [PATCH 10/18] dont remember inodes --- pkg/blobfs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/blobfs.go b/pkg/blobfs.go index 6d4d218..b7661ec 100644 --- a/pkg/blobfs.go +++ b/pkg/blobfs.go @@ -119,7 +119,7 @@ func Mount(ctx context.Context, opts BlobFsSystemOpts) (func() error, <-chan err DisableXAttrs: true, EnableSymlinkCaching: false, SyncRead: false, - RememberInodes: true, + RememberInodes: false, MaxReadAhead: maxReadAheadKB * 1024, }) if err != nil { From 3ed6877de375ba289bab321f5c486124c772631a Mon Sep 17 00:00:00 2001 From: luke-lombardi <33990301+luke-lombardi@users.noreply.github.com> Date: Sat, 14 Sep 2024 10:21:16 -0400 Subject: [PATCH 11/18] cleanup --- pkg/blobfs_node.go | 82 ++++++++++++++++++++-------------------------- pkg/client.go | 1 + 2 files changed, 37 insertions(+), 46 deletions(-) diff --git a/pkg/blobfs_node.go b/pkg/blobfs_node.go index 4b5c150..0cfa9fc 100644 --- a/pkg/blobfs_node.go +++ b/pkg/blobfs_node.go @@ -80,80 +80,70 @@ func metaToAttr(metadata *BlobFsMetadata) fuse.Attr { } } +func (n *FSNode) inodeFromFsId(ctx context.Context, fsId string) (*fs.Inode, *fuse.Attr, error) { + metadata, err := n.filesystem.Metadata.GetFsNode(ctx, fsId) + if err != nil { + return nil, nil, syscall.ENOENT + } + + // Fill out the child node's attributes + attr := metaToAttr(metadata) + + // Create a new Inode on lookup + node := n.NewInode(ctx, + &FSNode{filesystem: n.filesystem, bfsNode: &BlobFsNode{ + Path: metadata.Path, + ID: metadata.ID, + PID: metadata.PID, + Name: metadata.Name, + Hash: metadata.Hash, + Attr: attr, + Target: "", + }, attr: attr}, + fs.StableAttr{Mode: metadata.Mode, Ino: metadata.Ino}, + ) + + return node, &attr, nil +} + func (n *FSNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) (*fs.Inode, syscall.Errno) { n.log("Lookup called with name: %s", name) // Construct the full of this file path from root fullPath := path.Join(n.bfsNode.Path, name) - log.Println("LOOKING UP USING FULL PATH: ", fullPath) + n.log("Lookup called with full path: %s", fullPath) + + // Force caching of a specific full path if the path contains a special illegal character '%' + // This is a hack to trigger caching from external callers without going through the GRPC service directly if strings.Contains(fullPath, "%") { + sourcePath := strings.ReplaceAll(fullPath, "%", "/") + if !n.filesystem.Client.HostsAvailable() { return nil, syscall.ENOENT } - sourcePath := strings.ReplaceAll(fullPath, "%", "/") - log.Println("Storing content from source with path: ", sourcePath) - _, err := n.filesystem.Client.StoreContentFromSource(sourcePath, 0) if err != nil { return nil, syscall.ENOENT } - metadata, err := n.filesystem.Metadata.GetFsNode(ctx, GenerateFsID(fullPath)) + node, attr, err := n.inodeFromFsId(ctx, GenerateFsID(sourcePath)) if err != nil { return nil, syscall.ENOENT } - // Fill out the child node's attributes - attr := metaToAttr(metadata) - out.Attr = attr - - // Create a new Inode on lookup - node := n.NewInode(ctx, - &FSNode{filesystem: n.filesystem, bfsNode: &BlobFsNode{ - Path: metadata.Path, - ID: metadata.ID, - PID: metadata.PID, - Name: metadata.Name, - Hash: metadata.Hash, - Attr: attr, - Target: "", - }, attr: attr}, - fs.StableAttr{Mode: metadata.Mode, Ino: metadata.Ino}, - ) - + out.Attr = *attr return node, fs.OK } - id := GenerateFsID(fullPath) - metadata, err := n.filesystem.Metadata.GetFsNode(ctx, id) + node, attr, err := n.inodeFromFsId(ctx, GenerateFsID(fullPath)) if err != nil { return nil, syscall.ENOENT } - // Fill out the child node's attributes - attr := metaToAttr(metadata) - log.Printf("attr: %+v\n", attr) - out.Attr = attr - - // Create a new Inode on lookup - node := n.NewInode(ctx, - &FSNode{filesystem: n.filesystem, bfsNode: &BlobFsNode{ - Path: metadata.Path, - ID: metadata.ID, - PID: metadata.PID, - Name: metadata.Name, - Hash: metadata.Hash, - Attr: attr, - Target: "", - }, attr: attr}, - fs.StableAttr{Mode: metadata.Mode, Ino: metadata.Ino}, - ) - - log.Printf("node metadata attr: %+v", attr) - + out.Attr = *attr return node, fs.OK } diff --git a/pkg/client.go b/pkg/client.go index 4a08529..519719e 100644 --- a/pkg/client.go +++ b/pkg/client.go @@ -108,6 +108,7 @@ func NewBlobCacheClient(ctx context.Context, cfg BlobCacheConfig) (*BlobCacheCli Config: cfg, Metadata: metadata, Client: bc, + Verbose: true, }) if err != nil { return nil, err From 835253e07325068fa87d28add8584ae272c54c51 Mon Sep 17 00:00:00 2001 From: luke-lombardi <33990301+luke-lombardi@users.noreply.github.com> Date: Sat, 14 Sep 2024 10:42:52 -0400 Subject: [PATCH 12/18] store more attrs --- pkg/blobfs_node.go | 6 ++++-- pkg/metadata.go | 3 --- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/pkg/blobfs_node.go b/pkg/blobfs_node.go index 0cfa9fc..358f59c 100644 --- a/pkg/blobfs_node.go +++ b/pkg/blobfs_node.go @@ -3,7 +3,6 @@ package blobcache import ( "context" "fmt" - "log" "path" "strings" "syscall" @@ -53,6 +52,9 @@ func (n *FSNode) Getattr(ctx context.Context, fh fs.FileHandle, out *fuse.AttrOu out.Mode = node.Attr.Mode out.Nlink = node.Attr.Nlink out.Owner = node.Attr.Owner + out.Atimensec = node.Attr.Atimensec + out.Mtimensec = node.Attr.Mtimensec + out.Ctimensec = node.Attr.Ctimensec return fs.OK } @@ -123,7 +125,7 @@ func (n *FSNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) (* return nil, syscall.ENOENT } - log.Println("Storing content from source with path: ", sourcePath) + n.log("Storing content from source with path: ", sourcePath) _, err := n.filesystem.Client.StoreContentFromSource(sourcePath, 0) if err != nil { return nil, syscall.ENOENT diff --git a/pkg/metadata.go b/pkg/metadata.go index a3d3b1a..37621e8 100644 --- a/pkg/metadata.go +++ b/pkg/metadata.go @@ -3,7 +3,6 @@ package blobcache import ( "context" "fmt" - "log" "os" "path/filepath" "strings" @@ -244,8 +243,6 @@ func (m *BlobCacheMetadata) GetFsNode(ctx context.Context, id string) (*BlobFsMe func (m *BlobCacheMetadata) SetFsNode(ctx context.Context, id string, metadata *BlobFsMetadata) error { key := MetadataKeys.MetadataFsNode(id) - log.Printf("setting fs mode: %+v\n", metadata) - err := m.rdb.HSet(ctx, key, ToSlice(metadata)).Err() if err != nil { return fmt.Errorf("failed to set blobfs node metadata <%v>: %w", key, err) From 10cdad52714d60a4eb736ac835aaa207a7338a11 Mon Sep 17 00:00:00 2001 From: luke-lombardi <33990301+luke-lombardi@users.noreply.github.com> Date: Sat, 14 Sep 2024 11:06:26 -0400 Subject: [PATCH 13/18] cleanup, disable caching --- pkg/blobfs.go | 4 ++-- pkg/blobfs_node.go | 10 +++------- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/pkg/blobfs.go b/pkg/blobfs.go index b7661ec..03cc613 100644 --- a/pkg/blobfs.go +++ b/pkg/blobfs.go @@ -97,8 +97,8 @@ func Mount(ctx context.Context, opts BlobFsSystemOpts) (func() error, <-chan err } root, _ := blobfs.Root() - attrTimeout := time.Second * 60 - entryTimeout := time.Second * 60 + attrTimeout := time.Second * 0 + entryTimeout := time.Second * 0 fsOptions := &fs.Options{ AttrTimeout: &attrTimeout, EntryTimeout: &entryTimeout, diff --git a/pkg/blobfs_node.go b/pkg/blobfs_node.go index 358f59c..f98bf97 100644 --- a/pkg/blobfs_node.go +++ b/pkg/blobfs_node.go @@ -109,12 +109,8 @@ func (n *FSNode) inodeFromFsId(ctx context.Context, fsId string) (*fs.Inode, *fu } func (n *FSNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) (*fs.Inode, syscall.Errno) { - n.log("Lookup called with name: %s", name) - - // Construct the full of this file path from root - fullPath := path.Join(n.bfsNode.Path, name) - - n.log("Lookup called with full path: %s", fullPath) + fullPath := path.Join(n.bfsNode.Path, name) // Construct the full of this file path from root + n.log("Lookup called with path: %s", fullPath) // Force caching of a specific full path if the path contains a special illegal character '%' // This is a hack to trigger caching from external callers without going through the GRPC service directly @@ -125,7 +121,7 @@ func (n *FSNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) (* return nil, syscall.ENOENT } - n.log("Storing content from source with path: ", sourcePath) + n.log("Storing content from source with path: %s", sourcePath) _, err := n.filesystem.Client.StoreContentFromSource(sourcePath, 0) if err != nil { return nil, syscall.ENOENT From 2e3c5940fca19948e9613d4e49a10515e2f5ed61 Mon Sep 17 00:00:00 2001 From: luke-lombardi <33990301+luke-lombardi@users.noreply.github.com> Date: Sat, 14 Sep 2024 11:18:52 -0400 Subject: [PATCH 14/18] try incrementing generation # --- pkg/blobfs.go | 5 +++-- pkg/blobfs_node.go | 2 +- pkg/metadata.go | 19 +++++++++++++++++-- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/pkg/blobfs.go b/pkg/blobfs.go index 03cc613..b03e877 100644 --- a/pkg/blobfs.go +++ b/pkg/blobfs.go @@ -38,6 +38,7 @@ type BlobFsMetadata struct { Padding uint32 `redis:"padding" json:"padding"` Uid uint32 `redis:"uid" json:"uid"` Gid uint32 `redis:"gid" json:"gid"` + Gen uint64 `redis:"gen" json:"gen"` } type StorageLayer interface { @@ -97,8 +98,8 @@ func Mount(ctx context.Context, opts BlobFsSystemOpts) (func() error, <-chan err } root, _ := blobfs.Root() - attrTimeout := time.Second * 0 - entryTimeout := time.Second * 0 + attrTimeout := time.Second * 60 + entryTimeout := time.Second * 60 fsOptions := &fs.Options{ AttrTimeout: &attrTimeout, EntryTimeout: &entryTimeout, diff --git a/pkg/blobfs_node.go b/pkg/blobfs_node.go index f98bf97..2d4cb7b 100644 --- a/pkg/blobfs_node.go +++ b/pkg/blobfs_node.go @@ -102,7 +102,7 @@ func (n *FSNode) inodeFromFsId(ctx context.Context, fsId string) (*fs.Inode, *fu Attr: attr, Target: "", }, attr: attr}, - fs.StableAttr{Mode: metadata.Mode, Ino: metadata.Ino}, + fs.StableAttr{Mode: metadata.Mode, Ino: metadata.Ino, Gen: metadata.Gen}, ) return node, &attr, nil diff --git a/pkg/metadata.go b/pkg/metadata.go index 37621e8..f8fdcf3 100644 --- a/pkg/metadata.go +++ b/pkg/metadata.go @@ -223,7 +223,7 @@ func (m *BlobCacheMetadata) StoreContentInBlobFs(ctx context.Context, path strin func (m *BlobCacheMetadata) GetFsNode(ctx context.Context, id string) (*BlobFsMetadata, error) { key := MetadataKeys.MetadataFsNode(id) - res, err := m.rdb.HGetAll(context.TODO(), key).Result() + res, err := m.rdb.HGetAll(ctx, key).Result() if err != nil && err != redis.Nil { return nil, err } @@ -243,7 +243,22 @@ func (m *BlobCacheMetadata) GetFsNode(ctx context.Context, id string) (*BlobFsMe func (m *BlobCacheMetadata) SetFsNode(ctx context.Context, id string, metadata *BlobFsMetadata) error { key := MetadataKeys.MetadataFsNode(id) - err := m.rdb.HSet(ctx, key, ToSlice(metadata)).Err() + // If metadata exists, increment inode generation # + res, err := m.rdb.HGetAll(ctx, key).Result() + if err != nil && err != redis.Nil { + return err + } + + if len(res) > 0 { + existingMetadata := &BlobFsMetadata{} + if err = ToStruct(res, existingMetadata); err != nil { + return err + } + + metadata.Gen = existingMetadata.Gen + 1 + } + + err = m.rdb.HSet(ctx, key, ToSlice(metadata)).Err() if err != nil { return fmt.Errorf("failed to set blobfs node metadata <%v>: %w", key, err) } From 81d78704dec8c6f0ee4ca937640d5ad63ff546be Mon Sep 17 00:00:00 2001 From: luke-lombardi <33990301+luke-lombardi@users.noreply.github.com> Date: Sat, 14 Sep 2024 11:44:03 -0400 Subject: [PATCH 15/18] wip --- pkg/blobfs.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/blobfs.go b/pkg/blobfs.go index b03e877..4fea3cc 100644 --- a/pkg/blobfs.go +++ b/pkg/blobfs.go @@ -98,8 +98,8 @@ func Mount(ctx context.Context, opts BlobFsSystemOpts) (func() error, <-chan err } root, _ := blobfs.Root() - attrTimeout := time.Second * 60 - entryTimeout := time.Second * 60 + attrTimeout := time.Second * 10 + entryTimeout := time.Second * 10 fsOptions := &fs.Options{ AttrTimeout: &attrTimeout, EntryTimeout: &entryTimeout, From 356ff2befee1095accce93be2d9a8d11626a887c Mon Sep 17 00:00:00 2001 From: luke-lombardi <33990301+luke-lombardi@users.noreply.github.com> Date: Sat, 14 Sep 2024 11:49:38 -0400 Subject: [PATCH 16/18] lower attr timeouts --- pkg/blobfs.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/blobfs.go b/pkg/blobfs.go index 4fea3cc..cf65625 100644 --- a/pkg/blobfs.go +++ b/pkg/blobfs.go @@ -98,8 +98,8 @@ func Mount(ctx context.Context, opts BlobFsSystemOpts) (func() error, <-chan err } root, _ := blobfs.Root() - attrTimeout := time.Second * 10 - entryTimeout := time.Second * 10 + attrTimeout := time.Second * 5 + entryTimeout := time.Second * 5 fsOptions := &fs.Options{ AttrTimeout: &attrTimeout, EntryTimeout: &entryTimeout, From 6d72de876aad17548b62feff4789361445fce5ca Mon Sep 17 00:00:00 2001 From: luke-lombardi <33990301+luke-lombardi@users.noreply.github.com> Date: Sat, 14 Sep 2024 11:49:49 -0400 Subject: [PATCH 17/18] wip --- pkg/blobfs.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/blobfs.go b/pkg/blobfs.go index cf65625..a5875bd 100644 --- a/pkg/blobfs.go +++ b/pkg/blobfs.go @@ -118,9 +118,9 @@ func Mount(ctx context.Context, opts BlobFsSystemOpts) (func() error, <-chan err server, err := fuse.NewServer(fs.NewNodeFS(root, fsOptions), mountPoint, &fuse.MountOptions{ MaxBackground: maxBackgroundTasks, DisableXAttrs: true, - EnableSymlinkCaching: false, + EnableSymlinkCaching: true, SyncRead: false, - RememberInodes: false, + RememberInodes: true, MaxReadAhead: maxReadAheadKB * 1024, }) if err != nil { From 1457b57c041bb0f0a8f920e0260f13c45e201e14 Mon Sep 17 00:00:00 2001 From: luke-lombardi <33990301+luke-lombardi@users.noreply.github.com> Date: Sat, 14 Sep 2024 11:51:13 -0400 Subject: [PATCH 18/18] fix verbose flag --- pkg/client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/client.go b/pkg/client.go index 519719e..8f99e85 100644 --- a/pkg/client.go +++ b/pkg/client.go @@ -108,7 +108,7 @@ func NewBlobCacheClient(ctx context.Context, cfg BlobCacheConfig) (*BlobCacheCli Config: cfg, Metadata: metadata, Client: bc, - Verbose: true, + Verbose: cfg.DebugMode, }) if err != nil { return nil, err