feat: Adds a new raw file metadata storage for clients (#347)

* Added first draft for a raw json local file storage. Signed-off-by: Fredrik Skogman <kommendorkapten@github.com> * Ignore emacs temporary files Signed-off-by: Fredrik Skogman <kommendorkapten@github.com> * moved isMetaFile to util directory for reuse in other packages. Signed-off-by: Fredrik Skogman <kommendorkapten@github.com> * Added unit tests and refactored code to match the local store for repository side. Signed-off-by: Fredrik Skogman <kommendorkapten@github.com> * Added test case for non json file in metadata directory. Changed package to client to align with leveldb storage. Signed-off-by: Fredrik Skogman <kommendorkapten@github.com> * Use os.MkdirAll when creating metadata cache. Signed-off-by: Fredrik Skogman <kommendorkapten@github.com> * More consistent naming, added comments and a unit test. Signed-off-by: Fredrik Skogman <kommendorkapten@github.com> * Added a localstore wrapper for concurrent access. Signed-off-by: Fredrik Skogman <kommendorkapten@github.com> * Update client/filejsonstore/filejsonstore_test.go Fixed spelling error found during review. Co-authored-by: Joshua Lock <jlock@vmware.com> Signed-off-by: Fredrik Skogman <kommendorkapten@github.com> * Added tests to make sure returned struct implements LocalStore interface. Signed-off-by: Fredrik Skogman <kommendorkapten@github.com> * Update client/filejsonstore/filejsonstore.go Co-authored-by: Ethan Lowman <53835328+ethan-lowman-dd@users.noreply.github.com> Signed-off-by: Fredrik Skogman <kommendorkapten@github.com> * Update client/filejsonstore/filejsonstore_test.go Co-authored-by: Ethan Lowman <53835328+ethan-lowman-dd@users.noreply.github.com> Signed-off-by: Fredrik Skogman <kommendorkapten@github.com> * Update client/filejsonstore/filejsonstore.go Co-authored-by: Ethan Lowman <53835328+ethan-lowman-dd@users.noreply.github.com> Signed-off-by: Fredrik Skogman <kommendorkapten@github.com> * Update client/filejsonstore/filejsonstore_test.go Co-authored-by: Ethan Lowman <53835328+ethan-lowman-dd@users.noreply.github.com> Signed-off-by: Fredrik Skogman <kommendorkapten@github.com> * Made FileJSONStore safe for concurrent access. Moved IsMetaFile to new pkg, internal/fsutil Permission bits are validated when access the cache. Signed-off-by: Fredrik Skogman <kommendorkapten@github.com> * Spelling error. Signed-off-by: Fredrik Skogman <kommendorkapten@github.com> * Updates based on PR comments. Removed a test for satisfying an interface, and replaced with a compile time check. Signed-off-by: Fredrik Skogman <kommendorkapten@github.com> * Added first draft for a raw json local file storage. Signed-off-by: Fredrik Skogman <kommendorkapten@github.com> * Ignore emacs temporary files Signed-off-by: Fredrik Skogman <kommendorkapten@github.com> * moved isMetaFile to util directory for reuse in other packages. Signed-off-by: Fredrik Skogman <kommendorkapten@github.com> * Added unit tests and refactored code to match the local store for repository side. Signed-off-by: Fredrik Skogman <kommendorkapten@github.com> * Added test case for non json file in metadata directory. Changed package to client to align with leveldb storage. Signed-off-by: Fredrik Skogman <kommendorkapten@github.com> * Use os.MkdirAll when creating metadata cache. Signed-off-by: Fredrik Skogman <kommendorkapten@github.com> * More consistent naming, added comments and a unit test. Signed-off-by: Fredrik Skogman <kommendorkapten@github.com> * Added a localstore wrapper for concurrent access. Signed-off-by: Fredrik Skogman <kommendorkapten@github.com> * Update client/filejsonstore/filejsonstore_test.go Fixed spelling error found during review. Co-authored-by: Joshua Lock <jlock@vmware.com> Signed-off-by: Fredrik Skogman <kommendorkapten@github.com> * Added tests to make sure returned struct implements LocalStore interface. Signed-off-by: Fredrik Skogman <kommendorkapten@github.com> * Update client/filejsonstore/filejsonstore.go Co-authored-by: Ethan Lowman <53835328+ethan-lowman-dd@users.noreply.github.com> Signed-off-by: Fredrik Skogman <kommendorkapten@github.com> * Update client/filejsonstore/filejsonstore_test.go Co-authored-by: Ethan Lowman <53835328+ethan-lowman-dd@users.noreply.github.com> Signed-off-by: Fredrik Skogman <kommendorkapten@github.com> * Update client/filejsonstore/filejsonstore.go Co-authored-by: Ethan Lowman <53835328+ethan-lowman-dd@users.noreply.github.com> Signed-off-by: Fredrik Skogman <kommendorkapten@github.com> * Update client/filejsonstore/filejsonstore_test.go Co-authored-by: Ethan Lowman <53835328+ethan-lowman-dd@users.noreply.github.com> Signed-off-by: Fredrik Skogman <kommendorkapten@github.com> * Made FileJSONStore safe for concurrent access. Moved IsMetaFile to new pkg, internal/fsutil Permission bits are validated when access the cache. Signed-off-by: Fredrik Skogman <kommendorkapten@github.com> * Spelling error. Signed-off-by: Fredrik Skogman <kommendorkapten@github.com> * Updates based on PR comments. Removed a test for satisfying an interface, and replaced with a compile time check. Signed-off-by: Fredrik Skogman <kommendorkapten@github.com> * Disabled filesystem permission checks for windows. Windows filesystem permission is a bit different from UNIX like systems, and there is no good API in go to manipulate it. Nor is having these checks a requirement by the TUF spec. This commit relies on build tags to inject the correct filesystem permssion check, the Windows version always return "satisfied". Signed-off-by: Fredrik Skogman <kommendorkapten@github.com> * Moved the tests that rely on permission bits to a new test file. The permission bits are not executed during windows builds. Signed-off-by: Fredrik Skogman <kommendorkapten@github.com> * Clarify permissions check naming and comment, add tests Signed-off-by: Ethan Lowman <ethan.lowman@datadoghq.com> * Update internal/fsutil/fsutil.go Co-authored-by: Ethan Lowman <53835328+ethan-lowman-dd@users.noreply.github.com> Signed-off-by: Fredrik Skogman <kommendorkapten@github.com> * Updates based on feeback. Mostly minor things like removing sentinel errors, wrapping errors from std library. Signed-off-by: Fredrik Skogman <kommendorkapten@github.com> * Use fs.ErrNotExist instead of os.ErrNotExist as recommended by Go docs Signed-off-by: Ethan Lowman <ethan.lowman@datadoghq.com> * Clean up remaining unnecessary usage of filepath.FromSlash Signed-off-by: Ethan Lowman <ethan.lowman@datadoghq.com> * Add missing error checks in tests Signed-off-by: Ethan Lowman <ethan.lowman@datadoghq.com> * Make sure to test that returned err is nil in the tests. Signed-off-by: Fredrik Skogman <kommendorkapten@github.com> Signed-off-by: Fredrik Skogman <kommendorkapten@github.com> Signed-off-by: Ethan Lowman <ethan.lowman@datadoghq.com> Co-authored-by: Joshua Lock <jlock@vmware.com> Co-authored-by: Ethan Lowman <53835328+ethan-lowman-dd@users.noreply.github.com> Co-authored-by: Ethan Lowman <ethan.lowman@datadoghq.com>
theupdateframework · Sep 13, 2022 · 7097fd8 · 7097fd8
1 parent a9ddd89
commit 7097fd8
Show file tree

Hide file tree

Showing 9 changed files with 541 additions and 15 deletions.
diff --git a/.gitignore b/.gitignore
@@ -2,3 +2,4 @@
 cmd/tuf/tuf
 cmd/tuf-client/tuf-client
 .vscode
+*~
diff --git a/client/filejsonstore/filejsonstore.go b/client/filejsonstore/filejsonstore.go
@@ -0,0 +1,148 @@
+package client
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"sync"
+
+	"github.com/theupdateframework/go-tuf/client"
+	"github.com/theupdateframework/go-tuf/internal/fsutil"
+	"github.com/theupdateframework/go-tuf/util"
+)
+
+const (
+	// user:  rwx
+	// group: r-x
+	// other: ---
+	dirCreateMode = os.FileMode(0750)
+	// user:  rw-
+	// group: r--
+	// other: ---
+	fileCreateMode = os.FileMode(0640)
+)
+
+// FileJSONStore represents a local metadata cache relying on raw JSON files
+// as retrieved from the remote repository.
+type FileJSONStore struct {
+	mtx     sync.RWMutex
+	baseDir string
+}
+
+var _ client.LocalStore = (*FileJSONStore)(nil)
+
+// NewFileJSONStore returns a new metadata cache, implemented using raw JSON
+// files, stored in a directory provided by the client.
+// If the provided directory does not exist on disk, it will be created.
+// The provided metadata cache is safe for concurrent access.
+func NewFileJSONStore(baseDir string) (*FileJSONStore, error) {
+	f := &FileJSONStore{
+		baseDir: baseDir,
+	}
+
+	// Does the directory exist?
+	fi, err := os.Stat(baseDir)
+	if err != nil {
+		if errors.Is(err, fs.ErrNotExist) {
+			// Create the directory
+			if err = os.MkdirAll(baseDir, dirCreateMode); err != nil {
+				return nil, fmt.Errorf("error creating directory for metadata cache: %w", err)
+			}
+		} else {
+			return nil, fmt.Errorf("error getting FileInfo for %s: %w", baseDir, err)
+		}
+	} else {
+		// Verify that it is a directory
+		if !fi.IsDir() {
+			return nil, fmt.Errorf("can not open %s, not a directory", baseDir)
+		}
+		// Verify file mode is not too permissive.
+		if err = fsutil.EnsureMaxPermissions(fi, dirCreateMode); err != nil {
+			return nil, err
+		}
+	}
+
+	return f, nil
+}
+
+// GetMeta returns the currently cached set of metadata files.
+func (f *FileJSONStore) GetMeta() (map[string]json.RawMessage, error) {
+	f.mtx.RLock()
+	defer f.mtx.RUnlock()
+
+	names, err := os.ReadDir(f.baseDir)
+	if err != nil {
+		return nil, fmt.Errorf("error reading directory %s: %w", f.baseDir, err)
+	}
+
+	meta := map[string]json.RawMessage{}
+	for _, name := range names {
+		ok, err := fsutil.IsMetaFile(name)
+		if err != nil {
+			return nil, err
+		}
+		if !ok {
+			continue
+		}
+
+		// Verify permissions
+		info, err := name.Info()
+		if err != nil {
+			return nil, fmt.Errorf("error retrieving FileInfo for %s: %w", name.Name(), err)
+		}
+		if err = fsutil.EnsureMaxPermissions(info, fileCreateMode); err != nil {
+			return nil, err
+		}
+
+		p := filepath.Join(f.baseDir, name.Name())
+		b, err := os.ReadFile(p)
+		if err != nil {
+			return nil, fmt.Errorf("error reading file %s: %w", name.Name(), err)
+		}
+		meta[name.Name()] = b
+	}
+
+	return meta, nil
+}
+
+// SetMeta stores a metadata file in the cache. If the metadata file exist,
+// it will be overwritten.
+func (f *FileJSONStore) SetMeta(name string, meta json.RawMessage) error {
+	f.mtx.Lock()
+	defer f.mtx.Unlock()
+
+	if filepath.Ext(name) != ".json" {
+		return fmt.Errorf("file %s is not a JSON file", name)
+	}
+
+	p := filepath.Join(f.baseDir, name)
+	err := util.AtomicallyWriteFile(p, meta, fileCreateMode)
+	return err
+}
+
+// DeleteMeta deletes a metadata file from the cache.
+// If the file does not exist, an *os.PathError is returned.
+func (f *FileJSONStore) DeleteMeta(name string) error {
+	f.mtx.Lock()
+	defer f.mtx.Unlock()
+
+	if filepath.Ext(name) != ".json" {
+		return fmt.Errorf("file %s is not a JSON file", name)
+	}
+
+	p := filepath.Join(f.baseDir, name)
+	err := os.Remove(p)
+	if err == nil {
+		return nil
+	}
+
+	return fmt.Errorf("error deleting file %s: %w", name, err)
+}
+
+// Close closes the metadata cache. This is a no-op.
+func (f *FileJSONStore) Close() error {
+	return nil
+}
diff --git a/client/filejsonstore/filejsonstore_test.go b/client/filejsonstore/filejsonstore_test.go
@@ -0,0 +1,194 @@
+package client
+
+import (
+	"encoding/json"
+	"errors"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"gopkg.in/check.v1"
+)
+
+type FileJSONStoreSuite struct{}
+
+var _ = check.Suite(&FileJSONStoreSuite{})
+
+// Hook up gocheck into the "go test" runner
+func Test(t *testing.T) { check.TestingT(t) }
+
+func (FileJSONStoreSuite) TestNewOk(c *check.C) {
+	tmp := c.MkDir()
+	p := filepath.Join(tmp, "tuf_raw.db")
+
+	// Assert path does not exist
+	fi, err := os.Stat(p)
+	c.Assert(fi, check.IsNil)
+	c.Assert(errors.Is(err, os.ErrNotExist), check.Equals, true)
+
+	// Create implementation
+	s, err := NewFileJSONStore(p)
+	c.Assert(err, check.IsNil)
+	c.Assert(s, check.NotNil)
+
+	// Assert path does exist and is a directory
+	fi, err = os.Stat(p)
+	c.Assert(fi, check.NotNil)
+	c.Assert(err, check.IsNil)
+	c.Assert(fi.IsDir(), check.Equals, true)
+}
+
+func (FileJSONStoreSuite) TestNewFileExists(c *check.C) {
+	tmp := c.MkDir()
+	p := filepath.Join(tmp, "tuf_raw.db")
+
+	// Create an empty file
+	f, err := os.Create(p)
+	c.Assert(err, check.IsNil)
+	f.Close()
+
+	// Create implementation
+	s, err := NewFileJSONStore(p)
+	c.Assert(s, check.IsNil)
+	c.Assert(err, check.NotNil)
+	found := strings.Contains(err.Error(), ", not a directory")
+	c.Assert(found, check.Equals, true)
+}
+
+func (FileJSONStoreSuite) TestNewDirectoryExists(c *check.C) {
+	tmp := c.MkDir()
+	p := filepath.Join(tmp, "tuf_raw.db")
+
+	err := os.Mkdir(p, 0750)
+	c.Assert(err, check.IsNil)
+
+	// Create implementation
+	s, err := NewFileJSONStore(p)
+	c.Assert(s, check.NotNil)
+	c.Assert(err, check.IsNil)
+}
+
+func (FileJSONStoreSuite) TestGetMetaEmpty(c *check.C) {
+	tmp := c.MkDir()
+	p := filepath.Join(tmp, "tuf_raw.db")
+	s, err := NewFileJSONStore(p)
+	c.Assert(err, check.IsNil)
+
+	md, err := s.GetMeta()
+	c.Assert(err, check.IsNil)
+	c.Assert(md, check.HasLen, 0)
+}
+
+func (FileJSONStoreSuite) TestGetNoDirectory(c *check.C) {
+	tmp := c.MkDir()
+	p := filepath.Join(tmp, "tuf_raw.db")
+	s, err := NewFileJSONStore(p)
+	c.Assert(err, check.IsNil)
+
+	err = os.Remove(p)
+	c.Assert(err, check.IsNil)
+
+	md, err := s.GetMeta()
+	c.Assert(md, check.IsNil)
+	c.Assert(err, check.NotNil)
+}
+
+func (FileJSONStoreSuite) TestMetadataOperations(c *check.C) {
+	tmp := c.MkDir()
+	p := filepath.Join(tmp, "tuf_raw.db")
+	s, err := NewFileJSONStore(p)
+	c.Assert(err, check.IsNil)
+
+	expected := map[string]json.RawMessage{
+		"file1.json": []byte{0xf1, 0xe1, 0xd1},
+		"file2.json": []byte{0xf2, 0xe2, 0xd2},
+		"file3.json": []byte{0xf3, 0xe3, 0xd3},
+	}
+
+	for k, v := range expected {
+		err := s.SetMeta(k, v)
+		c.Assert(err, check.IsNil)
+	}
+
+	md, err := s.GetMeta()
+	c.Assert(err, check.IsNil)
+	c.Assert(md, check.HasLen, 3)
+	c.Assert(md, check.DeepEquals, expected)
+
+	// Delete all items
+	count := 3
+	for k := range expected {
+		err = s.DeleteMeta(k)
+		c.Assert(err, check.IsNil)
+
+		md, err := s.GetMeta()
+		c.Assert(err, check.IsNil)
+
+		count--
+		c.Assert(md, check.HasLen, count)
+	}
+
+	md, err = s.GetMeta()
+	c.Assert(err, check.IsNil)
+	c.Assert(md, check.HasLen, 0)
+}
+
+func (FileJSONStoreSuite) TestGetNoJSON(c *check.C) {
+	tmp := c.MkDir()
+	p := filepath.Join(tmp, "tuf_raw.db")
+	s, err := NewFileJSONStore(p)
+	c.Assert(s, check.NotNil)
+	c.Assert(err, check.IsNil)
+
+	// Create a file which does not end with '.json'
+	fp := filepath.Join(p, "meta.xml")
+	err = os.WriteFile(fp, []byte{}, 0644)
+	c.Assert(err, check.IsNil)
+
+	md, err := s.GetMeta()
+	c.Assert(err, check.IsNil)
+	c.Assert(md, check.HasLen, 0)
+}
+
+func (FileJSONStoreSuite) TestNoJSON(c *check.C) {
+	tmp := c.MkDir()
+	p := filepath.Join(tmp, "tuf_raw.db")
+	s, err := NewFileJSONStore(p)
+	c.Assert(s, check.NotNil)
+	c.Assert(err, check.IsNil)
+
+	files := []string{
+		"file.xml",
+		"file",
+		"",
+	}
+	for _, f := range files {
+		err := s.SetMeta(f, []byte{})
+		c.Assert(err, check.ErrorMatches, "file.*is not a JSON file")
+	}
+}
+
+func (FileJSONStoreSuite) TestClose(c *check.C) {
+	tmp := c.MkDir()
+	p := filepath.Join(tmp, "tuf_raw.db")
+	s, err := NewFileJSONStore(p)
+	c.Assert(s, check.NotNil)
+	c.Assert(err, check.IsNil)
+
+	err = s.Close()
+	c.Assert(err, check.IsNil)
+}
+
+func (FileJSONStoreSuite) TestDelete(c *check.C) {
+	tmp := c.MkDir()
+	p := filepath.Join(tmp, "tuf_raw.db")
+	s, err := NewFileJSONStore(p)
+	c.Assert(s, check.NotNil)
+	c.Assert(err, check.IsNil)
+
+	err = s.DeleteMeta("not_json.yml")
+	c.Assert(err, check.ErrorMatches, "file not_json\\.yml is not a JSON file")
+	err = s.DeleteMeta("non_existing.json")
+	c.Assert(errors.Is(err, os.ErrNotExist), check.Equals, true)
+}