-
Notifications
You must be signed in to change notification settings - Fork 2.9k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
7 changed files
with
248 additions
and
15 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,11 @@ | ||
| package diskcache | ||
|
|
||
| import ( | ||
| "context" | ||
| ) | ||
|
|
||
| type Interface interface { | ||
| Store(ctx context.Context, key string, value []byte) error | ||
| Load(ctx context.Context, key string) (value []byte, exist bool, err error) | ||
| Delete(ctx context.Context, key string) error | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,110 @@ | ||
| package diskcache | ||
|
|
||
| import ( | ||
| "context" | ||
| "crypto/sha1" //nolint:gosec | ||
| "encoding/hex" | ||
| "errors" | ||
| "fmt" | ||
| "io/ioutil" | ||
| "os" | ||
| "path/filepath" | ||
| "sync" | ||
|
|
||
| "github.com/spf13/afero" | ||
| ) | ||
|
|
||
| type FileCache struct { | ||
| fs afero.Fs | ||
|
|
||
| // granular locks | ||
| scopedLocks struct { | ||
| sync.Mutex | ||
| sync.Once | ||
| locks map[string]sync.Locker | ||
| } | ||
| } | ||
|
|
||
| func New(fs afero.Fs, root string) *FileCache { | ||
| return &FileCache{ | ||
| fs: afero.NewBasePathFs(fs, root), | ||
| } | ||
| } | ||
|
|
||
| func (f *FileCache) Store(ctx context.Context, key string, value []byte) error { | ||
| mu := f.getScopedLocks(key) | ||
| mu.Lock() | ||
| defer mu.Unlock() | ||
|
|
||
| fileName := f.getFileName(key) | ||
| if err := f.fs.MkdirAll(filepath.Dir(fileName), 0700); err != nil { | ||
| return err | ||
| } | ||
|
|
||
| if err := afero.WriteFile(f.fs, fileName, value, 0700); err != nil { | ||
| return err | ||
| } | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| func (f *FileCache) Load(ctx context.Context, key string) (value []byte, exist bool, err error) { | ||
| r, ok, err := f.open(key) | ||
| if err != nil || !ok { | ||
| return nil, ok, err | ||
| } | ||
| defer r.Close() | ||
|
|
||
| value, err = ioutil.ReadAll(r) | ||
| if err != nil { | ||
| return nil, false, err | ||
| } | ||
| return value, true, nil | ||
| } | ||
|
|
||
| func (f *FileCache) Delete(ctx context.Context, key string) error { | ||
| mu := f.getScopedLocks(key) | ||
| mu.Lock() | ||
| defer mu.Unlock() | ||
|
|
||
| fileName := f.getFileName(key) | ||
| if err := f.fs.Remove(fileName); err != nil && err != os.ErrNotExist { | ||
| return err | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| func (f *FileCache) open(key string) (afero.File, bool, error) { | ||
| fileName := f.getFileName(key) | ||
| file, err := f.fs.Open(fileName) | ||
| if err != nil { | ||
| if errors.Is(err, os.ErrNotExist) { | ||
| return nil, false, nil | ||
| } | ||
| return nil, false, err | ||
| } | ||
|
|
||
| return file, true, nil | ||
| } | ||
|
|
||
| // getScopedLocks pull lock from the map if found or create a new one | ||
| func (f *FileCache) getScopedLocks(key string) (lock sync.Locker) { | ||
| f.scopedLocks.Do(func() { f.scopedLocks.locks = map[string]sync.Locker{} }) | ||
|
|
||
| f.scopedLocks.Lock() | ||
| lock, ok := f.scopedLocks.locks[key] | ||
| if !ok { | ||
| lock = &sync.Mutex{} | ||
| f.scopedLocks.locks[key] = lock | ||
| } | ||
| f.scopedLocks.Unlock() | ||
|
|
||
| return lock | ||
| } | ||
|
|
||
| func (f *FileCache) getFileName(key string) string { | ||
| hasher := sha1.New() //nolint:gosec | ||
| _, _ = hasher.Write([]byte(key)) | ||
| hash := hex.EncodeToString(hasher.Sum(nil)) | ||
| return fmt.Sprintf("%s/%s/%s", hash[:1], hash[1:3], hash) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,55 @@ | ||
| package diskcache | ||
|
|
||
| import ( | ||
| "context" | ||
| "path/filepath" | ||
| "testing" | ||
|
|
||
| "github.com/spf13/afero" | ||
| "github.com/stretchr/testify/require" | ||
| ) | ||
|
|
||
| func TestFileCache(t *testing.T) { | ||
| ctx := context.Background() | ||
| const ( | ||
| key = "key" | ||
| value = "some text" | ||
| newValue = "new text" | ||
| cacheRoot = "/cache" | ||
| cachedFilePath = "a/62/a62f2225bf70bfaccbc7f1ef2a397836717377de" | ||
| ) | ||
|
|
||
| fs := afero.NewMemMapFs() | ||
| cache := New(fs, "/cache") | ||
|
|
||
| // store new key | ||
| err := cache.Store(ctx, key, []byte(value)) | ||
| require.NoError(t, err) | ||
| checkValue(t, ctx, fs, filepath.Join(cacheRoot, cachedFilePath), cache, key, value) | ||
|
|
||
| // update existing key | ||
| err = cache.Store(ctx, key, []byte(newValue)) | ||
| require.NoError(t, err) | ||
| checkValue(t, ctx, fs, filepath.Join(cacheRoot, cachedFilePath), cache, key, newValue) | ||
|
|
||
| // delete key | ||
| err = cache.Delete(ctx, key) | ||
| require.NoError(t, err) | ||
| exists, err := afero.Exists(fs, filepath.Join(cacheRoot, cachedFilePath)) | ||
| require.NoError(t, err) | ||
| require.False(t, exists) | ||
| } | ||
|
|
||
| func checkValue(t *testing.T, ctx context.Context, fs afero.Fs, fileFullPath string, cache *FileCache, key, wantValue string) { //nolint:golint | ||
| t.Helper() | ||
| // check actual file content | ||
| b, err := afero.ReadFile(fs, fileFullPath) | ||
| require.NoError(t, err) | ||
| require.Equal(t, wantValue, string(b)) | ||
|
|
||
| // check cache content | ||
| b, ok, err := cache.Load(ctx, key) | ||
| require.NoError(t, err) | ||
| require.True(t, ok) | ||
| require.Equal(t, wantValue, string(b)) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,24 @@ | ||
| package diskcache | ||
|
|
||
| import ( | ||
| "context" | ||
| ) | ||
|
|
||
| type NoOp struct { | ||
| } | ||
|
|
||
| func NewNoOp() *NoOp { | ||
| return &NoOp{} | ||
| } | ||
|
|
||
| func (n *NoOp) Store(ctx context.Context, key string, value []byte) error { | ||
| return nil | ||
| } | ||
|
|
||
| func (n *NoOp) Load(ctx context.Context, key string) (value []byte, exist bool, err error) { | ||
| return nil, false, nil | ||
| } | ||
|
|
||
| func (n *NoOp) Delete(ctx context.Context, key string) error { | ||
| return nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters