From ef36dd97c437c1a4120480e5306fafe46a43ae56 Mon Sep 17 00:00:00 2001 From: Alex Guidi <122013838+aguidirh@users.noreply.github.com> Date: Thu, 23 Nov 2023 17:34:27 +0100 Subject: [PATCH] feat: history file reader and writer (#736) --- v2/pkg/history/const.go | 8 + v2/pkg/history/history.go | 163 ++++++++++++++++++ v2/pkg/history/history_test.go | 154 +++++++++++++++++ v2/pkg/history/interface.go | 12 ++ .../.history-2023-11-21T11:47:46Z | 1 + .../.history-2023-11-22T11:24:14Z | 4 + v2/vendor/modules.txt | 16 -- 7 files changed, 342 insertions(+), 16 deletions(-) create mode 100644 v2/pkg/history/const.go create mode 100644 v2/pkg/history/history.go create mode 100644 v2/pkg/history/history_test.go create mode 100644 v2/pkg/history/interface.go create mode 100644 v2/tests/.history-fake/.history-2023-11-21T11:47:46Z create mode 100644 v2/tests/.history-fake/.history-2023-11-22T11:24:14Z diff --git a/v2/pkg/history/const.go b/v2/pkg/history/const.go new file mode 100644 index 000000000..d98756f82 --- /dev/null +++ b/v2/pkg/history/const.go @@ -0,0 +1,8 @@ +package history + +const ( + //TODO Alex during the integration of the building blocks, add the historyPath after the workingDir when NewHistory is called + historyPath = ".history/" + historyNamePrefix = ".history-" + historyFakePath = "../../tests/.history-fake/" +) diff --git a/v2/pkg/history/history.go b/v2/pkg/history/history.go new file mode 100644 index 000000000..3fc39b1d2 --- /dev/null +++ b/v2/pkg/history/history.go @@ -0,0 +1,163 @@ +package history + +import ( + "bufio" + "io" + "io/fs" + "os" + "strings" + "time" + + clog "github.com/openshift/oc-mirror/v2/pkg/log" +) + +var log clog.PluggableLoggerInterface + +type OSFileCreator struct{} +type history struct { + workingDir string + before time.Time + fileCreator FileCreator +} + +func NewHistory(workingDir string, before time.Time, logg clog.PluggableLoggerInterface, fileCreator FileCreator) (History, error) { + if logg == nil { + log = clog.New("error") + } else { + log = logg + } + + return history{ + workingDir: workingDir, + before: before, + fileCreator: fileCreator, + }, nil +} + +func (o history) Read() (map[string]string, error) { + historyFile, err := o.getHistoryFile(o.before) + if err != nil { + return nil, err + } + + file, err := os.Open(o.workingDir + historyFile.Name()) + if err != nil { + log.Error("unable to open history file: %s", err.Error()) + return nil, err + } + defer file.Close() + + scanner := bufio.NewScanner(file) + historyMap := make(map[string]string) + + for scanner.Scan() { + blob := scanner.Text() + historyMap[blob] = "" + } + + if err := scanner.Err(); err != nil { + log.Error("unable to read history file: %s", err.Error()) + return nil, err + } + + return historyMap, nil +} + +func (o history) getHistoryFile(before time.Time) (fs.DirEntry, error) { + historyFiles, err := os.ReadDir(o.workingDir) + if err != nil { + log.Error("unable to read history directory: %s", err.Error()) + return nil, err + } + + var latestFile fs.DirEntry + var latestTime time.Time + + for _, historyFile := range historyFiles { + if isHistoryFile(historyFile) { + fileTime, err := getFileDate(historyFile) + if err != nil { + return nil, err + } + + if !before.IsZero() { + if fileTime.After(latestTime) && fileTime.Before(before) { + latestFile = historyFile + latestTime = fileTime + } + } else { + if fileTime.After(latestTime) { + latestFile = historyFile + latestTime = fileTime + } + } + } + } + + return latestFile, err +} + +func isHistoryFile(historyFile fs.DirEntry) bool { + return !historyFile.IsDir() && strings.HasPrefix(historyFile.Name(), historyNamePrefix) +} + +func getFileDate(historyFile fs.DirEntry) (time.Time, error) { + fileDate := strings.TrimPrefix(historyFile.Name(), historyNamePrefix) + dateTime, err := time.Parse(time.RFC3339, fileDate) + if err != nil { + log.Error("unable to parse time from filename %s: %s", historyFile.Name(), err.Error()) + return time.Time{}, err + } + return dateTime, err +} + +func (o history) Append(blobsToAppend map[string]string) (map[string]string, error) { + + filename := o.newFileName() + + historyBlobs, err := o.Read() + if err != nil { + return historyBlobs, err + } + + for k, v := range blobsToAppend { + historyBlobs[k] = v + } + + file, err := o.fileCreator.Create(filename) + if err != nil { + return historyBlobs, err + } + defer file.Close() + + writer := bufio.NewWriter(file) + + for blob := range historyBlobs { + _, err := writer.WriteString(blob + "\n") + if err != nil { + log.Error("unable to write to history file: %s", err.Error()) + return historyBlobs, err + } + } + + err = writer.Flush() + if err != nil { + log.Error("unable to flush history file: %s", err.Error()) + return historyBlobs, err + } + + return historyBlobs, err + +} + +func (o history) newFileName() string { + return o.workingDir + historyNamePrefix + time.Now().UTC().Format(time.RFC3339) +} + +func (OSFileCreator) Create(filename string) (io.WriteCloser, error) { + file, err := os.Create(filename) + if err != nil { + log.Error("unable to create file: %s", err.Error()) + } + return file, err +} diff --git a/v2/pkg/history/history_test.go b/v2/pkg/history/history_test.go new file mode 100644 index 000000000..e0297df30 --- /dev/null +++ b/v2/pkg/history/history_test.go @@ -0,0 +1,154 @@ +package history + +import ( + "bytes" + "io" + "testing" + "time" + + clog "github.com/openshift/oc-mirror/v2/pkg/log" + "github.com/stretchr/testify/assert" +) + +type MockFileCreator struct { + Buffer *bytes.Buffer +} + +type nopCloser struct { + io.Writer +} + +func (m MockFileCreator) Create(name string) (io.WriteCloser, error) { + m.Buffer = new(bytes.Buffer) + return nopCloser{m.Buffer}, nil +} + +func (nopCloser) Close() error { return nil } + +func TestNewHistory(t *testing.T) { + history, err := NewHistory(historyFakePath, time.Time{}, clog.New("trace"), MockFileCreator{}) + assert.NoError(t, err) + assert.NotNil(t, history) +} + +func TestRead(t *testing.T) { + + type testCase struct { + caseName string + workingDir string + before time.Time + expectedError string + expectedHist map[string]string + } + + testCases := []testCase{ + { + caseName: "valid history file - without specified time", + workingDir: historyFakePath, + before: time.Time{}, + expectedError: "", + expectedHist: map[string]string{ + "sha256:1dddb0988d16": "", + "sha256:3658954f1990": "", + "sha256:e3dad360d035": "", + "sha256:422e4fbe1ed8": "", + }, + }, + { + caseName: "valid history file - with specified time", + workingDir: historyFakePath, + before: time.Date(2023, 11, 22, 0, 0, 0, 0, time.UTC), + expectedError: "", + expectedHist: map[string]string{ + "sha256:1dddb0988d16": "", + }, + }, + { + caseName: "invalid working dir", + workingDir: "./invalid-workindir", + before: time.Time{}, + expectedError: "open ./invalid-workindir: no such file or directory", + expectedHist: nil, + }, + } + + for _, test := range testCases { + t.Run(test.caseName, func(t *testing.T) { + history, err := NewHistory(test.workingDir, test.before, clog.New("trace"), MockFileCreator{}) + assert.NoError(t, err) + + historyMap, err := history.Read() + if test.expectedError != "" { + assert.EqualError(t, err, test.expectedError) + } + assert.Equal(t, test.expectedHist, historyMap) + }) + } +} + +func TestAppend(t *testing.T) { + + type testCase struct { + caseName string + workingDir string + before time.Time + blobsToAppend map[string]string + expectedError string + expectedHist map[string]string + } + + testCases := []testCase{ + { + caseName: "valid history file - without specified time", + workingDir: historyFakePath, + before: time.Time{}, + blobsToAppend: map[string]string{ + "sha256:20f695d2a913": "", + }, + expectedError: "", + expectedHist: map[string]string{ + "sha256:422e4fbe1ed8": "", + "sha256:1dddb0988d16": "", + "sha256:3658954f1990": "", + "sha256:e3dad360d035": "", + "sha256:20f695d2a913": "", + }, + }, + { + caseName: "valid history file - with specified time", + workingDir: historyFakePath, + before: time.Date(2023, 11, 22, 0, 0, 0, 0, time.UTC), + blobsToAppend: map[string]string{ + "sha256:20f695d2a913": "", + }, + expectedError: "", + expectedHist: map[string]string{ + "sha256:1dddb0988d16": "", + "sha256:20f695d2a913": "", + }, + }, + { + caseName: "invalid working dir", + workingDir: "./invalid-workindir", + before: time.Time{}, + expectedError: "open ./invalid-workindir: no such file or directory", + expectedHist: nil, + }, + } + + for _, test := range testCases { + t.Run(test.caseName, func(t *testing.T) { + history, err := NewHistory(test.workingDir, test.before, clog.New("trace"), MockFileCreator{}) + assert.NoError(t, err) + historyBlobs, err := history.Append(test.blobsToAppend) + + if test.expectedError != "" { + assert.EqualError(t, err, test.expectedError) + } else { + assert.NoError(t, err) + assert.Equal(t, test.expectedHist, historyBlobs) + } + }) + } + +} diff --git a/v2/pkg/history/interface.go b/v2/pkg/history/interface.go new file mode 100644 index 000000000..04ad4ce43 --- /dev/null +++ b/v2/pkg/history/interface.go @@ -0,0 +1,12 @@ +package history + +import "io" + +type History interface { + Read() (map[string]string, error) + Append(map[string]string) (map[string]string, error) +} + +type FileCreator interface { + Create(name string) (io.WriteCloser, error) +} diff --git a/v2/tests/.history-fake/.history-2023-11-21T11:47:46Z b/v2/tests/.history-fake/.history-2023-11-21T11:47:46Z new file mode 100644 index 000000000..bcb014437 --- /dev/null +++ b/v2/tests/.history-fake/.history-2023-11-21T11:47:46Z @@ -0,0 +1 @@ +sha256:1dddb0988d16 \ No newline at end of file diff --git a/v2/tests/.history-fake/.history-2023-11-22T11:24:14Z b/v2/tests/.history-fake/.history-2023-11-22T11:24:14Z new file mode 100644 index 000000000..e3df5805b --- /dev/null +++ b/v2/tests/.history-fake/.history-2023-11-22T11:24:14Z @@ -0,0 +1,4 @@ +sha256:1dddb0988d16 +sha256:3658954f1990 +sha256:e3dad360d035 +sha256:422e4fbe1ed8 diff --git a/v2/vendor/modules.txt b/v2/vendor/modules.txt index 611203967..027d86d27 100644 --- a/v2/vendor/modules.txt +++ b/v2/vendor/modules.txt @@ -308,7 +308,6 @@ github.com/docker/docker/api/types/volume github.com/docker/docker/client github.com/docker/docker/errdefs github.com/docker/docker/pkg/homedir -github.com/docker/docker/testutil/registry # github.com/docker/docker-credential-helpers v0.7.0 ## explicit; go 1.18 github.com/docker/docker-credential-helpers/client @@ -396,13 +395,6 @@ github.com/golang/protobuf/ptypes/timestamp # github.com/gomodule/redigo v1.8.2 ## explicit; go 1.14 github.com/gomodule/redigo/redis -# github.com/google/go-cmp v0.5.9 -## explicit; go 1.13 -github.com/google/go-cmp/cmp -github.com/google/go-cmp/cmp/internal/diff -github.com/google/go-cmp/cmp/internal/flags -github.com/google/go-cmp/cmp/internal/function -github.com/google/go-cmp/cmp/internal/value # github.com/google/go-containerregistry v0.15.2 ## explicit; go 1.18 github.com/google/go-containerregistry/internal/and @@ -907,14 +899,6 @@ gopkg.in/yaml.v2 # gopkg.in/yaml.v3 v3.0.1 ## explicit gopkg.in/yaml.v3 -# gotest.tools/v3 v3.4.0 -## explicit; go 1.13 -gotest.tools/v3/assert -gotest.tools/v3/assert/cmp -gotest.tools/v3/internal/assert -gotest.tools/v3/internal/difflib -gotest.tools/v3/internal/format -gotest.tools/v3/internal/source # k8s.io/api v0.26.3 ## explicit; go 1.19 k8s.io/api/core/v1