feat: history file reader and writer (#736)

v2/pkg/history/const.go

@@ -0,0 +1,8 @@
+package history
+
+const (
+	//TODO Alex during the integration of the building blocks, add the historyPath after the workingDir when NewHistory is called
+	historyPath       = ".history/"
+	historyNamePrefix = ".history-"
+	historyFakePath   = "../../tests/.history-fake/"
+)

v2/pkg/history/history.go

@@ -0,0 +1,163 @@
+package history
+
+import (
+	"bufio"
+	"io"
+	"io/fs"
+	"os"
+	"strings"
+	"time"
+
+	clog "github.com/openshift/oc-mirror/v2/pkg/log"
+)
+
+var log clog.PluggableLoggerInterface
+
+type OSFileCreator struct{}
+type history struct {
+	workingDir  string
+	before      time.Time
+	fileCreator FileCreator
+}
+
+func NewHistory(workingDir string, before time.Time, logg clog.PluggableLoggerInterface, fileCreator FileCreator) (History, error) {
+	if logg == nil {
+		log = clog.New("error")
+	} else {
+		log = logg
+	}
+
+	return history{
+		workingDir:  workingDir,
+		before:      before,
+		fileCreator: fileCreator,
+	}, nil
+}
+
+func (o history) Read() (map[string]string, error) {
+	historyFile, err := o.getHistoryFile(o.before)
+	if err != nil {
+		return nil, err
+	}
+
+	file, err := os.Open(o.workingDir + historyFile.Name())
+	if err != nil {
+		log.Error("unable to open history file: %s", err.Error())
+		return nil, err
+	}
+	defer file.Close()
+
+	scanner := bufio.NewScanner(file)
+	historyMap := make(map[string]string)
+
+	for scanner.Scan() {
+		blob := scanner.Text()
+		historyMap[blob] = ""
+	}
+
+	if err := scanner.Err(); err != nil {
+		log.Error("unable to read history file: %s", err.Error())
+		return nil, err
+	}
+
+	return historyMap, nil
+}
+
+func (o history) getHistoryFile(before time.Time) (fs.DirEntry, error) {
+	historyFiles, err := os.ReadDir(o.workingDir)
+	if err != nil {
+		log.Error("unable to read history directory: %s", err.Error())
+		return nil, err
+	}
+
+	var latestFile fs.DirEntry
+	var latestTime time.Time
+
+	for _, historyFile := range historyFiles {
+		if isHistoryFile(historyFile) {
+			fileTime, err := getFileDate(historyFile)
+			if err != nil {
+				return nil, err
+			}
+
+			if !before.IsZero() {
+				if fileTime.After(latestTime) && fileTime.Before(before) {
+					latestFile = historyFile
+					latestTime = fileTime
+				}
+			} else {
+				if fileTime.After(latestTime) {
+					latestFile = historyFile
+					latestTime = fileTime
+				}
+			}
+		}
+	}
+
+	return latestFile, err
+}
+
+func isHistoryFile(historyFile fs.DirEntry) bool {
+	return !historyFile.IsDir() && strings.HasPrefix(historyFile.Name(), historyNamePrefix)
+}
+
+func getFileDate(historyFile fs.DirEntry) (time.Time, error) {
+	fileDate := strings.TrimPrefix(historyFile.Name(), historyNamePrefix)
+	dateTime, err := time.Parse(time.RFC3339, fileDate)
+	if err != nil {
+		log.Error("unable to parse time from filename %s: %s", historyFile.Name(), err.Error())
+		return time.Time{}, err
+	}
+	return dateTime, err
+}
+
+func (o history) Append(blobsToAppend map[string]string) (map[string]string, error) {
+
+	filename := o.newFileName()
+
+	historyBlobs, err := o.Read()
+	if err != nil {
+		return historyBlobs, err
+	}
+
+	for k, v := range blobsToAppend {
+		historyBlobs[k] = v
+	}
+
+	file, err := o.fileCreator.Create(filename)
+	if err != nil {
+		return historyBlobs, err
+	}
+	defer file.Close()
+
+	writer := bufio.NewWriter(file)
+
+	for blob := range historyBlobs {
+		_, err := writer.WriteString(blob + "\n")
+		if err != nil {
+			log.Error("unable to write to history file: %s", err.Error())
+			return historyBlobs, err
+		}
+	}
+
+	err = writer.Flush()
+	if err != nil {
+		log.Error("unable to flush history file: %s", err.Error())
+		return historyBlobs, err
+	}
+
+	return historyBlobs, err
+
+}
+
+func (o history) newFileName() string {
+	return o.workingDir + historyNamePrefix + time.Now().UTC().Format(time.RFC3339)
+}
+
+func (OSFileCreator) Create(filename string) (io.WriteCloser, error) {
+	file, err := os.Create(filename)
+	if err != nil {
+		log.Error("unable to create file: %s", err.Error())
+	}
+	return file, err
+}

v2/pkg/history/history_test.go

@@ -0,0 +1,154 @@
+package history
+
+import (
+	"bytes"
+	"io"
+	"testing"
+	"time"
+
+	clog "github.com/openshift/oc-mirror/v2/pkg/log"
+	"github.com/stretchr/testify/assert"
+)
+
+type MockFileCreator struct {
+	Buffer *bytes.Buffer
+}
+
+type nopCloser struct {
+	io.Writer
+}
+
+func (m MockFileCreator) Create(name string) (io.WriteCloser, error) {
+	m.Buffer = new(bytes.Buffer)
+	return nopCloser{m.Buffer}, nil
+}
+
+func (nopCloser) Close() error { return nil }
+
+func TestNewHistory(t *testing.T) {
+	history, err := NewHistory(historyFakePath, time.Time{}, clog.New("trace"), MockFileCreator{})
+	assert.NoError(t, err)
+	assert.NotNil(t, history)
+}
+
+func TestRead(t *testing.T) {
+
+	type testCase struct {
+		caseName      string
+		workingDir    string
+		before        time.Time
+		expectedError string
+		expectedHist  map[string]string
+	}
+
+	testCases := []testCase{
+		{
+			caseName:      "valid history file - without specified time",
+			workingDir:    historyFakePath,
+			before:        time.Time{},
+			expectedError: "",
+			expectedHist: map[string]string{
+				"sha256:1dddb0988d16": "",
+				"sha256:3658954f1990": "",
+				"sha256:e3dad360d035": "",
+				"sha256:422e4fbe1ed8": "",
+			},
+		},
+		{
+			caseName:      "valid history file - with specified time",
+			workingDir:    historyFakePath,
+			before:        time.Date(2023, 11, 22, 0, 0, 0, 0, time.UTC),
+			expectedError: "",
+			expectedHist: map[string]string{
+				"sha256:1dddb0988d16": "",
+			},
+		},
+		{
+			caseName:      "invalid working dir",
+			workingDir:    "./invalid-workindir",
+			before:        time.Time{},
+			expectedError: "open ./invalid-workindir: no such file or directory",
+			expectedHist:  nil,
+		},
+	}
+
+	for _, test := range testCases {
+		t.Run(test.caseName, func(t *testing.T) {
+			history, err := NewHistory(test.workingDir, test.before, clog.New("trace"), MockFileCreator{})
+			assert.NoError(t, err)
+
+			historyMap, err := history.Read()
+			if test.expectedError != "" {
+				assert.EqualError(t, err, test.expectedError)
+			}
+			assert.Equal(t, test.expectedHist, historyMap)
+		})
+	}
+}
+
+func TestAppend(t *testing.T) {
+
+	type testCase struct {
+		caseName      string
+		workingDir    string
+		before        time.Time
+		blobsToAppend map[string]string
+		expectedError string
+		expectedHist  map[string]string
+	}
+
+	testCases := []testCase{
+		{
+			caseName:   "valid history file - without specified time",
+			workingDir: historyFakePath,
+			before:     time.Time{},
+			blobsToAppend: map[string]string{
+				"sha256:20f695d2a913": "",
+			},
+			expectedError: "",
+			expectedHist: map[string]string{
+				"sha256:422e4fbe1ed8": "",
+				"sha256:1dddb0988d16": "",
+				"sha256:3658954f1990": "",
+				"sha256:e3dad360d035": "",
+				"sha256:20f695d2a913": "",
+			},
+		},
+		{
+			caseName:   "valid history file - with specified time",
+			workingDir: historyFakePath,
+			before:     time.Date(2023, 11, 22, 0, 0, 0, 0, time.UTC),
+			blobsToAppend: map[string]string{
+				"sha256:20f695d2a913": "",
+			},
+			expectedError: "",
+			expectedHist: map[string]string{
+				"sha256:1dddb0988d16": "",
+				"sha256:20f695d2a913": "",
+			},
+		},
+		{
+			caseName:      "invalid working dir",
+			workingDir:    "./invalid-workindir",
+			before:        time.Time{},
+			expectedError: "open ./invalid-workindir: no such file or directory",
+			expectedHist:  nil,
+		},
+	}
+
+	for _, test := range testCases {
+		t.Run(test.caseName, func(t *testing.T) {
+			history, err := NewHistory(test.workingDir, test.before, clog.New("trace"), MockFileCreator{})
+			assert.NoError(t, err)
+			historyBlobs, err := history.Append(test.blobsToAppend)
+
+			if test.expectedError != "" {
+				assert.EqualError(t, err, test.expectedError)
+			} else {
+				assert.NoError(t, err)
+				assert.Equal(t, test.expectedHist, historyBlobs)
+			}
+		})
+	}
+
+}

v2/pkg/history/interface.go

@@ -0,0 +1,12 @@
+package history
+
+import "io"
+
+type History interface {
+	Read() (map[string]string, error)
+	Append(map[string]string) (map[string]string, error)
+}
+
+type FileCreator interface {
+	Create(name string) (io.WriteCloser, error)
+}

v2/tests/.history-fake/.history-2023-11-21T11:47:46Z

@@ -0,0 +1 @@
+sha256:1dddb0988d16

v2/tests/.history-fake/.history-2023-11-22T11:24:14Z

@@ -0,0 +1,4 @@
+sha256:1dddb0988d16
+sha256:3658954f1990
+sha256:e3dad360d035
+sha256:422e4fbe1ed8

v2/vendor/modules.txt

@@ -308,7 +308,6 @@ github.com/docker/docker/api/types/volume
 github.com/docker/docker/client
 github.com/docker/docker/errdefs
 github.com/docker/docker/pkg/homedir
-github.com/docker/docker/testutil/registry
 # github.com/docker/docker-credential-helpers v0.7.0
 ## explicit; go 1.18
 github.com/docker/docker-credential-helpers/client
@@ -396,13 +395,6 @@ github.com/golang/protobuf/ptypes/timestamp
 # github.com/gomodule/redigo v1.8.2
 ## explicit; go 1.14
 github.com/gomodule/redigo/redis
-# github.com/google/go-cmp v0.5.9
-## explicit; go 1.13
-github.com/google/go-cmp/cmp
-github.com/google/go-cmp/cmp/internal/diff
-github.com/google/go-cmp/cmp/internal/flags
-github.com/google/go-cmp/cmp/internal/function
-github.com/google/go-cmp/cmp/internal/value
 # github.com/google/go-containerregistry v0.15.2
 ## explicit; go 1.18
 github.com/google/go-containerregistry/internal/and
@@ -907,14 +899,6 @@ gopkg.in/yaml.v2
 # gopkg.in/yaml.v3 v3.0.1
 ## explicit
 gopkg.in/yaml.v3
-# gotest.tools/v3 v3.4.0
-## explicit; go 1.13
-gotest.tools/v3/assert
-gotest.tools/v3/assert/cmp
-gotest.tools/v3/internal/assert
-gotest.tools/v3/internal/difflib
-gotest.tools/v3/internal/format
-gotest.tools/v3/internal/source
 # k8s.io/api v0.26.3
 ## explicit; go 1.19
 k8s.io/api/core/v1