New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
CFE-977, CFE-978, CFE-982: Introduce history interface with its Reader and Append implementation #736
Merged
Merged
CFE-977, CFE-978, CFE-982: Introduce history interface with its Reader and Append implementation #736
Changes from all commits
Commits
File filter
Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,8 @@ | ||
package history | ||
|
||
const ( | ||
//TODO Alex during the integration of the building blocks, add the historyPath after the workingDir when NewHistory is called | ||
historyPath = ".history/" | ||
historyNamePrefix = ".history-" | ||
historyFakePath = "../../tests/.history-fake/" | ||
) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,163 @@ | ||
package history | ||
|
||
import ( | ||
"bufio" | ||
"io" | ||
"io/fs" | ||
"os" | ||
"strings" | ||
"time" | ||
|
||
clog "github.com/openshift/oc-mirror/v2/pkg/log" | ||
) | ||
|
||
var log clog.PluggableLoggerInterface | ||
|
||
type OSFileCreator struct{} | ||
type history struct { | ||
workingDir string | ||
before time.Time | ||
fileCreator FileCreator | ||
} | ||
|
||
func NewHistory(workingDir string, before time.Time, logg clog.PluggableLoggerInterface, fileCreator FileCreator) (History, error) { | ||
if logg == nil { | ||
log = clog.New("error") | ||
} else { | ||
log = logg | ||
} | ||
|
||
return history{ | ||
workingDir: workingDir, | ||
before: before, | ||
fileCreator: fileCreator, | ||
}, nil | ||
} | ||
|
||
func (o history) Read() (map[string]string, error) { | ||
historyFile, err := o.getHistoryFile(o.before) | ||
if err != nil { | ||
return nil, err | ||
} | ||
|
||
file, err := os.Open(o.workingDir + historyFile.Name()) | ||
if err != nil { | ||
log.Error("unable to open history file: %s", err.Error()) | ||
return nil, err | ||
} | ||
defer file.Close() | ||
|
||
scanner := bufio.NewScanner(file) | ||
historyMap := make(map[string]string) | ||
|
||
for scanner.Scan() { | ||
blob := scanner.Text() | ||
historyMap[blob] = "" | ||
} | ||
|
||
if err := scanner.Err(); err != nil { | ||
log.Error("unable to read history file: %s", err.Error()) | ||
return nil, err | ||
} | ||
|
||
return historyMap, nil | ||
} | ||
|
||
func (o history) getHistoryFile(before time.Time) (fs.DirEntry, error) { | ||
historyFiles, err := os.ReadDir(o.workingDir) | ||
if err != nil { | ||
log.Error("unable to read history directory: %s", err.Error()) | ||
return nil, err | ||
} | ||
|
||
var latestFile fs.DirEntry | ||
var latestTime time.Time | ||
|
||
for _, historyFile := range historyFiles { | ||
if isHistoryFile(historyFile) { | ||
fileTime, err := getFileDate(historyFile) | ||
if err != nil { | ||
return nil, err | ||
} | ||
|
||
if !before.IsZero() { | ||
if fileTime.After(latestTime) && fileTime.Before(before) { | ||
latestFile = historyFile | ||
latestTime = fileTime | ||
} | ||
} else { | ||
if fileTime.After(latestTime) { | ||
latestFile = historyFile | ||
latestTime = fileTime | ||
} | ||
} | ||
} | ||
} | ||
|
||
return latestFile, err | ||
} | ||
|
||
func isHistoryFile(historyFile fs.DirEntry) bool { | ||
return !historyFile.IsDir() && strings.HasPrefix(historyFile.Name(), historyNamePrefix) | ||
} | ||
|
||
func getFileDate(historyFile fs.DirEntry) (time.Time, error) { | ||
fileDate := strings.TrimPrefix(historyFile.Name(), historyNamePrefix) | ||
dateTime, err := time.Parse(time.RFC3339, fileDate) | ||
if err != nil { | ||
log.Error("unable to parse time from filename %s: %s", historyFile.Name(), err.Error()) | ||
return time.Time{}, err | ||
} | ||
return dateTime, err | ||
} | ||
|
||
func (o history) Append(blobsToAppend map[string]string) (map[string]string, error) { | ||
|
||
filename := o.newFileName() | ||
|
||
historyBlobs, err := o.Read() | ||
if err != nil { | ||
return historyBlobs, err | ||
} | ||
|
||
for k, v := range blobsToAppend { | ||
historyBlobs[k] = v | ||
} | ||
|
||
file, err := o.fileCreator.Create(filename) | ||
if err != nil { | ||
return historyBlobs, err | ||
} | ||
defer file.Close() | ||
|
||
writer := bufio.NewWriter(file) | ||
|
||
for blob := range historyBlobs { | ||
_, err := writer.WriteString(blob + "\n") | ||
if err != nil { | ||
log.Error("unable to write to history file: %s", err.Error()) | ||
return historyBlobs, err | ||
} | ||
} | ||
|
||
err = writer.Flush() | ||
if err != nil { | ||
log.Error("unable to flush history file: %s", err.Error()) | ||
return historyBlobs, err | ||
} | ||
|
||
return historyBlobs, err | ||
|
||
} | ||
|
||
func (o history) newFileName() string { | ||
return o.workingDir + historyNamePrefix + time.Now().UTC().Format(time.RFC3339) | ||
} | ||
|
||
func (OSFileCreator) Create(filename string) (io.WriteCloser, error) { | ||
file, err := os.Create(filename) | ||
if err != nil { | ||
log.Error("unable to create file: %s", err.Error()) | ||
} | ||
return file, err | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,154 @@ | ||
package history | ||
|
||
import ( | ||
"bytes" | ||
"io" | ||
"testing" | ||
"time" | ||
|
||
clog "github.com/openshift/oc-mirror/v2/pkg/log" | ||
"github.com/stretchr/testify/assert" | ||
) | ||
|
||
type MockFileCreator struct { | ||
Buffer *bytes.Buffer | ||
} | ||
|
||
type nopCloser struct { | ||
io.Writer | ||
} | ||
|
||
func (m MockFileCreator) Create(name string) (io.WriteCloser, error) { | ||
m.Buffer = new(bytes.Buffer) | ||
return nopCloser{m.Buffer}, nil | ||
} | ||
Comment on lines
+21
to
+24
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I like the idea! no tmp folder, no files on disk! |
||
|
||
func (nopCloser) Close() error { return nil } | ||
|
||
func TestNewHistory(t *testing.T) { | ||
history, err := NewHistory(historyFakePath, time.Time{}, clog.New("trace"), MockFileCreator{}) | ||
assert.NoError(t, err) | ||
assert.NotNil(t, history) | ||
} | ||
|
||
func TestRead(t *testing.T) { | ||
|
||
type testCase struct { | ||
caseName string | ||
workingDir string | ||
before time.Time | ||
expectedError string | ||
expectedHist map[string]string | ||
} | ||
|
||
testCases := []testCase{ | ||
{ | ||
caseName: "valid history file - without specified time", | ||
workingDir: historyFakePath, | ||
before: time.Time{}, | ||
expectedError: "", | ||
expectedHist: map[string]string{ | ||
"sha256:1dddb0988d16": "", | ||
"sha256:3658954f1990": "", | ||
"sha256:e3dad360d035": "", | ||
"sha256:422e4fbe1ed8": "", | ||
}, | ||
}, | ||
{ | ||
caseName: "valid history file - with specified time", | ||
workingDir: historyFakePath, | ||
before: time.Date(2023, 11, 22, 0, 0, 0, 0, time.UTC), | ||
expectedError: "", | ||
expectedHist: map[string]string{ | ||
"sha256:1dddb0988d16": "", | ||
}, | ||
}, | ||
{ | ||
caseName: "invalid working dir", | ||
workingDir: "./invalid-workindir", | ||
before: time.Time{}, | ||
expectedError: "open ./invalid-workindir: no such file or directory", | ||
expectedHist: nil, | ||
}, | ||
} | ||
|
||
for _, test := range testCases { | ||
t.Run(test.caseName, func(t *testing.T) { | ||
history, err := NewHistory(test.workingDir, test.before, clog.New("trace"), MockFileCreator{}) | ||
assert.NoError(t, err) | ||
|
||
historyMap, err := history.Read() | ||
if test.expectedError != "" { | ||
assert.EqualError(t, err, test.expectedError) | ||
} | ||
assert.Equal(t, test.expectedHist, historyMap) | ||
}) | ||
} | ||
} | ||
|
||
func TestAppend(t *testing.T) { | ||
|
||
type testCase struct { | ||
caseName string | ||
workingDir string | ||
before time.Time | ||
blobsToAppend map[string]string | ||
expectedError string | ||
expectedHist map[string]string | ||
} | ||
|
||
testCases := []testCase{ | ||
{ | ||
caseName: "valid history file - without specified time", | ||
workingDir: historyFakePath, | ||
before: time.Time{}, | ||
blobsToAppend: map[string]string{ | ||
"sha256:20f695d2a913": "", | ||
}, | ||
expectedError: "", | ||
expectedHist: map[string]string{ | ||
"sha256:422e4fbe1ed8": "", | ||
"sha256:1dddb0988d16": "", | ||
"sha256:3658954f1990": "", | ||
"sha256:e3dad360d035": "", | ||
"sha256:20f695d2a913": "", | ||
}, | ||
}, | ||
{ | ||
caseName: "valid history file - with specified time", | ||
workingDir: historyFakePath, | ||
before: time.Date(2023, 11, 22, 0, 0, 0, 0, time.UTC), | ||
blobsToAppend: map[string]string{ | ||
"sha256:20f695d2a913": "", | ||
}, | ||
expectedError: "", | ||
expectedHist: map[string]string{ | ||
"sha256:1dddb0988d16": "", | ||
"sha256:20f695d2a913": "", | ||
}, | ||
}, | ||
{ | ||
caseName: "invalid working dir", | ||
workingDir: "./invalid-workindir", | ||
before: time.Time{}, | ||
expectedError: "open ./invalid-workindir: no such file or directory", | ||
expectedHist: nil, | ||
}, | ||
} | ||
|
||
for _, test := range testCases { | ||
t.Run(test.caseName, func(t *testing.T) { | ||
history, err := NewHistory(test.workingDir, test.before, clog.New("trace"), MockFileCreator{}) | ||
assert.NoError(t, err) | ||
historyBlobs, err := history.Append(test.blobsToAppend) | ||
|
||
if test.expectedError != "" { | ||
assert.EqualError(t, err, test.expectedError) | ||
} else { | ||
assert.NoError(t, err) | ||
assert.Equal(t, test.expectedHist, historyBlobs) | ||
} | ||
}) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Any way you could also compare the new history content to expectedHist? |
||
} | ||
|
||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,12 @@ | ||
package history | ||
|
||
import "io" | ||
|
||
type History interface { | ||
Read() (map[string]string, error) | ||
Append(map[string]string) (map[string]string, error) | ||
} | ||
|
||
type FileCreator interface { | ||
Create(name string) (io.WriteCloser, error) | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
sha256:1dddb0988d16 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,4 @@ | ||
sha256:1dddb0988d16 | ||
sha256:3658954f1990 | ||
sha256:e3dad360d035 | ||
sha256:422e4fbe1ed8 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I like this idea!