From 3282bc02026d2cfbda37b8192e2e96fd329e8d79 Mon Sep 17 00:00:00 2001 From: Adrian Wobito Date: Tue, 11 Apr 2023 15:53:24 -0400 Subject: [PATCH] Preserve time and expose link strategy (#166) * bind file times to metadata Signed-off-by: Adrian Wobito * expose link strategy to walkConditions Signed-off-by: Adrian Wobito * update: test times Signed-off-by: Adrian Wobito * update: link options api Signed-off-by: Adrian Wobito * stub mod time for dynamic tar test fixtures Signed-off-by: Alex Goodman * set mtime on tar fixture explicitly Signed-off-by: Alex Goodman * always interpret tar header timestamps as UTC Signed-off-by: Alex Goodman * interpret all file metadata timestamps as UTC Signed-off-by: Alex Goodman --------- Signed-off-by: Adrian Wobito Signed-off-by: Alex Goodman Co-authored-by: Alex Goodman --- pkg/file/metadata.go | 23 +++++++++++------ pkg/file/metadata_test.go | 25 ++++++++++++------- pkg/file/tarutil_test.go | 7 ++++++ .../test-fixtures/generators/fixture-1.sh | 2 +- pkg/filetree/depth_first_path_walker.go | 20 ++++++++++----- pkg/filetree/depth_first_path_walker_test.go | 5 ++-- pkg/filetree/filetree_test.go | 3 ++- pkg/filetree/search_test.go | 3 ++- pkg/image/file_catalog_test.go | 19 +++++++------- 9 files changed, 71 insertions(+), 36 deletions(-) diff --git a/pkg/file/metadata.go b/pkg/file/metadata.go index 990e7a4f..90c6d764 100644 --- a/pkg/file/metadata.go +++ b/pkg/file/metadata.go @@ -6,6 +6,7 @@ import ( "os" "path" "path/filepath" + "time" "github.com/anchore/stereoscope/internal/log" @@ -19,13 +20,16 @@ type Metadata struct { // LinkDestination is populated only for hardlinks / symlinks, can be an absolute or relative LinkDestination string // Size of the file in bytes - Size int64 - UserID int - GroupID int - Type Type - IsDir bool - Mode os.FileMode - MIMEType string + Size int64 + UserID int + GroupID int + Type Type + IsDir bool + Mode os.FileMode + MIMEType string + ModTime time.Time + AccessTime time.Time + ChangeTime time.Time } func NewMetadata(header tar.Header, content io.Reader) Metadata { @@ -38,6 +42,9 @@ func NewMetadata(header tar.Header, content io.Reader) Metadata { UserID: header.Uid, GroupID: header.Gid, IsDir: header.FileInfo().IsDir(), + ModTime: header.ModTime.UTC(), + AccessTime: header.AccessTime.UTC(), + ChangeTime: header.ChangeTime.UTC(), MIMEType: MIMEType(content), } } @@ -79,6 +86,7 @@ func NewMetadataFromSquashFSFile(path string, f *squashfs.File) (Metadata, error Size: fi.Size(), IsDir: f.IsDir(), Mode: fi.Mode(), + ModTime: fi.ModTime().UTC(), Type: ty, } @@ -121,5 +129,6 @@ func NewMetadataFromPath(path string, info os.FileInfo) Metadata { Size: info.Size(), MIMEType: mimeType, IsDir: info.IsDir(), + ModTime: info.ModTime().UTC(), } } diff --git a/pkg/file/metadata_test.go b/pkg/file/metadata_test.go index aa60bc95..7d043f35 100644 --- a/pkg/file/metadata_test.go +++ b/pkg/file/metadata_test.go @@ -4,12 +4,14 @@ package file import ( - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" "io" "os" "strings" "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/go-test/deep" ) @@ -18,13 +20,13 @@ func TestFileMetadataFromTar(t *testing.T) { tarReader := getTarFixture(t, "fixture-1") expected := []Metadata{ - {Path: "/path", Type: TypeDirectory, LinkDestination: "", Size: 0, Mode: os.ModeDir | 0o755, UserID: 1337, GroupID: 5432, IsDir: true, MIMEType: ""}, - {Path: "/path/branch", Type: TypeDirectory, LinkDestination: "", Size: 0, Mode: os.ModeDir | 0o755, UserID: 1337, GroupID: 5432, IsDir: true, MIMEType: ""}, - {Path: "/path/branch/one", Type: TypeDirectory, LinkDestination: "", Size: 0, Mode: os.ModeDir | 0o700, UserID: 1337, GroupID: 5432, IsDir: true, MIMEType: ""}, - {Path: "/path/branch/one/file-1.txt", Type: TypeRegular, LinkDestination: "", Size: 11, Mode: 0o700, UserID: 1337, GroupID: 5432, IsDir: false, MIMEType: "text/plain"}, - {Path: "/path/branch/two", Type: TypeDirectory, LinkDestination: "", Size: 0, Mode: os.ModeDir | 0o755, UserID: 1337, GroupID: 5432, IsDir: true, MIMEType: ""}, - {Path: "/path/branch/two/file-2.txt", Type: TypeRegular, LinkDestination: "", Size: 12, Mode: 0o755, UserID: 1337, GroupID: 5432, IsDir: false, MIMEType: "text/plain"}, - {Path: "/path/file-3.txt", Type: TypeRegular, LinkDestination: "", Size: 11, Mode: 0o664, UserID: 1337, GroupID: 5432, IsDir: false, MIMEType: "text/plain"}, + {Path: "/path", Type: TypeDirectory, LinkDestination: "", Size: 0, Mode: os.ModeDir | 0o755, UserID: 1337, GroupID: 5432, IsDir: true, MIMEType: "", ModTime: time.Time{}, AccessTime: time.Time{}, ChangeTime: time.Time{}}, + {Path: "/path/branch", Type: TypeDirectory, LinkDestination: "", Size: 0, Mode: os.ModeDir | 0o755, UserID: 1337, GroupID: 5432, IsDir: true, MIMEType: "", ModTime: time.Time{}, AccessTime: time.Time{}, ChangeTime: time.Time{}}, + {Path: "/path/branch/one", Type: TypeDirectory, LinkDestination: "", Size: 0, Mode: os.ModeDir | 0o700, UserID: 1337, GroupID: 5432, IsDir: true, MIMEType: "", ModTime: time.Time{}, AccessTime: time.Time{}, ChangeTime: time.Time{}}, + {Path: "/path/branch/one/file-1.txt", Type: TypeRegular, LinkDestination: "", Size: 11, Mode: 0o700, UserID: 1337, GroupID: 5432, IsDir: false, MIMEType: "text/plain", ModTime: time.Time{}, AccessTime: time.Time{}, ChangeTime: time.Time{}}, + {Path: "/path/branch/two", Type: TypeDirectory, LinkDestination: "", Size: 0, Mode: os.ModeDir | 0o755, UserID: 1337, GroupID: 5432, IsDir: true, MIMEType: "", ModTime: time.Time{}, AccessTime: time.Time{}, ChangeTime: time.Time{}}, + {Path: "/path/branch/two/file-2.txt", Type: TypeRegular, LinkDestination: "", Size: 12, Mode: 0o755, UserID: 1337, GroupID: 5432, IsDir: false, MIMEType: "text/plain", ModTime: time.Time{}, AccessTime: time.Time{}, ChangeTime: time.Time{}}, + {Path: "/path/file-3.txt", Type: TypeRegular, LinkDestination: "", Size: 11, Mode: 0o664, UserID: 1337, GroupID: 5432, IsDir: false, MIMEType: "text/plain", ModTime: time.Time{}, AccessTime: time.Time{}, ChangeTime: time.Time{}}, } var actual []Metadata @@ -33,6 +35,11 @@ func TestFileMetadataFromTar(t *testing.T) { if strings.HasSuffix(entry.Header.Name, ".txt") { contents = strings.NewReader("#!/usr/bin/env bash\necho 'awesome script'") } + + entry.Header.ModTime = time.Time{} + entry.Header.ChangeTime = time.Time{} + entry.Header.AccessTime = time.Time{} + actual = append(actual, NewMetadata(entry.Header, contents)) return nil } diff --git a/pkg/file/tarutil_test.go b/pkg/file/tarutil_test.go index 2103f242..e9488c04 100644 --- a/pkg/file/tarutil_test.go +++ b/pkg/file/tarutil_test.go @@ -12,6 +12,7 @@ import ( "path" "path/filepath" "testing" + "time" "github.com/stretchr/testify/assert" ) @@ -71,6 +72,9 @@ func TestMetadataFromTar(t *testing.T) { IsDir: false, Mode: 0x1ed, MIMEType: "application/octet-stream", + ModTime: time.Date(2019, time.September, 16, 0, 0, 0, 0, time.UTC), + AccessTime: time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC), + ChangeTime: time.Time{}, }, }, { @@ -86,6 +90,9 @@ func TestMetadataFromTar(t *testing.T) { IsDir: true, Mode: 0x800001ed, MIMEType: "", + ModTime: time.Date(2019, time.September, 16, 0, 0, 0, 0, time.UTC), + AccessTime: time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC), + ChangeTime: time.Time{}, }, }, } diff --git a/pkg/file/test-fixtures/generators/fixture-1.sh b/pkg/file/test-fixtures/generators/fixture-1.sh index ac79a48c..09349ed7 100755 --- a/pkg/file/test-fixtures/generators/fixture-1.sh +++ b/pkg/file/test-fixtures/generators/fixture-1.sh @@ -34,7 +34,7 @@ pushd /tmp/stereoscope # tar + owner # note: sort by name is important for test file header entry ordering - tar --sort=name --owner=1337 --group=5432 -cvf "/scratch/${FIXTURE_NAME}" path/ + tar --sort=name --owner=1337 --group=5432 --mtime='UTC 2019-09-16' -cvf "/scratch/${FIXTURE_NAME}" path/ popd EOF diff --git a/pkg/filetree/depth_first_path_walker.go b/pkg/filetree/depth_first_path_walker.go index 71d3d7f4..9b542185 100644 --- a/pkg/filetree/depth_first_path_walker.go +++ b/pkg/filetree/depth_first_path_walker.go @@ -31,6 +31,8 @@ type WalkConditions struct { // Whether we should consider children of this Node to be included in the traversal path. // Return true to traverse children of this Node. ShouldContinueBranch func(file.Path, filenode.FileNode) bool + + LinkOptions []LinkResolutionOption } // DepthFirstPathWalker implements stateful depth-first Tree traversal. @@ -64,17 +66,23 @@ func (w *DepthFirstPathWalker) Walk(from file.Path) (file.Path, *filenode.FileNo err error ) + linkOpts := []LinkResolutionOption{followAncestorLinks} + // Setup link options defaults + if w.conditions.LinkOptions == nil { + linkOpts = []LinkResolutionOption{followAncestorLinks, DoNotFollowDeadBasenameLinks, FollowBasenameLinks} + } + + linkOpts = append(linkOpts, w.conditions.LinkOptions...) + linkStrat := newLinkResolutionStrategy(linkOpts...) + for w.pathStack.Size() > 0 { currentPath = w.pathStack.Pop() - // TODO: should we make these link resolutions configurable so you can observe the links on walk as well? (take link resolution options as a parameter) - currentNode, err = w.tree.node(currentPath, linkResolutionStrategy{ - FollowAncestorLinks: true, - FollowBasenameLinks: true, - DoNotFollowDeadBasenameLinks: true, - }) + + currentNode, err = w.tree.node(currentPath, linkStrat) if err != nil { return "", nil, err } + if !currentNode.HasFileNode() { return "", nil, fmt.Errorf("nil Node at path=%q", currentPath) } diff --git a/pkg/filetree/depth_first_path_walker_test.go b/pkg/filetree/depth_first_path_walker_test.go index 74678709..9db3fd3c 100644 --- a/pkg/filetree/depth_first_path_walker_test.go +++ b/pkg/filetree/depth_first_path_walker_test.go @@ -2,11 +2,12 @@ package filetree import ( "errors" + "strings" + "testing" + "github.com/anchore/stereoscope/pkg/file" "github.com/anchore/stereoscope/pkg/filetree/filenode" "github.com/go-test/deep" - "strings" - "testing" ) func dfsTestTree(t *testing.T) (*FileTree, map[string]*file.Reference) { diff --git a/pkg/filetree/filetree_test.go b/pkg/filetree/filetree_test.go index a6aac4a7..178ca174 100644 --- a/pkg/filetree/filetree_test.go +++ b/pkg/filetree/filetree_test.go @@ -3,9 +3,10 @@ package filetree import ( "errors" "fmt" - "github.com/scylladb/go-set/strset" "testing" + "github.com/scylladb/go-set/strset" + "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" "github.com/stretchr/testify/require" diff --git a/pkg/filetree/search_test.go b/pkg/filetree/search_test.go index 6033646c..73fcc334 100644 --- a/pkg/filetree/search_test.go +++ b/pkg/filetree/search_test.go @@ -2,13 +2,14 @@ package filetree import ( "fmt" + "testing" + "github.com/anchore/stereoscope/pkg/file" "github.com/anchore/stereoscope/pkg/filetree/filenode" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "testing" ) func Test_searchContext_SearchByPath(t *testing.T) { diff --git a/pkg/image/file_catalog_test.go b/pkg/image/file_catalog_test.go index a6100156..bcf6cab4 100644 --- a/pkg/image/file_catalog_test.go +++ b/pkg/image/file_catalog_test.go @@ -6,11 +6,6 @@ package image import ( "crypto/sha256" "fmt" - "github.com/anchore/stereoscope/pkg/filetree" - "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" "io" "os" "os/exec" @@ -19,6 +14,12 @@ import ( "strings" "testing" + "github.com/anchore/stereoscope/pkg/filetree" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/go-test/deep" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/types" @@ -359,7 +360,7 @@ func TestFileCatalog_GetByExtension(t *testing.T) { if d := cmp.Diff(tt.want, actual, cmpopts.EquateEmpty(), cmpopts.IgnoreUnexported(file.Reference{}), - cmpopts.IgnoreFields(file.Metadata{}, "Mode", "GroupID", "UserID", "Size"), + cmpopts.IgnoreFields(file.Metadata{}, "Mode", "GroupID", "UserID", "Size", "ModTime", "AccessTime", "ChangeTime"), ); d != "" { t.Errorf("diff: %s", d) } @@ -461,7 +462,7 @@ func TestFileCatalog_GetByBasename(t *testing.T) { if d := cmp.Diff(tt.want, actual, cmpopts.EquateEmpty(), cmpopts.IgnoreUnexported(file.Reference{}), - cmpopts.IgnoreFields(file.Metadata{}, "Mode", "GroupID", "UserID", "Size"), + cmpopts.IgnoreFields(file.Metadata{}, "Mode", "GroupID", "UserID", "Size", "ModTime", "AccessTime", "ChangeTime"), ); d != "" { t.Errorf("diff: %s", d) } @@ -571,7 +572,7 @@ func TestFileCatalog_GetByBasenameGlob(t *testing.T) { if d := cmp.Diff(tt.want, actual, cmpopts.EquateEmpty(), cmpopts.IgnoreUnexported(file.Reference{}), - cmpopts.IgnoreFields(file.Metadata{}, "Mode", "GroupID", "UserID", "Size"), + cmpopts.IgnoreFields(file.Metadata{}, "Mode", "GroupID", "UserID", "Size", "ModTime", "AccessTime", "ChangeTime"), ); d != "" { t.Errorf("diff: %s", d) } @@ -672,7 +673,7 @@ func TestFileCatalog_GetByMimeType(t *testing.T) { if d := cmp.Diff(tt.want, actual, cmpopts.EquateEmpty(), cmpopts.IgnoreUnexported(file.Reference{}), - cmpopts.IgnoreFields(file.Metadata{}, "Mode", "GroupID", "UserID", "Size"), + cmpopts.IgnoreFields(file.Metadata{}, "Mode", "GroupID", "UserID", "Size", "ModTime", "AccessTime", "ChangeTime"), ); d != "" { t.Errorf("diff: %s", d) }