Skip to content

Commit

Permalink
Preserve time and expose link strategy (#166)
Browse files Browse the repository at this point in the history 
* bind file times to metadata

Signed-off-by: Adrian Wobito <adrian.wobito@gmail.com>

* expose link strategy to walkConditions

Signed-off-by: Adrian Wobito <adrian.wobito@gmail.com>

* update: test times

Signed-off-by: Adrian Wobito <adrian.wobito@gmail.com>

* update: link options api

Signed-off-by: Adrian Wobito <adrian.wobito@gmail.com>

* stub mod time for dynamic tar test fixtures

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* set mtime on tar fixture explicitly

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* always interpret tar header timestamps as UTC

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* interpret all file metadata timestamps as UTC

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

---------

Signed-off-by: Adrian Wobito <adrian.wobito@gmail.com>
Signed-off-by: Alex Goodman <alex.goodman@anchore.com>
Co-authored-by: Alex Goodman <alex.goodman@anchore.com>
  • Loading branch information
@wobito @wagoodman
wobito and wagoodman committed Apr 11, 2023
1 parent e95d60a commit 3282bc0
Show file tree
Hide file tree
Showing 9 changed files with 71 additions and 36 deletions.
23 changes: 16 additions & 7 deletions pkg/file/metadata.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"os"
"path"
"path/filepath"
"time"

"github.com/anchore/stereoscope/internal/log"

Expand All @@ -19,13 +20,16 @@ type Metadata struct {
// LinkDestination is populated only for hardlinks / symlinks, can be an absolute or relative
LinkDestination string
// Size of the file in bytes
Size int64
UserID int
GroupID int
Type Type
IsDir bool
Mode os.FileMode
MIMEType string
Size int64
UserID int
GroupID int
Type Type
IsDir bool
Mode os.FileMode
MIMEType string
ModTime time.Time
AccessTime time.Time
ChangeTime time.Time
}

func NewMetadata(header tar.Header, content io.Reader) Metadata {
Expand All @@ -38,6 +42,9 @@ func NewMetadata(header tar.Header, content io.Reader) Metadata {
UserID: header.Uid,
GroupID: header.Gid,
IsDir: header.FileInfo().IsDir(),
ModTime: header.ModTime.UTC(),
AccessTime: header.AccessTime.UTC(),
ChangeTime: header.ChangeTime.UTC(),
MIMEType: MIMEType(content),
}
}
Expand Down Expand Up @@ -79,6 +86,7 @@ func NewMetadataFromSquashFSFile(path string, f *squashfs.File) (Metadata, error
Size: fi.Size(),
IsDir: f.IsDir(),
Mode: fi.Mode(),
ModTime: fi.ModTime().UTC(),
Type: ty,
}

Expand Down Expand Up @@ -121,5 +129,6 @@ func NewMetadataFromPath(path string, info os.FileInfo) Metadata {
Size: info.Size(),
MIMEType: mimeType,
IsDir: info.IsDir(),
ModTime: info.ModTime().UTC(),
}
}
25 changes: 16 additions & 9 deletions pkg/file/metadata_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,14 @@
package file

import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"io"
"os"
"strings"
"testing"
"time"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"

"github.com/go-test/deep"
)
Expand All @@ -18,13 +20,13 @@ func TestFileMetadataFromTar(t *testing.T) {
tarReader := getTarFixture(t, "fixture-1")

expected := []Metadata{
{Path: "/path", Type: TypeDirectory, LinkDestination: "", Size: 0, Mode: os.ModeDir | 0o755, UserID: 1337, GroupID: 5432, IsDir: true, MIMEType: ""},
{Path: "/path/branch", Type: TypeDirectory, LinkDestination: "", Size: 0, Mode: os.ModeDir | 0o755, UserID: 1337, GroupID: 5432, IsDir: true, MIMEType: ""},
{Path: "/path/branch/one", Type: TypeDirectory, LinkDestination: "", Size: 0, Mode: os.ModeDir | 0o700, UserID: 1337, GroupID: 5432, IsDir: true, MIMEType: ""},
{Path: "/path/branch/one/file-1.txt", Type: TypeRegular, LinkDestination: "", Size: 11, Mode: 0o700, UserID: 1337, GroupID: 5432, IsDir: false, MIMEType: "text/plain"},
{Path: "/path/branch/two", Type: TypeDirectory, LinkDestination: "", Size: 0, Mode: os.ModeDir | 0o755, UserID: 1337, GroupID: 5432, IsDir: true, MIMEType: ""},
{Path: "/path/branch/two/file-2.txt", Type: TypeRegular, LinkDestination: "", Size: 12, Mode: 0o755, UserID: 1337, GroupID: 5432, IsDir: false, MIMEType: "text/plain"},
{Path: "/path/file-3.txt", Type: TypeRegular, LinkDestination: "", Size: 11, Mode: 0o664, UserID: 1337, GroupID: 5432, IsDir: false, MIMEType: "text/plain"},
{Path: "/path", Type: TypeDirectory, LinkDestination: "", Size: 0, Mode: os.ModeDir | 0o755, UserID: 1337, GroupID: 5432, IsDir: true, MIMEType: "", ModTime: time.Time{}, AccessTime: time.Time{}, ChangeTime: time.Time{}},
{Path: "/path/branch", Type: TypeDirectory, LinkDestination: "", Size: 0, Mode: os.ModeDir | 0o755, UserID: 1337, GroupID: 5432, IsDir: true, MIMEType: "", ModTime: time.Time{}, AccessTime: time.Time{}, ChangeTime: time.Time{}},
{Path: "/path/branch/one", Type: TypeDirectory, LinkDestination: "", Size: 0, Mode: os.ModeDir | 0o700, UserID: 1337, GroupID: 5432, IsDir: true, MIMEType: "", ModTime: time.Time{}, AccessTime: time.Time{}, ChangeTime: time.Time{}},
{Path: "/path/branch/one/file-1.txt", Type: TypeRegular, LinkDestination: "", Size: 11, Mode: 0o700, UserID: 1337, GroupID: 5432, IsDir: false, MIMEType: "text/plain", ModTime: time.Time{}, AccessTime: time.Time{}, ChangeTime: time.Time{}},
{Path: "/path/branch/two", Type: TypeDirectory, LinkDestination: "", Size: 0, Mode: os.ModeDir | 0o755, UserID: 1337, GroupID: 5432, IsDir: true, MIMEType: "", ModTime: time.Time{}, AccessTime: time.Time{}, ChangeTime: time.Time{}},
{Path: "/path/branch/two/file-2.txt", Type: TypeRegular, LinkDestination: "", Size: 12, Mode: 0o755, UserID: 1337, GroupID: 5432, IsDir: false, MIMEType: "text/plain", ModTime: time.Time{}, AccessTime: time.Time{}, ChangeTime: time.Time{}},
{Path: "/path/file-3.txt", Type: TypeRegular, LinkDestination: "", Size: 11, Mode: 0o664, UserID: 1337, GroupID: 5432, IsDir: false, MIMEType: "text/plain", ModTime: time.Time{}, AccessTime: time.Time{}, ChangeTime: time.Time{}},
}

var actual []Metadata
Expand All @@ -33,6 +35,11 @@ func TestFileMetadataFromTar(t *testing.T) {
if strings.HasSuffix(entry.Header.Name, ".txt") {
contents = strings.NewReader("#!/usr/bin/env bash\necho 'awesome script'")
}

entry.Header.ModTime = time.Time{}
entry.Header.ChangeTime = time.Time{}
entry.Header.AccessTime = time.Time{}

actual = append(actual, NewMetadata(entry.Header, contents))
return nil
}
Expand Down
7 changes: 7 additions & 0 deletions pkg/file/tarutil_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
"path"
"path/filepath"
"testing"
"time"

"github.com/stretchr/testify/assert"
)
Expand Down Expand Up @@ -71,6 +72,9 @@ func TestMetadataFromTar(t *testing.T) {
IsDir: false,
Mode: 0x1ed,
MIMEType: "application/octet-stream",
ModTime: time.Date(2019, time.September, 16, 0, 0, 0, 0, time.UTC),
AccessTime: time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC),
ChangeTime: time.Time{},
},
},
{
Expand All @@ -86,6 +90,9 @@ func TestMetadataFromTar(t *testing.T) {
IsDir: true,
Mode: 0x800001ed,
MIMEType: "",
ModTime: time.Date(2019, time.September, 16, 0, 0, 0, 0, time.UTC),
AccessTime: time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC),
ChangeTime: time.Time{},
},
},
}
Expand Down
2 changes: 1 addition & 1 deletion pkg/file/test-fixtures/generators/fixture-1.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ pushd /tmp/stereoscope
# tar + owner
# note: sort by name is important for test file header entry ordering
tar --sort=name --owner=1337 --group=5432 -cvf "/scratch/${FIXTURE_NAME}" path/
tar --sort=name --owner=1337 --group=5432 --mtime='UTC 2019-09-16' -cvf "/scratch/${FIXTURE_NAME}" path/
popd
EOF
20 changes: 14 additions & 6 deletions pkg/filetree/depth_first_path_walker.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@ type WalkConditions struct {
// Whether we should consider children of this Node to be included in the traversal path.
// Return true to traverse children of this Node.
ShouldContinueBranch func(file.Path, filenode.FileNode) bool

LinkOptions []LinkResolutionOption
}

// DepthFirstPathWalker implements stateful depth-first Tree traversal.
Expand Down Expand Up @@ -64,17 +66,23 @@ func (w *DepthFirstPathWalker) Walk(from file.Path) (file.Path, *filenode.FileNo
err error
)

linkOpts := []LinkResolutionOption{followAncestorLinks}
// Setup link options defaults
if w.conditions.LinkOptions == nil {
linkOpts = []LinkResolutionOption{followAncestorLinks, DoNotFollowDeadBasenameLinks, FollowBasenameLinks}
}

linkOpts = append(linkOpts, w.conditions.LinkOptions...)
linkStrat := newLinkResolutionStrategy(linkOpts...)

for w.pathStack.Size() > 0 {
currentPath = w.pathStack.Pop()
// TODO: should we make these link resolutions configurable so you can observe the links on walk as well? (take link resolution options as a parameter)
currentNode, err = w.tree.node(currentPath, linkResolutionStrategy{
FollowAncestorLinks: true,
FollowBasenameLinks: true,
DoNotFollowDeadBasenameLinks: true,
})

currentNode, err = w.tree.node(currentPath, linkStrat)
if err != nil {
return "", nil, err
}

if !currentNode.HasFileNode() {
return "", nil, fmt.Errorf("nil Node at path=%q", currentPath)
}
Expand Down
5 changes: 3 additions & 2 deletions pkg/filetree/depth_first_path_walker_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,12 @@ package filetree

import (
"errors"
"strings"
"testing"

"github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/stereoscope/pkg/filetree/filenode"
"github.com/go-test/deep"
"strings"
"testing"
)

func dfsTestTree(t *testing.T) (*FileTree, map[string]*file.Reference) {
Expand Down
3 changes: 2 additions & 1 deletion pkg/filetree/filetree_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,10 @@ package filetree
import (
"errors"
"fmt"
"github.com/scylladb/go-set/strset"
"testing"

"github.com/scylladb/go-set/strset"

"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/stretchr/testify/require"
Expand Down
3 changes: 2 additions & 1 deletion pkg/filetree/search_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,14 @@ package filetree

import (
"fmt"
"testing"

"github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/stereoscope/pkg/filetree/filenode"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"testing"
)

func Test_searchContext_SearchByPath(t *testing.T) {
Expand Down
19 changes: 10 additions & 9 deletions pkg/image/file_catalog_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,6 @@ package image
import (
"crypto/sha256"
"fmt"
"github.com/anchore/stereoscope/pkg/filetree"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"io"
"os"
"os/exec"
Expand All @@ -19,6 +14,12 @@ import (
"strings"
"testing"

"github.com/anchore/stereoscope/pkg/filetree"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"

"github.com/go-test/deep"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/types"
Expand Down Expand Up @@ -359,7 +360,7 @@ func TestFileCatalog_GetByExtension(t *testing.T) {
if d := cmp.Diff(tt.want, actual,
cmpopts.EquateEmpty(),
cmpopts.IgnoreUnexported(file.Reference{}),
cmpopts.IgnoreFields(file.Metadata{}, "Mode", "GroupID", "UserID", "Size"),
cmpopts.IgnoreFields(file.Metadata{}, "Mode", "GroupID", "UserID", "Size", "ModTime", "AccessTime", "ChangeTime"),
); d != "" {
t.Errorf("diff: %s", d)
}
Expand Down Expand Up @@ -461,7 +462,7 @@ func TestFileCatalog_GetByBasename(t *testing.T) {
if d := cmp.Diff(tt.want, actual,
cmpopts.EquateEmpty(),
cmpopts.IgnoreUnexported(file.Reference{}),
cmpopts.IgnoreFields(file.Metadata{}, "Mode", "GroupID", "UserID", "Size"),
cmpopts.IgnoreFields(file.Metadata{}, "Mode", "GroupID", "UserID", "Size", "ModTime", "AccessTime", "ChangeTime"),
); d != "" {
t.Errorf("diff: %s", d)
}
Expand Down Expand Up @@ -571,7 +572,7 @@ func TestFileCatalog_GetByBasenameGlob(t *testing.T) {
if d := cmp.Diff(tt.want, actual,
cmpopts.EquateEmpty(),
cmpopts.IgnoreUnexported(file.Reference{}),
cmpopts.IgnoreFields(file.Metadata{}, "Mode", "GroupID", "UserID", "Size"),
cmpopts.IgnoreFields(file.Metadata{}, "Mode", "GroupID", "UserID", "Size", "ModTime", "AccessTime", "ChangeTime"),
); d != "" {
t.Errorf("diff: %s", d)
}
Expand Down Expand Up @@ -672,7 +673,7 @@ func TestFileCatalog_GetByMimeType(t *testing.T) {
if d := cmp.Diff(tt.want, actual,
cmpopts.EquateEmpty(),
cmpopts.IgnoreUnexported(file.Reference{}),
cmpopts.IgnoreFields(file.Metadata{}, "Mode", "GroupID", "UserID", "Size"),
cmpopts.IgnoreFields(file.Metadata{}, "Mode", "GroupID", "UserID", "Size", "ModTime", "AccessTime", "ChangeTime"),
); d != "" {
t.Errorf("diff: %s", d)
}
Expand Down

0 comments on commit 3282bc0

Please sign in to comment.