test: sif image_simple / image_symlinks adaptations

Modify the TestSimpleImage and TestImageSymlinks code to incorporate
testing of a singularity sif image source.

This requires signficant adaptations as singularity squashes
containers down to a single layer.

Singularity is expected to be available, and is is now installed in
the ci-bootstrap Makefile target (from a GitHub release).

Signed-off-by: David Trudgian <david.trudgian@sylabs.io>

Makefile

@@ -44,6 +44,7 @@ help:
 .PHONY: ci-bootstrap
 ci-bootstrap: bootstrap
 	sudo apt install -y bc
+	sudo apt install -y https://github.com/sylabs/singularity/releases/download/v3.10.0/singularity-ce_3.10.0-focal_amd64.deb
 
 $(RESULTSDIR):
 	mkdir -p $(RESULTSDIR)

pkg/image/sif/image.go

@@ -14,6 +14,8 @@ import (
 	"github.com/sylabs/sif/v2/pkg/sif"
 )
 
+const SingularityMediaType = "application/vnd.sylabs.sif.layer.v1.sif"
+
 // fileSectionReader implements an io.ReadCloser that reads from r and closes c.
 type fileSectionReader struct {
 	*io.SectionReader
@@ -135,7 +137,7 @@ func (im *sifImage) RawConfigFile() ([]byte, error) {
 
 // MediaType of this image's manifest.
 func (im *sifImage) MediaType() (types.MediaType, error) {
-	return "application/vnd.sylabs.sif.layer.v1.sif", nil
+	return SingularityMediaType, nil
 }
 
 // LayerByDiffID is a variation on the v1.Image method, which returns an UncompressedLayer instead.

pkg/imagetest/image_fixtures.go

@@ -48,6 +48,8 @@ func PrepareFixtureImage(t testing.TB, source, name string) string {
 			skopeoCopyDockerArchiveToPath(t, dockerArchivePath, fmt.Sprintf("oci:%s", ociDirPath))
 		}
 		location = ociDirPath
+	case image.SingularitySource:
+		location = GetFixtureImageSIFPath(t, name)
 	default:
 		t.Fatalf("could not determine source: %+v", source)
 	}
@@ -251,3 +253,64 @@ func saveImage(t testing.TB, image, path string) error {
 	cmd.Stdin = os.Stdin
 	return cmd.Run()
 }
+
+func GetFixtureImageSIFPath(t testing.TB, name string) string {
+	imageName, imageVersion := getFixtureImageInfo(t, name)
+	sifFileName := fmt.Sprintf("%s-%s.sif", imageName, imageVersion)
+	return getFixtureImageSIFPath(t, name, CacheDir, sifFileName)
+}
+
+func getFixtureImageSIFPath(t testing.TB, fixtureName, sifStoreDir, sifFileName string) string {
+	imageName, imageVersion := getFixtureImageInfo(t, fixtureName)
+	fullImageName := fmt.Sprintf("%s:%s", imageName, imageVersion)
+	sifPath := path.Join(sifStoreDir, sifFileName)
+
+	// create the cache dir if it does not already exist...
+	if !fileOrDirExists(t, CacheDir) {
+		err := os.Mkdir(CacheDir, 0o755)
+		if err != nil {
+			t.Fatalf("could not create sif cache dir (%s): %+v", CacheDir, err)
+		}
+	}
+
+	// if the image sif does not exist, make it
+	if !fileOrDirExists(t, sifPath) {
+		if !isImageInDocker(fullImageName) {
+			contextPath := path.Join(testutils.TestFixturesDir, fixtureName)
+			buildDockerImage(t, contextPath, imageName, imageVersion)
+		}
+		err := buildSIFFromDocker(t, fullImageName, sifPath)
+		if err != nil {
+			t.Fatal("could not save fixture image:", err)
+		}
+	}
+
+	return sifPath
+}
+
+func buildSIFFromDocker(t testing.TB, image, path string) error {
+	singularity, err := exec.LookPath("singularity")
+	if err != nil {
+		t.Skipf("singularity not found: %v", err)
+	}
+
+	outfile, err := os.Create(path)
+	if err != nil {
+		t.Fatal("unable to create file for SIF image:", err)
+	}
+	defer func() {
+		err := outfile.Close()
+		if err != nil {
+			t.Fatalf("unable to close file path=%q : %+v", path, err)
+		}
+	}()
+
+	cmdArgs := []string{"build", "--disable-cache", "--force", path, "docker-daemon:" + image}
+	cmd := exec.Command(singularity, cmdArgs...)
+	cmd.Env = os.Environ()
+
+	cmd.Stdout = outfile
+	cmd.Stderr = os.Stderr
+	cmd.Stdin = os.Stdin
+	return cmd.Run()
+}

test/integration/fixture_image_simple_test.go

@@ -15,49 +15,95 @@ import (
 	"github.com/anchore/stereoscope/pkg/file"
 	"github.com/anchore/stereoscope/pkg/filetree"
 	"github.com/anchore/stereoscope/pkg/image"
+	"github.com/anchore/stereoscope/pkg/image/sif"
 	"github.com/anchore/stereoscope/pkg/imagetest"
 	v1Types "github.com/google/go-containerregistry/pkg/v1/types"
 	"github.com/scylladb/go-set"
 	"github.com/stretchr/testify/require"
 )
 
+// Common layer metadata for OCI / Docker / Podman. MediaType will be filled in during test.
+var simpleImageLayers = []image.LayerMetadata{
+	{
+		Index: 0,
+		Size:  22,
+	},
+	{
+		Index: 1,
+		Size:  16,
+	},
+	{
+		Index: 2,
+		Size:  27,
+	},
+}
+
+// Singularity images are squashed to a single layer.
+var simpleImageSingularityLayer = []image.LayerMetadata{
+	{
+		Index: 0,
+		// Disable size check. Size can vary - image build embeds variable length timestamps etc.
+		Size: -1,
+	},
+}
+
 var simpleImageTestCases = []testCase{
 	{
 		name:           "FromTarball",
 		source:         "docker-archive",
 		imageMediaType: v1Types.DockerManifestSchema2,
 		layerMediaType: v1Types.DockerLayer,
+		layers:         simpleImageLayers,
 		tagCount:       1,
+		size:           65,
 	},
 	{
 		name:           "FromDocker",
 		source:         "docker",
 		imageMediaType: v1Types.DockerManifestSchema2,
 		layerMediaType: v1Types.DockerLayer,
+		layers:         simpleImageLayers,
 		// name:hash
 		// name:latest
 		tagCount: 2,
+		size:     65,
 	},
 	{
 		name:           "FromPodman",
 		source:         "podman",
 		imageMediaType: v1Types.DockerManifestSchema2,
 		layerMediaType: v1Types.DockerLayer,
+		layers:         simpleImageLayers,
 		tagCount:       2,
+		size:           65,
 	},
 	{
 		name:           "FromOciTarball",
 		source:         "oci-archive",
 		imageMediaType: v1Types.OCIManifestSchema1,
 		layerMediaType: v1Types.OCILayer,
+		layers:         simpleImageLayers,
 		tagCount:       0,
+		size:           65,
 	},
 	{
 		name:           "FromOciDirectory",
 		source:         "oci-dir",
 		imageMediaType: v1Types.OCIManifestSchema1,
 		layerMediaType: v1Types.OCILayer,
+		layers:         simpleImageLayers,
+		tagCount:       0,
+		size:           65,
+	},
+	{
+		name:           "FromSingularity",
+		source:         "singularity",
+		imageMediaType: sif.SingularityMediaType,
+		layerMediaType: image.SingularitySquashFSLayer,
+		layers:         simpleImageSingularityLayer,
 		tagCount:       0,
+		// Disable size check. Size can vary - image build embeds timestamps etc.
+		size: -1,
 	},
 }
 
@@ -66,7 +112,9 @@ type testCase struct {
 	source         string
 	imageMediaType v1Types.MediaType
 	layerMediaType v1Types.MediaType
+	layers         []image.LayerMetadata
 	tagCount       int
+	size           int
 }
 
 func TestSimpleImage(t *testing.T) {
@@ -81,8 +129,11 @@ func TestSimpleImage(t *testing.T) {
 			i := imagetest.GetFixtureImage(t, c.source, "image-simple")
 
 			assertImageSimpleMetadata(t, i, c)
-			assertImageSimpleTrees(t, i)
-			assertImageSimpleSquashedTrees(t, i)
+			// Singularity images are a single layer. Don't verify content per layer.
+			if c.source != "singularity" {
+				assertImageSimpleTrees(t, i)
+				assertImageSimpleSquashedTrees(t, i)
+			}
 			assertImageSimpleContents(t, i)
 		})
 	}
@@ -148,9 +199,6 @@ func BenchmarkSimpleImage_FetchSquashedContents(b *testing.B) {
 func assertImageSimpleMetadata(t *testing.T, i *image.Image, expectedValues testCase) {
 	t.Helper()
 	t.Log("Asserting metadata...")
-	if i.Metadata.Size != 65 {
-		t.Errorf("unexpected image size: %d", i.Metadata.Size)
-	}
 	if i.Metadata.MediaType != expectedValues.imageMediaType {
 		t.Errorf("unexpected image media type: %+v", i.Metadata.MediaType)
 	}
@@ -162,36 +210,24 @@ func assertImageSimpleMetadata(t *testing.T, i *image.Image, expectedValues test
 		}
 	}
 
-	expected := []image.LayerMetadata{
-		{
-			Index:     0,
-			Size:      22,
-			MediaType: expectedValues.layerMediaType,
-		},
-		{
-			Index:     1,
-			Size:      16,
-			MediaType: expectedValues.layerMediaType,
-		},
-		{
-			Index:     2,
-			Size:      27,
-			MediaType: expectedValues.layerMediaType,
-		},
+	if expectedValues.size >= 0 && i.Metadata.Size != int64(expectedValues.size) {
+		t.Errorf("unexpected image size: %d", i.Metadata.Size)
 	}
 
-	if len(expected) != len(i.Layers) {
+	if len(expectedValues.layers) != len(i.Layers) {
 		t.Fatal("unexpected number of layers:", len(i.Layers))
 	}
 
 	for idx, l := range i.Layers {
-		if expected[idx].Size != l.Metadata.Size {
+		expected := expectedValues.layers[idx]
+		expected.MediaType = expectedValues.layerMediaType
+		if expected.Size >= 0 && expected.Size != l.Metadata.Size {
 			t.Errorf("mismatched layer 'Size' (layer %d): %+v", idx, l.Metadata.Size)
 		}
-		if expected[idx].MediaType != l.Metadata.MediaType {
+		if expected.MediaType != l.Metadata.MediaType {
 			t.Errorf("mismatched layer 'MediaType' (layer %d): %+v", idx, l.Metadata.MediaType)
 		}
-		if expected[idx].Index != l.Metadata.Index {
+		if expected.Index != l.Metadata.Index {
 			t.Errorf("mismatched layer 'Index' (layer %d): %+v", idx, l.Metadata.Index)
 		}
 	}

test/integration/fixture_image_symlinks_test.go

@@ -50,6 +50,10 @@ func TestImageSymlinks(t *testing.T) {
 			name:   "FromOciDirectory",
 			source: "oci-dir",
 		},
+		{
+			name:   "FromSingularity",
+			source: "singularity",
+		},
 	}
 
 	expectedSet := set.NewIntSet()
@@ -61,6 +65,12 @@ func TestImageSymlinks(t *testing.T) {
 	for _, c := range cases {
 		t.Run(c.name, func(t *testing.T) {
 			i := imagetest.GetFixtureImage(t, c.source, "image-symlinks")
+
+			if c.source == "singularity" {
+				assertSquashedSymlinkLinkResolution(t, i)
+				return
+			}
+
 			assertImageSymlinkLinkResolution(t, i)
 		})
 	}
@@ -236,3 +246,60 @@ func assertImageSymlinkLinkResolution(t *testing.T, i *image.Image) {
 		})
 	}
 }
+
+// Check symlinks in image after it has been squashed to a single layer (Singularity)
+func assertSquashedSymlinkLinkResolution(t *testing.T, i *image.Image) {
+	tests := []linkFetchConfig{
+		// # link with previous data
+		// LAYER 1 > ADD file-1.txt .
+		// LAYER 2 > RUN ln -s ./file-1.txt link-1
+		{
+			linkLayer:        0,
+			linkPath:         "/link-1",
+			resolveLayer:     0,
+			expectedPath:     "/file-1.txt",
+			perspectiveLayer: 0,
+			contents:         "file 1!",
+		},
+
+		// # link with current data
+		// LAYER 5 > RUN echo "file 3" > file-3.txt && ln -s ./file-3.txt link-within
+		{
+			linkLayer:        0,
+			linkPath:         "/link-within",
+			resolveLayer:     0,
+			expectedPath:     "/file-3.txt",
+			perspectiveLayer: 0,
+			// since echo was used a newline character will be present
+			contents: "file 3\n",
+		},
+
+		// # dead link (link-indirect > [non-existant file])
+		// LAYER 8 > RUN unlink link-2
+		{
+			linkLayer:        0,
+			linkPath:         "/link-indirect",
+			resolveLayer:     0,
+			expectedPath:     "/link-indirect",
+			perspectiveLayer: 0,
+			linkOptions:      []filetree.LinkResolutionOption{filetree.DoNotFollowDeadBasenameLinks},
+		},
+	}
+
+	for _, cfg := range tests {
+		name := fmt.Sprintf("[%d:%s]-->[%d:%s]@%d", cfg.linkLayer, cfg.linkPath, cfg.resolveLayer, cfg.expectedPath, cfg.perspectiveLayer)
+		t.Run(name, func(t *testing.T) {
+			expectedResolve, actualResolve := fetchRefs(t, i, cfg)
+			assertMatch(t, i, cfg, expectedResolve, actualResolve)
+
+			if cfg.contents == "" {
+				return
+			}
+
+			actualContents := fetchContents(t, i, cfg)
+			if actualContents != cfg.contents {
+				t.Errorf("mismatched contents: '%+v'!='%+v'", cfg.contents, actualContents)
+			}
+		})
+	}
+}