AWS ECR uni-arch image support

.env_example

@@ -1,5 +1,16 @@
-STARLIGHT_CONTAINER_REGISTRY=http://nagi.lan:5002
+STARLIGHT_CONTAINER_REGISTRY=harbor.yuri.moe
 STARLIGHT_SANDBOX_DIR=../.sandbox
 
-TEST_ECR_IMAGE_FROM=
-TEST_ECR_IMAGE_TO=
+TEST_DOCKER_SOURCE_IMAGE=docker.io/library/redis:6.2.1
+
+TEST_HARBOR_REGISTRY=harbor.yuri.moe
+TEST_HARBOR_IMAGE_FROM=harbor.yuri.moe/starlight/test-harbor:test
+TEST_HARBOR_IMAGE_TO=harbor.yuri.moe/starlight/test-harbor123:starlight2
+
+HARBOR_USERNAME=
+HARBOR_PASSWORD=
+
+TEST_ECR_IMAGE_FROM=public.ecr.aws/______/image-name:test
+TEST_ECR_IMAGE_TO=public.ecr.aws/______/image-name:starlight2
+
+

proxy/extractor.go

@@ -21,18 +21,20 @@ package proxy
 import (
 	"bytes"
 	"fmt"
+	"io"
+	"net/http"
+	"path"
+
 	"github.com/containerd/containerd/log"
 	"github.com/google/go-containerregistry/pkg/authn"
 	"github.com/google/go-containerregistry/pkg/name"
 	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/google/go-containerregistry/pkg/v1/remote"
+	"github.com/google/go-containerregistry/pkg/v1/types"
 	"github.com/mc256/starlight/util/common"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/sync/errgroup"
-	"io"
-	"net/http"
-	"path"
 )
 
 // Extractor extract ToC from starlight-formatted container image and save it to the database
@@ -137,9 +139,7 @@ func (ex *Extractor) saveLayer(imageSerial, idx int64, layer v1.Layer) error {
 		// chunks
 		for k, v := range chunks {
 			extEntry := entBuffer[k]
-			for _, c := range v {
-				extEntry.Chunks = append(extEntry.Chunks, c)
-			}
+			extEntry.Chunks = append(extEntry.Chunks, v...)
 		}
 
 		err = ex.server.db.InsertFiles(txn, layerRef, entBuffer)
@@ -152,6 +152,69 @@ func (ex *Extractor) saveLayer(imageSerial, idx int64, layer v1.Layer) error {
 
 }
 
+func (ex *Extractor) saveToCPerImage(img v1.Image, m v1.Descriptor) error {
+	pltStr := "uni-arch"
+	if plt := m.Platform; plt != nil {
+		pltStr = path.Join(plt.OS, plt.Architecture, plt.Variant)
+	}
+
+	log.G(ex.server.ctx).WithFields(logrus.Fields{
+		"image":    ex.ParsedName,
+		"tag":      ex.ParsedTag,
+		"hash":     m.Digest.String(),
+		"platform": pltStr,
+	}).Trace("found image")
+
+	// Layers
+	layers, err := img.Layers()
+
+	// Insert into the "image" table
+	var (
+		existing = false
+		serial   int64
+	)
+	serial, existing, err = ex.saveImage(&img)
+	if err != nil {
+		return errors.Wrapf(err, "failed to save image")
+	}
+
+	// Insert into the "layer" - "filesystem" - "file" tables
+	if !existing {
+		var errGrp errgroup.Group
+		for idx, layer := range layers {
+			idx, layer, serial := int64(idx), layer, serial
+			errGrp.Go(func() error {
+				return ex.saveLayer(serial, idx, layer)
+			})
+		}
+
+		if err = errGrp.Wait(); err != nil {
+			return errors.Wrapf(err, "failed to cache ToC")
+		}
+
+		if err = ex.enableImage(serial); err != nil {
+			return errors.Wrapf(err, "failed to enable image")
+		}
+	} else if serial == 0 && err != nil {
+		return errors.Wrapf(err, "image exists")
+	}
+
+	// Insert into the "tag" table (tag.imageId - image.id)
+	if err = ex.setImageTag(serial, pltStr); err != nil {
+		return errors.Wrapf(err, "failed to cache ToC")
+	}
+
+	log.G(ex.server.ctx).WithFields(logrus.Fields{
+		"image":    ex.ParsedName,
+		"tag":      ex.ParsedTag,
+		"hash":     m.Digest.String(),
+		"serial":   serial,
+		"platform": pltStr,
+	}).Debug("saved ToC")
+
+	return nil
+}
+
 // SaveToC save ToC to the backend database and return ApiResponse if success.
 // It does require the container registry is functioning correctly.
 func (ex *Extractor) SaveToC() (res *ApiResponse, err error) {
@@ -162,79 +225,47 @@ func (ex *Extractor) SaveToC() (res *ApiResponse, err error) {
 		return nil, errors.Wrapf(err, "failed to cache ToC")
 	}
 
-	// Container image index
-	imgIdx, err := desc.ImageIndex()
-	if err != nil {
-		return nil, errors.Wrapf(err, "failed to get image index")
-	}
+	if desc.MediaType == types.DockerManifestSchema2 {
+		// single manifest image
+		// "application/vnd.docker.distribution.manifest.v2+json"
 
-	idxMan, err := imgIdx.IndexManifest()
-	if err != nil {
-		return nil, errors.Wrapf(err, "failed to get index manifest")
-	}
+		img, err := desc.Image()
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to get single image")
+		}
 
-	for _, m := range idxMan.Manifests {
-		img, err := imgIdx.Image(m.Digest)
+		err = ex.saveToCPerImage(img, desc.Descriptor)
 		if err != nil {
-			return nil, errors.Wrapf(err, "failed to get image")
+			return nil, errors.Wrapf(err, "failed to cache ToC")
 		}
 
-		plt := m.Platform
-		pltStr := path.Join(plt.OS, plt.Architecture, plt.Variant)
-		log.G(ex.server.ctx).WithFields(logrus.Fields{
-			"image":    ex.ParsedName,
-			"tag":      ex.ParsedTag,
-			"hash":     m.Digest.String(),
-			"platform": pltStr,
-		}).Trace("found image")
-
-		// Layers
-		layers, err := img.Layers()
-
-		// Insert into the "image" table
-		var (
-			existing = false
-			serial   int64
-		)
-		serial, existing, err = ex.saveImage(&img)
+	} else {
+		// image index
+		// "application/vnd.docker.distribution.manifest.list.v2+json"
+
+		// Container image index
+		imgIdx, err := desc.ImageIndex()
 		if err != nil {
-			return nil, errors.Wrapf(err, "failed to save image")
+			return nil, errors.Wrapf(err, "failed to get image index")
 		}
 
-		// Insert into the "layer" - "filesystem" - "file" tables
-		if !existing {
-			var errGrp errgroup.Group
-			for idx, layer := range layers {
-				idx, layer, serial := int64(idx), layer, serial
-				errGrp.Go(func() error {
-					return ex.saveLayer(serial, idx, layer)
-				})
-			}
+		idxMan, err := imgIdx.IndexManifest()
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to get index manifest")
+		}
 
-			if err = errGrp.Wait(); err != nil {
-				return nil, errors.Wrapf(err, "failed to cache ToC")
+		for _, m := range idxMan.Manifests {
+			img, err := imgIdx.Image(m.Digest)
+			if err != nil {
+				return nil, errors.Wrapf(err, "failed to get image")
 			}
 
-			if err = ex.enableImage(serial); err != nil {
-				return nil, errors.Wrapf(err, "failed to enable image")
+			err = ex.saveToCPerImage(img, m)
+			if err != nil {
+				return nil, errors.Wrapf(err, "failed to cache ToC")
 			}
-		} else if serial == 0 && err != nil {
-			return nil, errors.Wrapf(err, "image exists")
 		}
 
-		// Insert into the "tag" table (tag.imageId - image.id)
-		if err = ex.setImageTag(serial, pltStr); err != nil {
-			return nil, errors.Wrapf(err, "failed to cache ToC")
-		}
-
-		log.G(ex.server.ctx).WithFields(logrus.Fields{
-			"image":    ex.ParsedName,
-			"tag":      ex.ParsedTag,
-			"hash":     m.Digest.String(),
-			"serial":   serial,
-			"platform": pltStr,
-		}).Debug("saved ToC")
-
 	}
 
 	return &ApiResponse{

proxy/extractor_test.go

@@ -9,9 +9,11 @@ import (
 	"context"
 	"fmt"
 	"net/http"
+	"os"
 	"testing"
 
 	"github.com/containerd/containerd/log"
+	"github.com/mc256/starlight/test"
 )
 
 func TestNewExtractor(t *testing.T) {
@@ -32,11 +34,41 @@ func TestNewExtractor(t *testing.T) {
 	fmt.Println(ext)
 }
 
-func TestExtractor_SaveToC(t *testing.T) {
-	t.Skip("for dev only")
+func TestNewExtractor2(t *testing.T) {
+	ctx := context.Background()
+	cfg, _, _, _ := LoadConfig("")
+	server := &Server{
+		ctx: ctx,
+		Server: http.Server{
+			Addr: fmt.Sprintf("%s:%d", cfg.ListenAddress, cfg.ListenPort),
+		},
+		config: cfg,
+	}
+
+	//docker pull harbor.yuri.moe/starlight/test-harbor123@sha256:cfe501ce6892b1c9f5ca31a45ecef2d5bd3dddf21c57044bde33e930da3409d6
+	ext, err := NewExtractor(server, "starlight/test-harbor123@sha256:cfe501ce6892b1c9f5ca31a45ecef2d5bd3dddf21c57044bde33e930da3409d6", true)
+	if err != nil {
+		t.Error(err)
+	}
+	fmt.Println(ext)
+}
+
+func TestExtractor_SaveToC_Goharbor(t *testing.T) {
+	test.LoadEnvironmentVariables()
+	if test.HasLoginStarlightGoharbor() == false {
+		t.Skip(">>>>> Skip: no container registry credentials for goharbor")
+	}
 
 	ctx := context.Background()
 	cfg, _, _, _ := LoadConfig("")
+
+	if os.Getenv("TEST_HARBOR_REGISTRY") != "" {
+		cfg.DefaultRegistry = os.Getenv("TEST_HARBOR_REGISTRY")
+	}
+	if os.Getenv("POSTGRES_CONNECTION_URL") != "" {
+		cfg.PostgresConnectionString = os.Getenv("POSTGRES_CONNECTION_URL")
+	}
+
 	server := &Server{
 		ctx: ctx,
 		Server: http.Server{
@@ -61,3 +93,50 @@ func TestExtractor_SaveToC(t *testing.T) {
 	}
 	fmt.Println(res)
 }
+
+func TestExtractor_SaveToC_AWSECR(t *testing.T) {
+	test.LoadEnvironmentVariables()
+	if test.HasLoginAWSECR() == false {
+		t.Skip(">>>>> Skip: no container registry credentials for AWS ECR")
+	}
+
+	ctx := context.Background()
+	cfg, _, _, _ := LoadConfig("")
+
+	if os.Getenv("TEST_HARBOR_REGISTRY") != "" {
+		cfg.DefaultRegistry = os.Getenv("TEST_HARBOR_REGISTRY")
+	}
+	if os.Getenv("POSTGRES_CONNECTION_URL") != "" {
+		cfg.PostgresConnectionString = os.Getenv("POSTGRES_CONNECTION_URL")
+	}
+
+	if os.Getenv("TEST_ECR_IMAGE_TO") == "" {
+		t.Skip(">>>>> Skip: no ECR image set in TEST_ECR_IMAGE_TO")
+	}
+
+	awsecrImage := os.Getenv("TEST_ECR_IMAGE_TO")
+
+	server := &Server{
+		ctx: ctx,
+		Server: http.Server{
+			Addr: fmt.Sprintf("%s:%d", cfg.ListenAddress, cfg.ListenPort),
+		},
+		config: cfg,
+	}
+	if db, err := NewDatabase(ctx, cfg.PostgresConnectionString); err != nil {
+		log.G(ctx).Errorf("failed to connect to database: %v\n", err)
+	} else {
+		server.db = db
+	}
+
+	ext, err := NewExtractor(server, awsecrImage, true)
+	if err != nil {
+		t.Error(err)
+	}
+	fmt.Println(ext)
+	res, err := ext.SaveToC()
+	if err != nil {
+		t.Error(err)
+	}
+	fmt.Println(res)
+}