From 5cfa756ba6d6c6cf4cbc2d8bf23703f1fa8a9298 Mon Sep 17 00:00:00 2001 From: John Reese Date: Wed, 30 Sep 2020 18:00:05 -0400 Subject: [PATCH] Validate images can be pulled before pull operation (#39) --- Dockerfile | 32 ++++++++++------------------ Makefile | 25 +++++++++++----------- internal/commands/check.go | 2 +- internal/commands/pull.go | 13 ++++++++++-- internal/commands/push.go | 6 +++--- internal/docker/docker.go | 43 ++++++++++++++++++++++---------------- 6 files changed, 64 insertions(+), 57 deletions(-) diff --git a/Dockerfile b/Dockerfile index 14d5c1e..b9e1fd0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,33 +1,23 @@ -ARG GOLANG_VERSION=1.15.0 -ARG ALPINE_VERSION=3.12.0 +FROM golang:1.15.0 AS builder -FROM golang:${GOLANG_VERSION} AS builder -WORKDIR /build -COPY . /build - -# Enable static builds ENV CGO_ENABLED=0 -RUN go get && \ - go build - -FROM alpine:${ALPINE_VERSION} +WORKDIR /build +COPY . /build -# OCI annotations (https://github.com/opencontainers/image-spec/blob/master/annotations.md) -LABEL org.opencontainers.image.source="https://github.com/plexsystems/sinker" \ - org.opencontainers.image.title="sinker" \ - org.opencontainers.image.authors="John Reese " \ - org.opencontainers.image.description="Application to sync images from one registry to another" +# SINKER_VERSION is set during the release process +ARG SINKER_VERSION=0.0.0 +RUN go build -ldflags="-X 'github.com/plexsystems/sinker/internal/commands.sinkerVersion=${SINKER_VERSION}'" -# explicitly set user/group IDs -# RUN set -eux \ -# && addgroup -g 1001 -S sinker \ -# && adduser -S -D -H -u 1001 -s /sbin/nologin -G sinker -g sinker sinker +FROM alpine:3.12.0 RUN apk update && apk add --no-cache docker-cli COPY --from=builder /build/sinker /usr/bin/ -# USER sinker +LABEL org.opencontainers.image.source="https://github.com/plexsystems/sinker" +LABEL org.opencontainers.image.title="sinker" +LABEL org.opencontainers.image.authors="John Reese " +LABEL org.opencontainers.image.description="Sync container images from one registry to another" ENTRYPOINT ["/usr/bin/sinker"] diff --git a/Makefile b/Makefile index 1b41083..8ca5fbd 100644 --- a/Makefile +++ b/Makefile @@ -1,23 +1,24 @@ .PHONY: build build: - go build - -.PHONY: remove-images -remove-images: - docker rmi `docker images -a -q` + @go build .PHONY: test test: - go test -v ./... -count=1 + @go test -v ./... -count=1 .PHONY: acceptance -acceptance: - go build - bats acceptance.bats +acceptance: build + @bats acceptance.bats + +.PHONY: all +all: build test acceptance +# When using the release target a version must be specified. +# e.g. make release version=v0.1.0 .PHONY: release release: @test $(version) - GOOS=darwin GOARCH=amd64 go build -o sinker-darwin-amd64 -ldflags="-X 'github.com/plexsystems/sinker/internal/commands.sinkerVersion=$(version)'" - GOOS=windows GOARCH=amd64 go build -o sinker-windows-amd64 -ldflags="-X 'github.com/plexsystems/sinker/internal/commands.sinkerVersion=$(version)'" - GOOS=linux GOARCH=amd64 go build -o sinker-linux-amd64 -ldflags="-X 'github.com/plexsystems/sinker/internal/commands.sinkerVersion=$(version)'" + @docker build --build-arg SINKER_VERSION=$(version) -t plexsystems/sinker:$(version) . + @GOOS=darwin GOARCH=amd64 go build -o sinker-darwin-amd64 -ldflags="-X 'github.com/plexsystems/sinker/internal/commands.sinkerVersion=$(version)'" + @GOOS=windows GOARCH=amd64 go build -o sinker-windows-amd64 -ldflags="-X 'github.com/plexsystems/sinker/internal/commands.sinkerVersion=$(version)'" + @GOOS=linux GOARCH=amd64 go build -o sinker-linux-amd64 -ldflags="-X 'github.com/plexsystems/sinker/internal/commands.sinkerVersion=$(version)'" diff --git a/internal/commands/check.go b/internal/commands/check.go index b852898..8a4c1d6 100644 --- a/internal/commands/check.go +++ b/internal/commands/check.go @@ -47,7 +47,7 @@ func runCheckCommand(input string) error { ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() - client, err := docker.NewClient(log.Infof) + client, err := docker.New(log.Infof) if err != nil { return fmt.Errorf("new client: %w", err) } diff --git a/internal/commands/pull.go b/internal/commands/pull.go index da13056..635d7c1 100644 --- a/internal/commands/pull.go +++ b/internal/commands/pull.go @@ -48,7 +48,7 @@ func runPullCommand(origin string, manifestPath string) error { ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) defer cancel() - client, err := docker.NewClient(log.Infof) + client, err := docker.New(log.Infof) if err != nil { return fmt.Errorf("new client: %w", err) } @@ -77,9 +77,18 @@ func runPullCommand(origin string, manifestPath string) error { } } + // Iterate through each of the images to pull and verify if the client has + // the proper authorization to be able to successfully pull the images before + // performing the pull operation. + for image := range imagesToPull { + if _, err := client.ImageExistsAtRemote(ctx, image); err != nil { + return fmt.Errorf("validating remote image: %w", err) + } + } + for image, auth := range imagesToPull { log.Infof("Pulling %s", image) - if err := client.PullImageAndWait(ctx, image, auth); err != nil { + if err := client.PullAndWait(ctx, image, auth); err != nil { log.Errorf("pull image and wait: " + err.Error()) } } diff --git a/internal/commands/push.go b/internal/commands/push.go index 674fd60..b33160f 100644 --- a/internal/commands/push.go +++ b/internal/commands/push.go @@ -56,7 +56,7 @@ func runPushCommand(manifestPath string) error { ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) defer cancel() - client, err := docker.NewClient(log.Infof) + client, err := docker.New(log.Infof) if err != nil { return fmt.Errorf("new client: %w", err) } @@ -112,7 +112,7 @@ func runPushCommand(manifestPath string) error { if err != nil { return fmt.Errorf("get source auth: %w", err) } - if err := client.PullImageAndWait(ctx, source.Image(), sourceAuth); err != nil { + if err := client.PullAndWait(ctx, source.Image(), sourceAuth); err != nil { return fmt.Errorf("pull image and wait: %w", err) } } @@ -133,7 +133,7 @@ func runPushCommand(manifestPath string) error { if err != nil { return fmt.Errorf("get target auth: %w", err) } - if err := client.PushImageAndWait(ctx, source.TargetImage(), targetAuth); err != nil { + if err := client.PushAndWait(ctx, source.TargetImage(), targetAuth); err != nil { return fmt.Errorf("push image and wait: %w", err) } } diff --git a/internal/docker/docker.go b/internal/docker/docker.go index 819f9fc..4059dc5 100644 --- a/internal/docker/docker.go +++ b/internal/docker/docker.go @@ -23,8 +23,8 @@ type Client struct { logInfo func(format string, args ...interface{}) } -// NewClient returns a Docker client configured with the given information logger. -func NewClient(logInfo func(format string, args ...interface{})) (Client, error) { +// New returns a Docker client configured with the given information logger. +func New(logInfo func(format string, args ...interface{})) (Client, error) { retry.DefaultDelay = 5 * time.Second retry.DefaultAttempts = 2 @@ -41,11 +41,11 @@ func NewClient(logInfo func(format string, args ...interface{})) (Client, error) return client, nil } -// PushImageAndWait pushes an image and waits for it to finish pushing. +// PushAndWait pushes an image and waits for it to finish pushing. // If an error occurs when pushing an image, the push will be attempted again before failing. -func (c Client) PushImageAndWait(ctx context.Context, image string, auth string) error { +func (c Client) PushAndWait(ctx context.Context, image string, auth string) error { push := func() error { - if err := c.tryPushImageAndWait(ctx, image, auth); err != nil { + if err := c.tryPushAndWait(ctx, image, auth); err != nil { return fmt.Errorf("try push image: %w", err) } @@ -63,11 +63,11 @@ func (c Client) PushImageAndWait(ctx context.Context, image string, auth string) return nil } -// PullImageAndWait pulls an image and waits for it to finish pulling. +// PullAndWait pulls an image and waits for it to finish pulling. // If an error occurs when pulling an image, the pull will be attempted again before failing. -func (c Client) PullImageAndWait(ctx context.Context, image string, auth string) error { +func (c Client) PullAndWait(ctx context.Context, image string, auth string) error { pull := func() error { - if err := c.tryPullImageAndWait(ctx, image, auth); err != nil { + if err := c.tryPullAndWait(ctx, image, auth); err != nil { return fmt.Errorf("try pull image: %w", err) } @@ -170,10 +170,6 @@ func (c Client) Tag(ctx context.Context, sourceImage string, targetImage string) // ImageExistsAtRemote returns true if the image exists at the remote registry. func (c Client) ImageExistsAtRemote(ctx context.Context, image string) (bool, error) { - if hasLatestTag(image) { - return false, nil - } - reference, err := name.ParseReference(image, name.WeakValidation) if err != nil { return false, fmt.Errorf("parse ref: %w", err) @@ -181,24 +177,35 @@ func (c Client) ImageExistsAtRemote(ctx context.Context, image string) (bool, er if _, err := remote.Get(reference, remote.WithAuthFromKeychain(authn.DefaultKeychain)); err != nil { - // If the error is a transport error, check that the error code is of type MANIFEST_UNKNOWN. - // This is the expected error if an image does not exist. + // If the error is a transport error, check that the error code is of type + // MANIFEST_UNKNOWN or NOT_FOUND. These errors are expected if an image does + // not exist in the registry. if t, exists := err.(*transport.Error); exists { for _, diagnostic := range t.Errors { if strings.EqualFold("MANIFEST_UNKNOWN", string(diagnostic.Code)) { return false, nil } + if strings.EqualFold("NOT_FOUND", string(diagnostic.Code)) { return false, nil } } } - // If the error is not a transport error, some other error occured - // that is unrelated to checking if an image exists and it should be returned. return false, fmt.Errorf("get image: %w", err) } + // Always return false if the image has the latest tag as this method + // is used to determine if the image should be pushed or not. The latest + // tag is assumed to always need to be pushed, but a better approach + // would be to compare digests. + // + // This check must also be performed after the Get request to the remote + // registry to ensure that the client has appropriate access to pull the image. + if hasLatestTag(image) { + return false, nil + } + return true, nil } @@ -245,7 +252,7 @@ func (c Client) waitForScannerComplete(clientScanner *bufio.Scanner, image strin return nil } -func (c Client) tryPullImageAndWait(ctx context.Context, image string, auth string) error { +func (c Client) tryPullAndWait(ctx context.Context, image string, auth string) error { opts := types.ImagePullOptions{ RegistryAuth: auth, } @@ -266,7 +273,7 @@ func (c Client) tryPullImageAndWait(ctx context.Context, image string, auth stri return nil } -func (c Client) tryPushImageAndWait(ctx context.Context, image string, auth string) error { +func (c Client) tryPushAndWait(ctx context.Context, image string, auth string) error { opts := types.ImagePushOptions{ RegistryAuth: auth, }