Merge pull request #835 from HeavyWombat/add/local-source

Local source code feature (using bundle images)

.github/workflows/ci.yml

@@ -12,7 +12,7 @@ jobs:
     unit:
         strategy:
           matrix:
-            go-version: [1.15.x]
+            go-version: [1.16.x]
             os: [ubuntu-latest]
         runs-on: ${{ matrix.os }}
         steps:
@@ -30,7 +30,7 @@ jobs:
         strategy:
           fail-fast: false
           matrix:
-            go-version: [1.15.x]
+            go-version: [1.16.x]
             os: [ubuntu-latest]
             kubernetes:
               # Only v1.20 is currently enabled because of the flakiness in the tests, specifically API calls failing with "etcdserver: request timed out"
@@ -92,7 +92,7 @@ jobs:
         strategy:
           fail-fast: false
           matrix:
-            go-version: [1.15.x]
+            go-version: [1.16.x]
             os: [ubuntu-latest]
             kubernetes:
               # Only v1.20 is currently enabled because of the flakiness in the tests, specifically API calls failing with "etcdserver: request timed out"

.github/workflows/verify.yaml

@@ -20,7 +20,7 @@ jobs:
     - name: Set up Go 1.x
       uses: actions/setup-go@v2
       with:
-        go-version: ^1.15
+        go-version: ^1.16
 
     - name: Check out code into the Go module directory
       uses: actions/checkout@v2

cmd/bundle/bundle_suite_test.go

@@ -0,0 +1,17 @@
+// Copyright The Shipwright Contributors
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package main_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestBundle(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Bundle Suite")
+}

cmd/bundle/main.go

@@ -0,0 +1,124 @@
+// Copyright The Shipwright Contributors
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/docker/cli/cli/config"
+	"github.com/google/go-containerregistry/pkg/authn"
+	"github.com/google/go-containerregistry/pkg/name"
+	"github.com/google/go-containerregistry/pkg/v1/remote"
+	"github.com/shipwright-io/build/pkg/bundle"
+	"github.com/spf13/pflag"
+)
+
+var flagValues struct {
+	image      string
+	target     string
+	secretPath string
+}
+
+func init() {
+	pflag.StringVar(&flagValues.image, "image", "", "Location of the bundle image (mandatory)")
+	pflag.StringVar(&flagValues.target, "target", "/workspace/source", "The target directory to place the code")
+	pflag.StringVar(&flagValues.secretPath, "secret-path", "", "A directory that contains access credentials (optional)")
+}
+
+func main() {
+	if err := Do(context.Background()); err != nil {
+		log.Fatal(err.Error())
+	}
+}
+
+// Do is the main entry point of the bundle command
+func Do(ctx context.Context) error {
+	pflag.Parse()
+
+	if flagValues.image == "" {
+		return fmt.Errorf("mandatory flag --image is not set")
+	}
+
+	ref, err := name.ParseReference(flagValues.image)
+	if err != nil {
+		return err
+	}
+
+	auth, err := resolveAuthBasedOnTarget(ref)
+	if err != nil {
+		return err
+	}
+
+	log.Printf("Pulling image %q", ref)
+	if err := bundle.PullAndUnpack(
+		ref,
+		flagValues.target,
+		remote.WithContext(ctx),
+		remote.WithAuth(auth)); err != nil {
+		return err
+	}
+
+	log.Printf("Image content was extracted to %s\n", flagValues.target)
+	return nil
+}
+
+func resolveAuthBasedOnTarget(ref name.Reference) (authn.Authenticator, error) {
+	// In case no secret is mounted, use anonymous
+	if flagValues.secretPath == "" {
+		log.Printf("No access credentials provided, using anonymous mode")
+		return authn.Anonymous, nil
+	}
+
+	// Read the registry credentials from the well-known location
+	file, err := os.Open(filepath.Join(flagValues.secretPath, ".dockerconfigjson"))
+	if err != nil {
+		return nil, err
+	}
+	defer file.Close()
+
+	cf, err := config.LoadFromReader(file)
+	if err != nil {
+		return nil, err
+	}
+
+	// Look-up the respective registry server inside the credentials
+	registryName := ref.Context().RegistryStr()
+	if registryName == name.DefaultRegistry {
+		registryName = authn.DefaultAuthKey
+	}
+
+	authConfig, err := cf.GetAuthConfig(registryName)
+	if err != nil {
+		return nil, err
+	}
+
+	// Return an error in case the credentials do not match the desired
+	// registry and list all servers that actually are available
+	if authConfig.ServerAddress != registryName {
+		var servers []string
+		for name := range cf.GetAuthConfigs() {
+			servers = append(servers, name)
+		}
+
+		return nil, fmt.Errorf("failed to find registry credentials for %s, credentials are available for: %s",
+			registryName,
+			strings.Join(servers, ", "),
+		)
+	}
+
+	log.Printf("Using provided access credentials for %s", registryName)
+	return authn.FromConfig(authn.AuthConfig{
+		Username:      authConfig.Username,
+		Password:      authConfig.Password,
+		Auth:          authConfig.Auth,
+		IdentityToken: authConfig.IdentityToken,
+		RegistryToken: authConfig.RegistryToken,
+	}), nil
+}

cmd/bundle/main_test.go

@@ -0,0 +1,57 @@
+// Copyright The Shipwright Contributors
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package main_test
+
+import (
+	"context"
+	"io/ioutil"
+	"log"
+	"os"
+	"path/filepath"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	. "github.com/shipwright-io/build/cmd/bundle"
+)
+
+var _ = Describe("Bundle Loader", func() {
+	var run = func(args ...string) error {
+		log.SetOutput(ioutil.Discard)
+		os.Args = append([]string{"tool"}, args...)
+		return Do(context.Background())
+	}
+
+	var withTempDir = func(f func(target string)) {
+		path, err := ioutil.TempDir(os.TempDir(), "bundle")
+		Expect(err).ToNot(HaveOccurred())
+		defer os.RemoveAll(path)
+
+		f(path)
+	}
+
+	Context("Error cases", func() {
+		It("should fail in case the image is not specified", func() {
+			Expect(run(
+				"--image", "",
+			)).To(HaveOccurred())
+		})
+	})
+
+	Context("Pulling image anonymously", func() {
+		const exampleImage = "quay.io/shipwright/source-bundle:latest"
+
+		It("should pull and unbundle an image from a public registry", func() {
+			withTempDir(func(target string) {
+				Expect(run(
+					"--image", exampleImage,
+					"--target", target,
+				)).ToNot(HaveOccurred())
+
+				Expect(filepath.Join(target, "LICENSE")).To(BeAnExistingFile())
+			})
+		})
+	})
+})

deploy/500-controller.yaml

@@ -36,6 +36,8 @@ spec:
               value: ko://github.com/shipwright-io/build/cmd/git
             - name: MUTATE_IMAGE_CONTAINER_IMAGE
               value: ko://github.com/shipwright-io/build/cmd/mutate-image
+            - name: BUNDLE_CONTAINER_IMAGE
+              value: ko://github.com/shipwright-io/build/cmd/bundle
           ports:
             - containerPort: 8383
               name: metrics-port

deploy/crds/shipwright.io_buildruns.yaml

@@ -202,6 +202,15 @@ spec:
                   source:
                     description: Source refers to the Git repository containing the source code to be built.
                     properties:
+                      bundleContainer:
+                        description: BundleContainer
+                        properties:
+                          image:
+                            description: Image reference, i.e. quay.io/org/image:tag
+                            type: string
+                        required:
+                        - image
+                        type: object
                       contextDir:
                         description: ContextDir is a path to subfolder in the repo. Optional.
                         type: string
@@ -218,8 +227,6 @@ spec:
                       url:
                         description: URL describes the URL of the Git repository.
                         type: string
-                    required:
-                    - url
                     type: object
                   sources:
                     description: Sources slice of BuildSource, defining external build artifacts complementary to VCS (`.spec.source`) data.

deploy/crds/shipwright.io_builds.yaml

@@ -125,6 +125,15 @@ spec:
               source:
                 description: Source refers to the Git repository containing the source code to be built.
                 properties:
+                  bundleContainer:
+                    description: BundleContainer
+                    properties:
+                      image:
+                        description: Image reference, i.e. quay.io/org/image:tag
+                        type: string
+                    required:
+                    - image
+                    type: object
                   contextDir:
                     description: ContextDir is a path to subfolder in the repo. Optional.
                     type: string
@@ -141,8 +150,6 @@ spec:
                   url:
                     description: URL describes the URL of the Git repository.
                     type: string
-                required:
-                - url
                 type: object
               sources:
                 description: Sources slice of BuildSource, defining external build artifacts complementary to VCS (`.spec.source`) data.

go.mod

@@ -1,6 +1,6 @@
 module github.com/shipwright-io/build
 
-go 1.15
+go 1.16
 
 require (
 	github.com/docker/cli v20.10.7+incompatible