Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: support for loading images in the K3s module #1622

Merged
merged 12 commits into from
Sep 21, 2023
16 changes: 0 additions & 16 deletions docs/modules/k3s.md
Original file line number Diff line number Diff line change
Expand Up @@ -79,19 +79,3 @@ to the Kubernetes Rest Client API using a Kubernetes client. It'll be returned i
<!--codeinclude-->
[Get KubeConifg](../../modules/k3s/k3s_test.go) inside_block:GetKubeConfig
<!--/codeinclude-->

mdelapenya marked this conversation as resolved.
Show resolved Hide resolved
#### LoadImages

The `LoadImages` method loads images from a `.tar` file into the kubernetes cluster and makes them available to pods. This is useful for testing images generated locally without having to push them to a docker registry.

The `.tar` file can be created, for example, using the [docker save](https://docs.docker.com/engine/reference/commandline/save/) command:

```sh
docker save -o images.tar <image> <image> ...
```

Also, the [DockerProvider](https://pkg.go.dev/github.com/testcontainers/testcontainers-go#DockerProvider) offers methods for pulling and saving images, which can be used from the test code, as shown in the following example:

<!--codeinclude-->
[Load Images](../../modules/k3s/k3s_image_test.go) inside_block:LoadImages
<!--/codeinclude-->
28 changes: 23 additions & 5 deletions modules/k3s/k3s.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"
"fmt"
"io"
"os"
"path/filepath"

"github.com/docker/docker/api/types/container"
Expand Down Expand Up @@ -166,11 +167,28 @@ func unmarshal(bytes []byte) (*KubeConfigValue, error) {
}

// LoadImages loads images into the k3s container.
func (c *K3sContainer) LoadImages(ctx context.Context, images string) error {
imageFile := filepath.Base(images)
containerPath := fmt.Sprintf("/tmp/%s", imageFile)
func (c *K3sContainer) LoadImages(ctx context.Context, images ...string) error {
provider, err := testcontainers.ProviderDocker.GetProvider()
if err != nil {
return fmt.Errorf("getting docker provider %w", err)
}

// save image
imagesTar, err := os.CreateTemp(os.TempDir(), "images*.tar")
if err != nil {
return fmt.Errorf("creating temporary images file %w", err)
}
defer func() {
_ = os.Remove(imagesTar.Name())
}()

err := c.Container.CopyFileToContainer(ctx, images, containerPath, 0x644)
err = provider.SaveImages(context.Background(), imagesTar.Name(), images...)
if err != nil {
return fmt.Errorf("saving images %w", err)
}

containerPath := fmt.Sprintf("/tmp/%s", filepath.Base(imagesTar.Name()))
err = c.Container.CopyFileToContainer(ctx, imagesTar.Name(), containerPath, 0x644)
if err != nil {
return fmt.Errorf("copying image to container %w", err)
}
Expand All @@ -181,4 +199,4 @@ func (c *K3sContainer) LoadImages(ctx context.Context, images string) error {
}

return nil
}
}
102 changes: 52 additions & 50 deletions modules/k3s/k3s_image_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,106 +2,108 @@ package k3s_test

import (
"context"
"fmt"
"os"
"path/filepath"
"testing"
"time"

corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

"k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/clientcmd"

"github.com/testcontainers/testcontainers-go"
"github.com/testcontainers/testcontainers-go/modules/k3s"
"github.com/testcontainers/testcontainers-go/wait"
mdelapenya marked this conversation as resolved.
Show resolved Hide resolved
)

func ExampleLoadImages() {
func Test_LoadImages(t *testing.T) {
ctx := context.Background()

k3sContainer, err := RunContainer(ctx,
k3sContainer, err := k3s.RunContainer(ctx,
testcontainers.WithImage("docker.io/rancher/k3s:v1.27.1-k3s1"),
testcontainers.WithWaitStrategy(wait.ForLog(".*Node controller sync successful.*").AsRegexp()),
)
if err != nil {
panic(err)
t.Fatal(err)
}

// Clean up the container
defer func() {
if err := k3sContainer.Terminate(ctx); err != nil {
panic(err)
t.Fatal(err)
}
}()

kubeConfigYaml, err := k3sContainer.GetKubeConfig(ctx)
if err != nil {
panic(err)
t.Fatal(err)
}

restcfg, err := clientcmd.RESTConfigFromKubeConfig(kubeConfigYaml)
if err != nil {
panic(err)
t.Fatal(err)
}

k8s, err := kubernetes.NewForConfig(restcfg)
if err != nil {
panic(err)
t.Fatal(err)
}

provider, err := testcontainers.ProviderDocker.GetProvider()
if err != nil {
panic(err)
t.Fatal(err)
}

// ensure nginx image is available locally
err = provider.PullImage(context.Background(), "nginx")
if err != nil {
panic(err)
t.Fatal(err)
}

output := filepath.Join(os.TempDir(), "nginx.tar")
err = provider.SaveImages(context.Background(), output, "nginx")
if err != nil {
panic(err)
}
t.Run("Test load image not available", func(t *testing.T) {
err = k3sContainer.LoadImages(context.Background(), "nginx", "fake.registry/fake:non-existing")
if err == nil {
t.Fatal("should had failed")
}
})

err = k3sContainer.LoadImages(context.Background(), output)
if err != nil {
panic(err)
}
t.Run("Test load image in cluster", func(t *testing.T) {
err = k3sContainer.LoadImages(context.Background(), "nginx")
if err != nil {
t.Fatal(err)
}

pod := &corev1.Pod{
TypeMeta: metav1.TypeMeta{
Kind: "Pod",
APIVersion: "v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: "test-pod",
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "nginx",
Image: "nginx",
ImagePullPolicy: corev1.PullNever, // use image only if already present
pod := &corev1.Pod{
TypeMeta: metav1.TypeMeta{
Kind: "Pod",
APIVersion: "v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: "test-pod",
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "nginx",
Image: "nginx",
ImagePullPolicy: corev1.PullNever, // use image only if already present
},
},
},
},
}
}

_, err = k8s.CoreV1().Pods("default").Create(context.Background(), pod, metav1.CreateOptions{})
if err != nil {
panic(err)
}
_, err = k8s.CoreV1().Pods("default").Create(context.Background(), pod, metav1.CreateOptions{})
if err != nil {
t.Fatal(err)
}

time.Sleep(1 * time.Second)
pod, err = k8s.CoreV1().Pods("default").Get(context.Background(), "test-pod", metav1.GetOptions{})
if err != nil {
panic(err)
}
time.Sleep(1 * time.Second)
pod, err = k8s.CoreV1().Pods("default").Get(context.Background(), "test-pod", metav1.GetOptions{})
if err != nil {
t.Fatal(err)
}

if pod.Status.ContainerStatuses[0].State.Waiting.Reason == "ErrImageNeverPull" {
panic(fmt.Errorf("Image was not loaded"))
}
if pod.Status.ContainerStatuses[0].State.Waiting.Reason == "ErrImageNeverPull" {
mdelapenya marked this conversation as resolved.
Show resolved Hide resolved
t.Fatal("Image was not loaded")
}
})
}
Loading