Skip to content

Commit

Permalink
e2e: Add new option to test rootless docker (#1742)
Browse files Browse the repository at this point in the history 
Related to #1644

Signed-off-by: Yusuke Kuoka <ykuoka@gmail.com>

Signed-off-by: Yusuke Kuoka <ykuoka@gmail.com>
  • Loading branch information
@mumoshu
mumoshu committed Aug 24, 2022
1 parent 0cac005 commit ea94b3c
Show file tree
Hide file tree
Showing 3 changed files with 164 additions and 35 deletions.
2 changes: 1 addition & 1 deletion acceptance/testdata/runnerdeploy.envsubst.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ spec:

# # Uncomment the below to enable the kubernetes container mode
# # See https://github.com/actions-runner-controller/actions-runner-controller#runner-with-k8s-jobs
containerMode: kubernetes
containerMode: ${RUNNER_CONTAINER_MODE}
workVolumeClaimTemplate:
accessModes:
- ReadWriteOnce
Expand Down
194 changes: 160 additions & 34 deletions test/e2e/e2e_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -123,10 +123,18 @@ func TestE2E(t *testing.T) {
})
}

if t.Failed() {
return
}

t.Run("install actions-runner-controller v0.24.1", func(t *testing.T) {
env.installActionsRunnerController(t, "summerwind/actions-runner-controller", "v0.24.1", testID)
})

if t.Failed() {
return
}

t.Run("install argo-tunnel", func(t *testing.T) {
env.installArgoTunnel(t)
})
Expand All @@ -137,6 +145,10 @@ func TestE2E(t *testing.T) {
})
}

if t.Failed() {
return
}

t.Run("deploy runners", func(t *testing.T) {
env.deploy(t, RunnerSets, testID)
})
Expand All @@ -147,6 +159,10 @@ func TestE2E(t *testing.T) {
})
}

if t.Failed() {
return
}

t.Run("install edge actions-runner-controller", func(t *testing.T) {
env.installActionsRunnerController(t, vars.controllerImageRepo, vars.controllerImageTag, testID)
})
Expand Down Expand Up @@ -183,10 +199,18 @@ func TestE2E(t *testing.T) {
})
}

if t.Failed() {
return
}

t.Run("install actions-runner-controller v0.24.1", func(t *testing.T) {
env.installActionsRunnerController(t, "summerwind/actions-runner-controller", "v0.24.1", testID)
})

if t.Failed() {
return
}

t.Run("install argo-tunnel", func(t *testing.T) {
env.installArgoTunnel(t)
})
Expand All @@ -197,6 +221,10 @@ func TestE2E(t *testing.T) {
})
}

if t.Failed() {
return
}

t.Run("deploy runners", func(t *testing.T) {
env.deploy(t, RunnerDeployments, testID)
})
Expand All @@ -207,6 +235,10 @@ func TestE2E(t *testing.T) {
})
}

if t.Failed() {
return
}

t.Run("install edge actions-runner-controller", func(t *testing.T) {
env.installActionsRunnerController(t, vars.controllerImageRepo, vars.controllerImageTag, testID)
})
Expand Down Expand Up @@ -252,6 +284,8 @@ type env struct {
scaleDownDelaySecondsAfterScaleOut int64
minReplicas int64
dockerdWithinRunnerContainer bool
rootlessDocker bool
containerMode string
remoteKubeconfig string
imagePullSecretName string
imagePullPolicy string
Expand All @@ -263,8 +297,9 @@ type env struct {
type vars struct {
controllerImageRepo, controllerImageTag string

runnerImageRepo string
runnerDindImageRepo string
runnerImageRepo string
runnerDindImageRepo string
runnerRootlessDindImageRepo string

prebuildImages []testing.ContainerImage
builds []testing.DockerBuild
Expand All @@ -278,20 +313,23 @@ func buildVars(repo string) vars {
}

var (
controllerImageRepo = repo + "/actions-runner-controller"
controllerImageTag = "e2e"
controllerImage = testing.Img(controllerImageRepo, controllerImageTag)
runnerImageRepo = repo + "/actions-runner"
runnerDindImageRepo = repo + "/actions-runner-dind"
runnerImageTag = "e2e"
runnerImage = testing.Img(runnerImageRepo, runnerImageTag)
runnerDindImage = testing.Img(runnerDindImageRepo, runnerImageTag)
controllerImageRepo = repo + "/actions-runner-controller"
controllerImageTag = "e2e"
controllerImage = testing.Img(controllerImageRepo, controllerImageTag)
runnerImageRepo = repo + "/actions-runner"
runnerDindImageRepo = repo + "/actions-runner-dind"
runnerRootlessDindImageRepo = repo + "/actions-runner-rootless-dind"
runnerImageTag = "e2e"
runnerImage = testing.Img(runnerImageRepo, runnerImageTag)
runnerDindImage = testing.Img(runnerDindImageRepo, runnerImageTag)
runnerRootlessDindImage = testing.Img(runnerRootlessDindImageRepo, runnerImageTag)
)

var vs vars

vs.controllerImageRepo, vs.controllerImageTag = controllerImageRepo, controllerImageTag
vs.runnerDindImageRepo = runnerDindImageRepo
vs.runnerRootlessDindImageRepo = runnerRootlessDindImageRepo
vs.runnerImageRepo = runnerImageRepo

// vs.controllerImage, vs.controllerImageTag
Expand All @@ -300,6 +338,7 @@ func buildVars(repo string) vars {
controllerImage,
runnerImage,
runnerDindImage,
runnerRootlessDindImage,
}

vs.builds = []testing.DockerBuild{
Expand Down Expand Up @@ -331,6 +370,17 @@ func buildVars(repo string) vars {
Image: runnerDindImage,
EnableBuildX: true,
},
{
Dockerfile: "../../runner/actions-runner-dind-rootless.dockerfile",
Args: []testing.BuildArg{
{
Name: "RUNNER_VERSION",
Value: "2.294.0",
},
},
Image: runnerRootlessDindImage,
EnableBuildX: true,
},
}

vs.commonScriptEnv = []string{
Expand Down Expand Up @@ -397,6 +447,16 @@ func initTestEnv(t *testing.T, k8sMinorVer string, vars vars) *env {
panic(fmt.Sprintf("unable to parse bool from TEST_RUNNER_DOCKERD_WITHIN_RUNNER_CONTAINER: %v", err))
}

e.rootlessDocker, err = strconv.ParseBool(testing.Getenv(t, "TEST_RUNNER_ROOTLESS_DOCKER", "false"))
if err != nil {
panic(fmt.Sprintf("unable to parse bool from TEST_RUNNER_ROOTLESS_DOCKER: %v", err))
}

e.containerMode = testing.Getenv(t, "TEST_CONTAINER_MODE", "")
if err != nil {
panic(fmt.Sprintf("unable to parse bool from TEST_CONTAINER_MODE: %v", err))
}

return e
}

Expand Down Expand Up @@ -527,13 +587,27 @@ func (e *env) do(t *testing.T, op string, kind DeployKind, testID string) {
fmt.Sprintf("REPO_RUNNER_MIN_REPLICAS=%d", e.minReplicas),
fmt.Sprintf("ORG_RUNNER_MIN_REPLICAS=%d", e.minReplicas),
fmt.Sprintf("ENTERPRISE_RUNNER_MIN_REPLICAS=%d", e.minReplicas),
"RUNNER_CONTAINER_MODE=" + e.containerMode,
}

if e.dockerdWithinRunnerContainer && e.containerMode == "kubernetes" {
t.Fatalf("TEST_RUNNER_DOCKERD_WITHIN_RUNNER_CONTAINER cannot be set along with TEST_CONTAINER_MODE=kubernetes")
t.FailNow()
}

if e.dockerdWithinRunnerContainer {
varEnv = append(varEnv,
"RUNNER_DOCKERD_WITHIN_RUNNER_CONTAINER=true",
"RUNNER_NAME="+e.vars.runnerDindImageRepo,
)
if e.rootlessDocker {
varEnv = append(varEnv,
"RUNNER_NAME="+e.vars.runnerRootlessDindImageRepo,
)
} else {
varEnv = append(varEnv,
"RUNNER_NAME="+e.vars.runnerDindImageRepo,
)
}
} else {
varEnv = append(varEnv,
"RUNNER_DOCKERD_WITHIN_RUNNER_CONTAINER=false",
Expand Down Expand Up @@ -583,7 +657,7 @@ func (e *env) createControllerNamespaceAndServiceAccount(t *testing.T) {
func (e *env) installActionsWorkflow(t *testing.T, kind DeployKind, testID string) {
t.Helper()

installActionsWorkflow(t, e.testName+" "+testID, e.runnerLabel(testID), testResultCMNamePrefix, e.repoToCommit, kind, e.testJobs(testID))
installActionsWorkflow(t, e.testName+" "+testID, e.runnerLabel(testID), testResultCMNamePrefix, e.repoToCommit, kind, e.testJobs(testID), !e.rootlessDocker)
}

func (e *env) testJobs(testID string) []job {
Expand Down Expand Up @@ -624,7 +698,8 @@ func createTestJobs(id, testResultCMNamePrefix string, numJobs int) []job {

const Branch = "main"

func installActionsWorkflow(t *testing.T, testName, runnerLabel, testResultCMNamePrefix, testRepo string, kind DeployKind, testJobs []job) {
// useSudo also implies rootful docker and the use of buildx cache export/import
func installActionsWorkflow(t *testing.T, testName, runnerLabel, testResultCMNamePrefix, testRepo string, kind DeployKind, testJobs []job, useSudo bool) {
t.Helper()

ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
Expand Down Expand Up @@ -655,11 +730,41 @@ func installActionsWorkflow(t *testing.T, testName, runnerLabel, testResultCMNam
},
}

var sudo string
if useSudo {
sudo = "sudo "
}

if !kubernetesContainerMode {
if kind == RunnerDeployments {
steps = append(steps,
testing.Step{
Run: "sudo mkdir -p \"${RUNNER_TOOL_CACHE}\" \"${HOME}/.cache\" \"/var/lib/docker\"",
Run: sudo + "mkdir -p \"${RUNNER_TOOL_CACHE}\" \"${HOME}/.cache\"",
},
)

if useSudo {
steps = append(steps,
testing.Step{
// This might be the easiest way to handle permissions without use of securityContext
// https://stackoverflow.com/questions/50156124/kubernetes-nfs-persistent-volumes-permission-denied#comment107483717_53186320
Run: sudo + "mkdir -p \"/var/lib/docker\"",
},
)
}
}

steps = append(steps,
testing.Step{
// This might be the easiest way to handle permissions without use of securityContext
// https://stackoverflow.com/questions/50156124/kubernetes-nfs-persistent-volumes-permission-denied#comment107483717_53186320
Run: sudo + "chmod 777 -R \"${RUNNER_TOOL_CACHE}\" \"${HOME}/.cache\"",
},
)
if useSudo {
steps = append(steps,
testing.Step{
Run: sudo + "chmod 777 -R \"/var/lib/docker\"",
},
)
}
Expand All @@ -668,12 +773,12 @@ func installActionsWorkflow(t *testing.T, testName, runnerLabel, testResultCMNam
testing.Step{
// This might be the easiest way to handle permissions without use of securityContext
// https://stackoverflow.com/questions/50156124/kubernetes-nfs-persistent-volumes-permission-denied#comment107483717_53186320
Run: "sudo chmod 777 -R \"${RUNNER_TOOL_CACHE}\" \"${HOME}/.cache\" \"/var/lib/docker\"",
Run: "ls -lah \"${RUNNER_TOOL_CACHE}\" \"${HOME}/.cache\"",
},
testing.Step{
// This might be the easiest way to handle permissions without use of securityContext
// https://stackoverflow.com/questions/50156124/kubernetes-nfs-persistent-volumes-permission-denied#comment107483717_53186320
Run: "ls -lah \"${RUNNER_TOOL_CACHE}\" \"${HOME}/.cache\" \"/var/lib/docker\"",
Run: "ls -lah \"/var/lib/docker\" || echo ls failed.",
},
testing.Step{
Uses: "actions/setup-go@v3",
Expand All @@ -694,6 +799,26 @@ func installActionsWorkflow(t *testing.T, testName, runnerLabel, testResultCMNam
)

if !kubernetesContainerMode {
setupBuildXActionWith := &testing.With{
BuildkitdFlags: "--debug",
Endpoint: "mycontext",
// As the consequence of setting `install: false`, it doesn't install buildx as an alias to `docker build`
// so we need to use `docker buildx build` in the next step
Install: false,
}
var dockerBuildCache, dockerfile string
if useSudo {
// This needs to be set only when rootful docker mode.
// When rootless, we need to use the `docker` buildx driver, which doesn't support cache export
// so we end up with the below error on docker-build:
// error: cache export feature is currently not supported for docker driver. Please switch to a different driver (eg. "docker buildx create --use")
dockerBuildCache = "--cache-from=type=local,src=/home/runner/.cache/buildx " +
"--cache-to=type=local,dest=/home/runner/.cache/buildx-new,mode=max "
dockerfile = "Dockerfile"
} else {
setupBuildXActionWith.Driver = "docker"
dockerfile = "Dockerfile.nocache"
}
steps = append(steps,
testing.Step{
// https://github.com/docker/buildx/issues/413#issuecomment-710660155
Expand All @@ -707,28 +832,29 @@ func installActionsWorkflow(t *testing.T, testName, runnerLabel, testResultCMNam
testing.Step{
Name: "Set up Docker Buildx",
Uses: "docker/setup-buildx-action@v1",
With: &testing.With{
BuildkitdFlags: "--debug",
Endpoint: "mycontext",
// As the consequence of setting `install: false`, it doesn't install buildx as an alias to `docker build`
// so we need to use `docker buildx build` in the next step
Install: false,
},
With: setupBuildXActionWith,
},
testing.Step{
Run: "docker buildx build --platform=linux/amd64 " +
"--cache-from=type=local,src=/home/runner/.cache/buildx " +
"--cache-to=type=local,dest=/home/runner/.cache/buildx-new,mode=max " +
".",
},
testing.Step{
// https://github.com/docker/build-push-action/blob/master/docs/advanced/cache.md#local-cache
// See https://github.com/moby/buildkit/issues/1896 for why this is needed
Run: "rm -rf /home/runner/.cache/buildx && mv /home/runner/.cache/buildx-new /home/runner/.cache/buildx",
},
testing.Step{
Run: "ls -lah /home/runner/.cache/*",
dockerBuildCache +
fmt.Sprintf("-f %s .", dockerfile),
},
)

if useSudo {
steps = append(steps,
testing.Step{
// https://github.com/docker/build-push-action/blob/master/docs/advanced/cache.md#local-cache
// See https://github.com/moby/buildkit/issues/1896 for why this is needed
Run: "rm -rf /home/runner/.cache/buildx && mv /home/runner/.cache/buildx-new /home/runner/.cache/buildx",
},
testing.Step{
Run: "ls -lah /home/runner/.cache/*",
},
)
}

steps = append(steps,
testing.Step{
Uses: "azure/setup-kubectl@v1",
With: &testing.With{
Expand Down
3 changes: 3 additions & 0 deletions testing/workflow.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,4 +52,7 @@ type With struct {
// This can be either the address or the context name
// https://github.com/docker/buildx/blob/master/docs/reference/buildx_create.md#description
Endpoint string `json:"endpoint,omitempty"`
// Needs to be "docker" in rootless mode
// https://stackoverflow.com/questions/66142872/how-to-solve-error-with-rootless-docker-in-github-actions-self-hosted-runner-wr
Driver string `json:"driver,omitempty"`
}

0 comments on commit ea94b3c

Please sign in to comment.