Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

e2e: Add new option to test rootless docker #1742

Merged
merged 1 commit into from
Aug 24, 2022
Merged

e2e: Add new option to test rootless docker #1742

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion acceptance/testdata/runnerdeploy.envsubst.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ spec:

# # Uncomment the below to enable the kubernetes container mode
# # See https://github.com/actions-runner-controller/actions-runner-controller#runner-with-k8s-jobs
containerMode: kubernetes
containerMode: ${RUNNER_CONTAINER_MODE}
workVolumeClaimTemplate:
accessModes:
- ReadWriteOnce
Expand Down
194 changes: 160 additions & 34 deletions test/e2e/e2e_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,10 +123,18 @@ func TestE2E(t *testing.T) {
})
}

if t.Failed() {
return
}

t.Run("install actions-runner-controller v0.24.1", func(t *testing.T) {
env.installActionsRunnerController(t, "summerwind/actions-runner-controller", "v0.24.1", testID)
})

if t.Failed() {
return
}

t.Run("install argo-tunnel", func(t *testing.T) {
env.installArgoTunnel(t)
})
Expand All @@ -137,6 +145,10 @@ func TestE2E(t *testing.T) {
})
}

if t.Failed() {
return
}

t.Run("deploy runners", func(t *testing.T) {
env.deploy(t, RunnerSets, testID)
})
Expand All @@ -147,6 +159,10 @@ func TestE2E(t *testing.T) {
})
}

if t.Failed() {
return
}

t.Run("install edge actions-runner-controller", func(t *testing.T) {
env.installActionsRunnerController(t, vars.controllerImageRepo, vars.controllerImageTag, testID)
})
Expand Down Expand Up @@ -183,10 +199,18 @@ func TestE2E(t *testing.T) {
})
}

if t.Failed() {
return
}

t.Run("install actions-runner-controller v0.24.1", func(t *testing.T) {
env.installActionsRunnerController(t, "summerwind/actions-runner-controller", "v0.24.1", testID)
})

if t.Failed() {
return
}

t.Run("install argo-tunnel", func(t *testing.T) {
env.installArgoTunnel(t)
})
Expand All @@ -197,6 +221,10 @@ func TestE2E(t *testing.T) {
})
}

if t.Failed() {
return
}

t.Run("deploy runners", func(t *testing.T) {
env.deploy(t, RunnerDeployments, testID)
})
Expand All @@ -207,6 +235,10 @@ func TestE2E(t *testing.T) {
})
}

if t.Failed() {
return
}

t.Run("install edge actions-runner-controller", func(t *testing.T) {
env.installActionsRunnerController(t, vars.controllerImageRepo, vars.controllerImageTag, testID)
})
Expand Down Expand Up @@ -252,6 +284,8 @@ type env struct {
scaleDownDelaySecondsAfterScaleOut int64
minReplicas int64
dockerdWithinRunnerContainer bool
rootlessDocker bool
containerMode string
remoteKubeconfig string
imagePullSecretName string
imagePullPolicy string
Expand All @@ -263,8 +297,9 @@ type env struct {
type vars struct {
controllerImageRepo, controllerImageTag string

runnerImageRepo string
runnerDindImageRepo string
runnerImageRepo string
runnerDindImageRepo string
runnerRootlessDindImageRepo string

prebuildImages []testing.ContainerImage
builds []testing.DockerBuild
Expand All @@ -278,20 +313,23 @@ func buildVars(repo string) vars {
}

var (
controllerImageRepo = repo + "/actions-runner-controller"
controllerImageTag = "e2e"
controllerImage = testing.Img(controllerImageRepo, controllerImageTag)
runnerImageRepo = repo + "/actions-runner"
runnerDindImageRepo = repo + "/actions-runner-dind"
runnerImageTag = "e2e"
runnerImage = testing.Img(runnerImageRepo, runnerImageTag)
runnerDindImage = testing.Img(runnerDindImageRepo, runnerImageTag)
controllerImageRepo = repo + "/actions-runner-controller"
controllerImageTag = "e2e"
controllerImage = testing.Img(controllerImageRepo, controllerImageTag)
runnerImageRepo = repo + "/actions-runner"
runnerDindImageRepo = repo + "/actions-runner-dind"
runnerRootlessDindImageRepo = repo + "/actions-runner-rootless-dind"
runnerImageTag = "e2e"
runnerImage = testing.Img(runnerImageRepo, runnerImageTag)
runnerDindImage = testing.Img(runnerDindImageRepo, runnerImageTag)
runnerRootlessDindImage = testing.Img(runnerRootlessDindImageRepo, runnerImageTag)
)

var vs vars

vs.controllerImageRepo, vs.controllerImageTag = controllerImageRepo, controllerImageTag
vs.runnerDindImageRepo = runnerDindImageRepo
vs.runnerRootlessDindImageRepo = runnerRootlessDindImageRepo
vs.runnerImageRepo = runnerImageRepo

// vs.controllerImage, vs.controllerImageTag
Expand All @@ -300,6 +338,7 @@ func buildVars(repo string) vars {
controllerImage,
runnerImage,
runnerDindImage,
runnerRootlessDindImage,
}

vs.builds = []testing.DockerBuild{
Expand Down Expand Up @@ -331,6 +370,17 @@ func buildVars(repo string) vars {
Image: runnerDindImage,
EnableBuildX: true,
},
{
Dockerfile: "../../runner/actions-runner-dind-rootless.dockerfile",
Args: []testing.BuildArg{
{
Name: "RUNNER_VERSION",
Value: "2.294.0",
},
},
Image: runnerRootlessDindImage,
EnableBuildX: true,
},
}

vs.commonScriptEnv = []string{
Expand Down Expand Up @@ -397,6 +447,16 @@ func initTestEnv(t *testing.T, k8sMinorVer string, vars vars) *env {
panic(fmt.Sprintf("unable to parse bool from TEST_RUNNER_DOCKERD_WITHIN_RUNNER_CONTAINER: %v", err))
}

e.rootlessDocker, err = strconv.ParseBool(testing.Getenv(t, "TEST_RUNNER_ROOTLESS_DOCKER", "false"))
if err != nil {
panic(fmt.Sprintf("unable to parse bool from TEST_RUNNER_ROOTLESS_DOCKER: %v", err))
}

e.containerMode = testing.Getenv(t, "TEST_CONTAINER_MODE", "")
if err != nil {
panic(fmt.Sprintf("unable to parse bool from TEST_CONTAINER_MODE: %v", err))
}

return e
}

Expand Down Expand Up @@ -527,13 +587,27 @@ func (e *env) do(t *testing.T, op string, kind DeployKind, testID string) {
fmt.Sprintf("REPO_RUNNER_MIN_REPLICAS=%d", e.minReplicas),
fmt.Sprintf("ORG_RUNNER_MIN_REPLICAS=%d", e.minReplicas),
fmt.Sprintf("ENTERPRISE_RUNNER_MIN_REPLICAS=%d", e.minReplicas),
"RUNNER_CONTAINER_MODE=" + e.containerMode,
}

if e.dockerdWithinRunnerContainer && e.containerMode == "kubernetes" {
t.Fatalf("TEST_RUNNER_DOCKERD_WITHIN_RUNNER_CONTAINER cannot be set along with TEST_CONTAINER_MODE=kubernetes")
t.FailNow()
}

if e.dockerdWithinRunnerContainer {
varEnv = append(varEnv,
"RUNNER_DOCKERD_WITHIN_RUNNER_CONTAINER=true",
"RUNNER_NAME="+e.vars.runnerDindImageRepo,
)
if e.rootlessDocker {
varEnv = append(varEnv,
"RUNNER_NAME="+e.vars.runnerRootlessDindImageRepo,
)
} else {
varEnv = append(varEnv,
"RUNNER_NAME="+e.vars.runnerDindImageRepo,
)
}
} else {
varEnv = append(varEnv,
"RUNNER_DOCKERD_WITHIN_RUNNER_CONTAINER=false",
Expand Down Expand Up @@ -583,7 +657,7 @@ func (e *env) createControllerNamespaceAndServiceAccount(t *testing.T) {
func (e *env) installActionsWorkflow(t *testing.T, kind DeployKind, testID string) {
t.Helper()

installActionsWorkflow(t, e.testName+" "+testID, e.runnerLabel(testID), testResultCMNamePrefix, e.repoToCommit, kind, e.testJobs(testID))
installActionsWorkflow(t, e.testName+" "+testID, e.runnerLabel(testID), testResultCMNamePrefix, e.repoToCommit, kind, e.testJobs(testID), !e.rootlessDocker)
}

func (e *env) testJobs(testID string) []job {
Expand Down Expand Up @@ -624,7 +698,8 @@ func createTestJobs(id, testResultCMNamePrefix string, numJobs int) []job {

const Branch = "main"

func installActionsWorkflow(t *testing.T, testName, runnerLabel, testResultCMNamePrefix, testRepo string, kind DeployKind, testJobs []job) {
// useSudo also implies rootful docker and the use of buildx cache export/import
func installActionsWorkflow(t *testing.T, testName, runnerLabel, testResultCMNamePrefix, testRepo string, kind DeployKind, testJobs []job, useSudo bool) {
t.Helper()

ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
Expand Down Expand Up @@ -655,11 +730,41 @@ func installActionsWorkflow(t *testing.T, testName, runnerLabel, testResultCMNam
},
}

var sudo string
if useSudo {
sudo = "sudo "
}

if !kubernetesContainerMode {
if kind == RunnerDeployments {
steps = append(steps,
testing.Step{
Run: "sudo mkdir -p \"${RUNNER_TOOL_CACHE}\" \"${HOME}/.cache\" \"/var/lib/docker\"",
Run: sudo + "mkdir -p \"${RUNNER_TOOL_CACHE}\" \"${HOME}/.cache\"",
},
)

if useSudo {
steps = append(steps,
testing.Step{
// This might be the easiest way to handle permissions without use of securityContext
// https://stackoverflow.com/questions/50156124/kubernetes-nfs-persistent-volumes-permission-denied#comment107483717_53186320
Run: sudo + "mkdir -p \"/var/lib/docker\"",
},
)
}
}

steps = append(steps,
testing.Step{
// This might be the easiest way to handle permissions without use of securityContext
// https://stackoverflow.com/questions/50156124/kubernetes-nfs-persistent-volumes-permission-denied#comment107483717_53186320
Run: sudo + "chmod 777 -R \"${RUNNER_TOOL_CACHE}\" \"${HOME}/.cache\"",
},
)
if useSudo {
steps = append(steps,
testing.Step{
Run: sudo + "chmod 777 -R \"/var/lib/docker\"",
},
)
}
Expand All @@ -668,12 +773,12 @@ func installActionsWorkflow(t *testing.T, testName, runnerLabel, testResultCMNam
testing.Step{
// This might be the easiest way to handle permissions without use of securityContext
// https://stackoverflow.com/questions/50156124/kubernetes-nfs-persistent-volumes-permission-denied#comment107483717_53186320
Run: "sudo chmod 777 -R \"${RUNNER_TOOL_CACHE}\" \"${HOME}/.cache\" \"/var/lib/docker\"",
Run: "ls -lah \"${RUNNER_TOOL_CACHE}\" \"${HOME}/.cache\"",
},
testing.Step{
// This might be the easiest way to handle permissions without use of securityContext
// https://stackoverflow.com/questions/50156124/kubernetes-nfs-persistent-volumes-permission-denied#comment107483717_53186320
Run: "ls -lah \"${RUNNER_TOOL_CACHE}\" \"${HOME}/.cache\" \"/var/lib/docker\"",
Run: "ls -lah \"/var/lib/docker\" || echo ls failed.",
},
testing.Step{
Uses: "actions/setup-go@v3",
Expand All @@ -694,6 +799,26 @@ func installActionsWorkflow(t *testing.T, testName, runnerLabel, testResultCMNam
)

if !kubernetesContainerMode {
setupBuildXActionWith := &testing.With{
BuildkitdFlags: "--debug",
Endpoint: "mycontext",
// As the consequence of setting `install: false`, it doesn't install buildx as an alias to `docker build`
// so we need to use `docker buildx build` in the next step
Install: false,
}
var dockerBuildCache, dockerfile string
if useSudo {
// This needs to be set only when rootful docker mode.
// When rootless, we need to use the `docker` buildx driver, which doesn't support cache export
// so we end up with the below error on docker-build:
// error: cache export feature is currently not supported for docker driver. Please switch to a different driver (eg. "docker buildx create --use")
dockerBuildCache = "--cache-from=type=local,src=/home/runner/.cache/buildx " +
"--cache-to=type=local,dest=/home/runner/.cache/buildx-new,mode=max "
dockerfile = "Dockerfile"
} else {
setupBuildXActionWith.Driver = "docker"
dockerfile = "Dockerfile.nocache"
}
steps = append(steps,
testing.Step{
// https://github.com/docker/buildx/issues/413#issuecomment-710660155
Expand All @@ -707,28 +832,29 @@ func installActionsWorkflow(t *testing.T, testName, runnerLabel, testResultCMNam
testing.Step{
Name: "Set up Docker Buildx",
Uses: "docker/setup-buildx-action@v1",
With: &testing.With{
BuildkitdFlags: "--debug",
Endpoint: "mycontext",
// As the consequence of setting `install: false`, it doesn't install buildx as an alias to `docker build`
// so we need to use `docker buildx build` in the next step
Install: false,
},
With: setupBuildXActionWith,
},
testing.Step{
Run: "docker buildx build --platform=linux/amd64 " +
"--cache-from=type=local,src=/home/runner/.cache/buildx " +
"--cache-to=type=local,dest=/home/runner/.cache/buildx-new,mode=max " +
".",
},
testing.Step{
// https://github.com/docker/build-push-action/blob/master/docs/advanced/cache.md#local-cache
// See https://github.com/moby/buildkit/issues/1896 for why this is needed
Run: "rm -rf /home/runner/.cache/buildx && mv /home/runner/.cache/buildx-new /home/runner/.cache/buildx",
},
testing.Step{
Run: "ls -lah /home/runner/.cache/*",
dockerBuildCache +
fmt.Sprintf("-f %s .", dockerfile),
},
)

if useSudo {
steps = append(steps,
testing.Step{
// https://github.com/docker/build-push-action/blob/master/docs/advanced/cache.md#local-cache
// See https://github.com/moby/buildkit/issues/1896 for why this is needed
Run: "rm -rf /home/runner/.cache/buildx && mv /home/runner/.cache/buildx-new /home/runner/.cache/buildx",
},
testing.Step{
Run: "ls -lah /home/runner/.cache/*",
},
)
}

steps = append(steps,
testing.Step{
Uses: "azure/setup-kubectl@v1",
With: &testing.With{
Expand Down
3 changes: 3 additions & 0 deletions testing/workflow.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,4 +52,7 @@ type With struct {
// This can be either the address or the context name
// https://github.com/docker/buildx/blob/master/docs/reference/buildx_create.md#description
Endpoint string `json:"endpoint,omitempty"`
// Needs to be "docker" in rootless mode
// https://stackoverflow.com/questions/66142872/how-to-solve-error-with-rootless-docker-in-github-actions-self-hosted-runner-wr
Driver string `json:"driver,omitempty"`
}