chore(test): Add configurable opts to e2e tests (#239)

* Add configurable opts to e2e tests * Only run e2e nightly if on this repo and pr * Bump timeout for dmsetup I have added the initial frame for custom test flags. So far just the options to: - Skip setting up the devmapper thinpool - Skip deleting VMs - Skip all teardown steps - Set log level for containerd - Set log level for flintlockd Unfortunately custom test flags can only be added on `init()` which is bleh but I don't see another nice was of doing this. These flags can only be used while running `go test ...` directly. This means you cannot pass in flags as part of a `make` or a `docker run -it ...` command, but you can just call `go test etc` from inside the container or in the metal host and configure what you like. My next step is some more work around the python tooling so that these (and more) options can be passed up.
liquidmetal-dev · Nov 11, 2021 · 1980c5c · 1980c5c
1 parent 2c7797d
commit 1980c5c
Show file tree

Hide file tree

Showing 6 changed files with 164 additions and 69 deletions.
diff --git a/.github/workflows/nightly_e2e.yml b/.github/workflows/nightly_e2e.yml
@@ -8,6 +8,7 @@ on:
 jobs:
   e2e:
     runs-on: ubuntu-latest
+    if: ${{ github.event_name != 'pull_request' && github.repository_owner == 'weaveworks' }}
     env:
       PROJECT_NAME: "flintlock_nightly_e2e"
     name: e2e tests

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -332,42 +332,7 @@ make test
 
 ### Running the end to end tests
 
-There are several ways to run the end to end tests.
-
-#### In your local environment
-
-```
-make test-e2e
-```
-
-This will run the tests directly on your host with minimal fuss.
-You must ensure that you have installed all the dependencies per the
-[Quick-Start guide][quick-start].
-
-#### In a local docker container
-
-```
-make test-e2e-docker
-```
-
-This will run the tests in a Docker container running on your host machine.
-Note that due to the nature of flintlock, the container will be run with
-high privileges and will share some devices and process memory with the host.
-
-#### In an Equinix device
-
-```bash
-export METAL_AUTH_TOKEN=<your token>
-export EQUINIX_ORG_ID=<your org id>
-make test-e2e-metal
-```
-
-This will use the tool at `./test/tools/run.py` to create a new project and device
-with the credentials provided above, and then run the tests within that device.
-
-This exact command will run tests against main of the upstream branch, and only with
-minimal configuration. Read the tool [usage docs](test/tools/README.md) for information
-on how to configure and use the tool in your development.
+See the dedicated docs for the end to end tests [here](test/e2e/README.md).
 
 ### Writing your solution
 

diff --git a/test/e2e/README.md b/test/e2e/README.md
@@ -0,0 +1,64 @@
+## E2E tests
+
+The end to end tests are written in Go.
+They are fairly simple, and currently cover a simple CRUD happy path.
+We aim to test as much complexity as possible in lighter weight unit and
+integration tests.
+
+There are several ways to run the end to end tests.
+
+### In your local environment
+
+```
+make test-e2e
+```
+
+This will run the tests directly on your host with minimal fuss.
+You must ensure that you have installed all the dependencies per the
+[Quick-Start guide][quick-start].
+
+### In a local docker container
+
+```
+make test-e2e-docker
+```
+
+This will run the tests in a Docker container running on your host machine.
+Note that due to the nature of flintlock, the container will be run with
+high privileges and will share some devices and process memory with the host.
+
+### In an Equinix device
+
+```bash
+export METAL_AUTH_TOKEN=<your token>
+export EQUINIX_ORG_ID=<your org id>
+make test-e2e-metal
+```
+
+This will use the tool at `./test/tools/run.py` to create a new project and device
+with the credentials provided above, and then run the tests within that device.
+
+This exact command will run tests against main of the upstream branch, and only with
+minimal configuration. Read the tool [usage docs](test/tools/README.md) for information
+on how to configure and use the tool in your development.
+
+### Configuration
+
+There are a couple of custom test flags which you can set to alter the behaviour
+of the tests.
+
+At the time of writing these are:
+- `skip.setup.thinpool`: skips the setup of devicemapper thinpools.
+- `skip.delete`: skip the Delete step of the tests and leave the mVMs around for debugging.
+  This will also leave containerd and flintlockd running. All cleanup will be manual.
+- `skip.teardown`: skip stopping containerd and flintlockd processes.
+- `level.containerd`: set the containerd log level.
+- `level.flintlockd`: set the flintlockd log level.
+
+You can pass in these flags to the test like so:
+
+```bash
+go test -timeout 30m -p 1 -v -tags=e2e ./test/e2e/... -level.flintlockd=9
+```
+
+All the flags can be found at [`params.go`](test/e2e/utils/params.go).
diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go
@@ -13,6 +13,15 @@ import (
 	u "github.com/weaveworks/flintlock/test/e2e/utils"
 )
 
+var params *u.Params
+
+func init() {
+	// Call testing.Init() prior to tests.NewParams(), as otherwise custom test flags
+	// will not be recognised.
+	testing.Init()
+	params = u.NewParams()
+}
+
 func TestE2E(t *testing.T) {
 	RegisterTestingT(t)
 
@@ -26,7 +35,7 @@ func TestE2E(t *testing.T) {
 		mvmPid2 int
 	)
 
-	r := u.Runner{}
+	r := u.NewRunner(params)
 	defer func() {
 		log.Println("TEST STEP: cleaning up running processes")
 		r.Teardown()
@@ -79,6 +88,11 @@ func TestE2E(t *testing.T) {
 		return nil
 	}, "120s").Should(Succeed())
 
+	if params.SkipDelete {
+		log.Println("TEST STEP: skipping delete")
+		return
+	}
+
 	log.Println("TEST STEP: deleting existing MicroVMs")
 	Expect(u.DeleteMVM(flintlockClient, mvmID, mvmNS)).To(Succeed())
 	Expect(u.DeleteMVM(flintlockClient, secondMvmID, mvmNS)).To(Succeed())

diff --git a/test/e2e/utils/params.go b/test/e2e/utils/params.go
@@ -0,0 +1,30 @@
+//go:build e2e
+// +build e2e
+
+package utils
+
+import "flag"
+
+// Params groups all param.
+type Params struct {
+	SkipSetupThinpool  bool
+	SkipTeardown       bool
+	SkipDelete         bool
+	ContainerdLogLevel string
+	FlintlockdLogLevel string
+}
+
+// NewParams returns a new Params based on provided flags.
+func NewParams() *Params {
+	params := Params{}
+
+	flag.BoolVar(&params.SkipSetupThinpool, "skip.setup.thinpool", false, "Skip setting up devicemapper thinpools")
+	flag.BoolVar(&params.SkipDelete, "skip.delete", false, "Skip running the 'delete vm' step of the tests (useful for debugging, this will also leave containerd and flintlockd running)")
+	flag.BoolVar(&params.SkipTeardown, "skip.teardown", false, "Do not stop containerd or flintlockd after test exit (note: will require manual cleanup)")
+	flag.StringVar(&params.ContainerdLogLevel, "level.containerd", "debug", "Set containerd's log level [trace, *debug*, info, warn, error, fatal, panic]")
+	flag.StringVar(&params.FlintlockdLogLevel, "level.flintlockd", "0", "Set flintlockd's log level [A level of 2 and above is debug logging. A level of 9 and above is tracing.]")
+
+	flag.Parse()
+
+	return &params
+}
diff --git a/test/e2e/utils/runner.go b/test/e2e/utils/runner.go
@@ -1,3 +1,6 @@
+//go:build e2e
+// +build e2e
+
 package utils
 
 import (
@@ -34,17 +37,22 @@ const (
 	devMapperRoot         = containerdRootDir + "/snapshotter/devmapper"
 )
 
-// Runner is a very poorly named thing and honestly idk what to call it.
-// What it does is compile flintlockd and start containerd and flintlockd.
-// So 'TestSetterUpper' did not sound as slick, but that is what it is.
-// I am happy for literally any suggestions.
+// Runner holds test runner configuration.
 type Runner struct {
+	params            *Params
 	flintlockdBin     string
 	containerdSession *gexec.Session
 	flintlockdSession *gexec.Session
 	flintlockdConn    *grpc.ClientConn
 }
 
+// NewRunner creates a new instance of Runner with a set of Params.
+func NewRunner(params *Params) Runner {
+	return Runner{
+		params: params,
+	}
+}
+
 // Setup is a helper for the e2e tests which:
 // - sets up up devicemapper thinpools
 // - writes containerd config
@@ -57,8 +65,8 @@ type Runner struct {
 // Teardown should be called before Setup in a defer.
 func (r *Runner) Setup() v1alpha1.MicroVMClient {
 	makeDirectories()
-	createThinPools()
-	writeContainerdConfig()
+	r.createThinPools()
+	r.writeContainerdConfig()
 	r.buildFLBinary()
 	r.startContainerd()
 	r.startFlintlockd()
@@ -80,6 +88,13 @@ func (r *Runner) Teardown() {
 		r.flintlockdConn.Close()
 	}
 
+	// If either of these is true, we should still close the connection held
+	// by the runner itself. The other processes can be killed manually after
+	// debugging.
+	if r.params.SkipTeardown || r.params.SkipDelete {
+		return
+	}
+
 	if r.flintlockdSession != nil {
 		r.flintlockdSession.Terminate().Wait()
 	}
@@ -88,7 +103,7 @@ func (r *Runner) Teardown() {
 		r.containerdSession.Terminate().Wait()
 	}
 
-	cleanupThinPools()
+	r.cleanupThinPools()
 	cleanupDirectories()
 
 	gexec.CleanupBuildArtifacts()
@@ -106,16 +121,24 @@ func cleanupDirectories() {
 	gm.Expect(os.RemoveAll(containerdStateDir)).To(gm.Succeed())
 }
 
-func createThinPools() {
+func (r *Runner) createThinPools() {
+	if r.params.SkipSetupThinpool {
+		return
+	}
+
 	scriptPath := filepath.Join(baseDir(), "hack", "scripts", "devpool.sh")
 	command := exec.Command(scriptPath, thinpoolName, loopDeviceTag)
 	session, err := gexec.Start(command, gk.GinkgoWriter, gk.GinkgoWriter)
 
 	gm.Expect(err).NotTo(gm.HaveOccurred())
-	gm.Eventually(session).Should(gexec.Exit(0))
+	gm.Eventually(session, "20s").Should(gexec.Exit(0))
 }
 
-func cleanupThinPools() {
+func (r *Runner) cleanupThinPools() {
+	if r.params.SkipSetupThinpool {
+		return
+	}
+
 	gm.Expect(dmsetup.RemoveDevice(thinpoolName, dmsetup.RemoveWithForce)).To(gm.Succeed())
 
 	cmd := exec.Command("losetup")
@@ -129,16 +152,7 @@ func cleanupThinPools() {
 	}
 }
 
-func writeContainerdConfig() {
-	dmplug := map[string]interface{}{
-		"pool_name":       thinpoolName,
-		"root_path":       devMapperRoot,
-		"base_image_size": "10GB",
-		"discard_blocks":  "true",
-	}
-	pluginTree, err := toml.TreeFromMap(dmplug)
-	gm.Expect(err).NotTo(gm.HaveOccurred())
-
+func (r *Runner) writeContainerdConfig() {
 	cfg := ccfg.Config{
 		Version: 2,
 		Root:    containerdRootDir,
@@ -149,14 +163,25 @@ func writeContainerdConfig() {
 		Metrics: ccfg.MetricsConfig{
 			Address: "127.0.0.1:1338",
 		},
-		Plugins: map[string]toml.Tree{
-			"io.containerd.snapshotter.v1.devmapper": *pluginTree,
-		},
 		Debug: ccfg.Debug{
-			Level: "trace",
+			Level: r.params.ContainerdLogLevel,
 		},
 	}
 
+	if !r.params.SkipSetupThinpool {
+		dmplug := map[string]interface{}{
+			"pool_name":       thinpoolName,
+			"root_path":       devMapperRoot,
+			"base_image_size": "10GB",
+			"discard_blocks":  "true",
+		}
+		pluginTree, err := toml.TreeFromMap(dmplug)
+		gm.Expect(err).NotTo(gm.HaveOccurred())
+		cfg.Plugins = map[string]toml.Tree{
+			"io.containerd.snapshotter.v1.devmapper": *pluginTree,
+		}
+	}
+
 	f, err := os.Create(containerdCfg)
 	gm.Expect(err).NotTo(gm.HaveOccurred())
 
@@ -185,14 +210,10 @@ func (r *Runner) startFlintlockd() {
 	gm.Expect(err).NotTo(gm.HaveOccurred())
 
 	//nolint: gosec // We know what we're doing.
-	flCmd := exec.Command(
-		r.flintlockdBin,
-		"run",
-		"--containerd-socket",
-		containerdSocket,
-		"--parent-iface",
-		parentIface,
-	)
+	flCmd := exec.Command(r.flintlockdBin, "run",
+		"--containerd-socket", containerdSocket,
+		"--parent-iface", parentIface,
+		"--verbosity", r.params.FlintlockdLogLevel)
 	flSess, err := gexec.Start(flCmd, gk.GinkgoWriter, gk.GinkgoWriter)
 	gm.Expect(err).NotTo(gm.HaveOccurred())