From 9e0871e17c156bb7011902b11cdc56377820f49c Mon Sep 17 00:00:00 2001 From: Chris Kyrouac Date: Mon, 6 May 2024 11:24:01 -0400 Subject: [PATCH] tests: Add e2e tests This initial set of tests covers the basic use case of running and removing VMs/disk images. Signed-off-by: Chris Kyrouac --- .github/workflows/ci.yml | 2 +- .gitignore | 1 + Makefile | 8 +- test/e2e/e2e_test.go | 310 +++++++++++++++++++++++++++++++++ test/e2e/e2e_utils.go | 203 +++++++++++++++++++++ test/e2e/e2e_utils_darwin.go | 50 ++++++ test/e2e/e2e_utils_linux.go | 65 +++++++ test/e2e/test_vm.go | 69 ++++++++ test/resources/Containerfile.1 | 2 + test/resources/Containerfile.2 | 2 + test/resources/README.md | 4 + test/resources/build.images.sh | 19 ++ 12 files changed, 732 insertions(+), 3 deletions(-) create mode 100644 test/e2e/e2e_test.go create mode 100644 test/e2e/e2e_utils.go create mode 100644 test/e2e/e2e_utils_darwin.go create mode 100644 test/e2e/e2e_utils_linux.go create mode 100644 test/e2e/test_vm.go create mode 100644 test/resources/Containerfile.1 create mode 100644 test/resources/Containerfile.2 create mode 100644 test/resources/README.md create mode 100755 test/resources/build.images.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c80645ef..7894cc3c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,4 +21,4 @@ jobs: go install github.com/onsi/ginkgo/v2/ginkgo@latest make GOOPTS=-buildvcs=false export PATH=$PATH:$HOME/go/bin - make test + make integration_tests diff --git a/.gitignore b/.gitignore index 5e56e040..95f150c9 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ /bin +test/e2e/e2e.test diff --git a/Makefile b/Makefile index 21b367db..5162ab12 100644 --- a/Makefile +++ b/Makefile @@ -8,8 +8,12 @@ all: out_dir out_dir: mkdir -p $(output_dir) -test: - ginkgo -tags $(build_tags) ./... +integration_tests: + ginkgo run -tags $(build_tags) --skip-package test ./... + +# !! These tests will modify your system's resources. See note in e2e_test.go. !! +e2e_test: all + ginkgo -tags $(build_tags) ./test/... clean: rm -f $(output_dir)/* diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go new file mode 100644 index 00000000..7cbc84bf --- /dev/null +++ b/test/e2e/e2e_test.go @@ -0,0 +1,310 @@ +package e2e_test + +// **************************************************************************** +// These are end-to-end tests that run the podman-bootc binary. +// A rootful podman machine is assumed to already be running. +// The tests interact directly with libvirt (on linux), qemu (on darwin), +// podman-bootc cache dirs, and podman images and containers. +// +// Running these tests will create/delete VMs, pull/remove podman images +// and containers, and remove the entire podman-bootc cache dir. +// +// These tests depend on the quay.io/ckyrouac/podman-bootc-test image +// which is built from the Containerfiles in the test/resources directory. +// **************************************************************************** + +import ( + "encoding/json" + "os" + "path/filepath" + "sync" + "testing" + + "gitlab.com/bootc-org/podman-bootc/pkg/config" + "gitlab.com/bootc-org/podman-bootc/test/e2e" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestPodmanBootcE2E(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "End to End Test Suite") +} + +var _ = BeforeSuite(func() { + err := e2e.Cleanup() + Expect(err).To(Not(HaveOccurred())) +}) + +var _ = AfterSuite(func() { + err := e2e.Cleanup() + Expect(err).To(Not(HaveOccurred())) +}) + +var _ = Describe("E2E", func() { + Context("Run with no args from a fresh install", Ordered, func() { + // Create the disk/VM once to avoid the overhead of creating it for each test + var vm *e2e.TestVM + + BeforeAll(func() { + var err error + vm, err = e2e.BootVM(e2e.BaseImage) + Expect(err).To(Not(HaveOccurred())) + }) + + It("should pull the container image", func() { + imagesListOutput, _, err := e2e.RunPodman("images", e2e.BaseImage, "--format", "json") + Expect(err).To(Not(HaveOccurred())) + imagesList := []map[string]interface{}{} + json.Unmarshal([]byte(imagesListOutput), &imagesList) + Expect(imagesList).To(HaveLen(1)) + }) + + It("should create a bootc disk image", func() { + vmDirs, err := e2e.ListCacheDirs() + Expect(err).To(Not(HaveOccurred())) + Expect(vmDirs).To(HaveLen(1)) + + _, err = os.Stat(filepath.Join(vmDirs[0], config.DiskImage)) + Expect(err).To(Not(HaveOccurred())) + }) + + It("should create a new virtual machine", func() { + vmExists, err := e2e.VMExists(vm.Id) + Expect(err).To(Not(HaveOccurred())) + Expect(vmExists).To(BeTrue()) + }) + + It("should start an ssh session into the VM", func() { + // Send a command to the VM and check the output + vm.SendCommand("echo 'hello'", "hello") + Expect(vm.StdOut[len(vm.StdOut)-1]).To(ContainSubstring("hello")) + }) + + It("should keep the VM running after the initial ssh session is closed", func() { + vm.StdIn.Close() // this closes the ssh session + + vmIsRunning, err := e2e.VMIsRunning(vm.Id) + Expect(err).To(Not(HaveOccurred())) + Expect(vmIsRunning).To(BeTrue()) + }) + + It("should open a new ssh session into the VM via the ssh cmd", func() { + _, _, err := e2e.RunPodmanBootc("ssh", vm.Id) //TODO: test the output, send a command + Expect(err).To(Not(HaveOccurred())) + }) + + It("Should delete the VM and persist the disk image when calling stop", func() { + _, _, err := e2e.RunPodmanBootc("stop", vm.Id) + Expect(err).To(Not(HaveOccurred())) + + //qemu doesn't immediately stop the VM, so we need to wait for it to stop + Eventually(func() bool { + vmExists, err := e2e.VMExists(vm.Id) + Expect(err).To(Not(HaveOccurred())) + return vmExists + }).Should(BeFalse()) + + vmDirs, err := e2e.ListCacheDirs() + Expect(err).To(Not(HaveOccurred())) + + _, err = os.Stat(filepath.Join(vmDirs[0], config.DiskImage)) + Expect(err).To(Not(HaveOccurred())) + }) + + It("Should remove the disk image when calling rm", func() { + _, _, err := e2e.RunPodmanBootc("rm", vm.Id) + Expect(err).To(Not(HaveOccurred())) + + vmDirs, err := e2e.ListCacheDirs() + Expect(err).To(Not(HaveOccurred())) + + Expect(vmDirs).To(HaveLen(0)) + }) + + It("Should recreate the disk and VM when calling run", func() { + var err error + vm, err = e2e.BootVM(e2e.BaseImage) + Expect(err).To(Not(HaveOccurred())) + + vmDirs, err := e2e.ListCacheDirs() + Expect(err).To(Not(HaveOccurred())) + Expect(vmDirs).To(HaveLen(1)) + + vmExists, err := e2e.VMExists(vm.Id) + Expect(err).To(Not(HaveOccurred())) + Expect(vmExists).To(BeTrue()) + }) + + It("Should prevent removing a VM with an active SSH session", func() { + _, _, err := e2e.RunPodmanBootc("rm", "-f", vm.Id) + Expect(err).To(HaveOccurred()) + + Eventually(func() int { + vmDirs, err := e2e.ListCacheDirs() + Expect(err).To(Not(HaveOccurred())) + return len(vmDirs) + }).Should(Equal(1)) + }) + + It("Should remove the cache directory when calling rm -f while VM is running", func() { + // the SSH connection needs to be closed before attempting rm -f + err := vm.StdIn.Close() + Expect(err).To(Not(HaveOccurred())) + + _, _, err = e2e.RunPodmanBootc("rm", "-f", vm.Id) + Expect(err).To(Not(HaveOccurred())) + + Eventually(func() int { + vmDirs, err := e2e.ListCacheDirs() + Expect(err).To(Not(HaveOccurred())) + return len(vmDirs) + }).Should(Equal(0)) + }) + + AfterAll(func() { + vm.StdIn.Close() + e2e.Cleanup() + }) + }) + + Context("Multiple VMs exist", Ordered, func() { + var activeVM *e2e.TestVM + var inactiveVM *e2e.TestVM + var stoppedVM *e2e.TestVM + + BeforeAll(func() { + var err error + errors := make(chan error) + + var wg sync.WaitGroup + wg.Add(1) + go func() { + // create an "active" VM + // running with an active SSH session + println("**** STARTING ACTIVE VM") + activeVM, err = e2e.BootVM(e2e.TestImageTwo) + if err != nil { + errors <- err + } + wg.Done() + }() + + wg.Add(1) + go func() { + // create an "inactive" VM + // running with no active SSH session + inactiveVM, err = e2e.BootVM(e2e.TestImageOne) + if err != nil { + errors <- err + } + err = inactiveVM.StdIn.Close() + if err != nil { + errors <- err + } + wg.Done() + }() + + wg.Add(1) + go func() { + // create a "stopped" VM + // VM does not exist but the VM directory containing the cached disk image does + stoppedVM, err = e2e.BootVM(e2e.BaseImage) + if err != nil { + errors <- err + } + err = stoppedVM.StdIn.Close() //ssh needs to be closed before stopping the VM + if err != nil { + errors <- err + } + _, _, err = e2e.RunPodmanBootc("stop", stoppedVM.Id) + if err != nil { + errors <- err + } + wg.Done() + }() + + wg.Wait() + close(errors) + + if err := <-errors; err != nil { + Fail(err.Error()) + } + + // validate there are 3 vm directories + vmDirs, err := e2e.ListCacheDirs() + Expect(err).To(Not(HaveOccurred())) + Expect(vmDirs).To(HaveLen(3)) + }) + + It("Should list multiple VMs", func() { + stdout, _, err := e2e.RunPodmanBootc("list") + Expect(err).To(Not(HaveOccurred())) + + listOutput := e2e.ParseListOutput(stdout) + Expect(listOutput).To(HaveLen(3)) + Expect(listOutput).To(ContainElement(e2e.ListEntry{ + Id: activeVM.Id, + Repo: e2e.TestImageTwo, + Running: "true", + })) + + Expect(listOutput).To(ContainElement(e2e.ListEntry{ + Id: inactiveVM.Id, + Repo: e2e.TestImageOne, + Running: "true", + })) + + Expect(listOutput).To(ContainElement(e2e.ListEntry{ + Id: stoppedVM.Id, + Repo: e2e.BaseImage, + Running: "false", + })) + }) + + It("Should remove all inactive VMs and caches when calling rm -f --all", func() { + _, _, err := e2e.RunPodmanBootc("rm", "-f", "--all") + Expect(err).To(Not(HaveOccurred())) + + stdout, _, err := e2e.RunPodmanBootc("list") + Expect(err).To(Not(HaveOccurred())) + + // should keep the active VM that has an ssh session open + Expect(stdout).To(ContainSubstring(activeVM.Id)) + + // should remove the other VMs + Expect(stdout).To(Not(ContainSubstring(stoppedVM.Id))) + Expect(stdout).To(Not(ContainSubstring(inactiveVM.Id))) + + vmDirs, err := e2e.ListCacheDirs() + Expect(err).To(Not(HaveOccurred())) + Expect(vmDirs).To(HaveLen(1)) + Expect(vmDirs[0]).To(ContainSubstring(activeVM.Id)) + }) + + It("Should no-op and return successfully when rm -f --all with no VMs", func() { + //cleanup the remaining active VM first + err := activeVM.StdIn.Close() + Expect(err).To(Not(HaveOccurred())) + _, _, err = e2e.RunPodmanBootc("rm", "-f", activeVM.Id) + Expect(err).To(Not(HaveOccurred())) + + // verify there are no VMs + vmDirs, err := e2e.ListCacheDirs() + Expect(err).To(Not(HaveOccurred())) + Expect(vmDirs).To(HaveLen(0)) + + // attempt rm -f --all + _, _, err = e2e.RunPodmanBootc("rm", "-f", "--all") + Expect(err).To(Not(HaveOccurred())) + }) + + AfterAll(func() { + activeVM.StdIn.Close() + inactiveVM.StdIn.Close() + stoppedVM.StdIn.Close() + e2e.Cleanup() + }) + }) +}) diff --git a/test/e2e/e2e_utils.go b/test/e2e/e2e_utils.go new file mode 100644 index 00000000..95ea17c6 --- /dev/null +++ b/test/e2e/e2e_utils.go @@ -0,0 +1,203 @@ +package e2e + +import ( + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + + "gitlab.com/bootc-org/podman-bootc/pkg/user" +) + +const DefaultBaseImage = "quay.io/centos-bootc/centos-bootc-dev:stream9" +const TestImageOne = "quay.io/ckyrouac/podman-bootc-test:one" +const TestImageTwo = "quay.io/ckyrouac/podman-bootc-test:two" + +var BaseImage = GetBaseImage() + +func GetBaseImage() string { + if os.Getenv("BASE_IMAGE") != "" { + return os.Getenv("BASE_IMAGE") + } else { + return DefaultBaseImage + } +} + +func PodmanBootcBinary() string { + return ProjectRoot() + "/../../bin/podman-bootc" +} + +func ProjectRoot() string { + ex, err := os.Executable() + if err != nil { + panic(err) + } + projectRoot := filepath.Dir(ex) + return projectRoot +} + +func RunCmd(cmd string, args ...string) (stdout string, stderr string, err error) { + execCmd := exec.Command(cmd, args...) + + var stdOut strings.Builder + execCmd.Stdout = &stdOut + + var stdErr strings.Builder + execCmd.Stderr = &stdErr + + err = execCmd.Run() + if err != nil { + println(stdOut.String()) + println(stdErr.String()) + return + } + + return stdOut.String(), stdErr.String(), nil +} + +func RunPodmanBootc(args ...string) (stdout string, stderr string, err error) { + return RunCmd(PodmanBootcBinary(), args...) +} + +func RunPodman(args ...string) (stdout string, stderr string, err error) { + podmanArgs := append([]string{"-c", "podman-machine-default-root"}, args...) + return RunCmd("podman", podmanArgs...) +} + +func ListCacheDirs() (vmDirs []string, err error) { + user, err := user.NewUser() + if err != nil { + return + } + cacheDirContents, err := os.ReadDir(user.CacheDir()) + if err != nil { + return + } + + for _, dir := range cacheDirContents { + if dir.IsDir() { + vmDirs = append(vmDirs, filepath.Join(user.CacheDir(), dir.Name())) + } + } + return +} + +func GetVMIdFromContainerImage(image string) (vmId string, err error) { + imagesListOutput, _, err := RunPodman("images", image, "--format", "json") + if err != nil { + return + } + + imagesList := []map[string]interface{}{} + err = json.Unmarshal([]byte(imagesListOutput), &imagesList) + if err != nil { + return + } + + if len(imagesList) != 1 { + err = fmt.Errorf("Expected 1 image, got %d", len(imagesList)) + return + } + + vmId = imagesList[0]["Id"].(string)[:12] + return +} + +func BootVM(image string) (vm *TestVM, err error) { + runActiveCmd := exec.Command(PodmanBootcBinary(), "run", image) + stdIn, err := runActiveCmd.StdinPipe() + + if err != nil { + return + } + + vm = &TestVM{ + StdIn: stdIn, + } + + runActiveCmd.Stdout = vm + runActiveCmd.Stderr = vm + + go func() { + err = runActiveCmd.Run() + if err != nil { + return + } + }() + + err = vm.WaitForBoot() + + // populate the vm id after podman-bootc run + // so we can get the id from the pulled container image + vmId, err := GetVMIdFromContainerImage(image) + if err != nil { + return + } + vm.Id = vmId + + return +} + +func Cleanup() (err error) { + _, _, err = RunPodmanBootc("rm", "--all", "-f") + if err != nil { + return + } + + _, _, err = RunPodman("rmi", BaseImage, "-f") + if err != nil { + return + } + + _, _, err = RunPodman("rmi", TestImageTwo, "-f") + if err != nil { + return + } + + _, _, err = RunPodman("rmi", TestImageOne, "-f") + if err != nil { + return + } + + user, err := user.NewUser() + if err != nil { + return + } + + err = user.RemoveOSCDirs() + return +} + +type ListEntry struct { + Id string + Repo string + Running string +} + +// ParseListOutput parses the output of the podman bootc list command for easier comparison +func ParseListOutput(stdout string) (listOutput []ListEntry) { + listOuputLines := strings.Split(stdout, "\n") + + for i, line := range listOuputLines { + if i == 0 { + continue //skip the header + } + + if len(strings.Fields(line)) == 0 { + continue //skip the empty line + } + + entryArray := strings.Fields(line) + entry := ListEntry{ + Id: string(entryArray[0]), + Repo: string(entryArray[1]), + Running: string(entryArray[len(entryArray)-2]), + } + + listOutput = append(listOutput, entry) + } + + return +} diff --git a/test/e2e/e2e_utils_darwin.go b/test/e2e/e2e_utils_darwin.go new file mode 100644 index 00000000..1503888d --- /dev/null +++ b/test/e2e/e2e_utils_darwin.go @@ -0,0 +1,50 @@ +package e2e + +import ( + "path/filepath" + + "gitlab.com/bootc-org/podman-bootc/pkg/config" + "gitlab.com/bootc-org/podman-bootc/pkg/user" + "gitlab.com/bootc-org/podman-bootc/pkg/utils" + "gitlab.com/bootc-org/podman-bootc/pkg/vm" +) + +func pidFilePath(id string) (pidFilePath string, err error) { + user, err := user.NewUser() + if err != nil { + return + } + + _, cacheDir, err := vm.GetVMCachePath(id, user) + if err != nil { + return + } + + return filepath.Join(cacheDir, config.RunPidFile), nil +} + +func VMExists(id string) (exits bool, err error) { + pidFilePath, err := pidFilePath(id) + if err != nil { + return false, err + } + return utils.FileExists(pidFilePath) +} + +func VMIsRunning(id string) (exits bool, err error) { + pidFilePath, err := pidFilePath(id) + if err != nil { + return false, err + } + + pid, err := utils.ReadPidFile(pidFilePath) + if err != nil { + return false, err + } + + if pid != -1 && utils.IsProcessAlive(pid) { + return true, nil + } else { + return false, nil + } +} diff --git a/test/e2e/e2e_utils_linux.go b/test/e2e/e2e_utils_linux.go new file mode 100644 index 00000000..43311e33 --- /dev/null +++ b/test/e2e/e2e_utils_linux.go @@ -0,0 +1,65 @@ +package e2e + +import ( + "gitlab.com/bootc-org/podman-bootc/pkg/config" + + "libvirt.org/go/libvirt" +) + +func VMExists(id string) (exits bool, err error) { + vmName := "podman-bootc-" + id + + libvirtConnection, err := libvirt.NewConnect(config.LibvirtUri) + if err != nil { + return false, err + } + + defer libvirtConnection.Close() + + domains, err := libvirtConnection.ListAllDomains(libvirt.ConnectListAllDomainsFlags(0)) + if err != nil { + return false, err + } + for _, domain := range domains { + name, err := domain.GetName() + if err != nil { + return false, err + } + + if name == vmName { + return true, nil + } + } + + return false, nil +} + +func VMIsRunning(id string) (exits bool, err error) { + vmName := "podman-bootc-" + id + + libvirtConnection, err := libvirt.NewConnect(config.LibvirtUri) + if err != nil { + return false, err + } + defer libvirtConnection.Close() + + domain, err := libvirtConnection.LookupDomainByName(vmName) + if err != nil { + return false, err + } + + if domain == nil { + return false, nil + } + + state, _, err := domain.GetState() + if err != nil { + return false, err + } + + if state == libvirt.DOMAIN_RUNNING { + return true, nil + } else { + return false, nil + } +} diff --git a/test/e2e/test_vm.go b/test/e2e/test_vm.go new file mode 100644 index 00000000..1fa7116d --- /dev/null +++ b/test/e2e/test_vm.go @@ -0,0 +1,69 @@ +package e2e + +import ( + "fmt" + "io" + "strings" + "time" +) + +type TestVM struct { + StdIn io.WriteCloser + StdOut []string + IsBooted bool + Id string +} + +func (w *TestVM) SetId(id string) { + w.Id = id +} + +func (w *TestVM) GetId() string { + return w.Id +} + +func (w *TestVM) Write(p []byte) (n int, err error) { + if strings.Contains(string(p), "Connecting to vm") { + w.IsBooted = true + } + print(string(p)) + w.StdOut = append(w.StdOut, string(p)) + return len(p), nil +} + +func (w *TestVM) WaitForBoot() (err error) { + timeout := 10 * time.Minute + interval := 1 * time.Second + for { + if w.IsBooted { + break + } + time.Sleep(interval) + timeout -= interval + if timeout <= 0 { + return fmt.Errorf("VM did not boot within timeout") + } + } + + return +} + +// SendCommand sends a command to the VM's stdin and waits for the output +func (w *TestVM) SendCommand(cmd string, output string) (err error) { + w.StdIn.Write([]byte(cmd + "\n")) + + timeout := 2 * time.Minute + interval := 1 * time.Second + for { + if strings.Index(w.StdOut[len(w.StdOut)-1], output) == 0 { + break + } + time.Sleep(interval) + timeout -= interval + if timeout <= 0 { + return fmt.Errorf("VM did not output expected string within timeout") + } + } + + return +} diff --git a/test/resources/Containerfile.1 b/test/resources/Containerfile.1 new file mode 100644 index 00000000..189d9563 --- /dev/null +++ b/test/resources/Containerfile.1 @@ -0,0 +1,2 @@ +FROM quay.io/centos-bootc/centos-bootc:stream9 +RUN dnf install -y tmux diff --git a/test/resources/Containerfile.2 b/test/resources/Containerfile.2 new file mode 100644 index 00000000..2ec0523d --- /dev/null +++ b/test/resources/Containerfile.2 @@ -0,0 +1,2 @@ +FROM quay.io/centos-bootc/centos-bootc:stream9 +RUN dnf install -y gcc diff --git a/test/resources/README.md b/test/resources/README.md new file mode 100644 index 00000000..ab317877 --- /dev/null +++ b/test/resources/README.md @@ -0,0 +1,4 @@ +These Containerfiles are used to build test images for the e2e tests. +They are built with a multi-arch manifest and pushed to quay.io/ckyrouac/podman-bootc-test:[one|two] + +See build.images.sh diff --git a/test/resources/build.images.sh b/test/resources/build.images.sh new file mode 100755 index 00000000..7b928ea8 --- /dev/null +++ b/test/resources/build.images.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +set -e + +podman build --platform linux/amd64 -f Containerfile.1 -t quay.io/ckyrouac/podman-bootc-test:one-amd64 . +podman build --platform linux/arm64 -f Containerfile.1 -t quay.io/ckyrouac/podman-bootc-test:one-arm64 . +podman push quay.io/ckyrouac/podman-bootc-test:one-amd64 +podman push quay.io/ckyrouac/podman-bootc-test:one-arm64 +podman manifest create quay.io/ckyrouac/podman-bootc-test:one quay.io/ckyrouac/podman-bootc-test:one-arm64 quay.io/ckyrouac/podman-bootc-test:one-amd64 +podman manifest push quay.io/ckyrouac/podman-bootc-test:one +podman manifest rm quay.io/ckyrouac/podman-bootc-test:one + +podman build --platform linux/amd64 -f Containerfile.2 -t quay.io/ckyrouac/podman-bootc-test:two-amd64 . +podman build --platform linux/arm64 -f Containerfile.2 -t quay.io/ckyrouac/podman-bootc-test:two-arm64 . +podman push quay.io/ckyrouac/podman-bootc-test:two-amd64 +podman push quay.io/ckyrouac/podman-bootc-test:two-arm64 +podman manifest create quay.io/ckyrouac/podman-bootc-test:two quay.io/ckyrouac/podman-bootc-test:two-arm64 quay.io/ckyrouac/podman-bootc-test:two-amd64 +podman manifest push quay.io/ckyrouac/podman-bootc-test:two +podman manifest rm quay.io/ckyrouac/podman-bootc-test:two