Enable end-to-end docker tests (#185)

* Enable e2e docker tests

* Make e2e test more reliable

.github/workflows/e2e_test.yml

@@ -206,7 +206,7 @@ jobs:
       fail-fast: false
       matrix:
         algorithm: [vpg, dql]
-        context: [metal]
+        context: [metal, docker]
     steps:
       - uses: actions/checkout@v2
 

.vscode/launch.json

@@ -22,6 +22,14 @@
       "program": "${workspaceFolder}/cmd/spice/main.go",
       "args": ["action", "add", "hold"]
     },
+    {
+      "name": "spice e2e test docker",
+      "type": "go",
+      "request": "launch",
+      "mode": "auto",
+      "program": "${workspaceFolder}/test/e2e/e2e_test.go",
+      "args": ["-e2e"]
+    },
     {
       "name": "Test Current File",
       "type": "go",

pkg/cli/runtime/runtime.go

@@ -6,6 +6,7 @@ import (
 	"os"
 
 	"github.com/spiceai/spiceai/pkg/context"
+	"github.com/spiceai/spiceai/pkg/util"
 )
 
 func Run(contextFlag string, manifestPath string) error {
@@ -52,10 +53,10 @@ func Run(contextFlag string, manifestPath string) error {
 	cmd.Stderr = os.Stderr
 	cmd.Stdout = os.Stdout
 
-	err = cmd.Start()
+	err = util.RunCommand(cmd)
 	if err != nil {
 		return err
 	}
 
-	return cmd.Wait()
+	return nil
 }

pkg/util/command.go

@@ -0,0 +1,51 @@
+package util
+
+import (
+	"os"
+	"os/exec"
+	"os/signal"
+	"syscall"
+)
+
+func RunCommand(cmd *exec.Cmd) error {
+	if cmd == nil {
+		return nil
+	}
+
+	cmdErr := make(chan error, 1)
+	cmdStopped := make(chan bool, 1)
+	sigCh := make(chan os.Signal, 1)
+	signal.Notify(sigCh, syscall.SIGTERM, os.Interrupt)
+
+	err := cmd.Start()
+	if err != nil {
+		return err
+	}
+
+	go func() {
+		appErr := cmd.Wait()
+
+		if appErr != nil {
+			cmdErr <- appErr
+		}
+		cmdStopped <- true
+		sigCh <- os.Interrupt
+	}()
+
+	<-sigCh
+
+	if cmd.ProcessState == nil || !cmd.ProcessState.Exited() {
+		err := cmd.Process.Signal(os.Interrupt)
+		if err != nil {
+			return err
+		}
+	}
+
+	<-cmdStopped
+
+	if len(cmdErr) > 0 {
+		return <-cmdErr
+	}
+
+	return nil
+}

test/e2e/cli.go

@@ -11,14 +11,29 @@ type cli struct {
 }
 
 func (c *cli) runCliCmd(args ...string) error {
+	cmd, err := c.startCliCmd(args...)
+	if err != nil {
+		return err
+	}
+
+	err = cmd.Wait()
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (c *cli) startCliCmd(args ...string) (*exec.Cmd, error) {
 	cmd := exec.Command(c.cliPath, args...)
 	cmd.Dir = c.workingDir
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
-	err := cmd.Run()
+
+	err := cmd.Start()
 	if err != nil {
-		return err
+		return nil, err
 	}
 
-	return nil
+	return cmd, nil
 }

test/e2e/e2e_test.go

@@ -26,7 +26,6 @@ var (
 	testDir          string
 	repoRoot         string
 	workingDirectory string
-	spicePodsDir     string
 	runtimePath      string
 	cliClient        *cli
 	runtime          *runtimeServer
@@ -78,14 +77,7 @@ func TestMain(m *testing.M) {
 		os.Exit(1)
 	}
 
-	stat, err := os.Stat(testDir)
-	if err != nil {
-		log.Println(err.Error())
-		os.Exit(1)
-	}
-
-	spicePodsDir = filepath.Join(testDir, ".spice", "pods")
-	err = os.MkdirAll(spicePodsDir, stat.Mode())
+	_, err = os.Stat(testDir)
 	if err != nil {
 		log.Println(err.Error())
 		os.Exit(1)
@@ -97,7 +89,11 @@ func TestMain(m *testing.M) {
 	}
 
 	runtime = &runtimeServer{
-		baseUrl: BaseUrl,
+		baseUrl:          BaseUrl,
+		runtimePath:      runtimePath,
+		workingDirectory: testDir,
+		cli:              cliClient,
+		context:          spicedContext,
 	}
 
 	err = copyFile(filepath.Join(repoRoot, "test/assets/data/csv/COINBASE_BTCUSD, 30.csv"), testDir)
@@ -106,12 +102,6 @@ func TestMain(m *testing.M) {
 		os.Exit(1)
 	}
 
-	err = copyFile(filepath.Join(workingDirectory, "pods/trader.yaml"), spicePodsDir)
-	if err != nil {
-		log.Println(err.Error())
-		os.Exit(1)
-	}
-
 	err = cliClient.runCliCmd("add", testPod)
 	if err != nil {
 		log.Println(err.Error())
@@ -134,7 +124,7 @@ func TestObservations(t *testing.T) {
 		return
 	}
 
-	runtimeCmd, err := runtime.startRuntime(runtimePath, testDir)
+	runtimeCmd, err := runtime.startRuntime()
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -170,6 +160,8 @@ func TestObservations(t *testing.T) {
 		t.Fatal(err)
 	}
 
+	time.Sleep(1 * time.Second)
+
 	observation, err = runtime.getObservations("trader")
 	if err != nil {
 		t.Fatal(err)
@@ -187,7 +179,7 @@ func TestTrainingOutput(t *testing.T) {
 		return
 	}
 
-	runtimeCmd, err := runtime.startRuntime(runtimePath, testDir)
+	runtimeCmd, err := runtime.startRuntime()
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -243,7 +235,7 @@ func TestImportExport(t *testing.T) {
 		return
 	}
 
-	runtimeCmd, err := runtime.startRuntime(runtimePath, testDir)
+	runtimeCmd, err := runtime.startRuntime()
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -297,7 +289,7 @@ func TestImportExport(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	runtimeCmd, err = runtime.startRuntime(runtimePath, testDir)
+	runtimeCmd, err = runtime.startRuntime()
 	if err != nil {
 		t.Fatal(err)
 	}

test/e2e/runtime.go

@@ -16,15 +16,22 @@ import (
 )
 
 type runtimeServer struct {
-	baseUrl string
+	baseUrl          string
+	runtimePath      string
+	workingDirectory string
+	cli              *cli
+	context          string
 }
 
-func (r *runtimeServer) startRuntime(runtimePath string, workingDirectory string) (*exec.Cmd, error) {
-	runtimeCmd := exec.Command(runtimePath)
-	runtimeCmd.Dir = workingDirectory
-	runtimeCmd.Stdout = os.Stdout
-	runtimeCmd.Stderr = os.Stderr
-	err := runtimeCmd.Start()
+func (r *runtimeServer) startRuntime() (*exec.Cmd, error) {
+	var runtimeCmd *exec.Cmd
+	var err error
+
+	if r.context == "docker" {
+		runtimeCmd, err = r.startDockerRuntime()
+	} else {
+		runtimeCmd, err = r.startMetalRuntime()
+	}
 	if err != nil {
 		return nil, err
 	}
@@ -37,6 +44,28 @@ func (r *runtimeServer) startRuntime(runtimePath string, workingDirectory string
 	return runtimeCmd, nil
 }
 
+func (r *runtimeServer) startDockerRuntime() (*exec.Cmd, error) {
+	runtimeCmd, err := r.cli.startCliCmd("run")
+	if err != nil {
+		return nil, err
+	}
+
+	return runtimeCmd, nil
+}
+
+func (r *runtimeServer) startMetalRuntime() (*exec.Cmd, error) {
+	runtimeCmd := exec.Command(r.runtimePath)
+	runtimeCmd.Dir = r.workingDirectory
+	runtimeCmd.Stdout = os.Stdout
+	runtimeCmd.Stderr = os.Stderr
+	err := runtimeCmd.Start()
+	if err != nil {
+		return nil, err
+	}
+
+	return runtimeCmd, nil
+}
+
 func (r *runtimeServer) getRecommendation(podName string, tag string) (*aiengine_pb.InferenceResult, error) {
 	var inference aiengine_pb.InferenceResult
 	url := fmt.Sprintf("%s/api/v0.1/pods/%s/models/%s/recommendation", r.baseUrl, podName, tag)