refactoring

mariner/config.go

@@ -43,6 +43,8 @@ const (
 	commonsPrefix     = "COMMONS/"
 	userPrefix        = "USER/"
 	conformancePrefix = "CONFORMANCE/"
+	workspacePrefix   = "/" + engineWorkspaceVolumeName
+	gatewayPrefix     = "/" + commonsDataVolumeName
 
 	notStarted = "not-started" // 3
 	running    = "running"     // 2

mariner/engine.go

@@ -42,7 +42,6 @@ type K8sEngine struct {
 	Manifest        *Manifest           // to pass the manifest to the gen3fuse container of each task pod
 	Log             *MainLog            //
 	KeepFiles       map[string]bool     // all the paths to not delete during basic file cleanup
-	IsInitWorkDir   string
 }
 
 // Tool represents a leaf in the graph of a workflow
@@ -63,7 +62,6 @@ type Tool struct {
 	ExpressionResult map[string]interface{}
 	Task             *Task
 	S3Input          []*ToolS3Input
-	initWorkDirFiles []string
 
 	// dev'ing
 	// need to load this with runtime context as per CWL spec
@@ -84,9 +82,9 @@ type TaskRuntimeJSContext struct {
 
 // ToolS3Input ..
 type ToolS3Input struct {
-	URL         string `json:"url"`            // S3 URL
-	Path        string `json:"path"`           // Local path for dl
-	InitWorkDir bool   `json:"init_work_dir"`  // is this an initwkdir requirement?
+	URL         string `json:"url"`           // S3 URL
+	Path        string `json:"path"`          // Local path for dl
+	InitWorkDir bool   `json:"init_work_dir"` // is this an initwkdir requirement?
 }
 
 // Engine runs an instance of the mariner engine job
@@ -283,7 +281,7 @@ func (task *Task) tool(runID string) *Tool {
 	tool := &Tool{
 		Task:       task,
 		WorkingDir: task.workingDir(runID),
-		S3Input: []*ToolS3Input{},
+		S3Input:    []*ToolS3Input{},
 	}
 	tool.JSVM = tool.newJSVM()
 	task.infof("end make tool object")

mariner/k8s.go

@@ -271,17 +271,8 @@ func (tool *Tool) env() (env []k8sv1.EnvVar, err error) {
 
 // for marinerTask job
 func (engine *K8sEngine) s3SidecarEnv(tool *Tool) (env []k8sv1.EnvVar) {
-	initWorkDirFiles := strings.Join(tool.initWorkDirFiles, ",")
 	engine.infof("load s3 sidecar env for task: %v", tool.Task.Root.ID)
 	env = []k8sv1.EnvVar{
-		{
-			Name:  "InitWorkDirFiles",
-			Value: initWorkDirFiles,
-		},
-		{
-			Name:  "IsInitWorkDir",
-			Value: engine.IsInitWorkDir,
-		},
 		{
 			Name:      "AWSCREDS",
 			ValueFrom: envVarAWSUserCreds,

mariner/tool.go

@@ -4,62 +4,54 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
-	"path/filepath"
 	pathLib "path"
+	"path/filepath"
 	"strings"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/s3/s3manager"
 	log "github.com/sirupsen/logrus"
 )
 
+func isValidPath(path string) bool {
+	validPrefixes := [...]string{userPrefix, commonsPrefix, workspacePrefix, gatewayPrefix}
+	for _, prefix := range validPrefixes {
+		if strings.HasPrefix(path, prefix) {
+			return true
+		}
+	}
+	return false
+}
+
 // this file contains some methods/functions for setting up and working with Tools (i.e., commandlinetools and expressiontools)
 
 func pathHelper(path string, tool *Tool) (err error) {
+	if !isValidPath(path) {
+		log.Errorf("unsupported initwkdir path: %v", path)
+		return tool.Task.errorf("unsupported initwkdir path: %v", path)
+	}
+	url := ""
 	if strings.HasPrefix(path, userPrefix) {
 		trimmedPath := strings.TrimPrefix(path, userPrefix)
 		path = strings.Join([]string{"/", engineWorkspaceVolumeName, "/", trimmedPath}, "")
-		tool.Task.infof("adding initwkdir path: %v", path)
-		tool.S3Input = append(tool.S3Input, &ToolS3Input{
-			Path: path,
-			InitWorkDir: true,
-		})
-		tool.initWorkDirFiles = append(tool.initWorkDirFiles, path)
-	} else if strings.HasPrefix(path, "/"+engineWorkspaceVolumeName) {
-		tool.Task.infof("adding initwkdir path: %v", path)
-		tool.S3Input = append(tool.S3Input, &ToolS3Input{
-			Path: path,
-			InitWorkDir: true,
-		})
-		tool.initWorkDirFiles = append(tool.initWorkDirFiles, path)
-		tool.Task.infof("*File - Path: %v", path)
 	} else if strings.HasPrefix(path, commonsPrefix) {
 		guid := pathLib.Base(path)
 		indexFile, err := getIndexedFileInfo(guid)
 		if err != nil {
 			return tool.Task.errorf("Unable to get indexed record: %v; error: %v", guid, err)
 		}
 		path = pathLib.Join(pathToCommonsData, indexFile.Filename)
-		tool.Task.infof("adding initwkdir path: %v", path)
-		tool.S3Input = append(tool.S3Input, &ToolS3Input{
-			URL: indexFile.URLs[0],
-			Path: path,
-			InitWorkDir: true,
-		})
-		tool.initWorkDirFiles = append(tool.initWorkDirFiles, path)
-	} else if strings.HasPrefix(path, "/"+commonsDataVolumeName) {
-		tool.Task.infof("adding initwkdir path: %v", path)
-		tool.S3Input = append(tool.S3Input, &ToolS3Input{
-			Path: path,
-			InitWorkDir: true,
-		})
-		tool.initWorkDirFiles = append(tool.initWorkDirFiles, path)
-		tool.Task.infof("*File - Path: %v", path)
-	} else {
-		log.Errorf("unsupported initwkdir path: %v", path)
-		return tool.Task.errorf("unsupported initwkdir path: %v", path)
+		url = indexFile.URLs[0]
 	}
 
+	tool.Task.infof("adding initwkdir path: %v", path)
+	tool.S3Input = append(tool.S3Input, &ToolS3Input{
+		URL:         url,
+		Path:        path,
+		InitWorkDir: true,
+	})
+	tool.Task.infof("*File - Path: %v", path)
+
 	return nil
 }
 
@@ -146,9 +138,7 @@ func (engine *K8sEngine) initWorkDirReq(tool *Tool) (err error) {
 							return tool.Task.errorf("unsupported initwkdir type: %T; value: %v", output, output)
 						}
 					}
-					engine.IsInitWorkDir = "true"
 					tool.Task.infof("s3input paths: %v", tool.S3Input)
-					tool.Task.infof("initWorkDirFiles: %v", tool.initWorkDirFiles)
 					continue
 				}
 
@@ -204,7 +194,6 @@ func (engine *K8sEngine) initWorkDirReq(tool *Tool) (err error) {
 					tool.Task.infof("raw key: %v", key)
 					tool.Task.infof("tool workdir: %v", tool.WorkingDir)
 
-
 					var b []byte
 					switch contents.(type) {
 					case string:
@@ -232,11 +221,10 @@ func (engine *K8sEngine) initWorkDirReq(tool *Tool) (err error) {
 					}
 					log.Infof("init working directory request recieved")
 					tool.S3Input = append(tool.S3Input, &ToolS3Input{
-						URL: "s3://" + filepath.Join(engine.S3FileManager.S3BucketName, key),
-						Path: filepath.Join(tool.WorkingDir, entryName),
+						URL:         "s3://" + filepath.Join(engine.S3FileManager.S3BucketName, key),
+						Path:        filepath.Join(tool.WorkingDir, entryName),
 						InitWorkDir: true,
 					})
-					engine.IsInitWorkDir = "true"
 				}
 			}
 		}