Merge b134af2 into 68430cc

engines/enginetest/artifacts.go

@@ -23,7 +23,7 @@ type ArtifactTestCase struct {
 	FileNotFoundPath string
 	// Path to a folder that doesn't exist, and will return ErrResourceNotFound
 	FolderNotFoundPath string
-	// Files to expect in NestedFolderPath
+	// Files to expect in NestedFolderPath (must be relative and slash separated)
 	NestedFolderFiles []string
 	// Path to a folder that contains files NestedFolderFiles each containing
 	// Text
@@ -85,6 +85,12 @@ func (c *ArtifactTestCase) TestExtractFolderNotFound() {
 // NestedFolderFiles
 func (c *ArtifactTestCase) TestExtractNestedFolderPath() {
 	debug("## TestExtractNestedFolderPath")
+	for _, f := range c.NestedFolderFiles {
+		assert(f[0] != '/', "NestedFolderFiles must be relative paths")
+		assert(!strings.Contains(f, "\\"),
+			"NestedFolderFiles must be slash separated")
+	}
+
 	r := c.newRun()
 	defer r.Dispose()
 	r.NewSandboxBuilder(c.Payload)

engines/mock/mockengine_test.go

@@ -96,7 +96,7 @@ var artifactTestCase = enginetest.ArtifactTestCase{
 	TextFilePath:       "/folder/a.txt",
 	FileNotFoundPath:   "/not-found.txt",
 	FolderNotFoundPath: "/no-folder/",
-	NestedFolderFiles:  []string{"/folder/a.txt", "/folder/b.txt", "/folder/c/c.txt"},
+	NestedFolderFiles:  []string{"a.txt", "b.txt", "c/c.txt"},
 	NestedFolderPath:   "/folder/",
 	Payload: `{
 		"delay": 0,

engines/mock/mocksandbox.go

@@ -239,24 +239,20 @@ func (s *sandbox) ExtractFolder(folder string, handler engines.FileHandler) erro
 	m := sync.Mutex{}
 	handlerError := false
 	foundFolder := false
-	for path, data := range s.files {
-		if strings.HasPrefix(path, folder) {
+	for p, data := range s.files {
+		if strings.HasPrefix(p, folder) {
 			foundFolder = true
-			if path == folder {
-				// In this engine a filename ending with / is a folder, and its content
-				// is ignored, it's only used as indicator of folder existence
-				continue
-			}
 			wg.Add(1)
-			go func(path string, data []byte) {
-				err := handler(path, ioext.NopCloser(bytes.NewReader(data)))
+			go func(p string, data []byte) {
+				p = p[len(folder):] // Note: folder always ends with slash
+				err := handler(p, ioext.NopCloser(bytes.NewReader(data)))
 				if err != nil {
 					m.Lock()
 					handlerError = true
 					m.Unlock()
 				}
 				wg.Done()
-			}(path, data)
+			}(p, data)
 		}
 	}
 	wg.Wait()

engines/qemu/image/extract.go

@@ -11,6 +11,12 @@ import (
 	"github.com/taskcluster/taskcluster-worker/runtime/ioext"
 )
 
+// Image formats
+const (
+	formatQCOW2 = "qcow2"
+	formatRaw   = "raw"
+)
+
 const maxImageSize = int64(30 * 1024 * 1024 * 1024) // Use int64 for i386 builds
 
 // RandomMAC generates a new random MAC with the local bit set.
@@ -83,7 +89,7 @@ func extractImage(imageFile, imageFolder string) (*vm.Machine, error) {
 	// Inspect the raw disk file
 	diskFile := filepath.Join(imageFolder, "disk.img")
 	diskInfo := inspectImageFile(diskFile, imageRawFormat)
-	if diskInfo == nil || diskInfo.Format != "raw" {
+	if diskInfo == nil || diskInfo.Format != formatRaw {
 		return nil, engines.NewMalformedPayloadError("Image file contains ",
 			"'disk.img' which is not a RAW image file")
 	}
@@ -103,7 +109,7 @@ func extractImage(imageFile, imageFolder string) (*vm.Machine, error) {
 	// Inspect the QCOW2 layer file
 	layerFile := filepath.Join(imageFolder, "layer.qcow2")
 	layerInfo := inspectImageFile(layerFile, imageQCOW2Format)
-	if layerInfo == nil || layerInfo.Format != "qcow2" {
+	if layerInfo == nil || layerInfo.Format != formatQCOW2 {
 		return nil, engines.NewMalformedPayloadError("Image file contains ",
 			"'layer.qcow2' which is not a QCOW2 file")
 	}
@@ -119,7 +125,7 @@ func extractImage(imageFile, imageFolder string) (*vm.Machine, error) {
 		return nil, engines.NewMalformedPayloadError("Image file contains ",
 			"'layer.qcow2' which has a backing file that isn't: 'disk.img'")
 	}
-	if layerInfo.BackingFormat != "raw" {
+	if layerInfo.BackingFormat != formatRaw {
 		return nil, engines.NewMalformedPayloadError("Image file contains ",
 			"'layer.qcow2' which has a backing file format that isn't 'raw'")
 	}

engines/qemu/image/manager.go

@@ -18,7 +18,7 @@ type Manager struct {
 	m           sync.Mutex
 	images      map[string]*image
 	imageFolder string
-	gc          *gc.GarbageCollector
+	gc          gc.ResourceTracker
 	log         *logrus.Entry
 	sentry      *raven.Client
 }
@@ -48,7 +48,7 @@ type Instance struct {
 
 // NewManager creates a new image manager using the imageFolder for storing
 // images and instances of images.
-func NewManager(imageFolder string, gc *gc.GarbageCollector, log *logrus.Entry, sentry *raven.Client) (*Manager, error) {
+func NewManager(imageFolder string, gc gc.ResourceTracker, log *logrus.Entry, sentry *raven.Client) (*Manager, error) {
 	// Ensure the image folder is created
 	err := os.MkdirAll(imageFolder, 0777)
 	if err != nil {
@@ -221,7 +221,7 @@ func (i *Instance) DiskFile() string {
 
 // Format returns the image format: 'qcow2'
 func (i *Instance) Format() string {
-	return "qcow2"
+	return formatQCOW2
 }
 
 // Release frees the resources held by an instance.

engines/qemu/image/manager_test.go

@@ -67,7 +67,7 @@ func TestImageManager(t *testing.T) {
 	fmt.Println(" - Inspect file for sanity check: ", diskImage)
 	info := inspectImageFile(diskImage, imageQCOW2Format)
 	assert(info != nil, "Expected a qcow2 file")
-	assert(info.Format == "qcow2")
+	assert(info.Format == formatQCOW2)
 	assert(!info.DirtyFlag)
 	assert(info.BackingFile != "", "Missing backing file in qcow2")
 

engines/qemu/qemuengine_test.go

@@ -56,11 +56,9 @@ func TestMain(m *testing.M) {
 var provider = &enginetest.EngineProvider{
 	Engine: "qemu",
 	Config: `{
-		"qemu": {
-			"maxConcurrency":   5,
-			"imageFolder":      "/tmp/images/",
-			"socketFolder":     "/tmp/"
-		}
+		"maxConcurrency":   5,
+		"imageFolder":      "/tmp/images/",
+		"socketFolder":     "/tmp/"
   }`,
 }
 
@@ -70,22 +68,16 @@ func TestLogging(t *testing.T) {
 		EngineProvider: provider,
 		Target:         "hello-world",
 		TargetPayload: `{
-	    "start": {
-	      "image": "` + s.URL + `",
-	      "command": ["sh", "-c", "echo 'hello-world' && true"]
-	    }
+      "image": "` + s.URL + `",
+      "command": ["sh", "-c", "echo 'hello-world' && true"]
 	  }`,
 		FailingPayload: `{
-	    "start": {
-		    "image": "` + s.URL + `",
-		    "command": ["sh", "-c", "echo 'hello-world' && false"]
-	    }
+	    "image": "` + s.URL + `",
+	    "command": ["sh", "-c", "echo 'hello-world' && false"]
 	  }`,
 		SilentPayload: `{
-	    "start": {
-		    "image": "` + s.URL + `",
-		    "command": ["sh", "-c", "echo 'no hello' && true"]
-	    }
+	    "image": "` + s.URL + `",
+	    "command": ["sh", "-c", "echo 'no hello' && true"]
 	  }`,
 	}
 
@@ -104,10 +96,8 @@ func TestEnvironmentVariables(t *testing.T) {
 			"#=#",
 		},
 		Payload: `{
-	    "start": {
-	      "image": "` + s.URL + `",
-	      "command": ["sh", "-c", "echo $TEST_ENV_VAR && true"]
-	    }
+      "image": "` + s.URL + `",
+      "command": ["sh", "-c", "echo $TEST_ENV_VAR && true"]
 	  }`,
 	}
 
@@ -123,10 +113,8 @@ func TestAttachProxy(t *testing.T) {
 		EngineProvider: provider,
 		ProxyName:      "test-proxy",
 		PingProxyPayload: `{
-			"start": {
-				"image": "` + s.URL + `",
-				"command": ["sh", "-ec", "echo 'Pinging'; STATUS=$(curl -s -o /tmp/output -w '%{http_code}' http://taskcluster/test-proxy/v1/ping); cat /tmp/output; test $STATUS -eq 200;"]
-			}
+			"image": "` + s.URL + `",
+			"command": ["sh", "-ec", "echo 'Pinging'; STATUS=$(curl -s -o /tmp/output -w '%{http_code}' http://taskcluster/test-proxy/v1/ping); cat /tmp/output; test $STATUS -eq 200;"]
 		}`,
 	}
 
@@ -145,15 +133,13 @@ func TestArtifacts(t *testing.T) {
 		FileNotFoundPath:   "/home/tc/no-such-file.txt",
 		FolderNotFoundPath: "/home/tc/no-such-folder/",
 		NestedFolderFiles: []string{
-			"/home/tc/folder/hello.txt",
-			"/home/tc/folder/sub-folder/hello2.txt",
+			"hello.txt",
+			"sub-folder/hello2.txt",
 		},
 		NestedFolderPath: "/home/tc/folder/",
 		Payload: `{
-			"start": {
-				"image": "` + s.URL + `",
-				"command": ["sh", "-ec", "mkdir -p /home/tc/folder/sub-folder; echo '[hello-world]' > /home/tc/folder/hello.txt; echo '[hello-world]' > /home/tc/folder/sub-folder/hello2.txt"]
-			}
+			"image": "` + s.URL + `",
+			"command": ["sh", "-ec", "mkdir -p /home/tc/folder/sub-folder; echo '[hello-world]' > /home/tc/folder/hello.txt; echo '[hello-world]' > /home/tc/folder/sub-folder/hello2.txt"]
 		}`,
 	}
 
@@ -174,10 +160,8 @@ func TestShell(t *testing.T) {
 		BadCommand:     "exit 1;\n",
 		SleepCommand:   "sleep 30;\n",
 		Payload: `{
-	    "start": {
-	      "image": "` + s.URL + `",
-	      "command": ["sh", "-c", "true"]
-	    }
+      "image": "` + s.URL + `",
+      "command": ["sh", "-c", "true"]
 	  }`,
 	}
 
@@ -200,11 +184,9 @@ func TestDisplay(t *testing.T) {
 		},
 		InvalidDisplayName: "invalid-screen",
 		Payload: `{
-	    "start": {
-	      "image": "` + s.URL + `",
-	      "command": ["sh", "-c", "true"]
-	    }
-	  }`,
+      "image": "` + s.URL + `",
+      "command": ["sh", "-c", "true"]
+    }`,
 	}
 
 	c.TestListDisplays()

engines/qemu/resultset.go

@@ -1,6 +1,8 @@
 package qemuengine
 
 import (
+	"strings"
+
 	"github.com/taskcluster/taskcluster-worker/engines"
 	"github.com/taskcluster/taskcluster-worker/engines/qemu/metaservice"
 	"github.com/taskcluster/taskcluster-worker/engines/qemu/vm"
@@ -44,6 +46,12 @@ func (r *resultSet) ExtractFolder(path string, handler engines.FileHandler) erro
 		if err != nil {
 			return err
 		}
+		// If guest uses backslashes our input paths should have that, but the ones
+		// we return should be intepreted as names.
+		p = strings.Replace(p[len(path):], "\\", "/", 0)
+		if len(p) > 0 && p[0] == '\\' {
+			p = p[1:]
+		}
 		if handler(p, f) != nil {
 			return engines.ErrHandlerInterrupt
 		}

runtime/environment.go

@@ -9,7 +9,7 @@ import (
 
 // Environment is a collection of objects that makes up a runtime environment.
 type Environment struct {
-	GarbageCollector *gc.GarbageCollector
+	GarbageCollector gc.ResourceTracker
 	//TODO: Add some sort of interface to the system logger
 	//TODO: Add some interface to submit statistics for influxdb/signalfx
 	//TODO: Add some interface to attach a http.Handler to public facing server

runtime/gc/gc.go

@@ -13,6 +13,15 @@ func indexOfResource(resources []Disposable, resource Disposable) int {
 	return -1
 }
 
+// A ResourceTracker is an object capable of tracking resources.
+//
+// This is the interface for the GarbageCollector that should be exposed to
+// engines and plugins. So they can't initiate garbage collection.
+type ResourceTracker interface {
+	Register(resource Disposable)
+	Unregister(resource Disposable) bool
+}
+
 // GarbageCollector can be used register Disposable resources which will then
 // be diposed when not in use and the system is low on available disk space
 // or memory.