Merge pull request #6788 from taskcluster/matt-boris/gwMountsCopyBefo…

…reExtracting

fix(generic-worker): copy archive to task user dir before unarchiving

changelog/Tvnvy_n2T5SCY2pHwdYHVA.md

@@ -0,0 +1,4 @@
+audience: general
+level: patch
+---
+Generic Worker now copies archives to the task user's directory before unarchiving.

workers/generic-worker/mounts.go

@@ -714,24 +714,38 @@ func ensureCached(fsContent FSContent, taskMount *TaskMount) (file string, err e
 	return
 }
 
-func extract(fsContent FSContent, format string, dir string, taskMount *TaskMount) error {
-	cacheFile, err := ensureCached(fsContent, taskMount)
+func extract(fsContent FSContent, format string, dir string, taskMount *TaskMount) (err error) {
+	var cacheFile string
+	cacheFile, err = ensureCached(fsContent, taskMount)
 	if err != nil {
 		log.Printf("Could not cache content: %v", err)
-		return err
+		return
 	}
 	err = MkdirAll(taskMount, dir)
 	if err != nil {
-		return err
+		return
 	}
-	err = makeFileReadWritableForTaskUser(taskMount, cacheFile)
+	copyToPath := filepath.Join(taskContext.TaskDir, filepath.Base(cacheFile))
+	defer func() {
+		taskMount.Infof("Removing file '%v'", copyToPath)
+		err2 := os.Remove(copyToPath)
+		if err == nil {
+			err = err2
+		}
+	}()
+	taskMount.Infof("Copying file '%v' to '%v'", cacheFile, copyToPath)
+	_, err = fileutil.Copy(copyToPath, cacheFile)
 	if err != nil {
-		return err
+		return
+	}
+	err = makeFileReadWritableForTaskUser(taskMount, copyToPath)
+	if err != nil {
+		return
 	}
-	taskMount.Infof("Extracting %v file %v to '%v'", format, cacheFile, dir)
+	taskMount.Infof("Extracting %v file %v to '%v'", format, copyToPath, dir)
 	// Useful for worker logs too (not just task logs)
-	log.Printf("[mounts] Extracting %v file %v to '%v'", format, cacheFile, dir)
-	return unarchive(cacheFile, dir, format)
+	log.Printf("[mounts] Extracting %v file %v to '%v'", format, copyToPath, dir)
+	return unarchive(copyToPath, dir, format)
 }
 
 func decompress(fsContent FSContent, format string, file string, taskMount *TaskMount) error {

workers/generic-worker/mounts_test.go

@@ -332,11 +332,13 @@ func TestValidSHA256(t *testing.T) {
 		`Downloaded 4220 bytes with SHA256 625554ec8ce731e486a5fb904f3331d18cf84a944dd9e40c19550686d4e8492e from task ` + taskID + ` artifact public/build/unknown_issuer_app_1.zip to .*`,
 		`Content from task ` + taskID + ` artifact public/build/unknown_issuer_app_1.zip \(.*\) matches required SHA256 625554ec8ce731e486a5fb904f3331d18cf84a944dd9e40c19550686d4e8492e`,
 		`Creating directory .*unknown_issuer_app_1`,
+		`Copying file '.*' to '.*'`,
 	},
 		grantingCacheFile...,
 	)
 	pass1 = append(pass1,
 		`Extracting zip file .* to '.*unknown_issuer_app_1'`,
+		`Removing file '.*'`,
 	)
 	pass1 = append(pass1,
 		grantingDir...,
@@ -346,11 +348,13 @@ func TestValidSHA256(t *testing.T) {
 	pass2 := append([]string{
 		`Found existing download for artifact:` + taskID + `:public/build/unknown_issuer_app_1.zip \(.*\) with correct SHA256 625554ec8ce731e486a5fb904f3331d18cf84a944dd9e40c19550686d4e8492e`,
 		`Creating directory .*unknown_issuer_app_1`,
+		`Copying file '.*' to '.*'`,
 	},
 		grantingCacheFile...,
 	)
 	pass2 = append(pass2,
 		`Extracting zip file .* to '.*unknown_issuer_app_1'`,
+		`Removing file '.*'`,
 	)
 	pass2 = append(pass2,
 		grantingDir...,
@@ -554,11 +558,13 @@ func TestWritableDirectoryCacheNoSHA256(t *testing.T) {
 		`Downloaded 4220 bytes with SHA256 625554ec8ce731e486a5fb904f3331d18cf84a944dd9e40c19550686d4e8492e from task ` + taskID + ` artifact public/build/unknown_issuer_app_1.zip to .*`,
 		`Download .* of task ` + taskID + ` artifact public/build/unknown_issuer_app_1.zip has SHA256 625554ec8ce731e486a5fb904f3331d18cf84a944dd9e40c19550686d4e8492e but task payload does not declare a required value, so content authenticity cannot be verified`,
 		`Creating directory .*` + t.Name(),
+		`Copying file '.*' to '.*'`,
 	},
 		grantingCacheFile...,
 	)
 	pass1 = append(pass1,
 		`Extracting zip file .* to '.*`+t.Name()+`'`,
+		`Removing file '.*'`,
 	)
 	pass1 = append(pass1,
 		grantingDir...,
@@ -853,11 +859,13 @@ func TestCacheMoved(t *testing.T) {
 		`Downloaded 4220 bytes with SHA256 625554ec8ce731e486a5fb904f3331d18cf84a944dd9e40c19550686d4e8492e from task ` + taskID + ` artifact public/build/unknown_issuer_app_1.zip to .*`,
 		`Content from task ` + taskID + ` artifact public/build/unknown_issuer_app_1.zip \(.*\) matches required SHA256 625554ec8ce731e486a5fb904f3331d18cf84a944dd9e40c19550686d4e8492e`,
 		`Creating directory .*` + t.Name(),
+		`Copying file '.*' to '.*'`,
 	},
 		grantingCacheFile...,
 	)
 	pass1 = append(pass1,
 		`Extracting zip file .* to '.*`+t.Name()+`'`,
+		`Removing file '.*'`,
 	)
 	pass1 = append(pass1,
 		grantingDir...,
@@ -875,11 +883,13 @@ func TestCacheMoved(t *testing.T) {
 		`No existing writable directory cache 'banana-cache' - creating .*`,
 		`Found existing download for artifact:` + taskID + `:public/build/unknown_issuer_app_1.zip \(.*\) with correct SHA256 625554ec8ce731e486a5fb904f3331d18cf84a944dd9e40c19550686d4e8492e`,
 		`Creating directory .*` + t.Name(),
+		`Copying file '.*' to '.*'`,
 	},
 		grantingCacheFile...,
 	)
 	pass2 = append(pass2,
 		`Extracting zip file .* to '.*`+t.Name()+`'`,
+		`Removing file '.*'`,
 	)
 	pass2 = append(pass2,
 		grantingDir...,