diff --git a/all-unix-style.yml b/all-unix-style.yml index 9bbad9a9..49f93398 100644 --- a/all-unix-style.yml +++ b/all-unix-style.yml @@ -377,3 +377,40 @@ definitions: additionalProperties: false required: - url + - title: Raw Content + description: |- + Import raw content. + + Since: generic-worker 10.11.3 + type: object + properties: + raw: + type: string + maxLength: 8000 + title: Raw + description: |- + Raw content. + + Since: generic-worker 10.11.3 + additionalProperties: false + required: + - raw + - title: Base64 Content + description: |- + Import Base64 encoded content. + + Since: generic-worker 10.11.3 + type: object + properties: + base64: + type: string + maxLength: 8000 + title: Base64 + description: |- + Base64 content. + + Since: generic-worker 10.11.3 + pattern: '^[A-Za-z0-9/+]+[=]{0,2}$' + additionalProperties: false + required: + - base64 diff --git a/generated_all-unix-style.go b/generated_all-unix-style.go index d3db09d0..87203438 100644 --- a/generated_all-unix-style.go +++ b/generated_all-unix-style.go @@ -78,6 +78,20 @@ type ( TaskID string `json:"taskId"` } + // Import Base64 encoded content. + // + // Since: generic-worker 10.11.3 + Base64Content struct { + + // Base64 content. + // + // Since: generic-worker 10.11.3 + // + // Syntax: ^[A-Za-z0-9/+]+[=]{0,2}$ + // Max length: 8000 + Base64 string `json:"base64"` + } + // By default tasks will be resolved with `state/reasonResolved`: `completed/completed` // if all task commands have a zero exit code, or `failed/failed` if any command has a // non-zero exit code. This payload property allows customsation of the task resolution @@ -92,9 +106,6 @@ type ( // See [itermittent tasks](https://docs.taskcluster.net/docs/reference/platform/taskcluster-queue/docs/worker-interaction#intermittent-tasks) for more detail. // // Since: generic-worker 10.10.0 - // - // Array items: - // Mininum: 1 Retry []int64 `json:"retry,omitempty"` } @@ -124,6 +135,8 @@ type ( // One of: // * ArtifactContent // * URLContent + // * RawContent + // * Base64Content Content json.RawMessage `json:"content"` // The filesystem location to mount the file. @@ -145,9 +158,6 @@ type ( // for several commands. // // Since: generic-worker 0.0.1 - // - // Array items: - // Array items: Command [][]string `json:"command"` // Env vars must be string to __string__ mappings (not number or boolean). For example: @@ -161,8 +171,6 @@ type ( // ``` // // Since: generic-worker 0.0.1 - // - // Map entries: Env map[string]string `json:"env,omitempty"` // Feature flags enable additional functionality. @@ -181,12 +189,6 @@ type ( // Directories and/or files to be mounted. // // Since: generic-worker 5.4.0 - // - // Array items: - // One of: - // * FileMount - // * WritableDirectoryCache - // * ReadOnlyDirectory Mounts []json.RawMessage `json:"mounts,omitempty"` // By default tasks will be resolved with `state/reasonResolved`: `completed/completed` @@ -200,8 +202,6 @@ type ( // provided. // // Since: generic-worker 6.0.0 - // - // Array items: OSGroups []string `json:"osGroups,omitempty"` // URL of a service that can indicate tasks superseding this one; the current `taskId` @@ -216,11 +216,26 @@ type ( SupersederURL string `json:"supersederUrl,omitempty"` } + // Import raw content. + // + // Since: generic-worker 10.11.3 + RawContent struct { + + // Raw content. + // + // Since: generic-worker 10.11.3 + // + // Max length: 8000 + Raw string `json:"raw"` + } + ReadOnlyDirectory struct { // One of: // * ArtifactContent // * URLContent + // * RawContent + // * Base64Content Content json.RawMessage `json:"content"` // The filesystem location to mount the directory volume. @@ -271,6 +286,8 @@ type ( // One of: // * ArtifactContent // * URLContent + // * RawContent + // * Base64Content Content json.RawMessage `json:"content,omitempty"` // The filesystem location to mount the directory volume. @@ -358,6 +375,41 @@ func taskPayloadSchema() string { ], "title": "URL Content", "type": "object" + }, + { + "additionalProperties": false, + "description": "Import raw content.\n\nSince: generic-worker 10.11.3", + "properties": { + "raw": { + "description": "Raw content.\n\nSince: generic-worker 10.11.3", + "maxLength": 8000, + "title": "Raw", + "type": "string" + } + }, + "required": [ + "raw" + ], + "title": "Raw Content", + "type": "object" + }, + { + "additionalProperties": false, + "description": "Import Base64 encoded content.\n\nSince: generic-worker 10.11.3", + "properties": { + "base64": { + "description": "Base64 content.\n\nSince: generic-worker 10.11.3", + "maxLength": 8000, + "pattern": "^[A-Za-z0-9/+]+[=]{0,2}$", + "title": "Base64", + "type": "string" + } + }, + "required": [ + "base64" + ], + "title": "Base64 Content", + "type": "object" } ] }, diff --git a/generated_windows.go b/generated_windows.go index 14b6a7d2..1159b0ad 100644 --- a/generated_windows.go +++ b/generated_windows.go @@ -76,6 +76,20 @@ type ( TaskID string `json:"taskId"` } + // Import Base64 encoded content. + // + // Since: generic-worker 10.11.3 + Base64Content struct { + + // Base64 content. + // + // Since: generic-worker 10.11.3 + // + // Syntax: ^[A-Za-z0-9/+]+[=]{0,2}$ + // Max length: 8000 + Base64 string `json:"base64"` + } + // By default tasks will be resolved with `state/reasonResolved`: `completed/completed` // if all task commands have a zero exit code, or `failed/failed` if any command has a // non-zero exit code. This payload property allows customsation of the task resolution @@ -90,9 +104,6 @@ type ( // See [itermittent tasks](https://docs.taskcluster.net/docs/reference/platform/taskcluster-queue/docs/worker-interaction#intermittent-tasks) for more detail. // // Since: generic-worker 10.10.0 - // - // Array items: - // Mininum: 1 Retry []int64 `json:"retry,omitempty"` } @@ -142,6 +153,8 @@ type ( // One of: // * ArtifactContent // * URLContent + // * RawContent + // * Base64Content Content json.RawMessage `json:"content"` // The filesystem location to mount the file. @@ -170,8 +183,6 @@ type ( // ``` // // Since: generic-worker 0.0.1 - // - // Array items: Command []string `json:"command"` // Env vars must be string to __string__ mappings (not number or boolean). For example: @@ -185,8 +196,6 @@ type ( // ``` // // Since: generic-worker 0.0.1 - // - // Map entries: Env map[string]string `json:"env,omitempty"` // Feature flags enable additional functionality. @@ -205,12 +214,6 @@ type ( // Directories and/or files to be mounted. // // Since: generic-worker 5.4.0 - // - // Array items: - // One of: - // * FileMount - // * WritableDirectoryCache - // * ReadOnlyDirectory Mounts []json.RawMessage `json:"mounts,omitempty"` // By default tasks will be resolved with `state/reasonResolved`: `completed/completed` @@ -224,8 +227,6 @@ type ( // group listed. // // Since: generic-worker 6.0.0 - // - // Array items: OSGroups []string `json:"osGroups,omitempty"` // Specifies an artifact name for publishing RDP connection information. @@ -268,11 +269,26 @@ type ( SupersederURL string `json:"supersederUrl,omitempty"` } + // Import raw content. + // + // Since: generic-worker 10.11.3 + RawContent struct { + + // Raw content. + // + // Since: generic-worker 10.11.3 + // + // Max length: 8000 + Raw string `json:"raw"` + } + ReadOnlyDirectory struct { // One of: // * ArtifactContent // * URLContent + // * RawContent + // * Base64Content Content json.RawMessage `json:"content"` // The filesystem location to mount the directory volume. @@ -323,6 +339,8 @@ type ( // One of: // * ArtifactContent // * URLContent + // * RawContent + // * Base64Content Content json.RawMessage `json:"content,omitempty"` // The filesystem location to mount the directory volume. @@ -410,6 +428,41 @@ func taskPayloadSchema() string { ], "title": "URL Content", "type": "object" + }, + { + "additionalProperties": false, + "description": "Import raw content.\n\nSince: generic-worker 10.11.3", + "properties": { + "raw": { + "description": "Raw content.\n\nSince: generic-worker 10.11.3", + "maxLength": 8000, + "title": "Raw", + "type": "string" + } + }, + "required": [ + "raw" + ], + "title": "Raw Content", + "type": "object" + }, + { + "additionalProperties": false, + "description": "Import Base64 encoded content.\n\nSince: generic-worker 10.11.3", + "properties": { + "base64": { + "description": "Base64 content.\n\nSince: generic-worker 10.11.3", + "maxLength": 8000, + "pattern": "^[A-Za-z0-9/+]+[=]{0,2}$", + "title": "Base64", + "type": "string" + } + }, + "required": [ + "base64" + ], + "title": "Base64 Content", + "type": "object" } ] }, diff --git a/generic-worker b/generic-worker new file mode 100755 index 00000000..86bf2479 Binary files /dev/null and b/generic-worker differ diff --git a/mounts.go b/mounts.go index 56d21027..a5e27480 100644 --- a/mounts.go +++ b/mounts.go @@ -178,8 +178,8 @@ type MountEntry interface { } // FSContent represents file system content - it is based on the auto-generated -// type Content which is json.RawMessage, which can be ArtifactContent or -// URLContent concrete types. This is the interface which represents these +// type Content which is json.RawMessage, which can be ArtifactContent, URLContent, +// RawContent or Base64Content concrete types. This is the interface which represents these // underlying concrete types. type FSContent interface { // Keep it simple and just return a []string, rather than scopes.Required @@ -212,6 +212,15 @@ func (ac *ArtifactContent) RequiredScopes() []string { return []string{"queue:get-artifact:" + ac.Artifact} } +//No scopes required to mount files in a task +func (rc *RawContent) RequiredScopes() []string { + return []string{} +} + +func (bc *Base64Content) RequiredScopes() []string { + return []string{} +} + // Since mounts are protected by scopes per mount, no reason to have // a feature flag to enable. Having mounts in the payload is enough. func (feature *MountsFeature) IsEnabled(task *TaskRun) bool { @@ -384,7 +393,7 @@ func (w *WritableDirectoryCache) RequiredScopes() []string { return []string{"generic-worker:cache:" + w.CacheName} } -// Returns either a *URLContent or *ArtifactContent that is listed in the given +// Returns either a *URLContent *ArtifactContent, *RawContent or *Base64Content that is listed in the given // *WritableDirectoryCache func (w *WritableDirectoryCache) FSContent() (FSContent, error) { // no content if an empty cache folder, e.g. object directory @@ -400,7 +409,7 @@ func (r *ReadOnlyDirectory) RequiredScopes() []string { return []string{} } -// Returns either a *URLContent or *ArtifactContent that is listed in the given +// Returns either a *URLContent, *ArtifactContent, *RawContent or *Base64Content that is listed in the given // *ReadOnlyDirectory func (r *ReadOnlyDirectory) FSContent() (FSContent, error) { return FSContentFrom(r.Content) @@ -412,7 +421,7 @@ func (f *FileMount) RequiredScopes() []string { return []string{} } -// Returns either a *URLContent or *ArtifactContent that is listed in the given +// Returns either a *URLContent, *ArtifactContent, *RawContent or *Base64Content that is listed in the given // *FileMount func (f *FileMount) FSContent() (FSContent, error) { return FSContentFrom(f.Content) @@ -662,12 +671,14 @@ func extract(fsContent FSContent, format string, dir string, task *TaskRun) erro return fmt.Errorf("Unsupported archive format %v", format) } -// Returns either a *ArtifactContent or *URLContent based on the content +// Returns either a *ArtifactContent or *URLContent or *RawContent or *Base64Content based on the content // (json.RawMessage) func FSContentFrom(c json.RawMessage) (FSContent, error) { // c must be one of: // * ArtifactContent // * URLContent + // * RawContent + // * Base64Content // We have to check keys to find out... var m map[string]interface{} if err := json.Unmarshal(c, &m); err != nil { @@ -678,7 +689,12 @@ func FSContentFrom(c json.RawMessage) (FSContent, error) { return UnmarshalInto(c, &ArtifactContent{}) case m["url"] != nil: return UnmarshalInto(c, &URLContent{}) + case m["raw"] != nil: + return UnmarshalInto(c, &RawContent{}) + case m["base64"] != nil: + return UnmarshalInto(c, &Base64Content{}) } + return nil, errors.New("Unrecognised mount entry in payload") } @@ -792,6 +808,75 @@ func downloadURLToFile(url, contentSource, file string, task *TaskRun) (sha256 s return } +//RawContent to file +func (rc *RawContent) Download(task *TaskRun) (file string, sha256 string, err error) { + basename := slugid.Nice() + file = filepath.Join(config.DownloadsDir, basename) + sha256 = "" + err = writeStringtoFile(rc.Raw, rc.String(), file, task) + return +} + +func (rc *RawContent) String() string { + return "Raw (" + rc.Raw + ")" +} + +func (rc *RawContent) UniqueKey() string { + return "Raw content: " + rc.Raw +} + +func (rc *RawContent) RequiredSHA256() string { + return "" +} + +func (rc *RawContent) TaskDependencies() []string { + return []string{} +} + +//Base64Content to file +func (bc *Base64Content) Download(task *TaskRun) (file string, sha256 string, err error) { + basename := slugid.Nice() + file = filepath.Join(config.DownloadsDir, basename) + sha256 = "" + err = writeStringtoFile(bc.Base64, bc.String(), file, task) + return +} + +func (bc *Base64Content) String() string { + return "Base64 (" + bc.Base64 + ")" +} + +func (bc *Base64Content) UniqueKey() string { + return "Base64 content: " + bc.Base64 +} + +func (bc *Base64Content) RequiredSHA256() string { + return "" +} + +func (bc *Base64Content) TaskDependencies() []string { + return []string{} +} + +//Copying String to File +func writeStringtoFile(content, contentSource, file string, task *TaskRun) (err error) { + task.Infof("[mounts] Copying %v to %v", contentSource, file) + f, err := os.OpenFile(file, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600) + if err != nil { + task.Errorf("[mounts] Could not open file %v: %v", file, err) + // permanent error! + return err + } + defer f.Close() + contentSize, err := f.WriteString(content) + if err != nil { + task.Errorf("[mounts] Could not copy %v to %v", contentSource, file) + return err + } + task.Infof("[mounts] Copied %v bytes from %v to %v", contentSize, contentSource, file) + return err +} + func (taskMount *TaskMount) purgeCaches() error { // Don't bother to query purge cache service if this task uses no writable // caches, and we queried already less than 6 hours ago. Service keeps diff --git a/mounts_test.go b/mounts_test.go index d56c968d..eabfef1a 100644 --- a/mounts_test.go +++ b/mounts_test.go @@ -49,6 +49,22 @@ func TestMounts(t *testing.T) { }`), }, + //file mount from raw + &FileMount{ + File: filepath.Join("preloaded", "raw.txt"), + Content: json.RawMessage(`{ + "raw": "Hello Raw Content!" + }`), + }, + + //file mount from base64 + &FileMount{ + File: filepath.Join("preloaded", "base64"), + Content: json.RawMessage(`{ + "base64": "ZWNobyAiSGVsbG8gQmFzZTY0ISI=" + }`), + }, + // empty writable directory cache &WritableDirectoryCache{ CacheName: "banana-cache", diff --git a/windows.yml b/windows.yml index dbea5bae..0e3cb313 100644 --- a/windows.yml +++ b/windows.yml @@ -431,3 +431,40 @@ definitions: additionalProperties: false required: - url + - title: Raw Content + description: |- + Import raw content. + + Since: generic-worker 10.11.3 + type: object + properties: + raw: + type: string + maxLength: 8000 + title: Raw + description: |- + Raw content. + + Since: generic-worker 10.11.3 + additionalProperties: false + required: + - raw + - title: Base64 Content + description: |- + Import Base64 encoded content. + + Since: generic-worker 10.11.3 + type: object + properties: + base64: + type: string + maxLength: 8000 + title: Base64 + description: |- + Base64 content. + + Since: generic-worker 10.11.3 + pattern: '^[A-Za-z0-9/+]+[=]{0,2}$' + additionalProperties: false + required: + - base64