diff --git a/Gopkg.lock b/Gopkg.lock index 9133a37f..298503ba 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -200,7 +200,7 @@ version = "v1.1.1" [[projects]] - digest = "1:48f2d47b8f38bc88e0381a2a44ed2a8ccb12f268b71fe5aefa528e25a6f7256d" + digest = "1:b74af793d3b5df6ff997741106b71fc503abbbe40088355ccbd52260542de8be" name = "github.com/deislabs/cnab-go" packages = [ "action", @@ -217,8 +217,8 @@ "utils/crud", ] pruneopts = "NUT" - revision = "93515c713a91d6da48b5a9c68e4b0502d8d39963" - version = "v0.4.0-beta1" + revision = "406808480de1f033040608d1924f96dcd662c2d0" + version = "v0.6.0-beta1" [[projects]] digest = "1:7a6852b35eb5bbc184561443762d225116ae630c26a7c4d90546619f1e7d2ad2" diff --git a/Gopkg.toml b/Gopkg.toml index 9004f408..b49355c8 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -57,7 +57,7 @@ [[constraint]] name = "github.com/deislabs/cnab-go" - version = "v0.4.0-beta1" + version = "v0.6.0-beta1" [[override]] name = "github.com/google/go-containerregistry" diff --git a/vendor/github.com/deislabs/cnab-go/action/action.go b/vendor/github.com/deislabs/cnab-go/action/action.go index 5faec05a..1fb919bb 100644 --- a/vendor/github.com/deislabs/cnab-go/action/action.go +++ b/vendor/github.com/deislabs/cnab-go/action/action.go @@ -175,18 +175,6 @@ func getImageMap(b *bundle.Bundle) ([]byte, error) { return json.Marshal(imgs) } -func appliesToAction(action string, parameter bundle.Parameter) bool { - if len(parameter.ApplyTo) == 0 { - return true - } - for _, act := range parameter.ApplyTo { - if action == act { - return true - } - } - return false -} - func opFromClaim(action string, stateless bool, c *claim.Claim, ii bundle.InvocationImage, creds credentials.Set) (*driver.Operation, error) { env, files, err := creds.Expand(c.Bundle, stateless) if err != nil { @@ -204,6 +192,13 @@ func opFromClaim(action string, stateless bool, c *claim.Claim, ii bundle.Invoca return nil, err } + bundleBytes, err := json.Marshal(c.Bundle) + if err != nil { + return nil, fmt.Errorf("failed to marshal bundle contents: %s", err) + } + + files["/cnab/bundle.json"] = string(bundleBytes) + imgMap, err := getImageMap(c.Bundle) if err != nil { return nil, fmt.Errorf("unable to generate image map: %s", err) @@ -231,6 +226,7 @@ func opFromClaim(action string, stateless bool, c *claim.Claim, ii bundle.Invoca Environment: env, Files: files, Outputs: outputs, + Bundle: c.Bundle, }, nil } @@ -238,7 +234,7 @@ func injectParameters(action string, c *claim.Claim, env, files map[string]strin for k, param := range c.Bundle.Parameters { rawval, ok := c.Parameters[k] if !ok { - if param.Required && appliesToAction(action, param) { + if param.Required && param.AppliesTo(action) { return fmt.Errorf("missing required parameter %q for action %q", k, action) } continue diff --git a/vendor/github.com/deislabs/cnab-go/bundle/definition/schema.go b/vendor/github.com/deislabs/cnab-go/bundle/definition/schema.go index e756cc34..25269f7c 100644 --- a/vendor/github.com/deislabs/cnab-go/bundle/definition/schema.go +++ b/vendor/github.com/deislabs/cnab-go/bundle/definition/schema.go @@ -6,7 +6,6 @@ import ( "strings" "github.com/pkg/errors" - "github.com/qri-io/jsonschema" ) type Definitions map[string]*Schema @@ -96,7 +95,7 @@ func (s *Schema) UnmarshalJSON(data []byte) error { // Before we unmarshal into the cnab-go bundle/definition/Schema type, unmarshal into // the library struct so we can handle any validation errors in the schema. If there // are any errors, return those. - js := new(jsonschema.RootSchema) + js := NewRootSchema() if err := js.UnmarshalJSON(data); err != nil { return err } diff --git a/vendor/github.com/deislabs/cnab-go/bundle/definition/validation.go b/vendor/github.com/deislabs/cnab-go/bundle/definition/validation.go index 4a1511e2..058d9f92 100644 --- a/vendor/github.com/deislabs/cnab-go/bundle/definition/validation.go +++ b/vendor/github.com/deislabs/cnab-go/bundle/definition/validation.go @@ -4,7 +4,6 @@ import ( "encoding/json" "github.com/pkg/errors" - "github.com/qri-io/jsonschema" ) // ValidationError error represents a validation error @@ -24,7 +23,7 @@ func (s *Schema) Validate(data interface{}) ([]ValidationError, error) { if err != nil { return nil, errors.Wrap(err, "unable to load schema") } - def := new(jsonschema.RootSchema) + def := NewRootSchema() err = json.Unmarshal([]byte(b), def) if err != nil { return nil, errors.Wrap(err, "unable to build schema") diff --git a/vendor/github.com/deislabs/cnab-go/bundle/definition/validators.go b/vendor/github.com/deislabs/cnab-go/bundle/definition/validators.go new file mode 100644 index 00000000..ddc56a81 --- /dev/null +++ b/vendor/github.com/deislabs/cnab-go/bundle/definition/validators.go @@ -0,0 +1,45 @@ +package definition + +import ( + "encoding/base64" + "fmt" + + "github.com/qri-io/jsonschema" +) + +// ContentEncoding represents a "custom" Schema property +type ContentEncoding string + +// NewContentEncoding allocates a new ContentEncoding validator +func NewContentEncoding() jsonschema.Validator { + return new(ContentEncoding) +} + +// Validate implements the Validator interface for ContentEncoding +// which, as of writing, isn't included by default in the jsonschema library we consume +func (c ContentEncoding) Validate(propPath string, data interface{}, errs *[]jsonschema.ValError) { + if obj, ok := data.(string); ok { + switch c { + case "base64": + _, err := base64.StdEncoding.DecodeString(obj) + if err != nil { + jsonschema.AddError(errs, propPath, data, fmt.Sprintf("invalid %s value: %s", c, obj)) + } + // Add validation support for other encodings as needed + // See https://json-schema.org/latest/json-schema-validation.html#rfc.section.8.3 + default: + jsonschema.AddError(errs, propPath, data, fmt.Sprintf("unsupported or invalid contentEncoding type of %s", c)) + } + } +} + +// NewRootSchema returns a jsonschema.RootSchema with any needed custom +// jsonschema.Validators pre-registered +func NewRootSchema() *jsonschema.RootSchema { + // Register custom validators here + // Note: as of writing, jsonschema doesn't have a stock validator for instances of type `contentEncoding` + // There may be others missing in the library that exist in http://json-schema.org/draft-07/schema# + // and thus, we'd need to create/register them here (if not included upstream) + jsonschema.RegisterValidator("contentEncoding", NewContentEncoding) + return new(jsonschema.RootSchema) +} diff --git a/vendor/github.com/deislabs/cnab-go/bundle/outputs.go b/vendor/github.com/deislabs/cnab-go/bundle/outputs.go index a17cfa80..c5aa5ded 100644 --- a/vendor/github.com/deislabs/cnab-go/bundle/outputs.go +++ b/vendor/github.com/deislabs/cnab-go/bundle/outputs.go @@ -6,3 +6,17 @@ type Output struct { Description string `json:"description,omitempty" yaml:"description,omitempty"` Path string `json:"path" yaml:"path"` } + +// AppliesTo returns a boolean value specifying whether or not +// the Output applies to the provided action +func (output *Output) AppliesTo(action string) bool { + if len(output.ApplyTo) == 0 { + return true + } + for _, act := range output.ApplyTo { + if action == act { + return true + } + } + return false +} diff --git a/vendor/github.com/deislabs/cnab-go/bundle/parameters.go b/vendor/github.com/deislabs/cnab-go/bundle/parameters.go index c461c577..de3992f7 100644 --- a/vendor/github.com/deislabs/cnab-go/bundle/parameters.go +++ b/vendor/github.com/deislabs/cnab-go/bundle/parameters.go @@ -8,3 +8,17 @@ type Parameter struct { Destination *Location `json:"destination,omitemtpty" yaml:"destination,omitempty"` Required bool `json:"required,omitempty" yaml:"required,omitempty"` } + +// AppliesTo returns a boolean value specifying whether or not +// the Parameter applies to the provided action +func (parameter *Parameter) AppliesTo(action string) bool { + if len(parameter.ApplyTo) == 0 { + return true + } + for _, act := range parameter.ApplyTo { + if action == act { + return true + } + } + return false +} diff --git a/vendor/github.com/deislabs/cnab-go/claim/claim.go b/vendor/github.com/deislabs/cnab-go/claim/claim.go index 1355c512..95c83202 100644 --- a/vendor/github.com/deislabs/cnab-go/claim/claim.go +++ b/vendor/github.com/deislabs/cnab-go/claim/claim.go @@ -48,7 +48,7 @@ type Claim struct { } // ValidName is a regular expression that indicates whether a name is a valid claim name. -var ValidName = regexp.MustCompile("^[a-zA-Z0-9_-]+$") +var ValidName = regexp.MustCompile("^[a-zA-Z0-9._-]+$") // New creates a new Claim initialized for an installation operation. func New(name string) (*Claim, error) { diff --git a/vendor/github.com/deislabs/cnab-go/driver/docker/docker.go b/vendor/github.com/deislabs/cnab-go/driver/docker/docker.go index 0e8433bf..5d96830c 100644 --- a/vendor/github.com/deislabs/cnab-go/driver/docker/docker.go +++ b/vendor/github.com/deislabs/cnab-go/driver/docker/docker.go @@ -55,11 +55,20 @@ func (d *Driver) Config() map[string]string { "PULL_ALWAYS": "Always pull image, even if locally available (0|1)", "DOCKER_DRIVER_QUIET": "Make the Docker driver quiet (only print container stdout/stderr)", "OUTPUTS_MOUNT_PATH": "Absolute path to where Docker driver can create temporary directories to bundle outputs. Defaults to temp dir.", + "CLEANUP_CONTAINERS": "If true, the docker container will be destroyed when it finishes running. If false, it will not be destroyed. The supported values are true and false. Defaults to true.", } } // SetConfig sets Docker driver configuration func (d *Driver) SetConfig(settings map[string]string) { + // Set default and provide feedback on acceptable input values. + value, ok := settings["CLEANUP_CONTAINERS"] + if !ok { + settings["CLEANUP_CONTAINERS"] = "true" + } else if value != "true" && value != "false" { + fmt.Printf("CLEANUP_CONTAINERS environment variable has unexpected value %q. Supported values are 'true', 'false', or unset.", value) + } + d.config = settings } @@ -175,7 +184,9 @@ func (d *Driver) exec(op *driver.Operation) (driver.OperationResult, error) { return driver.OperationResult{}, fmt.Errorf("cannot create container: %v", err) } - defer cli.Client().ContainerRemove(ctx, resp.ID, types.ContainerRemoveOptions{}) + if d.config["CLEANUP_CONTAINERS"] == "true" { + defer cli.Client().ContainerRemove(ctx, resp.ID, types.ContainerRemoveOptions{}) + } tarContent, err := generateTar(op.Files) if err != nil { @@ -301,9 +312,36 @@ func (d *Driver) fetchOutputs(ctx context.Context, container string, op *driver. return opResult, err } + // if an applicable output is expected but does not exist and it has a + // non-empty default value, create an entry in the map with the + // default value as its contents + for name, output := range op.Bundle.Outputs { + filepath := unix_path.Join("/cnab", "app", "outputs", name) + if !existsInOutputsMap(opResult.Outputs, filepath) && output.AppliesTo(op.Action) { + if outputDefinition, exists := op.Bundle.Definitions[output.Definition]; exists { + outputDefault := outputDefinition.Default + if outputDefault != nil { + contents := fmt.Sprintf("%v", outputDefault) + opResult.Outputs[filepath] = contents + } else { + return opResult, fmt.Errorf("required output %s is missing and has no default", name) + } + } + } + } + return opResult, nil } +func existsInOutputsMap(outputsMap map[string]string, path string) bool { + for outputPath := range outputsMap { + if outputPath == path { + return true + } + } + return false +} + func generateTar(files map[string]string) (io.Reader, error) { r, w := io.Pipe() tw := tar.NewWriter(w) diff --git a/vendor/github.com/deislabs/cnab-go/driver/driver.go b/vendor/github.com/deislabs/cnab-go/driver/driver.go index d338ce25..7c93e2df 100644 --- a/vendor/github.com/deislabs/cnab-go/driver/driver.go +++ b/vendor/github.com/deislabs/cnab-go/driver/driver.go @@ -36,6 +36,8 @@ type Operation struct { Outputs []string `json:"outputs"` // Output stream for log messages from the driver Out io.Writer `json:"-"` + // Bundle represents the bundle information for use by the operation + Bundle *bundle.Bundle } // ResolvedCred is a credential that has been resolved and is ready for injection into the runtime. diff --git a/vendor/github.com/deislabs/cnab-go/driver/kubernetes/kubernetes.go b/vendor/github.com/deislabs/cnab-go/driver/kubernetes/kubernetes.go index b77aef12..30e5e360 100644 --- a/vendor/github.com/deislabs/cnab-go/driver/kubernetes/kubernetes.go +++ b/vendor/github.com/deislabs/cnab-go/driver/kubernetes/kubernetes.go @@ -6,14 +6,13 @@ import ( "os" "path/filepath" "strings" + "time" // load credential helpers _ "k8s.io/client-go/plugin/pkg/client/auth" // Convert transitive deps to direct deps so that we can use constraints in our Gopkg.toml _ "github.com/Azure/go-autorest/autorest" - "github.com/deislabs/cnab-go/bundle" - "github.com/deislabs/cnab-go/driver" batchv1 "k8s.io/api/batch/v1" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -23,19 +22,25 @@ import ( coreclientv1 "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" + + "github.com/deislabs/cnab-go/bundle" + "github.com/deislabs/cnab-go/driver" ) const ( k8sContainerName = "invocation" k8sFileSecretVolume = "files" + numBackoffLoops = 6 ) // Driver runs an invocation image in a Kubernetes cluster. type Driver struct { Namespace string ServiceAccountName string + Annotations map[string]string LimitCPU resource.Quantity LimitMemory resource.Quantity + Tolerations []v1.Toleration ActiveDeadlineSeconds int64 BackoffLimit int32 SkipCleanup bool @@ -68,7 +73,7 @@ func (k *Driver) Config() map[string]string { return map[string]string{ "KUBE_NAMESPACE": "Kubernetes namespace in which to run the invocation image", "SERVICE_ACCOUNT": "Kubernetes service account to be mounted by the invocation image (if empty, no service account token will be mounted)", - "KUBE_CONFIG": "Absolute path to the kubeconfig file", + "KUBECONFIG": "Absolute path to the kubeconfig file", "MASTER_URL": "Kubernetes master endpoint", } } @@ -80,7 +85,7 @@ func (k *Driver) SetConfig(settings map[string]string) { k.ServiceAccountName = settings["SERVICE_ACCOUNT"] var kubeconfig string - if kpath := settings["KUBE_CONFIG"]; kpath != "" { + if kpath := settings["KUBECONFIG"]; kpath != "" { kubeconfig = kpath } else if home := homeDir(); home != "" { kubeconfig = filepath.Join(home, ".kube", "config") @@ -141,12 +146,14 @@ func (k *Driver) Run(op *driver.Operation) (driver.OperationResult, error) { BackoffLimit: &k.BackoffLimit, Template: v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Labels: labelMap, + Labels: labelMap, + Annotations: k.Annotations, }, Spec: v1.PodSpec{ ServiceAccountName: k.ServiceAccountName, AutomountServiceAccountToken: &mountServiceAccountToken, RestartPolicy: v1.RestartPolicyNever, + Tolerations: k.Tolerations, }, }, }, @@ -230,22 +237,29 @@ func (k *Driver) Run(op *driver.Operation) (driver.OperationResult, error) { return driver.OperationResult{}, nil } - selector := metav1.ListOptions{ + // Create a selector to detect the job just created + jobSelector := metav1.ListOptions{ LabelSelector: labels.Set(job.ObjectMeta.Labels).String(), + FieldSelector: newSingleFieldSelector("metadata.name", job.ObjectMeta.Name), + } + + // Prevent detecting pods from prior jobs by adding the job name to the labels + podSelector := metav1.ListOptions{ + LabelSelector: newSingleFieldSelector("job-name", job.ObjectMeta.Name), } - return driver.OperationResult{}, k.watchJobStatusAndLogs(selector, op.Out) + return driver.OperationResult{}, k.watchJobStatusAndLogs(podSelector, jobSelector, op.Out) } -func (k *Driver) watchJobStatusAndLogs(selector metav1.ListOptions, out io.Writer) error { +func (k *Driver) watchJobStatusAndLogs(podSelector metav1.ListOptions, jobSelector metav1.ListOptions, out io.Writer) error { // Stream Pod logs in the background logsStreamingComplete := make(chan bool) - err := k.streamPodLogs(selector, out, logsStreamingComplete) + err := k.streamPodLogs(podSelector, out, logsStreamingComplete) if err != nil { return err } // Watch job events and exit on failure/success - watch, err := k.jobs.Watch(selector) + watch, err := k.jobs.Watch(jobSelector) if err != nil { return err } @@ -302,22 +316,36 @@ func (k *Driver) streamPodLogs(options metav1.ListOptions, out io.Writer, done c // The event was for a pod whose logs have already been streamed, so do nothing. continue } - req := k.pods.GetLogs(podName, &v1.PodLogOptions{ - Container: k8sContainerName, - Follow: true, - }) - reader, err := req.Stream() - // There was an error connecting to the pod, so continue the loop and attempt streaming - // logs again next time there is an event for the same pod. - if err != nil { - continue + + for i := 0; i < numBackoffLoops; i++ { + time.Sleep(time.Duration(i*i/2) * time.Second) + req := k.pods.GetLogs(podName, &v1.PodLogOptions{ + Container: k8sContainerName, + Follow: true, + }) + reader, err := req.Stream() + if err != nil { + // There was an error connecting to the pod, so continue the loop and attempt streaming + // the logs again. + continue + } + + // Block the loop until all logs from the pod have been processed. + bytesRead, err := io.Copy(out, reader) + reader.Close() + if err != nil { + continue + } + if bytesRead == 0 { + // There is a chance where we have connected to the pod, but it has yet to write something. + // In that case, we continue to to keep streaming until it does. + continue + } + // Set the pod to have successfully streamed data. + streamedLogs[podName] = true + break } - // We successfully connected to the pod, so mark it as having streamed logs. - streamedLogs[podName] = true - // Block the loop until all logs from the pod have been processed. - io.Copy(out, reader) - reader.Close() done <- true } }() @@ -373,6 +401,12 @@ func generateFileSecret(files map[string]string) (*v1.Secret, []v1.VolumeMount) return secret, mounts } +func newSingleFieldSelector(k, v string) string { + return labels.Set(map[string]string{ + k: v, + }).String() +} + func homeDir() string { if h := os.Getenv("HOME"); h != "" { return h