diff --git a/apis/swagger.yml b/apis/swagger.yml index fac5b0c9c..0fcb90631 100644 --- a/apis/swagger.yml +++ b/apis/swagger.yml @@ -2486,15 +2486,27 @@ definitions: ContainerUpgradeConfig: description: | - ContainerUpgradeConfig is used for API "POST /containers/upgrade". - It wraps all kinds of config used in container upgrade. - It can be used to encode client params in client and unmarshal request body in daemon side. - allOf: - - $ref: "#/definitions/ContainerConfig" - - type: "object" - properties: - HostConfig: - $ref: "#/definitions/HostConfig" + ContainerUpgradeConfig is used for API "POST /containers/{name:.*}/upgrade". when upgrade a container, + we must specify new image used to create a new container, and also can specify `Cmd` and `Entrypoint` for + new container. There is all parameters that upgrade a container, if want to change other parameters, i + think you should use `update` API interface. + required: [Image] + properties: + Image: + type: "string" + x-nullable: false + Cmd: + type: "array" + description: "Execution commands and args" + items: + type: "string" + Entrypoint: + description: | + The entrypoint for the container as a string or an array of strings. + If the array consists of exactly one empty string (`[""]`) then the entry point is reset to system default. + type: "array" + items: + type: "string" LogConfig: description: "The logging configuration for this container" diff --git a/apis/types/container_upgrade_config.go b/apis/types/container_upgrade_config.go index 00bb61400..68b4151ff 100644 --- a/apis/types/container_upgrade_config.go +++ b/apis/types/container_upgrade_config.go @@ -10,78 +10,86 @@ import ( "github.com/go-openapi/errors" "github.com/go-openapi/swag" + "github.com/go-openapi/validate" ) -// ContainerUpgradeConfig ContainerUpgradeConfig is used for API "POST /containers/upgrade". -// It wraps all kinds of config used in container upgrade. -// It can be used to encode client params in client and unmarshal request body in daemon side. +// ContainerUpgradeConfig ContainerUpgradeConfig is used for API "POST /containers/{name:.*}/upgrade". when upgrade a container, +// we must specify new image used to create a new container, and also can specify `Cmd` and `Entrypoint` for +// new container. There is all parameters that upgrade a container, if want to change other parameters, i +// think you should use `update` API interface. // // swagger:model ContainerUpgradeConfig type ContainerUpgradeConfig struct { - ContainerConfig - // host config - HostConfig *HostConfig `json:"HostConfig,omitempty"` + // Execution commands and args + Cmd []string `json:"Cmd"` + + // The entrypoint for the container as a string or an array of strings. + // If the array consists of exactly one empty string (`[""]`) then the entry point is reset to system default. + // + Entrypoint []string `json:"Entrypoint"` + + // image + // Required: true + Image string `json:"Image"` } -// UnmarshalJSON unmarshals this object from a JSON structure -func (m *ContainerUpgradeConfig) UnmarshalJSON(raw []byte) error { +/* polymorph ContainerUpgradeConfig Cmd false */ - var aO0 ContainerConfig - if err := swag.ReadJSON(raw, &aO0); err != nil { - return err - } - m.ContainerConfig = aO0 +/* polymorph ContainerUpgradeConfig Entrypoint false */ - var data struct { - HostConfig *HostConfig `json:"HostConfig,omitempty"` +/* polymorph ContainerUpgradeConfig Image false */ + +// Validate validates this container upgrade config +func (m *ContainerUpgradeConfig) Validate(formats strfmt.Registry) error { + var res []error + + if err := m.validateCmd(formats); err != nil { + // prop + res = append(res, err) } - if err := swag.ReadJSON(raw, &data); err != nil { - return err + + if err := m.validateEntrypoint(formats); err != nil { + // prop + res = append(res, err) } - m.HostConfig = data.HostConfig + if err := m.validateImage(formats); err != nil { + // prop + res = append(res, err) + } + if len(res) > 0 { + return errors.CompositeValidationError(res...) + } return nil } -// MarshalJSON marshals this object to a JSON structure -func (m ContainerUpgradeConfig) MarshalJSON() ([]byte, error) { - var _parts [][]byte +func (m *ContainerUpgradeConfig) validateCmd(formats strfmt.Registry) error { - aO0, err := swag.WriteJSON(m.ContainerConfig) - if err != nil { - return nil, err + if swag.IsZero(m.Cmd) { // not required + return nil } - _parts = append(_parts, aO0) - var data struct { - HostConfig *HostConfig `json:"HostConfig,omitempty"` - } + return nil +} - data.HostConfig = m.HostConfig +func (m *ContainerUpgradeConfig) validateEntrypoint(formats strfmt.Registry) error { - jsonData, err := swag.WriteJSON(data) - if err != nil { - return nil, err + if swag.IsZero(m.Entrypoint) { // not required + return nil } - _parts = append(_parts, jsonData) - return swag.ConcatJSON(_parts...), nil + return nil } -// Validate validates this container upgrade config -func (m *ContainerUpgradeConfig) Validate(formats strfmt.Registry) error { - var res []error +func (m *ContainerUpgradeConfig) validateImage(formats strfmt.Registry) error { - if err := m.ContainerConfig.Validate(formats); err != nil { - res = append(res, err) + if err := validate.RequiredString("Image", "body", string(m.Image)); err != nil { + return err } - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } return nil } diff --git a/cli/upgrade.go b/cli/upgrade.go index 801a0c931..8ae443097 100644 --- a/cli/upgrade.go +++ b/cli/upgrade.go @@ -3,24 +3,31 @@ package main import ( "context" "fmt" + "strings" + + "github.com/alibaba/pouch/apis/types" "github.com/spf13/cobra" ) // upgradeDescription is used to describe upgrade command in detail and auto generate command doc. -var upgradeDescription = "Upgrade a container with new image and args" +var upgradeDescription = "upgrade is a feature to replace a container's image." + + "You can specify the new Entrypoint and Cmd for the new container. When you want to update" + + "a container's image, but inherit the network and volumes of the old container, then you should" + + "think about the upgrade feature." // UpgradeCommand use to implement 'upgrade' command, it is used to upgrade a container. type UpgradeCommand struct { baseCommand - *container + entrypoint string + image string } // Init initialize upgrade command. func (ug *UpgradeCommand) Init(c *Cli) { ug.cli = c ug.cmd = &cobra.Command{ - Use: "upgrade [OPTIONS] IMAGE [COMMAND] [ARG...]", + Use: "upgrade [OPTIONS] CONTAINER [COMMAND] [ARG...]", Short: "Upgrade a container with new image and args", Long: upgradeDescription, Args: cobra.MinimumNArgs(1), @@ -36,47 +43,52 @@ func (ug *UpgradeCommand) Init(c *Cli) { func (ug *UpgradeCommand) addFlags() { flagSet := ug.cmd.Flags() flagSet.SetInterspersed(false) - - c := addCommonFlags(flagSet) - ug.container = c + flagSet.StringVar(&ug.entrypoint, "entrypoint", "", "Overwrite the default ENTRYPOINT of the image") + flagSet.StringVar(&ug.image, "image", "", "Specify image of the new container") } // runUpgrade is the entry of UpgradeCommand command. func (ug *UpgradeCommand) runUpgrade(args []string) error { - config, err := ug.config() - if err != nil { - return fmt.Errorf("failed to upgrade container: %v", err) - } + var cmd []string - config.Image = args[0] + name := args[0] + if name == "" { + return fmt.Errorf("failed to upgrade container: must specify container name") + } if len(args) > 1 { - config.Cmd = args[1:] + cmd = args[1:] } - containerName := ug.name - if containerName == "" { - return fmt.Errorf("failed to upgrade container: must specify container name") + image := ug.image + if image == "" { + return fmt.Errorf("failed to upgrade container: must specify new image") + } + + upgradeConfig := &types.ContainerUpgradeConfig{ + Image: image, + Cmd: cmd, + Entrypoint: strings.Fields(ug.entrypoint), } ctx := context.Background() apiClient := ug.cli.Client() - if err := pullMissingImage(ctx, apiClient, config.Image, false); err != nil { + if err := pullMissingImage(ctx, apiClient, image, false); err != nil { return err } - err = apiClient.ContainerUpgrade(ctx, containerName, config.ContainerConfig, config.HostConfig) - if err == nil { - fmt.Println(containerName) + if err := apiClient.ContainerUpgrade(ctx, name, upgradeConfig); err != nil { + return err } - return err + fmt.Println(name) + return nil } //upgradeExample shows examples in exec command, and is used in auto-generated cli docs. func upgradeExample() string { - return ` $ pouch run -d -m 20m --name test1 registry.hub.docker.com/library/busybox:latest + return ` $ pouch run -d -m 20m --name test registry.hub.docker.com/library/busybox:latest 4c58d27f58d38776dda31c01c897bbf554c802a9b80ae4dc20be1337f8a969f2 -$ pouch upgrade --name test1 registry.hub.docker.com/library/hello-world:latest +$ pouch upgrade --image registry.hub.docker.com/library/hello-world:latest test test1` } diff --git a/client/container_upgrade.go b/client/container_upgrade.go index 24d97aeef..b15959f04 100644 --- a/client/container_upgrade.go +++ b/client/container_upgrade.go @@ -8,12 +8,8 @@ import ( ) // ContainerUpgrade upgrade a container with new image and args. -func (client *APIClient) ContainerUpgrade(ctx context.Context, name string, config types.ContainerConfig, hostConfig *types.HostConfig) error { - upgradeConfig := types.ContainerUpgradeConfig{ - ContainerConfig: config, - HostConfig: hostConfig, - } - resp, err := client.post(ctx, "/containers/"+name+"/upgrade", url.Values{}, upgradeConfig, nil) +func (client *APIClient) ContainerUpgrade(ctx context.Context, name string, config *types.ContainerUpgradeConfig) error { + resp, err := client.post(ctx, "/containers/"+name+"/upgrade", url.Values{}, config, nil) ensureCloseReader(resp) return err diff --git a/client/interface.go b/client/interface.go index e6d387f2e..b0a8a50ea 100644 --- a/client/interface.go +++ b/client/interface.go @@ -36,7 +36,7 @@ type ContainerAPIClient interface { ContainerPause(ctx context.Context, name string) error ContainerUnpause(ctx context.Context, name string) error ContainerUpdate(ctx context.Context, name string, config *types.UpdateConfig) error - ContainerUpgrade(ctx context.Context, name string, config types.ContainerConfig, hostConfig *types.HostConfig) error + ContainerUpgrade(ctx context.Context, name string, config *types.ContainerUpgradeConfig) error ContainerTop(ctx context.Context, name string, arguments []string) (types.ContainerProcessList, error) ContainerLogs(ctx context.Context, name string, options types.ContainerLogsOptions) (io.ReadCloser, error) ContainerResize(ctx context.Context, name, height, width string) error diff --git a/cri/v1alpha1/cri_utils.go b/cri/v1alpha1/cri_utils.go index f2a2ccc94..1160a7af8 100644 --- a/cri/v1alpha1/cri_utils.go +++ b/cri/v1alpha1/cri_utils.go @@ -897,7 +897,7 @@ func (c *CriManager) getContainerMetrics(ctx context.Context, meta *mgr.Containe } // snapshot key may not equals container ID later - sn, err := c.SnapshotStore.Get(meta.ID) + sn, err := c.SnapshotStore.Get(meta.SnapshotID) if err == nil { usedBytes = sn.Size inodesUsed = sn.Inodes diff --git a/cri/v1alpha2/cri_utils.go b/cri/v1alpha2/cri_utils.go index 1b3943594..89f32e397 100644 --- a/cri/v1alpha2/cri_utils.go +++ b/cri/v1alpha2/cri_utils.go @@ -953,7 +953,7 @@ func (c *CriManager) getContainerMetrics(ctx context.Context, meta *mgr.Containe return nil, fmt.Errorf("failed to get metadata of container %q: %v", meta.ID, err) } - sn, err := c.SnapshotStore.Get(meta.ID) + sn, err := c.SnapshotStore.Get(meta.SnapshotID) if err == nil { usedBytes = sn.Size inodesUsed = sn.Inodes diff --git a/ctrd/container.go b/ctrd/container.go index fd5ac4347..dbed7c221 100644 --- a/ctrd/container.go +++ b/ctrd/container.go @@ -490,10 +490,10 @@ func (c *Client) createContainer(ctx context.Context, ref, id, checkpointDir str rootFSPath = container.BaseFS } else { // containers created by pouch must first create snapshot // check snapshot exist or not. - if _, err := c.GetSnapshot(ctx, id); err != nil { + if _, err := c.GetSnapshot(ctx, container.SnapshotID); err != nil { return errors.Wrapf(err, "failed to create container %s", id) } - options = append(options, containerd.WithSnapshot(id)) + options = append(options, containerd.WithSnapshot(container.SnapshotID)) } // specify Spec for new container diff --git a/ctrd/container_types.go b/ctrd/container_types.go index b9e09dbdc..52a16f266 100644 --- a/ctrd/container_types.go +++ b/ctrd/container_types.go @@ -11,12 +11,13 @@ import ( // then create container by specifying the snapshot; // The other is create container by specify container rootfs, we use `RootFSProvided` flag to mark it, type Container struct { - ID string - Image string - Runtime string - Labels map[string]string - IO *containerio.IO - Spec *specs.Spec + ID string + Image string + Runtime string + Labels map[string]string + IO *containerio.IO + Spec *specs.Spec + SnapshotID string // BaseFS is rootfs used by containerd container BaseFS string diff --git a/daemon/mgr/container.go b/daemon/mgr/container.go index 9eebbe93b..4d22f55bb 100644 --- a/daemon/mgr/container.go +++ b/daemon/mgr/container.go @@ -33,12 +33,10 @@ import ( "github.com/containerd/cgroups" containerdtypes "github.com/containerd/containerd/api/types" - "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/mount" "github.com/docker/docker/pkg/stdcopy" "github.com/docker/libnetwork" "github.com/go-openapi/strfmt" - "github.com/imdario/mergo" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -294,6 +292,7 @@ func (mgr *ContainerManager) Create(ctx context.Context, name string, config *ty if err != nil { return nil, err } + config.Image = primaryRef.String() // TODO: check request validate. if config.HostConfig == nil { @@ -329,9 +328,9 @@ func (mgr *ContainerManager) Create(ctx context.Context, name string, config *ty return nil, errors.Wrapf(errtypes.ErrInvalidParam, "unknown runtime %s", config.HostConfig.Runtime) } - config.Image = primaryRef.String() + snapID := id // create a snapshot with image. - if err := mgr.Client.CreateSnapshot(ctx, id, config.Image); err != nil { + if err := mgr.Client.CreateSnapshot(ctx, snapID, config.Image); err != nil { return nil, err } cleanups = append(cleanups, func() error { @@ -365,26 +364,19 @@ func (mgr *ContainerManager) Create(ctx context.Context, name string, config *ty Config: &config.ContainerConfig, Created: time.Now().UTC().Format(utils.TimeLayout), HostConfig: config.HostConfig, + SnapshotID: snapID, } // merge image's config into container if err := container.merge(func() (ocispec.ImageConfig, error) { - img, err := mgr.Client.GetImage(ctx, config.Image) - if err != nil { - return ocispec.ImageConfig{}, err - } - ociImage, err := containerdImageToOciImage(ctx, img) - if err != nil { - return ocispec.ImageConfig{}, err - } - return ociImage.Config, nil + return mgr.ImageMgr.GetOCIImageConfig(ctx, config.Image) }); err != nil { return nil, err } // set container basefs, basefs is not created in pouchd, it will created // after create options passed to containerd. - mgr.setBaseFS(ctx, container, id) + mgr.setBaseFS(ctx, container) // init container storage module, such as: set volumes, set diskquota, set /etc/mtab, copy image's data to volume. if err := mgr.initContainerStorage(ctx, container); err != nil { @@ -496,7 +488,12 @@ func (mgr *ContainerManager) Start(ctx context.Context, id string, options *type return errors.Wrapf(errtypes.ErrNotModified, "container already started") } - return mgr.start(ctx, c, options) + err = mgr.start(ctx, c, options) + if err == nil { + mgr.LogContainerEvent(ctx, c, "start") + } + + return err } func (mgr *ContainerManager) start(ctx context.Context, c *Container, options *types.ContainerStartOptions) error { @@ -541,8 +538,6 @@ func (mgr *ContainerManager) start(ctx context.Context, c *Container, options *t return errors.Wrapf(err, "failed to create container(%s) on containerd", c.ID) } - mgr.LogContainerEvent(ctx, c, "start") - return nil } @@ -682,6 +677,7 @@ func (mgr *ContainerManager) createContainerdContainer(ctx context.Context, c *C IO: io, RootFSProvided: c.RootFSProvided, BaseFS: c.BaseFS, + SnapshotID: c.SnapshotID, } c.Unlock() @@ -1087,7 +1083,7 @@ func (mgr *ContainerManager) Remove(ctx context.Context, name string, options *t if err := mount.Unmount(c.BaseFS, 0); err != nil { logrus.Errorf("failed to umount rootfs when remove the container %s: %v", c.ID, err) } - } else if err := mgr.Client.RemoveSnapshot(ctx, c.ID); err != nil { + } else if err := mgr.Client.RemoveSnapshot(ctx, c.SnapshotKey()); err != nil { // if the container is created by normal method, remove the // snapshot when delete it. logrus.Errorf("failed to remove snapshot of container %s: %v", c.ID, err) @@ -1228,132 +1224,6 @@ func (mgr *ContainerManager) updateContainerResources(c *Container, resources ty return nil } -// Upgrade upgrades a container with new image and args. -func (mgr *ContainerManager) Upgrade(ctx context.Context, name string, config *types.ContainerUpgradeConfig) error { - c, err := mgr.container(name) - if err != nil { - return err - } - - // check the image existed or not, and convert image id to image ref - _, _, primaryRef, err := mgr.ImageMgr.CheckReference(ctx, config.Image) - if err != nil { - return errors.Wrap(err, "failed to get image") - } - config.Image = primaryRef.String() - - // Nothing changed, no need upgrade. - if config.Image == c.Config.Image { - return fmt.Errorf("failed to upgrade container: image not changed") - } - - var ( - needRollback = false - // FIXME: do a deep copy of container? - backupContainer = c - ) - - defer func() { - if needRollback { - c = backupContainer - if err := mgr.Client.CreateSnapshot(ctx, c.ID, c.Image); err != nil { - logrus.Errorf("failed to create snapshot when rollback upgrade action: %v", err) - return - } - // FIXME: create new containerd container may failed - _ = mgr.createContainerdContainer(ctx, c, "", "") - } - }() - - // FIXME(ziren): mergo.Merge() use AppendSlice to merge slice. - // that is to say, t1 = ["a", "b"], t2 = ["a", "c"], the merge - // result will be ["a", "b", "a", "c"] - // This may occur errors, just take notes to record this. - if err := mergo.MergeWithOverwrite(c.Config, config.ContainerConfig); err != nil { - return errors.Wrapf(err, "failed to merge ContainerConfig") - } - if err := mergo.MergeWithOverwrite(c.HostConfig, config.HostConfig); err != nil { - return errors.Wrapf(err, "failed to merge HostConfig") - } - c.Image = config.Image - - // If container is not running, we just store this data. - if c.State.Status != types.StatusRunning { - // Works fine, store new container info to disk. - if err := c.Write(mgr.Store); err != nil { - logrus.Errorf("failed to update container %s in meta store: %v", c.ID, err) - return err - } - return nil - } - // If container is running, we need change - // configuration and recreate it. Else we just store new meta - // into disk, next time when starts container, the new configurations - // will take effect. - - // Inherit volume configurations from old container, - // New volume configurations may cover the old one. - // c.VolumesFrom = []string{c.ID} - - // FIXME(ziren): here will forcely stop container afer 3s. - // If DestroyContainer failed, we think the old container - // not changed, so just return error, no need recover it. - if _, err := mgr.Client.DestroyContainer(ctx, c.ID, 3); err != nil { - return errors.Wrapf(err, "failed to destroy container %s", c.ID) - } - - // remove snapshot of old container - if err := mgr.Client.RemoveSnapshot(ctx, c.ID); err != nil { - return errors.Wrapf(err, "failed to remove snapshot of container %s", c.ID) - } - - // wait util old snapshot to be deleted - wait := make(chan struct{}) - go func() { - for { - // FIXME(ziren) Ensure the removed snapshot be removed - // by garbage collection. - time.Sleep(100 * time.Millisecond) - - _, err := mgr.Client.GetSnapshot(ctx, c.ID) - if err != nil && errdefs.IsNotFound(err) { - close(wait) - return - } - } - }() - - select { - case <-wait: - // TODO delete snapshot succeeded - case <-time.After(30 * time.Second): - needRollback = true - return fmt.Errorf("failed to deleted old snapshot: wait old snapshot %s to be deleted timeout(30s)", c.ID) - } - - // create a snapshot with image for new container. - if err := mgr.Client.CreateSnapshot(ctx, c.ID, config.Image); err != nil { - needRollback = true - return errors.Wrap(err, "failed to create snapshot") - } - - if err := mgr.createContainerdContainer(ctx, c, "", ""); err != nil { - needRollback = true - return errors.Wrap(err, "failed to create new container") - } - - // Upgrade succeeded, refresh the cache - mgr.cache.Put(c.ID, c) - - // Works fine, store new container info to disk. - if err := c.Write(mgr.Store); err != nil { - logrus.Errorf("failed to update container %s in meta store: %v", c.ID, err) - return err - } - - return nil -} - // Top lists the processes running inside of the given container func (mgr *ContainerManager) Top(ctx context.Context, name string, psArgs string) (*types.ContainerProcessList, error) { if psArgs == "" { @@ -1952,16 +1822,17 @@ func (mgr *ContainerManager) buildContainerEndpoint(c *Container) *networktypes. } // setBaseFS keeps container basefs in meta. -func (mgr *ContainerManager) setBaseFS(ctx context.Context, c *Container, id string) { - info, err := mgr.Client.GetSnapshot(ctx, id) +func (mgr *ContainerManager) setBaseFS(ctx context.Context, c *Container) { + snapshotID := c.SnapshotKey() + _, err := mgr.Client.GetSnapshot(ctx, snapshotID) if err != nil { - logrus.Infof("failed to get container %s snapshot", id) + logrus.Errorf("failed to get container %s snapshot %s: %v", c.Key(), snapshotID, err) return } // io.containerd.runtime.v1.linux as a const used by runc c.Lock() - c.BaseFS = filepath.Join(mgr.Config.HomeDir, "containerd/state", "io.containerd.runtime.v1.linux", mgr.Config.DefaultNamespace, info.Name, "rootfs") + c.BaseFS = filepath.Join(mgr.Config.HomeDir, "containerd/state", "io.containerd.runtime.v1.linux", mgr.Config.DefaultNamespace, c.ID, "rootfs") c.Unlock() } diff --git a/daemon/mgr/container_types.go b/daemon/mgr/container_types.go index 2409086fe..b1d9c373c 100644 --- a/daemon/mgr/container_types.go +++ b/daemon/mgr/container_types.go @@ -261,6 +261,9 @@ type Container struct { // MountFS is used to mark the directory of mount overlayfs for pouch daemon to operate the image. MountFS string `json:"-"` + + // SnapshotID specify id of the snapshot that container using. + SnapshotID string } // Key returns container's id. @@ -270,6 +273,25 @@ func (c *Container) Key() string { return c.ID } +// SnapshotKey returns id of container's snapshot +func (c *Container) SnapshotKey() string { + c.Lock() + defer c.Unlock() + // for old container, SnapshotKey equals to Container ID + if c.SnapshotID == "" { + return c.ID + } + + return c.SnapshotID +} + +// SetSnapshotID sets the snapshot id of container +func (c *Container) SetSnapshotID(snapID string) { + c.Lock() + defer c.Unlock() + c.SnapshotID = snapID +} + // Write writes container's meta data into meta store. func (c *Container) Write(store *meta.Store) error { return store.Put(c) diff --git a/daemon/mgr/container_upgrade.go b/daemon/mgr/container_upgrade.go new file mode 100644 index 000000000..0cb848beb --- /dev/null +++ b/daemon/mgr/container_upgrade.go @@ -0,0 +1,214 @@ +package mgr + +import ( + "context" + "fmt" + + "github.com/alibaba/pouch/apis/types" + "github.com/alibaba/pouch/pkg/utils" + + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" +) + +// Upgrade a container with new image and args. when upgrade a container, +// we only support specify cmd and entrypoint. if you want to change other +// parameters of the container, you should think about the update API first. +func (mgr *ContainerManager) Upgrade(ctx context.Context, name string, config *types.ContainerUpgradeConfig) error { + var err error + c, err := mgr.container(name) + if err != nil { + return err + } + + var ( + needRollback = false + oldConfig = *c.Config + oldHostconfig = *c.HostConfig + oldImage = c.Image + oldSnapID = c.SnapshotKey() + IsRunning = false + ) + + // use err to determine if we should recover old container configure. + defer func() { + if err == nil { + return + } + + c.Lock() + // recover old container config + c.Config = &oldConfig + c.HostConfig = &oldHostconfig + c.Image = oldImage + c.SnapshotID = oldSnapID + c.Unlock() + + // even if the err is not nil, we may still no need to rollback the container + if !needRollback { + return + } + + if err := mgr.start(ctx, c, &types.ContainerStartOptions{}); err != nil { + logrus.Errorf("failed to rollback upgrade action: %s", err.Error()) + if err := mgr.markStoppedAndRelease(c, nil); err != nil { + logrus.Errorf("failed to mark container %s stop status: %s", c.ID, err.Error()) + } + } + }() + + // merge image config to container config + err = mgr.mergeImageConfigForUpgrade(ctx, c, config) + if err != nil { + return errors.Wrap(err, "failed to upgrade container") + } + + // if the container is running, we need first stop it. + if c.IsRunning() { + IsRunning = true + err = mgr.stop(ctx, c, 10) + if err != nil { + return errors.Wrapf(err, "failed to stop container %s when upgrade", c.Key()) + } + needRollback = true + } + + // prepare new snapshot for the new container + newSnapID, err := mgr.prepareSnapshotForUpgrade(ctx, c.Key(), c.SnapshotKey(), config.Image) + if err != nil { + return err + } + c.SetSnapshotID(newSnapID) + + // initialize container storage config before container started + err = mgr.initContainerStorage(ctx, c) + if err != nil { + return errors.Wrapf(err, "failed to init container storage, id: (%s)", c.Key()) + } + + // If container is running, we also should start the container + // after recreate it. + if IsRunning { + err = mgr.start(ctx, c, &types.ContainerStartOptions{}) + if err != nil { + if err := mgr.Client.RemoveSnapshot(ctx, newSnapID); err != nil { + logrus.Errorf("failed to remove snapshot %s: %v", newSnapID, err) + } + } + + return errors.Wrap(err, "failed to create new container") + } + + // Upgrade success, remove snapshot of old container + if err := mgr.Client.RemoveSnapshot(ctx, oldSnapID); err != nil { + // TODO(ziren): remove old snapshot failed, may cause dirty data + logrus.Errorf("failed to remove snapshot %s: %v", oldSnapID, err) + } + + // Upgrade succeeded, refresh the cache + mgr.cache.Put(c.ID, c) + + // Works fine, store new container info to disk. + if err := c.Write(mgr.Store); err != nil { + logrus.Errorf("failed to update container %s in meta store: %v", c.ID, err) + return err + } + + return nil +} + +func (mgr *ContainerManager) prepareContainerEntrypointForUpgrade(ctx context.Context, c *Container, config *types.ContainerUpgradeConfig) error { + // Firstly, try to use the entrypoint specified by ContainerUpgradeConfig + if len(config.Entrypoint) > 0 || len(config.Cmd) > 0 { + return nil + } + + // Secondly, try to use the entrypoint of the old container. + // because of the entrypoints of old container's CreateConfig and old image being merged, + // so we cannot decide which config that the old container's entrypoint belongs to, so just check if the old + // container's entrypoint is different with the old image. + c.Lock() + defer c.Unlock() + + oldImgConfig, err := mgr.ImageMgr.GetOCIImageConfig(ctx, c.Config.Image) + if err != nil { + return err + } + + // if the entrypoints of old container and the old image is empty, we should use to CMD of old container, + // else if entrypoints are different, we use the CMD of old container. + if (c.Config.Entrypoint == nil && oldImgConfig.Entrypoint == nil) || !utils.StringSliceEqual(c.Config.Entrypoint, oldImgConfig.Entrypoint) { + config.Entrypoint = c.Config.Entrypoint + if len(config.Cmd) == 0 { + config.Cmd = c.Config.Cmd + } + + return nil + } + + // Thirdly, just use the entrypoint of the new image + newImgConfig, err := mgr.ImageMgr.GetOCIImageConfig(ctx, config.Image) + if err != nil { + return err + } + config.Entrypoint = newImgConfig.Entrypoint + if len(config.Cmd) == 0 { + config.Cmd = newImgConfig.Cmd + } + return nil +} + +func (mgr *ContainerManager) prepareSnapshotForUpgrade(ctx context.Context, cID, oldSnapID, image string) (string, error) { + newSnapID := "" + // get a ID for the new snapshot + for { + newSnapID = utils.RandString(8, cID, "") + if newSnapID != oldSnapID { + break + } + } + + // create a snapshot with image for new container. + if err := mgr.Client.CreateSnapshot(ctx, newSnapID, image); err != nil { + return "", errors.Wrap(err, "failed to create snapshot") + } + + return newSnapID, nil +} + +func (mgr *ContainerManager) mergeImageConfigForUpgrade(ctx context.Context, c *Container, config *types.ContainerUpgradeConfig) error { + // check the image existed or not, and convert image id to image ref + imgID, _, primaryRef, err := mgr.ImageMgr.CheckReference(ctx, config.Image) + if err != nil { + return errors.Wrap(err, "failed to get image") + } + + config.Image = primaryRef.String() + // Nothing changed, no need upgrade. + if config.Image == c.Config.Image { + return fmt.Errorf("failed to upgrade container: image not changed") + } + + // prepare entrypoint for the new container of upgrade + if err := mgr.prepareContainerEntrypointForUpgrade(ctx, c, config); err != nil { + return errors.Wrap(err, "failed to check entrypoint for container upgrade") + } + + // merge image's config into the new container of upgrade + if err := c.merge(func() (ocispec.ImageConfig, error) { + return mgr.ImageMgr.GetOCIImageConfig(ctx, config.Image) + }); err != nil { + return errors.Wrap(err, "failed to merge image config when upgrade container") + } + + // set image and entrypoint for new container. + c.Lock() + c.Image = imgID.String() + c.Config.Image = config.Image + c.Config.Entrypoint = config.Entrypoint + c.Config.Cmd = config.Cmd + c.Unlock() + + return nil +} diff --git a/daemon/mgr/image.go b/daemon/mgr/image.go index 92d0e9b91..5a4e548b6 100644 --- a/daemon/mgr/image.go +++ b/daemon/mgr/image.go @@ -66,6 +66,9 @@ type ImageMgr interface { // StoreImageReference update image reference. StoreImageReference(ctx context.Context, img containerd.Image) error + + // GetOCIImageConfig returns the image config of OCI + GetOCIImageConfig(ctx context.Context, image string) (ocispec.ImageConfig, error) } // ImageManager is an implementation of interface ImageMgr. @@ -420,6 +423,19 @@ func (mgr *ImageManager) ListReferences(ctx context.Context, imageID digest.Dige return mgr.localStore.GetPrimaryReferences(imageID), nil } +// GetOCIImageConfig returns the image config of OCI +func (mgr *ImageManager) GetOCIImageConfig(ctx context.Context, image string) (ocispec.ImageConfig, error) { + img, err := mgr.client.GetImage(ctx, image) + if err != nil { + return ocispec.ImageConfig{}, err + } + ociImage, err := containerdImageToOciImage(ctx, img) + if err != nil { + return ocispec.ImageConfig{}, err + } + return ociImage.Config, nil +} + // updateLocalStore updates the local store. func (mgr *ImageManager) updateLocalStore() error { ctx, cancel := context.WithTimeout(context.Background(), deadlineLoadImagesAtBootup) diff --git a/pkg/utils/random_string.go b/pkg/utils/random_string.go new file mode 100644 index 000000000..1d8ffdd88 --- /dev/null +++ b/pkg/utils/random_string.go @@ -0,0 +1,29 @@ +package utils + +import ( + "math/rand" + "time" +) + +const ( + letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ123456789" + stringSeparator = "_" +) + +// RandString is a thread/goroutine safe solution to generate a random string +// of a fixed length +func RandString(n int, prefix, suffix string) string { + r := rand.New(rand.NewSource(time.Now().UnixNano())) + b := make([]byte, n) + for i := range b { + b[i] = letterBytes[r.Intn(len(letterBytes))] + } + randStr := string(b) + if prefix != "" { + randStr = prefix + stringSeparator + randStr + } + if suffix != "" { + randStr += stringSeparator + suffix + } + return randStr +} diff --git a/pkg/utils/random_string_test.go b/pkg/utils/random_string_test.go new file mode 100644 index 000000000..47ca50a10 --- /dev/null +++ b/pkg/utils/random_string_test.go @@ -0,0 +1,24 @@ +package utils + +import ( + "testing" + "time" + + "github.com/sirupsen/logrus" +) + +func TestRandString(t *testing.T) { + start := time.Now() + results := []string{} + for i := 0; i < 1000; i++ { + str := RandString(8, "", "") + if StringInSlice(results, str) { + t.Errorf("RandString got a same random string in the test: %s", str) + } + + results = append(results, str) + } + end := time.Now() + elapsed := end.Sub(start) + logrus.Infof("TestRandString generate 1000 random strings costs: %v s", elapsed.Seconds()) +} diff --git a/test/cli_upgrade_test.go b/test/cli_upgrade_test.go index f51bd0f18..cee9370c0 100644 --- a/test/cli_upgrade_test.go +++ b/test/cli_upgrade_test.go @@ -2,9 +2,6 @@ package main import ( "encoding/json" - "os" - "os/exec" - "reflect" "strings" "github.com/alibaba/pouch/apis/types" @@ -29,7 +26,7 @@ func (suite *PouchUpgradeSuite) SetUpSuite(c *check.C) { environment.PruneAllContainers(apiClient) PullImage(c, busyboxImage) - command.PouchRun("pull", busyboxImage125).Assert(c, icmd.Success) + PullImage(c, busyboxImage125) } // TearDownTest does cleanup work in the end of each test. @@ -41,16 +38,28 @@ func (suite *PouchUpgradeSuite) TeadDownTest(c *check.C) { func (suite *PouchUpgradeSuite) TestPouchUpgrade(c *check.C) { name := "TestPouchUpgrade" - res := command.PouchRun("run", "-d", "--name", name, busyboxImage, "top") + command.PouchRun("run", "-d", "--name", name, busyboxImage, "top").Assert(c, icmd.Success) defer DelContainerForceMultyTime(c, name) - res.Assert(c, icmd.Success) - res = command.PouchRun("upgrade", "--name", name, busyboxImage125) + res := command.PouchRun("upgrade", "--image", busyboxImage125, name) res.Assert(c, icmd.Success) - if out := res.Combined(); !strings.Contains(out, name) { c.Fatalf("unexpected output: %s, expected: %s", out, name) } + + // check if the new container is running after upgade a running container + output := command.PouchRun("inspect", name).Stdout() + result := []types.ContainerJSON{} + if err := json.Unmarshal([]byte(output), &result); err != nil { + c.Errorf("failed to decode inspect output: %v", err) + } + c.Assert(result[0].State.Running, check.Equals, true) + + // double check if container is running by executing a exec command + out := command.PouchRun("exec", name, "echo", "test").Stdout() + if !strings.Contains(out, "test") { + c.Errorf("failed to exec in container, expected test got %s", out) + } } // TestPouchUpgradeNoChange is to verify pouch upgrade command with same image. @@ -61,7 +70,7 @@ func (suite *PouchUpgradeSuite) TestPouchUpgradeNoChange(c *check.C) { defer DelContainerForceMultyTime(c, name) res.Assert(c, icmd.Success) - res = command.PouchRun("upgrade", "--name", name, busyboxImage) + res = command.PouchRun("upgrade", "--image", busyboxImage, name) c.Assert(res.Stderr(), check.NotNil) expectedStr := "failed to upgrade container: image not changed" @@ -74,96 +83,71 @@ func (suite *PouchUpgradeSuite) TestPouchUpgradeNoChange(c *check.C) { func (suite *PouchUpgradeSuite) TestPouchUpgradeStoppedContainer(c *check.C) { name := "TestPouchUpgradeStoppedContainer" - res := command.PouchRun("create", "--name", name, busyboxImage) + command.PouchRun("run", "-d", "--name", name, busyboxImage, "top").Assert(c, icmd.Success) defer DelContainerForceMultyTime(c, name) - res.Assert(c, icmd.Success) - res = command.PouchRun("upgrade", "--name", name, busyboxImage125) + command.PouchRun("stop", name).Assert(c, icmd.Success) + + res := command.PouchRun("upgrade", "--image", busyboxImage125, name) res.Assert(c, icmd.Success) if out := res.Combined(); !strings.Contains(out, name) { c.Fatalf("unexpected output: %s, expected %s", out, name) } - command.PouchRun("start", name).Assert(c, icmd.Success) -} - -// TestPouchUpgradeContainerMemCpu is to verify pouch upgrade container's memory -func (suite *PouchUpgradeSuite) TestPouchUpgradeContainerMemCpu(c *check.C) { - name := "TestPouchUpgradeContainerMemCpu" - - res := command.PouchRun("run", "-d", "-m", "300m", - "--cpu-shares", "20", "--name", name, busyboxImage, "top") - defer DelContainerForceMultyTime(c, name) - res.Assert(c, icmd.Success) - - command.PouchRun("upgrade", "-m", "500m", - "--cpu-shares", "40", "--name", name, busyboxImage125).Assert(c, icmd.Success) - + // check if the new container is running after upgade a running container output := command.PouchRun("inspect", name).Stdout() result := []types.ContainerJSON{} if err := json.Unmarshal([]byte(output), &result); err != nil { c.Errorf("failed to decode inspect output: %v", err) } - containerID := result[0].ID - - // Check if metajson has changed - c.Assert(result[0].HostConfig.Memory, check.Equals, int64(524288000)) - c.Assert(result[0].HostConfig.CPUShares, check.Equals, int64(40)) + c.Assert(result[0].State.Status, check.Equals, types.StatusStopped) - // Check if cgroup file has changed - memFile := "/sys/fs/cgroup/memory/default/" + containerID + "/memory.limit_in_bytes" - if _, err := os.Stat(memFile); err != nil { - c.Fatalf("container %s cgroup mountpoint not exists", containerID) - } - - out, err := exec.Command("cat", memFile).Output() - if err != nil { - c.Fatalf("execute cat command failed: %v", err) - } - - if !strings.Contains(string(out), "524288000") { - c.Fatalf("unexpected output %s expected %s\n", string(out), "524288000") - } - - cpuFile := "/sys/fs/cgroup/cpu/default/" + containerID + "/cpu.shares" - if _, err := os.Stat(cpuFile); err != nil { - c.Fatalf("container %s cgroup mountpoint not exists", containerID) - } + command.PouchRun("start", name).Assert(c, icmd.Success) +} - out, err = exec.Command("cat", cpuFile).Output() - if err != nil { - c.Fatalf("execute cat command failed: %v", err) - } +// TestPouchUpgradeWithDifferentImage is to verify pouch upgrade command. +func (suite *PouchUpgradeSuite) TestPouchUpgradeWithDifferentImage(c *check.C) { + name := "TestPouchUpgradeWithDifferentImage" + command.PouchRun("run", "-d", "--name", name, busyboxImage).Assert(c, icmd.Success) + defer DelContainerForceMultyTime(c, name) - if !strings.Contains(string(out), "40") { - c.Fatalf("unexpected output %s expected %s\n", string(out), "40") + res := command.PouchRun("upgrade", "--image", helloworldImage, name, "/hello") + c.Assert(res.Error, check.IsNil) + if out := res.Combined(); !strings.Contains(out, name) { + c.Fatalf("unexpected output: %s, expected: %s", out, name) } } -// TestPouchUpgradeContainerLabels is to verify pouch upgrade container's labels -func (suite *PouchUpgradeSuite) TestPouchUpgradeContainerLabels(c *check.C) { - name := "TestPouchUpgradeContainerLabels" +// TestPouchUpgradeCheckVolume is to verify if inherit old container's volume +// after upgrade a container +func (suite *PouchUpgradeSuite) TestPouchUpgradeCheckVolume(c *check.C) { + name := "TestPouchUpgradeCheckVolume" - res := command.PouchRun("run", "-d", "--label", "test=foo", "--name", name, busyboxImage, "top") + // create container with a /data volume + command.PouchRun("run", "-d", "-v", "/data", "--name", name, busyboxImage, "top").Assert(c, icmd.Success) defer DelContainerForceMultyTime(c, name) - res.Assert(c, icmd.Success) - command.PouchRun("upgrade", "--label", "test1=bar", - "--name", name, busyboxImage125).Assert(c, icmd.Success) + // create a file in volume and write some data to the file + command.PouchRun("exec", name, "sh", "-c", "echo '5678' >> /data/test").Assert(c, icmd.Success) + + res := command.PouchRun("upgrade", "--image", busyboxImage125, name) + res.Assert(c, icmd.Success) + if out := res.Combined(); !strings.Contains(out, name) { + c.Fatalf("unexpected output: %s, expected: %s", out, name) + } + // check if the new container is running after upgade a running container output := command.PouchRun("inspect", name).Stdout() result := []types.ContainerJSON{} if err := json.Unmarshal([]byte(output), &result); err != nil { c.Errorf("failed to decode inspect output: %v", err) } + c.Assert(result[0].State.Running, check.Equals, true) - labels := map[string]string{ - "test": "foo", - "test1": "bar", - } - - if !reflect.DeepEqual(result[0].Config.Labels, labels) { - c.Errorf("unexpected output: %s, expected: %s", result[0].Config.Labels, labels) + // double check if container is running by executing a exec command + out := command.PouchRun("exec", name, "cat", "/data/test").Stdout() + if !strings.Contains(out, "5678") { + c.Errorf("failed to exec in container, expected 5678 got %s", out) } } diff --git a/vendor/github.com/imdario/mergo/CODE_OF_CONDUCT.md b/vendor/github.com/imdario/mergo/CODE_OF_CONDUCT.md deleted file mode 100644 index 469b44907..000000000 --- a/vendor/github.com/imdario/mergo/CODE_OF_CONDUCT.md +++ /dev/null @@ -1,46 +0,0 @@ -# Contributor Covenant Code of Conduct - -## Our Pledge - -In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation. - -## Our Standards - -Examples of behavior that contributes to creating a positive environment include: - -* Using welcoming and inclusive language -* Being respectful of differing viewpoints and experiences -* Gracefully accepting constructive criticism -* Focusing on what is best for the community -* Showing empathy towards other community members - -Examples of unacceptable behavior by participants include: - -* The use of sexualized language or imagery and unwelcome sexual attention or advances -* Trolling, insulting/derogatory comments, and personal or political attacks -* Public or private harassment -* Publishing others' private information, such as a physical or electronic address, without explicit permission -* Other conduct which could reasonably be considered inappropriate in a professional setting - -## Our Responsibilities - -Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior. - -Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. - -## Scope - -This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers. - -## Enforcement - -Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at i@dario.im. The project team will review and investigate all complaints, and will respond in a way that it deems appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. - -Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership. - -## Attribution - -This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at [http://contributor-covenant.org/version/1/4][version] - -[homepage]: http://contributor-covenant.org -[version]: http://contributor-covenant.org/version/1/4/ diff --git a/vendor/github.com/imdario/mergo/LICENSE b/vendor/github.com/imdario/mergo/LICENSE deleted file mode 100644 index 686680298..000000000 --- a/vendor/github.com/imdario/mergo/LICENSE +++ /dev/null @@ -1,28 +0,0 @@ -Copyright (c) 2013 Dario Castañé. All rights reserved. -Copyright (c) 2012 The Go Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/imdario/mergo/README.md b/vendor/github.com/imdario/mergo/README.md deleted file mode 100644 index de9a4467e..000000000 --- a/vendor/github.com/imdario/mergo/README.md +++ /dev/null @@ -1,219 +0,0 @@ -# Mergo - -A helper to merge structs and maps in Golang. Useful for configuration default values, avoiding messy if-statements. - -Also a lovely [comune](http://en.wikipedia.org/wiki/Mergo) (municipality) in the Province of Ancona in the Italian region of Marche. - -## Status - -It is ready for production use. [It is used in several projects by Docker, Google, The Linux Foundation, VMWare, Shopify, etc](https://github.com/imdario/mergo#mergo-in-the-wild). - -[![GoDoc][3]][4] -[![GoCard][5]][6] -[![Build Status][1]][2] -[![Coverage Status][7]][8] -[![Sourcegraph][9]][10] - -[1]: https://travis-ci.org/imdario/mergo.png -[2]: https://travis-ci.org/imdario/mergo -[3]: https://godoc.org/github.com/imdario/mergo?status.svg -[4]: https://godoc.org/github.com/imdario/mergo -[5]: https://goreportcard.com/badge/imdario/mergo -[6]: https://goreportcard.com/report/github.com/imdario/mergo -[7]: https://coveralls.io/repos/github/imdario/mergo/badge.svg?branch=master -[8]: https://coveralls.io/github/imdario/mergo?branch=master -[9]: https://sourcegraph.com/github.com/imdario/mergo/-/badge.svg -[10]: https://sourcegraph.com/github.com/imdario/mergo?badge - -### Latest release - -[Release 0.3.2](https://github.com/imdario/mergo/releases/tag/0.3.2) is an important release because it changes `Merge()`and `Map()` signatures to support [transformers](#transformers). An optional/variadic argument has been added, so it won't break existing code. - -### Important note - -If you were using Mergo **before** April 6th 2015, please check your project works as intended after updating your local copy with ```go get -u github.com/imdario/mergo```. I apologize for any issue caused by its previous behavior and any future bug that Mergo could cause (I hope it won't!) in existing projects after the change (release 0.2.0). - -### Donations - -If Mergo is useful to you, consider buying me a coffe, a beer or making a monthly donation so I can keep building great free software. :heart_eyes: - -Buy Me a Coffee at ko-fi.com -[![Beerpay](https://beerpay.io/imdario/mergo/badge.svg)](https://beerpay.io/imdario/mergo) -[![Beerpay](https://beerpay.io/imdario/mergo/make-wish.svg)](https://beerpay.io/imdario/mergo) -Donate using Liberapay - -### Mergo in the wild - -- [moby/moby](https://github.com/moby/moby) -- [kubernetes/kubernetes](https://github.com/kubernetes/kubernetes) -- [vmware/dispatch](https://github.com/vmware/dispatch) -- [Shopify/themekit](https://github.com/Shopify/themekit) -- [imdario/zas](https://github.com/imdario/zas) -- [matcornic/hermes](https://github.com/matcornic/hermes) -- [OpenBazaar/openbazaar-go](https://github.com/OpenBazaar/openbazaar-go) -- [kataras/iris](https://github.com/kataras/iris) -- [michaelsauter/crane](https://github.com/michaelsauter/crane) -- [go-task/task](https://github.com/go-task/task) -- [sensu/uchiwa](https://github.com/sensu/uchiwa) -- [ory/hydra](https://github.com/ory/hydra) -- [sisatech/vcli](https://github.com/sisatech/vcli) -- [dairycart/dairycart](https://github.com/dairycart/dairycart) -- [projectcalico/felix](https://github.com/projectcalico/felix) -- [resin-os/balena](https://github.com/resin-os/balena) -- [go-kivik/kivik](https://github.com/go-kivik/kivik) -- [Telefonica/govice](https://github.com/Telefonica/govice) -- [supergiant/supergiant](supergiant/supergiant) -- [SergeyTsalkov/brooce](https://github.com/SergeyTsalkov/brooce) -- [soniah/dnsmadeeasy](https://github.com/soniah/dnsmadeeasy) -- [ohsu-comp-bio/funnel](https://github.com/ohsu-comp-bio/funnel) -- [EagerIO/Stout](https://github.com/EagerIO/Stout) -- [lynndylanhurley/defsynth-api](https://github.com/lynndylanhurley/defsynth-api) -- [russross/canvasassignments](https://github.com/russross/canvasassignments) -- [rdegges/cryptly-api](https://github.com/rdegges/cryptly-api) -- [casualjim/exeggutor](https://github.com/casualjim/exeggutor) -- [divshot/gitling](https://github.com/divshot/gitling) -- [RWJMurphy/gorl](https://github.com/RWJMurphy/gorl) -- [andrerocker/deploy42](https://github.com/andrerocker/deploy42) -- [elwinar/rambler](https://github.com/elwinar/rambler) -- [tmaiaroto/gopartman](https://github.com/tmaiaroto/gopartman) -- [jfbus/impressionist](https://github.com/jfbus/impressionist) -- [Jmeyering/zealot](https://github.com/Jmeyering/zealot) -- [godep-migrator/rigger-host](https://github.com/godep-migrator/rigger-host) -- [Dronevery/MultiwaySwitch-Go](https://github.com/Dronevery/MultiwaySwitch-Go) -- [thoas/picfit](https://github.com/thoas/picfit) -- [mantasmatelis/whooplist-server](https://github.com/mantasmatelis/whooplist-server) -- [jnuthong/item_search](https://github.com/jnuthong/item_search) -- [bukalapak/snowboard](https://github.com/bukalapak/snowboard) - -## Installation - - go get github.com/imdario/mergo - - // use in your .go code - import ( - "github.com/imdario/mergo" - ) - -## Usage - -You can only merge same-type structs with exported fields initialized as zero value of their type and same-types maps. Mergo won't merge unexported (private) fields but will do recursively any exported one. Also maps will be merged recursively except for structs inside maps (because they are not addressable using Go reflection). - -```go -if err := mergo.Merge(&dst, src); err != nil { - // ... -} -``` - -Also, you can merge overwriting values using the transformer `WithOverride`. - -```go -if err := mergo.Merge(&dst, src, WithOverride); err != nil { - // ... -} -``` - -Additionally, you can map a `map[string]interface{}` to a struct (and otherwise, from struct to map), following the same restrictions as in `Merge()`. Keys are capitalized to find each corresponding exported field. - -```go -if err := mergo.Map(&dst, srcMap); err != nil { - // ... -} -``` - -Warning: if you map a struct to map, it won't do it recursively. Don't expect Mergo to map struct members of your struct as `map[string]interface{}`. They will be just assigned as values. - -More information and examples in [godoc documentation](http://godoc.org/github.com/imdario/mergo). - -### Nice example - -```go -package main - -import ( - "fmt" - "github.com/imdario/mergo" -) - -type Foo struct { - A string - B int64 -} - -func main() { - src := Foo{ - A: "one", - B: 2, - } - dest := Foo{ - A: "two", - } - mergo.Merge(&dest, src) - fmt.Println(dest) - // Will print - // {two 2} -} -``` - -Note: if test are failing due missing package, please execute: - - go get gopkg.in/yaml.v2 - -### Transformers - -Transformers allow to merge specific types differently than in the default behavior. In other words, now you can customize how some types are merged. For example, `time.Time` is a struct; it doesn't have zero value but IsZero can return true because it has fields with zero value. How can we merge a non-zero `time.Time`? - -```go -package main - -import ( - "fmt" - "reflect" - "time" -) - -type timeTransfomer struct { -} - -func (t timeTransfomer) Transformer(typ reflect.Type) func(dst, src reflect.Value) error { - if typ == reflect.TypeOf(time.Time{}) { - return func(dst, src reflect.Value) error { - if dst.CanSet() { - isZero := dst.MethodByName("IsZero") - result := isZero.Call([]reflect.Value{}) - if result[0].Bool() { - dst.Set(src) - } - } - return nil - } - } - return nil -} - -type Snapshot struct { - Time time.Time - // ... -} - -func main() { - src := Snapshot{time.Now()} - dest := Snapshot{} - mergo.Merge(&dest, src, WithTransformers(timeTransfomer{})) - fmt.Println(dest) - // Will print - // { 2018-01-12 01:15:00 +0000 UTC m=+0.000000001 } -} -``` - - -## Contact me - -If I can help you, you have an idea or you are using Mergo in your projects, don't hesitate to drop me a line (or a pull request): [@im_dario](https://twitter.com/im_dario) - -## About - -Written by [Dario Castañé](http://dario.im). - -## License - -[BSD 3-Clause](http://opensource.org/licenses/BSD-3-Clause) license, as [Go language](http://golang.org/LICENSE). diff --git a/vendor/github.com/imdario/mergo/doc.go b/vendor/github.com/imdario/mergo/doc.go deleted file mode 100644 index 6e9aa7baf..000000000 --- a/vendor/github.com/imdario/mergo/doc.go +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright 2013 Dario Castañé. All rights reserved. -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -/* -Package mergo merges same-type structs and maps by setting default values in zero-value fields. - -Mergo won't merge unexported (private) fields but will do recursively any exported one. It also won't merge structs inside maps (because they are not addressable using Go reflection). - -Usage - -From my own work-in-progress project: - - type networkConfig struct { - Protocol string - Address string - ServerType string `json: "server_type"` - Port uint16 - } - - type FssnConfig struct { - Network networkConfig - } - - var fssnDefault = FssnConfig { - networkConfig { - "tcp", - "127.0.0.1", - "http", - 31560, - }, - } - - // Inside a function [...] - - if err := mergo.Merge(&config, fssnDefault); err != nil { - log.Fatal(err) - } - - // More code [...] - -*/ -package mergo diff --git a/vendor/github.com/imdario/mergo/map.go b/vendor/github.com/imdario/mergo/map.go deleted file mode 100644 index 209814329..000000000 --- a/vendor/github.com/imdario/mergo/map.go +++ /dev/null @@ -1,174 +0,0 @@ -// Copyright 2014 Dario Castañé. All rights reserved. -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Based on src/pkg/reflect/deepequal.go from official -// golang's stdlib. - -package mergo - -import ( - "fmt" - "reflect" - "unicode" - "unicode/utf8" -) - -func changeInitialCase(s string, mapper func(rune) rune) string { - if s == "" { - return s - } - r, n := utf8.DecodeRuneInString(s) - return string(mapper(r)) + s[n:] -} - -func isExported(field reflect.StructField) bool { - r, _ := utf8.DecodeRuneInString(field.Name) - return r >= 'A' && r <= 'Z' -} - -// Traverses recursively both values, assigning src's fields values to dst. -// The map argument tracks comparisons that have already been seen, which allows -// short circuiting on recursive types. -func deepMap(dst, src reflect.Value, visited map[uintptr]*visit, depth int, config *config) (err error) { - overwrite := config.overwrite - if dst.CanAddr() { - addr := dst.UnsafeAddr() - h := 17 * addr - seen := visited[h] - typ := dst.Type() - for p := seen; p != nil; p = p.next { - if p.ptr == addr && p.typ == typ { - return nil - } - } - // Remember, remember... - visited[h] = &visit{addr, typ, seen} - } - zeroValue := reflect.Value{} - switch dst.Kind() { - case reflect.Map: - dstMap := dst.Interface().(map[string]interface{}) - for i, n := 0, src.NumField(); i < n; i++ { - srcType := src.Type() - field := srcType.Field(i) - if !isExported(field) { - continue - } - fieldName := field.Name - fieldName = changeInitialCase(fieldName, unicode.ToLower) - if v, ok := dstMap[fieldName]; !ok || (isEmptyValue(reflect.ValueOf(v)) || overwrite) { - dstMap[fieldName] = src.Field(i).Interface() - } - } - case reflect.Ptr: - if dst.IsNil() { - v := reflect.New(dst.Type().Elem()) - dst.Set(v) - } - dst = dst.Elem() - fallthrough - case reflect.Struct: - srcMap := src.Interface().(map[string]interface{}) - for key := range srcMap { - srcValue := srcMap[key] - fieldName := changeInitialCase(key, unicode.ToUpper) - dstElement := dst.FieldByName(fieldName) - if dstElement == zeroValue { - // We discard it because the field doesn't exist. - continue - } - srcElement := reflect.ValueOf(srcValue) - dstKind := dstElement.Kind() - srcKind := srcElement.Kind() - if srcKind == reflect.Ptr && dstKind != reflect.Ptr { - srcElement = srcElement.Elem() - srcKind = reflect.TypeOf(srcElement.Interface()).Kind() - } else if dstKind == reflect.Ptr { - // Can this work? I guess it can't. - if srcKind != reflect.Ptr && srcElement.CanAddr() { - srcPtr := srcElement.Addr() - srcElement = reflect.ValueOf(srcPtr) - srcKind = reflect.Ptr - } - } - - if !srcElement.IsValid() { - continue - } - if srcKind == dstKind { - if err = deepMerge(dstElement, srcElement, visited, depth+1, config); err != nil { - return - } - } else if dstKind == reflect.Interface && dstElement.Kind() == reflect.Interface { - if err = deepMerge(dstElement, srcElement, visited, depth+1, config); err != nil { - return - } - } else if srcKind == reflect.Map { - if err = deepMap(dstElement, srcElement, visited, depth+1, config); err != nil { - return - } - } else { - return fmt.Errorf("type mismatch on %s field: found %v, expected %v", fieldName, srcKind, dstKind) - } - } - } - return -} - -// Map sets fields' values in dst from src. -// src can be a map with string keys or a struct. dst must be the opposite: -// if src is a map, dst must be a valid pointer to struct. If src is a struct, -// dst must be map[string]interface{}. -// It won't merge unexported (private) fields and will do recursively -// any exported field. -// If dst is a map, keys will be src fields' names in lower camel case. -// Missing key in src that doesn't match a field in dst will be skipped. This -// doesn't apply if dst is a map. -// This is separated method from Merge because it is cleaner and it keeps sane -// semantics: merging equal types, mapping different (restricted) types. -func Map(dst, src interface{}, opts ...func(*config)) error { - return _map(dst, src, opts...) -} - -// MapWithOverwrite will do the same as Map except that non-empty dst attributes will be overriden by -// non-empty src attribute values. -// Deprecated: Use Map(…) with WithOverride -func MapWithOverwrite(dst, src interface{}, opts ...func(*config)) error { - return _map(dst, src, append(opts, WithOverride)...) -} - -func _map(dst, src interface{}, opts ...func(*config)) error { - var ( - vDst, vSrc reflect.Value - err error - ) - config := &config{} - - for _, opt := range opts { - opt(config) - } - - if vDst, vSrc, err = resolveValues(dst, src); err != nil { - return err - } - // To be friction-less, we redirect equal-type arguments - // to deepMerge. Only because arguments can be anything. - if vSrc.Kind() == vDst.Kind() { - return deepMerge(vDst, vSrc, make(map[uintptr]*visit), 0, config) - } - switch vSrc.Kind() { - case reflect.Struct: - if vDst.Kind() != reflect.Map { - return ErrExpectedMapAsDestination - } - case reflect.Map: - if vDst.Kind() != reflect.Struct { - return ErrExpectedStructAsDestination - } - default: - return ErrNotSupported - } - return deepMap(vDst, vSrc, make(map[uintptr]*visit), 0, config) -} diff --git a/vendor/github.com/imdario/mergo/merge.go b/vendor/github.com/imdario/mergo/merge.go deleted file mode 100644 index 520ef40bf..000000000 --- a/vendor/github.com/imdario/mergo/merge.go +++ /dev/null @@ -1,219 +0,0 @@ -// Copyright 2013 Dario Castañé. All rights reserved. -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Based on src/pkg/reflect/deepequal.go from official -// golang's stdlib. - -package mergo - -import "reflect" - -func hasExportedField(dst reflect.Value) (exported bool) { - for i, n := 0, dst.NumField(); i < n; i++ { - field := dst.Type().Field(i) - if field.Anonymous && dst.Field(i).Kind() == reflect.Struct { - exported = exported || hasExportedField(dst.Field(i)) - } else { - exported = exported || len(field.PkgPath) == 0 - } - } - return -} - -type config struct { - overwrite bool - transformers transformers -} - -type transformers interface { - Transformer(reflect.Type) func(dst, src reflect.Value) error -} - -// Traverses recursively both values, assigning src's fields values to dst. -// The map argument tracks comparisons that have already been seen, which allows -// short circuiting on recursive types. -func deepMerge(dst, src reflect.Value, visited map[uintptr]*visit, depth int, config *config) (err error) { - overwrite := config.overwrite - - if !src.IsValid() { - return - } - if dst.CanAddr() { - addr := dst.UnsafeAddr() - h := 17 * addr - seen := visited[h] - typ := dst.Type() - for p := seen; p != nil; p = p.next { - if p.ptr == addr && p.typ == typ { - return nil - } - } - // Remember, remember... - visited[h] = &visit{addr, typ, seen} - } - - if config.transformers != nil && !isEmptyValue(dst) { - if fn := config.transformers.Transformer(dst.Type()); fn != nil { - err = fn(dst, src) - return - } - } - - switch dst.Kind() { - case reflect.Struct: - if hasExportedField(dst) { - for i, n := 0, dst.NumField(); i < n; i++ { - if err = deepMerge(dst.Field(i), src.Field(i), visited, depth+1, config); err != nil { - return - } - } - } else { - if dst.CanSet() && !isEmptyValue(src) && (overwrite || isEmptyValue(dst)) { - dst.Set(src) - } - } - case reflect.Map: - if len(src.MapKeys()) == 0 && !src.IsNil() && len(dst.MapKeys()) == 0 { - dst.Set(reflect.MakeMap(dst.Type())) - return - } - for _, key := range src.MapKeys() { - srcElement := src.MapIndex(key) - if !srcElement.IsValid() { - continue - } - dstElement := dst.MapIndex(key) - switch srcElement.Kind() { - case reflect.Chan, reflect.Func, reflect.Map, reflect.Interface, reflect.Slice: - if srcElement.IsNil() { - continue - } - fallthrough - default: - if !srcElement.CanInterface() { - continue - } - switch reflect.TypeOf(srcElement.Interface()).Kind() { - case reflect.Struct: - fallthrough - case reflect.Ptr: - fallthrough - case reflect.Map: - if err = deepMerge(dstElement, srcElement, visited, depth+1, config); err != nil { - return - } - case reflect.Slice: - srcSlice := reflect.ValueOf(srcElement.Interface()) - - var dstSlice reflect.Value - if !dstElement.IsValid() || dstElement.IsNil() { - dstSlice = reflect.MakeSlice(srcSlice.Type(), 0, srcSlice.Len()) - } else { - dstSlice = reflect.ValueOf(dstElement.Interface()) - } - - dstSlice = reflect.AppendSlice(dstSlice, srcSlice) - dst.SetMapIndex(key, dstSlice) - } - } - if dstElement.IsValid() && reflect.TypeOf(srcElement.Interface()).Kind() == reflect.Map { - continue - } - - if srcElement.IsValid() && (overwrite || (!dstElement.IsValid() || isEmptyValue(dst))) { - if dst.IsNil() { - dst.Set(reflect.MakeMap(dst.Type())) - } - dst.SetMapIndex(key, srcElement) - } - } - case reflect.Slice: - dst.Set(reflect.AppendSlice(dst, src)) - case reflect.Ptr: - fallthrough - case reflect.Interface: - if src.IsNil() { - break - } - if src.Kind() != reflect.Interface { - if dst.IsNil() || overwrite { - if dst.CanSet() && (overwrite || isEmptyValue(dst)) { - dst.Set(src) - } - } else if src.Kind() == reflect.Ptr { - if err = deepMerge(dst.Elem(), src.Elem(), visited, depth+1, config); err != nil { - return - } - } else if dst.Elem().Type() == src.Type() { - if err = deepMerge(dst.Elem(), src, visited, depth+1, config); err != nil { - return - } - } else { - return ErrDifferentArgumentsTypes - } - break - } - if dst.IsNil() || overwrite { - if dst.CanSet() && (overwrite || isEmptyValue(dst)) { - dst.Set(src) - } - } else if err = deepMerge(dst.Elem(), src.Elem(), visited, depth+1, config); err != nil { - return - } - default: - if dst.CanSet() && !isEmptyValue(src) && (overwrite || isEmptyValue(dst)) { - dst.Set(src) - } - } - return -} - -// Merge will fill any empty for value type attributes on the dst struct using corresponding -// src attributes if they themselves are not empty. dst and src must be valid same-type structs -// and dst must be a pointer to struct. -// It won't merge unexported (private) fields and will do recursively any exported field. -func Merge(dst, src interface{}, opts ...func(*config)) error { - return merge(dst, src, opts...) -} - -// MergeWithOverwrite will do the same as Merge except that non-empty dst attributes will be overriden by -// non-empty src attribute values. -// Deprecated: use Merge(…) with WithOverride -func MergeWithOverwrite(dst, src interface{}, opts ...func(*config)) error { - return merge(dst, src, append(opts, WithOverride)...) -} - -// WithTransformers adds transformers to merge, allowing to customize the merging of some types. -func WithTransformers(transformers transformers) func(*config) { - return func(config *config) { - config.transformers = transformers - } -} - -// WithOverride will make merge override non-empty dst attributes with non-empty src attributes values. -func WithOverride(config *config) { - config.overwrite = true -} - -func merge(dst, src interface{}, opts ...func(*config)) error { - var ( - vDst, vSrc reflect.Value - err error - ) - - config := &config{} - - for _, opt := range opts { - opt(config) - } - - if vDst, vSrc, err = resolveValues(dst, src); err != nil { - return err - } - if vDst.Type() != vSrc.Type() { - return ErrDifferentArgumentsTypes - } - return deepMerge(vDst, vSrc, make(map[uintptr]*visit), 0, config) -} diff --git a/vendor/github.com/imdario/mergo/mergo.go b/vendor/github.com/imdario/mergo/mergo.go deleted file mode 100644 index 785618cd0..000000000 --- a/vendor/github.com/imdario/mergo/mergo.go +++ /dev/null @@ -1,92 +0,0 @@ -// Copyright 2013 Dario Castañé. All rights reserved. -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Based on src/pkg/reflect/deepequal.go from official -// golang's stdlib. - -package mergo - -import ( - "errors" - "reflect" -) - -// Errors reported by Mergo when it finds invalid arguments. -var ( - ErrNilArguments = errors.New("src and dst must not be nil") - ErrDifferentArgumentsTypes = errors.New("src and dst must be of same type") - ErrNotSupported = errors.New("only structs and maps are supported") - ErrExpectedMapAsDestination = errors.New("dst was expected to be a map") - ErrExpectedStructAsDestination = errors.New("dst was expected to be a struct") -) - -// During deepMerge, must keep track of checks that are -// in progress. The comparison algorithm assumes that all -// checks in progress are true when it reencounters them. -// Visited are stored in a map indexed by 17 * a1 + a2; -type visit struct { - ptr uintptr - typ reflect.Type - next *visit -} - -// From src/pkg/encoding/json/encode.go. -func isEmptyValue(v reflect.Value) bool { - switch v.Kind() { - case reflect.Array, reflect.Map, reflect.Slice, reflect.String: - return v.Len() == 0 - case reflect.Bool: - return !v.Bool() - case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: - return v.Int() == 0 - case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: - return v.Uint() == 0 - case reflect.Float32, reflect.Float64: - return v.Float() == 0 - case reflect.Interface, reflect.Ptr, reflect.Func: - return v.IsNil() - case reflect.Invalid: - return true - } - return false -} - -func resolveValues(dst, src interface{}) (vDst, vSrc reflect.Value, err error) { - if dst == nil || src == nil { - err = ErrNilArguments - return - } - vDst = reflect.ValueOf(dst).Elem() - if vDst.Kind() != reflect.Struct && vDst.Kind() != reflect.Map { - err = ErrNotSupported - return - } - vSrc = reflect.ValueOf(src) - // We check if vSrc is a pointer to dereference it. - if vSrc.Kind() == reflect.Ptr { - vSrc = vSrc.Elem() - } - return -} - -// Traverses recursively both values, assigning src's fields values to dst. -// The map argument tracks comparisons that have already been seen, which allows -// short circuiting on recursive types. -func deeper(dst, src reflect.Value, visited map[uintptr]*visit, depth int) (err error) { - if dst.CanAddr() { - addr := dst.UnsafeAddr() - h := 17 * addr - seen := visited[h] - typ := dst.Type() - for p := seen; p != nil; p = p.next { - if p.ptr == addr && p.typ == typ { - return nil - } - } - // Remember, remember... - visited[h] = &visit{addr, typ, seen} - } - return // TODO refactor -} diff --git a/vendor/vendor.json b/vendor/vendor.json index ad4075661..2591decdd 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -1424,12 +1424,6 @@ "revision": "984a73625de3138f44deb38d00878fab39eb6447", "revisionTime": "2018-05-30T15:59:58Z" }, - { - "checksumSHA1": "/MbkohdDPqGPeAkHq6ZkInB/WD0=", - "path": "github.com/imdario/mergo", - "revision": "0d4b488675fdec1dde48751b05ab530cf0b630e1", - "revisionTime": "2018-01-26T22:59:47Z" - }, { "checksumSHA1": "Ex8204BJkD9d0nd8mLqw5YrTpNU=", "path": "github.com/ishidawataru/sctp",