diff --git a/apis/server/copy_bridge.go b/apis/server/copy_bridge.go index 53ffbea450..2a247bffcb 100644 --- a/apis/server/copy_bridge.go +++ b/apis/server/copy_bridge.go @@ -5,7 +5,6 @@ import ( "encoding/base64" "encoding/json" "errors" - "fmt" "io" "net/http" @@ -34,8 +33,6 @@ func (s *Server) headContainersArchive(ctx context.Context, rw http.ResponseWrit name := mux.Vars(req)["name"] path := req.FormValue("path") - fmt.Print(name) - if name == "" { return httputils.NewHTTPError(errors.New("name can't be empty"), http.StatusBadRequest) } @@ -58,7 +55,6 @@ func (s *Server) headContainersArchive(ctx context.Context, rw http.ResponseWrit base64.StdEncoding.EncodeToString(statJSON), ) - rw.WriteHeader(http.StatusOK) return nil } @@ -95,6 +91,5 @@ func (s *Server) getContainersArchive(ctx context.Context, rw http.ResponseWrite return err } - rw.WriteHeader(http.StatusOK) return nil } diff --git a/apis/swagger.yml b/apis/swagger.yml index 9416e4085f..baf9e0c907 100644 --- a/apis/swagger.yml +++ b/apis/swagger.yml @@ -4506,6 +4506,24 @@ definitions: type: "string" description: "ID uniquely identifies an image committed by a container" + ContainerPathStat: + description: "ContainerPathStat is used to describe the stat of file" + type: "object" + properties: + name: + type: "string" + size: + type: "string" + mode: + type: "integer" + format: "uint32" + mtime: + description: "modification time." + type: "string" + format: date-time + path: + type: "string" + parameters: id: name: id diff --git a/apis/types/container_path_stat.go b/apis/types/container_path_stat.go new file mode 100644 index 0000000000..c0f0cd6131 --- /dev/null +++ b/apis/types/container_path_stat.go @@ -0,0 +1,79 @@ +// Code generated by go-swagger; DO NOT EDIT. + +package types + +// This file was generated by the swagger tool. +// Editing this file might prove futile when you re-run the swagger generate command + +import ( + "github.com/go-openapi/errors" + strfmt "github.com/go-openapi/strfmt" + "github.com/go-openapi/swag" + "github.com/go-openapi/validate" +) + +// ContainerPathStat ContainerPathStat is used to describe the stat of file +// swagger:model ContainerPathStat +type ContainerPathStat struct { + + // mode + Mode uint32 `json:"mode,omitempty"` + + // modification time. + // Format: date-time + Mtime strfmt.DateTime `json:"mtime,omitempty"` + + // name + Name string `json:"name,omitempty"` + + // path + Path string `json:"path,omitempty"` + + // size + Size string `json:"size,omitempty"` +} + +// Validate validates this container path stat +func (m *ContainerPathStat) Validate(formats strfmt.Registry) error { + var res []error + + if err := m.validateMtime(formats); err != nil { + res = append(res, err) + } + + if len(res) > 0 { + return errors.CompositeValidationError(res...) + } + return nil +} + +func (m *ContainerPathStat) validateMtime(formats strfmt.Registry) error { + + if swag.IsZero(m.Mtime) { // not required + return nil + } + + if err := validate.FormatOf("mtime", "body", "date-time", m.Mtime.String(), formats); err != nil { + return err + } + + return nil +} + +// MarshalBinary interface implementation +func (m *ContainerPathStat) MarshalBinary() ([]byte, error) { + if m == nil { + return nil, nil + } + return swag.WriteJSON(m) +} + +// UnmarshalBinary interface implementation +func (m *ContainerPathStat) UnmarshalBinary(b []byte) error { + var res ContainerPathStat + if err := swag.ReadJSON(b, &res); err != nil { + return err + } + *m = res + return nil +} diff --git a/cli/cp.go b/cli/cp.go new file mode 100644 index 0000000000..dd2c106f6a --- /dev/null +++ b/cli/cp.go @@ -0,0 +1,237 @@ +package main + +import ( + "context" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "github.com/docker/docker/pkg/archive" + "github.com/spf13/cobra" +) + +// createDescription is used to describe create command in detail and auto generate command doc. +var copyDescription = "Copy files/folders between a container and the local filesystem\n" + + "\nUse '-' as the source to read a tar archive from stdin\n" + + "and extract it to a directory destination in a container.\n" + + "Use '-' as the destination to stream a tar archive of a\n" + + "container source to stdout." + +// CopyCommand use to implement 'copy' command, it copy files between host and container. +type CopyCommand struct { + *container + baseCommand +} + +type copyOptions struct { + source string + destination string +} + +// Init initialize copy command. +func (cc *CopyCommand) Init(c *Cli) { + var opts copyOptions + + cc.cli = c + cc.cmd = &cobra.Command{ + Use: "cp [OPTIONS] CONTAINER:SRC_PATH DEST_PATH|-\n pouch cp [OPTIONS] SRC_PATH|- CONTAINER:DEST_PATH", + Short: "Copy files/folders between a container and the local filesystem", + Long: copyDescription, + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + if args[0] == "" { + return fmt.Errorf("source can not be empty") + } + if args[1] == "" { + return fmt.Errorf("destination can not be empty") + } + + opts.source = args[0] + opts.destination = args[1] + return cc.runCopy(opts) + }, + Example: copyExample(), + } + + cc.addFlags() +} + +// addFlags adds flags for specific command. +func (cc *CopyCommand) addFlags() { + flagSet := cc.cmd.Flags() + flagSet.SetInterspersed(false) + + // TODO: add more flags here +} + +func splitCpArg(arg string) (container, path string) { + if filepath.IsAbs(arg) { + // Explicit local absolute path, e.g., `/foo`. + return "", arg + } + + parts := strings.SplitN(arg, ":", 2) + + if len(parts) == 1 { + return "", arg + } + + return parts[0], parts[1] +} + +type copyDirection int + +const ( + fromContainer copyDirection = 1 << iota + toContainer + acrossContainers = fromContainer | toContainer +) + +// runCopy is the entry of Copy command. +func (cc *CopyCommand) runCopy(opts copyOptions) error { + srcContainer, srcPath := splitCpArg(opts.source) + dstContainer, dstPath := splitCpArg(opts.destination) + + var direction copyDirection + if srcContainer != "" { + direction |= fromContainer + } + if dstContainer != "" { + direction |= toContainer + } + + ctx := context.Background() + + switch direction { + case fromContainer: + return copyFromContainer(ctx, cc.cli, srcContainer, srcPath, dstPath) + case toContainer: + return copyToContainer(ctx, cc.cli, srcPath, dstContainer, dstPath) + case acrossContainers: + // Copying between containers isn't supported. + return fmt.Errorf("copying between containers is not supported") + default: + // User didn't specify any container. + return fmt.Errorf("must specify at least one container source") + } +} + +func resolveLocalPath(localPath string) (absPath string, err error) { + if absPath, err = filepath.Abs(localPath); err != nil { + return + } + + return archive.PreserveTrailingDotOrSeparator(absPath, localPath, os.PathSeparator), nil +} + +func copyFromContainer(ctx context.Context, cli *Cli, srcContainer, srcPath, dstPath string) (err error) { + apiClient := cli.Client() + + if dstPath != "-" { + // Get an absolute destination path. + dstPath, err = resolveLocalPath(dstPath) + if err != nil { + return err + } + } + + content, stat, err := apiClient.CopyFromContainer(ctx, srcContainer, srcPath) + if err != nil { + return err + } + defer content.Close() + + if dstPath == "-" { + // Send the response to STDOUT. + _, err = io.Copy(os.Stdout, content) + return err + } + + // Prepare source copy info. + srcInfo := archive.CopyInfo{ + Path: srcPath, + Exists: true, + IsDir: os.FileMode(stat.Mode).IsDir(), + RebaseName: "", + } + + preArchive := content + if len(srcInfo.RebaseName) != 0 { + _, srcBase := archive.SplitPathDirEntry(srcInfo.Path) + preArchive = archive.RebaseArchiveEntries(content, srcBase, srcInfo.RebaseName) + } + // See comments in the implementation of `archive.CopyTo` for exactly what + // goes into deciding how and whether the source archive needs to be + // altered for the correct copy behavior. + return archive.CopyTo(preArchive, srcInfo, dstPath) +} + +func copyToContainer(ctx context.Context, cli *Cli, srcPath, dstContainer, dstPath string) (err error) { + apiClient := cli.Client() + + if srcPath != "-" { + // Get an absolute source path. + srcPath, err = resolveLocalPath(srcPath) + if err != nil { + return err + } + } + + dstInfo := archive.CopyInfo{Path: dstPath} + dstStat, err := apiClient.ContainerStatPath(ctx, dstContainer, dstPath) + + // Ignore any error and assume that the parent directory of the destination + // path exists, in which case the copy may still succeed. If there is any + // type of conflict (e.g., non-directory overwriting an existing directory + // or vice versa) the extraction will fail. If the destination simply did + // not exist, but the parent directory does, the extraction will still + // succeed. + if err == nil { + dstInfo.Exists, dstInfo.IsDir = true, os.FileMode(dstStat.Mode).IsDir() + } + + var ( + content io.Reader + resolvedDstPath string + ) + + if srcPath == "-" { + // Use STDIN. + content = os.Stdin + resolvedDstPath = dstInfo.Path + if !dstInfo.IsDir { + return fmt.Errorf("destination %q must be a directory", fmt.Sprintf("%s:%s", dstContainer, dstPath)) + } + } else { + // Prepare source copy info. + srcInfo, err := archive.CopyInfoSourcePath(srcPath, false) + if err != nil { + return err + } + + srcArchive, err := archive.TarResource(srcInfo) + if err != nil { + return err + } + defer srcArchive.Close() + + dstDir, preparedArchive, err := archive.PrepareArchiveCopy(srcArchive, srcInfo, dstInfo) + if err != nil { + return err + } + defer preparedArchive.Close() + + resolvedDstPath = dstDir + content = preparedArchive + } + + return apiClient.CopyToContainer(ctx, dstContainer, resolvedDstPath, content) +} + +// copyExample shows examples in copy command, and is used in auto-generated cli docs. +func copyExample() string { + return `$ pouch cp 8assd1234:/root/foo /home +$ pouch cp /home/bar 712yasbc:/root` +} diff --git a/cli/main.go b/cli/main.go index 9a952e8f72..d5717b0f46 100644 --- a/cli/main.go +++ b/cli/main.go @@ -55,6 +55,7 @@ func main() { cli.AddCommand(base, &CommitCommand{}) cli.AddCommand(base, &StatsCommand{}) cli.AddCommand(base, &BuildCommand{}) + cli.AddCommand(base, &CopyCommand{}) // add generate doc command cli.AddCommand(base, &GenDocCommand{}) diff --git a/client/container_copy.go b/client/container_copy.go new file mode 100644 index 0000000000..df4ec2594d --- /dev/null +++ b/client/container_copy.go @@ -0,0 +1,87 @@ +package client + +import ( + "context" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strings" + + "github.com/alibaba/pouch/apis/types" +) + +// ContainerStatPath returns Stat information about a path inside the container filesystem. +func (client *APIClient) ContainerStatPath(ctx context.Context, name string, path string) (types.ContainerPathStat, error) { + query := url.Values{} + query.Set("path", path) + + urlStr := fmt.Sprintf("/containers/%s/archive", name) + + response, err := client.head(ctx, urlStr, query, nil) + if err != nil { + return types.ContainerPathStat{}, err + } + ensureCloseReader(response) + return getContainerPathStatFromHeader(response.Header) +} + +// CopyFromContainer gets the content from the container and returns it as a Reader +// to manipulate it in the host. It's up to the caller to close the reader. +func (client *APIClient) CopyFromContainer(ctx context.Context, container, srcPath string) (io.ReadCloser, types.ContainerPathStat, error) { + query := url.Values{} + query.Set("path", srcPath) + + apiPath := fmt.Sprintf("/containers/%s/archive", container) + response, err := client.get(ctx, apiPath, query, nil) + if err != nil { + return nil, types.ContainerPathStat{}, err + } + + if response.StatusCode != http.StatusOK { + return nil, types.ContainerPathStat{}, fmt.Errorf("unexpected status code from daemon: %d", response.StatusCode) + } + + stat, err := getContainerPathStatFromHeader(response.Header) + if err != nil { + return nil, stat, fmt.Errorf("unable to get resource stat from response: %s", err) + } + return response.Body, stat, err +} + +// CopyToContainer copies content into the container filesystem. +func (client *APIClient) CopyToContainer(ctx context.Context, container, path string, content io.Reader) error { + query := url.Values{} + query.Set("noOverwriteDirNonDir", "true") + query.Set("path", path) + + apiPath := fmt.Sprintf("/containers/%s/archive", container) + + response, err := client.putRawData(ctx, apiPath, query, content, nil) + if err != nil { + return err + } + ensureCloseReader(response) + + if response.StatusCode != http.StatusOK { + return fmt.Errorf("unexpected status code from daemon: %d", response.StatusCode) + } + + return nil +} + +func getContainerPathStatFromHeader(header http.Header) (types.ContainerPathStat, error) { + var stat types.ContainerPathStat + + encodedStat := header.Get("X-Docker-Container-Path-Stat") + statDecoder := base64.NewDecoder(base64.StdEncoding, strings.NewReader(encodedStat)) + + err := json.NewDecoder(statDecoder).Decode(&stat) + if err != nil { + err = fmt.Errorf("unable to decode container path stat header: %s", err) + } + + return stat, err +} diff --git a/client/interface.go b/client/interface.go index 7ddf171543..e7f5bb70af 100644 --- a/client/interface.go +++ b/client/interface.go @@ -46,6 +46,9 @@ type ContainerAPIClient interface { ContainerCheckpointDelete(ctx context.Context, name string, options types.CheckpointDeleteOptions) error ContainerCommit(ctx context.Context, name string, options types.ContainerCommitOptions) (*types.ContainerCommitResp, error) ContainerStats(ctx context.Context, name string, stream bool) (io.ReadCloser, error) + ContainerStatPath(ctx context.Context, name string, path string) (types.ContainerPathStat, error) + CopyFromContainer(ctx context.Context, container, srcPath string) (io.ReadCloser, types.ContainerPathStat, error) + CopyToContainer(ctx context.Context, container, path string, content io.Reader) error } // ImageAPIClient defines methods of Image client. diff --git a/client/request.go b/client/request.go index b526d6c7b7..f1f12f83a7 100644 --- a/client/request.go +++ b/client/request.go @@ -55,7 +55,7 @@ func (client *APIClient) head(ctx context.Context, path string, query url.Values return client.sendRequest(ctx, "HEAD", path, query, nil, headers) } -func (client *APIClient) putRaw(ctx context.Context, path string, query url.Values, data io.Reader, headers map[string][]string) (*Response, error) { +func (client *APIClient) putRawData(ctx context.Context, path string, query url.Values, data io.Reader, headers map[string][]string) (*Response, error) { return client.sendRequest(ctx, "PUT", path, query, data, headers) } diff --git a/daemon/mgr/container.go b/daemon/mgr/container.go index 8cd411822f..727a70ec7d 100644 --- a/daemon/mgr/container.go +++ b/daemon/mgr/container.go @@ -166,10 +166,10 @@ type ContainerMgr interface { Commit(ctx context.Context, name string, options *types.ContainerCommitOptions) (*types.ContainerCommitResp, error) // StatPath stats the dir info at the specified path in the container. - StatPath(ctx context.Context, name, path string) (stat *ContainerPathStat, err error) + StatPath(ctx context.Context, name, path string) (stat *types.ContainerPathStat, err error) // ArchivePath return an archive and dir info at the specified path in the container. - ArchivePath(ctx context.Context, name, path string) (content io.ReadCloser, stat *ContainerPathStat, err error) + ArchivePath(ctx context.Context, name, path string) (content io.ReadCloser, stat *types.ContainerPathStat, err error) // ExtractToDir extracts the given archive at the specified path in the container. ExtractToDir(ctx context.Context, name, path string, copyUIDGID, noOverwriteDirNonDir bool, content io.Reader) error diff --git a/daemon/mgr/container_copy.go b/daemon/mgr/container_copy.go index 02f19559fb..b89e8a947d 100644 --- a/daemon/mgr/container_copy.go +++ b/daemon/mgr/container_copy.go @@ -3,18 +3,23 @@ package mgr import ( "context" "errors" - "fmt" "io" "os" "path/filepath" + "strconv" "strings" + "github.com/alibaba/pouch/apis/types" + "github.com/alibaba/pouch/pkg/ioutils" + "github.com/docker/docker/pkg/archive" "github.com/docker/docker/pkg/chrootarchive" + "github.com/docker/docker/pkg/mount" + "github.com/go-openapi/strfmt" ) // StatPath stats the dir info at the specified path in the container. -func (mgr *ContainerManager) StatPath(ctx context.Context, name, path string) (stat *ContainerPathStat, err error) { +func (mgr *ContainerManager) StatPath(ctx context.Context, name, path string) (stat *types.ContainerPathStat, err error) { c, err := mgr.container(name) if err != nil { return nil, err @@ -22,99 +27,100 @@ func (mgr *ContainerManager) StatPath(ctx context.Context, name, path string) (s c.Lock() defer c.Unlock() - cleanup := true - // judge the file exists or not - _, err = os.Stat(path) - if err == nil { - cleanup = false - } - - err = mgr.Mount(ctx, c, true) + err = mgr.Mount(ctx, c, false) if err != nil { return nil, err } - defer mgr.Unmount(ctx, c, true, cleanup) - err = mgr.attachVolumes(ctx, c) - if err != nil { - return nil, err - } - defer mgr.detachVolumes(ctx, c, false) + defer mgr.Unmount(ctx, c, false, !c.IsRunningOrPaused()) - // Consider the given path as an absolute path in the container. - absPath := path - if !filepath.IsAbs(absPath) { - absPath = archive.PreserveTrailingDotOrSeparator(filepath.Join("/", path), path, os.PathSeparator) + if !c.IsRunningOrPaused() { + err = mgr.attachVolumes(ctx, c) + if err != nil { + return nil, err + } + defer mgr.detachVolumes(ctx, c, false) } - resolvedDirPath := c.GetResourcePath(c.BaseFS, filepath.Dir(absPath)) - - resolvedPath := fmt.Sprintf("%s/%s", resolvedDirPath, filepath.Base(absPath)) + err = c.mountVolumes(!c.IsRunningOrPaused()) + if err != nil { + return nil, err + } + defer c.unmountVolumes(!c.IsRunningOrPaused()) + resolvedPath, absPath := c.getResolvedPath(path) lstat, err := os.Lstat(resolvedPath) + if err != nil { return nil, err } - return &ContainerPathStat{ + return &types.ContainerPathStat{ Name: lstat.Name(), Path: absPath, - Size: lstat.Size(), - Mode: lstat.Mode(), - Mtime: lstat.ModTime(), + Size: strconv.FormatInt(lstat.Size(), 10), + Mode: uint32(lstat.Mode()), + Mtime: strfmt.DateTime(lstat.ModTime()), }, nil } // ArchivePath return an archive and dir info at the specified path in the container. -func (mgr *ContainerManager) ArchivePath(ctx context.Context, name, path string) (content io.ReadCloser, stat *ContainerPathStat, err error) { +func (mgr *ContainerManager) ArchivePath(ctx context.Context, name, path string) (content io.ReadCloser, stat *types.ContainerPathStat, err error) { c, err := mgr.container(name) if err != nil { return nil, nil, err } c.Lock() - defer c.Unlock() - - cleanup := true - // judge the file exists or not - _, err = os.Stat(path) - if err == nil { - cleanup = false - } + defer func() { + if err != nil { + c.Unlock() + } + }() - err = mgr.Mount(ctx, c, true) + err = mgr.Mount(ctx, c, false) if err != nil { return nil, nil, err } - defer mgr.Unmount(ctx, c, true, cleanup) + defer func() { + if err != nil { + mgr.Unmount(ctx, c, false, !c.IsRunningOrPaused()) + } + }() - err = mgr.attachVolumes(ctx, c) - if err != nil { - return nil, nil, err + if !c.IsRunningOrPaused() { + err = mgr.attachVolumes(ctx, c) + if err != nil { + return nil, nil, err + } + defer func() { + if err != nil { + mgr.detachVolumes(ctx, c, false) + } + }() } - defer mgr.detachVolumes(ctx, c, false) - // Consider the given path as an absolute path in the container. - absPath := path - if !filepath.IsAbs(absPath) { - absPath = archive.PreserveTrailingDotOrSeparator(filepath.Join("/", path), path, os.PathSeparator) + err = c.mountVolumes(!c.IsRunningOrPaused()) + if err != nil { + return nil, nil, err } + defer func() { + if err != nil { + defer c.unmountVolumes(!c.IsRunningOrPaused()) + } + }() - // get the real path on the host - resolvedDirPath := c.GetResourcePath(c.BaseFS, filepath.Dir(absPath)) - - resolvedPath := fmt.Sprintf("%s/%s", resolvedDirPath, filepath.Base(absPath)) - fmt.Println(resolvedPath) + resolvedPath, absPath := c.getResolvedPath(path) lstat, err := os.Lstat(resolvedPath) if err != nil { return nil, nil, err } - stat = &ContainerPathStat{ + stat = &types.ContainerPathStat{ Name: lstat.Name(), Path: absPath, - Size: lstat.Size(), - Mode: lstat.Mode(), - Mtime: lstat.ModTime(), + Size: strconv.FormatInt(lstat.Size(), 10), + Mode: uint32(lstat.Mode()), + Mtime: strfmt.DateTime(lstat.ModTime()), } // TODO: support follow link in container rootfs copyInfo, err := archive.CopyInfoSourcePath(resolvedPath, false) @@ -126,7 +132,19 @@ func (mgr *ContainerManager) ArchivePath(ctx context.Context, name, path string) return nil, nil, err } - return data, stat, nil + // wait for io finish, then unmount the rootfs + content = ioutils.NewReadCloserWrapper(data, func() error { + err := data.Close() + if !c.IsRunningOrPaused() { + mgr.detachVolumes(ctx, c, false) + } + c.unmountVolumes(!c.IsRunningOrPaused()) + mgr.Unmount(ctx, c, false, !c.IsRunningOrPaused()) + c.Unlock() + return err + }) + + return content, stat, nil } // ExtractToDir extracts the given archive at the specified path in the container. @@ -138,35 +156,27 @@ func (mgr *ContainerManager) ExtractToDir(ctx context.Context, name, path string c.Lock() defer c.Unlock() - cleanup := true - // judge the file exists or not - _, err = os.Stat(path) - if err == nil { - cleanup = false - } - - err = mgr.Mount(ctx, c, true) + err = mgr.Mount(ctx, c, false) if err != nil { return err } - defer mgr.Unmount(ctx, c, true, cleanup) + defer mgr.Unmount(ctx, c, false, !c.IsRunningOrPaused()) - err = mgr.attachVolumes(ctx, c) - if err != nil { - return err + if !c.IsRunningOrPaused() { + err = mgr.attachVolumes(ctx, c) + if err != nil { + return err + } + defer mgr.detachVolumes(ctx, c, false) } - defer mgr.detachVolumes(ctx, c, false) - // Consider the given path as an absolute path in the container. - absPath := path - if !filepath.IsAbs(absPath) { - absPath = archive.PreserveTrailingDotOrSeparator(filepath.Join("/", path), path, os.PathSeparator) + err = c.mountVolumes(!c.IsRunningOrPaused()) + if err != nil { + return err } + defer c.unmountVolumes(!c.IsRunningOrPaused()) - // get the real path on the host - resolvedDirPath := c.GetResourcePath(c.BaseFS, filepath.Dir(absPath)) - - resolvedPath := fmt.Sprintf("%s/%s", resolvedDirPath, filepath.Base(absPath)) + resolvedPath, _ := c.getResolvedPath(path) lstat, err := os.Lstat(resolvedPath) if err != nil { @@ -201,3 +211,72 @@ func (mgr *ContainerManager) ExtractToDir(ctx context.Context, name, path string return chrootarchive.Untar(content, resolvedPath, opts) } + +func (c *Container) getResolvedPath(path string) (resolvedPath, absPath string) { + // consider the given path as an absolute path in the container. + absPath = path + if !filepath.IsAbs(absPath) { + absPath = archive.PreserveTrailingDotOrSeparator(filepath.Join(string(os.PathSeparator), path), path, os.PathSeparator) + } + + // get the real path on the host + resolvedPath = filepath.Join(c.BaseFS, absPath) + resolvedPath = filepath.Clean(resolvedPath) + + return resolvedPath, absPath +} + +func (c *Container) mountVolumes(created bool) error { + + for _, m := range c.Mounts { + dest, _ := c.getResolvedPath(m.Destination) + + _, err := os.Stat(m.Source) + if err != nil { + return err + } + + if created { + if e := os.MkdirAll(dest, 0755); e != nil { + return e + } + } + + writeMode := "ro" + if m.RW { + writeMode = "rw" + } + + // mountVolumes() seems to be called for temporary mounts + // outside the container. Soon these will be unmounted with + // lazy unmount option and given we have mounted the rbind, + // all the submounts will propagate if these are shared. If + // daemon is running in host namespace and has / as shared + // then these unmounts will propagate and unmount original + // mount as well. So make all these mounts rprivate. + // Do not use propagation property of volume as that should + // apply only when mounting happens inside the container. + opts := strings.Join([]string{"bind", writeMode, "rprivate"}, ",") + if err := mount.Mount(m.Source, dest, "", opts); err != nil { + return err + } + } + + return nil +} + +func (c *Container) unmountVolumes(remove bool) error { + for _, m := range c.Mounts { + dest, _ := c.getResolvedPath(m.Destination) + + if err := mount.Unmount(dest); err != nil { + return err + } + if remove { + if err := os.RemoveAll(dest); err != nil { + return err + } + } + } + return nil +} diff --git a/daemon/mgr/container_storage.go b/daemon/mgr/container_storage.go index 4fa8e599b1..16055103bb 100644 --- a/daemon/mgr/container_storage.go +++ b/daemon/mgr/container_storage.go @@ -690,9 +690,9 @@ func (mgr *ContainerManager) setMountFS(ctx context.Context, c *Container) { } // Mount sets the container rootfs -// online = true, mount rootfs at c.BaseFS -// online = false, mount rootfs at c.MountFS -func (mgr *ContainerManager) Mount(ctx context.Context, c *Container, online bool) error { +// preCreate = false, mount rootfs at c.BaseFS +// preCreate = true, pouchd does some initial job before container created, so we mount rootfs at c.MountFS +func (mgr *ContainerManager) Mount(ctx context.Context, c *Container, preCreate bool) error { mounts, err := mgr.Client.GetMounts(ctx, c.ID) if err != nil { @@ -703,7 +703,7 @@ func (mgr *ContainerManager) Mount(ctx context.Context, c *Container, online boo rootfs := c.BaseFS - if !online { + if preCreate { if c.MountFS == "" { mgr.setMountFS(ctx, c) } @@ -720,35 +720,39 @@ func (mgr *ContainerManager) Mount(ctx context.Context, c *Container, online boo // Unmount unsets the container rootfs // cleanup decides whether to clean up the dir or not -func (mgr *ContainerManager) Unmount(ctx context.Context, c *Container, online bool, cleanup bool) error { +func (mgr *ContainerManager) Unmount(ctx context.Context, c *Container, preCreate bool, cleanup bool) error { // TODO: if umount is failed, and how to deal it. rootfs := c.MountFS - if online { + if !preCreate { rootfs = c.BaseFS } var err error err = mount.Unmount(rootfs, 0) + if err != nil { - return errors.Wrapf(err, "failed to umount mountfs(%s)", c.MountFS) + return errors.Wrapf(err, "failed to umount rootfs(%s)", rootfs) } if cleanup { - err = os.RemoveAll(rootfs) - if err != nil { - return err + if preCreate { + return os.RemoveAll(rootfs) } + // also need to remove dir named by container ID + cleanPath, _ := filepath.Split(rootfs) + logrus.Debugf("clean path of unmount is %s", cleanPath) + return os.RemoveAll(cleanPath) } return err } func (mgr *ContainerManager) initContainerStorage(ctx context.Context, c *Container) (err error) { - if err = mgr.Mount(ctx, c, false); err != nil { + if err = mgr.Mount(ctx, c, true); err != nil { return errors.Wrapf(err, "failed to mount rootfs(%s)", c.MountFS) } defer func() { - if umountErr := mgr.Unmount(ctx, c, false, true); umountErr != nil { + if umountErr := mgr.Unmount(ctx, c, true, true); umountErr != nil { if err != nil { err = errors.Wrapf(err, "failed to umount rootfs(%s), err(%v)", c.MountFS, umountErr) } else { @@ -815,13 +819,13 @@ func (mgr *ContainerManager) getRootfs(ctx context.Context, c *Container, mounte } rootfs = basefs } else if !mounted { - if err = mgr.Mount(ctx, c, false); err != nil { + if err = mgr.Mount(ctx, c, true); err != nil { return "", errors.Wrapf(err, "failed to mount rootfs: (%s)", c.MountFS) } rootfs = c.MountFS defer func() { - if err = mgr.Unmount(ctx, c, false, true); err != nil { + if err = mgr.Unmount(ctx, c, true, true); err != nil { logrus.Errorf("failed to umount rootfs: (%s), err: (%v)", c.MountFS, err) } }() diff --git a/daemon/mgr/container_types.go b/daemon/mgr/container_types.go index 7523e6be23..114d38c416 100644 --- a/daemon/mgr/container_types.go +++ b/daemon/mgr/container_types.go @@ -166,25 +166,6 @@ type ContainerStatsConfig struct { OutStream io.Writer } -// ContainerPathStat is used to encode the header from -// GET "/containers/{name:.*}/archive" -type ContainerPathStat struct { - // the file or directory's name. - Name string `json:"name"` - - // the path of the file or directory - Path string `json:"path"` - - // the size of the file or directory - Size int64 `json:"size"` - - // FileMode represents a file's mode and permission bits. - Mode os.FileMode `json:"mode"` - - // modify time of the file or directory - Mtime time.Time `json:"mtime"` -} - // Container represents the container's meta data. type Container struct { sync.Mutex diff --git a/pkg/ioutils/reader.go b/pkg/ioutils/reader.go new file mode 100644 index 0000000000..61cf391b47 --- /dev/null +++ b/pkg/ioutils/reader.go @@ -0,0 +1,20 @@ +package ioutils + +import "io" + +type readCloserWrapper struct { + io.Reader + closeFunc func() error +} + +func (r *readCloserWrapper) Close() error { + return r.closeFunc() +} + +// NewReadCloserWrapper provides the ability to handle the cleanup during closer. +func NewReadCloserWrapper(r io.Reader, closeFunc func() error) io.ReadCloser { + return &readCloserWrapper{ + Reader: r, + closeFunc: closeFunc, + } +} diff --git a/test/api_container_cp_test.go b/test/api_container_cp_test.go index 5d2388b6cb..8fd8b34c66 100644 --- a/test/api_container_cp_test.go +++ b/test/api_container_cp_test.go @@ -9,7 +9,7 @@ import ( "net/url" "strings" - "github.com/alibaba/pouch/daemon/mgr" + "github.com/alibaba/pouch/apis/types" "github.com/alibaba/pouch/test/command" "github.com/alibaba/pouch/test/environment" "github.com/alibaba/pouch/test/request" @@ -72,8 +72,8 @@ func (suite *APIContainerCopySuite) TestCopyWorks(c *check.C) { //TODO: add test case copy file to container } -func getContainerPathStatFromHeader(header http.Header) (mgr.ContainerPathStat, error) { - var stat mgr.ContainerPathStat +func getContainerPathStatFromHeader(header http.Header) (types.ContainerPathStat, error) { + var stat types.ContainerPathStat encodedStat := header.Get("X-Docker-Container-Path-Stat") statDecoder := base64.NewDecoder(base64.StdEncoding, strings.NewReader(encodedStat)) diff --git a/test/cli_container_cp_test.go b/test/cli_container_cp_test.go new file mode 100644 index 0000000000..5d479fb3bd --- /dev/null +++ b/test/cli_container_cp_test.go @@ -0,0 +1,131 @@ +package main + +import ( + "fmt" + "os" + + "github.com/alibaba/pouch/test/command" + "github.com/alibaba/pouch/test/environment" + "github.com/alibaba/pouch/test/util" + + "github.com/go-check/check" + "github.com/gotestyourself/gotestyourself/icmd" +) + +// APIContainerCopySuite is the test suite for container cp CLI. +type PouchContainerCopySuite struct{} + +var testDataPath = "testdata/cp" + +func init() { + check.Suite(&PouchContainerCopySuite{}) +} + +// SetUpSuite does common setup in the beginning of each test suite. +func (suite *PouchContainerCopySuite) SetUpSuite(c *check.C) { + SkipIfFalse(c, environment.IsLinux) + PullImage(c, busyboxImage) + err := os.Mkdir(testDataPath, 0755) + c.Assert(err, check.IsNil) +} + +// TearDownSuite does cleanup work in the end of each test suite. +func (suite *PouchContainerCopySuite) TearDownSuite(c *check.C) { + err := os.RemoveAll(testDataPath) + c.Assert(err, check.IsNil) +} + +// Test pouch cp, basic usage +func (suite *PouchContainerCopySuite) TestPouchCopy(c *check.C) { + // test copy from container + name := "TestPouchCopy" + command.PouchRun("run", + "--name", name, + "-d", busyboxImage, + "sh", "-c", + "echo 'test pouch cp' >> data.txt && sleep 10000").Assert(c, icmd.Success) + defer DelContainerForceMultyTime(c, name) + + localTestPath := fmt.Sprintf("%s/%s", testDataPath, "data.txt") + containerTestPath := fmt.Sprintf("%s:%s", name, "test.txt") + + command.PouchRun("cp", fmt.Sprintf("%s:%s", name, "data.txt"), localTestPath).Assert(c, icmd.Success) + checkFileContains(c, localTestPath, "test pouch cp") + + // test copy to container + command.PouchRun("cp", localTestPath, containerTestPath).Assert(c, icmd.Success) + res := command.PouchRun("exec", name, "cat", "test.txt") + res.Assert(c, icmd.Success) + err := util.PartialEqual(res.Stdout(), "test pouch cp") + c.Assert(err, check.IsNil) + + // test copy to container with non-dir + res = command.PouchRun("cp", testDataPath, containerTestPath) + err = util.PartialEqual(res.Combined(), "Error: cannot copy directory") + c.Assert(err, check.IsNil) +} + +// Test pouch cp, where dir locate in volume +func (suite *PouchContainerCopySuite) TestVolumeCopy(c *check.C) { + // test mount rw and copy from container + name := "TestVolumeCopy" + command.PouchRun("volume", "create", "--name", name).Assert(c, icmd.Success) + defer command.PouchRun("volume", "remove", name) + command.PouchRun("run", + "--name", name, + "-d", + "-v", fmt.Sprintf("%s:%s:rw", name, "/test"), + busyboxImage, + "sh", "-c", + "echo 'test pouch cp' >> /test/data.txt && sleep 10000").Assert(c, icmd.Success) + defer DelContainerForceMultyTime(c, name) + + localTestPath := fmt.Sprintf("%s/%s", testDataPath, "data.txt") + containerTestPath := fmt.Sprintf("%s:%s", name, "/test/test.txt") + + command.PouchRun("cp", fmt.Sprintf("%s:%s", name, "/test/data.txt"), localTestPath).Assert(c, icmd.Success) + checkFileContains(c, localTestPath, "test pouch cp") + + // test mount rw and copy to container + command.PouchRun("cp", localTestPath, containerTestPath).Assert(c, icmd.Success) + res := command.PouchRun("exec", name, "cat", "/test/test.txt") + res.Assert(c, icmd.Success) + err := util.PartialEqual(res.Stdout(), "test pouch cp") + c.Assert(err, check.IsNil) + + // test mount only ro + nameRO := "TestVolumeCopyRO" + command.PouchRun("volume", "create", "--name", nameRO).Assert(c, icmd.Success) + defer command.PouchRun("volume", "remove", nameRO) + command.PouchRun("run", + "--name", nameRO, + "-d", + "-v", fmt.Sprintf("%s:%s:ro", nameRO, "/test"), + busyboxImage).Assert(c, icmd.Success) + defer DelContainerForceMultyTime(c, nameRO) + + command.PouchRun("cp", localTestPath, containerTestPath).Assert(c, icmd.Success) + err = util.PartialEqual(res.Stdout(), "can't extract to dir because volume read only") + c.Assert(err, check.NotNil) +} + +// TestStopContainerCopy tests stopped container can work well +func (suite *PouchContainerCopySuite) TestStopContainerCopy(c *check.C) { + name := "TestStopContainerCopy" + command.PouchRun("run", "-d", + "--name", name, + busyboxImage, + "sh", "-c", + "echo 'test pouch cp' >> data.txt && top").Assert(c, icmd.Success) + command.PouchRun("stop", "-t", "1", name).Assert(c, icmd.Success) + defer DelContainerForceMultyTime(c, name) + + // test copy from container + localTestPath := fmt.Sprintf("%s/%s", testDataPath, "TestStopContainerCopy.txt") + containerTestPath := fmt.Sprintf("%s:%s", name, "data.txt") + command.PouchRun("cp", containerTestPath, localTestPath).Assert(c, icmd.Success) + checkFileContains(c, localTestPath, "test pouch cp") + + // test stopped container can start after cp + command.PouchRun("start", name).Assert(c, icmd.Success) +}