From 3c59e535f48c673291f0c1429275e40cd77b2edb Mon Sep 17 00:00:00 2001 From: Bird Date: Thu, 21 Jul 2022 13:20:10 +0800 Subject: [PATCH 1/2] refactor: extract docker shell operator from gitlab-ce-docker Signed-off-by: Bird --- .lycheeignore | 1 + docs/plugins/gitlab-ce-docker.md | 2 +- docs/plugins/gitlab-ce-docker.zh.md | 2 +- internal/pkg/plugin/gitlabcedocker/create.go | 17 ++- internal/pkg/plugin/gitlabcedocker/delete.go | 22 ++-- .../plugin/gitlabcedocker/gitlabcedocker.go | 48 +++----- .../gitlabcedocker_suite_test.go | 2 +- internal/pkg/plugin/gitlabcedocker/options.go | 43 +++++-- .../pkg/plugin/gitlabcedocker/options_test.go | 36 +++++- internal/pkg/plugin/gitlabcedocker/read.go | 19 +-- .../plugin/gitlabcedocker/sshoperator_test.go | 39 ------- internal/pkg/plugin/gitlabcedocker/update.go | 48 +++----- .../show/config/plugins/gitlab-ce-docker.yaml | 2 +- pkg/util/docker/docker.go | 62 ++++++++++ pkg/util/docker/docker_suite_test.go | 13 +++ .../docker/dockersh/dockersh_suite_test.go | 13 +++ .../util/docker/dockersh}/exec.go | 2 +- .../util/docker/dockersh/operator.go | 110 +++++++++++------- pkg/util/docker/dockersh/operator_test.go | 77 ++++++++++++ pkg/util/docker/option.go | 78 +++++++++++++ pkg/util/docker/option_test.go | 59 ++++++++++ 21 files changed, 510 insertions(+), 185 deletions(-) delete mode 100644 internal/pkg/plugin/gitlabcedocker/sshoperator_test.go create mode 100644 pkg/util/docker/docker.go create mode 100644 pkg/util/docker/docker_suite_test.go create mode 100644 pkg/util/docker/dockersh/dockersh_suite_test.go rename {internal/pkg/plugin/gitlabcedocker => pkg/util/docker/dockersh}/exec.go (98%) rename internal/pkg/plugin/gitlabcedocker/sshoperator.go => pkg/util/docker/dockersh/operator.go (54%) create mode 100644 pkg/util/docker/dockersh/operator_test.go create mode 100644 pkg/util/docker/option.go create mode 100644 pkg/util/docker/option_test.go diff --git a/.lycheeignore b/.lycheeignore index d348eb60e..3f022bc18 100644 --- a/.lycheeignore +++ b/.lycheeignore @@ -4,6 +4,7 @@ https://fonts.gstatic.com/ .*foo.* .*bar.* .*xxx.* +.*example.* https://id.atlassian.net https://JIRA_ID.atlassian.net https://jira_id.atlassian.net diff --git a/docs/plugins/gitlab-ce-docker.md b/docs/plugins/gitlab-ce-docker.md index cd6daa7b3..9d4d53ec5 100644 --- a/docs/plugins/gitlab-ce-docker.md +++ b/docs/plugins/gitlab-ce-docker.md @@ -1,6 +1,6 @@ # gitlab-ce-docker plugin -This plugin installs [Gitlab-CE](https://about.gitlab.com/) in an existing docker. +This plugin installs [Gitlab-CE](https://about.gitlab.com/) in an existing docker, and the container name is `gitlab`. ## Usage Note: diff --git a/docs/plugins/gitlab-ce-docker.zh.md b/docs/plugins/gitlab-ce-docker.zh.md index 2e3c69555..edfce1b21 100644 --- a/docs/plugins/gitlab-ce-docker.zh.md +++ b/docs/plugins/gitlab-ce-docker.zh.md @@ -1,6 +1,6 @@ # gitlab-ce-docker 插件 -这个插件用来在本机已存在的 Docker 上安装 [Gitlab-CE](https://about.gitlab.com/) +这个插件用来在本机已存在的 Docker 上安装 [Gitlab-CE](https://about.gitlab.com/), 容器名为 `gitlab`。 ## 使用 注意: diff --git a/internal/pkg/plugin/gitlabcedocker/create.go b/internal/pkg/plugin/gitlabcedocker/create.go index dc4855716..70bff0d4a 100644 --- a/internal/pkg/plugin/gitlabcedocker/create.go +++ b/internal/pkg/plugin/gitlabcedocker/create.go @@ -7,6 +7,7 @@ import ( "github.com/mitchellh/mapstructure" + "github.com/devstream-io/devstream/pkg/util/docker" "github.com/devstream-io/devstream/pkg/util/log" ) @@ -16,6 +17,8 @@ func Create(options map[string]interface{}) (map[string]interface{}, error) { return nil, err } + defaults(&opts) + if errs := validate(&opts); len(errs) != 0 { for _, e := range errs { log.Errorf("Options error: %s.", e) @@ -23,30 +26,34 @@ func Create(options map[string]interface{}) (map[string]interface{}, error) { return nil, fmt.Errorf("opts are illegal") } - op := getDockerOperator(opts) + op := GetDockerOperator(opts) // 1. try to pull the image // always pull the image because docker will check the image existence - if err := op.PullImage(getImageNameWithTag(opts)); err != nil { + if err := op.ImagePull(getImageNameWithTag(opts)); err != nil { return nil, err } // 2. try to run the container log.Info("Running container as the name ") - if err := op.RunContainer(opts); err != nil { + if err := op.ContainerRun(buildDockerRunOptions(opts), dockerRunShmSizeParam); err != nil { return nil, fmt.Errorf("failed to run container: %v", err) } // 3. check if the container is started successfully - if ok := op.IfContainerRunning(gitlabContainerName); !ok { + if ok := op.ContainerIfRunning(gitlabContainerName); !ok { return nil, fmt.Errorf("failed to run container") } // 4. check if the volume is created successfully - volumes, err := op.ListContainerMounts(gitlabContainerName) + mounts, err := op.ContainerListMounts(gitlabContainerName) if err != nil { return nil, fmt.Errorf("failed to get container mounts: %v", err) } + volumes := mounts.ExtractSources() + if docker.IfVolumesDiffer(volumes, getVolumesDirFromOptions(opts)) { + return nil, fmt.Errorf("failed to create volumes") + } // 5. show the access url showGitLabURL(opts) diff --git a/internal/pkg/plugin/gitlabcedocker/delete.go b/internal/pkg/plugin/gitlabcedocker/delete.go index 8890a88c4..81b68d0d8 100644 --- a/internal/pkg/plugin/gitlabcedocker/delete.go +++ b/internal/pkg/plugin/gitlabcedocker/delete.go @@ -15,6 +15,8 @@ func Delete(options map[string]interface{}) (bool, error) { return false, err } + defaults(&opts) + if errs := validate(&opts); len(errs) != 0 { for _, e := range errs { log.Errorf("Options error: %s.", e) @@ -22,25 +24,25 @@ func Delete(options map[string]interface{}) (bool, error) { return false, fmt.Errorf("opts are illegal") } - op := getDockerOperator(opts) + op := GetDockerOperator(opts) // 1. stop the container if it is running - if ok := op.IfContainerRunning(gitlabContainerName); ok { - if err := op.StopContainer(gitlabContainerName); err != nil { + if ok := op.ContainerIfRunning(gitlabContainerName); ok { + if err := op.ContainerStop(gitlabContainerName); err != nil { log.Errorf("Failed to stop container: %v", err) } } // 2. remove the container if it exists - if ok := op.IfContainerExists(gitlabContainerName); ok { - if err := op.RemoveContainer(gitlabContainerName); err != nil { + if ok := op.ContainerIfExist(gitlabContainerName); ok { + if err := op.ContainerRemove(gitlabContainerName); err != nil { log.Errorf("failed to remove container %v: %v", gitlabContainerName, err) } } // 3. remove the image if it exists - if ok := op.IfImageExists(getImageNameWithTag(opts)); ok { - if err := op.RemoveImage(getImageNameWithTag(opts)); err != nil { + if ok := op.ImageIfExist(getImageNameWithTag(opts)); ok { + if err := op.ImageRemove(getImageNameWithTag(opts)); err != nil { log.Errorf("failed to remove image %v: %v", getImageNameWithTag(opts), err) } } @@ -58,15 +60,15 @@ func Delete(options map[string]interface{}) (bool, error) { var errs []error // 1. check if the container is stopped and deleted - if ok := op.IfContainerRunning(gitlabContainerName); ok { + if ok := op.ContainerIfRunning(gitlabContainerName); ok { errs = append(errs, fmt.Errorf("failed to delete/stop container %s", gitlabContainerName)) } - if ok := op.IfContainerExists(gitlabContainerName); ok { + if ok := op.ContainerIfExist(gitlabContainerName); ok { errs = append(errs, fmt.Errorf("failed to delete container %s", gitlabContainerName)) } // 2. check if the image is removed - if ok := op.IfImageExists(getImageNameWithTag(opts)); ok { + if ok := op.ImageIfExist(getImageNameWithTag(opts)); ok { errs = append(errs, fmt.Errorf("failed to delete image %s", getImageNameWithTag(opts))) } diff --git a/internal/pkg/plugin/gitlabcedocker/gitlabcedocker.go b/internal/pkg/plugin/gitlabcedocker/gitlabcedocker.go index dac01e47b..2183eb1be 100644 --- a/internal/pkg/plugin/gitlabcedocker/gitlabcedocker.go +++ b/internal/pkg/plugin/gitlabcedocker/gitlabcedocker.go @@ -2,48 +2,32 @@ package gitlabcedocker import ( "strings" + + "github.com/devstream-io/devstream/pkg/util/docker" + "github.com/devstream-io/devstream/pkg/util/docker/dockersh" ) const ( - gitlabImageName = "gitlab/gitlab-ce" - defaultImageTag = "rc" - gitlabContainerName = "gitlab" - tcp = "tcp" + gitlabImageName = "gitlab/gitlab-ce" + defaultImageTag = "rc" + gitlabContainerName = "gitlab" + tcp = "tcp" + dockerRunShmSizeParam = "--shm-size 256m" ) func getImageNameWithTag(opt Options) string { - var imageTag string - if opt.ImageTag == "" { - imageTag = defaultImageTag - } else { - imageTag = opt.ImageTag - } - return gitlabImageName + ":" + imageTag + return gitlabImageName + ":" + opt.ImageTag } -// dockerOperator is an interface for docker operations -// It is implemented by sshDockerOperator -// in the future, we can add other implementations such as sshDockerOperator -type dockerOperator interface { - IfImageExists(imageName string) bool - PullImage(image string) error - RemoveImage(image string) error - - IfContainerExists(container string) bool - IfContainerRunning(container string) bool - RunContainer(options Options) error - StopContainer(container string) error - RemoveContainer(container string) error - - ListContainerMounts(container string) ([]string, error) - - GetContainerHostname(container string) (string, error) - GetContainerPortBinding(container, containerPort, protocol string) (hostPort string, err error) +func defaults(opts *Options) { + if opts.ImageTag == "" { + opts.ImageTag = defaultImageTag + } } -func getDockerOperator(_ Options) dockerOperator { - // just return a sshDockerOperator for now - return &sshDockerOperator{} +func GetDockerOperator(_ Options) docker.Operator { + // just return a ShellOperator for now + return &dockersh.ShellOperator{} } type gitlabResource struct { diff --git a/internal/pkg/plugin/gitlabcedocker/gitlabcedocker_suite_test.go b/internal/pkg/plugin/gitlabcedocker/gitlabcedocker_suite_test.go index 7bb80709b..42ca05d6b 100644 --- a/internal/pkg/plugin/gitlabcedocker/gitlabcedocker_suite_test.go +++ b/internal/pkg/plugin/gitlabcedocker/gitlabcedocker_suite_test.go @@ -1,4 +1,4 @@ -package gitlabcedocker_test +package gitlabcedocker import ( "testing" diff --git a/internal/pkg/plugin/gitlabcedocker/options.go b/internal/pkg/plugin/gitlabcedocker/options.go index 50a2ea516..ce3a5f9a7 100644 --- a/internal/pkg/plugin/gitlabcedocker/options.go +++ b/internal/pkg/plugin/gitlabcedocker/options.go @@ -1,6 +1,10 @@ package gitlabcedocker -import "path/filepath" +import ( + "path/filepath" + + "github.com/devstream-io/devstream/pkg/util/docker" +) // Options is the struct for configurations of the gitlab-ce-docker plugin. type Options struct { @@ -14,13 +18,38 @@ type Options struct { ImageTag string `mapstructure:"image_tag"` } -// getVolumesDirFromOptions returns the volumes' directory from the options. +// getVolumesDirFromOptions returns host directories of the volumes from the options. func getVolumesDirFromOptions(opts Options) []string { - volumesDirFromOptions := []string{ - filepath.Join(opts.GitLabHome, "config"), - filepath.Join(opts.GitLabHome, "data"), - filepath.Join(opts.GitLabHome, "logs"), + volumes := buildDockerVolumes(opts) + return volumes.ExtractHostPaths() +} + +func buildDockerVolumes(opts Options) docker.Volumes { + volumes := []docker.Volume{ + {HostPath: filepath.Join(opts.GitLabHome, "config"), ContainerPath: "/etc/gitlab"}, + {HostPath: filepath.Join(opts.GitLabHome, "data"), ContainerPath: "/var/opt/gitlab"}, + {HostPath: filepath.Join(opts.GitLabHome, "logs"), ContainerPath: "/var/log/gitlab"}, } - return volumesDirFromOptions + return volumes +} + +func buildDockerRunOptions(opts Options) docker.RunOptions { + dockerRunOpts := docker.RunOptions{} + dockerRunOpts.ImageName = gitlabImageName + dockerRunOpts.ImageTag = opts.ImageTag + dockerRunOpts.Hostname = opts.Hostname + dockerRunOpts.ContainerName = gitlabContainerName + dockerRunOpts.RestartAlways = true + + portPublishes := []docker.PortPublish{ + {HostPort: opts.SSHPort, ContainerPort: 22}, + {HostPort: opts.HTTPPort, ContainerPort: 80}, + {HostPort: opts.HTTPSPort, ContainerPort: 443}, + } + dockerRunOpts.PortPublishes = portPublishes + + dockerRunOpts.Volumes = buildDockerVolumes(opts) + + return dockerRunOpts } diff --git a/internal/pkg/plugin/gitlabcedocker/options_test.go b/internal/pkg/plugin/gitlabcedocker/options_test.go index 786c0c29d..1e552641f 100644 --- a/internal/pkg/plugin/gitlabcedocker/options_test.go +++ b/internal/pkg/plugin/gitlabcedocker/options_test.go @@ -3,6 +3,8 @@ package gitlabcedocker import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + + "github.com/devstream-io/devstream/pkg/util/docker" ) var _ = Describe("Options", func() { @@ -13,16 +15,16 @@ var _ = Describe("Options", func() { opts = Options{ GitLabHome: "/srv/gitlab", Hostname: "gitlab.example.com", - SSHPort: 22, - HTTPPort: 80, - HTTPSPort: 443, + SSHPort: 8122, + HTTPPort: 8180, + HTTPSPort: 8443, RmDataAfterDelete: false, ImageTag: "rc", } }) Describe("getVolumesDirFromOptions func", func() { - Context("when the options is valid", func() { + When("the options is valid", func() { It("should return the volumes' directory", func() { volumesDirFromOptions := getVolumesDirFromOptions(opts) Expect(volumesDirFromOptions).To(Equal([]string{ @@ -33,4 +35,30 @@ var _ = Describe("Options", func() { }) }) }) + + Describe("buildDockerRunOptions func", func() { + It("should build the docker run options successfully", func() { + runOptsBuild := buildDockerRunOptions(opts) + runOptsExpect := docker.RunOptions{ + ImageName: "gitlab/gitlab-ce", + ImageTag: "rc", + Hostname: "gitlab.example.com", + ContainerName: "gitlab", + RestartAlways: true, + PortPublishes: []docker.PortPublish{ + {HostPort: 8122, ContainerPort: 22}, + {HostPort: 8180, ContainerPort: 80}, + {HostPort: 8443, ContainerPort: 443}, + }, + Volumes: []docker.Volume{ + {HostPath: "/srv/gitlab/config", ContainerPath: "/etc/gitlab"}, + {HostPath: "/srv/gitlab/data", ContainerPath: "/var/opt/gitlab"}, + {HostPath: "/srv/gitlab/logs", ContainerPath: "/var/log/gitlab"}, + }, + } + + Expect(runOptsBuild).To(Equal(runOptsExpect)) + }) + + }) }) diff --git a/internal/pkg/plugin/gitlabcedocker/read.go b/internal/pkg/plugin/gitlabcedocker/read.go index 493addcff..274ec988a 100644 --- a/internal/pkg/plugin/gitlabcedocker/read.go +++ b/internal/pkg/plugin/gitlabcedocker/read.go @@ -14,6 +14,8 @@ func Read(options map[string]interface{}) (map[string]interface{}, error) { return nil, err } + defaults(&opts) + if errs := validate(&opts); len(errs) != 0 { for _, e := range errs { log.Errorf("Options error: %s.", e) @@ -21,16 +23,16 @@ func Read(options map[string]interface{}) (map[string]interface{}, error) { return nil, fmt.Errorf("opts are illegal") } - op := getDockerOperator(opts) + op := GetDockerOperator(opts) // 1. get running status - running := op.IfContainerRunning(gitlabContainerName) + running := op.ContainerIfRunning(gitlabContainerName) if !running { return (&gitlabResource{}).toMap(), nil } - // 2. get volumes(gitlab_home) - volumes, err := op.ListContainerMounts(gitlabContainerName) + // 2. get volumes + mounts, err := op.ContainerListMounts(gitlabContainerName) if err != nil { // `Read` shouldn't return errors even if failed to read ports, volumes, hostname. // because: @@ -38,23 +40,24 @@ func Read(options map[string]interface{}) (map[string]interface{}, error) { // 2. if Read failed, the following steps contain the docker's restart will be aborted. log.Errorf("failed to get container mounts: %v", err) } + volumes := mounts.ExtractSources() // 3. get hostname - hostname, err := op.GetContainerHostname(gitlabContainerName) + hostname, err := op.ContainerGetHostname(gitlabContainerName) if err != nil { log.Errorf("failed to get container hostname: %v", err) } // 4. get port bindings - SSHPort, err := op.GetContainerPortBinding(gitlabContainerName, "22", tcp) + SSHPort, err := op.ContainerGetPortBinding(gitlabContainerName, "22", tcp) if err != nil { log.Errorf("failed to get container ssh port: %v", err) } - HTTPPort, err := op.GetContainerPortBinding(gitlabContainerName, "80", tcp) + HTTPPort, err := op.ContainerGetPortBinding(gitlabContainerName, "80", tcp) if err != nil { log.Errorf("failed to get container http port: %v", err) } - HTTPSPort, err := op.GetContainerPortBinding(gitlabContainerName, "443", tcp) + HTTPSPort, err := op.ContainerGetPortBinding(gitlabContainerName, "443", tcp) if err != nil { log.Errorf("failed to get container https port: %v", err) } diff --git a/internal/pkg/plugin/gitlabcedocker/sshoperator_test.go b/internal/pkg/plugin/gitlabcedocker/sshoperator_test.go deleted file mode 100644 index 69df42fae..000000000 --- a/internal/pkg/plugin/gitlabcedocker/sshoperator_test.go +++ /dev/null @@ -1,39 +0,0 @@ -package gitlabcedocker_test - -import ( - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - "github.com/devstream-io/devstream/internal/pkg/plugin/gitlabcedocker" -) - -var _ = Describe("SSH operator", func() { - var options gitlabcedocker.Options - - BeforeEach(func() { - options = gitlabcedocker.Options{ - Hostname: "gitlab.devstream.io", - GitLabHome: "/srv/gitlab", - SSHPort: 8122, - HTTPPort: 8180, - HTTPSPort: 8443, - } - }) - - It("docker run command should be built correctly", func() { - cmd := gitlabcedocker.BuildDockerRunCommand(options) - expect := ` - docker run --detach \ - --hostname gitlab.devstream.io \ - --publish 8443:443 --publish 8180:80 --publish 8122:22 \ - --name gitlab \ - --restart always \ - --volume /srv/gitlab/config:/etc/gitlab \ - --volume /srv/gitlab/logs:/var/log/gitlab \ - --volume /srv/gitlab/data:/var/opt/gitlab \ - --shm-size 256m \ - gitlab/gitlab-ce:rc - ` - Expect(cmd).To(Equal(expect)) - }) -}) diff --git a/internal/pkg/plugin/gitlabcedocker/update.go b/internal/pkg/plugin/gitlabcedocker/update.go index b6fafec6e..97a0eaa0e 100644 --- a/internal/pkg/plugin/gitlabcedocker/update.go +++ b/internal/pkg/plugin/gitlabcedocker/update.go @@ -7,6 +7,7 @@ import ( "github.com/mitchellh/mapstructure" + "github.com/devstream-io/devstream/pkg/util/docker" "github.com/devstream-io/devstream/pkg/util/log" ) @@ -16,6 +17,8 @@ func Update(options map[string]interface{}) (map[string]interface{}, error) { return nil, err } + defaults(&opts) + if errs := validate(&opts); len(errs) != 0 { for _, e := range errs { log.Errorf("Options error: %s.", e) @@ -23,44 +26,45 @@ func Update(options map[string]interface{}) (map[string]interface{}, error) { return nil, fmt.Errorf("opts are illegal") } - op := getDockerOperator(opts) + op := GetDockerOperator(opts) // 0. check if the volumes are the same - volumesDockerNow, err := op.ListContainerMounts(gitlabContainerName) + mounts, err := op.ContainerListMounts(gitlabContainerName) if err != nil { return nil, fmt.Errorf("failed to get container mounts: %v", err) } + volumesFromRunningContainer := mounts.ExtractSources() volumesDirFromOptions := getVolumesDirFromOptions(opts) - if ifVolumesDiffer(volumesDockerNow, volumesDirFromOptions) { + if docker.IfVolumesDiffer(volumesFromRunningContainer, volumesDirFromOptions) { log.Warnf("You changed volumes of the container or change the gitlab home directory") - log.Infof("Your volumes of the current container were: %v", strings.Join(volumesDockerNow, " ")) + log.Infof("Your volumes of the current container were: %v", strings.Join(volumesFromRunningContainer, " ")) return nil, fmt.Errorf("sorry, you can't change the gitlab_home of the container once it's already been created") } // 1. stop the container - if ok := op.IfContainerRunning(gitlabContainerName); ok { - if err := op.StopContainer(gitlabContainerName); err != nil { + if ok := op.ContainerIfRunning(gitlabContainerName); ok { + if err := op.ContainerStop(gitlabContainerName); err != nil { log.Warnf("Failed to stop container: %v", err) } } // 2. remove the container if it exists - if exists := op.IfContainerExists(gitlabContainerName); exists { - if err := op.RemoveContainer(gitlabContainerName); err != nil { + if exists := op.ContainerIfExist(gitlabContainerName); exists { + if err := op.ContainerRemove(gitlabContainerName); err != nil { return nil, fmt.Errorf("failed to remove container: %v", err) } } // 3. run the container with the new options - if err := op.RunContainer(opts); err != nil { + if err := op.ContainerRun(buildDockerRunOptions(opts), dockerRunShmSizeParam); err != nil { return nil, fmt.Errorf("failed to run container: %v", err) } resource := gitlabResource{ ContainerRunning: true, - Volumes: volumesDockerNow, + Volumes: volumesFromRunningContainer, Hostname: opts.Hostname, SSHPort: strconv.Itoa(int(opts.SSHPort)), HTTPPort: strconv.Itoa(int(opts.HTTPPort)), @@ -69,27 +73,3 @@ func Update(options map[string]interface{}) (map[string]interface{}, error) { return resource.toMap(), nil } - -func ifVolumesDiffer(volumesBefore, volumesCurrent []string) bool { - if len(volumesBefore) != len(volumesCurrent) { - return true - } - - beforeMap := make(map[string]struct{}) - currentMap := make(map[string]struct{}) - - for _, v := range volumesBefore { - beforeMap[v] = struct{}{} - } - for _, v := range volumesCurrent { - currentMap[v] = struct{}{} - } - - for _, v := range volumesBefore { - if _, ok := currentMap[v]; !ok { - return true - } - } - - return false -} diff --git a/internal/pkg/show/config/plugins/gitlab-ce-docker.yaml b/internal/pkg/show/config/plugins/gitlab-ce-docker.yaml index 37ae31d07..9dba0d2e0 100644 --- a/internal/pkg/show/config/plugins/gitlab-ce-docker.yaml +++ b/internal/pkg/show/config/plugins/gitlab-ce-docker.yaml @@ -8,7 +8,7 @@ tools: # options for the plugin options: # hostname for running docker - hostname: gitlab.devstream.io + hostname: gitlab.example.com # the directory where you store docker volumes of gitlab # 1. it should be a absolute path # 2. once the tool is applied, it can't be changed diff --git a/pkg/util/docker/docker.go b/pkg/util/docker/docker.go new file mode 100644 index 000000000..7ba9635b3 --- /dev/null +++ b/pkg/util/docker/docker.go @@ -0,0 +1,62 @@ +package docker + +import ( + "sort" + + mapset "github.com/deckarep/golang-set/v2" +) + +// Operator is an interface for docker operations +// It is implemented by shDockerOperator +// in the future, we can add other implementations such as SdkDockerOperator +type Operator interface { + ImageIfExist(imageNameWithTag string) bool + ImagePull(imageNameWithTag string) error + ImageRemove(imageNameWithTag string) error + + ContainerIfExist(containerName string) bool + ContainerIfRunning(containerName string) bool + // ContainerRun runs a container with the given options + // params is a list of additional parameters for docker run + // params will be appended to the end of the command + ContainerRun(opts RunOptions, params ...string) error + ContainerStop(containerName string) error + ContainerRemove(containerName string) error + + // ContainerListMounts lists container mounts + ContainerListMounts(containerName string) (Mounts, error) + + ContainerGetHostname(containerName string) (string, error) + ContainerGetPortBinding(containerName, containerPort, protocol string) (hostPort string, err error) +} + +type MountPoint struct { + Type string `json:"Type"` + Source string `json:"Source"` + Destination string `json:"Destination"` + Mode string `json:"Mode"` + Rw bool `json:"RW"` + Propagation string `json:"Propagation"` +} + +type Mounts []MountPoint + +// ExtractSources returns a list of sources for the given mounts +func (ms Mounts) ExtractSources() []string { + sources := make([]string, 0) + for _, mount := range ms { + sources = append(sources, mount.Source) + } + sort.Slice(sources, func(i, j int) bool { + return sources[i] < sources[j] + }) + + return sources +} + +func IfVolumesDiffer(volumesBefore, volumesCurrent []string) bool { + beforeSet := mapset.NewSet[string](volumesBefore...) + currentSet := mapset.NewSet[string](volumesCurrent...) + + return !beforeSet.Equal(currentSet) +} diff --git a/pkg/util/docker/docker_suite_test.go b/pkg/util/docker/docker_suite_test.go new file mode 100644 index 000000000..aab992003 --- /dev/null +++ b/pkg/util/docker/docker_suite_test.go @@ -0,0 +1,13 @@ +package docker + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestDocker(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Docker Suite") +} diff --git a/pkg/util/docker/dockersh/dockersh_suite_test.go b/pkg/util/docker/dockersh/dockersh_suite_test.go new file mode 100644 index 000000000..e07468de5 --- /dev/null +++ b/pkg/util/docker/dockersh/dockersh_suite_test.go @@ -0,0 +1,13 @@ +package dockersh + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestDockersh(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Dockersh Suite") +} diff --git a/internal/pkg/plugin/gitlabcedocker/exec.go b/pkg/util/docker/dockersh/exec.go similarity index 98% rename from internal/pkg/plugin/gitlabcedocker/exec.go rename to pkg/util/docker/dockersh/exec.go index 64d947a8a..84ffdab02 100644 --- a/internal/pkg/plugin/gitlabcedocker/exec.go +++ b/pkg/util/docker/dockersh/exec.go @@ -1,4 +1,4 @@ -package gitlabcedocker +package dockersh import ( "bufio" diff --git a/internal/pkg/plugin/gitlabcedocker/sshoperator.go b/pkg/util/docker/dockersh/operator.go similarity index 54% rename from internal/pkg/plugin/gitlabcedocker/sshoperator.go rename to pkg/util/docker/dockersh/operator.go index 4276a5395..b0e3d3522 100644 --- a/internal/pkg/plugin/gitlabcedocker/sshoperator.go +++ b/pkg/util/docker/dockersh/operator.go @@ -1,18 +1,23 @@ -package gitlabcedocker +package dockersh import ( "bytes" + "encoding/json" "fmt" "os/exec" - "sort" "strings" + "github.com/devstream-io/devstream/pkg/util/docker" "github.com/devstream-io/devstream/pkg/util/log" ) -type sshDockerOperator struct{} +// ShellOperator is an implementation of /pkg/util/docker.Operator interface by using shell commands +type ShellOperator struct{} -func (op *sshDockerOperator) IfImageExists(imageNameWithTag string) bool { +// TODO(aFlyBird0): maybe use one param "ImageNameWithTag" is not a good idea, +// because we have to extract the bare image name from the image name with tag +// we could use (a struct)/(a interface)/(two params) to represent the image name and tag +func (op *ShellOperator) ImageIfExist(imageNameWithTag string) bool { // eg. docker image ls gitlab/gitlab-ce:rc cmdString := fmt.Sprintf("docker image ls %v", imageNameWithTag) // output: eg. @@ -36,13 +41,13 @@ func extractImageName(imageNameWithTag string) string { return strings.Split(imageNameWithTag, ":")[0] } -func (op *sshDockerOperator) PullImage(imageName string) error { +func (op *ShellOperator) ImagePull(imageName string) error { err := ExecInSystemWithParams(".", []string{"docker", "pull", imageName}, nil, true) return err } -func (op *sshDockerOperator) RemoveImage(imageName string) error { +func (op *ShellOperator) ImageRemove(imageName string) error { log.Infof("Removing image %v ...", imageName) cmdString := fmt.Sprintf("docker rmi %s", imageName) @@ -51,7 +56,7 @@ func (op *sshDockerOperator) RemoveImage(imageName string) error { return err } -func (op *sshDockerOperator) IfContainerExists(containerName string) bool { +func (op *ShellOperator) ContainerIfExist(containerName string) bool { cmdString := fmt.Sprintf("docker inspect %s", containerName) outputBuffer := &bytes.Buffer{} err := ExecInSystem(".", cmdString, outputBuffer, false) @@ -66,7 +71,7 @@ func (op *sshDockerOperator) IfContainerExists(containerName string) bool { return true } -func (op *sshDockerOperator) IfContainerRunning(containerName string) bool { +func (op *ShellOperator) ContainerIfRunning(containerName string) bool { command := exec.Command("docker", "inspect", "--format='{{.State.Status}}'", containerName) output, err := command.Output() if err != nil { @@ -80,11 +85,16 @@ func (op *sshDockerOperator) IfContainerRunning(containerName string) bool { return false } -func (op *sshDockerOperator) RunContainer(options Options) error { - cmdString := BuildDockerRunCommand(options) +func (op *ShellOperator) ContainerRun(opts docker.RunOptions, params ...string) error { + // build the command + cmdString, err := BuildContainerRunCommand(opts, params...) + if err != nil { + return err + } log.Debugf("Docker run command: %s", cmdString) - cmdStringOneline := strings.Replace(cmdString, "\\\n", " ", -1) - err := ExecInSystem(".", cmdStringOneline, nil, true) + + // run the command + err = ExecInSystem(".", cmdString, nil, true) if err != nil { return fmt.Errorf("docker run failed: %v", err) } @@ -92,25 +102,36 @@ func (op *sshDockerOperator) RunContainer(options Options) error { return nil } -func BuildDockerRunCommand(options Options) string { - cmdTemplate := ` - docker run --detach \ - --hostname %s \ - --publish %d:443 --publish %d:80 --publish %d:22 \ - --name %s \ - --restart always \ - --volume %[6]s/config:/etc/gitlab \ - --volume %[6]s/logs:/var/log/gitlab \ - --volume %[6]s/data:/var/opt/gitlab \ - --shm-size 256m \ - %s - ` - cmdString := fmt.Sprintf(cmdTemplate, options.Hostname, options.HTTPSPort, - options.HTTPPort, options.SSHPort, gitlabContainerName, options.GitLabHome, getImageNameWithTag(options)) - return cmdString +// BuildContainerRunCommand builds the docker run command string from the given options and additional params +func BuildContainerRunCommand(opts docker.RunOptions, params ...string) (string, error) { + if err := opts.Validate(); err != nil { + return "", err + } + + cmdBuilder := strings.Builder{} + cmdBuilder.WriteString("docker run --detach ") + if opts.Hostname != "" { + cmdBuilder.WriteString(fmt.Sprintf("--hostname %s ", opts.Hostname)) + } + for _, publish := range opts.PortPublishes { + cmdBuilder.WriteString(fmt.Sprintf("--publish %d:%d ", publish.HostPort, publish.ContainerPort)) + } + cmdBuilder.WriteString(fmt.Sprintf("--name %s ", opts.ContainerName)) + if opts.RestartAlways { + cmdBuilder.WriteString("--restart always ") + } + for _, volume := range opts.Volumes { + cmdBuilder.WriteString(fmt.Sprintf("--volume %s:%s ", volume.HostPath, volume.ContainerPath)) + } + for _, param := range params { + cmdBuilder.WriteString(param + " ") + } + cmdBuilder.WriteString(docker.CombineImageNameAndTag(opts.ImageName, opts.ImageTag)) + + return cmdBuilder.String(), nil } -func (op *sshDockerOperator) StopContainer(containerName string) error { +func (op *ShellOperator) ContainerStop(containerName string) error { log.Infof("Stopping container %v ...", containerName) cmdString := fmt.Sprintf("docker stop %s", containerName) @@ -119,7 +140,7 @@ func (op *sshDockerOperator) StopContainer(containerName string) error { return err } -func (op *sshDockerOperator) RemoveContainer(containerName string) error { +func (op *ShellOperator) ContainerRemove(containerName string) error { log.Infof("Removing container %v ...", containerName) cmdString := fmt.Sprintf("docker rm %s", containerName) @@ -128,8 +149,15 @@ func (op *sshDockerOperator) RemoveContainer(containerName string) error { return err } -func (op *sshDockerOperator) ListContainerMounts(containerName string) ([]string, error) { - cmdString := fmt.Sprintf(`docker inspect --format='{{range .Mounts}}{{.Source}}{{"\n"}}{{end}}' %s`, containerName) +func (op *ShellOperator) ContainerListMounts(containerName string) (docker.Mounts, error) { + // result: the list of mounts, format: "SourcePath:DestinationPath" + // e.g. + // /srv/gitlab/config:/etc/gitlab + // /srv/gitlab/logs:/var/log/gitlab + // /srv/gitlab/data:/var/opt/gitlab + + cmdString := fmt.Sprintf(`docker inspect --format='{{json .Mounts}}' %s`, containerName) + outputBuffer := &bytes.Buffer{} err := ExecInSystem(".", cmdString, outputBuffer, false) @@ -137,18 +165,18 @@ func (op *sshDockerOperator) ListContainerMounts(containerName string) ([]string return nil, err } - volumes := strings.Split(strings.TrimSpace(outputBuffer.String()), "\n") - - sort.Slice(volumes, func(i, j int) bool { - return volumes[i] < volumes[j] - }) + mounts := make([]docker.MountPoint, 0) + err = json.Unmarshal([]byte(strings.TrimSpace(outputBuffer.String())), &mounts) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal docker inspect output when list mounts: %v", err) + } - log.Debugf("Container %v volumes: %v", containerName, volumes) + log.Debugf("Container %v mounts: %v", containerName, mounts) - return volumes, nil + return mounts, nil } -func (op *sshDockerOperator) GetContainerHostname(container string) (string, error) { +func (op *ShellOperator) ContainerGetHostname(container string) (string, error) { cmdString := fmt.Sprintf("docker inspect --format='{{.Config.Hostname}}' %s", container) outputBuffer := &bytes.Buffer{} @@ -161,7 +189,7 @@ func (op *sshDockerOperator) GetContainerHostname(container string) (string, err } -func (op *sshDockerOperator) GetContainerPortBinding(container, containerPort, protocol string) (hostPort string, err error) { +func (op *ShellOperator) ContainerGetPortBinding(container, containerPort, protocol string) (hostPort string, err error) { // get container port binding map // the result is like: diff --git a/pkg/util/docker/dockersh/operator_test.go b/pkg/util/docker/dockersh/operator_test.go new file mode 100644 index 000000000..b6df3ae06 --- /dev/null +++ b/pkg/util/docker/dockersh/operator_test.go @@ -0,0 +1,77 @@ +package dockersh + +import ( + "path/filepath" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/devstream-io/devstream/pkg/util/docker" +) + +var _ = Describe("Operator", func() { + Describe("BuildContainerRunCommand method", func() { + var ( + opts docker.RunOptions + params []string + ) + + When(" the options are invalid", func() { + BeforeEach(func() { + opts = docker.RunOptions{} + }) + + It("should return an error", func() { + _, err := BuildContainerRunCommand(opts, params...) + Expect(err).To(HaveOccurred()) + }) + }) + + When(" the options are valid(e.g. gitlab-ce)", func() { + BeforeEach(func() { + buildOpts := func() docker.RunOptions { + opts := docker.RunOptions{} + opts.ImageName = "gitlab/gitlab-ce" + opts.ImageTag = "rc" + opts.Hostname = "gitlab.example.com" + opts.ContainerName = "gitlab" + opts.RestartAlways = true + + portPublishes := []docker.PortPublish{ + {HostPort: 8122, ContainerPort: 22}, + {HostPort: 8180, ContainerPort: 80}, + {HostPort: 8443, ContainerPort: 443}, + } + opts.PortPublishes = portPublishes + + gitLabHome := "/srv/gitlab" + + opts.Volumes = []docker.Volume{ + {HostPath: filepath.Join(gitLabHome, "config"), ContainerPath: "/etc/gitlab"}, + {HostPath: filepath.Join(gitLabHome, "data"), ContainerPath: "/var/opt/gitlab"}, + {HostPath: filepath.Join(gitLabHome, "logs"), ContainerPath: "/var/log/gitlab"}, + } + + return opts + } + + opts = buildOpts() + params = []string{"--shm-size 256m"} + }) + + It("should return the correct command", func() { + cmdBuild, err := BuildContainerRunCommand(opts, params...) + Expect(err).NotTo(HaveOccurred()) + cmdExpect := "docker run --detach --hostname gitlab.example.com" + + " --publish 8122:22 --publish 8180:80 --publish 8443:443" + + " --name gitlab --restart always" + + " --volume /srv/gitlab/config:/etc/gitlab" + + " --volume /srv/gitlab/data:/var/opt/gitlab" + + " --volume /srv/gitlab/logs:/var/log/gitlab" + + " --shm-size 256m gitlab/gitlab-ce:rc" + Expect(cmdBuild).To(Equal(cmdExpect)) + }) + + }) + }) +}) diff --git a/pkg/util/docker/option.go b/pkg/util/docker/option.go new file mode 100644 index 000000000..3e7377e6b --- /dev/null +++ b/pkg/util/docker/option.go @@ -0,0 +1,78 @@ +package docker + +import ( + "fmt" + "strings" +) + +// RunOptions is used to pass options to ContainerRunWithOptions +type ( + RunOptions struct { + ImageName string + ImageTag string + Hostname string + ContainerName string + PortPublishes []PortPublish + Volumes []Volume + RestartAlways bool + } + + Volume struct { + HostPath string + ContainerPath string + } + Volumes []Volume + + PortPublish struct { + HostPort uint + ContainerPort uint + } +) + +func (opts *RunOptions) Validate() error { + var errs []error + if strings.TrimSpace(opts.ImageName) == "" { + errs = append(errs, fmt.Errorf("image name is required")) + } + if strings.TrimSpace(opts.ImageTag) == "" { + errs = append(errs, fmt.Errorf("image tag is required")) + } + if strings.TrimSpace(opts.ContainerName) == "" { + errs = append(errs, fmt.Errorf("container name is required")) + } + for _, volume := range opts.Volumes { + if volume.HostPath == "" { + errs = append(errs, fmt.Errorf("HostPath can not be empty")) + } + if volume.ContainerPath == "" { + errs = append(errs, fmt.Errorf("ContainerPath can not be empty")) + } + } + + return CombineErrs(errs) +} + +func CombineImageNameAndTag(imageName, tag string) string { + return imageName + ":" + tag +} + +func CombineErrs(errs []error) error { + if len(errs) == 0 { + return nil + } + + errsString := make([]string, len(errs)) + for _, err := range errs { + errsString = append(errsString, err.Error()) + } + + return fmt.Errorf(strings.Join(errsString, ";")) +} + +func (volumes Volumes) ExtractHostPaths() []string { + hostPaths := make([]string, len(volumes)) + for i, volume := range volumes { + hostPaths[i] = volume.HostPath + } + return hostPaths +} diff --git a/pkg/util/docker/option_test.go b/pkg/util/docker/option_test.go new file mode 100644 index 000000000..c9fc5f43d --- /dev/null +++ b/pkg/util/docker/option_test.go @@ -0,0 +1,59 @@ +package docker + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Option", func() { + Describe("IfVolumesDiffer func", func() { + var ( + volumes1 = []string{"/srv/gitlab/config", "/srv/gitlab/data", "/srv/gitlab/logs"} + Volumes1ChangeOrder = []string{"/srv/gitlab/data", "/srv/gitlab/logs", "/srv/gitlab/config"} + volumes1Missing = []string{"/srv/gitlab/data", "/srv/gitlab/logs"} + volumes2 = []string{"totally/different/path"} + ) + + var ( + volumesSrc, volumesDest []string + differ bool + ) + + JustBeforeEach(func() { + differ = IfVolumesDiffer(volumesSrc, volumesDest) + }) + + When("the volumes are the same but the order is changed", func() { + BeforeEach(func() { + volumesSrc = volumes1 + volumesDest = Volumes1ChangeOrder + }) + + It("should return false", func() { + Expect(differ).To(BeFalse()) + }) + }) + + When("the volumes are different(missing)", func() { + BeforeEach(func() { + volumesSrc = volumes1 + volumesDest = volumes1Missing + }) + + It("should return true", func() { + Expect(differ).To(BeTrue()) + }) + }) + + When("the volumes are totally different", func() { + BeforeEach(func() { + volumesSrc = volumes1 + volumesDest = volumes2 + }) + + It("should return true", func() { + Expect(differ).To(BeTrue()) + }) + }) + }) +}) From 66bca45af7c2a464775c1a0e05e4c704ae5a39ec Mon Sep 17 00:00:00 2001 From: Bird Date: Thu, 21 Jul 2022 23:31:10 +0800 Subject: [PATCH 2/2] chore: remove useless comment Signed-off-by: Bird --- pkg/util/docker/dockersh/operator.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/pkg/util/docker/dockersh/operator.go b/pkg/util/docker/dockersh/operator.go index b0e3d3522..002e329f5 100644 --- a/pkg/util/docker/dockersh/operator.go +++ b/pkg/util/docker/dockersh/operator.go @@ -150,12 +150,6 @@ func (op *ShellOperator) ContainerRemove(containerName string) error { } func (op *ShellOperator) ContainerListMounts(containerName string) (docker.Mounts, error) { - // result: the list of mounts, format: "SourcePath:DestinationPath" - // e.g. - // /srv/gitlab/config:/etc/gitlab - // /srv/gitlab/logs:/var/log/gitlab - // /srv/gitlab/data:/var/opt/gitlab - cmdString := fmt.Sprintf(`docker inspect --format='{{json .Mounts}}' %s`, containerName) outputBuffer := &bytes.Buffer{} @@ -190,7 +184,6 @@ func (op *ShellOperator) ContainerGetHostname(container string) (string, error) } func (op *ShellOperator) ContainerGetPortBinding(container, containerPort, protocol string) (hostPort string, err error) { - // get container port binding map // the result is like: // 22/tcp->8122