From 06969134b64caf3aee833c601ea808941ceae036 Mon Sep 17 00:00:00 2001 From: Bird Date: Mon, 1 Aug 2022 21:55:22 +0800 Subject: [PATCH 1/3] feat: docker installer Signed-off-by: Bird --- go.mod | 2 + go.sum | 2 + internal/pkg/plugin/gitlabcedocker/create.go | 90 +++------- internal/pkg/plugin/gitlabcedocker/delete.go | 106 +++--------- .../pkg/plugin/gitlabcedocker/delete_test.go | 71 -------- .../plugin/gitlabcedocker/gitlabcedocker.go | 46 ++--- internal/pkg/plugin/gitlabcedocker/options.go | 47 +++-- .../pkg/plugin/gitlabcedocker/options_test.go | 25 +-- internal/pkg/plugin/gitlabcedocker/read.go | 77 ++------- internal/pkg/plugin/gitlabcedocker/update.go | 79 +++------ .../pkg/plugin/gitlabcedocker/validate.go | 36 +++- .../docker/docker_suite_test.go | 13 ++ .../pkg/plugininstaller/docker/installer.go | 163 ++++++++++++++++++ internal/pkg/plugininstaller/docker/option.go | 75 ++++++++ internal/pkg/plugininstaller/docker/state.go | 86 +++++++++ internal/pkg/plugininstaller/docker/util.go | 23 +++ .../pkg/plugininstaller/docker/util_test.go | 68 ++++++++ pkg/util/docker/docker.go | 6 +- pkg/util/docker/dockersh/operator.go | 108 +++++++----- pkg/util/docker/dockersh/operator_test.go | 136 ++++++++------- pkg/util/docker/option.go | 20 +-- 21 files changed, 749 insertions(+), 530 deletions(-) delete mode 100644 internal/pkg/plugin/gitlabcedocker/delete_test.go create mode 100644 internal/pkg/plugininstaller/docker/docker_suite_test.go create mode 100644 internal/pkg/plugininstaller/docker/installer.go create mode 100644 internal/pkg/plugininstaller/docker/option.go create mode 100644 internal/pkg/plugininstaller/docker/state.go create mode 100644 internal/pkg/plugininstaller/docker/util.go create mode 100644 internal/pkg/plugininstaller/docker/util_test.go diff --git a/go.mod b/go.mod index 92e034c52..c4d78b4d1 100644 --- a/go.mod +++ b/go.mod @@ -30,6 +30,7 @@ require ( github.com/tcnksm/go-input v0.0.0-20180404061846-548a7d7a8ee8 github.com/withfig/autocomplete-tools/integrations/cobra v0.0.0-20220721102007-67b2515c5ea4 github.com/xanzy/go-gitlab v0.55.1 + go.uber.org/multierr v1.6.0 golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3 golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 gopkg.in/gookit/color.v1 v1.1.6 @@ -185,6 +186,7 @@ require ( github.com/xeipuuv/gojsonschema v1.2.0 // indirect github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca // indirect go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect + go.uber.org/atomic v1.7.0 // indirect golang.org/x/exp v0.0.0-20210901193431-a062eea981d2 // indirect golang.org/x/net v0.0.0-20220225172249-27dd8689420f // indirect golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect diff --git a/go.sum b/go.sum index c5ca0029a..c3bda336e 100644 --- a/go.sum +++ b/go.sum @@ -1345,11 +1345,13 @@ go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= +go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.8.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= diff --git a/internal/pkg/plugin/gitlabcedocker/create.go b/internal/pkg/plugin/gitlabcedocker/create.go index 70bff0d4a..e2c71b278 100644 --- a/internal/pkg/plugin/gitlabcedocker/create.go +++ b/internal/pkg/plugin/gitlabcedocker/create.go @@ -1,83 +1,45 @@ package gitlabcedocker import ( - "fmt" - "strconv" - "strings" - - "github.com/mitchellh/mapstructure" - - "github.com/devstream-io/devstream/pkg/util/docker" + "github.com/devstream-io/devstream/internal/pkg/plugininstaller" + dockerInstaller "github.com/devstream-io/devstream/internal/pkg/plugininstaller/docker" "github.com/devstream-io/devstream/pkg/util/log" ) func Create(options map[string]interface{}) (map[string]interface{}, error) { - var opts Options - if err := mapstructure.Decode(options, &opts); err != nil { + // 1. create config and pre-handle operations + opts, err := preHandleOptions(options) + if err != nil { return nil, err } - defaults(&opts) + gitlabURL = getGitLabURL(opts) - if errs := validate(&opts); len(errs) != 0 { - for _, e := range errs { - log.Errorf("Options error: %s.", e) - } - return nil, fmt.Errorf("opts are illegal") + // 2. config install operations + runner := &plugininstaller.Runner{ + PreExecuteOperations: []plugininstaller.MutableOperation{ + dockerInstaller.Validate, + }, + ExecuteOperations: []plugininstaller.BaseOperation{ + dockerInstaller.InstallOrUpdate, + showGitLabURL, + }, + TerminateOperations: []plugininstaller.BaseOperation{ + dockerInstaller.HandleRunFailure, + }, + GetStatusOperation: dockerInstaller.GetStaticStateFromOptions, } - op := GetDockerOperator(opts) - - // 1. try to pull the image - // always pull the image because docker will check the image existence - if err := op.ImagePull(getImageNameWithTag(opts)); err != nil { + // 3. execute installer get status and error + rawOptions, err := buildDockerOptions(opts).Encode() + if err != nil { return nil, err } - - // 2. try to run the container - log.Info("Running container as the name ") - if err := op.ContainerRun(buildDockerRunOptions(opts), dockerRunShmSizeParam); err != nil { - return nil, fmt.Errorf("failed to run container: %v", err) - } - - // 3. check if the container is started successfully - if ok := op.ContainerIfRunning(gitlabContainerName); !ok { - return nil, fmt.Errorf("failed to run container") - } - - // 4. check if the volume is created successfully - mounts, err := op.ContainerListMounts(gitlabContainerName) + status, err := runner.Execute(rawOptions) if err != nil { - return nil, fmt.Errorf("failed to get container mounts: %v", err) - } - volumes := mounts.ExtractSources() - if docker.IfVolumesDiffer(volumes, getVolumesDirFromOptions(opts)) { - return nil, fmt.Errorf("failed to create volumes") - } - - // 5. show the access url - showGitLabURL(opts) - - resource := gitlabResource{ - ContainerRunning: true, - Volumes: volumes, - Hostname: opts.Hostname, - SSHPort: strconv.Itoa(int(opts.SSHPort)), - HTTPPort: strconv.Itoa(int(opts.HTTPPort)), - HTTPSPort: strconv.Itoa(int(opts.HTTPSPort)), - } - - return resource.toMap(), nil -} - -func showGitLabURL(opts Options) { - accessUrl := opts.Hostname - if opts.HTTPPort != 80 { - accessUrl += ":" + strconv.Itoa(int(opts.HTTPPort)) - } - if !strings.HasPrefix(accessUrl, "http") { - accessUrl = "http://" + accessUrl + return nil, err } + log.Debugf("Return map: %v", status) - log.Infof("GitLab access URL: %s", accessUrl) + return status, nil } diff --git a/internal/pkg/plugin/gitlabcedocker/delete.go b/internal/pkg/plugin/gitlabcedocker/delete.go index 81b68d0d8..96d3f903a 100644 --- a/internal/pkg/plugin/gitlabcedocker/delete.go +++ b/internal/pkg/plugin/gitlabcedocker/delete.go @@ -1,102 +1,38 @@ package gitlabcedocker import ( - "fmt" - "os" - - "github.com/mitchellh/mapstructure" - - "github.com/devstream-io/devstream/pkg/util/log" + "github.com/devstream-io/devstream/internal/pkg/plugininstaller" + dockerInstaller "github.com/devstream-io/devstream/internal/pkg/plugininstaller/docker" ) func Delete(options map[string]interface{}) (bool, error) { - var opts Options - if err := mapstructure.Decode(options, &opts); err != nil { + // 1. create config and pre-handle operations + opts, err := preHandleOptions(options) + if err != nil { return false, err } - defaults(&opts) - - if errs := validate(&opts); len(errs) != 0 { - for _, e := range errs { - log.Errorf("Options error: %s.", e) - } - return false, fmt.Errorf("opts are illegal") - } - - op := GetDockerOperator(opts) - - // 1. stop the container if it is running - if ok := op.ContainerIfRunning(gitlabContainerName); ok { - if err := op.ContainerStop(gitlabContainerName); err != nil { - log.Errorf("Failed to stop container: %v", err) - } - } - - // 2. remove the container if it exists - if ok := op.ContainerIfExist(gitlabContainerName); ok { - if err := op.ContainerRemove(gitlabContainerName); err != nil { - log.Errorf("failed to remove container %v: %v", gitlabContainerName, err) - } - } - - // 3. remove the image if it exists - if ok := op.ImageIfExist(getImageNameWithTag(opts)); ok { - if err := op.ImageRemove(getImageNameWithTag(opts)); err != nil { - log.Errorf("failed to remove image %v: %v", getImageNameWithTag(opts), err) - } - } - - // 4. remove the volume if it exists - volumesDirFromOptions := getVolumesDirFromOptions(opts) - if opts.RmDataAfterDelete { - for _, volume := range volumesDirFromOptions { - if err := os.RemoveAll(volume); err != nil { - log.Errorf("failed to remove data %v: %v", volume, err) - } - } - } - - var errs []error - - // 1. check if the container is stopped and deleted - if ok := op.ContainerIfRunning(gitlabContainerName); ok { - errs = append(errs, fmt.Errorf("failed to delete/stop container %s", gitlabContainerName)) - } - if ok := op.ContainerIfExist(gitlabContainerName); ok { - errs = append(errs, fmt.Errorf("failed to delete container %s", gitlabContainerName)) - } + gitlabURL = getGitLabURL(opts) - // 2. check if the image is removed - if ok := op.ImageIfExist(getImageNameWithTag(opts)); ok { - errs = append(errs, fmt.Errorf("failed to delete image %s", getImageNameWithTag(opts))) + // 2. config delete operations + runner := &plugininstaller.Runner{ + PreExecuteOperations: []plugininstaller.MutableOperation{ + dockerInstaller.Validate, + }, + ExecuteOperations: []plugininstaller.BaseOperation{ + dockerInstaller.Delete, + }, } - // 3. check if the data volume is removed - if opts.RmDataAfterDelete { - errs = append(errs, RemoveDirs(volumesDirFromOptions)...) + // 3. delete and get status + rawOptions, err := buildDockerOptions(opts).Encode() + if err != nil { + return false, err } - - // splice the errors - if len(errs) != 0 { - errsString := "" - for _, e := range errs { - errsString += e.Error() + "; " - } - return false, fmt.Errorf(errsString) + _, err = runner.Execute(rawOptions) + if err != nil { + return false, err } return true, nil } - -// RemoveDirs removes the all the directories in the given list recursively -func RemoveDirs(dirs []string) []error { - var errs []error - for _, dir := range dirs { - if err := os.RemoveAll(dir); err != nil { - errs = append(errs, err) - } - } - - return errs -} diff --git a/internal/pkg/plugin/gitlabcedocker/delete_test.go b/internal/pkg/plugin/gitlabcedocker/delete_test.go deleted file mode 100644 index 3557b330e..000000000 --- a/internal/pkg/plugin/gitlabcedocker/delete_test.go +++ /dev/null @@ -1,71 +0,0 @@ -package gitlabcedocker - -import ( - "os" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -var _ = Describe("Delete", func() { - - Describe("RemoveDirs func", func() { - - var ( - errs []error - dirs []string - ) - - AfterEach(func() { - // check directories are removed - for _, dir := range dirs { - _, err := os.Stat(dir) - Expect(os.IsNotExist(err)).To(BeTrue()) - } - }) - - When("the directories are not exist", func() { - - BeforeEach(func() { - dirs = []string{"dir/not/exist/1", "dir/not/exist/2"} - }) - - It("should return no error", func() { - // Remove the directories - errs = RemoveDirs(dirs) - for _, e := range errs { - Expect(e).ToNot(HaveOccurred()) - } - }) - }) - - When("the directories are exist", func() { - - BeforeEach(func() { - // create temp dir, it will be removed automatically after the test - parentDir := GinkgoT().TempDir() - dirs = []string{ - parentDir + "dir1", - parentDir + "dir2", - parentDir + "dir3/dir3-1", - } - // create directories - for _, dir := range dirs { - err := os.MkdirAll(dir, 0755) - Expect(err).ToNot(HaveOccurred()) - } - // create files - _, err := os.CreateTemp(dirs[0], "file1*") - Expect(err).ToNot(HaveOccurred()) - }) - - It("should remove all directories successfully", func() { - // Remove the directories - errs = RemoveDirs(dirs) - for _, e := range errs { - Expect(e).ToNot(HaveOccurred()) - } - }) - }) - }) -}) diff --git a/internal/pkg/plugin/gitlabcedocker/gitlabcedocker.go b/internal/pkg/plugin/gitlabcedocker/gitlabcedocker.go index 2183eb1be..49c8a4335 100644 --- a/internal/pkg/plugin/gitlabcedocker/gitlabcedocker.go +++ b/internal/pkg/plugin/gitlabcedocker/gitlabcedocker.go @@ -1,51 +1,37 @@ package gitlabcedocker import ( + "strconv" "strings" - "github.com/devstream-io/devstream/pkg/util/docker" - "github.com/devstream-io/devstream/pkg/util/docker/dockersh" + "github.com/devstream-io/devstream/internal/pkg/plugininstaller" + "github.com/devstream-io/devstream/pkg/util/log" ) const ( gitlabImageName = "gitlab/gitlab-ce" defaultImageTag = "rc" gitlabContainerName = "gitlab" - tcp = "tcp" dockerRunShmSizeParam = "--shm-size 256m" ) -func getImageNameWithTag(opt Options) string { - return gitlabImageName + ":" + opt.ImageTag -} +// gitlabURL is the access URL of GitLab. +var gitlabURL string -func defaults(opts *Options) { - if opts.ImageTag == "" { - opts.ImageTag = defaultImageTag +func getGitLabURL(opts *Options) string { + accessUrl := opts.Hostname + if opts.HTTPPort != 80 { + accessUrl += ":" + strconv.Itoa(int(opts.HTTPPort)) + } + if !strings.HasPrefix(accessUrl, "http") { + accessUrl = "http://" + accessUrl } -} -func GetDockerOperator(_ Options) docker.Operator { - // just return a ShellOperator for now - return &dockersh.ShellOperator{} + return accessUrl } -type gitlabResource struct { - ContainerRunning bool - Volumes []string - Hostname string - SSHPort string - HTTPPort string - HTTPSPort string -} +func showGitLabURL(options plugininstaller.RawOptions) error { + log.Infof("GitLab access URL: %s", gitlabURL) -func (res *gitlabResource) toMap() map[string]interface{} { - return map[string]interface{}{ - "containerRunning": res.ContainerRunning, - "volumes": strings.Join(res.Volumes, ","), - "hostname": res.Hostname, - "SSHPort": res.SSHPort, - "HTTPPort": res.HTTPPort, - "HTTPSPort": res.HTTPSPort, - } + return nil } diff --git a/internal/pkg/plugin/gitlabcedocker/options.go b/internal/pkg/plugin/gitlabcedocker/options.go index ce3a5f9a7..1b7608799 100644 --- a/internal/pkg/plugin/gitlabcedocker/options.go +++ b/internal/pkg/plugin/gitlabcedocker/options.go @@ -3,6 +3,7 @@ package gitlabcedocker import ( "path/filepath" + dockerInstaller "github.com/devstream-io/devstream/internal/pkg/plugininstaller/docker" "github.com/devstream-io/devstream/pkg/util/docker" ) @@ -18,13 +19,29 @@ type Options struct { ImageTag string `mapstructure:"image_tag"` } -// getVolumesDirFromOptions returns host directories of the volumes from the options. -func getVolumesDirFromOptions(opts Options) []string { - volumes := buildDockerVolumes(opts) - return volumes.ExtractHostPaths() +func buildDockerOptions(opts *Options) *dockerInstaller.Options { + dockerOpts := &dockerInstaller.Options{} + dockerOpts.ImageName = gitlabImageName + dockerOpts.ImageTag = opts.ImageTag + dockerOpts.Hostname = opts.Hostname + dockerOpts.ContainerName = gitlabContainerName + dockerOpts.RestartAlways = true + + portPublishes := []docker.PortPublish{ + {HostPort: opts.SSHPort, ContainerPort: 22}, + {HostPort: opts.HTTPPort, ContainerPort: 80}, + {HostPort: opts.HTTPSPort, ContainerPort: 443}, + } + dockerOpts.PortPublishes = portPublishes + + dockerOpts.Volumes = buildDockerVolumes(opts) + + dockerOpts.RunParams = []string{dockerRunShmSizeParam} + + return dockerOpts } -func buildDockerVolumes(opts Options) docker.Volumes { +func buildDockerVolumes(opts *Options) docker.Volumes { volumes := []docker.Volume{ {HostPath: filepath.Join(opts.GitLabHome, "config"), ContainerPath: "/etc/gitlab"}, {HostPath: filepath.Join(opts.GitLabHome, "data"), ContainerPath: "/var/opt/gitlab"}, @@ -33,23 +50,3 @@ func buildDockerVolumes(opts Options) docker.Volumes { return volumes } - -func buildDockerRunOptions(opts Options) docker.RunOptions { - dockerRunOpts := docker.RunOptions{} - dockerRunOpts.ImageName = gitlabImageName - dockerRunOpts.ImageTag = opts.ImageTag - dockerRunOpts.Hostname = opts.Hostname - dockerRunOpts.ContainerName = gitlabContainerName - dockerRunOpts.RestartAlways = true - - portPublishes := []docker.PortPublish{ - {HostPort: opts.SSHPort, ContainerPort: 22}, - {HostPort: opts.HTTPPort, ContainerPort: 80}, - {HostPort: opts.HTTPSPort, ContainerPort: 443}, - } - dockerRunOpts.PortPublishes = portPublishes - - dockerRunOpts.Volumes = buildDockerVolumes(opts) - - return dockerRunOpts -} diff --git a/internal/pkg/plugin/gitlabcedocker/options_test.go b/internal/pkg/plugin/gitlabcedocker/options_test.go index 1e552641f..d6b464742 100644 --- a/internal/pkg/plugin/gitlabcedocker/options_test.go +++ b/internal/pkg/plugin/gitlabcedocker/options_test.go @@ -4,15 +4,16 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + dockerInstaller "github.com/devstream-io/devstream/internal/pkg/plugininstaller/docker" "github.com/devstream-io/devstream/pkg/util/docker" ) var _ = Describe("Options", func() { - var opts Options + var opts *Options BeforeEach(func() { - opts = Options{ + opts = &Options{ GitLabHome: "/srv/gitlab", Hostname: "gitlab.example.com", SSHPort: 8122, @@ -23,23 +24,10 @@ var _ = Describe("Options", func() { } }) - Describe("getVolumesDirFromOptions func", func() { - When("the options is valid", func() { - It("should return the volumes' directory", func() { - volumesDirFromOptions := getVolumesDirFromOptions(opts) - Expect(volumesDirFromOptions).To(Equal([]string{ - "/srv/gitlab/config", - "/srv/gitlab/data", - "/srv/gitlab/logs", - })) - }) - }) - }) - Describe("buildDockerRunOptions func", func() { It("should build the docker run options successfully", func() { - runOptsBuild := buildDockerRunOptions(opts) - runOptsExpect := docker.RunOptions{ + OptsBuild := *buildDockerOptions(opts) + OptsExpect := dockerInstaller.Options{ ImageName: "gitlab/gitlab-ce", ImageTag: "rc", Hostname: "gitlab.example.com", @@ -55,9 +43,10 @@ var _ = Describe("Options", func() { {HostPath: "/srv/gitlab/data", ContainerPath: "/var/opt/gitlab"}, {HostPath: "/srv/gitlab/logs", ContainerPath: "/var/log/gitlab"}, }, + RunParams: []string{dockerRunShmSizeParam}, } - Expect(runOptsBuild).To(Equal(runOptsExpect)) + Expect(OptsBuild).To(Equal(OptsExpect)) }) }) diff --git a/internal/pkg/plugin/gitlabcedocker/read.go b/internal/pkg/plugin/gitlabcedocker/read.go index 274ec988a..894bbb82e 100644 --- a/internal/pkg/plugin/gitlabcedocker/read.go +++ b/internal/pkg/plugin/gitlabcedocker/read.go @@ -1,77 +1,38 @@ package gitlabcedocker import ( - "fmt" - - "github.com/mitchellh/mapstructure" - + "github.com/devstream-io/devstream/internal/pkg/plugininstaller" + dockerInstaller "github.com/devstream-io/devstream/internal/pkg/plugininstaller/docker" "github.com/devstream-io/devstream/pkg/util/log" ) func Read(options map[string]interface{}) (map[string]interface{}, error) { - var opts Options - if err := mapstructure.Decode(options, &opts); err != nil { + // 1. create config and pre-handle operations + opts, err := preHandleOptions(options) + if err != nil { return nil, err } - defaults(&opts) - - if errs := validate(&opts); len(errs) != 0 { - for _, e := range errs { - log.Errorf("Options error: %s.", e) - } - return nil, fmt.Errorf("opts are illegal") - } - - op := GetDockerOperator(opts) - - // 1. get running status - running := op.ContainerIfRunning(gitlabContainerName) - if !running { - return (&gitlabResource{}).toMap(), nil - } - - // 2. get volumes - mounts, err := op.ContainerListMounts(gitlabContainerName) - if err != nil { - // `Read` shouldn't return errors even if failed to read ports, volumes, hostname. - // because: - // 1. when the docker is stopped it could cause these errors. - // 2. if Read failed, the following steps contain the docker's restart will be aborted. - log.Errorf("failed to get container mounts: %v", err) - } - volumes := mounts.ExtractSources() + gitlabURL = getGitLabURL(opts) - // 3. get hostname - hostname, err := op.ContainerGetHostname(gitlabContainerName) - if err != nil { - log.Errorf("failed to get container hostname: %v", err) + // 2. config read operations + runner := &plugininstaller.Runner{ + PreExecuteOperations: []plugininstaller.MutableOperation{ + dockerInstaller.Validate, + }, + GetStatusOperation: dockerInstaller.GetRunningState, } - // 4. get port bindings - SSHPort, err := op.ContainerGetPortBinding(gitlabContainerName, "22", tcp) - if err != nil { - log.Errorf("failed to get container ssh port: %v", err) - } - HTTPPort, err := op.ContainerGetPortBinding(gitlabContainerName, "80", tcp) + // 3. get status + rawOptions, err := buildDockerOptions(opts).Encode() if err != nil { - log.Errorf("failed to get container http port: %v", err) + return nil, err } - HTTPSPort, err := op.ContainerGetPortBinding(gitlabContainerName, "443", tcp) + status, err := runner.Execute(rawOptions) if err != nil { - log.Errorf("failed to get container https port: %v", err) - } - - // if the previous steps failed, the parameters will be empty - // so dtm will find the resource is drifted and restart docker - resource := gitlabResource{ - ContainerRunning: running, - Volumes: volumes, - Hostname: hostname, - SSHPort: SSHPort, - HTTPPort: HTTPPort, - HTTPSPort: HTTPSPort, + return nil, err } - return resource.toMap(), nil + log.Debugf("Return map: %v", status) + return status, nil } diff --git a/internal/pkg/plugin/gitlabcedocker/update.go b/internal/pkg/plugin/gitlabcedocker/update.go index 97a0eaa0e..2486f5049 100644 --- a/internal/pkg/plugin/gitlabcedocker/update.go +++ b/internal/pkg/plugin/gitlabcedocker/update.go @@ -1,75 +1,38 @@ package gitlabcedocker import ( - "fmt" - "strconv" - "strings" - - "github.com/mitchellh/mapstructure" - - "github.com/devstream-io/devstream/pkg/util/docker" + "github.com/devstream-io/devstream/internal/pkg/plugininstaller" + dockerInstaller "github.com/devstream-io/devstream/internal/pkg/plugininstaller/docker" "github.com/devstream-io/devstream/pkg/util/log" ) func Update(options map[string]interface{}) (map[string]interface{}, error) { - var opts Options - if err := mapstructure.Decode(options, &opts); err != nil { + // 1. create config and pre-handle operations + opts, err := preHandleOptions(options) + if err != nil { return nil, err } - defaults(&opts) + gitlabURL = getGitLabURL(opts) - if errs := validate(&opts); len(errs) != 0 { - for _, e := range errs { - log.Errorf("Options error: %s.", e) - } - return nil, fmt.Errorf("opts are illegal") + // 2. config install operations + runner := &plugininstaller.Runner{ + PreExecuteOperations: []plugininstaller.MutableOperation{ + dockerInstaller.Validate, + }, + ExecuteOperations: []plugininstaller.BaseOperation{ + dockerInstaller.InstallOrUpdate, + showGitLabURL, + }, + GetStatusOperation: dockerInstaller.GetRunningState, } - op := GetDockerOperator(opts) - - // 0. check if the volumes are the same - mounts, err := op.ContainerListMounts(gitlabContainerName) + // 3. update and get status + status, err := runner.Execute(options) if err != nil { - return nil, fmt.Errorf("failed to get container mounts: %v", err) - } - volumesFromRunningContainer := mounts.ExtractSources() - - volumesDirFromOptions := getVolumesDirFromOptions(opts) - - if docker.IfVolumesDiffer(volumesFromRunningContainer, volumesDirFromOptions) { - log.Warnf("You changed volumes of the container or change the gitlab home directory") - log.Infof("Your volumes of the current container were: %v", strings.Join(volumesFromRunningContainer, " ")) - return nil, fmt.Errorf("sorry, you can't change the gitlab_home of the container once it's already been created") - } - - // 1. stop the container - if ok := op.ContainerIfRunning(gitlabContainerName); ok { - if err := op.ContainerStop(gitlabContainerName); err != nil { - log.Warnf("Failed to stop container: %v", err) - } - } - - // 2. remove the container if it exists - if exists := op.ContainerIfExist(gitlabContainerName); exists { - if err := op.ContainerRemove(gitlabContainerName); err != nil { - return nil, fmt.Errorf("failed to remove container: %v", err) - } - } - - // 3. run the container with the new options - if err := op.ContainerRun(buildDockerRunOptions(opts), dockerRunShmSizeParam); err != nil { - return nil, fmt.Errorf("failed to run container: %v", err) - } - - resource := gitlabResource{ - ContainerRunning: true, - Volumes: volumesFromRunningContainer, - Hostname: opts.Hostname, - SSHPort: strconv.Itoa(int(opts.SSHPort)), - HTTPPort: strconv.Itoa(int(opts.HTTPPort)), - HTTPSPort: strconv.Itoa(int(opts.HTTPSPort)), + return nil, err } + log.Debugf("Return map: %v", status) - return resource.toMap(), nil + return status, nil } diff --git a/internal/pkg/plugin/gitlabcedocker/validate.go b/internal/pkg/plugin/gitlabcedocker/validate.go index 9b2889f40..d7cd46692 100644 --- a/internal/pkg/plugin/gitlabcedocker/validate.go +++ b/internal/pkg/plugin/gitlabcedocker/validate.go @@ -4,17 +4,47 @@ import ( "fmt" "path" + "github.com/mitchellh/mapstructure" + + "github.com/devstream-io/devstream/pkg/util/log" "github.com/devstream-io/devstream/pkg/util/validator" ) +func preHandleOptions(options map[string]interface{}) (*Options, error) { + var opts *Options + if err := mapstructure.Decode(options, &opts); err != nil { + return nil, err + } + + defaults(opts) + + if err := validate(opts); err != nil { + return nil, err + } + + return opts, nil +} + +func defaults(opts *Options) { + if opts.ImageTag == "" { + opts.ImageTag = defaultImageTag + } +} + // validate validates the options provided by the core. -func validate(options *Options) []error { +func validate(options *Options) error { errs := validator.Struct(options) - // volume directory must be absolute path if !path.IsAbs(options.GitLabHome) { errs = append(errs, fmt.Errorf("GitLabHome must be an absolute path")) } - return errs + if len(errs) > 0 { + for _, e := range errs { + log.Errorf("Options error: %s.", e) + } + return fmt.Errorf("opts are illegal") + } + + return nil } diff --git a/internal/pkg/plugininstaller/docker/docker_suite_test.go b/internal/pkg/plugininstaller/docker/docker_suite_test.go new file mode 100644 index 000000000..4001151b1 --- /dev/null +++ b/internal/pkg/plugininstaller/docker/docker_suite_test.go @@ -0,0 +1,13 @@ +package docker_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestDocker(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Docker Suite") +} diff --git a/internal/pkg/plugininstaller/docker/installer.go b/internal/pkg/plugininstaller/docker/installer.go new file mode 100644 index 000000000..6ee6115f3 --- /dev/null +++ b/internal/pkg/plugininstaller/docker/installer.go @@ -0,0 +1,163 @@ +package docker + +import ( + "fmt" + + "go.uber.org/multierr" + + "github.com/devstream-io/devstream/internal/pkg/plugininstaller" + "github.com/devstream-io/devstream/pkg/util/docker" + "github.com/devstream-io/devstream/pkg/util/docker/dockersh" + "github.com/devstream-io/devstream/pkg/util/log" +) + +var op docker.Operator + +func init() { + // default to shell operator + op = &dockersh.ShellOperator{} +} + +func UseShellOperator() { + op = &dockersh.ShellOperator{} +} + +// InstallOrUpdate runs or updates the docker container +// note: any update will stop and remove the container, then run the new container +func InstallOrUpdate(options plugininstaller.RawOptions) error { + opts, err := NewOptions(options) + if err != nil { + return err + } + + // 1. try to pull the image + // always pull the image because docker will check the image existence + if err := op.ImagePull(opts.GetImageNameWithTag()); err != nil { + return err + } + + // 2. try to run the container + log.Infof("Running container as the name <%s>", opts.ContainerName) + if err := op.ContainerRun(opts.GetRunOpts()); err != nil { + return fmt.Errorf("failed to run container: %v", err) + } + + // 3. check if the container is started successfully + if ok := op.ContainerIfRunning(opts.ContainerName); !ok { + return fmt.Errorf("failed to run container") + } + + // 4. check if the volume is created successfully + mounts, err := op.ContainerListMounts(opts.ContainerName) + if err != nil { + return fmt.Errorf("failed to get container mounts: %v", err) + } + volumes := mounts.ExtractSources() + if docker.IfVolumesDiffer(volumes, opts.Volumes.ExtractHostPaths()) { + return fmt.Errorf("failed to create volumes") + } + + return nil +} + +// HandleRunFailure will delete the container if the container fails to run +func HandleRunFailure(options plugininstaller.RawOptions) error { + opts, err := NewOptions(options) + if err != nil { + return err + } + + // 1. stop the container if it is running + if ok := op.ContainerIfRunning(opts.ContainerName); ok { + if err := op.ContainerStop(opts.ContainerName); err != nil { + log.Errorf("Failed to stop container %s: %v", opts.ContainerName, err) + } + } + + // 2. remove the container if it exists + if ok := op.ContainerIfExist(opts.ContainerName); ok { + if err := op.ContainerRemove(opts.ContainerName); err != nil { + log.Errorf("failed to remove container %v: %v", opts.ContainerName, err) + } + } + + var errs []error + + // 3. check if the container is stopped + if ok := op.ContainerIfRunning(opts.ContainerName); ok { + errs = append(errs, fmt.Errorf("failed to stop container %s", opts.ContainerName)) + } + + // 4. check if the container is removed + if ok := op.ContainerIfExist(opts.ContainerName); ok { + errs = append(errs, fmt.Errorf("failed to delete container %s", opts.ContainerName)) + } + + return multierr.Combine(errs...) +} + +// Delete will delete the container/image/volumes +func Delete(options plugininstaller.RawOptions) error { + opts, err := NewOptions(options) + if err != nil { + return err + } + + // 1. stop the container if it is running + if ok := op.ContainerIfRunning(opts.ContainerName); ok { + if err := op.ContainerStop(opts.ContainerName); err != nil { + log.Errorf("Failed to stop container %s: %v", opts.ContainerName, err) + } + } + + // 2. remove the container if it exists + if ok := op.ContainerIfExist(opts.ContainerName); ok { + if err := op.ContainerRemove(opts.ContainerName); err != nil { + log.Errorf("failed to remove container %v: %v", opts.ContainerName, err) + } + } + + // 3. remove the image if it exists + if ok := op.ImageIfExist(opts.GetImageNameWithTag()); ok { + if err := op.ImageRemove(opts.GetImageNameWithTag()); err != nil { + log.Errorf("failed to remove image %v: %v", opts.GetImageNameWithTag(), err) + } + } + + // 4. remove the volume if it exists + if opts.RmDataAfterDelete { + volumesDirFromOptions := opts.Volumes.ExtractHostPaths() + for _, err := range RemoveDirs(volumesDirFromOptions) { + log.Error(err) + } + } + + var errs []error + + // 5. check if the container is stopped and deleted + if ok := op.ContainerIfRunning(opts.ContainerName); ok { + errs = append(errs, fmt.Errorf("failed to stop container %s", opts.ContainerName)) + } + + // 6. check if the container is removed + if ok := op.ContainerIfExist(opts.ContainerName); ok { + errs = append(errs, fmt.Errorf("failed to delete container %s", opts.ContainerName)) + } + + // 7. check if the image is removed + if ok := op.ImageIfExist(opts.GetImageNameWithTag()); ok { + errs = append(errs, fmt.Errorf("failed to delete image %s", opts.GetImageNameWithTag())) + } + + // 8. check if the volume is removed + if opts.RmDataAfterDelete { + volumesDirFromOptions := opts.Volumes.ExtractHostPaths() + for _, volume := range volumesDirFromOptions { + if exist := pathExist(volume); exist { + errs = append(errs, fmt.Errorf("failed to delete volume %s", volume)) + } + } + } + + return multierr.Combine(errs...) +} diff --git a/internal/pkg/plugininstaller/docker/option.go b/internal/pkg/plugininstaller/docker/option.go new file mode 100644 index 000000000..fd8d98fcc --- /dev/null +++ b/internal/pkg/plugininstaller/docker/option.go @@ -0,0 +1,75 @@ +package docker + +import ( + "fmt" + + "github.com/mitchellh/mapstructure" + + "github.com/devstream-io/devstream/internal/pkg/plugininstaller" + "github.com/devstream-io/devstream/pkg/util/docker" + "github.com/devstream-io/devstream/pkg/util/log" + "github.com/devstream-io/devstream/pkg/util/validator" +) + +type ( + Options struct { + ImageName string `validate:"required"` + ImageTag string `validate:"required"` + ContainerName string `validate:"required"` + RmDataAfterDelete bool + + RunParams []string + Hostname string + PortPublishes []docker.PortPublish + Volumes docker.Volumes + RestartAlways bool + } +) + +// NewOptions create options by raw options +func NewOptions(options plugininstaller.RawOptions) (Options, error) { + var opts Options + if err := mapstructure.Decode(options, &opts); err != nil { + return opts, err + } + return opts, nil +} + +func (opts *Options) GetImageNameWithTag() string { + return opts.ImageName + ":" + opts.ImageTag +} + +func (opts *Options) GetRunOpts() *docker.RunOptions { + return &docker.RunOptions{ + ImageName: opts.ImageName, + ImageTag: opts.ImageTag, + ContainerName: opts.ContainerName, + Hostname: opts.Hostname, + PortPublishes: opts.PortPublishes, + Volumes: opts.Volumes, + RestartAlways: opts.RestartAlways, + } +} + +func (opts *Options) Encode() (map[string]interface{}, error) { + var options map[string]interface{} + if err := mapstructure.Decode(opts, &options); err != nil { + return nil, err + } + return options, nil +} + +func Validate(options plugininstaller.RawOptions) (plugininstaller.RawOptions, error) { + opts, err := NewOptions(options) + if err != nil { + return nil, err + } + errs := validator.Struct(opts) + if len(errs) > 0 { + for _, e := range errs { + log.Errorf("Options error: %s.", e) + } + return nil, fmt.Errorf("opts are illegal") + } + return options, nil +} diff --git a/internal/pkg/plugininstaller/docker/state.go b/internal/pkg/plugininstaller/docker/state.go new file mode 100644 index 000000000..9fea94dbb --- /dev/null +++ b/internal/pkg/plugininstaller/docker/state.go @@ -0,0 +1,86 @@ +package docker + +import ( + "github.com/devstream-io/devstream/internal/pkg/plugininstaller" + "github.com/devstream-io/devstream/pkg/util/docker" + "github.com/devstream-io/devstream/pkg/util/log" +) + +type State struct { + ContainerRunning bool + Volumes []string + Hostname string + PortPublishes []docker.PortPublish +} + +func (s *State) toMap() map[string]interface{} { + return map[string]interface{}{ + "ContainerRunning": s.ContainerRunning, + "Volumes": s.Volumes, + "Hostname": s.Hostname, + "PortPublishes": s.PortPublishes, + } +} + +func GetStaticStateFromOptions(options plugininstaller.RawOptions) (map[string]interface{}, error) { + opts, err := NewOptions(options) + if err != nil { + return nil, err + } + + staticState := &State{ + ContainerRunning: true, + Volumes: opts.Volumes.ExtractHostPaths(), + Hostname: opts.Hostname, + PortPublishes: opts.PortPublishes, + } + + return staticState.toMap(), nil +} + +func GetRunningState(options plugininstaller.RawOptions) (map[string]interface{}, error) { + opts, err := NewOptions(options) + if err != nil { + return nil, err + } + + // 1. get running status + running := op.ContainerIfRunning(opts.ContainerName) + if !running { + return map[string]interface{}{}, nil + } + + // 2. get volumes + mounts, err := op.ContainerListMounts(opts.ContainerName) + if err != nil { + // `Read` shouldn't return errors even if failed to read ports, volumes, hostname. + // because: + // 1. when the docker is stopped it could cause these errors. + // 2. if Read failed, the following steps contain the docker's restart will be aborted. + log.Errorf("failed to get container mounts: %v", err) + } + volumes := mounts.ExtractSources() + + // 3. get hostname + hostname, err := op.ContainerGetHostname(opts.ContainerName) + if err != nil { + log.Errorf("failed to get container hostname: %v", err) + } + + // 4. get port bindings + PortPublishes, err := op.ContainerListPortPublishes(opts.ContainerName) + if err != nil { + log.Errorf("failed to get container port publishes: %v", err) + } + + // if the previous steps failed, the parameters will be empty + // so dtm will find the resource is drifted and restart docker + resource := &State{ + ContainerRunning: running, + Volumes: volumes, + Hostname: hostname, + PortPublishes: PortPublishes, + } + + return resource.toMap(), nil +} diff --git a/internal/pkg/plugininstaller/docker/util.go b/internal/pkg/plugininstaller/docker/util.go new file mode 100644 index 000000000..ac68f7063 --- /dev/null +++ b/internal/pkg/plugininstaller/docker/util.go @@ -0,0 +1,23 @@ +package docker + +import ( + "fmt" + "os" +) + +// RemoveDirs removes the all the directories in the given list recursively +func RemoveDirs(dirs []string) []error { + var errs []error + for _, dir := range dirs { + if err := os.RemoveAll(dir); err != nil { + errs = append(errs, fmt.Errorf("failed to remove data %v: %v", dir, err)) + } + } + + return errs +} + +func pathExist(path string) bool { + _, err := os.Stat(path) + return err == nil +} diff --git a/internal/pkg/plugininstaller/docker/util_test.go b/internal/pkg/plugininstaller/docker/util_test.go new file mode 100644 index 000000000..2c227218b --- /dev/null +++ b/internal/pkg/plugininstaller/docker/util_test.go @@ -0,0 +1,68 @@ +package docker + +import ( + "os" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("RemoveDirs func", func() { + + var ( + errs []error + dirs []string + ) + + AfterEach(func() { + // check directories are removed + for _, dir := range dirs { + _, err := os.Stat(dir) + Expect(os.IsNotExist(err)).To(BeTrue()) + } + }) + + When("the directories are not exist", func() { + + BeforeEach(func() { + dirs = []string{"dir/not/exist/1", "dir/not/exist/2"} + }) + + It("should return no error", func() { + // Remove the directories + errs = RemoveDirs(dirs) + for _, e := range errs { + Expect(e).ToNot(HaveOccurred()) + } + }) + }) + + When("the directories are exist", func() { + + BeforeEach(func() { + // create temp dir, it will be removed automatically after the test + parentDir := GinkgoT().TempDir() + dirs = []string{ + parentDir + "dir1", + parentDir + "dir2", + parentDir + "dir3/dir3-1", + } + // create directories + for _, dir := range dirs { + err := os.MkdirAll(dir, 0755) + Expect(err).ToNot(HaveOccurred()) + } + // create files + _, err := os.CreateTemp(dirs[0], "file1*") + Expect(err).ToNot(HaveOccurred()) + }) + + It("should remove all directories successfully", func() { + // Remove the directories + errs = RemoveDirs(dirs) + for _, e := range errs { + Expect(e).ToNot(HaveOccurred()) + } + }) + }) +}) diff --git a/pkg/util/docker/docker.go b/pkg/util/docker/docker.go index 7ba9635b3..491901d8c 100644 --- a/pkg/util/docker/docker.go +++ b/pkg/util/docker/docker.go @@ -19,7 +19,7 @@ type Operator interface { // ContainerRun runs a container with the given options // params is a list of additional parameters for docker run // params will be appended to the end of the command - ContainerRun(opts RunOptions, params ...string) error + ContainerRun(opts *RunOptions) error ContainerStop(containerName string) error ContainerRemove(containerName string) error @@ -27,9 +27,9 @@ type Operator interface { ContainerListMounts(containerName string) (Mounts, error) ContainerGetHostname(containerName string) (string, error) - ContainerGetPortBinding(containerName, containerPort, protocol string) (hostPort string, err error) + ContainerListPortPublishes(containerName string) ([]PortPublish, error) + ContainerGetPortBinding(containerName string, containerPort uint) (hostPort uint, err error) } - type MountPoint struct { Type string `json:"Type"` Source string `json:"Source"` diff --git a/pkg/util/docker/dockersh/operator.go b/pkg/util/docker/dockersh/operator.go index 002e329f5..f8fce8c88 100644 --- a/pkg/util/docker/dockersh/operator.go +++ b/pkg/util/docker/dockersh/operator.go @@ -5,6 +5,8 @@ import ( "encoding/json" "fmt" "os/exec" + "regexp" + "strconv" "strings" "github.com/devstream-io/devstream/pkg/util/docker" @@ -14,43 +16,30 @@ import ( // ShellOperator is an implementation of /pkg/util/docker.Operator interface by using shell commands type ShellOperator struct{} -// TODO(aFlyBird0): maybe use one param "ImageNameWithTag" is not a good idea, -// because we have to extract the bare image name from the image name with tag -// we could use (a struct)/(a interface)/(two params) to represent the image name and tag func (op *ShellOperator) ImageIfExist(imageNameWithTag string) bool { - // eg. docker image ls gitlab/gitlab-ce:rc - cmdString := fmt.Sprintf("docker image ls %v", imageNameWithTag) - // output: eg. - // REPOSITORY TAG IMAGE ID CREATED SIZE - // gitlab/gitlab-ce rc a8543d702e39 4 days ago 2.49GB + // eg. docker image ls gitlab/gitlab-ce:rc -q + // output: image id (if exist) + cmdString := fmt.Sprintf("docker image ls %v -q", imageNameWithTag) outputBuffer := &bytes.Buffer{} err := ExecInSystem(".", cmdString, outputBuffer, false) if err != nil { return false } - // eg. gitlab/gitlab-ce - imageNameWithoutTag := extractImageName(imageNameWithTag) - return strings.Contains(outputBuffer.String(), imageNameWithoutTag) + return strings.TrimSpace(outputBuffer.String()) != "" } -func extractImageName(imageNameWithTag string) string { - // the imageNameWithTag is in the format of "registry/image:tag" - // we only want to return the image name "registry/image" - return strings.Split(imageNameWithTag, ":")[0] -} - -func (op *ShellOperator) ImagePull(imageName string) error { - err := ExecInSystemWithParams(".", []string{"docker", "pull", imageName}, nil, true) +func (op *ShellOperator) ImagePull(imageNameWithTag string) error { + err := ExecInSystemWithParams(".", []string{"docker", "pull", imageNameWithTag}, nil, true) return err } -func (op *ShellOperator) ImageRemove(imageName string) error { - log.Infof("Removing image %v ...", imageName) +func (op *ShellOperator) ImageRemove(imageNameWithTag string) error { + log.Infof("Removing image %v ...", imageNameWithTag) - cmdString := fmt.Sprintf("docker rmi %s", imageName) + cmdString := fmt.Sprintf("docker rmi %s", imageNameWithTag) err := ExecInSystem(".", cmdString, nil, true) return err @@ -85,9 +74,9 @@ func (op *ShellOperator) ContainerIfRunning(containerName string) bool { return false } -func (op *ShellOperator) ContainerRun(opts docker.RunOptions, params ...string) error { +func (op *ShellOperator) ContainerRun(opts *docker.RunOptions) error { // build the command - cmdString, err := BuildContainerRunCommand(opts, params...) + cmdString, err := BuildContainerRunCommand(opts) if err != nil { return err } @@ -103,7 +92,7 @@ func (op *ShellOperator) ContainerRun(opts docker.RunOptions, params ...string) } // BuildContainerRunCommand builds the docker run command string from the given options and additional params -func BuildContainerRunCommand(opts docker.RunOptions, params ...string) (string, error) { +func BuildContainerRunCommand(opts *docker.RunOptions) (string, error) { if err := opts.Validate(); err != nil { return "", err } @@ -123,7 +112,7 @@ func BuildContainerRunCommand(opts docker.RunOptions, params ...string) (string, for _, volume := range opts.Volumes { cmdBuilder.WriteString(fmt.Sprintf("--volume %s:%s ", volume.HostPath, volume.ContainerPath)) } - for _, param := range params { + for _, param := range opts.RunParams { cmdBuilder.WriteString(param + " ") } cmdBuilder.WriteString(docker.CombineImageNameAndTag(opts.ImageName, opts.ImageTag)) @@ -183,38 +172,73 @@ func (op *ShellOperator) ContainerGetHostname(container string) (string, error) } -func (op *ShellOperator) ContainerGetPortBinding(container, containerPort, protocol string) (hostPort string, err error) { +func (op *ShellOperator) ContainerListPortPublishes(containerName string) ([]docker.PortPublish, error) { // get container port binding map // the result is like: // 22/tcp->8122 // 443/tcp->8443 // 80/tcp->8180 format := "'{{range $p,$conf := .NetworkSettings.Ports}}{{$p}}->{{(index $conf 0).HostPort}}{{println}}{{end}}'" - cmdString := fmt.Sprintf("docker inspect --format=%s %s", format, container) + cmdString := fmt.Sprintf("docker inspect --format=%s %s", format, containerName) outputBuffer := &bytes.Buffer{} - err = ExecInSystem(".", cmdString, outputBuffer, false) + err := ExecInSystem(".", cmdString, outputBuffer, false) if err != nil { - return "", err + return nil, err } portBindings := strings.Split(strings.TrimSpace(outputBuffer.String()), "\n") - log.Debugf("Container %v port bindings: %v", container, portBindings) + log.Debugf("Container %v port bindings: %v", containerName, portBindings) + + publishes, err := buildPortPublishes(portBindings) + if err != nil { + return publishes, err + } + + return publishes, nil +} + +func buildPortPublishes(portBindings []string) (PortPublishes []docker.PortPublish, err error) { + // 22/tcp->8122 + // 443/tcp->8443 + // 80/tcp->8180 + re := regexp.MustCompile(`^(\d+)/(tcp|udp)->(\d+)$`) - // transfer port bindings to map - portBindingsMap := make(map[string]string) for _, portBinding := range portBindings { - portBindingParts := strings.Split(portBinding, "->") - if len(portBindingParts) != 2 { - return "", fmt.Errorf("Invalid port binding: %v", portBinding) + match := re.FindStringSubmatch(portBinding) + // match e.g. ["22/tcp->8122", "22", "tcp", "8122"] + if len(match) != 4 { + return nil, fmt.Errorf("invalid port binding: %v", portBinding) + } + + hostPort, err := strconv.Atoi(match[3]) + if err != nil { + return nil, fmt.Errorf("invalid port binding: %v", portBinding) + } + containerPort, err := strconv.Atoi(match[1]) + if err != nil { + return nil, fmt.Errorf("invalid port binding: %v", portBinding) + } + + portPublish := docker.PortPublish{ + ContainerPort: uint(containerPort), + HostPort: uint(hostPort), } - portBindingsMap[portBindingParts[0]] = portBindingParts[1] + PortPublishes = append(PortPublishes, portPublish) } - portKey := fmt.Sprintf("%s/%s", containerPort, protocol) - hostPort, ok := portBindingsMap[portKey] - if !ok { - return "", fmt.Errorf("No port binding for %v", portKey) + return PortPublishes, nil +} + +func (op *ShellOperator) ContainerGetPortBinding(container string, containerPort uint) (hostPort uint, err error) { + portBindings, err := op.ContainerListPortPublishes(container) + if err != nil { + return 0, err } - return hostPort, nil + for _, portBinding := range portBindings { + if portBinding.ContainerPort == containerPort { + return portBinding.HostPort, nil + } + } + return 0, fmt.Errorf("container %v does not have port binding for port %v", container, containerPort) } diff --git a/pkg/util/docker/dockersh/operator_test.go b/pkg/util/docker/dockersh/operator_test.go index b6df3ae06..f3af9005f 100644 --- a/pkg/util/docker/dockersh/operator_test.go +++ b/pkg/util/docker/dockersh/operator_test.go @@ -9,69 +9,89 @@ import ( "github.com/devstream-io/devstream/pkg/util/docker" ) -var _ = Describe("Operator", func() { - Describe("BuildContainerRunCommand method", func() { - var ( - opts docker.RunOptions - params []string - ) - - When(" the options are invalid", func() { - BeforeEach(func() { - opts = docker.RunOptions{} - }) - - It("should return an error", func() { - _, err := BuildContainerRunCommand(opts, params...) - Expect(err).To(HaveOccurred()) - }) +var _ = Describe("BuildContainerRunCommand method", func() { + var opts *docker.RunOptions + + When(" the options are invalid", func() { + BeforeEach(func() { + opts = &docker.RunOptions{} + }) + + It("should return an error", func() { + _, err := BuildContainerRunCommand(opts) + Expect(err).To(HaveOccurred()) }) + }) + + When(" the options are valid(e.g. gitlab-ce)", func() { + BeforeEach(func() { + buildOpts := func() *docker.RunOptions { + opts := &docker.RunOptions{} + opts.ImageName = "gitlab/gitlab-ce" + opts.ImageTag = "rc" + opts.Hostname = "gitlab.example.com" + opts.ContainerName = "gitlab" + opts.RestartAlways = true - When(" the options are valid(e.g. gitlab-ce)", func() { - BeforeEach(func() { - buildOpts := func() docker.RunOptions { - opts := docker.RunOptions{} - opts.ImageName = "gitlab/gitlab-ce" - opts.ImageTag = "rc" - opts.Hostname = "gitlab.example.com" - opts.ContainerName = "gitlab" - opts.RestartAlways = true - - portPublishes := []docker.PortPublish{ - {HostPort: 8122, ContainerPort: 22}, - {HostPort: 8180, ContainerPort: 80}, - {HostPort: 8443, ContainerPort: 443}, - } - opts.PortPublishes = portPublishes - - gitLabHome := "/srv/gitlab" - - opts.Volumes = []docker.Volume{ - {HostPath: filepath.Join(gitLabHome, "config"), ContainerPath: "/etc/gitlab"}, - {HostPath: filepath.Join(gitLabHome, "data"), ContainerPath: "/var/opt/gitlab"}, - {HostPath: filepath.Join(gitLabHome, "logs"), ContainerPath: "/var/log/gitlab"}, - } - - return opts + portPublishes := []docker.PortPublish{ + {HostPort: 8122, ContainerPort: 22}, + {HostPort: 8180, ContainerPort: 80}, + {HostPort: 8443, ContainerPort: 443}, } + opts.PortPublishes = portPublishes + + gitLabHome := "/srv/gitlab" + + opts.Volumes = []docker.Volume{ + {HostPath: filepath.Join(gitLabHome, "config"), ContainerPath: "/etc/gitlab"}, + {HostPath: filepath.Join(gitLabHome, "data"), ContainerPath: "/var/opt/gitlab"}, + {HostPath: filepath.Join(gitLabHome, "logs"), ContainerPath: "/var/log/gitlab"}, + } + + opts.RunParams = []string{"--shm-size 256m"} + + return opts + } - opts = buildOpts() - params = []string{"--shm-size 256m"} - }) - - It("should return the correct command", func() { - cmdBuild, err := BuildContainerRunCommand(opts, params...) - Expect(err).NotTo(HaveOccurred()) - cmdExpect := "docker run --detach --hostname gitlab.example.com" + - " --publish 8122:22 --publish 8180:80 --publish 8443:443" + - " --name gitlab --restart always" + - " --volume /srv/gitlab/config:/etc/gitlab" + - " --volume /srv/gitlab/data:/var/opt/gitlab" + - " --volume /srv/gitlab/logs:/var/log/gitlab" + - " --shm-size 256m gitlab/gitlab-ce:rc" - Expect(cmdBuild).To(Equal(cmdExpect)) - }) + opts = buildOpts() + }) + + It("should return the correct command", func() { + cmdBuild, err := BuildContainerRunCommand(opts) + Expect(err).NotTo(HaveOccurred()) + cmdExpect := "docker run --detach --hostname gitlab.example.com" + + " --publish 8122:22 --publish 8180:80 --publish 8443:443" + + " --name gitlab --restart always" + + " --volume /srv/gitlab/config:/etc/gitlab" + + " --volume /srv/gitlab/data:/var/opt/gitlab" + + " --volume /srv/gitlab/logs:/var/log/gitlab" + + " --shm-size 256m gitlab/gitlab-ce:rc" + Expect(cmdBuild).To(Equal(cmdExpect)) + }) + + }) +}) + +var _ = Describe("build[] PortPublish func ", func() { + var portBindings []string + + BeforeEach(func() { + portBindings = []string{ + "22/tcp->8122", + "443/tcp->8443", + "80/tcp->8180", + } + }) + When(" the options are valid", func() { + It("should return the correct port publishes", func() { + publishes, err := buildPortPublishes(portBindings) + Expect(err).NotTo(HaveOccurred()) + Expect(publishes).To(Equal([]docker.PortPublish{ + {HostPort: 8122, ContainerPort: 22}, + {HostPort: 8443, ContainerPort: 443}, + {HostPort: 8180, ContainerPort: 80}, + })) }) }) }) diff --git a/pkg/util/docker/option.go b/pkg/util/docker/option.go index 3e7377e6b..4b4a8d8fb 100644 --- a/pkg/util/docker/option.go +++ b/pkg/util/docker/option.go @@ -3,6 +3,8 @@ package docker import ( "fmt" "strings" + + "go.uber.org/multierr" ) // RunOptions is used to pass options to ContainerRunWithOptions @@ -13,8 +15,9 @@ type ( Hostname string ContainerName string PortPublishes []PortPublish - Volumes []Volume + Volumes Volumes RestartAlways bool + RunParams []string } Volume struct { @@ -49,26 +52,13 @@ func (opts *RunOptions) Validate() error { } } - return CombineErrs(errs) + return multierr.Combine(errs...) } func CombineImageNameAndTag(imageName, tag string) string { return imageName + ":" + tag } -func CombineErrs(errs []error) error { - if len(errs) == 0 { - return nil - } - - errsString := make([]string, len(errs)) - for _, err := range errs { - errsString = append(errsString, err.Error()) - } - - return fmt.Errorf(strings.Join(errsString, ";")) -} - func (volumes Volumes) ExtractHostPaths() []string { hostPaths := make([]string, len(volumes)) for i, volume := range volumes { From 9f34d7a9daa0f298aba801debf31d746c451a0cc Mon Sep 17 00:00:00 2001 From: Bird Date: Tue, 2 Aug 2022 20:20:06 +0800 Subject: [PATCH 2/3] fix: forget to assign `RmDataAfterDelete` Signed-off-by: Bird --- internal/pkg/plugin/gitlabcedocker/options.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/pkg/plugin/gitlabcedocker/options.go b/internal/pkg/plugin/gitlabcedocker/options.go index 1b7608799..b53c5ff53 100644 --- a/internal/pkg/plugin/gitlabcedocker/options.go +++ b/internal/pkg/plugin/gitlabcedocker/options.go @@ -21,6 +21,7 @@ type Options struct { func buildDockerOptions(opts *Options) *dockerInstaller.Options { dockerOpts := &dockerInstaller.Options{} + dockerOpts.RmDataAfterDelete = opts.RmDataAfterDelete dockerOpts.ImageName = gitlabImageName dockerOpts.ImageTag = opts.ImageTag dockerOpts.Hostname = opts.Hostname From 68ad6b28ef7d5c92e8566a39373f3d6655d15cdb Mon Sep 17 00:00:00 2001 From: Bird Date: Wed, 3 Aug 2022 15:26:30 +0800 Subject: [PATCH 3/3] chore: minor changes in gitlab-ce-docker and docker installer Signed-off-by: Bird --- internal/pkg/plugin/gitlabcedocker/create.go | 2 +- internal/pkg/plugin/gitlabcedocker/delete.go | 2 -- .../plugin/gitlabcedocker/gitlabcedocker.go | 15 +++---------- internal/pkg/plugin/gitlabcedocker/read.go | 2 +- internal/pkg/plugin/gitlabcedocker/update.go | 2 +- .../pkg/plugin/gitlabcedocker/validate.go | 4 ++-- .../pkg/plugininstaller/docker/installer.go | 2 +- internal/pkg/plugininstaller/docker/state.go | 21 ++++++++++++------- internal/pkg/plugininstaller/docker/util.go | 7 +++++-- 9 files changed, 27 insertions(+), 30 deletions(-) diff --git a/internal/pkg/plugin/gitlabcedocker/create.go b/internal/pkg/plugin/gitlabcedocker/create.go index e2c71b278..5641e7aab 100644 --- a/internal/pkg/plugin/gitlabcedocker/create.go +++ b/internal/pkg/plugin/gitlabcedocker/create.go @@ -13,7 +13,7 @@ func Create(options map[string]interface{}) (map[string]interface{}, error) { return nil, err } - gitlabURL = getGitLabURL(opts) + gitlabURL = opts.getGitLabURL() // 2. config install operations runner := &plugininstaller.Runner{ diff --git a/internal/pkg/plugin/gitlabcedocker/delete.go b/internal/pkg/plugin/gitlabcedocker/delete.go index 96d3f903a..d4ae468b4 100644 --- a/internal/pkg/plugin/gitlabcedocker/delete.go +++ b/internal/pkg/plugin/gitlabcedocker/delete.go @@ -12,8 +12,6 @@ func Delete(options map[string]interface{}) (bool, error) { return false, err } - gitlabURL = getGitLabURL(opts) - // 2. config delete operations runner := &plugininstaller.Runner{ PreExecuteOperations: []plugininstaller.MutableOperation{ diff --git a/internal/pkg/plugin/gitlabcedocker/gitlabcedocker.go b/internal/pkg/plugin/gitlabcedocker/gitlabcedocker.go index 49c8a4335..215169026 100644 --- a/internal/pkg/plugin/gitlabcedocker/gitlabcedocker.go +++ b/internal/pkg/plugin/gitlabcedocker/gitlabcedocker.go @@ -1,8 +1,7 @@ package gitlabcedocker import ( - "strconv" - "strings" + "fmt" "github.com/devstream-io/devstream/internal/pkg/plugininstaller" "github.com/devstream-io/devstream/pkg/util/log" @@ -18,16 +17,8 @@ const ( // gitlabURL is the access URL of GitLab. var gitlabURL string -func getGitLabURL(opts *Options) string { - accessUrl := opts.Hostname - if opts.HTTPPort != 80 { - accessUrl += ":" + strconv.Itoa(int(opts.HTTPPort)) - } - if !strings.HasPrefix(accessUrl, "http") { - accessUrl = "http://" + accessUrl - } - - return accessUrl +func (opts *Options) getGitLabURL() string { + return fmt.Sprintf("http://%s:%d", opts.Hostname, opts.HTTPPort) } func showGitLabURL(options plugininstaller.RawOptions) error { diff --git a/internal/pkg/plugin/gitlabcedocker/read.go b/internal/pkg/plugin/gitlabcedocker/read.go index 894bbb82e..7581c1744 100644 --- a/internal/pkg/plugin/gitlabcedocker/read.go +++ b/internal/pkg/plugin/gitlabcedocker/read.go @@ -13,7 +13,7 @@ func Read(options map[string]interface{}) (map[string]interface{}, error) { return nil, err } - gitlabURL = getGitLabURL(opts) + gitlabURL = opts.getGitLabURL() // 2. config read operations runner := &plugininstaller.Runner{ diff --git a/internal/pkg/plugin/gitlabcedocker/update.go b/internal/pkg/plugin/gitlabcedocker/update.go index 2486f5049..a9e959bab 100644 --- a/internal/pkg/plugin/gitlabcedocker/update.go +++ b/internal/pkg/plugin/gitlabcedocker/update.go @@ -13,7 +13,7 @@ func Update(options map[string]interface{}) (map[string]interface{}, error) { return nil, err } - gitlabURL = getGitLabURL(opts) + gitlabURL = opts.getGitLabURL() // 2. config install operations runner := &plugininstaller.Runner{ diff --git a/internal/pkg/plugin/gitlabcedocker/validate.go b/internal/pkg/plugin/gitlabcedocker/validate.go index d7cd46692..6e99405ec 100644 --- a/internal/pkg/plugin/gitlabcedocker/validate.go +++ b/internal/pkg/plugin/gitlabcedocker/validate.go @@ -2,7 +2,7 @@ package gitlabcedocker import ( "fmt" - "path" + "path/filepath" "github.com/mitchellh/mapstructure" @@ -35,7 +35,7 @@ func defaults(opts *Options) { func validate(options *Options) error { errs := validator.Struct(options) // volume directory must be absolute path - if !path.IsAbs(options.GitLabHome) { + if !filepath.IsAbs(options.GitLabHome) { errs = append(errs, fmt.Errorf("GitLabHome must be an absolute path")) } diff --git a/internal/pkg/plugininstaller/docker/installer.go b/internal/pkg/plugininstaller/docker/installer.go index 6ee6115f3..3516fad5b 100644 --- a/internal/pkg/plugininstaller/docker/installer.go +++ b/internal/pkg/plugininstaller/docker/installer.go @@ -153,7 +153,7 @@ func Delete(options plugininstaller.RawOptions) error { if opts.RmDataAfterDelete { volumesDirFromOptions := opts.Volumes.ExtractHostPaths() for _, volume := range volumesDirFromOptions { - if exist := pathExist(volume); exist { + if exist := PathExist(volume); exist { errs = append(errs, fmt.Errorf("failed to delete volume %s", volume)) } } diff --git a/internal/pkg/plugininstaller/docker/state.go b/internal/pkg/plugininstaller/docker/state.go index 9fea94dbb..8423b9b81 100644 --- a/internal/pkg/plugininstaller/docker/state.go +++ b/internal/pkg/plugininstaller/docker/state.go @@ -1,6 +1,10 @@ package docker import ( + "fmt" + + "github.com/mitchellh/mapstructure" + "github.com/devstream-io/devstream/internal/pkg/plugininstaller" "github.com/devstream-io/devstream/pkg/util/docker" "github.com/devstream-io/devstream/pkg/util/log" @@ -13,13 +17,14 @@ type State struct { PortPublishes []docker.PortPublish } -func (s *State) toMap() map[string]interface{} { - return map[string]interface{}{ - "ContainerRunning": s.ContainerRunning, - "Volumes": s.Volumes, - "Hostname": s.Hostname, - "PortPublishes": s.PortPublishes, +func (s *State) toMap() (map[string]interface{}, error) { + m := make(map[string]interface{}) + err := mapstructure.Decode(s, &m) + if err != nil { + return nil, fmt.Errorf("failed to convert state to map: %v", err) } + + return m, nil } func GetStaticStateFromOptions(options plugininstaller.RawOptions) (map[string]interface{}, error) { @@ -35,7 +40,7 @@ func GetStaticStateFromOptions(options plugininstaller.RawOptions) (map[string]i PortPublishes: opts.PortPublishes, } - return staticState.toMap(), nil + return staticState.toMap() } func GetRunningState(options plugininstaller.RawOptions) (map[string]interface{}, error) { @@ -82,5 +87,5 @@ func GetRunningState(options plugininstaller.RawOptions) (map[string]interface{} PortPublishes: PortPublishes, } - return resource.toMap(), nil + return resource.toMap() } diff --git a/internal/pkg/plugininstaller/docker/util.go b/internal/pkg/plugininstaller/docker/util.go index ac68f7063..c310d8a1a 100644 --- a/internal/pkg/plugininstaller/docker/util.go +++ b/internal/pkg/plugininstaller/docker/util.go @@ -17,7 +17,10 @@ func RemoveDirs(dirs []string) []error { return errs } -func pathExist(path string) bool { +func PathExist(path string) bool { _, err := os.Stat(path) - return err == nil + if err == nil { + return true + } + return os.IsExist(err) }