diff --git a/.github/ISSUE_TEMPLATE/documentation.yaml b/.github/ISSUE_TEMPLATE/documentation.yaml index 6989a8dfd..9c252042a 100644 --- a/.github/ISSUE_TEMPLATE/documentation.yaml +++ b/.github/ISSUE_TEMPLATE/documentation.yaml @@ -14,9 +14,6 @@ body: id: terms attributes: label: Please read the documents below. - description: | - 1. [Creating a Documentation for DevStream](https://docs.devstream.io/en/latest/development/mkdocs/) - 2. [Document Translation(Chinese only)](https://docs.devstream.io/en/latest/development/translation.zh/) options: - - label: I've read the documents above. (If you want to submit a document type contribution.) + - label: I've read the document [Creating a Documentation for DevStream](https://docs.devstream.io/en/latest/development/mkdocs/) and [Document Translation(Chinese only)](https://docs.devstream.io/en/latest/development/translation.zh/). (If you want to submit a document type contribution.) required: false diff --git a/.github/workflows/link-pr.yml b/.github/workflows/link-pr.yml index b2e6d26fe..83f34c7fc 100644 --- a/.github/workflows/link-pr.yml +++ b/.github/workflows/link-pr.yml @@ -5,6 +5,7 @@ on: branches: [ main ] paths: - '**.md' + - '.lycheeignore' jobs: linkChecker: @@ -12,10 +13,6 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Download Exclude Path - run: | - curl https://raw.githubusercontent.com/devstream-io/devstream/main/.lycheeignore --output .lycheeignore - # replace site base url to build MkDocs in local - name: Replace Base URL run: | diff --git a/.gitignore b/.gitignore index cf654253d..4d312757d 100644 --- a/.gitignore +++ b/.gitignore @@ -40,7 +40,7 @@ e2e-test-local.yaml # .devstream .devstream/ -devstream.state +*.state # e2e testbin/ diff --git a/.lycheeignore b/.lycheeignore index 3f022bc18..0fec81f42 100644 --- a/.lycheeignore +++ b/.lycheeignore @@ -1,6 +1,8 @@ https://github.com/.*/pull/[0-9]+.* https://github.com/.*/issues/[0-9]+.* https://fonts.gstatic.com/ +.*\$\{.*\}.* +.*ssh.* .*foo.* .*bar.* .*xxx.* diff --git a/docs/plugins/argocd.md b/docs/plugins/argocd.md index 6ce5b6792..3d07e36c7 100644 --- a/docs/plugins/argocd.md +++ b/docs/plugins/argocd.md @@ -8,4 +8,15 @@ This plugin installs [ArgoCD](https://argoproj.github.io/cd/) in an existing Kub --8<-- "argocd.yaml" ``` -Currently, except for `values_yaml`, all the parameters in the example above are mandatory. +### Default Configs + +| key | default value | description | +| ---- | ---- | ---- | +| chart.chart_name | argo/argo-cd | argocd's official chart name | +| chart.timeout | 5m | this config will wait 5 minutes to deploy argocd | +| upgradeCRDs | true | default update CRD config | +| chart.wait | true | whether to wait until install is complete | +| repo.url | https://argoproj.github.io/argo-helm | helm repo address | +| repo.name | argo | helm repo name | + +Currently, except for `values_yaml` and default configs, all the parameters in the example above are mandatory. \ No newline at end of file diff --git a/docs/plugins/artifactory.md b/docs/plugins/artifactory.md index 184048a03..fdaaa76e4 100644 --- a/docs/plugins/artifactory.md +++ b/docs/plugins/artifactory.md @@ -10,15 +10,16 @@ If you want to **test the plugin locally**， The following `values_yaml` config ```yaml values_yaml: | - artifactory: - service: - type: NodePort - nodePort: 30002 - nginx: - enabled: false + artifactory: + service: + type: NodePort + nodePort: 30002 + nginx: + enabled: false ``` In this configuration + - Postgresql dependencies are automatically created. - local disks on machines in the cluster are defaulted used for data mounting. - Using `nodePort` to expose service, You can access `artifactory` by domain `http://{{k8s node IP}}:30002`. The default account name and password are admin/password (please replace the default account password in the production environment). @@ -43,4 +44,4 @@ This plugin support`Ingress`, `ClusterIP`, `NodePort` and `LoadBalancer` , You c --8<-- "artifactory.yaml" ``` -Currently, except for `values_yaml`, all the parameters in the example above are mandatory. \ No newline at end of file +Currently, except for `values_yaml`, all the parameters in the example above are mandatory. diff --git a/docs/plugins/artifactory.zh.md b/docs/plugins/artifactory.zh.md index 6db7e2d49..51edbb9ae 100644 --- a/docs/plugins/artifactory.zh.md +++ b/docs/plugins/artifactory.zh.md @@ -10,15 +10,16 @@ ```yaml values_yaml: | - artifactory: - service: - type: NodePort - nodePort: 30002 - nginx: - enabled: false + artifactory: + service: + type: NodePort + nodePort: 30002 + nginx: + enabled: false ``` 在该配置下 + - helm 会自动创建依赖的 Postgresql； - 数据挂载的磁盘默认会使用集群上机器的本地磁盘； - 通过 `NodePort` 对外暴露服务，可使用 `http://{{k8s 节点ip}}:30002` 域名来访问，默认账号名密码为 admin/password (生产环境请替换默认账号密码)。 @@ -34,6 +35,7 @@ values_yaml: | 可以设置 `customVolumes` 和 `customVolumeMounts` 来配置挂载磁盘，具体配置可参考 [Config](https://www.jfrog.com/confluence/display/JFROG/Configuring+the+Filestore)。 #### 网络层配置 + 该插件支持 `Ingress`, `ClusterIP`, `NodePort`, `LoadBalancer` 对外暴露的模式，可以基于需求进行选择。 ### 配置 diff --git a/docs/plugins/gitlab-ce-docker.md b/docs/plugins/gitlab-ce-docker.md index 9d4d53ec5..18e7bb825 100644 --- a/docs/plugins/gitlab-ce-docker.md +++ b/docs/plugins/gitlab-ce-docker.md @@ -1,37 +1,83 @@ -# gitlab-ce-docker plugin +# gitlab-ce-docker Plugin -This plugin installs [Gitlab-CE](https://about.gitlab.com/) in an existing docker, and the container name is `gitlab`. -## Usage +This plugin installs [GitLab](https://about.gitlab.com/) CE(Community Edition) on Docker. + +_NOTICE: currently, this plugin support Linux only._ + +## Background + +GitLab officially provides an image [gitlab-ce](https://registry.hub.docker.com/r/gitlab/gitlab-ce). We can use this image to start a container: + +```shell +docker run --detach \ + --hostname gitlab.example.com \ + --publish 443:443 --publish 80:80 --publish 22:22 \ + --name gitlab \ + --restart always \ + --volume $GITLAB_HOME/config:/etc/gitlab \ + --volume $GITLAB_HOME/logs:/var/log/gitlab \ + --volume $GITLAB_HOME/data:/var/opt/gitlab \ + --shm-size 256m \ + gitlab/gitlab-ce:rc +``` + +The variable `$GITLAB_HOME` here pointing to the directory where the configuration, logs, and data files will reside. + +We can set this variable by the `export` command: + +```shell +export GITLAB_HOME=/srv/gitlab +``` + +The GitLab container uses host mounted volumes to store persistent data: + +| Local location |Container location | Usage | +| --------------------- | ----------------- | ------------------------------------------ | +| `$GITLAB_HOME/data` | `/var/opt/gitlab` | For storing application data | +| `$GITLAB_HOME/logs` | `/var/log/gitlab` | For storing logs | +| `$GITLAB_HOME/config` | `/etc/gitlab` | For storing the GitLab configuration files | + +So, we can customize the following configurations: + +1. hostname +2. host port +3. persistent data path +4. docker image tag + +## Configuration Note: -1. the user must be `root` or in `docker` group. -2. https not support now(todo). +1. the user you are using must be `root` or in the `docker` group; +2. `https` isn't supported for now. ```yaml - --8<-- "gitlab-ce-docker.yaml" +``` + +## Some Commands That May Help + +- clone code +```shell +export hostname=YOUR_HOSTNAME +export username=YOUR_USERNAME +export project=YOUR_PROJECT_NAME ``` -## Next -Here are some commands that may help you: +1. ssh -get password of user root in gitlab-ce-docker ```shell -sudo docker exec -it gitlab grep 'Password:' /etc/gitlab/initial_root_password +# port is 22 +git clone git@${hostname}/${username}/${project}.git +# port is not 22, 2022 as a sample +git clone ssh://git@${hostname}:2022/${username}/${project}.git ``` -git clone: +2. http + ```shell -#ssh -# 22 port -git clone git@hostname/.../xxx.git -# if not 22 port -git clone ssh://git@hostname:port/.../xxx.git - -# http -# 80 port -git clone http://hostname/.../xxx.git -# if not 80 port -git clone http://hostname:port/.../xxx.git +# port is 80 +git clone http://${hostname}/${username}/${project}.git +# port is not 80, 8080 as a sample +git clone http://${hostname}:8080/${username}/${project}.git ``` diff --git a/docs/plugins/gitlab-ce-docker.zh.md b/docs/plugins/gitlab-ce-docker.zh.md index edfce1b21..2c2c5f97f 100644 --- a/docs/plugins/gitlab-ce-docker.zh.md +++ b/docs/plugins/gitlab-ce-docker.zh.md @@ -1,36 +1,81 @@ # gitlab-ce-docker 插件 -这个插件用来在本机已存在的 Docker 上安装 [Gitlab-CE](https://about.gitlab.com/), 容器名为 `gitlab`。 -## 使用 +这个插件用于以 Docker 的方式安装 [GitLab](https://about.gitlab.com/) CE（社区版）。 + +_注意：目前本插件仅支持 Linux。_ + +## 背景知识 + +GitLab 官方提供了 [gitlab-ce](https://registry.hub.docker.com/r/gitlab/gitlab-ce) 镜像，通过这个镜像我们可以实现类似这样的命令来启动一个 GitLab 容器： + +```shell +docker run --detach \ + --hostname gitlab.example.com \ + --publish 443:443 --publish 80:80 --publish 22:22 \ + --name gitlab \ + --restart always \ + --volume $GITLAB_HOME/config:/etc/gitlab \ + --volume $GITLAB_HOME/logs:/var/log/gitlab \ + --volume $GITLAB_HOME/data:/var/opt/gitlab \ + --shm-size 256m \ + gitlab/gitlab-ce:rc +``` + +其中 $GITLAB_HOME 表示的是本地存储卷路径，比如我们可以通过 export 命令来设置这个变量： + +```shell +export GITLAB_HOME=/srv/gitlab +``` + +在上述命令中，我们可以看到这个容器使用了3个存储卷，含义分别如下： + +| 本地路径 | 容器内路径 | 用途 | +| --------------------- | ----------------- | ----------------- | +| `$GITLAB_HOME/data` | `/var/opt/gitlab` | 保存应用数据 | +| `$GITLAB_HOME/logs` | `/var/log/gitlab` | 保存日志 | +| `$GITLAB_HOME/config` | `/etc/gitlab` | 保存 GitLab 配置文件 | + +在此基础上，我们可以自定义如下一些配置： + +1. hostname +2. 本机端口 +3. 存储卷路径 +4. 镜像版本 + +## 配置 注意: -1. 执行本插件的用户，必须在 `docker` 用户组内，或者是 `root` -2. 目前暂不支持 `https` 访问 gitlab +1. 你使用的用户必须是 `root` 或者在 `docker` 用户组里； +2. 目前暂不支持 `https` 方式访问 GitLab。 ```yaml - --8<-- "gitlab-ce-docker.yaml" +``` + +## 一些可能有用的命令 + +- 克隆项目 +```shell +export hostname=YOUR_HOSTNAME +export username=YOUR_USERNAME +export project=YOUR_PROJECT_NAME ``` -## 可能会用到的命令 +1. ssh 方式 -查看 gitlab 的 root 用户的密码: ```shell -sudo docker exec -it gitlab grep 'Password:' /etc/gitlab/initial_root_password +# port is 22 +git clone git@${hostname}/${username}/${project}.git +# port is not 22, 2022 as a sample +git clone ssh://git@${hostname}:2022/${username}/${project}.git ``` -克隆项目： +2. http 方式 + ```shell -#ssh -# 22 port -git clone git@hostname/.../xxx.git -# if not 22 port -git clone ssh://git@hostname:port/.../xxx.git - -# http -# 80 port -git clone http://hostname/.../xxx.git -# if not 80 port -git clone http://hostname:port/.../xxx.git +# port is 80 +git clone http://${hostname}/${username}/${project}.git +# port is not 80, 8080 as a sample +git clone http://${hostname}:8080/${username}/${project}.git ``` diff --git a/docs/plugins/gitlabci-java.md b/docs/plugins/gitlabci-java.md index 387a32aa9..e23e87b2e 100644 --- a/docs/plugins/gitlabci-java.md +++ b/docs/plugins/gitlabci-java.md @@ -13,7 +13,5 @@ This plugin set up Gitlab Pipeline in an existing Gitlab Java repository. 3. If `Deploy` is enabled, you need to offer the Gitlab Kubernetes agent name(see [Gitlab-Kubernetes](https://docs.gitlab.cn/jh/user/clusters/agent/) for more details). This will deploy the new built application to your Kubernetes cluster. This step will use `deployment.yaml` to automatically deploy the application. Please create `manifests` directory in the repository root and create your `deployment.yaml` configuration file in it. ```yaml - --8<-- "gitlabci-java.yaml" - ``` diff --git a/docs/plugins/gitlabci-java.zh.md b/docs/plugins/gitlabci-java.zh.md index 9a78d1af0..d38542825 100644 --- a/docs/plugins/gitlabci-java.zh.md +++ b/docs/plugins/gitlabci-java.zh.md @@ -11,7 +11,5 @@ 3. 如果`Deploy`选项被开启，你需要提供Gitlab配置的Kubernetes代理名称（设置详情参照[Gitlab-Kubernetes](https://docs.gitlab.cn/jh/user/clusters/agent/))。这会将新构建的应用部署至上面提供的Kubernetes集群中。该步骤会使用`deployment.yaml`来自动部署应用，请在仓库根目录下创建`manifests`目录，并在其中新建你的`deployment.yaml`配置文件。 ```yaml - --8<-- "gitlabci-java.yaml" - ``` diff --git a/docs/plugins/harbor.md b/docs/plugins/harbor.md index bebea3fa1..77e28ebd3 100644 --- a/docs/plugins/harbor.md +++ b/docs/plugins/harbor.md @@ -10,6 +10,7 @@ If you want to **test the plugin locally**， The following `values_yaml` config ```yaml values_yaml: | + externalURL: http://127.0.0.1 expose: type: nodePort tls: @@ -25,6 +26,7 @@ values_yaml: | ``` In this configuration + - Postgresql and Redis dependencies are automatically created. - local disks on machines in the cluster are defaulted used for data mounting. - Only the `harbor` main program is installed, not the rest of the plug-ins. diff --git a/docs/plugins/harbor.zh.md b/docs/plugins/harbor.zh.md index a06a3171e..5717597a4 100644 --- a/docs/plugins/harbor.zh.md +++ b/docs/plugins/harbor.zh.md @@ -10,6 +10,7 @@ ```yaml values_yaml: | + externalURL: http://127.0.0.1 expose: type: nodePort tls: @@ -25,6 +26,7 @@ values_yaml: | ``` 在该配置下 + - helm 会自动创建依赖的 Postgresql 和 Redis； - 数据挂载的磁盘默认会使用集群上机器的本地磁盘； - 只安装 `harbor` 主程序而不会安装其余的插件； diff --git a/docs/plugins/jenkins-github-integ.md b/docs/plugins/jenkins-github-integ.md index 427efe681..530127d58 100644 --- a/docs/plugins/jenkins-github-integ.md +++ b/docs/plugins/jenkins-github-integ.md @@ -4,7 +4,5 @@ TODO(aFlyBird0): Add your document here. ## Usage ```yaml - --8<-- "jenkins-github-integ.yaml" - ``` diff --git a/docs/plugins/jenkins-github-integ.zh.md b/docs/plugins/jenkins-github-integ.zh.md index ea31e847b..c56d4c865 100644 --- a/docs/plugins/jenkins-github-integ.zh.md +++ b/docs/plugins/jenkins-github-integ.zh.md @@ -32,9 +32,7 @@ tools: ## 用例 ```yaml - --8<-- "jenkins-github-integ.yaml" - ``` ## 和 `jenkins` 插件一起使用 diff --git a/docs/plugins/jenkins-pipeline-kubernetes.md b/docs/plugins/jenkins-pipeline-kubernetes.md index 58776403d..ff273092e 100644 --- a/docs/plugins/jenkins-pipeline-kubernetes.md +++ b/docs/plugins/jenkins-pipeline-kubernetes.md @@ -4,7 +4,5 @@ TODO(aFlyBird0): Add your document here. ## Usage ```yaml - --8<-- "jenkins-pipeline-kubernetes.yaml" - ``` diff --git a/docs/plugins/jenkins-pipeline-kubernetes.zh.md b/docs/plugins/jenkins-pipeline-kubernetes.zh.md index ccefb8840..bcccdc4e5 100644 --- a/docs/plugins/jenkins-pipeline-kubernetes.zh.md +++ b/docs/plugins/jenkins-pipeline-kubernetes.zh.md @@ -12,9 +12,7 @@ ## 用例 ```yaml - --8<-- "jenkins-pipeline-kubernetes.yaml" - ``` ## 和 `jenkins` 插件一起使用 diff --git a/docs/plugins/tekton.md b/docs/plugins/tekton.md index 5c5531628..196c0a106 100644 --- a/docs/plugins/tekton.md +++ b/docs/plugins/tekton.md @@ -1,42 +1,10 @@ # tekton Plugin -This plugin installs [tekton]("https://tekton.dev/") in an existing Kubernetes cluster using the Helm chart. +This plugin installs [tekton](https://tekton.dev/) in an existing Kubernetes cluster using the Helm chart. ## Usage ```yaml -tools: - # name of the tool - - name: tekton - # id of the tool instance - instanceID: default - # format: name.instanceID; If specified, dtm will make sure the dependency is applied first before handling this tool. - dependsOn: [ ] - # options for the plugin - options: - # need to create the namespace or not, default: false - create_namespace: true - repo: - # name of the Helm repo - name: tekton - # url of the Helm repo, use self host helm config beacuse official helm does'nt support namespace config - url: https://steinliber.github.io/tekton-helm-chart/ - # Helm chart information - chart: - # name of the chart - chart_name: tekton/tekton-pipeline - # k8s namespace where Tekton will be installed - namespace: tekton - # release name of the chart - release_name: tekton - # whether to wait for the release to be deployed or not - wait: true - # the time to wait for any individual Kubernetes operation (like Jobs for hooks). This defaults to 5m0s - timeout: 5m - # whether to perform a CRD upgrade during installation - upgradeCRDs: true - values_yaml: | - serviceaccount: - enabled: true +--8<-- "tekton.yaml" ``` Currently, except for `values_yaml`, all the parameters in the example above are mandatory. diff --git a/docs/plugins/tekton.zh.md b/docs/plugins/tekton.zh.md new file mode 100644 index 000000000..e76ad4d6e --- /dev/null +++ b/docs/plugins/tekton.zh.md @@ -0,0 +1,9 @@ +# tekton 插件 + +TODO(dtm): 在这里添加文档. + +## 用例 + +```yaml +--8<-- "tekton.yaml" +``` diff --git a/go.mod b/go.mod index 92e034c52..c4d78b4d1 100644 --- a/go.mod +++ b/go.mod @@ -30,6 +30,7 @@ require ( github.com/tcnksm/go-input v0.0.0-20180404061846-548a7d7a8ee8 github.com/withfig/autocomplete-tools/integrations/cobra v0.0.0-20220721102007-67b2515c5ea4 github.com/xanzy/go-gitlab v0.55.1 + go.uber.org/multierr v1.6.0 golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3 golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 gopkg.in/gookit/color.v1 v1.1.6 @@ -185,6 +186,7 @@ require ( github.com/xeipuuv/gojsonschema v1.2.0 // indirect github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca // indirect go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect + go.uber.org/atomic v1.7.0 // indirect golang.org/x/exp v0.0.0-20210901193431-a062eea981d2 // indirect golang.org/x/net v0.0.0-20220225172249-27dd8689420f // indirect golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect diff --git a/go.sum b/go.sum index c5ca0029a..c3bda336e 100644 --- a/go.sum +++ b/go.sum @@ -1345,11 +1345,13 @@ go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= +go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.8.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= diff --git a/hack/quick-start/quickstart.sh b/hack/quick-start/quickstart.sh index 5c0344c56..430b93cd5 100755 --- a/hack/quick-start/quickstart.sh +++ b/hack/quick-start/quickstart.sh @@ -23,7 +23,11 @@ function init() { } function getLatestReleaseVersion() { - latestVersion=$(curl -s https://api.github.com/repos/devstream-io/devstream/releases/latest | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/') + if [ -n "${GITHUB_TOKEN}" ]; then + AUTH_HEADER="-H Authorization: token ${GITHUB_TOKEN}" + fi + + latestVersion=$(curl ${AUTH_HEADER} -s https://api.github.com/repos/devstream-io/devstream/releases/latest | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/') if [ -z "$latestVersion" ]; then echo "Failed to get latest release version" exit 1 diff --git a/internal/pkg/develop/plugin/template/NAME_plugin.md.go b/internal/pkg/develop/plugin/template/NAME_plugin.md.go index 09f852918..c4a11fc2f 100644 --- a/internal/pkg/develop/plugin/template/NAME_plugin.md.go +++ b/internal/pkg/develop/plugin/template/NAME_plugin.md.go @@ -4,7 +4,7 @@ var NAME_plugin_md_nameTpl = "{{ .Name }}.md" var NAME_plugin_md_dirTpl = "docs/plugins/" // TODO(daniel-hutao): * -> ` -var NAME_plugin_md_contentTpl = "# {{ .Name }} plugin\n\nTODO(dtm): Add your document here.\n## Usage\n\n" + "```" + "yaml\n\n--8<-- \"{{ .Name }}.yaml\"\n\n" + "```" +var NAME_plugin_md_contentTpl = "# {{ .Name }} plugin\n\nTODO(dtm): Add your document here.\n## Usage\n\n" + "```" + "yaml\n--8<-- \"{{ .Name }}.yaml\"\n" + "```" func init() { diff --git a/internal/pkg/develop/plugin/template/NAME_plugin.zh.md.go b/internal/pkg/develop/plugin/template/NAME_plugin.zh.md.go index 034f2a16d..85bbfb561 100644 --- a/internal/pkg/develop/plugin/template/NAME_plugin.zh.md.go +++ b/internal/pkg/develop/plugin/template/NAME_plugin.zh.md.go @@ -4,7 +4,7 @@ var NAME_plugin_zh_md_nameTpl = "{{ .Name }}.zh.md" var NAME_plugin_zh_md_dirTpl = "docs/plugins/" // TODO(daniel-hutao): * -> ` -var NAME_plugin_zh_md_contentTpl = "# {{ .Name }} 插件\n\nTODO(dtm): 在这里添加文档.\n## 用例\n\n" + "```" + "yaml\n\n--8<-- \"{{ .Name }}.yaml\"\n\n" + "```" +var NAME_plugin_zh_md_contentTpl = "# {{ .Name }} 插件\n\nTODO(dtm): 在这里添加文档.\n\n## 用例\n\n" + "```" + "yaml\n--8<-- \"{{ .Name }}.yaml\"\n" + "```" func init() { diff --git a/internal/pkg/plugin/argocd/argocd.go b/internal/pkg/plugin/argocd/argocd.go index 1ed389e97..61104fca5 100644 --- a/internal/pkg/plugin/argocd/argocd.go +++ b/internal/pkg/plugin/argocd/argocd.go @@ -1,25 +1,20 @@ package argocd import ( - "github.com/devstream-io/devstream/internal/pkg/plugininstaller" "github.com/devstream-io/devstream/internal/pkg/plugininstaller/helm" + helmCommon "github.com/devstream-io/devstream/pkg/util/helm" + "github.com/devstream-io/devstream/pkg/util/types" ) -var ( - defaultRepoURL = "https://argoproj.github.io/argo-helm" - defaultRepoName = "argo" -) - -func defaultMissedOption(options plugininstaller.RawOptions) (plugininstaller.RawOptions, error) { - opts, err := helm.NewOptions(options) - if err != nil { - return nil, err - } - if opts.Repo.URL == "" { - opts.Repo.URL = defaultRepoURL - } - if opts.Repo.Name == "" { - opts.Repo.Name = defaultRepoName - } - return opts.Encode() +var defaultHelmConfig = helm.Options{ + Chart: helmCommon.Chart{ + ChartName: "argo/argo-cd", + Timeout: "5m", + UpgradeCRDs: types.Bool(true), + Wait: types.Bool(true), + }, + Repo: helmCommon.Repo{ + URL: "https://argoproj.github.io/argo-helm", + Name: "argo", + }, } diff --git a/internal/pkg/plugin/argocd/create.go b/internal/pkg/plugin/argocd/create.go index 81cade770..bf2789b94 100644 --- a/internal/pkg/plugin/argocd/create.go +++ b/internal/pkg/plugin/argocd/create.go @@ -11,7 +11,7 @@ func Create(options map[string]interface{}) (map[string]interface{}, error) { // 1. config install operations runner := &plugininstaller.Runner{ PreExecuteOperations: []plugininstaller.MutableOperation{ - defaultMissedOption, + helm.SetDefaultConfig(&defaultHelmConfig), helm.Validate, }, ExecuteOperations: helm.DefaultCreateOperations, diff --git a/internal/pkg/plugin/argocd/delete.go b/internal/pkg/plugin/argocd/delete.go index 73fddba80..f39ed557f 100644 --- a/internal/pkg/plugin/argocd/delete.go +++ b/internal/pkg/plugin/argocd/delete.go @@ -9,7 +9,7 @@ func Delete(options map[string]interface{}) (bool, error) { // 1. config delete operations runner := &plugininstaller.Runner{ PreExecuteOperations: []plugininstaller.MutableOperation{ - defaultMissedOption, + helm.SetDefaultConfig(&defaultHelmConfig), helm.Validate, }, ExecuteOperations: helm.DefaultDeleteOperations, diff --git a/internal/pkg/plugin/argocd/read.go b/internal/pkg/plugin/argocd/read.go index 058a4a62a..82d81cbdf 100644 --- a/internal/pkg/plugin/argocd/read.go +++ b/internal/pkg/plugin/argocd/read.go @@ -14,7 +14,7 @@ func Read(options map[string]interface{}) (map[string]interface{}, error) { // 1. config read operations runner := &plugininstaller.Runner{ PreExecuteOperations: []plugininstaller.MutableOperation{ - defaultMissedOption, + helm.SetDefaultConfig(&defaultHelmConfig), helm.Validate, }, GetStatusOperation: helm.GetPluginAllState, diff --git a/internal/pkg/plugin/argocd/update.go b/internal/pkg/plugin/argocd/update.go index 5e8257d22..2080e60d6 100644 --- a/internal/pkg/plugin/argocd/update.go +++ b/internal/pkg/plugin/argocd/update.go @@ -10,7 +10,7 @@ func Update(options map[string]interface{}) (map[string]interface{}, error) { // 1. config update operations runner := &plugininstaller.Runner{ PreExecuteOperations: []plugininstaller.MutableOperation{ - defaultMissedOption, + helm.SetDefaultConfig(&defaultHelmConfig), helm.Validate, }, ExecuteOperations: helm.DefaultUpdateOperations, diff --git a/internal/pkg/plugin/argocdapp/argocdapp.go b/internal/pkg/plugin/argocdapp/argocdapp.go index 2fe4dd1c8..b6c939681 100644 --- a/internal/pkg/plugin/argocdapp/argocdapp.go +++ b/internal/pkg/plugin/argocdapp/argocdapp.go @@ -1,27 +1,8 @@ package argocdapp -var argoCDAppTemplate = `--- -apiVersion: argoproj.io/v1alpha1 -kind: Application -metadata: - name: {{.app.name}} - namespace: {{.app.namespace}} - finalizers: - - resources-finalizer.argocd.argoproj.io -spec: - destination: - namespace: {{.destination.namespace}} - server: {{.destination.server}} - project: default - source: - helm: - valueFiles: - - {{.source.valuefile}} - path: {{.source.path}} - repoURL: {{.source.repoURL}} - targetRevision: HEAD - syncPolicy: - automated: - prune: true - selfHeal: true -` +import ( + _ "embed" +) + +//go:embed tpl/argocd.tpl.yaml +var templateFileLoc string diff --git a/internal/pkg/plugin/argocdapp/create.go b/internal/pkg/plugin/argocdapp/create.go index 4161316ff..a6c8e0e5a 100644 --- a/internal/pkg/plugin/argocdapp/create.go +++ b/internal/pkg/plugin/argocdapp/create.go @@ -3,6 +3,7 @@ package argocdapp import ( "github.com/devstream-io/devstream/internal/pkg/plugininstaller" "github.com/devstream-io/devstream/internal/pkg/plugininstaller/kubectl" + "github.com/devstream-io/devstream/pkg/util/file" "github.com/devstream-io/devstream/pkg/util/log" ) @@ -14,7 +15,9 @@ func Create(options map[string]interface{}) (map[string]interface{}, error) { validate, }, ExecuteOperations: []plugininstaller.BaseOperation{ - kubectl.ProcessByContent("create", "", argoCDAppTemplate), + kubectl.ProcessByContent( + "create", file.NewTemplate().FromContent(templateFileLoc), + ), }, GetStatusOperation: getStaticState, } diff --git a/internal/pkg/plugin/argocdapp/delete.go b/internal/pkg/plugin/argocdapp/delete.go index a57ddb235..87ac7f354 100644 --- a/internal/pkg/plugin/argocdapp/delete.go +++ b/internal/pkg/plugin/argocdapp/delete.go @@ -3,6 +3,7 @@ package argocdapp import ( "github.com/devstream-io/devstream/internal/pkg/plugininstaller" "github.com/devstream-io/devstream/internal/pkg/plugininstaller/kubectl" + "github.com/devstream-io/devstream/pkg/util/file" ) func Delete(options map[string]interface{}) (bool, error) { @@ -12,7 +13,9 @@ func Delete(options map[string]interface{}) (bool, error) { validate, }, ExecuteOperations: []plugininstaller.BaseOperation{ - kubectl.ProcessByContent("delete", "", argoCDAppTemplate), + kubectl.ProcessByContent( + "delete", file.NewTemplate().FromContent(templateFileLoc), + ), }, } diff --git a/internal/pkg/plugin/argocdapp/state.go b/internal/pkg/plugin/argocdapp/state.go index aea9a144e..480a87aca 100644 --- a/internal/pkg/plugin/argocdapp/state.go +++ b/internal/pkg/plugin/argocdapp/state.go @@ -6,7 +6,7 @@ import ( "github.com/cenkalti/backoff" "github.com/devstream-io/devstream/internal/pkg/plugininstaller" - "github.com/devstream-io/devstream/internal/pkg/plugininstaller/util" + "github.com/devstream-io/devstream/pkg/util/k8s" ) func getStaticState(options plugininstaller.RawOptions) (map[string]interface{}, error) { @@ -43,7 +43,7 @@ func getDynamicState(options plugininstaller.RawOptions) (map[string]interface{} state := make(map[string]interface{}) operation := func() error { - err := util.GetArgoCDAppFromK8sAndSetState(state, opts.App.Name, opts.App.Namespace) + err := getArgoCDAppFromK8sAndSetState(state, opts.App.Name, opts.App.Namespace) if err != nil { return err } @@ -57,3 +57,22 @@ func getDynamicState(options plugininstaller.RawOptions) (map[string]interface{} } return state, nil } + +func getArgoCDAppFromK8sAndSetState(state map[string]interface{}, name, namespace string) error { + kubeClient, err := k8s.NewClient() + if err != nil { + return err + } + + app, err := kubeClient.GetArgocdApplication(namespace, name) + if err != nil { + return err + } + + d := kubeClient.DescribeArgocdApp(app) + state["app"] = d["app"] + state["src"] = d["src"] + state["dest"] = d["dest"] + + return nil +} diff --git a/internal/pkg/plugin/argocdapp/tpl/argocd.tpl.yaml b/internal/pkg/plugin/argocdapp/tpl/argocd.tpl.yaml new file mode 100644 index 000000000..c5e04fe82 --- /dev/null +++ b/internal/pkg/plugin/argocdapp/tpl/argocd.tpl.yaml @@ -0,0 +1,24 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: "[[ .app.name ]]" + namespace: "[[ .app.namespace ]]" + finalizers: + - resources-finalizer.argocd.argoproj.io +spec: + destination: + namespace: "[[ .destination.namespace ]]" + server: "[[ .destination.server ]]" + project: default + source: + helm: + valueFiles: + - "[[ .source.valuefile ]]" + path: "[[ .source.path ]]" + repoURL: "[[ .source.repoURL ]]" + targetRevision: HEAD + syncPolicy: + automated: + prune: true + selfHeal: true + diff --git a/internal/pkg/plugin/devlake/create.go b/internal/pkg/plugin/devlake/create.go index d8a79bf0d..86affe362 100644 --- a/internal/pkg/plugin/devlake/create.go +++ b/internal/pkg/plugin/devlake/create.go @@ -3,6 +3,7 @@ package devlake import ( "github.com/devstream-io/devstream/internal/pkg/plugininstaller" "github.com/devstream-io/devstream/internal/pkg/plugininstaller/kubectl" + "github.com/devstream-io/devstream/pkg/util/file" "github.com/devstream-io/devstream/pkg/util/log" ) @@ -10,8 +11,9 @@ func Create(options map[string]interface{}) (map[string]interface{}, error) { // 1. config install operations runner := &plugininstaller.Runner{ ExecuteOperations: []plugininstaller.BaseOperation{ - kubectl.ProcessByContent("create", devLakeInstallYAMLDownloadURL, ""), - kubectl.WaitDeployReadyWithDeployList(defaultNamespace, devLakeDeployments), + kubectl.ProcessByContent( + "create", file.NewTemplate().FromRemote(devLakeInstallYAMLDownloadURL), + ), }, GetStatusOperation: getStaticState, } diff --git a/internal/pkg/plugin/devlake/delete.go b/internal/pkg/plugin/devlake/delete.go index 83cb68bfe..204d1b4a2 100644 --- a/internal/pkg/plugin/devlake/delete.go +++ b/internal/pkg/plugin/devlake/delete.go @@ -3,13 +3,16 @@ package devlake import ( "github.com/devstream-io/devstream/internal/pkg/plugininstaller" "github.com/devstream-io/devstream/internal/pkg/plugininstaller/kubectl" + "github.com/devstream-io/devstream/pkg/util/file" ) func Delete(options map[string]interface{}) (bool, error) { // 1. config delete operations runner := &plugininstaller.Runner{ ExecuteOperations: []plugininstaller.BaseOperation{ - kubectl.ProcessByContent("delete", devLakeInstallYAMLDownloadURL, ""), + kubectl.ProcessByContent( + "delete", file.NewTemplate().FromRemote(devLakeInstallYAMLDownloadURL), + ), }, } diff --git a/internal/pkg/plugin/devlake/state.go b/internal/pkg/plugin/devlake/state.go index 12b803820..9680400b5 100644 --- a/internal/pkg/plugin/devlake/state.go +++ b/internal/pkg/plugin/devlake/state.go @@ -2,7 +2,7 @@ package devlake import ( "github.com/devstream-io/devstream/internal/pkg/plugininstaller" - "github.com/devstream-io/devstream/internal/pkg/plugininstaller/util" + "github.com/devstream-io/devstream/internal/pkg/plugininstaller/common" ) func getStaticState(opts plugininstaller.RawOptions) (map[string]interface{}, error) { @@ -17,5 +17,8 @@ func getStaticState(opts plugininstaller.RawOptions) (map[string]interface{}, er } func getDynamicState(opts plugininstaller.RawOptions) (map[string]interface{}, error) { - return util.ReadDepAndServiceState(defaultNamespace, devLakeDeployments) + labelFilter := map[string]string{ + "app": "devlake", + } + return common.GetPluginAllK8sState(defaultNamespace, map[string]string{}, labelFilter) } diff --git a/internal/pkg/plugin/gitlabcedocker/create.go b/internal/pkg/plugin/gitlabcedocker/create.go index 70bff0d4a..5a96d47c2 100644 --- a/internal/pkg/plugin/gitlabcedocker/create.go +++ b/internal/pkg/plugin/gitlabcedocker/create.go @@ -1,83 +1,43 @@ package gitlabcedocker import ( - "fmt" - "strconv" - "strings" - - "github.com/mitchellh/mapstructure" - - "github.com/devstream-io/devstream/pkg/util/docker" + "github.com/devstream-io/devstream/internal/pkg/plugininstaller" + dockerInstaller "github.com/devstream-io/devstream/internal/pkg/plugininstaller/docker" "github.com/devstream-io/devstream/pkg/util/log" ) func Create(options map[string]interface{}) (map[string]interface{}, error) { - var opts Options - if err := mapstructure.Decode(options, &opts); err != nil { + // 1. create config and pre-handle operations + opts, err := validateAndDefault(options) + if err != nil { return nil, err } - defaults(&opts) - - if errs := validate(&opts); len(errs) != 0 { - for _, e := range errs { - log.Errorf("Options error: %s.", e) - } - return nil, fmt.Errorf("opts are illegal") + // 2. config install operations + runner := &plugininstaller.Runner{ + PreExecuteOperations: []plugininstaller.MutableOperation{ + dockerInstaller.Validate, + }, + ExecuteOperations: []plugininstaller.BaseOperation{ + dockerInstaller.InstallOrUpdate, + showHelpMsg, + }, + TerminateOperations: []plugininstaller.BaseOperation{ + dockerInstaller.HandleRunFailure, + }, + GetStatusOperation: dockerInstaller.GetStaticStateFromOptions, } - op := GetDockerOperator(opts) - - // 1. try to pull the image - // always pull the image because docker will check the image existence - if err := op.ImagePull(getImageNameWithTag(opts)); err != nil { + // 3. execute installer get status and error + rawOptions, err := buildDockerOptions(opts).Encode() + if err != nil { return nil, err } - - // 2. try to run the container - log.Info("Running container as the name ") - if err := op.ContainerRun(buildDockerRunOptions(opts), dockerRunShmSizeParam); err != nil { - return nil, fmt.Errorf("failed to run container: %v", err) - } - - // 3. check if the container is started successfully - if ok := op.ContainerIfRunning(gitlabContainerName); !ok { - return nil, fmt.Errorf("failed to run container") - } - - // 4. check if the volume is created successfully - mounts, err := op.ContainerListMounts(gitlabContainerName) + status, err := runner.Execute(rawOptions) if err != nil { - return nil, fmt.Errorf("failed to get container mounts: %v", err) - } - volumes := mounts.ExtractSources() - if docker.IfVolumesDiffer(volumes, getVolumesDirFromOptions(opts)) { - return nil, fmt.Errorf("failed to create volumes") - } - - // 5. show the access url - showGitLabURL(opts) - - resource := gitlabResource{ - ContainerRunning: true, - Volumes: volumes, - Hostname: opts.Hostname, - SSHPort: strconv.Itoa(int(opts.SSHPort)), - HTTPPort: strconv.Itoa(int(opts.HTTPPort)), - HTTPSPort: strconv.Itoa(int(opts.HTTPSPort)), - } - - return resource.toMap(), nil -} - -func showGitLabURL(opts Options) { - accessUrl := opts.Hostname - if opts.HTTPPort != 80 { - accessUrl += ":" + strconv.Itoa(int(opts.HTTPPort)) - } - if !strings.HasPrefix(accessUrl, "http") { - accessUrl = "http://" + accessUrl + return nil, err } + log.Debugf("Return map: %v", status) - log.Infof("GitLab access URL: %s", accessUrl) + return status, nil } diff --git a/internal/pkg/plugin/gitlabcedocker/delete.go b/internal/pkg/plugin/gitlabcedocker/delete.go index 81b68d0d8..91b24f255 100644 --- a/internal/pkg/plugin/gitlabcedocker/delete.go +++ b/internal/pkg/plugin/gitlabcedocker/delete.go @@ -1,102 +1,36 @@ package gitlabcedocker import ( - "fmt" - "os" - - "github.com/mitchellh/mapstructure" - - "github.com/devstream-io/devstream/pkg/util/log" + "github.com/devstream-io/devstream/internal/pkg/plugininstaller" + dockerInstaller "github.com/devstream-io/devstream/internal/pkg/plugininstaller/docker" ) func Delete(options map[string]interface{}) (bool, error) { - var opts Options - if err := mapstructure.Decode(options, &opts); err != nil { + // 1. create config and pre-handle operations + opts, err := validateAndDefault(options) + if err != nil { return false, err } - defaults(&opts) - - if errs := validate(&opts); len(errs) != 0 { - for _, e := range errs { - log.Errorf("Options error: %s.", e) - } - return false, fmt.Errorf("opts are illegal") + // 2. config delete operations + runner := &plugininstaller.Runner{ + PreExecuteOperations: []plugininstaller.MutableOperation{ + dockerInstaller.Validate, + }, + ExecuteOperations: []plugininstaller.BaseOperation{ + dockerInstaller.Delete, + }, } - op := GetDockerOperator(opts) - - // 1. stop the container if it is running - if ok := op.ContainerIfRunning(gitlabContainerName); ok { - if err := op.ContainerStop(gitlabContainerName); err != nil { - log.Errorf("Failed to stop container: %v", err) - } - } - - // 2. remove the container if it exists - if ok := op.ContainerIfExist(gitlabContainerName); ok { - if err := op.ContainerRemove(gitlabContainerName); err != nil { - log.Errorf("failed to remove container %v: %v", gitlabContainerName, err) - } - } - - // 3. remove the image if it exists - if ok := op.ImageIfExist(getImageNameWithTag(opts)); ok { - if err := op.ImageRemove(getImageNameWithTag(opts)); err != nil { - log.Errorf("failed to remove image %v: %v", getImageNameWithTag(opts), err) - } - } - - // 4. remove the volume if it exists - volumesDirFromOptions := getVolumesDirFromOptions(opts) - if opts.RmDataAfterDelete { - for _, volume := range volumesDirFromOptions { - if err := os.RemoveAll(volume); err != nil { - log.Errorf("failed to remove data %v: %v", volume, err) - } - } - } - - var errs []error - - // 1. check if the container is stopped and deleted - if ok := op.ContainerIfRunning(gitlabContainerName); ok { - errs = append(errs, fmt.Errorf("failed to delete/stop container %s", gitlabContainerName)) - } - if ok := op.ContainerIfExist(gitlabContainerName); ok { - errs = append(errs, fmt.Errorf("failed to delete container %s", gitlabContainerName)) - } - - // 2. check if the image is removed - if ok := op.ImageIfExist(getImageNameWithTag(opts)); ok { - errs = append(errs, fmt.Errorf("failed to delete image %s", getImageNameWithTag(opts))) - } - - // 3. check if the data volume is removed - if opts.RmDataAfterDelete { - errs = append(errs, RemoveDirs(volumesDirFromOptions)...) + // 3. delete and get status + rawOptions, err := buildDockerOptions(opts).Encode() + if err != nil { + return false, err } - - // splice the errors - if len(errs) != 0 { - errsString := "" - for _, e := range errs { - errsString += e.Error() + "; " - } - return false, fmt.Errorf(errsString) + _, err = runner.Execute(rawOptions) + if err != nil { + return false, err } return true, nil } - -// RemoveDirs removes the all the directories in the given list recursively -func RemoveDirs(dirs []string) []error { - var errs []error - for _, dir := range dirs { - if err := os.RemoveAll(dir); err != nil { - errs = append(errs, err) - } - } - - return errs -} diff --git a/internal/pkg/plugin/gitlabcedocker/delete_test.go b/internal/pkg/plugin/gitlabcedocker/delete_test.go deleted file mode 100644 index 3557b330e..000000000 --- a/internal/pkg/plugin/gitlabcedocker/delete_test.go +++ /dev/null @@ -1,71 +0,0 @@ -package gitlabcedocker - -import ( - "os" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -var _ = Describe("Delete", func() { - - Describe("RemoveDirs func", func() { - - var ( - errs []error - dirs []string - ) - - AfterEach(func() { - // check directories are removed - for _, dir := range dirs { - _, err := os.Stat(dir) - Expect(os.IsNotExist(err)).To(BeTrue()) - } - }) - - When("the directories are not exist", func() { - - BeforeEach(func() { - dirs = []string{"dir/not/exist/1", "dir/not/exist/2"} - }) - - It("should return no error", func() { - // Remove the directories - errs = RemoveDirs(dirs) - for _, e := range errs { - Expect(e).ToNot(HaveOccurred()) - } - }) - }) - - When("the directories are exist", func() { - - BeforeEach(func() { - // create temp dir, it will be removed automatically after the test - parentDir := GinkgoT().TempDir() - dirs = []string{ - parentDir + "dir1", - parentDir + "dir2", - parentDir + "dir3/dir3-1", - } - // create directories - for _, dir := range dirs { - err := os.MkdirAll(dir, 0755) - Expect(err).ToNot(HaveOccurred()) - } - // create files - _, err := os.CreateTemp(dirs[0], "file1*") - Expect(err).ToNot(HaveOccurred()) - }) - - It("should remove all directories successfully", func() { - // Remove the directories - errs = RemoveDirs(dirs) - for _, e := range errs { - Expect(e).ToNot(HaveOccurred()) - } - }) - }) - }) -}) diff --git a/internal/pkg/plugin/gitlabcedocker/gitlabcedocker.go b/internal/pkg/plugin/gitlabcedocker/gitlabcedocker.go index 2183eb1be..547001a18 100644 --- a/internal/pkg/plugin/gitlabcedocker/gitlabcedocker.go +++ b/internal/pkg/plugin/gitlabcedocker/gitlabcedocker.go @@ -1,51 +1,19 @@ package gitlabcedocker -import ( - "strings" - - "github.com/devstream-io/devstream/pkg/util/docker" - "github.com/devstream-io/devstream/pkg/util/docker/dockersh" -) +import "github.com/devstream-io/devstream/pkg/util/types" const ( - gitlabImageName = "gitlab/gitlab-ce" + defaultHostname = "gitlab.example.com" + defaultGitlabHome = "/srv/gitlab" + defaultSSHPort = 22 + defaultHTTPPort = 80 + defaultHTTPSPort = 443 defaultImageTag = "rc" + gitlabImageName = "gitlab/gitlab-ce" gitlabContainerName = "gitlab" - tcp = "tcp" dockerRunShmSizeParam = "--shm-size 256m" ) -func getImageNameWithTag(opt Options) string { - return gitlabImageName + ":" + opt.ImageTag -} - -func defaults(opts *Options) { - if opts.ImageTag == "" { - opts.ImageTag = defaultImageTag - } -} - -func GetDockerOperator(_ Options) docker.Operator { - // just return a ShellOperator for now - return &dockersh.ShellOperator{} -} - -type gitlabResource struct { - ContainerRunning bool - Volumes []string - Hostname string - SSHPort string - HTTPPort string - HTTPSPort string -} - -func (res *gitlabResource) toMap() map[string]interface{} { - return map[string]interface{}{ - "containerRunning": res.ContainerRunning, - "volumes": strings.Join(res.Volumes, ","), - "hostname": res.Hostname, - "SSHPort": res.SSHPort, - "HTTPPort": res.HTTPPort, - "HTTPSPort": res.HTTPSPort, - } -} +var ( + defaultRMDataAfterDelete = types.Bool(false) +) diff --git a/internal/pkg/plugin/gitlabcedocker/options.go b/internal/pkg/plugin/gitlabcedocker/options.go index ce3a5f9a7..451a4d111 100644 --- a/internal/pkg/plugin/gitlabcedocker/options.go +++ b/internal/pkg/plugin/gitlabcedocker/options.go @@ -1,55 +1,96 @@ package gitlabcedocker import ( + "fmt" "path/filepath" + "github.com/devstream-io/devstream/internal/pkg/plugininstaller" + dockerInstaller "github.com/devstream-io/devstream/internal/pkg/plugininstaller/docker" "github.com/devstream-io/devstream/pkg/util/docker" + "github.com/devstream-io/devstream/pkg/util/log" ) // Options is the struct for configurations of the gitlab-ce-docker plugin. type Options struct { + Hostname string `validate:"hostname" mapstructure:"hostname"` // GitLab home directory, we assume the path set by user is always correct. - GitLabHome string `validate:"required" mapstructure:"gitlab_home"` - Hostname string `validate:"required,hostname" mapstructure:"hostname"` - SSHPort uint `validate:"required" mapstructure:"ssh_port"` - HTTPPort uint `validate:"required" mapstructure:"http_port"` - HTTPSPort uint `validate:"required" mapstructure:"https_port"` - RmDataAfterDelete bool `mapstructure:"rm_data_after_delete"` + GitLabHome string `mapstructure:"gitlab_home"` + SSHPort uint `mapstructure:"ssh_port"` + HTTPPort uint `mapstructure:"http_port"` + HTTPSPort uint `mapstructure:"https_port"` + RmDataAfterDelete *bool `mapstructure:"rm_data_after_delete"` ImageTag string `mapstructure:"image_tag"` } -// getVolumesDirFromOptions returns host directories of the volumes from the options. -func getVolumesDirFromOptions(opts Options) []string { - volumes := buildDockerVolumes(opts) - return volumes.ExtractHostPaths() +func (opts *Options) Defaults() { + if opts.Hostname == "" { + opts.Hostname = defaultHostname + } + if opts.GitLabHome == "" { + opts.GitLabHome = defaultGitlabHome + } + if opts.SSHPort == 0 { + opts.SSHPort = defaultSSHPort + } + if opts.HTTPPort == 0 { + opts.HTTPPort = defaultHTTPPort + } + if opts.HTTPSPort == 0 { + opts.HTTPSPort = defaultHTTPSPort + } + if opts.RmDataAfterDelete == nil { + opts.RmDataAfterDelete = defaultRMDataAfterDelete + } + if opts.ImageTag == "" { + opts.ImageTag = defaultImageTag + } } -func buildDockerVolumes(opts Options) docker.Volumes { - volumes := []docker.Volume{ - {HostPath: filepath.Join(opts.GitLabHome, "config"), ContainerPath: "/etc/gitlab"}, - {HostPath: filepath.Join(opts.GitLabHome, "data"), ContainerPath: "/var/opt/gitlab"}, - {HostPath: filepath.Join(opts.GitLabHome, "logs"), ContainerPath: "/var/log/gitlab"}, - } +// gitlabURL is the access URL of GitLab. +var gitlabURL string - return volumes +func (opts *Options) setGitLabURL() { + if gitlabURL != "" { + return + } + gitlabURL = fmt.Sprintf("http://%s:%d", opts.Hostname, opts.HTTPPort) } -func buildDockerRunOptions(opts Options) docker.RunOptions { - dockerRunOpts := docker.RunOptions{} - dockerRunOpts.ImageName = gitlabImageName - dockerRunOpts.ImageTag = opts.ImageTag - dockerRunOpts.Hostname = opts.Hostname - dockerRunOpts.ContainerName = gitlabContainerName - dockerRunOpts.RestartAlways = true +func showHelpMsg(options plugininstaller.RawOptions) error { + log.Infof("GitLab access URL: %s", gitlabURL) + log.Infof("GitLab initial root password: execute the command -> docker exec -it gitlab grep 'Password:' /etc/gitlab/initial_root_password") + + return nil +} +func buildDockerOptions(opts *Options) *dockerInstaller.Options { portPublishes := []docker.PortPublish{ {HostPort: opts.SSHPort, ContainerPort: 22}, {HostPort: opts.HTTPPort, ContainerPort: 80}, {HostPort: opts.HTTPSPort, ContainerPort: 443}, } - dockerRunOpts.PortPublishes = portPublishes - dockerRunOpts.Volumes = buildDockerVolumes(opts) + dockerOpts := &dockerInstaller.Options{ + RmDataAfterDelete: opts.RmDataAfterDelete, + ImageName: gitlabImageName, + ImageTag: opts.ImageTag, + Hostname: opts.Hostname, + ContainerName: gitlabContainerName, + RestartAlways: true, + Volumes: buildDockerVolumes(opts), + RunParams: []string{dockerRunShmSizeParam}, + PortPublishes: portPublishes, + } - return dockerRunOpts + return dockerOpts +} + +func buildDockerVolumes(opts *Options) docker.Volumes { + volumes := []docker.Volume{ + {HostPath: filepath.Join(opts.GitLabHome, "config"), ContainerPath: "/etc/gitlab"}, + {HostPath: filepath.Join(opts.GitLabHome, "data"), ContainerPath: "/var/opt/gitlab"}, + {HostPath: filepath.Join(opts.GitLabHome, "logs"), ContainerPath: "/var/log/gitlab"}, + } + + return volumes } diff --git a/internal/pkg/plugin/gitlabcedocker/options_test.go b/internal/pkg/plugin/gitlabcedocker/options_test.go index 1e552641f..de78cb906 100644 --- a/internal/pkg/plugin/gitlabcedocker/options_test.go +++ b/internal/pkg/plugin/gitlabcedocker/options_test.go @@ -4,42 +4,29 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + dockerInstaller "github.com/devstream-io/devstream/internal/pkg/plugininstaller/docker" "github.com/devstream-io/devstream/pkg/util/docker" ) var _ = Describe("Options", func() { - var opts Options - + var opts *Options BeforeEach(func() { - opts = Options{ + opts = &Options{ GitLabHome: "/srv/gitlab", Hostname: "gitlab.example.com", SSHPort: 8122, HTTPPort: 8180, HTTPSPort: 8443, - RmDataAfterDelete: false, + RmDataAfterDelete: nil, ImageTag: "rc", } }) - Describe("getVolumesDirFromOptions func", func() { - When("the options is valid", func() { - It("should return the volumes' directory", func() { - volumesDirFromOptions := getVolumesDirFromOptions(opts) - Expect(volumesDirFromOptions).To(Equal([]string{ - "/srv/gitlab/config", - "/srv/gitlab/data", - "/srv/gitlab/logs", - })) - }) - }) - }) - Describe("buildDockerRunOptions func", func() { It("should build the docker run options successfully", func() { - runOptsBuild := buildDockerRunOptions(opts) - runOptsExpect := docker.RunOptions{ + OptsBuild := *buildDockerOptions(opts) + OptsExpect := dockerInstaller.Options{ ImageName: "gitlab/gitlab-ce", ImageTag: "rc", Hostname: "gitlab.example.com", @@ -55,9 +42,10 @@ var _ = Describe("Options", func() { {HostPath: "/srv/gitlab/data", ContainerPath: "/var/opt/gitlab"}, {HostPath: "/srv/gitlab/logs", ContainerPath: "/var/log/gitlab"}, }, + RunParams: []string{dockerRunShmSizeParam}, } - Expect(runOptsBuild).To(Equal(runOptsExpect)) + Expect(OptsBuild).To(Equal(OptsExpect)) }) }) diff --git a/internal/pkg/plugin/gitlabcedocker/read.go b/internal/pkg/plugin/gitlabcedocker/read.go index 274ec988a..b8074aa24 100644 --- a/internal/pkg/plugin/gitlabcedocker/read.go +++ b/internal/pkg/plugin/gitlabcedocker/read.go @@ -1,77 +1,36 @@ package gitlabcedocker import ( - "fmt" - - "github.com/mitchellh/mapstructure" - + "github.com/devstream-io/devstream/internal/pkg/plugininstaller" + dockerInstaller "github.com/devstream-io/devstream/internal/pkg/plugininstaller/docker" "github.com/devstream-io/devstream/pkg/util/log" ) func Read(options map[string]interface{}) (map[string]interface{}, error) { - var opts Options - if err := mapstructure.Decode(options, &opts); err != nil { - return nil, err - } - - defaults(&opts) - - if errs := validate(&opts); len(errs) != 0 { - for _, e := range errs { - log.Errorf("Options error: %s.", e) - } - return nil, fmt.Errorf("opts are illegal") - } - - op := GetDockerOperator(opts) - - // 1. get running status - running := op.ContainerIfRunning(gitlabContainerName) - if !running { - return (&gitlabResource{}).toMap(), nil - } - - // 2. get volumes - mounts, err := op.ContainerListMounts(gitlabContainerName) + // 1. create config and pre-handle operations + opts, err := validateAndDefault(options) if err != nil { - // `Read` shouldn't return errors even if failed to read ports, volumes, hostname. - // because: - // 1. when the docker is stopped it could cause these errors. - // 2. if Read failed, the following steps contain the docker's restart will be aborted. - log.Errorf("failed to get container mounts: %v", err) + return nil, err } - volumes := mounts.ExtractSources() - // 3. get hostname - hostname, err := op.ContainerGetHostname(gitlabContainerName) - if err != nil { - log.Errorf("failed to get container hostname: %v", err) + // 2. config read operations + runner := &plugininstaller.Runner{ + PreExecuteOperations: []plugininstaller.MutableOperation{ + dockerInstaller.Validate, + }, + GetStatusOperation: dockerInstaller.GetRunningState, } - // 4. get port bindings - SSHPort, err := op.ContainerGetPortBinding(gitlabContainerName, "22", tcp) - if err != nil { - log.Errorf("failed to get container ssh port: %v", err) - } - HTTPPort, err := op.ContainerGetPortBinding(gitlabContainerName, "80", tcp) + // 3. get status + rawOptions, err := buildDockerOptions(opts).Encode() if err != nil { - log.Errorf("failed to get container http port: %v", err) + return nil, err } - HTTPSPort, err := op.ContainerGetPortBinding(gitlabContainerName, "443", tcp) + status, err := runner.Execute(rawOptions) if err != nil { - log.Errorf("failed to get container https port: %v", err) - } - - // if the previous steps failed, the parameters will be empty - // so dtm will find the resource is drifted and restart docker - resource := gitlabResource{ - ContainerRunning: running, - Volumes: volumes, - Hostname: hostname, - SSHPort: SSHPort, - HTTPPort: HTTPPort, - HTTPSPort: HTTPSPort, + return nil, err } - return resource.toMap(), nil + log.Debugf("Return map: %v", status) + return status, nil } diff --git a/internal/pkg/plugin/gitlabcedocker/update.go b/internal/pkg/plugin/gitlabcedocker/update.go index 97a0eaa0e..f5b24b790 100644 --- a/internal/pkg/plugin/gitlabcedocker/update.go +++ b/internal/pkg/plugin/gitlabcedocker/update.go @@ -1,75 +1,36 @@ package gitlabcedocker import ( - "fmt" - "strconv" - "strings" - - "github.com/mitchellh/mapstructure" - - "github.com/devstream-io/devstream/pkg/util/docker" + "github.com/devstream-io/devstream/internal/pkg/plugininstaller" + dockerInstaller "github.com/devstream-io/devstream/internal/pkg/plugininstaller/docker" "github.com/devstream-io/devstream/pkg/util/log" ) func Update(options map[string]interface{}) (map[string]interface{}, error) { - var opts Options - if err := mapstructure.Decode(options, &opts); err != nil { + // 1. create config and pre-handle operations + _, err := validateAndDefault(options) + if err != nil { return nil, err } - defaults(&opts) - - if errs := validate(&opts); len(errs) != 0 { - for _, e := range errs { - log.Errorf("Options error: %s.", e) - } - return nil, fmt.Errorf("opts are illegal") + // 2. config install operations + runner := &plugininstaller.Runner{ + PreExecuteOperations: []plugininstaller.MutableOperation{ + dockerInstaller.Validate, + }, + ExecuteOperations: []plugininstaller.BaseOperation{ + dockerInstaller.InstallOrUpdate, + showHelpMsg, + }, + GetStatusOperation: dockerInstaller.GetRunningState, } - op := GetDockerOperator(opts) - - // 0. check if the volumes are the same - mounts, err := op.ContainerListMounts(gitlabContainerName) + // 3. update and get status + status, err := runner.Execute(options) if err != nil { - return nil, fmt.Errorf("failed to get container mounts: %v", err) - } - volumesFromRunningContainer := mounts.ExtractSources() - - volumesDirFromOptions := getVolumesDirFromOptions(opts) - - if docker.IfVolumesDiffer(volumesFromRunningContainer, volumesDirFromOptions) { - log.Warnf("You changed volumes of the container or change the gitlab home directory") - log.Infof("Your volumes of the current container were: %v", strings.Join(volumesFromRunningContainer, " ")) - return nil, fmt.Errorf("sorry, you can't change the gitlab_home of the container once it's already been created") - } - - // 1. stop the container - if ok := op.ContainerIfRunning(gitlabContainerName); ok { - if err := op.ContainerStop(gitlabContainerName); err != nil { - log.Warnf("Failed to stop container: %v", err) - } - } - - // 2. remove the container if it exists - if exists := op.ContainerIfExist(gitlabContainerName); exists { - if err := op.ContainerRemove(gitlabContainerName); err != nil { - return nil, fmt.Errorf("failed to remove container: %v", err) - } - } - - // 3. run the container with the new options - if err := op.ContainerRun(buildDockerRunOptions(opts), dockerRunShmSizeParam); err != nil { - return nil, fmt.Errorf("failed to run container: %v", err) - } - - resource := gitlabResource{ - ContainerRunning: true, - Volumes: volumesFromRunningContainer, - Hostname: opts.Hostname, - SSHPort: strconv.Itoa(int(opts.SSHPort)), - HTTPPort: strconv.Itoa(int(opts.HTTPPort)), - HTTPSPort: strconv.Itoa(int(opts.HTTPSPort)), + return nil, err } + log.Debugf("Return map: %v", status) - return resource.toMap(), nil + return status, nil } diff --git a/internal/pkg/plugin/gitlabcedocker/validate.go b/internal/pkg/plugin/gitlabcedocker/validate.go index 9b2889f40..08f53e1da 100644 --- a/internal/pkg/plugin/gitlabcedocker/validate.go +++ b/internal/pkg/plugin/gitlabcedocker/validate.go @@ -2,19 +2,41 @@ package gitlabcedocker import ( "fmt" - "path" + "os" + "path/filepath" + "github.com/mitchellh/mapstructure" + + "github.com/devstream-io/devstream/pkg/util/log" "github.com/devstream-io/devstream/pkg/util/validator" ) -// validate validates the options provided by the core. -func validate(options *Options) []error { - errs := validator.Struct(options) +func validateAndDefault(options map[string]interface{}) (*Options, error) { + var opts *Options + if err := mapstructure.Decode(options, &opts); err != nil { + return nil, err + } + + opts.Defaults() + // validate + errs := validator.Struct(opts) // volume directory must be absolute path - if !path.IsAbs(options.GitLabHome) { + if !filepath.IsAbs(opts.GitLabHome) { errs = append(errs, fmt.Errorf("GitLabHome must be an absolute path")) } + if len(errs) > 0 { + for _, e := range errs { + log.Errorf("Options error: %s.", e) + } + return nil, fmt.Errorf("opts are illegal") + } + + if err := os.MkdirAll(opts.GitLabHome, 0755); err != nil { + return nil, err + } + + opts.setGitLabURL() - return errs + return opts, nil } diff --git a/internal/pkg/plugin/jenkinsgithub/jcasc.go b/internal/pkg/plugin/jenkinsgithub/jcasc.go index 62f97b36b..b1cfa07d8 100644 --- a/internal/pkg/plugin/jenkinsgithub/jcasc.go +++ b/internal/pkg/plugin/jenkinsgithub/jcasc.go @@ -6,6 +6,7 @@ import ( _ "embed" "fmt" "text/template" + "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" corev1 "k8s.io/client-go/applyconfigurations/core/v1" @@ -75,6 +76,10 @@ func applyJCasC(namespace, chartReleaseName, configName, fileContent string) err log.Debugf("Created configmap %+v", configMapRes) + // wait for the config map and the sidecar to be ready + // TODO(aFlyBird0): read JCasC to judge if JCasC is ready + time.Sleep(time.Second * 3) + return nil } diff --git a/internal/pkg/plugininstaller/reposcaffolding/dstrepo.go b/internal/pkg/plugininstaller/common/repo.go similarity index 66% rename from internal/pkg/plugininstaller/reposcaffolding/dstrepo.go rename to internal/pkg/plugininstaller/common/repo.go index 01f786e54..79e1ea9b0 100644 --- a/internal/pkg/plugininstaller/reposcaffolding/dstrepo.go +++ b/internal/pkg/plugininstaller/common/repo.go @@ -1,20 +1,21 @@ -package reposcaffolding +package common import ( "io/fs" "os" "path/filepath" + "regexp" "strings" - "github.com/devstream-io/devstream/internal/pkg/plugininstaller/util" + "github.com/devstream-io/devstream/pkg/util/file" "github.com/devstream-io/devstream/pkg/util/github" "github.com/devstream-io/devstream/pkg/util/gitlab" "github.com/devstream-io/devstream/pkg/util/log" "github.com/devstream-io/devstream/pkg/util/template" ) -// DstRepo is the destination repo to push scaffolding project -type DstRepo struct { +// Repo is the repo info of github or gitlab +type Repo struct { Owner string `validate:"required_without=Org"` Org string `validate:"required_without=Owner"` Repo string `validate:"required"` @@ -26,7 +27,8 @@ type DstRepo struct { Visibility string `validate:"omitempty,oneof=public private internal"` } -func (d *DstRepo) createLocalRepoPath(workpath string) (string, error) { +// CreateLocalRepoPath create local path for repo +func (d *Repo) CreateLocalRepoPath(workpath string) (string, error) { localPath := filepath.Join(workpath, d.Repo) if err := os.MkdirAll(localPath, os.ModePerm); err != nil { return "", err @@ -34,9 +36,9 @@ func (d *DstRepo) createLocalRepoPath(workpath string) (string, error) { return localPath, nil } -// this method generate a walker func to render and copy files from srcRepoPath to dstRepoPath -func (d *DstRepo) generateRenderWalker( - srcRepoPath, dstRepoPath string, renderConfig map[string]interface{}, +// Generate is a walker func to render and copy files from srcRepoPath to dstRepoPath +func (d *Repo) GenerateRenderWalker( + srcRepoPath, dstRepoPath, appNamePlaceHolder string, renderConfig map[string]interface{}, ) func(path string, info fs.FileInfo, err error) error { return func(path string, info fs.FileInfo, err error) error { if err != nil { @@ -56,7 +58,7 @@ func (d *DstRepo) generateRenderWalker( } // replace template with appName - outputWithRepoName, err := replaceAppNameInPathStr(relativePath, d.Repo) + outputWithRepoName, err := replaceAppNameInPathStr(relativePath, appNamePlaceHolder, d.Repo) if err != nil { log.Debugf("Walk: Replace file name failed %s.", path) return err @@ -83,11 +85,12 @@ func (d *DstRepo) generateRenderWalker( outputPath = strings.TrimSuffix(outputPath, ".tpl") return template.RenderForFile("repo-scaffolding", path, outputPath, renderConfig) } - return util.CopyFile(path, outputPath) + return file.CopyFile(path, outputPath) } } -func (d *DstRepo) createRepoRenderConfig() map[string]interface{} { +// CreateRepoRenderConfig will generate template render variables +func (d *Repo) CreateRepoRenderConfig() map[string]interface{} { var owner = d.Owner if d.Org != "" { owner = d.Org @@ -103,7 +106,8 @@ func (d *DstRepo) createRepoRenderConfig() map[string]interface{} { return renderConfigMap } -func (d *DstRepo) createGithubClient(needAuth bool) (*github.Client, error) { +// CreateGithubClient build github client connection info +func (d *Repo) CreateGithubClient(needAuth bool) (*github.Client, error) { ghOptions := &github.Option{ Owner: d.Owner, Org: d.Org, @@ -117,11 +121,13 @@ func (d *DstRepo) createGithubClient(needAuth bool) (*github.Client, error) { return ghClient, nil } -func (d *DstRepo) createGitlabClient() (*gitlab.Client, error) { +// CreateGitlabClient build gitlab connection info +func (d *Repo) CreateGitlabClient() (*gitlab.Client, error) { return gitlab.NewClient(gitlab.WithBaseURL(d.BaseURL)) } -func (d *DstRepo) buildgitlabOpts() *gitlab.CreateProjectOptions { +// BuildgitlabOpts build gitlab connection options +func (d *Repo) BuildgitlabOpts() *gitlab.CreateProjectOptions { return &gitlab.CreateProjectOptions{ Name: d.Repo, Branch: d.Branch, @@ -129,3 +135,12 @@ func (d *DstRepo) buildgitlabOpts() *gitlab.CreateProjectOptions { Visibility: d.Visibility, } } + +func replaceAppNameInPathStr(filePath, appNamePlaceHolder, appName string) (string, error) { + if !strings.Contains(filePath, appNamePlaceHolder) { + return filePath, nil + } + newFilePath := regexp.MustCompile(appNamePlaceHolder).ReplaceAllString(filePath, appName) + log.Debugf("Replace file path place holder. Before: %s, after: %s.", filePath, newFilePath) + return newFilePath, nil +} diff --git a/internal/pkg/plugininstaller/common/state.go b/internal/pkg/plugininstaller/common/state.go new file mode 100644 index 000000000..fde08af96 --- /dev/null +++ b/internal/pkg/plugininstaller/common/state.go @@ -0,0 +1,39 @@ +package common + +import ( + "github.com/devstream-io/devstream/pkg/util/helm" + "github.com/devstream-io/devstream/pkg/util/k8s" + "github.com/devstream-io/devstream/pkg/util/log" +) + +func GetPluginAllK8sState(nameSpace string, anFilter, labelFilter map[string]string) (map[string]interface{}, error) { + + // 1. init kube client + kubeClient, err := k8s.NewClient() + if err != nil { + return nil, err + } + + // 2. get all related resource + allResource, err := kubeClient.GetResourceStatus(nameSpace, anFilter, labelFilter) + if err != nil { + log.Debugf("helm status: get status failed: %s", err) + return nil, err + } + + // 3. transfer resources status to workflows + state := &helm.InstanceState{} + for _, dep := range allResource.Deployment { + state.Workflows.AddDeployment(dep.Name, dep.Ready) + } + for _, sts := range allResource.StatefulSet { + state.Workflows.AddStatefulset(sts.Name, sts.Ready) + } + for _, ds := range allResource.DaemonSet { + state.Workflows.AddDaemonset(ds.Name, ds.Ready) + } + + retMap := state.ToStringInterfaceMap() + log.Debugf("Return map: %v.", retMap) + return retMap, nil +} diff --git a/internal/pkg/plugininstaller/docker/docker_suite_test.go b/internal/pkg/plugininstaller/docker/docker_suite_test.go new file mode 100644 index 000000000..4001151b1 --- /dev/null +++ b/internal/pkg/plugininstaller/docker/docker_suite_test.go @@ -0,0 +1,13 @@ +package docker_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestDocker(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Docker Suite") +} diff --git a/internal/pkg/plugininstaller/docker/installer.go b/internal/pkg/plugininstaller/docker/installer.go new file mode 100644 index 000000000..6d79bf476 --- /dev/null +++ b/internal/pkg/plugininstaller/docker/installer.go @@ -0,0 +1,163 @@ +package docker + +import ( + "fmt" + + "go.uber.org/multierr" + + "github.com/devstream-io/devstream/internal/pkg/plugininstaller" + "github.com/devstream-io/devstream/pkg/util/docker" + "github.com/devstream-io/devstream/pkg/util/docker/dockersh" + "github.com/devstream-io/devstream/pkg/util/log" +) + +var op docker.Operator + +func init() { + // default to shell operator + op = &dockersh.ShellOperator{} +} + +func UseShellOperator() { + op = &dockersh.ShellOperator{} +} + +// InstallOrUpdate runs or updates the docker container +// note: any update will stop and remove the container, then run the new container +func InstallOrUpdate(options plugininstaller.RawOptions) error { + opts, err := NewOptions(options) + if err != nil { + return err + } + + // 1. try to pull the image + // always pull the image because docker will check the image existence + if err := op.ImagePull(opts.GetImageNameWithTag()); err != nil { + return err + } + + // 2. try to run the container + log.Infof("Running container as the name <%s>", opts.ContainerName) + if err := op.ContainerRun(opts.GetRunOpts()); err != nil { + return fmt.Errorf("failed to run container: %v", err) + } + + // 3. check if the container is started successfully + if ok := op.ContainerIfRunning(opts.ContainerName); !ok { + return fmt.Errorf("failed to run container") + } + + // 4. check if the volume is created successfully + mounts, err := op.ContainerListMounts(opts.ContainerName) + if err != nil { + return fmt.Errorf("failed to get container mounts: %v", err) + } + volumes := mounts.ExtractSources() + if docker.IfVolumesDiffer(volumes, opts.Volumes.ExtractHostPaths()) { + return fmt.Errorf("failed to create volumes") + } + + return nil +} + +// HandleRunFailure will delete the container if the container fails to run +func HandleRunFailure(options plugininstaller.RawOptions) error { + opts, err := NewOptions(options) + if err != nil { + return err + } + + // 1. stop the container if it is running + if ok := op.ContainerIfRunning(opts.ContainerName); ok { + if err := op.ContainerStop(opts.ContainerName); err != nil { + log.Errorf("Failed to stop container %s: %v", opts.ContainerName, err) + } + } + + // 2. remove the container if it exists + if ok := op.ContainerIfExist(opts.ContainerName); ok { + if err := op.ContainerRemove(opts.ContainerName); err != nil { + log.Errorf("failed to remove container %v: %v", opts.ContainerName, err) + } + } + + var errs []error + + // 3. check if the container is stopped + if ok := op.ContainerIfRunning(opts.ContainerName); ok { + errs = append(errs, fmt.Errorf("failed to stop container %s", opts.ContainerName)) + } + + // 4. check if the container is removed + if ok := op.ContainerIfExist(opts.ContainerName); ok { + errs = append(errs, fmt.Errorf("failed to delete container %s", opts.ContainerName)) + } + + return multierr.Combine(errs...) +} + +// Delete will delete the container/image/volumes +func Delete(options plugininstaller.RawOptions) error { + opts, err := NewOptions(options) + if err != nil { + return err + } + + // 1. stop the container if it is running + if ok := op.ContainerIfRunning(opts.ContainerName); ok { + if err := op.ContainerStop(opts.ContainerName); err != nil { + log.Errorf("Failed to stop container %s: %v", opts.ContainerName, err) + } + } + + // 2. remove the container if it exists + if ok := op.ContainerIfExist(opts.ContainerName); ok { + if err := op.ContainerRemove(opts.ContainerName); err != nil { + log.Errorf("failed to remove container %v: %v", opts.ContainerName, err) + } + } + + // 3. remove the image if it exists + //if ok := op.ImageIfExist(opts.GetImageNameWithTag()); ok { + // if err := op.ImageRemove(opts.GetImageNameWithTag()); err != nil { + // log.Errorf("failed to remove image %v: %v", opts.GetImageNameWithTag(), err) + // } + //} + + // 4. remove the volume if it exists + if *opts.RmDataAfterDelete { + volumesDirFromOptions := opts.Volumes.ExtractHostPaths() + for _, err := range RemoveDirs(volumesDirFromOptions) { + log.Error(err) + } + } + + var errs []error + + // 5. check if the container is stopped and deleted + if ok := op.ContainerIfRunning(opts.ContainerName); ok { + errs = append(errs, fmt.Errorf("failed to stop container %s", opts.ContainerName)) + } + + // 6. check if the container is removed + if ok := op.ContainerIfExist(opts.ContainerName); ok { + errs = append(errs, fmt.Errorf("failed to delete container %s", opts.ContainerName)) + } + + // 7. check if the image is removed + if ok := op.ImageIfExist(opts.GetImageNameWithTag()); ok { + errs = append(errs, fmt.Errorf("failed to delete image %s", opts.GetImageNameWithTag())) + } + + // 8. check if the volume is removed + if *opts.RmDataAfterDelete { + volumesDirFromOptions := opts.Volumes.ExtractHostPaths() + for _, volume := range volumesDirFromOptions { + if exist := PathExist(volume); exist { + errs = append(errs, fmt.Errorf("failed to delete volume %s", volume)) + } + } + } + + return multierr.Combine(errs...) +} diff --git a/internal/pkg/plugininstaller/docker/option.go b/internal/pkg/plugininstaller/docker/option.go new file mode 100644 index 000000000..173030d0a --- /dev/null +++ b/internal/pkg/plugininstaller/docker/option.go @@ -0,0 +1,73 @@ +package docker + +import ( + "fmt" + + "github.com/mitchellh/mapstructure" + + "github.com/devstream-io/devstream/internal/pkg/plugininstaller" + "github.com/devstream-io/devstream/pkg/util/docker" + "github.com/devstream-io/devstream/pkg/util/log" + "github.com/devstream-io/devstream/pkg/util/validator" +) + +type Options struct { + ImageName string `validate:"required"` + ImageTag string `validate:"required"` + ContainerName string `validate:"required"` + RmDataAfterDelete *bool + + RunParams []string + Hostname string + PortPublishes []docker.PortPublish + Volumes docker.Volumes + RestartAlways bool +} + +// NewOptions create options by raw options +func NewOptions(options plugininstaller.RawOptions) (Options, error) { + var opts Options + if err := mapstructure.Decode(options, &opts); err != nil { + return opts, err + } + return opts, nil +} + +func (opts *Options) GetImageNameWithTag() string { + return fmt.Sprintf("%s:%s", opts.ImageName, opts.ImageTag) +} + +func (opts *Options) GetRunOpts() *docker.RunOptions { + return &docker.RunOptions{ + ImageName: opts.ImageName, + ImageTag: opts.ImageTag, + ContainerName: opts.ContainerName, + Hostname: opts.Hostname, + PortPublishes: opts.PortPublishes, + Volumes: opts.Volumes, + RestartAlways: opts.RestartAlways, + } +} + +func (opts *Options) Encode() (map[string]interface{}, error) { + var options map[string]interface{} + if err := mapstructure.Decode(opts, &options); err != nil { + return nil, err + } + return options, nil +} + +func Validate(options plugininstaller.RawOptions) (plugininstaller.RawOptions, error) { + opts, err := NewOptions(options) + if err != nil { + return nil, err + } + errs := validator.Struct(opts) + if len(errs) > 0 { + for _, e := range errs { + log.Errorf("Options error: %s.", e) + } + return nil, fmt.Errorf("opts are illegal") + } + return options, nil +} diff --git a/internal/pkg/plugininstaller/docker/state.go b/internal/pkg/plugininstaller/docker/state.go new file mode 100644 index 000000000..8423b9b81 --- /dev/null +++ b/internal/pkg/plugininstaller/docker/state.go @@ -0,0 +1,91 @@ +package docker + +import ( + "fmt" + + "github.com/mitchellh/mapstructure" + + "github.com/devstream-io/devstream/internal/pkg/plugininstaller" + "github.com/devstream-io/devstream/pkg/util/docker" + "github.com/devstream-io/devstream/pkg/util/log" +) + +type State struct { + ContainerRunning bool + Volumes []string + Hostname string + PortPublishes []docker.PortPublish +} + +func (s *State) toMap() (map[string]interface{}, error) { + m := make(map[string]interface{}) + err := mapstructure.Decode(s, &m) + if err != nil { + return nil, fmt.Errorf("failed to convert state to map: %v", err) + } + + return m, nil +} + +func GetStaticStateFromOptions(options plugininstaller.RawOptions) (map[string]interface{}, error) { + opts, err := NewOptions(options) + if err != nil { + return nil, err + } + + staticState := &State{ + ContainerRunning: true, + Volumes: opts.Volumes.ExtractHostPaths(), + Hostname: opts.Hostname, + PortPublishes: opts.PortPublishes, + } + + return staticState.toMap() +} + +func GetRunningState(options plugininstaller.RawOptions) (map[string]interface{}, error) { + opts, err := NewOptions(options) + if err != nil { + return nil, err + } + + // 1. get running status + running := op.ContainerIfRunning(opts.ContainerName) + if !running { + return map[string]interface{}{}, nil + } + + // 2. get volumes + mounts, err := op.ContainerListMounts(opts.ContainerName) + if err != nil { + // `Read` shouldn't return errors even if failed to read ports, volumes, hostname. + // because: + // 1. when the docker is stopped it could cause these errors. + // 2. if Read failed, the following steps contain the docker's restart will be aborted. + log.Errorf("failed to get container mounts: %v", err) + } + volumes := mounts.ExtractSources() + + // 3. get hostname + hostname, err := op.ContainerGetHostname(opts.ContainerName) + if err != nil { + log.Errorf("failed to get container hostname: %v", err) + } + + // 4. get port bindings + PortPublishes, err := op.ContainerListPortPublishes(opts.ContainerName) + if err != nil { + log.Errorf("failed to get container port publishes: %v", err) + } + + // if the previous steps failed, the parameters will be empty + // so dtm will find the resource is drifted and restart docker + resource := &State{ + ContainerRunning: running, + Volumes: volumes, + Hostname: hostname, + PortPublishes: PortPublishes, + } + + return resource.toMap() +} diff --git a/internal/pkg/plugininstaller/docker/util.go b/internal/pkg/plugininstaller/docker/util.go new file mode 100644 index 000000000..c310d8a1a --- /dev/null +++ b/internal/pkg/plugininstaller/docker/util.go @@ -0,0 +1,26 @@ +package docker + +import ( + "fmt" + "os" +) + +// RemoveDirs removes the all the directories in the given list recursively +func RemoveDirs(dirs []string) []error { + var errs []error + for _, dir := range dirs { + if err := os.RemoveAll(dir); err != nil { + errs = append(errs, fmt.Errorf("failed to remove data %v: %v", dir, err)) + } + } + + return errs +} + +func PathExist(path string) bool { + _, err := os.Stat(path) + if err == nil { + return true + } + return os.IsExist(err) +} diff --git a/internal/pkg/plugininstaller/docker/util_test.go b/internal/pkg/plugininstaller/docker/util_test.go new file mode 100644 index 000000000..2c227218b --- /dev/null +++ b/internal/pkg/plugininstaller/docker/util_test.go @@ -0,0 +1,68 @@ +package docker + +import ( + "os" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("RemoveDirs func", func() { + + var ( + errs []error + dirs []string + ) + + AfterEach(func() { + // check directories are removed + for _, dir := range dirs { + _, err := os.Stat(dir) + Expect(os.IsNotExist(err)).To(BeTrue()) + } + }) + + When("the directories are not exist", func() { + + BeforeEach(func() { + dirs = []string{"dir/not/exist/1", "dir/not/exist/2"} + }) + + It("should return no error", func() { + // Remove the directories + errs = RemoveDirs(dirs) + for _, e := range errs { + Expect(e).ToNot(HaveOccurred()) + } + }) + }) + + When("the directories are exist", func() { + + BeforeEach(func() { + // create temp dir, it will be removed automatically after the test + parentDir := GinkgoT().TempDir() + dirs = []string{ + parentDir + "dir1", + parentDir + "dir2", + parentDir + "dir3/dir3-1", + } + // create directories + for _, dir := range dirs { + err := os.MkdirAll(dir, 0755) + Expect(err).ToNot(HaveOccurred()) + } + // create files + _, err := os.CreateTemp(dirs[0], "file1*") + Expect(err).ToNot(HaveOccurred()) + }) + + It("should remove all directories successfully", func() { + // Remove the directories + errs = RemoveDirs(dirs) + for _, e := range errs { + Expect(e).ToNot(HaveOccurred()) + } + }) + }) +}) diff --git a/internal/pkg/plugininstaller/helm/helm_suit_test.go b/internal/pkg/plugininstaller/helm/helm_suit_test.go new file mode 100644 index 000000000..a7a9da004 --- /dev/null +++ b/internal/pkg/plugininstaller/helm/helm_suit_test.go @@ -0,0 +1,13 @@ +package helm_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestPlanmanager(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "PluginInstaller Helm Suite") +} diff --git a/internal/pkg/plugininstaller/helm/installer.go b/internal/pkg/plugininstaller/helm/installer.go index f1f5d2ef9..f36d28d37 100644 --- a/internal/pkg/plugininstaller/helm/installer.go +++ b/internal/pkg/plugininstaller/helm/installer.go @@ -111,7 +111,7 @@ func Delete(options plugininstaller.RawOptions) error { return err } - log.Info("Uninstalling argocd helm chart.") + log.Infof("Uninstalling %s helm chart.", opts.GetReleaseName()) if err = h.UninstallHelmChartRelease(); err != nil { return err } diff --git a/internal/pkg/plugininstaller/helm/option.go b/internal/pkg/plugininstaller/helm/option.go index bc2d71d9b..abf741bdb 100644 --- a/internal/pkg/plugininstaller/helm/option.go +++ b/internal/pkg/plugininstaller/helm/option.go @@ -9,9 +9,9 @@ import ( // Options is the struct for parameters used by the helm install config. type Options struct { - CreateNamespace bool `mapstructure:"create_namespace"` - Repo helm.Repo - Chart helm.Chart + CreateNamespace bool `mapstructure:"create_namespace"` + Repo helm.Repo `mapstructure:"repo"` + Chart helm.Chart `mapstructure:"chart"` } func (opts *Options) GetHelmParam() *helm.HelmParam { @@ -41,11 +41,18 @@ func (opts *Options) Encode() (map[string]interface{}, error) { return options, nil } +func (opts *Options) fillDefaultValue(defaultOpts *Options) { + chart := &opts.Chart + chart.FillDefaultValue(&defaultOpts.Chart) + repo := &opts.Repo + repo.FillDefaultValue(&defaultOpts.Repo) +} + // NewOptions create options by raw options -func NewOptions(options plugininstaller.RawOptions) (Options, error) { +func NewOptions(options plugininstaller.RawOptions) (*Options, error) { var opts Options if err := mapstructure.Decode(options, &opts); err != nil { - return opts, err + return nil, err } - return opts, nil + return &opts, nil } diff --git a/internal/pkg/plugininstaller/helm/option_test.go b/internal/pkg/plugininstaller/helm/option_test.go new file mode 100644 index 000000000..eed7d65bd --- /dev/null +++ b/internal/pkg/plugininstaller/helm/option_test.go @@ -0,0 +1,117 @@ +package helm_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/devstream-io/devstream/internal/pkg/plugininstaller" + "github.com/devstream-io/devstream/internal/pkg/plugininstaller/helm" + helmCommon "github.com/devstream-io/devstream/pkg/util/helm" +) + +var _ = Describe("Options struct", func() { + var ( + testOpts helm.Options + testChartName string + testRepoName string + testNameSpace string + expectMap map[string]interface{} + emptyBool *bool + ) + + BeforeEach(func() { + testChartName = "test_chart" + testRepoName = "test_repo" + testNameSpace = "test_nameSpace" + testOpts = helm.Options{ + Chart: helmCommon.Chart{ + ChartName: testChartName, + Namespace: testNameSpace, + }, + Repo: helmCommon.Repo{ + Name: testRepoName, + }, + } + expectMap = map[string]interface{}{ + "create_namespace": false, + "repo": map[string]interface{}{ + "name": "test_repo", + "url": "", + }, + "chart": map[string]interface{}{ + "version": "", + "release_name": "", + "wait": emptyBool, + "chart_name": "test_chart", + "namespace": "test_nameSpace", + "create_namespace": emptyBool, + "timeout": "", + "upgradeCRDs": emptyBool, + "values_yaml": "", + }, + } + }) + + Context("GetHelmParam method", func() { + It("should pass chart and repo field", func() { + helmParam := testOpts.GetHelmParam() + Expect(helmParam.Chart).Should(Equal(testOpts.Chart)) + Expect(helmParam.Repo).Should(Equal(testOpts.Repo)) + }) + }) + + Context("CheckIfCreateNamespace method", func() { + It("should equal opts config", func() { + Expect(testOpts.CheckIfCreateNamespace()).Should(Equal(testOpts.CreateNamespace)) + }) + }) + + Context("GetNamespace method", func() { + It("should return chart's nameSpace", func() { + Expect(testOpts.GetNamespace()).Should(Equal(testOpts.Chart.Namespace)) + }) + }) + + Context("GetReleaseName method", func() { + It("should return chart's ReleaseName", func() { + Expect(testOpts.GetReleaseName()).Should(Equal(testOpts.Chart.ReleaseName)) + }) + }) + + Context("Encode method", func() { + It("should return opts map", func() { + result, err := testOpts.Encode() + Expect(err).Error().ShouldNot(HaveOccurred()) + Expect(result).Should(Equal(expectMap)) + }) + }) +}) + +var _ = Describe("NewOptions func", func() { + + var ( + inputOptions plugininstaller.RawOptions + testRepoName string + testChartName string + ) + + BeforeEach(func() { + testRepoName = "test_repo" + testChartName = "test_chart" + inputOptions = map[string]interface{}{ + "repo": map[string]interface{}{ + "name": testRepoName, + }, + "chart": map[string]interface{}{ + "chart_name": testChartName, + }, + } + }) + + It("should work normal", func() { + opts, err := helm.NewOptions(inputOptions) + Expect(err).Error().ShouldNot(HaveOccurred()) + Expect(opts.Chart.ChartName).Should(Equal(testChartName)) + Expect(opts.Repo.Name).Should(Equal(testRepoName)) + }) +}) diff --git a/internal/pkg/plugininstaller/helm/state.go b/internal/pkg/plugininstaller/helm/state.go index 2ac287b48..75090a1a1 100644 --- a/internal/pkg/plugininstaller/helm/state.go +++ b/internal/pkg/plugininstaller/helm/state.go @@ -2,9 +2,8 @@ package helm import ( "github.com/devstream-io/devstream/internal/pkg/plugininstaller" + "github.com/devstream-io/devstream/internal/pkg/plugininstaller/common" "github.com/devstream-io/devstream/pkg/util/helm" - "github.com/devstream-io/devstream/pkg/util/k8s" - "github.com/devstream-io/devstream/pkg/util/log" ) // GetPlugAllStateWrapper will get deploy, ds, statefulset status @@ -14,35 +13,9 @@ func GetPluginAllState(options plugininstaller.RawOptions) (map[string]interface return nil, err } - // 1. init kube client - kubeClient, err := k8s.NewClient() - if err != nil { - return nil, err - } anFilter := map[string]string{ helm.GetAnnotationName(): opts.GetReleaseName(), } - - // 2. get all related resource - allResource, err := kubeClient.GetResourceStatus(opts.GetNamespace(), anFilter) - if err != nil { - log.Debugf("helm status: get status failed: %s", err) - return nil, err - } - - // 3. transfer resources status to workflows - state := &helm.InstanceState{} - for _, dep := range allResource.Deployment { - state.Workflows.AddDeployment(dep.Name, dep.Ready) - } - for _, sts := range allResource.StatefulSet { - state.Workflows.AddStatefulset(sts.Name, sts.Ready) - } - for _, ds := range allResource.DaemonSet { - state.Workflows.AddDaemonset(ds.Name, ds.Ready) - } - - retMap := state.ToStringInterfaceMap() - log.Debugf("Return map: %v.", retMap) - return retMap, nil + labelFilter := map[string]string{} + return common.GetPluginAllK8sState(opts.GetNamespace(), anFilter, labelFilter) } diff --git a/internal/pkg/plugininstaller/helm/validate.go b/internal/pkg/plugininstaller/helm/validate.go index d3d014dca..c15e00abd 100644 --- a/internal/pkg/plugininstaller/helm/validate.go +++ b/internal/pkg/plugininstaller/helm/validate.go @@ -14,7 +14,7 @@ func Validate(options plugininstaller.RawOptions) (plugininstaller.RawOptions, e if err != nil { return nil, err } - errs := helm.DefaultsAndValidate(opts.GetHelmParam()) + errs := helm.Validate(opts.GetHelmParam()) if len(errs) > 0 { for _, e := range errs { log.Errorf("Options error: %s.", e) @@ -23,3 +23,15 @@ func Validate(options plugininstaller.RawOptions) (plugininstaller.RawOptions, e } return options, nil } + +// SetDefaultConfig will update options empty values base on import options +func SetDefaultConfig(defaultConfig *Options) plugininstaller.MutableOperation { + return func(options plugininstaller.RawOptions) (plugininstaller.RawOptions, error) { + opts, err := NewOptions(options) + if err != nil { + return nil, err + } + opts.fillDefaultValue(defaultConfig) + return opts.Encode() + } +} diff --git a/internal/pkg/plugininstaller/helm/validate_test.go b/internal/pkg/plugininstaller/helm/validate_test.go new file mode 100644 index 000000000..0514544fa --- /dev/null +++ b/internal/pkg/plugininstaller/helm/validate_test.go @@ -0,0 +1,108 @@ +package helm_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/devstream-io/devstream/internal/pkg/plugininstaller" + "github.com/devstream-io/devstream/internal/pkg/plugininstaller/helm" + helmCommon "github.com/devstream-io/devstream/pkg/util/helm" + "github.com/devstream-io/devstream/pkg/util/types" +) + +var _ = Describe("Validate func", func() { + var testOption plugininstaller.RawOptions + + When("options is not valid", func() { + BeforeEach(func() { + testOption = map[string]interface{}{ + "chart": map[string]string{}, + "repo": map[string]string{}, + } + }) + It("should return error", func() { + _, err := helm.Validate(testOption) + Expect(err).Error().Should(HaveOccurred()) + }) + }) + + When("options is valid", func() { + BeforeEach(func() { + testOption = map[string]interface{}{ + "chart": map[string]string{ + "chart_name": "test", + }, + "repo": map[string]string{ + "url": "http://test.com", + "name": "test", + }, + } + }) + It("should return success", func() { + opt, err := helm.Validate(testOption) + Expect(err).Error().ShouldNot(HaveOccurred()) + Expect(opt).ShouldNot(BeEmpty()) + }) + }) +}) + +var _ = Describe("SetDefaultConfig func", func() { + var ( + testChartName string + testRepoURL string + testRepoName string + testBool *bool + defaultConfig helm.Options + testOptions plugininstaller.RawOptions + expectChart map[string]interface{} + expectRepo map[string]interface{} + ) + BeforeEach(func() { + testChartName = "test_chart" + testRepoName = "test_repo" + testRepoURL = "http://test.com" + testBool = types.Bool(true) + testOptions = map[string]interface{}{ + "chart": map[string]string{}, + "repo": map[string]string{}, + } + defaultConfig = helm.Options{ + Chart: helmCommon.Chart{ + ChartName: testChartName, + Wait: testBool, + UpgradeCRDs: testBool, + CreateNamespace: testBool, + }, + Repo: helmCommon.Repo{ + URL: testRepoURL, + Name: testRepoName, + }, + } + expectChart = map[string]interface{}{ + "chart_name": testChartName, + "wait": testBool, + "namespace": "", + "version": "", + "release_name": "", + "values_yaml": "", + "timeout": "", + "create_namespace": testBool, + "upgradeCRDs": testBool, + } + expectRepo = map[string]interface{}{ + "url": testRepoURL, + "name": testRepoName, + } + }) + It("should update default value", func() { + updateFunc := helm.SetDefaultConfig(&defaultConfig) + o, err := updateFunc(testOptions) + Expect(err).Error().ShouldNot(HaveOccurred()) + oRepo, exist := o["repo"] + Expect(exist).Should(BeTrue()) + oChart, exist := o["chart"] + Expect(exist).Should(BeTrue()) + Expect(oRepo).Should(Equal(expectRepo)) + Expect(oChart).Should(Equal(expectChart)) + }) +}) diff --git a/internal/pkg/plugininstaller/kubectl/installer.go b/internal/pkg/plugininstaller/kubectl/installer.go index e76200840..fe668cab8 100644 --- a/internal/pkg/plugininstaller/kubectl/installer.go +++ b/internal/pkg/plugininstaller/kubectl/installer.go @@ -2,32 +2,20 @@ package kubectl import ( "fmt" - "os" - "time" - - "github.com/cenkalti/backoff" "github.com/devstream-io/devstream/internal/pkg/plugininstaller" - "github.com/devstream-io/devstream/internal/pkg/plugininstaller/util" + "github.com/devstream-io/devstream/pkg/util/file" "github.com/devstream-io/devstream/pkg/util/kubectl" - "github.com/devstream-io/devstream/pkg/util/log" ) // InstallByDownload will download file for apply -func ProcessByContent(action, downloadUrl, content string) plugininstaller.BaseOperation { +func ProcessByContent(action string, templateConfig *file.TemplateConfig) plugininstaller.BaseOperation { return func(options plugininstaller.RawOptions) error { // generate k8s config file for apply - configFileName, err := createKubectlFile(downloadUrl, content, options) + configFileName, err := templateConfig.RenderFile("kubectl", options).Run() if err != nil { return err } - - defer func() { - err := os.Remove(configFileName) - if err != nil { - log.Debugf("kubectl delete temp file failed: %s", err) - } - }() // kubectl apply -f switch action { case "create": @@ -43,23 +31,3 @@ func ProcessByContent(action, downloadUrl, content string) plugininstaller.BaseO return nil } } - -// WaitDeployReadyWithDeployList will wait all deploy in deployList get ready -func WaitDeployReadyWithDeployList(namespace string, deployList []string) plugininstaller.BaseOperation { - waitFunc := func(options plugininstaller.RawOptions) error { - operation := func() error { - if err := util.CheckAllDeployAndServiceReady(namespace, deployList); err != nil { - return err - } - return nil - } - bkoff := backoff.NewExponentialBackOff() - bkoff.MaxElapsedTime = 3 * time.Minute - err := backoff.Retry(operation, bkoff) - if err != nil { - return err - } - return nil - } - return waitFunc -} diff --git a/internal/pkg/plugininstaller/kubectl/utils.go b/internal/pkg/plugininstaller/kubectl/utils.go deleted file mode 100644 index ca97a59ee..000000000 --- a/internal/pkg/plugininstaller/kubectl/utils.go +++ /dev/null @@ -1,67 +0,0 @@ -package kubectl - -import ( - "fmt" - "os" - "strings" - "text/template" - - "github.com/devstream-io/devstream/internal/pkg/plugininstaller" - "github.com/devstream-io/devstream/pkg/util/downloader" - "github.com/devstream-io/devstream/pkg/util/log" -) - -const defaultKubectlFileName = "kubectl-create-file_" - -func writeContentToTmpFile(output *os.File, content string, opts *plugininstaller.RawOptions) error { - t, err := template.New("app").Option("missingkey=error").Parse(content) - if err != nil { - return err - } - - log.Infof("All opts %+v", opts) - err = t.Execute(output, opts) - if err != nil { - if strings.Contains(err.Error(), "can't evaluate field name") { - msg := err.Error() - start := strings.Index(msg, "<") - end := strings.Index(msg, ">") - return fmt.Errorf("plugin argocdapp needs options%s but it's missing from the config file", msg[start+1:end]) - } else { - return fmt.Errorf("executing tpl error: %s", err) - } - } - return nil -} - -func createKubectlFile(downloadUrl, content string, options plugininstaller.RawOptions) (string, error) { - // create temp file for kubectl apply - tempFile, err := os.CreateTemp("", defaultKubectlFileName) - if err != nil { - return "", err - } - - defer func() { - err := tempFile.Close() - if err != nil { - log.Debugf("kubectl file close failed: %s", err) - } - }() - - if downloadUrl != "" { - // download config file - _, err := downloader.DownloadToFile(downloadUrl, tempFile) - if err != nil { - log.Debugf("Failed to download K8s deploy YAML file from %s.", downloadUrl) - return "", err - } - } else if content != "" { - // use content - if err = writeContentToTmpFile(tempFile, content, &options); err != nil { - return "", err - } - } else { - return "", fmt.Errorf("No Install plugin config is set") - } - return tempFile.Name(), nil -} diff --git a/internal/pkg/plugininstaller/reposcaffolding/installer.go b/internal/pkg/plugininstaller/reposcaffolding/installer.go index f91aa7d7f..906be5028 100644 --- a/internal/pkg/plugininstaller/reposcaffolding/installer.go +++ b/internal/pkg/plugininstaller/reposcaffolding/installer.go @@ -2,11 +2,8 @@ package reposcaffolding import ( "fmt" - "os" - "path/filepath" "github.com/devstream-io/devstream/internal/pkg/plugininstaller" - "github.com/devstream-io/devstream/pkg/util/log" ) // InstallRepo will install repo by opts config @@ -16,33 +13,20 @@ func InstallRepo(options plugininstaller.RawOptions) error { return err } - // 1. Create temp dir - dirName, err := os.MkdirTemp("", "") + // 1. Create and render repo get from given url + dirPath, err := opts.CreateAndRenderLocalRepo() if err != nil { return err } - defer func() { - if err := os.RemoveAll(dirName); err != nil { - log.Errorf("Failed to clear workpath %s: %s.", dirName, err) - } - }() - // 2. Create and render repo get from given url - err = opts.CreateAndRenderLocalRepo(dirName) - if err != nil { - return err - } - - dstRepo := opts.DestinationRepo // 2. Push local repo to remote - repoLoc := filepath.Join(dirName, opts.DestinationRepo.Repo) - switch dstRepo.RepoType { + switch opts.DestinationRepo.RepoType { case "github": - err = opts.PushToRemoteGithub(repoLoc) + err = opts.PushToRemoteGithub(dirPath) case "gitlab": - err = opts.PushToRemoteGitlab(repoLoc) + err = opts.PushToRemoteGitlab(dirPath) default: - err = fmt.Errorf("scaffolding not support repo destination: %s", dstRepo.RepoType) + err = fmt.Errorf("scaffolding not support repo destination: %s", opts.DestinationRepo.RepoType) } if err != nil { return err @@ -62,14 +46,14 @@ func DeleteRepo(options plugininstaller.RawOptions) error { switch dstRepo.RepoType { case "github": // 1. create ghClient - ghClient, err := opts.DestinationRepo.createGithubClient(true) + ghClient, err := dstRepo.CreateGithubClient(true) if err != nil { return err } // 2. delete github repo return ghClient.DeleteRepo() case "gitlab": - gLclient, err := dstRepo.createGitlabClient() + gLclient, err := dstRepo.CreateGitlabClient() if err != nil { return err } diff --git a/internal/pkg/plugininstaller/reposcaffolding/option.go b/internal/pkg/plugininstaller/reposcaffolding/option.go index d64e228ee..ea10bc742 100644 --- a/internal/pkg/plugininstaller/reposcaffolding/option.go +++ b/internal/pkg/plugininstaller/reposcaffolding/option.go @@ -1,21 +1,24 @@ package reposcaffolding import ( + "path/filepath" + "github.com/mitchellh/mapstructure" "github.com/devstream-io/devstream/internal/pkg/plugininstaller" + "github.com/devstream-io/devstream/internal/pkg/plugininstaller/common" + "github.com/devstream-io/devstream/pkg/util/file" "github.com/devstream-io/devstream/pkg/util/log" ) const ( - transitBranch = "init-with-devstream" - appNamePlaceHolder = "_app_name_" - defaultCommitMsg = "init with devstream" + transitBranch = "init-with-devstream" + defaultCommitMsg = "init with devstream" ) type Options struct { - SourceRepo *SrcRepo `validate:"required" mapstructure:"source_repo"` - DestinationRepo *DstRepo `validate:"required" mapstructure:"destination_repo"` + SourceRepo *SrcRepo `validate:"required" mapstructure:"source_repo"` + DestinationRepo *common.Repo `validate:"required" mapstructure:"destination_repo"` Vars map[string]interface{} } @@ -35,32 +38,39 @@ func (opts *Options) Encode() (map[string]interface{}, error) { return options, nil } -func (opts *Options) CreateAndRenderLocalRepo(workpath string) error { - // 1. download template scaffolding repo - err := opts.SourceRepo.DownloadRepo(workpath) +// CreateAndRenderLocalRepo will download repo from source repo and render it locally +func (opts *Options) CreateAndRenderLocalRepo() (string, error) { + // 1. get download url + githubCodeZipDownloadURL, err := opts.SourceRepo.getDownloadURL() if err != nil { - return err + log.Debugf("reposcaffolding get download url failed: %s", err) + return "", err } - // 2. walk iter repo files to render template - if err := walkLocalRepoPath(workpath, opts); err != nil { - log.Debugf("create local repo failed walk: %s.", err) - return err + // 2. download zip file and unzip this file then render folders + projectDir, err := file.NewTemplate().FromRemote(githubCodeZipDownloadURL).UnzipFile().RenderRepoDIr( + opts.DestinationRepo.Repo, opts.renderTplConfig(), + ).Run() + if err != nil { + log.Debugf("reposcaffolding process file error: %s", err) + return "", err } - return nil + // 3. join download path and repo name to get repo path + repoDirName := opts.SourceRepo.getRepoName() + return filepath.Join(projectDir, repoDirName), nil } // PushToRemoteGitLab push local repo to remote gitlab repo func (opts *Options) PushToRemoteGitlab(repoPath string) error { dstRepo := opts.DestinationRepo // 1. init gitlab client - c, err := dstRepo.createGitlabClient() + c, err := dstRepo.CreateGitlabClient() if err != nil { log.Debugf("Gitlab push: init gitlab client failed %s", err) return err } // 2. create the project - if err := c.CreateProject(dstRepo.buildgitlabOpts()); err != nil { + if err := c.CreateProject(dstRepo.BuildgitlabOpts()); err != nil { log.Errorf("Failed to create repo: %s.", err) return err } @@ -91,7 +101,7 @@ func (opts *Options) PushToRemoteGitlab(repoPath string) error { func (opts *Options) PushToRemoteGithub(repoPath string) error { dstRepo := opts.DestinationRepo // 1. init github client - ghClient, err := dstRepo.createGithubClient(true) + ghClient, err := dstRepo.CreateGithubClient(true) if err != nil { log.Debugf("Github push: init github client failed %s", err) return err @@ -123,7 +133,7 @@ func (opts *Options) PushToRemoteGithub(repoPath string) error { } func (opts *Options) renderTplConfig() map[string]interface{} { - renderConfig := opts.DestinationRepo.createRepoRenderConfig() + renderConfig := opts.DestinationRepo.CreateRepoRenderConfig() for k, v := range opts.Vars { renderConfig[k] = v } diff --git a/internal/pkg/plugininstaller/reposcaffolding/srcrepo.go b/internal/pkg/plugininstaller/reposcaffolding/srcrepo.go index 778a756d9..0cc807d7e 100644 --- a/internal/pkg/plugininstaller/reposcaffolding/srcrepo.go +++ b/internal/pkg/plugininstaller/reposcaffolding/srcrepo.go @@ -2,10 +2,8 @@ package reposcaffolding import ( "fmt" - "path/filepath" "github.com/devstream-io/devstream/pkg/util/github" - "github.com/devstream-io/devstream/pkg/util/zip" ) // default get main branch of repo for scaffolding project @@ -18,20 +16,19 @@ type SrcRepo struct { RepoType string `validate:"oneof=github" mapstructure:"repo_type"` } -func (t *SrcRepo) DownloadRepo(workpath string) error { - // 1. download scaffolding repo from github - if err := downloadGithubRepo(t.Org, t.Repo, workpath); err != nil { - return err - } - - // 2. unzip downloaded zip file - unzipPath := filepath.Join(workpath, github.DefaultLatestCodeZipfileName) - if err := zip.UnZip(unzipPath, workpath); err != nil { - return err - } - return nil +func (t *SrcRepo) getRepoName() string { + return fmt.Sprintf("%s-%s", t.Repo, srcDefaultBranch) } -func (t *SrcRepo) getLocalRepoPath(workpath string) string { - return filepath.Join(workpath, fmt.Sprintf("%s-%s", t.Repo, srcDefaultBranch)) +func (t *SrcRepo) getDownloadURL() (string, error) { + ghOption := &github.Option{ + Org: t.Org, + Repo: t.Repo, + NeedAuth: false, + } + ghClient, err := github.NewClient(ghOption) + if err != nil { + return "", err + } + return ghClient.GetLatestCodeZipURL(), nil } diff --git a/internal/pkg/plugininstaller/reposcaffolding/state.go b/internal/pkg/plugininstaller/reposcaffolding/state.go index f63b59cd0..56e8e42fe 100644 --- a/internal/pkg/plugininstaller/reposcaffolding/state.go +++ b/internal/pkg/plugininstaller/reposcaffolding/state.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/devstream-io/devstream/internal/pkg/plugininstaller" + "github.com/devstream-io/devstream/internal/pkg/plugininstaller/common" "github.com/devstream-io/devstream/pkg/util/gitlab" "github.com/devstream-io/devstream/pkg/util/log" ) @@ -66,8 +67,8 @@ func GetDynamicState(options plugininstaller.RawOptions) (map[string]interface{} } -func getGithubStatus(dstRepo *DstRepo) (map[string]interface{}, error) { - ghClient, err := dstRepo.createGithubClient(true) +func getGithubStatus(dstRepo *common.Repo) (map[string]interface{}, error) { + ghClient, err := dstRepo.CreateGithubClient(true) if err != nil { return nil, err } @@ -105,8 +106,8 @@ func getGithubStatus(dstRepo *DstRepo) (map[string]interface{}, error) { } -func getGitlabStatus(dstRepo *DstRepo) (map[string]interface{}, error) { - c, err := dstRepo.createGitlabClient() +func getGitlabStatus(dstRepo *common.Repo) (map[string]interface{}, error) { + c, err := dstRepo.CreateGitlabClient() if err != nil { return nil, err } diff --git a/internal/pkg/plugininstaller/reposcaffolding/utils.go b/internal/pkg/plugininstaller/reposcaffolding/utils.go deleted file mode 100644 index 20bf1c441..000000000 --- a/internal/pkg/plugininstaller/reposcaffolding/utils.go +++ /dev/null @@ -1,61 +0,0 @@ -package reposcaffolding - -import ( - "path/filepath" - "regexp" - "strings" - - "github.com/devstream-io/devstream/pkg/util/github" - "github.com/devstream-io/devstream/pkg/util/log" -) - -func downloadGithubRepo(org, repo, workpath string) error { - ghOption := &github.Option{ - Org: org, - Repo: repo, - NeedAuth: false, - WorkPath: workpath, - } - ghClient, err := github.NewClient(ghOption) - if err != nil { - return err - } - - if err = ghClient.DownloadLatestCodeAsZipFile(); err != nil { - return err - } - - return nil -} - -func replaceAppNameInPathStr(filePath, appName string) (string, error) { - if !strings.Contains(filePath, appNamePlaceHolder) { - return filePath, nil - } - newFilePath := regexp.MustCompile(appNamePlaceHolder).ReplaceAllString(filePath, appName) - log.Debugf("Replace file path place holder. Before: %s, after: %s.", filePath, newFilePath) - return newFilePath, nil -} - -func walkLocalRepoPath(workpath string, opts *Options) error { - // 1. get src path and dst path - srcRepoPath := opts.SourceRepo.getLocalRepoPath(workpath) - dstOpts := opts.DestinationRepo - dstRepoPath, err := dstOpts.createLocalRepoPath(workpath) - if err != nil { - log.Debugf("Walk: create output dir failed: %s", err) - return err - } - - // 2. config template render config - renderConfig := opts.renderTplConfig() - - // 3. create walk func - walkFunc := dstOpts.generateRenderWalker(srcRepoPath, dstRepoPath, renderConfig) - - // 4. walk iter srcRepoPath to execuate walk func logic - if err := filepath.Walk(srcRepoPath, walkFunc); err != nil { - return err - } - return nil -} diff --git a/internal/pkg/plugininstaller/util/file.go b/internal/pkg/plugininstaller/util/file.go deleted file mode 100644 index 36ab22fb1..000000000 --- a/internal/pkg/plugininstaller/util/file.go +++ /dev/null @@ -1,14 +0,0 @@ -package util - -import ( - "io/ioutil" -) - -func CopyFile(src, dest string) error { - bytesRead, err := ioutil.ReadFile(src) - if err != nil { - return err - } - - return ioutil.WriteFile(dest, bytesRead, 0644) -} diff --git a/internal/pkg/plugininstaller/util/k8s.go b/internal/pkg/plugininstaller/util/k8s.go deleted file mode 100644 index a46c00902..000000000 --- a/internal/pkg/plugininstaller/util/k8s.go +++ /dev/null @@ -1,90 +0,0 @@ -package util - -import ( - "fmt" - - "github.com/devstream-io/devstream/pkg/util/k8s" - "github.com/devstream-io/devstream/pkg/util/log" -) - -func CheckAllDeployAndServiceReady(namespace string, deployList []string) error { - kubeClient, err := k8s.NewClient() - if err != nil { - return err - } - - // check if all deployments are ready - for _, d := range deployList { - dp, err := kubeClient.GetDeployment(namespace, d) - if err != nil { - return err - } - - if !kubeClient.IsDeploymentReady(dp) { - log.Infof("The deployment %s is not ready yet.", dp.Name) - return fmt.Errorf("deployment %s not ready", dp.Name) - } - log.Infof("The deployment %s is ready.", dp.Name) - } - - // check if all services exist - for _, d := range deployList { - _, err := kubeClient.GetService(namespace, d) - if err != nil { - return err - } - } - return nil -} - -func ReadDepAndServiceState(namespace string, deployList []string) (map[string]interface{}, error) { - kubeClient, err := k8s.NewClient() - if err != nil { - return nil, err - } - - res := make(map[string]interface{}) - res["deployments"] = make(map[string]interface{}) - res["services"] = make(map[string]interface{}) - - // check if all deployments are ready - for _, d := range deployList { - // deployment - dp, err := kubeClient.GetDeployment(namespace, d) - if err == nil && kubeClient.IsDeploymentReady(dp) { - res["deployments"].(map[string]interface{})[d] = true - } else { - res["deployments"].(map[string]interface{})[d] = true - } - - // services - _, err = kubeClient.GetService(namespace, d) - if err == nil { - res["services"].(map[string]interface{})[d] = true - } else { - res["services"].(map[string]interface{})[d] = false - } - } - - log.Debugf("Resource read returns: %v.", res) - return res, nil -} - -func GetArgoCDAppFromK8sAndSetState(state map[string]interface{}, name, namespace string) error { - kubeClient, err := k8s.NewClient() - if err != nil { - return err - } - - app, err := kubeClient.GetArgocdApplication(namespace, name) - if err != nil { - return err - } - - d := kubeClient.DescribeArgocdApp(app) - state["app"] = d["app"] - state["src"] = d["src"] - state["dest"] = d["dest"] - - return nil -} diff --git a/internal/pkg/show/config/gen_embed_var.go b/internal/pkg/show/config/gen_embed_var.go index 5ed4a84c0..0167aafa6 100644 --- a/internal/pkg/show/config/gen_embed_var.go +++ b/internal/pkg/show/config/gen_embed_var.go @@ -17,7 +17,7 @@ import ( "strings" "text/template" - "github.com/devstream-io/devstream/internal/pkg/plugininstaller/util" + "github.com/devstream-io/devstream/pkg/util/file" "golang.org/x/text/cases" "golang.org/x/text/language" @@ -72,7 +72,7 @@ func copyTemplates(src, dst string) error { if d.IsDir() || !strings.HasSuffix(path, ".yaml") { return nil } - return util.CopyFile(path, filepath.Join(dst, filepath.Base(path))) + return file.CopyFile(path, filepath.Join(dst, filepath.Base(path))) }) } diff --git a/internal/pkg/show/config/plugins/gitlab-ce-docker.yaml b/internal/pkg/show/config/plugins/gitlab-ce-docker.yaml index 9dba0d2e0..010e056cb 100644 --- a/internal/pkg/show/config/plugins/gitlab-ce-docker.yaml +++ b/internal/pkg/show/config/plugins/gitlab-ce-docker.yaml @@ -7,20 +7,22 @@ tools: dependsOn: [ ] # options for the plugin options: - # hostname for running docker + # hostname for running docker. (default: gitlab.example.com) hostname: gitlab.example.com - # the directory where you store docker volumes of gitlab + # pointing to the directory where the configuration, logs, and data files will reside. + # (default: /srv/gitlab) # 1. it should be a absolute path - # 2. once the tool is applied, it can't be changed + # 2. once the tool is installed, it can't be changed gitlab_home: /srv/gitlab - # ssh port exposed in the host machine + # ssh port exposed in the host machine. (default: 22) ssh_port: 22 - # http port exposed in the host machine + # http port exposed in the host machine. (default: 80) http_port: 80 - # https port exposed in the host machine + # https port exposed in the host machine. + # (default: 443) # todo: support https, reference: https://docs.gitlab.com/omnibus/settings/nginx.html#enable-https https_port: 443 - # whether to delete the gitlab_home directory when the tool is removed + # whether to delete the gitlab_home directory when the tool is removed. (default: false) rm_data_after_delete: false - # gitlab-ce tag, default tag is rc + # gitlab-ce tag. (default: "rc") image_tag: "rc" diff --git a/pkg/util/docker/docker.go b/pkg/util/docker/docker.go index 7ba9635b3..491901d8c 100644 --- a/pkg/util/docker/docker.go +++ b/pkg/util/docker/docker.go @@ -19,7 +19,7 @@ type Operator interface { // ContainerRun runs a container with the given options // params is a list of additional parameters for docker run // params will be appended to the end of the command - ContainerRun(opts RunOptions, params ...string) error + ContainerRun(opts *RunOptions) error ContainerStop(containerName string) error ContainerRemove(containerName string) error @@ -27,9 +27,9 @@ type Operator interface { ContainerListMounts(containerName string) (Mounts, error) ContainerGetHostname(containerName string) (string, error) - ContainerGetPortBinding(containerName, containerPort, protocol string) (hostPort string, err error) + ContainerListPortPublishes(containerName string) ([]PortPublish, error) + ContainerGetPortBinding(containerName string, containerPort uint) (hostPort uint, err error) } - type MountPoint struct { Type string `json:"Type"` Source string `json:"Source"` diff --git a/pkg/util/docker/dockersh/operator.go b/pkg/util/docker/dockersh/operator.go index 002e329f5..f8fce8c88 100644 --- a/pkg/util/docker/dockersh/operator.go +++ b/pkg/util/docker/dockersh/operator.go @@ -5,6 +5,8 @@ import ( "encoding/json" "fmt" "os/exec" + "regexp" + "strconv" "strings" "github.com/devstream-io/devstream/pkg/util/docker" @@ -14,43 +16,30 @@ import ( // ShellOperator is an implementation of /pkg/util/docker.Operator interface by using shell commands type ShellOperator struct{} -// TODO(aFlyBird0): maybe use one param "ImageNameWithTag" is not a good idea, -// because we have to extract the bare image name from the image name with tag -// we could use (a struct)/(a interface)/(two params) to represent the image name and tag func (op *ShellOperator) ImageIfExist(imageNameWithTag string) bool { - // eg. docker image ls gitlab/gitlab-ce:rc - cmdString := fmt.Sprintf("docker image ls %v", imageNameWithTag) - // output: eg. - // REPOSITORY TAG IMAGE ID CREATED SIZE - // gitlab/gitlab-ce rc a8543d702e39 4 days ago 2.49GB + // eg. docker image ls gitlab/gitlab-ce:rc -q + // output: image id (if exist) + cmdString := fmt.Sprintf("docker image ls %v -q", imageNameWithTag) outputBuffer := &bytes.Buffer{} err := ExecInSystem(".", cmdString, outputBuffer, false) if err != nil { return false } - // eg. gitlab/gitlab-ce - imageNameWithoutTag := extractImageName(imageNameWithTag) - return strings.Contains(outputBuffer.String(), imageNameWithoutTag) + return strings.TrimSpace(outputBuffer.String()) != "" } -func extractImageName(imageNameWithTag string) string { - // the imageNameWithTag is in the format of "registry/image:tag" - // we only want to return the image name "registry/image" - return strings.Split(imageNameWithTag, ":")[0] -} - -func (op *ShellOperator) ImagePull(imageName string) error { - err := ExecInSystemWithParams(".", []string{"docker", "pull", imageName}, nil, true) +func (op *ShellOperator) ImagePull(imageNameWithTag string) error { + err := ExecInSystemWithParams(".", []string{"docker", "pull", imageNameWithTag}, nil, true) return err } -func (op *ShellOperator) ImageRemove(imageName string) error { - log.Infof("Removing image %v ...", imageName) +func (op *ShellOperator) ImageRemove(imageNameWithTag string) error { + log.Infof("Removing image %v ...", imageNameWithTag) - cmdString := fmt.Sprintf("docker rmi %s", imageName) + cmdString := fmt.Sprintf("docker rmi %s", imageNameWithTag) err := ExecInSystem(".", cmdString, nil, true) return err @@ -85,9 +74,9 @@ func (op *ShellOperator) ContainerIfRunning(containerName string) bool { return false } -func (op *ShellOperator) ContainerRun(opts docker.RunOptions, params ...string) error { +func (op *ShellOperator) ContainerRun(opts *docker.RunOptions) error { // build the command - cmdString, err := BuildContainerRunCommand(opts, params...) + cmdString, err := BuildContainerRunCommand(opts) if err != nil { return err } @@ -103,7 +92,7 @@ func (op *ShellOperator) ContainerRun(opts docker.RunOptions, params ...string) } // BuildContainerRunCommand builds the docker run command string from the given options and additional params -func BuildContainerRunCommand(opts docker.RunOptions, params ...string) (string, error) { +func BuildContainerRunCommand(opts *docker.RunOptions) (string, error) { if err := opts.Validate(); err != nil { return "", err } @@ -123,7 +112,7 @@ func BuildContainerRunCommand(opts docker.RunOptions, params ...string) (string, for _, volume := range opts.Volumes { cmdBuilder.WriteString(fmt.Sprintf("--volume %s:%s ", volume.HostPath, volume.ContainerPath)) } - for _, param := range params { + for _, param := range opts.RunParams { cmdBuilder.WriteString(param + " ") } cmdBuilder.WriteString(docker.CombineImageNameAndTag(opts.ImageName, opts.ImageTag)) @@ -183,38 +172,73 @@ func (op *ShellOperator) ContainerGetHostname(container string) (string, error) } -func (op *ShellOperator) ContainerGetPortBinding(container, containerPort, protocol string) (hostPort string, err error) { +func (op *ShellOperator) ContainerListPortPublishes(containerName string) ([]docker.PortPublish, error) { // get container port binding map // the result is like: // 22/tcp->8122 // 443/tcp->8443 // 80/tcp->8180 format := "'{{range $p,$conf := .NetworkSettings.Ports}}{{$p}}->{{(index $conf 0).HostPort}}{{println}}{{end}}'" - cmdString := fmt.Sprintf("docker inspect --format=%s %s", format, container) + cmdString := fmt.Sprintf("docker inspect --format=%s %s", format, containerName) outputBuffer := &bytes.Buffer{} - err = ExecInSystem(".", cmdString, outputBuffer, false) + err := ExecInSystem(".", cmdString, outputBuffer, false) if err != nil { - return "", err + return nil, err } portBindings := strings.Split(strings.TrimSpace(outputBuffer.String()), "\n") - log.Debugf("Container %v port bindings: %v", container, portBindings) + log.Debugf("Container %v port bindings: %v", containerName, portBindings) + + publishes, err := buildPortPublishes(portBindings) + if err != nil { + return publishes, err + } + + return publishes, nil +} + +func buildPortPublishes(portBindings []string) (PortPublishes []docker.PortPublish, err error) { + // 22/tcp->8122 + // 443/tcp->8443 + // 80/tcp->8180 + re := regexp.MustCompile(`^(\d+)/(tcp|udp)->(\d+)$`) - // transfer port bindings to map - portBindingsMap := make(map[string]string) for _, portBinding := range portBindings { - portBindingParts := strings.Split(portBinding, "->") - if len(portBindingParts) != 2 { - return "", fmt.Errorf("Invalid port binding: %v", portBinding) + match := re.FindStringSubmatch(portBinding) + // match e.g. ["22/tcp->8122", "22", "tcp", "8122"] + if len(match) != 4 { + return nil, fmt.Errorf("invalid port binding: %v", portBinding) + } + + hostPort, err := strconv.Atoi(match[3]) + if err != nil { + return nil, fmt.Errorf("invalid port binding: %v", portBinding) + } + containerPort, err := strconv.Atoi(match[1]) + if err != nil { + return nil, fmt.Errorf("invalid port binding: %v", portBinding) + } + + portPublish := docker.PortPublish{ + ContainerPort: uint(containerPort), + HostPort: uint(hostPort), } - portBindingsMap[portBindingParts[0]] = portBindingParts[1] + PortPublishes = append(PortPublishes, portPublish) } - portKey := fmt.Sprintf("%s/%s", containerPort, protocol) - hostPort, ok := portBindingsMap[portKey] - if !ok { - return "", fmt.Errorf("No port binding for %v", portKey) + return PortPublishes, nil +} + +func (op *ShellOperator) ContainerGetPortBinding(container string, containerPort uint) (hostPort uint, err error) { + portBindings, err := op.ContainerListPortPublishes(container) + if err != nil { + return 0, err } - return hostPort, nil + for _, portBinding := range portBindings { + if portBinding.ContainerPort == containerPort { + return portBinding.HostPort, nil + } + } + return 0, fmt.Errorf("container %v does not have port binding for port %v", container, containerPort) } diff --git a/pkg/util/docker/dockersh/operator_test.go b/pkg/util/docker/dockersh/operator_test.go index b6df3ae06..f3af9005f 100644 --- a/pkg/util/docker/dockersh/operator_test.go +++ b/pkg/util/docker/dockersh/operator_test.go @@ -9,69 +9,89 @@ import ( "github.com/devstream-io/devstream/pkg/util/docker" ) -var _ = Describe("Operator", func() { - Describe("BuildContainerRunCommand method", func() { - var ( - opts docker.RunOptions - params []string - ) - - When(" the options are invalid", func() { - BeforeEach(func() { - opts = docker.RunOptions{} - }) - - It("should return an error", func() { - _, err := BuildContainerRunCommand(opts, params...) - Expect(err).To(HaveOccurred()) - }) +var _ = Describe("BuildContainerRunCommand method", func() { + var opts *docker.RunOptions + + When(" the options are invalid", func() { + BeforeEach(func() { + opts = &docker.RunOptions{} + }) + + It("should return an error", func() { + _, err := BuildContainerRunCommand(opts) + Expect(err).To(HaveOccurred()) }) + }) + + When(" the options are valid(e.g. gitlab-ce)", func() { + BeforeEach(func() { + buildOpts := func() *docker.RunOptions { + opts := &docker.RunOptions{} + opts.ImageName = "gitlab/gitlab-ce" + opts.ImageTag = "rc" + opts.Hostname = "gitlab.example.com" + opts.ContainerName = "gitlab" + opts.RestartAlways = true - When(" the options are valid(e.g. gitlab-ce)", func() { - BeforeEach(func() { - buildOpts := func() docker.RunOptions { - opts := docker.RunOptions{} - opts.ImageName = "gitlab/gitlab-ce" - opts.ImageTag = "rc" - opts.Hostname = "gitlab.example.com" - opts.ContainerName = "gitlab" - opts.RestartAlways = true - - portPublishes := []docker.PortPublish{ - {HostPort: 8122, ContainerPort: 22}, - {HostPort: 8180, ContainerPort: 80}, - {HostPort: 8443, ContainerPort: 443}, - } - opts.PortPublishes = portPublishes - - gitLabHome := "/srv/gitlab" - - opts.Volumes = []docker.Volume{ - {HostPath: filepath.Join(gitLabHome, "config"), ContainerPath: "/etc/gitlab"}, - {HostPath: filepath.Join(gitLabHome, "data"), ContainerPath: "/var/opt/gitlab"}, - {HostPath: filepath.Join(gitLabHome, "logs"), ContainerPath: "/var/log/gitlab"}, - } - - return opts + portPublishes := []docker.PortPublish{ + {HostPort: 8122, ContainerPort: 22}, + {HostPort: 8180, ContainerPort: 80}, + {HostPort: 8443, ContainerPort: 443}, } + opts.PortPublishes = portPublishes + + gitLabHome := "/srv/gitlab" + + opts.Volumes = []docker.Volume{ + {HostPath: filepath.Join(gitLabHome, "config"), ContainerPath: "/etc/gitlab"}, + {HostPath: filepath.Join(gitLabHome, "data"), ContainerPath: "/var/opt/gitlab"}, + {HostPath: filepath.Join(gitLabHome, "logs"), ContainerPath: "/var/log/gitlab"}, + } + + opts.RunParams = []string{"--shm-size 256m"} + + return opts + } - opts = buildOpts() - params = []string{"--shm-size 256m"} - }) - - It("should return the correct command", func() { - cmdBuild, err := BuildContainerRunCommand(opts, params...) - Expect(err).NotTo(HaveOccurred()) - cmdExpect := "docker run --detach --hostname gitlab.example.com" + - " --publish 8122:22 --publish 8180:80 --publish 8443:443" + - " --name gitlab --restart always" + - " --volume /srv/gitlab/config:/etc/gitlab" + - " --volume /srv/gitlab/data:/var/opt/gitlab" + - " --volume /srv/gitlab/logs:/var/log/gitlab" + - " --shm-size 256m gitlab/gitlab-ce:rc" - Expect(cmdBuild).To(Equal(cmdExpect)) - }) + opts = buildOpts() + }) + + It("should return the correct command", func() { + cmdBuild, err := BuildContainerRunCommand(opts) + Expect(err).NotTo(HaveOccurred()) + cmdExpect := "docker run --detach --hostname gitlab.example.com" + + " --publish 8122:22 --publish 8180:80 --publish 8443:443" + + " --name gitlab --restart always" + + " --volume /srv/gitlab/config:/etc/gitlab" + + " --volume /srv/gitlab/data:/var/opt/gitlab" + + " --volume /srv/gitlab/logs:/var/log/gitlab" + + " --shm-size 256m gitlab/gitlab-ce:rc" + Expect(cmdBuild).To(Equal(cmdExpect)) + }) + + }) +}) + +var _ = Describe("build[] PortPublish func ", func() { + var portBindings []string + + BeforeEach(func() { + portBindings = []string{ + "22/tcp->8122", + "443/tcp->8443", + "80/tcp->8180", + } + }) + When(" the options are valid", func() { + It("should return the correct port publishes", func() { + publishes, err := buildPortPublishes(portBindings) + Expect(err).NotTo(HaveOccurred()) + Expect(publishes).To(Equal([]docker.PortPublish{ + {HostPort: 8122, ContainerPort: 22}, + {HostPort: 8443, ContainerPort: 443}, + {HostPort: 8180, ContainerPort: 80}, + })) }) }) }) diff --git a/pkg/util/docker/option.go b/pkg/util/docker/option.go index 3e7377e6b..4b4a8d8fb 100644 --- a/pkg/util/docker/option.go +++ b/pkg/util/docker/option.go @@ -3,6 +3,8 @@ package docker import ( "fmt" "strings" + + "go.uber.org/multierr" ) // RunOptions is used to pass options to ContainerRunWithOptions @@ -13,8 +15,9 @@ type ( Hostname string ContainerName string PortPublishes []PortPublish - Volumes []Volume + Volumes Volumes RestartAlways bool + RunParams []string } Volume struct { @@ -49,26 +52,13 @@ func (opts *RunOptions) Validate() error { } } - return CombineErrs(errs) + return multierr.Combine(errs...) } func CombineImageNameAndTag(imageName, tag string) string { return imageName + ":" + tag } -func CombineErrs(errs []error) error { - if len(errs) == 0 { - return nil - } - - errsString := make([]string, len(errs)) - for _, err := range errs { - errsString = append(errsString, err.Error()) - } - - return fmt.Errorf(strings.Join(errsString, ";")) -} - func (volumes Volumes) ExtractHostPaths() []string { hostPaths := make([]string, len(volumes)) for i, volume := range volumes { diff --git a/pkg/util/file/file.go b/pkg/util/file/file.go new file mode 100644 index 000000000..9230deb1e --- /dev/null +++ b/pkg/util/file/file.go @@ -0,0 +1,120 @@ +package file + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" +) + +const ( + defaultTempName = "pkg-util-file-create_" + appNamePlaceHolder = "_app_name_" +) + +type fileProcesser func(string) (string, error) +type renderProcesser func(string, string, map[string]interface{}) (string, error) + +// TemplateConfig contains all file template getter, process and render info +type TemplateConfig struct { + info string + templateName string + vars map[string]interface{} + getter fileProcesser + processor fileProcesser + render renderProcesser + processErr error +} + +func NewTemplate() *TemplateConfig { + return &TemplateConfig{} +} + +// FromLocal will check if local file exist +func (c *TemplateConfig) FromLocal(path string) *TemplateConfig { + ex, err := os.Executable() + if err != nil { + c.processErr = err + } else { + exPath := filepath.Dir(ex) + c.info = filepath.Join(exPath, path) + } + c.getter = getFileFromLocal + return c +} + +// FromRemote will create a file from remote url +func (c *TemplateConfig) FromRemote(url string) *TemplateConfig { + c.getter = getFileFromURL + c.info = url + return c +} + +// FromContent will create a file for content +func (c *TemplateConfig) FromContent(content string) *TemplateConfig { + c.getter = getFileFromContent + c.info = content + return c +} + +func (c *TemplateConfig) UnzipFile() *TemplateConfig { + c.processor = unZipFileProcesser + return c +} + +func (c *TemplateConfig) RenderRepoDIr(templateName string, vars map[string]interface{}) *TemplateConfig { + c.render = renderGitRepoDir + c.templateName = templateName + c.vars = vars + return c +} + +// RenderFile will create render config +func (c *TemplateConfig) RenderFile(templateName string, vars map[string]interface{}) *TemplateConfig { + c.render = renderFile + c.templateName = templateName + c.vars = vars + return c +} + +func (c *TemplateConfig) Run() (string, error) { + // check if has error before + if c.processErr != nil { + return "", c.processErr + } + + // check if info is empty + if c.info == "" { + return "", fmt.Errorf("file util: content is not setted") + } + var ( + outPutName string + err error + ) + // 1. run Getter func to get file + outPutName, err = c.getter(c.info) + if err != nil { + return "", err + } + // 2. if need file process, run processer + if c.processor != nil { + outPutName, err = c.processor(outPutName) + if err != nil { + return "", err + } + } + // 3. if need render, render func + if c.render != nil { + return c.render(c.templateName, outPutName, c.vars) + } + return outPutName, nil +} + +// CopyFile will copy file content from src to dst +func CopyFile(src, dest string) error { + bytesRead, err := ioutil.ReadFile(src) + if err != nil { + return err + } + return ioutil.WriteFile(dest, bytesRead, 0644) +} diff --git a/pkg/util/file/file_suite_test.go b/pkg/util/file/file_suite_test.go new file mode 100644 index 000000000..ac2674664 --- /dev/null +++ b/pkg/util/file/file_suite_test.go @@ -0,0 +1,13 @@ +package file_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestPlanmanager(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Util File Suite") +} diff --git a/pkg/util/file/file_test.go b/pkg/util/file/file_test.go new file mode 100644 index 000000000..f9358daa2 --- /dev/null +++ b/pkg/util/file/file_test.go @@ -0,0 +1,157 @@ +package file + +import ( + "errors" + "io/ioutil" + "os" + "path/filepath" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("file struct", func() { + var ( + templateConfig *TemplateConfig + ) + BeforeEach(func() { + templateConfig = NewTemplate() + }) + + Context("FromLocal method", func() { + It("should set getter method and info", func() { + testLocation := "test_location" + Expect(templateConfig.getter).Should(BeNil()) + config := templateConfig.FromLocal(testLocation) + Expect(config.getter).ShouldNot(BeNil()) + Expect(config.info).Should(ContainSubstring(testLocation)) + }) + }) + + Context("FromContent method", func() { + It("should set getter method and info", func() { + testContent := "this is a test content" + Expect(templateConfig.getter).Should(BeNil()) + config := templateConfig.FromContent(testContent) + Expect(config.getter).ShouldNot(BeNil()) + Expect(config.info).Should(ContainSubstring(testContent)) + }) + }) + + Context("FromRemote method", func() { + It("should set url method and info", func() { + Expect(templateConfig.getter).Should(BeNil()) + testURL := "http://www.test.com" + config := templateConfig.FromRemote(testURL) + Expect(config.getter).ShouldNot(BeNil()) + Expect(config.info).Should(Equal(testURL)) + }) + }) + + Context("Run method", func() { + var ( + inputFileName string + outputFileName string + mockErr error + mockFunc fileProcesser + mockFailedFunc fileProcesser + mockRenderFunc renderProcesser + mockRenderFailedFunc renderProcesser + ) + BeforeEach(func() { + inputFileName = "test" + outputFileName = "changed_file_name" + mockErr = errors.New("mock test") + mockFunc = func(input string) (string, error) { + return outputFileName, nil + } + mockFailedFunc = func(input string) (string, error) { + return "", mockErr + } + mockRenderFunc = func(templateName, filePath string, vars map[string]interface{}) (string, error) { + return filePath, nil + } + mockRenderFailedFunc = func(templateName, filePath string, vars map[string]interface{}) (string, error) { + return "", mockErr + } + }) + + When("getter method process error", func() { + It("should return err", func() { + templateConfig.info = inputFileName + templateConfig.getter = mockFailedFunc + _, err := templateConfig.Run() + Expect(err).Error().Should(HaveOccurred()) + + }) + }) + + When("render method process error", func() { + It("should return err", func() { + templateConfig.info = inputFileName + templateConfig.getter = mockFunc + templateConfig.render = mockRenderFailedFunc + _, err := templateConfig.Run() + Expect(err).Error().Should(HaveOccurred()) + }) + }) + + When("struct has error", func() { + It("should return err driectly", func() { + errMsg := "test_err" + templateConfig.processErr = errors.New(errMsg) + _, err := templateConfig.Run() + Expect(err).Error().Should(HaveOccurred()) + Expect(err.Error()).Should(Equal(errMsg)) + }) + It("should return empty error", func() { + templateConfig.info = "" + _, err := templateConfig.Run() + Expect(err).Error().Should(HaveOccurred()) + }) + AfterEach(func() { + templateConfig = NewTemplate() + }) + }) + When("all config is right", func() { + It("should return name and path", func() { + templateConfig.info = inputFileName + templateConfig.getter = mockFunc + templateConfig.render = mockRenderFunc + output, err := templateConfig.Run() + Expect(err).Error().ShouldNot(HaveOccurred()) + Expect(output).Should(Equal(outputFileName)) + }) + }) + }) +}) + +var _ = Describe("CopyFile func", func() { + var ( + tempDir, srcPath, dstPath string + testContent []byte + ) + + BeforeEach(func() { + testContent = []byte("test_content") + tempDir = GinkgoT().TempDir() + srcPath = filepath.Join(tempDir, "src") + dstPath = filepath.Join(tempDir, "dst") + f1, err := os.Create(srcPath) + Expect(err).Error().ShouldNot(HaveOccurred()) + defer f1.Close() + f2, err := os.Create(dstPath) + Expect(err).Error().ShouldNot(HaveOccurred()) + defer f2.Close() + }) + + It("should copy content form src to dst", func() { + err := ioutil.WriteFile(srcPath, testContent, 0666) + Expect(err).Error().ShouldNot(HaveOccurred()) + err = CopyFile(srcPath, dstPath) + Expect(err).Error().ShouldNot(HaveOccurred()) + data, err := ioutil.ReadFile(dstPath) + Expect(err).Error().ShouldNot(HaveOccurred()) + Expect(data).Should(Equal(testContent)) + }) +}) diff --git a/pkg/util/file/local.go b/pkg/util/file/local.go new file mode 100644 index 000000000..eecc22dbc --- /dev/null +++ b/pkg/util/file/local.go @@ -0,0 +1,36 @@ +package file + +import ( + "bytes" + "io" + "os" + + "github.com/devstream-io/devstream/pkg/util/log" +) + +// getFileFromLocal will check if local file exist +func getFileFromLocal(location string) (string, error) { + _, err := os.Stat(location) + if err != nil { + return "", err + } + return location, nil +} + +// getFileFromContent will create a temp file based on content +func getFileFromContent(content string) (string, error) { + // 1. create temp file for save content + tempFile, err := os.CreateTemp("", defaultTempName) + if err != nil { + return "", err + } + defer tempFile.Close() + + // 2. save content to file + _, err = io.Copy(tempFile, bytes.NewBufferString(content)) + if err != nil { + log.Debugf("Download file copy content failed: %s", err) + return "", err + } + return tempFile.Name(), nil +} diff --git a/pkg/util/file/local_test.go b/pkg/util/file/local_test.go new file mode 100644 index 000000000..528463ae0 --- /dev/null +++ b/pkg/util/file/local_test.go @@ -0,0 +1,59 @@ +package file + +import ( + "io/ioutil" + "os" + "path/filepath" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("getFileFromLocal func", func() { + var ( + tempDir string + notExistFile string + existFile string + ) + BeforeEach(func() { + tempDir = GinkgoT().TempDir() + notExistFile = filepath.Join(tempDir, "not_exist") + existFile = filepath.Join(tempDir, "exist") + f, err := os.Create(existFile) + Expect(err).Error().ShouldNot(HaveOccurred()) + defer f.Close() + }) + + When("file not exist", func() { + It("should contain error", func() { + _, err := getFileFromLocal(notExistFile) + Expect(err).Error().Should(HaveOccurred()) + }) + }) + + When("file exist", func() { + It("should config normal", func() { + loc, err := getFileFromLocal(existFile) + Expect(err).ShouldNot(HaveOccurred()) + Expect(loc).Should(Equal(existFile)) + }) + }) +}) + +var _ = Describe("getFileFromContent func", func() { + var ( + testContent string + ) + BeforeEach(func() { + testContent = "This is a test Content" + + }) + + It("should return a file for content", func() { + fileName, _ := getFileFromContent(testContent) + // check file exist + content, err := ioutil.ReadFile(fileName) + Expect(err).ShouldNot(HaveOccurred()) + Expect(string(content)).Should(Equal(testContent)) + }) +}) diff --git a/pkg/util/file/processer.go b/pkg/util/file/processer.go new file mode 100644 index 000000000..4f491b5e2 --- /dev/null +++ b/pkg/util/file/processer.go @@ -0,0 +1,23 @@ +package file + +import ( + "os" + "path/filepath" + + "github.com/devstream-io/devstream/pkg/util/zip" +) + +// unZipFileProcesser will unzip zip file and return unzip files dir path +func unZipFileProcesser(zipFilePath string) (string, error) { + // 1. create tempDir to save unzip files + dirName := filepath.Dir(zipFilePath) + tempDirName, err := os.MkdirTemp(dirName, defaultTempName) + if err != nil { + return "", err + } + err = zip.UnZip(zipFilePath, tempDirName) + if err != nil { + return "", err + } + return tempDirName, nil +} diff --git a/pkg/util/file/processer_test.go b/pkg/util/file/processer_test.go new file mode 100644 index 000000000..5bdcc7798 --- /dev/null +++ b/pkg/util/file/processer_test.go @@ -0,0 +1,50 @@ +package file + +import ( + "archive/zip" + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("unZipFileProcesser func", func() { + var ( + zipFileName, tempDir, zipLocation, tempFile string + ) + BeforeEach(func() { + zipLocation = "test" + tempFile = "testfile" + tempDir = GinkgoT().TempDir() + zipFile, err := os.CreateTemp(tempDir, "*.zip") + zipFileName = zipFile.Name() + Expect(err).Error().ShouldNot(HaveOccurred()) + defer zipFile.Close() + writer := zip.NewWriter(zipFile) + defer writer.Close() + + newFile, err := os.CreateTemp(tempDir, tempFile) + Expect(err).Error().ShouldNot(HaveOccurred()) + defer newFile.Close() + w1, _ := writer.Create(fmt.Sprintf("%s/%s", zipLocation, tempFile)) + _, err = io.Copy(w1, newFile) + Expect(err).Error().ShouldNot(HaveOccurred()) + }) + + It("should work", func() { + dstPath, err := unZipFileProcesser(zipFileName) + Expect(err).Error().ShouldNot(HaveOccurred()) + dirFiles, err := ioutil.ReadDir(dstPath) + Expect(err).Error().ShouldNot(HaveOccurred()) + Expect(len(dirFiles)).Should(Equal(1)) + Expect(dirFiles[0].Name()).Should(Equal(zipLocation)) + zipDirFiles, err := ioutil.ReadDir(filepath.Join(dstPath, zipLocation)) + Expect(err).Error().ShouldNot(HaveOccurred()) + Expect(len(dirFiles)).Should(Equal(1)) + Expect(zipDirFiles[0].Name()).Should(Equal(tempFile)) + }) +}) diff --git a/pkg/util/file/remote.go b/pkg/util/file/remote.go new file mode 100644 index 000000000..e90743ce5 --- /dev/null +++ b/pkg/util/file/remote.go @@ -0,0 +1,42 @@ +package file + +import ( + "fmt" + "io" + "net/http" + "os" + + "github.com/devstream-io/devstream/pkg/util/log" +) + +// getFileFromURL will download file from url and return file path +func getFileFromURL(url string) (string, error) { + // 1. create temp file for save content + tempFile, err := os.CreateTemp("", defaultTempName) + if err != nil { + return "", err + } + defer tempFile.Close() + + // 2. download content to file + log.Debugf("Download URL: %s to filename %s", url, tempFile.Name()) + resp, err := http.Get(url) + + // 3. check response error + if err != nil { + log.Debugf("Download file from url failed: %s", err) + return "", err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("Download file from url failed: %+v", resp) + } + + // 4. copy response body to template file + _, err = io.Copy(tempFile, resp.Body) + if err != nil { + log.Debugf("Download file copy content failed: %s", err) + return "", err + } + return tempFile.Name(), nil +} diff --git a/pkg/util/file/remote_test.go b/pkg/util/file/remote_test.go new file mode 100644 index 000000000..09e1c6fbe --- /dev/null +++ b/pkg/util/file/remote_test.go @@ -0,0 +1,65 @@ +package file + +import ( + "fmt" + "io/ioutil" + "net/http" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/onsi/gomega/ghttp" +) + +var _ = Describe("getFileFromURL", func() { + var ( + server *ghttp.Server + testPath, remoteContent string + ) + + BeforeEach(func() { + testPath = "/testPath" + server = ghttp.NewServer() + }) + + When("server return error code", func() { + BeforeEach(func() { + server.AppendHandlers( + ghttp.CombineHandlers( + ghttp.VerifyRequest("GET", testPath), + ghttp.RespondWith(http.StatusNotFound, ""), + ), + ) + + }) + It("should return err", func() { + reqURL := fmt.Sprintf("%s%s", server.URL(), testPath) + _, err := getFileFromURL(reqURL) + Expect(err).Error().Should(HaveOccurred()) + }) + }) + + When("server return success", func() { + BeforeEach(func() { + remoteContent = "download content" + server.AppendHandlers( + ghttp.CombineHandlers( + ghttp.VerifyRequest("GET", testPath), + ghttp.RespondWith(http.StatusOK, remoteContent), + ), + ) + }) + + It("should create file with content", func() { + reqURL := fmt.Sprintf("%s%s", server.URL(), testPath) + fileName, err := getFileFromURL(reqURL) + Expect(err).Error().ShouldNot(HaveOccurred()) + fileContent, err := ioutil.ReadFile(fileName) + Expect(err).Error().ShouldNot(HaveOccurred()) + Expect(string(fileContent)).Should(Equal(remoteContent)) + }) + }) + + AfterEach(func() { + server.Close() + }) +}) diff --git a/pkg/util/file/render.go b/pkg/util/file/render.go new file mode 100644 index 000000000..1490e3ed9 --- /dev/null +++ b/pkg/util/file/render.go @@ -0,0 +1,96 @@ +package file + +import ( + "io/fs" + "os" + "path/filepath" + "regexp" + "strings" + + "github.com/devstream-io/devstream/pkg/util/log" + "github.com/devstream-io/devstream/pkg/util/template" +) + +// renderGitRepoDir will render files in srcPath to dstPath, It will render two things +// 1. replace filename with appNamePlaceHolder to templateName +// 2. render files in srcPath and output to dstPath +func renderGitRepoDir(templateName, srcPath string, vars map[string]interface{}) (string, error) { + // 1. create temp dir for destination + dstPath, err := os.MkdirTemp("", defaultTempName) + if err != nil { + return "", err + } + if err := filepath.Walk(srcPath, func(path string, info fs.FileInfo, err error) error { + if err != nil { + log.Debugf("Walk error: %s.", err) + return err + } + + relativePath := strings.Replace(path, srcPath, "", 1) + if strings.Contains(relativePath, ".git/") { + log.Debugf("Walk: ignore file %s.", "./git/xxx") + return nil + } + + if strings.HasSuffix(relativePath, "README.md") { + log.Debugf("Walk: ignore file %s.", "README.md") + return nil + } + + // replace template with appName + outputWithRepoName, err := replaceAppNameInPathStr(relativePath, appNamePlaceHolder, templateName) + if err != nil { + log.Debugf("Walk: Replace file name failed %s.", path) + return err + } + outputPath := filepath.Join(dstPath, outputWithRepoName) + + if info.IsDir() { + log.Debugf("Walk: found dir: %s.", path) + if err != nil { + return err + } + + if err := os.MkdirAll(outputPath, os.ModePerm); err != nil { + return err + } + log.Debugf("Walk: new output dir created: %s.", outputPath) + return nil + } + + log.Debugf("Walk: found file: %s.", path) + + // if file endswith tpl, render this file, else copy this file directly + if strings.Contains(path, "tpl") { + outputPath = strings.TrimSuffix(outputPath, ".tpl") + return template.RenderForFile(templateName, path, outputPath, vars) + } + return CopyFile(path, outputPath) + }); err != nil { + return "", err + } + return dstPath, nil +} + +func replaceAppNameInPathStr(filePath, appNamePlaceHolder, appName string) (string, error) { + if !strings.Contains(filePath, appNamePlaceHolder) { + return filePath, nil + } + newFilePath := regexp.MustCompile(appNamePlaceHolder).ReplaceAllString(filePath, appName) + log.Debugf("Replace file path place holder. Before: %s, after: %s.", filePath, newFilePath) + return newFilePath, nil +} + +func renderFile(templateName string, srcPath string, vars map[string]interface{}) (string, error) { + tempFile, err := os.CreateTemp("", defaultTempName) + if err != nil { + return "", err + } + defer tempFile.Close() + + err = template.RenderForFile(templateName, srcPath, tempFile.Name(), vars) + if err != nil { + return "", err + } + return filepath.Abs(tempFile.Name()) +} diff --git a/pkg/util/file/render_test.go b/pkg/util/file/render_test.go new file mode 100644 index 000000000..8a1c62a0f --- /dev/null +++ b/pkg/util/file/render_test.go @@ -0,0 +1,183 @@ +package file + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("renderFile func", func() { + var ( + tempFilePath, templateContent string + ) + BeforeEach(func() { + templateContent = ` + metadata: + name: "[[ .App.Name ]]" + namespace: "[[ .App.NameSpace ]]"` + tempFile, err := os.CreateTemp("", "test_temp") + Expect(err).Error().ShouldNot(HaveOccurred()) + defer tempFile.Close() + tempFilePath = tempFile.Name() + err = ioutil.WriteFile(tempFilePath, []byte(templateContent), 0666) + Expect(err).Error().ShouldNot(HaveOccurred()) + }) + + When("srcPath file is not exist", func() { + It("should return err", func() { + _, err := renderFile("test_app", "not_exist_path", map[string]interface{}{}) + Expect(err).Error().Should(HaveOccurred()) + }) + }) + + When("input vars is empty", func() { + It("should return err", func() { + _, err := renderFile("test_app", tempFilePath, map[string]interface{}{}) + Expect(err).Error().Should(HaveOccurred()) + }) + }) + + When("input vars and right srcPath", func() { + var rightContent string + BeforeEach(func() { + rightContent = ` + metadata: + name: "test" + namespace: "test_namespace"` + }) + + It("should work normal", func() { + dstPath, err := renderFile("test_app", tempFilePath, map[string]interface{}{ + "App": map[string]interface{}{ + "Name": "test", + "NameSpace": "test_namespace", + }, + }) + Expect(err).Error().ShouldNot(HaveOccurred()) + content, err := ioutil.ReadFile(dstPath) + Expect(err).Error().ShouldNot(HaveOccurred()) + Expect(string(content)).Should(Equal(rightContent)) + }) + }) +}) + +var _ = Describe("replaceAppNameInPathStr func", func() { + var ( + placeHolder string + filePath string + appName string + ) + BeforeEach(func() { + placeHolder = "__app__" + appName = "test" + }) + When("filePath not contains placeHolder", func() { + BeforeEach(func() { + filePath = "/app/dev" + }) + It("should return same filePath", func() { + newPath, err := replaceAppNameInPathStr(filePath, placeHolder, appName) + Expect(err).Error().ShouldNot(HaveOccurred()) + Expect(newPath).Should(Equal(filePath)) + }) + }) + When("filPath contains placeHolder", func() { + BeforeEach(func() { + filePath = fmt.Sprintf("app/%s/dev", placeHolder) + }) + It("should replace placeHolder with app name", func() { + newPath, err := replaceAppNameInPathStr(filePath, placeHolder, appName) + Expect(err).Error().ShouldNot(HaveOccurred()) + Expect(newPath).Should(Equal(fmt.Sprintf("app/%s/dev", appName))) + }) + }) +}) + +var _ = Describe("renderGitRepoDir func", func() { + var ( + vars map[string]interface{} + srcPath, contentDir, rawContent, tplContent, renderdContent string + ) + + createFile := func(filePath, content string) { + f, err := os.Create(filePath) + Expect(err).Error().ShouldNot(HaveOccurred()) + defer f.Close() + err = ioutil.WriteFile(filePath, []byte(content), 0755) + Expect(err).Error().ShouldNot(HaveOccurred()) + } + createDir := func(dirPath string) { + err := os.Mkdir(dirPath, 0755) + Expect(err).Error().ShouldNot(HaveOccurred()) + } + BeforeEach(func() { + rawContent = "This is a file without template variable" + tplContent = ` + metadata: + name: "[[ .App.Name ]]" + namespace: "[[ .App.NameSpace ]]"` + renderdContent = ` + metadata: + name: "test" + namespace: "test_namespace"` + + }) + + When("srcPath is not exist", func() { + BeforeEach(func() { + srcPath = "not_exist_path" + }) + It("should return err", func() { + _, err := renderGitRepoDir("test", srcPath, vars) + Expect(err).Error().Should(HaveOccurred()) + }) + }) + When("all config is right", func() { + BeforeEach(func() { + contentDir = "content" + vars = map[string]interface{}{ + "App": map[string]interface{}{ + "Name": "test", + "NameSpace": "test_namespace", + }, + } + srcPath = GinkgoT().TempDir() + gitPath := filepath.Join(srcPath, ".git") + createDir(gitPath) + createFile(filepath.Join(gitPath, "gitFile"), tplContent) + createFile(filepath.Join(srcPath, "README.md"), "") + contentDirPath := filepath.Join(srcPath, contentDir) + createDir(contentDirPath) + createFile(filepath.Join(contentDirPath, "test.yaml.tpl"), tplContent) + createFile(filepath.Join(contentDirPath, "raw.txt"), rawContent) + }) + It("should render all dir", func() { + dstPath, err := renderGitRepoDir("test", srcPath, vars) + Expect(err).Error().ShouldNot(HaveOccurred()) + files, err := ioutil.ReadDir(dstPath) + Expect(err).Error().ShouldNot(HaveOccurred()) + // test README.md dir is not copied + Expect(len(files)).Should(Equal(2)) + // test git dir files should not copied + gitDirFiles, err := ioutil.ReadDir(filepath.Join(dstPath, ".git")) + Expect(err).Error().ShouldNot(HaveOccurred()) + Expect(len(gitDirFiles)).Should(Equal(0)) + // test content dir files is copied + contentDirLoc := filepath.Join(dstPath, contentDir) + contentFiles, err := ioutil.ReadDir(contentDirLoc) + Expect(err).Error().ShouldNot(HaveOccurred()) + Expect(len(contentFiles)).Should(Equal(2)) + // test file content + tplFileContent, err := ioutil.ReadFile(filepath.Join(contentDirLoc, "test.yaml")) + Expect(err).Error().ShouldNot(HaveOccurred()) + Expect(string(tplFileContent)).Should(Equal(renderdContent)) + rawFileContent, err := ioutil.ReadFile(filepath.Join(contentDirLoc, "raw.txt")) + Expect(err).Error().ShouldNot(HaveOccurred()) + Expect(string(rawFileContent)).Should(Equal(rawContent)) + }) + }) +}) diff --git a/pkg/util/github/download.go b/pkg/util/github/download.go index 0d9e8725b..3acf3a0c3 100644 --- a/pkg/util/github/download.go +++ b/pkg/util/github/download.go @@ -80,21 +80,12 @@ func (c *Client) DownloadAsset(tagName, assetName, fileName string) error { return nil } -func (c *Client) DownloadLatestCodeAsZipFile() error { +func (c *Client) GetLatestCodeZipURL() string { var owner = c.Owner if c.Org != "" { owner = c.Org } - - latestCodeZipfileDownloadUrl := fmt.Sprintf(DefaultLatestCodeZipfileDownloadUrlFormat, owner, c.Repo) - log.Debugf("LatestCodeZipfileDownloadUrl: %s.", latestCodeZipfileDownloadUrl) - - n, err := downloader.Download(latestCodeZipfileDownloadUrl, DefaultLatestCodeZipfileName, c.WorkPath) - if err != nil { - log.Debugf("Failed to download zip file from %s.", latestCodeZipfileDownloadUrl) - return err - } - - log.Debugf("Downloaded <%d> bytes.", n) - return nil + latestCodeZipfileDownloadURL := fmt.Sprintf(DefaultLatestCodeZipfileDownloadUrlFormat, owner, c.Repo) + log.Debugf("LatestCodeZipfileDownloadUrl: %s.", latestCodeZipfileDownloadURL) + return latestCodeZipfileDownloadURL } diff --git a/pkg/util/github/download_test.go b/pkg/util/github/download_test.go index b3444583b..ea835aabf 100644 --- a/pkg/util/github/download_test.go +++ b/pkg/util/github/download_test.go @@ -199,8 +199,7 @@ var _ = Describe("DownloadAsset", func() { var _ = Describe("DownloadLatestCodeAsZipFile", func() { const ( - owner, repo, org = "owner", "repo", "org" - rightWorkPath, wrongWorkPath = "./", "//" + owner, repo, org = "owner", "repo", "org" ) var ( @@ -218,34 +217,14 @@ var _ = Describe("DownloadLatestCodeAsZipFile", func() { }) When("the url is correct", func() { - BeforeEach(func() { - workPath = rightWorkPath - }) It("should return no error", func() { ghClient, err := github.NewClientWithOption(opts, serverURL) Expect(err).NotTo(HaveOccurred()) Expect(ghClient).NotTo(Equal(nil)) - err = ghClient.DownloadLatestCodeAsZipFile() - Expect(err).To(Succeed()) - }) - }) - - When("the url is incorrect(caused by wrong work path)", func() { - BeforeEach(func() { - workPath = wrongWorkPath - }) - - It("should return an error", func() { - ghClient, err := github.NewClientWithOption(opts, serverURL) - Expect(err).NotTo(HaveOccurred()) - Expect(ghClient).NotTo(Equal(nil)) - err = ghClient.DownloadLatestCodeAsZipFile() - Expect(err).NotTo(Succeed()) + url := ghClient.GetLatestCodeZipURL() + Expect(url).ShouldNot(BeEmpty()) }) }) - AfterEach(func() { - DeferCleanup(io.DeleteFile, workPath+github.DefaultLatestCodeZipfileName) - }) }) diff --git a/pkg/util/github/workflow_test.go b/pkg/util/github/workflow_test.go index c921855ce..58df96e91 100644 --- a/pkg/util/github/workflow_test.go +++ b/pkg/util/github/workflow_test.go @@ -5,10 +5,11 @@ import ( "net/http" "strconv" - "github.com/devstream-io/devstream/pkg/util/github" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/onsi/gomega/ghttp" + + "github.com/devstream-io/devstream/pkg/util/github" ) var _ = Describe("Workflow", func() { diff --git a/pkg/util/helm/helm.go b/pkg/util/helm/helm.go index 27f3b38c5..a37ad8d04 100644 --- a/pkg/util/helm/helm.go +++ b/pkg/util/helm/helm.go @@ -43,7 +43,7 @@ func NewHelm(param *HelmParam, option ...Option) (*Helm, error) { PassCredentialsAll: false, } atomic := true - if !param.Chart.Wait { + if !*param.Chart.Wait { atomic = false } tmout, err := time.ParseDuration(param.Chart.Timeout) @@ -59,14 +59,14 @@ func NewHelm(param *HelmParam, option ...Option) (*Helm, error) { CreateNamespace: false, DisableHooks: false, Replace: true, - Wait: param.Chart.Wait, + Wait: *param.Chart.Wait, DependencyUpdate: false, Timeout: tmout, GenerateName: false, NameTemplate: "", Atomic: atomic, SkipCRDs: false, - UpgradeCRDs: param.Chart.UpgradeCRDs, + UpgradeCRDs: *param.Chart.UpgradeCRDs, SubNotes: false, Force: false, ResetValues: false, @@ -134,3 +134,8 @@ func (h *Helm) UninstallHelmChartRelease() (err error) { func GetAnnotationName() string { return "meta.helm.sh/release-name" } + +// GetAnnotationName will return label key for service created by helm +func GetLabelName() string { + return "app.kubernetes.io/instance" +} diff --git a/pkg/util/helm/helm_operation_test.go b/pkg/util/helm/helm_operation_test.go index eb3819bef..6c0e2e550 100644 --- a/pkg/util/helm/helm_operation_test.go +++ b/pkg/util/helm/helm_operation_test.go @@ -10,7 +10,7 @@ import ( func TestInstallOrUpgradeChart(t *testing.T) { atomic := true - if !helmParam.Chart.Wait { + if !*helmParam.Chart.Wait { atomic = false } tmout, err := time.ParseDuration(helmParam.Chart.Timeout) @@ -26,14 +26,14 @@ func TestInstallOrUpgradeChart(t *testing.T) { CreateNamespace: false, DisableHooks: false, Replace: true, - Wait: helmParam.Chart.Wait, + Wait: *helmParam.Chart.Wait, DependencyUpdate: false, Timeout: tmout, GenerateName: false, NameTemplate: "", Atomic: atomic, SkipCRDs: false, - UpgradeCRDs: helmParam.Chart.UpgradeCRDs, + UpgradeCRDs: *helmParam.Chart.UpgradeCRDs, SubNotes: false, Force: false, ResetValues: false, @@ -77,7 +77,7 @@ func TestAddOrUpdateChartRepo(t *testing.T) { PassCredentialsAll: false, } atomic := true - if !helmParam.Chart.Wait { + if !*helmParam.Chart.Wait { atomic = false } tmout, err := time.ParseDuration(helmParam.Chart.Timeout) @@ -93,14 +93,14 @@ func TestAddOrUpdateChartRepo(t *testing.T) { CreateNamespace: false, DisableHooks: false, Replace: true, - Wait: helmParam.Chart.Wait, + Wait: *helmParam.Chart.Wait, DependencyUpdate: false, Timeout: tmout, GenerateName: false, NameTemplate: "", Atomic: atomic, SkipCRDs: false, - UpgradeCRDs: helmParam.Chart.UpgradeCRDs, + UpgradeCRDs: *helmParam.Chart.UpgradeCRDs, SubNotes: false, Force: false, ResetValues: false, @@ -124,7 +124,7 @@ func TestAddOrUpdateChartRepo(t *testing.T) { func TestHelm_UninstallHelmChartRelease(t *testing.T) { atomic := true - if !helmParam.Chart.Wait { + if !*helmParam.Chart.Wait { atomic = false } tmout, err := time.ParseDuration(helmParam.Chart.Timeout) @@ -140,14 +140,14 @@ func TestHelm_UninstallHelmChartRelease(t *testing.T) { CreateNamespace: false, DisableHooks: false, Replace: true, - Wait: helmParam.Chart.Wait, + Wait: *helmParam.Chart.Wait, DependencyUpdate: false, Timeout: tmout, GenerateName: false, NameTemplate: "", Atomic: atomic, SkipCRDs: false, - UpgradeCRDs: helmParam.Chart.UpgradeCRDs, + UpgradeCRDs: *helmParam.Chart.UpgradeCRDs, SubNotes: false, Force: false, ResetValues: false, diff --git a/pkg/util/helm/helm_test.go b/pkg/util/helm/helm_test.go index 1d7076e22..e353b202f 100644 --- a/pkg/util/helm/helm_test.go +++ b/pkg/util/helm/helm_test.go @@ -10,6 +10,8 @@ import ( helmclient "github.com/mittwald/go-helm-client" "helm.sh/helm/v3/pkg/release" "helm.sh/helm/v3/pkg/repo" + + "github.com/devstream-io/devstream/pkg/util/types" ) var ( @@ -26,7 +28,8 @@ var helmParam = &HelmParam{ Chart{ ReleaseName: "helm:v1.0.0", Timeout: "1m", - Wait: true, + Wait: types.Bool(false), + UpgradeCRDs: types.Bool(false), }, } @@ -101,7 +104,7 @@ func TestNewHelmWithOption(t *testing.T) { PassCredentialsAll: false, } atomic := true - if !helmParam.Chart.Wait { + if !*helmParam.Chart.Wait { atomic = false } tmout, err := time.ParseDuration(helmParam.Chart.Timeout) @@ -117,14 +120,14 @@ func TestNewHelmWithOption(t *testing.T) { CreateNamespace: false, DisableHooks: false, Replace: true, - Wait: helmParam.Chart.Wait, + Wait: *helmParam.Chart.Wait, DependencyUpdate: false, Timeout: tmout, GenerateName: false, NameTemplate: "", Atomic: atomic, SkipCRDs: false, - UpgradeCRDs: helmParam.Chart.UpgradeCRDs, + UpgradeCRDs: *helmParam.Chart.UpgradeCRDs, SubNotes: false, Force: false, ResetValues: false, diff --git a/pkg/util/helm/param.go b/pkg/util/helm/param.go index da4b70b9a..e4b582814 100644 --- a/pkg/util/helm/param.go +++ b/pkg/util/helm/param.go @@ -1,5 +1,9 @@ package helm +import ( + "github.com/devstream-io/devstream/pkg/util/types" +) + // HelmParam is the struct for parameters with helm style. type HelmParam struct { Repo Repo @@ -9,22 +13,55 @@ type HelmParam struct { // Repo is the struct containing details of a git repository. // TODO(daniel-hutao): make the Repo equals to repo.Entry type Repo struct { - Name string `validate:"required"` - URL string `validate:"required"` + Name string `validate:"required" mapstructure:"name"` + URL string `validate:"required" mapstructure:"url"` } // Chart is the struct containing details of a helm chart. // TODO(daniel-hutao): make the Chart equals to helmclient.ChartSpec type Chart struct { ChartName string `validate:"required" mapstructure:"chart_name"` - Version string + Version string `mapstructure:"version"` ReleaseName string `mapstructure:"release_name"` - Namespace string - CreateNamespace bool `mapstructure:"create_namespace"` - Wait bool - Timeout string // such as "1.5h" or "2h45m", valid time units are "s", "m", "h" - UpgradeCRDs bool `mapstructure:"upgradeCRDs"` + Namespace string `mapstructure:"namespace"` + CreateNamespace *bool `mapstructure:"create_namespace"` + Wait *bool `mapstructure:"wait"` + Timeout string `mapstructure:"timeout"` // such as "1.5h" or "2h45m", valid time units are "s", "m", "h" + UpgradeCRDs *bool `mapstructure:"upgradeCRDs"` // ValuesYaml is the values.yaml content. // use string instead of map[string]interface{} ValuesYaml string `mapstructure:"values_yaml"` } + +func (repo *Repo) FillDefaultValue(defaultRepo *Repo) { + if repo.Name == "" { + repo.Name = defaultRepo.Name + } + if repo.URL == "" { + repo.URL = defaultRepo.URL + } +} + +func (chart *Chart) FillDefaultValue(defaultChart *Chart) { + if chart.ChartName == "" { + chart.ChartName = defaultChart.ChartName + } + if chart.Timeout == "" { + chart.Timeout = defaultChart.Timeout + } + chart.UpgradeCRDs = getBoolValue(chart.UpgradeCRDs, defaultChart.UpgradeCRDs) + chart.Wait = getBoolValue(chart.Wait, defaultChart.Wait) + chart.CreateNamespace = getBoolValue(chart.CreateNamespace, defaultChart.CreateNamespace) +} + +func getBoolValue(field, defaultField *bool) *bool { + if field != nil { + return field + } + + if defaultField != nil { + return defaultField + } + + return types.Bool(false) +} diff --git a/pkg/util/helm/validation.go b/pkg/util/helm/validation.go index 77bdb535b..7da6356dc 100644 --- a/pkg/util/helm/validation.go +++ b/pkg/util/helm/validation.go @@ -2,19 +2,6 @@ package helm import "github.com/devstream-io/devstream/pkg/util/validator" -// defaults set the default value with HelmParam. -func defaults(param *HelmParam) { - if param.Chart.Timeout == "" { - // Make the timeout be same as the default value for `--timeout` with `helm install/upgrade/rollback` - param.Chart.Timeout = "5m0s" - } -} - -func validate(param *HelmParam) []error { +func Validate(param *HelmParam) []error { return validator.Struct(param) } - -func DefaultsAndValidate(param *HelmParam) []error { - defaults(param) - return validate(param) -} diff --git a/pkg/util/helm/validation_test.go b/pkg/util/helm/validation_test.go index 0a7bb8b5a..61299c405 100644 --- a/pkg/util/helm/validation_test.go +++ b/pkg/util/helm/validation_test.go @@ -1,50 +1,9 @@ package helm import ( - "reflect" "testing" ) -func Test_defaults(t *testing.T) { - // base - // timeout := 1h - tests := []struct { - name string - got HelmParam - want HelmParam - }{ - // TODO: Add test cases. - {"base", - HelmParam{Repo{"test", ""}, - Chart{ - Timeout: "", - }}, - HelmParam{ - Repo{"test", ""}, - Chart{ - Timeout: "5m0s", - }}}, - {"case timeout := 1h", - HelmParam{Repo{"test", ""}, - Chart{ - Timeout: "1h", - }}, - HelmParam{ - Repo{"test", ""}, - Chart{ - Timeout: "1h", - }}}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - defaults(&tt.got) - if !reflect.DeepEqual(tt.got, tt.want) { - t.Errorf("validate() = %v, want %v", tt.got, tt.want) - } - }) - } -} - func Test_validate(t *testing.T) { type args struct { param *HelmParam @@ -66,45 +25,10 @@ func Test_validate(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := validate(tt.args.param); len(got) != tt.want { + if got := Validate(tt.args.param); len(got) != tt.want { t.Logf("got errors' length: %d\n", len(got)) t.Errorf("validate() = %v, want %v", got, tt.want) } }) } } - -func TestDefaultsAndValidate(t *testing.T) { - type args struct { - param *HelmParam - } - type want struct { - HelmParam - errCount int - } - tests := []struct { - name string - args args - want want - }{ - // TODO: Add test cases. - {"base", args{&HelmParam{ - Repo{Name: "argo", URL: "https://argoproj.github.io/argo-helm"}, - Chart{ChartName: "argo/argo-cd"}, - }}, want{HelmParam{ - Repo{Name: "argo", URL: "https://argoproj.github.io/argo-helm"}, - Chart{ChartName: "argo/argo-cd", Timeout: "5m0s"}, - }, 0}}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := DefaultsAndValidate(tt.args.param) - if len(got) != tt.want.errCount { - t.Errorf("DefaultsAndValidate(): errorCount = %v, want %v", len(got), tt.want.errCount) - } - if !reflect.DeepEqual(*tt.args.param, tt.want.HelmParam) { - t.Errorf("DefaultsAndValidate(): HelmParam= %v, want %v", *tt.args.param, tt.want.HelmParam) - } - }) - } -} diff --git a/pkg/util/k8s/state.go b/pkg/util/k8s/state.go index 82c1f6d73..bfe1000fb 100644 --- a/pkg/util/k8s/state.go +++ b/pkg/util/k8s/state.go @@ -16,10 +16,10 @@ type AllResourceStatus struct { } //GetResourceStatus get all resource state by input nameSpace and filtermap -func (c *Client) GetResourceStatus(nameSpace string, anFilter map[string]string) (AllResourceStatus, error) { +func (c *Client) GetResourceStatus(nameSpace string, anFilter, labelFilter map[string]string) (AllResourceStatus, error) { stateMap := AllResourceStatus{} // 1. list deploy resource - dps, err := c.ListDeployments(nameSpace) + dps, err := c.ListDeploymentsWithLabel(nameSpace, labelFilter) if err != nil { log.Debugf("Failed to list deployments: %s.", err) return stateMap, err @@ -28,7 +28,8 @@ func (c *Client) GetResourceStatus(nameSpace string, anFilter map[string]string) for _, dp := range dps { matchFilterd := filterByAnnotation(dp.GetAnnotations(), anFilter) if !matchFilterd { - log.Debugf("Found unknown deployment: %s.", dp.GetName()) + log.Infof("Found unknown statefulSet: %s.", dp.GetName()) + continue } dpName := dp.GetName() ready := c.IsDeploymentReady(&dp) @@ -37,7 +38,7 @@ func (c *Client) GetResourceStatus(nameSpace string, anFilter map[string]string) } // 2. list statefulsets resource - sts, err := c.ListStatefulsets(nameSpace) + sts, err := c.ListStatefulsetsWithLabel(nameSpace, labelFilter) if err != nil { log.Debugf("Failed to list statefulsets: %s.", err) return stateMap, err @@ -47,6 +48,7 @@ func (c *Client) GetResourceStatus(nameSpace string, anFilter map[string]string) matchFilterd := filterByAnnotation(ss.GetAnnotations(), anFilter) if !matchFilterd { log.Infof("Found unknown statefulSet: %s.", ss.GetName()) + continue } ready := c.IsStatefulsetReady(&ss) @@ -56,7 +58,7 @@ func (c *Client) GetResourceStatus(nameSpace string, anFilter map[string]string) } // 3. list daemonset resource - dss, err := c.ListDaemonsets(nameSpace) + dss, err := c.ListDaemonsetsWithLabel(nameSpace, labelFilter) if err != nil { log.Debugf("Failed to list daemonsets: %s.", err) return stateMap, err @@ -64,8 +66,9 @@ func (c *Client) GetResourceStatus(nameSpace string, anFilter map[string]string) for _, ds := range dss { matchFilterd := filterByAnnotation(ds.GetAnnotations(), anFilter) - if matchFilterd { - log.Infof("Found unknown daemonSet: %s.", ds.GetName()) + if !matchFilterd { + log.Infof("Found unknown statefulSet: %s.", ds.GetName()) + continue } ready := c.IsDaemonsetReady(&ds) diff --git a/pkg/util/k8s/workload.go b/pkg/util/k8s/workload.go index 8d7bd2f51..594801217 100644 --- a/pkg/util/k8s/workload.go +++ b/pkg/util/k8s/workload.go @@ -7,18 +7,18 @@ import ( appsv1 "k8s.io/api/apps/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" "github.com/devstream-io/devstream/pkg/util/log" ) -func (c *Client) ListDeployments(namespace string) ([]appsv1.Deployment, error) { - dpList, err := c.AppsV1().Deployments(namespace).List(context.TODO(), metav1.ListOptions{}) +func (c *Client) ListDeploymentsWithLabel(namespace string, labelFilter map[string]string) ([]appsv1.Deployment, error) { + dpList, err := c.AppsV1().Deployments(namespace).List(context.TODO(), c.generateLabelFilterOption(labelFilter)) if err != nil { return nil, err } return dpList.Items, nil } - func (c *Client) GetDeployment(namespace, name string) (*appsv1.Deployment, error) { return c.AppsV1().Deployments(namespace).Get(context.TODO(), name, metav1.GetOptions{}) } @@ -57,8 +57,8 @@ func (c *Client) DeleteDeployment(namespace, deployName string) error { Delete(context.TODO(), deployName, metav1.DeleteOptions{}) } -func (c *Client) ListDaemonsets(namespace string) ([]appsv1.DaemonSet, error) { - dsList, err := c.AppsV1().DaemonSets(namespace).List(context.TODO(), metav1.ListOptions{}) +func (c *Client) ListDaemonsetsWithLabel(namespace string, labeFilter map[string]string) ([]appsv1.DaemonSet, error) { + dsList, err := c.AppsV1().DaemonSets(namespace).List(context.TODO(), c.generateLabelFilterOption(labeFilter)) if err != nil { return nil, err } @@ -73,8 +73,8 @@ func (c *Client) IsDaemonsetReady(daemonset *appsv1.DaemonSet) bool { return daemonset.Status.NumberReady == daemonset.Status.DesiredNumberScheduled } -func (c *Client) ListStatefulsets(namespace string) ([]appsv1.StatefulSet, error) { - ssList, err := c.AppsV1().StatefulSets(namespace).List(context.TODO(), metav1.ListOptions{}) +func (c *Client) ListStatefulsetsWithLabel(namespace string, labelFilter map[string]string) ([]appsv1.StatefulSet, error) { + ssList, err := c.AppsV1().StatefulSets(namespace).List(context.TODO(), c.generateLabelFilterOption(labelFilter)) if err != nil { return nil, err } @@ -88,3 +88,11 @@ func (c *Client) GetStatefulset(namespace, name string) (*appsv1.StatefulSet, er func (c *Client) IsStatefulsetReady(statefulset *appsv1.StatefulSet) bool { return statefulset.Status.ReadyReplicas == *statefulset.Spec.Replicas } + +func (c *Client) generateLabelFilterOption(labelFilter map[string]string) metav1.ListOptions { + labelSelector := metav1.LabelSelector{MatchLabels: labelFilter} + options := metav1.ListOptions{ + LabelSelector: labels.Set(labelSelector.MatchLabels).String(), + } + return options +} diff --git a/pkg/util/template/render.go b/pkg/util/template/render.go index 9dc3e7853..71aacc93a 100644 --- a/pkg/util/template/render.go +++ b/pkg/util/template/render.go @@ -9,7 +9,7 @@ import ( ) func Render(name, templateStr string, variable any) (string, error) { - t, err := template.New(name).Delims("[[", "]]").Parse(templateStr) + t, err := template.New(name).Option("missingkey=error").Delims("[[", "]]").Parse(templateStr) if err != nil { log.Debugf("Template parse file failed: %s.", err) return "", err diff --git a/pkg/util/types/bool.go b/pkg/util/types/bool.go new file mode 100644 index 000000000..027674b87 --- /dev/null +++ b/pkg/util/types/bool.go @@ -0,0 +1,3 @@ +package types + +func Bool(v bool) *bool { return &v }