feat: object storage cluster init. (#4510)

* feat: object storage cluster init. Signed-off-by: yy <lingdie.yy@outlook.com> Signed-off-by: yy <lingdie.yy@outlook.com> * chore: add more manifests for minio and prometheus Signed-off-by: yy <lingdie.yy@outlook.com> Signed-off-by: yy <lingdie.yy@outlook.com> * refine cluster image * refine init.sh * tmp images * tmp images * fix * fix env * rm '' * fix * test * add env * add env to Kubefile * use sed to replace env * fix path * fix * fix sed * fix policy path * add env * fix mc admin remove user * add app cr * fix prometheus deploy.yaml error * fix 1 * delete app cr and update images * add ns * fix * change admin username to admin * refine console ingress * run tars/xxx.tar --------- Signed-off-by: yy <lingdie.yy@outlook.com> Co-authored-by: xuziyi <nowinkey@tom.com>
labring · Feb 26, 2024 · 649c535 · 649c535
1 parent 4488fb6
commit 649c535
Show file tree

Hide file tree

Showing 13 changed files with 664 additions and 0 deletions.
diff --git a/.github/workflows/objectstorage.yaml b/.github/workflows/objectstorage.yaml
@@ -0,0 +1,147 @@
+name: Build Object Storage Cluster image
+
+on:
+  workflow_call:
+    inputs:
+      push_image:
+        description: 'Push image'
+        required: false
+        type: boolean
+        default: false
+      push_image_tag:
+        description: 'Push all-in-one image tag, default is latest'
+        default: 'latest'
+        required: false
+        type: string
+      build_from:
+        description: 'Build all-in-one image from components image tag, default is latest'
+        default: 'latest'
+        required: false
+        type: string
+  workflow_dispatch:
+    inputs:
+      push_image:
+        description: 'Push image'
+        required: false
+        type: boolean
+        default: false
+      push_image_tag:
+        description: 'Push all-in-one image tag, default is latest'
+        default: 'latest'
+        required: false
+        type: string
+      build_from:
+        description: 'Build all-in-one image from components image tag, default is latest'
+        default: 'latest'
+        required: false
+        type: string
+  push:
+    branches: [ "main" ]
+    paths:
+      - "deploy/objectstorage/**"
+      - ".github/workflows/objectstorage.yml"
+      - "!**/*.md"
+      - "!**/*.yaml"
+  pull_request:
+    branches: [ "*" ]
+    paths:
+      - "deploy/objectstorage/**"
+      - ".github/workflows/objectstorage.yml"
+      - "!**/*.md"
+      - "!**/*.yaml"
+
+env:
+  # Common versions
+  GO_VERSION: "1.20"
+  DEFAULT_OWNER: "labring"
+
+jobs:
+  save-sealos:
+    uses: ./.github/workflows/import-save-sealos.yml
+
+  build-cluster-image:
+    if: ${{ (github.event_name == 'release') ||(github.event_name == 'push')  || (inputs.push_image == true) }}
+    needs:
+      - save-sealos
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Expose git commit data
+        uses: rlespinasse/git-commit-data-action@v1
+      - name: Check if tag
+        id: check_tag
+        run: |
+          if [[ "${{ github.ref }}" == refs/tags/* ]]; then
+            echo "isTag=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "isTag=false" >> "$GITHUB_OUTPUT"
+          fi
+      - name: Prepare
+        id: prepare
+        run: |
+          bash ./scripts/resolve-tag-image.sh "${{ inputs.push_image }}" "${{ steps.check_tag.outputs.isTag }}"  "${{ inputs.push_image_tag }}"     
+          echo repo=ghcr.io/${{ github.repository_owner }}/sealos-cloud-objectstorage >> $GITHUB_OUTPUT
+      - name: Download sealos
+        uses: actions/download-artifact@v3
+        with:
+          name: sealos
+          path: /tmp/
+      - name: Verify sealos
+        run: |
+          sudo chmod a+x /tmp/sealos
+          sudo mv /tmp/sealos /usr/bin/sealos
+          sudo sealos version
+
+      # todo: mutate image tag in images/shim and scripts or change scripts to use changeable tags
+
+      - name: Sealos login to ghcr.io
+        # if push to master, then login to ghcr.io
+        run: |
+          sudo sealos login -u ${{ github.repository_owner }} -p ${{ secrets.GH_PAT }} --debug ghcr.io
+
+      - name: Build sealos cloud cluster image
+        working-directory: deploy/objectstorage
+        run: |
+          [ -z "${{ inputs.build_from }}" ] && BuildFromTag="latest" || BuildFromTag="${{ inputs.build_from }}"; echo "BuildFromTag=${BuildFromTag}" 
+          sed -i "s#labring#${{ github.repository_owner }}#g" init.sh
+          sed -i "s#latest#${BuildFromTag}#g" init.sh
+          
+          sudo bash init.sh amd64 
+          sudo sealos build -t ${{ steps.prepare.outputs.repo }}:${{ steps.prepare.outputs.tag_name }}-amd64 --platform linux/amd64 -f Kubefile
+          sudo sealos build -t ${{ steps.prepare.outputs.repo }}:latest-amd64 --platform linux/amd64 -f Kubefile
+          
+          
+          # delete old registry cache
+          sudo rm -rf registry
+          sudo rm -rf tars
+          
+          sudo bash init.sh arm64 
+          sudo sealos build -t ${{ steps.prepare.outputs.repo }}:${{ steps.prepare.outputs.tag_name }}-arm64 --platform linux/arm64 -f Kubefile
+          sudo sealos build -t ${{ steps.prepare.outputs.repo }}:latest-arm64 --platform linux/arm64 -f Kubefile
+
+      - name: Manifest Cluster Images
+        # if push to master, then patch images to ghcr.io
+        run: |
+          sudo sealos images
+          bash docker/patch/manifest-cluster-images.sh ${{ steps.prepare.outputs.repo }}:${{ steps.prepare.outputs.tag_name }}
+          bash docker/patch/manifest-cluster-images.sh ${{ steps.prepare.outputs.repo }}:latest
+        env:
+          OWNER: ${{ github.repository_owner }}
+
+      - name: Renew issue and Sync Images
+        uses: labring/gh-rebot@v0.0.6
+        if: ${{ github.repository_owner == env.DEFAULT_OWNER }}
+        with:
+          version: v0.0.8-rc1
+        env:
+          GH_TOKEN: "${{ secrets.GH_PAT }}"
+          SEALOS_TYPE: "issue_renew"
+          SEALOS_ISSUE_TITLE: "[DaylyReport] Auto build for sealos"
+          SEALOS_ISSUE_BODYFILE: "scripts/ISSUE_RENEW.md"
+          SEALOS_ISSUE_LABEL: "dayly-report"
+          SEALOS_ISSUE_TYPE: "day"
+          SEALOS_ISSUE_REPO: "labring-actions/cluster-image"
+          SEALOS_COMMENT_BODY: "/imagesync ghcr.io/${{ github.repository_owner }}/sealos-cloud:${{ steps.prepare.outputs.tag_name }}"
diff --git a/deploy/objectstorage/Kubefile b/deploy/objectstorage/Kubefile
@@ -0,0 +1,14 @@
+FROM scratch
+COPY tars tars
+COPY etc etc
+COPY scripts scripts
+COPY manifests manifests
+
+ENV cloudDomain=${cloudDomain:-"127.0.0.1.nip.io"}
+ENV cloudPort=""
+ENV minioStorageSize=${minioStorageSize:-1Gi}
+ENV promStorageSize=${promStorageSize:-1Gi}
+ENV minioAdminUser=${minioAdminUser:-"admin"}
+ENV minioAdminPassword=${minioAdminPassword:-"passw0rd"}
+
+CMD ["bash scripts/init.sh"]
diff --git a/deploy/objectstorage/README.md b/deploy/objectstorage/README.md
@@ -0,0 +1,6 @@
+# sealos cloud object storage cluster image
+## prepare
+
+1. install minio operator
+2. install prometheus operator
+3. run object storage cluster image
diff --git a/deploy/objectstorage/etc/minio/policy/kubeblocks.json b/deploy/objectstorage/etc/minio/policy/kubeblocks.json
@@ -0,0 +1,44 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": [
+        "s3:AbortMultipartUpload",
+        "s3:GetBucketTagging",
+        "s3:ListBucket",
+        "s3:ListMultipartUploadParts",
+        "s3:PutBucketTagging",
+        "s3:PutObject",
+        "s3:CreateBucket",
+        "s3:DeleteObject",
+        "s3:GetBucketLocation",
+        "s3:GetBucketPolicy",
+        "s3:GetObject"
+      ],
+      "Resource": [
+        "arn:aws:s3:::file-migration/*"
+      ]
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "s3:AbortMultipartUpload",
+        "s3:CreateBucket",
+        "s3:DeleteObject",
+        "s3:GetBucketLocation",
+        "s3:PutObject",
+        "s3:GetBucketPolicy",
+        "s3:GetBucketTagging",
+        "s3:GetObject",
+        "s3:ListBucket",
+        "s3:ListBucketMultipartUploads",
+        "s3:ListMultipartUploadParts",
+        "s3:PutBucketTagging"
+      ],
+      "Resource": [
+        "arn:aws:s3:::file-backup/*"
+      ]
+    }
+  ]
+}
diff --git a/deploy/objectstorage/etc/minio/policy/user_deny_write.json b/deploy/objectstorage/etc/minio/policy/user_deny_write.json
@@ -0,0 +1,14 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Deny",
+      "Action": [
+        "s3:PutObject"
+      ],
+      "Resource": [
+        "arn:aws:s3:::${aws:username}-*"
+      ]
+    }
+  ]
+}
diff --git a/deploy/objectstorage/etc/minio/policy/user_normal.json b/deploy/objectstorage/etc/minio/policy/user_normal.json
@@ -0,0 +1,23 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": [
+        "s3:ListBucket",
+        "s3:ListBucketMultipartUploads",
+        "s3:ListMultipartUploadParts",
+        "s3:GetBucketPolicy",
+        "s3:GetBucketLocation",
+        "s3:GetBucketTagging",
+        "s3:PutBucketTagging",
+        "s3:GetObject",
+        "s3:PutObject",
+        "s3:DeleteObject"
+      ],
+      "Resource": [
+        "arn:aws:s3:::${aws:username}-*"
+      ]
+    }
+  ]
+}
diff --git a/deploy/objectstorage/images/shim/imageList b/deploy/objectstorage/images/shim/imageList
@@ -0,0 +1,2 @@
+quay.io/prometheus/prometheus:v2.45.0
+minio/minio:RELEASE.2023-11-11T08-14-41Z
diff --git a/deploy/objectstorage/init.sh b/deploy/objectstorage/init.sh
@@ -0,0 +1,33 @@
+#!/bin/bash
+set -e
+export readonly ARCH=${1:-amd64}
+mkdir -p tars
+
+RetryPullImageInterval=3
+RetrySleepSeconds=3
+
+retryPullImage() {
+    local image=$1
+    local retry=0
+    local retryMax=3
+    set +e
+    while [ $retry -lt $RetryPullImageInterval ]; do
+        sealos pull --policy=always --platform=linux/"${ARCH}" $image >/dev/null && break
+        retry=$(($retry + 1))
+        echo "retry pull image $image, retry times: $retry"
+        sleep $RetrySleepSeconds
+    done
+    set -e
+    if [ $retry -eq $retryMax ]; then
+        echo "pull image $image failed"
+        exit 1
+    fi
+}
+
+retryPullImage ghcr.io/labring/sealos-cloud-objectstorage-controller:latest
+retryPullImage ghcr.io/labring/sealos-cloud-objectstorage-frontend:latest
+retryPullImage ghcr.io/labring/sealos-cloud-minio-service:latest
+
+sealos save -o tars/objectstorage-controller.tar ghcr.io/labring/sealos-cloud-objectstorage-controller:latest
+sealos save -o tars/objectstorage-frontend.tar ghcr.io/labring/sealos-cloud-objectstorage-frontend:latest
+sealos save -o tars/objectstorage-service.tar ghcr.io/labring/sealos-cloud-minio-service:latest