From 0e9de2ae737a450a5cedda636b4bf4eff5af169d Mon Sep 17 00:00:00 2001 From: yy Date: Thu, 25 Jan 2024 14:52:15 +0800 Subject: [PATCH 01/29] feat: object storage cluster init. Signed-off-by: yy Signed-off-by: yy --- .github/workflows/objectstorage.yaml | 147 ++++++++++++++++++ deploy/objectstorage/Kubefile | 13 ++ deploy/objectstorage/README.md | 6 + .../etc/minio/policy/migration.json | 23 +++ .../etc/minio/policy/user_deny_write.json | 14 ++ .../etc/minio/policy/user_normal.json | 23 +++ deploy/objectstorage/init.sh | 33 ++++ deploy/objectstorage/scripts/init.sh | 8 + 8 files changed, 267 insertions(+) create mode 100644 .github/workflows/objectstorage.yaml create mode 100644 deploy/objectstorage/Kubefile create mode 100644 deploy/objectstorage/README.md create mode 100644 deploy/objectstorage/etc/minio/policy/migration.json create mode 100644 deploy/objectstorage/etc/minio/policy/user_deny_write.json create mode 100644 deploy/objectstorage/etc/minio/policy/user_normal.json create mode 100644 deploy/objectstorage/init.sh create mode 100644 deploy/objectstorage/scripts/init.sh diff --git a/.github/workflows/objectstorage.yaml b/.github/workflows/objectstorage.yaml new file mode 100644 index 00000000000..13998423e39 --- /dev/null +++ b/.github/workflows/objectstorage.yaml @@ -0,0 +1,147 @@ +name: Build Object Storage Cluster image + +on: + workflow_call: + inputs: + push_image: + description: 'Push image' + required: false + type: boolean + default: false + push_image_tag: + description: 'Push all-in-one image tag, default is latest' + default: 'latest' + required: false + type: string + build_from: + description: 'Build all-in-one image from components image tag, default is latest' + default: 'latest' + required: false + type: string + workflow_dispatch: + inputs: + push_image: + description: 'Push image' + required: false + type: boolean + default: false + push_image_tag: + description: 'Push all-in-one image tag, default is latest' + default: 'latest' + required: false + type: string + build_from: + description: 'Build all-in-one image from components image tag, default is latest' + default: 'latest' + required: false + type: string + push: + branches: [ "main" ] + paths: + - "deploy/objectstorage/**" + - ".github/workflows/objectstorage.yml" + - "!**/*.md" + - "!**/*.yaml" + pull_request: + branches: [ "*" ] + paths: + - "deploy/objectstorage/**" + - ".github/workflows/objectstorage.yml" + - "!**/*.md" + - "!**/*.yaml" + +env: + # Common versions + GO_VERSION: "1.20" + DEFAULT_OWNER: "labring" + +jobs: + save-sealos: + uses: ./.github/workflows/import-save-sealos.yml + + build-cluster-image: + if: ${{ (github.event_name == 'release') ||(github.event_name == 'push') || (inputs.push_image == true) }} + needs: + - save-sealos + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Expose git commit data + uses: rlespinasse/git-commit-data-action@v1 + - name: Check if tag + id: check_tag + run: | + if [[ "${{ github.ref }}" == refs/tags/* ]]; then + echo "isTag=true" >> "$GITHUB_OUTPUT" + else + echo "isTag=false" >> "$GITHUB_OUTPUT" + fi + - name: Prepare + id: prepare + run: | + bash ./scripts/resolve-tag-image.sh "${{ inputs.push_image }}" "${{ steps.check_tag.outputs.isTag }}" "${{ inputs.push_image_tag }}" + echo repo=ghcr.io/${{ github.repository_owner }}/sealos-cloud-objectstorage >> $GITHUB_OUTPUT + - name: Download sealos + uses: actions/download-artifact@v3 + with: + name: sealos + path: /tmp/ + - name: Verify sealos + run: | + sudo chmod a+x /tmp/sealos + sudo mv /tmp/sealos /usr/bin/sealos + sudo sealos version + + # todo: mutate image tag in images/shim and scripts or change scripts to use changeable tags + + - name: Sealos login to ghcr.io + # if push to master, then login to ghcr.io + run: | + sudo sealos login -u ${{ github.repository_owner }} -p ${{ secrets.GH_PAT }} --debug ghcr.io + + - name: Build sealos cloud cluster image + working-directory: deploy/objectstorage + run: | + [ -z "${{ inputs.build_from }}" ] && BuildFromTag="latest" || BuildFromTag="${{ inputs.build_from }}"; echo "BuildFromTag=${BuildFromTag}" + sed -i "s#labring#${{ github.repository_owner }}#g" init.sh + sed -i "s#latest#${BuildFromTag}#g" init.sh + + sudo bash init.sh amd64 + sudo sealos build -t ${{ steps.prepare.outputs.repo }}:${{ steps.prepare.outputs.tag_name }}-amd64 --platform linux/amd64 -f Kubefile + sudo sealos build -t ${{ steps.prepare.outputs.repo }}:latest-amd64 --platform linux/amd64 -f Kubefile + + + # delete old registry cache + sudo rm -rf registry + sudo rm -rf tars + + sudo bash init.sh arm64 + sudo sealos build -t ${{ steps.prepare.outputs.repo }}:${{ steps.prepare.outputs.tag_name }}-arm64 --platform linux/arm64 -f Kubefile + sudo sealos build -t ${{ steps.prepare.outputs.repo }}:latest-arm64 --platform linux/arm64 -f Kubefile + + - name: Manifest Cluster Images + # if push to master, then patch images to ghcr.io + run: | + sudo sealos images + bash docker/patch/manifest-cluster-images.sh ${{ steps.prepare.outputs.repo }}:${{ steps.prepare.outputs.tag_name }} + bash docker/patch/manifest-cluster-images.sh ${{ steps.prepare.outputs.repo }}:latest + env: + OWNER: ${{ github.repository_owner }} + + - name: Renew issue and Sync Images + uses: labring/gh-rebot@v0.0.6 + if: ${{ github.repository_owner == env.DEFAULT_OWNER }} + with: + version: v0.0.8-rc1 + env: + GH_TOKEN: "${{ secrets.GH_PAT }}" + SEALOS_TYPE: "issue_renew" + SEALOS_ISSUE_TITLE: "[DaylyReport] Auto build for sealos" + SEALOS_ISSUE_BODYFILE: "scripts/ISSUE_RENEW.md" + SEALOS_ISSUE_LABEL: "dayly-report" + SEALOS_ISSUE_TYPE: "day" + SEALOS_ISSUE_REPO: "labring-actions/cluster-image" + SEALOS_COMMENT_BODY: "/imagesync ghcr.io/${{ github.repository_owner }}/sealos-cloud:${{ steps.prepare.outputs.tag_name }}" diff --git a/deploy/objectstorage/Kubefile b/deploy/objectstorage/Kubefile new file mode 100644 index 00000000000..586625ecc7b --- /dev/null +++ b/deploy/objectstorage/Kubefile @@ -0,0 +1,13 @@ +FROM scratch +COPY tars tars +COPY etc etc +COPY scripts scripts +COPY manifests manifests + +ENV cloudDomain="127.0.0.1.nip.io" +ENV cloudPort="" +ENV storageSize=${storageSize:-1Gi} +ENV minioAdminUser=${minioAdminUser:-"admin"} +ENV minioAdminPassword=${minioAdminPassword:-"passw0rd"} + +CMD ["bash scripts/init.sh"] diff --git a/deploy/objectstorage/README.md b/deploy/objectstorage/README.md new file mode 100644 index 00000000000..a4a7eb4e12e --- /dev/null +++ b/deploy/objectstorage/README.md @@ -0,0 +1,6 @@ +# sealos cloud object storage cluster image +## prepare + +1. install minio operator +2. install prometheus operator +3. run object storage cluster image \ No newline at end of file diff --git a/deploy/objectstorage/etc/minio/policy/migration.json b/deploy/objectstorage/etc/minio/policy/migration.json new file mode 100644 index 00000000000..ce87ae79aa1 --- /dev/null +++ b/deploy/objectstorage/etc/minio/policy/migration.json @@ -0,0 +1,23 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:ListBucket", + "s3:ListBucketMultipartUploads", + "s3:ListMultipartUploadParts", + "s3:GetBucketPolicy", + "s3:GetBucketLocation", + "s3:GetBucketTagging", + "s3:PutBucketTagging", + "s3:GetObject", + "s3:PutObject", + "s3:DeleteObject" + ], + "Resource": [ + "arn:aws:s3:::file-migration/*" + ] + } + ] +} \ No newline at end of file diff --git a/deploy/objectstorage/etc/minio/policy/user_deny_write.json b/deploy/objectstorage/etc/minio/policy/user_deny_write.json new file mode 100644 index 00000000000..07d0eb87c18 --- /dev/null +++ b/deploy/objectstorage/etc/minio/policy/user_deny_write.json @@ -0,0 +1,14 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Deny", + "Action": [ + "s3:PutObject" + ], + "Resource": [ + "arn:aws:s3:::${aws:username}-*" + ] + } + ] +} \ No newline at end of file diff --git a/deploy/objectstorage/etc/minio/policy/user_normal.json b/deploy/objectstorage/etc/minio/policy/user_normal.json new file mode 100644 index 00000000000..35c98adf9c8 --- /dev/null +++ b/deploy/objectstorage/etc/minio/policy/user_normal.json @@ -0,0 +1,23 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:ListBucket", + "s3:ListBucketMultipartUploads", + "s3:ListMultipartUploadParts", + "s3:GetBucketPolicy", + "s3:GetBucketLocation", + "s3:GetBucketTagging", + "s3:PutBucketTagging", + "s3:GetObject", + "s3:PutObject", + "s3:DeleteObject" + ], + "Resource": [ + "arn:aws:s3:::${aws:username}-*" + ] + } + ] +} \ No newline at end of file diff --git a/deploy/objectstorage/init.sh b/deploy/objectstorage/init.sh new file mode 100644 index 00000000000..d0b27b91c54 --- /dev/null +++ b/deploy/objectstorage/init.sh @@ -0,0 +1,33 @@ +#!/bin/bash +set -e +export readonly ARCH=${1:-amd64} +mkdir -p tars + +RetryPullImageInterval=3 +RetrySleepSeconds=3 + +retryPullImage() { + local image=$1 + local retry=0 + local retryMax=3 + set +e + while [ $retry -lt $RetryPullImageInterval ]; do + sealos pull --policy=always --platform=linux/"${ARCH}" $image >/dev/null && break + retry=$(($retry + 1)) + echo "retry pull image $image, retry times: $retry" + sleep $RetrySleepSeconds + done + set -e + if [ $retry -eq $retryMax ]; then + echo "pull image $image failed" + exit 1 + fi +} + +retryPullImage ghcr.io/labring/sealos-cloud-objectstorage-controller:latest +retryPullImage ghcr.io/labring/sealos-cloud-objectstorage-frontend:latest +retryPullImage ghcr.io/labring/sealos-cloud-minio-service:latest + +sealos save -o tars/objectstorage-controller.tar ghcr.io/labring/sealos-cloud-objectstorage-controller:latest +sealos save -o tars/objectstorage-frontend.tar ghcr.io/labring/sealos-cloud-objectstorage-frontend:latest +sealos save -o tars/objectstorage-service.tar ghcr.io/labring/sealos-cloud-minio-service:latest diff --git a/deploy/objectstorage/scripts/init.sh b/deploy/objectstorage/scripts/init.sh new file mode 100644 index 00000000000..9d484c8c70e --- /dev/null +++ b/deploy/objectstorage/scripts/init.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +# TODO +# 1. create minio instance +# 2. create prometheus instance +# 3. run objectstorage controller +# 4. run objectstorage frontend +# 5. run objectstorage monitor service From 697e19b1b2a272048a16c8e567aaeb609f479fff Mon Sep 17 00:00:00 2001 From: yy Date: Thu, 25 Jan 2024 15:02:09 +0800 Subject: [PATCH 02/29] chore: add more manifests for minio and prometheus Signed-off-by: yy Signed-off-by: yy --- deploy/objectstorage/images/shim/imageList | 2 + .../manifests/minio/deploy.yaml.tmpl | 44 +++++++ .../manifests/prometheus/deploy.yaml.tmpl | 114 ++++++++++++++++++ deploy/objectstorage/scripts/minio.sh | 1 + deploy/objectstorage/scripts/prometheus.sh | 1 + 5 files changed, 162 insertions(+) create mode 100644 deploy/objectstorage/images/shim/imageList create mode 100644 deploy/objectstorage/manifests/minio/deploy.yaml.tmpl create mode 100644 deploy/objectstorage/manifests/prometheus/deploy.yaml.tmpl create mode 100644 deploy/objectstorage/scripts/minio.sh create mode 100644 deploy/objectstorage/scripts/prometheus.sh diff --git a/deploy/objectstorage/images/shim/imageList b/deploy/objectstorage/images/shim/imageList new file mode 100644 index 00000000000..225166f8b80 --- /dev/null +++ b/deploy/objectstorage/images/shim/imageList @@ -0,0 +1,2 @@ +quay.io/prometheus/prometheus:v2.45.0 +# TODO ADD MINIO DOCKER IMAGE!! \ No newline at end of file diff --git a/deploy/objectstorage/manifests/minio/deploy.yaml.tmpl b/deploy/objectstorage/manifests/minio/deploy.yaml.tmpl new file mode 100644 index 00000000000..862c3a113a3 --- /dev/null +++ b/deploy/objectstorage/manifests/minio/deploy.yaml.tmpl @@ -0,0 +1,44 @@ +apiVersion: minio.min.io/v2 +kind: Tenant +metadata: + name: object-storage + namespace: ${BACKEND_NAMESPACE} +spec: + configuration: + name: object-storage-env-configuration + credsSecret: + name: object-storage-secret + exposeServices: + console: true + minio: true + features: {} + image: minio/minio:RELEASE.2023-11-11T08-14-41Z + imagePullSecret: {} + mountPath: /export + pools: + - name: pool-0 + resources: + limits: + cpu: 1000m + memory: 2Gi + requests: + cpu: 100m + memory: 256Mi + runtimeClassName: '' + servers: 4 + volumeClaimTemplate: + metadata: + name: data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: ${STORAGE_SIZE}Gi + status: {} + volumesPerServer: 1 + requestAutoCert: false + users: + - name: object-storage-user-0 +scheduler: + name: '' \ No newline at end of file diff --git a/deploy/objectstorage/manifests/prometheus/deploy.yaml.tmpl b/deploy/objectstorage/manifests/prometheus/deploy.yaml.tmpl new file mode 100644 index 00000000000..e1f2a7cf9b8 --- /dev/null +++ b/deploy/objectstorage/manifests/prometheus/deploy.yaml.tmpl @@ -0,0 +1,114 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: object-storage-sa + namespace: ${BACKEND_NAMESPACE} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: object-storage-role + namespace: ${BACKEND_NAMESPACE} +rules: + - verbs: + - get + - list + - watch + apiGroups: + - '' + resources: + - secrets + - verbs: + - create + - delete + - get + apiGroups: + - '' + resources: + - services + - verbs: + - get + - list + - watch + apiGroups: + - minio.min.io + resources: + - tenants +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: object-storage-rolebind + namespace: ${BACKEND_NAMESPACE} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: object-storage-role +subjects: + - kind: ServiceAccount + name: object-storage-sa + namespace: ${BACKEND_NAMESPACE} +--- +apiVersion: monitoring.coreos.com/v1 +kind: Prometheus +metadata: + labels: + app: prometheus-object-storage + name: object-storage + namespace: ${BACKEND_NAMESPACE} +spec: + podMetadata: + labels: + app: prometheus-object-storage + resources: + limits: + cpu: 200m + memory: 256Mi + requests: + cpu: 50m + memory: 128Mi + securityContext: + fsGroup: 2000 + runAsGroup: 2000 + runAsNonRoot: true + runAsUser: 1000 + seccompProfile: + type: RuntimeDefault + evaluationInterval: 60s + image: quay.io/prometheus/prometheus:v2.45.0 + serviceMonitorSelector: {} + probeSelector: {} + ruleSelector: {} + portName: http-web + retention: 10d + scrapeInterval: 60s + serviceAccountName: object-storage-sa + replicas: 1 + shards: 1 + storage: + volumeClaimTemplate: + metadata: + annotations: + path: /prometheus + value: ${STORAGE_SIZE}Gi + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: ${STORAGE_SIZE}Gi +--- +apiVersion: v1 +kind: Service +metadata: + name: prometheus-object-storage + namespace: ${BACKEND_NAMESPACE} +spec: + ports: + - port: 9090 + targetPort: 9090 + protocol: TCP + name: http-web + selector: + app: prometheus-object-storage + type: ClusterIP \ No newline at end of file diff --git a/deploy/objectstorage/scripts/minio.sh b/deploy/objectstorage/scripts/minio.sh new file mode 100644 index 00000000000..e273062fc43 --- /dev/null +++ b/deploy/objectstorage/scripts/minio.sh @@ -0,0 +1 @@ +# create minio instance diff --git a/deploy/objectstorage/scripts/prometheus.sh b/deploy/objectstorage/scripts/prometheus.sh new file mode 100644 index 00000000000..2ce5bd586ba --- /dev/null +++ b/deploy/objectstorage/scripts/prometheus.sh @@ -0,0 +1 @@ +# create prometheus instance From 6d3b74107a3d9ff9e9de7015cba804db35d6e140 Mon Sep 17 00:00:00 2001 From: xuziyi Date: Thu, 25 Jan 2024 17:38:20 +0800 Subject: [PATCH 03/29] refine cluster image --- deploy/objectstorage/Kubefile | 1 + .../etc/minio/policy/kubeblocks.json | 44 ++++++ .../etc/minio/policy/migration.json | 23 --- deploy/objectstorage/images/shim/imageList | 2 +- .../manifests/minio/deploy.yaml.tmpl | 148 +++++++++++++++++- .../manifests/prometheus/deploy.yaml.tmpl | 75 ++++++--- deploy/objectstorage/scripts/init.sh | 2 + deploy/objectstorage/scripts/minio.sh | 72 ++++++++- deploy/objectstorage/scripts/prometheus.sh | 14 +- 9 files changed, 332 insertions(+), 49 deletions(-) create mode 100644 deploy/objectstorage/etc/minio/policy/kubeblocks.json delete mode 100644 deploy/objectstorage/etc/minio/policy/migration.json diff --git a/deploy/objectstorage/Kubefile b/deploy/objectstorage/Kubefile index 586625ecc7b..92c2edbc656 100644 --- a/deploy/objectstorage/Kubefile +++ b/deploy/objectstorage/Kubefile @@ -9,5 +9,6 @@ ENV cloudPort="" ENV storageSize=${storageSize:-1Gi} ENV minioAdminUser=${minioAdminUser:-"admin"} ENV minioAdminPassword=${minioAdminPassword:-"passw0rd"} +ENV backendNamespace=${backendNamespace:-"objectstorage-system"} CMD ["bash scripts/init.sh"] diff --git a/deploy/objectstorage/etc/minio/policy/kubeblocks.json b/deploy/objectstorage/etc/minio/policy/kubeblocks.json new file mode 100644 index 00000000000..ad32526b1ca --- /dev/null +++ b/deploy/objectstorage/etc/minio/policy/kubeblocks.json @@ -0,0 +1,44 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:AbortMultipartUpload", + "s3:GetBucketTagging", + "s3:ListBucket", + "s3:ListMultipartUploadParts", + "s3:PutBucketTagging", + "s3:PutObject", + "s3:CreateBucket", + "s3:DeleteObject", + "s3:GetBucketLocation", + "s3:GetBucketPolicy", + "s3:GetObject" + ], + "Resource": [ + "arn:aws:s3:::file-migration/*" + ] + }, + { + "Effect": "Allow", + "Action": [ + "s3:AbortMultipartUpload", + "s3:CreateBucket", + "s3:DeleteObject", + "s3:GetBucketLocation", + "s3:PutObject", + "s3:GetBucketPolicy", + "s3:GetBucketTagging", + "s3:GetObject", + "s3:ListBucket", + "s3:ListBucketMultipartUploads", + "s3:ListMultipartUploadParts", + "s3:PutBucketTagging" + ], + "Resource": [ + "arn:aws:s3:::file-backup/*" + ] + } + ] +} \ No newline at end of file diff --git a/deploy/objectstorage/etc/minio/policy/migration.json b/deploy/objectstorage/etc/minio/policy/migration.json deleted file mode 100644 index ce87ae79aa1..00000000000 --- a/deploy/objectstorage/etc/minio/policy/migration.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": [ - "s3:ListBucket", - "s3:ListBucketMultipartUploads", - "s3:ListMultipartUploadParts", - "s3:GetBucketPolicy", - "s3:GetBucketLocation", - "s3:GetBucketTagging", - "s3:PutBucketTagging", - "s3:GetObject", - "s3:PutObject", - "s3:DeleteObject" - ], - "Resource": [ - "arn:aws:s3:::file-migration/*" - ] - } - ] -} \ No newline at end of file diff --git a/deploy/objectstorage/images/shim/imageList b/deploy/objectstorage/images/shim/imageList index 225166f8b80..f5536fe66fc 100644 --- a/deploy/objectstorage/images/shim/imageList +++ b/deploy/objectstorage/images/shim/imageList @@ -1,2 +1,2 @@ quay.io/prometheus/prometheus:v2.45.0 -# TODO ADD MINIO DOCKER IMAGE!! \ No newline at end of file +quay.io/minio/minio \ No newline at end of file diff --git a/deploy/objectstorage/manifests/minio/deploy.yaml.tmpl b/deploy/objectstorage/manifests/minio/deploy.yaml.tmpl index 862c3a113a3..057601700f7 100644 --- a/deploy/objectstorage/manifests/minio/deploy.yaml.tmpl +++ b/deploy/objectstorage/manifests/minio/deploy.yaml.tmpl @@ -1,8 +1,50 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: ${backendNamespace} +--- +apiVersion: v1 +kind: Secret +metadata: + name: object-storage-env-configuration + namespace: ${backendNamespace} + labels: + v1.min.io/tenant: object-storage +data: + config.env: >- + ${ENCODED_CONFIG_ENV} +type: Opaque +--- +apiVersion: v1 +kind: Secret +metadata: + name: object-storage-secret + namespace: ${backendNamespace} + labels: + v1.min.io/tenant: object-storage +data: + accesskey: '' + secretkey: '' +type: Opaque +--- +apiVersion: v1 +kind: Secret +metadata: + name: object-storage-user-0 + namespace: ${backendNamespace} + labels: + v1.min.io/tenant: object-storage +immutable: true +data: + CONSOLE_ACCESS_KEY: ${CONSOLE_ACCESS_KEY} + CONSOLE_SECRET_KEY: ${CONSOLE_SECRET_KEY} +type: Opaque +--- apiVersion: minio.min.io/v2 kind: Tenant metadata: name: object-storage - namespace: ${BACKEND_NAMESPACE} + namespace: ${backendNamespace} spec: configuration: name: object-storage-env-configuration @@ -12,7 +54,7 @@ spec: console: true minio: true features: {} - image: minio/minio:RELEASE.2023-11-11T08-14-41Z + image: quay.io/minio/minio imagePullSecret: {} mountPath: /export pools: @@ -34,11 +76,109 @@ spec: - ReadWriteOnce resources: requests: - storage: ${STORAGE_SIZE}Gi + storage: ${storageSize} status: {} volumesPerServer: 1 requestAutoCert: false users: - name: object-storage-user-0 scheduler: - name: '' \ No newline at end of file + name: '' +--- +apiVersion: v1 +kind: Service +metadata: + name: object-storage + namespace: ${backendNamespace} +spec: + ports: + - name: http-minio + protocol: TCP + port: 80 + targetPort: 9000 + selector: + v1.min.io/tenant: object-storage + type: LoadBalancer + sessionAffinity: None + externalTrafficPolicy: Cluster + ipFamilies: + - IPv4 + ipFamilyPolicy: SingleStack + allocateLoadBalancerNodePorts: true + internalTrafficPolicy: Cluster +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: object-storage-api + namespace: ${backendNamespace} + annotations: + kubernetes.io/ingress.class: nginx + nginx.ingress.kubernetes.io/proxy-body-size: 3g + nginx.ingress.kubernetes.io/server-snippet: | + client_header_buffer_size 64k; + large_client_header_buffers 4 128k; + nginx.ingress.kubernetes.io/ssl-redirect: 'false' + nginx.ingress.kubernetes.io/backend-protocol: HTTP + nginx.ingress.kubernetes.io/rewrite-target: /\$2 + nginx.ingress.kubernetes.io/client-body-buffer-size: 64k + nginx.ingress.kubernetes.io/proxy-buffer-size: 64k + nginx.ingress.kubernetes.io/configuration-snippet: | + if (\$request_uri ~* \.(js|css|gif|jpe?g|png)) { + expires 30d; + add_header Cache-Control "public"; + } +spec: + rules: + - host: objectstorageapi.${cloudDomain} + http: + paths: + - pathType: Prefix + path: /()(.*) + backend: + service: + name: object-storage + port: + number: 80 + tls: + - hosts: + - objectstorageapi.${cloudDomain} + secretName: wildcard-cert +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: object-storage-console + namespace: ${backendNamespace} + annotations: + kubernetes.io/ingress.class: nginx + nginx.ingress.kubernetes.io/proxy-body-size: 3g + nginx.ingress.kubernetes.io/server-snippet: | + client_header_buffer_size 64k; + large_client_header_buffers 4 128k; + nginx.ingress.kubernetes.io/ssl-redirect: 'false' + nginx.ingress.kubernetes.io/backend-protocol: HTTP + nginx.ingress.kubernetes.io/rewrite-target: /\$2 + nginx.ingress.kubernetes.io/client-body-buffer-size: 64k + nginx.ingress.kubernetes.io/proxy-buffer-size: 64k + nginx.ingress.kubernetes.io/configuration-snippet: | + if (\$request_uri ~* \.(js|css|gif|jpe?g|png)) { + expires 30d; + add_header Cache-Control "public"; + } +spec: + rules: + - host: osconsole.${cloudDomain} + http: + paths: + - pathType: Prefix + path: /()(.*) + backend: + service: + name: object-storage-console + port: + number: 9090 + tls: + - hosts: + - osconsole.${cloudDomain} + secretName: wildcard-cert \ No newline at end of file diff --git a/deploy/objectstorage/manifests/prometheus/deploy.yaml.tmpl b/deploy/objectstorage/manifests/prometheus/deploy.yaml.tmpl index e1f2a7cf9b8..06d84b8a828 100644 --- a/deploy/objectstorage/manifests/prometheus/deploy.yaml.tmpl +++ b/deploy/objectstorage/manifests/prometheus/deploy.yaml.tmpl @@ -2,13 +2,13 @@ apiVersion: v1 kind: ServiceAccount metadata: name: object-storage-sa - namespace: ${BACKEND_NAMESPACE} + namespace: ${backendNamespace} --- apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: name: object-storage-role - namespace: ${BACKEND_NAMESPACE} + namespace: ${backendNamespace} rules: - verbs: - get @@ -39,7 +39,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: object-storage-rolebind - namespace: ${BACKEND_NAMESPACE} + namespace: ${backendNamespace} roleRef: apiGroup: rbac.authorization.k8s.io kind: Role @@ -47,7 +47,7 @@ roleRef: subjects: - kind: ServiceAccount name: object-storage-sa - namespace: ${BACKEND_NAMESPACE} + namespace: ${backendNamespace} --- apiVersion: monitoring.coreos.com/v1 kind: Prometheus @@ -55,7 +55,7 @@ metadata: labels: app: prometheus-object-storage name: object-storage - namespace: ${BACKEND_NAMESPACE} + namespace: ${backendNamespace} spec: podMetadata: labels: @@ -90,25 +90,62 @@ spec: metadata: annotations: path: /prometheus - value: ${STORAGE_SIZE}Gi + value: ${storageSize} spec: accessModes: - ReadWriteOnce resources: requests: - storage: ${STORAGE_SIZE}Gi + storage: ${storageSize} --- apiVersion: v1 -kind: Service +kind: ServiceAccount metadata: - name: prometheus-object-storage - namespace: ${BACKEND_NAMESPACE} -spec: - ports: - - port: 9090 - targetPort: 9090 - protocol: TCP - name: http-web - selector: - app: prometheus-object-storage - type: ClusterIP \ No newline at end of file + name: object-storage-sa + namespace: ${backendNamespace} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: object-storage-role + namespace: ${backendNamespace} +rules: + - verbs: + - get + - list + - watch + apiGroups: + - '' + resources: + - secrets + - verbs: + - create + - delete + - get + apiGroups: + - '' + resources: + - services + - verbs: + - get + - list + - watch + apiGroups: + - minio.min.io + resources: + - tenants +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: object-storage-rolebind + namespace: ${backendNamespace} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: object-storage-role +subjects: + - kind: ServiceAccount + name: object-storage-sa + namespace: ${backendNamespace} + diff --git a/deploy/objectstorage/scripts/init.sh b/deploy/objectstorage/scripts/init.sh index 9d484c8c70e..5f4ee493215 100644 --- a/deploy/objectstorage/scripts/init.sh +++ b/deploy/objectstorage/scripts/init.sh @@ -2,7 +2,9 @@ # TODO # 1. create minio instance +bash ./minio.sh # 2. create prometheus instance +bash ./ # 3. run objectstorage controller # 4. run objectstorage frontend # 5. run objectstorage monitor service diff --git a/deploy/objectstorage/scripts/minio.sh b/deploy/objectstorage/scripts/minio.sh index e273062fc43..561a7cc52a7 100644 --- a/deploy/objectstorage/scripts/minio.sh +++ b/deploy/objectstorage/scripts/minio.sh @@ -1 +1,71 @@ -# create minio instance +#!/usr/bin/env bash +set -e + +function deploy_minio() { + MINIO_EXTERNAL_ENDPOINT="https://objectstorageapi.${cloudDomain}" + CONSOLE_ACCESS_KEY=$(echo -n "${minioAdminUser}" | base64 -w 0) + CONSOLE_SECRET_KEY=$(echo -n "${minioAdminPassword}" | base64 -w 0) + + + MINIO_ROOT_USER=$(openssl rand -hex 12 | head -c 16) + MINIO_ROOT_PASSWORD=$(openssl rand -hex 24 | head -c 32) + + CONFIG_ENV="export MINIO_STORAGE_CLASS_STANDARD=\"EC:2\" + export MINIO_BROWSER=\"on\" + export MINIO_ROOT_USER=\"${MINIO_ROOT_USER}\" + export MINIO_ROOT_PASSWORD=\"${MINIO_ROOT_PASSWORD}\"" + + ENCODED_CONFIG_ENV=$(echo -n "$CONFIG_ENV" | base64 -w 0) + + if kubectl get secret object-storage-env-configuration -n objectstorage-system 2>&1; then + ENCODED_CONFIG_ENV=$(kubectl get secret object-storage-env-configuration -n ${backendNamespace} -o jsonpath='{.data.config\.env}') + fi + + kubectl apply -f manifests/minio/deploy.yaml +} + +function init_minio() { + if [ ! -f "$HOME/minio-binaries/mc" ]; then + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o $HOME/minio-binaries/mc + fi + + chmod +x $HOME/minio-binaries/mc + export PATH=$PATH:$HOME/minio-binaries/ + + while kubectl wait -l statefulset.kubernetes.io/pod-name=object-storage-pool-0-0 --for=condition=ready pod -n ${backendNamespace} --timeout=-1s 2>&1 | grep -q "error: no matching resources found"; do + sleep 1 + done + + kubectl wait -l statefulset.kubernetes.io/pod-name=object-storage-pool-0-0 --for=condition=ready pod -n ${backendNamespace} --timeout=-1s + kubectl wait -l statefulset.kubernetes.io/pod-name=object-storage-pool-0-1 --for=condition=ready pod -n ${backendNamespace} --timeout=-1s + kubectl wait -l statefulset.kubernetes.io/pod-name=object-storage-pool-0-2 --for=condition=ready pod -n ${backendNamespace} --timeout=-1s + kubectl wait -l statefulset.kubernetes.io/pod-name=object-storage-pool-0-3 --for=condition=ready pod -n ${backendNamespace} --timeout=-1s + + while mc alias set objectstorage ${MINIO_EXTERNAL_ENDPOINT} ${minioAdminUser} ${minioAdminPassword} 2>&1 | grep -q "Unable to initialize new alias from the provided credentials."; do + sleep 1 + done + + mc admin policy create objectstorage userNormal ./manifests/policy/user_normal.json + mc admin policy create objectstorage userDenyWrite ./manifests/policy/user_deny_write.json + mc admin policy create objectstorage kubeblocks ./manifests/policy/kubeblocks.json + + mc admin user add objectstorage kubeblocks sealos.12345 + mc admin user add objectstorage testuser sealos2023 + mc admin group add objectstorage userNormal testuser + mc admin group add objectstorage userDenyWrite testuser + + mc admin user rm testuser + + mc admin policy attach objectstorage userNormal --group userNormal + mc admin policy attach objectstorage userDenyWrite --group userDenyWrite + mc admin policy attach objectstorage kubeblocks --user kubeblocks +} + +function install() { + deploy_minio + + init_minio +} + +install + diff --git a/deploy/objectstorage/scripts/prometheus.sh b/deploy/objectstorage/scripts/prometheus.sh index 2ce5bd586ba..0a065a81ad5 100644 --- a/deploy/objectstorage/scripts/prometheus.sh +++ b/deploy/objectstorage/scripts/prometheus.sh @@ -1 +1,13 @@ -# create prometheus instance +#!/usr/bin/env bash +set -e + +function deploy_prometheus() { + kubectl apply -f manifests/prometheus/deploy.yaml +} + + +function install() { + deploy_prometheus +} + +install \ No newline at end of file From cc73ba3166bc88c50ba8672d625b31cd2eefc16a Mon Sep 17 00:00:00 2001 From: xuziyi Date: Thu, 25 Jan 2024 17:56:49 +0800 Subject: [PATCH 04/29] refine init.sh --- deploy/objectstorage/scripts/init.sh | 6 ++++-- deploy/objectstorage/scripts/prometheus.sh | 1 - 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/deploy/objectstorage/scripts/init.sh b/deploy/objectstorage/scripts/init.sh index 5f4ee493215..5025eb93d04 100644 --- a/deploy/objectstorage/scripts/init.sh +++ b/deploy/objectstorage/scripts/init.sh @@ -1,10 +1,12 @@ #!/bin/bash -# TODO # 1. create minio instance bash ./minio.sh # 2. create prometheus instance -bash ./ +bash ./prometheus.sh # 3. run objectstorage controller +sealos run ghcr.io/labring/sealos-cloud-objectstorage-controller:latest # 4. run objectstorage frontend +sealos run ghcr.io/labring/sealos-cloud-objectstorage-frontend:latest # 5. run objectstorage monitor service +sealos run ghcr.io/labring/sealos-cloud-minio-service:latest diff --git a/deploy/objectstorage/scripts/prometheus.sh b/deploy/objectstorage/scripts/prometheus.sh index 0a065a81ad5..dbe78193419 100644 --- a/deploy/objectstorage/scripts/prometheus.sh +++ b/deploy/objectstorage/scripts/prometheus.sh @@ -5,7 +5,6 @@ function deploy_prometheus() { kubectl apply -f manifests/prometheus/deploy.yaml } - function install() { deploy_prometheus } From c931de0ab5fd137401f9fcd7da47d9df9249c3fc Mon Sep 17 00:00:00 2001 From: xuziyi Date: Sun, 28 Jan 2024 11:34:46 +0800 Subject: [PATCH 05/29] tmp images --- deploy/objectstorage/images/shim/imageList | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/deploy/objectstorage/images/shim/imageList b/deploy/objectstorage/images/shim/imageList index f5536fe66fc..306118a0a47 100644 --- a/deploy/objectstorage/images/shim/imageList +++ b/deploy/objectstorage/images/shim/imageList @@ -1,2 +1,5 @@ quay.io/prometheus/prometheus:v2.45.0 -quay.io/minio/minio \ No newline at end of file +quay.io/minio/minio +ghcr.io/nowinkeyy/sealos-cloud-objectstorage-controller:latest +ghcr.io/nowinkeyy/sealos-cloud-objectstorage-frontend:latest +ghcr.io/nowinkeyy/sealos-cloud-minio-service:latest \ No newline at end of file From d9c183bc40dec7e1a0b34cd054c28350ac910fea Mon Sep 17 00:00:00 2001 From: xuziyi Date: Sun, 28 Jan 2024 14:59:01 +0800 Subject: [PATCH 06/29] tmp images --- deploy/objectstorage/init.sh | 12 ++++++------ deploy/objectstorage/scripts/init.sh | 10 +++++----- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/deploy/objectstorage/init.sh b/deploy/objectstorage/init.sh index d0b27b91c54..4d571cbb748 100644 --- a/deploy/objectstorage/init.sh +++ b/deploy/objectstorage/init.sh @@ -24,10 +24,10 @@ retryPullImage() { fi } -retryPullImage ghcr.io/labring/sealos-cloud-objectstorage-controller:latest -retryPullImage ghcr.io/labring/sealos-cloud-objectstorage-frontend:latest -retryPullImage ghcr.io/labring/sealos-cloud-minio-service:latest +retryPullImage ghcr.io/nowinkeyy/sealos-cloud-objectstorage-controller:latest +retryPullImage ghcr.io/nowinkeyy/sealos-cloud-objectstorage-frontend:latest +retryPullImage ghcr.io/nowinkeyy/sealos-cloud-minio-service:latest -sealos save -o tars/objectstorage-controller.tar ghcr.io/labring/sealos-cloud-objectstorage-controller:latest -sealos save -o tars/objectstorage-frontend.tar ghcr.io/labring/sealos-cloud-objectstorage-frontend:latest -sealos save -o tars/objectstorage-service.tar ghcr.io/labring/sealos-cloud-minio-service:latest +sealos save -o tars/objectstorage-controller.tar ghcr.io/nowinkeyy/sealos-cloud-objectstorage-controller:latest +sealos save -o tars/objectstorage-frontend.tar ghcr.io/nowinkeyy/sealos-cloud-objectstorage-frontend:latest +sealos save -o tars/objectstorage-service.tar ghcr.io/nowinkeyy/sealos-cloud-minio-service:latest diff --git a/deploy/objectstorage/scripts/init.sh b/deploy/objectstorage/scripts/init.sh index 5025eb93d04..3e942b1c06f 100644 --- a/deploy/objectstorage/scripts/init.sh +++ b/deploy/objectstorage/scripts/init.sh @@ -1,12 +1,12 @@ #!/bin/bash # 1. create minio instance -bash ./minio.sh +bash minio.sh # 2. create prometheus instance -bash ./prometheus.sh +bash prometheus.sh # 3. run objectstorage controller -sealos run ghcr.io/labring/sealos-cloud-objectstorage-controller:latest +sealos run ghcr.io/nowinkeyy/sealos-cloud-objectstorage-controller:latest # 4. run objectstorage frontend -sealos run ghcr.io/labring/sealos-cloud-objectstorage-frontend:latest +sealos run ghcr.io/nowinkeyy/sealos-cloud-objectstorage-frontend:latest # 5. run objectstorage monitor service -sealos run ghcr.io/labring/sealos-cloud-minio-service:latest +sealos run ghcr.io/nowinkeyy/sealos-cloud-minio-service:latest From db8ccd1cb3679da47ac413b6fbbbf64a61565cd4 Mon Sep 17 00:00:00 2001 From: xuziyi Date: Sun, 28 Jan 2024 15:29:08 +0800 Subject: [PATCH 07/29] fix --- deploy/objectstorage/Kubefile | 1 - .../manifests/minio/deploy.yaml.tmpl | 16 ++++++++-------- .../manifests/prometheus/deploy.yaml.tmpl | 18 +++++++++--------- deploy/objectstorage/scripts/init.sh | 4 ++-- deploy/objectstorage/scripts/minio.sh | 12 ++++++------ 5 files changed, 25 insertions(+), 26 deletions(-) diff --git a/deploy/objectstorage/Kubefile b/deploy/objectstorage/Kubefile index 92c2edbc656..586625ecc7b 100644 --- a/deploy/objectstorage/Kubefile +++ b/deploy/objectstorage/Kubefile @@ -9,6 +9,5 @@ ENV cloudPort="" ENV storageSize=${storageSize:-1Gi} ENV minioAdminUser=${minioAdminUser:-"admin"} ENV minioAdminPassword=${minioAdminPassword:-"passw0rd"} -ENV backendNamespace=${backendNamespace:-"objectstorage-system"} CMD ["bash scripts/init.sh"] diff --git a/deploy/objectstorage/manifests/minio/deploy.yaml.tmpl b/deploy/objectstorage/manifests/minio/deploy.yaml.tmpl index 057601700f7..4aae5057028 100644 --- a/deploy/objectstorage/manifests/minio/deploy.yaml.tmpl +++ b/deploy/objectstorage/manifests/minio/deploy.yaml.tmpl @@ -1,13 +1,13 @@ apiVersion: v1 kind: Namespace metadata: - name: ${backendNamespace} + name: objectstorage-system --- apiVersion: v1 kind: Secret metadata: name: object-storage-env-configuration - namespace: ${backendNamespace} + namespace: objectstorage-system labels: v1.min.io/tenant: object-storage data: @@ -19,7 +19,7 @@ apiVersion: v1 kind: Secret metadata: name: object-storage-secret - namespace: ${backendNamespace} + namespace: objectstorage-system labels: v1.min.io/tenant: object-storage data: @@ -31,7 +31,7 @@ apiVersion: v1 kind: Secret metadata: name: object-storage-user-0 - namespace: ${backendNamespace} + namespace: objectstorage-system labels: v1.min.io/tenant: object-storage immutable: true @@ -44,7 +44,7 @@ apiVersion: minio.min.io/v2 kind: Tenant metadata: name: object-storage - namespace: ${backendNamespace} + namespace: objectstorage-system spec: configuration: name: object-storage-env-configuration @@ -89,7 +89,7 @@ apiVersion: v1 kind: Service metadata: name: object-storage - namespace: ${backendNamespace} + namespace: objectstorage-system spec: ports: - name: http-minio @@ -111,7 +111,7 @@ apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: object-storage-api - namespace: ${backendNamespace} + namespace: objectstorage-system annotations: kubernetes.io/ingress.class: nginx nginx.ingress.kubernetes.io/proxy-body-size: 3g @@ -149,7 +149,7 @@ apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: object-storage-console - namespace: ${backendNamespace} + namespace: objectstorage-system annotations: kubernetes.io/ingress.class: nginx nginx.ingress.kubernetes.io/proxy-body-size: 3g diff --git a/deploy/objectstorage/manifests/prometheus/deploy.yaml.tmpl b/deploy/objectstorage/manifests/prometheus/deploy.yaml.tmpl index 06d84b8a828..e054a869644 100644 --- a/deploy/objectstorage/manifests/prometheus/deploy.yaml.tmpl +++ b/deploy/objectstorage/manifests/prometheus/deploy.yaml.tmpl @@ -2,13 +2,13 @@ apiVersion: v1 kind: ServiceAccount metadata: name: object-storage-sa - namespace: ${backendNamespace} + namespace: objectstorage-system --- apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: name: object-storage-role - namespace: ${backendNamespace} + namespace: objectstorage-system rules: - verbs: - get @@ -39,7 +39,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: object-storage-rolebind - namespace: ${backendNamespace} + namespace: objectstorage-system roleRef: apiGroup: rbac.authorization.k8s.io kind: Role @@ -47,7 +47,7 @@ roleRef: subjects: - kind: ServiceAccount name: object-storage-sa - namespace: ${backendNamespace} + namespace: objectstorage-system --- apiVersion: monitoring.coreos.com/v1 kind: Prometheus @@ -55,7 +55,7 @@ metadata: labels: app: prometheus-object-storage name: object-storage - namespace: ${backendNamespace} + namespace: objectstorage-system spec: podMetadata: labels: @@ -102,13 +102,13 @@ apiVersion: v1 kind: ServiceAccount metadata: name: object-storage-sa - namespace: ${backendNamespace} + namespace: objectstorage-system --- apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: name: object-storage-role - namespace: ${backendNamespace} + namespace: objectstorage-system rules: - verbs: - get @@ -139,7 +139,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: object-storage-rolebind - namespace: ${backendNamespace} + namespace: objectstorage-system roleRef: apiGroup: rbac.authorization.k8s.io kind: Role @@ -147,5 +147,5 @@ roleRef: subjects: - kind: ServiceAccount name: object-storage-sa - namespace: ${backendNamespace} + namespace: objectstorage-system diff --git a/deploy/objectstorage/scripts/init.sh b/deploy/objectstorage/scripts/init.sh index 3e942b1c06f..21046b4fa46 100644 --- a/deploy/objectstorage/scripts/init.sh +++ b/deploy/objectstorage/scripts/init.sh @@ -1,9 +1,9 @@ #!/bin/bash # 1. create minio instance -bash minio.sh +bash scripts/minio.sh # 2. create prometheus instance -bash prometheus.sh +bash scripts/prometheus.sh # 3. run objectstorage controller sealos run ghcr.io/nowinkeyy/sealos-cloud-objectstorage-controller:latest # 4. run objectstorage frontend diff --git a/deploy/objectstorage/scripts/minio.sh b/deploy/objectstorage/scripts/minio.sh index 561a7cc52a7..3ba96dcf560 100644 --- a/deploy/objectstorage/scripts/minio.sh +++ b/deploy/objectstorage/scripts/minio.sh @@ -18,7 +18,7 @@ function deploy_minio() { ENCODED_CONFIG_ENV=$(echo -n "$CONFIG_ENV" | base64 -w 0) if kubectl get secret object-storage-env-configuration -n objectstorage-system 2>&1; then - ENCODED_CONFIG_ENV=$(kubectl get secret object-storage-env-configuration -n ${backendNamespace} -o jsonpath='{.data.config\.env}') + ENCODED_CONFIG_ENV=$(kubectl get secret object-storage-env-configuration -n objectstorage-system -o jsonpath='{.data.config\.env}') fi kubectl apply -f manifests/minio/deploy.yaml @@ -32,14 +32,14 @@ function init_minio() { chmod +x $HOME/minio-binaries/mc export PATH=$PATH:$HOME/minio-binaries/ - while kubectl wait -l statefulset.kubernetes.io/pod-name=object-storage-pool-0-0 --for=condition=ready pod -n ${backendNamespace} --timeout=-1s 2>&1 | grep -q "error: no matching resources found"; do + while kubectl wait -l statefulset.kubernetes.io/pod-name=object-storage-pool-0-0 --for=condition=ready pod -n objectstorage-system --timeout=-1s 2>&1 | grep -q "error: no matching resources found"; do sleep 1 done - kubectl wait -l statefulset.kubernetes.io/pod-name=object-storage-pool-0-0 --for=condition=ready pod -n ${backendNamespace} --timeout=-1s - kubectl wait -l statefulset.kubernetes.io/pod-name=object-storage-pool-0-1 --for=condition=ready pod -n ${backendNamespace} --timeout=-1s - kubectl wait -l statefulset.kubernetes.io/pod-name=object-storage-pool-0-2 --for=condition=ready pod -n ${backendNamespace} --timeout=-1s - kubectl wait -l statefulset.kubernetes.io/pod-name=object-storage-pool-0-3 --for=condition=ready pod -n ${backendNamespace} --timeout=-1s + kubectl wait -l statefulset.kubernetes.io/pod-name=object-storage-pool-0-0 --for=condition=ready pod -n objectstorage-system --timeout=-1s + kubectl wait -l statefulset.kubernetes.io/pod-name=object-storage-pool-0-1 --for=condition=ready pod -n objectstorage-system --timeout=-1s + kubectl wait -l statefulset.kubernetes.io/pod-name=object-storage-pool-0-2 --for=condition=ready pod -n objectstorage-system --timeout=-1s + kubectl wait -l statefulset.kubernetes.io/pod-name=object-storage-pool-0-3 --for=condition=ready pod -n objectstorage-system --timeout=-1s while mc alias set objectstorage ${MINIO_EXTERNAL_ENDPOINT} ${minioAdminUser} ${minioAdminPassword} 2>&1 | grep -q "Unable to initialize new alias from the provided credentials."; do sleep 1 From 8abbf57fc2545bd30b0e43285de1ed1c3fd7f354 Mon Sep 17 00:00:00 2001 From: xuziyi Date: Sun, 28 Jan 2024 16:22:02 +0800 Subject: [PATCH 08/29] fix env --- .../manifests/minio/deploy.yaml.tmpl | 16 ++++++++-------- .../manifests/prometheus/deploy.yaml.tmpl | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/deploy/objectstorage/manifests/minio/deploy.yaml.tmpl b/deploy/objectstorage/manifests/minio/deploy.yaml.tmpl index 4aae5057028..5d1a2ed2977 100644 --- a/deploy/objectstorage/manifests/minio/deploy.yaml.tmpl +++ b/deploy/objectstorage/manifests/minio/deploy.yaml.tmpl @@ -12,7 +12,7 @@ metadata: v1.min.io/tenant: object-storage data: config.env: >- - ${ENCODED_CONFIG_ENV} + '{{ .ENCODED_CONFIG_ENV }}' type: Opaque --- apiVersion: v1 @@ -36,8 +36,8 @@ metadata: v1.min.io/tenant: object-storage immutable: true data: - CONSOLE_ACCESS_KEY: ${CONSOLE_ACCESS_KEY} - CONSOLE_SECRET_KEY: ${CONSOLE_SECRET_KEY} + CONSOLE_ACCESS_KEY: '{{ .CONSOLE_ACCESS_KEY }}' + CONSOLE_SECRET_KEY: '{{ .CONSOLE_SECRET_KEY }}' type: Opaque --- apiVersion: minio.min.io/v2 @@ -76,7 +76,7 @@ spec: - ReadWriteOnce resources: requests: - storage: ${storageSize} + storage: '{{ .storageSize }}' status: {} volumesPerServer: 1 requestAutoCert: false @@ -130,7 +130,7 @@ metadata: } spec: rules: - - host: objectstorageapi.${cloudDomain} + - host: objectstorageapi.'{{ .cloudDomain }}' http: paths: - pathType: Prefix @@ -142,7 +142,7 @@ spec: number: 80 tls: - hosts: - - objectstorageapi.${cloudDomain} + - objectstorageapi.'{{ .cloudDomain }}' secretName: wildcard-cert --- apiVersion: networking.k8s.io/v1 @@ -168,7 +168,7 @@ metadata: } spec: rules: - - host: osconsole.${cloudDomain} + - host: osconsole.'{{ .cloudDomain }}' http: paths: - pathType: Prefix @@ -180,5 +180,5 @@ spec: number: 9090 tls: - hosts: - - osconsole.${cloudDomain} + - osconsole.'{{ .cloudDomain }}' secretName: wildcard-cert \ No newline at end of file diff --git a/deploy/objectstorage/manifests/prometheus/deploy.yaml.tmpl b/deploy/objectstorage/manifests/prometheus/deploy.yaml.tmpl index e054a869644..43c3eda55ca 100644 --- a/deploy/objectstorage/manifests/prometheus/deploy.yaml.tmpl +++ b/deploy/objectstorage/manifests/prometheus/deploy.yaml.tmpl @@ -90,13 +90,13 @@ spec: metadata: annotations: path: /prometheus - value: ${storageSize} + value: '{{ .storageSize }}' spec: accessModes: - ReadWriteOnce resources: requests: - storage: ${storageSize} + storage: '{{ .storageSize }}' --- apiVersion: v1 kind: ServiceAccount From 5bc5e3917b450b88aeea9d16ee2abaee4f86dd08 Mon Sep 17 00:00:00 2001 From: xuziyi Date: Sun, 28 Jan 2024 16:31:09 +0800 Subject: [PATCH 09/29] rm '' --- .../manifests/minio/deploy.yaml.tmpl | 16 ++++++++-------- .../manifests/prometheus/deploy.yaml.tmpl | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/deploy/objectstorage/manifests/minio/deploy.yaml.tmpl b/deploy/objectstorage/manifests/minio/deploy.yaml.tmpl index 5d1a2ed2977..9d3b6baee80 100644 --- a/deploy/objectstorage/manifests/minio/deploy.yaml.tmpl +++ b/deploy/objectstorage/manifests/minio/deploy.yaml.tmpl @@ -12,7 +12,7 @@ metadata: v1.min.io/tenant: object-storage data: config.env: >- - '{{ .ENCODED_CONFIG_ENV }}' + {{ .ENCODED_CONFIG_ENV }} type: Opaque --- apiVersion: v1 @@ -36,8 +36,8 @@ metadata: v1.min.io/tenant: object-storage immutable: true data: - CONSOLE_ACCESS_KEY: '{{ .CONSOLE_ACCESS_KEY }}' - CONSOLE_SECRET_KEY: '{{ .CONSOLE_SECRET_KEY }}' + CONSOLE_ACCESS_KEY: {{ .CONSOLE_ACCESS_KEY }} + CONSOLE_SECRET_KEY: {{ .CONSOLE_SECRET_KEY }} type: Opaque --- apiVersion: minio.min.io/v2 @@ -76,7 +76,7 @@ spec: - ReadWriteOnce resources: requests: - storage: '{{ .storageSize }}' + storage: {{ .storageSize }} status: {} volumesPerServer: 1 requestAutoCert: false @@ -130,7 +130,7 @@ metadata: } spec: rules: - - host: objectstorageapi.'{{ .cloudDomain }}' + - host: objectstorageapi.{{ .cloudDomain }} http: paths: - pathType: Prefix @@ -142,7 +142,7 @@ spec: number: 80 tls: - hosts: - - objectstorageapi.'{{ .cloudDomain }}' + - objectstorageapi.{{ .cloudDomain }} secretName: wildcard-cert --- apiVersion: networking.k8s.io/v1 @@ -168,7 +168,7 @@ metadata: } spec: rules: - - host: osconsole.'{{ .cloudDomain }}' + - host: osconsole.{{ .cloudDomain }} http: paths: - pathType: Prefix @@ -180,5 +180,5 @@ spec: number: 9090 tls: - hosts: - - osconsole.'{{ .cloudDomain }}' + - osconsole.{{ .cloudDomain }} secretName: wildcard-cert \ No newline at end of file diff --git a/deploy/objectstorage/manifests/prometheus/deploy.yaml.tmpl b/deploy/objectstorage/manifests/prometheus/deploy.yaml.tmpl index 43c3eda55ca..139a991f0c0 100644 --- a/deploy/objectstorage/manifests/prometheus/deploy.yaml.tmpl +++ b/deploy/objectstorage/manifests/prometheus/deploy.yaml.tmpl @@ -90,13 +90,13 @@ spec: metadata: annotations: path: /prometheus - value: '{{ .storageSize }}' + value: {{ .storageSize }} spec: accessModes: - ReadWriteOnce resources: requests: - storage: '{{ .storageSize }}' + storage: {{ .storageSize }} --- apiVersion: v1 kind: ServiceAccount From 474e1a5392d56a1747cbadf9e7cf64965a784866 Mon Sep 17 00:00:00 2001 From: xuziyi Date: Sun, 28 Jan 2024 17:51:22 +0800 Subject: [PATCH 10/29] fix --- deploy/objectstorage/Kubefile | 4 ++-- deploy/objectstorage/manifests/minio/deploy.yaml.tmpl | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/deploy/objectstorage/Kubefile b/deploy/objectstorage/Kubefile index 586625ecc7b..6cb563f0c08 100644 --- a/deploy/objectstorage/Kubefile +++ b/deploy/objectstorage/Kubefile @@ -7,7 +7,7 @@ COPY manifests manifests ENV cloudDomain="127.0.0.1.nip.io" ENV cloudPort="" ENV storageSize=${storageSize:-1Gi} -ENV minioAdminUser=${minioAdminUser:-"admin"} +ENV minioAdminUser=${minioAdminUser:-"username"} ENV minioAdminPassword=${minioAdminPassword:-"passw0rd"} -CMD ["bash scripts/init.sh"] +CMD ["bash init.sh"] diff --git a/deploy/objectstorage/manifests/minio/deploy.yaml.tmpl b/deploy/objectstorage/manifests/minio/deploy.yaml.tmpl index 9d3b6baee80..29b45d51de1 100644 --- a/deploy/objectstorage/manifests/minio/deploy.yaml.tmpl +++ b/deploy/objectstorage/manifests/minio/deploy.yaml.tmpl @@ -120,11 +120,11 @@ metadata: large_client_header_buffers 4 128k; nginx.ingress.kubernetes.io/ssl-redirect: 'false' nginx.ingress.kubernetes.io/backend-protocol: HTTP - nginx.ingress.kubernetes.io/rewrite-target: /\$2 + nginx.ingress.kubernetes.io/rewrite-target: /$2 nginx.ingress.kubernetes.io/client-body-buffer-size: 64k nginx.ingress.kubernetes.io/proxy-buffer-size: 64k nginx.ingress.kubernetes.io/configuration-snippet: | - if (\$request_uri ~* \.(js|css|gif|jpe?g|png)) { + if ($request_uri ~* \.(js|css|gif|jpe?g|png)) { expires 30d; add_header Cache-Control "public"; } @@ -158,11 +158,11 @@ metadata: large_client_header_buffers 4 128k; nginx.ingress.kubernetes.io/ssl-redirect: 'false' nginx.ingress.kubernetes.io/backend-protocol: HTTP - nginx.ingress.kubernetes.io/rewrite-target: /\$2 + nginx.ingress.kubernetes.io/rewrite-target: /$2 nginx.ingress.kubernetes.io/client-body-buffer-size: 64k nginx.ingress.kubernetes.io/proxy-buffer-size: 64k nginx.ingress.kubernetes.io/configuration-snippet: | - if (\$request_uri ~* \.(js|css|gif|jpe?g|png)) { + if ($request_uri ~* \.(js|css|gif|jpe?g|png)) { expires 30d; add_header Cache-Control "public"; } From 6ee3a3d78e1fa62f9b417a2e53f9f915cbf25be3 Mon Sep 17 00:00:00 2001 From: xuziyi Date: Mon, 29 Jan 2024 10:51:31 +0800 Subject: [PATCH 11/29] test --- deploy/objectstorage/Kubefile | 2 +- deploy/objectstorage/manifests/minio/deploy.yaml.tmpl | 6 +++--- deploy/objectstorage/scripts/minio.sh | 10 +++++----- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/deploy/objectstorage/Kubefile b/deploy/objectstorage/Kubefile index 6cb563f0c08..5c97b92e00d 100644 --- a/deploy/objectstorage/Kubefile +++ b/deploy/objectstorage/Kubefile @@ -10,4 +10,4 @@ ENV storageSize=${storageSize:-1Gi} ENV minioAdminUser=${minioAdminUser:-"username"} ENV minioAdminPassword=${minioAdminPassword:-"passw0rd"} -CMD ["bash init.sh"] +CMD ["bash scripts/init.sh"] diff --git a/deploy/objectstorage/manifests/minio/deploy.yaml.tmpl b/deploy/objectstorage/manifests/minio/deploy.yaml.tmpl index 29b45d51de1..3376247ee08 100644 --- a/deploy/objectstorage/manifests/minio/deploy.yaml.tmpl +++ b/deploy/objectstorage/manifests/minio/deploy.yaml.tmpl @@ -12,7 +12,7 @@ metadata: v1.min.io/tenant: object-storage data: config.env: >- - {{ .ENCODED_CONFIG_ENV }} + "{{ .ENCODED_CONFIG_ENV }}" type: Opaque --- apiVersion: v1 @@ -36,8 +36,8 @@ metadata: v1.min.io/tenant: object-storage immutable: true data: - CONSOLE_ACCESS_KEY: {{ .CONSOLE_ACCESS_KEY }} - CONSOLE_SECRET_KEY: {{ .CONSOLE_SECRET_KEY }} + CONSOLE_ACCESS_KEY: "{{ .CONSOLE_ACCESS_KEY }}" + CONSOLE_SECRET_KEY: "{{ .CONSOLE_SECRET_KEY }}" type: Opaque --- apiVersion: minio.min.io/v2 diff --git a/deploy/objectstorage/scripts/minio.sh b/deploy/objectstorage/scripts/minio.sh index 3ba96dcf560..305231152e9 100644 --- a/deploy/objectstorage/scripts/minio.sh +++ b/deploy/objectstorage/scripts/minio.sh @@ -10,14 +10,14 @@ function deploy_minio() { MINIO_ROOT_USER=$(openssl rand -hex 12 | head -c 16) MINIO_ROOT_PASSWORD=$(openssl rand -hex 24 | head -c 32) - CONFIG_ENV="export MINIO_STORAGE_CLASS_STANDARD=\"EC:2\" - export MINIO_BROWSER=\"on\" - export MINIO_ROOT_USER=\"${MINIO_ROOT_USER}\" - export MINIO_ROOT_PASSWORD=\"${MINIO_ROOT_PASSWORD}\"" +CONFIG_ENV="export MINIO_STORAGE_CLASS_STANDARD=\"EC:2\" +export MINIO_BROWSER=\"on\" +export MINIO_ROOT_USER=\"${MINIO_ROOT_USER}\" +export MINIO_ROOT_PASSWORD=\"${MINIO_ROOT_PASSWORD}\"" ENCODED_CONFIG_ENV=$(echo -n "$CONFIG_ENV" | base64 -w 0) - if kubectl get secret object-storage-env-configuration -n objectstorage-system 2>&1; then + if kubectl get secret object-storage-env-configuration -n objectstorage-system 2>/dev/null >/dev/; then ENCODED_CONFIG_ENV=$(kubectl get secret object-storage-env-configuration -n objectstorage-system -o jsonpath='{.data.config\.env}') fi From 20a48eeb5d9a2ede6885aa191122441470ce3824 Mon Sep 17 00:00:00 2001 From: xuziyi Date: Mon, 29 Jan 2024 14:55:45 +0800 Subject: [PATCH 12/29] add env --- deploy/objectstorage/Kubefile | 1 + 1 file changed, 1 insertion(+) diff --git a/deploy/objectstorage/Kubefile b/deploy/objectstorage/Kubefile index 5c97b92e00d..8318d8d901b 100644 --- a/deploy/objectstorage/Kubefile +++ b/deploy/objectstorage/Kubefile @@ -9,5 +9,6 @@ ENV cloudPort="" ENV storageSize=${storageSize:-1Gi} ENV minioAdminUser=${minioAdminUser:-"username"} ENV minioAdminPassword=${minioAdminPassword:-"passw0rd"} +ENV ENCODED_CONFIG_ENV="" CMD ["bash scripts/init.sh"] From a9e974a56078daeff7b052071657825d4f60f57c Mon Sep 17 00:00:00 2001 From: xuziyi Date: Mon, 29 Jan 2024 15:17:03 +0800 Subject: [PATCH 13/29] add env to Kubefile --- deploy/objectstorage/Kubefile | 4 +++- deploy/objectstorage/manifests/minio/deploy.yaml.tmpl | 6 +++--- deploy/objectstorage/scripts/init.sh | 2 ++ deploy/objectstorage/scripts/minio.sh | 8 ++++---- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/deploy/objectstorage/Kubefile b/deploy/objectstorage/Kubefile index 8318d8d901b..e72d5a38f9d 100644 --- a/deploy/objectstorage/Kubefile +++ b/deploy/objectstorage/Kubefile @@ -9,6 +9,8 @@ ENV cloudPort="" ENV storageSize=${storageSize:-1Gi} ENV minioAdminUser=${minioAdminUser:-"username"} ENV minioAdminPassword=${minioAdminPassword:-"passw0rd"} -ENV ENCODED_CONFIG_ENV="" +ENV ENCODED_CONFIG_ENV="dXNlcm5hbWU=" +ENV CONSOLE_ACCESS_KEY="dXNlcm5hbWU=" +ENV CONSOLE_SECRET_KEY="dXNlcm5hbWU=" CMD ["bash scripts/init.sh"] diff --git a/deploy/objectstorage/manifests/minio/deploy.yaml.tmpl b/deploy/objectstorage/manifests/minio/deploy.yaml.tmpl index 3376247ee08..29b45d51de1 100644 --- a/deploy/objectstorage/manifests/minio/deploy.yaml.tmpl +++ b/deploy/objectstorage/manifests/minio/deploy.yaml.tmpl @@ -12,7 +12,7 @@ metadata: v1.min.io/tenant: object-storage data: config.env: >- - "{{ .ENCODED_CONFIG_ENV }}" + {{ .ENCODED_CONFIG_ENV }} type: Opaque --- apiVersion: v1 @@ -36,8 +36,8 @@ metadata: v1.min.io/tenant: object-storage immutable: true data: - CONSOLE_ACCESS_KEY: "{{ .CONSOLE_ACCESS_KEY }}" - CONSOLE_SECRET_KEY: "{{ .CONSOLE_SECRET_KEY }}" + CONSOLE_ACCESS_KEY: {{ .CONSOLE_ACCESS_KEY }} + CONSOLE_SECRET_KEY: {{ .CONSOLE_SECRET_KEY }} type: Opaque --- apiVersion: minio.min.io/v2 diff --git a/deploy/objectstorage/scripts/init.sh b/deploy/objectstorage/scripts/init.sh index 21046b4fa46..0540dc14eca 100644 --- a/deploy/objectstorage/scripts/init.sh +++ b/deploy/objectstorage/scripts/init.sh @@ -1,5 +1,7 @@ #!/bin/bash +# 0.create minio-operator +# sealos run ghcr.io/labring/minio-operator:v5.0.6 # 1. create minio instance bash scripts/minio.sh # 2. create prometheus instance diff --git a/deploy/objectstorage/scripts/minio.sh b/deploy/objectstorage/scripts/minio.sh index 305231152e9..3ee6cd02049 100644 --- a/deploy/objectstorage/scripts/minio.sh +++ b/deploy/objectstorage/scripts/minio.sh @@ -10,10 +10,10 @@ function deploy_minio() { MINIO_ROOT_USER=$(openssl rand -hex 12 | head -c 16) MINIO_ROOT_PASSWORD=$(openssl rand -hex 24 | head -c 32) -CONFIG_ENV="export MINIO_STORAGE_CLASS_STANDARD=\"EC:2\" -export MINIO_BROWSER=\"on\" -export MINIO_ROOT_USER=\"${MINIO_ROOT_USER}\" -export MINIO_ROOT_PASSWORD=\"${MINIO_ROOT_PASSWORD}\"" + CONFIG_ENV="export MINIO_STORAGE_CLASS_STANDARD=\"EC:2\" + export MINIO_BROWSER=\"on\" + export MINIO_ROOT_USER=\"${MINIO_ROOT_USER}\" + export MINIO_ROOT_PASSWORD=\"${MINIO_ROOT_PASSWORD}\"" ENCODED_CONFIG_ENV=$(echo -n "$CONFIG_ENV" | base64 -w 0) From ad638f1697162314d00ba21bd9fc31832ab8c69d Mon Sep 17 00:00:00 2001 From: xuziyi Date: Mon, 29 Jan 2024 16:22:38 +0800 Subject: [PATCH 14/29] use sed to replace env --- deploy/objectstorage/Kubefile | 3 --- deploy/objectstorage/manifests/minio/deploy.yaml.tmpl | 6 +++--- deploy/objectstorage/scripts/minio.sh | 4 ++++ 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/deploy/objectstorage/Kubefile b/deploy/objectstorage/Kubefile index e72d5a38f9d..5c97b92e00d 100644 --- a/deploy/objectstorage/Kubefile +++ b/deploy/objectstorage/Kubefile @@ -9,8 +9,5 @@ ENV cloudPort="" ENV storageSize=${storageSize:-1Gi} ENV minioAdminUser=${minioAdminUser:-"username"} ENV minioAdminPassword=${minioAdminPassword:-"passw0rd"} -ENV ENCODED_CONFIG_ENV="dXNlcm5hbWU=" -ENV CONSOLE_ACCESS_KEY="dXNlcm5hbWU=" -ENV CONSOLE_SECRET_KEY="dXNlcm5hbWU=" CMD ["bash scripts/init.sh"] diff --git a/deploy/objectstorage/manifests/minio/deploy.yaml.tmpl b/deploy/objectstorage/manifests/minio/deploy.yaml.tmpl index 29b45d51de1..53987c472c0 100644 --- a/deploy/objectstorage/manifests/minio/deploy.yaml.tmpl +++ b/deploy/objectstorage/manifests/minio/deploy.yaml.tmpl @@ -12,7 +12,7 @@ metadata: v1.min.io/tenant: object-storage data: config.env: >- - {{ .ENCODED_CONFIG_ENV }} + ${ENCODED_CONFIG_ENV} type: Opaque --- apiVersion: v1 @@ -36,8 +36,8 @@ metadata: v1.min.io/tenant: object-storage immutable: true data: - CONSOLE_ACCESS_KEY: {{ .CONSOLE_ACCESS_KEY }} - CONSOLE_SECRET_KEY: {{ .CONSOLE_SECRET_KEY }} + CONSOLE_ACCESS_KEY: ${CONSOLE_ACCESS_KEY} + CONSOLE_SECRET_KEY: ${CONSOLE_SECRET_KEY} type: Opaque --- apiVersion: minio.min.io/v2 diff --git a/deploy/objectstorage/scripts/minio.sh b/deploy/objectstorage/scripts/minio.sh index 3ee6cd02049..b35245bc6f6 100644 --- a/deploy/objectstorage/scripts/minio.sh +++ b/deploy/objectstorage/scripts/minio.sh @@ -21,6 +21,10 @@ function deploy_minio() { ENCODED_CONFIG_ENV=$(kubectl get secret object-storage-env-configuration -n objectstorage-system -o jsonpath='{.data.config\.env}') fi + sed -i 's/${ENCODED_CONFIG_ENV}/$ENCODED_CONFIG_ENV/g' manifests/minio.deploy.yaml + sed -i 's/${CONSOLE_ACCESS_KEY}/$CONSOLE_ACCESS_KEY/g' manifests/minio.deploy.yaml + sed -i 's/${CONSOLE_SECRET_KEY}/$CONSOLE_SECRET_KEY/g' manifests/minio.deploy.yaml + kubectl apply -f manifests/minio/deploy.yaml } From af8e230d67e554aa66ae8d3383307f1e067e9f3c Mon Sep 17 00:00:00 2001 From: xuziyi Date: Mon, 29 Jan 2024 16:38:55 +0800 Subject: [PATCH 15/29] fix path --- deploy/objectstorage/scripts/minio.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/deploy/objectstorage/scripts/minio.sh b/deploy/objectstorage/scripts/minio.sh index b35245bc6f6..552ec2611e8 100644 --- a/deploy/objectstorage/scripts/minio.sh +++ b/deploy/objectstorage/scripts/minio.sh @@ -21,9 +21,9 @@ function deploy_minio() { ENCODED_CONFIG_ENV=$(kubectl get secret object-storage-env-configuration -n objectstorage-system -o jsonpath='{.data.config\.env}') fi - sed -i 's/${ENCODED_CONFIG_ENV}/$ENCODED_CONFIG_ENV/g' manifests/minio.deploy.yaml - sed -i 's/${CONSOLE_ACCESS_KEY}/$CONSOLE_ACCESS_KEY/g' manifests/minio.deploy.yaml - sed -i 's/${CONSOLE_SECRET_KEY}/$CONSOLE_SECRET_KEY/g' manifests/minio.deploy.yaml + sed -i 's/${ENCODED_CONFIG_ENV}/$ENCODED_CONFIG_ENV/g' manifests/minio/deploy.yaml + sed -i 's/${CONSOLE_ACCESS_KEY}/$CONSOLE_ACCESS_KEY/g' manifests/minio/deploy.yaml + sed -i 's/${CONSOLE_SECRET_KEY}/$CONSOLE_SECRET_KEY/g' manifests/minio/deploy.yaml kubectl apply -f manifests/minio/deploy.yaml } From 5fd4bb6e952f56a2b0e422263faa797f75504858 Mon Sep 17 00:00:00 2001 From: xuziyi Date: Mon, 29 Jan 2024 16:44:51 +0800 Subject: [PATCH 16/29] fix --- deploy/objectstorage/manifests/minio/deploy.yaml.tmpl | 6 +++--- deploy/objectstorage/scripts/minio.sh | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/deploy/objectstorage/manifests/minio/deploy.yaml.tmpl b/deploy/objectstorage/manifests/minio/deploy.yaml.tmpl index 53987c472c0..fb76f240e5e 100644 --- a/deploy/objectstorage/manifests/minio/deploy.yaml.tmpl +++ b/deploy/objectstorage/manifests/minio/deploy.yaml.tmpl @@ -12,7 +12,7 @@ metadata: v1.min.io/tenant: object-storage data: config.env: >- - ${ENCODED_CONFIG_ENV} + {ENCODED_CONFIG_ENV} type: Opaque --- apiVersion: v1 @@ -36,8 +36,8 @@ metadata: v1.min.io/tenant: object-storage immutable: true data: - CONSOLE_ACCESS_KEY: ${CONSOLE_ACCESS_KEY} - CONSOLE_SECRET_KEY: ${CONSOLE_SECRET_KEY} + CONSOLE_ACCESS_KEY: {CONSOLE_ACCESS_KEY} + CONSOLE_SECRET_KEY: {CONSOLE_SECRET_KEY} type: Opaque --- apiVersion: minio.min.io/v2 diff --git a/deploy/objectstorage/scripts/minio.sh b/deploy/objectstorage/scripts/minio.sh index 552ec2611e8..5cf255e5f53 100644 --- a/deploy/objectstorage/scripts/minio.sh +++ b/deploy/objectstorage/scripts/minio.sh @@ -21,9 +21,9 @@ function deploy_minio() { ENCODED_CONFIG_ENV=$(kubectl get secret object-storage-env-configuration -n objectstorage-system -o jsonpath='{.data.config\.env}') fi - sed -i 's/${ENCODED_CONFIG_ENV}/$ENCODED_CONFIG_ENV/g' manifests/minio/deploy.yaml - sed -i 's/${CONSOLE_ACCESS_KEY}/$CONSOLE_ACCESS_KEY/g' manifests/minio/deploy.yaml - sed -i 's/${CONSOLE_SECRET_KEY}/$CONSOLE_SECRET_KEY/g' manifests/minio/deploy.yaml + sed -i 's/{ENCODED_CONFIG_ENV}/$ENCODED_CONFIG_ENV/g' manifests/minio/deploy.yaml + sed -i 's/{CONSOLE_ACCESS_KEY}/$CONSOLE_ACCESS_KEY/g' manifests/minio/deploy.yaml.tmpl + sed -i 's/{CONSOLE_SECRET_KEY}/$CONSOLE_SECRET_KEY/g' manifests/minio/deploy.yaml.tmpl kubectl apply -f manifests/minio/deploy.yaml } From 44e847636afa6d8802bc26ac437bb8fe6eb4b4ba Mon Sep 17 00:00:00 2001 From: xuziyi Date: Mon, 29 Jan 2024 17:34:35 +0800 Subject: [PATCH 17/29] fix sed --- deploy/objectstorage/scripts/minio.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/deploy/objectstorage/scripts/minio.sh b/deploy/objectstorage/scripts/minio.sh index 5cf255e5f53..102ceac4c78 100644 --- a/deploy/objectstorage/scripts/minio.sh +++ b/deploy/objectstorage/scripts/minio.sh @@ -21,9 +21,9 @@ function deploy_minio() { ENCODED_CONFIG_ENV=$(kubectl get secret object-storage-env-configuration -n objectstorage-system -o jsonpath='{.data.config\.env}') fi - sed -i 's/{ENCODED_CONFIG_ENV}/$ENCODED_CONFIG_ENV/g' manifests/minio/deploy.yaml - sed -i 's/{CONSOLE_ACCESS_KEY}/$CONSOLE_ACCESS_KEY/g' manifests/minio/deploy.yaml.tmpl - sed -i 's/{CONSOLE_SECRET_KEY}/$CONSOLE_SECRET_KEY/g' manifests/minio/deploy.yaml.tmpl + sed -i 's/{ENCODED_CONFIG_ENV}/'${ENCODED_CONFIG_ENV}'/g' manifests/minio/deploy.yaml + sed -i 's/{CONSOLE_ACCESS_KEY}/'${CONSOLE_ACCESS_KEY}'/g' manifests/minio/deploy.yaml + sed -i 's/{CONSOLE_SECRET_KEY}/'${CONSOLE_SECRET_KEY}'/g' manifests/minio/deploy.yaml kubectl apply -f manifests/minio/deploy.yaml } From e802638bb47f8ae7339294b0dae2d52e6c8976e0 Mon Sep 17 00:00:00 2001 From: xuziyi Date: Mon, 29 Jan 2024 17:45:33 +0800 Subject: [PATCH 18/29] fix policy path --- deploy/objectstorage/scripts/minio.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/deploy/objectstorage/scripts/minio.sh b/deploy/objectstorage/scripts/minio.sh index 102ceac4c78..9b3122a9741 100644 --- a/deploy/objectstorage/scripts/minio.sh +++ b/deploy/objectstorage/scripts/minio.sh @@ -49,9 +49,9 @@ function init_minio() { sleep 1 done - mc admin policy create objectstorage userNormal ./manifests/policy/user_normal.json - mc admin policy create objectstorage userDenyWrite ./manifests/policy/user_deny_write.json - mc admin policy create objectstorage kubeblocks ./manifests/policy/kubeblocks.json + mc admin policy create objectstorage userNormal etc/minio/policy/user_normal.json + mc admin policy create objectstorage userDenyWrite etc/minio/policy/user_deny_write.json + mc admin policy create objectstorage kubeblocks etc/minio/policy/kubeblocks.json mc admin user add objectstorage kubeblocks sealos.12345 mc admin user add objectstorage testuser sealos2023 From 317570b477c92c90804942802a2bb695eb638485 Mon Sep 17 00:00:00 2001 From: xuziyi Date: Tue, 30 Jan 2024 14:50:01 +0800 Subject: [PATCH 19/29] add env --- deploy/objectstorage/Kubefile | 2 +- deploy/objectstorage/scripts/init.sh | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/deploy/objectstorage/Kubefile b/deploy/objectstorage/Kubefile index 5c97b92e00d..eb3411953ce 100644 --- a/deploy/objectstorage/Kubefile +++ b/deploy/objectstorage/Kubefile @@ -4,7 +4,7 @@ COPY etc etc COPY scripts scripts COPY manifests manifests -ENV cloudDomain="127.0.0.1.nip.io" +ENV cloudDomain=${cloudDomain:-"127.0.0.1.nip.io"} ENV cloudPort="" ENV storageSize=${storageSize:-1Gi} ENV minioAdminUser=${minioAdminUser:-"username"} diff --git a/deploy/objectstorage/scripts/init.sh b/deploy/objectstorage/scripts/init.sh index 0540dc14eca..feebb876ede 100644 --- a/deploy/objectstorage/scripts/init.sh +++ b/deploy/objectstorage/scripts/init.sh @@ -7,8 +7,8 @@ bash scripts/minio.sh # 2. create prometheus instance bash scripts/prometheus.sh # 3. run objectstorage controller -sealos run ghcr.io/nowinkeyy/sealos-cloud-objectstorage-controller:latest +sealos run ghcr.io/nowinkeyy/sealos-cloud-objectstorage-controller:latest -e cloudDomain=${cloudDomain} # 4. run objectstorage frontend -sealos run ghcr.io/nowinkeyy/sealos-cloud-objectstorage-frontend:latest +sealos run ghcr.io/nowinkeyy/sealos-cloud-objectstorage-frontend:latest -e cloudDomain=${cloudDomain} # 5. run objectstorage monitor service -sealos run ghcr.io/nowinkeyy/sealos-cloud-minio-service:latest +sealos run ghcr.io/nowinkeyy/sealos-cloud-minio-service:latest -e cloudDomain=${cloudDomain} From 4fbb8a32bab272cd02d96606794eb45d75441553 Mon Sep 17 00:00:00 2001 From: xuziyi Date: Tue, 30 Jan 2024 16:16:48 +0800 Subject: [PATCH 20/29] fix mc admin remove user --- deploy/objectstorage/scripts/init.sh | 2 +- deploy/objectstorage/scripts/minio.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/objectstorage/scripts/init.sh b/deploy/objectstorage/scripts/init.sh index feebb876ede..943086e63f0 100644 --- a/deploy/objectstorage/scripts/init.sh +++ b/deploy/objectstorage/scripts/init.sh @@ -1,7 +1,7 @@ #!/bin/bash # 0.create minio-operator -# sealos run ghcr.io/labring/minio-operator:v5.0.6 +sealos run docker.io/nowinkey/minio-operator:v5.0.6 # 1. create minio instance bash scripts/minio.sh # 2. create prometheus instance diff --git a/deploy/objectstorage/scripts/minio.sh b/deploy/objectstorage/scripts/minio.sh index 9b3122a9741..be8e07932cf 100644 --- a/deploy/objectstorage/scripts/minio.sh +++ b/deploy/objectstorage/scripts/minio.sh @@ -58,7 +58,7 @@ function init_minio() { mc admin group add objectstorage userNormal testuser mc admin group add objectstorage userDenyWrite testuser - mc admin user rm testuser + mc admin user remove objectstorage testuser mc admin policy attach objectstorage userNormal --group userNormal mc admin policy attach objectstorage userDenyWrite --group userDenyWrite From 16f35d55337027cb205521efd564f4c296911d45 Mon Sep 17 00:00:00 2001 From: xuziyi Date: Tue, 30 Jan 2024 16:23:32 +0800 Subject: [PATCH 21/29] add app cr --- .../manifests/app/deploy.yaml.tmpl | 21 +++++++++++++++++++ deploy/objectstorage/scripts/app.sh | 12 +++++++++++ deploy/objectstorage/scripts/init.sh | 2 ++ 3 files changed, 35 insertions(+) create mode 100644 deploy/objectstorage/manifests/app/deploy.yaml.tmpl create mode 100644 deploy/objectstorage/scripts/app.sh diff --git a/deploy/objectstorage/manifests/app/deploy.yaml.tmpl b/deploy/objectstorage/manifests/app/deploy.yaml.tmpl new file mode 100644 index 00000000000..3baecf3bf4c --- /dev/null +++ b/deploy/objectstorage/manifests/app/deploy.yaml.tmpl @@ -0,0 +1,21 @@ +apiVersion: app.sealos.io/v1 +kind: App +metadata: + name: objectstorage + namespace: app-system +spec: + data: + desc: object storage + url: https://objectstorage.{{ .cloudDomain }}:443 + displayType: normal + i18n: + zh: + name: 对象存储 + zh-Hans: + name: 对象存储 + icon: https://objectstorage.{{ .cloudDomain }}:443/logo.svg + menuData: + helpDropDown: false + nameColor: text-black + name: Object Storage + type: iframe \ No newline at end of file diff --git a/deploy/objectstorage/scripts/app.sh b/deploy/objectstorage/scripts/app.sh new file mode 100644 index 00000000000..bc5f1ab8d2a --- /dev/null +++ b/deploy/objectstorage/scripts/app.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -e + +function deploy_app() { + kubectl apply -f manifests/app/deploy.yaml +} + +function install() { + deploy_app +} + +install \ No newline at end of file diff --git a/deploy/objectstorage/scripts/init.sh b/deploy/objectstorage/scripts/init.sh index 943086e63f0..7eea2a72341 100644 --- a/deploy/objectstorage/scripts/init.sh +++ b/deploy/objectstorage/scripts/init.sh @@ -12,3 +12,5 @@ sealos run ghcr.io/nowinkeyy/sealos-cloud-objectstorage-controller:latest -e clo sealos run ghcr.io/nowinkeyy/sealos-cloud-objectstorage-frontend:latest -e cloudDomain=${cloudDomain} # 5. run objectstorage monitor service sealos run ghcr.io/nowinkeyy/sealos-cloud-minio-service:latest -e cloudDomain=${cloudDomain} +# 6. create app cr +bash scripts/app.sh From a98ae590a4a2b63ae1ef425a6745bf1ec3b5386b Mon Sep 17 00:00:00 2001 From: xuziyi Date: Tue, 30 Jan 2024 17:41:59 +0800 Subject: [PATCH 22/29] fix prometheus deploy.yaml error --- .../manifests/prometheus/deploy.yaml.tmpl | 97 ++++--------------- 1 file changed, 19 insertions(+), 78 deletions(-) diff --git a/deploy/objectstorage/manifests/prometheus/deploy.yaml.tmpl b/deploy/objectstorage/manifests/prometheus/deploy.yaml.tmpl index 139a991f0c0..20fd072ca20 100644 --- a/deploy/objectstorage/manifests/prometheus/deploy.yaml.tmpl +++ b/deploy/objectstorage/manifests/prometheus/deploy.yaml.tmpl @@ -1,54 +1,3 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: object-storage-sa - namespace: objectstorage-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: object-storage-role - namespace: objectstorage-system -rules: - - verbs: - - get - - list - - watch - apiGroups: - - '' - resources: - - secrets - - verbs: - - create - - delete - - get - apiGroups: - - '' - resources: - - services - - verbs: - - get - - list - - watch - apiGroups: - - minio.min.io - resources: - - tenants ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: object-storage-rolebind - namespace: objectstorage-system -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: object-storage-role -subjects: - - kind: ServiceAccount - name: object-storage-sa - namespace: objectstorage-system ---- apiVersion: monitoring.coreos.com/v1 kind: Prometheus metadata: @@ -99,47 +48,40 @@ spec: storage: {{ .storageSize }} --- apiVersion: v1 +kind: Service +metadata: + name: prometheus-object-storage +spec: + ports: + - port: 9090 + targetPort: http-web + protocol: TCP + name: http-web + selector: + app: prometheus-object-storage + type: ClusterIP +--- +apiVersion: v1 kind: ServiceAccount metadata: name: object-storage-sa - namespace: objectstorage-system --- apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: name: object-storage-role - namespace: objectstorage-system rules: - - verbs: - - get - - list - - watch - apiGroups: - - '' - resources: - - secrets - - verbs: - - create - - delete - - get - apiGroups: - - '' + - apiGroups: + - "*" resources: - - services - - verbs: - - get - - list - - watch - apiGroups: - - minio.min.io - resources: - - tenants + - "*" + verbs: + - "*" --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: object-storage-rolebind - namespace: objectstorage-system roleRef: apiGroup: rbac.authorization.k8s.io kind: Role @@ -147,5 +89,4 @@ roleRef: subjects: - kind: ServiceAccount name: object-storage-sa - namespace: objectstorage-system From 0c02748989614b4027524547d863565cfdcd67a4 Mon Sep 17 00:00:00 2001 From: xuziyi Date: Wed, 31 Jan 2024 13:33:53 +0800 Subject: [PATCH 23/29] fix 1 --- deploy/objectstorage/Kubefile | 3 ++- deploy/objectstorage/images/shim/imageList | 5 +---- deploy/objectstorage/manifests/minio/deploy.yaml.tmpl | 2 +- deploy/objectstorage/manifests/prometheus/deploy.yaml.tmpl | 4 ++-- deploy/objectstorage/scripts/init.sh | 2 -- 5 files changed, 6 insertions(+), 10 deletions(-) diff --git a/deploy/objectstorage/Kubefile b/deploy/objectstorage/Kubefile index eb3411953ce..1583b0fc878 100644 --- a/deploy/objectstorage/Kubefile +++ b/deploy/objectstorage/Kubefile @@ -6,7 +6,8 @@ COPY manifests manifests ENV cloudDomain=${cloudDomain:-"127.0.0.1.nip.io"} ENV cloudPort="" -ENV storageSize=${storageSize:-1Gi} +ENV minioStorageSize=${minioStorageSize:-1Gi} +ENV promStorageSize=${promStorageSize:-1Gi} ENV minioAdminUser=${minioAdminUser:-"username"} ENV minioAdminPassword=${minioAdminPassword:-"passw0rd"} diff --git a/deploy/objectstorage/images/shim/imageList b/deploy/objectstorage/images/shim/imageList index 306118a0a47..7f78d7bfb15 100644 --- a/deploy/objectstorage/images/shim/imageList +++ b/deploy/objectstorage/images/shim/imageList @@ -1,5 +1,2 @@ quay.io/prometheus/prometheus:v2.45.0 -quay.io/minio/minio -ghcr.io/nowinkeyy/sealos-cloud-objectstorage-controller:latest -ghcr.io/nowinkeyy/sealos-cloud-objectstorage-frontend:latest -ghcr.io/nowinkeyy/sealos-cloud-minio-service:latest \ No newline at end of file +quay.io/minio/minio:RELEASE.2023-11-20T22-40-07Z \ No newline at end of file diff --git a/deploy/objectstorage/manifests/minio/deploy.yaml.tmpl b/deploy/objectstorage/manifests/minio/deploy.yaml.tmpl index fb76f240e5e..1ad42d6cee3 100644 --- a/deploy/objectstorage/manifests/minio/deploy.yaml.tmpl +++ b/deploy/objectstorage/manifests/minio/deploy.yaml.tmpl @@ -76,7 +76,7 @@ spec: - ReadWriteOnce resources: requests: - storage: {{ .storageSize }} + storage: {{ .minioStorageSize }} status: {} volumesPerServer: 1 requestAutoCert: false diff --git a/deploy/objectstorage/manifests/prometheus/deploy.yaml.tmpl b/deploy/objectstorage/manifests/prometheus/deploy.yaml.tmpl index 20fd072ca20..4e4dfc4171b 100644 --- a/deploy/objectstorage/manifests/prometheus/deploy.yaml.tmpl +++ b/deploy/objectstorage/manifests/prometheus/deploy.yaml.tmpl @@ -39,13 +39,13 @@ spec: metadata: annotations: path: /prometheus - value: {{ .storageSize }} + value: {{ .promStorageSize }} spec: accessModes: - ReadWriteOnce resources: requests: - storage: {{ .storageSize }} + storage: {{ .promStorageSize }} --- apiVersion: v1 kind: Service diff --git a/deploy/objectstorage/scripts/init.sh b/deploy/objectstorage/scripts/init.sh index 7eea2a72341..b4819cfc51f 100644 --- a/deploy/objectstorage/scripts/init.sh +++ b/deploy/objectstorage/scripts/init.sh @@ -1,7 +1,5 @@ #!/bin/bash -# 0.create minio-operator -sealos run docker.io/nowinkey/minio-operator:v5.0.6 # 1. create minio instance bash scripts/minio.sh # 2. create prometheus instance From 8fcbdf00e3b16ccad7132b0e0218c4791408d063 Mon Sep 17 00:00:00 2001 From: xuziyi Date: Wed, 31 Jan 2024 17:09:38 +0800 Subject: [PATCH 24/29] delete app cr and update images --- deploy/objectstorage/init.sh | 12 +++++------ .../manifests/app/deploy.yaml.tmpl | 21 ------------------- deploy/objectstorage/scripts/app.sh | 12 ----------- deploy/objectstorage/scripts/init.sh | 8 +++---- 4 files changed, 9 insertions(+), 44 deletions(-) delete mode 100644 deploy/objectstorage/manifests/app/deploy.yaml.tmpl delete mode 100644 deploy/objectstorage/scripts/app.sh diff --git a/deploy/objectstorage/init.sh b/deploy/objectstorage/init.sh index 4d571cbb748..d0b27b91c54 100644 --- a/deploy/objectstorage/init.sh +++ b/deploy/objectstorage/init.sh @@ -24,10 +24,10 @@ retryPullImage() { fi } -retryPullImage ghcr.io/nowinkeyy/sealos-cloud-objectstorage-controller:latest -retryPullImage ghcr.io/nowinkeyy/sealos-cloud-objectstorage-frontend:latest -retryPullImage ghcr.io/nowinkeyy/sealos-cloud-minio-service:latest +retryPullImage ghcr.io/labring/sealos-cloud-objectstorage-controller:latest +retryPullImage ghcr.io/labring/sealos-cloud-objectstorage-frontend:latest +retryPullImage ghcr.io/labring/sealos-cloud-minio-service:latest -sealos save -o tars/objectstorage-controller.tar ghcr.io/nowinkeyy/sealos-cloud-objectstorage-controller:latest -sealos save -o tars/objectstorage-frontend.tar ghcr.io/nowinkeyy/sealos-cloud-objectstorage-frontend:latest -sealos save -o tars/objectstorage-service.tar ghcr.io/nowinkeyy/sealos-cloud-minio-service:latest +sealos save -o tars/objectstorage-controller.tar ghcr.io/labring/sealos-cloud-objectstorage-controller:latest +sealos save -o tars/objectstorage-frontend.tar ghcr.io/labring/sealos-cloud-objectstorage-frontend:latest +sealos save -o tars/objectstorage-service.tar ghcr.io/labring/sealos-cloud-minio-service:latest diff --git a/deploy/objectstorage/manifests/app/deploy.yaml.tmpl b/deploy/objectstorage/manifests/app/deploy.yaml.tmpl deleted file mode 100644 index 3baecf3bf4c..00000000000 --- a/deploy/objectstorage/manifests/app/deploy.yaml.tmpl +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: app.sealos.io/v1 -kind: App -metadata: - name: objectstorage - namespace: app-system -spec: - data: - desc: object storage - url: https://objectstorage.{{ .cloudDomain }}:443 - displayType: normal - i18n: - zh: - name: 对象存储 - zh-Hans: - name: 对象存储 - icon: https://objectstorage.{{ .cloudDomain }}:443/logo.svg - menuData: - helpDropDown: false - nameColor: text-black - name: Object Storage - type: iframe \ No newline at end of file diff --git a/deploy/objectstorage/scripts/app.sh b/deploy/objectstorage/scripts/app.sh deleted file mode 100644 index bc5f1ab8d2a..00000000000 --- a/deploy/objectstorage/scripts/app.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash -set -e - -function deploy_app() { - kubectl apply -f manifests/app/deploy.yaml -} - -function install() { - deploy_app -} - -install \ No newline at end of file diff --git a/deploy/objectstorage/scripts/init.sh b/deploy/objectstorage/scripts/init.sh index b4819cfc51f..920926529c2 100644 --- a/deploy/objectstorage/scripts/init.sh +++ b/deploy/objectstorage/scripts/init.sh @@ -5,10 +5,8 @@ bash scripts/minio.sh # 2. create prometheus instance bash scripts/prometheus.sh # 3. run objectstorage controller -sealos run ghcr.io/nowinkeyy/sealos-cloud-objectstorage-controller:latest -e cloudDomain=${cloudDomain} +sealos run ghcr.io/labring/sealos-cloud-objectstorage-controller:latest -e cloudDomain=${cloudDomain} # 4. run objectstorage frontend -sealos run ghcr.io/nowinkeyy/sealos-cloud-objectstorage-frontend:latest -e cloudDomain=${cloudDomain} +sealos run ghcr.io/labring/sealos-cloud-objectstorage-frontend:latest -e cloudDomain=${cloudDomain} # 5. run objectstorage monitor service -sealos run ghcr.io/nowinkeyy/sealos-cloud-minio-service:latest -e cloudDomain=${cloudDomain} -# 6. create app cr -bash scripts/app.sh +sealos run ghcr.io/labring/sealos-cloud-minio-service:latest -e cloudDomain=${cloudDomain} From be27857926270c7524d8fbe4b15372ac1ac30f60 Mon Sep 17 00:00:00 2001 From: xuziyi Date: Thu, 1 Feb 2024 18:51:33 +0800 Subject: [PATCH 25/29] add ns --- deploy/objectstorage/manifests/prometheus/deploy.yaml.tmpl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/deploy/objectstorage/manifests/prometheus/deploy.yaml.tmpl b/deploy/objectstorage/manifests/prometheus/deploy.yaml.tmpl index 4e4dfc4171b..6cb6e0db89e 100644 --- a/deploy/objectstorage/manifests/prometheus/deploy.yaml.tmpl +++ b/deploy/objectstorage/manifests/prometheus/deploy.yaml.tmpl @@ -51,6 +51,7 @@ apiVersion: v1 kind: Service metadata: name: prometheus-object-storage + namespace: objectstorage-system spec: ports: - port: 9090 @@ -65,11 +66,13 @@ apiVersion: v1 kind: ServiceAccount metadata: name: object-storage-sa + namespace: objectstorage-system --- apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: name: object-storage-role + namespace: objectstorage-system rules: - apiGroups: - "*" @@ -82,6 +85,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: object-storage-rolebind + namespace: objectstorage-system roleRef: apiGroup: rbac.authorization.k8s.io kind: Role @@ -89,4 +93,5 @@ roleRef: subjects: - kind: ServiceAccount name: object-storage-sa + namespace: objectstorage-system From 12e691cec4e016fb300ff3d22f523f23d567e1e1 Mon Sep 17 00:00:00 2001 From: xuziyi Date: Wed, 21 Feb 2024 16:22:02 +0800 Subject: [PATCH 26/29] fix --- deploy/objectstorage/images/shim/imageList | 2 +- deploy/objectstorage/manifests/minio/deploy.yaml.tmpl | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/deploy/objectstorage/images/shim/imageList b/deploy/objectstorage/images/shim/imageList index 7f78d7bfb15..3ee611a7073 100644 --- a/deploy/objectstorage/images/shim/imageList +++ b/deploy/objectstorage/images/shim/imageList @@ -1,2 +1,2 @@ quay.io/prometheus/prometheus:v2.45.0 -quay.io/minio/minio:RELEASE.2023-11-20T22-40-07Z \ No newline at end of file +minio/minio:RELEASE.2023-11-11T08-14-41Z \ No newline at end of file diff --git a/deploy/objectstorage/manifests/minio/deploy.yaml.tmpl b/deploy/objectstorage/manifests/minio/deploy.yaml.tmpl index 1ad42d6cee3..1c206d6140d 100644 --- a/deploy/objectstorage/manifests/minio/deploy.yaml.tmpl +++ b/deploy/objectstorage/manifests/minio/deploy.yaml.tmpl @@ -54,7 +54,7 @@ spec: console: true minio: true features: {} - image: quay.io/minio/minio + image: minio/minio:RELEASE.2023-11-11T08-14-41Z imagePullSecret: {} mountPath: /export pools: @@ -90,6 +90,8 @@ kind: Service metadata: name: object-storage namespace: objectstorage-system + labels: + v1.min.io/tenant: object-storage spec: ports: - name: http-minio From 88af169d1bbce56b4510758577356fa9e6d99691 Mon Sep 17 00:00:00 2001 From: xuziyi Date: Mon, 26 Feb 2024 14:20:41 +0800 Subject: [PATCH 27/29] change admin username to admin --- deploy/objectstorage/Kubefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/objectstorage/Kubefile b/deploy/objectstorage/Kubefile index 1583b0fc878..5e8ef94547a 100644 --- a/deploy/objectstorage/Kubefile +++ b/deploy/objectstorage/Kubefile @@ -8,7 +8,7 @@ ENV cloudDomain=${cloudDomain:-"127.0.0.1.nip.io"} ENV cloudPort="" ENV minioStorageSize=${minioStorageSize:-1Gi} ENV promStorageSize=${promStorageSize:-1Gi} -ENV minioAdminUser=${minioAdminUser:-"username"} +ENV minioAdminUser=${minioAdminUser:-"admin"} ENV minioAdminPassword=${minioAdminPassword:-"passw0rd"} CMD ["bash scripts/init.sh"] From 23701db1699effde0e0775c7ea3addbedf501281 Mon Sep 17 00:00:00 2001 From: xuziyi Date: Mon, 26 Feb 2024 14:27:51 +0800 Subject: [PATCH 28/29] refine console ingress --- deploy/objectstorage/manifests/minio/deploy.yaml.tmpl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/deploy/objectstorage/manifests/minio/deploy.yaml.tmpl b/deploy/objectstorage/manifests/minio/deploy.yaml.tmpl index 1c206d6140d..b98eb33c778 100644 --- a/deploy/objectstorage/manifests/minio/deploy.yaml.tmpl +++ b/deploy/objectstorage/manifests/minio/deploy.yaml.tmpl @@ -160,7 +160,6 @@ metadata: large_client_header_buffers 4 128k; nginx.ingress.kubernetes.io/ssl-redirect: 'false' nginx.ingress.kubernetes.io/backend-protocol: HTTP - nginx.ingress.kubernetes.io/rewrite-target: /$2 nginx.ingress.kubernetes.io/client-body-buffer-size: 64k nginx.ingress.kubernetes.io/proxy-buffer-size: 64k nginx.ingress.kubernetes.io/configuration-snippet: | @@ -174,7 +173,7 @@ spec: http: paths: - pathType: Prefix - path: /()(.*) + path: / backend: service: name: object-storage-console From c7d0122bc3cc3251e0b03164e4385899f21f4a1c Mon Sep 17 00:00:00 2001 From: xuziyi Date: Mon, 26 Feb 2024 15:55:56 +0800 Subject: [PATCH 29/29] run tars/xxx.tar --- deploy/objectstorage/scripts/init.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/deploy/objectstorage/scripts/init.sh b/deploy/objectstorage/scripts/init.sh index 920926529c2..30a751dc66f 100644 --- a/deploy/objectstorage/scripts/init.sh +++ b/deploy/objectstorage/scripts/init.sh @@ -5,8 +5,8 @@ bash scripts/minio.sh # 2. create prometheus instance bash scripts/prometheus.sh # 3. run objectstorage controller -sealos run ghcr.io/labring/sealos-cloud-objectstorage-controller:latest -e cloudDomain=${cloudDomain} +sealos run tars/objectstorage-controller.tar -e cloudDomain=${cloudDomain} # 4. run objectstorage frontend -sealos run ghcr.io/labring/sealos-cloud-objectstorage-frontend:latest -e cloudDomain=${cloudDomain} +sealos run tars/objectstorage-frontend.tar -e cloudDomain=${cloudDomain} # 5. run objectstorage monitor service -sealos run ghcr.io/labring/sealos-cloud-minio-service:latest -e cloudDomain=${cloudDomain} +sealos run tars/objectstorage-service.tar -e cloudDomain=${cloudDomain}