OpenMined · yashgorana · Apr 11, 2024 · Apr 3, 2024 · Apr 3, 2024 · Apr 3, 2024
diff --git a/packages/grid/backend/grid/core/node.py b/packages/grid/backend/grid/core/node.py
@@ -4,6 +4,7 @@
 from syft.node.domain import Node
 from syft.node.enclave import Enclave
 from syft.node.gateway import Gateway
+from syft.node.node import get_default_bucket_name
 from syft.node.node import get_enable_warnings
 from syft.node.node import get_node_name
 from syft.node.node import get_node_side_type
@@ -61,7 +62,7 @@ def seaweedfs_config() -> SeaweedFSConfig:
         access_key=settings.S3_ROOT_USER,
         secret_key=settings.S3_ROOT_PWD,
         region=settings.S3_REGION,
-        default_bucket_name=get_node_uid_env(),
+        default_bucket_name=get_default_bucket_name(),
         mount_port=settings.SEAWEED_MOUNT_PORT,
     )
 

diff --git a/packages/grid/default.env b/packages/grid/default.env
@@ -63,13 +63,11 @@ BACKEND_API_BASE_URL="/api/v2"
 
 # SeaweedFS
 S3_ENDPOINT="seaweedfs"
-S3_PORT=8333
 S3_ROOT_USER="admin"
 S3_ROOT_PWD="admin" # needs randomizing
 S3_REGION="us-east-1"
  #not-using
 S3_PRESIGNED_TIMEOUT_SECS=1800
-S3_VOLUME_SIZE_MB=1024
 
 
 # Jax

diff --git a/packages/grid/docker-compose.yml b/packages/grid/docker-compose.yml
@@ -246,11 +246,10 @@ services:
       - .env
     image: "${DOCKER_IMAGE_SEAWEEDFS?Variable not set}:${VERSION-latest}"
     environment:
-      - S3_VOLUME_SIZE_MB=${S3_VOLUME_SIZE_MB:-1024}
+      - SWFS_VOLUME_SIZE_LIMIT_MB=${SWFS_VOLUME_SIZE_LIMIT_MB:-1000}
       - S3_ROOT_USER=${S3_ROOT_USER:-admin}
       - S3_ROOT_PWD=${S3_ROOT_PWD:-admin}
-      - S3_PORT=${S3_PORT:-8888}
-      - SEAWEED_MOUNT_PORT=${SEAWEED_MOUNT_PORT:-4001}
+      - MOUNT_API_PORT=${MOUNT_API_PORT:-4001}
     volumes:
       - seaweedfs-data:/data
       - ./seaweedfs/filer.toml:/etc/seaweedfs/filer.toml

diff --git a/packages/grid/helm/syft/templates/backend/backend-statefulset.yaml b/packages/grid/helm/syft/templates/backend/backend-statefulset.yaml
@@ -64,6 +64,10 @@ spec:
               value: "{{ .Values.global.registry }}/openmined/grid-backend:{{ .Values.global.version }}"
             - name: DEFAULT_WORKER_POOL_COUNT
               value: {{ .Values.node.defaultWorkerPoolCount | quote }}
+            {{- if .Values.node.defaultBucketName }}
+            - name: DEFAULT_BUCKET_NAME
+              value: {{ .Values.node.defaultBucketName | quote }}
+            {{- end }}
             {{- if .Values.node.debuggerEnabled }}
             - name: DEBUGGER_ENABLED
               value: "true"
@@ -100,8 +104,6 @@ spec:
                 secretKeyRef:
                     name: {{ .Values.seaweedfs.secretKeyName | required "seaweedfs.secretKeyName is required" }}
                     key: s3RootPassword
-            - name: S3_PORT
-              value: {{ .Values.seaweedfs.s3.port | quote }}
             - name: SEAWEED_MOUNT_PORT
               value: {{ .Values.seaweedfs.mountApi.port | quote }}
             {{- end }}

diff --git a/packages/grid/helm/syft/templates/seaweedfs/seaweedfs-configmap.yaml b/packages/grid/helm/syft/templates/seaweedfs/seaweedfs-configmap.yaml
@@ -7,15 +7,25 @@ metadata:
     {{- include "common.labels" . | nindent 4 }}
     app.kubernetes.io/component: seaweedfs
 data:
+  {{- if .Values.seaweedfs.master.config }}
+  master.toml: |
+    # User provisioned master config
+    {{ .Values.seaweedfs.master.config | nindent 4 }}
+    # End master config
+  {{- end }}
+
+  {{- if .Values.seaweedfs.filer.config }}
   filer.toml: |
-    [leveldb2]
-    enabled = true
-    dir = "./filerldb2"
-  start.sh: |
-    #! /usr/bin/env bash
+    # User provisioned filer config
+    {{ .Values.seaweedfs.filer.config | nindent 4 }}
+    # End filer config
+  {{- end }}
+
+  {{- if .Values.seaweedfs.mountApi.autoMountConfig }}
+  automount.yaml: |
+    # User provisioned automount config
+    {{ .Values.seaweedfs.mountApi.autoMountConfig | nindent 4 }}
+    # End automount config
+  {{- end }}
 
-    sleep 30 &&
-    echo "s3.configure -access_key ${S3_ROOT_USER} -secret_key ${S3_ROOT_PWD} -user iam -actions Read,Write,List,Tagging,Admin -apply" \
-    | weed shell > /dev/null 2>&1 \
-    & weed server -s3 -s3.port=${S3_PORT} -master.volumeSizeLimitMB=${S3_VOLUME_SIZE_MB}
 {{ end }}
diff --git a/packages/grid/helm/syft/templates/seaweedfs/seaweedfs-mount-secret.yaml b/packages/grid/helm/syft/templates/seaweedfs/seaweedfs-mount-secret.yaml
@@ -0,0 +1,18 @@
+{{- if and (ne .Values.node.type "gateway") (.Values.seaweedfs.secret.mountCredentials) }}
+{{- $secretName := "seaweedfs-mount-secret" }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ $secretName }}
+  labels:
+    {{- include "common.labels" . | nindent 4 }}
+    app.kubernetes.io/component: seaweedfs
+type: Opaque
+data:
+  # we don't need to call "common.secrets.set" because:
+  # 1. secrets are not generated randomly
+  # 2. secrets can expire and might require a force update
+  {{- range $obj := $.Values.seaweedfs.secret.mountCredentials }}
+  {{ $obj.name }}: {{ $obj.value | b64enc }}
+  {{- end }}
+{{ end }}
diff --git a/packages/grid/helm/syft/templates/seaweedfs/seaweedfs-secret.yaml b/packages/grid/helm/syft/templates/seaweedfs/seaweedfs-secret.yaml
@@ -1,4 +1,4 @@
-{{- if ne .Values.node.type "gateway"}}
+{{- if ne .Values.node.type "gateway" }}
 {{- $secretName := "seaweedfs-secret" }}
 apiVersion: v1
 kind: Secret

diff --git a/packages/grid/helm/syft/templates/seaweedfs/seaweedfs-service.yaml b/packages/grid/helm/syft/templates/seaweedfs/seaweedfs-service.yaml
@@ -12,11 +12,15 @@ spec:
     {{- include "common.selectorLabels" . | nindent 4 }}
     app.kubernetes.io/component: seaweedfs
   ports:
-    - name: filer
+    - name: master-api
+      protocol: TCP
+      port: 9333
+      targetPort: 9333
+    - name: filer-ui
       protocol: TCP
       port: 8888
       targetPort: 8888
-    - name: s3
+    - name: s3-api
       protocol: TCP
       port: 8333
       targetPort: 8333

diff --git a/packages/grid/helm/syft/templates/seaweedfs/seaweedfs-statefulset.yaml b/packages/grid/helm/syft/templates/seaweedfs/seaweedfs-statefulset.yaml
@@ -28,40 +28,57 @@ spec:
           imagePullPolicy: Always
           resources: {{ include "common.resources.set" (dict "resources" .Values.seaweedfs.resources "preset" .Values.seaweedfs.resourcesPreset) | nindent 12 }}
           env:
-            - name: S3_VOLUME_SIZE_MB
-              value: {{ .Values.seaweedfs.s3.volumeSizeMB | quote }}
+            - name: SWFS_VOLUME_SIZE_LIMIT_MB
+              value: {{ .Values.seaweedfs.master.volumeSizeMB | quote }}
             - name: S3_ROOT_USER
               value: {{ .Values.seaweedfs.s3.rootUser | quote }}
             - name: S3_ROOT_PWD
               valueFrom:
                 secretKeyRef:
                     name: {{ .Values.seaweedfs.secretKeyName | required "seaweedfs.secretKeyName is required" }}
                     key: s3RootPassword
-            - name: S3_PORT
-              value: {{ .Values.seaweedfs.s3.port | quote }}
-            - name: SEAWEED_MOUNT_PORT
+            - name: MOUNT_API_PORT
               value: {{ .Values.seaweedfs.mountApi.port | quote }}
             {{- if .Values.seaweedfs.env }}
             {{- toYaml .Values.seaweedfs.env | nindent 12 }}
             {{- end }}
           volumeMounts:
-            - mountPath: /etc/seaweedfs/filer.toml
-              name: seaweedfs-config
+            # master + filer + volume data
+            - mountPath: /data/
+              name: seaweedfs-data
               readOnly: false
+            {{- if .Values.seaweedfs.filer.config -}}
+            # filer conf
+            - mountPath: /root/swfs/filer.toml
               subPath: filer.toml
-            - mountPath: /etc/seaweedfs/start.sh
               name: seaweedfs-config
               readOnly: false
-              subPath: start.sh
-            - mountPath: /data/blob
-              name: seaweedfs-data
+            {{- end }}
+            {{- if .Values.seaweedfs.master.config -}}
+            # filer conf
+            - mountPath: /root/swfs/master.toml
+              subPath: master.toml
+              name: seaweedfs-config
               readOnly: false
-              subPath: ''
+            {{- end }}
+            {{- if .Values.seaweedfs.mountApi.autoMountConfig -}}
+            # automount config
+            - mountPath: /root/swfs/automount.yaml
+              subPath: automount.yaml
+              name: seaweedfs-config
+              readOnly: false
+            {{- end }}
+            {{- if .Values.seaweedfs.volumeMounts }}
+            {{- toYaml .Values.seaweedfs.volumeMounts | nindent 12 }}
+            {{- end }}
       terminationGracePeriodSeconds: 5
       volumes:
-        - configMap:
+        - name: seaweedfs-config
+          configMap:
             name: seaweedfs-config
-          name: seaweedfs-config
+        {{- if .Values.seaweedfs.volumes }}
+        {{- toYaml .Values.seaweedfs.volumes | nindent 8 }}
+        {{- end }}
   volumeClaimTemplates:
   - metadata:
       name: seaweedfs-data

diff --git a/packages/grid/helm/syft/values.yaml b/packages/grid/helm/syft/values.yaml
@@ -37,34 +37,51 @@ frontend:
   resources: null
 
 seaweedfs:
-  # S3 settings
+  # Master
+  master:
+    # Seaweed Volume Size in MB set on master. This is not PVC.
+    volumeSizeMB: 1000
+    # master.toml For details check weed scaffold -config=master
+    config: null
+
+  # filer
+  filer:
+    # filer.toml For details check weed scaffold -config=filer
+    config: null
+
+  # S3
   s3:
-    port: 8333
     rootUser: admin
-    volumeSizeMB: 1024
 
-  # Mount API settings
+  # Mount API
   mountApi:
     port: 4001
+    # automount.yaml
+    autoMountConfig: null
 
   # Extra environment vars
   env: null
 
   # PVC storage size
-  storageSize: 5Gi
+  storageSize: 10Gi
 
   # Seaweed secret name. Override this if you want to use a self-managed secret.
   # Secret must contain the following keys:
   # - s3RootPassword
   secretKeyName: seaweedfs-secret
 
   # Pod Resource Limits
-  resourcesPreset: nano
+  resourcesPreset: medium
   resources: null
 
+  # extra volume mounts
+  volumes: null
+  volumeMounts: null
+
   # custom secret values
   secret:
     s3RootPassword: null
+    mountCredentials: null
 
 proxy:
   # Extra environment vars
@@ -79,7 +96,7 @@ registry:
   env: null
 
   # PVC storage size
-  storageSize: 5Gi
+  storageSize: 10Gi
 
   # Pod Resource Limits
   resourcesPreset: nano
@@ -93,6 +110,7 @@ node:
   side: high
   inMemoryWorkers: false
   defaultWorkerPoolCount: 1
+  defaultBucketName: null
   queuePort: 5556
   logLevel: info
   debuggerEnabled: false
@@ -114,7 +132,7 @@ node:
   env: null
 
   # Pod Resource Limits
-  resourcesPreset: small
+  resourcesPreset: medium
   resources: null
 
   # Seaweed secret name. Override this if you want to use a self-managed secret.

diff --git a/packages/grid/podman/podman-kube/podman-syft-kube-config.yaml b/packages/grid/podman/podman-kube/podman-syft-kube-config.yaml
@@ -65,7 +65,6 @@ data:
   S3_ROOT_PWD: "admin"
   S3_REGION: "us-east-1"
   S3_PRESIGNED_TIMEOUT_SECS: 1800
-  S3_VOLUME_SIZE_MB: 1024
 
   # Jax
   JAX_ENABLE_X64: True

diff --git a/packages/grid/seaweedfs/app.py b/packages/grid/seaweedfs/app.py
diff --git a/packages/grid/seaweedfs/config/automount.yaml b/packages/grid/seaweedfs/config/automount.yaml
@@ -0,0 +1,43 @@
+# Auto mount remote buckets on seaweed startup
+# Example:
+
+# mounts:
+#   - local_bucket: gcs-test-bucket
+#     remote_bucket:
+#       type: gcs
+#       bucket_name: test-bucket
+#       creds: /tmp/test-bucket-gcs-credentials.json
+#       # ------- test-bucket-gcs-credentials.json -------
+#       # {
+#       #   "type": "service_account",
+#       #   "project_id": "...",
+#       #   ...
+#       #   "universe_domain": "googleapis.com",
+#       # }
+#       # ------------------------------------------------
+
+#   - local_bucket: s3-test-bucket
+#     remote_bucket:
+#       type: s3
+#       bucket_name: test-bucket
+#       creds: /tmp/test-bucket-s3-credentials.json
+#       # ------- test-bucket-azure-credentials.json -------
+#       # {
+#       #   "aws_access_key_id": "example"
+#       #   "aws_secret_access_key": "example"
+#       # }
+#       # --------------------------------------------------
+
+#   - local_bucket: azure-test-bucket
+#     remote_bucket:
+#       type: azure
+#       bucket_name: test-bucket
+#       creds: /tmp/test-bucket-azure-credentials.json
+#       # ------- test-bucket-azure-credentials.json -------
+#       # {
+#       #   "azure_account_name": "example"
+#       #   "azure_account_key": "example"
+#       # }
+#       # --------------------------------------------------
+
+mounts: []
diff --git a/packages/grid/seaweedfs/config/filer.toml b/packages/grid/seaweedfs/config/filer.toml
@@ -0,0 +1,9 @@
+[leveldb2]
+enabled = false
+dir = "/data/filerldb2"
+
+[leveldb3]
+# similar to leveldb2.
+# each bucket has its own meta store.
+enabled = true
+dir = "/data/filerldb3"
diff --git a/packages/grid/seaweedfs/config/master.toml b/packages/grid/seaweedfs/config/master.toml
@@ -0,0 +1,2 @@
+[master.volume_growth]
+copy_1 = 5