feat: onyx clusters

Taskfile.yml

@@ -65,3 +65,5 @@ tasks:
           CHART_NAME: vcluster
       - >-
         echo "sulfoxide-bromine: $(skopeo list-tags docker://ghcr.io/atomicloud/sulfoxide.bromine/sulfoxide-bromine | jq -r '.Tags[]' | sort -V | tail -n 1)"
+      - >-
+        echo "ectd: $(skopeo list-tags docker://registry-1.docker.io/bitnamicharts/etcd | jq -r '.Tags[]' | sort -V | tail -n 1)"

chart/Chart.lock

@@ -1,9 +1,12 @@
 dependencies:
 - name: vcluster
   repository: https://charts.loft.sh
-  version: 0.16.4
+  version: 0.18.1
 - name: sulfoxide-bromine
   repository: oci://ghcr.io/atomicloud/sulfoxide.bromine
   version: 1.2.3
-digest: sha256:99b16ceb516f4d4c0c2adc361b09aeb813374f898101002682470878f6d94923
-generated: "2023-10-30T10:10:16.308216+08:00"
+- name: etcd
+  repository: oci://registry-1.docker.io/bitnamicharts
+  version: 9.10.5
+digest: sha256:3bcdad6f9965955280ab5113f5711fcc335c7c1ca81b6fcb5b58c018c6e37c9b
+generated: "2024-02-06T20:59:04.025595+08:00"

chart/Chart.yaml

@@ -3,11 +3,15 @@ name: sulfoxide-iodine
 description: Helm chart to install virtual cluster on a physical cluster
 type: application
 version: 1.15.4
-appVersion: "v0.16.3"
+appVersion: "v0.18.1"
 dependencies:
   - name: vcluster
-    version: v0.16.4
+    version: v0.18.1
     repository: https://charts.loft.sh
   - name: sulfoxide-bromine
-    version: 1.2.3
+    version: 1.3.0
     repository: oci://ghcr.io/atomicloud/sulfoxide.bromine
+  - name: etcd
+    repository: oci://registry-1.docker.io/bitnamicharts
+    version: 9.10.5
+    condition: etcd.enabled

chart/README.md

@@ -1,15 +1,16 @@
 # sulfoxide-iodine
 
-![Version: 1.15.4](https://img.shields.io/badge/Version-1.15.4-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v0.16.3](https://img.shields.io/badge/AppVersion-v0.16.3-informational?style=flat-square)
+![Version: 1.15.4](https://img.shields.io/badge/Version-1.15.4-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v0.18.1](https://img.shields.io/badge/AppVersion-v0.18.1-informational?style=flat-square)
 
 Helm chart to install virtual cluster on a physical cluster
 
 ## Requirements
 
 | Repository | Name | Version |
 |------------|------|---------|
-| https://charts.loft.sh | vcluster | v0.16.4 |
-| oci://ghcr.io/atomicloud/sulfoxide.bromine | sulfoxide-bromine | 1.2.3 |
+| https://charts.loft.sh | vcluster | v0.18.1 |
+| oci://ghcr.io/atomicloud/sulfoxide.bromine | sulfoxide-bromine | 1.3.0 |
+| oci://registry-1.docker.io/bitnamicharts | etcd | 9.10.5 |
 
 ## Values
 
@@ -26,7 +27,8 @@ Helm chart to install virtual cluster on a physical cluster
 | auth.secretStore.kind | string | `"ClusterSecretStore"` | kind of the secret store to reference |
 | auth.secretStore.name | string | `"doppler"` | name of the secret store to reference |
 | auth.upsyncNamespace | string | `"sulfoxide"` | upsync namespace |
-| datastore | object | `{"name":"datastore-endpoint","policy":{"creation":"Owner","deletion":"Retain"},"refreshInterval":"1h","remoteName":"PICHU_OPAL_K3S_DATASTORE_ENDPOINT","secretKey":"K3S_DATASTORE_ENDPOINT","secretStore":{"kind":"SecretStore","name":"doppler-iodine"}}` | K3S state (postgresql) auth |
+| datastore | object | `{"enable":true,"name":"datastore-endpoint","policy":{"creation":"Owner","deletion":"Retain"},"refreshInterval":"1h","remoteName":"PICHU_OPAL_K3S_DATASTORE_ENDPOINT","secretKey":"K3S_DATASTORE_ENDPOINT","secretStore":{"kind":"SecretStore","name":"doppler-iodine"}}` | K3S state (postgresql) auth |
+| datastore.enable | bool | `true` | Attempt to obtain datastore's connection string |
 | datastore.name | string | `"datastore-endpoint"` | name of the secret to be created |
 | datastore.policy.creation | string | `"Owner"` | External Secret creation policy |
 | datastore.policy.deletion | string | `"Retain"` | External Secret deletion policy |
@@ -47,6 +49,7 @@ Helm chart to install virtual cluster on a physical cluster
 | datastoreCa.secretStore | object | `{"kind":"SecretStore","name":"doppler-iodine"}` | Secret store to reference |
 | datastoreCa.secretStore.kind | string | `"SecretStore"` | kind of the secret store to reference |
 | datastoreCa.secretStore.name | string | `"doppler-iodine"` | name of the secret store to reference |
+| etcd | object | `{"commonAnnotations":{"<<":{"atomi.cloud/layer":"1","atomi.cloud/platform":"sulfoxide","atomi.cloud/service":"iodine"},"atomi.cloud/module":"etcd"},"commonLabels":{"<<":{"atomi.cloud/layer":"1","atomi.cloud/platform":"sulfoxide","atomi.cloud/service":"iodine"},"atomi.cloud/module":"etcd"},"persistence":{"enabled":false},"podAnnotations":{"<<":{"atomi.cloud/layer":"1","atomi.cloud/platform":"sulfoxide","atomi.cloud/service":"iodine"},"atomi.cloud/module":"etcd"},"podLabels":{"<<":{"atomi.cloud/layer":"1","atomi.cloud/platform":"sulfoxide","atomi.cloud/service":"iodine"},"atomi.cloud/module":"etcd"},"replicaCount":3,"topologySpreadConstraints":[{"labelSelector":{"matchLabels":{"<<":{"atomi.cloud/layer":"1","atomi.cloud/platform":"sulfoxide","atomi.cloud/service":"iodine"},"atomi.cloud/module":"etcd"}},"maxSkew":1,"topologyKey":"topology.kubernetes.io/zone","whenUnsatisfiable":"ScheduleAnyway"}]}` | ETCD Cluster configuration. See [etcd documentation](https://artifacthub.io/packages/helm/bitnami/etcd) |
 | k3sSyncToken | object | `{"name":"k3s-sync-token","policy":{"creation":"Owner","deletion":"Retain"},"refreshInterval":"1h","remoteName":"PIKACHU_K3S_TOKEN","secretKey":"K3S_TOKEN","secretStore":{"kind":"SecretStore","name":"doppler-iodine"}}` | K3S sync token |
 | k3sSyncToken.name | string | `"k3s-sync-token"` | name of the secret to be created |
 | k3sSyncToken.policy.creation | string | `"Owner"` | External Secret creation policy |
@@ -58,13 +61,13 @@ Helm chart to install virtual cluster on a physical cluster
 | k3sSyncToken.secretStore.kind | string | `"SecretStore"` | kind of the secret store to reference |
 | k3sSyncToken.secretStore.name | string | `"doppler-iodine"` | name of the secret store to reference |
 | secretAnnotation | object | `{"argocd.argoproj.io/sync-wave":"-3"}` | Secret Annotations (External Secrets) to control synchronization |
-| serviceTree | object | `{"layer":"1","module":"apiserver","platform":"sulfoxide","service":"iodine"}` | AtomiCloud Service Tree. See [ServiceTree](https://atomicloud.larksuite.com/wiki/OkfJwTXGFiMJkrk6W3RuwRrZs64?theme=DARK&contentTheme=DARK#MHw5d76uDo2tBLx86cduFQMRsBb) |
+| serviceTree | object | `{"layer":"1","platform":"sulfoxide","service":"iodine"}` | AtomiCloud Service Tree. See [ServiceTree](https://atomicloud.larksuite.com/wiki/OkfJwTXGFiMJkrk6W3RuwRrZs64?theme=DARK&contentTheme=DARK#MHw5d76uDo2tBLx86cduFQMRsBb) |
 | sulfoxide-bromine | object | `{"annotations":{"argocd.argoproj.io/sync-wave":"-5"},"rootSecret":{"ref":"SULFOXIDE_IODINE"},"storeName":"doppler-iodine"}` | Create SecretStore via secret of secrets pattern |
 | sulfoxide-bromine.rootSecret | object | `{"ref":"SULFOXIDE_IODINE"}` | Secret of Secrets reference |
 | sulfoxide-bromine.rootSecret.ref | string | `"SULFOXIDE_IODINE"` | DOPPLER Token Reference |
 | sulfoxide-bromine.storeName | string | `"doppler-iodine"` | Store name to create |
-| tags | object | `{"atomi.cloud/layer":"1","atomi.cloud/module":"apiserver","atomi.cloud/platform":"sulfoxide","atomi.cloud/service":"iodine"}` | Kubernetes labels and annotations, following Service Tree |
-| vcluster | object | `{"annotations":{"<<":{"atomi.cloud/layer":"1","atomi.cloud/module":"apiserver","atomi.cloud/platform":"sulfoxide","atomi.cloud/service":"iodine"}},"coredns":{"replicas":1},"enableHA":true,"ingress":{"enabled":true,"host":"kubernetes.atomi.cloud","ingressClassName":"nginx"},"init":{"manifests":"apiVersion: v1\nkind: Namespace\nmetadata:\n  labels:\n    kubernetes.io/metadata.name: sulfoxide\n  name: sulfoxide\n"},"labels":{"<<":{"atomi.cloud/layer":"1","atomi.cloud/module":"apiserver","atomi.cloud/platform":"sulfoxide","atomi.cloud/service":"iodine"}},"mapServices":{"fromHost":[{"from":"sulfoxide/entei-silicon-otlp-collector","to":"sulfoxide/silicon-otlp-collector"}]},"plugin":{"secret-syncer":{"image":"ghcr.io/kirinnee/vcluster-secret-syncer/secret-syncer-amd:1.0.0","imagePullPolicy":"IfNotPresent"}},"podAnnotations":{"<<":{"atomi.cloud/layer":"1","atomi.cloud/module":"apiserver","atomi.cloud/platform":"sulfoxide","atomi.cloud/service":"iodine"}},"podLabels":{"<<":{"atomi.cloud/layer":"1","atomi.cloud/module":"apiserver","atomi.cloud/platform":"sulfoxide","atomi.cloud/service":"iodine"}},"proxy":{"metricsServer":{"nodes":{"enabled":false},"pods":{"enabled":false}}},"replicas":1,"serverToken":{"secretKeyRef":{"key":"K3S_TOKEN","name":"k3s-sync-token"}},"storage":{"persistence":false},"sync":{"configmaps":{"all":true},"ingresses":{"enabled":true},"nodes":{"enableScheduler":true,"enabled":true,"fakeKubeletIPs":true,"syncAllNodes":true,"syncNodeChanges":true},"pods":{"enabled":true,"ephemeralContainers":true,"status":true},"secrets":{"all":true}},"syncer":{"extraArgs":["--tls-san=https://kubernetes.atomi.cloud"]},"telemetry":{"disabled":true},"vcluster":{"env":[{"name":"K3S_DATASTORE_ENDPOINT","valueFrom":{"secretKeyRef":{"key":"K3S_DATASTORE_ENDPOINT","name":"datastore-endpoint"}}},{"name":"K3S_DATASTORE_CAFILE","value":"/etc/certs/cert.ca"}],"extraVolumeMounts":[{"mountPath":"/etc/certs","name":"datastore-tls"}]},"volumes":[{"name":"datastore-tls","secret":{"items":[{"key":"K3S_DATASTORE_CAFILE","path":"cert.ca"}],"secretName":"datastore-ca"}}]}` | Virtual Cluster Configuration. See [vcluster documentation](https://artifacthub.io/packages/helm/loft/vcluster) |
+| tags | object | `{"atomi.cloud/layer":"1","atomi.cloud/platform":"sulfoxide","atomi.cloud/service":"iodine"}` | Kubernetes labels and annotations, following Service Tree |
+| vcluster | object | `{"annotations":{"<<":{"atomi.cloud/layer":"1","atomi.cloud/platform":"sulfoxide","atomi.cloud/service":"iodine"},"atomi.cloud/module":"apiserver"},"coredns":{"replicas":1},"enableHA":true,"ingress":{"enabled":true,"host":"kubernetes.atomi.cloud","ingressClassName":"nginx"},"init":{"manifests":"apiVersion: v1\nkind: Namespace\nmetadata:\n  labels:\n    kubernetes.io/metadata.name: sulfoxide\n  name: sulfoxide\n"},"labels":{"<<":{"atomi.cloud/layer":"1","atomi.cloud/platform":"sulfoxide","atomi.cloud/service":"iodine"},"atomi.cloud/module":"apiserver"},"mapServices":{"fromHost":[{"from":"sulfoxide/entei-silicon-otlp-collector","to":"sulfoxide/silicon-otlp-collector"}]},"plugin":{"secret-syncer":{"image":"ghcr.io/kirinnee/vcluster-secret-syncer/secret-syncer-amd:1.0.0","imagePullPolicy":"IfNotPresent"}},"podAnnotations":{"<<":{"atomi.cloud/layer":"1","atomi.cloud/platform":"sulfoxide","atomi.cloud/service":"iodine"},"atomi.cloud/module":"apiserver"},"podLabels":{"<<":{"atomi.cloud/layer":"1","atomi.cloud/platform":"sulfoxide","atomi.cloud/service":"iodine"},"atomi.cloud/module":"apiserver"},"proxy":{"metricsServer":{"nodes":{"enabled":false},"pods":{"enabled":false}}},"replicas":1,"serverToken":{"secretKeyRef":{"key":"K3S_TOKEN","name":"k3s-sync-token"}},"storage":{"persistence":false},"sync":{"configmaps":{"all":true},"ingresses":{"enabled":true},"nodes":{"enableScheduler":true,"enabled":true,"fakeKubeletIPs":true,"syncAllNodes":true,"syncNodeChanges":true},"pods":{"enabled":true,"ephemeralContainers":true,"status":true},"secrets":{"all":true}},"syncer":{"extraArgs":["--tls-san=https://kubernetes.atomi.cloud"]},"telemetry":{"disabled":true}}` | Virtual Cluster Configuration. See [vcluster documentation](https://artifacthub.io/packages/helm/loft/vcluster) |
 
 ----------------------------------------------
 Autogenerated from chart metadata using [helm-docs v1.11.1](https://github.com/norwoodj/helm-docs/releases/v1.11.1)

chart/templates/datastore-secret.yaml

@@ -1,3 +1,4 @@
+{{- if .Values.datastore.enable }}
 apiVersion: external-secrets.io/v1beta1
 kind: ExternalSecret
 metadata:
@@ -17,4 +18,5 @@ spec:
   data:
     - secretKey: "{{ .Values.datastore.secretKey }}"
       remoteRef:
-        key: "{{ .Values.datastore.remoteName }}"
+        key: "{{ .Values.datastore.remoteName }}"
+  {{- end }}

chart/values.pichu.onyx.yaml

@@ -0,0 +1,60 @@
+serviceTree:
+  landscape: &landscape pichu
+  cluster: &cluster onyx
+
+tags: &tags
+  atomi.cloud/landscape: *landscape
+  atomi.cloud/cluster: *cluster
+
+auth:
+  remoteName: PICHU_SULFOXIDE_SOS
+
+datastore:
+  enable: false
+
+datastoreCa:
+  enable: false
+
+k3sSyncToken:
+  remoteName: PICHU_K3S_TOKEN
+
+etcd:
+  enabled: true
+  commonLabels:
+    <<: *tags
+  podLabels:
+    <<: *tags
+  commonAnnotations:
+    <<: *tags
+  podAnnotations:
+    <<: *tags
+  persistence:
+    enabled: true
+    accessModes: [ "ReadWriteOnce" ]
+    storageClass: "ebs"
+  replicaCount: 3
+  fullnameOverride: iodine-etcd
+vcluster:
+  replicas: 1
+  coredns:
+    replicas: 1
+  podLabels:
+    <<: *tags
+  podAnnotations:
+    <<: *tags
+  labels:
+    <<: *tags
+  annotations:
+    <<: *tags
+  syncer:
+    extraArgs:
+      - --kube-config-context-name=pichu-onyx
+      - --out-kube-config-server=https://pichu.onyx.kubernetes.atomi.cloud
+  ingress:
+    host: pichu.onyx.kubernetes.atomi.cloud
+  vcluster:
+    env:
+      - name: K3S_DATASTORE_ENDPOINT
+        value: http://iodine-etcd:2379
+
+

chart/values.pichu.opal.yaml

@@ -10,6 +10,7 @@ auth:
   remoteName: PICHU_SULFOXIDE_SOS
 
 datastore:
+  enable: true
   remoteName: PICHU_OPAL_K3S_DATASTORE_ENDPOINT
 
 datastoreCa:
@@ -19,6 +20,9 @@ datastoreCa:
 k3sSyncToken:
   remoteName: PICHU_K3S_TOKEN
 
+etcd:
+  enabled: false
+
 vcluster:
   replicas: 1
   coredns:
@@ -37,3 +41,23 @@ vcluster:
       - --out-kube-config-server=https://pichu.opal.kubernetes.atomi.cloud
   ingress:
     host: pichu.opal.kubernetes.atomi.cloud
+
+  volumes:
+    - name: datastore-tls
+      secret:
+        secretName: datastore-ca
+        items:
+          - key: K3S_DATASTORE_CAFILE
+            path: cert.ca
+  vcluster:
+    extraVolumeMounts:
+      - mountPath: /etc/certs
+        name: datastore-tls
+    env:
+      - name: K3S_DATASTORE_ENDPOINT
+        valueFrom:
+          secretKeyRef:
+            name: datastore-endpoint
+            key: K3S_DATASTORE_ENDPOINT
+      - name: K3S_DATASTORE_CAFILE
+        value: /etc/certs/cert.ca

chart/values.pichu.ruby.yaml

@@ -10,6 +10,7 @@ auth:
   remoteName: PICHU_SULFOXIDE_SOS
 
 datastore:
+  enable: true
   remoteName: PICHU_RUBY_K3S_DATASTORE_ENDPOINT
 
 datastoreCa:
@@ -19,6 +20,9 @@ datastoreCa:
 k3sSyncToken:
   remoteName: PICHU_K3S_TOKEN
 
+etcd:
+  enabled: false
+
 vcluster:
   replicas: 1
   coredns:
@@ -37,3 +41,23 @@ vcluster:
       - --out-kube-config-server=https://pichu.ruby.kubernetes.atomi.cloud
   ingress:
     host: pichu.ruby.kubernetes.atomi.cloud
+
+  volumes:
+    - name: datastore-tls
+      secret:
+        secretName: datastore-ca
+        items:
+          - key: K3S_DATASTORE_CAFILE
+            path: cert.ca
+  vcluster:
+    extraVolumeMounts:
+      - mountPath: /etc/certs
+        name: datastore-tls
+    env:
+      - name: K3S_DATASTORE_ENDPOINT
+        valueFrom:
+          secretKeyRef:
+            name: datastore-endpoint
+            key: K3S_DATASTORE_ENDPOINT
+      - name: K3S_DATASTORE_CAFILE
+        value: /etc/certs/cert.ca

chart/values.pikachu.onyx.yaml

@@ -0,0 +1,60 @@
+serviceTree:
+  landscape: &landscape pikachu
+  cluster: &cluster onyx
+
+tags: &tags
+  atomi.cloud/landscape: *landscape
+  atomi.cloud/cluster: *cluster
+
+auth:
+  remoteName: PIKACHU_SULFOXIDE_SOS
+
+datastore:
+  enable: false
+
+datastoreCa:
+  enable: false
+
+k3sSyncToken:
+  remoteName: PIKACHU_K3S_TOKEN
+
+etcd:
+  enabled: true
+  commonLabels:
+    <<: *tags
+  podLabels:
+    <<: *tags
+  commonAnnotations:
+    <<: *tags
+  podAnnotations:
+    <<: *tags
+  persistence:
+    enabled: true
+    accessModes: [ "ReadWriteOnce" ]
+    storageClass: "ebs"
+  replicaCount: 3
+  fullnameOverride: iodine-etcd
+vcluster:
+  replicas: 1
+  coredns:
+    replicas: 1
+  podLabels:
+    <<: *tags
+  podAnnotations:
+    <<: *tags
+  labels:
+    <<: *tags
+  annotations:
+    <<: *tags
+  syncer:
+    extraArgs:
+      - --kube-config-context-name=pikachu-onyx
+      - --out-kube-config-server=https://pikachu.onyx.kubernetes.atomi.cloud
+  ingress:
+    host: pikachu.onyx.kubernetes.atomi.cloud
+  vcluster:
+    env:
+      - name: K3S_DATASTORE_ENDPOINT
+        value: http://iodine-etcd:2379
+
+

chart/values.pikachu.opal.yaml

@@ -10,6 +10,7 @@ auth:
   remoteName: PIKACHU_SULFOXIDE_SOS
 
 datastore:
+  enable: true
   remoteName: PIKACHU_OPAL_K3S_DATASTORE_ENDPOINT
 
 datastoreCa:
@@ -19,6 +20,9 @@ datastoreCa:
 k3sSyncToken:
   remoteName: PIKACHU_K3S_TOKEN
 
+etcd:
+  enabled: false
+
 vcluster:
   replicas: 1
   coredns:
@@ -37,3 +41,23 @@ vcluster:
       - --out-kube-config-server=https://pikachu.opal.kubernetes.atomi.cloud
   ingress:
     host: pikachu.opal.kubernetes.atomi.cloud
+
+  volumes:
+    - name: datastore-tls
+      secret:
+        secretName: datastore-ca
+        items:
+          - key: K3S_DATASTORE_CAFILE
+            path: cert.ca
+  vcluster:
+    extraVolumeMounts:
+      - mountPath: /etc/certs
+        name: datastore-tls
+    env:
+      - name: K3S_DATASTORE_ENDPOINT
+        valueFrom:
+          secretKeyRef:
+            name: datastore-endpoint
+            key: K3S_DATASTORE_ENDPOINT
+      - name: K3S_DATASTORE_CAFILE
+        value: /etc/certs/cert.ca