Feature: Add support for Kubernetes deployment through Kustomize (#187)

faros-ai · Jun 27, 2022 · 0f15b23 · 0f15b23
1 parent d0b7eb5
commit 0f15b23
Show file tree

Hide file tree

Showing 23 changed files with 753 additions and 0 deletions.
diff --git a/.env b/.env
@@ -42,6 +42,7 @@ HASURA_VERSION=v2.1.1
 
 ############################## Metabase #######################################
 METABASE_IMAGE="metabase/metabase"
+METABASE_VERSION=v0.41.6
 METABASE_DB_NAME=metabase
 METABASE_PORT=3000
 METABASE_USER=admin@admin.com

diff --git a/kube/README.md b/kube/README.md
@@ -0,0 +1,47 @@
+## Deploying FarosCE on Kubernetes
+
+We provide all required Kubernetes manifests for a full stack deployment in this repository under `kube/base`. These manifests can be used raw but are also exposed using [Kustomize](https://kustomize.io/) for a more flexible and scalable deployment model.
+
+> Local Kubernetes clusters can be setup in a few different ways. Here's a list of the few most common tools for local K8s deployment: [kind](https://kind.sigs.k8s.io/), [k3d](https://k3d.io/v5.4.3/) or [minikube](https://minikube.sigs.k8s.io/docs/start/).
+
+Provided you have have access to a Kubernetes cluster and have configured access to it via `kubectl`, FarosCE can be deployed with the following command:
+
+```base
+kustomize build --load-restrictor LoadRestrictionsNone https://github.com/faros-ai/faros-community-edition/kube/base | kubectl apply -f -
+```
+
+Kustomize will load the deployment definition in this repository over Github, build the manifests and pipe them over to `kubectl` for deployment in the configured cluster.
+
+To access the deployed applications you can port forward onto the relevant services, for example:
+
+```yaml
+# Expose Airbyte to localhost
+kubectl port-forward svc/airbyte-webapp-svc 8000:80
+# Expose Metabase to localhost
+kubectl port-forward svc/metabase 3000:80
+# Expose Hasura to localhost
+kubectl port-forward svc/hasura 8080:80
+# Expose N8N to localhost
+kubectl port-forward svc/n8n 5678:80
+```
+
+> Kubernetes services can also be exposed using NodePort, Ingress, or External LoadBalancers. This will vary depending on your Kubernetes cluster.
+
+### Using kustomization.yaml file
+
+For a more declarative and GitOps friendly deployment approach you can create your own local `kustomization.yaml` file, put it under a source control system and reference the same path in the resources attribute.
+
+```yaml
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - https://github.com/faros-ai/faros-community-edition/kube/base
+```
+
+> This allows you to customize the Kubernetes resources objects in any way you want to meet whatever desired cluster requirements you might have
+
+With this file available you can run the following command from the same path to build and apply the manifests:
+
+```yaml
+kustomize build --load-restrictor LoadRestrictionsNone . | kubectl apply -f -
+```
diff --git a/kube/base/airbyte/bootloader.yaml b/kube/base/airbyte/bootloader.yaml
@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: airbyte-bootloader
+  labels:
+    app.kubernetes.io/name: airbyte-bootloader
+    app.kubernetes.io/component: bootloader
diff --git a/kube/base/airbyte/db.yaml b/kube/base/airbyte/db.yaml
@@ -0,0 +1,68 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: airbyte-db
+  labels:
+    app.kubernetes.io/name: airbyte-db
+    app.kubernetes.io/component: database
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: airbyte-db
+      app.kubernetes.io/component: database
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: airbyte-db
+        app.kubernetes.io/component: database
+    spec:
+      containers:
+        - name: airbyte-db-container
+          env:
+            - name: CONFIG_DATABASE_PASSWORD
+              valueFrom:
+                configMapKeyRef:
+                  name: faros-config
+                  key: CONFIG_DATABASE_PASSWORD
+                  optional: true
+            - name: CONFIG_DATABASE_URL
+              valueFrom:
+                configMapKeyRef:
+                  name: faros-config
+                  key: CONFIG_DATABASE_URL
+                  optional: true
+            - name: CONFIG_DATABASE_USER
+              valueFrom:
+                configMapKeyRef:
+                  name: faros-config
+                  key: CONFIG_DATABASE_USER
+                  optional: true
+            - name: DATABASE_PASSWORD
+              valueFrom:
+                configMapKeyRef:
+                  name: faros-config
+                  key: DATABASE_PASSWORD
+            - name: DATABASE_URL
+              valueFrom:
+                configMapKeyRef:
+                  name: faros-config
+                  key: DATABASE_URL
+            - name: DATABASE_USER
+              valueFrom:
+                configMapKeyRef:
+                  name: faros-config
+                  key: DATABASE_USER
+            - name: POSTGRES_PASSWORD
+              value: ""
+              valueFrom:
+                configMapKeyRef:
+                  name: faros-config
+                  key: DATABASE_PASSWORD
+            - name: POSTGRES_USER
+              value: ""
+              valueFrom:
+                configMapKeyRef:
+                  name: faros-config
+                  key: DATABASE_USER
+            - name: POSTGRES_DB
+              $patch: delete
diff --git a/kube/base/airbyte/scheduler.yaml b/kube/base/airbyte/scheduler.yaml
@@ -0,0 +1,17 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: airbyte-scheduler
+  labels:
+    app.kubernetes.io/name: airbyte-scheduler
+    app.kubernetes.io/component: scheduler
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: airbyte-scheduler
+      app.kubernetes.io/component: scheduler
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: airbyte-scheduler
+        app.kubernetes.io/component: scheduler
diff --git a/kube/base/airbyte/server.yaml b/kube/base/airbyte/server.yaml
@@ -0,0 +1,29 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: airbyte-server-svc
+  labels:
+    app.kubernetes.io/name: airbyte-server
+    app.kubernetes.io/component: server
+spec:
+  selector:
+    app.kubernetes.io/name: airbyte-server
+    app.kubernetes.io/component: server
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: airbyte-server
+  labels:
+    app.kubernetes.io/name: airbyte-server
+    app.kubernetes.io/component: server
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: airbyte-server
+      app.kubernetes.io/component: server
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: airbyte-server
+        app.kubernetes.io/component: server
diff --git a/kube/base/airbyte/temporal.yaml b/kube/base/airbyte/temporal.yaml
@@ -0,0 +1,26 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: airbyte-temporal-svc
+spec:
+  selector:
+    app.kubernetes.io/name: airbyte-temporal
+    app.kubernetes.io/component: temporal
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: airbyte-temporal
+  labels:
+    app.kubernetes.io/name: airbyte-temporal
+    app.kubernetes.io/component: temporal
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: airbyte-temporal
+      app.kubernetes.io/component: temporal
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: airbyte-temporal
+        app.kubernetes.io/component: temporal
diff --git a/kube/base/airbyte/webapp.yaml b/kube/base/airbyte/webapp.yaml
@@ -0,0 +1,29 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: airbyte-webapp-svc
+  labels:
+    app.kubernetes.io/name: airbyte-webapp-svc
+    app.kubernetes.io/component: webapp
+spec:
+  selector:
+    app.kubernetes.io/name: airbyte-webapp
+    app.kubernetes.io/component: webapp
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: airbyte-webapp
+  labels:
+    app.kubernetes.io/name: airbyte-webapp
+    app.kubernetes.io/component: webapp
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: airbyte-webapp
+      app.kubernetes.io/component: webapp
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: airbyte-webapp
+        app.kubernetes.io/component: webapp
diff --git a/kube/base/airbyte/worker.yaml b/kube/base/airbyte/worker.yaml
@@ -0,0 +1,17 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: airbyte-worker
+  labels:
+    app.kubernetes.io/name: airbyte-worker
+    app.kubernetes.io/component: worker
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: airbyte-worker
+      app.kubernetes.io/component: worker
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: airbyte-worker
+        app.kubernetes.io/component: worker
diff --git a/kube/base/faros/config/.env b/kube/base/faros/config/.env
@@ -0,0 +1 @@
+../../../../.env
diff --git a/kube/base/faros/config/.k8s.env b/kube/base/faros/config/.k8s.env
@@ -0,0 +1,13 @@
+############################## Faros ##########################################
+FAROS_DB_HOST=airbyte-db-svc
+FAROS_CONFIG_DB_HOST=airbyte-db-svc
+
+############################## Airbyte ########################################
+AIRBYTE_URL=http://airbyte-webapp-svc
+
+############################## Hasura ########################################
+HASURA_URL=http://hasura
+
+############################## Metabase ########################################
+METABASE_URL=http://metabase
+METABASE_FAROS_DB_HOST=airbyte-db-svc
diff --git a/kube/base/faros/init.yaml b/kube/base/faros/init.yaml
@@ -0,0 +1,126 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: faros-init
+spec:
+  ttlSecondsAfterFinished: 300
+  template:
+    spec:
+      restartPolicy: OnFailure
+      containers:
+        - name: faros-init
+          image: farosai.docker.scarf.sh/farosai/faros-ce-init:latest
+          env:
+            - name: AIRBYTE_URL
+              valueFrom:
+                configMapKeyRef:
+                  name: faros-config
+                  key: AIRBYTE_URL
+            - name: AIRBYTE_FORCE_SETUP
+              value: "false"
+            - name: AIRBYTE_DESTINATION_HASURA_URL
+              valueFrom:
+                configMapKeyRef:
+                  name: faros-config
+                  key: HASURA_URL
+            - name: FAROS_DB_HOST
+              valueFrom:
+                configMapKeyRef:
+                  name: faros-config
+                  key: FAROS_DB_HOST
+            - name: FAROS_DB_PORT
+              valueFrom:
+                configMapKeyRef:
+                  name: faros-config
+                  key: FAROS_DB_PORT
+            - name: FAROS_DB_PASSWORD
+              valueFrom:
+                configMapKeyRef:
+                  name: faros-config
+                  key: FAROS_DB_PASSWORD
+            - name: FAROS_DB_NAME
+              valueFrom:
+                configMapKeyRef:
+                  name: faros-config
+                  key: FAROS_DB_NAME
+            - name: FAROS_DB_USER
+              valueFrom:
+                configMapKeyRef:
+                  name: faros-config
+                  key: FAROS_DB_USER
+            - name: FAROS_CONFIG_DB_HOST
+              valueFrom:
+                configMapKeyRef:
+                  name: faros-config
+                  key: FAROS_CONFIG_DB_HOST
+            - name: FAROS_CONFIG_DB_PORT
+              valueFrom:
+                configMapKeyRef:
+                  name: faros-config
+                  key: FAROS_CONFIG_DB_PORT
+            - name: FAROS_CONFIG_DB_USER
+              valueFrom:
+                configMapKeyRef:
+                  name: faros-config
+                  key: FAROS_CONFIG_DB_USER
+            - name: FAROS_CONFIG_DB_PASSWORD
+              valueFrom:
+                configMapKeyRef:
+                  name: faros-config
+                  key: FAROS_CONFIG_DB_PASSWORD
+            - name: HASURA_DB_NAME
+              valueFrom:
+                configMapKeyRef:
+                  name: faros-config
+                  key: HASURA_DB_NAME
+            - name: HASURA_URL
+              valueFrom:
+                configMapKeyRef:
+                  name: faros-config
+                  key: HASURA_URL
+            - name: HASURA_GRAPHQL_ADMIN_SECRET
+              valueFrom:
+                configMapKeyRef:
+                  name: faros-config
+                  key: HASURA_GRAPHQL_ADMIN_SECRET
+            - name: HASURA_GRAPHQL_DATABASE_URL
+              valueFrom:
+                configMapKeyRef:
+                  name: faros-config
+                  key: HASURA_GRAPHQL_DATABASE_URL
+                  optional: true
+            - name: METABASE_DB_NAME
+              valueFrom:
+                configMapKeyRef:
+                  name: faros-config
+                  key: METABASE_DB_NAME
+            - name: METABASE_FAROS_DB_HOST
+              valueFrom:
+                configMapKeyRef:
+                  name: faros-config
+                  key: METABASE_FAROS_DB_HOST
+            - name: METABASE_USE_SSL
+              valueFrom:
+                configMapKeyRef:
+                  name: faros-config
+                  key: METABASE_USE_SSL
+            - name: METABASE_PASSWORD
+              valueFrom:
+                configMapKeyRef:
+                  name: faros-config
+                  key: METABASE_PASSWORD
+            - name: METABASE_URL
+              valueFrom:
+                configMapKeyRef:
+                  name: faros-config
+                  key: METABASE_URL
+            - name: METABASE_USER
+              valueFrom:
+                configMapKeyRef:
+                  name: faros-config
+                  key: METABASE_USER
+            - name: N8N_DB_NAME
+              valueFrom:
+                configMapKeyRef:
+                  name: faros-config
+                  key: N8N_DB_NAME