From feacac3ef66bf133732803abc1a1327c370391d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Massot?= Date: Mon, 4 Mar 2024 02:04:44 +0100 Subject: [PATCH 1/3] Add EKS tutorial with service account setup. --- docs/deployment/kubernetes/eks.md | 191 ++++++++++++++++++++++++++++++ docs/deployment/kubernetes/gke.md | 15 ++- 2 files changed, 201 insertions(+), 5 deletions(-) create mode 100644 docs/deployment/kubernetes/eks.md diff --git a/docs/deployment/kubernetes/eks.md b/docs/deployment/kubernetes/eks.md new file mode 100644 index 00000000000..8b28b764e95 --- /dev/null +++ b/docs/deployment/kubernetes/eks.md @@ -0,0 +1,191 @@ +--- +title: EKS + Helm +sidebar_position: 3 +--- + +This guide will help you set up a Quickwit cluster on EKS with the correct S3 permissions. + +## Prerequisites +- Running Elastic Kubernetes cluster (EKS) +- `kubectl` +- Permission to create the IAM role and Policies +- AWS CLI +- `eksctl` if you don't have an IAM OIDC provider for your cluster. + +## Set up + +Let's use the following environment variables: + +```bash +export NAMESPACE=qw-tutorial +export EKS_CLUSTER=qw-cluster +export S3_BUCKET={your-bucket} +export SERVICE_ACCOUNT_NAME=qw-sa +export REGION={your-region} +export CLUSTER_ID={your-cluster-id} +``` + +Create the namespace for our playground: + +```bash +kubectl create ns ${NAMESPACE} +``` + +And set this namespace as the default one: + +```bash +kubectl config set-context --current --namespace=${NAMESPACE} +``` + + +### Create IAM OIDC provider if you don't have one + +To check if you have one provider for your EKS cluster, just run: + +```bash +aws iam list-open-id-connect-providers +``` + +If you have one, you will get a response similar to this one: + +```json +{ + "OpenIDConnectProviderList": [ + { + "Arn": "arn:aws:iam::(some-ID):oidc-provider/oidc.eks.{your-region}.amazonaws.com/id/{your-cluster-id}" + } + ] +} +``` + +If you don't, run the following command: + +```bash +eksctl utils associate-iam-oidc-provider --cluster ${EKS_CLUSTER} --approve +``` + +You can run again `aws iam list-open-id-connect-providers` to get the ARN of the provider. + +### Create an IAM policy + +You need to set the following policy to allow Quickwit to access your S3 bucket. + +Then create the policy using the AWS CLI: + +```bash +cat > s3-policy.json < s3-role.json << EOF +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Federated": "arn:aws:iam::${IAM_ID}:oidc-provider/oidc.eks.${REGION}.amazonaws.com/id/${CLUSTER_ID}" + }, + "Action": "sts:AssumeRoleWithWebIdentity", + "Condition": { + "StringEquals": { + "oidc.eks.${REGION}.amazonaws.com/id/${CLUSTER_ID}:aud": "sts.amazonaws.com", + "oidc.eks.${REGION}.amazonaws.com/id/${CLUSTER_ID}:sub": "system:serviceaccount:${S3_BUCKET}:${SERVICE_ACCOUNT_NAME}" + } + } + } + ] +} +EOF +``` + +```bash +aws iam create-role --role-name s3-role --assume-role-policy-document file://s3-role.json +``` + +And then attach the policy to the role: + +```bash +aws iam attach-role-policy --role-name s3-role --policy-arn=arn:aws:iam::${IAM_ID}:policy/s3-policy +``` + +## Install Quickwit using Helm + +We are now ready to install Quickwit on EKS. If you'd like to know more about Helm, consult our [comprehensive guide](./helm.md) for installing Quickwit on Kubernetes. + +```bash +helm repo add quickwit https://helm.quickwit.io +helm repo update quickwit +``` + +Let's set Quickwit `values.yaml`: + +```yaml +# We use the edge version here as we recently fixed +# a bug which prevents the metastore from running on GCS. +image: + repository: quickwit/quickwit + pullPolicy: Always + +serviceAccount: + create: true + name: ${SERVICE_ACCOUNT_NAME} + annotations: + eks.amazonaws.com/role-arn: arn:aws:iam::${ARN_ID}:role/${SERVICE_ACCOUNT_NAME} + +config: + default_index_root_uri: s3://${S3_BUCKET}/qw-indexes + metastore_uri: s3://${S3_BUCKET}/qw-indexes + +``` + +We're ready to deploy: + +```bash +helm install quickwit/quickwit -f values.yaml +``` + +## Check that Quickwit is running + +It should take a few seconds for the cluster to start. During the startup process, individual pods might restart themselves several times. + +To access the UI, you can run the following command and then open your browser at [http://localhost:7280](http://localhost:7280): + +``` +kubectl port-forward svc/{release-name}-quickwit-searcher 7280:7280 +``` + +## Uninstall the deployment + +Run the following Helm command to uninstall the deployment + +```bash +helm uninstall +``` + +And don't forget to clean your bucket, Quickwit should have stored 3 files in `s3://${S3_BUCKET}/qw-indexes`. diff --git a/docs/deployment/kubernetes/gke.md b/docs/deployment/kubernetes/gke.md index 7c821779aa9..ee444e0e642 100644 --- a/docs/deployment/kubernetes/gke.md +++ b/docs/deployment/kubernetes/gke.md @@ -4,7 +4,7 @@ sidebar_label: Google GKE sidebar_position: 2 --- -This guide will help you set up a Quickwit cluster with the correct GCS permissions. +This guide will help you set up a Quickwit cluster on GKE with the correct GCS permissions. ## Set up @@ -19,7 +19,6 @@ kubectl create ns ${NS} Quickwit stores its index on an object storage. We will use GCS, which is natively supported since the 0.7 version (for versions < 0.7, you should use an S3 interoperability key). The following steps create a GCP and a GKE service account and bind them together. -We are going to create them, set the right permissions and bind them. ```bash export PROJECT_ID={your-project-id} @@ -66,12 +65,19 @@ image: tag: edge serviceAccount: +<<<<<<< HEAD create: false name: quickwit-sa +======= + create: true + name: ${GKE_SERVICE_ACCOUNT} + annotations: + iam.gke.io/gcp-service-account: ${GCP_SERVICE_ACCOUNT}@${PROJECT_ID}.iam.gserviceaccount.com +>>>>>>> 99c372cec (Add EKS tutorial with service account setup.) config: - default_index_root_uri: gs://{BUCKET}/qw-indexes - metastore_uri: gs://{BUCKET}/qw-indexes + default_index_root_uri: gs://${BUCKET}/qw-indexes + metastore_uri: gs://${BUCKET}/qw-indexes ``` @@ -91,7 +97,6 @@ To access the UI, you can run the following command and then open your browser a kubectl port-forward svc/release-name-quickwit-searcher 7280:7280 ``` - ## Uninstall the deployment Run the following Helm command to uninstall the deployment From 01c3dd0212eabe97280bcb8d1b81594fe76d8ab2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Massot?= Date: Mon, 21 Oct 2024 14:23:50 +0200 Subject: [PATCH 2/3] Update docs/deployment/kubernetes/eks.md Co-authored-by: Harrison Burt <57491488+ChillFish8@users.noreply.github.com> --- docs/deployment/kubernetes/eks.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/deployment/kubernetes/eks.md b/docs/deployment/kubernetes/eks.md index 8b28b764e95..21b02592158 100644 --- a/docs/deployment/kubernetes/eks.md +++ b/docs/deployment/kubernetes/eks.md @@ -115,7 +115,7 @@ cat > s3-role.json << EOF "Condition": { "StringEquals": { "oidc.eks.${REGION}.amazonaws.com/id/${CLUSTER_ID}:aud": "sts.amazonaws.com", - "oidc.eks.${REGION}.amazonaws.com/id/${CLUSTER_ID}:sub": "system:serviceaccount:${S3_BUCKET}:${SERVICE_ACCOUNT_NAME}" + "oidc.eks.${REGION}.amazonaws.com/id/${CLUSTER_ID}:sub": "system:serviceaccount:${NAMESPACE}:${SERVICE_ACCOUNT_NAME}" } } } From 2d13b2def80f4503463fdaab87c69c8ad6a8b37f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Massot?= Date: Mon, 28 Oct 2024 10:08:47 +0100 Subject: [PATCH 3/3] WIP --- docs/deployment/kubernetes/aws-marketplace.md | 202 ++++++++++++++++++ docs/deployment/kubernetes/eks.md | 14 +- docs/deployment/kubernetes/gke.md | 7 - docs/deployment/kubernetes/glasskube.md | 2 +- 4 files changed, 215 insertions(+), 10 deletions(-) create mode 100644 docs/deployment/kubernetes/aws-marketplace.md diff --git a/docs/deployment/kubernetes/aws-marketplace.md b/docs/deployment/kubernetes/aws-marketplace.md new file mode 100644 index 00000000000..7e8e75a1171 --- /dev/null +++ b/docs/deployment/kubernetes/aws-marketplace.md @@ -0,0 +1,202 @@ +--- +title: AWS Marketplace Install +sidebar_label: AWS Markeplace +sidebar_position: 5 +--- + +This guide will help you install Quickwit on EKS from the AWS marketplace. + +## Prerequisites +- Running Elastic Kubernetes cluster (EKS) +- `kubectl` +- Permission to create the IAM role and Policies +- AWS CLI +- `eksctl` if you don't have an IAM OIDC provider for your cluster. + +## Target platforms +Quickwit containers can be run in ECS (including Fargate), or EKS. + +## Set up + +Let's use the following environment variables: + +```bash +export NAMESPACE=qw-tutorial +export EKS_CLUSTER=qw-cluster +export S3_BUCKET={your-bucket} +export SERVICE_ACCOUNT_NAME=qw-sa +export REGION={your-region} +export CLUSTER_ID={your-cluster-id} +``` + +Create the namespace for our playground: + +```bash +kubectl create ns ${NAMESPACE} +``` + +And set this namespace as the default one: + +```bash +kubectl config set-context --current --namespace=${NAMESPACE} +``` + + +### Create IAM OIDC provider if you don't have one + +To check if you have one provider for your EKS cluster, just run: + +```bash +aws iam list-open-id-connect-providers +``` + +If you have one, you will get a response similar to this one: + +```json +{ + "OpenIDConnectProviderList": [ + { + "Arn": "arn:aws:iam::(some-ID):oidc-provider/oidc.eks.{your-region}.amazonaws.com/id/{your-cluster-id}" + } + ] +} +``` + +If you don't, run the following command: + +```bash +eksctl utils associate-iam-oidc-provider --cluster ${EKS_CLUSTER} --approve +``` + +You can run again `aws iam list-open-id-connect-providers` to get the ARN of the provider. + +### Create an IAM policy + +You need to set the following policy to allow Quickwit to access your S3 bucket. + +Then create the policy using the AWS CLI: + +```bash +cat > s3-policy.json < s3-role.json << EOF +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Federated": "arn:aws:iam::${IAM_ID}:oidc-provider/oidc.eks.${REGION}.amazonaws.com/id/${CLUSTER_ID}" + }, + "Action": "sts:AssumeRoleWithWebIdentity", + "Condition": { + "StringEquals": { + "oidc.eks.${REGION}.amazonaws.com/id/${CLUSTER_ID}:aud": "sts.amazonaws.com", + "oidc.eks.${REGION}.amazonaws.com/id/${CLUSTER_ID}:sub": "system:serviceaccount:${NAMESPACE}:${SERVICE_ACCOUNT_NAME}" + } + } + } + ] +} +EOF +``` + +```bash +aws iam create-role --role-name s3-role --assume-role-policy-document file://s3-role.json +``` + +And then attach the policy to the role: + +```bash +aws iam attach-role-policy --role-name s3-role --policy-arn=arn:aws:iam::${IAM_ID}:policy/s3-policy +``` + +## Install Quickwit using Helm + +We are now ready to install Quickwit on EKS. If you'd like to know more about Helm, consult our [comprehensive guide](./helm.md) for installing Quickwit on Kubernetes. + +```bash +helm repo add quickwit https://helm.quickwit.io +helm repo update quickwit +``` + +Let's set Quickwit `values.yaml`: + +```yaml +image: + repository: quickwit/quickwit + pullPolicy: Always + +serviceAccount: + create: true + name: ${SERVICE_ACCOUNT_NAME} + annotations: + eks.amazonaws.com/role-arn: arn:aws:iam::${ARN_ID}:role/${SERVICE_ACCOUNT_NAME} + +config: + default_index_root_uri: s3://${S3_BUCKET}/qw-indexes + metastore_uri: s3://${S3_BUCKET}/qw-indexes + +``` + +We're ready to deploy: + +```bash +helm install quickwit/quickwit -f values.yaml +``` + +## Check that Quickwit is running + +It should take a few seconds for the cluster to start. During the startup process, individual pods might restart themselves several times. + +To access the UI, you can run the following command and then open your browser at [http://localhost:7280](http://localhost:7280): + +``` +kubectl port-forward svc/quickwit-searcher 7280:7280 +``` + +## Uninstall the deployment + +Run the following Helm command to uninstall the deployment + +```bash +helm uninstall +``` + +And don't forget to clean your bucket, Quickwit should have stored 3 files in `s3://${S3_BUCKET}/qw-indexes`. diff --git a/docs/deployment/kubernetes/eks.md b/docs/deployment/kubernetes/eks.md index 21b02592158..2dce610f7b1 100644 --- a/docs/deployment/kubernetes/eks.md +++ b/docs/deployment/kubernetes/eks.md @@ -1,5 +1,6 @@ --- -title: EKS + Helm +title: Install Quickwit on AWS EKS +sidebar_label: AWS EKS sidebar_position: 3 --- @@ -76,6 +77,15 @@ Then create the policy using the AWS CLI: cat > s3-policy.json <>>>>>> 99c372cec (Add EKS tutorial with service account setup.) config: default_index_root_uri: gs://${BUCKET}/qw-indexes diff --git a/docs/deployment/kubernetes/glasskube.md b/docs/deployment/kubernetes/glasskube.md index 4712d2a7e51..56a49b5673d 100644 --- a/docs/deployment/kubernetes/glasskube.md +++ b/docs/deployment/kubernetes/glasskube.md @@ -1,7 +1,7 @@ --- title: Install Quickwit with Glasskube sidebar_label: Glasskube -sidebar_position: 3 +sidebar_position: 4 --- [Glasskube](https://glasskube.dev) is a package manager for Kubernetes that empowers you to effortlessly install, upgrade, configure, and manage your Kubernetes cluster packages, all while streamlining repetitive and cumbersome maintenance tasks.