From 9cad28fe91eeb349d34d30bae4a1ff371fd61a54 Mon Sep 17 00:00:00 2001 From: davidshtian Date: Wed, 2 Jun 2021 21:11:49 +0800 Subject: [PATCH 1/5] Update metadata.go Add on-premise using environment variables comments. --- pkg/cloud/metadata.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/cloud/metadata.go b/pkg/cloud/metadata.go index 5111df574..7563f9270 100644 --- a/pkg/cloud/metadata.go +++ b/pkg/cloud/metadata.go @@ -58,7 +58,7 @@ func (m *metadata) GetAvailabilityZone() string { return m.availabilityZone } -// NewMetadataService return either EC2 or ECS Task MetadataServiceImplementation. +// NewMetadataService return either EC2, ECS Task MetadataServiceImplementation or on-premise using environment variables. func NewMetadataService(sess *session.Session) (MetadataService, error) { // check if it is running in on-premise environment otherwise turn to to ECS if onPremiseEnv := os.Getenv("onPremise"); onPremiseEnv == "true" { From 94394cdda7e0c8a5aa21b60dde2378be3231c809 Mon Sep 17 00:00:00 2001 From: davidshtian Date: Wed, 2 Jun 2021 22:03:25 +0800 Subject: [PATCH 2/5] Create ON-PREMISE.md --- docs/ON-PREMISE.md | 58 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 docs/ON-PREMISE.md diff --git a/docs/ON-PREMISE.md b/docs/ON-PREMISE.md new file mode 100644 index 000000000..687f46a58 --- /dev/null +++ b/docs/ON-PREMISE.md @@ -0,0 +1,58 @@ +## Notes for on-premise Kubernetes environment + +### Decouple EC2 metadata service (IMDS) +Since on-premise Kubernetes environment cannot access Amazon EC2 metadata service and cannot get information about instanceID, region and availabilityZone, additional environment variables need to be set, otherwise it will throw "could not get metadata from AWS: EC2 instance metadata is not available" described in [issue 468](https://github.com/kubernetes-sigs/aws-efs-csi-driver/issues/468). + +Environment variables need to be added for efs-plugin container in [controller-deployment.yaml](../deploy/kubernetes/base/controller-deployment.yaml), specify onPremise to let driver know it's a on-premise Kubernetes environment, and then follow the deployment guide above. Examples are shown below (instanceID can be mocked): + +``` +... + - name: onPremise + value: "true" + - name: instanceID + value: i-0123456789012345 + - name: region + value: us-east-1 + - name: availabilityZone + value: us-east-1a +... +``` +For IAM permission, you could set it using environment variables with [AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html) or mount secret to container for [configuration and credential file settings](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html). + +### Configure region for efs-utils +Besides, you might encounter errors when mounting file system "Output: Error retrieving region. Please set the "region" parameter in the efs-utils configuration file", the binary entrypoint aws-efs-csi-driver would dynamically generate configuration file, but the region information need to specify in the conf file which will be override by aws-efs-csi-driver. Follow below procedure to fix this issue: +* Get the original efs-utils configuration file +``` +kubectl -n kube-system exec -it efs-csi-node- -c efs-plugin cat /etc/amazon/efs/efs-utils.conf +``` +* Configure region information `region = us-east-1` and add disable fetch ec2 metadata setting `disable_fetch_ec2_metadata_token = true` +* Create new configmap +``` +kubectl -n kube-system create configmap efs-utils-conf --from-file=./efs-utils.conf +``` +* Edit efs-plugin in daemon set efs-csi-node +``` +kubectl -n kube-system edit daemonsets.apps efs-csi-node +``` +Add the configurations below: +``` +... + lifecycle: + postStart: + exec: + command: ["/bin/sh", "-c", "cp -f /tmp/efs-utils.conf /etc/amazon/efs/efs-utils.conf"] +... + - mountPath: /tmp/efs-utils.conf + subPath: efs-utils.conf + name: efs-utils-conf +... + - configMap: + name: efs-utils-conf + name: efs-utils-conf +... +``` + +### DNS resolve issue +And if you still got errors 'Output: Failed to resolve "fs-01234567.efs.us-east-1.amazonaws.com" - check that your file system ID is correct, and ensure that the VPC has an EFS mount target for this file system ID." Follow below procedure to fix this issue: +* Configure IP address in /etc/hosts on each host, refer to [Walkthrough: Create and mount a file system on-premises with AWS Direct Connect and VPN](https://docs.aws.amazon.com/efs/latest/ug/efs-onpremises.html) +* Or install botocore on each host and set `fall_back_to_mount_target_ip_address_enabled = true` in efs-utils.conf, refer to [Using botocore to retrieve mount target ip address when dns name cannot be resolved](https://github.com/aws/efs-utils). From 171794d63017f5f2225406da3d7c754976a68024 Mon Sep 17 00:00:00 2001 From: davidshtian Date: Wed, 2 Jun 2021 22:05:46 +0800 Subject: [PATCH 3/5] Update README.md Add doc link to ON-PREMISE.md instruction. --- docs/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/README.md b/docs/README.md index e0c28e9ed..770d28c83 100644 --- a/docs/README.md +++ b/docs/README.md @@ -82,6 +82,8 @@ The following sections are Kubernetes specific. If you are a Kubernetes user, us **Notes**: * Since EFS is an elastic file system it doesn't really enforce any file system capacity. The actual storage capacity value in persistent volume and persistent volume claim is not used when creating the file system. However, since the storage capacity is a required field by Kubernetes, you must specify the value and you can use any valid value for the capacity. +* If you are deploying Amazon EFS CSI Driver in on-premise environment, please refer to the doc [ON-PREMISE.md](./ON-PREMISE.md). + ### Installation #### Set up driver permission: The driver requires IAM permission to talk to Amazon EFS to manage the volume on user's behalf. There are several methods to grant driver IAM permission: From 4dc84e9173bb4e586159987d7db01159e941e1e2 Mon Sep 17 00:00:00 2001 From: davidshtian Date: Wed, 2 Jun 2021 22:06:24 +0800 Subject: [PATCH 4/5] Update controller-deployment.yaml Restore back to default settings. --- deploy/kubernetes/base/controller-deployment.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/deploy/kubernetes/base/controller-deployment.yaml b/deploy/kubernetes/base/controller-deployment.yaml index 6a7f10fbb..ff56155ba 100644 --- a/deploy/kubernetes/base/controller-deployment.yaml +++ b/deploy/kubernetes/base/controller-deployment.yaml @@ -42,12 +42,6 @@ spec: env: - name: CSI_ENDPOINT value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock - - name: instanceID - value: - - name: region - value: - - name: availabilityZone - value: volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ From a913b4cb26ab702739a3cbe992f9af4a5457c813 Mon Sep 17 00:00:00 2001 From: davidshtian Date: Wed, 2 Jun 2021 22:17:52 +0800 Subject: [PATCH 5/5] Update metadata.go --- pkg/cloud/metadata.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/cloud/metadata.go b/pkg/cloud/metadata.go index 7563f9270..821a92180 100644 --- a/pkg/cloud/metadata.go +++ b/pkg/cloud/metadata.go @@ -60,7 +60,7 @@ func (m *metadata) GetAvailabilityZone() string { // NewMetadataService return either EC2, ECS Task MetadataServiceImplementation or on-premise using environment variables. func NewMetadataService(sess *session.Session) (MetadataService, error) { - // check if it is running in on-premise environment otherwise turn to to ECS + // check if it is running in on-premise environment otherwise turn to ECS if onPremiseEnv := os.Getenv("onPremise"); onPremiseEnv == "true" { return &metadata{ instanceID: os.Getenv("instanceID"),