Skip to content

Commit

Permalink
add end to end test for dynamicfile backend
Browse files Browse the repository at this point in the history
to run it:

make start-dev-dynamicfile
  • Loading branch information
nnmin-aws committed Jan 17, 2023
1 parent d72e1b4 commit f2b6111
Show file tree
Hide file tree
Showing 8 changed files with 264 additions and 1 deletion.
8 changes: 8 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,14 @@ start-dev: bin
stop-dev:
./hack/stop-dev-env.sh

.PHONY: start-dev-dynamicfile-e2e
start-dev-dynamicfile:
./hack/start-dev-env-dynamicfile.sh

.PHONY: stop-dev-dynamicfile-e2e
stop-dev-dynamicfile:
./hack/stop-dev-env.sh

# Use make kill-dev when you want to remove a dev environment
# and clean everything up in preparation for creating another
# in the future.
Expand Down
Empty file added hack/dev/access-entries.json
Empty file.
12 changes: 12 additions & 0 deletions hack/dev/access-entries.template
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
{
"mapRoles": [
{
"rolearn": "arn:aws:iam::{{AWS_ACCOUNT}}:role/{{AWS_TEST_ROLE}}",
"username": "kubernetes-admin",
"groups": [
"system:masters"
]
}
]
}

14 changes: 14 additions & 0 deletions hack/dev/authenticator_with_dynamicfile_mode.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
clusterID: {{CLUSTER_NAME}}
server:
address: {{AUTHENTICATOR_IP}}
port: {{AUTHENTICATOR_PORT}}
stateDir: {{AUTHENTICATOR_STATE_DIR}}
generateKubeconfig: {{AUTHENTICATOR_GENERATE_KUBECONFIG}}
kubeconfig: {{AUTHENTICATOR_KUBECONFIG}}
backendmode: [ "MountedFile", "DynamicFile" ]
dynamicfilepath: {{AUTHENTICATOR_DYNAMICFILE_PATH}}
mapUsers:
- userARN: {{ADMIN_ARN}}
username: kubernetes-admin
groups:
- system:masters
12 changes: 12 additions & 0 deletions hack/dev/policies.template
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Principal": {
"AWS": "arn:aws:iam::{{AWS_ACCOUNT}}:root"
},
"Action": "sts:AssumeRole"
}
]
}
106 changes: 106 additions & 0 deletions hack/e2e-dynamicfile.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,106 @@
#!/bin/bash

# Copyright 2021 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


#
# An easy way to quickly do end to end test for aws-iam-authenticator
# after creating the kind cluster and running authenticator in docker.
#

REPO_ROOT="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )/.." &> /dev/null && pwd )"
OUTPUT="${OUTPUT:-${REPO_ROOT}/_output}"

# Location of templates, config files, mounts
policies_template="${REPO_ROOT}/hack/dev/policies.template"
access_entry_template="${REPO_ROOT}/hack/dev/access-entries.template"
policies_json="${OUTPUT}/dev/authenticator/policies.json"
access_entry_tmp="${OUTPUT}/dev/authenticator/access-entry/access-entries.tmp"
access_entry_json="${OUTPUT}/dev/authenticator/access-entry/access-entries.json"
client_dir="${OUTPUT}/dev/client"
kubectl_kubeconfig="${client_dir}/kubeconfig.yaml"

REGION=${AWS_REGION:-us-west-2}
AWS_ACCOUNT=$(aws sts get-caller-identity --query "Account" --output text)
AWS_TEST_ROLE=${AWS_TEST_ROLE-authenticator-dev-cluster-testrole}

function e2e_dynamicfile(){
set +e
RoleOutput=$(aws iam get-role --role-name authenticator-dev-cluster-testrole 2>/dev/null)

if [ -z "$RoleOutput" ]; then
sed -e "s|{{AWS_ACCOUNT}}|${AWS_ACCOUNT}|g" \
"${policies_template}" > "${policies_json}"
aws iam create-role --role-name ${AWS_TEST_ROLE} --assume-role-policy-document file://${policies_json} 1>/dev/null
sleep 10
fi

set -e
OUT=$(aws sts assume-role --role-arn arn:aws:iam::${AWS_ACCOUNT}:role/${AWS_TEST_ROLE} --role-session-name aaa);\
export AWS_ACCESS_KEY_ID=$(echo $OUT | jq -r '.Credentials''.AccessKeyId');\
export AWS_SECRET_ACCESS_KEY=$(echo $OUT | jq -r '.Credentials''.SecretAccessKey');\
export AWS_SESSION_TOKEN=$(echo $OUT | jq -r '.Credentials''.SessionToken');

OUT=$(aws sts get-caller-identity|grep "${AWS_TEST_ROLE}")
echo "assumed role: "$OUT
if [ -z "$OUT" ]
then
echo "can't assume-role: "${AWS_TEST_ROLE}
exit 1
fi

#run kubectl cmd without adding the role into access entry
if [ -f ${access_entry_json} ]
then
mv "${access_entry_json}" "${access_entry_tmp}"
fi

sleep 10
set +e
OUT=$(kubectl --kubeconfig=${kubectl_kubeconfig} --context="test-authenticator" get nodes 2>/dev/null)

if [ ! -z "$OUT" ]
then
echo "testing failed"
exit 1
fi
#update access entry to add the test role

sed -e "s|{{AWS_ACCOUNT}}|${AWS_ACCOUNT}|g" \
-e "s|{{AWS_TEST_ROLE}}|${AWS_TEST_ROLE}|g" \
"${access_entry_template}" > "${access_entry_tmp}"
mv "${access_entry_tmp}" "${access_entry_json}"
#sleep 10 seconds to make access entry effective
sleep 10

OUT=$(kubectl --kubeconfig=${kubectl_kubeconfig} --context="test-authenticator" get nodes|grep Ready)
if [ ! -z "$OUT" ]
then
echo $OUT
unset AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN
aws iam delete-role --role-name ${AWS_TEST_ROLE}
echo "end to end testing for dynamicfile mode succeeded"
exit 0

else
echo "testing failed"
exit 1
fi
}





52 changes: 51 additions & 1 deletion hack/lib/dev-env.sh
Original file line number Diff line number Diff line change
Expand Up @@ -53,13 +53,20 @@ authenticator_healthz_port=21363

# Location of templates, config files, mounts
authenticator_config_template="${REPO_ROOT}/hack/dev/authenticator.yaml"
authenticator_access_entry_template="${REPO_ROOT}/hack/dev/access-entries.json"
authenticator_dynamicfile_mode_config_template="${REPO_ROOT}/hack/dev/authenticator_with_dynamicfile_mode.yaml"
kind_config_template="${REPO_ROOT}/hack/dev/env.yaml"
kind_config_host_dir="${OUTPUT}/dev/kind"
kind_config="${kind_config_host_dir}/env.yaml"
authenticator_config_host_dir="${OUTPUT}/dev/authenticator/config"
authenticator_config="${authenticator_config_host_dir}/authenticator.yaml"
authenticator_dynamicfile_mode_config="${authenticator_config_host_dir}/authenticator_dynamicfile_mode.yaml"
authenticator_export_host_dir="${OUTPUT}/dev/authenticator/export"
authenticator_state_host_dir="${OUTPUT}/dev/authenticator/state"
authenticator_dynamicfile_host_path="${OUTPUT}/dev/authenticator/access-entry"
authenticator_access_entry_host_file="${authenticator_dynamicfile_host_path}/access-entries.json"
authenticator_dynamicfile_dest_path="/var/authenticator/access-entry"
authenticator_access_entry_dest_file="${authenticator_dynamicfile_dest_path}/access-entries.json"
authenticator_config_dest_dir="/etc/authenticator"
authenticator_export_dest_dir="/var/authenticator/export"
authenticator_state_dest_dir="/var/authenticator/state"
Expand Down Expand Up @@ -111,6 +118,22 @@ function write_authenticator_config() {
"${authenticator_config_template}" > "${authenticator_config}"
}

function write_authenticator_with_dynamicfile_mode_config() {
mkdir -p "${authenticator_config_host_dir}"
mkdir -p "${authenticator_dynamicfile_host_path}"
sed -e "s|{{ADMIN_ARN}}|${ADMIN_ARN}|g" \
-e "s|{{AUTHENTICATOR_STATE_DIR}}|${authenticator_state_dest_dir}|g" \
-e "s|{{AUTHENTICATOR_GENERATE_KUBECONFIG}}|${authenticator_generate_kubeconfig}|g" \
-e "s|{{AUTHENTICATOR_KUBECONFIG}}|${authenticator_kubeconfig}|g" \
-e "s|{{AUTHENTICATOR_PORT}}|${AUTHENTICATOR_PORT}|g" \
-e "s|{{AUTHENTICATOR_IP}}|${AUTHENTICATOR_IP}|g" \
-e "s|{{CLUSTER_NAME}}|${CLUSTER_NAME}|g" \
-e "s|{{AUTHENTICATOR_DYNAMICFILE_PATH}}|${authenticator_access_entry_dest_file}|g" \
"${authenticator_dynamicfile_mode_config_template}" > "${authenticator_dynamicfile_mode_config}"
cat "${authenticator_dynamicfile_mode_config}"
cp "${authenticator_access_entry_template}" "${authenticator_access_entry_host_file}"
}

function write_kind_config() {
mkdir -p "${kind_config_host_dir}"
sed -e "s|{{AUTHENTICATOR_EXPORT_HOST_DIR}}|${authenticator_export_host_dir}|g" \
Expand Down Expand Up @@ -141,6 +164,33 @@ function start_authenticator() {
--config "${authenticator_config_dest_dir}/authenticator.yaml"
}

function start_authenticator_with_dynamicfile() {
mkdir -p "${authenticator_state_host_dir}"
mkdir -p "${authenticator_export_host_dir}"
chmod -R 777 "${authenticator_state_host_dir}"
chmod -R 777 "${authenticator_export_host_dir}"
chmod -R 777 "${authenticator_dynamicfile_host_path}"
chmod 777 "${authenticator_access_entry_host_file}"

docker run \
--detach \
--ip "${AUTHENTICATOR_IP}" \
--mount "type=bind,src=${authenticator_config_host_dir},dst=${authenticator_config_dest_dir}" \
--mount "type=bind,src=${authenticator_state_host_dir},dst=${authenticator_state_dest_dir}" \
--mount "type=bind,src=${authenticator_export_host_dir},dst=${authenticator_export_dest_dir}" \
--mount "type=bind,src=${authenticator_dynamicfile_host_path},dst=${authenticator_dynamicfile_dest_path}" \
--name aws-iam-authenticator \
--network "${NETWORK_NAME}" \
--publish ${authenticator_healthz_port}:${authenticator_healthz_port} \
--publish ${AUTHENTICATOR_PORT}:${AUTHENTICATOR_PORT} \
--env AWS_REGION="us-west-2" \
--rm \
"${AUTHENTICATOR_IMAGE}" \
server \
--config "${authenticator_config_dest_dir}/authenticator_dynamicfile_mode.yaml"

}

function kill_authenticator() {
docker kill aws-iam-authenticator || true
}
Expand Down Expand Up @@ -179,4 +229,4 @@ function create_kind_cluster() {

function delete_kind_cluster() {
kind delete cluster --name "${CLUSTER_NAME}"
}
}
61 changes: 61 additions & 0 deletions hack/start-dev-env-dynamicfile.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
#!/bin/bash

# Copyright 2021 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -o errexit
set -o pipefail
set -o nounset

#
# An easy way to quickly test local changes.
#
# The approach taken was to start the aws-iam-authenticator as
# a separate docker container first, with a fixed IP and in the
# kind node network. Then, generate the webhook
# config with that IP and start the kind cluster.
#
# Alternative approach: we could run the authenticator pod in
# the kind cluster. This might be a future improvement:
# pregenerate all certificates and config, start the cluster,
# load the target authenticator image, run authenticator as a
# host network pod (same as API server), and all communication
# between them is over localhost and fixed port.

REPO_ROOT="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )/.." &> /dev/null && pwd )"
ADMIN_ARN=$(aws sts get-caller-identity --query Arn --output text)
REPO_NAME=${REPO_NAME:-aws-iam-authenticator}

source "${REPO_ROOT}/hack/lib/dev-env.sh"
source "${REPO_ROOT}/hack/e2e-dynamicfile.sh"

make image
LOCAL_IMAGE_TAG=$(docker images | awk '{print $2}' | awk 'NR==2')
AUTHENTICATOR_IMAGE="${REPO_NAME}":"${LOCAL_IMAGE_TAG}"

echo ${ADMIN_ARN} ${AUTHENTICATOR_IMAGE}

create_network
write_authenticator_with_dynamicfile_mode_config
start_authenticator_with_dynamicfile
sleep 5
echo "Authenticator running at $AUTHENTICATOR_IP"
replace_authenticator_ip
write_kind_config
create_kind_cluster
certificate_authority_data="$(extract_certificate_authority_data)"
write_kubectl_kubeconfig
echo "starting end to end testing for dynamicfile mode"
e2e_dynamicfile

0 comments on commit f2b6111

Please sign in to comment.