-
Notifications
You must be signed in to change notification settings - Fork 38.6k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #44394 from rthallisey/pre-existing-provider
Automatic merge from submit-queue Launch kubemark with an existing Kubemark master In order to expand the use of kubemark, allow developers to use kubemark with a pre-existing Kubernetes cluster. Ref issue #44393
- Loading branch information
Showing
6 changed files
with
230 additions
and
24 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,39 @@ | ||
#!/bin/bash | ||
# Copyright 2017 The Kubernetes Authors. | ||
# | ||
# Licensed under the Apache License, Version 2.0 (the "License"); | ||
# you may not use this file except in compliance with the License. | ||
# You may obtain a copy of the License at | ||
# | ||
# http://www.apache.org/licenses/LICENSE-2.0 | ||
# | ||
# Unless required by applicable law or agreed to in writing, software | ||
# distributed under the License is distributed on an "AS IS" BASIS, | ||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
# See the License for the specific language governing permissions and | ||
# limitations under the License. | ||
|
||
# Configuration for landing a Kubemark cluster on a pre-existing Kubernetes | ||
# cluster. | ||
|
||
# Pre-existing provider expects a MASTER_IP. | ||
# If you need to specify a port that's not the default (443), add it to MASTER_IP. | ||
# | ||
# Example: Connect to the Master on the secure port 6443 | ||
# MASTER_IP=192.168.122.5:6443 | ||
# | ||
MASTER_IP="${MASTER_IP:-}" | ||
|
||
# The container registry and project given to the kubemark container: | ||
# $CONTAINER_REGISTRY/$PROJECT/kubemark | ||
# | ||
CONTAINER_REGISTRY="${CONTAINER_REGISTRY:-}" | ||
PROJECT="${PROJECT:-}" | ||
|
||
NUM_NODES="${NUM_NODES:-1}" | ||
|
||
TEST_CLUSTER_API_CONTENT_TYPE="${TEST_CLUSTER_API_CONTENT_TYPE:-}" | ||
KUBELET_TEST_LOG_LEVEL="${KUBELET_TEST_LOG_LEVEL:-}" | ||
KUBEPROXY_TEST_LOG_LEVEL="${KUBEPROXY_TEST_LOG_LEVEL:-}" | ||
MASTER_NAME="${MASTER_NAME:-}" | ||
USE_REAL_PROXIER="${USE_REAL_PROXIER:-true}" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,62 @@ | ||
#!/bin/bash | ||
|
||
# Copyright 2017 The Kubernetes Authors. | ||
# | ||
# Licensed under the Apache License, Version 2.0 (the "License"); | ||
# you may not use this file except in compliance with the License. | ||
# You may obtain a copy of the License at | ||
# | ||
# http://www.apache.org/licenses/LICENSE-2.0 | ||
# | ||
# Unless required by applicable law or agreed to in writing, software | ||
# distributed under the License is distributed on an "AS IS" BASIS, | ||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
# See the License for the specific language governing permissions and | ||
# limitations under the License. | ||
|
||
# A library of helper functions for landing kubemark containers on a | ||
# pre-existing Kubernetes master. See test/kubemark/pre-existing/README.md | ||
# for me details on using a pre-existing provider. | ||
|
||
KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../.. | ||
|
||
source "${KUBE_ROOT}/cluster/common.sh" | ||
source "${KUBE_ROOT}/hack/lib/util.sh" | ||
|
||
function detect-project() { | ||
if [[ -z "${MASTER_IP:-}" ]]; then | ||
echo "Set 'MASTER_IP' to the instance assigned to be the Kubernetes master" 1>&2 | ||
exit 1 | ||
fi | ||
|
||
if [[ -z "${PROJECT:-}" ]]; then | ||
echo "Set 'PROJECT' to the name of the container project: $CONTAINER_REGISTRY/$PROJECT/kubemark" >&2 | ||
exit 1 | ||
fi | ||
|
||
if [[ -z "${SERVICE_CLUSTER_IP_RANGE:-}" ]]; then | ||
cluster_range=$(echo "${MASTER_IP}" | awk -F '.' '{printf("%d.%d.%d.0", $1, $2, $3)}') | ||
SERVICE_CLUSTER_IP_RANGE="${SERVICE_CLUSTER_IP_RANGE:-$cluster_range/16}" | ||
fi | ||
} | ||
|
||
function create-certs { | ||
rm /tmp/kubeconfig | ||
|
||
execute-cmd-on-pre-existing-master-with-retries "sudo cat /etc/kubernetes/admin.conf" > /tmp/kubeconfig | ||
CA_CERT_BASE64=$(cat /tmp/kubeconfig | grep certificate-authority | awk '{print $2}' | head -n 1) | ||
KUBELET_CERT_BASE64=$(cat /tmp/kubeconfig | grep client-certificate-data | awk '{print $2}' | head -n 1) | ||
KUBELET_KEY_BASE64=$(cat /tmp/kubeconfig | grep client-key-data | awk '{print $2}' | head -n 1) | ||
|
||
# Local kubeconfig.kubemark vars | ||
KUBECFG_CERT_BASE64="${KUBELET_CERT_BASE64}" | ||
KUBECFG_KEY_BASE64="${KUBELET_KEY_BASE64}" | ||
|
||
# The pre-existing Kubernetes master already has these setup | ||
# Set these vars but don't use them | ||
CA_KEY_BASE64=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null) | ||
MASTER_CERT_BASE64=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null) | ||
MASTER_KEY_BASE64=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null) | ||
KUBEAPISERVER_CERT_BASE64=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null) | ||
KUBEAPISERVER_KEY_BASE64=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null) | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,43 @@ | ||
#!/bin/bash | ||
|
||
# Copyright 2017 The Kubernetes Authors. | ||
# | ||
# Licensed under the Apache License, Version 2.0 (the "License"); | ||
# you may not use this file except in compliance with the License. | ||
# You may obtain a copy of the License at | ||
# | ||
# http://www.apache.org/licenses/LICENSE-2.0 | ||
# | ||
# Unless required by applicable law or agreed to in writing, software | ||
# distributed under the License is distributed on an "AS IS" BASIS, | ||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
# See the License for the specific language governing permissions and | ||
# limitations under the License. | ||
|
||
# Running cmd $RETRIES times in case of failures. | ||
function run-cmd-with-retries { | ||
RETRIES="${RETRIES:-3}" | ||
for attempt in $(seq 1 ${RETRIES}); do | ||
exec 5>&1 # Duplicate &1 to &5 for use below. | ||
# We don't use 'local' to declare result as then ret_val always gets value 0. | ||
# We use tee to output to &5 (redirected to stdout) while also storing it in the variable. | ||
result=$("$@" 2>&1 | tee >(cat - >&5)) || local ret_val="$?" | ||
if [[ "${ret_val:-0}" -ne "0" ]]; then | ||
if [[ $(echo "${result}" | grep -c "already exists") -gt 0 ]]; then | ||
if [[ "${attempt}" == 1 ]]; then | ||
echo -e "${color_red}Failed to $1 $2 $3 as the resource hasn't been deleted from a previous run.${color_norm}" >& 2 | ||
exit 1 | ||
fi | ||
echo -e "${color_yellow}Succeeded to $1 $2 $3 in the previous attempt, but status response wasn't received.${color_norm}" | ||
return 0 | ||
fi | ||
echo -e "${color_yellow}Attempt $attempt failed to $1 $2 $3. Retrying.${color_norm}" >& 2 | ||
sleep $(($attempt * 5)) | ||
else | ||
echo -e "${color_green}Succeeded to $1 $2 $3.${color_norm}" | ||
return 0 | ||
fi | ||
done | ||
echo -e "${color_red}Failed to $1 $2 $3.${color_norm}" >& 2 | ||
exit 1 | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,54 @@ | ||
# Kubemark Pre-existing Provider Guide | ||
|
||
**Kubemark Master** | ||
- A set of Kubernetes control plane components running in a VM | ||
|
||
**Kubernetes Cluster** | ||
- A real Kubernetes Cluster that has master and minions. The hollow-node pods | ||
are run in this cluster, but appear as nodes to the Kubemark Master | ||
|
||
## Introduction | ||
|
||
Every running Kubemark setup looks like the following: | ||
1) A running Kubernetes cluster pointed to by the local kubeconfig | ||
2) A separate VM where the kubemark master is running | ||
3) Some hollow-nodes that run on the Kubernetes Cluster from #1 | ||
4) The hollow-nodes are configured to talk with the kubemark master at #2 | ||
|
||
When using the pre-existing provider, the developer is responsible for creating | ||
#1 and #2. Therefore, the kubemark scripts will not create any infrastructure | ||
or start a kubemark master like in other providers. Instead, the existing | ||
resources provided by the VM at $MASTER_IP will serve as the kubemark master. | ||
|
||
## Use Case | ||
|
||
The goal of the pre-existing provider is to use the kubemark tools with an | ||
existing kubermark master. It's meant to provide the developer with | ||
additional flexibility to customize the cluster infrastructure and still use | ||
the kubemark setup tools. The pre-existing provider is an **advanced** use | ||
case that requires the developer to have knowledge of setting up a kubemark | ||
master. | ||
|
||
## Requirements | ||
|
||
To use the pre-existing provider, the expectation is that there's a kubemark | ||
master that is rechable at $MASTER_IP. The machine that the kubemark master is | ||
on has to be ssh able from the host that's executing the kubemark scripts. And | ||
the user on that machine has to be 'kubernetes'. | ||
|
||
Requirement checklist: | ||
- Set MASTER_IP to ip address to the kubemark master | ||
- The host where you execute the kubemark scripts must be able to ssh to | ||
kubernetes@$MASTER_IP | ||
|
||
## Example Configuration | ||
|
||
_test/kubemark/cloud-provider-config.sh_ | ||
|
||
``` | ||
CLOUD_PROVIDER="pre-existing" | ||
KUBEMARK_IMAGE_MAKE_TARGET="push" | ||
CONTAINER_REGISTRY=docker.io | ||
PROJECT="rthallisey" | ||
MASTER_IP="192.168.121.29:6443" | ||
``` |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,27 @@ | ||
#!/bin/bash | ||
|
||
# Copyright 2017 The Kubernetes Authors. | ||
# | ||
# Licensed under the Apache License, Version 2.0 (the "License"); | ||
# you may not use this file except in compliance with the License. | ||
# You may obtain a copy of the License at | ||
# | ||
# http://www.apache.org/licenses/LICENSE-2.0 | ||
# | ||
# Unless required by applicable law or agreed to in writing, software | ||
# distributed under the License is distributed on an "AS IS" BASIS, | ||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
# See the License for the specific language governing permissions and | ||
# limitations under the License. | ||
|
||
KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../../.. | ||
|
||
source "${KUBE_ROOT}/test/kubemark/common/util.sh" | ||
|
||
# Leave the skeleton definition of execute-cmd-on-master-with-retries | ||
# so only the pre-existing provider functions will target this. | ||
function execute-cmd-on-pre-existing-master-with-retries() { | ||
IP_WITHOUT_PORT=$(echo "${MASTER_IP}" | cut -f 1 -d ':') || "${MASTER_IP}" | ||
|
||
RETRIES="${2:-1}" run-cmd-with-retries ssh kubernetes@"${IP_WITHOUT_PORT}" $1 | ||
} |