Skip to content

Commit

Permalink
Address PR comments
Browse files Browse the repository at this point in the history
  • Loading branch information
Ernest Wong committed Jun 1, 2020
1 parent 893ebbb commit a2dfcf2
Show file tree
Hide file tree
Showing 16 changed files with 136 additions and 399 deletions.
4 changes: 1 addition & 3 deletions docs/development.md
Expand Up @@ -335,8 +335,6 @@ To generate the mocks you can run
make generate-go
```



#### E2E Testing

To run E2E locally, set `AZURE_CLIENT_ID`, `AZURE_CLIENT_SECRET`, `AZURE_SUBSCRIPTION_ID`, `AZURE_TENANT_ID` and run:
Expand Down Expand Up @@ -377,7 +375,7 @@ You can optionally set the following variables:
| `PARALLEL` | Skip serial tests and set --ginkgo-parallel. |
| `USE_CI_ARTIFACTS` | Use a CI version of Kubernetes, ie. not a released version (eg. `v1.19.0-alpha.1.426+0926c9c47677e9`) |
| `CI_VERSION` | Provide a custom CI version of Kubernetes. By default, the latest master commit will be used. |
| `USE_MACHINE_POOL` | Use [Machine Pool](topics/machinepools.md) for worker machines. |
| `FEATURE_GATE_MACHINE_POOL` | Use [Machine Pool](topics/machinepools.md) for worker machines. |

You can also customize the configuration of the CAPZ cluster (assuming that `SKIP_CREATE_WORKLOAD_CLUSTER` is not set). See [Customizing the cluster deployment](#customizing-the-cluster-deployment) for more details.

Expand Down
2 changes: 1 addition & 1 deletion hack/gen-flavors.sh
Expand Up @@ -30,4 +30,4 @@ find "${flavors_dir}"* -maxdepth 0 -type d -print0 | xargs -0 -I {} basename {}
mv "${root}/templates/cluster-template-default.yaml" "${root}/templates/cluster-template.yaml"

rm -f "${test_dir}cluster-template"*
find "${test_dir}"* -maxdepth 0 -type d -print0 | xargs -0 -I {} basename {} | xargs -I {} sh -c "${kustomize} build --reorder none ${test_dir}{} > ${test_dir}cluster-template-{}.yaml"
find "${test_dir}"* -maxdepth 0 -type d -print0 | xargs -0 -I {} basename {} | grep -v patches | xargs -I {} sh -c "${kustomize} build --load_restrictor none --reorder none ${test_dir}{} > ${test_dir}cluster-template-{}.yaml"
23 changes: 10 additions & 13 deletions scripts/ci-entrypoint.sh
Expand Up @@ -82,8 +82,8 @@ create_cluster() {
export CLUSTER_TEMPLATE="test/cluster-template-prow.yaml"
fi

if [[ "${USE_MACHINE_POOL:-}" == "true" ]]; then
export CLUSTER_TEMPLATE="${CLUSTER_TEMPLATE/prow/prow-machinepool}"
if [[ "${FEATURE_GATE_MACHINE_POOL:-}" == "true" ]]; then
export CLUSTER_TEMPLATE="${CLUSTER_TEMPLATE/prow/prow-machine-pool}"
fi

export CLUSTER_NAME="capz-$(head /dev/urandom | LC_ALL=C tr -dc a-z0-9 | head -c 6 ; echo '')"
Expand All @@ -101,7 +101,11 @@ wait_for_nodes() {
echo "Waiting for ${CONTROL_PLANE_MACHINE_COUNT} control plane machine and ${WORKER_MACHINE_COUNT} worker machine to become Ready"

local total_nodes="$((${CONTROL_PLANE_MACHINE_COUNT} + ${WORKER_MACHINE_COUNT}))"
timeout --foreground 1800 bash -c "while [[ $(kubectl get nodes | grep "Ready" | wc -l) -ne "${total_nodes}" ]]; do sleep 10; done"
while [[ $(kubectl get nodes -ojson | grep -c '"Ready"') -ne "${total_nodes}" ]]; do
sleep 10
done

kubectl get nodes -owide
}

run_upstream_e2e_tests() {
Expand All @@ -118,19 +122,11 @@ run_upstream_e2e_tests() {
fi
fi

# get the number of worker nodes
NUM_NODES="$(kubectl get nodes --kubeconfig="$KUBECONFIG" \
-o=jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.spec.taints}{"\n"}{end}' \
| grep -cv "node-role.kubernetes.io/master" )"

# wait for all the nodes to be ready
kubectl wait --for=condition=Ready node --kubeconfig="$KUBECONFIG" --all || true

# setting this env prevents ginkg e2e from trying to run provider setup
export KUBERNETES_CONFORMANCE_TEST="y"
# run the tests
(cd "$(go env GOPATH)/src/k8s.io/kubernetes" && ./hack/ginkgo-e2e.sh \
'--provider=skeleton' "--num-nodes=${NUM_NODES}" \
'--provider=skeleton' \
"--ginkgo.focus=${FOCUS}" "--ginkgo.skip=${SKIP}" \
"--report-dir=${ARTIFACTS}" '--disable-log-dump=true')

Expand Down Expand Up @@ -174,7 +170,8 @@ fi
# export the target cluster KUBECONFIG if not already set
export KUBECONFIG="${KUBECONFIG:-${PWD}/kubeconfig}"

wait_for_nodes
export -f wait_for_nodes
timeout --foreground 1800 bash -c wait_for_nodes

# build k8s binaries and run upstream e2e tests
if [[ -z "${SKIP_UPSTREAM_E2E_TESTS:-}" ]]; then
Expand Down
Expand Up @@ -121,121 +121,3 @@ spec:
offer: capi
sku: k8s-1dot18dot2-ubuntu-1804
version: "2020.04.17"
---
apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3
kind: KubeadmConfig
metadata:
name: ${CLUSTER_NAME}-mp-0
spec:
preKubeadmCommands:
- bash -c /tmp/kubeadm-bootstrap.sh
files:
- path: /tmp/kubeadm-bootstrap.sh
owner: "root:root"
permissions: "0744"
content: |
#!/bin/bash
set -o nounset
set -o pipefail
set -o errexit
[[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO=""
GSUTIL=gsutil
if ! command -v $${GSUTIL} > /dev/null; then
curl -sSL https://sdk.cloud.google.com > /tmp/gcl && bash /tmp/gcl --install-dir=~/gcloud --disable-prompts > /dev/null 2>&1
GSUTIL=~/gcloud/google-cloud-sdk/bin/gsutil
# For faster downloads
pip install --no-cache-dir -U crcmod
fi
$${GSUTIL} version
# This test installs release packages or binaries that are a result of the CI and release builds.
# It runs '... --version' commands to verify that the binaries are correctly installed
# and finally uninstalls the packages.
# For the release packages it tests all versions in the support skew.
LINE_SEPARATOR="*************************************************"
echo "$$LINE_SEPARATOR"
CI_VERSION=${CI_VERSION}
if [[ "$${CI_VERSION}" != "" ]]; then
CI_DIR=/tmp/k8s-ci
mkdir -p $$CI_DIR
declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm")
declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler")
CONTAINER_EXT="tar"
echo "* testing CI version $$CI_VERSION"
# Check for semver
if [[ "$${CI_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
CI_URL="gs://kubernetes-release/release/$$CI_VERSION/bin/linux/amd64"
VERSION_WITHOUT_PREFIX="${CI_VERSION#v}"
DEBIAN_FRONTEND=noninteractive apt-get install -y apt-transport-https curl
curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
echo 'deb https://apt.kubernetes.io/ kubernetes-xenial main' > /etc/apt/sources.list.d/kubernetes.list
apt-get update
# replace . with \.
VERSION_REGEX="${VERSION_WITHOUT_PREFIX//./\\.}"
PACKAGE_VERSION="$(apt-cache madison kubelet|grep $${VERSION_REGEX}- | head -n1 | cut -d '|' -f 2 | tr -d '[:space:]')"
for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do
echo "* installing package: $$CI_PACKAGE $${PACKAGE_VERSION}"
DEBIAN_FRONTEND=noninteractive apt-get install -y $$CI_PACKAGE=$$PACKAGE_VERSION
done
else
CI_URL="gs://kubernetes-release-dev/ci/$$CI_VERSION-bazel/bin/linux/amd64"
for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do
echo "* downloading binary: $$CI_URL/$$CI_PACKAGE"
$${GSUTIL} cp "$$CI_URL/$$CI_PACKAGE" "$$CI_DIR/$$CI_PACKAGE"
chmod +x "$$CI_DIR/$$CI_PACKAGE"
mv "$$CI_DIR/$$CI_PACKAGE" "/usr/bin/$$CI_PACKAGE"
done
systemctl restart kubelet
fi
for CI_CONTAINER in "$${CONTAINERS_TO_TEST[@]}"; do
echo "* downloading package: $$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT"
$${GSUTIL} cp "$$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT" "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT"
$${SUDO} ctr -n k8s.io images import "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT" || echo "* ignoring expected 'ctr images import' result"
$${SUDO} ctr -n k8s.io images tag k8s.gcr.io/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" k8s.gcr.io/$$CI_CONTAINER:"$${CI_VERSION//+/_}"
$${SUDO} ctr -n k8s.io images tag k8s.gcr.io/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" gcr.io/kubernetes-ci-images/$$CI_CONTAINER:"$${CI_VERSION//+/_}"
done
fi
echo "* checking binary versions"
echo "ctr version: " $(ctr version)
echo "kubeadm version: " $(kubeadm version -o=short)
echo "kubectl version: " $(kubectl version --client=true --short=true)
echo "kubelet version: " $(kubelet --version)
echo "$$LINE_SEPARATOR"
- path: /etc/kubernetes/azure.json
owner: "root:root"
permissions: "0644"
content: |
{
"cloud": "AzurePublicCloud",
"tenantId": "${AZURE_TENANT_ID}",
"subscriptionId": "${AZURE_SUBSCRIPTION_ID}",
"aadClientId": "${AZURE_CLIENT_ID}",
"aadClientSecret": "${AZURE_CLIENT_SECRET}",
"resourceGroup": "${CLUSTER_NAME}",
"securityGroupName": "${CLUSTER_NAME}-node-nsg",
"location": "${AZURE_LOCATION}",
"vmType": "vmss",
"vnetName": "${CLUSTER_NAME}-vnet",
"vnetResourceGroup": "${CLUSTER_NAME}",
"subnetName": "${CLUSTER_NAME}-node-subnet",
"routeTableName": "${CLUSTER_NAME}-node-routetable",
"loadBalancerSku": "standard",
"maximumLoadBalancerRuleCount": 250,
"useManagedIdentityExtension": false,
"useInstanceMetadata": true
}
---
apiVersion: exp.infrastructure.cluster.x-k8s.io/v1alpha3
kind: AzureMachinePool
metadata:
name: ${CLUSTER_NAME}-mp-0
spec:
template:
image:
# we use the 1.18.2 image as a workaround there is no published marketplace image for k8s CI versions.
# 1.18.2 binaries and images will get replaced to the desired version by the script above.
marketplace:
publisher: cncf-upstream
offer: capi
sku: k8s-1dot18dot2-ubuntu-1804
version: "2020.04.17"
File renamed without changes.
3 changes: 2 additions & 1 deletion templates/test/prow-ci-version/kustomization.yaml
Expand Up @@ -4,4 +4,5 @@ namespace: default
resources:
- ../prow
patchesStrategicMerge:
- patches/ci-version.yaml
- ../patches/control-plane-ci-version.yaml
- patches/machine-deployment-ci-version.yaml
Expand Up @@ -2,6 +2,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namespace: default
resources:
- ../prow-machinepool
- ../prow-machine-pool
patchesStrategicMerge:
- patches/ci-version.yaml
- ../patches/control-plane-ci-version.yaml
- patches/machine-pool-ci-version.yaml
@@ -0,0 +1,117 @@
apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3
kind: KubeadmConfig
metadata:
name: ${CLUSTER_NAME}-mp-0
spec:
preKubeadmCommands:
- bash -c /tmp/kubeadm-bootstrap.sh
files:
- path: /tmp/kubeadm-bootstrap.sh
owner: "root:root"
permissions: "0744"
content: |
#!/bin/bash
set -o nounset
set -o pipefail
set -o errexit
[[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO=""
GSUTIL=gsutil
if ! command -v $${GSUTIL} > /dev/null; then
curl -sSL https://sdk.cloud.google.com > /tmp/gcl && bash /tmp/gcl --install-dir=~/gcloud --disable-prompts > /dev/null 2>&1
GSUTIL=~/gcloud/google-cloud-sdk/bin/gsutil
# For faster downloads
pip install --no-cache-dir -U crcmod
fi
$${GSUTIL} version
# This test installs release packages or binaries that are a result of the CI and release builds.
# It runs '... --version' commands to verify that the binaries are correctly installed
# and finally uninstalls the packages.
# For the release packages it tests all versions in the support skew.
LINE_SEPARATOR="*************************************************"
echo "$$LINE_SEPARATOR"
CI_VERSION=${CI_VERSION}
if [[ "$${CI_VERSION}" != "" ]]; then
CI_DIR=/tmp/k8s-ci
mkdir -p $$CI_DIR
declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm")
declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler")
CONTAINER_EXT="tar"
echo "* testing CI version $$CI_VERSION"
# Check for semver
if [[ "$${CI_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
CI_URL="gs://kubernetes-release/release/$$CI_VERSION/bin/linux/amd64"
VERSION_WITHOUT_PREFIX="${CI_VERSION#v}"
DEBIAN_FRONTEND=noninteractive apt-get install -y apt-transport-https curl
curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
echo 'deb https://apt.kubernetes.io/ kubernetes-xenial main' > /etc/apt/sources.list.d/kubernetes.list
apt-get update
# replace . with \.
VERSION_REGEX="${VERSION_WITHOUT_PREFIX//./\\.}"
PACKAGE_VERSION="$(apt-cache madison kubelet|grep $${VERSION_REGEX}- | head -n1 | cut -d '|' -f 2 | tr -d '[:space:]')"
for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do
echo "* installing package: $$CI_PACKAGE $${PACKAGE_VERSION}"
DEBIAN_FRONTEND=noninteractive apt-get install -y $$CI_PACKAGE=$$PACKAGE_VERSION
done
else
CI_URL="gs://kubernetes-release-dev/ci/$$CI_VERSION-bazel/bin/linux/amd64"
for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do
echo "* downloading binary: $$CI_URL/$$CI_PACKAGE"
$${GSUTIL} cp "$$CI_URL/$$CI_PACKAGE" "$$CI_DIR/$$CI_PACKAGE"
chmod +x "$$CI_DIR/$$CI_PACKAGE"
mv "$$CI_DIR/$$CI_PACKAGE" "/usr/bin/$$CI_PACKAGE"
done
systemctl restart kubelet
fi
for CI_CONTAINER in "$${CONTAINERS_TO_TEST[@]}"; do
echo "* downloading package: $$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT"
$${GSUTIL} cp "$$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT" "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT"
$${SUDO} ctr -n k8s.io images import "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT" || echo "* ignoring expected 'ctr images import' result"
$${SUDO} ctr -n k8s.io images tag k8s.gcr.io/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" k8s.gcr.io/$$CI_CONTAINER:"$${CI_VERSION//+/_}"
$${SUDO} ctr -n k8s.io images tag k8s.gcr.io/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" gcr.io/kubernetes-ci-images/$$CI_CONTAINER:"$${CI_VERSION//+/_}"
done
fi
echo "* checking binary versions"
echo "ctr version: " $(ctr version)
echo "kubeadm version: " $(kubeadm version -o=short)
echo "kubectl version: " $(kubectl version --client=true --short=true)
echo "kubelet version: " $(kubelet --version)
echo "$$LINE_SEPARATOR"
- path: /etc/kubernetes/azure.json
owner: "root:root"
permissions: "0644"
content: |
{
"cloud": "AzurePublicCloud",
"tenantId": "${AZURE_TENANT_ID}",
"subscriptionId": "${AZURE_SUBSCRIPTION_ID}",
"aadClientId": "${AZURE_CLIENT_ID}",
"aadClientSecret": "${AZURE_CLIENT_SECRET}",
"resourceGroup": "${CLUSTER_NAME}",
"securityGroupName": "${CLUSTER_NAME}-node-nsg",
"location": "${AZURE_LOCATION}",
"vmType": "vmss",
"vnetName": "${CLUSTER_NAME}-vnet",
"vnetResourceGroup": "${CLUSTER_NAME}",
"subnetName": "${CLUSTER_NAME}-node-subnet",
"routeTableName": "${CLUSTER_NAME}-node-routetable",
"loadBalancerSku": "standard",
"maximumLoadBalancerRuleCount": 250,
"useManagedIdentityExtension": false,
"useInstanceMetadata": true
}
---
apiVersion: exp.infrastructure.cluster.x-k8s.io/v1alpha3
kind: AzureMachinePool
metadata:
name: ${CLUSTER_NAME}-mp-0
spec:
template:
image:
# we use the 1.18.2 image as a workaround there is no published marketplace image for k8s CI versions.
# 1.18.2 binaries and images will get replaced to the desired version by the script above.
marketplace:
publisher: cncf-upstream
offer: capi
sku: k8s-1dot18dot2-ubuntu-1804
version: "2020.04.17"
Expand Up @@ -4,4 +4,4 @@ namespace: default
resources:
- ../../flavors/machinepool
patchesStrategicMerge:
- patches/tags.yaml
- ../patches/tags.yaml
2 changes: 1 addition & 1 deletion templates/test/prow/kustomization.yaml
Expand Up @@ -4,4 +4,4 @@ namespace: default
resources:
- ../../flavors/default
patchesStrategicMerge:
- patches/tags.yaml
- ../patches/tags.yaml
8 changes: 0 additions & 8 deletions templates/test/prow/patches/tags.yaml

This file was deleted.

7 changes: 0 additions & 7 deletions templates/test/prow/prow-ci-version/kustomization.yaml

This file was deleted.

0 comments on commit a2dfcf2

Please sign in to comment.