diff --git a/cluster/lib/logging.sh b/cluster/lib/logging.sh new file mode 100644 index 0000000000..1c74f66528 --- /dev/null +++ b/cluster/lib/logging.sh @@ -0,0 +1,171 @@ +#!/bin/bash + +# Copyright 2014 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Controls verbosity of the script output and logging. +KUBE_VERBOSE="${KUBE_VERBOSE:-5}" + +# Handler for when we exit automatically on an error. +# Borrowed from https://gist.github.com/ahendrix/7030300 +kube::log::errexit() { + local err="${PIPESTATUS[@]}" + + # If the shell we are in doesn't have errexit set (common in subshells) then + # don't dump stacks. + set +o | grep -qe "-o errexit" || return + + set +o xtrace + local code="${1:-1}" + # Print out the stack trace described by $function_stack + if [ ${#FUNCNAME[@]} -gt 2 ] + then + kube::log::error "Call tree:" + for ((i=1;i<${#FUNCNAME[@]}-1;i++)) + do + kube::log::error " $i: ${BASH_SOURCE[$i+1]}:${BASH_LINENO[$i]} ${FUNCNAME[$i]}(...)" + done + fi + kube::log::error_exit "Error in ${BASH_SOURCE[1]}:${BASH_LINENO[0]}. '${BASH_COMMAND}' exited with status $err" "${1:-1}" 1 +} + +kube::log::install_errexit() { + # trap ERR to provide an error handler whenever a command exits nonzero this + # is a more verbose version of set -o errexit + trap 'kube::log::errexit' ERR + + # setting errtrace allows our ERR trap handler to be propagated to functions, + # expansions and subshells + set -o errtrace +} + +# Print out the stack trace +# +# Args: +# $1 The number of stack frames to skip when printing. +kube::log::stack() { + local stack_skip=${1:-0} + stack_skip=$((stack_skip + 1)) + if [[ ${#FUNCNAME[@]} -gt $stack_skip ]]; then + echo "Call stack:" >&2 + local i + for ((i=1 ; i <= ${#FUNCNAME[@]} - $stack_skip ; i++)) + do + local frame_no=$((i - 1 + stack_skip)) + local source_file=${BASH_SOURCE[$frame_no]} + local source_lineno=${BASH_LINENO[$((frame_no - 1))]} + local funcname=${FUNCNAME[$frame_no]} + echo " $i: ${source_file}:${source_lineno} ${funcname}(...)" >&2 + done + fi +} + +# Log an error and exit. +# Args: +# $1 Message to log with the error +# $2 The error code to return +# $3 The number of stack frames to skip when printing. +kube::log::error_exit() { + local message="${1:-}" + local code="${2:-1}" + local stack_skip="${3:-0}" + stack_skip=$((stack_skip + 1)) + + if [[ ${KUBE_VERBOSE} -ge 4 ]]; then + local source_file=${BASH_SOURCE[$stack_skip]} + local source_line=${BASH_LINENO[$((stack_skip - 1))]} + echo "!!! Error in ${source_file}:${source_line}" >&2 + [[ -z ${1-} ]] || { + echo " ${1}" >&2 + } + + kube::log::stack $stack_skip + + echo "Exiting with status ${code}" >&2 + fi + + exit "${code}" +} + +# Log an error but keep going. Don't dump the stack or exit. +kube::log::error() { + timestamp=$(date +"[%m%d %H:%M:%S]") + echo "!!! $timestamp ${1-}" >&2 + shift + for message; do + echo " $message" >&2 + done +} + +# Print an usage message to stderr. The arguments are printed directly. +kube::log::usage() { + echo >&2 + local message + for message; do + echo "$message" >&2 + done + echo >&2 +} + +kube::log::usage_from_stdin() { + local messages=() + while read -r line; do + messages+=("$line") + done + + kube::log::usage "${messages[@]}" +} + +# Print out some info that isn't a top level status line +kube::log::info() { + local V="${V:-0}" + if [[ $KUBE_VERBOSE < $V ]]; then + return + fi + + for message; do + echo "$message" + done +} + +# Just like kube::log::info, but no \n, so you can make a progress bar +kube::log::progress() { + for message; do + echo -e -n "$message" + done +} + +kube::log::info_from_stdin() { + local messages=() + while read -r line; do + messages+=("$line") + done + + kube::log::info "${messages[@]}" +} + +# Print a status line. Formatted to show up in a stream of output. +kube::log::status() { + local V="${V:-0}" + if [[ $KUBE_VERBOSE < $V ]]; then + return + fi + + timestamp=$(date +"[%m%d %H:%M:%S]") + echo "+++ $timestamp $1" + shift + for message; do + echo " $message" + done +} diff --git a/hack/dind-cluster-v1.12.sh b/hack/dind-cluster-v1.12.sh new file mode 100644 index 0000000000..1003e6722f --- /dev/null +++ b/hack/dind-cluster-v1.12.sh @@ -0,0 +1,2389 @@ +#!/bin/bash +# Copyright 2018 Mirantis +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -o errexit +set -o nounset +set -o pipefail +set -o errtrace + +if [ $(uname) = Darwin ]; then + readlinkf(){ perl -MCwd -e 'print Cwd::abs_path shift' "$1";} +else + readlinkf(){ readlink -f "$1"; } +fi +DIND_ROOT="$(cd $(dirname "$(readlinkf "${BASH_SOURCE}")"); pwd)" + +RUN_ON_BTRFS_ANYWAY="${RUN_ON_BTRFS_ANYWAY:-}" +if [[ ! ${RUN_ON_BTRFS_ANYWAY} ]] && docker info| grep -q '^Storage Driver: btrfs'; then + echo "ERROR: Docker is using btrfs storage driver which is unsupported by kubeadm-dind-cluster" >&2 + echo "Please refer to the documentation for more info." >&2 + echo "Set RUN_ON_BTRFS_ANYWAY to non-empty string to continue anyway." >&2 + exit 1 +fi + +# In case of linuxkit / moby linux, -v will not work so we can't +# mount /lib/modules and /boot. Also we'll be using localhost +# to access the apiserver. +using_linuxkit= +if ! docker info|grep -s '^Operating System: .*Docker for Windows' > /dev/null 2>&1 ; then + if docker info|grep -s '^Kernel Version: .*-moby$' >/dev/null 2>&1 || + docker info|grep -s '^Kernel Version: .*-linuxkit' > /dev/null 2>&1 ; then + using_linuxkit=1 + fi +fi + +# Determine when using Linux and docker daemon running locally +using_local_linuxdocker= +if [[ $(uname) == Linux && -z ${DOCKER_HOST:-} ]]; then + using_local_linuxdocker=1 +fi + +EMBEDDED_CONFIG=y;DOWNLOAD_KUBECTL=y;DIND_K8S_VERSION=v1.12;DIND_IMAGE_DIGEST=sha256:a6534a4425b0427c32f420a2d38c09964327c5515da4f5620f82876507cc8afd;DIND_COMMIT=596f7d093470c1dc3a3e4466bcdfb34438a99b90 + +# dind::localhost provides the local host IP based on the address family used for service subnet. +function dind::localhost() { + if [[ ${SERVICE_NET_MODE} = "ipv6" ]]; then + echo '[::1]' + else + echo '127.0.0.1' + fi +} + +# dind::family-for indicates whether the CIDR or IP is for an IPv6 or IPv4 family. +function dind::family-for { + local addr=$1 + if [[ "$addr" = *":"* ]]; then + echo "ipv6" + else + echo "ipv4" + fi +} + +# dind::cluster-suffix builds a suffix used for resources, based on the DIND_LABEL. +function dind::cluster-suffix { + if [ "$DIND_LABEL" != "$DEFAULT_DIND_LABEL" ]; then + echo "-${DIND_LABEL}" + else + echo '' + fi +} + +function dind::net-name { + echo "kubeadm-dind-net$( dind::cluster-suffix )" +} + +# dind::add-cluster will inject the cluster ID to the IP address. For IPv4, it is +# assumed that the IP is a /24 with the third part of the address available for cluster ID. +# For IPv6, it is assumed that there is enough space for the cluster to be added, and the +# cluster ID will be added to the 16 bits before the double colon. For example: +# +# 10.192.0.0/24 => 10.192.5.0/24 +# fd00:77:20::/64 => fd00:77:20:5::/64 +# +# This function is intended to be used for management networks. +# +# TODO: Validate that there is enough space for cluster ID. +# TODO: For IPv6 could get fancy and handle case where cluster ID is placed in upper 8 bits of hextet +# TODO: Consider if want to do /16 for IPv4 management subnet. +# +function dind::add-cluster { + local cidr=$1 + local ip_mode=$2 + + if [[ ${ip_mode} = "ipv4" ]]; then + echo ${cidr} | sed "s/^\([0-9]*\.[0-9]*\.\).*\/24$/\1${CLUSTER_ID}.0\/24/" + else # IPv6 + echo ${cidr} | sed "s/^\(.*\)\(\:\:\/[0-9]*\)$/\1:${CLUSTER_ID}\2/" + fi +} + +# dind::get-and-validate-cidrs takes a list of CIDRs and validates them based on the ip +# mode, returning them. For IPv4 only and IPv6 only modes, only one CIDR is expected. For +# dual stack, two CIDRS are expected. It verifies that the CIDRs are the right family and +# will use the provided defaults, when CIDRs are missing. For dual-stack, the IPv4 address +# will be first. +# +# For the management network, the cluster ID will be injected into the CIDR. Also, if no +# MGMT_CIDRS value is specified, but the legacy DIND_SUBNET/DIND_SUBNET_SIZE is provided, +# that will be used for the (first) CIDR. +# +# NOTE: It is expected that the CIDR size is /24 for IPv4 management networks. +# +# For pod CIDRs, the size will be increased by 8, to leave room for the node ID to be +# injected into the address. +# +# NOTE: For IPv4, the pod size is expected to be /16 -> /24 in usage. +# +function dind::get-and-validate-cidrs { + IFS=', ' read -r -a cidrs <<< "$1" + IFS=', ' read -r -a defaults <<< "$2" + local is_mgmt=$3 + case ${IP_MODE} in + ipv4) + case ${#cidrs[@]} in + 0) + cidrs[0]="${defaults[0]}" + ;; + 1) + ;; + *) + echo "ERROR! More than one CIDR provided '$1'" + exit 1 + ;; + esac + if [[ $( dind::family-for "${cidrs[0]}" ) != "ipv4" ]]; then + echo "ERROR! CIDR must be IPv4 value" + exit 1 + fi + if [[ ${is_mgmt} = true ]]; then + cidrs[0]="$( dind::add-cluster "${cidrs[0]}" "${IP_MODE}" )" + fi + ;; + + ipv6) + case ${#cidrs[@]} in + 0) + cidrs[0]="${defaults[0]}" + ;; + 1) + ;; + *) + echo "ERROR! More than one CIDR provided '$1'" + exit 1 + ;; + esac + if [[ $( dind::family-for "${cidrs[0]}" ) != "ipv6" ]]; then + echo "ERROR! CIDR must be IPv6 value" + exit 1 + fi + if [[ ${is_mgmt} = true ]]; then + cidrs[0]="$( dind::add-cluster "${cidrs[0]}" "${IP_MODE}" )" + fi + ;; + + dual-stack) + case ${#cidrs[@]} in + 0) + cidrs[0]="${defaults[0]}" + cidrs[1]="${defaults[1]}" + ;; + 1) + if [[ $( dind::family-for "${cidrs[0]}" ) = "ipv6" ]]; then + cidrs[1]=${cidrs[0]} + cidrs[0]="${defaults[0]}" # Assuming first default is a V4 address + else + cidrs[1]="${defaults[1]}" + fi + ;; + 2) + # Force ordering to have V4 address first + if [[ $( dind::family-for "${cidrs[0]}" ) = "ipv6" ]]; then + local temp=${cidrs[0]} + cidrs[0]=${cidrs[1]} + cidrs[1]=${temp} + fi + ;; + *) + echo "ERROR! More than two CIDRs provided '$1'" + exit 1 + ;; + esac + local have_v4="" + local have_v6="" + for cidr in ${cidrs[@]}; do + if [[ $( dind::family-for "${cidr}" ) = "ipv6" ]]; then + have_v6=1 + else + have_v4=1 + fi + done + if [[ -z ${have_v4} ]]; then + echo "ERROR! Missing IPv4 CIDR in '$1'" + exit 1 + fi + if [[ -z ${have_v6} ]]; then + echo "ERROR! Missing IPv6 CIDR in '$1'" + exit 1 + fi + if [[ ${is_mgmt} = true ]]; then + cidrs[0]="$( dind::add-cluster "${cidrs[0]}" "${IP_MODE}" )" + cidrs[1]="$( dind::add-cluster "${cidrs[1]}" "${IP_MODE}" )" + fi + ;; + esac + echo "${cidrs[@]}" +} + +# dind::make-ip-from-cidr strips off the slash and size, and appends the +# interface part to the prefix to form an IP. For IPv4, it strips off the +# fourth part of the prefix, so that it can be replaced. It assumes that the +# resulting prefix will be of sufficient size. It also will use hex for the +# appended part for IPv6, and decimal for IPv4. +# +# fd00:20::/64 -> fd00:20::a +# 10.96.0.0/12 -> 10.96.0.10 +# +function dind::make-ip-from-cidr { + prefix="$(echo $1 | sed 's,/.*,,')" + if [[ $( dind::family-for ${prefix} ) == "ipv4" ]]; then + printf "%s%d" $( echo ${prefix} | sed 's/0$//' ) $2 + else + printf "%s%x" ${prefix} $2 + fi +} + +# dind::add-cluster-id-and-validate-nat64-prefix will modify the IPv4 mapping +# subnet prefix, by adding the cluster ID (default 0) to the second octet. +# It will produce an error, if the prefix is not in the 10.0.0.0/8 or +# 172.16.0.0/12 private networks. +function dind::add-cluster-id-and-validate-nat64-prefix { + local parts + IFS="." read -a parts <<<${NAT64_V4_SUBNET_PREFIX} + if [[ ${#parts[@]} -ne 2 ]]; then + echo "ERROR! NAT64_V4_SUBNET_PREFIX must be two octets (have '${NAT64_V4_SUBNET_PREFIX}')" + exit 1 + fi + (( parts[1]+=${CLUSTER_ID} )) + NAT64_V4_SUBNET_PREFIX="${parts[0]}.${parts[1]}" + echo "Added cluster ID offset (${CLUSTER_ID}) to NAT64_V4_SUBNET_PREFIX giving prefix '${NAT64_V4_SUBNET_PREFIX}'" + if [[ ${parts[0]} -eq 10 ]]; then + if [[ ${parts[1]} > 253 ]]; then + echo "ERROR! NAT64_V4_SUBNET_PREFIX is too large for 10.0.0.0/8 private net" + exit 1 + fi + elif [[ ${parts[0]} -eq 172 ]]; then + if [[ ${parts[1]} -lt 16 || ${parts[1]} -gt 31 ]]; then + echo "ERROR! NAT64_V4_SUBNET_PREFIX is outside of range for 172.16.0.0/12 private net" + exit 1 + fi + else + echo "ERROR! NAT64_V4_SUBNET_PREFIX is not in 10.0.0.0/8 or 172.16.0.0/12 private networks" + exit 1 + fi + echo "Using NAT64 V4 mapping network prefix: ${NAT64_V4_SUBNET_PREFIX}" +} + + +# START OF PROCESSING... + +IP_MODE="${IP_MODE:-ipv4}" # ipv4, ipv6, dual-stack +# FUTURE: Once dual-stack support is released, check K8s version, and reject for older versions. +if [[ ! ${EMBEDDED_CONFIG:-} ]]; then + source "${DIND_ROOT}/config.sh" +fi + +# Multicluster support +# Users can specify a cluster ID number from 1..254, represented as a string. +# This will be used to form resource names "cluster-#", and will be used in the +# management subnet to give unique networks for each cluster. If the cluster ID +# is not specified, or zero, it will be considered a single cluster or the first +# in the multi-cluster. This is the recommended usage. +# +# For legacy support, the user can specify DIND_LABEL, which will be used in the +# resource names. If a cluster ID is specified (a hybrid case, where people are +# using the new method, but want custom names), the resourse name will have the +# suffix "-#" with the cluster ID. If no cluster ID is specified (for backward +# compatibility), then the resource name will be just the DIND_LABEL, and a pseudo- +# random number from 1..13 will be generated for the cluster ID to be used in +# management network. The range is limited, because, in IPv6 mode, the cluster ID +# is used in the NAT64 V4 subnet prefix, which must be in a private network. +# The default is 172.18, so the cluster ID cannot be larger than 13 to guarantee +# a valid value. +# +# To get around that limitation, you can set the cluster ID, in addition to the +# DIND_LABEL, and optionally, change the NAT64_V4_SUBNET_PREFIX value. +# +DEFAULT_DIND_LABEL='mirantis.kubeadm_dind_cluster_runtime' +if [[ -z ${DIND_LABEL+x} ]]; then # No legacy DIND_LABEL set + if [[ -z ${CLUSTER_ID+x} ]]; then # No cluster ID set + DIND_LABEL=${DEFAULT_DIND_LABEL} # Single cluster mode + CLUSTER_ID="0" + else # Have cluster ID + if [[ ${CLUSTER_ID} = "0" ]]; then + DIND_LABEL=${DEFAULT_DIND_LABEL} # Single cluster mode or first cluster of multi-cluster + else + DIND_LABEL="cluster-${CLUSTER_ID}" # Multi-cluster + fi + fi +else # Legacy DIND_LABEL set for multi-cluster + if [[ -z ${CLUSTER_ID+x} ]]; then # No cluster ID set, make one from 1..13, but don't use in resource names + CLUSTER_ID="$(( ($RANDOM % 12) + 1 ))" + else + if [[ ${CLUSTER_ID} = "0" ]]; then + CLUSTER_ID="$(( ($RANDOM % 12) + 1 ))" # Force a pseudo-random cluster for additional legacy cluster + else + DIND_LABEL="${DIND_LABEL}-${CLUSTER_ID}" + fi + fi +fi + +CNI_PLUGIN="${CNI_PLUGIN:-bridge}" +GCE_HOSTED="${GCE_HOSTED:-}" +DIND_ALLOW_AAAA_USE="${DIND_ALLOW_AAAA_USE:-}" # Default is to use DNS64 always for IPv6 mode +KUBE_ROUTER_VERSION="${KUBE_ROUTER_VERSION:-v0.2.0}" + +# Use legacy DIND_SUBNET/DIND_SUBNET_SIZE, only if MGMT_CIDRS is not set. +legacy_mgmt_cidr="" +if [[ ${DIND_SUBNET:-} && ${DIND_SUBNET_SIZE:-} ]]; then + legacy_mgmt_cidr="${DIND_SUBNET}/${DIND_SUBNET_SIZE}" +fi + +if [[ ${IP_MODE} = "dual-stack" ]]; then + mgmt_net_defaults="10.192.0.0/24, fd00:20::/64" + + KUBE_RSYNC_ADDR="${KUBE_RSYNC_ADDR:-::1}" + SERVICE_CIDR="${SERVICE_CIDR:-fd00:30::/110}" # Will default to IPv6 service net family + + pod_net_defaults="10.244.0.0/16, fd00:40::/72" + + USE_HAIRPIN="${USE_HAIRPIN:-true}" # Default is to use hairpin for dual-stack + DIND_ALLOW_AAAA_USE=true # Forced, so can access external hosts via IPv6 + if [[ ${DIND_ALLOW_AAAA_USE} && ${GCE_HOSTED} ]]; then + echo "ERROR! GCE does not support use of IPv6 for external addresses - aborting." + exit 1 + fi +elif [[ ${IP_MODE} = "ipv6" ]]; then + mgmt_net_defaults="fd00:20::/64" + + KUBE_RSYNC_ADDR="${KUBE_RSYNC_ADDR:-::1}" + SERVICE_CIDR="${SERVICE_CIDR:-fd00:30::/110}" + + pod_net_defaults="fd00:40::/72" + + USE_HAIRPIN="${USE_HAIRPIN:-true}" # Default is to use hairpin for IPv6 + if [[ ${DIND_ALLOW_AAAA_USE} && ${GCE_HOSTED} ]]; then + echo "ERROR! GCE does not support use of IPv6 for external addresses - aborting." + exit 1 + fi +else # IPv4 mode + mgmt_net_defaults="10.192.0.0/24" + + KUBE_RSYNC_ADDR="${KUBE_RSYNC_ADDR:-127.0.0.1}" + SERVICE_CIDR="${SERVICE_CIDR:-10.96.0.0/12}" + + pod_net_defaults="10.244.0.0/16" + + USE_HAIRPIN="${USE_HAIRPIN:-false}" # Disabled for IPv4, as issue with Virtlet networking + if [[ ${DIND_ALLOW_AAAA_USE} ]]; then + echo "WARNING! The DIND_ALLOW_AAAA_USE option is for IPv6 mode - ignoring setting." + DIND_ALLOW_AAAA_USE= + fi + if [[ ${CNI_PLUGIN} = "calico" || ${CNI_PLUGIN} = "calico-kdd" ]]; then + pod_net_defaults="192.168.0.0/16" + fi +fi + +IFS=' ' read -r -a mgmt_net_cidrs <<<$( dind::get-and-validate-cidrs "${MGMT_CIDRS:-${legacy_mgmt_cidr}}" "${mgmt_net_defaults[@]}" true ) + +REMOTE_DNS64_V4SERVER="${REMOTE_DNS64_V4SERVER:-8.8.8.8}" +if [[ ${IP_MODE} == "ipv6" ]]; then + # Uses local DNS64 container + dns_server="$( dind::make-ip-from-cidr ${mgmt_net_cidrs[0]} 0x100 )" + DNS64_PREFIX="${DNS64_PREFIX:-fd00:10:64:ff9b::}" + DNS64_PREFIX_SIZE="${DNS64_PREFIX_SIZE:-96}" + DNS64_PREFIX_CIDR="${DNS64_PREFIX}/${DNS64_PREFIX_SIZE}" + + LOCAL_NAT64_SERVER="$( dind::make-ip-from-cidr ${mgmt_net_cidrs[0]} 0x200 )" + NAT64_V4_SUBNET_PREFIX="${NAT64_V4_SUBNET_PREFIX:-172.18}" + dind::add-cluster-id-and-validate-nat64-prefix +else + dns_server="${REMOTE_DNS64_V4SERVER}" +fi + +SERVICE_NET_MODE="$( dind::family-for ${SERVICE_CIDR} )" +DNS_SVC_IP="$( dind::make-ip-from-cidr ${SERVICE_CIDR} 10 )" + +ETCD_HOST="${ETCD_HOST:-$( dind::localhost )}" + +IFS=' ' read -r -a pod_net_cidrs <<<$( dind::get-and-validate-cidrs "${POD_NETWORK_CIDR:-}" "${pod_net_defaults[@]}" false ) + +declare -a pod_prefixes +declare -a pod_sizes +# Extract the prefix and size from the provided pod CIDR(s), based on the IP mode of each. The +# size will be increased by 8, to make room for the node ID to be added to the prefix later. +# Bridge and PTP plugins can process IPv4 and IPv6 pod CIDRs, other plugins must be IPv4 only. +for pod_cidr in "${pod_net_cidrs[@]}"; do + if [[ $( dind::family-for "${pod_cidr}" ) = "ipv4" ]]; then + actual_size=$( echo ${pod_cidr} | sed 's,.*/,,' ) + if [[ ${actual_size} -ne 16 ]]; then + echo "ERROR! For IPv4 CIDRs, the size must be /16. Have '${pod_cidr}'" + exit 1 + fi + pod_sizes+=( 24 ) + pod_prefixes+=( "$(echo ${pod_cidr} | sed 's/^\([0-9]*\.[0-9]*\.\).*/\1/')" ) + else # IPv6 + if [[ ${CNI_PLUGIN} != "bridge" && ${CNI_PLUGIN} != "ptp" ]]; then + echo "ERROR! IPv6 pod networks are only supported by bridge and PTP CNI plugins" + exit 1 + fi + # There are several cases to address. First, is normal split of prefix and size: + # fd00:10:20:30::/64 ---> fd00:10:20:30: /72 + # + # Second, is when the prefix needs to be padded, so that node ID can be added later: + # fd00:10::/64 ---> fd00:10:0:0: /72 + # + # Third, is when the low order part of the address, must be removed for the prefix, + # as the node ID will be placed in the lower byte: + # fd00:10:20:30:4000::/72 ---> fd00:10:20:30:40 /80 + # + # We will attempt to check for three error cases. One is when the address part is + # way too big for the size specified: + # fd00:10:20:30:40::/48 ---> fd00:10:20: /56 desired, but conflict with 30:40: + # + # Another is when the address part, once trimmed for the size, would loose info: + # fd00:10:20:1234::/56 ---> fd00:10:20:12 /64, but lost 34:, which conflicts + # + # Lastly, again, trimming would leave high byte in hextet, conflicting with + # the node ID: + # fd00:10:20:30:1200::/64 ---> fd00:10:20:30:12 /72, but 12 conflicts + # + # Note: later, the node ID will be appended to the prefix generated. + # + cluster_size="$(echo ${pod_cidr} | sed 's,.*::/,,')" + pod_sizes+=( $((${cluster_size}+8)) ) + + pod_prefix="$(echo ${pod_cidr} | sed 's,::/.*,:,')" + num_colons="$(grep -o ":" <<< "${pod_prefix}" | wc -l)" + need_zero_pads=$((${cluster_size}/16)) + + if [[ ${num_colons} -gt $((need_zero_pads + 1)) ]]; then + echo "ERROR! Address part of CIDR (${pod_prefix}) is too large for /${cluster_size}" + exit 1 + fi + if [[ ${num_colons} -gt ${need_zero_pads} ]]; then + # Will be replacing lowest byte with node ID, so pull off lower byte and colon + if [[ ${pod_prefix: -3} != "00:" ]]; then # last byte is not zero + echo "ERROR! Cannot trim address part of CIDR (${pod_prefix}) to fit in /${cluster_size}" + exit 1 + fi + pod_prefix=${pod_prefix::-3} + if [[ $(( ${cluster_size} % 16 )) -eq 0 && $( ${pod_prefix: -1} ) != ":" ]]; then # should not be upper byte for this size CIDR + echo "ERROR! Trimmed address part of CIDR (${pod_prefix}) is still too large for /${cluster_size}" + exit 1 + fi + fi + # Add in zeros to pad 16 bits at a time, up to the padding needed, which is + # need_zero_pads - num_colons. + while [ ${num_colons} -lt ${need_zero_pads} ]; do + pod_prefix+="0:" + ((num_colons++)) + done + pod_prefixes+=( "${pod_prefix}" ) + fi +done + +DIND_IMAGE_BASE="${DIND_IMAGE_BASE:-mirantis/kubeadm-dind-cluster}" +if [[ ${DIND_COMMIT:-} ]]; then + if [[ ${DIND_COMMIT} = current ]]; then + DIND_COMMIT="$(cd "${DIND_ROOT}"; git rev-parse HEAD)" + fi + DIND_K8S_VERSION="${DIND_K8S_VERSION:-v1.13}" + DIND_IMAGE="${DIND_IMAGE_BASE}:${DIND_COMMIT}-${DIND_K8S_VERSION}" +else + DIND_IMAGE="${DIND_IMAGE:-${DIND_IMAGE_BASE}:local}" +fi +if [[ ${DIND_IMAGE_DIGEST:-} ]]; then + DIND_IMAGE="${DIND_IMAGE}@${DIND_IMAGE_DIGEST}" +fi + +BUILD_KUBEADM="${BUILD_KUBEADM:-}" +BUILD_HYPERKUBE="${BUILD_HYPERKUBE:-}" +if [[ ! -z ${DIND_K8S_BIN_DIR:-} ]]; then + BUILD_KUBEADM="" + BUILD_HYPERKUBE="" +fi +KUBEADM_SOURCE="${KUBEADM_SOURCE-}" +HYPERKUBE_SOURCE="${HYPERKUBE_SOURCE-}" +NUM_NODES=${NUM_NODES:-2} +EXTRA_PORTS="${EXTRA_PORTS:-}" +KUBECTL_DIR="${KUBECTL_DIR:-${HOME}/.kubeadm-dind-cluster}" +DASHBOARD_URL="${DASHBOARD_URL:-https://rawgit.com/kubernetes/dashboard/bfab10151f012d1acc5dfb1979f3172e2400aa3c/src/deploy/kubernetes-dashboard.yaml}" +SKIP_SNAPSHOT="${SKIP_SNAPSHOT:-}" +E2E_REPORT_DIR="${E2E_REPORT_DIR:-}" +DIND_NO_PARALLEL_E2E="${DIND_NO_PARALLEL_E2E:-}" +DNS_SERVICE="${DNS_SERVICE:-coredns}" +DIND_STORAGE_DRIVER="${DIND_STORAGE_DRIVER:-overlay2}" + +DIND_CA_CERT_URL="${DIND_CA_CERT_URL:-}" +DIND_PROPAGATE_HTTP_PROXY="${DIND_PROPAGATE_HTTP_PROXY:-}" +DIND_HTTP_PROXY="${DIND_HTTP_PROXY:-}" +DIND_HTTPS_PROXY="${DIND_HTTPS_PROXY:-}" +DIND_NO_PROXY="${DIND_NO_PROXY:-}" + +DIND_DAEMON_JSON_FILE="${DIND_DAEMON_JSON_FILE:-/etc/docker/daemon.json}" # can be set to /dev/null +DIND_REGISTRY_MIRROR="${DIND_REGISTRY_MIRROR:-}" # plain string format +DIND_INSECURE_REGISTRIES="${DIND_INSECURE_REGISTRIES:-}" # json list format + +FEATURE_GATES="${FEATURE_GATES:-MountPropagation=true}" +# you can set special value 'none' not to set any kubelet's feature gates. +KUBELET_FEATURE_GATES="${KUBELET_FEATURE_GATES:-MountPropagation=true,DynamicKubeletConfig=true}" + +ENABLE_CEPH="${ENABLE_CEPH:-}" + +DIND_CRI="${DIND_CRI:-docker}" +case "${DIND_CRI}" in + docker) + CRI_SOCKET=/var/run/dockershim.sock + ;; + containerd) + CRI_SOCKET=/var/run/containerd/containerd.sock + ;; + *) + echo >&2 "Bad DIND_CRI. Please specify 'docker' or 'containerd'" + ;; +esac + +# TODO: Test multi-cluster for IPv6, before enabling +if [[ "${DIND_LABEL}" != "${DEFAULT_DIND_LABEL}" && "${IP_MODE}" == 'dual-stack' ]]; then + echo "Multiple parallel clusters currently not supported for dual-stack mode" >&2 + exit 1 +fi + +# not configurable for now, would need to setup context for kubectl _inside_ the cluster +readonly INTERNAL_APISERVER_PORT=8080 + +function dind::need-source { + if [[ ! -f cluster/kubectl.sh ]]; then + echo "$0 must be called from the Kubernetes repository root directory" 1>&2 + exit 1 + fi +} + +build_tools_dir="build" +use_k8s_source=y +if [[ ! ${BUILD_KUBEADM} && ! ${BUILD_HYPERKUBE} ]]; then + use_k8s_source= +fi +if [[ ${use_k8s_source} ]]; then + dind::need-source + kubectl=cluster/kubectl.sh + if [[ ! -f ${build_tools_dir}/common.sh ]]; then + build_tools_dir="build-tools" + fi +else + if [[ ! ${DOWNLOAD_KUBECTL:-} ]] && ! hash kubectl 2>/dev/null; then + echo "You need kubectl binary in your PATH to use prebuilt DIND image" 1>&2 + exit 1 + fi + kubectl=kubectl +fi + +function dind::retry { + # based on retry function in hack/jenkins/ scripts in k8s source + for i in {1..10}; do + "$@" && return 0 || sleep ${i} + done + "$@" +} + +busybox_image="busybox:1.26.2" +e2e_base_image="golang:1.10.5" +sys_volume_args=() +build_volume_args=() + +function dind::set-build-volume-args { + if [ ${#build_volume_args[@]} -gt 0 ]; then + return 0 + fi + build_container_name= + if [ -n "${KUBEADM_DIND_LOCAL:-}" ]; then + build_volume_args=(-v "$PWD:/go/src/k8s.io/kubernetes") + else + build_container_name="$(KUBE_ROOT=${PWD} ETCD_HOST=${ETCD_HOST} && + . ${build_tools_dir}/common.sh && + kube::build::verify_prereqs >&2 && + echo "${KUBE_DATA_CONTAINER_NAME:-${KUBE_BUILD_DATA_CONTAINER_NAME}}")" + build_volume_args=(--volumes-from "${build_container_name}") + fi +} + +function dind::volume-exists { + local name="$1" + if docker volume inspect "${name}" >& /dev/null; then + return 0 + fi + return 1 +} + +function dind::create-volume { + local name="$1" + docker volume create --label "${DIND_LABEL}" --name "${name}" >/dev/null +} + +# We mount /boot and /lib/modules into the container +# below to in case some of the workloads need them. +# This includes virtlet, for instance. Also this may be +# useful in future if we want DIND nodes to pass +# preflight checks. +# Unfortunately we can't do this when using Mac Docker +# (unless a remote docker daemon on Linux is used) +# NB: there's no /boot on recent Mac dockers +function dind::prepare-sys-mounts { + if [[ ! ${using_linuxkit} ]]; then + sys_volume_args=() + if [[ -d /boot ]]; then + sys_volume_args+=(-v /boot:/boot) + fi + if [[ -d /lib/modules ]]; then + sys_volume_args+=(-v /lib/modules:/lib/modules) + fi + return 0 + fi + local dind_sys_vol_name + dind_sys_vol_name="kubeadm-dind-sys$( dind::cluster-suffix )" + if ! dind::volume-exists "$dind_sys_vol_name"; then + dind::step "Saving a copy of docker host's /lib/modules" + dind::create-volume "$dind_sys_vol_name" + # Use a dirty nsenter trick to fool Docker on Mac and grab system + # /lib/modules into sys.tar file on kubeadm-dind-sys volume. + local nsenter="nsenter --mount=/proc/1/ns/mnt --" + docker run \ + --rm \ + --privileged \ + -v "$dind_sys_vol_name":/dest \ + --pid=host \ + "${busybox_image}" \ + /bin/sh -c \ + "if ${nsenter} test -d /lib/modules; then ${nsenter} tar -C / -c lib/modules >/dest/sys.tar; fi" + fi + sys_volume_args=(-v "$dind_sys_vol_name":/dind-sys) +} + +tmp_containers=() + +function dind::cleanup { + if [ ${#tmp_containers[@]} -gt 0 ]; then + for name in "${tmp_containers[@]}"; do + docker rm -vf "${name}" 2>/dev/null + done + fi +} + +trap dind::cleanup EXIT + +function dind::check-image { + local name="$1" + if docker inspect --format 'x' "${name}" >&/dev/null; then + return 0 + else + return 1 + fi +} + +function dind::filter-make-output { + # these messages make output too long and make Travis CI choke + egrep -v --line-buffered 'I[0-9][0-9][0-9][0-9] .*(parse|conversion|defaulter|deepcopy)\.go:[0-9]+\]' +} + +function dind::run-build-command { + # this is like build/run.sh, but it doesn't rsync back the binaries, + # only the generated files. + local cmd=("$@") + ( + # The following is taken from build/run.sh and build/common.sh + # of Kubernetes source tree. It differs in + # --filter='+ /_output/dockerized/bin/**' + # being removed from rsync + . ${build_tools_dir}/common.sh + kube::build::verify_prereqs + kube::build::build_image + kube::build::run_build_command "$@" + + kube::log::status "Syncing out of container" + + kube::build::start_rsyncd_container + + local rsync_extra="" + if (( ${KUBE_VERBOSE} >= 6 )); then + rsync_extra="-iv" + fi + + # The filter syntax for rsync is a little obscure. It filters on files and + # directories. If you don't go in to a directory you won't find any files + # there. Rules are evaluated in order. The last two rules are a little + # magic. '+ */' says to go in to every directory and '- /**' says to ignore + # any file or directory that isn't already specifically allowed. + # + # We are looking to copy out all of the built binaries along with various + # generated files. + kube::build::rsync \ + --filter='- /vendor/' \ + --filter='- /_temp/' \ + --filter='+ zz_generated.*' \ + --filter='+ generated.proto' \ + --filter='+ *.pb.go' \ + --filter='+ types.go' \ + --filter='+ */' \ + --filter='- /**' \ + "rsync://k8s@${KUBE_RSYNC_ADDR}/k8s/" "${KUBE_ROOT}" + + kube::build::stop_rsyncd_container + ) +} + +function dind::make-for-linux { + local copy="$1" + shift + dind::step "Building binaries:" "$*" + if [ -n "${KUBEADM_DIND_LOCAL:-}" ]; then + dind::step "+ make WHAT=\"$*\"" + make WHAT="$*" 2>&1 | dind::filter-make-output + elif [ "${copy}" = "y" ]; then + dind::step "+ ${build_tools_dir}/run.sh make WHAT=\"$*\"" + "${build_tools_dir}/run.sh" make WHAT="$*" 2>&1 | dind::filter-make-output + else + dind::step "+ [using the build container] make WHAT=\"$*\"" + dind::run-build-command make WHAT="$*" 2>&1 | dind::filter-make-output + fi +} + +function dind::check-binary { + local filename="$1" + local dockerized="_output/dockerized/bin/linux/amd64/${filename}" + local plain="_output/local/bin/linux/amd64/${filename}" + dind::set-build-volume-args + # FIXME: don't hardcode amd64 arch + if [ -n "${KUBEADM_DIND_LOCAL:-${force_local:-}}" ]; then + if [ -f "${dockerized}" -o -f "${plain}" ]; then + return 0 + fi + elif docker run --rm "${build_volume_args[@]}" \ + "${busybox_image}" \ + test -f "/go/src/k8s.io/kubernetes/${dockerized}" >&/dev/null; then + return 0 + fi + return 1 +} + +function dind::ensure-downloaded-kubectl { + local kubectl_url + local kubectl_sha1 + local kubectl_sha1_linux + local kubectl_sha1_darwin + local kubectl_link + local kubectl_os + + if [[ ! ${DOWNLOAD_KUBECTL:-} ]]; then + return 0 + fi + + export PATH="${KUBECTL_DIR}:$PATH" + + eval "$(docker run --entrypoint /bin/bash --rm "${DIND_IMAGE}" -c "cat /dind-env")" + + if [ $(uname) = Darwin ]; then + kubectl_sha1="${KUBECTL_DARWIN_SHA1}" + kubectl_url="${KUBECTL_DARWIN_URL}" + else + kubectl_sha1="${KUBECTL_LINUX_SHA1}" + kubectl_url="${KUBECTL_LINUX_URL}" + fi + local link_target="kubectl-${KUBECTL_VERSION}" + local link_name="${KUBECTL_DIR}"/kubectl + if [[ -h "${link_name}" && "$(readlink "${link_name}")" = "${link_target}" ]]; then + return 0 + fi + + local path="${KUBECTL_DIR}/${link_target}" + if [[ ! -f "${path}" ]]; then + mkdir -p "${KUBECTL_DIR}" + curl -sSLo "${path}" "${kubectl_url}" + echo "${kubectl_sha1} ${path}" | sha1sum -c + chmod +x "${path}" + fi + + ln -fs "${link_target}" "${KUBECTL_DIR}/kubectl" +} + +function dind::ensure-kubectl { + if [[ ! ${use_k8s_source} ]]; then + # already checked on startup + dind::ensure-downloaded-kubectl + return 0 + fi + if [ $(uname) = Darwin ]; then + if [ ! -f _output/local/bin/darwin/amd64/kubectl ]; then + dind::step "Building kubectl" + dind::step "+ make WHAT=cmd/kubectl" + make WHAT=cmd/kubectl 2>&1 | dind::filter-make-output + fi + elif ! force_local=y dind::check-binary kubectl; then + dind::make-for-linux y cmd/kubectl + fi +} + +function dind::ensure-binaries { + local -a to_build=() + for name in "$@"; do + if ! dind::check-binary "$(basename "${name}")"; then + to_build+=("${name}") + fi + done + if [ "${#to_build[@]}" -gt 0 ]; then + dind::make-for-linux n "${to_build[@]}" + fi + return 0 +} + +# dind::ensure-network creates the management network for the cluster. For IPv4 +# only it will have the management network CIDR. For IPv6 only, it will have +# the IPv6 management network CIDR and the NAT64 V4 mapping network CIDR. For +# dual stack, it will have the IPv4 and IPv6 management CIDRs. Each of the +# management networks (not the NAT64 network) will have a gateway specified. +# +function dind::ensure-network { + if ! docker network inspect $(dind::net-name) >&/dev/null; then + local -a args + for cidr in "${mgmt_net_cidrs[@]}"; do + if [[ $( dind::family-for ${cidr} ) = "ipv6" ]]; then + args+=(--ipv6) + fi + args+=(--subnet="${cidr}") + local gw=$( dind::make-ip-from-cidr ${cidr} 1 ) + args+=(--gateway="${gw}") + done + if [[ ${IP_MODE} = "ipv6" ]]; then + # Need second network for NAT64 V4 mapping network + args+=(--subnet=${NAT64_V4_SUBNET_PREFIX}.0.0/16) + fi + docker network create ${args[@]} $(dind::net-name) >/dev/null + fi +} + +function dind::ensure-volume { + local reuse_volume= + if [[ $1 = -r ]]; then + reuse_volume=1 + shift + fi + local name="$1" + if dind::volume-exists "${name}"; then + if [[ ! ${reuse_volume} ]]; then + docker volume rm "${name}" >/dev/null + fi + fi + dind::create-volume "${name}" +} + +function dind::ensure-dns { + if [[ ${IP_MODE} = "ipv6" ]]; then + local dns64_name="bind9$( dind::cluster-suffix )" + if ! docker inspect ${dns64_name} >&/dev/null; then + local force_dns64_for="" + if [[ ! ${DIND_ALLOW_AAAA_USE} ]]; then + # Normally, if have an AAAA record, it is used. This clause tells + # bind9 to do ignore AAAA records for the specified networks + # and/or addresses and lookup A records and synthesize new AAAA + # records. In this case, we select "any" networks that have AAAA + # records meaning we ALWAYS use A records and do NAT64. + force_dns64_for="exclude { any; };" + fi + read -r -d '' bind9_conf </named.conf && named -c /named.conf -g -u named' >/dev/null + ipv4_addr="$(docker exec ${dns64_name} ip addr list eth0 | grep "inet" | awk '$1 == "inet" {print $2}')" + docker exec ${dns64_name} ip addr del ${ipv4_addr} dev eth0 + docker exec ${dns64_name} ip -6 route add ${DNS64_PREFIX_CIDR} via ${LOCAL_NAT64_SERVER} + fi + fi +} + +function dind::ensure-nat { + if [[ ${IP_MODE} = "ipv6" ]]; then + local nat64_name="tayga$( dind::cluster-suffix )" + if ! docker ps | grep ${nat64_name} >&/dev/null; then + docker run -d --name ${nat64_name} --hostname ${nat64_name} --net "$(dind::net-name)" --label "dind-support$( dind::cluster-suffix )" \ + --sysctl net.ipv6.conf.all.disable_ipv6=0 --sysctl net.ipv6.conf.all.forwarding=1 \ + --privileged=true --ip ${NAT64_V4_SUBNET_PREFIX}.0.200 --ip6 ${LOCAL_NAT64_SERVER} --dns ${REMOTE_DNS64_V4SERVER} --dns ${dns_server} \ + -e TAYGA_CONF_PREFIX=${DNS64_PREFIX_CIDR} -e TAYGA_CONF_IPV4_ADDR=${NAT64_V4_SUBNET_PREFIX}.0.200 \ + -e TAYGA_CONF_DYNAMIC_POOL=${NAT64_V4_SUBNET_PREFIX}.0.128/25 danehans/tayga:latest >/dev/null + # Need to check/create, as "clean" may remove route + local route="$(ip route | egrep "^${NAT64_V4_SUBNET_PREFIX}.0.128/25")" + if [[ -z "${route}" ]]; then + docker run --net=host --rm --privileged ${busybox_image} ip route add ${NAT64_V4_SUBNET_PREFIX}.0.128/25 via ${NAT64_V4_SUBNET_PREFIX}.0.200 + fi + fi + fi +} + +function dind::run { + local reuse_volume= + if [[ $1 = -r ]]; then + reuse_volume="-r" + shift + fi + local container_name="${1:-}" + local node_id=${2:-0} + local portforward="${3:-}" + if [[ $# -gt 3 ]]; then + shift 3 + else + shift $# + fi + + local -a opts=("$@") + local ip_mode="--ip" + for cidr in "${mgmt_net_cidrs[@]}"; do + if [[ $( dind::family-for ${cidr} ) = "ipv6" ]]; then + ip_mode="--ip6" + fi + opts+=("${ip_mode}" "$( dind::make-ip-from-cidr ${cidr} $((${node_id}+1)) )") + done + opts+=("$@") + + local -a args=("systemd.setenv=CNI_PLUGIN=${CNI_PLUGIN}") + args+=("systemd.setenv=IP_MODE=${IP_MODE}") + args+=("systemd.setenv=DIND_STORAGE_DRIVER=${DIND_STORAGE_DRIVER}") + args+=("systemd.setenv=DIND_CRI=${DIND_CRI}") + + if [[ ${IP_MODE} != "ipv4" ]]; then + opts+=(--sysctl net.ipv6.conf.all.disable_ipv6=0) + opts+=(--sysctl net.ipv6.conf.all.forwarding=1) + fi + + if [[ ${IP_MODE} = "ipv6" ]]; then + opts+=(--dns ${dns_server}) + args+=("systemd.setenv=DNS64_PREFIX_CIDR=${DNS64_PREFIX_CIDR}") + args+=("systemd.setenv=LOCAL_NAT64_SERVER=${LOCAL_NAT64_SERVER}") + fi + + declare -a pod_nets + local i=0 + if [[ ${IP_MODE} = "ipv4" || ${IP_MODE} = "dual-stack" ]]; then + pod_nets+=("${pod_prefixes[$i]}${node_id}") + i=$((i+1)) + fi + if [[ ${IP_MODE} = "ipv6" || ${IP_MODE} = "dual-stack" ]]; then + # For prefix, if node ID will be in the upper byte, push it over + if [[ $((${pod_sizes[$i]} % 16)) -ne 0 ]]; then + n_id=$(printf "%02x00\n" "${node_id}") + else + if [[ "${pod_prefixes[$i]: -1}" = ":" ]]; then + n_id=$(printf "%x\n" "${node_id}") + else + n_id=$(printf "%02x\n" "${node_id}") # In lower byte, so ensure two chars + fi + fi + pod_nets+=("${pod_prefixes[$i]}${n_id}") + fi + + args+=("systemd.setenv=POD_NET_PREFIX=\"${pod_nets[0]}\"") + args+=("systemd.setenv=POD_NET_SIZE=\"${pod_sizes[0]}\"") + args+=("systemd.setenv=POD_NET2_PREFIX=\"${pod_nets[1]:-}\"") + args+=("systemd.setenv=POD_NET2_SIZE=\"${pod_sizes[1]:-}\"") + args+=("systemd.setenv=SERVICE_NET_MODE=${SERVICE_NET_MODE}") + args+=("systemd.setenv=USE_HAIRPIN=${USE_HAIRPIN}") + args+=("systemd.setenv=DNS_SVC_IP=${DNS_SVC_IP}") + args+=("systemd.setenv=DNS_SERVICE=${DNS_SERVICE}") + if [[ ! "${container_name}" ]]; then + echo >&2 "Must specify container name" + exit 1 + fi + + # remove any previously created containers with the same name + docker rm -vf "${container_name}" >&/dev/null || true + + if [[ "${portforward}" ]]; then + IFS=';' read -ra array <<< "${portforward}" + for element in "${array[@]}"; do + opts+=(-p "${element}") + done + fi + + opts+=(${sys_volume_args[@]+"${sys_volume_args[@]}"}) + + dind::step "Starting DIND container:" "${container_name}" + + if [[ ! -z ${DIND_K8S_BIN_DIR:-} ]]; then + opts+=(-v ${DIND_K8S_BIN_DIR}:/k8s) + fi + if [[ ! ${using_linuxkit} ]]; then + opts+=(-v /boot:/boot -v /lib/modules:/lib/modules) + fi + + if [[ ${ENABLE_CEPH} ]]; then + opts+=(-v /dev:/dev + -v /sys/bus:/sys/bus + -v /var/run/docker.sock:/opt/outer-docker.sock) + fi + + local volume_name="kubeadm-dind-${container_name}" + dind::ensure-network + dind::ensure-volume ${reuse_volume} "${volume_name}" + dind::ensure-nat + dind::ensure-dns + + # TODO: create named volume for binaries and mount it to /k8s + # in case of the source build + + # Start the new container. + docker run \ + -e IP_MODE="${IP_MODE}" \ + -e KUBEADM_SOURCE="${KUBEADM_SOURCE}" \ + -e HYPERKUBE_SOURCE="${HYPERKUBE_SOURCE}" \ + -d --privileged \ + --net "$(dind::net-name)" \ + --name "${container_name}" \ + --hostname "${container_name}" \ + -l "${DIND_LABEL}" \ + -v "${volume_name}:/dind" \ + ${opts[@]+"${opts[@]}"} \ + "${DIND_IMAGE}" \ + ${args[@]+"${args[@]}"} +} + +function dind::kubeadm { + local container_id="$1" + shift + dind::step "Running kubeadm:" "$*" + status=0 + # See image/bare/wrapkubeadm. + # Capturing output is necessary to grab flags for 'kubeadm join' + local -a env=(-e KUBELET_FEATURE_GATES="${KUBELET_FEATURE_GATES}" + -e DIND_CRI="${DIND_CRI}") + if ! docker exec "${env[@]}" "${container_id}" /usr/local/bin/wrapkubeadm "$@" 2>&1 | tee /dev/fd/2; then + echo "*** kubeadm failed" >&2 + return 1 + fi + return ${status} +} + +# function dind::bare { +# local container_name="${1:-}" +# if [[ ! "${container_name}" ]]; then +# echo >&2 "Must specify container name" +# exit 1 +# fi +# shift +# run_opts=(${@+"$@"}) +# dind::run "${container_name}" +# } + +function dind::configure-kubectl { + dind::step "Setting cluster config" + local host="$(dind::localhost)" + if [[ -z "$using_local_linuxdocker" ]]; then + host="127.0.0.1" + fi + local context_name cluster_name + context_name="$(dind::context-name)" + cluster_name="$(dind::context-name)" + "${kubectl}" config set-cluster "$cluster_name" \ + --server="http://${host}:$(dind::apiserver-port)" \ + --insecure-skip-tls-verify=true + "${kubectl}" config set-context "$context_name" --cluster="$cluster_name" + if [[ ${DIND_LABEL} = ${DEFAULT_DIND_LABEL} ]]; then + # Single cluster mode + "${kubectl}" config use-context "$context_name" + fi +} + +force_make_binaries= +function dind::set-master-opts { + master_opts=() + if [[ ${BUILD_KUBEADM} || ${BUILD_HYPERKUBE} ]]; then + # share binaries pulled from the build container between nodes + local dind_k8s_bin_vol_name + dind_k8s_bin_vol_name="dind-k8s-binaries$(dind::cluster-suffix)" + dind::ensure-volume -r "${dind_k8s_bin_vol_name}" + dind::set-build-volume-args + master_opts+=("${build_volume_args[@]}" -v "${dind_k8s_bin_vol_name}:/k8s") + local -a bins + if [[ ${BUILD_KUBEADM} ]]; then + master_opts+=(-e KUBEADM_SOURCE=build://) + bins+=(cmd/kubeadm) + else + master_opts+=(-e ${KUBEADM_SOURCE}) + fi + if [[ ${BUILD_HYPERKUBE} ]]; then + master_opts+=(-e HYPERKUBE_SOURCE=build://) + bins+=(cmd/hyperkube) + fi + if [[ ${force_make_binaries} ]]; then + dind::make-for-linux n "${bins[@]}" + else + dind::ensure-binaries "${bins[@]}" + fi + fi + if [[ ${MASTER_EXTRA_OPTS:-} ]]; then + master_opts+=( ${MASTER_EXTRA_OPTS} ) + fi +} + +function dind::ensure-dashboard-clusterrolebinding { + local ctx + ctx="$(dind::context-name)" + # 'create' may cause etcd timeout, yet create the clusterrolebinding. + # So use 'apply' to actually create it + "${kubectl}" --context "$ctx" create clusterrolebinding add-on-cluster-admin \ + --clusterrole=cluster-admin \ + --serviceaccount=kube-system:default \ + -o json --dry-run | + docker exec -i "$(dind::master-name)" jq '.apiVersion="rbac.authorization.k8s.io/v1beta1"|.kind|="ClusterRoleBinding"' | + "${kubectl}" --context "$ctx" apply -f - +} + +function dind::deploy-dashboard { + dind::step "Deploying k8s dashboard" + dind::retry "${kubectl}" --context "$(dind::context-name)" apply -f "${DASHBOARD_URL}" + # https://kubernetes-io-vnext-staging.netlify.com/docs/admin/authorization/rbac/#service-account-permissions + # Thanks @liggitt for the hint + dind::retry dind::ensure-dashboard-clusterrolebinding +} + +function dind::kubeadm-version { + if [[ ${use_k8s_source} ]]; then + (cluster/kubectl.sh version --short 2>/dev/null || true) | + grep Client | + sed 's/^.*: v\([0-9.]*\).*/\1/' + else + docker exec "$(dind::master-name)" \ + /bin/bash -c 'kubeadm version -o json | jq -r .clientVersion.gitVersion' | + sed 's/^v\([0-9.]*\).*/\1/' + fi +} + +function dind::kubeadm-version-at-least { + local major="${1}" + local minor="${2}" + if [[ ! ( $(dind::kubeadm-version) =~ ^([0-9]+)\.([0-9]+) ) ]]; then + echo >&2 "WARNING: can't parse kubeadm version: $(dind::kubeadm-version)" + return 1 + fi + local act_major="${BASH_REMATCH[1]}" + local act_minor="${BASH_REMATCH[2]}" + if [[ ${act_major} -gt ${major} ]]; then + return 0 + fi + if [[ ${act_major} -lt ${major} ]]; then + return 1 + fi + if [[ ${act_minor} -ge ${minor} ]]; then + return 0 + fi + return 1 +} + +function dind::verify-image-compatibility { + # We can't tell in advance, if the image selected supports dual-stack, + # but will do the next best thing, and check as soon as start up kube-master + local master_name=$1 + if [[ ${IP_MODE} = "dual-stack" ]]; then + local dual_stack_support="$(docker exec ${master_name} cat /node-info 2>/dev/null | grep "dual-stack-support" | wc -l)" + if [[ ${dual_stack_support} -eq 0 ]]; then + echo "ERROR! DinD image (${DIND_IMAGE}) does not support dual-stack mode - aborting!" + dind::remove-images "${DIND_LABEL}" + exit 1 + fi + fi +} + +function dind::check-dns-service-type { + if [[ ${DNS_SERVICE} = "kube-dns" ]] && dind::kubeadm-version-at-least 1 13; then + echo >&2 "WARNING: for 1.13+, only coredns can be used as the DNS service" + DNS_SERVICE="coredns" + fi +} + +function dind::init { + local -a opts + dind::set-master-opts + local local_host master_name container_id + master_name="$(dind::master-name)" + local_host="$( dind::localhost )" + container_id=$(dind::run "${master_name}" 1 "${local_host}:$(dind::apiserver-port):${INTERNAL_APISERVER_PORT}" ${master_opts[@]+"${master_opts[@]}"}) + + dind::verify-image-compatibility ${master_name} + + # FIXME: I tried using custom tokens with 'kubeadm ex token create' but join failed with: + # 'failed to parse response as JWS object [square/go-jose: compact JWS format must have three parts]' + # So we just pick the line from 'kubeadm init' output + # Using a template file in the image (based on version) to build a kubeadm.conf file and to customize + # it based on CNI plugin, IP mode, and environment settings. User can add additional + # customizations to template and then rebuild the image used (build/build-local.sh). + local pod_subnet_disable="# " + # TODO: May want to specify each of the plugins that require --pod-network-cidr + if [[ ${CNI_PLUGIN} != "bridge" && ${CNI_PLUGIN} != "ptp" ]]; then + pod_subnet_disable="" + fi + local bind_address="0.0.0.0" + if [[ ${SERVICE_NET_MODE} = "ipv6" ]]; then + bind_address="::" + fi + dind::proxy "$master_name" + dind::custom-docker-opts "$master_name" + + # HACK: Indicating mode, so that wrapkubeadm will not set a cluster CIDR for kube-proxy + # in IPv6 (only) mode. + if [[ ${SERVICE_NET_MODE} = "ipv6" ]]; then + docker exec --privileged -i "$master_name" touch /v6-mode + fi + + feature_gates="{CoreDNS: false}" + if [[ ${DNS_SERVICE} == "coredns" ]]; then + feature_gates="{CoreDNS: true}" + fi + + kubeadm_version="$(dind::kubeadm-version)" + case "${kubeadm_version}" in + 1\.9\.* | 1\.10\.*) + template="1.10" + ;; + 1\.11\.*) + template="1.11" + ;; + 1\.12\.*) + template="1.12" + ;; + *) # Includes 1.13 master branch + # Will make a separate template if/when it becomes incompatible + template="1.13" + # CoreDNS can no longer be switched off + feature_gates="{}" + ;; + esac + dind::check-dns-service-type + + component_feature_gates="" + if [ "${FEATURE_GATES}" != "none" ]; then + component_feature_gates="feature-gates: \\\"${FEATURE_GATES}\\\"" + fi + + apiserver_extra_args="" + for e in $(set -o posix ; set | grep -E "^APISERVER_[a-z_]+=" | cut -d'=' -f 1); do + opt_name=$(echo ${e#APISERVER_} | sed 's/_/-/g') + apiserver_extra_args+=" ${opt_name}: \\\"$(eval echo \$$e)\\\"\\n" + done + + controller_manager_extra_args="" + for e in $(set -o posix ; set | grep -E "^CONTROLLER_MANAGER_[a-z_]+=" | cut -d'=' -f 1); do + opt_name=$(echo ${e#CONTROLLER_MANAGER_} | sed 's/_/-/g') + controller_manager_extra_args+=" ${opt_name}: \\\"$(eval echo \$$e)\\\"\\n" + done + + scheduler_extra_args="" + for e in $(set -o posix ; set | grep -E "^SCHEDULER_[a-z_]+=" | cut -d'=' -f 1); do + opt_name=$(echo ${e#SCHEDULER_} | sed 's/_/-/g') + scheduler_extra_args+=" ${opt_name}: \\\"$(eval echo \$$e)\\\"\\n" + done + + local mgmt_cidr=${mgmt_net_cidrs[0]} + if [[ ${IP_MODE} = "dual-stack" && ${SERVICE_NET_MODE} = "ipv6" ]]; then + mgmt_cidr=${mgmt_net_cidrs[1]} + fi + local master_ip=$( dind::make-ip-from-cidr ${mgmt_cidr} 2 ) + docker exec -i "$master_name" bash < /etc/kubeadm.conf +EOF + init_args=(--config /etc/kubeadm.conf) + # required when building from source + if [[ ${BUILD_KUBEADM} || ${BUILD_HYPERKUBE} ]]; then + docker exec "$master_name" mount --make-shared /k8s + fi + kubeadm_join_flags="$(dind::kubeadm "${container_id}" init "${init_args[@]}" --ignore-preflight-errors=all "$@" | grep '^ *kubeadm join' | sed 's/^ *kubeadm join //')" + dind::configure-kubectl + dind::start-port-forwarder +} + +function dind::create-node-container { + local reuse_volume next_node_index node_name + reuse_volume='' + if [[ ${1:-} = -r ]]; then + reuse_volume="-r" + shift + fi + # if there's just one node currently, it's master, thus we need to use + # kube-node-1 hostname, if there are two nodes, we should pick + # kube-node-2 and so on + next_node_index=${1:-$(docker ps -q --filter=label="${DIND_LABEL}" | wc -l | sed 's/^ *//g')} + local -a opts + if [[ ${BUILD_KUBEADM} || ${BUILD_HYPERKUBE} ]]; then + opts+=(-v "dind-k8s-binaries$(dind::cluster-suffix)":/k8s) + if [[ ${BUILD_KUBEADM} ]]; then + opts+=(-e KUBEADM_SOURCE=build://) + fi + if [[ ${BUILD_HYPERKUBE} ]]; then + opts+=(-e HYPERKUBE_SOURCE=build://) + fi + fi + node_name="$(dind::node-name ${next_node_index})" + dind::run ${reuse_volume} "$node_name" $((next_node_index + 1)) "${EXTRA_PORTS}" ${opts[@]+"${opts[@]}"} +} + +function dind::join { + local container_id="$1" + shift + dind::proxy "${container_id}" + dind::custom-docker-opts "${container_id}" + local -a join_opts=(--ignore-preflight-errors=all + --cri-socket="${CRI_SOCKET}") + dind::kubeadm "${container_id}" join "${join_opts[@]}" "$@" >/dev/null +} + +function dind::escape-e2e-name { + sed 's/[]\$*.^()[]/\\&/g; s/\s\+/\\s+/g' <<< "$1" | tr -d '\n' +} + +function dind::accelerate-kube-dns { + if [[ ${DNS_SERVICE} == "kube-dns" ]]; then + dind::step "Patching kube-dns deployment to make it start faster" + # Could do this on the host, too, but we don't want to require jq here + # TODO: do this in wrapkubeadm + docker exec "$(dind::master-name)" /bin/bash -c \ + "kubectl get deployment kube-dns -n kube-system -o json | jq '.spec.template.spec.containers[0].readinessProbe.initialDelaySeconds = 3|.spec.template.spec.containers[0].readinessProbe.periodSeconds = 3' | kubectl apply --force -f -" + fi +} + +function dind::component-ready { + local label="$1" + local out + if ! out="$("${kubectl}" --context "$(dind::context-name)" get pod -l "${label}" -n kube-system \ + -o jsonpath='{ .items[*].status.conditions[?(@.type == "Ready")].status }' 2>/dev/null)"; then + return 1 + fi + if ! grep -v False <<<"${out}" | grep -q True; then + return 1 + fi + return 0 +} + +function dind::kill-failed-pods { + local pods ctx + ctx="$(dind::context-name)" + # workaround for https://github.com/kubernetes/kubernetes/issues/36482 + if ! pods="$(kubectl --context "$ctx" get pod -n kube-system -o jsonpath='{ .items[?(@.status.phase == "Failed")].metadata.name }' 2>/dev/null)"; then + return + fi + for name in ${pods}; do + kubectl --context "$ctx" delete pod --now -n kube-system "${name}" >&/dev/null || true + done +} + +function dind::create-static-routes { + echo "Creating static routes for bridge/PTP plugin" + for ((i=0; i <= NUM_NODES; i++)); do + if [[ ${i} -eq 0 ]]; then + node="$(dind::master-name)" + else + node="$(dind::node-name $i)" + fi + for ((j=0; j <= NUM_NODES; j++)); do + if [[ ${i} -eq ${j} ]]; then + continue + fi + if [[ ${j} -eq 0 ]]; then + dest_node="$(dind::master-name)" + else + dest_node="$(dind::node-name $j)" + fi + id=$((${j}+1)) + if [[ ${IP_MODE} = "ipv4" || ${IP_MODE} = "dual-stack" ]]; then + # Assuming pod subnets will all be /24 + dest="${pod_prefixes[0]}${id}.0/24" + gw=`docker exec ${dest_node} ip addr show eth0 | grep -w inet | awk '{ print $2 }' | sed 's,/.*,,'` + docker exec "${node}" ip route add "${dest}" via "${gw}" + fi + if [[ ${IP_MODE} = "ipv6" || ${IP_MODE} = "dual-stack" ]]; then + local position=0 + if [[ ${IP_MODE} = "dual-stack" ]]; then + position=1 + fi + instance=$(printf "%02x" ${id}) + if [[ $((${pod_sizes[$position]} % 16)) -ne 0 ]]; then + instance+="00" # Move node ID to upper byte + fi + dest="${pod_prefixes[$position]}${instance}::/${pod_sizes[$position]}" + gw=`docker exec ${dest_node} ip addr show eth0 | grep -w inet6 | grep -i global | head -1 | awk '{ print $2 }' | sed 's,/.*,,'` + docker exec "${node}" ip route add "${dest}" via "${gw}" + fi + done + done +} + +# If we are allowing AAAA record use, then provide SNAT for IPv6 packets from +# node containers, and forward packets to bridge used for $(dind::net-name). +# This gives pods access to external IPv6 sites, when using IPv6 addresses. +function dind::setup_external_access_on_host { + if [[ ! ${DIND_ALLOW_AAAA_USE} ]]; then + return + fi + local main_if=`ip route | grep default | awk '{print $5}'` + dind::ip6tables-on-hostnet -t nat -A POSTROUTING -o $main_if -j MASQUERADE + if [[ ${IP_MODE} = "dual-stack" ]]; then + return + fi + local bridge_if=`ip route | grep ${NAT64_V4_SUBNET_PREFIX}.0.0 | awk '{print $3}'` + if [[ -n "$bridge_if" ]]; then + dind::ip6tables-on-hostnet -A FORWARD -i $bridge_if -j ACCEPT + else + echo "WARNING! No $(dind::net-name) bridge with NAT64 - unable to setup forwarding/SNAT" + fi +} + +# Remove ip6tables rules for SNAT and forwarding, if they exist. +function dind::remove_external_access_on_host { + if [[ ! ${DIND_ALLOW_AAAA_USE} ]]; then + return + fi + local have_rule + local main_if="$(ip route | grep default | awk '{print $5}')" + have_rule="$(dind::ip6tables-on-hostnet -S -t nat | grep "\-o $main_if" || true)" + if [[ -n "$have_rule" ]]; then + dind::ip6tables-on-hostnet -t nat -D POSTROUTING -o $main_if -j MASQUERADE + else + echo "Skipping delete of ip6tables rule for SNAT, as rule non-existent" + fi + + if [[ ${IP_MODE} = "dual-stack" ]]; then + return + fi + local bridge_if="$(ip route | grep ${NAT64_V4_SUBNET_PREFIX}.0.0 | awk '{print $3}')" + if [[ -n "$bridge_if" ]]; then + have_rule="$(dind::ip6tables-on-hostnet -S | grep "\-i $bridge_if" || true)" + if [[ -n "$have_rule" ]]; then + dind::ip6tables-on-hostnet -D FORWARD -i $bridge_if -j ACCEPT + else + echo "Skipping delete of ip6tables rule for forwarding, as rule non-existent" + fi + else + echo "Skipping delete of ip6tables rule for forwarding, as no bridge interface using NAT64" + fi +} + +function dind::ip6tables-on-hostnet { + local mod_path='/lib/modules' + docker run -v "${mod_path}:${mod_path}" --entrypoint /sbin/ip6tables --net=host --rm --privileged "${DIND_IMAGE}" "$@" +} + +function dind::wait-for-ready { + local app="kube-proxy" + if [[ ${CNI_PLUGIN} = "kube-router" ]]; then + app=kube-router + fi + dind::step "Waiting for ${app} and the nodes" + local app_ready + local nodes_ready + local n=3 + local ntries=200 + local ctx + ctx="$(dind::context-name)" + while true; do + dind::kill-failed-pods + if "${kubectl}" --context "$ctx" get nodes 2>/dev/null | grep -q NotReady; then + nodes_ready= + else + nodes_ready=y + fi + if dind::component-ready k8s-app=${app}; then + app_ready=y + else + app_ready= + fi + if [[ ${nodes_ready} && ${app_ready} ]]; then + if ((--n == 0)); then + echo "[done]" >&2 + break + fi + else + n=3 + fi + if ((--ntries == 0)); then + echo "Error waiting for ${app} and the nodes" >&2 + exit 1 + fi + echo -n "." >&2 + sleep 1 + done + + dind::step "Bringing up ${DNS_SERVICE} and kubernetes-dashboard" + # on Travis 'scale' sometimes fails with 'error: Scaling the resource failed with: etcdserver: request timed out; Current resource version 442' here + dind::retry "${kubectl}" --context "$ctx" scale deployment --replicas=1 -n kube-system ${DNS_SERVICE} + dind::retry "${kubectl}" --context "$ctx" scale deployment --replicas=1 -n kube-system kubernetes-dashboard + + ntries=200 + while ! dind::component-ready k8s-app=kube-dns || ! dind::component-ready app=kubernetes-dashboard; do + if ((--ntries == 0)); then + echo "Error bringing up ${DNS_SERVICE} and kubernetes-dashboard" >&2 + exit 1 + fi + echo -n "." >&2 + dind::kill-failed-pods + sleep 1 + done + echo "[done]" >&2 + + dind::retry "${kubectl}" --context "$ctx" get nodes >&2 + + local local_host + local_host="$( dind::localhost )" + dind::step "Access dashboard at:" "http://${local_host}:$(dind::apiserver-port)/api/v1/namespaces/kube-system/services/kubernetes-dashboard:/proxy" +} + +# dind::make-kube-router-yaml creates a temp file with contents of the configuration needed for the kube-router CNI +# plugin at a specific version, instead of using the publically available file, which uses the latest version. This +# allows us to control the version used. If/when updating, be sure to update the KUBE_ROUTER_VERSION env variable +# ensure the YAML contents below, reflect the configuration in: +# +# https://raw.githubusercontent.com/cloudnativelabs/kube-router/master/daemonset/kubeadm-kuberouter-all-feature.yaml +# +# FUTURE: May be able to remove this, if/when kube-router "latest" is stable, and use the public YAML file instead. +function dind::make-kube-router-yaml { + tmp_yaml=$(mktemp /tmp/kube-router-yaml.XXXXXX) + cat >${tmp_yaml} <&2 "*** Failed to start node container ${n}" + exit 1 + else + node_containers+=(${container_id}) + dind::step "Node container started:" ${n} + fi + done + dind::fix-mounts + status=0 + local -a pids + for ((n=1; n <= NUM_NODES; n++)); do + ( + dind::step "Joining node:" ${n} + container_id="${node_containers[${n}-1]}" + if ! dind::join ${container_id} ${kubeadm_join_flags}; then + echo >&2 "*** Failed to start node container ${n}" + exit 1 + else + dind::step "Node joined:" ${n} + fi + )& + pids[${n}]=$! + done + if ((NUM_NODES > 0)); then + for pid in ${pids[*]}; do + wait ${pid} + done + else + # FIXME: this may fail depending on k8s/kubeadm version + # FIXME: check for taint & retry if it's there + "${kubectl}" --context "$ctx" taint nodes $(dind::master-name) node-role.kubernetes.io/master- || true + fi + case "${CNI_PLUGIN}" in + bridge | ptp) + dind::create-static-routes + dind::setup_external_access_on_host + ;; + flannel) + # without --validate=false this will fail on older k8s versions + dind::retry "${kubectl}" --context "$ctx" apply --validate=false -f "https://github.com/coreos/flannel/blob/master/Documentation/kube-flannel.yml?raw=true" + ;; + calico) + manifest_base=https://docs.projectcalico.org/${CALICO_VERSION:-v3.3}/getting-started/kubernetes/installation + dind::retry "${kubectl}" --context "$ctx" apply -f ${manifest_base}/hosted/etcd.yaml + if [ "${CALICO_VERSION:-v3.3}" != master ]; then + dind::retry "${kubectl}" --context "$ctx" apply -f ${manifest_base}/rbac.yaml + fi + dind::retry "${kubectl}" --context "$ctx" apply -f ${manifest_base}/hosted/calico.yaml + dind::retry "${kubectl}" --context "$ctx" apply -f ${manifest_base}/hosted/calicoctl.yaml + ;; + calico-kdd) + manifest_base=https://docs.projectcalico.org/${CALICO_VERSION:-v3.3}/getting-started/kubernetes/installation + dind::retry "${kubectl}" --context "$ctx" apply -f ${manifest_base}/hosted/rbac-kdd.yaml + dind::retry "${kubectl}" --context "$ctx" apply -f ${manifest_base}/hosted/kubernetes-datastore/calico-networking/1.7/calico.yaml + ;; + weave) + dind::retry "${kubectl}" --context "$ctx" apply -f "https://cloud.weave.works/k8s/net?k8s-version=$(${kubectl} --context "$ctx" version | base64 | tr -d '\n')" + ;; + kube-router) + kube_router_config="$( dind::make-kube-router-yaml )" + dind::retry "${kubectl}" --context "$ctx" apply -f ${kube_router_config} + rm "${kube_router_config}" + dind::retry "${kubectl}" --context "$ctx" -n kube-system delete ds kube-proxy + docker run --privileged --net=host k8s.gcr.io/kube-proxy-amd64:v1.10.2 kube-proxy --cleanup + ;; + *) + echo "Unsupported CNI plugin '${CNI_PLUGIN}'" >&2 + ;; + esac + dind::deploy-dashboard + dind::accelerate-kube-dns + if [[ (${CNI_PLUGIN} != "bridge" && ${CNI_PLUGIN} != "ptp") || ${SKIP_SNAPSHOT} ]]; then + # This is especially important in case of Calico - + # the cluster will not recover after snapshotting + # (at least not after restarting from the snapshot) + # if Calico installation is interrupted + dind::wait-for-ready + fi + dind::step "Cluster Info" + echo "Network Mode: ${IP_MODE}" + echo "Cluster context: $( dind::context-name )" + echo "Cluster ID: ${CLUSTER_ID}" + echo "Management CIDR(s): ${mgmt_net_cidrs[@]}" + echo "Service CIDR/mode: ${SERVICE_CIDR}/${SERVICE_NET_MODE}" + echo "Pod CIDR(s): ${pod_net_cidrs[@]}" +} + +function dind::fix-mounts { + local node_name + for ((n=0; n <= NUM_NODES; n++)); do + node_name="$(dind::master-name)" + if ((n > 0)); then + node_name="$(dind::node-name $n)" + fi + docker exec "${node_name}" mount --make-shared /run + if [[ ! ${using_linuxkit} ]]; then + docker exec "${node_name}" mount --make-shared /lib/modules/ + fi + # required when building from source + if [[ ${BUILD_KUBEADM} || ${BUILD_HYPERKUBE} ]]; then + docker exec "${node_name}" mount --make-shared /k8s + fi + # docker exec "${node_name}" mount --make-shared /sys/kernel/debug + done +} + +function dind::snapshot_container { + local container_name="$1" + # we must pass DIND_CRI here because in case of containerd + # a special care must be taken to stop the containers during + # the snapshot + docker exec -e DIND_CRI="${DIND_CRI}" -i ${container_name} \ + /usr/local/bin/snapshot prepare + # remove the hidden *plnk directories + docker diff ${container_name} | grep -v plnk | docker exec -i ${container_name} /usr/local/bin/snapshot save +} + +function dind::snapshot { + dind::step "Taking snapshot of the cluster" + dind::snapshot_container "$(dind::master-name)" + for ((n=1; n <= NUM_NODES; n++)); do + dind::snapshot_container "$(dind::node-name $n)" + done + dind::wait-for-ready +} + +restore_cmd=restore +function dind::restore_container { + local container_id="$1" + docker exec ${container_id} /usr/local/bin/snapshot "${restore_cmd}" +} + +function dind::restore { + local apiserver_port local_host pid pids + dind::down + dind::check-dns-service-type + dind::step "Restoring containers" + dind::set-master-opts + local_host="$( dind::localhost )" + apiserver_port="$( dind::apiserver-port )" + for ((n=0; n <= NUM_NODES; n++)); do + ( + if [[ n -eq 0 ]]; then + dind::step "Restoring master container" + dind::restore_container "$(dind::run -r "$(dind::master-name)" 1 "${local_host}:${apiserver_port}:${INTERNAL_APISERVER_PORT}" ${master_opts[@]+"${master_opts[@]}"})" + dind::verify-image-compatibility "$(dind::master-name)" + dind::step "Master container restored" + else + dind::step "Restoring node container:" ${n} + if ! container_id="$(dind::create-node-container -r ${n})"; then + echo >&2 "*** Failed to start node container ${n}" + exit 1 + else + dind::restore_container "${container_id}" + dind::step "Node container restored:" ${n} + fi + fi + )& + pids[${n}]=$! + done + for pid in ${pids[*]}; do + wait ${pid} + done + if [[ ${CNI_PLUGIN} = "bridge" || ${CNI_PLUGIN} = "ptp" ]]; then + dind::create-static-routes + dind::setup_external_access_on_host + fi + dind::fix-mounts + # Recheck kubectl config. It's possible that the cluster was started + # on this docker from different host + dind::configure-kubectl + dind::start-port-forwarder + dind::wait-for-ready +} + +function dind::down { + dind::remove-images "${DIND_LABEL}" + if [[ ${CNI_PLUGIN} = "bridge" || ${CNI_PLUGIN} = "ptp" ]]; then + dind::remove_external_access_on_host + elif [[ "${CNI_PLUGIN}" = "kube-router" ]]; then + if [[ ${COMMAND} = "down" || ${COMMAND} = "clean" ]]; then + # FUTURE: Updated pinned version, after verifying operation + docker run --privileged --net=host cloudnativelabs/kube-router:${KUBE_ROUTER_VERSION} --cleanup-config + fi + fi +} + +function dind::apiserver-port { + # APISERVER_PORT is explicitely set + if [ -n "${APISERVER_PORT:-}" ] + then + echo "$APISERVER_PORT" + return + fi + + # Get the port from the master + local master port + master="$(dind::master-name)" + # 8080/tcp -> 127.0.0.1:8082 => 8082 + port="$( docker port "$master" 2>/dev/null | awk -F: "/^${INTERNAL_APISERVER_PORT}/{ print \$NF }" )" + if [ -n "$port" ] + then + APISERVER_PORT="$port" + echo "$APISERVER_PORT" + return + fi + + # get a random free port + APISERVER_PORT=0 + echo "$APISERVER_PORT" +} + +function dind::master-name { + echo "kube-master$( dind::cluster-suffix )" +} + +function dind::node-name { + local nr="$1" + echo "kube-node-${nr}$( dind::cluster-suffix )" +} + +function dind::context-name { + echo "dind$( dind::cluster-suffix )" +} + +function dind::remove-volumes { + # docker 1.13+: docker volume ls -q -f label="${DIND_LABEL}" + local nameRE + nameRE="^kubeadm-dind-(sys|kube-master|kube-node-[0-9]+)$(dind::cluster-suffix)$" + docker volume ls -q | (grep -E "$nameRE" || true) | while read -r volume_id; do + dind::step "Removing volume:" "${volume_id}" + docker volume rm "${volume_id}" + done +} + +function dind::remove-images { + local which=$1 + docker ps -a -q --filter=label="${which}" | while read container_id; do + dind::step "Removing container:" "${container_id}" + docker rm -fv "${container_id}" + done +} + +function dind::remove-cluster { + cluster_name="dind$(dind::cluster-suffix)" + if ${kubectl} config get-clusters | grep -qE "^${cluster_name}$"; then + dind::step "Removing cluster from config:" "${cluster_name}" + ${kubectl} config delete-cluster ${cluster_name} 2>/dev/null || true + fi +} + +function dind::remove-context { + context_name="$(dind::context-name)" + if ${kubectl} config get-contexts | grep -qE "${context_name}\\s"; then + dind::step "Removing context from config:" "${context_name}" + ${kubectl} config delete-context ${context_name} 2>/dev/null || true + fi +} + +function dind::start-port-forwarder { + local fwdr port + fwdr="${DIND_PORT_FORWARDER:-}" + + [ -n "$fwdr" ] || return 0 + + [ -x "$fwdr" ] || { + echo "'${fwdr}' is not executable." >&2 + return 1 + } + + port="$( dind::apiserver-port )" + dind::step "+ Setting up port-forwarding for :${port}" + "$fwdr" "$port" +} + +function dind::check-for-snapshot { + if ! dind::volume-exists "kubeadm-dind-$(dind::master-name)"; then + return 1 + fi + for ((n=1; n <= NUM_NODES; n++)); do + if ! dind::volume-exists "kubeadm-dind-$(dind::node-name ${n})"; then + return 1 + fi + done +} + +function dind::do-run-e2e { + local parallel="${1:-}" + local focus="${2:-}" + local skip="${3:-}" + local host="$(dind::localhost)" + if [[ -z "$using_local_linuxdocker" ]]; then + host="127.0.0.1" + fi + dind::need-source + local kubeapi test_args term= + local -a e2e_volume_opts=() + kubeapi="http://${host}:$(dind::apiserver-port)" + test_args="--host=${kubeapi}" + if [[ ${focus} ]]; then + test_args="--ginkgo.focus=${focus} ${test_args}" + fi + if [[ ${skip} ]]; then + test_args="--ginkgo.skip=${skip} ${test_args}" + fi + if [[ ${E2E_REPORT_DIR} ]]; then + test_args="--report-dir=/report ${test_args}" + e2e_volume_opts=(-v "${E2E_REPORT_DIR}:/report") + fi + dind::make-for-linux n "cmd/kubectl test/e2e/e2e.test vendor/github.com/onsi/ginkgo/ginkgo" + dind::step "Running e2e tests with args:" "${test_args}" + dind::set-build-volume-args + if [ -t 1 ] ; then + term="-it" + test_args="--ginkgo.noColor --num-nodes=2 ${test_args}" + fi + docker run \ + --rm ${term} \ + --net=host \ + "${build_volume_args[@]}" \ + -e KUBERNETES_PROVIDER=dind \ + -e KUBE_MASTER_IP="${kubeapi}" \ + -e KUBE_MASTER=local \ + -e KUBERNETES_CONFORMANCE_TEST=y \ + -e GINKGO_PARALLEL=${parallel} \ + ${e2e_volume_opts[@]+"${e2e_volume_opts[@]}"} \ + -w /go/src/k8s.io/kubernetes \ + "${e2e_base_image}" \ + bash -c "cluster/kubectl.sh config set-cluster dind --server='${kubeapi}' --insecure-skip-tls-verify=true && + cluster/kubectl.sh config set-context dind --cluster=dind && + cluster/kubectl.sh config use-context dind && + go run hack/e2e.go -- --v 6 --test --check-version-skew=false --test_args='${test_args}'" +} + +function dind::clean { + dind::ensure-downloaded-kubectl + dind::down + dind::remove-images "dind-support$( dind::cluster-suffix )" + dind::remove-volumes + local net_name + net_name="$(dind::net-name)" + if docker network inspect "$net_name" >&/dev/null; then + docker network rm "$net_name" + fi + dind::remove-cluster + dind::remove-context +} + +function dind::copy-image { + local image="${2:-}" + local image_path="/tmp/save_${image//\//_}" + if [[ -f "${image_path}" ]]; then + rm -fr "${image_path}" + fi + docker save "${image}" -o "${image_path}" + docker ps -a -q --filter=label="${DIND_LABEL}" | while read container_id; do + cat "${image_path}" | docker exec -i "${container_id}" docker load + done + rm -fr "${image_path}" +} + +function dind::run-e2e { + local focus="${1:-}" + local skip="${2:-[Serial]}" + skip="$(dind::escape-e2e-name "${skip}")" + if [[ "$focus" ]]; then + focus="$(dind::escape-e2e-name "${focus}")" + else + focus="\[Conformance\]" + fi + local parallel=y + if [[ ${DIND_NO_PARALLEL_E2E} ]]; then + parallel= + fi + dind::do-run-e2e "${parallel}" "${focus}" "${skip}" +} + +function dind::run-e2e-serial { + local focus="${1:-}" + local skip="${2:-}" + skip="$(dind::escape-e2e-name "${skip}")" + dind::need-source + if [[ "$focus" ]]; then + focus="$(dind::escape-e2e-name "${focus}")" + else + focus="\[Serial\].*\[Conformance\]" + fi + dind::do-run-e2e n "${focus}" "${skip}" + # TBD: specify filter +} + +function dind::step { + local OPTS="" + if [ "$1" = "-n" ]; then + shift + OPTS+="-n" + fi + GREEN="$1" + shift + if [ -t 2 ] ; then + echo -e ${OPTS} "\x1B[97m* \x1B[92m${GREEN}\x1B[39m $*" 1>&2 + else + echo ${OPTS} "* ${GREEN} $*" 1>&2 + fi +} + +function dind::dump { + set +e + echo "*** Dumping cluster state ***" + for node in $(docker ps --format '{{.Names}}' --filter label="${DIND_LABEL}"); do + for service in kubelet.service dindnet.service criproxy.service dockershim.service; do + if docker exec "${node}" systemctl is-enabled "${service}" >&/dev/null; then + echo "@@@ service-${node}-${service}.log @@@" + docker exec "${node}" systemctl status "${service}" + docker exec "${node}" journalctl -xe -n all -u "${service}" + fi + done + echo "@@@ psaux-${node}.txt @@@" + docker exec "${node}" ps auxww + echo "@@@ dockerps-a-${node}.txt @@@" + docker exec "${node}" docker ps -a + echo "@@@ ip-a-${node}.txt @@@" + docker exec "${node}" ip a + echo "@@@ ip-r-${node}.txt @@@" + docker exec "${node}" ip r + done + local ctx master_name + master_name="$(dind::master-name)" + ctx="$(dind::context-name)" + docker exec "$master_name" kubectl get pods --all-namespaces \ + -o go-template='{{range $x := .items}}{{range $x.spec.containers}}{{$x.spec.nodeName}}{{" "}}{{$x.metadata.namespace}}{{" "}}{{$x.metadata.name}}{{" "}}{{.name}}{{"\n"}}{{end}}{{end}}' | + while read node ns pod container; do + echo "@@@ pod-${node}-${ns}-${pod}--${container}.log @@@" + docker exec "$master_name" kubectl logs -n "${ns}" -c "${container}" "${pod}" + done + echo "@@@ kubectl-all.txt @@@" + docker exec "$master_name" kubectl get all --all-namespaces -o wide + echo "@@@ describe-all.txt @@@" + docker exec "$master_name" kubectl describe all --all-namespaces + echo "@@@ nodes.txt @@@" + docker exec "$master_name" kubectl get nodes -o wide +} + +function dind::dump64 { + echo "%%% start-base64 %%%" + dind::dump | docker exec -i "$(dind::master-name)" /bin/sh -c "lzma | base64 -w 100" + echo "%%% end-base64 %%%" +} + +function dind::split-dump { + mkdir -p cluster-dump + cd cluster-dump + awk '!/^@@@ .* @@@$/{print >out}; /^@@@ .* @@@$/{out=$2}' out=/dev/null + ls -l +} + +function dind::split-dump64 { + decode_opt=-d + if base64 --help | grep -q '^ *-D'; then + # Mac OS X + decode_opt=-D + fi + sed -n '/^%%% start-base64 %%%$/,/^%%% end-base64 %%%$/p' | + sed '1d;$d' | + base64 "${decode_opt}" | + lzma -dc | + dind::split-dump +} + +function dind::proxy { + local container_id="$1" + if [[ ${DIND_CA_CERT_URL} ]] ; then + dind::step "+ Adding certificate on ${container_id}" + docker exec ${container_id} /bin/sh -c "cd /usr/local/share/ca-certificates; curl -sSO ${DIND_CA_CERT_URL}" + docker exec ${container_id} update-ca-certificates + fi + if [[ "${DIND_PROPAGATE_HTTP_PROXY}" || "${DIND_HTTP_PROXY}" || "${DIND_HTTPS_PROXY}" || "${DIND_NO_PROXY}" ]]; then + dind::step "+ Setting *_PROXY for docker service on ${container_id}" + local proxy_env="[Service]"$'\n'"Environment=" + if [[ "${DIND_PROPAGATE_HTTP_PROXY}" ]]; then + # take *_PROXY values from container environment + proxy_env+=$(docker exec ${container_id} env | grep -i _proxy | awk '{ print "\""$0"\""}' | xargs -d'\n') + else + if [[ "${DIND_HTTP_PROXY}" ]] ; then proxy_env+="\"HTTP_PROXY=${DIND_HTTP_PROXY}\" "; fi + if [[ "${DIND_HTTPS_PROXY}" ]] ; then proxy_env+="\"HTTPS_PROXY=${DIND_HTTPS_PROXY}\" "; fi + if [[ "${DIND_NO_PROXY}" ]] ; then proxy_env+="\"NO_PROXY=${DIND_NO_PROXY}\" "; fi + fi + docker exec -i ${container_id} /bin/sh -c "cat > /etc/systemd/system/docker.service.d/30-proxy.conf" <<< "${proxy_env}" + docker exec ${container_id} systemctl daemon-reload + docker exec ${container_id} systemctl restart docker + fi +} + +function dind::custom-docker-opts { + local container_id="$1" + local -a jq=() + local got_changes="" + if [[ ! -f ${DIND_DAEMON_JSON_FILE} ]] ; then + jq[0]="{}" + else + jq+=("$(cat ${DIND_DAEMON_JSON_FILE})") + if [[ ${DIND_DAEMON_JSON_FILE} != "/etc/docker/daemon.json" ]]; then + got_changes=1 + fi + fi + if [[ ${DIND_REGISTRY_MIRROR} ]] ; then + dind::step "+ Setting up registry mirror on ${container_id}" + jq+=("{\"registry-mirrors\": [\"${DIND_REGISTRY_MIRROR}\"]}") + got_changes=1 + fi + if [[ ${DIND_INSECURE_REGISTRIES} ]] ; then + dind::step "+ Setting up insecure-registries on ${container_id}" + jq+=("{\"insecure-registries\": ${DIND_INSECURE_REGISTRIES}}") + got_changes=1 + fi + if [[ ${got_changes} ]] ; then + local json=$(IFS="+"; echo "${jq[*]}") + docker exec -i ${container_id} /bin/sh -c "mkdir -p /etc/docker && jq -n '${json}' > /etc/docker/daemon.json" + docker exec ${container_id} systemctl daemon-reload + docker exec ${container_id} systemctl restart docker + fi +} + +COMMAND="${1:-}" + +case ${COMMAND} in + up) + if [[ ! ( ${DIND_IMAGE} =~ local ) && ! ${DIND_SKIP_PULL:-} ]]; then + dind::step "Making sure DIND image is up to date" + docker pull "${DIND_IMAGE}" >&2 + fi + + dind::prepare-sys-mounts + dind::ensure-kubectl + if [[ ${SKIP_SNAPSHOT} ]]; then + force_make_binaries=y dind::up + elif ! dind::check-for-snapshot; then + force_make_binaries=y dind::up + dind::snapshot + else + dind::restore + fi + ;; + reup) + dind::prepare-sys-mounts + dind::ensure-kubectl + if [[ ${SKIP_SNAPSHOT} ]]; then + force_make_binaries=y dind::up + elif ! dind::check-for-snapshot; then + force_make_binaries=y dind::up + dind::snapshot + else + force_make_binaries=y + restore_cmd=update_and_restore + dind::restore + fi + ;; + down) + dind::down + ;; + init) + shift + dind::prepare-sys-mounts + dind::ensure-kubectl + dind::init "$@" + ;; + join) + shift + dind::prepare-sys-mounts + dind::ensure-kubectl + dind::join "$(dind::create-node-container)" "$@" + ;; + # bare) + # shift + # dind::bare "$@" + # ;; + snapshot) + shift + dind::snapshot + ;; + restore) + shift + dind::restore + ;; + clean) + dind::clean + ;; + copy-image) + dind::copy-image "$@" + ;; + e2e) + shift + dind::run-e2e "$@" + ;; + e2e-serial) + shift + dind::run-e2e-serial "$@" + ;; + dump) + dind::dump + ;; + dump64) + dind::dump64 + ;; + split-dump) + dind::split-dump + ;; + split-dump64) + dind::split-dump64 + ;; + apiserver-port) + dind::apiserver-port + ;; + *) + echo "usage:" >&2 + echo " $0 up" >&2 + echo " $0 reup" >&2 + echo " $0 down" >&2 + echo " $0 init kubeadm-args..." >&2 + echo " $0 join kubeadm-args..." >&2 + # echo " $0 bare container_name [docker_options...]" + echo " $0 clean" + echo " $0 copy-image [image_name]" >&2 + echo " $0 e2e [test-name-substring]" >&2 + echo " $0 e2e-serial [test-name-substring]" >&2 + echo " $0 dump" >&2 + echo " $0 dump64" >&2 + echo " $0 split-dump" >&2 + echo " $0 split-dump64" >&2 + exit 1 + ;; +esac diff --git a/hack/lib/golang.sh b/hack/lib/golang.sh new file mode 100644 index 0000000000..db54e476ad --- /dev/null +++ b/hack/lib/golang.sh @@ -0,0 +1,716 @@ +#!/bin/bash + +# Copyright 2014 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# The golang package that we are building. +readonly KUBE_GO_PACKAGE=github.com/kubernetes-sigs/kube-batch +readonly KUBE_GOPATH="${GOPATH}" + +# The set of server targets that we are only building for Linux +# If you update this list, please also update build/BUILD. +kube::golang::server_targets() { + local targets=( + cmd/kube-proxy + cmd/kube-apiserver + cmd/kube-controller-manager + cmd/cloud-controller-manager + cmd/kubelet + cmd/kubeadm + cmd/hyperkube + vendor/k8s.io/kube-aggregator + vendor/k8s.io/apiextensions-apiserver + plugin/cmd/kube-scheduler + ) + echo "${targets[@]}" +} + +readonly KUBE_SERVER_TARGETS=($(kube::golang::server_targets)) +readonly KUBE_SERVER_BINARIES=("${KUBE_SERVER_TARGETS[@]##*/}") + +# The set of server targets that we are only building for Kubernetes nodes +# If you update this list, please also update build/BUILD. +kube::golang::node_targets() { + local targets=( + cmd/kube-proxy + cmd/kubelet + ) + echo "${targets[@]}" +} + +readonly KUBE_NODE_TARGETS=($(kube::golang::node_targets)) +readonly KUBE_NODE_BINARIES=("${KUBE_NODE_TARGETS[@]##*/}") +readonly KUBE_NODE_BINARIES_WIN=("${KUBE_NODE_BINARIES[@]/%/.exe}") + +if [[ -n "${KUBE_BUILD_PLATFORMS:-}" ]]; then + readonly KUBE_SERVER_PLATFORMS=(${KUBE_BUILD_PLATFORMS}) + readonly KUBE_NODE_PLATFORMS=(${KUBE_BUILD_PLATFORMS}) + readonly KUBE_TEST_PLATFORMS=(${KUBE_BUILD_PLATFORMS}) + readonly KUBE_CLIENT_PLATFORMS=(${KUBE_BUILD_PLATFORMS}) +elif [[ "${KUBE_FASTBUILD:-}" == "true" ]]; then + readonly KUBE_SERVER_PLATFORMS=(linux/amd64) + readonly KUBE_NODE_PLATFORMS=(linux/amd64) + if [[ "${KUBE_BUILDER_OS:-}" == "darwin"* ]]; then + readonly KUBE_TEST_PLATFORMS=( + darwin/amd64 + linux/amd64 + ) + readonly KUBE_CLIENT_PLATFORMS=( + darwin/amd64 + linux/amd64 + ) + else + readonly KUBE_TEST_PLATFORMS=(linux/amd64) + readonly KUBE_CLIENT_PLATFORMS=(linux/amd64) + fi +else + + # The server platform we are building on. + readonly KUBE_SERVER_PLATFORMS=( + linux/amd64 + linux/arm + linux/arm64 + linux/s390x + linux/ppc64le + ) + + # The node platforms we build for + readonly KUBE_NODE_PLATFORMS=( + linux/amd64 + linux/arm + linux/arm64 + linux/s390x + linux/ppc64le + windows/amd64 + ) + + # If we update this we should also update the set of platforms whose standard library is precompiled for in build/build-image/cross/Dockerfile + readonly KUBE_CLIENT_PLATFORMS=( + linux/amd64 + linux/386 + linux/arm + linux/arm64 + linux/s390x + linux/ppc64le + darwin/amd64 + darwin/386 + windows/amd64 + windows/386 + ) + + # Which platforms we should compile test targets for. Not all client platforms need these tests + readonly KUBE_TEST_PLATFORMS=( + linux/amd64 + linux/arm + linux/arm64 + linux/s390x + linux/ppc64le + darwin/amd64 + windows/amd64 + ) +fi + +# The set of client targets that we are building for all platforms +# If you update this list, please also update build/BUILD. +readonly KUBE_CLIENT_TARGETS=( + cmd/kubectl +) +readonly KUBE_CLIENT_BINARIES=("${KUBE_CLIENT_TARGETS[@]##*/}") +readonly KUBE_CLIENT_BINARIES_WIN=("${KUBE_CLIENT_BINARIES[@]/%/.exe}") + +# The set of test targets that we are building for all platforms +# If you update this list, please also update build/BUILD. +kube::golang::test_targets() { + local targets=( + cmd/gendocs + cmd/genkubedocs + cmd/genman + cmd/genyaml + cmd/genswaggertypedocs + cmd/linkcheck + vendor/github.com/onsi/ginkgo/ginkgo + ) + echo "${targets[@]}" +} +readonly KUBE_TEST_TARGETS=($(kube::golang::test_targets)) +readonly KUBE_TEST_BINARIES=("${KUBE_TEST_TARGETS[@]##*/}") +readonly KUBE_TEST_BINARIES_WIN=("${KUBE_TEST_BINARIES[@]/%/.exe}") +# If you update this list, please also update build/BUILD. +readonly KUBE_TEST_PORTABLE=( + test/kubemark + hack/get-build.sh + hack/lib +) + +# Test targets which run on the Kubernetes clusters directly, so we only +# need to target server platforms. +# These binaries will be distributed in the kubernetes-test tarball. +# If you update this list, please also update build/BUILD. +kube::golang::server_test_targets() { + local targets=( + cmd/kubemark + vendor/github.com/onsi/ginkgo/ginkgo + ) + + echo "${targets[@]}" +} + +readonly KUBE_TEST_SERVER_TARGETS=($(kube::golang::server_test_targets)) +readonly KUBE_TEST_SERVER_BINARIES=("${KUBE_TEST_SERVER_TARGETS[@]##*/}") +readonly KUBE_TEST_SERVER_PLATFORMS=("${KUBE_SERVER_PLATFORMS[@]}") + +# Gigabytes desired for parallel platform builds. 11 is fairly +# arbitrary, but is a reasonable splitting point for 2015 +# laptops-versus-not. +readonly KUBE_PARALLEL_BUILD_MEMORY=11 + +# TODO(pipejakob) gke-certificates-controller is included here to exercise its +# compilation, but it doesn't need to be distributed in any of our tars. Its +# code is only living in this repo temporarily until it finds a new home. +readonly KUBE_ALL_TARGETS=( + "${KUBE_SERVER_TARGETS[@]}" + "${KUBE_CLIENT_TARGETS[@]}" + "${KUBE_TEST_TARGETS[@]}" + "${KUBE_TEST_SERVER_TARGETS[@]}" + cmd/gke-certificates-controller +) +readonly KUBE_ALL_BINARIES=("${KUBE_ALL_TARGETS[@]##*/}") + +readonly KUBE_STATIC_LIBRARIES=( + cloud-controller-manager + kube-apiserver + kube-controller-manager + kube-scheduler + kube-proxy + kube-aggregator + kubeadm + kubectl +) + +# Add any files with those //generate annotations in the array below. +readonly KUBE_BINDATAS=( +) + +kube::golang::is_statically_linked_library() { + local e + for e in "${KUBE_STATIC_LIBRARIES[@]}"; do [[ "$1" == *"/$e" ]] && return 0; done; + # Allow individual overrides--e.g., so that you can get a static build of + # kubectl for inclusion in a container. + if [ -n "${KUBE_STATIC_OVERRIDES:+x}" ]; then + for e in "${KUBE_STATIC_OVERRIDES[@]}"; do [[ "$1" == *"/$e" ]] && return 0; done; + fi + return 1; +} + +# kube::binaries_from_targets take a list of build targets and return the +# full go package to be built +kube::golang::binaries_from_targets() { + local target + for target; do + # If the target starts with what looks like a domain name, assume it has a + # fully-qualified package name rather than one that needs the Kubernetes + # package prepended. + if [[ "${target}" =~ ^([[:alnum:]]+".")+[[:alnum:]]+"/" ]]; then + echo "${target}" + else + echo "${KUBE_GO_PACKAGE}/${target}" + fi + done +} + +# Asks golang what it thinks the host platform is. The go tool chain does some +# slightly different things when the target platform matches the host platform. +kube::golang::host_platform() { + echo "$(go env GOHOSTOS)/$(go env GOHOSTARCH)" +} + +kube::golang::current_platform() { + local os="${GOOS-}" + if [[ -z $os ]]; then + os=$(go env GOHOSTOS) + fi + + local arch="${GOARCH-}" + if [[ -z $arch ]]; then + arch=$(go env GOHOSTARCH) + fi + + echo "$os/$arch" +} + +# Takes the platform name ($1) and sets the appropriate golang env variables +# for that platform. +kube::golang::set_platform_envs() { + [[ -n ${1-} ]] || { + kube::log::error_exit "!!! Internal error. No platform set in kube::golang::set_platform_envs" + } + + export GOOS=${platform%/*} + export GOARCH=${platform##*/} + + # Do not set CC when building natively on a platform, only if cross-compiling from linux/amd64 + if [[ $(kube::golang::host_platform) == "linux/amd64" ]]; then + # Dynamic CGO linking for other server architectures than linux/amd64 goes here + # If you want to include support for more server platforms than these, add arch-specific gcc names here + case "${platform}" in + "linux/arm") + export CGO_ENABLED=1 + export CC=arm-linux-gnueabihf-gcc + ;; + "linux/arm64") + export CGO_ENABLED=1 + export CC=aarch64-linux-gnu-gcc + ;; + "linux/ppc64le") + export CGO_ENABLED=1 + export CC=powerpc64le-linux-gnu-gcc + ;; + "linux/s390x") + export CGO_ENABLED=1 + export CC=s390x-linux-gnu-gcc + ;; + esac + fi +} + +kube::golang::unset_platform_envs() { + unset GOOS + unset GOARCH + unset GOROOT + unset CGO_ENABLED + unset CC +} + +# Create the GOPATH tree under $KUBE_OUTPUT +kube::golang::create_gopath_tree() { + local go_pkg_dir="${KUBE_GOPATH}/src/${KUBE_GO_PACKAGE}" + local go_pkg_basedir=$(dirname "${go_pkg_dir}") + + mkdir -p "${go_pkg_basedir}" + + # TODO: This symlink should be relative. + # if [[ ! -e "${go_pkg_dir}" || "$(readlink ${go_pkg_dir})" != "${KUBE_ROOT}" ]]; then + # ln -snf "${KUBE_ROOT}" "${go_pkg_dir}" + # fi + + cat >"${KUBE_GOPATH}/BUILD" <` only works for a single pkg. + local subdir + subdir=$(kube::realpath . | sed "s|$KUBE_ROOT||") + cd "${KUBE_GOPATH}/src/${KUBE_GO_PACKAGE}/${subdir}" + + # Set GOROOT so binaries that parse code can work properly. + export GOROOT=$(go env GOROOT) + + # Unset GOBIN in case it already exists in the current session. + unset GOBIN + + # This seems to matter to some tools (godep, ginkgo...) + export GO15VENDOREXPERIMENT=1 +} + +# This will take binaries from $GOPATH/bin and copy them to the appropriate +# place in ${KUBE_OUTPUT_BINDIR} +# +# Ideally this wouldn't be necessary and we could just set GOBIN to +# KUBE_OUTPUT_BINDIR but that won't work in the face of cross compilation. 'go +# install' will place binaries that match the host platform directly in $GOBIN +# while placing cross compiled binaries into `platform_arch` subdirs. This +# complicates pretty much everything else we do around packaging and such. +kube::golang::place_bins() { + local host_platform + host_platform=$(kube::golang::host_platform) + + V=2 kube::log::status "Placing binaries" + + local platform + for platform in "${KUBE_CLIENT_PLATFORMS[@]}"; do + # The substitution on platform_src below will replace all slashes with + # underscores. It'll transform darwin/amd64 -> darwin_amd64. + local platform_src="/${platform//\//_}" + if [[ $platform == $host_platform ]]; then + platform_src="" + rm -f "${THIS_PLATFORM_BIN}" + ln -s "${KUBE_OUTPUT_BINPATH}/${platform}" "${THIS_PLATFORM_BIN}" + fi + + local full_binpath_src="${KUBE_GOPATH}/bin${platform_src}" + if [[ -d "${full_binpath_src}" ]]; then + mkdir -p "${KUBE_OUTPUT_BINPATH}/${platform}" + find "${full_binpath_src}" -maxdepth 1 -type f -exec \ + rsync -pc {} "${KUBE_OUTPUT_BINPATH}/${platform}" \; + fi + done +} + +kube::golang::fallback_if_stdlib_not_installable() { + local go_root_dir=$(go env GOROOT); + local go_host_os=$(go env GOHOSTOS); + local go_host_arch=$(go env GOHOSTARCH); + local cgo_pkg_dir=${go_root_dir}/pkg/${go_host_os}_${go_host_arch}_cgo; + + if [ -e ${cgo_pkg_dir} ]; then + return 0; + fi + + if [ -w ${go_root_dir}/pkg ]; then + return 0; + fi + + kube::log::status "+++ Warning: stdlib pkg with cgo flag not found."; + kube::log::status "+++ Warning: stdlib pkg cannot be rebuilt since ${go_root_dir}/pkg is not writable by `whoami`"; + kube::log::status "+++ Warning: Make ${go_root_dir}/pkg writable for `whoami` for a one-time stdlib install, Or" + kube::log::status "+++ Warning: Rebuild stdlib using the command 'CGO_ENABLED=0 go install -a -installsuffix cgo std'"; + kube::log::status "+++ Falling back to go build, which is slower"; + + use_go_build=true +} + +# Builds the toolchain necessary for building kube. This needs to be +# built only on the host platform. +# TODO: Find this a proper home. +# Ideally, not a shell script because testing shell scripts is painful. +kube::golang::build_kube_toolchain() { + local targets=( + hack/cmd/teststale + vendor/github.com/jteeuwen/go-bindata/go-bindata + ) + + local binaries + binaries=($(kube::golang::binaries_from_targets "${targets[@]}")) + + kube::log::status "Building the toolchain targets:" "${binaries[@]}" + go install "${goflags[@]:+${goflags[@]}}" \ + -gcflags "${gogcflags}" \ + -ldflags "${goldflags}" \ + "${binaries[@]:+${binaries[@]}}" +} + +# Try and replicate the native binary placement of go install without +# calling go install. +kube::golang::output_filename_for_binary() { + local binary=$1 + local platform=$2 + local output_path="${KUBE_GOPATH}/bin" + if [[ $platform != $host_platform ]]; then + output_path="${output_path}/${platform//\//_}" + fi + local bin=$(basename "${binary}") + if [[ ${GOOS} == "windows" ]]; then + bin="${bin}.exe" + fi + echo "${output_path}/${bin}" +} + +kube::golang::build_binaries_for_platform() { + local platform=$1 + local use_go_build=${2-} + + local -a statics=() + local -a nonstatics=() + local -a tests=() + + V=2 kube::log::info "Env for ${platform}: GOOS=${GOOS-} GOARCH=${GOARCH-} GOROOT=${GOROOT-} CGO_ENABLED=${CGO_ENABLED-} CC=${CC-}" + + for binary in "${binaries[@]}"; do + if [[ "${binary}" =~ ".test"$ ]]; then + tests+=($binary) + elif kube::golang::is_statically_linked_library "${binary}"; then + statics+=($binary) + else + nonstatics+=($binary) + fi + done + + if [[ "${#statics[@]}" != 0 ]]; then + kube::golang::fallback_if_stdlib_not_installable; + fi + + if [[ -n ${use_go_build:-} ]]; then + kube::log::progress " " + for binary in "${statics[@]:+${statics[@]}}"; do + local outfile=$(kube::golang::output_filename_for_binary "${binary}" "${platform}") + CGO_ENABLED=0 go build -o "${outfile}" \ + "${goflags[@]:+${goflags[@]}}" \ + -gcflags "${gogcflags}" \ + -ldflags "${goldflags}" \ + "${binary}" + kube::log::progress "*" + done + for binary in "${nonstatics[@]:+${nonstatics[@]}}"; do + local outfile=$(kube::golang::output_filename_for_binary "${binary}" "${platform}") + go build -o "${outfile}" \ + "${goflags[@]:+${goflags[@]}}" \ + -gcflags "${gogcflags}" \ + -ldflags "${goldflags}" \ + "${binary}" + kube::log::progress "*" + done + kube::log::progress "\n" + else + # Use go install. + if [[ "${#nonstatics[@]}" != 0 ]]; then + go install "${goflags[@]:+${goflags[@]}}" \ + -gcflags "${gogcflags}" \ + -ldflags "${goldflags}" \ + "${nonstatics[@]:+${nonstatics[@]}}" + fi + if [[ "${#statics[@]}" != 0 ]]; then + CGO_ENABLED=0 go install -installsuffix cgo "${goflags[@]:+${goflags[@]}}" \ + -gcflags "${gogcflags}" \ + -ldflags "${goldflags}" \ + "${statics[@]:+${statics[@]}}" + fi + fi + + for test in "${tests[@]:+${tests[@]}}"; do + local outfile=$(kube::golang::output_filename_for_binary "${test}" \ + "${platform}") + + local testpkg="$(dirname ${test})" + + # Staleness check always happens on the host machine, so we don't + # have to locate the `teststale` binaries for the other platforms. + # Since we place the host binaries in `$KUBE_GOPATH/bin`, we can + # assume that the binary exists there, if it exists at all. + # Otherwise, something has gone wrong with building the `teststale` + # binary and we should safely proceed building the test binaries + # assuming that they are stale. There is no good reason to error + # out. + if test -x "${KUBE_GOPATH}/bin/teststale" && ! "${KUBE_GOPATH}/bin/teststale" -binary "${outfile}" -package "${testpkg}" + then + continue + fi + + # `go test -c` below directly builds the binary. It builds the packages, + # but it never installs them. `go test -i` only installs the dependencies + # of the test, but not the test package itself. So neither `go test -c` + # nor `go test -i` installs, for example, test/e2e.a. And without that, + # doing a staleness check on k8s.io/kubernetes/test/e2e package always + # returns true (always stale). And that's why we need to install the + # test package. + go install "${goflags[@]:+${goflags[@]}}" \ + -gcflags "${gogcflags}" \ + -ldflags "${goldflags}" \ + "${testpkg}" + + mkdir -p "$(dirname ${outfile})" + go test -i -c \ + "${goflags[@]:+${goflags[@]}}" \ + -gcflags "${gogcflags}" \ + -ldflags "${goldflags}" \ + -o "${outfile}" \ + "${testpkg}" + done +} + +# Return approximate physical memory available in gigabytes. +kube::golang::get_physmem() { + local mem + + # Linux kernel version >=3.14, in kb + if mem=$(grep MemAvailable /proc/meminfo | awk '{ print $2 }'); then + echo $(( ${mem} / 1048576 )) + return + fi + + # Linux, in kb + if mem=$(grep MemTotal /proc/meminfo | awk '{ print $2 }'); then + echo $(( ${mem} / 1048576 )) + return + fi + + # OS X, in bytes. Note that get_physmem, as used, should only ever + # run in a Linux container (because it's only used in the multiple + # platform case, which is a Dockerized build), but this is provided + # for completeness. + if mem=$(sysctl -n hw.memsize 2>/dev/null); then + echo $(( ${mem} / 1073741824 )) + return + fi + + # If we can't infer it, just give up and assume a low memory system + echo 1 +} + +# Build binaries targets specified +# +# Input: +# $@ - targets and go flags. If no targets are set then all binaries targets +# are built. +# KUBE_BUILD_PLATFORMS - Incoming variable of targets to build for. If unset +# then just the host architecture is built. +kube::golang::build_binaries() { + # Create a sub-shell so that we don't pollute the outer environment + ( + # Check for `go` binary and set ${GOPATH}. + kube::golang::setup_env + V=2 kube::log::info "Go version: $(go version)" + + local host_platform + host_platform=$(kube::golang::host_platform) + + # Use eval to preserve embedded quoted strings. + local goflags goldflags gogcflags + eval "goflags=(${GOFLAGS:-})" + goldflags="${GOLDFLAGS:-} $(kube::version::ldflags)" + gogcflags="${GOGCFLAGS:-}" + + local use_go_build + local -a targets=() + local arg + + for arg; do + if [[ "${arg}" == "--use_go_build" ]]; then + use_go_build=true + elif [[ "${arg}" == -* ]]; then + # Assume arguments starting with a dash are flags to pass to go. + goflags+=("${arg}") + else + targets+=("${arg}") + fi + done + + if [[ ${#targets[@]} -eq 0 ]]; then + targets=("${KUBE_ALL_TARGETS[@]}") + fi + + local -a platforms=(${KUBE_BUILD_PLATFORMS:-}) + if [[ ${#platforms[@]} -eq 0 ]]; then + platforms=("${host_platform}") + fi + + local binaries + binaries=($(kube::golang::binaries_from_targets "${targets[@]}")) + + local parallel=false + if [[ ${#platforms[@]} -gt 1 ]]; then + local gigs + gigs=$(kube::golang::get_physmem) + + if [[ ${gigs} -ge ${KUBE_PARALLEL_BUILD_MEMORY} ]]; then + kube::log::status "Multiple platforms requested and available ${gigs}G >= threshold ${KUBE_PARALLEL_BUILD_MEMORY}G, building platforms in parallel" + parallel=true + else + kube::log::status "Multiple platforms requested, but available ${gigs}G < threshold ${KUBE_PARALLEL_BUILD_MEMORY}G, building platforms in serial" + parallel=false + fi + fi + + # First build the toolchain before building any other targets + kube::golang::build_kube_toolchain + + kube::log::status "Generating bindata:" "${KUBE_BINDATAS[@]}" + for bindata in ${KUBE_BINDATAS[@]}; do + # Only try to generate bindata if the file exists, since in some cases + # one-off builds of individual directories may exclude some files. + if [[ -f "${KUBE_ROOT}/${bindata}" ]]; then + go generate "${goflags[@]:+${goflags[@]}}" "${KUBE_ROOT}/${bindata}" + fi + done + + if [[ "${parallel}" == "true" ]]; then + kube::log::status "Building go targets for {${platforms[*]}} in parallel (output will appear in a burst when complete):" "${targets[@]}" + local platform + for platform in "${platforms[@]}"; do ( + kube::golang::set_platform_envs "${platform}" + kube::log::status "${platform}: go build started" + kube::golang::build_binaries_for_platform ${platform} ${use_go_build:-} + kube::log::status "${platform}: go build finished" + ) &> "/tmp//${platform//\//_}.build" & + done + + local fails=0 + for job in $(jobs -p); do + wait ${job} || let "fails+=1" + done + + for platform in "${platforms[@]}"; do + cat "/tmp//${platform//\//_}.build" + done + + exit ${fails} + else + for platform in "${platforms[@]}"; do + kube::log::status "Building go targets for ${platform}:" "${targets[@]}" + ( + kube::golang::set_platform_envs "${platform}" + kube::golang::build_binaries_for_platform ${platform} ${use_go_build:-} + ) + done + fi + ) +} diff --git a/hack/lib/init.sh b/hack/lib/init.sh new file mode 100644 index 0000000000..f22a4aeef0 --- /dev/null +++ b/hack/lib/init.sh @@ -0,0 +1,172 @@ +#!/bin/bash + +# Copyright 2014 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +# The root of the build/dist directory +KUBE_ROOT="$(cd "$(dirname "${BASH_SOURCE}")/../.." && pwd -P)" + +KUBE_OUTPUT_SUBPATH="${KUBE_OUTPUT_SUBPATH:-_output/local}" +KUBE_OUTPUT="${KUBE_ROOT}/${KUBE_OUTPUT_SUBPATH}" +KUBE_OUTPUT_BINPATH="${KUBE_OUTPUT}/bin" + +# This controls rsync compression. Set to a value > 0 to enable rsync +# compression for build container +KUBE_RSYNC_COMPRESS="${KUBE_RSYNC_COMPRESS:-0}" + +# Set no_proxy for localhost if behind a proxy, otherwise, +# the connections to localhost in scripts will time out +export no_proxy=127.0.0.1,localhost + +# This is a symlink to binaries for "this platform", e.g. build tools. +THIS_PLATFORM_BIN="${KUBE_ROOT}/_output/bin" + +source "${KUBE_ROOT}/hack/lib/util.sh" +source "${KUBE_ROOT}/cluster/lib/logging.sh" + +kube::log::install_errexit + +source "${KUBE_ROOT}/hack/lib/version.sh" +source "${KUBE_ROOT}/hack/lib/golang.sh" +source "${KUBE_ROOT}/hack/lib/etcd.sh" + +KUBE_OUTPUT_HOSTBIN="${KUBE_OUTPUT_BINPATH}/$(kube::util::host_platform)" + +# list of all available group versions. This should be used when generated code +# or when starting an API server that you want to have everything. +# most preferred version for a group should appear first +KUBE_AVAILABLE_GROUP_VERSIONS="${KUBE_AVAILABLE_GROUP_VERSIONS:-\ +v1 \ +admissionregistration.k8s.io/v1alpha1 \ +admission.k8s.io/v1alpha1 \ +apps/v1beta1 \ +apps/v1beta2 \ +authentication.k8s.io/v1 \ +authentication.k8s.io/v1beta1 \ +authorization.k8s.io/v1 \ +authorization.k8s.io/v1beta1 \ +autoscaling/v1 \ +autoscaling/v2beta1 \ +batch/v1 \ +batch/v1beta1 \ +batch/v2alpha1 \ +certificates.k8s.io/v1beta1 \ +extensions/v1beta1 \ +imagepolicy.k8s.io/v1alpha1 \ +networking.k8s.io/v1 \ +policy/v1beta1 \ +rbac.authorization.k8s.io/v1 \ +rbac.authorization.k8s.io/v1beta1 \ +rbac.authorization.k8s.io/v1alpha1 \ +scheduling.k8s.io/v1alpha1 \ +settings.k8s.io/v1alpha1 \ +storage.k8s.io/v1beta1 \ +storage.k8s.io/v1 \ +}" + +# not all group versions are exposed by the server. This list contains those +# which are not available so we don't generate clients or swagger for them +KUBE_NONSERVER_GROUP_VERSIONS=" + abac.authorization.kubernetes.io/v0 \ + abac.authorization.kubernetes.io/v1beta1 \ + componentconfig/v1alpha1 \ + imagepolicy.k8s.io/v1alpha1\ + admission.k8s.io/v1alpha1\ +" + +# This emulates "readlink -f" which is not available on MacOS X. +# Test: +# T=/tmp/$$.$RANDOM +# mkdir $T +# touch $T/file +# mkdir $T/dir +# ln -s $T/file $T/linkfile +# ln -s $T/dir $T/linkdir +# function testone() { +# X=$(readlink -f $1 2>&1) +# Y=$(kube::readlinkdashf $1 2>&1) +# if [ "$X" != "$Y" ]; then +# echo readlinkdashf $1: expected "$X", got "$Y" +# fi +# } +# testone / +# testone /tmp +# testone $T +# testone $T/file +# testone $T/dir +# testone $T/linkfile +# testone $T/linkdir +# testone $T/nonexistant +# testone $T/linkdir/file +# testone $T/linkdir/dir +# testone $T/linkdir/linkfile +# testone $T/linkdir/linkdir +function kube::readlinkdashf { + # run in a subshell for simpler 'cd' + ( + if [[ -d "$1" ]]; then # This also catch symlinks to dirs. + cd "$1" + pwd -P + else + cd $(dirname "$1") + local f + f=$(basename "$1") + if [[ -L "$f" ]]; then + readlink "$f" + else + echo "$(pwd -P)/${f}" + fi + fi + ) +} + +# This emulates "realpath" which is not available on MacOS X +# Test: +# T=/tmp/$$.$RANDOM +# mkdir $T +# touch $T/file +# mkdir $T/dir +# ln -s $T/file $T/linkfile +# ln -s $T/dir $T/linkdir +# function testone() { +# X=$(realpath $1 2>&1) +# Y=$(kube::realpath $1 2>&1) +# if [ "$X" != "$Y" ]; then +# echo realpath $1: expected "$X", got "$Y" +# fi +# } +# testone / +# testone /tmp +# testone $T +# testone $T/file +# testone $T/dir +# testone $T/linkfile +# testone $T/linkdir +# testone $T/nonexistant +# testone $T/linkdir/file +# testone $T/linkdir/dir +# testone $T/linkdir/linkfile +# testone $T/linkdir/linkdir +kube::realpath() { + if [[ ! -e "$1" ]]; then + echo "$1: No such file or directory" >&2 + return 1 + fi + kube::readlinkdashf "$1" +} + diff --git a/hack/lib/util.sh b/hack/lib/util.sh new file mode 100644 index 0000000000..a2981b8286 --- /dev/null +++ b/hack/lib/util.sh @@ -0,0 +1,765 @@ +#!/bin/bash + +# Copyright 2014 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +kube::util::sortable_date() { + date "+%Y%m%d-%H%M%S" +} + +kube::util::wait_for_url() { + local url=$1 + local prefix=${2:-} + local wait=${3:-1} + local times=${4:-30} + + which curl >/dev/null || { + kube::log::usage "curl must be installed" + exit 1 + } + + local i + for i in $(seq 1 $times); do + local out + if out=$(curl --max-time 1 -gkfs $url 2>/dev/null); then + kube::log::status "On try ${i}, ${prefix}: ${out}" + return 0 + fi + sleep ${wait} + done + kube::log::error "Timed out waiting for ${prefix} to answer at ${url}; tried ${times} waiting ${wait} between each" + return 1 +} + +# returns a random port +kube::util::get_random_port() { + awk -v min=1024 -v max=65535 'BEGIN{srand(); print int(min+rand()*(max-min+1))}' +} + +# use netcat to check if the host($1):port($2) is free (return 0 means free, 1 means used) +kube::util::test_host_port_free() { + local host=$1 + local port=$2 + local success=0 + local fail=1 + + which nc >/dev/null || { + kube::log::usage "netcat isn't installed, can't verify if ${host}:${port} is free, skipping the check..." + return ${success} + } + + if [ ! $(nc -vz "${host}" "${port}") ]; then + kube::log::status "${host}:${port} is free, proceeding..." + return ${success} + else + kube::log::status "${host}:${port} is already used" + return ${fail} + fi +} + +# Example: kube::util::trap_add 'echo "in trap DEBUG"' DEBUG +# See: http://stackoverflow.com/questions/3338030/multiple-bash-traps-for-the-same-signal +kube::util::trap_add() { + local trap_add_cmd + trap_add_cmd=$1 + shift + + for trap_add_name in "$@"; do + local existing_cmd + local new_cmd + + # Grab the currently defined trap commands for this trap + existing_cmd=`trap -p "${trap_add_name}" | awk -F"'" '{print $2}'` + + if [[ -z "${existing_cmd}" ]]; then + new_cmd="${trap_add_cmd}" + else + new_cmd="${existing_cmd};${trap_add_cmd}" + fi + + # Assign the test + trap "${new_cmd}" "${trap_add_name}" + done +} + +# Opposite of kube::util::ensure-temp-dir() +kube::util::cleanup-temp-dir() { + rm -rf "${KUBE_TEMP}" +} + +# Create a temp dir that'll be deleted at the end of this bash session. +# +# Vars set: +# KUBE_TEMP +kube::util::ensure-temp-dir() { + if [[ -z ${KUBE_TEMP-} ]]; then + KUBE_TEMP=$(mktemp -d 2>/dev/null || mktemp -d -t kubernetes.XXXXXX) + kube::util::trap_add kube::util::cleanup-temp-dir EXIT + fi +} + +# This figures out the host platform without relying on golang. We need this as +# we don't want a golang install to be a prerequisite to building yet we need +# this info to figure out where the final binaries are placed. +kube::util::host_platform() { + local host_os + local host_arch + case "$(uname -s)" in + Darwin) + host_os=darwin + ;; + Linux) + host_os=linux + ;; + *) + kube::log::error "Unsupported host OS. Must be Linux or Mac OS X." + exit 1 + ;; + esac + + case "$(uname -m)" in + x86_64*) + host_arch=amd64 + ;; + i?86_64*) + host_arch=amd64 + ;; + amd64*) + host_arch=amd64 + ;; + aarch64*) + host_arch=arm64 + ;; + arm64*) + host_arch=arm64 + ;; + arm*) + host_arch=arm + ;; + i?86*) + host_arch=x86 + ;; + s390x*) + host_arch=s390x + ;; + ppc64le*) + host_arch=ppc64le + ;; + *) + kube::log::error "Unsupported host arch. Must be x86_64, 386, arm, arm64, s390x or ppc64le." + exit 1 + ;; + esac + echo "${host_os}/${host_arch}" +} + +kube::util::find-binary-for-platform() { + local -r lookfor="$1" + local -r platform="$2" + local locations=( + "${KUBE_ROOT}/_output/bin/${lookfor}" + "${KUBE_ROOT}/_output/dockerized/bin/${platform}/${lookfor}" + "${KUBE_ROOT}/_output/local/bin/${platform}/${lookfor}" + "${KUBE_ROOT}/platforms/${platform}/${lookfor}" + ) + # Also search for binary in bazel build tree. + # In some cases we have to name the binary $BINARY_bin, since there was a + # directory named $BINARY next to it. + locations+=($(find "${KUBE_ROOT}/bazel-bin/" -type f -executable \ + \( -name "${lookfor}" -o -name "${lookfor}_bin" \) 2>/dev/null || true) ) + + # List most recently-updated location. + local -r bin=$( (ls -t "${locations[@]}" 2>/dev/null || true) | head -1 ) + echo -n "${bin}" +} + +kube::util::find-binary() { + kube::util::find-binary-for-platform "$1" "$(kube::util::host_platform)" +} + +# Run all known doc generators (today gendocs and genman for kubectl) +# $1 is the directory to put those generated documents +kube::util::gen-docs() { + local dest="$1" + + # Find binary + gendocs=$(kube::util::find-binary "gendocs") + genkubedocs=$(kube::util::find-binary "genkubedocs") + genman=$(kube::util::find-binary "genman") + genyaml=$(kube::util::find-binary "genyaml") + genfeddocs=$(kube::util::find-binary "genfeddocs") + + mkdir -p "${dest}/docs/user-guide/kubectl/" + "${gendocs}" "${dest}/docs/user-guide/kubectl/" + mkdir -p "${dest}/docs/admin/" + "${genkubedocs}" "${dest}/docs/admin/" "kube-apiserver" + "${genkubedocs}" "${dest}/docs/admin/" "kube-controller-manager" + "${genkubedocs}" "${dest}/docs/admin/" "cloud-controller-manager" + "${genkubedocs}" "${dest}/docs/admin/" "kube-proxy" + "${genkubedocs}" "${dest}/docs/admin/" "kube-scheduler" + "${genkubedocs}" "${dest}/docs/admin/" "kubelet" + + "${genfeddocs}" "${dest}/docs/admin/" "kubefed" + + mkdir -p "${dest}/docs/man/man1/" + "${genman}" "${dest}/docs/man/man1/" "kube-apiserver" + "${genman}" "${dest}/docs/man/man1/" "kube-controller-manager" + "${genman}" "${dest}/docs/man/man1/" "cloud-controller-manager" + "${genman}" "${dest}/docs/man/man1/" "kube-proxy" + "${genman}" "${dest}/docs/man/man1/" "kube-scheduler" + "${genman}" "${dest}/docs/man/man1/" "kubelet" + "${genman}" "${dest}/docs/man/man1/" "kubectl" + + mkdir -p "${dest}/docs/yaml/kubectl/" + "${genyaml}" "${dest}/docs/yaml/kubectl/" + + # create the list of generated files + pushd "${dest}" > /dev/null + touch docs/.generated_docs + find . -type f | cut -sd / -f 2- | LC_ALL=C sort > docs/.generated_docs + popd > /dev/null +} + +# Puts a placeholder for every generated doc. This makes the link checker work. +kube::util::set-placeholder-gen-docs() { + local list_file="${KUBE_ROOT}/docs/.generated_docs" + if [ -e ${list_file} ]; then + # remove all of the old docs; we don't want to check them in. + while read file; do + if [[ "${list_file}" != "${KUBE_ROOT}/${file}" ]]; then + cp "${KUBE_ROOT}/hack/autogenerated_placeholder.txt" "${KUBE_ROOT}/${file}" + fi + done <"${list_file}" + # The docs/.generated_docs file lists itself, so we don't need to explicitly + # delete it. + fi +} + +# Removes previously generated docs-- we don't want to check them in. $KUBE_ROOT +# must be set. +kube::util::remove-gen-docs() { + if [ -e "${KUBE_ROOT}/docs/.generated_docs" ]; then + # remove all of the old docs; we don't want to check them in. + while read file; do + rm "${KUBE_ROOT}/${file}" 2>/dev/null || true + done <"${KUBE_ROOT}/docs/.generated_docs" + # The docs/.generated_docs file lists itself, so we don't need to explicitly + # delete it. + fi +} + +# Takes a path $1 to traverse for md files to append the ga-beacon tracking +# link to, if needed. If $2 is set, just print files that are missing +# the link. +kube::util::gen-analytics() { + local path="$1" + local dryrun="${2:-}" + local mdfiles dir link + # find has some strange inconsistencies between darwin/linux. The + # path to search must end in '/' for linux, but darwin will put an extra + # slash in results if there is a trailing '/'. + if [[ $( uname ) == 'Linux' ]]; then + dir="${path}/" + else + dir="${path}" + fi + # We don't touch files in special dirs, and the kubectl docs are + # autogenerated by gendocs. + # Don't descend into .directories + mdfiles=($( find "${dir}" -name "*.md" -type f \ + -not -path '*/\.*' \ + -not -path "${path}/vendor/*" \ + -not -path "${path}/staging/*" \ + -not -path "${path}/third_party/*" \ + -not -path "${path}/_gopath/*" \ + -not -path "${path}/_output/*" \ + -not -path "${path}/docs/user-guide/kubectl/kubectl*" )) + for f in "${mdfiles[@]}"; do + link=$(kube::util::analytics-link "${f#${path}/}") + if grep -q -F -x "${link}" "${f}"; then + continue + elif [[ -z "${dryrun}" ]]; then + echo -e "\n\n${link}" >> "${f}" + else + echo "$f" + fi + done +} + +# Prints analytics link to append to a file at path $1. +kube::util::analytics-link() { + local path="$1" + echo "[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/${path}?pixel)]()" +} + +# Takes a group/version and returns the path to its location on disk, sans +# "pkg". E.g.: +# * default behavior: extensions/v1beta1 -> apis/extensions/v1beta1 +# * default behavior for only a group: experimental -> apis/experimental +# * Special handling for empty group: v1 -> api/v1, unversioned -> api/unversioned +# * Special handling for groups suffixed with ".k8s.io": foo.k8s.io/v1 -> apis/foo/v1 +# * Very special handling for when both group and version are "": / -> api +kube::util::group-version-to-pkg-path() { + staging_apis=( + $( + pushd ${KUBE_ROOT}/staging/src/k8s.io/api > /dev/null + find . -name types.go | xargs -n1 dirname | sed "s|\./||g" | sort + popd > /dev/null + ) + ) + + local group_version="$1" + + if [[ " ${staging_apis[@]} " =~ " ${group_version/.*k8s.io/} " ]]; then + echo "vendor/k8s.io/api/${group_version/.*k8s.io/}" + return + fi + + # "v1" is the API GroupVersion + if [[ "${group_version}" == "v1" ]]; then + echo "vendor/k8s.io/api/core/v1" + return + fi + + # Special cases first. + # TODO(lavalamp): Simplify this by moving pkg/api/v1 and splitting pkg/api, + # moving the results to pkg/apis/api. + case "${group_version}" in + # both group and version are "", this occurs when we generate deep copies for internal objects of the legacy v1 API. + __internal) + echo "pkg/api" + ;; + meta/v1) + echo "vendor/k8s.io/apimachinery/pkg/apis/meta/v1" + ;; + meta/v1) + echo "../vendor/k8s.io/apimachinery/pkg/apis/meta/v1" + ;; + meta/v1alpha1) + echo "vendor/k8s.io/apimachinery/pkg/apis/meta/v1alpha1" + ;; + meta/v1alpha1) + echo "../vendor/k8s.io/apimachinery/pkg/apis/meta/v1alpha1" + ;; + unversioned) + echo "pkg/api/unversioned" + ;; + *.k8s.io) + echo "pkg/apis/${group_version%.*k8s.io}" + ;; + *.k8s.io/*) + echo "pkg/apis/${group_version/.*k8s.io/}" + ;; + *) + echo "pkg/apis/${group_version%__internal}" + ;; + esac +} + +# Takes a group/version and returns the swagger-spec file name. +# default behavior: extensions/v1beta1 -> extensions_v1beta1 +# special case for v1: v1 -> v1 +kube::util::gv-to-swagger-name() { + local group_version="$1" + case "${group_version}" in + v1) + echo "v1" + ;; + *) + echo "${group_version%/*}_${group_version#*/}" + ;; + esac +} + + +# Fetches swagger spec from apiserver. +# Assumed vars: +# SWAGGER_API_PATH: Base path for swaggerapi on apiserver. Ex: +# http://localhost:8080/swaggerapi. +# SWAGGER_ROOT_DIR: Root dir where we want to save the fetched spec. +# VERSIONS: Array of group versions to include in swagger spec. +kube::util::fetch-swagger-spec() { + for ver in ${VERSIONS}; do + if [[ " ${KUBE_NONSERVER_GROUP_VERSIONS} " == *" ${ver} "* ]]; then + continue + fi + # fetch the swagger spec for each group version. + if [[ ${ver} == "v1" ]]; then + SUBPATH="api" + else + SUBPATH="apis" + fi + SUBPATH="${SUBPATH}/${ver}" + SWAGGER_JSON_NAME="$(kube::util::gv-to-swagger-name ${ver}).json" + curl -w "\n" -fs "${SWAGGER_API_PATH}${SUBPATH}" > "${SWAGGER_ROOT_DIR}/${SWAGGER_JSON_NAME}" + + # fetch the swagger spec for the discovery mechanism at group level. + if [[ ${ver} == "v1" ]]; then + continue + fi + SUBPATH="apis/"${ver%/*} + SWAGGER_JSON_NAME="${ver%/*}.json" + curl -w "\n" -fs "${SWAGGER_API_PATH}${SUBPATH}" > "${SWAGGER_ROOT_DIR}/${SWAGGER_JSON_NAME}" + done + + # fetch swagger specs for other discovery mechanism. + curl -w "\n" -fs "${SWAGGER_API_PATH}" > "${SWAGGER_ROOT_DIR}/resourceListing.json" + curl -w "\n" -fs "${SWAGGER_API_PATH}version" > "${SWAGGER_ROOT_DIR}/version.json" + curl -w "\n" -fs "${SWAGGER_API_PATH}api" > "${SWAGGER_ROOT_DIR}/api.json" + curl -w "\n" -fs "${SWAGGER_API_PATH}apis" > "${SWAGGER_ROOT_DIR}/apis.json" + curl -w "\n" -fs "${SWAGGER_API_PATH}logs" > "${SWAGGER_ROOT_DIR}/logs.json" +} + +# Returns the name of the upstream remote repository name for the local git +# repo, e.g. "upstream" or "origin". +kube::util::git_upstream_remote_name() { + git remote -v | grep fetch |\ + grep -E 'github.com[/:]kubernetes/kubernetes|k8s.io/kubernetes' |\ + head -n 1 | awk '{print $1}' +} + +# Exits script if working directory is dirty. If it's run interactively in the terminal +# the user can commit changes in a second terminal. This script will wait. +kube::util::ensure_clean_working_dir() { + while ! git diff HEAD --exit-code &>/dev/null; do + echo -e "\nUnexpected dirty working directory:\n" + if tty -s; then + git status -s + else + git diff -a # be more verbose in log files without tty + exit 1 + fi | sed 's/^/ /' + echo -e "\nCommit your changes in another terminal and then continue here by pressing enter." + read + done 1>&2 +} + + +# Installs the specified go package at a particular commit. +kube::util::go_install_from_commit() { + local -r pkg=$1 + local -r commit=$2 + + kube::util::ensure-temp-dir + mkdir -p "${KUBE_TEMP}/go/src" + GOPATH="${KUBE_TEMP}/go" go get -d -u "${pkg}" + ( + cd "${KUBE_TEMP}/go/src/${pkg}" + git checkout -q "${commit}" + GOPATH="${KUBE_TEMP}/go" go install "${pkg}" + ) + PATH="${KUBE_TEMP}/go/bin:${PATH}" + hash -r # force bash to clear PATH cache +} + +# Checks that the GOPATH is simple, i.e. consists only of one directory, not multiple. +kube::util::ensure_single_dir_gopath() { + if [[ "${GOPATH}" == *:* ]]; then + echo "GOPATH must consist of a single directory." 1>&2 + exit 1 + fi +} + +# Checks whether there are any files matching pattern $2 changed between the +# current branch and upstream branch named by $1. +# Returns 1 (false) if there are no changes, 0 (true) if there are changes +# detected. +kube::util::has_changes_against_upstream_branch() { + local -r git_branch=$1 + local -r pattern=$2 + local -r not_pattern=${3:-totallyimpossiblepattern} + local full_branch + + full_branch="$(kube::util::git_upstream_remote_name)/${git_branch}" + echo "Checking for '${pattern}' changes against '${full_branch}'" + # make sure the branch is valid, otherwise the check will pass erroneously. + if ! git describe "${full_branch}" >/dev/null; then + # abort! + exit 1 + fi + # notice this uses ... to find the first shared ancestor + if git diff --name-only "${full_branch}...HEAD" | grep -v -E "${not_pattern}" | grep "${pattern}" > /dev/null; then + return 0 + fi + # also check for pending changes + if git status --porcelain | grep -v -E "${not_pattern}" | grep "${pattern}" > /dev/null; then + echo "Detected '${pattern}' uncommitted changes." + return 0 + fi + echo "No '${pattern}' changes detected." + return 1 +} + +kube::util::download_file() { + local -r url=$1 + local -r destination_file=$2 + + rm ${destination_file} 2&> /dev/null || true + + for i in $(seq 5) + do + if ! curl -fsSL --retry 3 --keepalive-time 2 ${url} -o ${destination_file}; then + echo "Downloading ${url} failed. $((5-i)) retries left." + sleep 1 + else + echo "Downloading ${url} succeed" + return 0 + fi + done + return 1 +} + +# Test whether openssl is installed. +# Sets: +# OPENSSL_BIN: The path to the openssl binary to use +function kube::util::test_openssl_installed { + openssl version >& /dev/null + if [ "$?" != "0" ]; then + echo "Failed to run openssl. Please ensure openssl is installed" + exit 1 + fi + OPENSSL_BIN=$(command -v openssl) +} + +# creates a client CA, args are sudo, dest-dir, ca-id, purpose +# purpose is dropped in after "key encipherment", you usually want +# '"client auth"' +# '"server auth"' +# '"client auth","server auth"' +function kube::util::create_signing_certkey { + local sudo=$1 + local dest_dir=$2 + local id=$3 + local purpose=$4 + # Create client ca + ${sudo} /bin/bash -e < "${dest_dir}/${id}-ca-config.json" +EOF +} + +# signs a client certificate: args are sudo, dest-dir, CA, filename (roughly), username, groups... +function kube::util::create_client_certkey { + local sudo=$1 + local dest_dir=$2 + local ca=$3 + local id=$4 + local cn=${5:-$4} + local groups="" + local SEP="" + shift 5 + while [ -n "${1:-}" ]; do + groups+="${SEP}{\"O\":\"$1\"}" + SEP="," + shift 1 + done + ${sudo} /bin/bash -e < /dev/null +apiVersion: v1 +kind: Config +clusters: + - cluster: + certificate-authority: ${ca_file} + server: https://${api_host}:${api_port}/ + name: local-up-cluster +users: + - user: + token: ${token} + client-certificate: ${dest_dir}/client-${client_id}.crt + client-key: ${dest_dir}/client-${client_id}.key + name: local-up-cluster +contexts: + - context: + cluster: local-up-cluster + user: local-up-cluster + name: local-up-cluster +current-context: local-up-cluster +EOF + + # flatten the kubeconfig files to make them self contained + username=$(whoami) + ${sudo} /bin/bash -e < "/tmp/${client_id}.kubeconfig" + mv -f "/tmp/${client_id}.kubeconfig" "${dest_dir}/${client_id}.kubeconfig" + chown ${username} "${dest_dir}/${client_id}.kubeconfig" +EOF +} + +# Determines if docker can be run, failures may simply require that the user be added to the docker group. +function kube::util::ensure_docker_daemon_connectivity { + DOCKER=(docker ${DOCKER_OPTS}) + if ! "${DOCKER[@]}" info > /dev/null 2>&1 ; then + cat <<'EOF' >&2 +Can't connect to 'docker' daemon. please fix and retry. + +Possible causes: + - Docker Daemon not started + - Linux: confirm via your init system + - macOS w/ docker-machine: run `docker-machine ls` and `docker-machine start ` + - macOS w/ Docker for Mac: Check the menu bar and start the Docker application + - DOCKER_HOST hasn't been set or is set incorrectly + - Linux: domain socket is used, DOCKER_* should be unset. In Bash run `unset ${!DOCKER_*}` + - macOS w/ docker-machine: run `eval "$(docker-machine env )"` + - macOS w/ Docker for Mac: domain socket is used, DOCKER_* should be unset. In Bash run `unset ${!DOCKER_*}` + - Other things to check: + - Linux: User isn't in 'docker' group. Add and relogin. + - Something like 'sudo usermod -a -G docker ${USER}' + - RHEL7 bug and workaround: https://bugzilla.redhat.com/show_bug.cgi?id=1119282#c8 +EOF + return 1 + fi +} + +# Wait for background jobs to finish. Return with +# an error status if any of the jobs failed. +kube::util::wait-for-jobs() { + local fail=0 + local job + for job in $(jobs -p); do + wait "${job}" || fail=$((fail + 1)) + done + return ${fail} +} + +# kube::util::join +# Concatenates the list elements with the delimiter passed as first parameter +# +# Ex: kube::util::join , a b c +# -> a,b,c +function kube::util::join { + local IFS="$1" + shift + echo "$*" +} + +# Downloads cfssl/cfssljson into $1 directory if they do not already exist in PATH +# +# Assumed vars: +# $1 (cfssl directory) (optional) +# +# Sets: +# CFSSL_BIN: The path of the installed cfssl binary +# CFSSLJSON_BIN: The path of the installed cfssljson binary +# +function kube::util::ensure-cfssl { + if command -v cfssl &>/dev/null && command -v cfssljson &>/dev/null; then + CFSSL_BIN=$(command -v cfssl) + CFSSLJSON_BIN=$(command -v cfssljson) + return 0 + fi + + # Create a temp dir for cfssl if no directory was given + local cfssldir=${1:-} + if [[ -z "${cfssldir}" ]]; then + kube::util::ensure-temp-dir + cfssldir="${KUBE_TEMP}/cfssl" + fi + + mkdir -p "${cfssldir}" + pushd "${cfssldir}" > /dev/null + + echo "Unable to successfully run 'cfssl' from $PATH; downloading instead..." + kernel=$(uname -s) + case "${kernel}" in + Linux) + curl -s -L -o cfssl https://pkg.cfssl.org/R1.2/cfssl_linux-amd64 + curl -s -L -o cfssljson https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64 + ;; + Darwin) + curl -s -L -o cfssl https://pkg.cfssl.org/R1.2/cfssl_darwin-amd64 + curl -s -L -o cfssljson https://pkg.cfssl.org/R1.2/cfssljson_darwin-amd64 + ;; + *) + echo "Unknown, unsupported platform: ${kernel}." >&2 + echo "Supported platforms: Linux, Darwin." >&2 + exit 2 + esac + + chmod +x cfssl || true + chmod +x cfssljson || true + + CFSSL_BIN="${cfssldir}/cfssl" + CFSSLJSON_BIN="${cfssldir}/cfssljson" + if [[ ! -x ${CFSSL_BIN} || ! -x ${CFSSLJSON_BIN} ]]; then + echo "Failed to download 'cfssl'. Please install cfssl and cfssljson and verify they are in \$PATH." + echo "Hint: export PATH=\$PATH:\$GOPATH/bin; go get -u github.com/cloudflare/cfssl/cmd/..." + exit 1 + fi + popd > /dev/null +} + +# kube::util::ensure_dockerized +# Confirms that the script is being run inside a kube-build image +# +function kube::util::ensure_dockerized { + if [[ -f /kube-build-image ]]; then + return 0 + else + echo "ERROR: This script is designed to be run inside a kube-build container" + exit 1 + fi +} + +# Some useful colors. +if [[ -z "${color_start-}" ]]; then + declare -r color_start="\033[" + declare -r color_red="${color_start}0;31m" + declare -r color_yellow="${color_start}0;33m" + declare -r color_green="${color_start}0;32m" + declare -r color_norm="${color_start}0m" +fi + +# ex: ts=2 sw=2 et filetype=sh diff --git a/hack/lib/version.sh b/hack/lib/version.sh new file mode 100644 index 0000000000..7a1d676e5a --- /dev/null +++ b/hack/lib/version.sh @@ -0,0 +1,154 @@ +#!/bin/bash + +# Copyright 2014 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ----------------------------------------------------------------------------- +# Version management helpers. These functions help to set, save and load the +# following variables: +# +# KUBE_GIT_COMMIT - The git commit id corresponding to this +# source code. +# KUBE_GIT_TREE_STATE - "clean" indicates no changes since the git commit id +# "dirty" indicates source code changes after the git commit id +# KUBE_GIT_VERSION - "vX.Y" used to indicate the last release version. +# KUBE_GIT_MAJOR - The major part of the version +# KUBE_GIT_MINOR - The minor component of the version + +# Grovels through git to set a set of env variables. +# +# If KUBE_GIT_VERSION_FILE, this function will load from that file instead of +# querying git. +kube::version::get_version_vars() { + if [[ -n ${KUBE_GIT_VERSION_FILE-} ]]; then + kube::version::load_version_vars "${KUBE_GIT_VERSION_FILE}" + return + fi + + local git=(git --work-tree "${KUBE_ROOT}") + + if [[ -n ${KUBE_GIT_COMMIT-} ]] || KUBE_GIT_COMMIT=$("${git[@]}" rev-parse "HEAD^{commit}" 2>/dev/null); then + if [[ -z ${KUBE_GIT_TREE_STATE-} ]]; then + # Check if the tree is dirty. default to dirty + if git_status=$("${git[@]}" status --porcelain 2>/dev/null) && [[ -z ${git_status} ]]; then + KUBE_GIT_TREE_STATE="clean" + else + KUBE_GIT_TREE_STATE="dirty" + fi + fi + + # Use git describe to find the version based on annotated tags. + if [[ -n ${KUBE_GIT_VERSION-} ]] || KUBE_GIT_VERSION=$("${git[@]}" describe --tags --abbrev=14 "${KUBE_GIT_COMMIT}^{commit}" 2>/dev/null); then + # This translates the "git describe" to an actual semver.org + # compatible semantic version that looks something like this: + # v1.1.0-alpha.0.6+84c76d1142ea4d + # + # TODO: We continue calling this "git version" because so many + # downstream consumers are expecting it there. + DASHES_IN_VERSION=$(echo "${KUBE_GIT_VERSION}" | sed "s/[^-]//g") + if [[ "${DASHES_IN_VERSION}" == "---" ]] ; then + # We have distance to subversion (v1.1.0-subversion-1-gCommitHash) + KUBE_GIT_VERSION=$(echo "${KUBE_GIT_VERSION}" | sed "s/-\([0-9]\{1,\}\)-g\([0-9a-f]\{14\}\)$/.\1\+\2/") + elif [[ "${DASHES_IN_VERSION}" == "--" ]] ; then + # We have distance to base tag (v1.1.0-1-gCommitHash) + KUBE_GIT_VERSION=$(echo "${KUBE_GIT_VERSION}" | sed "s/-g\([0-9a-f]\{14\}\)$/+\1/") + fi + if [[ "${KUBE_GIT_TREE_STATE}" == "dirty" ]]; then + # git describe --dirty only considers changes to existing files, but + # that is problematic since new untracked .go files affect the build, + # so use our idea of "dirty" from git status instead. + KUBE_GIT_VERSION+="-dirty" + fi + + + # Try to match the "git describe" output to a regex to try to extract + # the "major" and "minor" versions and whether this is the exact tagged + # version or whether the tree is between two tagged versions. + if [[ "${KUBE_GIT_VERSION}" =~ ^v([0-9]+)\.([0-9]+)(\.[0-9]+)?([-].*)?$ ]]; then + KUBE_GIT_MAJOR=${BASH_REMATCH[1]} + KUBE_GIT_MINOR=${BASH_REMATCH[2]} + if [[ -n "${BASH_REMATCH[4]}" ]]; then + KUBE_GIT_MINOR+="+" + fi + fi + fi + fi +} + +# Saves the environment flags to $1 +kube::version::save_version_vars() { + local version_file=${1-} + [[ -n ${version_file} ]] || { + echo "!!! Internal error. No file specified in kube::version::save_version_vars" + return 1 + } + + cat <"${version_file}" +KUBE_GIT_COMMIT='${KUBE_GIT_COMMIT-}' +KUBE_GIT_TREE_STATE='${KUBE_GIT_TREE_STATE-}' +KUBE_GIT_VERSION='${KUBE_GIT_VERSION-}' +KUBE_GIT_MAJOR='${KUBE_GIT_MAJOR-}' +KUBE_GIT_MINOR='${KUBE_GIT_MINOR-}' +EOF +} + +# Loads up the version variables from file $1 +kube::version::load_version_vars() { + local version_file=${1-} + [[ -n ${version_file} ]] || { + echo "!!! Internal error. No file specified in kube::version::load_version_vars" + return 1 + } + + source "${version_file}" +} + +kube::version::ldflag() { + local key=${1} + local val=${2} + + # If you update these, also update the list pkg/version/def.bzl. + echo "-X ${KUBE_GO_PACKAGE}/pkg/version.${key}=${val}" + echo "-X ${KUBE_GO_PACKAGE}/vendor/k8s.io/client-go/pkg/version.${key}=${val}" +} + +# Prints the value that needs to be passed to the -ldflags parameter of go build +# in order to set the Kubernetes based on the git tree status. +# IMPORTANT: if you update any of these, also update the lists in +# pkg/version/def.bzl and hack/print-workspace-status.sh. +kube::version::ldflags() { + kube::version::get_version_vars + + local buildDate= + [[ -z ${SOURCE_DATE_EPOCH-} ]] || buildDate="--date=@${SOURCE_DATE_EPOCH}" + local -a ldflags=($(kube::version::ldflag "buildDate" "$(date ${buildDate} -u +'%Y-%m-%dT%H:%M:%SZ')")) + if [[ -n ${KUBE_GIT_COMMIT-} ]]; then + ldflags+=($(kube::version::ldflag "gitCommit" "${KUBE_GIT_COMMIT}")) + ldflags+=($(kube::version::ldflag "gitTreeState" "${KUBE_GIT_TREE_STATE}")) + fi + + if [[ -n ${KUBE_GIT_VERSION-} ]]; then + ldflags+=($(kube::version::ldflag "gitVersion" "${KUBE_GIT_VERSION}")) + fi + + if [[ -n ${KUBE_GIT_MAJOR-} && -n ${KUBE_GIT_MINOR-} ]]; then + ldflags+=( + $(kube::version::ldflag "gitMajor" "${KUBE_GIT_MAJOR}") + $(kube::version::ldflag "gitMinor" "${KUBE_GIT_MINOR}") + ) + fi + + # The -ldflags parameter takes a single string, so join the output. + echo "${ldflags[*]-}" +} diff --git a/hack/run-e2e.sh b/hack/run-e2e.sh index 5ebd2cdde4..421c502382 100755 --- a/hack/run-e2e.sh +++ b/hack/run-e2e.sh @@ -16,6 +16,7 @@ ${dind_dest} up kubectl create -f config/crds/scheduling_v1alpha1_podgroup.yaml kubectl create -f config/crds/scheduling_v1alpha1_queue.yaml kubectl create -f config/crds/batch_v1alpha1_job.yaml +kubectl create -f config/crds/bus_v1alpha1_command.yaml # start controller nohup ${VK_BIN}/vk-controllers --kubeconfig ${HOME}/.kube/config --logtostderr --v ${LOG_LEVEL} > controller.log 2>&1 & diff --git a/hack/update-gofmt.sh b/hack/update-gofmt.sh new file mode 100644 index 0000000000..992f81d0bc --- /dev/null +++ b/hack/update-gofmt.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +# Copyright 2014 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# GoFmt apparently is changing @ head... + +set -o errexit +set -o nounset +set -o pipefail + +KUBE_ROOT=$(dirname "${BASH_SOURCE}")/.. +source "${KUBE_ROOT}/hack/lib/init.sh" + +kube::golang::verify_go_version + +cd "${KUBE_ROOT}" + +find_files() { + find . -not \( \ + \( \ + -wholename './output' \ + -o -wholename './_output' \ + -o -wholename './_gopath' \ + -o -wholename './release' \ + -o -wholename './target' \ + -o -wholename '*/third_party/*' \ + -o -wholename '*/vendor/*' \ + -o -wholename '*/contrib/*' \ + -o -wholename './staging/src/k8s.io/client-go/*vendor/*' \ + \) -prune \ + \) -name '*.go' +} + +GOFMT="gofmt -s -w" +find_files | xargs $GOFMT