Skip to content

Commit

Permalink
Automatically discover kubectl binaries in KOTS_KUBECTL_BIN_DIR
Browse files Browse the repository at this point in the history
  • Loading branch information
emosbaugh committed Oct 20, 2021
1 parent 4fcbc93 commit 5a0ad9a
Show file tree
Hide file tree
Showing 9 changed files with 355 additions and 176 deletions.
22 changes: 14 additions & 8 deletions deploy/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -10,14 +10,20 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
&& rm -rf /var/lib/apt/lists/* \
&& rm -rf /usr/share/man/man*

# KOTS can be configured to use a specific version of kubectl by setting kubectlVersion in the
# kots.io/v1beta1.Application spec. The github.com/replicatedhq/kots/pkg/binaries package will
# discover all kubectl binaries in the KOTS_KUBECTL_BIN_DIR directory for use by KOTS.

ENV KOTS_KUBECTL_BIN_DIR=/usr/local/bin

# Install Kubectl 1.14
ENV KUBECTL_1_14_VERSION=v1.14.10
ENV KUBECTL_1_14_URL=https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_1_14_VERSION}/bin/linux/amd64/kubectl
ENV KUBECTL_1_14_SHA256SUM=7729c6612bec76badc7926a79b26e0d9b06cc312af46dbb80ea7416d1fce0b36
RUN curl -fsSLO "${KUBECTL_1_14_URL}" \
&& echo "${KUBECTL_1_14_SHA256SUM} kubectl" | sha256sum -c - \
&& chmod +x kubectl \
&& mv kubectl "/usr/local/bin/kubectl-v1.14"
&& mv kubectl "${KOTS_KUBECTL_BIN_DIR}/kubectl-v1.14"

# Install Kubectl 1.16
ENV KUBECTL_1_16_VERSION=v1.16.15
Expand All @@ -26,7 +32,7 @@ ENV KUBECTL_1_16_SHA256SUM=e8913069293156ddf55f243814a22d2384fc18b165efb6200606f
RUN curl -fsSLO "${KUBECTL_1_16_URL}" \
&& echo "${KUBECTL_1_16_SHA256SUM} kubectl" | sha256sum -c - \
&& chmod +x kubectl \
&& mv kubectl "/usr/local/bin/kubectl-v1.16"
&& mv kubectl "${KOTS_KUBECTL_BIN_DIR}/kubectl-v1.16"

# Install Kubectl 1.17
ENV KUBECTL_1_17_VERSION=v1.17.17
Expand All @@ -35,7 +41,7 @@ ENV KUBECTL_1_17_SHA256SUM=8329fac94c66bf7a475b630972a8c0b036bab1f28a5584115e8dd
RUN curl -fsSLO "${KUBECTL_1_17_URL}" \
&& echo "${KUBECTL_1_17_SHA256SUM} kubectl" | sha256sum -c - \
&& chmod +x kubectl \
&& mv kubectl "/usr/local/bin/kubectl-v1.17"
&& mv kubectl "${KOTS_KUBECTL_BIN_DIR}/kubectl-v1.17"

# Install Kubectl 1.18
ENV KUBECTL_1_18_VERSION=v1.18.20
Expand All @@ -44,7 +50,7 @@ ENV KUBECTL_1_18_SHA256SUM=66a9bb8e9843050340844ca6e72e67632b75b9ebb651559c49db2
RUN curl -fsSLO "${KUBECTL_1_18_URL}" \
&& echo "${KUBECTL_1_18_SHA256SUM} kubectl" | sha256sum -c - \
&& chmod +x kubectl \
&& mv kubectl "/usr/local/bin/kubectl-v1.18"
&& mv kubectl "${KOTS_KUBECTL_BIN_DIR}/kubectl-v1.18"

# Install Kubectl 1.19
ENV KUBECTL_1_19_VERSION=v1.19.15
Expand All @@ -53,7 +59,7 @@ ENV KUBECTL_1_19_SHA256SUM=6f2ac7db8cfd59f660abc9891c1bb7da2dabd1cf5e114d836f2ff
RUN curl -fsSLO "${KUBECTL_1_19_URL}" \
&& echo "${KUBECTL_1_19_SHA256SUM} kubectl" | sha256sum -c - \
&& chmod +x kubectl \
&& mv kubectl "/usr/local/bin/kubectl-v1.19"
&& mv kubectl "${KOTS_KUBECTL_BIN_DIR}/kubectl-v1.19"

# Install Kubectl 1.20
ENV KUBECTL_1_20_VERSION=v1.20.11
Expand All @@ -62,7 +68,7 @@ ENV KUBECTL_1_20_SHA256SUM=3a2bf981939df89f807858a481f6f5f2e33a7b9708bd029c8bece
RUN curl -fsSLO "${KUBECTL_1_20_URL}" \
&& echo "${KUBECTL_1_20_SHA256SUM} kubectl" | sha256sum -c - \
&& chmod +x kubectl \
&& mv kubectl "/usr/local/bin/kubectl-v1.20"
&& mv kubectl "${KOTS_KUBECTL_BIN_DIR}/kubectl-v1.20"

# Install Kubectl 1.21
ENV KUBECTL_1_21_VERSION=v1.21.5
Expand All @@ -71,8 +77,8 @@ ENV KUBECTL_1_21_SHA256SUM=060ede75550c63bdc84e14fcc4c8ab3017f7ffc032fc4cac3bf20
RUN curl -fsSLO "${KUBECTL_1_21_URL}" \
&& echo "${KUBECTL_1_21_SHA256SUM} kubectl" | sha256sum -c - \
&& chmod +x kubectl \
&& mv kubectl "/usr/local/bin/kubectl-v1.21" \
&& ln -s "/usr/local/bin/kubectl-v1.21" /usr/local/bin/kubectl
&& mv kubectl "${KOTS_KUBECTL_BIN_DIR}/kubectl-v1.21" \
&& ln -s "${KOTS_KUBECTL_BIN_DIR}/kubectl-v1.21" "${KOTS_KUBECTL_BIN_DIR}/kubectl"

# Install helm v3.4.2
RUN curl -L "https://get.helm.sh/helm-v3.4.2-linux-amd64.tar.gz" -o /tmp/helm.tar.gz && \
Expand Down
23 changes: 15 additions & 8 deletions hack/dev/Dockerfile.skaffold
Original file line number Diff line number Diff line change
Expand Up @@ -26,14 +26,21 @@ RUN apt-get update && apt-get install -y --no-install-recommends curl gnupg2 \
ENV GO111MODULE=on
ENV PATH="/usr/local/bin:$PATH"

# KOTS can be configured to use a specific version of kubectl by setting kubectlVersion in the
# kots.io/v1beta1.Application spec. The github.com/replicatedhq/kots/pkg/binaries package will
# discover all kubectl binaries in the KOTS_KUBECTL_BIN_DIR directory for use by KOTS.

ENV KOTS_KUBECTL_BIN_DIR=/usr/local/bin

# Install Kubectl 1.14
ENV Install Kubectl 1.14
ENV KUBECTL_1_14_VERSION=v1.14.10
ENV KUBECTL_1_14_URL=https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_1_14_VERSION}/bin/linux/amd64/kubectl
ENV KUBECTL_1_14_SHA256SUM=7729c6612bec76badc7926a79b26e0d9b06cc312af46dbb80ea7416d1fce0b36
RUN curl -fsSLO "${KUBECTL_1_14_URL}" \
&& echo "${KUBECTL_1_14_SHA256SUM} kubectl" | sha256sum -c - \
&& chmod +x kubectl \
&& mv kubectl "/usr/local/bin/kubectl-v1.14"
&& mv kubectl "${KOTS_KUBECTL_BIN_DIR}/kubectl-v1.14"

# Install Kubectl 1.16
ENV KUBECTL_1_16_VERSION=v1.16.15
Expand All @@ -42,7 +49,7 @@ ENV KUBECTL_1_16_SHA256SUM=e8913069293156ddf55f243814a22d2384fc18b165efb6200606f
RUN curl -fsSLO "${KUBECTL_1_16_URL}" \
&& echo "${KUBECTL_1_16_SHA256SUM} kubectl" | sha256sum -c - \
&& chmod +x kubectl \
&& mv kubectl "/usr/local/bin/kubectl-v1.16"
&& mv kubectl "${KOTS_KUBECTL_BIN_DIR}/kubectl-v1.16"

# Install Kubectl 1.17
ENV KUBECTL_1_17_VERSION=v1.17.17
Expand All @@ -51,7 +58,7 @@ ENV KUBECTL_1_17_SHA256SUM=8329fac94c66bf7a475b630972a8c0b036bab1f28a5584115e8dd
RUN curl -fsSLO "${KUBECTL_1_17_URL}" \
&& echo "${KUBECTL_1_17_SHA256SUM} kubectl" | sha256sum -c - \
&& chmod +x kubectl \
&& mv kubectl "/usr/local/bin/kubectl-v1.17"
&& mv kubectl "${KOTS_KUBECTL_BIN_DIR}/kubectl-v1.17"

# Install Kubectl 1.18
ENV KUBECTL_1_18_VERSION=v1.18.20
Expand All @@ -60,7 +67,7 @@ ENV KUBECTL_1_18_SHA256SUM=66a9bb8e9843050340844ca6e72e67632b75b9ebb651559c49db2
RUN curl -fsSLO "${KUBECTL_1_18_URL}" \
&& echo "${KUBECTL_1_18_SHA256SUM} kubectl" | sha256sum -c - \
&& chmod +x kubectl \
&& mv kubectl "/usr/local/bin/kubectl-v1.18"
&& mv kubectl "${KOTS_KUBECTL_BIN_DIR}/kubectl-v1.18"

# Install Kubectl 1.19
ENV KUBECTL_1_19_VERSION=v1.19.15
Expand All @@ -69,7 +76,7 @@ ENV KUBECTL_1_19_SHA256SUM=6f2ac7db8cfd59f660abc9891c1bb7da2dabd1cf5e114d836f2ff
RUN curl -fsSLO "${KUBECTL_1_19_URL}" \
&& echo "${KUBECTL_1_19_SHA256SUM} kubectl" | sha256sum -c - \
&& chmod +x kubectl \
&& mv kubectl "/usr/local/bin/kubectl-v1.19"
&& mv kubectl "${KOTS_KUBECTL_BIN_DIR}/kubectl-v1.19"

# Install Kubectl 1.20
ENV KUBECTL_1_20_VERSION=v1.20.11
Expand All @@ -78,7 +85,7 @@ ENV KUBECTL_1_20_SHA256SUM=3a2bf981939df89f807858a481f6f5f2e33a7b9708bd029c8bece
RUN curl -fsSLO "${KUBECTL_1_20_URL}" \
&& echo "${KUBECTL_1_20_SHA256SUM} kubectl" | sha256sum -c - \
&& chmod +x kubectl \
&& mv kubectl "/usr/local/bin/kubectl-v1.20"
&& mv kubectl "${KOTS_KUBECTL_BIN_DIR}/kubectl-v1.20"

# Install Kubectl 1.21
ENV KUBECTL_1_21_VERSION=v1.21.5
Expand All @@ -87,8 +94,8 @@ ENV KUBECTL_1_21_SHA256SUM=060ede75550c63bdc84e14fcc4c8ab3017f7ffc032fc4cac3bf20
RUN curl -fsSLO "${KUBECTL_1_21_URL}" \
&& echo "${KUBECTL_1_21_SHA256SUM} kubectl" | sha256sum -c - \
&& chmod +x kubectl \
&& mv kubectl "/usr/local/bin/kubectl-v1.21" \
&& ln -s "/usr/local/bin/kubectl-v1.21" /usr/local/bin/kubectl
&& mv kubectl "${KOTS_KUBECTL_BIN_DIR}/kubectl-v1.21" \
&& ln -s "${KOTS_KUBECTL_BIN_DIR}/kubectl-v1.21" "${KOTS_KUBECTL_BIN_DIR}/kubectl"

# Install kustomize 3
RUN curl -L "https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2Fv3.5.4/kustomize_v3.5.4_linux_amd64.tar.gz" > /tmp/kustomize.tar.gz && \
Expand Down
6 changes: 6 additions & 0 deletions pkg/apiserver/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (
"github.com/gorilla/mux"
"github.com/pkg/errors"
"github.com/replicatedhq/kots/pkg/automation"
"github.com/replicatedhq/kots/pkg/binaries"
"github.com/replicatedhq/kots/pkg/handlers"
"github.com/replicatedhq/kots/pkg/informers"
"github.com/replicatedhq/kots/pkg/k8sutil"
Expand Down Expand Up @@ -72,6 +73,11 @@ func Start(params *APIServerParams) {

store.GetStore().RunMigrations()

if err := binaries.InitKubectl(); err != nil {
log.Println("error initializing kubectl binaries package")
panic(err)
}

if err := operator.Start(params.AutocreateClusterToken); err != nil {
log.Println("error starting the operator")
panic(err)
Expand Down
174 changes: 174 additions & 0 deletions pkg/binaries/kubectl.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,174 @@
package binaries

import (
"fmt"
"io/fs"
"os"
"os/exec"
"path/filepath"
"regexp"
"sort"
"strconv"
"strings"

"github.com/blang/semver"
"github.com/pkg/errors"
"github.com/replicatedhq/kots/pkg/logger"
"github.com/replicatedhq/kots/pkg/persistence"
)

var (
knownKubectlVersions kubectlVersions
)

// InitKubectl will discover kubectl versions from the environment and populate a list of known
// kubectl versions for later use.
func InitKubectl() (err error) {
knownKubectlVersions, err = discoverKubectlVersions(os.DirFS("/"))
if err != nil {
return errors.Wrap(err, "discover kubectl versions")
}
logger.Infof("Found kubectl binary versions %s", knownKubectlVersions)
return nil
}

// GetKubectlPathForVersion gets the path to a known kubectl version that matches the provided
// semver or semver range.
func GetKubectlPathForVersion(userString string) (string, error) {
for _, knownVersion := range knownKubectlVersions {
if knownVersion.Match(userString) {
return knownVersion.Path, nil
}
}

return "", errors.New("kubectl binary not found")
}

type kubectlVersions []kubectlFuzzyVersion

func (a kubectlVersions) Len() int {
return len(a)
}

func (a kubectlVersions) Less(i, j int) bool {
return a[i].Version.GT(a[j].Version) // sort in descending order
}

func (a kubectlVersions) Swap(i, j int) {
a[i], a[j] = a[j], a[i]
}

type kubectlFuzzyVersion struct {
Version semver.Version
Path string
}

func newKubectlFuzzyVersion(major, minor uint64, path string) kubectlFuzzyVersion {
patch := uint64(0)
// allow for zero value catch all
if major != 0 && minor != 0 {
// I am making an assumption here that likely the package that we are bundling will
// be greater than x.x.0 thus setting patch version to 1 to prevent a semver range
// such as in kots.io docs (>1.16.0 <1.17.0) from matching an incorrect version.
patch = 1
}
return kubectlFuzzyVersion{
Version: semver.Version{
Major: major,
Minor: minor,
Patch: patch,
},
Path: path,
}
}

func (v kubectlFuzzyVersion) String() string {
if v.Version.Equals(semver.Version{}) {
return ""
}
return fmt.Sprintf("v%d.%d", v.Version.Major, v.Version.Minor)
}

func (v kubectlFuzzyVersion) Match(userString string) bool {
if v.Version.Equals(semver.Version{}) {
return true // catch all
}

if userString == "" || userString == "latest" {
return false
}

// ignore error here as this could be a semver range
if exactVer, err := semver.Parse(userString); err == nil {
// fuzzy match major and minor, not patch
return exactVer.Major == v.Version.Major && exactVer.Minor == v.Version.Minor
}

rangeVer, err := semver.ParseRange(userString)
if err != nil {
return false
}

return rangeVer(v.Version)
}

func discoverKubectlVersions(fileSystem fs.FS) ([]kubectlFuzzyVersion, error) {
// in the kots run workflow, binaries exist under {kotsdatadir}/binaries
if persistence.IsSQlite() {
version := newKubectlFuzzyVersion(0, 0, filepath.Join(os.Getenv("KOTS_DATA_DIR"), "binaries/kubectl"))
return []kubectlFuzzyVersion{version}, nil
}

if binDirPath := os.Getenv("KOTS_KUBECTL_BIN_DIR"); binDirPath != "" {
versions, err := discoverKubectlVersionsFromDir(fileSystem, binDirPath)
return versions, errors.Wrap(err, "discover kubectl versions from dir")
}

versions, err := discoverKubectlVersionsFromPath(fileSystem)
return versions, errors.Wrap(err, "discover kubectl versions from path")
}

var kubectlVerRegexp = regexp.MustCompile(`^kubectl(-v(\d+)\.(\d+))?$`)

func discoverKubectlVersionsFromDir(fileSystem fs.FS, dirPath string) ([]kubectlFuzzyVersion, error) {
versions := []kubectlFuzzyVersion{}

entries, err := fs.ReadDir(fileSystem, strings.TrimLeft(dirPath, "/"))
if err != nil {
return versions, errors.Wrap(err, "read dir")
}

for _, entry := range entries {
matches := kubectlVerRegexp.FindStringSubmatch(entry.Name())
if len(matches) > 0 {
var major, minor uint64
// default version has no version suffix
if matches[2] != "" && matches[3] != "" {
var err error
major, err = strconv.ParseUint(matches[2], 10, 64)
if err != nil {
continue
}
minor, err = strconv.ParseUint(matches[3], 10, 64)
if err != nil {
continue
}
}
versions = append(versions, newKubectlFuzzyVersion(major, minor, filepath.Join(dirPath, entry.Name())))
}
}

sort.Sort(kubectlVersions(versions))

return versions, nil
}

func discoverKubectlVersionsFromPath(fileSystem fs.FS) ([]kubectlFuzzyVersion, error) {
// NOTE: exec.LookPath does not yet support io/fs
binPath, err := exec.LookPath("kubectl")
if err != nil {
return nil, errors.Wrap(err, "look path")
}

return []kubectlFuzzyVersion{newKubectlFuzzyVersion(0, 0, binPath)}, nil
}

0 comments on commit 5a0ad9a

Please sign in to comment.