From f637541df50d13c64549d9262e52efaf7409c291 Mon Sep 17 00:00:00 2001 From: Nick Santos Date: Mon, 21 Sep 2020 18:03:04 -0400 Subject: [PATCH] document local minikube. Fixes https://github.com/tilt-dev/tilt/issues/3193 --- .circleci/Dockerfile | 38 ++++++++++++ .circleci/README.md | 66 ++++++++++++++++++++ .circleci/config.yml | 10 +++ .circleci/portforward.sh | 48 +++++++++++++++ .circleci/start-portforward-service.sh | 34 +++++++++++ .circleci/with-minikube-cluster.sh | 85 ++++++++++++++++++++++++++ README.md | 69 +++++++++++++++++++++ minikube-with-registry.sh | 60 ++++++++++++++++++ test/pod.yaml | 13 ++++ test/test.sh | 14 +++++ 10 files changed, 437 insertions(+) create mode 100644 .circleci/Dockerfile create mode 100644 .circleci/README.md create mode 100644 .circleci/config.yml create mode 100755 .circleci/portforward.sh create mode 100755 .circleci/start-portforward-service.sh create mode 100755 .circleci/with-minikube-cluster.sh create mode 100755 minikube-with-registry.sh create mode 100644 test/pod.yaml create mode 100755 test/test.sh diff --git a/.circleci/Dockerfile b/.circleci/Dockerfile new file mode 100644 index 0000000..6aefbae --- /dev/null +++ b/.circleci/Dockerfile @@ -0,0 +1,38 @@ +FROM debian:buster + +RUN apt update +RUN apt install -y curl ca-certificates liblz4-tool rsync socat + +# Install docker +# Adapted from https://github.com/circleci/circleci-images/blob/staging/shared/images/Dockerfile-basic.template +# Check https://download.docker.com/linux/static/stable/x86_64/ for latest versions +ENV DOCKER_VERSION=19.03.5 +RUN set -exu \ + && DOCKER_URL="https://download.docker.com/linux/static/stable/x86_64/docker-${DOCKER_VERSION}.tgz" \ + && echo Docker URL: $DOCKER_URL \ + && curl --silent --show-error --location --fail --retry 3 --output /tmp/docker.tgz "${DOCKER_URL}" \ + && ls -lha /tmp/docker.tgz \ + && tar -xz -C /tmp -f /tmp/docker.tgz \ + && mv /tmp/docker/* /usr/bin \ + && rm -rf /tmp/docker /tmp/docker.tgz \ + && which docker \ + && (docker version || true) + +# Install kubectl client +RUN apt install -y apt-transport-https gnupg \ + && curl -fsS https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - \ + && touch /etc/apt/sources.list.d/kubernetes.list \ + && echo "deb http://apt.kubernetes.io/ kubernetes-xenial main" | tee -a /etc/apt/sources.list.d/kubernetes.list \ + && apt update && apt install -y kubectl + +# install Minikube +ENV MINIKUBE_VERSION=v1.13.1 +RUN set -exu \ + && curl -fLo ./minikube-linux-amd64 "https://github.com/kubernetes/minikube/releases/download/${MINIKUBE_VERSION}/minikube-linux-amd64" \ + && chmod +x ./minikube-linux-amd64 \ + && mv ./minikube-linux-amd64 /usr/local/bin/minikube + +# Install the script for portforwarding connections from the remote cluster +ADD start-portforward-service.sh /usr/local/bin/start-portforward-service.sh +ADD portforward.sh /usr/local/bin/portforward.sh +ADD with-minikube-cluster.sh /usr/local/bin/with-minikube-cluster.sh diff --git a/.circleci/README.md b/.circleci/README.md new file mode 100644 index 0000000..4d192a3 --- /dev/null +++ b/.circleci/README.md @@ -0,0 +1,66 @@ +# Minikube on CircleCI with Remote Docker + +This config demonstrates how to run Minikube +with a local registry on CircleCI. + +## Why is it hard to run Minikube on CircleCI? + +CircleCI doesn't run Docker the same way you run Docker on your +local computer. + +Instead, CircleCI creates a [remote environment](https://circleci.com/docs/2.0/building-docker-images/) +and runs Docker there. + +This means that when you run Minikube, you need to make sure +that whenever you want to talk to the Minikube cluster, +you're talking to the remote docker, not to localhost. + +## How do I use it? + +Create a circleci job that looks like this: + +``` +version: 2.1 +jobs: + build: + docker: + - image: tiltdev/circleci-minikube:v1.13.1 + + steps: + - setup_remote_docker + - checkout + - run: with-minikube-cluster.sh [YOUR TEST COMMAND] +``` + +The `circleci-minikube` image has all the tools you need to set up +and talk to the cluster. You can build on this image with: + +``` +FROM tiltdev/circleci-minikube:v1.13.1 +``` + +Or copy out the helper scripts: + +``` +COPY --from=tiltdev/circleci-minikube:v1.13.1 /usr/local/bin/start-portforward-service.sh /usr/local/bin/ +COPY --from=tiltdev/circleci-minikube:v1.13.1 /usr/local/bin/portforward.sh /usr/local/bin/ +COPY --from=tiltdev/circleci-minikube:v1.13.1 /usr/local/bin/with-minikube-cluster.sh /usr/local/bin/ +``` + +See [Dockerfile](Dockerfile) for instructions on how to add these tools to +your own docker image. + +## How does it work? + +There are five steps + +1) Create a port-forward service on the remote machine + that knows how to forward connections. + +2) Create an image registry. + +3) Create a Minikube cluster connected to the image registry. + +4) Tell the port-forward service to connect the image registry to localhost. + +5) Tell the port-forward service to connect the Minikube cluster to localhost. diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000..93ff6e1 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,10 @@ +version: 2.1 +jobs: + build: + docker: + - image: tiltdev/circleci-minikube:v1.13.1 + + steps: + - setup_remote_docker + - checkout + - run: with-minikube-cluster.sh test/test.sh \ No newline at end of file diff --git a/.circleci/portforward.sh b/.circleci/portforward.sh new file mode 100755 index 0000000..00eda43 --- /dev/null +++ b/.circleci/portforward.sh @@ -0,0 +1,48 @@ +#!/bin/bash +# +# Sets up portforwarding for the specified port. +# +# Usage: +# start-portforward-service.sh +# portforward.sh [port] +# +# Adapted from +# https://github.com/kubernetes-retired/kubeadm-dind-cluster/blob/master/build/portforward.sh +# +# Copyright 2020 The Kubernetes Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +port="${1}" +if [[ "$port" == "" ]]; then + echo "Must specify a port" + exit 1 +fi + +mode="" +localhost="localhost" +if [[ "${IP_MODE:-ipv4}" = "ipv6" ]]; then + mode="6" + localhost="[::1]" +fi + +socat "TCP-LISTEN:${port},reuseaddr,fork" \ + EXEC:"'docker exec -i portforward socat STDIO TCP${mode}:${localhost}:${port}'" & + +# Wait for a successful connection. +for ((n = 0; n < 20; n++)); do + if socat - "TCP${mode}:localhost:${port}" initializing Docker registry" + +# create registry container unless it already exists +running="$(docker inspect -f '{{.State.Running}}' "${reg_name}" 2>/dev/null || true)" +if [ "${running}" != 'true' ]; then + docker run \ + -d --restart=always -p "${reg_port}:5000" --name "${reg_name}" \ + registry:2 +fi + +# create a cluster +reg_host="$(docker inspect -f '{{.NetworkSettings.IPAddress}}' "${reg_name}")" + +echo "> initializing Minikube cluster: ${MINIKUBE_PROFILE_NAME} with registry ${reg_name}" +minikube start -p "$MINIKUBE_PROFILE_NAME" --driver=docker --container-runtime=containerd + +# patch the container runtime +# this is the most annoying sed expression i've ever had to write +minikube ssh sudo sed "\-i" "s,\\\[plugins.cri.registry.mirrors\\\],[plugins.cri.registry.mirrors]\\\n\ \ \ \ \ \ \ \ [plugins.cri.registry.mirrors.\\\"localhost:${reg_port}\\\"]\\\n\ \ \ \ \ \ \ \ \ \ endpoint\ =\ [\\\"http://${reg_host}:${reg_port}\\\"]," /etc/containerd/config.toml + +# restart the container runtime +minikube ssh sudo systemctl restart containerd + +echo "> port-forwarding k8s API server" +/usr/local/bin/start-portforward-service.sh start + +APISERVER_PORT=$(kubectl config view -o jsonpath='{.clusters[].cluster.server}' | cut -d: -f 3 -) +/usr/local/bin/portforward.sh $APISERVER_PORT +kubectl get nodes # make sure it worked + +echo "> port-forwarding local registry" +/usr/local/bin/portforward.sh $reg_port + +echo "> applying local-registry docs" + +cat < waiting for kubernetes node(s) become ready" +kubectl wait --for=condition=ready node --all --timeout=60s + +echo "> with-minikube-cluster.sh setup complete! Running user script: $@" +exec "$@" diff --git a/README.md b/README.md index 7e7424c..e1111a8 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,71 @@ # minikube-local + The best way to set minikube up for local development + +[![Build Status](https://circleci.com/gh/tilt-dev/minikube-local/tree/master.svg?style=shield)](https://circleci.com/gh/tilt-dev/minikube-local) + +When using Tilt with a [Minikube](https://minikube.sigs.k8s.io/docs/) cluster, +we recommend using a local registry for faster image pushing and pulling. + +This repo documents the best way to set it up. + +## Why use Minikube with a local registry? + +When developing locally, you want to push images to the cluster as fast as possible. + +Pushing to a local image registry skips a lot of overhead: + +- Unlike with a remote registry, the image stays local to your machine, with no network traffic + +- Unlike with loading into the container runtime, docker will skip pushing any layers that already exist in the registry + +- Unlike with in-cluster build daemons, there's no additional auth configuration in-cluster. + +- Unlike with an in-cluster registry, you can reset the cluster without deleting the image cache. + +This makes it a great solution for iterative local development. But setting it up is awkward and fiddly. This script makes it easy. + +## How to Try It + +1) Install [Minikube](https://minikube.sigs.k8s.io/docs/) + +2) Copy the [minikube-with-registry.sh](minikube-with-registry.sh) somewhere on your path. + +3) Create a cluster with `minikube-with-registry.sh`. Currently it creates the registry at port 5000. + +``` +minikube-with-registry.sh +``` + +4) Try pushing an image. + +``` +docker tag alpine localhost:5000/alpine +docker push localhost:5000/alpine +``` + +You can now use the image name `localhost:5000/alpine` in any resources you deploy to the cluster. + +[Tilt](https://tilt.dev) will automatically detect the local registry created by this script. + +## How to Use it in CI + +We also have instructions for setting Minikube up with a local registry in + +- [.circleci](.circleci) + +## Thanks to + +High five to [MicroK8s](https://github.com/ubuntu/microk8s) for the initial local registry feature +that inspired a lot of this work. + +The Kind team ran with this, writing up documentation and hooks for how to [set up a local registry](https://kind.sigs.k8s.io/docs/user/local-registry/) with Kind. + +This repo adapts the Kind team's approach and applies the local registry configmap, so that tools +like Tilt can discover the local-registry. This protocol is a [Kubernetes Enhancement Proposal](https://github.com/kubernetes/enhancements/issues/1755). + +## License + +Copyright 2020 Windmill Engineering + +Licensed under [the Apache License, Version 2.0](LICENSE) diff --git a/minikube-with-registry.sh b/minikube-with-registry.sh new file mode 100755 index 0000000..35a1762 --- /dev/null +++ b/minikube-with-registry.sh @@ -0,0 +1,60 @@ +#!/bin/sh +# +# Adapted from: +# https://github.com/kubernetes-sigs/kind/commits/master/site/static/examples/kind-with-registry.sh +# +# Copyright 2020 The Kubernetes Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit + +# desired profile name; default is "" +MINIKUBE_PROFILE_NAME="${MINIKUBE_PROFILE_NAME:-minikube}" + +reg_name='minikube-registry' +reg_port='5000' + +# create registry container unless it already exists +running="$(docker inspect -f '{{.State.Running}}' "${reg_name}" 2>/dev/null || true)" +if [ "${running}" != 'true' ]; then + docker run \ + -d --restart=always -p "${reg_port}:5000" --name "${reg_name}" \ + registry:2 +fi + +reg_host="$(docker inspect -f '{{.NetworkSettings.IPAddress}}' "${reg_name}")" +echo "Registry Host: ${reg_host}" + +# create a cluster +minikube start -p "$MINIKUBE_PROFILE_NAME" --driver=docker --container-runtime=containerd + +# patch the container runtime +# this is the most annoying sed expression i've ever had to write +minikube ssh sudo sed "\-i" "s,\\\[plugins.cri.registry.mirrors\\\],[plugins.cri.registry.mirrors]\\\n\ \ \ \ \ \ \ \ [plugins.cri.registry.mirrors.\\\"localhost:${reg_port}\\\"]\\\n\ \ \ \ \ \ \ \ \ \ endpoint\ =\ [\\\"http://${reg_host}:${reg_port}\\\"]," /etc/containerd/config.toml + +# restart the container runtime +minikube ssh sudo systemctl restart containerd + +# document the registry +cat <