document local minikube. Fixes tilt-dev/tilt#3193

.circleci/Dockerfile

@@ -0,0 +1,38 @@
+FROM debian:buster
+
+RUN apt update
+RUN apt install -y curl ca-certificates liblz4-tool rsync socat
+
+# Install docker
+# Adapted from https://github.com/circleci/circleci-images/blob/staging/shared/images/Dockerfile-basic.template
+# Check https://download.docker.com/linux/static/stable/x86_64/ for latest versions
+ENV DOCKER_VERSION=19.03.5
+RUN set -exu \
+  && DOCKER_URL="https://download.docker.com/linux/static/stable/x86_64/docker-${DOCKER_VERSION}.tgz" \
+  && echo Docker URL: $DOCKER_URL \
+  && curl --silent --show-error --location --fail --retry 3 --output /tmp/docker.tgz "${DOCKER_URL}" \
+  && ls -lha /tmp/docker.tgz \
+  && tar -xz -C /tmp -f /tmp/docker.tgz \
+  && mv /tmp/docker/* /usr/bin \
+  && rm -rf /tmp/docker /tmp/docker.tgz \
+  && which docker \
+  && (docker version || true)
+
+# Install kubectl client
+RUN apt install -y apt-transport-https gnupg \
+  && curl -fsS https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - \
+  && touch /etc/apt/sources.list.d/kubernetes.list \
+  && echo "deb http://apt.kubernetes.io/ kubernetes-xenial main" | tee -a /etc/apt/sources.list.d/kubernetes.list \
+  && apt update && apt install -y kubectl
+
+# install Minikube
+ENV MINIKUBE_VERSION=v1.13.1
+RUN set -exu \
+  && curl -fLo ./minikube-linux-amd64 "https://github.com/kubernetes/minikube/releases/download/${MINIKUBE_VERSION}/minikube-linux-amd64" \
+  && chmod +x ./minikube-linux-amd64 \
+  && mv ./minikube-linux-amd64 /usr/local/bin/minikube
+
+# Install the script for portforwarding connections from the remote cluster
+ADD start-portforward-service.sh /usr/local/bin/start-portforward-service.sh
+ADD portforward.sh /usr/local/bin/portforward.sh
+ADD with-minikube-cluster.sh /usr/local/bin/with-minikube-cluster.sh

.circleci/README.md

@@ -0,0 +1,66 @@
+# Minikube on CircleCI with Remote Docker
+
+This config demonstrates how to run Minikube 
+with a local registry on CircleCI.
+
+## Why is it hard to run Minikube on CircleCI?
+
+CircleCI doesn't run Docker the same way you run Docker on your
+local computer. 
+
+Instead, CircleCI creates a [remote environment](https://circleci.com/docs/2.0/building-docker-images/)
+and runs Docker there.
+
+This means that when you run Minikube, you need to make sure
+that whenever you want to talk to the Minikube cluster,
+you're talking to the remote docker, not to localhost.
+
+## How do I use it?
+
+Create a circleci job that looks like this:
+
+```
+version: 2.1
+jobs:
+  build:
+    docker:
+      - image: tiltdev/circleci-minikube:v1.13.1
+
+    steps:
+      - setup_remote_docker
+      - checkout
+      - run: with-minikube-cluster.sh [YOUR TEST COMMAND]
+```
+
+The `circleci-minikube` image has all the tools you need to set up
+and talk to the cluster. You can build on this image with:
+
+```
+FROM tiltdev/circleci-minikube:v1.13.1
+```
+
+Or copy out the helper scripts:
+
+```
+COPY --from=tiltdev/circleci-minikube:v1.13.1 /usr/local/bin/start-portforward-service.sh /usr/local/bin/
+COPY --from=tiltdev/circleci-minikube:v1.13.1 /usr/local/bin/portforward.sh /usr/local/bin/
+COPY --from=tiltdev/circleci-minikube:v1.13.1 /usr/local/bin/with-minikube-cluster.sh /usr/local/bin/
+```
+
+See [Dockerfile](Dockerfile) for instructions on how to add these tools to
+your own docker image.
+
+## How does it work?
+
+There are five steps
+
+1) Create a port-forward service on the remote machine
+   that knows how to forward connections.
+
+2) Create an image registry.
+
+3) Create a Minikube cluster connected to the image registry.
+
+4) Tell the port-forward service to connect the image registry to localhost.
+
+5) Tell the port-forward service to connect the Minikube cluster to localhost.

.circleci/config.yml

@@ -0,0 +1,15 @@
+version: 2.1
+jobs:
+  build:
+    docker:
+      - image: tiltdev/circleci-minikube:v1.13.1
+
+    steps:
+      - setup_remote_docker
+      - checkout
+      - run: |
+          set -ex
+          export DOCKER_ADDR=$(echo $DOCKER_HOST | sed -s 's,tcp://,,')
+          sudo apt install -y socat
+          sudo socat UNIX-LISTEN:/var/run/docker.sock,user=root,fork "TCP:$DOCKER_ADDR" &
+          with-minikube-cluster.sh test/test.sh

.circleci/portforward.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+#
+# Sets up portforwarding for the specified port.
+#
+# Usage:
+#   start-portforward-service.sh
+#   portforward.sh [port]
+#
+# Adapted from
+# https://github.com/kubernetes-retired/kubeadm-dind-cluster/blob/master/build/portforward.sh
+#
+# Copyright 2020 The Kubernetes Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+port="${1}"
+if [[ "$port" == "" ]]; then
+  echo "Must specify a port"
+  exit 1
+fi
+
+mode=""
+localhost="localhost"
+if [[ "${IP_MODE:-ipv4}" = "ipv6" ]]; then
+  mode="6"
+  localhost="[::1]"
+fi
+
+socat "TCP-LISTEN:${port},reuseaddr,fork" \
+      EXEC:"'docker exec -i portforward socat STDIO TCP${mode}:${localhost}:${port}'" &
+
+# Wait for a successful connection.
+for ((n = 0; n < 20; n++)); do
+  if socat - "TCP${mode}:localhost:${port}" </dev/null; then
+    break
+  fi
+    sleep 0.5
+done

.circleci/start-portforward-service.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+#
+# Starts a portforwarding service on a remote Docker.
+#
+# Usage:
+#   start-portforward-service.sh
+#   portforward.sh [port]
+#
+# Adapted from
+# https://github.com/kubernetes-retired/kubeadm-dind-cluster/blob/master/build/portforward.sh
+#
+# Copyright 2020 The Kubernetes Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+set -o errtrace
+
+docker run -d -it \
+    --name portforward --net=host \
+    --entrypoint /bin/sh \
+    bobrik/socat -c "while true; do sleep 1000; done"

.circleci/with-minikube-cluster.sh

@@ -0,0 +1,90 @@
+#!/bin/bash
+#
+# Starts a Minikube cluster and runs a command against it.
+#
+# Usage:
+#   with-minikube-cluster.sh kubectl cluster-info
+#
+# Adapted from:
+# https://github.com/kubernetes-sigs/kind/commits/master/site/static/examples/kind-with-registry.sh
+#
+# Copyright 2020 The Kubernetes Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -oe errexit
+
+# desired profile name; default is ""
+MINIKUBE_PROFILE_NAME="${MINIKUBE_PROFILE_NAME:-minikube}"
+
+# default registry name and port
+reg_name='minikube-registry'
+reg_port='5000'
+
+echo "> initializing Docker registry"
+
+# create registry container unless it already exists
+running="$(docker inspect -f '{{.State.Running}}' "${reg_name}" 2>/dev/null || true)"
+if [ "${running}" != 'true' ]; then
+  docker run \
+    -d --restart=always -p "${reg_port}:5000" --name "${reg_name}" \
+    registry:2
+fi
+
+# create a cluster
+reg_host="$(docker inspect -f '{{.NetworkSettings.IPAddress}}' "${reg_name}")"
+
+
+sudo socat UNIX-LISTEN:/var/run/docker.sock,user=root,fork "TCP:$DOCKER_HOST" &
+
+echo "> initializing Minikube cluster: ${MINIKUBE_PROFILE_NAME} with registry ${reg_name}"
+
+
+minikube start -p "$MINIKUBE_PROFILE_NAME" --driver=docker --container-runtime=containerd
+
+# patch the container runtime
+# this is the most annoying sed expression i've ever had to write
+minikube ssh sudo sed "\-i" "s,\\\[plugins.cri.registry.mirrors\\\],[plugins.cri.registry.mirrors]\\\n\ \ \ \ \ \ \ \ [plugins.cri.registry.mirrors.\\\"localhost:${reg_port}\\\"]\\\n\ \ \ \ \ \ \ \ \ \ endpoint\ =\ [\\\"http://${reg_host}:${reg_port}\\\"]," /etc/containerd/config.toml
+
+# restart the container runtime
+minikube ssh sudo systemctl restart containerd
+
+echo "> port-forwarding k8s API server"
+/usr/local/bin/start-portforward-service.sh start
+
+APISERVER_PORT=$(kubectl config view -o jsonpath='{.clusters[].cluster.server}' | cut -d: -f 3 -)
+/usr/local/bin/portforward.sh $APISERVER_PORT
+kubectl get nodes # make sure it worked
+
+echo "> port-forwarding local registry"
+/usr/local/bin/portforward.sh $reg_port
+
+echo "> applying local-registry docs"
+
+cat <<EOF | kubectl apply -f -
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: local-registry-hosting
+  namespace: kube-public
+data:
+  localRegistryHosting.v1: |
+    host: "localhost:${reg_port}"
+    help: "https://github.com/tilt-dev/minikube-local"
+EOF
+
+echo "> waiting for kubernetes node(s) become ready"
+kubectl wait --for=condition=ready node --all --timeout=60s
+
+echo "> with-minikube-cluster.sh setup complete! Running user script: $@"
+exec "$@"