This repository has been archived by the owner on Apr 3, 2023. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 6
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
document local minikube. Fixes tilt-dev/tilt#3193
- Loading branch information
Showing
6 changed files
with
287 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,38 @@ | ||
FROM debian:buster | ||
|
||
RUN apt update | ||
RUN apt install -y curl ca-certificates liblz4-tool rsync socat | ||
|
||
# Install docker | ||
# Adapted from https://github.com/circleci/circleci-images/blob/staging/shared/images/Dockerfile-basic.template | ||
# Check https://download.docker.com/linux/static/stable/x86_64/ for latest versions | ||
ENV DOCKER_VERSION=19.03.5 | ||
RUN set -exu \ | ||
&& DOCKER_URL="https://download.docker.com/linux/static/stable/x86_64/docker-${DOCKER_VERSION}.tgz" \ | ||
&& echo Docker URL: $DOCKER_URL \ | ||
&& curl --silent --show-error --location --fail --retry 3 --output /tmp/docker.tgz "${DOCKER_URL}" \ | ||
&& ls -lha /tmp/docker.tgz \ | ||
&& tar -xz -C /tmp -f /tmp/docker.tgz \ | ||
&& mv /tmp/docker/* /usr/bin \ | ||
&& rm -rf /tmp/docker /tmp/docker.tgz \ | ||
&& which docker \ | ||
&& (docker version || true) | ||
|
||
# Install kubectl client | ||
RUN apt install -y apt-transport-https gnupg \ | ||
&& curl -fsS https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - \ | ||
&& touch /etc/apt/sources.list.d/kubernetes.list \ | ||
&& echo "deb http://apt.kubernetes.io/ kubernetes-xenial main" | tee -a /etc/apt/sources.list.d/kubernetes.list \ | ||
&& apt update && apt install -y kubectl | ||
|
||
# install Minikube | ||
ENV MINIKUBE_VERSION=v1.13.1 | ||
RUN set -exu \ | ||
&& curl -fLo ./minikube-linux-amd64 "https://github.com/kubernetes/minikube/releases/download/${MINIKUBE_VERSION}/minikube-linux-amd64" \ | ||
&& chmod +x ./minikube-linux-amd64 \ | ||
&& mv ./minikube-linux-amd64 /usr/local/bin/minikube | ||
|
||
# Install the script for portforwarding connections from the remote cluster | ||
ADD start-portforward-service.sh /usr/local/bin/start-portforward-service.sh | ||
ADD portforward.sh /usr/local/bin/portforward.sh | ||
ADD with-minikube-cluster.sh /usr/local/bin/with-minikube-cluster.sh |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,66 @@ | ||
# Minikube on CircleCI with Remote Docker | ||
|
||
This config demonstrates how to run Minikube | ||
with a local registry on CircleCI. | ||
|
||
## Why is it hard to run Minikube on CircleCI? | ||
|
||
CircleCI doesn't run Docker the same way you run Docker on your | ||
local computer. | ||
|
||
Instead, CircleCI creates a [remote environment](https://circleci.com/docs/2.0/building-docker-images/) | ||
and runs Docker there. | ||
|
||
This means that when you run Minikube, you need to make sure | ||
that whenever you want to talk to the Minikube cluster, | ||
you're talking to the remote docker, not to localhost. | ||
|
||
## How do I use it? | ||
|
||
Create a circleci job that looks like this: | ||
|
||
``` | ||
version: 2.1 | ||
jobs: | ||
build: | ||
docker: | ||
- image: tiltdev/circleci-minikube:v1.13.1 | ||
steps: | ||
- setup_remote_docker | ||
- checkout | ||
- run: with-minikube-cluster.sh [YOUR TEST COMMAND] | ||
``` | ||
|
||
The `circleci-minikube` image has all the tools you need to set up | ||
and talk to the cluster. You can build on this image with: | ||
|
||
``` | ||
FROM tiltdev/circleci-minikube:v1.13.1 | ||
``` | ||
|
||
Or copy out the helper scripts: | ||
|
||
``` | ||
COPY --from=tiltdev/circleci-minikube:v1.13.1 /usr/local/bin/start-portforward-service.sh /usr/local/bin/ | ||
COPY --from=tiltdev/circleci-minikube:v1.13.1 /usr/local/bin/portforward.sh /usr/local/bin/ | ||
COPY --from=tiltdev/circleci-minikube:v1.13.1 /usr/local/bin/with-minikube-cluster.sh /usr/local/bin/ | ||
``` | ||
|
||
See [Dockerfile](Dockerfile) for instructions on how to add these tools to | ||
your own docker image. | ||
|
||
## How does it work? | ||
|
||
There are five steps | ||
|
||
1) Create a port-forward service on the remote machine | ||
that knows how to forward connections. | ||
|
||
2) Create an image registry. | ||
|
||
3) Create a Minikube cluster connected to the image registry. | ||
|
||
4) Tell the port-forward service to connect the image registry to localhost. | ||
|
||
5) Tell the port-forward service to connect the Minikube cluster to localhost. |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,11 @@ | ||
version: 2.1 | ||
jobs: | ||
build: | ||
docker: | ||
- image: tiltdev/circleci-minikube:v1.13.1 | ||
|
||
steps: | ||
- setup_remote_docker | ||
- checkout | ||
- run: DOCKER_ADDR=$(echo $DOCKER_HOST | sed -s 's,tcp://,,') sudo socat UNIX-LISTEN:/var/run/docker.sock,user=root,fork "TCP:$DOCKER_ADDR" & \ | ||
./with-minikube-cluster.sh test/test.sh |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,48 @@ | ||
#!/bin/bash | ||
# | ||
# Sets up portforwarding for the specified port. | ||
# | ||
# Usage: | ||
# start-portforward-service.sh | ||
# portforward.sh [port] | ||
# | ||
# Adapted from | ||
# https://github.com/kubernetes-retired/kubeadm-dind-cluster/blob/master/build/portforward.sh | ||
# | ||
# Copyright 2020 The Kubernetes Project | ||
# | ||
# Licensed under the Apache License, Version 2.0 (the "License"); | ||
# you may not use this file except in compliance with the License. | ||
# You may obtain a copy of the License at | ||
# | ||
# http://www.apache.org/licenses/LICENSE-2.0 | ||
# | ||
# Unless required by applicable law or agreed to in writing, software | ||
# distributed under the License is distributed on an "AS IS" BASIS, | ||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
# See the License for the specific language governing permissions and | ||
# limitations under the License. | ||
|
||
port="${1}" | ||
if [[ "$port" == "" ]]; then | ||
echo "Must specify a port" | ||
exit 1 | ||
fi | ||
|
||
mode="" | ||
localhost="localhost" | ||
if [[ "${IP_MODE:-ipv4}" = "ipv6" ]]; then | ||
mode="6" | ||
localhost="[::1]" | ||
fi | ||
|
||
socat "TCP-LISTEN:${port},reuseaddr,fork" \ | ||
EXEC:"'docker exec -i portforward socat STDIO TCP${mode}:${localhost}:${port}'" & | ||
|
||
# Wait for a successful connection. | ||
for ((n = 0; n < 20; n++)); do | ||
if socat - "TCP${mode}:localhost:${port}" </dev/null; then | ||
break | ||
fi | ||
sleep 0.5 | ||
done |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,34 @@ | ||
#!/bin/bash | ||
# | ||
# Starts a portforwarding service on a remote Docker. | ||
# | ||
# Usage: | ||
# start-portforward-service.sh | ||
# portforward.sh [port] | ||
# | ||
# Adapted from | ||
# https://github.com/kubernetes-retired/kubeadm-dind-cluster/blob/master/build/portforward.sh | ||
# | ||
# Copyright 2020 The Kubernetes Project | ||
# | ||
# Licensed under the Apache License, Version 2.0 (the "License"); | ||
# you may not use this file except in compliance with the License. | ||
# You may obtain a copy of the License at | ||
# | ||
# http://www.apache.org/licenses/LICENSE-2.0 | ||
# | ||
# Unless required by applicable law or agreed to in writing, software | ||
# distributed under the License is distributed on an "AS IS" BASIS, | ||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
# See the License for the specific language governing permissions and | ||
# limitations under the License. | ||
|
||
set -o errexit | ||
set -o nounset | ||
set -o pipefail | ||
set -o errtrace | ||
|
||
docker run -d -it \ | ||
--name portforward --net=host \ | ||
--entrypoint /bin/sh \ | ||
bobrik/socat -c "while true; do sleep 1000; done" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,90 @@ | ||
#!/bin/bash | ||
# | ||
# Starts a Minikube cluster and runs a command against it. | ||
# | ||
# Usage: | ||
# with-minikube-cluster.sh kubectl cluster-info | ||
# | ||
# Adapted from: | ||
# https://github.com/kubernetes-sigs/kind/commits/master/site/static/examples/kind-with-registry.sh | ||
# | ||
# Copyright 2020 The Kubernetes Project | ||
# | ||
# Licensed under the Apache License, Version 2.0 (the "License"); | ||
# you may not use this file except in compliance with the License. | ||
# You may obtain a copy of the License at | ||
# | ||
# http://www.apache.org/licenses/LICENSE-2.0 | ||
# | ||
# Unless required by applicable law or agreed to in writing, software | ||
# distributed under the License is distributed on an "AS IS" BASIS, | ||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
# See the License for the specific language governing permissions and | ||
# limitations under the License. | ||
|
||
set -oe errexit | ||
|
||
# desired profile name; default is "" | ||
MINIKUBE_PROFILE_NAME="${MINIKUBE_PROFILE_NAME:-minikube}" | ||
|
||
# default registry name and port | ||
reg_name='minikube-registry' | ||
reg_port='5000' | ||
|
||
echo "> initializing Docker registry" | ||
|
||
# create registry container unless it already exists | ||
running="$(docker inspect -f '{{.State.Running}}' "${reg_name}" 2>/dev/null || true)" | ||
if [ "${running}" != 'true' ]; then | ||
docker run \ | ||
-d --restart=always -p "${reg_port}:5000" --name "${reg_name}" \ | ||
registry:2 | ||
fi | ||
|
||
# create a cluster | ||
reg_host="$(docker inspect -f '{{.NetworkSettings.IPAddress}}' "${reg_name}")" | ||
|
||
|
||
sudo socat UNIX-LISTEN:/var/run/docker.sock,user=root,fork "TCP:$DOCKER_HOST" & | ||
|
||
echo "> initializing Minikube cluster: ${MINIKUBE_PROFILE_NAME} with registry ${reg_name}" | ||
|
||
|
||
minikube start -p "$MINIKUBE_PROFILE_NAME" --driver=docker --container-runtime=containerd | ||
|
||
# patch the container runtime | ||
# this is the most annoying sed expression i've ever had to write | ||
minikube ssh sudo sed "\-i" "s,\\\[plugins.cri.registry.mirrors\\\],[plugins.cri.registry.mirrors]\\\n\ \ \ \ \ \ \ \ [plugins.cri.registry.mirrors.\\\"localhost:${reg_port}\\\"]\\\n\ \ \ \ \ \ \ \ \ \ endpoint\ =\ [\\\"http://${reg_host}:${reg_port}\\\"]," /etc/containerd/config.toml | ||
|
||
# restart the container runtime | ||
minikube ssh sudo systemctl restart containerd | ||
|
||
echo "> port-forwarding k8s API server" | ||
/usr/local/bin/start-portforward-service.sh start | ||
|
||
APISERVER_PORT=$(kubectl config view -o jsonpath='{.clusters[].cluster.server}' | cut -d: -f 3 -) | ||
/usr/local/bin/portforward.sh $APISERVER_PORT | ||
kubectl get nodes # make sure it worked | ||
|
||
echo "> port-forwarding local registry" | ||
/usr/local/bin/portforward.sh $reg_port | ||
|
||
echo "> applying local-registry docs" | ||
|
||
cat <<EOF | kubectl apply -f - | ||
apiVersion: v1 | ||
kind: ConfigMap | ||
metadata: | ||
name: local-registry-hosting | ||
namespace: kube-public | ||
data: | ||
localRegistryHosting.v1: | | ||
host: "localhost:${reg_port}" | ||
help: "https://github.com/tilt-dev/minikube-local" | ||
EOF | ||
|
||
echo "> waiting for kubernetes node(s) become ready" | ||
kubectl wait --for=condition=ready node --all --timeout=60s | ||
|
||
echo "> with-minikube-cluster.sh setup complete! Running user script: $@" | ||
exec "$@" |