diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index c60fd63..3b9200f 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,5 +1,11 @@ FROM ubuntu:bionic +ARG TARGETARCH + +RUN if [ "${TARGETARCH}" != "amd64" ] && [ "${TARGETARCH}" != "arm64" ]; then \ + echo "${TARGETARCH} is not a supported architecture. Please use linux/amd64 or linux/arm64"; \ + exit 1; fi + RUN apt-get update && \ export DEBIAN_FRONTEND=noninteractive && \ apt-get install -y \ @@ -37,8 +43,8 @@ ENV DEBIAN_FRONTEND=noninteractive RUN pip3 install --upgrade pip # Install AWS CLI v2 -RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \ - unzip awscliv2.zip && \ +RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-$(uname -m).zip" -o "awscliv2.zip" && \ + unzip -q awscliv2.zip && \ ./aws/install && \ rm -rf awscliv2.zip aws @@ -50,28 +56,36 @@ RUN curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-c RUN chmod +x /usr/local/bin/docker-compose # Install Go. Keep in sync with other Dockerfiles. -ENV GOLANG_VERSION 1.17.1 -ENV GOLANG_DOWNLOAD_SHA256 dab7d9c34361dc21ec237d584590d72500652e7c909bf082758fb63064fca0ef -RUN curl -fsSL "https://dl.google.com/go/go${GOLANG_VERSION}.linux-amd64.tar.gz" -o golang.tar.gz \ +ENV GOLANG_VERSION 1.18 +RUN GOLANG_DOWNLOAD_SHA256=$(if [ $TARGETARCH = "arm64" ]; \ + then echo "7ac7b396a691e588c5fb57687759e6c4db84a2a3bbebb0765f4b38e5b1c5b00e"; \ + else echo "e85278e98f57cdb150fe8409e6e5df5343ecb13cebf03a5d5ff12bd55a80264f"; fi) \ + && curl -fsSL "https://dl.google.com/go/go${GOLANG_VERSION}.linux-${TARGETARCH}.tar.gz" -o golang.tar.gz \ && echo "$GOLANG_DOWNLOAD_SHA256 golang.tar.gz" | sha256sum -c - \ && tar -C /usr/local -xzf golang.tar.gz \ && rm golang.tar.gz # Install Go tools -RUN PATH="$PATH:/usr/local/go/bin" GOPATH=/usr/local/gotools go get -u \ - golang.org/x/tools/... \ - github.com/ramya-rao-a/go-outline \ - github.com/fatih/gomodifytags \ - github.com/josharian/impl \ - github.com/haya14busa/goplay/cmd/goplay \ - honnef.co/go/tools/cmd/staticcheck@latest \ - github.com/uudashr/gopkgs/v2/cmd/gopkgs +RUN PATH="$PATH:/usr/local/go/bin" GOPATH=/usr/local/gotools go install \ + golang.org/x/tools/...@latest +RUN PATH="$PATH:/usr/local/go/bin" GOPATH=/usr/local/gotools go install \ + github.com/ramya-rao-a/go-outline@latest +RUN PATH="$PATH:/usr/local/go/bin" GOPATH=/usr/local/gotools go install \ + github.com/fatih/gomodifytags@latest +RUN PATH="$PATH:/usr/local/go/bin" GOPATH=/usr/local/gotools go install \ + github.com/josharian/impl@latest +RUN PATH="$PATH:/usr/local/go/bin" GOPATH=/usr/local/gotools go install \ + github.com/haya14busa/goplay/cmd/goplay@latest +RUN PATH="$PATH:/usr/local/go/bin" GOPATH=/usr/local/gotools go install \ + honnef.co/go/tools/cmd/staticcheck@latest +RUN PATH="$PATH:/usr/local/go/bin" GOPATH=/usr/local/gotools go install \ + github.com/uudashr/gopkgs/v2/cmd/gopkgs@latest RUN PATH="$PATH:/usr/local/go/bin" go install github.com/go-delve/delve/cmd/dlv@latest -RUN GO111MODULE=on PATH="$PATH:/usr/local/go/bin" GOPATH=/usr/local/gotools go get -v \ +RUN GO111MODULE=on PATH="$PATH:/usr/local/go/bin" GOPATH=/usr/local/gotools go install \ golang.org/x/tools/gopls@latest RUN PATH="$PATH:/usr/local/go/bin" GOPATH=/usr/local/gotools go install \ github.com/cweill/gotests/... -RUN GO111MODULE=on PATH="$PATH:/usr/local/go/bin" GOPATH=/usr/local/gotools GOBIN=/tmp/ go get -v \ +RUN PATH="$PATH:/usr/local/go/bin" GOPATH=/usr/local/gotools GOBIN=/tmp/ go install \ github.com/go-delve/delve/cmd/dlv@master && \ mv /tmp/dlv $GOPATH/bin/dlv-dap ENV PATH="$PATH:/usr/local/gotools/bin:/usr/local/go/bin" diff --git a/.github/actions/test/Dockerfile b/.github/actions/test/Dockerfile index 364c0c5..443a702 100644 --- a/.github/actions/test/Dockerfile +++ b/.github/actions/test/Dockerfile @@ -39,7 +39,7 @@ RUN groupadd -g $JENKINS_GID jenkins && \ echo "Defaults secure_path='/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin:/usr/local/pyenv/bin'" >> /etc/sudoers # AWS CLI -RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \ +RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-$(uname -m).zip" -o "awscliv2.zip" && \ unzip -q awscliv2.zip && \ ./aws/install && \ rm awscliv2.zip diff --git a/README.md b/README.md index a86c524..265b225 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,7 @@ We use `just` to run project specific commands. See the examples. You will also need [Docker](https://docs.docker.com/get-docker/) if you wish to -run the integration tests. +run the integration and end-to-end tests. ## Testing @@ -92,15 +92,63 @@ MODULE=pkg/rsnotify just test -v github.com/rstudio/platform-lib/pkg/rsnotify/lo # Run the PgxNotifySuite suite tests with docker-compose MODULE=pkg/rsnotify just test-integration -v github.com/rstudio/platform-lib/pkg/rsnotify/pgxlistener -check.f=PgxNotifySuite +``` -# Run the end-to-end tests +### Testing with Docker + +End-to-end and integration tests are run with Docker. + +By default, Docker will use your host machine to infer which [platform to use +when building images](https://docs.docker.com/engine/reference/builder/#from). +You can specify another architecture with the `--platform` flag. Supported +options are `linux/amd64` and `linux/arm64`. + +If your machine is running on the same platform as the image you plan to use for +testing, you can simply build the code natively. If not you will need to build a +separate Docker image to cross-compile your code. + +>Note: On Apple silicon, Docker will build linux/arm64 images by default. +> However, you will need to cross-compile your code for the end-to-end tests +> because binaries built on Darwin are not compatible with the Linux environment +> in the Docker images and vice-versa. + +Examples: + +```bash +# Build the docker image used for end-to-end testing +just build-e2e-env + +# Build the end-to-end testing image for ARM +just build-e2e-env --platform=linux/arm64 + +# Build the build-env image to cross-compile for ARM +just build-build-env --platform=linux/arm64 + +# Build the code natively just build + +# Build the code in Docker +just build-docker + +# Run the end-to-end tests just test-e2e # Open an interactive container for end-to-end testing just start-e2e-env just test exit + +# Run the integration tests (uses the build-env image) +just test-integration +``` + +When re-building images for a different platform, Docker may ignore the platform +flag and use cached layers from the previous build. To resolve this issue, +remove the images and clear the build cache. You can do this manually for +a single image or use the `just` target to clean up both images at once: + +```bash +just clean-docker ``` ## Licenses diff --git a/docker/bionic/Dockerfile b/docker/bionic/Dockerfile index 4652ff7..4123cad 100644 --- a/docker/bionic/Dockerfile +++ b/docker/bionic/Dockerfile @@ -1,5 +1,11 @@ FROM ubuntu:bionic +ARG TARGETARCH + +RUN if [ "${TARGETARCH}" != "amd64" ] && [ "${TARGETARCH}" != "arm64" ]; then \ + echo "${TARGETARCH} is not a supported architecture. Please use linux/amd64 or linux/arm64"; \ + exit 1; fi + RUN apt-get -qq update && \ export DEBIAN_FRONTEND=noninteractive && \ apt-get install -qq \ @@ -33,11 +39,12 @@ ENV LANGUAGE en_US:en ENV LC_ALL en_US.UTF-8 ENV DEBIAN_FRONTEND=noninteractive + # Upgrade pip RUN pip3 install --upgrade pip # Install AWS CLI v2 -RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \ +RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-$(uname -m).zip" -o "awscliv2.zip" && \ unzip -q awscliv2.zip && \ ./aws/install && \ rm -rf awscliv2.zip aws @@ -51,9 +58,11 @@ RUN chmod +x /usr/local/bin/docker-compose # Install Go. Keep in sync with other Dockerfiles. ENV GOLANG_VERSION 1.18 -ENV GOLANG_DOWNLOAD_SHA256 e85278e98f57cdb150fe8409e6e5df5343ecb13cebf03a5d5ff12bd55a80264f -RUN curl -fsSL "https://dl.google.com/go/go${GOLANG_VERSION}.linux-amd64.tar.gz" -o golang.tar.gz \ - && echo "$GOLANG_DOWNLOAD_SHA256 golang.tar.gz" | sha256sum -c - \ +RUN GOLANG_DOWNLOAD_SHA256=$(if [ "${TARGETARCH}" = "arm64" ]; \ + then echo "7ac7b396a691e588c5fb57687759e6c4db84a2a3bbebb0765f4b38e5b1c5b00e"; \ + else echo "e85278e98f57cdb150fe8409e6e5df5343ecb13cebf03a5d5ff12bd55a80264f"; fi) \ + && curl -fsSL "https://dl.google.com/go/go${GOLANG_VERSION}.linux-${TARGETARCH}.tar.gz" -o golang.tar.gz \ + && echo "${GOLANG_DOWNLOAD_SHA256} golang.tar.gz" | sha256sum -c - \ && tar -C /usr/local -xzf golang.tar.gz \ && rm golang.tar.gz @@ -73,11 +82,11 @@ RUN PATH="$PATH:/usr/local/go/bin" GOPATH=/usr/local/gotools go install \ RUN PATH="$PATH:/usr/local/go/bin" GOPATH=/usr/local/gotools go install \ github.com/uudashr/gopkgs/v2/cmd/gopkgs@latest RUN PATH="$PATH:/usr/local/go/bin" go install github.com/go-delve/delve/cmd/dlv@latest -RUN GO111MODULE=on PATH="$PATH:/usr/local/go/bin" GOPATH=/usr/local/gotools go install \ +RUN PATH="$PATH:/usr/local/go/bin" GOPATH=/usr/local/gotools go install \ golang.org/x/tools/gopls@latest RUN PATH="$PATH:/usr/local/go/bin" GOPATH=/usr/local/gotools go install \ github.com/cweill/gotests/... -RUN GO111MODULE=on PATH="$PATH:/usr/local/go/bin" GOPATH=/usr/local/gotools GOBIN=/tmp/ go install \ +RUN PATH="$PATH:/usr/local/go/bin" GOPATH=/usr/local/gotools GOBIN=/tmp/ go install \ github.com/go-delve/delve/cmd/dlv@master && \ mv /tmp/dlv $GOPATH/bin/dlv-dap ENV PATH="$PATH:/usr/local/gotools/bin:/usr/local/go/bin" diff --git a/justfile b/justfile index b54deed..61d4135 100644 --- a/justfile +++ b/justfile @@ -83,13 +83,20 @@ clean: rm -rf data/ rm -rf .chart/ +# Remove docker images and clear build cache (useful to run before building cross platform images) +clean-docker: + docker image rm rstudio/platform-lib:lib-build rstudio/platform-lib:lib-e2e + docker builder prune + # Builds the docker image used for building Go code -build-build-env: - docker build -t rstudio/platform-lib:lib-build -f docker/bionic/Dockerfile docker/bionic +# * args - Optional additional docker build args +build-build-env *args: + DOCKER_BUILDKIT=1 docker build {{args}} -t rstudio/platform-lib:lib-build -f docker/bionic/Dockerfile docker/bionic # Builds the docker image for e2e testing -build-e2e-env: - docker build --network host -t rstudio/platform-lib:lib-e2e -f .github/actions/test/Dockerfile .github/actions/test +# * args - Optional additional docker build args +build-e2e-env *args: + docker build {{args}} --network host -t rstudio/platform-lib:lib-e2e -f .github/actions/test/Dockerfile .github/actions/test # Creates a container for e2e testing # * name - The container name