Merge branch 'maint' into wm-WX_MACOS_NON_GUI_APP

.github/dockerfiles/Dockerfile.64-bit

@@ -1,9 +1,8 @@
 ARG BASE=docker.pkg.github.com/erlang/otp/ubuntu-base
 FROM $BASE
 
-ARG MAKEFLAGS=-j4
+ARG MAKEFLAGS=$MAKEFLAGS
 ENV MAKEFLAGS=$MAKEFLAGS \
-    ERLC_USE_SERVER=yes \
     ERL_TOP=/buildroot/otp \
     PATH=/otp/bin:/buildroot/otp/bin:$PATH
 
@@ -40,3 +39,10 @@ RUN latest () { \
 ENV ERL_LIBS=/buildroot/proper:/buildroot/jsx
 
 WORKDIR /buildroot/otp/
+
+## Update init.sh with correct env vars
+RUN echo "export MAKEFLAGS=$MAKEFLAGS" > /buildroot/env.sh && \
+    echo "export ERLC_USE_SERVER=$ERLC_USE_SERVER" >> /buildroot/env.sh && \
+    echo "export ERL_TOP=$ERL_TOP" >> /buildroot/env.sh && \
+    echo "export PATH=$PATH" >> /buildroot/env.sh && \
+    echo "export ERL_LIBS=$ERL_LIBS" >> /buildroot/env.sh

.github/dockerfiles/Dockerfile.ubuntu-base

@@ -19,6 +19,7 @@ RUN apt-get update && apt-get -y upgrade && \
     unixodbc odbc-postgresql postgresql \
     tzdata ssh openssh-server groff-base sudo gdb tinyproxy bind9 nsd expect vsftpd python \
     linux-tools-common linux-tools-generic linux-tools-`uname -r` curl jq \
+    xvfb libgl1-mesa-dri \
     ${INSTALL_LIBS} && \
     for lib in ${EXTRA_LIBS}; do apt-get install -y ${lib}; done && \
     if [ ! -f /etc/apache2/apache2.conf ]; then apt-get install -y apache2; fi && \
@@ -28,6 +29,24 @@ RUN apt-get update && apt-get -y upgrade && \
 ## EXTRA_LIBS are installed using a for loop because of bugs in the erlang-doc deb package
 ## Apache2 may already be installed, if so we do not want to install it again
 
+ARG MAKEFLAGS=-j4
+ENV MAKEFLAGS=$MAKEFLAGS \
+    ERLC_USE_SERVER=yes
+
+## We install the latest version of the previous three releases in order to do
+## backwards compatability testing of the Erlang distribution.
+RUN apt-get install -y git && \
+    curl -L https://raw.githubusercontent.com/kerl/kerl/master/kerl > /usr/bin/kerl && \
+    chmod +x /usr/bin/kerl && \
+    kerl update releases && \
+    LATEST=$(kerl list releases | tail -1 | awk -F '.' '{print $1}') && \
+    for release in $(seq $(( LATEST - 3 )) $(( LATEST - 1 ))); do \
+      VSN=$(kerl list releases | grep "^$release" | tail -1); \
+      kerl build ${VSN} ${VSN} && \
+      kerl install ${VSN} /usr/local/lib/erlang-${VSN}; \
+    done && \
+    rm -rf ~/.kerl
+
 ENV LANG=en_US.UTF-8 LC_ALL=en_US.UTF-8
 
 ARG USER=gitpod

.github/dockerfiles/init.sh

@@ -1,5 +1,9 @@
 #!/bin/sh
 
+if [ -f "/buildroot/env.sh" ]; then
+    . "/buildroot/env.sh"
+fi
+
 sudo mkdir -p -m0755 /var/run/sshd
 
 sudo /usr/sbin/sshd
@@ -8,4 +12,9 @@ sudo service postgresql start
 
 sudo -E bash -c "apt-get update && apt-get install -y linux-tools-common linux-tools-generic linux-tools-`uname -r`"
 
+sudo bash -c "Xvfb :99 -ac -screen 0 1920x1080x24 -nolisten tcp" &
+export DISPLAY=:99
+
+PATH=$PATH:$(ls -1d /usr/local/lib/erlang-*/bin | tr '\n' ':')
+
 exec /bin/bash -c "$1"

.github/scripts/build-base-image.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+
+if [ -z "${BASE_TAG}" ]; then
+    BASE_TAG=$(grep "ARG BASE=" ".github/dockerfiles/Dockerfile.${1}" | head -1 | tr '=' ' ' | awk '{print $3}')
+fi
+
+case "${BASE_TAG}" in
+    *i386-debian-base)
+        BASE="i386/debian:bullseye"
+        BASE_TYPE=debian-base
+        ;;
+    *debian-base)
+        BASE="debian:bullseye"
+        BASE_TYPE=debian-base
+        ;;
+    *ubuntu-base)
+        BASE="ubuntu:20.04"
+        BASE_TYPE=ubuntu-base
+        ;;
+esac
+
+echo "::set-output name=BASE::${BASE}"
+echo "::set-output name=BASE_TAG::${BASE_TAG}"
+echo "::set-output name=BASE_TYPE::${BASE_TYPE}"
+
+if [ -f "otp_docker_base.tar" ]; then
+    docker load -i "otp_docker_base.tar"
+    echo "::set-output name=BASE_BUILD::loaded"
+elif [ -f "otp_docker_base/otp_docker_base.tar" ]; then
+    docker load -i "otp_docker_base/otp_docker_base.tar"
+    echo "::set-output name=BASE_BUILD::loaded"
+else
+    if [ "${BASE_USE_CACHE}" != "false" ]; then
+        docker pull "${BASE_TAG}"
+        BASE_CACHE="--cache-from ${BASE_TAG}"
+    fi
+
+    BASE_IMAGE_ID=$(docker images -q "${BASE_TAG}")
+
+    docker build --pull --tag "${BASE_TAG}" \
+       ${BASE_CACHE} \
+       --file ".github/dockerfiles/Dockerfile.${BASE_TYPE}" \
+       --build-arg MAKEFLAGS=-j$(($(nproc) + 2)) \
+       --build-arg USER=otptest --build-arg GROUP=uucp \
+       --build-arg uid="$(id -u)" \
+       --build-arg BASE="${BASE}" .github/
+
+    NEW_BASE_IMAGE_ID=$(docker images -q "${BASE_TAG}")
+    if [ "${BASE_IMAGE_ID}" = "${NEW_BASE_IMAGE_ID}" ]; then
+        echo "::set-output name=BASE_BUILD::cached"
+    else
+        echo "::set-output name=BASE_BUILD::re-built"
+        docker save "${BASE_TAG}" > "otp_docker_base.tar"
+    fi
+fi

.github/scripts/restore-otp-image.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+BASE_TAG=$(grep "ARG BASE=" ".github/dockerfiles/Dockerfile.64-bit" | head -1 | tr '=' ' ' | awk '{print $3}')
+
+BASE_TAG="$BASE_TAG" .github/scripts/build-base-image.sh
+
+cat > Dockerfile <<EOF
+FROM ${BASE_TAG}
+ADD otp-ubuntu-20.04.tar.gz /
+WORKDIR /buildroot/otp/
+EOF
+docker build -t otp .

.github/workflows/main.yaml

@@ -25,6 +25,8 @@ jobs:
   pack:
     name: Build Erlang/OTP (64-bit)
     runs-on: ubuntu-latest
+    outputs:
+        BASE_BUILD: ${{ steps.base-build.outputs.BASE_BUILD }}
     steps:
       - uses: actions/checkout@v2
       - name: Create initial pre-release tar
@@ -40,40 +42,35 @@ jobs:
           registry: docker.pkg.github.com
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}
-      - name: Calculate BASE image
-        id: base
-        run: |
-          BASE_TAG=$(grep "ARG BASE=" .github/dockerfiles/Dockerfile.64-bit | head -1 | tr '=' ' ' | awk '{print $3}')
-          echo "::set-output name=BASE_TAG::${BASE_TAG}"
-          .github/scripts/base-tag "${BASE_TAG}"
-      - name: Pull BASE image
-        run: docker pull ${{ steps.base.outputs.BASE_TAG }}
       - name: Build BASE image
-        run: |
-          docker build --pull --tag ${{ steps.base.outputs.BASE_TAG }} \
-            --cache-from ${{ steps.base.outputs.BASE_TAG }} \
-            --file .github/dockerfiles/Dockerfile.${{ steps.base.outputs.BASE_TYPE }} \
-            --build-arg USER=otptest --build-arg GROUP=uucp \
-            --build-arg uid=$(id -u) \
-            --build-arg BASE=${{ steps.base.outputs.BASE }} .github/
-      - name: Build 64-bit image
+        id: base-build
+        run: .github/scripts/build-base-image.sh 64-bit
+      - name: Save BASE image
+        if: steps.base-build.outputs.BASE_BUILD == 're-built'
+        uses: actions/upload-artifact@v2
+        with:
+            name: otp_docker_base
+            path: otp_docker_base.tar
+      - name: Build image
         run: |
           mv otp_src.tar.gz .github/otp.tar.gz
           docker build --tag otp \
             --build-arg MAKEFLAGS=-j$(($(nproc) + 2)) \
-            --file .github/dockerfiles/Dockerfile.64-bit \
+            --file ".github/dockerfiles/Dockerfile.64-bit" \
             .github/
-      - name: Save docker image
-        run: docker save otp > otp_docker.tar
-      - name: Upload docker image
+      - name: Save Erlang/OTP image
+        run: |
+            docker run -v $PWD:/github --entrypoint "" otp \
+              tar czf /github/otp-ubuntu-20.04.tar.gz /buildroot/ /otp/
+      - name: Upload otp ubuntu image
         uses: actions/upload-artifact@v2
         with:
-          name: otp_docker
-          path: otp_docker.tar
+          name: otp-ubuntu-20.04
+          path: otp-ubuntu-20.04.tar.gz
       - name: Build pre-built tar archives
         run: |
-          docker run -v $PWD:/github otp \
-            "scripts/build-otp-tar -o /github/otp_clean_src.tar.gz /github/otp_src.tar.gz -b /buildroot/otp/ /buildroot/otp.tar.gz"
+          docker run -v $PWD:/github --entrypoint "" otp \
+            scripts/build-otp-tar -o /github/otp_clean_src.tar.gz /github/otp_src.tar.gz -b /buildroot/otp/ /buildroot/otp.tar.gz
       - name: Upload pre-built tar archive
         uses: actions/upload-artifact@v2
         with:
@@ -92,7 +89,7 @@ jobs:
         id: apps
         run: |
           .github/scripts/path-filters.sh > .github/scripts/path-filters.yaml
-          ALL_APPS=$(grep '^[a-z_]*:$' .github/scripts/path-filters.yaml | sed 's/:$//')
+          ALL_APPS=$(grep '^[a-z_]*:' .github/scripts/path-filters.yaml | sed 's/:.*$//')
           ALL_APPS=$(jq -n --arg inarr "${ALL_APPS}" '$inarr | split("\n")' | tr '\n' ' ')
           echo "::set-output name=all::${ALL_APPS}"
       - uses: dorny/paths-filter@v2
@@ -293,47 +290,45 @@ jobs:
         uses: actions/download-artifact@v2
         with:
           name: otp_prebuilt_no_chunks
-      - run: echo "${{ needs.pack.outputs.changes }}"
       - name: Docker login
         uses: docker/login-action@v1
         with:
           registry: docker.pkg.github.com
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}
-      - name: Calculate BASE image
-        id: base
-        run: |
-          BASE_TAG=$(grep "ARG BASE=" .github/dockerfiles/Dockerfile.${{ matrix.type }} | head -1 | tr '=' ' ' | awk '{print $3}')
-          echo "::set-output name=BASE_TAG::${BASE_TAG}"
-          .github/scripts/base-tag "${BASE_TAG}"
-      - name: Pull BASE image
-        run: docker pull ${{ steps.base.outputs.BASE_TAG }}
-      - name: Build BASE image
-        run: |
-          docker build --pull --tag ${{ steps.base.outputs.BASE_TAG }} \
-            --cache-from ${{ steps.base.outputs.BASE_TAG }} \
-            --file .github/dockerfiles/Dockerfile.${{ steps.base.outputs.BASE_TYPE }} \
-            --build-arg USER=otptest --build-arg GROUP=uucp \
-            --build-arg uid=$(id -u) \
-            --build-arg BASE=${{ steps.base.outputs.BASE }} .github/
+      - name: Build base image
+        run: .github/scripts/build-base-image.sh ${{ matrix.type }}
       - name: Build ${{ matrix.type }} image
         run: |
           mv otp_src.tar.gz .github/otp.tar.gz
           docker build --tag otp --file .github/dockerfiles/Dockerfile.${{ matrix.type }} \
-            .github/
+            .github
 
   documentation:
     name: Build and check documentation
     runs-on: ubuntu-latest
     needs: pack
     steps:
+      - uses: actions/checkout@v2
       ## Download docker images
-      - name: Download source archive
+      - name: Docker login
+        uses: docker/login-action@v1
+        with:
+          registry: docker.pkg.github.com
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Download base build
+        if: needs.pack.outputs.BASE_BUILD == 're-built'
         uses: actions/download-artifact@v2
         with:
-          name: otp_docker
+          name: otp_docker_base
+      - name: Download otp build
+        uses: actions/download-artifact@v2
+        with:
+          name: otp-ubuntu-20.04
       - name: Restore docker image
-        run: docker load < otp_docker.tar
+        run: .github/scripts/restore-otp-image.sh
+
       ## Build pre-built tar with chunks
       - name: Build doc chunks
         run: |
@@ -387,13 +382,25 @@ jobs:
     runs-on: ubuntu-latest
     needs: pack
     steps:
+      - uses: actions/checkout@v2
       ## Download docker images
-      - name: Download source archive
+      - name: Docker login
+        uses: docker/login-action@v1
+        with:
+          registry: docker.pkg.github.com
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Download base build
+        if: needs.pack.outputs.BASE_BUILD == 're-built'
+        uses: actions/download-artifact@v2
+        with:
+          name: otp_docker_base
+      - name: Download otp build
         uses: actions/download-artifact@v2
         with:
-          name: otp_docker
+          name: otp-ubuntu-20.04
       - name: Restore docker image
-        run: docker load < otp_docker.tar
+        run: .github/scripts/restore-otp-image.sh
       - name: Install clang-format
         run: |
           docker build -t otp - <<EOF
@@ -419,13 +426,25 @@ jobs:
         # type: ["os_mon","sasl"]
       fail-fast: false
     steps:
+      - uses: actions/checkout@v2
       ## Download docker images
-      - name: Download source archive
+      - name: Download base build
+        if: needs.pack.outputs.BASE_BUILD == 're-built'
+        uses: actions/download-artifact@v2
+        with:
+          name: otp_docker_base
+      - name: Download otp build
         uses: actions/download-artifact@v2
         with:
-          name: otp_docker
+          name: otp-ubuntu-20.04
+      - name: Docker login
+        uses: docker/login-action@v1
+        with:
+          registry: docker.pkg.github.com
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
       - name: Restore docker image
-        run: docker load < otp_docker.tar
+        run: .github/scripts/restore-otp-image.sh
       - name: Run tests
         id: run-tests
         run: |
@@ -469,16 +488,25 @@ jobs:
     if: always() # Run even if the need has failed
     needs: test
     steps:
+      - uses: actions/checkout@v2
+      - name: Docker login
+        uses: docker/login-action@v1
+        with:
+          registry: docker.pkg.github.com
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
       - name: Download test results
         uses: actions/download-artifact@v2
+      - run: mv otp-ubuntu-20.04/otp-ubuntu-20.04.tar.gz .
+      - name: Restore docker image
+        run: .github/scripts/restore-otp-image.sh
       - name: Merge test results
         run: |
           shopt -s nullglob
           mkdir -p make_test_dir
           for file in *_test_results/*.tar.gz; do
             tar xzf $file
           done
-          docker load < otp_docker/otp_docker.tar
           docker run -v $PWD/make_test_dir:/buildroot/otp/erts/make_test_dir otp \
             "ct_run -refresh_logs /buildroot/otp/erts/make_test_dir"
       - name: Run system tests