upload logs and reports to s3 on GH Actions jobs failures

.github/workflows/ci.yml

@@ -10,6 +10,10 @@ env:
   # required for run_common_test.erl
   KEEP_COVER_RUNNING: '1'
   SKIP_AUTO_COMPILE: 'true'
+  # required for tools/gh-upload-to-s3.sh script
+  AWS_DEFAULT_REGION: "${{ secrets.AWS_DEFAULT_REGION }}"
+  AWS_ACCESS_KEY_ID: "${{ secrets.AWS_ACCESS_KEY_ID }}"
+  AWS_SECRET_ACCESS_KEY: "${{ secrets.AWS_SECRET_ACCESS_KEY }}"
 
 jobs:
   small_tests:
@@ -42,6 +46,9 @@ jobs:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           flag-name: small_tests on OTP ${{matrix.otp}}
           path-to-lcov: ./lcov.info
+      - name: upload common test results on failure
+        if: ${{ failure() }}
+        run: tools/gh-upload-to-s3.sh _build/test/logs test_logs
 
   big_tests:
     name: ${{matrix.preset}} on OTP ${{matrix.otp}}
@@ -70,6 +77,9 @@ jobs:
           preset: ${{matrix.preset}}
           gh-token: ${{secrets.GITHUB_TOKEN}}
           test-spec: ${{matrix.test-spec}}
+      - name: upload common test results on failure
+        if: ${{ failure() }}
+        run: tools/gh-upload-to-s3.sh big_tests/ct_report
 
   dynamic_domains_big_tests:
     name: dynamic domains ${{matrix.preset}} on OTP ${{matrix.otp}}
@@ -92,6 +102,9 @@ jobs:
           preset: ${{matrix.preset}}
           gh-token: ${{secrets.GITHUB_TOKEN}}
           test-spec: ${{matrix.test-spec}}
+      - name: upload common test results on failure
+        if: ${{ failure() }}
+        run: tools/gh-upload-to-s3.sh big_tests/ct_report
 
   coveralls_webhook:
     needs: [big_tests, small_tests, dynamic_domains_big_tests]

tools/gh-actions-configure-preset.sh

@@ -45,5 +45,6 @@ case "$PRESET" in
 esac
 
 if [ ! -z "$GITHUB_ENV" ]; then
+  ## $PRESET is required for gh-upload-to-s3.sh script
   env | grep -E "^(DB|REL_CONFIG|TLS_DIST|PRESET)=" >> $GITHUB_ENV
 fi

tools/gh-upload-to-s3.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+dir="$1"
+
+[ -d "$dir" ] || { echo "ERROR: directory '${dir}' does not exist"; exit 1; }
+
+dest_dir="${2:-$(basename "$dir")}"
+
+[ -z "$AWS_ACCESS_KEY_ID" ] && { echo "ERROR: AWS_ACCESS_KEY_ID env is not set"; exit 1; }
+[ -z "$AWS_SECRET_ACCESS_KEY" ] && { echo "ERROR: AWS_SECRET_ACCESS_KEY env is not set"; exit 1; }
+[ -z "$AWS_DEFAULT_REGION" ] && { echo "ERROR: AWS_DEFAULT_REGION env is not set"; exit 1; }
+
+[ -z "$GITHUB_RUN_ID" ] && { echo "ERROR: GITHUB_RUN_ID env is not set"; exit 1; }
+[ -z "$GITHUB_RUN_ATTEMPT" ] && { echo "ERROR: GITHUB_RUN_ATTEMPT env is not set"; exit 1; }
+
+which aws || { echo "aws tool is missing"; exit 1; }
+
+aws configure set aws_access_key_id "$AWS_ACCESS_KEY_ID"
+aws configure set aws_secret_access_key "$AWS_SECRET_ACCESS_KEY"
+aws configure set default.region "$AWS_DEFAULT_REGION"
+aws configure set default.s3.max_concurrent_requests 64
+
+file_count=$(find "$dir" -type f | wc -l)
+echo "Uploading ${file_count} files"
+
+## $PRESET is not unique, so we have to add $RANDOM to avoid collisions
+prefix="GH/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}/${PRESET}.${RANDOM}/${dest_dir}"
+
+echo "directory '${dir}' is uploaded here:"
+echo "   https://esl.github.io/circleci-mim-results/s3_reports.html?prefix=${prefix}"
+
+time aws s3 cp "$dir" s3://circleci-mim-results/"${prefix}" --acl public-read --recursive --quiet