Skip to content
Back to pull request #4235

[documentation] #4234: Iroha 2 README YAC #737

Sign in to view logs

[documentation] #4234: Iroha 2 README YAC

[documentation] #4234: Iroha 2 README YAC #737

Workflow file for this run

.github/workflows/build-iroha1-fork.yml at 6f1ddc8
## DO NOT EDIT
## Generated from build-iroha1-fork.src.yml with make-workflows.sh
name: Iroha1-fork
# The only write permission required is packages
permissions:
actions: read
checks: read
contents: read
deployments: read
issues: read
packages: write
pull-requests: read
repository-projects: read
security-events: read
statuses: read
## This workflow is created for pull requests from forks and has less permissions than build-iroha1 workflow
on:
pull_request_target:
branches: [iroha1-main, support/1.*, edge]
env:
DOCKERHUB_ORG: hyperledger
jobs:
## This job checks if PRs is from fork
## Also checks that .github folder, Dockerfiles and scripts in docker directory are not changed
check_if_pull_request_comes_from_fork:
runs-on: ubuntu-20.04 #ubuntu-latest
permissions: read-all
name: Pull requests from forks should use this workflow
if: github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name
steps:
- name: Show context
run: |
echo $JSON_github | jq .
echo $JSON_needs | jq .
env:
JSON_github: ${{ toJSON(github) }}
JSON_needs: ${{ toJson(needs) }}
- name: Checkout head
uses: actions/checkout@v2
with:
ref: ${{ github.event.pull_request.head.sha }}
- name: Filter files
uses: dorny/paths-filter@v2
id: filter
with:
filters: |
github:
- ".github/**"
dockerfile:
- "**/Dockerfile"
- "docker/release/entrypoint.sh"
- "docker/release/wait-for-it.sh"
build_dependecies:
- "vcpkg/build_iroha_deps.sh"
- name: verify .github folder is not changed
if: steps.filter.outputs.github == 'true'
run: |
echo "Pull requests from forks are not allowed to change .github folder"
false
- name: verify Dockerfiles and scripts in docker directory are not changed
if: steps.filter.outputs.dockerfile == 'true'
run: |
echo "Pull requests from forks are not allowed to change Dockerfiles"
false
- name: verify build depedencies script is not changed
if: steps.filter.outputs.build_dependecies == 'true'
run: |
echo "Pull requests from forks are not allowed to change build dependencies script"
false
## This job is to generate build matrixes for build jobs
## The matrixes depend on what is requeted to be build
## At the moment there are several options:
## - default on pushes, pull requests
## - on comment to pull request according to comment message (chat-ops)
## - on workflow_dispatch according to its build_spec
## - on schedule '/build all'
generate_matrixes:
environment: test-env
runs-on: ubuntu-20.04
permissions: read-all
needs: check_if_pull_request_comes_from_fork
steps:
- name: Show context
run: |
echo $JSON_github | jq .
echo $JSON_needs | jq .
env:
JSON_github: ${{ toJSON(github) }}
JSON_needs: ${{ toJson(needs) }}
- name: REF and SHA of commented PR to ENV
if: github.event.comment
run: >
curl -fsSL ${{github.event.issue.pull_request.url}} -H "Authorization: token ${{github.token}}" | jq -r '
"PR_REF="+.head.ref,
"PR_SHA="+.head.sha,
"PR_NUM="+(.number|tostring),
"PR_REPO="+.head.repo.full_name' >>$GITHUB_ENV
- name: Checkout head
uses: actions/checkout@v2
with:
ref: ${{ github.event.pull_request.head.sha }}
- name: Generate default matrix for regular builds
run: |
set -x
git fetch origin ${{github.event.pull_request.head.sha}} --depth=2 ## depth=2 to detect if fetched commit is merge commit
git log -1 FETCH_HEAD
commit_message_body_build_spec(){
git log -n1 $1 --format=%B | grep '^/build '
}
git_is_merge_commit(){
git rev-parse ${1:-HEAD}^2 &>/dev/null
}
commit_was_merged_build_spec(){
git_is_merge_commit $1 &&
git log -n1 $1 --format=%s | grep -q '^Merge branch' &&
echo "/build ubuntu debug release normal gcc-10"
}
case ${{github.event_name}} in
pull_request_target) if commit_message_body_build_spec FETCH_HEAD >/tmp/comment_body ;then
if git_is_merge_commit FETCH_HEAD ;then
echo ::warning::'/build directive in merge commit overrides default "/build ubuntu debug release normal gcc-10"'
fi
elif commit_was_merged_build_spec FETCH_HEAD >/tmp/comment_body ;then
true
else
#echo >/tmp/comment_body "/build debug; /build ubuntu release debug normal"
echo >/tmp/comment_body "/build ubuntu debug release normal gcc-10"
fi ;;
esac
- name: Generate matrixes
id: matrixes
run: |
set -x
cat /tmp/comment_body | .github/chatops-gen-matrix.sh
echo "::set-output name=matrix_ubuntu::$(cat matrix_ubuntu)"
echo "::set-output name=matrix_ubuntu_release::$(cat matrix_ubuntu_release)"
echo "::set-output name=matrix_ubuntu_debug::$(cat matrix_ubuntu_debug)"
echo "::set-output name=matrix_macos::$(cat matrix_macos)"
echo "::set-output name=matrix_windows::$(cat matrix_windows)"
echo "::set-output name=matrix_dockerimage_release::$(cat matrix_dockerimage_release)"
echo "::set-output name=matrix_dockerimage_debug::$(cat matrix_dockerimage_debug)"
##TODO report errors and warnings as answer to issue comment (chat-ops)
- name: Reaction confused
if: failure() && github.event.comment
run: |
# send reaction to comment to show build was triggered
curl ${{github.event.comment.url}}/reactions \
-X POST \
-d '{"content":"confused"}' \
-H "Accept: application/vnd.github.squirrel-girl-preview+json" \
-H "Authorization: token ${{github.token}}"
- name: Reaction rocket
if: github.event.comment
run: |
# send reaction to comment to show build was triggered
curl ${{github.event.comment.url}}/reactions \
-X POST \
-d '{"content":"rocket"}' \
-H "Accept: application/vnd.github.squirrel-girl-preview+json" \
-H "Authorization: token ${{github.token}}"
outputs:
matrix_ubuntu: ${{steps.matrixes.outputs.matrix_ubuntu}}
matrix_ubuntu_release: ${{steps.matrixes.outputs.matrix_ubuntu_release}}
matrix_ubuntu_debug: ${{steps.matrixes.outputs.matrix_ubuntu_debug}}
matrix_macos: ${{steps.matrixes.outputs.matrix_macos}}
matrix_windows: ${{steps.matrixes.outputs.matrix_windows}}
matrix_dockerimage_release: ${{steps.matrixes.outputs.matrix_dockerimage_release}}
matrix_dockerimage_debug: ${{steps.matrixes.outputs.matrix_dockerimage_debug}}
## Build docker image named 'hyperledger/iroha-builder' with all stuff to compile iroha and its dependencies
## The result docker image is pushed with tags :pr-NUMBER, :commit-HASH, :branch-name, :tag-name,
## and conditional tags :edge for development branch, and :latest for git-tags.
Docker-iroha-builder:
environment: test-env
needs: check_if_pull_request_comes_from_fork
runs-on: ubuntu-20.04 #ubuntu-latest #[ self-hosted, Linux ]
steps:
- name: Show context
run: |
echo $JSON_github | jq .
echo $JSON_needs | jq .
env:
JSON_github: ${{ toJSON(github) }}
JSON_needs: ${{ toJson(needs) }}
- name: System info
run: |
set -x
whoami
id $(whoami)
free || vm_stat | perl -ne '/page size of (\d+)/ and $size=$1;
/Pages\s+([^:]+)[^\d]+(\d+)/ and printf("%-16s % 16.2f Mi\n", "$1:", $2 * $size / 1048576);'
df -h
- name: Build info
run: |
cat << 'END'
ref:$github_ref
sha:${{github.sha}}
run_number:${{github.run_number}}
event_name:${{github.event_name}}
event.action:${{github.event.action}}
event.issue.number:${{ github.event.issue.number }}
END
env:
github_ref: ${{ github.ref }}
- name: REF and SHA of commented PR to ENV
if: github.event.comment
run: >
curl -fsSL ${{github.event.issue.pull_request.url}} -H "Authorization: token ${{github.token}}" | jq -r '
"PR_REF="+.head.ref,
"PR_SHA="+.head.sha,
"PR_NUM="+(.number|tostring),
"PR_REPO="+.head.repo.full_name' >>$GITHUB_ENV
- name: Checkout base
uses: actions/checkout@v2
with:
ref: ${{ github.event.pull_request.base.sha }}
- name: Determine dockertag
id: dockertag
env:
dockertag: ${{ hashFiles('docker/iroha-builder/**') }}
run: |
echo "::set-output name=dockertag::$dockertag"
echo >>$GITHUB_ENV dockertag=$dockertag
test -n "$DOCKERHUB_ORG" || {
echo ::error::"DOCKERHUB_ORG must contain value"
false
}
- name: Login to DockerHub
#if: ${{ secrets.DOCKERHUB_TOKEN != '' && secrets.DOCKERHUB_USERNAME != '' }}
id: docker_login
uses: docker/login-action@v1
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Log in to the Container registry GHCR
uses: docker/login-action@v1
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Possible WARNING
if: ${{ steps.docker_login.outcome == 'skipped' }}
run: echo "::warning::DOCKERHUB_TOKEN and DOCKERHUB_USERNAME are empty. Will build but NOT push."
- name: Docker meta
id: meta
uses: docker/metadata-action@v3
with:
images: ${{ env.DOCKERHUB_ORG }}/iroha-builder
tags: |
type=raw,value=${{env.dockertag}}
type=ref,event=branch
type=ref,event=pr
type=ref,event=tag
type=schedule
## Docker image will be pushed with tags:
## - hash of file Dockerfile.builder
## - branchname, when branch is pushed
## - pr-NUMBER, when pushed to PR
## - git tag when tag is pushed
## - schedule, see the docs
- uses: docker/metadata-action@v3
name: Docker meta GHCR
id: meta_ghcr
with:
tags: |
type=raw,value=${{env.dockertag}}
type=ref,event=branch
type=ref,event=pr
type=ref,event=tag
type=schedule
## Docker image will be pushed with tags:
## - hash of file Dockerfile.builder
## - branchname, when branch is pushed
## - pr-NUMBER, when pushed to PR
## - git tag when tag is pushed
## - schedule, see the docs
images: ghcr.io/${{ github.repository }}-builder
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
- name: Cache Docker layers
uses: actions/cache@v2
with:
path: /tmp/.buildx-cache
key: ${{ runner.os }}-buildx-${{env.dockertag}}
restore-keys: ${{ runner.os }}-buildx-
- id: build_and_push
name: Build and push
uses: docker/build-push-action@v2
with:
context: docker/iroha-builder/
cache-from: type=local,src=/tmp/.buildx-cache
cache-to: type=local,dest=/tmp/.buildx-cache-new
push: ${{ steps.docker_login.outcome == 'success' }}
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
- uses: docker/build-push-action@v2
id: build_and_push_ghcr
name: Build and push to GHCR
with:
context: docker/iroha-builder/
cache-from: type=local,src=/tmp/.buildx-cache
cache-to: type=local,dest=/tmp/.buildx-cache-new
push: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo == github.event.pull_request.base.repo }}
tags: ${{ steps.meta_ghcr.outputs.tags }}
labels: ${{ steps.meta_ghcr.outputs.labels }}
- # Temp fix
# https://github.com/docker/build-push-action/issues/252
# https://github.com/moby/buildkit/issues/1896
name: Move cache
run: |
rm -rf /tmp/.buildx-cache
mv /tmp/.buildx-cache-new /tmp/.buildx-cache
- name: Check if dockertaghash exists in remote registry
id: dockertag_already
run: |
echo "::set-output name=container::$DOCKERHUB_ORG/iroha-builder:$dockertag"
docker pull "$DOCKERHUB_ORG/iroha-builder:$dockertag"
outputs:
## WARN secret dropped from output!, output may not contain secret,
## and secret cannot be used in job:container directly, and there is no github non-secret variables...
## if dockertag is already pushed then use it. But let it be empty when tag does not exist remotely.
container: ${{steps.dockertag_already.outputs.container}}
dockertag: ${{steps.dockertag.outputs.dockertag}}
pushed: ${{ steps.docker_login.outcome == 'success' && steps.build_and_push.outcome == 'success' }}
## Build iroha in a container made of the image earlier prepared
## Result artifacts are
## - stand-alone irohad (linked statically)
## - iroha.deb (with irohad, migration-tool, wsv_checker inside)
build-UD:
needs:
- Docker-iroha-builder
- generate_matrixes
runs-on: [self-hosted, Linux, iroha]
permissions: read-all
container: ## Container is taken from previous job
image: ${{needs.Docker-iroha-builder.outputs.container}}
options: --user root
strategy:
fail-fast: false
matrix: ${{ fromJSON( needs.generate_matrixes.outputs.matrix_ubuntu_debug ) }}
if: ${{ fromJSON( needs.generate_matrixes.outputs.matrix_ubuntu_debug ).include[0] }}
defaults:
run:
shell: bash
steps:
- name: Show context
run: |
echo $JSON_github | jq .
echo $JSON_needs | jq .
env:
JSON_github: ${{ toJSON(github) }}
JSON_needs: ${{ toJson(needs) }}
- name: Show needs
run: |
cat >/dev/null <<'END'
${{ toJson(needs) }}
END
- env:
container: ${{needs.Docker-iroha-builder.outputs.container}}
run: test -n "$container"
- name: System info
run: |
set -x
whoami
id $(whoami)
free || vm_stat | perl -ne '/page size of (\d+)/ and $size=$1;
/Pages\s+([^:]+)[^\d]+(\d+)/ and printf("%-16s % 16.2f Mi\n", "$1:", $2 * $size / 1048576);'
df -h
- name: Build info
run: |
cat << 'END'
ref:$github_ref
sha:${{github.sha}}
run_number:${{github.run_number}}
event_name:${{github.event_name}}
event.action:${{github.event.action}}
event.issue.number:${{ github.event.issue.number }}
END
env:
github_ref: ${{ github.ref }}
- name: export CC,BuildType from matrix.buildspec
run: |
echo >>$GITHUB_ENV OS=$(echo ${{matrix.buildspec}} | awk '{print $1}')
echo >>$GITHUB_ENV CC_NAME=$(echo ${{matrix.buildspec}} | awk '{print $2}')
echo >>$GITHUB_ENV BuildType=$(echo ${{matrix.buildspec}} | awk '{print $3}')
features=$(echo ${{matrix.buildspec}} | awk '{print $4}')
case $features in
normal) echo >>$GITHUB_ENV CMAKE_USE=""; features= ;;
ursa) echo >>$GITHUB_ENV CMAKE_USE="-DUSE_LIBURSA=ON";;
burrow) echo >>$GITHUB_ENV CMAKE_USE="-DUSE_BURROW=ON";;
*) echo "::error::Unknown features '$features'"; false ;;
esac
echo >>$GITHUB_ENV features="$features"
echo >>$GITHUB_ENV skip_testing=$(echo ${{matrix.buildspec}} | grep -Fo skip_testing)
- name: REF and SHA of commented PR to ENV
if: github.event.comment
run: >
curl -fsSL ${{github.event.issue.pull_request.url}} -H "Authorization: token ${{github.token}}" | jq -r '
"PR_REF="+.head.ref,
"PR_SHA="+.head.sha,
"PR_NUM="+(.number|tostring),
"PR_REPO="+.head.repo.full_name' >>$GITHUB_ENV
- name: Checkout head
uses: actions/checkout@v2
with:
ref: ${{ github.event.pull_request.head.sha }}
fetch-depth: 0 ## full history
- name: export CC and CXX
env:
CCACHE_PATH: /usr/lib/ccache
run: |
set -xeu #o pipefail
if test $CC_NAME = llvm
then CC=/usr/local/opt/llvm/bin/clang
else CC=$CC_NAME
fi
echo >>$GITHUB_ENV CC=$CC
echo >>$GITHUB_ENV CXX=$(echo $CC | sed -es,gcc,g++, -es,clang,clang++,)
echo >>$GITHUB_PATH $CCACHE_PATH
ls -lA $CCACHE_PATH
$(realpath $CCACHE_PATH/gcc) --show-config
echo >>$GITHUB_ENV _CCACHE_DIR=$($(realpath $CCACHE_PATH/gcc) --show-config | sed -nE 's,.*cache_dir = ,,p')
echo >>$GITHUB_ENV NPROC=$(nproc | awk '{printf("%.0f",$1*0.77)}')
echo >>$GITHUB_ENV HOME=$HOME
- name: Restore cache CCache
uses: actions/cache@v2
with:
path: ${{ env._CCACHE_DIR }}
key: ${{runner.os}}-ccache-${{ github.event.pull_request.head.sha }}
restore-keys: ${{runner.os}}-ccache-
- run: ccache --show-stats | tee /tmp/ccache-stats
- ## Read the docs https://vcpkg.readthedocs.io/en/latest/users/binarycaching/ https://github.com/microsoft/vcpkg/blob/master/docs/users/binarycaching.md
name: Restore cache Vcpkg binarycache ## NOTE not useng NuGet because on ubuntu nuget needs mono of 433MB, unusable.
uses: actions/cache@v2
with:
path: |
${{env.HOME}}/.cache/vcpkg/archives
key: ${{runner.os}}-vcpkg-${{env.CC_NAME}}.${{ hashFiles('vcpkg/**') }}
restore-keys: ${{runner.os}}-vcpkg-${{env.CC_NAME}}.
- name: Build iroha vcpkg dependencies
run: ./vcpkg/build_iroha_deps.sh $PWD/vcpkg-build; test -f $PWD/vcpkg-build/scripts/buildsystems/vcpkg.cmake
## Takes 48m16s on default GitHub runner with 2 cores
## Takes 13m41s on self-hosted AWS EC2 c5.x4large
# ________________________________________________________
# Executed in 32,08 mins fish external
# usr time 110,52 mins 0,24 millis 110,52 mins
# sys time 12,26 mins 1,34 millis 12,26 mins
#
# All requested packages are currently installed.
# ________________________________________________________
# Executed in 3,17 secs fish external
# usr time 2,05 secs 128,00 micros 2,05 secs
# sys time 0,70 secs 575,00 micros 0,70 secs
- name: CMake configure
## Takes 13s on regular GitHub runner
run: cmake -B build -DCMAKE_TOOLCHAIN_FILE=$PWD/vcpkg-build/scripts/buildsystems/vcpkg.cmake -DCMAKE_BUILD_TYPE=${{ env.BuildType }} -GNinja $CMAKE_USE -DTESTING=$( test "$skip_testing" = skip_testing && echo OFF || echo ON ) -DBENCHMARKING=$( test "$skip_testing" = skip_testing && echo OFF || echo ON ) -DPACKAGE_DEB=ON
#-DCMAKE_VERBOSE_MAKEFILE=ON ## Note: use for debug
- name: CMake build
run: |
set -x
## reduce memory usage to do not overflow
cmake --build build --config ${{ env.BuildType }} -- -j$(nproc | awk '{printf("%.0f",$1*0.77)}')
echo ::notice::"$(./build/bin/irohad --version)"
## Debug takes 18m44s on regular GitHub runner
## Debug takes 7m41s on self-hosted AWS EC2 c5.x4large
## Release takes 2m58s on self-hosted AWS EC2 c5.x4large
- name: CPack (linux only)
run: cd build; cpack; ## cmake --build build --target package
- run: ccache --show-stats | diff --side-by-side /tmp/ccache-stats - ||true
- name: Show free space and disk usage
if: ${{ always() }}
run: |
df -h || true
- name: Generate artifact suffix depending on matrix to env.ARTIFACT_SUFFIX
run: |
set -x
cc=$(echo $CC_NAME | sed -Ee's,[-/],,g' )
build_type=$(echo $BuildType | tr A-Z a-z | sed -E -es,debug,dbg, -es,release,rel, )
test $build_type = dbg -o $build_type = rel
uses=$(echo "$features" | tr ' ' - | tr A-Z a-z)
_os=${OS:+-$OS} _cc=${cc:+-$cc} _build_type=${build_type:+-$build_type} _uses=${uses:+-$uses}
echo >>$GITHUB_ENV ARTIFACT_SUFFIX=$_os$_cc$_build_type$_uses
echo >>$GITHUB_ENV _uses_suffix=$_uses
echo >>$GITHUB_ENV _compiler_suffix=$(test $cc != gcc10 && echo $_cc)
echo >>$GITHUB_ENV _debug_suffix=$(test "$build_type" = dbg && echo -debug || true)
- name: Upload artifact irohad
uses: actions/upload-artifact@v2
with:
name: irohad${{env.ARTIFACT_SUFFIX}}
path: |
build/bin/irohad
build/bin/iroha-cli
- name: Upload artifact iroha-deb
uses: actions/upload-artifact@v2
with:
name: iroha-deb${{env.ARTIFACT_SUFFIX}}
path: |
build/*.deb
- if: ${{ false }} ## Maybe test in another job
name: Upload artifact tests
uses: actions/upload-artifact@v2
with:
name: iroha-tests-ubuntu${{env.ARTIFACT_SUFFIX}}
path: |
build/test_bin/**
build/test_data/**
- timeout-minutes: 40
if: env.skip_testing == ''
name: CTest
run: |
echo ::group::'boilerplate'
set -euo pipefail
if test $(uname) = Darwin ;then
## This is a common portable solution, but Debian and Ubuntu have their own wrappers
initdb --locale=C --encoding=UTF-8 --username=postgres $PWD/postgres_database
postgres -D $PWD/postgres_database -p5432 2>&1 >/tmp/postgres.log & { sleep .3; kill -0 $!; } ## use pg_ctl no need &
else ## Debian or Ubuntu
## Need to go debian-specific way because
## initdb is not allowed to be run as root, but we need to run as root
## because GitHub actions runners have much issues with permissions.
mkdir postgres_database && chown iroha-ci postgres_database
echo /usr/lib/postgresql/12/bin/initdb --locale=C --encoding=UTF-8 --username=postgres $PWD/postgres_database | su iroha-ci
echo /usr/lib/postgresql/12/bin/pg_ctl start -D $PWD/postgres_database --log=$PWD/postgres_database/log | su iroha-ci
# ## Need to go debian-specific way because
# ## initdb is not allowed to be run as root, but we need to run as root
# ## because GitHub actions runners have much issues with permissions.
# cat <<END >/etc/postgresql/12/main/pg_hba.conf
# # TYPE DATABASE USER ADDRESS METHOD
# local all all trust
# host all all 127.0.0.1/32 trust
# host all all ::1/128 trust
# local replication all trust
# host replication all 127.0.0.1/32 trust
# host replication all ::1/128 trust
# END
# pg_ctlcluster 12 main start ## Cluster 'main' exist by default
# #OR pg_createcluster -p 5432 --start 12 iroha -- --locale=C --encoding=UTF-8 --username=postgres
fi
cd build
## This is just a small lifehack to TEMPORARY allow some tests to fail
cat ../.github/TESTS_ALLOWED_TO_FAIL | sort -u >ALLOW_TO_FAIL || true
if test -e ALLOW_TO_FAIL
then echo "::warning:: There are TESTS_ALLOWED_TO_FAIL: "$(cat ALLOW_TO_FAIL)
fi
grep_failed_tests(){
grep 'The following tests FAILED:' -A10000 "$@" | tail +2 #| cut -d- -f2 | cut -d' ' -f2 | sort
}
exclude_allowed_to_fail(){
grep -Fvf ALLOW_TO_FAIL "$@"
}
only_allowed_to_fail(){
grep -Ff ALLOW_TO_FAIL "$@"
}
list_to_line(){
comma=''
while read N d name sta ;do
echo -n "$comma$N-$name$sta"
comma=', '
done
}
echo ::endgroup::
## Run module_* tests in parallel and others subsequently
## Cathgories sorted in order of importancy
CTEST_CATHEGORIES=( module tool framework regression system integration )
## Add rest available cathgories
CTEST_CATHEGORIES+=( $(
ctest --show-only=json-v1 -R "^(module|tool|framework|regression|system|integration)" |
jq -r .tests[].name |
cut -f1 -d_ |
sort -u |
grep -Fvf <( printf '%s\n' ${CTEST_CATHEGORIES[@]} )
)
) || true
CTEST_DEFAULT_timeout=80
CTEST_module_timeout=120 CTEST_module_parallel=4
CTEST_tool_timeout=200
CTEST_integration_timeout=120
CTEST_integration_args='--repeat until-pass:10' ## FIXME remove this hack
CTEST_system_args='--repeat until-pass:10' ## FIXME remove this hack
for cathegory in ${CTEST_CATHEGORIES[@]} ;do
echo >&2 ::group::"$cathegory tests"
set -x
timeout_name=CTEST_${cathegory}_timeout; timeout=${!timeout_name:-$CTEST_DEFAULT_timeout}
parallel_name=CTEST_${cathegory}_parallel; parallel=${!parallel_name:-}
args_name=CTEST_${cathegory}_args; args=${!args_name:-}
ctest -R "^${cathegory}_" ${parallel:+--parallel $parallel} --output-on-failure --no-tests=error --timeout $timeout ${args:-} \
| tee ctest_$cathegory.out \
|| true
set +x
echo >&2 ::endgroup::
done
tests_passed=true
for t in ${CTEST_CATHEGORIES[@]} ;do
f=ctest_$t.out
if a=$(grep_failed_tests $f | exclude_allowed_to_fail | list_to_line) ;then
echo "::error::The following $(echo $t | tr a-z A-Z) tests FAILED but not in list ALLOW_TO_FAIL: $a"
tests_passed=false
fi
if o=$(grep_failed_tests $f | only_allowed_to_fail | list_to_line) ;then
echo "::warning::The following $t tests FAILED and ALLOWED TO FAIL: $o"
fi
done
$tests_passed
## Just because release is built 2..3 times faster make it a different job
build-UR:
needs:
- Docker-iroha-builder
- generate_matrixes
runs-on: [self-hosted, Linux, iroha]
permissions: read-all
container: ## Container is taken from previous job
image: ${{needs.Docker-iroha-builder.outputs.container}}
options: --user root
defaults:
run:
shell: bash
steps:
- name: Show context
run: |
echo $JSON_github | jq .
echo $JSON_needs | jq .
env:
JSON_github: ${{ toJSON(github) }}
JSON_needs: ${{ toJson(needs) }}
- name: Show needs
run: |
cat >/dev/null <<'END'
${{ toJson(needs) }}
END
- env:
container: ${{needs.Docker-iroha-builder.outputs.container}}
run: test -n "$container"
- name: System info
run: |
set -x
whoami
id $(whoami)
free || vm_stat | perl -ne '/page size of (\d+)/ and $size=$1;
/Pages\s+([^:]+)[^\d]+(\d+)/ and printf("%-16s % 16.2f Mi\n", "$1:", $2 * $size / 1048576);'
df -h
- name: Build info
run: |
cat << 'END'
ref:$github_ref
sha:${{github.sha}}
run_number:${{github.run_number}}
event_name:${{github.event_name}}
event.action:${{github.event.action}}
event.issue.number:${{ github.event.issue.number }}
END
env:
github_ref: ${{ github.ref }}
- name: export CC,BuildType from matrix.buildspec
run: |
echo >>$GITHUB_ENV OS=$(echo ${{matrix.buildspec}} | awk '{print $1}')
echo >>$GITHUB_ENV CC_NAME=$(echo ${{matrix.buildspec}} | awk '{print $2}')
echo >>$GITHUB_ENV BuildType=$(echo ${{matrix.buildspec}} | awk '{print $3}')
features=$(echo ${{matrix.buildspec}} | awk '{print $4}')
case $features in
normal) echo >>$GITHUB_ENV CMAKE_USE=""; features= ;;
ursa) echo >>$GITHUB_ENV CMAKE_USE="-DUSE_LIBURSA=ON";;
burrow) echo >>$GITHUB_ENV CMAKE_USE="-DUSE_BURROW=ON";;
*) echo "::error::Unknown features '$features'"; false ;;
esac
echo >>$GITHUB_ENV features="$features"
echo >>$GITHUB_ENV skip_testing=$(echo ${{matrix.buildspec}} | grep -Fo skip_testing)
- name: REF and SHA of commented PR to ENV
if: github.event.comment
run: >
curl -fsSL ${{github.event.issue.pull_request.url}} -H "Authorization: token ${{github.token}}" | jq -r '
"PR_REF="+.head.ref,
"PR_SHA="+.head.sha,
"PR_NUM="+(.number|tostring),
"PR_REPO="+.head.repo.full_name' >>$GITHUB_ENV
- name: Checkout head
uses: actions/checkout@v2
with:
ref: ${{ github.event.pull_request.head.sha }}
fetch-depth: 0 ## full history
- name: export CC and CXX
env:
CCACHE_PATH: /usr/lib/ccache
run: |
set -xeu #o pipefail
if test $CC_NAME = llvm
then CC=/usr/local/opt/llvm/bin/clang
else CC=$CC_NAME
fi
echo >>$GITHUB_ENV CC=$CC
echo >>$GITHUB_ENV CXX=$(echo $CC | sed -es,gcc,g++, -es,clang,clang++,)
echo >>$GITHUB_PATH $CCACHE_PATH
ls -lA $CCACHE_PATH
$(realpath $CCACHE_PATH/gcc) --show-config
echo >>$GITHUB_ENV _CCACHE_DIR=$($(realpath $CCACHE_PATH/gcc) --show-config | sed -nE 's,.*cache_dir = ,,p')
echo >>$GITHUB_ENV NPROC=$(nproc | awk '{printf("%.0f",$1*0.77)}')
echo >>$GITHUB_ENV HOME=$HOME
- name: Restore cache CCache
uses: actions/cache@v2
with:
path: ${{ env._CCACHE_DIR }}
key: ${{runner.os}}-ccache-${{ github.event.pull_request.head.sha }}
restore-keys: ${{runner.os}}-ccache-
- run: ccache --show-stats | tee /tmp/ccache-stats
- ## Read the docs https://vcpkg.readthedocs.io/en/latest/users/binarycaching/ https://github.com/microsoft/vcpkg/blob/master/docs/users/binarycaching.md
name: Restore cache Vcpkg binarycache ## NOTE not useng NuGet because on ubuntu nuget needs mono of 433MB, unusable.
uses: actions/cache@v2
with:
path: |
${{env.HOME}}/.cache/vcpkg/archives
key: ${{runner.os}}-vcpkg-${{env.CC_NAME}}.${{ hashFiles('vcpkg/**') }}
restore-keys: ${{runner.os}}-vcpkg-${{env.CC_NAME}}.
- name: Build iroha vcpkg dependencies
run: ./vcpkg/build_iroha_deps.sh $PWD/vcpkg-build; test -f $PWD/vcpkg-build/scripts/buildsystems/vcpkg.cmake
## Takes 48m16s on default GitHub runner with 2 cores
## Takes 13m41s on self-hosted AWS EC2 c5.x4large
# ________________________________________________________
# Executed in 32,08 mins fish external
# usr time 110,52 mins 0,24 millis 110,52 mins
# sys time 12,26 mins 1,34 millis 12,26 mins
#
# All requested packages are currently installed.
# ________________________________________________________
# Executed in 3,17 secs fish external
# usr time 2,05 secs 128,00 micros 2,05 secs
# sys time 0,70 secs 575,00 micros 0,70 secs
- name: CMake configure
## Takes 13s on regular GitHub runner
run: cmake -B build -DCMAKE_TOOLCHAIN_FILE=$PWD/vcpkg-build/scripts/buildsystems/vcpkg.cmake -DCMAKE_BUILD_TYPE=${{ env.BuildType }} -GNinja $CMAKE_USE -DTESTING=$( test "$skip_testing" = skip_testing && echo OFF || echo ON ) -DBENCHMARKING=$( test "$skip_testing" = skip_testing && echo OFF || echo ON ) -DPACKAGE_DEB=ON
#-DCMAKE_VERBOSE_MAKEFILE=ON ## Note: use for debug
- name: CMake build
run: |
set -x
## reduce memory usage to do not overflow
cmake --build build --config ${{ env.BuildType }} -- -j$(nproc | awk '{printf("%.0f",$1*0.77)}')
echo ::notice::"$(./build/bin/irohad --version)"
## Debug takes 18m44s on regular GitHub runner
## Debug takes 7m41s on self-hosted AWS EC2 c5.x4large
## Release takes 2m58s on self-hosted AWS EC2 c5.x4large
- name: CPack (linux only)
run: cd build; cpack; ## cmake --build build --target package
- run: ccache --show-stats | diff --side-by-side /tmp/ccache-stats - ||true
- name: Show free space and disk usage
if: ${{ always() }}
run: |
df -h || true
- name: Generate artifact suffix depending on matrix to env.ARTIFACT_SUFFIX
run: |
set -x
cc=$(echo $CC_NAME | sed -Ee's,[-/],,g' )
build_type=$(echo $BuildType | tr A-Z a-z | sed -E -es,debug,dbg, -es,release,rel, )
test $build_type = dbg -o $build_type = rel
uses=$(echo "$features" | tr ' ' - | tr A-Z a-z)
_os=${OS:+-$OS} _cc=${cc:+-$cc} _build_type=${build_type:+-$build_type} _uses=${uses:+-$uses}
echo >>$GITHUB_ENV ARTIFACT_SUFFIX=$_os$_cc$_build_type$_uses
echo >>$GITHUB_ENV _uses_suffix=$_uses
echo >>$GITHUB_ENV _compiler_suffix=$(test $cc != gcc10 && echo $_cc)
echo >>$GITHUB_ENV _debug_suffix=$(test "$build_type" = dbg && echo -debug || true)
- name: Upload artifact irohad
uses: actions/upload-artifact@v2
with:
name: irohad${{env.ARTIFACT_SUFFIX}}
path: |
build/bin/irohad
build/bin/iroha-cli
- name: Upload artifact iroha-deb
uses: actions/upload-artifact@v2
with:
name: iroha-deb${{env.ARTIFACT_SUFFIX}}
path: |
build/*.deb
- if: ${{ false }} ## Maybe test in another job
name: Upload artifact tests
uses: actions/upload-artifact@v2
with:
name: iroha-tests-ubuntu${{env.ARTIFACT_SUFFIX}}
path: |
build/test_bin/**
build/test_data/**
- timeout-minutes: 40
if: env.skip_testing == ''
name: CTest
run: |
echo ::group::'boilerplate'
set -euo pipefail
if test $(uname) = Darwin ;then
## This is a common portable solution, but Debian and Ubuntu have their own wrappers
initdb --locale=C --encoding=UTF-8 --username=postgres $PWD/postgres_database
postgres -D $PWD/postgres_database -p5432 2>&1 >/tmp/postgres.log & { sleep .3; kill -0 $!; } ## use pg_ctl no need &
else ## Debian or Ubuntu
## Need to go debian-specific way because
## initdb is not allowed to be run as root, but we need to run as root
## because GitHub actions runners have much issues with permissions.
mkdir postgres_database && chown iroha-ci postgres_database
echo /usr/lib/postgresql/12/bin/initdb --locale=C --encoding=UTF-8 --username=postgres $PWD/postgres_database | su iroha-ci
echo /usr/lib/postgresql/12/bin/pg_ctl start -D $PWD/postgres_database --log=$PWD/postgres_database/log | su iroha-ci
# ## Need to go debian-specific way because
# ## initdb is not allowed to be run as root, but we need to run as root
# ## because GitHub actions runners have much issues with permissions.
# cat <<END >/etc/postgresql/12/main/pg_hba.conf
# # TYPE DATABASE USER ADDRESS METHOD
# local all all trust
# host all all 127.0.0.1/32 trust
# host all all ::1/128 trust
# local replication all trust
# host replication all 127.0.0.1/32 trust
# host replication all ::1/128 trust
# END
# pg_ctlcluster 12 main start ## Cluster 'main' exist by default
# #OR pg_createcluster -p 5432 --start 12 iroha -- --locale=C --encoding=UTF-8 --username=postgres
fi
cd build
## This is just a small lifehack to TEMPORARY allow some tests to fail
cat ../.github/TESTS_ALLOWED_TO_FAIL | sort -u >ALLOW_TO_FAIL || true
if test -e ALLOW_TO_FAIL
then echo "::warning:: There are TESTS_ALLOWED_TO_FAIL: "$(cat ALLOW_TO_FAIL)
fi
grep_failed_tests(){
grep 'The following tests FAILED:' -A10000 "$@" | tail +2 #| cut -d- -f2 | cut -d' ' -f2 | sort
}
exclude_allowed_to_fail(){
grep -Fvf ALLOW_TO_FAIL "$@"
}
only_allowed_to_fail(){
grep -Ff ALLOW_TO_FAIL "$@"
}
list_to_line(){
comma=''
while read N d name sta ;do
echo -n "$comma$N-$name$sta"
comma=', '
done
}
echo ::endgroup::
## Run module_* tests in parallel and others subsequently
## Cathgories sorted in order of importancy
CTEST_CATHEGORIES=( module tool framework regression system integration )
## Add rest available cathgories
CTEST_CATHEGORIES+=( $(
ctest --show-only=json-v1 -R "^(module|tool|framework|regression|system|integration)" |
jq -r .tests[].name |
cut -f1 -d_ |
sort -u |
grep -Fvf <( printf '%s\n' ${CTEST_CATHEGORIES[@]} )
)
) || true
CTEST_DEFAULT_timeout=80
CTEST_module_timeout=120 CTEST_module_parallel=4
CTEST_tool_timeout=200
CTEST_integration_timeout=120
CTEST_integration_args='--repeat until-pass:10' ## FIXME remove this hack
CTEST_system_args='--repeat until-pass:10' ## FIXME remove this hack
for cathegory in ${CTEST_CATHEGORIES[@]} ;do
echo >&2 ::group::"$cathegory tests"
set -x
timeout_name=CTEST_${cathegory}_timeout; timeout=${!timeout_name:-$CTEST_DEFAULT_timeout}
parallel_name=CTEST_${cathegory}_parallel; parallel=${!parallel_name:-}
args_name=CTEST_${cathegory}_args; args=${!args_name:-}
ctest -R "^${cathegory}_" ${parallel:+--parallel $parallel} --output-on-failure --no-tests=error --timeout $timeout ${args:-} \
| tee ctest_$cathegory.out \
|| true
set +x
echo >&2 ::endgroup::
done
tests_passed=true
for t in ${CTEST_CATHEGORIES[@]} ;do
f=ctest_$t.out
if a=$(grep_failed_tests $f | exclude_allowed_to_fail | list_to_line) ;then
echo "::error::The following $(echo $t | tr a-z A-Z) tests FAILED but not in list ALLOW_TO_FAIL: $a"
tests_passed=false
fi
if o=$(grep_failed_tests $f | only_allowed_to_fail | list_to_line) ;then
echo "::warning::The following $t tests FAILED and ALLOWED TO FAIL: $o"
fi
done
$tests_passed
strategy:
fail-fast: false
matrix: ${{ fromJSON( needs.generate_matrixes.outputs.matrix_ubuntu_release ) }}
if: ${{ fromJSON( needs.generate_matrixes.outputs.matrix_ubuntu_release ).include[0] }}
## Just to align picture
prepare-macos-env:
needs: check_if_pull_request_comes_from_fork
environment: test-env
runs-on: macos-latest
permissions: read-all
steps:
- name: Show context
run: |
echo $JSON_github | jq .
echo $JSON_needs | jq .
env:
JSON_github: ${{ toJSON(github) }}
JSON_needs: ${{ toJson(needs) }}
build-M:
needs:
- prepare-macos-env
- generate_matrixes
runs-on: macos-latest #[ self-hosted, MacOS ] #
permissions: read-all
strategy:
fail-fast: false
matrix: ${{ fromJSON( needs.generate_matrixes.outputs.matrix_macos ) }}
if: ${{ fromJSON( needs.generate_matrixes.outputs.matrix_macos ).include[0] }}
defaults:
run:
shell: bash
steps:
- name: Show context
run: |
echo $JSON_github | jq .
echo $JSON_needs | jq .
env:
JSON_github: ${{ toJSON(github) }}
JSON_needs: ${{ toJson(needs) }}
- name: System info
run: |
set -x
whoami
id $(whoami)
free || vm_stat | perl -ne '/page size of (\d+)/ and $size=$1;
/Pages\s+([^:]+)[^\d]+(\d+)/ and printf("%-16s % 16.2f Mi\n", "$1:", $2 * $size / 1048576);'
df -h
- name: Build info
run: |
cat << 'END'
ref:$github_ref
sha:${{github.sha}}
run_number:${{github.run_number}}
event_name:${{github.event_name}}
event.action:${{github.event.action}}
event.issue.number:${{ github.event.issue.number }}
END
env:
github_ref: ${{ github.ref }}
- name: export CC,BuildType from matrix.buildspec
run: |
echo >>$GITHUB_ENV OS=$(echo ${{matrix.buildspec}} | awk '{print $1}')
echo >>$GITHUB_ENV CC_NAME=$(echo ${{matrix.buildspec}} | awk '{print $2}')
echo >>$GITHUB_ENV BuildType=$(echo ${{matrix.buildspec}} | awk '{print $3}')
features=$(echo ${{matrix.buildspec}} | awk '{print $4}')
case $features in
normal) echo >>$GITHUB_ENV CMAKE_USE=""; features= ;;
ursa) echo >>$GITHUB_ENV CMAKE_USE="-DUSE_LIBURSA=ON";;
burrow) echo >>$GITHUB_ENV CMAKE_USE="-DUSE_BURROW=ON";;
*) echo "::error::Unknown features '$features'"; false ;;
esac
echo >>$GITHUB_ENV features="$features"
echo >>$GITHUB_ENV skip_testing=$(echo ${{matrix.buildspec}} | grep -Fo skip_testing)
- name: Homebrew
run: brew install cmake ninja coreutils ccache bash
## Takes 22 seconds with default github runner
- if: ${{ contains(env.CC_NAME, 'gcc-10') }}
name: Homebrew GCC
run: brew install gcc@10
- if: ${{ contains(env.CC_NAME, 'llvm') }}
name: Homebrew LLVM
run: brew install llvm
- if: ${{ contains(env.features, 'burrow') }}
name: Install protoc-gen-go for -DUSE_BURROW=ON
run: |
go get github.com/golang/protobuf/protoc-gen-go
echo >>$GITHUB_PATH $HOME/go/bin
- name: REF and SHA of commented PR to ENV
if: github.event.comment
run: >
curl -fsSL ${{github.event.issue.pull_request.url}} -H "Authorization: token ${{github.token}}" | jq -r '
"PR_REF="+.head.ref,
"PR_SHA="+.head.sha,
"PR_NUM="+(.number|tostring),
"PR_REPO="+.head.repo.full_name' >>$GITHUB_ENV
- name: Checkout head
uses: actions/checkout@v2
with:
ref: ${{ github.event.pull_request.head.sha }}
fetch-depth: 0 ## full history
- name: export CC and CXX
run: |
set -xeu #o pipefail
if test $CC_NAME = llvm
then CC=/usr/local/opt/llvm/bin/clang
else CC=$CC_NAME
fi
echo >>$GITHUB_ENV CC=$CC
echo >>$GITHUB_ENV CXX=$(echo $CC | sed -es,gcc,g++, -es,clang,clang++,)
echo >>$GITHUB_PATH $CCACHE_PATH
ls -lA $CCACHE_PATH
$(realpath $CCACHE_PATH/gcc) --show-config
echo >>$GITHUB_ENV _CCACHE_DIR=$($(realpath $CCACHE_PATH/gcc) --show-config | sed -nE 's,.*cache_dir = ,,p')
echo >>$GITHUB_ENV NPROC=$(nproc | awk '{printf("%.0f",$1*0.77)}')
echo >>$GITHUB_ENV HOME=$HOME
env:
CCACHE_PATH: /usr/local/opt/ccache/libexec
- name: Restore cache CCache
uses: actions/cache@v2
with:
path: ${{ env._CCACHE_DIR }}
key: ${{runner.os}}-ccache-${{ github.event.pull_request.head.sha }}
restore-keys: ${{runner.os}}-ccache-
- run: ccache --show-stats | tee /tmp/ccache-stats
- ## Read the docs https://vcpkg.readthedocs.io/en/latest/users/binarycaching/ https://github.com/microsoft/vcpkg/blob/master/docs/users/binarycaching.md
name: Restore cache Vcpkg binarycache ## NOTE not useng NuGet because on ubuntu nuget needs mono of 433MB, unusable.
uses: actions/cache@v2
with:
path: |
${{env.HOME}}/.cache/vcpkg/archives
key: ${{runner.os}}-vcpkg-${{env.CC_NAME}}.${{ hashFiles('vcpkg/**') }}
restore-keys: ${{runner.os}}-vcpkg-${{env.CC_NAME}}.
- name: Build iroha vcpkg dependencies
run: ./vcpkg/build_iroha_deps.sh $PWD/vcpkg-build; test -f $PWD/vcpkg-build/scripts/buildsystems/vcpkg.cmake
## Takes 48m16s on default GitHub runner with 2 cores
## Takes 13m41s on self-hosted AWS EC2 c5.x4large
# ________________________________________________________
# Executed in 32,08 mins fish external
# usr time 110,52 mins 0,24 millis 110,52 mins
# sys time 12,26 mins 1,34 millis 12,26 mins
#
# All requested packages are currently installed.
# ________________________________________________________
# Executed in 3,17 secs fish external
# usr time 2,05 secs 128,00 micros 2,05 secs
# sys time 0,70 secs 575,00 micros 0,70 secs
- name: CMake configure
## Takes 13s on regular GitHub runner
run: cmake -B build -DCMAKE_TOOLCHAIN_FILE=$PWD/vcpkg-build/scripts/buildsystems/vcpkg.cmake -DCMAKE_BUILD_TYPE=${{ env.BuildType }} -GNinja $CMAKE_USE -DTESTING=$( test "$skip_testing" = skip_testing && echo OFF || echo ON ) -DBENCHMARKING=$( test "$skip_testing" = skip_testing && echo OFF || echo ON ) -DPACKAGE_DEB=ON
#-DCMAKE_VERBOSE_MAKEFILE=ON ## Note: use for debug
- name: CMake build
run: |
set -x
## reduce memory usage to do not overflow
cmake --build build --config ${{ env.BuildType }} -- -j$(nproc | awk '{printf("%.0f",$1*0.77)}')
echo ::notice::"$(./build/bin/irohad --version)"
## Debug takes 18m44s on regular GitHub runner
## Debug takes 7m41s on self-hosted AWS EC2 c5.x4large
## Release takes 2m58s on self-hosted AWS EC2 c5.x4large
- run: ccache --show-stats | diff --side-by-side /tmp/ccache-stats - ||true
- name: Show free space and disk usage
if: ${{ always() }}
run: |
df -h || true
- name: Generate artifact suffix depending on matrix to env.ARTIFACT_SUFFIX
run: |
set -x
cc=$(echo $CC_NAME | sed -Ee's,[-/],,g' )
build_type=$(echo $BuildType | tr A-Z a-z | sed -E -es,debug,dbg, -es,release,rel, )
test $build_type = dbg -o $build_type = rel
uses=$(echo "$features" | tr ' ' - | tr A-Z a-z)
_os=${OS:+-$OS} _cc=${cc:+-$cc} _build_type=${build_type:+-$build_type} _uses=${uses:+-$uses}
echo >>$GITHUB_ENV ARTIFACT_SUFFIX=$_os$_cc$_build_type$_uses
echo >>$GITHUB_ENV _uses_suffix=$_uses
echo >>$GITHUB_ENV _compiler_suffix=$(test $cc != gcc10 && echo $_cc)
echo >>$GITHUB_ENV _debug_suffix=$(test "$build_type" = dbg && echo -debug || true)
- name: Upload artifact irohad
uses: actions/upload-artifact@v2
with:
name: irohad-macos${{env.ARTIFACT_SUFFIX}}
path: |
build/bin/irohad
build/bin/iroha-cli
- if: ${{ false }} ## Maybe test in another job
name: Upload artifact tests
uses: actions/upload-artifact@v2
with:
name: iroha-tests-ubuntu${{env.ARTIFACT_SUFFIX}}
path: |
build/test_bin/**
build/test_data/**
- name: Install Postgres on MacOS
run: brew install postgresql
- if: env.skip_testing == ''
name: CTest
run: |
echo ::group::'boilerplate'
set -euo pipefail
if test $(uname) = Darwin ;then
## This is a common portable solution, but Debian and Ubuntu have their own wrappers
initdb --locale=C --encoding=UTF-8 --username=postgres $PWD/postgres_database
postgres -D $PWD/postgres_database -p5432 2>&1 >/tmp/postgres.log & { sleep .3; kill -0 $!; } ## use pg_ctl no need &
else ## Debian or Ubuntu
## Need to go debian-specific way because
## initdb is not allowed to be run as root, but we need to run as root
## because GitHub actions runners have much issues with permissions.
mkdir postgres_database && chown iroha-ci postgres_database
echo /usr/lib/postgresql/12/bin/initdb --locale=C --encoding=UTF-8 --username=postgres $PWD/postgres_database | su iroha-ci
echo /usr/lib/postgresql/12/bin/pg_ctl start -D $PWD/postgres_database --log=$PWD/postgres_database/log | su iroha-ci
# ## Need to go debian-specific way because
# ## initdb is not allowed to be run as root, but we need to run as root
# ## because GitHub actions runners have much issues with permissions.
# cat <<END >/etc/postgresql/12/main/pg_hba.conf
# # TYPE DATABASE USER ADDRESS METHOD
# local all all trust
# host all all 127.0.0.1/32 trust
# host all all ::1/128 trust
# local replication all trust
# host replication all 127.0.0.1/32 trust
# host replication all ::1/128 trust
# END
# pg_ctlcluster 12 main start ## Cluster 'main' exist by default
# #OR pg_createcluster -p 5432 --start 12 iroha -- --locale=C --encoding=UTF-8 --username=postgres
fi
cd build
## This is just a small lifehack to TEMPORARY allow some tests to fail
cat ../.github/TESTS_ALLOWED_TO_FAIL | sort -u >ALLOW_TO_FAIL || true
if test -e ALLOW_TO_FAIL
then echo "::warning:: There are TESTS_ALLOWED_TO_FAIL: "$(cat ALLOW_TO_FAIL)
fi
grep_failed_tests(){
grep 'The following tests FAILED:' -A10000 "$@" | tail +2 #| cut -d- -f2 | cut -d' ' -f2 | sort
}
exclude_allowed_to_fail(){
grep -Fvf ALLOW_TO_FAIL "$@"
}
only_allowed_to_fail(){
grep -Ff ALLOW_TO_FAIL "$@"
}
list_to_line(){
comma=''
while read N d name sta ;do
echo -n "$comma$N-$name$sta"
comma=', '
done
}
echo ::endgroup::
## Run module_* tests in parallel and others subsequently
## Cathgories sorted in order of importancy
CTEST_CATHEGORIES=( module tool framework regression system integration )
## Add rest available cathgories
CTEST_CATHEGORIES+=( $(
ctest --show-only=json-v1 -R "^(module|tool|framework|regression|system|integration)" |
jq -r .tests[].name |
cut -f1 -d_ |
sort -u |
grep -Fvf <( printf '%s\n' ${CTEST_CATHEGORIES[@]} )
)
) || true
CTEST_DEFAULT_timeout=80
CTEST_module_timeout=120 CTEST_module_parallel=4
CTEST_tool_timeout=200
CTEST_integration_timeout=120
CTEST_integration_args='--repeat until-pass:10' ## FIXME remove this hack
CTEST_system_args='--repeat until-pass:10' ## FIXME remove this hack
for cathegory in ${CTEST_CATHEGORIES[@]} ;do
echo >&2 ::group::"$cathegory tests"
set -x
timeout_name=CTEST_${cathegory}_timeout; timeout=${!timeout_name:-$CTEST_DEFAULT_timeout}
parallel_name=CTEST_${cathegory}_parallel; parallel=${!parallel_name:-}
args_name=CTEST_${cathegory}_args; args=${!args_name:-}
ctest -R "^${cathegory}_" ${parallel:+--parallel $parallel} --output-on-failure --no-tests=error --timeout $timeout ${args:-} \
| tee ctest_$cathegory.out \
|| true
set +x
echo >&2 ::endgroup::
done
tests_passed=true
for t in ${CTEST_CATHEGORIES[@]} ;do
f=ctest_$t.out
if a=$(grep_failed_tests $f | exclude_allowed_to_fail | list_to_line) ;then
echo "::error::The following $(echo $t | tr a-z A-Z) tests FAILED but not in list ALLOW_TO_FAIL: $a"
tests_passed=false
fi
if o=$(grep_failed_tests $f | only_allowed_to_fail | list_to_line) ;then
echo "::warning::The following $t tests FAILED and ALLOWED TO FAIL: $o"
fi
done
$tests_passed
timeout-minutes: 70
## Just to align picture
prepare-windows-env:
needs: check_if_pull_request_comes_from_fork
environment: test-env
runs-on: windows-latest
permissions: read-all
steps:
- name: Show context
run: |
echo $JSON_github | jq .
echo $JSON_needs | jq .
env:
JSON_github: ${{ toJSON(github) }}
JSON_needs: ${{ toJson(needs) }}
defaults:
run:
shell: bash
build-W:
needs:
- prepare-windows-env
- generate_matrixes
runs-on: windows-latest
permissions: read-all
strategy:
fail-fast: false
matrix: ${{ fromJSON( needs.generate_matrixes.outputs.matrix_windows ) }}
if: ${{ false && ( fromJSON( needs.generate_matrixes.outputs.matrix_windows ).include[0] ) }}
defaults:
run:
shell: bash #pwsh
working-directory: 'C:\github\iroha' ## Use disk C: because D: is out of space
steps:
- name: Create working-directory, export WORKDIR
run: |
set -x
mkdir -p "$WORKDIR"
echo $PWD
echo >>$GITHUB_ENV WORKDIR="$WORKDIR"
working-directory: 'C:\'
env: {WORKDIR: 'C:\github\iroha'}
- name: uname in bash
run: uname
shell: bash
- name: uname in [default] pwsh shell
run: uname
shell: pwsh
- name: Chocolatey install
run: choco install cmake ninja #ccache
- name: Checkout head
uses: actions/checkout@v2
with:
ref: ${{ github.event.pull_request.head.sha }}
fetch-depth: 0 ## full history
- name: move to workdir
run: |
set -x
echo $PWD
shopt -s dotglob nullglob
mv -vf * -t "$WORKDIR"
working-directory:
#- *step_restore_ccache
#- *step_vcpkg_cache
- name: Build iroha vcpkg dependencies
run: ./vcpkg/build_iroha_deps.sh $PWD/vcpkg-build; test -f $PWD/vcpkg-build/scripts/buildsystems/vcpkg.cmake
## Takes 48m16s on default GitHub runner with 2 cores
## Takes 13m41s on self-hosted AWS EC2 c5.x4large
# ________________________________________________________
# Executed in 32,08 mins fish external
# usr time 110,52 mins 0,24 millis 110,52 mins
# sys time 12,26 mins 1,34 millis 12,26 mins
#
# All requested packages are currently installed.
# ________________________________________________________
# Executed in 3,17 secs fish external
# usr time 2,05 secs 128,00 micros 2,05 secs
# sys time 0,70 secs 575,00 micros 0,70 secs
- name: CMake configure
## Takes 13s on regular GitHub runner
run: cmake -B build -DCMAKE_TOOLCHAIN_FILE=$PWD/vcpkg-build/scripts/buildsystems/vcpkg.cmake -DCMAKE_BUILD_TYPE=${{ env.BuildType }} -GNinja $CMAKE_USE -DTESTING=$( test "$skip_testing" = skip_testing && echo OFF || echo ON ) -DBENCHMARKING=$( test "$skip_testing" = skip_testing && echo OFF || echo ON ) -DPACKAGE_DEB=ON
#-DCMAKE_VERBOSE_MAKEFILE=ON ## Note: use for debug
- name: CMake build
run: |
set -x
## reduce memory usage to do not overflow
cmake --build build --config ${{ env.BuildType }} -- -j$(nproc | awk '{printf("%.0f",$1*0.77)}')
echo ::notice::"$(./build/bin/irohad --version)"
## Debug takes 18m44s on regular GitHub runner
## Debug takes 7m41s on self-hosted AWS EC2 c5.x4large
## Release takes 2m58s on self-hosted AWS EC2 c5.x4large
- name: Show free space and disk usage
if: ${{ always() }}
run: |
df -h || true
- name: Install Postgres on Windows
run: choco install postgresql
# - *step_ctest
## Build and publish docker image named 'hyperledger/iroha' with irohad and iroha tools inside.
## The result docker image is pushed with tags :branch_name, :pr-NUMBER, :tag_name,
## and :latest (for git-tags).
## Those docker image tags could be extended with suffixes with compiler and build type like
## -gcc10, -clang, -debug, -gcc10-debug. Like :latest-debug, :main-debug, :1.3.0-gcc10-debug.
## Result image name could look like: hyperledger/iroha:pr-1117, hyperledger/iroha:nightly.
docker-R:
needs:
- build-UR
- generate_matrixes
runs-on: [self-hosted, Linux, iroha] #ubuntu-latest
# strategy: *strategy_ubuntu_release
# if: *if_ubuntu_release
strategy:
fail-fast: false
matrix: ${{ fromJSON( needs.generate_matrixes.outputs.matrix_dockerimage_release ) }}
if: ${{ fromJSON( needs.generate_matrixes.outputs.matrix_dockerimage_release ).include[0] }}
env:
IMAGE_NAME: iroha
steps:
- name: Show context
run: |
echo $JSON_github | jq .
echo $JSON_needs | jq .
env:
JSON_github: ${{ toJSON(github) }}
JSON_needs: ${{ toJson(needs) }}
- name: System info
run: |
set -x
whoami
id $(whoami)
free || vm_stat | perl -ne '/page size of (\d+)/ and $size=$1;
/Pages\s+([^:]+)[^\d]+(\d+)/ and printf("%-16s % 16.2f Mi\n", "$1:", $2 * $size / 1048576);'
df -h
- name: Build info
run: |
cat << 'END'
ref:$github_ref
sha:${{github.sha}}
run_number:${{github.run_number}}
event_name:${{github.event_name}}
event.action:${{github.event.action}}
event.issue.number:${{ github.event.issue.number }}
END
env:
github_ref: ${{ github.ref }}
- name: export CC,BuildType from matrix.buildspec
run: |
echo >>$GITHUB_ENV OS=$(echo ${{matrix.buildspec}} | awk '{print $1}')
echo >>$GITHUB_ENV CC_NAME=$(echo ${{matrix.buildspec}} | awk '{print $2}')
echo >>$GITHUB_ENV BuildType=$(echo ${{matrix.buildspec}} | awk '{print $3}')
features=$(echo ${{matrix.buildspec}} | awk '{print $4}')
case $features in
normal) echo >>$GITHUB_ENV CMAKE_USE=""; features= ;;
ursa) echo >>$GITHUB_ENV CMAKE_USE="-DUSE_LIBURSA=ON";;
burrow) echo >>$GITHUB_ENV CMAKE_USE="-DUSE_BURROW=ON";;
*) echo "::error::Unknown features '$features'"; false ;;
esac
echo >>$GITHUB_ENV features="$features"
echo >>$GITHUB_ENV skip_testing=$(echo ${{matrix.buildspec}} | grep -Fo skip_testing)
- name: REF and SHA of commented PR to ENV
if: github.event.comment
run: >
curl -fsSL ${{github.event.issue.pull_request.url}} -H "Authorization: token ${{github.token}}" | jq -r '
"PR_REF="+.head.ref,
"PR_SHA="+.head.sha,
"PR_NUM="+(.number|tostring),
"PR_REPO="+.head.repo.full_name' >>$GITHUB_ENV
- name: Checkout base
uses: actions/checkout@v2
with:
ref: ${{ github.event.pull_request.base.sha }}
- name: Generate artifact suffix depending on matrix to env.ARTIFACT_SUFFIX
run: |
set -x
cc=$(echo $CC_NAME | sed -Ee's,[-/],,g' )
build_type=$(echo $BuildType | tr A-Z a-z | sed -E -es,debug,dbg, -es,release,rel, )
test $build_type = dbg -o $build_type = rel
uses=$(echo "$features" | tr ' ' - | tr A-Z a-z)
_os=${OS:+-$OS} _cc=${cc:+-$cc} _build_type=${build_type:+-$build_type} _uses=${uses:+-$uses}
echo >>$GITHUB_ENV ARTIFACT_SUFFIX=$_os$_cc$_build_type$_uses
echo >>$GITHUB_ENV _uses_suffix=$_uses
echo >>$GITHUB_ENV _compiler_suffix=$(test $cc != gcc10 && echo $_cc)
echo >>$GITHUB_ENV _debug_suffix=$(test "$build_type" = dbg && echo -debug || true)
- name: Download artifact
uses: actions/download-artifact@v2
with:
name: iroha-deb${{env.ARTIFACT_SUFFIX}}
- name: Rename artifact debs
run: |
mv *iroha_shepherd.deb docker/release/iroha_shepherd.deb
mv *irohad.deb docker/release/iroha.deb
- name: Determine dockertag
id: dockertag
run: |
echo "::set-output name=dockertag::$dockertag"
echo >>$GITHUB_ENV dockertag=$dockertag
test -n "$DOCKERHUB_ORG" || {
echo ::error::"DOCKERHUB_ORG must contain value"
false
}
env:
dockertag: ${{ hashFiles('docker/release/**') }}
- name: Docker meta
id: meta
uses: docker/metadata-action@v3
with:
images: |
${{ env.DOCKERHUB_ORG }}/${{ env.IMAGE_NAME }}${{ env._uses_suffix }}
tags: |
type=ref,event=branch
type=ref,event=pr
type=ref,event=tag
type=schedule
flavor: |
suffix=${{env._compiler_suffix}}${{env._debug_suffix}},onlatest=true
#maybetodo flavor: prefix=${{ env.USES_PREFIX }} ## In case creating repository hyperledger/iroha-burrow denied, Use tag prefix hyperledger/iroha:burrow-xxxx
- uses: docker/metadata-action@v3
name: Docker meta GHCR
id: meta_ghcr
with:
tags: |
type=ref,event=branch
type=ref,event=pr
type=ref,event=tag
type=schedule
flavor: |
suffix=${{env._compiler_suffix}}${{env._debug_suffix}},onlatest=true
#maybetodo flavor: prefix=${{ env.USES_PREFIX }} ## In case creating repository hyperledger/iroha-burrow denied, Use tag prefix hyperledger/iroha:burrow-xxxx
images: ghcr.io/${{ github.repository }}${{ env._uses_suffix }}
- name: Login to DockerHub
#if: ${{ secrets.DOCKERHUB_TOKEN != '' && secrets.DOCKERHUB_USERNAME != '' }}
id: docker_login
uses: docker/login-action@v1
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Log in to the Container registry GHCR
uses: docker/login-action@v1
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Possible WARNING
if: ${{ steps.docker_login.outcome == 'skipped' }}
run: echo "::warning::DOCKERHUB_TOKEN and DOCKERHUB_USERNAME are empty. Will build but NOT push."
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
- name: Cache Docker layers
uses: actions/cache@v2
with:
path: /tmp/.buildx-cache
key: ${{ runner.os }}-buildx-release-${{env.dockertag}}
restore-keys: ${{ runner.os }}-buildx-release
- id: build_and_push
name: Build and push
uses: docker/build-push-action@v2
with:
cache-from: type=local,src=/tmp/.buildx-cache
cache-to: type=local,dest=/tmp/.buildx-cache-new
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
context: docker/release/
push: ${{ steps.docker_login.outcome == 'success' }}
- uses: docker/build-push-action@v2
id: build_and_push_ghcr
name: Build and push to GHCR
with:
cache-from: type=local,src=/tmp/.buildx-cache
cache-to: type=local,dest=/tmp/.buildx-cache-new
push: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo == github.event.pull_request.base.repo }}
tags: ${{ steps.meta_ghcr.outputs.tags }}
labels: ${{ steps.meta_ghcr.outputs.labels }}
context: docker/release/
- # Temp fix
# https://github.com/docker/build-push-action/issues/252
# https://github.com/moby/buildkit/issues/1896
name: Move cache
run: |
rm -rf /tmp/.buildx-cache
mv /tmp/.buildx-cache-new /tmp/.buildx-cache
docker-D:
runs-on: [self-hosted, Linux, iroha] #ubuntu-latest
env:
IMAGE_NAME: iroha
steps:
- name: Show context
run: |
echo $JSON_github | jq .
echo $JSON_needs | jq .
env:
JSON_github: ${{ toJSON(github) }}
JSON_needs: ${{ toJson(needs) }}
- name: System info
run: |
set -x
whoami
id $(whoami)
free || vm_stat | perl -ne '/page size of (\d+)/ and $size=$1;
/Pages\s+([^:]+)[^\d]+(\d+)/ and printf("%-16s % 16.2f Mi\n", "$1:", $2 * $size / 1048576);'
df -h
- name: Build info
run: |
cat << 'END'
ref:$github_ref
sha:${{github.sha}}
run_number:${{github.run_number}}
event_name:${{github.event_name}}
event.action:${{github.event.action}}
event.issue.number:${{ github.event.issue.number }}
END
env:
github_ref: ${{ github.ref }}
- name: export CC,BuildType from matrix.buildspec
run: |
echo >>$GITHUB_ENV OS=$(echo ${{matrix.buildspec}} | awk '{print $1}')
echo >>$GITHUB_ENV CC_NAME=$(echo ${{matrix.buildspec}} | awk '{print $2}')
echo >>$GITHUB_ENV BuildType=$(echo ${{matrix.buildspec}} | awk '{print $3}')
features=$(echo ${{matrix.buildspec}} | awk '{print $4}')
case $features in
normal) echo >>$GITHUB_ENV CMAKE_USE=""; features= ;;
ursa) echo >>$GITHUB_ENV CMAKE_USE="-DUSE_LIBURSA=ON";;
burrow) echo >>$GITHUB_ENV CMAKE_USE="-DUSE_BURROW=ON";;
*) echo "::error::Unknown features '$features'"; false ;;
esac
echo >>$GITHUB_ENV features="$features"
echo >>$GITHUB_ENV skip_testing=$(echo ${{matrix.buildspec}} | grep -Fo skip_testing)
- name: REF and SHA of commented PR to ENV
if: github.event.comment
run: >
curl -fsSL ${{github.event.issue.pull_request.url}} -H "Authorization: token ${{github.token}}" | jq -r '
"PR_REF="+.head.ref,
"PR_SHA="+.head.sha,
"PR_NUM="+(.number|tostring),
"PR_REPO="+.head.repo.full_name' >>$GITHUB_ENV
- name: Checkout base
uses: actions/checkout@v2
with:
ref: ${{ github.event.pull_request.base.sha }}
- name: Generate artifact suffix depending on matrix to env.ARTIFACT_SUFFIX
run: |
set -x
cc=$(echo $CC_NAME | sed -Ee's,[-/],,g' )
build_type=$(echo $BuildType | tr A-Z a-z | sed -E -es,debug,dbg, -es,release,rel, )
test $build_type = dbg -o $build_type = rel
uses=$(echo "$features" | tr ' ' - | tr A-Z a-z)
_os=${OS:+-$OS} _cc=${cc:+-$cc} _build_type=${build_type:+-$build_type} _uses=${uses:+-$uses}
echo >>$GITHUB_ENV ARTIFACT_SUFFIX=$_os$_cc$_build_type$_uses
echo >>$GITHUB_ENV _uses_suffix=$_uses
echo >>$GITHUB_ENV _compiler_suffix=$(test $cc != gcc10 && echo $_cc)
echo >>$GITHUB_ENV _debug_suffix=$(test "$build_type" = dbg && echo -debug || true)
- name: Download artifact
uses: actions/download-artifact@v2
with:
name: iroha-deb${{env.ARTIFACT_SUFFIX}}
- name: Rename artifact debs
run: |
mv *iroha_shepherd.deb docker/release/iroha_shepherd.deb
mv *irohad.deb docker/release/iroha.deb
- name: Determine dockertag
id: dockertag
run: |
echo "::set-output name=dockertag::$dockertag"
echo >>$GITHUB_ENV dockertag=$dockertag
test -n "$DOCKERHUB_ORG" || {
echo ::error::"DOCKERHUB_ORG must contain value"
false
}
env:
dockertag: ${{ hashFiles('docker/release/**') }}
- name: Docker meta
id: meta
uses: docker/metadata-action@v3
with:
images: |
${{ env.DOCKERHUB_ORG }}/${{ env.IMAGE_NAME }}${{ env._uses_suffix }}
tags: |
type=ref,event=branch
type=ref,event=pr
type=ref,event=tag
type=schedule
flavor: |
suffix=${{env._compiler_suffix}}${{env._debug_suffix}},onlatest=true
#maybetodo flavor: prefix=${{ env.USES_PREFIX }} ## In case creating repository hyperledger/iroha-burrow denied, Use tag prefix hyperledger/iroha:burrow-xxxx
- uses: docker/metadata-action@v3
name: Docker meta GHCR
id: meta_ghcr
with:
tags: |
type=ref,event=branch
type=ref,event=pr
type=ref,event=tag
type=schedule
flavor: |
suffix=${{env._compiler_suffix}}${{env._debug_suffix}},onlatest=true
#maybetodo flavor: prefix=${{ env.USES_PREFIX }} ## In case creating repository hyperledger/iroha-burrow denied, Use tag prefix hyperledger/iroha:burrow-xxxx
images: ghcr.io/${{ github.repository }}${{ env._uses_suffix }}
- name: Login to DockerHub
#if: ${{ secrets.DOCKERHUB_TOKEN != '' && secrets.DOCKERHUB_USERNAME != '' }}
id: docker_login
uses: docker/login-action@v1
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Log in to the Container registry GHCR
uses: docker/login-action@v1
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Possible WARNING
if: ${{ steps.docker_login.outcome == 'skipped' }}
run: echo "::warning::DOCKERHUB_TOKEN and DOCKERHUB_USERNAME are empty. Will build but NOT push."
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
- name: Cache Docker layers
uses: actions/cache@v2
with:
path: /tmp/.buildx-cache
key: ${{ runner.os }}-buildx-release-${{env.dockertag}}
restore-keys: ${{ runner.os }}-buildx-release
- id: build_and_push
name: Build and push
uses: docker/build-push-action@v2
with:
cache-from: type=local,src=/tmp/.buildx-cache
cache-to: type=local,dest=/tmp/.buildx-cache-new
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
context: docker/release/
push: ${{ steps.docker_login.outcome == 'success' }}
- uses: docker/build-push-action@v2
id: build_and_push_ghcr
name: Build and push to GHCR
with:
cache-from: type=local,src=/tmp/.buildx-cache
cache-to: type=local,dest=/tmp/.buildx-cache-new
push: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo == github.event.pull_request.base.repo }}
tags: ${{ steps.meta_ghcr.outputs.tags }}
labels: ${{ steps.meta_ghcr.outputs.labels }}
context: docker/release/
- # Temp fix
# https://github.com/docker/build-push-action/issues/252
# https://github.com/moby/buildkit/issues/1896
name: Move cache
run: |
rm -rf /tmp/.buildx-cache
mv /tmp/.buildx-cache-new /tmp/.buildx-cache
needs:
- build-UD
- generate_matrixes
strategy:
fail-fast: false
matrix: ${{ fromJSON( needs.generate_matrixes.outputs.matrix_dockerimage_debug ) }}
if: ${{ fromJSON( needs.generate_matrixes.outputs.matrix_dockerimage_debug ).include[0] }}