From a8330e697d77c259801574c4c9a0b5360c901c13 Mon Sep 17 00:00:00 2001 From: Denys Gonchar Date: Tue, 15 Dec 2020 21:56:51 +0100 Subject: [PATCH 01/19] Creating ci.yml workflow with small_tests job --- .github/workflows/ci.yml | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000000..83b1b2f75e0 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,31 @@ +name: CI + +on: + push: + branches: [ master ] + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +env: + # required for run_common_test.erl + KEEP_COVER_RUNNING: '1' + SKIP_AUTO_COMPILE: 'true' + +jobs: + small_tests: + name: small_tests on OTP ${{matrix.otp}} + strategy: + matrix: + otp: ['23.0.3', '22.3'] + runs-on: ubuntu-latest + env: + PRESET: 'small_tests' + steps: + - uses: actions/checkout@v2 + - uses: ErlGang/setup-erlang@v1.0.0 + with: + otp-version: ${{ matrix.otp }} + - name: create certificates + run: make certs + - name: run test + run: tools/travis-test.sh -p $PRESET -s true -e true From 79937fd02613e9e07741bce56561f1c1f65c0908 Mon Sep 17 00:00:00 2001 From: Denys Gonchar Date: Wed, 16 Dec 2020 00:28:03 +0100 Subject: [PATCH 02/19] adding rebar dependencies caching --- .github/workflows/ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 83b1b2f75e0..815a530a95d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,6 +25,10 @@ jobs: - uses: ErlGang/setup-erlang@v1.0.0 with: otp-version: ${{ matrix.otp }} + - uses: actions/cache@v2 + with: + path: ~/.cache/rebar3 + key: rebar3-${{ hashFiles('rebar.lock') }} - name: create certificates run: make certs - name: run test From c38541c2635b34b14a1ac75eedac7d565f55118e Mon Sep 17 00:00:00 2001 From: Denys Gonchar Date: Wed, 16 Dec 2020 01:33:20 +0100 Subject: [PATCH 03/19] adding internal_mnesia job --- .github/workflows/ci.yml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 815a530a95d..7fe8c3dff55 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,3 +33,31 @@ jobs: run: make certs - name: run test run: tools/travis-test.sh -p $PRESET -s true -e true + + internal_mnesia: + name: internal_mnesia on OTP 23.0.3 + runs-on: ubuntu-latest + env: + PRESET: 'internal_mnesia' + DB: 'mnesia minio redis' + REL_CONFIG: 'with-all' + TLS_DIST: 'true' + steps: + - uses: actions/checkout@v2 + - uses: ErlGang/setup-erlang@v1.0.0 + with: + otp-version: '23.0.3' + - uses: actions/cache@v2 + with: + path: ~/.cache/rebar3 + key: rebar3-${{ hashFiles('rebar.lock') }} + - name: create certificates + run: make certs + - name: build releases + run: tools/build-releases.sh + - name: build big tests + run: tools/travis-build-tests.sh + - name: start backends + run: tools/travis-setup-db.sh + - name: run test + run: tools/travis-test.sh -p $PRESET -s false -e false From 28e2a72e18dc7bae9f64f73aca2b456656a551c3 Mon Sep 17 00:00:00 2001 From: DenysGonchar Date: Wed, 16 Dec 2020 15:13:54 +0100 Subject: [PATCH 04/19] temporary enabling CI workflow execution on PR to master branch --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7fe8c3dff55..49604c668b8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,6 +3,8 @@ name: CI on: push: branches: [ master ] + pull_request: + branches: [ master ] # Allows you to run this workflow manually from the Actions tab workflow_dispatch: From 7a19893eec446a0cb9ba84da74052f0bbf37226c Mon Sep 17 00:00:00 2001 From: DenysGonchar Date: Thu, 17 Dec 2020 14:09:26 +0100 Subject: [PATCH 05/19] renaming internal_mnesia job into big_tests and making it more generic --- .github/workflows/ci.yml | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 49604c668b8..e959b570c39 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,25 +36,31 @@ jobs: - name: run test run: tools/travis-test.sh -p $PRESET -s true -e true - internal_mnesia: - name: internal_mnesia on OTP 23.0.3 + big_tests: + name: ${{matrix.preset}} on OTP ${{matrix.otp}} + strategy: + matrix: + preset: [internal_mnesia] + otp: ['23.0.3'] runs-on: ubuntu-latest env: - PRESET: 'internal_mnesia' - DB: 'mnesia minio redis' - REL_CONFIG: 'with-all' - TLS_DIST: 'true' + PRESET: ${{matrix.preset}} steps: - uses: actions/checkout@v2 - uses: ErlGang/setup-erlang@v1.0.0 with: - otp-version: '23.0.3' + otp-version: ${{matrix.otp}} - uses: actions/cache@v2 with: path: ~/.cache/rebar3 key: rebar3-${{ hashFiles('rebar.lock') }} - name: create certificates run: make certs + - if: ${{ matrix.preset == 'internal_mnesia' }} + run: + echo "DB=mnesia minio redis" >> $GITHUB_ENV + echo "REL_CONFIG=with-all" >> $GITHUB_ENV + echo "TLS_DIST=true" >> $GITHUB_ENV - name: build releases run: tools/build-releases.sh - name: build big tests From a2cb4721ae3193716bd7a62b3fa91855fee22873 Mon Sep 17 00:00:00 2001 From: DenysGonchar Date: Fri, 18 Dec 2020 12:15:30 +0100 Subject: [PATCH 06/19] adding coveralls reporting --- .github/workflows/ci.yml | 13 +++++++++++++ rebar.config | 3 +++ rebar.config.script | 22 +++++++++++++++++++++- 3 files changed, 37 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e959b570c39..9f832fa1deb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -69,3 +69,16 @@ jobs: run: tools/travis-setup-db.sh - name: run test run: tools/travis-test.sh -p $PRESET -s false -e false + - name: report coverage + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DEBUG: '1' + run: | + ls -ahl _build/test/cover || true + cp _build/test/cover/*.coverdata /tmp || true + ls -ahl /tmp/*.coverdata || true + ./rebar3 coveralls send || { + echo --rebar3.crashdump-- + cat rebar3.crashdump + exit 1 + } diff --git a/rebar.config b/rebar.config index e061e40514f..b8ec62bc5ee 100644 --- a/rebar.config +++ b/rebar.config @@ -154,6 +154,7 @@ {pc, "1.11.0"}, {provider_asn1, "0.2.3"}, {rebar3_codecov, {git, "https://github.com/esl/rebar3_codecov.git", {ref, "6bd31cc"}}}, + {coveralls, "2.2.0"}, {rebar3_lint, "0.1.11"} ]}. @@ -184,6 +185,8 @@ {cover_enabled, true}. {cover_print_enabled, true}. {cover_export_enabled, true}. +{coveralls_coverdata, ["/tmp/*.coverdata", "_build/test/cover/*.coverdata"]}. +{coveralls_service_name, "gh-actions"}. {codecov_opts, [ diff --git a/rebar.config.script b/rebar.config.script index b0ec4af566f..ec654e756f2 100644 --- a/rebar.config.script +++ b/rebar.config.script @@ -84,5 +84,25 @@ MaybeFIPSSupport = fun(Config) -> end end, -Config1 = SetupIncludedApps(CONFIG, EnvApps), +MaybeConfigureCoveralls = fun(CONFIG) -> + case {os:getenv("GITHUB_ACTIONS"), os:getenv("GITHUB_TOKEN")} of + {"true", Token} when is_list(Token) -> + CONFIG1 = [{coveralls_repo_token, Token}, + {coveralls_service_job_id, os:getenv("GITHUB_RUN_ID")}, + {coveralls_commit_sha, os:getenv("GITHUB_SHA")}, + {coveralls_service_number, os:getenv("GITHUB_RUN_NUMBER")} | CONFIG], + case os:getenv("GITHUB_EVENT_NAME") =:= "pull_request" andalso + string:tokens(os:getenv("GITHUB_REF"), "/") of + [_, "pull", PRNO, _] -> + [{coveralls_service_pull_request, PRNO} | CONFIG1]; + _ -> + CONFIG1 + end; + _ -> + CONFIG + end +end, + +Config0 = MaybeConfigureCoveralls(CONFIG), +Config1 = SetupIncludedApps(Config0, EnvApps), MaybeFIPSSupport(Config1). From 3640ae8ae614b7368ab041bdb8503e328d6b7c11 Mon Sep 17 00:00:00 2001 From: DenysGonchar Date: Fri, 18 Dec 2020 16:48:16 +0100 Subject: [PATCH 07/19] fix coveralls_coverdata configuration --- .github/workflows/ci.yml | 4 +++- rebar.config | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9f832fa1deb..3394bf8beaa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,7 +62,9 @@ jobs: echo "REL_CONFIG=with-all" >> $GITHUB_ENV echo "TLS_DIST=true" >> $GITHUB_ENV - name: build releases - run: tools/build-releases.sh + run: | + ./rebar3 compile + tools/build-releases.sh - name: build big tests run: tools/travis-build-tests.sh - name: start backends diff --git a/rebar.config b/rebar.config index b8ec62bc5ee..1c790f4bcf8 100644 --- a/rebar.config +++ b/rebar.config @@ -185,8 +185,8 @@ {cover_enabled, true}. {cover_print_enabled, true}. {cover_export_enabled, true}. -{coveralls_coverdata, ["/tmp/*.coverdata", "_build/test/cover/*.coverdata"]}. -{coveralls_service_name, "gh-actions"}. +{coveralls_coverdata, "/tmp/*.coverdata"}. +{coveralls_service_name, "github"}. {codecov_opts, [ From d755e15362a92b56167ebd02bc384994a96abc73 Mon Sep 17 00:00:00 2001 From: DenysGonchar Date: Mon, 21 Dec 2020 15:52:23 +0100 Subject: [PATCH 08/19] adding all the big_tests presents --- .github/workflows/ci.yml | 20 +++++--------------- tools/gh-actions-configure-preset.sh | 24 ++++++++++++++++++++++++ tools/test_runner/presets_to_dbs.erl | 5 +++-- 3 files changed, 32 insertions(+), 17 deletions(-) create mode 100755 tools/gh-actions-configure-preset.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3394bf8beaa..52f44708609 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,7 +40,8 @@ jobs: name: ${{matrix.preset}} on OTP ${{matrix.otp}} strategy: matrix: - preset: [internal_mnesia] + preset: [internal_mnesia, odbc_mssql_mnesia, mysql_redis, riak_mnesia, + ldap_mnesia, elasticsearch_and_cassandra_mnesia] otp: ['23.0.3'] runs-on: ubuntu-latest env: @@ -56,11 +57,8 @@ jobs: key: rebar3-${{ hashFiles('rebar.lock') }} - name: create certificates run: make certs - - if: ${{ matrix.preset == 'internal_mnesia' }} - run: - echo "DB=mnesia minio redis" >> $GITHUB_ENV - echo "REL_CONFIG=with-all" >> $GITHUB_ENV - echo "TLS_DIST=true" >> $GITHUB_ENV + - name: set required env variables for preset + run: tools/gh-actions-configure-preset.sh "$PRESET" - name: build releases run: | ./rebar3 compile @@ -74,13 +72,5 @@ jobs: - name: report coverage env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - DEBUG: '1' run: | - ls -ahl _build/test/cover || true - cp _build/test/cover/*.coverdata /tmp || true - ls -ahl /tmp/*.coverdata || true - ./rebar3 coveralls send || { - echo --rebar3.crashdump-- - cat rebar3.crashdump - exit 1 - } + ./rebar3 coveralls send diff --git a/tools/gh-actions-configure-preset.sh b/tools/gh-actions-configure-preset.sh new file mode 100755 index 00000000000..ad911cff200 --- /dev/null +++ b/tools/gh-actions-configure-preset.sh @@ -0,0 +1,24 @@ +#!/bin/bash +## GITHUB_ENV=/dev/tty ./gh-actions-configure-preset.sh internal-mnesia + +DB_ARRAY=( $(./tools/test_runner/presets_to_dbs.sh "$1" ) ) +[ "${#DB_ARRAY[@]}" -gt 0 ] && export DB="${DB_ARRAY[@]}" + +case "$1" in + internal_mnesia) + export REL_CONFIG="with-all" TLS_DIST=true ;; + odbc_mssql_mnesia) + export REL_CONFIG="with-odbc" ;; + mysql_redis) + export REL_CONFIG="with-mysql with-redis with-amqp_client" ;; + riak_mnesia) + export REL_CONFIG="with-riak" ;; + ldap_mnesia) + export REL_CONFIG="with-none" ;; + elasticsearch_and_cassandra_mnesia) + export REL_CONFIG="with-elasticsearch with-cassandra" TESTSPEC=mam.spec +esac + +if [ ! -z "$GITHUB_ENV" ]; then + env | egrep "^(DB|REL_CONFIG|TLS_DIST)=" >> $GITHUB_ENV +fi diff --git a/tools/test_runner/presets_to_dbs.erl b/tools/test_runner/presets_to_dbs.erl index e0c6bdf5a79..176e135d431 100644 --- a/tools/test_runner/presets_to_dbs.erl +++ b/tools/test_runner/presets_to_dbs.erl @@ -14,8 +14,9 @@ main(Opts) -> -spec presets_to_dbs(atom()) -> [atom()]. presets_to_dbs(PresetNames) -> {ok, Config} = file:consult("big_tests/test.config"), - Presets = proplists:get_value(ejabberd_presets, Config, []), - Presets2 = lists:filter(fun({Name,_}) -> lists:member(Name, PresetNames) end, Presets), + Presets = proplists:get_value(presets, Config, []), + Toml = proplists:get_value(toml, Presets, []), + Presets2 = lists:filter(fun({Name,_}) -> lists:member(Name, PresetNames) end, Toml), Dbs = lists:flatmap(fun({_,Opts}) -> proplists:get_value(dbs, Opts, []) end, Presets2), lists:usort(Dbs). From 147e3db329c49a376ece081180994a294fef9efb Mon Sep 17 00:00:00 2001 From: DenysGonchar Date: Mon, 21 Dec 2020 16:21:27 +0100 Subject: [PATCH 09/19] temporarily disabling mssql preset --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 52f44708609..64454009f3c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,7 +40,7 @@ jobs: name: ${{matrix.preset}} on OTP ${{matrix.otp}} strategy: matrix: - preset: [internal_mnesia, odbc_mssql_mnesia, mysql_redis, riak_mnesia, + preset: [internal_mnesia, mysql_redis, riak_mnesia, ldap_mnesia, elasticsearch_and_cassandra_mnesia] otp: ['23.0.3'] runs-on: ubuntu-latest From f3d49ee7eb509ba4c47c22c81f810d2cd3b6bd4f Mon Sep 17 00:00:00 2001 From: DenysGonchar Date: Tue, 22 Dec 2020 00:39:09 +0100 Subject: [PATCH 10/19] re-enabling odbc_mssql_mnesia preset --- .github/workflows/ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 64454009f3c..8003982671b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,7 +40,7 @@ jobs: name: ${{matrix.preset}} on OTP ${{matrix.otp}} strategy: matrix: - preset: [internal_mnesia, mysql_redis, riak_mnesia, + preset: [internal_mnesia, odbc_mssql_mnesia, mysql_redis, riak_mnesia, ldap_mnesia, elasticsearch_and_cassandra_mnesia] otp: ['23.0.3'] runs-on: ubuntu-latest @@ -59,6 +59,8 @@ jobs: run: make certs - name: set required env variables for preset run: tools/gh-actions-configure-preset.sh "$PRESET" + - if: ${{ matrix.preset == 'odbc_mssql_mnesia' }} + run: sudo apt-get install -y tdsodbc - name: build releases run: | ./rebar3 compile From 8961e2ba45af4abbf68adf0a4e614613a7b27252 Mon Sep 17 00:00:00 2001 From: DenysGonchar Date: Tue, 22 Dec 2020 01:40:30 +0100 Subject: [PATCH 11/19] enabling coveralls parallel reporting --- .github/workflows/ci.yml | 21 +++++++++++++++++++++ rebar.config | 1 + rebar.config.script | 4 ++-- 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8003982671b..21790660020 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,7 +46,11 @@ jobs: runs-on: ubuntu-latest env: PRESET: ${{matrix.preset}} + GITHUB_JOB_ID: ${{github.run_id}}.${{matrix.preset}}.${{matrix.otp}} steps: + - run: | + echo "!!! job id: $GITHUB_JOB_ID" + echo "!!! run id: $GITHUB_RUN_ID" - uses: actions/checkout@v2 - uses: ErlGang/setup-erlang@v1.0.0 with: @@ -76,3 +80,20 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | ./rebar3 coveralls send + + finish: + needs: big_tests + runs-on: ubuntu-latest + steps: + - name: Finish coveralls parallel build + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: > + curl -vk "https://coveralls.io/webhook?repo_token=${GITHUB_TOKEN}" + -d "payload[build_num]=${GITHUB_RUN_ID}&payload[status]=done" + - env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: > + echo + curl -vk "https://coveralls.io/webhook?repo_token=${GITHUB_TOKEN}" + -d "payload[build_num]=${GITHUB_RUN_ID}&payload[status]=done" diff --git a/rebar.config b/rebar.config index 1c790f4bcf8..370a392df90 100644 --- a/rebar.config +++ b/rebar.config @@ -187,6 +187,7 @@ {cover_export_enabled, true}. {coveralls_coverdata, "/tmp/*.coverdata"}. {coveralls_service_name, "github"}. +{coveralls_parallel, true}. {codecov_opts, [ diff --git a/rebar.config.script b/rebar.config.script index ec654e756f2..8bdd9e746ea 100644 --- a/rebar.config.script +++ b/rebar.config.script @@ -88,9 +88,9 @@ MaybeConfigureCoveralls = fun(CONFIG) -> case {os:getenv("GITHUB_ACTIONS"), os:getenv("GITHUB_TOKEN")} of {"true", Token} when is_list(Token) -> CONFIG1 = [{coveralls_repo_token, Token}, - {coveralls_service_job_id, os:getenv("GITHUB_RUN_ID")}, + {coveralls_service_job_id, os:getenv("GITHUB_JOB_ID")}, {coveralls_commit_sha, os:getenv("GITHUB_SHA")}, - {coveralls_service_number, os:getenv("GITHUB_RUN_NUMBER")} | CONFIG], + {coveralls_service_number, os:getenv("GITHUB_RUN_ID")} | CONFIG], case os:getenv("GITHUB_EVENT_NAME") =:= "pull_request" andalso string:tokens(os:getenv("GITHUB_REF"), "/") of [_, "pull", PRNO, _] -> From e24f5547d64bbc92d491e0583535496c1b3bc476 Mon Sep 17 00:00:00 2001 From: DenysGonchar Date: Wed, 23 Dec 2020 01:33:24 +0100 Subject: [PATCH 12/19] webhook fails with GH token, switching to coveralls generated repo token (only for webhook) --- .github/workflows/ci.yml | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 21790660020..196ac85105e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -87,13 +87,7 @@ jobs: steps: - name: Finish coveralls parallel build env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: > - curl -vk "https://coveralls.io/webhook?repo_token=${GITHUB_TOKEN}" - -d "payload[build_num]=${GITHUB_RUN_ID}&payload[status]=done" - - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }} run: > - echo - curl -vk "https://coveralls.io/webhook?repo_token=${GITHUB_TOKEN}" + curl -k "https://coveralls.io/webhook?repo_token=${REPO_TOKEN}" -d "payload[build_num]=${GITHUB_RUN_ID}&payload[status]=done" From fc6620a82836225bdd86733e4c307b8a8eb53541 Mon Sep 17 00:00:00 2001 From: DenysGonchar Date: Wed, 23 Dec 2020 01:25:52 +0100 Subject: [PATCH 13/19] reporting coverage for small_tests setup --- .github/workflows/ci.yml | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 196ac85105e..d8156524232 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,7 +34,17 @@ jobs: - name: create certificates run: make certs - name: run test - run: tools/travis-test.sh -p $PRESET -s true -e true + run: | + ./rebar3 compile #required for coverage reporting + tools/travis-test.sh -p $PRESET -s true -e true + - name: report coverage + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_JOB_ID: ${{github.run_id}}.small_tests.${{matrix.otp}} + DEBUG: 1 + run: | + cp _build/test/cover/*.coverdata /tmp + ./rebar3 coveralls send big_tests: name: ${{matrix.preset}} on OTP ${{matrix.otp}} @@ -46,11 +56,7 @@ jobs: runs-on: ubuntu-latest env: PRESET: ${{matrix.preset}} - GITHUB_JOB_ID: ${{github.run_id}}.${{matrix.preset}}.${{matrix.otp}} steps: - - run: | - echo "!!! job id: $GITHUB_JOB_ID" - echo "!!! run id: $GITHUB_RUN_ID" - uses: actions/checkout@v2 - uses: ErlGang/setup-erlang@v1.0.0 with: @@ -67,7 +73,7 @@ jobs: run: sudo apt-get install -y tdsodbc - name: build releases run: | - ./rebar3 compile + ./rebar3 compile #required for coverage reporting tools/build-releases.sh - name: build big tests run: tools/travis-build-tests.sh @@ -78,11 +84,11 @@ jobs: - name: report coverage env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - ./rebar3 coveralls send + GITHUB_JOB_ID: ${{github.run_id}}.${{matrix.preset}}.${{matrix.otp}} + run: ./rebar3 coveralls send - finish: - needs: big_tests + coveralls_webhook: + needs: [big_tests, small_tests] runs-on: ubuntu-latest steps: - name: Finish coveralls parallel build From 9e9b3b25e91688fadcc5211df25b18bac758ead0 Mon Sep 17 00:00:00 2001 From: DenysGonchar Date: Wed, 23 Dec 2020 12:34:35 +0100 Subject: [PATCH 14/19] switching back to GITHUB_TOKEN (it additionally requires 'repo_name' parameter) --- .github/workflows/ci.yml | 13 ++++--------- rebar.config.script | 5 ++--- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d8156524232..fad5740b76c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,7 +19,7 @@ jobs: strategy: matrix: otp: ['23.0.3', '22.3'] - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 env: PRESET: 'small_tests' steps: @@ -40,8 +40,6 @@ jobs: - name: report coverage env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_JOB_ID: ${{github.run_id}}.small_tests.${{matrix.otp}} - DEBUG: 1 run: | cp _build/test/cover/*.coverdata /tmp ./rebar3 coveralls send @@ -53,7 +51,7 @@ jobs: preset: [internal_mnesia, odbc_mssql_mnesia, mysql_redis, riak_mnesia, ldap_mnesia, elasticsearch_and_cassandra_mnesia] otp: ['23.0.3'] - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 env: PRESET: ${{matrix.preset}} steps: @@ -84,16 +82,13 @@ jobs: - name: report coverage env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_JOB_ID: ${{github.run_id}}.${{matrix.preset}}.${{matrix.otp}} run: ./rebar3 coveralls send coveralls_webhook: needs: [big_tests, small_tests] - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 steps: - name: Finish coveralls parallel build - env: - REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }} run: > - curl -k "https://coveralls.io/webhook?repo_token=${REPO_TOKEN}" + curl -k "https://coveralls.io/webhook?repo_token=${{secrets.GITHUB_TOKEN}}&repo_name=${{github.repository}}" -d "payload[build_num]=${GITHUB_RUN_ID}&payload[status]=done" diff --git a/rebar.config.script b/rebar.config.script index 8bdd9e746ea..3ae58d3ad37 100644 --- a/rebar.config.script +++ b/rebar.config.script @@ -88,9 +88,8 @@ MaybeConfigureCoveralls = fun(CONFIG) -> case {os:getenv("GITHUB_ACTIONS"), os:getenv("GITHUB_TOKEN")} of {"true", Token} when is_list(Token) -> CONFIG1 = [{coveralls_repo_token, Token}, - {coveralls_service_job_id, os:getenv("GITHUB_JOB_ID")}, - {coveralls_commit_sha, os:getenv("GITHUB_SHA")}, - {coveralls_service_number, os:getenv("GITHUB_RUN_ID")} | CONFIG], + {coveralls_service_job_id, os:getenv("GITHUB_RUN_ID")}, + {coveralls_commit_sha, os:getenv("GITHUB_SHA")} | CONFIG], case os:getenv("GITHUB_EVENT_NAME") =:= "pull_request" andalso string:tokens(os:getenv("GITHUB_REF"), "/") of [_, "pull", PRNO, _] -> From e8f5d2ba4e887d0ed2e432be62a6d9597411b5be Mon Sep 17 00:00:00 2001 From: DenysGonchar Date: Mon, 28 Dec 2020 17:23:30 +0100 Subject: [PATCH 15/19] using GH coveralls action to finalize parallel reporting --- .github/workflows/ci.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fad5740b76c..322b976f94c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -89,6 +89,7 @@ jobs: runs-on: ubuntu-20.04 steps: - name: Finish coveralls parallel build - run: > - curl -k "https://coveralls.io/webhook?repo_token=${{secrets.GITHUB_TOKEN}}&repo_name=${{github.repository}}" - -d "payload[build_num]=${GITHUB_RUN_ID}&payload[status]=done" + uses: coverallsapp/github-action@v1.1.2 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + parallel-finished: true From cd11040db43a38bf2efab13a918d5674d1ff2294 Mon Sep 17 00:00:00 2001 From: DenysGonchar Date: Tue, 29 Dec 2020 05:28:44 +0100 Subject: [PATCH 16/19] adding dialyzer & pkg building jobs --- .github/workflows/ci.yml | 30 ++++++++++++++++++++++++++-- tools/gh-actions-configure-preset.sh | 2 ++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 322b976f94c..cc16bb08bad 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,8 +48,8 @@ jobs: name: ${{matrix.preset}} on OTP ${{matrix.otp}} strategy: matrix: - preset: [internal_mnesia, odbc_mssql_mnesia, mysql_redis, riak_mnesia, - ldap_mnesia, elasticsearch_and_cassandra_mnesia] + preset: [internal_mnesia, pgsql_mnesia, mysql_redis, odbc_mssql_mnesia, + ldap_mnesia, riak_mnesia, elasticsearch_and_cassandra_mnesia] otp: ['23.0.3'] runs-on: ubuntu-20.04 env: @@ -93,3 +93,29 @@ jobs: with: github-token: ${{ secrets.GITHUB_TOKEN }} parallel-finished: true + + dialyzer: + name: dialyzer on OTP ${{matrix.otp}} + strategy: + matrix: + otp: [ '23.0.3'] + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + - uses: ErlGang/setup-erlang@v1.0.0 + with: + otp-version: ${{matrix.otp}} + - run: tools/travis-test.sh -p dialyzer_only + + pkg: + name: ${{matrix.pkg}} package + strategy: + matrix: + pkg: [centos_7, debian_stretch] + runs-on: ubuntu-20.04 + env: + ESL_ERLANG_PKG_VER: "22.3.4.9-1" + pkg_PLATFORM: ${{matrix.pkg}} + steps: + - uses: actions/checkout@v2 + - run: tools/travis-test.sh -p pkg diff --git a/tools/gh-actions-configure-preset.sh b/tools/gh-actions-configure-preset.sh index ad911cff200..5d4ef988d59 100755 --- a/tools/gh-actions-configure-preset.sh +++ b/tools/gh-actions-configure-preset.sh @@ -11,6 +11,8 @@ case "$1" in export REL_CONFIG="with-odbc" ;; mysql_redis) export REL_CONFIG="with-mysql with-redis with-amqp_client" ;; + pgsql_mnesia) + export REL_CONFIG="with-pgsql" ;; riak_mnesia) export REL_CONFIG="with-riak" ;; ldap_mnesia) From 644eb68d5f1cfd0be178974e4389c7507adbb138 Mon Sep 17 00:00:00 2001 From: DenysGonchar Date: Tue, 29 Dec 2020 15:10:12 +0100 Subject: [PATCH 17/19] disabling GH CI workflow execution on PRs --- .github/workflows/ci.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cc16bb08bad..d12ee8f1f94 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,8 +3,6 @@ name: CI on: push: branches: [ master ] - pull_request: - branches: [ master ] # Allows you to run this workflow manually from the Actions tab workflow_dispatch: From fba74fc1b07eb511daf1f2884eef3f26c5a69876 Mon Sep 17 00:00:00 2001 From: DenysGonchar Date: Thu, 31 Dec 2020 13:41:02 +0100 Subject: [PATCH 18/19] report lcov file to coveralls using gh action --- .github/workflows/ci.yml | 32 ++++++++++++++++++-------------- rebar.config | 3 +-- rebar.config.script | 23 ++--------------------- 3 files changed, 21 insertions(+), 37 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d12ee8f1f94..61857d04702 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,15 +32,16 @@ jobs: - name: create certificates run: make certs - name: run test - run: | - ./rebar3 compile #required for coverage reporting - tools/travis-test.sh -p $PRESET -s true -e true + run: tools/travis-test.sh -p $PRESET -s true -e true + - name: prepare coverage report + run: ./rebar3 codecov analyze --lcov --json false - name: report coverage - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - cp _build/test/cover/*.coverdata /tmp - ./rebar3 coveralls send + uses: coverallsapp/github-action@master + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + flag-name: small_tests on OTP ${{matrix.otp}} + path-to-lcov: ./lcov.info + parallel: true big_tests: name: ${{matrix.preset}} on OTP ${{matrix.otp}} @@ -68,19 +69,22 @@ jobs: - if: ${{ matrix.preset == 'odbc_mssql_mnesia' }} run: sudo apt-get install -y tdsodbc - name: build releases - run: | - ./rebar3 compile #required for coverage reporting - tools/build-releases.sh + run: tools/build-releases.sh - name: build big tests run: tools/travis-build-tests.sh - name: start backends run: tools/travis-setup-db.sh - name: run test run: tools/travis-test.sh -p $PRESET -s false -e false + - name: prepare coverage report + run: ./rebar3 codecov analyze --lcov --json false - name: report coverage - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: ./rebar3 coveralls send + uses: coverallsapp/github-action@master + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + flag-name: ${{matrix.preset}} on OTP ${{matrix.otp}} + path-to-lcov: ./lcov.info + parallel: true coveralls_webhook: needs: [big_tests, small_tests] diff --git a/rebar.config b/rebar.config index 370a392df90..d2df6beb111 100644 --- a/rebar.config +++ b/rebar.config @@ -153,8 +153,7 @@ [ {pc, "1.11.0"}, {provider_asn1, "0.2.3"}, - {rebar3_codecov, {git, "https://github.com/esl/rebar3_codecov.git", {ref, "6bd31cc"}}}, - {coveralls, "2.2.0"}, + {rebar3_codecov, {git, "https://github.com/esl/rebar3_codecov.git", {branch, "lcov"}}}, {rebar3_lint, "0.1.11"} ]}. diff --git a/rebar.config.script b/rebar.config.script index 3ae58d3ad37..d1511fc1f2d 100644 --- a/rebar.config.script +++ b/rebar.config.script @@ -84,24 +84,5 @@ MaybeFIPSSupport = fun(Config) -> end end, -MaybeConfigureCoveralls = fun(CONFIG) -> - case {os:getenv("GITHUB_ACTIONS"), os:getenv("GITHUB_TOKEN")} of - {"true", Token} when is_list(Token) -> - CONFIG1 = [{coveralls_repo_token, Token}, - {coveralls_service_job_id, os:getenv("GITHUB_RUN_ID")}, - {coveralls_commit_sha, os:getenv("GITHUB_SHA")} | CONFIG], - case os:getenv("GITHUB_EVENT_NAME") =:= "pull_request" andalso - string:tokens(os:getenv("GITHUB_REF"), "/") of - [_, "pull", PRNO, _] -> - [{coveralls_service_pull_request, PRNO} | CONFIG1]; - _ -> - CONFIG1 - end; - _ -> - CONFIG - end -end, - -Config0 = MaybeConfigureCoveralls(CONFIG), -Config1 = SetupIncludedApps(Config0, EnvApps), -MaybeFIPSSupport(Config1). +Config0 = SetupIncludedApps(CONFIG, EnvApps), +MaybeFIPSSupport(Config0). From 4048808300a01b187f1b545c340144efd7d23dec Mon Sep 17 00:00:00 2001 From: DenysGonchar Date: Thu, 31 Dec 2020 13:47:43 +0100 Subject: [PATCH 19/19] drop --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 61857d04702..121928bcad8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,6 +3,8 @@ name: CI on: push: branches: [ master ] + pull_request: + branches: [ master ] # Allows you to run this workflow manually from the Actions tab workflow_dispatch: