From 291705ab3dd9aae0f0475a516fbc209010529a16 Mon Sep 17 00:00:00 2001 From: Pierre Millot Date: Mon, 10 Jan 2022 19:11:28 +0100 Subject: [PATCH 01/49] first try --- .github/actions/setup/action.yml | 9 ++ .github/workflows/check.yml | 257 ++----------------------------- 2 files changed, 18 insertions(+), 248 deletions(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index cc7afff4730..b2d3ed5558f 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -52,6 +52,11 @@ runs: echo "::set-output name=JAVA_CLIENT_CHANGED::$(git diff --shortstat origin/${{ github.base_ref }}..HEAD -- clients/algoliasearch-client-java-2 | wc -l)" echo "::set-output name=JAVA_TEMPLATE_CHANGED::$(git diff --shortstat origin/${{ github.base_ref }}..HEAD -- templates/java | wc -l)" + - name: matrix + id: set-matrix + shell: bash + run: echo "::set-output name=specs_matrix::{\"client\": ["search", "recommend"]}" + outputs: # specs variables RUN_SPECS_SEARCH: @@ -102,3 +107,7 @@ outputs: RUN_CTS: description: Determine if the `cts` job should run value: ${{ github.ref_name == 'main' || steps.diff.outputs.GITHUB_ACTIONS_CHANGED > 0 || steps.diff.outputs.SPECS_CHANGED > 0 || steps.diff.outputs.TESTS_CHANGED > 0 || steps.diff.outputs.JS_CLIENT_CHANGED > 0 || steps.diff.outputs.JS_TEMPLATE_CHANGED > 0 || steps.diff.outputs.JAVA_CLIENT_CHANGED > 0 || steps.diff.outputs.JAVA_TEMPLATE_CHANGED > 0 }} + + specs_matrix: + description: Generate the matrix for specs + value: ${{ steps.set-matrix.outputs.specs_matrix }} diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index bda25d1b967..e3a5d13debc 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -25,6 +25,7 @@ jobs: - name: Lint GitHub actions run: yarn eslint --ext=yml .github/actions .github/workflows + outputs: RUN_SPECS_SEARCH: ${{ steps.setup.outputs.RUN_SPECS_SEARCH }} RUN_SPECS_RECOMMEND: ${{ steps.setup.outputs.RUN_SPECS_RECOMMEND }} @@ -43,261 +44,21 @@ jobs: RUN_JAVA_CLIENT: ${{ steps.setup.outputs.RUN_JAVA_CLIENT }} RUN_CTS: ${{ steps.setup.outputs.RUN_CTS }} + specs_matrix: ${{ steps.setup.outputs.specs_matrix }} - specs_search: - runs-on: ubuntu-20.04 - needs: setup - if: ${{ always() && needs.setup.outputs.RUN_SPECS_SEARCH == 'true' }} - steps: - - uses: actions/checkout@v2 - - - name: Restore cache - uses: ./.github/actions/cache - - - name: Checking search specs - run: yarn build:specs search - - - name: Lint search specs - run: yarn eslint --ext=yml specs/search - - specs_recommend: - runs-on: ubuntu-20.04 - needs: setup - if: ${{ always() && needs.setup.outputs.RUN_SPECS_RECOMMEND == 'true' }} - steps: - - uses: actions/checkout@v2 - - - name: Restore cache - uses: ./.github/actions/cache - - - name: Checking recommend specs - run: yarn build:specs recommend - - - name: Lint recommend specs - run: yarn eslint --ext=yml specs/recommend - - specs_perso: - runs-on: ubuntu-20.04 - needs: setup - if: ${{ always() && needs.setup.outputs.RUN_SPECS_PERSO == 'true' }} - steps: - - uses: actions/checkout@v2 - - - name: Restore cache - uses: ./.github/actions/cache - - - name: Checking personalization specs - run: yarn build:specs personalization - - - name: Lint personalization specs - run: yarn eslint --ext=yml specs/personalization - - specs_analytics: + specs: runs-on: ubuntu-20.04 needs: setup - if: ${{ always() && needs.setup.outputs.RUN_SPECS_ANALYTICS == 'true' }} - steps: - - uses: actions/checkout@v2 - - - name: Restore cache - uses: ./.github/actions/cache - - - name: Checking analytics specs - run: yarn build:specs analytics - - - name: Lint analytics specs - run: yarn eslint --ext=yml specs/analytics - - specs_insights: - runs-on: ubuntu-20.04 - needs: setup - if: ${{ always() && needs.setup.outputs.RUN_SPECS_INSIGHTS == 'true' }} - steps: - - uses: actions/checkout@v2 - - - name: Restore cache - uses: ./.github/actions/cache - - - name: Checking insights specs - run: yarn build:specs insights - - - name: Lint insights specs - run: yarn eslint --ext=yml specs/insights - - client_javascript_search: - runs-on: ubuntu-20.04 - needs: [specs_search] - if: ${{ always() && needs.setup.outputs.RUN_JS_CLIENT_SEARCH == 'true' }} + strategy: + matrix: ${{ fromJSON(needs.setup.outputs.specs_matrix)}} steps: - uses: actions/checkout@v2 - name: Restore cache uses: ./.github/actions/cache - - name: Cache search client - id: cache - uses: actions/cache@v2 - with: - path: /home/runner/work/api-clients-automation/api-clients-automation/clients/algoliasearch-client-javascript/client-search/dist - key: ${{ runner.os }}-js-client-search-${{ hashFiles('clients/algoliasearch-client-javascript/client-search/**') }} - - - name: Generate search client - if: steps.cache.outputs.cache-hit != 'true' - run: yarn generate javascript search - - - name: Build search client - if: steps.cache.outputs.cache-hit != 'true' - run: yarn build:clients javascript search - - client_javascript_recommend: - runs-on: ubuntu-20.04 - needs: [specs_recommend] - if: ${{ always() && needs.setup.outputs.RUN_JS_CLIENT_RECOMMEND == 'true' }} - steps: - - uses: actions/checkout@v2 - - - name: Restore cache - uses: ./.github/actions/cache - - - name: Cache recommend client - id: cache - uses: actions/cache@v2 - with: - path: /home/runner/work/api-clients-automation/api-clients-automation/clients/algoliasearch-client-javascript/recommend/dist - key: ${{ runner.os }}-js-client-recommend-${{ hashFiles('clients/algoliasearch-client-javascript/recommend/**') }} - - - name: Generate recommend client - if: steps.cache.outputs.cache-hit != 'true' - run: yarn generate javascript recommend - - - name: Build recommend client - if: steps.cache.outputs.cache-hit != 'true' - run: yarn build:clients javascript recommend - - client_javascript_perso: - runs-on: ubuntu-20.04 - needs: [specs_perso] - if: ${{ always() && needs.setup.outputs.RUN_JS_CLIENT_PERSO == 'true' }} - steps: - - uses: actions/checkout@v2 - - - name: Restore cache - uses: ./.github/actions/cache - - - name: Cache personalization client - id: cache - uses: actions/cache@v2 - with: - path: /home/runner/work/api-clients-automation/api-clients-automation/clients/algoliasearch-client-javascript/client-personalization/dist - key: ${{ runner.os }}-js-client-personalization-${{ hashFiles('clients/algoliasearch-client-javascript/client-personalization/**') }} - - - name: Generate personalization client - if: steps.cache.outputs.cache-hit != 'true' - run: yarn generate javascript personalization - - - name: Build personalization client - if: steps.cache.outputs.cache-hit != 'true' - run: yarn build:clients javascript personalization - - client_javascript_analytics: - runs-on: ubuntu-20.04 - needs: [specs_analytics] - if: ${{ always() && needs.setup.outputs.RUN_JS_CLIENT_ANALYTICS == 'true' }} - steps: - - uses: actions/checkout@v2 - - - name: Restore cache - uses: ./.github/actions/cache - - - name: Cache analytics client - id: cache - uses: actions/cache@v2 - with: - path: /home/runner/work/api-clients-automation/api-clients-automation/clients/algoliasearch-client-javascript/client-analytics/dist - key: ${{ runner.os }}-js-client-analytics-${{ hashFiles('clients/algoliasearch-client-javascript/client-analytics/**') }} - - - name: Generate analytics client - if: steps.cache.outputs.cache-hit != 'true' - run: yarn generate javascript analytics - - - name: Build analytics client - if: steps.cache.outputs.cache-hit != 'true' - run: yarn build:clients javascript analytics - - client_javascript_insights: - runs-on: ubuntu-20.04 - needs: [specs_insights] - if: ${{ always() && needs.setup.outputs.RUN_JS_CLIENT_INSIGHTS == 'true' }} - steps: - - uses: actions/checkout@v2 - - - name: Restore cache - uses: ./.github/actions/cache - - - name: Cache insights client - id: cache - uses: actions/cache@v2 - with: - path: /home/runner/work/api-clients-automation/api-clients-automation/clients/algoliasearch-client-javascript/client-insights/dist - key: ${{ runner.os }}-js-client-insights-${{ hashFiles('clients/algoliasearch-client-javascript/client-insights/**') }} - - - name: Generate insights client - if: steps.cache.outputs.cache-hit != 'true' - run: yarn generate javascript insights - - - name: Build insights client - if: steps.cache.outputs.cache-hit != 'true' - run: yarn build:clients javascript insights - - client_java_search: - runs-on: ubuntu-20.04 - needs: [specs_search] - if: ${{ always() && needs.setup.outputs.RUN_JAVA_CLIENT == 'true' }} - steps: - - uses: actions/checkout@v2 - - - name: Restore cache - uses: ./.github/actions/cache - with: - job: java - - - name: Cache built clients - id: cache - uses: actions/cache@v2 - with: - path: /home/runner/work/api-clients-automation/api-clients-automation/clients/algoliasearch-client-java-2/target - key: ${{ runner.os }}-java-client-${{ hashFiles('clients/algoliasearch-client-java-2/**') }} - - - name: Generate search client - if: steps.cache.outputs.cache-hit != 'true' - run: yarn generate java search - - - name: Build search client - if: steps.cache.outputs.cache-hit != 'true' - run: yarn build:clients java search - - cts: - runs-on: ubuntu-20.04 - needs: - - client_javascript_search - - client_javascript_recommend - - client_javascript_perso - - client_javascript_analytics - - client_javascript_insights - - client_java_search - - if: ${{ always() && needs.setup.outputs.RUN_CTS == 'true' }} - steps: - - uses: actions/checkout@v2 - - - name: Restore cache - id: restore - uses: ./.github/actions/cache - with: - job: cts - - - name: Generate CTS - run: yarn cts:generate + - name: Checking ${{ matrix.client }} specs + run: yarn build:specs ${{ matrix.client }} - - name: Run CTS - run: yarn cts:test + - name: Lint ${{ matrix.client }} specs + run: yarn eslint --ext=yml specs/${{ matrix.client }} From 887dfe7c44493a31366e3ab266401e6efebd45b1 Mon Sep 17 00:00:00 2001 From: Pierre Millot Date: Mon, 10 Jan 2022 19:13:34 +0100 Subject: [PATCH 02/49] escape --- .github/actions/setup/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index b2d3ed5558f..301ed9ec633 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -55,7 +55,7 @@ runs: - name: matrix id: set-matrix shell: bash - run: echo "::set-output name=specs_matrix::{\"client\": ["search", "recommend"]}" + run: echo "::set-output name=specs_matrix::{\"client\": [\"search\", \"recommend\"]}" outputs: # specs variables From 8d34990d5610244778be97c9e15a16e85b00ad88 Mon Sep 17 00:00:00 2001 From: Pierre Millot Date: Mon, 10 Jan 2022 19:16:27 +0100 Subject: [PATCH 03/49] json --- .github/actions/setup/action.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 301ed9ec633..1596007a1d9 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -55,7 +55,8 @@ runs: - name: matrix id: set-matrix shell: bash - run: echo "::set-output name=specs_matrix::{\"client\": [\"search\", \"recommend\"]}" + run: | + echo ::set-output name=specs_matrix::'{"client": ["search", "recommend"]}' outputs: # specs variables From 8339ceec83b5cd9660fe3f2271445bc47cc47514 Mon Sep 17 00:00:00 2001 From: Pierre Millot Date: Tue, 11 Jan 2022 10:32:58 +0100 Subject: [PATCH 04/49] try things out --- .github/actions/setup/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 1596007a1d9..d3ce282b0de 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -111,4 +111,4 @@ outputs: specs_matrix: description: Generate the matrix for specs - value: ${{ steps.set-matrix.outputs.specs_matrix }} + value: '{"client": [${{ COMMON_CHANGE=$(github.ref == "refs/heads/main" || steps.diff.outputs.GITHUB_ACTIONS_CHANGED > 0 || steps.diff.outputs.COMMON_SPECS_CHANGED > 0) && echo $COMMON_CHANGE }}]}' From 44f038200c4ee30141ca840d4a91f723c416b2b3 Mon Sep 17 00:00:00 2001 From: Pierre Millot Date: Tue, 11 Jan 2022 10:42:55 +0100 Subject: [PATCH 05/49] generate array --- .github/actions/setup/action.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index d3ce282b0de..8cf4eb19f4a 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -52,11 +52,17 @@ runs: echo "::set-output name=JAVA_CLIENT_CHANGED::$(git diff --shortstat origin/${{ github.base_ref }}..HEAD -- clients/algoliasearch-client-java-2 | wc -l)" echo "::set-output name=JAVA_TEMPLATE_CHANGED::$(git diff --shortstat origin/${{ github.base_ref }}..HEAD -- templates/java | wc -l)" + - name: Set temporary variables + id: tmpvar + shell: bash + run: | + echo ::set-output name=BASE_SPEC::${{ github.ref == 'refs/heads/main' || steps.diff.outputs.GITHUB_ACTIONS_CHANGED > 0 || steps.diff.outputs.SCRIPTS_CHANGED > 0 || steps.diff.outputs.COMMON_SPECS_CHANGED > 0 }} + - name: matrix id: set-matrix shell: bash run: | - echo ::set-output name=specs_matrix::'{"client": ["search", "recommend"]}' + echo ::set-output name=specs_matrix::'{"client": [${{ (steps.tmpvar.output.BASE_SPEC > 0 || steps.diff.outputs.SEARCH_SPECS_CHANGED > 0) && "search" }}, "recommend"]}' outputs: # specs variables @@ -111,4 +117,4 @@ outputs: specs_matrix: description: Generate the matrix for specs - value: '{"client": [${{ COMMON_CHANGE=$(github.ref == "refs/heads/main" || steps.diff.outputs.GITHUB_ACTIONS_CHANGED > 0 || steps.diff.outputs.COMMON_SPECS_CHANGED > 0) && echo $COMMON_CHANGE }}]}' + value: ${{ steps.set-matrix.output.specs_matrix }} From 4a7c0a9f84c0f14b3d513207d989ac50e7cf105c Mon Sep 17 00:00:00 2001 From: Pierre Millot Date: Tue, 11 Jan 2022 10:44:30 +0100 Subject: [PATCH 06/49] quotes --- .github/actions/setup/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 8cf4eb19f4a..54190dd7c26 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -62,7 +62,7 @@ runs: id: set-matrix shell: bash run: | - echo ::set-output name=specs_matrix::'{"client": [${{ (steps.tmpvar.output.BASE_SPEC > 0 || steps.diff.outputs.SEARCH_SPECS_CHANGED > 0) && "search" }}, "recommend"]}' + echo ::set-output name=specs_matrix::'{"client": ["${{ (steps.tmpvar.output.BASE_SPEC > 0 || steps.diff.outputs.SEARCH_SPECS_CHANGED > 0) && 'search' }}", "recommend"]}' outputs: # specs variables From ff018c6934244dd53714e458304d5162c58a7ca2 Mon Sep 17 00:00:00 2001 From: Pierre Millot Date: Tue, 11 Jan 2022 11:03:25 +0100 Subject: [PATCH 07/49] jq --- .github/actions/setup/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 54190dd7c26..9ac1e2c9e9d 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -62,7 +62,7 @@ runs: id: set-matrix shell: bash run: | - echo ::set-output name=specs_matrix::'{"client": ["${{ (steps.tmpvar.output.BASE_SPEC > 0 || steps.diff.outputs.SEARCH_SPECS_CHANGED > 0) && 'search' }}", "recommend"]}' + echo ::set-output name=specs_matrix::$(jq -n -c -M --arg s "${{ (steps.tmpvar.outputs.BASE_SPEC > 0 || steps.diff.outputs.SEARCH_SPECS_CHANGED > 0) && 'search ' || ' '}}${{ (steps.tmpvar.outputs.BASE_SPEC > 0 || steps.diff.outputs.RECOMMEND_SPECS_CHANGED > 0) && 'recommend ' || ' '}}" '{client: ($s|split(" "))}') outputs: # specs variables From 4884ce968f5afcc20ab2f9b99ac704b969593630 Mon Sep 17 00:00:00 2001 From: Pierre Millot Date: Tue, 11 Jan 2022 11:07:07 +0100 Subject: [PATCH 08/49] log --- .github/workflows/check.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index e3a5d13debc..97fce63c31b 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -23,6 +23,12 @@ jobs: id: setup uses: ./.github/actions/setup + - name: Log outputs + shell: bash + run: | + echo "Hello world !" + echo ${{ steps.setup.outputs.specs_matrix}} + - name: Lint GitHub actions run: yarn eslint --ext=yml .github/actions .github/workflows From a6a237b5adb2ab9ed446e136bef2e1b74d83b807 Mon Sep 17 00:00:00 2001 From: Pierre Millot Date: Tue, 11 Jan 2022 11:11:09 +0100 Subject: [PATCH 09/49] logic --- .github/actions/setup/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 9ac1e2c9e9d..8f810b52b22 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -62,7 +62,7 @@ runs: id: set-matrix shell: bash run: | - echo ::set-output name=specs_matrix::$(jq -n -c -M --arg s "${{ (steps.tmpvar.outputs.BASE_SPEC > 0 || steps.diff.outputs.SEARCH_SPECS_CHANGED > 0) && 'search ' || ' '}}${{ (steps.tmpvar.outputs.BASE_SPEC > 0 || steps.diff.outputs.RECOMMEND_SPECS_CHANGED > 0) && 'recommend ' || ' '}}" '{client: ($s|split(" "))}') + echo ::set-output name=specs_matrix::$(jq -n -c -M --arg s "${{ steps.tmpvar.outputs.BASE_SPEC > 0 && 'search ' }}${{ (steps.tmpvar.outputs.BASE_SPEC > 0 || steps.diff.outputs.RECOMMEND_SPECS_CHANGED > 0) && 'recommend ' || ' '}}" '{client: ($s|split(" "))}') outputs: # specs variables From cca503947dc1933d1411ff96748bbab4696c51ba Mon Sep 17 00:00:00 2001 From: Pierre Millot Date: Tue, 11 Jan 2022 11:33:28 +0100 Subject: [PATCH 10/49] forgort s on output --- .github/actions/setup/action.yml | 6 +++--- .github/workflows/check.yml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 8f810b52b22..feed9b55063 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -62,7 +62,7 @@ runs: id: set-matrix shell: bash run: | - echo ::set-output name=specs_matrix::$(jq -n -c -M --arg s "${{ steps.tmpvar.outputs.BASE_SPEC > 0 && 'search ' }}${{ (steps.tmpvar.outputs.BASE_SPEC > 0 || steps.diff.outputs.RECOMMEND_SPECS_CHANGED > 0) && 'recommend ' || ' '}}" '{client: ($s|split(" "))}') + echo ::set-output name=SPECS_MATRIX::$(jq -n -c -M --arg s "${{ steps.tmpvar.outputs.BASE_SPEC > 0 && 'search ' }}${{ (steps.tmpvar.outputs.BASE_SPEC > 0 || steps.diff.outputs.RECOMMEND_SPECS_CHANGED > 0) && 'recommend ' || ' '}}" '{client: ($s|split(" "))}') outputs: # specs variables @@ -115,6 +115,6 @@ outputs: description: Determine if the `cts` job should run value: ${{ github.ref_name == 'main' || steps.diff.outputs.GITHUB_ACTIONS_CHANGED > 0 || steps.diff.outputs.SPECS_CHANGED > 0 || steps.diff.outputs.TESTS_CHANGED > 0 || steps.diff.outputs.JS_CLIENT_CHANGED > 0 || steps.diff.outputs.JS_TEMPLATE_CHANGED > 0 || steps.diff.outputs.JAVA_CLIENT_CHANGED > 0 || steps.diff.outputs.JAVA_TEMPLATE_CHANGED > 0 }} - specs_matrix: + SPECS_MATRIX: description: Generate the matrix for specs - value: ${{ steps.set-matrix.output.specs_matrix }} + value: ${{ steps.set-matrix.outputs.SPECS_MATRIX }} diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 97fce63c31b..583bca8ae93 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -27,7 +27,7 @@ jobs: shell: bash run: | echo "Hello world !" - echo ${{ steps.setup.outputs.specs_matrix}} + echo ${{ steps.setup.outputs.SPECS_MATRIX}} - name: Lint GitHub actions run: yarn eslint --ext=yml .github/actions .github/workflows @@ -50,13 +50,13 @@ jobs: RUN_JAVA_CLIENT: ${{ steps.setup.outputs.RUN_JAVA_CLIENT }} RUN_CTS: ${{ steps.setup.outputs.RUN_CTS }} - specs_matrix: ${{ steps.setup.outputs.specs_matrix }} + SPECS_MATRIX: ${{ steps.setup.outputs.SPECS_MATRIX }} specs: runs-on: ubuntu-20.04 needs: setup strategy: - matrix: ${{ fromJSON(needs.setup.outputs.specs_matrix)}} + matrix: ${{ fromJSON(needs.setup.outputs.SPECS_MATRIX)}} steps: - uses: actions/checkout@v2 From ce70e8d5b8c4880cf025530dcb490bdb8740e1ad Mon Sep 17 00:00:00 2001 From: Pierre Millot Date: Tue, 11 Jan 2022 11:49:50 +0100 Subject: [PATCH 11/49] try toJSON --- .github/actions/setup/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index feed9b55063..21d59df9c4b 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -62,7 +62,7 @@ runs: id: set-matrix shell: bash run: | - echo ::set-output name=SPECS_MATRIX::$(jq -n -c -M --arg s "${{ steps.tmpvar.outputs.BASE_SPEC > 0 && 'search ' }}${{ (steps.tmpvar.outputs.BASE_SPEC > 0 || steps.diff.outputs.RECOMMEND_SPECS_CHANGED > 0) && 'recommend ' || ' '}}" '{client: ($s|split(" "))}') + echo ::set-output name=SPECS_MATRIX::${{ toJSON(join(((steps.tmpvar.outputs.BASE_SPEC || steps.diff.outputs.SEARCH_SPECS_CHANGED) && 'search', (steps.tmpvar.outputs.BASE_SPEC || steps.diff.outputs.RECOMMENT_SPECS_CHANGED) && 'recommend'))) }} outputs: # specs variables From cebc54e05eb30effc426ec405a633f6ab2cd92bb Mon Sep 17 00:00:00 2001 From: Pierre Millot Date: Tue, 11 Jan 2022 12:00:26 +0100 Subject: [PATCH 12/49] try with array --- .github/actions/setup/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 21d59df9c4b..61940b7ae79 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -62,7 +62,7 @@ runs: id: set-matrix shell: bash run: | - echo ::set-output name=SPECS_MATRIX::${{ toJSON(join(((steps.tmpvar.outputs.BASE_SPEC || steps.diff.outputs.SEARCH_SPECS_CHANGED) && 'search', (steps.tmpvar.outputs.BASE_SPEC || steps.diff.outputs.RECOMMENT_SPECS_CHANGED) && 'recommend'))) }} + echo ::set-output name=SPECS_MATRIX::${{ toJSON(join([(steps.tmpvar.outputs.BASE_SPEC || steps.diff.outputs.SEARCH_SPECS_CHANGED) && 'search', (steps.tmpvar.outputs.BASE_SPEC || steps.diff.outputs.RECOMMENT_SPECS_CHANGED) && 'recommend'])) }} outputs: # specs variables From 34db1f05d78cbb65ce8150e2160d0b6eb7e66549 Mon Sep 17 00:00:00 2001 From: Pierre Millot Date: Tue, 11 Jan 2022 13:15:13 +0100 Subject: [PATCH 13/49] using openapitools.json --- .github/actions/setup/action.yml | 17 ++++++++++++++++- .github/workflows/check.yml | 6 ------ 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 61940b7ae79..25f3920c887 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -62,7 +62,22 @@ runs: id: set-matrix shell: bash run: | - echo ::set-output name=SPECS_MATRIX::${{ toJSON(join([(steps.tmpvar.outputs.BASE_SPEC || steps.diff.outputs.SEARCH_SPECS_CHANGED) && 'search', (steps.tmpvar.outputs.BASE_SPEC || steps.diff.outputs.RECOMMENT_SPECS_CHANGED) && 'recommend'])) }} + specs="" + specs_openapi=( $(cat openapitools.json | jq -r '."generator-cli".generators[] | .glob') ) + + for spec in "${specs_openapi[@]}"; do + if [[ ! ${specs[*]} =~ $spec ]]; then + specs="$specs $spec" + fi + done + specs_matrix=$(( + echo '{ "client" : [' + echo $specs | xargs | sed 's/ /, /g' | xargs -n 1 echo | sed -r 's/^([^,]*)(,?)$/"\1"\2/' + echo " ]}" + ) | jq -c .) + echo $specs_matrix + echo $specs_matrix | jq . + echo "::set-output name=SPECS_MATRIX::$specs_matrix" outputs: # specs variables diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 583bca8ae93..d221946a29a 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -23,12 +23,6 @@ jobs: id: setup uses: ./.github/actions/setup - - name: Log outputs - shell: bash - run: | - echo "Hello world !" - echo ${{ steps.setup.outputs.SPECS_MATRIX}} - - name: Lint GitHub actions run: yarn eslint --ext=yml .github/actions .github/workflows From 29e01e7875bc9e18d19f41363cc9d066d8e77731 Mon Sep 17 00:00:00 2001 From: Pierre Millot Date: Tue, 11 Jan 2022 13:25:34 +0100 Subject: [PATCH 14/49] test with git --- .github/actions/setup/action.yml | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 25f3920c887..94f9d576995 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -62,18 +62,25 @@ runs: id: set-matrix shell: bash run: | + # Read from openapitools.json and put each client in the specs array specs="" - specs_openapi=( $(cat openapitools.json | jq -r '."generator-cli".generators[] | .glob') ) - - for spec in "${specs_openapi[@]}"; do - if [[ ! ${specs[*]} =~ $spec ]]; then - specs="$specs $spec" + generators=( $(cat openapitools.json | jq '."generator-cli".generators' | jq -r 'keys[]') ) + + for generator in "${generators[@]}"; do + client=${generator#*-} + if [[ ! ${specs[*]} =~ $client ]]; then + changed=$(git diff --shortstat origin/${{ github.base_ref }}..HEAD -- specs/$client | wc -l) + if [[ $changed > 0 ]]: then + specs="$specs $client" + fi fi done + + # Convert the array to json for the matrix specs_matrix=$(( - echo '{ "client" : [' - echo $specs | xargs | sed 's/ /, /g' | xargs -n 1 echo | sed -r 's/^([^,]*)(,?)$/"\1"\2/' - echo " ]}" + echo '{ "client" : [' + echo $specs | xargs | sed 's/ /, /g' | xargs -n 1 echo | sed -r 's/^([^,]*)(,?)$/"\1"\2/' + echo " ]}" ) | jq -c .) echo $specs_matrix echo $specs_matrix | jq . From e0449cbb18810c2fbfaa84deaef5f40cbb32e630 Mon Sep 17 00:00:00 2001 From: Pierre Millot Date: Tue, 11 Jan 2022 13:29:05 +0100 Subject: [PATCH 15/49] typo --- .github/actions/setup/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 94f9d576995..1d475ea2fee 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -70,7 +70,7 @@ runs: client=${generator#*-} if [[ ! ${specs[*]} =~ $client ]]; then changed=$(git diff --shortstat origin/${{ github.base_ref }}..HEAD -- specs/$client | wc -l) - if [[ $changed > 0 ]]: then + if [[ $changed > 0 ]]; then specs="$specs $client" fi fi From 50254f97bb41ac9af197afe54c83d2eafc690692 Mon Sep 17 00:00:00 2001 From: Pierre Millot Date: Tue, 11 Jan 2022 13:35:00 +0100 Subject: [PATCH 16/49] try with empty array --- .github/actions/setup/action.yml | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 1d475ea2fee..824d441e2e6 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -77,13 +77,16 @@ runs: done # Convert the array to json for the matrix - specs_matrix=$(( - echo '{ "client" : [' - echo $specs | xargs | sed 's/ /, /g' | xargs -n 1 echo | sed -r 's/^([^,]*)(,?)$/"\1"\2/' - echo " ]}" - ) | jq -c .) + if [[ $specs == "" ]]; then + specs_matrix='{"client":[]}' + else + specs_matrix=$(( + echo '{ "client" : [' + echo $specs | xargs | sed 's/ /, /g' | xargs -n 1 echo | sed -r 's/^([^,]*)(,?)$/"\1"\2/' + echo " ]}" + ) | jq -c .) + fi echo $specs_matrix - echo $specs_matrix | jq . echo "::set-output name=SPECS_MATRIX::$specs_matrix" outputs: From 46a58822a0c0dcdabb0e8c341853d805c7796e1d Mon Sep 17 00:00:00 2001 From: Pierre Millot Date: Tue, 11 Jan 2022 13:44:56 +0100 Subject: [PATCH 17/49] skip if no specs --- .github/actions/setup/action.yml | 70 +++----------------------------- .github/workflows/check.yml | 21 ++-------- 2 files changed, 10 insertions(+), 81 deletions(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 824d441e2e6..429962d6a95 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -29,29 +29,11 @@ runs: echo "::set-output name=SPECS_CHANGED::$(git diff --shortstat origin/${{ github.base_ref }}..HEAD -- specs | wc -l)" echo "::set-output name=COMMON_SPECS_CHANGED::$(git diff --shortstat origin/${{ github.base_ref }}..HEAD -- specs/common | wc -l)" - echo "::set-output name=SEARCH_SPECS_CHANGED::$(git diff --shortstat origin/${{ github.base_ref }}..HEAD -- specs/search | wc -l)" - echo "::set-output name=RECOMMEND_SPECS_CHANGED::$(git diff --shortstat origin/${{ github.base_ref }}..HEAD -- specs/recommend | wc -l)" - echo "::set-output name=QS_SPECS_CHANGED::$(git diff --shortstat origin/${{ github.base_ref }}..HEAD -- specs/query_suggestions | wc -l)" - echo "::set-output name=PERSO_SPECS_CHANGED::$(git diff --shortstat origin/${{ github.base_ref }}..HEAD -- specs/personalization | wc -l)" - echo "::set-output name=INSIGHTS_SPECS_CHANGED::$(git diff --shortstat origin/${{ github.base_ref }}..HEAD -- specs/insights | wc -l)" - echo "::set-output name=ANALYTICS_SPECS_CHANGED::$(git diff --shortstat origin/${{ github.base_ref }}..HEAD -- specs/analytics | wc -l)" - echo "::set-output name=AB_SPECS_CHANGED::$(git diff --shortstat origin/${{ github.base_ref }}..HEAD -- specs/abtesting | wc -l)" echo "::set-output name=TESTS_CHANGED::$(git diff --shortstat origin/${{ github.base_ref }}..HEAD -- tests | wc -l)" echo "::set-output name=SCRIPTS_CHANGED::$(git diff --shortstat origin/${{ github.base_ref }}..HEAD -- scripts | wc -l)" - echo "::set-output name=JS_CLIENT_CHANGED::$(git diff --shortstat origin/${{ github.base_ref }}..HEAD -- clients/algoliasearch-client-javascript | wc -l)" - echo "::set-output name=JS_SEARCH_CLIENT_CHANGED::$(git diff --shortstat origin/${{ github.base_ref }}..HEAD -- clients/algoliasearch-client-javascript/client-search | wc -l)" - echo "::set-output name=JS_RECOMMEND_CLIENT_CHANGED::$(git diff --shortstat origin/${{ github.base_ref }}..HEAD -- clients/algoliasearch-client-javascript/recommend | wc -l)" - echo "::set-output name=JS_PERSO_CLIENT_CHANGED::$(git diff --shortstat origin/${{ github.base_ref }}..HEAD -- clients/algoliasearch-client-javascript/client-personalization | wc -l)" - echo "::set-output name=JS_ANALYTICS_CLIENT_CHANGED::$(git diff --shortstat origin/${{ github.base_ref }}..HEAD -- clients/algoliasearch-client-javascript/client-analytics | wc -l)" - echo "::set-output name=JS_INSIGHTS_CLIENT_CHANGED::$(git diff --shortstat origin/${{ github.base_ref }}..HEAD -- clients/algoliasearch-client-javascript/client-insights | wc -l)" - echo "::set-output name=JS_TEMPLATE_CHANGED::$(git diff --shortstat origin/${{ github.base_ref }}..HEAD -- templates/javascript | wc -l)" - - echo "::set-output name=JAVA_CLIENT_CHANGED::$(git diff --shortstat origin/${{ github.base_ref }}..HEAD -- clients/algoliasearch-client-java-2 | wc -l)" - echo "::set-output name=JAVA_TEMPLATE_CHANGED::$(git diff --shortstat origin/${{ github.base_ref }}..HEAD -- templates/java | wc -l)" - - name: Set temporary variables id: tmpvar shell: bash @@ -79,63 +61,23 @@ runs: # Convert the array to json for the matrix if [[ $specs == "" ]]; then specs_matrix='{"client":[]}' + echo ::set-output name=RUN_SPECS::1 else specs_matrix=$(( echo '{ "client" : [' echo $specs | xargs | sed 's/ /, /g' | xargs -n 1 echo | sed -r 's/^([^,]*)(,?)$/"\1"\2/' echo " ]}" ) | jq -c .) + echo ::set-output name=RUN_SPECS::0 fi echo $specs_matrix - echo "::set-output name=SPECS_MATRIX::$specs_matrix" + echo ::set-output name=SPECS_MATRIX::$specs_matrix outputs: - # specs variables - RUN_SPECS_SEARCH: - description: Determine if the `specs_search` job should run - value: ${{ github.ref == 'refs/heads/main' || steps.diff.outputs.GITHUB_ACTIONS_CHANGED > 0 || steps.diff.outputs.SCRIPTS_CHANGED > 0 || steps.diff.outputs.COMMON_SPECS_CHANGED > 0 || steps.diff.outputs.SEARCH_SPECS_CHANGED > 0 }} - RUN_SPECS_RECOMMEND: - description: Determine if the `specs_recommend` job should run - value: ${{ github.ref == 'refs/heads/main' || steps.diff.outputs.GITHUB_ACTIONS_CHANGED > 0 || steps.diff.outputs.SCRIPTS_CHANGED > 0 || steps.diff.outputs.COMMON_SPECS_CHANGED > 0 || steps.diff.outputs.RECOMMEND_SPECS_CHANGED > 0 }} - RUN_SPECS_QS: - description: Determine if the `specs_qs` job should run - value: ${{ github.ref == 'refs/heads/main' || steps.diff.outputs.GITHUB_ACTIONS_CHANGED > 0 || steps.diff.outputs.SCRIPTS_CHANGED > 0 || steps.diff.outputs.COMMON_SPECS_CHANGED > 0 || steps.diff.outputs.QS_SPECS_CHANGED > 0 }} - RUN_SPECS_PERSO: - description: Determine if the `specs_perso` job should run - value: ${{ github.ref == 'refs/heads/main' || steps.diff.outputs.GITHUB_ACTIONS_CHANGED > 0 || steps.diff.outputs.SCRIPTS_CHANGED > 0 || steps.diff.outputs.COMMON_SPECS_CHANGED > 0 || steps.diff.outputs.PERSO_SPECS_CHANGED > 0 }} - RUN_SPECS_INSIGHTS: - description: Determine if the `specs_insights` job should run - value: ${{ github.ref == 'refs/heads/main' || steps.diff.outputs.GITHUB_ACTIONS_CHANGED > 0 || steps.diff.outputs.SCRIPTS_CHANGED > 0 || steps.diff.outputs.COMMON_SPECS_CHANGED > 0 || steps.diff.outputs.INSIGHTS_SPECS_CHANGED > 0 }} - RUN_SPECS_ANALYTICS: - description: Determine if the `specs_analytics` job should run - value: ${{ github.ref == 'refs/heads/main' || steps.diff.outputs.GITHUB_ACTIONS_CHANGED > 0 || steps.diff.outputs.SCRIPTS_CHANGED > 0 || steps.diff.outputs.COMMON_SPECS_CHANGED > 0 || steps.diff.outputs.ANALYTICS_SPECS_CHANGED > 0 }} - RUN_SPECS_AB: - description: Determine if the `specs_ab` job should run - value: ${{ github.ref == 'refs/heads/main' || steps.diff.outputs.GITHUB_ACTIONS_CHANGED > 0 || steps.diff.outputs.SCRIPTS_CHANGED > 0 || steps.diff.outputs.COMMON_SPECS_CHANGED > 0 || steps.diff.outputs.AB_SPECS_CHANGED > 0 }} - - # js client variables - RUN_JS_CLIENT_SEARCH: - description: Determine if the `client_javascript_search` job should run - value: ${{ github.ref == 'refs/heads/main' || steps.diff.outputs.GITHUB_ACTIONS_CHANGED > 0 || steps.diff.outputs.COMMON_SPECS_CHANGED > 0 || steps.diff.outputs.SEARCH_SPECS_CHANGED > 0 || steps.diff.outputs.SCRIPTS_CHANGED > 0 || steps.diff.outputs.JS_SEARCH_CLIENT_CHANGED > 0 || steps.diff.outputs.JS_TEMPLATE_CHANGED > 0 }} - RUN_JS_CLIENT_RECOMMEND: - description: Determine if the `client_javascript_recommend` job should run - value: ${{ github.ref == 'refs/heads/main' || steps.diff.outputs.GITHUB_ACTIONS_CHANGED > 0 || steps.diff.outputs.COMMON_SPECS_CHANGED > 0 || steps.diff.outputs.RECOMMEND_SPECS_CHANGED > 0 || steps.diff.outputs.SCRIPTS_CHANGED > 0 || steps.diff.outputs.JS_RECOMMEND_CLIENT_CHANGED > 0 || steps.diff.outputs.JS_TEMPLATE_CHANGED > 0 }} - RUN_JS_CLIENT_PERSO: - description: Determine if the `client_javascript_perso` job should run - value: ${{ github.ref == 'refs/heads/main' || steps.diff.outputs.GITHUB_ACTIONS_CHANGED > 0 || steps.diff.outputs.COMMON_SPECS_CHANGED > 0 || steps.diff.outputs.PERSO_SPECS_CHANGED > 0 || steps.diff.outputs.SCRIPTS_CHANGED > 0 || steps.diff.outputs.JS_PERSO_CLIENT_CHANGED > 0 || steps.diff.outputs.JS_TEMPLATE_CHANGED > 0 }} - RUN_JS_CLIENT_ANALYTICS: - description: Determine if the `client_javascript_analytics` job should run - value: ${{ github.ref == 'refs/heads/main' || steps.diff.outputs.GITHUB_ACTIONS_CHANGED > 0 || steps.diff.outputs.COMMON_SPECS_CHANGED > 0 || steps.diff.outputs.ANALYTICS_SPECS_CHANGED > 0 || steps.diff.outputs.SCRIPTS_CHANGED > 0 || steps.diff.outputs.JS_ANALYTICS_CLIENT_CHANGED > 0 || steps.diff.outputs.JS_TEMPLATE_CHANGED > 0 }} - RUN_JS_CLIENT_INSIGHTS: - description: Determine if the `client_javascript_insights` job should run - value: ${{ github.ref == 'refs/heads/main' || steps.diff.outputs.GITHUB_ACTIONS_CHANGED > 0 || steps.diff.outputs.COMMON_SPECS_CHANGED > 0 || steps.diff.outputs.INSIGHTS_SPECS_CHANGED > 0 || steps.diff.outputs.SCRIPTS_CHANGED > 0 || steps.diff.outputs.JS_INSIGHTS_CLIENT_CHANGED > 0 || steps.diff.outputs.JS_TEMPLATE_CHANGED > 0 }} - - # java client variables - RUN_JAVA_CLIENT: - description: Determine if the `client_java_*` job should run - value: ${{ github.ref == 'refs/heads/main' || steps.diff.outputs.GITHUB_ACTIONS_CHANGED > 0 || steps.diff.outputs.COMMON_SPECS_CHANGED > 0 || steps.diff.outputs.SEARCH_SPECS_CHANGED > 0 || steps.diff.outputs.SCRIPTS_CHANGED > 0 || steps.diff.outputs.JAVA_CLIENT_CHANGED > 0 || steps.diff.outputs.JAVA_TEMPLATE_CHANGED > 0 }} + RUN_SPECS: + description: Whether to run specs or not + value: ${{ steps.set-matrix.outputs.RUN_SPECS }} - # cts variables RUN_CTS: description: Determine if the `cts` job should run value: ${{ github.ref_name == 'main' || steps.diff.outputs.GITHUB_ACTIONS_CHANGED > 0 || steps.diff.outputs.SPECS_CHANGED > 0 || steps.diff.outputs.TESTS_CHANGED > 0 || steps.diff.outputs.JS_CLIENT_CHANGED > 0 || steps.diff.outputs.JS_TEMPLATE_CHANGED > 0 || steps.diff.outputs.JAVA_CLIENT_CHANGED > 0 || steps.diff.outputs.JAVA_TEMPLATE_CHANGED > 0 }} diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index d221946a29a..b83c52d7fa9 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -27,30 +27,17 @@ jobs: run: yarn eslint --ext=yml .github/actions .github/workflows outputs: - RUN_SPECS_SEARCH: ${{ steps.setup.outputs.RUN_SPECS_SEARCH }} - RUN_SPECS_RECOMMEND: ${{ steps.setup.outputs.RUN_SPECS_RECOMMEND }} - RUN_SPECS_QS: ${{ steps.setup.outputs.RUN_SPECS_QS }} - RUN_SPECS_PERSO: ${{ steps.setup.outputs.RUN_SPECS_PERSO }} - RUN_SPECS_INSIGHTS: ${{ steps.setup.outputs.RUN_SPECS_INSIGHTS }} - RUN_SPECS_ANALYTICS: ${{ steps.setup.outputs.RUN_SPECS_ANALYTICS }} - RUN_SPECS_AB: ${{ steps.setup.outputs.RUN_SPECS_AB }} - - RUN_JS_CLIENT_SEARCH: ${{ steps.setup.outputs.RUN_JS_CLIENT_SEARCH }} - RUN_JS_CLIENT_RECOMMEND: ${{ steps.setup.outputs.RUN_JS_CLIENT_RECOMMEND }} - RUN_JS_CLIENT_PERSO: ${{ steps.setup.outputs.RUN_JS_CLIENT_PERSO }} - RUN_JS_CLIENT_ANALYTICS: ${{ steps.setup.outputs.RUN_JS_CLIENT_ANALYTICS }} - RUN_JS_CLIENT_INSIGHTS: ${{ steps.setup.outputs.RUN_JS_CLIENT_INSIGHTS }} - - RUN_JAVA_CLIENT: ${{ steps.setup.outputs.RUN_JAVA_CLIENT }} - + RUN_SPECS: ${{ steps.setup.outputs.RUN_SPECS }} RUN_CTS: ${{ steps.setup.outputs.RUN_CTS }} + SPECS_MATRIX: ${{ steps.setup.outputs.SPECS_MATRIX }} specs: runs-on: ubuntu-20.04 needs: setup + if: ${{ always() && needs.setup.outputs.RUN_SPECS }} strategy: - matrix: ${{ fromJSON(needs.setup.outputs.SPECS_MATRIX)}} + matrix: ${{ fromJSON(needs.setup.outputs.SPECS_MATRIX) }} steps: - uses: actions/checkout@v2 From 6d126a277786d66b8d2bebc8cdb1ca655305d949 Mon Sep 17 00:00:00 2001 From: Pierre Millot Date: Tue, 11 Jan 2022 13:46:52 +0100 Subject: [PATCH 18/49] fix --- .github/actions/setup/action.yml | 2 +- .github/workflows/check.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 429962d6a95..05c16a522ee 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -60,7 +60,7 @@ runs: # Convert the array to json for the matrix if [[ $specs == "" ]]; then - specs_matrix='{"client":[]}' + specs_matrix='{"client":["no-run"]}' echo ::set-output name=RUN_SPECS::1 else specs_matrix=$(( diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index b83c52d7fa9..09fa9973d09 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -35,7 +35,7 @@ jobs: specs: runs-on: ubuntu-20.04 needs: setup - if: ${{ always() && needs.setup.outputs.RUN_SPECS }} + if: ${{ always() && needs.setup.outputs.RUN_SPECS > 0 }} strategy: matrix: ${{ fromJSON(needs.setup.outputs.SPECS_MATRIX) }} steps: From b481de7e37352ce6f39d23222d88a4edacacf064 Mon Sep 17 00:00:00 2001 From: Pierre Millot Date: Tue, 11 Jan 2022 13:48:57 +0100 Subject: [PATCH 19/49] swap 1 and 0 --- .github/actions/setup/action.yml | 6 +++--- .github/workflows/check.yml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 05c16a522ee..aec3fc67e7c 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -60,15 +60,15 @@ runs: # Convert the array to json for the matrix if [[ $specs == "" ]]; then - specs_matrix='{"client":["no-run"]}' - echo ::set-output name=RUN_SPECS::1 + specs_matrix='{"client":[]}' + echo ::set-output name=RUN_SPECS::0 else specs_matrix=$(( echo '{ "client" : [' echo $specs | xargs | sed 's/ /, /g' | xargs -n 1 echo | sed -r 's/^([^,]*)(,?)$/"\1"\2/' echo " ]}" ) | jq -c .) - echo ::set-output name=RUN_SPECS::0 + echo ::set-output name=RUN_SPECS::1 fi echo $specs_matrix echo ::set-output name=SPECS_MATRIX::$specs_matrix diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 09fa9973d09..b83c52d7fa9 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -35,7 +35,7 @@ jobs: specs: runs-on: ubuntu-20.04 needs: setup - if: ${{ always() && needs.setup.outputs.RUN_SPECS > 0 }} + if: ${{ always() && needs.setup.outputs.RUN_SPECS }} strategy: matrix: ${{ fromJSON(needs.setup.outputs.SPECS_MATRIX) }} steps: From 7b2af63f7bb5dcdddb8f4c2b65166f0673148f14 Mon Sep 17 00:00:00 2001 From: Pierre Millot Date: Tue, 11 Jan 2022 13:52:07 +0100 Subject: [PATCH 20/49] log run --- .github/actions/setup/action.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index aec3fc67e7c..e2303861438 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -60,18 +60,20 @@ runs: # Convert the array to json for the matrix if [[ $specs == "" ]]; then - specs_matrix='{"client":[]}' - echo ::set-output name=RUN_SPECS::0 + specs_matrix='{"client":["no-run"]}' + run_specs=0 else specs_matrix=$(( echo '{ "client" : [' echo $specs | xargs | sed 's/ /, /g' | xargs -n 1 echo | sed -r 's/^([^,]*)(,?)$/"\1"\2/' echo " ]}" ) | jq -c .) - echo ::set-output name=RUN_SPECS::1 + run_specs=1 fi echo $specs_matrix + echo "Run specs: $run_specs" echo ::set-output name=SPECS_MATRIX::$specs_matrix + echo ::set-output name=RUN_SPECS::$run_specs outputs: RUN_SPECS: From 47c0e615e7e83d9bde600be4ed692c08fc573755 Mon Sep 17 00:00:00 2001 From: Pierre Millot Date: Tue, 11 Jan 2022 13:53:48 +0100 Subject: [PATCH 21/49] condition --- .github/workflows/check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index b83c52d7fa9..09fa9973d09 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -35,7 +35,7 @@ jobs: specs: runs-on: ubuntu-20.04 needs: setup - if: ${{ always() && needs.setup.outputs.RUN_SPECS }} + if: ${{ always() && needs.setup.outputs.RUN_SPECS > 0 }} strategy: matrix: ${{ fromJSON(needs.setup.outputs.SPECS_MATRIX) }} steps: From 8eb47765a029271ceb75477fb8c589b01e3fe60a Mon Sep 17 00:00:00 2001 From: Pierre Millot Date: Tue, 11 Jan 2022 13:57:43 +0100 Subject: [PATCH 22/49] working? --- .github/actions/setup/action.yml | 25 +++++++++++-------------- .github/workflows/check.yml | 4 ++-- 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index e2303861438..397f1685035 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -34,25 +34,21 @@ runs: echo "::set-output name=SCRIPTS_CHANGED::$(git diff --shortstat origin/${{ github.base_ref }}..HEAD -- scripts | wc -l)" - - name: Set temporary variables - id: tmpvar - shell: bash - run: | - echo ::set-output name=BASE_SPEC::${{ github.ref == 'refs/heads/main' || steps.diff.outputs.GITHUB_ACTIONS_CHANGED > 0 || steps.diff.outputs.SCRIPTS_CHANGED > 0 || steps.diff.outputs.COMMON_SPECS_CHANGED > 0 }} - - - name: matrix - id: set-matrix + - name: Compute specs matrix + id: spec-matrix shell: bash run: | # Read from openapitools.json and put each client in the specs array specs="" generators=( $(cat openapitools.json | jq '."generator-cli".generators' | jq -r 'keys[]') ) + base_changed=${{ github.ref == 'refs/heads/main' || steps.diff.outputs.GITHUB_ACTIONS_CHANGED > 0 || steps.diff.outputs.SCRIPTS_CHANGED > 0 || steps.diff.outputs.COMMON_SPECS_CHANGED > 0 }} + for generator in "${generators[@]}"; do client=${generator#*-} if [[ ! ${specs[*]} =~ $client ]]; then changed=$(git diff --shortstat origin/${{ github.base_ref }}..HEAD -- specs/$client | wc -l) - if [[ $changed > 0 ]]; then + if [[ $base_changed || $changed > 0 ]]; then specs="$specs $client" fi fi @@ -60,6 +56,7 @@ runs: # Convert the array to json for the matrix if [[ $specs == "" ]]; then + # client cannot be empty or the matrix will fail specs_matrix='{"client":["no-run"]}' run_specs=0 else @@ -78,12 +75,12 @@ runs: outputs: RUN_SPECS: description: Whether to run specs or not - value: ${{ steps.set-matrix.outputs.RUN_SPECS }} + value: ${{ steps.spec-matrix.outputs.RUN_SPECS }} + + SPECS_MATRIX: + description: Generate the matrix for specs + value: ${{ steps.spec-matrix.outputs.SPECS_MATRIX }} RUN_CTS: description: Determine if the `cts` job should run value: ${{ github.ref_name == 'main' || steps.diff.outputs.GITHUB_ACTIONS_CHANGED > 0 || steps.diff.outputs.SPECS_CHANGED > 0 || steps.diff.outputs.TESTS_CHANGED > 0 || steps.diff.outputs.JS_CLIENT_CHANGED > 0 || steps.diff.outputs.JS_TEMPLATE_CHANGED > 0 || steps.diff.outputs.JAVA_CLIENT_CHANGED > 0 || steps.diff.outputs.JAVA_TEMPLATE_CHANGED > 0 }} - - SPECS_MATRIX: - description: Generate the matrix for specs - value: ${{ steps.set-matrix.outputs.SPECS_MATRIX }} diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 09fa9973d09..e417b550845 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -28,10 +28,10 @@ jobs: outputs: RUN_SPECS: ${{ steps.setup.outputs.RUN_SPECS }} - RUN_CTS: ${{ steps.setup.outputs.RUN_CTS }} - SPECS_MATRIX: ${{ steps.setup.outputs.SPECS_MATRIX }} + RUN_CTS: ${{ steps.setup.outputs.RUN_CTS }} + specs: runs-on: ubuntu-20.04 needs: setup From c7a7bef5c8368eac2a0e9ee90165a9f4510fc996 Mon Sep 17 00:00:00 2001 From: Pierre Millot Date: Tue, 11 Jan 2022 15:33:27 +0100 Subject: [PATCH 23/49] add clients --- .github/actions/setup/action.yml | 53 ++++++++++++++++++++++++++++++-- .github/workflows/check.yml | 38 ++++++++++++++++++++++- 2 files changed, 88 insertions(+), 3 deletions(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 397f1685035..0b2e63b8328 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -34,6 +34,9 @@ runs: echo "::set-output name=SCRIPTS_CHANGED::$(git diff --shortstat origin/${{ github.base_ref }}..HEAD -- scripts | wc -l)" + echo "::set-output name=JS_CLIENT_CHANGED::$(git diff --shortstat origin/${{ github.base_ref }}..HEAD -- clients/algoliasearch-client-javascript | wc -l)" + echo "::set-output name=JS_TEMPLATE_CHANGED::$(git diff --shortstat origin/${{ github.base_ref }}..HEAD -- templates/javascript | wc -l)" + - name: Compute specs matrix id: spec-matrix shell: bash @@ -58,20 +61,58 @@ runs: if [[ $specs == "" ]]; then # client cannot be empty or the matrix will fail specs_matrix='{"client":["no-run"]}' - run_specs=0 + run_specs="false" else specs_matrix=$(( echo '{ "client" : [' echo $specs | xargs | sed 's/ /, /g' | xargs -n 1 echo | sed -r 's/^([^,]*)(,?)$/"\1"\2/' echo " ]}" ) | jq -c .) - run_specs=1 + run_specs="true" fi echo $specs_matrix echo "Run specs: $run_specs" echo ::set-output name=SPECS_MATRIX::$specs_matrix echo ::set-output name=RUN_SPECS::$run_specs + - name: Compute JS client build matrix + id: js-matrix + shell: bash + run: | + # Read from openapitools.json to get the clients and folders (not pretty jq) + clients=$(cat openapitools.json | jq '."generator-cli".generators | with_entries(if (.key | test("javascript-.*")) then ({key:.key,value:.value}) else empty end) | to_entries | map({client:.key | sub("javascript-";""),folder:.value.output | sub("#{cwd}/";"")}) | .[]') + + base_changed=${{ github.ref == 'refs/heads/main' || steps.diff.outputs.GITHUB_ACTIONS_CHANGED > 0 || steps.diff.outputs.COMMON_SPECS_CHANGED > 0 || steps.diff.outputs.SCRIPTS_CHANGED > 0 || steps.diff.outputs.JS_TEMPLATE_CHANGED > 0 }} + + to_test='{"client": []}' + for pair in "${clients[@]}"; do + client=$(echo $pair | jq '.client') + folder=$(echo $pair | jq '.folder') + spec_changed=$(git diff --shortstat origin/${{ github.base_ref }}..HEAD -- specs/$client | wc -l) + client_changed=$(git diff --shortstat origin/${{ github.base_ref }}..HEAD -- $folder | wc -l) + if [[ $base_changed || $spec_changed > 0 || $client_changed > 0 ]]; then + to_test="$to_test $client" + fi + done + + # Convert the array to json for the matrix + if [[ $to_test == "" ]]; then + # client cannot be empty or the matrix will fail + js_matrix='{"client":["no-run"]}' + run_js="false" + else + js_matrix=$(( + echo '{ "client" : [' + echo $to_test | xargs | sed 's/ /, /g' | xargs -n 1 echo | sed -r 's/^([^,]*)(,?)$/"\1"\2/' + echo " ]}" + ) | jq -c .) + run_js="true" + fi + echo $js_matrix + echo "Run JS: $run_js" + echo ::set-output name=JS_MATRIX::$js_matrix + echo ::set-output name=RUN_JS::$run_js + outputs: RUN_SPECS: description: Whether to run specs or not @@ -81,6 +122,14 @@ outputs: description: Generate the matrix for specs value: ${{ steps.spec-matrix.outputs.SPECS_MATRIX }} + RUN_JS: + description: Whether to run specs or not + value: ${{ steps.js-matrix.outputs.RUN_JS }} + + JS_MATRIX: + description: Generate the matrix for specs + value: ${{ steps.js-matrix.outputs.JS_MATRIX }} + RUN_CTS: description: Determine if the `cts` job should run value: ${{ github.ref_name == 'main' || steps.diff.outputs.GITHUB_ACTIONS_CHANGED > 0 || steps.diff.outputs.SPECS_CHANGED > 0 || steps.diff.outputs.TESTS_CHANGED > 0 || steps.diff.outputs.JS_CLIENT_CHANGED > 0 || steps.diff.outputs.JS_TEMPLATE_CHANGED > 0 || steps.diff.outputs.JAVA_CLIENT_CHANGED > 0 || steps.diff.outputs.JAVA_TEMPLATE_CHANGED > 0 }} diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index e417b550845..2dd765b5b2a 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -30,12 +30,15 @@ jobs: RUN_SPECS: ${{ steps.setup.outputs.RUN_SPECS }} SPECS_MATRIX: ${{ steps.setup.outputs.SPECS_MATRIX }} + RUN_JS: ${{ steps.setup.outputs.RUN_JS }} + JS_MATRIX: ${{ steps.setup.outputs.JS_MATRIX }} + RUN_CTS: ${{ steps.setup.outputs.RUN_CTS }} specs: runs-on: ubuntu-20.04 needs: setup - if: ${{ always() && needs.setup.outputs.RUN_SPECS > 0 }} + if: ${{ always() && needs.setup.outputs.RUN_SPECS == 'true' }} strategy: matrix: ${{ fromJSON(needs.setup.outputs.SPECS_MATRIX) }} steps: @@ -49,3 +52,36 @@ jobs: - name: Lint ${{ matrix.client }} specs run: yarn eslint --ext=yml specs/${{ matrix.client }} + + client_javascript: + runs-on: ubuntu-20.04 + needs: [specs] + if: ${{ always() && needs.setup.outputs.RUN_JS == 'true' }} + strategy: + matrix: ${{ fromJSON(needs.setup.outputs.JS_MATRIX) }} + steps: + - uses: actions/checkout@v2 + + - name: Restore cache + uses: ./.github/actions/cache + + - name: Get client folder + id: folder + shell: bash + run: | + echo ::set-output name=CLIENT_FOLDER::$($(cat openapitools.json | jq '."generator-cli".generators."javascript-${{ matrix.client }}".output' | sed 's/#{cwd}\///g')) + + - name: Cache ${{ matrix.client }} client + id: cache + uses: actions/cache@v2 + with: + path: '/home/runner/work/api-clients-automation/api-clients-automation/${{ steps.folder.outputs.CLIENT_FOLDER }}/dist' + key: ${{ runner.os }}-js-client-${{ matrix.client }}-${{ hashFiles(format('{0}/**', steps.folder.outputs.CLIENT_FOLDER)) }} + + - name: Generate ${{ matrix.client }} client + if: steps.cache.outputs.cache-hit != 'true' + run: yarn generate javascript ${{ matrix.client }} + + - name: Build ${{ matrix.client }} client + if: steps.cache.outputs.cache-hit != 'true' + run: yarn build:clients javascript ${{ matrix.client }} From de11397af1f87fc2defaa07f688db76a14391e0d Mon Sep 17 00:00:00 2001 From: Pierre Millot Date: Tue, 11 Jan 2022 15:39:39 +0100 Subject: [PATCH 24/49] messed up --- .github/actions/setup/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 0b2e63b8328..e994daefd97 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -84,7 +84,7 @@ runs: base_changed=${{ github.ref == 'refs/heads/main' || steps.diff.outputs.GITHUB_ACTIONS_CHANGED > 0 || steps.diff.outputs.COMMON_SPECS_CHANGED > 0 || steps.diff.outputs.SCRIPTS_CHANGED > 0 || steps.diff.outputs.JS_TEMPLATE_CHANGED > 0 }} - to_test='{"client": []}' + to_test="" for pair in "${clients[@]}"; do client=$(echo $pair | jq '.client') folder=$(echo $pair | jq '.folder') @@ -127,7 +127,7 @@ outputs: value: ${{ steps.js-matrix.outputs.RUN_JS }} JS_MATRIX: - description: Generate the matrix for specs + description: Generate the matrix for javascript client build value: ${{ steps.js-matrix.outputs.JS_MATRIX }} RUN_CTS: From b1382d66e1950b366839e090712e29951d3e36fe Mon Sep 17 00:00:00 2001 From: Pierre Millot Date: Tue, 11 Jan 2022 15:59:26 +0100 Subject: [PATCH 25/49] use json --- .github/actions/setup/action.yml | 14 +++++--------- .github/workflows/check.yml | 22 ++++++++-------------- 2 files changed, 13 insertions(+), 23 deletions(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index e994daefd97..abd85c61da7 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -75,7 +75,7 @@ runs: echo ::set-output name=SPECS_MATRIX::$specs_matrix echo ::set-output name=RUN_SPECS::$run_specs - - name: Compute JS client build matrix + - name: Compute the JS client build matrix id: js-matrix shell: bash run: | @@ -84,28 +84,24 @@ runs: base_changed=${{ github.ref == 'refs/heads/main' || steps.diff.outputs.GITHUB_ACTIONS_CHANGED > 0 || steps.diff.outputs.COMMON_SPECS_CHANGED > 0 || steps.diff.outputs.SCRIPTS_CHANGED > 0 || steps.diff.outputs.JS_TEMPLATE_CHANGED > 0 }} - to_test="" + to_test='{"client": []}' for pair in "${clients[@]}"; do client=$(echo $pair | jq '.client') folder=$(echo $pair | jq '.folder') spec_changed=$(git diff --shortstat origin/${{ github.base_ref }}..HEAD -- specs/$client | wc -l) client_changed=$(git diff --shortstat origin/${{ github.base_ref }}..HEAD -- $folder | wc -l) if [[ $base_changed || $spec_changed > 0 || $client_changed > 0 ]]; then - to_test="$to_test $client" + to_test=$(echo $to_test | jq '.client |= .+ ['$pair']') fi done # Convert the array to json for the matrix - if [[ $to_test == "" ]]; then + if [[ $(echo $to_test | jq '.client | length') == 0 ]]; then # client cannot be empty or the matrix will fail js_matrix='{"client":["no-run"]}' run_js="false" else - js_matrix=$(( - echo '{ "client" : [' - echo $to_test | xargs | sed 's/ /, /g' | xargs -n 1 echo | sed -r 's/^([^,]*)(,?)$/"\1"\2/' - echo " ]}" - ) | jq -c .) + js_matrix=$to_test run_js="true" fi echo $js_matrix diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 2dd765b5b2a..f1791793d29 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -55,7 +55,7 @@ jobs: client_javascript: runs-on: ubuntu-20.04 - needs: [specs] + needs: specs if: ${{ always() && needs.setup.outputs.RUN_JS == 'true' }} strategy: matrix: ${{ fromJSON(needs.setup.outputs.JS_MATRIX) }} @@ -65,23 +65,17 @@ jobs: - name: Restore cache uses: ./.github/actions/cache - - name: Get client folder - id: folder - shell: bash - run: | - echo ::set-output name=CLIENT_FOLDER::$($(cat openapitools.json | jq '."generator-cli".generators."javascript-${{ matrix.client }}".output' | sed 's/#{cwd}\///g')) - - - name: Cache ${{ matrix.client }} client + - name: Cache ${{ matrix.client.client }} client id: cache uses: actions/cache@v2 with: - path: '/home/runner/work/api-clients-automation/api-clients-automation/${{ steps.folder.outputs.CLIENT_FOLDER }}/dist' - key: ${{ runner.os }}-js-client-${{ matrix.client }}-${{ hashFiles(format('{0}/**', steps.folder.outputs.CLIENT_FOLDER)) }} + path: '/home/runner/work/api-clients-automation/api-clients-automation/${{ matrix.client.folder }}/dist' + key: ${{ runner.os }}-js-client-${{ matrix.client.client }}-${{ hashFiles(format('{0}/**', matrix.client.folder)) }} - - name: Generate ${{ matrix.client }} client + - name: Generate ${{ matrix.client.client }} client if: steps.cache.outputs.cache-hit != 'true' - run: yarn generate javascript ${{ matrix.client }} + run: yarn generate javascript ${{ matrix.client.client }} - - name: Build ${{ matrix.client }} client + - name: Build ${{ matrix.client.client }} client if: steps.cache.outputs.cache-hit != 'true' - run: yarn build:clients javascript ${{ matrix.client }} + run: yarn build:clients javascript ${{ matrix.client.client }} From f2318f9ae14c1694907352116fa532936116111c Mon Sep 17 00:00:00 2001 From: Pierre Millot Date: Tue, 11 Jan 2022 16:04:10 +0100 Subject: [PATCH 26/49] argjson --- .github/actions/setup/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index abd85c61da7..fd6776d5320 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -91,7 +91,7 @@ runs: spec_changed=$(git diff --shortstat origin/${{ github.base_ref }}..HEAD -- specs/$client | wc -l) client_changed=$(git diff --shortstat origin/${{ github.base_ref }}..HEAD -- $folder | wc -l) if [[ $base_changed || $spec_changed > 0 || $client_changed > 0 ]]; then - to_test=$(echo $to_test | jq '.client |= .+ ['$pair']') + to_test=$(echo $to_test | jq --argjson pair $pair '.client |= .+ [$pair]') fi done From 94e9916da49fa08d0187b16c2a061a09a81df955 Mon Sep 17 00:00:00 2001 From: Pierre Millot Date: Tue, 11 Jan 2022 16:10:12 +0100 Subject: [PATCH 27/49] log --- .github/actions/setup/action.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index fd6776d5320..8fccff98241 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -86,6 +86,7 @@ runs: to_test='{"client": []}' for pair in "${clients[@]}"; do + echo $pair client=$(echo $pair | jq '.client') folder=$(echo $pair | jq '.folder') spec_changed=$(git diff --shortstat origin/${{ github.base_ref }}..HEAD -- specs/$client | wc -l) From 7aca413dcfc1219b5c0d97d46ad0b8f954af37fd Mon Sep 17 00:00:00 2001 From: Pierre Millot Date: Tue, 11 Jan 2022 16:14:25 +0100 Subject: [PATCH 28/49] iterate --- .github/actions/setup/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 8fccff98241..1fb6bd603c3 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -80,7 +80,7 @@ runs: shell: bash run: | # Read from openapitools.json to get the clients and folders (not pretty jq) - clients=$(cat openapitools.json | jq '."generator-cli".generators | with_entries(if (.key | test("javascript-.*")) then ({key:.key,value:.value}) else empty end) | to_entries | map({client:.key | sub("javascript-";""),folder:.value.output | sub("#{cwd}/";"")}) | .[]') + clients=$(cat openapitools.json | jq -c '."generator-cli".generators | with_entries(if (.key | test("javascript-.*")) then ({key:.key,value:.value}) else empty end) | to_entries | map({client:.key | sub("javascript-";""),folder:.value.output | sub("#{cwd}/";"")}) | .[]') base_changed=${{ github.ref == 'refs/heads/main' || steps.diff.outputs.GITHUB_ACTIONS_CHANGED > 0 || steps.diff.outputs.COMMON_SPECS_CHANGED > 0 || steps.diff.outputs.SCRIPTS_CHANGED > 0 || steps.diff.outputs.JS_TEMPLATE_CHANGED > 0 }} From 912606ad24d2eaedd226fa09e7a90a4e67027a3f Mon Sep 17 00:00:00 2001 From: Pierre Millot Date: Tue, 11 Jan 2022 16:20:04 +0100 Subject: [PATCH 29/49] simple iteration --- .github/actions/setup/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 1fb6bd603c3..b67c5ad2e07 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -85,7 +85,7 @@ runs: base_changed=${{ github.ref == 'refs/heads/main' || steps.diff.outputs.GITHUB_ACTIONS_CHANGED > 0 || steps.diff.outputs.COMMON_SPECS_CHANGED > 0 || steps.diff.outputs.SCRIPTS_CHANGED > 0 || steps.diff.outputs.JS_TEMPLATE_CHANGED > 0 }} to_test='{"client": []}' - for pair in "${clients[@]}"; do + for pair in $clients; do echo $pair client=$(echo $pair | jq '.client') folder=$(echo $pair | jq '.folder') From 1ef1c9eb4c4309498050d28929c6f0bd3402f6a2 Mon Sep 17 00:00:00 2001 From: Pierre Millot Date: Tue, 11 Jan 2022 16:24:41 +0100 Subject: [PATCH 30/49] compact --- .github/actions/setup/action.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index b67c5ad2e07..6e48366fd08 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -86,7 +86,6 @@ runs: to_test='{"client": []}' for pair in $clients; do - echo $pair client=$(echo $pair | jq '.client') folder=$(echo $pair | jq '.folder') spec_changed=$(git diff --shortstat origin/${{ github.base_ref }}..HEAD -- specs/$client | wc -l) @@ -102,7 +101,7 @@ runs: js_matrix='{"client":["no-run"]}' run_js="false" else - js_matrix=$to_test + js_matrix=$(echo $to_test | jq -c .) run_js="true" fi echo $js_matrix From 87835700a209f23105e07013a7f0f8b0399906bf Mon Sep 17 00:00:00 2001 From: Pierre Millot Date: Tue, 11 Jan 2022 16:40:51 +0100 Subject: [PATCH 31/49] escape --- .github/actions/setup/action.yml | 2 +- .github/workflows/check.yml | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 6e48366fd08..af31bcd912e 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -101,7 +101,7 @@ runs: js_matrix='{"client":["no-run"]}' run_js="false" else - js_matrix=$(echo $to_test | jq -c .) + js_matrix=$(echo $to_test | jq tostring) run_js="true" fi echo $js_matrix diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index f1791793d29..04e2e7d64ee 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -79,3 +79,22 @@ jobs: - name: Build ${{ matrix.client.client }} client if: steps.cache.outputs.cache-hit != 'true' run: yarn build:clients javascript ${{ matrix.client.client }} + + cts: + runs-on: ubuntu-20.04 + needs: client_javascript + if: ${{ always() && needs.setup.outputs.RUN_CTS == 'true' }} + steps: + - uses: actions/checkout@v2 + + - name: Restore cache + id: restore + uses: ./.github/actions/cache + with: + job: cts + + - name: Generate CTS + run: yarn cts:generate + + - name: Run CTS + run: yarn cts:test From b43aa35499ffcc4fe667ff9a7e0b276087293e9f Mon Sep 17 00:00:00 2001 From: Pierre Millot Date: Tue, 11 Jan 2022 17:46:01 +0100 Subject: [PATCH 32/49] try log --- .github/actions/setup/action.yml | 2 +- .github/workflows/check.yml | 14 ++++++++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index af31bcd912e..0de7dfda6cb 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -104,7 +104,7 @@ runs: js_matrix=$(echo $to_test | jq tostring) run_js="true" fi - echo $js_matrix + echo $js_matrix | jq echo "Run JS: $run_js" echo ::set-output name=JS_MATRIX::$js_matrix echo ::set-output name=RUN_JS::$run_js diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 04e2e7d64ee..475989dafa2 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -53,9 +53,17 @@ jobs: - name: Lint ${{ matrix.client }} specs run: yarn eslint --ext=yml specs/${{ matrix.client }} + - name: Log + shell: bash + run: | + echo ${{ needs.setup.outputs.JS_MATRIX }} + echo ${{ fromJSON(needs.setup.outputs.JS_MATRIX) }} + client_javascript: runs-on: ubuntu-20.04 - needs: specs + needs: + - setup + - specs if: ${{ always() && needs.setup.outputs.RUN_JS == 'true' }} strategy: matrix: ${{ fromJSON(needs.setup.outputs.JS_MATRIX) }} @@ -82,7 +90,9 @@ jobs: cts: runs-on: ubuntu-20.04 - needs: client_javascript + needs: + - setup + - client_javascript if: ${{ always() && needs.setup.outputs.RUN_CTS == 'true' }} steps: - uses: actions/checkout@v2 From 68d8e97863436e8c86fd6db448864a32e4705af8 Mon Sep 17 00:00:00 2001 From: Pierre Millot Date: Tue, 11 Jan 2022 18:22:45 +0100 Subject: [PATCH 33/49] try fixing json bug --- .github/actions/setup/action.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 0de7dfda6cb..a1fae0ee1ab 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -104,6 +104,7 @@ runs: js_matrix=$(echo $to_test | jq tostring) run_js="true" fi + js_matrix="{\"include\":[{\"project\":\"foo\",\"config\":\"Debug\"},{\"project\":\"bar\",\"config\":\"Release\"}]}" echo $js_matrix | jq echo "Run JS: $run_js" echo ::set-output name=JS_MATRIX::$js_matrix From c7e0289c52a8d1baa3b252a0f19a5a2b217d492e Mon Sep 17 00:00:00 2001 From: Pierre Millot Date: Tue, 11 Jan 2022 18:29:23 +0100 Subject: [PATCH 34/49] change name --- .github/actions/setup/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index a1fae0ee1ab..019aedbcce4 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -104,7 +104,7 @@ runs: js_matrix=$(echo $to_test | jq tostring) run_js="true" fi - js_matrix="{\"include\":[{\"project\":\"foo\",\"config\":\"Debug\"},{\"project\":\"bar\",\"config\":\"Release\"}]}" + js_matrix="{\"client\":[{\"client\":\"foo\",\"config\":\"Debug\"},{\"client\":\"bar\",\"config\":\"Release\"}]}" echo $js_matrix | jq echo "Run JS: $run_js" echo ::set-output name=JS_MATRIX::$js_matrix From e4b968e0d14dde4dd20d5c949bda4cf04aaa433e Mon Sep 17 00:00:00 2001 From: Pierre Millot Date: Tue, 11 Jan 2022 18:39:32 +0100 Subject: [PATCH 35/49] =?UTF-8?q?remove=20quotes=C3=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/actions/setup/action.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 019aedbcce4..d33b3c25ff7 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -101,10 +101,9 @@ runs: js_matrix='{"client":["no-run"]}' run_js="false" else - js_matrix=$(echo $to_test | jq tostring) + js_matrix=$(echo $to_test | jq tostring | sed -e 's/^"//' -e 's/"$//') run_js="true" fi - js_matrix="{\"client\":[{\"client\":\"foo\",\"config\":\"Debug\"},{\"client\":\"bar\",\"config\":\"Release\"}]}" echo $js_matrix | jq echo "Run JS: $run_js" echo ::set-output name=JS_MATRIX::$js_matrix From 77679843b145c692b9e6f138a04758133a303008 Mon Sep 17 00:00:00 2001 From: Pierre Millot Date: Tue, 11 Jan 2022 18:43:18 +0100 Subject: [PATCH 36/49] this is taking way too long --- .github/actions/setup/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index d33b3c25ff7..20780d7389b 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -104,7 +104,7 @@ runs: js_matrix=$(echo $to_test | jq tostring | sed -e 's/^"//' -e 's/"$//') run_js="true" fi - echo $js_matrix | jq + echo $js_matrix echo "Run JS: $run_js" echo ::set-output name=JS_MATRIX::$js_matrix echo ::set-output name=RUN_JS::$run_js From 6e35b595b06be4a8bfec86fd2df56799577684a3 Mon Sep 17 00:00:00 2001 From: Pierre Millot Date: Tue, 11 Jan 2022 18:50:13 +0100 Subject: [PATCH 37/49] check --- .github/actions/setup/action.yml | 6 +++--- .github/workflows/check.yml | 9 +++++++++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 20780d7389b..2cb2d48119a 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -101,12 +101,12 @@ runs: js_matrix='{"client":["no-run"]}' run_js="false" else - js_matrix=$(echo $to_test | jq tostring | sed -e 's/^"//' -e 's/"$//') + js_matrix=$(echo $to_test | jq -c) run_js="true" fi - echo $js_matrix + echo $js_matrix | jq echo "Run JS: $run_js" - echo ::set-output name=JS_MATRIX::$js_matrix + echo "::set-output name=JS_MATRIX::$js_matrix" echo ::set-output name=RUN_JS::$run_js outputs: diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 475989dafa2..6ceb0696342 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -23,6 +23,15 @@ jobs: id: setup uses: ./.github/actions/setup + - name: Check matrices + shell: bash + run: | + sudo npm install -g json2yaml + matrix='${{ steps.setup.outputs.JS_MATRIX }}' + echo $matrix + echo $matrix | jq . + echo $matrix | json2yaml + - name: Lint GitHub actions run: yarn eslint --ext=yml .github/actions .github/workflows From 6020cfa6964d5f1ae42bd8194509584d535352ab Mon Sep 17 00:00:00 2001 From: Pierre Millot Date: Tue, 11 Jan 2022 19:11:07 +0100 Subject: [PATCH 38/49] add java --- .github/actions/setup/action.yml | 53 +++++++++++++++++++++++++++++--- .github/workflows/check.yml | 53 ++++++++++++++++++++------------ 2 files changed, 83 insertions(+), 23 deletions(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 2cb2d48119a..03732ab132f 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -37,6 +37,9 @@ runs: echo "::set-output name=JS_CLIENT_CHANGED::$(git diff --shortstat origin/${{ github.base_ref }}..HEAD -- clients/algoliasearch-client-javascript | wc -l)" echo "::set-output name=JS_TEMPLATE_CHANGED::$(git diff --shortstat origin/${{ github.base_ref }}..HEAD -- templates/javascript | wc -l)" + echo "::set-output name=JAVA_CLIENT_CHANGED::$(git diff --shortstat origin/${{ github.base_ref }}..HEAD -- clients/algoliasearch-client-java-2 | wc -l)" + echo "::set-output name=JAVA_TEMPLATE_CHANGED::$(git diff --shortstat origin/${{ github.base_ref }}..HEAD -- templates/java | wc -l)" + - name: Compute specs matrix id: spec-matrix shell: bash @@ -72,8 +75,8 @@ runs: fi echo $specs_matrix echo "Run specs: $run_specs" - echo ::set-output name=SPECS_MATRIX::$specs_matrix - echo ::set-output name=RUN_SPECS::$run_specs + echo "::set-output name=SPECS_MATRIX::$specs_matrix" + echo "::set-output name=RUN_SPECS::$run_specs" - name: Compute the JS client build matrix id: js-matrix @@ -107,7 +110,41 @@ runs: echo $js_matrix | jq echo "Run JS: $run_js" echo "::set-output name=JS_MATRIX::$js_matrix" - echo ::set-output name=RUN_JS::$run_js + echo "::set-output name=RUN_JS::$run_js" + + - name: Compute the Java client build matrix + id: java-matrix + shell: bash + run: | + # Read from openapitools.json to get the clients and folders (not pretty jq) + clients=$(cat openapitools.json | jq -c '."generator-cli".generators | with_entries(if (.key | test("java-.*")) then ({key:.key,value:.value}) else empty end) | to_entries | map({client:.key | sub("javascript-";""),folder:.value.output | sub("#{cwd}/";"")}) | .[]') + + base_changed=${{ github.ref == 'refs/heads/main' || steps.diff.outputs.GITHUB_ACTIONS_CHANGED > 0 || steps.diff.outputs.COMMON_SPECS_CHANGED > 0 || steps.diff.outputs.SCRIPTS_CHANGED > 0 || steps.diff.outputs.JAVA_TEMPLATE_CHANGED > 0 }} + + to_test='{"client": []}' + for pair in $clients; do + client=$(echo $pair | jq '.client') + folder=$(echo $pair | jq '.folder') + spec_changed=$(git diff --shortstat origin/${{ github.base_ref }}..HEAD -- specs/$client | wc -l) + client_changed=$(git diff --shortstat origin/${{ github.base_ref }}..HEAD -- $folder | wc -l) + if [[ $base_changed || $spec_changed > 0 || $client_changed > 0 ]]; then + to_test=$(echo $to_test | jq --argjson pair $pair '.client |= .+ [$pair]') + fi + done + + # Convert the array to json for the matrix + if [[ $(echo $to_test | jq '.client | length') == 0 ]]; then + # client cannot be empty or the matrix will fail + java_matrix='{"client":["no-run"]}' + run_java="false" + else + java_matrix=$(echo $to_test | jq -c) + run_java="true" + fi + echo $java_matrix | jq + echo "Run Java: $run_java" + echo "::set-output name=JAVA_MATRIX::$java_matrix" + echo "::set-output name=RUN_JAVA::$run_java" outputs: RUN_SPECS: @@ -119,13 +156,21 @@ outputs: value: ${{ steps.spec-matrix.outputs.SPECS_MATRIX }} RUN_JS: - description: Whether to run specs or not + description: Whether to run js client or not value: ${{ steps.js-matrix.outputs.RUN_JS }} JS_MATRIX: description: Generate the matrix for javascript client build value: ${{ steps.js-matrix.outputs.JS_MATRIX }} + RUN_JAVA: + description: Whether to run java client or not + value: ${{ steps.js-matrix.outputs.RUN_JAVA }} + + JAVA_MATRIX: + description: Generate the matrix for javascript client build + value: ${{ steps.js-matrix.outputs.JAVA_MATRIX }} + RUN_CTS: description: Determine if the `cts` job should run value: ${{ github.ref_name == 'main' || steps.diff.outputs.GITHUB_ACTIONS_CHANGED > 0 || steps.diff.outputs.SPECS_CHANGED > 0 || steps.diff.outputs.TESTS_CHANGED > 0 || steps.diff.outputs.JS_CLIENT_CHANGED > 0 || steps.diff.outputs.JS_TEMPLATE_CHANGED > 0 || steps.diff.outputs.JAVA_CLIENT_CHANGED > 0 || steps.diff.outputs.JAVA_TEMPLATE_CHANGED > 0 }} diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 6ceb0696342..ce2122a3d4c 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -23,15 +23,6 @@ jobs: id: setup uses: ./.github/actions/setup - - name: Check matrices - shell: bash - run: | - sudo npm install -g json2yaml - matrix='${{ steps.setup.outputs.JS_MATRIX }}' - echo $matrix - echo $matrix | jq . - echo $matrix | json2yaml - - name: Lint GitHub actions run: yarn eslint --ext=yml .github/actions .github/workflows @@ -42,6 +33,9 @@ jobs: RUN_JS: ${{ steps.setup.outputs.RUN_JS }} JS_MATRIX: ${{ steps.setup.outputs.JS_MATRIX }} + RUN_JAVA: ${{ steps.setup.outputs.RUN_JAVA }} + JAVA_MATRIX: ${{ steps.setup.outputs.JAVA_MATRIX }} + RUN_CTS: ${{ steps.setup.outputs.RUN_CTS }} specs: @@ -62,17 +56,9 @@ jobs: - name: Lint ${{ matrix.client }} specs run: yarn eslint --ext=yml specs/${{ matrix.client }} - - name: Log - shell: bash - run: | - echo ${{ needs.setup.outputs.JS_MATRIX }} - echo ${{ fromJSON(needs.setup.outputs.JS_MATRIX) }} - client_javascript: runs-on: ubuntu-20.04 - needs: - - setup - - specs + needs: specs if: ${{ always() && needs.setup.outputs.RUN_JS == 'true' }} strategy: matrix: ${{ fromJSON(needs.setup.outputs.JS_MATRIX) }} @@ -97,11 +83,40 @@ jobs: if: steps.cache.outputs.cache-hit != 'true' run: yarn build:clients javascript ${{ matrix.client.client }} + client_java: + runs-on: ubuntu-20.04 + needs: specs + if: ${{ always() && needs.setup.outputs.RUN_JAVA == 'true' }} + strategy: + matrix: ${{ fromJSON(needs.setup.outputs.JAVA_MATRIX) }} + steps: + - uses: actions/checkout@v2 + + - name: Restore cache + uses: ./.github/actions/cache + with: + job: java + + - name: Cache ${{ matrix.client.client }} client + id: cache + uses: actions/cache@v2 + with: + path: '/home/runner/work/api-clients-automation/api-clients-automation/${{ matrix.client.folder }}/dist' + key: ${{ runner.os }}-java-client-${{ matrix.client.client }}-${{ hashFiles(format('{0}/**', matrix.client.folder)) }} + + - name: Generate ${{ matrix.client.client }} client + if: steps.cache.outputs.cache-hit != 'true' + run: yarn generate java ${{ matrix.client.client }} + + - name: Build ${{ matrix.client.client }} client + if: steps.cache.outputs.cache-hit != 'true' + run: yarn build:clients java ${{ matrix.client.client }} + cts: runs-on: ubuntu-20.04 needs: - - setup - client_javascript + - client_java if: ${{ always() && needs.setup.outputs.RUN_CTS == 'true' }} steps: - uses: actions/checkout@v2 From 893ec6faec7c92ba54b37180e593540f7c27b557 Mon Sep 17 00:00:00 2001 From: Pierre Millot Date: Wed, 12 Jan 2022 11:55:28 +0100 Subject: [PATCH 39/49] needs ? --- .github/workflows/check.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index ce2122a3d4c..1695f1a3e54 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -58,7 +58,9 @@ jobs: client_javascript: runs-on: ubuntu-20.04 - needs: specs + needs: + - setup + - specs if: ${{ always() && needs.setup.outputs.RUN_JS == 'true' }} strategy: matrix: ${{ fromJSON(needs.setup.outputs.JS_MATRIX) }} @@ -85,7 +87,9 @@ jobs: client_java: runs-on: ubuntu-20.04 - needs: specs + needs: + - setup + - specs if: ${{ always() && needs.setup.outputs.RUN_JAVA == 'true' }} strategy: matrix: ${{ fromJSON(needs.setup.outputs.JAVA_MATRIX) }} From 1b0a198373fb3ca8e9f46aeea4b3e64cc74cda16 Mon Sep 17 00:00:00 2001 From: Pierre Millot Date: Wed, 12 Jan 2022 14:29:58 +0100 Subject: [PATCH 40/49] use local script --- .github/actions/setup/action.yml | 37 ++++++---------------------- scripts/ci/create-matrix.sh | 42 ++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 30 deletions(-) create mode 100755 scripts/ci/create-matrix.sh diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 03732ab132f..fa06c2276c8 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -82,35 +82,12 @@ runs: id: js-matrix shell: bash run: | - # Read from openapitools.json to get the clients and folders (not pretty jq) - clients=$(cat openapitools.json | jq -c '."generator-cli".generators | with_entries(if (.key | test("javascript-.*")) then ({key:.key,value:.value}) else empty end) | to_entries | map({client:.key | sub("javascript-";""),folder:.value.output | sub("#{cwd}/";"")}) | .[]') - base_changed=${{ github.ref == 'refs/heads/main' || steps.diff.outputs.GITHUB_ACTIONS_CHANGED > 0 || steps.diff.outputs.COMMON_SPECS_CHANGED > 0 || steps.diff.outputs.SCRIPTS_CHANGED > 0 || steps.diff.outputs.JS_TEMPLATE_CHANGED > 0 }} - to_test='{"client": []}' - for pair in $clients; do - client=$(echo $pair | jq '.client') - folder=$(echo $pair | jq '.folder') - spec_changed=$(git diff --shortstat origin/${{ github.base_ref }}..HEAD -- specs/$client | wc -l) - client_changed=$(git diff --shortstat origin/${{ github.base_ref }}..HEAD -- $folder | wc -l) - if [[ $base_changed || $spec_changed > 0 || $client_changed > 0 ]]; then - to_test=$(echo $to_test | jq --argjson pair $pair '.client |= .+ [$pair]') - fi - done + read matrix run <<< $(./scripts/ci/create-matrix.sh javascript $base_changed ${{ github.base_ref }} | xargs) - # Convert the array to json for the matrix - if [[ $(echo $to_test | jq '.client | length') == 0 ]]; then - # client cannot be empty or the matrix will fail - js_matrix='{"client":["no-run"]}' - run_js="false" - else - js_matrix=$(echo $to_test | jq -c) - run_js="true" - fi - echo $js_matrix | jq - echo "Run JS: $run_js" - echo "::set-output name=JS_MATRIX::$js_matrix" - echo "::set-output name=RUN_JS::$run_js" + echo "::set-output name=MATRIX::$matrix" + echo "::set-output name=RUN::$run" - name: Compute the Java client build matrix id: java-matrix @@ -157,19 +134,19 @@ outputs: RUN_JS: description: Whether to run js client or not - value: ${{ steps.js-matrix.outputs.RUN_JS }} + value: ${{ steps.js-matrix.outputs.RUN }} JS_MATRIX: description: Generate the matrix for javascript client build - value: ${{ steps.js-matrix.outputs.JS_MATRIX }} + value: ${{ steps.js-matrix.outputs.MATRIX }} RUN_JAVA: description: Whether to run java client or not - value: ${{ steps.js-matrix.outputs.RUN_JAVA }} + value: ${{ steps.java-matrix.outputs.RUN_JAVA }} JAVA_MATRIX: description: Generate the matrix for javascript client build - value: ${{ steps.js-matrix.outputs.JAVA_MATRIX }} + value: ${{ steps.java-matrix.outputs.JAVA_MATRIX }} RUN_CTS: description: Determine if the `cts` job should run diff --git a/scripts/ci/create-matrix.sh b/scripts/ci/create-matrix.sh new file mode 100755 index 00000000000..23fb72500dc --- /dev/null +++ b/scripts/ci/create-matrix.sh @@ -0,0 +1,42 @@ +LANGUAGE=$1 +BASE_CHANGED=$2 +BASE_BRANCH=$3 + +clients=$(cat openapitools.json | jq --arg lang $LANGUAGE -c '."generator-cli".generators + | with_entries( + if (.key | test($lang + "-.*")) then + ({key:.key,value:.value}) + else + empty + end + ) + | to_entries + | map({ + client:.key | sub($lang + "-";""), + folder:.value.output | sub("#{cwd}/";"") + }) + | .[]') + +to_test='{"client": []}' +for pair in $clients; do + client=$(echo $pair | jq '.client') + folder=$(echo $pair | jq '.folder') + spec_changed=$(git diff --shortstat origin/$BASE_BRANCH..HEAD -- specs/$client | wc -l) + client_changed=$(git diff --shortstat origin/$BASE_BRANCH..HEAD -- $folder | wc -l) + if [[ $BASE_CHANGED || $spec_changed > 0 || $client_changed > 0 ]]; then + to_test=$(echo $to_test | jq --argjson pair $pair '.client |= .+ [$pair]') + fi +done + +# Convert the array to json for the matrix +if [[ $(echo $to_test | jq '.client | length') == 0 ]]; then + # client cannot be empty or the matrix will fail + matrix='{"client":["no-run"]}' + run="false" +else + matrix=$(echo $to_test | jq -c) + run="true" +fi + +echo $matrix +echo $run From b48ae9e15d4eb226542fe4ed158e277fe2c939d3 Mon Sep 17 00:00:00 2001 From: Pierre Millot Date: Wed, 12 Jan 2022 14:47:14 +0100 Subject: [PATCH 41/49] use -d --- .github/actions/setup/action.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index fa06c2276c8..3e0a47684e8 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -84,7 +84,10 @@ runs: run: | base_changed=${{ github.ref == 'refs/heads/main' || steps.diff.outputs.GITHUB_ACTIONS_CHANGED > 0 || steps.diff.outputs.COMMON_SPECS_CHANGED > 0 || steps.diff.outputs.SCRIPTS_CHANGED > 0 || steps.diff.outputs.JS_TEMPLATE_CHANGED > 0 }} - read matrix run <<< $(./scripts/ci/create-matrix.sh javascript $base_changed ${{ github.base_ref }} | xargs) + read -d "\n" matrix run <<< $(./scripts/ci/create-matrix.sh javascript $base_changed ${{ github.base_ref }}) + + echo $matrix + echo $matrix | jq echo "::set-output name=MATRIX::$matrix" echo "::set-output name=RUN::$run" @@ -94,7 +97,7 @@ runs: shell: bash run: | # Read from openapitools.json to get the clients and folders (not pretty jq) - clients=$(cat openapitools.json | jq -c '."generator-cli".generators | with_entries(if (.key | test("java-.*")) then ({key:.key,value:.value}) else empty end) | to_entries | map({client:.key | sub("javascript-";""),folder:.value.output | sub("#{cwd}/";"")}) | .[]') + clients=$(cat openapitools.json | jq -c '."generator-cli".generators | with_entries(if (.key | test("java-.*")) then ({key:.key,value:.value}) else empty end) | to_entries | map({client:.key | sub("java-";""),folder:.value.output | sub("#{cwd}/";"")}) | .[]') base_changed=${{ github.ref == 'refs/heads/main' || steps.diff.outputs.GITHUB_ACTIONS_CHANGED > 0 || steps.diff.outputs.COMMON_SPECS_CHANGED > 0 || steps.diff.outputs.SCRIPTS_CHANGED > 0 || steps.diff.outputs.JAVA_TEMPLATE_CHANGED > 0 }} From a760f437e3c1c35b43668704783cc0f0d97bf8f5 Mon Sep 17 00:00:00 2001 From: Pierre Millot Date: Wed, 12 Jan 2022 14:51:37 +0100 Subject: [PATCH 42/49] log --- .github/actions/setup/action.yml | 2 ++ scripts/ci/create-matrix.sh | 8 +++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 3e0a47684e8..a82737862fd 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -84,6 +84,8 @@ runs: run: | base_changed=${{ github.ref == 'refs/heads/main' || steps.diff.outputs.GITHUB_ACTIONS_CHANGED > 0 || steps.diff.outputs.COMMON_SPECS_CHANGED > 0 || steps.diff.outputs.SCRIPTS_CHANGED > 0 || steps.diff.outputs.JS_TEMPLATE_CHANGED > 0 }} + ./scripts/ci/create-matrix.sh javascript $base_changed ${{ github.base_ref }} + read -d "\n" matrix run <<< $(./scripts/ci/create-matrix.sh javascript $base_changed ${{ github.base_ref }}) echo $matrix diff --git a/scripts/ci/create-matrix.sh b/scripts/ci/create-matrix.sh index 23fb72500dc..81200dd9164 100755 --- a/scripts/ci/create-matrix.sh +++ b/scripts/ci/create-matrix.sh @@ -17,14 +17,20 @@ clients=$(cat openapitools.json | jq --arg lang $LANGUAGE -c '."generator-cli".g }) | .[]') +echo $clients + to_test='{"client": []}' for pair in $clients; do client=$(echo $pair | jq '.client') folder=$(echo $pair | jq '.folder') spec_changed=$(git diff --shortstat origin/$BASE_BRANCH..HEAD -- specs/$client | wc -l) client_changed=$(git diff --shortstat origin/$BASE_BRANCH..HEAD -- $folder | wc -l) + + echo $spec_changed + echo $client_changed + echo "why" if [[ $BASE_CHANGED || $spec_changed > 0 || $client_changed > 0 ]]; then - to_test=$(echo $to_test | jq --argjson pair $pair '.client |= .+ [$pair]') + to_test=$(echo $to_test | jq --argjson pair $pair '.client |= .+ [$pair]') fi done From f0b1e89c5a3acde912c8f6d5c730d95e88805c3b Mon Sep 17 00:00:00 2001 From: Pierre Millot Date: Wed, 12 Jan 2022 15:10:14 +0100 Subject: [PATCH 43/49] test split --- .github/actions/setup/action.yml | 6 ++++++ scripts/ci/create-matrix.sh | 6 ------ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index a82737862fd..5b631612573 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -86,6 +86,12 @@ runs: ./scripts/ci/create-matrix.sh javascript $base_changed ${{ github.base_ref }} + echo "can it split ?" + + read -d "\n" v1 v2 <<< "test1\ntest2" + echo $v1 + echo $v2 + read -d "\n" matrix run <<< $(./scripts/ci/create-matrix.sh javascript $base_changed ${{ github.base_ref }}) echo $matrix diff --git a/scripts/ci/create-matrix.sh b/scripts/ci/create-matrix.sh index 81200dd9164..e6ade391143 100755 --- a/scripts/ci/create-matrix.sh +++ b/scripts/ci/create-matrix.sh @@ -17,18 +17,12 @@ clients=$(cat openapitools.json | jq --arg lang $LANGUAGE -c '."generator-cli".g }) | .[]') -echo $clients - to_test='{"client": []}' for pair in $clients; do client=$(echo $pair | jq '.client') folder=$(echo $pair | jq '.folder') spec_changed=$(git diff --shortstat origin/$BASE_BRANCH..HEAD -- specs/$client | wc -l) client_changed=$(git diff --shortstat origin/$BASE_BRANCH..HEAD -- $folder | wc -l) - - echo $spec_changed - echo $client_changed - echo "why" if [[ $BASE_CHANGED || $spec_changed > 0 || $client_changed > 0 ]]; then to_test=$(echo $to_test | jq --argjson pair $pair '.client |= .+ [$pair]') fi From 70549dc4dc604ed68c9b6abbbbea5f11e62305d4 Mon Sep 17 00:00:00 2001 From: Pierre Millot Date: Wed, 12 Jan 2022 15:22:16 +0100 Subject: [PATCH 44/49] use one echo --- .github/actions/setup/action.yml | 14 ++++++-------- scripts/ci/create-matrix.sh | 6 +++--- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 5b631612573..393218d642a 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -84,15 +84,13 @@ runs: run: | base_changed=${{ github.ref == 'refs/heads/main' || steps.diff.outputs.GITHUB_ACTIONS_CHANGED > 0 || steps.diff.outputs.COMMON_SPECS_CHANGED > 0 || steps.diff.outputs.SCRIPTS_CHANGED > 0 || steps.diff.outputs.JS_TEMPLATE_CHANGED > 0 }} - ./scripts/ci/create-matrix.sh javascript $base_changed ${{ github.base_ref }} + matrix=$(./scripts/ci/create-matrix.sh javascript $base_changed ${{ github.base_ref }}) - echo "can it split ?" - - read -d "\n" v1 v2 <<< "test1\ntest2" - echo $v1 - echo $v2 - - read -d "\n" matrix run <<< $(./scripts/ci/create-matrix.sh javascript $base_changed ${{ github.base_ref }}) + if [[ $? == 0 ]]; then + run="false" + else + run="true" + fi echo $matrix echo $matrix | jq diff --git a/scripts/ci/create-matrix.sh b/scripts/ci/create-matrix.sh index e6ade391143..edf909f7e8b 100755 --- a/scripts/ci/create-matrix.sh +++ b/scripts/ci/create-matrix.sh @@ -32,11 +32,11 @@ done if [[ $(echo $to_test | jq '.client | length') == 0 ]]; then # client cannot be empty or the matrix will fail matrix='{"client":["no-run"]}' - run="false" + run=0 else matrix=$(echo $to_test | jq -c) - run="true" + run=1 fi echo $matrix -echo $run +exit $run From 1cf77be728688346cd0ddfad383b35482f34ee2d Mon Sep 17 00:00:00 2001 From: Pierre Millot Date: Wed, 12 Jan 2022 15:29:40 +0100 Subject: [PATCH 45/49] wrong commit --- .github/actions/setup/action.yml | 5 ++++- scripts/ci/create-matrix.sh | 5 ++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 393218d642a..8cdb02e7093 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -86,7 +86,9 @@ runs: matrix=$(./scripts/ci/create-matrix.sh javascript $base_changed ${{ github.base_ref }}) - if [[ $? == 0 ]]; then + echo $matrix + + if [[ $matrix == '{"client":["no-run"]}' ]]; then run="false" else run="true" @@ -94,6 +96,7 @@ runs: echo $matrix echo $matrix | jq + echo $run echo "::set-output name=MATRIX::$matrix" echo "::set-output name=RUN::$run" diff --git a/scripts/ci/create-matrix.sh b/scripts/ci/create-matrix.sh index edf909f7e8b..45a40238c66 100755 --- a/scripts/ci/create-matrix.sh +++ b/scripts/ci/create-matrix.sh @@ -1,3 +1,5 @@ +#!/bin/bash + LANGUAGE=$1 BASE_CHANGED=$2 BASE_BRANCH=$3 @@ -32,11 +34,8 @@ done if [[ $(echo $to_test | jq '.client | length') == 0 ]]; then # client cannot be empty or the matrix will fail matrix='{"client":["no-run"]}' - run=0 else matrix=$(echo $to_test | jq -c) - run=1 fi echo $matrix -exit $run From 059481b573245c87009fb37c3011064c91c5b040 Mon Sep 17 00:00:00 2001 From: Pierre Millot Date: Wed, 12 Jan 2022 16:07:47 +0100 Subject: [PATCH 46/49] remove echo --- .github/actions/setup/action.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 8cdb02e7093..442ae6fc3ce 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -86,18 +86,12 @@ runs: matrix=$(./scripts/ci/create-matrix.sh javascript $base_changed ${{ github.base_ref }}) - echo $matrix - if [[ $matrix == '{"client":["no-run"]}' ]]; then run="false" else run="true" fi - echo $matrix - echo $matrix | jq - echo $run - echo "::set-output name=MATRIX::$matrix" echo "::set-output name=RUN::$run" From 1f2152046c7900712ff34a405e6eea1f28b8eee7 Mon Sep 17 00:00:00 2001 From: Pierre Millot Date: Wed, 12 Jan 2022 16:18:20 +0100 Subject: [PATCH 47/49] use script for java too --- .github/actions/setup/action.yml | 35 ++++++++------------------------ 1 file changed, 9 insertions(+), 26 deletions(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 442ae6fc3ce..596b4c35b53 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -99,35 +99,18 @@ runs: id: java-matrix shell: bash run: | - # Read from openapitools.json to get the clients and folders (not pretty jq) - clients=$(cat openapitools.json | jq -c '."generator-cli".generators | with_entries(if (.key | test("java-.*")) then ({key:.key,value:.value}) else empty end) | to_entries | map({client:.key | sub("java-";""),folder:.value.output | sub("#{cwd}/";"")}) | .[]') - base_changed=${{ github.ref == 'refs/heads/main' || steps.diff.outputs.GITHUB_ACTIONS_CHANGED > 0 || steps.diff.outputs.COMMON_SPECS_CHANGED > 0 || steps.diff.outputs.SCRIPTS_CHANGED > 0 || steps.diff.outputs.JAVA_TEMPLATE_CHANGED > 0 }} - to_test='{"client": []}' - for pair in $clients; do - client=$(echo $pair | jq '.client') - folder=$(echo $pair | jq '.folder') - spec_changed=$(git diff --shortstat origin/${{ github.base_ref }}..HEAD -- specs/$client | wc -l) - client_changed=$(git diff --shortstat origin/${{ github.base_ref }}..HEAD -- $folder | wc -l) - if [[ $base_changed || $spec_changed > 0 || $client_changed > 0 ]]; then - to_test=$(echo $to_test | jq --argjson pair $pair '.client |= .+ [$pair]') - fi - done + matrix=$(./scripts/ci/create-matrix.sh java $base_changed ${{ github.base_ref }}) - # Convert the array to json for the matrix - if [[ $(echo $to_test | jq '.client | length') == 0 ]]; then - # client cannot be empty or the matrix will fail - java_matrix='{"client":["no-run"]}' - run_java="false" + if [[ $matrix == '{"client":["no-run"]}' ]]; then + run="false" else - java_matrix=$(echo $to_test | jq -c) - run_java="true" + run="true" fi - echo $java_matrix | jq - echo "Run Java: $run_java" - echo "::set-output name=JAVA_MATRIX::$java_matrix" - echo "::set-output name=RUN_JAVA::$run_java" + + echo "::set-output name=MATRIX::$matrix" + echo "::set-output name=RUN::$run" outputs: RUN_SPECS: @@ -148,11 +131,11 @@ outputs: RUN_JAVA: description: Whether to run java client or not - value: ${{ steps.java-matrix.outputs.RUN_JAVA }} + value: ${{ steps.java-matrix.outputs.RUN }} JAVA_MATRIX: description: Generate the matrix for javascript client build - value: ${{ steps.java-matrix.outputs.JAVA_MATRIX }} + value: ${{ steps.java-matrix.outputs.MATRIX }} RUN_CTS: description: Determine if the `cts` job should run From ddf5319722e9432a059e5a09c939c2352dfe113d Mon Sep 17 00:00:00 2001 From: Pierre Millot Date: Wed, 12 Jan 2022 18:00:46 +0100 Subject: [PATCH 48/49] review and spec script --- .github/actions/setup/action.yml | 51 ++++++------------- .github/workflows/check.yml | 24 ++++----- ...eate-matrix.sh => create-client-matrix.sh} | 6 +-- scripts/ci/create-spec-matrix.sh | 27 ++++++++++ 4 files changed, 57 insertions(+), 51 deletions(-) rename scripts/ci/{create-matrix.sh => create-client-matrix.sh} (91%) create mode 100755 scripts/ci/create-spec-matrix.sh diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 596b4c35b53..c566ee64d3c 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -44,39 +44,18 @@ runs: id: spec-matrix shell: bash run: | - # Read from openapitools.json and put each client in the specs array - specs="" - generators=( $(cat openapitools.json | jq '."generator-cli".generators' | jq -r 'keys[]') ) - base_changed=${{ github.ref == 'refs/heads/main' || steps.diff.outputs.GITHUB_ACTIONS_CHANGED > 0 || steps.diff.outputs.SCRIPTS_CHANGED > 0 || steps.diff.outputs.COMMON_SPECS_CHANGED > 0 }} - for generator in "${generators[@]}"; do - client=${generator#*-} - if [[ ! ${specs[*]} =~ $client ]]; then - changed=$(git diff --shortstat origin/${{ github.base_ref }}..HEAD -- specs/$client | wc -l) - if [[ $base_changed || $changed > 0 ]]; then - specs="$specs $client" - fi - fi - done - - # Convert the array to json for the matrix - if [[ $specs == "" ]]; then - # client cannot be empty or the matrix will fail - specs_matrix='{"client":["no-run"]}' - run_specs="false" + matrix=$(./scripts/ci/create-spec-matrix.sh $base_changed ${{ github.base_ref }}) + + if [[ $matrix == '{"client":["no-run"]}' ]]; then + run="false" else - specs_matrix=$(( - echo '{ "client" : [' - echo $specs | xargs | sed 's/ /, /g' | xargs -n 1 echo | sed -r 's/^([^,]*)(,?)$/"\1"\2/' - echo " ]}" - ) | jq -c .) - run_specs="true" + run="true" fi - echo $specs_matrix - echo "Run specs: $run_specs" - echo "::set-output name=SPECS_MATRIX::$specs_matrix" - echo "::set-output name=RUN_SPECS::$run_specs" + + echo "::set-output name=MATRIX::$matrix" + echo "::set-output name=RUN_SPECS::$run" - name: Compute the JS client build matrix id: js-matrix @@ -84,7 +63,7 @@ runs: run: | base_changed=${{ github.ref == 'refs/heads/main' || steps.diff.outputs.GITHUB_ACTIONS_CHANGED > 0 || steps.diff.outputs.COMMON_SPECS_CHANGED > 0 || steps.diff.outputs.SCRIPTS_CHANGED > 0 || steps.diff.outputs.JS_TEMPLATE_CHANGED > 0 }} - matrix=$(./scripts/ci/create-matrix.sh javascript $base_changed ${{ github.base_ref }}) + matrix=$(./scripts/ci/create-client-matrix.sh javascript $base_changed ${{ github.base_ref }}) if [[ $matrix == '{"client":["no-run"]}' ]]; then run="false" @@ -93,7 +72,7 @@ runs: fi echo "::set-output name=MATRIX::$matrix" - echo "::set-output name=RUN::$run" + echo "::set-output name=RUN_CLIENT::$run" - name: Compute the Java client build matrix id: java-matrix @@ -101,7 +80,7 @@ runs: run: | base_changed=${{ github.ref == 'refs/heads/main' || steps.diff.outputs.GITHUB_ACTIONS_CHANGED > 0 || steps.diff.outputs.COMMON_SPECS_CHANGED > 0 || steps.diff.outputs.SCRIPTS_CHANGED > 0 || steps.diff.outputs.JAVA_TEMPLATE_CHANGED > 0 }} - matrix=$(./scripts/ci/create-matrix.sh java $base_changed ${{ github.base_ref }}) + matrix=$(./scripts/ci/create-client-matrix.sh java $base_changed ${{ github.base_ref }}) if [[ $matrix == '{"client":["no-run"]}' ]]; then run="false" @@ -110,7 +89,7 @@ runs: fi echo "::set-output name=MATRIX::$matrix" - echo "::set-output name=RUN::$run" + echo "::set-output name=RUN_CLIENT::$run" outputs: RUN_SPECS: @@ -119,11 +98,11 @@ outputs: SPECS_MATRIX: description: Generate the matrix for specs - value: ${{ steps.spec-matrix.outputs.SPECS_MATRIX }} + value: ${{ steps.spec-matrix.outputs.MATRIX }} RUN_JS: description: Whether to run js client or not - value: ${{ steps.js-matrix.outputs.RUN }} + value: ${{ steps.js-matrix.outputs.RUN_CLIENT }} JS_MATRIX: description: Generate the matrix for javascript client build @@ -131,7 +110,7 @@ outputs: RUN_JAVA: description: Whether to run java client or not - value: ${{ steps.java-matrix.outputs.RUN }} + value: ${{ steps.java-matrix.outputs.RUN_CLIENT }} JAVA_MATRIX: description: Generate the matrix for javascript client build diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 1695f1a3e54..e417cb0d5c1 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -70,20 +70,20 @@ jobs: - name: Restore cache uses: ./.github/actions/cache - - name: Cache ${{ matrix.client.client }} client + - name: Cache ${{ matrix.client.name }} client id: cache uses: actions/cache@v2 with: path: '/home/runner/work/api-clients-automation/api-clients-automation/${{ matrix.client.folder }}/dist' - key: ${{ runner.os }}-js-client-${{ matrix.client.client }}-${{ hashFiles(format('{0}/**', matrix.client.folder)) }} + key: ${{ runner.os }}-js-client-${{ matrix.client.name }}-${{ hashFiles(format('{0}/**', matrix.client.folder)) }} - - name: Generate ${{ matrix.client.client }} client + - name: Generate ${{ matrix.client.name }} client if: steps.cache.outputs.cache-hit != 'true' - run: yarn generate javascript ${{ matrix.client.client }} + run: yarn generate javascript ${{ matrix.client.name }} - - name: Build ${{ matrix.client.client }} client + - name: Build ${{ matrix.client.name }} client if: steps.cache.outputs.cache-hit != 'true' - run: yarn build:clients javascript ${{ matrix.client.client }} + run: yarn build:clients javascript ${{ matrix.client.name }} client_java: runs-on: ubuntu-20.04 @@ -101,20 +101,20 @@ jobs: with: job: java - - name: Cache ${{ matrix.client.client }} client + - name: Cache ${{ matrix.client.name }} client id: cache uses: actions/cache@v2 with: path: '/home/runner/work/api-clients-automation/api-clients-automation/${{ matrix.client.folder }}/dist' - key: ${{ runner.os }}-java-client-${{ matrix.client.client }}-${{ hashFiles(format('{0}/**', matrix.client.folder)) }} + key: ${{ runner.os }}-java-client-${{ matrix.client.name }}-${{ hashFiles(format('{0}/**', matrix.client.folder)) }} - - name: Generate ${{ matrix.client.client }} client + - name: Generate ${{ matrix.client.name }} client if: steps.cache.outputs.cache-hit != 'true' - run: yarn generate java ${{ matrix.client.client }} + run: yarn generate java ${{ matrix.client.name }} - - name: Build ${{ matrix.client.client }} client + - name: Build ${{ matrix.client.name }} client if: steps.cache.outputs.cache-hit != 'true' - run: yarn build:clients java ${{ matrix.client.client }} + run: yarn build:clients java ${{ matrix.client.name }} cts: runs-on: ubuntu-20.04 diff --git a/scripts/ci/create-matrix.sh b/scripts/ci/create-client-matrix.sh similarity index 91% rename from scripts/ci/create-matrix.sh rename to scripts/ci/create-client-matrix.sh index 45a40238c66..a927606c044 100755 --- a/scripts/ci/create-matrix.sh +++ b/scripts/ci/create-client-matrix.sh @@ -14,16 +14,16 @@ clients=$(cat openapitools.json | jq --arg lang $LANGUAGE -c '."generator-cli".g ) | to_entries | map({ - client:.key | sub($lang + "-";""), + name:.key | sub($lang + "-";""), folder:.value.output | sub("#{cwd}/";"") }) | .[]') to_test='{"client": []}' for pair in $clients; do - client=$(echo $pair | jq '.client') + name=$(echo $pair | jq '.name') folder=$(echo $pair | jq '.folder') - spec_changed=$(git diff --shortstat origin/$BASE_BRANCH..HEAD -- specs/$client | wc -l) + spec_changed=$(git diff --shortstat origin/$BASE_BRANCH..HEAD -- specs/$name | wc -l) client_changed=$(git diff --shortstat origin/$BASE_BRANCH..HEAD -- $folder | wc -l) if [[ $BASE_CHANGED || $spec_changed > 0 || $client_changed > 0 ]]; then to_test=$(echo $to_test | jq --argjson pair $pair '.client |= .+ [$pair]') diff --git a/scripts/ci/create-spec-matrix.sh b/scripts/ci/create-spec-matrix.sh new file mode 100755 index 00000000000..cff3aeb123e --- /dev/null +++ b/scripts/ci/create-spec-matrix.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +BASE_CHANGED=$1 +BASE_BRANCH=$2 + +specs='{"client":[]}' +generators=( $(cat openapitools.json | jq '."generator-cli".generators' | jq -r 'keys[]') ) + +for generator in "${generators[@]}"; do + client=${generator#*-} + if [[ ! ${specs[*]} =~ $client ]]; then + changed=$(git diff --shortstat origin/$BASE_BRANCH..HEAD -- specs/$client | wc -l) + if [[ $BASE_CHANGED || $changed > 0 ]]; then + specs=$(echo $specs | jq --arg client $client '.client |= .+ [$client]') + fi + fi +done + +# Convert the array to json for the matrix +if [[ $(echo $specs | jq '.client | length') == 0 ]]; then + # client cannot be empty or the matrix will fail + matrix='{"client":["no-run"]}' +else + matrix=$(echo $specs | jq -c) +fi + +echo $matrix From 560feadec8361427cab263a9274428a735da218d Mon Sep 17 00:00:00 2001 From: Pierre Millot Date: Thu, 13 Jan 2022 09:38:01 +0100 Subject: [PATCH 49/49] review --- .github/actions/setup/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index c566ee64d3c..177a55c5c04 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -105,7 +105,7 @@ outputs: value: ${{ steps.js-matrix.outputs.RUN_CLIENT }} JS_MATRIX: - description: Generate the matrix for javascript client build + description: Generate the matrix for the Javascript client value: ${{ steps.js-matrix.outputs.MATRIX }} RUN_JAVA: @@ -113,7 +113,7 @@ outputs: value: ${{ steps.java-matrix.outputs.RUN_CLIENT }} JAVA_MATRIX: - description: Generate the matrix for javascript client build + description: Generate the matrix for the Java client value: ${{ steps.java-matrix.outputs.MATRIX }} RUN_CTS: