From c75d3c5f6ccb4e403e858c94b53ef7028c72eff4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20Torres=20Marroqu=C3=ADn?= Date: Wed, 14 May 2025 14:21:47 -0600 Subject: [PATCH 01/22] splitting cypress tests --- .github/workflows/cypress_admin-ui.yml | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cypress_admin-ui.yml b/.github/workflows/cypress_admin-ui.yml index 3618838b3e6..2282d8afd93 100644 --- a/.github/workflows/cypress_admin-ui.yml +++ b/.github/workflows/cypress_admin-ui.yml @@ -15,6 +15,22 @@ env: jobs: Admin-UI-Cypress: + strategy: + matrix: + specs: + - "cypress/e2e/a**.*" + - "cypress/e2e/c**.*" + - "cypress/e2e/d**.*" + - "cypress/e2e/h**.*" + - "cypress/e2e/l**.*" + - "cypress/e2e/m**.*" + - "cypress/e2e/n**.*" + - "cypress/e2e/o**.*" + - "cypress/e2e/p**.*" + - "cypress/e2e/r**.*" + - "cypress/e2e/s**.*" + - "cypress/e2e/t**.*" + - "cypress/e2e/u**.*" runs-on: ubuntu-latest defaults: run: @@ -40,7 +56,7 @@ jobs: with: working-directory: clients/admin-ui install: false - start: npm run cy:start + start: npm run cy:run -- --spec ${{ matrix.specs }} wait-on: "http://localhost:3000" wait-on-timeout: 180 From 20ef085a2f4c5ab35742010cd7bd6d723093dfe1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20Torres=20Marroqu=C3=ADn?= Date: Wed, 14 May 2025 14:37:44 -0600 Subject: [PATCH 02/22] QA --- .github/workflows/cypress_admin-ui.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cypress_admin-ui.yml b/.github/workflows/cypress_admin-ui.yml index 2282d8afd93..62482a383a2 100644 --- a/.github/workflows/cypress_admin-ui.yml +++ b/.github/workflows/cypress_admin-ui.yml @@ -56,9 +56,10 @@ jobs: with: working-directory: clients/admin-ui install: false - start: npm run cy:run -- --spec ${{ matrix.specs }} + start: npm run cy:start wait-on: "http://localhost:3000" wait-on-timeout: 180 + spec: ${{ matrix.specs }} - uses: actions/upload-artifact@v4 if: failure() From 1a0bb02e945b3cf08bb70fab7356aaf499f03e12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20Torres=20Marroqu=C3=ADn?= Date: Wed, 14 May 2025 14:56:48 -0600 Subject: [PATCH 03/22] QA --- .github/workflows/cypress_admin-ui.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/cypress_admin-ui.yml b/.github/workflows/cypress_admin-ui.yml index 62482a383a2..12c54c5df4c 100644 --- a/.github/workflows/cypress_admin-ui.yml +++ b/.github/workflows/cypress_admin-ui.yml @@ -20,14 +20,14 @@ jobs: specs: - "cypress/e2e/a**.*" - "cypress/e2e/c**.*" - - "cypress/e2e/d**.*" - - "cypress/e2e/h**.*" + - "cypress/e2e/datamap.*" + - "cypress/e2e/datasets.*" + - "cypress/e2e/h**.*,cypress/e2e/m**.*,cypress/e2e/n**.*,cypress/e2e/o**.*" - "cypress/e2e/l**.*" - - "cypress/e2e/m**.*" - - "cypress/e2e/n**.*" - - "cypress/e2e/o**.*" - - "cypress/e2e/p**.*" - - "cypress/e2e/r**.*" + - "cypress/e2e/privacy-experiences.*" + - "cypress/e2e/privacy-notices.*" + - "cypress/e2e/privacy-requests.*" + - "cypress/e2e/properties.*,cypress/e2e/r.*" - "cypress/e2e/s**.*" - "cypress/e2e/t**.*" - "cypress/e2e/u**.*" From 73898abda60cb8bdd00797644abbe86fe20358b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20Torres=20Marroqu=C3=ADn?= Date: Wed, 14 May 2025 15:04:33 -0600 Subject: [PATCH 04/22] QA --- .github/workflows/cypress_admin-ui.yml | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/.github/workflows/cypress_admin-ui.yml b/.github/workflows/cypress_admin-ui.yml index 12c54c5df4c..23a7b926ad4 100644 --- a/.github/workflows/cypress_admin-ui.yml +++ b/.github/workflows/cypress_admin-ui.yml @@ -14,23 +14,21 @@ env: CI: true jobs: + list-specs: + runs-on: ubuntu-latest + outputs: + specs: ${{ steps.list-specs.outputs.matrix }} + steps: + - name: Checkout + uses: actions/checkout@v4 + - id: list-specs + run: echo "::set-output name=specs::$(ls clients/admin-ui/cypress/e2e/* | jq -R -s -c 'split("\n")[:-1]')" + Admin-UI-Cypress: + needs: list-specs strategy: matrix: - specs: - - "cypress/e2e/a**.*" - - "cypress/e2e/c**.*" - - "cypress/e2e/datamap.*" - - "cypress/e2e/datasets.*" - - "cypress/e2e/h**.*,cypress/e2e/m**.*,cypress/e2e/n**.*,cypress/e2e/o**.*" - - "cypress/e2e/l**.*" - - "cypress/e2e/privacy-experiences.*" - - "cypress/e2e/privacy-notices.*" - - "cypress/e2e/privacy-requests.*" - - "cypress/e2e/properties.*,cypress/e2e/r.*" - - "cypress/e2e/s**.*" - - "cypress/e2e/t**.*" - - "cypress/e2e/u**.*" + specs: ${{ fromJson(needs.list-specs.outputs.matrix) }} runs-on: ubuntu-latest defaults: run: From 6a9897a4de6a16a2ec64f4ab60015476c6d5d3dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20Torres=20Marroqu=C3=ADn?= Date: Wed, 14 May 2025 15:06:27 -0600 Subject: [PATCH 05/22] QA --- .github/workflows/cypress_admin-ui.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cypress_admin-ui.yml b/.github/workflows/cypress_admin-ui.yml index 23a7b926ad4..1d3600b5514 100644 --- a/.github/workflows/cypress_admin-ui.yml +++ b/.github/workflows/cypress_admin-ui.yml @@ -17,18 +17,19 @@ jobs: list-specs: runs-on: ubuntu-latest outputs: - specs: ${{ steps.list-specs.outputs.matrix }} + specs: ${{ steps.list-specs.outputs.specs }} steps: - name: Checkout uses: actions/checkout@v4 - id: list-specs - run: echo "::set-output name=specs::$(ls clients/admin-ui/cypress/e2e/* | jq -R -s -c 'split("\n")[:-1]')" + run: | + find clients/admin-ui/cypress/e2e -name "*.cy.ts" | sed 's/^/ - "/' | sed 's/$/",/' | tee specs.txt Admin-UI-Cypress: needs: list-specs strategy: matrix: - specs: ${{ fromJson(needs.list-specs.outputs.matrix) }} + specs: ${{ needs.list-specs.outputs.specs }} runs-on: ubuntu-latest defaults: run: From 10c0b35db4a7d59c1cded9d4a705e1105ba01b2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20Torres=20Marroqu=C3=ADn?= Date: Wed, 14 May 2025 15:10:03 -0600 Subject: [PATCH 06/22] qA --- .github/workflows/cypress_admin-ui.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cypress_admin-ui.yml b/.github/workflows/cypress_admin-ui.yml index 1d3600b5514..e502fcae0da 100644 --- a/.github/workflows/cypress_admin-ui.yml +++ b/.github/workflows/cypress_admin-ui.yml @@ -17,13 +17,13 @@ jobs: list-specs: runs-on: ubuntu-latest outputs: - specs: ${{ steps.list-specs.outputs.specs }} + specs: ${{ steps.step2.outputs.specs }} steps: - name: Checkout uses: actions/checkout@v4 - - id: list-specs + - id: step2 run: | - find clients/admin-ui/cypress/e2e -name "*.cy.ts" | sed 's/^/ - "/' | sed 's/$/",/' | tee specs.txt + echo "specs=$(ls clients/admin-ui/cypress/e2e)" >> "$GITHUB_OUTPUT" Admin-UI-Cypress: needs: list-specs From 6e5730f16ce4a4191b51b3fe65a9d336d2f67315 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20Torres=20Marroqu=C3=ADn?= Date: Wed, 14 May 2025 15:14:54 -0600 Subject: [PATCH 07/22] QA --- .github/workflows/cypress_admin-ui.yml | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/.github/workflows/cypress_admin-ui.yml b/.github/workflows/cypress_admin-ui.yml index e502fcae0da..774a3972267 100644 --- a/.github/workflows/cypress_admin-ui.yml +++ b/.github/workflows/cypress_admin-ui.yml @@ -14,22 +14,26 @@ env: CI: true jobs: - list-specs: + prepare-matrix: runs-on: ubuntu-latest outputs: - specs: ${{ steps.step2.outputs.specs }} + specs: ${{ steps.set-matrix.outputs.specs }} steps: - name: Checkout uses: actions/checkout@v4 - - id: step2 + + - name: Set matrix + id: set-matrix + shell: bash run: | - echo "specs=$(ls clients/admin-ui/cypress/e2e)" >> "$GITHUB_OUTPUT" + cd clients/admin-ui/cypress/e2e + echo "specs=$(find . -name "*.cy.ts" | sed 's|^\./||'" >> $GITHUB_OUTPUT Admin-UI-Cypress: - needs: list-specs + needs: prepare-matrix strategy: matrix: - specs: ${{ needs.list-specs.outputs.specs }} + specs: ${{ fromJson(needs.prepare-matrix.outputs.specs) }} runs-on: ubuntu-latest defaults: run: From 1ba32459cfc313f31a1ab8f978695dce0eacaf8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20Torres=20Marroqu=C3=ADn?= Date: Wed, 14 May 2025 15:15:32 -0600 Subject: [PATCH 08/22] QA --- .github/workflows/cypress_admin-ui.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cypress_admin-ui.yml b/.github/workflows/cypress_admin-ui.yml index 774a3972267..053f1360e7f 100644 --- a/.github/workflows/cypress_admin-ui.yml +++ b/.github/workflows/cypress_admin-ui.yml @@ -27,7 +27,7 @@ jobs: shell: bash run: | cd clients/admin-ui/cypress/e2e - echo "specs=$(find . -name "*.cy.ts" | sed 's|^\./||'" >> $GITHUB_OUTPUT + echo "specs=$(find . -name "*.cy.ts" | sed 's|^\./||')" >> $GITHUB_OUTPUT Admin-UI-Cypress: needs: prepare-matrix From 023f29fb890a6ffa352ca37b6d52fbc0ce386443 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20Torres=20Marroqu=C3=ADn?= Date: Wed, 14 May 2025 15:16:24 -0600 Subject: [PATCH 09/22] QA --- .github/workflows/cypress_admin-ui.yml | 50 +++++++++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cypress_admin-ui.yml b/.github/workflows/cypress_admin-ui.yml index 053f1360e7f..9504d47d8c7 100644 --- a/.github/workflows/cypress_admin-ui.yml +++ b/.github/workflows/cypress_admin-ui.yml @@ -27,7 +27,55 @@ jobs: shell: bash run: | cd clients/admin-ui/cypress/e2e - echo "specs=$(find . -name "*.cy.ts" | sed 's|^\./||')" >> $GITHUB_OUTPUT + # Group files by first letter to create balanced test groups + # The grouping logic aims to distribute tests more evenly + + # Generate groups based on file prefixes + echo "specs=$(find . -name "*.cy.ts" | sed 's|^\./||' | sort | python3 -c ' + import sys, json + files = [line.strip() for line in sys.stdin] + + # Create prefix groups + groups = {} + for file in files: + prefix = file[0].lower() + if prefix not in groups: + groups[prefix] = [] + groups[prefix].append(file) + + # Combine small groups + result = [] + current_group = [] + current_size = 0 + + # Sort keys to ensure consistent output + sorted_keys = sorted(groups.keys()) + + for prefix in sorted_keys: + prefix_files = groups[prefix] + prefix_size = len(prefix_files) + + # If prefix has many files, create its own group + if prefix_size >= 3: + result.append(",".join(prefix_files)) + # Add to current group + elif current_size + prefix_size <= 3: + current_group.extend(prefix_files) + current_size += prefix_size + # Current group is full, start a new one + else: + if current_group: + result.append(",".join(current_group)) + current_group = prefix_files.copy() + current_size = prefix_size + + # Add the last group if it has any files + if current_group: + result.append(",".join(current_group)) + + # Format for GitHub Actions + print(json.dumps(result)) + ')" >> $GITHUB_OUTPUT Admin-UI-Cypress: needs: prepare-matrix From b50744f5443aa5ceae8fcadf0effcfc83626f11b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20Torres=20Marroqu=C3=ADn?= Date: Wed, 14 May 2025 15:21:36 -0600 Subject: [PATCH 10/22] QA --- .github/workflows/cypress_admin-ui.yml | 48 ++------------------------ 1 file changed, 3 insertions(+), 45 deletions(-) diff --git a/.github/workflows/cypress_admin-ui.yml b/.github/workflows/cypress_admin-ui.yml index 9504d47d8c7..4e6af6f9efc 100644 --- a/.github/workflows/cypress_admin-ui.yml +++ b/.github/workflows/cypress_admin-ui.yml @@ -27,54 +27,12 @@ jobs: shell: bash run: | cd clients/admin-ui/cypress/e2e - # Group files by first letter to create balanced test groups - # The grouping logic aims to distribute tests more evenly - - # Generate groups based on file prefixes + # Find all test files and create a matrix with individual files echo "specs=$(find . -name "*.cy.ts" | sed 's|^\./||' | sort | python3 -c ' import sys, json files = [line.strip() for line in sys.stdin] - - # Create prefix groups - groups = {} - for file in files: - prefix = file[0].lower() - if prefix not in groups: - groups[prefix] = [] - groups[prefix].append(file) - - # Combine small groups - result = [] - current_group = [] - current_size = 0 - - # Sort keys to ensure consistent output - sorted_keys = sorted(groups.keys()) - - for prefix in sorted_keys: - prefix_files = groups[prefix] - prefix_size = len(prefix_files) - - # If prefix has many files, create its own group - if prefix_size >= 3: - result.append(",".join(prefix_files)) - # Add to current group - elif current_size + prefix_size <= 3: - current_group.extend(prefix_files) - current_size += prefix_size - # Current group is full, start a new one - else: - if current_group: - result.append(",".join(current_group)) - current_group = prefix_files.copy() - current_size = prefix_size - - # Add the last group if it has any files - if current_group: - result.append(",".join(current_group)) - - # Format for GitHub Actions - print(json.dumps(result)) + # Format for GitHub Actions - each file becomes its own matrix entry + print(json.dumps(files)) ')" >> $GITHUB_OUTPUT Admin-UI-Cypress: From 477499288d0ad8f9648e6be531981bd2a3dfa02b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20Torres=20Marroqu=C3=ADn?= Date: Wed, 14 May 2025 15:26:08 -0600 Subject: [PATCH 11/22] QA --- .github/workflows/cypress_admin-ui.yml | 45 +++++++++++++++++++++++--- 1 file changed, 40 insertions(+), 5 deletions(-) diff --git a/.github/workflows/cypress_admin-ui.yml b/.github/workflows/cypress_admin-ui.yml index 4e6af6f9efc..072db1a714a 100644 --- a/.github/workflows/cypress_admin-ui.yml +++ b/.github/workflows/cypress_admin-ui.yml @@ -35,8 +35,39 @@ jobs: print(json.dumps(files)) ')" >> $GITHUB_OUTPUT + build-admin-ui: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Use Node.js 20.x + uses: actions/setup-node@v4 + with: + node-version: 20.x + + - name: Install dependencies + working-directory: clients + run: npm install + + - name: Build FidesJS + working-directory: clients/fides-js + run: npm run build + + - name: Build Admin UI + working-directory: clients/admin-ui + run: npm run build + + - name: Cache build output + uses: actions/cache@v3 + with: + path: | + clients/admin-ui/.next + clients/fides-js/dist + key: ${{ runner.os }}-admin-ui-build-${{ github.sha }} + Admin-UI-Cypress: - needs: prepare-matrix + needs: [prepare-matrix, build-admin-ui] strategy: matrix: specs: ${{ fromJson(needs.prepare-matrix.outputs.specs) }} @@ -56,16 +87,20 @@ jobs: - name: Install dependencies run: npm install - - name: Build FidesJS - working-directory: clients/fides-js - run: npm run build + - name: Restore build output + uses: actions/cache@v3 + with: + path: | + clients/admin-ui/.next + clients/fides-js/dist + key: ${{ runner.os }}-admin-ui-build-${{ github.sha }} - name: Cypress Admin UI E2E Tests uses: cypress-io/github-action@v6 with: working-directory: clients/admin-ui install: false - start: npm run cy:start + start: npm run start wait-on: "http://localhost:3000" wait-on-timeout: 180 spec: ${{ matrix.specs }} From 3a9fba7897f5a7833f708d61619b7996b8143d33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20Torres=20Marroqu=C3=ADn?= Date: Wed, 14 May 2025 15:33:46 -0600 Subject: [PATCH 12/22] QA --- .github/workflows/cypress_admin-ui.yml | 45 +++----------------------- 1 file changed, 5 insertions(+), 40 deletions(-) diff --git a/.github/workflows/cypress_admin-ui.yml b/.github/workflows/cypress_admin-ui.yml index 072db1a714a..4e6af6f9efc 100644 --- a/.github/workflows/cypress_admin-ui.yml +++ b/.github/workflows/cypress_admin-ui.yml @@ -35,39 +35,8 @@ jobs: print(json.dumps(files)) ')" >> $GITHUB_OUTPUT - build-admin-ui: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Use Node.js 20.x - uses: actions/setup-node@v4 - with: - node-version: 20.x - - - name: Install dependencies - working-directory: clients - run: npm install - - - name: Build FidesJS - working-directory: clients/fides-js - run: npm run build - - - name: Build Admin UI - working-directory: clients/admin-ui - run: npm run build - - - name: Cache build output - uses: actions/cache@v3 - with: - path: | - clients/admin-ui/.next - clients/fides-js/dist - key: ${{ runner.os }}-admin-ui-build-${{ github.sha }} - Admin-UI-Cypress: - needs: [prepare-matrix, build-admin-ui] + needs: prepare-matrix strategy: matrix: specs: ${{ fromJson(needs.prepare-matrix.outputs.specs) }} @@ -87,20 +56,16 @@ jobs: - name: Install dependencies run: npm install - - name: Restore build output - uses: actions/cache@v3 - with: - path: | - clients/admin-ui/.next - clients/fides-js/dist - key: ${{ runner.os }}-admin-ui-build-${{ github.sha }} + - name: Build FidesJS + working-directory: clients/fides-js + run: npm run build - name: Cypress Admin UI E2E Tests uses: cypress-io/github-action@v6 with: working-directory: clients/admin-ui install: false - start: npm run start + start: npm run cy:start wait-on: "http://localhost:3000" wait-on-timeout: 180 spec: ${{ matrix.specs }} From 71d70c07886037fa47eb0dc692719a2e2c42521a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20Torres=20Marroqu=C3=ADn?= Date: Wed, 14 May 2025 15:44:02 -0600 Subject: [PATCH 13/22] QA --- .github/workflows/cypress_admin-ui.yml | 48 ++++++++++++++++++++++---- 1 file changed, 42 insertions(+), 6 deletions(-) diff --git a/.github/workflows/cypress_admin-ui.yml b/.github/workflows/cypress_admin-ui.yml index 4e6af6f9efc..13e12592305 100644 --- a/.github/workflows/cypress_admin-ui.yml +++ b/.github/workflows/cypress_admin-ui.yml @@ -35,11 +35,7 @@ jobs: print(json.dumps(files)) ')" >> $GITHUB_OUTPUT - Admin-UI-Cypress: - needs: prepare-matrix - strategy: - matrix: - specs: ${{ fromJson(needs.prepare-matrix.outputs.specs) }} + build-admin-ui: runs-on: ubuntu-latest defaults: run: @@ -60,6 +56,46 @@ jobs: working-directory: clients/fides-js run: npm run build + - name: Build Admin UI + working-directory: clients/admin-ui + run: npm run build + + - name: Cache build artifacts + uses: actions/cache@v3 + with: + path: | + clients/admin-ui/dist + clients/fides-js/dist + clients/node_modules + key: ${{ runner.os }}-admin-ui-build-${{ github.sha }} + + Admin-UI-Cypress: + needs: [prepare-matrix, build-admin-ui] + strategy: + matrix: + specs: ${{ fromJson(needs.prepare-matrix.outputs.specs) }} + runs-on: ubuntu-latest + defaults: + run: + working-directory: clients + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Use Node.js 20.x + uses: actions/setup-node@v4 + with: + node-version: 20.x + + - name: Restore cached build artifacts + uses: actions/cache@v3 + with: + path: | + clients/admin-ui/dist + clients/fides-js/dist + clients/node_modules + key: ${{ runner.os }}-admin-ui-build-${{ github.sha }} + - name: Cypress Admin UI E2E Tests uses: cypress-io/github-action@v6 with: @@ -68,7 +104,7 @@ jobs: start: npm run cy:start wait-on: "http://localhost:3000" wait-on-timeout: 180 - spec: ${{ matrix.specs }} + spec: cypress/e2e/${{ matrix.specs }} - uses: actions/upload-artifact@v4 if: failure() From 716b7fa91465d70ec5f6dd529c71567d6193d4c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20Torres=20Marroqu=C3=ADn?= Date: Wed, 14 May 2025 15:54:38 -0600 Subject: [PATCH 14/22] QA --- .github/workflows/cypress_admin-ui.yml | 83 +++++++++++--------------- 1 file changed, 36 insertions(+), 47 deletions(-) diff --git a/.github/workflows/cypress_admin-ui.yml b/.github/workflows/cypress_admin-ui.yml index 13e12592305..7ee527caf05 100644 --- a/.github/workflows/cypress_admin-ui.yml +++ b/.github/workflows/cypress_admin-ui.yml @@ -17,7 +17,7 @@ jobs: prepare-matrix: runs-on: ubuntu-latest outputs: - specs: ${{ steps.set-matrix.outputs.specs }} + spec_groups: ${{ steps.set-matrix.outputs.spec_groups }} steps: - name: Checkout uses: actions/checkout@v4 @@ -27,53 +27,44 @@ jobs: shell: bash run: | cd clients/admin-ui/cypress/e2e - # Find all test files and create a matrix with individual files - echo "specs=$(find . -name "*.cy.ts" | sed 's|^\./||' | sort | python3 -c ' - import sys, json - files = [line.strip() for line in sys.stdin] - # Format for GitHub Actions - each file becomes its own matrix entry - print(json.dumps(files)) - ')" >> $GITHUB_OUTPUT + # Group test files into balanced groups based on file size + echo "spec_groups=$(find . -name "*.cy.ts" | python3 -c ' + import sys, os, json, math - build-admin-ui: - runs-on: ubuntu-latest - defaults: - run: - working-directory: clients - steps: - - name: Checkout - uses: actions/checkout@v4 + # Number of groups to create (adjust based on your needs) + NUM_GROUPS = 4 - - name: Use Node.js 20.x - uses: actions/setup-node@v4 - with: - node-version: 20.x + # Get all test files with their sizes + files = [] + for line in sys.stdin: + filepath = line.strip() + if os.path.exists(filepath): + size = os.path.getsize(filepath) + files.append((filepath.replace("./", ""), size)) - - name: Install dependencies - run: npm install + # Sort files by size in descending order + files.sort(key=lambda x: x[1], reverse=True) - - name: Build FidesJS - working-directory: clients/fides-js - run: npm run build + # Initialize groups + groups = [[] for _ in range(NUM_GROUPS)] + group_sizes = [0] * NUM_GROUPS - - name: Build Admin UI - working-directory: clients/admin-ui - run: npm run build + # Distribute files using greedy algorithm (largest file first) + for file, size in files: + # Find the group with the smallest total size + min_group_idx = group_sizes.index(min(group_sizes)) + groups[min_group_idx].append(file) + group_sizes[min_group_idx] += size - - name: Cache build artifacts - uses: actions/cache@v3 - with: - path: | - clients/admin-ui/dist - clients/fides-js/dist - clients/node_modules - key: ${{ runner.os }}-admin-ui-build-${{ github.sha }} + # Format for GitHub Actions + print(json.dumps([",".join(group) for group in groups])) + ')" >> $GITHUB_OUTPUT Admin-UI-Cypress: - needs: [prepare-matrix, build-admin-ui] + needs: prepare-matrix strategy: matrix: - specs: ${{ fromJson(needs.prepare-matrix.outputs.specs) }} + spec_group: ${{ fromJson(needs.prepare-matrix.outputs.spec_groups) }} runs-on: ubuntu-latest defaults: run: @@ -87,14 +78,12 @@ jobs: with: node-version: 20.x - - name: Restore cached build artifacts - uses: actions/cache@v3 - with: - path: | - clients/admin-ui/dist - clients/fides-js/dist - clients/node_modules - key: ${{ runner.os }}-admin-ui-build-${{ github.sha }} + - name: Install dependencies + run: npm install + + - name: Build FidesJS + working-directory: clients/fides-js + run: npm run build - name: Cypress Admin UI E2E Tests uses: cypress-io/github-action@v6 @@ -104,7 +93,7 @@ jobs: start: npm run cy:start wait-on: "http://localhost:3000" wait-on-timeout: 180 - spec: cypress/e2e/${{ matrix.specs }} + spec: cypress/e2e/{${{ matrix.spec_group }}} - uses: actions/upload-artifact@v4 if: failure() From 269c3a83e28ead20e2691d37d69bb53e6a6bb15f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20Torres=20Marroqu=C3=ADn?= Date: Wed, 14 May 2025 16:06:10 -0600 Subject: [PATCH 15/22] QA --- .github/workflows/cypress_admin-ui.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/cypress_admin-ui.yml b/.github/workflows/cypress_admin-ui.yml index 7ee527caf05..edb5a1134d2 100644 --- a/.github/workflows/cypress_admin-ui.yml +++ b/.github/workflows/cypress_admin-ui.yml @@ -87,6 +87,7 @@ jobs: - name: Cypress Admin UI E2E Tests uses: cypress-io/github-action@v6 + continue-on-error: true with: working-directory: clients/admin-ui install: false From 3763274bbdc762b279a0326e9d0eadda6794db2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20Torres=20Marroqu=C3=ADn?= Date: Wed, 14 May 2025 16:21:40 -0600 Subject: [PATCH 16/22] QA --- .github/workflows/cypress_admin-ui.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cypress_admin-ui.yml b/.github/workflows/cypress_admin-ui.yml index edb5a1134d2..7c006edb7d9 100644 --- a/.github/workflows/cypress_admin-ui.yml +++ b/.github/workflows/cypress_admin-ui.yml @@ -32,7 +32,7 @@ jobs: import sys, os, json, math # Number of groups to create (adjust based on your needs) - NUM_GROUPS = 4 + NUM_GROUPS = 6 # Get all test files with their sizes files = [] From 805459d7ba2b0519a65c3c7ab681a5efce95be3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20Torres=20Marroqu=C3=ADn?= Date: Wed, 14 May 2025 16:42:43 -0600 Subject: [PATCH 17/22] QA --- .github/workflows/cypress_admin-ui.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/cypress_admin-ui.yml b/.github/workflows/cypress_admin-ui.yml index 7c006edb7d9..0ae2ef72fe4 100644 --- a/.github/workflows/cypress_admin-ui.yml +++ b/.github/workflows/cypress_admin-ui.yml @@ -87,7 +87,6 @@ jobs: - name: Cypress Admin UI E2E Tests uses: cypress-io/github-action@v6 - continue-on-error: true with: working-directory: clients/admin-ui install: false From 2773c25087d1032040d7c283db77aed6adb7c5c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20Torres=20Marroqu=C3=ADn?= Date: Wed, 14 May 2025 16:47:15 -0600 Subject: [PATCH 18/22] QA --- .github/workflows/cypress_admin-ui.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cypress_admin-ui.yml b/.github/workflows/cypress_admin-ui.yml index 0ae2ef72fe4..5f76a6749ed 100644 --- a/.github/workflows/cypress_admin-ui.yml +++ b/.github/workflows/cypress_admin-ui.yml @@ -32,7 +32,7 @@ jobs: import sys, os, json, math # Number of groups to create (adjust based on your needs) - NUM_GROUPS = 6 + NUM_GROUPS = 3 # Get all test files with their sizes files = [] @@ -63,6 +63,7 @@ jobs: Admin-UI-Cypress: needs: prepare-matrix strategy: + fail-fast: false matrix: spec_group: ${{ fromJson(needs.prepare-matrix.outputs.spec_groups) }} runs-on: ubuntu-latest From fe8dc77da94cde371c65ed8e5391a0cd6392d4c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20Torres=20Marroqu=C3=ADn?= Date: Wed, 14 May 2025 17:13:27 -0600 Subject: [PATCH 19/22] QA --- .github/workflows/cypress_admin-ui.yml | 36 ++++++++++++++++---------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/.github/workflows/cypress_admin-ui.yml b/.github/workflows/cypress_admin-ui.yml index 5f76a6749ed..919341653ac 100644 --- a/.github/workflows/cypress_admin-ui.yml +++ b/.github/workflows/cypress_admin-ui.yml @@ -27,34 +27,44 @@ jobs: shell: bash run: | cd clients/admin-ui/cypress/e2e - # Group test files into balanced groups based on file size + # Group test files into balanced groups based on number of tests in each file echo "spec_groups=$(find . -name "*.cy.ts" | python3 -c ' - import sys, os, json, math + import sys, os, json, math, re # Number of groups to create (adjust based on your needs) NUM_GROUPS = 3 - # Get all test files with their sizes + # Get all test files with their test counts files = [] for line in sys.stdin: filepath = line.strip() if os.path.exists(filepath): - size = os.path.getsize(filepath) - files.append((filepath.replace("./", ""), size)) - - # Sort files by size in descending order + # Count the number of test occurrences (it, describe, context) + test_count = 0 + with open(filepath, "r") as f: + content = f.read() + # Count occurrences of test definitions like "it(", "test(", "specify(" + test_count += len(re.findall(r"\bit\s*\(", content)) + test_count += len(re.findall(r"\btest\s*\(", content)) + test_count += len(re.findall(r"\bspecify\s*\(", content)) + # If no tests found, set minimum of 1 to ensure file is included + if test_count == 0: + test_count = 1 + files.append((filepath.replace("./", ""), test_count)) + + # Sort files by test count in descending order files.sort(key=lambda x: x[1], reverse=True) # Initialize groups groups = [[] for _ in range(NUM_GROUPS)] - group_sizes = [0] * NUM_GROUPS + group_counts = [0] * NUM_GROUPS - # Distribute files using greedy algorithm (largest file first) - for file, size in files: - # Find the group with the smallest total size - min_group_idx = group_sizes.index(min(group_sizes)) + # Distribute files using greedy algorithm (most tests first) + for file, count in files: + # Find the group with the smallest total test count + min_group_idx = group_counts.index(min(group_counts)) groups[min_group_idx].append(file) - group_sizes[min_group_idx] += size + group_counts[min_group_idx] += count # Format for GitHub Actions print(json.dumps([",".join(group) for group in groups])) From c5b3b2ea097c60056d2461730b27d7b7ac4fb22b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20Torres=20Marroqu=C3=ADn?= Date: Thu, 15 May 2025 04:10:46 -0600 Subject: [PATCH 20/22] QA --- .../admin-ui/cypress/e2e/integration-management.cy.ts | 10 +++++----- .../add-integration/ConfigureIntegrationForm.tsx | 6 +++--- .../configure-monitor/ConfigureMonitorForm.tsx | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/clients/admin-ui/cypress/e2e/integration-management.cy.ts b/clients/admin-ui/cypress/e2e/integration-management.cy.ts index dae8ee45c5d..57a7d05d211 100644 --- a/clients/admin-ui/cypress/e2e/integration-management.cy.ts +++ b/clients/admin-ui/cypress/e2e/integration-management.cy.ts @@ -14,7 +14,7 @@ describe("Integration management for data detection & discovery", () => { test_status: "succeeded", }, }).as("testConnection"); - cy.intercept("GET", "/api/v1/connection_type", { + cy.intercept("GET", "/api/v1/connection_type?*", { fixture: "connectors/connection_types.json", }).as("getConnectionTypes"); cy.intercept("GET", "/api/v1/connection_type/*/secret", { @@ -90,9 +90,9 @@ describe("Integration management for data detection & discovery", () => { describe("adding an integration", () => { beforeEach(() => { cy.intercept("GET", "/api/v1/connection?*", { - fixture: "connectors/bigquery_connection_list.json", + // fixture: "connectors/bigquery_connection_list.json", }).as("getConnections"); - cy.intercept("GET", "/api/v1/connection_type", { + cy.intercept("GET", "/api/v1/connection_type?*", { fixture: "connectors/connection_types.json", }).as("getConnectionTypes"); cy.visit(INTEGRATION_MANAGEMENT_ROUTE); @@ -178,7 +178,7 @@ describe("Integration management for data detection & discovery", () => { cy.intercept("GET", "/api/v1/connection?*", { fixture: "connectors/bigquery_connection_list.json", }).as("getConnections"); - cy.intercept("GET", "/api/v1/connection_type", { + cy.intercept("GET", "/api/v1/connection_type?*", { fixture: "connectors/connection_types.json", }).as("getConnectionTypes"); cy.intercept("GET", "/api/v1/system", { @@ -461,7 +461,7 @@ describe("Integration management for data detection & discovery", () => { cy.intercept("GET", "/api/v1/plus/discovery-monitor*", { fixture: "detection-discovery/monitors/website_monitor_list.json", }).as("getMonitors"); - cy.intercept("GET", "/api/v1/connection_type", { + cy.intercept("GET", "/api/v1/connection_type?*", { fixture: "connectors/connection_types.json", }).as("getConnectionTypes"); cy.getByTestId("tab-Data discovery").click(); diff --git a/clients/admin-ui/src/features/integrations/add-integration/ConfigureIntegrationForm.tsx b/clients/admin-ui/src/features/integrations/add-integration/ConfigureIntegrationForm.tsx index 84176db85b0..c7c86ffa001 100644 --- a/clients/admin-ui/src/features/integrations/add-integration/ConfigureIntegrationForm.tsx +++ b/clients/admin-ui/src/features/integrations/add-integration/ConfigureIntegrationForm.tsx @@ -64,7 +64,7 @@ const ConfigureIntegrationForm = ({ usePatchSystemConnectionConfigsMutation(); const { data: secrets, isLoading: secretsSchemaIsLoading } = - useGetConnectionTypeSecretSchemaQuery(connectionOption?.identifier); + useGetConnectionTypeSecretSchemaQuery(connectionOption.identifier); const { data: allSystems } = useGetAllSystemsQuery(); @@ -129,7 +129,7 @@ const ConfigureIntegrationForm = ({ : { name: values.name, key: formatKey(values.name), - connection_type: connectionOption?.identifier as ConnectionType, + connection_type: connectionOption.identifier as ConnectionType, access: AccessLevel.READ, disabled: false, description: values.description, @@ -292,7 +292,7 @@ const ConfigureIntegrationForm = ({ layout="stacked" /> )} - {connectionOption?.identifier === ConnectionType.DATAHUB && ( + {connectionOption.identifier === ConnectionType.DATAHUB && ( Date: Thu, 15 May 2025 04:11:48 -0600 Subject: [PATCH 21/22] QA --- .github/workflows/cypress_admin-ui.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cypress_admin-ui.yml b/.github/workflows/cypress_admin-ui.yml index 919341653ac..1e6d30cc137 100644 --- a/.github/workflows/cypress_admin-ui.yml +++ b/.github/workflows/cypress_admin-ui.yml @@ -32,7 +32,7 @@ jobs: import sys, os, json, math, re # Number of groups to create (adjust based on your needs) - NUM_GROUPS = 3 + NUM_GROUPS = 4 # Get all test files with their test counts files = [] From bc4833be6b21d8fdbd8475d7d107ab7165cbc728 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20Torres=20Marroqu=C3=ADn?= Date: Thu, 15 May 2025 04:23:54 -0600 Subject: [PATCH 22/22] QA --- .github/workflows/cypress_admin-ui.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cypress_admin-ui.yml b/.github/workflows/cypress_admin-ui.yml index 1e6d30cc137..919341653ac 100644 --- a/.github/workflows/cypress_admin-ui.yml +++ b/.github/workflows/cypress_admin-ui.yml @@ -32,7 +32,7 @@ jobs: import sys, os, json, math, re # Number of groups to create (adjust based on your needs) - NUM_GROUPS = 4 + NUM_GROUPS = 3 # Get all test files with their test counts files = []