Skip to content

Commit

Permalink
test(real): Make real data E2E tests more robust and scalable (#4774)
Browse files Browse the repository at this point in the history
* test(real): Make real data E2E tests more robust and scalable

ignore .env

* github actions

* github actions

* github actions

* github actions

* github actions

* github actions

* github actions

* github actions

* github actions

* Fix mock tests

* Fix mock tests

* shard real tests

* fix thing

* fix thing

* fix thing

* fix thing

* fix thing

* fix thing
  • Loading branch information
hartra344 committed May 6, 2024
1 parent 0f960c9 commit 8eaf2e1
Show file tree
Hide file tree
Showing 30 changed files with 1,246 additions and 1,055 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/playwright.yml
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ jobs:
- name: Install Playwright Browsers
run: pnpm run e2e:setup
- name: Run Playwright tests
run: pnpm run test:e2e --grep @mock
run: pnpm run test:e2e --grep @mock --workers 4
- uses: actions/upload-artifact@v4
if: always()
with:
Expand Down
165 changes: 89 additions & 76 deletions .github/workflows/real-e2e.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,86 +3,72 @@ on:
workflow_dispatch:
schedule:
- cron: '0 */6 * * *'

concurrency:
group: e2e_tests
cancel-in-progress: true

jobs:

deploy-test-app:
run-tests:
runs-on: ubuntu-latest
environment: E2ETesting
permissions:
id-token: write
strategy:
fail-fast: false
matrix:
browser: [chromium, firefox]
shardIndex: [1, 2, 3, 4, 5, 6]
shardTotal: [6]
env:
RGName : rg${{ github.run_id }}${{ github.run_attempt }}${{ matrix.shardIndex }}
steps:
- uses: actions/checkout@v4

- uses: vimtor/action-zip@v1.2
with:
files: e2e/testSetup/BlankLogicApp/test-app-1
dest: drop/test-app-1.zip

- name: Azure login
uses: azure/login@v2
with:
client-id: ${{ secrets.AZURE_TESTING_CLIENT_ID }}
tenant-id: ${{ secrets.AZURE_TESTING_TENANT_ID }}
subscription-id: ${{ secrets.AZURE_TESTING_SUBSCRIPTION_ID }}

- name: "Upload Playwright Report to Azure Blob Storage for static site access"
shell: bash
if: always()
subscription-id: ${{ secrets.AZURE_TESTING_SUBSCRIPTION_ID }}

- name: Deploy Logic Apps For Testing
run: |
ZIP_DIR='run-${{ github.run_id }}-${{ github.run_attempt }}-${{matrix.browser}}'
az storage blob upload-batch --account-name lauxtestreport --auth-mode key -d "deploymentzips/$ZIP_DIR" -s "./drop"
echo "ZIP-URL=https://lauxtestreport.blob.core.windows.net/deploymentzips/$ZIP_DIR/test-app-1.zip" >> $GITHUB_ENV
az group create -l westus --subscription ${{ secrets.AZURE_TESTING_SUBSCRIPTION_ID }} -n ${{ env.RGName }}
az deployment group create \
--resource-group ${{ env.RGName }} \
--subscription ${{ secrets.AZURE_TESTING_SUBSCRIPTION_ID }} \
--template-file ./e2e/testSetup/BlankLogicApp/logicAppStandard.bicep
- name: Deploy Logic Apps For Testing
uses: azure/cli@v2
- uses: pnpm/action-setup@v3
with:
azcliversion: 2.30.0
inlineScript: |
az group create -l westus --subscription ${{ secrets.AZURE_TESTING_SUBSCRIPTION_ID }} -n lauxtest2${{matrix.browser}}
az deployment group create \
--resource-group lauxtest2${{matrix.browser}} \
--subscription ${{ secrets.AZURE_TESTING_SUBSCRIPTION_ID }} \
--template-file ./e2e/testSetup/BlankLogicApp/logicAppStandard.bicep \
--parameters "{ \"zipDeploymentLocation\": { \"value\": \"${{env.ZIP-URL}}\" } }"
version: 8
run_install: |
- recursive: true
args: [--frozen-lockfile, --strict-peer-dependencies]
run-tests:
runs-on: ubuntu-latest
environment: E2ETesting
needs: deploy-test-app
permissions:
id-token: write
steps:
- uses: actions/checkout@v4
- name: Install Playwright Browsers
run: pnpm run e2e:setup

- name: Azure login
uses: azure/login@v2
with:
client-id: ${{ secrets.AZURE_TESTING_CLIENT_ID }}
tenant-id: ${{ secrets.AZURE_TESTING_TENANT_ID }}
subscription-id: ${{ secrets.AZURE_TESTING_SUBSCRIPTION_ID }}

- name: "extract AAD access token for Test Runs (valid for about an hour)"
run: |
export AZURE_MANAGEMENT_TOKEN=$(az account get-access-token --tenant ${{ secrets.AZURE_TESTING_TENANT_ID }} | jq -r .accessToken)
az account get-access-token --tenant ${{ secrets.AZURE_TESTING_TENANT_ID }} >> apps/Standalone/src/environments/jsonImport/armToken.json
- uses: pnpm/action-setup@v3
with:
version: 8
run_install: |
- recursive: true
args: [--frozen-lockfile, --strict-peer-dependencies]
- name: Install Playwright Browsers
run: pnpm run e2e:setup
echo "AZURE_MANAGEMENT_TOKEN=$AZURE_MANAGEMENT_TOKEN" >> $GITHUB_ENV
- name: Run Playwright tests
run: pnpm run test:e2e --grep @real

run: pnpm run test:e2e --grep @real --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
env:
AZURE_RESOURCE_GROUP: ${{env.RGName}}
AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_TESTING_SUBSCRIPTION_ID }}
AZURE_SITE_NAME: wapp-${{env.RGName}}
AZURE_MANAGEMENT_TOKEN: ${{ env.AZURE_MANAGEMENT_TOKEN }}
TEST_SHARDED: true
- name: Scrub Secrets From Trace
if: always()
run: pnpm run scrub-secrets:e2e
Expand All @@ -99,32 +85,59 @@ jobs:
shell: bash
if: always()
run: |
az group delete -n lauxtest2chromium --yes --no-wait
az group delete -n lauxtest2firefox --yes --no-wait
- name: "Upload Playwright Report to Azure Blob Storage for static site access"
shell: bash
if: always()
run: |
REPORT_DIR='run-${{ github.run_id }}-${{ github.run_attempt }}'
az storage blob upload-batch --account-name lauxtestreport --auth-mode key -d "\$web/$REPORT_DIR" -s "./playwright-report"
echo "::notice title=HTML report url::https://lauxtestreport.z13.web.core.windows.net/$REPORT_DIR/index.html"
az group delete -n ${{env.RGName}} --yes --no-wait
- uses: actions/upload-artifact@v4
if: always()
- name: Upload blob report to GitHub Actions Artifacts
if: ${{ !cancelled() }}
uses: actions/upload-artifact@v4
with:
name: playwright-report
path: playwright-report/
retention-days: 30

# notify:
# name: Notify failed build
# needs: run-tests
# if: ${{ always() && needs.run-tests.result == 'failure' }}
# runs-on: ubuntu-latest
# permissions:
# issues: write
# steps:
# - uses: jayqi/failed-build-issue-action@v1
# with:
# github-token: ${{ secrets.GITHUB_TOKEN }}
name: blob-report-${{ matrix.shardIndex }}
path: blob-report
retention-days: 1

merge-reports:
# Merge reports after playwright-tests, even if some shards have failed
if: ${{ !cancelled() }}
needs: [run-tests]
runs-on: ubuntu-latest
environment: E2ETesting
permissions:
id-token: write
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: 18

- name: Download blob reports from GitHub Actions Artifacts
uses: actions/download-artifact@v4
with:
path: all-blob-reports
pattern: blob-report-*
merge-multiple: true

- name: Merge into HTML Report
run: npx playwright merge-reports --reporter html ./all-blob-reports

- name: Azure login
uses: azure/login@v2
if: always()
with:
client-id: ${{ secrets.AZURE_TESTING_CLIENT_ID }}
tenant-id: ${{ secrets.AZURE_TESTING_TENANT_ID }}
subscription-id: ${{ secrets.AZURE_TESTING_SUBSCRIPTION_ID }}

- name: "Upload Playwright Report to Azure Blob Storage for static site access"
shell: bash
if: always()
run: |
REPORT_DIR='run-${{ github.run_id }}-${{ github.run_attempt }}'
az storage blob upload-batch --account-name lauxtestreport --auth-mode key -d "\$web/$REPORT_DIR" -s "./playwright-report"
echo "::notice title=HTML report url::https://lauxtestreport.z13.web.core.windows.net/$REPORT_DIR/index.html"
- name: Upload HTML report
uses: actions/upload-artifact@v4
with:
name: html-report--attempt-${{ github.run_attempt }}
path: playwright-report
retention-days: 14
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,7 @@ dist-ssr
CHANGELOG.MD
# Misc
.DS_Store
.env
.env.local
.env.development.local
.env.test.local
Expand Down
2 changes: 1 addition & 1 deletion e2e/designer/app.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ test(

await GoToMockWorkflow(page, 'Simple Big Workflow');

await page.getByTestId('card-Increment variable').getByRole('button').click();
await page.getByTestId('card-increment_variable').getByRole('button').click();
await page.getByLabel('Value').getByRole('paragraph').click();
await page.getByLabel('Value').press('Escape');
await page.getByRole('tab', { name: 'Code View' }).click();
Expand Down
4 changes: 1 addition & 3 deletions e2e/designer/dragAndDrop.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,6 @@ test(
await page.goto('/');

await GoToMockWorkflow(page, 'Panel');
await page
.getByLabel('HTTP operation, HTTP connector')
.dragTo(page.getByTestId('rf__edge-manual-Initialize_ArrayVariable').getByLabel('Insert a new step between'));
await page.getByTestId('card-http').dragTo(page.getByTestId('msla-plus-button-manual-initialize_arrayvariable'));
}
);
108 changes: 108 additions & 0 deletions e2e/designer/fixtures/real-api.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
import { test as base, expect } from '@playwright/test';
import type { APIRequestContext, Page } from '@playwright/test';
import * as Constants from '../utils/Constants';

export class RealDataApi {
private workflowName: string;
private siteId: string;
public siteName: string;
constructor(
public readonly page: Page,
public readonly request: APIRequestContext
) {
this.workflowName = 'testworkflow2';
this.siteName = process.env.AZURE_SITE_NAME ?? '';
this.siteId = `/subscriptions/${process.env.AZURE_SUBSCRIPTION_ID}/resourceGroups/${process.env.AZURE_RESOURCE_GROUP}/providers/Microsoft.Web/sites/${this.siteName}`;
}
async goToWorkflow() {
await this.page.getByPlaceholder('Select an App').click();
await this.page.getByPlaceholder('Select an App').fill(this.siteName);
await this.page.getByPlaceholder('Select an App').press('Enter');
await this.page.getByLabel('Workflow').locator('span').filter({ hasText: '' }).click();
await this.page.getByRole('option', { name: this.workflowName, exact: true }).click();
await this.page.getByRole('button', { name: 'Toolbox' }).click();
await this.page.getByLabel('fit view').click({ force: true });
}
async saveWorkflow() {
const responsePromise = this.page.waitForResponse(
`${Constants.managementUrl}${this.siteId}/deployWorkflowArtifacts?api-version=${Constants.siteApiVersion}`
);
await this.page.getByRole('menuitem', { name: 'Save Save' }).click();
await responsePromise;
}
async verifyWorkflowSaveWithRequest(expectedStatus: number, expectedBody: string, triggerName: string, dataToSend?: any) {
let listCallbackUrlCall = await this.request.post(
`${Constants.managementUrl}${this.siteId}/hostruntime/runtime/webhooks/workflow/api/management/workflows/${this.workflowName}/triggers/${triggerName}/listCallbackUrl?api-version=${Constants.siteApiVersion}`,
{
headers: {
'Content-Type': 'application/json',
Authorization: `Bearer ${process.env.AZURE_MANAGEMENT_TOKEN}`,
'If-Match': '*',
},
}
);

let listCallbackUrlResponseValue = await listCallbackUrlCall.json();
let listCallbackUrl: string = listCallbackUrlResponseValue.value;
let callbackMethod = listCallbackUrlResponseValue.method;
let LAResult = await this.request.fetch(listCallbackUrl, {
data: dataToSend,
method: callbackMethod,
headers: {
'Content-Type': 'text/plain',
},
});
while (LAResult && (LAResult.status() !== expectedStatus || await LAResult.text() !== expectedBody)) {
await this.page.waitForTimeout(4000);
listCallbackUrlCall = await this.request.post(
`${Constants.managementUrl}${this.siteId}/hostruntime/runtime/webhooks/workflow/api/management/workflows/${this.workflowName}/triggers/${triggerName}/listCallbackUrl?api-version=${Constants.siteApiVersion}`,
{
headers: {
'Content-Type': 'application/json',
Authorization: `Bearer ${process.env.AZURE_MANAGEMENT_TOKEN}`,
'If-Match': '*',
},
}
);

listCallbackUrlResponseValue = await listCallbackUrlCall.json();
listCallbackUrl = listCallbackUrlResponseValue.value;
callbackMethod = listCallbackUrlResponseValue.method;
LAResult = await this.request.fetch(listCallbackUrl, {
data: dataToSend,
method: callbackMethod,
});
}

expect(LAResult.status()).toBe(expectedStatus);
expect(await LAResult.text()).toBe(expectedBody);
}
async deployWorkflow(workflowData: any) {
if ((workflowData.kind as string).toLowerCase() === 'stateless') {
this.workflowName = `${this.workflowName}-stateless`;
}
return this.request.post(`${Constants.managementUrl}${this.siteId}/deployWorkflowArtifacts?api-version=${Constants.siteApiVersion}`, {
data: {
files: {
[`${this.workflowName}/workflow.json`]: workflowData,
},
},
headers: {
'Content-Type': 'application/json',
Authorization: `Bearer ${process.env.AZURE_MANAGEMENT_TOKEN}`,
'If-Match': '*',
},
});
}
}

type MyFixtures = {
realDataApi: RealDataApi;
};

export const test = base.extend<MyFixtures>({
realDataApi: async ({ page, request }, use) => {
await use(new RealDataApi(page, request));
},
});
export { expect } from '@playwright/test';
17 changes: 9 additions & 8 deletions e2e/designer/mock-copypastescope.spec.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import { test, expect } from '@playwright/test';
import { getSerializedWorkflowFromState } from './utils/designerFunctions';
import { GoToMockWorkflow } from './utils/GoToWorkflow';

test(
'Mock: Expect Copy and Paste of Scopes to work on single workflow',
Expand All @@ -8,14 +9,14 @@ test(
},
async ({ page }) => {
await page.goto('/');
await page.getByText('Select an option').click();
await page.getByRole('option', { name: 'Conditionals', exact: true }).click();
await page.getByRole('button', { name: 'Toolbox' }).click();
await page.waitForLoadState('networkidle');
await page.getByTestId('rf__node-Condition-#scope').getByRole('button', { name: 'Condition' }).focus();
await page.keyboard.press('Control+C');
await page.getByTestId('rf__edge-Initialize_variable-Condition').getByLabel('Insert a new step between').focus();
await page.keyboard.press('Control+V');
await GoToMockWorkflow(page, 'Conditionals');
await page.getByTestId('card-condition').click({
button: 'right',
});
await page.getByTestId('msla-copy-menu-option').click();

await page.getByTestId('msla-plus-button-initialize_variable-condition').click();
await page.getByTestId('msla-paste-button-initialize_variable-condition').click();
await page.waitForTimeout(1000);
const serialized: any = await getSerializedWorkflowFromState(page);
expect(serialized.definition).toEqual(verificationWorkflow);
Expand Down
Loading

0 comments on commit 8eaf2e1

Please sign in to comment.