[TE-3912] Add support for tagging executions in playwright

README.md

@@ -141,6 +141,14 @@ Official [Buildkite Test Engine](https://buildkite.com/platform/test-engine) col
     ],
     ```
 
+    If you would like to pass execution tags through to Test Engine, then you can use Playwright's tagging syntax as follows:
+
+    ```
+    test('has tags', { tag: ['@type:feature'] }, ...)
+    ```
+
+    This will be threaded through to Test Engine as an execution tag with key set to `type` and value set to `feature`.
+
    ### Cypress
 
    Update your [Cypress configuration](https://docs.cypress.io/guides/references/configuration):<br>

e2e/playwright.test.js

@@ -61,25 +61,28 @@ describe('examples/playwright', () => {
     expect(data).toHaveProperty("run_env.version")
     expect(data).toHaveProperty("run_env.collector", "js-buildkite-test-collector")
 
-    expect(data).toHaveProperty("tags", { "hello": "playwright" }) // examples/playwright/playwright.config.js
-
     expect(data).toHaveProperty("data[0].scope", ' chromium example.spec.js has title')
     expect(data).toHaveProperty("data[0].name", 'has title')
     expect(data).toHaveProperty("data[0].location", "tests/example.spec.js:3:1")
     expect(data).toHaveProperty("data[0].file_name", "tests/example.spec.js")
-    expect(data).toHaveProperty("data[0].result", 'passed')
+    expect(data).toHaveProperty("data[0].result", "passed")
 
     expect(data).toHaveProperty("data[1].scope", " chromium example.spec.js says hello")
     expect(data).toHaveProperty("data[1].name", "says hello")
     expect(data).toHaveProperty("data[1].location", "tests/example.spec.js:9:1")
     expect(data).toHaveProperty("data[1].file_name", "tests/example.spec.js")
     expect(data).toHaveProperty("data[1].result", "failed")
-    expect(data).toHaveProperty("data[1].failure_reason", "Timed out 5000ms waiting for expect(received).toHaveText(expected)")
+    expect(data).toHaveProperty("data[1].failure_reason", "Error: Timed out 5000ms waiting for expect(locator).toHaveText(expected)")
     expect(data).toHaveProperty("data[1].failure_expanded", expect.arrayContaining([
       expect.objectContaining({
-        expanded: expect.arrayContaining(['Expected string: "Hello, World!"', 'Received string: ""'])
+        expanded: expect.arrayContaining(['Expected string: "Hello, World!"', 'Received: <element(s) not found>'])
       })
     ]))
+
+    expect(data).toHaveProperty("data[2].tags", { foo: "bar" });
+    expect(data).toHaveProperty("data[3].tags", { foo: "bar", baz: 'qux' });
+    expect(data).toHaveProperty("data[4].tags", {});
+
     expect(stdout).toMatch(/Test Engine .* response/m)
   }, TIMEOUT);
 
@@ -117,10 +120,10 @@ describe('examples/playwright', () => {
       expect(data).toHaveProperty("data[0].location", "tests/example.spec.js:3:1")
       expect(data).toHaveProperty("data[0].file_name", "tests/example.spec.js")
       expect(data).toHaveProperty("data[0].result", 'failed')
-      expect(data).toHaveProperty("data[1].failure_reason", "Test timeout of 1ms exceeded.")
+      expect(data).toHaveProperty("data[1].failure_reason", "Test timeout of 1ms exceeded while setting up \"browserName\".")
       expect(data).toHaveProperty("data[1].failure_expanded", expect.arrayContaining([
         expect.objectContaining({
-          expanded: expect.arrayContaining(["Test timeout of 1ms exceeded."])
+          expanded: expect.arrayContaining(["Test timeout of 1ms exceeded while setting up \"browserName\"."])
         })
       ]))
 
@@ -129,10 +132,10 @@ describe('examples/playwright', () => {
       expect(data).toHaveProperty("data[1].location", "tests/example.spec.js:9:1")
       expect(data).toHaveProperty("data[1].file_name", "tests/example.spec.js")
       expect(data).toHaveProperty("data[1].result", "failed")
-      expect(data).toHaveProperty("data[1].failure_reason", "Test timeout of 1ms exceeded.")
+      expect(data).toHaveProperty("data[1].failure_reason", "Test timeout of 1ms exceeded while setting up \"browserName\".")
       expect(data).toHaveProperty("data[1].failure_expanded", expect.arrayContaining([
         expect.objectContaining({
-          expanded: expect.arrayContaining(["Test timeout of 1ms exceeded."])
+          expanded: expect.arrayContaining(["Test timeout of 1ms exceeded while setting up \"browserName\"."])
         })
       ]))
       expect(stdout).toMatch(/Test Engine .* response/m)

examples/playwright/package.json

@@ -1,7 +1,7 @@
 {
   "name": "buildkite-test-collector-playwright-example",
   "devDependencies": {
-    "@playwright/test": "^1.38.1",
+    "@playwright/test": "^1.43.0",
     "buildkite-test-collector": "file:../.."
   },
   "scripts": {

examples/playwright/tests/example.spec.js

@@ -13,3 +13,21 @@ test('says hello', async ({ page }, { retry }) => {
   const h1 = await page.locator(retry == 0 ? 'h2' : 'h1');
   await expect(h1).toHaveText('Hello, World!');
 });
+
+test('has scalar tag', { tag: '@foo:bar' }, async ({ page }) => {
+  await page.goto('/');
+
+  await expect(page).toHaveTitle(/Buildkite/);
+});
+
+test('has array of tags', { tag: ['@foo:bar', '@baz:qux'] }, async ({ page }) => {
+  await page.goto('/');
+
+  await expect(page).toHaveTitle(/Buildkite/);
+});
+
+test("has a tag we don't care about", { tag: '@foobar' }, async ({ page }) => {
+  await page.goto('/');
+
+  await expect(page).toHaveTitle(/Buildkite/);
+});

package.json

@@ -29,7 +29,7 @@
     "uuid": "^8.3.2"
   },
   "devDependencies": {
-    "@playwright/test": "^1.38.1",
+    "@playwright/test": "^1.43.0",
     "jasmine": "^4.2.1",
     "jest": "^29.3.1",
     "mocha": "^10.0.0",

playwright/reporter.js

@@ -27,6 +27,7 @@ class PlaywrightBuildkiteTestEngineReporter {
     this._tags = options?.tags;
     this._options = options;
     this._paths = new Paths({ cwd: process.cwd() }, this._testEnv.location_prefix);
+    this._tagPattern = /^@[^:]+:[^:]+$/;
   }
 
   onBegin() { }
@@ -49,6 +50,14 @@ class PlaywrightBuildkiteTestEngineReporter {
     const fileName = this._paths.prefixTestPath(test.location.file);
     const location = [fileName, test.location.line, test.location.column].join(':');
 
+    const filteredTags = test.tags.filter(tag => this._tagPattern.test(tag));
+    const tags = filteredTags.reduce((acc, test) => {
+      const [key, value] = test.slice(1).split(':');
+      acc[key] = value;
+
+      return acc;
+    }, {});
+
     this._testResults.push({
       'id': test.id,
       'name': test.title,
@@ -62,7 +71,8 @@ class PlaywrightBuildkiteTestEngineReporter {
         'section': 'top',
         'start_at': testResult.startTime.getTime(),
         'duration': testResult.duration / 1000,
-      }
+      },
+      'tags': tags
     });
   }