Add other CI jobs

.circleci/config.yml

@@ -478,14 +478,133 @@ jobs:
           command: yarn wait-on http://localhost:6000
           working_directory: code
       - run:
-          name: Preparing Sandboxes
-          command: yarn build-sandboxes --cadence ci
+          name: Creating Sandboxes
+          command: yarn create-sandboxes --cadence ci --junit test-results/create.xml
           working_directory: code
       - persist_to_workspace:
           root: .
           paths:
             - sandbox
-
+      - store_test_results:
+          path: code/test-results
+  smoke-test-sandboxes:
+    executor:
+      class: medium+
+      name: sb_node_14_browsers
+    parallelism: 2
+    steps:
+      - git-shallow-clone/checkout_advanced:
+          clone_options: '--depth 1 --verbose'
+      - attach_workspace:
+          at: .
+      - run:
+          name: Smoke Testing Sandboxes
+          command: yarn smoke-test-sandboxes --cadence ci --junit test-results/smoke-test.xml
+          working_directory: code
+      - store_test_results:
+          path: code/test-results
+  build-sandboxes:
+    executor:
+      class: medium+
+      name: sb_node_14_browsers
+    parallelism: 2
+    steps:
+      - git-shallow-clone/checkout_advanced:
+          clone_options: '--depth 1 --verbose'
+      - attach_workspace:
+          at: .
+      - run:
+          name: Building Sandboxes
+          command: yarn build-sandboxes --cadence ci --junit test-results/build.xml
+          working_directory: code
+      - run:
+          name: Build index.html
+          command: yarn create-built-sandboxes-index --cadence ci --output "../built-sandboxes"
+          background: true
+          working_directory: code
+      - store_test_results:
+          path: code/test-results
+      - persist_to_workspace:
+          root: .
+          paths:
+            - built-sandboxes
+  test-runner-sandboxes:
+    executor:
+      class: medium+
+      name: sb_node_14_browsers
+    parallelism: 2
+    steps:
+      - git-shallow-clone/checkout_advanced:
+          clone_options: '--depth 1 --verbose'
+      - attach_workspace:
+          at: .
+      - run:
+          name: Running local registry
+          command: yarn local-registry --port 6000 --open
+          background: true
+          working_directory: code
+      - run:
+          name: Wait for registry
+          command: yarn wait-on http://localhost:6000
+          working_directory: code
+      - run:
+          name: Serve Sandboxes
+          command: yarn http-server ../built-sandboxes -p 8001
+          background: true
+          working_directory: code
+      - run:
+          name: Await Serving Sandboxes
+          command: yarn wait-on http://localhost:8001
+          working_directory: code
+      - run:
+          name: Running Test Runner
+          command: yarn test-runner-sandboxes --cadence ci --junit test-results/test-runner.xml
+          working_directory: code
+      - store_test_results:
+          path: code/test-results
+  chromatic-sandboxes:
+    executor:
+      class: medium+
+      name: sb_node_14_browsers
+    parallelism: 2
+    steps:
+      - git-shallow-clone/checkout_advanced:
+          clone_options: '--depth 1 --verbose'
+      - attach_workspace:
+          at: .
+      - run:
+          name: Running Chromatic
+          command: yarn chromatic-sandboxes --cadence ci --junit test-results/chromatic.xml
+          working_directory: code
+      - store_test_results:
+          path: code/test-results
+  e2e-sandboxes:
+    docker:
+      - image: mcr.microsoft.com/playwright:v1.24.0-focal
+    parallelism: 2
+    steps:
+      - git-shallow-clone/checkout_advanced:
+          clone_options: '--depth 1 --verbose'
+      - attach_workspace:
+          at: .
+      - run:
+          name: Serve Sandboxes
+          command: yarn http-server ../built-sandboxes -p 8001
+          background: true
+          working_directory: code
+      - run:
+          name: Await Serving Sandboxes
+          command: yarn wait-on http://localhost:8001
+          working_directory: code
+      - run:
+          name: Running E2E Tests
+          command: yarn e2e-sandboxes --cadence ci --junit test-results/e2e.xml
+          working_directory: code
+      - store_test_results:
+          path: code/test-results
+      - store_artifacts: # this is where playwright puts more complex stuff
+          path: code/playwright-results/
+          destination: playwright
 
 workflows:
   test:
@@ -535,4 +654,19 @@ workflows:
       - create-sandboxes:
           requires:
             - publish
+      - smoke-test-sandboxes:
+          requires:
+            - create-sandboxes
+      - build-sandboxes:
+          requires:
+            - create-sandboxes
+      - test-runner-sandboxes:
+          requires:
+            - build-sandboxes
+      - chromatic-sandboxes:
+          requires:
+            - build-sandboxes
+      - e2e-sandboxes:
+          requires:
+            - build-sandboxes
 

.gitignore

@@ -9,6 +9,7 @@ dist
 junit.xml
 /repros
 /sandbox
+/built-sandboxes
 .verdaccio-cache
 
 # Yarn stuff

code/.eslintrc.js

@@ -95,5 +95,11 @@ module.exports = {
         'react/no-unknown-property': 'off', // Need to deactivate otherwise eslint replaces some unknown properties with React ones
       },
     },
+    {
+      files: ['**/e2e-tests/**/*'],
+      rules: {
+        'jest/no-test-callback': 'off', // These aren't jest tests
+      },
+    },
   ],
 };

code/.gitignore

@@ -36,6 +36,12 @@ examples/angular-cli/addon-jest.testresults.json
 junit.xml
 .next
 
+/test-results/
+/playwright-results/
+/playwright-report/
+/playwright/.cache/
+
+
 # Yarn stuff
 /**/.yarn/*
 !/**/.yarn/releases

code/e2e-tests/example.spec.ts

@@ -0,0 +1,14 @@
+import { test, expect } from '@playwright/test';
+import process from 'process';
+
+const storybookUrl = process.env.STORYBOOK_URL || 'http://localhost:8001';
+
+test('Basic story test', async ({ page }) => {
+  await page.goto(storybookUrl);
+
+  const preview = page.frameLocator('#storybook-preview-iframe');
+  const root = preview.locator('#root:visible, #docs-root:visible');
+
+  // Specific check for introduction story
+  await expect(root).toContainText('Welcome');
+});

code/package.json

@@ -60,9 +60,13 @@
     "changelog": "pr-log --sloppy --cherry-pick",
     "changelog:next": "pr-log --sloppy --since-prerelease",
     "check": "NODE_ENV=production node ../scripts/check-package.js",
+    "chromatic-sandboxes": "ts-node ../scripts/chromatic-sandboxes.ts",
     "clean:dist": "del **/dist",
     "coverage": "codecov",
+    "create-built-sandboxes-index": "ts-node ../scripts/create-built-sandboxes-index",
+    "create-sandboxes": "ts-node ../scripts/create-sandboxes.ts",
     "danger": "danger",
+    "e2e-sandboxes": "ts-node ../scripts/e2e-sandboxes.ts",
     "generate-repros": "zx ../scripts/repros-generator/index.mjs",
     "github-release": "github-release-from-changelog",
     "linear-export": "ts-node --project=../scripts/tsconfig.json ../scripts/linear-export.ts",
@@ -83,13 +87,16 @@
     "run-chromatics": "node -r esm ../scripts/run-chromatics.js",
     "sandbox": "ts-node ../scripts/sandbox.ts",
     "serve-storybooks": "http-server ./built-storybooks -p 8001",
+    "smoke-test-sandboxes": "ts-node ../scripts/smoke-test-sandboxes.ts",
     "smoketest-storybooks": "cross-env STORYBOOK_DISPLAY_WARNING=true DISPLAY_WARNING=true node -r esm ../scripts/smoketest-storybooks.js",
     "start": "yarn workspace official-storybook storybook --no-manager-cache",
     "test": "NODE_OPTIONS=--max_old_space_size=4096 jest --config ./jest.config.js",
     "test-puppeteer": "jest --projects examples/official-storybook/storyshots-puppeteer",
+    "test-runner-sandboxes": "ts-node ../scripts/test-runner-sandboxes.ts",
     "test:cli": "npm --prefix lib/cli run test",
     "test:e2e-examples": "cypress run",
     "test:e2e-examples-gui": "concurrently --success first --kill-others \"cypress open\" \"yarn serve-storybooks\"",
+    "test:e2e-examples-playwright": "playwright test",
     "test:e2e-framework": "ts-node --project=../scripts/tsconfig.json ../scripts/run-e2e.ts"
   },
   "husky": {
@@ -153,6 +160,7 @@
     "@nrwl/nx-cloud": "12.1.1",
     "@nrwl/tao": "12.3.4",
     "@nrwl/workspace": "12.3.4",
+    "@playwright/test": "^1.24.2",
     "@rollup/plugin-babel": "^5.3.1",
     "@rollup/plugin-commonjs": "^21.0.1",
     "@rollup/plugin-json": "^4.1.0",
@@ -316,6 +324,7 @@
     "node-gyp": "^8.4.0",
     "npmlog": "^5.0.1",
     "p-limit": "^3.1.0",
+    "playwright": "^1.24.2",
     "postcss-loader": "^6.2.1",
     "prettier": ">=2.2.1 <=2.3.0",
     "prompts": "^2.4.0",

code/playwright.config.ts

@@ -0,0 +1,107 @@
+import type { PlaywrightTestConfig } from '@playwright/test';
+import { devices } from '@playwright/test';
+
+/**
+ * Read environment variables from file.
+ * https://github.com/motdotla/dotenv
+ */
+// require('dotenv').config();
+
+/**
+ * See https://playwright.dev/docs/test-configuration.
+ */
+const config: PlaywrightTestConfig = {
+  testDir: './e2e-tests',
+  /* Maximum time one test can run for. */
+  timeout: 30 * 1000,
+  expect: {
+    /**
+     * Maximum time expect() should wait for the condition to be met.
+     * For example in `await expect(locator).toHaveText();`
+     */
+    timeout: 5000,
+  },
+  /* Run tests in files in parallel */
+  fullyParallel: true,
+  /* Fail the build on CI if you accidentally left test.only in the source code. */
+  forbidOnly: !!process.env.CI,
+  /* Retry on CI only */
+  retries: process.env.CI ? 2 : 0,
+  /* Opt out of parallel tests on CI. */
+  workers: process.env.CI ? 1 : undefined,
+  /* Reporter to use. See https://playwright.dev/docs/test-reporters */
+  reporter: 'html',
+  /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
+  use: {
+    /* Maximum time each action such as `click()` can take. Defaults to 0 (no limit). */
+    actionTimeout: 0,
+    /* Base URL to use in actions like `await page.goto('/')`. */
+    // baseURL: 'http://localhost:3000',
+
+    /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
+    trace: 'on-first-retry',
+  },
+
+  /* Configure projects for major browsers */
+  projects: [
+    {
+      name: 'chromium',
+      use: {
+        ...devices['Desktop Chrome'],
+      },
+    },
+
+    {
+      name: 'firefox',
+      use: {
+        ...devices['Desktop Firefox'],
+      },
+    },
+
+    {
+      name: 'webkit',
+      use: {
+        ...devices['Desktop Safari'],
+      },
+    },
+
+    /* Test against mobile viewports. */
+    // {
+    //   name: 'Mobile Chrome',
+    //   use: {
+    //     ...devices['Pixel 5'],
+    //   },
+    // },
+    // {
+    //   name: 'Mobile Safari',
+    //   use: {
+    //     ...devices['iPhone 12'],
+    //   },
+    // },
+
+    /* Test against branded browsers. */
+    // {
+    //   name: 'Microsoft Edge',
+    //   use: {
+    //     channel: 'msedge',
+    //   },
+    // },
+    // {
+    //   name: 'Google Chrome',
+    //   use: {
+    //     channel: 'chrome',
+    //   },
+    // },
+  ],
+
+  /* Folder for test artifacts such as screenshots, videos, traces, etc. */
+  outputDir: 'playwright-results/',
+
+  /* Run your local dev server before starting the tests */
+  // webServer: {
+  //   command: 'npm run start',
+  //   port: 3000,
+  // },
+};
+
+export default config;