feat: Implement pipeline e2e test (#280)

Because

- #270

This commit

- Implement pipeline e2e test

integration-test/README.md

@@ -1,14 +1,23 @@
-# Caveats
 
-## About the usage of `click({force: true})`
+## Future improvemnet
+
+- Use page object model
+
+## Caveats
+
+### Use `locator.fill()` instead of `locator.type()`
+
+`locator.type()` may act different among browser and cause troubles, if you don't need to test typing feature, we encourage you to use `locator.fill()`.
+
+### About the usage of `click({force: true})`
 
 - Our SingleSelect have multiple clickable item, including the label and the select itself. Sometimes the label will intercept
   the click event, so we recommend to bypass the [actionaility](https://playwright.dev/docs/actionability) check on these elements.
 - related issues
   - [subtree intercepts pointer events, Unable to click](https://github.com/microsoft/playwright/issues/13576)
   - [Chromium: Cannot click, element intercepts pointer events](https://github.com/microsoft/playwright/issues/12821)
 
-## About the env variables
+### About the env variables
 
 - We use `.env` and `dotenv` to digest environment variables.
 - Make sure your `.env` file have following variables:
@@ -20,14 +29,50 @@
   - NEXT_PUBLIC_API_VERSION=v1alpha
   - NEXT_PUBLIC_INSTILL_AI_USER_COOKIE_NAME=instill-ai-user
 
-## About the config `fullyParallel`
+### About the config `fullyParallel`
 
 - This config will force every test run in parallel even you specific `test.describe.serial`.
 - We recommend you set `fullyParallel: false` and control this behavior with fine-grained control.
 
-## About the flaky test
+### About the flaky test
 
 - If the test behavior is related to backend, remember that backend can only handle a request at a time. So if the test run in sequence and the time between requests is too short, the request will fail.
 - We have to limit the test worker to one, because the test suite might run to quick to make backend panic.
 - Remember to `make down` backend every time you want have another round of test.
-- use `expect().to` after every behavior to make sure the behavior succeeded.
+- use `expect().to` after every behavior to make sure the behavior succeeded.
+- `page.waitForResponse` is not particularly reliable. If you are facing some flaky test, try to rewrite the whole part with some visual hint, like.
+
+```js
+
+// waitForResponse is flaky
+
+const saveButton = page.locator("button", { hasText: "Save" });
+expect(await saveButton.isEnabled()).toBeTruthy();
+const succeedMessage = page.locator("h3", { hasText: "Succeed" });
+await Promise.all([saveButton.click(), page.waitForResponse("your url"));
+
+// Rewrite with visual hint
+
+const saveButton = page.locator("button", { hasText: "Save" });
+expect(await saveButton.isEnabled()).toBeTruthy();
+const succeedMessage = page.locator("h3", { hasText: "Succeed" });
+await Promise.all([saveButton.click(), succeedMessage.isVisible()]);
+```
+
+- If your interaction will trigger some action on other element, wrap them with promise.all.
+
+```js
+
+// This will be flaky
+
+await editButton.click();
+await expect(pipelineDescriptionField.isEditable()).toBeTruthy(),
+
+// Wrap with promise.all to avoid flaky test
+
+await Promise.all([
+  editButton.click(),
+  expect(pipelineDescriptionField.isEditable()).toBeTruthy(),
+]);
+
+```

integration-test/common/mgmt.ts

@@ -1,6 +1,52 @@
+import { BrowserContext, Page, expect } from "@playwright/test";
+
 export const removeRegisteredUser = async () => {
   await fetch(
     `${process.env.NEXT_PUBLIC_MGMT_BACKEND_BASE_URL}/${process.env.NEXT_PUBLIC_API_VERSION}/local-user`,
     { method: "patch", body: JSON.stringify({ cookie_token: "" }) }
   );
 };
+
+export const expectToOnboardUser = async (
+  page: Page,
+  context: BrowserContext,
+  browserName: string
+) => {
+  await page.goto("/onboarding", { waitUntil: "networkidle" });
+
+  // Should input email
+  const emailField = page.locator("input#email");
+  await emailField.fill("droplet@instill.tech");
+
+  // Should input company name
+  const companyField = page.locator("input#companyName");
+  await companyField.fill("instill-ai");
+
+  // Shoyld select role
+  await page.locator("#role").click({ force: true });
+  await page.locator("#react-select-role-option-0").click();
+  await expect(page.locator("data-testid=role-selected-option")).toHaveText(
+    "Manager (who makes decisions)"
+  );
+
+  // Should accept newsletter subscription
+  await page.locator("#newsletterSubscription").check();
+
+  const startButton = page.locator("button", { hasText: "Start" });
+  expect(await startButton.isEnabled()).toBeTruthy();
+
+  // Should submit the onboarding form
+  await Promise.all([page.waitForNavigation(), startButton.click()]);
+
+  expect(page.url()).toBe(`${process.env.NEXT_PUBLIC_MAIN_URL}/pipelines`);
+
+  // Should have cookie
+  // Safari have issue when setting up cookies.
+  if (browserName !== "webkit") {
+    const cookies = await context.cookies();
+    const instillCookies = cookies.find(
+      (e) => e.name === process.env.NEXT_PUBLIC_INSTILL_AI_USER_COOKIE_NAME
+    );
+    expect(instillCookies).toBeDefined();
+  }
+};

integration-test/common/model.ts

@@ -0,0 +1,222 @@
+import { Page, expect, Locator } from "@playwright/test";
+
+export const expectToDeleteModel = async (page: Page, modelId: string) => {
+  await page.goto(`/models/${modelId}`, { waitUntil: "networkidle" });
+
+  // Should enable open delete model modal button
+  const openDeleteModelModalButton = page.locator("button", {
+    hasText: "Delete",
+  });
+  expect(await openDeleteModelModalButton.isEnabled()).toBeTruthy();
+
+  // Should open delete model modal
+  const deleteResourceModal = page.locator("data-testid=delete-resource-modal");
+
+  await Promise.all([
+    openDeleteModelModalButton.click(),
+    deleteResourceModal.isVisible(),
+  ]);
+
+  // Should have proper modal title
+  const modalTitle = deleteResourceModal.locator("h2", {
+    hasText: "Delete This Model",
+  });
+  await expect(modalTitle).toHaveCount(1);
+
+  // Should have proper confirmation code hint
+  const confirmationCodeHint = deleteResourceModal.locator("label", {
+    hasText: `Please type "${modelId}" to confirm.`,
+  });
+  await expect(confirmationCodeHint).toHaveCount(1);
+
+  // Should disable delete model button
+  const deleteButton = deleteResourceModal.locator("button", {
+    hasText: "Delete",
+  });
+  expect(await deleteButton.isDisabled()).toBeTruthy();
+
+  // Should enable cancel button
+  const cancelButton = deleteResourceModal.locator("button", {
+    hasText: "Cancel",
+  });
+  expect(await cancelButton.isEnabled()).toBeTruthy();
+
+  // Should input confirmation code
+  const confirmationCodeInput =
+    deleteResourceModal.locator("#confirmationCode");
+
+  await Promise.all([
+    confirmationCodeInput.fill(modelId),
+    deleteButton.isEnabled(),
+  ]);
+
+  // Should delete model and navigate to models page
+  await Promise.all([page.waitForNavigation(), deleteButton.click()]);
+  expect(page.url()).toEqual(`${process.env.NEXT_PUBLIC_MAIN_URL}/models`);
+
+  // Should not list model item
+  const modelItemTitle = page.locator("h3", { hasText: modelId });
+  await expect(modelItemTitle).toHaveCount(0);
+};
+
+export const expectToUpdateModelDescription = async (
+  page: Page,
+  modelId: string,
+  modelDescription: string
+) => {
+  await page.goto(`/models/${modelId}`, { waitUntil: "networkidle" });
+
+  // Should enable edit button
+  const editButton = page.locator("button", { hasText: "Edit" });
+  expect(await editButton.isEnabled()).toBeTruthy();
+
+  // Should disable description field
+  const modelDescriptionField = page.locator("#description");
+  expect(await modelDescriptionField.isDisabled()).toBeTruthy();
+
+  // Should enable description field
+  await Promise.all([editButton.click(), modelDescriptionField.isEnabled()]);
+
+  // Should display save button
+  const saveButton = page.locator("button", { hasText: "Save" });
+  expect(await saveButton.isEnabled()).toBeTruthy();
+
+  // Should update model description
+  const succeedMessage = page.locator("h3", { hasText: "Succeed" });
+  await modelDescriptionField.fill(modelDescription);
+  await Promise.all([saveButton.click(), succeedMessage.isVisible()]);
+
+  // Reload page
+  await page.goto(`/models/${modelId}`);
+
+  // Should have updated model description
+  await modelDescriptionField.isVisible();
+  await expect(modelDescriptionField).toHaveValue(modelDescription);
+};
+
+export const expectCorrectModelList = async (page: Page, modelId: string) => {
+  await page.goto("/models", { waitUntil: "networkidle" });
+
+  // Should list model item
+  const modelItemTitle = page.locator("h3", { hasText: modelId });
+  await expect(modelItemTitle).toHaveCount(1);
+
+  // Should navigate to model details page
+  await Promise.all([
+    page.locator("h3", { hasText: modelId }).click(),
+    page.waitForNavigation(),
+  ]);
+  expect(page.url()).toEqual(
+    `${process.env.NEXT_PUBLIC_MAIN_URL}/models/${modelId}`
+  );
+};
+
+export type ExpectCorrectModelDetailsProps = {
+  page: Page;
+  modelId: string;
+  modelDescription: string;
+  modelInstanceTag: string;
+  modelState:
+    | "STATE_ONLINE"
+    | "STATE_OFFLINE"
+    | "STATE_UNSPECIFIED"
+    | "STATE_ERROR";
+  modelTask: string;
+  additionalRule?: () => Promise<void>;
+};
+
+export const expectCorrectModelDetails = async ({
+  page,
+  modelId,
+  modelDescription,
+  modelInstanceTag,
+  modelState,
+  modelTask,
+  additionalRule,
+}: ExpectCorrectModelDetailsProps) => {
+  await page.goto(`/models/${modelId}`, { waitUntil: "networkidle" });
+
+  // Should have proper title
+  const modelDetailsPageTitle = page.locator("h2", { hasText: modelId });
+  await expect(modelDetailsPageTitle).toHaveCount(2);
+
+  // Should have proper model description
+  const modelDescriptionField = page.locator("#description");
+  await expect(modelDescriptionField).toHaveValue(modelDescription);
+
+  // Should have correct model instance tag
+  const modelInstanceTagOption = page.locator(
+    "data-testid=modelInstanceTag-selected-option"
+  );
+  await expect(modelInstanceTagOption).toHaveText(modelInstanceTag);
+
+  // Should display task fill classification
+  const modelTaskLabel = page.locator("data-testid=model-task-label");
+  await expect(modelTaskLabel).toHaveText(modelTask);
+
+  // Should display online and have correct toggle button state
+  const modelStateLabel = page.locator("data-testid=state-label");
+  const stateToggle = page.locator("#pipelineStateToggleButton");
+  if (modelState === "STATE_ONLINE") {
+    await expect(modelStateLabel).toHaveText("Online");
+    expect(await stateToggle.isChecked()).toBeTruthy();
+  } else if (modelState === "STATE_OFFLINE") {
+    await expect(modelStateLabel).toHaveText("Offline");
+    expect(await stateToggle.isChecked()).not.toBeTruthy();
+  } else if (modelState === "STATE_UNSPECIFIED") {
+    await expect(modelStateLabel).toHaveText("Unspecified");
+    expect(await stateToggle.isChecked()).not.toBeTruthy();
+  } else {
+    await expect(modelStateLabel).toHaveText("Error");
+    expect(await stateToggle.isChecked()).not.toBeTruthy();
+  }
+
+  if (additionalRule) await additionalRule();
+};
+
+export const expectToDeployModel = async (
+  page: Page,
+  modelInstanceTag: string,
+  setupButton: Locator,
+  timeout?: number
+) => {
+  // Should create model and display model instance section
+  const modelInstanceTitle = page.locator("h3", {
+    hasText: "Deploy a model instance",
+  });
+  const modelInstanceIdOption = page.locator(
+    "#react-select-modelInstanceId-input"
+  );
+  const deployButton = page.locator("button", { hasText: "Deploy" });
+
+  await Promise.all([
+    modelInstanceTitle.isVisible(),
+    modelInstanceIdOption.isVisible(),
+    deployButton.isVisible(),
+    setupButton.click(),
+  ]);
+
+  // Should disable deploy button
+  expect(await deployButton.isDisabled()).toBeTruthy();
+
+  // Should select latest model instance
+  await modelInstanceIdOption.click({ force: true });
+  await page
+    .locator("data-testid=modelInstanceId-selected-option", {
+      hasText: modelInstanceTag,
+    })
+    .click();
+  await expect(
+    page.locator("data-testid=modelInstanceId-selected-option")
+  ).toHaveText(modelInstanceTag);
+
+  // Should enable deploy button
+  expect(await deployButton.isEnabled()).toBeTruthy();
+
+  // Should deploy model
+  await Promise.all([
+    page.waitForNavigation({ timeout }),
+    deployButton.click(),
+  ]);
+  expect(page.url()).toEqual(`${process.env.NEXT_PUBLIC_MAIN_URL}/models`);
+};