test: add new tests based on new basic examples + vector store tests (l…

…angflow-ai#2879) * 🔧 (.github/workflows/typescript_test.yml): add BRAVE_SEARCH_API_KEY secret to workflow environment variables ✨ (frontend/src/modals/IOModal/components/chatView/chatMessage/index.tsx): add data-testid attribute to div element for testing purposes ✨ (frontend/tests/end-to-end/Hierarchical Tasks Agent.spec.ts): add end-to-end test for Hierarchical Tasks Agent ✨ (frontend/tests/end-to-end/Sequential Tasks Agent.spec.ts): add end-to-end test for Sequential Tasks Agent * updating hierarchical task json * 🔧 (.github/workflows/typescript_test.yml): add secrets for ASTRA_DB_APPLICATION_TOKEN and ASTRA_DB_API_ENDPOINT ♻️ (playwright.config.ts): refactor timeout value to 3 minutes 📝 (Complex Agent.spec.ts): update test to use BRAVE_SEARCH_API_KEY instead of OPENAI_API_KEY 📝 (Complex Agent.spec.ts): update test to use "apple" instead of "agile" 📝 (Hierarchical Tasks Agent.spec.ts): update test to use "langflow" instead of "agile" 📝 (Sequential Tasks Agent.spec.ts): update test to use waitForTimeout instead of isVisible ✨ (Vector Store.spec.ts): Add checks to skip tests if required environment variables are not set 🔧 (Vector Store.spec.ts): Update test logic to fill in required environment variables before running tests to ensure proper test execution
datastax · Jul 22, 2024 · ae096a3 · ae096a3
1 parent 9ecf101
commit ae096a3
Show file tree

Hide file tree

Showing 8 changed files with 776 additions and 388 deletions.
diff --git a/.github/workflows/typescript_test.yml b/.github/workflows/typescript_test.yml
@@ -56,6 +56,9 @@ jobs:
     env:
       OPENAI_API_KEY: ${{ inputs.openai_api_key || secrets.OPENAI_API_KEY }}
       STORE_API_KEY: ${{ inputs.store_api_key || secrets.STORE_API_KEY }}
+      BRAVE_SEARCH_API_KEY: "${{ secrets.BRAVE_SEARCH_API_KEY }}"
+      ASTRA_DB_APPLICATION_TOKEN: "${{ secrets.ASTRA_DB_APPLICATION_TOKEN }}"
+      ASTRA_DB_API_ENDPOINT: "${{ secrets.ASTRA_DB_API_ENDPOINT }}"
     outputs:
       failed: ${{ steps.check-failure.outputs.failed }}
     steps:

diff --git a/src/backend/base/langflow/initial_setup/starter_projects/Hierarchical Agent.json b/src/backend/base/langflow/initial_setup/starter_projects/Hierarchical Agent.json
diff --git a/src/frontend/playwright.config.ts b/src/frontend/playwright.config.ts
@@ -23,7 +23,7 @@ export default defineConfig({
   /* Opt out of parallel tests on CI. */
   workers: 2,
   /* Reporter to use. See https://playwright.dev/docs/test-reporters */
-  timeout: 120 * 1000,
+  timeout: 3 * 60 * 1000,
   // reporter: [
   //   ["html", { open: "never", outputFolder: "playwright-report/test-results" }],
   // ],

diff --git a/src/frontend/src/modals/IOModal/components/chatView/chatMessage/index.tsx b/src/frontend/src/modals/IOModal/components/chatView/chatMessage/index.tsx
@@ -173,7 +173,10 @@ export default function ChatMessage({
               )}
               {chat.thought && chat.thought !== "" && !hidden && <br></br>}
               <div className="flex w-full flex-col">
-                <div className="flex w-full flex-col dark:text-white">
+                <div
+                  className="flex w-full flex-col dark:text-white"
+                  data-testid="div-chat-message"
+                >
                   <div
                     data-testid={
                       "chat-message-" + chat.sender_name + "-" + chatMessage

diff --git a/src/frontend/tests/end-to-end/Complex Agent.spec.ts b/src/frontend/tests/end-to-end/Complex Agent.spec.ts
@@ -0,0 +1,102 @@
+import { expect, test } from "@playwright/test";
+import * as dotenv from "dotenv";
+import path from "path";
+
+test("Complex Agent", async ({ page }) => {
+  test.skip(
+    !process?.env?.OPENAI_API_KEY,
+    "OPENAI_API_KEY required to run this test",
+  );
+
+  test.skip(
+    !process?.env?.BRAVE_SEARCH_API_KEY,
+    "BRAVE_SEARCH_API_KEY required to run this test",
+  );
+
+  if (!process.env.CI) {
+    dotenv.config({ path: path.resolve(__dirname, "../../.env") });
+  }
+
+  await page.goto("/");
+  await page.waitForTimeout(2000);
+
+  let modalCount = 0;
+  try {
+    const modalTitleElement = await page?.getByTestId("modal-title");
+    if (modalTitleElement) {
+      modalCount = await modalTitleElement.count();
+    }
+  } catch (error) {
+    modalCount = 0;
+  }
+
+  while (modalCount === 0) {
+    await page.getByText("New Project", { exact: true }).click();
+    await page.waitForTimeout(5000);
+    modalCount = await page.getByTestId("modal-title")?.count();
+  }
+
+  await page.getByRole("heading", { name: "Complex Agent" }).click();
+
+  await page.waitForSelector('[title="fit view"]', {
+    timeout: 100000,
+  });
+
+  await page.getByTitle("fit view").click();
+  await page.getByTitle("zoom out").click();
+  await page.getByTitle("zoom out").click();
+  await page.getByTitle("zoom out").click();
+
+  let outdatedComponents = await page.getByTestId("icon-AlertTriangle").count();
+
+  while (outdatedComponents > 0) {
+    await page.getByTestId("icon-AlertTriangle").first().click();
+    await page.waitForTimeout(1000);
+    outdatedComponents = await page.getByTestId("icon-AlertTriangle").count();
+  }
+
+  await page
+    .getByTestId("popover-anchor-input-api_key")
+    .last()
+    .fill(process.env.BRAVE_SEARCH_API_KEY ?? "");
+
+  await page.waitForTimeout(2000);
+
+  let openAiLlms = await page.getByText("OpenAI", { exact: true }).count();
+
+  for (let i = 0; i < openAiLlms; i++) {
+    await page
+      .getByTestId("popover-anchor-input-api_key")
+      .nth(i)
+      .fill(process.env.OPENAI_API_KEY ?? "");
+
+    await page.getByTestId("dropdown-model_name").nth(i).click();
+    await page.getByTestId("gpt-4o-1-option").last().click();
+
+    await page.waitForTimeout(1000);
+  }
+
+  await page.getByTestId("button_run_chat output").click();
+  await page.waitForSelector("text=built successfully", { timeout: 60000 * 3 });
+
+  await page.getByText("built successfully").last().click({
+    timeout: 15000,
+  });
+
+  await page.getByText("Playground", { exact: true }).click();
+
+  await page.waitForTimeout(2000);
+
+  expect(
+    page.getByText("Could you search info about AAPL?", { exact: true }).last(),
+  ).toBeVisible();
+
+  const textContents = await page
+    .getByTestId("div-chat-message")
+    .allTextContents();
+
+  const concatAllText = textContents.join(" ");
+  expect(concatAllText.toLocaleLowerCase()).toContain("apple");
+  const allTextLength = concatAllText.length;
+  expect(allTextLength).toBeGreaterThan(500);
+});
diff --git a/src/frontend/tests/end-to-end/Hierarchical Tasks Agent.spec.ts b/src/frontend/tests/end-to-end/Hierarchical Tasks Agent.spec.ts
@@ -0,0 +1,106 @@
+import { expect, test } from "@playwright/test";
+import * as dotenv from "dotenv";
+import path from "path";
+
+test("Hierarchical Tasks Agent", async ({ page }) => {
+  test.skip(
+    !process?.env?.OPENAI_API_KEY,
+    "OPENAI_API_KEY required to run this test",
+  );
+
+  test.skip(
+    !process?.env?.BRAVE_SEARCH_API_KEY,
+    "BRAVE_SEARCH_API_KEY required to run this test",
+  );
+
+  if (!process.env.CI) {
+    dotenv.config({ path: path.resolve(__dirname, "../../.env") });
+  }
+
+  await page.goto("/");
+  await page.waitForTimeout(2000);
+
+  let modalCount = 0;
+  try {
+    const modalTitleElement = await page?.getByTestId("modal-title");
+    if (modalTitleElement) {
+      modalCount = await modalTitleElement.count();
+    }
+  } catch (error) {
+    modalCount = 0;
+  }
+
+  while (modalCount === 0) {
+    await page.getByText("New Project", { exact: true }).click();
+    await page.waitForTimeout(5000);
+    modalCount = await page.getByTestId("modal-title")?.count();
+  }
+
+  await page.getByRole("heading", { name: "Hierarchical Tasks Agent" }).click();
+
+  await page.waitForSelector('[title="fit view"]', {
+    timeout: 100000,
+  });
+
+  await page.getByTitle("fit view").click();
+  await page.getByTitle("zoom out").click();
+  await page.getByTitle("zoom out").click();
+  await page.getByTitle("zoom out").click();
+
+  let outdatedComponents = await page.getByTestId("icon-AlertTriangle").count();
+
+  while (outdatedComponents > 0) {
+    await page.getByTestId("icon-AlertTriangle").first().click();
+    await page.waitForTimeout(1000);
+    outdatedComponents = await page.getByTestId("icon-AlertTriangle").count();
+  }
+
+  await page
+    .getByTestId("popover-anchor-input-api_key")
+    .first()
+    .fill(process.env.OPENAI_API_KEY ?? "");
+
+  await page
+    .getByTestId("popover-anchor-input-api_key")
+    .nth(1)
+    .fill(process.env.OPENAI_API_KEY ?? "");
+
+  await page.getByTestId("dropdown-model_name").first().click();
+  await page.getByTestId("gpt-4o-1-option").first().click();
+
+  await page.waitForTimeout(2000);
+
+  await page.getByTestId("dropdown-model_name").last().click();
+  await page.getByTestId("gpt-4o-1-option").last().click();
+
+  await page.waitForTimeout(2000);
+
+  await page
+    .getByTestId("popover-anchor-input-api_key")
+    .last()
+    .fill(process.env.BRAVE_SEARCH_API_KEY ?? "");
+
+  await page.waitForTimeout(2000);
+
+  await page.getByTestId("button_run_chat output").click();
+  await page.waitForSelector("text=built successfully", { timeout: 60000 * 3 });
+
+  await page.getByText("built successfully").last().click({
+    timeout: 15000,
+  });
+
+  await page.getByText("Playground", { exact: true }).click();
+
+  await page.waitForTimeout(2000);
+
+  expect(page.getByText("What is Langflow?", { exact: true })).toBeVisible();
+
+  const textContents = await page
+    .getByTestId("div-chat-message")
+    .allTextContents();
+
+  const concatAllText = textContents.join(" ");
+  expect(concatAllText.toLocaleLowerCase()).toContain("langflow");
+  const allTextLength = concatAllText.length;
+  expect(allTextLength).toBeGreaterThan(500);
+});
diff --git a/src/frontend/tests/end-to-end/Sequential Tasks Agent.spec.ts b/src/frontend/tests/end-to-end/Sequential Tasks Agent.spec.ts
@@ -0,0 +1,106 @@
+import { expect, test } from "@playwright/test";
+import * as dotenv from "dotenv";
+import path from "path";
+
+test("Sequential Tasks Agent", async ({ page }) => {
+  test.skip(
+    !process?.env?.OPENAI_API_KEY,
+    "OPENAI_API_KEY required to run this test",
+  );
+
+  test.skip(
+    !process?.env?.BRAVE_SEARCH_API_KEY,
+    "BRAVE_SEARCH_API_KEY required to run this test",
+  );
+
+  if (!process.env.CI) {
+    dotenv.config({ path: path.resolve(__dirname, "../../.env") });
+  }
+
+  await page.goto("/");
+  await page.waitForTimeout(2000);
+
+  let modalCount = 0;
+  try {
+    const modalTitleElement = await page?.getByTestId("modal-title");
+    if (modalTitleElement) {
+      modalCount = await modalTitleElement.count();
+    }
+  } catch (error) {
+    modalCount = 0;
+  }
+
+  while (modalCount === 0) {
+    await page.getByText("New Project", { exact: true }).click();
+    await page.waitForTimeout(5000);
+    modalCount = await page.getByTestId("modal-title")?.count();
+  }
+
+  await page.getByRole("heading", { name: "Sequential Tasks Agent" }).click();
+
+  await page.waitForSelector('[title="fit view"]', {
+    timeout: 100000,
+  });
+
+  await page.getByTitle("fit view").click();
+  await page.getByTitle("zoom out").click();
+  await page.getByTitle("zoom out").click();
+  await page.getByTitle("zoom out").click();
+
+  let outdatedComponents = await page.getByTestId("icon-AlertTriangle").count();
+
+  while (outdatedComponents > 0) {
+    await page.getByTestId("icon-AlertTriangle").first().click();
+    await page.waitForTimeout(1000);
+    outdatedComponents = await page.getByTestId("icon-AlertTriangle").count();
+  }
+
+  await page
+    .getByTestId("popover-anchor-input-api_key")
+    .first()
+    .fill(process.env.OPENAI_API_KEY ?? "");
+
+  await page.getByTestId("dropdown-model_name").click();
+  await page.getByTestId("gpt-4o-1-option").click();
+
+  await page.waitForTimeout(2000);
+
+  await page
+    .getByTestId("popover-anchor-input-api_key")
+    .last()
+    .fill(process.env.BRAVE_SEARCH_API_KEY ?? "");
+
+  await page.waitForTimeout(2000);
+
+  await page.getByTestId("button_run_chat output").click();
+  await page.waitForSelector("text=built successfully", { timeout: 60000 * 3 });
+
+  await page.getByText("built successfully").last().click({
+    timeout: 15000,
+  });
+
+  await page.getByText("Playground", { exact: true }).click();
+
+  await page.waitForTimeout(2000);
+
+  expect(
+    page
+      .getByPlaceholder("No chat input variables found. Click to run your flow")
+      .last(),
+  ).toBeVisible();
+
+  await page.getByText("Topic", { exact: true }).nth(1).isVisible();
+  await page.getByText("Topic", { exact: true }).nth(1).click();
+  expect(await page.getByPlaceholder("Enter text...").inputValue()).toBe(
+    "Agile",
+  );
+
+  const textContents = await page
+    .getByTestId("div-chat-message")
+    .allTextContents();
+
+  const concatAllText = textContents.join(" ");
+  expect(concatAllText.toLocaleLowerCase()).toContain("agile");
+  const allTextLength = concatAllText.length;
+  expect(allTextLength).toBeGreaterThan(500);
+});