From a78c7554162f0da2ca726ec6deca56dc6e8a810d Mon Sep 17 00:00:00 2001 From: Taylor Price Date: Wed, 19 Nov 2025 16:11:28 -0700 Subject: [PATCH 1/7] wip: run matrix on dispatch from PR Signed-off-by: Taylor Price --- .github/workflows/automation.yml | 2 - .github/workflows/pr-trigger.yml | 117 +++++++++++++++++++++++++++++++ 2 files changed, 117 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/pr-trigger.yml diff --git a/.github/workflows/automation.yml b/.github/workflows/automation.yml index f44a6cf..f6e09c8 100644 --- a/.github/workflows/automation.yml +++ b/.github/workflows/automation.yml @@ -2,8 +2,6 @@ name: Run WDIO Tests with OBOT Docker on: workflow_dispatch: - repository_dispatch: - types: [pr-created] jobs: wdio-tests: diff --git a/.github/workflows/pr-trigger.yml b/.github/workflows/pr-trigger.yml new file mode 100644 index 0000000..9c2f464 --- /dev/null +++ b/.github/workflows/pr-trigger.yml @@ -0,0 +1,117 @@ +name: Run WDIO Tests with OBOT Docker + +on: + repository_dispatch: + types: [pr-created] + +jobs: + prepare-matrix: + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + pr_number: ${{ steps.set-matrix.outputs.pr_number }} + steps: + - name: Set matrix data + id: set-matrix + run: | + if [ "${{ github.event_name }}" = "repository_dispatch" ]; then + echo "matrix=${{ toJson(github.event.client_payload.changed_files) }}" >> $GITHUB_OUTPUT + echo "pr_number=${{ github.event.client_payload.pr_number }}" >> $GITHUB_OUTPUT + else + # Default empty matrix for workflow_dispatch + echo 'matrix=[{"file":"default","containerizedConfig":null,"env":null}]' >> $GITHUB_OUTPUT + echo "pr_number=" >> $GITHUB_OUTPUT + fi + + wdio-tests: + needs: prepare-matrix + runs-on: ubuntu-latest + if: needs.prepare-matrix.outputs.matrix != '[]' + strategy: + fail-fast: false + matrix: + server: ${{ fromJson(needs.prepare-matrix.outputs.matrix) }} + + permissions: + id-token: write + actions: read + checks: read + + steps: + - name: Checkout Repository + uses: actions/checkout@v5 + + - name: Set up Node.js + uses: actions/setup-node@v6 + with: + node-version: 22 + + - name: Cache NPM Dependencies + uses: actions/cache@v4 + with: + path: ~/.npm + key: ${{ runner.os }}-node-${{ hashFiles('package-lock.json') }} + restore-keys: | + ${{ runner.os }}-node- + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Run OBOT container + run: | + docker run -d --name obot \ + -p 8080:8080 \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -e OPENAI_API_KEY=${{ secrets.OPENAI_API_KEY }} \ + -e OBOT_SERVER_DISABLE_UPDATE_CHECK=true \ + ghcr.io/obot-platform/obot:latest + + - name: Wait for OBOT to be ready + run: | + echo "Waiting for OBOT container..." + for i in {1..12}; do + if curl -sf http://localhost:8080/api/me | grep -q '"username":"nobody"'; then + echo "OBOT API is ready." + break + fi + echo "Not ready yet... retry #$i" + sleep 5 + done + + - name: Create catalog entry + run: | + curl localhost:8080/api/mcp-catalogs/default/entries -X POST -H "Content-Type: application/json" \ + -d '{ + "name": "test-${{ replace(matrix.server.file, '.yaml', '') }}", + "description": "testing ${{ replace(matrix.server.file, '.yaml', '') }} mcp", + "icon": "https://avatars.githubusercontent.com/u/9919?v=4", + "repoURL": "https://github.com/testing/testing", + "runtime": "containerized", + "containerizedConfig": { + "image": "${{ matrix.server.containerizedConfig.image }}", + "port": ${{ matrix.server.containerizedConfig.port }}, + "path": "${{ matrix.server.containerizedConfig.path }}", + "args": ${{ toJson(matrix.server.containerizedConfig.args) } + }, + "metadata": { + "categories": "testing" + } + }' + + - name: Install dependencies + run: npm ci + + - name: Run WDIO Tests + env: + WP_URL: ${{ secrets.WP_URL }} + WP_USERNAME: ${{ secrets.WP_USERNAME }} + WP_PASSWORD: ${{ secrets.WP_PASSWORD }} + OBOT_URL: ${{ secrets.OBOT_URL }} + GITLAB_TOKEN: ${{ secrets.GITLAB_TOKEN }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY}} + run: | + npx wdio run wdio.conf.ts --cucumberOpts.tagExpression='@${{ replace(matrix.server.file, '.yaml', '') }}' From 7d1bfda04ea0a7f0ccd61efcf0125d54b815dd5d Mon Sep 17 00:00:00 2001 From: Taylor Price Date: Wed, 19 Nov 2025 16:13:46 -0700 Subject: [PATCH 2/7] chore: trigger specific scenarios per mcp server Signed-off-by: Taylor Price --- README.md | 4 +- src/core/selectors.ts | 507 ++++++++++++++++++--------------- src/features/mcpServer.feature | 10 +- src/steps/mcpServer.step.ts | 145 ++++++---- 4 files changed, 372 insertions(+), 294 deletions(-) diff --git a/README.md b/README.md index ce4249b..fd04489 100644 --- a/README.md +++ b/README.md @@ -34,9 +34,9 @@ Once you've installed the dependencies and set up your environment, you can run 2. **Run a specific test**: If you want to run a specific scenario or feature, use: ```bash - npm run wdio:byScenario --spec src/features/obot.feature:10 + npx wdio run wdio.conf.ts --cucumberOpts.tagExpression='@gitlab' ``` - Replace `10` with the line number of the scenario you want to execute. + Replace `@gitlab` with the tag corresponding to the scenario you want to execute. 3. **Run in headless mode** (useful for CI/CD): You can run the tests in headless mode (without opening the browser window) by configuring the `wdio.conf.ts` file to enable headless execution, and then run the tests with: diff --git a/src/core/selectors.ts b/src/core/selectors.ts index f51a671..a0e9465 100644 --- a/src/core/selectors.ts +++ b/src/core/selectors.ts @@ -1,231 +1,280 @@ const Selectors = { - loginButton: '//button[text()="Login"]', - obotPopUp: '//dialog[contains(@class, "fixed top-1/2 left-1/2")]', - continueWithOkta: '//span[text()="Continue with Okta"]', - continueWithGoogle: '//span[text()="Continue with Google"]', - clickonAccount: '//div[text()="Use another account"]', - emailId: '//input[@type="email"]', - usernameField: '//input[@type="text"]', - passwordField: '//input[@type="password"]', - submitButton: '//input[@type="submit"]', - createnewobotButton: "//button[contains(@class, 'button') and contains(., 'Create New Agent')]", - editpencilIcon: '//*[contains(@class,"pencil h-5")]/ancestor::button[contains(@class,"group text")]', - messageContent: '//div[@class="flex w-full items-center gap-4 p-2"]//textarea[@placeholder="Your message..."]', - updateUsername: '//input[@class="bg-surface grow rounded-lg p-2"]', - exitEditor: '//button[@class="group relative mr-1 flex items-center rounded-full border p-2 text-xs transition-[background-color] duration-200 border-blue bg-blue text-white md:px-4"]', - fileIcon:'//label[text()="Your messages"]/following-sibling::div//button', - obotUpdatedname: '//h4[@class="mb-1!"]', - deleteButton: '//button[@class="button-destructive"]', - deleteTool:'(//button[@class="icon-button-small"])[1]', - fileuploadPopup:"//div[contains(@class, 'default-dialog') and @style='left: 166px; top: 12.833px;']", - fileuploadButton:"(//label[contains(@class, 'button') and contains(@class, 'flex') and contains(., 'Upload')])[2]", - filenameVerify:'//span[text()="AI_test.txt"]', - fileUpload:'//label[contains(normalize-space(), "Upload")]//input[@type="file" and @accept=".pdf, .txt, .doc, .docx, .ppt, .pptx, .md, .rtf, .html, .odt, .ipynb, .json,.csv, .png, .jpg, .jpeg, .webp"]', - confirmDeleteObotButton: '//h3[text()="Delete the current Obot?"]/following-sibling::button[contains(text(), "Yes") and contains(@class, "bg-red-600")]', - clickChatbot:'//button[@class="icon-button group relative flex items-center gap-2 p-0 shadow-md"]', - clickOnDiv:'//div[@id="click-catch"]', - clickshowOutput:'//button[text()="Show Output"]', - chatOutputElement:'//p[text()="Output"]/following-sibling::pre[contains(@class,"text")]', - task: { - expandTaskMenu: '//div[@class="flex items-center justify-between"]//button[@class="icon-button"][1]', - createNewTaskButton: '//button[contains(., "New Task")]', - strongTask: '//strong[text()="TASK"]', - newTaskTitle: '//input[@class="ghost-input text-2xl font-semibold"]', - deleteTask: '//strong[text()="TASK"]/../..//button[@class="button-destructive p-4"]', - confirmDeleteTask: '//h3[contains(text(),"delete this task")]/following-sibling::button[contains(text(), "Yes") and contains(@class, "bg-red-600")]', - stepsInput: '//textarea[@class="ghost-input border-surface2 ml-1 grow resize-none scrollbar-none"]', - runButton: "(//button[contains(@class, 'bg-blue') and contains(@class, 'text-white')])", - advancedOptionBtn:'//button[text()="Show Advanced Options..."]', - argumentBtn: '(//button[contains(text(),"Argument")])', - inputArgument: '//input[@placeholder="Enter Name"]', - argumentPopup: '//h4[text()="Arguments"]/following-sibling::div//input[@id="param-tax calculator"]', - argumentValue:'//input[@id="param-Enter your income"]', - submitArgs: "(//button[text()='Run' and contains(@class, 'button-primary')])[2]", - inputResult: "(//div[contains(@class, 'relative') and contains(@class, 'my-3')])[3]", - inputTextHeader: '//p[contains(text(),"Here is the tax on your income")]', - inputText:'//span[@class="text-sm font-semibold" and contains(text(),"Automation")]', - searchBot: '//textarea[@id="chat"]', - submitButton:'//button[@type="submit" and contains(@class, "button-colors") and contains(@class, "text-blue")]', - triggerTypeBtn:'(//button[normalize-space(text())="on demand"])[1]', - selectTriggerType: (option: string) => `//button[normalize-space(text())='${option}']`, - webHookURL:'//div[contains(@class, "bg-surface2") and contains(., "https://")]', - runTask:'//div[contains(@class, "flex") and contains(@class, "truncate")]', - taskResult:'//ul[@class="ml-4 flex flex-col text-xs"]//li[1]//div', - tableResult:'//span[text()="Tables "]/../following-sibling::div/div/div/ul/li/button', - tableRefresh:'//button[text()=" Refresh"]', - TaskMenu:'//button[.//span[contains(text(), "Tasks")]]', - sideBar:'(//button[@class="icon-button"])[1]', - timeTypeBtn: (option: string) => `(//button[normalize-space(text())='${option}'])[1]`, - clickTask:'//span[text()="Tasks "]', - createNewTask:'//button[text()=" New Task"]', - loopStep:'//button[@data-testid="step-loop-btn"]', - btnrun:'//button[text()="Run "]', - }, - table:{ - id:'//tbody//tr//td[1]', - name:'//tbody//tr//td[2]', - email:'//tbody//tr//td[3]', - tableTask:'(//div/child::textarea)[2]', - submitInput:'(//button[@type="submit" and contains(@class,"button-colors")])[2]', - }, - toolIcon: '//button[@class="button-icon-primary"]', - addTool: '//button[contains(text(),"Tools")]', - searchTools: '//input[contains(@placeholder,"Search tools")]', - selectToolQuery: `(//span[normalize-space(text())="{{toolName}}"])[2]`, - applyTool: '//button[text()="Apply"]', - uploadButton: '//label[text()=" Upload "]', - expandknowledgeFileicon: '//span[text()="Knowledge "]', - expandFileicon:'//span[text()="Files"]/parent::button[@class="flex items-center gap-2 px-5 py-2"]', - scrollableDiv: '//div[@class="default-scrollbar-thin flex grow flex-col"]', - verifytheKnowlegefileName: '//span[@class="ms-3 truncate"]', - verifythefileName:'//span[@class="truncate"]', - knowledgeFileUpload: '//label[contains(normalize-space(), "Upload")]//input[@type="file" and @accept=".pdf, .txt, .doc, .docx"]', - showDetails: '//button[text()="Show Details"]', - chatbotTextArea: '//div/textarea[@placeholder="Your message..."]', - submitChat: '//button[@type="submit" and contains(@class,"button-colors")]', - showAdvanceOptions: '//span[text()="Show Advanced Options..."]', - advancedOptionTabs: { - interfaceTab: '//span[text()="Interface"]', - tasksTab: '//span[text()="Tasks"]', - }, - chatIntroductionTextArea: '//div/textarea[@placeholder="Introduction"]', - starterMsgButton: '//span[text()="Starter Message"]/ancestor::button[contains(@class,"button")]', - starterMsgInput: '//textarea[@id="project-instructions"]', - chatIntroductionInObot: '//div[contains(@class,"message-content")]', - chatStarterMsgInObot: '//button/span[@class="line-clamp-3"]', - agentCatalog:'//a[text()="Agent Catalog"]', - createNewAgent:'//button[contains(@class, "bg-surface1")]', - agentEditHover:'//div[@class="flex flex-col items-center justify-center text-center"]', - agentEditPencilIcon:'(//button[contains(@class, "rounded-full")])[1]', - updateAgentName:'//input[contains(@class, "ghost-input")]', - confirmDelete:'(//button[normalize-space(text())="Yes, I\'m sure"])[3]', - tableInput:'//pre[contains(@class, "whitespace-pre-wrap")]//span[contains(@class, "text-gray-500")]', - showDetails2: '(//button[text()="Show Details"])[2]', - showDetails3: '(//button[text()="Show Details"])[3]', + loginButton: '//button[text()="Login"]', + obotPopUp: '//dialog[contains(@class, "fixed top-1/2 left-1/2")]', + continueWithOkta: '//span[text()="Continue with Okta"]', + continueWithGoogle: '//span[text()="Continue with Google"]', + clickonAccount: '//div[text()="Use another account"]', + emailId: '//input[@type="email"]', + usernameField: '//input[@type="text"]', + passwordField: '//input[@type="password"]', + submitButton: '//input[@type="submit"]', + createnewobotButton: + "//button[contains(@class, 'button') and contains(., 'Create New Agent')]", + editpencilIcon: + '//*[contains(@class,"pencil h-5")]/ancestor::button[contains(@class,"group text")]', + messageContent: + '//div[@class="flex w-full items-center gap-4 p-2"]//textarea[@placeholder="Your message..."]', + updateUsername: '//input[@class="bg-surface grow rounded-lg p-2"]', + exitEditor: + '//button[@class="group relative mr-1 flex items-center rounded-full border p-2 text-xs transition-[background-color] duration-200 border-blue bg-blue text-white md:px-4"]', + fileIcon: '//label[text()="Your messages"]/following-sibling::div//button', + obotUpdatedname: '//h4[@class="mb-1!"]', + deleteButton: '//button[@class="button-destructive"]', + deleteTool: '(//button[@class="icon-button-small"])[1]', + fileuploadPopup: + "//div[contains(@class, 'default-dialog') and @style='left: 166px; top: 12.833px;']", + fileuploadButton: + "(//label[contains(@class, 'button') and contains(@class, 'flex') and contains(., 'Upload')])[2]", + filenameVerify: '//span[text()="AI_test.txt"]', + fileUpload: + '//label[contains(normalize-space(), "Upload")]//input[@type="file" and @accept=".pdf, .txt, .doc, .docx, .ppt, .pptx, .md, .rtf, .html, .odt, .ipynb, .json,.csv, .png, .jpg, .jpeg, .webp"]', + confirmDeleteObotButton: + '//h3[text()="Delete the current Obot?"]/following-sibling::button[contains(text(), "Yes") and contains(@class, "bg-red-600")]', + clickChatbot: + '//button[@class="icon-button group relative flex items-center gap-2 p-0 shadow-md"]', + clickOnDiv: '//div[@id="click-catch"]', + clickshowOutput: '//button[text()="Show Output"]', + chatOutputElement: + '//p[text()="Output"]/following-sibling::pre[contains(@class,"text")]', + task: { + expandTaskMenu: + '//div[@class="flex items-center justify-between"]//button[@class="icon-button"][1]', + createNewTaskButton: '//button[contains(., "New Task")]', + strongTask: '//strong[text()="TASK"]', + newTaskTitle: '//input[@class="ghost-input text-2xl font-semibold"]', + deleteTask: + '//strong[text()="TASK"]/../..//button[@class="button-destructive p-4"]', + confirmDeleteTask: + '//h3[contains(text(),"delete this task")]/following-sibling::button[contains(text(), "Yes") and contains(@class, "bg-red-600")]', + stepsInput: + '//textarea[@class="ghost-input border-surface2 ml-1 grow resize-none scrollbar-none"]', + runButton: + "(//button[contains(@class, 'bg-blue') and contains(@class, 'text-white')])", + advancedOptionBtn: '//button[text()="Show Advanced Options..."]', + argumentBtn: '(//button[contains(text(),"Argument")])', + inputArgument: '//input[@placeholder="Enter Name"]', + argumentPopup: + '//h4[text()="Arguments"]/following-sibling::div//input[@id="param-tax calculator"]', + argumentValue: '//input[@id="param-Enter your income"]', + submitArgs: + "(//button[text()='Run' and contains(@class, 'button-primary')])[2]", + inputResult: + "(//div[contains(@class, 'relative') and contains(@class, 'my-3')])[3]", + inputTextHeader: '//p[contains(text(),"Here is the tax on your income")]', + inputText: + '//span[@class="text-sm font-semibold" and contains(text(),"Automation")]', + searchBot: '//textarea[@id="chat"]', + submitButton: + '//button[@type="submit" and contains(@class, "button-colors") and contains(@class, "text-blue")]', + triggerTypeBtn: '(//button[normalize-space(text())="on demand"])[1]', + selectTriggerType: (option: string) => + `//button[normalize-space(text())='${option}']`, + webHookURL: + '//div[contains(@class, "bg-surface2") and contains(., "https://")]', + runTask: '//div[contains(@class, "flex") and contains(@class, "truncate")]', + taskResult: '//ul[@class="ml-4 flex flex-col text-xs"]//li[1]//div', + tableResult: + '//span[text()="Tables "]/../following-sibling::div/div/div/ul/li/button', + tableRefresh: '//button[text()=" Refresh"]', + TaskMenu: '//button[.//span[contains(text(), "Tasks")]]', + sideBar: '(//button[@class="icon-button"])[1]', + timeTypeBtn: (option: string) => + `(//button[normalize-space(text())='${option}'])[1]`, + clickTask: '//span[text()="Tasks "]', + createNewTask: '//button[text()=" New Task"]', + loopStep: '//button[@data-testid="step-loop-btn"]', + btnrun: '//button[text()="Run "]', + }, + table: { + id: "//tbody//tr//td[1]", + name: "//tbody//tr//td[2]", + email: "//tbody//tr//td[3]", + tableTask: "(//div/child::textarea)[2]", + submitInput: + '(//button[@type="submit" and contains(@class,"button-colors")])[2]', + }, + toolIcon: '//button[@class="button-icon-primary"]', + addTool: '//button[contains(text(),"Tools")]', + searchTools: '//input[contains(@placeholder,"Search tools")]', + selectToolQuery: `(//span[normalize-space(text())="{{toolName}}"])[2]`, + applyTool: '//button[text()="Apply"]', + uploadButton: '//label[text()=" Upload "]', + expandknowledgeFileicon: '//span[text()="Knowledge "]', + expandFileicon: + '//span[text()="Files"]/parent::button[@class="flex items-center gap-2 px-5 py-2"]', + scrollableDiv: '//div[@class="default-scrollbar-thin flex grow flex-col"]', + verifytheKnowlegefileName: '//span[@class="ms-3 truncate"]', + verifythefileName: '//span[@class="truncate"]', + knowledgeFileUpload: + '//label[contains(normalize-space(), "Upload")]//input[@type="file" and @accept=".pdf, .txt, .doc, .docx"]', + showDetails: '//button[text()="Show Details"]', + chatbotTextArea: '//div/textarea[@placeholder="Your message..."]', + submitChat: '//button[@type="submit" and contains(@class,"button-colors")]', + showAdvanceOptions: '//span[text()="Show Advanced Options..."]', + advancedOptionTabs: { + interfaceTab: '//span[text()="Interface"]', + tasksTab: '//span[text()="Tasks"]', + }, + chatIntroductionTextArea: '//div/textarea[@placeholder="Introduction"]', + starterMsgButton: + '//span[text()="Starter Message"]/ancestor::button[contains(@class,"button")]', + starterMsgInput: '//textarea[@id="project-instructions"]', + chatIntroductionInObot: '//div[contains(@class,"message-content")]', + chatStarterMsgInObot: '//button/span[@class="line-clamp-3"]', + agentCatalog: '//a[text()="Agent Catalog"]', + createNewAgent: '//button[contains(@class, "bg-surface1")]', + agentEditHover: + '//div[@class="flex flex-col items-center justify-center text-center"]', + agentEditPencilIcon: '(//button[contains(@class, "rounded-full")])[1]', + updateAgentName: '//input[contains(@class, "ghost-input")]', + confirmDelete: '(//button[normalize-space(text())="Yes, I\'m sure"])[3]', + tableInput: + '//pre[contains(@class, "whitespace-pre-wrap")]//span[contains(@class, "text-gray-500")]', + showDetails2: '(//button[text()="Show Details"])[2]', + showDetails3: '(//button[text()="Show Details"])[3]', - admin:{ - oktaLogin:'//button[normalize-space(.//div) = "Sign in with Okta"]', - crateNewAgent:'//button[normalize-space(.//div) = "New Agent"]', - agentName:'(//div/child::input)[1]', - agentDes:'(//div/child::input)[2]', - tryItOutBtn:'//div/child::a', - editAgent:'(//a[contains(@href, "/admin/agents/")])[1]', - deleteAgent:'(//a[contains(@href, "/admin/agents/")]/following-sibling::button)[1]', - confirmDelete:'//div[text()="Confirm"]', - agenetDeletedMessagge:'//div[@data-content]//div[contains(text(), "Agent deleted")]', - memoryDropdown:'//p[text()="Memory"]/ancestor::div[contains(@class, "flex")]/following-sibling::div//button', - setAlwaysOn:'//div[contains(@role, "option")]/span[contains(text(),"Always On")]', - addTool:'//div[text()=" Add Tools"]', - addKnowledge:'//div[text()="Add Knowledge"]', - localFiles:'//div[text()="Local Files"]', - searchTool:'//input[@placeholder="Search tools..."]', - googleSearch:'//span[text()="Google Search "]', - weather:'//span[text()="Weather "]', - viewAllMemories:'//button[text()="View All Memories"]', - advanced:'//h4[text()="Advanced"]', - labelClick: `(//label[normalize-space(text())="{{labelClick}}"])`, - userleftsideMenu:'//span[text()="Users"]', - userFilter:'//span[text()="User"]', - searchText:'//input[@placeholder="Filter..."]', - clickonthreedotBtn:'//div[@class="flex justify-end"]//button[@type="button"]', - updateroleBtn:'//div[text()="Update Role"]', - clickonroleDrp:'//button[@aria-controls="radix-:rceg:"]', - selectRole:'//span[text()="{{role}}"]', - updateBtn:'//div[text()="Update"]', - deleteroleBtn:'//div[text()="Delete User"]', - deleteBtn:'//div[text()="Delete"]', - chatthreadMenu:'//a//span[text()="Chat Threads"]', - }, - agentDes:'//p[@class="text-gray w-sm font-light md:w-md"]', - authLink:'//a[contains(@class, "bg-blue") and contains(@class, "rounded-3xl") and contains(@class, "text-white")]', - scheduleDay:'(//button[normalize-space(text())="daily"])[1]', - scheduleTime:'(//button[normalize-space(text())="on the hour"])[1]', - homeButton:'//a[@id="navbar-home-link"]', - continueBtn:'//button[contains(text(), "I dunno")]/following-sibling::button', - lunchAgent:'//button[normalize-space(text())="Launch Agent"]', - obotEditArea:'//button[@id="edit-basic-details-button"]', - editAreaPencil: '(//button[@id="edit-basic-details-button"]/child::div)[2]', - editAgentName:'//input[@id="project-name"]', - obotNav:'(//div/child::span/following-sibling::button)[1]', - closeEditArea:'//button[@class="icon-button absolute top-2 right-2"]', - addKnowledge:'//p[text()="Knowledge"]/following-sibling::div', - clicksNewThread:'//a[.//img[@alt="Obot icon"]]/following-sibling::button', - memoryValidate:'//tr//td[2]', - refreashMemory:'//span[text()="Most recent memories"]/../div', - memoryIcon:'//button[@data-memories-btn and contains(@class, "icon-button")]', - closeMemory:'(//button[contains(@class, "absolute top-0 right-0 p-3")])[1]', - deleteAllMemory:'(//div//button[contains(text()," Delete All")])[2]', - deleteMemoryValidate:'//p[text()="No memories stored"]', - startFromScratch:'//button[text()=" Create From Scratch"]', - configuration:'//span[text()="Configuration "]', - editObotName:'//span[text()="Name & Description "]', - mcpServer:'//span[text()="MCP Servers "]', - addMCPServer:'//button[text()=" Add MCP Server"]', - searchMCPTool: '//input[@placeholder="Search MCP Servers..."]', - selectedTool: `(//h4[normalize-space(text())="{{mcpTool}}"])`, - mcpStep: `(//button[(text())="{{mcpStep}}"])`, - selectServer:'(//button[text()="Select Server "])', - addServer:'//button[text()="Add server"]', - validateMCPTool: (option: string) => `//p[normalize-space(text())="${option}"]`, - knowledge:'//span[text()="Knowledge "]', - selectFromSidebar:`(//span[(text())="{{select}}"])`, - enableChatBot:'//h5[text()="Enable"]/../label', - copyTemplates:'//div/a[@class="overflow-hidden text-sm text-ellipsis hover:underline"]', - copyChatBot:'//div/a[@class="overflow-hidden text-ellipsis hover:underline"]', - validateThread:'//button[text()="New Thread"]', - // btnIUnderstand:'//button[text()="I Understand"]', - btnClick: `(//button[normalize-space(text())="{{btnclick}}"])`, - templatePencil:'(//button[contains(@class, "text-gray-500 hover:text-gray-700 dark:text-gray-400 dark:hover:text-gray-200")])[2]', - publicTemplate:'(//span[@class="slider rounded-2xl"])[3]', - templateName:'//div//div//h3[@class="text-base font-medium"]', - modelApi:'//input[@id="OBOT_OPENAI_MODEL_PROVIDER_API_KEY"]', - modelSelect: `(//label[(text())="{{modelSelect}}"])`, - ValidateModel:'//button[@title="Select model for this thread"]', - gpt3btn:'//div[text()="gpt-3.5-turbo"]', - gpt4btn:'//div[text()="gpt-4"]', - message:'//label[text()="Your messages"]', - databaseOn:'//span[text()="Toggle Capability"]/../span[2]', - profileIcon:'//a[@id="navbar-home-link"]/following-sibling::button', - logOut:'//a[text()=" Log out"]', - cancelTool:'//span[text()=" AWS S3"]/following-sibling::button', - clickherelink:'//span[text()="click here"]', - signinHubspot:"//i18n-string[normalize-space()='Sign in to your HubSpot account']", - loginHubspotButton:'//i18n-string[text()="Log in"]', - emailHubspot:'//input[@type="email"]', - nextButton:'//button[@type="submit"]//i18n-string', - passwordHubspot:'//input[@type="password"]', - authentication: '//p[text()="Authentication is required."]', - // fillEmail: '//input[@placeholder="Email, phone, or Skype"]', - fillEmail: '//input[@autocomplete="username"]', - clickNext:'//input[@value="Next"]', - getCode:'//button[text()="Send code"]', - - MCP:{ - googlebtn: '//span[text()="Continue with Google"]', - emailInput: '//input[@type="email"]', - nextbtn: '//span[text()="Next"]', - passwordInput: '//input[@type="password"]', - cntBtn: '//span[text()="Continue"]', - navigationbtn: '//button[@class="icon-button z-20"]', - clickChatObot: '//button[normalize-space(text())="Chat"]', - connectorbtn: '//p[normalize-space(text())="Connectors"]/following-sibling::button', - mcpSearchInput: '//input[normalize-space(@placeholder)="Search by name..."]', - // selectMCPServer: '//p[normalize-space(text())="WordPress1"]', - selectMCPServer: (option: string) => `//p[normalize-space(text())="${option}"]/ancestor::div[contains(@class, 'flex')]/button`, - wpSiteURL: '//input[normalize-space(@id)="WORDPRESS_SITE"]', - wpUsername: '//input[normalize-space(@id)="WORDPRESS_USERNAME"]', - wpPassword: '//input[normalize-space(@id)="WordPress App Password"]', - btnClick: (option: string) => `//button[normalize-space(text())="${option}"]`, - promptInput: '//div[@class="plaintext-editor text-md relative w-full flex-1 grow resize-none p-2 leading-8 outline-none"]', - // submitPrompt: '//button[@type="submit"]', - // obotInput: '//div[@class="ProseMirror editor"]', - gitlabToken: '//input[@name="GitLab Personal Access Token"]', - // messageContainer: "//div[contains(@class, 'flex-1') and contains(@class, 'flex-col') and contains(@class, 'justify-start') and contains(@class, 'gap-8')]", - obotInput: "//div[contains(@class,'ProseMirror') and @contenteditable='true']", - submitPrompt: '//button[@type="submit"]', - lastBotReply: '//div[@class="message-content"]', - messageContainer: "//div[contains(@class, 'flex-1') and contains(@class, 'flex-col') and contains(@class, 'justify-start') and contains(@class, 'gap-8')]" - - } -} -export default Selectors; \ No newline at end of file + admin: { + oktaLogin: '//button[normalize-space(.//div) = "Sign in with Okta"]', + crateNewAgent: '//button[normalize-space(.//div) = "New Agent"]', + agentName: "(//div/child::input)[1]", + agentDes: "(//div/child::input)[2]", + tryItOutBtn: "//div/child::a", + editAgent: '(//a[contains(@href, "/admin/agents/")])[1]', + deleteAgent: + '(//a[contains(@href, "/admin/agents/")]/following-sibling::button)[1]', + confirmDelete: '//div[text()="Confirm"]', + agenetDeletedMessagge: + '//div[@data-content]//div[contains(text(), "Agent deleted")]', + memoryDropdown: + '//p[text()="Memory"]/ancestor::div[contains(@class, "flex")]/following-sibling::div//button', + setAlwaysOn: + '//div[contains(@role, "option")]/span[contains(text(),"Always On")]', + addTool: '//div[text()=" Add Tools"]', + addKnowledge: '//div[text()="Add Knowledge"]', + localFiles: '//div[text()="Local Files"]', + searchTool: '//input[@placeholder="Search tools..."]', + googleSearch: '//span[text()="Google Search "]', + weather: '//span[text()="Weather "]', + viewAllMemories: '//button[text()="View All Memories"]', + advanced: '//h4[text()="Advanced"]', + labelClick: `(//label[normalize-space(text())="{{labelClick}}"])`, + userleftsideMenu: '//span[text()="Users"]', + userFilter: '//span[text()="User"]', + searchText: '//input[@placeholder="Filter..."]', + clickonthreedotBtn: + '//div[@class="flex justify-end"]//button[@type="button"]', + updateroleBtn: '//div[text()="Update Role"]', + clickonroleDrp: '//button[@aria-controls="radix-:rceg:"]', + selectRole: '//span[text()="{{role}}"]', + updateBtn: '//div[text()="Update"]', + deleteroleBtn: '//div[text()="Delete User"]', + deleteBtn: '//div[text()="Delete"]', + chatthreadMenu: '//a//span[text()="Chat Threads"]', + }, + agentDes: '//p[@class="text-gray w-sm font-light md:w-md"]', + authLink: + '//a[contains(@class, "bg-blue") and contains(@class, "rounded-3xl") and contains(@class, "text-white")]', + scheduleDay: '(//button[normalize-space(text())="daily"])[1]', + scheduleTime: '(//button[normalize-space(text())="on the hour"])[1]', + homeButton: '//a[@id="navbar-home-link"]', + continueBtn: + '//button[contains(text(), "I dunno")]/following-sibling::button', + lunchAgent: '//button[normalize-space(text())="Launch Agent"]', + obotEditArea: '//button[@id="edit-basic-details-button"]', + editAreaPencil: '(//button[@id="edit-basic-details-button"]/child::div)[2]', + editAgentName: '//input[@id="project-name"]', + obotNav: "(//div/child::span/following-sibling::button)[1]", + closeEditArea: '//button[@class="icon-button absolute top-2 right-2"]', + addKnowledge: '//p[text()="Knowledge"]/following-sibling::div', + clicksNewThread: '//a[.//img[@alt="Obot icon"]]/following-sibling::button', + memoryValidate: "//tr//td[2]", + refreashMemory: '//span[text()="Most recent memories"]/../div', + memoryIcon: + '//button[@data-memories-btn and contains(@class, "icon-button")]', + closeMemory: '(//button[contains(@class, "absolute top-0 right-0 p-3")])[1]', + deleteAllMemory: '(//div//button[contains(text()," Delete All")])[2]', + deleteMemoryValidate: '//p[text()="No memories stored"]', + startFromScratch: '//button[text()=" Create From Scratch"]', + configuration: '//span[text()="Configuration "]', + editObotName: '//span[text()="Name & Description "]', + mcpServer: '//span[text()="MCP Servers "]', + addMCPServer: '//button[text()=" Add MCP Server"]', + searchMCPTool: '//input[@placeholder="Search MCP Servers..."]', + selectedTool: `(//h4[normalize-space(text())="{{mcpTool}}"])`, + mcpStep: `(//button[(text())="{{mcpStep}}"])`, + selectServer: '(//button[text()="Select Server "])', + addServer: '//button[text()="Add server"]', + validateMCPTool: (option: string) => + `//p[normalize-space(text())="${option}"]`, + knowledge: '//span[text()="Knowledge "]', + selectFromSidebar: `(//span[(text())="{{select}}"])`, + enableChatBot: '//h5[text()="Enable"]/../label', + copyTemplates: + '//div/a[@class="overflow-hidden text-sm text-ellipsis hover:underline"]', + copyChatBot: + '//div/a[@class="overflow-hidden text-ellipsis hover:underline"]', + validateThread: '//button[text()="New Thread"]', + // btnIUnderstand:'//button[text()="I Understand"]', + btnClick: `(//button[normalize-space(text())="{{btnclick}}"])`, + templatePencil: + '(//button[contains(@class, "text-gray-500 hover:text-gray-700 dark:text-gray-400 dark:hover:text-gray-200")])[2]', + publicTemplate: '(//span[@class="slider rounded-2xl"])[3]', + templateName: '//div//div//h3[@class="text-base font-medium"]', + modelApi: '//input[@id="OBOT_OPENAI_MODEL_PROVIDER_API_KEY"]', + modelSelect: `(//label[(text())="{{modelSelect}}"])`, + ValidateModel: '//button[@title="Select model for this thread"]', + gpt3btn: '//div[text()="gpt-3.5-turbo"]', + gpt4btn: '//div[text()="gpt-4"]', + message: '//label[text()="Your messages"]', + databaseOn: '//span[text()="Toggle Capability"]/../span[2]', + profileIcon: '//a[@id="navbar-home-link"]/following-sibling::button', + logOut: '//a[text()=" Log out"]', + cancelTool: '//span[text()=" AWS S3"]/following-sibling::button', + clickherelink: '//span[text()="click here"]', + signinHubspot: + "//i18n-string[normalize-space()='Sign in to your HubSpot account']", + loginHubspotButton: '//i18n-string[text()="Log in"]', + emailHubspot: '//input[@type="email"]', + nextButton: '//button[@type="submit"]//i18n-string', + passwordHubspot: '//input[@type="password"]', + authentication: '//p[text()="Authentication is required."]', + // fillEmail: '//input[@placeholder="Email, phone, or Skype"]', + fillEmail: '//input[@autocomplete="username"]', + clickNext: '//input[@value="Next"]', + getCode: '//button[text()="Send code"]', + + MCP: { + googlebtn: '//span[text()="Continue with Google"]', + emailInput: '//input[@type="email"]', + nextbtn: '//span[text()="Next"]', + passwordInput: '//input[@type="password"]', + cntBtn: '//span[text()="Continue"]', + navigationbtn: '//button[@class="icon-button z-20"]', + clickChatObot: '//button[normalize-space(text())="Chat"]', + connectorbtn: + '//p[normalize-space(text())="Connectors"]/following-sibling::button', + mcpSearchInput: + '//input[normalize-space(@placeholder)="Search by name..."]', + // selectMCPServer: '//p[normalize-space(text())="WordPress"]', + selectMCPServer: (option: string) => + `//p[normalize-space(text())="${option}"]/ancestor::div[contains(@class, 'flex')]/button`, + wpSiteURL: '//input[normalize-space(@id)="WORDPRESS_SITE"]', + wpUsername: '//input[normalize-space(@id)="WORDPRESS_USERNAME"]', + wpPassword: '//input[normalize-space(@id)="WordPress App Password"]', + btnClick: (option: string) => + `//button[normalize-space(text())="${option}"]`, + promptInput: + '//div[@class="plaintext-editor text-md relative w-full flex-1 grow resize-none p-2 leading-8 outline-none"]', + // submitPrompt: '//button[@type="submit"]', + // obotInput: '//div[@class="ProseMirror editor"]', + gitlabToken: '//input[@name="GitLab Personal Access Token"]', + // messageContainer: "//div[contains(@class, 'flex-1') and contains(@class, 'flex-col') and contains(@class, 'justify-start') and contains(@class, 'gap-8')]", + obotInput: + "//div[contains(@class,'ProseMirror') and @contenteditable='true']", + submitPrompt: '//button[@type="submit"]', + lastBotReply: '//div[@class="message-content"]', + messageContainer: + "//div[contains(@class, 'flex-1') and contains(@class, 'flex-col') and contains(@class, 'justify-start') and contains(@class, 'gap-8')]", + }, +}; +export default Selectors; diff --git a/src/features/mcpServer.feature b/src/features/mcpServer.feature index 20459b5..de75661 100644 --- a/src/features/mcpServer.feature +++ b/src/features/mcpServer.feature @@ -1,22 +1,24 @@ -Feature: Connecte MCP server on Obot +Feature: Connect MCP server on Obot Background: Navigate to Obot Given I setup context for assertion When User navigates the Obot main login page Then User open chat Obot + @wordpress Scenario: Validate Wordpress sequential prompts on Obot When User open MCP connector page And User select "WordPress" MCP server And User select "Connect To Server" button - And User connect to the WordPress1 MCP server + And User connect to the WordPress MCP server When User sends prompts to Obot AI chat for "Wordpress" MCP server Then All prompts results should be validated and report generated for selected "Wordpress" MCP Server - Scenario: Validate GitLab sequential prompts on Obot + @gitlab + Scenario: Validate GitLab sequential prompts on Obot When User open MCP connector page And User select "GitLab" MCP server And User select "Connect To Server" button And User connect to the GitLab MCP server When User sends prompts to Obot AI chat for "Gitlab" MCP server - Then All prompts results should be validated and report generated for selected "Gitlab" MCP Server \ No newline at end of file + Then All prompts results should be validated and report generated for selected "Gitlab" MCP Server diff --git a/src/steps/mcpServer.step.ts b/src/steps/mcpServer.step.ts index 86e5bd0..438dda6 100644 --- a/src/steps/mcpServer.step.ts +++ b/src/steps/mcpServer.step.ts @@ -1,91 +1,118 @@ import { When, Then, Given } from "@wdio/cucumber-framework"; import Selectors from "../core/selectors"; -import { clickToElement,isElementDisplayed,slowInputFilling} from "../core/func"; +import { + clickToElement, + isElementDisplayed, + slowInputFilling, +} from "../core/func"; import { LONG_PAUSE, SHORT_PAUSE } from "../core/timeouts"; -import { aggregateToolResponses, saveMCPReport, sendPromptValidateAndCollect } from "../core/mcpFunc"; -import path from 'path'; -import { promises as fs } from 'fs'; +import { + aggregateToolResponses, + saveMCPReport, + sendPromptValidateAndCollect, +} from "../core/mcpFunc"; +import path from "path"; +import { promises as fs } from "fs"; -Given(/^User navigates the Obot main login page$/, async() => { - const url = process.env.OBOT_URL ; - await browser.url(url); +Given(/^User navigates the Obot main login page$/, async () => { + const url = process.env.OBOT_URL; + await browser.url(url); }); Then(/^User open chat Obot$/, async () => { - await clickToElement(Selectors.MCP.navigationbtn); - await clickToElement(Selectors.MCP.clickChatObot); + await clickToElement(Selectors.MCP.navigationbtn); + await clickToElement(Selectors.MCP.clickChatObot); }); When(/^User open MCP connector page$/, async () => { - await clickToElement(Selectors.MCP.connectorbtn); + await clickToElement(Selectors.MCP.connectorbtn); }); Then(/^User select "([^"]*)" MCP server$/, async (MCPServer) => { - await slowInputFilling(Selectors.MCP.mcpSearchInput, MCPServer); - await isElementDisplayed(Selectors.MCP.selectMCPServer(MCPServer), LONG_PAUSE); - // Wait until matching elements appear - const allServers = await $$(Selectors.MCP.selectMCPServer(MCPServer)); - if (await allServers.length === 0) throw new Error(`No MCP server found matching: ${MCPServer}`); + await slowInputFilling(Selectors.MCP.mcpSearchInput, MCPServer); + await isElementDisplayed( + Selectors.MCP.selectMCPServer(MCPServer), + LONG_PAUSE, + ); + // Wait until matching elements appear + const allServers = await $$(Selectors.MCP.selectMCPServer(MCPServer)); + if ((await allServers.length) === 0) + throw new Error(`No MCP server found matching: ${MCPServer}`); - // Click the last one - const lastServer = allServers[await allServers.length - 1]; - await lastServer.waitForDisplayed({ timeout: LONG_PAUSE }); - await lastServer.click(); + // Click the last one + const lastServer = allServers[(await allServers.length) - 1]; + await lastServer.waitForDisplayed({ timeout: LONG_PAUSE }); + await lastServer.click(); - await browser.pause(SHORT_PAUSE); + await browser.pause(SHORT_PAUSE); }); Then(/^User select "([^"]*)" button$/, async (Button) => { - await isElementDisplayed(Selectors.MCP.btnClick(Button),SHORT_PAUSE); - await clickToElement(Selectors.MCP.btnClick(Button)); + await isElementDisplayed(Selectors.MCP.btnClick(Button), SHORT_PAUSE); + await clickToElement(Selectors.MCP.btnClick(Button)); }); -Then(/^User connect to the WordPress1 MCP server$/, async () => { - await slowInputFilling(Selectors.MCP.wpSiteURL,process.env.WP_URL); - await slowInputFilling(Selectors.MCP.wpUsername,process.env.WP_USERNAME); - await slowInputFilling(Selectors.MCP.wpPassword, process.env.WP_PASSWORD); - await clickToElement(Selectors.MCP.btnClick("Launch")); - await browser.pause(LONG_PAUSE*2); +Then(/^User connect to the WordPress MCP server$/, async () => { + await slowInputFilling(Selectors.MCP.wpSiteURL, process.env.WP_URL); + await slowInputFilling(Selectors.MCP.wpUsername, process.env.WP_USERNAME); + await slowInputFilling(Selectors.MCP.wpPassword, process.env.WP_PASSWORD); + await clickToElement(Selectors.MCP.btnClick("Launch")); + await browser.pause(LONG_PAUSE * 2); }); - + Then(/^User asks obot "([^"]*)"$/, async (prompt) => { - await slowInputFilling(Selectors.MCP.obotInput, prompt); - await clickToElement(Selectors.MCP.submitPrompt); - await browser.pause(LONG_PAUSE); + await slowInputFilling(Selectors.MCP.obotInput, prompt); + await clickToElement(Selectors.MCP.submitPrompt); + await browser.pause(LONG_PAUSE); }); Then(/^User connect to the GitLab MCP server$/, async () => { - await slowInputFilling(Selectors.MCP.gitlabToken,process.env.GITLAB_TOKEN); - await clickToElement(Selectors.MCP.btnClick("Launch")); - await browser.pause(LONG_PAUSE); + await slowInputFilling(Selectors.MCP.gitlabToken, process.env.GITLAB_TOKEN); + await clickToElement(Selectors.MCP.btnClick("Launch")); + await browser.pause(LONG_PAUSE); }); -When(/^User sends prompts to Obot AI chat for "([^"]*)" MCP server$/, { timeout: 15 * 60 * 1000 }, async function(serverName: string) { - const jsonPath = path.resolve(process.cwd(), 'src', 'data', `${serverName.toLowerCase()}.MCP.json`); - const data = await fs.readFile(jsonPath, 'utf-8'); - const { prompts, tools } = JSON.parse(data); +When( + /^User sends prompts to Obot AI chat for "([^"]*)" MCP server$/, + { timeout: 15 * 60 * 1000 }, + async function (serverName: string) { + const jsonPath = path.resolve( + process.cwd(), + "src", + "data", + `${serverName.toLowerCase()}.MCP.json`, + ); + const data = await fs.readFile(jsonPath, "utf-8"); + const { prompts, tools } = JSON.parse(data); - this.promptResults = []; - const toolList = tools + this.promptResults = []; + const toolList = tools; - for (let i = 0; i < prompts.length; i++) { - try { - const result = await sendPromptValidateAndCollect(prompts[i], toolList, i); - this.promptResults.push(result); - } catch (err: any) { - console.error(`Error in prompt #${i+1}: ${err.message}`); - this.promptResults.push({ prompt: prompts[i], error: err.message }); + for (let i = 0; i < prompts.length; i++) { + try { + const result = await sendPromptValidateAndCollect( + prompts[i], + toolList, + i, + ); + this.promptResults.push(result); + } catch (err: any) { + console.error(`Error in prompt #${i + 1}: ${err.message}`); + this.promptResults.push({ prompt: prompts[i], error: err.message }); + } } - } -}); + }, +); -Then(/^All prompts results should be validated and report generated for selected "([^"]*)" MCP Server$/, async function(serverName: string) { - const report = aggregateToolResponses(this.promptResults); - saveMCPReport(serverName, report); - - const errors = this.promptResults.filter(r => r.error); - if (errors.length > 0) { - console.warn(`${errors.length} prompts had issues.`); - } -}); +Then( + /^All prompts results should be validated and report generated for selected "([^"]*)" MCP Server$/, + async function (serverName: string) { + const report = aggregateToolResponses(this.promptResults); + saveMCPReport(serverName, report); + const errors = this.promptResults.filter((r) => r.error); + if (errors.length > 0) { + console.warn(`${errors.length} prompts had issues.`); + } + }, +); From aa93dabfe1f9a750904648d93f921c688f3d6d74 Mon Sep 17 00:00:00 2001 From: Taylor Price Date: Thu, 20 Nov 2025 11:08:23 -0700 Subject: [PATCH 3/7] fix: replace function does not exist Signed-off-by: Taylor Price --- .github/workflows/pr-trigger.yml | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pr-trigger.yml b/.github/workflows/pr-trigger.yml index 9c2f464..cde6990 100644 --- a/.github/workflows/pr-trigger.yml +++ b/.github/workflows/pr-trigger.yml @@ -82,12 +82,19 @@ jobs: sleep 5 done + - name: Remove file extension + env: + MATRIX_SERVER_FILE: ${{ matrix.server.file }} + run: | + MCP_SERVER_NAME="${MATRIX_SERVER_FILE/.yaml/}" + echo "MCP_SERVER_NAME=$MCP_SERVER_NAME" >> $GITHUB_ENV + - name: Create catalog entry run: | curl localhost:8080/api/mcp-catalogs/default/entries -X POST -H "Content-Type: application/json" \ -d '{ - "name": "test-${{ replace(matrix.server.file, '.yaml', '') }}", - "description": "testing ${{ replace(matrix.server.file, '.yaml', '') }} mcp", + "name": "test-${MCP_SERVER_NAME}", + "description": "testing ${MCP_SERVER_NAME} mcp", "icon": "https://avatars.githubusercontent.com/u/9919?v=4", "repoURL": "https://github.com/testing/testing", "runtime": "containerized", @@ -114,4 +121,4 @@ jobs: GITLAB_TOKEN: ${{ secrets.GITLAB_TOKEN }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY}} run: | - npx wdio run wdio.conf.ts --cucumberOpts.tagExpression='@${{ replace(matrix.server.file, '.yaml', '') }}' + npx wdio run wdio.conf.ts --cucumberOpts.tagExpression='@${MCP_SERVER_NAME}' From d88daf6210e131483cb4614377aee21198dcafc2 Mon Sep 17 00:00:00 2001 From: Taylor Price Date: Thu, 20 Nov 2025 11:30:23 -0700 Subject: [PATCH 4/7] fix: close escape sequence Signed-off-by: Taylor Price --- .github/workflows/pr-trigger.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr-trigger.yml b/.github/workflows/pr-trigger.yml index cde6990..278297a 100644 --- a/.github/workflows/pr-trigger.yml +++ b/.github/workflows/pr-trigger.yml @@ -102,7 +102,7 @@ jobs: "image": "${{ matrix.server.containerizedConfig.image }}", "port": ${{ matrix.server.containerizedConfig.port }}, "path": "${{ matrix.server.containerizedConfig.path }}", - "args": ${{ toJson(matrix.server.containerizedConfig.args) } + "args": ${{ toJson(matrix.server.containerizedConfig.args) }} }, "metadata": { "categories": "testing" @@ -119,6 +119,6 @@ jobs: WP_PASSWORD: ${{ secrets.WP_PASSWORD }} OBOT_URL: ${{ secrets.OBOT_URL }} GITLAB_TOKEN: ${{ secrets.GITLAB_TOKEN }} - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY}} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} run: | npx wdio run wdio.conf.ts --cucumberOpts.tagExpression='@${MCP_SERVER_NAME}' From 80fcdcdb4b5b1389740f1690c5cc08819d312dd2 Mon Sep 17 00:00:00 2001 From: Shwetapsdet Date: Thu, 6 Nov 2025 12:34:47 -0700 Subject: [PATCH 5/7] Updated the github workflow to add OpenAI Key and updated the eval logic --- auto_eval.ts | 4 ---- src/core/mcpFunc.ts | 12 +++++------- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/auto_eval.ts b/auto_eval.ts index e813e01..b71222d 100644 --- a/auto_eval.ts +++ b/auto_eval.ts @@ -44,7 +44,6 @@ interface GradeInfo { interface ToolData { responses?: string[]; - errors?: string[]; task_done?: boolean | null; failure_reason?: string[]; status?: string; @@ -118,10 +117,7 @@ async function enhanceReportWithEval( // Merge reasons const reasons: string[] = []; if (gradeInfo.reason) reasons.push(gradeInfo.reason); - if (toolData.errors?.length) reasons.push(...toolData.errors); - toolData.failure_reason = reasons; - delete toolData.errors; // Set status based on grading if (gradeInfo.result === "FAILURE") toolData.status = "Failure"; diff --git a/src/core/mcpFunc.ts b/src/core/mcpFunc.ts index d82a060..9962c41 100644 --- a/src/core/mcpFunc.ts +++ b/src/core/mcpFunc.ts @@ -119,7 +119,6 @@ export async function sendPromptValidateAndCollect(promptText: string, toolList: replyElement: currReply, tools: toolsTexts, status: hasSuccess ? 'Success' : (hasFailure ? 'Failure' : 'Unknown'), - error: errorMessage || null, }; } @@ -131,13 +130,13 @@ function maxStatus(s1: string, s2: string): string { export function aggregateToolResponses(promptResults: any[]) { const report: Record + tools: Record }> = {}; for (let i = 0; i < promptResults.length; i++) { const result = promptResults[i]; - const { prompt, tools, reply, status, error } = result; - if (!reply && !error) continue; + const { prompt, tools, reply, status } = result; + if (!reply) continue; const promptKey = `Prompt #${i + 1}`; @@ -153,11 +152,10 @@ export function aggregateToolResponses(promptResults: any[]) { for (const tool of toolsToUse) { if (!report[promptKey].tools[tool]) { - report[promptKey].tools[tool] = { responses: [], status: 'Unknown', errors: [] }; + report[promptKey].tools[tool] = { responses: [], status: 'Unknown'}; } if (reply) report[promptKey].tools[tool].responses.push(reply); - if (error) report[promptKey].tools[tool].errors.push(error); report[promptKey].tools[tool].status = maxStatus(status, report[promptKey].tools[tool].status); @@ -170,7 +168,7 @@ export function aggregateToolResponses(promptResults: any[]) { export function saveMCPReport(mcpName: string, reportJson: any) { const folderName = `MCP Server Reports`; const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); - const fileName = `${mcpName}_MCP_Report_${timestamp}.json`; + const fileName = `${mcpName.toLowerCase().replace(/\s+/g, '_')}_MCP_Report_${timestamp}.json`; const dirPath = path.join(process.cwd(), folderName); const filePath = path.join(dirPath, fileName); From 6fcb9629c0a91ead392a05e0ab60759cf522e594 Mon Sep 17 00:00:00 2001 From: Shwetapsdet Date: Mon, 17 Nov 2025 13:57:36 -0700 Subject: [PATCH 6/7] Added new MCP server and updated the feature file to enhance readability --- src/core/mcpFunc.ts | 26 +- src/core/selectors.ts | 554 ++++++++++++++--------------- src/data/bigquery_toolbox.MCP.json | 20 ++ src/data/brave_search.MCP.json | 25 ++ src/data/chroma_cloud.MCP.json | 32 ++ src/data/databrick_genie.MCP.json | 20 ++ src/data/databrick_unity.MCP.json | 17 + src/data/databrick_vector.MCP.json | 25 ++ src/data/datadog.MCP.json | 51 +++ src/data/deepwiki.MCP.json | 13 + src/data/exa_search.MCP.Json | 11 + src/data/firecrawl.MCP.json | 20 ++ src/data/gitmcp.MCP.json | 16 + src/data/postman.MCP.json | 139 ++++++++ src/data/redis.MCP.json | 21 ++ src/data/tavily_search.MCP.json | 15 + src/features/mcpServer.feature | 48 ++- src/steps/mcpServer.step.ts | 213 +++++++---- src/utils/projectName.model.ts | 12 + wdio.conf.ts | 2 +- 20 files changed, 888 insertions(+), 392 deletions(-) create mode 100644 src/data/bigquery_toolbox.MCP.json create mode 100644 src/data/brave_search.MCP.json create mode 100644 src/data/chroma_cloud.MCP.json create mode 100644 src/data/databrick_genie.MCP.json create mode 100644 src/data/databrick_unity.MCP.json create mode 100644 src/data/databrick_vector.MCP.json create mode 100644 src/data/datadog.MCP.json create mode 100644 src/data/deepwiki.MCP.json create mode 100644 src/data/exa_search.MCP.Json create mode 100644 src/data/firecrawl.MCP.json create mode 100644 src/data/gitmcp.MCP.json create mode 100644 src/data/postman.MCP.json create mode 100644 src/data/redis.MCP.json create mode 100644 src/data/tavily_search.MCP.json create mode 100644 src/utils/projectName.model.ts diff --git a/src/core/mcpFunc.ts b/src/core/mcpFunc.ts index 9962c41..07b2878 100644 --- a/src/core/mcpFunc.ts +++ b/src/core/mcpFunc.ts @@ -68,7 +68,7 @@ export async function sendPromptValidateAndCollect(promptText: string, toolList: // Send and wait for reply const reply = await sendPromptAndWaitForReply(promptText); - await browser.pause(2000); + await browser.pause(10000); // Wait until a new message-content div appears await browser.waitUntil(async () => { @@ -98,27 +98,12 @@ export async function sendPromptValidateAndCollect(promptText: string, toolList: const currReply = promptReplies[await promptReplies.length - 1]; if (!currReply) throw new Error(`No reply container found even after waiting for prompt: "${promptText}"`); - // Validation regex - const successRegex = /(success|completed|connected|created|retrieved|posted|updated|closed|deleted|functioning|valid|available|ready to use)/i; - const failureRegex = /(not valid|failed|error|cannot access|do not have|insufficient|not available|required|troubleshooting)/i; - - const hasSuccess = successRegex.test(reply); - const hasFailure = failureRegex.test(reply); - - let errorMessage = ''; - if (!hasSuccess && !hasFailure) { - errorMessage = `No success or actionable failure detected in prompt #${index + 1} response.`; - } - - console.log(`Prompt #${index + 1}: Tools used: ${toolsTexts.length ? toolsTexts.join(', ') : 'None'} | Status: ${hasSuccess ? 'Success' : (hasFailure ? 'Failure' : 'Unknown')}`); - // Return data for reporting return { prompt: promptText, reply, replyElement: currReply, tools: toolsTexts, - status: hasSuccess ? 'Success' : (hasFailure ? 'Failure' : 'Unknown'), }; } @@ -130,12 +115,12 @@ function maxStatus(s1: string, s2: string): string { export function aggregateToolResponses(promptResults: any[]) { const report: Record + tools: Record }> = {}; for (let i = 0; i < promptResults.length; i++) { const result = promptResults[i]; - const { prompt, tools, reply, status } = result; + const { prompt, tools, reply } = result; if (!reply) continue; const promptKey = `Prompt #${i + 1}`; @@ -152,13 +137,10 @@ export function aggregateToolResponses(promptResults: any[]) { for (const tool of toolsToUse) { if (!report[promptKey].tools[tool]) { - report[promptKey].tools[tool] = { responses: [], status: 'Unknown'}; + report[promptKey].tools[tool] = { responses: []}; } if (reply) report[promptKey].tools[tool].responses.push(reply); - - report[promptKey].tools[tool].status = - maxStatus(status, report[promptKey].tools[tool].status); } } diff --git a/src/core/selectors.ts b/src/core/selectors.ts index a0e9465..132b74c 100644 --- a/src/core/selectors.ts +++ b/src/core/selectors.ts @@ -1,280 +1,280 @@ const Selectors = { - loginButton: '//button[text()="Login"]', - obotPopUp: '//dialog[contains(@class, "fixed top-1/2 left-1/2")]', - continueWithOkta: '//span[text()="Continue with Okta"]', - continueWithGoogle: '//span[text()="Continue with Google"]', - clickonAccount: '//div[text()="Use another account"]', - emailId: '//input[@type="email"]', - usernameField: '//input[@type="text"]', - passwordField: '//input[@type="password"]', - submitButton: '//input[@type="submit"]', - createnewobotButton: - "//button[contains(@class, 'button') and contains(., 'Create New Agent')]", - editpencilIcon: - '//*[contains(@class,"pencil h-5")]/ancestor::button[contains(@class,"group text")]', - messageContent: - '//div[@class="flex w-full items-center gap-4 p-2"]//textarea[@placeholder="Your message..."]', - updateUsername: '//input[@class="bg-surface grow rounded-lg p-2"]', - exitEditor: - '//button[@class="group relative mr-1 flex items-center rounded-full border p-2 text-xs transition-[background-color] duration-200 border-blue bg-blue text-white md:px-4"]', - fileIcon: '//label[text()="Your messages"]/following-sibling::div//button', - obotUpdatedname: '//h4[@class="mb-1!"]', - deleteButton: '//button[@class="button-destructive"]', - deleteTool: '(//button[@class="icon-button-small"])[1]', - fileuploadPopup: - "//div[contains(@class, 'default-dialog') and @style='left: 166px; top: 12.833px;']", - fileuploadButton: - "(//label[contains(@class, 'button') and contains(@class, 'flex') and contains(., 'Upload')])[2]", - filenameVerify: '//span[text()="AI_test.txt"]', - fileUpload: - '//label[contains(normalize-space(), "Upload")]//input[@type="file" and @accept=".pdf, .txt, .doc, .docx, .ppt, .pptx, .md, .rtf, .html, .odt, .ipynb, .json,.csv, .png, .jpg, .jpeg, .webp"]', - confirmDeleteObotButton: - '//h3[text()="Delete the current Obot?"]/following-sibling::button[contains(text(), "Yes") and contains(@class, "bg-red-600")]', - clickChatbot: - '//button[@class="icon-button group relative flex items-center gap-2 p-0 shadow-md"]', - clickOnDiv: '//div[@id="click-catch"]', - clickshowOutput: '//button[text()="Show Output"]', - chatOutputElement: - '//p[text()="Output"]/following-sibling::pre[contains(@class,"text")]', - task: { - expandTaskMenu: - '//div[@class="flex items-center justify-between"]//button[@class="icon-button"][1]', - createNewTaskButton: '//button[contains(., "New Task")]', - strongTask: '//strong[text()="TASK"]', - newTaskTitle: '//input[@class="ghost-input text-2xl font-semibold"]', - deleteTask: - '//strong[text()="TASK"]/../..//button[@class="button-destructive p-4"]', - confirmDeleteTask: - '//h3[contains(text(),"delete this task")]/following-sibling::button[contains(text(), "Yes") and contains(@class, "bg-red-600")]', - stepsInput: - '//textarea[@class="ghost-input border-surface2 ml-1 grow resize-none scrollbar-none"]', - runButton: - "(//button[contains(@class, 'bg-blue') and contains(@class, 'text-white')])", - advancedOptionBtn: '//button[text()="Show Advanced Options..."]', - argumentBtn: '(//button[contains(text(),"Argument")])', - inputArgument: '//input[@placeholder="Enter Name"]', - argumentPopup: - '//h4[text()="Arguments"]/following-sibling::div//input[@id="param-tax calculator"]', - argumentValue: '//input[@id="param-Enter your income"]', - submitArgs: - "(//button[text()='Run' and contains(@class, 'button-primary')])[2]", - inputResult: - "(//div[contains(@class, 'relative') and contains(@class, 'my-3')])[3]", - inputTextHeader: '//p[contains(text(),"Here is the tax on your income")]', - inputText: - '//span[@class="text-sm font-semibold" and contains(text(),"Automation")]', - searchBot: '//textarea[@id="chat"]', - submitButton: - '//button[@type="submit" and contains(@class, "button-colors") and contains(@class, "text-blue")]', - triggerTypeBtn: '(//button[normalize-space(text())="on demand"])[1]', - selectTriggerType: (option: string) => - `//button[normalize-space(text())='${option}']`, - webHookURL: - '//div[contains(@class, "bg-surface2") and contains(., "https://")]', - runTask: '//div[contains(@class, "flex") and contains(@class, "truncate")]', - taskResult: '//ul[@class="ml-4 flex flex-col text-xs"]//li[1]//div', - tableResult: - '//span[text()="Tables "]/../following-sibling::div/div/div/ul/li/button', - tableRefresh: '//button[text()=" Refresh"]', - TaskMenu: '//button[.//span[contains(text(), "Tasks")]]', - sideBar: '(//button[@class="icon-button"])[1]', - timeTypeBtn: (option: string) => - `(//button[normalize-space(text())='${option}'])[1]`, - clickTask: '//span[text()="Tasks "]', - createNewTask: '//button[text()=" New Task"]', - loopStep: '//button[@data-testid="step-loop-btn"]', - btnrun: '//button[text()="Run "]', - }, - table: { - id: "//tbody//tr//td[1]", - name: "//tbody//tr//td[2]", - email: "//tbody//tr//td[3]", - tableTask: "(//div/child::textarea)[2]", - submitInput: - '(//button[@type="submit" and contains(@class,"button-colors")])[2]', - }, - toolIcon: '//button[@class="button-icon-primary"]', - addTool: '//button[contains(text(),"Tools")]', - searchTools: '//input[contains(@placeholder,"Search tools")]', - selectToolQuery: `(//span[normalize-space(text())="{{toolName}}"])[2]`, - applyTool: '//button[text()="Apply"]', - uploadButton: '//label[text()=" Upload "]', - expandknowledgeFileicon: '//span[text()="Knowledge "]', - expandFileicon: - '//span[text()="Files"]/parent::button[@class="flex items-center gap-2 px-5 py-2"]', - scrollableDiv: '//div[@class="default-scrollbar-thin flex grow flex-col"]', - verifytheKnowlegefileName: '//span[@class="ms-3 truncate"]', - verifythefileName: '//span[@class="truncate"]', - knowledgeFileUpload: - '//label[contains(normalize-space(), "Upload")]//input[@type="file" and @accept=".pdf, .txt, .doc, .docx"]', - showDetails: '//button[text()="Show Details"]', - chatbotTextArea: '//div/textarea[@placeholder="Your message..."]', - submitChat: '//button[@type="submit" and contains(@class,"button-colors")]', - showAdvanceOptions: '//span[text()="Show Advanced Options..."]', - advancedOptionTabs: { - interfaceTab: '//span[text()="Interface"]', - tasksTab: '//span[text()="Tasks"]', - }, - chatIntroductionTextArea: '//div/textarea[@placeholder="Introduction"]', - starterMsgButton: - '//span[text()="Starter Message"]/ancestor::button[contains(@class,"button")]', - starterMsgInput: '//textarea[@id="project-instructions"]', - chatIntroductionInObot: '//div[contains(@class,"message-content")]', - chatStarterMsgInObot: '//button/span[@class="line-clamp-3"]', - agentCatalog: '//a[text()="Agent Catalog"]', - createNewAgent: '//button[contains(@class, "bg-surface1")]', - agentEditHover: - '//div[@class="flex flex-col items-center justify-center text-center"]', - agentEditPencilIcon: '(//button[contains(@class, "rounded-full")])[1]', - updateAgentName: '//input[contains(@class, "ghost-input")]', - confirmDelete: '(//button[normalize-space(text())="Yes, I\'m sure"])[3]', - tableInput: - '//pre[contains(@class, "whitespace-pre-wrap")]//span[contains(@class, "text-gray-500")]', - showDetails2: '(//button[text()="Show Details"])[2]', - showDetails3: '(//button[text()="Show Details"])[3]', + loginButton: '//button[text()="Login"]', + obotPopUp: '//dialog[contains(@class, "fixed top-1/2 left-1/2")]', + continueWithOkta: '//span[text()="Continue with Okta"]', + continueWithGoogle: '//span[text()="Continue with Google"]', + clickonAccount: '//div[text()="Use another account"]', + emailId: '//input[@type="email"]', + usernameField: '//input[@type="text"]', + passwordField: '//input[@type="password"]', + submitButton: '//input[@type="submit"]', + createnewobotButton: "//button[contains(@class, 'button') and contains(., 'Create New Agent')]", + editpencilIcon: '//*[contains(@class,"pencil h-5")]/ancestor::button[contains(@class,"group text")]', + messageContent: '//div[@class="flex w-full items-center gap-4 p-2"]//textarea[@placeholder="Your message..."]', + updateUsername: '//input[@class="bg-surface grow rounded-lg p-2"]', + exitEditor: '//button[@class="group relative mr-1 flex items-center rounded-full border p-2 text-xs transition-[background-color] duration-200 border-blue bg-blue text-white md:px-4"]', + fileIcon:'//label[text()="Your messages"]/following-sibling::div//button', + obotUpdatedname: '//h4[@class="mb-1!"]', + deleteButton: '//button[@class="button-destructive"]', + deleteTool:'(//button[@class="icon-button-small"])[1]', + fileuploadPopup:"//div[contains(@class, 'default-dialog') and @style='left: 166px; top: 12.833px;']", + fileuploadButton:"(//label[contains(@class, 'button') and contains(@class, 'flex') and contains(., 'Upload')])[2]", + filenameVerify:'//span[text()="AI_test.txt"]', + fileUpload:'//label[contains(normalize-space(), "Upload")]//input[@type="file" and @accept=".pdf, .txt, .doc, .docx, .ppt, .pptx, .md, .rtf, .html, .odt, .ipynb, .json,.csv, .png, .jpg, .jpeg, .webp"]', + confirmDeleteObotButton: '//h3[text()="Delete the current Obot?"]/following-sibling::button[contains(text(), "Yes") and contains(@class, "bg-red-600")]', + clickChatbot:'//button[@class="icon-button group relative flex items-center gap-2 p-0 shadow-md"]', + clickOnDiv:'//div[@id="click-catch"]', + clickshowOutput:'//button[text()="Show Output"]', + chatOutputElement:'//p[text()="Output"]/following-sibling::pre[contains(@class,"text")]', + task: { + expandTaskMenu: '//div[@class="flex items-center justify-between"]//button[@class="icon-button"][1]', + createNewTaskButton: '//button[contains(., "New Task")]', + strongTask: '//strong[text()="TASK"]', + newTaskTitle: '//input[@class="ghost-input text-2xl font-semibold"]', + deleteTask: '//strong[text()="TASK"]/../..//button[@class="button-destructive p-4"]', + confirmDeleteTask: '//h3[contains(text(),"delete this task")]/following-sibling::button[contains(text(), "Yes") and contains(@class, "bg-red-600")]', + stepsInput: '//textarea[@class="ghost-input border-surface2 ml-1 grow resize-none scrollbar-none"]', + runButton: "(//button[contains(@class, 'bg-blue') and contains(@class, 'text-white')])", + advancedOptionBtn:'//button[text()="Show Advanced Options..."]', + argumentBtn: '(//button[contains(text(),"Argument")])', + inputArgument: '//input[@placeholder="Enter Name"]', + argumentPopup: '//h4[text()="Arguments"]/following-sibling::div//input[@id="param-tax calculator"]', + argumentValue:'//input[@id="param-Enter your income"]', + submitArgs: "(//button[text()='Run' and contains(@class, 'button-primary')])[2]", + inputResult: "(//div[contains(@class, 'relative') and contains(@class, 'my-3')])[3]", + inputTextHeader: '//p[contains(text(),"Here is the tax on your income")]', + inputText:'//span[@class="text-sm font-semibold" and contains(text(),"Automation")]', + searchBot: '//textarea[@id="chat"]', + submitButton:'//button[@type="submit" and contains(@class, "button-colors") and contains(@class, "text-blue")]', + triggerTypeBtn:'(//button[normalize-space(text())="on demand"])[1]', + selectTriggerType: (option: string) => `//button[normalize-space(text())='${option}']`, + webHookURL:'//div[contains(@class, "bg-surface2") and contains(., "https://")]', + runTask:'//div[contains(@class, "flex") and contains(@class, "truncate")]', + taskResult:'//ul[@class="ml-4 flex flex-col text-xs"]//li[1]//div', + tableResult:'//span[text()="Tables "]/../following-sibling::div/div/div/ul/li/button', + tableRefresh:'//button[text()=" Refresh"]', + TaskMenu:'//button[.//span[contains(text(), "Tasks")]]', + sideBar:'(//button[@class="icon-button"])[1]', + timeTypeBtn: (option: string) => `(//button[normalize-space(text())='${option}'])[1]`, + clickTask:'//span[text()="Tasks "]', + createNewTask:'//button[text()=" New Task"]', + loopStep:'//button[@data-testid="step-loop-btn"]', + btnrun:'//button[text()="Run "]', + }, + table:{ + id:'//tbody//tr//td[1]', + name:'//tbody//tr//td[2]', + email:'//tbody//tr//td[3]', + tableTask:'(//div/child::textarea)[2]', + submitInput:'(//button[@type="submit" and contains(@class,"button-colors")])[2]', + }, + toolIcon: '//button[@class="button-icon-primary"]', + addTool: '//button[contains(text(),"Tools")]', + searchTools: '//input[contains(@placeholder,"Search tools")]', + selectToolQuery: `(//span[normalize-space(text())="{{toolName}}"])[2]`, + applyTool: '//button[text()="Apply"]', + uploadButton: '//label[text()=" Upload "]', + expandknowledgeFileicon: '//span[text()="Knowledge "]', + expandFileicon:'//span[text()="Files"]/parent::button[@class="flex items-center gap-2 px-5 py-2"]', + scrollableDiv: '//div[@class="default-scrollbar-thin flex grow flex-col"]', + verifytheKnowlegefileName: '//span[@class="ms-3 truncate"]', + verifythefileName:'//span[@class="truncate"]', + knowledgeFileUpload: '//label[contains(normalize-space(), "Upload")]//input[@type="file" and @accept=".pdf, .txt, .doc, .docx"]', + showDetails: '//button[text()="Show Details"]', + chatbotTextArea: '//div/textarea[@placeholder="Your message..."]', + submitChat: '//button[@type="submit" and contains(@class,"button-colors")]', + showAdvanceOptions: '//span[text()="Show Advanced Options..."]', + advancedOptionTabs: { + interfaceTab: '//span[text()="Interface"]', + tasksTab: '//span[text()="Tasks"]', + }, + chatIntroductionTextArea: '//div/textarea[@placeholder="Introduction"]', + starterMsgButton: '//span[text()="Starter Message"]/ancestor::button[contains(@class,"button")]', + starterMsgInput: '//textarea[@id="project-instructions"]', + chatIntroductionInObot: '//div[contains(@class,"message-content")]', + chatStarterMsgInObot: '//button/span[@class="line-clamp-3"]', + agentCatalog:'//a[text()="Agent Catalog"]', + createNewAgent:'//button[contains(@class, "bg-surface1")]', + agentEditHover:'//div[@class="flex flex-col items-center justify-center text-center"]', + agentEditPencilIcon:'(//button[contains(@class, "rounded-full")])[1]', + updateAgentName:'//input[contains(@class, "ghost-input")]', + confirmDelete:'(//button[normalize-space(text())="Yes, I\'m sure"])[3]', + tableInput:'//pre[contains(@class, "whitespace-pre-wrap")]//span[contains(@class, "text-gray-500")]', + showDetails2: '(//button[text()="Show Details"])[2]', + showDetails3: '(//button[text()="Show Details"])[3]', + connectionsList: `//div[@class="flex flex-col"]`, + currentProjectButton: `//span[text()="Project"]/ancestor::button`, + createNewProjectButton: `//button[text()=" Create New Project"]`, + inputNewProjectName: '//input[@id="project-name"]', + saveButton: '//button[text()="Save"]', - admin: { - oktaLogin: '//button[normalize-space(.//div) = "Sign in with Okta"]', - crateNewAgent: '//button[normalize-space(.//div) = "New Agent"]', - agentName: "(//div/child::input)[1]", - agentDes: "(//div/child::input)[2]", - tryItOutBtn: "//div/child::a", - editAgent: '(//a[contains(@href, "/admin/agents/")])[1]', - deleteAgent: - '(//a[contains(@href, "/admin/agents/")]/following-sibling::button)[1]', - confirmDelete: '//div[text()="Confirm"]', - agenetDeletedMessagge: - '//div[@data-content]//div[contains(text(), "Agent deleted")]', - memoryDropdown: - '//p[text()="Memory"]/ancestor::div[contains(@class, "flex")]/following-sibling::div//button', - setAlwaysOn: - '//div[contains(@role, "option")]/span[contains(text(),"Always On")]', - addTool: '//div[text()=" Add Tools"]', - addKnowledge: '//div[text()="Add Knowledge"]', - localFiles: '//div[text()="Local Files"]', - searchTool: '//input[@placeholder="Search tools..."]', - googleSearch: '//span[text()="Google Search "]', - weather: '//span[text()="Weather "]', - viewAllMemories: '//button[text()="View All Memories"]', - advanced: '//h4[text()="Advanced"]', - labelClick: `(//label[normalize-space(text())="{{labelClick}}"])`, - userleftsideMenu: '//span[text()="Users"]', - userFilter: '//span[text()="User"]', - searchText: '//input[@placeholder="Filter..."]', - clickonthreedotBtn: - '//div[@class="flex justify-end"]//button[@type="button"]', - updateroleBtn: '//div[text()="Update Role"]', - clickonroleDrp: '//button[@aria-controls="radix-:rceg:"]', - selectRole: '//span[text()="{{role}}"]', - updateBtn: '//div[text()="Update"]', - deleteroleBtn: '//div[text()="Delete User"]', - deleteBtn: '//div[text()="Delete"]', - chatthreadMenu: '//a//span[text()="Chat Threads"]', - }, - agentDes: '//p[@class="text-gray w-sm font-light md:w-md"]', - authLink: - '//a[contains(@class, "bg-blue") and contains(@class, "rounded-3xl") and contains(@class, "text-white")]', - scheduleDay: '(//button[normalize-space(text())="daily"])[1]', - scheduleTime: '(//button[normalize-space(text())="on the hour"])[1]', - homeButton: '//a[@id="navbar-home-link"]', - continueBtn: - '//button[contains(text(), "I dunno")]/following-sibling::button', - lunchAgent: '//button[normalize-space(text())="Launch Agent"]', - obotEditArea: '//button[@id="edit-basic-details-button"]', - editAreaPencil: '(//button[@id="edit-basic-details-button"]/child::div)[2]', - editAgentName: '//input[@id="project-name"]', - obotNav: "(//div/child::span/following-sibling::button)[1]", - closeEditArea: '//button[@class="icon-button absolute top-2 right-2"]', - addKnowledge: '//p[text()="Knowledge"]/following-sibling::div', - clicksNewThread: '//a[.//img[@alt="Obot icon"]]/following-sibling::button', - memoryValidate: "//tr//td[2]", - refreashMemory: '//span[text()="Most recent memories"]/../div', - memoryIcon: - '//button[@data-memories-btn and contains(@class, "icon-button")]', - closeMemory: '(//button[contains(@class, "absolute top-0 right-0 p-3")])[1]', - deleteAllMemory: '(//div//button[contains(text()," Delete All")])[2]', - deleteMemoryValidate: '//p[text()="No memories stored"]', - startFromScratch: '//button[text()=" Create From Scratch"]', - configuration: '//span[text()="Configuration "]', - editObotName: '//span[text()="Name & Description "]', - mcpServer: '//span[text()="MCP Servers "]', - addMCPServer: '//button[text()=" Add MCP Server"]', - searchMCPTool: '//input[@placeholder="Search MCP Servers..."]', - selectedTool: `(//h4[normalize-space(text())="{{mcpTool}}"])`, - mcpStep: `(//button[(text())="{{mcpStep}}"])`, - selectServer: '(//button[text()="Select Server "])', - addServer: '//button[text()="Add server"]', - validateMCPTool: (option: string) => - `//p[normalize-space(text())="${option}"]`, - knowledge: '//span[text()="Knowledge "]', - selectFromSidebar: `(//span[(text())="{{select}}"])`, - enableChatBot: '//h5[text()="Enable"]/../label', - copyTemplates: - '//div/a[@class="overflow-hidden text-sm text-ellipsis hover:underline"]', - copyChatBot: - '//div/a[@class="overflow-hidden text-ellipsis hover:underline"]', - validateThread: '//button[text()="New Thread"]', - // btnIUnderstand:'//button[text()="I Understand"]', - btnClick: `(//button[normalize-space(text())="{{btnclick}}"])`, - templatePencil: - '(//button[contains(@class, "text-gray-500 hover:text-gray-700 dark:text-gray-400 dark:hover:text-gray-200")])[2]', - publicTemplate: '(//span[@class="slider rounded-2xl"])[3]', - templateName: '//div//div//h3[@class="text-base font-medium"]', - modelApi: '//input[@id="OBOT_OPENAI_MODEL_PROVIDER_API_KEY"]', - modelSelect: `(//label[(text())="{{modelSelect}}"])`, - ValidateModel: '//button[@title="Select model for this thread"]', - gpt3btn: '//div[text()="gpt-3.5-turbo"]', - gpt4btn: '//div[text()="gpt-4"]', - message: '//label[text()="Your messages"]', - databaseOn: '//span[text()="Toggle Capability"]/../span[2]', - profileIcon: '//a[@id="navbar-home-link"]/following-sibling::button', - logOut: '//a[text()=" Log out"]', - cancelTool: '//span[text()=" AWS S3"]/following-sibling::button', - clickherelink: '//span[text()="click here"]', - signinHubspot: - "//i18n-string[normalize-space()='Sign in to your HubSpot account']", - loginHubspotButton: '//i18n-string[text()="Log in"]', - emailHubspot: '//input[@type="email"]', - nextButton: '//button[@type="submit"]//i18n-string', - passwordHubspot: '//input[@type="password"]', - authentication: '//p[text()="Authentication is required."]', - // fillEmail: '//input[@placeholder="Email, phone, or Skype"]', - fillEmail: '//input[@autocomplete="username"]', - clickNext: '//input[@value="Next"]', - getCode: '//button[text()="Send code"]', - - MCP: { - googlebtn: '//span[text()="Continue with Google"]', - emailInput: '//input[@type="email"]', - nextbtn: '//span[text()="Next"]', - passwordInput: '//input[@type="password"]', - cntBtn: '//span[text()="Continue"]', - navigationbtn: '//button[@class="icon-button z-20"]', - clickChatObot: '//button[normalize-space(text())="Chat"]', - connectorbtn: - '//p[normalize-space(text())="Connectors"]/following-sibling::button', - mcpSearchInput: - '//input[normalize-space(@placeholder)="Search by name..."]', - // selectMCPServer: '//p[normalize-space(text())="WordPress"]', - selectMCPServer: (option: string) => - `//p[normalize-space(text())="${option}"]/ancestor::div[contains(@class, 'flex')]/button`, - wpSiteURL: '//input[normalize-space(@id)="WORDPRESS_SITE"]', - wpUsername: '//input[normalize-space(@id)="WORDPRESS_USERNAME"]', - wpPassword: '//input[normalize-space(@id)="WordPress App Password"]', - btnClick: (option: string) => - `//button[normalize-space(text())="${option}"]`, - promptInput: - '//div[@class="plaintext-editor text-md relative w-full flex-1 grow resize-none p-2 leading-8 outline-none"]', - // submitPrompt: '//button[@type="submit"]', - // obotInput: '//div[@class="ProseMirror editor"]', - gitlabToken: '//input[@name="GitLab Personal Access Token"]', - // messageContainer: "//div[contains(@class, 'flex-1') and contains(@class, 'flex-col') and contains(@class, 'justify-start') and contains(@class, 'gap-8')]", - obotInput: - "//div[contains(@class,'ProseMirror') and @contenteditable='true']", - submitPrompt: '//button[@type="submit"]', - lastBotReply: '//div[@class="message-content"]', - messageContainer: - "//div[contains(@class, 'flex-1') and contains(@class, 'flex-col') and contains(@class, 'justify-start') and contains(@class, 'gap-8')]", - }, -}; + admin:{ + oktaLogin:'//button[normalize-space(.//div) = "Sign in with Okta"]', + crateNewAgent:'//button[normalize-space(.//div) = "New Agent"]', + agentName:'(//div/child::input)[1]', + agentDes:'(//div/child::input)[2]', + tryItOutBtn:'//div/child::a', + editAgent:'(//a[contains(@href, "/admin/agents/")])[1]', + deleteAgent:'(//a[contains(@href, "/admin/agents/")]/following-sibling::button)[1]', + confirmDelete:'//div[text()="Confirm"]', + agenetDeletedMessagge:'//div[@data-content]//div[contains(text(), "Agent deleted")]', + memoryDropdown:'//p[text()="Memory"]/ancestor::div[contains(@class, "flex")]/following-sibling::div//button', + setAlwaysOn:'//div[contains(@role, "option")]/span[contains(text(),"Always On")]', + addTool:'//div[text()=" Add Tools"]', + addKnowledge:'//div[text()="Add Knowledge"]', + localFiles:'//div[text()="Local Files"]', + searchTool:'//input[@placeholder="Search tools..."]', + googleSearch:'//span[text()="Google Search "]', + weather:'//span[text()="Weather "]', + viewAllMemories:'//button[text()="View All Memories"]', + advanced:'//h4[text()="Advanced"]', + labelClick: `(//label[normalize-space(text())="{{labelClick}}"])`, + userleftsideMenu:'//span[text()="Users"]', + userFilter:'//span[text()="User"]', + searchText:'//input[@placeholder="Filter..."]', + clickonthreedotBtn:'//div[@class="flex justify-end"]//button[@type="button"]', + updateroleBtn:'//div[text()="Update Role"]', + clickonroleDrp:'//button[@aria-controls="radix-:rceg:"]', + selectRole:'//span[text()="{{role}}"]', + updateBtn:'//div[text()="Update"]', + deleteroleBtn:'//div[text()="Delete User"]', + deleteBtn:'//div[text()="Delete"]', + chatthreadMenu:'//a//span[text()="Chat Threads"]', + }, + agentDes:'//p[@class="text-gray w-sm font-light md:w-md"]', + authLink:'//a[contains(@class, "bg-blue") and contains(@class, "rounded-3xl") and contains(@class, "text-white")]', + scheduleDay:'(//button[normalize-space(text())="daily"])[1]', + scheduleTime:'(//button[normalize-space(text())="on the hour"])[1]', + homeButton:'//a[@id="navbar-home-link"]', + continueBtn:'//button[contains(text(), "I dunno")]/following-sibling::button', + lunchAgent:'//button[normalize-space(text())="Launch Agent"]', + obotEditArea:'//button[@id="edit-basic-details-button"]', + editAreaPencil: '(//button[@id="edit-basic-details-button"]/child::div)[2]', + editAgentName:'//input[@id="project-name"]', + obotNav:'(//div/child::span/following-sibling::button)[1]', + closeEditArea:'//button[@class="icon-button absolute top-2 right-2"]', + addKnowledge:'//p[text()="Knowledge"]/following-sibling::div', + clicksNewThread:'//a[.//img[@alt="Obot icon"]]/following-sibling::button', + memoryValidate:'//tr//td[2]', + refreashMemory:'//span[text()="Most recent memories"]/../div', + memoryIcon:'//button[@data-memories-btn and contains(@class, "icon-button")]', + closeMemory:'(//button[contains(@class, "absolute top-0 right-0 p-3")])[1]', + deleteAllMemory:'(//div//button[contains(text()," Delete All")])[2]', + deleteMemoryValidate:'//p[text()="No memories stored"]', + startFromScratch:'//button[text()=" Create From Scratch"]', + configuration:'//span[text()="Configuration "]', + editObotName:'//span[text()="Name & Description "]', + mcpServer:'//span[text()="MCP Servers "]', + addMCPServer:'//button[text()=" Add MCP Server"]', + searchMCPTool: '//input[@placeholder="Search MCP Servers..."]', + selectedTool: `(//h4[normalize-space(text())="{{mcpTool}}"])`, + mcpStep: `(//button[(text())="{{mcpStep}}"])`, + selectServer:'(//button[text()="Select Server "])', + addServer:'//button[text()="Add server"]', + validateMCPTool: (option: string) => `//p[normalize-space(text())="${option}"]`, + knowledge:'//span[text()="Knowledge "]', + selectFromSidebar:`(//span[(text())="{{select}}"])`, + enableChatBot:'//h5[text()="Enable"]/../label', + copyTemplates:'//div/a[@class="overflow-hidden text-sm text-ellipsis hover:underline"]', + copyChatBot:'//div/a[@class="overflow-hidden text-ellipsis hover:underline"]', + validateThread:'//button[text()="New Thread"]', + // btnIUnderstand:'//button[text()="I Understand"]', + btnClick: `(//button[normalize-space(text())="{{btnclick}}"])`, + templatePencil:'(//button[contains(@class, "text-gray-500 hover:text-gray-700 dark:text-gray-400 dark:hover:text-gray-200")])[2]', + publicTemplate:'(//span[@class="slider rounded-2xl"])[3]', + templateName:'//div//div//h3[@class="text-base font-medium"]', + modelApi:'//input[@id="OBOT_OPENAI_MODEL_PROVIDER_API_KEY"]', + modelSelect: `(//label[(text())="{{modelSelect}}"])`, + ValidateModel:'//button[@title="Select model for this thread"]', + gpt3btn:'//div[text()="gpt-3.5-turbo"]', + gpt4btn:'//div[text()="gpt-4"]', + message:'//label[text()="Your messages"]', + databaseOn:'//span[text()="Toggle Capability"]/../span[2]', + profileIcon:'//a[@id="navbar-home-link"]/following-sibling::button', + logOut:'//a[text()=" Log out"]', + cancelTool:'//span[text()=" AWS S3"]/following-sibling::button', + clickherelink:'//span[text()="click here"]', + signinHubspot:"//i18n-string[normalize-space()='Sign in to your HubSpot account']", + loginHubspotButton:'//i18n-string[text()="Log in"]', + emailHubspot:'//input[@type="email"]', + nextButton:'//button[@type="submit"]//i18n-string', + passwordHubspot:'//input[@type="password"]', + authentication: '//p[text()="Authentication is required."]', + // fillEmail: '//input[@placeholder="Email, phone, or Skype"]', + fillEmail: '//input[@autocomplete="username"]', + clickNext:'//input[@value="Next"]', + getCode:'//button[text()="Send code"]', + + MCP:{ + googlebtn: '//span[text()="Continue with Google"]', + emailInput: '//input[@type="email"]', + nextbtn: '//span[text()="Next"]', + passwordInput: '//input[@type="password"]', + cntBtn: '//span[text()="Continue"]', + navigationbtn: '//button[@class="icon-button z-20"]', + clickChatObot: '//button[normalize-space(text())="Chat"]', + connectorbtn: '//p[normalize-space(text())="Connectors"]/following-sibling::button', + mcpSearchInput: '//input[normalize-space(@placeholder)="Search by name..."]', + selectMCPServer: (option: string) => `//p[normalize-space(text())="${option}"]/ancestor::div[contains(@class, 'flex')]/button`, + btnClick: (option: string) => `//button[normalize-space(text())="${option}"]`, + promptInput: '//div[@class="plaintext-editor text-md relative w-full flex-1 grow resize-none p-2 leading-8 outline-none"]', + obotInput: "//div[contains(@class,'ProseMirror') and @contenteditable='true']", + submitPrompt: '//button[@type="submit"]', + lastBotReply: '//div[@class="message-content"]', + messageContainer: "//div[contains(@class, 'flex-1') and contains(@class, 'flex-col') and contains(@class, 'justify-start') and contains(@class, 'gap-8')]", + wordpressMCP:{ + wpSiteURL: '//input[normalize-space(@id)="WORDPRESS_SITE"]', + wpUsername: '//input[normalize-space(@id)="WORDPRESS_USERNAME"]', + wpPassword: '//input[normalize-space(@id)="WordPress App Password"]', + }, + gitlabMCP:{ + gitlabToken: '//input[@name="GitLab Personal Access Token"]', + }, + bigQuery:{ + googleCloudProjectID: '//input[@id="GOOGLE_CLOUD_PROJECT"]', + googleCloudCredentials: '//input[@name="Google Application Credentials"]//following-sibling::div[1]', + }, + datadog:{ + datadogAPIKey: `//input[@id="Datadog API Key"]`, + datadogAPPKey: `//input[@id="Datadog App Key"]`, + }, + databricks:{ + utility:{ + workspaceHostname: `//input[@id="DATABRICKS_WORKSPACE_URL"]`, + functionCatalog: `//input[@id="DATABRICKS_FUNCTIONS_CATALOG"]`, + functionalSchema: `//input[@id="DATABRICKS_FUNCTIONS_SCHEMA"]`, + PAT: `//input[@id="Personal Access Token"]`, + }, + vector:{ + vectorCatalog: `//input[@id="DATABRICKS_VECTOR_SEARCH_CATALOG"]`, + vectorSchema: `//input[@id="DATABRICKS_VECTOR_SEARCH_SCHEMA"]`, + }, + genie: { + genieSpaceID: `//input[@id="DATABRICKS_GENIE_SPACE_ID"]` + } + }, + brave: { + braveAPIKey: `//input[@id="Brave API Key"]` + }, + chromaCloud: { + tenentID: `//input[@id="CHROMA_TENANT"]`, + DBName: `input[@id="CHROMA_DATABASE"]`, + APIKey: `//input[@id="Chroma Cloud API Key"]` + }, + fireCrawl: { + API_key: `//input[@id="Firecrawl API Key"]` + }, + gitMCP: { + urlLink: `//input[@id="url-manifest-url"]` + }, + redis: { + urlLink: `//input[@id="REDIS_URI"]` + }, + postman: { + hostURL: `//input[@id="HOST_URL"]`, + toolCOnfig: `//input[@id="TOOL_CONFIGURATION"]`, + postmanAPIKey: `//input[@id="Postman API Key"]`, + } + } +} export default Selectors; diff --git a/src/data/bigquery_toolbox.MCP.json b/src/data/bigquery_toolbox.MCP.json new file mode 100644 index 0000000..701a509 --- /dev/null +++ b/src/data/bigquery_toolbox.MCP.json @@ -0,0 +1,20 @@ +{ + "prompts": [ + "Is BigQuery Toolbox connected?", + "Retrieve all columns from the table company_data.Employees where Salary is greater than 60000.", + "List the first 10 employees' first and last names from company_data.Employees.", + "Count the total number of employees in company_data.Employees.", + "Find the average salary of all employees in company_data.Employees.", + "Find all employees from company_data.Employees who were hired after January 1, 2023.", + "Get the total salary amount paid to all employees in company_data.Employees.", + "List employees from company_data.Employees with their email addresses ordered by HireDate descending.", + "Use service account credentials to query the company_data.Employees table.", + "Run SELECT * FROM company_data.Employees LIMIT 5.", + "Find the employee with the highest salary in company_data.Employees.", + "Find the employee with the lowest salary in company_data.Employees.", + "Find the number of employees hired per year from company_data.Employees." + ], + "tools": [ + "query" + ] +} diff --git a/src/data/brave_search.MCP.json b/src/data/brave_search.MCP.json new file mode 100644 index 0000000..c727c55 --- /dev/null +++ b/src/data/brave_search.MCP.json @@ -0,0 +1,25 @@ +{ + "prompts": [ + "Is Brave Search MCP server connected?", + "Search the web for 'best productivity apps 2025' and return the top 5 results.", + "Find web pages about 'remote work trends' published in the last 7 days.", + "Search for the latest news about 'AI regulation' and return the 5 most recent articles.", + "Find breaking news on 'electric vehicles' from the last 24 hours.", + "Search for images of 'modern home office setups' and return the top 5 results.", + "Find safe-for-work images of 'healthy breakfast ideas'.", + "Search for videos about 'machine learning tutorials' and return the top 3 results.", + "Find recent videos on '2025 tech conferences' published in the last month.", + "Find coffee shops near 'Denver, CO' with ratings and reviews.", + "Search for top-rated vegan restaurants in 'San Francisco'.", + "Summarize the top web results for 'benefits of meditation' with inline references.", + "Generate a concise summary of news articles about 'renewable energy investments'." + ], + "tools": [ + "brave_web_search", + "brave_news_search", + "brave_image_search", + "brave_video_search", + "brave_local_search", + "brave_summarizer" + ] +} \ No newline at end of file diff --git a/src/data/chroma_cloud.MCP.json b/src/data/chroma_cloud.MCP.json new file mode 100644 index 0000000..db1c58a --- /dev/null +++ b/src/data/chroma_cloud.MCP.json @@ -0,0 +1,32 @@ +{ + "prompts": [ + "Create a new collection called 'test_collection' using the default embedding function.", + "Add the following documents to a collection named 'test_collection': 'This is the first test document.' (ID: doc1), 'This is the second test document.' (ID: doc2).", + "Retrieve all documents from the collection 'test_collection'.", + "Get information about the collection 'test_collection'.", + "Get the number of documents in the collection 'test_collection'.", + "Peek at the first 3 documents in the collection 'test_collection'.", + "Query the collection 'test_collection' for documents similar to 'test document' and return the top 2 results.", + "Update the document with ID 'doc2' in 'test_collection' to have the content: 'This is the updated second test document.'", + "Delete the document with ID 'doc1' from the collection 'test_collection'.", + "Fork the collection 'test_collection' into a new collection called 'test_collection_fork'.", + "Rename the collection 'test_collection' to 'test_collection_renamed'.", + "List all collections in Chroma Cloud.", + "Delete the collection named 'test_collection_renamed'." + ], + "tools": [ + "chroma_create_collection", + "chroma_add_documents", + "chroma_get_documents", + "chroma_get_collection_info", + "chroma_get_collection_count", + "chroma_peek_collection", + "chroma_query_documents", + "chroma_update_documents", + "chroma_delete_documents", + "chroma_fork_collection", + "chroma_modify_collection", + "chroma_list_collections", + "chroma_delete_collection" + ] +} \ No newline at end of file diff --git a/src/data/databrick_genie.MCP.json b/src/data/databrick_genie.MCP.json new file mode 100644 index 0000000..1accccc --- /dev/null +++ b/src/data/databrick_genie.MCP.json @@ -0,0 +1,20 @@ +{ + "prompts": [ + "Connect to the Databricks Genie Spaces MCP server.", + "List all available tools under Databricks Genie Spaces.", + "Attempt to fetch data without specifying any parameters.", + "What is the total sales for Q1 2024?", + "Give me insights on the trend of product sales over the last year.", + "How many new customers did we acquire in the last 3 months?", + "Which products had the highest sales last week in region A?", + "Run a serverless SQL query to fetch the top 10 customers by revenue.", + "Use serverless SQL to aggregate sales data by product category.", + "Run a complex serverless SQL query on a large dataset.", + "Show me the trend of website traffic for the last 6 months.", + "Give me the top 5 sales-performing regions over the past quarter.", + "Request data insights across multiple datasets concurrently.", + "Run multiple BI queries about different products and regions at the same time.", + "Search for product sales data across multiple years and regions." + ], + "tools": [] +} diff --git a/src/data/databrick_unity.MCP.json b/src/data/databrick_unity.MCP.json new file mode 100644 index 0000000..c0f42c3 --- /dev/null +++ b/src/data/databrick_unity.MCP.json @@ -0,0 +1,17 @@ +{ + "prompts": [ + "Is Databricks Unity Catalog Functions connected?", + "Execute the function get_user_activity from analytics_catalog.user_schema catalog with the following parameters: start_date='2025-10-01', end_date='2025-10-10'.", + "Execute the function current_timestamp from system.default catalog with no parameters.", + "Execute the function date_add from system.default catalog with the following parameters: start_date='2025-10-01', days=5.", + "Run the function get_total_sales from sales_catalog.monthly_reports with the parameter month='September'.", + "Execute the function get_user_activity from analytics_catalog.user_schema with the parameters start_date='invalid_date' and end_date='2025-10-10' to test error handling.", + "Run the function calculate_sales with an invalid parameter region_code='XYZ' that does not exist in the valid list.", + "Execute the function concat from system.default catalog with the following parameters: str1='Hello', str2='World'.", + "Execute the function non_existent_function from default.test_schema catalog to test function not found error.", + "Execute the function current_date from system.default catalog with invalid parameter test_param='xyz' to test parameter validation.", + "SELECT analytics_catalog.user_schema.get_user_activity('2025-10-01', '2025-10-10');", + "Run the SQL query: SELECT analytics_catalog.user_schema.get_user_activity('2025-10-01', '2025-10-10');" + ], + "tools": [] +} \ No newline at end of file diff --git a/src/data/databrick_vector.MCP.json b/src/data/databrick_vector.MCP.json new file mode 100644 index 0000000..7aeb54d --- /dev/null +++ b/src/data/databrick_vector.MCP.json @@ -0,0 +1,25 @@ +{ + "prompts": [ + "Connect to the Databricks Vector Search MCP server.", + "List all available tools under Databricks Vector Search.", + "Fetch data from Databricks using the vector model without specifying parameters.", + "Search for documents related to 'machine learning models'.", + "Perform semantic search using the vector embedding for the phrase 'neural networks in deep learning'.", + "Find similar items to the embedding for the phrase 'cloud computing infrastructure'.", + "Search for articles about 'distributed systems' and return results that are more relevant to 'data replication in databases'.", + "Find similar items to a non-existent or irrelevant embedding.", + "List all available vector search indexes in the system.", + "Create a new vector search index for the dataset 'customer reviews'.", + "Modify the index 'product_reviews' by adding a new field for 'review_rating'.", + "Delete the 'outdated_documents' index from the search database.", + "Create an index without specifying a dataset.", + "Use serverless compute to run a semantic search over the 'tech_education' dataset.", + "Check if the serverless compute is being utilized for this search operation.", + "Perform a search on a very large dataset that may result in a timeout.", + "Run the same semantic search query twice: 'data analytics tools for 2024'.", + "Search for content related to 'AWS EC2' from documents published after 2020.", + "Fetch results for 'cloud services' and include 'cloud_security' tag.", + "Perform a semantic search on the entire AWS documentation with a high query volume." + ], + "tools": [] +} diff --git a/src/data/datadog.MCP.json b/src/data/datadog.MCP.json new file mode 100644 index 0000000..c0076d7 --- /dev/null +++ b/src/data/datadog.MCP.json @@ -0,0 +1,51 @@ +{ + "prompts": [ + "Is Datadog tool connected?", + "List all dashboards in my Datadog account.", + "Fetch details for the first dashboard returned by the dashboard list.", + "Get the count of active hosts in Datadog.", + "List all hosts currently registered in Datadog.", + "Extract all unique service names from Datadog logs.", + "Query logs from the last 24 hours in Datadog.", + "Search for logs with status:error in Datadog.", + "List all incidents currently present in Datadog.", + "Fetch details for the most recent incident in Datadog.", + "List all monitors configured in Datadog.", + "List all monitors currently in alert state in Datadog.", + "Schedule a downtime for a test monitor in Datadog.", + "List all currently scheduled downtimes in Datadog.", + "Cancel a downtime by its ID in Datadog.", + "Mute a test host in Datadog.", + "Unmute the test host in Datadog.", + "List all RUM applications in Datadog.", + "Query recent RUM events in Datadog.", + "Group RUM events by application.name and count them in Datadog.", + "Get page performance metrics for RUM data in Datadog.", + "Get the RUM page waterfall for a given application and session in Datadog.", + "Query the system.cpu.user metric for the last hour in Datadog.", + "List recent APM traces for a test service in Datadog." + ], + "tools": [ + "cancel_downtime", + "get_active_hosts_count", + "get_all_services", + "get_dashboard", + "get_incident", + "get_logs", + "get_monitors", + "get_rum_applications", + "get_rum_events", + "get_rum_grouped_event_count", + "get_rum_page_performance", + "get_rum_page_waterfall", + "list_dashboards", + "list_downtimes", + "list_hosts", + "list_incidents", + "list_traces", + "mute_host", + "query_metrics", + "schedule_downtime", + "unmute_host" +] +} \ No newline at end of file diff --git a/src/data/deepwiki.MCP.json b/src/data/deepwiki.MCP.json new file mode 100644 index 0000000..4e892f9 --- /dev/null +++ b/src/data/deepwiki.MCP.json @@ -0,0 +1,13 @@ +{ + "prompts": [ + "Is DeepWiki MCP tool connected?", + "Show me the documentation structure for the GitHub repository facebook/react using read_wiki_structure.", + "Display the main documentation contents for the GitHub repository facebook/react using read_wiki_contents.", + "What is the purpose of the useState hook in the facebook/react repository using ask_question?" + ], + "tools": [ + "read_wiki_structure", + "read_wiki_contents", + "ask_question" + ] +} \ No newline at end of file diff --git a/src/data/exa_search.MCP.Json b/src/data/exa_search.MCP.Json new file mode 100644 index 0000000..51568b3 --- /dev/null +++ b/src/data/exa_search.MCP.Json @@ -0,0 +1,11 @@ +{ + "prompts": [ + "Is Exa Search tool connected?", + "Show me Python code examples for reading and writing JSON files using the built-in json module with get_code_context_exa.", + "What are the most important new features in React 19 as of November 2025 using web_search_exa?" + ], + "tools": [ + "get_code_context_exa", + "web_search_exa" + ] +} diff --git a/src/data/firecrawl.MCP.json b/src/data/firecrawl.MCP.json new file mode 100644 index 0000000..3faebfc --- /dev/null +++ b/src/data/firecrawl.MCP.json @@ -0,0 +1,20 @@ +{ + "prompts": [ + "Is Firecrawl tool connected?", + "Validate the connection to the Firecrawl service.", + "Search the web for the official homepage of 'Obot AI'.", + "Scrape the main heading (h1) from https://obot.ai/ in markdown format.", + "List the first 5 URLs found on https://obot.ai/.", + "Crawl https://obot.ai/ with a depth of 0 and a limit of 1 page.", + "Extract the page title and the first sentence from https://obot.ai/.", + "Check the status of the most recent crawl job." + ], + "tools": [ + "firecrawl_search", + "firecrawl_scrape", + "firecrawl_map", + "firecrawl_crawl", + "firecrawl_extract", + "firecrawl_check_crawl_status" + ] +} \ No newline at end of file diff --git a/src/data/gitmcp.MCP.json b/src/data/gitmcp.MCP.json new file mode 100644 index 0000000..bf20dce --- /dev/null +++ b/src/data/gitmcp.MCP.json @@ -0,0 +1,16 @@ +{ + "prompts": [ + "What is the owner and repo for the library 'react'?", + "Fetch the documentation for the repository 'facebook/react'.", + "Search the documentation in 'facebook/react' for the term 'hooks'.", + "Search the code in 'facebook/react' for the keyword 'useState'.", + "Fetch the content from this URL: https://raw.githubusercontent.com/facebook/react/main/README.md" + ], + "tools": [ + "gitmcpMatchCommonLibsOwnerRepoMapping", + "gitmcpFetchGenericDocumentation", + "gitmcpSearchGenericDocumentation", + "gitmcpSearchGenericCode", + "gitmcpFetchGenericUrlContent" + ] +} \ No newline at end of file diff --git a/src/data/postman.MCP.json b/src/data/postman.MCP.json new file mode 100644 index 0000000..f1e3480 --- /dev/null +++ b/src/data/postman.MCP.json @@ -0,0 +1,139 @@ +{ + "prompts": [ + "Create a new collection named 'Dummy Collection for Tool Testing' in workspace 1213ab29-47cc-4f5d-810c-0a307b54a9ce.", + "Add a comment 'This is a test comment' to the collection with ID fe28eb66-9a9d-4d48-821e-cbba279181e5.", + "Create a folder named 'Test Folder' in the collection with ID fe28eb66-9a9d-4d48-821e-cbba279181e5.", + "Fork the collection with ID fe28eb66-9a9d-4d48-821e-cbba279181e5 into workspace 1213ab29-47cc-4f5d-810c-0a307b54a9ce and label it 'Test Fork'.", + "Add a GET request named 'Test Request' to the collection with ID fe28eb66-9a9d-4d48-821e-cbba279181e5.", + "Add a response to the request with ID 85bb120b-5802-b5e8-a16c-ef48d25c1c42 in collection fe28eb66-9a9d-4d48-821e-cbba279181e5. (Note: Response ID creation may require manual setup)", + "Create a new environment named 'Test Environment' in workspace 1213ab29-47cc-4f5d-810c-0a307b54a9ce.", + "Add a comment 'Test folder comment' to folder a818298d-d548-21bd-eef6-d18f85f9811f in collection fe28eb66-9a9d-4d48-821e-cbba279181e5.", + "Create a mock server for collection 31616058-fe28eb66-9a9d-4d48-821e-cbba279181e5 in workspace 1213ab29-47cc-4f5d-810c-0a307b54a9ce.", + "Add a comment 'Test request comment' to request 85bb120b-5802-b5e8-a16c-ef48d-25c1c42 in collection fe28eb66-9a9d-4d48-821e-cbba279181e5.", + "Create a new OpenAPI 3.0 spec named 'Test Spec' in workspace 1213ab29-47cc-4f5d-810c-0a307b54a9ce.", + "Add a file named 'test.yaml' to spec 8d7ad57a-8d7b-4ad3-b1e0-df901d3903e7 with the content 'openapi: 3.0.0'.", + "Create a new workspace named 'Test Workspace' of type 'personal'.", + "Update the name of collection fe28eb66-9a9d-4d48-821e-cbba279181e5 to 'Updated Test Collection'.", + "Update the variable 'baseUrl' in environment a4fe7252-9078-4d08-8cb3-8a812468c78f to 'https://example.com'.", + "Run the monitor with ID 1f0be5c4-9f53-4240-8b8f-75bade8b5686 now.", + "Publish documentation for collection fe28eb66-9a9d-4d48-821e-cbba279181e5 with the default layout.", + "Unpublish documentation for collection fe28eb66-9a9d-4d48-821e-cbba279181e5.", + "Publish the mock server with ID c35efb81-cbcf-44b2-8602-eacf84ae877d.", + "Unpublish the mock server with ID c35efb81-cbcf-44b2-8602-eacf84ae877d.", + "Update the monitor 1f0be5c4-9f53-4240-8b8f-75bade8b5686 to run every 30 minutes.", + "Update the name of spec 8d7ad57a-8d7b-4ad3-b1e0-df901d3903e7 to 'Updated Spec Name'.", + "Update the file 'test.yaml' in spec 8d7ad57a-8d7b-4ad3-b1e0-df901d3903e7 with new content.", + "Update the name of workspace 1213ab29-47cc-4f5d-810c-0a307b54a9ce to 'Updated Workspace'.", + "Update tags for workspace 1213ab29-47cc-4f5d-810c-0a307b54a9ce to include 'test-tag'.", + "Update the global variable 'apiKey' in workspace 1213ab29-47cc-4f5d-810c-0a307b54a9ce to '12345'." + ], + "tools": [ + "createCollection", + "createCollectionComment", + "createCollectionFolder", + "createCollectionFork", + "createCollectionRequest", + "createCollectionResponse", + "createEnvironment", + "createFolderComment", + "createMock", + "createMonitor", + "createRequestComment", + "createResponseComment", + "createSpec", + "createSpecFile", + "createWorkspace", + "deleteApiCollectionComment", + "deleteCollection", + "deleteCollectionComment", + "deleteCollectionFolder", + "deleteCollectionRequest", + "deleteCollectionResponse", + "deleteEnvironment", + "deleteFolderComment", + "deleteMock", + "deleteMonitor", + "deletePanElementOrFolder", + "deleteRequestComment", + "deleteResponseComment", + "deleteSpec", + "deleteSpecFile", + "deleteWorkspace", + "duplicateCollection", + "generateCollection", + "generateSpecFromCollection", + "getAllElementsAndFolders", + "getAllPanAddElementRequests", + "getAllSpecs", + "getAsyncSpecTaskStatus", + "getAuthenticatedUser", + "getCollection", + "getCollectionComments", + "getCollectionFolder", + "getCollectionForks", + "getCollectionRequest", + "getCollectionResponse", + "getCollectionTags", + "getCollectionUpdatesTasks", + "getCollections", + "getCollectionsForkedByUser", + "getDuplicateCollectionTaskStatus", + "getEnvironment", + "getEnvironments", + "getFolderComments", + "getGeneratedCollectionSpecs", + "getMock", + "getMocks", + "getMonitor", + "getMonitors", + "getRequestComments", + "getResponseComments", + "getSourceCollectionStatus", + "getSpec", + "getSpecCollections", + "getSpecDefinition", + "getSpecFile", + "getSpecFiles", + "getStatusOfAnAsyncApiTask", + "getTaggedEntities", + "getWorkspace", + "getWorkspaceGlobalVariables", + "getWorkspaceTags", + "getWorkspaces", + "mergeCollectionFork", + "patchCollection", + "patchEnvironment", + "postPanElementOrFolder", + "publishDocumentation", + "publishMock", + "pullCollectionChanges", + "putCollection", + "putEnvironment", + "resolveCommentThread", + "runMonitor", + "syncCollectionWithSpec", + "syncSpecWithCollection", + "transferCollectionFolders", + "transferCollectionRequests", + "transferCollectionResponses", + "unpublishDocumentation", + "unpublishMock", + "updateApiCollectionComment", + "updateCollectionComment", + "updateCollectionFolder", + "updateCollectionRequest", + "updateCollectionResponse", + "updateCollectionTags", + "updateFolderComment", + "updateMock", + "updateMonitor", + "updatePanElementOrFolder", + "updateRequestComment", + "updateResponseComment", + "updateSpecFile", + "updateSpecProperties", + "updateWorkspace", + "updateWorkspaceGlobalVariables", + "updateWorkspaceTags" + ] +} \ No newline at end of file diff --git a/src/data/redis.MCP.json b/src/data/redis.MCP.json new file mode 100644 index 0000000..1c120d9 --- /dev/null +++ b/src/data/redis.MCP.json @@ -0,0 +1,21 @@ +{ + "prompts": [ + "Show me the list of connected Redis clients.", + "Create a vector similarity index named 'test_vector_index' on hashes with prefix 'testdoc:', vector field 'vector', dimension 4, and distance metric 'COSINE'. Then show the schema and info for the index, and list all indexes in the database.", + "How many keys are currently stored in the Redis database? Scan and return all keys matching the pattern 'test*' and scan for keys matching the pattern 'test*' (first batch).", + "Set the key 'testkey' to the value 'testvalue', get its value, set it to expire in 60 seconds, check its type, and then delete it.", + "Create hash 'testhash' with field 'field1' set to 'value1', check if 'field1' exists, get its value, get all fields and values, delete 'field1', and set it again.", + "Store the vector [0.1, 0.2, 0.3, 0.4] in the field 'vector' of hash 'testdoc:1', get the vector from hash 'testdoc:1', and search for the 2 nearest neighbors to the vector [0.1, 0.2, 0.3, 0.4] in index 'test_vector_index'.", + "Set the root JSON value of key 'testjson' to {\"foo\": \"bar\"}, get the root JSON value, and then delete it.", + "Push 'item1' onto the left and 'item2' onto the right of the list 'testlist', get all elements, get the length, remove and return the first and last elements.", + "Add 'member1' and 'member2' to the set 'testset', get all members, and remove 'member1'.", + "Add the member 'zmember1' with score 1.5 and 'zmember2' with score 2.5 to the sorted set 'testzset', get all members (with scores), and remove 'zmember1'.", + "Add two entries with field 'foo' and values 'bar' and 'baz' to the stream 'teststream', read the first 2 entries, and delete the entry with ID '123-0'.", + "Publish the message 'hello world' to the channel 'testchannel', subscribe and unsubscribe from the channel.", + "Set the key 'oldkey' to the value 'somevalue' and rename it to 'newkey'.", + "Show me the default Redis server info." + ], + "tools": [ + "client_list", "create_vector_index_hash", "get_index_info", "get_indexes", "dbsize", "scan_all_keys", "scan_keys", "set", "get", "expire", "type", "delete", "hset", "hexists", "hget", "hgetall", "hdel", "set_vector_in_hash", "get_vector_from_hash", "vector_search_hash", "json_set", "json_get", "json_del", "lpush", "rpush", "lrange", "llen", "lpop", "rpop", "sadd", "smembers", "srem", "zadd", "zrange", "zrem", "xadd", "xrange", "xdel", "publish", "subscribe", "unsubscribe", "rename", "info" + ] +} \ No newline at end of file diff --git a/src/data/tavily_search.MCP.json b/src/data/tavily_search.MCP.json new file mode 100644 index 0000000..e40dc1a --- /dev/null +++ b/src/data/tavily_search.MCP.json @@ -0,0 +1,15 @@ +{ + "prompts": [ + "Is Tavily Search tool connected?", + "Search for the latest advancements in AI using tavily_search.", + "Extract the main content from https://obot.ai/ using tavily_extract and summarize the result.", + "Crawl the first 3 pages of https://docs.python.org/3/ using tavily_crawl and summarize the topics.", + "Map the structure of https://developer.mozilla.org/ using tavily_map to list main documentation sections." + ], + "tools": [ + "tavily_search", + "tavily_extract", + "tavily_crawl", + "tavily_map" + ] +} \ No newline at end of file diff --git a/src/features/mcpServer.feature b/src/features/mcpServer.feature index de75661..afb9d3d 100644 --- a/src/features/mcpServer.feature +++ b/src/features/mcpServer.feature @@ -1,24 +1,34 @@ -Feature: Connect MCP server on Obot +Feature: Connect MCP servers on Obot Background: Navigate to Obot Given I setup context for assertion - When User navigates the Obot main login page - Then User open chat Obot + When User navigates to the Obot main login page + Then User opens chat Obot + And User creates a new Project with no existing connections - @wordpress - Scenario: Validate Wordpress sequential prompts on Obot - When User open MCP connector page - And User select "WordPress" MCP server - And User select "Connect To Server" button - And User connect to the WordPress MCP server - When User sends prompts to Obot AI chat for "Wordpress" MCP server - Then All prompts results should be validated and report generated for selected "Wordpress" MCP Server + Scenario Outline: Validate sequential prompts on Obot + When User opens the MCP connector page + And User selects "" MCP server + And User selects "Connect To Server" button + And User connects to the "" MCP server + When User sends prompts to Obot AI chat for "" MCP server + Then All prompt results should be validated and a report generated for the selected "" MCP server - @gitlab - Scenario: Validate GitLab sequential prompts on Obot - When User open MCP connector page - And User select "GitLab" MCP server - And User select "Connect To Server" button - And User connect to the GitLab MCP server - When User sends prompts to Obot AI chat for "Gitlab" MCP server - Then All prompts results should be validated and report generated for selected "Gitlab" MCP Server + Examples: + | ServerName | ConnectionName | PromptName | ReportName | + | WordPress | WordPress | WordPress | WordPress | + | GitLab | GitLab | GitLab | GitLab | + | BigQuery Toolbox | BigQuery | BigQuery Toolbox | BigQuery Toolbox | + # | Datadog | Datadog | Datadog | Datadog | + | Databricks Unity Catalog Functions | Databricks Unity Catalog Functions | Databrick Unity | Databrick Unity | + | Databricks Genie Spaces | Databricks Genie Space | Databrick Genie | Databrick Genie | + | Databricks Vector Search | Databricks Vector Space | Databrick Vector | Databrick Vector | + | Brave Search | Brave Search | Brave Search | Brave Search | + | Chroma Cloud | Chroma Cloud | Chroma Cloud | Chroma Cloud | + | Firecrawl | Firecrawl | Firecrawl | Firecrawl | + | GitMCP | GitMCP | GitMCP | GitMCP | + | Redis | Redis | Redis | Redis | + # | Postman | Postman | Postman | Postman | + | Tavily Search | Tavily Search | Tavily Search | Tavily Search | + | Exa Search | Exa Search | Exa Search | Exa Search | + | DeepWiki | DeepWiki | DeepWiki | DeepWiki | diff --git a/src/steps/mcpServer.step.ts b/src/steps/mcpServer.step.ts index 438dda6..b4b8590 100644 --- a/src/steps/mcpServer.step.ts +++ b/src/steps/mcpServer.step.ts @@ -1,43 +1,32 @@ import { When, Then, Given } from "@wdio/cucumber-framework"; import Selectors from "../core/selectors"; -import { - clickToElement, - isElementDisplayed, - slowInputFilling, -} from "../core/func"; -import { LONG_PAUSE, SHORT_PAUSE } from "../core/timeouts"; -import { - aggregateToolResponses, - saveMCPReport, - sendPromptValidateAndCollect, -} from "../core/mcpFunc"; -import path from "path"; -import { promises as fs } from "fs"; - -Given(/^User navigates the Obot main login page$/, async () => { - const url = process.env.OBOT_URL; - await browser.url(url); +import { clickToElement,isElementDisplayed,slowInputFilling} from "../core/func"; +import { LONG_PAUSE, MEDIUM_PAUSE, SHORT_PAUSE } from "../core/timeouts"; +import { aggregateToolResponses, saveMCPReport, sendPromptValidateAndCollect } from "../core/mcpFunc"; +import path from 'path'; +import { promises as fs } from 'fs'; +import { generateProjectName } from '../utils/projectName.model'; + +Given(/^User navigates to the Obot main login page$/, async() => { + const url = process.env.OBOT_URL ; + await browser.url(url); }); -Then(/^User open chat Obot$/, async () => { - await clickToElement(Selectors.MCP.navigationbtn); - await clickToElement(Selectors.MCP.clickChatObot); +Then(/^User opens chat Obot$/, async () => { + await clickToElement(Selectors.MCP.navigationbtn); + await clickToElement(Selectors.MCP.clickChatObot); }); -When(/^User open MCP connector page$/, async () => { - await clickToElement(Selectors.MCP.connectorbtn); +When(/^User opens the MCP connector page$/, async () => { + await clickToElement(Selectors.MCP.connectorbtn); }); -Then(/^User select "([^"]*)" MCP server$/, async (MCPServer) => { - await slowInputFilling(Selectors.MCP.mcpSearchInput, MCPServer); - await isElementDisplayed( - Selectors.MCP.selectMCPServer(MCPServer), - LONG_PAUSE, - ); - // Wait until matching elements appear - const allServers = await $$(Selectors.MCP.selectMCPServer(MCPServer)); - if ((await allServers.length) === 0) - throw new Error(`No MCP server found matching: ${MCPServer}`); +Then(/^User selects "([^"]*)" MCP server$/, async (MCPServer) => { + await slowInputFilling(Selectors.MCP.mcpSearchInput, MCPServer); + await isElementDisplayed(Selectors.MCP.selectMCPServer(MCPServer), LONG_PAUSE); + // Wait until matching elements appear + const allServers = await $$(Selectors.MCP.selectMCPServer(MCPServer)); + if (await allServers.length === 0) throw new Error(`No MCP server found matching: ${MCPServer}`); // Click the last one const lastServer = allServers[(await allServers.length) - 1]; @@ -47,43 +36,21 @@ Then(/^User select "([^"]*)" MCP server$/, async (MCPServer) => { await browser.pause(SHORT_PAUSE); }); -Then(/^User select "([^"]*)" button$/, async (Button) => { - await isElementDisplayed(Selectors.MCP.btnClick(Button), SHORT_PAUSE); - await clickToElement(Selectors.MCP.btnClick(Button)); -}); - -Then(/^User connect to the WordPress MCP server$/, async () => { - await slowInputFilling(Selectors.MCP.wpSiteURL, process.env.WP_URL); - await slowInputFilling(Selectors.MCP.wpUsername, process.env.WP_USERNAME); - await slowInputFilling(Selectors.MCP.wpPassword, process.env.WP_PASSWORD); - await clickToElement(Selectors.MCP.btnClick("Launch")); - await browser.pause(LONG_PAUSE * 2); +Then(/^User selects "([^"]*)" button$/, async (Button) => { + await isElementDisplayed(Selectors.MCP.btnClick(Button),SHORT_PAUSE); + await clickToElement(Selectors.MCP.btnClick(Button)); }); - + Then(/^User asks obot "([^"]*)"$/, async (prompt) => { await slowInputFilling(Selectors.MCP.obotInput, prompt); await clickToElement(Selectors.MCP.submitPrompt); await browser.pause(LONG_PAUSE); }); -Then(/^User connect to the GitLab MCP server$/, async () => { - await slowInputFilling(Selectors.MCP.gitlabToken, process.env.GITLAB_TOKEN); - await clickToElement(Selectors.MCP.btnClick("Launch")); - await browser.pause(LONG_PAUSE); -}); - -When( - /^User sends prompts to Obot AI chat for "([^"]*)" MCP server$/, - { timeout: 15 * 60 * 1000 }, - async function (serverName: string) { - const jsonPath = path.resolve( - process.cwd(), - "src", - "data", - `${serverName.toLowerCase()}.MCP.json`, - ); - const data = await fs.readFile(jsonPath, "utf-8"); - const { prompts, tools } = JSON.parse(data); +When(/^User sends prompts to Obot AI chat for "([^"]*)" MCP server$/, { timeout: 15 * 60 * 1000 }, async function(serverName: string) { + const jsonPath = path.resolve(process.cwd(), 'src', 'data', `${serverName.toLowerCase().replace(/\s+/g, '_')}.MCP.json`); + const data = await fs.readFile(jsonPath, 'utf-8'); + const { prompts, tools } = JSON.parse(data); this.promptResults = []; const toolList = tools; @@ -104,15 +71,115 @@ When( }, ); -Then( - /^All prompts results should be validated and report generated for selected "([^"]*)" MCP Server$/, - async function (serverName: string) { - const report = aggregateToolResponses(this.promptResults); - saveMCPReport(serverName, report); +Then(/^All prompt results should be validated and a report generated for the selected "(.*)" MCP server$/, async function(serverName: string) { + const report = aggregateToolResponses(this.promptResults); + saveMCPReport(serverName, report); - const errors = this.promptResults.filter((r) => r.error); - if (errors.length > 0) { - console.warn(`${errors.length} prompts had issues.`); - } - }, -); + const errors = this.promptResults.filter(r => r.error); + if (errors.length > 0) { + console.warn(`${errors.length} prompts had issues.`); + } +}); + +Then(/^User connects to the "(.*)" MCP server$/, async (mcpServer: string) => { + switch (mcpServer.toLowerCase()) { + case 'wordpress': + await slowInputFilling(Selectors.MCP.wordpressMCP.wpSiteURL, process.env.WP_URL); + await slowInputFilling(Selectors.MCP.wordpressMCP.wpUsername, process.env.WP_USERNAME); + await slowInputFilling(Selectors.MCP.wordpressMCP.wpPassword, process.env.WP_PASSWORD); + break; + + case 'gitlab': + await slowInputFilling(Selectors.MCP.gitlabMCP.gitlabToken, process.env.GITLAB_TOKEN); + break; + + case 'bigquery': + await slowInputFilling(Selectors.MCP.bigQuery.googleCloudProjectID, process.env.BQ_PROJECTID); + await $(Selectors.MCP.bigQuery.googleCloudCredentials).setValue(process.env.BQ_APP_CREDS); + break; + + case 'datadog': + await slowInputFilling(Selectors.MCP.datadog.datadogAPIKey, process.env.DD_API_KEY); + await slowInputFilling(Selectors.MCP.datadog.datadogAPPKey, process.env.DD_APP_KEY); + break; + + case 'databricks utility': + await slowInputFilling(Selectors.MCP.databricks.utility.workspaceHostname, process.env.DB_UTILITY_WORKSPACENAME); + await slowInputFilling(Selectors.MCP.databricks.utility.functionCatalog, process.env.DB_UTILITY_FUNCATALOG || 'workspace'); + await slowInputFilling(Selectors.MCP.databricks.utility.functionalSchema, process.env.DB_UTILITY_FUNSCHEMA || 'information_schema'); + await slowInputFilling(Selectors.MCP.databricks.utility.PAT, process.env.DB_UTILITY_PAT); + break; + + case 'databricks vector space': + await slowInputFilling(Selectors.MCP.databricks.utility.workspaceHostname, process.env.DB_UTILITY_WORKSPACENAME); + await slowInputFilling(Selectors.MCP.databricks.vector.vectorCatalog, process.env.DB_UTILITY_FUNCATALOG || 'workspace'); + await slowInputFilling(Selectors.MCP.databricks.vector.vectorSchema, process.env.DB_UTILITY_FUNSCHEMA || 'information_schema'); + await slowInputFilling(Selectors.MCP.databricks.utility.PAT, process.env.DB_UTILITY_PAT); + break; + + case 'databricks genie space': + await slowInputFilling(Selectors.MCP.databricks.utility.workspaceHostname, process.env.DB_UTILITY_WORKSPACENAME); + await slowInputFilling(Selectors.MCP.databricks.genie.genieSpaceID, process.env.DB_GENIE_ID); + await slowInputFilling(Selectors.MCP.databricks.utility.PAT, process.env.DB_UTILITY_PAT); + break; + + case 'brave search': + await slowInputFilling(Selectors.MCP.brave.braveAPIKey, process.env.BRAVE_API_KEY); + break; + + case 'chroma cloud': + await slowInputFilling(Selectors.MCP.chromaCloud.tenentID, process.env.CHROMA_TENENT_ID); + await slowInputFilling(Selectors.MCP.chromaCloud.DBName, process.env.CHROMA_DB_NAME || 'obot-test'); + await slowInputFilling(Selectors.MCP.chromaCloud.APIKey, process.env.CHROMA_API_KEY); + break; + + case 'firecrawl': + await slowInputFilling(Selectors.MCP.fireCrawl.API_key, process.env.FIRECRAWL_API_KEY); + break; + + case 'gitmcp': + await slowInputFilling(Selectors.MCP.gitMCP.urlLink, "https://gitmcp.io/docs"); + break; + + case 'redis': + await slowInputFilling(Selectors.MCP.redis.urlLink, process.env.REDIS_URL); + break; + + case 'tavily search': + await slowInputFilling('//input[@id="Tavily API Key"]', process.env.TAVILY_API_KEY); + break; + + case 'exa search': + await slowInputFilling('//input[@id="Exa API Key"]', process.env.TAVILY_API_KEY); + break; + + case 'postman': + await slowInputFilling(Selectors.MCP.postman.hostURL, 'https://mcp.postman.com'); + await slowInputFilling(Selectors.MCP.postman.toolCOnfig, 'mcp'); + await slowInputFilling(Selectors.MCP.postman.postmanAPIKey, process.env.POSTMAN_API_KEY); + break; + + case 'deepwiki': + break; + + default: + throw new Error(`Unknown MCP Server: ${mcpServer}`); + } + + await clickToElement(Selectors.MCP.btnClick('Launch')); + await browser.pause(LONG_PAUSE * 2); +}); + +When("User creates a new Project with no existing connections", async () => { + await browser.pause(MEDIUM_PAUSE); + const connectionElement = await $(Selectors.connectionsList).isDisplayed(); + const projectName = generateProjectName(); + if(connectionElement) { + await clickToElement(Selectors.currentProjectButton); + await clickToElement(Selectors.createNewProjectButton); + await slowInputFilling(Selectors.inputNewProjectName, projectName); + await clickToElement(Selectors.saveButton); + } + await browser.pause(MEDIUM_PAUSE); + await browser.refresh(); +}) diff --git a/src/utils/projectName.model.ts b/src/utils/projectName.model.ts new file mode 100644 index 0000000..c9a9023 --- /dev/null +++ b/src/utils/projectName.model.ts @@ -0,0 +1,12 @@ +import { faker } from '@faker-js/faker'; + +/** + * Generate a random project name + * Format: Automation-Obot-MCP-XXXXXX + */ +export function generateProjectName(): string { + const randomNumber = faker.number.int({ min: 100000, max: 999999 }); + return `Automation-Obot-MCP-${randomNumber}`; +} + +console.log(generateProjectName()); \ No newline at end of file diff --git a/wdio.conf.ts b/wdio.conf.ts index a6cf302..7396897 100644 --- a/wdio.conf.ts +++ b/wdio.conf.ts @@ -182,7 +182,7 @@ export const config: WebdriverIO.Config = { // add cucumber tags to feature or scenario name tagsInTitle: false, // timeout for step definitions - timeout: 120000, + timeout: 180000, }, ...hooks, }; From 38d5245df98eb79f7ff1f21b077dda1c25ad930c Mon Sep 17 00:00:00 2001 From: Shwetapsdet Date: Mon, 17 Nov 2025 14:13:09 -0700 Subject: [PATCH 7/7] Updated workflow file to tag all github secrets rather then configuring each --- .github/workflows/automation.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.github/workflows/automation.yml b/.github/workflows/automation.yml index f6e09c8..6bc5647 100644 --- a/.github/workflows/automation.yml +++ b/.github/workflows/automation.yml @@ -62,13 +62,7 @@ jobs: run: npm ci - name: Run WDIO Tests - env: - WP_URL: ${{ secrets.WP_URL }} - WP_USERNAME: ${{ secrets.WP_USERNAME }} - WP_PASSWORD: ${{ secrets.WP_PASSWORD }} - OBOT_URL: ${{ secrets.OBOT_URL }} - GITLAB_TOKEN: ${{ secrets.GITLAB_TOKEN }} - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY}} + env: ${{ secrets }} run: | npm run wdio:byScenario npm run eval