diff --git a/packages/core/package.json b/packages/core/package.json index 3ebaa115f..b182daed5 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -27,6 +27,10 @@ "build:types": "tsc --emitDeclarationOnly --declaration --outDir dist/types -p tsconfig.dts.json", "build:jsbundle": "cross-env rollup -c", "build": "pnpm run build:jsbundle && pnpm run build:types", + "test:unit": "cd ../.. && vitest run packages/core/tests/unit", + "test:unit:watch": "cd ../.. && vitest watch packages/core/tests/unit", + "test:integration": "cd ../.. && vitest run packages/core/tests/integration", + "test:integration:watch": "cd ../.. && vitest watch packages/core/tests/integration", "gen:docs": "typedoc", "doc:graphgen": "npx depcruise src --config ./doc/.dep-minimal.json --output-type dot > ./doc/dep-graph.dot && dot -Tpng ./doc/dep-graph.dot -o ./doc/dep-graph.png", "knip": "knip" diff --git a/packages/core/src/Components/ECMASandbox.class.ts b/packages/core/src/Components/ECMASandbox.class.ts index 1d13df98a..5773a0aac 100644 --- a/packages/core/src/Components/ECMASandbox.class.ts +++ b/packages/core/src/Components/ECMASandbox.class.ts @@ -38,14 +38,15 @@ export class ECMASandbox extends Component { } } - const inputVarsCode = this.generateInputVarCode(codeInputs); - const code = inputVarsCode + '\n' + config.data.code; + //const inputVarsCode = this.generateInputVarCode(codeInputs); + //const code = inputVarsCode + '\n' + config.data.code; + const code = config.data.code; logger.debug(`Running code: \n${code}\n`); const ecmaCodeConnector = ConnectorService.getCodeConnector('ECMASandbox'); - const executionResponse: CodeExecutionResult = await ecmaCodeConnector.agent(agent.id).execute(config.id, { code }); + const executionResponse: CodeExecutionResult = await ecmaCodeConnector.agent(agent.id).execute(config.id, { code, inputs: input }); if (executionResponse.success) { Output = executionResponse.output; } else { diff --git a/packages/core/src/helpers/AWSLambdaCode.helper.ts b/packages/core/src/helpers/AWSLambdaCode.helper.ts index a9ca2756e..e40e867f5 100644 --- a/packages/core/src/helpers/AWSLambdaCode.helper.ts +++ b/packages/core/src/helpers/AWSLambdaCode.helper.ts @@ -2,7 +2,17 @@ import crypto from 'crypto'; import { ConnectorService } from '@sre/Core/ConnectorsService'; import { AccessCandidate } from '@sre/Security/AccessControl/AccessCandidate.class'; import zl from 'zip-lib'; -import { InvokeCommand, Runtime, LambdaClient, UpdateFunctionCodeCommand, CreateFunctionCommand, GetFunctionCommand, GetFunctionCommandOutput, InvokeCommandOutput, UpdateFunctionConfigurationCommand } from '@aws-sdk/client-lambda'; +import { + InvokeCommand, + Runtime, + LambdaClient, + UpdateFunctionCodeCommand, + CreateFunctionCommand, + GetFunctionCommand, + GetFunctionCommandOutput, + InvokeCommandOutput, + UpdateFunctionConfigurationCommand, +} from '@aws-sdk/client-lambda'; import { GetRoleCommand, CreateRoleCommand, IAMClient, GetRoleCommandOutput, CreateRoleCommandOutput } from '@aws-sdk/client-iam'; import fs from 'fs'; import { AWSConfig, AWSCredentials, AWSRegionConfig } from '@sre/types/AWS.types'; @@ -19,7 +29,6 @@ export function getLambdaFunctionName(agentId: string, componentId: string) { return `${agentId}-${componentId}`; } - export function generateCodeHash(code_body: string, codeInputs: string[], envVariables: string[]) { const bodyHash = getSanitizeCodeHash(code_body); const inputsHash = getSanitizeCodeHash(JSON.stringify(codeInputs)); @@ -79,9 +88,7 @@ export async function getDeployedCodeHash(agentId: string, componentId: string) export async function setDeployedCodeHash(agentId: string, componentId: string, codeHash: string) { const redisCache = ConnectorService.getCacheConnector(); - await redisCache - .user(AccessCandidate.agent(agentId)) - .set(`${cachePrefix}_${agentId}-${componentId}`, codeHash, null, null, cacheTTL); + await redisCache.user(AccessCandidate.agent(agentId)).set(`${cachePrefix}_${agentId}-${componentId}`, codeHash, null, null, cacheTTL); } function replaceVaultKeysTemplateVars(code: string, envVariables: Record) { @@ -121,7 +128,7 @@ export async function zipCode(directory: string) { }, function (err) { reject(err); - }, + } ); }); } @@ -276,7 +283,6 @@ export function getLambdaRolePolicy() { }); } - export async function updateDeployedCodeTTL(agentId: string, componentId: string, ttl: number) { const redisCache = ConnectorService.getCacheConnector(); await redisCache.user(AccessCandidate.agent(agentId)).updateTTL(`${cachePrefix}_${agentId}-${componentId}`, ttl); @@ -285,7 +291,7 @@ export async function updateDeployedCodeTTL(agentId: string, componentId: string export async function invokeLambdaFunction( functionName: string, inputs: { [key: string]: any }, - awsCredentials: AWSCredentials & AWSRegionConfig, + awsCredentials: AWSCredentials & AWSRegionConfig ): Promise { try { const client = new LambdaClient({ @@ -384,11 +390,11 @@ export function reportUsage({ cost, agentId, teamId }: { cost: number; agentId: export function validateAsyncMainFunction(rawCode: string): { isValid: boolean; error?: string; parameters?: string[]; dependencies?: string[] } { try { - const code = replaceVaultKeysTemplateVars(rawCode, {}); + const code = replaceVaultKeysTemplateVars(rawCode.trim(), {}); // Parse the code using acorn const ast = acorn.parse(code, { ecmaVersion: 'latest', - sourceType: 'module' + sourceType: 'module', }); // Extract library imports @@ -414,11 +420,13 @@ export function validateAsyncMainFunction(rawCode: string): { isValid: boolean; } // Handle CallExpression (require() calls) - if (node.type === 'CallExpression' && + if ( + node.type === 'CallExpression' && node.callee.type === 'Identifier' && node.callee.name === 'require' && node.arguments.length > 0 && - node.arguments[0].type === 'Literal') { + node.arguments[0].type === 'Literal' + ) { const modulePath = node.arguments[0].value; if (modulePath && !modulePath.startsWith('.') && !modulePath.startsWith('/')) { libraries.add(extractPackageName(modulePath)); @@ -426,10 +434,12 @@ export function validateAsyncMainFunction(rawCode: string): { isValid: boolean; } // Handle dynamic import() calls - if (node.type === 'CallExpression' && + if ( + node.type === 'CallExpression' && node.callee.type === 'Import' && node.arguments.length > 0 && - node.arguments[0].type === 'Literal') { + node.arguments[0].type === 'Literal' + ) { const modulePath = node.arguments[0].value; if (modulePath && !modulePath.startsWith('.') && !modulePath.startsWith('/')) { libraries.add(extractPackageName(modulePath)); @@ -503,7 +513,7 @@ export function validateAsyncMainFunction(rawCode: string): { isValid: boolean; return { isValid: false, error: 'No main function found at root level', - dependencies + dependencies, }; } @@ -511,7 +521,7 @@ export function validateAsyncMainFunction(rawCode: string): { isValid: boolean; return { isValid: false, error: 'Main function exists but is not async', - dependencies + dependencies, }; } @@ -519,7 +529,7 @@ export function validateAsyncMainFunction(rawCode: string): { isValid: boolean; } catch (error) { return { isValid: false, - error: `Failed to parse code: ${error.message}` + error: `Failed to parse code: ${error.message}`, }; } } @@ -549,7 +559,7 @@ export function generateCodeFromLegacyComponent(code_body: string, code_imports: async function main(${codeInputs.join(', ')}) { ${code_body} } - ` + `; return code; } @@ -565,14 +575,12 @@ export function extractAllKeyNamesFromTemplateVars(input: string): string[] { return matches; } - -async function fetchVaultSecret(keyName: string, agentTeamId: string): Promise<{ value: string, key: string }> { +async function fetchVaultSecret(keyName: string, agentTeamId: string): Promise<{ value: string; key: string }> { const vaultSecret = await VaultHelper.getAgentKey(keyName, agentTeamId); return { value: vaultSecret, key: keyName, }; - } export async function getCurrentEnvironmentVariables(agentTeamId: string, code: string): Promise> { @@ -588,4 +596,4 @@ export async function getCurrentEnvironmentVariables(agentTeamId: string, code: export function getSortedObjectValues(obj: Record): string[] { const sortedKeys = Object.keys(obj).sort(); return sortedKeys.map((key) => obj[key]); -} \ No newline at end of file +} diff --git a/packages/core/src/helpers/Conversation.helper.ts b/packages/core/src/helpers/Conversation.helper.ts index 30d050da1..9171c80cf 100644 --- a/packages/core/src/helpers/Conversation.helper.ts +++ b/packages/core/src/helpers/Conversation.helper.ts @@ -132,9 +132,9 @@ export class Conversation extends EventEmitter { console.warn('Conversation Error: ', error?.message); }); this._maxContextSize = - _settings.maxContextSize || (this._model as TLLMModel).tokens || (this._model as TLLMModel).keyOptions?.tokens || this._maxContextSize; + _settings?.maxContextSize || (this._model as TLLMModel).tokens || (this._model as TLLMModel).keyOptions?.tokens || this._maxContextSize; this._maxOutputTokens = - _settings.maxOutputTokens || + _settings?.maxOutputTokens || (this._model as TLLMModel).completionTokens || (this._model as TLLMModel).keyOptions?.completionTokens || this._maxOutputTokens; @@ -928,6 +928,7 @@ export class Conversation extends EventEmitter { //is this a valid agent data? if (typeof specSource?.behavior === 'string' && specSource?.components && specSource?.connections) { this.agentData = specSource; //agent loaded from data directly + this._agentId = specSource.id; return await this.loadSpecFromAgent(specSource); } diff --git a/packages/core/src/helpers/ECMASandbox.helper.ts b/packages/core/src/helpers/ECMASandbox.helper.ts index c8eb389fd..a78e023c5 100644 --- a/packages/core/src/helpers/ECMASandbox.helper.ts +++ b/packages/core/src/helpers/ECMASandbox.helper.ts @@ -1,11 +1,22 @@ import 'ses'; - - - export function runJs(code: string) { - // Call lockdown to secure the environment - lockdown(); + // Ensure SES lockdown happens only once per process + // and tolerate environments already locked down elsewhere + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const globalAny: any = globalThis as any; + if (globalAny.__SRE_SES_LOCKED_DOWN__ !== true) { + try { + lockdown(); + } catch (err) { + const msg = String(err || ''); + if (!(msg.includes('Already locked down') || msg.includes('SES_ALREADY_LOCKED_DOWN'))) { + throw err; + } + } finally { + globalAny.__SRE_SES_LOCKED_DOWN__ = true; + } + } try { // Endow the compartment with necessary APIs const compartment = new Compartment({ @@ -27,7 +38,6 @@ export function runJs(code: string) { console.error(error); throw error; } - } function getParametersString(parameters: string[], inputs: Record) { @@ -49,6 +59,6 @@ export function generateExecutableCode(code: string, parameters: string[], input const result = await main(${getParametersString(parameters, inputs)}); return result; })(); - ` + `; return executableCode; } diff --git a/packages/core/src/subsystems/AgentManager/AgentData.service/connectors/LocalAgentDataConnector.class.ts b/packages/core/src/subsystems/AgentManager/AgentData.service/connectors/LocalAgentDataConnector.class.ts index d9d132183..3cbf02192 100644 --- a/packages/core/src/subsystems/AgentManager/AgentData.service/connectors/LocalAgentDataConnector.class.ts +++ b/packages/core/src/subsystems/AgentManager/AgentData.service/connectors/LocalAgentDataConnector.class.ts @@ -2,6 +2,9 @@ import fs from 'fs'; import path from 'path'; import { AgentDataConnector } from '../AgentDataConnector'; import { uid } from '@sre/utils/general.utils'; +import { Logger } from '@sre/helpers/Log.helper'; + +const console = Logger('LocalAgentDataConnector'); export type LocalAgentDataSettings = { devDir: string; prodDir: string }; diff --git a/packages/core/src/subsystems/IO/VectorDB.service/connectors/RAMVecrtorDB.class.ts b/packages/core/src/subsystems/IO/VectorDB.service/connectors/RAMVecrtorDB.class.ts index f32366910..8c0349726 100644 --- a/packages/core/src/subsystems/IO/VectorDB.service/connectors/RAMVecrtorDB.class.ts +++ b/packages/core/src/subsystems/IO/VectorDB.service/connectors/RAMVecrtorDB.class.ts @@ -5,7 +5,7 @@ import { IAccessCandidate, IACL, TAccessLevel } from '@sre/types/ACL.types'; import { AccessRequest } from '@sre/Security/AccessControl/AccessRequest.class'; import { AccessCandidate } from '@sre/Security/AccessControl/AccessCandidate.class'; import { SecureConnector } from '@sre/Security/SecureConnector.class'; -import { VectorDBConnector } from '../VectorDBConnector'; +import { VectorDBConnector, DeleteTarget } from '../VectorDBConnector'; import { DatasourceDto, IStorageVectorDataSource, @@ -21,6 +21,8 @@ import { OpenAIEmbeds } from '@sre/IO/VectorDB.service/embed/OpenAIEmbedding'; import crypto from 'crypto'; import { BaseEmbedding, TEmbeddings } from '../embed/BaseEmbedding'; import { EmbeddingsFactory } from '../embed'; +import { chunkText } from '@sre/utils/string.utils'; +import { jsonrepair } from 'jsonrepair'; const console = Logger('RAM VectorDB'); @@ -66,6 +68,7 @@ export class RAMVectorDB extends VectorDBConnector { if (!_settings.embeddings) { _settings.embeddings = { provider: 'OpenAI', model: 'text-embedding-3-large', params: { dimensions: 1024 } }; } + if (!_settings.embeddings.params) _settings.embeddings.params = { dimensions: 1024 }; if (!_settings.embeddings.params?.dimensions) _settings.embeddings.params.dimensions = 1024; this.embedder = EmbeddingsFactory.create(_settings.embeddings.provider, _settings.embeddings); @@ -182,12 +185,26 @@ export class RAMVectorDB extends VectorDBConnector { for (const vector of namespaceData) { const similarity = this.cosineSimilarity(queryVector as number[], vector.values); + + let userMetadata = undefined; + if (options.includeMetadata) { + if (vector.metadata?.[this.USER_METADATA_KEY]) { + try { + userMetadata = JSON.parse(vector.metadata[this.USER_METADATA_KEY]); + } catch { + userMetadata = vector.metadata[this.USER_METADATA_KEY]; + } + } else { + userMetadata = {}; // Return empty object when no metadata exists, like Milvus + } + } + results.push({ id: vector.id, score: similarity, values: vector.values, - metadata: options.includeMetadata ? vector.metadata : undefined, - text: vector.metadata?.text, + text: vector.metadata?.text as string | undefined, + metadata: options.includeMetadata ? userMetadata : undefined, }); } @@ -207,28 +224,28 @@ export class RAMVectorDB extends VectorDBConnector { //const teamId = await this.accountConnector.getCandidateTeam(acRequest.candidate); const preparedNs = this.constructNsName(acRequest.candidate as AccessCandidate, namespace); - const sources = Array.isArray(sourceWrapper) ? sourceWrapper : [sourceWrapper]; - const insertedIds: string[] = []; + sourceWrapper = Array.isArray(sourceWrapper) ? sourceWrapper : [sourceWrapper]; - if (!this.vectors[preparedNs]) { - this.vectors[preparedNs] = []; + // make sure that all sources are of the same type (source.source) + if (sourceWrapper.some((s) => this.embedder.detectSourceType(s.source) !== this.embedder.detectSourceType(sourceWrapper[0].source))) { + throw new Error('All sources must be of the same type'); } - for (const source of sources) { - let vector: number[] = []; + const sourceType = this.embedder.detectSourceType(sourceWrapper[0].source); + if (sourceType === 'unknown' || sourceType === 'url') throw new Error('Invalid source type'); - if (typeof source.source === 'string') { - // Text embedding + const transformedSource = await this.embedder.transformSource(sourceWrapper, sourceType, acRequest.candidate as AccessCandidate); - vector = await this.embedder.embedText(source.source, acRequest.candidate as AccessCandidate); - } else { - // Direct vector - vector = source.source; - } + if (!this.vectors[preparedNs]) { + this.vectors[preparedNs] = []; + } + + const insertedIds: string[] = []; + for (const source of transformedSource) { const vectorData: VectorData = { id: source.id, - values: vector, + values: source.source as number[], datasource: source.metadata?.datasourceId || 'unknown', metadata: source.metadata, }; @@ -248,107 +265,96 @@ export class RAMVectorDB extends VectorDBConnector { } @SecureConnector.AccessControl - protected async delete(acRequest: AccessRequest, namespace: string, id: string | string[]): Promise { + protected async delete(acRequest: AccessRequest, namespace: string, deleteTarget: DeleteTarget): Promise { //const teamId = await this.accountConnector.getCandidateTeam(acRequest.candidate); const preparedNs = this.constructNsName(acRequest.candidate as AccessCandidate, namespace); - const ids = Array.isArray(id) ? id : [id]; + const isDeleteByFilter = typeof deleteTarget === 'object' && !Array.isArray(deleteTarget); - if (this.vectors[preparedNs]) { - this.vectors[preparedNs] = this.vectors[preparedNs].filter((vector) => !ids.includes(vector.id)); + if (isDeleteByFilter) { + // Handle delete by filter (e.g., by datasourceId) + if ('datasourceId' in deleteTarget && deleteTarget.datasourceId) { + if (this.vectors[preparedNs]) { + this.vectors[preparedNs] = this.vectors[preparedNs].filter((vector) => vector.datasource !== deleteTarget.datasourceId); + } + } else { + throw new Error('Unsupported delete filter'); + } + } else { + // Handle delete by ID(s) + const ids = Array.isArray(deleteTarget) ? deleteTarget : [deleteTarget]; + if (this.vectors[preparedNs]) { + this.vectors[preparedNs] = this.vectors[preparedNs].filter((vector) => !ids.includes(vector.id)); + } } } @SecureConnector.AccessControl protected async createDatasource(acRequest: AccessRequest, namespace: string, datasource: DatasourceDto): Promise { //const teamId = await this.accountConnector.getCandidateTeam(acRequest.candidate); - const preparedNs = this.constructNsName(acRequest.candidate as AccessCandidate, namespace); - const datasourceId = datasource.id || crypto.randomUUID(); - - // Ensure namespace exists - if (!this.namespaces[preparedNs]) { - await this.createNamespace(acRequest, namespace); - } - - // Process text and create vectors - const vectorIds: string[] = []; - - // Split text into chunks if needed - const chunkSize = datasource.chunkSize || 1000; - const chunkOverlap = datasource.chunkOverlap || 200; - const chunks = this.splitTextIntoChunks(datasource.text, chunkSize, chunkOverlap); - - // Initialize namespace vectors if not exists (should already exist if namespace was created properly) - if (!this.vectors[preparedNs]) { - this.vectors[preparedNs] = []; - } - - for (let i = 0; i < chunks.length; i++) { - const chunkId = `${datasourceId}_chunk_${i}`; - const vector = await this.embedder.embedText(chunks[i], acRequest.candidate as AccessCandidate); - - const vectorData: VectorData = { - id: chunkId, - values: vector, - datasource: datasourceId, + const acl = new ACL().addAccess(acRequest.candidate.role, acRequest.candidate.id, TAccessLevel.Owner); + const dsId = datasource.id || crypto.randomUUID(); + + const formattedNs = this.constructNsName(acRequest.candidate as AccessCandidate, namespace); + const chunkedText = chunkText(datasource.text, { + chunkSize: datasource.chunkSize, + chunkOverlap: datasource.chunkOverlap, + }); + const label = datasource.label || 'Untitled'; + const ids = Array.from({ length: chunkedText.length }, (_, i) => `${dsId}_${crypto.randomUUID()}`); + const source: IVectorDataSourceDto[] = chunkedText.map((doc, i) => { + return { + id: ids[i], + source: doc, metadata: { - ...datasource.metadata, - text: chunks[i], - chunkIndex: i, - totalChunks: chunks.length, + acl: acl.serializedACL, + namespaceId: formattedNs, + datasourceId: dsId, + datasourceLabel: label, + [this.USER_METADATA_KEY]: datasource.metadata ? jsonrepair(JSON.stringify(datasource.metadata)) : undefined, }, }; + }); - this.vectors[preparedNs].push(vectorData); - vectorIds.push(chunkId); - } + const _vIds = await this.insert(acRequest, namespace, source); - const storageDataSource: IStorageVectorDataSource = { - namespaceId: preparedNs, + const dsData: IStorageVectorDataSource = { + namespaceId: formattedNs, candidateId: acRequest.candidate.id, candidateRole: acRequest.candidate.role, - name: datasource.label || `Datasource ${datasourceId}`, - metadata: JSON.stringify(datasource.metadata || {}), + name: datasource.label || 'Untitled', + metadata: datasource.metadata ? jsonrepair(JSON.stringify(datasource.metadata)) : undefined, text: datasource.text, - vectorIds, - id: datasourceId, + vectorIds: _vIds, + id: dsId, }; // Store datasource metadata in memory - if (!this.datasources[preparedNs]) { - this.datasources[preparedNs] = {}; - } - if (!this.datasources[preparedNs][datasourceId]) { - this.datasources[preparedNs][datasourceId] = storageDataSource; - } else { - this.datasources[preparedNs][datasourceId].vectorIds.push(...vectorIds); + if (!this.datasources[formattedNs]) { + this.datasources[formattedNs] = {}; } + this.datasources[formattedNs][dsId] = dsData; - return storageDataSource; + return dsData; } @SecureConnector.AccessControl protected async deleteDatasource(acRequest: AccessRequest, namespace: string, datasourceId: string): Promise { //const teamId = await this.accountConnector.getCandidateTeam(acRequest.candidate); - const preparedNs = this.constructNsName(acRequest.candidate as AccessCandidate, namespace); + const formattedNs = this.constructNsName(acRequest.candidate as AccessCandidate, namespace); - // Ensure namespace exists - if (!this.namespaces[preparedNs]) { - throw new Error('Namespace does not exist'); + // Get datasource info to get vector IDs + const ds = this.datasources[formattedNs]?.[datasourceId]; + if (!ds) { + throw new Error(`Data source not found with id: ${datasourceId}`); } - // Get datasource info - const datasource = this.datasources[preparedNs]?.[datasourceId]; - if (datasource) { - // Delete all vectors belonging to this datasource - if (this.vectors[preparedNs]) { - this.vectors[preparedNs] = this.vectors[preparedNs].filter((vector) => vector.datasource !== datasourceId); - } - } + // Delete all vectors belonging to this datasource using the delete method + await this.delete(acRequest, namespace, ds.vectorIds || []); // Delete datasource metadata - if (this.datasources[preparedNs]) { - delete this.datasources[preparedNs][datasourceId]; + if (this.datasources[formattedNs]) { + delete this.datasources[formattedNs][datasourceId]; } } @@ -362,15 +368,12 @@ export class RAMVectorDB extends VectorDBConnector { } @SecureConnector.AccessControl - protected async getDatasource(acRequest: AccessRequest, namespace: string, datasourceId: string): Promise { + protected async getDatasource(acRequest: AccessRequest, namespace: string, datasourceId: string): Promise { //const teamId = await this.accountConnector.getCandidateTeam(acRequest.candidate); const preparedNs = this.constructNsName(acRequest.candidate as AccessCandidate, namespace); const datasource = this.datasources[preparedNs]?.[datasourceId]; - if (!datasource) { - throw new Error(`Datasource ${datasourceId} not found`); - } - return datasource; + return datasource; // Return undefined if not found, like MilvusVectorDB } /** @@ -400,22 +403,4 @@ export class RAMVectorDB extends VectorDBConnector { return dotProduct / (normA * normB); } - - /** - * Split text into chunks with overlap - */ - private splitTextIntoChunks(text: string, chunkSize: number, overlap: number): string[] { - const chunks: string[] = []; - let start = 0; - - while (start < text.length) { - const end = Math.min(start + chunkSize, text.length); - chunks.push(text.slice(start, end)); - - if (end === text.length) break; - start = end - overlap; - } - - return chunks; - } } diff --git a/packages/core/src/subsystems/IO/VectorDB.service/embed/GoogleEmbedding.ts b/packages/core/src/subsystems/IO/VectorDB.service/embed/GoogleEmbedding.ts index 6dccbe943..175dd28c2 100644 --- a/packages/core/src/subsystems/IO/VectorDB.service/embed/GoogleEmbedding.ts +++ b/packages/core/src/subsystems/IO/VectorDB.service/embed/GoogleEmbedding.ts @@ -50,7 +50,7 @@ export class GoogleEmbeds extends BaseEmbedding { const modelInfo: TLLMModel = { provider: 'GoogleAI', modelId: this.model, - credentials: this.settings?.credentials as unknown as TLLMCredentials, + credentials: (this.settings?.credentials as unknown as TLLMCredentials) || [TLLMCredentials.Internal, TLLMCredentials.Vault], }; const credentials = await getLLMCredentials(candidate, modelInfo); apiKey = (credentials as BasicCredentials)?.apiKey; diff --git a/packages/core/src/subsystems/IO/VectorDB.service/embed/OpenAIEmbedding.ts b/packages/core/src/subsystems/IO/VectorDB.service/embed/OpenAIEmbedding.ts index 482653cca..fcba13c32 100644 --- a/packages/core/src/subsystems/IO/VectorDB.service/embed/OpenAIEmbedding.ts +++ b/packages/core/src/subsystems/IO/VectorDB.service/embed/OpenAIEmbedding.ts @@ -82,7 +82,7 @@ export class OpenAIEmbeds extends BaseEmbedding { const modelInfo: TLLMModel = { provider: 'OpenAI', modelId: this.model, - credentials: this.settings?.credentials as unknown as TLLMCredentials, + credentials: (this.settings?.credentials as unknown as TLLMCredentials) || [TLLMCredentials.Internal, TLLMCredentials.Vault], }; const credentials = await getLLMCredentials(candidate, modelInfo); diff --git a/packages/core/src/subsystems/LLMManager/LLM.inference.ts b/packages/core/src/subsystems/LLMManager/LLM.inference.ts index b70866fb7..043917481 100644 --- a/packages/core/src/subsystems/LLMManager/LLM.inference.ts +++ b/packages/core/src/subsystems/LLMManager/LLM.inference.ts @@ -127,6 +127,7 @@ export class LLMInference { return this.llmConnector.user(AccessCandidate.agent(params.agentId)).imageEditRequest(params); } + //@deprecated public async streamRequest(params: any, agent: string | IAgent) { const agentId = isAgent(agent) ? (agent as IAgent).id : agent; try { @@ -149,6 +150,7 @@ export class LLMInference { } } + //@deprecated public async multimodalStreamRequest(params: any, fileSources, agent: string | IAgent) { const agentId = isAgent(agent) ? (agent as IAgent).id : agent; @@ -180,6 +182,7 @@ export class LLMInference { } } + //@deprecated public async multimodalStreamRequestLegacy(prompt, files: string[], config: any = {}, agent: string | IAgent) { const agentId = isAgent(agent) ? (agent as IAgent).id : agent; diff --git a/packages/core/src/subsystems/LLMManager/models.ts b/packages/core/src/subsystems/LLMManager/models.ts index 1df04d2d1..856dcf4f8 100644 --- a/packages/core/src/subsystems/LLMManager/models.ts +++ b/packages/core/src/subsystems/LLMManager/models.ts @@ -68,11 +68,11 @@ export const models = { // keep the gpt-4o-mini as default model for now 'gpt-4o-mini': { llm: 'OpenAI', - alias: 'gpt-4o-mini-2024-07-18', + components: ['PromptGenerator', 'LLMAssistant', 'Classifier', 'VisionLLM', 'AgentPlugin', 'Chatbot', 'GPTPlugin', 'GenAILLM'], label: 'GPT 4o Mini', - modelId: 'gpt-4o-mini-2024-07-18', + modelId: 'gpt-4o-mini', provider: 'OpenAI', features: ['text', 'tools', 'image', 'search'], tags: ['Personal'], diff --git a/packages/core/src/subsystems/Security/Account.service/connectors/JSONFileAccount.class.ts b/packages/core/src/subsystems/Security/Account.service/connectors/JSONFileAccount.class.ts index c5cb2275c..4bd2d59de 100644 --- a/packages/core/src/subsystems/Security/Account.service/connectors/JSONFileAccount.class.ts +++ b/packages/core/src/subsystems/Security/Account.service/connectors/JSONFileAccount.class.ts @@ -74,8 +74,14 @@ export class JSONFileAccount extends AccountConnector { } public async isTeamMember(team: string, candidate: IAccessCandidate): Promise { + if (team === DEFAULT_TEAM_ID) return true; //everyone is a member of the default team + if (!this.data[team]) return false; + if (candidate.role === TAccessRole.Team && team === candidate.id) { + return true; + } + if (candidate.role === TAccessRole.User) { return !!this.data[team].users?.[candidate.id]; } else if (candidate.role === TAccessRole.Agent) { diff --git a/packages/core/src/types/LLM.types.ts b/packages/core/src/types/LLM.types.ts index f5a844955..c432307b7 100644 --- a/packages/core/src/types/LLM.types.ts +++ b/packages/core/src/types/LLM.types.ts @@ -221,7 +221,7 @@ export type TLLMModel = { tokens: number; completionTokens: number; }; - credentials?: TLLMCredentials; + credentials?: TLLMCredentials | TLLMCredentials[]; //models can come with predefined params //this can also be used to pass a preconfigured model object diff --git a/packages/core/tests/README.md b/packages/core/tests/README.md new file mode 100644 index 000000000..193a45268 --- /dev/null +++ b/packages/core/tests/README.md @@ -0,0 +1,70 @@ +# How to run the tests + +We have two types of tests here: + +- unit tests +- integration tests + +# Unit Tests + +The unit tests are located in the `tests/unit` folder and do not rely on external services (except APICall cases that use httpbin.org). + +Run from `packages/core`: + +```bash +pnpm run test:unit +pnpm run test:unit:watch +``` + +If you are an external contributor, please run unit tests before and after your changes to ensure nothing breaks. + +# Integration Tests + +The integration tests live in `tests/integration` and rely on external services (LLM providers, storage, etc.). + +**⚠️ Important: Integration tests require a Vault file for API keys before they can run.** + +## Vault setup (required for integration tests) + +Create a Vault file in your user home directory: + +- macOS/Linux: `~/.smyth/.sre/vault.json` +- Windows: `%USERPROFILE%\.smyth\.sre\vault.json` + +Example `vault.json` (you can reference environment variables using `$env(VAR_NAME)`): + +```json +{ + "default": { + "echo": "", + "openai": "...", + "anthropic": "...", + "googleai": "...", + "groq": "...", + "togetherai": "...", + "xai": "...", + "perplexity": "..." + } +} +``` + +## Running integration tests + +Once your vault is configured, run from the repository root: + +```bash +pnpm -F @smythos/sre test:integration +pnpm -F @smythos/sre test:integration:watch +``` + +Or from `packages/core`: + +```bash +pnpm run test:integration +pnpm run test:integration:watch +``` + +# Important Notes + +- Running LLM integration tests may incur costs with your providers. +- Always use pnpm in this monorepo. diff --git a/packages/core/tests/data/AgentData/async-await-foreach-tests.smyth b/packages/core/tests/data/AgentData/async-await-foreach-tests.smyth new file mode 100644 index 000000000..48d5f59e3 --- /dev/null +++ b/packages/core/tests/data/AgentData/async-await-foreach-tests.smyth @@ -0,0 +1 @@ +{"version":"1.0.0","id":"clzh0ec9h1gg4blqzp3ae4fw1","components":[{"id":"CLZH0ELUNYD","name":"Async","outputs":[{"name":"JobID","color":"#3C89F9","index":0,"default":true},{"name":"prompt","color":"#3C89F9","index":1,"default":false}],"inputs":[{"name":"prompt","type":"Any","color":"#F35063","optional":false,"index":0,"default":false}],"data":{},"top":"305px","left":"522px","width":"","height":"","displayName":"Async","title":"Async","aiTitle":"","description":"Runs workflow asynchronously"},{"id":"CLZH0UKYVWM7","name":"APIEndpoint","outputs":[{"name":"headers","color":"#3C89F9","index":0,"default":true},{"name":"body","color":"#3C89F9","index":1,"default":true},{"name":"query","color":"#3C89F9","index":2,"default":true},{"name":"body.prompt","color":"#3C89F9","index":3,"default":false}],"inputs":[{"name":"prompt","type":"Any","color":"#F35063","optional":false,"index":0,"default":false}],"data":{"endpoint":"no-await-async","description":"","ai_exposed":true,"summary":"","method":"POST"},"top":"169.432px","left":"101.513px","width":"","height":"","displayName":"APIEndpoint","title":"No Await","aiTitle":"","description":"HTTP API Endpoint - Expose your agent as a REST API"},{"id":"CLZH0VZ5381","name":"FSleep","outputs":[{"name":"Output","color":"#3C89F9","index":0,"default":true}],"inputs":[{"name":"Input","type":"Any","color":"#F35063","optional":false,"index":0,"default":true}],"data":{"delay":"30"},"top":"239px","left":"929px","width":"130px","height":"","displayName":"F:Sleep","title":"Sleep for 30s","aiTitle":"","description":""},{"id":"CLZH0W7PG2HK","name":"PromptGenerator","outputs":[{"name":"Reply","color":"#3C89F9","index":0,"default":true}],"inputs":[{"name":"Input","type":"Any","color":"#F35063","optional":false,"index":0,"default":false}],"data":{"model":"Echo","prompt":"Summarize the input text\nInput : {{Input}}","temperature":"1","maxTokens":"256","stopSequences":"","topP":"1","topK":"0","frequencyPenalty":"0","presencePenalty":"0"},"top":"299px","left":"1169px","width":"","height":"","displayName":"LLM Prompt","title":"LLM Prompt","aiTitle":"","description":"LLM - Generates output based on prompt and input variables"},{"id":"LZH0WX1K4R6","name":"APIEndpoint","outputs":[{"name":"headers","color":"#3C89F9","index":0,"default":true},{"name":"body","color":"#3C89F9","index":1,"default":true},{"name":"query","color":"#3C89F9","index":2,"default":true},{"name":"body.prompt","color":"#3C89F9","index":3,"default":false}],"inputs":[{"name":"prompt","type":"Any","color":"#F35063","optional":false,"index":0,"default":false}],"data":{"endpoint":"async-job","description":"","ai_exposed":true,"summary":"","method":"POST"},"top":"755px","left":"49px","width":"","height":"","displayName":"APIEndpoint","title":"Await Async Job","aiTitle":"","description":"HTTP API Endpoint - Expose your agent as a REST API"},{"id":"LZH0XJXSR","name":"Async","outputs":[{"name":"JobID","color":"#3C89F9","index":0,"default":true},{"name":"prompt","color":"#3C89F9","index":1,"default":false}],"inputs":[{"name":"prompt","type":"Any","color":"#F35063","optional":false,"index":0,"default":false}],"data":{},"top":"848px","left":"478px","width":"","height":"","displayName":"Async","title":"Async","aiTitle":"","description":"Runs workflow asynchronously"},{"id":"CLZH0XZZ4TCH","name":"FSleep","outputs":[{"name":"Output","color":"#3C89F9","index":0,"default":true}],"inputs":[{"name":"Input","type":"Any","color":"#F35063","optional":false,"index":0,"default":true}],"data":{"delay":"10"},"top":"867px","left":"885px","width":"130px","height":"","displayName":"F:Sleep","title":"Sleep for 10s","aiTitle":"","description":""},{"id":"LZH0Y7INYWG","name":"PromptGenerator","outputs":[{"name":"Reply","color":"#3C89F9","index":0,"default":true}],"inputs":[{"name":"Input","type":"Any","color":"#F35063","optional":false,"index":0,"default":false}],"data":{"model":"Echo","prompt":"Summarize the input text\nInput : {{Input}}","temperature":"1","maxTokens":"256","stopSequences":"","topP":"1","topK":"0","frequencyPenalty":"0","presencePenalty":"0"},"top":"803px","left":"1141px","width":"","height":"","displayName":"LLM Prompt","title":"LLM Prompt","aiTitle":"","description":"LLM - Generates output based on prompt and input variables"},{"id":"CLZH0YHTYJZU","name":"Await","outputs":[{"name":"Results","color":"#3C89F9","index":0,"default":true}],"inputs":[{"name":"Jobs","type":"Any","color":"#F35063","optional":false,"index":0,"default":true}],"data":{"jobs_count":"1","max_time":"60"},"top":"1037.15px","left":"829.25px","width":"","height":"","displayName":"Await","title":"Await","aiTitle":"","description":"Wait for asynchronous jobs"},{"id":"LZH8CSZSHW7","name":"APIEndpoint","outputs":[{"name":"headers","color":"#3C89F9","index":0,"default":true},{"name":"query","color":"#3C89F9","index":1,"default":true},{"name":"body","color":"#3C89F9","index":2,"default":true},{"name":"body.prompts","color":"#3C89F9","index":3,"default":false}],"inputs":[{"name":"prompts","type":"Array","color":"#F35063","optional":false,"index":0,"default":false}],"data":{"endpoint":"for-each-async-job","description":"","ai_exposed":true,"summary":"","method":"POST"},"top":"2172.71px","left":"-378.985px","width":"134.986px","height":"","displayName":"APIEndpoint","title":"Multi Async Jobs","aiTitle":"","description":"HTTP API Endpoint - Expose your agent as a REST API"},{"id":"LZH8CSZSERK","name":"Async","outputs":[{"name":"JobID","color":"#3C89F9","index":0,"default":true},{"name":"prompt","color":"#3C89F9","index":1,"default":false}],"inputs":[{"name":"prompt","type":"Any","color":"#F35063","optional":false,"index":0,"default":false}],"data":{},"top":"1931.83px","left":"427.418px","width":"273.207px","height":"","displayName":"Async","title":"Async","aiTitle":"","description":"Runs workflow asynchronously"},{"id":"LZH8CSZSO8N","name":"FSleep","outputs":[{"name":"Output","color":"#3C89F9","index":0,"default":true}],"inputs":[{"name":"Input","type":"Any","color":"#F35063","optional":false,"index":0,"default":true}],"data":{"delay":"3"},"top":"2074.43px","left":"792.629px","width":"130px","height":"","displayName":"F:Sleep","title":"Sleep for 3s","aiTitle":"","description":""},{"id":"LZH8CSZSXOS","name":"PromptGenerator","outputs":[{"name":"Reply","color":"#3C89F9","index":0,"default":true}],"inputs":[{"name":"Input","type":"Any","color":"#F35063","optional":false,"index":0,"default":false}],"data":{"model":"Echo","prompt":"{{Input}}","temperature":"1","maxTokens":"256","stopSequences":"","topP":"1","topK":"0","frequencyPenalty":"0","presencePenalty":"0"},"top":"1949.69px","left":"1040.42px","width":"","height":"","displayName":"LLM Prompt","title":"LLM Prompt","aiTitle":"Transform Input Data","description":"LLM - Generates output based on prompt and input variables"},{"id":"LZH8CSZSWJN","name":"Await","outputs":[{"name":"Results","color":"#3C89F9","index":0,"default":true}],"inputs":[{"name":"Jobs","type":"Any","color":"#F35063","optional":false,"index":0,"default":true}],"data":{"jobs_count":"4","max_time":"60"},"top":"2308.78px","left":"499.37px","width":"","height":"","displayName":"Await","title":"Await","aiTitle":"","description":"Wait for asynchronous jobs"},{"id":"CLZH8ETAJRPN","name":"ForEach","outputs":[{"name":"Loop","color":"#3C89F9","index":0,"default":true},{"name":"Result","color":"#3C89F9","index":1,"default":true}],"inputs":[{"name":"Input","type":"Any","color":"#F35063","optional":false,"index":0,"default":true}],"data":{"format":"results-array"},"top":"2128.79px","left":"37.9564px","width":"282.397px","height":"","displayName":"For Each","title":"For Each","aiTitle":"","description":"Performs a for each loop on an array"},{"id":"CLZH8HAUHZ5","name":"APIOutput","outputs":[{"name":"Output","color":"#3C89F9","index":0,"default":true}],"inputs":[{"name":"results","type":"Any","color":"#F35063","optional":false,"index":0,"default":false}],"data":{"format":"full"},"top":"2377.23px","left":"844.448px","width":"","height":"","displayName":"APIOutput","title":"APIOutput","aiTitle":"","description":""},{"id":"LZI8BN1Y0L7","name":"APIEndpoint","outputs":[{"name":"headers","color":"#3C89F9","index":0,"default":true},{"name":"body","color":"#3C89F9","index":1,"default":true},{"name":"query","color":"#3C89F9","index":2,"default":true},{"name":"body.prompts","color":"#3C89F9","index":3,"default":false}],"inputs":[{"name":"prompts","type":"Array","color":"#F35063","optional":false,"index":0,"default":false}],"data":{"endpoint":"for-each-job","description":"","ai_exposed":true,"summary":"","method":"POST"},"top":"3298.57px","left":"-373.438px","width":"314.048px","height":"","displayName":"APIEndpoint","title":"ForEach Test","aiTitle":"","description":"HTTP API Endpoint - Expose your agent as a REST API"},{"id":"LZI8BN1Y3Z","name":"PromptGenerator","outputs":[{"name":"Reply","color":"#3C89F9","index":0,"default":true}],"inputs":[{"name":"Input","type":"Any","color":"#F35063","optional":false,"index":0,"default":false}],"data":{"model":"Echo","prompt":"{{Input}}","temperature":"1","maxTokens":"256","stopSequences":"","topP":"1","topK":"0","frequencyPenalty":"0","presencePenalty":"0"},"top":"2996.2px","left":"646.616px","width":"","height":"","displayName":"LLM Prompt","title":"Input Processing Analysis","aiTitle":"Input Processing Analysis","description":"LLM - Generates output based on prompt and input variables"},{"id":"LZI8BN1YYGB","name":"ForEach","outputs":[{"name":"Loop","color":"#3C89F9","index":0,"default":true},{"name":"Result","color":"#3C89F9","index":1,"default":true}],"inputs":[{"name":"Input","type":"Any","color":"#F35063","optional":false,"index":0,"default":true}],"data":{"format":"results-array"},"top":"3199.83px","left":"136.584px","width":"282.397px","height":"","displayName":"For Each","title":"For Each","aiTitle":"","description":"Performs a for each loop on an array"},{"id":"LZI8BN1YX1","name":"APIOutput","outputs":[{"name":"Output","color":"#3C89F9","index":0,"default":true}],"inputs":[{"name":"results","type":"Any","color":"#F35063","optional":false,"index":0,"default":false}],"data":{"format":"full"},"top":"3573.15px","left":"619.057px","width":"","height":"","displayName":"APIOutput","title":"APIOutput","aiTitle":"","description":""}],"connections":[{"sourceId":"CLZH0UKYVWM7","sourceIndex":3,"targetId":"CLZH0ELUNYD","targetIndex":0},{"sourceId":"CLZH0ELUNYD","sourceIndex":1,"targetId":"CLZH0VZ5381","targetIndex":0},{"sourceId":"CLZH0VZ5381","sourceIndex":0,"targetId":"CLZH0W7PG2HK","targetIndex":0},{"sourceId":"LZH0WX1K4R6","sourceIndex":3,"targetId":"LZH0XJXSR","targetIndex":0},{"sourceId":"LZH0XJXSR","sourceIndex":1,"targetId":"CLZH0XZZ4TCH","targetIndex":0},{"sourceId":"CLZH0XZZ4TCH","sourceIndex":0,"targetId":"LZH0Y7INYWG","targetIndex":0},{"sourceId":"LZH0XJXSR","sourceIndex":0,"targetId":"CLZH0YHTYJZU","targetIndex":0},{"sourceId":"LZH8CSZSERK","sourceIndex":1,"targetId":"LZH8CSZSO8N","targetIndex":0},{"sourceId":"LZH8CSZSO8N","sourceIndex":0,"targetId":"LZH8CSZSXOS","targetIndex":0},{"sourceId":"LZH8CSZSHW7","sourceIndex":3,"targetId":"CLZH8ETAJRPN","targetIndex":0},{"sourceId":"CLZH8ETAJRPN","sourceIndex":0,"targetId":"LZH8CSZSERK","targetIndex":0},{"sourceId":"CLZH8ETAJRPN","sourceIndex":1,"targetId":"LZH8CSZSWJN","targetIndex":0},{"sourceId":"LZH8CSZSWJN","sourceIndex":0,"targetId":"CLZH8HAUHZ5","targetIndex":0},{"sourceId":"LZI8BN1Y0L7","sourceIndex":3,"targetId":"LZI8BN1YYGB","targetIndex":0},{"sourceId":"LZI8BN1YYGB","sourceIndex":1,"targetId":"LZI8BN1YX1","targetIndex":0},{"sourceId":"LZI8BN1YYGB","sourceIndex":0,"targetId":"LZI8BN1Y3Z","targetIndex":0}],"description":"","shortDescription":"An efficient agent that utilizes asynchronous processing, executes API requests, manages sleep durations, and generates LLM prompts for creative content generation.","ui":{"panzoom":{"currentPan":{"x":92.9078822233488,"y":-1850.0802616003675},"currentZoom":0.7408182206817211}},"templateInfo":{"id":"async-test-lzi97afrf0r","name":"Async Test","description":"An efficient agent that utilizes asynchronous processing, executes API requests, manages sleep durations, and generates LLM prompts for creative content generation.","icon":"","category":"","publish":false,"color":"#000000","imageUrl":"","docLink":"","videoLink":""}} \ No newline at end of file diff --git a/packages/core/tests/data/AgentData/data-components.smyth b/packages/core/tests/data/AgentData/data-components.smyth new file mode 100644 index 000000000..c5603871b --- /dev/null +++ b/packages/core/tests/data/AgentData/data-components.smyth @@ -0,0 +1,119 @@ +{ + "version": "1.0.0", + "id": "cly3s2wv1023jgmrm8fodo69f", + "components": [ + { + "id": "CLZ1I1XCB42T", + "name": "DataSourceLookup", + "outputs": [ + { + "name": "Results", + "color": "#3C89F9", + "index": 0, + "default": true + } + ], + "inputs": [ + { + "name": "Query", + "type": "Any", + "color": "#F35063", + "optional": false, + "index": 0, + "default": true + } + ], + "data": { + "namespace": "clv1cv00t0001alug52hd69j8_test", + "topK": "3", + "model": "gpt-3.5-turbo", + "prompt": "provide brief results", + "includeMetadata": true, + "postprocess": false + }, + "top": "162px", + "left": "721px", + "width": "", + "height": "", + "displayName": "Data Lookup", + "title": "Data Lookup", + "aiTitle": "Results Summary Display", + "description": "Lookup data from data pool" + }, + { + "id": "CLZ1IE6AMT4L", + "name": "DataSourceIndexer", + "outputs": [ + { + "name": "Success", + "color": "#3C89F9", + "index": 0, + "default": true + } + ], + "inputs": [ + { + "name": "Source", + "type": "Any", + "color": "#F35063", + "optional": false, + "index": 0, + "default": true + } + ], + "data": { + "namespace": "clv1cv00t0001alug52hd69j8_test", + "id": "_random", + "name": "", + "metadata": "" + }, + "top": "544.962px", + "left": "757.962px", + "width": "", + "height": "", + "displayName": "Source Indexer", + "title": "Source Indexer", + "aiTitle": "", + "description": "Index data sources to data pool" + }, + { + "id": "CLZ1IE8XY94R", + "name": "DataSourceCleaner", + "outputs": [ + { + "name": "Success", + "color": "#3C89F9", + "index": 0, + "default": true + } + ], + "inputs": [], + "data": { + "namespaceId": "clv1cv00t0001alug52hd69j8_dara_with_space", + "id": "" + }, + "top": "273px", + "left": "393px", + "width": "", + "height": "", + "displayName": "Source Cleaner", + "title": "Source Cleaner", + "aiTitle": "", + "description": "Delete data sources from data pool" + } + ], + "connections": [], + "description": "", + "shortDescription": "An efficient agent that looks up data sources, indexes sources, and cleans up data for optimal processing.", + "behavior": "", + "debugSessionEnabled": true, + "ui": { + "panzoom": { + "currentPan": { + "x": 4, + "y": 50 + }, + "currentZoom": 1 + } + } + } \ No newline at end of file diff --git a/packages/core/tests/data/AgentData/functions-components.smyth b/packages/core/tests/data/AgentData/functions-components.smyth new file mode 100644 index 000000000..56c952c2d --- /dev/null +++ b/packages/core/tests/data/AgentData/functions-components.smyth @@ -0,0 +1,588 @@ +{ + "version": "1.0.0", + "components": [ + { + "id": "CLYQPG97PT0G", + "name": "FTimestamp", + "outputs": [ + { + "name": "Timestamp", + "color": "#3C89F9", + "index": 0, + "default": true + } + ], + "inputs": [ + { + "name": "Trigger", + "type": "Any", + "color": "#F35063", + "optional": false, + "index": 0, + "default": true + } + ], + "data": {}, + "top": "919px", + "left": "151px", + "width": "130px", + "height": "", + "displayName": "F:Timestamp", + "title": "Timestamp", + "aiTitle": "", + "description": "" + }, + { + "id": "CLYQPM7QOHE", + "name": "FHash", + "outputs": [ + { + "name": "Hash", + "color": "#3C89F9", + "index": 0, + "default": true + } + ], + "inputs": [ + { + "name": "Data", + "type": "Any", + "color": "#F35063", + "optional": false, + "index": 0, + "default": true + } + ], + "data": { + "algorithm": "md5", + "encoding": "hex" + }, + "top": "1190.36px", + "left": "159.107px", + "width": "130px", + "height": "", + "displayName": "F:Hash", + "title": "MD5 :: hex", + "aiTitle": "", + "description": "" + }, + { + "id": "CLYQPMF218O", + "name": "FEncDec", + "outputs": [ + { + "name": "Output", + "color": "#3C89F9", + "index": 0, + "default": true + } + ], + "inputs": [ + { + "name": "Data", + "type": "Any", + "color": "#F35063", + "optional": false, + "index": 0, + "default": true + } + ], + "data": { + "action": "Encode", + "encoding": "hex" + }, + "top": "1689.28px", + "left": "191.836px", + "width": "130px", + "height": "", + "displayName": "F:Encode/Decode", + "title": "hex Encode", + "aiTitle": "", + "description": "" + }, + { + "id": "CLYQPR6ESZF", + "name": "FSign", + "outputs": [ + { + "name": "Signature", + "color": "#3C89F9", + "index": 0, + "default": true + } + ], + "inputs": [ + { + "name": "Data", + "type": "Any", + "color": "#F35063", + "optional": false, + "index": 0, + "default": true + }, + { + "name": "Key", + "type": "Any", + "color": "#F35063", + "optional": false, + "index": 1, + "default": true + } + ], + "data": { + "signMethod": "HMAC", + "dataTransform": "Stringify", + "key": "", + "hashType": "md5", + "RSA_padding": "RSA_PKCS1_PADDING", + "RSA_saltLength": "", + "encoding": "hex" + }, + "top": "2139.32px", + "left": "230.976px", + "width": "130px", + "height": "", + "displayName": "F:Sign", + "title": "HMAC-MD5 :: hex", + "aiTitle": "", + "description": "" + }, + { + "id": "CLYQPRHIYEVD", + "name": "APIEndpoint", + "outputs": [ + { + "name": "headers", + "color": "#3C89F9", + "index": 0, + "default": true + }, + { + "name": "body", + "color": "#3C89F9", + "index": 1, + "default": true + }, + { + "name": "query", + "color": "#3C89F9", + "index": 2, + "default": true + } + ], + "inputs": [], + "data": { + "endpoint": "timestamp", + "description": "", + "ai_exposed": true, + "summary": "", + "method": "POST" + }, + "top": "757.813px", + "left": "-189.307px", + "width": "256.55px", + "height": "", + "displayName": "APIEndpoint", + "title": "Timestamp", + "aiTitle": "", + "description": "HTTP API Endpoint - Expose your agent as a REST API" + }, + { + "id": "LYQPSR9TQDJ", + "name": "APIEndpoint", + "outputs": [ + { + "name": "headers", + "color": "#3C89F9", + "index": 0, + "default": true + }, + { + "name": "body", + "color": "#3C89F9", + "index": 1, + "default": true + }, + { + "name": "query", + "color": "#3C89F9", + "index": 2, + "default": true + }, + { + "name": "body.data", + "color": "#3C89F9", + "index": 3, + "default": false + } + ], + "inputs": [ + { + "name": "data", + "type": "Any", + "color": "#F35063", + "optional": false, + "index": 0, + "default": false + } + ], + "data": { + "endpoint": "hash", + "description": "", + "ai_exposed": true, + "summary": "", + "method": "POST" + }, + "top": "1026px", + "left": "-189.36px", + "width": "", + "height": "", + "displayName": "APIEndpoint", + "title": "Hash", + "aiTitle": "", + "description": "HTTP API Endpoint - Expose your agent as a REST API" + }, + { + "id": "LYQPSTMTS3K", + "name": "APIEndpoint", + "outputs": [ + { + "name": "headers", + "color": "#3C89F9", + "index": 0, + "default": true + }, + { + "name": "body", + "color": "#3C89F9", + "index": 1, + "default": true + }, + { + "name": "query", + "color": "#3C89F9", + "index": 2, + "default": true + }, + { + "name": "body.data", + "color": "#3C89F9", + "index": 3, + "default": false + } + ], + "inputs": [ + { + "name": "data", + "type": "Any", + "color": "#F35063", + "optional": false, + "index": 0, + "default": false + } + ], + "data": { + "endpoint": "dec", + "description": "", + "ai_exposed": true, + "summary": "", + "method": "POST" + }, + "top": "1353.98px", + "left": "-192.037px", + "width": "247px", + "height": "", + "displayName": "APIEndpoint", + "title": "Decode Hex", + "aiTitle": "", + "description": "HTTP API Endpoint - Expose your agent as a REST API" + }, + { + "id": "LYQPSVGWYX", + "name": "APIEndpoint", + "outputs": [ + { + "name": "headers", + "color": "#3C89F9", + "index": 0, + "default": true + }, + { + "name": "body", + "color": "#3C89F9", + "index": 1, + "default": true + }, + { + "name": "query", + "color": "#3C89F9", + "index": 2, + "default": true + }, + { + "name": "body.data", + "color": "#3C89F9", + "index": 3, + "default": false + }, + { + "name": "body.key", + "color": "#3C89F9", + "index": 4, + "default": false + } + ], + "inputs": [ + { + "name": "key", + "type": "Any", + "color": "#F35063", + "optional": false, + "index": 0, + "default": false + }, + { + "name": "data", + "type": "Any", + "color": "#F35063", + "optional": false, + "index": 1, + "default": false + } + ], + "data": { + "endpoint": "sign", + "description": "", + "ai_exposed": true, + "summary": "", + "method": "POST" + }, + "top": "2024.62px", + "left": "-178.71px", + "width": "", + "height": "", + "displayName": "APIEndpoint", + "title": "Sign", + "aiTitle": "", + "description": "HTTP API Endpoint - Expose your agent as a REST API" + }, + { + "id": "LYQPTE87PO8", + "name": "APIEndpoint", + "outputs": [ + { + "name": "headers", + "color": "#3C89F9", + "index": 0, + "default": true + }, + { + "name": "body", + "color": "#3C89F9", + "index": 1, + "default": true + }, + { + "name": "query", + "color": "#3C89F9", + "index": 2, + "default": true + }, + { + "name": "body.data", + "color": "#3C89F9", + "index": 3, + "default": false + } + ], + "inputs": [ + { + "name": "data", + "type": "Any", + "color": "#F35063", + "optional": false, + "index": 0, + "default": false + } + ], + "data": { + "endpoint": "enc", + "description": "", + "ai_exposed": true, + "summary": "", + "method": "POST" + }, + "top": "1675px", + "left": "-184px", + "width": "", + "height": "", + "displayName": "APIEndpoint", + "title": "Encode Hex", + "aiTitle": "", + "description": "HTTP API Endpoint - Expose your agent as a REST API" + }, + { + "id": "LYQPVTHUTUJ", + "name": "FEncDec", + "outputs": [ + { + "name": "Output", + "color": "#3C89F9", + "index": 0, + "default": true + } + ], + "inputs": [ + { + "name": "Data", + "type": "Any", + "color": "#F35063", + "optional": false, + "index": 0, + "default": true + } + ], + "data": { + "action": "Decode", + "encoding": "hex" + }, + "top": "1491px", + "left": "186px", + "width": "130px", + "height": "", + "displayName": "F:Encode/Decode", + "title": "hex Decode", + "aiTitle": "", + "description": "" + }, + { + "id": "CLYRF6U1AZXS", + "name": "FSleep", + "outputs": [ + { + "name": "Output", + "color": "#3C89F9", + "index": 0, + "default": true + } + ], + "inputs": [ + { + "name": "Input", + "type": "Any", + "color": "#F35063", + "optional": false, + "index": 0, + "default": true + } + ], + "data": { + "delay": "10" + }, + "top": "2566.27px", + "left": "245.975px", + "width": "130px", + "height": "", + "displayName": "F:Sleep", + "title": "Sleep for 10s", + "aiTitle": "", + "description": "" + }, + { + "id": "LYRF6ZI3QBN", + "name": "APIEndpoint", + "outputs": [ + { + "name": "headers", + "color": "#3C89F9", + "index": 0, + "default": true + }, + { + "name": "body", + "color": "#3C89F9", + "index": 1, + "default": true + }, + { + "name": "query", + "color": "#3C89F9", + "index": 2, + "default": true + } + ], + "inputs": [], + "data": { + "endpoint": "sleep_10", + "description": "", + "ai_exposed": true, + "summary": "", + "method": "POST" + }, + "top": "2506px", + "left": "-208px", + "width": "", + "height": "", + "displayName": "APIEndpoint", + "title": "Delay 10s", + "aiTitle": "", + "description": "HTTP API Endpoint - Expose your agent as a REST API" + } + ], + "connections": [ + { + "sourceId": "CLYQPRHIYEVD", + "sourceIndex": 1, + "targetId": "CLYQPG97PT0G", + "targetIndex": 0 + }, + { + "sourceId": "LYQPSVGWYX", + "sourceIndex": 4, + "targetId": "CLYQPR6ESZF", + "targetIndex": 1 + }, + { + "sourceId": "LYQPSVGWYX", + "sourceIndex": 3, + "targetId": "CLYQPR6ESZF", + "targetIndex": 1 + }, + { + "sourceId": "LYQPTE87PO8", + "sourceIndex": 3, + "targetId": "CLYQPMF218O", + "targetIndex": 0 + }, + { + "sourceId": "LYQPSR9TQDJ", + "sourceIndex": 3, + "targetId": "CLYQPM7QOHE", + "targetIndex": 0 + }, + { + "sourceId": "LYQPSTMTS3K", + "sourceIndex": 3, + "targetId": "LYQPVTHUTUJ", + "targetIndex": 0 + }, + { + "sourceId": "LYRF6ZI3QBN", + "sourceIndex": 1, + "targetId": "CLYRF6U1AZXS", + "targetIndex": 0 + } + ], + "description": "", + "behavior": "", + "debugSessionEnabled": false, + "ui": { + "panzoom": { + "currentPan": { + "x": 560.4753966599353, + "y": -2086.042344478873 + }, + "currentZoom": 1.000000000000004 + } + } + } \ No newline at end of file diff --git a/packages/core/tests/data/general-unit-tests.smyth b/packages/core/tests/data/AgentData/general-unit-tests.smyth similarity index 100% rename from packages/core/tests/data/general-unit-tests.smyth rename to packages/core/tests/data/AgentData/general-unit-tests.smyth diff --git a/packages/core/tests/data/AgentData/sre-llm.smyth b/packages/core/tests/data/AgentData/sre-llm.smyth new file mode 100644 index 000000000..73837ab70 --- /dev/null +++ b/packages/core/tests/data/AgentData/sre-llm.smyth @@ -0,0 +1 @@ +{"version":"1.0.0","id":"cm0zjhkzx0dfvhxf81u76taiz","name":"[SRE] LLM","teamId":"cloilcrl9001v9tkguilsu8dx","components":[{"id":"CM0ZJIDVPEPO","name":"GPTPlugin","outputs":[{"name":"Output","color":"#3C89F9","index":0,"default":true}],"inputs":[{"name":"Query","type":"Any","color":"#F35063","optional":false,"index":0,"default":true}],"data":{"openAiModel":"gpt-4o-mini","specUrl":"https://byby.ai/openapi.yaml","descForModel":"Plugin for displaying a table of products. Each product consists of properties such as: Title, Description, Link, Number of stars, Image URL. The user should provide a searchTerm and optionally the quantity of products (default is 5). The responses should be represented in a Markdown table with the following columns: Title, Description, Rating, Image. The images will optionally be displayed in the table, only if the user requests it. The Title should be a link with the Link provided by the API.","name":"ByByAI","desc":"Search for the best Amazon products ordered by rating.","logoUrl":"https://byby.ai/logo.jpeg"},"top":"-2214.9px","left":"535.603px","width":"","height":"","displayName":"ByByAI","title":"ByByAI","aiTitle":"","description":"Search for the best Amazon products ordered by rating."},{"id":"CM0ZJO5YFRQ","name":"APIEndpoint","outputs":[{"name":"headers","color":"#3C89F9","index":0,"default":true},{"name":"body","color":"#3C89F9","index":1,"default":true},{"name":"query","color":"#3C89F9","index":2,"default":true},{"name":"body.Input","color":"#3C89F9","index":3,"default":false}],"inputs":[{"name":"Input","type":"Any","color":"#F35063","optional":false,"index":0,"default":false}],"data":{"endpoint":"test-gpt-plugin","description":"","ai_exposed":true,"summary":"","method":"POST"},"top":"-2243.15px","left":"113.7px","width":"","height":"","displayName":"APIEndpoint","title":"APIEndpoint","aiTitle":"","description":"HTTP API Endpoint - Expose your agent as a REST API"},{"id":"CM19HJVYHMEU","name":"PromptGenerator","outputs":[{"name":"Reply","color":"#3C89F9","index":0,"default":true}],"inputs":[{"name":"Input","type":"Any","color":"#F35063","optional":false,"index":0,"default":false}],"data":{"model":"SRE - Bedrock with AI21 Labs - Jamba-Instruct","prompt":"{{Input}}","temperature":"1","maxTokens":"256","stopSequences":"","topP":"1","topK":"0","frequencyPenalty":"0","presencePenalty":"0"},"top":"1015.64px","left":"495.905px","width":"249.917px","height":"","displayName":"LLM Prompt","title":"Amazon Bedrock","aiTitle":"","description":"LLM - Generates output based on prompt and input variables"},{"id":"CM19HK1HH08","name":"APIEndpoint","outputs":[{"name":"headers","color":"#3C89F9","index":0,"default":true},{"name":"body","color":"#3C89F9","index":1,"default":true},{"name":"query","color":"#3C89F9","index":2,"default":true},{"name":"body.Input","color":"#3C89F9","index":3,"default":false}],"inputs":[{"name":"Input","type":"Any","color":"#F35063","optional":false,"index":0,"default":false}],"data":{"endpoint":"test-bedrock-model","description":"","ai_exposed":true,"summary":"","method":"POST"},"top":"958.715px","left":"117.712px","width":"","height":"","displayName":"APIEndpoint","title":"APIEndpoint","aiTitle":"","description":"HTTP API Endpoint - Expose your agent as a REST API"},{"id":"M19SQ4813E","name":"PromptGenerator","outputs":[{"name":"Reply","color":"#3C89F9","index":0,"default":true}],"inputs":[{"name":"Input","type":"Any","color":"#F35063","optional":false,"index":0,"default":false}],"data":{"model":"SRE - Vertex AI with Gemini Flash","prompt":"{{Input}}","temperature":"1","maxTokens":"256","stopSequences":"","topP":"1","topK":"1","frequencyPenalty":"0","presencePenalty":"0"},"top":"1757.74px","left":"501.686px","width":"251px","height":"","displayName":"LLM Prompt","title":"VertexAI","aiTitle":"","description":"LLM - Generates output based on prompt and input variables"},{"id":"M19SQ481DPA","name":"APIEndpoint","outputs":[{"name":"headers","color":"#3C89F9","index":0,"default":true},{"name":"body","color":"#3C89F9","index":1,"default":true},{"name":"query","color":"#3C89F9","index":2,"default":true},{"name":"body.Input","color":"#3C89F9","index":3,"default":false}],"inputs":[{"name":"Input","type":"Any","color":"#F35063","optional":false,"index":0,"default":false}],"data":{"endpoint":"test-vertexai-model","description":"","ai_exposed":true,"summary":"","method":"POST"},"top":"1704.62px","left":"126.208px","width":"","height":"","displayName":"APIEndpoint","title":"APIEndpoint","aiTitle":"","description":"HTTP API Endpoint - Expose your agent as a REST API"},{"id":"M19SUJSVY7","name":"PromptGenerator","outputs":[{"name":"Reply","color":"#3C89F9","index":0,"default":true}],"inputs":[{"name":"Input","type":"Any","color":"#F35063","optional":false,"index":0,"default":false}],"data":{"model":"meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo","prompt":"{{Input}}","temperature":"1","maxTokens":"512","stopSequences":"","topP":"1","topK":"1","frequencyPenalty":"0","presencePenalty":"0"},"top":"608.524px","left":"501.285px","width":"279.153px","height":"","displayName":"LLM Prompt","title":"TogetherAI","aiTitle":"","description":"LLM - Generates output based on prompt and input variables"},{"id":"M19SUJSVM5C","name":"APIEndpoint","outputs":[{"name":"headers","color":"#3C89F9","index":0,"default":true},{"name":"body","color":"#3C89F9","index":1,"default":true},{"name":"query","color":"#3C89F9","index":2,"default":true},{"name":"body.Input","color":"#3C89F9","index":3,"default":false}],"inputs":[{"name":"Input","type":"Any","color":"#F35063","optional":false,"index":0,"default":false}],"data":{"endpoint":"test-togetherai-model","description":"","ai_exposed":true,"summary":"","method":"POST"},"top":"563px","left":"117px","width":"","height":"","displayName":"APIEndpoint","title":"APIEndpoint","aiTitle":"","description":"HTTP API Endpoint - Expose your agent as a REST API"},{"id":"M19UF7AK3P","name":"APIEndpoint","outputs":[{"name":"headers","color":"#3C89F9","index":0,"default":true},{"name":"body","color":"#3C89F9","index":1,"default":true},{"name":"query","color":"#3C89F9","index":2,"default":true},{"name":"body.Input","color":"#3C89F9","index":3,"default":false}],"inputs":[{"name":"Input","type":"Any","color":"#F35063","optional":false,"index":0,"default":false}],"data":{"endpoint":"read-video-file-with-multimodal-llm","description":"","ai_exposed":true,"summary":"","method":"POST"},"top":"2106.36px","left":"128.164px","width":"247px","height":"","displayName":"APIEndpoint","title":"APIEndpoint","aiTitle":"","description":"HTTP API Endpoint - Expose your agent as a REST API"},{"id":"CM19UG6IXRWA","name":"MultimodalLLM","outputs":[{"name":"Reply","color":"#3C89F9","index":0,"default":true}],"inputs":[{"name":"Input","type":"Any","color":"#F35063","optional":false,"index":0,"default":true}],"data":{"model":"gemini-1.5-flash-stable","prompt":"Describe the main events in this video.","maxTokens":"300"},"top":"2139.57px","left":"486.401px","width":"","height":"","displayName":"Multimodal LLM","title":"Video Summary Breakdown","aiTitle":"Video Summary Breakdown","description":"Processes multimodal inputs (images, videos, text) to generate context-aware outputs."},{"id":"M19UJ2K7MA","name":"APIEndpoint","outputs":[{"name":"headers","color":"#3C89F9","index":0,"default":true},{"name":"body","color":"#3C89F9","index":1,"default":true},{"name":"query","color":"#3C89F9","index":2,"default":true},{"name":"body.Input","color":"#3C89F9","index":3,"default":false}],"inputs":[{"name":"Input","type":"Any","color":"#F35063","optional":false,"index":0,"default":false}],"data":{"endpoint":"read-image-file-with-vision-llm","description":"","ai_exposed":true,"summary":"","method":"POST"},"top":"2489.13px","left":"126.26px","width":"249px","height":"","displayName":"APIEndpoint","title":"APIEndpoint","aiTitle":"","description":"HTTP API Endpoint - Expose your agent as a REST API"},{"id":"CM19UJ5P1S3","name":"VisionLLM","outputs":[{"name":"Reply","color":"#3C89F9","index":0,"default":true}],"inputs":[{"name":"Images","type":"Any","color":"#F35063","optional":false,"index":0,"default":true}],"data":{"model":"gemini-1.5-flash-stable","prompt":"What’s in this image?","maxTokens":"300"},"top":"2539.93px","left":"487.781px","width":"","height":"","displayName":"Vision LLM","title":"Vision LLM","aiTitle":"","description":"Answers questions related given image(s)."},{"id":"M1EUADI6GVT","name":"APIEndpoint","outputs":[{"name":"headers","color":"#3C89F9","index":0,"default":true},{"name":"body","color":"#3C89F9","index":1,"default":true},{"name":"query","color":"#3C89F9","index":2,"default":true}],"inputs":[],"data":{"endpoint":"test-agent-plugin","description":"","ai_exposed":true,"summary":"","method":"GET"},"top":"-1857.28px","left":"115.563px","width":"","height":"","displayName":"APIEndpoint","title":"APIEndpoint","aiTitle":"","description":"HTTP API Endpoint - Expose your agent as a REST API"},{"id":"CM1EUBDZSS7O","name":"AgentPlugin","outputs":[{"name":"Response","color":"#3C89F9","index":0,"default":true}],"inputs":[{"name":"Prompt","type":"Any","color":"#F35063","optional":false,"index":0,"default":true}],"data":{"openAiModel":"gpt-4o-mini","version":"same-as-parent","descForModel":"Get posts and give me only the title for the top 2 posts","agentId":"clp1tl4tx00129tq5owb0kfxh","id":"clp1tl4tx00129tq5owb0kfxh","name":"Sub Agent","desc":"Description goes here","logoUrl":"https://app.smythos.dev/img/smythos-logo.png","model":"gpt-4o-mini"},"top":"-1863.77px","left":"520.621px","width":"","height":"","displayName":"Sub Agent","title":"Sub Agent (SRE)","aiTitle":"","description":"Description goes here"},{"id":"M1F1SJUTJO","name":"APIEndpoint","outputs":[{"name":"headers","color":"#3C89F9","index":0,"default":true},{"name":"body","color":"#3C89F9","index":1,"default":true},{"name":"query","color":"#3C89F9","index":2,"default":true}],"inputs":[],"data":{"endpoint":"test-agent-plugin","description":"","ai_exposed":true,"summary":"","method":"POST"},"top":"-1482.43px","left":"115.447px","width":"","height":"","displayName":"APIEndpoint","title":"APIEndpoint","aiTitle":"","description":"HTTP API Endpoint - Expose your agent as a REST API"},{"id":"M1F1SJUTHXH","name":"AgentPlugin","outputs":[{"name":"Response","color":"#3C89F9","index":0,"default":true}],"inputs":[{"name":"Prompt","type":"Any","color":"#F35063","optional":false,"index":0,"default":true}],"data":{"openAiModel":"gpt-4o-mini","version":"same-as-parent","descForModel":"Create posts by giving details","agentId":"clp1tl4tx00129tq5owb0kfxh","id":"clp1tl4tx00129tq5owb0kfxh","name":"Sub Agent","desc":"Description goes here","logoUrl":"https://app.smythos.dev/img/smythos-logo.png","model":"gpt-4o-mini"},"top":"-1488.44px","left":"520.447px","width":"","height":"","displayName":"Sub Agent","title":"Sub Agent (SRE)","aiTitle":"","description":"Description goes here"},{"id":"M1Z79T7UISH","name":"PromptGenerator","outputs":[{"name":"Reply","color":"#3C89F9","index":0,"default":true}],"inputs":[{"name":"Input","type":"Any","color":"#F35063","optional":false,"index":0,"default":false}],"data":{"model":"Echo","prompt":"{{Input}}","temperature":"1","maxTokens":"256","stopSequences":"","topP":"1","topK":"0","frequencyPenalty":"0","presencePenalty":"0"},"top":"-2638.43px","left":"498.019px","width":"251px","height":"","displayName":"LLM Prompt","title":"JSON Data Structure","aiTitle":"JSON Data Structure","description":"LLM - Generates output based on prompt and input variables"},{"id":"M1Z79T7V4NE","name":"APIEndpoint","outputs":[{"name":"headers","color":"#3C89F9","index":0,"default":true},{"name":"body","color":"#3C89F9","index":1,"default":true},{"name":"query","color":"#3C89F9","index":2,"default":true},{"name":"body.Input","color":"#3C89F9","index":3,"default":false}],"inputs":[{"name":"Input","type":"Any","color":"#F35063","optional":false,"index":0,"default":false}],"data":{"endpoint":"test-echo","description":"","ai_exposed":true,"summary":"","method":"POST"},"top":"-2697.23px","left":"115.495px","width":"","height":"","displayName":"APIEndpoint","title":"APIEndpoint","aiTitle":"","description":"HTTP API Endpoint - Expose your agent as a REST API"},{"id":"M2BWAL0I9VN","name":"PromptGenerator","outputs":[{"name":"Reply","color":"#3C89F9","index":0,"default":true}],"inputs":[{"name":"Input","type":"Any","color":"#F35063","optional":false,"index":0,"default":false}],"data":{"model":"groq-gemma2-9b","prompt":"{{Input}}","temperature":"1","maxTokens":"256","stopSequences":"","topP":"1","topK":"0","frequencyPenalty":"0","presencePenalty":"0"},"top":"233px","left":"502px","width":"278.058px","height":"","displayName":"LLM Prompt","title":"Groq","aiTitle":"","description":"LLM - Generates output based on prompt and input variables"},{"id":"M2BWAL0JJZU","name":"APIEndpoint","outputs":[{"name":"headers","color":"#3C89F9","index":0,"default":true},{"name":"body","color":"#3C89F9","index":1,"default":true},{"name":"query","color":"#3C89F9","index":2,"default":true},{"name":"body.Input","color":"#3C89F9","index":3,"default":false}],"inputs":[{"name":"Input","type":"Any","color":"#F35063","optional":false,"index":0,"default":false}],"data":{"endpoint":"test-groq-model","description":"","ai_exposed":true,"summary":"","method":"POST"},"top":"184px","left":"118px","width":"","height":"","displayName":"APIEndpoint","title":"APIEndpoint","aiTitle":"","description":"HTTP API Endpoint - Expose your agent as a REST API"},{"id":"M2BWB3G4PJ6","name":"PromptGenerator","outputs":[{"name":"Reply","color":"#3C89F9","index":0,"default":true}],"inputs":[{"name":"Input","type":"Any","color":"#F35063","optional":false,"index":0,"default":false}],"data":{"model":"gemini-1.5-flash-stable","prompt":"{{Input}}","temperature":"1","maxTokens":"256","stopSequences":"","topP":"1","topK":"0","frequencyPenalty":"0","presencePenalty":"0"},"top":"-150.802px","left":"498.802px","width":"279.153px","height":"","displayName":"LLM Prompt","title":"GoogleAI","aiTitle":"","description":"LLM - Generates output based on prompt and input variables"},{"id":"M2BWB3G5F6G","name":"APIEndpoint","outputs":[{"name":"headers","color":"#3C89F9","index":0,"default":true},{"name":"body","color":"#3C89F9","index":1,"default":true},{"name":"query","color":"#3C89F9","index":2,"default":true},{"name":"body.Input","color":"#3C89F9","index":3,"default":false}],"inputs":[{"name":"Input","type":"Any","color":"#F35063","optional":false,"index":0,"default":false}],"data":{"endpoint":"test-googleai-model","description":"","ai_exposed":true,"summary":"","method":"POST"},"top":"-195px","left":"114px","width":"","height":"","displayName":"APIEndpoint","title":"APIEndpoint","aiTitle":"","description":"HTTP API Endpoint - Expose your agent as a REST API"},{"id":"M2BWBC2AU1","name":"PromptGenerator","outputs":[{"name":"Reply","color":"#3C89F9","index":0,"default":true}],"inputs":[{"name":"Input","type":"Any","color":"#F35063","optional":false,"index":0,"default":false}],"data":{"model":"claude-3.5-haiku","prompt":"{{Input}}","temperature":"1","maxTokens":"2048","stopSequences":"","topP":"1","topK":"0","frequencyPenalty":"0","presencePenalty":"0"},"top":"-533px","left":"498px","width":"279.153px","height":"","displayName":"LLM Prompt","title":"Anthropic","aiTitle":"","description":"LLM - Generates output based on prompt and input variables"},{"id":"M2BWBC2AX4","name":"APIEndpoint","outputs":[{"name":"headers","color":"#3C89F9","index":0,"default":true},{"name":"body","color":"#3C89F9","index":1,"default":true},{"name":"query","color":"#3C89F9","index":2,"default":true},{"name":"body.Input","color":"#3C89F9","index":3,"default":false}],"inputs":[{"name":"Input","type":"Any","color":"#F35063","optional":false,"index":0,"default":false}],"data":{"endpoint":"test-anthropic-model","description":"","ai_exposed":true,"summary":"","method":"POST"},"top":"-578px","left":"114px","width":"","height":"","displayName":"APIEndpoint","title":"APIEndpoint","aiTitle":"","description":"HTTP API Endpoint - Expose your agent as a REST API"},{"id":"M2BWBG7ESVQ","name":"PromptGenerator","outputs":[{"name":"Reply","color":"#3C89F9","index":0,"default":true}],"inputs":[{"name":"Input","type":"Any","color":"#F35063","optional":false,"index":0,"default":false}],"data":{"model":"gpt-4o-mini","prompt":"{{Input}}","temperature":"1","maxTokens":"256","stopSequences":"","topP":"1","topK":"0","frequencyPenalty":"0","presencePenalty":"0"},"top":"-914px","left":"504px","width":"279.153px","height":"","displayName":"LLM Prompt","title":"OpenAI","aiTitle":"","description":"LLM - Generates output based on prompt and input variables"},{"id":"M2BWBG7GGQG","name":"APIEndpoint","outputs":[{"name":"headers","color":"#3C89F9","index":0,"default":true},{"name":"body","color":"#3C89F9","index":1,"default":true},{"name":"query","color":"#3C89F9","index":2,"default":true},{"name":"body.Input","color":"#3C89F9","index":3,"default":false}],"inputs":[{"name":"Input","type":"Any","color":"#F35063","optional":false,"index":0,"default":false}],"data":{"endpoint":"test-openai-model","description":"","ai_exposed":true,"summary":"","method":"POST"},"top":"-959px","left":"120px","width":"","height":"","displayName":"APIEndpoint","title":"APIEndpoint","aiTitle":"","description":"HTTP API Endpoint - Expose your agent as a REST API"},{"id":"M3E9MK4TF3","name":"PromptGenerator","outputs":[{"name":"Reply","color":"#3C89F9","index":0,"default":true}],"inputs":[{"name":"Input","type":"Any","color":"#F35063","optional":false,"index":0,"default":false}],"data":{"model":"SRE - Bedrock does not support System Prompt","prompt":"{{Input}}","temperature":"1","maxTokens":"256","stopSequences":"","topP":"1","topK":"0","frequencyPenalty":"0","presencePenalty":"0"},"top":"1392.4px","left":"501.81px","width":"249.917px","height":"","displayName":"LLM Prompt","title":"Amazon Bedrock 2","aiTitle":"","description":"LLM - Generates output based on prompt and input variables"},{"id":"M3E9MK4UV2","name":"APIEndpoint","outputs":[{"name":"headers","color":"#3C89F9","index":0,"default":true},{"name":"body","color":"#3C89F9","index":1,"default":true},{"name":"query","color":"#3C89F9","index":2,"default":true},{"name":"body.Input","color":"#3C89F9","index":3,"default":false}],"inputs":[{"name":"Input","type":"Any","color":"#F35063","optional":false,"index":0,"default":false}],"data":{"endpoint":"test-bedrock-model-that-does-not-support-system","description":"","ai_exposed":true,"summary":"","method":"POST"},"top":"1335.4px","left":"123.81px","width":"","height":"","displayName":"APIEndpoint","title":"APIEndpoint","aiTitle":"","description":"HTTP API Endpoint - Expose your agent as a REST API"}],"connections":[{"sourceId":"CM0ZJO5YFRQ","sourceIndex":3,"targetId":"CM0ZJIDVPEPO","targetIndex":0},{"sourceId":"CM19HK1HH08","sourceIndex":3,"targetId":"CM19HJVYHMEU","targetIndex":0},{"sourceId":"M19SQ481DPA","sourceIndex":3,"targetId":"M19SQ4813E","targetIndex":0},{"sourceId":"M19SUJSVM5C","sourceIndex":3,"targetId":"M19SUJSVY7","targetIndex":0},{"sourceId":"M19UF7AK3P","sourceIndex":3,"targetId":"CM19UG6IXRWA","targetIndex":0},{"sourceId":"M19UJ2K7MA","sourceIndex":3,"targetId":"CM19UJ5P1S3","targetIndex":0},{"sourceId":"M1F1SJUTJO","sourceIndex":1,"targetId":"M1F1SJUTHXH","targetIndex":0},{"sourceId":"M1Z79T7V4NE","sourceIndex":3,"targetId":"M1Z79T7UISH","targetIndex":0},{"sourceId":"M2BWAL0JJZU","sourceIndex":3,"targetId":"M2BWAL0I9VN","targetIndex":0},{"sourceId":"M2BWB3G5F6G","sourceIndex":3,"targetId":"M2BWB3G4PJ6","targetIndex":0},{"sourceId":"M2BWBC2AX4","sourceIndex":3,"targetId":"M2BWBC2AU1","targetIndex":0},{"sourceId":"M2BWBG7GGQG","sourceIndex":3,"targetId":"M2BWBG7ESVQ","targetIndex":0},{"sourceId":"M3E9MK4UV2","sourceIndex":3,"targetId":"M3E9MK4TF3","targetIndex":0},{"sourceId":"M1EUADI6GVT","sourceIndex":1,"targetId":"CM1EUBDZSS7O","targetIndex":0}],"description":"","shortDescription":"A dynamic agent that integrates with the ByByAI GPTPlugin and utilizes a POST method to efficiently connect and interact with API endpoints.","behavior":"","ui":{"panzoom":{"currentPan":{"x":5.739286084181686,"y":920.041974719531},"currentZoom":1.02077123907096}},"templateInfo":{"id":"sre-llm-m3vfd1nc62c","name":"[SRE] LLM","description":"A dynamic agent that integrates with the ByByAI GPTPlugin and utilizes a POST method to efficiently connect and interact with API endpoints.","icon":"","category":"","publish":false,"color":"#000000","imageUrl":"","docLink":"","videoLink":""}} \ No newline at end of file diff --git a/packages/core/tests/data/AgentData/unit-tests-agent-randnum-randwords.smyth b/packages/core/tests/data/AgentData/unit-tests-agent-randnum-randwords.smyth new file mode 100644 index 000000000..7f6d16a9b --- /dev/null +++ b/packages/core/tests/data/AgentData/unit-tests-agent-randnum-randwords.smyth @@ -0,0 +1 @@ +{"version":"1.0.0","id":"cme6bzglr1pw1wzc2wpzb9kkv","name":"Unit Tests Agent","teamId":"9","parentTeamId":"9","components":[{"id":"MEQYL55P3HB","name":"APIEndpoint","outputs":[{"name":"headers","color":"#3C89F9","index":0,"default":true},{"name":"body","color":"#3C89F9","index":1,"default":true},{"name":"query","color":"#3C89F9","index":2,"default":true},{"name":"user_query","color":"#3C89F9","expression":"body.user_query","index":3,"default":false}],"inputs":[{"name":"user_query","type":"Any","friendlyInputType":"","coreInputType":"","color":"#F35063","optional":false,"description":"The user query to search for","index":0,"default":false}],"data":{"method":"POST","endpoint":"version","endpointLabel":"version","description":"Returns agent version number","ai_exposed":true,"summary":"","advancedModeEnabled":false},"top":"53.3005px","left":"-3px","width":"","height":"","displayName":"APIEndpoint","title":"Version","aiTitle":"","description":"An API Endpoint with input parameters for your AI agent."},{"id":"MEQYL55PUUA","name":"GenAILLM","outputs":[{"name":"Reply","color":"#3C89F9","index":0,"default":true}],"inputs":[{"name":"Input","type":"Any","friendlyInputType":"","coreInputType":"","color":"#F35063","optional":false,"index":0,"default":false}],"data":{"model":"Echo","prompt":"v1.0.5","temperature":"1","maxTokens":"1024","stopSequences":"","topP":"1","topK":"0","frequencyPenalty":"0","presencePenalty":"0","passthrough":false,"useSystemPrompt":false,"useContextWindow":false,"maxContextWindowLength":"100","useWebSearch":false,"webSearchContextSize":"medium","webSearchCity":"","webSearchCountry":"","webSearchRegion":"","webSearchTimezone":"","useSearch":false,"searchMode":"auto","returnCitations":false,"maxSearchResults":"10","searchDataSources":["web","x"],"searchCountry":"","excludedWebsites":"","allowedWebsites":"","includedXHandles":"","excludedXHandles":"","postFavoriteCount":0,"postViewCount":0,"rssLinks":"","safeSearch":false,"fromDate":"","toDate":"","useReasoning":false,"maxThinkingTokens":"4096","reasoningEffort":"","verbosity":"medium"},"top":"33px","left":"345px","width":"","height":"","displayName":"GenAI LLM","title":"Version data","aiTitle":"","description":"Make a GenAI request to an AI model. Works with many input file types."},{"id":"CMFF8M36DM5O","name":"APIOutput","outputs":[{"name":"Output","color":"#3C89F9","index":0,"default":true}],"inputs":[{"name":"result","type":"Any","friendlyInputType":"","coreInputType":"Any","color":"#F35063","optional":false,"index":0,"default":false}],"data":{"format":"minimal"},"top":"105px","left":"835px","width":"250px","height":"","displayName":"APIOutput","title":"APIOutput","aiTitle":"","description":""},{"id":"CMFQX73B9EQ","name":"APIEndpoint","outputs":[{"name":"headers","color":"#3C89F9","index":0,"default":true},{"name":"body","color":"#3C89F9","index":1,"default":true},{"name":"query","color":"#3C89F9","index":2,"default":true},{"name":"min","color":"#3C89F9","expression":"body.min","index":3,"default":false},{"name":"max","color":"#3C89F9","expression":"body.max","index":4,"default":false}],"inputs":[{"name":"min","type":"Number","friendlyInputType":"Others","coreInputType":"Number","color":"#F35063","optional":false,"description":"the minimum number","index":0,"default":false},{"name":"max","type":"Number","friendlyInputType":"Others","coreInputType":"Number","color":"#F35063","optional":false,"description":"the max number","index":1,"default":false}],"data":{"method":"POST","endpoint":"rand_number","endpointLabel":"rand_number","description":"This skill can be used to generate a random number between min and max","ai_exposed":true,"summary":"","advancedModeEnabled":false,"status_message":""},"top":"524.376px","left":"26.5895px","width":"","height":"","displayName":"APIEndpoint","title":"Agent Skill","aiTitle":"","description":"An API Endpoint with input parameters for your AI agent."},{"id":"CMFQX9TE856","name":"GenAILLM","outputs":[{"name":"Reply","color":"#3C89F9","index":0,"default":true},{"name":"number","color":"#3C89F9","index":1,"default":false}],"inputs":[{"name":"min","type":"Any","friendlyInputType":"","coreInputType":"Any","color":"#F35063","optional":false,"index":0,"default":false},{"name":"max","type":"Any","friendlyInputType":"","coreInputType":"Any","color":"#F35063","optional":false,"index":1,"default":false}],"data":{"model":"gpt-4o-mini","prompt":"generate a random number between {{min}} and {{max}}","temperature":"1","maxTokens":"8192","stopSequences":"","topP":"1","topK":"0","frequencyPenalty":"0","presencePenalty":"0","passthrough":false,"useSystemPrompt":false,"useContextWindow":false,"maxContextWindowLength":"4096","useWebSearch":false,"webSearchContextSize":"medium","webSearchCity":"","webSearchCountry":"","webSearchRegion":"","webSearchTimezone":"","useSearch":false,"searchMode":"auto","returnCitations":false,"maxSearchResults":"10","searchDataSources":["web","x"],"searchCountry":"","excludedWebsites":"","allowedWebsites":"","includedXHandles":"","excludedXHandles":"","postFavoriteCount":0,"postViewCount":0,"rssLinks":"","safeSearch":false,"fromDate":"","toDate":"","useReasoning":false,"maxThinkingTokens":"4096","reasoningEffort":"","verbosity":"medium"},"top":"505.844px","left":"392.025px","width":"","height":"","displayName":"GenAI LLM","title":"GenAI LLM","aiTitle":"Random Number Generator","description":"Make a GenAI request to an AI model. Works with many input file types."},{"id":"MFQXBYVO2KE","name":"APIOutput","outputs":[{"name":"Output","color":"#3C89F9","index":0,"default":true}],"inputs":[{"name":"number","type":"Any","friendlyInputType":"","coreInputType":"Any","color":"#F35063","optional":false,"index":0,"default":false}],"data":{"format":"minimal"},"top":"562.422px","left":"846.169px","width":"250px","height":"","displayName":"APIOutput","title":"APIOutput","aiTitle":"","description":""},{"id":"CMFQYJP4XM5O","name":"APIEndpoint","outputs":[{"name":"headers","color":"#3C89F9","index":0,"default":true},{"name":"body","color":"#3C89F9","index":1,"default":true},{"name":"query","color":"#3C89F9","index":2,"default":true},{"name":"topic","color":"#3C89F9","expression":"body.topic","index":3,"default":false},{"name":"number","color":"#3C89F9","expression":"body.number","index":4,"default":false}],"inputs":[{"name":"topic","type":"Text","friendlyInputType":"Text","coreInputType":"Any","color":"#F35063","optional":false,"description":"The topic to generate keywords from","index":0,"default":false},{"name":"number","type":"Number","friendlyInputType":"Others","coreInputType":"Number","color":"#F35063","optional":false,"description":"The number of keywords to generate","index":1,"default":false}],"data":{"method":"POST","endpoint":"rand_words","endpointLabel":"rand_words","description":"generates a list of random words from a topic","ai_exposed":true,"summary":"","advancedModeEnabled":false,"status_message":""},"top":"960px","left":"71px","width":"","height":"","displayName":"APIEndpoint","title":"Agent Skill","aiTitle":"","description":"An API Endpoint with input parameters for your AI agent."},{"id":"CMFQYKMAD2U7","name":"GenAILLM","outputs":[{"name":"Reply","color":"#3C89F9","index":0,"default":true},{"name":"keywords_list","color":"#3C89F9","index":1,"default":false}],"inputs":[{"name":"topic","type":"Any","friendlyInputType":"","coreInputType":"Any","color":"#F35063","optional":false,"index":0,"default":false},{"name":"number","type":"Number","friendlyInputType":"","coreInputType":"Number","color":"#F35063","optional":false,"index":1,"default":false}],"data":{"model":"gpt-4o-mini","prompt":"Generate {{number}} keywords about : {{topic}}","temperature":"1","maxTokens":"8192","stopSequences":"","topP":"1","topK":"0","frequencyPenalty":"0","presencePenalty":"0","passthrough":false,"useSystemPrompt":false,"useContextWindow":false,"maxContextWindowLength":"4096","useWebSearch":false,"webSearchContextSize":"medium","webSearchCity":"","webSearchCountry":"","webSearchRegion":"","webSearchTimezone":"","useSearch":false,"searchMode":"auto","returnCitations":false,"maxSearchResults":"10","searchDataSources":["web","x"],"searchCountry":"","excludedWebsites":"","allowedWebsites":"","includedXHandles":"","excludedXHandles":"","postFavoriteCount":0,"postViewCount":0,"rssLinks":"","safeSearch":false,"fromDate":"","toDate":"","useReasoning":false,"maxThinkingTokens":"4096","reasoningEffort":"medium","verbosity":"medium"},"top":"977px","left":"461px","width":"","height":"","displayName":"GenAI LLM","title":"GenAI LLM","aiTitle":"Keyword Generation Tool","description":"Make a GenAI request to an AI model. Works with many input file types."},{"id":"MFQYNG8R61P","name":"APIOutput","outputs":[{"name":"Output","color":"#3C89F9","index":0,"default":true}],"inputs":[{"name":"keywords","type":"Any","friendlyInputType":"","coreInputType":"Any","color":"#F35063","optional":false,"index":0,"default":false}],"data":{"format":"minimal"},"top":"1037px","left":"912px","width":"250px","height":"","displayName":"APIOutput","title":"APIOutput","aiTitle":"","description":""}],"connections":[{"sourceId":"MEQYL55P3HB","sourceIndex":3,"targetId":"MEQYL55PUUA","targetIndex":0},{"sourceId":"MEQYL55PUUA","sourceIndex":0,"targetId":"CMFF8M36DM5O","targetIndex":0},{"sourceId":"CMFQX73B9EQ","sourceIndex":3,"targetId":"CMFQX9TE856","targetIndex":0},{"sourceId":"CMFQX73B9EQ","sourceIndex":4,"targetId":"CMFQX9TE856","targetIndex":1},{"sourceId":"CMFQX9TE856","sourceIndex":1,"targetId":"MFQXBYVO2KE","targetIndex":0},{"sourceId":"CMFQYJP4XM5O","sourceIndex":3,"targetId":"CMFQYKMAD2U7","targetIndex":0},{"sourceId":"CMFQYJP4XM5O","sourceIndex":4,"targetId":"CMFQYKMAD2U7","targetIndex":1},{"sourceId":"CMFQYKMAD2U7","sourceIndex":1,"targetId":"MFQYNG8R61P","targetIndex":0}],"description":"","shortDescription":"Test Agent","behavior":"A test agent to check LLM capabilities and tool calls","ui":{"panzoom":{"currentPan":{"x":316.2973464389816,"y":-290.40344575157724},"currentZoom":0.7700431458051551},"agentCard":{"left":"-567.54px","top":"344.629px"}},"introMessage":"Hi, I'm Unit Tests Agent. How can I help you today?","templateInfo":{"id":"unit-tests-agent-mfqzgvy314","name":"Unit Tests Agent","description":"Test Agent","icon":"","category":"","publish":false,"color":"#000000","imageUrl":"","docLink":"","videoLink":"","valueProposition":""}} \ No newline at end of file diff --git a/packages/core/tests/data/account.json b/packages/core/tests/data/account.json index f79218669..3a60b3db0 100644 --- a/packages/core/tests/data/account.json +++ b/packages/core/tests/data/account.json @@ -24,7 +24,114 @@ }, "settings": { "custom-llm": "{\"m192qxzvp6i\":{\"id\":\"m192qxzvp6i\",\"name\":\"SRE - Bedrock with AI21 Labs - Jamba-Instruct\",\"provider\":\"Bedrock\",\"features\":[\"text-completion\"],\"components\":[\"PromptGenerator\",\"LLMAssistant\"],\"tags\":[\"Bedrock\"],\"tokens\":256000,\"completionTokens\":4096,\"settings\":{\"foundationModel\":\"ai21.jamba-instruct-v1:0\",\"customModel\":\"\",\"region\":\"us-east-1\",\"keyIDName\":\"AWS Key ID (m192qxzvp6i)\",\"secretKeyName\":\"AWS Secret Key (m192qxzvp6i)\",\"sessionKeyName\":\"\"}},\"m19sryru23\":{\"id\":\"m19sryru23\",\"name\":\"SRE - Vertex AI with Gemini Flash\",\"provider\":\"VertexAI\",\"features\":[\"text-completion\"],\"components\":[\"PromptGenerator\",\"LLMAssistant\"],\"tags\":[\"VertexAI\"],\"tokens\":1048576,\"completionTokens\":8192,\"settings\":{\"foundationModel\":\"gemini-1.5-flash\",\"customModel\":\"\",\"region\":\"us-central1\",\"projectId\":\"opt-smythos-vertexia-432522\",\"jsonCredentialsName\":\"Google JSON Credentials (m19sryru23)\"}},\"m3e980g1y1\":{\"id\":\"m3e980g1y1\",\"name\":\"SRE - Bedrock does not support System Prompt\",\"provider\":\"Bedrock\",\"features\":[\"text-completion\"],\"tags\":[\"Bedrock\"],\"settings\":{\"foundationModel\":\"mistral.mistral-7b-instruct-v0:2\",\"customModel\":\"\",\"region\":\"us-east-1\",\"keyIDName\":\"AWS Key ID (m3e980g1y1)\",\"secretKeyName\":\"AWS Secret Key (m3e980g1y1)\",\"sessionKeyName\":\"\"}},\"m3iws4lisbh\":{\"id\":\"m3iws4lisbh\",\"name\":\"SRE - Bedrock for Tool Use\",\"provider\":\"Bedrock\",\"features\":[\"text-completion\"],\"tags\":[\"Bedrock\"],\"settings\":{\"foundationModel\":\"us.anthropic.claude-3-5-haiku-20241022-v1:0\",\"customModel\":\"\",\"region\":\"us-east-1\",\"keyIDName\":\"AWS Key ID (m3iws4lisbh)\",\"secretKeyName\":\"AWS Secret Key (m3iws4lisbh)\",\"sessionKeyName\":\"\"}},\"m3x7hrhm9ue\":{\"id\":\"m3x7hrhm9ue\",\"name\":\"Bedrock Plugin\",\"provider\":\"Bedrock\",\"features\":[\"text-completion\",\"tool-use\"],\"tags\":[\"Bedrock\"],\"settings\":{\"foundationModel\":\"ai21.jamba-1-5-mini-v1:0\",\"customModel\":\"\",\"region\":\"us-east-1\",\"keyIDName\":\"AWS Key ID (m3x7hrhm9ue)\",\"secretKeyName\":\"AWS Secret Key (m3x7hrhm9ue)\",\"sessionKeyName\":\"\"}},\"m6c65ou1b8d\":{\"id\":\"m6c65ou1b8d\",\"name\":\"Bedrock LLama\",\"provider\":\"Bedrock\",\"features\":[\"text\"],\"tags\":[\"Bedrock\"],\"settings\":{\"foundationModel\":\"us.meta.llama3-2-1b-instruct-v1:0\",\"customModel\":\"\",\"region\":\"us-east-1\",\"keyIDName\":\"AWS Key ID (m6c65ou1b8d)\",\"secretKeyName\":\"AWS Secret Key (m6c65ou1b8d)\",\"sessionKeyName\":\"\"}}}", + "vault": "default-vault-config", "setting2": "value2" } + }, + "TeamAPICall": { + "agents": { + "image-agent": { + "settings": {} + } + } + }, + "Team2": { + "users": { + "user1": { + "settings": { + "setting1": "value1", + "setting2": "value2" + } + } + }, + "agents": { + "agent-123456": { + "settings": { + "setting1": "value1", + "setting2": "value2" + } + }, + "agent-007": { + "settings": {} + } + } + }, + "development": { + "users": { + "dev-user1": { + "settings": { + "setting1": "dev-value1", + "environment": "development", + "debug": "true" + } + }, + "dev-user2": { + "settings": { + "setting1": "dev-value1", + "role": "developer", + "access-level": "standard" + } + } + }, + "agents": { + "dev-agent1": { + "settings": { + "setting1": "dev-value1", + "mode": "development", + "logging": "verbose" + } + }, + "test-agent": { + "settings": { + "setting1": "test-value", + "environment": "testing" + } + } + }, + "settings": { + "api-endpoint": "https://dev-api.smythos.com", + "vault": "dev-vault-config", + "debug-mode": "enabled" + } + }, + "production": { + "users": { + "prod-user1": { + "settings": { + "setting1": "prod-value1", + "environment": "production", + "audit-logging": "enabled" + } + }, + "admin-user": { + "settings": { + "setting1": "admin-value", + "role": "administrator", + "access-level": "full" + } + } + }, + "agents": { + "prod-agent1": { + "settings": { + "setting1": "prod-value1", + "mode": "production", + "monitoring": "enabled" + } + }, + "critical-agent": { + "settings": { + "setting1": "critical-value", + "priority": "high", + "failover": "enabled" + } + } + }, + "settings": { + "api-endpoint": "https://api.smythos.com", + "vault": "prod-vault-config", + "security-level": "maximum", + "backup-interval": "hourly" + } } } diff --git a/packages/core/tests/data/dummy-article.txt b/packages/core/tests/data/dummy-article.txt new file mode 100644 index 000000000..c7fc2a524 --- /dev/null +++ b/packages/core/tests/data/dummy-article.txt @@ -0,0 +1,40 @@ +**Title: The Transformative Impact of Artificial Intelligence in the Medical Industry** + +### **Introduction: The Transformative Impact of AI in the Medical Industry** + +Artificial Intelligence (AI) has emerged as a groundbreaking force in various fields, with the medical industry standing out as one of the most significantly affected. The intersection of AI and healthcare represents a pivotal moment in the evolution of medicine, promising advancements that could reshape diagnostics, treatment, patient care, and operational efficiencies. As AI technologies continue to evolve, their integration into the medical field offers unprecedented opportunities for improving patient outcomes and optimizing healthcare systems. + +**Historical Context and Evolution** + +The concept of AI in medicine is not entirely new. Early developments in medical AI were primarily focused on rule-based systems and expert systems designed to assist with diagnosis and treatment planning. However, the real breakthrough came with the advent of machine learning and deep learning technologies, which enabled more sophisticated and accurate models capable of analyzing vast amounts of data. This evolution has paved the way for the current era of AI-driven healthcare innovations. + +**AI in Diagnostics** + +One of the most prominent areas where AI is making a significant impact is in medical diagnostics. AI-powered tools are revolutionizing the way diseases are detected and diagnosed, offering enhanced accuracy and speed compared to traditional methods. In radiology, for example, AI algorithms are used to analyze medical images, such as X-rays, CT scans, and MRIs, to identify abnormalities and assist radiologists in making more precise diagnoses. Similarly, in pathology, AI systems can analyze tissue samples and detect cancerous cells with remarkable accuracy. + +**AI in Treatment** + +AI is also transforming treatment modalities. Robotic surgery, powered by AI, allows for greater precision and control during surgical procedures, reducing recovery times and improving patient outcomes. Additionally, AI is playing a crucial role in drug discovery and development, accelerating the process of identifying potential new treatments and streamlining clinical trials. By analyzing large datasets and predicting patient responses, AI helps in personalizing treatment plans, ensuring that patients receive therapies tailored to their specific needs. + +**AI in Patient Care** + +In the realm of patient care, AI technologies are enhancing the way patients interact with healthcare providers. Virtual health assistants, powered by AI chatbots, provide patients with real-time support and information, improving engagement and adherence to treatment plans. Remote monitoring and telemedicine platforms, enabled by AI, allow for continuous patient monitoring and virtual consultations, making healthcare more accessible and convenient. Furthermore, AI applications in mental health are offering new ways to assess and treat psychological conditions, providing support through digital therapeutic interventions. + +**AI in Healthcare Operations** + +Beyond clinical applications, AI is optimizing healthcare operations. Administrative tasks, such as medical billing and coding, are being streamlined with AI technologies, reducing errors and increasing efficiency. AI is also playing a role in resource management, predicting patient admission rates and optimizing supply chains. Data management and security are further enhanced by AI, which helps in analyzing health data while ensuring compliance with privacy regulations. + +**Ethical Considerations and Challenges** + +While AI presents numerous benefits, it also raises important ethical considerations. Addressing bias and ensuring fairness in AI algorithms is crucial to prevent disparities in healthcare delivery. Regulatory compliance and patient privacy concerns are also significant, requiring careful management to balance innovation with ethical standards. As AI continues to advance, it is essential to address these challenges proactively to ensure equitable and responsible use of technology in healthcare. + +**Future Directions and Innovations** + +The future of AI in healthcare holds exciting possibilities. Emerging technologies, such as AI-powered genomics and advanced diagnostic tools, promise to further enhance medical practice. Integrating AI with other technologies, such as blockchain and IoT, could lead to even more innovative solutions. The long-term impact of AI on healthcare delivery will likely involve changes in the roles of healthcare professionals and the evolution of healthcare systems, driven by continuous advancements in AI technology. + +**Conclusion** + +The transformative impact of AI in the medical industry is already evident, with ongoing advancements promising even greater improvements in the future. As AI technologies continue to evolve, they will play a crucial role in shaping the future of healthcare, driving innovations that enhance patient care, optimize operations, and revolutionize medical practices. The integration of AI into the medical field represents a significant step forward, offering the potential to improve health outcomes and transform the way healthcare is delivered. + + +This introduction and outline should provide a solid foundation for a comprehensive 10,000-word article on the impact of AI in the medical industry. Each section can be expanded with detailed information, case studies, and examples to reach the desired length. diff --git a/packages/core/tests/data/dummy-input-messages.json b/packages/core/tests/data/dummy-input-messages.json new file mode 100644 index 000000000..b181494d3 --- /dev/null +++ b/packages/core/tests/data/dummy-input-messages.json @@ -0,0 +1,50 @@ +[ + { + "role": "system", + "content": "You are an expert astronomer with extensive knowledge of our solar system and beyond. Provide detailed, scientifically accurate information about various celestial bodies, their characteristics, and their roles in the cosmos. Include interesting facts and recent discoveries when relevant." + }, + { + "role": "user", + "content": "Can you give me a comprehensive overview of the Sun, including its structure, importance to our solar system, and any recent discoveries?" + }, + { + "role": "assistant", + "content": "Certainly! The Sun is the heart of our solar system, a massive ball of plasma held together by its own gravity. It's classified as a G-type main-sequence star, also known as a yellow dwarf. The Sun's structure consists of several layers: the core, radiative zone, convective zone, photosphere, chromosphere, and corona. In the core, nuclear fusion converts hydrogen into helium, releasing enormous amounts of energy that powers life on Earth. Recent discoveries from missions like Parker Solar Probe have revealed new insights into the Sun's atmosphere and magnetic field, helping us better understand solar wind and space weather phenomena." + }, + { + "role": "user", + "content": "That's fascinating! Now, could you elaborate on Earth's unique features that make it habitable, and discuss some of the current environmental challenges we're facing?" + }, + { + "role": "assistant", + "content": "Earth, the third planet from the Sun, is often called the \"Blue Planet\" due to its abundant water. It's the only known planet to support life, thanks to a combination of factors: its distance from the Sun (within the habitable zone), its protective magnetic field, a breathable atmosphere rich in oxygen, and liquid water on its surface. Earth's atmosphere consists mainly of nitrogen and oxygen, with trace amounts of other gases. The planet's surface is dynamic, shaped by plate tectonics, which drive the formation of mountains, earthquakes, and volcanoes. Currently, Earth faces significant environmental challenges, primarily due to human activities. These include climate change caused by greenhouse gas emissions, deforestation, ocean acidification, and loss of biodiversity. Scientists are working on solutions to these issues, including renewable energy technologies and conservation efforts." + }, + { + "role": "user", + "content": "Thank you for that comprehensive explanation. Let's move on to the Moon. Can you describe its formation, its effects on Earth, and its importance in space exploration?" + }, + { + "role": "assistant", + "content": "The Moon, Earth's only natural satellite, is believed to have formed about 4.5 billion years ago, likely from the debris of a collision between Earth and a Mars-sized object. It's about a quarter the diameter of Earth and has a significant impact on our planet. The Moon's gravitational pull causes ocean tides and slightly lengthens Earth's day. It has no substantial atmosphere and is covered in craters, mountains, and vast plains called maria. The Moon has been crucial in space exploration, serving as the first celestial body humans have set foot on during the Apollo missions. It continues to be a focus of space agencies worldwide, with plans for establishing permanent bases and using it as a stepping stone for deeper space exploration. Recent missions have discovered water ice in permanently shadowed craters at the lunar poles, which could be a valuable resource for future lunar inhabitants." + }, + { + "role": "user", + "content": "Intriguing! Now, let's discuss Mars. What makes it a target for potential human colonization, and what are some of the major discoveries made by recent Mars missions?" + }, + { + "role": "assistant", + "content": "Mars, often called the \"Red Planet\" due to its reddish appearance caused by iron oxide on its surface, is the fourth planet from the Sun. It's a prime target for potential human colonization due to several factors: its day length and axial tilt are similar to Earth's, it has a thin atmosphere, and there's evidence of subsurface water ice. Recent missions like NASA's Perseverance rover and China's Tianwen-1 have made significant discoveries. Perseverance has successfully extracted oxygen from the Martian atmosphere, a crucial step for future human missions. It has also collected rock samples for future return to Earth. The Ingenuity helicopter, deployed by Perseverance, achieved the first powered, controlled flight on another planet. Earlier missions discovered complex organic molecules in rocks and seasonal variations in methane levels, hinting at the possibility of past or present microbial life. The planet has two small, irregularly shaped moons: Phobos and Deimos." + }, + { + "role": "user", + "content": "That's remarkable progress in Mars exploration! Lastly, can you tell me about Jupiter, its unique features, and its role in our solar system?" + }, + { + "role": "assistant", + "content": "Jupiter, the largest planet in our solar system, is a gas giant primarily composed of hydrogen and helium. It's so massive that it has more than twice the mass of all other planets combined. Jupiter plays a crucial role in the solar system as a \"cosmic vacuum cleaner,\" its strong gravity attracting and capturing many comets and asteroids that might otherwise threaten Earth. The planet is known for its Great Red Spot, a massive storm that has been raging for at least 400 years. Jupiter has a complex system of at least 79 known moons, including the four large Galilean moons: Io, Europa, Ganymede, and Callisto. Each of these moons is unique: Io is the most volcanically active body in the solar system, Europa likely has a subsurface ocean that could potentially harbor life, Ganymede is the largest moon in the solar system and has its own magnetic field, and Callisto is one of the most heavily cratered objects in the solar system. Recent missions like Juno have provided new insights into Jupiter's interior structure, magnetic field, and atmospheric dynamics, revealing that the planet's iconic bands extend far deeper than previously thought." + }, + { + "role": "user", + "content": "Fascinating information about Jupiter! Can you tell me about some of the dwarf planets in our solar system, particularly Pluto and its unique characteristics?" + } +] diff --git a/packages/core/tests/data/smythos.png b/packages/core/tests/data/smythos.png new file mode 100644 index 000000000..101e0cd9d Binary files /dev/null and b/packages/core/tests/data/smythos.png differ diff --git a/packages/core/tests/data/vault.fake.json b/packages/core/tests/data/vault.fake.json new file mode 100644 index 000000000..9c29f261d --- /dev/null +++ b/packages/core/tests/data/vault.fake.json @@ -0,0 +1,22 @@ +{ + "default": { + "echo": "", + "openai": "sk-proj-openai", + "anthropic": "sk-ant-anthropic", + "googleai": "#", + "groq": "#", + "togetherai": "#", + "xai": "#", + "deepseek": "#", + "my-key": "Hello world", + "DIFFBOT_API": "THIS_IS_A_FAKE_DIFFBOT_API_KEY" + }, + "Team2": { + "echo": "", + "openai": "sk-proj-team2-openai", + "anthropic": "sk-ant-team2-anthropic", + "team2-secret": "team2-secret-value", + "TEAM2_API": "THIS_IS_A_FAKE_TEAM2_API_KEY", + "SRE TEST KEY": "sdl7k8lsd93ko4iu39" + } +} diff --git a/packages/core/tests/integration/003-LLM/conversation.test.ts b/packages/core/tests/integration/003-LLM/conversation.test.ts new file mode 100644 index 000000000..21e3d0ed1 --- /dev/null +++ b/packages/core/tests/integration/003-LLM/conversation.test.ts @@ -0,0 +1,129 @@ +import { describe, expect, it } from 'vitest'; +import { SmythRuntime, TLLMEvent } from 'index'; +import { Conversation } from '@sre/helpers/Conversation.helper'; +import { setupSRE } from '../../utils/sre'; +import fs from 'fs'; +import { testData } from '../../utils/test-data-manager'; + +setupSRE(); +const TIMEOUT = 30000; +const LLM_OUTPUT_VALIDATOR = 'Yohohohooooo!'; +const WORD_INCLUSION_PROMPT = `\nIMPORTANT : The response should start with "${LLM_OUTPUT_VALIDATOR}".`; + +function runTestCases(model: string) { + it( + 'runs a conversation with tool use', + async () => { + const spec = JSON.parse(fs.readFileSync(testData.getDataPath('AgentData/unit-tests-agent-randnum-randwords.smyth'), 'utf8')); + + const conv = new Conversation(model, spec); + + const prompt = 'What is your version number ?'; + + const result = await conv.prompt(prompt); + + expect(result).toBeDefined(); + expect(result).toContain('v1.0.5'); // This version number is hardcoded in the skill, if the correct version is returned it means that the agent called the skill correctly + }, + TIMEOUT + ); + + it( + 'runs a conversation with tool use in stream mode', + async () => { + const spec = JSON.parse(fs.readFileSync(testData.getDataPath('AgentData/unit-tests-agent-randnum-randwords.smyth'), 'utf8')); + const conv = new Conversation(model, spec); + + let streamResult = ''; + + // * The order is important to ensure proper event handling: + // 1. Set up event listeners before calling streamPrompt to capture all events. ie. const streamComplete = new Promise((resolve) => {... + // 2. Call streamPrompt to initiate the streaming process. ie. const result = await conv.streamPrompt(...); + // 3. Wait for the stream to complete to ensure all content is received. ie. await streamComplete; + const streamComplete = new Promise((resolve) => { + conv.on(TLLMEvent.Content, (content) => { + streamResult += content; + }); + conv.on(TLLMEvent.End, resolve); + }); + + const prompt = 'What is your version number ?'; + + const result = await conv.streamPrompt(prompt); + + await streamComplete; + + expect(result).toBeDefined(); + expect(streamResult).toBeTruthy(); + expect(streamResult).toContain('v1.0.5'); // This version number is hardcoded in the skill, if the correct version is returned it means that the agent called the skill correctly + }, + TIMEOUT + ); + + it( + 'handles multiple tool calls in a single conversation', + async () => { + const spec = JSON.parse(fs.readFileSync(testData.getDataPath('AgentData/unit-tests-agent-randnum-randwords.smyth'), 'utf8')); + const conv = new Conversation(model, spec); + + let streamResult = ''; + + // * The order is important to ensure proper event handling: + // 1. Set up event listeners before calling streamPrompt to capture all events. ie. const streamComplete = new Promise((resolve) => {... + // 2. Call streamPrompt to initiate the streaming process. ie. const result = await conv.streamPrompt(...); + // 3. Wait for the stream to complete to ensure all content is received. ie. await streamComplete; + const streamComplete = new Promise((resolve) => { + conv.on(TLLMEvent.Content, (content) => { + streamResult += content; + }); + conv.on(TLLMEvent.End, resolve); + }); + + const prompt = + "First, Generate a random number between 42 and 100, then generate 5 keywords about The Hitchhiker's Guide to the Galaxy" + + WORD_INCLUSION_PROMPT; + + const result = await conv.streamPrompt(prompt); + + await streamComplete; + + expect(result).toBeTruthy(); + expect(streamResult).toBeTruthy(); + expect(streamResult).toContain(LLM_OUTPUT_VALIDATOR); + }, + TIMEOUT * 2 + ); + + it( + 'handles follow-up questions correctly', + async () => { + const spec = JSON.parse(fs.readFileSync(testData.getDataPath('AgentData/unit-tests-agent-randnum-randwords.smyth'), 'utf8')); + const conv = new Conversation(model, spec); + + const prompt = 'What is your version number ?' + WORD_INCLUSION_PROMPT; + + await conv.prompt(prompt); + + const followUpPrompt = "Generate 5 keywords about The Hitchhiker's Guide to the Galaxy" + WORD_INCLUSION_PROMPT; + const followUpResult = await conv.prompt(followUpPrompt); + + expect(followUpResult).toBeTruthy(); + expect(followUpResult).toContain(LLM_OUTPUT_VALIDATOR); + }, + TIMEOUT * 2 + ); +} + +const models = [ + { provider: 'OpenAI', id: 'gpt-4o-mini' }, + { provider: 'Anthropic', id: 'claude-3.5-haiku' }, + { provider: 'GoogleAI', id: 'gemini-1.5-flash' }, + /* { provider: 'Groq', id: 'gemma2-9b-it' }, + { provider: 'TogetherAI', id: 'meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo' }, */ +]; + +for (const model of models) { + describe(`Conversation Tests: ${model.provider} (${model.id})`, async () => { + await runTestCases(model.id); + }); +} diff --git a/packages/core/tests/unit/embeddings/GoogleEmbedding.test.ts b/packages/core/tests/integration/003-LLM/googleai/Embedding.test.ts similarity index 100% rename from packages/core/tests/unit/embeddings/GoogleEmbedding.test.ts rename to packages/core/tests/integration/003-LLM/googleai/Embedding.test.ts diff --git a/packages/core/tests/integration/003-LLM/multimodal.test.ts b/packages/core/tests/integration/003-LLM/multimodal.test.ts new file mode 100644 index 000000000..43ee3b3c8 --- /dev/null +++ b/packages/core/tests/integration/003-LLM/multimodal.test.ts @@ -0,0 +1,175 @@ +import { Conversation } from '@sre/helpers/Conversation.helper'; +import fs from 'fs'; +import { describe, expect, it } from 'vitest'; +import { setupSRE } from '../../utils/sre'; +import { testData } from '../../utils/test-data-manager'; + +setupSRE(); + +const TIMEOUT = 30000; +const LLM_OUTPUT_VALIDATOR = 'Yohohohooooo!'; +const WORD_INCLUSION_PROMPT = `\nIMPORTANT : The response should start with "${LLM_OUTPUT_VALIDATOR}".`; + +function runMultimodalTestCases(model: string) { + const imageUrl1 = 'https://images.unsplash.com/photo-1721332155637-8b339526cf4c?q=10&w=300'; + const imageUrl2 = 'https://plus.unsplash.com/premium_photo-1732410903106-3379bbe6e9db?q=10&w=300'; + const audioUrl = 'https://actions.google.com/sounds/v1/foley/play_in_pile_of_leaves.ogg'; + const videoUrl = 'https://storage.googleapis.com/generativeai-downloads/images/GreatRedSpot.mp4'; + const pdfUrl = 'https://www.princexml.com/samples/invoice/invoicesample.pdf'; + + it( + `runs a simple multimodal request with a single image for Model: ${model}`, + async () => { + const spec = JSON.parse(fs.readFileSync(testData.getDataPath('AgentData/unit-tests-agent-randnum-randwords.smyth'), 'utf8')); + const conv = new Conversation(model, spec); + + const prompt = 'What is in this image?' + WORD_INCLUSION_PROMPT; + const fileSources = [imageUrl1]; + const result: any = await conv.prompt({ message: prompt, files: fileSources }); + expect(result).toBeTruthy(); + expect(result.length).toBeGreaterThan(20); + expect(result).toContain(LLM_OUTPUT_VALIDATOR); + }, + TIMEOUT + ); + + it( + `handles multiple images in a single request for Model: ${model}`, + async () => { + const spec = JSON.parse(fs.readFileSync(testData.getDataPath('AgentData/unit-tests-agent-randnum-randwords.smyth'), 'utf8')); + const conv = new Conversation(model, spec); + + const prompt = 'Compare these two images' + WORD_INCLUSION_PROMPT; + const fileSources = [imageUrl1, imageUrl2]; + const result: any = await conv.prompt({ message: prompt, files: fileSources }); + expect(result).toBeTruthy(); + expect(result.length).toBeGreaterThan(20); + expect(result).toContain(LLM_OUTPUT_VALIDATOR); + }, + TIMEOUT + ); + + it( + `handles empty file sources array for Model: ${model}`, + async () => { + const spec = JSON.parse(fs.readFileSync(testData.getDataPath('AgentData/unit-tests-agent-randnum-randwords.smyth'), 'utf8')); + const conv = new Conversation(model, spec); + + const fileSources = []; + const result: any = await conv.prompt({ message: 'Analyze this data', files: fileSources }); + expect(result).toBeTruthy(); + expect(result.length).toBeGreaterThan(20); + expect(result).toContain(LLM_OUTPUT_VALIDATOR); + }, + TIMEOUT + ); + + it( + `handles complex prompts with multiple file types for Model: ${model}`, + async () => { + const spec = JSON.parse(fs.readFileSync(testData.getDataPath('AgentData/unit-tests-agent-randnum-randwords.smyth'), 'utf8')); + const conv = new Conversation(model, spec); + + const complexPrompt = + 'IMPORTANT INSTRUCTION: First include the word "' + + LLM_OUTPUT_VALIDATOR + + '" in your response.\n\n' + + 'Then analyze these files in detail. Describe the visual elements in the image, the audio content, and the document content. Then, speculate about how they might be related.' + + WORD_INCLUSION_PROMPT; + const fileSources = [imageUrl1, audioUrl, pdfUrl]; + const result: any = await conv.prompt({ message: complexPrompt, files: fileSources }); + expect(result).toBeTruthy(); + expect(result.length).toBeGreaterThan(20); + expect(result).toContain(LLM_OUTPUT_VALIDATOR); + }, + TIMEOUT * 5 + ); + + it( + `handles prompts with special characters and Unicode for Model: ${model}`, + async () => { + const spec = JSON.parse(fs.readFileSync(testData.getDataPath('AgentData/unit-tests-agent-randnum-randwords.smyth'), 'utf8')); + const conv = new Conversation(model, spec); + + const specialCharsPrompt = 'Describe these files: 🌍🚀 こんにちは! 你好! مرحبا!' + WORD_INCLUSION_PROMPT; + const fileSources = [imageUrl1, audioUrl]; + const result: any = await conv.prompt({ message: specialCharsPrompt, files: fileSources }); + expect(result).toBeTruthy(); + expect(result.length).toBeGreaterThan(20); + expect(result).toContain(LLM_OUTPUT_VALIDATOR); + }, + TIMEOUT + ); + + it( + `handles a mix of image and text files for Model: ${model}`, + async () => { + const spec = JSON.parse(fs.readFileSync(testData.getDataPath('AgentData/unit-tests-agent-randnum-randwords.smyth'), 'utf8')); + const conv = new Conversation(model, spec); + + const prompt = 'Compare the content of the image with the text file. Are they related?' + WORD_INCLUSION_PROMPT; + const fileSources = [imageUrl1, pdfUrl]; + const result: any = await conv.prompt({ message: prompt, files: fileSources }); + expect(result).toBeTruthy(); + expect(result.length).toBeGreaterThan(20); + expect(result).toContain(LLM_OUTPUT_VALIDATOR); + }, + TIMEOUT * 2 + ); + + it( + `processes a video file correctly for Model: ${model}`, + async () => { + const spec = JSON.parse(fs.readFileSync(testData.getDataPath('AgentData/unit-tests-agent-randnum-randwords.smyth'), 'utf8')); + const conv = new Conversation(model, spec); + + const prompt = 'Describe the main events in this video.' + WORD_INCLUSION_PROMPT; + const fileSources = [videoUrl]; + const result: any = await conv.prompt({ message: prompt, files: fileSources }); + expect(result).toBeTruthy(); + expect(result.length).toBeGreaterThan(20); + expect(result).toContain(LLM_OUTPUT_VALIDATOR); + }, + TIMEOUT * 30 // 15 mins, it takes long time to process video file + ); + + it( + `handles a combination of audio and image files for Model: ${model}`, + async () => { + const spec = JSON.parse(fs.readFileSync(testData.getDataPath('AgentData/unit-tests-agent-randnum-randwords.smyth'), 'utf8')); + const conv = new Conversation(model, spec); + + const prompt = 'Is the audio describing the image? If not, how are they different?' + WORD_INCLUSION_PROMPT; + const fileSources = [audioUrl, imageUrl1]; + const result: any = await conv.prompt({ message: prompt, files: fileSources }); + expect(result).toBeTruthy(); + expect(result.length).toBeGreaterThan(20); + expect(result).toContain(LLM_OUTPUT_VALIDATOR); + }, + TIMEOUT + ); + + it( + `should throw error when there are video file with other file types for Model: ${model}`, + async () => { + const spec = JSON.parse(fs.readFileSync(testData.getDataPath('AgentData/unit-tests-agent-randnum-randwords.smyth'), 'utf8')); + const conv = new Conversation(model, spec); + + const fileSources = [imageUrl1, audioUrl, videoUrl, pdfUrl]; + await expect(conv.prompt({ message: 'Analyze these files', files: fileSources })).rejects.toThrow(); + }, + TIMEOUT * 20 // 10 mins + ); +} + +const models = [ + //{ provider: 'OpenAI', id: 'gpt-4o-mini' }, + //{ provider: 'Anthropic', id: 'claude-3-5-haiku' }, + { provider: 'GoogleAI', id: 'gemini-1.5-flash' }, +]; + +for (const model of models) { + describe(`LLM Multimodal Tests: ${model.provider} (${model.id})`, async () => { + runMultimodalTestCases(model.id); + }); +} diff --git a/packages/core/tests/unit/openai/ChatCompletionsApiInterface.unit.test.ts b/packages/core/tests/integration/003-LLM/openai/ChatCompletionsApiInterface.unit.test.ts similarity index 100% rename from packages/core/tests/unit/openai/ChatCompletionsApiInterface.unit.test.ts rename to packages/core/tests/integration/003-LLM/openai/ChatCompletionsApiInterface.unit.test.ts diff --git a/packages/core/tests/integration/003-LLM/openai/Embedding.test.ts b/packages/core/tests/integration/003-LLM/openai/Embedding.test.ts new file mode 100644 index 000000000..2f69b9fa6 --- /dev/null +++ b/packages/core/tests/integration/003-LLM/openai/Embedding.test.ts @@ -0,0 +1,331 @@ +import { describe, expect, it, beforeEach, vi, afterEach } from 'vitest'; +import { OpenAIEmbeds } from '@sre/IO/VectorDB.service/embed/OpenAIEmbedding'; +import { AccessCandidate } from '@sre/Security/AccessControl/AccessCandidate.class'; +import { getLLMCredentials } from '@sre/LLMManager/LLM.service/LLMCredentials.helper'; +import OpenAI, { OpenAI as OpenAIClient } from 'openai'; + +// Mock the OpenAI SDK +vi.mock('openai', () => { + // We expose a mock class for the client constructor via named export `OpenAI` + const OpenAICtor = vi.fn(); + // The default export must expose APIError for createOpenAIError + class APIError extends Error { + status: number; + code?: string; + type?: string; + constructor(status: number, data: any, message: string, _headers: any) { + super(message); + this.status = status; + this.code = data?.code; + this.type = data?.type; + } + } + return { + OpenAI: OpenAICtor, + default: { APIError }, + } as any; +}); + +// Mock the LLM credentials helper +vi.mock('@sre/LLMManager/LLM.service/LLMCredentials.helper', () => ({ + getLLMCredentials: vi.fn(), +})); + +describe('OpenAIEmbeds - Unit Tests', () => { + let openaiEmbeds: OpenAIEmbeds; + let mockAccessCandidate: AccessCandidate; + let mockClient: any; + + beforeEach(() => { + // Reset all mocks + vi.clearAllMocks(); + + // Setup mock OpenAI client + mockClient = { + embeddings: { + create: vi.fn(), + }, + }; + + (OpenAIClient as any).mockImplementation(() => mockClient); + + // Setup mock access candidate + mockAccessCandidate = { + teamId: 'test-team', + agentId: 'test-agent', + } as unknown as AccessCandidate; + + // Default mock for getLLMCredentials + (getLLMCredentials as any).mockResolvedValue({ + apiKey: 'test-api-key', + }); + }); + + afterEach(() => { + vi.restoreAllMocks(); + delete (process as any).env.OPENAI_API_KEY; + }); + + describe('constructor', () => { + it('should initialize with default model', () => { + openaiEmbeds = new OpenAIEmbeds(); + expect(openaiEmbeds.model).toBe('text-embedding-ada-002'); + }); + + it('should initialize with custom model', () => { + openaiEmbeds = new OpenAIEmbeds({ model: 'text-embedding-3-large' } as any); + expect(openaiEmbeds.model).toBe('text-embedding-3-large'); + }); + + it('should initialize with custom settings', () => { + const settings = { + model: 'text-embedding-3-large', + params: { + chunkSize: 256, + dimensions: 512, + stripNewLines: false, + timeout: 1234, + }, + } as any; + openaiEmbeds = new OpenAIEmbeds(settings); + + expect(openaiEmbeds.model).toBe('text-embedding-3-large'); + expect(openaiEmbeds.chunkSize).toBe(256); + expect(openaiEmbeds.dimensions).toBe(512); + expect(openaiEmbeds.stripNewLines).toBe(false); + }); + + it('should support dimension specification except for ada-002', () => { + openaiEmbeds = new OpenAIEmbeds(); + expect(openaiEmbeds.canSpecifyDimensions).toBe(false); + + openaiEmbeds = new OpenAIEmbeds({ model: 'text-embedding-3-large' } as any); + expect(openaiEmbeds.canSpecifyDimensions).toBe(true); + }); + + it('should have correct available models', () => { + expect(OpenAIEmbeds.models).toEqual(['text-embedding-ada-002', 'text-embedding-3-large']); + }); + }); + + describe('embedText', () => { + beforeEach(() => { + openaiEmbeds = new OpenAIEmbeds(); + }); + + it('should successfully embed a single text', async () => { + const mockEmbedding = [0.1, 0.2, 0.3, 0.4, 0.5]; + mockClient.embeddings.create.mockResolvedValue({ + data: [{ embedding: mockEmbedding }], + }); + + const result = await openaiEmbeds.embedText('test text', mockAccessCandidate); + + expect(result).toEqual(mockEmbedding); + expect(mockClient.embeddings.create).toHaveBeenCalledWith({ + model: 'text-embedding-ada-002', + input: 'test text', + }); + // Validate client created with credentials + expect(OpenAIClient).toHaveBeenCalledWith({ + dangerouslyAllowBrowser: true, + apiKey: 'test-api-key', + timeout: undefined, + maxRetries: 0, + }); + }); + + it('should process text by stripping newlines when stripNewLines is true', async () => { + mockClient.embeddings.create.mockResolvedValue({ data: [{ embedding: [0.1] }] }); + + await openaiEmbeds.embedText('test\ntext\nwith\nnewlines', mockAccessCandidate); + + expect(mockClient.embeddings.create).toHaveBeenCalledWith({ + model: 'text-embedding-ada-002', + input: 'test text with newlines', + }); + }); + + it('should preserve newlines when stripNewLines is false', async () => { + openaiEmbeds = new OpenAIEmbeds({ params: { stripNewLines: false } } as any); + mockClient.embeddings.create.mockResolvedValue({ data: [{ embedding: [0.1] }] }); + + await openaiEmbeds.embedText('test\ntext\nwith\nnewlines', mockAccessCandidate); + + expect(mockClient.embeddings.create).toHaveBeenCalledWith({ + model: 'text-embedding-ada-002', + input: 'test\ntext\nwith\nnewlines', + }); + }); + + it('should include dimensions for models that support it', async () => { + openaiEmbeds = new OpenAIEmbeds({ model: 'text-embedding-3-large', params: { dimensions: 128 } } as any); + mockClient.embeddings.create.mockResolvedValue({ data: [{ embedding: [0.1] }] }); + + await openaiEmbeds.embedText('hello', mockAccessCandidate); + + expect(mockClient.embeddings.create).toHaveBeenCalledWith({ + model: 'text-embedding-3-large', + input: 'hello', + dimensions: 128, + }); + }); + + it('should NOT include dimensions for ada-002 even if specified', async () => { + openaiEmbeds = new OpenAIEmbeds({ model: 'text-embedding-ada-002', params: { dimensions: 64 } } as any); + mockClient.embeddings.create.mockResolvedValue({ data: [{ embedding: [0.1] }] }); + + await openaiEmbeds.embedText('hello', mockAccessCandidate); + + const callArg = mockClient.embeddings.create.mock.calls[0][0]; + expect(callArg).toEqual({ model: 'text-embedding-ada-002', input: 'hello' }); + expect((callArg as any).dimensions).toBeUndefined(); + }); + + it('should pass timeout client option from settings', async () => { + openaiEmbeds = new OpenAIEmbeds({ params: { timeout: 4321 } } as any); + mockClient.embeddings.create.mockResolvedValue({ data: [{ embedding: [0.2] }] }); + + await openaiEmbeds.embedText('test', mockAccessCandidate); + + expect(OpenAIClient).toHaveBeenCalledWith({ + dangerouslyAllowBrowser: true, + apiKey: 'test-api-key', + timeout: 4321, + maxRetries: 0, + }); + }); + + it('should reuse client instance across multiple calls', async () => { + mockClient.embeddings.create.mockResolvedValue({ data: [{ embedding: [0.1] }] }); + + await openaiEmbeds.embedText('test1', mockAccessCandidate); + await openaiEmbeds.embedText('test2', mockAccessCandidate); + + expect(OpenAIClient).toHaveBeenCalledTimes(1); + expect(mockClient.embeddings.create).toHaveBeenCalledTimes(2); + }); + + it('should pass custom credentials when provided in settings', async () => { + const customCredentials = { apiKey: 'custom-key' } as any; + openaiEmbeds = new OpenAIEmbeds({ credentials: customCredentials } as any); + mockClient.embeddings.create.mockResolvedValue({ data: [{ embedding: [0.3] }] }); + + await openaiEmbeds.embedText('test', mockAccessCandidate); + + expect(getLLMCredentials).toHaveBeenCalledWith(mockAccessCandidate, { + provider: 'OpenAI', + modelId: 'text-embedding-ada-002', + credentials: customCredentials, + }); + }); + }); + + describe('embedTexts', () => { + beforeEach(() => { + openaiEmbeds = new OpenAIEmbeds({ params: { chunkSize: 2 } } as any); + }); + + it('should successfully embed multiple texts', async () => { + const mockEmbeddings = [ + [0.1, 0.2, 0.3], + [0.4, 0.5, 0.6], + [0.7, 0.8, 0.9], + ]; + + mockClient.embeddings.create.mockImplementation((req: any) => { + const inputs = Array.isArray(req.input) ? req.input : [req.input]; + const data = inputs.map((t: string) => { + if (t === 'text1') return { embedding: mockEmbeddings[0] }; + if (t === 'text2') return { embedding: mockEmbeddings[1] }; + if (t === 'text3') return { embedding: mockEmbeddings[2] }; + throw new Error('Unexpected text'); + }); + return Promise.resolve({ data }); + }); + + const texts = ['text1', 'text2', 'text3']; + const result = await openaiEmbeds.embedTexts(texts, mockAccessCandidate); + + expect(result).toEqual(mockEmbeddings); + expect(mockClient.embeddings.create).toHaveBeenCalledTimes(2); // chunkSize = 2 => 2 batches + expect(mockClient.embeddings.create).toHaveBeenCalledWith({ + model: 'text-embedding-ada-002', + input: ['text1', 'text2'], + }); + expect(mockClient.embeddings.create).toHaveBeenCalledWith({ + model: 'text-embedding-ada-002', + input: ['text3'], + }); + }); + + it('should handle empty texts array', async () => { + const result = await openaiEmbeds.embedTexts([], mockAccessCandidate); + expect(result).toEqual([]); + expect(mockClient.embeddings.create).not.toHaveBeenCalled(); + }); + + it('should process texts consistently with embedText', async () => { + mockClient.embeddings.create.mockResolvedValue({ data: [{ embedding: [0.1, 0.2, 0.3] }] }); + const texts = ['text\nwith\nnewlines']; + await openaiEmbeds.embedTexts(texts, mockAccessCandidate); + + expect(mockClient.embeddings.create).toHaveBeenCalledWith({ + model: 'text-embedding-ada-002', + input: ['text with newlines'], + }); + }); + }); + + describe('error handling', () => { + beforeEach(() => { + openaiEmbeds = new OpenAIEmbeds(); + }); + + it('should wrap OpenAI API errors', async () => { + const apiError: any = { statusCode: 429, message: 'API quota exceeded', code: 'rate_limit_exceeded', name: 'Error' }; + mockClient.embeddings.create.mockRejectedValue(apiError); + + await expect(openaiEmbeds.embedText('test', mockAccessCandidate)).rejects.toEqual(expect.any((OpenAI as any).APIError)); + }); + }); + + describe('text processing', () => { + it('should handle empty string input', async () => { + openaiEmbeds = new OpenAIEmbeds(); + mockClient.embeddings.create.mockResolvedValue({ data: [{ embedding: [0.1, 0.2, 0.3] }] }); + + const result = await openaiEmbeds.embedText('', mockAccessCandidate); + expect(result).toEqual([0.1, 0.2, 0.3]); + expect(mockClient.embeddings.create).toHaveBeenCalledWith({ + model: 'text-embedding-ada-002', + input: '', + }); + }); + + it('should handle strings with only whitespace', async () => { + openaiEmbeds = new OpenAIEmbeds(); + mockClient.embeddings.create.mockResolvedValue({ data: [{ embedding: [0.1, 0.2, 0.3] }] }); + + const result = await openaiEmbeds.embedText(' \t ', mockAccessCandidate); + expect(result).toEqual([0.1, 0.2, 0.3]); + expect(mockClient.embeddings.create).toHaveBeenCalledWith({ + model: 'text-embedding-ada-002', + input: ' \t ', + }); + }); + + it('should handle very long text inputs', async () => { + openaiEmbeds = new OpenAIEmbeds(); + const longText = 'Lorem ipsum '.repeat(1000); + mockClient.embeddings.create.mockResolvedValue({ data: [{ embedding: [0.1, 0.2, 0.3] }] }); + + const result = await openaiEmbeds.embedText(longText, mockAccessCandidate); + expect(result).toEqual([0.1, 0.2, 0.3]); + expect(mockClient.embeddings.create).toHaveBeenCalledWith({ + model: 'text-embedding-ada-002', + input: longText, + }); + }); + }); +}); diff --git a/packages/core/tests/unit/openai/ResponsesApiInterface.unit.test.ts b/packages/core/tests/integration/003-LLM/openai/ResponsesApiInterface.unit.test.ts similarity index 100% rename from packages/core/tests/unit/openai/ResponsesApiInterface.unit.test.ts rename to packages/core/tests/integration/003-LLM/openai/ResponsesApiInterface.unit.test.ts diff --git a/packages/core/tests/integration/003-LLM/prompt.test.ts b/packages/core/tests/integration/003-LLM/prompt.test.ts new file mode 100644 index 000000000..cc9cb40d6 --- /dev/null +++ b/packages/core/tests/integration/003-LLM/prompt.test.ts @@ -0,0 +1,158 @@ +import fs from 'fs'; +import { describe, expect, it, beforeEach } from 'vitest'; +import { LLMInference } from '@sre/LLMManager/LLM.inference'; +import { AccessCandidate } from '@sre/Security/AccessControl/AccessCandidate.class'; + +import { TLLMMessageRole } from '@sre/types/LLM.types'; +import { setupSRE } from '../../utils/sre'; +import { testData } from '../../utils/test-data-manager'; + +setupSRE(); + +const agentId = 'cm0zjhkzx0dfvhxf81u76taiz'; + +const TIMEOUT = 30000; +const LLM_OUTPUT_VALIDATOR = 'Yohohohooooo!'; +const WORD_INCLUSION_PROMPT = `\nThe response must includes "${LLM_OUTPUT_VALIDATOR}". If the response is JSON, then include an additional key-value pair with key as "${LLM_OUTPUT_VALIDATOR}" and value as "${LLM_OUTPUT_VALIDATOR}"`; + +async function runTestCases(model: string) { + let config; + + beforeEach(() => { + config = { + data: { + model, + maxTokens: 100, + temperature: 0.5, + stopSequences: '', + topP: 0.9, + topK: 10, + frequencyPenalty: 0, + presencePenalty: 0, + responseFormat: 'json', + }, + }; + }); + + const llmInference: LLMInference = await LLMInference.getInstance(model, AccessCandidate.team('default')); + + it( + `runs a simple prompt with Model: ${model}`, + async () => { + const prompt = 'Hello, what is the smallest country in the world?' + WORD_INCLUSION_PROMPT; + const result: any = await llmInference.prompt({ + query: prompt, + params: { ...config.data, agentId }, + }); + + expect(result).toBeTruthy(); + expect(JSON.stringify(result)).toContain(LLM_OUTPUT_VALIDATOR); + }, + TIMEOUT + ); + + it( + `runs a prompt with system message with Model: ${model}`, + async () => { + const prompt = 'What can you do?' + WORD_INCLUSION_PROMPT; + + const consistentMessages = [ + { role: TLLMMessageRole.System, content: 'You are a helpful assistant' }, + { role: TLLMMessageRole.User, content: prompt }, + ]; + + const result = await llmInference.prompt({ + contextWindow: consistentMessages, + params: { ...config.data, agentId }, + }); + expect(result).toBeTruthy(); + expect(JSON.stringify(result)).toContain(LLM_OUTPUT_VALIDATOR); + }, + TIMEOUT + ); + + it( + `handles long prompts correctly with Model: ${model}`, + async () => { + let longPrompt = fs.readFileSync(testData.getDataPath('dummy-article.txt'), 'utf8'); + longPrompt += '\n\nWhat is the main topic of this article?' + WORD_INCLUSION_PROMPT; + + const result = await llmInference.prompt({ + query: longPrompt, + params: { ...config.data, agentId }, + }); + expect(result).toBeTruthy(); + expect(JSON.stringify(result)).toContain(LLM_OUTPUT_VALIDATOR); + }, + TIMEOUT + ); + + it( + `handles complex multi-turn conversations with system message for Model: ${model}`, + async () => { + // * Note: WORD_INCLUSION_PROMPT does not work properly here + const messages = JSON.parse(fs.readFileSync(testData.getDataPath('dummy-input-messages.json'), 'utf8')); + + config.data.responseFormat = ''; + const result = await llmInference.prompt({ + contextWindow: messages, + params: { ...config.data, agentId }, + }); + expect(result).toBeTruthy(); + expect(result?.length).toBeGreaterThan(200); + }, + TIMEOUT + ); + + it( + `correctly handles special characters and Unicode with Model: ${model}`, + async () => { + const specialCharsPrompt = 'Hello! こんにちは! 你好! مرحبا! 🌍🚀' + WORD_INCLUSION_PROMPT; + const result = await llmInference.prompt({ + query: specialCharsPrompt, + params: { ...config.data, agentId }, + }); + expect(result).toBeTruthy(); + expect(JSON.stringify(result)).toContain(LLM_OUTPUT_VALIDATOR); + }, + TIMEOUT + ); + + it( + `handles prompts with code snippets correctly with Model: ${model}`, + async () => { + const codePrompt = 'Explain this code:\n\nfunction add(a, b) {\n return a + b;\n}' + WORD_INCLUSION_PROMPT; + const result = await llmInference.prompt({ + query: codePrompt, + params: { ...config.data, agentId }, + }); + expect(result).toBeTruthy(); + expect(JSON.stringify(result)).toContain(LLM_OUTPUT_VALIDATOR); + }, + TIMEOUT + ); + + it( + `handles errors gracefully with Model: ${model}`, + async () => { + const result = await llmInference.prompt({ query: '', params: { ...config.data, agentId } }); + await expect(result).toBeDefined(); + }, + TIMEOUT + ); +} + +const models = [ + { provider: 'OpenAI', id: 'gpt-4o-mini' }, + { provider: 'Anthropic', id: 'claude-3.5-haiku' }, + { provider: 'GoogleAI', id: 'gemini-1.5-flash' }, + { provider: 'Groq', id: 'gemma2-9b-it' }, + // { provider: 'TogetherAI', id: 'meta-llama/Meta-Llama-3-8B-Instruct-Lite' }, + // { provider: 'xAI', id: 'grok-beta' }, +]; + +for (const model of models) { + describe(`LLM Prompt Tests: ${model.provider} (${model.id})`, async () => { + await runTestCases(model.id); + }); +} diff --git a/packages/core/tests/integration/003-LLM/tools.test.ts b/packages/core/tests/integration/003-LLM/tools.test.ts new file mode 100644 index 000000000..c1b4178e7 --- /dev/null +++ b/packages/core/tests/integration/003-LLM/tools.test.ts @@ -0,0 +1,435 @@ +import { describe, expect, it, beforeEach } from 'vitest'; +import { LLMInference } from '@sre/LLMManager/LLM.inference'; +import { AccessCandidate } from '@sre/Security/AccessControl/AccessCandidate.class'; +import { setupSRE } from '../../utils/sre'; +import EventEmitter from 'events'; +import { TLLMEvent } from '@sre/types/LLM.types'; +import { delay } from '@sre/utils/index'; + +/* + * This file contains tests for the `toolRequest` and `streamRequest` functions. + * These tests ensure that the responses include the correct tool information + * and handle various scenarios, such as using multiple tools, handling errors, + * and streaming responses. + */ + +setupSRE(); + +const agentId = 'cm0zjhkzx0dfvhxf81u76taiz'; + +const TIMEOUT = 30000; + +async function runToolTestCases(model: string) { + const llmInference: LLMInference = await LLMInference.getInstance(model, AccessCandidate.team('default')); + + it( + 'should execute a simple tool request', + async () => { + const toolDefinitions = [ + { + name: 'get_weather', + description: 'Get the current weather', + properties: { + location: { type: 'string' }, + }, + requiredFields: ['location'], + }, + ]; + + const toolsConfig: any = llmInference.connector.formatToolsConfig({ + type: 'function', + toolDefinitions, + toolChoice: 'auto', + }) as any; + + const stream: any = await llmInference.promptStream({ + query: "What's current weather in New York?", + params: { model, toolsConfig, agentId }, + }); + + let toolsData: any[] = []; + const streamComplete = new Promise((resolve) => { + stream.on(TLLMEvent.ToolInfo, (data) => { + toolsData = toolsData.concat(data); + }); + stream.on(TLLMEvent.End, resolve); + }); + + await streamComplete; + + expect(toolsData).toBeInstanceOf(Array); + expect(toolsData.length).toBeGreaterThan(0); + expect(toolsData[0].name).toBe('get_weather'); + }, + TIMEOUT + ); + + it( + 'should handle tool requests with no tools used', + async () => { + const toolDefinitions = [ + { + name: 'get_weather', + description: 'Get the current weather', + properties: { + location: { type: 'string' }, + }, + requiredFields: ['location'], + }, + ]; + + const stream: any = await llmInference.promptStream({ + query: 'Hello, how are you?', + params: { + model, + toolsConfig: llmInference.connector.formatToolsConfig({ + type: 'function', + toolDefinitions, + toolChoice: 'auto', + }) as any, + agentId, + }, + }); + + let usedTool = false; + let content = ''; + const streamComplete = new Promise((resolve) => { + stream.on(TLLMEvent.ToolInfo, () => { + usedTool = true; + }); + stream.on(TLLMEvent.Content, (chunk) => { + content += chunk; + }); + stream.on(TLLMEvent.End, resolve); + }); + + await streamComplete; + + expect(usedTool).toBe(false); + expect(content).toBeTruthy(); + }, + TIMEOUT + ); + + it( + 'should handle requests with empty toolDefinitions', + async () => { + const stream: any = await llmInference.promptStream({ + query: "What's the weather like today?", + params: { + model, + toolsConfig: llmInference.connector.formatToolsConfig({ + type: 'function', + toolDefinitions: [], // Empty tools array + toolChoice: 'auto', + }) as any, + agentId, + }, + }); + + let usedTool = false; + let content = ''; + const streamComplete = new Promise((resolve) => { + stream.on(TLLMEvent.ToolInfo, () => { + usedTool = true; + }); + stream.on(TLLMEvent.Content, (chunk) => { + content += chunk; + }); + stream.on(TLLMEvent.End, resolve); + }); + + await streamComplete; + + expect(usedTool).toBe(false); + expect(content).toBeTruthy(); + }, + TIMEOUT + ); + + it( + 'should handle errors in toolRequest gracefully', + async () => { + const stream: any = await llmInference.promptStream({ params: { model, agentId } }); + + let error; + const streamComplete = new Promise((resolve) => { + stream.on(TLLMEvent.Error, (e) => (error = e)); + stream.on(TLLMEvent.End, resolve); + }); + + await streamComplete; + + expect(error).toBeInstanceOf(Error); + }, + TIMEOUT + ); +} + +async function runStreamRequestTestCases(model: string) { + const llmInference: LLMInference = await LLMInference.getInstance(model, AccessCandidate.team('default')); + + it( + 'should stream a simple request', + async () => { + const params = { + messages: [{ role: 'user', content: 'Tell me a short story.' }], + model, + }; + + const stream = await llmInference.promptStream({ + contextWindow: params.messages, + params: { model: params.model, agentId }, + }); + expect(stream).toBeInstanceOf(EventEmitter); + + let content = ''; + + const streamComplete = new Promise((resolve) => { + stream.on(TLLMEvent.Content, (chunk) => { + content += chunk; + }); + + stream.on(TLLMEvent.End, resolve); + }); + + await streamComplete; + + expect(content).toBeTruthy(); + }, + TIMEOUT + ); + + it( + 'should handle streaming with tools', + async () => { + const toolDefinitions = [ + { + name: 'get_weather', + description: 'Get the current weather', + properties: { + location: { type: 'string' }, + }, + requiredFields: ['location'], + }, + ]; + + const toolsConfig = llmInference.connector.formatToolsConfig({ + type: 'function', + toolDefinitions, + toolChoice: 'auto', + }); + + const params = { + messages: [{ role: 'user', content: "What's the current weather in Bangladesh?" }], + toolsConfig, + model, + }; + + const stream = await llmInference.promptStream({ + contextWindow: params.messages, + params: { model: params.model, toolsConfig: params.toolsConfig as any, agentId }, + }); + expect(stream).toBeInstanceOf(EventEmitter); + + let toolsData; + + const streamComplete = new Promise((resolve) => { + stream.on(TLLMEvent.ToolInfo, (data) => { + toolsData = data; + resolve(); + }); + }); + + await streamComplete; + + expect(toolsData).toBeTruthy(); + expect(toolsData[0].name).toBe('get_weather'); + }, + TIMEOUT * 2 + ); + + it( + 'should handle errors in stream gracefully', + async () => { + const params = { + messages: [], // Empty messages array should cause an error + model, + }; + + const stream = await llmInference.promptStream({ + contextWindow: params.messages, + params: { model: params.model, agentId }, + }); + expect(stream).toBeInstanceOf(EventEmitter); + + let error; + + const streamComplete = new Promise((resolve) => { + stream.on(TLLMEvent.Error, (e) => { + error = e; + }); + stream.on(TLLMEvent.End, resolve); + }); + + await streamComplete; + + expect(error).toBeInstanceOf(Error); + }, + TIMEOUT + ); +} + +async function runMultipleToolRequestTestCases(model: string, provider?: string) { + const llmInference: LLMInference = await LLMInference.getInstance(model, AccessCandidate.team('default')); + let toolDefinitions; + let toolsConfig; + let params; + + beforeEach(() => { + toolDefinitions = [ + { + name: 'get_weather', + description: 'Get the current weather', + properties: { + location: { type: 'string' }, + }, + requiredFields: ['location'], + }, + { + name: 'get_population', + description: 'Get the population of a city', + properties: { + city: { type: 'string' }, + }, + requiredFields: ['city'], + }, + ]; + + toolsConfig = llmInference.connector.formatToolsConfig({ + type: 'function', + toolDefinitions, + toolChoice: 'auto', + }) as any; + + params = { + messages: [ + { + role: 'user', + content: + "I need two pieces of information in a single response: 1) What's the current weather in New York City? 2) What's the exact population of New York City? Please ensure both tools are used simultaneously to provide a comprehensive answer.", + }, + ], + toolsConfig, + model, + }; + }); + + it( + 'should return multiple tools info with promptStream()', + async () => { + const stream: any = await llmInference.promptStream({ + params: { ...params, model, agentId }, + contextWindow: params.messages, + }); + + let toolsData: any[] = []; + let error; + const streamComplete = new Promise((resolve) => { + stream.on(TLLMEvent.Error, (e) => { + error = e; + resolve(); + }); + stream.on(TLLMEvent.ToolInfo, (data) => { + toolsData = toolsData.concat(data); + }); + stream.on(TLLMEvent.End, resolve); + }); + + await streamComplete; + + expect(error).toBeFalsy(); + expect(toolsData).toBeInstanceOf(Array); + expect(toolsData.length).toBe(2); + expect(toolsData[0].name).toBe('get_weather'); + expect(toolsData[1].name).toBe('get_population'); + }, + TIMEOUT + ); + + it( + 'should return multiple tools info with promptStream()', + async () => { + // wait 10 seconds to prevent error like "Request was rejected due to request rate limiting..." for TogetherAI + if (provider === 'TogetherAI') { + await delay(10000); + } + + const stream = await llmInference.promptStream({ + contextWindow: params.messages, + params: { model: params.model, toolsConfig: params.toolsConfig as any, agentId }, + }); + expect(stream).toBeInstanceOf(EventEmitter); + + let toolsData: any[] = []; + let error; + + const streamComplete = new Promise((resolve) => { + stream.on(TLLMEvent.Error, (e) => { + error = e; + resolve(); + }); + stream.on(TLLMEvent.ToolInfo, (data) => { + toolsData = toolsData.concat(data); + }); + stream.on(TLLMEvent.End, resolve); + }); + + await streamComplete; + + expect(error).toBeFalsy(); + expect(toolsData).toBeInstanceOf(Array); + expect(toolsData.length).toBe(2); + expect(toolsData[0].name).toBe('get_weather'); + expect(toolsData[1].name).toBe('get_population'); + }, + TIMEOUT * 2 + ); +} + +const models = [ + { provider: 'OpenAI', id: 'gpt-4o-mini' }, + //{ provider: 'Anthropic', id: 'claude-3-5-haiku-latest' }, + //{ provider: 'GoogleAI', id: 'gemini-1.5-flash' }, + //{ provider: 'Groq', id: 'gemma2-9b-it' }, + //{ provider: 'TogetherAI', id: 'meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo' }, + //{ provider: 'xAI', id: 'grok-beta' }, +]; + +for (const model of models) { + describe(`Tool Request Tests: ${model.provider} (${model.id})`, async () => { + await runToolTestCases(model.id); + }); + + describe(`Stream Request Tests: ${model.provider} (${model.id})`, async () => { + await runStreamRequestTestCases(model.id); + }); +} + +/* + * Google AI and Groq do not return multiple tool data in a single response. + * Therefore, the expectation "(result.data.toolsData.length).toBe(2)" does not apply to them. + * They may provide additional tool data in subsequent requests. + * Tests for the sequence of tool responses are available in conversation.test.ts. + */ +const modelsWithMultipleToolsResponse = [ + { provider: 'OpenAI', id: 'gpt-4o-mini' }, + //{ provider: 'Anthropic', id: 'claude-3-5-haiku-latest' }, + //{ provider: 'TogetherAI', id: 'meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo' }, + /* { provider: 'xAI', id: 'grok-beta' }, */ // xAI is not able to handle multiple tools use properly +]; +for (const model of modelsWithMultipleToolsResponse) { + describe(`Multiple Tools Request Tests: ${model.provider} (${model.id})`, async () => { + await runMultipleToolRequestTestCases(model.id, model.provider); + }); +} diff --git a/packages/core/tests/integration/003-LLM/usage.test.ts b/packages/core/tests/integration/003-LLM/usage.test.ts new file mode 100644 index 000000000..1b4dc9107 --- /dev/null +++ b/packages/core/tests/integration/003-LLM/usage.test.ts @@ -0,0 +1,221 @@ +import { describe, expect, it, vi, beforeEach } from 'vitest'; +import { AccessCandidate, ConnectorService, SystemEvents } from 'index'; +import { LLMInference } from '@sre/LLMManager/LLM.inference'; +import EventEmitter from 'events'; +import { delay } from '@sre/utils/index'; +import { APIKeySource, SmythLLMUsage, TLLMParams, TLLMEvent } from '@sre/types/LLM.types'; +import { setupSRE } from '../../utils/sre'; +import { AccessRequest } from '@sre/Security/AccessControl/AccessRequest.class'; + +setupSRE(); +const agentId = 'cm0zjhkzx0dfvhxf81u76taiz'; + +// import {DummyAccount} from "@sre/Security/Account.service/connectors/DummyAccount.class" + +vi.mock('@sre/Security/Account.service/connectors/DummyAccount.class', async () => { + let DummyAccount = (await import('@sre/Security/Account.service/connectors/DummyAccount.class')).DummyAccount; + class MockedDummyAccount extends DummyAccount { + public getTeamSetting(acRequest: AccessRequest, teamId: string, settingKey: string): Promise { + if (settingKey === 'custom-llm') { + return Promise.resolve( + JSON.stringify({ + m5zlsw6gduo: { + id: 'm5zlsw6gduo', + name: 'NEW_LLM', + provider: 'Bedrock', + features: ['text-completion'], + tags: ['Bedrock'], + settings: { + foundationModel: 'ai21.jamba-instruct-v1:0', + customModel: '', + region: 'us-east-1', + keyIDName: 'BEDROCK_TESINTG_AWS_KEY_ID', + secretKeyName: 'BEDROCK_TESINTG_AWS_SECRET_KEY', + sessionKeyName: '', + }, + }, + }) + ); + } + return super.getTeamSetting(acRequest, teamId, settingKey); + } + } + return { DummyAccount: MockedDummyAccount }; +}); + +// SmythRuntime initialization replaced by setupSRE() + +const models = [ + { + provider: 'OpenAI', + id: 'gpt-4o-mini', + supportedMethods: [ + 'chatRequest', + 'visionRequest', + 'multimodalRequest', + 'toolRequest', + 'streamRequest', + 'multimodalStreamRequest', + 'imageGenRequest', + ], + }, + // { + // provider: 'Anthropic', + // id: 'claude-3.5-sonnet', + // supportedMethods: ['chatRequest', 'visionRequest', 'multimodalRequest', 'toolRequest', 'streamRequest', 'multimodalStreamRequest'], + // }, + // { provider: 'Groq', id: 'gemma2-9b-it', supportedMethods: ['chatRequest', 'toolRequest', 'streamRequest'] }, + // { + // provider: 'GoogleAI', + // id: 'gemini-1.5-flash', + // supportedMethods: ['chatRequest', 'visionRequest', 'multimodalRequest', 'toolRequest', 'streamRequest', 'multimodalStreamRequest'], + // }, + // { provider: 'Bedrock', id: 'm5zlsw6gduo', supportedMethods: ['chatRequest', 'toolRequest', 'streamRequest'] }, + //* disabled for now since we have no valid access to VertexAI + // { provider: 'VertexAI', id: 'gemini-1.5-flash', supportedMethods: ['chatRequest'] }, +]; + +// Use static agentId. No Agent instance needed. + +function listenForUsageEvent() { + let usageEvent: SmythLLMUsage = undefined; + SystemEvents.once('USAGE:LLM', (usage) => { + console.log('USAGE:LLM received', usage); + usageEvent = usage; + }); + return { + get(): SmythLLMUsage { + return usageEvent; + }, + }; +} + +async function consumeStream(stream) { + // stream.on('end', resolve); + return new Promise((resolve) => { + stream.on('end', resolve); + }); +} + +describe.each(models)('LLM Usage Reporting Tests: $provider ($id)', async ({ provider, id, supportedMethods }) => { + let config; + + beforeEach(() => { + config = { + data: { + model: id, + maxTokens: 100, + temperature: 0.5, + stopSequences: '', + topP: 0.9, + topK: 10, + frequencyPenalty: 0, + presencePenalty: 0, + responseFormat: 'json', + cache: true, + }, + }; + + // make sure to info the user to put the needed vault keys in vault.json before running + // "keyIDName": "BEDROCK_TESINTG_AWS_KEY_ID", + // "secretKeyName": "BEDROCK_TESINTG_AWS_SECRET_KEY", + console.warn('|----------------------------------------------------------|'); + console.warn('| Make sure to put the following keys in vault.json to make sure all tests pass |'); + console.warn('| BEDROCK_TESINTG_AWS_KEY_ID |'); + console.warn('| BEDROCK_TESINTG_AWS_SECRET_KEY |'); + console.warn('|----------------------------------------------------------|'); + }); + + const llmInference: LLMInference = await LLMInference.getInstance(id, AccessCandidate.team('default')); + const isSupported = (method: string) => supportedMethods.includes(method); + const vaultConnector = ConnectorService.getVaultConnector(); + const apiKey = await vaultConnector + .user(AccessCandidate.agent(agentId)) + .get(provider) + .catch(() => ''); + + //let expectedKeySource = apiKey ? APIKeySource.User : APIKeySource.Smyth; + + isSupported('chatRequest') && + it('should report usage for chatRequest', async () => { + const usageEvent = listenForUsageEvent(); + const prompt = 'Hello, what is the smallest country in the world?'; + await llmInference.prompt({ query: prompt, params: { ...config.data, agentId } }); + const eventValue = usageEvent.get(); + expect(eventValue, 'Did not receive usage event').toBeDefined(); + expect(eventValue.input_tokens, 'Input tokens should be greater than 0').toBeGreaterThan(0); + expect(eventValue.output_tokens, 'Output tokens should be greater than 0').toBeGreaterThan(0); + expect(eventValue.sourceId, 'LLM sourceId mismatch').toContain('llm:'); + expect(eventValue.keySource, 'Key source mismatch').toBe(APIKeySource.User); + }); + // Vision/multimodal usage is covered by prompt with files in other tests; skipping here. + isSupported('toolRequest') && + it('should report usage for toolRequest', async () => { + const usageEvent = listenForUsageEvent(); + const toolDefinitions = [ + { + name: 'get_weather', + description: 'Get the current weather', + properties: { + location: { type: 'string' }, + }, + requiredFields: ['location'], + }, + ]; + await llmInference.prompt({ + query: 'Hello, how are you?', + params: { + model: id, + toolsConfig: llmInference.connector.formatToolsConfig({ + type: 'function', + toolDefinitions, + toolChoice: 'auto', + }) as any, + agentId, + }, + }); + const eventValue = usageEvent.get(); + expect(eventValue, 'Did not receive usage event').toBeDefined(); + expect(eventValue.input_tokens, 'Input tokens should be greater than 0').toBeGreaterThan(0); + expect(eventValue.output_tokens, 'Output tokens should be greater than 0').toBeGreaterThan(0); + expect(eventValue.sourceId, 'LLM sourceId mismatch').toContain('llm:'); + expect(eventValue.keySource, 'Key source mismatch').toBe(APIKeySource.User); + }); + + isSupported('streamRequest') && + it('should report usage for streamRequest', async () => { + const usageEvent = listenForUsageEvent(); + const msgs = []; + // 30*2 messages with same q&a to test prompt caching (for eg. OpenAI starts caching when tokens >= 1024) + for (let i = 0; i < 30; i++) { + msgs.push({ role: 'user', content: ' Explain quantum physics in simple terms.' }); + msgs.push({ + role: 'assistant', + content: + 'Quantum physics is the study of the behavior of matter and energy at the smallest scales, where it behaves differently than it does at larger scales.', + }); + } + const contextWindow = [...msgs, { role: 'user', content: ' Explain quantum physics in simple terms.' }]; + const stream: any = await llmInference.promptStream({ + contextWindow, + params: { cache: true, model: id, agentId }, + }); + await new Promise((resolve) => { + stream.on(TLLMEvent.End, resolve); + stream.on(TLLMEvent.Error, resolve); + }); + let eventValue = usageEvent.get(); + // if the event was not emitted even after the stream ended, + // wait for additional 500ms in case the usage is reported after the content stream ends + if (!eventValue) { + await delay(500); + eventValue = usageEvent.get(); + } + expect(eventValue, 'Did not receive usage event').toBeDefined(); + expect(eventValue.input_tokens, 'Input tokens should be greater than 0').toBeGreaterThan(0); + expect(eventValue.output_tokens, 'Output tokens should be greater than 0').toBeGreaterThan(0); + expect(eventValue.sourceId, 'LLM sourceId mismatch').toContain('llm:'); + expect(eventValue.keySource, 'Key source mismatch').toBe(APIKeySource.User); + }); + // multimodal/image generation usage tests are deprecated for this migration scope. +}); diff --git a/packages/core/tests/integration/connectors/vectordb/milvus.test.ts b/packages/core/tests/integration/004-VectorDB/milvus.test.ts similarity index 99% rename from packages/core/tests/integration/connectors/vectordb/milvus.test.ts rename to packages/core/tests/integration/004-VectorDB/milvus.test.ts index 4e1f25608..74c78821c 100644 --- a/packages/core/tests/integration/connectors/vectordb/milvus.test.ts +++ b/packages/core/tests/integration/004-VectorDB/milvus.test.ts @@ -1,5 +1,5 @@ import { afterEach, beforeAll, describe, expect, it, vi } from 'vitest'; -import { setupSRE } from '../../../utils/sre'; +import { setupSRE } from '../../utils/sre'; import { ConnectorService } from '@sre/Core/ConnectorsService'; import { AccessCandidate } from '@sre/Security/AccessControl/AccessCandidate.class'; diff --git a/packages/core/tests/integration/connectors/vectordb/pinecone.test.ts b/packages/core/tests/integration/004-VectorDB/pinecone.test.ts similarity index 92% rename from packages/core/tests/integration/connectors/vectordb/pinecone.test.ts rename to packages/core/tests/integration/004-VectorDB/pinecone.test.ts index 386a4a16b..a8c756726 100644 --- a/packages/core/tests/integration/connectors/vectordb/pinecone.test.ts +++ b/packages/core/tests/integration/004-VectorDB/pinecone.test.ts @@ -1,7 +1,10 @@ import { afterEach, beforeAll, describe, expect, it, vi } from 'vitest'; -import { setupSRE } from '../../../utils/sre'; +import { setupSRE } from '../../utils/sre'; import { ConnectorService } from '@sre/Core/ConnectorsService'; import { AccessCandidate } from '@sre/Security/AccessControl/AccessCandidate.class'; +import dotenv from 'dotenv'; +import { delay } from '@sre/utils/index'; +dotenv.config(); // Deterministic, offline embedding mock vi.mock('@sre/IO/VectorDB.service/embed', async () => { @@ -80,6 +83,9 @@ describe('Pinecone - VectorDB connector', () => { // Create namespace and verify await client.createNamespace('docs', { env: 'test' }); + + //cool down : sometimes created namespace is not available immediately + await delay(3000); await expect(client.namespaceExists('docs')).resolves.toBe(true); // Create datasource with chunking @@ -119,7 +125,9 @@ describe('Pinecone - VectorDB connector', () => { // Delete datasource and verify await client.deleteDatasource('docs', 'pc-ds1'); - await expect(client.getDatasource('docs', 'pc-ds1')).rejects.toThrow('Data source not found'); + + const datasource = await client.getDatasource('docs', 'pc-ds1'); + expect(datasource).toBeUndefined(); // Delete namespace await client.deleteNamespace('docs'); diff --git a/packages/core/tests/integration/008-Components/GPTPlugin.test.ts b/packages/core/tests/integration/008-Components/GPTPlugin.test.ts new file mode 100644 index 000000000..092bef865 --- /dev/null +++ b/packages/core/tests/integration/008-Components/GPTPlugin.test.ts @@ -0,0 +1,149 @@ +import { AgentProcess } from '@sre/Core/AgentProcess.helper'; +import config from '@sre/config'; +import { CLIAgentDataConnector, ConnectorService, SmythRuntime } from '@sre/index'; +import { TConnectorService } from '@sre/types/SRE.types'; +import fs from 'fs'; + +import { describe, expect, it } from 'vitest'; + +const sre = SmythRuntime.Instance.init({ + Storage: { + Connector: 'S3', + Settings: { + bucket: config.env.AWS_S3_BUCKET_NAME || '', + region: config.env.AWS_S3_REGION || '', + accessKeyId: config.env.AWS_ACCESS_KEY_ID || '', + secretAccessKey: config.env.AWS_SECRET_ACCESS_KEY || '', + }, + }, + AgentData: { + Connector: 'Local', + Settings: { + devDir: './tests/data/AgentData', + prodDir: './tests/data/AgentData', + }, + }, + Vault: { + Connector: 'JSONFileVault', + Settings: { + file: './tests/data/vault.json', + }, + }, +}); + +ConnectorService.register(TConnectorService.AgentData, 'CLI', CLIAgentDataConnector); +ConnectorService.init(TConnectorService.AgentData, 'CLI'); + +describe('GPTPlugin Component', () => { + it('runs a simple OpenAPI Plugin request', async () => { + let error; + try { + const agentData = fs.readFileSync('./tests/data/sre-llm.smyth', 'utf-8'); + const data = JSON.parse(agentData); + + const agentProcess = AgentProcess.load(data); + + let res = await agentProcess.run({ + method: 'POST', + path: '/api/test-gpt-plugin', + body: { + Input: 'Monitors', + }, + }); + + const output = res?.data?.result?.Output; + + expect(output).toBeDefined(); + expect(output).toBeTypeOf('string'); + } catch (e) { + error = e; + console.error(e.message); + } + expect(error).toBeUndefined(); + }); + + it('should handle missing prompt', async () => { + let error; + try { + const agentData = fs.readFileSync('./tests/data/sre-llm.smyth', 'utf-8'); + const data = JSON.parse(agentData); + + const agentProcess = AgentProcess.load(data); + + let output = await agentProcess.run({ + method: 'POST', + path: '/api/test-gpt-plugin', + body: { + Input: '', + }, + }); + + expect(output?.data?.result?._error).toBe('Please provide a prompt'); + } catch (e) { + error = e; + console.error(e.message); + } + expect(error).toBeUndefined(); + }); + + it('should use template string for descForModel', async () => { + let error; + try { + const agentData = fs.readFileSync('./tests/data/sre-llm.smyth', 'utf-8'); + const data = JSON.parse(agentData); + data.components[0].data.descForModel = 'Description for {{Query}}'; + + const agentProcess = AgentProcess.load(data); + + let output = await agentProcess.run({ + method: 'POST', + path: '/api/test-gpt-plugin', + body: { + Input: 'Monitors', + }, + }); + + expect(output?.data?.result?.Output).toBeDefined(); + expect(output?.data?.result?.Output).toBeTypeOf('string'); + } catch (e) { + error = e; + console.error(e.message); + } + expect(error).toBeUndefined(); + }); + + it('should handle different input types', async () => { + let error; + try { + const agentData = fs.readFileSync('./tests/data/sre-llm.smyth', 'utf-8'); + const data = JSON.parse(agentData); + + const agentProcess = AgentProcess.load(data); + + const testCases = [ + { Input: 'LED Monitors' }, + { Input: JSON.stringify({ complex: 'Monitors', with: ['Energy Saving', 'Eye protector'] }) }, + { Input: 42 }, + { Input: true }, + ]; + + for (let testCase of testCases) { + let output = await agentProcess.run({ + method: 'POST', + path: '/api/test-gpt-plugin', + body: testCase, + }); + agentProcess.reset(); + + expect(output?.data?.result?.Output).toBeDefined(); + expect(output?.data?.result?.Output).toBeTypeOf('string'); + } + } catch (e) { + error = e; + console.error(e.message); + } + expect(error).toBeUndefined(); + }); + + // TODO [Forhad]: Need to add test case for large input after implementing the token limit check +}); diff --git a/packages/core/tests/integration/008-Components/HuggingFace.test.ts b/packages/core/tests/integration/008-Components/HuggingFace.test.ts new file mode 100644 index 000000000..67f294a06 --- /dev/null +++ b/packages/core/tests/integration/008-Components/HuggingFace.test.ts @@ -0,0 +1,351 @@ +import Agent from '@sre/AgentManager/Agent.class'; +import HuggingFace from '@sre/Components/HuggingFace.class'; +import LLMAssistant from '@sre/Components/LLMAssistant.class'; +import { config, ConnectorService, SmythRuntime } from '@sre/index'; +import { delay } from '@sre/utils/date-time.utils'; +import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; +import fs from 'fs'; +import util from 'util'; +import path from 'path'; +import { AccessCandidate } from '@sre/Security/AccessControl/AccessCandidate.class'; +import { HfInference, textClassification, zeroShotClassification, zeroShotImageClassification } from '@huggingface/inference'; +import { TestAccountConnector } from '../../utils/TestConnectors'; +import { IAccessCandidate } from '@sre/types/ACL.types'; +import { TConnectorService } from '@sre/types/SRE.types'; + +const imagePath = path.resolve(__dirname, '../../data/avatar.png'); +const imageBlob = await util.promisify(fs.readFile)(imagePath); + +// Specific getter for HuggingFace API key +const getApiKeyVaultKeyName = (): string => { + // const apiKey = process.env.__TEST__HUGGINGFACE_API_KEY; + // if (!apiKey) { + // throw new Error('Zapier testing API Key is not set. Please set the __TEST__HUGGINGFACE_API_KEY environment variable to run this test.'); + // } + // // return apiKey; + return `{{KEY(HUGGINGFACE_API_KEY)}}`; +}; + +class CustomAccountConnector extends TestAccountConnector { + public getCandidateTeam(candidate: IAccessCandidate): Promise { + if (candidate.id === 'agent-123456') { + return Promise.resolve('9'); + } else if (candidate.id === 'agent-654321') { + return Promise.resolve('5'); + } + + return super.getCandidateTeam(candidate); + } +} +ConnectorService.register(TConnectorService.Account, 'MyCustomAccountConnector', CustomAccountConnector); + +// We need SRE to be loaded because LLMAssistant uses internal SRE functions +const sre = SmythRuntime.Instance.init({ + CLI: { + Connector: 'CLI', + }, + Storage: { + Connector: 'S3', + Settings: { + bucket: config.env.AWS_S3_BUCKET_NAME || '', + region: config.env.AWS_S3_REGION || '', + accessKeyId: config.env.AWS_ACCESS_KEY_ID || '', + secretAccessKey: config.env.AWS_SECRET_ACCESS_KEY || '', + }, + }, + Cache: { + Connector: 'Redis', + Settings: { + hosts: config.env.REDIS_SENTINEL_HOSTS, + name: config.env.REDIS_MASTER_NAME || '', + password: config.env.REDIS_PASSWORD || '', + }, + }, + AgentData: { + Connector: 'Local', + Settings: { + devDir: './tests/data/AgentData', + prodDir: './tests/data/AgentData', + }, + }, + Vault: { + Connector: 'JSONFileVault', + Settings: { + file: './tests/data/vault.json', + }, + }, + Account: { + Connector: 'DummyAccount', + Settings: {}, + }, +}); + +// Mock Agent class to keep the test isolated from the actual Agent implementation +vi.mock('@sre/AgentManager/Agent.class', () => { + const MockedAgent = vi.fn().mockImplementation(() => ({ + id: 'agent-123456', + teamId: 'default', + agentRuntime: { debug: true }, // used inside createComponentLogger() + })); + return { default: MockedAgent }; +}); + +vi.mock('@huggingface/inference', async () => { + const originalHfInference = (await vi.importActual('@huggingface/inference')).HfInference; + + return { + HfInference: vi.fn().mockImplementation((apiKey) => { + const hfInference = new originalHfInference(apiKey); + return { + // dummy blob of a png image + textToImage: vi.fn().mockResolvedValue(new Blob([imageBlob], { type: 'image/png' })), + zeroShotClassification: hfInference.zeroShotClassification.bind(hfInference), + zeroShotImageClassification: hfInference.zeroShotImageClassification.bind(hfInference), + textClassification: hfInference.textClassification.bind(hfInference), + textGeneration: hfInference.textGeneration.bind(hfInference), + objectDetection: hfInference.objectDetection.bind(hfInference), + accessToken: apiKey, + custom: true, + }; + }), + }; +}); + +describe('HuggingFace Component', () => { + beforeAll(async () => { + // This will throw an error if the API key is not set + const vaultConnector = ConnectorService.getVaultConnector(); + const team = AccessCandidate.team('default'); + + const apiKey = await vaultConnector + .user(team) + .get('HUGGINGFACE_API_KEY') + .catch((e) => { + console.log(e); + throw new Error('Failed to get HuggingFace API Key from vault. Please add HUGGINGFACE_API_KEY to your vault.'); + }); + + if (!apiKey) { + throw new Error('HuggingFace testing API Key is not set. Please set the key in vault.json to run this test.'); + } + }); + + it('should pass prompt with a text input', async () => { + // @ts-ignore + const agent = new Agent(); + const hfComp = new HuggingFace(); + + const output = await hfComp.process( + { + Text: 'A photo', + }, + { + data: { + accessToken: getApiKeyVaultKeyName(), + desc: '', + disableCache: false, + displayName: 'fasttext-language-identification', + logoUrl: '', + modelName: 'facebook/fasttext-language-identification', + modelTask: 'text-classification', + name: 'facebook/fasttext-language-identification', + parameters: JSON.stringify({}), + }, + }, + agent + ); + + const response = output.Output; + + expect(response).toBeDefined(); + expect(output._error).toBeUndefined(); + }, 60_000); + + it('should pass prompt with a local binary input', async () => { + // const imagePath = '../../data/avatar.png'; + const imagePath = path.resolve(__dirname, '../../data/avatar.png'); + + const base64Str = await util.promisify(fs.readFile)(imagePath, { encoding: 'base64' }); + const base64Url = `data:image/png;base64,${base64Str}`; + // @ts-ignore + const agent = new Agent(); + const hfComp = new HuggingFace(); + + const output = await hfComp.process( + { + Image: base64Url, + }, + { + data: { + accessToken: getApiKeyVaultKeyName(), + desc: "Zero-shot image classification based on OpenAI's CLIP model using Vision Transformer with large patches.", + disableCache: false, + displayName: 'clip-vit-large-patch14', + logoUrl: '', + modelName: 'openai/clip-vit-large-patch14', + modelTask: 'zero-shot-image-classification', + name: 'openai/clip-vit-large-patch14', + parameters: JSON.stringify({ + candidate_labels: ['woman', 'cat', 'dog'], + }), + }, + }, + agent + ); + + const response = output.Output; + + expect(response).toBeDefined(); + expect(output._error).toBeUndefined(); + }, 60_000); + + it('should pass prompt with a remote binary input', async () => { + // @ts-ignore + const agent = new Agent(); + const hfComp = new HuggingFace(); + + const output = await hfComp.process( + { + Image: 'https://i.imgur.com/LY998xU.jpeg', + }, + { + data: { + accessToken: getApiKeyVaultKeyName(), + desc: "Zero-shot image classification based on OpenAI's CLIP model using Vision Transformer with large patches.", + disableCache: false, + displayName: 'clip-vit-large-patch14', + logoUrl: '', + modelName: 'openai/clip-vit-large-patch14', + modelTask: 'zero-shot-image-classification', + name: 'openai/clip-vit-large-patch14', + parameters: JSON.stringify({ + candidate_labels: ['dog', 'cat', 'lion'], + }), + }, + }, + agent + ); + + const response = output.Output; + + expect(response).toBeDefined(); + expect(output._error).toBeUndefined(); + }, 60_000); + + it('should return a binary output with a smythfs:// uri', async () => { + // vi.mock('@huggingface/inference', () => ({ + // HfInference: vi.fn().mockImplementation(() => ({ + // // dummy blob of a png image + // textToImage: vi.fn().mockResolvedValue(new Blob()), + // })), + // })); + + // @ts-ignore + const agent = new Agent(); + const hfComp = new HuggingFace(); + + const output = await hfComp.process( + { + Text: 'anime artwork, anime style', + }, + { + data: { + accessToken: getApiKeyVaultKeyName(), + desc: '', + disableCache: false, + displayName: 'aipicasso/emi', + logoUrl: '', + modelName: 'aipicasso/emi', + modelTask: 'text-to-image', + name: 'aipicasso/emi', + parameters: JSON.stringify({}), + }, + }, + agent + ); + + const response = output.Output; + + expect(output._error).toBeUndefined(); + expect(response).toBeDefined(); + + const previewUrl = response?.url; + expect(previewUrl).toBeDefined(); + // expect(previewUrl, 'The output should be a valid URL to an image file').toMatch(/^https:\/\/.*\.(jpg|jpeg|png|gif)$/); + + // should match: smythfs://.team//_temp/ + expect(previewUrl, 'The output should be a valid SmythFS URI that points to the image file').toMatch(/^smythfs:\/\/.*\.team\/.*\/_temp\/.*$/); + + expect(response).toBeDefined(); + expect(output._error).toBeUndefined(); + }, 90_000); + + it('should corectly pass a binary HuggingFace ouptut to another binary HuggingFace input', async () => { + // @ts-ignore + const agent = new Agent(); + const hfComp = new HuggingFace(); + + const output1 = await hfComp.process( + { + Text: 'anime artwork, anime style', + }, + { + data: { + accessToken: getApiKeyVaultKeyName(), + desc: '', + disableCache: false, + displayName: 'aipicasso/emi', + logoUrl: '', + modelName: 'aipicasso/emi', + modelTask: 'text-to-image', + name: 'aipicasso/emi', + parameters: JSON.stringify({}), + }, + }, + agent + ); + + const response1 = output1.Output; + + expect(output1._error).toBeUndefined(); + expect(response1).toBeDefined(); + + const previewUrl = response1?.url; + expect(previewUrl).toBeDefined(); + // expect(previewUrl, 'The output should be a valid URL to an image file').toMatch(/^https:\/\/.*\.(jpg|jpeg|png|gif)$/); + + // should match: smythfs://.team//_temp/ + expect(previewUrl, 'The output should be a valid SmythFS URI that points to the image file').toMatch(/^smythfs:\/\/.*\.team\/.*\/_temp\/.*$/); + + expect(response1).toBeDefined(); + expect(output1._error).toBeUndefined(); + + const output2 = await hfComp.process( + { + Image: response1, + }, + { + data: { + accessToken: getApiKeyVaultKeyName(), + disableCache: false, + parameters: JSON.stringify({}), + modelName: 'facebook/detr-resnet-50', + modelTask: 'object-detection', + inputConfig: '{"Image":"URL | base64 | file | SmythFileObject"}', + name: 'facebook/detr-resnet-50', + displayName: 'detr-resnet-50', + desc: 'An object detection model developed by Facebook using a ResNet-50 backbone network architecture.', + logoUrl: + 'https://aeiljuispo.cloudimg.io/v7/https://cdn-uploads.huggingface.co/production/uploads/1592839207516-noauth.png?w=200&h=200&f=face', + }, + }, + agent + ); + + const response2 = output2.Output; + + console.log(output2._error); + expect(output2._error).toBeUndefined(); + expect(response2).toBeDefined(); + }, 90_000); +}); diff --git a/packages/core/tests/integration/008-Components/ImageGenerator.test.ts b/packages/core/tests/integration/008-Components/ImageGenerator.test.ts new file mode 100644 index 000000000..1826ef4f1 --- /dev/null +++ b/packages/core/tests/integration/008-Components/ImageGenerator.test.ts @@ -0,0 +1,124 @@ +import Agent from '@sre/AgentManager/Agent.class'; +import HuggingFace from '@sre/Components/HuggingFace.class'; +import LLMAssistant from '@sre/Components/LLMAssistant.class'; +import { config, SmythRuntime } from '@sre/index'; +import { delay } from '@sre/utils/date-time.utils'; +import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; +import fs from 'fs'; +import util from 'util'; +import path from 'path'; +import Classifier from '@sre/Components/Classifier.class'; +import ImageGenerator from '@sre/Components/ImageGenerator.class'; +import { GenerateImageConfig } from '@sre/types/LLM.types'; + +const sre = SmythRuntime.Instance.init({ + CLI: { + Connector: 'CLI', + }, + Storage: { + Connector: 'S3', + Settings: { + bucket: config.env.AWS_S3_BUCKET_NAME || '', + region: config.env.AWS_S3_REGION || '', + accessKeyId: config.env.AWS_ACCESS_KEY_ID || '', + secretAccessKey: config.env.AWS_SECRET_ACCESS_KEY || '', + }, + }, + Cache: { + Connector: 'Redis', + Settings: { + hosts: config.env.REDIS_SENTINEL_HOSTS, + name: config.env.REDIS_MASTER_NAME || '', + password: config.env.REDIS_PASSWORD || '', + }, + }, + AgentData: { + Connector: 'Local', + Settings: { + devDir: './tests/data/AgentData', + prodDir: './tests/data/AgentData', + }, + }, + Vault: { + Connector: 'JSONFileVault', + Settings: { + file: './tests/data/vault.json', + }, + }, + Account: { + Connector: 'DummyAccount', + Settings: { + oAuthAppID: process.env.LOGTO_M2M_APP_ID, + oAuthAppSecret: process.env.LOGTO_M2M_APP_SECRET, + oAuthBaseUrl: `${process.env.LOGTO_SERVER}/oidc/token`, + oAuthResource: process.env.LOGTO_API_RESOURCE, + oAuthScope: '', + smythAPIBaseUrl: process.env.SMYTH_API_BASE_URL, + }, + }, +}); + +// Mock Agent class to keep the test isolated from the actual Agent implementation +vi.mock('@sre/AgentManager/Agent.class', () => { + const MockedAgent = vi.fn().mockImplementation(() => ({ + id: 'agent-123456', + agentRuntime: { debug: true }, // used inside createComponentLogger() + })); + return { default: MockedAgent }; +}); + +const TIMEOUT = 30000; + +describe('ImageGenerator Component', () => { + it('should generate an image and return the URL', async () => { + const imageGenerator = new ImageGenerator(); + const configData: GenerateImageConfig & { responseFormat: 'url' | 'b64_json'; prompt: string } = { + model: 'dall-e-3', + responseFormat: 'url', + prompt: 'A beautiful landscape with a river and mountains', + }; + + // @ts-ignore + const agent = new Agent(); + const result = await imageGenerator.process({}, { data: configData }, agent); + + expect(result._error).toBeUndefined(); + expect(result.Output).toBeDefined(); + // match any valid URL. + expect(result.Output).toMatch(/^https:\/\/[^ ]+$/); + }, TIMEOUT); + + it('should generate an image and return the base64', async () => { + const imageGenerator = new ImageGenerator(); + const configData = { + model: 'dall-e-3', + responseFormat: 'b64_json', + prompt: 'A beautiful landscape with a river and mountains', + }; + + // @ts-ignore + const agent = new Agent(); + const result = await imageGenerator.process({}, { data: configData }, agent); + + expect(result._error).toBeUndefined(); + expect(result.Output).toBeDefined(); + expect(result.Output).toMatch(/^[A-Za-z0-9+/]+={0,2}$/); + //* it is a base64 image but not base64 URL + }, TIMEOUT); + + it('should throw an error when no prompt is given', async () => { + const imageGenerator = new ImageGenerator(); + const configData: GenerateImageConfig & { responseFormat: 'url' | 'b64_json'; prompt: string } = { + model: 'dall-e-3', + responseFormat: 'url', + prompt: '', + }; + + // @ts-ignore + const agent = new Agent(); + + const result = await imageGenerator.process({}, { data: configData }, agent); + + expect(result._error).toBeDefined(); + }, TIMEOUT); +}); diff --git a/packages/core/tests/integration/008-Components/LLMAssistant.test.ts b/packages/core/tests/integration/008-Components/LLMAssistant.test.ts new file mode 100644 index 000000000..aa9ff3a2d --- /dev/null +++ b/packages/core/tests/integration/008-Components/LLMAssistant.test.ts @@ -0,0 +1,320 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import Agent from '@sre/AgentManager/Agent.class'; +import LLMAssistant from '@sre/Components/LLMAssistant.class'; +import { SmythRuntime } from '@sre/index'; +import { delay } from '@sre/utils/date-time.utils'; +import { TLLMMessageRole } from '@sre/types/LLM.types'; +import { ConnectorService } from '@sre/Core/ConnectorsService'; +import { LLMRegistry } from '@sre/LLMManager/LLMRegistry.class'; +import { type LLMConnector } from '@sre/LLMManager/LLM.service/LLMConnector'; + +//We need SRE to be loaded because LLMAssistant uses internal SRE functions +const sre = SmythRuntime.Instance.init({ + Storage: { + Connector: 'S3', + Settings: { + bucket: process.env.AWS_S3_BUCKET_NAME || '', + region: process.env.AWS_S3_REGION || '', + accessKeyId: process.env.AWS_ACCESS_KEY_ID || '', + secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY || '', + }, + }, + Cache: { + Connector: 'Redis', + Settings: { + hosts: process.env.REDIS_SENTINEL_HOSTS, + name: process.env.REDIS_MASTER_NAME || '', + password: process.env.REDIS_PASSWORD || '', + }, + }, + AgentData: { + Connector: 'Local', + Settings: { + devDir: './tests/data/AgentData', + prodDir: './tests/data/AgentData', + }, + }, + Account: { + Connector: 'SmythAccount', + Settings: { + oAuthAppID: process.env.LOGTO_M2M_APP_ID, + oAuthAppSecret: process.env.LOGTO_M2M_APP_SECRET, + oAuthBaseUrl: `${process.env.LOGTO_SERVER}/oidc/token`, + oAuthResource: process.env.LOGTO_API_RESOURCE, + oAuthScope: '', + smythAPIBaseUrl: process.env.SMYTH_API_BASE_URL, + }, + }, + Vault: { + Connector: 'SmythVault', + Settings: { + oAuthAppID: process.env.LOGTO_M2M_APP_ID, + oAuthAppSecret: process.env.LOGTO_M2M_APP_SECRET, + oAuthBaseUrl: `${process.env.LOGTO_SERVER}/oidc/token`, + oAuthResource: process.env.LOGTO_API_RESOURCE, + oAuthScope: '', + vaultAPIBaseUrl: process.env.SMYTH_VAULT_API_BASE_URL, + }, + }, +}); + +const TEAM_ID = 'cloilcrl9001v9tkguilsu8dx'; // Team ID of Forhad + +// Mock Agent class to keep the test isolated from the actual Agent implementation +vi.mock('@sre/AgentManager/Agent.class', () => { + const MockedAgent = vi.fn().mockImplementation(() => { + // Inherit Agent.prototype for proper instanceof Agent checks + return Object.create(Agent.prototype, { + id: { value: 'cm0zjhkzx0dfvhxf81u76taiz' }, + teamId: { value: TEAM_ID }, + agentRuntime: { value: { debug: true } }, // used inside createComponentLogger() + }); + }); + return { default: MockedAgent }; +}); + +const TIMEOUT = 30000; +const LLM_OUTPUT_VALIDATOR = 'Yohohohooooo!'; + +function testProcessFunction(model) { + let llmAssistant: LLMAssistant; + let agent: Agent; + let config: any; + + beforeEach(() => { + // @ts-ignore (Ignore required arguments, as we are using the mocked Agent) + agent = new Agent(); + llmAssistant = new LLMAssistant(); + config = { + name: 'LLMAssistant', + inputs: [], + data: { + model, + ttl: 5 * 60, // default expiration time for conversation cache + behavior: `You are a friendly and funny assistant. It is absolutely mandatory that you start and finish every message with the exact phrase "${LLM_OUTPUT_VALIDATOR}". Failure to include "${LLM_OUTPUT_VALIDATOR}" at both the beginning and end of your response will result in an incorrect output. If you don't know an answer, simply say "I don't know". This requirement is non-negotiable and must be adhered to in every response.`, + }, + }; + }); + + afterEach(() => { + // ⚠️ Warning from vitest doc (https://vitest.dev/guide/mocking#mocking) - "Always remember to clear or restore mocks before or after each test run to undo mock state changes between runs!" + vi.clearAllMocks(); + }); + + it( + 'Conversation with no context - UserId and ConversationId are empty ', + async () => { + const input = { UserInput: 'What is your prefered movie?', UserId: '', ConversationId: '' }; + + config.inputs = [{ name: 'UserInput' }, { name: 'UserId' }, { name: 'ConversationId' }]; + + const result = await llmAssistant.process(input, config, agent); + + expect(result.Response).toContain(LLM_OUTPUT_VALIDATOR); + }, + TIMEOUT * 2 + ); + + it( + 'Conversation with context ', + async () => { + const input = { UserInput: 'Hi, my name is Smyth, who are you?', UserId: '', ConversationId: 'SmythTestConversation0001' }; + + config.inputs = [{ name: 'UserInput' }, { name: 'UserId' }, { name: 'ConversationId' }]; + + let result = await llmAssistant.process(input, config, agent); + + expect(result.Response).toContain(LLM_OUTPUT_VALIDATOR); + + input.UserInput = 'What is LLM?'; + result = await llmAssistant.process(input, config, agent); + expect(result.Response).toContain(LLM_OUTPUT_VALIDATOR); + + input.UserInput = 'Hi again, Do you remember my name ?'; + result = await llmAssistant.process(input, config, agent); + expect(result.Response).toContain(LLM_OUTPUT_VALIDATOR); + expect(result.Response).toContain('Smyth'); + }, + TIMEOUT * 5 + ); + + it( + 'Conversation with context that expires', + async () => { + const input = { UserInput: 'Hi, my name is Smyth, who are you ?', UserId: '', ConversationId: 'SmythTestConversation0002' }; + config.data.ttl = 10; // 10 seconds + config.inputs = [{ name: 'UserInput' }, { name: 'UserId' }, { name: 'ConversationId' }]; + + let result = await llmAssistant.process(input, config, agent); + + expect(result.Response).toContain(LLM_OUTPUT_VALIDATOR); + + input.UserInput = 'What is your prefered movie?'; + result = await llmAssistant.process(input, config, agent); + expect(result.Response).toContain(LLM_OUTPUT_VALIDATOR); + + await delay(15 * 1000); // wait for the conversation context to expire + + input.UserInput = 'Hi again, Do you remember my name ?'; + result = await llmAssistant.process(input, config, agent); + expect(result.Response.toLowerCase().indexOf('smyth')).toBe(-1); + }, + TIMEOUT * 5 + ); +} + +const models = [ + { provider: 'OpenAI', id: 'gpt-4o-mini' }, + { provider: 'Anthropic', id: 'claude-3-haiku-20240307' }, + { provider: 'GoogleAI', id: 'gemini-1.5-flash' }, + { provider: 'Groq', id: 'gemma2-9b-it' }, + { provider: 'TogetherAI', id: 'meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo' }, + { provider: 'Bedrock', id: 'SRE - Bedrock with AI21 Labs - Jamba-Instruct' }, + { provider: 'Bedrock', id: 'SRE - Bedrock does not support System Prompt' }, + { provider: 'VertexAI', id: 'SRE - Vertex AI with Gemini Flash' }, +]; + +models.forEach((model, index) => { + describe(`LLMAssistant: test process function: ${model.provider} (${model.id})`, () => { + testProcessFunction(model.id); + }); +}); + +describe('LLMAssistant: test process function with model switching', () => { + it( + 'should switch models for the same conversation and maintain context', + async () => { + const llmAssistant = new LLMAssistant(); + // @ts-ignore (Ignore required arguments, as we are using the mocked Agent) + const agent = new Agent(); + const conversationId = 'ModelSwitchingTest001'; + + const config = { + name: 'LLMAssistant', + inputs: [{ name: 'UserInput' }, { name: 'UserId' }, { name: 'ConversationId' }], + data: { + model: 'gpt-4o-mini', + ttl: 30 * models.length, + behavior: '', + }, + }; + + for (const model of models) { + config.data.model = model.id; + config.data.behavior = `You are an AI assistant specializing in different types of Large Language Models (LLMs). Follow these instructions: + 1. Begin and end every message with ${LLM_OUTPUT_VALIDATOR} + 2. Always include the name of the current model (${model.id}) in lowercase somewhere in your response. + 3. If you don't know something, simply say "I don't know" instead of guessing. + 4. Maintain context from previous messages in the conversation. + 5. Provide concise, accurate answers related to LLMs and general knowledge. + 6. Be friendly and engaging in your responses.`; + + const input = { + UserInput: `What is the provider of model ${model.id} and is it a good model?`, + UserId: '', + ConversationId: conversationId, + }; + + const result = await llmAssistant.process(input, config, agent); + const response = result.Response; + + expect(response).toBeTruthy(); + expect(response).toContain(LLM_OUTPUT_VALIDATOR); + expect(response.toLowerCase()).toContain(model.id.toLowerCase()); + } + + // Set up the final test with GPT-4 + config.data.model = 'gpt-4o-mini'; + config.data.behavior = `You are an AI assistant tasked with summarizing a conversation about various LLM models. Follow these instructions: + 1. Begin and end every message with ${LLM_OUTPUT_VALIDATOR} + 2. Provide a concise summary of the previous conversation, mentioning each model discussed. + 3. Include at least one strength or characteristic for each model mentioned. + 4. If you're unsure about any details, state "I'm not certain about [specific detail]" rather than guessing. + 5. Keep your response friendly and engaging.`; + + const input = { + UserInput: `Summarize our conversation about the different LLM models we discussed.`, + UserId: '', + ConversationId: conversationId, + }; + + const result = await llmAssistant.process(input, config, agent); + const response = result.Response; + + expect(response).toBeTruthy(); + expect(response).toContain(LLM_OUTPUT_VALIDATOR); + expect(response.split(' ').length).toBeGreaterThan(20); // Ensure a substantial summary + }, + TIMEOUT * (models.length + 1) // Additional 30 seconds for getting credentials of custom LLM + ); +}); + +describe('LLMAssistant: test getConsistentMessages()', () => { + for (const model of models) { + let llmConnector: LLMConnector; + + const isStandardLLM = LLMRegistry.isStandardLLM(model.id); + + if (isStandardLLM) { + llmConnector = ConnectorService.getLLMConnector(model.provider); + } else { + llmConnector = ConnectorService.getLLMConnector(model.provider); + } + + it(`should remove duplicate user messages at the beginning and end: ${model.provider} (${model.id})`, async () => { + const content1 = 'Hello, how are you?'; + const content2 = 'Hello, how are you doing?'; + const content3 = 'I am fine, thank you!'; + const content4 = 'What can you do?'; + const content5 = ''; + + const messages = [ + { role: TLLMMessageRole.User, content: content1 }, + { role: TLLMMessageRole.User, content: content2 }, // Duplicate at the beginning + { role: TLLMMessageRole.Assistant, content: content3 }, + { role: TLLMMessageRole.User, content: content4 }, + { role: TLLMMessageRole.User, content: content5 }, // Duplicate at the end + ]; + + const consistentMessages = llmConnector.getConsistentMessages(messages); + + expect(consistentMessages).toHaveLength(messages.length - 2); + + expect(consistentMessages[0].role).toBe(TLLMMessageRole.User); + expect(_getMessageContent(model.provider, consistentMessages[0])).toBe(content2); + + expect(consistentMessages[1].role).toBe(_getLLMMessageRole(model.provider)); + expect(_getMessageContent(model.provider, consistentMessages[1])).toBe(content3); + + expect(consistentMessages[2].role).toBe(TLLMMessageRole.User); + expect(_getMessageContent(model.provider, consistentMessages[2])).toBe(content4); + }); + } +}); + +function _getLLMMessageRole(provider): String { + let role = ''; + + if (provider === 'GoogleAI') { + role = TLLMMessageRole.Model; + } else { + role = TLLMMessageRole.Assistant; + } + + return role; +} + +function _getMessageContent(provider, message): String { + let content = ''; + + if (['GoogleAI', 'VertexAI'].includes(provider)) { + content = message?.parts?.[0]?.text; + } else if (provider === 'Bedrock') { + content = message?.content?.[0]?.text; + } else { + content = message?.content; + } + + return content; +} diff --git a/packages/core/tests/integration/008-Components/MultimodalLLM.test.ts b/packages/core/tests/integration/008-Components/MultimodalLLM.test.ts new file mode 100644 index 000000000..44a5b9d71 --- /dev/null +++ b/packages/core/tests/integration/008-Components/MultimodalLLM.test.ts @@ -0,0 +1,83 @@ +import fs from 'fs'; +import { describe, expect, it, vi, beforeEach } from 'vitest'; + +import { AgentProcess } from '@sre/Core/AgentProcess.helper'; +import { SmythRuntime } from '@sre/index'; + +const sre = SmythRuntime.Instance.init({ + Storage: { + Connector: 'S3', + Settings: { + bucket: process.env.AWS_S3_BUCKET_NAME || '', + region: process.env.AWS_S3_REGION || '', + accessKeyId: process.env.AWS_ACCESS_KEY_ID || '', + secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY || '', + }, + }, + AgentData: { + Connector: 'Local', + Settings: { + devDir: './tests/data/AgentData', + prodDir: './tests/data/AgentData', + }, + }, + Vault: { + Connector: 'JSONFileVault', + Settings: { + file: './tests/data/vault.json', + }, + }, + Account: { + Connector: 'DummyAccount', + Settings: { + oAuthAppID: process.env.LOGTO_M2M_APP_ID, + oAuthAppSecret: process.env.LOGTO_M2M_APP_SECRET, + oAuthBaseUrl: `${process.env.LOGTO_SERVER}/oidc/token`, + oAuthResource: process.env.LOGTO_API_RESOURCE, + oAuthScope: '', + smythAPIBaseUrl: process.env.SMYTH_API_BASE_URL, + }, + }, +}); + +const TIMEOUT = 30000; + +// TODO [Forhad]: Need to write more test cases + +function runTestCases(endpoint: string) { + const videoUrl = 'https://storage.googleapis.com/generativeai-downloads/images/GreatRedSpot.mp4'; + + it( + 'should generate a relevant response for a given prompt with attached file', + async () => { + let error; + try { + const agentData = fs.readFileSync('./tests/data/sre-llm.smyth', 'utf-8'); + const data = JSON.parse(agentData); + + const agentProcess = AgentProcess.load(data); + + let res = await agentProcess.run({ + method: 'POST', + path: endpoint, + body: { + Input: videoUrl, + }, + }); + + const output = res?.data?.result?.Reply; + + expect(output).toBeTruthy(); + } catch (e) { + error = e; + console.error(e.message); + } + expect(error).toBeUndefined(); + }, + TIMEOUT * 20 // 10 mins + ); +} + +describe('MultimodalLLM', () => { + runTestCases('/api/read-video-file-with-multimodal-llm'); +}); diff --git a/packages/core/tests/integration/008-Components/PromptGenerator.test.ts b/packages/core/tests/integration/008-Components/PromptGenerator.test.ts new file mode 100644 index 000000000..40d9bf6b7 --- /dev/null +++ b/packages/core/tests/integration/008-Components/PromptGenerator.test.ts @@ -0,0 +1,138 @@ +import fs from 'fs'; +import { describe, expect, it } from 'vitest'; + +import { AgentProcess } from '@sre/Core/AgentProcess.helper'; +import config from '@sre/config'; +import { CLIAgentDataConnector, ConnectorService, SmythRuntime } from '@sre/index'; +import { TConnectorService } from '@sre/types/SRE.types'; + +const sre = SmythRuntime.Instance.init({ + Storage: { + Connector: 'S3', + Settings: { + bucket: config.env.AWS_S3_BUCKET_NAME || '', + region: config.env.AWS_S3_REGION || '', + accessKeyId: config.env.AWS_ACCESS_KEY_ID || '', + secretAccessKey: config.env.AWS_SECRET_ACCESS_KEY || '', + }, + }, + AgentData: { + Connector: 'Local', + Settings: { + devDir: './tests/data/AgentData', + prodDir: './tests/data/AgentData', + }, + }, + Account: { + Connector: 'SmythAccount', + Settings: { + oAuthAppID: process.env.LOGTO_M2M_APP_ID, + oAuthAppSecret: process.env.LOGTO_M2M_APP_SECRET, + oAuthBaseUrl: `${process.env.LOGTO_SERVER}/oidc/token`, + oAuthResource: process.env.LOGTO_API_RESOURCE, + oAuthScope: '', + smythAPIBaseUrl: process.env.SMYTH_API_BASE_URL, + }, + }, + Vault: { + Connector: 'SmythVault', + Settings: { + oAuthAppID: process.env.LOGTO_M2M_APP_ID, + oAuthAppSecret: process.env.LOGTO_M2M_APP_SECRET, + oAuthBaseUrl: `${process.env.LOGTO_SERVER}/oidc/token`, + oAuthResource: process.env.LOGTO_API_RESOURCE, + oAuthScope: '', + vaultAPIBaseUrl: process.env.SMYTH_VAULT_API_BASE_URL, + }, + }, +}); + +ConnectorService.register(TConnectorService.AgentData, 'CLI', CLIAgentDataConnector); +ConnectorService.init(TConnectorService.AgentData, 'CLI'); + +// TODO [Forhad]: Need to implement more test cases for PromptGenerator +// - expect JSON output +// - expect error when model is not supported +// - run test cases for all providers +// - Need to separate test cases for custom models, as custom models require SmythAccount account connector +// - provide some outputs params and check if it's response with those params properly + +const TIMEOUT = 30000; +const LLM_OUTPUT_VALIDATOR = 'Yohohohooooo!'; +const WORD_INCLUSION_PROMPT = `\nThe response must includes "${LLM_OUTPUT_VALIDATOR}". If the response is JSON, then include an additional key-value pair with key as "${LLM_OUTPUT_VALIDATOR}" and value as "${LLM_OUTPUT_VALIDATOR}"`; + +describe('PromptGenerator Component - Echo', () => { + it('should echo the prompt', async () => { + const agentData = fs.readFileSync('./tests/data/sre-llm.smyth', 'utf-8'); + const data = JSON.parse(agentData); + + const agentProcess = AgentProcess.load(data); + + const obj = { a: 1, b: 2, c: [1, 2] }; + + let res = await agentProcess.run({ + method: 'POST', + path: '/api/test-echo', + body: { + Input: JSON.stringify(obj), + }, + }); + + const output = res?.data?.result?.Reply; + + expect(output).toEqual(obj); + }); +}); + +function runTestCasesWithAgent(endpoint: string) { + it( + 'should generate a relevant response for a given prompt', + async () => { + let error; + try { + const agentData = fs.readFileSync('./tests/data/sre-llm.smyth', 'utf-8'); + const data = JSON.parse(agentData); + + const agentProcess = AgentProcess.load(data); + + let res = await agentProcess.run({ + method: 'POST', + path: endpoint, + body: { + Input: + 'What is the largest planet in our solar system, and how does it compare to other planets in the Milky Way galaxy?' + + WORD_INCLUSION_PROMPT, + }, + }); + + const output = res?.data?.result?.Reply; + + expect(output).toBeTruthy(); + expect(output).toBeTypeOf('object'); + expect(JSON.stringify(output)).toContain(LLM_OUTPUT_VALIDATOR); + } catch (e) { + error = e; + console.error(e.message); + } + expect(error).toBeUndefined(); + }, + TIMEOUT * 2 + ); +} + +const llmProviderEndpoints = [ + { provider: 'OpenAI', endpoint: '/api/test-openai-model' }, + { provider: 'Anthropic', endpoint: '/api/test-anthropic-model' }, + { provider: 'GoogleAI', endpoint: '/api/test-googleai-model' }, + { provider: 'Groq', endpoint: '/api/test-groq-model' }, + { provider: 'TogetherAI', endpoint: '/api/test-togetherai-model' }, + { provider: 'Bedrock', endpoint: '/api/test-bedrock-model' }, + { provider: 'Bedrock', endpoint: '/api/test-bedrock-model-that-does-not-support-system' }, + { provider: 'VertexAI', endpoint: '/api/test-vertexai-model' }, +]; + +for (const endpoint of llmProviderEndpoints) { + describe(`PromptGenerator Component - ${endpoint.provider} (${endpoint.endpoint})`, () => { + runTestCasesWithAgent(endpoint.endpoint); + }); +} diff --git a/packages/core/tests/integration/008-Components/ServerlessCode.test.ts b/packages/core/tests/integration/008-Components/ServerlessCode.test.ts new file mode 100644 index 000000000..8ef547c01 --- /dev/null +++ b/packages/core/tests/integration/008-Components/ServerlessCode.test.ts @@ -0,0 +1,77 @@ +import { AgentProcess } from '@sre/Core/AgentProcess.helper'; +import config from '@sre/config'; +import { CLIAgentDataConnector, ConnectorService, SmythRuntime } from '@sre/index'; +import { TConnectorService } from '@sre/types/SRE.types'; +import fs from 'fs'; + +import { describe, expect, it } from 'vitest'; +const sre = SmythRuntime.Instance.init({ + Storage: { + Connector: 'S3', + Settings: { + bucket: process.env.AWS_S3_BUCKET_NAME || '', + region: process.env.AWS_S3_REGION || '', + accessKeyId: process.env.AWS_ACCESS_KEY_ID || '', + secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY || '', + }, + }, + AgentData: { + Connector: 'Local', + Settings: { + devDir: './tests/data/AgentData', + prodDir: './tests/data/AgentData', + }, + }, + Vault: { + Connector: 'SecretsManager', + Settings: { + region: process.env.AWS_REGION, + }, + }, + Account: { + Connector: 'DummyAccount', + Settings: { + oAuthAppID: process.env.LOGTO_M2M_APP_ID, + oAuthAppSecret: process.env.LOGTO_M2M_APP_SECRET, + oAuthBaseUrl: `${process.env.LOGTO_SERVER}/oidc/token`, + oAuthResource: process.env.LOGTO_API_RESOURCE, + oAuthScope: '', + smythAPIBaseUrl: process.env.SMYTH_API_BASE_URL, + }, + }, + Log: { + Connector: 'ConsoleLog', + }, +}); + +ConnectorService.register(TConnectorService.AgentData, 'CLI', CLIAgentDataConnector); +ConnectorService.init(TConnectorService.AgentData, 'CLI'); +describe('ServerlessCode Component', () => { + it('ServerlessCode : Run lambda Code and return output', async () => { + let error; + try { + const agentData = fs.readFileSync('./tests/data/serverless-code-sample.smyth', 'utf-8'); + const data = JSON.parse(agentData); + const date = new Date(); + + const agentProcess = AgentProcess.load(data); + + let output = await agentProcess.run({ + method: 'POST', + path: '/api/test', + body: { + name: 'test', + description: 'test', + }, + }); + + console.log('>>>>>>', output?.data?.result); + expect(output?.data?.result?.Output).toBeDefined(); + } catch (e) { + error = e; + console.error(e.message); + } + expect(error).toBeUndefined(); + }); + +}); diff --git a/packages/core/tests/integration/008-Components/VisionLLM.test.ts b/packages/core/tests/integration/008-Components/VisionLLM.test.ts new file mode 100644 index 000000000..558229279 --- /dev/null +++ b/packages/core/tests/integration/008-Components/VisionLLM.test.ts @@ -0,0 +1,74 @@ +import fs from 'fs'; +import { describe, expect, it } from 'vitest'; +import { AgentProcess } from '@sre/Core/AgentProcess.helper'; +import { SmythRuntime } from '@sre/index'; + +const sre = SmythRuntime.Instance.init({ + Storage: { + Connector: 'S3', + Settings: { + bucket: process.env.AWS_S3_BUCKET_NAME || '', + region: process.env.AWS_S3_REGION || '', + accessKeyId: process.env.AWS_ACCESS_KEY_ID || '', + secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY || '', + }, + }, + AgentData: { + Connector: 'Local', + Settings: { + devDir: './tests/data/AgentData', + prodDir: './tests/data/AgentData', + }, + }, + Vault: { + Connector: 'JSONFileVault', + Settings: { + file: './tests/data/vault.json', + }, + }, + Account: { + Connector: 'DummyAccount', + Settings: { + oAuthAppID: process.env.LOGTO_M2M_APP_ID, + oAuthAppSecret: process.env.LOGTO_M2M_APP_SECRET, + oAuthBaseUrl: `${process.env.LOGTO_SERVER}/oidc/token`, + oAuthResource: process.env.LOGTO_API_RESOURCE, + oAuthScope: '', + smythAPIBaseUrl: process.env.SMYTH_API_BASE_URL, + }, + }, +}); + +const TIMEOUT = 30000; + +// TODO [Forhad]: Need to write more test cases + +function runTestCases(endpoint: string) { + const imageUrl1 = 'https://images.unsplash.com/photo-1721332155637-8b339526cf4c?q=10&w=300'; + + it( + 'should analyze a single image', + async () => { + const agentData = fs.readFileSync('./tests/data/sre-llm.smyth', 'utf-8'); + const data = JSON.parse(agentData); + const agentProcess = AgentProcess.load(data); + + const res = await agentProcess.run({ + method: 'POST', + path: endpoint, + body: { + Input: imageUrl1, + }, + }); + + const output = res?.data?.result?.Reply; + expect(output).toBeTruthy(); + expect(output.length).toBeGreaterThan(20); + }, + TIMEOUT + ); +} + +describe('VisionLLM', () => { + runTestCases('/api/read-image-file-with-vision-llm'); +}); diff --git a/packages/core/tests/integration/008-Components/ZapierAction.test.ts b/packages/core/tests/integration/008-Components/ZapierAction.test.ts new file mode 100644 index 000000000..42417980a --- /dev/null +++ b/packages/core/tests/integration/008-Components/ZapierAction.test.ts @@ -0,0 +1,207 @@ +import Agent from '@sre/AgentManager/Agent.class'; +import HuggingFace from '@sre/Components/HuggingFace.class'; +import LLMAssistant from '@sre/Components/LLMAssistant.class'; +import { config, SmythRuntime } from '@sre/index'; +import { delay } from '@sre/utils/date-time.utils'; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; +import fs from 'fs'; +import util from 'util'; +import path from 'path'; +import ZapierAction from '@sre/Components/ZapierAction.class'; +import { ConnectorService, ConnectorServiceProvider } from '@sre/Core/ConnectorsService'; +import { AccessCandidate } from '@sre/Security/AccessControl/AccessCandidate.class'; +import { AccountConnector } from '@sre/Security/Account.service/AccountConnector'; +import { IAccessCandidate } from '@sre/types/ACL.types'; +import { TConnectorService } from '@sre/types/SRE.types'; +import { BinaryInput } from '@sre/helpers/BinaryInput.helper'; +import axios from 'axios'; +import http from 'http'; +import { promisify } from 'util'; +import express, { Router } from 'express'; + +// Specific getter for Zapier API key +const apiKeyVaultKeyName = (): string => { + // const apiKey = process.env.__TEST__ZAPIER_API_KEY; + // if (!apiKey) { + // throw new Error('Zapier testing API Key is not set. Please set the __TEST__ZAPIER_API_KEY environment variable to run this test.'); + // } + // // return apiKey; + return `{{KEY(ZAPIER_API_KEY)}}`; +}; + +//We need SRE to be loaded because LLMAssistant uses internal SRE functions + +const PORT = 8084; +const BASE_URL = `http://localhost:${PORT}`; +const app = express(); + +const sre = SmythRuntime.Instance.init({ + CLI: { + Connector: 'CLI', + }, + Storage: { + Connector: 'S3', + Settings: { + bucket: config.env.AWS_S3_BUCKET_NAME || '', + region: config.env.AWS_S3_REGION || '', + accessKeyId: config.env.AWS_ACCESS_KEY_ID || '', + secretAccessKey: config.env.AWS_SECRET_ACCESS_KEY || '', + }, + }, + + Account: { + Connector: 'DummyAccount', + }, + + Cache: { + Connector: 'Redis', + Settings: { + hosts: config.env.REDIS_SENTINEL_HOSTS, + name: config.env.REDIS_MASTER_NAME || '', + password: config.env.REDIS_PASSWORD || '', + }, + }, + AgentData: { + Connector: 'Local', + Settings: { + devDir: './tests/data/AgentData', + prodDir: './tests/data/AgentData', + }, + }, + Vault: { + Connector: 'JSONFileVault', + Settings: { + file: './tests/data/vault.json', + }, + }, + + Router: { + Connector: 'ExpressRouter', + Settings: { + router: app, + baseUrl: BASE_URL, + }, + }, +}); + +const server = http.createServer(app); + +const ZAPIER_DEFAULT_ACTION_ID = 'bef9dc04-e6f9-482a-9c45-513375182818'; + +// Mock Agent class to keep the test isolated from the actual Agent implementation +vi.mock('@sre/AgentManager/Agent.class', () => { + const MockedAgent = vi.fn().mockImplementation(() => ({ + id: 'agent-123456', + teamId: 'default', + agentRuntime: { debug: true }, // used inside createComponentLogger() + })); + return { default: MockedAgent }; +}); + +describe('ZapierAction Component', () => { + beforeAll(async () => { + // This will throw an error if the API key is not set + const vaultConnector = ConnectorService.getVaultConnector(); + const agent = AccessCandidate.agent('agent-123456'); + + const apiKey = await vaultConnector + .user(agent) + .get('ZAPIER_API_KEY') + .catch((e) => { + throw new Error('Failed to get Zapier API Key from vault. Please add ZAPIER_API_KEY to your vault.'); + }); + + console.log('apiKey', apiKey); + + if (!apiKey) { + throw new Error('Zapier testing API Key is not set. Please set the key in vault.json to run this test.'); + } + }); + + beforeAll(async () => { + const listen = promisify(server.listen.bind(server)); + await listen(PORT); + console.log(`Server is running on port ${PORT}`); + }); + + afterAll(async () => { + const close = promisify(server.close.bind(server)); + await close(); + console.log('Server has been shut down'); + }); + + it('triggers a zapier action', async () => { + // @ts-ignore + const agent = new Agent(); + const zapierAction = new ZapierAction(); + + //* the zapier code action code snippet: `output = [{isOk: true}];` + + const output = await zapierAction.process( + { + instructions: 'run code', + }, + { + data: { + actionId: ZAPIER_DEFAULT_ACTION_ID, + actionName: 'ANY NAME', + // apiKey: '{{KEY(Zapier (3))}}', + apiKey: apiKeyVaultKeyName(), + logoUrl: 'https://app.smythos.dev/img/zapier.png', + params: '{"instructions":"run 1+1"}', + }, + }, + agent + ); + + const response = output.Output; + expect(output._error).toBeUndefined(); + expect(response).toBeDefined(); + + expect(response?.result?.isOk).toBe(true); + }, 60_000); + + it('should pass a temp pub url of smyth file input to the action', async () => { + //@ts-ignore + const agent = new Agent(); + const zapierAction = new ZapierAction(); + const img = await fs.promises.readFile(path.join(__dirname, '../../data/avatar.png')); + const obj = await BinaryInput.from(img).getJsonData(AccessCandidate.agent(agent.id)); + + //* the zapier code action code snippet: `output = [{isOk: true}];` + + // mock axios.post to see the passed inputs + const spy = vi.spyOn(axios, 'post').mockResolvedValueOnce({ data: { result: { isOk: true } } }); + + const output = await zapierAction.process( + { + instructions: 'run code', + img: obj, + }, + { + data: { + actionId: ZAPIER_DEFAULT_ACTION_ID, + actionName: 'ANY NAME', + // apiKey: '{{KEY(Zapier (3))}}', + apiKey: apiKeyVaultKeyName(), + logoUrl: 'https://app.smythos.dev/img/zapier.png', + params: '{"instructions":"str", "img": "{{FILE(img)}}"}', + }, + }, + agent + ); + + //get the first argument of the first call to axios.post + const firstCallArguments = spy.mock.calls[0]; + // const url = (firstCallArguments[1] as any)?.img as Awaited>; + const url = (firstCallArguments[1] as any)?.img as string; + expect(url).toBeDefined(); + expect(url.startsWith(`${BASE_URL}/_temp/`)).toBe(true); + + // expect the pub url access to be deleted after the the agent finishes processing + // wait for 5 seconds because the temp url deletion run in the background + await delay(5000); + const responseErr = await axios.get(url).catch((e) => e); + expect(responseErr?.response?.status || responseErr?.status, 'temp url should not be accessible after agent processing finished').toBe(404); + }); +}); diff --git a/packages/core/tests/integration/llm/conversation.test.ts b/packages/core/tests/integration/llm/conversation.test.ts deleted file mode 100644 index 0f2d8b0de..000000000 --- a/packages/core/tests/integration/llm/conversation.test.ts +++ /dev/null @@ -1,170 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { Conversation } from '@sre/helpers/Conversation.helper'; -import { setupSRE } from '../../utils/sre'; -import { models as LLM_MODELS } from '@sre/LLMManager/models'; - -const models = { - 'gpt-4o-mini': LLM_MODELS['gpt-4o-mini'], - 'claude-3-5-haiku-latest': LLM_MODELS['claude-3-5-haiku-latest'], - 'gemini-1.5-flash': LLM_MODELS['gemini-1.5-flash'], -}; - -setupSRE({ - ModelsProvider: { - Connector: 'JSONModelsProvider', - }, - - Log: { - Connector: 'ConsoleLog', - }, -}); - -const TIMEOUT = 120_000; -const LLM_OUTPUT_VALIDATOR = 'Yohohohooooo!'; -const WORD_INCLUSION_PROMPT = `\nThe response must includes "${LLM_OUTPUT_VALIDATOR}".`; - -describe.each(Object.keys(models))('Conversation Tests: %s', (id: string) => { - it( - 'runs a conversation with tool use', - async () => { - const specUrl = 'https://clzddo5xy19zg3mjrmr3urtfd.agent.stage.smyth.ai/api-docs/openapi-llm.json'; - const system = - `You are a helpful assistant that can answer questions about SmythOS. - If the user asks any question, use /ask endpoint to get information and be able to answer it.` + WORD_INCLUSION_PROMPT; - - const conv = new Conversation(id, specUrl, { systemPrompt: system }); - - const prompt = 'What can you help me with?'; - - const result = await conv.prompt(prompt); - - expect(result).toBeTruthy(); - expect(result).toContain(LLM_OUTPUT_VALIDATOR); - }, - TIMEOUT - ); - - it( - 'runs a conversation with tool use in stream mode', - async () => { - const specUrl = 'https://clzddo5xy19zg3mjrmr3urtfd.agent.stage.smyth.ai/api-docs/openapi-llm.json'; - const system = - `You are a helpful assistant that can answer questions about SmythOS. - If the user asks any question, use /ask endpoint to get information and be able to answer it.` + WORD_INCLUSION_PROMPT; - const conv = new Conversation(id, specUrl, { systemPrompt: system }); - - let streamResult = ''; - - const streamComplete = new Promise((resolve) => { - conv.on('content', (content) => { - streamResult += content; - }); - conv.on('end', resolve); - }); - - const prompt = 'What can you help me with?'; - - const result = await conv.streamPrompt(prompt); - - await streamComplete; - - expect(result).toBeTruthy(); - expect(streamResult).toBeTruthy(); - expect(streamResult).toContain(LLM_OUTPUT_VALIDATOR); - }, - TIMEOUT * 10 - ); - - it( - 'handles multiple tool calls in a single conversation', - async () => { - const specUrl = 'https://clzddo5xy19zg3mjrmr3urtfd.agent.stage.smyth.ai/api-docs/openapi-llm.json'; - const system = - `You are a helpful assistant that can answer questions about SmythOS. - If the user asks any question, use /ask endpoint to get information and be able to answer it.` + WORD_INCLUSION_PROMPT; - const conv = new Conversation(id, specUrl, { systemPrompt: system }); - - let streamResult = ''; - - const streamComplete = new Promise((resolve) => { - conv.on('content', (content) => { - streamResult += content; - }); - conv.on('end', resolve); - }); - - const prompt = 'First, tell me about SmythOS. Then, explain how it handles data storage.'; - - const result = await conv.streamPrompt(prompt); - - await streamComplete; - - expect(result).toBeTruthy(); - expect(streamResult).toBeTruthy(); - expect(streamResult).toContain(LLM_OUTPUT_VALIDATOR); - }, - TIMEOUT * 2 - ); - - it( - 'handles follow-up questions correctly', - async () => { - const specUrl = 'https://clzddo5xy19zg3mjrmr3urtfd.agent.stage.smyth.ai/api-docs/openapi-llm.json'; - const conv = new Conversation(id, specUrl); - - const prompt = 'What is SmythOS?' + WORD_INCLUSION_PROMPT; - - await conv.prompt(prompt); - - const followUpPrompt = 'Can you provide more details about its features?' + WORD_INCLUSION_PROMPT; - const followUpResult = await conv.prompt(followUpPrompt); - - expect(followUpResult).toBeTruthy(); - expect(followUpResult).toContain(LLM_OUTPUT_VALIDATOR); - }, - TIMEOUT * 2 - ); - - describe('Passthrough Mode', () => { - it( - 'should passthrough responses immediately midst agent execution', - async () => { - const conv = new Conversation(id, 'passthrough-llm-test'); - - const prompt = 'Generate story with first name as John. call the endpoint /passthrough_story'; - - const WAIT_THRESHOLD_SEC = 20; - const TOTAL_AGENT_EXECUTION_TIME = 500; - let executionTimeSec = process.hrtime(); - conv.streamPrompt(prompt); - - let error = null; - const isResponseEmitted = await new Promise((resolve) => { - let timeout = setTimeout(() => { - resolve(false); - }, WAIT_THRESHOLD_SEC * 1000); - - conv.on('content', (content) => { - console.log('content', content); - clearTimeout(timeout); - resolve(true); - executionTimeSec = process.hrtime(executionTimeSec); - }); - - conv.on('error', (err) => { - error = err; - clearTimeout(timeout); - resolve(false); - }); - }); - - if (isResponseEmitted) { - console.log(`Benchmark took ${executionTimeSec[0] + executionTimeSec[1] / 1e9} seconds`); - } - expect(error, error).toBeFalsy(); - expect(isResponseEmitted).toBe(true); - }, - TIMEOUT * 2 - ); - }); -}); diff --git a/packages/core/tests/unit/core/AgentProcess.test.ts b/packages/core/tests/unit/001-Core/AgentProcess.test.ts similarity index 96% rename from packages/core/tests/unit/core/AgentProcess.test.ts rename to packages/core/tests/unit/001-Core/AgentProcess.test.ts index 28fd23503..690d6201f 100644 --- a/packages/core/tests/unit/core/AgentProcess.test.ts +++ b/packages/core/tests/unit/001-Core/AgentProcess.test.ts @@ -13,7 +13,7 @@ describe('AgentProcess Tests', () => { let output; try { // Use the test data manager to load agent data - const data = loadAgentData('sre-echo-LLMPrompt.smyth'); + const data = loadAgentData('AgentData/sre-echo-LLMPrompt.smyth'); output = await AgentProcess.load(data).run({ method: 'POST', @@ -68,7 +68,7 @@ describe('AgentProcess Tests', () => { try { const sre = SmythRuntime.Instance; // Use the test data manager to load agent data - const data = loadAgentData('sre-echo-LLMPrompt.smyth'); + const data = loadAgentData('AgentData/sre-echo-LLMPrompt.smyth'); const output = await AgentProcess.load(data).post('/api/say', { message: 'Write a poem about flowers, the word "Rose" should be mentioned at least once', diff --git a/packages/core/tests/unit/core/BinaryInput.test.ts b/packages/core/tests/unit/001-Core/BinaryInput.test.ts similarity index 100% rename from packages/core/tests/unit/core/BinaryInput.test.ts rename to packages/core/tests/unit/001-Core/BinaryInput.test.ts diff --git a/packages/core/tests/unit/core/Connector.test.ts b/packages/core/tests/unit/001-Core/Connector.test.ts similarity index 100% rename from packages/core/tests/unit/core/Connector.test.ts rename to packages/core/tests/unit/001-Core/Connector.test.ts diff --git a/packages/core/tests/unit/001-Core/JSONFileVaultConnector.test.ts b/packages/core/tests/unit/001-Core/JSONFileVaultConnector.test.ts new file mode 100644 index 000000000..aa72b5a82 --- /dev/null +++ b/packages/core/tests/unit/001-Core/JSONFileVaultConnector.test.ts @@ -0,0 +1,41 @@ +import { describe, expect, it } from 'vitest'; +import { setupSRE } from '../../utils/sre'; +import { ConnectorService } from '@sre/Core/ConnectorsService'; +import { IAccessCandidate, TAccessRole } from 'index'; +import { testData } from '../../utils/test-data-manager'; + +setupSRE({ + Vault: { + Connector: 'JSONFileVault', + Settings: { + file: testData.getDataPath('vault.fake.json'), + }, + }, + Log: { + Connector: 'ConsoleLog', + }, +}); + +describe('JSONFileVault Tests', () => { + it('List all keys in the vault', async () => { + const mockCandidate: IAccessCandidate = { + id: 'default', + role: TAccessRole.Team, + }; + + const vaultConnector = ConnectorService.getVaultConnector('JSONFileVault'); + const result = await vaultConnector.team(mockCandidate.id).listKeys(); + expect(result).toBeDefined(); + }); + + it('Get a key from the vault', async () => { + const mockCandidate: IAccessCandidate = { + id: 'default', + role: TAccessRole.Team, + }; + + const vaultConnector = ConnectorService.getVaultConnector('JSONFileVault'); + const result = await vaultConnector.team(mockCandidate.id).get('my-key'); + expect(result).toBe('Hello world'); + }); +}); diff --git a/packages/core/tests/unit/core/ecma-sandbox.test.ts b/packages/core/tests/unit/001-Core/ecma-sandbox.test.ts similarity index 100% rename from packages/core/tests/unit/core/ecma-sandbox.test.ts rename to packages/core/tests/unit/001-Core/ecma-sandbox.test.ts diff --git a/packages/core/tests/unit/core/sre.test.ts b/packages/core/tests/unit/001-Core/sre.test.ts similarity index 100% rename from packages/core/tests/unit/core/sre.test.ts rename to packages/core/tests/unit/001-Core/sre.test.ts diff --git a/packages/core/tests/unit/core/strong-data-typing.test.ts b/packages/core/tests/unit/001-Core/strong-data-typing.test.ts similarity index 93% rename from packages/core/tests/unit/core/strong-data-typing.test.ts rename to packages/core/tests/unit/001-Core/strong-data-typing.test.ts index e4aca7995..b4f8e049a 100644 --- a/packages/core/tests/unit/core/strong-data-typing.test.ts +++ b/packages/core/tests/unit/001-Core/strong-data-typing.test.ts @@ -16,7 +16,7 @@ describe('Strong Data typing Features', () => { let error; try { // const agentData = fs.readFileSync('./tests/data/general-unit-tests.smyth', 'utf-8'); - const data = loadAgentData('general-unit-tests.smyth'); + const data = loadAgentData('AgentData/general-unit-tests.smyth'); const date = new Date(); const agentProcess = AgentProcess.load(data); @@ -61,7 +61,7 @@ describe('Strong Data typing Features', () => { it('APIEndpoint with Array variation', async () => { let error; try { - const data = loadAgentData('general-unit-tests.smyth'); + const data = loadAgentData('AgentData/general-unit-tests.smyth'); const date = new Date(); const agentProcess = AgentProcess.load(data); @@ -90,7 +90,7 @@ describe('Strong Data typing Features', () => { it('APIEndpoint : should detect invalid formats', async () => { let error; try { - const data = loadAgentData('general-unit-tests.smyth'); + const data = loadAgentData('AgentData/general-unit-tests.smyth'); const agentProcess = AgentProcess.load(data); @@ -142,7 +142,7 @@ describe('Strong Data typing Features', () => { }); it('APIEndpoint : default values', async () => { - const data = loadAgentData('general-unit-tests.smyth'); + const data = loadAgentData('AgentData/general-unit-tests.smyth'); const agentProcess = AgentProcess.load(data); @@ -162,6 +162,6 @@ describe('Strong Data typing Features', () => { expect(outputBody.array).toEqual(['a', 'b', 'c', 'd']); expect(outputBody.object).toEqual({ message: 'hello world' }); expect(outputBody.binary.size).toEqual(9); - expect(outputBody.date).toEqual('2024-01-19T22:00:00.000Z'); + expect(outputBody.date).toEqual('2024-01-19T23:00:00.000Z'); }); }); diff --git a/packages/core/tests/unit/002-Security/acl.test.ts b/packages/core/tests/unit/002-Security/acl.test.ts new file mode 100644 index 000000000..45bd614ac --- /dev/null +++ b/packages/core/tests/unit/002-Security/acl.test.ts @@ -0,0 +1,75 @@ +import { describe, expect, it } from 'vitest'; +import { ACL } from '@sre/Security/AccessControl/ACL.class'; +import { IAccessRequest, TAccessLevel, TAccessRole } from '@sre/types/ACL.types'; +//import SRE, { AgentRequest } from '../../dist'; +describe('SRE ACL Tests', () => { + it('Create an ACL', async () => { + const acl: ACL = new ACL(); + expect(acl).toBeInstanceOf(ACL); + }); + it('Create an ACL with access rights and get the ACL object', async () => { + // prettier-ignore + const acl = new ACL() + .addAccess(TAccessRole.Agent, 'agent1', TAccessLevel.Read) + .addAccess(TAccessRole.Agent, 'agent1', TAccessLevel.Write) + .addAccess(TAccessRole.Team, 'team1', TAccessLevel.Read) + .ACL; + + expect(acl).toBeTypeOf('object'); + }); + it('Creates Compressed ACL', async () => { + const aclObj = new ACL() + .addAccess(TAccessRole.User, 'user1', TAccessLevel.Owner) + .addAccess(TAccessRole.Agent, 'agent1', TAccessLevel.Read) + .addAccess(TAccessRole.Agent, 'agent1', TAccessLevel.Write) + .addAccess(TAccessRole.Agent, 'agent2', TAccessLevel.Read) + .addAccess(TAccessRole.Team, 'team1', TAccessLevel.Read); + + const sACL = aclObj.serializedACL; + + expect(sACL).toBeTypeOf('string'); + + const acl2 = new ACL(sACL).ACL; + + expect(acl2).toEqual(aclObj.ACL); + }); + + it('Check Access Rights Granted', async () => { + // prettier-ignore + const acl = new ACL() + .addAccess(TAccessRole.Agent, 'agent1', TAccessLevel.Read) + .addAccess(TAccessRole.Agent, 'agent1', TAccessLevel.Write) + .addAccess(TAccessRole.Team, 'team1', TAccessLevel.Read) + .ACL + + const hasAccess = new ACL(acl).checkExactAccess({ + resourceId: 'resource1', + candidate: { + role: TAccessRole.Agent, + id: 'agent1', + }, + level: TAccessLevel.Write, + } as IAccessRequest); + + expect(hasAccess).toBeTruthy(); + }); + it('Check Access Rights Refused', async () => { + // prettier-ignore + const acl = new ACL() + .addAccess(TAccessRole.Agent, 'agent1', TAccessLevel.Read) + .addAccess(TAccessRole.Agent, 'agent1', TAccessLevel.Write) + .addAccess(TAccessRole.Team, 'team1', TAccessLevel.Read) + .ACL + + const hasAccess = new ACL(acl).checkExactAccess({ + resourceId: 'resource1', + candidate: { + role: TAccessRole.Agent, + id: 'agent2', + }, + level: TAccessLevel.Write, + } as IAccessRequest); + + expect(hasAccess).toBeFalsy(); + }); +}); diff --git a/packages/core/tests/unit/002-Security/jsonAccount.test.ts b/packages/core/tests/unit/002-Security/jsonAccount.test.ts new file mode 100644 index 000000000..26667a117 --- /dev/null +++ b/packages/core/tests/unit/002-Security/jsonAccount.test.ts @@ -0,0 +1,257 @@ +import { describe, expect, it } from 'vitest'; +import { AccessCandidate } from '@sre/Security/AccessControl/AccessCandidate.class'; +import { ConnectorService, JSONFileAccount, SmythRuntime } from 'index'; + +import { AccountConnector } from '@sre/Security/Account.service/AccountConnector'; +import { testData } from '../../utils/test-data-manager'; + +const SREInstance = SmythRuntime.Instance.init({ + Account: { + Connector: 'JSONFileAccount', + Settings: { + file: testData.getDataPath('account.json'), + }, + }, +}); + +describe('JSON Account Tests', () => { + it('Smyth Account loaded', async () => { + const jsonAccount: AccountConnector = ConnectorService.getAccountConnector(); + expect(jsonAccount).toBeInstanceOf(JSONFileAccount); + }); + + it('Verify user to be team member', async () => { + const jsonAccount: AccountConnector = ConnectorService.getAccountConnector(); + const value = await jsonAccount.user('user1').isTeamMember('default'); + expect(value).toEqual(true); + }); + + it('Verify user not to be team member', async () => { + const jsonAccount: AccountConnector = ConnectorService.getAccountConnector(); + const value = await jsonAccount.user('nonexistent-user').isTeamMember('default'); + expect(value).toEqual(false); + }); + + it('Verify correct team is returning for user', async () => { + const jsonAccount: AccountConnector = ConnectorService.getAccountConnector(); + const team = await jsonAccount.user('user1').getCandidateTeam(); + expect(team).toEqual('default'); + }); + + it('Verify correct team is returning for team', async () => { + const jsonAccount: AccountConnector = ConnectorService.getAccountConnector(); + const team = await jsonAccount.team('default').getCandidateTeam(); + expect(team).toEqual('default'); + }); + + it('Verify correct team is returning for agent', async () => { + const jsonAccount: AccountConnector = ConnectorService.getAccountConnector(); + const team = await jsonAccount.agent('agent1').getCandidateTeam(); + expect(team).toEqual('default'); + }); + + it('Verify all account settings are returning', async () => { + const jsonAccount: AccountConnector = ConnectorService.getAccountConnector(); + const userSettings = await jsonAccount.user('user1').getAllUserSettings(); + const setting1 = userSettings.find((setting) => setting.key === 'setting1'); + expect(setting1).toEqual({ key: 'setting1', value: 'value1' }); + }); + + it('Verify all team settings are returning', async () => { + const jsonAccount: AccountConnector = ConnectorService.getAccountConnector(); + const teamSettings = await jsonAccount.team('default').getAllTeamSettings(); + const customLlmSettings = teamSettings.find((setting) => setting.key === 'custom-llm'); + expect(customLlmSettings).toBeDefined(); + }); + + it('Verify specific account setting is returning', async () => { + const jsonAccount: AccountConnector = ConnectorService.getAccountConnector(); + const userSettings = await jsonAccount.user('user1').getUserSetting('setting1'); + expect(userSettings).toEqual('value1'); + }); + + it('Verify specific team setting is returning', async () => { + const jsonAccount: AccountConnector = ConnectorService.getAccountConnector(); + const teamSettings = await jsonAccount.team('default').getTeamSetting('custom-llm'); + expect(teamSettings).toBeTypeOf('string'); + }); + + it('Verify agent can access account setting', async () => { + const jsonAccount: AccountConnector = ConnectorService.getAccountConnector(); + const team = await jsonAccount.agent('agent1').getCandidateTeam(); + const teamSettings = await jsonAccount.team(team).getTeamSetting('custom-llm'); + expect(teamSettings).toBeTypeOf('string'); + }); + + it('Invalid setting key to be returned as null', async () => { + const jsonAccount: AccountConnector = ConnectorService.getAccountConnector(); + const teamSettings = await jsonAccount.team('default').getTeamSetting('nonexistent-setting'); + expect(teamSettings).toBe(''); + }); + + // Multi-team tests + describe('Development Team Tests', () => { + it('Verify development team user membership', async () => { + const jsonAccount: AccountConnector = ConnectorService.getAccountConnector(); + const value = await jsonAccount.user('dev-user1').isTeamMember('development'); + expect(value).toEqual(true); + }); + + it('Verify development team settings', async () => { + const jsonAccount: AccountConnector = ConnectorService.getAccountConnector(); + const teamSettings = await jsonAccount.team('development').getTeamSetting('api-endpoint'); + expect(teamSettings).toEqual('https://dev-api.smythos.com'); + }); + + it('Verify development user settings', async () => { + const jsonAccount: AccountConnector = ConnectorService.getAccountConnector(); + const userSettings = await jsonAccount.user('dev-user1').getUserSetting('environment'); + expect(userSettings).toEqual('development'); + }); + + it('Verify development agent settings', async () => { + const jsonAccount: AccountConnector = ConnectorService.getAccountConnector(); + const team = await jsonAccount.agent('dev-agent1').getCandidateTeam(); + expect(team).toEqual('development'); + }); + }); + + describe('Production Team Tests', () => { + it('Verify production team user membership', async () => { + const jsonAccount: AccountConnector = ConnectorService.getAccountConnector(); + const value = await jsonAccount.user('prod-user1').isTeamMember('production'); + expect(value).toEqual(true); + }); + + it('Verify production team settings', async () => { + const jsonAccount: AccountConnector = ConnectorService.getAccountConnector(); + const teamSettings = await jsonAccount.team('production').getTeamSetting('security-level'); + expect(teamSettings).toEqual('maximum'); + }); + + it('Verify admin user access level', async () => { + const jsonAccount: AccountConnector = ConnectorService.getAccountConnector(); + const userSettings = await jsonAccount.user('admin-user').getUserSetting('access-level'); + expect(userSettings).toEqual('full'); + }); + + it('Verify critical agent settings', async () => { + const jsonAccount: AccountConnector = ConnectorService.getAccountConnector(); + const agentSettings = await jsonAccount.agent('critical-agent').getAgentSetting('priority'); + expect(agentSettings).toEqual('high'); + }); + }); + + describe('Cross-Team Access Tests', () => { + it('Verify user cannot access wrong team', async () => { + const jsonAccount: AccountConnector = ConnectorService.getAccountConnector(); + const value = await jsonAccount.user('dev-user1').isTeamMember('production'); + expect(value).toEqual(false); + }); + + it('Verify default team user cannot access development team', async () => { + const jsonAccount: AccountConnector = ConnectorService.getAccountConnector(); + const value = await jsonAccount.user('user1').isTeamMember('development'); + expect(value).toEqual(false); + }); + + it('Verify production user cannot access development team', async () => { + const jsonAccount: AccountConnector = ConnectorService.getAccountConnector(); + const value = await jsonAccount.user('prod-user1').isTeamMember('development'); + expect(value).toEqual(false); + }); + }); + + describe('Team Isolation Tests', () => { + it('Verify each team has different vault configurations', async () => { + const jsonAccount: AccountConnector = ConnectorService.getAccountConnector(); + const defaultVault = await jsonAccount.team('default').getTeamSetting('vault'); + const devVault = await jsonAccount.team('development').getTeamSetting('vault'); + const prodVault = await jsonAccount.team('production').getTeamSetting('vault'); + + expect(defaultVault).toEqual('default-vault-config'); + expect(devVault).toEqual('dev-vault-config'); + expect(prodVault).toEqual('prod-vault-config'); + + // Ensure they are all different + expect(defaultVault).not.toEqual(devVault); + expect(devVault).not.toEqual(prodVault); + expect(defaultVault).not.toEqual(prodVault); + }); + + it('Verify teams have different API endpoints', async () => { + const jsonAccount: AccountConnector = ConnectorService.getAccountConnector(); + const devEndpoint = await jsonAccount.team('development').getTeamSetting('api-endpoint'); + const prodEndpoint = await jsonAccount.team('production').getTeamSetting('api-endpoint'); + + expect(devEndpoint).toEqual('https://dev-api.smythos.com'); + expect(prodEndpoint).toEqual('https://api.smythos.com'); + expect(devEndpoint).not.toEqual(prodEndpoint); + }); + + it('Verify agents belong to correct teams', async () => { + const jsonAccount: AccountConnector = ConnectorService.getAccountConnector(); + const defaultAgentTeam = await jsonAccount.agent('agent1').getCandidateTeam(); + const devAgentTeam = await jsonAccount.agent('dev-agent1').getCandidateTeam(); + const prodAgentTeam = await jsonAccount.agent('prod-agent1').getCandidateTeam(); + + expect(defaultAgentTeam).toEqual('default'); + expect(devAgentTeam).toEqual('development'); + expect(prodAgentTeam).toEqual('production'); + }); + }); + + describe('Environment-Specific Settings Tests', () => { + it('Verify environment-specific user settings', async () => { + const jsonAccount: AccountConnector = ConnectorService.getAccountConnector(); + const devUserEnv = await jsonAccount.user('dev-user1').getUserSetting('environment'); + const prodUserEnv = await jsonAccount.user('prod-user1').getUserSetting('environment'); + + expect(devUserEnv).toEqual('development'); + expect(prodUserEnv).toEqual('production'); + }); + + it('Verify role-based access settings', async () => { + const jsonAccount: AccountConnector = ConnectorService.getAccountConnector(); + const devUserRole = await jsonAccount.user('dev-user2').getUserSetting('role'); + const adminUserRole = await jsonAccount.user('admin-user').getUserSetting('role'); + + expect(devUserRole).toEqual('developer'); + expect(adminUserRole).toEqual('administrator'); + }); + + it('Verify agent mode settings across teams', async () => { + const jsonAccount: AccountConnector = ConnectorService.getAccountConnector(); + const devAgentMode = await jsonAccount.agent('dev-agent1').getAgentSetting('mode'); + const prodAgentMode = await jsonAccount.agent('prod-agent1').getAgentSetting('mode'); + + expect(devAgentMode).toEqual('development'); + expect(prodAgentMode).toEqual('production'); + }); + }); + + describe('Nonexistent Entity Tests', () => { + it('Verify nonexistent user in any team returns false', async () => { + const jsonAccount: AccountConnector = ConnectorService.getAccountConnector(); + const defaultResult = await jsonAccount.user('nonexistent-user').isTeamMember('default'); + const devResult = await jsonAccount.user('nonexistent-user').isTeamMember('development'); + const prodResult = await jsonAccount.user('nonexistent-user').isTeamMember('production'); + + expect(defaultResult).toEqual(false); + expect(devResult).toEqual(false); + expect(prodResult).toEqual(false); + }); + + it('Verify nonexistent team returns empty string for settings', async () => { + const jsonAccount: AccountConnector = ConnectorService.getAccountConnector(); + const teamSettings = await jsonAccount.team('nonexistent-team').getTeamSetting('any-setting'); + expect(teamSettings).toBe(''); + }); + + it('Verify nonexistent agent returns "default" string for team result', async () => { + const jsonAccount: AccountConnector = ConnectorService.getAccountConnector(); + const team = await jsonAccount.agent('nonexistent-agent').getCandidateTeam(); + expect(team).toBe('default'); + }); + }); +}); diff --git a/packages/core/tests/unit/002-Security/jsonVault.test.ts b/packages/core/tests/unit/002-Security/jsonVault.test.ts new file mode 100644 index 000000000..6af6a66a0 --- /dev/null +++ b/packages/core/tests/unit/002-Security/jsonVault.test.ts @@ -0,0 +1,65 @@ +import { describe, expect, it } from 'vitest'; + +//import SRE, { AgentRequest } from '../../dist'; + +import { AccessCandidate } from '@sre/Security/AccessControl/AccessCandidate.class'; + +import { VaultConnector } from '@sre/Security/Vault.service/VaultConnector'; +import { JSONFileVault } from '@sre/Security/Vault.service/connectors/JSONFileVault.class'; +import { ConnectorService, SmythRuntime } from 'index'; +import { Match, TemplateString, TPLProcessor } from '@sre/helpers/TemplateString.helper'; +import { VaultHelper } from '@sre/Security/Vault.service/Vault.helper'; +import { testData } from '../../utils/test-data-manager'; + +const SREInstance = SmythRuntime.Instance.init({ + Vault: { + Connector: 'JSONFileVault', + Settings: { + file: testData.getDataPath('vault.fake.json'), + }, + }, + Account: { + Connector: 'DummyAccount', + }, +}); + +describe('Vault Tests', () => { + it('Vault loaded', async () => { + const vault: VaultConnector = ConnectorService.getVaultConnector(); + expect(vault).toBeInstanceOf(JSONFileVault); + }); + + it('Read vault key', async () => { + const vault: VaultConnector = ConnectorService.getVaultConnector(); + + //by default current team resolver resolves every user team to "default" + const value = await vault.user(AccessCandidate.user('test')).get('DIFFBOT_API'); + + expect(value).toEqual('THIS_IS_A_FAKE_DIFFBOT_API_KEY'); + }); + + it('Do not allow reading key from different team', async () => { + const vault: VaultConnector = ConnectorService.getVaultConnector(); + + //we use a team candidate here in order to test another team access + const value = await vault + .user(AccessCandidate.team('Team2')) + .get('DIFFBOT_API') + .catch((e) => undefined); + + expect(value).toBeUndefined(); + }); + + it('Parse a template string containing vault keys', async () => { + const tpl = `using a vault key : {{KEY(DIFFBOT_API)}} and a simple template variable : {{MyVAR}}`; + const teamId = 'default'; + + //prettier-ignore + const result = await TemplateString(tpl) + .parse({ MyVAR: 'Hello' }) + .parseTeamKeysAsync(teamId) + .asyncResult; //since parseTeamKeys is async, we use asyncResult with await to wait for the result + + expect(result).toEqual('using a vault key : THIS_IS_A_FAKE_DIFFBOT_API_KEY and a simple template variable : Hello'); + }); +}); diff --git a/packages/core/tests/unit/002-Security/secretsManager.test.ts b/packages/core/tests/unit/002-Security/secretsManager.test.ts new file mode 100644 index 000000000..961043ec8 --- /dev/null +++ b/packages/core/tests/unit/002-Security/secretsManager.test.ts @@ -0,0 +1,53 @@ +import { describe, expect, it } from 'vitest'; +import { AccessCandidate } from '@sre/Security/AccessControl/AccessCandidate.class'; +import { VaultConnector } from '@sre/Security/Vault.service/VaultConnector'; +import { SecretsManager } from '@sre/Security/Vault.service/connectors/SecretsManager.class'; + +import { TemplateString } from '@sre/helpers/TemplateString.helper'; +import { ConnectorService, SmythRuntime } from 'index'; + +const SREInstance = SmythRuntime.Instance.init({ + Vault: { + Connector: 'SecretsManager', + Settings: { + accessKeyId: process.env.AWS_ACCESS_KEY_ID, + secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY, + region: process.env.AWS_REGION, + }, + }, + Account: { + Connector: 'SmythAccount', + }, +}); + +describe('Secret Manager Tests', () => { + it('Secret Manager loaded', async () => { + const vault: VaultConnector = ConnectorService.getVaultConnector('SecretsManager'); + expect(vault).toBeInstanceOf(SecretsManager); + }); + + it('Read secret', async () => { + const vault: VaultConnector = ConnectorService.getVaultConnector(); + const value = await vault.user(AccessCandidate.team('test')).get('secret'); + expect(value).toEqual('test_value'); + }); + + it('Do not allow random secrets to be read', async () => { + const vault: VaultConnector = ConnectorService.getVaultConnector(); + + const value = await vault + .user(AccessCandidate.team('randomSecretId')) + .get('test') + .catch((e) => undefined); + expect(value).toBeUndefined(); + }); + + it('Parse a template string containing secret', async () => { + const tpl = `using a vault key : {{secret}} and a simple template variable : {{MyVAR}}`; + const teamId = 'test'; + const vault: VaultConnector = ConnectorService.getVaultConnector(); + const value = await vault.user(AccessCandidate.team(teamId)).get('secret'); + const result = await TemplateString(tpl).parse({ MyVAR: 'Hello', secret: value }).result; + expect(result).toEqual('using a vault key : test_value and a simple template variable : Hello'); + }); +}); diff --git a/packages/core/tests/unit/002-Security/secretsManagerManagedVault.test.ts b/packages/core/tests/unit/002-Security/secretsManagerManagedVault.test.ts new file mode 100644 index 000000000..0f03e44ae --- /dev/null +++ b/packages/core/tests/unit/002-Security/secretsManagerManagedVault.test.ts @@ -0,0 +1,58 @@ +import { describe, expect, it } from 'vitest'; +import { AccessCandidate } from '@sre/Security/AccessControl/AccessCandidate.class'; +import { SecretManagerManagedVault } from '@sre/Security/ManagedVault.service/connectors/SecretManagerManagedVault'; +import { ConnectorService, SmythRuntime } from 'index'; +import { TemplateString } from '@sre/helpers/TemplateString.helper'; +import { ManagedVaultConnector } from '@sre/Security/ManagedVault.service/ManagedVaultConnector'; + +const SREInstance = SmythRuntime.Instance.init({ + ManagedVault: { + Connector: 'SecretManagerManagedVault', + Settings: { + accessKeyId: process.env.AWS_ACCESS_KEY_ID, + secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY, + region: process.env.AWS_REGION, + }, + }, + Account: { + Connector: 'SmythAccount', + }, +}); + +describe('Secret Manager Tests', () => { + it('Secret Manager loaded', async () => { + const vault: ManagedVaultConnector = ConnectorService.getManagedVaultConnector('SecretManagerManagedVault'); + expect(vault).toBeInstanceOf(SecretManagerManagedVault); + }); + + it('Read secret', async () => { + const vault: ManagedVaultConnector = ConnectorService.getManagedVaultConnector('SecretManagerManagedVault'); + const value = await vault.user(AccessCandidate.team('test')).get('secret_key'); + expect(value).toEqual('secret_value'); + }); + + // it('Create secret', async () => { + // const vault: ManagedVaultConnector = ConnectorService.getManagedVaultConnector('SecretManagerManagedVault'); + // await vault.user(AccessCandidate.team('test')).set('secret_key_1', 'secret_value_1'); + // expect(true).toBeTruthy(); + // }); + + it('Do not allow random secrets to be read', async () => { + const vault: ManagedVaultConnector = ConnectorService.getManagedVaultConnector('SecretManagerManagedVault'); + + const value = await vault + .user(AccessCandidate.team('randomSecretId')) + .get('test') + .catch((e) => undefined); + expect(value).toBeUndefined(); + }); + + it('Parse a template string containing secret', async () => { + const tpl = `using a vault key : {{secret}} and a simple template variable : {{MyVAR}}`; + const teamId = 'test'; + const vault: ManagedVaultConnector = ConnectorService.getManagedVaultConnector('SecretManagerManagedVault'); + const value = await vault.user(AccessCandidate.team(teamId)).get('secret_key'); + const result = await TemplateString(tpl).parse({ MyVAR: 'Hello', secret: value }).result; + expect(result).toEqual('using a vault key : secret_value and a simple template variable : Hello'); + }); +}); diff --git a/packages/core/tests/integration/connectors/vectordb/ramvec.test.ts b/packages/core/tests/unit/004-VectorDB/ramvec.test.ts similarity index 68% rename from packages/core/tests/integration/connectors/vectordb/ramvec.test.ts rename to packages/core/tests/unit/004-VectorDB/ramvec.test.ts index b70804b89..40afb68d1 100644 --- a/packages/core/tests/integration/connectors/vectordb/ramvec.test.ts +++ b/packages/core/tests/unit/004-VectorDB/ramvec.test.ts @@ -1,5 +1,5 @@ import { afterEach, beforeAll, describe, expect, it, vi } from 'vitest'; -import { setupSRE } from '../../../utils/sre'; +import { setupSRE } from '../../utils/sre'; import { ConnectorService } from '@sre/Core/ConnectorsService'; import { AccessCandidate } from '@sre/Security/AccessControl/AccessCandidate.class'; @@ -197,8 +197,9 @@ describe('RAMVec - VectorDB connector (in-memory)', () => { list = await client.listDatasources('workspace'); expect(list.map((d) => d.id)).not.toContain('dsX'); - // getDatasource for deleted should throw - await expect(client.getDatasource('workspace', 'dsX')).rejects.toThrow('Datasource dsX not found'); + // getDatasource for deleted should return undefined + const deletedDs = await client.getDatasource('workspace', 'dsX'); + expect(deletedDs).toBeUndefined(); }); it('should throw when searching non-existing namespace and after deleting namespace', async () => { @@ -219,4 +220,88 @@ describe('RAMVec - VectorDB connector (in-memory)', () => { const list = await client.listDatasources('temp'); expect(list).toEqual([]); }); + + it('should get namespace metadata and handle non-existing namespace', async () => { + const vdb = ConnectorService.getVectorDBConnector('RAMVec'); + const user = AccessCandidate.user('test-user'); + + // Test getNamespace - this is an internal method, so we access it directly + const ramVectorDB = vdb as any; + + // Should throw for non-existing namespace + await expect(ramVectorDB.getNamespace(user.readRequest, 'non-existing')).rejects.toThrow('Namespace non-existing not found'); + + // Create namespace with metadata + await vdb.requester(user).createNamespace('project-alpha', { + project: 'alpha', + description: 'Test project namespace', + version: '1.0.0', + }); + + // Should return namespace data + const nsData = await ramVectorDB.getNamespace(user.readRequest, 'project-alpha'); + expect(nsData).toBeDefined(); + expect(nsData.displayName).toBe('project-alpha'); + expect(nsData.candidateId).toBe('test-user'); + expect(nsData.candidateRole).toBe('user'); + expect(nsData.metadata).toBeDefined(); + expect(nsData.metadata.project).toBe('alpha'); + expect(nsData.metadata.description).toBe('Test project namespace'); + expect(nsData.metadata.version).toBe('1.0.0'); + expect(nsData.metadata.storageType).toBe('RAM'); + expect(nsData.namespace).toContain('test-user'); + expect(nsData.namespace).toContain('project-alpha'); + }); + + it('should handle deleteDatasource edge cases', async () => { + const vdb = ConnectorService.getVectorDBConnector('RAMVec'); + const user = AccessCandidate.user('test-user'); + const client = vdb.requester(user); + + await client.createNamespace('edge-cases'); + + // Should return undefined for non-existing datasource (graceful handling) + const nonExistingDs = await client.getDatasource('edge-cases', 'non-existing-ds'); + expect(nonExistingDs).toBeUndefined(); + + // Should throw when trying to delete non-existing datasource + await expect(client.deleteDatasource('edge-cases', 'non-existing-ds')).rejects.toThrow('Data source not found with id: non-existing-ds'); + + // Create a datasource and verify it exists + await client.createDatasource('edge-cases', { + id: 'test-ds', + label: 'Test Datasource', + text: 'This is a test datasource with some content for testing deletion', + chunkSize: 15, + chunkOverlap: 3, + metadata: { type: 'test', category: 'edge-case' }, + }); + + // Verify datasource exists + const ds = await client.getDatasource('edge-cases', 'test-ds'); + expect(ds.id).toBe('test-ds'); + expect(ds.vectorIds.length).toBeGreaterThan(0); + + // Verify vectors exist in search + const searchBefore = await client.search('edge-cases', 'test datasource', { topK: 5 }); + expect(searchBefore.length).toBeGreaterThan(0); + + // Delete the datasource + await client.deleteDatasource('edge-cases', 'test-ds'); + + // Verify datasource no longer exists + const deletedDs = await client.getDatasource('edge-cases', 'test-ds'); + expect(deletedDs).toBeUndefined(); + + // Verify datasource is not in list + const list = await client.listDatasources('edge-cases'); + expect(list.map((d) => d.id)).not.toContain('test-ds'); + + // Verify vectors are removed from search (should return fewer or no results) + const searchAfter = await client.search('edge-cases', 'test datasource', { topK: 5 }); + expect(searchAfter.length).toBeLessThan(searchBefore.length); + + // Should throw when trying to delete the same datasource again + await expect(client.deleteDatasource('edge-cases', 'test-ds')).rejects.toThrow('Data source not found with id: test-ds'); + }); }); diff --git a/packages/core/tests/unit/005-Storage/LocalStorage.test.ts b/packages/core/tests/unit/005-Storage/LocalStorage.test.ts new file mode 100644 index 000000000..02197e952 --- /dev/null +++ b/packages/core/tests/unit/005-Storage/LocalStorage.test.ts @@ -0,0 +1,171 @@ +import xxhash from 'xxhashjs'; +import { describe, expect, it } from 'vitest'; +import { TAccessLevel, TAccessRole } from '@sre/types/ACL.types'; +import { StorageConnector } from '@sre/IO/Storage.service/StorageConnector'; +import { AccessCandidate } from '@sre/Security/AccessControl/AccessCandidate.class'; +import { ConnectorService } from '@sre/Core/ConnectorsService'; +import { setupSRE } from '../../utils/sre'; +import { LocalStorage } from '@sre/IO/Storage.service/connectors/LocalStorage.class'; + +function xxh3(source) { + const h64 = xxhash.h64(); // Use xxhashjs's h64 function + return h64.update(source.toString()).digest().toString(16); +} + +setupSRE(); + +const testFile = 'test.txt'; +const testAdditionalACLMetadata = { + hashAlgorithm: 'xxh3', + entries: { + [TAccessRole.Team]: { + //hash 'team1' + [xxh3('team1')]: [TAccessLevel.Read, TAccessLevel.Write], + }, + [TAccessRole.Agent]: { + //hash 'team1' + [xxh3('agent1')]: [TAccessLevel.Owner], + }, + }, +}; + +const testFileWithMeta = 'test-meta.txt'; +const testOriginalACLMetadata = { + hashAlgorithm: 'xxh3', + entries: { + [TAccessRole.Team]: { + teamMeta: [TAccessLevel.Read, TAccessLevel.Write], + }, + }, +}; + +const agentCandidate = AccessCandidate.agent('agent1'); + +const testOriginalMetadata = { + 'Content-Type': 'text/plain', + 'x-amz-meta-test': 'test', +}; + +describe('Local Storage Tests', () => { + it('Get Default Storage instance', async () => { + const localStorage: StorageConnector = ConnectorService.getStorageConnector(); + + expect(localStorage).toBeInstanceOf(LocalStorage); + }); + + it('Write a file in LocalStorage', async () => { + let error; + + try { + const localStorage: StorageConnector = ConnectorService.getStorageConnector(); + await localStorage.requester(agentCandidate).write(testFile, 'Hello World!'); + } catch (e) { + console.error(e); + error = e; + } + + expect(error).toBeUndefined(); + }); + + it('Write a file with metadata in LocalStorage', async () => { + let error; + + try { + const localStorage: StorageConnector = ConnectorService.getStorageConnector(); + await localStorage.requester(agentCandidate).write(testFileWithMeta, 'I have metadata', testOriginalACLMetadata, testOriginalMetadata); + + const storageReq = await localStorage.requester(agentCandidate); + storageReq.write(testFileWithMeta, 'I have metadata'); + } catch (e) { + console.error(e); + error = e; + } + + expect(error).toBeUndefined(); + }); + + it('Does the files exist?', async () => { + const localStorage: StorageConnector = ConnectorService.getStorageConnector(); + const files = [testFile, testFileWithMeta]; + + const promises = files.map((file) => { + return localStorage.requester(agentCandidate).exists(file); + }); + + const results = await Promise.all(promises); + + expect(results).toEqual([true, true]); + }); + + it('Is Metadata present', async () => { + const localStorage: StorageConnector = ConnectorService.getStorageConnector(); + const metadata = await localStorage.requester(agentCandidate).getMetadata(testFileWithMeta); + + expect(metadata).toBeDefined(); + }); + + it('Set ACL Metadata', async () => { + let error; + try { + const localStorage: StorageConnector = ConnectorService.getStorageConnector(); + //we set the metadata for the file created in the previous test + + await localStorage.requester(agentCandidate).setACL(testFile, testAdditionalACLMetadata); + } catch (e) { + console.error(e); + error = e; + } + + expect(error).toBeUndefined(); + }); + + it('Does ACL metadata exist?', async () => { + const localStorage: StorageConnector = ConnectorService.getStorageConnector(); + //here we need to build an access request for the agent1 because we changed the ACL metadata to have agent1 as owner + let metadata: any = await localStorage.requester(agentCandidate).getACL(testFile); + + expect(metadata.entries?.team).toEqual(testAdditionalACLMetadata.entries.team); + }); + + it('Read files from LocalStorage', async () => { + const localStorage: StorageConnector = ConnectorService.getStorageConnector(); + + const data = await localStorage.requester(agentCandidate).read(testFile); + const strData = data.toString(); + expect(strData).toEqual('Hello World!'); + + const dataWithMeta = await localStorage.requester(agentCandidate).read(testFileWithMeta); + const strDataWithMeta = dataWithMeta.toString(); + expect(strDataWithMeta).toEqual('I have metadata'); + }); + + it('Delete files from LocalStorage', async () => { + let error; + + try { + const localStorage: StorageConnector = ConnectorService.getStorageConnector(); + await Promise.all([ + localStorage.requester(agentCandidate).delete(testFile), + localStorage.requester(agentCandidate).delete(testFileWithMeta), + ]); + } catch (e) { + console.error(e); + error = e; + } + + expect(error).toBeUndefined(); + }); + + it('The file should be deleted', async () => { + const localStorage: StorageConnector = ConnectorService.getStorageConnector(); + const files = [testFile, testFileWithMeta]; + + const promises = files.map((file) => { + return localStorage.requester(agentCandidate).exists(file); + }); + + const results = await Promise.all(promises); + + expect(results).toEqual([false, false]); + }); +}); diff --git a/packages/core/tests/unit/005-Storage/SmythFS.test.ts b/packages/core/tests/unit/005-Storage/SmythFS.test.ts new file mode 100644 index 000000000..080e41d74 --- /dev/null +++ b/packages/core/tests/unit/005-Storage/SmythFS.test.ts @@ -0,0 +1,324 @@ +import { afterAll, describe, expect, it, beforeAll } from 'vitest'; +import { SmythFS } from '@sre/IO/Storage.service/SmythFS.class'; +import express from 'express'; +import { IAccessCandidate, TAccessRole } from '@sre/types/ACL.types'; + +import { ConnectorService } from '@sre/Core/ConnectorsService'; +import { setupSRE } from '../../utils/sre'; +import http, { Server } from 'http'; +import { AccessCandidate } from '@sre/Security/AccessControl/AccessCandidate.class'; +import axios from 'axios'; +import { promisify } from 'util'; +import fs from 'fs'; +import path from 'path'; +import mime from 'mime'; +import { testData } from '../../utils/test-data-manager'; +const PORT = 8083; +const BASE_URL = `http://localhost:${PORT}`; + +const app = express(); + +const { SREInstance } = setupSRE({ + Router: { + Connector: 'ExpressRouter', + Settings: { + router: app, + baseUrl: BASE_URL, + }, + }, +}); + +// make router listen on port 3000 +const server = http.createServer(app); + +if (!SREInstance.ready()) { + process.exit(1); +} //force SmythRuntime to initialize + +describe('Smyth FileSystem Tests', () => { + beforeAll(async () => { + const listen = promisify(server.listen.bind(server)); + await listen(PORT); + console.log(`Server is running on port ${PORT}`); + }); + + afterAll(async () => { + const close = promisify(server.close.bind(server)); + await close(); + console.log('Server has been shut down'); + }); + + it('initializes SmythFS', () => { + const smythFS = SmythFS.Instance; + expect(smythFS).toBeDefined(); + }); + it('Writes a SmythFS file from uri', async () => { + const smythFS = SmythFS.Instance; + let error; + try { + const uri = 'smythfs://default.team/myTestAgent/myTestFile.txt'; + + const candidate: IAccessCandidate = { + role: TAccessRole.Agent, + id: 'agent-123456', + }; + + await smythFS.write(uri, 'Hello World!', candidate); + } catch (e) { + error = e; + } + + expect(error).toBeUndefined(); + }); + + it('Reads a SmythFS file from uri', async () => { + const smythFS = SmythFS.Instance; + let error; + try { + const uri = 'smythfs://default.team/myTestAgent/myTestFile.txt'; + + const candidate: IAccessCandidate = { + role: TAccessRole.Agent, + id: 'agent-123456', + }; + const data = await smythFS.read(uri, candidate); + + expect(data.toString()).toEqual('Hello World!'); + } catch (e) { + error = e; + } + + expect(error).toBeUndefined(); + }); + + it('Does not allow Read to a different agent ', async () => { + const smythFS = SmythFS.Instance; + let error; + try { + const uri = 'smythfs://default.team/myTestAgent/myTestFile.txt'; + + const candidate: IAccessCandidate = { + role: TAccessRole.Agent, + id: 'agent-000000', + }; + const data = await smythFS.read(uri, candidate); + } catch (e) { + error = e; + } + + expect(error.message).toEqual('Access Denied'); + }); + + it('Deletes a SmythFS file from uri', async () => { + const smythFS = SmythFS.Instance; + let error; + try { + const uri = 'smythfs://default.team/myTestAgent/myTestFile.txt'; + + const candidate: IAccessCandidate = { + role: TAccessRole.Agent, + id: 'agent-123456', + }; + + await smythFS.delete(uri, candidate); + + const exists = await smythFS.exists(uri, candidate); + + expect(exists).toBeFalsy(); + } catch (e) { + error = e; + } + + expect(error).toBeUndefined(); + }); + + it.each([ + { contentType: 'text/plain', content: 'Hello World!' }, + { contentType: 'image/png', content: 'smythos.png' }, + ])('Generate temp url to serve $contentType content', async ({ contentType, content }) => { + const smythFS = SmythFS.Instance; + let error; + try { + const candidate: IAccessCandidate = AccessCandidate.team('TeamAPICall'); + + const uri = `smythfs://${candidate.id}.team/image-agent/myTestFile`; + + // write the file + let _preparedContent; + if (contentType === 'text/plain') { + _preparedContent = content; + } else if (contentType === 'image/png') { + const image = testData.readBinaryData(content); + + _preparedContent = image; + } + + await smythFS.write(uri, _preparedContent, candidate); + + const tempUrl = await smythFS.genTempUrl(uri, candidate); + console.log('tempUrl', tempUrl); + + expect(tempUrl).toBeDefined(); + + const response = await axios.get(tempUrl, { + responseType: 'arraybuffer', + }); + + expect(response.status).toBe(200); + expect(response.headers['content-type']).toBe(contentType); + + expect(Buffer.from(response.data).equals(Buffer.from(_preparedContent))).toBeTruthy(); + + // delete the file + // await smythFS.destroyTempUrl(tempUrl, { delResource: true }).catch((e) => {}); // destroyTempUrl wil be tested in separate test + } catch (e) { + error = e; + } + + expect(error).toBeUndefined(); + }); + + it('Delete content after the temp url is destroyed', async () => { + const smythFS = SmythFS.Instance; + let error; + try { + const candidate: IAccessCandidate = AccessCandidate.team('TeamAPICall'); + const uri = `smythfs://${candidate.id}.team/image-agent/myTestFile_unqiue`; + + // write the file + await smythFS.write(uri, 'Hello World!', candidate); + + // generate temp url + const tempUrl = await smythFS.genTempUrl(uri, candidate); + + // delete the file + await smythFS.destroyTempUrl(tempUrl, { delResource: true }); + + // try to reach the destroyed content + const responseErr = await axios.get(tempUrl).catch((e) => e); + expect(responseErr?.response?.status).toBe(404); + + // check if the file still exists + const exists = await smythFS.exists(uri, candidate); + expect(exists).toBeFalsy(); + } catch (e) { + error = e; + } + + expect(error).toBeUndefined(); + }); + + it('Expire temp url after TTL', async () => { + const smythFS = SmythFS.Instance; + let error; + try { + const candidate: IAccessCandidate = AccessCandidate.team('TeamAPICall'); + const uri = `smythfs://${candidate.id}.team/image-agent/myTestFile_unqiue`; + + // write the file + await smythFS.write(uri, 'Hello World!', candidate); + + // set the ttl + const tempUrl = await smythFS.genTempUrl(uri, candidate, 2); // 1 second ttl + + // wait for the ttl to expire + await new Promise((resolve) => setTimeout(resolve, 3000)); + + const responseErr = await axios.get(tempUrl).catch((e) => e); + expect(responseErr?.response?.status).toBe(404); + } catch (e) { + error = e; + } + + console.log('error', error); + expect(error).toBeUndefined(); + }); + + it.each([ + { contentType: 'text/plain', content: 'Hello World!' }, + { contentType: 'image/png', content: 'smythos.png' }, + ])('Generate resource url to serve $contentType content', async ({ contentType, content }) => { + const smythFS = SmythFS.Instance; + let error; + try { + const candidate: IAccessCandidate = { + role: TAccessRole.Agent, + id: 'agent-123456', + }; + + // const uri = `smythfs://default.team/myTestAgent/resourceFile`; + const uri = `smythfs://default.team/components_data/resourceFile`; + + // Write the file + let _preparedContent; + if (contentType === 'text/plain') { + _preparedContent = content; + } else if (contentType === 'image/png') { + const image = await fs.promises.readFile(path.join(__dirname, `../../data/${content}`)); + _preparedContent = image; + } + + await smythFS.write(uri, _preparedContent, candidate, { ContentType: contentType }); + + const resourceUrl = await smythFS.genResourceUrl(uri, candidate); + const cfg: any = ConnectorService.getAgentDataConnector().getAgentConfig(candidate.id); + const agentDomain = cfg?.agentStageDomain ? `https://${cfg.agentStageDomain}` : BASE_URL; + + expect(resourceUrl).toBeDefined(); + console.log('agent domain', agentDomain); + expect(resourceUrl.startsWith(agentDomain)).toBeTruthy(); + expect(resourceUrl.endsWith(mime.getExtension(contentType))).toBeTruthy(); + + // Test serving the resource + const testUrl = resourceUrl.replace(agentDomain, BASE_URL); + // for testing, we will use the the BaseUrl to fetch the resource + const response = await axios.get(testUrl, { + responseType: 'arraybuffer', + }); + + expect(response.status).toBe(200); + expect(response.headers['content-type']).toBe(contentType); + expect(Buffer.from(response.data).equals(Buffer.from(_preparedContent))).toBeTruthy(); + } catch (e) { + error = e; + } + + expect(error).toBeUndefined(); + }); + + it('Should not allow non-agent users to generate resource urls', async () => { + const smythFS = SmythFS.Instance; + let error; + try { + const candidate: IAccessCandidate = AccessCandidate.team('TeamAPICall'); + const uri = `smythfs://${candidate.id}.team/image-agent/resourceFile`; + + await smythFS.write(uri, 'Hello World!', candidate); + await smythFS.genResourceUrl(uri, candidate); + } catch (e) { + error = e; + } + + expect(error?.message).toBe('Only agents can generate resource urls'); + }); + + /* this test need to be fixed + it('Should handle invalid resource urls gracefully', async () => { + const smythFS = SmythFS.Instance; + let error; + try { + // Try to access an invalid resource URL + const invalidUrl = `${BASE_URL}/storage/invalid-uuid`; + const responseErr = await axios.get(invalidUrl).catch((e) => e); + expect(responseErr?.response?.status).toBe(404); + expect(responseErr?.response?.data).toBe('Invalid Resource URL'); + } catch (e) { + error = e; + } + + expect(error).toBeUndefined(); + }); + */ + + //TODO: test auto ContentTypes +}); diff --git a/packages/core/tests/unit/008-Components/APICall.test.ts b/packages/core/tests/unit/008-Components/APICall.test.ts new file mode 100644 index 000000000..4f813a6d4 --- /dev/null +++ b/packages/core/tests/unit/008-Components/APICall.test.ts @@ -0,0 +1,1640 @@ +import axios from 'axios'; +import express from 'express'; +import { describe, expect, it, vi, beforeEach } from 'vitest'; + +import { Agent } from '@sre/AgentManager/Agent.class'; +import { APICall } from '@sre/Components/APICall/APICall.class'; +import { setupSRE } from '../../utils/sre'; +import { ConnectorService } from '@sre/Core/ConnectorsService'; +import { testData } from '../../utils/test-data-manager'; +import { AccessCandidate, SmythFS } from 'index'; + +const app = express(); +const BASE_URL = `http://agents-server.smyth.stage`; + +setupSRE({ + Vault: { + Connector: 'JSONFileVault', + Settings: { + file: testData.getDataPath('vault.fake.json'), + }, + }, + Router: { + Connector: 'ExpressRouter', + Settings: { + router: app, + baseUrl: BASE_URL, + }, + }, +}); + +// Mock Agent class to keep the test isolated from the actual Agent implementation +vi.mock('@sre/AgentManager/Agent.class', () => { + const MockedAgent = vi.fn().mockImplementation(() => ({ + id: 'agent-123456', + agentRuntime: { debug: true }, // used inside createComponentLogger() + teamId: 'Team2', + isKilled: () => false, + modelsProvider: ConnectorService.getModelsProviderConnector(), + })); + return { Agent: MockedAgent }; +}); + +// @ts-ignore (Ignore required arguments, as we are using the mocked Agent) +const agent = new Agent(); +const apiCall = new APICall(); + +const VAULT_KEY_TEMPLATE_VAR = '{{KEY(SRE TEST KEY)}}'; +const DUMMY_KEY = 'sdl7k8lsd93ko4iu39'; + +const IMAGE_URL = 'https://app.smythos.dev/img/smythos-logo.png'; + +// TODO [Forhad]: Need to write more advance tests for URL +//- {{baseUrl}}/path/goes/here - baseUrl = https://httpbin.org +//- https://httpbin.org/{{path}} - path = /path/goes/here?q=some+query+params + +describe('APICall Component - HTTP Methods', () => { + const methods = ['GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'HEAD', 'OPTIONS']; + methods.forEach((method) => { + it(`handle ${method} method`, async () => { + const path = ['HEAD', 'OPTIONS'].includes(method) ? 'get' : method.toLowerCase(); + const url = `https://httpbin.org/${path}`; + + const config = { + data: { + method, + url, + headers: '', + contentType: 'none', + oauthService: 'None', + body: '', + }, + }; + const output = await apiCall.process({}, config, agent); + const headers = output.Headers; + + expect(headers).toBeInstanceOf(Object); + }); + }); +}); + +describe('APICall Component - Headers', () => { + it('handle default headers', async () => { + const config = { + data: { + method: 'GET', + url: 'https://httpbin.org/headers', + headers: '{"User-Agent": "APICall-Test", "Accept": "application/json"}', + contentType: 'none', + oauthService: 'None', + body: '', + }, + }; + const output = await apiCall.process({}, config, agent); + const response = output.Response; + + expect(response.headers['User-Agent']).toEqual('APICall-Test'); + expect(response.headers['Accept']).toEqual('application/json'); + }); + + it('handle custom headers', async () => { + const authToken = 'Bearer token'; + const contentType = 'application/json'; + + const config = { + data: { + method: 'GET', + url: 'https://httpbin.org/headers', + headers: `{"Authorization": "${authToken}", "Content-Type": "${contentType}"}`, + contentType: 'none', + oauthService: 'None', + body: '', + }, + }; + const output = await apiCall.process({}, config, agent); + const response = output.Response; + + expect(response.headers['Content-Type']).toEqual(contentType); + expect(response.headers['Authorization']).toEqual(authToken); + }); + + it('should override contentType header', async () => { + const config = { + data: { + method: 'GET', + url: 'https://httpbin.org/headers', + headers: '{"Content-Type": "application/xml"}', + contentType: 'application/json', + oauthService: 'None', + body: '', + }, + }; + const output = await apiCall.process({}, config, agent); + const response = output.Response; + + expect(response.headers['Content-Type']).toEqual('application/xml'); + }); + + it('resolve input template variable in headers', async () => { + const userName = 'John Doe'; + const config = { + data: { + method: 'GET', + url: 'https://httpbin.org/headers', + headers: `{"Authorization": "Bearer {{key}}", X-User-Name: "{{userName}}"}`, + contentType: 'none', + oauthService: 'None', + }, + }; + + const output = await apiCall.process({ key: DUMMY_KEY, userName }, config, agent); + const response = output.Response; + + expect(response.headers['Authorization']).toEqual(`Bearer ${DUMMY_KEY}`); + expect(response.headers['X-User-Name']).toEqual(userName); + }); + + it('resolve component template variable in headers', async () => { + const config = { + data: { + method: 'GET', + url: 'https://httpbin.org/headers', + headers: `{"Authorization": "Bearer {{VARVAULTINPUT:Authentication Key:[""]}}"}`, + contentType: 'none', + oauthService: 'None', + body: '', + _templateVars: { + 'VARVAULTINPUT-LTH3E8AB028': DUMMY_KEY, + }, + }, + + template: { + settings: { + 'VARVAULTINPUT-LTH3E8AB028': { + id: 'VARVAULTINPUT-LTH3E8AB028', + type: 'INPUT', + label: 'Authentication Key', + value: '', + options: [''], + attributes: { + 'data-template-vars': 'true', + 'data-vault': 'APICall,ALL', + }, + _templateEntry: true, + }, + }, + }, + }; + const output = await apiCall.process({}, config, agent); + const response = output.Response; + + expect(response.headers['Authorization']).toEqual(`Bearer ${DUMMY_KEY}`); + }); + + it('resolve vault key in headers', async () => { + const config = { + data: { + method: 'GET', + url: 'https://httpbin.org/headers', + headers: `{"Authorization": "Bearer ${VAULT_KEY_TEMPLATE_VAR}`, + contentType: 'none', + oauthService: 'None', + body: '', + }, + }; + + const output = await apiCall.process({}, config, agent); + const response = output.Response; + + expect(response.headers['Authorization']).toEqual(`Bearer ${DUMMY_KEY}`); + }); + + it('resolve multiple variable types in headers', async () => { + const config = { + data: { + method: 'GET', + url: 'https://httpbin.org/headers', + headers: `{ + "Authorization": "Bearer ${VAULT_KEY_TEMPLATE_VAR}", + "X-User-Name": "{{name}}", + "X-Api-Key": '{{VARVAULTINPUT:Authentication Key:[""]}}' + }`, + contentType: 'none', + oauthService: 'None', + body: '', + _templateVars: { + 'VARVAULTINPUT-LTH3E8AB028': DUMMY_KEY, + }, + }, + + template: { + settings: { + 'VARVAULTINPUT-LTH3E8AB028': { + id: 'VARVAULTINPUT-LTH3E8AB028', + type: 'INPUT', + label: 'Authentication Key', + value: '', + options: [''], + attributes: { + 'data-template-vars': 'true', + 'data-vault': 'APICall,ALL', + }, + _templateEntry: true, + }, + }, + }, + }; + + const name = 'John Doe'; + + const output = await apiCall.process({ name }, config, agent); + const response = output.Response; + + expect(response.headers['Authorization']).toEqual(`Bearer ${DUMMY_KEY}`); + expect(response.headers['X-User-Name']).toEqual(name); + expect(response.headers['X-Api-Key']).toEqual(DUMMY_KEY); + }); +}); + +describe('APICall Component - URL Formats', () => { + const url = 'https://httpbin.org/get?a=hello%20world&b=robot'; + + it('handle URL with query parameters', async () => { + const config = { + data: { + method: 'GET', + url, + headers: '', + contentType: 'none', + oauthService: 'None', + body: '', + }, + }; + const output = await apiCall.process({}, config, agent); + const response = output.Response; + + expect(response.args.a).toEqual('hello world'); + expect(response.args.b).toEqual('robot'); + }); + + it('handle URL with array query parameters', async () => { + const url = 'https://httpbin.org/get?ids[]=1&ids[]=2&ids[]=3'; + const config = { + data: { + method: 'GET', + url, + headers: '', + contentType: 'none', + oauthService: 'None', + body: '', + }, + }; + const output = await apiCall.process({}, config, agent); + const response = output.Response; + + expect(response.args['ids[]']).toEqual(['1', '2', '3']); + expect(response.url).toEqual(url); + }); + + it('handle URL with object query parameters', async () => { + const url = 'https://httpbin.org/get?filter[name]=John&filter[age]=30'; + const config = { + data: { + method: 'GET', + url, + headers: '', + contentType: 'none', + oauthService: 'None', + body: '', + }, + }; + const output = await apiCall.process({}, config, agent); + const response = output.Response; + + expect(response.url).toEqual(url); + expect(response.args['filter[age]']).toEqual('30'); + expect(response.args['filter[name]']).toEqual('John'); + }); + + it('handle URL with multiple occurrences of the same parameter', async () => { + const url = 'https://httpbin.org/get?color=red&color=blue&color=green'; + const config = { + data: { + method: 'GET', + url, + headers: '', + contentType: 'none', + oauthService: 'None', + body: '', + }, + }; + const output = await apiCall.process({}, config, agent); + const response = output.Response; + + expect(response.url).toEqual(url); + expect(response.args.color).toEqual(['red', 'blue', 'green']); + }); + + it('handle URL with nested object parameters', async () => { + const url = 'https://httpbin.org/get?user[name][first]=John&user[name][last]=Doe&user[age]=30'; + const config = { + data: { + method: 'GET', + url, + headers: '', + contentType: 'none', + oauthService: 'None', + body: '', + }, + }; + const output = await apiCall.process({}, config, agent); + const response = output.Response; + + expect(response.url).toEqual(url); + expect(response.args['user[name][first]']).toEqual('John'); + expect(response.args['user[name][last]']).toEqual('Doe'); + expect(response.args['user[age]']).toEqual('30'); + }); + + it('handle URL with empty parameter values', async () => { + const url = 'https://httpbin.org/get?empty=&null=&undefined='; + const config = { + data: { + method: 'GET', + url, + headers: '', + contentType: 'none', + oauthService: 'None', + body: '', + }, + }; + const output = await apiCall.process({}, config, agent); + const response = output.Response; + + expect(response.url).toEqual(url); + expect(response.args.empty).toEqual(''); + expect(response.args.null).toEqual(''); + expect(response.args.undefined).toEqual(''); + }); + + it('handle URL with encoded spaces and plus signs', async () => { + const url = 'https://httpbin.org/get?message=hello%20world&operation=1+1'; + const config = { + data: { + method: 'GET', + url, + headers: '', + contentType: 'none', + oauthService: 'None', + body: '', + }, + }; + const output = await apiCall.process({}, config, agent); + const response = output.Response; + + expect(response.args.message).toEqual('hello world'); + expect(response.args.operation).toEqual('1 1'); + }); + + //#region test cases with symbols and special characters + // * Note: Following commented test cases in includes characters that that could be used in very rare cases, we will check later + /* it('handle URL with all types of raw characters and symbols', async () => { + const allChars = `!@$%^*()_+-={}[]|\:;"'<>,.?/~\`∑πΔ∞≠≤≥±×÷√∫∂$€£¥₹₽₩₪áéíóúñüçãõâêîôûäëïöü😀🌍🚀🎉🍕🐱‍👤©®™♥♠♣♦☢☣☮☯Hello, 世界! ¿Cómo estás? 123 + 456 = 579 ©️ 🌈#&`; // we should keep # and & in the end of the string for it's special meaning in URL + const url = `https://httpbin.org/get?all=${allChars}`; + + const config = { + data: { + method: 'GET', + url, + headers: '', + contentType: 'none', + oauthService: 'None', + body: '', + }, + + }; + const output = await apiCall.process({}, config, agent); + const response = output.Response; + + // The expected arguments and URL encoding differs between browsers and Postman, We're expecting the Postman version. + const expectedChars = `!@$%^*()_ -={}[]|\\:;\"'<>,.?/~\`\u2211\u03c0\u0394\u221e\u2260\u2264\u2265\u00b1\u00d7\u00f7\u221a\u222b\u2202$\u20ac\u00a3\u00a5\u20b9\u20bd\u20a9\u20aa\u00e1\u00e9\u00ed\u00f3\u00fa\u00f1\u00fc\u00e7\u00e3\u00f5\u00e2\u00ea\u00ee\u00f4\u00fb\u00e4\u00eb\u00ef\u00f6\u00fc\ud83d\ude00\ud83c\udf0d\ud83d\ude80\ud83c\udf89\ud83c\udf55\ud83d\udc31\u200d\ud83d\udc64\u00a9\u00ae\u2122\u2665\u2660\u2663\u2666\u2622\u2623\u262e\u262fHello, \u4e16\u754c! \u00bfC\u00f3mo est\u00e1s? 123 456 = 579 \u00a9\ufe0f \ud83c\udf08`; + const expectedUrl = `https://httpbin.org/get?all=!%40$%^*()_+-={}[]|\\:%3B\"'<>,.%3F%2F~\`\u2211\u03c0\u0394\u221e\u2260\u2264\u2265\u00b1\u00d7\u00f7\u221a\u222b\u2202$\u20ac\u00a3\u00a5\u20b9\u20bd\u20a9\u20aa\u00e1\u00e9\u00ed\u00f3\u00fa\u00f1\u00fc\u00e7\u00e3\u00f5\u00e2\u00ea\u00ee\u00f4\u00fb\u00e4\u00eb\u00ef\u00f6\u00fc\ud83d\ude00\ud83c\udf0d\ud83d\ude80\ud83c\udf89\ud83c\udf55\ud83d\udc31\u200d\ud83d\udc64\u00a9\u00ae\u2122\u2665\u2660\u2663\u2666\u2622\u2623\u262e\u262fHello, \u4e16\u754c! \u00bfC\u00f3mo est\u00e1s%3F 123 + 456 = 579 \u00a9\ufe0f \ud83c\udf08`; + + expect(response.args.all).toEqual(expectedChars); + expect(response.url).toEqual(expectedUrl); + }); + + it('handle URL with all types of encoded characters and symbols', async () => { + const allChars = + '!@$%^*()_+-={}[]|\\:;"\'<>,.?/~`∑πΔ∞≠≤≥±×÷√∫∂$€£¥₹₽₩₪áéíóúñüçãõâêîôûäëïöü😀🌍🚀🎉🍕🐱‍👤©®™♥♠♣♦☢☣☮☯Hello, 世界! ¿Cómo estás? 123 + 456 = 579 ©️ 🌈#&'; // we should keep # and & in the end of the string for it's special meaning in URL + const url = `https://httpbin.org/get?all=${encodeURIComponent(allChars)}`; + + const config = { + data: { + method: 'GET', + url, + headers: '', + contentType: 'none', + oauthService: 'None', + body: '', + }, + + }; + const output = await apiCall.process({}, config, agent); + const response = output.Response; + + const expectedChars = + '!@$%^*()_+-={}[]|\\:;"\'<>,.?/~`∑πΔ∞≠≤≥±×÷√∫∂$€£¥₹₽₩₪áéíóúñüçãõâêîôûäëïöü😀🌍🚀🎉🍕🐱‍👤©®™♥♠♣♦☢☣☮☯Hello, 世界! ¿Cómo estás? 123 + 456 = 579 ©️ 🌈#&'; + const expectedUrl = + 'https://httpbin.org/get?all=!%40%24%25^*()_%2B-%3D{}[]|\\%3A%3B"\'<>%2C.%3F%2F~`∑πΔ∞≠≤≥±×÷√∫∂%24€£¥₹₽₩₪áéíóúñüçãõâêîôûäëïöü😀🌍🚀🎉🍕🐱‍👤©®™♥♠♣♦☢☣☮☯Hello%2C 世界! ¿Cómo estás%3F 123 %2B 456 %3D 579 ©️ 🌈%23%26'; + + expect(response.args.all).toEqual(expectedChars); + expect(response.url).toEqual(expectedUrl); + }); */ + //#endregion test cases with symbols and special characters + + it('handle URL with common symbols and special characters', async () => { + const specialChars = "!@$'()*+,;=-._~:/?[]#&"; + const url = `https://httpbin.org/get?special=${specialChars}`; + + const config = { + data: { + method: 'GET', + url, + headers: '', + contentType: 'none', + oauthService: 'None', + body: '', + }, + }; + const output = await apiCall.process({}, config, agent); + const response = output.Response; + + const expectedSpecialChars = "!@$'()* ,;=-._~:/?[]"; + const expectedUrl = "https://httpbin.org/get?special=!%40$'()* ,%3B=-._~:%2F%3F[]"; + + expect(response.args.special).toEqual(expectedSpecialChars); + expect(response.url).toEqual(expectedUrl); + }); + + it('handle URL with encoded common symbols and special characters', async () => { + const specialChars = "!@$'()*,;=-._~:/?[]"; + const url = `https://httpbin.org/get?special=${encodeURIComponent(specialChars)}`; + + const config = { + data: { + method: 'GET', + url, + headers: '', + contentType: 'none', + oauthService: 'None', + body: '', + }, + }; + const output = await apiCall.process({}, config, agent); + const response = output.Response; + + const expectedSpecialChars = "!@$'()*,;=-._~:/?[]"; + + expect(response.args.special).toEqual(expectedSpecialChars); + + // TODO: We have difference in returned URL and expected URL for some of the special characters. Need to check it later. + // response.url is "https://httpbin.org/get?special=!%40$'()*,%3B=-._~:%2F%3F[]"; + // const expectedUrl = "https://httpbin.org/get?special=!%40%24'()*%2C%3B%3D-._~%3A%2F%3F[]"; // According to Postman + + // expect(response.url).toEqual(expectedUrl); + }); + + it('handle URL with fragment identifier', async () => { + const fragment = '#section1'; + const urlWithoutFragment = `https://httpbin.org/get?param=value`; + const url = `${urlWithoutFragment}${fragment}`; + const config = { + data: { + method: 'GET', + url, + headers: '', + contentType: 'none', + oauthService: 'None', + body: '', + }, + }; + const output = await apiCall.process({}, config, agent); + const response = output.Response; + + expect(response.url).toEqual(urlWithoutFragment); + expect(response.args.param).toEqual('value'); + }); + + it('handle URL with basic auth credentials', async () => { + const url = 'https://user:pass@httpbin.org/basic-auth/user/pass'; + const config = { + data: { + method: 'GET', + url, + headers: '', + contentType: 'none', + oauthService: 'None', + body: '', + }, + }; + const output = await apiCall.process({}, config, agent); + const response = output.Response; + + expect(response.authenticated).toEqual(true); + expect(response.user).toEqual('user'); + }); + + it('handle wrong URL', async () => { + const url = 'https://httpbin.org/wrong-url'; + const config = { + data: { + method: 'GET', + url, + headers: '', + contentType: 'none', + oauthService: 'None', + body: '', + }, + }; + const output = await apiCall.process({}, config, agent); + + expect(output._error).toBeDefined(); + expect(output._error).toContain('404'); + }); + + it('resolve input template variable in URL', async () => { + const user = 'John Doe'; + const url = 'https://httpbin.org/get?user={{user}}'; + const config = { + data: { + method: 'GET', + url, + contentType: 'none', + oauthService: 'None', + body: '', + }, + }; + const output = await apiCall.process({ user }, config, agent); + const response = output.Response; + + expect(response.args.user).toEqual(user); + expect(response.url).toEqual(`https://httpbin.org/get?user=${user}`); + }); + + it('resolve component template variable in URL', async () => { + const url = 'https://httpbin.org/get?key={{VARVAULTINPUT:Authentication Key:[""]}}'; + const config = { + data: { + method: 'GET', + url, + contentType: 'none', + oauthService: 'None', + body: '', + _templateVars: { + 'VARVAULTINPUT-LTH3E8AB028': DUMMY_KEY, + }, + }, + template: { + settings: { + 'VARVAULTINPUT-LTH3E8AB028': { + id: 'VARVAULTINPUT-LTH3E8AB028', + type: 'INPUT', + label: 'Authentication Key', + value: '', + options: [''], + attributes: { + 'data-template-vars': 'true', + 'data-vault': 'APICall,ALL', + }, + _templateEntry: true, + }, + }, + }, + }; + const output = await apiCall.process({}, config, agent); + const response = output.Response; + + expect(response.args.key).toEqual(DUMMY_KEY); + expect(response.url).toEqual(`https://httpbin.org/get?key=${DUMMY_KEY}`); + }); + + it('resolve vault key in URL', async () => { + const url = `https://httpbin.org/get?key=${VAULT_KEY_TEMPLATE_VAR}`; + const config = { + data: { + method: 'GET', + url, + contentType: 'none', + oauthService: 'None', + body: '', + }, + }; + + const output = await apiCall.process({}, config, agent); + const response = output.Response; + + expect(response.args.key).toEqual(DUMMY_KEY); + expect(response.url).toEqual(`https://httpbin.org/get?key=${DUMMY_KEY}`); + }); + + it('resolve multiple variable types in URL', async () => { + const url = `https://httpbin.org/get?user={{user}}&key={{VARVAULTINPUT:Authentication Key:[""]}}&secret=${VAULT_KEY_TEMPLATE_VAR}`; + const config = { + data: { + method: 'GET', + url, + contentType: 'none', + oauthService: 'None', + body: '', + _templateVars: { + 'VARVAULTINPUT-LTH3E8AB028': DUMMY_KEY, + }, + }, + template: { + settings: { + 'VARVAULTINPUT-LTH3E8AB028': { + id: 'VARVAULTINPUT-LTH3E8AB028', + type: 'INPUT', + label: 'Authentication Key', + value: '', + options: [''], + attributes: { + 'data-template-vars': 'true', + 'data-vault': 'APICall,ALL', + }, + _templateEntry: true, + }, + }, + }, + }; + const user = 'John Doe'; + const output = await apiCall.process({ user }, config, agent); + const response = output.Response; + + expect(response.args.user).toEqual(user); + expect(response.args.key).toEqual(DUMMY_KEY); + expect(response.args.secret).toEqual(DUMMY_KEY); + }); + + it('resolve smythfs:// URI in public URL', async () => { + // write a file in Smyth File System (will be used to test smythfs:// uris passed as arguments) + await SmythFS.Instance.write('smythfs://Team2.team/agent-123456/_temp/file.txt', 'Hello, world!', AccessCandidate.agent('agent-123456')); + const url = 'https://httpbin.org/get?image=smythfs://Team2.team/agent-123456/_temp/file.txt'; + const config = { + data: { + method: 'GET', + url, + contentType: 'none', + oauthService: 'None', + body: '', + headers: '', + }, + }; + + const output = await apiCall.process({}, config, agent); + const response = output.Response; + + const regex = new RegExp(`${BASE_URL}`); + + // delete the file + await SmythFS.Instance.delete('smythfs://Team2.team/agent-123456/_temp/file.txt', AccessCandidate.agent('agent-123456')); + expect(response.args.image).toMatch(regex); + }); + + it('does not resolve smythfs:// URI if it does not belong to the agent', async () => { + await SmythFS.Instance.write('smythfs://Team2.team/agent-007/_temp/file.txt', 'Hello, world!', AccessCandidate.agent('agent-007')); + const url = 'https://httpbin.org/get?image=smythfs://AnotherTeam.team/agent-007/_temp/M4I8A5XIDKJ.jpeg'; + const config = { + data: { + method: 'GET', + url, + contentType: 'none', + oauthService: 'None', + body: '', + headers: '', + }, + }; + + const output = await apiCall.process({}, config, agent); + const response = output.Response; + + const regex = new RegExp(`${BASE_URL}`); + // delete the file + await SmythFS.Instance.delete('smythfs://Team2.team/agent-007/_temp/file.txt', AccessCandidate.agent('agent-007')); + expect(response).toBeUndefined(); + expect(output).toHaveProperty('_error'); + expect(output._error).toContain('Access Denied'); + }); +}); + +describe('APICall Component - Content Types', () => { + const contentTypes = ['none', 'application/json', 'multipart/form-data', 'binary', 'application/x-www-form-urlencoded', 'text/plain']; + contentTypes.forEach((contentType) => { + it(`handle ${contentType} content type`, async () => { + const config = { + data: { + method: 'GET', + url: 'https://httpbin.org/get', + headers: '', + contentType, + oauthService: 'None', + }, + }; + const output = await apiCall.process({}, config, agent); + const response = output.Response; + + const expectedContentType = contentType === 'none' ? undefined : contentType; + expect(response.headers['Content-Type']).toEqual(expectedContentType); + }); + }); +}); + +describe('APICall Component - Body', () => { + it('handle application/json content type', async () => { + const body = { name: 'John Doe', age: 30 }; + const config = { + data: { + method: 'POST', + url: 'https://httpbin.org/post', + headers: '', + contentType: 'application/json', + body: JSON.stringify(body), + oauthService: 'None', + }, + }; + const output = await apiCall.process({}, config, agent); + const response = output.Response; + + expect(response.headers['Content-Type']).toContain('application/json'); + expect(response.json).toEqual(body); + }); + + it('handle application/x-www-form-urlencoded content type', async () => { + const config = { + data: { + method: 'POST', + url: 'https://httpbin.org/post', + headers: '', + contentType: 'application/x-www-form-urlencoded', + body: 'name=John+Doe&age=30', + oauthService: 'None', + }, + }; + const output = await apiCall.process({}, config, agent); + const response = output.Response; + + expect(response.headers['Content-Type']).toContain('application/x-www-form-urlencoded'); + expect(response.form).toEqual({ name: 'John Doe', age: '30' }); + }); + + it('handle text/plain content type', async () => { + const config = { + data: { + method: 'POST', + url: 'https://httpbin.org/post', + headers: '', + contentType: 'text/plain', + body: 'Hello, world!', + oauthService: 'None', + }, + }; + const output = await apiCall.process({}, config, agent); + const response = output.Response; + + expect(response.headers['Content-Type']).toContain('text/plain'); + expect(response.data).toEqual('Hello, world!'); + }); + + const fetchFileInfoAndContent = async (fileUrl: string): Promise<{ mimetype: string; size: number; buffer: Buffer | null }> => { + if (!fileUrl) return { mimetype: '', size: 0, buffer: null }; + + try { + const response = await axios.get(fileUrl, { responseType: 'arraybuffer' }); + const data = response.data || ''; + const buffer = Buffer.from(data, 'binary'); + const size = buffer.byteLength; + + return { mimetype: response.headers['content-type'], size, buffer }; + } catch (error: any) { + return { mimetype: '', size: 0, buffer: null }; + } + }; + + it('handle multipart/form-data with base64 input', async () => { + const config = { + data: { + method: 'POST', + url: 'https://httpbin.org/post', + contentType: 'multipart/form-data', + body: '{"image": "{{image}}"}', + oauthService: 'None', + }, + inputs: [ + { + name: 'image', + type: 'Binary', + color: '#F35063', + optional: false, + index: 0, + default: false, + }, + ], + }; + + const { mimetype, buffer } = await fetchFileInfoAndContent(IMAGE_URL); + + // Convert buffer to base64 URL + const base64Data = buffer ? buffer.toString('base64') : ''; + const base64Url = `data:${mimetype};base64,${base64Data}`; + + const output = await apiCall.process({ image: base64Url }, config, agent); + const response = output.Response; + + expect(response.headers['Content-Type']).toMatch(/^multipart\/form-data; boundary=/); + expect(response).toHaveProperty('files'); + expect(response.files).toHaveProperty('image'); + expect(response.files.image).toMatch(/^data:image\/png;base64,/); + }); + + it('handle multipart/form-data with SmythFile object input', async () => { + const config = { + data: { + method: 'POST', + url: 'https://httpbin.org/post', + contentType: 'multipart/form-data', + body: '{"image": "{{image}}"}', + oauthService: 'None', + }, + }; + + const { mimetype, size } = await fetchFileInfoAndContent(IMAGE_URL); + + const output = await apiCall.process( + { + image: { + mimetype, + size, + url: IMAGE_URL, + }, + }, + config, + agent + ); + const response = output.Response; + + expect(response.headers['Content-Type']).toMatch(/^multipart\/form-data; boundary=/); + expect(response).toHaveProperty('files'); + expect(response.files).toHaveProperty('image'); + expect(response.files.image).toMatch(/^data:image\/png;base64,/); + }); + + it('handle multipart/form-data with SmythFile object input as Binary type', async () => { + const config = { + data: { + method: 'POST', + url: 'https://httpbin.org/post', + contentType: 'multipart/form-data', + body: '{"image": "{{image}}"}', + oauthService: 'None', + }, + inputs: [ + { + name: 'image', + type: 'Binary', + color: '#F35063', + optional: false, + index: 0, + default: false, + }, + ], + }; + + const { mimetype, size } = await fetchFileInfoAndContent(IMAGE_URL); + + const output = await apiCall.process( + { + image: { + mimetype, + size, + url: IMAGE_URL, + }, + }, + config, + agent + ); + const response = output.Response; + + expect(response.headers['Content-Type']).toMatch(/^multipart\/form-data; boundary=/); + expect(response).toHaveProperty('files'); + expect(response.files).toHaveProperty('image'); + expect(response.files.image).toMatch(/^data:image\/png;base64,/); + }); + + it('handle multipart/form-data with smythfs:// URI', async () => { + const imageData = testData.readBinaryData('smythos.png'); + await SmythFS.Instance.write('smythfs://Team2.team/agent-123456/_temp/smythos.png', imageData, AccessCandidate.agent('agent-123456')); + + const input = { + image: { + mimetype: 'image/png', + size: 13179, + url: 'smythfs://Team2.team/agent-123456/_temp/smythos.png', + name: 'smythos.png', + }, + }; + const config = { + data: { + method: 'POST', + url: 'https://httpbin.org/post', + contentType: 'multipart/form-data', + body: '{"image": "{{image}}"}', + oauthService: 'None', + }, + inputs: [ + { + name: 'image', + type: 'Binary', + color: '#F35063', + optional: false, + index: 0, + default: false, + }, + ], + }; + + const output = await apiCall.process(input, config, agent); + const response = output.Response; + + // delete the file + await SmythFS.Instance.delete('smythfs://Team2.team/agent-123456/_temp/smythos.png', AccessCandidate.agent('agent-123456')); + + expect(response.headers['Content-Type']).toMatch(/^multipart\/form-data; boundary=/); + expect(response).toHaveProperty('files'); + expect(response.files).toHaveProperty('image'); + expect(response.files.image).toMatch(/^data:image\/png;base64,/); + }); + + it('handle binary content type with base64 input', async () => { + const config = { + data: { + method: 'POST', + url: 'https://httpbin.org/post', + headers: '', + contentType: 'binary', + body: '{{file}}', + oauthService: 'None', + }, + inputs: [ + { + name: 'file', + type: 'Binary', + color: '#F35063', + optional: false, + index: 0, + default: false, + }, + ], + }; + + const { mimetype, size, buffer } = await fetchFileInfoAndContent(IMAGE_URL); + + // Convert buffer to base64 URL + const base64Data = buffer ? buffer.toString('base64') : ''; + const base64Url = `data:${mimetype};base64,${base64Data}`; + + const output = await apiCall.process({ file: base64Url }, config, agent); + const response = output.Response; + + expect(response.headers['Content-Type']).toMatch(mimetype); + expect(response.headers['Content-Length']).toEqual(size.toString()); + // for some reason httpbin returns data as application/octet-stream + expect(response.data).toMatch(/^data:application\/octet-stream;base64,/); + }); + + it('handle binary with SmythFile object', async () => { + const config = { + data: { + method: 'POST', + url: 'https://httpbin.org/post', + headers: '', + contentType: 'binary', + body: '{{file}}', + oauthService: 'None', + }, + }; + + const { mimetype, size } = await fetchFileInfoAndContent(IMAGE_URL); + + const output = await apiCall.process( + { + file: { + mimetype, + size, + url: IMAGE_URL, + }, + }, + config, + agent + ); + const response = output.Response; + + expect(response.headers['Content-Type']).toMatch(mimetype); + expect(response.headers['Content-Length']).toEqual(size.toString()); + // for some reason httpbin returns data as application/octet-stream + expect(response.data).toMatch(/^data:application\/octet-stream;base64,/); + }); + + it('handle binary with SmythFile object as binary input', async () => { + const config = { + data: { + method: 'POST', + url: 'https://httpbin.org/post', + headers: '', + contentType: 'binary', + body: '{{file}}', + oauthService: 'None', + }, + inputs: [ + { + name: 'file', + type: 'Binary', + color: '#F35063', + optional: false, + index: 0, + default: false, + }, + ], + }; + + const { mimetype, size } = await fetchFileInfoAndContent(IMAGE_URL); + + const output = await apiCall.process( + { + file: { + mimetype, + size, + url: IMAGE_URL, + }, + }, + config, + agent + ); + const response = output.Response; + + expect(response.headers['Content-Type']).toMatch(mimetype); + expect(response.headers['Content-Length']).toEqual(size.toString()); + // for some reason httpbin returns data as application/octet-stream + expect(response.data).toMatch(/^data:application\/octet-stream;base64,/); + }); + + it('handle empty body', async () => { + const config = { + data: { + method: 'POST', + url: 'https://httpbin.org/post', + headers: '', + contentType: 'none', + body: '', + oauthService: 'None', + }, + }; + const output = await apiCall.process({}, config, agent); + const response = output.Response; + + expect(response.headers['Content-Type']).toEqual('application/x-www-form-urlencoded'); + expect(response.data).toEqual(''); + expect(response.headers['Content-Length']).toEqual('0'); + }); + + it('handle application/xml content type', async () => { + const config = { + data: { + method: 'POST', + url: 'https://httpbin.org/post', + headers: '', + contentType: 'application/xml', + body: 'John Doe30', + oauthService: 'None', + }, + }; + const output = await apiCall.process({}, config, agent); + const response = output.Response; + + expect(response.headers['Content-Type']).toContain('application/xml'); + expect(response.data).toContain('John Doe30'); + }); + + // TODO [Forhad]: Need to make it work + it('resolve input template variable in body', async () => { + const body = { name: 'John Doe', age: 30 }; + const config = { + data: { + method: 'POST', + url: 'https://httpbin.org/post', + headers: '', + contentType: 'application/json', + body: '{{body}}', + oauthService: 'None', + }, + }; + const output = await apiCall.process({ body: JSON.stringify(body) }, config, agent); + const response = output.Response; + + expect(response.headers['Content-Type']).toContain('application/json'); + expect(response.json).toEqual(body); + }); + + it('resolve input template variable inside body properties', async () => { + const name = 'John Doe'; + const age = 30; + const config = { + data: { + method: 'POST', + url: 'https://httpbin.org/post', + headers: '', + contentType: 'application/json', + body: '{name: "{{name}}", age: {{age}}}', + oauthService: 'None', + }, + }; + const output = await apiCall.process({ name, age }, config, agent); + const response = output.Response; + + expect(response.headers['Content-Type']).toContain('application/json'); + expect(response.json).toEqual({ name, age }); + }); + + it('resolve input template variable in body with URL Encoded content type', async () => { + const url = 'https://httpbin.org/post'; + const config = { + data: { + method: 'POST', + url, + headers: '', + contentType: 'application/x-www-form-urlencoded', + body: `{\n \"To\": \"{{to}}\",\n \"From\": \"{{from}}\",\n \"Body\": \"{{body}}\"\n}`, + oauthService: 'None', + }, + inputs: [ + { + name: 'to', + type: 'Number', + color: '#F35063', + optional: false, + index: 0, + default: false, + }, + { + name: 'from', + type: 'Any', + color: '#F35063', + optional: false, + index: 1, + default: false, + }, + { + name: 'body', + type: 'Any', + color: '#F35063', + optional: false, + index: 2, + default: false, + }, + ], + }; + + const to = '123456789'; + const from = '987654321'; + const body = 'Hello, how are you?'; + const input = { + to, + from, + body, + }; + const output = await apiCall.process(input, config, agent); + const response = output.Response; + + expect(response.form.To).toEqual(to); + expect(response.form.From).toEqual(from); + expect(response.form.Body).toEqual(body); + + expect(response.url).toEqual(url); + }); + + it('resolve component template variable in body', async () => { + const userData = { name: 'John Doe', age: 30 }; + const config = { + data: { + method: 'POST', + url: 'https://httpbin.org/post', + headers: '', + contentType: 'application/json', + body: '{{VARVAULTINPUT:User Data:[""]}}', + oauthService: 'None', + _templateVars: { + 'VARVAULTINPUT-LTH3E8AB028': JSON.stringify(userData), + }, + }, + template: { + settings: { + 'VARVAULTINPUT-LTH3E8AB028': { + id: 'VARVAULTINPUT-LTH3E8AB028', + type: 'INPUT', + label: 'User Data', + value: '', + options: [''], + attributes: { + 'data-template-vars': 'true', + 'data-vault': 'APICall,ALL', + }, + _templateEntry: true, + }, + }, + }, + }; + const output = await apiCall.process({}, config, agent); + const response = output.Response; + + expect(response.headers['Content-Type']).toContain('application/json'); + expect(response.json).toEqual(userData); + }); + + it('resolve vault key in body', async () => { + const config = { + data: { + method: 'POST', + url: 'https://httpbin.org/post', + headers: '', + contentType: 'application/json', + body: `{"key": ${VAULT_KEY_TEMPLATE_VAR}}`, + oauthService: 'None', + }, + }; + const output = await apiCall.process({}, config, agent); + const response = output.Response; + + expect(response.headers['Content-Type']).toContain('application/json'); + expect(response.json.key).toEqual(DUMMY_KEY); + }); + + it('resolve multiple variable types in Body', async () => { + const userName = 'John Doe'; + const userData = { name: userName, age: 30 }; + + const config = { + data: { + method: 'POST', + url: 'https://httpbin.org/post', + headers: '', + contentType: 'application/json', + body: `{"name": "{{name}}", "userData": {{VARVAULTINPUT:User Data:[""]}}, "key": ${VAULT_KEY_TEMPLATE_VAR}}`, + oauthService: 'None', + _templateVars: { + 'VARVAULTINPUT-LTH3E8AB028': JSON.stringify(userData), + }, + }, + template: { + settings: { + 'VARVAULTINPUT-LTH3E8AB028': { + id: 'VARVAULTINPUT-LTH3E8AB028', + type: 'INPUT', + label: 'User Data', + value: '', + options: [''], + attributes: { + 'data-template-vars': 'true', + 'data-vault': 'APICall,ALL', + }, + _templateEntry: true, + }, + }, + }, + }; + + const output = await apiCall.process({ name: userName }, config, agent); + const response = output.Response; + + expect(response.headers['Content-Type']).toContain('application/json'); + expect(response.json).toEqual({ + name: userName, + userData: userData, + key: DUMMY_KEY, + }); + }); + + it(`should handle falsy values (0, '', false) correctly in request body`, async () => { + const input = { + num: '0', + int: '0.11', + str: '', + bool: 'false', + }; + const config = { + data: { + method: 'POST', + url: 'https://httpbin.org/post', + headers: '', + contentType: 'application/json', + body: '{\n "number": {{num}},\n "integer": {{int}},\n "string": "{{str}}",\n "boolean": {{bool}}\n}', + oauthService: 'None', + }, + inputs: [ + { + name: 'num', + type: 'Number', + color: '#F35063', + optional: false, + index: 0, + default: false, + }, + { + name: 'int', + type: 'Integer', + color: '#F35063', + optional: false, + index: 1, + default: false, + }, + { + name: 'str', + type: 'String', + color: '#F35063', + optional: false, + index: 2, + default: false, + }, + { + name: 'bool', + type: 'Boolean', + color: '#F35063', + optional: false, + index: 3, + default: false, + }, + ], + }; + const output = await apiCall.process(input, config, agent); + const response = output.Response; + + expect(response.json.integer).toEqual(0); + expect(response.json.number).toEqual(0); + expect(response.json.string).toEqual(''); + expect(response.json.boolean).toEqual(false); + }); + + it('should handle request body with only template variable that hold object', async () => { + const input = { + obj: { + name: 'test1', + email: 'test1@example.com', + age: 30, + }, + }; + const config = { + data: { + method: 'POST', + url: 'https://httpbin.org/post', + headers: '', + contentType: 'application/json', + body: '{{obj}}', + oauthService: 'None', + }, + inputs: [ + { + name: 'obj', + type: 'Object', + color: '#F35063', + optional: false, + index: 0, + default: false, + }, + ], + }; + const output = await apiCall.process(input, config, agent); + const response = output.Response; + + expect(response.json).toEqual(input.obj); + }); + + it('should resolve template variables containing objects', async () => { + const input = { + obj: { + name: 'test1', + email: 'test1@example.com', + age: 30, + }, + nestedObj: { + name: 'test2', + email: 'test1@example.com', + age: 40, + }, + }; + const config = { + data: { + method: 'POST', + url: 'https://httpbin.org/post', + headers: '', + contentType: 'application/json', + body: '{\n "obj": {{obj}},\n "nestedObj": {{nestedObj}}\n}', + oauthService: 'None', + }, + inputs: [ + { + name: 'obj', + type: 'Object', + color: '#F35063', + optional: false, + index: 0, + default: false, + }, + { + name: 'nestedObj', + type: 'Object', + color: '#F35063', + optional: false, + index: 1, + default: false, + }, + ], + }; + const output = await apiCall.process(input, config, agent); + const response = output.Response; + + expect(response.json.obj).toEqual(input.obj); + expect(response.json.nestedObj).toEqual(input.nestedObj); + }); + + it('should resolve template variables containing array', async () => { + const input = { + arr: ['Item1', 'Item2', 'Item3', 'Item4', 'Item5'], + }; + const config = { + data: { + method: 'POST', + contentType: 'application/json', + oauthService: 'None', + url: 'https://httpbin.org/post', + headers: '', + body: '{\n "arr": {{arr}}\n}', + }, + inputs: [ + { + name: 'arr', + type: 'Array', + color: '#F35063', + optional: false, + defaultVal: '{{array}}', + index: 0, + default: false, + }, + ], + }; + const output = await apiCall.process(input, config, agent); + const response = output.Response; + + expect(response.json.arr).toEqual(input.arr); + }); +}); + +/* +//OAuth tests and proxy tests need to be re-implemented +//the oauth approach below is deprecated + +describe('APICall Component - OAuth', () => { + it('handle OAuth2 authentication', async () => { + const config = { + id: 'M1LWWLNL1V', + name: 'APICall', + title: 'OAuth2 with Google', + data: { + method: 'POST', + url: 'https://httpbin.org/post', + headers: '', + contentType: 'none', + body: '', + proxy: '', + oauthService: 'Google', + scope: 'https://www.googleapis.com/auth/gmail.readonly', + authorizationURL: 'https://accounts.google.com/o/oauth2/v2/auth', + tokenURL: 'https://oauth2.googleapis.com/token', + clientID: '{{KEY(Google Client ID)}}', + clientSecret: '{{KEY(Google Client Secret)}}', + requestTokenURL: '', + accessTokenURL: '', + userAuthorizationURL: '', + consumerKey: '', + consumerSecret: '', + authenticate: '', + }, + }; + const output = await apiCall.process({}, config, agent); + const response = output.Response; + + expect(response.headers['Authorization']).toMatch(/^Bearer .{200,}$/); + }); + + it('handle OAuth1 authentication', async () => { + const config = { + id: 'CM1LXC1LAZV9', + name: 'APICall', + title: 'OAuth1 with X', + data: { + method: 'POST', + url: 'https://httpbin.org/post', + headers: '', + contentType: 'none', + body: '', + proxy: '', + oauthService: 'Twitter', + scope: '', + authorizationURL: '', + tokenURL: '', + clientID: '', + clientSecret: '', + requestTokenURL: 'https://api.twitter.com/oauth/request_token', + accessTokenURL: 'https://api.twitter.com/oauth/access_token', + userAuthorizationURL: 'https://api.twitter.com/oauth/authorize', + consumerKey: '{{KEY(X Consumer Key)}}', + consumerSecret: '{{KEY(X Consumer Secret)}}', + authenticate: '', + }, + }; + const output = await apiCall.process({}, config, agent); + const response = output.Response; + expect(response.headers['Authorization']).toMatch( + /OAuth oauth_consumer_key="[^"]+", oauth_nonce="[^"]+", oauth_signature="[^"]+", oauth_signature_method="HMAC-SHA1", oauth_timestamp="[^"]+", oauth_token="[^"]+", oauth_version="1.0"/ + ); + }); +}); + +describe('APICall Component - Proxy', () => { + const proxyHost = '207.244.217.165'; + + function handleProxy(scheme: string, proxyUrl: string) { + it(`handle proxy settings with ${scheme} protocol`, async () => { + const config = { + data: { + method: 'GET', + url: 'https://httpbin.org/get', + headers: '', + contentType: 'none', + body: '', + proxy: proxyUrl, + oauthService: 'None', + }, + }; + const output = await apiCall.process({}, config, agent); + const response = output.Response; + const headers = output.Headers; + + expect(headers['access-control-allow-credentials']).toEqual('true'); + expect(response.origin).toEqual(proxyHost); + }); + } + + const proxyUser = '{{KEY(WEBSHARE PROXY USERNAME)}}'; + const proxyPass = '{{KEY(WEBSHARE PROXY PASSWORD)}}'; + const proxyPort = 6712; + + const proxyUrls = { + http: `http://${proxyUser}:${proxyPass}@${proxyHost}:${proxyPort}`, + socks5: `socks5://${proxyUser}:${proxyPass}@${proxyHost}:${proxyPort}`, + multiple: `http://${proxyUser}:${proxyPass}@${proxyHost}:${proxyPort}\nsocks5://${proxyUser}:${proxyPass}@${proxyHost}:${proxyPort}`, + }; + + for (const [scheme, proxyUrl] of Object.entries(proxyUrls)) { + handleProxy(scheme, proxyUrl); + } +}); +*/ +describe('APICall Component - Error Handling', () => { + it('handle network errors', async () => { + const config = { + data: { + method: 'GET', + url: 'https://invalid.url', + headers: '', + contentType: 'none', + body: '', + oauthService: 'None', + }, + }; + const output = await apiCall.process({}, config, agent); + expect(output._error).toContain('ENOTFOUND'); + }); + + it('handle invalid URL errors', async () => { + const config = { + data: { + method: 'GET', + url: 'invalid-url', + headers: '', + contentType: 'none', + body: '', + }, + }; + const output = await apiCall.process({}, config, agent); + expect(output._error).toContain('Invalid URL'); + }); +}); diff --git a/packages/core/tests/unit/008-Components/APIEndpoint.test.ts b/packages/core/tests/unit/008-Components/APIEndpoint.test.ts new file mode 100644 index 000000000..981cbdd29 --- /dev/null +++ b/packages/core/tests/unit/008-Components/APIEndpoint.test.ts @@ -0,0 +1,166 @@ +import { AgentProcess } from '@sre/Core/AgentProcess.helper'; +import { describe, expect, it } from 'vitest'; +import { setupSRE } from '../../utils/sre'; +import { loadAgentData } from '../../utils/test-data-manager'; + +setupSRE(); +describe('APIEndpoint Component', () => { + it('APIEndpoint : nominal case', async () => { + let error; + try { + const data = loadAgentData('AgentData/general-unit-tests.smyth'); + const date = new Date(); + + const agentProcess = AgentProcess.load(data); + + let output = await agentProcess.run({ + method: 'POST', + path: '/api/test_strong_typing', + body: { + any: 'some data', + string: 'Hello world', + number: '1337.42', + integer: '42', + boolean: 'true', + array: '[1,2,3]', + object: '{"key":"value"}', + binary: 'https://smythos.com/wp-content/themes/generatepress_child/img/smythos-light.svg', + date: date.toISOString(), + }, + }); + + let outputBody = output?.data?.result?.body; + expect(outputBody).toBeDefined(); + + expect(outputBody?.string).toBeTypeOf('string'); + expect(outputBody?.number).toBeTypeOf('number'); + expect(Math.round(outputBody?.integer)).toEqual(42); + expect(outputBody?.boolean).toBeTypeOf('boolean'); + + expect(outputBody?.array).toBeInstanceOf(Array); + expect(outputBody?.array).toEqual([1, 2, 3]); + + expect(outputBody?.object).toBeInstanceOf(Object); + expect(outputBody?.binary?.url).toContain('smythfs://'); + expect(new Date(outputBody?.date)).toEqual(date); + } catch (e) { + error = e; + console.error(e.message); + } + expect(error).toBeUndefined(); + }); + + it('APIEndpoint with Array variation', async () => { + let error; + try { + const data = loadAgentData('AgentData/general-unit-tests.smyth'); + const date = new Date(); + + const agentProcess = AgentProcess.load(data); + + let output = await agentProcess.run({ + method: 'POST', + path: '/api/test_strong_typing', + body: { + any: 'some data', + array: 'a,b,c', + }, + }); + + let outputBody = output?.data?.result?.body; + expect(outputBody).toBeDefined(); + + expect(outputBody?.array).toBeInstanceOf(Array); + expect(outputBody?.array).toEqual(['a', 'b', 'c']); + } catch (e) { + error = e; + console.error(e.message); + } + expect(error).toBeUndefined(); + }); + + it('APIEndpoint : should detect invalid formats', async () => { + let error; + try { + const data = loadAgentData('AgentData/general-unit-tests.smyth'); + + const agentProcess = AgentProcess.load(data); + + const testCases = [ + { + fields: { number: 'abc' }, + message: 'Invalid number format', + }, + { + fields: { integer: 'abc' }, + message: 'Invalid integer format', + }, + { + fields: { boolean: 'abc' }, + message: 'Invalid boolean format', + }, + { + fields: { object: 'abc' }, + message: 'Invalid object format', + }, + { + fields: { date: 'abc' }, + message: 'Invalid date format', + }, + ]; + + for (let testCase of testCases) { + let output = await agentProcess.run({ + method: 'POST', + path: '/api/test_strong_typing', + body: { + ...testCase.fields, + }, + }); + agentProcess.reset(); + + if (!output?.data?.result?._error) { + console.log(testCase.message, testCase.fields); + console.log('Received', output?.data?.result?.body); + } + + expect(output?.data?.result?._error).toBeDefined(); + } + } catch (e) { + error = e; + console.error(e.message); + } + expect(error).toBeUndefined(); + }); + + it('APIEndpoint : default values', async () => { + let error; + try { + const data = loadAgentData('AgentData/general-unit-tests.smyth'); + + const agentProcess = AgentProcess.load(data); + + let output = await agentProcess.run({ + method: 'POST', + path: '/api/test_strong_typing_default_vals', + body: {}, + }); + + let outputBody = output?.data?.result?.body; + + console.log('>>>>>>', outputBody); + expect(outputBody.string).toBe('Hello world'); + expect(outputBody.number).toBe(123); + expect(outputBody.integer).toBe(1234); + expect(outputBody.boolean).toBe(true); + expect(outputBody.array).toEqual(['a', 'b', 'c', 'd']); + expect(outputBody.object).toEqual({ message: 'hello world' }); + expect(outputBody.binary.size).toEqual(9); + expect(outputBody.date).toEqual('2024-01-19T23:00:00.000Z'); + } catch (e) { + error = e; + console.error(e.message); + } + expect(error).toBeUndefined(); + }); +}); diff --git a/packages/core/tests/unit/008-Components/AgentPlugin.test.ts b/packages/core/tests/unit/008-Components/AgentPlugin.test.ts new file mode 100644 index 000000000..8a9bdb9bf --- /dev/null +++ b/packages/core/tests/unit/008-Components/AgentPlugin.test.ts @@ -0,0 +1,120 @@ +import { describe, expect, it } from 'vitest'; +import { AgentProcess } from '@sre/Core/AgentProcess.helper'; +import { Agent } from '@sre/AgentManager/Agent.class'; +import { AgentSettings } from '@sre/AgentManager/AgentSettings.class'; +import { AgentPlugin } from '@sre/Components/AgentPlugin.class'; +import { setupSRE } from '../../utils/sre'; +import { loadAgentData } from '../../utils/test-data-manager'; + +setupSRE(); + +// TODO [Forhad]: Need to add more test cases for AgentPlugin + +describe('AgentPlugin Component', () => { + it('runs a simple Agent Plugin with GET request', async () => { + let error; + try { + const data = loadAgentData('AgentData/sre-llm.smyth'); + + const agentProcess = AgentProcess.load(data); + + let res = await agentProcess.run({ + method: 'GET', + path: '/api/test-agent-plugin', + }); + + const output = res?.data?.result?.Response; + + expect(output).toBeDefined(); + expect(output?.length).toBeGreaterThan(20); + expect(output).toBeTypeOf('string'); + } catch (e) { + error = e; + console.error(e.message); + } + expect(error).toBeUndefined(); + }); + + it('runs a simple Agent Plugin with POST request', async () => { + let error; + try { + const data = loadAgentData('AgentData/sre-llm.smyth'); + + const agentProcess = AgentProcess.load(data); + + let res = await agentProcess.run({ + method: 'POST', + path: '/api/test-agent-plugin', + body: { + title: 'SmythOS - Design AI Agents with Drag & Drop Ease', + body: 'Seamlessly integrate AI, APIs, and data sources through our no-code platform. Just drag and drop. Simplify complexity, enhance control, and accelerate innovation — all in an afternoon.', + userId: 1, + }, + }); + + const output = res?.data?.result?.Response; + + expect(output).toBeDefined(); + expect(output?.length).toBeGreaterThan(20); + expect(output).toBeTypeOf('string'); + } catch (e) { + error = e; + console.error(e.message); + } + expect(error).toBeUndefined(); + }); + + it('test process function of AgentPlugin', async () => { + const input = { + Prompt: 'Which country is considered the middle of the world?', + }; + const subAgentId = 'clp1tl4tx00129tq5owb0kfxh'; + const config = { + id: '1', + name: 'AgentPlugin', + inputs: [ + { + name: 'Prompt', + type: 'Any', + color: '#F35063', + optional: false, + index: 0, + default: true, + }, + ], + data: { + model: 'gpt-4o-mini', + version: 'same-as-parent', + descForModel: + 'A dynamic agent that utilizes a POST API endpoint for interactions and generates prompts for effective communication with language models.', + agentId: subAgentId, + id: subAgentId, + name: 'Sub Agent', + desc: 'A dynamic agent that utilizes a POST API endpoint for interactions and generates prompts for effective communication with language models.', + }, + }; + + let error; + + try { + const agentId = 'clp1tnwli001h9tq56c9m6i7j'; + const agentSettings = new AgentSettings(agentId); + const data = loadAgentData('AgentData/parent-agent.smyth'); + const agent = new Agent(agentId, { data }, agentSettings); + + const agentPlugin = new AgentPlugin(); + + const result = await agentPlugin.process(input, config, agent); + const output = result?.Response; + + // The sub-agent has an Endpoint and a LLM Prompt component that echo "Tell the user that the system is busy and that he should retry later" + expect(output).toBeDefined(); + expect(output?.length).toBeGreaterThan(20); + expect(output).toBeTypeOf('string'); + } catch (e) { + error = e; + console.error(e.message); + } + expect(error).toBeUndefined(); + }); +}); diff --git a/packages/core/tests/unit/008-Components/Async-Await.test.ts b/packages/core/tests/unit/008-Components/Async-Await.test.ts new file mode 100644 index 000000000..7ef1586eb --- /dev/null +++ b/packages/core/tests/unit/008-Components/Async-Await.test.ts @@ -0,0 +1,84 @@ +import { Async } from '@sre/Components/Async.class'; +import { Await } from '@sre/Components/Await.class'; +import { AgentProcess } from '@sre/Core/AgentProcess.helper'; +import { describe, expect, it } from 'vitest'; +import { setupSRE } from '../../utils/sre'; +import { loadAgentData } from '../../utils/test-data-manager'; + +setupSRE(); + +describe('Async and Await Components', () => { + it('should not wait for a job to be done without await', async () => { + const data = loadAgentData('AgentData/async-await-foreach-tests.smyth'); + const date = new Date(); + + const agentProcess = AgentProcess.load(data); + const testJobTimeSec = 30_000; + + const start = process.hrtime(); + let output = await agentProcess.run({ + method: 'POST', + path: '/api/no-await-async', + body: { + prompt: 'Hello', + }, + }); + const end = process.hrtime(start); + const elapsedSeconds = Math.round((end[0] + end[1] / 1e9) * 1000) / 1000; + + expect(elapsedSeconds).toBeLessThan(testJobTimeSec); + expect(output.data).toEqual([]); + }); + + it('should resolve and wait for a job to be done with await', async () => { + const data = loadAgentData('AgentData/async-await-foreach-tests.smyth'); + + const agentProcess = AgentProcess.load(data); + const minTestJobTimeSec = 10; + + const start = process.hrtime(); + let output = await agentProcess.run({ + method: 'POST', + path: '/api/async-job', + body: { + prompt: 'Hello', + }, + }); + + const end = process.hrtime(start); + const elapsedSeconds = Math.round((end[0] + end[1] / 1e9) * 1000) / 1000; + + const results = Object.entries(output.data?.result?.Results || {}) as [string, any][]; + expect(results.length).toBe(1); + + const jobStatus = results[0][1]?.status; + expect(jobStatus).toBe('done'); + + expect(elapsedSeconds).toBeGreaterThanOrEqual(minTestJobTimeSec); + }); + + // TODO: implement For-Each component first to test this + it('should resolve and wait for multiple concurrent jobs with await', async () => { + const data = loadAgentData('AgentData/async-await-foreach-tests.smyth'); + const date = new Date(); + + const agentProcess = AgentProcess.load(data); + + let output = await agentProcess.run({ + method: 'POST', + path: '/api/for-each-async-job', + body: { + prompts: ['Hello', 'World', 'Foo', 'Bar'], + }, + }); + + console.log(output); + const results = output.data?.result?.Output?.results; + expect(results).toBeDefined(); + + const replies = Object.values(results).map((r: any) => r?.output?.result?.Reply); + expect(replies).toHaveLength(4); + + expect(replies).toEqual(['Hello', 'World', 'Foo', 'Bar']); + }); +}); diff --git a/packages/core/tests/unit/008-Components/Classifier.test.ts b/packages/core/tests/unit/008-Components/Classifier.test.ts new file mode 100644 index 000000000..89910c322 --- /dev/null +++ b/packages/core/tests/unit/008-Components/Classifier.test.ts @@ -0,0 +1,58 @@ +import { Agent } from '@sre/AgentManager/Agent.class'; +import { Classifier } from '@sre/Components/Classifier.class'; +import { ConnectorService } from '@sre/Core/ConnectorsService'; +import { describe, expect, it, vi } from 'vitest'; +import { setupSRE } from '../../utils/sre'; + +setupSRE(); + +// Mock Agent class to keep the test isolated from the actual Agent implementation +vi.mock('@sre/AgentManager/Agent.class', () => { + const MockedAgent = vi.fn().mockImplementation(() => ({ + id: 'agent-0000', + agentRuntime: { debug: true }, // used inside createComponentLogger() + isKilled: () => false, + modelsProvider: ConnectorService.getModelsProviderConnector(), + })); + return { Agent: MockedAgent }; +}); + +describe('Classifier Component', () => { + it('should correctly classify an input using one of the options', async () => { + // @ts-ignore + const agent = new Agent(); + const input = `I'm upset`; + const options = ['happy', 'sad', 'excited']; + const answer = 'sad'; + const classifier = new Classifier(); + + const output = await classifier.process( + { + Input: input, + }, + { + name: 'Classifier', + data: { + model: 'gpt-4o', + prompt: `Classify the input content to one of the categories. Set the selected category to true and the others to empty value`, + }, + outputs: options.map((option) => ({ + name: option, + description: '', + })), + }, + agent + ); + + expect(output).toBeDefined(); + // expect(output[answer]).toBe(true); + + for (let option of options) { + if (option === answer) { + expect(output[option]).toBeTruthy(); + } else { + expect(output[option]).toBeFalsy(); + } + } + }); +}); diff --git a/packages/core/tests/unit/008-Components/DataSourceCleaner.test.ts b/packages/core/tests/unit/008-Components/DataSourceCleaner.test.ts new file mode 100644 index 000000000..cca258a73 --- /dev/null +++ b/packages/core/tests/unit/008-Components/DataSourceCleaner.test.ts @@ -0,0 +1,113 @@ +import { faker } from '@faker-js/faker'; +import { DataSourceIndexer } from '@sre/Components/DataSourceIndexer.class'; +// import { VectorsHelper } from '@sre/IO/VectorDB.service/Vectors.helper'; +import { AccessCandidate } from '@sre/Security/AccessControl/AccessCandidate.class'; +import { Agent } from '@sre/AgentManager/Agent.class'; +import { AgentSettings } from '@sre/AgentManager/AgentSettings.class'; +import { ConnectorService } from '@sre/Core/ConnectorsService'; +import { describe, expect, it } from 'vitest'; +import crypto from 'crypto'; +import { DataSourceCleaner } from '@sre/Components/DataSourceCleaner.class'; +import { setupSRE } from '../../utils/sre'; +import { loadAgentData } from '../../utils/test-data-manager'; + +setupSRE({ + VectorDB: { + Connector: 'RAMVec', + Settings: { + embeddings: { + provider: 'OpenAI', + model: 'text-embedding-3-large', + params: { + dimensions: 1024, + }, + }, + }, + }, +}); + +describe('DataSourceCleaner Component', () => { + it( + 'deletes datasources created by DataSourceIndexer', + async () => { + const data = loadAgentData('AgentData/data-components.smyth'); + const date = new Date(); + + const agent = new Agent(10, data, new AgentSettings(10)); + agent.teamId = 'default'; + + const cleaner = new DataSourceCleaner(); + const indexer = new DataSourceIndexer(); + + // index some data using the connector + const namespace = faker.lorem.word(); + const vectorDbConnector = ConnectorService.getVectorDBConnector(); + await vectorDbConnector.user(AccessCandidate.team(agent.teamId)).createNamespace(namespace); + + const sourceText = ['What is the capital of France?', 'Paris']; + + const dynamic_id = crypto.randomBytes(16).toString('hex'); + + const res = await indexer.process( + { + Source: sourceText.join(' '), + }, + { + data: { + namespace, + id: dynamic_id, + name: faker.lorem.word(), + metadata: faker.lorem.sentence(), + }, + outputs: [], + }, + agent + ); + + // expect that the datasource file exists now + // const existsAfterInsert = await SmythFS.Instance.exists(dsUrl, AccessCandidate.team(agent.teamId)); + const id = res.Success?.id; + + expect(id).toBeDefined(); + + const dsBeforeDel = await vectorDbConnector + .user(AccessCandidate.team(agent.teamId)) + .getDatasource(namespace, DataSourceIndexer.genDsId(dynamic_id, agent.teamId, namespace)); + + expect(dsBeforeDel).toBeDefined(); + + await cleaner.process( + { + Source: sourceText.join(' '), + }, + { + data: { + namespaceId: namespace, + id: dynamic_id, + }, + outputs: [], + }, + agent + ); + + // expect that the datasource file does not exist now + // const existsAfterDelete = await SmythFS.Instance.exists(dsUrl, AccessCandidate.team(agent.teamId)); + + const dsAfterDel = await vectorDbConnector + .user(AccessCandidate.team(agent.teamId)) + .getDatasource(namespace, DataSourceIndexer.genDsId(dynamic_id, agent.teamId, namespace)); + + expect(dsAfterDel).toBeUndefined(); + + // expect that all the embeddings are deleted. we can do that by doing a similar search on the data we indexed + + const vectors = await vectorDbConnector.user(AccessCandidate.team(agent.teamId)).search(namespace, 'Paris'); + + expect(vectors).toBeDefined(); + expect(vectors.length).toBe(0); + }, + { + timeout: 35_000, + } + ); +}); diff --git a/packages/core/tests/unit/008-Components/DataSourceIndexer.test.ts b/packages/core/tests/unit/008-Components/DataSourceIndexer.test.ts new file mode 100644 index 000000000..1cf53c4e7 --- /dev/null +++ b/packages/core/tests/unit/008-Components/DataSourceIndexer.test.ts @@ -0,0 +1,195 @@ +import { faker } from '@faker-js/faker'; +import { DataSourceIndexer } from '@sre/Components/DataSourceIndexer.class'; +// import { VectorsHelper } from '@sre/IO/VectorDB.service/Vectors.helper'; +import { Agent } from '@sre/AgentManager/Agent.class'; +import { AgentSettings } from '@sre/AgentManager/AgentSettings.class'; +import { ConnectorService } from '@sre/Core/ConnectorsService'; +import { AccessCandidate } from '@sre/Security/AccessControl/AccessCandidate.class'; +import crypto from 'crypto'; +import { describe, expect, it } from 'vitest'; + +import { setupSRE } from '../../utils/sre'; +import { loadAgentData } from '../../utils/test-data-manager'; + +setupSRE({ + VectorDB: { + Connector: 'RAMVec', + Settings: { + embeddings: { + provider: 'OpenAI', + model: 'text-embedding-3-large', + params: { + dimensions: 1024, + }, + }, + }, + }, +}); +const EVENTUAL_CONSISTENCY_DELAY = 5_000; + +describe('DataSourceIndexer Component', () => { + it('inserts data on global storage', async () => { + const data = loadAgentData('AgentData/data-components.smyth'); + const date = new Date(); + + const agent = new Agent(10, data, new AgentSettings(10)); + agent.teamId = 'default'; + + const indexer = new DataSourceIndexer(); + + // index some data using the connector + const namespace = faker.lorem.word(); + const vectorDbConnector = ConnectorService.getVectorDBConnector(); + await vectorDbConnector.user(AccessCandidate.team(agent.teamId)).createNamespace(namespace); + + const sourceText = ['What is the capital of France?', 'Paris']; + + const dynamic_id = crypto.randomBytes(16).toString('hex'); + + await indexer.process( + { + Source: sourceText.join(' '), + dynamic_id, + }, + { + data: { + namespace, + name: 'Paris Datasource', + id: '{{dynamic_id}}', + metadata: 'Paris', + }, + outputs: [], + }, + agent + ); + + await new Promise((resolve) => setTimeout(resolve, EVENTUAL_CONSISTENCY_DELAY)); + + const vectors = await vectorDbConnector.user(AccessCandidate.team('default')).search(namespace, 'Paris'); + + expect(vectors).toBeDefined(); + expect(vectors.length).toBeGreaterThan(0); + + // expect(vectors[0].metadata).toBe('Paris'); + expect(vectors.some((result) => result.text?.includes('Paris'))).toBeTruthy(); + + // make sure that the datasource was created + + const ds = await vectorDbConnector + .user(AccessCandidate.team(agent.teamId)) + .getDatasource(namespace, DataSourceIndexer.genDsId(dynamic_id, agent.teamId, namespace)); + + expect(ds).toBeDefined(); + }); + + it('inserts data on non-existing namespace (implicitly creates it)', async () => { + const data = loadAgentData('AgentData/data-components.smyth'); + const date = new Date(); + + const agent = new Agent(10, data, new AgentSettings(10)); + agent.teamId = 'default'; + + const indexer = new DataSourceIndexer(); + + // index some data using the connector + const namespace = faker.lorem.word(); + const vectorDbConnector = ConnectorService.getVectorDBConnector(); + + const sourceText = ['What is the capital of France?', 'Paris']; + + const dynamic_id = crypto.randomBytes(16).toString('hex'); + + await indexer.process( + { + Source: sourceText.join(' '), + dynamic_id, + }, + { + data: { + namespace, + name: 'Paris Datasource', + id: '{{dynamic_id}}', + metadata: 'Paris', + }, + outputs: [], + }, + agent + ); + + await new Promise((resolve) => setTimeout(resolve, EVENTUAL_CONSISTENCY_DELAY)); + + const vectors = await vectorDbConnector.user(AccessCandidate.team('default')).search(namespace, 'Paris'); + + expect(vectors).toBeDefined(); + expect(vectors.length).toBeGreaterThan(0); + + // expect(vectors[0].metadata).toBe('Paris'); + expect(vectors.some((result) => result.text?.includes('Paris'))).toBeTruthy(); + + // make sure that the datasource was created + + const ds = await vectorDbConnector + .user(AccessCandidate.team(agent.teamId)) + .getDatasource(namespace, DataSourceIndexer.genDsId(dynamic_id, agent.teamId, namespace)); + + expect(ds).toBeDefined(); + }); + + it('inserts data on custom storage', async () => { + const data = loadAgentData('AgentData/data-components.smyth'); + const agent = new Agent(10, data, new AgentSettings(10)); + agent.teamId = 'default'; + + const indexer = new DataSourceIndexer(); + + // index some data using the connector + const namespace = faker.lorem.word(); + // const vectorDBHelper = await VectorsHelper.forTeam(agent.teamId); // load an instance that can access the custom storage (if it exists) + const vectorDbConnector = ConnectorService.getVectorDBConnector(); + await vectorDbConnector.team(agent.teamId).createNamespace(namespace); + + const sourceText = ['What is the capital of France?', 'Paris']; + + const dynamic_id = crypto.randomBytes(16).toString('hex'); + + await indexer.process( + { + Source: sourceText.join(' '), + dynamic_id, + }, + { + data: { + namespace, + name: 'Paris Datasource', + id: '{{dynamic_id}}', + metadata: 'Paris', + }, + outputs: [], + }, + agent + ); + + await new Promise((resolve) => setTimeout(resolve, EVENTUAL_CONSISTENCY_DELAY)); + + // make sure that the datasource was created + + const ds = await vectorDbConnector + .user(AccessCandidate.team(agent.teamId)) + .getDatasource(namespace, DataSourceIndexer.genDsId(dynamic_id, agent.teamId, namespace)); + expect(ds).toBeDefined(); + + const vectors = await vectorDbConnector.user(AccessCandidate.team(agent.teamId)).search(namespace, 'Paris'); + expect(vectors).toBeDefined(); + expect(vectors.length).toBeGreaterThan(0); + expect(vectors.some((result) => result.text?.includes('Paris'))).toBeTruthy(); + + const globalVectorDbConnector = ConnectorService.getVectorDBConnector(); + //* expect an error because we tried to access a namespace that exists on custom storage + const globalVectors = await globalVectorDbConnector + .user(AccessCandidate.team('different-team')) + .search(namespace, 'Paris') + .catch((e) => []); + expect(globalVectors).toBeDefined(); + expect(globalVectors.length).toBe(0); + }); +}); diff --git a/packages/core/tests/unit/008-Components/DataSourceLookup.test.ts b/packages/core/tests/unit/008-Components/DataSourceLookup.test.ts new file mode 100644 index 000000000..82fbe2816 --- /dev/null +++ b/packages/core/tests/unit/008-Components/DataSourceLookup.test.ts @@ -0,0 +1,267 @@ +import { faker } from '@faker-js/faker'; +import { DataSourceLookup } from '@sre/Components/DataSourceLookup.class'; +// import { VectorsHelper } from '@sre/IO/VectorDB.service/Vectors.helper'; +import { Agent } from '@sre/AgentManager/Agent.class'; +import { AgentSettings } from '@sre/AgentManager/AgentSettings.class'; +import { ConnectorService } from '@sre/Core/ConnectorsService'; +import { AccessCandidate } from '@sre/Security/AccessControl/AccessCandidate.class'; +import fs from 'fs'; +import { describe, expect, it } from 'vitest'; + +import { setupSRE } from '../../utils/sre'; +import { loadAgentData } from '../../utils/test-data-manager'; + +setupSRE({ + VectorDB: { + Connector: 'RAMVec', + Settings: { + embeddings: { + provider: 'OpenAI', + model: 'text-embedding-3-large', + params: { + dimensions: 1024, + }, + }, + }, + }, +}); +const EVENTUAL_CONSISTENCY_DELAY = 5_000; + +describe('DataSourceLookup Component', () => { + it('match similar data correctly', async () => { + let error; + const data = loadAgentData('AgentData/data-components.smyth'); + const date = new Date(); + + const agent = new Agent(10, data, new AgentSettings(10)); + agent.teamId = 'default'; + + const lookupComp = new DataSourceLookup(); + + // index some data using the connector + const namespace = faker.lorem.word(); + const vectorDbConnector = ConnectorService.getVectorDBConnector(); + + await vectorDbConnector.user(AccessCandidate.team(agent.teamId)).createNamespace(namespace); + + const sourceText = ['What is the capital of France?', 'Paris']; + + await vectorDbConnector.user(AccessCandidate.team('default')).createDatasource(namespace, { + text: sourceText.join(' '), + chunkSize: 1000, + chunkOverlap: 0, + metadata: { + text: 'Paris', + }, + }); + + await new Promise((resolve) => setTimeout(resolve, EVENTUAL_CONSISTENCY_DELAY)); + + const output = await lookupComp.process( + { + Query: sourceText[0], + }, + { + data: { + namespace, + postprocess: false, + prompt: '', + includeMetadata: false, + topK: 10, + }, + outputs: [], + }, + agent + ); + + const results = output.Results; + + expect(results).toBeDefined(); + expect(results.length).toBeGreaterThan(0); + expect(results.length).toBeLessThanOrEqual(10); + expect(results.some((result) => result.includes('Paris'))).toBeTruthy(); + + expect(output._error).toBeUndefined(); + + expect(error).toBeUndefined(); + }); + + it('include metadata', async () => { + let error; + const data = loadAgentData('AgentData/data-components.smyth'); + const date = new Date(); + + const agent = new Agent(10, data, new AgentSettings(10)); + agent.teamId = 'default'; + + const lookupComp = new DataSourceLookup(); + + // index some data using the connector + const namespace = faker.lorem.word(); + + const vectorDbConnector = ConnectorService.getVectorDBConnector(); + await vectorDbConnector.user(AccessCandidate.team(agent.teamId)).createNamespace(namespace); + const id = faker.lorem.word(); + const sourceText = ['What is the capital of France?', 'Paris']; + + // await vectorDbConnector.user(AccessCandidate.team(agent.teamId)).insert(namespace, { + // id, + // source: Array.from({ length: 1536 }, () => Math.floor(Math.random() * 100)), + // metadata: { + // user: VectorsHelper.stringifyMetadata({ + // text: 'Paris', + // meta2: 'meta2', + // }), + // }, + // }); + const text = 'Any matching text'; + await vectorDbConnector.user(AccessCandidate.team(agent.teamId)).createDatasource(namespace, { + id, + text, + metadata: { + text: 'Paris', + meta2: 'meta2', + }, + }); + + await new Promise((resolve) => setTimeout(resolve, EVENTUAL_CONSISTENCY_DELAY)); + + const output = await lookupComp.process( + { + Query: sourceText[0], + }, + { + data: { + namespace, + postprocess: false, + prompt: '', + includeMetadata: true, + topK: 10, + }, + outputs: [], + }, + agent + ); + + const results = output.Results; + + expect(results).toBeDefined(); + expect(results.length).toBeGreaterThan(0); + expect(results[0]).not.toBeTypeOf('string'); + expect(results[0]).toBeTypeOf('object'); + + expect(results.some((result) => result.metadata.text === 'Paris')).toBeTruthy(); + expect(results.some((result) => result.metadata.meta2 === 'meta2')).toBeTruthy(); + + expect(output._error).toBeUndefined(); + + expect(error).toBeUndefined; + }); + + it('lookup data in custom storage', async () => { + let error; + + const data = loadAgentData('AgentData/data-components.smyth'); + const date = new Date(); + + const agent = new Agent(10, data, new AgentSettings(10)); + agent.teamId = 'default'; + + const lookupComp = new DataSourceLookup(); + + const namespace = faker.lorem.word(); + // const vectorDbHelper = await VectorsHelper.forTeam(agent.teamId); + const vectorDbConnector = ConnectorService.getVectorDBConnector(); + await vectorDbConnector.user(AccessCandidate.team(agent.teamId)).createNamespace(namespace); + const id = faker.lorem.word(); + const sourceText = ['What is the capital of France?', 'Paris']; + + await vectorDbConnector.user(AccessCandidate.team('default')).createDatasource(namespace, { + text: sourceText.join(' '), + chunkSize: 1000, + chunkOverlap: 0, + metadata: { + text: 'Paris', + }, + }); + + await new Promise((resolve) => setTimeout(resolve, EVENTUAL_CONSISTENCY_DELAY)); + + const output = await lookupComp.process( + { + Query: sourceText[0], + }, + { + data: { + namespace, + postprocess: false, + prompt: '', + includeMetadata: false, + topK: 10, + }, + outputs: [], + }, + agent + ); + + const results = output.Results; + + expect(results).toBeDefined(); + expect(results.length).toBeGreaterThan(0); + expect(results.some((result) => result.includes('Paris'))).toBeTruthy(); + }); + + // it('postprocess data', async () => { + // let error; + // try { + // const agentData = fs.readFileSync('./tests/data/data-components.smyth', 'utf-8'); + // const data = JSON.parse(agentData); + // const date = new Date(); + + // const agent = new Agent(10, data, new AgentSettings(10)); + + // const lookupComp = new DataSourceLookup(); + + // // index some data using the connector + // const namespace = faker.lorem.word(); + + // const sourceText = ['What is the capital of France?', 'Paris']; + + // await VectorsHelper.load().ingestText(sourceText.join(' '), namespace, { + // teamId: agent.teamId, + // chunkSize: 1000, + // chunkOverlap: 0, + // metadata: { + // text: 'Paris', + // }, + // }); + + // await new Promise((resolve) => setTimeout(resolve, EVENTUAL_CONSISTENCY_DELAY)); + + // const output = await lookupComp.process( + // { + // Query: 'What is the capital of France?', + // }, + // { + // namespace, + // postprocess: true, + // includeMetadata: true, + // model: 'gpt-3.5-turbo', + // prompt: 'What is the capital of {{result}}?', + // }, + // agent + // ); + + // const results = output.Results; + + // expect(results).toBeDefined(); + // expect(results.length).toBeGreaterThan(0); + + // expect(output._error).toBeUndefined(); + // } catch (e) { + // error = e; + // console.error(e.message); + // } + // expect(error).toBeUndefined(); + // }); +}); diff --git a/packages/core/tests/unit/008-Components/ECMASandbox.test.ts b/packages/core/tests/unit/008-Components/ECMASandbox.test.ts new file mode 100644 index 000000000..bccd4bf75 --- /dev/null +++ b/packages/core/tests/unit/008-Components/ECMASandbox.test.ts @@ -0,0 +1,144 @@ +import { Agent } from '@sre/AgentManager/Agent.class'; +import { AgentProcess } from '@sre/Core/AgentProcess.helper'; +import { describe, expect, it, vi } from 'vitest'; +import { ECMASandbox } from '@sre/Components/ECMASandbox.class'; +import { setupSRE } from '../../utils/sre'; +import { ConnectorService } from '@sre/Core/ConnectorsService'; + +setupSRE({ + Code: { + Connector: 'ECMASandbox', + }, +}); + +// Mock Agent class to keep the test isolated from the actual Agent implementation +vi.mock('@sre/AgentManager/Agent.class', () => { + const MockedAgent = vi.fn().mockImplementation(() => ({ + id: 'agent-123456', + agentRuntime: { debug: true }, // used inside createComponentLogger() + teamId: 'Team2', + isKilled: () => false, + modelsProvider: ConnectorService.getModelsProviderConnector(), + })); + return { Agent: MockedAgent }; +}); + +describe('Code Component', () => { + it('runs code without vars', async () => { + const code = ` + async function main() { + const a = 1; + const b = 2; + const c = a + b; + return { c }; + } + `; + + const agent: any = new (Agent as any)(); + + const codeComp = new ECMASandbox(); + const output = await codeComp.process( + {}, + { + data: { + code, + }, + }, + agent + ); + + const result = output.Output; + + expect(result).toBeDefined(); + expect(result.c).toBe(3); + }); + + it('runs code with vars', async () => { + const bo = true; + const num = 1; + const str = 'Hello World!'; + const letterObj = '{ a: 1, b: 2, c: 3 }'; + const numArr = '[1, 2, 3]'; + + const code = ` + async function main(bo, num, str, letterObj, numArr) { + return { bo, num, str, letterObj, numArr }; + } + `; + + const agent: any = new (Agent as any)(); + + const codeComp = new ECMASandbox(); + const output = await codeComp.process( + { bo, num, str, letterObj, numArr }, // inputs + { + data: { + code, + }, + }, + agent + ); + + const result = output.Output; + + expect(result).toBeDefined(); + expect(result.bo).toBe(bo); + expect(result.num).toBe(num); + expect(result.str).toBe(str); + expect(result.letterObj).toStrictEqual(letterObj); + expect(result.numArr).toStrictEqual(numArr); + }); + + it("rejects code with 'require' statement", async () => { + const code = ` + const fs = require('fs'); + async function main() { + return { fs }; + } + `; + + const agent: any = new (Agent as any)(); + + const codeComp = new ECMASandbox(); + const output = await codeComp.process( + {}, + { + data: { + code, + }, + }, + agent + ); + + const result = output.Output; + + expect(result).toBeUndefined(); + expect(output._error).toBeDefined(); + }); + + // it('rejects code with infinite loop', async () => { + // const code = ` + // async function main() { + // while (true) {} + // } + // `; + + // const agent: any = new (Agent as any)(); + + // const codeComp = new ECMASandbox(); + // const output = await codeComp.process( + // {}, + // { + // data: { + // code, + // }, + // }, + // agent + // ); + + // const result = output.Output; + + // expect(result).toBeUndefined(); + // expect(output._error).toBeDefined(); + // }); +}); diff --git a/packages/core/tests/unit/008-Components/FEncDec.test.ts b/packages/core/tests/unit/008-Components/FEncDec.test.ts new file mode 100644 index 000000000..5500a258a --- /dev/null +++ b/packages/core/tests/unit/008-Components/FEncDec.test.ts @@ -0,0 +1,48 @@ +import { FEncDec } from '@sre/Components/FEncDec.class'; +import { Agent } from '@sre/AgentManager/Agent.class'; +import { AgentSettings } from '@sre/AgentManager/AgentSettings.class'; +import { describe, expect, it } from 'vitest'; +import { setupSRE } from '../../utils/sre'; +import { loadAgentData } from '../../utils/test-data-manager'; + +setupSRE(); + +describe('FEncDec Component', () => { + it('encodes data', async () => { + let error; + try { + const data = loadAgentData('AgentData/functions-components.smyth'); + const date = new Date(); + + const agent = new Agent(10, data, new AgentSettings(10)); + + const fEncDec = new FEncDec(); + const decodedData = 'Hello World'; + const encodeOutput = await fEncDec.process({ Data: decodedData }, { data: { action: 'Encode', encoding: 'hex' } }, agent); + expect(encodeOutput.Output).toBe(Buffer.from(decodedData).toString('hex')); + } catch (e) { + error = e; + console.error(e.message); + } + expect(error).toBeUndefined(); + }); + + it('decodes data', async () => { + let error; + try { + const data = loadAgentData('AgentData/functions-components.smyth'); + const date = new Date(); + + const agent = new Agent(10, data, new AgentSettings(10)); + + const fEncDec = new FEncDec(); + const encodedData = Buffer.from('Hello World').toString('hex'); + const decodeOutput = await fEncDec.process({ Data: encodedData }, { data: { action: 'Decode', encoding: 'hex' } }, agent); + expect(decodeOutput.Output).toBe(Buffer.from(encodedData, 'hex').toString('utf8')); + } catch (e) { + error = e; + console.error(e.message); + } + expect(error).toBeUndefined(); + }); +}); diff --git a/packages/core/tests/unit/008-Components/FHash.test.ts b/packages/core/tests/unit/008-Components/FHash.test.ts new file mode 100644 index 000000000..8b082656d --- /dev/null +++ b/packages/core/tests/unit/008-Components/FHash.test.ts @@ -0,0 +1,60 @@ +import { FHash } from '@sre/Components/FHash.class'; +import { Agent } from '@sre/AgentManager/Agent.class'; +import { AgentSettings } from '@sre/AgentManager/AgentSettings.class'; +import { loadAgentData } from '../../utils/test-data-manager'; +import { setupSRE } from '../../utils/sre'; +import crypto from 'crypto'; + +import { describe, expect, it } from 'vitest'; + +setupSRE(); + +describe('FHash Component', () => { + it('generate correct md5 hash', async () => { + let error; + try { + const data = loadAgentData('AgentData/functions-components.smyth'); + const date = new Date(); + + const agent = new Agent(10, data, new AgentSettings(10)); + + const fHash = new FHash(); + const dataToHash = 'Hello World'; + const output = await fHash.process({ Data: dataToHash }, { data: { algorithm: 'md5', encoding: 'hex' } }, agent); + const expectedHash = crypto.createHash('md5').update(dataToHash).digest('hex'); + expect(output.Hash).toBe(expectedHash); + + console.log(output); + + // agent should wait for 10 seconds + } catch (e) { + error = e; + console.error(e.message); + } + expect(error).toBeUndefined(); + }); + + it('generate correct sha256 hash', async () => { + let error; + try { + const data = loadAgentData('AgentData/functions-components.smyth'); + const date = new Date(); + + const agent = new Agent(10, data, new AgentSettings(10)); + + const fHash = new FHash(); + const dataToHash = 'Hello World'; + const output = await fHash.process({ Data: dataToHash }, { data: { algorithm: 'sha256', encoding: 'hex' } }, agent); + const expectedHash = crypto.createHash('sha256').update(dataToHash).digest('hex'); + expect(output.Hash).toBe(expectedHash); + + console.log(output); + + // agent should wait for 10 seconds + } catch (e) { + error = e; + console.error(e.message); + } + expect(error).toBeUndefined(); + }); +}); diff --git a/packages/core/tests/unit/008-Components/FSign.test.ts b/packages/core/tests/unit/008-Components/FSign.test.ts new file mode 100644 index 000000000..2101fa82d --- /dev/null +++ b/packages/core/tests/unit/008-Components/FSign.test.ts @@ -0,0 +1,125 @@ +import { FSign } from '@sre/Components/FSign.class'; +import { Agent } from '@sre/AgentManager/Agent.class'; +import { AgentSettings } from '@sre/AgentManager/AgentSettings.class'; +import { loadAgentData } from '../../utils/test-data-manager'; +import { setupSRE } from '../../utils/sre'; +import crypto from 'crypto'; + +import { describe, expect, it, vi, beforeEach } from 'vitest'; +import * as tempStringHelpers from '@sre/helpers/TemplateString.helper'; + +setupSRE(); + +beforeEach(() => { + // reset all mocks + vi.restoreAllMocks(); +}); + +describe('FSign Component', () => { + it('should sign data using passed key', async () => { + let error; + try { + const data = loadAgentData('AgentData/functions-components.smyth'); + const date = new Date(); + + const agent = new Agent(10, data, new AgentSettings(10)); + + const fsign = new FSign(); + const signingKey = crypto.randomBytes(16).toString('hex'); + const dataToSign = { test: 'data to sign' }; + const output = await fsign.process( + { Data: dataToSign, Key: signingKey }, + { + data: { + signMethod: 'HMAC', + encoding: 'hex', + dataTransform: 'Stringify', + hashType: 'md5', + }, + }, + agent + ); + + expect(output.Signature).toBeDefined(); + + const hmac = crypto.createHmac('md5', signingKey); + hmac.update(JSON.stringify(dataToSign)); + const expected = hmac.digest('hex' as crypto.BinaryToTextEncoding); + + expect(output.Signature).toBe(expected); + } catch (e) { + error = e; + console.error(e.message); + } + expect(error).toBeUndefined(); + }); + + it('should sign data using vault key', async () => { + let error; + try { + const data = loadAgentData('AgentData/functions-components.smyth'); + const date = new Date(); + + // const agentProcess = AgentProcess.load(data); + + // const + // let output = await agentProcess.run({ + // method: 'POST', + // path: '/api/sleep_10', + // body: {}, + // }); + + // let outputResult = output?.result; + + const agent = new Agent(10, data, new AgentSettings(10)); + + const fsign = new FSign(); + const dataToSign = { test: 'data to sign' }; + const vaultKey = '{{KEY(VAULT_KEY)}}'; + const vaultSigningKey = '0'.repeat(32); + + //@ts-ignore + const spy = vi.spyOn(tempStringHelpers, 'TemplateString').mockImplementation((templateString: string) => { + return { + parse: () => { + return { + parseTeamKeysAsync: () => { + return { + asyncResult: vaultSigningKey, + }; + }, + }; + }, + }; + }); + + const output = await fsign.process( + { Data: dataToSign }, + { + data: { + key: vaultKey, + signMethod: 'HMAC', + encoding: 'hex', + dataTransform: 'Stringify', + hashType: 'md5', + }, + }, + agent + ); + + expect(output.Signature).toBeDefined(); + + expect(spy).toHaveBeenCalledWith(vaultKey); + + const hmac = crypto.createHmac('md5', vaultSigningKey); + hmac.update(JSON.stringify(dataToSign)); + const expected = hmac.digest('hex' as crypto.BinaryToTextEncoding); + + expect(output.Signature).toBe(expected); + } catch (e) { + error = e; + console.error(e.message); + } + expect(error).toBeUndefined(); + }); +}); diff --git a/packages/core/tests/unit/008-Components/FSleep.test.ts b/packages/core/tests/unit/008-Components/FSleep.test.ts new file mode 100644 index 000000000..5c1ea9b4b --- /dev/null +++ b/packages/core/tests/unit/008-Components/FSleep.test.ts @@ -0,0 +1,47 @@ +import { FSleep } from '@sre/Components/FSleep.class'; +import { Agent } from '@sre/AgentManager/Agent.class'; +import { AgentSettings } from '@sre/AgentManager/AgentSettings.class'; +import { describe, expect, it } from 'vitest'; +import { setupSRE } from '../../utils/sre'; +import { loadAgentData } from '../../utils/test-data-manager'; + +setupSRE(); + +describe('FSleep Component', () => { + it('agent should wait until sleep duration finishes', async () => { + let error; + try { + const data = loadAgentData('AgentData/functions-components.smyth'); + const date = new Date(); + + // const agentProcess = AgentProcess.load(data); + + // const + // let output = await agentProcess.run({ + // method: 'POST', + // path: '/api/sleep_10', + // body: {}, + // }); + + // let outputResult = output?.result; + + const agent = new Agent(10, data, new AgentSettings(10)); + + const fSleepComponent = new FSleep(); + const start = process.hrtime(); + const output = await fSleepComponent.process({}, { name: 'sleep', data: { delay: 3 } }, agent); + const end = process.hrtime(start); + const durationSec = end[0] + end[1] / 1e9; + + expect(durationSec).toBeGreaterThanOrEqual(3); + + console.log(output); + + // agent should wait for 10 seconds + } catch (e) { + error = e; + console.error(e.message); + } + expect(error).toBeUndefined(); + }); +}); diff --git a/packages/core/tests/unit/components/FTimestamp.test.ts b/packages/core/tests/unit/008-Components/FTimestamp.new.test.ts similarity index 100% rename from packages/core/tests/unit/components/FTimestamp.test.ts rename to packages/core/tests/unit/008-Components/FTimestamp.new.test.ts diff --git a/packages/core/tests/unit/008-Components/FTimestamp.test.ts b/packages/core/tests/unit/008-Components/FTimestamp.test.ts new file mode 100644 index 000000000..d077b665f --- /dev/null +++ b/packages/core/tests/unit/008-Components/FTimestamp.test.ts @@ -0,0 +1,115 @@ +import { AgentProcess } from '@sre/Core/AgentProcess.helper'; +import { setupSRE } from '../../utils/sre'; +import { loadAgentData } from '../../utils/test-data-manager'; +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { FTimestamp } from '../../../src/Components/FTimestamp.class'; +import { IAgent } from '../../../src/types/Agent.types'; + +setupSRE(); + +const mockAgent: IAgent = { + id: 'test-agent-id', + agentRuntime: { debug: false }, + sse: null, + isKilled: vi.fn(() => false), +} as any; +const mockTimestamp = 1642780800000; // January 21, 2022 12:00:00 UTC +describe('FTimestamp Component', () => { + let component: FTimestamp; + let originalDateNow: () => number; + + beforeEach(() => { + component = new FTimestamp(); + originalDateNow = Date.now; + Date.now = vi.fn(() => mockTimestamp); + }); + + afterEach(() => { + Date.now = originalDateNow; + }); + + it('should return unix timestamp by default', async () => { + const result = await component.process({}, { data: {}, name: 'test' }, mockAgent); + + expect(result.Timestamp).toBe(mockTimestamp); + expect(result._error).toBeUndefined(); + expect(typeof result.Timestamp).toBe('number'); + }); + + it('should return unix timestamp when format is "unix"', async () => { + const config = { data: { format: 'unix' }, name: 'test' }; + const result = await component.process({}, config, mockAgent); + + expect(result.Timestamp).toBe(mockTimestamp); + expect(typeof result.Timestamp).toBe('number'); + }); + + it('should return ISO string when format is "iso"', async () => { + const config = { data: { format: 'iso' }, name: 'test' }; + const result = await component.process({}, config, mockAgent); + + expect(typeof result.Timestamp).toBe('string'); + expect(typeof result.Timestamp).toBe('string'); + }); + + it('should format using custom dayjs patterns', async () => { + const config = { data: { format: 'YYYY-MM-DD' }, name: 'test' }; + const result = await component.process({}, config, mockAgent); + + expect(typeof result.Timestamp).toBe('string'); + expect(result.Timestamp).toMatch(/^\d{4}-\d{2}-\d{2}$/); + }); + + it('should include standard component output properties', async () => { + const result = await component.process({}, { data: {}, name: 'test' }, mockAgent); + + expect(result).toHaveProperty('Timestamp'); + expect(result).toHaveProperty('_error'); + expect(result).toHaveProperty('_debug'); + expect(result).toHaveProperty('_debug_time'); + }); + + it('should return error for invalid custom format strings', async () => { + const config = { data: { format: 'INVALID_FORMAT' }, name: 'test' }; + const result = await component.process({}, config, mockAgent); + + expect(result._error).toBeDefined(); + }); + + it('should handle null format gracefully', async () => { + const invalidConfig = { data: { format: null }, name: 'test' }; + const result = await component.process({}, invalidConfig, mockAgent); + + // Should fallback to default behavior since null becomes 'unix' + expect(result.Timestamp).toBeDefined(); + expect(result._error).toBeUndefined(); + }); + + it('Call from agent (.smyth file)', async () => { + let error; + try { + const data = loadAgentData('AgentData/functions-components.smyth'); + const date = new Date(); + + const agentProcess = AgentProcess.load(data); + + let output = await agentProcess.run({ + method: 'POST', + path: '/api/timestamp', + body: {}, + }); + + let outputResult = output?.data?.result; + expect(outputResult).toBeDefined(); + + expect(outputResult?.Timestamp).toBeDefined(); + expect(outputResult?.Timestamp).toEqual(mockTimestamp); // Timestamp should be greater than current time + + expect(outputResult?._error).toBeUndefined(); + } catch (e) { + error = e; + console.error(e.message); + } + expect(error).toBeUndefined(); + }); +}); diff --git a/packages/core/tests/unit/008-Components/ForEach.test.ts b/packages/core/tests/unit/008-Components/ForEach.test.ts new file mode 100644 index 000000000..d07598743 --- /dev/null +++ b/packages/core/tests/unit/008-Components/ForEach.test.ts @@ -0,0 +1,29 @@ +import { AgentProcess } from '@sre/Core/AgentProcess.helper'; +import { describe, expect, it } from 'vitest'; +import { setupSRE } from '../../utils/sre'; +import { loadAgentData } from '../../utils/test-data-manager'; + +setupSRE(); + +describe('ForEach Component', () => { + it('should process input array', async () => { + const data = loadAgentData('AgentData/async-await-foreach-tests.smyth'); + const date = new Date(); + + const agentProcess = AgentProcess.load(data); + + let output = await agentProcess.run({ + method: 'POST', + path: '/api/for-each-job', + body: { + prompts: ['Hello', 'World', 'Foo', 'Bar'], + }, + }); + + const results = output.data?.result?.Output?.results; + + expect(results).toHaveLength(4); + + expect(results).toEqual(['Hello', 'World', 'Foo', 'Bar']); + }); +}); diff --git a/packages/core/tests/unit/008-Components/JSONFilter.test.ts b/packages/core/tests/unit/008-Components/JSONFilter.test.ts new file mode 100644 index 000000000..4b62fb424 --- /dev/null +++ b/packages/core/tests/unit/008-Components/JSONFilter.test.ts @@ -0,0 +1,215 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; + +import { JSONFilter } from '@sre/Components/JSONFilter.class'; + +import { Agent } from '@sre/AgentManager/Agent.class'; +import { setupSRE } from '../../utils/sre'; +import { ConnectorService } from '@sre/Core/ConnectorsService'; + +setupSRE(); + +// Mock Agent class to keep the test isolated from the actual Agent implementation +vi.mock('@sre/AgentManager/Agent.class', () => { + const MockedAgent = vi.fn().mockImplementation(() => ({ + id: 'agent-0000', + agentRuntime: { debug: true }, // used inside createComponentLogger() + isKilled: () => false, + modelsProvider: ConnectorService.getModelsProviderConnector(), + })); + return { Agent: MockedAgent }; +}); + +describe('jsonFilter: filter some specific properties from a JSON Object', () => { + let agent; + let jsonFilter; + let input; + let config; + + beforeEach(() => { + // @ts-ignore (Ignore required arguments, as we are using the mocked Agent) + agent = new Agent(); + jsonFilter = new JSONFilter(); + input = { + id: 1, + name: 'John Doe', + email: 'johndoe@example.com', + profile: { + age: 30, + gender: 'male', + location: { + city: 'New York', + state: 'NY', + country: 'USA', + }, + }, + }; + config = { + id: 1, + name: 'JSONFilter', + data: { + fields: 'name,email', + }, + }; + }); + + afterEach(() => { + // ⚠️ Warning from vitest doc (https://vitest.dev/guide/mocking#mocking) - "Always remember to clear or restore mocks before or after each test run to undo mock state changes between runs!" + vi.clearAllMocks(); + }); + + // Test case: Filtering with top level fields + it('should return object with top level properties', async () => { + let error; + + try { + const expected = { + name: 'John Doe', + email: 'johndoe@example.com', + }; + const result = await jsonFilter.process({ Input: input }, config, agent); + + expect(result.Output).toEqual(expected); + } catch (e) { + error = e; + console.error(e.message); + } + + expect(error).toBeUndefined(); + }); + + // Test case: Filtering nested properties + it('should correctly filter nested properties', async () => { + let error; + try { + config.data.fields = 'name,email,profile,location,city,state'; + + const expected = { + name: 'John Doe', + email: 'johndoe@example.com', + profile: { + location: { + city: 'New York', + state: 'NY', + }, + }, + }; + + const result = await jsonFilter.process({ Input: input }, config, agent); + + expect(result.Output).toEqual(expected); + } catch (e) { + error = e; + console.error(e.message); + } + + expect(error).toBeUndefined(); + }); + + // Test case: Filtering with missing properties + it('should handle missing properties gracefully', async () => { + let error; + try { + delete input.email; + delete input.profile.gender; + delete input.profile.location; + + config.data.fields = 'name,email,profile,location,city,state'; + + const expected = { + name: 'John Doe', + profile: {}, // TODO: It could be improved by removing empty parent objects from the `Output` when nested properties are missing. + }; + + const result = await jsonFilter.process({ Input: input }, config, agent); + + // Expected to only return the name, as email is missing + expect(result.Output).toEqual(expected); + } catch (e) { + error = e; + console.error(e.message); + } + + expect(error).toBeUndefined(); + }); + + // Test case: Filtering with empty input + it('should return an empty object when input is empty', async () => { + let error; + try { + const expected = {}; + + const result = await jsonFilter.process({ Input: {} }, config, agent); + + expect(result.Output).toEqual(expected); + } catch (e) { + error = e; + console.error(e.message); + } + expect(error).toBeUndefined(); + }); + + it('should filter fields in an array', async () => { + const _input = { + Input: [ + { id: 1, name: 'John Doe' }, + { email: 'johndoe@example.com', phone: '+1826234343' }, + ], + }; + const expected = [{ name: 'John Doe' }, { email: 'johndoe@example.com' }]; + + const result = await jsonFilter.process(_input, config, agent); + + expect(result.Output).toEqual(expected); + }); + + it('should return the input if it is not an object or array', async () => { + const _input = { Input: 'string' }; + const expected = 'string'; + + const result = await jsonFilter.process(_input, config, agent); + + expect(result.Output).toEqual(expected); + }); + + // Test case: Empty config.data.fields + it('should handle empty config.data.fields gracefully', async () => { + const _input = { Input: {} }; + + config.data.fields = ''; + + const expected = {}; + + const result = await jsonFilter.process(_input, config, agent); + + expect(result.Output).toEqual(expected); + }); + + it('should return Output as null if the Input is null', async () => { + const _input = { Input: null }; + const expected = null; + + const result = await jsonFilter.process(_input, config, agent); + + expect(result.Output).toEqual(expected); + }); + + // Test case: throw error for null input + it('should throw an Error if the Input is null', async () => { + await expect(jsonFilter.process(null, config, agent)).rejects.toThrowError(); + }); + + // Test case: Missing config.data.fields + it('should handle missing config.data.fields gracefully', async () => { + const _input = { Input: {} }; + + delete config.data.fields; + + const expected = { + Output: {}, + _error: null, + _debug: agent?.agentRuntime?.debug ? expect.anything() : undefined, + }; + + await expect(jsonFilter.process(_input, config, agent)).resolves.toEqual(expect.objectContaining(expected)); + }); +}); diff --git a/packages/core/tests/unit/008-Components/LogicAND.test.ts b/packages/core/tests/unit/008-Components/LogicAND.test.ts new file mode 100644 index 000000000..fa40211ca --- /dev/null +++ b/packages/core/tests/unit/008-Components/LogicAND.test.ts @@ -0,0 +1,99 @@ +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; +import { LogicAND } from '@sre/Components/LogicAND.class'; +import { Agent } from '@sre/AgentManager/Agent.class'; +import { ConnectorService } from '@sre/Core/ConnectorsService'; +import { setupSRE } from '../../utils/sre'; + +setupSRE(); + +// Mock Agent class to keep the test isolated from the actual Agent implementation +vi.mock('@sre/AgentManager/Agent.class', () => { + const MockedAgent = vi.fn().mockImplementation(() => ({ + id: 'agent-0000', + agentRuntime: { debug: true }, // used inside createComponentLogger() + isKilled: () => false, + modelsProvider: ConnectorService.getModelsProviderConnector(), + })); + return { Agent: MockedAgent }; +}); + +describe('LogicAND: process function', () => { + let logicAND: LogicAND; + let agent: Agent; + let config: any; + + beforeEach(() => { + // @ts-ignore (Ignore required arguments, as we are using the mocked Agent) + agent = new Agent(); + logicAND = new LogicAND(); + config = { + name: 'LogicAND', + inputs: [], + }; + }); + + afterEach(() => { + // ⚠️ Warning from vitest doc (https://vitest.dev/guide/mocking#mocking) - "Always remember to clear or restore mocks before or after each test run to undo mock state changes between runs!" + vi.clearAllMocks(); + }); + + it('should return `Output` and `Verified` as `true` when all expected inputs are provided', async () => { + const input = { a: true, b: 'string', c: {}, d: [] }; + + config.inputs = [{ name: 'a' }, { name: 'b' }, { name: 'c' }, { name: 'd' }]; + + const expected = { + Output: true, + Verified: true, + }; + + const result = await logicAND.process(input, config, agent); + + expect(result).toEqual(expected); + }); + + it('should return `Output` as `undefined` and `Unverified` as `true` when some expected inputs are missing', async () => { + const input = { a: true }; + + config.inputs = [{ name: 'a' }, { name: 'b' }]; + + const expected = { + Output: undefined, + Unverified: true, + }; + + const result = await logicAND.process(input, config, agent); + + expect(result).toEqual(expected); + }); + + it('should return `Output` as `undefined` and `Unverified` as `true` when no inputs are provided', async () => { + const input = {}; + + config.inputs = [{ name: 'a' }, { name: 'b' }]; + + const expected = { + Output: undefined, + Unverified: true, + }; + + const result = await logicAND.process(input, config, agent); + + expect(result).toEqual(expected); + }); + + it('should return `Output` as `true` and `Verified` as `true` when both input and config.inputs are empty', async () => { + const input = {}; + + config.inputs = []; + + const expected = { + Output: true, + Verified: true, + }; + + const result = await logicAND.process(input, config, agent); + + expect(result).toEqual(expected); + }); +}); diff --git a/packages/core/tests/unit/008-Components/LogicAtLeast.test.ts b/packages/core/tests/unit/008-Components/LogicAtLeast.test.ts new file mode 100644 index 000000000..0dcd95285 --- /dev/null +++ b/packages/core/tests/unit/008-Components/LogicAtLeast.test.ts @@ -0,0 +1,153 @@ +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; +import { LogicAtLeast } from '@sre/Components/LogicAtLeast.class'; +import { Agent } from '@sre/AgentManager/Agent.class'; +import { ConnectorService } from '@sre/Core/ConnectorsService'; +import { setupSRE } from '../../utils/sre'; + +setupSRE(); + +// Mock Agent class to keep the test isolated from the actual Agent implementation +vi.mock('@sre/AgentManager/Agent.class', () => { + const MockedAgent = vi.fn().mockImplementation(() => ({ + id: 'agent-0000', + agentRuntime: { debug: true }, // used inside createComponentLogger() + isKilled: () => false, + modelsProvider: ConnectorService.getModelsProviderConnector(), + })); + return { Agent: MockedAgent }; +}); + +describe('LogicAtLeast: process function', () => { + let logicAtLeast: LogicAtLeast; + let agent: Agent; + let config: any; + + beforeEach(() => { + // @ts-ignore (Ignore required arguments, as we are using the mocked Agent) + agent = new Agent(); + logicAtLeast = new LogicAtLeast(); + config = { + name: 'LogicAtLeast', + inputs: [], + data: { minSetInputs: '2' }, // TODO: minSetInputs must be string, maybe need to adjust this behavior in the process function + }; + }); + + afterEach(() => { + // ⚠️ Warning from vitest doc (https://vitest.dev/guide/mocking#mocking) - "Always remember to clear or restore mocks before or after each test run to undo mock state changes between runs!" + vi.clearAllMocks(); + }); + + it('should return `Output` as `true` and `Verified` as `true` when at least minSetInputs are truthy', async () => { + const input = { a: true, b: true, c: '', d: false }; + + config.inputs = [{ name: 'a' }, { name: 'b' }, { name: 'c' }, { name: 'd' }]; + + const expected = { Output: true, Verified: true }; + + const result = await logicAtLeast.process(input, config, agent); + + expect(result).toEqual(expected); + }); + + it('should return `Output` as `undefined` and `Unverified` as `true` when less than minSetInputs are truthy', async () => { + const input = { a: true, b: false, c: false }; + + config.inputs = [{ name: 'a' }, { name: 'b' }, { name: 'c' }]; + + const expected = { Output: undefined, Unverified: true }; + + const result = await logicAtLeast.process(input, config, agent); + + expect(result).toEqual(expected); + }); + + it('should return `Output` as `undefined` and `Unverified` as `true` when no inputs are provided', async () => { + const input = {}; + config.inputs = [{ name: 'a' }, { name: 'b' }]; + const expected = { Output: undefined, Unverified: true }; + + const result = await logicAtLeast.process(input, config, agent); + + expect(result).toEqual(expected); + }); + + it('should return `Output` as `undefined` when minSetInputs is not a valid number', async () => { + const input = { a: true, b: true }; + + config.data.minSetInputs = 'invalid'; + config.inputs = [{ name: 'a' }, { name: 'b' }]; + + const expected = { Output: undefined }; + + const result = await logicAtLeast.process(input, config, agent); + + expect(result).toEqual(expected); + }); + + it('should return `Output` as `true` and `Verified` as `true` when minSetInputs is less than 0', async () => { + const input = { a: true, b: true }; + config.data.minSetInputs = '-1'; + config.inputs = [{ name: 'a' }, { name: 'b' }]; + const expected = { Output: true, Verified: true }; + + const result = await logicAtLeast.process(input, config, agent); + + expect(result).toEqual(expected); + }); + + it('should return `Output` as `undefined` when minSetInputs is greater than 9', async () => { + const input = { a: true, b: true }; + config.data.minSetInputs = '10'; + config.inputs = [{ name: 'a' }, { name: 'b' }]; + const expected = { Output: undefined }; + + const result = await logicAtLeast.process(input, config, agent); + + expect(result).toEqual(expected); + }); + + it('should return `Output` as `true` and `Verified` as `true` when exactly minSetInputs are truthy', async () => { + const input = { a: true, b: true, c: false }; + config.data.minSetInputs = '2'; + config.inputs = [{ name: 'a' }, { name: 'b' }, { name: 'c' }]; + const expected = { Output: true, Verified: true }; + + const result = await logicAtLeast.process(input, config, agent); + + expect(result).toEqual(expected); + }); + + it('should return `Output` as `true` and `Verified` as `true` when more than minSetInputs are truthy', async () => { + const input = { a: true, b: true, c: true }; + config.data.minSetInputs = '2'; + config.inputs = [{ name: 'a' }, { name: 'b' }, { name: 'c' }]; + const expected = { Output: true, Verified: true }; + + const result = await logicAtLeast.process(input, config, agent); + + expect(result).toEqual(expected); + }); + + it('should return `Output` as `true` and `Verified` as `true` when minSetInputs is 0 and no inputs are truthy', async () => { + const input = { a: false, b: false }; + config.data.minSetInputs = '0'; + config.inputs = [{ name: 'a' }, { name: 'b' }]; + const expected = { Output: true, Verified: true }; + + const result = await logicAtLeast.process(input, config, agent); + + expect(result).toEqual(expected); + }); + + it('should return `Output` as `true` and `Verified` as `true` when minSetInputs is 0 and at least one input is truthy', async () => { + const input = { a: true, b: false }; + config.data.minSetInputs = '0'; + config.inputs = [{ name: 'a' }, { name: 'b' }]; + const expected = { Output: true, Verified: true }; + + const result = await logicAtLeast.process(input, config, agent); + + expect(result).toEqual(expected); + }); +}); diff --git a/packages/core/tests/unit/008-Components/LogicAtMost.test.ts b/packages/core/tests/unit/008-Components/LogicAtMost.test.ts new file mode 100644 index 000000000..ef21b5cbe --- /dev/null +++ b/packages/core/tests/unit/008-Components/LogicAtMost.test.ts @@ -0,0 +1,137 @@ +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; +import { LogicAtMost } from '@sre/Components/LogicAtMost.class'; +import { Agent } from '@sre/AgentManager/Agent.class'; +import { ConnectorService } from '@sre/Core/ConnectorsService'; +import { setupSRE } from '../../utils/sre'; + +setupSRE(); + +// Mock Agent class to keep the test isolated from the actual Agent implementation +vi.mock('@sre/AgentManager/Agent.class', () => { + const MockedAgent = vi.fn().mockImplementation(() => ({ + id: 'agent-0000', + agentRuntime: { debug: true }, // used inside createComponentLogger() + isKilled: () => false, + modelsProvider: ConnectorService.getModelsProviderConnector(), + })); + return { Agent: MockedAgent }; +}); + +describe('LogicAtMost: process function', () => { + let logicAtMost: LogicAtMost; + let agent: Agent; + let config: any; + + beforeEach(() => { + // @ts-ignore (Ignore required arguments, as we are using the mocked Agent) + agent = new Agent(); + logicAtMost = new LogicAtMost(); + config = { + name: 'LogicAtMost', + inputs: [], + data: { maxSetInputs: '2' }, + }; + }); + + afterEach(() => { + // ⚠️ Warning from vitest doc (https://vitest.dev/guide/mocking#mocking) - "Always remember to clear or restore mocks before or after each test run to undo mock state changes between runs!" + vi.clearAllMocks(); + }); + + it('should return `Output` as `undefined` when maxSetInputs is a non-numeric string', async () => { + const input = {}; + + config.data.maxSetInputs = 'abc'; + + const result = await logicAtMost.process(input, config, agent); + + expect(result).toEqual({ Output: undefined }); + }); + + it('should return `Output` as `undefined` when maxSetInputs is an empty string', async () => { + const input = {}; + + config.data.maxSetInputs = ''; + + const expected = { Output: undefined }; + + const result = await logicAtMost.process(input, config, agent); + + expect(result).toEqual(expected); + }); + + it('should return `Output` as `undefined` when maxSetInputs is NaN', async () => { + const input = {}; + + config.data.maxSetInputs = 'NaN'; + + const expected = { Output: undefined }; + + const result = await logicAtMost.process(input, config, agent); + + expect(result).toEqual(expected); + }); + + it('should return `Output` as `true` and `Verified` as `true` when at most maxSetInputs are truthy', async () => { + const input = { a: true, b: true, c: false }; + + config.inputs = [{ name: 'a' }, { name: 'b' }, { name: 'c' }]; + + const expected = { Output: true, Verified: true }; + + const result = await logicAtMost.process(input, config, agent); + + expect(result).toEqual(expected); + }); + + it('should return `Output` as `undefined` and `Unverified` as `true` when more than maxSetInputs are truthy', async () => { + const input = { a: true, b: true, c: true }; + + config.inputs = [{ name: 'a' }, { name: 'b' }, { name: 'c' }]; + + const expected = { Output: undefined, Unverified: true }; + + const result = await logicAtMost.process(input, config, agent); + + expect(result).toEqual(expected); + }); + + it('should return `Output` as `true` and `Verified` as `true` when exactly maxSetInputs are truthy', async () => { + const input = { a: true, b: true, c: false }; + + config.data.maxSetInputs = '2'; + config.inputs = [{ name: 'a' }, { name: 'b' }, { name: 'c' }]; + + const expected = { Output: true, Verified: true }; + + const result = await logicAtMost.process(input, config, agent); + + expect(result).toEqual(expected); + }); + + it('should return `Output` as `undefined` and `Unverified` as `true` when maxSetInputs is less than 0', async () => { + const input = { a: true, b: true }; + + config.data.maxSetInputs = '-1'; + config.inputs = [{ name: 'a' }, { name: 'b' }]; + + const expected = { Output: undefined, Unverified: true }; + + const result = await logicAtMost.process(input, config, agent); + + expect(result).toEqual(expected); + }); + + it('should return `Output` as `undefined` when maxSetInputs is greater than 9', async () => { + const input = { a: true, b: true }; + + config.data.maxSetInputs = '10'; + config.inputs = [{ name: 'a' }, { name: 'b' }]; + + const expected = { Output: undefined }; + + const result = await logicAtMost.process(input, config, agent); + + expect(result).toEqual(expected); + }); +}); diff --git a/packages/core/tests/unit/008-Components/LogicOR.test.ts b/packages/core/tests/unit/008-Components/LogicOR.test.ts new file mode 100644 index 000000000..2fa492308 --- /dev/null +++ b/packages/core/tests/unit/008-Components/LogicOR.test.ts @@ -0,0 +1,114 @@ +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; +import { LogicOR } from '@sre/Components/LogicOR.class'; +import { ConnectorService } from '@sre/Core/ConnectorsService'; +import { Agent } from '@sre/AgentManager/Agent.class'; +import { setupSRE } from '../../utils/sre'; + +setupSRE(); + +// Mock Agent class to keep the test isolated from the actual Agent implementation +vi.mock('@sre/AgentManager/Agent.class', () => { + const MockedAgent = vi.fn().mockImplementation(() => ({ + id: 'agent-0000', + agentRuntime: { debug: true }, // used inside createComponentLogger() + isKilled: () => false, + modelsProvider: ConnectorService.getModelsProviderConnector(), + })); + return { Agent: MockedAgent }; +}); + +describe('LogicOR: process function', () => { + let agent; + let logicOR; + let config; + + beforeEach(() => { + // @ts-ignore (Ignore required arguments, as we are using the mocked Agent) + agent = new Agent(); + logicOR = new LogicOR(); + config = { + name: 'LogicOR', + inputs: [], + }; + }); + + afterEach(() => { + // ⚠️ Warning from vitest doc (https://vitest.dev/guide/mocking#mocking) - "Always remember to clear or restore mocks before or after each test run to undo mock state changes between runs!" + vi.clearAllMocks(); + }); + + it('should return `Output` as `true` and `Verified` as `true` when at least one expected input is truthy', async () => { + const input = { a: true, b: false, c: 0, d: null }; + + config.inputs = [{ name: 'a' }, { name: 'b' }, { name: 'c' }, { name: 'd' }]; + + const expected = { + Output: true, + Verified: true, + }; + + const result = await logicOR.process(input, config, agent); + + expect(result).toEqual(expected); + }); + + it('should return `Output` as `undefined` and `Unverified` as `true` when all expected inputs are falsy', async () => { + const input = { a: false, b: 0, c: null, d: '' }; + + config.inputs = [{ name: 'a' }, { name: 'b' }, { name: 'c' }, { name: 'd' }]; + + const expected = { + Output: undefined, + Unverified: true, + }; + + const result = await logicOR.process(input, config, agent); + + expect(result).toEqual(expected); + }); + + it('should return `Output` as `true` and `Verified` as `true` when some expected inputs are missing but at least one is truthy', async () => { + const input = { a: true }; + + config.inputs = [{ name: 'a' }, { name: 'b' }]; + + const expected = { + Output: true, + Verified: true, + }; + + const result = await logicOR.process(input, config, agent); + + expect(result).toEqual(expected); + }); + + it('should return `Output` as `undefined` and `Unverified` as `true` when no inputs are provided', async () => { + const input = {}; + + config.inputs = [{ name: 'a' }, { name: 'b' }]; + + const expected = { + Output: undefined, + Unverified: true, + }; + + const result = await logicOR.process(input, config, agent); + + expect(result).toEqual(expected); + }); + + it('should return `Output` as `undefined` and `Unverified` as `true` when configuration is empty', async () => { + const input = { a: true }; + + config.inputs = []; + + const expected = { + Output: undefined, + Unverified: true, + }; + + const result = await logicOR.process(input, config, agent); + + expect(result).toEqual(expected); + }); +}); diff --git a/packages/core/tests/unit/008-Components/LogicXOR.test.ts b/packages/core/tests/unit/008-Components/LogicXOR.test.ts new file mode 100644 index 000000000..b390ca25b --- /dev/null +++ b/packages/core/tests/unit/008-Components/LogicXOR.test.ts @@ -0,0 +1,129 @@ +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; +import { LogicXOR } from '@sre/Components/LogicXOR.class'; +import { ConnectorService } from '@sre/Core/ConnectorsService'; +import { Agent } from '@sre/AgentManager/Agent.class'; +import { setupSRE } from '../../utils/sre'; + +setupSRE(); + +// Mock Agent class to keep the test isolated from the actual Agent implementation +vi.mock('@sre/AgentManager/Agent.class', () => { + const MockedAgent = vi.fn().mockImplementation(() => ({ + id: 'agent-0000', + agentRuntime: { debug: true }, // used inside createComponentLogger() + isKilled: () => false, + modelsProvider: ConnectorService.getModelsProviderConnector(), + })); + return { Agent: MockedAgent }; +}); + +describe('LogicXOR: process function', () => { + let agent; + let logicXOR; + let config; + + beforeEach(() => { + // @ts-ignore (Ignore required arguments, as we are using the mocked Agent) + agent = new Agent(); + logicXOR = new LogicXOR(); + config = { + name: 'LogicXOR', + inputs: [], + }; + }); + + afterEach(() => { + // ⚠️ Warning from vitest doc (https://vitest.dev/guide/mocking#mocking) - "Always remember to clear or restore mocks before or after each test run to undo mock state changes between runs!" + vi.clearAllMocks(); + }); + + it('should return `Output` as `true` and `Verified` as `true` when exactly one expected input is truthy', async () => { + const input = { a: true, b: false, c: 0, d: null }; + + config.inputs = [{ name: 'a' }, { name: 'b' }, { name: 'c' }, { name: 'd' }]; + + const expected = { + Output: true, + Verified: true, + }; + + const result = await logicXOR.process(input, config, agent); + + expect(result).toEqual(expected); + }); + + it('should return `Output` as `undefined` and `Unverified` as `true` when more than one expected input is truthy', async () => { + const input = { a: true, b: true, c: 0, d: null }; + + config.inputs = [{ name: 'a' }, { name: 'b' }, { name: 'c' }, { name: 'd' }]; + + const expected = { + Output: undefined, + Unverified: true, + }; + + const result = await logicXOR.process(input, config, agent); + + expect(result).toEqual(expected); + }); + + it('should return `Output` as `undefined` and `Unverified` as `true` when no expected inputs are truthy', async () => { + const input = { a: false, b: 0, c: null, d: '' }; + + config.inputs = [{ name: 'a' }, { name: 'b' }, { name: 'c' }, { name: 'd' }]; + + const expected = { + Output: undefined, + Unverified: true, + }; + + const result = await logicXOR.process(input, config, agent); + + expect(result).toEqual(expected); + }); + + it('should return `Output` as `true` and `Verified` as `true` when some expected inputs are missing but exactly one is truthy', async () => { + const input = { a: true }; + + config.inputs = [{ name: 'a' }, { name: 'b' }]; + + const expected = { + Output: true, + Verified: true, + }; + + const result = await logicXOR.process(input, config, agent); + + expect(result).toEqual(expected); + }); + + it('should return `Output` as `undefined` and `Unverified` as `true` when no inputs are provided', async () => { + const input = {}; + + config.inputs = [{ name: 'a' }, { name: 'b' }]; + + const expected = { + Output: undefined, + Unverified: true, + }; + + const result = await logicXOR.process(input, config, agent); + + expect(result).toEqual(expected); + }); + + it('should return `Output` as `undefined` and `Unverified` as `true` when configuration is empty', async () => { + const input = { a: true }; + + config.inputs = []; + + const expected = { + Output: undefined, + Unverified: true, + }; + + const result = await logicXOR.process(input, config, agent); + + expect(result).toEqual(expected); + }); +}); diff --git a/packages/core/tests/unit/utils/templateString.test.ts b/packages/core/tests/unit/999-Utils/templateString.test.ts similarity index 100% rename from packages/core/tests/unit/utils/templateString.test.ts rename to packages/core/tests/unit/999-Utils/templateString.test.ts diff --git a/packages/core/tests/unit/core/JSONFileVaultConnector.test.ts b/packages/core/tests/unit/core/JSONFileVaultConnector.test.ts deleted file mode 100644 index f56b7d455..000000000 --- a/packages/core/tests/unit/core/JSONFileVaultConnector.test.ts +++ /dev/null @@ -1,46 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { setupSRE } from '../../utils/sre'; -import { ConnectorService } from '@sre/Core/ConnectorsService'; -import { IAccessCandidate, TAccessRole } from 'index'; - -setupSRE({ - Vault: { - Connector: 'JSONFileVault', - Settings: { - file: '/Users/zubair/Zubair/SmythOS/smyth-opensource/smythos-ui/vault.json', - }, - }, - Log: { - Connector: 'ConsoleLog', - }, -}); - -describe('JSONFileVault Tests', () => { - it( - 'List all keys in the vault', - async () => { - const mockCandidate: IAccessCandidate = { - id: 'default', - role: TAccessRole.Team, - }; - - const vaultConnector = ConnectorService.getVaultConnector('JSONFileVault'); - const result = await vaultConnector.team(mockCandidate.id).listKeys(); - expect(result).toBeDefined(); - }, - ); - - it( - 'Get a key from the vault', - async () => { - const mockCandidate: IAccessCandidate = { - id: 'default', - role: TAccessRole.Team, - }; - - const vaultConnector = ConnectorService.getVaultConnector('JSONFileVault'); - const result = await vaultConnector.team(mockCandidate.id).get('testKey'); - expect(result).toBe('testValue'); - }, - ); -}); \ No newline at end of file diff --git a/packages/core/tests/utils/TestConnectors.ts b/packages/core/tests/utils/TestConnectors.ts new file mode 100644 index 000000000..94f519879 --- /dev/null +++ b/packages/core/tests/utils/TestConnectors.ts @@ -0,0 +1,39 @@ +import { AccessCandidate } from '@sre/Security/AccessControl/AccessCandidate.class'; +import { AccessRequest } from '@sre/Security/AccessControl/AccessRequest.class'; +import { ACL } from '@sre/Security/AccessControl/ACL.class'; +import { AccountConnector, ISmythAccountRequest } from '@sre/Security/Account.service/AccountConnector'; +import { IAccessCandidate, TAccessRole } from '@sre/types/ACL.types'; + +export class TestAccountConnector extends AccountConnector { + public user(candidate: AccessCandidate): ISmythAccountRequest { + throw new Error('Method not implemented.'); + } + public getResourceACL(resourceId: string, candidate: IAccessCandidate): Promise { + throw new Error('Method not implemented.'); + } + public isTeamMember(teamId: string, candidate: IAccessCandidate): Promise { + return Promise.resolve(true); + } + public getAllTeamSettings(acRequest: AccessRequest, teamId: string): Promise { + throw new Error('Method not implemented.'); + } + public getAllUserSettings(acRequest: AccessRequest, accountId: string): Promise { + throw new Error('Method not implemented.'); + } + public getAgentSetting(acRequest: AccessRequest, agentId: string, settingKey: string): Promise { + throw new Error('Method not implemented.'); + } + public getTeamSetting(acRequest: AccessRequest, teamId: string, settingKey: string): Promise { + throw new Error('Method not implemented.'); + } + public getUserSetting(acRequest: AccessRequest, accountId: string, settingKey: string): Promise { + throw new Error('Method not implemented.'); + } + public getCandidateTeam(candidate: IAccessCandidate): Promise { + if (candidate.role === TAccessRole.Team) { + return Promise.resolve(candidate.id); + } + + return Promise.resolve('default'); + } +} diff --git a/packages/core/tests/utils/test-data-manager.ts b/packages/core/tests/utils/test-data-manager.ts index ef5f2482f..c8a5c520b 100644 --- a/packages/core/tests/utils/test-data-manager.ts +++ b/packages/core/tests/utils/test-data-manager.ts @@ -1,3 +1,4 @@ +import { ConnectorService } from '@sre/Core/ConnectorsService'; import fs from 'fs'; import path from 'path'; import { fileURLToPath } from 'url'; @@ -62,6 +63,13 @@ export class TestDataManager { return fs.readFileSync(filePath, 'utf-8'); } + public readBinaryData(filename: string): Buffer { + const filePath = this.getDataPath(filename); + if (!fs.existsSync(filePath)) { + throw new Error(`Test data file not found: ${filePath}`); + } + return fs.readFileSync(filePath); + } /** * Read and parse a .smyth agent file */