diff --git a/package.json b/package.json index 64ad2ce2..6d4611b8 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "sre", - "version": "0.1.0", + "version": "0.1.1", "description": "", "author": "Alaa-eddine KADDOURI", "license": "MIT", diff --git a/packages/core/package.json b/packages/core/package.json index 21dc1e93..b97afdee 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -1,6 +1,6 @@ { "name": "@smythos/sre", - "version": "1.5.20", + "version": "1.5.21", "description": "Smyth Runtime Environment", "author": "Alaa-eddine KADDOURI", "license": "MIT", diff --git a/packages/core/src/Components/MultimodalLLM.class.ts b/packages/core/src/Components/MultimodalLLM.class.ts new file mode 100644 index 00000000..85b6db31 --- /dev/null +++ b/packages/core/src/Components/MultimodalLLM.class.ts @@ -0,0 +1,128 @@ +import Joi from 'joi'; +import { Component } from './Component.class'; +import { LLMInference } from '@sre/LLMManager/LLM.inference'; +import { TemplateString } from '@sre/helpers/TemplateString.helper'; + +import { AccessCandidate } from '@sre/Security/AccessControl/AccessCandidate.class'; + +export class MultimodalLLM extends Component { + protected configSchema = Joi.object({ + prompt: Joi.string().required().max(8_000_000).label('Prompt'), // 2M tokens is around 8M characters + maxTokens: Joi.number().min(1).label('Maximum Tokens'), + model: Joi.string().max(200).required(), + passthrough: Joi.boolean().optional().label('Passthrough'), + }); + + constructor() { + super(); + } + + init() {} + + async process(input, config, agent) { + await super.process(input, config, agent); + + const logger = this.createComponentLogger(agent, config); + + logger.debug(`=== Multimodal LLM Log ===`); + + try { + const passThrough: boolean = config.data.passthrough || false; + const model: string = config.data.model || 'gpt-4o-mini'; + const llmInference: LLMInference = await LLMInference.getInstance(model, AccessCandidate.agent(agent.id)); + + if (!llmInference.connector) { + return { + _error: `The model '${model}' is not available. Please try a different one.`, + _debug: logger.output, + }; + } + + const modelId = await agent.modelsProvider.getModelId(model); + logger.debug(` Model : ${modelId || model}`); + + let prompt: any = TemplateString(config.data.prompt).parse(input).result; + + logger.debug(` Prompt\n`, prompt, '\n'); + + const outputs = {}; + for (let con of config.outputs) { + if (con.default) continue; + outputs[con.name] = con?.description ? `<${con?.description}>` : ''; + } + + const excludedKeys = ['_debug', '_error']; + const outputKeys = Object.keys(outputs).filter((key) => !excludedKeys.includes(key)); + + if (outputKeys.length > 0) { + const outputFormat = {}; + outputKeys.forEach((key) => (outputFormat[key] = '')); + + prompt += + '\n\nExpected output format = ' + + JSON.stringify(outputFormat) + + '\n\n The output JSON should only use the entries from the output format.'; + + logger.debug(`[Component enhanced prompt]\n${prompt}\n\n`); + } + + const files = Array.isArray(input.Input) ? input.Input : [input.Input]; + + let response: any; + if (passThrough) { + const contentPromise = new Promise(async (resolve, reject) => { + let _content = ''; + const eventEmitter: any = await llmInference.multimodalStreamRequestLegacy(prompt, files, config, agent).catch((error) => { + console.error('Error on multimodalStreamRequest: ', error); + reject(error); + }); + eventEmitter.on('content', (content) => { + if (typeof agent.callback === 'function') { + agent.callback({ content }); + } + agent.sse.send('llm/passthrough/content', content); + _content += content; + }); + eventEmitter.on('thinking', (thinking) => { + if (typeof agent.callback === 'function') { + agent.callback({ thinking }); + } + agent.sse.send('llm/passthrough/thinking', thinking); + }); + eventEmitter.on('end', () => { + console.log('end'); + resolve(_content); + }); + }); + response = await contentPromise; + } else { + response = await llmInference.prompt({ query: prompt, files, params: { ...config, agentId: agent.id } }); + } + + // in case we have the response but it's empty string, undefined or null + if (!response) { + return { _error: ' LLM Error = Empty Response!', _debug: logger.output }; + } + + if (response?.error) { + const error = response?.error + ' ' + (response?.details || ''); + logger.error(` LLM Error=`, error); + return { Reply: response?.data, _error: error, _debug: logger.output }; + } + + logger.debug(' Response \n', response); + + const result = { Reply: response }; + result['_debug'] = logger.output; + + return result; + } catch (error: any) { + const _error = `${error?.error || ''} ${error?.details || ''}`.trim() || error?.message || 'Something went wrong!'; + logger.error(`Error processing File(s)!`, _error); + return { + _error, + _debug: logger.output, + }; + } + } +} diff --git a/packages/core/src/Components/OpenAPI.class.ts b/packages/core/src/Components/OpenAPI.class.ts new file mode 100644 index 00000000..5124b7b2 --- /dev/null +++ b/packages/core/src/Components/OpenAPI.class.ts @@ -0,0 +1,72 @@ +import Joi from 'joi'; + +import { Agent } from '@sre/AgentManager/Agent.class'; +import { Conversation } from '@sre/helpers/Conversation.helper'; +import { TemplateString } from '@sre/helpers/TemplateString.helper'; + +import { Component } from './Component.class'; + +export class OpenAPI extends Component { + protected configSchema = Joi.object({ + model: Joi.string().optional(), + openAiModel: Joi.string().optional(), // for backward compatibility + specUrl: Joi.string().max(2048).uri().required().description('URL of the OpenAPI specification'), + descForModel: Joi.string().max(5000).required().allow('').label('Description for Model'), + name: Joi.string().max(500).required().allow(''), + desc: Joi.string().max(5000).required().allow('').label('Description'), + logoUrl: Joi.string().max(8192).allow(''), + id: Joi.string().max(200), + version: Joi.string().max(100).allow(''), + domain: Joi.string().max(253).allow(''), + }); + + constructor() { + super(); + } + + init() {} + + async process(input, config, agent: Agent) { + await super.process(input, config, agent); + const logger = this.createComponentLogger(agent, config); + + logger.debug(`=== Open API Log ===`); + + try { + const specUrl = config?.data?.specUrl; + + if (!specUrl) { + return { _error: 'Please provide a Open API Specification URL!', _debug: logger.output }; + } + + const model = config?.data?.model || config?.data?.openAiModel; + const descForModel = TemplateString(config?.data?.descForModel).parse(input).result; + let prompt = ''; + + if (input?.Prompt) { + prompt = typeof input?.Prompt === 'string' ? input?.Prompt : JSON.stringify(input?.Prompt); + } else if (input?.Query) { + prompt = typeof input?.Query === 'string' ? input?.Query : JSON.stringify(input?.Query); + } + + if (!prompt) { + return { _error: 'Please provide a prompt', _debug: logger.output }; + } + + // TODO [Forhad]: Need to check and validate input prompt token + + const conv = new Conversation(model, specUrl, { systemPrompt: descForModel, agentId: agent?.id }); + + const result = await conv.prompt(prompt); + + logger.debug(`Response:\n`, result, '\n'); + + return { Output: result, _debug: logger.output }; + } catch (error: any) { + console.error('Error on running Open API: ', error); + return { _error: `Error on running Open API!\n${error?.message || JSON.stringify(error)}`, _debug: logger.output }; + } + } +} + +export default OpenAPI; diff --git a/packages/core/src/Components/VisionLLM.class.ts b/packages/core/src/Components/VisionLLM.class.ts new file mode 100644 index 00000000..75557975 --- /dev/null +++ b/packages/core/src/Components/VisionLLM.class.ts @@ -0,0 +1,104 @@ +import Joi from 'joi'; + +import { TemplateString } from '@sre/helpers/TemplateString.helper'; +import { Component } from './Component.class'; +import { LLMInference } from '@sre/LLMManager/LLM.inference'; +import { AccessCandidate } from '@sre/Security/AccessControl/AccessCandidate.class'; + +export class VisionLLM extends Component { + protected configSchema = Joi.object({ + prompt: Joi.string().required().max(8_000_000).label('Prompt'), // 2M tokens is around 8M characters + maxTokens: Joi.number().min(1).label('Maximum Tokens'), + model: Joi.string().max(200).required(), + passthrough: Joi.boolean().optional().label('Passthrough'), + }); + + constructor() { + super(); + } + + init() {} + + async process(input, config, agent) { + await super.process(input, config, agent); + + const logger = this.createComponentLogger(agent, config); + try { + logger.debug(`=== Vision LLM Log ===`); + + const passThrough: boolean = config.data.passthrough || false; + const model: string = config.data?.model; + + const llmInference: LLMInference = await LLMInference.getInstance(model, AccessCandidate.agent(agent.id)); + // if the llm is undefined, then it means we removed the model from our system + if (!llmInference.connector) { + return { + _error: `The model '${model}' is not available. Please try a different one.`, + _debug: logger.output, + }; + } + + const modelId = await agent.modelsProvider.getModelId(model); + logger.debug(` Model : ${modelId || model}`); + + let prompt: any = TemplateString(config.data.prompt).parse(input).result; + + logger.debug(` Prompt\n`, prompt, '\n'); + + const files = Array.isArray(input.Images) ? input.Images : [input.Images]; + + let response: any; + if (passThrough) { + const contentPromise = new Promise(async (resolve, reject) => { + let _content = ''; + const eventEmitter: any = await llmInference.multimodalStreamRequestLegacy(prompt, files, config, agent).catch((error) => { + console.error('Error on multimodalStreamRequest: ', error); + reject(error); + }); + eventEmitter.on('content', (content) => { + if (typeof agent.callback === 'function') { + agent.callback({ content }); + } + agent.sse.send('llm/passthrough/content', content); + _content += content; + }); + eventEmitter.on('thinking', (thinking) => { + if (typeof agent.callback === 'function') { + agent.callback({ thinking }); + } + agent.sse.send('llm/passthrough/thinking', thinking); + }); + eventEmitter.on('end', () => { + console.log('end'); + resolve(_content); + }); + }); + response = await contentPromise; + } else { + response = await llmInference.prompt({ query: prompt, files, params: { ...config, agentId: agent.id } }); + } + + // in case we have the response but it's empty string, undefined or null + if (!response) { + return { _error: ' LLM Error = Empty Response!', _debug: logger.output }; + } + + if (response?.error) { + const error = response?.error + ' ' + (response?.details || ''); + logger.error(` LLM Error=`, error); + + return { Reply: response?.data, _error: error, _debug: logger.output }; + } + + logger.debug(' Response \n', response); + + const result = { Reply: response }; + + result['_debug'] = logger.output; + + return result; + } catch (error: any) { + return { _error: error.message, _debug: logger.output }; + } + } +} diff --git a/packages/core/src/Components/ZapierAction.class.ts b/packages/core/src/Components/ZapierAction.class.ts new file mode 100644 index 00000000..05708adc --- /dev/null +++ b/packages/core/src/Components/ZapierAction.class.ts @@ -0,0 +1,127 @@ +import axios from 'axios'; +import { Component } from './Component.class'; +import { IAgent as Agent } from '@sre/types/Agent.types'; +import Joi from 'joi'; +import { TemplateStringHelper } from '@sre/helpers/TemplateString.helper'; +import { isSmythFileObject } from '../utils'; +import { SmythFS } from '@sre/IO/Storage.service/SmythFS.class'; +import { AccessCandidate } from '@sre/Security/AccessControl/AccessCandidate.class'; + +function validateAndParseJson(value, helpers) { + let parsedJson: any = null; + + // Try parsing the JSON string + try { + parsedJson = JSON.parse(value); + } catch (error) { + // If parsing fails, return an error + return helpers.error('string.invalidJson', { value }); + } + + // Check if the result is an object + if (typeof parsedJson !== 'object' || parsedJson === null) { + return helpers.error('string.notJsonObject', { value }); + } + + // Check for empty keys + for (const key in parsedJson) { + if (key.trim() === '') { + return helpers.error('object.emptyKey', { value }); + } + } + + // Return the parsed JSON if all validations pass + return parsedJson; +} + +export class ZapierAction extends Component { + protected configSchema = Joi.object({ + actionName: Joi.string().max(100).required(), + actionId: Joi.string().max(100).required(), + logoUrl: Joi.string().max(500).allow(''), + apiKey: Joi.string().max(350).required(), + params: Joi.string().custom(validateAndParseJson, 'custom JSON validation').allow(''), + }); + constructor() { + super(); + } + + init() {} + + async process(input, config, agent: Agent) { + await super.process(input, config, agent); + + const logger = this.createComponentLogger(agent, config); + + logger.debug(`=== Zapier Action Log ===`); + + const teamId = agent?.teamId; + // const apiKey = await parseKey(config?.data?.apiKey, teamId); + const apiKey = await TemplateStringHelper.create(config?.data?.apiKey).parseTeamKeysAsync(teamId).asyncResult; + + if (!apiKey) { + return { _error: 'You are not authorized to run the Zapier Action!', _debug: logger.output }; + } + + const actionId = config?.data?.actionId; + + if (!actionId) { + return { _error: 'Zapier Action ID is required!', _debug: logger.output }; + } + + if (!Object.keys(input || {})?.length) { + return { _error: 'Give a plain english description of exact action you want to do!', _debug: logger.output }; + } + + let _input = {}; + let _pubUrlsCreated: string[] = []; + + for (const [key, value] of Object.entries(input)) { + if (isSmythFileObject(value)) { + // _input[key] = (value as SmythFileObject)?.url; + const pubUrl = await SmythFS.Instance.genTempUrl((value as any)?.url, AccessCandidate.agent(agent.id)); + _pubUrlsCreated.push(pubUrl); + _input[key] = pubUrl; + } else { + _input[key] = value; + } + } + + try { + const url = `https://actions.zapier.com/api/v1/exposed/${actionId}/execute/?api_key=${apiKey}`; + const res = await axios.post(url, { ..._input }); + + logger.debug(`Output:\n`, res?.data); + + Promise.all(_pubUrlsCreated.map((url) => SmythFS.Instance.destroyTempUrl(url))) + .then(() => { + console.log('Cleaned up all temp urls'); + }) + .catch((e) => { + console.log('Error cleaning up temp urls', e); + }); + + return { Output: res?.data, _debug: logger.output }; + } catch (error: any) { + console.log('Error Running Zapier Action: \n', error); + + // Sometimes 'error?.response?.data' is an empty Object then we need to use 'error?.message' + let message = Object.keys(error?.response?.data || {})?.length ? error?.response?.data : error?.message; + + if (typeof message === 'object') message = JSON.stringify(message); + + logger.error(`Error running Zapier Action!`, message); + logger.error('Error Inputs ', input); + + Promise.all(_pubUrlsCreated.map((url) => SmythFS.Instance.destroyTempUrl(url))) + .then(() => { + console.log('Cleaned up all temp urls'); + }) + .catch((e) => { + console.log('Error cleaning up temp urls', e); + }); + + return { _error: `Zapier Error: ${message}`, _debug: logger.output }; + } + } +} diff --git a/packages/core/src/Components/index.ts b/packages/core/src/Components/index.ts index a7bcaee2..d947530f 100644 --- a/packages/core/src/Components/index.ts +++ b/packages/core/src/Components/index.ts @@ -3,6 +3,7 @@ import { APIEndpoint } from './APIEndpoint.class'; import { APIOutput } from './APIOutput.class'; import { PromptGenerator } from './PromptGenerator.class'; import { APICall } from './APICall/APICall.class'; +import { VisionLLM } from './VisionLLM.class'; import { FSleep } from './FSleep.class'; import { FHash } from './FHash.class'; import { FEncDec } from './FEncDec.class'; @@ -22,17 +23,20 @@ import { Async } from './Async.class'; import { Await } from './Await.class'; import { ForEach } from './ForEach.class'; import { HuggingFace } from './HuggingFace.class'; +import { ZapierAction } from './ZapierAction.class'; import { GPTPlugin } from './GPTPlugin.class'; import { Classifier } from './Classifier.class'; import { FSign } from './FSign.class'; +import { MultimodalLLM } from './MultimodalLLM.class'; import { GenAILLM } from './GenAILLM.class'; import { FileStore } from './FileStore.class'; import { ScrapflyWebScrape } from './ScrapflyWebScrape.class'; import { TavilyWebSearch } from './TavilyWebSearch.class'; import { ComponentHost } from './ComponentHost.class'; +import { ServerlessCode } from './ServerlessCode.class'; import { ImageGenerator } from './ImageGenerator.class'; // Legacy import { MCPClient } from './MCPClient.class'; -import { ServerlessCode } from './ServerlessCode.class'; +import { OpenAPI } from './OpenAPI.class'; const components = { Component: new Component(), @@ -42,6 +46,7 @@ const components = { PromptGenerator: new PromptGenerator(), LLMPrompt: new PromptGenerator(), APICall: new APICall(), + VisionLLM: new VisionLLM(), FSleep: new FSleep(), FHash: new FHash(), FEncDec: new FEncDec(), @@ -62,8 +67,10 @@ const components = { Await: new Await(), ForEach: new ForEach(), HuggingFace: new HuggingFace(), + ZapierAction: new ZapierAction(), GPTPlugin: new GPTPlugin(), Classifier: new Classifier(), + MultimodalLLM: new MultimodalLLM(), GenAILLM: new GenAILLM(), FileStore: new FileStore(), WebSearch: new TavilyWebSearch(), @@ -71,9 +78,10 @@ const components = { TavilyWebSearch: new TavilyWebSearch(), ScrapflyWebScrape: new ScrapflyWebScrape(), ComponentHost: new ComponentHost(), + ServerlessCode: new ServerlessCode(), ImageGenerator: new ImageGenerator(), MCPClient: new MCPClient(), - ServerlessCode: new ServerlessCode() + OpenAPI: new OpenAPI(), }; export const ComponentInstances = components; diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index e59f33a1..f330efc5 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -38,10 +38,13 @@ export * from './Components/LogicAtMost.class'; export * from './Components/LogicOR.class'; export * from './Components/LogicXOR.class'; export * from './Components/MCPClient.class'; +export * from './Components/MultimodalLLM.class'; export * from './Components/PromptGenerator.class'; export * from './Components/ScrapflyWebScrape.class'; export * from './Components/ServerlessCode.class'; export * from './Components/TavilyWebSearch.class'; +export * from './Components/VisionLLM.class'; +export * from './Components/ZapierAction.class'; export * from './Core/AgentProcess.helper'; export * from './Core/boot'; export * from './Core/Connector.class';