From 00c77837ac4f5ac6b768f58e0cf33435535b3040 Mon Sep 17 00:00:00 2001 From: Mike Goitia Date: Thu, 15 May 2025 15:24:28 -0400 Subject: [PATCH 01/11] add polling, cleanup --- examples/example_evaluate.ts | 67 --- examples/example_logs.ts | 38 +- package-lock.json | 35 +- package.json | 6 +- quotientai/index.ts | 50 -- quotientai/logger.ts | 119 ++++- quotientai/resources/datasets.ts | 312 ----------- quotientai/resources/logs.ts | 300 +++++++++-- quotientai/resources/metrics.ts | 20 - quotientai/resources/models.ts | 42 -- quotientai/resources/prompts.ts | 135 ----- quotientai/resources/runs.ts | 186 ------- quotientai/types.ts | 287 +++++++--- tests/logger.test.ts | 208 +++----- tests/resources/datasets.test.ts | 887 ------------------------------- tests/resources/logs.test.ts | 42 +- tests/resources/metrics.test.ts | 37 -- tests/resources/models.test.ts | 64 --- tests/resources/prompt.test.ts | 132 ----- tests/resources/runs.test.ts | 366 ------------- tsconfig.json | 2 +- 21 files changed, 733 insertions(+), 2602 deletions(-) delete mode 100644 examples/example_evaluate.ts delete mode 100644 quotientai/resources/datasets.ts delete mode 100644 quotientai/resources/metrics.ts delete mode 100644 quotientai/resources/models.ts delete mode 100644 quotientai/resources/prompts.ts delete mode 100644 quotientai/resources/runs.ts delete mode 100644 tests/resources/datasets.test.ts delete mode 100644 tests/resources/metrics.test.ts delete mode 100644 tests/resources/models.test.ts delete mode 100644 tests/resources/prompt.test.ts delete mode 100644 tests/resources/runs.test.ts diff --git a/examples/example_evaluate.ts b/examples/example_evaluate.ts deleted file mode 100644 index 87f9621..0000000 --- a/examples/example_evaluate.ts +++ /dev/null @@ -1,67 +0,0 @@ -import { QuotientAI } from '../quotientai'; - -async function main() { - const quotient = new QuotientAI(); - - // create a prompt - const prompt = await quotient.prompts.create({ - name: "quotient-demo-prompt", - system_prompt: "I have a problem", - user_prompt: "Here is a user's inquiry {{input}}, and the context {{context}}", - }); - - console.log(`Prompt ID: ${prompt?.id}`); - - // create a dataset - const dataset = await quotient.datasets.create({ - name: "quotient-demo-dataset", - rows: [ - { - input: "I have a problem", - context: "here is a support ticket", - expected: "I'm sorry to hear that. What's the problem?", - }, - { - input: "I need help", - context: "here is a support ticket", - expected: "I'm sorry to hear that. What's the problem?", - }, - { - input: "I want to cancel my subscription", - expected: "I'm sorry to hear that. I can help you with that. Please provide me with your account information.", - }, - ], - }); - - console.log(`Dataset ID: ${dataset?.id}`); - - // list out the models - const models = await quotient.models.list(); - - // get gpt4o-mini model - const model = models.find(model => model.name === "gpt-4o-mini-2024-07-18"); - - if (!model) { - throw new Error("Model not found"); - } - - console.log(`Model ID: ${model.id}`); - - // create a run - const run = await quotient.runs.create({ - prompt: prompt, - dataset: dataset, - model: model, - parameters: { - temperature: 0.7, - top_k: 50, - top_p: 0.9, - max_tokens: 100, - }, - metrics: ["exactmatch", "rouge1", "sacrebleu"], - }); - - console.log(`Run ID: ${run?.id}`); -} - -main(); diff --git a/examples/example_logs.ts b/examples/example_logs.ts index f06abd9..5388246 100644 --- a/examples/example_logs.ts +++ b/examples/example_logs.ts @@ -5,30 +5,31 @@ async function main() { console.log("QuotientAI client initialized") // configure the logger - const quotient_logger = quotient.logger.init({ - app_name: "my-app", + const quotientLogger = quotient.logger.init({ + appName: "my-app-test", environment: "dev", - sample_rate: 1.0, + sampleRate: 1.0, tags: { model: "gpt-4o", feature: "customer-support" }, - hallucination_detection: true, + hallucinationDetection: true, + hallucinationDetectionSampleRate: 1.0, }) console.log("Logger initialized") // mock retrieved documents - const retrieved_documents = [ + const retrievedDocuments = [ "Sample document 1", - {"page_content": "Sample document 2", "metadata": {"source": "website.com"}}, - {"page_content": "Sample document 3"} + {"pageContent": "Sample document 2", "metadata": {"source": "website.com"}}, + {"pageContent": "Sample document 3"} ] console.log("Preparing to log with quotient_logger") try { - const response = await quotient_logger.log({ - user_query: "How do I cook a goose?", - model_output: "The capital of France is Paris", - documents: retrieved_documents, - message_history: [ + const logId= await quotientLogger.log({ + userQuery: "How do I cook a test?", + modelOutput: "The capital of France is Paris", + documents: retrievedDocuments, + messageHistory: [ {"role": "system", "content": "You are an expert on geography."}, {"role": "user", "content": "What is the capital of France?"}, {"role": "assistant", "content": "The capital of France is Paris"}, @@ -37,10 +38,17 @@ async function main() { "You are a helpful assistant that answers questions about the world.", "Answer the question in a concise manner. If you are not sure, say 'I don't know'.", ], - hallucination_detection: true, - inconsistency_detection: true, + hallucinationDetection: true, + inconsistencyDetection: true, }); - console.log(response.message) + console.log('pollForDetectionResults with logId: ', logId) + + // poll for the detection results + const detectionResults = await quotientLogger.pollForDetectionResults(logId); + console.log('documentEvaluations', detectionResults?.evaluations[0].documentEvaluations) + console.log('messageHistoryEvaluations', detectionResults?.evaluations[0].messageHistoryEvaluations) + console.log('instructionEvaluations', detectionResults?.evaluations[0].instructionEvaluations) + console.log('fullDocContextEvaluation', detectionResults?.evaluations[0].fullDocContextEvaluation) } catch (error) { console.error(error) } diff --git a/package-lock.json b/package-lock.json index c1117b1..3eb091b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,20 +1,22 @@ { "name": "quotientai", - "version": "0.0.1", + "version": "0.0.7", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "quotientai", - "version": "0.0.1", + "version": "0.0.7", "license": "ISC", "dependencies": { "axios": "^1.6.7", - "jsonwebtoken": "^9.0.2" + "jsonwebtoken": "^9.0.2", + "uuid": "^11.1.0" }, "devDependencies": { "@types/jsonwebtoken": "^9.0.5", - "@types/node": "^20.11.19", + "@types/node": "^20.17.47", + "@types/uuid": "^10.0.0", "@typescript-eslint/eslint-plugin": "^7.0.1", "@typescript-eslint/parser": "^7.0.1", "@vitest/coverage-v8": "^1.3.1", @@ -1061,15 +1063,20 @@ "license": "MIT" }, "node_modules/@types/node": { - "version": "20.17.24", - "resolved": "https://registry.npmjs.org/@types/node/-/node-20.17.24.tgz", - "integrity": "sha512-d7fGCyB96w9BnWQrOsJtpyiSaBcAYYr75bnK6ZRjDbql2cGLj/3GsL5OYmLPNq76l7Gf2q4Rv9J2o6h5CrD9sA==", + "version": "20.17.47", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.17.47.tgz", + "integrity": "sha512-3dLX0Upo1v7RvUimvxLeXqwrfyKxUINk0EAM83swP2mlSUcwV73sZy8XhNz8bcZ3VbsfQyC/y6jRdL5tgCNpDQ==", "dev": true, - "license": "MIT", "dependencies": { "undici-types": "~6.19.2" } }, + "node_modules/@types/uuid": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-10.0.0.tgz", + "integrity": "sha512-7gqG38EyHgyP1S+7+xomFtL+ZNHcKv6DwNaCZmJmo1vgMugyF3TCnXVg4t1uk89mLNwnLtnY3TpOpCOyp1/xHQ==", + "dev": true + }, "node_modules/@typescript-eslint/eslint-plugin": { "version": "7.18.0", "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-7.18.0.tgz", @@ -3882,6 +3889,18 @@ "punycode": "^2.1.0" } }, + "node_modules/uuid": { + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-11.1.0.tgz", + "integrity": "sha512-0/A9rDy9P7cJ+8w1c9WD9V//9Wj15Ce2MPz8Ri6032usz+NfePxx5AcN3bN+r6ZL6jEo066/yNYB3tn4pQEx+A==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "bin": { + "uuid": "dist/esm/bin/uuid" + } + }, "node_modules/vite": { "version": "5.4.15", "resolved": "https://registry.npmjs.org/vite/-/vite-5.4.15.tgz", diff --git a/package.json b/package.json index 86eaa0c..e7b5661 100644 --- a/package.json +++ b/package.json @@ -19,11 +19,13 @@ }, "dependencies": { "axios": "^1.6.7", - "jsonwebtoken": "^9.0.2" + "jsonwebtoken": "^9.0.2", + "uuid": "^11.1.0" }, "devDependencies": { "@types/jsonwebtoken": "^9.0.5", - "@types/node": "^20.11.19", + "@types/node": "^20.17.47", + "@types/uuid": "^10.0.0", "@typescript-eslint/eslint-plugin": "^7.0.1", "@typescript-eslint/parser": "^7.0.1", "@vitest/coverage-v8": "^1.3.1", diff --git a/quotientai/index.ts b/quotientai/index.ts index ad0d909..cdb6ead 100644 --- a/quotientai/index.ts +++ b/quotientai/index.ts @@ -1,23 +1,11 @@ import { BaseQuotientClient } from './client'; import { QuotientLogger } from './logger'; -import { Prompt, Model, Dataset } from './types'; -import { Run } from './resources/runs'; import { AuthResource } from './resources/auth'; -import { PromptsResource } from './resources/prompts'; -import { DatasetsResource } from './resources/datasets'; -import { ModelsResource } from './resources/models'; -import { RunsResource } from './resources/runs'; -import { MetricsResource } from './resources/metrics'; import { LogsResource } from './resources/logs'; import { logError } from './exceptions'; export class QuotientAI { public auth: AuthResource = null!; - public prompts: PromptsResource = null!; - public datasets: DatasetsResource = null!; - public models: ModelsResource = null!; - public runs: RunsResource = null!; - public metrics: MetricsResource = null!; public logs: LogsResource = null!; public logger: QuotientLogger = null!; @@ -40,11 +28,6 @@ export class QuotientAI { private initializeResources(client: BaseQuotientClient): void { // Initialize resources this.auth = new AuthResource(client); - this.prompts = new PromptsResource(client); - this.datasets = new DatasetsResource(client); - this.models = new ModelsResource(client); - this.runs = new RunsResource(client); - this.metrics = new MetricsResource(client); this.logs = new LogsResource(client); // Create an unconfigured logger instance @@ -62,37 +45,4 @@ export class QuotientAI { return; } } - - async evaluate(params: { - prompt: Prompt; - dataset: Dataset; - model: Model; - parameters: Record; - metrics: string[]; - }): Promise | null> { - const { prompt, dataset, model, parameters, metrics } = params; - - // Validate parameters - const validParameters = ['temperature', 'top_k', 'top_p', 'max_tokens']; - const invalidParameters = Object.keys(parameters).filter( - key => !validParameters.includes(key) - ); - - if (invalidParameters.length > 0) { - const error = new Error( - `Invalid parameters: ${invalidParameters.join(', ')}. ` + - `Valid parameters are: ${validParameters.join(', ')}` - ); - logError(error, 'QuotientAI.evaluate'); - return null; - } - - return this.runs.create({ - prompt, - dataset, - model, - parameters, - metrics - }); - } } \ No newline at end of file diff --git a/quotientai/logger.ts b/quotientai/logger.ts index c690faf..4fbc515 100644 --- a/quotientai/logger.ts +++ b/quotientai/logger.ts @@ -1,9 +1,10 @@ -import { LogEntry, LoggerConfig, LogDocument } from './types'; +import { LogEntry, LoggerConfig, LogDocument, LOG_STATUS, DetectionResults } from './types'; import { ValidationError, logError } from './exceptions'; - +import { v4 as uuidv4 } from 'uuid'; interface LogsResource { create(params: LogEntry): Promise; list(): Promise; + getDetections(logId: string): Promise; } export class QuotientLogger { @@ -22,17 +23,17 @@ export class QuotientLogger { } init(config: LoggerConfig): QuotientLogger { - this.appName = config.app_name; + this.appName = config.appName; this.environment = config.environment; this.tags = config.tags || {}; - this.sampleRate = config.sample_rate || 1.0; - this.hallucinationDetection = config.hallucination_detection || false; - this.inconsistencyDetection = config.inconsistency_detection || false; - this.hallucinationDetectionSampleRate = config.hallucination_detection_sample_rate || 0.0; + this.sampleRate = config.sampleRate || 1.0; + this.hallucinationDetection = config.hallucinationDetection || false; + this.inconsistencyDetection = config.inconsistencyDetection || false; + this.hallucinationDetectionSampleRate = config.hallucinationDetectionSampleRate || 0.0; this.configured = true; if (this.sampleRate < 0 || this.sampleRate > 1) { - logError(new Error('sample_rate must be between 0.0 and 1.0')); + logError(new Error('sampleRate must be between 0.0 and 1.0')); return this; } @@ -46,19 +47,19 @@ export class QuotientLogger { // Type guard function to check if an object is a valid LogDocument private isValidLogDocument(obj: any): { valid: boolean; error?: string } { try { - // Check if it has the required page_content property - if (!('page_content' in obj)) { + // Check if it has the required pageContent property + if (!('pageContent' in obj)) { return { valid: false, - error: "Missing required 'page_content' property" + error: "Missing required 'pageContent' property" }; } - // Check if page_content is a string - if (typeof obj.page_content !== 'string') { + // Check if pageContent is a string + if (typeof obj.pageContent !== 'string') { return { valid: false, - error: `The 'page_content' property must be a string, found ${typeof obj.page_content}` + error: `The 'pageContent' property must be a string, found ${typeof obj.pageContent}` }; } @@ -91,15 +92,15 @@ export class QuotientLogger { if (!validation.valid) { logError(new ValidationError( `Invalid document format at index ${i}: ${validation.error}. ` + - "Documents must be either strings or JSON objects with a 'page_content' string property and an optional 'metadata' object. " + - "To fix this, ensure each document follows the format: { page_content: 'your text content', metadata?: { key: 'value' } }" + "Documents must be either strings or JSON objects with a 'pageContent' string property and an optional 'metadata' object. " + + "To fix this, ensure each document follows the format: { pageContent: 'your text content', metadata?: { key: 'value' } }" )); return false; } } else { logError(new ValidationError( - `Invalid document type at index ${i}. Found ${typeof doc}, but documents must be either strings or JSON objects with a 'page_content' property. ` + - "To fix this, provide documents as either simple strings or properly formatted objects: { page_content: 'your text content' }" + `Invalid document type at index ${i}. Found ${typeof doc}, but documents must be either strings or JSON objects with a 'pageContent' property. ` + + "To fix this, provide documents as either simple strings or properly formatted objects: { pageContent: 'your text content' }" )); return false; } @@ -108,15 +109,14 @@ export class QuotientLogger { } // log a message - // params: Omit - async log(params: Omit): Promise { + async log(params: Omit): Promise { if (!this.configured) { logError(new Error('Logger is not configured. Please call init() before logging.')); return null; } if (!this.appName || !this.environment) { - logError(new Error('Logger is not properly configured. app_name and environment must be set.')); + logError(new Error('Logger is not properly configured. appName and environment must be set.')); return null; } @@ -132,21 +132,82 @@ export class QuotientLogger { const mergedTags = { ...this.tags, ...(params.tags || {}) }; // Use instance variables as defaults if not provided - const hallucinationDetection = params.hallucination_detection ?? this.hallucinationDetection; - const inconsistencyDetection = params.inconsistency_detection ?? this.inconsistencyDetection; + const hallucinationDetection = params.hallucinationDetection ?? this.hallucinationDetection; + const inconsistencyDetection = params.inconsistencyDetection ?? this.inconsistencyDetection; if (this.shouldSample()) { - const response = await this.logsResource.create({ + // generate a random id + const id = uuidv4(); + // generate iso string for createdAt + const createdAt = new Date().toISOString(); + await this.logsResource.create({ ...params, - app_name: this.appName, + id: id, + createdAt: createdAt, + appName: this.appName, environment: this.environment, tags: mergedTags, - hallucination_detection: hallucinationDetection, - inconsistency_detection: inconsistencyDetection, - hallucination_detection_sample_rate: this.hallucinationDetectionSampleRate, + hallucinationDetection: hallucinationDetection, + inconsistencyDetection: inconsistencyDetection, + hallucinationDetectionSampleRate: this.hallucinationDetectionSampleRate, }); - return response; + return id; + } + } + + // poll for the detection results using log id + async pollForDetectionResults( + logId: string, + timeout: number = 300, + pollInterval: number = 2.0 + ): Promise { + if (!this.configured) { + logError(new Error('Logger is not configured. Please call init() before polling for detection results.')); + return null; + } + + if (!logId) { + logError(new Error('Log ID is required for detection polling.')); + return null; + } + + const startTime = Date.now(); + const timeoutMs = timeout * 1000; // Convert timeout to milliseconds + let currentPollInterval = pollInterval * 1000; // Convert poll interval to milliseconds + const baseInterval = pollInterval * 1000; // Keep track of the base interval + + while ((Date.now() - startTime) < timeoutMs) { + try { + const results = await this.logsResource.getDetections(logId); + + // Reset interval on successful response + currentPollInterval = baseInterval; + + if (results && results.log) { + const status = results.log.status; + + // Check if we're in a final state + if (status === LOG_STATUS.LOG_CREATED_NO_DETECTIONS_PENDING || + status === LOG_STATUS.LOG_CREATED_AND_DETECTION_COMPLETED) { + return results; + } + + } + + // Wait for poll interval before trying again + await new Promise(resolve => setTimeout(resolve, currentPollInterval)); + } catch (error) { + // Handle event loop errors specifically + if (error instanceof Error && error.message.includes('Event loop is closed')) { + await new Promise(resolve => setTimeout(resolve, currentPollInterval)); + continue; + } + await new Promise(resolve => setTimeout(resolve, currentPollInterval)); + } } + + logError(new Error(`Timed out waiting for detection results after ${timeout} seconds`)); + return null; } } \ No newline at end of file diff --git a/quotientai/resources/datasets.ts b/quotientai/resources/datasets.ts deleted file mode 100644 index 88da4db..0000000 --- a/quotientai/resources/datasets.ts +++ /dev/null @@ -1,312 +0,0 @@ -import { BaseQuotientClient } from '../client'; -import { Dataset, DatasetRow, DatasetRowMetadata } from '../types'; -import { logError } from '../exceptions'; -export interface DatasetRowResponse { - id: string; - input: string; - context?: string; - expected?: string; - annotation?: string; - annotation_note?: string; - created_by: string; - created_at: string; - updated_at: string; -} - -interface DatasetResponse { - id: string; - name: string; - created_by: string; - created_at: string; - updated_at: string; - description?: string; - dataset_rows?: DatasetRowResponse[]; -} - -interface ListOptions { - includeRows?: boolean; -} - -interface CreateDatasetParams { - name: string; - description?: string; - rows?: CreateDatasetRowParams[]; - model_id?: string; - user_id?: string; - tags?: Record; -} - -interface CreateDatasetRowParams { - input: string; - context?: string; - expected?: string; - metadata?: CreateDatasetRowMetadataParams; -} - -interface CreateDatasetRowMetadataParams { - annotation?: string; - annotation_note?: string; -} - -interface AppendDatasetParams { - dataset: Dataset; - rows?: CreateDatasetRowParams[]; -} - -interface UpdateDatasetParams extends AppendDatasetParams { - name?: string; - description?: string; -} - -interface DeleteDatasetParams { - dataset: Dataset; - rows?: DatasetRow[]; -} - -export class DatasetsResource { - protected client: BaseQuotientClient; - - constructor(client: BaseQuotientClient) { - this.client = client; - } - - // list all datasets - // options: ListOptions - async list(options: ListOptions = {}): Promise { - const response = await this.client.get('/datasets') as DatasetResponse[]; - const datasets: Dataset[] = []; - - for (const dataset of response) { - let rows: DatasetRow[] | undefined; - if (options.includeRows) { - const rowsResponse = await this.client.get(`/datasets/${dataset.id}/rows`) as DatasetRowResponse[]; - rows = rowsResponse.map(row => ({ - id: row.id, - input: row.input, - context: row.context, - expected: row.expected, - metadata: { - annotation: row.annotation, - annotation_note: row.annotation_note - } as DatasetRowMetadata, - created_by: row.created_by, - created_at: new Date(row.created_at), - updated_at: new Date(row.updated_at) - })); - } - datasets.push({ - id: dataset.id, - name: dataset.name, - created_by: dataset.created_by, - description: dataset.description, - created_at: new Date(dataset.created_at), - updated_at: new Date(dataset.updated_at), - rows - }); - } - return datasets; - } - - // get a dataset - // id: string - async get(id: string): Promise { - const response = await this.client.get(`/datasets/${id}`) as DatasetResponse; - - const rows = response.dataset_rows?.map(row => ({ - id: row.id, - input: row.input, - context: row.context, - expected: row.expected, - metadata: { - annotation: row.annotation, - annotation_note: row.annotation_note - } as DatasetRowMetadata, - created_by: row.created_by, - created_at: new Date(row.created_at), - updated_at: new Date(row.updated_at) - })); - - return { - id: response.id, - name: response.name, - created_by: response.created_by, - description: response.description, - created_at: new Date(response.created_at), - updated_at: new Date(response.updated_at), - rows - }; - } - - // create a dataset - // options: CreateDatasetParams - async create(options: CreateDatasetParams): Promise { - const response = await this.client.post('/datasets', { - name: options.name, - description: options.description, - rows: options.rows?.map(row => ({ - input: row.input, - context: row.context, - expected: row.expected, - metadata: row.metadata - })), - model_id: options.model_id, - user_id: options.user_id, - tags: options.tags - }) as DatasetResponse; - - const id = response.id; - const rowResponses: DatasetRowResponse[] = []; - let datasetRows: DatasetRow[] = []; - if (options.rows) { - const results = await this.batchCreateRows(id, options.rows, rowResponses); - datasetRows = results.map(result => ({ - id: result.id, - input: result.input, - context: result.context, - expected: result.expected, - metadata: { - annotation: result.annotation, - annotation_note: result.annotation_note - }, - created_by: result.created_by, - created_at: new Date(result.created_at), - updated_at: new Date(result.updated_at) - })); - } else { - datasetRows = []; - } - - return { - id: response.id, - name: response.name, - created_by: response.created_by, - description: response.description, - created_at: new Date(response.created_at), - updated_at: new Date(response.updated_at), - rows: datasetRows - }; - } - - // update a dataset - // dataset: Dataset, name?: string, description?: string, rows?: DatasetRow[] - async update(options: UpdateDatasetParams): Promise { - const payload = { - name: options.name, - description: options.description, - rows: options.rows?.map(row => ({ - input: row.input, - context: row.context, - expected: row.expected, - annotation: row.metadata?.annotation, - annotation_note: row.metadata?.annotation_note - })), - } - const response = await this.client.patch(`/datasets/${options.dataset.id}`, payload) as DatasetResponse; - - return { - id: response.id, - name: response.name, - created_by: response.created_by, - description: response.description, - created_at: new Date(response.created_at), - updated_at: new Date(response.updated_at), - rows: response.dataset_rows?.map(row => ({ - id: row.id, - input: row.input, - context: row.context, - expected: row.expected, - metadata: { - annotation: row.annotation, - annotation_note: row.annotation_note - }, - created_by: row.created_by, - created_at: new Date(row.created_at), - updated_at: new Date(row.updated_at) - })) - }; - } - - // append rows to a dataset - // options: AppendDatasetParams - async append(options: AppendDatasetParams): Promise { - if (!options.rows) { - logError(new Error('rows are required')); - return null; - } - const rowResponses: DatasetRowResponse[] = []; - let datasetRows: DatasetRow[] = []; - const results = await this.batchCreateRows(options.dataset.id, options.rows, rowResponses); - datasetRows = results.map(result => ({ - id: result.id, - input: result.input, - context: result.context, - expected: result.expected, - metadata: { - annotation: result.annotation, - annotation_note: result.annotation_note - }, - created_by: result.created_by, - created_at: new Date(result.created_at), - updated_at: new Date(result.updated_at) - })); - - return { - id: options.dataset.id, - name: options.dataset.name, - created_by: options.dataset.created_by, - description: options.dataset.description, - created_at: options.dataset.created_at, - updated_at: options.dataset.updated_at, - rows: options.dataset.rows ? [...options.dataset.rows, ...datasetRows] : datasetRows - }; - } - - // delete a dataset - // options: DeleteDatasetParams - async delete(options: DeleteDatasetParams): Promise { - if (options.rows) { - for (const row of options.rows) { - await this.client.patch(`/datasets/${options.dataset.id}/dataset_rows/${row.id}`, { - id: row.id, - input: row.input, - context: row.context, - expected: row.expected, - annotation: row.metadata.annotation, - annotation_note: row.metadata.annotation_note, - is_deleted: true - }); - } - } else { - await this.client.patch(`/datasets/${options.dataset.id}`, { - name: options.dataset.name, - description: options.dataset.description, - is_deleted: true - }); - } - } - - // batch create rows - // datasetID: string, rows: DatasetRow[], rowResponses: DatasetRowResponse[], batchSize: number = 10 - async batchCreateRows(datasetID: string, rows: CreateDatasetRowParams[], rowResponses: DatasetRowResponse[], batchSize: number = 10): Promise { - // Process rows in batches - for (let i = 0; i < rows.length; i += batchSize) { - const batch = rows.slice(i, i + batchSize); - try { - const response = await this.client.post( - `/datasets/${datasetID}/dataset_rows/batch`, - { rows: batch } - ) as DatasetRowResponse[]; - rowResponses.push(...response); - } catch (e) { - // If batch create fails, divide batch size by two and try recursively - if (batchSize === 1) { - logError(e as Error, 'DatasetsResource.batchCreateRows'); - return []; - } else { - await this.batchCreateRows(datasetID, batch, rowResponses, Math.floor(batchSize / 2)); - } - } - } - return rowResponses; - } -} \ No newline at end of file diff --git a/quotientai/resources/logs.ts b/quotientai/resources/logs.ts index d0f7773..1d2a517 100644 --- a/quotientai/resources/logs.ts +++ b/quotientai/resources/logs.ts @@ -1,7 +1,8 @@ import { logError } from '../exceptions'; import { BaseQuotientClient } from '../client'; -import { LogDocument } from '../types'; +import { LogDocument, DetectionResultsResponse, DetectionResults, Evaluation, LogDetail, DocumentLog, LogMessageHistory, LogInstruction, DocumentEvaluation, MessageHistoryEvaluation, InstructionEvaluation, FullDocContextEvaluation, DocumentEvaluationResponse, MessageHistoryEvaluationResponse, InstructionEvaluationResponse, FullDocContextEvaluationResponse } from '../types'; +// Snake case interface for API responses interface LogResponse { id: string; app_name: string; @@ -10,7 +11,7 @@ interface LogResponse { inconsistency_detection: boolean; user_query: string; model_output: string; - documents: (string | LogDocument)[]; + documents: (string | { page_content: string; metadata?: Record })[]; message_history: any[] | null; instructions: string[] | null; tags: Record; @@ -21,60 +22,78 @@ interface LogsResponse { logs: LogResponse[]; } +// CamelCase interface for client-side params, will be converted to snake_case for API interface CreateLogParams { - app_name: string; + id?: string; + createdAt?: Date; + appName: string; environment: string; - hallucination_detection: boolean; - inconsistency_detection: boolean; - user_query: string; - model_output: string; + hallucinationDetection: boolean; + inconsistencyDetection: boolean; + userQuery: string; + modelOutput: string; documents: (string | LogDocument)[]; - message_history?: any[] | null; + messageHistory?: any[] | null; instructions?: string[] | null; tags?: Record; - hallucination_detection_sample_rate?: number; + hallucinationDetectionSampleRate?: number; } +// CamelCase interface for client-side params, will be converted to snake_case for API interface ListLogsParams { - app_name?: string; + appName?: string; environment?: string; - start_date?: Date; - end_date?: Date; + startDate?: Date; + endDate?: Date; limit?: number; offset?: number; } export class Log { id: string; - app_name: string; + appName: string; environment: string; - hallucination_detection: boolean; - inconsistency_detection: boolean; - user_query: string; - model_output: string; + hallucinationDetection: boolean; + inconsistencyDetection: boolean; + userQuery: string; + modelOutput: string; documents: (string | LogDocument)[]; - message_history: any[] | null; + messageHistory: any[] | null; instructions: string[] | null; tags: Record; - created_at: Date; + createdAt: Date; constructor(data: LogResponse) { this.id = data.id; - this.app_name = data.app_name; + this.appName = data.app_name; this.environment = data.environment; - this.hallucination_detection = data.hallucination_detection; - this.inconsistency_detection = data.inconsistency_detection; - this.user_query = data.user_query; - this.model_output = data.model_output; - this.documents = data.documents; - this.message_history = data.message_history; + this.hallucinationDetection = data.hallucination_detection; + this.inconsistencyDetection = data.inconsistency_detection; + this.userQuery = data.user_query; + this.modelOutput = data.model_output; + + // Convert documents with page_content to pageContent format for client-side use + this.documents = data.documents.map(doc => { + if (typeof doc === 'string') { + return doc; + } else if (doc && typeof doc === 'object' && 'page_content' in doc) { + const { page_content, metadata } = doc; + return { + pageContent: page_content, + metadata + } as LogDocument; + } + return doc; + }); + + this.messageHistory = data.message_history; this.instructions = data.instructions; this.tags = data.tags; - this.created_at = new Date(data.created_at); + this.createdAt = new Date(data.created_at); } toString(): string { - return `Log(id="${this.id}", app_name="${this.app_name}", environment="${this.environment}", created_at="${this.created_at.toISOString()}")`; + return `Log(id="${this.id}", appName="${this.appName}", environment="${this.environment}", createdAt="${this.createdAt.toISOString()}")`; } } @@ -85,11 +104,41 @@ export class LogsResource { this.client = client; } - // create a log - // params: CreateLogParams + // Create a log async create(params: CreateLogParams): Promise { try { - const response = await this.client.post('/logs', params); + // Convert document objects with pageContent to page_content format for API + const convertedDocuments = params.documents.map(doc => { + if (typeof doc === 'string') { + return doc; + } else if (doc && typeof doc === 'object' && 'pageContent' in doc) { + const { pageContent, metadata } = doc; + return { + page_content: pageContent, + metadata + }; + } + return doc; + }); + + // Convert camelCase params to snake_case for API + const apiParams = { + id: params.id, + created_at: params.createdAt, + app_name: params.appName, + environment: params.environment, + hallucination_detection: params.hallucinationDetection, + inconsistency_detection: params.inconsistencyDetection, + user_query: params.userQuery, + model_output: params.modelOutput, + documents: convertedDocuments, + message_history: params.messageHistory, + instructions: params.instructions, + tags: params.tags, + hallucination_detection_sample_rate: params.hallucinationDetectionSampleRate + }; + + const response = await this.client.post('/logs', apiParams); return response; } catch (error) { logError(error as Error, 'LogsResource.create'); @@ -97,15 +146,15 @@ export class LogsResource { } } - // list logs - // params: ListLogsParams + // List logs async list(params: ListLogsParams = {}): Promise { + // Convert camelCase params to snake_case for API const queryParams: Record = {}; - if (params.app_name) queryParams.app_name = params.app_name; + if (params.appName) queryParams.app_name = params.appName; if (params.environment) queryParams.environment = params.environment; - if (params.start_date) queryParams.start_date = params.start_date.toISOString(); - if (params.end_date) queryParams.end_date = params.end_date.toISOString(); + if (params.startDate) queryParams.start_date = params.startDate.toISOString(); + if (params.endDate) queryParams.end_date = params.endDate.toISOString(); if (params.limit !== undefined) queryParams.limit = params.limit; if (params.offset !== undefined) queryParams.offset = params.offset; @@ -125,4 +174,183 @@ export class LogsResource { return []; } } + + /** + * Get detection results for a log + * @param logId The ID of the log to get detection results for + * @returns Promise resolving to the detection results when available + * @throws Error if the API call fails, to allow for proper retry handling + */ + async getDetections(logId: string): Promise { + try { + if (!logId) { + throw new Error('Log ID is required for detection polling'); + } + + // The path should match the Python implementation which uses `/logs/{log_id}/rca` + const path = `/logs/${logId}/rca`; + const response = await this.client.get(path) as DetectionResultsResponse; + + if (!response) { + return null; + } + + // Convert snake_case response to camelCase + return this.convertToDetectionResults(response); + } catch (error) { + return null; + } + } + + /** + * Converts snake_case API response to camelCase DetectionResults + */ + private convertToDetectionResults(response: DetectionResultsResponse): DetectionResults { + // Convert LogDetail + const logDetail: LogDetail = { + id: response.log.id, + createdAt: response.log.created_at, + appName: response.log.app_name, + environment: response.log.environment, + tags: response.log.tags, + inconsistencyDetection: response.log.inconsistency_detection, + hallucinationDetection: response.log.hallucination_detection, + userQuery: response.log.user_query, + modelOutput: response.log.model_output, + hallucinationDetectionSampleRate: response.log.hallucination_detection_sample_rate, + updatedAt: response.log.updated_at, + status: response.log.status, + hasHallucination: response.log.has_hallucination, + hasInconsistency: response.log.has_inconsistency, + documents: response.log.documents, + messageHistory: response.log.message_history, + instructions: response.log.instructions + }; + + // Convert LogDocuments + const logDocuments = response.log_documents?.map(doc => { + const documentLog: DocumentLog = { + id: doc.id, + content: doc.content, + metadata: doc.metadata, + logId: doc.log_id, + createdAt: doc.created_at, + updatedAt: doc.updated_at, + index: doc.index + }; + return documentLog; + }) || null; + + // Convert LogMessageHistory + const logMessageHistory = response.log_message_history?.map(msg => { + const messageHistory: LogMessageHistory = { + id: msg.id, + content: msg.content, + logId: msg.log_id, + createdAt: msg.created_at, + updatedAt: msg.updated_at, + index: msg.index + }; + return messageHistory; + }) || null; + + // Convert LogInstructions + const logInstructions = response.log_instructions?.map(inst => { + const instruction: LogInstruction = { + id: inst.id, + content: inst.content, + logId: inst.log_id, + createdAt: inst.created_at, + updatedAt: inst.updated_at, + index: inst.index + }; + return instruction; + }) || null; + + // Convert Evaluations + const evaluations = response.evaluations.map(evalItem => { + // Convert document evaluations + const documentEvaluations = evalItem.document_evaluations.map((docEval: DocumentEvaluationResponse) => { + const documentEvaluation: DocumentEvaluation = { + id: docEval.id, + evaluationId: docEval.evaluation_id, + reasoning: docEval.reasoning, + score: docEval.score, + index: docEval.index, + createdAt: docEval.created_at, + updatedAt: docEval.updated_at, + logDocumentId: docEval.log_document_id + }; + return documentEvaluation; + }); + + // Convert message history evaluations + const messageHistoryEvaluations = evalItem.message_history_evaluations.map((msgEval: MessageHistoryEvaluationResponse) => { + const messageHistoryEvaluation: MessageHistoryEvaluation = { + id: msgEval.id, + evaluationId: msgEval.evaluation_id, + reasoning: msgEval.reasoning, + score: msgEval.score, + index: msgEval.index, + createdAt: msgEval.created_at, + updatedAt: msgEval.updated_at, + logMessageHistoryId: msgEval.log_message_history_id + }; + return messageHistoryEvaluation; + }); + + // Convert instruction evaluations + const instructionEvaluations = evalItem.instruction_evaluations.map((instEval: InstructionEvaluationResponse) => { + const instructionEvaluation: InstructionEvaluation = { + id: instEval.id, + evaluationId: instEval.evaluation_id, + reasoning: instEval.reasoning, + score: instEval.score, + index: instEval.index, + createdAt: instEval.created_at, + updatedAt: instEval.updated_at, + logInstructionId: instEval.log_instruction_id + }; + return instructionEvaluation; + }); + + // Convert full doc context evaluation + const fullDocEval = evalItem.full_doc_context_evaluation; + const fullDocContextEvaluation: FullDocContextEvaluation = { + id: fullDocEval.id, + evaluationId: fullDocEval.evaluation_id, + reasoning: fullDocEval.reasoning, + score: fullDocEval.score, + index: fullDocEval.index, + createdAt: fullDocEval.created_at, + updatedAt: fullDocEval.updated_at, + logDocumentIds: fullDocEval.log_document_ids + }; + + const evaluation: Evaluation = { + id: evalItem.id, + sentence: evalItem.sentence, + supportingDocumentIds: evalItem.supporting_document_ids, + supportingMessageHistoryIds: evalItem.supporting_message_history_ids, + supportingInstructionIds: evalItem.supporting_instruction_ids, + isHallucinated: evalItem.is_hallucinated, + fullDocContextHasHallucination: evalItem.full_doc_context_has_hallucination, + index: evalItem.index, + documentEvaluations, + messageHistoryEvaluations, + instructionEvaluations, + fullDocContextEvaluation + }; + return evaluation; + }); + + // Construct and return the camelCase DetectionResults + return { + log: logDetail, + logDocuments, + logMessageHistory, + logInstructions, + evaluations + }; + } } \ No newline at end of file diff --git a/quotientai/resources/metrics.ts b/quotientai/resources/metrics.ts deleted file mode 100644 index a70665f..0000000 --- a/quotientai/resources/metrics.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { BaseQuotientClient } from '../client'; - -interface MetricsResponse { - data: string[]; -} - -export class MetricsResource { - protected client: BaseQuotientClient; - - constructor(client: BaseQuotientClient) { - this.client = client; - } - - // list all metrics - // no params - async list(): Promise { - const response = await this.client.get('/runs/metrics') as MetricsResponse; - return response.data; - } -} \ No newline at end of file diff --git a/quotientai/resources/models.ts b/quotientai/resources/models.ts deleted file mode 100644 index c7623f2..0000000 --- a/quotientai/resources/models.ts +++ /dev/null @@ -1,42 +0,0 @@ -import { logError } from '../exceptions'; -import { BaseQuotientClient } from '../client'; -import { Model, ModelProvider } from '../types'; - -interface ModelResponse { - id: string; - name: string; - provider: ModelProvider; - created_at: string; -} - -export class ModelsResource { - protected client: BaseQuotientClient; - - constructor(client: BaseQuotientClient) { - this.client = client; - } - - // list all models - // no params - async list(): Promise { - const response = await this.client.get('/models') as ModelResponse[]; - return response.map(model => ({ - ...model, - created_at: new Date(model.created_at) - })); - } - - // get a model - // name: string - async getModel(name: string): Promise { - const response = await this.client.get(`/models/${name}`) as ModelResponse; - if (!response) { - logError(new Error(`Model with name ${name} not found. Please check the list of available models using quotient.models.list()`)); - return null; - } - return { - ...response, - created_at: new Date(response.created_at) - }; - } -} \ No newline at end of file diff --git a/quotientai/resources/prompts.ts b/quotientai/resources/prompts.ts deleted file mode 100644 index 01b63ed..0000000 --- a/quotientai/resources/prompts.ts +++ /dev/null @@ -1,135 +0,0 @@ -import { logError } from '../exceptions'; -import { BaseQuotientClient } from '../client'; -import { Prompt } from '../types'; - -interface PromptResponse { - id: string; - name: string; - content: string; - version: number; - system_prompt?: string; - user_prompt: string; - created_at: string; - updated_at: string; -} - -interface GetPromptParams { - id: string; - version?: string; -} - -interface CreatePromptParams { - name: string; - system_prompt?: string; - user_prompt?: string; -} - -interface UpdatePromptParams { - prompt: Prompt; -} - - -export class PromptsResource { - protected client: BaseQuotientClient; - - constructor(client: BaseQuotientClient) { - this.client = client; - } - - // list all prompts - // no params - async list(): Promise { - const response = await this.client.get('/prompts') as PromptResponse[]; - const prompts: Prompt[] = []; - - for (const prompt of response) { - prompts.push({ - id: prompt.id, - name: prompt.name, - content: prompt.content, - version: prompt.version, - system_prompt: prompt.system_prompt, - user_prompt: prompt.user_prompt, - created_at: new Date(prompt.created_at), - updated_at: new Date(prompt.updated_at) - }); - } - return prompts; - } - - // get a prompt - // options: GetPromptParams - async getPrompt(options: GetPromptParams): Promise { - let path = `/prompts/${options.id}`; - if (options.version) { - path += `/versions/${options.version}`; - } - const response = await this.client.get(path) as PromptResponse; - if (!response) { - logError(new Error('Prompt not found')); - return null; - } - return { - id: response.id, - name: response.name, - content: response.content, - version: response.version, - system_prompt: response.system_prompt, - user_prompt: response.user_prompt, - created_at: new Date(response.created_at), - updated_at: new Date(response.updated_at) - }; - } - - // create a prompt - // options: CreatePromptParams - async create(params: CreatePromptParams): Promise { - const response = await this.client.post('/prompts', { - name: params.name, - system_prompt: params.system_prompt, - user_prompt: params.user_prompt - }) as PromptResponse; - return { - id: response.id, - name: response.name, - content: response.content, - version: response.version, - system_prompt: response.system_prompt, - user_prompt: response.user_prompt, - created_at: new Date(response.created_at), - updated_at: new Date(response.updated_at) - }; - } - - // update a prompt - // options: UpdatePromptParams - async update(options: UpdatePromptParams): Promise { - const response = await this.client.patch(`/prompts/${options.prompt.id}`, { - name: options.prompt.name, - system_prompt: options.prompt.system_prompt, - user_prompt: options.prompt.user_prompt - }) as PromptResponse; - return { - id: response.id, - name: response.name, - content: response.content, - version: response.version, - system_prompt: response.system_prompt, - user_prompt: response.user_prompt, - created_at: new Date(response.created_at), - updated_at: new Date(response.updated_at) - }; - } - - // delete a prompt - // options: UpdatePromptParams - async deletePrompt(options: UpdatePromptParams): Promise { - await this.client.patch(`/prompts/${options.prompt.id}`, { - id: options.prompt.id, - name: options.prompt.name, - system_prompt: options.prompt.system_prompt, - user_prompt: options.prompt.user_prompt, - is_deleted: true - }); - } -} diff --git a/quotientai/resources/runs.ts b/quotientai/resources/runs.ts deleted file mode 100644 index 32dc33f..0000000 --- a/quotientai/resources/runs.ts +++ /dev/null @@ -1,186 +0,0 @@ -import { logError } from '../exceptions'; -import { BaseQuotientClient } from '../client'; -import { RunResult, Prompt, Dataset, Model } from '../types'; - -interface RunResponse { - id: string; - prompt: string; - dataset: string; - model: string; - parameters: Record; - metrics: string[]; - status: string; - results: RunResult[]; - created_at: string; - finished_at: string | null; -} - -interface MetricSummary { - avg: number; - stddev: number; -} - -interface RunSummary { - id: string; - model: string; - parameters: Record; - metrics: Record; - created_at: Date; - [key: string]: any; -} - -interface CreateRunParams { - prompt: Prompt; - dataset: Dataset; - model: Model; - parameters: Record; - metrics: string[]; -} - - -export class Run { - id: string; - prompt: string; - dataset: string; - model: string; - parameters: Record; - metrics: string[]; - status: string; - results: RunResult[]; - created_at: Date; - finished_at?: Date; - - constructor(private client: BaseQuotientClient, data: RunResponse) { - this.id = data.id; - this.prompt = data.prompt; - this.dataset = data.dataset; - this.model = data.model; - this.parameters = data.parameters; - this.metrics = data.metrics; - this.status = data.status; - this.results = data.results; - this.created_at = new Date(data.created_at); - this.finished_at = data.finished_at ? new Date(data.finished_at) : undefined; - } - - // summarize the run - // best_n: number = 3, worst_n: number = 3 - summarize(best_n: number = 3, worst_n: number = 3): RunSummary | null { - if (!this.results || this.results.length === 0) { - return null; - } - - // Calculate metrics for each result - const resultMetrics = this.results.map(result => { - const avg = this.metrics.reduce((sum, metric) => sum + result.values[metric], 0) / this.metrics.length; - return { result, avg }; - }); - - // Sort results by average metric value - resultMetrics.sort((a, b) => b.avg - a.avg); - - // Get best and worst results - const bestResults = best_n > 0 ? resultMetrics.slice(0, best_n).map(r => r.result) : undefined; - const worstResults = worst_n > 0 ? resultMetrics.slice(-worst_n).map(r => r.result) : undefined; - - // Calculate metrics summary - const metricsSummary = this.metrics.reduce((acc, metric) => { - const values = this.results.map(r => r.values[metric]); - const avg = values.reduce((sum, val) => sum + val, 0) / values.length; - const variance = values.reduce((sum, val) => sum + Math.pow(val - avg, 2), 0) / values.length; - const stddev = Math.sqrt(variance); - - acc[metric] = { avg, stddev }; - return acc; - }, {} as Record); - - return { - id: this.id, - model: this.model, - parameters: this.parameters, - metrics: metricsSummary, - created_at: this.created_at, - ...(bestResults && { [`best_${best_n}`]: bestResults }), - ...(worstResults && { [`worst_${worst_n}`]: worstResults }) - }; - } -} - -export class RunsResource { - protected client: BaseQuotientClient; - - constructor(client: BaseQuotientClient) { - this.client = client; - } - - // list all runs - // no params - async list(): Promise[]> { - const response = await this.client.get('/runs') as RunResponse[]; - return response.map(runData => new Run(this.client, runData)); - } - - // get a run - // run_id: string - async get(run_id: string): Promise | null> { - const response = await this.client.get(`/runs/${run_id}`) as RunResponse; - return new Run(this.client, response); - } - - // create a run - // options: CreateRunParams - async create(options: CreateRunParams): Promise | null> { - const response = await this.client.post('/runs', { - prompt_id: options.prompt.id, - dataset_id: options.dataset.id, - model_id: options.model.id, - parameters: options.parameters, - metrics: options.metrics - }) as RunResponse; - return new Run(this.client, response); - } - - // compare runs - // runs: Run[] - async compare(runs: Run[]): Promise | null | undefined> { - if (runs.length <= 1) { - return null; - } - - if (new Set(runs.map(run => run.dataset)).size > 1) { - logError(new Error("All runs must be on the same dataset to compare them")); - return null; - } - - if (new Set(runs.map(run => run.prompt)).size > 1 && - new Set(runs.map(run => run.model)).size > 1) { - logError(new Error("All runs must be on the same prompt or model to compare them")); - return null; - } - - const summaries = runs.map(run => run.summarize()); - - if (runs.length === 2) { - const comparison: Record = {}; - for (const metric of runs[0].metrics) { - comparison[metric] = { - avg: summaries[0]!.metrics[metric].avg - summaries[1]!.metrics[metric].avg, - stddev: summaries[0]!.metrics[metric].stddev - }; - } - return comparison; - } else if (runs.length > 2) { - const comparison: Record> = {}; - for (const run of runs) { - comparison[run.id] = {}; - for (const metric of run.metrics) { - comparison[run.id][metric] = { - avg: summaries[0]!.metrics[metric].avg - summaries[1]!.metrics[metric].avg, - stddev: summaries[0]!.metrics[metric].stddev - }; - } - } - return comparison; - } - } -} \ No newline at end of file diff --git a/quotientai/types.ts b/quotientai/types.ts index 4e5f1fd..4ea5516 100644 --- a/quotientai/types.ts +++ b/quotientai/types.ts @@ -15,97 +15,252 @@ export interface BaseResource { client: BaseQuotientClient; } -export interface Prompt { +export interface LogDocument { + pageContent: string; + metadata?: Record; +} + +export interface LogEntry { + id?: string; + createdAt?: string | Date; + appName: string; + environment: string; + userQuery: string; + modelOutput: string; + documents: (string | LogDocument)[]; + messageHistory?: Array> | null; + instructions?: string[] | null; + tags?: Record; + hallucinationDetection: boolean; + hallucinationDetectionSampleRate?: number; + inconsistencyDetection: boolean; +} + +export interface LoggerConfig { + id?: string; + createdAt?: string | Date; + appName: string; + environment: string; + tags?: Record; + sampleRate?: number; + hallucinationDetection?: boolean; + inconsistencyDetection?: boolean; + hallucinationDetectionSampleRate?: number; +} + +export enum LOG_STATUS { + LOG_NOT_FOUND = "log_not_found", + LOG_CREATION_IN_PROGRESS = "log_creation_in_progress", + LOG_CREATED_NO_DETECTIONS_PENDING = "log_created_no_detections_pending", + LOG_CREATED_AND_DETECTION_IN_PROGRESS = "log_created_and_detection_in_progress", + LOG_CREATED_AND_DETECTION_COMPLETED = "log_created_and_detection_completed", +} + +export enum EVALUATION_SCORE { + PASS = "PASS", + FAIL = "FAIL", + INCONCLUSIVE = "INCONCLUSIVE", +} + +export interface QuotientAIError extends Error { + status?: number; + code?: string; +} + +// Common evaluation properties - API Response format (snake_case) +export interface BaseEvaluationResponse { + id: string; + evaluation_id: string; + reasoning: string; + score: EVALUATION_SCORE; + index: number; + created_at: string; + updated_at: string; +} + +// Snake case interfaces - API Response format +export interface DocumentEvaluationResponse extends BaseEvaluationResponse { + log_document_id: string; +} + +export interface MessageHistoryEvaluationResponse extends BaseEvaluationResponse { + log_message_history_id: string; +} + +export interface InstructionEvaluationResponse extends BaseEvaluationResponse { + log_instruction_id: string; +} + +export interface FullDocContextEvaluationResponse extends BaseEvaluationResponse { + log_document_ids: string[]; +} + +export interface EvaluationResponse { + id: string; + sentence: string; + supporting_document_ids: string[]; + supporting_message_history_ids: string[]; + supporting_instruction_ids: string[]; + is_hallucinated: boolean; + full_doc_context_has_hallucination: boolean; + index: number; + document_evaluations: DocumentEvaluationResponse[]; + message_history_evaluations: MessageHistoryEvaluationResponse[]; + instruction_evaluations: InstructionEvaluationResponse[]; + full_doc_context_evaluation: FullDocContextEvaluationResponse; +} + +export interface LogDetailResponse { + id: string; + created_at: string; + app_name: string; + environment: string; + tags?: Record; + inconsistency_detection: boolean; + hallucination_detection: boolean; + user_query: string; + model_output: string; + hallucination_detection_sample_rate: number; + updated_at: string; + status: string; + has_hallucination: boolean | null; + has_inconsistency: boolean | null; + documents: any[] | null; + message_history: any[] | null; + instructions: any[] | null; +} + +export interface DocumentLogResponse { id: string; - name: string; content: string; - version: number; - user_prompt: string; - created_at: Date; - updated_at: Date; - system_prompt?: string; + metadata: Record | null; + log_id: string; + created_at: string; + updated_at: string; + index: number; } -export interface ModelProvider { +export interface LogMessageHistoryResponse { id: string; - name: string; + content: Record; + log_id: string; + created_at: string; + updated_at: string; + index: number; } -export interface Model { +export interface LogInstructionResponse { id: string; - name: string; - provider: ModelProvider; - created_at: Date; + content: string; + log_id: string; + created_at: string; + updated_at: string; + index: number; +} + +export interface DetectionResultsResponse { + log: LogDetailResponse; + log_documents: DocumentLogResponse[] | null; + log_message_history: LogMessageHistoryResponse[] | null; + log_instructions: LogInstructionResponse[] | null; + evaluations: EvaluationResponse[]; } -export interface Dataset { +// Common evaluation properties - Client side format (camelCase) +export interface BaseEvaluation { id: string; - name: string; - created_by: string; - created_at: Date; - updated_at: Date; - description?: string; - rows?: DatasetRow[]; + evaluationId: string; + reasoning: string; + score: EVALUATION_SCORE; + index: number; + createdAt: string; + updatedAt: string; +} + +// CamelCase interfaces - Client side format +export interface DocumentEvaluation extends BaseEvaluation { + logDocumentId: string; +} + +export interface MessageHistoryEvaluation extends BaseEvaluation { + logMessageHistoryId: string; +} +export interface InstructionEvaluation extends BaseEvaluation { + logInstructionId: string; } -export interface DatasetRowMetadata { - annotation?: string; - annotation_note?: string +export interface FullDocContextEvaluation extends BaseEvaluation { + logDocumentIds: string[]; } -export interface DatasetRow { +export interface Evaluation { id: string; - input: string; - context?: string; - expected?: string; - metadata: DatasetRowMetadata; - created_by: string; - created_at: Date; - updated_at: Date; + sentence: string; + supportingDocumentIds: string[]; + supportingMessageHistoryIds: string[]; + supportingInstructionIds: string[]; + isHallucinated: boolean; + fullDocContextHasHallucination: boolean; + index: number; + documentEvaluations: DocumentEvaluation[]; + messageHistoryEvaluations: MessageHistoryEvaluation[]; + instructionEvaluations: InstructionEvaluation[]; + fullDocContextEvaluation: FullDocContextEvaluation; } -export interface RunResult { +export interface LogDetail { id: string; - input: string; - output: string; - values: Record; - created_at: Date; - created_by: string; - context: string; - expected: string; + createdAt: string; + appName: string; + environment: string; + tags?: Record; + inconsistencyDetection: boolean; + hallucinationDetection: boolean; + userQuery: string; + modelOutput: string; + hallucinationDetectionSampleRate: number; + updatedAt: string; + status: string; + hasHallucination: boolean | null; + hasInconsistency: boolean | null; + documents: any[] | null; + messageHistory: any[] | null; + instructions: any[] | null; } -export interface LogDocument { - page_content: string; - metadata?: Record; +export interface DocumentLog { + id: string; + content: string; + metadata: Record | null; + logId: string; + createdAt: string; + updatedAt: string; + index: number; } -export interface LogEntry { - app_name: string; - environment: string; - user_query: string; - model_output: string; - documents: (string | LogDocument)[]; - message_history?: Array> | null; - instructions?: string[] | null; - tags?: Record; - hallucination_detection: boolean; - inconsistency_detection: boolean; - hallucination_detection_sample_rate?: number; +export interface LogMessageHistory { + id: string; + content: Record; + logId: string; + createdAt: string; + updatedAt: string; + index: number; } -export interface LoggerConfig { - app_name: string; - environment: string; - tags?: Record; - sample_rate?: number; - hallucination_detection?: boolean; - inconsistency_detection?: boolean; - hallucination_detection_sample_rate?: number; +export interface LogInstruction { + id: string; + content: string; + logId: string; + createdAt: string; + updatedAt: string; + index: number; } -export interface QuotientAIError extends Error { - status?: number; - code?: string; -} \ No newline at end of file +export interface DetectionResults { + log: LogDetail; + logDocuments: DocumentLog[] | null; + logMessageHistory: LogMessageHistory[] | null; + logInstructions: LogInstruction[] | null; + evaluations: Evaluation[]; +} \ No newline at end of file diff --git a/tests/logger.test.ts b/tests/logger.test.ts index f9fb9fc..bed98b9 100644 --- a/tests/logger.test.ts +++ b/tests/logger.test.ts @@ -31,7 +31,7 @@ describe('QuotientLogger', () => { const privateLogger = logger as any; privateLogger.init({ - app_name: 'test_app', + appName: 'test_app', environment: 'test_environment' }); @@ -50,13 +50,13 @@ describe('QuotientLogger', () => { const privateLogger = logger as any; privateLogger.init({ - app_name: 'test_app', + appName: 'test_app', environment: 'test_environment', tags: { test: 'test' }, - sample_rate: 0.5, - hallucination_detection: true, - inconsistency_detection: true, - hallucination_detection_sample_rate: 0.5, + sampleRate: 0.5, + hallucinationDetection: true, + inconsistencyDetection: true, + hallucinationDetectionSampleRate: 0.5, }); expect(privateLogger.appName).toBe('test_app'); expect(privateLogger.environment).toBe('test_environment'); @@ -72,9 +72,9 @@ describe('QuotientLogger', () => { const mockLogsResource = { create: vi.fn(), list: vi.fn() }; const logger = new QuotientLogger(mockLogsResource); const privateLogger = logger as any; - const result = privateLogger.init({ sample_rate: 1.5 }); + const result = privateLogger.init({ sampleRate: 1.5 }); expect(result).toBe(logger); - expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('sample_rate must be between 0.0 and 1.0')); + expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('sampleRate must be between 0.0 and 1.0')); }); it('should log error and return null if you attempt to log before initializing', async () => { @@ -91,16 +91,16 @@ describe('QuotientLogger', () => { const mockLogsResource = { create: vi.fn(), list: vi.fn() }; const logger = new QuotientLogger(mockLogsResource); const privateLogger = logger as any; - privateLogger.init({ app_name: 'test_app', environment: 'test_environment', tags: { test: 'test' }, sample_rate: 1.0 }); + privateLogger.init({ appName: 'test_app', environment: 'test_environment', tags: { test: 'test' }, sampleRate: 1.0 }); await privateLogger.log({ message: 'test' }); expect(mockLogsResource.create).toHaveBeenCalledWith({ - app_name: 'test_app', + appName: 'test_app', environment: 'test_environment', tags: { test: 'test' }, message: 'test', - hallucination_detection: false, - hallucination_detection_sample_rate: 0, - inconsistency_detection: false + hallucinationDetection: false, + hallucinationDetectionSampleRate: 0, + inconsistencyDetection: false }); }); @@ -111,7 +111,7 @@ describe('QuotientLogger', () => { // Mock shouldSample to always return false vi.spyOn(privateLogger, 'shouldSample').mockReturnValue(false); - privateLogger.init({ app_name: 'test_app', environment: 'test_environment', tags: { test: 'test' }, sample_rate: 0.5 }); + privateLogger.init({ appName: 'test_app', environment: 'test_environment', tags: { test: 'test' }, sampleRate: 0.5 }); await privateLogger.log({ message: 'test' }); expect(mockLogsResource.create).not.toHaveBeenCalled(); }); @@ -120,7 +120,7 @@ describe('QuotientLogger', () => { const mockLogsResource = { create: vi.fn(), list: vi.fn() }; const logger = new QuotientLogger(mockLogsResource); const privateLogger = logger as any; - privateLogger.init({ sample_rate: 0.5 }); + privateLogger.init({ sampleRate: 0.5 }); // Test when random is less than sample rate vi.spyOn(Math, 'random').mockReturnValue(0.4); @@ -133,7 +133,7 @@ describe('QuotientLogger', () => { vi.spyOn(Math, 'random').mockRestore(); }); - it('should log error and return null if required app_name or environment is missing after initialization', async () => { + it('should log error and return null if required appName or environment is missing after initialization', async () => { const mockLogsResource = { create: vi.fn(), list: vi.fn() }; const logger = new QuotientLogger(mockLogsResource); const privateLogger = logger as any; @@ -146,7 +146,7 @@ describe('QuotientLogger', () => { const result1 = await privateLogger.log({ message: 'test' }); expect(result1).toBeNull(); - expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('app_name and environment must be set')); + expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('appName and environment must be set')); expect(mockLogsResource.create).not.toHaveBeenCalled(); privateLogger.appName = 'test'; @@ -154,7 +154,7 @@ describe('QuotientLogger', () => { const result2 = await privateLogger.log({ message: 'test' }); expect(result2).toBeNull(); - expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('app_name and environment must be set')); + expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('appName and environment must be set')); expect(mockLogsResource.create).not.toHaveBeenCalled(); }); @@ -164,11 +164,11 @@ describe('QuotientLogger', () => { const logger = new QuotientLogger(mockLogsResource); const privateLogger = logger as any; - privateLogger.init({ app_name: 'test_app', environment: 'test_environment' }); + privateLogger.init({ appName: 'test_app', environment: 'test_environment' }); await privateLogger.log({ - user_query: 'test', - model_output: 'test', + userQuery: 'test', + modelOutput: 'test', documents: ['This is a string document'] }); @@ -180,54 +180,52 @@ describe('QuotientLogger', () => { const logger = new QuotientLogger(mockLogsResource); const privateLogger = logger as any; - privateLogger.init({ app_name: 'test_app', environment: 'test_environment' }); + privateLogger.init({ appName: 'test_app', environment: 'test_environment' }); await privateLogger.log({ - user_query: 'test', - model_output: 'test', + userQuery: 'test', + modelOutput: 'test', documents: [ - { page_content: 'This is valid content' }, - { page_content: 'Also valid', metadata: { source: 'test' } } + { pageContent: 'This is valid content' }, + { pageContent: 'Also valid', metadata: { source: 'test' } } ] }); expect(mockLogsResource.create).toHaveBeenCalled(); }); - it('should log error and return null when document is missing page_content', async () => { + it('should log error and return null when document is missing pageContent', async () => { const mockLogsResource = { create: vi.fn(), list: vi.fn() }; const logger = new QuotientLogger(mockLogsResource); const privateLogger = logger as any; - privateLogger.init({ app_name: 'test_app', environment: 'test_environment' }); + privateLogger.init({ appName: 'test_app', environment: 'test_environment' }); - const result = await privateLogger.log({ - user_query: 'test', - model_output: 'test', - documents: [{ not_page_content: 'invalid' }] + await privateLogger.log({ + userQuery: 'test', + modelOutput: 'test', + documents: [{}] }); - expect(result).toBeNull(); - expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('Missing required \'page_content\' property')); expect(mockLogsResource.create).not.toHaveBeenCalled(); + expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining("Missing required 'pageContent' property")); }); - it('should log error and return null when page_content is not a string', async () => { + it('should log error and return null when pageContent is not a string', async () => { const mockLogsResource = { create: vi.fn(), list: vi.fn() }; const logger = new QuotientLogger(mockLogsResource); const privateLogger = logger as any; - privateLogger.init({ app_name: 'test_app', environment: 'test_environment' }); + privateLogger.init({ appName: 'test_app', environment: 'test_environment' }); - const result = await privateLogger.log({ - user_query: 'test', - model_output: 'test', - documents: [{ page_content: 123 }] + await privateLogger.log({ + userQuery: 'test', + modelOutput: 'test', + documents: [{ pageContent: 123 }] }); - expect(result).toBeNull(); - expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('page_content\' property must be a string')); expect(mockLogsResource.create).not.toHaveBeenCalled(); + expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining("The 'pageContent' property must be a string")); }); it('should log error and return null when metadata is not an object', async () => { @@ -235,142 +233,100 @@ describe('QuotientLogger', () => { const logger = new QuotientLogger(mockLogsResource); const privateLogger = logger as any; - privateLogger.init({ app_name: 'test_app', environment: 'test_environment' }); + privateLogger.init({ appName: 'test_app', environment: 'test_environment' }); - const result = await privateLogger.log({ - user_query: 'test', - model_output: 'test', - documents: [{ page_content: 'Valid content', metadata: 'not an object' }] + await privateLogger.log({ + userQuery: 'test', + modelOutput: 'test', + documents: [{ pageContent: 'test', metadata: 'not-object' }] }); - expect(result).toBeNull(); - expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('metadata\' property must be an object')); expect(mockLogsResource.create).not.toHaveBeenCalled(); + expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining("The 'metadata' property must be an object")); }); - it('should log error and return null when document is not a string or object', async () => { + it('should accept null metadata', async () => { const mockLogsResource = { create: vi.fn(), list: vi.fn() }; const logger = new QuotientLogger(mockLogsResource); const privateLogger = logger as any; - privateLogger.init({ app_name: 'test_app', environment: 'test_environment' }); + privateLogger.init({ appName: 'test_app', environment: 'test_environment' }); - const result = await privateLogger.log({ - user_query: 'test', - model_output: 'test', - documents: [123] + await privateLogger.log({ + userQuery: 'test', + modelOutput: 'test', + documents: [{ pageContent: 'test', metadata: null }] }); - expect(result).toBeNull(); - expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('documents must be either strings or JSON objects')); - expect(mockLogsResource.create).not.toHaveBeenCalled(); + expect(mockLogsResource.create).toHaveBeenCalled(); }); - it('should handle unexpected errors during validation', async () => { + it('should validate documents as part of the log method', async () => { const mockLogsResource = { create: vi.fn(), list: vi.fn() }; const logger = new QuotientLogger(mockLogsResource); const privateLogger = logger as any; - privateLogger.init({ app_name: 'test_app', environment: 'test_environment' }); + privateLogger.init({ appName: 'test_app', environment: 'test_environment' }); - // Create a malformed object that will cause an error during validation - const malformedObj = new Proxy({}, { - get: () => { throw new Error('Unexpected error'); } - }); + const validateSpy = vi.spyOn(privateLogger, 'validateDocuments'); + validateSpy.mockReturnValue(false); - const result = await privateLogger.log({ - user_query: 'test', - model_output: 'test', - documents: [malformedObj] + await privateLogger.log({ + userQuery: 'test', + modelOutput: 'test', + documents: ['test'] }); - expect(result).toBeNull(); - expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('ValidationError: Invalid document format')); + expect(validateSpy).toHaveBeenCalled(); expect(mockLogsResource.create).not.toHaveBeenCalled(); }); - - it('should handle unexpected errors during document validation', async () => { + + it('should skip validation if no documents are provided', async () => { const mockLogsResource = { create: vi.fn(), list: vi.fn() }; const logger = new QuotientLogger(mockLogsResource); const privateLogger = logger as any; - privateLogger.init({ app_name: 'test_app', environment: 'test_environment' }); + privateLogger.init({ appName: 'test_app', environment: 'test_environment' }); - // Create a document that will cause an unexpected error during validation - const problematicDoc = { - get page_content() { - throw new Error('Unexpected error during validation'); - } - }; + const validateSpy = vi.spyOn(privateLogger, 'validateDocuments'); - const result = await privateLogger.log({ - user_query: 'test', - model_output: 'test', - documents: [problematicDoc] + await privateLogger.log({ + userQuery: 'test', + modelOutput: 'test' }); - expect(result).toBeNull(); - expect(mockLogsResource.create).not.toHaveBeenCalled(); - expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('Invalid document format')); + expect(validateSpy).not.toHaveBeenCalled(); + expect(mockLogsResource.create).toHaveBeenCalled(); }); - }); - - describe('Direct Method Testing', () => { + it('should directly test isValidLogDocument with various inputs', () => { const mockLogsResource = { create: vi.fn(), list: vi.fn() }; const logger = new QuotientLogger(mockLogsResource); const privateLogger = logger as any; // Valid document - expect(privateLogger.isValidLogDocument({ page_content: 'test' })).toEqual({ valid: true }); + expect(privateLogger.isValidLogDocument({ pageContent: 'test' })).toEqual({ valid: true }); - // Missing page_content + // Missing pageContent expect(privateLogger.isValidLogDocument({})).toEqual({ valid: false, - error: "Missing required 'page_content' property" + error: "Missing required 'pageContent' property" }); - // Non-string page_content - expect(privateLogger.isValidLogDocument({ page_content: 123 })).toEqual({ + // pageContent not a string + expect(privateLogger.isValidLogDocument({ pageContent: 123 })).toEqual({ valid: false, - error: "The 'page_content' property must be a string, found number" + error: "The 'pageContent' property must be a string, found number" }); - // Invalid metadata - expect(privateLogger.isValidLogDocument({ page_content: 'test', metadata: 'not-object' })).toEqual({ + // metadata not an object + expect(privateLogger.isValidLogDocument({ pageContent: 'test', metadata: 'not-object' })).toEqual({ valid: false, error: "The 'metadata' property must be an object, found string" }); - // null metadata should be valid - expect(privateLogger.isValidLogDocument({ page_content: 'test', metadata: null })).toEqual({ valid: true }); - }); - - it('should directly test validateDocuments with various inputs', () => { - const mockLogsResource = { create: vi.fn(), list: vi.fn() }; - const logger = new QuotientLogger(mockLogsResource); - const privateLogger = logger as any; - - // Empty array should not throw - expect(() => privateLogger.validateDocuments([])).not.toThrow(); - - // Null should not throw - expect(() => privateLogger.validateDocuments(null)).not.toThrow(); - - // Valid strings should not throw - expect(() => privateLogger.validateDocuments(['test', 'test2'])).not.toThrow(); - - // Valid objects should not throw - expect(() => privateLogger.validateDocuments([ - { page_content: 'test' }, - { page_content: 'test2', metadata: { source: 'test' } } - ])).not.toThrow(); - - // Mixed valid types should not throw - expect(() => privateLogger.validateDocuments([ - 'test', - { page_content: 'test' } - ])).not.toThrow(); + // null metadata is valid + expect(privateLogger.isValidLogDocument({ pageContent: 'test', metadata: null })).toEqual({ valid: true }); }); }); }); diff --git a/tests/resources/datasets.test.ts b/tests/resources/datasets.test.ts deleted file mode 100644 index 8798037..0000000 --- a/tests/resources/datasets.test.ts +++ /dev/null @@ -1,887 +0,0 @@ -import { describe, it, expect, vi } from 'vitest'; -import { DatasetsResource, DatasetRowResponse } from '../../quotientai/resources/datasets'; -import { BaseQuotientClient } from '../../quotientai/client'; - -describe('DatasetsResource', () => { - it('should list datasets', async () => { - const client = new BaseQuotientClient('test'); - const mockDate = new Date('2024-01-01').toISOString(); - - const getMock = vi.spyOn(client, 'get'); - getMock.mockImplementation((path: string) => { - if (path === '/datasets') { - return Promise.resolve([{ - id: 'test_id', - name: 'test_dataset', - created_by: 'test_user', - created_at: mockDate, - updated_at: mockDate, - description: "test description" - }]); - } - if (path === '/datasets/test_id/rows') { - return Promise.resolve([{ - id: 'test_row_id', - dataset_id: 'test_id', - created_at: mockDate, - updated_at: mockDate, - input: 'test input', - context: 'test context', - expected: 'test expected', - annotation: 'test annotation', - annotation_note: 'test note', - created_by: 'test_user' - }]); - } - return Promise.resolve([]); - }); - - const datasetsResource = new DatasetsResource(client); - const datasets = await datasetsResource.list(); - - expect(datasets).toBeDefined(); - expect(datasets).toHaveLength(1); - expect(datasets[0]).toEqual({ - id: 'test_id', - name: 'test_dataset', - created_by: 'test_user', - created_at: new Date(mockDate), - updated_at: new Date(mockDate), - description: "test description", - rows: undefined - }); - expect(client.get).toHaveBeenCalledWith('/datasets'); - }); - - it('should list datasets with rows', async () => { - const client = new BaseQuotientClient('test'); - const mockDate = new Date('2024-01-01').toISOString(); - - const getMock = vi.spyOn(client, 'get'); - getMock.mockImplementation((path: string) => { - if (path === '/datasets') { - return Promise.resolve([{ - id: 'test_id', - name: 'test_dataset', - created_by: 'test_user', - created_at: mockDate, - updated_at: mockDate, - description: 'test description' - }]); - } - if (path === '/datasets/test_id/rows') { - return Promise.resolve([{ - id: 'test_row_id', - input: 'test input', - context: 'test context', - expected: 'test expected', - annotation: 'test annotation', - annotation_note: 'test note', - created_by: 'test_user', - created_at: mockDate, - updated_at: mockDate - }]); - } - return Promise.resolve([]); - }); - - const datasetsResource = new DatasetsResource(client); - const datasets = await datasetsResource.list({ includeRows: true }); - - expect(datasets).toBeDefined(); - expect(datasets).toHaveLength(1); - expect(datasets[0].rows).toBeDefined(); - expect(datasets[0].rows).toHaveLength(1); - expect(datasets[0].rows?.[0]).toEqual({ - id: 'test_row_id', - input: 'test input', - context: 'test context', - expected: 'test expected', - metadata: { - annotation: 'test annotation', - annotation_note: 'test note' - }, - created_by: 'test_user', - created_at: new Date(mockDate), - updated_at: new Date(mockDate) - }); - expect(client.get).toHaveBeenCalledWith('/datasets'); - expect(client.get).toHaveBeenCalledWith('/datasets/test_id/rows'); - }); - - it('should convert dates to Date objects', async () => { - const client = new BaseQuotientClient('test'); - const mockDate = new Date('2024-01-01').toISOString(); - - const getMock = vi.spyOn(client, 'get'); - getMock.mockImplementation((path: string) => { - if (path === '/datasets') { - return Promise.resolve([{ - id: 'test_id', - name: 'test_dataset', - created_by: 'test_user', - created_at: mockDate, - updated_at: mockDate, - description: 'test description' - }]); - } - return Promise.resolve([]); - }); - - const datasetsResource = new DatasetsResource(client); - const datasets = await datasetsResource.list(); - - expect(datasets).toBeDefined(); - expect(datasets).toHaveLength(1); - expect(datasets[0].created_at).toBeInstanceOf(Date); - expect(datasets[0].updated_at).toBeInstanceOf(Date); - }) - - it('should convert rows dates to Date objects', async () => { - const client = new BaseQuotientClient('test'); - const mockDate = new Date('2024-01-01').toISOString(); - - const getMock = vi.spyOn(client, 'get'); - getMock.mockImplementation((path: string) => { - if (path === '/datasets') { - return Promise.resolve([{ - id: 'test_id', - name: 'test_dataset', - created_by: 'test_user', - created_at: mockDate, - updated_at: mockDate, - description: 'test description' - }]); - } - if (path === '/datasets/test_id/rows') { - return Promise.resolve([{ - id: 'test_row_id', - dataset_id: 'test_id', - created_at: mockDate, - updated_at: mockDate, - input: 'test input', - context: 'test context', - expected: 'test expected', - annotation: 'test annotation', - annotation_note: 'test note', - created_by: 'test_user' - }]); - } - return Promise.resolve([]); - }); - - const datasetsResource = new DatasetsResource(client); - const datasets = await datasetsResource.list({ includeRows: true }); - - expect(datasets).toBeDefined(); - expect(datasets).toHaveLength(1); - expect(datasets[0].rows?.[0].created_at).toBeInstanceOf(Date); - expect(datasets[0].rows?.[0].updated_at).toBeInstanceOf(Date); - }) - - it('should get a dataset', async () => { - const client = new BaseQuotientClient('test'); - const mockDate = new Date('2024-01-01').toISOString(); - - const getMock = vi.spyOn(client, 'get'); - getMock.mockImplementation((path: string) => { - if (path === '/datasets/test_id') { - return Promise.resolve({ - id: 'test_id', - name: 'test_dataset', - created_by: 'test_user', - created_at: mockDate, - updated_at: mockDate, - description: 'test description', - dataset_rows: [{ - id: 'test_row_id', - input: 'test input', - context: 'test context', - expected: 'test expected', - annotation: 'test annotation', - annotation_note: 'test note', - created_by: 'test_user', - created_at: mockDate, - updated_at: mockDate - }] - }); - } - return Promise.resolve([]); - }); - - const datasetsResource = new DatasetsResource(client); - const dataset = await datasetsResource.get('test_id'); - - expect(dataset).toBeDefined(); - expect(dataset.created_at).toBeInstanceOf(Date); - expect(dataset.updated_at).toBeInstanceOf(Date); - expect(dataset.rows?.[0].created_at).toBeInstanceOf(Date); - expect(dataset.rows?.[0].updated_at).toBeInstanceOf(Date); - }) - - it('should create a dataset without rows', async () => { - const client = new BaseQuotientClient('test'); - const mockDate = new Date('2024-01-01').toISOString(); - - const postMock = vi.spyOn(client, 'post'); - postMock.mockImplementation((path: string) => { - if (path === '/datasets') { - return Promise.resolve({ - id: 'test_id', - name: 'test_dataset', - created_by: 'test_user', - created_at: mockDate, - updated_at: mockDate, - description: 'test description' - }); - } - return Promise.resolve([]); - }); - - const datasetsResource = new DatasetsResource(client); - const dataset = await datasetsResource.create({ - name: 'test_dataset', - description: 'test description' - }); - - expect(dataset).toBeDefined(); - expect(dataset.rows).toEqual([]); - expect(client.post).toHaveBeenCalledWith('/datasets', { - name: 'test_dataset', - description: 'test description' - }); - }); - - it('should create a dataset with rows', async () => { - const client = new BaseQuotientClient('test'); - const mockDate = new Date('2024-01-01').toISOString(); - - const postMock = vi.spyOn(client, 'post'); - postMock.mockImplementation((path: string) => { - if (path === '/datasets') { - return Promise.resolve({ - id: 'test_id', - name: 'test_dataset', - created_by: 'test_user', - created_at: mockDate, - updated_at: mockDate, - description: 'test description' - }); - } - if (path === '/datasets/test_id/dataset_rows/batch') { - return Promise.resolve([{ - id: 'test_row_id', - input: 'test input', - context: 'test context', - expected: 'test expected', - annotation: 'test annotation', - annotation_note: 'test note', - created_by: 'test_user', - created_at: mockDate, - updated_at: mockDate - }]); - } - return Promise.resolve([]); - }); - - const datasetsResource = new DatasetsResource(client); - const dataset = await datasetsResource.create({ - name: 'test_dataset', - description: 'test description', - rows: [{ - input: 'test input', - context: 'test context', - expected: 'test expected', - metadata: { - annotation: 'test annotation', - annotation_note: 'test note' - } - }] - }); - - expect(dataset).toBeDefined(); - expect(dataset.rows).toEqual([{ - id: 'test_row_id', - input: 'test input', - context: 'test context', - expected: 'test expected', - metadata: { - annotation: 'test annotation', - annotation_note: 'test note' - }, - created_by: 'test_user', - created_at: new Date(mockDate), - updated_at: new Date(mockDate) - }]); - - expect(client.post).toHaveBeenNthCalledWith(1, '/datasets', { - name: 'test_dataset', - description: 'test description', - rows: [{ - input: 'test input', - context: 'test context', - expected: 'test expected', - metadata: { - annotation: 'test annotation', - annotation_note: 'test note' - } - }], - model_id: undefined, - user_id: undefined, - tags: undefined - }); - expect(client.post).toHaveBeenNthCalledWith(2, '/datasets/test_id/dataset_rows/batch', { - rows: expect.any(Array) - }); - }); - - it('should update a dataset', async () => { - const client = new BaseQuotientClient('test'); - const mockDate = new Date('2024-01-01').toISOString(); - - const patchMock = vi.spyOn(client, 'patch'); - patchMock.mockImplementation((path: string) => { - if (path === '/datasets/test_id') { - return Promise.resolve({ - id: 'test_id', - name: 'test_dataset', - created_by: 'test_user', - created_at: mockDate, - updated_at: mockDate, - description: 'test description', - dataset_rows: [] - }); - } - return Promise.resolve([]); - }); - - const datasetsResource = new DatasetsResource(client); - const dataset = await datasetsResource.update({ - dataset: { - id: 'test_id', - name: 'test_dataset', - created_by: 'test_user', - created_at: new Date(mockDate), - updated_at: new Date(mockDate), - description: 'test description' - }, - name: 'test_dataset', - description: 'test description' - }); - - expect(dataset).toBeDefined(); - expect(dataset.rows).toEqual([]); - expect(client.patch).toHaveBeenCalledWith('/datasets/test_id', { - name: 'test_dataset', - description: 'test description' - }); - }); - - it('should update a dataset with rows', async () => { - const client = new BaseQuotientClient('test'); - const mockDate = new Date('2024-01-01').toISOString(); - - const patchMock = vi.spyOn(client, 'patch'); - patchMock.mockImplementation((path: string) => { - if (path === '/datasets/test_id') { - return Promise.resolve({ - id: 'test_id', - name: 'test_dataset', - created_by: 'test_user', - created_at: mockDate, - updated_at: mockDate, - description: 'test description', - dataset_rows: [{ - id: 'test_row_id', - input: 'test input', - context: 'test context', - expected: 'test expected', - annotation: 'test annotation', - annotation_note: 'test note', - created_by: 'test_user', - created_at: mockDate, - updated_at: mockDate - }] - }); - } - return Promise.resolve([]); - }); - - const datasetsResource = new DatasetsResource(client); - const dataset = await datasetsResource.update({ - dataset: { - id: 'test_id', - name: 'test_dataset', - created_by: 'test_user', - created_at: new Date(mockDate), - updated_at: new Date(mockDate), - description: 'test description', - rows: [{ - id: 'test_row_id', - input: 'test input', - context: 'test context', - expected: 'test expected', - metadata: { - annotation: 'test annotation', - annotation_note: 'test note' - }, - created_by: 'test_user', - created_at: new Date(mockDate), - updated_at: new Date(mockDate) - }] - }, - name: 'test_dataset', - description: 'test description', - rows: [{ - input: 'test input', - context: 'test context', - expected: 'test expected', - metadata: { - annotation: 'test annotation', - annotation_note: 'test note' - } - }] - }); - - expect(dataset).toBeDefined(); - expect(dataset.rows).toEqual([{ - id: 'test_row_id', - input: 'test input', - context: 'test context', - expected: 'test expected', - metadata: { - annotation: 'test annotation', - annotation_note: 'test note' - }, - created_by: 'test_user', - created_at: new Date(mockDate), - updated_at: new Date(mockDate) - }]); - - expect(client.patch).toHaveBeenCalledWith('/datasets/test_id', { - name: 'test_dataset', - description: 'test description', - rows: expect.any(Array) - }); - }); - - it('should append rows to a dataset', async () => { - const client = new BaseQuotientClient('test'); - const mockDate = new Date('2024-01-01').toISOString(); - - const postMock = vi.spyOn(client, 'post'); - let attemptCount = 0; - postMock.mockImplementation((path: string) => { - if (path === '/datasets/test_id/dataset_rows/batch') { - attemptCount++; - if (attemptCount === 1) { - return Promise.reject(new Error('Test error')); - } - return Promise.resolve([{ - id: 'new_row_id', - input: 'test input', - context: 'test context', - expected: 'test expected', - annotation: 'test annotation', - annotation_note: 'test note', - created_by: 'test_user', - created_at: mockDate, - updated_at: mockDate - }]); - } - return Promise.resolve([]); - }); - - const datasetsResource = new DatasetsResource(client); - const dataset = await datasetsResource.append({ - dataset: { - id: 'test_id', - name: 'test_dataset', - created_by: 'test_user', - created_at: new Date(mockDate), - updated_at: new Date(mockDate), - description: 'test description' - }, - rows: [{ - input: 'test input', - context: 'test context', - expected: 'test expected', - metadata: { - annotation: 'test annotation', - annotation_note: 'test note' - } - }] - }); - - expect(dataset).toBeDefined(); - - expect(dataset?.rows).toEqual([ - { - id: 'new_row_id', - input: 'test input', - context: 'test context', - expected: 'test expected', - metadata: { - annotation: 'test annotation', - annotation_note: 'test note' - }, - created_by: 'test_user', - created_at: new Date(mockDate), - updated_at: new Date(mockDate) - } - ]); - - expect(client.post).toHaveBeenCalledWith('/datasets/test_id/dataset_rows/batch', { - rows: expect.any(Array) - }); - }); - - it ('should append rows to a dataset that already has rows', async () => { - const client = new BaseQuotientClient('test'); - const mockDate = new Date('2024-01-01').toISOString(); - - const postMock = vi.spyOn(client, 'post'); - let attemptCount = 0; - postMock.mockImplementation((path: string) => { - if (path === '/datasets/test_id/dataset_rows/batch') { - attemptCount++; - if (attemptCount === 1) { - return Promise.reject(new Error('Test error')); - } - return Promise.resolve([{ - id: 'new_row_id', - input: 'test input', - context: 'test context', - expected: 'test expected', - annotation: 'test annotation', - annotation_note: 'test note', - created_by: 'test_user', - created_at: mockDate, - updated_at: mockDate - }]); - } - return Promise.resolve([]); - }); - - const datasetsResource = new DatasetsResource(client); - const dataset = await datasetsResource.append({ - dataset: { - id: 'test_id', - name: 'test_dataset', - created_by: 'test_user', - created_at: new Date(mockDate), - updated_at: new Date(mockDate), - description: 'test description', - rows: [{ - id: 'existing_row_id', - input: 'test input', - context: 'test context', - expected: 'test expected', - metadata: { - annotation: 'test annotation', - annotation_note: 'test note' - }, - created_by: 'test_user', - created_at: new Date(mockDate), - updated_at: new Date(mockDate) - }] - }, - rows: [{ - input: 'test input', - context: 'test context', - expected: 'test expected', - metadata: { - annotation: 'test annotation', - annotation_note: 'test note' - } - }] - }); - - expect(dataset).toBeDefined(); - - expect(dataset?.rows).toEqual([ - { - id: 'existing_row_id', - input: 'test input', - context: 'test context', - expected: 'test expected', - metadata: { - annotation: 'test annotation', - annotation_note: 'test note' - }, - created_by: 'test_user', - created_at: new Date(mockDate), - updated_at: new Date(mockDate) - }, - { - id: 'new_row_id', - input: 'test input', - context: 'test context', - expected: 'test expected', - metadata: { - annotation: 'test annotation', - annotation_note: 'test note' - }, - created_by: 'test_user', - created_at: new Date(mockDate), - updated_at: new Date(mockDate) - } - ]); - - expect(client.post).toHaveBeenCalledWith('/datasets/test_id/dataset_rows/batch', { - rows: expect.any(Array) - }); - }); - - it('should delete rows if rows are passed as an argument', async () => { - const client = new BaseQuotientClient('test'); - const mockDate = new Date('2024-01-01').toISOString(); - - const patchMock = vi.spyOn(client, 'patch'); - patchMock.mockImplementation((path: string) => { - if (path === '/datasets/test_id/dataset_rows/test_row_id') { - return Promise.resolve({}); - } - return Promise.resolve([]); - }); - - const datasetsResource = new DatasetsResource(client); - await datasetsResource.delete({ - dataset: { - id: 'test_id', - name: 'test_dataset', - created_by: 'test_user', - created_at: new Date(mockDate), - updated_at: new Date(mockDate), - description: 'test description' - }, - rows: [{ - id: 'test_row_id', - input: 'test input', - context: 'test context', - expected: 'test expected', - metadata: { - annotation: 'test annotation', - annotation_note: 'test note' - }, - created_by: 'test_user', - created_at: new Date(mockDate), - updated_at: new Date(mockDate) - }] - }); - - expect(patchMock).toHaveBeenCalledWith('/datasets/test_id/dataset_rows/test_row_id', { - id: 'test_row_id', - input: 'test input', - context: 'test context', - expected: 'test expected', - annotation: 'test annotation', - annotation_note: 'test note', - is_deleted: true - }); - - }) - - it('should delete a dataset if no rows are passed as an argument', async () => { - const client = new BaseQuotientClient('test'); - const mockDate = new Date('2024-01-01').toISOString(); - - const patchMock = vi.spyOn(client, 'patch'); - patchMock.mockImplementation((path: string) => { - if (path === '/datasets/test_id') { - return Promise.resolve({}); - } - return Promise.resolve([]); - }); - - const datasetsResource = new DatasetsResource(client); - await datasetsResource.delete({ - dataset: { - id: 'test_id', - name: 'test_dataset', - created_by: 'test_user', - created_at: new Date(mockDate), - updated_at: new Date(mockDate), - description: 'test description' - } - }); - - expect(patchMock).toHaveBeenCalledWith('/datasets/test_id', { - name: 'test_dataset', - description: 'test description', - is_deleted: true - }); - }); - - it('should batch create rows', async () => { - const client = new BaseQuotientClient('test'); - const mockDate = new Date('2024-01-01').toISOString(); - - const postMock = vi.spyOn(client, 'post'); - postMock.mockImplementation((path: string) => { - if (path === '/datasets/test_id/dataset_rows/batch') { - return Promise.resolve([{ - id: 'new_row_id', - input: 'test input', - context: 'test context', - expected: 'test expected', - annotation: 'test annotation', - annotation_note: 'test note', - created_by: 'test_user', - created_at: mockDate, - updated_at: mockDate - }]); - } - return Promise.resolve([]); - }); - const rowResponses: DatasetRowResponse[] = []; - const datasetsResource = new DatasetsResource(client); - await datasetsResource.batchCreateRows('test_id', [{ - input: 'test input', - context: 'test context', - expected: 'test expected', - metadata: { - annotation: 'test annotation', - annotation_note: 'test note' - } - }], rowResponses); - - expect(rowResponses).toEqual([{ - id: 'new_row_id', - input: 'test input', - context: 'test context', - expected: 'test expected', - annotation: 'test annotation', - annotation_note: 'test note', - created_by: 'test_user', - created_at: mockDate, - updated_at: mockDate - }]); - - expect(client.post).toHaveBeenCalledWith('/datasets/test_id/dataset_rows/batch', { - rows: expect.any(Array) - }); - }); - - it('should recursively retry if the initial request fails', async () => { - const client = new BaseQuotientClient('test'); - const mockDate = new Date('2024-01-01').toISOString(); - - const postMock = vi.spyOn(client, 'post'); - let attemptCount = 0; - postMock.mockImplementation((path: string) => { - if (path === '/datasets/test_id/dataset_rows/batch') { - attemptCount++; - if (attemptCount === 1) { - return Promise.reject(new Error('Test error')); - } - return Promise.resolve([{ - id: 'new_row_id', - input: 'test input', - context: 'test context', - expected: 'test expected', - annotation: 'test annotation', - annotation_note: 'test note', - created_by: 'test_user', - created_at: mockDate, - updated_at: mockDate - }]); - } - return Promise.resolve([]); - }); - - const datasetsResource = new DatasetsResource(client); - await datasetsResource.batchCreateRows('test_id', [{ - input: 'test input', - context: 'test context', - expected: 'test expected', - metadata: { - annotation: 'test annotation', - annotation_note: 'test note' - } - }], []); - - expect(postMock).toHaveBeenCalledWith('/datasets/test_id/dataset_rows/batch', { - rows: expect.any(Array) - }); - - expect(postMock).toHaveBeenCalledTimes(2); - expect(postMock).toHaveBeenNthCalledWith(2, '/datasets/test_id/dataset_rows/batch', { - rows: expect.any(Array) - }); - }); - - it('should fail if batch size is one and the initial request fails', async () => { - const client = new BaseQuotientClient('test'); - const consoleErrorSpy = vi.spyOn(console, 'error'); - - const postMock = vi.spyOn(client, 'post'); - postMock.mockRejectedValue(new Error('Test error')); - - const datasetsResource = new DatasetsResource(client); - const result = await datasetsResource.batchCreateRows('test_id', [{ - input: 'test input', - context: 'test context', - expected: 'test expected', - metadata: { - annotation: 'test annotation', - annotation_note: 'test note' - } - }], [], 1); - - expect(result).toEqual([]); - expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('[DatasetsResource.batchCreateRows] Error: Test error')); - expect(postMock).toHaveBeenCalledWith('/datasets/test_id/dataset_rows/batch', { - rows: expect.any(Array) - }); - expect(postMock).toHaveBeenCalledTimes(1); - }); - - it('should handle missing rows when appending to a dataset', async () => { - const client = new BaseQuotientClient('test'); - const consoleErrorSpy = vi.spyOn(console, 'error'); - const mockDate = new Date('2024-01-01').toISOString(); - - const datasetsResource = new DatasetsResource(client); - const result = await datasetsResource.append({ - dataset: { - id: 'test_id', - name: 'test_dataset', - created_by: 'test_user', - created_at: new Date(mockDate), - updated_at: new Date(mockDate), - description: 'test description' - } - }); - - expect(result).toBeNull(); - expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('Error: rows are required')); - }); - - it('should handle batch creation failure with size 1', async () => { - const client = new BaseQuotientClient('test'); - const consoleErrorSpy = vi.spyOn(console, 'error'); - - const postMock = vi.spyOn(client, 'post'); - postMock.mockRejectedValue(new Error('Test error')); - - const datasetsResource = new DatasetsResource(client); - const result = await datasetsResource.batchCreateRows('test_id', [{ - input: 'test input', - context: 'test context', - expected: 'test expected', - metadata: { - annotation: 'test annotation', - annotation_note: 'test note' - } - }], [], 1); - - expect(result).toEqual([]); - expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('[DatasetsResource.batchCreateRows] Error: Test error')); - expect(postMock).toHaveBeenCalledWith('/datasets/test_id/dataset_rows/batch', { - rows: expect.any(Array) - }); - expect(postMock).toHaveBeenCalledTimes(1); - }); - -}); \ No newline at end of file diff --git a/tests/resources/logs.test.ts b/tests/resources/logs.test.ts index dc74185..8cb93d6 100644 --- a/tests/resources/logs.test.ts +++ b/tests/resources/logs.test.ts @@ -12,7 +12,7 @@ describe('LogsResource', () => { inconsistency_detection: false, user_query: 'What is the capital of France?', model_output: 'Paris is the capital of France.', - documents: ['doc1', 'doc2', { page_content: 'doc3', metadata: { source: 'test' } }], + documents: ['doc1', 'doc2', { pageContent: 'doc3', metadata: { source: 'test' } }], message_history: [ { role: 'user', content: 'What is the capital of France?' }, { role: 'assistant', content: 'Paris is the capital of France.' } @@ -40,7 +40,7 @@ describe('LogsResource', () => { describe('Log class', () => { it('should format log as string', () => { const log = new Log(mockLogs[0]); - const expectedString = `Log(id="log-1", app_name="test-app", environment="development", created_at="2024-03-20T10:00:00.000Z")`; + const expectedString = `Log(id="log-1", appName="test-app", environment="development", createdAt="2024-03-20T10:00:00.000Z")`; expect(log.toString()).toBe(expectedString); }); }); @@ -57,9 +57,9 @@ describe('LogsResource', () => { expect(logs).toHaveLength(2); expect(logs[0]).toBeInstanceOf(Log); expect(logs[0].id).toBe('log-1'); - expect(logs[0].app_name).toBe('test-app'); + expect(logs[0].appName).toBe('test-app'); expect(logs[0].environment).toBe('development'); - expect(logs[0].created_at).toBeInstanceOf(Date); + expect(logs[0].createdAt).toBeInstanceOf(Date); expect(client.get).toHaveBeenCalledWith('/logs', {}); }); @@ -74,10 +74,10 @@ describe('LogsResource', () => { const endDate = new Date('2024-03-20T23:59:59Z'); const logs = await logsResource.list({ - app_name: 'test-app', + appName: 'test-app', environment: 'development', - start_date: startDate, - end_date: endDate, + startDate: startDate, + endDate: endDate, limit: 10, offset: 0 }); @@ -157,20 +157,20 @@ describe('LogsResource', () => { const logsResource = new LogsResource(client); await logsResource.create({ - app_name: 'test-app', + appName: 'test-app', environment: 'development', - hallucination_detection: true, - inconsistency_detection: false, - user_query: 'What is the capital of France?', - model_output: 'Paris is the capital of France.', + hallucinationDetection: true, + inconsistencyDetection: false, + userQuery: 'What is the capital of France?', + modelOutput: 'Paris is the capital of France.', documents: ['doc1', 'doc2'], - message_history: [ + messageHistory: [ { role: 'user', content: 'What is the capital of France?' }, { role: 'assistant', content: 'Paris is the capital of France.' } ], instructions: ['Be concise', 'Be accurate'], tags: { user_id: '123' }, - hallucination_detection_sample_rate: 0.5 + hallucinationDetectionSampleRate: 0.5 }); expect(client.post).toHaveBeenCalledWith('/logs', { @@ -198,20 +198,20 @@ describe('LogsResource', () => { const logsResource = new LogsResource(client); const result = await logsResource.create({ - app_name: 'test-app', + appName: 'test-app', environment: 'development', - hallucination_detection: true, - inconsistency_detection: false, - user_query: 'What is the capital of France?', - model_output: 'Paris is the capital of France.', + hallucinationDetection: true, + inconsistencyDetection: false, + userQuery: 'What is the capital of France?', + modelOutput: 'Paris is the capital of France.', documents: ['doc1', 'doc2'], - message_history: [ + messageHistory: [ { role: 'user', content: 'What is the capital of France?' }, { role: 'assistant', content: 'Paris is the capital of France.' } ], instructions: ['Be concise', 'Be accurate'], tags: { user_id: '123' }, - hallucination_detection_sample_rate: 0.5 + hallucinationDetectionSampleRate: 0.5 }); expect(result).toBeNull(); diff --git a/tests/resources/metrics.test.ts b/tests/resources/metrics.test.ts deleted file mode 100644 index b12707e..0000000 --- a/tests/resources/metrics.test.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { vi, describe, it, expect } from 'vitest'; -import { BaseQuotientClient } from '../../quotientai/client'; -import { MetricsResource } from '../../quotientai/resources/metrics'; - -const SAMPLE_METRICS = [ - "bertscore", - "exactmatch", - "faithfulness_selfcheckgpt", - "sentence_tranformers_similarity", - "f1score", - "jaccard_similarity", - "knowledge_f1score", - "meteor", - "normalized_exactmatch", - "rouge_for_context", - "rouge1", - "rouge2", - "rougeL", - "rougeLsum", - "sacrebleu", - "verbosity_ratio", -] - -describe('MetricsResource', () => { - it('should list metrics', async () => { - const client = new BaseQuotientClient('test'); - const getMock = vi.spyOn(client, 'get'); - getMock.mockResolvedValue({ - data: SAMPLE_METRICS - }); - const metricsResource = new MetricsResource(client); - const metrics = await metricsResource.list(); - expect(metrics).toEqual(SAMPLE_METRICS); - expect(getMock).toHaveBeenCalledWith('/runs/metrics'); - }); - -}); \ No newline at end of file diff --git a/tests/resources/models.test.ts b/tests/resources/models.test.ts deleted file mode 100644 index c1eecb0..0000000 --- a/tests/resources/models.test.ts +++ /dev/null @@ -1,64 +0,0 @@ -import { vi, describe, it, expect } from 'vitest'; -import { BaseQuotientClient } from '../../quotientai/client'; -import { ModelsResource } from '../../quotientai/resources/models'; - -const SAMPLE_MODELS = [ - { - "id": "model-123", - "name": "gpt-4", - "provider": { - "id": "provider-123", - "name": "OpenAI" - }, - "created_at": "2024-01-01T00:00:00" - }, - { - "id": "model-456", - "name": "claude-3", - "provider": { - "id": "provider-456", - "name": "Anthropic" - }, - "created_at": "2024-01-02T00:00:00" - } -] - -describe('ModelsResource', () => { - it('should list models', async () => { - const client = new BaseQuotientClient('test'); - const modelsResource = new ModelsResource(client); - const getMock = vi.spyOn(client, 'get'); - getMock.mockResolvedValue(SAMPLE_MODELS); - const models = await modelsResource.list(); - expect(models).toEqual(SAMPLE_MODELS.map(model => ({ - ...model, - created_at: new Date(model.created_at) - }))); - expect(getMock).toHaveBeenCalledWith('/models'); - }); - - it('should get a model', async () => { - const client = new BaseQuotientClient('test'); - const modelsResource = new ModelsResource(client); - const getMock = vi.spyOn(client, 'get'); - getMock.mockResolvedValue(SAMPLE_MODELS[0]); - const model = await modelsResource.getModel('gpt-4'); - expect(model).toEqual({ - ...SAMPLE_MODELS[0], - created_at: new Date(SAMPLE_MODELS[0].created_at) - }); - expect(getMock).toHaveBeenCalledWith('/models/gpt-4'); - }); - - it('should handle null response when getting a model', async () => { - const client = new BaseQuotientClient('test'); - const modelsResource = new ModelsResource(client); - const getMock = vi.spyOn(client, 'get'); - const consoleErrorSpy = vi.spyOn(console, 'error'); - getMock.mockResolvedValue(null); - const model = await modelsResource.getModel('non-existent-model'); - expect(model).toBeNull(); - expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('Model with name non-existent-model not found')); - expect(getMock).toHaveBeenCalledWith('/models/non-existent-model'); - }); -}); diff --git a/tests/resources/prompt.test.ts b/tests/resources/prompt.test.ts deleted file mode 100644 index 7bf4b6c..0000000 --- a/tests/resources/prompt.test.ts +++ /dev/null @@ -1,132 +0,0 @@ -import { describe, it, expect, vi } from 'vitest'; -import { BaseQuotientClient } from '../../quotientai/client'; -import { PromptsResource } from '../../quotientai/resources/prompts'; - -const SAMPLE_PROMPTS = [ - { - id: 'test', - name: 'test', - content: 'test', - version: 1, - user_prompt: 'test', - system_prompt: 'test', - created_at: '2024-01-01', - updated_at: '2024-01-01' - }, - { - id: 'test2', - name: 'test2', - content: 'test2', - version: 2, - user_prompt: 'test2', - system_prompt: 'test2', - created_at: '2024-01-01', - updated_at: '2024-01-01' - } -] -describe('PromptsResource', () => { - it('should list prompts', async () => { - const client = new BaseQuotientClient('test'); - const promptsResource = new PromptsResource(client); - const getMock = vi.spyOn(client, 'get'); - getMock.mockResolvedValue(SAMPLE_PROMPTS); - const prompts = await promptsResource.list(); - expect(prompts).toEqual(SAMPLE_PROMPTS.map(prompt => ({ - ...prompt, - created_at: new Date(prompt.created_at), - updated_at: new Date(prompt.updated_at) - }))); - expect(getMock).toHaveBeenCalledWith('/prompts'); - }); - - it('should get a prompt', async () => { - const client = new BaseQuotientClient('test'); - const promptsResource = new PromptsResource(client); - const getMock = vi.spyOn(client, 'get'); - getMock.mockResolvedValue(SAMPLE_PROMPTS[0]); - const prompt = await promptsResource.getPrompt({ id: 'test' }); - expect(prompt).toEqual({ - ...SAMPLE_PROMPTS[0], - created_at: new Date(SAMPLE_PROMPTS[0].created_at), - updated_at: new Date(SAMPLE_PROMPTS[0].updated_at) - }); - expect(getMock).toHaveBeenCalledWith('/prompts/test'); - }); - - it('should get a prompt by version', async () => { - const client = new BaseQuotientClient('test'); - const promptsResource = new PromptsResource(client); - const getMock = vi.spyOn(client, 'get'); - getMock.mockResolvedValue(SAMPLE_PROMPTS[0]); - const prompt = await promptsResource.getPrompt({ id: 'test', version: '1' }); - expect(prompt).toEqual({ - ...SAMPLE_PROMPTS[0], - created_at: new Date(SAMPLE_PROMPTS[0].created_at), - updated_at: new Date(SAMPLE_PROMPTS[0].updated_at) - }); - expect(getMock).toHaveBeenCalledWith('/prompts/test/versions/1'); - }); - - it('should create a prompt', async () => { - const client = new BaseQuotientClient('test'); - const promptsResource = new PromptsResource(client); - const postMock = vi.spyOn(client, 'post'); - postMock.mockResolvedValue(SAMPLE_PROMPTS[0]); - const prompt = await promptsResource.create({ - name: 'test', - system_prompt: 'test', - user_prompt: 'test' - }); - expect(prompt).toEqual({ - ...SAMPLE_PROMPTS[0], - created_at: new Date(SAMPLE_PROMPTS[0].created_at), - updated_at: new Date(SAMPLE_PROMPTS[0].updated_at) - }); - expect(postMock).toHaveBeenCalledWith('/prompts', { name: 'test', system_prompt: 'test', user_prompt: 'test' }); - }); - - it('should update a prompt', async () => { - const client = new BaseQuotientClient('test'); - const promptsResource = new PromptsResource(client); - const patchMock = vi.spyOn(client, 'patch'); - patchMock.mockResolvedValue(SAMPLE_PROMPTS[0]); - const prompt = await promptsResource.update({ - prompt: { - ...SAMPLE_PROMPTS[0], - created_at: new Date(SAMPLE_PROMPTS[0].created_at), - updated_at: new Date(SAMPLE_PROMPTS[0].updated_at) - } - }); - expect(prompt).toEqual({ - ...SAMPLE_PROMPTS[0], - created_at: new Date(SAMPLE_PROMPTS[0].created_at), - updated_at: new Date(SAMPLE_PROMPTS[0].updated_at) - }); - }); - it('should delete a prompt', async () => { - const client = new BaseQuotientClient('test'); - const promptsResource = new PromptsResource(client); - const patchMock = vi.spyOn(client, 'patch'); - patchMock.mockResolvedValue(SAMPLE_PROMPTS[0]); - await promptsResource.deletePrompt({ - prompt: { - ...SAMPLE_PROMPTS[0], - created_at: new Date(SAMPLE_PROMPTS[0].created_at), - updated_at: new Date(SAMPLE_PROMPTS[0].updated_at) - } - }); - expect(patchMock).toHaveBeenCalledWith('/prompts/test', { id: 'test', name: 'test', system_prompt: 'test', user_prompt: 'test', is_deleted: true }); - }); - - it('should handle null response when getting a prompt', async () => { - const client = new BaseQuotientClient('test'); - const promptsResource = new PromptsResource(client); - const getMock = vi.spyOn(client, 'get'); - const consoleErrorSpy = vi.spyOn(console, 'error'); - getMock.mockResolvedValue(null); - const prompt = await promptsResource.getPrompt({ id: 'test' }); - expect(prompt).toBeNull(); - expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('Error: Prompt not found')); - expect(getMock).toHaveBeenCalledWith('/prompts/test'); - }); -}); diff --git a/tests/resources/runs.test.ts b/tests/resources/runs.test.ts deleted file mode 100644 index 9aed284..0000000 --- a/tests/resources/runs.test.ts +++ /dev/null @@ -1,366 +0,0 @@ -import { describe, it, expect, vi } from 'vitest'; -import { BaseQuotientClient } from '../../quotientai/client'; -import { RunsResource } from '../../quotientai/resources/runs'; -import { Run } from '../../quotientai/resources/runs'; - -const SAMPLE_RUNS = [ - { - id: 'test', - prompt: 'test', - dataset: 'test', - model: 'test', - parameters: {}, - metrics: [], - status: 'test', - results: [], - created_at: '2024-01-01', - finished_at: '2024-01-01' - }, - { - id: 'test2', - prompt: 'test2', - dataset: 'test2', - model: 'test2', - parameters: {}, - metrics: [], - status: 'test2', - results: [], - created_at: '2024-01-01', - finished_at: '2024-01-01' - } -] -describe('RunsResource', () => { - it('should list runs', async () => { - const client = new BaseQuotientClient('test'); - const runsResource = new RunsResource(client); - const getMock = vi.spyOn(client, 'get'); - getMock.mockResolvedValue(SAMPLE_RUNS); - const runs = await runsResource.list(); - expect(runs).toEqual(SAMPLE_RUNS.map(run => new Run(client, run))); - }); - - it('should get a run', async () => { - const client = new BaseQuotientClient('test'); - const runsResource = new RunsResource(client); - const getMock = vi.spyOn(client, 'get'); - getMock.mockResolvedValue(SAMPLE_RUNS[0]); - const run = await runsResource.get('test'); - expect(run).toBeInstanceOf(Run); - expect(run?.id).toBe('test'); - expect(run?.prompt).toBe('test'); - expect(run?.dataset).toBe('test'); - expect(run?.model).toBe('test'); - expect(run?.parameters).toEqual({}); - expect(run?.metrics).toEqual([]); - expect(run?.results).toEqual([]); - expect(run?.created_at).toEqual(new Date('2024-01-01')); - expect(run?.finished_at).toEqual(new Date('2024-01-01')); - expect(getMock).toHaveBeenCalledWith('/runs/test'); - }); - - it('should create a run', async () => { - const client = new BaseQuotientClient('test'); - const runsResource = new RunsResource(client); - const postMock = vi.spyOn(client, 'post'); - postMock.mockResolvedValue(SAMPLE_RUNS[0]); - const mockPrompt = { id: 'test', name: 'test', content: 'test', version: 1, user_prompt: 'test', created_at: new Date(), updated_at: new Date() }; - const mockDataset = { id: 'test', name: 'test', created_by: 'test', created_at: new Date(), updated_at: new Date() }; - const mockModel = { id: 'test', name: 'test', provider: { id: 'test', name: 'test' }, created_at: new Date() }; - const run = await runsResource.create({ - prompt: mockPrompt, - dataset: mockDataset, - model: mockModel, - parameters: {}, - metrics: [] - }); - expect(run).toBeInstanceOf(Run); - expect(run?.id).toBe('test'); - expect(run?.prompt).toBe('test'); - expect(run?.dataset).toBe('test'); - expect(run?.model).toBe('test'); - expect(run?.parameters).toEqual({}); - expect(run?.metrics).toEqual([]); - expect(run?.results).toEqual([]); - expect(run?.created_at).toEqual(new Date('2024-01-01')); - expect(run?.finished_at).toEqual(new Date('2024-01-01')); - expect(postMock).toHaveBeenCalledWith('/runs', { - prompt_id: 'test', - dataset_id: 'test', - model_id: 'test', - parameters: {}, - metrics: [] - }); - }); - - it('should compare runs with different datasets and log an error', async () => { - const client = new BaseQuotientClient('test'); - const runsResource = new RunsResource(client); - - const run1 = new Run(client, { - ...SAMPLE_RUNS[0], - dataset: 'dataset1' - }); - - const run2 = new Run(client, { - ...SAMPLE_RUNS[0], - dataset: 'dataset2' - }); - - // Spy on console.error - const consoleErrorSpy = vi.spyOn(console, 'error'); - - const result = await runsResource.compare([run1, run2]); - expect(result).toBeNull(); - expect(consoleErrorSpy).toHaveBeenCalledWith( - expect.stringContaining('All runs must be on the same dataset to compare them') - ); - }); - - it('should compare runs with different prompts and models and log an error', async () => { - const client = new BaseQuotientClient('test'); - const runsResource = new RunsResource(client); - - const run1 = new Run(client, { - ...SAMPLE_RUNS[0], - prompt: 'prompt1', - model: 'model1' - }); - - const run2 = new Run(client, { - ...SAMPLE_RUNS[0], - prompt: 'prompt2', - model: 'model2' - }); - - // Spy on console.error - const consoleErrorSpy = vi.spyOn(console, 'error'); - - const result = await runsResource.compare([run1, run2]); - expect(result).toBeNull(); - expect(consoleErrorSpy).toHaveBeenCalledWith( - expect.stringContaining('All runs must be on the same prompt or model to compare them') - ); - }); - - it('should compare runs with different prompts and the same model', async () => { - const client = new BaseQuotientClient('test'); - const runsResource = new RunsResource(client); - - const run1 = new Run(client, { - ...SAMPLE_RUNS[0], - prompt: 'prompt1', - model: 'model1', - metrics: ['accuracy'], - results: [ - { values: { accuracy: 0.8 }, id: '1', input: '', output: '', created_at: new Date('2024-01-01'), created_by: '', context: '', expected: '' } - ] - }); - - const run2 = new Run(client, { - ...SAMPLE_RUNS[0], - prompt: 'prompt2', - model: 'model1', - metrics: ['accuracy'], - results: [ - { values: { accuracy: 0.6 }, id: '2', input: '', output: '', created_at: new Date('2024-01-01'), created_by: '', context: '', expected: '' } - ] - }); - - const comparison = await runsResource.compare([run1, run2]); - expect(comparison).toEqual({ - accuracy: { - avg: expect.any(Number), - stddev: expect.any(Number) - } - }); - expect(comparison?.accuracy.avg).toBeCloseTo(0.2, 10); - expect(comparison?.accuracy.stddev).toBeCloseTo(0, 10); - }); - - it('should compare runs with the same prompts and different models', async () => { - const client = new BaseQuotientClient('test'); - const runsResource = new RunsResource(client); - - const run1 = new Run(client, { - ...SAMPLE_RUNS[0], - prompt: 'prompt1', - model: 'model1', - metrics: ['accuracy'], - results: [ - { values: { accuracy: 0.9 }, id: '1', input: '', output: '', created_at: new Date('2024-01-01'), created_by: '', context: '', expected: '' } - ] - }); - - const run2 = new Run(client, { - ...SAMPLE_RUNS[0], - prompt: 'prompt1', - model: 'model2', - metrics: ['accuracy'], - results: [ - { values: { accuracy: 0.7 }, id: '2', input: '', output: '', created_at: new Date('2024-01-01'), created_by: '', context: '', expected: '' } - ] - }); - - const comparison = await runsResource.compare([run1, run2]); - expect(comparison).toEqual({ - accuracy: { - avg: expect.any(Number), - stddev: expect.any(Number) - } - }); - expect(comparison?.accuracy.avg).toBeCloseTo(0.2, 10); - }); - - it('should compare multiple runs with valid results', async () => { - const client = new BaseQuotientClient('test'); - const runsResource = new RunsResource(client); - - const run1 = new Run(client, { - ...SAMPLE_RUNS[0], - id: 'run1', - metrics: ['accuracy'], - results: [ - { values: { accuracy: 0.9 }, id: '1', input: '', output: '', created_at: new Date('2024-01-01'), created_by: '', context: '', expected: '' } - ] - }); - - const run2 = new Run(client, { - ...SAMPLE_RUNS[0], - id: 'run2', - metrics: ['accuracy'], - results: [ - { values: { accuracy: 0.7 }, id: '2', input: '', output: '', created_at: new Date('2024-01-01'), created_by: '', context: '', expected: '' } - ] - }); - - const run3 = new Run(client, { - ...SAMPLE_RUNS[0], - id: 'run3', - metrics: ['accuracy'], - results: [ - { values: { accuracy: 0.8 }, id: '3', input: '', output: '', created_at: new Date('2024-01-01'), created_by: '', context: '', expected: '' } - ] - }); - - const comparison = await runsResource.compare([run1, run2, run3]); - expect(comparison).toEqual({ - run1: { - accuracy: { - avg: expect.any(Number), - stddev: expect.any(Number) - } - }, - run2: { - accuracy: { - avg: expect.any(Number), - stddev: expect.any(Number) - } - }, - run3: { - accuracy: { - avg: expect.any(Number), - stddev: expect.any(Number) - } - } - }); - - // Check the actual values with toBeCloseTo - expect(comparison?.run1.accuracy.avg).toBeCloseTo(0.2, 10); - expect(comparison?.run2.accuracy.avg).toBeCloseTo(0.2, 10); - expect(comparison?.run3.accuracy.avg).toBeCloseTo(0.2, 10); - expect(comparison?.run1.accuracy.stddev).toBeCloseTo(0, 10); - expect(comparison?.run2.accuracy.stddev).toBeCloseTo(0, 10); - expect(comparison?.run3.accuracy.stddev).toBeCloseTo(0, 10); - }); - - it('should return null if only one run is provided', async () => { - const client = new BaseQuotientClient('test'); - const runsResource = new RunsResource(client); - - const run = new Run(client, { - ...SAMPLE_RUNS[0], - metrics: ['accuracy'], - results: [ - { values: { accuracy: 0.9 }, id: '1', input: '', output: '', created_at: new Date('2024-01-01'), created_by: '', context: '', expected: '' } - ] - }); - - // Explicitly await the result and store it - const result = await runsResource.compare([run]); - - // Verify it's exactly null (not undefined) - expect(result).toBe(null); - }); - - it('should return null when summarizing a run with no results', () => { - const client = new BaseQuotientClient('test'); - const run = new Run(client, { - ...SAMPLE_RUNS[0], - metrics: ['accuracy'], - results: [] - }); - - const summary = run.summarize(); - expect(summary).toBe(null); - }); - - it('should correctly summarize a run with results', () => { - const client = new BaseQuotientClient('test'); - const run = new Run(client, { - ...SAMPLE_RUNS[0], - metrics: ['accuracy'], - results: [ - { values: { accuracy: 0.8 }, id: '1', input: '', output: '', created_at: new Date('2024-01-01'), created_by: '', context: '', expected: '' }, - { values: { accuracy: 0.6 }, id: '2', input: '', output: '', created_at: new Date('2024-01-01'), created_by: '', context: '', expected: '' }, - { values: { accuracy: 0.7 }, id: '3', input: '', output: '', created_at: new Date('2024-01-01'), created_by: '', context: '', expected: '' } - ] - }); - - const summary = run.summarize(2, 2); - expect(summary).not.toBe(null); - expect(summary?.metrics.accuracy.avg).toBeCloseTo(0.7, 10); - expect(summary?.metrics.accuracy.stddev).toBeCloseTo(0.0816, 4); - const bestResults = (summary as any)['best_2']; - const worstResults = (summary as any)['worst_2']; - expect(bestResults).toBeDefined(); - expect(worstResults).toBeDefined(); - if (bestResults && worstResults) { - expect(bestResults).toHaveLength(2); - expect(worstResults).toHaveLength(2); - expect(bestResults[0].values.accuracy).toBe(0.8); - expect(bestResults[1].values.accuracy).toBe(0.7); - expect(worstResults[0].values.accuracy).toBe(0.7); - expect(worstResults[1].values.accuracy).toBe(0.6); - } - }); - - it('should handle zero values for best_n and worst_n', () => { - const client = new BaseQuotientClient('test'); - const run = new Run(client, { - ...SAMPLE_RUNS[0], - metrics: ['accuracy'], - results: [ - { values: { accuracy: 0.8 }, id: '1', input: '', output: '', created_at: new Date('2024-01-01'), created_by: '', context: '', expected: '' }, - { values: { accuracy: 0.6 }, id: '2', input: '', output: '', created_at: new Date('2024-01-01'), created_by: '', context: '', expected: '' }, - { values: { accuracy: 0.7 }, id: '3', input: '', output: '', created_at: new Date('2024-01-01'), created_by: '', context: '', expected: '' } - ] - }); - - const summary = run.summarize(0, 0); - expect(summary).not.toBe(null); - expect(summary?.metrics.accuracy.avg).toBeCloseTo(0.7, 10); - expect(summary?.metrics.accuracy.stddev).toBeCloseTo(0.0816, 4); - expect((summary as any)['best_0']).toBeUndefined(); - expect((summary as any)['worst_0']).toBeUndefined(); - }); - - it('should handle null finished_at in response', () => { - const client = new BaseQuotientClient('test'); - const run = new Run(client, { - ...SAMPLE_RUNS[0], - finished_at: null - }); - expect(run.finished_at).toBeUndefined(); - }); -}); - diff --git a/tsconfig.json b/tsconfig.json index 79d28ff..e5de723 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -14,7 +14,7 @@ "paths": { "*": ["node_modules/*"] }, - "types": ["vitest/globals"] + "types": ["vitest/globals", "node"] }, "include": ["quotientai/**/*", "tests/**/*"], "exclude": ["node_modules", "dist"] From 48eac40442bb3cb0fae64f83d0574262ded730f1 Mon Sep 17 00:00:00 2001 From: Mike Goitia Date: Thu, 15 May 2025 15:33:50 -0400 Subject: [PATCH 02/11] readme --- README.md | 84 ++++++++++++++++++++++-------------- quotientai/resources/logs.ts | 18 +++++++- 2 files changed, 69 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index e072580..3cabb72 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ ## Overview -`quotientai` is an SDK built to manage artifacts (prompts, datasets), and run evaluations on [Quotient](https://quotientai.co). +`quotientai` is an SDK built to manage logs and detect hallucinations and inconsistencies in AI responses with [Quotient](https://quotientai.co). ## Installation @@ -16,8 +16,7 @@ npm install quotientai Create an API key on Quotient and set it as an environment variable called `QUOTIENT_API_KEY`. Then check out the examples in the `examples/` directory or see our [docs](https://docs.quotientai.co) for a more comprehensive walkthrough. The examples directory contains: -- `example_evaluate.ts` - Running an evaluation against a dataset -- `example_logs.ts` - Logging with hallucination detection +- `example_logs.ts` - Logging with hallucination and inconsistency detection ### QuotientAI @@ -31,25 +30,9 @@ new QuotientAI(apiKey?: string) - `apiKey`: Optional API key. If not provided, will attempt to read from `QUOTIENT_API_KEY` environment variable. -#### Methods - -##### evaluate - -```typescript -evaluate(params: { - prompt: Prompt; - dataset: Dataset; - model: Model; - parameters: Record; - metrics: string[]; -}): Promise -``` - -Evaluates a model on a dataset using a prompt. - ### QuotientLogger -A logger interface for tracking model interactions. +A logger interface for tracking model interactions and detecting hallucinations. #### Methods @@ -57,13 +40,13 @@ A logger interface for tracking model interactions. ```typescript init(config: { - app_name: string; + appName: string; environment: string; tags?: Record; - sample_rate?: number; - hallucination_detection?: boolean; - inconsistency_detection?: boolean; - hallucination_detection_sample_rate?: number; + sampleRate?: number; + hallucinationDetection?: boolean; + inconsistencyDetection?: boolean; + hallucinationDetectionSampleRate?: number; }): QuotientLogger ``` @@ -73,26 +56,63 @@ Configures the logger with the provided parameters. ```typescript log(params: { - user_query: string; - model_output: string; - documents?: string[]; - message_history?: Array>; + userQuery: string; + modelOutput: string; + documents?: (string | LogDocument)[]; + messageHistory?: Array>; instructions?: string[]; tags?: Record; - hallucination_detection?: boolean; - inconsistency_detection?: boolean; + hallucinationDetection?: boolean; + inconsistencyDetection?: boolean; }): Promise ``` Logs a model interaction. +##### pollForDetections + +```typescript +pollForDetections(logId: string): Promise +``` + +Retrieves the detection results for a specific log entry, including hallucination and inconsistency evaluations. This method polls the API until results are available or a timeout is reached. + +## Detection Results + +The SDK provides strongly-typed interfaces for working with detection results: + +```typescript +interface DetectionResults { + log: LogDetail; // Main log information + logDocuments: DocumentLog[] | null; // Reference documents + logMessageHistory: LogMessageHistory[] | null; // Conversation history + logInstructions: LogInstruction[] | null; // System instructions + evaluations: Evaluation[]; // Hallucination evaluations +} +``` + +Each evaluation includes detailed information about whether content is hallucinated: + +```typescript +interface Evaluation { + id: string; + sentence: string; + isHallucinated: boolean; + // ... additional evaluation details + documentEvaluations: DocumentEvaluation[]; + messageHistoryEvaluations: MessageHistoryEvaluation[]; + instructionEvaluations: InstructionEvaluation[]; + fullDocContextEvaluation: FullDocContextEvaluation; +} +``` + ## Error Handling The client uses a custom `QuotientAIError` class for error handling: ```typescript try { - await client.evaluate({ ... }); + const results = await logger.pollForDetections('log-id'); } catch (error) { if (error instanceof QuotientAIError) { console.error(`Error ${error.status}: ${error.message}`); diff --git a/quotientai/resources/logs.ts b/quotientai/resources/logs.ts index 1d2a517..0884079 100644 --- a/quotientai/resources/logs.ts +++ b/quotientai/resources/logs.ts @@ -1,6 +1,22 @@ import { logError } from '../exceptions'; import { BaseQuotientClient } from '../client'; -import { LogDocument, DetectionResultsResponse, DetectionResults, Evaluation, LogDetail, DocumentLog, LogMessageHistory, LogInstruction, DocumentEvaluation, MessageHistoryEvaluation, InstructionEvaluation, FullDocContextEvaluation, DocumentEvaluationResponse, MessageHistoryEvaluationResponse, InstructionEvaluationResponse, FullDocContextEvaluationResponse } from '../types'; +import { + LogDocument, + DetectionResultsResponse, + DetectionResults, + Evaluation, + LogDetail, + DocumentLog, + LogMessageHistory, + LogInstruction, + DocumentEvaluation, + MessageHistoryEvaluation, + InstructionEvaluation, + FullDocContextEvaluation, + DocumentEvaluationResponse, + MessageHistoryEvaluationResponse, + InstructionEvaluationResponse +} from '../types'; // Snake case interface for API responses interface LogResponse { From cb1cccbde4e7d57aa7ab2314e6d9804791d61cb7 Mon Sep 17 00:00:00 2001 From: Mike Goitia Date: Thu, 15 May 2025 15:45:37 -0400 Subject: [PATCH 03/11] lint and pre commit --- .eslintrc.json | 12 +- .husky/pre-commit | 4 + .prettierignore | 5 + .prettierrc | 8 + examples/example_logs.ts | 98 +-- package-lock.json | 620 ++++++++++++++ package.json | 19 +- quotientai/client.ts | 26 +- quotientai/exceptions.ts | 445 +++++------ quotientai/index.ts | 12 +- quotientai/logger.ts | 89 ++- quotientai/resources/auth.ts | 1 - quotientai/resources/logs.ts | 685 ++++++++-------- quotientai/types.ts | 26 +- tests/client.test.ts | 866 ++++++++++---------- tests/index.test.ts | 308 +++---- tests/logger.test.ts | 637 ++++++++------- tests/resources/auth.test.ts | 24 +- tests/resources/exceptions.test.ts | 1197 +++++++++++++++------------- tests/resources/logs.test.ts | 444 ++++++----- vitest.config.ts | 8 +- 21 files changed, 3158 insertions(+), 2376 deletions(-) create mode 100755 .husky/pre-commit create mode 100644 .prettierignore create mode 100644 .prettierrc diff --git a/.eslintrc.json b/.eslintrc.json index b6963ab..fbc0194 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -1,10 +1,7 @@ { "parser": "@typescript-eslint/parser", - "plugins": ["@typescript-eslint"], - "extends": [ - "eslint:recommended", - "plugin:@typescript-eslint/recommended" - ], + "plugins": ["@typescript-eslint", "prettier"], + "extends": ["eslint:recommended", "plugin:@typescript-eslint/recommended", "prettier"], "env": { "node": true, "es2022": true @@ -16,6 +13,7 @@ "rules": { "@typescript-eslint/explicit-function-return-type": "off", "@typescript-eslint/no-explicit-any": "off", - "@typescript-eslint/no-unused-vars": ["error", { "argsIgnorePattern": "^_" }] + "@typescript-eslint/no-unused-vars": ["error", { "argsIgnorePattern": "^_" }], + "prettier/prettier": "error" } -} \ No newline at end of file +} diff --git a/.husky/pre-commit b/.husky/pre-commit new file mode 100755 index 0000000..d24fdfc --- /dev/null +++ b/.husky/pre-commit @@ -0,0 +1,4 @@ +#!/usr/bin/env sh +. "$(dirname -- "$0")/_/husky.sh" + +npx lint-staged diff --git a/.prettierignore b/.prettierignore new file mode 100644 index 0000000..b7cc02b --- /dev/null +++ b/.prettierignore @@ -0,0 +1,5 @@ +dist +node_modules +build +coverage +package-lock.json \ No newline at end of file diff --git a/.prettierrc b/.prettierrc new file mode 100644 index 0000000..d81e44a --- /dev/null +++ b/.prettierrc @@ -0,0 +1,8 @@ +{ + "semi": true, + "trailingComma": "es5", + "singleQuote": true, + "printWidth": 100, + "tabWidth": 2, + "endOfLine": "auto" +} \ No newline at end of file diff --git a/examples/example_logs.ts b/examples/example_logs.ts index 5388246..2a17b59 100644 --- a/examples/example_logs.ts +++ b/examples/example_logs.ts @@ -1,57 +1,63 @@ import { QuotientAI } from '../quotientai'; async function main() { - const quotient = new QuotientAI(); - console.log("QuotientAI client initialized") + const quotient = new QuotientAI(); + console.log('QuotientAI client initialized'); - // configure the logger - const quotientLogger = quotient.logger.init({ - appName: "my-app-test", - environment: "dev", - sampleRate: 1.0, - tags: { model: "gpt-4o", feature: "customer-support" }, - hallucinationDetection: true, - hallucinationDetectionSampleRate: 1.0, - }) + // configure the logger + const quotientLogger = quotient.logger.init({ + appName: 'my-app-test', + environment: 'dev', + sampleRate: 1.0, + tags: { model: 'gpt-4o', feature: 'customer-support' }, + hallucinationDetection: true, + hallucinationDetectionSampleRate: 1.0, + }); - console.log("Logger initialized") + console.log('Logger initialized'); - // mock retrieved documents - const retrievedDocuments = [ - "Sample document 1", - {"pageContent": "Sample document 2", "metadata": {"source": "website.com"}}, - {"pageContent": "Sample document 3"} - ] + // mock retrieved documents + const retrievedDocuments = [ + 'Sample document 1', + { pageContent: 'Sample document 2', metadata: { source: 'website.com' } }, + { pageContent: 'Sample document 3' }, + ]; - console.log("Preparing to log with quotient_logger") - try { - const logId= await quotientLogger.log({ - userQuery: "How do I cook a test?", - modelOutput: "The capital of France is Paris", - documents: retrievedDocuments, - messageHistory: [ - {"role": "system", "content": "You are an expert on geography."}, - {"role": "user", "content": "What is the capital of France?"}, - {"role": "assistant", "content": "The capital of France is Paris"}, - ], - instructions: [ - "You are a helpful assistant that answers questions about the world.", - "Answer the question in a concise manner. If you are not sure, say 'I don't know'.", - ], - hallucinationDetection: true, - inconsistencyDetection: true, - }); - console.log('pollForDetectionResults with logId: ', logId) + console.log('Preparing to log with quotient_logger'); + try { + const logId = await quotientLogger.log({ + userQuery: 'How do I cook a test?', + modelOutput: 'The capital of France is Paris', + documents: retrievedDocuments, + messageHistory: [ + { role: 'system', content: 'You are an expert on geography.' }, + { role: 'user', content: 'What is the capital of France?' }, + { role: 'assistant', content: 'The capital of France is Paris' }, + ], + instructions: [ + 'You are a helpful assistant that answers questions about the world.', + "Answer the question in a concise manner. If you are not sure, say 'I don't know'.", + ], + hallucinationDetection: true, + inconsistencyDetection: true, + }); + console.log('pollForDetectionResults with logId: ', logId); - // poll for the detection results - const detectionResults = await quotientLogger.pollForDetectionResults(logId); - console.log('documentEvaluations', detectionResults?.evaluations[0].documentEvaluations) - console.log('messageHistoryEvaluations', detectionResults?.evaluations[0].messageHistoryEvaluations) - console.log('instructionEvaluations', detectionResults?.evaluations[0].instructionEvaluations) - console.log('fullDocContextEvaluation', detectionResults?.evaluations[0].fullDocContextEvaluation) - } catch (error) { - console.error(error) - } + // poll for the detection results + const detectionResults = await quotientLogger.pollForDetectionResults(logId); + console.log('documentEvaluations', detectionResults?.evaluations[0].documentEvaluations); + console.log( + 'messageHistoryEvaluations', + detectionResults?.evaluations[0].messageHistoryEvaluations + ); + console.log('instructionEvaluations', detectionResults?.evaluations[0].instructionEvaluations); + console.log( + 'fullDocContextEvaluation', + detectionResults?.evaluations[0].fullDocContextEvaluation + ); + } catch (error) { + console.error(error); + } } main().catch(console.error); diff --git a/package-lock.json b/package-lock.json index 3eb091b..4665ee3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -21,6 +21,11 @@ "@typescript-eslint/parser": "^7.0.1", "@vitest/coverage-v8": "^1.3.1", "eslint": "^8.56.0", + "eslint-config-prettier": "^10.1.5", + "eslint-plugin-prettier": "^5.4.0", + "husky": "^9.1.7", + "lint-staged": "^16.0.0", + "prettier": "^3.5.3", "typescript": "^5.3.3", "vitest": "^1.3.1" } @@ -750,6 +755,18 @@ "node": ">= 8" } }, + "node_modules/@pkgr/core": { + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/@pkgr/core/-/core-0.2.4.tgz", + "integrity": "sha512-ROFF39F6ZrnzSUEmQQZUar0Jt4xVoP9WnDRdWwF4NNcXs3xBTLgBUDoOwW141y1jP+S8nahIbdxbFC7IShw9Iw==", + "dev": true, + "engines": { + "node": "^12.20.0 || ^14.18.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/pkgr" + } + }, "node_modules/@rollup/rollup-android-arm-eabi": { "version": "4.37.0", "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.37.0.tgz", @@ -1461,6 +1478,21 @@ "url": "https://github.com/sponsors/epoberezkin" } }, + "node_modules/ansi-escapes": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-7.0.0.tgz", + "integrity": "sha512-GdYO7a61mR0fOlAsvC9/rIHf7L96sBc6dEWzeOu+KAea5bZyQRPIpojrVoI4AXGJS/ycu/fBTdLrUkA4ODrvjw==", + "dev": true, + "dependencies": { + "environment": "^1.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/ansi-regex": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", @@ -1649,6 +1681,37 @@ "node": "*" } }, + "node_modules/cli-cursor": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-5.0.0.tgz", + "integrity": "sha512-aCj4O5wKyszjMmDT4tZj93kxyydN/K5zPWSCe6/0AV/AA1pqe5ZBIw0a2ZfPQV7lL5/yb5HsUreJ6UFAF1tEQw==", + "dev": true, + "dependencies": { + "restore-cursor": "^5.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/cli-truncate": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/cli-truncate/-/cli-truncate-4.0.0.tgz", + "integrity": "sha512-nPdaFdQ0h/GEigbPClz11D0v/ZJEwxmeVZGeMo3Z5StPtUTkA9o1lD6QwoirYiSDzbcwn2XcjwmCp68W1IS4TA==", + "dev": true, + "dependencies": { + "slice-ansi": "^5.0.0", + "string-width": "^7.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/color-convert": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", @@ -1669,6 +1732,12 @@ "dev": true, "license": "MIT" }, + "node_modules/colorette": { + "version": "2.0.20", + "resolved": "https://registry.npmjs.org/colorette/-/colorette-2.0.20.tgz", + "integrity": "sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w==", + "dev": true + }, "node_modules/combined-stream": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", @@ -1681,6 +1750,15 @@ "node": ">= 0.8" } }, + "node_modules/commander": { + "version": "13.1.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-13.1.0.tgz", + "integrity": "sha512-/rFeCpNJQbhSZjGVwO9RFV3xPqbnERS8MmIQzCtD/zl6gpJuV/bMLuN92oG3F7d8oDEHHRrujSXNUr8fpjntKw==", + "dev": true, + "engines": { + "node": ">=18" + } + }, "node_modules/concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -1816,6 +1894,24 @@ "safe-buffer": "^5.0.1" } }, + "node_modules/emoji-regex": { + "version": "10.4.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-10.4.0.tgz", + "integrity": "sha512-EC+0oUMY1Rqm4O6LLrgjtYDvcVYTy7chDnM4Q7030tP4Kwj3u/pR6gP9ygnp2CJMK5Gq+9Q2oqmrFJAz01DXjw==", + "dev": true + }, + "node_modules/environment": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/environment/-/environment-1.1.0.tgz", + "integrity": "sha512-xUtoPkMggbz0MPyPiIWr1Kp4aeWJjDZ6SMvURhimjdZgsRuDplF5/s9hcgGhyXMhs+6vpnuoiZ2kFiu3FMnS8Q==", + "dev": true, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/es-define-property": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", @@ -1970,6 +2066,51 @@ "url": "https://opencollective.com/eslint" } }, + "node_modules/eslint-config-prettier": { + "version": "10.1.5", + "resolved": "https://registry.npmjs.org/eslint-config-prettier/-/eslint-config-prettier-10.1.5.tgz", + "integrity": "sha512-zc1UmCpNltmVY34vuLRV61r1K27sWuX39E+uyUnY8xS2Bex88VV9cugG+UZbRSRGtGyFboj+D8JODyme1plMpw==", + "dev": true, + "bin": { + "eslint-config-prettier": "bin/cli.js" + }, + "funding": { + "url": "https://opencollective.com/eslint-config-prettier" + }, + "peerDependencies": { + "eslint": ">=7.0.0" + } + }, + "node_modules/eslint-plugin-prettier": { + "version": "5.4.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-prettier/-/eslint-plugin-prettier-5.4.0.tgz", + "integrity": "sha512-BvQOvUhkVQM1i63iMETK9Hjud9QhqBnbtT1Zc642p9ynzBuCe5pybkOnvqZIBypXmMlsGcnU4HZ8sCTPfpAexA==", + "dev": true, + "dependencies": { + "prettier-linter-helpers": "^1.0.0", + "synckit": "^0.11.0" + }, + "engines": { + "node": "^14.18.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint-plugin-prettier" + }, + "peerDependencies": { + "@types/eslint": ">=8.0.0", + "eslint": ">=8.0.0", + "eslint-config-prettier": ">= 7.0.0 <10.0.0 || >=10.1.0", + "prettier": ">=3.0.0" + }, + "peerDependenciesMeta": { + "@types/eslint": { + "optional": true + }, + "eslint-config-prettier": { + "optional": true + } + } + }, "node_modules/eslint-scope": { "version": "7.2.2", "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-7.2.2.tgz", @@ -2098,6 +2239,12 @@ "node": ">=0.10.0" } }, + "node_modules/eventemitter3": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-5.0.1.tgz", + "integrity": "sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA==", + "dev": true + }, "node_modules/execa": { "version": "8.0.1", "resolved": "https://registry.npmjs.org/execa/-/execa-8.0.1.tgz", @@ -2129,6 +2276,12 @@ "dev": true, "license": "MIT" }, + "node_modules/fast-diff": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/fast-diff/-/fast-diff-1.3.0.tgz", + "integrity": "sha512-VxPP4NqbUjj6MaAOafWeUn2cXWLcCtljklUtZf0Ind4XQ+QPtmA0b18zZy0jIQx+ExRVCR/ZQpBmik5lXshNsw==", + "dev": true + }, "node_modules/fast-glob": { "version": "3.3.3", "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.3.tgz", @@ -2314,6 +2467,18 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/get-east-asian-width": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-east-asian-width/-/get-east-asian-width-1.3.0.tgz", + "integrity": "sha512-vpeMIQKxczTD/0s2CdEWHcb0eeJe6TFjxb+J5xgX7hScxqrGuyjmv4c1D4A/gelKfyox0gJJwIHF+fLjeaM8kQ==", + "dev": true, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/get-func-name": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/get-func-name/-/get-func-name-2.0.2.tgz", @@ -2555,6 +2720,21 @@ "node": ">=16.17.0" } }, + "node_modules/husky": { + "version": "9.1.7", + "resolved": "https://registry.npmjs.org/husky/-/husky-9.1.7.tgz", + "integrity": "sha512-5gs5ytaNjBrh5Ow3zrvdUUY+0VxIuWVL4i9irt6friV+BqdCfmV11CQTWMiBYWHbXhco+J1kHfTOUkePhCDvMA==", + "dev": true, + "bin": { + "husky": "bin.js" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/typicode" + } + }, "node_modules/ignore": { "version": "5.3.2", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", @@ -2621,6 +2801,18 @@ "node": ">=0.10.0" } }, + "node_modules/is-fullwidth-code-point": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-4.0.0.tgz", + "integrity": "sha512-O4L094N2/dZ7xqVdrXhh9r1KODPJpFms8B5sGdJLPy664AgvXsreZUyCQQNItZRDlYug4xStLjNp/sz3HvBowQ==", + "dev": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/is-glob": { "version": "4.0.3", "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", @@ -2836,6 +3028,74 @@ "node": ">= 0.8.0" } }, + "node_modules/lilconfig": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/lilconfig/-/lilconfig-3.1.3.tgz", + "integrity": "sha512-/vlFKAoH5Cgt3Ie+JLhRbwOsCQePABiU3tJ1egGvyQ+33R/vcwM2Zl2QR/LzjsBeItPt3oSVXapn+m4nQDvpzw==", + "dev": true, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/antonk52" + } + }, + "node_modules/lint-staged": { + "version": "16.0.0", + "resolved": "https://registry.npmjs.org/lint-staged/-/lint-staged-16.0.0.tgz", + "integrity": "sha512-sUCprePs6/rbx4vKC60Hez6X10HPkpDJaGcy3D1NdwR7g1RcNkWL8q9mJMreOqmHBTs+1sNFp+wOiX9fr+hoOQ==", + "dev": true, + "dependencies": { + "chalk": "^5.4.1", + "commander": "^13.1.0", + "debug": "^4.4.0", + "lilconfig": "^3.1.3", + "listr2": "^8.3.3", + "micromatch": "^4.0.8", + "nano-spawn": "^1.0.0", + "pidtree": "^0.6.0", + "string-argv": "^0.3.2", + "yaml": "^2.7.1" + }, + "bin": { + "lint-staged": "bin/lint-staged.js" + }, + "engines": { + "node": ">=20.18" + }, + "funding": { + "url": "https://opencollective.com/lint-staged" + } + }, + "node_modules/lint-staged/node_modules/chalk": { + "version": "5.4.1", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.4.1.tgz", + "integrity": "sha512-zgVZuo2WcZgfUEmsn6eO3kINexW8RAE4maiQ8QNs8CtpPCSyMiYsULR3HQYkm3w8FIA3SberyMJMSldGsW+U3w==", + "dev": true, + "engines": { + "node": "^12.17.0 || ^14.13 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/listr2": { + "version": "8.3.3", + "resolved": "https://registry.npmjs.org/listr2/-/listr2-8.3.3.tgz", + "integrity": "sha512-LWzX2KsqcB1wqQ4AHgYb4RsDXauQiqhjLk+6hjbaeHG4zpjjVAB6wC/gz6X0l+Du1cN3pUB5ZlrvTbhGSNnUQQ==", + "dev": true, + "dependencies": { + "cli-truncate": "^4.0.0", + "colorette": "^2.0.20", + "eventemitter3": "^5.0.1", + "log-update": "^6.1.0", + "rfdc": "^1.4.1", + "wrap-ansi": "^9.0.0" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/local-pkg": { "version": "0.5.1", "resolved": "https://registry.npmjs.org/local-pkg/-/local-pkg-0.5.1.tgz", @@ -2918,6 +3178,95 @@ "integrity": "sha512-Sb487aTOCr9drQVL8pIxOzVhafOjZN9UU54hiN8PU3uAiSV7lx1yYNpbNmex2PK6dSJoNTSJUUswT651yww3Mg==", "license": "MIT" }, + "node_modules/log-update": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/log-update/-/log-update-6.1.0.tgz", + "integrity": "sha512-9ie8ItPR6tjY5uYJh8K/Zrv/RMZ5VOlOWvtZdEHYSTFKZfIBPQa9tOAEeAWhd+AnIneLJ22w5fjOYtoutpWq5w==", + "dev": true, + "dependencies": { + "ansi-escapes": "^7.0.0", + "cli-cursor": "^5.0.0", + "slice-ansi": "^7.1.0", + "strip-ansi": "^7.1.0", + "wrap-ansi": "^9.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/log-update/node_modules/ansi-regex": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.1.0.tgz", + "integrity": "sha512-7HSX4QQb4CspciLpVFwyRe79O3xsIZDDLER21kERQ71oaPodF8jL725AgJMFAYbooIqolJoRLuM81SpeUkpkvA==", + "dev": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" + } + }, + "node_modules/log-update/node_modules/ansi-styles": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz", + "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==", + "dev": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/log-update/node_modules/is-fullwidth-code-point": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-5.0.0.tgz", + "integrity": "sha512-OVa3u9kkBbw7b8Xw5F9P+D/T9X+Z4+JruYVNapTjPYZYUznQ5YfWeFkOj606XYYW8yugTfC8Pj0hYqvi4ryAhA==", + "dev": true, + "dependencies": { + "get-east-asian-width": "^1.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/log-update/node_modules/slice-ansi": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-7.1.0.tgz", + "integrity": "sha512-bSiSngZ/jWeX93BqeIAbImyTbEihizcwNjFoRUIY/T1wWQsfsm2Vw1agPKylXvQTU7iASGdHhyqRlqQzfz+Htg==", + "dev": true, + "dependencies": { + "ansi-styles": "^6.2.1", + "is-fullwidth-code-point": "^5.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/chalk/slice-ansi?sponsor=1" + } + }, + "node_modules/log-update/node_modules/strip-ansi": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz", + "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==", + "dev": true, + "dependencies": { + "ansi-regex": "^6.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" + } + }, "node_modules/loupe": { "version": "2.3.7", "resolved": "https://registry.npmjs.org/loupe/-/loupe-2.3.7.tgz", @@ -3040,6 +3389,18 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/mimic-function": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/mimic-function/-/mimic-function-5.0.1.tgz", + "integrity": "sha512-VP79XUPxV2CigYP3jWwAUFSku2aKqBH7uTAapFWCBqutsbmDo96KY5o8uh6U+/YSIn5OxJnXp73beVkpqMIGhA==", + "dev": true, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/minimatch": { "version": "9.0.5", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", @@ -3082,6 +3443,18 @@ "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", "license": "MIT" }, + "node_modules/nano-spawn": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/nano-spawn/-/nano-spawn-1.0.1.tgz", + "integrity": "sha512-BfcvzBlUTxSDWfT+oH7vd6CbUV+rThLLHCIym/QO6GGLBsyVXleZs00fto2i2jzC/wPiBYk5jyOmpXWg4YopiA==", + "dev": true, + "engines": { + "node": ">=20.18" + }, + "funding": { + "url": "https://github.com/sindresorhus/nano-spawn?sponsor=1" + } + }, "node_modules/nanoid": { "version": "3.3.11", "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.11.tgz", @@ -3303,6 +3676,18 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, + "node_modules/pidtree": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/pidtree/-/pidtree-0.6.0.tgz", + "integrity": "sha512-eG2dWTVw5bzqGRztnHExczNxt5VGsE6OwTeCG3fdUf9KBsZzO3R5OIIIzWR+iZA0NtZ+RDVdaoE2dK1cn6jH4g==", + "dev": true, + "bin": { + "pidtree": "bin/pidtree.js" + }, + "engines": { + "node": ">=0.10" + } + }, "node_modules/pkg-types": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/pkg-types/-/pkg-types-1.3.1.tgz", @@ -3361,6 +3746,33 @@ "node": ">= 0.8.0" } }, + "node_modules/prettier": { + "version": "3.5.3", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.5.3.tgz", + "integrity": "sha512-QQtaxnoDJeAkDvDKWCLiwIXkTgRhwYDEQCghU9Z6q03iyek/rxRh/2lC3HB7P8sWT2xC/y5JDctPLBIGzHKbhw==", + "dev": true, + "bin": { + "prettier": "bin/prettier.cjs" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/prettier/prettier?sponsor=1" + } + }, + "node_modules/prettier-linter-helpers": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/prettier-linter-helpers/-/prettier-linter-helpers-1.0.0.tgz", + "integrity": "sha512-GbK2cP9nraSSUF9N2XwUwqfzlAFlMNYYl+ShE/V+H8a9uNl/oUqB1w2EL54Jh0OlyRSd8RfWYJ3coVS4TROP2w==", + "dev": true, + "dependencies": { + "fast-diff": "^1.1.2" + }, + "engines": { + "node": ">=6.0.0" + } + }, "node_modules/pretty-format": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-29.7.0.tgz", @@ -3443,6 +3855,37 @@ "node": ">=4" } }, + "node_modules/restore-cursor": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-5.1.0.tgz", + "integrity": "sha512-oMA2dcrw6u0YfxJQXm342bFKX/E4sG9rbTzO9ptUcR/e8A33cHuvStiYOwH7fszkZlZ1z/ta9AAoPk2F4qIOHA==", + "dev": true, + "dependencies": { + "onetime": "^7.0.0", + "signal-exit": "^4.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/restore-cursor/node_modules/onetime": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/onetime/-/onetime-7.0.0.tgz", + "integrity": "sha512-VXJjc87FScF88uafS3JllDgvAm+c/Slfz06lorj2uAY34rlUu0Nt+v8wreiImcrgAjjIHp1rXpTDlLOGw29WwQ==", + "dev": true, + "dependencies": { + "mimic-function": "^5.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/reusify": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.1.0.tgz", @@ -3454,6 +3897,12 @@ "node": ">=0.10.0" } }, + "node_modules/rfdc": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/rfdc/-/rfdc-1.4.1.tgz", + "integrity": "sha512-q1b3N5QkRUWUl7iyylaaj3kOpIT0N2i9MqIEQXP73GVsN9cw3fdx8X63cEmWhJGi2PPCF23Ijp7ktmd39rawIA==", + "dev": true + }, "node_modules/rimraf": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", @@ -3627,6 +4076,34 @@ "node": ">=8" } }, + "node_modules/slice-ansi": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-5.0.0.tgz", + "integrity": "sha512-FC+lgizVPfie0kkhqUScwRu1O/lF6NOgJmlCgK+/LYxDCTk8sGelYaHDhFcDN+Sn3Cv+3VSa4Byeo+IMCzpMgQ==", + "dev": true, + "dependencies": { + "ansi-styles": "^6.0.0", + "is-fullwidth-code-point": "^4.0.0" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/slice-ansi?sponsor=1" + } + }, + "node_modules/slice-ansi/node_modules/ansi-styles": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz", + "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==", + "dev": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, "node_modules/source-map-js": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", @@ -3651,6 +4128,59 @@ "dev": true, "license": "MIT" }, + "node_modules/string-argv": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/string-argv/-/string-argv-0.3.2.tgz", + "integrity": "sha512-aqD2Q0144Z+/RqG52NeHEkZauTAUWJO8c6yTftGJKO3Tja5tUgIfmIl6kExvhtxSDP7fXB6DvzkfMpCd/F3G+Q==", + "dev": true, + "engines": { + "node": ">=0.6.19" + } + }, + "node_modules/string-width": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-7.2.0.tgz", + "integrity": "sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ==", + "dev": true, + "dependencies": { + "emoji-regex": "^10.3.0", + "get-east-asian-width": "^1.0.0", + "strip-ansi": "^7.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/string-width/node_modules/ansi-regex": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.1.0.tgz", + "integrity": "sha512-7HSX4QQb4CspciLpVFwyRe79O3xsIZDDLER21kERQ71oaPodF8jL725AgJMFAYbooIqolJoRLuM81SpeUkpkvA==", + "dev": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" + } + }, + "node_modules/string-width/node_modules/strip-ansi": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz", + "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==", + "dev": true, + "dependencies": { + "ansi-regex": "^6.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" + } + }, "node_modules/strip-ansi": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", @@ -3716,6 +4246,22 @@ "node": ">=8" } }, + "node_modules/synckit": { + "version": "0.11.5", + "resolved": "https://registry.npmjs.org/synckit/-/synckit-0.11.5.tgz", + "integrity": "sha512-frqvfWyDA5VPVdrWfH24uM6SI/O8NLpVbIIJxb8t/a3YGsp4AW9CYgSKC0OaSEfexnp7Y1pVh2Y6IHO8ggGDmA==", + "dev": true, + "dependencies": { + "@pkgr/core": "^0.2.4", + "tslib": "^2.8.1" + }, + "engines": { + "node": "^14.18.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/synckit" + } + }, "node_modules/test-exclude": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/test-exclude/-/test-exclude-6.0.0.tgz", @@ -3815,6 +4361,12 @@ "typescript": ">=4.2.0" } }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "dev": true + }, "node_modules/type-check": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", @@ -4093,6 +4645,62 @@ "node": ">=0.10.0" } }, + "node_modules/wrap-ansi": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-9.0.0.tgz", + "integrity": "sha512-G8ura3S+3Z2G+mkgNRq8dqaFZAuxfsxpBB8OCTGRTCtp+l/v9nbFNmCUP1BZMts3G1142MsZfn6eeUKrr4PD1Q==", + "dev": true, + "dependencies": { + "ansi-styles": "^6.2.1", + "string-width": "^7.0.0", + "strip-ansi": "^7.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrap-ansi/node_modules/ansi-regex": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.1.0.tgz", + "integrity": "sha512-7HSX4QQb4CspciLpVFwyRe79O3xsIZDDLER21kERQ71oaPodF8jL725AgJMFAYbooIqolJoRLuM81SpeUkpkvA==", + "dev": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" + } + }, + "node_modules/wrap-ansi/node_modules/ansi-styles": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz", + "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==", + "dev": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/wrap-ansi/node_modules/strip-ansi": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz", + "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==", + "dev": true, + "dependencies": { + "ansi-regex": "^6.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" + } + }, "node_modules/wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", @@ -4100,6 +4708,18 @@ "dev": true, "license": "ISC" }, + "node_modules/yaml": { + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.8.0.tgz", + "integrity": "sha512-4lLa/EcQCB0cJkyts+FpIRx5G/llPxfP6VQU5KByHEhLxY3IJCH0f0Hy1MHI8sClTvsIb8qwRJ6R/ZdlDJ/leQ==", + "dev": true, + "bin": { + "yaml": "bin.mjs" + }, + "engines": { + "node": ">= 14.6" + } + }, "node_modules/yocto-queue": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", diff --git a/package.json b/package.json index e7b5661..a111aea 100644 --- a/package.json +++ b/package.json @@ -15,7 +15,19 @@ "test": "vitest run", "coverage": "vitest run --coverage", "lint": "eslint . --ext .ts", - "test:watch": "vitest" + "lint:fix": "eslint . --ext .ts --fix", + "format": "prettier --write \"**/*.{ts,tsx,js,jsx,json,md}\"", + "test:watch": "vitest", + "prepare": "husky" + }, + "lint-staged": { + "*.{ts,tsx}": [ + "eslint --fix", + "prettier --write" + ], + "*.{js,jsx,json,md}": [ + "prettier --write" + ] }, "dependencies": { "axios": "^1.6.7", @@ -30,6 +42,11 @@ "@typescript-eslint/parser": "^7.0.1", "@vitest/coverage-v8": "^1.3.1", "eslint": "^8.56.0", + "eslint-config-prettier": "^10.1.5", + "eslint-plugin-prettier": "^5.4.0", + "husky": "^9.0.11", + "lint-staged": "^16.0.0", + "prettier": "^3.5.3", "typescript": "^5.3.3", "vitest": "^1.3.1" }, diff --git a/quotientai/client.ts b/quotientai/client.ts index a2487af..3e6b11b 100644 --- a/quotientai/client.ts +++ b/quotientai/client.ts @@ -16,7 +16,7 @@ export class BaseQuotientClient { constructor(apiKey: string) { this.apiKey = apiKey; - + // Determine token directory let tokenDir: string; try { @@ -37,8 +37,8 @@ export class BaseQuotientClient { this.client = axios.create({ baseURL: 'https://api.quotientai.co/api/v1', headers: { - Authorization: `Bearer ${apiKey}` - } + Authorization: `Bearer ${apiKey}`, + }, }); // Load existing token @@ -80,7 +80,11 @@ export class BaseQuotientClient { JSON.stringify({ token, expires_at: expiry, api_key: this.apiKey }) ); } catch (error) { - logError(new QuotientAIError('Could not create directory for token. If you see this error please notify us at contact@quotientai.co')); + logError( + new QuotientAIError( + 'Could not create directory for token. If you see this error please notify us at contact@quotientai.co' + ) + ); } } @@ -96,7 +100,7 @@ export class BaseQuotientClient { } // With 5-minute buffer - return Date.now() / 1000 < (this.tokenExpiry - 300); + return Date.now() / 1000 < this.tokenExpiry - 300; } private updateAuthHeader(): void { @@ -137,10 +141,8 @@ export class BaseQuotientClient { // Filter out null values const filteredData = Array.isArray(data) - ? data.filter(v => v !== null) - : Object.fromEntries( - Object.entries(data).filter(([_, v]) => v !== null) - ); + ? data.filter((v) => v !== null) + : Object.fromEntries(Object.entries(data).filter(([_, v]) => v !== null)); const response = await this.client.post(path, filteredData, { timeout }); return response.data; @@ -149,9 +151,7 @@ export class BaseQuotientClient { public async patch(path: string, data: any = {}, timeout?: number): Promise { this.updateAuthHeader(); - const filteredData = Object.fromEntries( - Object.entries(data).filter(([_, v]) => v !== null) - ); + const filteredData = Object.fromEntries(Object.entries(data).filter(([_, v]) => v !== null)); const response = await this.client.patch(path, filteredData, { timeout }); return response.data; @@ -162,4 +162,4 @@ export class BaseQuotientClient { const response = await this.client.delete(path, { timeout }); return response.data; } -} \ No newline at end of file +} diff --git a/quotientai/exceptions.ts b/quotientai/exceptions.ts index 390541a..427a859 100644 --- a/quotientai/exceptions.ts +++ b/quotientai/exceptions.ts @@ -1,295 +1,296 @@ -import axios, { AxiosError, AxiosRequestConfig, AxiosResponse, InternalAxiosRequestConfig } from 'axios'; +import axios, { + AxiosError, + AxiosRequestConfig, + AxiosResponse, + InternalAxiosRequestConfig, +} from 'axios'; export function logError(error: Error, context?: string) { - const timestamp = new Date().toISOString(); - const contextStr = context ? `[${context}] ` : ''; - const stack = error.stack || ''; - - console.error(`[${timestamp}] ${contextStr}${error.name}: ${error.message}`); - console.error(stack); + const timestamp = new Date().toISOString(); + const contextStr = context ? `[${context}] ` : ''; + const stack = error.stack || ''; + + console.error(`[${timestamp}] ${contextStr}${error.name}: ${error.message}`); + console.error(stack); } export class QuotientAIError extends Error { - constructor(message: string) { - super(message); - this.name = 'QuotientAIError'; - } + constructor(message: string) { + super(message); + this.name = 'QuotientAIError'; + } } export class ValidationError extends QuotientAIError { - constructor(message: string) { - super(message); - this.name = 'ValidationError'; - } + constructor(message: string) { + super(message); + this.name = 'ValidationError'; + } } export class APIError extends QuotientAIError { - request: AxiosRequestConfig; - body: any; - code?: string; - param?: string; - type?: string; + request: AxiosRequestConfig; + body: any; + code?: string; + param?: string; + type?: string; - constructor(message: string, request: AxiosRequestConfig, body?: any) { - super(message); - this.name = 'APIError'; - this.request = request; - this.body = body; + constructor(message: string, request: AxiosRequestConfig, body?: any) { + super(message); + this.name = 'APIError'; + this.request = request; + this.body = body; - if (body && typeof body === 'object') { - this.code = body.code; - this.param = body.param; - this.type = body.type; - } + if (body && typeof body === 'object') { + this.code = body.code; + this.param = body.param; + this.type = body.type; } + } } export class APIResponseValidationError extends APIError { - response: AxiosResponse; - status: number; + response: AxiosResponse; + status: number; - constructor(response: AxiosResponse, body: any, message?: string) { - super( - message || 'Data returned by API invalid for expected schema.', - response.config, - body - ); - this.name = 'APIResponseValidationError'; - this.response = response; - this.status = response.status; - } + constructor(response: AxiosResponse, body: any, message?: string) { + super(message || 'Data returned by API invalid for expected schema.', response.config, body); + this.name = 'APIResponseValidationError'; + this.response = response; + this.status = response.status; + } } export class APIStatusError extends APIError { - response: AxiosResponse; - status: number; + response: AxiosResponse; + status: number; - constructor(message: string, response: AxiosResponse, body?: any) { - super(message, response.config, body); - this.name = 'APIStatusError'; - this.response = response; - this.status = response.status; - } + constructor(message: string, response: AxiosResponse, body?: any) { + super(message, response.config, body); + this.name = 'APIStatusError'; + this.response = response; + this.status = response.status; + } } export class APIConnectionError extends APIError { - constructor(message: string = 'Connection error.', request: AxiosRequestConfig) { - super(message, request); - this.name = 'APIConnectionError'; - } + constructor(message: string = 'Connection error.', request: AxiosRequestConfig) { + super(message, request); + this.name = 'APIConnectionError'; + } } export class APITimeoutError extends APIConnectionError { - constructor(request: InternalAxiosRequestConfig | undefined) { - super('Request timed out.', request || { url: 'unknown' }); - this.name = 'APITimeoutError'; - } + constructor(request: InternalAxiosRequestConfig | undefined) { + super('Request timed out.', request || { url: 'unknown' }); + this.name = 'APITimeoutError'; + } } export class BadRequestError extends APIStatusError { - status = 400; - constructor(message: string, response: AxiosResponse, body?: any) { - super(message, response, body); - this.name = 'BadRequestError'; - } + status = 400; + constructor(message: string, response: AxiosResponse, body?: any) { + super(message, response, body); + this.name = 'BadRequestError'; + } } export class AuthenticationError extends APIStatusError { - status = 401; - constructor(message: string, response: AxiosResponse, body?: any) { - super(message, response, body); - this.name = 'AuthenticationError'; - } + status = 401; + constructor(message: string, response: AxiosResponse, body?: any) { + super(message, response, body); + this.name = 'AuthenticationError'; + } } export class PermissionDeniedError extends APIStatusError { - status = 403; - constructor(message: string, response: AxiosResponse, body?: any) { - super(message, response, body); - this.name = 'PermissionDeniedError'; - } + status = 403; + constructor(message: string, response: AxiosResponse, body?: any) { + super(message, response, body); + this.name = 'PermissionDeniedError'; + } } export class NotFoundError extends APIStatusError { - status = 404; - constructor(message: string, response: AxiosResponse, body?: any) { - super(message, response, body); - this.name = 'NotFoundError'; - } + status = 404; + constructor(message: string, response: AxiosResponse, body?: any) { + super(message, response, body); + this.name = 'NotFoundError'; + } } export class ConflictError extends APIStatusError { - status = 409; - constructor(message: string, response: AxiosResponse, body?: any) { - super(message, response, body); - this.name = 'ConflictError'; - } + status = 409; + constructor(message: string, response: AxiosResponse, body?: any) { + super(message, response, body); + this.name = 'ConflictError'; + } } export class UnprocessableEntityError extends APIStatusError { - status = 422; - constructor(message: string, response: AxiosResponse, body?: any) { - super(message, response, body); - this.name = 'UnprocessableEntityError'; - } + status = 422; + constructor(message: string, response: AxiosResponse, body?: any) { + super(message, response, body); + this.name = 'UnprocessableEntityError'; + } } export class RateLimitError extends APIStatusError { - status = 429; - constructor(message: string, response: AxiosResponse, body?: any) { - super(message, response, body); - this.name = 'RateLimitError'; - } + status = 429; + constructor(message: string, response: AxiosResponse, body?: any) { + super(message, response, body); + this.name = 'RateLimitError'; + } } export class InternalServerError extends APIStatusError { - status = 500; - constructor(message: string, response: AxiosResponse, body?: any) { - super(message, response, body); - this.name = 'InternalServerError'; - } + status = 500; + constructor(message: string, response: AxiosResponse, body?: any) { + super(message, response, body); + this.name = 'InternalServerError'; + } } export function parseUnprocessableEntityError(response: AxiosResponse): string { - try { - const body = response.data; - if ('detail' in body) { - const missingFields: string[] = []; - for (const detail of body.detail) { - if (detail.type === 'missing') { - missingFields.push(detail.loc[detail.loc.length - 1]); - } - } - if (missingFields.length) { - return `missing required fields: ${missingFields.join(', ')}`; - } + try { + const body = response.data; + if ('detail' in body) { + const missingFields: string[] = []; + for (const detail of body.detail) { + if (detail.type === 'missing') { + missingFields.push(detail.loc[detail.loc.length - 1]); } - const error = new APIResponseValidationError(response, body); - logError(error, 'parseUnprocessableEntityError'); - return 'Invalid response format'; - } catch (error) { - const apiError = new APIResponseValidationError(response, null); - logError(apiError, 'parseUnprocessableEntityError'); - return 'Invalid response format'; + } + if (missingFields.length) { + return `missing required fields: ${missingFields.join(', ')}`; + } } + const error = new APIResponseValidationError(response, body); + logError(error, 'parseUnprocessableEntityError'); + return 'Invalid response format'; + } catch (error) { + const apiError = new APIResponseValidationError(response, null); + logError(apiError, 'parseUnprocessableEntityError'); + return 'Invalid response format'; + } } export function parseBadRequestError(response: AxiosResponse): string { - try { - const body = response.data; - if ('detail' in body) { - return body.detail; - } - const error = new APIResponseValidationError(response, body); - logError(error, 'parseBadRequestError'); - return 'Invalid request format'; - } catch (error) { - const apiError = new APIResponseValidationError(response, null); - logError(apiError, 'parseBadRequestError'); - return 'Invalid request format'; + try { + const body = response.data; + if ('detail' in body) { + return body.detail; } + const error = new APIResponseValidationError(response, body); + logError(error, 'parseBadRequestError'); + return 'Invalid request format'; + } catch (error) { + const apiError = new APIResponseValidationError(response, null); + logError(apiError, 'parseBadRequestError'); + return 'Invalid request format'; + } } export function handleErrors() { - return function(target: any, propertyKey: string, descriptor: PropertyDescriptor) { - const originalMethod = descriptor.value; + return function (target: any, propertyKey: string, descriptor: PropertyDescriptor) { + const originalMethod = descriptor.value; - descriptor.value = async function(...args: any[]) { - let retries = 3; - let delay = 1000; + descriptor.value = async function (...args: any[]) { + let retries = 3; + let delay = 1000; - while (retries > 0) { - try { - const response = await originalMethod.apply(this, args); - return response.data; - } catch (err) { - if (axios.isAxiosError(err)) { - const axiosError = err as AxiosError; + while (retries > 0) { + try { + const response = await originalMethod.apply(this, args); + return response.data; + } catch (err) { + if (axios.isAxiosError(err)) { + const axiosError = err as AxiosError; - if (axiosError.response) { - const { status, data } = axiosError.response; - - switch (status) { - case 400: { - const message = parseBadRequestError(axiosError.response); - const error = new BadRequestError(message, axiosError.response, data); - logError(error, `${target.constructor.name}.${propertyKey}`); - return null; - } - case 401: { - const error = new AuthenticationError( - 'unauthorized: the request requires user authentication. ensure your API key is correct.', - axiosError.response, - data - ); - logError(error, `${target.constructor.name}.${propertyKey}`); - return null; - } - case 403: { - const error = new PermissionDeniedError( - 'forbidden: the server understood the request, but it refuses to authorize it.', - axiosError.response, - data - ); - logError(error, `${target.constructor.name}.${propertyKey}`); - return null; - } - case 404: { - const error = new NotFoundError( - 'not found: the server can not find the requested resource.', - axiosError.response, - data - ); - logError(error, `${target.constructor.name}.${propertyKey}`); - return null; - } - case 422: { - const unprocessableMessage = parseUnprocessableEntityError(axiosError.response); - const error = new UnprocessableEntityError( - unprocessableMessage, - axiosError.response, - data - ); - logError(error, `${target.constructor.name}.${propertyKey}`); - return null; - } - default: { - const error = new APIStatusError( - `unexpected status code: ${status}. contact support@quotientai.co for help.`, - axiosError.response, - data - ); - logError(error, `${target.constructor.name}.${propertyKey}`); - return null; - } - } - } - - if (axiosError.code === 'ECONNABORTED') { - if (retries > 1) { - retries--; - await new Promise(resolve => setTimeout(resolve, delay)); - delay *= 2; // Exponential backoff - continue; - } - const error = new APITimeoutError(axiosError.config); - logError(error, `${target.constructor.name}.${propertyKey}`); - return null; - } + if (axiosError.response) { + const { status, data } = axiosError.response; - const connectionError = new APIConnectionError( - 'connection error. please try again later.', - axiosError.config || { url: 'unknown' } - ); - logError(connectionError, `${target.constructor.name}.${propertyKey}`); - return null; - } - logError(err as Error, `${target.constructor.name}.${propertyKey}`); - return null; + switch (status) { + case 400: { + const message = parseBadRequestError(axiosError.response); + const error = new BadRequestError(message, axiosError.response, data); + logError(error, `${target.constructor.name}.${propertyKey}`); + return null; + } + case 401: { + const error = new AuthenticationError( + 'unauthorized: the request requires user authentication. ensure your API key is correct.', + axiosError.response, + data + ); + logError(error, `${target.constructor.name}.${propertyKey}`); + return null; + } + case 403: { + const error = new PermissionDeniedError( + 'forbidden: the server understood the request, but it refuses to authorize it.', + axiosError.response, + data + ); + logError(error, `${target.constructor.name}.${propertyKey}`); + return null; } + case 404: { + const error = new NotFoundError( + 'not found: the server can not find the requested resource.', + axiosError.response, + data + ); + logError(error, `${target.constructor.name}.${propertyKey}`); + return null; + } + case 422: { + const unprocessableMessage = parseUnprocessableEntityError(axiosError.response); + const error = new UnprocessableEntityError( + unprocessableMessage, + axiosError.response, + data + ); + logError(error, `${target.constructor.name}.${propertyKey}`); + return null; + } + default: { + const error = new APIStatusError( + `unexpected status code: ${status}. contact support@quotientai.co for help.`, + axiosError.response, + data + ); + logError(error, `${target.constructor.name}.${propertyKey}`); + return null; + } + } } - }; - return descriptor; + if (axiosError.code === 'ECONNABORTED') { + if (retries > 1) { + retries--; + await new Promise((resolve) => setTimeout(resolve, delay)); + delay *= 2; // Exponential backoff + continue; + } + const error = new APITimeoutError(axiosError.config); + logError(error, `${target.constructor.name}.${propertyKey}`); + return null; + } + + const connectionError = new APIConnectionError( + 'connection error. please try again later.', + axiosError.config || { url: 'unknown' } + ); + logError(connectionError, `${target.constructor.name}.${propertyKey}`); + return null; + } + logError(err as Error, `${target.constructor.name}.${propertyKey}`); + return null; + } + } }; + + return descriptor; + }; } diff --git a/quotientai/index.ts b/quotientai/index.ts index cdb6ead..6684b84 100644 --- a/quotientai/index.ts +++ b/quotientai/index.ts @@ -4,7 +4,7 @@ import { AuthResource } from './resources/auth'; import { LogsResource } from './resources/logs'; import { logError } from './exceptions'; -export class QuotientAI { +export class QuotientAI { public auth: AuthResource = null!; public logs: LogsResource = null!; public logger: QuotientLogger = null!; @@ -14,8 +14,8 @@ export class QuotientAI { if (!key) { const error = new Error( 'Could not find API key. Either pass apiKey to QuotientAI() or ' + - 'set the QUOTIENT_API_KEY environment variable. ' + - 'If you do not have an API key, you can create one at https://app.quotientai.co in your settings page' + 'set the QUOTIENT_API_KEY environment variable. ' + + 'If you do not have an API key, you can create one at https://app.quotientai.co in your settings page' ); logError(error, 'QuotientAI.constructor'); return; @@ -25,7 +25,7 @@ export class QuotientAI { } } - private initializeResources(client: BaseQuotientClient): void { + private initializeResources(client: BaseQuotientClient): void { // Initialize resources this.auth = new AuthResource(client); this.logs = new LogsResource(client); @@ -40,9 +40,9 @@ export class QuotientAI { logError( error as Error, 'If you are seeing this error, please check that your API key is correct.\n' + - 'If the issue persists, please contact support@quotientai.co' + 'If the issue persists, please contact support@quotientai.co' ); return; } } -} \ No newline at end of file +} diff --git a/quotientai/logger.ts b/quotientai/logger.ts index 4fbc515..7c7ba7d 100644 --- a/quotientai/logger.ts +++ b/quotientai/logger.ts @@ -49,31 +49,31 @@ export class QuotientLogger { try { // Check if it has the required pageContent property if (!('pageContent' in obj)) { - return { - valid: false, - error: "Missing required 'pageContent' property" + return { + valid: false, + error: "Missing required 'pageContent' property", }; } - + // Check if pageContent is a string if (typeof obj.pageContent !== 'string') { - return { - valid: false, - error: `The 'pageContent' property must be a string, found ${typeof obj.pageContent}` + return { + valid: false, + error: `The 'pageContent' property must be a string, found ${typeof obj.pageContent}`, }; } - + // If metadata exists, check if it's an object if ('metadata' in obj && obj.metadata !== null && typeof obj.metadata !== 'object') { - return { - valid: false, - error: `The 'metadata' property must be an object, found ${typeof obj.metadata}` + return { + valid: false, + error: `The 'metadata' property must be an object, found ${typeof obj.metadata}`, }; } - + return { valid: true }; } catch (error) { - return { valid: false, error: "Unexpected error validating document" }; + return { valid: false, error: 'Unexpected error validating document' }; } } @@ -90,18 +90,22 @@ export class QuotientLogger { } else if (typeof doc === 'object' && doc !== null) { const validation = this.isValidLogDocument(doc); if (!validation.valid) { - logError(new ValidationError( - `Invalid document format at index ${i}: ${validation.error}. ` + - "Documents must be either strings or JSON objects with a 'pageContent' string property and an optional 'metadata' object. " + - "To fix this, ensure each document follows the format: { pageContent: 'your text content', metadata?: { key: 'value' } }" - )); + logError( + new ValidationError( + `Invalid document format at index ${i}: ${validation.error}. ` + + "Documents must be either strings or JSON objects with a 'pageContent' string property and an optional 'metadata' object. " + + "To fix this, ensure each document follows the format: { pageContent: 'your text content', metadata?: { key: 'value' } }" + ) + ); return false; } } else { - logError(new ValidationError( - `Invalid document type at index ${i}. Found ${typeof doc}, but documents must be either strings or JSON objects with a 'pageContent' property. ` + - "To fix this, provide documents as either simple strings or properly formatted objects: { pageContent: 'your text content' }" - )); + logError( + new ValidationError( + `Invalid document type at index ${i}. Found ${typeof doc}, but documents must be either strings or JSON objects with a 'pageContent' property. ` + + "To fix this, provide documents as either simple strings or properly formatted objects: { pageContent: 'your text content' }" + ) + ); return false; } } @@ -116,7 +120,9 @@ export class QuotientLogger { } if (!this.appName || !this.environment) { - logError(new Error('Logger is not properly configured. appName and environment must be set.')); + logError( + new Error('Logger is not properly configured. appName and environment must be set.') + ); return null; } @@ -158,12 +164,16 @@ export class QuotientLogger { // poll for the detection results using log id async pollForDetectionResults( - logId: string, - timeout: number = 300, + logId: string, + timeout: number = 300, pollInterval: number = 2.0 ): Promise { if (!this.configured) { - logError(new Error('Logger is not configured. Please call init() before polling for detection results.')); + logError( + new Error( + 'Logger is not configured. Please call init() before polling for detection results.' + ) + ); return null; } @@ -177,37 +187,38 @@ export class QuotientLogger { let currentPollInterval = pollInterval * 1000; // Convert poll interval to milliseconds const baseInterval = pollInterval * 1000; // Keep track of the base interval - while ((Date.now() - startTime) < timeoutMs) { + while (Date.now() - startTime < timeoutMs) { try { const results = await this.logsResource.getDetections(logId); - + // Reset interval on successful response currentPollInterval = baseInterval; - + if (results && results.log) { const status = results.log.status; - + // Check if we're in a final state - if (status === LOG_STATUS.LOG_CREATED_NO_DETECTIONS_PENDING || - status === LOG_STATUS.LOG_CREATED_AND_DETECTION_COMPLETED) { + if ( + status === LOG_STATUS.LOG_CREATED_NO_DETECTIONS_PENDING || + status === LOG_STATUS.LOG_CREATED_AND_DETECTION_COMPLETED + ) { return results; } - } - + // Wait for poll interval before trying again - await new Promise(resolve => setTimeout(resolve, currentPollInterval)); + await new Promise((resolve) => setTimeout(resolve, currentPollInterval)); } catch (error) { // Handle event loop errors specifically if (error instanceof Error && error.message.includes('Event loop is closed')) { - await new Promise(resolve => setTimeout(resolve, currentPollInterval)); + await new Promise((resolve) => setTimeout(resolve, currentPollInterval)); continue; } - await new Promise(resolve => setTimeout(resolve, currentPollInterval)); + await new Promise((resolve) => setTimeout(resolve, currentPollInterval)); } } - + logError(new Error(`Timed out waiting for detection results after ${timeout} seconds`)); return null; } -} \ No newline at end of file +} diff --git a/quotientai/resources/auth.ts b/quotientai/resources/auth.ts index cb3d7a1..a3f00ff 100644 --- a/quotientai/resources/auth.ts +++ b/quotientai/resources/auth.ts @@ -14,4 +14,3 @@ export class AuthResource { return response; } } - diff --git a/quotientai/resources/logs.ts b/quotientai/resources/logs.ts index 0884079..224343b 100644 --- a/quotientai/resources/logs.ts +++ b/quotientai/resources/logs.ts @@ -1,372 +1,381 @@ import { logError } from '../exceptions'; import { BaseQuotientClient } from '../client'; -import { - LogDocument, - DetectionResultsResponse, - DetectionResults, - Evaluation, - LogDetail, - DocumentLog, - LogMessageHistory, - LogInstruction, - DocumentEvaluation, - MessageHistoryEvaluation, - InstructionEvaluation, - FullDocContextEvaluation, - DocumentEvaluationResponse, - MessageHistoryEvaluationResponse, - InstructionEvaluationResponse +import { + LogDocument, + DetectionResultsResponse, + DetectionResults, + Evaluation, + LogDetail, + DocumentLog, + LogMessageHistory, + LogInstruction, + DocumentEvaluation, + MessageHistoryEvaluation, + InstructionEvaluation, + FullDocContextEvaluation, + DocumentEvaluationResponse, + MessageHistoryEvaluationResponse, + InstructionEvaluationResponse, } from '../types'; // Snake case interface for API responses interface LogResponse { - id: string; - app_name: string; - environment: string; - hallucination_detection: boolean; - inconsistency_detection: boolean; - user_query: string; - model_output: string; - documents: (string | { page_content: string; metadata?: Record })[]; - message_history: any[] | null; - instructions: string[] | null; - tags: Record; - created_at: string; + id: string; + app_name: string; + environment: string; + hallucination_detection: boolean; + inconsistency_detection: boolean; + user_query: string; + model_output: string; + documents: (string | { page_content: string; metadata?: Record })[]; + message_history: any[] | null; + instructions: string[] | null; + tags: Record; + created_at: string; } interface LogsResponse { - logs: LogResponse[]; + logs: LogResponse[]; } // CamelCase interface for client-side params, will be converted to snake_case for API interface CreateLogParams { - id?: string; - createdAt?: Date; - appName: string; - environment: string; - hallucinationDetection: boolean; - inconsistencyDetection: boolean; - userQuery: string; - modelOutput: string; - documents: (string | LogDocument)[]; - messageHistory?: any[] | null; - instructions?: string[] | null; - tags?: Record; - hallucinationDetectionSampleRate?: number; + id?: string; + createdAt?: Date; + appName: string; + environment: string; + hallucinationDetection: boolean; + inconsistencyDetection: boolean; + userQuery: string; + modelOutput: string; + documents: (string | LogDocument)[]; + messageHistory?: any[] | null; + instructions?: string[] | null; + tags?: Record; + hallucinationDetectionSampleRate?: number; } // CamelCase interface for client-side params, will be converted to snake_case for API interface ListLogsParams { - appName?: string; - environment?: string; - startDate?: Date; - endDate?: Date; - limit?: number; - offset?: number; + appName?: string; + environment?: string; + startDate?: Date; + endDate?: Date; + limit?: number; + offset?: number; } export class Log { - id: string; - appName: string; - environment: string; - hallucinationDetection: boolean; - inconsistencyDetection: boolean; - userQuery: string; - modelOutput: string; - documents: (string | LogDocument)[]; - messageHistory: any[] | null; - instructions: string[] | null; - tags: Record; - createdAt: Date; - - constructor(data: LogResponse) { - this.id = data.id; - this.appName = data.app_name; - this.environment = data.environment; - this.hallucinationDetection = data.hallucination_detection; - this.inconsistencyDetection = data.inconsistency_detection; - this.userQuery = data.user_query; - this.modelOutput = data.model_output; - - // Convert documents with page_content to pageContent format for client-side use - this.documents = data.documents.map(doc => { - if (typeof doc === 'string') { - return doc; - } else if (doc && typeof doc === 'object' && 'page_content' in doc) { - const { page_content, metadata } = doc; - return { - pageContent: page_content, - metadata - } as LogDocument; - } - return doc; - }); - - this.messageHistory = data.message_history; - this.instructions = data.instructions; - this.tags = data.tags; - this.createdAt = new Date(data.created_at); - } + id: string; + appName: string; + environment: string; + hallucinationDetection: boolean; + inconsistencyDetection: boolean; + userQuery: string; + modelOutput: string; + documents: (string | LogDocument)[]; + messageHistory: any[] | null; + instructions: string[] | null; + tags: Record; + createdAt: Date; - toString(): string { - return `Log(id="${this.id}", appName="${this.appName}", environment="${this.environment}", createdAt="${this.createdAt.toISOString()}")`; - } + constructor(data: LogResponse) { + this.id = data.id; + this.appName = data.app_name; + this.environment = data.environment; + this.hallucinationDetection = data.hallucination_detection; + this.inconsistencyDetection = data.inconsistency_detection; + this.userQuery = data.user_query; + this.modelOutput = data.model_output; + + // Convert documents with page_content to pageContent format for client-side use + this.documents = data.documents.map((doc) => { + if (typeof doc === 'string') { + return doc; + } else if (doc && typeof doc === 'object' && 'page_content' in doc) { + const { page_content, metadata } = doc; + return { + pageContent: page_content, + metadata, + } as LogDocument; + } + return doc; + }); + + this.messageHistory = data.message_history; + this.instructions = data.instructions; + this.tags = data.tags; + this.createdAt = new Date(data.created_at); + } + + toString(): string { + return `Log(id="${this.id}", appName="${this.appName}", environment="${this.environment}", createdAt="${this.createdAt.toISOString()}")`; + } } export class LogsResource { - protected client: BaseQuotientClient; + protected client: BaseQuotientClient; - constructor(client: BaseQuotientClient) { - this.client = client; - } + constructor(client: BaseQuotientClient) { + this.client = client; + } - // Create a log - async create(params: CreateLogParams): Promise { - try { - // Convert document objects with pageContent to page_content format for API - const convertedDocuments = params.documents.map(doc => { - if (typeof doc === 'string') { - return doc; - } else if (doc && typeof doc === 'object' && 'pageContent' in doc) { - const { pageContent, metadata } = doc; - return { - page_content: pageContent, - metadata - }; - } - return doc; - }); - - // Convert camelCase params to snake_case for API - const apiParams = { - id: params.id, - created_at: params.createdAt, - app_name: params.appName, - environment: params.environment, - hallucination_detection: params.hallucinationDetection, - inconsistency_detection: params.inconsistencyDetection, - user_query: params.userQuery, - model_output: params.modelOutput, - documents: convertedDocuments, - message_history: params.messageHistory, - instructions: params.instructions, - tags: params.tags, - hallucination_detection_sample_rate: params.hallucinationDetectionSampleRate - }; - - const response = await this.client.post('/logs', apiParams); - return response; - } catch (error) { - logError(error as Error, 'LogsResource.create'); - return null; + // Create a log + async create(params: CreateLogParams): Promise { + try { + // Convert document objects with pageContent to page_content format for API + const convertedDocuments = params.documents.map((doc) => { + if (typeof doc === 'string') { + return doc; + } else if (doc && typeof doc === 'object' && 'pageContent' in doc) { + const { pageContent, metadata } = doc; + return { + page_content: pageContent, + metadata, + }; } + return doc; + }); + + // Convert camelCase params to snake_case for API + const apiParams = { + id: params.id, + created_at: params.createdAt, + app_name: params.appName, + environment: params.environment, + hallucination_detection: params.hallucinationDetection, + inconsistency_detection: params.inconsistencyDetection, + user_query: params.userQuery, + model_output: params.modelOutput, + documents: convertedDocuments, + message_history: params.messageHistory, + instructions: params.instructions, + tags: params.tags, + hallucination_detection_sample_rate: params.hallucinationDetectionSampleRate, + }; + + const response = await this.client.post('/logs', apiParams); + return response; + } catch (error) { + logError(error as Error, 'LogsResource.create'); + return null; } + } - // List logs - async list(params: ListLogsParams = {}): Promise { - // Convert camelCase params to snake_case for API - const queryParams: Record = {}; - - if (params.appName) queryParams.app_name = params.appName; - if (params.environment) queryParams.environment = params.environment; - if (params.startDate) queryParams.start_date = params.startDate.toISOString(); - if (params.endDate) queryParams.end_date = params.endDate.toISOString(); - if (params.limit !== undefined) queryParams.limit = params.limit; - if (params.offset !== undefined) queryParams.offset = params.offset; - - try { - const response = await this.client.get('/logs', queryParams) as LogsResponse; - - // Check if response has logs property and it's an array - if (!response || !response.logs || !Array.isArray(response.logs)) { - console.warn('No logs found. Please check your query parameters and try again.'); - return []; - } - - // Map the logs to Log objects - return response.logs.map(logData => new Log(logData)); - } catch (error) { - logError(error as Error, 'LogsResource.list'); - return []; - } + // List logs + async list(params: ListLogsParams = {}): Promise { + // Convert camelCase params to snake_case for API + const queryParams: Record = {}; + + if (params.appName) queryParams.app_name = params.appName; + if (params.environment) queryParams.environment = params.environment; + if (params.startDate) queryParams.start_date = params.startDate.toISOString(); + if (params.endDate) queryParams.end_date = params.endDate.toISOString(); + if (params.limit !== undefined) queryParams.limit = params.limit; + if (params.offset !== undefined) queryParams.offset = params.offset; + + try { + const response = (await this.client.get('/logs', queryParams)) as LogsResponse; + + // Check if response has logs property and it's an array + if (!response || !response.logs || !Array.isArray(response.logs)) { + console.warn('No logs found. Please check your query parameters and try again.'); + return []; + } + + // Map the logs to Log objects + return response.logs.map((logData) => new Log(logData)); + } catch (error) { + logError(error as Error, 'LogsResource.list'); + return []; } + } - /** - * Get detection results for a log - * @param logId The ID of the log to get detection results for - * @returns Promise resolving to the detection results when available - * @throws Error if the API call fails, to allow for proper retry handling - */ - async getDetections(logId: string): Promise { - try { - if (!logId) { - throw new Error('Log ID is required for detection polling'); - } - - // The path should match the Python implementation which uses `/logs/{log_id}/rca` - const path = `/logs/${logId}/rca`; - const response = await this.client.get(path) as DetectionResultsResponse; - - if (!response) { - return null; - } - - // Convert snake_case response to camelCase - return this.convertToDetectionResults(response); - } catch (error) { - return null; - } + /** + * Get detection results for a log + * @param logId The ID of the log to get detection results for + * @returns Promise resolving to the detection results when available + * @throws Error if the API call fails, to allow for proper retry handling + */ + async getDetections(logId: string): Promise { + try { + if (!logId) { + throw new Error('Log ID is required for detection polling'); + } + + // The path should match the Python implementation which uses `/logs/{log_id}/rca` + const path = `/logs/${logId}/rca`; + const response = (await this.client.get(path)) as DetectionResultsResponse; + + if (!response) { + return null; + } + + // Convert snake_case response to camelCase + return this.convertToDetectionResults(response); + } catch (error) { + return null; } - - /** - * Converts snake_case API response to camelCase DetectionResults - */ - private convertToDetectionResults(response: DetectionResultsResponse): DetectionResults { - // Convert LogDetail - const logDetail: LogDetail = { - id: response.log.id, - createdAt: response.log.created_at, - appName: response.log.app_name, - environment: response.log.environment, - tags: response.log.tags, - inconsistencyDetection: response.log.inconsistency_detection, - hallucinationDetection: response.log.hallucination_detection, - userQuery: response.log.user_query, - modelOutput: response.log.model_output, - hallucinationDetectionSampleRate: response.log.hallucination_detection_sample_rate, - updatedAt: response.log.updated_at, - status: response.log.status, - hasHallucination: response.log.has_hallucination, - hasInconsistency: response.log.has_inconsistency, - documents: response.log.documents, - messageHistory: response.log.message_history, - instructions: response.log.instructions + } + + /** + * Converts snake_case API response to camelCase DetectionResults + */ + private convertToDetectionResults(response: DetectionResultsResponse): DetectionResults { + // Convert LogDetail + const logDetail: LogDetail = { + id: response.log.id, + createdAt: response.log.created_at, + appName: response.log.app_name, + environment: response.log.environment, + tags: response.log.tags, + inconsistencyDetection: response.log.inconsistency_detection, + hallucinationDetection: response.log.hallucination_detection, + userQuery: response.log.user_query, + modelOutput: response.log.model_output, + hallucinationDetectionSampleRate: response.log.hallucination_detection_sample_rate, + updatedAt: response.log.updated_at, + status: response.log.status, + hasHallucination: response.log.has_hallucination, + hasInconsistency: response.log.has_inconsistency, + documents: response.log.documents, + messageHistory: response.log.message_history, + instructions: response.log.instructions, + }; + + // Convert LogDocuments + const logDocuments = + response.log_documents?.map((doc) => { + const documentLog: DocumentLog = { + id: doc.id, + content: doc.content, + metadata: doc.metadata, + logId: doc.log_id, + createdAt: doc.created_at, + updatedAt: doc.updated_at, + index: doc.index, }; - - // Convert LogDocuments - const logDocuments = response.log_documents?.map(doc => { - const documentLog: DocumentLog = { - id: doc.id, - content: doc.content, - metadata: doc.metadata, - logId: doc.log_id, - createdAt: doc.created_at, - updatedAt: doc.updated_at, - index: doc.index - }; - return documentLog; - }) || null; - - // Convert LogMessageHistory - const logMessageHistory = response.log_message_history?.map(msg => { - const messageHistory: LogMessageHistory = { - id: msg.id, - content: msg.content, - logId: msg.log_id, - createdAt: msg.created_at, - updatedAt: msg.updated_at, - index: msg.index - }; - return messageHistory; - }) || null; - - // Convert LogInstructions - const logInstructions = response.log_instructions?.map(inst => { - const instruction: LogInstruction = { - id: inst.id, - content: inst.content, - logId: inst.log_id, - createdAt: inst.created_at, - updatedAt: inst.updated_at, - index: inst.index - }; - return instruction; - }) || null; - - // Convert Evaluations - const evaluations = response.evaluations.map(evalItem => { - // Convert document evaluations - const documentEvaluations = evalItem.document_evaluations.map((docEval: DocumentEvaluationResponse) => { - const documentEvaluation: DocumentEvaluation = { - id: docEval.id, - evaluationId: docEval.evaluation_id, - reasoning: docEval.reasoning, - score: docEval.score, - index: docEval.index, - createdAt: docEval.created_at, - updatedAt: docEval.updated_at, - logDocumentId: docEval.log_document_id - }; - return documentEvaluation; - }); - - // Convert message history evaluations - const messageHistoryEvaluations = evalItem.message_history_evaluations.map((msgEval: MessageHistoryEvaluationResponse) => { - const messageHistoryEvaluation: MessageHistoryEvaluation = { - id: msgEval.id, - evaluationId: msgEval.evaluation_id, - reasoning: msgEval.reasoning, - score: msgEval.score, - index: msgEval.index, - createdAt: msgEval.created_at, - updatedAt: msgEval.updated_at, - logMessageHistoryId: msgEval.log_message_history_id - }; - return messageHistoryEvaluation; - }); - - // Convert instruction evaluations - const instructionEvaluations = evalItem.instruction_evaluations.map((instEval: InstructionEvaluationResponse) => { - const instructionEvaluation: InstructionEvaluation = { - id: instEval.id, - evaluationId: instEval.evaluation_id, - reasoning: instEval.reasoning, - score: instEval.score, - index: instEval.index, - createdAt: instEval.created_at, - updatedAt: instEval.updated_at, - logInstructionId: instEval.log_instruction_id - }; - return instructionEvaluation; - }); - - // Convert full doc context evaluation - const fullDocEval = evalItem.full_doc_context_evaluation; - const fullDocContextEvaluation: FullDocContextEvaluation = { - id: fullDocEval.id, - evaluationId: fullDocEval.evaluation_id, - reasoning: fullDocEval.reasoning, - score: fullDocEval.score, - index: fullDocEval.index, - createdAt: fullDocEval.created_at, - updatedAt: fullDocEval.updated_at, - logDocumentIds: fullDocEval.log_document_ids - }; - - const evaluation: Evaluation = { - id: evalItem.id, - sentence: evalItem.sentence, - supportingDocumentIds: evalItem.supporting_document_ids, - supportingMessageHistoryIds: evalItem.supporting_message_history_ids, - supportingInstructionIds: evalItem.supporting_instruction_ids, - isHallucinated: evalItem.is_hallucinated, - fullDocContextHasHallucination: evalItem.full_doc_context_has_hallucination, - index: evalItem.index, - documentEvaluations, - messageHistoryEvaluations, - instructionEvaluations, - fullDocContextEvaluation - }; - return evaluation; - }); - - // Construct and return the camelCase DetectionResults - return { - log: logDetail, - logDocuments, - logMessageHistory, - logInstructions, - evaluations + return documentLog; + }) || null; + + // Convert LogMessageHistory + const logMessageHistory = + response.log_message_history?.map((msg) => { + const messageHistory: LogMessageHistory = { + id: msg.id, + content: msg.content, + logId: msg.log_id, + createdAt: msg.created_at, + updatedAt: msg.updated_at, + index: msg.index, }; - } -} \ No newline at end of file + return messageHistory; + }) || null; + + // Convert LogInstructions + const logInstructions = + response.log_instructions?.map((inst) => { + const instruction: LogInstruction = { + id: inst.id, + content: inst.content, + logId: inst.log_id, + createdAt: inst.created_at, + updatedAt: inst.updated_at, + index: inst.index, + }; + return instruction; + }) || null; + + // Convert Evaluations + const evaluations = response.evaluations.map((evalItem) => { + // Convert document evaluations + const documentEvaluations = evalItem.document_evaluations.map( + (docEval: DocumentEvaluationResponse) => { + const documentEvaluation: DocumentEvaluation = { + id: docEval.id, + evaluationId: docEval.evaluation_id, + reasoning: docEval.reasoning, + score: docEval.score, + index: docEval.index, + createdAt: docEval.created_at, + updatedAt: docEval.updated_at, + logDocumentId: docEval.log_document_id, + }; + return documentEvaluation; + } + ); + + // Convert message history evaluations + const messageHistoryEvaluations = evalItem.message_history_evaluations.map( + (msgEval: MessageHistoryEvaluationResponse) => { + const messageHistoryEvaluation: MessageHistoryEvaluation = { + id: msgEval.id, + evaluationId: msgEval.evaluation_id, + reasoning: msgEval.reasoning, + score: msgEval.score, + index: msgEval.index, + createdAt: msgEval.created_at, + updatedAt: msgEval.updated_at, + logMessageHistoryId: msgEval.log_message_history_id, + }; + return messageHistoryEvaluation; + } + ); + + // Convert instruction evaluations + const instructionEvaluations = evalItem.instruction_evaluations.map( + (instEval: InstructionEvaluationResponse) => { + const instructionEvaluation: InstructionEvaluation = { + id: instEval.id, + evaluationId: instEval.evaluation_id, + reasoning: instEval.reasoning, + score: instEval.score, + index: instEval.index, + createdAt: instEval.created_at, + updatedAt: instEval.updated_at, + logInstructionId: instEval.log_instruction_id, + }; + return instructionEvaluation; + } + ); + + // Convert full doc context evaluation + const fullDocEval = evalItem.full_doc_context_evaluation; + const fullDocContextEvaluation: FullDocContextEvaluation = { + id: fullDocEval.id, + evaluationId: fullDocEval.evaluation_id, + reasoning: fullDocEval.reasoning, + score: fullDocEval.score, + index: fullDocEval.index, + createdAt: fullDocEval.created_at, + updatedAt: fullDocEval.updated_at, + logDocumentIds: fullDocEval.log_document_ids, + }; + + const evaluation: Evaluation = { + id: evalItem.id, + sentence: evalItem.sentence, + supportingDocumentIds: evalItem.supporting_document_ids, + supportingMessageHistoryIds: evalItem.supporting_message_history_ids, + supportingInstructionIds: evalItem.supporting_instruction_ids, + isHallucinated: evalItem.is_hallucinated, + fullDocContextHasHallucination: evalItem.full_doc_context_has_hallucination, + index: evalItem.index, + documentEvaluations, + messageHistoryEvaluations, + instructionEvaluations, + fullDocContextEvaluation, + }; + return evaluation; + }); + + // Construct and return the camelCase DetectionResults + return { + log: logDetail, + logDocuments, + logMessageHistory, + logInstructions, + evaluations, + }; + } +} diff --git a/quotientai/types.ts b/quotientai/types.ts index 4ea5516..0c90164 100644 --- a/quotientai/types.ts +++ b/quotientai/types.ts @@ -28,9 +28,9 @@ export interface LogEntry { userQuery: string; modelOutput: string; documents: (string | LogDocument)[]; - messageHistory?: Array> | null; + messageHistory?: Array> | null; instructions?: string[] | null; - tags?: Record; + tags?: Record; hallucinationDetection: boolean; hallucinationDetectionSampleRate?: number; inconsistencyDetection: boolean; @@ -41,7 +41,7 @@ export interface LoggerConfig { createdAt?: string | Date; appName: string; environment: string; - tags?: Record; + tags?: Record; sampleRate?: number; hallucinationDetection?: boolean; inconsistencyDetection?: boolean; @@ -49,23 +49,23 @@ export interface LoggerConfig { } export enum LOG_STATUS { - LOG_NOT_FOUND = "log_not_found", - LOG_CREATION_IN_PROGRESS = "log_creation_in_progress", - LOG_CREATED_NO_DETECTIONS_PENDING = "log_created_no_detections_pending", - LOG_CREATED_AND_DETECTION_IN_PROGRESS = "log_created_and_detection_in_progress", - LOG_CREATED_AND_DETECTION_COMPLETED = "log_created_and_detection_completed", + LOG_NOT_FOUND = 'log_not_found', + LOG_CREATION_IN_PROGRESS = 'log_creation_in_progress', + LOG_CREATED_NO_DETECTIONS_PENDING = 'log_created_no_detections_pending', + LOG_CREATED_AND_DETECTION_IN_PROGRESS = 'log_created_and_detection_in_progress', + LOG_CREATED_AND_DETECTION_COMPLETED = 'log_created_and_detection_completed', } export enum EVALUATION_SCORE { - PASS = "PASS", - FAIL = "FAIL", - INCONCLUSIVE = "INCONCLUSIVE", + PASS = 'PASS', + FAIL = 'FAIL', + INCONCLUSIVE = 'INCONCLUSIVE', } export interface QuotientAIError extends Error { status?: number; code?: string; -} +} // Common evaluation properties - API Response format (snake_case) export interface BaseEvaluationResponse { @@ -263,4 +263,4 @@ export interface DetectionResults { logMessageHistory: LogMessageHistory[] | null; logInstructions: LogInstruction[] | null; evaluations: Evaluation[]; -} \ No newline at end of file +} diff --git a/tests/client.test.ts b/tests/client.test.ts index d384b2a..1a449cb 100644 --- a/tests/client.test.ts +++ b/tests/client.test.ts @@ -5,463 +5,471 @@ import * as jwt from 'jsonwebtoken'; import * as os from 'os'; vi.mock('fs', () => { - return { - existsSync: vi.fn().mockReturnValue(false), - readFileSync: vi.fn().mockReturnValue('{}'), - mkdirSync: vi.fn(), - writeFileSync: vi.fn() - }; + return { + existsSync: vi.fn().mockReturnValue(false), + readFileSync: vi.fn().mockReturnValue('{}'), + mkdirSync: vi.fn(), + writeFileSync: vi.fn(), + }; }); vi.mock('jsonwebtoken', () => { - return { - decode: vi.fn().mockReturnValue({ exp: Math.floor(Date.now() / 1000) + 3600 }) - }; + return { + decode: vi.fn().mockReturnValue({ exp: Math.floor(Date.now() / 1000) + 3600 }), + }; }); vi.mock('os', () => ({ - homedir: vi.fn().mockReturnValue('/home/user') + homedir: vi.fn().mockReturnValue('/home/user'), })); describe('BaseQuotientClient', () => { - beforeEach(() => { - vi.clearAllMocks(); - }); + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('should initialize with the correct api key', () => { + const client = new BaseQuotientClient('test_api_key'); + const privateClient = client as any; + + expect(privateClient.apiKey).toBe('test_api_key'); + expect(privateClient.token).toBeNull(); + expect(privateClient.tokenExpiry).toBe(0); + }); + + it('should properly process and store tokens from a response', async () => { + const client = new BaseQuotientClient('test_api_key'); + const privateClient = client as any; + + // Spy on the saveToken method + const saveTokenSpy = vi.spyOn(privateClient, 'saveToken'); + + const response = { + data: { + token: 'test_token', + expires_at: Date.now() + 1000 * 60 * 60 * 24, // 1 day from now + }, + headers: { + 'x-jwt-token': 'test_token', + }, + }; - it('should initialize with the correct api key', () => { - const client = new BaseQuotientClient('test_api_key'); - const privateClient = client as any; - - expect(privateClient.apiKey).toBe('test_api_key'); - expect(privateClient.token).toBeNull(); - expect(privateClient.tokenExpiry).toBe(0); - }); - - it('should properly process and store tokens from a response', async () => { - const client = new BaseQuotientClient('test_api_key'); - const privateClient = client as any; - - // Spy on the saveToken method - const saveTokenSpy = vi.spyOn(privateClient, 'saveToken'); - - const response = { - data: { - token: 'test_token', - expires_at: Date.now() + 1000 * 60 * 60 * 24 // 1 day from now - }, - headers: { - 'x-jwt-token': 'test_token' - } - }; - - vi.mocked(fs.existsSync).mockReturnValue(false); - vi.mocked(fs.readFileSync).mockReturnValue(JSON.stringify(response)); - - await privateClient.handleResponse(response); - - expect(privateClient.token).toBe('test_token'); - expect(privateClient.tokenExpiry).toBeCloseTo(Math.floor(Date.now() / 1000) + 3600, -1); - expect(saveTokenSpy).toHaveBeenCalledOnce(); - expect(saveTokenSpy).toHaveBeenCalledWith('test_token', expect.any(Number)); - }); - - it('should save the token to the file system', async () => { - const client = new BaseQuotientClient('test_api_key'); - const privateClient = client as any; - - // Mock fs operations - const mkdirSpy = vi.spyOn(fs, 'mkdirSync'); - const writeSpy = vi.spyOn(fs, 'writeFileSync'); - - const response = { - data: { - token: 'test_token', - expires_at: Date.now() + 1000 * 60 * 60 * 24 // 1 day from now - }, - headers: { - 'x-jwt-token': 'test_token' - } - }; - - await privateClient.handleResponse(response); - - // Verify token is in memory - expect(privateClient.token).toBe('test_token'); - expect(privateClient.tokenExpiry).toBeCloseTo(Math.floor(Date.now() / 1000) + 3600, -1); - - // Verify file system operations - expect(mkdirSpy).toHaveBeenCalledWith(expect.any(String), { recursive: true }); - expect(writeSpy).toHaveBeenCalledWith( - expect.any(String), - expect.stringContaining('"token":"test_token"') - ); - const writeCall = writeSpy.mock.calls[0]; - const writtenData = writeCall[1] as string; - expect(typeof JSON.parse(writtenData).expires_at).toBe('number'); - }); + vi.mocked(fs.existsSync).mockReturnValue(false); + vi.mocked(fs.readFileSync).mockReturnValue(JSON.stringify(response)); + + await privateClient.handleResponse(response); + + expect(privateClient.token).toBe('test_token'); + expect(privateClient.tokenExpiry).toBeCloseTo(Math.floor(Date.now() / 1000) + 3600, -1); + expect(saveTokenSpy).toHaveBeenCalledOnce(); + expect(saveTokenSpy).toHaveBeenCalledWith('test_token', expect.any(Number)); + }); + + it('should save the token to the file system', async () => { + const client = new BaseQuotientClient('test_api_key'); + const privateClient = client as any; + + // Mock fs operations + const mkdirSpy = vi.spyOn(fs, 'mkdirSync'); + const writeSpy = vi.spyOn(fs, 'writeFileSync'); + + const response = { + data: { + token: 'test_token', + expires_at: Date.now() + 1000 * 60 * 60 * 24, // 1 day from now + }, + headers: { + 'x-jwt-token': 'test_token', + }, + }; - it('should read the token from the file system', async () => { - const client = new BaseQuotientClient('test_api_key'); - const privateClient = client as any; - - const mockExpiry = Math.floor(Date.now() / 1000) + 3600; - - vi.mocked(fs.existsSync).mockReturnValue(true); - vi.mocked(fs.readFileSync).mockReturnValue(JSON.stringify({ - token: 'test_token', - expires_at: mockExpiry - })); - - await privateClient.loadToken(); - - expect(privateClient.token).toBe('test_token'); - expect(privateClient.tokenExpiry).toBe(mockExpiry); + await privateClient.handleResponse(response); + + // Verify token is in memory + expect(privateClient.token).toBe('test_token'); + expect(privateClient.tokenExpiry).toBeCloseTo(Math.floor(Date.now() / 1000) + 3600, -1); + + // Verify file system operations + expect(mkdirSpy).toHaveBeenCalledWith(expect.any(String), { recursive: true }); + expect(writeSpy).toHaveBeenCalledWith( + expect.any(String), + expect.stringContaining('"token":"test_token"') + ); + const writeCall = writeSpy.mock.calls[0]; + const writtenData = writeCall[1] as string; + expect(typeof JSON.parse(writtenData).expires_at).toBe('number'); + }); + + it('should read the token from the file system', async () => { + const client = new BaseQuotientClient('test_api_key'); + const privateClient = client as any; + + const mockExpiry = Math.floor(Date.now() / 1000) + 3600; + + vi.mocked(fs.existsSync).mockReturnValue(true); + vi.mocked(fs.readFileSync).mockReturnValue( + JSON.stringify({ + token: 'test_token', + expires_at: mockExpiry, + }) + ); + + await privateClient.loadToken(); + + expect(privateClient.token).toBe('test_token'); + expect(privateClient.tokenExpiry).toBe(mockExpiry); + }); + + it('should handle token loading failures gracefully', async () => { + const client = new BaseQuotientClient('test_api_key'); + const privateClient = client as any; + + // Reset token state + privateClient.token = null; + privateClient.tokenExpiry = 0; + + // Mock file exists but reading fails + vi.mocked(fs.existsSync).mockReturnValue(true); + vi.mocked(fs.readFileSync).mockImplementationOnce(() => { + throw new Error('Failed to read file'); }); - it('should handle token loading failures gracefully', async () => { - const client = new BaseQuotientClient('test_api_key'); - const privateClient = client as any; - - // Reset token state - privateClient.token = null; - privateClient.tokenExpiry = 0; - - // Mock file exists but reading fails - vi.mocked(fs.existsSync).mockReturnValue(true); - vi.mocked(fs.readFileSync).mockImplementationOnce(() => { - throw new Error('Failed to read file'); - }); - - privateClient.loadToken(); - - // Token should remain null if loading fails - expect(privateClient.token).toBeNull(); - expect(privateClient.tokenExpiry).toBe(0); + privateClient.loadToken(); + + // Token should remain null if loading fails + expect(privateClient.token).toBeNull(); + expect(privateClient.tokenExpiry).toBe(0); + }); + + it('should correctly validate token status', () => { + const client = new BaseQuotientClient('test_api_key'); + const privateClient = client as any; + + // Mock loadToken to not reset our test state + const originalLoadToken = privateClient.loadToken; + privateClient.loadToken = vi.fn(); + + // Test valid token + privateClient.token = 'valid_token'; + privateClient.tokenExpiry = Math.floor(Date.now() / 1000) + 3600; // 1 hour from now + privateClient.tokenApiKey = 'test_api_key'; + expect(privateClient.isTokenValid()).toBe(true); + + // Test expired token + privateClient.token = 'expired_token'; + privateClient.tokenExpiry = Math.floor(Date.now() / 1000) - 3600; // 1 hour ago + privateClient.tokenApiKey = 'test_api_key'; + expect(privateClient.isTokenValid()).toBe(false); + + // Test token expiring soon (within 5-minute buffer) + privateClient.token = 'expiring_token'; + privateClient.tokenExpiry = Math.floor(Date.now() / 1000) + 240; // 4 minutes from now + privateClient.tokenApiKey = 'test_api_key'; + expect(privateClient.isTokenValid()).toBe(false); + + // Test null token + privateClient.token = null; + privateClient.tokenExpiry = Math.floor(Date.now() / 1000) + 3600; + privateClient.tokenApiKey = 'test_api_key'; + expect(privateClient.isTokenValid()).toBe(false); + + // Test different API key + privateClient.token = 'valid_token'; + privateClient.tokenExpiry = Math.floor(Date.now() / 1000) + 3600; + privateClient.tokenApiKey = 'different_api_key'; + expect(privateClient.isTokenValid()).toBe(false); + + // Restore original loadToken + privateClient.loadToken = originalLoadToken; + }); + + it('should update auth header with valid token', () => { + const client = new BaseQuotientClient('test_api_key'); + const privateClient = client as any; + + // Mock loadToken to not reset our test state + const originalLoadToken = privateClient.loadToken; + privateClient.loadToken = vi.fn(); + + // Test valid token + privateClient.token = 'valid_token'; + privateClient.tokenExpiry = Math.floor(Date.now() / 1000) + 3600; // 1 hour from now + privateClient.tokenApiKey = 'test_api_key'; + privateClient.updateAuthHeader(); + expect(privateClient.client.defaults.headers.common['Authorization']).toBe( + 'Bearer valid_token' + ); + + // Test expired token + privateClient.token = 'expired_token'; + privateClient.tokenExpiry = Math.floor(Date.now() / 1000) - 3600; // 1 hour ago + privateClient.tokenApiKey = 'test_api_key'; + privateClient.updateAuthHeader(); + expect(privateClient.client.defaults.headers.common['Authorization']).toBe( + 'Bearer test_api_key' + ); + + // Test null token + privateClient.token = null; + privateClient.tokenExpiry = Math.floor(Date.now() / 1000) + 3600; + privateClient.tokenApiKey = 'test_api_key'; + privateClient.updateAuthHeader(); + expect(privateClient.client.defaults.headers.common['Authorization']).toBe( + 'Bearer test_api_key' + ); + + // Test token expiring soon (within 5-minute buffer) + privateClient.token = 'expiring_token'; + privateClient.tokenExpiry = Math.floor(Date.now() / 1000) + 240; // 4 minutes from now + privateClient.tokenApiKey = 'test_api_key'; + privateClient.updateAuthHeader(); + expect(privateClient.client.defaults.headers.common['Authorization']).toBe( + 'Bearer test_api_key' + ); + + // Test different API key + privateClient.token = 'valid_token'; + privateClient.tokenExpiry = Math.floor(Date.now() / 1000) + 3600; + privateClient.tokenApiKey = 'different_api_key'; + privateClient.updateAuthHeader(); + expect(privateClient.client.defaults.headers.common['Authorization']).toBe( + 'Bearer test_api_key' + ); + + // Restore original loadToken + privateClient.loadToken = originalLoadToken; + }); + + it('should handle token directory fallbacks', () => { + // Mock os.homedir to throw + vi.mocked(os.homedir).mockImplementation(() => { + throw new Error('homedir not available'); }); - it('should correctly validate token status', () => { - const client = new BaseQuotientClient('test_api_key'); - const privateClient = client as any; - - // Mock loadToken to not reset our test state - const originalLoadToken = privateClient.loadToken; - privateClient.loadToken = vi.fn(); - - // Test valid token - privateClient.token = 'valid_token'; - privateClient.tokenExpiry = Math.floor(Date.now() / 1000) + 3600; // 1 hour from now - privateClient.tokenApiKey = 'test_api_key'; - expect(privateClient.isTokenValid()).toBe(true); - - // Test expired token - privateClient.token = 'expired_token'; - privateClient.tokenExpiry = Math.floor(Date.now() / 1000) - 3600; // 1 hour ago - privateClient.tokenApiKey = 'test_api_key'; - expect(privateClient.isTokenValid()).toBe(false); - - // Test token expiring soon (within 5-minute buffer) - privateClient.token = 'expiring_token'; - privateClient.tokenExpiry = Math.floor(Date.now() / 1000) + 240; // 4 minutes from now - privateClient.tokenApiKey = 'test_api_key'; - expect(privateClient.isTokenValid()).toBe(false); - - // Test null token - privateClient.token = null; - privateClient.tokenExpiry = Math.floor(Date.now() / 1000) + 3600; - privateClient.tokenApiKey = 'test_api_key'; - expect(privateClient.isTokenValid()).toBe(false); - - // Test different API key - privateClient.token = 'valid_token'; - privateClient.tokenExpiry = Math.floor(Date.now() / 1000) + 3600; - privateClient.tokenApiKey = 'different_api_key'; - expect(privateClient.isTokenValid()).toBe(false); - - // Restore original loadToken - privateClient.loadToken = originalLoadToken; - }); + // First test: when /root exists + vi.mocked(fs.existsSync).mockReturnValueOnce(true); + let client = new BaseQuotientClient('test_api_key'); + let privateClient = client as any; + expect(privateClient.tokenPath).toContain('/root/.quotient/pi_keyauth_token.json'); + + // Second test: when /root doesn't exist + vi.mocked(fs.existsSync).mockReturnValueOnce(false); + client = new BaseQuotientClient('test_api_key'); + privateClient = client as any; + expect(privateClient.tokenPath).toContain(process.cwd() + '/.quotient/pi_keyauth_token.json'); + + // Reset the mock to test homedir success case + vi.mocked(os.homedir).mockReturnValue('/home/user'); + client = new BaseQuotientClient('test_api_key'); + privateClient = client as any; + expect(privateClient.tokenPath).toContain('/home/user/.quotient/pi_keyauth_token.json'); + }); + + it('should handle successful JWT decoding', async () => { + const client = new BaseQuotientClient('test_api_key'); + const privateClient = client as any; + + const mockExpiry = Math.floor(Date.now() / 1000) + 3600; + vi.mocked(jwt.decode).mockReturnValueOnce({ exp: mockExpiry }); + + // Mock file system data to include API key + vi.mocked(fs.readFileSync).mockReturnValue( + JSON.stringify({ + token: 'valid_jwt_token', + expires_at: mockExpiry, + api_key: 'test_api_key', + }) + ); + vi.mocked(fs.existsSync).mockReturnValue(true); + + const response = { + data: {}, + headers: { + 'x-jwt-token': 'valid_jwt_token', + }, + }; - it('should update auth header with valid token', () => { - const client = new BaseQuotientClient('test_api_key'); - const privateClient = client as any; - - // Mock loadToken to not reset our test state - const originalLoadToken = privateClient.loadToken; - privateClient.loadToken = vi.fn(); - - // Test valid token - privateClient.token = 'valid_token'; - privateClient.tokenExpiry = Math.floor(Date.now() / 1000) + 3600; // 1 hour from now - privateClient.tokenApiKey = 'test_api_key'; - privateClient.updateAuthHeader(); - expect(privateClient.client.defaults.headers.common['Authorization']).toBe('Bearer valid_token'); - - // Test expired token - privateClient.token = 'expired_token'; - privateClient.tokenExpiry = Math.floor(Date.now() / 1000) - 3600; // 1 hour ago - privateClient.tokenApiKey = 'test_api_key'; - privateClient.updateAuthHeader(); - expect(privateClient.client.defaults.headers.common['Authorization']).toBe('Bearer test_api_key'); - - // Test null token - privateClient.token = null; - privateClient.tokenExpiry = Math.floor(Date.now() / 1000) + 3600; - privateClient.tokenApiKey = 'test_api_key'; - privateClient.updateAuthHeader(); - expect(privateClient.client.defaults.headers.common['Authorization']).toBe('Bearer test_api_key'); - - // Test token expiring soon (within 5-minute buffer) - privateClient.token = 'expiring_token'; - privateClient.tokenExpiry = Math.floor(Date.now() / 1000) + 240; // 4 minutes from now - privateClient.tokenApiKey = 'test_api_key'; - privateClient.updateAuthHeader(); - expect(privateClient.client.defaults.headers.common['Authorization']).toBe('Bearer test_api_key'); - - // Test different API key - privateClient.token = 'valid_token'; - privateClient.tokenExpiry = Math.floor(Date.now() / 1000) + 3600; - privateClient.tokenApiKey = 'different_api_key'; - privateClient.updateAuthHeader(); - expect(privateClient.client.defaults.headers.common['Authorization']).toBe('Bearer test_api_key'); - - // Restore original loadToken - privateClient.loadToken = originalLoadToken; - }); - - - it('should handle token directory fallbacks', () => { - // Mock os.homedir to throw - vi.mocked(os.homedir).mockImplementation(() => { - throw new Error('homedir not available'); - }); - - // First test: when /root exists - vi.mocked(fs.existsSync).mockReturnValueOnce(true); - let client = new BaseQuotientClient('test_api_key'); - let privateClient = client as any; - expect(privateClient.tokenPath).toContain('/root/.quotient/pi_keyauth_token.json'); - - // Second test: when /root doesn't exist - vi.mocked(fs.existsSync).mockReturnValueOnce(false); - client = new BaseQuotientClient('test_api_key'); - privateClient = client as any; - expect(privateClient.tokenPath).toContain(process.cwd() + '/.quotient/pi_keyauth_token.json'); - - // Reset the mock to test homedir success case - vi.mocked(os.homedir).mockReturnValue('/home/user'); - client = new BaseQuotientClient('test_api_key'); - privateClient = client as any; - expect(privateClient.tokenPath).toContain('/home/user/.quotient/pi_keyauth_token.json'); - }); + await privateClient.handleResponse(response); + + expect(privateClient.token).toBe('valid_jwt_token'); + expect(privateClient.tokenExpiry).toBe(mockExpiry); + expect(privateClient.tokenApiKey).toBe('test_api_key'); + }); + + it('should handle JWT tokens without expiration', async () => { + const client = new BaseQuotientClient('test_api_key'); + const privateClient = client as any; + + // Mock JWT decode to return token without exp + vi.mocked(jwt.decode).mockReturnValueOnce({}); + + // Mock file system data to include API key + const beforeTime = Math.floor(Date.now() / 1000); + vi.mocked(fs.readFileSync).mockReturnValue( + JSON.stringify({ + token: 'token_without_exp', + expires_at: beforeTime + 3600, + api_key: 'test_api_key', + }) + ); + vi.mocked(fs.existsSync).mockReturnValue(true); + + const response = { + data: {}, + headers: { + 'x-jwt-token': 'token_without_exp', + }, + }; - it('should handle successful JWT decoding', async () => { - const client = new BaseQuotientClient('test_api_key'); - const privateClient = client as any; - - const mockExpiry = Math.floor(Date.now() / 1000) + 3600; - vi.mocked(jwt.decode).mockReturnValueOnce({ exp: mockExpiry }); - - // Mock file system data to include API key - vi.mocked(fs.readFileSync).mockReturnValue(JSON.stringify({ - token: 'valid_jwt_token', - expires_at: mockExpiry, - api_key: 'test_api_key' - })); - vi.mocked(fs.existsSync).mockReturnValue(true); - - const response = { - data: {}, - headers: { - 'x-jwt-token': 'valid_jwt_token' - } - }; - - await privateClient.handleResponse(response); - - expect(privateClient.token).toBe('valid_jwt_token'); - expect(privateClient.tokenExpiry).toBe(mockExpiry); - expect(privateClient.tokenApiKey).toBe('test_api_key'); - }); + await privateClient.handleResponse(response); - it('should handle JWT tokens without expiration', async () => { - const client = new BaseQuotientClient('test_api_key'); - const privateClient = client as any; - - // Mock JWT decode to return token without exp - vi.mocked(jwt.decode).mockReturnValueOnce({}); - - // Mock file system data to include API key - const beforeTime = Math.floor(Date.now() / 1000); - vi.mocked(fs.readFileSync).mockReturnValue(JSON.stringify({ - token: 'token_without_exp', - expires_at: beforeTime + 3600, - api_key: 'test_api_key' - })); - vi.mocked(fs.existsSync).mockReturnValue(true); - - const response = { - data: {}, - headers: { - 'x-jwt-token': 'token_without_exp' - } - }; - - await privateClient.handleResponse(response); - - expect(privateClient.token).toBe('token_without_exp'); - // Should default to 1 hour from now - expect(privateClient.tokenExpiry).toBeGreaterThanOrEqual(beforeTime + 3600); - expect(privateClient.tokenApiKey).toBe('test_api_key'); - }); + expect(privateClient.token).toBe('token_without_exp'); + // Should default to 1 hour from now + expect(privateClient.tokenExpiry).toBeGreaterThanOrEqual(beforeTime + 3600); + expect(privateClient.tokenApiKey).toBe('test_api_key'); + }); - it('should handle JWT decode failures', async () => { - const client = new BaseQuotientClient('test_api_key'); - const privateClient = client as any; - - // Save initial state - const initialToken = privateClient.token; - const initialExpiry = privateClient.tokenExpiry; - - // Mock JWT decode to throw - vi.mocked(jwt.decode).mockImplementationOnce(() => { - throw new Error('Invalid token'); - }); - - const response = { - data: {}, - headers: { - 'x-jwt-token': 'invalid_jwt_token' - } - }; - - await privateClient.handleResponse(response); - - // Verify state remains unchanged after failure - expect(privateClient.token).toBe(initialToken); - expect(privateClient.tokenExpiry).toBe(initialExpiry); - }); + it('should handle JWT decode failures', async () => { + const client = new BaseQuotientClient('test_api_key'); + const privateClient = client as any; - it('should handle directory creation failures', async () => { - const client = new BaseQuotientClient('test_api_key'); - const privateClient = client as any; - - // Mock mkdirSync to throw - vi.mocked(fs.mkdirSync).mockImplementationOnce(() => { - throw new Error('Permission denied'); - }); - - // Spy on console.error - const consoleErrorSpy = vi.spyOn(console, 'error'); - - // Attempt to save token - privateClient.saveToken('test_token', Math.floor(Date.now() / 1000) + 3600); - - // Verify token is still set in memory even though file save failed - expect(privateClient.token).toBe('test_token'); - expect(privateClient.tokenExpiry).toBe(Math.floor(Date.now() / 1000) + 3600); - - // Verify error was logged - expect(consoleErrorSpy).toHaveBeenCalledWith( - expect.stringContaining('Could not create directory for token') - ); - }); + // Save initial state + const initialToken = privateClient.token; + const initialExpiry = privateClient.tokenExpiry; - it('should handle GET requests correctly', async () => { - const client = new BaseQuotientClient('test_api_key'); - const privateClient = client as any; - - const mockResponse = { data: { result: 'success' } }; - privateClient.client.get = vi.fn().mockResolvedValue(mockResponse); - - const result = await client.get('/test', { param: 'value' }, 1000); - - expect(privateClient.client.get).toHaveBeenCalledWith( - '/test', - { params: { param: 'value' }, timeout: 1000 } - ); - expect(result).toBe(mockResponse.data); + // Mock JWT decode to throw + vi.mocked(jwt.decode).mockImplementationOnce(() => { + throw new Error('Invalid token'); }); - it('should handle POST requests and filter null values', async () => { - const client = new BaseQuotientClient('test_api_key'); - const privateClient = client as any; - - const mockResponse = { data: { result: 'success' } }; - privateClient.client.post = vi.fn().mockResolvedValue(mockResponse); - - const data = { - valid: 'value', - nullValue: null, - nested: { valid: true } - }; - - const result = await client.post('/test', data, 1000); - - expect(privateClient.client.post).toHaveBeenCalledWith( - '/test', - { valid: 'value', nested: { valid: true } }, - { timeout: 1000 } - ); - expect(result).toBe(mockResponse.data); - }); + const response = { + data: {}, + headers: { + 'x-jwt-token': 'invalid_jwt_token', + }, + }; + + await privateClient.handleResponse(response); + + // Verify state remains unchanged after failure + expect(privateClient.token).toBe(initialToken); + expect(privateClient.tokenExpiry).toBe(initialExpiry); + }); - it('should handle POST requests with array data', async () => { - const client = new BaseQuotientClient('test_api_key'); - const privateClient = client as any; - - const mockResponse = { data: { result: 'success' } }; - privateClient.client.post = vi.fn().mockResolvedValue(mockResponse); - - const data = ['value1', null, 'value2']; - - const result = await client.post('/test', data, 1000); - - expect(privateClient.client.post).toHaveBeenCalledWith( - '/test', - ['value1', 'value2'], - { timeout: 1000 } - ); - expect(result).toBe(mockResponse.data); + it('should handle directory creation failures', async () => { + const client = new BaseQuotientClient('test_api_key'); + const privateClient = client as any; + + // Mock mkdirSync to throw + vi.mocked(fs.mkdirSync).mockImplementationOnce(() => { + throw new Error('Permission denied'); }); - it('should handle PATCH requests and filter null values', async () => { - const client = new BaseQuotientClient('test_api_key'); - const privateClient = client as any; - - const mockResponse = { data: { result: 'success' } }; - privateClient.client.patch = vi.fn().mockResolvedValue(mockResponse); - - const data = { - valid: 'value', - nullValue: null, - nested: { valid: true } - }; - - const result = await client.patch('/test', data, 1000); - - expect(privateClient.client.patch).toHaveBeenCalledWith( - '/test', - { valid: 'value', nested: { valid: true } }, - { timeout: 1000 } - ); - expect(result).toBe(mockResponse.data); + // Spy on console.error + const consoleErrorSpy = vi.spyOn(console, 'error'); + + // Attempt to save token + privateClient.saveToken('test_token', Math.floor(Date.now() / 1000) + 3600); + + // Verify token is still set in memory even though file save failed + expect(privateClient.token).toBe('test_token'); + expect(privateClient.tokenExpiry).toBe(Math.floor(Date.now() / 1000) + 3600); + + // Verify error was logged + expect(consoleErrorSpy).toHaveBeenCalledWith( + expect.stringContaining('Could not create directory for token') + ); + }); + + it('should handle GET requests correctly', async () => { + const client = new BaseQuotientClient('test_api_key'); + const privateClient = client as any; + + const mockResponse = { data: { result: 'success' } }; + privateClient.client.get = vi.fn().mockResolvedValue(mockResponse); + + const result = await client.get('/test', { param: 'value' }, 1000); + + expect(privateClient.client.get).toHaveBeenCalledWith('/test', { + params: { param: 'value' }, + timeout: 1000, }); + expect(result).toBe(mockResponse.data); + }); + + it('should handle POST requests and filter null values', async () => { + const client = new BaseQuotientClient('test_api_key'); + const privateClient = client as any; + + const mockResponse = { data: { result: 'success' } }; + privateClient.client.post = vi.fn().mockResolvedValue(mockResponse); + + const data = { + valid: 'value', + nullValue: null, + nested: { valid: true }, + }; + + const result = await client.post('/test', data, 1000); + + expect(privateClient.client.post).toHaveBeenCalledWith( + '/test', + { valid: 'value', nested: { valid: true } }, + { timeout: 1000 } + ); + expect(result).toBe(mockResponse.data); + }); + + it('should handle POST requests with array data', async () => { + const client = new BaseQuotientClient('test_api_key'); + const privateClient = client as any; - it('should handle DELETE requests', async () => { - const client = new BaseQuotientClient('test_api_key'); - const privateClient = client as any; - - const mockResponse = { data: { result: 'success' } }; - privateClient.client.delete = vi.fn().mockResolvedValue(mockResponse); - - const result = await client.delete('/test', 1000); - - expect(privateClient.client.delete).toHaveBeenCalledWith( - '/test', - { timeout: 1000 } - ); - expect(result).toBe(mockResponse.data); + const mockResponse = { data: { result: 'success' } }; + privateClient.client.post = vi.fn().mockResolvedValue(mockResponse); + + const data = ['value1', null, 'value2']; + + const result = await client.post('/test', data, 1000); + + expect(privateClient.client.post).toHaveBeenCalledWith('/test', ['value1', 'value2'], { + timeout: 1000, }); + expect(result).toBe(mockResponse.data); + }); + + it('should handle PATCH requests and filter null values', async () => { + const client = new BaseQuotientClient('test_api_key'); + const privateClient = client as any; + + const mockResponse = { data: { result: 'success' } }; + privateClient.client.patch = vi.fn().mockResolvedValue(mockResponse); + + const data = { + valid: 'value', + nullValue: null, + nested: { valid: true }, + }; + + const result = await client.patch('/test', data, 1000); + + expect(privateClient.client.patch).toHaveBeenCalledWith( + '/test', + { valid: 'value', nested: { valid: true } }, + { timeout: 1000 } + ); + expect(result).toBe(mockResponse.data); + }); + it('should handle DELETE requests', async () => { + const client = new BaseQuotientClient('test_api_key'); + const privateClient = client as any; -}); \ No newline at end of file + const mockResponse = { data: { result: 'success' } }; + privateClient.client.delete = vi.fn().mockResolvedValue(mockResponse); + + const result = await client.delete('/test', 1000); + + expect(privateClient.client.delete).toHaveBeenCalledWith('/test', { timeout: 1000 }); + expect(result).toBe(mockResponse.data); + }); +}); diff --git a/tests/index.test.ts b/tests/index.test.ts index c063c73..3b586df 100644 --- a/tests/index.test.ts +++ b/tests/index.test.ts @@ -3,166 +3,186 @@ import { QuotientAI } from '../quotientai/index'; import { BaseQuotientClient } from '../quotientai/client'; vi.mock('../quotientai/client', () => { - return { - BaseQuotientClient: vi.fn().mockImplementation((apiKey) => { - return { - apiKey, - client: { - defaults: { - headers: { - common: {} - } - } - }, - get: vi.fn(), - post: vi.fn(), - patch: vi.fn(), - delete: vi.fn() - }; - }) - }; + return { + BaseQuotientClient: vi.fn().mockImplementation((apiKey) => { + return { + apiKey, + client: { + defaults: { + headers: { + common: {}, + }, + }, + }, + get: vi.fn(), + post: vi.fn(), + patch: vi.fn(), + delete: vi.fn(), + }; + }), + }; }); // Mock AuthResource at the module level const mockAuthenticate = vi.fn().mockResolvedValue({}); vi.mock('../quotientai/resources/auth', () => { - return { - AuthResource: vi.fn().mockImplementation(() => ({ - authenticate: mockAuthenticate, - client: new BaseQuotientClient('test_api_key') - })) - }; + return { + AuthResource: vi.fn().mockImplementation(() => ({ + authenticate: mockAuthenticate, + client: new BaseQuotientClient('test_api_key'), + })), + }; }); describe('QuotientAI', () => { - beforeEach(() => { - // Reset environment variable and mocks before each test - process.env.QUOTIENT_API_KEY = undefined; - vi.clearAllMocks(); - }); + beforeEach(() => { + // Reset environment variable and mocks before each test + process.env.QUOTIENT_API_KEY = undefined; + vi.clearAllMocks(); + }); - it('should initialize with the correct api key', () => { - new QuotientAI('test_api_key'); - expect(BaseQuotientClient).toHaveBeenCalledWith('test_api_key'); - }); + it('should initialize with the correct api key', () => { + new QuotientAI('test_api_key'); + expect(BaseQuotientClient).toHaveBeenCalledWith('test_api_key'); + }); - it('should initialize with the correct api key from environment variable', () => { - process.env.QUOTIENT_API_KEY = 'test_api_key'; - new QuotientAI(); - expect(BaseQuotientClient).toHaveBeenCalledWith('test_api_key'); - }); + it('should initialize with the correct api key from environment variable', () => { + process.env.QUOTIENT_API_KEY = 'test_api_key'; + new QuotientAI(); + expect(BaseQuotientClient).toHaveBeenCalledWith('test_api_key'); + }); - it('should log an error if no api key is provided', () => { - process.env.QUOTIENT_API_KEY = ''; - const consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); - new QuotientAI() - expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('Could not find API key. Either pass apiKey to QuotientAI() or set the QUOTIENT_API_KEY environment variable. If you do not have an API key, you can create one at https://app.quotientai.co in your settings page')); - }); - - it('should call the auth resource on initialization', async () => { - new QuotientAI('test_api_key'); - await new Promise(resolve => setTimeout(resolve, 0)); - expect(mockAuthenticate).toHaveBeenCalledOnce(); + it('should log an error if no api key is provided', () => { + process.env.QUOTIENT_API_KEY = ''; + const consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + new QuotientAI(); + expect(consoleErrorSpy).toHaveBeenCalledWith( + expect.stringContaining( + 'Could not find API key. Either pass apiKey to QuotientAI() or set the QUOTIENT_API_KEY environment variable. If you do not have an API key, you can create one at https://app.quotientai.co in your settings page' + ) + ); + }); + + it('should call the auth resource on initialization', async () => { + new QuotientAI('test_api_key'); + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(mockAuthenticate).toHaveBeenCalledOnce(); + }); + + it('should log an error if parameters are invalid', async () => { + const consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + const quotientAI = new QuotientAI('test_api_key'); + await quotientAI.evaluate({ + prompt: { + id: 'test_id', + name: 'test_name', + content: 'test_content', + version: 1, + user_prompt: 'test_prompt', + created_at: new Date(), + updated_at: new Date(), + }, + dataset: { + id: 'test_dataset', + name: 'test', + created_at: new Date(), + created_by: 'test_user', + updated_at: new Date(), + }, + model: { + id: 'test_model', + name: 'test', + provider: { id: 'test', name: 'test' }, + created_at: new Date(), + }, + parameters: { + invalid_param: 'value', + }, + metrics: ['test_metric'], }); + expect(consoleErrorSpy).toHaveBeenCalledWith( + expect.stringContaining( + 'Invalid parameters: invalid_param. Valid parameters are: temperature, top_k, top_p, max_tokens' + ) + ); + }); - it('should log an error if parameters are invalid', async () => { - const consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); - const quotientAI = new QuotientAI('test_api_key'); - await quotientAI.evaluate({ - prompt: { - id: 'test_id', - name: 'test_name', - content: 'test_content', - version: 1, - user_prompt: 'test_prompt', - created_at: new Date(), - updated_at: new Date() - }, - dataset: { - id: 'test_dataset', - name: 'test', - created_at: new Date(), - created_by: 'test_user', - updated_at: new Date() - }, - model: { id: 'test_model', name: 'test', provider: { id: 'test', name: 'test' }, created_at: new Date() }, - parameters: { - invalid_param: 'value' - }, - metrics: ['test_metric'] - }) - expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('Invalid parameters: invalid_param. Valid parameters are: temperature, top_k, top_p, max_tokens')); + it('should successfully evaluate if all parameters are valid', async () => { + const quotientAI = new QuotientAI('test_api_key'); + const mockRun = { + id: 'test_run', + status: 'completed', + }; + + // Mock the runs.create method + quotientAI.runs.create = vi.fn().mockResolvedValue(mockRun); + + const result = await quotientAI.evaluate({ + prompt: { + id: 'test_id', + name: 'test_name', + content: 'test_content', + version: 1, + user_prompt: 'test_prompt', + created_at: new Date(), + updated_at: new Date(), + }, + dataset: { + id: 'test_dataset', + name: 'test', + created_at: new Date(), + created_by: 'test_user', + updated_at: new Date(), + }, + model: { + id: 'test_model', + name: 'test', + provider: { id: 'test', name: 'test' }, + created_at: new Date(), + }, + parameters: { + temperature: 0.5, + top_k: 10, + top_p: 0.9, + max_tokens: 100, + }, + metrics: ['test_metric'], }); - it('should successfully evaluate if all parameters are valid', async () => { - const quotientAI = new QuotientAI('test_api_key'); - const mockRun = { - id: 'test_run', - status: 'completed' - }; - - // Mock the runs.create method - quotientAI.runs.create = vi.fn().mockResolvedValue(mockRun); - - const result = await quotientAI.evaluate({ - prompt: { - id: 'test_id', - name: 'test_name', - content: 'test_content', - version: 1, - user_prompt: 'test_prompt', - created_at: new Date(), - updated_at: new Date() - }, - dataset: { - id: 'test_dataset', - name: 'test', - created_at: new Date(), - created_by: 'test_user', - updated_at: new Date() - }, - model: { id: 'test_model', name: 'test', provider: { id: 'test', name: 'test' }, created_at: new Date() }, - parameters: { - temperature: 0.5, - top_k: 10, - top_p: 0.9, - max_tokens: 100, - }, - metrics: ['test_metric'], - }); - - expect(result).toBe(mockRun); - expect(quotientAI.runs.create).toHaveBeenCalledWith({ - prompt: expect.objectContaining({ id: 'test_id' }), - dataset: expect.objectContaining({ id: 'test_dataset' }), - model: expect.objectContaining({ id: 'test_model' }), - parameters: { - max_tokens: 100, - temperature: 0.5, - top_k: 10, - top_p: 0.9, - }, - metrics: ['test_metric'] - }); + expect(result).toBe(mockRun); + expect(quotientAI.runs.create).toHaveBeenCalledWith({ + prompt: expect.objectContaining({ id: 'test_id' }), + dataset: expect.objectContaining({ id: 'test_dataset' }), + model: expect.objectContaining({ id: 'test_model' }), + parameters: { + max_tokens: 100, + temperature: 0.5, + top_k: 10, + top_p: 0.9, + }, + metrics: ['test_metric'], }); + }); - it('should handle authentication errors during initialization', () => { - // Mock the authenticate method to throw an error - const error = new Error('Authentication failed'); - mockAuthenticate.mockImplementationOnce(() => { - throw error; - }); - - // Spy on console.error - const consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); - - // Create a new instance - const quotient = new QuotientAI('test_api_key'); - - // Verify error was logged with correct message - expect(consoleErrorSpy).toHaveBeenCalledWith( - expect.stringContaining('If you are seeing this error, please check that your API key is correct.') - ); - expect(quotient).toBeDefined(); + it('should handle authentication errors during initialization', () => { + // Mock the authenticate method to throw an error + const error = new Error('Authentication failed'); + mockAuthenticate.mockImplementationOnce(() => { + throw error; }); + + // Spy on console.error + const consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + + // Create a new instance + const quotient = new QuotientAI('test_api_key'); + + // Verify error was logged with correct message + expect(consoleErrorSpy).toHaveBeenCalledWith( + expect.stringContaining( + 'If you are seeing this error, please check that your API key is correct.' + ) + ); + expect(quotient).toBeDefined(); + }); }); diff --git a/tests/logger.test.ts b/tests/logger.test.ts index bed98b9..74656c1 100644 --- a/tests/logger.test.ts +++ b/tests/logger.test.ts @@ -2,332 +2,359 @@ import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; import { QuotientLogger } from '../quotientai/logger'; describe('QuotientLogger', () => { - let consoleErrorSpy: any; + let consoleErrorSpy: any; - beforeEach(() => { - consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + beforeEach(() => { + consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + }); + + afterEach(() => { + consoleErrorSpy.mockRestore(); + }); + + it('should initialize without being configured', () => { + const mockLogsResource = { create: vi.fn(), list: vi.fn() }; + const logger = new QuotientLogger(mockLogsResource); + const privateLogger = logger as any; + expect(logger).toBeDefined(); + expect(privateLogger.configured).toBe(false); + expect(privateLogger.sampleRate).toBe(1.0); + expect(privateLogger.tags).toEqual({}); + expect(privateLogger.hallucinationDetection).toBe(false); + expect(privateLogger.inconsistencyDetection).toBe(false); + expect(privateLogger.hallucinationDetectionSampleRate).toBe(0.0); + }); + + it('should use default values when not provided', () => { + const mockLogsResource = { create: vi.fn(), list: vi.fn() }; + const logger = new QuotientLogger(mockLogsResource); + const privateLogger = logger as any; + + privateLogger.init({ + appName: 'test_app', + environment: 'test_environment', }); - afterEach(() => { - consoleErrorSpy.mockRestore(); + expect(privateLogger.sampleRate).toBe(1.0); + expect(privateLogger.appName).toBe('test_app'); + expect(privateLogger.environment).toBe('test_environment'); + expect(privateLogger.tags).toEqual({}); + expect(privateLogger.hallucinationDetection).toBe(false); + expect(privateLogger.inconsistencyDetection).toBe(false); + expect(privateLogger.hallucinationDetectionSampleRate).toBe(0.0); + }); + + it('should initialize with the correct properties', () => { + const mockLogsResource = { create: vi.fn(), list: vi.fn() }; + const logger = new QuotientLogger(mockLogsResource); + const privateLogger = logger as any; + + privateLogger.init({ + appName: 'test_app', + environment: 'test_environment', + tags: { test: 'test' }, + sampleRate: 0.5, + hallucinationDetection: true, + inconsistencyDetection: true, + hallucinationDetectionSampleRate: 0.5, }); + expect(privateLogger.appName).toBe('test_app'); + expect(privateLogger.environment).toBe('test_environment'); + expect(privateLogger.tags).toEqual({ test: 'test' }); + expect(privateLogger.sampleRate).toBe(0.5); + expect(privateLogger.hallucinationDetection).toBe(true); + expect(privateLogger.inconsistencyDetection).toBe(true); + expect(privateLogger.hallucinationDetectionSampleRate).toBe(0.5); + expect(privateLogger.configured).toBe(true); + }); + + it('should log error and return this if sample rate is not between 0 and 1', () => { + const mockLogsResource = { create: vi.fn(), list: vi.fn() }; + const logger = new QuotientLogger(mockLogsResource); + const privateLogger = logger as any; + const result = privateLogger.init({ sampleRate: 1.5 }); + expect(result).toBe(logger); + expect(consoleErrorSpy).toHaveBeenCalledWith( + expect.stringContaining('sampleRate must be between 0.0 and 1.0') + ); + }); - it('should initialize without being configured', () => { - const mockLogsResource = { create: vi.fn(), list: vi.fn() }; - const logger = new QuotientLogger(mockLogsResource); - const privateLogger = logger as any; - expect(logger).toBeDefined(); - expect(privateLogger.configured).toBe(false); - expect(privateLogger.sampleRate).toBe(1.0); - expect(privateLogger.tags).toEqual({}); - expect(privateLogger.hallucinationDetection).toBe(false); - expect(privateLogger.inconsistencyDetection).toBe(false); - expect(privateLogger.hallucinationDetectionSampleRate).toBe(0.0); + it('should log error and return null if you attempt to log before initializing', async () => { + const mockLogsResource = { create: vi.fn(), list: vi.fn() }; + const logger = new QuotientLogger(mockLogsResource); + const privateLogger = logger as any; + const result = await privateLogger.log({ message: 'test' }); + expect(result).toBeNull(); + expect(consoleErrorSpy).toHaveBeenCalledWith( + expect.stringContaining('Logger is not configured') + ); + expect(mockLogsResource.create).not.toHaveBeenCalled(); + }); + + it('should log a message if initialized', async () => { + const mockLogsResource = { create: vi.fn(), list: vi.fn() }; + const logger = new QuotientLogger(mockLogsResource); + const privateLogger = logger as any; + privateLogger.init({ + appName: 'test_app', + environment: 'test_environment', + tags: { test: 'test' }, + sampleRate: 1.0, + }); + await privateLogger.log({ message: 'test' }); + expect(mockLogsResource.create).toHaveBeenCalledWith({ + appName: 'test_app', + environment: 'test_environment', + tags: { test: 'test' }, + message: 'test', + hallucinationDetection: false, + hallucinationDetectionSampleRate: 0, + inconsistencyDetection: false, }); + }); - it('should use default values when not provided', () => { - const mockLogsResource = { create: vi.fn(), list: vi.fn() }; - const logger = new QuotientLogger(mockLogsResource); - const privateLogger = logger as any; - - privateLogger.init({ - appName: 'test_app', - environment: 'test_environment' - }); - - expect(privateLogger.sampleRate).toBe(1.0); - expect(privateLogger.appName).toBe('test_app'); - expect(privateLogger.environment).toBe('test_environment'); - expect(privateLogger.tags).toEqual({}); - expect(privateLogger.hallucinationDetection).toBe(false); - expect(privateLogger.inconsistencyDetection).toBe(false); - expect(privateLogger.hallucinationDetectionSampleRate).toBe(0.0); + it('should not log a message if shouldSample returns false', async () => { + const mockLogsResource = { create: vi.fn(), list: vi.fn() }; + const logger = new QuotientLogger(mockLogsResource); + const privateLogger = logger as any; + // Mock shouldSample to always return false + vi.spyOn(privateLogger, 'shouldSample').mockReturnValue(false); + + privateLogger.init({ + appName: 'test_app', + environment: 'test_environment', + tags: { test: 'test' }, + sampleRate: 0.5, }); - - it('should initialize with the correct properties', () => { - const mockLogsResource = { create: vi.fn(), list: vi.fn() }; - const logger = new QuotientLogger(mockLogsResource); - const privateLogger = logger as any; - - privateLogger.init({ - appName: 'test_app', - environment: 'test_environment', - tags: { test: 'test' }, - sampleRate: 0.5, - hallucinationDetection: true, - inconsistencyDetection: true, - hallucinationDetectionSampleRate: 0.5, - }); - expect(privateLogger.appName).toBe('test_app'); - expect(privateLogger.environment).toBe('test_environment'); - expect(privateLogger.tags).toEqual({ test: 'test' }); - expect(privateLogger.sampleRate).toBe(0.5); - expect(privateLogger.hallucinationDetection).toBe(true); - expect(privateLogger.inconsistencyDetection).toBe(true); - expect(privateLogger.hallucinationDetectionSampleRate).toBe(0.5); - expect(privateLogger.configured).toBe(true); + await privateLogger.log({ message: 'test' }); + expect(mockLogsResource.create).not.toHaveBeenCalled(); + }); + + it('should verify shouldSample behavior based on Math.random', () => { + const mockLogsResource = { create: vi.fn(), list: vi.fn() }; + const logger = new QuotientLogger(mockLogsResource); + const privateLogger = logger as any; + privateLogger.init({ sampleRate: 0.5 }); + + // Test when random is less than sample rate + vi.spyOn(Math, 'random').mockReturnValue(0.4); + expect(privateLogger.shouldSample()).toBe(true); + + // Test when random is greater than sample rate + vi.spyOn(Math, 'random').mockReturnValue(0.6); + expect(privateLogger.shouldSample()).toBe(false); + + vi.spyOn(Math, 'random').mockRestore(); + }); + + it('should log error and return null if required appName or environment is missing after initialization', async () => { + const mockLogsResource = { create: vi.fn(), list: vi.fn() }; + const logger = new QuotientLogger(mockLogsResource); + const privateLogger = logger as any; + + // Initialize but with null values + privateLogger.init({}); + privateLogger.configured = true; + privateLogger.appName = null; + privateLogger.environment = 'test'; + + const result1 = await privateLogger.log({ message: 'test' }); + expect(result1).toBeNull(); + expect(consoleErrorSpy).toHaveBeenCalledWith( + expect.stringContaining('appName and environment must be set') + ); + expect(mockLogsResource.create).not.toHaveBeenCalled(); + + privateLogger.appName = 'test'; + privateLogger.environment = null; + + const result2 = await privateLogger.log({ message: 'test' }); + expect(result2).toBeNull(); + expect(consoleErrorSpy).toHaveBeenCalledWith( + expect.stringContaining('appName and environment must be set') + ); + expect(mockLogsResource.create).not.toHaveBeenCalled(); + }); + + describe('Document Validation', () => { + it('should accept string documents', async () => { + const mockLogsResource = { create: vi.fn(), list: vi.fn() }; + const logger = new QuotientLogger(mockLogsResource); + const privateLogger = logger as any; + + privateLogger.init({ appName: 'test_app', environment: 'test_environment' }); + + await privateLogger.log({ + userQuery: 'test', + modelOutput: 'test', + documents: ['This is a string document'], + }); + + expect(mockLogsResource.create).toHaveBeenCalled(); + }); + + it('should accept valid LogDocument objects', async () => { + const mockLogsResource = { create: vi.fn(), list: vi.fn() }; + const logger = new QuotientLogger(mockLogsResource); + const privateLogger = logger as any; + + privateLogger.init({ appName: 'test_app', environment: 'test_environment' }); + + await privateLogger.log({ + userQuery: 'test', + modelOutput: 'test', + documents: [ + { pageContent: 'This is valid content' }, + { pageContent: 'Also valid', metadata: { source: 'test' } }, + ], + }); + + expect(mockLogsResource.create).toHaveBeenCalled(); }); - - it('should log error and return this if sample rate is not between 0 and 1', () => { - const mockLogsResource = { create: vi.fn(), list: vi.fn() }; - const logger = new QuotientLogger(mockLogsResource); - const privateLogger = logger as any; - const result = privateLogger.init({ sampleRate: 1.5 }); - expect(result).toBe(logger); - expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('sampleRate must be between 0.0 and 1.0')); + + it('should log error and return null when document is missing pageContent', async () => { + const mockLogsResource = { create: vi.fn(), list: vi.fn() }; + const logger = new QuotientLogger(mockLogsResource); + const privateLogger = logger as any; + + privateLogger.init({ appName: 'test_app', environment: 'test_environment' }); + + await privateLogger.log({ + userQuery: 'test', + modelOutput: 'test', + documents: [{}], + }); + + expect(mockLogsResource.create).not.toHaveBeenCalled(); + expect(consoleErrorSpy).toHaveBeenCalledWith( + expect.stringContaining("Missing required 'pageContent' property") + ); }); - - it('should log error and return null if you attempt to log before initializing', async () => { - const mockLogsResource = { create: vi.fn(), list: vi.fn() }; - const logger = new QuotientLogger(mockLogsResource); - const privateLogger = logger as any; - const result = await privateLogger.log({ message: 'test' }); - expect(result).toBeNull(); - expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('Logger is not configured')); - expect(mockLogsResource.create).not.toHaveBeenCalled(); + + it('should log error and return null when pageContent is not a string', async () => { + const mockLogsResource = { create: vi.fn(), list: vi.fn() }; + const logger = new QuotientLogger(mockLogsResource); + const privateLogger = logger as any; + + privateLogger.init({ appName: 'test_app', environment: 'test_environment' }); + + await privateLogger.log({ + userQuery: 'test', + modelOutput: 'test', + documents: [{ pageContent: 123 }], + }); + + expect(mockLogsResource.create).not.toHaveBeenCalled(); + expect(consoleErrorSpy).toHaveBeenCalledWith( + expect.stringContaining("The 'pageContent' property must be a string") + ); }); - it('should log a message if initialized', async () => { - const mockLogsResource = { create: vi.fn(), list: vi.fn() }; - const logger = new QuotientLogger(mockLogsResource); - const privateLogger = logger as any; - privateLogger.init({ appName: 'test_app', environment: 'test_environment', tags: { test: 'test' }, sampleRate: 1.0 }); - await privateLogger.log({ message: 'test' }); - expect(mockLogsResource.create).toHaveBeenCalledWith({ - appName: 'test_app', - environment: 'test_environment', - tags: { test: 'test' }, - message: 'test', - hallucinationDetection: false, - hallucinationDetectionSampleRate: 0, - inconsistencyDetection: false - }); + it('should log error and return null when metadata is not an object', async () => { + const mockLogsResource = { create: vi.fn(), list: vi.fn() }; + const logger = new QuotientLogger(mockLogsResource); + const privateLogger = logger as any; + + privateLogger.init({ appName: 'test_app', environment: 'test_environment' }); + + await privateLogger.log({ + userQuery: 'test', + modelOutput: 'test', + documents: [{ pageContent: 'test', metadata: 'not-object' }], + }); + + expect(mockLogsResource.create).not.toHaveBeenCalled(); + expect(consoleErrorSpy).toHaveBeenCalledWith( + expect.stringContaining("The 'metadata' property must be an object") + ); }); - - it('should not log a message if shouldSample returns false', async () => { - const mockLogsResource = { create: vi.fn(), list: vi.fn() }; - const logger = new QuotientLogger(mockLogsResource); - const privateLogger = logger as any; - // Mock shouldSample to always return false - vi.spyOn(privateLogger, 'shouldSample').mockReturnValue(false); - - privateLogger.init({ appName: 'test_app', environment: 'test_environment', tags: { test: 'test' }, sampleRate: 0.5 }); - await privateLogger.log({ message: 'test' }); - expect(mockLogsResource.create).not.toHaveBeenCalled(); + + it('should accept null metadata', async () => { + const mockLogsResource = { create: vi.fn(), list: vi.fn() }; + const logger = new QuotientLogger(mockLogsResource); + const privateLogger = logger as any; + + privateLogger.init({ appName: 'test_app', environment: 'test_environment' }); + + await privateLogger.log({ + userQuery: 'test', + modelOutput: 'test', + documents: [{ pageContent: 'test', metadata: null }], + }); + + expect(mockLogsResource.create).toHaveBeenCalled(); }); - it('should verify shouldSample behavior based on Math.random', () => { - const mockLogsResource = { create: vi.fn(), list: vi.fn() }; - const logger = new QuotientLogger(mockLogsResource); - const privateLogger = logger as any; - privateLogger.init({ sampleRate: 0.5 }); + it('should validate documents as part of the log method', async () => { + const mockLogsResource = { create: vi.fn(), list: vi.fn() }; + const logger = new QuotientLogger(mockLogsResource); + const privateLogger = logger as any; - // Test when random is less than sample rate - vi.spyOn(Math, 'random').mockReturnValue(0.4); - expect(privateLogger.shouldSample()).toBe(true); + privateLogger.init({ appName: 'test_app', environment: 'test_environment' }); - // Test when random is greater than sample rate - vi.spyOn(Math, 'random').mockReturnValue(0.6); - expect(privateLogger.shouldSample()).toBe(false); + const validateSpy = vi.spyOn(privateLogger, 'validateDocuments'); + validateSpy.mockReturnValue(false); - vi.spyOn(Math, 'random').mockRestore(); + await privateLogger.log({ + userQuery: 'test', + modelOutput: 'test', + documents: ['test'], + }); + + expect(validateSpy).toHaveBeenCalled(); + expect(mockLogsResource.create).not.toHaveBeenCalled(); }); - it('should log error and return null if required appName or environment is missing after initialization', async () => { - const mockLogsResource = { create: vi.fn(), list: vi.fn() }; - const logger = new QuotientLogger(mockLogsResource); - const privateLogger = logger as any; - - // Initialize but with null values - privateLogger.init({}); - privateLogger.configured = true; - privateLogger.appName = null; - privateLogger.environment = 'test'; - - const result1 = await privateLogger.log({ message: 'test' }); - expect(result1).toBeNull(); - expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('appName and environment must be set')); - expect(mockLogsResource.create).not.toHaveBeenCalled(); - - privateLogger.appName = 'test'; - privateLogger.environment = null; - - const result2 = await privateLogger.log({ message: 'test' }); - expect(result2).toBeNull(); - expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('appName and environment must be set')); - expect(mockLogsResource.create).not.toHaveBeenCalled(); + it('should skip validation if no documents are provided', async () => { + const mockLogsResource = { create: vi.fn(), list: vi.fn() }; + const logger = new QuotientLogger(mockLogsResource); + const privateLogger = logger as any; + + privateLogger.init({ appName: 'test_app', environment: 'test_environment' }); + + const validateSpy = vi.spyOn(privateLogger, 'validateDocuments'); + + await privateLogger.log({ + userQuery: 'test', + modelOutput: 'test', + }); + + expect(validateSpy).not.toHaveBeenCalled(); + expect(mockLogsResource.create).toHaveBeenCalled(); }); - describe('Document Validation', () => { - it('should accept string documents', async () => { - const mockLogsResource = { create: vi.fn(), list: vi.fn() }; - const logger = new QuotientLogger(mockLogsResource); - const privateLogger = logger as any; - - privateLogger.init({ appName: 'test_app', environment: 'test_environment' }); - - await privateLogger.log({ - userQuery: 'test', - modelOutput: 'test', - documents: ['This is a string document'] - }); - - expect(mockLogsResource.create).toHaveBeenCalled(); - }); - - it('should accept valid LogDocument objects', async () => { - const mockLogsResource = { create: vi.fn(), list: vi.fn() }; - const logger = new QuotientLogger(mockLogsResource); - const privateLogger = logger as any; - - privateLogger.init({ appName: 'test_app', environment: 'test_environment' }); - - await privateLogger.log({ - userQuery: 'test', - modelOutput: 'test', - documents: [ - { pageContent: 'This is valid content' }, - { pageContent: 'Also valid', metadata: { source: 'test' } } - ] - }); - - expect(mockLogsResource.create).toHaveBeenCalled(); - }); - - it('should log error and return null when document is missing pageContent', async () => { - const mockLogsResource = { create: vi.fn(), list: vi.fn() }; - const logger = new QuotientLogger(mockLogsResource); - const privateLogger = logger as any; - - privateLogger.init({ appName: 'test_app', environment: 'test_environment' }); - - await privateLogger.log({ - userQuery: 'test', - modelOutput: 'test', - documents: [{}] - }); - - expect(mockLogsResource.create).not.toHaveBeenCalled(); - expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining("Missing required 'pageContent' property")); - }); - - it('should log error and return null when pageContent is not a string', async () => { - const mockLogsResource = { create: vi.fn(), list: vi.fn() }; - const logger = new QuotientLogger(mockLogsResource); - const privateLogger = logger as any; - - privateLogger.init({ appName: 'test_app', environment: 'test_environment' }); - - await privateLogger.log({ - userQuery: 'test', - modelOutput: 'test', - documents: [{ pageContent: 123 }] - }); - - expect(mockLogsResource.create).not.toHaveBeenCalled(); - expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining("The 'pageContent' property must be a string")); - }); - - it('should log error and return null when metadata is not an object', async () => { - const mockLogsResource = { create: vi.fn(), list: vi.fn() }; - const logger = new QuotientLogger(mockLogsResource); - const privateLogger = logger as any; - - privateLogger.init({ appName: 'test_app', environment: 'test_environment' }); - - await privateLogger.log({ - userQuery: 'test', - modelOutput: 'test', - documents: [{ pageContent: 'test', metadata: 'not-object' }] - }); - - expect(mockLogsResource.create).not.toHaveBeenCalled(); - expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining("The 'metadata' property must be an object")); - }); - - it('should accept null metadata', async () => { - const mockLogsResource = { create: vi.fn(), list: vi.fn() }; - const logger = new QuotientLogger(mockLogsResource); - const privateLogger = logger as any; - - privateLogger.init({ appName: 'test_app', environment: 'test_environment' }); - - await privateLogger.log({ - userQuery: 'test', - modelOutput: 'test', - documents: [{ pageContent: 'test', metadata: null }] - }); - - expect(mockLogsResource.create).toHaveBeenCalled(); - }); - - it('should validate documents as part of the log method', async () => { - const mockLogsResource = { create: vi.fn(), list: vi.fn() }; - const logger = new QuotientLogger(mockLogsResource); - const privateLogger = logger as any; - - privateLogger.init({ appName: 'test_app', environment: 'test_environment' }); - - const validateSpy = vi.spyOn(privateLogger, 'validateDocuments'); - validateSpy.mockReturnValue(false); - - await privateLogger.log({ - userQuery: 'test', - modelOutput: 'test', - documents: ['test'] - }); - - expect(validateSpy).toHaveBeenCalled(); - expect(mockLogsResource.create).not.toHaveBeenCalled(); - }); - - it('should skip validation if no documents are provided', async () => { - const mockLogsResource = { create: vi.fn(), list: vi.fn() }; - const logger = new QuotientLogger(mockLogsResource); - const privateLogger = logger as any; - - privateLogger.init({ appName: 'test_app', environment: 'test_environment' }); - - const validateSpy = vi.spyOn(privateLogger, 'validateDocuments'); - - await privateLogger.log({ - userQuery: 'test', - modelOutput: 'test' - }); - - expect(validateSpy).not.toHaveBeenCalled(); - expect(mockLogsResource.create).toHaveBeenCalled(); - }); - - it('should directly test isValidLogDocument with various inputs', () => { - const mockLogsResource = { create: vi.fn(), list: vi.fn() }; - const logger = new QuotientLogger(mockLogsResource); - const privateLogger = logger as any; - - // Valid document - expect(privateLogger.isValidLogDocument({ pageContent: 'test' })).toEqual({ valid: true }); - - // Missing pageContent - expect(privateLogger.isValidLogDocument({})).toEqual({ - valid: false, - error: "Missing required 'pageContent' property" - }); - - // pageContent not a string - expect(privateLogger.isValidLogDocument({ pageContent: 123 })).toEqual({ - valid: false, - error: "The 'pageContent' property must be a string, found number" - }); - - // metadata not an object - expect(privateLogger.isValidLogDocument({ pageContent: 'test', metadata: 'not-object' })).toEqual({ - valid: false, - error: "The 'metadata' property must be an object, found string" - }); - - // null metadata is valid - expect(privateLogger.isValidLogDocument({ pageContent: 'test', metadata: null })).toEqual({ valid: true }); - }); + it('should directly test isValidLogDocument with various inputs', () => { + const mockLogsResource = { create: vi.fn(), list: vi.fn() }; + const logger = new QuotientLogger(mockLogsResource); + const privateLogger = logger as any; + + // Valid document + expect(privateLogger.isValidLogDocument({ pageContent: 'test' })).toEqual({ valid: true }); + + // Missing pageContent + expect(privateLogger.isValidLogDocument({})).toEqual({ + valid: false, + error: "Missing required 'pageContent' property", + }); + + // pageContent not a string + expect(privateLogger.isValidLogDocument({ pageContent: 123 })).toEqual({ + valid: false, + error: "The 'pageContent' property must be a string, found number", + }); + + // metadata not an object + expect( + privateLogger.isValidLogDocument({ pageContent: 'test', metadata: 'not-object' }) + ).toEqual({ + valid: false, + error: "The 'metadata' property must be an object, found string", + }); + + // null metadata is valid + expect(privateLogger.isValidLogDocument({ pageContent: 'test', metadata: null })).toEqual({ + valid: true, + }); }); + }); }); - diff --git a/tests/resources/auth.test.ts b/tests/resources/auth.test.ts index dc0eb3c..61dedc3 100644 --- a/tests/resources/auth.test.ts +++ b/tests/resources/auth.test.ts @@ -4,16 +4,16 @@ import { BaseQuotientClient } from '../../quotientai/client'; import { vi } from 'vitest'; describe('QuotientAuth', () => { - it('should be defined', () => { - expect(AuthResource).toBeDefined(); - }); + it('should be defined', () => { + expect(AuthResource).toBeDefined(); + }); - it('should authenticate', async () => { - const client = new BaseQuotientClient('test'); - vi.spyOn(client, 'get').mockResolvedValue({ id: 'test_id', email: 'test@test.com' }); - const authResource = new AuthResource(client); - - const response = await authResource.authenticate(); - expect(response).toEqual({ id: 'test_id', email: 'test@test.com' }); - }); -}); \ No newline at end of file + it('should authenticate', async () => { + const client = new BaseQuotientClient('test'); + vi.spyOn(client, 'get').mockResolvedValue({ id: 'test_id', email: 'test@test.com' }); + const authResource = new AuthResource(client); + + const response = await authResource.authenticate(); + expect(response).toEqual({ id: 'test_id', email: 'test@test.com' }); + }); +}); diff --git a/tests/resources/exceptions.test.ts b/tests/resources/exceptions.test.ts index d798991..f124a35 100644 --- a/tests/resources/exceptions.test.ts +++ b/tests/resources/exceptions.test.ts @@ -1,621 +1,666 @@ import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; import { - QuotientAIError, - APIError, - APIResponseValidationError, - APIStatusError, - APIConnectionError, - APITimeoutError, - BadRequestError, - AuthenticationError, - PermissionDeniedError, - NotFoundError, - ConflictError, - UnprocessableEntityError, - RateLimitError, - InternalServerError, - parseUnprocessableEntityError, - parseBadRequestError, - handleErrors, - logError } from '../../quotientai/exceptions'; + QuotientAIError, + APIError, + APIResponseValidationError, + APIStatusError, + APIConnectionError, + APITimeoutError, + BadRequestError, + AuthenticationError, + PermissionDeniedError, + NotFoundError, + ConflictError, + UnprocessableEntityError, + RateLimitError, + InternalServerError, + parseUnprocessableEntityError, + parseBadRequestError, + handleErrors, + logError, +} from '../../quotientai/exceptions'; import axios from 'axios'; describe('QuotientAIError', () => { - it('should create an error with the correct message', () => { - const error = new QuotientAIError('Test error'); - expect(error.message).toBe('Test error'); - expect(error.name).toBe('QuotientAIError'); - }); + it('should create an error with the correct message', () => { + const error = new QuotientAIError('Test error'); + expect(error.message).toBe('Test error'); + expect(error.name).toBe('QuotientAIError'); + }); }); describe('APIError', () => { - it('should create an API error with request and body', () => { - const request = { url: 'test', headers: {} } as any; - const body = { code: 'test_code', param: 'test_param', type: 'test_type' }; - const error = new APIError('Test error', request, body); - expect(error.message).toBe('Test error'); - expect(error.name).toBe('APIError'); - expect(error.request).toBe(request); - expect(error.body).toBe(body); - expect(error.code).toBe('test_code'); - expect(error.param).toBe('test_param'); - expect(error.type).toBe('test_type'); - }); + it('should create an API error with request and body', () => { + const request = { url: 'test', headers: {} } as any; + const body = { code: 'test_code', param: 'test_param', type: 'test_type' }; + const error = new APIError('Test error', request, body); + expect(error.message).toBe('Test error'); + expect(error.name).toBe('APIError'); + expect(error.request).toBe(request); + expect(error.body).toBe(body); + expect(error.code).toBe('test_code'); + expect(error.param).toBe('test_param'); + expect(error.type).toBe('test_type'); + }); }); describe('APIResponseValidationError', () => { - it('should create a validation error with response and status', () => { - const response = { status: 400, config: { url: 'test', headers: {} } } as any; - const error = new APIResponseValidationError(response, {}); - expect(error.message).toBe('Data returned by API invalid for expected schema.'); - expect(error.name).toBe('APIResponseValidationError'); - expect(error.response).toBe(response); - expect(error.status).toBe(400); - }); + it('should create a validation error with response and status', () => { + const response = { status: 400, config: { url: 'test', headers: {} } } as any; + const error = new APIResponseValidationError(response, {}); + expect(error.message).toBe('Data returned by API invalid for expected schema.'); + expect(error.name).toBe('APIResponseValidationError'); + expect(error.response).toBe(response); + expect(error.status).toBe(400); + }); }); describe('APIStatusError', () => { - it('should create a status error with response and status', () => { - const response = { status: 500, config: { url: 'test', headers: {} } } as any; - const error = new APIStatusError('Test error', response); - expect(error.message).toBe('Test error'); - expect(error.name).toBe('APIStatusError'); - expect(error.response).toBe(response); - expect(error.status).toBe(500); - }); + it('should create a status error with response and status', () => { + const response = { status: 500, config: { url: 'test', headers: {} } } as any; + const error = new APIStatusError('Test error', response); + expect(error.message).toBe('Test error'); + expect(error.name).toBe('APIStatusError'); + expect(error.response).toBe(response); + expect(error.status).toBe(500); + }); }); describe('APIConnectionError', () => { - it('should create a connection error', () => { - const request = { url: 'test', headers: {} } as any; - const error = new APIConnectionError('Test error', request); - expect(error.message).toBe('Test error'); - expect(error.name).toBe('APIConnectionError'); - expect(error.request).toBe(request); - }); + it('should create a connection error', () => { + const request = { url: 'test', headers: {} } as any; + const error = new APIConnectionError('Test error', request); + expect(error.message).toBe('Test error'); + expect(error.name).toBe('APIConnectionError'); + expect(error.request).toBe(request); + }); }); describe('APITimeoutError', () => { - it('should create a timeout error', () => { - const request = { url: 'test', headers: {} } as any; - const error = new APITimeoutError(request); - expect(error.message).toBe('Request timed out.'); - expect(error.name).toBe('APITimeoutError'); - expect(error.request).toBe(request); - }); - - it('should create a timeout error with unknown url when request is undefined', () => { - const error = new APITimeoutError(undefined); - expect(error.message).toBe('Request timed out.'); - expect(error.name).toBe('APITimeoutError'); - expect(error.request).toEqual({ url: 'unknown' }); - }); + it('should create a timeout error', () => { + const request = { url: 'test', headers: {} } as any; + const error = new APITimeoutError(request); + expect(error.message).toBe('Request timed out.'); + expect(error.name).toBe('APITimeoutError'); + expect(error.request).toBe(request); + }); + + it('should create a timeout error with unknown url when request is undefined', () => { + const error = new APITimeoutError(undefined); + expect(error.message).toBe('Request timed out.'); + expect(error.name).toBe('APITimeoutError'); + expect(error.request).toEqual({ url: 'unknown' }); + }); }); describe('HTTP Status Errors', () => { - it('should create BadRequestError', () => { - const response = { status: 400, config: { url: 'test', headers: {} } } as any; - const error = new BadRequestError('Test error', response); - expect(error.status).toBe(400); - }); - - it('should create AuthenticationError', () => { - const response = { status: 401, config: { url: 'test', headers: {} } } as any; - const error = new AuthenticationError('Test error', response); - expect(error.status).toBe(401); - }); - - it('should create PermissionDeniedError', () => { - const response = { status: 403, config: { url: 'test', headers: {} } } as any; - const error = new PermissionDeniedError('Test error', response); - expect(error.status).toBe(403); - }); - - it('should create NotFoundError', () => { - const response = { status: 404, config: { url: 'test', headers: {} } } as any; - const error = new NotFoundError('Test error', response); - expect(error.status).toBe(404); - }); - - it('should create ConflictError', () => { - const response = { status: 409, config: { url: 'test', headers: {} } } as any; - const error = new ConflictError('Test error', response); - expect(error.status).toBe(409); - }); - - it('should create UnprocessableEntityError', () => { - const response = { status: 422, config: { url: 'test', headers: {} } } as any; - const error = new UnprocessableEntityError('Test error', response); - expect(error.status).toBe(422); - }); - - it('should create RateLimitError', () => { - const response = { status: 429, config: { url: 'test', headers: {} } } as any; - const error = new RateLimitError('Test error', response); - expect(error.status).toBe(429); - }); - - it('should create InternalServerError', () => { - const response = { status: 500, config: { url: 'test', headers: {} } } as any; - const error = new InternalServerError('Test error', response); - expect(error.status).toBe(500); - }); + it('should create BadRequestError', () => { + const response = { status: 400, config: { url: 'test', headers: {} } } as any; + const error = new BadRequestError('Test error', response); + expect(error.status).toBe(400); + }); + + it('should create AuthenticationError', () => { + const response = { status: 401, config: { url: 'test', headers: {} } } as any; + const error = new AuthenticationError('Test error', response); + expect(error.status).toBe(401); + }); + + it('should create PermissionDeniedError', () => { + const response = { status: 403, config: { url: 'test', headers: {} } } as any; + const error = new PermissionDeniedError('Test error', response); + expect(error.status).toBe(403); + }); + + it('should create NotFoundError', () => { + const response = { status: 404, config: { url: 'test', headers: {} } } as any; + const error = new NotFoundError('Test error', response); + expect(error.status).toBe(404); + }); + + it('should create ConflictError', () => { + const response = { status: 409, config: { url: 'test', headers: {} } } as any; + const error = new ConflictError('Test error', response); + expect(error.status).toBe(409); + }); + + it('should create UnprocessableEntityError', () => { + const response = { status: 422, config: { url: 'test', headers: {} } } as any; + const error = new UnprocessableEntityError('Test error', response); + expect(error.status).toBe(422); + }); + + it('should create RateLimitError', () => { + const response = { status: 429, config: { url: 'test', headers: {} } } as any; + const error = new RateLimitError('Test error', response); + expect(error.status).toBe(429); + }); + + it('should create InternalServerError', () => { + const response = { status: 500, config: { url: 'test', headers: {} } } as any; + const error = new InternalServerError('Test error', response); + expect(error.status).toBe(500); + }); }); describe('Error Parsing Functions', () => { - describe('parseUnprocessableEntityError', () => { - it('should parse missing fields error', () => { - const response = { - data: { - detail: [ - { type: 'missing', loc: ['body', 'name'] }, - { type: 'missing', loc: ['body', 'email'] } - ] - } - } as any; - const message = parseUnprocessableEntityError(response); - expect(message).toBe('missing required fields: name, email'); - }); - - it('should handle invalid detail format', () => { - const consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); - const response = { - data: { - detail: 'Invalid format' - } - } as any; - const message = parseUnprocessableEntityError(response); - expect(message).toBe('Invalid response format'); - expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('APIResponseValidationError')); - }); - - it('should handle invalid body', () => { - const consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); - const response = { - data: null - } as any; - const message = parseUnprocessableEntityError(response); - expect(message).toBe('Invalid response format'); - expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('APIResponseValidationError')); - }); - }); - - describe('parseBadRequestError', () => { - it('should parse detail message from response', () => { - const response = { - data: { - detail: 'Invalid input data' - } - } as any; - const message = parseBadRequestError(response); - expect(message).toBe('Invalid input data'); - }); - - it('should handle missing detail', () => { - const consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); - const response = { - data: { - message: 'Invalid format' - } - } as any; - const message = parseBadRequestError(response); - expect(message).toBe('Invalid request format'); - expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('APIResponseValidationError')); - }); - - it('should handle invalid body', () => { - const consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); - const response = { - data: null - } as any; - const message = parseBadRequestError(response); - expect(message).toBe('Invalid request format'); - expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('APIResponseValidationError')); - }); - }); + describe('parseUnprocessableEntityError', () => { + it('should parse missing fields error', () => { + const response = { + data: { + detail: [ + { type: 'missing', loc: ['body', 'name'] }, + { type: 'missing', loc: ['body', 'email'] }, + ], + }, + } as any; + const message = parseUnprocessableEntityError(response); + expect(message).toBe('missing required fields: name, email'); + }); + + it('should handle invalid detail format', () => { + const consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + const response = { + data: { + detail: 'Invalid format', + }, + } as any; + const message = parseUnprocessableEntityError(response); + expect(message).toBe('Invalid response format'); + expect(consoleErrorSpy).toHaveBeenCalledWith( + expect.stringContaining('APIResponseValidationError') + ); + }); + + it('should handle invalid body', () => { + const consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + const response = { + data: null, + } as any; + const message = parseUnprocessableEntityError(response); + expect(message).toBe('Invalid response format'); + expect(consoleErrorSpy).toHaveBeenCalledWith( + expect.stringContaining('APIResponseValidationError') + ); + }); + }); + + describe('parseBadRequestError', () => { + it('should parse detail message from response', () => { + const response = { + data: { + detail: 'Invalid input data', + }, + } as any; + const message = parseBadRequestError(response); + expect(message).toBe('Invalid input data'); + }); + + it('should handle missing detail', () => { + const consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + const response = { + data: { + message: 'Invalid format', + }, + } as any; + const message = parseBadRequestError(response); + expect(message).toBe('Invalid request format'); + expect(consoleErrorSpy).toHaveBeenCalledWith( + expect.stringContaining('APIResponseValidationError') + ); + }); + + it('should handle invalid body', () => { + const consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + const response = { + data: null, + } as any; + const message = parseBadRequestError(response); + expect(message).toBe('Invalid request format'); + expect(consoleErrorSpy).toHaveBeenCalledWith( + expect.stringContaining('APIResponseValidationError') + ); + }); + }); }); describe('Error Handling', () => { - it('should handle successful responses', async () => { - const response = { data: { success: true } }; - const handler = handleErrors(); - const descriptor = handler({}, 'testMethod', { value: async () => response }); - const result = await descriptor.value(); - expect(result).toEqual({ success: true }); - }); - - it('should handle BadRequestError (400)', async () => { - const consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); - const axiosError = { - isAxiosError: true, - response: { - status: 400, - data: { detail: 'Invalid request' }, - config: { url: 'test', headers: {} } - } - } as any; - vi.spyOn(axios, 'isAxiosError').mockReturnValue(true); - - const handler = handleErrors(); - const descriptor = handler({}, 'testMethod', { - value: async () => { throw axiosError; } - }); - - const result = await descriptor.value(); - expect(result).toBeNull(); - expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('BadRequestError')); - expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('Invalid request')); - }); - - it('should handle AuthenticationError (401)', async () => { - const consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); - const axiosError = { - isAxiosError: true, - response: { - status: 401, - data: { detail: 'Unauthorized' }, - config: { url: 'test', headers: {} } - } - } as any; - vi.spyOn(axios, 'isAxiosError').mockReturnValue(true); - - const handler = handleErrors(); - const descriptor = handler({}, 'testMethod', { - value: async () => { throw axiosError; } - }); - - const result = await descriptor.value(); - expect(result).toBeNull(); - expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('AuthenticationError')); - expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('unauthorized')); - }); - - it('should handle PermissionDeniedError (403)', async () => { - const consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); - const axiosError = { - isAxiosError: true, - response: { - status: 403, - data: { detail: 'Forbidden' }, - config: { url: 'test', headers: {} } - } - } as any; - vi.spyOn(axios, 'isAxiosError').mockReturnValue(true); - - const handler = handleErrors(); - const descriptor = handler({}, 'testMethod', { - value: async () => { throw axiosError; } - }); - - const result = await descriptor.value(); - expect(result).toBeNull(); - expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('PermissionDeniedError')); - expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('forbidden')); - }); - - it('should handle NotFoundError (404)', async () => { - const consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); - const axiosError = { - isAxiosError: true, - response: { - status: 404, - data: { detail: 'Not found' }, - config: { url: 'test', headers: {} } - } - } as any; - vi.spyOn(axios, 'isAxiosError').mockReturnValue(true); - - const handler = handleErrors(); - const descriptor = handler({}, 'testMethod', { - value: async () => { throw axiosError; } - }); - - const result = await descriptor.value(); - expect(result).toBeNull(); - expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('NotFoundError')); - expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('not found')); - }); - - it('should handle UnprocessableEntityError (422)', async () => { - const consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); - const axiosError = { - isAxiosError: true, - response: { - status: 422, - data: { - detail: [ - { type: 'missing', loc: ['body', 'name'] } - ] - }, - config: { url: 'test', headers: {} } - } - } as any; - vi.spyOn(axios, 'isAxiosError').mockReturnValue(true); - - const handler = handleErrors(); - const descriptor = handler({}, 'testMethod', { - value: async () => { throw axiosError; } - }); - - const result = await descriptor.value(); - expect(result).toBeNull(); - expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('UnprocessableEntityError')); - expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('missing required fields')); - }); - - it('should handle APITimeoutError with retries', async () => { - const consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); - const axiosError = { - isAxiosError: true, - code: 'ECONNABORTED', - config: { url: 'test', headers: {} } - } as any; - vi.spyOn(axios, 'isAxiosError').mockReturnValue(true); - - let attempts = 0; - const handler = handleErrors(); - const descriptor = handler({}, 'testMethod', { - value: async () => { - attempts++; - throw axiosError; - } - }); - - const result = await descriptor.value(); - expect(result).toBeNull(); - expect(attempts).toBe(3); // Should have tried 3 times - expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('APITimeoutError')); - expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('Request timed out')); - }); - - it('should handle APIConnectionError', async () => { - const consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); - const axiosError = { - isAxiosError: true, - response: null, - code: 'NETWORK_ERROR', - config: { url: 'test', headers: {} } - } as any; - vi.spyOn(axios, 'isAxiosError').mockReturnValue(true); - - const handler = handleErrors(); - const descriptor = handler({}, 'testMethod', { - value: async () => { throw axiosError; } - }); - - const result = await descriptor.value(); - expect(result).toBeNull(); - expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('APIConnectionError')); - expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('connection error')); - }); - - it('should handle Axios error with response but no status code', async () => { - const consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); - const axiosError = { - isAxiosError: true, - response: { - data: { detail: 'Invalid response' }, - config: { url: 'test', headers: {} } - } - } as any; - vi.spyOn(axios, 'isAxiosError').mockReturnValue(true); - - const handler = handleErrors(); - const descriptor = handler({}, 'testMethod', { - value: async () => { throw axiosError; } - }); - - const result = await descriptor.value(); - expect(result).toBeNull(); - expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('APIStatusError')); - expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('unexpected status code: undefined')); - }); - - it('should handle Axios error with undefined response', async () => { - const consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); - const axiosError = { - isAxiosError: true, - response: undefined, - config: { url: 'test', headers: {} } - } as any; - vi.spyOn(axios, 'isAxiosError').mockReturnValue(true); - - const handler = handleErrors(); - const descriptor = handler({}, 'testMethod', { - value: async () => { throw axiosError; } - }); - - const result = await descriptor.value(); - expect(result).toBeNull(); - expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('APIConnectionError')); - expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('connection error')); - }); - - it('should handle Axios error with undefined config', async () => { - const consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); - const axiosError = { - isAxiosError: true, - response: undefined, - config: undefined - } as any; - vi.spyOn(axios, 'isAxiosError').mockReturnValue(true); - - const handler = handleErrors(); - const descriptor = handler({}, 'testMethod', { - value: async () => { throw axiosError; } - }); - - const result = await descriptor.value(); - expect(result).toBeNull(); - expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('APIConnectionError')); - expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('connection error')); - }); - - it('should handle Axios error without response property', async () => { - const consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); - const axiosError = { - isAxiosError: true, - code: 'NETWORK_ERROR', - config: { url: 'test', headers: {} } - } as any; - vi.spyOn(axios, 'isAxiosError').mockReturnValue(true); - - const handler = handleErrors(); - const descriptor = handler({}, 'testMethod', { - value: async () => { throw axiosError; } - }); - - const result = await descriptor.value(); - expect(result).toBeNull(); - expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('APIConnectionError')); - expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('connection error')); - }); - - it('should handle Axios error with false response', async () => { - const consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); - const axiosError = { - isAxiosError: true, - response: false, - code: 'NETWORK_ERROR', - config: { url: 'test', headers: {} } - } as any; - vi.spyOn(axios, 'isAxiosError').mockReturnValue(true); - - const handler = handleErrors(); - const descriptor = handler({}, 'testMethod', { - value: async () => { throw axiosError; } - }); - - const result = await descriptor.value(); - expect(result).toBeNull(); - expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('APIConnectionError')); - expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('connection error')); - }); + it('should handle successful responses', async () => { + const response = { data: { success: true } }; + const handler = handleErrors(); + const descriptor = handler({}, 'testMethod', { value: async () => response }); + const result = await descriptor.value(); + expect(result).toEqual({ success: true }); + }); + + it('should handle BadRequestError (400)', async () => { + const consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + const axiosError = { + isAxiosError: true, + response: { + status: 400, + data: { detail: 'Invalid request' }, + config: { url: 'test', headers: {} }, + }, + } as any; + vi.spyOn(axios, 'isAxiosError').mockReturnValue(true); + + const handler = handleErrors(); + const descriptor = handler({}, 'testMethod', { + value: async () => { + throw axiosError; + }, + }); + + const result = await descriptor.value(); + expect(result).toBeNull(); + expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('BadRequestError')); + expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('Invalid request')); + }); + + it('should handle AuthenticationError (401)', async () => { + const consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + const axiosError = { + isAxiosError: true, + response: { + status: 401, + data: { detail: 'Unauthorized' }, + config: { url: 'test', headers: {} }, + }, + } as any; + vi.spyOn(axios, 'isAxiosError').mockReturnValue(true); + + const handler = handleErrors(); + const descriptor = handler({}, 'testMethod', { + value: async () => { + throw axiosError; + }, + }); + + const result = await descriptor.value(); + expect(result).toBeNull(); + expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('AuthenticationError')); + expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('unauthorized')); + }); + + it('should handle PermissionDeniedError (403)', async () => { + const consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + const axiosError = { + isAxiosError: true, + response: { + status: 403, + data: { detail: 'Forbidden' }, + config: { url: 'test', headers: {} }, + }, + } as any; + vi.spyOn(axios, 'isAxiosError').mockReturnValue(true); + + const handler = handleErrors(); + const descriptor = handler({}, 'testMethod', { + value: async () => { + throw axiosError; + }, + }); + + const result = await descriptor.value(); + expect(result).toBeNull(); + expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('PermissionDeniedError')); + expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('forbidden')); + }); + + it('should handle NotFoundError (404)', async () => { + const consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + const axiosError = { + isAxiosError: true, + response: { + status: 404, + data: { detail: 'Not found' }, + config: { url: 'test', headers: {} }, + }, + } as any; + vi.spyOn(axios, 'isAxiosError').mockReturnValue(true); + + const handler = handleErrors(); + const descriptor = handler({}, 'testMethod', { + value: async () => { + throw axiosError; + }, + }); + + const result = await descriptor.value(); + expect(result).toBeNull(); + expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('NotFoundError')); + expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('not found')); + }); + + it('should handle UnprocessableEntityError (422)', async () => { + const consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + const axiosError = { + isAxiosError: true, + response: { + status: 422, + data: { + detail: [{ type: 'missing', loc: ['body', 'name'] }], + }, + config: { url: 'test', headers: {} }, + }, + } as any; + vi.spyOn(axios, 'isAxiosError').mockReturnValue(true); + + const handler = handleErrors(); + const descriptor = handler({}, 'testMethod', { + value: async () => { + throw axiosError; + }, + }); + + const result = await descriptor.value(); + expect(result).toBeNull(); + expect(consoleErrorSpy).toHaveBeenCalledWith( + expect.stringContaining('UnprocessableEntityError') + ); + expect(consoleErrorSpy).toHaveBeenCalledWith( + expect.stringContaining('missing required fields') + ); + }); + + it('should handle APITimeoutError with retries', async () => { + const consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + const axiosError = { + isAxiosError: true, + code: 'ECONNABORTED', + config: { url: 'test', headers: {} }, + } as any; + vi.spyOn(axios, 'isAxiosError').mockReturnValue(true); + + let attempts = 0; + const handler = handleErrors(); + const descriptor = handler({}, 'testMethod', { + value: async () => { + attempts++; + throw axiosError; + }, + }); + + const result = await descriptor.value(); + expect(result).toBeNull(); + expect(attempts).toBe(3); // Should have tried 3 times + expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('APITimeoutError')); + expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('Request timed out')); + }); + + it('should handle APIConnectionError', async () => { + const consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + const axiosError = { + isAxiosError: true, + response: null, + code: 'NETWORK_ERROR', + config: { url: 'test', headers: {} }, + } as any; + vi.spyOn(axios, 'isAxiosError').mockReturnValue(true); + + const handler = handleErrors(); + const descriptor = handler({}, 'testMethod', { + value: async () => { + throw axiosError; + }, + }); + + const result = await descriptor.value(); + expect(result).toBeNull(); + expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('APIConnectionError')); + expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('connection error')); + }); + + it('should handle Axios error with response but no status code', async () => { + const consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + const axiosError = { + isAxiosError: true, + response: { + data: { detail: 'Invalid response' }, + config: { url: 'test', headers: {} }, + }, + } as any; + vi.spyOn(axios, 'isAxiosError').mockReturnValue(true); + + const handler = handleErrors(); + const descriptor = handler({}, 'testMethod', { + value: async () => { + throw axiosError; + }, + }); + + const result = await descriptor.value(); + expect(result).toBeNull(); + expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('APIStatusError')); + expect(consoleErrorSpy).toHaveBeenCalledWith( + expect.stringContaining('unexpected status code: undefined') + ); + }); + + it('should handle Axios error with undefined response', async () => { + const consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + const axiosError = { + isAxiosError: true, + response: undefined, + config: { url: 'test', headers: {} }, + } as any; + vi.spyOn(axios, 'isAxiosError').mockReturnValue(true); + + const handler = handleErrors(); + const descriptor = handler({}, 'testMethod', { + value: async () => { + throw axiosError; + }, + }); + + const result = await descriptor.value(); + expect(result).toBeNull(); + expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('APIConnectionError')); + expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('connection error')); + }); + + it('should handle Axios error with undefined config', async () => { + const consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + const axiosError = { + isAxiosError: true, + response: undefined, + config: undefined, + } as any; + vi.spyOn(axios, 'isAxiosError').mockReturnValue(true); + + const handler = handleErrors(); + const descriptor = handler({}, 'testMethod', { + value: async () => { + throw axiosError; + }, + }); + + const result = await descriptor.value(); + expect(result).toBeNull(); + expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('APIConnectionError')); + expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('connection error')); + }); + + it('should handle Axios error without response property', async () => { + const consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + const axiosError = { + isAxiosError: true, + code: 'NETWORK_ERROR', + config: { url: 'test', headers: {} }, + } as any; + vi.spyOn(axios, 'isAxiosError').mockReturnValue(true); + + const handler = handleErrors(); + const descriptor = handler({}, 'testMethod', { + value: async () => { + throw axiosError; + }, + }); + + const result = await descriptor.value(); + expect(result).toBeNull(); + expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('APIConnectionError')); + expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('connection error')); + }); + + it('should handle Axios error with false response', async () => { + const consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + const axiosError = { + isAxiosError: true, + response: false, + code: 'NETWORK_ERROR', + config: { url: 'test', headers: {} }, + } as any; + vi.spyOn(axios, 'isAxiosError').mockReturnValue(true); + + const handler = handleErrors(); + const descriptor = handler({}, 'testMethod', { + value: async () => { + throw axiosError; + }, + }); + + const result = await descriptor.value(); + expect(result).toBeNull(); + expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('APIConnectionError')); + expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('connection error')); + }); + + it('should handle non-axios errors', async () => { + const consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + const handler = handleErrors(); + const descriptor = handler({}, 'testMethod', { + value: async () => { + throw new Error('Test error'); + }, + }); + vi.spyOn(axios, 'isAxiosError').mockReturnValue(false); + + await descriptor.value(); + expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('Test error')); + }); + + it('should handle unexpected status codes', async () => { + // spy on console.error + const consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + const axiosError = { + isAxiosError: true, + response: { + status: 418, + data: { detail: 'I am a teapot' }, + config: { url: 'test', headers: {} }, + }, + } as any; + vi.spyOn(axios, 'isAxiosError').mockReturnValue(true); + + const handler = handleErrors(); + const descriptor = handler({}, 'testMethod', { + value: async () => { + throw axiosError; + }, + }); + + await descriptor.value(); + + // expect console.error to have been called + expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('APIStatusError')); + // expect console.error to have been called with the message + expect(consoleErrorSpy).toHaveBeenCalledWith( + expect.stringContaining('unexpected status code: 418') + ); + }); + + it('should return null when retries reach 0', async () => { + const consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + const axiosError = { + isAxiosError: true, + code: 'ECONNABORTED', + config: { url: 'test', headers: {} }, + } as any; + vi.spyOn(axios, 'isAxiosError').mockReturnValue(true); + + let attempts = 0; + const handler = handleErrors(); + const descriptor = handler({}, 'testMethod', { + value: async () => { + attempts++; + throw axiosError; + }, + }); + + // Mock setTimeout to immediately resolve + vi.spyOn(global, 'setTimeout').mockImplementation((fn) => { + fn(); + return {} as unknown as NodeJS.Timeout; + }); + + const result = await descriptor.value(); + expect(result).toBeNull(); + expect(attempts).toBe(3); // Should have tried 3 times + expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('APITimeoutError')); + expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('Request timed out')); + }); +}); - it('should handle non-axios errors', async () => { - const consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); - const handler = handleErrors(); - const descriptor = handler({}, 'testMethod', { - value: async () => { throw new Error('Test error'); } - }); - vi.spyOn(axios, 'isAxiosError').mockReturnValue(false); +describe('logError', () => { + let consoleErrorSpy: any; + const mockDate = new Date('2024-01-01T12:00:00.000Z'); - await descriptor.value() - expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('Test error')); - }); + beforeEach(() => { + consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + vi.setSystemTime(mockDate); + }); - it('should handle unexpected status codes', async () => { - // spy on console.error - const consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); - const axiosError = { - isAxiosError: true, - response: { - status: 418, - data: { detail: 'I am a teapot' }, - config: { url: 'test', headers: {} } - } - } as any; - vi.spyOn(axios, 'isAxiosError').mockReturnValue(true); - - const handler = handleErrors(); - const descriptor = handler({}, 'testMethod', { - value: async () => { throw axiosError; } - }); - - await descriptor.value(); - - // expect console.error to have been called - expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('APIStatusError')); - // expect console.error to have been called with the message - expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('unexpected status code: 418')); - }); + afterEach(() => { + consoleErrorSpy.mockRestore(); + vi.useRealTimers(); + }); - it('should return null when retries reach 0', async () => { - const consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); - const axiosError = { - isAxiosError: true, - code: 'ECONNABORTED', - config: { url: 'test', headers: {} } - } as any; - vi.spyOn(axios, 'isAxiosError').mockReturnValue(true); - - let attempts = 0; - const handler = handleErrors(); - const descriptor = handler({}, 'testMethod', { - value: async () => { - attempts++; - throw axiosError; - } - }); - - // Mock setTimeout to immediately resolve - vi.spyOn(global, 'setTimeout').mockImplementation((fn) => { - fn(); - return {} as unknown as NodeJS.Timeout; - }); - - const result = await descriptor.value(); - expect(result).toBeNull(); - expect(attempts).toBe(3); // Should have tried 3 times - expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('APITimeoutError')); - expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('Request timed out')); - }); -}); + it('should log error with timestamp and stack trace when no context provided', () => { + const error = new Error('Test error'); + error.stack = 'Error: Test error\n at Test.stack'; -describe('logError', () => { - let consoleErrorSpy: any; - const mockDate = new Date('2024-01-01T12:00:00.000Z'); + logError(error); - beforeEach(() => { - consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); - vi.setSystemTime(mockDate); - }); + expect(consoleErrorSpy).toHaveBeenCalledWith('[2024-01-01T12:00:00.000Z] Error: Test error'); + expect(consoleErrorSpy).toHaveBeenCalledWith('Error: Test error\n at Test.stack'); + }); - afterEach(() => { - consoleErrorSpy.mockRestore(); - vi.useRealTimers(); - }); + it('should log error with timestamp, context, and stack trace when context provided', () => { + const error = new Error('Test error'); + error.stack = 'Error: Test error\n at Test.stack'; + const context = 'TestContext'; - it('should log error with timestamp and stack trace when no context provided', () => { - const error = new Error('Test error'); - error.stack = 'Error: Test error\n at Test.stack'; - - logError(error); + logError(error, context); - expect(consoleErrorSpy).toHaveBeenCalledWith('[2024-01-01T12:00:00.000Z] Error: Test error'); - expect(consoleErrorSpy).toHaveBeenCalledWith('Error: Test error\n at Test.stack'); - }); + expect(consoleErrorSpy).toHaveBeenCalledWith( + '[2024-01-01T12:00:00.000Z] [TestContext] Error: Test error' + ); + expect(consoleErrorSpy).toHaveBeenCalledWith('Error: Test error\n at Test.stack'); + }); - it('should log error with timestamp, context, and stack trace when context provided', () => { - const error = new Error('Test error'); - error.stack = 'Error: Test error\n at Test.stack'; - const context = 'TestContext'; - - logError(error, context); + it('should handle error without stack trace', () => { + const error = new Error('Test error'); + error.stack = ''; - expect(consoleErrorSpy).toHaveBeenCalledWith('[2024-01-01T12:00:00.000Z] [TestContext] Error: Test error'); - expect(consoleErrorSpy).toHaveBeenCalledWith('Error: Test error\n at Test.stack'); - }); + logError(error); - it('should handle error without stack trace', () => { - const error = new Error('Test error'); - error.stack = ''; - - logError(error); + expect(consoleErrorSpy).toHaveBeenCalledWith('[2024-01-01T12:00:00.000Z] Error: Test error'); + expect(consoleErrorSpy).toHaveBeenCalledWith(''); + }); - expect(consoleErrorSpy).toHaveBeenCalledWith('[2024-01-01T12:00:00.000Z] Error: Test error'); - expect(consoleErrorSpy).toHaveBeenCalledWith(''); - }); + it('should handle custom error types', () => { + const error = new QuotientAIError('Custom error'); + error.stack = 'QuotientAIError: Custom error\n at Test.stack'; - it('should handle custom error types', () => { - const error = new QuotientAIError('Custom error'); - error.stack = 'QuotientAIError: Custom error\n at Test.stack'; - - logError(error); + logError(error); - expect(consoleErrorSpy).toHaveBeenCalledWith('[2024-01-01T12:00:00.000Z] QuotientAIError: Custom error'); - expect(consoleErrorSpy).toHaveBeenCalledWith('QuotientAIError: Custom error\n at Test.stack'); - }); + expect(consoleErrorSpy).toHaveBeenCalledWith( + '[2024-01-01T12:00:00.000Z] QuotientAIError: Custom error' + ); + expect(consoleErrorSpy).toHaveBeenCalledWith( + 'QuotientAIError: Custom error\n at Test.stack' + ); + }); }); - - diff --git a/tests/resources/logs.test.ts b/tests/resources/logs.test.ts index 8cb93d6..a3e9018 100644 --- a/tests/resources/logs.test.ts +++ b/tests/resources/logs.test.ts @@ -1,240 +1,244 @@ -import {describe, it, expect, vi} from 'vitest'; +import { describe, it, expect, vi } from 'vitest'; import { BaseQuotientClient } from '../../quotientai/client'; import { LogsResource, Log } from '../../quotientai/resources/logs'; describe('LogsResource', () => { - const mockLogs = [ - { - id: 'log-1', - app_name: 'test-app', - environment: 'development', - hallucination_detection: true, - inconsistency_detection: false, - user_query: 'What is the capital of France?', - model_output: 'Paris is the capital of France.', - documents: ['doc1', 'doc2', { pageContent: 'doc3', metadata: { source: 'test' } }], - message_history: [ - { role: 'user', content: 'What is the capital of France?' }, - { role: 'assistant', content: 'Paris is the capital of France.' } - ], - instructions: ['Be concise', 'Be accurate'], - tags: { user_id: '123' }, - created_at: '2024-03-20T10:00:00Z' - }, - { - id: 'log-2', - app_name: 'test-app', - environment: 'production', - hallucination_detection: false, - inconsistency_detection: true, - user_query: 'What is the population of Tokyo?', - model_output: 'Tokyo has a population of approximately 37.4 million people.', - documents: ['doc3'], - message_history: null, - instructions: null, - tags: { user_id: '456' }, - created_at: '2024-03-20T11:00:00Z' - } - ]; - - describe('Log class', () => { - it('should format log as string', () => { - const log = new Log(mockLogs[0]); - const expectedString = `Log(id="log-1", appName="test-app", environment="development", createdAt="2024-03-20T10:00:00.000Z")`; - expect(log.toString()).toBe(expectedString); - }); - }); + const mockLogs = [ + { + id: 'log-1', + app_name: 'test-app', + environment: 'development', + hallucination_detection: true, + inconsistency_detection: false, + user_query: 'What is the capital of France?', + model_output: 'Paris is the capital of France.', + documents: ['doc1', 'doc2', { pageContent: 'doc3', metadata: { source: 'test' } }], + message_history: [ + { role: 'user', content: 'What is the capital of France?' }, + { role: 'assistant', content: 'Paris is the capital of France.' }, + ], + instructions: ['Be concise', 'Be accurate'], + tags: { user_id: '123' }, + created_at: '2024-03-20T10:00:00Z', + }, + { + id: 'log-2', + app_name: 'test-app', + environment: 'production', + hallucination_detection: false, + inconsistency_detection: true, + user_query: 'What is the population of Tokyo?', + model_output: 'Tokyo has a population of approximately 37.4 million people.', + documents: ['doc3'], + message_history: null, + instructions: null, + tags: { user_id: '456' }, + created_at: '2024-03-20T11:00:00Z', + }, + ]; - it('should list logs with default parameters', async () => { - const client = new BaseQuotientClient('test'); - vi.spyOn(client, 'get').mockResolvedValue({ - logs: mockLogs - }); - - const logsResource = new LogsResource(client); - const logs = await logsResource.list(); - - expect(logs).toHaveLength(2); - expect(logs[0]).toBeInstanceOf(Log); - expect(logs[0].id).toBe('log-1'); - expect(logs[0].appName).toBe('test-app'); - expect(logs[0].environment).toBe('development'); - expect(logs[0].createdAt).toBeInstanceOf(Date); - expect(client.get).toHaveBeenCalledWith('/logs', {}); + describe('Log class', () => { + it('should format log as string', () => { + const log = new Log(mockLogs[0]); + const expectedString = `Log(id="log-1", appName="test-app", environment="development", createdAt="2024-03-20T10:00:00.000Z")`; + expect(log.toString()).toBe(expectedString); }); + }); - it('should list logs with query parameters', async () => { - const client = new BaseQuotientClient('test'); - vi.spyOn(client, 'get').mockResolvedValue({ - logs: [mockLogs[0]] - }); - - const logsResource = new LogsResource(client); - const startDate = new Date('2024-03-20T00:00:00Z'); - const endDate = new Date('2024-03-20T23:59:59Z'); - - const logs = await logsResource.list({ - appName: 'test-app', - environment: 'development', - startDate: startDate, - endDate: endDate, - limit: 10, - offset: 0 - }); - - expect(logs).toHaveLength(1); - expect(logs[0].environment).toBe('development'); - expect(client.get).toHaveBeenCalledWith('/logs', { - app_name: 'test-app', - environment: 'development', - start_date: startDate.toISOString(), - end_date: endDate.toISOString(), - limit: 10, - offset: 0 - }); + it('should list logs with default parameters', async () => { + const client = new BaseQuotientClient('test'); + vi.spyOn(client, 'get').mockResolvedValue({ + logs: mockLogs, }); - it('should handle errors when listing logs', async () => { - const client = new BaseQuotientClient('test'); - const consoleErrorSpy = vi.spyOn(console, 'error'); - vi.spyOn(client, 'get').mockRejectedValue(new Error('Test error')); - - const logsResource = new LogsResource(client); - const result = await logsResource.list(); - - expect(result).toEqual([]); - expect(consoleErrorSpy).toHaveBeenCalledWith( - expect.stringContaining('[LogsResource.list] Error: Test error') - ); - expect(client.get).toHaveBeenCalledWith('/logs', {}); + const logsResource = new LogsResource(client); + const logs = await logsResource.list(); + + expect(logs).toHaveLength(2); + expect(logs[0]).toBeInstanceOf(Log); + expect(logs[0].id).toBe('log-1'); + expect(logs[0].appName).toBe('test-app'); + expect(logs[0].environment).toBe('development'); + expect(logs[0].createdAt).toBeInstanceOf(Date); + expect(client.get).toHaveBeenCalledWith('/logs', {}); + }); + + it('should list logs with query parameters', async () => { + const client = new BaseQuotientClient('test'); + vi.spyOn(client, 'get').mockResolvedValue({ + logs: [mockLogs[0]], }); - // Tests for the edge cases in list method (lines 117-119) - it('should handle null response from API', async () => { - const client = new BaseQuotientClient('test'); - vi.spyOn(client, 'get').mockResolvedValue(null); - const consoleSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}); - - const logsResource = new LogsResource(client); - const logs = await logsResource.list(); - - expect(logs).toHaveLength(0); - expect(consoleSpy).toHaveBeenCalledWith('No logs found. Please check your query parameters and try again.'); - expect(client.get).toHaveBeenCalledWith('/logs', {}); + const logsResource = new LogsResource(client); + const startDate = new Date('2024-03-20T00:00:00Z'); + const endDate = new Date('2024-03-20T23:59:59Z'); + + const logs = await logsResource.list({ + appName: 'test-app', + environment: 'development', + startDate: startDate, + endDate: endDate, + limit: 10, + offset: 0, }); - - it('should handle response with missing logs property', async () => { - const client = new BaseQuotientClient('test'); - vi.spyOn(client, 'get').mockResolvedValue({}); - const consoleSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}); - - const logsResource = new LogsResource(client); - const logs = await logsResource.list(); - - expect(logs).toHaveLength(0); - expect(consoleSpy).toHaveBeenCalledWith('No logs found. Please check your query parameters and try again.'); - expect(client.get).toHaveBeenCalledWith('/logs', {}); + + expect(logs).toHaveLength(1); + expect(logs[0].environment).toBe('development'); + expect(client.get).toHaveBeenCalledWith('/logs', { + app_name: 'test-app', + environment: 'development', + start_date: startDate.toISOString(), + end_date: endDate.toISOString(), + limit: 10, + offset: 0, }); - - it('should handle response with logs not being an array', async () => { - const client = new BaseQuotientClient('test'); - vi.spyOn(client, 'get').mockResolvedValue({ logs: 'not an array' }); - const consoleSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}); - - const logsResource = new LogsResource(client); - const logs = await logsResource.list(); - - expect(logs).toHaveLength(0); - expect(consoleSpy).toHaveBeenCalledWith('No logs found. Please check your query parameters and try again.'); - expect(client.get).toHaveBeenCalledWith('/logs', {}); + }); + + it('should handle errors when listing logs', async () => { + const client = new BaseQuotientClient('test'); + const consoleErrorSpy = vi.spyOn(console, 'error'); + vi.spyOn(client, 'get').mockRejectedValue(new Error('Test error')); + + const logsResource = new LogsResource(client); + const result = await logsResource.list(); + + expect(result).toEqual([]); + expect(consoleErrorSpy).toHaveBeenCalledWith( + expect.stringContaining('[LogsResource.list] Error: Test error') + ); + expect(client.get).toHaveBeenCalledWith('/logs', {}); + }); + + // Tests for the edge cases in list method (lines 117-119) + it('should handle null response from API', async () => { + const client = new BaseQuotientClient('test'); + vi.spyOn(client, 'get').mockResolvedValue(null); + const consoleSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}); + + const logsResource = new LogsResource(client); + const logs = await logsResource.list(); + + expect(logs).toHaveLength(0); + expect(consoleSpy).toHaveBeenCalledWith( + 'No logs found. Please check your query parameters and try again.' + ); + expect(client.get).toHaveBeenCalledWith('/logs', {}); + }); + + it('should handle response with missing logs property', async () => { + const client = new BaseQuotientClient('test'); + vi.spyOn(client, 'get').mockResolvedValue({}); + const consoleSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}); + + const logsResource = new LogsResource(client); + const logs = await logsResource.list(); + + expect(logs).toHaveLength(0); + expect(consoleSpy).toHaveBeenCalledWith( + 'No logs found. Please check your query parameters and try again.' + ); + expect(client.get).toHaveBeenCalledWith('/logs', {}); + }); + + it('should handle response with logs not being an array', async () => { + const client = new BaseQuotientClient('test'); + vi.spyOn(client, 'get').mockResolvedValue({ logs: 'not an array' }); + const consoleSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}); + + const logsResource = new LogsResource(client); + const logs = await logsResource.list(); + + expect(logs).toHaveLength(0); + expect(consoleSpy).toHaveBeenCalledWith( + 'No logs found. Please check your query parameters and try again.' + ); + expect(client.get).toHaveBeenCalledWith('/logs', {}); + }); + + it('should create a log', async () => { + const client = new BaseQuotientClient('test'); + vi.spyOn(client, 'post').mockResolvedValue({ + id: 'log-1', }); - it('should create a log', async () => { - const client = new BaseQuotientClient('test'); - vi.spyOn(client, 'post').mockResolvedValue({ - id: 'log-1' - }); - - const logsResource = new LogsResource(client); - await logsResource.create({ - appName: 'test-app', - environment: 'development', - hallucinationDetection: true, - inconsistencyDetection: false, - userQuery: 'What is the capital of France?', - modelOutput: 'Paris is the capital of France.', - documents: ['doc1', 'doc2'], - messageHistory: [ - { role: 'user', content: 'What is the capital of France?' }, - { role: 'assistant', content: 'Paris is the capital of France.' } - ], - instructions: ['Be concise', 'Be accurate'], - tags: { user_id: '123' }, - hallucinationDetectionSampleRate: 0.5 - }); - - expect(client.post).toHaveBeenCalledWith('/logs', { - app_name: 'test-app', - environment: 'development', - hallucination_detection: true, - inconsistency_detection: false, - user_query: 'What is the capital of France?', - model_output: 'Paris is the capital of France.', - documents: ['doc1', 'doc2'], - message_history: [ - { role: 'user', content: 'What is the capital of France?' }, - { role: 'assistant', content: 'Paris is the capital of France.' } - ], - instructions: ['Be concise', 'Be accurate'], - tags: { user_id: '123' }, - hallucination_detection_sample_rate: 0.5 - }); + const logsResource = new LogsResource(client); + await logsResource.create({ + appName: 'test-app', + environment: 'development', + hallucinationDetection: true, + inconsistencyDetection: false, + userQuery: 'What is the capital of France?', + modelOutput: 'Paris is the capital of France.', + documents: ['doc1', 'doc2'], + messageHistory: [ + { role: 'user', content: 'What is the capital of France?' }, + { role: 'assistant', content: 'Paris is the capital of France.' }, + ], + instructions: ['Be concise', 'Be accurate'], + tags: { user_id: '123' }, + hallucinationDetectionSampleRate: 0.5, }); - it('should handle errors when creating a log', async () => { - const client = new BaseQuotientClient('test'); - const consoleErrorSpy = vi.spyOn(console, 'error'); - vi.spyOn(client, 'post').mockRejectedValue(new Error('Test error')); - - const logsResource = new LogsResource(client); - const result = await logsResource.create({ - appName: 'test-app', - environment: 'development', - hallucinationDetection: true, - inconsistencyDetection: false, - userQuery: 'What is the capital of France?', - modelOutput: 'Paris is the capital of France.', - documents: ['doc1', 'doc2'], - messageHistory: [ - { role: 'user', content: 'What is the capital of France?' }, - { role: 'assistant', content: 'Paris is the capital of France.' } - ], - instructions: ['Be concise', 'Be accurate'], - tags: { user_id: '123' }, - hallucinationDetectionSampleRate: 0.5 - }); - - expect(result).toBeNull(); - expect(consoleErrorSpy).toHaveBeenCalledWith( - expect.stringContaining('[LogsResource.create] Error: Test error') - ); - expect(client.post).toHaveBeenCalledWith('/logs', { - app_name: 'test-app', - environment: 'development', - hallucination_detection: true, - inconsistency_detection: false, - user_query: 'What is the capital of France?', - model_output: 'Paris is the capital of France.', - documents: ['doc1', 'doc2'], - message_history: [ - { role: 'user', content: 'What is the capital of France?' }, - { role: 'assistant', content: 'Paris is the capital of France.' } - ], - instructions: ['Be concise', 'Be accurate'], - tags: { user_id: '123' }, - hallucination_detection_sample_rate: 0.5 - }); + expect(client.post).toHaveBeenCalledWith('/logs', { + app_name: 'test-app', + environment: 'development', + hallucination_detection: true, + inconsistency_detection: false, + user_query: 'What is the capital of France?', + model_output: 'Paris is the capital of France.', + documents: ['doc1', 'doc2'], + message_history: [ + { role: 'user', content: 'What is the capital of France?' }, + { role: 'assistant', content: 'Paris is the capital of France.' }, + ], + instructions: ['Be concise', 'Be accurate'], + tags: { user_id: '123' }, + hallucination_detection_sample_rate: 0.5, }); + }); + it('should handle errors when creating a log', async () => { + const client = new BaseQuotientClient('test'); + const consoleErrorSpy = vi.spyOn(console, 'error'); + vi.spyOn(client, 'post').mockRejectedValue(new Error('Test error')); -}); \ No newline at end of file + const logsResource = new LogsResource(client); + const result = await logsResource.create({ + appName: 'test-app', + environment: 'development', + hallucinationDetection: true, + inconsistencyDetection: false, + userQuery: 'What is the capital of France?', + modelOutput: 'Paris is the capital of France.', + documents: ['doc1', 'doc2'], + messageHistory: [ + { role: 'user', content: 'What is the capital of France?' }, + { role: 'assistant', content: 'Paris is the capital of France.' }, + ], + instructions: ['Be concise', 'Be accurate'], + tags: { user_id: '123' }, + hallucinationDetectionSampleRate: 0.5, + }); + + expect(result).toBeNull(); + expect(consoleErrorSpy).toHaveBeenCalledWith( + expect.stringContaining('[LogsResource.create] Error: Test error') + ); + expect(client.post).toHaveBeenCalledWith('/logs', { + app_name: 'test-app', + environment: 'development', + hallucination_detection: true, + inconsistency_detection: false, + user_query: 'What is the capital of France?', + model_output: 'Paris is the capital of France.', + documents: ['doc1', 'doc2'], + message_history: [ + { role: 'user', content: 'What is the capital of France?' }, + { role: 'assistant', content: 'Paris is the capital of France.' }, + ], + instructions: ['Be concise', 'Be accurate'], + tags: { user_id: '123' }, + hallucination_detection_sample_rate: 0.5, + }); + }); +}); diff --git a/vitest.config.ts b/vitest.config.ts index f8449ce..5a26130 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -4,7 +4,7 @@ export default defineConfig({ test: { coverage: { include: ['quotientai/**/*.ts'], - exclude: ['tests/**/*.ts', 'examples/**/*.ts'] - } - } -}); \ No newline at end of file + exclude: ['tests/**/*.ts', 'examples/**/*.ts'], + }, + }, +}); From f1543904901090ca8a6e147cf310cbcf13838f4d Mon Sep 17 00:00:00 2001 From: Mike Goitia Date: Thu, 15 May 2025 15:49:00 -0400 Subject: [PATCH 04/11] update readme --- README.md | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 3cabb72..3d9cea8 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ # quotientai + [![npm version](https://img.shields.io/npm/v/quotientai)](https://www.npmjs.com/package/quotientai) ## Overview @@ -16,8 +17,15 @@ npm install quotientai Create an API key on Quotient and set it as an environment variable called `QUOTIENT_API_KEY`. Then check out the examples in the `examples/` directory or see our [docs](https://docs.quotientai.co) for a more comprehensive walkthrough. The examples directory contains: + - `example_logs.ts` - Logging with hallucination and inconsistency detection +run the example + +```typescript +QUOTIENT_API_KEY={{YOUR_API_KEY}} ts-node examples/example_logs.ts +``` + ### QuotientAI The main client class that provides access to all QuotientAI resources. @@ -83,11 +91,11 @@ The SDK provides strongly-typed interfaces for working with detection results: ```typescript interface DetectionResults { - log: LogDetail; // Main log information - logDocuments: DocumentLog[] | null; // Reference documents + log: LogDetail; // Main log information + logDocuments: DocumentLog[] | null; // Reference documents logMessageHistory: LogMessageHistory[] | null; // Conversation history logInstructions: LogInstruction[] | null; // System instructions - evaluations: Evaluation[]; // Hallucination evaluations + evaluations: Evaluation[]; // Hallucination evaluations } ``` @@ -122,4 +130,4 @@ try { ## License -MIT \ No newline at end of file +MIT From 4aa1179272da0885f90374cda74cbded2ff9d932 Mon Sep 17 00:00:00 2001 From: Mike Goitia Date: Thu, 15 May 2025 17:39:06 -0400 Subject: [PATCH 05/11] feedback --- examples/example_logs.ts | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/examples/example_logs.ts b/examples/example_logs.ts index 2a17b59..731c068 100644 --- a/examples/example_logs.ts +++ b/examples/example_logs.ts @@ -1,4 +1,4 @@ -import { QuotientAI } from '../quotientai'; +import { QuotientAI } from 'quotientai'; async function main() { const quotient = new QuotientAI(); @@ -6,7 +6,7 @@ async function main() { // configure the logger const quotientLogger = quotient.logger.init({ - appName: 'my-app-test', + appName: 'my-app', environment: 'dev', sampleRate: 1.0, tags: { model: 'gpt-4o', feature: 'customer-support' }, @@ -41,20 +41,11 @@ async function main() { hallucinationDetection: true, inconsistencyDetection: true, }); - console.log('pollForDetectionResults with logId: ', logId); + console.log('pollForDetections with logId: ', logId); // poll for the detection results - const detectionResults = await quotientLogger.pollForDetectionResults(logId); - console.log('documentEvaluations', detectionResults?.evaluations[0].documentEvaluations); - console.log( - 'messageHistoryEvaluations', - detectionResults?.evaluations[0].messageHistoryEvaluations - ); - console.log('instructionEvaluations', detectionResults?.evaluations[0].instructionEvaluations); - console.log( - 'fullDocContextEvaluation', - detectionResults?.evaluations[0].fullDocContextEvaluation - ); + const detectionResults = await quotientLogger.pollForDetections(logId); + console.log('detectionResults', detectionResults); } catch (error) { console.error(error); } From 952300db9ee5b4676994b0bca0f5a5631017d8d8 Mon Sep 17 00:00:00 2001 From: Mike Goitia Date: Thu, 15 May 2025 18:02:33 -0400 Subject: [PATCH 06/11] cleanup --- package-lock.json | 2 +- quotientai/logger.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 4665ee3..6879664 100644 --- a/package-lock.json +++ b/package-lock.json @@ -23,7 +23,7 @@ "eslint": "^8.56.0", "eslint-config-prettier": "^10.1.5", "eslint-plugin-prettier": "^5.4.0", - "husky": "^9.1.7", + "husky": "^9.0.11", "lint-staged": "^16.0.0", "prettier": "^3.5.3", "typescript": "^5.3.3", diff --git a/quotientai/logger.ts b/quotientai/logger.ts index 7c7ba7d..64afad3 100644 --- a/quotientai/logger.ts +++ b/quotientai/logger.ts @@ -162,8 +162,8 @@ export class QuotientLogger { } } - // poll for the detection results using log id - async pollForDetectionResults( + // poll for detection results using log id + async pollForDetections( logId: string, timeout: number = 300, pollInterval: number = 2.0 From 6b4df844a07964e9c5be5086ad3f7bcb8a186d04 Mon Sep 17 00:00:00 2001 From: Mike Goitia Date: Thu, 15 May 2025 18:51:34 -0400 Subject: [PATCH 07/11] update readme --- README.md | 129 +++++++++--------------------------------------------- 1 file changed, 20 insertions(+), 109 deletions(-) diff --git a/README.md b/README.md index 3d9cea8..68c5ea0 100644 --- a/README.md +++ b/README.md @@ -14,120 +14,31 @@ npm install quotientai ## Usage -Create an API key on Quotient and set it as an environment variable called `QUOTIENT_API_KEY`. Then check out the examples in the `examples/` directory or see our [docs](https://docs.quotientai.co) for a more comprehensive walkthrough. +Create an API key on Quotient and set it as an environment variable called `QUOTIENT_API_KEY`. Check out the examples in the `examples/` directory or see our [docs](https://docs.quotientai.co) for a more comprehensive walkthrough. -The examples directory contains: - -- `example_logs.ts` - Logging with hallucination and inconsistency detection - -run the example - -```typescript -QUOTIENT_API_KEY={{YOUR_API_KEY}} ts-node examples/example_logs.ts -``` - -### QuotientAI - -The main client class that provides access to all QuotientAI resources. - -#### Constructor +Send your first log and detect hallucinations. Run the code below: ```typescript -new QuotientAI(apiKey?: string) -``` - -- `apiKey`: Optional API key. If not provided, will attempt to read from `QUOTIENT_API_KEY` environment variable. +import { QuotientAI } from 'quotientai'; -### QuotientLogger +const quotient = new QuotientAI(); -A logger interface for tracking model interactions and detecting hallucinations. +// initialize the logger +const quotientLogger = quotient.logger.init({ + app_name: 'my-app', + environment: 'dev', + sample_rate: 1.0, + hallucination_detection: true, + hallucination_detection_sample_rate: 1.0, +}); -#### Methods +// create a log +const logId = await quotientLogger.log({ + user_query: 'How do I cook a goose?', + model_output: 'The capital of France is Paris', + documents: ['Here is an excellent goose recipe...'], +}); -##### init - -```typescript -init(config: { - appName: string; - environment: string; - tags?: Record; - sampleRate?: number; - hallucinationDetection?: boolean; - inconsistencyDetection?: boolean; - hallucinationDetectionSampleRate?: number; -}): QuotientLogger -``` - -Configures the logger with the provided parameters. - -##### log - -```typescript -log(params: { - userQuery: string; - modelOutput: string; - documents?: (string | LogDocument)[]; - messageHistory?: Array>; - instructions?: string[]; - tags?: Record; - hallucinationDetection?: boolean; - inconsistencyDetection?: boolean; -}): Promise +// optionally, you can poll for detection results for further actions +const detectionResults = await quotientLogger.pollForDetections(logId); ``` - -Logs a model interaction. - -##### pollForDetections - -```typescript -pollForDetections(logId: string): Promise -``` - -Retrieves the detection results for a specific log entry, including hallucination and inconsistency evaluations. This method polls the API until results are available or a timeout is reached. - -## Detection Results - -The SDK provides strongly-typed interfaces for working with detection results: - -```typescript -interface DetectionResults { - log: LogDetail; // Main log information - logDocuments: DocumentLog[] | null; // Reference documents - logMessageHistory: LogMessageHistory[] | null; // Conversation history - logInstructions: LogInstruction[] | null; // System instructions - evaluations: Evaluation[]; // Hallucination evaluations -} -``` - -Each evaluation includes detailed information about whether content is hallucinated: - -```typescript -interface Evaluation { - id: string; - sentence: string; - isHallucinated: boolean; - // ... additional evaluation details - documentEvaluations: DocumentEvaluation[]; - messageHistoryEvaluations: MessageHistoryEvaluation[]; - instructionEvaluations: InstructionEvaluation[]; - fullDocContextEvaluation: FullDocContextEvaluation; -} -``` - -## Error Handling - -The client uses a custom `QuotientAIError` class for error handling: - -```typescript -try { - const results = await logger.pollForDetections('log-id'); -} catch (error) { - if (error instanceof QuotientAIError) { - console.error(`Error ${error.status}: ${error.message}`); - } -} -``` - -## License - -MIT From 3567c2974948d7d132480219039ff72a7c8d62fa Mon Sep 17 00:00:00 2001 From: Mike Goitia Date: Thu, 15 May 2025 18:57:49 -0400 Subject: [PATCH 08/11] readme --- README.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 68c5ea0..eb0843f 100644 --- a/README.md +++ b/README.md @@ -14,9 +14,9 @@ npm install quotientai ## Usage -Create an API key on Quotient and set it as an environment variable called `QUOTIENT_API_KEY`. Check out the examples in the `examples/` directory or see our [docs](https://docs.quotientai.co) for a more comprehensive walkthrough. +Create an API key on [Quotient](https://app.quotientai.co/dashboard) and set it as an environment variable called `QUOTIENT_API_KEY`. -Send your first log and detect hallucinations. Run the code below: +Send your first log and detect hallucinations. Run the code below and see your Logs and Detections on your [Quotient Dashboard](https://app.quotientai.co/dashboard). ```typescript import { QuotientAI } from 'quotientai'; @@ -42,3 +42,7 @@ const logId = await quotientLogger.log({ // optionally, you can poll for detection results for further actions const detectionResults = await quotientLogger.pollForDetections(logId); ``` + +## Docs + +For comprehensive documentation, please visit our [docs](https://docs.quotientai.co). From 50c6d12d556d434baae2ce1ebe44077a34a14cb5 Mon Sep 17 00:00:00 2001 From: Mike Goitia Date: Thu, 15 May 2025 19:00:13 -0400 Subject: [PATCH 09/11] readme --- README.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index eb0843f..7822273 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ Send your first log and detect hallucinations. Run the code below and see your L ```typescript import { QuotientAI } from 'quotientai'; -const quotient = new QuotientAI(); +const quotient = new QuotientAI(apiKey?: string); // initialize the logger const quotientLogger = quotient.logger.init({ @@ -43,6 +43,18 @@ const logId = await quotientLogger.log({ const detectionResults = await quotientLogger.pollForDetections(logId); ``` +### QuotientAI Client + +The main client class that provides access to all QuotientAI resources. + +#### Constructor + +```typescript +new QuotientAI(apiKey?: string) +``` + +- `apiKey`: Optional API key. If not provided, will attempt to read from `QUOTIENT_API_KEY` environment variable. + ## Docs For comprehensive documentation, please visit our [docs](https://docs.quotientai.co). From 27f9f7b4272f5bb8d0129a03b69851cf547ed4cd Mon Sep 17 00:00:00 2001 From: Freddie Vargus Date: Thu, 15 May 2025 23:07:52 -0400 Subject: [PATCH 10/11] Update logger.ts --- quotientai/logger.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/quotientai/logger.ts b/quotientai/logger.ts index 64afad3..423c97e 100644 --- a/quotientai/logger.ts +++ b/quotientai/logger.ts @@ -163,7 +163,7 @@ export class QuotientLogger { } // poll for detection results using log id - async pollForDetections( + async pollForDetection( logId: string, timeout: number = 300, pollInterval: number = 2.0 From 5f5e41ec238335ed664b99b317dfbee940a88f6f Mon Sep 17 00:00:00 2001 From: Freddie Vargus Date: Thu, 15 May 2025 23:12:08 -0400 Subject: [PATCH 11/11] Update README.md --- README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 7822273..90f1659 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ ## Overview -`quotientai` is an SDK built to manage logs and detect hallucinations and inconsistencies in AI responses with [Quotient](https://quotientai.co). +`quotientai` is an SDK and CLI for logging data to [Quotient](https://quotientai.co), and running hallucination and document attribution detections for retrieval and search-augmented AI systems. ## Installation @@ -14,7 +14,7 @@ npm install quotientai ## Usage -Create an API key on [Quotient](https://app.quotientai.co/dashboard) and set it as an environment variable called `QUOTIENT_API_KEY`. +Create an API key on [Quotient](https://app.quotientai.co) and set it as an environment variable called `QUOTIENT_API_KEY`. Then follow the examples below or see our [docs](https://docs.quotientai.co) for a more comprehensive walkthrough. Send your first log and detect hallucinations. Run the code below and see your Logs and Detections on your [Quotient Dashboard](https://app.quotientai.co/dashboard). @@ -25,17 +25,17 @@ const quotient = new QuotientAI(apiKey?: string); // initialize the logger const quotientLogger = quotient.logger.init({ - app_name: 'my-app', + appName: 'my-app', environment: 'dev', - sample_rate: 1.0, - hallucination_detection: true, + sampleRate: 1.0, + hallucinationDetection: true, hallucination_detection_sample_rate: 1.0, }); // create a log const logId = await quotientLogger.log({ - user_query: 'How do I cook a goose?', - model_output: 'The capital of France is Paris', + userQuery: 'How do I cook a goose?', + modelOutput: 'The capital of France is Paris', documents: ['Here is an excellent goose recipe...'], });