diff --git a/package-lock.json b/package-lock.json index 6879664..eb25ded 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "quotientai", - "version": "0.0.7", + "version": "0.0.8", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "quotientai", - "version": "0.0.7", + "version": "0.0.8", "license": "ISC", "dependencies": { "axios": "^1.6.7", @@ -27,7 +27,7 @@ "lint-staged": "^16.0.0", "prettier": "^3.5.3", "typescript": "^5.3.3", - "vitest": "^1.3.1" + "vitest": "^1.6.1" } }, "node_modules/@ampproject/remapping": { diff --git a/package.json b/package.json index 37a2739..26a883b 100644 --- a/package.json +++ b/package.json @@ -48,7 +48,7 @@ "lint-staged": "^16.0.0", "prettier": "^3.5.3", "typescript": "^5.3.3", - "vitest": "^1.3.1" + "vitest": "^1.6.1" }, "directories": { "src": "quotientai", diff --git a/tests/index.test.ts b/tests/index.test.ts index 3b586df..63f1add 100644 --- a/tests/index.test.ts +++ b/tests/index.test.ts @@ -68,121 +68,4 @@ describe('QuotientAI', () => { await new Promise((resolve) => setTimeout(resolve, 0)); expect(mockAuthenticate).toHaveBeenCalledOnce(); }); - - it('should log an error if parameters are invalid', async () => { - const consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); - const quotientAI = new QuotientAI('test_api_key'); - await quotientAI.evaluate({ - prompt: { - id: 'test_id', - name: 'test_name', - content: 'test_content', - version: 1, - user_prompt: 'test_prompt', - created_at: new Date(), - updated_at: new Date(), - }, - dataset: { - id: 'test_dataset', - name: 'test', - created_at: new Date(), - created_by: 'test_user', - updated_at: new Date(), - }, - model: { - id: 'test_model', - name: 'test', - provider: { id: 'test', name: 'test' }, - created_at: new Date(), - }, - parameters: { - invalid_param: 'value', - }, - metrics: ['test_metric'], - }); - expect(consoleErrorSpy).toHaveBeenCalledWith( - expect.stringContaining( - 'Invalid parameters: invalid_param. Valid parameters are: temperature, top_k, top_p, max_tokens' - ) - ); - }); - - it('should successfully evaluate if all parameters are valid', async () => { - const quotientAI = new QuotientAI('test_api_key'); - const mockRun = { - id: 'test_run', - status: 'completed', - }; - - // Mock the runs.create method - quotientAI.runs.create = vi.fn().mockResolvedValue(mockRun); - - const result = await quotientAI.evaluate({ - prompt: { - id: 'test_id', - name: 'test_name', - content: 'test_content', - version: 1, - user_prompt: 'test_prompt', - created_at: new Date(), - updated_at: new Date(), - }, - dataset: { - id: 'test_dataset', - name: 'test', - created_at: new Date(), - created_by: 'test_user', - updated_at: new Date(), - }, - model: { - id: 'test_model', - name: 'test', - provider: { id: 'test', name: 'test' }, - created_at: new Date(), - }, - parameters: { - temperature: 0.5, - top_k: 10, - top_p: 0.9, - max_tokens: 100, - }, - metrics: ['test_metric'], - }); - - expect(result).toBe(mockRun); - expect(quotientAI.runs.create).toHaveBeenCalledWith({ - prompt: expect.objectContaining({ id: 'test_id' }), - dataset: expect.objectContaining({ id: 'test_dataset' }), - model: expect.objectContaining({ id: 'test_model' }), - parameters: { - max_tokens: 100, - temperature: 0.5, - top_k: 10, - top_p: 0.9, - }, - metrics: ['test_metric'], - }); - }); - - it('should handle authentication errors during initialization', () => { - // Mock the authenticate method to throw an error - const error = new Error('Authentication failed'); - mockAuthenticate.mockImplementationOnce(() => { - throw error; - }); - - // Spy on console.error - const consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); - - // Create a new instance - const quotient = new QuotientAI('test_api_key'); - - // Verify error was logged with correct message - expect(consoleErrorSpy).toHaveBeenCalledWith( - expect.stringContaining( - 'If you are seeing this error, please check that your API key is correct.' - ) - ); - expect(quotient).toBeDefined(); - }); }); diff --git a/tests/logger.test.ts b/tests/logger.test.ts index 74656c1..7e3d623 100644 --- a/tests/logger.test.ts +++ b/tests/logger.test.ts @@ -13,7 +13,7 @@ describe('QuotientLogger', () => { }); it('should initialize without being configured', () => { - const mockLogsResource = { create: vi.fn(), list: vi.fn() }; + const mockLogsResource = { create: vi.fn(), list: vi.fn(), getDetections: vi.fn() }; const logger = new QuotientLogger(mockLogsResource); const privateLogger = logger as any; expect(logger).toBeDefined(); @@ -26,7 +26,7 @@ describe('QuotientLogger', () => { }); it('should use default values when not provided', () => { - const mockLogsResource = { create: vi.fn(), list: vi.fn() }; + const mockLogsResource = { create: vi.fn(), list: vi.fn(), getDetections: vi.fn() }; const logger = new QuotientLogger(mockLogsResource); const privateLogger = logger as any; @@ -45,7 +45,7 @@ describe('QuotientLogger', () => { }); it('should initialize with the correct properties', () => { - const mockLogsResource = { create: vi.fn(), list: vi.fn() }; + const mockLogsResource = { create: vi.fn(), list: vi.fn(), getDetections: vi.fn() }; const logger = new QuotientLogger(mockLogsResource); const privateLogger = logger as any; @@ -69,7 +69,7 @@ describe('QuotientLogger', () => { }); it('should log error and return this if sample rate is not between 0 and 1', () => { - const mockLogsResource = { create: vi.fn(), list: vi.fn() }; + const mockLogsResource = { create: vi.fn(), list: vi.fn(), getDetections: vi.fn() }; const logger = new QuotientLogger(mockLogsResource); const privateLogger = logger as any; const result = privateLogger.init({ sampleRate: 1.5 }); @@ -80,7 +80,7 @@ describe('QuotientLogger', () => { }); it('should log error and return null if you attempt to log before initializing', async () => { - const mockLogsResource = { create: vi.fn(), list: vi.fn() }; + const mockLogsResource = { create: vi.fn(), list: vi.fn(), getDetections: vi.fn() }; const logger = new QuotientLogger(mockLogsResource); const privateLogger = logger as any; const result = await privateLogger.log({ message: 'test' }); @@ -92,7 +92,7 @@ describe('QuotientLogger', () => { }); it('should log a message if initialized', async () => { - const mockLogsResource = { create: vi.fn(), list: vi.fn() }; + const mockLogsResource = { create: vi.fn(), list: vi.fn(), getDetections: vi.fn() }; const logger = new QuotientLogger(mockLogsResource); const privateLogger = logger as any; privateLogger.init({ @@ -102,19 +102,21 @@ describe('QuotientLogger', () => { sampleRate: 1.0, }); await privateLogger.log({ message: 'test' }); - expect(mockLogsResource.create).toHaveBeenCalledWith({ - appName: 'test_app', - environment: 'test_environment', - tags: { test: 'test' }, - message: 'test', - hallucinationDetection: false, - hallucinationDetectionSampleRate: 0, - inconsistencyDetection: false, - }); + expect(mockLogsResource.create).toHaveBeenCalledWith( + expect.objectContaining({ + appName: 'test_app', + environment: 'test_environment', + tags: { test: 'test' }, + message: 'test', + hallucinationDetection: false, + hallucinationDetectionSampleRate: 0, + inconsistencyDetection: false, + }) + ); }); it('should not log a message if shouldSample returns false', async () => { - const mockLogsResource = { create: vi.fn(), list: vi.fn() }; + const mockLogsResource = { create: vi.fn(), list: vi.fn(), getDetections: vi.fn() }; const logger = new QuotientLogger(mockLogsResource); const privateLogger = logger as any; // Mock shouldSample to always return false @@ -131,7 +133,7 @@ describe('QuotientLogger', () => { }); it('should verify shouldSample behavior based on Math.random', () => { - const mockLogsResource = { create: vi.fn(), list: vi.fn() }; + const mockLogsResource = { create: vi.fn(), list: vi.fn(), getDetections: vi.fn() }; const logger = new QuotientLogger(mockLogsResource); const privateLogger = logger as any; privateLogger.init({ sampleRate: 0.5 }); @@ -148,7 +150,7 @@ describe('QuotientLogger', () => { }); it('should log error and return null if required appName or environment is missing after initialization', async () => { - const mockLogsResource = { create: vi.fn(), list: vi.fn() }; + const mockLogsResource = { create: vi.fn(), list: vi.fn(), getDetections: vi.fn() }; const logger = new QuotientLogger(mockLogsResource); const privateLogger = logger as any; @@ -178,7 +180,7 @@ describe('QuotientLogger', () => { describe('Document Validation', () => { it('should accept string documents', async () => { - const mockLogsResource = { create: vi.fn(), list: vi.fn() }; + const mockLogsResource = { create: vi.fn(), list: vi.fn(), getDetections: vi.fn() }; const logger = new QuotientLogger(mockLogsResource); const privateLogger = logger as any; @@ -194,7 +196,7 @@ describe('QuotientLogger', () => { }); it('should accept valid LogDocument objects', async () => { - const mockLogsResource = { create: vi.fn(), list: vi.fn() }; + const mockLogsResource = { create: vi.fn(), list: vi.fn(), getDetections: vi.fn() }; const logger = new QuotientLogger(mockLogsResource); const privateLogger = logger as any; @@ -213,7 +215,7 @@ describe('QuotientLogger', () => { }); it('should log error and return null when document is missing pageContent', async () => { - const mockLogsResource = { create: vi.fn(), list: vi.fn() }; + const mockLogsResource = { create: vi.fn(), list: vi.fn(), getDetections: vi.fn() }; const logger = new QuotientLogger(mockLogsResource); const privateLogger = logger as any; @@ -232,7 +234,7 @@ describe('QuotientLogger', () => { }); it('should log error and return null when pageContent is not a string', async () => { - const mockLogsResource = { create: vi.fn(), list: vi.fn() }; + const mockLogsResource = { create: vi.fn(), list: vi.fn(), getDetections: vi.fn() }; const logger = new QuotientLogger(mockLogsResource); const privateLogger = logger as any; @@ -251,7 +253,7 @@ describe('QuotientLogger', () => { }); it('should log error and return null when metadata is not an object', async () => { - const mockLogsResource = { create: vi.fn(), list: vi.fn() }; + const mockLogsResource = { create: vi.fn(), list: vi.fn(), getDetections: vi.fn() }; const logger = new QuotientLogger(mockLogsResource); const privateLogger = logger as any; @@ -270,7 +272,7 @@ describe('QuotientLogger', () => { }); it('should accept null metadata', async () => { - const mockLogsResource = { create: vi.fn(), list: vi.fn() }; + const mockLogsResource = { create: vi.fn(), list: vi.fn(), getDetections: vi.fn() }; const logger = new QuotientLogger(mockLogsResource); const privateLogger = logger as any; @@ -286,7 +288,7 @@ describe('QuotientLogger', () => { }); it('should validate documents as part of the log method', async () => { - const mockLogsResource = { create: vi.fn(), list: vi.fn() }; + const mockLogsResource = { create: vi.fn(), list: vi.fn(), getDetections: vi.fn() }; const logger = new QuotientLogger(mockLogsResource); const privateLogger = logger as any; @@ -306,7 +308,7 @@ describe('QuotientLogger', () => { }); it('should skip validation if no documents are provided', async () => { - const mockLogsResource = { create: vi.fn(), list: vi.fn() }; + const mockLogsResource = { create: vi.fn(), list: vi.fn(), getDetections: vi.fn() }; const logger = new QuotientLogger(mockLogsResource); const privateLogger = logger as any; @@ -324,7 +326,7 @@ describe('QuotientLogger', () => { }); it('should directly test isValidLogDocument with various inputs', () => { - const mockLogsResource = { create: vi.fn(), list: vi.fn() }; + const mockLogsResource = { create: vi.fn(), list: vi.fn(), getDetections: vi.fn() }; const logger = new QuotientLogger(mockLogsResource); const privateLogger = logger as any;