Skip to content

Fix broken test, update vitest #15

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
May 16, 2025
Merged

Fix broken test, update vitest #15

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
34c166e
Fix broken test
freddiev4 May 16, 2025
0558e4c
Fix broken logs test
freddiev4 May 16, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions package-lock.json
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@
"lint-staged": "^16.0.0",
"prettier": "^3.5.3",
"typescript": "^5.3.3",
"vitest": "^1.3.1"
"vitest": "^1.6.1"
},
"directories": {
"src": "quotientai",
Expand Down
117 changes: 0 additions & 117 deletions tests/index.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,121 +68,4 @@ describe('QuotientAI', () => {
await new Promise((resolve) => setTimeout(resolve, 0));
expect(mockAuthenticate).toHaveBeenCalledOnce();
});

it('should log an error if parameters are invalid', async () => {
const consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => {});
const quotientAI = new QuotientAI('test_api_key');
await quotientAI.evaluate({
prompt: {
id: 'test_id',
name: 'test_name',
content: 'test_content',
version: 1,
user_prompt: 'test_prompt',
created_at: new Date(),
updated_at: new Date(),
},
dataset: {
id: 'test_dataset',
name: 'test',
created_at: new Date(),
created_by: 'test_user',
updated_at: new Date(),
},
model: {
id: 'test_model',
name: 'test',
provider: { id: 'test', name: 'test' },
created_at: new Date(),
},
parameters: {
invalid_param: 'value',
},
metrics: ['test_metric'],
});
expect(consoleErrorSpy).toHaveBeenCalledWith(
expect.stringContaining(
'Invalid parameters: invalid_param. Valid parameters are: temperature, top_k, top_p, max_tokens'
)
);
});

it('should successfully evaluate if all parameters are valid', async () => {
const quotientAI = new QuotientAI('test_api_key');
const mockRun = {
id: 'test_run',
status: 'completed',
};

// Mock the runs.create method
quotientAI.runs.create = vi.fn().mockResolvedValue(mockRun);

const result = await quotientAI.evaluate({
prompt: {
id: 'test_id',
name: 'test_name',
content: 'test_content',
version: 1,
user_prompt: 'test_prompt',
created_at: new Date(),
updated_at: new Date(),
},
dataset: {
id: 'test_dataset',
name: 'test',
created_at: new Date(),
created_by: 'test_user',
updated_at: new Date(),
},
model: {
id: 'test_model',
name: 'test',
provider: { id: 'test', name: 'test' },
created_at: new Date(),
},
parameters: {
temperature: 0.5,
top_k: 10,
top_p: 0.9,
max_tokens: 100,
},
metrics: ['test_metric'],
});

expect(result).toBe(mockRun);
expect(quotientAI.runs.create).toHaveBeenCalledWith({
prompt: expect.objectContaining({ id: 'test_id' }),
dataset: expect.objectContaining({ id: 'test_dataset' }),
model: expect.objectContaining({ id: 'test_model' }),
parameters: {
max_tokens: 100,
temperature: 0.5,
top_k: 10,
top_p: 0.9,
},
metrics: ['test_metric'],
});
});

it('should handle authentication errors during initialization', () => {
// Mock the authenticate method to throw an error
const error = new Error('Authentication failed');
mockAuthenticate.mockImplementationOnce(() => {
throw error;
});

// Spy on console.error
const consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => {});

// Create a new instance
const quotient = new QuotientAI('test_api_key');

// Verify error was logged with correct message
expect(consoleErrorSpy).toHaveBeenCalledWith(
expect.stringContaining(
'If you are seeing this error, please check that your API key is correct.'
)
);
expect(quotient).toBeDefined();
});
});
56 changes: 29 additions & 27 deletions tests/logger.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ describe('QuotientLogger', () => {
});

it('should initialize without being configured', () => {
const mockLogsResource = { create: vi.fn(), list: vi.fn() };
const mockLogsResource = { create: vi.fn(), list: vi.fn(), getDetections: vi.fn() };
const logger = new QuotientLogger(mockLogsResource);
const privateLogger = logger as any;
expect(logger).toBeDefined();
Expand All @@ -26,7 +26,7 @@ describe('QuotientLogger', () => {
});

it('should use default values when not provided', () => {
const mockLogsResource = { create: vi.fn(), list: vi.fn() };
const mockLogsResource = { create: vi.fn(), list: vi.fn(), getDetections: vi.fn() };
const logger = new QuotientLogger(mockLogsResource);
const privateLogger = logger as any;

Expand All @@ -45,7 +45,7 @@ describe('QuotientLogger', () => {
});

it('should initialize with the correct properties', () => {
const mockLogsResource = { create: vi.fn(), list: vi.fn() };
const mockLogsResource = { create: vi.fn(), list: vi.fn(), getDetections: vi.fn() };
const logger = new QuotientLogger(mockLogsResource);
const privateLogger = logger as any;

Expand All @@ -69,7 +69,7 @@ describe('QuotientLogger', () => {
});

it('should log error and return this if sample rate is not between 0 and 1', () => {
const mockLogsResource = { create: vi.fn(), list: vi.fn() };
const mockLogsResource = { create: vi.fn(), list: vi.fn(), getDetections: vi.fn() };
const logger = new QuotientLogger(mockLogsResource);
const privateLogger = logger as any;
const result = privateLogger.init({ sampleRate: 1.5 });
Expand All @@ -80,7 +80,7 @@ describe('QuotientLogger', () => {
});

it('should log error and return null if you attempt to log before initializing', async () => {
const mockLogsResource = { create: vi.fn(), list: vi.fn() };
const mockLogsResource = { create: vi.fn(), list: vi.fn(), getDetections: vi.fn() };
const logger = new QuotientLogger(mockLogsResource);
const privateLogger = logger as any;
const result = await privateLogger.log({ message: 'test' });
Expand All @@ -92,7 +92,7 @@ describe('QuotientLogger', () => {
});

it('should log a message if initialized', async () => {
const mockLogsResource = { create: vi.fn(), list: vi.fn() };
const mockLogsResource = { create: vi.fn(), list: vi.fn(), getDetections: vi.fn() };
const logger = new QuotientLogger(mockLogsResource);
const privateLogger = logger as any;
privateLogger.init({
Expand All @@ -102,19 +102,21 @@ describe('QuotientLogger', () => {
sampleRate: 1.0,
});
await privateLogger.log({ message: 'test' });
expect(mockLogsResource.create).toHaveBeenCalledWith({
appName: 'test_app',
environment: 'test_environment',
tags: { test: 'test' },
message: 'test',
hallucinationDetection: false,
hallucinationDetectionSampleRate: 0,
inconsistencyDetection: false,
});
expect(mockLogsResource.create).toHaveBeenCalledWith(
expect.objectContaining({
appName: 'test_app',
environment: 'test_environment',
tags: { test: 'test' },
message: 'test',
hallucinationDetection: false,
hallucinationDetectionSampleRate: 0,
inconsistencyDetection: false,
})
);
});

it('should not log a message if shouldSample returns false', async () => {
const mockLogsResource = { create: vi.fn(), list: vi.fn() };
const mockLogsResource = { create: vi.fn(), list: vi.fn(), getDetections: vi.fn() };
const logger = new QuotientLogger(mockLogsResource);
const privateLogger = logger as any;
// Mock shouldSample to always return false
Expand All @@ -131,7 +133,7 @@ describe('QuotientLogger', () => {
});

it('should verify shouldSample behavior based on Math.random', () => {
const mockLogsResource = { create: vi.fn(), list: vi.fn() };
const mockLogsResource = { create: vi.fn(), list: vi.fn(), getDetections: vi.fn() };
const logger = new QuotientLogger(mockLogsResource);
const privateLogger = logger as any;
privateLogger.init({ sampleRate: 0.5 });
Expand All @@ -148,7 +150,7 @@ describe('QuotientLogger', () => {
});

it('should log error and return null if required appName or environment is missing after initialization', async () => {
const mockLogsResource = { create: vi.fn(), list: vi.fn() };
const mockLogsResource = { create: vi.fn(), list: vi.fn(), getDetections: vi.fn() };
const logger = new QuotientLogger(mockLogsResource);
const privateLogger = logger as any;

Expand Down Expand Up @@ -178,7 +180,7 @@ describe('QuotientLogger', () => {

describe('Document Validation', () => {
it('should accept string documents', async () => {
const mockLogsResource = { create: vi.fn(), list: vi.fn() };
const mockLogsResource = { create: vi.fn(), list: vi.fn(), getDetections: vi.fn() };
const logger = new QuotientLogger(mockLogsResource);
const privateLogger = logger as any;

Expand All @@ -194,7 +196,7 @@ describe('QuotientLogger', () => {
});

it('should accept valid LogDocument objects', async () => {
const mockLogsResource = { create: vi.fn(), list: vi.fn() };
const mockLogsResource = { create: vi.fn(), list: vi.fn(), getDetections: vi.fn() };
const logger = new QuotientLogger(mockLogsResource);
const privateLogger = logger as any;

Expand All @@ -213,7 +215,7 @@ describe('QuotientLogger', () => {
});

it('should log error and return null when document is missing pageContent', async () => {
const mockLogsResource = { create: vi.fn(), list: vi.fn() };
const mockLogsResource = { create: vi.fn(), list: vi.fn(), getDetections: vi.fn() };
const logger = new QuotientLogger(mockLogsResource);
const privateLogger = logger as any;

Expand All @@ -232,7 +234,7 @@ describe('QuotientLogger', () => {
});

it('should log error and return null when pageContent is not a string', async () => {
const mockLogsResource = { create: vi.fn(), list: vi.fn() };
const mockLogsResource = { create: vi.fn(), list: vi.fn(), getDetections: vi.fn() };
const logger = new QuotientLogger(mockLogsResource);
const privateLogger = logger as any;

Expand All @@ -251,7 +253,7 @@ describe('QuotientLogger', () => {
});

it('should log error and return null when metadata is not an object', async () => {
const mockLogsResource = { create: vi.fn(), list: vi.fn() };
const mockLogsResource = { create: vi.fn(), list: vi.fn(), getDetections: vi.fn() };
const logger = new QuotientLogger(mockLogsResource);
const privateLogger = logger as any;

Expand All @@ -270,7 +272,7 @@ describe('QuotientLogger', () => {
});

it('should accept null metadata', async () => {
const mockLogsResource = { create: vi.fn(), list: vi.fn() };
const mockLogsResource = { create: vi.fn(), list: vi.fn(), getDetections: vi.fn() };
const logger = new QuotientLogger(mockLogsResource);
const privateLogger = logger as any;

Expand All @@ -286,7 +288,7 @@ describe('QuotientLogger', () => {
});

it('should validate documents as part of the log method', async () => {
const mockLogsResource = { create: vi.fn(), list: vi.fn() };
const mockLogsResource = { create: vi.fn(), list: vi.fn(), getDetections: vi.fn() };
const logger = new QuotientLogger(mockLogsResource);
const privateLogger = logger as any;

Expand All @@ -306,7 +308,7 @@ describe('QuotientLogger', () => {
});

it('should skip validation if no documents are provided', async () => {
const mockLogsResource = { create: vi.fn(), list: vi.fn() };
const mockLogsResource = { create: vi.fn(), list: vi.fn(), getDetections: vi.fn() };
const logger = new QuotientLogger(mockLogsResource);
const privateLogger = logger as any;

Expand All @@ -324,7 +326,7 @@ describe('QuotientLogger', () => {
});

it('should directly test isValidLogDocument with various inputs', () => {
const mockLogsResource = { create: vi.fn(), list: vi.fn() };
const mockLogsResource = { create: vi.fn(), list: vi.fn(), getDetections: vi.fn() };
const logger = new QuotientLogger(mockLogsResource);
const privateLogger = logger as any;

Expand Down