Skip to content

Commit

Permalink
Removed rating from chili
Browse files Browse the repository at this point in the history
  • Loading branch information
@MathijsVerbeeck
MathijsVerbeeck committed May 17, 2024
1 parent 9dbe7a2 commit 6123809
Show file tree
Hide file tree
Showing 3 changed files with 8 additions and 406 deletions.
20 changes: 0 additions & 20 deletions docs/docs/user-guide/chili.mdx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,12 +53,6 @@ You can also start chili without the initial prompt by running `m365?` and then

Running this command will start the virtual assistant in the terminal. It will use the question you asked to search in CLI's documentation and answer your question, along with the list of sources from which it found the answer.

:::tip

After you get an answer, 🌶️ chili will prompt you to rate the response. While it's not required, we'd really appreciate it if you did. It will help us improve the quality of the answers.

:::

After you get an answer, you have the ability to follow up your conversation with additional questions or start from scratch. Start a new conversation to reset the assistant and ensure that it doesn't mix up the context of your questions.

## Options
Expand Down Expand Up @@ -92,17 +86,3 @@ or short:
```sh
m365? -h
```

### Disable rating prompts

If you don't want to be prompted to rate the response, you can disable it by starting 🌶️ chili with the `--no-rating` option:

```sh
m365? --no-rating
```

:::tip

While we offer this option, we'd appreciate your feedback, especially as we've just launched the assistant and want to improve its accuracy.

:::
310 changes: 7 additions & 303 deletions src/chili/chili.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ describe('chili', () => {
]);
});

it('starts a conversation using a prompt from args when specified', async () => {
it('starts a conversation using a prompt from args when specified with debug mode', async () => {
sinon.stub(request, 'post').callsFake(async options => {
switch (options.url) {
case 'https://api.mendable.ai/v1/newConversation':
Expand Down Expand Up @@ -70,7 +70,7 @@ describe('chili', () => {

throw `Prompt not found for '${config.message}'`;
});
await assert.doesNotReject(chili.startConversation(['Hello']));
await assert.doesNotReject(chili.startConversation(['Hello', '--debug']));
});

it('starts a conversation when a prompt specified as a single arg', async () => {
Expand Down Expand Up @@ -254,7 +254,7 @@ describe('chili', () => {

throw `Prompt not found for '${config.message}'`;
});
await chili.startConversation(['Hello', '--no-rating']);
await chili.startConversation(['Hello']);
assert(consoleLogSpy.calledWith('Hello back'));
});

Expand Down Expand Up @@ -336,302 +336,6 @@ describe('chili', () => {
assert(consoleLogSpy.calledWith('⬥ https://example.com/source-1'));
});

it('prompts for rating the response when rating is enabled', async () => {
let promptedForRating = false;
sinon.stub(request, 'post').callsFake(async options => {
switch (options.url) {
case 'https://api.mendable.ai/v1/newConversation':
return {
// eslint-disable-next-line camelcase
conversation_id: 1
};
case 'https://api.mendable.ai/v1/mendableChat':
if (options.data.question === 'Hello world') {
return {
answer: {
text: 'Hello back'
},
// eslint-disable-next-line camelcase
message_id: 1,
sources: []
};
}
break;
case 'https://api.mendable.ai/v1/rateMessage':
return {};
case 'https://api.mendable.ai/v1/endConversation':
return {};
}
throw `Invalid request: ${options.url}`;
});
sinon.stub(prompt, 'forInput').resolves('Hello world');
sinon.stub(prompt, 'forSelection').callsFake(async (config: SelectionConfig<unknown>): Promise<unknown> => {
if (config.message === 'Was this helpful?') {
promptedForRating = true;
return 1;
}
else if (config.message === 'What would you like to do next?') {
return 'end';
}

throw `Prompt not found for '${config.message}'`;
});
await chili.startConversation(['Hello world']);
assert.strictEqual(promptedForRating, true);
});

it(`doesn't prompt for rating the response when rating is disabled`, async () => {
let promptedForRating = false;
sinon.stub(request, 'post').callsFake(async options => {
switch (options.url) {
case 'https://api.mendable.ai/v1/newConversation':
return {
// eslint-disable-next-line camelcase
conversation_id: 1
};
case 'https://api.mendable.ai/v1/mendableChat':
if (options.data.question === 'Hello world') {
return {
answer: {
text: 'Hello back'
},
// eslint-disable-next-line camelcase
message_id: 1,
sources: []
};
}
break;
case 'https://api.mendable.ai/v1/endConversation':
return {};
}
throw `Invalid request: ${options.url}`;
});
sinon.stub(prompt, 'forInput').resolves('Hello world');
sinon.stub(prompt, 'forSelection').callsFake(async (config: SelectionConfig<unknown>): Promise<unknown> => {
if (config.message === 'Was this helpful?') {
promptedForRating = true;
return 1;
}
else if (config.message === 'What would you like to do next?') {
return 'end';
}

throw `Prompt not found for '${config.message}'`;
});
await chili.startConversation(['Hello world', '--no-rating']);
assert.strictEqual(promptedForRating, false);
});

it('sends positive rating to Mendable', async () => {
sinon.stub(request, 'post').callsFake(async options => {
switch (options.url) {
case 'https://api.mendable.ai/v1/newConversation':
return {
// eslint-disable-next-line camelcase
conversation_id: 1
};
case 'https://api.mendable.ai/v1/mendableChat':
if (options.data.question === 'Hello world') {
return {
answer: {
text: 'Hello back'
},
// eslint-disable-next-line camelcase
message_id: 1,
sources: []
};
}
break;
case 'https://api.mendable.ai/v1/rateMessage':
if (options.data.rating === 1) {
return {};
}
break;
case 'https://api.mendable.ai/v1/endConversation':
return {};
}
throw `Invalid request: ${options.url}`;
});
sinon.stub(prompt, 'forInput').resolves('Hello world');
sinon.stub(prompt, 'forSelection').callsFake(async (config: SelectionConfig<unknown>): Promise<unknown> => {
if (config.message === 'Was this helpful?') {
return 1;
}
if (config.message === 'What would you like to do next?') {
return 'end';
}

throw `Prompt not found for '${config.message}'`;
});
await assert.doesNotReject(chili.startConversation(['Hello world']));
});

it('sends negative rating to Mendable', async () => {
sinon.stub(request, 'post').callsFake(async options => {
switch (options.url) {
case 'https://api.mendable.ai/v1/newConversation':
return {
// eslint-disable-next-line camelcase
conversation_id: 1
};
case 'https://api.mendable.ai/v1/mendableChat':
if (options.data.question === 'Hello world') {
return {
answer: {
text: 'Hello back'
},
// eslint-disable-next-line camelcase
message_id: 1,
sources: []
};
}
break;
case 'https://api.mendable.ai/v1/rateMessage':
if (options.data.rating === -1) {
return {};
}
break;
case 'https://api.mendable.ai/v1/endConversation':
return {};
}
throw `Invalid request: ${options.url}`;
});
sinon.stub(prompt, 'forInput').resolves('Hello world');
sinon.stub(prompt, 'forSelection').callsFake(async (config: SelectionConfig<unknown>): Promise<unknown> => {
if (config.message === 'Was this helpful?') {
return -1;
}
if (config.message === 'What would you like to do next?') {
return 'end';
}

throw `Prompt not found for '${config.message}'`;
});
await assert.doesNotReject(chili.startConversation(['Hello world']));
});

it(`doesn't send rating to Mendable when user chose to skip`, async () => {
sinon.stub(request, 'post').callsFake(async options => {
switch (options.url) {
case 'https://api.mendable.ai/v1/newConversation':
return {
// eslint-disable-next-line camelcase
conversation_id: 1
};
case 'https://api.mendable.ai/v1/mendableChat':
if (options.data.question === 'Hello world') {
return {
answer: {
text: 'Hello back'
},
// eslint-disable-next-line camelcase
message_id: 1,
sources: []
};
}
break;
case 'https://api.mendable.ai/v1/endConversation':
return {};
}
throw `Invalid request: ${options.url}`;
});
sinon.stub(prompt, 'forInput').resolves('Hello world');
sinon.stub(prompt, 'forSelection').callsFake(async (config: SelectionConfig<unknown>): Promise<unknown> => {
if (config.message === 'Was this helpful?') {
return 0;
}
if (config.message === 'What would you like to do next?') {
return 'end';
}

throw `Prompt not found for '${config.message}'`;
});
await assert.doesNotReject(chili.startConversation(['Hello world']));
});

it(`doesn't fail when rating the response failed`, async () => {
sinon.stub(request, 'post').callsFake(async options => {
switch (options.url) {
case 'https://api.mendable.ai/v1/newConversation':
return {
// eslint-disable-next-line camelcase
conversation_id: 1
};
case 'https://api.mendable.ai/v1/mendableChat':
if (options.data.question === 'Hello world') {
return {
answer: {
text: 'Hello back'
},
// eslint-disable-next-line camelcase
message_id: 1,
sources: []
};
}
break;
case 'https://api.mendable.ai/v1/rateMessage':
throw 'An error has occurred';
case 'https://api.mendable.ai/v1/endConversation':
return {};
}
throw `Invalid request: ${options.url}`;
});
sinon.stub(prompt, 'forInput').resolves('Hello world');
sinon.stub(prompt, 'forSelection').callsFake(async (config: SelectionConfig<unknown>): Promise<unknown> => {
if (config.message === 'Was this helpful?') {
return 1;
}
if (config.message === 'What would you like to do next?') {
return 'end';
}

throw `Prompt not found for '${config.message}'`;
});
await assert.doesNotReject(chili.startConversation(['Hello world']));
});

it(`when rating the response failed, shows error in debug mode`, async () => {
sinon.stub(request, 'post').callsFake(async options => {
switch (options.url) {
case 'https://api.mendable.ai/v1/newConversation':
return {
// eslint-disable-next-line camelcase
conversation_id: 1
};
case 'https://api.mendable.ai/v1/mendableChat':
if (options.data.question === 'Hello world') {
return {
answer: {
text: 'Hello back'
},
// eslint-disable-next-line camelcase
message_id: 1,
sources: []
};
}
break;
case 'https://api.mendable.ai/v1/rateMessage':
throw 'An error has occurred';
case 'https://api.mendable.ai/v1/endConversation':
return {};
}
throw `Invalid request: ${options.url}`;
});
sinon.stub(prompt, 'forInput').resolves('Hello world');
sinon.stub(prompt, 'forSelection').callsFake(async (config: SelectionConfig<unknown>): Promise<unknown> => {
if (config.message === 'Was this helpful?') {
return 1;
}
else if (config.message === 'What would you like to do next?') {
return 'end';
}

throw `Prompt not found for '${config.message}'`;
});
await chili.startConversation(['Hello world', '--debug']);
assert(consoleErrorSpy.calledWith('An error has occurred while rating the response: An error has occurred'));
});

it('allows asking a follow-up question after a response', async () => {
let questionsAsked = 0;
sinon.stub(request, 'post').callsFake(async options => {
Expand Down Expand Up @@ -678,7 +382,7 @@ describe('chili', () => {

throw `Prompt not found for '${config.message}'`;
});
await assert.doesNotReject(chili.startConversation(['Hello', '--no-rating']));
await assert.doesNotReject(chili.startConversation(['Hello']));
});

it('for a follow-up question, includes the history', async () => {
Expand Down Expand Up @@ -724,7 +428,7 @@ describe('chili', () => {

return 'ask';
});
await assert.doesNotReject(chili.startConversation(['Hello', '--no-rating']));
await assert.doesNotReject(chili.startConversation(['Hello']));
});

it('allows ending conversation after a response', async () => {
Expand Down Expand Up @@ -758,7 +462,7 @@ describe('chili', () => {

throw `Prompt not found for '${config.message}'`;
});
await assert.doesNotReject(chili.startConversation(['Hello', '--no-rating']));
await assert.doesNotReject(chili.startConversation(['Hello']));
});

it('allows starting a new conversation after a response', async () => {
Expand Down Expand Up @@ -805,7 +509,7 @@ describe('chili', () => {

throw `Prompt not found for '${config.message}'`;
});
await assert.doesNotReject(chili.startConversation(['Hello', '--no-rating']));
await assert.doesNotReject(chili.startConversation(['Hello']));
});

it('throws exception when getting conversation ID failed', async () => {
Expand Down
Loading

0 comments on commit 6123809

Please sign in to comment.