Skip to content

Refactor askAI plugin: improve file exclusion logic and enhance conttext #611

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Dec 5, 2024
Merged

Refactor askAI plugin: improve file exclusion logic and enhance conttext #611

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
292937b
Refactor askAI plugin: improve file exclusion logic and enhance conte…
MishaKav Dec 3, 2024
a91da94
Merge branch 'main' into LINBEE-12502-fix-askai-plugin
PavelLinearB Dec 4, 2024
1e387ff
increase file diff from 800 to 1000
MishaKav Dec 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
98 changes: 51 additions & 47 deletions plugins/filters/askAI/index.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,11 @@
* @returns {Object} Returns the response from the AI model.
* @example {{ branch | generateDescription(pr, repo, source) }}
* @license MIT
**/
* */

const lockFiles = [
const MAX_TOKENS = 4096;
const OPEN_AI_ENDPOINT = 'https://api.openai.com/v1/chat/completions';
const LOCK_FILES = [
'package-lock.json',
'yarn.lock',
'npm-shrinkwrap.json',
Expand All @@ -36,8 +38,7 @@ const lockFiles = [
'flake.lock',
'pnpm-lock.yaml'
];

const excludeExpressionsList = [
const EXCLUDE_EXPRESSIONS_LIST = [
'.*\\.(ini|csv|xls|xlsx|xlr|doc|docx|txt|pps|ppt|pptx|dot|dotx|log|tar|rtf|dat|ipynb|po|profile|object|obj|dxf|twb|bcsymbolmap|tfstate|pdf|rbi|pem|crt|svg|png|jpeg|jpg|ttf)$',
'.*(package-lock|packages\\.lock|package)\\.json$',
'.*(yarn|gemfile|podfile|cargo|composer|pipfile|gopkg)\\.lock$',
Expand All @@ -47,55 +48,58 @@ const excludeExpressionsList = [
'.*public/assets/.*\\.js',
'.*ci\\.yml$'
];
const IGNORE_FILES_REGEX_LIST = [
...LOCK_FILES.map(f => f.replace('.', '\\.')),
...EXCLUDE_EXPRESSIONS_LIST
];
const EXCLUDE_PATTERN = new RegExp(IGNORE_FILES_REGEX_LIST.join('|'));

const ignoreFilesRegexList = lockFiles
.map(file => file.replace('.', '\\.'))
.concat(excludeExpressionsList);
const excludePattern = new RegExp(ignoreFilesRegexList.join('|'));

const filterExcludeFiles = file => {
return !excludePattern.test(file) || (file.diff?.split(' ').length ?? 0) < 800;
};

const buildArrayContext = context => {
return context.filter(element => {
if (typeof element !== 'object') {
return true;
}

return context.filter(filterExcludeFiles);
});
/**
* @description Check if a file should be excluded from the context like "package-lock.json"
* @param {*} fileObject
* @returns returns true if the file should be excluded
*/
const shouldExcludeFile = fileObject => {
const shouldExludeByName = EXCLUDE_PATTERN.test(fileObject.original_file);
const shouldExludeBySize = (fileObject.diff?.split(' ').length ?? 0) > 1000;

return shouldExludeByName || shouldExludeBySize;
};

const buildSourceContext = context => {
return context.diff.files.filter(filterExcludeFiles);
/**
* @description Check if a file should be included in the context
* @param {*} fileObject
* @returns returns true if the file should be included
*/
const shouldIncludeFile = fileObject => {
return !shouldExcludeFile(fileObject);
};

const buildContextForGPT = context => {
if (Array.isArray(context)) {
return buildArrayContext(context);
return context.filter(element =>
typeof element !== 'object' ? true : context.filter(shouldIncludeFile)
);
}

if (context?.diff?.files) {
return buildSourceContext(context);
const files = context.diff.files.filter(shouldIncludeFile);
return files;
}

return context;
};

const askAI = async (context, role = '', prompt, token, callback) => {
const cacheKey = `${__filename}${role}${prompt}`;

if (process.env[cacheKey]) {
return callback(null, process.env[cacheKey]);
}

const maxTokens = 4096;
const endpoint = 'https://api.openai.com/v1/chat/completions';

const askAI = async (context, role, prompt, token, callback) => {
const formattedContext = buildContextForGPT(context);

const response = await fetch(endpoint, {
if (!formattedContext?.length) {
const message = `There are no context files to analyze.\nAll ${context?.diff?.files?.length} files were excluded by pattern or size.`;
console.log(message);
return callback(null, message);
}

const response = await fetch(OPEN_AI_ENDPOINT, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
Expand All @@ -104,32 +108,32 @@ const askAI = async (context, role = '', prompt, token, callback) => {
body: JSON.stringify({
model: 'gpt-4o-2024-08-06',
messages: [
...(role ?
[
{
role: 'system',
content: `You are a ${role}. Answer only to the request, without any introductory or conclusion text.`
}]
: []),
{
role: 'system',
content: `You are a ${role}. Answer only to the request, without any introductory or conclusion text.`
},
{
role: 'user',
content: JSON.stringify(formattedContext)
},
{ role: 'user', content: prompt }
],
max_tokens: maxTokens
max_tokens: MAX_TOKENS
})
});

const data = await response.json();

if (data?.error?.message) {
console.error(data.error.message);
return callback(null, data.error.message);
}

const suggestion =
data.choices?.[0]?.message?.content ??
'context was too big for api, try with smaller context object';

process.env[cacheKey] = suggestion;

return callback(null, process.env[cacheKey]);
return callback(null, suggestion);
};

module.exports = {
Expand Down