From 292937b2fa94b6e79efb218d1680fc316e4a3a36 Mon Sep 17 00:00:00 2001 From: Misha Kav Date: Tue, 3 Dec 2024 16:04:10 +0200 Subject: [PATCH 1/2] Refactor askAI plugin: improve file exclusion logic and enhance context handling --- plugins/filters/askAI/index.js | 98 ++++++++++++++++++---------------- 1 file changed, 51 insertions(+), 47 deletions(-) diff --git a/plugins/filters/askAI/index.js b/plugins/filters/askAI/index.js index a543abff7..787ced9db 100644 --- a/plugins/filters/askAI/index.js +++ b/plugins/filters/askAI/index.js @@ -8,9 +8,11 @@ * @returns {Object} Returns the response from the AI model. * @example {{ branch | generateDescription(pr, repo, source) }} * @license MIT -**/ + * */ -const lockFiles = [ +const MAX_TOKENS = 4096; +const OPEN_AI_ENDPOINT = 'https://api.openai.com/v1/chat/completions'; +const LOCK_FILES = [ 'package-lock.json', 'yarn.lock', 'npm-shrinkwrap.json', @@ -36,8 +38,7 @@ const lockFiles = [ 'flake.lock', 'pnpm-lock.yaml' ]; - -const excludeExpressionsList = [ +const EXCLUDE_EXPRESSIONS_LIST = [ '.*\\.(ini|csv|xls|xlsx|xlr|doc|docx|txt|pps|ppt|pptx|dot|dotx|log|tar|rtf|dat|ipynb|po|profile|object|obj|dxf|twb|bcsymbolmap|tfstate|pdf|rbi|pem|crt|svg|png|jpeg|jpg|ttf)$', '.*(package-lock|packages\\.lock|package)\\.json$', '.*(yarn|gemfile|podfile|cargo|composer|pipfile|gopkg)\\.lock$', @@ -47,55 +48,58 @@ const excludeExpressionsList = [ '.*public/assets/.*\\.js', '.*ci\\.yml$' ]; +const IGNORE_FILES_REGEX_LIST = [ + ...LOCK_FILES.map(f => f.replace('.', '\\.')), + ...EXCLUDE_EXPRESSIONS_LIST +]; +const EXCLUDE_PATTERN = new RegExp(IGNORE_FILES_REGEX_LIST.join('|')); -const ignoreFilesRegexList = lockFiles - .map(file => file.replace('.', '\\.')) - .concat(excludeExpressionsList); -const excludePattern = new RegExp(ignoreFilesRegexList.join('|')); - -const filterExcludeFiles = file => { - return !excludePattern.test(file) || (file.diff?.split(' ').length ?? 0) < 800; -}; - -const buildArrayContext = context => { - return context.filter(element => { - if (typeof element !== 'object') { - return true; - } - - return context.filter(filterExcludeFiles); - }); +/** + * @description Check if a file should be excluded from the context like "package-lock.json" + * @param {*} fileObject + * @returns returns true if the file should be excluded + */ +const shouldExcludeFile = fileObject => { + const shouldExludeByName = EXCLUDE_PATTERN.test(fileObject.original_file); + const shouldExludeBySize = (fileObject.diff?.split(' ').length ?? 0) > 800; + + return shouldExludeByName || shouldExludeBySize; }; -const buildSourceContext = context => { - return context.diff.files.filter(filterExcludeFiles); +/** + * @description Check if a file should be included in the context + * @param {*} fileObject + * @returns returns true if the file should be included + */ +const shouldIncludeFile = fileObject => { + return !shouldExcludeFile(fileObject); }; const buildContextForGPT = context => { if (Array.isArray(context)) { - return buildArrayContext(context); + return context.filter(element => + typeof element !== 'object' ? true : context.filter(shouldIncludeFile) + ); } if (context?.diff?.files) { - return buildSourceContext(context); + const files = context.diff.files.filter(shouldIncludeFile); + return files; } return context; }; -const askAI = async (context, role = '', prompt, token, callback) => { - const cacheKey = `${__filename}${role}${prompt}`; - - if (process.env[cacheKey]) { - return callback(null, process.env[cacheKey]); - } - - const maxTokens = 4096; - const endpoint = 'https://api.openai.com/v1/chat/completions'; - +const askAI = async (context, role, prompt, token, callback) => { const formattedContext = buildContextForGPT(context); - const response = await fetch(endpoint, { + if (!formattedContext?.length) { + const message = `There are no context files to analyze.\nAll ${context?.diff?.files?.length} files were excluded by pattern or size.`; + console.log(message); + return callback(null, message); + } + + const response = await fetch(OPEN_AI_ENDPOINT, { method: 'POST', headers: { 'Content-Type': 'application/json', @@ -104,32 +108,32 @@ const askAI = async (context, role = '', prompt, token, callback) => { body: JSON.stringify({ model: 'gpt-4o-2024-08-06', messages: [ - ...(role ? - [ - { - role: 'system', - content: `You are a ${role}. Answer only to the request, without any introductory or conclusion text.` - }] - : []), + { + role: 'system', + content: `You are a ${role}. Answer only to the request, without any introductory or conclusion text.` + }, { role: 'user', content: JSON.stringify(formattedContext) }, { role: 'user', content: prompt } ], - max_tokens: maxTokens + max_tokens: MAX_TOKENS }) }); const data = await response.json(); + if (data?.error?.message) { + console.error(data.error.message); + return callback(null, data.error.message); + } + const suggestion = data.choices?.[0]?.message?.content ?? 'context was too big for api, try with smaller context object'; - process.env[cacheKey] = suggestion; - - return callback(null, process.env[cacheKey]); + return callback(null, suggestion); }; module.exports = { From 1e387ffae3e18557a3ff32f03af0365d6592e079 Mon Sep 17 00:00:00 2001 From: Misha Kav Date: Thu, 5 Dec 2024 16:00:03 +0200 Subject: [PATCH 2/2] increase file diff from 800 to 1000 --- plugins/filters/askAI/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/filters/askAI/index.js b/plugins/filters/askAI/index.js index 787ced9db..21a20f6b7 100644 --- a/plugins/filters/askAI/index.js +++ b/plugins/filters/askAI/index.js @@ -61,7 +61,7 @@ const EXCLUDE_PATTERN = new RegExp(IGNORE_FILES_REGEX_LIST.join('|')); */ const shouldExcludeFile = fileObject => { const shouldExludeByName = EXCLUDE_PATTERN.test(fileObject.original_file); - const shouldExludeBySize = (fileObject.diff?.split(' ').length ?? 0) > 800; + const shouldExludeBySize = (fileObject.diff?.split(' ').length ?? 0) > 1000; return shouldExludeByName || shouldExludeBySize; };