From 7f2511c9116dccaa14f842fc9d1de2e46fa614f5 Mon Sep 17 00:00:00 2001 From: ToonCodes Date: Tue, 11 Jun 2024 03:47:32 -0600 Subject: [PATCH 1/6] Updates ChatGPT to 4o model Adds support for image queries --- package-lock.json | 222 ++++++++++++++++++++++++++---- package.json | 2 +- src/api/routes/chatgpt.ts | 6 +- src/commands/utilities/chatgpt.ts | 67 ++++++--- src/discord-utils.ts | 20 +-- src/events/dms.ts | 1 + 6 files changed, 259 insertions(+), 59 deletions(-) diff --git a/package-lock.json b/package-lock.json index cf18464..ace0619 100644 --- a/package-lock.json +++ b/package-lock.json @@ -39,7 +39,7 @@ "node-cache": "^5.1.2", "node-fetch": "^2.7.0", "node-notifier": "^9.0.0", - "openai": "^3.3.0", + "openai": "^4.50.0", "p-limit": "^3.1.0", "pg": "^8.5.1", "play-dl": "^1.9.7", @@ -1871,7 +1871,6 @@ "version": "2.6.6", "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.6.tgz", "integrity": "sha512-95X8guJYhfqiuVVhRFxVQcf4hW/2bCuoPwDasMf/531STFoNoWTT7YDnWdXHEZKqAGUigmpG31r2FE70LwnzJw==", - "dev": true, "dependencies": { "@types/node": "*", "form-data": "^4.0.0" @@ -2265,6 +2264,17 @@ "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==" }, + "node_modules/abort-controller": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", + "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", + "dependencies": { + "event-target-shim": "^5.0.0" + }, + "engines": { + "node": ">=6.5" + } + }, "node_modules/accepts": { "version": "1.3.7", "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.7.tgz", @@ -2318,6 +2328,17 @@ "node": ">= 6.0.0" } }, + "node_modules/agentkeepalive": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.5.0.tgz", + "integrity": "sha512-5GG/5IbQQpC9FpkRGsSvZI5QYeSCzlJHdpBQntCsuTOxhKD8lqKhrleg2Yi7yvMIf82Ycmmqln9U8V9qwEiJew==", + "dependencies": { + "humanize-ms": "^1.2.1" + }, + "engines": { + "node": ">= 8.0.0" + } + }, "node_modules/ajv": { "version": "6.12.6", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", @@ -4358,6 +4379,14 @@ "node": ">= 0.6" } }, + "node_modules/event-target-shim": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", + "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==", + "engines": { + "node": ">=6" + } + }, "node_modules/execa": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/execa/-/execa-5.1.1.tgz", @@ -4720,6 +4749,31 @@ "node": ">= 6" } }, + "node_modules/form-data-encoder": { + "version": "1.7.2", + "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-1.7.2.tgz", + "integrity": "sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==" + }, + "node_modules/formdata-node": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/formdata-node/-/formdata-node-4.4.1.tgz", + "integrity": "sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==", + "dependencies": { + "node-domexception": "1.0.0", + "web-streams-polyfill": "4.0.0-beta.3" + }, + "engines": { + "node": ">= 12.20" + } + }, + "node_modules/formdata-node/node_modules/web-streams-polyfill": { + "version": "4.0.0-beta.3", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz", + "integrity": "sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==", + "engines": { + "node": ">= 14" + } + }, "node_modules/forwarded": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", @@ -5260,6 +5314,14 @@ "resolved": "https://registry.npmjs.org/humanize-duration/-/humanize-duration-3.27.1.tgz", "integrity": "sha512-jCVkMl+EaM80rrMrAPl96SGG4NRac53UyI1o/yAzebDntEY6K6/Fj2HOjdPg8omTqIe5Y0wPBai2q5xXrIbarA==" }, + "node_modules/humanize-ms": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz", + "integrity": "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==", + "dependencies": { + "ms": "^2.0.0" + } + }, "node_modules/iconv-lite": { "version": "0.4.24", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", @@ -7310,6 +7372,24 @@ "node": ">= 8.0.0" } }, + "node_modules/node-domexception": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", + "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "github", + "url": "https://paypal.me/jimmywarting" + } + ], + "engines": { + "node": ">=10.5.0" + } + }, "node_modules/node-fetch": { "version": "2.7.0", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", @@ -7613,20 +7693,29 @@ } }, "node_modules/openai": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/openai/-/openai-3.3.0.tgz", - "integrity": "sha512-uqxI/Au+aPRnsaQRe8CojU0eCR7I0mBiKjD3sNMzY6DaC1ZVrc85u98mtJW6voDug8fgGN+DIZmTDxTthxb7dQ==", - "dependencies": { - "axios": "^0.26.0", - "form-data": "^4.0.0" + "version": "4.50.0", + "resolved": "https://registry.npmjs.org/openai/-/openai-4.50.0.tgz", + "integrity": "sha512-2ADkNIU6Q589oYHr5pn9k7SbUcrBTK9X0rIXrYqwMVSoqOj1yK9/1OO0ExaWsqOOpD7o58UmRjeKlx9gKAcuKQ==", + "dependencies": { + "@types/node": "^18.11.18", + "@types/node-fetch": "^2.6.4", + "abort-controller": "^3.0.0", + "agentkeepalive": "^4.2.1", + "form-data-encoder": "1.7.2", + "formdata-node": "^4.3.2", + "node-fetch": "^2.6.7", + "web-streams-polyfill": "^3.2.1" + }, + "bin": { + "openai": "bin/cli" } }, - "node_modules/openai/node_modules/axios": { - "version": "0.26.1", - "resolved": "https://registry.npmjs.org/axios/-/axios-0.26.1.tgz", - "integrity": "sha512-fPwcX4EvnSHuInCMItEhAGnaSEXRBjtzh9fOtsE6E1G6p7vl7edEeZe11QHf18+6+9gR5PbKV/sGKNaD8YaMeA==", + "node_modules/openai/node_modules/@types/node": { + "version": "18.19.33", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.33.tgz", + "integrity": "sha512-NR9+KrpSajr2qBVp/Yt5TU/rp+b5Mayi3+OlMlcg2cVCfRmcG5PWZ7S4+MG9PZ5gWBoc9Pd0BKSRViuBCRPu0A==", "dependencies": { - "follow-redirects": "^1.14.8" + "undici-types": "~5.26.4" } }, "node_modules/optionator": { @@ -9570,6 +9659,11 @@ "node": ">=14.0" } }, + "node_modules/undici-types": { + "version": "5.26.5", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", + "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==" + }, "node_modules/unique-string": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/unique-string/-/unique-string-2.0.0.tgz", @@ -9737,6 +9831,14 @@ "makeerror": "1.0.12" } }, + "node_modules/web-streams-polyfill": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", + "integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==", + "engines": { + "node": ">= 8" + } + }, "node_modules/webidl-conversions": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", @@ -11461,7 +11563,6 @@ "version": "2.6.6", "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.6.tgz", "integrity": "sha512-95X8guJYhfqiuVVhRFxVQcf4hW/2bCuoPwDasMf/531STFoNoWTT7YDnWdXHEZKqAGUigmpG31r2FE70LwnzJw==", - "dev": true, "requires": { "@types/node": "*", "form-data": "^4.0.0" @@ -11722,6 +11823,14 @@ "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==" }, + "abort-controller": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", + "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", + "requires": { + "event-target-shim": "^5.0.0" + } + }, "accepts": { "version": "1.3.7", "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.7.tgz", @@ -11758,6 +11867,14 @@ "debug": "4" } }, + "agentkeepalive": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.5.0.tgz", + "integrity": "sha512-5GG/5IbQQpC9FpkRGsSvZI5QYeSCzlJHdpBQntCsuTOxhKD8lqKhrleg2Yi7yvMIf82Ycmmqln9U8V9qwEiJew==", + "requires": { + "humanize-ms": "^1.2.1" + } + }, "ajv": { "version": "6.12.6", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", @@ -13295,6 +13412,11 @@ "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", "integrity": "sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc=" }, + "event-target-shim": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", + "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==" + }, "execa": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/execa/-/execa-5.1.1.tgz", @@ -13584,6 +13706,27 @@ "mime-types": "^2.1.12" } }, + "form-data-encoder": { + "version": "1.7.2", + "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-1.7.2.tgz", + "integrity": "sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==" + }, + "formdata-node": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/formdata-node/-/formdata-node-4.4.1.tgz", + "integrity": "sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==", + "requires": { + "node-domexception": "1.0.0", + "web-streams-polyfill": "4.0.0-beta.3" + }, + "dependencies": { + "web-streams-polyfill": { + "version": "4.0.0-beta.3", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz", + "integrity": "sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==" + } + } + }, "forwarded": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", @@ -13976,6 +14119,14 @@ "resolved": "https://registry.npmjs.org/humanize-duration/-/humanize-duration-3.27.1.tgz", "integrity": "sha512-jCVkMl+EaM80rrMrAPl96SGG4NRac53UyI1o/yAzebDntEY6K6/Fj2HOjdPg8omTqIe5Y0wPBai2q5xXrIbarA==" }, + "humanize-ms": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz", + "integrity": "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==", + "requires": { + "ms": "^2.0.0" + } + }, "iconv-lite": { "version": "0.4.24", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", @@ -15478,6 +15629,11 @@ "clone": "2.x" } }, + "node-domexception": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", + "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==" + }, "node-fetch": { "version": "2.7.0", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", @@ -15695,20 +15851,26 @@ } }, "openai": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/openai/-/openai-3.3.0.tgz", - "integrity": "sha512-uqxI/Au+aPRnsaQRe8CojU0eCR7I0mBiKjD3sNMzY6DaC1ZVrc85u98mtJW6voDug8fgGN+DIZmTDxTthxb7dQ==", - "requires": { - "axios": "^0.26.0", - "form-data": "^4.0.0" + "version": "4.50.0", + "resolved": "https://registry.npmjs.org/openai/-/openai-4.50.0.tgz", + "integrity": "sha512-2ADkNIU6Q589oYHr5pn9k7SbUcrBTK9X0rIXrYqwMVSoqOj1yK9/1OO0ExaWsqOOpD7o58UmRjeKlx9gKAcuKQ==", + "requires": { + "@types/node": "^18.11.18", + "@types/node-fetch": "^2.6.4", + "abort-controller": "^3.0.0", + "agentkeepalive": "^4.2.1", + "form-data-encoder": "1.7.2", + "formdata-node": "^4.3.2", + "node-fetch": "^2.6.7", + "web-streams-polyfill": "^3.2.1" }, "dependencies": { - "axios": { - "version": "0.26.1", - "resolved": "https://registry.npmjs.org/axios/-/axios-0.26.1.tgz", - "integrity": "sha512-fPwcX4EvnSHuInCMItEhAGnaSEXRBjtzh9fOtsE6E1G6p7vl7edEeZe11QHf18+6+9gR5PbKV/sGKNaD8YaMeA==", + "@types/node": { + "version": "18.19.33", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.33.tgz", + "integrity": "sha512-NR9+KrpSajr2qBVp/Yt5TU/rp+b5Mayi3+OlMlcg2cVCfRmcG5PWZ7S4+MG9PZ5gWBoc9Pd0BKSRViuBCRPu0A==", "requires": { - "follow-redirects": "^1.14.8" + "undici-types": "~5.26.4" } } } @@ -17066,6 +17228,11 @@ "busboy": "^1.6.0" } }, + "undici-types": { + "version": "5.26.5", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", + "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==" + }, "unique-string": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/unique-string/-/unique-string-2.0.0.tgz", @@ -17189,6 +17356,11 @@ "makeerror": "1.0.12" } }, + "web-streams-polyfill": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", + "integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==" + }, "webidl-conversions": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", diff --git a/package.json b/package.json index 95f8468..b0094b5 100644 --- a/package.json +++ b/package.json @@ -77,7 +77,7 @@ "node-cache": "^5.1.2", "node-fetch": "^2.7.0", "node-notifier": "^9.0.0", - "openai": "^3.3.0", + "openai": "^4.50.0", "p-limit": "^3.1.0", "pg": "^8.5.1", "play-dl": "^1.9.7", diff --git a/src/api/routes/chatgpt.ts b/src/api/routes/chatgpt.ts index 17c178d..fe0c112 100644 --- a/src/api/routes/chatgpt.ts +++ b/src/api/routes/chatgpt.ts @@ -1,12 +1,11 @@ import express, { Response } from 'express'; -import { ChatCompletionRequestMessage } from 'openai'; import { IntentionalAny } from 'src/types'; import authMiddleware, { AuthRequest } from 'src/api/middlewares/auth'; -import { getChatGptResponse } from 'src/commands/utilities/chatgpt'; +import { getChatGptResponse, ChatMessage } from 'src/commands/utilities/chatgpt'; const router = express.Router(); -function validateConversation(conversation: IntentionalAny[]): conversation is ChatCompletionRequestMessage[] { +function validateConversation(conversation: IntentionalAny[]): conversation is ChatMessage[] { return conversation.every(message => { return typeof message === 'object' && message @@ -48,6 +47,7 @@ async function handleMessage({ // @ts-expect-error router.post('/message', authMiddleware, (req: AuthRequest, res) => { + // TODO: Add queryImage support const { query, conversation } = req.body; handleMessage({ query, diff --git a/src/commands/utilities/chatgpt.ts b/src/commands/utilities/chatgpt.ts index 4346b6b..0054231 100644 --- a/src/commands/utilities/chatgpt.ts +++ b/src/commands/utilities/chatgpt.ts @@ -1,7 +1,7 @@ import type { Command, CommandOrModalRunMethod } from 'src/types'; - +import type { Attachment } from 'discord.js'; import { SlashCommandBuilder } from '@discordjs/builders'; -import { Configuration, OpenAIApi, ChatCompletionRequestMessage } from 'openai'; +import OpenAI from 'openai'; import NodeCache from 'node-cache'; import { @@ -12,11 +12,11 @@ import { } from 'src/discord-utils'; import { ENV_LIMITER_SPLIT_REGEX } from 'src/constants'; +export type ChatMessage = OpenAI.ChatCompletionMessageParam; + const apiKey = process.env.OPENAI_SECRET_KEY; -const configuration = new Configuration({ - apiKey, -}); -const openai = new OpenAIApi(configuration); + +const openai = new OpenAI({ apiKey }); const conversationTimeLimit = process.env.CHATGPT_CONVERSATION_TIME_LIMIT; const conversations = conversationTimeLimit ? new NodeCache({ @@ -40,6 +40,12 @@ commandBuilder.addStringOption(option => { .setDescription('The query (a question for ChatGPT).') .setRequired(true); }); +commandBuilder.addAttachmentOption(option => { + return option + .setName('image') + .setDescription('An image to include with your query.') + .setRequired(false); +}); commandBuilder.addBooleanOption(option => { return option .setName('ephemeral') @@ -49,43 +55,57 @@ commandBuilder.addBooleanOption(option => { export async function getChatGptResponse(options: { query: string, + queryImage?: string, // URL userId: string, guildId?: string | null, - conversation?: ChatCompletionRequestMessage[], + conversation?: ChatMessage[], }): Promise { if (!apiKey) { throw new Error('ChatGPT is not configured on the bot.'); } - const { userId, guildId, query } = options; + const { userId, guildId, query, queryImage } = options; + + if (queryImage && !whiteListedUserIds.has(userId)) { + throw new Error('You are not permitted to submit images.'); + } // This throws an error if rate limited const rateLimiter = whiteListedUserIds.has(userId) ? whiteListedRateLimiter : regularRateLimiter; await rateLimiter.attempt({ userId, guildId }); const conversationKey = userId + guildId; - const conversation = options.conversation ?? conversations?.get(conversationKey) ?? []; - const chatCompletion = await openai.createChatCompletion({ - model: 'gpt-3.5-turbo', - messages: [ - ...conversation, + const conversation = options.conversation ?? conversations?.get(conversationKey) ?? []; + + const newMessage: ChatMessage = { + role: 'user', + content: queryImage ? [ + { type: 'text', text: query }, { - role: 'user', - content: query, + type: 'image_url', + image_url: { + url: queryImage, + detail: 'low', + }, }, + ] : query, + }; + + const chatCompletion = await openai.chat.completions.create({ + model: 'gpt-4o', + messages: [ + ...conversation, + newMessage, ], }); - const responseMessage = chatCompletion.data.choices[0].message; + const responseMessage = chatCompletion.choices[0].message; // Update cached conversation if (responseMessage && conversations && !options.conversation) { - const conversation = conversations.get(userId + guildId) ?? []; + const conversation = conversations.get(userId + guildId) ?? []; conversations.set(conversationKey, [ ...conversation, - { - role: 'user', - content: query, - }, + newMessage, responseMessage, ]); } @@ -99,12 +119,15 @@ const run: CommandOrModalRunMethod = async interaction => { const inputs = await parseInput({ slashCommandData: commandBuilder, interaction }); const query: string = inputs.query; - + const attachment: Attachment | undefined = inputs.image; const userId = interaction.user.id; const guildId = interaction.guildId || ''; + const queryImage = attachment?.url; + const content = await getChatGptResponse({ query, + queryImage, userId, guildId, }); diff --git a/src/discord-utils.ts b/src/discord-utils.ts index 17f8552..1c2b5b9 100644 --- a/src/discord-utils.ts +++ b/src/discord-utils.ts @@ -855,6 +855,10 @@ export async function parseInput({ resolvedInputs[option.name] = options.getUser(option.name); break; } + case ApplicationCommandOptionType.Attachment: { + resolvedInputs[option.name] = options.getAttachment(option.name); + break; + } default: { resolvedInputs[option.name] = option.value; } @@ -879,52 +883,52 @@ export async function parseInput({ return; } switch (option.type) { - case 4: { // Integer + case ApplicationCommandOptionType.Integer: { const int = parseInt(input, 10); if (Number.isNaN(int)) throw new Error(`Could not parse "${input}" to an integer.`); resolvedInputs[option.name] = int; break; } - case 10: { // Number + case ApplicationCommandOptionType.Number: { const num = Number(input); if (Number.isNaN(num)) throw new Error(`Could not parse "${input}" to a number.`); resolvedInputs[option.name] = num; break; } - case 5: { // Boolean + case ApplicationCommandOptionType.Boolean: { if (input) { resolvedInputs[option.name] = getBooleanFromValue(input); } break; } - case 6: { // User + case ApplicationCommandOptionType.User: { if (!interaction.guild || !input) break; const member = await resolveMember(input, interaction); if (!member) throw new Error(`Could not find member based on: ${input}`); resolvedInputs[option.name] = member; break; } - case 9: { // Mentionable + case ApplicationCommandOptionType.Mentionable: { // TODO // We don't currently have a command which uses this, so build this out later. resolvedInputs[option.name] = input; break; } - case 7: { // Channel + case ApplicationCommandOptionType.Channel: { if (!interaction.guild || !input) break; const channel = await resolveChannel(input, interaction); if (!channel) throw new Error(`Could not find channel based on: ${input}`); resolvedInputs[option.name] = channel; break; } - case 8: { // Role + case ApplicationCommandOptionType.Role: { if (!interaction.guild || !input) break; const role = await resolveRole(input, interaction); if (!role) throw new Error(`Could not find role based on: ${input}`); resolvedInputs[option.name] = role; break; } - case 3: // String + case ApplicationCommandOptionType.String: default: { if (!input) break; if (!interaction.guild) { diff --git a/src/events/dms.ts b/src/events/dms.ts index 2b98874..ad87183 100644 --- a/src/events/dms.ts +++ b/src/events/dms.ts @@ -10,6 +10,7 @@ const NewDmEvent: EventTrigger = ['messageCreate', async (message: Message): Pro try { const response = await getChatGptResponse({ query: message.content, + queryImage: message.attachments?.at(0)?.url, userId: message.author.id, guildId: message.guildId, }); From 46f8300f0beacc2efc8282a253f6e1a76a4c0d44 Mon Sep 17 00:00:00 2001 From: Michael Yaworski Date: Tue, 11 Jun 2024 14:51:37 -0400 Subject: [PATCH 2/6] Fixes OpenAI instantiation without API key Conditionally instantiates OpenAI if there is a key since OpenAI throws an error when trying to instantiate without one --- src/commands/utilities/chatgpt.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/commands/utilities/chatgpt.ts b/src/commands/utilities/chatgpt.ts index 0054231..4f07dac 100644 --- a/src/commands/utilities/chatgpt.ts +++ b/src/commands/utilities/chatgpt.ts @@ -15,8 +15,7 @@ import { ENV_LIMITER_SPLIT_REGEX } from 'src/constants'; export type ChatMessage = OpenAI.ChatCompletionMessageParam; const apiKey = process.env.OPENAI_SECRET_KEY; - -const openai = new OpenAI({ apiKey }); +const openai = apiKey ? new OpenAI({ apiKey }) : null; const conversationTimeLimit = process.env.CHATGPT_CONVERSATION_TIME_LIMIT; const conversations = conversationTimeLimit ? new NodeCache({ @@ -60,7 +59,7 @@ export async function getChatGptResponse(options: { guildId?: string | null, conversation?: ChatMessage[], }): Promise { - if (!apiKey) { + if (!openai) { throw new Error('ChatGPT is not configured on the bot.'); } From a3864bfd6555f5edfb6d7104808ad701b2a64608 Mon Sep 17 00:00:00 2001 From: Michael Yaworski Date: Tue, 11 Jun 2024 15:40:34 -0400 Subject: [PATCH 3/6] Nit --- src/commands/utilities/chatgpt.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/commands/utilities/chatgpt.ts b/src/commands/utilities/chatgpt.ts index 4f07dac..e783058 100644 --- a/src/commands/utilities/chatgpt.ts +++ b/src/commands/utilities/chatgpt.ts @@ -1,5 +1,6 @@ import type { Command, CommandOrModalRunMethod } from 'src/types'; import type { Attachment } from 'discord.js'; + import { SlashCommandBuilder } from '@discordjs/builders'; import OpenAI from 'openai'; import NodeCache from 'node-cache'; From 220ffc23ff8bd036a4109d923f63ba4944403238 Mon Sep 17 00:00:00 2001 From: Michael Yaworski Date: Tue, 11 Jun 2024 15:49:35 -0400 Subject: [PATCH 4/6] Adds support for hidden modal args --- src/commands/index.ts | 2 +- src/types/index.ts | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/commands/index.ts b/src/commands/index.ts index 293f732..1f15d04 100644 --- a/src/commands/index.ts +++ b/src/commands/index.ts @@ -171,7 +171,7 @@ export function listenToCommands(): void { .setStyle(option.name === 'message' ? TextInputStyle.Paragraph : TextInputStyle.Short); // Each row can only hold one input const row = new ActionRowBuilder().addComponents(input); - modal.addComponents(row); + if (!command.modalHiddenArgs?.includes(option.name)) modal.addComponents(row); }); try { await interaction.showModal(modal); diff --git a/src/types/index.ts b/src/types/index.ts index fcd88ed..2ce62dd 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -69,6 +69,7 @@ export interface Command { runModal?: (interaction: ModalSubmitInteraction) => Promise, modalLabels?: StringMapping, modalPlaceholders?: StringMapping, + modalHiddenArgs?: string[], showModalWithNoArgs?: boolean, buttonAction?: (interaction: ButtonInteraction) => Promise, guildOnly?: boolean, From ada240c68ee9846aa5144131e068263860159518 Mon Sep 17 00:00:00 2001 From: Michael Yaworski Date: Tue, 11 Jun 2024 15:50:37 -0400 Subject: [PATCH 5/6] [ChatGPT] Removes image arg on modal Improves label for ephemeral field on modal --- src/commands/utilities/chatgpt.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/commands/utilities/chatgpt.ts b/src/commands/utilities/chatgpt.ts index e783058..cfcf8c7 100644 --- a/src/commands/utilities/chatgpt.ts +++ b/src/commands/utilities/chatgpt.ts @@ -142,6 +142,13 @@ const run: CommandOrModalRunMethod = async interaction => { const ChatGptCommand: Command = { guildOnly: false, slashCommandData: commandBuilder, + modalHiddenArgs: ['image'], + modalLabels: { + ephemeral: 'Show only to you? (Defaults to "no")', + }, + modalPlaceholders: { + ephemeral: 'yes/no', + }, runCommand: run, runModal: run, }; From 668e9a625202f5c85eca6475df9d9326aee54780 Mon Sep 17 00:00:00 2001 From: Michael Yaworski Date: Tue, 11 Jun 2024 16:06:10 -0400 Subject: [PATCH 6/6] [ChatGPT] Adds error handling if attachment is not an image --- src/commands/utilities/chatgpt.ts | 2 ++ src/discord-utils.ts | 7 +++++++ src/events/dms.ts | 6 ++++-- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/commands/utilities/chatgpt.ts b/src/commands/utilities/chatgpt.ts index cfcf8c7..3260b56 100644 --- a/src/commands/utilities/chatgpt.ts +++ b/src/commands/utilities/chatgpt.ts @@ -10,6 +10,7 @@ import { getBooleanArg, getRateLimiterFromEnv, parseInput, + throwIfNotImageAttachment, } from 'src/discord-utils'; import { ENV_LIMITER_SPLIT_REGEX } from 'src/constants'; @@ -124,6 +125,7 @@ const run: CommandOrModalRunMethod = async interaction => { const guildId = interaction.guildId || ''; const queryImage = attachment?.url; + throwIfNotImageAttachment(attachment); const content = await getChatGptResponse({ query, diff --git a/src/discord-utils.ts b/src/discord-utils.ts index 1c2b5b9..51eec07 100644 --- a/src/discord-utils.ts +++ b/src/discord-utils.ts @@ -1,4 +1,5 @@ import { + Attachment, Message, User, PermissionResolvable, @@ -1187,3 +1188,9 @@ export function getRateLimiterFromEnv(userKey: string, guildKey: string): RateLi } : undefined, }); } + +export function throwIfNotImageAttachment(attachment: Attachment | undefined | null): void { + if (attachment && !(attachment.contentType && /^image\//.test(attachment.contentType))) { + throw new Error('Attachment must be an image.'); + } +} diff --git a/src/events/dms.ts b/src/events/dms.ts index ad87183..2cedc66 100644 --- a/src/events/dms.ts +++ b/src/events/dms.ts @@ -2,15 +2,17 @@ import type { Message } from 'discord.js'; import type { EventTrigger } from 'src/types'; import { getChatGptResponse } from 'src/commands/utilities/chatgpt'; -import { chunkReplies, getErrorMsg } from 'src/discord-utils'; +import { chunkReplies, getErrorMsg, throwIfNotImageAttachment } from 'src/discord-utils'; const NewDmEvent: EventTrigger = ['messageCreate', async (message: Message): Promise => { if (!message.inGuild() && !message.author.bot) { if (!message.content) return; try { + const attachment = message.attachments?.at(0); + throwIfNotImageAttachment(attachment); const response = await getChatGptResponse({ query: message.content, - queryImage: message.attachments?.at(0)?.url, + queryImage: attachment?.url, userId: message.author.id, guildId: message.guildId, });