Allow using images as input with gpt-4 vision

src/Chat/ChatBase.tsx

@@ -146,7 +146,7 @@ function ChatBase({ chat }: ChatBaseProps) {
 
   // Handle prompt form submission
   const onPrompt = useCallback(
-    async (prompt?: string) => {
+    async (prompt?: string, imageUrls?: string[]) => {
       setLoading(true);
 
       // Special-case for "help", to invoke /help command
@@ -198,7 +198,11 @@ function ChatBase({ chat }: ChatBaseProps) {
         // If the prompt text exist, package it up as a human message and add to the chat
         if (prompt) {
           // Add this prompt message to the chat
-          promptMessage = new ChatCraftHumanMessage({ text: prompt, user });
+          promptMessage = new ChatCraftHumanMessage({ text: prompt, imageUrls, user });
+          await chat.addMessage(promptMessage);
+        } else if (imageUrls?.length) {
+          // Add only image to the chat
+          promptMessage = new ChatCraftHumanMessage({ text: "", imageUrls, user });
           await chat.addMessage(promptMessage);
         } else {
           // If there isn't any prompt text, see if the final message in the chat was a human

src/components/ImageModal.tsx

@@ -0,0 +1,40 @@
+import React from "react";
+import {
+  Modal,
+  ModalOverlay,
+  ModalContent,
+  ModalCloseButton,
+  ModalBody,
+  Image,
+  Flex,
+} from "@chakra-ui/react";
+
+interface ImageModalProps {
+  isOpen: boolean;
+  onClose: () => void;
+  imageSrc: string;
+}
+
+const ImageModal: React.FC<ImageModalProps> = ({ isOpen, onClose, imageSrc }) => (
+  <Modal isOpen={isOpen} onClose={onClose} size="2xl" isCentered>
+    <ModalOverlay />
+    <ModalContent maxW="90vw" maxHeight="90vh">
+      <ModalCloseButton />
+      <ModalBody>
+        <Flex height={"100%"} justifyContent={"center"} alignItems={"center"}>
+          <Image
+            maxWidth="100%"
+            maxHeight="70vh"
+            overflow={"auto"}
+            src={imageSrc}
+            alt="Selected Image"
+            m="auto"
+            objectFit="contain"
+          />
+        </Flex>
+      </ModalBody>
+    </ModalContent>
+  </Modal>
+);
+
+export default ImageModal;

src/components/Message/MessageBase.tsx

@@ -21,6 +21,7 @@ import {
   Flex,
   Heading,
   IconButton,
+  Image,
   Link,
   Tag,
   Text,
@@ -53,6 +54,7 @@ import { ChatCraftModel } from "../../lib/ChatCraftModel";
 import { useModels } from "../../hooks/use-models";
 import { useSettings } from "../../hooks/use-settings";
 import { useAlert } from "../../hooks/use-alert";
+import ImageModal from "../ImageModal";
 
 // Styles for the message text are defined in CSS vs. Chakra-UI
 import "./Message.css";
@@ -103,7 +105,7 @@ function MessageBase({
   disableFork,
   disableEdit,
 }: MessageBaseProps) {
-  const { id, date, text } = message;
+  const { id, date, text, imageUrls } = message;
   const { models } = useModels();
   const { onCopy } = useClipboard(text);
   const { info, error } = useAlert();
@@ -114,6 +116,8 @@ function MessageBase({
   const messageForm = useRef<HTMLFormElement>(null);
   const messageContent = useRef<HTMLDivElement>(null);
   const meta = useMemo(getMetaKey, []);
+  const [imageModalOpen, setImageModalOpen] = useState<boolean>(false);
+  const [selectedImage, setSelectedImage] = useState<string>("");
   const { isOpen, onToggle: originalOnToggle } = useDisclosure();
   const isLongMessage = text.length > 5000;
   const displaySummaryText = !isOpen && (summaryText || isLongMessage);
@@ -269,6 +273,12 @@ function MessageBase({
     [message, onResubmitClick, chatId, error, onEditingChange]
   );
 
+  const openModalWithImage = (imageSrc: string) => {
+    setSelectedImage(imageSrc);
+    setImageModalOpen(true);
+  };
+  const closeModal = () => setImageModalOpen(false);
+
   return (
     <Box
       id={id}
@@ -481,6 +491,17 @@ function MessageBase({
                   // Add a single pixel of offset for rendering to canvas (offset handled above with m=-1)
                   p={1}
                 >
+                  {imageUrls.map((imageUrl, index) => (
+                    <Box key={`${id}-${index}`}>
+                      <Image
+                        src={imageUrl}
+                        alt={`Images# ${index}`}
+                        margin={"auto"}
+                        maxWidth={"100%"}
+                        onClick={() => openModalWithImage(imageUrl)}
+                      />
+                    </Box>
+                  ))}
                   <Markdown
                     previewCode={!hidePreviews && !displaySummaryText}
                     isLoading={isLoading}
@@ -497,6 +518,7 @@ function MessageBase({
                 </Box>
               )}
             </Box>
+            <ImageModal isOpen={imageModalOpen} onClose={closeModal} imageSrc={selectedImage} />
           </Flex>
         </CardBody>
         {footer && <CardFooter py={2}>{footer}</CardFooter>}

src/components/PromptForm/AttachFileButton.tsx

@@ -0,0 +1,61 @@
+import { useState, useRef } from "react";
+import { IconButton, Input } from "@chakra-ui/react";
+import { BsPaperclip } from "react-icons/bs";
+
+import useMobileBreakpoint from "../../hooks/use-mobile-breakpoint";
+
+type AttachProps = {
+  isDisabled: boolean;
+  onFileSelected: (base64: string) => void;
+};
+
+export default function AttachFileButton({ isDisabled = false, onFileSelected }: AttachProps) {
+  const isMobile = useMobileBreakpoint();
+  const [colorScheme] = useState<"blue" | "red">("blue");
+  const clipIconRef = useRef<HTMLButtonElement | null>(null);
+  const fileInputRef = useRef<HTMLInputElement>(null);
+
+  const handleFileChange = (event: React.ChangeEvent<HTMLInputElement>) => {
+    const files = event.target.files;
+    if (files) {
+      for (let i = 0; i < files.length; i++) {
+        const reader = new FileReader();
+        reader.onload = (e) => {
+          onFileSelected(e.target?.result as string);
+        };
+        reader.readAsDataURL(files[i]);
+      }
+      // Reset the input value after file read
+      event.target.value = "";
+    }
+  };
+
+  const handleClick = () => {
+    fileInputRef.current?.click();
+  };
+
+  return (
+    <>
+      <Input
+        multiple
+        type="file"
+        ref={fileInputRef}
+        hidden
+        onChange={handleFileChange}
+        accept="image/*"
+      />
+      <IconButton
+        onClick={handleClick}
+        isRound
+        isDisabled={isDisabled}
+        colorScheme={colorScheme}
+        variant={isMobile ? "outline" : "ghost"}
+        icon={<BsPaperclip />}
+        aria-label="Attach file"
+        size={isMobile ? "lg" : "md"}
+        fontSize="18px"
+        ref={clipIconRef}
+      />
+    </>
+  );
+}