From 646fd074709693c674e4b99c2153e95b2e0b5ae8 Mon Sep 17 00:00:00 2001 From: Tony Giorgio Date: Fri, 12 Sep 2025 18:44:08 -0500 Subject: [PATCH 1/6] feat: add TTS capabilities to chat messages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Added speaker icon button next to copy button for assistant messages - Integrated OpenAI TTS with kokoro model and af_sky+af_bella voice - Audio plays directly in browser without downloading - Click speaker icon to play/stop audio - Made copy and TTS buttons always visible (removed hover-only display) - Fixed button hover boundaries to prevent overlap 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude Bump 1.4.1 feat: add microphone recording with Whisper transcription - Add RecordRTC library for proper WAV audio recording - Implement mic button with visual states (orange when recording) - Configure audio with echo cancellation and noise suppression - Optimize for speech with 16kHz mono WAV format - Integrate with OpenSecret SDK's transcribeAudio API - Append transcribed text to message input field 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- frontend/bun.lock | 10 +- frontend/package.json | 4 +- frontend/src/components/ChatBox.tsx | 106 +++++++++++++++++++- frontend/src/routes/_auth.chat.$chatId.tsx | 108 ++++++++++++++++++--- 4 files changed, 213 insertions(+), 15 deletions(-) diff --git a/frontend/bun.lock b/frontend/bun.lock index 3ab60275..c6f3e148 100644 --- a/frontend/bun.lock +++ b/frontend/bun.lock @@ -4,7 +4,7 @@ "": { "name": "maple", "dependencies": { - "@opensecret/react": "1.4.0", + "@opensecret/react": "1.4.3", "@radix-ui/react-alert-dialog": "^1.1.1", "@radix-ui/react-avatar": "^1.1.0", "@radix-ui/react-dialog": "^1.1.1", @@ -30,6 +30,7 @@ "react": "^18.3.1", "react-dom": "^18.3.1", "react-markdown": "^9.0.1", + "recordrtc": "^5.6.2", "rehype-highlight": "^7.0.0", "rehype-katex": "^7.0.1", "rehype-sanitize": "^6.0.0", @@ -51,6 +52,7 @@ "@types/node": "^22.3.0", "@types/react": "^18.3.3", "@types/react-dom": "^18.3.0", + "@types/recordrtc": "^5.6.14", "@types/uuid": "^10.0.0", "@vitejs/plugin-react": "^4.3.1", "autoprefixer": "^10.4.20", @@ -217,7 +219,7 @@ "@nodelib/fs.walk": ["@nodelib/fs.walk@1.2.8", "", { "dependencies": { "@nodelib/fs.scandir": "2.1.5", "fastq": "^1.6.0" } }, "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg=="], - "@opensecret/react": ["@opensecret/react@1.4.0", "", { "dependencies": { "@peculiar/x509": "^1.12.2", "@stablelib/base64": "^2.0.0", "@stablelib/chacha20poly1305": "^2.0.0", "@stablelib/random": "^2.0.0", "cbor2": "^1.7.0", "tweetnacl": "^1.0.3", "zod": "^3.23.8" }, "peerDependencies": { "react": "^18.0.0 || ^19.0.0", "react-dom": "^18.0.0 || ^19.0.0" } }, "sha512-21L4V1AWoKTzcMKwe4OrM+sj3BKl5ATMODdFwsh+wdRvmYrG/OXf9AXmO7cP9LfsN7Gb1nj3fTmnwU8Gbx//Kw=="], + "@opensecret/react": ["@opensecret/react@1.4.3", "", { "dependencies": { "@peculiar/x509": "^1.12.2", "@stablelib/base64": "^2.0.0", "@stablelib/chacha20poly1305": "^2.0.0", "@stablelib/random": "^2.0.0", "cbor2": "^1.7.0", "tweetnacl": "^1.0.3", "zod": "^3.23.8" }, "peerDependencies": { "react": "^18.0.0 || ^19.0.0", "react-dom": "^18.0.0 || ^19.0.0" } }, "sha512-lsBsPRM9tsY9C8y7hHxLe8MOKlvNVI2B3X0XLWUqn/Prm51iAl5anWsRbEhKBvvNvWjlW/gfgHBfx+i2B0tvAw=="], "@peculiar/asn1-cms": ["@peculiar/asn1-cms@2.3.15", "", { "dependencies": { "@peculiar/asn1-schema": "^2.3.15", "@peculiar/asn1-x509": "^2.3.15", "@peculiar/asn1-x509-attr": "^2.3.15", "asn1js": "^3.0.5", "tslib": "^2.8.1" } }, "sha512-B+DoudF+TCrxoJSTjjcY8Mmu+lbv8e7pXGWrhNp2/EGJp9EEcpzjBCar7puU57sGifyzaRVM03oD5L7t7PghQg=="], @@ -469,6 +471,8 @@ "@types/react-dom": ["@types/react-dom@18.3.5", "", { "peerDependencies": { "@types/react": "^18.0.0" } }, "sha512-P4t6saawp+b/dFrUr2cvkVsfvPguwsxtH6dNIYRllMsefqFzkZk5UIjzyDOv5g1dXIPdG4Sp1yCR4Z6RCUsG/Q=="], + "@types/recordrtc": ["@types/recordrtc@5.6.14", "", {}, "sha512-Reiy1sl11xP0r6w8DW3iQjc1BgXFyNC7aDuutysIjpFoqyftbQps9xPA2FoBkfVXpJM61betgYPNt+v65zvMhA=="], + "@types/unist": ["@types/unist@3.0.3", "", {}, "sha512-ko/gIFJRv177XgZsZcBwnqJN5x/Gien8qNOn0D5bQU/zAzVf9Zt3BlcUiLqhV9y4ARk0GbT3tnUiPNgnTXzc/Q=="], "@types/uuid": ["@types/uuid@10.0.0", "", {}, "sha512-7gqG38EyHgyP1S+7+xomFtL+ZNHcKv6DwNaCZmJmo1vgMugyF3TCnXVg4t1uk89mLNwnLtnY3TpOpCOyp1/xHQ=="], @@ -1037,6 +1041,8 @@ "readdirp": ["readdirp@3.6.0", "", { "dependencies": { "picomatch": "^2.2.1" } }, "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA=="], + "recordrtc": ["recordrtc@5.6.2", "", {}, "sha512-1QNKKNtl7+KcwD1lyOgP3ZlbiJ1d0HtXnypUy7yq49xEERxk31PHvE9RCciDrulPCY7WJ+oz0R9hpNxgsIurGQ=="], + "reflect-metadata": ["reflect-metadata@0.2.2", "", {}, "sha512-urBwgfrvVP/eAyXx4hluJivBKzuEbSQs9rKWCrCkbSxNv8mxPcUZKeuoF3Uy4mJl3Lwprp6yy5/39VWigZ4K6Q=="], "rehype-highlight": ["rehype-highlight@7.0.2", "", { "dependencies": { "@types/hast": "^3.0.0", "hast-util-to-text": "^4.0.0", "lowlight": "^3.0.0", "unist-util-visit": "^5.0.0", "vfile": "^6.0.0" } }, "sha512-k158pK7wdC2qL3M5NcZROZ2tR/l7zOzjxXd5VGdcfIyoijjQqpHd3JKtYSBDpDZ38UI2WJWuFAtkMDxmx5kstA=="], diff --git a/frontend/package.json b/frontend/package.json index b2d1880c..1a687360 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -16,7 +16,7 @@ "mdast-util-gfm-autolink-literal": "2.0.0" }, "dependencies": { - "@opensecret/react": "1.4.0", + "@opensecret/react": "1.4.3", "@radix-ui/react-alert-dialog": "^1.1.1", "@radix-ui/react-avatar": "^1.1.0", "@radix-ui/react-dialog": "^1.1.1", @@ -42,6 +42,7 @@ "react": "^18.3.1", "react-dom": "^18.3.1", "react-markdown": "^9.0.1", + "recordrtc": "^5.6.2", "rehype-highlight": "^7.0.0", "rehype-katex": "^7.0.1", "rehype-sanitize": "^6.0.0", @@ -63,6 +64,7 @@ "@types/node": "^22.3.0", "@types/react": "^18.3.3", "@types/react-dom": "^18.3.0", + "@types/recordrtc": "^5.6.14", "@types/uuid": "^10.0.0", "@vitejs/plugin-react": "^4.3.1", "autoprefixer": "^10.4.20", diff --git a/frontend/src/components/ChatBox.tsx b/frontend/src/components/ChatBox.tsx index aa1eb17a..e4f4ff5e 100644 --- a/frontend/src/components/ChatBox.tsx +++ b/frontend/src/components/ChatBox.tsx @@ -1,4 +1,5 @@ -import { CornerRightUp, Bot, Image, X, FileText, Loader2, Plus } from "lucide-react"; +import { CornerRightUp, Bot, Image, X, FileText, Loader2, Plus, Mic } from "lucide-react"; +import RecordRTC from "recordrtc"; import { Button } from "@/components/ui/button"; import { Label } from "@/components/ui/label"; import { @@ -244,6 +245,12 @@ export default function Component({ const os = useOpenSecret(); const navigate = useNavigate(); + // Audio recording state + const [isRecording, setIsRecording] = useState(false); + const [isTranscribing, setIsTranscribing] = useState(false); + const recorderRef = useRef(null); + const streamRef = useRef(null); + // Find the first vision-capable model the user has access to const findFirstVisionModel = () => { // Check if user has Pro/Team access @@ -471,6 +478,83 @@ export default function Component({ setUploadedDocument(null); setDocumentError(null); }; + + // Audio recording functions + const startRecording = async () => { + try { + const stream = await navigator.mediaDevices.getUserMedia({ + audio: { + echoCancellation: true, + noiseSuppression: true, + sampleRate: 44100 + } + }); + + streamRef.current = stream; + + // Create RecordRTC instance configured for WAV + const recorder = new RecordRTC(stream, { + type: "audio", + mimeType: "audio/wav", + recorderType: RecordRTC.StereoAudioRecorder, + numberOfAudioChannels: 1, // Mono audio for smaller file size + desiredSampRate: 16000, // 16kHz is good for speech + timeSlice: 1000 // Get data every second (optional) + }); + + recorderRef.current = recorder; + recorder.startRecording(); + setIsRecording(true); + } catch (error) { + console.error("Failed to start recording:", error); + alert("Failed to access microphone. Please check your permissions."); + } + }; + + const stopRecording = () => { + if (recorderRef.current && isRecording) { + recorderRef.current.stopRecording(async () => { + const blob = recorderRef.current!.getBlob(); + + // Create a proper WAV file + const audioFile = new File([blob], "recording.wav", { + type: "audio/wav" + }); + + setIsTranscribing(true); + try { + const result = await os.transcribeAudio(audioFile, "whisper-large-v3"); + + // Append transcribed text to existing input + setInputValue((prev) => { + const newValue = prev ? `${prev} ${result.text}` : result.text; + return newValue; + }); + } catch (error) { + console.error("Transcription failed:", error); + } finally { + setIsTranscribing(false); + } + + // Clean up + if (streamRef.current) { + streamRef.current.getTracks().forEach((track) => track.stop()); + streamRef.current = null; + } + recorderRef.current = null; + }); + + setIsRecording(false); + } + }; + + const toggleRecording = () => { + if (isRecording) { + stopRecording(); + } else { + startRecording(); + } + }; const [isFocused, setIsFocused] = useState(false); const inputRef = useRef(null); const systemPromptRef = useRef(null); @@ -901,6 +985,26 @@ export default function Component({ className="hidden" /> + {/* Microphone button */} + + {/* Consolidated upload button - show for all users */} {!uploadedDocument && ( diff --git a/frontend/src/routes/_auth.chat.$chatId.tsx b/frontend/src/routes/_auth.chat.$chatId.tsx index 582bb11f..3977d6e6 100644 --- a/frontend/src/routes/_auth.chat.$chatId.tsx +++ b/frontend/src/routes/_auth.chat.$chatId.tsx @@ -1,6 +1,16 @@ import { useEffect, useRef, useState, useCallback } from "react"; import { createFileRoute } from "@tanstack/react-router"; -import { AsteriskIcon, Check, Copy, UserIcon, ChevronDown, Bot, SquarePenIcon } from "lucide-react"; +import { + AsteriskIcon, + Check, + Copy, + UserIcon, + ChevronDown, + Bot, + SquarePenIcon, + Volume2, + Square +} from "lucide-react"; import ChatBox from "@/components/ChatBox"; import { useOpenAI } from "@/ai/useOpenAi"; import { useLocalState } from "@/state/useLocalState"; @@ -71,6 +81,71 @@ function SystemMessage({ }) { const textWithoutThinking = stripThinkingTags(text); const { isCopied, handleCopy } = useCopyToClipboard(textWithoutThinking); + const [isPlaying, setIsPlaying] = useState(false); + const audioRef = useRef(null); + const openai = useOpenAI(); + + const handleTTS = useCallback(async () => { + if (isPlaying) { + // Stop playing + if (audioRef.current) { + audioRef.current.pause(); + audioRef.current = null; + } + setIsPlaying(false); + return; + } + + try { + setIsPlaying(true); + + // Generate speech using OpenAI TTS + const response = await openai.audio.speech.create({ + // eslint-disable-next-line @typescript-eslint/no-explicit-any + model: "kokoro" as any, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + voice: "af_sky+af_bella" as any, + input: textWithoutThinking, + response_format: "mp3" + }); + + // Convert response to blob and create audio URL + const blob = new Blob([await response.arrayBuffer()], { type: "audio/mp3" }); + const audioUrl = URL.createObjectURL(blob); + + // Create and play audio + const audio = new Audio(audioUrl); + audioRef.current = audio; + + audio.onended = () => { + setIsPlaying(false); + URL.revokeObjectURL(audioUrl); + audioRef.current = null; + }; + + audio.onerror = () => { + console.error("Error playing audio"); + setIsPlaying(false); + URL.revokeObjectURL(audioUrl); + audioRef.current = null; + }; + + await audio.play(); + } catch (error) { + console.error("TTS error:", error); + setIsPlaying(false); + } + }, [textWithoutThinking, isPlaying, openai]); + + // Cleanup on unmount + useEffect(() => { + return () => { + if (audioRef.current) { + audioRef.current.pause(); + audioRef.current = null; + } + }; + }, []); return (
@@ -80,15 +155,26 @@ function SystemMessage({
- +
+ + +
@@ -145,7 +231,7 @@ function SystemPromptMessage({ text }: { text: string }) { + {/* Microphone button - only show if whisper model is available */} + {hasWhisperModel && ( + + )} {/* Consolidated upload button - show for all users */} {!uploadedDocument && ( diff --git a/frontend/src/components/ModelSelector.tsx b/frontend/src/components/ModelSelector.tsx index 4d3843e6..9e58644e 100644 --- a/frontend/src/components/ModelSelector.tsx +++ b/frontend/src/components/ModelSelector.tsx @@ -28,7 +28,7 @@ export const MODEL_CONFIG: Record = { displayName: "Llama 3.3 70B", tokenLimit: 70000 }, - "llama3-3-70b": { + "llama-3.3-70b": { displayName: "Llama 3.3 70B", tokenLimit: 70000 }, @@ -89,7 +89,14 @@ export function ModelSelector({ messages?: ChatMessage[]; draftImages?: File[]; }) { - const { model, setModel, availableModels, setAvailableModels, billingStatus } = useLocalState(); + const { + model, + setModel, + availableModels, + setAvailableModels, + billingStatus, + setHasWhisperModel + } = useLocalState(); const os = useOpenSecret(); const navigate = useNavigate(); const isFetching = useRef(false); @@ -116,6 +123,10 @@ export function ModelSelector({ isFetching.current = true; os.fetchModels() .then((models) => { + // Check if whisper-large-v3 is available before filtering + const hasWhisper = models.some((model) => model.id === "whisper-large-v3"); + setHasWhisperModel(hasWhisper); + // Filter out embedding models and "latest" interface ModelWithTasks extends Model { tasks?: string[]; @@ -123,6 +134,16 @@ export function ModelSelector({ const filteredModels = models.filter((model) => { if (model.id === "latest") return false; + // Filter out whisper models (transcription) + if (model.id.toLowerCase().includes("whisper")) { + return false; + } + + // Filter out qwen3-coder-30b-a3b + if (model.id === "qwen3-coder-30b-a3b") { + return false; + } + // Filter out models with lowercase 'instruct' or 'embed' in their ID if (model.id.includes("instruct") || model.id.includes("embed")) { return false; @@ -140,11 +161,6 @@ export function ModelSelector({ return false; } - // Filter out transcription models like Whisper - if (modelWithTasks.tasks.includes("transcribe")) { - return false; - } - return true; }); @@ -167,7 +183,7 @@ export function ModelSelector({ isFetching.current = false; }); } - }, [os, setAvailableModels]); + }, [os, setAvailableModels, setHasWhisperModel]); // Check if user has access to a model based on their plan const hasAccessToModel = (modelId: string) => { diff --git a/frontend/src/components/apikeys/ProxyConfigSection.tsx b/frontend/src/components/apikeys/ProxyConfigSection.tsx index 7b91b61e..02d7718c 100644 --- a/frontend/src/components/apikeys/ProxyConfigSection.tsx +++ b/frontend/src/components/apikeys/ProxyConfigSection.tsx @@ -343,7 +343,7 @@ client = OpenAI( ) response = client.chat.completions.create( - model="llama3-3-70b", + model="llama-3.3-70b", messages=[{"role": "user", "content": "Hello!"}], stream=True ) @@ -361,7 +361,7 @@ for chunk in response: {`curl -N http://${config.host}:${config.port}/v1/chat/completions \\ -H "Content-Type: application/json" \\ -d '{ - "model": "llama3-3-70b", + "model": "llama-3.3-70b", "messages": [{"role": "user", "content": "Hello!"}], "stream": true }'`} diff --git a/frontend/src/state/LocalStateContext.tsx b/frontend/src/state/LocalStateContext.tsx index 9d92bacd..41f35fa7 100644 --- a/frontend/src/state/LocalStateContext.tsx +++ b/frontend/src/state/LocalStateContext.tsx @@ -12,7 +12,7 @@ export { type LocalState } from "./LocalStateContextDef"; -export const DEFAULT_MODEL_ID = "llama3-3-70b"; +export const DEFAULT_MODEL_ID = "llama-3.3-70b"; export const LocalStateProvider = ({ children }: { children: React.ReactNode }) => { /** The model that should be assumed when a chat doesn't yet have one */ @@ -39,6 +39,7 @@ export const LocalStateProvider = ({ children }: { children: React.ReactNode }) } })(), availableModels: [llamaModel] as OpenSecretModel[], + hasWhisperModel: true, // Default to true to avoid hiding button during loading billingStatus: null as BillingStatus | null, searchQuery: "", isSearchVisible: false, @@ -287,6 +288,10 @@ export const LocalStateProvider = ({ children }: { children: React.ReactNode }) setLocalState((prev) => ({ ...prev, availableModels: models })); } + function setHasWhisperModel(hasWhisper: boolean) { + setLocalState((prev) => ({ ...prev, hasWhisperModel: hasWhisper })); + } + return ( void; setAvailableModels: (models: OpenSecretModel[]) => void; + /** Whether the whisper transcription model is available */ + hasWhisperModel: boolean; + setHasWhisperModel: (hasWhisper: boolean) => void; userPrompt: string; systemPrompt: string | null; userImages: File[]; @@ -77,6 +80,8 @@ export const LocalStateContext = createContext({ availableModels: [], setModel: () => void 0, setAvailableModels: () => void 0, + hasWhisperModel: true, + setHasWhisperModel: () => void 0, userPrompt: "", systemPrompt: null, userImages: [], diff --git a/frontend/src/utils/utils.ts b/frontend/src/utils/utils.ts index bcba8028..3ff75e0c 100644 --- a/frontend/src/utils/utils.ts +++ b/frontend/src/utils/utils.ts @@ -81,9 +81,12 @@ export function useClickOutside( export function aliasModelName(modelName: string | undefined): string { if (!modelName) return ""; - // Map old complicated model name to new simplified name - if (modelName === "ibnzterrell/Meta-Llama-3.3-70B-Instruct-AWQ-INT4") { - return "llama3-3-70b"; + // Map old model names to new simplified name + if ( + modelName === "ibnzterrell/Meta-Llama-3.3-70B-Instruct-AWQ-INT4" || + modelName === "llama3-3-70b" + ) { + return "llama-3.3-70b"; } return modelName; From 6e8a78771ab57098dce2b1c1e583b9a750124497 Mon Sep 17 00:00:00 2001 From: Tony Giorgio Date: Sat, 13 Sep 2025 16:39:17 -0500 Subject: [PATCH 3/6] feat: add premium feature gating with upgrade prompts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Remove Plus dropdown button, add separate Image and Mic buttons - Add feature gating for Image (Starter+), Voice (Pro+), and TTS (Pro+) - Create reusable UpgradePromptDialog for all premium features - Add upgrade prompts for locked models in ModelSelector - Simplify pricing config with consolidated model listings - Emphasize privacy/encryption in all upgrade messaging - Grey out restricted features with 50% opacity for free users 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- frontend/src/components/ChatBox.tsx | 150 ++++++------- frontend/src/components/ModelSelector.tsx | 201 +++++++++--------- .../src/components/UpgradePromptDialog.tsx | 139 ++++++++++++ frontend/src/config/pricingConfig.tsx | 85 ++------ frontend/src/routes/_auth.chat.$chatId.tsx | 42 +++- 5 files changed, 359 insertions(+), 258 deletions(-) create mode 100644 frontend/src/components/UpgradePromptDialog.tsx diff --git a/frontend/src/components/ChatBox.tsx b/frontend/src/components/ChatBox.tsx index 161c3a27..3b0b38da 100644 --- a/frontend/src/components/ChatBox.tsx +++ b/frontend/src/components/ChatBox.tsx @@ -1,13 +1,8 @@ -import { CornerRightUp, Bot, Image, X, FileText, Loader2, Plus, Mic } from "lucide-react"; +import { CornerRightUp, Bot, Image, X, FileText, Loader2, Mic } from "lucide-react"; import RecordRTC from "recordrtc"; import { Button } from "@/components/ui/button"; import { Label } from "@/components/ui/label"; -import { - DropdownMenu, - DropdownMenuContent, - DropdownMenuItem, - DropdownMenuTrigger -} from "@/components/ui/dropdown-menu"; +import { UpgradePromptDialog } from "@/components/UpgradePromptDialog"; import { useEffect, useRef, useState, useMemo } from "react"; import { useLocalState } from "@/state/useLocalState"; import { cn, useIsMobile } from "@/utils/utils"; @@ -243,8 +238,9 @@ export default function Component({ const [imageError, setImageError] = useState(null); const fileInputRef = useRef(null); const documentInputRef = useRef(null); + const [upgradeDialogOpen, setUpgradeDialogOpen] = useState(false); + const [upgradeFeature, setUpgradeFeature] = useState<"image" | "voice">("image"); const os = useOpenSecret(); - const navigate = useNavigate(); // Audio recording state const [isRecording, setIsRecording] = useState(false); @@ -596,7 +592,16 @@ export default function Component({ freshBillingStatus.product_name?.toLowerCase().includes("max") || freshBillingStatus.product_name?.toLowerCase().includes("team")); - const canUseDocuments = hasProTeamAccess; + // Check if user has access to Starter features (Starter plan and above) + const hasStarterAccess = + freshBillingStatus && + (freshBillingStatus.product_name?.toLowerCase().includes("starter") || + freshBillingStatus.product_name?.toLowerCase().includes("pro") || + freshBillingStatus.product_name?.toLowerCase().includes("max") || + freshBillingStatus.product_name?.toLowerCase().includes("team")); + + const canUseImages = hasStarterAccess; + const canUseVoice = hasProTeamAccess; const handleSubmit = (e?: React.FormEvent) => { e?.preventDefault(); @@ -986,14 +991,51 @@ export default function Component({ className="hidden" /> - {/* Microphone button - only show if whisper model is available */} + {/* Image upload button - show for all users */} + {!uploadedDocument && ( + + )} + + {/* Microphone button - show if whisper model is available */} {hasWhisperModel && ( )} - {/* Consolidated upload button - show for all users */} - {!uploadedDocument && ( - - - - - - { - if (!hasProTeamAccess) { - navigate({ to: "/pricing" }); - } else { - // If not on a vision model, switch to one first - if (!supportsVision) { - const visionModelId = findFirstVisionModel(); - if (visionModelId) { - setModel(visionModelId); - } - } - fileInputRef.current?.click(); - } - }} - className={cn( - "flex items-center gap-2 group", - !hasProTeamAccess && "hover:bg-purple-50 dark:hover:bg-purple-950/20" - )} - > - - Upload Images - {!hasProTeamAccess && ( - <> - - Pro - - - Upgrade? - - - )} - - {/* Document upload temporarily removed - will be re-added later - { - e.preventDefault(); - // Temporarily disabled - remove this condition when re-enabling - // if (!canUseDocuments) { - // navigate({ to: "/pricing" }); - // } else { - // documentInputRef.current?.click(); - // } - }} - className={cn("flex items-center gap-2 cursor-not-allowed opacity-50")} - disabled - > - -
- Upload Document - Temporarily Unavailable -
- - */} - - - )} - - - - {availableModels && - Array.isArray(availableModels) && - // Sort models: vision-capable first (if images present), then available, then restricted, then disabled - [...availableModels] - .sort((a, b) => { - const aConfig = MODEL_CONFIG[a.id]; - const bConfig = MODEL_CONFIG[b.id]; - - // If chat has images, prioritize vision models - if (chatHasImages) { - const aHasVision = aConfig?.supportsVision || false; - const bHasVision = bConfig?.supportsVision || false; - if (aHasVision && !bHasVision) return -1; - if (!aHasVision && bHasVision) return 1; - } - - // Unknown models are treated as disabled - const aDisabled = aConfig?.disabled || !aConfig; - const bDisabled = bConfig?.disabled || !bConfig; - const aRestricted = - (aConfig?.requiresPro || aConfig?.requiresStarter || false) && - !hasAccessToModel(a.id); - const bRestricted = - (bConfig?.requiresPro || bConfig?.requiresStarter || false) && - !hasAccessToModel(b.id); - - // Disabled models go last - if (aDisabled && !bDisabled) return 1; - if (!aDisabled && bDisabled) return -1; - - // Restricted models go after available but before disabled - if (aRestricted && !bRestricted) return 1; - if (!aRestricted && bRestricted) return -1; - - return 0; - }) - .map((availableModel) => { - const config = MODEL_CONFIG[availableModel.id]; - // Unknown models are treated as disabled - const isDisabled = config?.disabled || !config; - const requiresPro = config?.requiresPro || false; - const requiresStarter = config?.requiresStarter || false; - const hasAccess = hasAccessToModel(availableModel.id); - const isRestricted = (requiresPro || requiresStarter) && !hasAccess; - - // Disable non-vision models if chat has images - const isDisabledDueToImages = chatHasImages && !config?.supportsVision; - const effectivelyDisabled = isDisabled || isDisabledDueToImages; - - return ( - { - if (effectivelyDisabled) return; - if (isRestricted) { - // Navigate to pricing page for upgrade - navigate({ to: "/pricing" }); - } else { - setModel(availableModel.id); - } - }} - className={`flex items-center justify-between group ${ - effectivelyDisabled ? "opacity-50 cursor-not-allowed" : "" - } ${isRestricted ? "hover:bg-purple-50 dark:hover:bg-purple-950/20" : ""}`} - disabled={effectivelyDisabled} - > -
-
{getDisplayName(availableModel.id, true)}
- {isRestricted && !isDisabledDueToImages && ( - - Upgrade? - - )} -
- {model === availableModel.id && } - - ); - })} - - + <> + + + + + + {availableModels && + Array.isArray(availableModels) && + // Sort models: vision-capable first (if images present), then available, then restricted, then disabled + [...availableModels] + .sort((a, b) => { + const aConfig = MODEL_CONFIG[a.id]; + const bConfig = MODEL_CONFIG[b.id]; + + // If chat has images, prioritize vision models + if (chatHasImages) { + const aHasVision = aConfig?.supportsVision || false; + const bHasVision = bConfig?.supportsVision || false; + if (aHasVision && !bHasVision) return -1; + if (!aHasVision && bHasVision) return 1; + } + + // Unknown models are treated as disabled + const aDisabled = aConfig?.disabled || !aConfig; + const bDisabled = bConfig?.disabled || !bConfig; + const aRestricted = + (aConfig?.requiresPro || aConfig?.requiresStarter || false) && + !hasAccessToModel(a.id); + const bRestricted = + (bConfig?.requiresPro || bConfig?.requiresStarter || false) && + !hasAccessToModel(b.id); + + // Disabled models go last + if (aDisabled && !bDisabled) return 1; + if (!aDisabled && bDisabled) return -1; + + // Restricted models go after available but before disabled + if (aRestricted && !bRestricted) return 1; + if (!aRestricted && bRestricted) return -1; + + return 0; + }) + .map((availableModel) => { + const config = MODEL_CONFIG[availableModel.id]; + // Unknown models are treated as disabled + const isDisabled = config?.disabled || !config; + const requiresPro = config?.requiresPro || false; + const requiresStarter = config?.requiresStarter || false; + const hasAccess = hasAccessToModel(availableModel.id); + const isRestricted = (requiresPro || requiresStarter) && !hasAccess; + + // Disable non-vision models if chat has images + const isDisabledDueToImages = chatHasImages && !config?.supportsVision; + const effectivelyDisabled = isDisabled || isDisabledDueToImages; + + return ( + { + if (effectivelyDisabled) return; + if (isRestricted) { + // Show upgrade dialog for restricted model + const modelConfig = MODEL_CONFIG[availableModel.id]; + setSelectedModelName(modelConfig?.displayName || availableModel.id); + setUpgradeDialogOpen(true); + } else { + setModel(availableModel.id); + } + }} + className={`flex items-center justify-between group ${ + effectivelyDisabled ? "opacity-50 cursor-not-allowed" : "" + } ${isRestricted ? "hover:bg-purple-50 dark:hover:bg-purple-950/20" : ""}`} + disabled={effectivelyDisabled} + > +
+
{getDisplayName(availableModel.id, true)}
+
+ {model === availableModel.id && } + + ); + })} + + + + + ); } diff --git a/frontend/src/components/UpgradePromptDialog.tsx b/frontend/src/components/UpgradePromptDialog.tsx new file mode 100644 index 00000000..55d45823 --- /dev/null +++ b/frontend/src/components/UpgradePromptDialog.tsx @@ -0,0 +1,139 @@ +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle +} from "@/components/ui/dialog"; +import { Button } from "@/components/ui/button"; +import { Image, Mic, Sparkles, Check, Cpu, Volume2 } from "lucide-react"; +import { useNavigate } from "@tanstack/react-router"; + +interface UpgradePromptDialogProps { + open: boolean; + onOpenChange: (open: boolean) => void; + feature: "image" | "voice" | "model" | "tts"; + modelName?: string; +} + +export function UpgradePromptDialog({ + open, + onOpenChange, + feature, + modelName +}: UpgradePromptDialogProps) { + const navigate = useNavigate(); + + const handleUpgrade = () => { + onOpenChange(false); + navigate({ to: "/pricing" }); + }; + + const getFeatureInfo = () => { + if (feature === "image") { + return { + icon: , + title: "Image Upload", + description: "Upload and analyze images with AI-powered vision models", + requiredPlan: "Starter", + benefits: [ + "Images stay private with end-to-end encryption", + "Upload JPEG, PNG, and WebP formats securely", + "Use advanced vision models like Gemma 3", + "Analyze diagrams, screenshots, and photos privately", + "Extract text from images without exposing data" + ] + }; + } else if (feature === "voice") { + return { + icon: , + title: "Voice Recording", + description: "Record and transcribe voice messages with Whisper AI", + requiredPlan: "Pro", + benefits: [ + "Voice recordings are end-to-end encrypted", + "Record messages directly in chat securely", + "Private transcription with Whisper Large v3", + "Support for multiple languages", + "No audio data is stored or used for training" + ] + }; + } else if (feature === "tts") { + return { + icon: , + title: "Text-to-Speech", + description: "Listen to AI responses with natural-sounding voices", + requiredPlan: "Pro", + benefits: [ + "Audio generation happens privately on secure servers", + "Natural-sounding AI voices", + "Perfect for accessibility or multitasking", + "Listen to long responses hands-free" + ] + }; + } else { + return { + icon: , + title: modelName ? `Access ${modelName}` : "Powerful AI Models", + description: "Get access to our most advanced AI models for superior performance", + requiredPlan: "Pro", + benefits: [ + "All models run in secure, encrypted environments", + "Access to DeepSeek R1 for advanced reasoning", + "OpenAI GPT-OSS, Mistral, Qwen, and more", + "Higher token limits for longer conversations", + "Priority access to new models as they launch" + ] + }; + } + }; + + const info = getFeatureInfo(); + + return ( + + + +
+
{info.icon}
+ {info.title} +
+ {info.description} + + +
+
+

+ Available with Pro plan and above +

+
    + {info.benefits.map((benefit, i) => ( +
  • + + {benefit} +
    • + ))} +
+
+ +
+

+ Plus access to 6 powerful models (including DeepSeek R1), API access, and more usage +

+
+
+ + + + + + + + ); +} diff --git a/frontend/src/config/pricingConfig.tsx b/frontend/src/config/pricingConfig.tsx index d01bc917..1d1b7ce9 100644 --- a/frontend/src/config/pricingConfig.tsx +++ b/frontend/src/config/pricingConfig.tsx @@ -44,24 +44,13 @@ export const PRICING_PLANS: PricingPlan[] = [ { text: "Rename Chats", included: true, icon: }, { text: "Image Upload", included: false, icon: }, { - text: "DeepSeek R1 0528 671B", + text: "Voice Recording (Whisper)", included: false, icon: }, + { text: "Text-to-Speech", included: false, icon: }, { - text: "OpenAI GPT-OSS 120B", - included: false, - icon: - }, - { text: "Gemma 3 27B", included: false, icon: }, - { - text: "Mistral Small 3.1 24B", - included: false, - icon: - }, - { text: "Qwen 2.5 72B", included: false, icon: }, - { - text: "Qwen3 Coder 480B", + text: "6 Powerful Models (including DeepSeek R1)", included: false, icon: }, @@ -90,28 +79,17 @@ export const PRICING_PLANS: PricingPlan[] = [ icon: }, { text: "Gemma 3 27B", included: true, icon: }, + { text: "Image Upload", included: true, icon: }, { - text: "DeepSeek R1 0528 671B", - included: false, - icon: - }, - { - text: "OpenAI GPT-OSS 120B", - included: false, - icon: - }, - { - text: "Mistral Small 3.1 24B", + text: "Voice Recording (Whisper)", included: false, icon: }, - { text: "Qwen 2.5 72B", included: false, icon: }, { - text: "Qwen3 Coder 480B", + text: "5 More Powerful Models", included: false, icon: }, - { text: "Image Upload", included: false, icon: }, { text: "API Access", included: false, icon: } ], ctaText: "Start Chatting" @@ -133,28 +111,17 @@ export const PRICING_PLANS: PricingPlan[] = [ }, { text: "Image Upload", included: true, icon: }, { - text: "DeepSeek R1 0528 671B", + text: "Voice Recording (Whisper Large v3)", included: true, icon: }, { - text: "OpenAI GPT-OSS 120B", - included: true, - icon: - }, - { text: "Gemma 3 27B", included: true, icon: }, - { - text: "Mistral Small 3.1 24B", + text: "Text-to-Speech", included: true, icon: }, { - text: "Qwen 2.5 72B", - included: true, - icon: - }, - { - text: "Qwen3 Coder 480B", + text: "6 Powerful Models (including DeepSeek R1)", included: true, icon: }, @@ -194,28 +161,17 @@ export const PRICING_PLANS: PricingPlan[] = [ }, { text: "Image Upload", included: true, icon: }, { - text: "DeepSeek R1 0528 671B", - included: true, - icon: - }, - { - text: "OpenAI GPT-OSS 120B", - included: true, - icon: - }, - { text: "Gemma 3 27B", included: true, icon: }, - { - text: "Mistral Small 3.1 24B", + text: "Voice Recording (Whisper Large v3)", included: true, icon: }, { - text: "Qwen 2.5 72B", + text: "Text-to-Speech", included: true, icon: }, { - text: "Qwen3 Coder 480B", + text: "6 Powerful Models (including DeepSeek R1)", included: true, icon: }, @@ -264,28 +220,17 @@ export const PRICING_PLANS: PricingPlan[] = [ }, { text: "Image Upload", included: true, icon: }, { - text: "DeepSeek R1 0528 671B", - included: true, - icon: - }, - { - text: "OpenAI GPT-OSS 120B", - included: true, - icon: - }, - { text: "Gemma 3 27B", included: true, icon: }, - { - text: "Mistral Small 3.1 24B", + text: "Voice Recording (Whisper Large v3)", included: true, icon: }, { - text: "Qwen 2.5 72B", + text: "Text-to-Speech", included: true, icon: }, { - text: "Qwen3 Coder 480B", + text: "6 Powerful Models (including DeepSeek R1)", included: true, icon: }, diff --git a/frontend/src/routes/_auth.chat.$chatId.tsx b/frontend/src/routes/_auth.chat.$chatId.tsx index 3977d6e6..c6e0c1c6 100644 --- a/frontend/src/routes/_auth.chat.$chatId.tsx +++ b/frontend/src/routes/_auth.chat.$chatId.tsx @@ -16,6 +16,10 @@ import { useOpenAI } from "@/ai/useOpenAi"; import { useLocalState } from "@/state/useLocalState"; import { Markdown, stripThinkingTags } from "@/components/markdown"; import { ChatMessage, DEFAULT_MODEL_ID } from "@/state/LocalStateContext"; +import { UpgradePromptDialog } from "@/components/UpgradePromptDialog"; +import { useQuery } from "@tanstack/react-query"; +import { getBillingService } from "@/billing/billingService"; +import { cn } from "@/utils/utils"; import { Sidebar, SidebarToggle } from "@/components/Sidebar"; import { useQueryClient } from "@tanstack/react-query"; import { Button } from "@/components/ui/button"; @@ -82,10 +86,38 @@ function SystemMessage({ const textWithoutThinking = stripThinkingTags(text); const { isCopied, handleCopy } = useCopyToClipboard(textWithoutThinking); const [isPlaying, setIsPlaying] = useState(false); + const [upgradeDialogOpen, setUpgradeDialogOpen] = useState(false); const audioRef = useRef(null); const openai = useOpenAI(); + const { setBillingStatus } = useLocalState(); + + // Fetch billing status to check if user has Pro/Team/Max access + const { data: billingStatus } = useQuery({ + queryKey: ["billingStatus"], + queryFn: async () => { + const billingService = getBillingService(); + const status = await billingService.getBillingStatus(); + setBillingStatus(status); + return status; + } + }); + + // Check if user has Pro/Team/Max access for TTS + const hasProTeamAccess = + billingStatus && + (billingStatus.product_name?.toLowerCase().includes("pro") || + billingStatus.product_name?.toLowerCase().includes("max") || + billingStatus.product_name?.toLowerCase().includes("team")); + + const canUseTTS = hasProTeamAccess; const handleTTS = useCallback(async () => { + // Check if user has access + if (!canUseTTS) { + setUpgradeDialogOpen(true); + return; + } + if (isPlaying) { // Stop playing if (audioRef.current) { @@ -135,7 +167,7 @@ function SystemMessage({ console.error("TTS error:", error); setIsPlaying(false); } - }, [textWithoutThinking, isPlaying, openai]); + }, [textWithoutThinking, isPlaying, openai, canUseTTS]); // Cleanup on unmount useEffect(() => { @@ -168,7 +200,7 @@ function SystemMessage({ +
+ {isRecording && ( + + )} +
+ {/* Simple System Prompt Section - just a gear button and input when expanded */} +
+
+ +
+ + {isSystemPromptExpanded && ( +