diff --git a/web/app/.gitignore b/web/app/.gitignore index 2072aeac28..417738589c 100644 --- a/web/app/.gitignore +++ b/web/app/.gitignore @@ -1,2 +1,3 @@ # Claude Code local files .claude/ +web/app/tsconfig.tsbuildinfo diff --git a/web/app/env.local.example b/web/app/env.local.example new file mode 100644 index 0000000000..167242a5d7 --- /dev/null +++ b/web/app/env.local.example @@ -0,0 +1,26 @@ +# --- Firebase (Public Web Config ONLY) --- + +NEXT_PUBLIC_FIREBASE_API_KEY= +NEXT_PUBLIC_FIREBASE_AUTH_DOMAIN= +NEXT_PUBLIC_FIREBASE_PROJECT_ID= +NEXT_PUBLIC_FIREBASE_STORAGE_BUCKET= + +# You MUST fetch these from Firebase Console -> Project Settings +NEXT_PUBLIC_FIREBASE_MESSAGING_SENDER_ID= +NEXT_PUBLIC_FIREBASE_APP_ID= +NEXT_PUBLIC_FIREBASE_MEASUREMENT_ID= + +GEMINI_API_KEY= + +# --- Push Notifications (Web Push / VAPID) --- +# Firebase Console -> Cloud Messaging -> Web Push certificates +NEXT_PUBLIC_FIREBASE_VAPID_KEY= + +# --- Backend API Connection --- +# Map from API_BASE_URL / BASE_API_URL +NEXT_PUBLIC_API_BASE_URL= + +NEXT_PUBLIC_WS_BASE_URL= + +# --- Analytics (Optional) --- +NEXT_PUBLIC_MIXPANEL_TOKEN= diff --git a/web/app/next-env.d.ts b/web/app/next-env.d.ts index c4b7818fbb..9edff1c7ca 100644 --- a/web/app/next-env.d.ts +++ b/web/app/next-env.d.ts @@ -1,6 +1,6 @@ /// /// -import "./.next/dev/types/routes.d.ts"; +import "./.next/types/routes.d.ts"; // NOTE: This file should not be edited // see https://nextjs.org/docs/app/api-reference/config/typescript for more information. diff --git a/web/app/src/app/actions/proactive.ts b/web/app/src/app/actions/proactive.ts new file mode 100644 index 0000000000..b746e9f5bd --- /dev/null +++ b/web/app/src/app/actions/proactive.ts @@ -0,0 +1,97 @@ +'use server'; + +import { GeminiResponseSchema } from '@/lib/geminiClient'; + +interface AnalyzeScreenParams { + imageBase64: string; + imageMimeType?: string; + prompt: string; + systemPrompt: string; + responseSchema?: GeminiResponseSchema; + model?: string; +} + +const DEFAULT_MODEL = 'gemini-2.0-flash'; +const ALLOWED_MODELS = ['gemini-2.0-flash', 'gemini-1.5-flash', 'gemini-1.5-pro']; + +export async function analyzeScreenAction(params: AnalyzeScreenParams): Promise { + const apiKey = process.env.GEMINI_API_KEY; + + if (!apiKey) { + throw new Error('Gemini API key not configured on server'); + } + + const { + imageBase64, + imageMimeType = 'image/jpeg', + prompt, + systemPrompt, + responseSchema, + model = DEFAULT_MODEL + } = params; + + // Validate model + if (!ALLOWED_MODELS.includes(model)) { + throw new Error(`Invalid model specified. Allowed: ${ALLOWED_MODELS.join(', ')}`); + } + + // Construct request payload + const requestBody: any = { + contents: [ + { + parts: [ + { text: prompt }, + { + inline_data: { + mime_type: imageMimeType, + data: imageBase64, + }, + }, + ], + }, + ], + system_instruction: { + parts: [{ text: systemPrompt }], + }, + }; + + if (responseSchema) { + requestBody.generation_config = { + response_mime_type: 'application/json', + response_schema: responseSchema, + }; + } + + const url = `https://generativelanguage.googleapis.com/v1beta/models/${model}:generateContent?key=${apiKey}`; + + try { + const response = await fetch(url, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify(requestBody), + }); + + if (!response.ok) { + const errorText = await response.text(); + throw new Error(`Gemini API error: ${response.status} ${response.statusText} - ${errorText}`); + } + + const data = await response.json(); + + if (data.error) { + throw new Error(data.error.message); + } + + const text = data.candidates?.[0]?.content?.parts?.[0]?.text; + if (!text) { + throw new Error('No text in response from Gemini API'); + } + + return text; + } catch (error: any) { + console.error('Server Action Analysis Failed:', error); + throw new Error(error.message || 'Analysis failed'); + } +} diff --git a/web/app/src/components/layout/MainLayout.tsx b/web/app/src/components/layout/MainLayout.tsx index 3d41d4b522..7d338f6540 100644 --- a/web/app/src/components/layout/MainLayout.tsx +++ b/web/app/src/components/layout/MainLayout.tsx @@ -9,6 +9,7 @@ import { ChatBubble } from '@/components/chat/ChatBubble'; import { BottomNavigation } from './BottomNavigation'; import { NotificationProvider, useNotificationContext } from '@/components/notifications/NotificationContext'; import { HeaderRecordingIndicator } from '@/components/recording'; +import { ProactiveProvider } from '@/components/proactive'; import { getChatApps } from '@/lib/api'; import { cn } from '@/lib/utils'; import { MemoriesPrefetcher } from '@/components/memories/MemoriesPrefetcher'; @@ -82,70 +83,72 @@ export function MainLayout({ children, title, hideHeader = false }: MainLayoutPr return ( - {/* Handle notification routing from chatApp query param */} - - {/* Prefetch memories in background for instant page load */} - -
- {/* Sidebar */} - setSidebarOpen(false)} - /> - - {/* Main content area - flex row to support push/slide panels */} -
- {/* Main content */} -
- {/* Header - conditionally shown */} - {!hideHeader && ( -
- setSidebarOpen(true)} /> - - {title && ( -

- {title} -

- )} -
- )} - - {/* Mobile menu button when header is hidden */} - {hideHeader && ( -
- setSidebarOpen(true)} /> + + {/* Handle notification routing from chatApp query param */} + + {/* Prefetch memories in background for instant page load */} + +
+ {/* Sidebar */} + setSidebarOpen(false)} + /> + + {/* Main content area - flex row to support push/slide panels */} +
+ {/* Main content */} +
+ {/* Header - conditionally shown */} + {!hideHeader && ( +
+ setSidebarOpen(true)} /> + + {title && ( +

+ {title} +

+ )} +
+ )} + + {/* Mobile menu button when header is hidden */} + {hideHeader && ( +
+ setSidebarOpen(true)} /> +
+ )} + + {/* Content */} +
+ {children}
- )} +
- {/* Content */} -
- {children} -
-
+ {/* Chat panel - push/slide from right */} + - {/* Chat panel - push/slide from right */} - + {/* Notification center - push/slide from right */} + +
- {/* Notification center - push/slide from right */} - -
- - {/* Chat bubble - floating button */} - + {/* Chat bubble - floating button */} + - {/* Bottom navigation - mobile only */} - setSidebarOpen(true)} /> + {/* Bottom navigation - mobile only */} + setSidebarOpen(true)} /> - {/* Recording indicator - handles its own fixed positioning and animates with panels */} - - + {/* Recording indicator - handles its own fixed positioning and animates with panels */} + + + ); diff --git a/web/app/src/components/layout/Sidebar.tsx b/web/app/src/components/layout/Sidebar.tsx index 46bc9d9b44..bf6865aaea 100644 --- a/web/app/src/components/layout/Sidebar.tsx +++ b/web/app/src/components/layout/Sidebar.tsx @@ -24,18 +24,9 @@ import { Bell, Mic, MessageSquare, - Smartphone, + Sparkles, } from 'lucide-react'; -// Apple logo SVG component -function AppleLogo({ className }: { className?: string }) { - return ( - - - - ); -} - // Discord icon SVG component function DiscordIcon({ className }: { className?: string }) { return ( @@ -114,6 +105,7 @@ const settingsMenuItems = [ { id: 'integrations', label: 'Integrations', icon: Puzzle }, { id: 'developer', label: 'Developer', icon: Code }, { id: 'account', label: 'Account', icon: Settings }, + { id: 'proactive', label: 'Proactive Assistant', icon: Sparkles }, ]; interface SidebarProps { @@ -134,7 +126,6 @@ export function Sidebar({ const [isExpanded, setIsExpanded] = useState(false); const [isHeaderHovered, setIsHeaderHovered] = useState(false); const [isTemporaryExpand, setIsTemporaryExpand] = useState(false); - const [mobileAppDismissed, setMobileAppDismissed] = useState(false); const isDesktop = useIsDesktop(); const sidebarRef = useRef(null); const userMenuRef = useRef(null); @@ -145,9 +136,6 @@ export function Sidebar({ if (saved === 'true') { setIsExpanded(true); } - if (localStorage.getItem('mobile-app-banner-dismissed') === 'true') { - setMobileAppDismissed(true); - } }, []); // Click outside handler to close menu and collapse if temporary @@ -254,11 +242,9 @@ export function Sidebar({ height={showText ? 24 : 13} className="object-contain" /> - {showText && ( - - Beta - - )} + + Beta + {/* Mobile close button */} diff --git a/web/app/src/components/proactive/ProactiveContext.tsx b/web/app/src/components/proactive/ProactiveContext.tsx new file mode 100644 index 0000000000..378860ed11 --- /dev/null +++ b/web/app/src/components/proactive/ProactiveContext.tsx @@ -0,0 +1,260 @@ +'use client'; + +import { createContext, useContext, useState, useCallback, useRef, ReactNode } from 'react'; +import type { ExtractedAdvice } from '@/lib/proactiveAnalysis'; +import type { ExtractedMemory } from '@/lib/memoryAnalysis'; +import type { FocusHistoryItem, FocusStatus } from '@/lib/focusAnalysis'; + +// Types +export type ProactiveState = 'idle' | 'monitoring' | 'analyzing'; + +export interface AssistantSettings { + enabled: boolean; + systemPrompt: string; +} + +export interface ProactiveSettings { + + + // Shared + analysisIntervalMs: number; + cooldownMs: number; + + // Advice Assistant + advice: AssistantSettings & { + confidenceThreshold: number; + }; + + // Focus Assistant + focus: AssistantSettings; + + // Memory Assistant + memory: AssistantSettings & { + confidenceThreshold: number; + maxMemoriesPerMinute: number; + }; + + // Legacy support (to avoid full break during transition) + enabled: boolean; + confidenceThreshold?: number; // deprecated + systemPrompt?: string; // deprecated +} + +// Default settings +const DEFAULT_SETTINGS: ProactiveSettings = { + analysisIntervalMs: 30000, + cooldownMs: 30000, + + advice: { + enabled: true, + confidenceThreshold: 0.6, + systemPrompt: '', + }, + + focus: { + enabled: false, + systemPrompt: '', + }, + + memory: { + enabled: true, + confidenceThreshold: 0.6, + maxMemoriesPerMinute: 3, + systemPrompt: '', + }, + + // Legacy + enabled: false, +}; + +// Storage key +const SETTINGS_STORAGE_KEY = 'omi-proactive-settings'; +const MAX_PREVIOUS_ADVICE = 10; + +function deepMerge>(target: T, source: Partial): T { + const result = { ...target }; + for (const key in source) { + if (source.hasOwnProperty(key)) { + const sourceValue = source[key]; + const targetValue = result[key]; + if ( + sourceValue && + typeof sourceValue === 'object' && + !Array.isArray(sourceValue) && + targetValue && + typeof targetValue === 'object' && + !Array.isArray(targetValue) + ) { + result[key] = deepMerge(targetValue, sourceValue); + } else if (sourceValue !== undefined) { + result[key] = sourceValue as any; + } + } + } + return result; +} + +function loadSettings(): ProactiveSettings { + if (typeof window === 'undefined') return DEFAULT_SETTINGS; + + try { + const stored = localStorage.getItem(SETTINGS_STORAGE_KEY); + if (!stored) return DEFAULT_SETTINGS; + const parsed = JSON.parse(stored); + + // Migration: If old setting exists but new ones don't, map them + if (parsed.enabled !== undefined && !parsed.advice) { + return { + ...DEFAULT_SETTINGS, + analysisIntervalMs: parsed.analysisIntervalMs || 30000, + cooldownMs: parsed.cooldownMs || 30000, + advice: { + enabled: parsed.enabled, + confidenceThreshold: parsed.confidenceThreshold || 0.6, + systemPrompt: parsed.systemPrompt || '', + }, + enabled: parsed.enabled, // Keep for legacy check + }; + } + + return deepMerge(DEFAULT_SETTINGS, parsed); + } catch { + return DEFAULT_SETTINGS; + } +} + +function saveSettings(settings: ProactiveSettings): void { + if (typeof window === 'undefined') return; + + try { + localStorage.setItem(SETTINGS_STORAGE_KEY, JSON.stringify(settings)); + } catch (error) { + console.error('Failed to save proactive settings:', error); + } +} + +interface ProactiveContextValue { + // State + state: ProactiveState; + settings: ProactiveSettings; + + // Data + previousAdvice: ExtractedAdvice[]; + previousMemories: ExtractedMemory[]; + focusHistory: FocusHistoryItem[]; + currentFocusStatus: FocusStatus | 'unknown'; + + lastNotificationTime: number; + error: string | null; + + // Actions + setState: (state: ProactiveState) => void; + updateSettings: (updates: Partial | ((prev: ProactiveSettings) => Partial)) => void; + addAdvice: (advice: ExtractedAdvice) => void; + addMemory: (memory: ExtractedMemory) => void; + addFocusEntry: (entry: FocusHistoryItem) => void; + clearHistory: () => void; + setPreviousAdvice: React.Dispatch>; + setPreviousMemories: React.Dispatch>; + setFocusHistory: React.Dispatch>; + setLastNotificationTime: (time: number) => void; + setError: (error: string | null) => void; + + // Refs for hook integration + startMonitoringRef: React.MutableRefObject<(() => Promise) | null>; + stopMonitoringRef: React.MutableRefObject<(() => void) | null>; +} + +const ProactiveContext = createContext(null); + +export function ProactiveProvider({ children }: { children: ReactNode }) { + // State + const [state, setState] = useState('idle'); + const [settings, setSettings] = useState(loadSettings); + const [previousAdvice, setPreviousAdvice] = useState([]); + const [previousMemories, setPreviousMemories] = useState([]); + const [focusHistory, setFocusHistory] = useState([]); + const [currentFocusStatus, setCurrentFocusStatus] = useState('unknown'); + const [lastNotificationTime, setLastNotificationTime] = useState(0); + const [error, setError] = useState(null); + + // Refs for hook integration + const startMonitoringRef = useRef<(() => Promise) | null>(null); + const stopMonitoringRef = useRef<(() => void) | null>(null); + + // Update settings and persist + const updateSettings = useCallback((updates: Partial | ((prev: ProactiveSettings) => Partial)) => { + setSettings((prev) => { + const newUpdates = typeof updates === 'function' ? updates(prev) : updates; + const updated = { ...prev, ...newUpdates }; + + if (newUpdates.advice) updated.advice = { ...prev.advice, ...newUpdates.advice }; + if (newUpdates.focus) updated.focus = { ...prev.focus, ...newUpdates.focus }; + if (newUpdates.memory) updated.memory = { ...prev.memory, ...newUpdates.memory }; + + saveSettings(updated); + return updated; + }); + }, []); + + // Add advice to history + const addAdvice = useCallback((advice: ExtractedAdvice) => { + setPreviousAdvice((prev) => [advice, ...prev].slice(0, MAX_PREVIOUS_ADVICE)); + }, []); + + // Add memory to history + const addMemory = useCallback((memory: ExtractedMemory) => { + setPreviousMemories((prev) => [memory, ...prev].slice(0, 20)); // Keep slightly more memories + }, []); + + // Add focus entry + const addFocusEntry = useCallback((entry: FocusHistoryItem) => { + setFocusHistory((prev) => [entry, ...prev].slice(0, MAX_PREVIOUS_ADVICE)); + setCurrentFocusStatus(entry.status); + }, []); + + const clearHistory = useCallback(() => { + setPreviousAdvice([]); + setPreviousMemories([]); + setFocusHistory([]); + setCurrentFocusStatus('unknown'); + }, []); + + return ( + + {children} + + ); +} + +export function useProactiveContext() { + const context = useContext(ProactiveContext); + if (!context) { + throw new Error('useProactiveContext must be used within a ProactiveProvider'); + } + return context; +} diff --git a/web/app/src/components/proactive/ProactiveMonitoringWidget.tsx b/web/app/src/components/proactive/ProactiveMonitoringWidget.tsx new file mode 100644 index 0000000000..f24bbd3b50 --- /dev/null +++ b/web/app/src/components/proactive/ProactiveMonitoringWidget.tsx @@ -0,0 +1,128 @@ +'use client'; + +import { useState } from 'react'; +import { useProactiveNotifications } from '@/hooks/useProactiveNotifications'; +import { ProactiveSettings } from './ProactiveSettings'; + +export function ProactiveMonitoringWidget() { + const { state, error, isMonitoring, startMonitoring, stopMonitoring, settings, clearError } = + useProactiveNotifications(); + const [showSettings, setShowSettings] = useState(false); + + // Determine status display + function getStatusInfo(): { color: string; text: string; pulse: boolean } { + switch (state) { + case 'monitoring': + return { color: 'bg-green-500', text: 'Monitoring', pulse: true }; + case 'analyzing': + return { color: 'bg-blue-500', text: 'Analyzing...', pulse: true }; + default: + return { color: 'bg-gray-500', text: 'Off', pulse: false }; + } + } + + const status = getStatusInfo(); + + async function handleToggle() { + if (isMonitoring) { + stopMonitoring(); + } else { + await startMonitoring(); + } + } + + // Don't render if not enabled in settings + if (!settings.enabled && !showSettings) { + return ( + + ); + } + + return ( +
+ {/* Main widget button */} +
{ + e.preventDefault(); + setShowSettings(true); + }} + className={`flex items-center gap-2 rounded-full px-3 py-1.5 text-xs transition-colors ${isMonitoring + ? 'bg-green-900/50 text-green-400 hover:bg-green-900/70' + : 'bg-gray-800 text-gray-400 hover:bg-gray-700 hover:text-white' + }`} + title={isMonitoring ? 'Click to stop monitoring' : 'Click to start monitoring'} + role="button" + tabIndex={0} + onKeyDown={(e) => { + if (e.key === 'Enter' || e.key === ' ') { + handleToggle(); + } + }} + > + {/* Status indicator */} + + {status.pulse && ( + + )} + + + + {/* Label */} + {status.text} + + {/* Settings gear */} + +
+ + {/* Error tooltip */} + {error && ( +
+
+ ⚠️ +
+

{error}

+ +
+
+
+ )} + + {/* Settings popup */} + {showSettings && ( + <> + {/* Backdrop */} +
setShowSettings(false)} + /> + + {/* Settings panel */} +
+ setShowSettings(false)} /> +
+ + )} +
+ ); +} diff --git a/web/app/src/components/proactive/ProactiveSettings.tsx b/web/app/src/components/proactive/ProactiveSettings.tsx new file mode 100644 index 0000000000..3cb444e4a6 --- /dev/null +++ b/web/app/src/components/proactive/ProactiveSettings.tsx @@ -0,0 +1,310 @@ +'use client'; + +import { useState } from 'react'; +import { useProactiveNotifications } from '@/hooks/useProactiveNotifications'; +import { DEFAULT_ANALYSIS_PROMPT } from '@/lib/proactiveAnalysis'; + +interface ProactiveSettingsProps { + onClose?: () => void; +} + +export function ProactiveSettings({ onClose }: ProactiveSettingsProps) { + const { settings, updateSettings, clearAdviceHistory, previousAdvice, clearMemoryHistory, previousMemories } = useProactiveNotifications(); + + const [showPrompt, setShowPrompt] = useState(false); + + // Format interval for display + function formatInterval(ms: number): string { + const seconds = ms / 1000; + return seconds >= 60 ? `${seconds / 60} min` : `${seconds} sec`; + } + + function handleResetPrompt() { + updateSettings({ advice: { ...settings.advice, systemPrompt: '' } }); + } + + return ( +
+ {/* Header */} +
+

Proactive Notifications

+ {onClose && ( + + )} +
+ +

+ Get contextual advice based on what's on your screen. Requires screen sharing and + browser notification permissions. +

+ + {/* Enable toggle */} +
+ + +
+ + + + {/* Analysis Interval */} +
+
+ + + {formatInterval(settings.analysisIntervalMs)} + +
+ updateSettings({ analysisIntervalMs: Number(e.target.value) })} + className="w-full" + /> +

How often to capture and analyze your screen

+
+ + {/* Confidence Threshold */} +
+
+ + + {Math.round((settings.advice?.confidenceThreshold ?? 0.6) * 100)}% + +
+ updateSettings({ advice: { ...settings.advice!, confidenceThreshold: Number(e.target.value) } })} + className="w-full" + /> +

+ Only show advice above this confidence level. Higher = fewer, more relevant notifications +

+
+ + {/* Cooldown */} +
+
+ + {formatInterval(settings.cooldownMs)} +
+ updateSettings({ cooldownMs: Number(e.target.value) })} + className="w-full" + /> +

Minimum time between notifications

+
+ + {/* Custom Prompt */} +
+
+ + +
+ {showPrompt && ( + <> +