diff --git a/apps/AppWithWearable/ios/Runner.xcodeproj/project.pbxproj b/apps/AppWithWearable/ios/Runner.xcodeproj/project.pbxproj index c9ca609987..90df96bd42 100644 --- a/apps/AppWithWearable/ios/Runner.xcodeproj/project.pbxproj +++ b/apps/AppWithWearable/ios/Runner.xcodeproj/project.pbxproj @@ -371,7 +371,7 @@ BUILD_LIBRARY_FOR_DISTRIBUTION = NO; CLANG_ENABLE_MODULES = YES; CODE_SIGN_ENTITLEMENTS = Runner/Runner.entitlements; - CURRENT_PROJECT_VERSION = 15; + CURRENT_PROJECT_VERSION = 16; DEVELOPMENT_TEAM = UA6Q3X7F6K; ENABLE_BITCODE = NO; FRAMEWORK_SEARCH_PATHS = ( @@ -387,7 +387,7 @@ "$(inherited)", "$(PROJECT_DIR)/Flutter", ); - MARKETING_VERSION = 1.0.6; + MARKETING_VERSION = 1.0.7; PRODUCT_BUNDLE_IDENTIFIER = "com.friend-app-with-wearable.ios12"; PRODUCT_NAME = "$(TARGET_NAME)"; SWIFT_OBJC_BRIDGING_HEADER = "Runner/Runner-Bridging-Header.h"; @@ -510,7 +510,7 @@ BUILD_LIBRARY_FOR_DISTRIBUTION = NO; CLANG_ENABLE_MODULES = YES; CODE_SIGN_ENTITLEMENTS = Runner/Runner.entitlements; - CURRENT_PROJECT_VERSION = 15; + CURRENT_PROJECT_VERSION = 16; DEVELOPMENT_TEAM = UA6Q3X7F6K; ENABLE_BITCODE = NO; FRAMEWORK_SEARCH_PATHS = ( @@ -526,7 +526,7 @@ "$(inherited)", "$(PROJECT_DIR)/Flutter", ); - MARKETING_VERSION = 1.0.6; + MARKETING_VERSION = 1.0.7; PRODUCT_BUNDLE_IDENTIFIER = "com.friend-app-with-wearable.ios12"; PRODUCT_NAME = "$(TARGET_NAME)"; SWIFT_OBJC_BRIDGING_HEADER = "Runner/Runner-Bridging-Header.h"; @@ -544,7 +544,7 @@ BUILD_LIBRARY_FOR_DISTRIBUTION = NO; CLANG_ENABLE_MODULES = YES; CODE_SIGN_ENTITLEMENTS = Runner/Runner.entitlements; - CURRENT_PROJECT_VERSION = 15; + CURRENT_PROJECT_VERSION = 16; DEVELOPMENT_TEAM = UA6Q3X7F6K; ENABLE_BITCODE = NO; FRAMEWORK_SEARCH_PATHS = ( @@ -560,7 +560,7 @@ "$(inherited)", "$(PROJECT_DIR)/Flutter", ); - MARKETING_VERSION = 1.0.6; + MARKETING_VERSION = 1.0.7; PRODUCT_BUNDLE_IDENTIFIER = "com.friend-app-with-wearable.ios12"; PRODUCT_NAME = "$(TARGET_NAME)"; SWIFT_OBJC_BRIDGING_HEADER = "Runner/Runner-Bridging-Header.h"; diff --git a/apps/AppWithWearable/lib/backend/api_requests/api_calls.dart b/apps/AppWithWearable/lib/backend/api_requests/api_calls.dart index 4ffdcd6f61..8ec00a6abe 100644 --- a/apps/AppWithWearable/lib/backend/api_requests/api_calls.dart +++ b/apps/AppWithWearable/lib/backend/api_requests/api_calls.dart @@ -158,6 +158,37 @@ Future generateTitleAndSummaryForMemory(String rawMemory, List adviseOnCurrentConversation(String transcript) async { + if (transcript.isEmpty) return ''; + // if (transcript.contains('Speaker 0') && + // (!transcript.contains('Speaker 1') && !transcript.contains('Speaker 2') && !transcript.contains('Speaker 3'))) { + // return ''; + // } + // TODO: eventually determine who am I, and improve diarization, deepgram is no good + var prompt = ''' + You are a conversation coach, you provide clear and concise advice for conversations in real time. + The following is a transcript of the conversation (in progress) where most likely I am "Speaker 0", \ + provide advice on my current way of speaking, and my interactions with the other speaker(s). + + Transcription: + ``` + $transcript + ``` + + Consider that the transcription is not perfect, so there might be mixed up words or sentences between speakers, try to work around that. + Also, it's possible that there's nothing word notifying the user about his interactions, in that case, output N/A. + Remember that the purpose of this advice, is to notify the user about his way of interacting in real time, so he can improve his communication skills. + Be concise and short, respond in 10 to 15 words. + ''' + .replaceAll(' ', '') + .replaceAll(' ', '') + .trim(); + debugPrint(prompt); + var result = await executeGptPrompt(prompt); + if (result.contains('N/A')) return ''; + return result; +} + Future requestSummary(List memories) async { var prompt = ''' Based on my recent memories below, summarize everything into 3-4 most important facts I need to remember. diff --git a/apps/AppWithWearable/lib/pages/home/page.dart b/apps/AppWithWearable/lib/pages/home/page.dart index 2cfa60ea51..f7e3529ff2 100644 --- a/apps/AppWithWearable/lib/pages/home/page.dart +++ b/apps/AppWithWearable/lib/pages/home/page.dart @@ -204,23 +204,26 @@ class _HomePageState extends State { void _saveSettings() async { final prefs = SharedPreferencesUtil(); prefs.openAIApiKey = _openaiApiKeyController.text.trim(); - prefs.deepgramApiKey = _deepgramApiKeyController.text.trim(); prefs.gcpCredentials = _gcpCredentialsController.text.trim(); prefs.gcpBucketName = _gcpBucketNameController.text.trim(); - prefs.useFriendApiKeys = _useFriendApiKeys; - prefs.recordingsLanguage = _selectedLanguage; - prefs.customWebsocketUrl = _customWebsocketUrlController.text.trim(); bool requiresReset = false; - if (_selectedLanguage != prefs.getString('recordingsLanguage')) { + if (_selectedLanguage != prefs.recordingsLanguage) { + prefs.recordingsLanguage = _selectedLanguage; requiresReset = true; } - if (_deepgramApiKeyController.text != prefs.getString('deepgramApiKey')) { + if (_deepgramApiKeyController.text != prefs.deepgramApiKey) { + prefs.deepgramApiKey = _deepgramApiKeyController.text.trim(); requiresReset = true; } - if (_customWebsocketUrlController.text != prefs.getString('customWebsocketUrl')) { + if (_customWebsocketUrlController.text != prefs.customWebsocketUrl) { + prefs.customWebsocketUrl = _customWebsocketUrlController.text.trim(); requiresReset = true; } + if (_useFriendApiKeys != prefs.useFriendApiKeys) { + requiresReset = true; + prefs.useFriendApiKeys = _useFriendApiKeys; + } if (requiresReset) childWidgetKey.currentState?.resetState(); if (_gcpCredentialsController.text.isNotEmpty && _gcpBucketNameController.text.isNotEmpty) { diff --git a/apps/AppWithWearable/lib/pages/home/widgets/transcript.dart b/apps/AppWithWearable/lib/pages/home/widgets/transcript.dart index 7956a60dc8..cf164d15b4 100644 --- a/apps/AppWithWearable/lib/pages/home/widgets/transcript.dart +++ b/apps/AppWithWearable/lib/pages/home/widgets/transcript.dart @@ -4,6 +4,7 @@ import 'dart:io'; import 'package:collection/collection.dart'; import 'package:flutter/material.dart'; import 'package:flutter/scheduler.dart'; +import 'package:friend_private/backend/api_requests/api_calls.dart'; import 'package:friend_private/utils/memories.dart'; import 'package:friend_private/backend/api_requests/cloud_storage.dart'; import 'package:friend_private/utils/notifications.dart'; @@ -49,6 +50,8 @@ class TranscriptWidgetState extends State with WidgetsBindingO String customWebsocketTranscript = ''; IOWebSocketChannel? channelCustomWebsocket; + Timer? _conversationAdvisorTimer; + @override void initState() { super.initState(); @@ -56,6 +59,7 @@ class TranscriptWidgetState extends State with WidgetsBindingO SchedulerBinding.instance.addPostFrameCallback((_) async { initBleConnection(); }); + _initiateConversationAdvisorTimer(); } @override @@ -73,38 +77,39 @@ class TranscriptWidgetState extends State with WidgetsBindingO @override void dispose() { WidgetsBinding.instance.removeObserver(this); + _conversationAdvisorTimer?.cancel(); + _memoryCreationTimer?.cancel(); + debugPrint('TranscriptWidget disposed'); super.dispose(); + + // TODO: cancel snackbars properly when in the background } - _pingWsKeepAlive() { - // TODO: try later - Timer.periodic(const Duration(seconds: 30), (timer) { - if (channelCustomWebsocket != null) { - channelCustomWebsocket!.sink.add('ping'); - } - }); + updateTranscript(Map transcriptBySpeaker) { + if (transcriptBySpeaker.isEmpty) return; + // debugPrint('Updating transcript with: $transcriptBySpeaker'); + var copy = Map.from(whispersDiarized.last); + transcriptBySpeaker.forEach((speaker, transcript) => copy[speaker] = transcript); + whispersDiarized[whispersDiarized.length - 1] = copy; + setState(() {}); + // debugPrint('Updated whispersDiarized: $transcriptBySpeaker'); } Future initBleConnection() async { Tuple4 data = await bleReceiveWAV( btDevice: widget.btDevice!, - speechFinalCallback: (_) { + speechFinalCallback: (Map transcriptBySpeaker, String transcriptItem) { debugPrint("Deepgram Finalized Callback received"); - setState(() { + updateTranscript(transcriptBySpeaker); + if (whispersDiarized.last.isNotEmpty) { whispersDiarized.add({}); - }); - _initiateTimer(); + } + _initiateMemoryCreationTimer(); }, - interimCallback: (String transcript, Map transcriptBySpeaker) { + interimCallback: (Map transcriptBySpeaker, String transcriptItem) { _memoryCreationTimer?.cancel(); - var copy = whispersDiarized[whispersDiarized.length - 1]; - transcriptBySpeaker.forEach((speaker, transcript) { - copy[speaker] = transcript; - }); - setState(() { - whispersDiarized[whispersDiarized.length - 1] = copy; - }); - _initiateTimer(); // sometimes final speech callback is not made + updateTranscript(transcriptBySpeaker); + // _initiateTimer(); // FIXME sometimes final speech callback is not made, thus memory timer not started }, onWebsocketConnectionSuccess: () { addEventToContext('Websocket Opened'); @@ -237,9 +242,23 @@ class TranscriptWidgetState extends State with WidgetsBindingO bool memoryCreating = false; - _initiateTimer() { + _initiateConversationAdvisorTimer() { + _conversationAdvisorTimer = Timer.periodic(const Duration(seconds: 60*10), (timer) async { + addEventToContext('Conversation Advisor Timer Triggered'); + var transcript = _buildDiarizedTranscriptMessage(); + debugPrint('_initiateConversationAdvisorTimer: $transcript'); + if (transcript.isEmpty || transcript.split(' ').length < 20) return; + var advice = await adviseOnCurrentConversation(transcript); + if (advice.isNotEmpty) { + clearNotification(3); + createNotification(notificationId: 3, title: 'Your Conversation Coach Says', body: advice); + } + }); + } + + _initiateMemoryCreationTimer() { _memoryCreationTimer?.cancel(); - _memoryCreationTimer = Timer(const Duration(seconds: 120), () async { + _memoryCreationTimer = Timer(const Duration(seconds: 5), () async { setState(() { memoryCreating = true; }); @@ -261,7 +280,6 @@ class TranscriptWidgetState extends State with WidgetsBindingO memoryCreating = false; }); audioStorage?.clearAudioBytes(); - // TODO: proactive audio, and sends notifications telling like "Dude, don't say x like this, how frequently? }); } @@ -306,8 +324,7 @@ class TranscriptWidgetState extends State with WidgetsBindingO ); } - var filteredNotEmptyWhispers = whispersDiarized.where((e) => e.isNotEmpty).toList(); - if (filteredNotEmptyWhispers.isEmpty) { + if (whispersDiarized[0].keys.isEmpty) { return const Padding( padding: EdgeInsets.only(top: 48.0), child: InfoButton(), @@ -317,13 +334,15 @@ class TranscriptWidgetState extends State with WidgetsBindingO padding: EdgeInsets.zero, shrinkWrap: true, scrollDirection: Axis.vertical, - itemCount: filteredNotEmptyWhispers.length, + itemCount: whispersDiarized.length, physics: const NeverScrollableScrollPhysics(), separatorBuilder: (_, __) => const SizedBox(height: 16.0), itemBuilder: (context, idx) { - final data = filteredNotEmptyWhispers[idx]; + final data = whispersDiarized[idx]; + var keys = data.keys.map((e) => e); + int totalSpeakers = keys.isNotEmpty ? (keys.reduce(max) + 1) : 0; String transcriptItem = ''; - for (int speaker = 0; speaker < data.length; speaker++) { + for (int speaker = 0; speaker < totalSpeakers; speaker++) { if (data.containsKey(speaker)) { transcriptItem += 'Speaker $speaker: ${data[speaker]!} '; } diff --git a/apps/AppWithWearable/lib/utils/memories.dart b/apps/AppWithWearable/lib/utils/memories.dart index fc56ccbf4a..e99a4c354b 100644 --- a/apps/AppWithWearable/lib/utils/memories.dart +++ b/apps/AppWithWearable/lib/utils/memories.dart @@ -20,6 +20,7 @@ Future memoryCreationBlock(BuildContext context, String rawMemory, String? } catch (e) { debugPrint('Error: $e'); changeAppStateMemoryCreating(); + ScaffoldMessenger.of(context).removeCurrentSnackBar(); ScaffoldMessenger.of(context).showSnackBar( const SnackBar(content: Text('There was an error creating your memory, please check your open AI API keys.'))); return; @@ -28,6 +29,7 @@ Future memoryCreationBlock(BuildContext context, String rawMemory, String? if (structuredMemory.contains("N/A")) { await saveFailureMemory(rawMemory, structuredMemory); changeAppStateMemoryCreating(); + ScaffoldMessenger.of(context).removeCurrentSnackBar(); ScaffoldMessenger.of(context).showSnackBar(const SnackBar( content: Text( 'Recent Memory Discarded! Nothing useful. 😄', @@ -37,6 +39,7 @@ Future memoryCreationBlock(BuildContext context, String rawMemory, String? )); } else { await finalizeMemoryRecord(rawMemory, structuredMemory, audioFileName); + ScaffoldMessenger.of(context).removeCurrentSnackBar(); ScaffoldMessenger.of(context).showSnackBar(const SnackBar( content: Text('New Memory Created! 🚀', style: TextStyle(color: Colors.white)), duration: Duration(seconds: 4), diff --git a/apps/AppWithWearable/lib/utils/stt/deepgram.dart b/apps/AppWithWearable/lib/utils/stt/deepgram.dart index dfe9784648..fb6d52ab5c 100644 --- a/apps/AppWithWearable/lib/utils/stt/deepgram.dart +++ b/apps/AppWithWearable/lib/utils/stt/deepgram.dart @@ -62,8 +62,8 @@ Future _initCustomStream({ } Future _initStream( - void Function(String) speechFinalCallback, - void Function(String, Map) interimCallback, + void Function(Map, String) speechFinalCallback, + void Function(Map, String) interimCallback, VoidCallback onWebsocketConnectionSuccess, void Function(dynamic) onWebsocketConnectionFailed, void Function(int?, String?) onWebsocketConnectionClosed, @@ -85,25 +85,31 @@ Future _initStream( if (parsedJson['channel'] == null || parsedJson['channel']['alternatives'] == null) return; final data = parsedJson['channel']['alternatives'][0]; + // debugPrint('parsedJson: ${data.toString()}'); final transcript = data['transcript']; final speechFinal = parsedJson['is_final']; if (transcript.length > 0) { - debugPrint('~~Transcript: $transcript ~ speechFinal: $speechFinal'); + debugPrint('~~Transcript: $transcript ~ speechFinal: $speechFinal ~ ${data.toString()}'); Map bySpeaker = {}; data['words'].forEach((word) { int speaker = word['speaker']; - bySpeaker[speaker] ??= ''; bySpeaker[speaker] = '${(bySpeaker[speaker] ?? '') + word['punctuated_word']} '; }); + int totalSpeakers = bySpeaker.keys.map((e) => e).reduce(max) + 1; + String transcriptItem = ''; + for (int speaker = 0; speaker < totalSpeakers; speaker++) { + if (bySpeaker.containsKey(speaker)) { + transcriptItem += 'Speaker $speaker: ${bySpeaker[speaker]!} '; + } + } // This is step 1 for diarization, but, sometimes "Speaker 1: Hello how" // but it says it's the previous speaker (e.g. speaker 0), but in the next stream it fixes the transcript, and says it's speaker 1. // debugPrint(bySpeaker.toString()); if (speechFinal) { - interimCallback(transcript, bySpeaker); - speechFinalCallback(''); + speechFinalCallback(bySpeaker, transcriptItem); } else { - interimCallback(transcript, bySpeaker); + interimCallback(bySpeaker, transcriptItem); } } }, @@ -133,8 +139,8 @@ Future _initStream( Future> bleReceiveWAV({ BTDeviceStruct? btDevice, - void Function(String)? speechFinalCallback, - void Function(String, Map)? interimCallback, + void Function(Map, String)? speechFinalCallback, + void Function(Map, String)? interimCallback, VoidCallback? onWebsocketConnectionSuccess, void Function(dynamic)? onWebsocketConnectionFailed, void Function(int?, String?)? onWebsocketConnectionClosed,