Skip to content

Feature/conversation coach notifications #171

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
josancamon19 merged 2 commits into from
May 17, 2024
Merged

Feature/conversation coach notifications #171

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
640fd3d
initial conversation coach notifications every 10 minutes of conversa…
josancamon19 May 17, 2024
618da45
memory creation when on the background, doesn't trigger tons of Snack…
josancamon19 May 17, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions apps/AppWithWearable/ios/Runner.xcodeproj/project.pbxproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -371,7 +371,7 @@
BUILD_LIBRARY_FOR_DISTRIBUTION = NO;
CLANG_ENABLE_MODULES = YES;
CODE_SIGN_ENTITLEMENTS = Runner/Runner.entitlements;
CURRENT_PROJECT_VERSION = 15;
CURRENT_PROJECT_VERSION = 16;
DEVELOPMENT_TEAM = UA6Q3X7F6K;
ENABLE_BITCODE = NO;
FRAMEWORK_SEARCH_PATHS = (
Expand All @@ -387,7 +387,7 @@
"$(inherited)",
"$(PROJECT_DIR)/Flutter",
);
MARKETING_VERSION = 1.0.6;
MARKETING_VERSION = 1.0.7;
PRODUCT_BUNDLE_IDENTIFIER = "com.friend-app-with-wearable.ios12";
PRODUCT_NAME = "$(TARGET_NAME)";
SWIFT_OBJC_BRIDGING_HEADER = "Runner/Runner-Bridging-Header.h";
Expand Down Expand Up @@ -510,7 +510,7 @@
BUILD_LIBRARY_FOR_DISTRIBUTION = NO;
CLANG_ENABLE_MODULES = YES;
CODE_SIGN_ENTITLEMENTS = Runner/Runner.entitlements;
CURRENT_PROJECT_VERSION = 15;
CURRENT_PROJECT_VERSION = 16;
DEVELOPMENT_TEAM = UA6Q3X7F6K;
ENABLE_BITCODE = NO;
FRAMEWORK_SEARCH_PATHS = (
Expand All @@ -526,7 +526,7 @@
"$(inherited)",
"$(PROJECT_DIR)/Flutter",
);
MARKETING_VERSION = 1.0.6;
MARKETING_VERSION = 1.0.7;
PRODUCT_BUNDLE_IDENTIFIER = "com.friend-app-with-wearable.ios12";
PRODUCT_NAME = "$(TARGET_NAME)";
SWIFT_OBJC_BRIDGING_HEADER = "Runner/Runner-Bridging-Header.h";
Expand All @@ -544,7 +544,7 @@
BUILD_LIBRARY_FOR_DISTRIBUTION = NO;
CLANG_ENABLE_MODULES = YES;
CODE_SIGN_ENTITLEMENTS = Runner/Runner.entitlements;
CURRENT_PROJECT_VERSION = 15;
CURRENT_PROJECT_VERSION = 16;
DEVELOPMENT_TEAM = UA6Q3X7F6K;
ENABLE_BITCODE = NO;
FRAMEWORK_SEARCH_PATHS = (
Expand All @@ -560,7 +560,7 @@
"$(inherited)",
"$(PROJECT_DIR)/Flutter",
);
MARKETING_VERSION = 1.0.6;
MARKETING_VERSION = 1.0.7;
PRODUCT_BUNDLE_IDENTIFIER = "com.friend-app-with-wearable.ios12";
PRODUCT_NAME = "$(TARGET_NAME)";
SWIFT_OBJC_BRIDGING_HEADER = "Runner/Runner-Bridging-Header.h";
Expand Down
31 changes: 31 additions & 0 deletions apps/AppWithWearable/lib/backend/api_requests/api_calls.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,37 @@ Future<String> generateTitleAndSummaryForMemory(String rawMemory, List<MemoryRec
return (await executeGptPrompt(prompt)).replaceAll('```', '').trim();
}

Future<String> adviseOnCurrentConversation(String transcript) async {
if (transcript.isEmpty) return '';
// if (transcript.contains('Speaker 0') &&
// (!transcript.contains('Speaker 1') && !transcript.contains('Speaker 2') && !transcript.contains('Speaker 3'))) {
// return '';
// }
// TODO: eventually determine who am I, and improve diarization, deepgram is no good
var prompt = '''
You are a conversation coach, you provide clear and concise advice for conversations in real time.
The following is a transcript of the conversation (in progress) where most likely I am "Speaker 0", \
provide advice on my current way of speaking, and my interactions with the other speaker(s).

Transcription:
```
$transcript
```

Consider that the transcription is not perfect, so there might be mixed up words or sentences between speakers, try to work around that.
Also, it's possible that there's nothing word notifying the user about his interactions, in that case, output N/A.
Remember that the purpose of this advice, is to notify the user about his way of interacting in real time, so he can improve his communication skills.
Be concise and short, respond in 10 to 15 words.
'''
.replaceAll(' ', '')
.replaceAll(' ', '')
.trim();
debugPrint(prompt);
var result = await executeGptPrompt(prompt);
if (result.contains('N/A')) return '';
return result;
}

Future<String> requestSummary(List<MemoryRecord> memories) async {
var prompt = '''
Based on my recent memories below, summarize everything into 3-4 most important facts I need to remember.
Expand Down
17 changes: 10 additions & 7 deletions apps/AppWithWearable/lib/pages/home/page.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -204,23 +204,26 @@ class _HomePageState extends State<HomePage> {
void _saveSettings() async {
final prefs = SharedPreferencesUtil();
prefs.openAIApiKey = _openaiApiKeyController.text.trim();
prefs.deepgramApiKey = _deepgramApiKeyController.text.trim();
prefs.gcpCredentials = _gcpCredentialsController.text.trim();
prefs.gcpBucketName = _gcpBucketNameController.text.trim();
prefs.useFriendApiKeys = _useFriendApiKeys;
prefs.recordingsLanguage = _selectedLanguage;
prefs.customWebsocketUrl = _customWebsocketUrlController.text.trim();

bool requiresReset = false;
if (_selectedLanguage != prefs.getString('recordingsLanguage')) {
if (_selectedLanguage != prefs.recordingsLanguage) {
prefs.recordingsLanguage = _selectedLanguage;
requiresReset = true;
}
if (_deepgramApiKeyController.text != prefs.getString('deepgramApiKey')) {
if (_deepgramApiKeyController.text != prefs.deepgramApiKey) {
prefs.deepgramApiKey = _deepgramApiKeyController.text.trim();
requiresReset = true;
}
if (_customWebsocketUrlController.text != prefs.getString('customWebsocketUrl')) {
if (_customWebsocketUrlController.text != prefs.customWebsocketUrl) {
prefs.customWebsocketUrl = _customWebsocketUrlController.text.trim();
requiresReset = true;
}
if (_useFriendApiKeys != prefs.useFriendApiKeys) {
requiresReset = true;
prefs.useFriendApiKeys = _useFriendApiKeys;
}
if (requiresReset) childWidgetKey.currentState?.resetState();

if (_gcpCredentialsController.text.isNotEmpty && _gcpBucketNameController.text.isNotEmpty) {
Expand Down
75 changes: 47 additions & 28 deletions apps/AppWithWearable/lib/pages/home/widgets/transcript.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import 'dart:io';
import 'package:collection/collection.dart';
import 'package:flutter/material.dart';
import 'package:flutter/scheduler.dart';
import 'package:friend_private/backend/api_requests/api_calls.dart';
import 'package:friend_private/utils/memories.dart';
import 'package:friend_private/backend/api_requests/cloud_storage.dart';
import 'package:friend_private/utils/notifications.dart';
Expand Down Expand Up @@ -49,13 +50,16 @@ class TranscriptWidgetState extends State<TranscriptWidget> with WidgetsBindingO
String customWebsocketTranscript = '';
IOWebSocketChannel? channelCustomWebsocket;

Timer? _conversationAdvisorTimer;

@override
void initState() {
super.initState();
WidgetsBinding.instance.addObserver(this);
SchedulerBinding.instance.addPostFrameCallback((_) async {
initBleConnection();
});
_initiateConversationAdvisorTimer();
}

@override
Expand All @@ -73,38 +77,39 @@ class TranscriptWidgetState extends State<TranscriptWidget> with WidgetsBindingO
@override
void dispose() {
WidgetsBinding.instance.removeObserver(this);
_conversationAdvisorTimer?.cancel();
_memoryCreationTimer?.cancel();
debugPrint('TranscriptWidget disposed');
super.dispose();

// TODO: cancel snackbars properly when in the background
}

_pingWsKeepAlive() {
// TODO: try later
Timer.periodic(const Duration(seconds: 30), (timer) {
if (channelCustomWebsocket != null) {
channelCustomWebsocket!.sink.add('ping');
}
});
updateTranscript(Map<int, String> transcriptBySpeaker) {
if (transcriptBySpeaker.isEmpty) return;
// debugPrint('Updating transcript with: $transcriptBySpeaker');
var copy = Map<int, String>.from(whispersDiarized.last);
transcriptBySpeaker.forEach((speaker, transcript) => copy[speaker] = transcript);
whispersDiarized[whispersDiarized.length - 1] = copy;
setState(() {});
// debugPrint('Updated whispersDiarized: $transcriptBySpeaker');
}

Future<void> initBleConnection() async {
Tuple4<IOWebSocketChannel?, StreamSubscription?, WavBytesUtil, IOWebSocketChannel?> data = await bleReceiveWAV(
btDevice: widget.btDevice!,
speechFinalCallback: (_) {
speechFinalCallback: (Map<int, String> transcriptBySpeaker, String transcriptItem) {
debugPrint("Deepgram Finalized Callback received");
setState(() {
updateTranscript(transcriptBySpeaker);
if (whispersDiarized.last.isNotEmpty) {
whispersDiarized.add({});
});
_initiateTimer();
}
_initiateMemoryCreationTimer();
},
interimCallback: (String transcript, Map<int, String> transcriptBySpeaker) {
interimCallback: (Map<int, String> transcriptBySpeaker, String transcriptItem) {
_memoryCreationTimer?.cancel();
var copy = whispersDiarized[whispersDiarized.length - 1];
transcriptBySpeaker.forEach((speaker, transcript) {
copy[speaker] = transcript;
});
setState(() {
whispersDiarized[whispersDiarized.length - 1] = copy;
});
_initiateTimer(); // sometimes final speech callback is not made
updateTranscript(transcriptBySpeaker);
// _initiateTimer(); // FIXME sometimes final speech callback is not made, thus memory timer not started
},
onWebsocketConnectionSuccess: () {
addEventToContext('Websocket Opened');
Expand Down Expand Up @@ -237,9 +242,23 @@ class TranscriptWidgetState extends State<TranscriptWidget> with WidgetsBindingO

bool memoryCreating = false;

_initiateTimer() {
_initiateConversationAdvisorTimer() {
_conversationAdvisorTimer = Timer.periodic(const Duration(seconds: 60*10), (timer) async {
addEventToContext('Conversation Advisor Timer Triggered');
var transcript = _buildDiarizedTranscriptMessage();
debugPrint('_initiateConversationAdvisorTimer: $transcript');
if (transcript.isEmpty || transcript.split(' ').length < 20) return;
var advice = await adviseOnCurrentConversation(transcript);
if (advice.isNotEmpty) {
clearNotification(3);
createNotification(notificationId: 3, title: 'Your Conversation Coach Says', body: advice);
}
});
}

_initiateMemoryCreationTimer() {
_memoryCreationTimer?.cancel();
_memoryCreationTimer = Timer(const Duration(seconds: 120), () async {
_memoryCreationTimer = Timer(const Duration(seconds: 5), () async {
setState(() {
memoryCreating = true;
});
Expand All @@ -261,7 +280,6 @@ class TranscriptWidgetState extends State<TranscriptWidget> with WidgetsBindingO
memoryCreating = false;
});
audioStorage?.clearAudioBytes();
// TODO: proactive audio, and sends notifications telling like "Dude, don't say x like this, how frequently?
});
}

Expand Down Expand Up @@ -306,8 +324,7 @@ class TranscriptWidgetState extends State<TranscriptWidget> with WidgetsBindingO
);
}

var filteredNotEmptyWhispers = whispersDiarized.where((e) => e.isNotEmpty).toList();
if (filteredNotEmptyWhispers.isEmpty) {
if (whispersDiarized[0].keys.isEmpty) {
return const Padding(
padding: EdgeInsets.only(top: 48.0),
child: InfoButton(),
Expand All @@ -317,13 +334,15 @@ class TranscriptWidgetState extends State<TranscriptWidget> with WidgetsBindingO
padding: EdgeInsets.zero,
shrinkWrap: true,
scrollDirection: Axis.vertical,
itemCount: filteredNotEmptyWhispers.length,
itemCount: whispersDiarized.length,
physics: const NeverScrollableScrollPhysics(),
separatorBuilder: (_, __) => const SizedBox(height: 16.0),
itemBuilder: (context, idx) {
final data = filteredNotEmptyWhispers[idx];
final data = whispersDiarized[idx];
var keys = data.keys.map((e) => e);
int totalSpeakers = keys.isNotEmpty ? (keys.reduce(max) + 1) : 0;
String transcriptItem = '';
for (int speaker = 0; speaker < data.length; speaker++) {
for (int speaker = 0; speaker < totalSpeakers; speaker++) {
if (data.containsKey(speaker)) {
transcriptItem += 'Speaker $speaker: ${data[speaker]!} ';
}
Expand Down
3 changes: 3 additions & 0 deletions apps/AppWithWearable/lib/utils/memories.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ Future<void> memoryCreationBlock(BuildContext context, String rawMemory, String?
} catch (e) {
debugPrint('Error: $e');
changeAppStateMemoryCreating();
ScaffoldMessenger.of(context).removeCurrentSnackBar();
ScaffoldMessenger.of(context).showSnackBar(
const SnackBar(content: Text('There was an error creating your memory, please check your open AI API keys.')));
return;
Expand All @@ -28,6 +29,7 @@ Future<void> memoryCreationBlock(BuildContext context, String rawMemory, String?
if (structuredMemory.contains("N/A")) {
await saveFailureMemory(rawMemory, structuredMemory);
changeAppStateMemoryCreating();
ScaffoldMessenger.of(context).removeCurrentSnackBar();
ScaffoldMessenger.of(context).showSnackBar(const SnackBar(
content: Text(
'Recent Memory Discarded! Nothing useful. 😄',
Expand All @@ -37,6 +39,7 @@ Future<void> memoryCreationBlock(BuildContext context, String rawMemory, String?
));
} else {
await finalizeMemoryRecord(rawMemory, structuredMemory, audioFileName);
ScaffoldMessenger.of(context).removeCurrentSnackBar();
ScaffoldMessenger.of(context).showSnackBar(const SnackBar(
content: Text('New Memory Created! 🚀', style: TextStyle(color: Colors.white)),
duration: Duration(seconds: 4),
Expand Down
24 changes: 15 additions & 9 deletions apps/AppWithWearable/lib/utils/stt/deepgram.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,8 +62,8 @@ Future<IOWebSocketChannel?> _initCustomStream({
}

Future<IOWebSocketChannel> _initStream(
void Function(String) speechFinalCallback,
void Function(String, Map<int, String>) interimCallback,
void Function(Map<int, String>, String) speechFinalCallback,
void Function(Map<int, String>, String) interimCallback,
VoidCallback onWebsocketConnectionSuccess,
void Function(dynamic) onWebsocketConnectionFailed,
void Function(int?, String?) onWebsocketConnectionClosed,
Expand All @@ -85,25 +85,31 @@ Future<IOWebSocketChannel> _initStream(
if (parsedJson['channel'] == null || parsedJson['channel']['alternatives'] == null) return;

final data = parsedJson['channel']['alternatives'][0];
// debugPrint('parsedJson: ${data.toString()}');
final transcript = data['transcript'];
final speechFinal = parsedJson['is_final'];

if (transcript.length > 0) {
debugPrint('~~Transcript: $transcript ~ speechFinal: $speechFinal');
debugPrint('~~Transcript: $transcript ~ speechFinal: $speechFinal ~ ${data.toString()}');
Map<int, String> bySpeaker = {};
data['words'].forEach((word) {
int speaker = word['speaker'];
bySpeaker[speaker] ??= '';
bySpeaker[speaker] = '${(bySpeaker[speaker] ?? '') + word['punctuated_word']} ';
});
int totalSpeakers = bySpeaker.keys.map((e) => e).reduce(max) + 1;
String transcriptItem = '';
for (int speaker = 0; speaker < totalSpeakers; speaker++) {
if (bySpeaker.containsKey(speaker)) {
transcriptItem += 'Speaker $speaker: ${bySpeaker[speaker]!} ';
}
}
// This is step 1 for diarization, but, sometimes "Speaker 1: Hello how"
// but it says it's the previous speaker (e.g. speaker 0), but in the next stream it fixes the transcript, and says it's speaker 1.
// debugPrint(bySpeaker.toString());
if (speechFinal) {
interimCallback(transcript, bySpeaker);
speechFinalCallback('');
speechFinalCallback(bySpeaker, transcriptItem);
} else {
interimCallback(transcript, bySpeaker);
interimCallback(bySpeaker, transcriptItem);
}
}
},
Expand Down Expand Up @@ -133,8 +139,8 @@ Future<IOWebSocketChannel> _initStream(

Future<Tuple4<IOWebSocketChannel?, StreamSubscription?, WavBytesUtil, IOWebSocketChannel?>> bleReceiveWAV({
BTDeviceStruct? btDevice,
void Function(String)? speechFinalCallback,
void Function(String, Map<int, String>)? interimCallback,
void Function(Map<int, String>, String)? speechFinalCallback,
void Function(Map<int, String>, String)? interimCallback,
VoidCallback? onWebsocketConnectionSuccess,
void Function(dynamic)? onWebsocketConnectionFailed,
void Function(int?, String?)? onWebsocketConnectionClosed,
Expand Down